From phk at FreeBSD.org  Mon Aug  2 04:13:07 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon,  2 Aug 2021 04:13:07 +0000 (UTC)
Subject: [master] 069b78a4b HAProxy has subtly changed their --version message.
Message-ID: <20210802041307.C3C72B2597@lists.varnish-cache.org>


commit 069b78a4b6653c3bbb16bf11798b658a149c92b5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 2 04:12:29 2021 +0000

    HAProxy has subtly changed their --version message.

diff --git a/bin/varnishtest/tests/h00001.vtc b/bin/varnishtest/tests/h00001.vtc
index 7f8c11599..7d38c9b93 100644
--- a/bin/varnishtest/tests/h00001.vtc
+++ b/bin/varnishtest/tests/h00001.vtc
@@ -2,7 +2,7 @@ varnishtest "Basic HAproxy test"
 
 feature ignore_unknown_macro
 
-feature cmd {haproxy --version 2>&1 | grep -q 'HA-Proxy version'}
+feature cmd {haproxy --version 2>&1 | grep -q 'HA-*Proxy version'}
 
 server s1 {
     rxreq
diff --git a/bin/varnishtest/tests/h00002.vtc b/bin/varnishtest/tests/h00002.vtc
index c8c9bc1dd..bfa9fcc16 100644
--- a/bin/varnishtest/tests/h00002.vtc
+++ b/bin/varnishtest/tests/h00002.vtc
@@ -2,7 +2,7 @@ varnishtest "Basic HAproxy test (daemon mode)"
 
 feature ignore_unknown_macro
 
-feature cmd {haproxy --version 2>&1 | grep -q 'HA-Proxy version'}
+feature cmd {haproxy --version 2>&1 | grep -q 'HA-*Proxy version'}
 
 server s1 {
     rxreq
diff --git a/bin/varnishtest/tests/h00003.vtc b/bin/varnishtest/tests/h00003.vtc
index 404b1eecb..46d2e7b21 100644
--- a/bin/varnishtest/tests/h00003.vtc
+++ b/bin/varnishtest/tests/h00003.vtc
@@ -2,7 +2,7 @@ varnishtest "Test -conf+backend"
 
 feature ignore_unknown_macro
 
-feature cmd {haproxy --version 2>&1 | grep -q 'HA-Proxy version'}
+feature cmd {haproxy --version 2>&1 | grep -q 'HA-*Proxy version'}
 
 server s1 {
     rxreq
diff --git a/bin/varnishtest/tests/h00004.vtc b/bin/varnishtest/tests/h00004.vtc
index 122e0411d..754354f9b 100644
--- a/bin/varnishtest/tests/h00004.vtc
+++ b/bin/varnishtest/tests/h00004.vtc
@@ -2,7 +2,7 @@ varnishtest "Test -conf+backend"
 
 feature ignore_unknown_macro
 
-feature cmd {haproxy --version 2>&1 | grep -q 'HA-Proxy version'}
+feature cmd {haproxy --version 2>&1 | grep -q 'HA-*Proxy version'}
 
 haproxy h1 -conf-OK {
     defaults
diff --git a/bin/varnishtest/tests/h00005.vtc b/bin/varnishtest/tests/h00005.vtc
index 52f1d8315..e59234bd3 100644
--- a/bin/varnishtest/tests/h00005.vtc
+++ b/bin/varnishtest/tests/h00005.vtc
@@ -1,7 +1,7 @@
 varnishtest "Exercise varnishtest syslog facility"
 
 feature ignore_unknown_macro
-feature cmd {haproxy --version 2>&1 | grep -q 'HA-Proxy version'}
+feature cmd {haproxy --version 2>&1 | grep -q 'HA-*Proxy version'}
 
 server s1 {
 	rxreq
diff --git a/bin/varnishtest/tests/h00006.vtc b/bin/varnishtest/tests/h00006.vtc
index 18e099162..516dcd053 100644
--- a/bin/varnishtest/tests/h00006.vtc
+++ b/bin/varnishtest/tests/h00006.vtc
@@ -4,7 +4,7 @@ varnishtest "haproxy tcp-mode, uds, send-proxy-v2, client ip and acl"
 
 feature ignore_unknown_macro
 
-feature cmd {haproxy --version 2>&1 | grep -q 'HA-Proxy version'}
+feature cmd {haproxy --version 2>&1 | grep -q 'HA-*Proxy version'}
 
 server s1 {
     rxreq
diff --git a/bin/varnishtest/tests/h00007.vtc b/bin/varnishtest/tests/h00007.vtc
index b153590ba..9951ef7fc 100644
--- a/bin/varnishtest/tests/h00007.vtc
+++ b/bin/varnishtest/tests/h00007.vtc
@@ -4,7 +4,7 @@ varnishtest "haproxy tcp-mode, tcp, send-proxy-v2, client ip and acl"
 
 feature ignore_unknown_macro
 
-feature cmd {haproxy --version 2>&1 | grep -q 'HA-Proxy version'}
+feature cmd {haproxy --version 2>&1 | grep -q 'HA-*Proxy version'}
 
 server s1 {
     rxreq

From dridi.boukelmoune at gmail.com  Mon Aug  2 09:55:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon,  2 Aug 2021 09:55:07 +0000 (UTC)
Subject: [master] 3fdce6cf2 VRE: bounds check back references in VRE_sub()
Message-ID: <20210802095507.7A35F913ED@lists.varnish-cache.org>


commit 3fdce6cf2c4d360c27c98d05e530d39317d23c68
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Aug 2 10:49:41 2021 +0200

    VRE: bounds check back references in VRE_sub()
    
    Before 6014912e74de7989e37b7be2737cef370b7147ba, VRE_sub() used an
    ovector of size 30, which always containted sufficient space to store
    the 10 possible back- references \0 thorugh \9.
    
    Now that we use pcre2_match_data_create_from_pattern() and later
    pcre2_get_ovector_pointer(), we only get space for the number of
    substrings in the pattern, see pcre2api(3):
    
            The ovector is created to be exactly the right size to hold
            all the substrings a pattern might capture.
    
    Consequently, we need to check that back references do not exceed the
    maximum ovector.

diff --git a/bin/varnishtest/tests/c00001.vtc b/bin/varnishtest/tests/c00001.vtc
index e301e4d09..5a338b2fb 100644
--- a/bin/varnishtest/tests/c00001.vtc
+++ b/bin/varnishtest/tests/c00001.vtc
@@ -23,6 +23,8 @@ varnish v1 -vcl+backend {
 		    regsub(beresp.http.Foobar, "(b)(a)(r)(f)", "\4\&\3\2p\");
 		set beresp.http.Snafu7 =
 		    regsub(beresp.http.Foobar, "ar", bereq.http.nosuchheader);
+		set beresp.http.inval =
+		    regsub(beresp.http.Foobar, "(b)(a)(r)f", "\9\8\7\6\5\4\3\2p");
 	}
 } -start
 
@@ -40,4 +42,5 @@ client c1 {
 	expect resp.http.snafu5 == "_barffra\\p_"
 	expect resp.http.snafu6 == "_f&rap\\_"
 	expect resp.http.snafu7 == "_bf_"
+	expect resp.http.inval == "_rap_"
 } -run
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index dd8eb70c3..9a8d2cc24 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -231,6 +231,7 @@ VRE_sub(const vre_t *code, const char *subject, const char *replacement,
 {
 	pcre2_match_data *data = NULL;
 	PCRE2_SIZE *ovector;
+	uint32_t nov;
 	int i, l;
 	const char *s;
 	unsigned x;
@@ -250,6 +251,7 @@ VRE_sub(const vre_t *code, const char *subject, const char *replacement,
 	do {
 		AN(data);
 		ovector = pcre2_get_ovector_pointer(data);
+		nov = pcre2_get_ovector_count(data);
 		AN(ovector);
 
 		/* Copy prefix to match */
@@ -262,6 +264,8 @@ VRE_sub(const vre_t *code, const char *subject, const char *replacement,
 			s++;
 			if (isdigit(*s)) {
 				x = *s - '0';
+				if (x >= nov)
+					continue;
 				l = ovector[2*x+1] - ovector[2*x];
 				VSB_bcat(vsb, subject + ovector[2*x], l);
 				continue;

From dridi.boukelmoune at gmail.com  Tue Aug  3 07:17:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue,  3 Aug 2021 07:17:07 +0000 (UTC)
Subject: [master] 92e700687 vre: Use pcre2_match_context
Message-ID: <20210803071707.7D6FE63206@lists.varnish-cache.org>


commit 92e700687ea6c41bfe9a31dd8db4ac98338acd38
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Aug 3 09:00:00 2021 +0200

    vre: Use pcre2_match_context

diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index 9a8d2cc24..ad0d31a3a 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -201,7 +201,7 @@ vre_match(const vre_t *code, const char *subject, size_t length, size_t offset,
 	}
 
 	matches =  pcre2_match(re, (PCRE2_SPTR)subject, length, offset,
-	    options, data, NULL);
+	    options, data, code->re_ctx);
 
 	if (datap != NULL && matches > VRE_ERROR_NOMATCH)
 		*datap = data;

From dridi.boukelmoune at gmail.com  Tue Aug  3 07:17:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue,  3 Aug 2021 07:17:07 +0000 (UTC)
Subject: [master] 8e6527302 vrt: Log the proper vre error message
Message-ID: <20210803071707.926D96320A@lists.varnish-cache.org>


commit 8e652730266386053f2838db890a5a764d0e9013
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 3 09:00:46 2021 +0200

    vrt: Log the proper vre error message

diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c
index 4fc352364..944a40bce 100644
--- a/bin/varnishd/cache/cache_vrt_re.c
+++ b/bin/varnishd/cache/cache_vrt_re.c
@@ -64,6 +64,8 @@ VPI_re_fini(vre_t *rep)
 VCL_BOOL
 VRT_re_match(VRT_CTX, const char *s, VCL_REGEX re)
 {
+	struct vsb vsb[1];
+	char errbuf[VRE_ERROR_LEN];
 	int i;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
@@ -73,8 +75,13 @@ VRT_re_match(VRT_CTX, const char *s, VCL_REGEX re)
 	i = VRE_match(re, s, 0, 0, &cache_param->vre_limits);
 	if (i >= 0)
 		return (1);
-	if (i < VRE_ERROR_NOMATCH )
-		VRT_fail(ctx, "Regexp matching returned %d", i);
+	if (i < VRE_ERROR_NOMATCH ) {
+		VSB_init(vsb, errbuf, sizeof errbuf);
+		AZ(VRE_error(vsb, i));
+		AZ(VSB_finish(vsb));
+		VSB_fini(vsb);
+		VRT_fail(ctx, "Regexp matching failed: %s", errbuf);
+	}
 	return (0);
 }
 

From dridi.boukelmoune at gmail.com  Tue Aug  3 07:17:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue,  3 Aug 2021 07:17:07 +0000 (UTC)
Subject: [master] a80a50ca8 param: Remove stale references to pcre error codes
Message-ID: <20210803071707.AD8166320E@lists.varnish-cache.org>


commit a80a50ca8a7fdcf602722eb44b5bf31c38dbf990
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 3 09:08:12 2021 +0200

    param: Remove stale references to pcre error codes

diff --git a/include/tbl/params.h b/include/tbl/params.h
index 66c14b1e6..074048e9e 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -1650,7 +1650,7 @@ PARAM_PCRE2(
 	" prevent crashes, at the cost of possible regexp"
 	" matching failures.\n\n"
 	"Matching failures will show up in the log as VCL_Error"
-	" messages with regexp errors -27 or -21."
+	" messages."
 )
 
 #  undef PARAM_ALL

From dridi.boukelmoune at gmail.com  Tue Aug  3 07:17:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue,  3 Aug 2021 07:17:07 +0000 (UTC)
Subject: [master] e2dd3884e vtc: Bring r01576 back
Message-ID: <20210803071707.C6F6463212@lists.varnish-cache.org>


commit e2dd3884e0edc16af4b72c472697607b0aa1eeda
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 3 09:09:27 2021 +0200

    vtc: Bring r01576 back
    
    Adapted to the pcre2 change, and (manually) tested both with and
    without jit compilation and various tweaks to trigger both match
    and depth limits.
    
    Closes #3658

diff --git a/bin/varnishtest/tests/r01576.vtc b/bin/varnishtest/tests/r01576.vtc
new file mode 100644
index 000000000..b95cc8301
--- /dev/null
+++ b/bin/varnishtest/tests/r01576.vtc
@@ -0,0 +1,53 @@
+varnishtest "Test recursive regexp's fail before consuming all the stack"
+
+# Use 64bit defaults also on 32bit
+varnish v1 -cliok "param.set workspace_client 64k"
+varnish v1 -cliok "param.set http_req_size 32k"
+
+# If you want to play around, uncomment the next lines and adjust
+# the length of the ABAB strings below to suit your needs.
+# Better yet:  Rewrite your regexps to avoid this madness.
+
+# Tweaks you may add to this test:
+# varnish v1 -cliok "param.set thread_pool_stack 48k"
+# varnish v1 -cliok "param.set pcre2_match_limit 100"
+# varnish v1 -cliok "param.set pcre2_depth_limit 10000"
+# varnish v1 -cliok "param.set pcre2_jit_compilation off"
+
+# Approximate formula for FreeBSD/amd64:
+#	pcre2_depth_limit = thread_pool_stack * 2 - 9
+
+varnish v1 -vcl+backend {
+	backend proforma none;
+
+	sub vcl_recv {
+		return (synth(200));
+	}
+	sub vcl_synth {
+		# shamelessly copied from "bugzilla77 at gmail dot com"
+		# https://bugs.php.net/bug.php?id=70110
+		if (req.url ~ "^/(A{1,2}B)+$") {
+			set resp.http.found = "1";
+		}
+	}
+} -start
+
+# This should succeed with default params and JIT/no-JIT
+client c1 {
+	txreq -url "/ABAABABAABABABAB"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.found == 1
+} -run
+
+logexpect l1 -v v1 {
+	expect * * VCL_Error "(match|depth) limit exceeded"
+} -start
+
+# This should fail with default params and JIT/no-JIT
+client c1 {
+	txreq -url "/${string,repeat,8192,AB}"
+	expect_close
+} -run
+
+logexpect l1 -wait

From dridi.boukelmoune at gmail.com  Tue Aug  3 08:06:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue,  3 Aug 2021 08:06:07 +0000 (UTC)
Subject: [master] a2cdf9ede vtc: An older pcre2 may say "recursion"
Message-ID: <20210803080607.190A6650E9@lists.varnish-cache.org>


commit a2cdf9ede244ef562023b07afcd906cdd110f908
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 3 10:04:07 2021 +0200

    vtc: An older pcre2 may say "recursion"

diff --git a/bin/varnishtest/tests/r01576.vtc b/bin/varnishtest/tests/r01576.vtc
index b95cc8301..7cc73e089 100644
--- a/bin/varnishtest/tests/r01576.vtc
+++ b/bin/varnishtest/tests/r01576.vtc
@@ -41,7 +41,7 @@ client c1 {
 } -run
 
 logexpect l1 -v v1 {
-	expect * * VCL_Error "(match|depth) limit exceeded"
+	expect * * VCL_Error "(match|depth|recursion) limit exceeded"
 } -start
 
 # This should fail with default params and JIT/no-JIT

From nils.goroll at uplex.de  Tue Aug  3 11:12:07 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  3 Aug 2021 11:12:07 +0000 (UTC)
Subject: [master] 4cb1faa24 Bring back SmartOS build instructions with
 s:pcre:pcre2:
Message-ID: <20210803111207.A2DE895E59@lists.varnish-cache.org>


commit 4cb1faa243c081fbfb41bf5afe8181876a8d199a
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Aug 3 13:11:29 2021 +0200

    Bring back SmartOS build instructions with s:pcre:pcre2:

diff --git a/doc/sphinx/installation/install_source.rst b/doc/sphinx/installation/install_source.rst
index a8e88b8b2..2d4908c6e 100644
--- a/doc/sphinx/installation/install_source.rst
+++ b/doc/sphinx/installation/install_source.rst
@@ -195,26 +195,24 @@ Then continue `Compiling Varnish`_, using the ``--with-unwind``
 
 .. _Alpine Community Repository: https://wiki.alpinelinux.org/wiki/Enable_Community_Repository
 
-.. XXX: no pcre2 on SmartOS
+Build dependencies on a SmartOS Zone
+------------------------------------
 ..
-.. Build dependencies on a SmartOS Zone
-.. ------------------------------------
+As of SmartOS pkgsrc 2019Q4, install the following packages::
 ..
-.. As of SmartOS pkgsrc 2019Q4, install the following packages::
+	pkgin in autoconf automake editline libtool ncurses \
+		 pcre2 python37 py37-sphinx py37-docutils gmake gcc8 pkg-config
 ..
-.. 	pkgin in autoconf automake editline libtool ncurses \
-.. 		 pcre python37 py37-sphinx py37-docutils gmake gcc8 pkg-config
+*Note:* you will probably need to add ``/opt/local/gcc8/bin`` to
+``PATH`` in order to have ``gcc`` available.
 ..
-.. *Note:* you will probably need to add ``/opt/local/gcc8/bin`` to
-.. ``PATH`` in order to have ``gcc`` available.
-..
-.. Optionally, to rebuild the svg files::
+Optionally, to rebuild the svg files::
 ..
-.. 	pkgin in graphviz
+	pkgin in graphviz
 ..
-.. Optionally, to pull from a repository::
+Optionally, to pull from a repository::
 ..
-.. 	pkgin in git
+	pkgin in git
 
 Building on Solaris and other Solaris-ish OSes
 ----------------------------------------------

From nils.goroll at uplex.de  Tue Aug  3 11:17:06 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  3 Aug 2021 11:17:06 +0000 (UTC)
Subject: [master] 8445ff8f9 No change is trivial enough to not require a test
 make :|
Message-ID: <20210803111706.58874962C4@lists.varnish-cache.org>


commit 8445ff8f98b5809077326b51ab5104e620c2d566
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Aug 3 13:16:09 2021 +0200

    No change is trivial enough to not require a test make :|

diff --git a/doc/sphinx/installation/install_source.rst b/doc/sphinx/installation/install_source.rst
index 2d4908c6e..0415513b2 100644
--- a/doc/sphinx/installation/install_source.rst
+++ b/doc/sphinx/installation/install_source.rst
@@ -197,21 +197,21 @@ Then continue `Compiling Varnish`_, using the ``--with-unwind``
 
 Build dependencies on a SmartOS Zone
 ------------------------------------
-..
+
 As of SmartOS pkgsrc 2019Q4, install the following packages::
-..
+
 	pkgin in autoconf automake editline libtool ncurses \
 		 pcre2 python37 py37-sphinx py37-docutils gmake gcc8 pkg-config
-..
+
 *Note:* you will probably need to add ``/opt/local/gcc8/bin`` to
 ``PATH`` in order to have ``gcc`` available.
-..
+
 Optionally, to rebuild the svg files::
-..
+
 	pkgin in graphviz
-..
+
 Optionally, to pull from a repository::
-..
+
 	pkgin in git
 
 Building on Solaris and other Solaris-ish OSes

From dridi.boukelmoune at gmail.com  Tue Aug  3 11:40:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue,  3 Aug 2021 11:40:07 +0000 (UTC)
Subject: [6.0] 31e3895ab vcl: Change the order of sess.* variables
Message-ID: <20210803114007.1A94696F89@lists.varnish-cache.org>


commit 31e3895ab66d79a69b553b67fb63288049cb591d
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 2 14:43:26 2021 +0200

    vcl: Change the order of sess.* variables
    
    This is just the order of their declaration in the VCL manual.
    
    As a side effect it works around a bug where the sess.xid syntax
    requirements would prevent sess.timeout_idle to be used in VCL 4.0,
    which is less intrusive than a proper fix.
    
    The bug was fixed in trunk without being noticed in the first place
    after many heavy changes to libvcc. For a stable branch this is less
    risky than a back-port since there are only two sess.* symbols.
    
    Fixes #3564

diff --git a/bin/varnishtest/tests/r03564.vtc b/bin/varnishtest/tests/r03564.vtc
new file mode 100644
index 000000000..7252d009b
--- /dev/null
+++ b/bin/varnishtest/tests/r03564.vtc
@@ -0,0 +1,31 @@
+varnishtest "sess.* symbols and vcl syntax"
+
+varnish v1 -vcl {
+	backend be none;
+
+	sub vcl_deliver {
+		set resp.http.sess-xid = sess.xid;
+		set resp.http.sess-timeout-idle = sess.timeout_idle;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.http.sess-xid == 1000
+	expect resp.http.sess-timeout-idle == 5.000
+} -run
+
+varnish v1 -syntax 4.0 -vcl {
+	backend be none;
+	sub vcl_deliver {
+		set resp.http.sess-timeout-idle = sess.timeout_idle;
+	}
+}
+
+varnish v1 -syntax 4.0 -errvcl "Symbol not found: 'sess.xid'" {
+	backend be none;
+	sub vcl_deliver {
+		set resp.http.sess-xid = sess.xid;
+	}
+}
diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst
index 55b0defca..b7266a103 100644
--- a/doc/sphinx/reference/vcl_var.rst
+++ b/doc/sphinx/reference/vcl_var.rst
@@ -1172,14 +1172,6 @@ transactions may take place. It may comprise the traffic over an
 HTTP/1 keep-alive connection, or the multiplexed traffic over an
 HTTP/2 connection.
 
-sess.xid	``VCL >= 4.1``
-
-	Type: STRING
-
-	Readable from: client, backend
-
-	Unique ID of this session.
-
 sess.timeout_idle
 
 	Type: DURATION
@@ -1191,6 +1183,14 @@ sess.timeout_idle
 	Idle timeout for this session, defaults to the
 	``timeout_idle`` parameter, see :ref:`varnishd(1)`
 
+sess.xid	``VCL >= 4.1``
+
+	Type: STRING
+
+	Readable from: client, backend
+
+	Unique ID of this session.
+
 storage
 ~~~~~~~
 

From dridi.boukelmoune at gmail.com  Tue Aug  3 11:54:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue,  3 Aug 2021 11:54:06 +0000 (UTC)
Subject: [master] 1bc7abb82 vtc: Import r03564 from the 6.0 branch
Message-ID: <20210803115406.1E048977FE@lists.varnish-cache.org>


commit 1bc7abb82981e54c05c1c710c5ac0f1bf0622482
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 3 13:41:39 2021 +0200

    vtc: Import r03564 from the 6.0 branch
    
    Refs 31e3895ab66d

diff --git a/bin/varnishtest/tests/r03564.vtc b/bin/varnishtest/tests/r03564.vtc
new file mode 100644
index 000000000..7252d009b
--- /dev/null
+++ b/bin/varnishtest/tests/r03564.vtc
@@ -0,0 +1,31 @@
+varnishtest "sess.* symbols and vcl syntax"
+
+varnish v1 -vcl {
+	backend be none;
+
+	sub vcl_deliver {
+		set resp.http.sess-xid = sess.xid;
+		set resp.http.sess-timeout-idle = sess.timeout_idle;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.http.sess-xid == 1000
+	expect resp.http.sess-timeout-idle == 5.000
+} -run
+
+varnish v1 -syntax 4.0 -vcl {
+	backend be none;
+	sub vcl_deliver {
+		set resp.http.sess-timeout-idle = sess.timeout_idle;
+	}
+}
+
+varnish v1 -syntax 4.0 -errvcl "Symbol not found: 'sess.xid'" {
+	backend be none;
+	sub vcl_deliver {
+		set resp.http.sess-xid = sess.xid;
+	}
+}

From nils.goroll at uplex.de  Tue Aug  3 12:34:05 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  3 Aug 2021 12:34:05 +0000 (UTC)
Subject: [master] 5c55d208b Add vre_pcre2.h to contain the backend specifics
 of VRE
Message-ID: <20210803123405.EF1B99BD0D@lists.varnish-cache.org>


commit 5c55d208bdff57bab7cb6ed047e96ad1247fa627
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Aug 2 16:07:38 2021 +0200

    Add vre_pcre2.h to contain the backend specifics of VRE
    
    For now, this is only VRE_unpack() to get the pcre2_code handle.
    
    If we ever change the regex backend again, we will remove
    vre_pcre2.h and add something else.
    
    Suggested by Dridi, thus authorship attributed to Dridi.

diff --git a/include/Makefile.am b/include/Makefile.am
index 818011d3d..1eeaa3537 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -67,6 +67,7 @@ nobase_pkginclude_HEADERS += \
 	vmod_abi.h \
 	vqueue.h \
 	vre.h \
+	vre_pcre2.h \
 	vdef.h \
 	vrt.h \
 	vrt_obj.h \
diff --git a/include/vre_pcre2.h b/include/vre_pcre2.h
new file mode 100644
index 000000000..f020875f8
--- /dev/null
+++ b/include/vre_pcre2.h
@@ -0,0 +1,45 @@
+/*-
+ * Copyright (c) 2021 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Access to the PCRE2 backend in VRE
+ *
+ */
+
+#ifndef VRE_H_INCLUDED
+#error Include vre.h before vre_pcre2.h
+#endif
+
+#ifndef VRE_PCRE2_H_INCLUDED
+#define VRE_PCRE2_H_INCLUDED
+
+#include <pcre2.h>
+
+pcre2_code *VRE_unpack(const vre_t *code);
+
+#endif /* VRE_PCRE2_H_INCLUDED */
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index ad0d31a3a..ca3029d3f 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -34,8 +34,6 @@
 #include <string.h>
 #include <unistd.h>
 
-#include <pcre2.h>
-
 #include "vdef.h"
 
 #include "vas.h"	// XXX Flexelint "not used" - but req'ed for assert()
@@ -43,6 +41,7 @@
 #include "miniobj.h"
 
 #include "vre.h"
+#include "vre_pcre2.h"
 
 #if !HAVE_PCRE2_SET_DEPTH_LIMIT
 #  define pcre2_set_depth_limit(r, d) pcre2_set_recursion_limit(r, d)
@@ -132,8 +131,8 @@ VRE_error(struct vsb *vsb, int err)
 	return (0);
 }
 
-static pcre2_code *
-vre_unpack(const vre_t *code)
+pcre2_code *
+VRE_unpack(const vre_t *code)
 {
 
 	CHECK_OBJ_NOTNULL(code, VRE_MAGIC);
@@ -168,7 +167,7 @@ VRE_export(const vre_t *code, size_t *sz)
 	vre_t *exp;
 
 	CHECK_OBJ_NOTNULL(code, VRE_MAGIC);
-	re = vre_unpack(code);
+	re = VRE_unpack(code);
 	AZ(pcre2_pattern_info(re, PCRE2_INFO_SIZE, sz));
 
 	exp = malloc(sizeof(*exp) + *sz);
@@ -190,7 +189,7 @@ vre_match(const vre_t *code, const char *subject, size_t length, size_t offset,
 	pcre2_code *re;
 	int matches;
 
-	re = vre_unpack(code);
+	re = VRE_unpack(code);
 
 	if (datap != NULL && *datap != NULL) {
 		data = *datap;

From nils.goroll at uplex.de  Tue Aug  3 12:34:06 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  3 Aug 2021 12:34:06 +0000 (UTC)
Subject: [master] 3c399b858 Move PCRE2_CODE_UNIT_WIDTH to vre_pcre2.h
Message-ID: <20210803123406.1304A9BD10@lists.varnish-cache.org>


commit 3c399b8585bd1abe26922792c33ec2fe836dad72
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Aug 2 17:59:30 2021 +0200

    Move PCRE2_CODE_UNIT_WIDTH to vre_pcre2.h
    
    We need to ensure that vmods use the same functions and data
    types as varnish-cache.
    
    Also it seems that having the define in autoconf might suggest
    that it could be changed, which would be misleading.

diff --git a/configure.ac b/configure.ac
index 2b96413d6..c396a3806 100644
--- a/configure.ac
+++ b/configure.ac
@@ -138,7 +138,6 @@ else
 fi
 AC_SUBST(PCRE2_CFLAGS)
 AC_SUBST(PCRE2_LIBS)
-AC_DEFINE([PCRE2_CODE_UNIT_WIDTH], [8], [Work with 8-bit characters for PCRE2])
 
 save_LIBS="${LIBS}"
 LIBS="${LIBS} ${PCRE2_LIBS}"
diff --git a/include/vre_pcre2.h b/include/vre_pcre2.h
index f020875f8..d8af96e8d 100644
--- a/include/vre_pcre2.h
+++ b/include/vre_pcre2.h
@@ -38,6 +38,8 @@
 #ifndef VRE_PCRE2_H_INCLUDED
 #define VRE_PCRE2_H_INCLUDED
 
+#define PCRE2_CODE_UNIT_WIDTH 8
+
 #include <pcre2.h>
 
 pcre2_code *VRE_unpack(const vre_t *code);

From phk at FreeBSD.org  Wed Aug  4 08:05:07 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed,  4 Aug 2021 08:05:07 +0000 (UTC)
Subject: [master] e7ed39857 Cater for the older kevent(2) API still in use on
 NetBSD
Message-ID: <20210804080507.E7B1793846@lists.varnish-cache.org>


commit e7ed398572051d0ea87d93722b28237c7755aaad
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 4 08:04:19 2021 +0000

    Cater for the older kevent(2) API still in use on NetBSD
    
    Fixes: #3642

diff --git a/bin/varnishd/waiter/cache_waiter_kqueue.c b/bin/varnishd/waiter/cache_waiter_kqueue.c
index 27ff1cca6..8e70af060 100644
--- a/bin/varnishd/waiter/cache_waiter_kqueue.c
+++ b/bin/varnishd/waiter/cache_waiter_kqueue.c
@@ -113,11 +113,11 @@ vwk_thread(void *priv)
 		now = VTIM_real();
 		for (kp = ke, j = 0; j < n; j++, kp++) {
 			assert(kp->filter == EVFILT_READ);
-			if (ke[j].udata == vwk) {
+			if ((uintptr_t)ke[j].udata == (uintptr_t)vwk) {
 				assert(read(vwk->pipe[0], &c, 1) == 1);
 				continue;
 			}
-			CAST_OBJ_NOTNULL(wp, ke[j].udata, WAITED_MAGIC);
+			CAST_OBJ_NOTNULL(wp, (void*)ke[j].udata, WAITED_MAGIC);
 			Lck_Lock(&vwk->mtx);
 			AN(Wait_HeapDelete(w, wp));
 			Lck_Unlock(&vwk->mtx);

From phk at FreeBSD.org  Wed Aug  4 09:11:06 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed,  4 Aug 2021 09:11:06 +0000 (UTC)
Subject: [master] 38506b339 Use "help -j" to get list for command completion.
Message-ID: <20210804091106.4B57495993@lists.varnish-cache.org>


commit 38506b339af043d64bef12b7c27fbfc7df62e657
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 4 09:07:43 2021 +0000

    Use "help -j" to get list for command completion.
    
    Do it on demand, so start/stop changes to available commands is properly reflected.

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index ebc1aa2cc..d95cc8845 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -61,10 +61,13 @@
 
 #include "vdef.h"
 
+#include "vqueue.h"
+
 #include "vapi/vsig.h"
 #include "vapi/vsm.h"
 #include "vas.h"
 #include "vcli.h"
+#include "vjsn.h"
 #include "vtcp.h"
 
 #define RL_EXIT(status) \
@@ -76,6 +79,7 @@
 
 static double timeout = 5;
 static int p_arg = 0;
+static int line_sock;
 
 static void
 cli_write(int sock, const char *s)
@@ -212,13 +216,12 @@ do_args(int sock, int argc, char * const *argv)
 /* Callback for readline, doesn't take a private pointer, so we need
  * to have a global variable.
  */
-static int _line_sock;
 static void v_matchproto_()
 send_line(char *l)
 {
 	if (l) {
-		cli_write(_line_sock, l);
-		cli_write(_line_sock, "\n");
+		cli_write(line_sock, l);
+		cli_write(line_sock, "\n");
 		if (*l)
 			add_history(l);
 		rl_callback_handler_install("varnish> ", send_line);
@@ -227,27 +230,49 @@ send_line(char *l)
 	}
 }
 
-static char *commands[256];
 static char *
 command_generator (const char *text, int state)
 {
-	static int list_index, len;
-	const char *name;
+	static struct vjsn *jsn_cmds;
+	static const struct vjsn_val *jv;
+	struct vjsn_val *jv2;
+	unsigned u;
+	char *answer = NULL;
+	const char *err;
 
-	/* If this is a new word to complete, initialize now.  This
-	   includes saving the length of TEXT for efficiency, and
-	   initializing the index variable to 0. */
 	if (!state) {
-		list_index = 0;
-		len = strlen(text);
+		cli_write(line_sock, "help -j\n");
+		u = VCLI_ReadResult(line_sock, NULL, &answer, timeout);
+		if (u) {
+			free(answer);
+			return (NULL);
+		}
+		jsn_cmds = vjsn_parse(answer, &err);
+		if (err != NULL)
+			return (NULL);
+		free(answer);
+		AN(jsn_cmds);
+		AN(jsn_cmds->value);
+		assert (jsn_cmds->value->type == VJSN_ARRAY);
+		jv = VTAILQ_FIRST(&jsn_cmds->value->children);
+		assert (jv->type == VJSN_NUMBER);
+		jv = VTAILQ_NEXT(jv, list);
+		assert (jv->type == VJSN_ARRAY);
+		jv = VTAILQ_NEXT(jv, list);
+		assert (jv->type == VJSN_NUMBER);
+		jv = VTAILQ_NEXT(jv, list);
 	}
-
-	while ((name = commands[list_index]) != NULL) {
-		list_index++;
-		if (strncmp (name, text, len) == 0)
-			return (strdup(name));
+	while (jv != NULL) {
+		assert (jv->type == VJSN_OBJECT);
+		jv2 = VTAILQ_FIRST(&jv->children);
+		AN(jv2);
+		jv = VTAILQ_NEXT(jv, list);
+		assert (jv2->type == VJSN_STRING);
+		assert (!strcmp(jv2->name, "request"));
+		if (!strncmp(text, jv2->value, strlen(text)))
+			return (strdup(jv2->value));
 	}
-	/* If no names matched, then return NULL. */
+	vjsn_delete(&jsn_cmds);
 	return (NULL);
 }
 
@@ -272,9 +297,7 @@ interactive(int sock)
 {
 	struct pollfd fds[2];
 	int i;
-	char *answer = NULL;
-	unsigned u, status;
-	_line_sock = sock;
+	line_sock = sock;
 	rl_already_prompted = 1;
 	rl_callback_handler_install("varnish> ", send_line);
 	rl_attempted_completion_function = varnishadm_completion;
@@ -284,34 +307,6 @@ interactive(int sock)
 	fds[1].fd = 0;
 	fds[1].events = POLLIN;
 
-	/* Grab the commands, for completion */
-	cli_write(sock, "help\n");
-	u = VCLI_ReadResult(fds[0].fd, &status, &answer, timeout);
-	if (!u) {
-		char *t, c[128];
-		if (status == CLIS_COMMS) {
-			RL_EXIT(0);
-		}
-		t = answer;
-
-		i = 0;
-		while (*t) {
-			if (sscanf(t, "%127s", c) == 1) {
-				commands[i++] = strdup(c);
-				while (*t != '\n' && *t != '\0')
-					t++;
-				if (*t == '\n')
-					t++;
-			} else {
-				/* what? */
-				fprintf(stderr, "Unknown command '%s' parsing "
-					"help output. Tab completion may be "
-					"broken\n", t);
-				break;
-			}
-		}
-	}
-	free(answer);
 	cli_write(sock, "banner\n");
 	while (1) {
 		i = poll(fds, 2, -1);

From dridi.boukelmoune at gmail.com  Fri Aug  6 11:51:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri,  6 Aug 2021 11:51:07 +0000 (UTC)
Subject: [master] 82081bab1 vre: Check return value when setting limits
Message-ID: <20210806115107.268B710F3F9@lists.varnish-cache.org>


commit 82081bab1ec376f276a9c04c1714b404b3f66c4f
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Aug 6 11:42:59 2021 +0200

    vre: Check return value when setting limits
    
    Spotted by flexelint.

diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index ca3029d3f..6a2040823 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -156,8 +156,8 @@ vre_limit(const vre_t *code, const volatile struct vre_limits *lim)
 
 	/* XXX: not reentrant */
 	AN(code->re_ctx);
-	pcre2_set_match_limit(code->re_ctx, lim->match);
-	pcre2_set_depth_limit(code->re_ctx, lim->depth);
+	AZ(pcre2_set_match_limit(code->re_ctx, lim->match));
+	AZ(pcre2_set_depth_limit(code->re_ctx, lim->depth));
 }
 
 vre_t *

From dridi.boukelmoune at gmail.com  Fri Aug  6 11:51:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri,  6 Aug 2021 11:51:07 +0000 (UTC)
Subject: [master] d311ad3a8 vtc-bisect: Missing variable initialization
Message-ID: <20210806115107.3C2E510F3FC@lists.varnish-cache.org>


commit d311ad3a828aba13cc55d885a3b8b3ea375dba57
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Aug 6 11:45:43 2021 +0200

    vtc-bisect: Missing variable initialization

diff --git a/tools/vtc-bisect.sh b/tools/vtc-bisect.sh
index e70c9fff9..69a27af03 100755
--- a/tools/vtc-bisect.sh
+++ b/tools/vtc-bisect.sh
@@ -31,6 +31,8 @@
 set -e
 set -u
 
+SCRIPT=$0
+
 usage() {
 	test $# -eq 1 &&
 	printf 'Error: %s.\n\n' "$1"

From dridi.boukelmoune at gmail.com  Fri Aug  6 11:51:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri,  6 Aug 2021 11:51:07 +0000 (UTC)
Subject: [master] 255da824f vtc-bisect: Don't document internal options
Message-ID: <20210806115107.5591710F400@lists.varnish-cache.org>


commit 255da824fc82de4ff043fb4f604a8a565eb208e9
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Aug 6 11:46:10 2021 +0200

    vtc-bisect: Don't document internal options

diff --git a/tools/vtc-bisect.sh b/tools/vtc-bisect.sh
index 69a27af03..e4c82e083 100755
--- a/tools/vtc-bisect.sh
+++ b/tools/vtc-bisect.sh
@@ -39,7 +39,6 @@ usage() {
 
 	cat <<-EOF
 	Usage: $SCRIPT [-b <rev>] [-g <rev>] [-j <jobs>] [<file>]
-	       $SCRIPT [-j <jobs>] -r <file>
 	       $SCRIPT -h
 
 	Automatically look for the change that introduced a regression with
@@ -50,7 +49,6 @@ usage() {
 	-b <rev>   : bad revision (defaults to HEAD)
 	-g <rev>   : good revision (defaults to latest tag before bad)
 	-j <jobs>  : number of jobs for make invocations (defaults to 8)
-	-r         : git-bisect(1) run mode
 
 	When <file> is empty or missing, bisect.vtc is expected to be found
 	at the root of the git repository. The current source tree is used
@@ -96,7 +94,7 @@ do
 	g) BISECT_GOOD=$OPTARG ;;
 	h) usage ;;
 	j) MAKE_JOBS=$OPTARG ;;
-	r) RUN_MODE=true ;;
+	r) RUN_MODE=true ;; # -r usage is strictly internal
 	*) usage "wrong usage" >&2 ;;
 	esac
 done

From dridi.boukelmoune at gmail.com  Fri Aug  6 11:51:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri,  6 Aug 2021 11:51:07 +0000 (UTC)
Subject: [master] f850b6086 vtc-bisect: New inverted mode
Message-ID: <20210806115107.6F74710F404@lists.varnish-cache.org>


commit f850b608648f6fcdddd6f13e2e94e7c87f4da046
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Aug 6 12:00:51 2021 +0200

    vtc-bisect: New inverted mode
    
    This is how I found the accidental bug fix for #3662:
    
        git show varnish-6.0.8:bin/varnishtest/tests/c00053.vtc >bisect.vtc
        tools/vtc-bisect.sh -g varnish-6.0.0 -j32 -i
    
    I hope this will help others, I use this script whenever the cause of a
    failure is not obvious.

diff --git a/tools/vtc-bisect.sh b/tools/vtc-bisect.sh
index e4c82e083..e0f2e8b92 100755
--- a/tools/vtc-bisect.sh
+++ b/tools/vtc-bisect.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Copyright (c) 2019 Varnish Software AS
+# Copyright (c) 2019, 2021 Varnish Software AS
 # All rights reserved.
 #
 # Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
@@ -38,7 +38,7 @@ usage() {
 	printf 'Error: %s.\n\n' "$1"
 
 	cat <<-EOF
-	Usage: $SCRIPT [-b <rev>] [-g <rev>] [-j <jobs>] [<file>]
+	Usage: $SCRIPT [-b <rev>] [-g <rev>] [-i] [-j <jobs>] [<file>]
 	       $SCRIPT -h
 
 	Automatically look for the change that introduced a regression with
@@ -48,12 +48,17 @@ usage() {
 	-h         : show this help and exit
 	-b <rev>   : bad revision (defaults to HEAD)
 	-g <rev>   : good revision (defaults to latest tag before bad)
+	-i         : inverted mode, look for a bug fix instead
 	-j <jobs>  : number of jobs for make invocations (defaults to 8)
 
 	When <file> is empty or missing, bisect.vtc is expected to be found
 	at the root of the git repository. The current source tree is used
 	and VPATH setups are not supported.
 
+	The -i option inverses the bisection behavior: the test case is now
+	expected to fail on the good revision and pass on the bad revision.
+	This is useful to track a bug that was fixed without being noticed.
+
 	This script is expected to run from the root of the git repository
 	as well.
 	EOF
@@ -77,22 +82,29 @@ run() {
 		exit 125
 	fi
 
-	bin/varnishtest/varnishtest -i "$VTC_FILE"
+	if [ -n "$INVERSE" ]
+	then
+		! bin/varnishtest/varnishtest -i "$VTC_FILE"
+	else
+		bin/varnishtest/varnishtest -i "$VTC_FILE"
+	fi
 	exit $?
 }
 
 BISECT_GOOD=
 BISECT_BAD=
 MAKE_JOBS=
+INVERSE=
 RUN_MODE=false
 VTC_FILE=
 
-while getopts b:g:hj:r OPT
+while getopts b:g:hij:r OPT
 do
 	case $OPT in
 	b) BISECT_BAD=$OPTARG ;;
 	g) BISECT_GOOD=$OPTARG ;;
 	h) usage ;;
+	i) INVERSE=-i ;;
 	j) MAKE_JOBS=$OPTARG ;;
 	r) RUN_MODE=true ;; # -r usage is strictly internal
 	*) usage "wrong usage" >&2 ;;
@@ -128,5 +140,6 @@ cd "$ROOT_DIR"
 git bisect start
 git bisect good "$BISECT_GOOD"
 git bisect bad "$BISECT_BAD"
-git bisect run "$TMP_DIR"/vtc-bisect.sh -r -j "$MAKE_JOBS" "$TMP_DIR"/bisect.vtc
+git bisect run "$TMP_DIR"/vtc-bisect.sh -r -j "$MAKE_JOBS" $INVERSE \
+	"$TMP_DIR"/bisect.vtc
 git bisect reset

From dridi.boukelmoune at gmail.com  Mon Aug  9 04:33:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon,  9 Aug 2021 04:33:06 +0000 (UTC)
Subject: [master] 1e1903483 vrt: Missing assertion
Message-ID: <20210809043306.E3784105520@lists.varnish-cache.org>


commit 1e1903483cb7ee601db4f859bd0d948d3f6da554
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 9 06:29:15 2021 +0200

    vrt: Missing assertion
    
    Spotted by flexelint.

diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c
index 944a40bce..58d980d1b 100644
--- a/bin/varnishd/cache/cache_vrt_re.c
+++ b/bin/varnishd/cache/cache_vrt_re.c
@@ -76,7 +76,7 @@ VRT_re_match(VRT_CTX, const char *s, VCL_REGEX re)
 	if (i >= 0)
 		return (1);
 	if (i < VRE_ERROR_NOMATCH ) {
-		VSB_init(vsb, errbuf, sizeof errbuf);
+		AN(VSB_init(vsb, errbuf, sizeof errbuf));
 		AZ(VRE_error(vsb, i));
 		AZ(VSB_finish(vsb));
 		VSB_fini(vsb);

From nils.goroll at uplex.de  Mon Aug  9 13:37:07 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon,  9 Aug 2021 13:37:07 +0000 (UTC)
Subject: [master] 886980154 Increase vsl_buffer and workspaces
Message-ID: <20210809133707.DA2DA1194ED@lists.varnish-cache.org>


commit 8869801544c52b59812ff7272fe753cc4569cd91
Author: P?l Hermunn Johansen <hermunn at varnish-software.com>
Date:   Wed Jul 14 17:36:24 2021 +0200

    Increase vsl_buffer and workspaces
    
    On many busy production systems, the VSL buffer for transactions often
    fills up, triggering a flush to the VSM. If such a transaction takes a
    long time, it can happen that the flushed data is overwritten before
    the whole transaction completes. The result is that these transactions
    are missed by varnishncsa and other tools. Increasing the vsl_buffer
    does the trick, at some cost in workspace usage.
    
    The cost implies that one should increase the workspaces, too. Since
    the old defaults of 64k each are too low for many common use cases
    today, it is natural to increse everything in one patch.
    
    Some test cases have been updated. These are all cases where the
    workspaces have been set very low, and to make sure we are still
    testing the same regression, the vsl_buffer has been set to the old
    default instead of increasing the workspace.

diff --git a/bin/varnishtest/tests/r01038.vtc b/bin/varnishtest/tests/r01038.vtc
index 540deb8be..6a3c7899c 100644
--- a/bin/varnishtest/tests/r01038.vtc
+++ b/bin/varnishtest/tests/r01038.vtc
@@ -45,7 +45,7 @@ server s1 {
 	txresp -body "foo8"
 } -start
 
-varnish v1 -arg "-p workspace_backend=10k" -vcl+backend {
+varnish v1 -arg "-p workspace_backend=10k -p vsl_buffer=4k" -vcl+backend {
 	sub vcl_backend_response {
 		if (bereq.url == "/") {
 			set beresp.do_esi = true;
diff --git a/bin/varnishtest/tests/r01120.vtc b/bin/varnishtest/tests/r01120.vtc
index d906096d7..577e57b2c 100644
--- a/bin/varnishtest/tests/r01120.vtc
+++ b/bin/varnishtest/tests/r01120.vtc
@@ -7,7 +7,7 @@ server s1 {
 	txresp -hdr "Vary: Foo" -body "yyyyy"
 } -start
 
-varnish v1 \
+varnish v1 -arg "-p vsl_buffer=4k" \
 	-cliok "param.set workspace_client 10k" \
 	-cliok "param.set workspace_backend 200k" \
 	-vcl+backend {
diff --git a/bin/varnishtest/tests/r02219.vtc b/bin/varnishtest/tests/r02219.vtc
index 5cc3abafc..3604392a0 100644
--- a/bin/varnishtest/tests/r02219.vtc
+++ b/bin/varnishtest/tests/r02219.vtc
@@ -9,7 +9,10 @@ server s1 {
 	txresp
 } -start
 
-varnish v1 -arg "-p workspace_client=9k" -proto PROXY -vcl+backend {
+varnish v1 -arg "-p workspace_client=9k" \
+	   -arg "-p vsl_buffer=4k" \
+	   -proto PROXY \
+	   -vcl+backend {
 	import std;
 	import vtc;
 	sub vcl_recv {
diff --git a/include/tbl/params.h b/include/tbl/params.h
index 074048e9e..9b30b37d5 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -1022,7 +1022,7 @@ PARAM_SIMPLE(
 	/* type */	vsl_buffer,
 	/* min */	"267",
 	/* max */	NULL,
-	/* def */	"4k",
+	/* def */	"16k",
 	/* units */	"bytes",
 	/* descr */
 	"Bytes of (req-/backend-)workspace dedicated to buffering VSL "
@@ -1082,7 +1082,7 @@ PARAM_SIMPLE(
 	/* type */	bytes_u,
 	/* min */	"1k",
 	/* max */	NULL,
-	/* def */	"64k",
+	/* def */	"96k",
 	/* units */	"bytes",
 	/* descr */
 	"Bytes of HTTP protocol workspace for backend HTTP req/resp.  If "
@@ -1095,7 +1095,7 @@ PARAM_SIMPLE(
 	/* type */	bytes_u,
 	/* min */	"9k",
 	/* max */	NULL,
-	/* def */	"64k",
+	/* def */	"96k",
 	/* units */	"bytes",
 	/* descr */
 	"Bytes of HTTP protocol workspace for clients HTTP req/resp.  Use a "

From dridi.boukelmoune at gmail.com  Mon Aug  9 15:09:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon,  9 Aug 2021 15:09:06 +0000 (UTC)
Subject: [master] a33ea4015 waiter: Adjust assertion based on usage
Message-ID: <20210809150906.7AF6211BB33@lists.varnish-cache.org>


commit a33ea4015bb87ae42252e0f42c8d01b23e081675
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 9 16:59:13 2021 +0200

    waiter: Adjust assertion based on usage
    
    This looks like a copy-pasta or autocomplete mistake.

diff --git a/bin/varnishd/waiter/waiter_priv.h b/bin/varnishd/waiter/waiter_priv.h
index e44c548b9..f0bef789c 100644
--- a/bin/varnishd/waiter/waiter_priv.h
+++ b/bin/varnishd/waiter/waiter_priv.h
@@ -63,7 +63,7 @@ struct waiter_impl {
 static inline double
 Wait_When(const struct waited *wp)
 {
-	CHECK_OBJ_ORNULL(wp, WAITED_MAGIC);
+	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
 	return (wp->idle + wp->tmo);
 }
 

From phk at FreeBSD.org  Tue Aug 10 07:28:08 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 10 Aug 2021 07:28:08 +0000 (UTC)
Subject: [master] 08132c6f2 Failing -I <file> processing should be ARGV_ERR()
Message-ID: <20210810072808.E8F4EA987A@lists.varnish-cache.org>


commit 08132c6f2e6e49d0759a467c4e922167c94a1c9e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 10 06:57:25 2021 +0000

    Failing -I <file> processing should be ARGV_ERR()

diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index 9d307f793..ab889506f 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -349,9 +349,8 @@ mgt_cli_cb_after(const struct cli *cli)
 	    cli->ident, cli->result, VSB_data(cli->sb));
 	if (cli->priv == stderr &&
 	    cli->result != CLIS_OK && *VSB_data(cli->cmd) != '-') {
-		MGT_Complain(C_ERR, "-I file CLI command failed (%d)\n%s\n",
+		ARGV_ERR("-I file CLI command failed (%d)\n%s\n",
 		    cli->result, VSB_data(cli->sb));
-		exit(2);
 	}
 }
 

From phk at FreeBSD.org  Tue Aug 10 07:28:09 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 10 Aug 2021 07:28:09 +0000 (UTC)
Subject: [master] 3dd6aa06e Only MGT_Complain() on stderr when -d or -F flags
 given.
Message-ID: <20210810072809.0AA9CA987D@lists.varnish-cache.org>


commit 3dd6aa06e53a2341b3a5a903a55e620f11eef8be
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 10 06:58:40 2021 +0000

    Only MGT_Complain() on stderr when -d or -F flags given.

diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 02a17f88a..bf0bacca9 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -176,6 +176,7 @@ extern const char C_INFO[];	// Normal stuff, keep a record for later
 extern const char C_DEBUG[];	// More detail than you'd normally want
 extern const char C_SECURITY[];	// Security issues
 extern const char C_CLI[];	// CLI traffic between master and child
+extern int complain_to_stderr;
 
 /* mgt_param.c */
 void MCF_InitParams(struct cli *);
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index f16e13127..dca5c4b52 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -579,6 +579,9 @@ main(int argc, char * const *argv)
 	if (!d_flag && b_arg == NULL && !f_flag)
 		ARGV_ERR("Neither -b nor -f given. (use -f '' to override)\n");
 
+	if (d_flag || F_flag)
+		complain_to_stderr = 1;
+
 	/*
 	 * Start out by closing all unwanted file descriptors we might
 	 * have inherited from sloppy process control daemons.
diff --git a/bin/varnishd/mgt/mgt_util.c b/bin/varnishd/mgt/mgt_util.c
index 889fdc69b..d9ec5a0f1 100644
--- a/bin/varnishd/mgt/mgt_util.c
+++ b/bin/varnishd/mgt/mgt_util.c
@@ -49,6 +49,8 @@
 #include "vav.h"
 #include "vct.h"
 
+int complain_to_stderr;
+
 /*--------------------------------------------------------------------*/
 
 char *
@@ -170,7 +172,7 @@ MGT_Complain(const char *loud, const char *fmt, ...)
 	else
 		WRONG("Wrong complaint loudness");
 
-	if (loud != C_CLI)
+	if (complain_to_stderr && loud != C_CLI)
 		fprintf(stderr, "%s %s\n", loud, VSB_data(vsb));
 
 	if (!MGT_DO_DEBUG(DBG_VTC_MODE))

From phk at FreeBSD.org  Tue Aug 10 07:28:09 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 10 Aug 2021 07:28:09 +0000 (UTC)
Subject: [master] ef23612bf Do not emit 'VCL compiled' message during argv VCL
 loading
Message-ID: <20210810072809.25E0FA9881@lists.varnish-cache.org>


commit ef23612bf67a4b96669dfde3487ebe264549374d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 10 06:59:50 2021 +0000

    Do not emit 'VCL compiled' message during argv VCL loading

diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c
index 5775a2fb9..faedb803b 100644
--- a/bin/varnishd/mgt/mgt_vcc.c
+++ b/bin/varnishd/mgt/mgt_vcc.c
@@ -384,8 +384,6 @@ mgt_VccCompile(struct cli *cli, struct vclprog *vcl, const char *vclname,
 	AN(p);
 	mgt_vcl_symtab(vcl, p);
 
-	VCLI_Out(cli, "VCL compiled.\n");
-
 	REPLACE(p, VSB_data(vp->libfile));
 	mgt_vcc_fini_vp(vp, 1);
 	return (p);
diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c
index c6c78de21..69cd27ec5 100644
--- a/bin/varnishd/mgt/mgt_vcl.c
+++ b/bin/varnishd/mgt/mgt_vcl.c
@@ -421,7 +421,7 @@ mgt_vcl_setstate(struct cli *cli, struct vclprog *vp, const struct vclstate *vs)
 
 /*--------------------------------------------------------------------*/
 
-static void
+static int
 mgt_new_vcl(struct cli *cli, const char *vclname, const char *vclsrc,
     const char *vclsrcfile, const char *state, int C_flag)
 {
@@ -437,7 +437,7 @@ mgt_new_vcl(struct cli *cli, const char *vclname, const char *vclsrc,
 		VCLI_Out(cli, "Too many (%d) VCLs already loaded\n", vcl_count);
 		VCLI_Out(cli, "(See max_vcl and max_vcl_handling parameters)");
 		VCLI_SetResult(cli, CLIS_CANT);
-		return;
+		return (0);
 	}
 
 	if (state == NULL)
@@ -446,13 +446,13 @@ mgt_new_vcl(struct cli *cli, const char *vclname, const char *vclsrc,
 		vs = mcf_vcl_parse_state(cli, state);
 
 	if (vs == NULL)
-		return;
+		return (0);
 
 	vp = mgt_vcl_add(vclname, vs);
 	lib = mgt_VccCompile(cli, vp, vclname, vclsrc, vclsrcfile, C_flag);
 	if (lib == NULL) {
 		mgt_vcl_del(vp);
-		return;
+		return (0);
 	}
 
 	AZ(C_flag);
@@ -470,7 +470,7 @@ mgt_new_vcl(struct cli *cli, const char *vclname, const char *vclsrc,
 	}
 
 	if (!MCH_Running())
-		return;
+		return (0);
 
 	if (mgt_cli_askchild(&status, &p, "vcl.load %s %s %d%s\n",
 	    vp->name, vp->fname, vp->warm, vp->state->name)) {
@@ -478,11 +478,12 @@ mgt_new_vcl(struct cli *cli, const char *vclname, const char *vclsrc,
 		VCLI_Out(cli, "%s", p);
 		VCLI_SetResult(cli, status);
 		free(p);
-		return;
+		return (0);
 	}
 	free(p);
 
 	mgt_vcl_set_cooldown(vp, VTIM_mono());
+	return (1);
 }
 
 /*--------------------------------------------------------------------*/
@@ -501,7 +502,7 @@ mgt_vcl_startup(struct cli *cli, const char *vclsrc, const char *vclname,
 		vclname = buf;
 	}
 	active_vcl = NULL;
-	mgt_new_vcl(cli, vclname, vclsrc, origin, NULL, C_flag);
+	(void)mgt_new_vcl(cli, vclname, vclsrc, origin, NULL, C_flag);
 }
 
 /*--------------------------------------------------------------------*/
@@ -571,7 +572,8 @@ mcf_vcl_inline(struct cli *cli, const char * const *av, void *priv)
 	if (!mcf_find_no_vcl(cli, av[2]))
 		return;
 
-	mgt_new_vcl(cli, av[2], av[3], "<vcl.inline>", av[4], 0);
+	if (!mgt_new_vcl(cli, av[2], av[3], "<vcl.inline>", av[4], 0))
+		VCLI_Out(cli, "VCL compiled.\n");
 }
 
 static void v_matchproto_(cli_func_t)
@@ -582,7 +584,8 @@ mcf_vcl_load(struct cli *cli, const char * const *av, void *priv)
 	if (!mcf_find_no_vcl(cli, av[2]))
 		return;
 
-	mgt_new_vcl(cli, av[2], NULL, av[3], av[4], 0);
+	if (!mgt_new_vcl(cli, av[2], NULL, av[3], av[4], 0))
+		VCLI_Out(cli, "VCL compiled.\n");
 }
 
 

From phk at FreeBSD.org  Tue Aug 10 07:28:09 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 10 Aug 2021 07:28:09 +0000 (UTC)
Subject: [master] e470f9177 Remove an old fall-back to stdout which has not
 been used for approx 15 years.
Message-ID: <20210810072809.3FBC3A9886@lists.varnish-cache.org>


commit e470f91775fc49d17b129fa1652ae17fe50d92ee
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 10 07:13:07 2021 +0000

    Remove an old fall-back to stdout which has not been used for approx 15 years.

diff --git a/lib/libvarnish/vcli_serve.c b/lib/libvarnish/vcli_serve.c
index 2fc7dfe5b..1dcbb7da5 100644
--- a/lib/libvarnish/vcli_serve.c
+++ b/lib/libvarnish/vcli_serve.c
@@ -638,16 +638,13 @@ VCLI_Out(struct cli *cli, const char *fmt, ...)
 {
 	va_list ap;
 
+	AN(cli);
 	va_start(ap, fmt);
-	if (cli != NULL) {
-		CHECK_OBJ_NOTNULL(cli, CLI_MAGIC);
-		if (VSB_len(cli->sb) < *cli->limit)
-			(void)VSB_vprintf(cli->sb, fmt, ap);
-		else if (cli->result == CLIS_OK)
-			cli->result = CLIS_TRUNCATED;
-	} else {
-		(void)vfprintf(stdout, fmt, ap);
-	}
+	CHECK_OBJ_NOTNULL(cli, CLI_MAGIC);
+	if (VSB_len(cli->sb) < *cli->limit)
+		(void)VSB_vprintf(cli->sb, fmt, ap);
+	else if (cli->result == CLIS_OK)
+		cli->result = CLIS_TRUNCATED;
 	va_end(ap);
 }
 

From phk at FreeBSD.org  Tue Aug 10 07:28:09 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 10 Aug 2021 07:28:09 +0000 (UTC)
Subject: [master] dcd1718bd White-space OCD
Message-ID: <20210810072809.62247A9889@lists.varnish-cache.org>


commit dcd1718bda341d0a6925cdb1845bcf7f1b2362ab
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 10 07:27:00 2021 +0000

    White-space OCD

diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c
index 67da6f2f4..7114398d8 100644
--- a/bin/varnishd/storage/storage_simple.c
+++ b/bin/varnishd/storage/storage_simple.c
@@ -691,7 +691,7 @@ SML_panic(struct vsb *vsb, const struct objcore *oc)
 	    #U, o->va_##l##_len, o->va_##l);
 
 #define OBJ_AUXATTR(U, l)						\
-        do {								\
+	do {								\
 		if (o->aa_##l != NULL) sml_panic_st(vsb, #U, o->aa_##l);\
 	} while(0);
 
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index eda2ff1f0..60427d526 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -83,14 +83,14 @@ vcc_vcl_met2c(struct vsb *vsb, unsigned method)
 
 	//lint -e{774} Boolean within 'if' always evaluates to False
 #define VCL_MET_MAC(l,U,t,b)				\
-        do {						\
+	do {						\
 		if (method & VCL_MET_##U) {		\
 			if (d)				\
 				VSB_putc(vsb, '|');	\
 			VSB_cat(vsb, "VCL_MET_" #U);	\
 			d = 1;				\
 		}					\
-        } while (0);
+	} while (0);
 #include "tbl/vcl_returns.h"
 	AN(d);
 }
diff --git a/vmod/vmod_blob.c b/vmod/vmod_blob.c
index 393f7fb23..2f97979a2 100644
--- a/vmod/vmod_blob.c
+++ b/vmod/vmod_blob.c
@@ -128,7 +128,7 @@ parse_encoding(VCL_ENUM e)
 static enum case_e
 parse_case(VCL_ENUM e)
 {
-#define VMODENUM(n) 				\
+#define VMODENUM(n)				\
 	do {					\
 		if (e == VENUM(n)) return (n);	\
 	} while (0);

From hermunn at varnish-software.com  Tue Aug 10 09:58:05 2021
From: hermunn at varnish-software.com (Pål Hermunn Johansen)
Date: Tue, 10 Aug 2021 09:58:05 +0000 (UTC)
Subject: [master] 560d9443e Old default vsl_buffer for 32 bit
Message-ID: <20210810095805.E0DD1AE229@lists.varnish-cache.org>


commit 560d9443eae8ace9d744c4171db44a8babbda2cc
Author: P?l Hermunn Johansen <hermunn at varnish-software.com>
Date:   Tue Aug 10 11:42:18 2021 +0200

    Old default vsl_buffer for 32 bit
    
    After 886980154 broke 32 bit, we need to adjust the default(s) which
    are special cased for 32 bit in mgt_param.c. Instead of increasing
    things, we make sure the old default vsl_buffer of 4k is maintained.

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 24c5bfcda..08dc48233 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -672,6 +672,7 @@ MCF_InitParams(struct cli *cli)
 		MCF_ParamConf(MCF_DEFAULT, "http_resp_size", "8k");
 		MCF_ParamConf(MCF_DEFAULT, "http_req_size", "12k");
 		MCF_ParamConf(MCF_DEFAULT, "gzip_buffer", "4k");
+		MCF_ParamConf(MCF_DEFAULT, "vsl_buffer", "4k");
 		MCF_ParamConf(MCF_MAXIMUM, "vsl_space", "1G");
 		def = 52 * 1024;
 	}

From hermunn at varnish-software.com  Tue Aug 10 14:59:06 2021
From: hermunn at varnish-software.com (Pål Hermunn Johansen)
Date: Tue, 10 Aug 2021 14:59:06 +0000 (UTC)
Subject: [master] 8eb2e4e5a Document the fact that vsl_buffer now is special
 for 32-bit
Message-ID: <20210810145906.D4A3DC12A1@lists.varnish-cache.org>


commit 8eb2e4e5a96e51140a78bed9041565e99316acef
Author: P?l Hermunn Johansen <hermunn at varnish-software.com>
Date:   Tue Aug 10 16:57:24 2021 +0200

    Document the fact that vsl_buffer now is special for 32-bit

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 5fad3096f..e87f48a14 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -526,6 +526,7 @@ space:
 * http_resp_size: 8k
 * http_req_size: 12k
 * gzip_buffer: 4k
+* vsl_buffer: 4k
 * vsl_space: 1G (maximum)
 * thread_pool_stack: 52k
 

From dridi.boukelmoune at gmail.com  Wed Aug 11 15:33:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 11 Aug 2021 15:33:07 +0000 (UTC)
Subject: [6.0] 63d35cc93 Add MAIN.esi_req counter for ESI subrequests
Message-ID: <20210811153307.E25255DD0@lists.varnish-cache.org>


commit 63d35cc937bb5fd23988dc0d1d64bddebe7495f5
Author: Reza Naghibi <reza at naghibi.com>
Date:   Tue Nov 10 16:05:55 2020 -0500

    Add MAIN.esi_req counter for ESI subrequests
    
     Conflicts:
            bin/varnishd/cache/cache_esi_deliver.c

diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc
index f6925f30b..d03144367 100644
--- a/bin/varnishd/VSC_main.vsc
+++ b/bin/varnishd/VSC_main.vsc
@@ -103,6 +103,12 @@
 
 	The count of parseable client requests seen.
 
+.. varnish_vsc:: esi_req
+	:group: wrk
+	:oneliner:	ESI subrequests
+
+	Number of ESI subrequests made.
+
 .. varnish_vsc:: cache_hit
 	:group: wrk
 	:oneliner:	Cache hits
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 2f3f8e86f..405bcaaf5 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -122,6 +122,8 @@ ved_include(struct req *preq, const char *src, const char *host,
 	req->vsl->wid = VXID_Get(wrk, VSL_CLIENTMARKER);
 	VSLb(req->vsl, SLT_Begin, "req %u esi", VXID(preq->vsl->wid));
 	VSLb(preq->vsl, SLT_Link, "req %u esi", VXID(req->vsl->wid));
+
+	wrk->stats->esi_req++;
 	req->esi_level = preq->esi_level + 1;
 
 	if (preq->esi_level == 0)
diff --git a/bin/varnishtest/tests/e00003.vtc b/bin/varnishtest/tests/e00003.vtc
index a541596bf..f621b5b4d 100644
--- a/bin/varnishtest/tests/e00003.vtc
+++ b/bin/varnishtest/tests/e00003.vtc
@@ -76,6 +76,7 @@ client c1 {
 }
 
 client c1 -run
+varnish v1 -expect esi_req == 2
 varnish v1 -expect esi_errors == 0
 
 logexpect l1 -wait

From dridi.boukelmoune at gmail.com  Thu Aug 12 05:26:08 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 12 Aug 2021 05:26:08 +0000 (UTC)
Subject: [master] c85c4ecb4 vtc: Another recursive resolver breaking m11
Message-ID: <20210812052608.1F3F7112A6B@lists.varnish-cache.org>


commit c85c4ecb4502ebc2da0f0cb394ff13ed02868bbb
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Aug 12 07:17:21 2021 +0200

    vtc: Another recursive resolver breaking m11
    
    Something DNS always breaks when I join a new network.

diff --git a/bin/varnishtest/tests/m00011.vtc b/bin/varnishtest/tests/m00011.vtc
index 243e5daf6..11cb16aec 100644
--- a/bin/varnishtest/tests/m00011.vtc
+++ b/bin/varnishtest/tests/m00011.vtc
@@ -37,7 +37,7 @@ varnish v1 -vcl+backend {
 		set resp.http.port4 = std.port(debug.get_ip());
 		std.timestamp("2001:db8::");
 
-		debug.store_ip(std.ip("2001::db8::", "[::1]"));
+		debug.store_ip(std.ip("2001::db8::", "[::1]", resolve = false));
 		set resp.http.ip5 = debug.get_ip();
 		set resp.http.port5 = std.port(debug.get_ip());
 		std.timestamp("2001::db8::");

From dridi.boukelmoune at gmail.com  Thu Aug 12 07:41:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 12 Aug 2021 07:41:05 +0000 (UTC)
Subject: [master] 46d6928da build: Check vsl_glob_test_coverage in
Message-ID: <20210812074105.ABB091161A6@lists.varnish-cache.org>


commit 46d6928da799b210fd8b396ad53aad005a55bca8
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Aug 12 08:35:08 2021 +0200

    build: Check vsl_glob_test_coverage in
    
    It should be more friendly to maintain than a script incrementally
    generated by a makefile. Make it responsible for also running
    vsl_glob_test without arguments instead of the test driver, and
    restore its output. In case of failure, the -x mode should help
    figure which vsl_glob_test failed. While at it give the script a
    proper shebang.
    
    Refs #3667

diff --git a/.gitignore b/.gitignore
index 183ecfddd..4f6a67eaf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -123,7 +123,6 @@ cscope.*out
 /lib/libvarnish/vnum_c_test
 /lib/libvarnish/vsb_test
 /lib/libvarnishapi/vsl_glob_test
-/lib/libvarnishapi/vsl_glob_test_coverage
 /lib/libvarnishapi/vxp_test
 
 # vtc-bisect.sh default vtc
diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index 15f8a208c..8e26239e7 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -85,20 +85,12 @@ vxp_test_LDADD = \
 	$(top_builddir)/lib/libvarnish/libvarnish.la \
 	${RT_LIBS} ${LIBM} ${PTHREAD_LIBS} ${SAN_LDFLAGS}
 
-TESTS = vsl_glob_test
-
 noinst_PROGRAMS += vsl_glob_test
 
 vsl_glob_test_SOURCES = vsl_glob_test.c
 vsl_glob_test_CFLAGS = @SAN_CFLAGS@
 vsl_glob_test_LDADD = libvarnishapi.la @SAN_LDFLAGS@
 
-TESTS += vsl_glob_test_coverage
-
-vsl_glob_test_coverage:
-	echo './vsl_glob_test 1 2 3 2> /dev/null || true' > ${builddir}/_
-	echo './vsl_glob_test "Req*" > /dev/null' >> ${builddir}/_
-	mv ${builddir}/_ ${builddir}/vsl_glob_test_coverage
-	chmod +x ${builddir}/vsl_glob_test_coverage
+dist_noinst_SCRIPTS = vsl_glob_test_coverage
 
-CLEANFILES += ${builddir}/vsl_glob_test_coverage
+TESTS = vsl_glob_test_coverage
diff --git a/lib/libvarnishapi/vsl_glob_test_coverage b/lib/libvarnishapi/vsl_glob_test_coverage
new file mode 100755
index 000000000..b93fb382d
--- /dev/null
+++ b/lib/libvarnishapi/vsl_glob_test_coverage
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -e
+set -x
+
+# wrong usage
+! ./vsl_glob_test too many arguments
+
+# built-in sanity checks
+./vsl_glob_test
+
+# coverage
+./vsl_glob_test 'Req*'
+./vsl_glob_test 'beresp*'

From dridi.boukelmoune at gmail.com  Thu Aug 12 07:41:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 12 Aug 2021 07:41:05 +0000 (UTC)
Subject: [master] 5343d84de build: Remove ad-hoc test target
Message-ID: <20210812074105.C05371161A9@lists.varnish-cache.org>


commit 5343d84de42695972d6d28b89a664b445d0d89a7
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Aug 12 08:51:08 2021 +0200

    build: Remove ad-hoc test target
    
    One can always run `make -C include/ check VERBOSE=1` for feature
    parity, and even get the other goodies from the regular test driver.

diff --git a/include/Makefile.am b/include/Makefile.am
index 1eeaa3537..dffea027d 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -169,6 +169,3 @@ _vrt.c: Makefile.am vdef.h vrt.h
 vrt_test: _vrt.c
 	echo "exec ${CC} -c -o _vrt_test _vrt.c" > $@
 	chmod +x $@
-
-test: ${TESTS}
-	@for test in ${TESTS} ; do ./$${test} ; done

From dridi.boukelmoune at gmail.com  Thu Aug 12 07:41:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 12 Aug 2021 07:41:05 +0000 (UTC)
Subject: [master] 5884e2f2f build: Turn vrt_test into a compile-time check
Message-ID: <20210812074105.DBE621161AD@lists.varnish-cache.org>


commit 5884e2f2fd55e0e7d2cfb9722076c1f3b6c8cac9
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Aug 12 09:21:43 2021 +0200

    build: Turn vrt_test into a compile-time check
    
    Instead of generating a test script that performs the compilation, we
    can give automake a no-op linker command. With that vrt.h is verified
    with a simple `make` or `make all` invocation, before `make check` even
    kicks in.
    
    Doing it this way also gives applicable warnings like strict-prototypes
    a chance to fail earlier.
    
    Refs #3667

diff --git a/include/Makefile.am b/include/Makefile.am
index dffea027d..16c63062e 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -157,15 +157,15 @@ CLEANFILES = \
 	_vrt_test \
 	_vrt.c
 
-TESTS = vbm_test vrt_test
-
-noinst_PROGRAMS = vbm_test
+noinst_PROGRAMS = vbm_test vrt_test
 
 vbm_test_SOURCES = vbm_test.c vbm.h
 
-_vrt.c: Makefile.am vdef.h vrt.h
+vrt_test.c: vdef.h vrt.h
 	cat $(srcdir)/vdef.h $(srcdir)/vrt.h > $@
 
-vrt_test: _vrt.c
-	echo "exec ${CC} -c -o _vrt_test _vrt.c" > $@
-	chmod +x $@
+vrt_test_BUILT_SOURCES = vrt_test.c
+vrt_test_CFLAGS = -c
+vrt_test_LINK = :
+
+TESTS = vbm_test

From dridi.boukelmoune at gmail.com  Thu Aug 12 08:34:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 12 Aug 2021 08:34:05 +0000 (UTC)
Subject: [master] 7f8ac60a5 build: Polish output for the vrt_test.c target
Message-ID: <20210812083405.AC7D0117B22@lists.varnish-cache.org>


commit 7f8ac60a5865cf9171d83e451e3f4de0b4ce9f1f
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Aug 12 10:32:46 2021 +0200

    build: Polish output for the vrt_test.c target

diff --git a/include/Makefile.am b/include/Makefile.am
index 16c63062e..d6afb0438 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -162,7 +162,7 @@ noinst_PROGRAMS = vbm_test vrt_test
 vbm_test_SOURCES = vbm_test.c vbm.h
 
 vrt_test.c: vdef.h vrt.h
-	cat $(srcdir)/vdef.h $(srcdir)/vrt.h > $@
+	$(AM_V_GEN) cat $(srcdir)/vdef.h $(srcdir)/vrt.h > $@
 
 vrt_test_BUILT_SOURCES = vrt_test.c
 vrt_test_CFLAGS = -c

From dridi.boukelmoune at gmail.com  Thu Aug 12 08:51:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 12 Aug 2021 08:51:06 +0000 (UTC)
Subject: [master] f3f0116c8 build: mv
 lib/libvarnishapi/vsl_glob_test_coverage{, .sh}
Message-ID: <20210812085106.2CE3311836A@lists.varnish-cache.org>


commit f3f0116c895cd37a69cd58ef21d4e0e670adf92f
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Aug 12 10:48:46 2021 +0200

    build: mv lib/libvarnishapi/vsl_glob_test_coverage{,.sh}
    
    Trying to unconfuse bmake on FreeBSD:
    
        make[6]: don't know how to make vsl_glob_test_coverage. Stop
    
    But I can't reproduce it locally with bmake on Linux.

diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index 8e26239e7..17adc831a 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -91,6 +91,7 @@ vsl_glob_test_SOURCES = vsl_glob_test.c
 vsl_glob_test_CFLAGS = @SAN_CFLAGS@
 vsl_glob_test_LDADD = libvarnishapi.la @SAN_LDFLAGS@
 
-dist_noinst_SCRIPTS = vsl_glob_test_coverage
+dist_noinst_SCRIPTS = vsl_glob_test_coverage.sh
 
-TESTS = vsl_glob_test_coverage
+TESTS = vsl_glob_test_coverage.sh
+TEST_EXTENSIONS = .sh
diff --git a/lib/libvarnishapi/vsl_glob_test_coverage b/lib/libvarnishapi/vsl_glob_test_coverage.sh
similarity index 100%
rename from lib/libvarnishapi/vsl_glob_test_coverage
rename to lib/libvarnishapi/vsl_glob_test_coverage.sh

From phk at FreeBSD.org  Mon Aug 16 09:40:06 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 16 Aug 2021 09:40:06 +0000 (UTC)
Subject: [master] 1e546e958 Add coverage of the vxp_test program
Message-ID: <20210816094006.5AD8460858@lists.varnish-cache.org>


commit 1e546e9587bf73c60b9509b3f5c8577d276cb5cb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 16 09:39:18 2021 +0000

    Add coverage of the vxp_test program

diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index 17adc831a..ed08064a0 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -93,5 +93,5 @@ vsl_glob_test_LDADD = libvarnishapi.la @SAN_LDFLAGS@
 
 dist_noinst_SCRIPTS = vsl_glob_test_coverage.sh
 
-TESTS = vsl_glob_test_coverage.sh
+TESTS = vsl_glob_test_coverage.sh vxp_test_coverage.sh
 TEST_EXTENSIONS = .sh
diff --git a/lib/libvarnishapi/vxp_test_coverage.sh b/lib/libvarnishapi/vxp_test_coverage.sh
new file mode 100755
index 000000000..316a7e9c1
--- /dev/null
+++ b/lib/libvarnishapi/vxp_test_coverage.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+./vxp_test -q '(*Error) or (BerespStatus >= 500)'
+./vxp_test -q 'ReqHeader:user-agent ~ "iPod" and Timestamp:Resp[2] > 1'

From phk at FreeBSD.org  Mon Aug 16 10:38:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 16 Aug 2021 10:38:05 +0000 (UTC)
Subject: [master] 8349386cc No,
 I clearly dont know how autocrap's `make distcheck` works
Message-ID: <20210816103805.A3B90624FF@lists.varnish-cache.org>


commit 8349386cc4117bd9ef533642a6b1a75fe440c1c4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 16 10:37:15 2021 +0000

    No, I clearly dont know how autocrap's `make distcheck` works

diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index ed08064a0..dcd34239d 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -91,7 +91,7 @@ vsl_glob_test_SOURCES = vsl_glob_test.c
 vsl_glob_test_CFLAGS = @SAN_CFLAGS@
 vsl_glob_test_LDADD = libvarnishapi.la @SAN_LDFLAGS@
 
-dist_noinst_SCRIPTS = vsl_glob_test_coverage.sh
+dist_noinst_SCRIPTS = vsl_glob_test_coverage.sh vxp_test_coverage.sh
 
 TESTS = vsl_glob_test_coverage.sh vxp_test_coverage.sh
 TEST_EXTENSIONS = .sh

From phk at FreeBSD.org  Mon Aug 16 11:12:06 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 16 Aug 2021 11:12:06 +0000 (UTC)
Subject: [master] eb24a4a6e Cover a more lines to a void a pointless red gcov
 entry
Message-ID: <20210816111206.28125637EE@lists.varnish-cache.org>


commit eb24a4a6e1992924dbfbd2756521b5c4044825f3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 16 11:11:07 2021 +0000

    Cover a more lines to a void a pointless red gcov entry

diff --git a/lib/libvarnishapi/vxp_test_coverage.sh b/lib/libvarnishapi/vxp_test_coverage.sh
index 316a7e9c1..65fee3f38 100755
--- a/lib/libvarnishapi/vxp_test_coverage.sh
+++ b/lib/libvarnishapi/vxp_test_coverage.sh
@@ -1,4 +1,10 @@
 #!/bin/sh
 
+set -ex
+
 ./vxp_test -q '(*Error) or (BerespStatus >= 500)'
 ./vxp_test -q 'ReqHeader:user-agent ~ "iPod" and Timestamp:Resp[2] > 1'
+
+! ./vxp_test -h 
+
+! ./vxp_test -q 'The head and in frontal attack on an english writer' 

From phk at FreeBSD.org  Tue Aug 17 06:16:08 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 17 Aug 2021 06:16:08 +0000 (UTC)
Subject: [master] 328540c6c tree-wide: add missing magic checks
Message-ID: <20210817061608.594C193C1@lists.varnish-cache.org>


commit 328540c6ca58b8009d739ea542442a9d24372ce2
Author: Asad Sajjad Ahmed <asadsa at varnish-software.com>
Date:   Tue Aug 10 20:35:58 2021 +0200

    tree-wide: add missing magic checks
    
    Signed-off-by: Asad Sajjad Ahmed <asadsa at varnish-software.com>

diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c
index f56a9fac6..5fa8b1b7e 100644
--- a/bin/varnishd/cache/cache_backend_probe.c
+++ b/bin/varnishd/cache/cache_backend_probe.c
@@ -102,6 +102,8 @@ static const unsigned char vbp_proxy_local[] = {
 static void
 vbp_delete(struct vbp_target *vt)
 {
+	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
+
 #define DN(x)	/**/
 	VRT_BACKEND_PROBE_HANDLE();
 #undef DN
diff --git a/bin/varnishd/cache/cache_conn_pool.c b/bin/varnishd/cache/cache_conn_pool.c
index c8d39e21d..4f4b1b210 100644
--- a/bin/varnishd/cache/cache_conn_pool.c
+++ b/bin/varnishd/cache/cache_conn_pool.c
@@ -156,6 +156,7 @@ vcp_handle(struct waited *w, enum wait_event ev, vtim_real now)
 	struct pfd *pfd;
 	struct conn_pool *cp;
 
+	CHECK_OBJ_NOTNULL(w, WAITED_MAGIC);
 	CAST_OBJ_NOTNULL(pfd, w->priv1, PFD_MAGIC);
 	(void)ev;
 	(void)now;
@@ -730,6 +731,7 @@ VCP_Ref(const struct vrt_endpoint *vep, const char *ident)
 	cp->refcnt = 1;
 	cp->holddown = 0;
 	cp->endpoint = VRT_Endpoint_Clone(vep);
+	CHECK_OBJ_NOTNULL(cp->endpoint, VRT_ENDPOINT_MAGIC);
 	memcpy(cp->ident, digest, sizeof cp->ident);
 	if (vep->uds_path != NULL)
 		cp->methods = &vus_methods;
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 082e32631..466e096a4 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -296,10 +296,8 @@ ved_vdp_esi_fini(struct vdp_ctx *vdc, void **priv)
 	struct ecx *ecx;
 
 	(void)vdc;
-	AN(priv);
-	CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC);
+	TAKE_OBJ_NOTNULL(ecx, priv, ECX_MAGIC);
 	FREE_OBJ(ecx);
-	*priv = NULL;
 	return (0);
 }
 
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 90c7a7ca8..1af3a3793 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -162,9 +162,9 @@ HSH_Cleanup(const struct worker *wrk)
 		ObjDestroy(wrk, &wrk->wpriv->nobjcore);
 
 	if (wrk->wpriv->nobjhead != NULL) {
+		CHECK_OBJ(wrk->wpriv->nobjhead, OBJHEAD_MAGIC);
 		Lck_Delete(&wrk->wpriv->nobjhead->mtx);
 		FREE_OBJ(wrk->wpriv->nobjhead);
-		wrk->wpriv->nobjhead = NULL;
 		wrk->stats->n_objecthead--;
 	}
 	if (wrk->wpriv->nhashpriv != NULL) {
@@ -177,6 +177,8 @@ HSH_Cleanup(const struct worker *wrk)
 void
 HSH_DeleteObjHead(const struct worker *wrk, struct objhead *oh)
 {
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 
 	AZ(oh->refcnt);
 	assert(VTAILQ_EMPTY(&oh->objcs));
diff --git a/bin/varnishd/cache/cache_mempool.c b/bin/varnishd/cache/cache_mempool.c
index b6d69bd61..eec6cfa65 100644
--- a/bin/varnishd/cache/cache_mempool.c
+++ b/bin/varnishd/cache/cache_mempool.c
@@ -115,8 +115,8 @@ mpl_guard(void *priv)
 
 		if (mi != NULL && (mpl->n_pool > mpl->param->max_pool ||
 		    mi->size < *mpl->cur_size)) {
+			CHECK_OBJ(mi, MEMITEM_MAGIC);
 			FREE_OBJ(mi);
-			mi = NULL;
 		}
 
 		if (mi == NULL && mpl->n_pool < mpl->param->min_pool)
@@ -160,8 +160,8 @@ mpl_guard(void *priv)
 				}
 				if (mi == NULL)
 					break;
+				CHECK_OBJ(mi, MEMITEM_MAGIC);
 				FREE_OBJ(mi);
-				mi = NULL;
 			}
 			VSC_mempool_Destroy(&mpl->vsc_seg);
 			Lck_Unlock(&mpl->mtx);
@@ -172,7 +172,7 @@ mpl_guard(void *priv)
 
 		if (mpl->n_pool < mpl->param->min_pool &&
 		    mi != NULL && mi->size >= *mpl->cur_size) {
-			CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
+			CHECK_OBJ(mi, MEMITEM_MAGIC);
 			mpl->vsc->pool = ++mpl->n_pool;
 			mi->touched = mpl->t_now;
 			VTAILQ_INSERT_HEAD(&mpl->list, mi, list);
@@ -215,8 +215,8 @@ mpl_guard(void *priv)
 		Lck_Unlock(&mpl->mtx);
 
 		if (mi != NULL) {
+			CHECK_OBJ(mi, MEMITEM_MAGIC);
 			FREE_OBJ(mi);
-			mi = NULL;
 		}
 	}
 	return (NULL);
@@ -288,7 +288,7 @@ MPL_Get(struct mempool *mpl, unsigned *size)
 			break;
 		}
 		mpl->vsc->pool = --mpl->n_pool;
-		CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
+		CHECK_OBJ(mi, MEMITEM_MAGIC);
 		VTAILQ_REMOVE(&mpl->list, mi, list);
 		if (mi->size < *mpl->cur_size) {
 			mpl->vsc->toosmall++;
@@ -305,7 +305,7 @@ MPL_Get(struct mempool *mpl, unsigned *size)
 		mi = mpl_alloc(mpl);
 	*size = mi->size - sizeof *mi;
 
-	CHECK_OBJ_NOTNULL(mi, MEMITEM_MAGIC);
+	CHECK_OBJ(mi, MEMITEM_MAGIC);
 	/* Throw away sizeof info for FlexeLint: */
 	return ((void *)(uintptr_t)(mi + 1));
 }
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 1bab08f36..6c77d5840 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -200,7 +200,7 @@ pool_poolherder(void *priv)
 				DO_DEBUG(DBG_DROP_POOLS)) {
 			Lck_Lock(&pool_mtx);
 			pp = VTAILQ_FIRST(&pools);
-			AN(pp);
+			CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 			VTAILQ_REMOVE(&pools, pp, list);
 			VTAILQ_INSERT_TAIL(&pools, pp, list);
 			if (!pp->die)
@@ -218,6 +218,8 @@ pool_poolherder(void *priv)
 		ppx = NULL;
 		Lck_Lock(&pool_mtx);
 		VTAILQ_FOREACH(pp, &pools, list) {
+			CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
+
 			if (pp->die && pp->nthr == 0)
 				ppx = pp;
 			u += pp->lqueue;
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index d70e9543f..0efded131 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -517,9 +517,7 @@ VCL_Close(struct vcl **vclp)
 {
 	struct vcl *vcl;
 
-	CHECK_OBJ_NOTNULL(*vclp, VCL_MAGIC);
-	vcl = *vclp;
-	*vclp = NULL;
+	TAKE_OBJ_NOTNULL(vcl, vclp, VCL_MAGIC);
 	assert(VTAILQ_EMPTY(&vcl->vfps));
 	assert(VTAILQ_EMPTY(&vcl->vdps));
 	AZ(dlclose(vcl->dlh));
@@ -878,7 +876,7 @@ vcl_cli_discard(struct cli *cli, const char * const *av, void *priv)
 	(void)cli;
 	AZ(priv);
 	vcl = vcl_find(av[2]);
-	AN(vcl);			// MGT ensures this
+	CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC);		// MGT ensures this
 	Lck_Lock(&vcl_mtx);
 	assert (vcl != vcl_active);	// MGT ensures this
 	AZ(vcl->nlabels);		// MGT ensures this
diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c
index 485915c11..1baf344f5 100644
--- a/bin/varnishd/cache/cache_vrt_filter.c
+++ b/bin/varnishd/cache/cache_vrt_filter.c
@@ -80,6 +80,7 @@ VRT_AddVFP(VRT_CTX, const struct vfp *filter)
 	}
 	if (ctx != NULL) {
 		ASSERT_CLI();
+		CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC);
 		hd = &ctx->vcl->vfps;
 		VTAILQ_FOREACH(vp, hd, list) {
 			xxxassert(vp->vfp != filter);
@@ -111,6 +112,7 @@ VRT_AddVDP(VRT_CTX, const struct vdp *filter)
 	}
 	if (ctx != NULL) {
 		ASSERT_CLI();
+		CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC);
 		hd = &ctx->vcl->vdps;
 		VTAILQ_FOREACH(vp, hd, list) {
 			xxxassert(vp->vdp != filter);
@@ -129,15 +131,18 @@ void
 VRT_RemoveVFP(VRT_CTX, const struct vfp *filter)
 {
 	struct vfilter *vp;
-	struct vfilter_head *hd = &ctx->vcl->vfps;
+	struct vfilter_head *hd;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC);
+	hd = &ctx->vcl->vfps;
 	AN(filter);
 	AN(filter->name);
 	AN(*filter->name);
 
 	ASSERT_CLI();
 	VTAILQ_FOREACH(vp, hd, list) {
+		CHECK_OBJ_NOTNULL(vp, VFILTER_MAGIC);
 		if (vp->vfp == filter)
 			break;
 	}
@@ -150,15 +155,18 @@ void
 VRT_RemoveVDP(VRT_CTX, const struct vdp *filter)
 {
 	struct vfilter *vp;
-	struct vfilter_head *hd = &ctx->vcl->vdps;
+	struct vfilter_head *hd;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC);
+	hd = &ctx->vcl->vdps;
 	AN(filter);
 	AN(filter->name);
 	AN(*filter->name);
 
 	ASSERT_CLI();
 	VTAILQ_FOREACH(vp, hd, list) {
+		CHECK_OBJ_NOTNULL(vp, VFILTER_MAGIC);
 		if (vp->vdp == filter)
 			break;
 	}
diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c
index 9b3c25a0f..f7bcf2017 100644
--- a/bin/varnishd/cache/cache_vrt_vcl.c
+++ b/bin/varnishd/cache/cache_vrt_vcl.c
@@ -143,6 +143,7 @@ vcldir_free(struct vcldir *vdir)
 {
 
 	CHECK_OBJ_NOTNULL(vdir, VCLDIR_MAGIC);
+	CHECK_OBJ_NOTNULL(vdir->dir, DIRECTOR_MAGIC);
 	Lck_Delete(&vdir->dlck);
 	free(vdir->cli_name);
 	FREE_OBJ(vdir->dir);
diff --git a/bin/varnishd/common/common_vsc.c b/bin/varnishd/common/common_vsc.c
index 94cb465cd..0e9128eaa 100644
--- a/bin/varnishd/common/common_vsc.c
+++ b/bin/varnishd/common/common_vsc.c
@@ -198,12 +198,11 @@ VRT_VSC_Destroy(const char *nm, struct vsc_seg *vsg)
 
 	AN(heritage.proc_vsmw);
 	CHECK_OBJ_NOTNULL(vsg, VSC_SEG_MAGIC);
+	CAST_OBJ_NOTNULL(dvsg, vsg->doc, VSC_SEG_MAGIC);
 	AZ(vsg->jp);
-	CHECK_OBJ_NOTNULL(vsg->doc, VSC_SEG_MAGIC);
 	assert(vsg->vsm == heritage.proc_vsmw);
 	assert(vsg->nm == nm);
 
-	dvsg = vsg->doc;
 	VSMW_Free(heritage.proc_vsmw, &vsg->seg);
 	VTAILQ_REMOVE(&vsc_seglist, vsg, list);
 	FREE_OBJ(vsg);
diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c
index c516a530c..2424947e3 100644
--- a/bin/varnishd/hash/hash_critbit.c
+++ b/bin/varnishd/hash/hash_critbit.c
@@ -316,10 +316,12 @@ hcb_cleaner(struct worker *wrk, void *priv)
 	(void)priv;
 	while (1) {
 		VSTAILQ_FOREACH_SAFE(y, &dead_y, list, y2) {
+			CHECK_OBJ_NOTNULL(y, HCB_Y_MAGIC);
 			VSTAILQ_REMOVE_HEAD(&dead_y, list);
 			FREE_OBJ(y);
 		}
 		VTAILQ_FOREACH_SAFE(oh, &dead_h, hoh_list, oh2) {
+			CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
 			VTAILQ_REMOVE(&dead_h, oh, hoh_list);
 			HSH_DeleteObjHead(wrk, oh);
 		}
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index dca5c4b52..9c2a8f0ef 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -827,6 +827,7 @@ main(int argc, char * const *argv)
 	u = 0;
 	while (!VTAILQ_EMPTY(&f_args)) {
 		fa = VTAILQ_FIRST(&f_args);
+		CHECK_OBJ_NOTNULL(fa, F_ARG_MAGIC);
 		VTAILQ_REMOVE(&f_args, fa, list);
 		mgt_vcl_startup(cli, fa->src,
 		    VTAILQ_EMPTY(&f_args) ? "boot" : NULL,
diff --git a/bin/varnishd/storage/storage_persistent_silo.c b/bin/varnishd/storage/storage_persistent_silo.c
index 95d3864d0..942103d33 100644
--- a/bin/varnishd/storage/storage_persistent_silo.c
+++ b/bin/varnishd/storage/storage_persistent_silo.c
@@ -96,6 +96,7 @@ smp_save_segs(struct smp_sc *sc)
 {
 	struct smp_seg *sg, *sg2;
 
+	CHECK_OBJ_NOTNULL(sc, SMP_SC_MAGIC);
 	Lck_AssertHeld(&sc->mtx);
 
 	/*
@@ -103,6 +104,8 @@ smp_save_segs(struct smp_sc *sc)
 	 * before we write the segments to disk.
 	 */
 	VTAILQ_FOREACH_SAFE(sg, &sc->segments, list, sg2) {
+		CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC);
+
 		if (sg->nobj > 0)
 			break;
 		if (sg == sc->cur_seg)
@@ -265,6 +268,7 @@ smp_close_seg(struct smp_sc *sc, struct smp_seg *sg)
 	uint64_t left, dst, len;
 	void *dp;
 
+	CHECK_OBJ_NOTNULL(sc, SMP_SC_MAGIC);
 	Lck_AssertHeld(&sc->mtx);
 
 	CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC);
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index a73cc0b87..06982c5bf 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -207,6 +207,7 @@ macro_undef(struct vtclog *vl, const char *instance, const char *name)
 	if (m != NULL) {
 		if (!vtc_stop)
 			vtc_log(vl, 4, "macro undef %s", name);
+		CHECK_OBJ(m, MACRO_MAGIC);
 		VTAILQ_REMOVE(&macro_list, m, list);
 		free(m->name);
 		free(m->val);
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 3e310d791..6e0c2846b 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -1777,7 +1777,9 @@ const struct cmds http_cmds[] = {
 static void
 http_process_cleanup(void *arg)
 {
-	struct http *hp = arg;
+	struct http *hp;
+
+	CAST_OBJ_NOTNULL(hp, arg, HTTP_MAGIC);
 
 	if (hp->h2)
 		stop_h2(hp);
diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c
index 15c1a7371..dae3ff0ec 100644
--- a/bin/varnishtest/vtc_log.c
+++ b/bin/varnishtest/vtc_log.c
@@ -98,9 +98,9 @@ vtc_logopen(const char *fmt, ...)
 void
 vtc_logclose(void *arg)
 {
-	struct vtclog *vl = arg;
+	struct vtclog *vl;
 
-	CHECK_OBJ_NOTNULL(vl, VTCLOG_MAGIC);
+	CAST_OBJ_NOTNULL(vl, arg, VTCLOG_MAGIC);
 	if (pthread_getspecific(log_key) == vl)
 		AZ(pthread_setspecific(log_key, NULL));
 	VSB_destroy(&vl->vsb);
diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index bd4d243eb..2194d6621 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -296,6 +296,7 @@ tst_cb(const struct vev *ve, int what)
 	char *p;
 
 	CAST_OBJ_NOTNULL(jp, ve->priv, JOB_MAGIC);
+	CHECK_OBJ_NOTNULL(jp->tst, TST_MAGIC);
 
 	// printf("CB %p %s %d\n", ve, jp->tst->filename, what);
 	if (what == 0) {
diff --git a/lib/libvarnish/vbh.c b/lib/libvarnish/vbh.c
index ce7bd5fd9..02afefa44 100644
--- a/lib/libvarnish/vbh.c
+++ b/lib/libvarnish/vbh.c
@@ -613,7 +613,7 @@ main(void)
 		for (u = 0; u < M; u++) {
 			v = VRND_RandomTestable() % N;
 			if (ff[v] != NULL) {
-				CHECK_OBJ_NOTNULL(ff[v], FOO_MAGIC);
+				CHECK_OBJ(ff[v], FOO_MAGIC);
 				AN(ff[v]->idx);
 				if (ff[v]->key & 1) {
 					VBH_delete(bh, ff[v]->idx);
@@ -639,6 +639,7 @@ main(void)
 		fprintf(stderr, "%d updates OK\n", M);
 	}
 	while ((fp = VBH_root(bh)) != NULL) {
+		CHECK_OBJ(fp, FOO_MAGIC);
 		VBH_delete(bh, fp->idx);
 		FREE_OBJ(fp);
 	}
diff --git a/lib/libvarnish/vfil.c b/lib/libvarnish/vfil.c
index 0720e7f74..8d1f3ab60 100644
--- a/lib/libvarnish/vfil.c
+++ b/lib/libvarnish/vfil.c
@@ -332,6 +332,7 @@ VFIL_setpath(struct vfil_path **pp, const char *path)
 	REPLACE(vp->str, path);
 	while (!VTAILQ_EMPTY(&vp->paths)) {
 		vd = VTAILQ_FIRST(&vp->paths);
+		CHECK_OBJ_NOTNULL(vd, VFIL_DIR_MAGIC);
 		VTAILQ_REMOVE(&vp->paths, vd, list);
 		FREE_OBJ(vd);
 	}
diff --git a/lib/libvarnish/vlu.c b/lib/libvarnish/vlu.c
index d8cb39c5c..217c50b12 100644
--- a/lib/libvarnish/vlu.c
+++ b/lib/libvarnish/vlu.c
@@ -66,10 +66,8 @@ VLU_New(vlu_f *func, void *priv, unsigned bufsize)
 		l->priv = priv;
 		l->bufl = bufsize - 1;
 		l->buf = malloc(l->bufl + 1L);
-		if (l->buf == NULL) {
+		if (l->buf == NULL)
 			FREE_OBJ(l);
-			l = NULL;
-		}
 	}
 	return (l);
 }
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index 6a2040823..1e62184f9 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -294,10 +294,9 @@ VRE_sub(const vre_t *code, const char *subject, const char *replacement,
 void
 VRE_free(vre_t **vv)
 {
-	vre_t *v = *vv;
+	vre_t *v;
 
-	*vv = NULL;
-	CHECK_OBJ(v, VRE_MAGIC);
+	TAKE_OBJ_NOTNULL(v, vv, VRE_MAGIC);
 
 	if (v->re == VRE_PACKED_RE) {
 		v->re = NULL;
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index 1a84d6aeb..2fdcfe740 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -319,7 +319,6 @@ vsc_del_seg(const struct vsc *vsc, struct vsm *vsm, struct vsc_seg **spp)
 	struct vsc_seg *sp;
 
 	CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);
-	AN(vsm);
 	TAKE_OBJ_NOTNULL(sp, spp, VSC_SEG_MAGIC);
 	AZ(VSM_Unmap(vsm, sp->fantom));
 	if (sp->vj != NULL) {
@@ -343,7 +342,6 @@ vsc_add_seg(const struct vsc *vsc, struct vsm *vsm, const struct vsm_fantom *fp)
 	struct vsc_pt *pp;
 
 	CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);
-	AN(vsm);
 
 	ALLOC_OBJ(sp, VSC_SEG_MAGIC);
 	AN(sp);
diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index 6b1284667..b06f7deb5 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -79,6 +79,7 @@ vslc_vsm_delete(const struct VSL_cursor *cursor)
 {
 	struct vslc_vsm *c;
 
+	AN(cursor);
 	CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_VSM_MAGIC);
 	AZ(VSM_Unmap(c->vsm, &c->vf));
 	assert(&c->cursor == cursor);
@@ -322,6 +323,7 @@ vslc_file_delete(const struct VSL_cursor *cursor)
 {
 	struct vslc_file *c;
 
+	AN(cursor);
 	CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_FILE_MAGIC);
 	assert(&c->cursor == cursor);
 	if (c->close_fd)
@@ -418,6 +420,7 @@ vslc_mmap_delete(const struct VSL_cursor *cursor)
 {
 	struct vslc_mmap *c;
 
+	AN(cursor);
 	CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_MMAP_MAGIC);
 	assert(&c->cursor == cursor);
 	AZ(munmap(c->b, c->e - c->b));
diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index e68fa4e29..4038273d0 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -375,12 +375,12 @@ chunk_newbuf(struct vtx *vtx, const uint32_t *ptr, size_t len)
 static void
 chunk_freebuf(struct chunk **pchunk)
 {
+	struct chunk *chunk;
 
-	CHECK_OBJ_NOTNULL(*pchunk, CHUNK_MAGIC);
-	assert((*pchunk)->type == chunk_t_buf);
-	free((*pchunk)->buf.data);
-	FREE_OBJ(*pchunk);
-	*pchunk = NULL;
+	TAKE_OBJ_NOTNULL(chunk, pchunk, CHUNK_MAGIC);
+	assert(chunk->type == chunk_t_buf);
+	free(chunk->buf.data);
+	FREE_OBJ(chunk);
 }
 
 /* Append a set of records to a chunk */
@@ -593,9 +593,9 @@ vtx_retire(struct VSLQ *vslq, struct vtx **pvtx)
 	if (vslq->n_cache < VTX_CACHE) {
 		VTAILQ_INSERT_HEAD(&vslq->cache, vtx, list_child);
 		vslq->n_cache++;
-	} else {
+	} else
 		FREE_OBJ(vtx);
-	}
+
 }
 
 /* Lookup a vtx by vxid from the managed list */
@@ -1166,6 +1166,7 @@ VSLQ_Delete(struct VSLQ **pvslq)
 	while (!VTAILQ_EMPTY(&vslq->cache)) {
 		AN(vslq->n_cache);
 		vtx = VTAILQ_FIRST(&vslq->cache);
+		CHECK_OBJ_NOTNULL(vtx, VTX_MAGIC);
 		VTAILQ_REMOVE(&vslq->cache, vtx, list_child);
 		vslq->n_cache--;
 		FREE_OBJ(vtx);
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 401f278dd..0040e85c8 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -306,9 +306,8 @@ vsm_delset(struct vsm_set **p)
 	struct vsm_set *vs;
 	struct vsm_seg *vg;
 
-	AN(p);
-	vs = *p;
-	*p = NULL;
+	TAKE_OBJ_NOTNULL(vs, p, VSM_SET_MAGIC);
+
 	if (vs->fd >= 0)
 		closefd(&vs->fd);
 	if (vs->dfd >= 0)
diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c
index 82077270a..7c275102e 100644
--- a/lib/libvarnishapi/vut.c
+++ b/lib/libvarnishapi/vut.c
@@ -378,7 +378,6 @@ VUT_Fini(struct VUT **vutp)
 	if (vut->vsm)
 		VSM_Destroy(&vut->vsm);
 
-	memset(vut, 0, sizeof *vut);
 	FREE_OBJ(vut);
 }
 
diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c
index 6d346d55d..01cbff89f 100644
--- a/lib/libvarnishapi/vxp_parse.c
+++ b/lib/libvarnishapi/vxp_parse.c
@@ -573,31 +573,37 @@ vxp_Parse(struct vxp *vxp)
 void
 vex_Free(struct vex **pvex)
 {
-
-	if ((*pvex)->lhs != NULL) {
-		if ((*pvex)->lhs->tags != NULL)
-			vbit_destroy((*pvex)->lhs->tags);
-		if ((*pvex)->lhs->prefix != NULL)
-			free((*pvex)->lhs->prefix);
-		FREE_OBJ((*pvex)->lhs);
+	struct vex *vex;
+	struct vex_lhs *lhs;
+	struct vex_rhs *rhs;
+
+	TAKE_OBJ_NOTNULL(vex, pvex, VEX_MAGIC);
+
+	if (vex->lhs) {
+		CAST_OBJ(lhs, vex->lhs, VEX_LHS_MAGIC);
+		if (lhs->tags)
+			vbit_destroy(lhs->tags);
+		if (lhs->prefix)
+			free(lhs->prefix);
+		FREE_OBJ(lhs);
 	}
-	if ((*pvex)->rhs != NULL) {
-		if ((*pvex)->rhs->val_string)
-			free((*pvex)->rhs->val_string);
-		if ((*pvex)->rhs->val_regex)
-			VRE_free(&(*pvex)->rhs->val_regex);
-		FREE_OBJ((*pvex)->rhs);
+	if (vex->rhs) {
+		CAST_OBJ(rhs, vex->rhs, VEX_RHS_MAGIC);
+		if (rhs->val_string)
+			free(rhs->val_string);
+		if (rhs->val_regex)
+			VRE_free(&rhs->val_regex);
+		FREE_OBJ(rhs);
 	}
-	if ((*pvex)->a != NULL) {
-		vex_Free(&(*pvex)->a);
-		AZ((*pvex)->a);
+	if (vex->a) {
+		vex_Free(&vex->a);
+		AZ(vex->a);
 	}
-	if ((*pvex)->b != NULL) {
-		vex_Free(&(*pvex)->b);
-		AZ((*pvex)->b);
+	if (vex->b) {
+		vex_Free(&vex->b);
+		AZ(vex->b);
 	}
-	FREE_OBJ(*pvex);
-	*pvex = NULL;
+	FREE_OBJ(vex);
 }
 
 #ifdef VXP_DEBUG
diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c
index 557b3a21f..96360b8d3 100644
--- a/lib/libvcc/vcc_acl.c
+++ b/lib/libvcc/vcc_acl.c
@@ -87,11 +87,11 @@ struct acl_e {
 static void
 vcl_acl_free(struct acl_e **aep)
 {
-	AN(aep);
-	CHECK_OBJ_NOTNULL(*aep, VCC_ACL_E_MAGIC);
-	if ((*aep)->addr != NULL)
-		free((*aep)->addr);
-	FREE_OBJ(*aep);
+	struct acl_e *a;
+
+	TAKE_OBJ_NOTNULL(a, aep, VCC_ACL_E_MAGIC);
+	free(a->addr);
+	FREE_OBJ(a);
 }
 
 static int
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index d40417baa..28a3f4402 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -118,7 +118,7 @@ vcc_delete_expr(struct expr *e)
 {
 	if (e == NULL)
 		return;
-	CHECK_OBJ_NOTNULL(e, EXPR_MAGIC);
+	CHECK_OBJ(e, EXPR_MAGIC);
 	VSB_destroy(&e->vsb);
 	FREE_OBJ(e);
 }
diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c
index f75016da0..240f5d281 100644
--- a/vmod/vmod_debug.c
+++ b/vmod/vmod_debug.c
@@ -422,7 +422,6 @@ priv_vcl_fini(VRT_CTX, void *priv)
 	AZ(priv_vcl->vclref_discard);
 	AZ(priv_vcl->vclref_cold);
 	FREE_OBJ(priv_vcl);
-	AZ(priv_vcl);
 }
 
 static const struct vmod_priv_methods priv_vcl_methods[1] = {{
@@ -747,11 +746,10 @@ VCL_VOID
 xyzzy_concat__fini(struct xyzzy_debug_concat **concatp)
 {
 	struct xyzzy_debug_concat *concat;
-	void *p;
+
 
 	TAKE_OBJ_NOTNULL(concat, concatp, CONCAT_MAGIC);
-	p = TRUST_ME(concat->s);
-	free(p);
+	free(TRUST_ME(concat->s));
 	FREE_OBJ(concat);
 }
 
@@ -1366,9 +1364,7 @@ xyzzy_caller__fini(struct VPFX(debug_caller) **callerp)
 
 	if (callerp == NULL || *callerp == NULL)
 		return;
-	CHECK_OBJ(*callerp, DEBUG_CALLER_MAGIC);
-	caller = *callerp;
-	*callerp = NULL;
+	TAKE_OBJ_NOTNULL(caller, callerp, DEBUG_CALLER_MAGIC);
 	FREE_OBJ(caller);
 }
 
diff --git a/vmod/vmod_debug_obj.c b/vmod/vmod_debug_obj.c
index f5e92a2e5..ac3bab0d1 100644
--- a/vmod/vmod_debug_obj.c
+++ b/vmod/vmod_debug_obj.c
@@ -339,9 +339,6 @@ xyzzy_obj_opt__fini(struct xyzzy_debug_obj_opt **op)
 {
 	struct xyzzy_debug_obj_opt *o;
 
-	AN(op);
-	AN(*op);
-
 	TAKE_OBJ_NOTNULL(o, op, VMOD_DEBUG_OBJ_OPT_MAGIC);
 
 	REPLACE(o->name, NULL);
@@ -350,7 +347,6 @@ xyzzy_obj_opt__fini(struct xyzzy_debug_obj_opt **op)
 		REPLACE(o->freeptr, NULL);
 	}
 	FREE_OBJ(o);
-	AZ(o);
 }
 
 VCL_STRING v_matchproto_()
diff --git a/vmod/vmod_directors_fall_back.c b/vmod/vmod_directors_fall_back.c
index ac79dae47..0e5d159cb 100644
--- a/vmod/vmod_directors_fall_back.c
+++ b/vmod/vmod_directors_fall_back.c
@@ -180,6 +180,7 @@ vmod_fallback_destroy(VCL_BACKEND dir)
 {
 	struct vmod_directors_fallback *fallback;
 
+	CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC);
 	CAST_OBJ_NOTNULL(fallback, dir->priv, VMOD_DIRECTORS_FALLBACK_MAGIC);
 	vdir_delete(&fallback->vd);
 	FREE_OBJ(fallback);
diff --git a/vmod/vmod_directors_hash.c b/vmod/vmod_directors_hash.c
index 2ab92fb7c..cfa212b1c 100644
--- a/vmod/vmod_directors_hash.c
+++ b/vmod/vmod_directors_hash.c
@@ -50,6 +50,7 @@ vmod_hash_destroy(VCL_BACKEND dir)
 {
 	struct vmod_directors_hash *rr;
 
+	CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC);
 	CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_HASH_MAGIC);
 	vdir_delete(&rr->vd);
 	FREE_OBJ(rr);
diff --git a/vmod/vmod_directors_random.c b/vmod/vmod_directors_random.c
index 8fe711931..f58acd702 100644
--- a/vmod/vmod_directors_random.c
+++ b/vmod/vmod_directors_random.c
@@ -91,6 +91,7 @@ vmod_random_destroy(VCL_BACKEND dir)
 {
 	struct vmod_directors_random *rr;
 
+	CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC);
 	CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_RANDOM_MAGIC);
 	vdir_delete(&rr->vd);
 	FREE_OBJ(rr);
diff --git a/vmod/vmod_directors_round_robin.c b/vmod/vmod_directors_round_robin.c
index b365a7116..1cc6ac82d 100644
--- a/vmod/vmod_directors_round_robin.c
+++ b/vmod/vmod_directors_round_robin.c
@@ -101,6 +101,7 @@ vmod_rr_destroy(VCL_BACKEND dir)
 {
 	struct vmod_directors_round_robin *rr;
 
+	CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC);
 	CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_ROUND_ROBIN_MAGIC);
 	vdir_delete(&rr->vd);
 	FREE_OBJ(rr);

From phk at FreeBSD.org  Tue Aug 17 06:29:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 17 Aug 2021 06:29:05 +0000 (UTC)
Subject: [master] ea96fd332 Rename vtc::h2's `struct txt` in preparation for
 #3659
Message-ID: <20210817062905.DDC669B5F@lists.varnish-cache.org>


commit ea96fd3322352bba0dfbe067b70ac7ef80c7e8ed
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 17 06:27:35 2021 +0000

    Rename vtc::h2's `struct txt` in preparation for #3659

diff --git a/bin/varnishtest/hpack.h b/bin/varnishtest/hpack.h
index c9d55977b..c38fc4d90 100644
--- a/bin/varnishtest/hpack.h
+++ b/bin/varnishtest/hpack.h
@@ -44,15 +44,15 @@ enum hpk_indexed {
 	hpk_never,
 };
 
-struct txt {
+struct hpk_txt {
 	char *ptr;
 	int len;
 	int huff;
 };
 
 struct hpk_hdr {
-	struct txt key;
-	struct txt value;
+	struct hpk_txt key;
+	struct hpk_txt value;
 	enum hpk_indexed t;
 	unsigned i;
 };
diff --git a/bin/varnishtest/vtc_h2_hpack.c b/bin/varnishtest/vtc_h2_hpack.c
index 64b5d0360..085741323 100644
--- a/bin/varnishtest/vtc_h2_hpack.c
+++ b/bin/varnishtest/vtc_h2_hpack.c
@@ -227,7 +227,7 @@ num_encode(struct hpk_iter *iter, uint8_t prefix, uint32_t num)
 }
 
 static enum hpk_result
-str_encode(struct hpk_iter *iter, const struct txt *t)
+str_encode(struct hpk_iter *iter, const struct hpk_txt *t)
 {
 	int slen = huff_simulate(t->ptr, t->len, t->huff);
 	assert(iter->buf < iter->end);
@@ -252,7 +252,7 @@ str_encode(struct hpk_iter *iter, const struct txt *t)
 }
 
 static enum hpk_result
-str_decode(struct hpk_iter *iter, struct txt *t)
+str_decode(struct hpk_iter *iter, struct hpk_txt *t)
 {
 	uint32_t num;
 	int huff;
@@ -292,7 +292,7 @@ str_decode(struct hpk_iter *iter, struct txt *t)
 }
 
 static inline void
-txtcpy(struct txt *to, const struct txt *from)
+txtcpy(struct hpk_txt *to, const struct hpk_txt *from)
 {
 	//AZ(to->ptr);
 	to->ptr = malloc(from->len + 1L);
@@ -311,7 +311,7 @@ enum hpk_result
 HPK_DecHdr(struct hpk_iter *iter, struct hpk_hdr *header)
 {
 	int pref = 0;
-	const struct txt *t;
+	const struct hpk_txt *t;
 	uint32_t num;
 	int must_index = 0;
 	assert(iter);
diff --git a/bin/varnishtest/vtc_h2_priv.h b/bin/varnishtest/vtc_h2_priv.h
index 683d020b3..48221f86e 100644
--- a/bin/varnishtest/vtc_h2_priv.h
+++ b/bin/varnishtest/vtc_h2_priv.h
@@ -44,7 +44,7 @@ struct hpk_iter {
 	uint8_t			*end;
 };
 
-const struct txt * tbl_get_key(const struct hpk_ctx *ctx, uint32_t index);
+const struct hpk_txt * tbl_get_key(const struct hpk_ctx *ctx, uint32_t index);
 
-const struct txt * tbl_get_value(const struct hpk_ctx *ctx, uint32_t index);
+const struct hpk_txt * tbl_get_value(const struct hpk_ctx *ctx, uint32_t index);
 void push_header (struct hpk_ctx *ctx, const struct hpk_hdr *h);
diff --git a/bin/varnishtest/vtc_h2_tbl.c b/bin/varnishtest/vtc_h2_tbl.c
index 5a5234254..658e9bc4b 100644
--- a/bin/varnishtest/vtc_h2_tbl.c
+++ b/bin/varnishtest/vtc_h2_tbl.c
@@ -179,7 +179,7 @@ HPK_ResizeTbl(struct hpk_ctx *ctx, uint32_t num)
 	return (hpk_done);
 }
 
-static const struct txt *
+static const struct hpk_txt *
 tbl_get_field(const struct hpk_ctx *ctx, uint32_t idx, int key)
 {
 	struct dynhdr *dh;
@@ -206,13 +206,13 @@ tbl_get_field(const struct hpk_ctx *ctx, uint32_t idx, int key)
 		return (NULL);
 }
 
-const struct txt *
+const struct hpk_txt *
 tbl_get_key(const struct hpk_ctx *ctx, uint32_t idx)
 {
 	return (tbl_get_field(ctx, idx, 1));
 }
 
-const struct txt *
+const struct hpk_txt *
 tbl_get_value(const struct hpk_ctx *ctx, uint32_t idx)
 {
 	return (tbl_get_field(ctx, idx, 0));

From phk at FreeBSD.org  Tue Aug 17 06:35:06 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 17 Aug 2021 06:35:06 +0000 (UTC)
Subject: [master] 40471b963 Keep FlexeLint happy
Message-ID: <20210817063506.78E6D60232@lists.varnish-cache.org>


commit 40471b9634406c84a08ef984f1ed0d25da0765dd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 17 06:34:46 2021 +0000

    Keep FlexeLint happy

diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c
index 01cbff89f..94896837a 100644
--- a/lib/libvarnishapi/vxp_parse.c
+++ b/lib/libvarnishapi/vxp_parse.c
@@ -580,7 +580,7 @@ vex_Free(struct vex **pvex)
 	TAKE_OBJ_NOTNULL(vex, pvex, VEX_MAGIC);
 
 	if (vex->lhs) {
-		CAST_OBJ(lhs, vex->lhs, VEX_LHS_MAGIC);
+		CAST_OBJ_NOTNULL(lhs, vex->lhs, VEX_LHS_MAGIC);
 		if (lhs->tags)
 			vbit_destroy(lhs->tags);
 		if (lhs->prefix)
@@ -588,7 +588,7 @@ vex_Free(struct vex **pvex)
 		FREE_OBJ(lhs);
 	}
 	if (vex->rhs) {
-		CAST_OBJ(rhs, vex->rhs, VEX_RHS_MAGIC);
+		CAST_OBJ_NOTNULL(rhs, vex->rhs, VEX_RHS_MAGIC);
 		if (rhs->val_string)
 			free(rhs->val_string);
 		if (rhs->val_regex)

From dridi.boukelmoune at gmail.com  Tue Aug 17 06:57:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 17 Aug 2021 06:57:05 +0000 (UTC)
Subject: [master] fd72157cd ws: Replace WS_Inside() with WS_Allocated()
Message-ID: <20210817065705.403C760E17@lists.varnish-cache.org>


commit fd72157cd52e974d42dff2260cb608a79f1b9403
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Jul 5 11:56:05 2021 +0200

    ws: Replace WS_Inside() with WS_Allocated()
    
    It was unfit for purpose anyway, since it would allow contents past the
    front pointer to be referenced. What we are really looking for where
    WS_Inside() was used is actual allocations. WS_Assert_Allocated() was
    made redundant by this change.
    
    Refs #3320

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index e770e23f1..f3fc6c688 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -780,8 +780,7 @@ void *WS_Copy(struct ws *ws, const void *str, int len);
 uintptr_t WS_Snapshot(struct ws *ws);
 int WS_Overflowed(const struct ws *ws);
 const char *WS_Printf(struct ws *ws, const char *fmt, ...) v_printflike_(2, 3);
-int WS_Inside(const struct ws *, const void *, const void *);
-void WS_Assert_Allocated(const struct ws *ws, const void *ptr, ssize_t len);
+int WS_Allocated(const struct ws *ws, const void *ptr, ssize_t len);
 
 void WS_VSB_new(struct vsb *, struct ws *);
 char *WS_VSB_finish(struct vsb *, struct ws *, size_t *);
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index ec0739ee6..dd4843c1c 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -1325,10 +1325,11 @@ http_CopyHome(const struct http *hp)
 			assert(u < HTTP_HDR_FIRST);
 			continue;
 		}
-		if (WS_Inside(hp->ws, hp->hd[u].b, hp->hd[u].e))
-			continue;
 
 		l = Tlen(hp->hd[u]);
+		if (WS_Allocated(hp->ws, hp->hd[u].b, l))
+			continue;
+
 		p = WS_Copy(hp->ws, hp->hd[u].b, l + 1L);
 		if (p == NULL) {
 			http_fail(hp);
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index d38587d36..8d8ab859b 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -94,7 +94,7 @@ VRT_synth(VRT_CTX, VCL_INT code, VCL_STRING reason)
 		return;
 	}
 
-	if (reason && !WS_Inside(ctx->ws, reason, NULL)) {
+	if (reason && !WS_Allocated(ctx->ws, reason, -1)) {
 		reason = WS_Copy(ctx->ws, reason, -1);
 		if (!reason) {
 			VRT_fail(ctx, "Workspace overflow");
@@ -445,14 +445,14 @@ VRT_String(struct ws *ws, const char *h, const char *p, va_list ap)
 	while (q == NULL || (q != vrt_magic_string_end && *q == '\0'));
 
 	if (h != NULL && p == NULL && q == vrt_magic_string_end &&
-	    WS_Inside(ws, h, NULL)) {
+	    WS_Allocated(ws, h, -1)) {
 		va_end(aq);
 		WS_Release(ws, 0);
 		return (h);
 	}
 
 	if (h == NULL && p != NULL && q == vrt_magic_string_end &&
-	    WS_Inside(ws, p, NULL)) {
+	    WS_Allocated(ws, p, -1)) {
 		va_end(aq);
 		WS_Release(ws, 0);
 		return (p);
@@ -467,7 +467,7 @@ VRT_String(struct ws *ws, const char *h, const char *p, va_list ap)
 		do
 			p = va_arg(aq, const char *);
 		while (p == NULL || (p != vrt_magic_string_end && *p == '\0'));
-		if (p == vrt_magic_string_end && WS_Inside(ws, q, NULL)) {
+		if (p == vrt_magic_string_end && WS_Allocated(ws, q, -1)) {
 			va_end(aq);
 			WS_Release(ws, 0);
 			return (q);
@@ -569,9 +569,9 @@ VRT_StrandsWS(struct ws *ws, const char *h, VCL_STRANDS s)
 	if (q == NULL) {
 		if (h == NULL)
 			return ("");
-		if (WS_Inside(ws, h, NULL))
+		if (WS_Allocated(ws, h, -1))
 			return (h);
-	} else if (h == NULL && WS_Inside(ws, q, NULL)) {
+	} else if (h == NULL && WS_Allocated(ws, q, -1)) {
 		for (i++; i < s->n; i++)
 			if (s->p[i] != NULL && *s->p[i] != '\0')
 				break;
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index d1f54a2fd..a82f1aa1f 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -66,28 +66,14 @@ WS_Assert(const struct ws *ws)
 }
 
 int
-WS_Inside(const struct ws *ws, const void *bb, const void *ee)
-{
-	const char *b = bb;
-	const char *e = ee;
-
-	WS_Assert(ws);
-	if (b < ws->s || b >= ws->e)
-		return (0);
-	if (e != NULL && (e < b || e > ws->e))
-		return (0);
-	return (1);
-}
-
-void
-WS_Assert_Allocated(const struct ws *ws, const void *ptr, ssize_t len)
+WS_Allocated(const struct ws *ws, const void *ptr, ssize_t len)
 {
 	const char *p = ptr;
 
 	WS_Assert(ws);
 	if (len < 0)
 		len = strlen(p) + 1;
-	assert(p >= ws->s && (p + len) <= ws->f);
+	return (p >= ws->s && (p + len) <= ws->f);
 }
 
 /*
@@ -369,7 +355,7 @@ WS_AtOffset(const struct ws *ws, unsigned off, unsigned len)
 
 	WS_Assert(ws);
 	ptr = ws->s + off;
-	WS_Assert_Allocated(ws, ptr, len);
+	AN(WS_Allocated(ws, ptr, len));
 	return (ptr);
 }
 
diff --git a/include/vrt.h b/include/vrt.h
index e61156401..e9bcd256e 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -60,6 +60,9 @@
  *	VRT_NULL_BLOB_TYPE added as the .type of vrt_null_blob
  *	VRT_blob() changed to return vrt_null_blob for
  *	    len == 0 or src == NULL arguments
+ *	[cache.h] WS_Inside() removed
+ *	[cache.h] WS_Assert_Allocated() removed
+ *	[cache.h] WS_Allocated() added
  * 13.0 (2021-03-15)
  *	Move VRT_synth_page() to deprecated status
  *	Add VRT_synth_strands() and VRT_synth_blob()
diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c
index 240f5d281..190773397 100644
--- a/vmod/vmod_debug.c
+++ b/vmod/vmod_debug.c
@@ -769,7 +769,7 @@ xyzzy_concatenate(VRT_CTX, VCL_STRANDS s)
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	r = VRT_StrandsWS(ctx->ws, NULL, s);
 	if (r != NULL && *r != '\0')
-		WS_Assert_Allocated(ctx->ws, r, strlen(r) + 1);
+		AN(WS_Allocated(ctx->ws, r, strlen(r) + 1));
 	return (r);
 }
 
@@ -781,7 +781,7 @@ xyzzy_collect(VRT_CTX, VCL_STRANDS s)
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	r = VRT_CollectStrands(ctx, s);
 	if (r != NULL && *r != '\0')
-		WS_Assert_Allocated(ctx->ws, r, strlen(r) + 1);
+		AN(WS_Allocated(ctx->ws, r, strlen(r) + 1));
 	return (r);
 }
 
@@ -813,7 +813,7 @@ xyzzy_sethdr(VRT_CTX, VCL_HEADER hdr, VCL_STRANDS s)
 			VSLb(ctx->vsl, SLT_LostHeader, "%s", hdr->what + 1);
 		} else {
 			if (*b != '\0')
-				WS_Assert_Allocated(hp->ws, b, strlen(b) + 1);
+				AN(WS_Allocated(hp->ws, b, strlen(b) + 1));
 			http_Unset(hp, hdr->what);
 			http_SetHeader(hp, b);
 		}
diff --git a/vmod/vmod_debug_obj.c b/vmod/vmod_debug_obj.c
index ac3bab0d1..94331f047 100644
--- a/vmod/vmod_debug_obj.c
+++ b/vmod/vmod_debug_obj.c
@@ -267,7 +267,7 @@ xyzzy_obj_test_priv_top(VRT_CTX, struct xyzzy_debug_obj *o, VCL_STRING s)
 	ws = req->ws;
 
 	/* copy to top req's workspace if need to */
-	if (ctx->ws != ws && WS_Inside(ctx->ws, s, NULL))
+	if (ctx->ws != ws && WS_Allocated(ctx->ws, s, -1))
 		s = WS_Copy(ws, s, -1);
 
 	if (p == NULL || s == NULL) {

From dridi.boukelmoune at gmail.com  Tue Aug 17 06:57:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 17 Aug 2021 06:57:05 +0000 (UTC)
Subject: [master] 46d9e79e6 ws: Remove deprecated WS_Front()
Message-ID: <20210817065705.551AC60E1A@lists.varnish-cache.org>


commit 46d9e79e61007f22d763c716feb8f58e9505e239
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Jul 5 12:22:40 2021 +0200

    ws: Remove deprecated WS_Front()

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index f3fc6c688..abfff698d 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -785,14 +785,6 @@ int WS_Allocated(const struct ws *ws, const void *ptr, ssize_t len);
 void WS_VSB_new(struct vsb *, struct ws *);
 char *WS_VSB_finish(struct vsb *, struct ws *, size_t *);
 
-/* REL_20210915 remove */
-static inline char*
-WS_Front(const struct ws *ws)
-{
-	AN(ws->r);
-	return ws->f;
-}
-
 static inline void *
 WS_Reservation(const struct ws *ws)
 {
diff --git a/include/vrt.h b/include/vrt.h
index e9bcd256e..180bbc8e3 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -60,6 +60,7 @@
  *	VRT_NULL_BLOB_TYPE added as the .type of vrt_null_blob
  *	VRT_blob() changed to return vrt_null_blob for
  *	    len == 0 or src == NULL arguments
+ *	[cache.h] WS_Front() removed
  *	[cache.h] WS_Inside() removed
  *	[cache.h] WS_Assert_Allocated() removed
  *	[cache.h] WS_Allocated() added

From dridi.boukelmoune at gmail.com  Tue Aug 17 06:57:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 17 Aug 2021 06:57:05 +0000 (UTC)
Subject: [master] 4c9e068df ws: Forbid zero-length allocations
Message-ID: <20210817065705.6EF8160E1D@lists.varnish-cache.org>


commit 4c9e068df853d82024c78b6b39a88e07e5b0d9d0
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Jul 7 12:20:51 2021 +0200

    ws: Forbid zero-length allocations
    
    Not even failing to allocate, but outright panicking.

diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index a82f1aa1f..f8c491ebb 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -181,6 +181,7 @@ WS_Alloc(struct ws *ws, unsigned bytes)
 	char *r;
 
 	WS_Assert(ws);
+	assert(bytes > 0);
 	bytes = PRNDUP(bytes);
 
 	assert(ws->r == NULL);
@@ -206,7 +207,7 @@ WS_Copy(struct ws *ws, const void *str, int len)
 
 	if (len == -1)
 		len = strlen(str) + 1;
-	assert(len >= 0);
+	assert(len > 0);
 
 	bytes = PRNDUP((unsigned)len);
 	if (ws->f + bytes > ws->e) {

From dridi.boukelmoune at gmail.com  Tue Aug 17 06:57:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 17 Aug 2021 06:57:05 +0000 (UTC)
Subject: [master] 94337ba52 ws: Reserve the requested size
Message-ID: <20210817065705.8778160E22@lists.varnish-cache.org>


commit 94337ba5222371dbe489340a4ed5c36c1ecb5d7f
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Jul 7 14:46:10 2021 +0200

    ws: Reserve the requested size
    
    Instead of guaranteeing the pointer alignment of ws->r at the expense of
    sometimes reserving more than requested, we can instead drop the pointer
    alignment requirement since that does not refer to the beginning of the
    allocation.
    
    Instead, it is in WS_Release() that we guarantee that the ws->f remains
    aligned for the next allocation. And WS_Release() already did that so we
    might as well avoid the potential overcommit from WS_ReserveSize().
    
    This is going to be helpful for a workspace emulator, to make sure that
    the alignment overhead wouldn't allow off-by-little overflows to sneak
    past the address sanitizer. This very change was initially made for a
    workspace sanitizer.
    
    It is also a good occasion to make the relevant checks around the ws->r
    pointer in WS_Assert().
    
    This change is visible in r03131.vtc that now exhibits this behavior.
    
    Refs #3320

diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index f8c491ebb..784e3b493 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -58,9 +58,8 @@ WS_Assert(const struct ws *ws)
 	assert(ws->f <= ws->e);
 	assert(PAOK(ws->f));
 	if (ws->r) {
-		assert(ws->r > ws->s);
+		assert(ws->r >= ws->f);
 		assert(ws->r <= ws->e);
-		assert(PAOK(ws->r));
 	}
 	assert(*ws->e == WS_REDZONE_END);
 }
@@ -284,25 +283,22 @@ WS_ReserveAll(struct ws *ws)
 unsigned
 WS_ReserveSize(struct ws *ws, unsigned bytes)
 {
-	unsigned b2;
+	unsigned l;
 
 	WS_Assert(ws);
 	assert(ws->r == NULL);
 	assert(bytes > 0);
 
-	b2 = PRNDDN(ws->e - ws->f);
-	if (bytes < b2)
-		b2 = PRNDUP(bytes);
-
-	if (bytes > b2) {
+	l = pdiff(ws->f, ws->e);
+	if (bytes > l) {
 		WS_MarkOverflow(ws);
 		return (0);
 	}
-	ws->r = ws->f + b2;
-	DSL(DBG_WORKSPACE, 0, "WS_ReserveSize(%p, %u/%u) = %zu",
-	    ws, b2, bytes, pdiff(ws->f, ws->r));
+	ws->r = ws->f + bytes;
+	DSL(DBG_WORKSPACE, 0, "WS_ReserveSize(%p, %u/%u) = %u",
+	    ws, bytes, l, bytes);
 	WS_Assert(ws);
-	return (pdiff(ws->f, ws->r));
+	return (bytes);
 }
 
 unsigned
@@ -315,12 +311,11 @@ void
 WS_Release(struct ws *ws, unsigned bytes)
 {
 	WS_Assert(ws);
-	bytes = PRNDUP(bytes);
 	assert(bytes <= ws->e - ws->f);
 	DSL(DBG_WORKSPACE, 0, "WS_Release(%p, %u)", ws, bytes);
 	assert(ws->r != NULL);
 	assert(ws->f + bytes <= ws->r);
-	ws->f += bytes;
+	ws->f += PRNDUP(bytes);
 	ws->r = NULL;
 	WS_Assert(ws);
 }
diff --git a/bin/varnishtest/tests/r03131.vtc b/bin/varnishtest/tests/r03131.vtc
index ad6fc0e95..309e03b82 100644
--- a/bin/varnishtest/tests/r03131.vtc
+++ b/bin/varnishtest/tests/r03131.vtc
@@ -22,7 +22,7 @@ varnish v1 -vcl {
 
 logexpect l1 -v v1 -g raw {
 	expect * * VCL_Log	{^\Qres(8) = 8\E$}
-	expect 0 = VCL_Log	{^\Qres(15) = 16\E$}
+	expect 0 = VCL_Log	{^\Qres(15) = 15\E$}
 	expect 0 = VCL_Log	{^\Qres(16) = 16\E$}
 
 	expect 0 = VCL_Log	{^\Qres(17) = 0\E$}

From dridi.boukelmoune at gmail.com  Tue Aug 17 06:57:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 17 Aug 2021 06:57:05 +0000 (UTC)
Subject: [master] 1d140b1a0 vca: Prevent conceptual workspace use-after-release
Message-ID: <20210817065705.A199A60E29@lists.varnish-cache.org>


commit 1d140b1a09588c31583fbb6ae7ca6e3f626bb419
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Jul 7 18:35:42 2021 +0200

    vca: Prevent conceptual workspace use-after-release
    
    Releasing a reservation or rolling back too early is harmless as long as
    nothing is allocated on the workspace that could overwrite its contents.
    
    Once you swap the workspace in its current form with a different kind of
    allocator it can turn into effective use-after-free, if rolled back or
    released state doesn't linger.

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index a57978e6a..33dc99130 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -400,8 +400,6 @@ vca_make_session(struct worker *wrk, void *arg)
 	    raddr, rport, wa->acceptlsock->name, laddr, lport,
 	    sp->t_open, sp->fd);
 
-	WS_Release(wrk->aws, 0);
-
 	vca_pace_good();
 	wrk->stats->sess_conn++;
 
@@ -416,6 +414,7 @@ vca_make_session(struct worker *wrk, void *arg)
 	req->htc->rfd = &sp->fd;
 
 	SES_SetTransport(wrk, sp, req, wa->acceptlsock->transport);
+	WS_Release(wrk->aws, 0);
 }
 
 /*--------------------------------------------------------------------

From dridi.boukelmoune at gmail.com  Tue Aug 17 06:57:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 17 Aug 2021 06:57:05 +0000 (UTC)
Subject: [master] 343e680a9 ws: New WS_Dump() facility
Message-ID: <20210817065705.BAD0D60E3B@lists.varnish-cache.org>


commit 343e680a900f7553c2d701da9fd596c0497e70ee
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Jul 8 08:03:39 2021 +0200

    ws: New WS_Dump() facility
    
    Getting rid of the very last direct access inside struct ws from
    vmod_vtc.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index abfff698d..7c12d7875 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -781,6 +781,7 @@ uintptr_t WS_Snapshot(struct ws *ws);
 int WS_Overflowed(const struct ws *ws);
 const char *WS_Printf(struct ws *ws, const char *fmt, ...) v_printflike_(2, 3);
 int WS_Allocated(const struct ws *ws, const void *ptr, ssize_t len);
+unsigned WS_Dump(const struct ws *ws, char, size_t off, void *buf, size_t len);
 
 void WS_VSB_new(struct vsb *, struct ws *);
 char *WS_VSB_finish(struct vsb *, struct ws *, size_t *);
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index 784e3b493..091179d47 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -411,6 +411,50 @@ WS_VSB_finish(struct vsb *vsb, struct ws *ws, size_t *szp)
 
 /*--------------------------------------------------------------------*/
 
+unsigned
+WS_Dump(const struct ws *ws, char where, size_t off, void *buf, size_t len)
+{
+	char *b, *p;
+	size_t l;
+
+	WS_Assert(ws);
+	AN(buf);
+	AN(len);
+
+	switch (where) {
+	case 's': p = ws->s; break;
+	case 'f': p = ws->f; break;
+	case 'r': p = ws->r; break;
+	default:
+		errno = EINVAL;
+		return (0);
+	}
+
+	if (p == NULL) {
+		errno = EAGAIN;
+		return (0);
+	}
+
+	p += off;
+	if (p >= ws->e) {
+		errno = EFAULT;
+		return (0);
+	}
+
+	l = pdiff(p, ws->e);
+	if (len <= l) {
+		memcpy(buf, p, len);
+		return (len);
+	}
+
+	b = buf;
+	memcpy(b, p, l);
+	memset(b + l, WS_REDZONE_END, len - l);
+	return (l);
+}
+
+/*--------------------------------------------------------------------*/
+
 void
 WS_Panic(struct vsb *vsb, const struct ws *ws)
 {
diff --git a/include/vrt.h b/include/vrt.h
index 180bbc8e3..4019f7e57 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -64,6 +64,7 @@
  *	[cache.h] WS_Inside() removed
  *	[cache.h] WS_Assert_Allocated() removed
  *	[cache.h] WS_Allocated() added
+ *	[cache.h] WS_Dump() added
  * 13.0 (2021-03-15)
  *	Move VRT_synth_page() to deprecated status
  *	Add VRT_synth_strands() and VRT_synth_blob()
diff --git a/vmod/vmod_vtc.c b/vmod/vmod_vtc.c
index 2b445d82b..7c09e79c3 100644
--- a/vmod/vmod_vtc.c
+++ b/vmod/vmod_vtc.c
@@ -244,9 +244,10 @@ vmod_workspace_dump(VRT_CTX, VCL_ENUM which, VCL_ENUM where,
 	struct ws *ws;
 	VCL_BYTES l, maxlen = 1024;
 	unsigned char buf[maxlen];
-	const char *p;
+	const char *p, *err;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	AN(where);
 
 	ws = vtc_ws_find(ctx, which);
 	if (ws == NULL)
@@ -259,32 +260,20 @@ vmod_workspace_dump(VRT_CTX, VCL_ENUM which, VCL_ENUM where,
 		return (NULL);
 	}
 
-	if (where == VENUM(s))
-		p = ws->s;
-	else if (where == VENUM(f))
-		p = ws->f;
-	else if (where == VENUM(r))
-		p = ws->r;
-	else
-		INCOMPL();
+	l = WS_Dump(ws, *where, off, buf, len);
 
-	if (p == NULL) {
-		VSLb(ctx->vsl, SLT_Error, "workspace_dump: NULL");
-		return (NULL);
-	}
-
-	p += off;
-	if (p >= ws->e) {
-		VSLb(ctx->vsl, SLT_Error, "workspace_dump: off limit");
+	if (l == 0) {
+		switch (errno) {
+		case EINVAL: WRONG(where); break;
+		case EAGAIN: err = "NULL"; break;
+		case EFAULT: err = "off limit"; break;
+		default: err = "unknown error"; break;
+		}
+		VRT_fail(ctx, "workspace_dump: %s", err);
 		return (NULL);
 	}
 
-	l = pdiff(p, ws->e);
-	if (len < l)
-		l = len;
-
 	assert(l < maxlen);
-	memcpy(buf, p, l);
 	p = WS_Copy(ctx->ws, buf, l);
 	if (p == NULL) {
 		VRT_fail(ctx, "workspace_dump: copy failed");

From dridi.boukelmoune at gmail.com  Tue Aug 17 06:57:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 17 Aug 2021 06:57:05 +0000 (UTC)
Subject: [master] 3fc784782 wrk: Track workers to dump them during a panic
Message-ID: <20210817065705.D49C560E40@lists.varnish-cache.org>


commit 3fc7847828963d0870b2eb4f4880512209304bfb
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Jul 12 07:31:05 2021 +0200

    wrk: Track workers to dump them during a panic
    
    In the most common case of a VCL transaction panicking this should only
    result in an extra "Already dumped, see above" message, but it may give
    more insights for other kinds of tasks.

diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index a60e8a584..4b98a8a0f 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -76,6 +76,7 @@ cache_vrnd_unlock(void)
 
 static pthread_key_t req_key;
 static pthread_key_t bo_key;
+static pthread_key_t wrk_key;
 pthread_key_t witness_key;
 
 void
@@ -106,6 +107,20 @@ THR_GetRequest(void)
 	return (pthread_getspecific(req_key));
 }
 
+void
+THR_SetWorker(const struct worker *wrk)
+{
+
+	AZ(pthread_setspecific(wrk_key, wrk));
+}
+
+struct worker *
+THR_GetWorker(void)
+{
+
+	return (pthread_getspecific(wrk_key));
+}
+
 /*--------------------------------------------------------------------
  * Name threads if our pthreads implementation supports it.
  */
@@ -351,6 +366,7 @@ child_main(int sigmagic, size_t altstksz)
 
 	AZ(pthread_key_create(&req_key, NULL));
 	AZ(pthread_key_create(&bo_key, NULL));
+	AZ(pthread_key_create(&wrk_key, NULL));
 	AZ(pthread_key_create(&witness_key, free));
 	AZ(pthread_key_create(&name_key, NULL));
 
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 68309705f..533c9e906 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -712,6 +712,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond,
 	const char *q;
 	struct req *req;
 	struct busyobj *bo;
+	struct worker *wrk;
 	struct sigaction sa;
 	int err = errno;
 
@@ -794,6 +795,9 @@ pan_ic(const char *func, const char *file, int line, const char *cond,
 		pan_busyobj(pan_vsb, bo);
 		if (bo != NULL)
 			VSL_Flush(bo->vsl, 0);
+		wrk = THR_GetWorker();
+		VSB_cat(pan_vsb, "thr.");
+		pan_wrk(pan_vsb, wrk);
 		VMOD_Panic(pan_vsb);
 		pan_pool(pan_vsb);
 	} else {
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index 96db999b1..3f5d929f3 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -309,6 +309,8 @@ void THR_SetBusyobj(const struct busyobj *);
 struct busyobj * THR_GetBusyobj(void);
 void THR_SetRequest(const struct req *);
 struct req * THR_GetRequest(void);
+void THR_SetWorker(const struct worker *);
+struct worker * THR_GetWorker(void);
 void THR_Init(void);
 
 /* cache_lck.c */
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 32b53024d..db4139a98 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -145,6 +145,7 @@ WRK_Thread(struct pool *qp, size_t stacksize, unsigned thread_workspace)
 	w->lastused = NAN;
 	memset(&ds, 0, sizeof ds);
 	w->stats = &ds;
+	THR_SetWorker(w);
 	AZ(pthread_cond_init(&w->cond, NULL));
 
 	WS_Init(w->aws, "wrk", ws, thread_workspace);

From dridi.boukelmoune at gmail.com  Tue Aug 17 06:57:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 17 Aug 2021 06:57:05 +0000 (UTC)
Subject: [master] 0b32af6dc panic: Optionally track miniobjs
Message-ID: <20210817065705.EEC1960E45@lists.varnish-cache.org>


commit 0b32af6dc78e422fa1fdc41033ad772f06e5e246
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Jul 12 08:25:38 2021 +0200

    panic: Optionally track miniobjs
    
    PAN_dump_struct2() is now renamed to PAN__DumpStruct() following the
    guidelines from apispaces.rst in addition to growing a track argument.
    This should allow dumping structures that we know ought to be dumped
    only once, in case they would be numerous.
    
    There seems to be no good reason to expose it to VMODs yet since they
    have no integration point with the panic output, so those definitions
    can be confined in cache_varnishd.h for now.
    
    Better diff with the --ignore-all-space option.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 7c12d7875..e65c76ea7 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -835,13 +835,3 @@ Tlen(const txt t)
  * extra timestamps in cache_pool.c.  Hide this detail with a macro
  */
 #define W_TIM_real(w) ((w)->lastused = VTIM_real())
-
-int PAN_dump_struct2(struct vsb *vsb, int block, const void *ptr,
-    const char *smagic, unsigned magic, const char *fmt, ...)
-    v_printflike_(6,7);
-
-#define PAN_dump_struct(vsb, ptr, magic, ...)		\
-    PAN_dump_struct2(vsb, 1, ptr, #magic, magic, __VA_ARGS__)
-
-#define PAN_dump_oneline(vsb, ptr, magic, ...)		\
-    PAN_dump_struct2(vsb, 0, ptr, #magic, magic, __VA_ARGS__)
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 533c9e906..14de7e11b 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -110,7 +110,7 @@ static const void *already_list[N_ALREADY];
 static int already_idx;
 
 int
-PAN_dump_struct2(struct vsb *vsb, int block, const void *ptr,
+PAN__DumpStruct(struct vsb *vsb, int block, int track, const void *ptr,
     const char *smagic, unsigned magic, const char *fmt, ...)
 {
 	va_list ap;
@@ -128,17 +128,19 @@ PAN_dump_struct2(struct vsb *vsb, int block, const void *ptr,
 	VSB_printf(vsb, " = %p {", ptr);
 	if (block)
 		VSB_putc(vsb, '\n');
-	for (i = 0; i < already_idx; i++) {
-		if (already_list[i] == ptr) {
-			VSB_cat(vsb, "  [Already dumped, see above]");
-			if (block)
-				VSB_putc(vsb, '\n');
-			VSB_cat(vsb, "},\n");
-			return (-2);
+	if (track) {
+		for (i = 0; i < already_idx; i++) {
+			if (already_list[i] == ptr) {
+				VSB_cat(vsb, "  [Already dumped, see above]");
+				if (block)
+					VSB_putc(vsb, '\n');
+				VSB_cat(vsb, "},\n");
+				return (-2);
+			}
 		}
+		if (already_idx < N_ALREADY)
+			already_list[already_idx++] = ptr;
 	}
-	if (already_idx < N_ALREADY)
-		already_list[already_idx++] = ptr;
 	uptr = ptr;
 	if (*uptr != magic) {
 		VSB_printf(vsb, "  .magic = 0x%08x", *uptr);
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index 3f5d929f3..b150ea5e0 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -367,6 +367,22 @@ void ObjUnsubscribeEvents(uintptr_t *);
 
 /* cache_panic.c */
 void PAN_Init(void);
+int PAN__DumpStruct(struct vsb *vsb, int block, int track, const void *ptr,
+    const char *smagic, unsigned magic, const char *fmt, ...)
+    v_printflike_(7,8);
+
+#define PAN_dump_struct(vsb, ptr, magic, ...)		\
+    PAN__DumpStruct(vsb, 1, 1, ptr, #magic, magic, __VA_ARGS__)
+
+#define PAN_dump_oneline(vsb, ptr, magic, ...)		\
+    PAN__DumpStruct(vsb, 0, 1, ptr, #magic, magic, __VA_ARGS__)
+
+#define PAN_dump_once(vsb, ptr, magic, ...)		\
+    PAN__DumpStruct(vsb, 1, 0, ptr, #magic, magic, __VA_ARGS__)
+
+#define PAN_dump_once_oneline(vsb, ptr, magic, ...)		\
+    PAN__DumpStruct(vsb, 0, 0, ptr, #magic, magic, __VA_ARGS__)
+
 const char *sess_close_2str(enum sess_close sc, int want_desc);
 
 /* cache_pool.c */

From dridi.boukelmoune at gmail.com  Tue Aug 17 06:57:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 17 Aug 2021 06:57:06 +0000 (UTC)
Subject: [master] 0632b8469 req: Prevent early rollback
Message-ID: <20210817065706.158C260E49@lists.varnish-cache.org>


commit 0632b84693f3e146b7208e5310bc59b0e2769296
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jul 13 07:50:05 2021 +0200

    req: Prevent early rollback
    
    After a cleanup we may still use the request's workspace, so the final
    rollback should instead happen at release time before we give the memory
    back to the pool.

diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c
index e2981e69b..eeb339b62 100644
--- a/bin/varnishd/cache/cache_req.c
+++ b/bin/varnishd/cache/cache_req.c
@@ -172,6 +172,7 @@ Req_Release(struct req *req)
 	AZ(req->vcl);
 	if (req->vsl->wid)
 		VSL_End(req->vsl);
+	WS_Rollback(req->ws, 0);
 	TAKE_OBJ_NOTNULL(sp, &req->sp, SESS_MAGIC);
 	pp = sp->pool;
 	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
@@ -261,7 +262,9 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 	if (WS_Overflowed(req->ws))
 		wrk->stats->ws_client_overflow++;
 
-	WS_Rollback(req->ws, 0);
+	/* no snapshot for h2 stream 0 */
+	if (req->ws_req)
+		WS_Rollback(req->ws, req->ws_req);
 }
 
 /*----------------------------------------------------------------------

From dridi.boukelmoune at gmail.com  Tue Aug 17 06:57:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 17 Aug 2021 06:57:06 +0000 (UTC)
Subject: [master] ce71896ae sess: Plug conceptual leak
Message-ID: <20210817065706.3159860E4E@lists.varnish-cache.org>


commit ce71896ae353387d1da4088f5c6c3921d98b249a
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jul 13 07:52:15 2021 +0200

    sess: Plug conceptual leak
    
    It's harmless with a regular workspace, but technically we are leaving
    outstanding allocations in the workspace.

diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index dbbce45d0..51d10b417 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -654,6 +654,7 @@ SES_Rel(struct sess *sp)
 	if (i)
 		return;
 	Lck_Delete(&sp->mtx);
+	WS_Rollback(sp->ws, 0);
 	MPL_Free(sp->pool->mpl_sess, sp);
 }
 

From dridi.boukelmoune at gmail.com  Tue Aug 17 06:57:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 17 Aug 2021 06:57:06 +0000 (UTC)
Subject: [master] 246b1eb1e busyobj: Plug conceptual leak
Message-ID: <20210817065706.5219460E67@lists.varnish-cache.org>


commit 246b1eb1e888d6b937008da95a7776c40e1db2bc
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jul 13 07:53:44 2021 +0200

    busyobj: Plug conceptual leak
    
    It's harmless with a regular workspace, but technically we are leaving
    outstanding allocations in the workspace.

diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index c6c85120e..139112854 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -181,6 +181,7 @@ VBO_ReleaseBusyObj(struct worker *wrk, struct busyobj **pbo)
 	}
 
 	VCL_Rel(&bo->vcl);
+	WS_Rollback(bo->ws, 0);
 
 	memset(&bo->retries, 0,
 	    sizeof *bo - offsetof(struct busyobj, retries));

From dridi.boukelmoune at gmail.com  Tue Aug 17 06:57:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 17 Aug 2021 06:57:06 +0000 (UTC)
Subject: [master] 9f1d6a906 v1l: Prevent conceptual use-after-free
Message-ID: <20210817065706.6DB5D60E75@lists.varnish-cache.org>


commit 9f1d6a906b1d99b4d4daea2caf99578e640544c8
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jul 13 07:55:15 2021 +0200

    v1l: Prevent conceptual use-after-free
    
    The miniobj resides in the workspace it's rolling back. To preserve its
    zeroing we need to roll back afterwards.

diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c
index 52e1c88b0..006119e98 100644
--- a/bin/varnishd/http1/cache_http1_line.c
+++ b/bin/varnishd/http1/cache_http1_line.c
@@ -127,6 +127,8 @@ enum sess_close
 V1L_Close(struct worker *wrk, uint64_t *cnt)
 {
 	struct v1l *v1l;
+	struct ws *ws;
+	uintptr_t ws_snap;
 	enum sess_close sc;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -136,8 +138,10 @@ V1L_Close(struct worker *wrk, uint64_t *cnt)
 	wrk->v1l = NULL;
 	CHECK_OBJ_NOTNULL(v1l, V1L_MAGIC);
 	*cnt = v1l->cnt;
-	WS_Rollback(v1l->ws, v1l->ws_snap);
+	ws = v1l->ws;
+	ws_snap = v1l->ws_snap;
 	ZERO_OBJ(v1l, sizeof *v1l);
+	WS_Rollback(ws, ws_snap);
 	return (sc);
 }
 

From dridi.boukelmoune at gmail.com  Tue Aug 17 06:57:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 17 Aug 2021 06:57:06 +0000 (UTC)
Subject: [master] 5d7511cf2 panic: Sanitizers consume more stack frames
Message-ID: <20210817065706.8DAD260E86@lists.varnish-cache.org>


commit 5d7511cf2e21db8aae45d2878cb8b6b67f44050b
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jul 13 09:10:29 2021 +0200

    panic: Sanitizers consume more stack frames

diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 14de7e11b..30235976f 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -618,7 +618,11 @@ pan_backtrace(struct vsb *vsb)
 
 #else /* WITH_UNWIND */
 
-#define BACKTRACE_LEVELS	10
+#if __SANITIZER
+#  define BACKTRACE_LEVELS	20
+#else
+#  define BACKTRACE_LEVELS	10
+#endif
 
 static void
 pan_backtrace(struct vsb *vsb)

From phk at FreeBSD.org  Tue Aug 17 08:07:06 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 17 Aug 2021 08:07:06 +0000 (UTC)
Subject: [master] 294b1ff29 Fail attempts to set parameters which have
 NOT_IMPLEMENTED set.
Message-ID: <20210817080706.2419A643BD@lists.varnish-cache.org>


commit 294b1ff29487ca3f980f793e51c50e0013eea02f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 17 07:53:51 2021 +0000

    Fail attempts to set parameters which have NOT_IMPLEMENTED set.

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 08dc48233..34e23d466 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -501,6 +501,14 @@ MCF_ParamSet(struct cli *cli, const char *param, const char *val)
 		VCLI_Out(cli, "Unknown parameter \"%s\".", param);
 		return;
 	}
+	if (pp->flags & NOT_IMPLEMENTED) {
+		VCLI_SetResult(cli, CLIS_CANT);
+		VCLI_Out(cli,
+		    "parameter \"%s\" is not available on this platform.",
+		    param
+		);
+		return;
+	}
 	if (pp->flags & PROTECTED) {
 		VCLI_SetResult(cli, CLIS_AUTH);
 		VCLI_Out(cli, "parameter \"%s\" is protected.", param);

From phk at FreeBSD.org  Tue Aug 17 08:07:06 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 17 Aug 2021 08:07:06 +0000 (UTC)
Subject: [master] dfbc77ac5 Correctly detect TCP_FASTOPEN on FreeBSD
Message-ID: <20210817080706.37F48643C0@lists.varnish-cache.org>


commit dfbc77ac546aa1d86b6fa410bcd8f293d2453c37
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 17 08:06:34 2021 +0000

    Correctly detect TCP_FASTOPEN on FreeBSD

diff --git a/configure.ac b/configure.ac
index c396a3806..df96a23ea 100644
--- a/configure.ac
+++ b/configure.ac
@@ -632,6 +632,10 @@ AC_CACHE_CHECK([for TCP_FASTOPEN socket option],
 #include <netinet/in.h>
 #include <netinet/tcp.h>
 #include <errno.h>
+
+#ifndef SOL_TCP
+#  define SOL_TCP IPPROTO_TCP
+#endif
     ]],[[
 int s = socket(AF_INET, SOCK_STREAM, 0);
 int i = 5;
diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index 85521b8c9..1dca3c6ce 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -189,26 +189,23 @@ VTCP_filter_http(int sock)
 
 /*--------------------------------------------------------------------*/
 
-#ifdef HAVE_TCP_FASTOPEN
 
 int
 VTCP_fastopen(int sock, int depth)
 {
+#ifdef HAVE_TCP_FASTOPEN
+#  ifndef SOL_TCP
+#    define SOL_TCP IPPROTO_TCP
+#  endif
 	return (setsockopt(sock, SOL_TCP, TCP_FASTOPEN, &depth, sizeof depth));
-}
-
 #else
-
-int
-VTCP_fastopen(int sock, int depth)
-{
 	errno = EOPNOTSUPP;
 	(void)sock;
 	(void)depth;
 	return (-1);
+#endif
 }
 
-#endif
 
 /*--------------------------------------------------------------------
  * Functions for controlling NONBLOCK mode.

From phk at FreeBSD.org  Tue Aug 17 08:34:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 17 Aug 2021 08:34:05 +0000 (UTC)
Subject: [master] dc7e849fe Dont depend on `accept_filter` parameter being
 settable
Message-ID: <20210817083405.42D0965343@lists.varnish-cache.org>


commit dc7e849fe1d4d02087558b3eed8b3f26d814b706
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 17 08:33:02 2021 +0000

    Dont depend on `accept_filter` parameter being settable

diff --git a/bin/varnishtest/tests/b00046.vtc b/bin/varnishtest/tests/b00046.vtc
index abf5663d5..bcdecf271 100644
--- a/bin/varnishtest/tests/b00046.vtc
+++ b/bin/varnishtest/tests/b00046.vtc
@@ -12,7 +12,9 @@ server s1 {
 	send_urgent " "
 } -start
 
-varnish v1 -cliok "param.set accept_filter off"
+# -cli because accept_filter may not be supported
+varnish v1 -cli "param.set accept_filter off"
+
 varnish v1 -vcl+backend {} -start
 
 client c1 {
diff --git a/bin/varnishtest/tests/r01914.vtc b/bin/varnishtest/tests/r01914.vtc
index 753051893..a86736ce2 100644
--- a/bin/varnishtest/tests/r01914.vtc
+++ b/bin/varnishtest/tests/r01914.vtc
@@ -13,7 +13,6 @@ varnish v1 \
     -arg "-s default,1MB" \
     -arg "-s default,1MB" \
     -arg "-s Transient=default" \
-    -arg "-p accept_filter=off" \
     -syntax 4.0 \
     -vcl+backend {
 	import std;
@@ -29,7 +28,12 @@ varnish v1 \
 	sub vcl_backend_response {
 		set beresp.storage = storage.s0;
 	}
-} -start
+}
+
+# -cli because accept_filter may not be supported
+varnish v1 -cli "param.set accept_filter off"
+
+varnish v1 -start
 
 client c1 {
 	txreq -url /0 -bodylen 100
diff --git a/bin/varnishtest/tests/r02105.vtc b/bin/varnishtest/tests/r02105.vtc
index 4dd2c9a28..4eddec94f 100644
--- a/bin/varnishtest/tests/r02105.vtc
+++ b/bin/varnishtest/tests/r02105.vtc
@@ -9,7 +9,9 @@ server s1 {
 	txresp
 } -start
 
-varnish v1 -cliok "param.set accept_filter off"
+# -cli because accept_filter may not be supported
+varnish v1 -cli "param.set accept_filter off"
+
 varnish v1 -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.ttl = 0.5s;
diff --git a/bin/varnishtest/tests/r02266.vtc b/bin/varnishtest/tests/r02266.vtc
index c0f4c9d18..068881313 100644
--- a/bin/varnishtest/tests/r02266.vtc
+++ b/bin/varnishtest/tests/r02266.vtc
@@ -16,7 +16,9 @@ server s1 {
 	send "line2\n"
 } -start
 
-varnish v1 -cliok "param.set accept_filter off"
+# -cli because accept_filter may not be supported
+varnish v1 -cli "param.set accept_filter off"
+
 varnish v1 -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.do_stream = false;

From dridi.boukelmoune at gmail.com  Tue Aug 17 09:05:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 17 Aug 2021 09:05:06 +0000 (UTC)
Subject: [master] c71ab01e0 vcl: Allow header names to be quoted
Message-ID: <20210817090506.B6D6D6E2FE@lists.varnish-cache.org>


commit c71ab01e054a896448326977769c811e59d0dddf
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Jun 30 14:31:22 2021 +0200

    vcl: Allow header names to be quoted
    
    Because we funnel HTTP header names through the symbol table they have
    to be valid VCL identifiers. It means that we can't support all valid
    header names, which are tokens in the HTTP grammar. To finally close this
    loophole without the help of a VMOD we allow header names to be quoted:
    
        req.http.regular-header
        req.http."quoted.header"
    
    However we don't want to allow any component of a symbol to be quoted:
    
        req."http".we-dont-want-this
    
    So we teach the symbol table that wildcard symbols may be quoted. There
    used to be several use cases for wildcards but it is now limited to HTTP
    headers.
    
    Refs #3246
    Refs #3379

diff --git a/bin/varnishtest/tests/b00076.vtc b/bin/varnishtest/tests/b00076.vtc
new file mode 100644
index 000000000..dbb01e81f
--- /dev/null
+++ b/bin/varnishtest/tests/b00076.vtc
@@ -0,0 +1,51 @@
+varnishtest "Non-symbolic HTTP headers names"
+
+varnish v1 -errvcl "Invalid character '\\n' in header name" {
+	backend be none;
+	sub vcl_recv {
+		set req.http.{"line
+break"} = "invalid";
+	}
+}
+
+varnish v1 -errvcl "Expected '=' got '.'" {
+	backend be none;
+	sub vcl_recv {
+		set req."http".wrong-quote = "invalid";
+	}
+}
+
+varnish v1 -syntax 4.0 -errvcl "Quoted headers are available for VCL >= 4.1" {
+	backend be none;
+	sub vcl_recv {
+		set req.http."quoted" = "invalid";
+	}
+}
+
+varnish v1 -vcl {
+	import std;
+	backend be none;
+	sub vcl_recv {
+		std.collect(req.http."...");
+		return (synth(200));
+	}
+	sub vcl_synth {
+		set resp.http."123" = "456";
+		set resp.http."456" = resp.http."123";
+		set resp.http.{"!!!"} = "???";
+		set resp.http."""resp.http.foo""" = "bar";
+		set resp.http.bar = resp.http."resp.http.foo".upper();
+		set resp.http."..." = req.http."...";
+	}
+} -start
+
+client c1 {
+	txreq -hdr "...: a" -hdr "...: b"
+	rxresp
+	expect resp.http.123 == 456
+	expect resp.http.456 == 456
+	expect resp.http.!!! == ???
+	expect resp.http.resp.http.foo == bar
+	expect resp.http.bar == BAR
+	expect resp.http.... == "a, b"
+} -run
diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst
index af4c7ecdf..cce173636 100644
--- a/doc/sphinx/reference/vcl_var.rst
+++ b/doc/sphinx/reference/vcl_var.rst
@@ -225,11 +225,15 @@ req.http.*
 	The headers of request, things like ``req.http.date``.
 
 	The RFCs allow multiple headers with the same name, and both
-	``set`` and ``unset`` will remove *all* headers with the name given.
+	``set`` and ``unset`` will remove *all* headers with the name
+	given.
 
 	The header name ``*`` is a VCL symbol and as such cannot, for
-	example, start with a numeral. Custom VMODs exist for handling
-	of such header names.
+	example, start with a numeral. To work with valid header that
+	can't be represented as VCL symbols it is possible to quote the
+	name, like ``req.http."grammatically.valid"``. None of the HTTP
+	headers present in IANA registries need to be quoted, so the
+	quoted syntax is discouraged but available for interoperability.
 
 
 req.restarts
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index 6c7c182d7..703553f27 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -141,21 +141,24 @@ vcc_symtab_new(const char *name)
 }
 
 static struct symtab *
-vcc_symtab_str(struct symtab *st, const char *b, const char *e)
+vcc_symtab_str(struct symtab *st, const char *b, const char *e, unsigned tok)
 {
 	struct symtab *st2, *st3;
 	size_t l;
 	int i;
 	const char *q;
 
+	assert(tok == ID || tok == CSTR);
 	if (e == NULL)
 		e = strchr(b, '\0');
+	q = e;
 
 	while (b < e) {
-		for (q = b; q < e && *q != '.'; q++)
-			continue;
-		AN(q);
-		l = q - b;
+		if (tok == ID) {
+			for (q = b; q < e && *q != '.'; q++)
+				continue;
+		}
+		l = pdiff(b, q);
 		VTAILQ_FOREACH(st2, &st->children, list) {
 			i = strncasecmp(st2->name, b, l);
 			if (i < 0)
@@ -208,10 +211,12 @@ vcc_sym_in_tab(struct vcc *tl, struct symtab *st,
 	VTAILQ_FOREACH(sym, &st->symbols, list) {
 		if (sym->lorev > vhi || sym->hirev < vlo)
 			continue;
-		if ((kind == SYM_NONE && kind == sym->kind))
+		if (kind == SYM_NONE && kind == sym->kind &&
+		    sym->wildcard == NULL)
 			continue;
 		if (tl->syntax < VCL_41 && strcmp(sym->name, "default") &&
-		     (kind != SYM_NONE && kind != sym->kind))
+		     kind != SYM_NONE && kind != sym->kind &&
+		     sym->wildcard == NULL)
 			continue;
 		return (sym);
 	}
@@ -285,8 +290,20 @@ VCC_SymbolGet(struct vcc *tl, vcc_ns_t ns, vcc_kind_t kind,
 	st = tl->syms[ns->id];
 	t0 = tl->t;
 	tn = tl->t;
+	assert(tn->tok == ID);
 	while (1) {
-		st = vcc_symtab_str(st, tn->b, tn->e);
+		assert(tn->tok == ID || tn->tok == CSTR);
+		if (tn->tok == CSTR && tl->syntax < VCL_41) {
+			VSB_cat(tl->sb,
+			    "Quoted headers are available for VCL >= 4.1.\n"
+			    "At:");
+			vcc_ErrWhere(tl, tn);
+			return (NULL);
+		}
+		if (tn->tok == ID)
+			st = vcc_symtab_str(st, tn->b, tn->e, tn->tok);
+		else
+			st = vcc_symtab_str(st, tn->dec, NULL, tn->tok);
 		sym2 = vcc_sym_in_tab(tl, st, kind, tl->syntax, tl->syntax);
 		if (sym2 != NULL) {
 			sym = sym2;
@@ -297,7 +314,11 @@ VCC_SymbolGet(struct vcc *tl, vcc_ns_t ns, vcc_kind_t kind,
 		if (tn1->tok != '.')
 			break;
 		tn1 = vcc_PeekTokenFrom(tl, tn1);
-		if (tn1->tok != ID)
+		if (tn1->tok == CSTR && sym == NULL)
+			break;
+		if (tn1->tok == CSTR && sym->wildcard == NULL)
+			break;
+		if (tn1->tok != CSTR && tn1->tok != ID)
 			break;
 		tn = tn1;
 	}
@@ -421,7 +442,7 @@ VCC_MkSym(struct vcc *tl, const char *b, vcc_ns_t ns, vcc_kind_t kind,
 
 	if (tl->syms[ns->id] == NULL)
 		tl->syms[ns->id] = vcc_symtab_new("");
-	st = vcc_symtab_str(tl->syms[ns->id], b, NULL);
+	st = vcc_symtab_str(tl->syms[ns->id], b, NULL, ID);
 	AN(st);
 	sym = vcc_sym_in_tab(tl, st, kind, vlo, vhi);
 	AZ(sym);
diff --git a/lib/libvcc/vcc_var.c b/lib/libvcc/vcc_var.c
index cdc071f9f..c71831bd6 100644
--- a/lib/libvcc/vcc_var.c
+++ b/lib/libvcc/vcc_var.c
@@ -36,12 +36,15 @@
 
 #include "vcc_compile.h"
 
+#include "vct.h"
+
 /*--------------------------------------------------------------------*/
 
 void v_matchproto_(sym_wildcard_t)
 vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent, struct symbol *sym)
 {
 	struct vsb *vsb;
+	const char *p;
 
 	assert(parent->type == HEADER);
 
@@ -52,6 +55,16 @@ vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent, struct symbol *sym)
 		return;
 	}
 
+	for (p = sym->name; *p != '\0'; p++) {
+		if (!vct_istchar(*p)) {
+			VSB_cat(tl->sb, "Invalid character '");
+			VSB_quote(tl->sb, p, 1, VSB_QUOTE_PLAIN);
+			VSB_cat(tl->sb, "' in header name.\n");
+			tl->err = 1;
+			return;
+		}
+	}
+
 	AN(sym);
 	sym->noref = 1;
 	sym->kind = SYM_VAR;

From dridi.boukelmoune at gmail.com  Tue Aug 17 12:33:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 17 Aug 2021 12:33:06 +0000 (UTC)
Subject: [master] 00eaed81e vcc: Flexelint still thinks sym could be null
Message-ID: <20210817123306.77D409C406@lists.varnish-cache.org>


commit 00eaed81e3d29ca26cc78c4d5ea9f1b10e80658b
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 17 14:31:08 2021 +0200

    vcc: Flexelint still thinks sym could be null
    
    When we check sym->wildcard for quoted headers.

diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index 703553f27..5125cda3e 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -314,9 +314,7 @@ VCC_SymbolGet(struct vcc *tl, vcc_ns_t ns, vcc_kind_t kind,
 		if (tn1->tok != '.')
 			break;
 		tn1 = vcc_PeekTokenFrom(tl, tn1);
-		if (tn1->tok == CSTR && sym == NULL)
-			break;
-		if (tn1->tok == CSTR && sym->wildcard == NULL)
+		if (tn1->tok == CSTR && (sym == NULL || sym->wildcard == NULL))
 			break;
 		if (tn1->tok != CSTR && tn1->tok != ID)
 			break;

From phk at FreeBSD.org  Thu Aug 19 06:31:07 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 19 Aug 2021 06:31:07 +0000 (UTC)
Subject: [master] dd16b6d0c Add VRT_UnsetHdr() and remove
 vrt_magic_string_unset
Message-ID: <20210819063108.066739558F@lists.varnish-cache.org>


commit dd16b6d0c54e13093acc29652b9e3e5a237c1115
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Aug 19 06:29:09 2021 +0000

    Add VRT_UnsetHdr() and remove vrt_magic_string_unset

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 8d8ab859b..17e406edf 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -55,7 +55,6 @@
 #include "proxy/cache_proxy.h"
 
 const void * const vrt_magic_string_end = &vrt_magic_string_end;
-const void * const vrt_magic_string_unset = &vrt_magic_string_unset;
 const struct strands *vrt_null_strands = &(struct strands){
 	.n = 0,
 	.p = (const char *[1]){NULL}
@@ -691,6 +690,19 @@ VRT_ValidHdr(VRT_CTX, VCL_STRANDS s)
 }
 /*--------------------------------------------------------------------*/
 
+VCL_VOID
+VRT_UnsetHdr(VRT_CTX , VCL_HEADER hs)
+{
+	VCL_HTTP hp;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	AN(hs);
+	AN(hs->what);
+	hp = VRT_selecthttp(ctx, hs->where);
+	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+	http_Unset(hp, hs->what);
+}
+
 VCL_VOID
 VRT_SetHdr(VRT_CTX , VCL_HEADER hs, const char *p, ...)
 {
@@ -703,23 +715,19 @@ VRT_SetHdr(VRT_CTX , VCL_HEADER hs, const char *p, ...)
 	AN(hs->what);
 	hp = VRT_selecthttp(ctx, hs->where);
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
-	if (p == vrt_magic_string_unset) {
-		http_Unset(hp, hs->what);
-	} else {
-		va_start(ap, p);
-		b = VRT_String(hp->ws, hs->what + 1, p, ap);
-		va_end(ap);
-		if (b == NULL) {
-			VSLb(ctx->vsl, SLT_LostHeader, "%s", hs->what + 1);
-			return;
-		}
-		if (FEATURE(FEATURE_VALIDATE_HEADERS) && ! validhdr(b)) {
-			VRT_fail(ctx, "Bad header %s", b);
-			return;
-		}
-		http_Unset(hp, hs->what);
-		http_SetHeader(hp, b);
+	va_start(ap, p);
+	b = VRT_String(hp->ws, hs->what + 1, p, ap);
+	va_end(ap);
+	if (b == NULL) {
+		VSLb(ctx->vsl, SLT_LostHeader, "%s", hs->what + 1);
+		return;
+	}
+	if (FEATURE(FEATURE_VALIDATE_HEADERS) && ! validhdr(b)) {
+		VRT_fail(ctx, "Bad header %s", b);
+		return;
 	}
+	http_Unset(hp, hs->what);
+	http_SetHeader(hp, b);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/include/vrt.h b/include/vrt.h
index 4019f7e57..8c6678935 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -54,6 +54,8 @@
  * binary/load-time compatible, increment MAJOR version
  *
  * NEXT (2021-09-15)
+ *	VRT_UnsetHdr() added
+ *	vrt_magic_string_unset removed (use VRT_UnsetHdr() instead)
  *	VNUMpfx() removed, SF_Parse_{Integer|Decimal|Number} added
  *	vrt_null_strands added
  *	vrt_null_blob added
@@ -593,6 +595,7 @@ VCL_VOID VRT_synth(VRT_CTX, VCL_INT, VCL_STRING);
 VCL_VOID VRT_hit_for_pass(VRT_CTX, VCL_DURATION);
 
 VCL_BOOL VRT_ValidHdr(VRT_CTX, VCL_STRANDS);
+VCL_VOID VRT_UnsetHdr(VRT_CTX, VCL_HEADER);
 VCL_VOID VRT_SetHdr(VRT_CTX, VCL_HEADER, const char *, ...);
 VCL_VOID VRT_handling(VRT_CTX, unsigned hand);
 unsigned VRT_handled(VRT_CTX);
@@ -709,5 +712,4 @@ void VRT_VCL_Allow_Discard(struct vclref **);
 
 VCL_VOID VRT_synth_page(VRT_CTX, VCL_STRANDS);
 extern const void * const vrt_magic_string_end;
-extern const void * const vrt_magic_string_unset;
 int VRT_Stv(const char *nm);
diff --git a/lib/libvcc/vcc_var.c b/lib/libvcc/vcc_var.c
index c71831bd6..286949e49 100644
--- a/lib/libvcc/vcc_var.c
+++ b/lib/libvcc/vcc_var.c
@@ -93,8 +93,7 @@ vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent, struct symbol *sym)
 	AZ(VSB_finish(vsb));
 	sym->lname = TlDup(tl, VSB_data(vsb));
 	VSB_clear(vsb);
-	VSB_printf(vsb, "VRT_SetHdr(ctx, %s, vrt_magic_string_unset)",
-	    sym->rname);
+	VSB_printf(vsb, "VRT_UnsetHdr(ctx, %s)", sym->rname);
 	AZ(VSB_finish(vsb));
 	sym->uname = TlDup(tl, VSB_data(vsb));
 	VSB_destroy(&vsb);

From phk at FreeBSD.org  Thu Aug 19 12:50:07 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 19 Aug 2021 12:50:07 +0000 (UTC)
Subject: [master] 1dbc68841 Change VRT_SetHdr() to take a `const char*` and
 VCL_STRANDS, either or both of which are allowed to be NULL.
Message-ID: <20210819125007.F392BA5F6A@lists.varnish-cache.org>


commit 1dbc688417c28789a8e1674e3dfce67e7ce6deab
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Aug 19 12:44:13 2021 +0000

    Change VRT_SetHdr() to take a `const char*` and VCL_STRANDS,
    either or both of which are allowed to be NULL.
    
    (Part of STRING_LIST eradication)

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 17e406edf..2cf320ac7 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -704,28 +704,55 @@ VRT_UnsetHdr(VRT_CTX , VCL_HEADER hs)
 }
 
 VCL_VOID
-VRT_SetHdr(VRT_CTX , VCL_HEADER hs, const char *p, ...)
+VRT_SetHdr(VRT_CTX , VCL_HEADER hs, const char *pfx, VCL_STRANDS s)
 {
 	VCL_HTTP hp;
-	va_list ap;
-	const char *b;
+	unsigned u, l;
+	char *p, *b;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	AN(hs);
 	AN(hs->what);
 	hp = VRT_selecthttp(ctx, hs->where);
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
-	va_start(ap, p);
-	b = VRT_String(hp->ws, hs->what + 1, p, ap);
-	va_end(ap);
-	if (b == NULL) {
+
+	u = WS_ReserveAll(hp->ws);
+	l = hs->what[0] + 1;
+	if (pfx != NULL)
+		l += strlen(pfx);
+	if (u <= l) {
+		WS_Release(hp->ws, 0);
+		WS_MarkOverflow(hp->ws);
 		VSLb(ctx->vsl, SLT_LostHeader, "%s", hs->what + 1);
 		return;
 	}
-	if (FEATURE(FEATURE_VALIDATE_HEADERS) && ! validhdr(b)) {
+	b = WS_Reservation(hp->ws);
+	if (s != NULL) {
+		p = VRT_Strands(b + l, u - l, s);
+		if (p == NULL) {
+			WS_Release(hp->ws, 0);
+			WS_MarkOverflow(hp->ws);
+			VSLb(ctx->vsl, SLT_LostHeader, "%s", hs->what + 1);
+			return;
+		}
+	} else {
+		b[l] = '\0';
+	}
+	p = b;
+	memcpy(p, hs->what + 1, hs->what[0]);
+	p += hs->what[0];
+	*p++ = ' ';
+	if (pfx != NULL) {
+		l = strlen(pfx);
+		memcpy(p, pfx, l);
+		p += l;
+	}
+	if (FEATURE(FEATURE_VALIDATE_HEADERS) && !validhdr(b)) {
+		WS_Release(hp->ws, 0);
 		VRT_fail(ctx, "Bad header %s", b);
 		return;
 	}
+	WS_ReleaseP(hp->ws, strchr(p, '\0') + 1);
 	http_Unset(hp, hs->what);
 	http_SetHeader(hp, b);
 }

From phk at FreeBSD.org  Thu Aug 19 13:02:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 19 Aug 2021 13:02:05 +0000 (UTC)
Subject: [master] 13ebfef8f The rest of the previous commit,
 sorry for the botch.
Message-ID: <20210819130205.A1C4AA66AC@lists.varnish-cache.org>


commit 13ebfef8f730aff9f54211257baa3ddb79632193
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Aug 19 13:01:29 2021 +0000

    The rest of the previous commit, sorry for the botch.

diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc
index 49c9cc234..d8a9960a1 100644
--- a/bin/varnishtest/tests/m00000.vtc
+++ b/bin/varnishtest/tests/m00000.vtc
@@ -234,10 +234,12 @@ varnish v1 -vcl {
 		};
 
 		#define SETHDR(type)					\
-			VRT_SetHdr(ctx,					\
+			VRT_SetHdr(					\
+			    ctx,					\
 			    &VGC_HDR_RESP_ ## type ## _2d_sizeof,	\
 			    VRT_INT_string(ctx, sizeof(struct type)),	\
-			    vrt_magic_string_end)
+			    0						\
+			)
 	}C
 
 	sub vcl_recv {
diff --git a/bin/varnishtest/tests/r01406.vtc b/bin/varnishtest/tests/r01406.vtc
index ee4e5e498..d4c3de428 100644
--- a/bin/varnishtest/tests/r01406.vtc
+++ b/bin/varnishtest/tests/r01406.vtc
@@ -15,7 +15,12 @@ varnish v1 -arg "-p vcc_allow_inline_c=true" -vcl+backend {
 
 	sub vcl_recv {
 		C{
-			VRT_SetHdr(ctx, &VGC_HDR_REQ_foo, 0, vrt_magic_string_end );
+			VRT_SetHdr(
+			    ctx,
+			    &VGC_HDR_REQ_foo,
+			    0,
+			    &(struct strands){.n = 0}
+			);
 		}C
 	}
 
diff --git a/include/vrt.h b/include/vrt.h
index 8c6678935..b0cae3b9b 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -54,6 +54,7 @@
  * binary/load-time compatible, increment MAJOR version
  *
  * NEXT (2021-09-15)
+ *	VRT_SetHdr() changed to take `const char*` & `STRANDS` arguments.
  *	VRT_UnsetHdr() added
  *	vrt_magic_string_unset removed (use VRT_UnsetHdr() instead)
  *	VNUMpfx() removed, SF_Parse_{Integer|Decimal|Number} added
@@ -596,7 +597,7 @@ VCL_VOID VRT_hit_for_pass(VRT_CTX, VCL_DURATION);
 
 VCL_BOOL VRT_ValidHdr(VRT_CTX, VCL_STRANDS);
 VCL_VOID VRT_UnsetHdr(VRT_CTX, VCL_HEADER);
-VCL_VOID VRT_SetHdr(VRT_CTX, VCL_HEADER, const char *, ...);
+VCL_VOID VRT_SetHdr(VRT_CTX, VCL_HEADER, const char *pfx, VCL_STRANDS);
 VCL_VOID VRT_handling(VRT_CTX, unsigned hand);
 unsigned VRT_handled(VRT_CTX);
 VCL_VOID VRT_fail(VRT_CTX, const char *fmt, ...) v_printflike_(2,3);
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 70e805504..25e2f7a8b 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -119,8 +119,8 @@ static const struct assign {
 	{ DURATION,	0,		DURATION },
 	{ STRING,	T_INCR,		STRING_LIST, "\v,\n" },
 	{ STRING,	'=',		STRING_LIST },
-	{ HEADER,	T_INCR,		STRING_LIST, "VRT_GetHdr(ctx, \v),\n" },
-	{ HEADER,	'=',		STRING_LIST },
+	{ HEADER,	T_INCR,		STRANDS, "VRT_GetHdr(ctx, \v),\n" },
+	{ HEADER,	'=',		STRANDS, "NULL,\n" },
 	{ BODY,		'=',		STRING_LIST, "LBODY_SET,\n" },
 	{ BODY,		T_INCR,		STRING_LIST, "LBODY_ADD,\n" },
 	{ VOID,		'=',		VOID }

From dridi.boukelmoune at gmail.com  Thu Aug 19 17:09:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 19 Aug 2021 17:09:05 +0000 (UTC)
Subject: [master] 63554eb91 param: Fix the description of thread_pool_stack
Message-ID: <20210819170905.799FBAD344@lists.varnish-cache.org>


commit 63554eb91b8446a3901a7d0431559e5df6295779
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Aug 19 17:02:36 2021 +0200

    param: Fix the description of thread_pool_stack
    
    It is the minimum value that matches the dynamic description, the
    default value gets adjusted if the one we use is below on a given
    system.

diff --git a/include/tbl/params.h b/include/tbl/params.h
index 9b30b37d5..edae76b28 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -1536,9 +1536,9 @@ PARAM_THREAD(
 	" increments is recommended until stack overflows"
 	" cease to occur.",
 	/* flags */	DELAYED_EFFECT,
-	/* dyn_min_reason */	NULL,
+	/* dyn_min_reason */	"sysconf(_SC_THREAD_STACK_MIN)",
 	/* dyn_max_reason */	NULL,
-	/* dyn_def_reason */	"sysconf(_SC_THREAD_STACK_MIN)"
+	/* dyn_def_reason */	NULL
 )
 
 #if defined(PARAM_ALL)

From dridi.boukelmoune at gmail.com  Thu Aug 19 17:09:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 19 Aug 2021 17:09:05 +0000 (UTC)
Subject: [master] 273513695 param: Increase the thread_pool_stack defaults
Message-ID: <20210819170905.966C5AD347@lists.varnish-cache.org>


commit 273513695fc212630cd6e4e7e5dde3f4ef77c839
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Aug 19 17:09:25 2021 +0200

    param: Increase the thread_pool_stack defaults
    
    And use the 64bit default as a dynamic default to have it show up in the
    varnishd manual. Since we explicitly document the 32bit default in the
    same manual it shouldn't be controversial.
    
    The reason to increase them is the PCRE2 jit compiler that produces
    stack-hungry code.
    
    Refs #3671

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 34e23d466..cdeaa3d18 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -666,7 +666,7 @@ MCF_InitParams(struct cli *cli)
 
 	MCF_TcpParams();
 
-	def = 56 * 1024;
+	def = 80 * 1024;
 
 	if (sizeof(void *) < 8) {		/*lint !e506 !e774  */
 		/*
@@ -682,14 +682,14 @@ MCF_InitParams(struct cli *cli)
 		MCF_ParamConf(MCF_DEFAULT, "gzip_buffer", "4k");
 		MCF_ParamConf(MCF_DEFAULT, "vsl_buffer", "4k");
 		MCF_ParamConf(MCF_MAXIMUM, "vsl_space", "1G");
-		def = 52 * 1024;
+		def = 64 * 1024;
 	}
 
 	low = sysconf(_SC_THREAD_STACK_MIN);
 	MCF_ParamConf(MCF_MINIMUM, "thread_pool_stack", "%jdb", (intmax_t)low);
 
 #if defined(__SANITIZER) || __has_feature(address_sanitizer) || defined(GCOVING)
-	def = 92 * 1024;
+	def = 192 * 1024;
 #endif
 
 	if (def < low)
diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index e87f48a14..301b8bd66 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -528,7 +528,7 @@ space:
 * gzip_buffer: 4k
 * vsl_buffer: 4k
 * vsl_space: 1G (maximum)
-* thread_pool_stack: 52k
+* thread_pool_stack: 64k
 
 .. _List of Parameters:
 
diff --git a/include/tbl/params.h b/include/tbl/params.h
index edae76b28..26345d6c3 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -1538,7 +1538,7 @@ PARAM_THREAD(
 	/* flags */	DELAYED_EFFECT,
 	/* dyn_min_reason */	"sysconf(_SC_THREAD_STACK_MIN)",
 	/* dyn_max_reason */	NULL,
-	/* dyn_def_reason */	NULL
+	/* dyn_def_reason */	"80k"
 )
 
 #if defined(PARAM_ALL)

From dridi.boukelmoune at gmail.com  Thu Aug 19 17:09:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 19 Aug 2021 17:09:05 +0000 (UTC)
Subject: [master] 2d9fd0258 vre: Actually trigger the jit compilation
Message-ID: <20210819170905.D2BEFAD34C@lists.varnish-cache.org>


commit 2d9fd0258dbb7d3426fb0e60b3e52d6f703a697a
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Aug 19 17:15:52 2021 +0200

    vre: Actually trigger the jit compilation
    
    Spotted by Simon Vikstr?m.
    
    Closes #3671

diff --git a/bin/varnishtest/tests/r01576.vtc b/bin/varnishtest/tests/r01576.vtc
index 7cc73e089..e866fb145 100644
--- a/bin/varnishtest/tests/r01576.vtc
+++ b/bin/varnishtest/tests/r01576.vtc
@@ -41,7 +41,7 @@ client c1 {
 } -run
 
 logexpect l1 -v v1 {
-	expect * * VCL_Error "(match|depth|recursion) limit exceeded"
+	expect * * VCL_Error "(match|depth|recursion|JIT stack) limit"
 } -start
 
 # This should fail with default params and JIT/no-JIT
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index 1e62184f9..e2efdcbc8 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -108,7 +108,7 @@ VRE_compile(const char *pattern, unsigned options,
 	}
 #if USE_PCRE2_JIT
 	if (jit)
-		(void)pcre2_jit_compile(v->re, 0);
+		(void)pcre2_jit_compile(v->re, PCRE2_JIT_COMPLETE);
 #else
 	(void)jit;
 #endif

From phk at FreeBSD.org  Fri Aug 20 08:22:06 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 20 Aug 2021 08:22:06 +0000 (UTC)
Subject: [master] 01bea6f62 Change VRT_l_{beresp|resp}_body to take STRANDS
 argument
Message-ID: <20210820082206.C0B82A8395@lists.varnish-cache.org>


commit 01bea6f620f75a39ee401b11568aabe324054f1c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Aug 20 08:20:24 2021 +0000

    Change VRT_l_{beresp|resp}_body to take STRANDS argument

diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 186de5092..35ef4ee29 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -950,26 +950,18 @@ VRT_r_resp_do_esi(VRT_CTX)
 
 #define VRT_BODY_L(which)					\
 VCL_VOID							\
-VRT_l_##which##_body(VRT_CTX, enum lbody_e type,		\
-    const char *str, ...)					\
+VRT_l_##which##_body(VRT_CTX, enum lbody_e type, VCL_STRANDS s)	\
 {								\
-	va_list ap;						\
-	const char *p;						\
+	int n;							\
 	struct vsb *vsb;					\
 								\
 	CAST_OBJ_NOTNULL(vsb, ctx->specific, VSB_MAGIC);	\
 	assert(type == LBODY_SET || type == LBODY_ADD);		\
 	if (type == LBODY_SET)					\
 		VSB_clear(vsb);					\
-	va_start(ap, str);					\
-	p = str;						\
-	while (p != vrt_magic_string_end) {			\
-		if (p == NULL)					\
-			p = "(null)";				\
-		VSB_cat(vsb, p);				\
-		p = va_arg(ap, const char *);			\
-	}							\
-	va_end(ap);						\
+	for (n = 0; n < s->n; n++)				\
+		if (s->p[n] != NULL)				\
+			VSB_cat(vsb, s->p[n]);			\
 }
 
 VRT_BODY_L(beresp)
diff --git a/include/vrt.h b/include/vrt.h
index b0cae3b9b..7c7264c34 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -54,6 +54,7 @@
  * binary/load-time compatible, increment MAJOR version
  *
  * NEXT (2021-09-15)
+ *	VRT_l_{beresp|resp}_body() changed to takes STRANDS argument
  *	VRT_SetHdr() changed to take `const char*` & `STRANDS` arguments.
  *	VRT_UnsetHdr() added
  *	vrt_magic_string_unset removed (use VRT_UnsetHdr() instead)
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index c1117905a..58b1f7858 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -219,7 +219,8 @@ class vardef(object):
             if self.typ == "STRING":
                 s += ctyp.c + ", ...)"
             elif self.typ == "BODY":
-                s += "enum lbody_e, " + ctyp.c + ", ...)"
+                #s += "enum lbody_e, " + ctyp.c + ", VCL_STRANDS)"
+                s += "enum lbody_e, VCL_STRANDS)"
             else:
                 s += "VCL_" + self.typ + ")"
             varproto(s)
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 25e2f7a8b..fbd0cd004 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -121,8 +121,8 @@ static const struct assign {
 	{ STRING,	'=',		STRING_LIST },
 	{ HEADER,	T_INCR,		STRANDS, "VRT_GetHdr(ctx, \v),\n" },
 	{ HEADER,	'=',		STRANDS, "NULL,\n" },
-	{ BODY,		'=',		STRING_LIST, "LBODY_SET,\n" },
-	{ BODY,		T_INCR,		STRING_LIST, "LBODY_ADD,\n" },
+	{ BODY,		'=',		STRANDS, "LBODY_SET,\n" },
+	{ BODY,		T_INCR,		STRANDS, "LBODY_ADD,\n" },
 	{ VOID,		'=',		VOID }
 };
 

From dridi.boukelmoune at gmail.com  Fri Aug 20 11:46:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 20 Aug 2021 11:46:05 +0000 (UTC)
Subject: [master] 4e0af4904 build: Make the fake linker produce a target
Message-ID: <20210820114605.B5AB74065@lists.varnish-cache.org>


commit 4e0af4904de07ca51fdec7494bbc7060a2dd0743
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Aug 20 13:42:21 2021 +0200

    build: Make the fake linker produce a target
    
    Otherwise we always try to rebuild vrt_test when running make.

diff --git a/include/Makefile.am b/include/Makefile.am
index d6afb0438..d5c5925c5 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -166,6 +166,6 @@ vrt_test.c: vdef.h vrt.h
 
 vrt_test_BUILT_SOURCES = vrt_test.c
 vrt_test_CFLAGS = -c
-vrt_test_LINK = :
+vrt_test_LINK = cat >$@
 
 TESTS = vbm_test

From dridi.boukelmoune at gmail.com  Fri Aug 20 11:49:04 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 20 Aug 2021 11:49:04 +0000 (UTC)
Subject: [master] 211c69609 build: Fake link with echo instead of cat
Message-ID: <20210820114904.7356A4361@lists.varnish-cache.org>


commit 211c6960995aad74f420bc5e85e953a07b1bceb3
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Aug 20 13:47:51 2021 +0200

    build: Fake link with echo instead of cat
    
    There might be linker flags that cat would fail to find.

diff --git a/include/Makefile.am b/include/Makefile.am
index d5c5925c5..ecf95affd 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -166,6 +166,6 @@ vrt_test.c: vdef.h vrt.h
 
 vrt_test_BUILT_SOURCES = vrt_test.c
 vrt_test_CFLAGS = -c
-vrt_test_LINK = cat >$@
+vrt_test_LINK = echo >$@
 
 TESTS = vbm_test

From phk at FreeBSD.org  Fri Aug 20 13:52:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 20 Aug 2021 13:52:05 +0000 (UTC)
Subject: [master] 28902ab94 Convert VRT_l_* to "const char *,
 VCL_STRANDS" format.
Message-ID: <20210820135205.A35E27E3E@lists.varnish-cache.org>


commit 28902ab948080069d588767d0db244669fa7ee1f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Aug 20 13:50:20 2021 +0000

    Convert VRT_l_* to "const char *, VCL_STRANDS" format.

diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c
index 1baf344f5..cab61ea74 100644
--- a/bin/varnishd/cache/cache_vrt_filter.c
+++ b/bin/varnishd/cache/cache_vrt_filter.c
@@ -406,15 +406,13 @@ resp_Get_Filter_List(struct req *req)
 	}								\
 									\
 	VCL_VOID							\
-	VRT_l_##vcl##_filters(VRT_CTX, const char *str, ...)		\
+	VRT_l_##vcl##_filters(VRT_CTX, const char *str, VCL_STRANDS s)	\
 	{								\
-		va_list ap;						\
 		const char *b;						\
 									\
+		(void)str;						\
 		CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);			\
-		va_start(ap, str);					\
-		b = VRT_String(ctx->in->ws, NULL, str, ap);		\
-		va_end(ap);						\
+		b = VRT_StrandsWS(ctx->in->ws, str, s);			\
 		if (b == NULL)						\
 			WS_MarkOverflow(ctx->in->ws);			\
 		else							\
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 35ef4ee29..2b05aac01 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -50,14 +50,14 @@ static char vrt_hostname[255] = "";
  */
 
 static void
-vrt_do_string(VRT_CTX, struct http *hp, int fld,
-    const char *err, const char *p, va_list ap)
+vrt_do_strands(VRT_CTX, struct http *hp, int fld,
+    const char *err, const char *str, VCL_STRANDS s)
 {
 	const char *b;
 
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 
-	b = VRT_String(hp->ws, NULL, p, ap);
+	b = VRT_StrandsWS(hp->ws, str, s);
 	if (b == NULL) {
 		VRT_fail(ctx, "Workspace overflow (%s)", err);
 		WS_MarkOverflow(hp->ws);
@@ -72,14 +72,11 @@ vrt_do_string(VRT_CTX, struct http *hp, int fld,
 
 #define VRT_HDR_L(obj, hdr, fld)					\
 VCL_VOID								\
-VRT_l_##obj##_##hdr(VRT_CTX, const char *p, ...)			\
+VRT_l_##obj##_##hdr(VRT_CTX, const char *str, VCL_STRANDS s)		\
 {									\
-	va_list ap;							\
 									\
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);				\
-	va_start(ap, p);						\
-	vrt_do_string(ctx, ctx->http_##obj, fld, #obj "." #hdr, p, ap);	\
-	va_end(ap);							\
+	vrt_do_strands(ctx, ctx->http_##obj, fld, #obj "." #hdr, str, s);	\
 }
 
 #define VRT_HDR_R(obj, hdr, fld)					\
@@ -316,16 +313,13 @@ VRT_r_client_identity(VRT_CTX)
 }
 
 VCL_VOID
-VRT_l_client_identity(VRT_CTX, const char *str, ...)
+VRT_l_client_identity(VRT_CTX, const char *str, VCL_STRANDS s)
 {
-	va_list ap;
 	const char *b;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
-	va_start(ap, str);
-	b = VRT_String(ctx->req->http->ws, NULL, str, ap);
-	va_end(ap);
+	b = VRT_StrandsWS(ctx->req->http->ws, str, s);
 	if (b == NULL) {
 		VSLb(ctx->vsl, SLT_LostHeader, "client.identity");
 		WS_MarkOverflow(ctx->req->http->ws);
@@ -441,24 +435,18 @@ VRT_r_beresp_storage_hint(VRT_CTX)
 }
 
 VCL_VOID
-VRT_l_beresp_storage_hint(VRT_CTX, const char *str, ...)
+VRT_l_beresp_storage_hint(VRT_CTX, const char *str, VCL_STRANDS s)
 {
 	const char *p;
-	va_list ap;
-	uintptr_t sn;
 	VCL_STEVEDORE stv;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
 
-	sn = WS_Snapshot(ctx->ws);
-	va_start(ap, str);
-	p = VRT_String(ctx->ws, NULL, str, ap);
-	va_end(ap);
+	p = VRT_StrandsWS(ctx->ws, str, s);
 
 	if (p == NULL) {
 		VSLb(ctx->vsl, SLT_LostHeader, "storage_hint");
-		WS_Reset(ctx->ws, sn);
 		WS_MarkOverflow(ctx->ws);
 		return;
 	}
@@ -466,8 +454,6 @@ VRT_l_beresp_storage_hint(VRT_CTX, const char *str, ...)
 	stv = VRT_stevedore(p);
 	if (stv != NULL)
 		ctx->bo->storage = stv;
-
-	WS_Reset(ctx->ws, sn);
 }
 
 /*--------------------------------------------------------------------*/
@@ -950,7 +936,8 @@ VRT_r_resp_do_esi(VRT_CTX)
 
 #define VRT_BODY_L(which)					\
 VCL_VOID							\
-VRT_l_##which##_body(VRT_CTX, enum lbody_e type, VCL_STRANDS s)	\
+VRT_l_##which##_body(VRT_CTX, enum lbody_e type,		\
+    const char *str, VCL_STRANDS s)				\
 {								\
 	int n;							\
 	struct vsb *vsb;					\
@@ -959,7 +946,9 @@ VRT_l_##which##_body(VRT_CTX, enum lbody_e type, VCL_STRANDS s)	\
 	assert(type == LBODY_SET || type == LBODY_ADD);		\
 	if (type == LBODY_SET)					\
 		VSB_clear(vsb);					\
-	for (n = 0; n < s->n; n++)				\
+	if (str != NULL)					\
+		VSB_cat(vsb, str);				\
+	for (n = 0; s != NULL && n < s->n; n++)			\
 		if (s->p[n] != NULL)				\
 			VSB_cat(vsb, s->p[n]);			\
 }
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 58b1f7858..f5fd13a26 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -217,10 +217,9 @@ class vardef(object):
             fo.write('\tsym->lname = "VRT_l_%s(ctx, ";\n' % cnam)
             s = "void VRT_l_%s(VRT_CTX, " % cnam
             if self.typ == "STRING":
-                s += ctyp.c + ", ...)"
+                s += ctyp.c + ", VCL_STRANDS)"
             elif self.typ == "BODY":
-                #s += "enum lbody_e, " + ctyp.c + ", VCL_STRANDS)"
-                s += "enum lbody_e, VCL_STRANDS)"
+                s += "enum lbody_e, " + ctyp.c + ", VCL_STRANDS)"
             else:
                 s += "VCL_" + self.typ + ")"
             varproto(s)
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index fbd0cd004..7c7a57714 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -117,12 +117,12 @@ static const struct assign {
 	{ DURATION,	T_DIV,		REAL, "\v / " },
 	{ DURATION,	'=',		DURATION },
 	{ DURATION,	0,		DURATION },
-	{ STRING,	T_INCR,		STRING_LIST, "\v,\n" },
-	{ STRING,	'=',		STRING_LIST },
+	{ STRING,	T_INCR,		STRANDS, "\v,\n" },
+	{ STRING,	'=',		STRANDS, "0,\n" },
 	{ HEADER,	T_INCR,		STRANDS, "VRT_GetHdr(ctx, \v),\n" },
-	{ HEADER,	'=',		STRANDS, "NULL,\n" },
-	{ BODY,		'=',		STRANDS, "LBODY_SET,\n" },
-	{ BODY,		T_INCR,		STRANDS, "LBODY_ADD,\n" },
+	{ HEADER,	'=',		STRANDS, "0,\n" },
+	{ BODY,		'=',		STRANDS, "LBODY_SET, 0,\n" },
+	{ BODY,		T_INCR,		STRANDS, "LBODY_ADD, 0,\n" },
 	{ VOID,		'=',		VOID }
 };
 

From dridi.boukelmoune at gmail.com  Fri Aug 20 14:32:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 20 Aug 2021 14:32:06 +0000 (UTC)
Subject: [6.0] 5dbca6f17 vcc: Insert the built-in source last
Message-ID: <20210820143206.8CA3A612C7@lists.varnish-cache.org>


commit 5dbca6f173bee9be9c0b5bd5a116a7e1bfa443e7
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Aug 4 17:32:38 2021 +0200

    vcc: Insert the built-in source last
    
    In the output of vcl.show -v, it means that the least useful file (in the
    sense that it is common to every single vcl.load) is now printed last.
    
    This change originates from a larger and more intrusive refactoring.
    
    It also helps get rid of spurious Wstring-contatenation warnings from
    clang 12 in the test suite, instead of disabling it altogether.
    
    Refs c8174af68956206115972e72e75ceddbd758116e

diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 50288c6ef..81a088c2e 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -504,7 +504,7 @@ static void
 vcc_resolve_includes(struct vcc *tl)
 {
 	struct token *t, *t1, *t2;
-	struct source *sp;
+	struct source *sp, *builtin_sp;
 	struct vsb *vsb;
 	const char *p;
 
@@ -559,7 +559,9 @@ vcc_resolve_includes(struct vcc *tl)
 			vcc_ErrWhere(tl, t1);
 			return;
 		}
-		VTAILQ_INSERT_TAIL(&tl->sources, sp, list);
+		builtin_sp = VTAILQ_LAST(&tl->sources, sourcehead);
+		AN(builtin_sp);
+		VTAILQ_INSERT_BEFORE(builtin_sp, sp, list);
 		sp->idx = tl->nsources++;
 		tl->t = t2;
 		vcc_Lexer(tl, sp);
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 52e0dd282..5d0186294 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -208,6 +208,7 @@ struct inifin {
 };
 
 VTAILQ_HEAD(inifinhead, inifin);
+VTAILQ_HEAD(sourcehead, source);
 
 struct vcc {
 	unsigned		magic;
@@ -228,7 +229,7 @@ struct vcc {
 
 	/* Instance section */
 	struct tokenhead	tokens;
-	VTAILQ_HEAD(, source)	sources;
+	struct sourcehead	sources;
 	unsigned		nsources;
 	struct source		*src;
 	struct token		*t;

From dridi.boukelmoune at gmail.com  Fri Aug 20 14:51:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 20 Aug 2021 14:51:05 +0000 (UTC)
Subject: [master] 91abe0a53 sess: Polish HTC_RxPipeline()
Message-ID: <20210820145105.BC61A61D8F@lists.varnish-cache.org>


commit 91abe0a534d9fd7c86faf2b19b2e04cd1d334fbf
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Jul 8 08:45:37 2021 +0200

    sess: Polish HTC_RxPipeline()
    
    All call sites pass a non-null char pointer.

diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 51d10b417..f8215dc13 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -244,16 +244,16 @@ HTC_RxInit(struct http_conn *htc, struct ws *ws)
 }
 
 void
-HTC_RxPipeline(struct http_conn *htc, void *p)
+HTC_RxPipeline(struct http_conn *htc, char *p)
 {
 
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
-	if (p == NULL || (char*)p == htc->rxbuf_e) {
+	assert(p >= htc->rxbuf_b);
+	assert(p <= htc->rxbuf_e);
+	if (p == htc->rxbuf_e) {
 		htc->pipeline_b = NULL;
 		htc->pipeline_e = NULL;
 	} else {
-		assert((char*)p >= htc->rxbuf_b);
-		assert((char*)p < htc->rxbuf_e);
 		htc->pipeline_b = p;
 		htc->pipeline_e = htc->rxbuf_e;
 	}
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index b150ea5e0..f9236cf0e 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -432,7 +432,7 @@ void SES_Rel(struct sess *sp);
 
 const char * HTC_Status(enum htc_status_e);
 void HTC_RxInit(struct http_conn *htc, struct ws *ws);
-void HTC_RxPipeline(struct http_conn *htc, void *);
+void HTC_RxPipeline(struct http_conn *htc, char *);
 enum htc_status_e HTC_RxStuff(struct http_conn *, htc_complete_f *,
     vtim_real *t1, vtim_real *t2, vtim_real ti, vtim_real tn, vtim_dur td,
     int maxbytes);

From dridi.boukelmoune at gmail.com  Fri Aug 20 16:28:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 20 Aug 2021 16:28:06 +0000 (UTC)
Subject: [master] 7456b1673 h2: Log the htc status name for easier debugging
Message-ID: <20210820162806.2958665865@lists.varnish-cache.org>


commit 7456b1673a9600779f92c3184c6a50036cc433ac
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Aug 20 14:23:23 2021 +0200

    h2: Log the htc status name for easier debugging

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 3597ec169..05944f49e 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -1115,6 +1115,7 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2)
 	enum htc_status_e hs;
 	h2_frame h2f;
 	h2_error h2e;
+	const char *s = NULL;
 	char b[8];
 
 	ASSERT_RXTHR(h2);
@@ -1134,8 +1135,12 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2)
 		/* FALLTHROUGH */
 	default:
 		/* XXX: HTC_S_OVERFLOW / FRAME_SIZE_ERROR handling */
+#define HTC_STATUS(e, n, d, l)		\
+		if (hs == HTC_S_ ## e)	\
+			s = #e;
+#include "tbl/htc.h"
 		Lck_Lock(&h2->sess->mtx);
-		VSLb(h2->vsl, SLT_Debug, "H2: No frame (hs=%d)", hs);
+		VSLb(h2->vsl, SLT_Debug, "H2: No frame (hs=%s)", s);
 		h2->error = H2CE_NO_ERROR;
 		Lck_Unlock(&h2->sess->mtx);
 		return (0);

From dridi.boukelmoune at gmail.com  Fri Aug 20 16:28:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 20 Aug 2021 16:28:06 +0000 (UTC)
Subject: [master] 67fb22653 vtc_sess: Simplify the main for loop
Message-ID: <20210820162806.3D9EA65868@lists.varnish-cache.org>


commit 67fb226532a7679b7717fd8ad121c2d44925b86f
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Aug 20 18:23:02 2021 +0200

    vtc_sess: Simplify the main for loop
    
    The nested while loop made more sense when the surroundings were more
    complicated.
    
    Better diff with the --ignore-all-space option.

diff --git a/bin/varnishtest/vtc_sess.c b/bin/varnishtest/vtc_sess.c
index d9766bf28..d28eb9eda 100644
--- a/bin/varnishtest/vtc_sess.c
+++ b/bin/varnishtest/vtc_sess.c
@@ -120,7 +120,7 @@ sess_thread(void *priv)
 	struct vtclog *vl;
 	struct vtc_sess *vsp;
 	struct thread_arg ta, *tap;
-	int i, fd;
+	int i, fd = -1;
 
 	CAST_OBJ_NOTNULL(tap, priv, THREAD_ARG_MAGIC);
 	ta = *tap;
@@ -135,15 +135,15 @@ sess_thread(void *priv)
 	vtc_log(vl, 2, "Started on %s (%u iterations%s)", ta.listen_addr,
 		vsp->repeat, vsp->keepalive ? " using keepalive" : "");
 	for (i = 0; i < vsp->repeat; i++) {
-		fd = ta.conn_f(ta.priv, vl);
+		if (fd < 0)
+			fd = ta.conn_f(ta.priv, vl);
+		fd = sess_process(vl, ta.vsp, ta.spec, fd,
+		    ta.asocket, ta.listen_addr);
 		if (! vsp->keepalive)
-			fd = sess_process(vl, ta.vsp, ta.spec, fd, ta.asocket, ta.listen_addr);
-		else
-			while (fd >= 0 && i++ < vsp->repeat)
-				fd = sess_process(vl, ta.vsp, ta.spec, fd,
-				    ta.asocket, ta.listen_addr);
-		ta.disc_f(ta.priv, vl, &fd);
+			ta.disc_f(ta.priv, vl, &fd);
 	}
+	if (vsp->keepalive)
+		ta.disc_f(ta.priv, vl, &fd);
 	vtc_log(vl, 2, "Ending");
 	pthread_cleanup_pop(0);
 	vtc_logclose(vl);

From phk at FreeBSD.org  Fri Aug 20 19:50:06 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 20 Aug 2021 19:50:06 +0000 (UTC)
Subject: [master] 656aa24b5 Rename VRT_CollectStrands() to VRT_STRANDS_string()
Message-ID: <20210820195006.14C2895D88@lists.varnish-cache.org>


commit 656aa24b55c6c749f00677f4d1fab5e64feff8b2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Aug 20 19:32:04 2021 +0000

    Rename VRT_CollectStrands() to VRT_STRANDS_string()

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 2cf320ac7..5894a95f9 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -593,7 +593,7 @@ VRT_StrandsWS(struct ws *ws, const char *h, VCL_STRANDS s)
  */
 
 VCL_STRING
-VRT_CollectStrands(VRT_CTX, VCL_STRANDS s)
+VRT_STRANDS_string(VRT_CTX, VCL_STRANDS s)
 {
 	const char *b;
 
diff --git a/include/vrt.h b/include/vrt.h
index 7c7264c34..a9eb450c2 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -54,6 +54,7 @@
  * binary/load-time compatible, increment MAJOR version
  *
  * NEXT (2021-09-15)
+ *	VRT_CollectStrands() renamed to VRT_STRANDS_string()
  *	VRT_l_{beresp|resp}_body() changed to takes STRANDS argument
  *	VRT_SetHdr() changed to take `const char*` & `STRANDS` arguments.
  *	VRT_UnsetHdr() added
@@ -454,7 +455,6 @@ VCL_BOOL VRT_Strands2Bool(VCL_STRANDS);
 uint32_t VRT_HashStrands32(VCL_STRANDS);
 char *VRT_Strands(char *, size_t, VCL_STRANDS);
 VCL_STRING VRT_StrandsWS(struct ws *, const char *, VCL_STRANDS);
-VCL_STRING VRT_CollectStrands(VRT_CTX, VCL_STRANDS);
 VCL_STRING VRT_UpperLowerStrands(VRT_CTX, VCL_STRANDS s, int up);
 
 /* VCL_SUB */
@@ -471,6 +471,7 @@ VCL_STRING VRT_INT_string(VRT_CTX, VCL_INT);
 VCL_STRING VRT_IP_string(VRT_CTX, VCL_IP);
 VCL_STRING VRT_REAL_string(VRT_CTX, VCL_REAL);
 VCL_STRING VRT_STEVEDORE_string(VCL_STEVEDORE);
+VCL_STRING VRT_STRANDS_string(VRT_CTX, VCL_STRANDS);
 VCL_STRING VRT_TIME_string(VRT_CTX, VCL_TIME);
 
 /* historical */
diff --git a/lib/libvcc/vcc_types.c b/lib/libvcc/vcc_types.c
index 7557e520e..0a83f0976 100644
--- a/lib/libvcc/vcc_types.c
+++ b/lib/libvcc/vcc_types.c
@@ -184,7 +184,7 @@ const struct type STRANDS[1] = {{
 	.magic =		TYPE_MAGIC,
 	.name =			"STRANDS",
 	.stringform =		1,
-	.tostring =		"VRT_CollectStrands(ctx,\v+\n\v1\v-\n)",
+	.tostring =		"VRT_STRANDS_string(ctx,\v+\n\v1\v-\n)",
 }};
 
 static const struct vcc_method strings_methods[] = {
diff --git a/vmod/vmod_blob.c b/vmod/vmod_blob.c
index 2f97979a2..19d2f38aa 100644
--- a/vmod/vmod_blob.c
+++ b/vmod/vmod_blob.c
@@ -470,7 +470,7 @@ vmod_transcode(VRT_CTX, VCL_ENUM decs, VCL_ENUM encs, VCL_ENUM case_s,
 		 * overflow. If there is only one string already in the
 		 * workspace, then it is re-used.
 		 */
-		return (VRT_CollectStrands(ctx, strings));
+		return (VRT_STRANDS_string(ctx, strings));
 
 	r = encode(ctx, enc, kase, &b);
 	return (r);
diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c
index 190773397..e467e4146 100644
--- a/vmod/vmod_debug.c
+++ b/vmod/vmod_debug.c
@@ -779,7 +779,7 @@ xyzzy_collect(VRT_CTX, VCL_STRANDS s)
 	VCL_STRING r;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	r = VRT_CollectStrands(ctx, s);
+	r = VRT_STRANDS_string(ctx, s);
 	if (r != NULL && *r != '\0')
 		AN(WS_Allocated(ctx->ws, r, strlen(r) + 1));
 	return (r);

From phk at FreeBSD.org  Fri Aug 20 19:50:06 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 20 Aug 2021 19:50:06 +0000 (UTC)
Subject: [master] 611baec6f STRING_LIST is no more.
Message-ID: <20210820195006.2D44195D8C@lists.varnish-cache.org>


commit 611baec6f29681bc1cc3836e635dfa14cca3d018
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Aug 20 19:48:27 2021 +0000

    STRING_LIST is no more.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index e65c76ea7..8b00b4fd4 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -748,15 +748,6 @@ VSLb_ts_busyobj(struct busyobj *bo, const char *event, vtim_real now)
 /* cache_vcl.c */
 const char *VCL_Name(const struct vcl *);
 
-/* cache_vrt.c */
-/*
- * These prototypes go here, because we do not want to pollute vrt.h
- * with va_list.  VCC never generates direct calls to them.
- * XXX: We should deprecate these (ref: STRANDS)
- */
-const char *VRT_String(struct ws *ws, const char *h, const char *p, va_list ap);
-char *VRT_StringList(char *d, unsigned dl, const char *p, va_list ap);
-
 /* cache_wrk.c */
 
 typedef void *bgthread_t(struct worker *, void *priv);
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 5894a95f9..fa62c8957 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -54,7 +54,6 @@
 #include "common/vsmw.h"
 #include "proxy/cache_proxy.h"
 
-const void * const vrt_magic_string_end = &vrt_magic_string_end;
 const struct strands *vrt_null_strands = &(struct strands){
 	.n = 0,
 	.p = (const char *[1]){NULL}
@@ -395,126 +394,6 @@ VRT_HashStrands32(VCL_STRANDS s)
 	return (vle32dec(sha256 + VSHA256_LEN - 4));
 }
 
-/*--------------------------------------------------------------------
- * Collapse a STRING_LIST in the space provided, or return NULL
- */
-
-char *
-VRT_StringList(char *d, unsigned dl, const char *p, va_list ap)
-{
-	char *b, *e;
-	unsigned x;
-
-	b = d;
-	e = b + dl;
-	while (p != vrt_magic_string_end && b < e) {
-		if (p != NULL && *p != '\0') {
-			x = strlen(p);
-			if (b + x < e)
-				memcpy(b, p, x);
-			b += x;
-		}
-		p = va_arg(ap, const char *);
-	}
-	if (b >= e)
-		return (NULL);
-	*b++ = '\0';
-	return (b);
-}
-
-/*--------------------------------------------------------------------
- * Copy and merge a STRING_LIST into a workspace.
- */
-
-const char *
-VRT_String(struct ws *ws, const char *h, const char *p, va_list ap)
-{
-	char *b, *e;
-	const char *q;
-	unsigned u, x;
-	va_list aq;
-
-	u = WS_ReserveAll(ws);
-	e = b = WS_Reservation(ws);
-	e += u;
-
-	va_copy(aq, ap);
-	do
-		q = va_arg(aq, const char *);
-	while (q == NULL || (q != vrt_magic_string_end && *q == '\0'));
-
-	if (h != NULL && p == NULL && q == vrt_magic_string_end &&
-	    WS_Allocated(ws, h, -1)) {
-		va_end(aq);
-		WS_Release(ws, 0);
-		return (h);
-	}
-
-	if (h == NULL && p != NULL && q == vrt_magic_string_end &&
-	    WS_Allocated(ws, p, -1)) {
-		va_end(aq);
-		WS_Release(ws, 0);
-		return (p);
-	}
-
-	if (h == NULL && p == NULL) {
-		if (q == vrt_magic_string_end) {
-			va_end(aq);
-			WS_Release(ws, 0);
-			return ("");
-		}
-		do
-			p = va_arg(aq, const char *);
-		while (p == NULL || (p != vrt_magic_string_end && *p == '\0'));
-		if (p == vrt_magic_string_end && WS_Allocated(ws, q, -1)) {
-			va_end(aq);
-			WS_Release(ws, 0);
-			return (q);
-		}
-		p = NULL;
-		va_end(aq);
-	}
-
-	if (h != NULL) {
-		x = strlen(h);
-		if (b + x < e)
-			memcpy(b, h, x);
-		b += x;
-		if (b < e)
-			*b = ' ';
-		b++;
-	}
-	b = VRT_StringList(b, e > b ? e - b : 0, p, ap);
-	if (b == NULL || b == e) {
-		WS_MarkOverflow(ws);
-		WS_Release(ws, 0);
-		return (NULL);
-	}
-	e = b;
-	b = WS_Reservation(ws);
-	WS_Release(ws, e - b);
-	return (b);
-}
-
-/*--------------------------------------------------------------------
- * Copy and merge a STRING_LIST on the current workspace
- */
-
-VCL_STRING
-VRT_CollectString(VRT_CTX, const char *p, ...)
-{
-	va_list ap;
-	const char *b;
-
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	CHECK_OBJ_NOTNULL(ctx->ws, WS_MAGIC);
-	va_start(ap, p);
-	b = VRT_String(ctx->ws, NULL, p, ap);
-	va_end(ap);
-	if (b == NULL)
-		VRT_fail(ctx, "Workspace overflow");
-	return (b);
-}
 
 /*--------------------------------------------------------------------
  * Collapse STRANDS into the space provided, or return NULL
diff --git a/include/vrt.h b/include/vrt.h
index a9eb450c2..fd678e572 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -54,6 +54,8 @@
  * binary/load-time compatible, increment MAJOR version
  *
  * NEXT (2021-09-15)
+ *	VCL_STRINGLIST, vrt_magic_string_end removed
+ *	VRT_String(), VRT_StringList(), VRT_CollectString() removed
  *	VRT_CollectStrands() renamed to VRT_STRANDS_string()
  *	VRT_l_{beresp|resp}_body() changed to takes STRANDS argument
  *	VRT_SetHdr() changed to take `const char*` & `STRANDS` arguments.
@@ -714,5 +716,4 @@ void VRT_VCL_Allow_Discard(struct vclref **);
  */
 
 VCL_VOID VRT_synth_page(VRT_CTX, VCL_STRANDS);
-extern const void * const vrt_magic_string_end;
 int VRT_Stv(const char *nm);
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index f5fd13a26..a2b318ba8 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -327,7 +327,6 @@ class vcltype(object):
 
 
 vcltype("STRINGS", "void", True)
-vcltype("STRING_LIST", "void*", True)
 vcltype("SUB", "void*", True)
 
 fi = open(join(srcroot, "include/vrt.h"))
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 28a3f4402..c2e25c871 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -48,7 +48,7 @@ struct expr {
 	uint8_t		constant;
 #define EXPR_VAR	(1<<0)
 #define EXPR_CONST	(1<<1)
-#define EXPR_STR_CONST	(1<<2)		// Last STRING_LIST elem is "..."
+#define EXPR_STR_CONST	(1<<2)		// Last string elem is "..."
 	struct token	*t1, *t2;
 	struct symbol	*instance;
 	int		nstr;
@@ -176,13 +176,18 @@ vcc_expr_edit(struct vcc *tl, vcc_type_t fmt, const char *p, struct expr *e1,
 			e3 = (*p == 'S' ? e1 : e2);
 			AN(e3);
 			assert(e1->fmt == STRINGS);
-			if (e3->nstr > 1)
+			if (e3->nstr > 1) {
 				VSB_cat(e->vsb,
-				    "\nVRT_CollectString(ctx,\v+\n");
-			VSB_cat(e->vsb, VSB_data(e3->vsb));
-			if (e3->nstr > 1)
+				    "\nVRT_STRANDS_string(ctx,\v+\n");
+				VSB_printf(e->vsb,
+				    "&(struct strands){.n = %d, .p = "
+				    "(const char *[%d]){\n%s\n}}",
+				    e3->nstr, e3->nstr, VSB_data(e3->vsb));
 				VSB_cat(e->vsb,
-				    ",\nvrt_magic_string_end)\v-\n");
+				    "\v-\n)\n");
+			} else {
+				VSB_cat(e->vsb, VSB_data(e3->vsb));
+			}
 			break;
 		case 'T':
 		case 't':
@@ -1396,9 +1401,7 @@ vcc_expr0(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 	}
 
 	if ((*e)->fmt == STRINGS && fmt->stringform) {
-		if (fmt == STRING_LIST)
-			(*e)->fmt = STRING_LIST;
-		else if (fmt == STRING)
+		if (fmt == STRING)
 			*e = vcc_expr_edit(tl, STRING, "\vS", *e, NULL);
 		else if (fmt == STRANDS)
 			*e = vcc_expr_edit(tl, STRANDS, "\vT", (*e), NULL);
@@ -1406,10 +1409,6 @@ vcc_expr0(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 			WRONG("Unhandled stringform");
 	}
 
-	if ((*e)->fmt == STRING_LIST)
-		*e = vcc_expr_edit(tl, STRING_LIST,
-		    "\n\v1,\nvrt_magic_string_end", *e, NULL);
-
 	if (fmt == BOOL) {
 		vcc_expr_tobool(tl, e);
 		ERRCHK(tl);
diff --git a/lib/libvcc/vcc_types.c b/lib/libvcc/vcc_types.c
index 0a83f0976..829bb2c2e 100644
--- a/lib/libvcc/vcc_types.c
+++ b/lib/libvcc/vcc_types.c
@@ -202,12 +202,6 @@ const struct type STRINGS[1] = {{
 	.tostring =		"",
 }};
 
-const struct type STRING_LIST[1] = {{
-	.magic =		TYPE_MAGIC,
-	.name =			"STRING_LIST",
-	.stringform =		1,
-}};
-
 const struct type SUB[1] = {{
 	.magic =		TYPE_MAGIC,
 	.name =			"SUB",

From phk at FreeBSD.org  Fri Aug 20 20:32:10 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 20 Aug 2021 20:32:10 +0000 (UTC)
Subject: [master] 17ee20f22 Silence FlexeLint
Message-ID: <20210820203210.7A4C097521@lists.varnish-cache.org>


commit 17ee20f22af5f7b95a3e05cd1a5af7b298d0b5c3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Aug 20 20:31:21 2021 +0000

    Silence FlexeLint

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 05944f49e..b668523a7 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -1135,9 +1135,11 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2)
 		/* FALLTHROUGH */
 	default:
 		/* XXX: HTC_S_OVERFLOW / FRAME_SIZE_ERROR handling */
-#define HTC_STATUS(e, n, d, l)		\
-		if (hs == HTC_S_ ## e)	\
-			s = #e;
+#define HTC_STATUS(e, n, d, l)			\
+		do {				\
+			if (hs == HTC_S_ ## e)	\
+				s = #e;		\
+		} while (0);
 #include "tbl/htc.h"
 		Lck_Lock(&h2->sess->mtx);
 		VSLb(h2->vsl, SLT_Debug, "H2: No frame (hs=%s)", s);

From dridi.boukelmoune at gmail.com  Sat Aug 21 06:17:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Sat, 21 Aug 2021 06:17:06 +0000 (UTC)
Subject: [master] 9824ae5b2 doc: Remove STRING_LIST references
Message-ID: <20210821061706.4FFEEAC283@lists.varnish-cache.org>


commit 9824ae5b2bdbe53e8eefc753adfb00ac4367522c
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Sat Aug 21 08:14:36 2021 +0200

    doc: Remove STRING_LIST references

diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index 1e3a2b422..84283745f 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -273,8 +273,7 @@ VCL data types are targeted at the job, so for instance, we have data
 types like "DURATION" and "HEADER", but they all have some kind of C
 language representation.  Here is a description of them.
 
-All but the PRIV and STRING_LIST types have typedefs: VCL_INT, VCL_REAL,
-etc.
+All but the PRIV types have typedefs: VCL_INT, VCL_REAL, etc.
 
 Notice that most of the non-native (C pointer) types are ``const``,
 which, if returned by a vmod function/method, are assumed to be
@@ -411,46 +410,11 @@ STEVEDORE
 
 	A storage backend.
 
-STRING_LIST
-	C-type: ``const char *, ...``
-
-	`Notice: New vmod developments for 6.0 LTS and later must
-	use STRANDS instead of STRING_LIST, which is going away.`
-
-	A multi-component text-string.  We try very hard to avoid
-	doing text-processing in Varnish, and this is one way we
-	to avoid that, by not editing separate pieces of a string
-	together to one string, unless we have to.
-
-	Consider this contrived example::
-
-		set req.http.foo = std.toupper(req.http.foo + req.http.bar);
-
-	The usual way to do this, would be be to allocate memory for
-	the concatenated string, then pass that to ``toupper()`` which in
-	turn would return another freshly allocated string with the
-	modified result.  Remember: strings in VCL are ``const``, we
-	cannot just modify the string in place.
-
-	What we do instead, is declare that ``toupper()`` takes a "STRING_LIST"
-	as argument.  This makes the C function implementing ``toupper()``
-	a vararg function (see the prototype above) and responsible for
-	considering all the ``const char *`` arguments it finds, until the
-	magic marker "vrt_magic_string_end" is encountered.
-
-	Bear in mind that the individual strings in a STRING_LIST can be
-	NULL, as described under STRING, that is why we do not use NULL
-	as the terminator.
-
-	STRING_LIST must be the last argument to a function and the
-	function must not contain optional arguments.
-
 STRANDS
 	C-Type: ``const struct strands *``
 
-	Strands are like STRING_LIST, but without the drawbacks of
-	variable arguments: The list of strings gets passed in a
-	struct with the following members:
+	Strands are a list of strings that gets passed in a struct with the
+	following members:
 
 	* ``int n``: the number of strings
 	* ``const char **p``: the array of strings with `n` elements

From dridi.boukelmoune at gmail.com  Sat Aug 21 06:17:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Sat, 21 Aug 2021 06:17:06 +0000 (UTC)
Subject: [master] 164ebeb47 vmodtool: Remove STRING_LIST support
Message-ID: <20210821061706.651D4AC286@lists.varnish-cache.org>


commit 164ebeb47696153c03a7fbcb0e1903551eccfbea
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Sat Aug 21 08:15:08 2021 +0200

    vmodtool: Remove STRING_LIST support

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 126fc8306..e37a75966 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -114,7 +114,6 @@ CTYPES = {
     'STEVEDORE':   "VCL_STEVEDORE",
     'STRANDS':     "VCL_STRANDS",
     'STRING':      "VCL_STRING",
-    'STRING_LIST': "const char *, ...",
     'SUB':         "VCL_SUB",
     'TIME':        "VCL_TIME",
     'VOID':        "VCL_VOID",
@@ -234,12 +233,6 @@ class CType(object):
         self.opt = False
 
         self.vt = wl.pop(0)
-        if self.vt == "STRING_LIST":
-            deprecated("STRING_LIST", '''
-STRING_LIST will be discontinued before the 2019-09-15 release
-
-Please switch to STRANDS
-''')
         self.ct = CTYPES.get(self.vt)
         if self.ct is None:
             err("Expected type got '%s'" % self.vt, warn=False)
@@ -275,7 +268,7 @@ Please switch to STRANDS
             assert w == ","
 
     def vcl(self, terse=False):
-        if self.vt in ("STRING_LIST", "STRANDS"):
+        if self.vt == "STRANDS":
             return "STRING"
         if terse:
             return self.vt
@@ -385,8 +378,6 @@ class ProtoType(object):
                 t = arg(wl, names, st.vcc.enums, ',')
             if t.vt == 'VOID':
                 err("arguments can not be of type '%s'" % t.vt, warn=False)
-            if t.vt == 'STRING_LIST' and len(wl) > 1:
-                err("'%s' must be the last argument" % t.vt, warn=False)
             if t.nm is None:
                 t.nm2 = "arg%d" % n
             else:

From dridi.boukelmoune at gmail.com  Mon Aug 23 04:23:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 23 Aug 2021 04:23:06 +0000 (UTC)
Subject: [master] f0ea90986 vtc: Work around a leak report with h2 tests
Message-ID: <20210823042306.87F35AC1AF@lists.varnish-cache.org>


commit f0ea90986c14d7769cb16acd41c8a32328a92891
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 23 06:13:22 2021 +0200

    vtc: Work around a leak report with h2 tests
    
    This will hopefully silence our continuous integration complaining way
    too often about server leaking what http_process_cleanup() should take
    care of.

diff --git a/bin/varnishtest/tests/a02001.vtc b/bin/varnishtest/tests/a02001.vtc
index 332957982..eb0253983 100644
--- a/bin/varnishtest/tests/a02001.vtc
+++ b/bin/varnishtest/tests/a02001.vtc
@@ -94,3 +94,5 @@ client c1 -connect ${s1_sock} {
 		expect resp.body == "floubidoutata"
 	} -run
 } -run
+
+server s1 -wait
diff --git a/bin/varnishtest/tests/a02002.vtc b/bin/varnishtest/tests/a02002.vtc
index 0ebd23e67..2f6c504bd 100644
--- a/bin/varnishtest/tests/a02002.vtc
+++ b/bin/varnishtest/tests/a02002.vtc
@@ -18,3 +18,5 @@ client c1 -connect ${s1_sock} {
 		expect goaway.debug == "compression_error"
 	} -run
 } -run
+
+server s1 -wait
diff --git a/bin/varnishtest/tests/a02003.vtc b/bin/varnishtest/tests/a02003.vtc
index 6e0bb50b7..eced2da04 100644
--- a/bin/varnishtest/tests/a02003.vtc
+++ b/bin/varnishtest/tests/a02003.vtc
@@ -15,3 +15,5 @@ client c1 -connect ${s1_sock} {
 		expect resp.bodylen == 7
 	} -run
 } -run
+
+server s1 -wait
diff --git a/bin/varnishtest/tests/a02004.vtc b/bin/varnishtest/tests/a02004.vtc
index 992b8a194..915446080 100644
--- a/bin/varnishtest/tests/a02004.vtc
+++ b/bin/varnishtest/tests/a02004.vtc
@@ -14,3 +14,5 @@ client c1 -connect ${s1_sock} {
 		expect resp.bodylen == 3
 	} -run
 } -run
+
+server s1 -wait
diff --git a/bin/varnishtest/tests/a02011.vtc b/bin/varnishtest/tests/a02011.vtc
index 031bc4068..d4a7719e6 100644
--- a/bin/varnishtest/tests/a02011.vtc
+++ b/bin/varnishtest/tests/a02011.vtc
@@ -20,3 +20,5 @@ client c1  -connect ${s1_sock} {
 		expect resp.http.:status == 200
 	} -run
 } -run
+
+server s1 -wait
diff --git a/bin/varnishtest/tests/a02012.vtc b/bin/varnishtest/tests/a02012.vtc
index 76ad72de6..83960b8e8 100644
--- a/bin/varnishtest/tests/a02012.vtc
+++ b/bin/varnishtest/tests/a02012.vtc
@@ -54,3 +54,5 @@ client c1 -connect ${s1_sock} {
 		expect frame.padding == 6
 	} -run
 } -run
+
+server s1 -wait
diff --git a/bin/varnishtest/tests/a02014.vtc b/bin/varnishtest/tests/a02014.vtc
index 0dd7d71e1..b026e56b9 100644
--- a/bin/varnishtest/tests/a02014.vtc
+++ b/bin/varnishtest/tests/a02014.vtc
@@ -20,6 +20,7 @@ server s1 {
 		expect stream.dependency == 7
 	} -run
 } -start
+
 client c1 -connect ${s1_sock} {
 	stream 1 {
 		txreq -method GET -url /1 \
@@ -49,4 +50,5 @@ client c1 -connect ${s1_sock} {
 		expect stream.dependency == <undef>
 	} -run
 } -run
+
 server s1 -wait
diff --git a/bin/varnishtest/tests/a02015.vtc b/bin/varnishtest/tests/a02015.vtc
index 9af3a7a0d..e1f7b59d0 100644
--- a/bin/varnishtest/tests/a02015.vtc
+++ b/bin/varnishtest/tests/a02015.vtc
@@ -30,6 +30,7 @@ server s1 {
 	} -run
 
 } -start
+
 client c1 -connect ${s1_sock} {
 	stream 1 {
 		txreq
diff --git a/bin/varnishtest/tests/a02019.vtc b/bin/varnishtest/tests/a02019.vtc
index 712d93e3f..388b6e506 100644
--- a/bin/varnishtest/tests/a02019.vtc
+++ b/bin/varnishtest/tests/a02019.vtc
@@ -1,4 +1,5 @@
 varnishtest "Static table encoding"
+
 server s1 {
 	stream 1 {
 		rxreq
diff --git a/bin/varnishtest/tests/a02020.vtc b/bin/varnishtest/tests/a02020.vtc
index 5b0702ba1..38aba8755 100644
--- a/bin/varnishtest/tests/a02020.vtc
+++ b/bin/varnishtest/tests/a02020.vtc
@@ -39,3 +39,5 @@ client c1 -connect ${s1_sock} {
 	} -run
 
 } -run
+
+server s1 -wait
diff --git a/bin/varnishtest/tests/a02022.vtc b/bin/varnishtest/tests/a02022.vtc
index c089cd6a6..1be052e05 100644
--- a/bin/varnishtest/tests/a02022.vtc
+++ b/bin/varnishtest/tests/a02022.vtc
@@ -21,3 +21,5 @@ server s1 {
 } -start
 
 shell { nghttp http://${s1_sock} -nu }
+
+server s1 -wait
diff --git a/bin/varnishtest/tests/a02025.vtc b/bin/varnishtest/tests/a02025.vtc
index b3088d4d9..396e29e6e 100644
--- a/bin/varnishtest/tests/a02025.vtc
+++ b/bin/varnishtest/tests/a02025.vtc
@@ -1,17 +1,21 @@
 varnishtest "Test -bodyfrom"
 
+shell {printf helloworld >body.txt}
+
 server s1 {
 	stream 1 {
 		rxreq
-		expect req.bodylen == 286
-		txresp -bodyfrom ${testdir}/a02025.vtc
+		expect req.body == helloworld
+		txresp -bodyfrom body.txt
 	} -run
 } -start
 
 client c1 -connect ${s1_sock} {
 	stream 1 {
-		txreq -bodyfrom ${testdir}/a02025.vtc
+		txreq -bodyfrom body.txt
 		rxresp
-		expect resp.bodylen == 286
+		expect resp.body == helloworld
 	} -run
 } -run
+
+server s1 -wait
diff --git a/bin/varnishtest/tests/a02026.vtc b/bin/varnishtest/tests/a02026.vtc
index 71bcca94d..bf61aebc9 100644
--- a/bin/varnishtest/tests/a02026.vtc
+++ b/bin/varnishtest/tests/a02026.vtc
@@ -27,3 +27,5 @@ client c1 -connect ${s1_sock} {
 		expect resp.bodylen == 10
 	} -run
 } -run
+
+server s1 -wait
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 6e0c2846b..d4265f4c4 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -1806,7 +1806,6 @@ http_process(struct vtclog *vl, struct vtc_sess *vsp, const char *spec,
 	hp->sess->fd = sock;
 	hp->timeout = vtc_maxdur * 1000 / 2;
 
-
 	if (rcvbuf) {
 		// XXX setsockopt() too late on SunOS
 		// https://github.com/varnishcache/varnish-cache/pull/2980#issuecomment-486214661
@@ -1853,6 +1852,12 @@ http_process(struct vtclog *vl, struct vtc_sess *vsp, const char *spec,
 		strcpy(hp->rem_port, "0");
 		hp->rem_path = strdup(addr);
 	}
+	/* XXX: After an upgrade to HTTP/2 the cleanup of a server that is
+	 * not -wait'ed before the test resets is subject to a race where the
+	 * cleanup does not happen, so ASAN reports leaks despite the push
+	 * of a cleanup handler. To easily reproduce, remove the server wait
+	 * from a02022.vtc and run with ASAN enabled.
+	 */
 	pthread_cleanup_push(http_process_cleanup, hp);
 	parse_string(vl, hp, spec);
 	retval = hp->sess->fd;

From dridi.boukelmoune at gmail.com  Mon Aug 23 21:31:09 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 23 Aug 2021 21:31:09 +0000 (UTC)
Subject: [master] 7b7227995 build: Clean up after the vrt_test changes
Message-ID: <20210823213109.9A4A4A3885@lists.varnish-cache.org>


commit 7b722799598ea3ee98fee3c79089f07c22c670be
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 23 22:37:42 2021 +0200

    build: Clean up after the vrt_test changes
    
    Refs 5884e2f2fd55

diff --git a/include/Makefile.am b/include/Makefile.am
index ecf95affd..62395ce2a 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -148,14 +148,12 @@ vmod_abi.h: vcs_version.h
 BUILT_SOURCES = \
 	$(GENERATED_H) \
 	vcs_version.h \
-	vmod_abi.h
+	vmod_abi.h \
+	vrt_test.c
 
 MAINTAINERCLEANFILES = $(GENERATED_H)
 
-CLEANFILES = \
-	vrt_test \
-	_vrt_test \
-	_vrt.c
+CLEANFILES = vrt_test.c
 
 noinst_PROGRAMS = vbm_test vrt_test
 
@@ -164,7 +162,6 @@ vbm_test_SOURCES = vbm_test.c vbm.h
 vrt_test.c: vdef.h vrt.h
 	$(AM_V_GEN) cat $(srcdir)/vdef.h $(srcdir)/vrt.h > $@
 
-vrt_test_BUILT_SOURCES = vrt_test.c
 vrt_test_CFLAGS = -c
 vrt_test_LINK = echo >$@
 

From dridi.boukelmoune at gmail.com  Mon Aug 23 21:31:09 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 23 Aug 2021 21:31:09 +0000 (UTC)
Subject: [master] f563d5779 circleci: 32bit build on the oldest Ubuntu
Message-ID: <20210823213109.8D300A3839@lists.varnish-cache.org>


commit f563d577918605fe94e09fdeffa78cb3cee8ba3d
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 23 20:37:19 2021 +0200

    circleci: 32bit build on the oldest Ubuntu

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 95df2a5e2..2a58cbf45 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -327,6 +327,11 @@ workflows:
           dist: debian
           release: buster
           extra_conf: --enable-asan --enable-ubsan
+      - distcheck:
+          name: distcheck_ubuntu_xenial
+          dist: ubuntu
+          release: xenial
+          extra_conf: CFLAGS='-g -O2 -m32'
       - distcheck:
           name: distcheck_alpine
           dist: alpine

From phk at FreeBSD.org  Tue Aug 24 05:40:08 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Aug 2021 05:40:08 +0000 (UTC)
Subject: [master] d9a393617 Remove deprecated VSB_new() and VSB_delete()
Message-ID: <20210824054008.51850B120C@lists.varnish-cache.org>


commit d9a3936171c9ed04a4d257b7d99acb731ac9b84d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 24 05:38:58 2021 +0000

    Remove deprecated VSB_new() and VSB_delete()

diff --git a/include/vrt.h b/include/vrt.h
index fd678e572..d65c19984 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -54,6 +54,7 @@
  * binary/load-time compatible, increment MAJOR version
  *
  * NEXT (2021-09-15)
+ *	VSB_new() and VSB_delete() removed
  *	VCL_STRINGLIST, vrt_magic_string_end removed
  *	VRT_String(), VRT_StringList(), VRT_CollectString() removed
  *	VRT_CollectStrands() renamed to VRT_STRANDS_string()
diff --git a/include/vsb.h b/include/vsb.h
index 7a32360e3..537254f4f 100644
--- a/include/vsb.h
+++ b/include/vsb.h
@@ -59,7 +59,6 @@ extern "C" {
 /*
  * API functions
  */
-struct vsb	*VSB_new(struct vsb *, char *, int, int) v_deprecated_;
 struct vsb	*VSB_init(struct vsb *, void *, ssize_t);
 struct vsb	*VSB_new_auto(void);
 void		 VSB_clear(struct vsb *);
@@ -76,7 +75,6 @@ int		 VSB_error(const struct vsb *);
 int		 VSB_finish(struct vsb *);
 char		*VSB_data(const struct vsb *);
 ssize_t		 VSB_len(const struct vsb *);
-void		 VSB_delete(struct vsb *) v_deprecated_;
 void		 VSB_fini(struct vsb *);
 void		 VSB_destroy(struct vsb **);
 
diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c
index e868cade0..f7e9e46c3 100644
--- a/lib/libvarnish/vsb.c
+++ b/lib/libvarnish/vsb.c
@@ -208,35 +208,6 @@ VSB_newbuf(struct vsb *s, char *buf, int length, int flags)
 	return (s);
 }
 
-/*
- * Initialize an vsb.
- * If buf is non-NULL, it points to a static or already-allocated string
- * big enough to hold at least length characters.
- */
-struct vsb *
-VSB_new(struct vsb *s, char *buf, int length, int flags)
-{
-
-	KASSERT(length >= 0,
-	    ("attempt to create an vsb of negative length (%d)", length));
-	KASSERT((flags & ~VSB_USRFLAGMSK) == 0,
-	    ("%s called with invalid flags", __func__));
-
-	flags &= VSB_USRFLAGMSK;
-	if (s != NULL)
-		return (VSB_newbuf(s, buf, length, flags));
-
-	s = SBMALLOC(sizeof(*s));
-	if (s == NULL)
-		return (NULL);
-	if (VSB_newbuf(s, buf, length, flags) == NULL) {
-		SBFREE(s);
-		return (NULL);
-	}
-	VSB_SETFLAG(s, VSB_DYNSTRUCT);
-	return (s);
-}
-
 struct vsb *
 VSB_init(struct vsb *s, void *buf, ssize_t length)
 {
@@ -510,25 +481,6 @@ VSB_len(const struct vsb *s)
 	return (s->s_len);
 }
 
-/*
- * Clear an vsb, free its buffer if necessary.
- */
-void
-VSB_delete(struct vsb *s)
-{
-	int isdyn;
-
-	assert_VSB_integrity(s);
-	/* don't care if it's finished or not */
-
-	if (VSB_ISDYNAMIC(s))
-		SBFREE(s->s_buf);
-	isdyn = VSB_ISDYNSTRUCT(s);
-	memset(s, 0, sizeof(*s));
-	if (isdyn)
-		SBFREE(s);
-}
-
 void
 VSB_fini(struct vsb *s)
 {

From dridi.boukelmoune at gmail.com  Tue Aug 24 06:11:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 24 Aug 2021 06:11:06 +0000 (UTC)
Subject: [master] 6d31a6ce6 libvarnishapi: Major soname bump
Message-ID: <20210824061106.2732FB2072@lists.varnish-cache.org>


commit 6d31a6ce6ddef77c2953e9736aa8364d9c622171
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 24 08:08:53 2021 +0200

    libvarnishapi: Major soname bump
    
    After the removal of VSB_new() and VSB_delete().

diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index dcd34239d..23e9a485c 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -8,7 +8,7 @@ AM_CPPFLAGS = \
 
 lib_LTLIBRARIES = libvarnishapi.la
 
-libvarnishapi_la_LDFLAGS = $(AM_LDFLAGS) -version-info 2:0:0
+libvarnishapi_la_LDFLAGS = $(AM_LDFLAGS) -version-info 3:0:0
 
 libvarnishapi_la_SOURCES = \
 	../../include/vcs_version.h \
diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map
index b98a2b761..87b374f2e 100644
--- a/lib/libvarnishapi/libvarnishapi.map
+++ b/lib/libvarnishapi/libvarnishapi.map
@@ -28,9 +28,10 @@
  * SUCH DAMAGE.
  */
 
-LIBVARNISHAPI_2.0 {
+LIBVARNISHAPI_3.0 {	/* 2021-09-15 release */
     global:
 	# vas.c
+		VAS_errtxt;
 		VAS_Fail;
 		VAS_Fail_Func;
 
@@ -41,33 +42,51 @@ LIBVARNISHAPI_2.0 {
 
 	# vcs.c
 		VCS_Message;
+		VCS_String;
 
 	# vsb.c
 		VSB_bcat;
 		VSB_cat;
 		VSB_clear;
 		VSB_data;
-		VSB_delete;
 		VSB_destroy;
 		VSB_error;
+		VSB_fini;
 		VSB_finish;
 		VSB_indent;
+		VSB_init;
 		VSB_len;
-		VSB_new;
+		VSB_new_auto;
 		VSB_printf;
 		VSB_putc;
 		VSB_quote;
 		VSB_quote_pfx;
+		VSB_tofile;
 		VSB_vprintf;
 
 	# vsc.c
 		VSC_Arg;
 		VSC_ChangeLevel;
 		VSC_Destroy;
+		VSC_IsRaw;
 		VSC_Iter;
 		VSC_New;
 		VSC_State;
 
+	# vsig.c
+		VSIG_int;
+		VSIG_Got_int;
+		VSIG_Arm_int;
+		VSIG_hup;
+		VSIG_Got_hup;
+		VSIG_Arm_hup;
+		VSIG_term;
+		VSIG_Got_term;
+		VSIG_Arm_term;
+		VSIG_usr1;
+		VSIG_Got_usr1;
+		VSIG_Arm_usr1;
+
 	# vsl*.c
 		VSLQ_Delete;
 		VSLQ_Dispatch;
@@ -138,76 +157,8 @@ LIBVARNISHAPI_2.0 {
 		VUT_Setup;
 		VUT_Signal;
 		VUT_Signaled;
+		VUT_Usage;
 
     local:
 	*;
 };
-
-LIBVARNISHAPI_2.1 {
-    global:
-	# vut.c
-	VUT_Usage;
-    local:
-	*;
-};
-
-LIBVARNISHAPI_2.2 {
-    global:
-	# vsig.c
-	VSIG_int;
-	VSIG_Got_int;
-	VSIG_Arm_int;
-	VSIG_hup;
-	VSIG_Got_hup;
-	VSIG_Arm_hup;
-	VSIG_term;
-	VSIG_Got_term;
-	VSIG_Arm_term;
-	VSIG_usr1;
-	VSIG_Got_usr1;
-	VSIG_Arm_usr1;
-    local:
-	*;
-};
-
-LIBVARNISHAPI_2.3 {	/* 2019-09-15 release */
-    global:
-	# vcs.c
-	VCS_String;
-    local:
-	*;
-};
-
-LIBVARNISHAPI_2.4 {	/* 2020-03-15 release */
-    global:
-	# vsb.c
-	VSB_tofile;
-    local:
-	*;
-};
-
-LIBVARNISHAPI_2.5 {	/* 2020-09-15 release */
-    global:
-	# vsb.c
-	VSB_init;
-	VSB_fini;
-	VSB_new_auto;
-    local:
-	*;
-};
-
-LIBVARNISHAPI_2.6 {	/* 2020-03-15 release */
-    global:
-	# vsc.c
-	VSC_IsRaw;
-    local:
-	*;
-};
-
-LIBVARNISHAPI_2.7 {	/* 2021-09-15 release */
-    global:
-	# vas.c
-	VAS_errtxt;
-    local:
-	*;
-};

From phk at FreeBSD.org  Tue Aug 24 11:30:08 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Aug 2021 11:30:08 +0000 (UTC)
Subject: [master] 879c9b37c Make `-n` default to `/var/run` unless overridden
 by configure(1)
Message-ID: <20210824113008.1E05064759@lists.varnish-cache.org>


commit 879c9b37ca108e365d4eb16ad391e28a8cfccd46
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 24 11:23:45 2021 +0000

    Make `-n` default to `/var/run` unless overridden by configure(1)
    
    Renovate `-n` and `-a` manual page descriptions
    
    Fixes: #3672

diff --git a/INSTALL b/INSTALL
index 4f2227f92..8f6e796d4 100644
--- a/INSTALL
+++ b/INSTALL
@@ -8,7 +8,7 @@ afterwards in order to update the shared library cache.
 If you obtained the sources directly from the Git repository, you will
 need to run autogen.sh first to create the configure script.
 
-Varnish will store run-time state in $localstatedir/varnish; you may
+Varnish will store run-time state in /var/run/varnish; you may
 want to tune this using configure's --localstatedir parameter.
 
 Additional configure options of interest:
diff --git a/configure.ac b/configure.ac
index df96a23ea..a2cfa6073 100644
--- a/configure.ac
+++ b/configure.ac
@@ -654,7 +654,11 @@ fi
 LIBS="${save_LIBS}"
 
 # Run-time directory
-VARNISH_STATE_DIR='${localstatedir}/varnish'
+if test "${localstatedir}" = '${prefix}/var' ; then
+   VARNISH_STATE_DIR='/var/run'
+else
+   VARNISH_STATE_DIR='${localstatedir}/varnish'
+fi
 AC_SUBST(VARNISH_STATE_DIR)
 
 # Default configuration directory.
diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 301b8bd66..0a5704848 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -20,7 +20,29 @@ HTTP accelerator daemon
 SYNOPSIS
 ========
 
-varnishd [-a [name=][address][:port][,PROTO][,user=<user>][,group=<group>][,mode=<mode>]] [-b [host[:port]|path]] [-C] [-d] [-F] [-f config] [-h type[,options]] [-I clifile] [-i identity] [-j jail[,jailoptions]] [-l vsl] [-M address:port] [-n name] [-P file] [-p param=value] [-r param[,param...]] [-S secret-file] [-s [name=]kind[,options]] [-T address[:port]] [-t TTL] [-V] [-W waiter]
+varnishd 
+    [-a [name=][listen_address[,PROTO]]
+    [-b [host[:port]|path]]
+    [-C]
+    [-d]
+    [-F]
+    [-f config]
+    [-h type[,options]]
+    [-I clifile]
+    [-i identity]
+    [-j jail[,jailoptions]]
+    [-l vsl]
+    [-M address:port]
+    [-n workdir]
+    [-P file]
+    [-p param=value]
+    [-r param[,param...]]
+    [-S secret-file]
+    [-s [name=]kind[,options]]
+    [-T address[:port]]
+    [-t TTL]
+    [-V]
+    [-W waiter]
 
 varnishd [-x parameter|vsl|cli|builtin|optstring]
 
@@ -41,32 +63,41 @@ OPTIONS
 Basic options
 -------------
 
--a <[name=][address][:port][,PROTO][,user=<user>][,group=<group>][,mode=<mode>]>
+-a <[name=][listen_address[,PROTO]]>
 
-  Listen for client requests on the specified address and port. The
-  address can be a host name ("localhost"), an IPv4 dotted-quad
-  ("127.0.0.1"), an IPv6 address enclosed in square brackets
-  ("[::1]"), or a path beginning with a '/' for a Unix domain socket
-  ("/path/to/listen.sock"). If address is not specified, `varnishd`
-  will listen on all available IPv4 and IPv6 interfaces. If port is
-  not specified, port 80 (http) is used. At least one of address or
-  port is required.
-
-  If a Unix domain socket is specified as the listen address, then the
-  user, group and mode sub-arguments may be used to specify the
-  permissions of the socket file -- use names for user and group, and
-  a 3-digit octal value for mode. These sub-arguments are not
-  permitted if an IP address is specified. When Unix domain socket
-  listeners are in use, all VCL configurations must have version >=
-  4.1.
+  Accept for client requests on the specified listen_address (see below).
 
   Name is referenced in logs. If name is not specified, "a0", "a1",
-  etc. is used. An additional protocol type can be set for the
-  listening socket with PROTO. Valid protocol types are: HTTP
-  (default), and PROXY.
+  etc. is used.
+
+  PROTO can be "HTTP" (the default) or "PROXY".  Both version 1
+  and 2 of the proxy protocol can be used.
+
+  Multiple -a arguments are allowed.
+
+  If no -a argument is given, the default `-a :80` will listen to
+  all IPv4 and IPv6 interfaces.
+
+-a <[name=][ip_address][:port][,PROTO]>
+
+  The ip_address can be a host name ("localhost"), an IPv4 dotted-quad
+  ("127.0.0.1") or an IPv6 address enclosed in square brackets
+  ("[::1]")
+
+  If port is not specified, port 80 (http) is used.
 
-  Multiple listening addresses can be specified by using different
-  -a arguments.
+  At least one of ip_address or port is required.
+
+-a <[name=][path][,PROTO][,user=name][,group=name][,mode=octal]>
+
+  (VCL4.1 and higher)
+
+  Accept connections on a Unix domain socket.  Path must be absolute
+  ("/path/to/listen.sock").
+
+  The user, group and mode sub-arguments may be used to specify the
+  permissions of the socket file -- use names for user and group, and
+  a 3-digit octal value for mode.
 
 -b <[host[:port]|path]>
 
@@ -111,12 +142,17 @@ Basic options
 
 .. _opt_n:
 
--n name
+-n workdir
+
+  Runtime directory for the shared memory, compiled VCLs etc.
+
+  In performance critical applications, this directory should be
+  on a RAM backed filesystem.
+
+  Relative paths will be appended to `/var/run/` (NB: Binary packages
+  of Varnish may have adjusted this to the platform.)
 
-  Specify the name for this instance.  This name is used to construct
-  the name of the directory in which `varnishd` keeps temporary files
-  and persistent state. If the specified name begins with a forward slash,
-  it is interpreted as the absolute path to the directory.
+  The default value is `/var/run/varnishd` (NB: as above.)
 
 Documentation options
 ---------------------
diff --git a/lib/libvarnish/vin.c b/lib/libvarnish/vin.c
index 581bbdba2..493486f8d 100644
--- a/lib/libvarnish/vin.c
+++ b/lib/libvarnish/vin.c
@@ -27,17 +27,14 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * XXX: NB: also used in libvarnishapi
+ * NB: also used in libvarnishapi
  */
 
 #include "config.h"
 
-#include <limits.h>
-#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <unistd.h>
 
 #include "vdef.h"
 
@@ -48,45 +45,27 @@
 int
 VIN_n_Arg(const char *n_arg, char **dir)
 {
-	char nm[PATH_MAX];
-	char dn[PATH_MAX];
-	struct vsb vsb[1];
-	int i;
+	struct vsb *vsb;
 
 	AN(dir);
 
-	/* First: determine the name */
-
-	if (n_arg == NULL || *n_arg == '\0') {
-		if (gethostname(nm, sizeof nm) != 0)
-			return (-1);
-	} else if (strlen(n_arg) >= sizeof nm) {
-		/* preliminary length check to avoid overflowing nm */
-		errno = ENAMETOOLONG;
-		return (-1);
-	} else
-		bprintf(nm, "%s", n_arg);
-
-	/* Second: find the directory name */
-
-	AN(VSB_init(vsb, dn, sizeof dn));
-
-	if (*nm == '/')
-		i = VSB_printf(vsb, "%s/", nm);
-	else
-		i = VSB_printf(vsb, "%s/%s/", VARNISH_STATE_DIR, nm);
-
-	if (i != 0) {
-		errno = ENAMETOOLONG;
-		return (-1);
+	vsb = VSB_new_auto();
+	AN(vsb);
+	if (n_arg == NULL || n_arg[0] == '\0') {
+		VSB_cat(vsb, VARNISH_STATE_DIR);
+		VSB_cat(vsb, "/varnishd");
+	} else if (n_arg[0] == '/') {
+		VSB_cat(vsb, n_arg);
+	} else {
+		VSB_cat(vsb, VARNISH_STATE_DIR);
+		VSB_cat(vsb, "/");
+		VSB_cat(vsb, n_arg);
 	}
-
 	AZ(VSB_finish(vsb));
 
 	*dir = strdup(VSB_data(vsb));
+	VSB_destroy(&vsb);
 	if (*dir == NULL)
 		return (-1);
-
-	VSB_fini(vsb);
 	return (0);
 }

From phk at FreeBSD.org  Tue Aug 24 12:51:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 24 Aug 2021 12:51:05 +0000 (UTC)
Subject: [master] 9d1986f92 Simplify VIN_n_Arg() to return the directory
 directly.
Message-ID: <20210824125105.CF3836EF7C@lists.varnish-cache.org>


commit 9d1986f92c6efccec7e3b8705026088c30f2f166
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 24 12:49:37 2021 +0000

    Simplify VIN_n_Arg() to return the directory directly.

diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 9c2a8f0ef..f5d2ede33 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -782,8 +782,8 @@ main(int argc, char * const *argv)
 		VJ_master(JAIL_MASTER_LOW);
 	}
 
-	if (VIN_n_Arg(n_arg, &workdir) != 0)
-		ARGV_ERR("Invalid instance (-n) name: %s\n", VAS_errtxt(errno));
+	workdir = VIN_n_Arg(n_arg);
+	AN(workdir);
 
 	if (i_arg == NULL || *i_arg == '\0')
 		i_arg = mgt_HostName();
diff --git a/include/vin.h b/include/vin.h
index 373dda0e1..e7dc3eb8d 100644
--- a/include/vin.h
+++ b/include/vin.h
@@ -34,5 +34,5 @@
 #define VIN_H_INCLUDED
 
 /* This function lives in both libvarnish and libvarnishapi */
-int VIN_n_Arg(const char *n_arg, char **dir);
+char *VIN_n_Arg(const char *n_arg);
 #endif
diff --git a/include/vrt.h b/include/vrt.h
index d65c19984..24d11fd10 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -54,6 +54,7 @@
  * binary/load-time compatible, increment MAJOR version
  *
  * NEXT (2021-09-15)
+ *	VIN_n_Arg() no directly returns the directory name.
  *	VSB_new() and VSB_delete() removed
  *	VCL_STRINGLIST, vrt_magic_string_end removed
  *	VRT_String(), VRT_StringList(), VRT_CollectString() removed
diff --git a/lib/libvarnish/vin.c b/lib/libvarnish/vin.c
index 493486f8d..89f132f4e 100644
--- a/lib/libvarnish/vin.c
+++ b/lib/libvarnish/vin.c
@@ -42,12 +42,11 @@
 #include "vin.h"
 #include "vsb.h"
 
-int
-VIN_n_Arg(const char *n_arg, char **dir)
+char *
+VIN_n_Arg(const char *n_arg)
 {
 	struct vsb *vsb;
-
-	AN(dir);
+	char *retval;
 
 	vsb = VSB_new_auto();
 	AN(vsb);
@@ -63,9 +62,7 @@ VIN_n_Arg(const char *n_arg, char **dir)
 	}
 	AZ(VSB_finish(vsb));
 
-	*dir = strdup(VSB_data(vsb));
+	retval = strdup(VSB_data(vsb));
 	VSB_destroy(&vsb);
-	if (*dir == NULL)
-		return (-1);
-	return (0);
+	return (retval);
 }
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 0040e85c8..af1b3b212 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -387,12 +387,12 @@ VSM_Arg(struct vsm *vd, char flag, const char *arg)
 		}
 		break;
 	case 'n':
-		if (VIN_n_Arg(arg, &p))
+		if (vd->wdname != NULL)
+			free(vd->wdname);
+		vd->wdname = VIN_n_Arg(arg);
+		if (vd->wdname == NULL)
 			return (vsm_diag(vd, "Invalid instance name: %s",
 			    strerror(errno)));
-		AN(p);
-		REPLACE(vd->wdname, p);
-		free(p);
 		break;
 	default:
 		return (vsm_diag(vd, "Unknown VSM_Arg('%c')", flag));

From phk at FreeBSD.org  Wed Aug 25 06:12:07 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 25 Aug 2021 06:12:07 +0000 (UTC)
Subject: [master] e099a39a6 Bump the minimum session workspace to 384 bytes
Message-ID: <20210825061207.AE0289CAD@lists.varnish-cache.org>


commit e099a39a64b6634a5a486dc0769caad33022f0de
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 25 06:10:37 2021 +0000

    Bump the minimum session workspace to 384 bytes
    
    Closes: #3318

diff --git a/bin/varnishtest/tests/o00005.vtc b/bin/varnishtest/tests/o00005.vtc
index 6a3c99446..53250dc84 100644
--- a/bin/varnishtest/tests/o00005.vtc
+++ b/bin/varnishtest/tests/o00005.vtc
@@ -216,8 +216,10 @@ logexpect l1 -wait
 # This value is fragile, ideally we would want something like
 # vtc.alloc(-x), yet there is no vcl code being run before we parse
 # proxy headers
+#
+# 20210825: Minimum possible seems to be 364, but param.min is 384 now.
 
-varnish v1 -cliok "param.set workspace_session 402"
+varnish v1 -cliok "param.set workspace_session 384"
 
 delay 1
 
diff --git a/include/tbl/params.h b/include/tbl/params.h
index 26345d6c3..7d383b8d2 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -1110,7 +1110,7 @@ PARAM_SIMPLE(
 PARAM_SIMPLE(
 	/* name */	workspace_session,
 	/* type */	bytes_u,
-	/* min */	"0.25k",
+	/* min */	"384b",
 	/* max */	NULL,
 	/* def */	"0.75k",
 	/* units */	"bytes",

From phk at FreeBSD.org  Wed Aug 25 07:03:06 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 25 Aug 2021 07:03:06 +0000 (UTC)
Subject: [master] 4c5911279 Keep release notes informed
Message-ID: <20210825070306.7ADD06275F@lists.varnish-cache.org>


commit 4c5911279c5d4bfd83e478485f4e45a363272176
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 25 06:50:31 2021 +0000

    Keep release notes informed

diff --git a/doc/changes.rst b/doc/changes.rst
index 92f3c533e..59ed9eef3 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -35,6 +35,12 @@ release process.
 Varnish Cache 7.x.x (2021-09-15)
 ================================
 
+* The minimum `session_workspace` is now 384 bytes
+
+* The default workdir (the default `-n` argument) is now `/var/run`
+  instead of `${prefix}/var`.
+  (Packages usually configure this to match local customs.)
+
 * 'Scientific Notation' numbers like 6.62607004e-34 are no longer
   supported in VCL.  (The preparation of RFC8941 made it clear that
   there are neither reason nor any need to support scientific notation

From phk at FreeBSD.org  Wed Aug 25 07:03:06 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 25 Aug 2021 07:03:06 +0000 (UTC)
Subject: [master] 58fefad06 Emit minimal 500 response if vcl_synth{} fails.
Message-ID: <20210825070306.AA91262762@lists.varnish-cache.org>


commit 58fefad0697b1ead28dfe6ac4bc9e77f8d00ccdf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 25 07:00:10 2021 +0000

    Emit minimal 500 response if vcl_synth{} fails.
    
    With the synth->filters koncept out of the picture this becomes
    possible.
    
    Fixes: #3441

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 683b08135..f584f63fa 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -329,7 +329,9 @@ cnt_synth(struct worker *wrk, struct req *req)
 
 	if (wrk->handling == VCL_RET_FAIL) {
 		VSB_destroy(&synth_body);
-		req->doclose = SC_VCL_FAILURE;
+		(void)VRB_Ignore(req);
+		(void)req->transport->minimal_response(req, 500);
+		req->doclose = SC_VCL_FAILURE; // XXX: Not necessary any more ?
 		VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
 		http_Teardown(req->resp);
 		return (REQ_FSM_DONE);
diff --git a/bin/varnishtest/tests/r01576.vtc b/bin/varnishtest/tests/r01576.vtc
index e866fb145..617b6df6f 100644
--- a/bin/varnishtest/tests/r01576.vtc
+++ b/bin/varnishtest/tests/r01576.vtc
@@ -47,6 +47,8 @@ logexpect l1 -v v1 {
 # This should fail with default params and JIT/no-JIT
 client c1 {
 	txreq -url "/${string,repeat,8192,AB}"
+	rxresp
+	expect resp.status == 500
 	expect_close
 } -run
 
diff --git a/bin/varnishtest/tests/r02339.vtc b/bin/varnishtest/tests/r02339.vtc
index a90479a6a..11f84fb27 100644
--- a/bin/varnishtest/tests/r02339.vtc
+++ b/bin/varnishtest/tests/r02339.vtc
@@ -154,6 +154,8 @@ client c1 {
 
 client c1 {
 	txreq -url synth
+	rxresp
+	expect resp.status == 500
 	expect_close
 } -run
 
diff --git a/bin/varnishtest/tests/r02488.vtc b/bin/varnishtest/tests/r02488.vtc
index 11844a60b..d4c0bb069 100644
--- a/bin/varnishtest/tests/r02488.vtc
+++ b/bin/varnishtest/tests/r02488.vtc
@@ -35,6 +35,8 @@ client c1 {
 	expect resp.http.bar == "bar"
 
 	txreq -url "/empty"
+	rxresp
+	expect resp.status == 500
 	expect_close
 } -run
 
diff --git a/bin/varnishtest/tests/v00051.vtc b/bin/varnishtest/tests/v00051.vtc
index 8b8453b32..8f5ffd606 100644
--- a/bin/varnishtest/tests/v00051.vtc
+++ b/bin/varnishtest/tests/v00051.vtc
@@ -150,6 +150,8 @@ logexpect l1007 -v v1 -g raw {
 
 client c1 {
 	txreq -hdr "foo: synth"
+	rxresp
+	expect resp.status == 500
 	expect_close
 } -run
 

From phk at FreeBSD.org  Wed Aug 25 07:29:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 25 Aug 2021 07:29:05 +0000 (UTC)
Subject: [master] 556007bb5 Don't change the C-L if we detect a failed stream.
Message-ID: <20210825072905.9C89863859@lists.varnish-cache.org>


commit 556007bb5ce9dae18b347324d70740d9c392f503
Author: Reza Naghibi <reza at naghibi.com>
Date:   Wed Mar 24 15:58:05 2021 -0400

    Don't change the C-L if we detect a failed stream.
    
    Previously we would read the response Content-Length from a failed oc,
    which would make the error response valid. Now, if this is detected,
    we don't touch the Content-Length.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index f584f63fa..75ea7b27c 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -425,7 +425,7 @@ cnt_transmit(struct worker *wrk, struct req *req)
 	status = http_GetStatus(req->resp);
 	head = !strcmp(req->http0->hd[HTTP_HDR_METHOD].b, "HEAD");
 
-	if (boc != NULL)
+	if (boc != NULL || (req->objcore->flags & (OC_F_FAILED)))
 		req->resp_len = clval;
 	else
 		req->resp_len = ObjGetLen(req->wrk, req->objcore);
@@ -477,6 +477,11 @@ cnt_transmit(struct worker *wrk, struct req *req)
 
 	HSH_Cancel(wrk, req->objcore, boc);
 
+	if (!req->doclose && (req->objcore->flags & OC_F_FAILED))
+		/* The object we delivered failed due to a streaming error.
+		 * Fail the request. */
+		req->doclose = SC_TX_ERROR;
+
 	if (boc != NULL)
 		HSH_DerefBoc(wrk, req->objcore);
 
diff --git a/bin/varnishtest/tests/r03560.vtc b/bin/varnishtest/tests/r03560.vtc
new file mode 100644
index 000000000..fc76bbe1d
--- /dev/null
+++ b/bin/varnishtest/tests/r03560.vtc
@@ -0,0 +1,29 @@
+varnishtest "A backend connection error gets returned as a valid short response"
+
+barrier b1 sock 2
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr "Content-Length: 10"
+	barrier b1 sync
+	send "12345"
+	# Early connection close error
+} -start
+
+varnish v1 -vcl+backend {
+	import vtc;
+
+	sub vcl_deliver {
+		# Make sure we are streaming and give the backend time to error out
+		vtc.barrier_sync("${b1_sock}");
+		vtc.sleep(1s);
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresphdrs
+	expect resp.http.Content-Length == "10"
+	recv 5
+	expect_close
+} -run

From phk at FreeBSD.org  Wed Aug 25 07:30:08 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 25 Aug 2021 07:30:08 +0000 (UTC)
Subject: [master] 7e46a249e Docfix for issue 3634
Message-ID: <20210825073008.5C2B363A65@lists.varnish-cache.org>


commit 7e46a249ee103f9b2bcbd8ce1479ccbd1c1f676d
Author: P?l Hermunn Johansen <hermunn at varnish-software.com>
Date:   Wed Jul 14 14:47:57 2021 +0200

    Docfix for issue 3634
    
    Fixes: #3634

diff --git a/vmod/vmod_cookie.vcc b/vmod/vmod_cookie.vcc
index 85e6b8519..1ae5ad020 100644
--- a/vmod/vmod_cookie.vcc
+++ b/vmod/vmod_cookie.vcc
@@ -150,8 +150,7 @@ Example::
 
 $Function STRING get(PRIV_TASK, STRING cookiename)
 
-Get the value of ``cookiename``, as stored in internal vmod storage. If
-``cookiename`` does not exist an empty string is returned.
+Get the value of ``cookiename``, as stored in internal vmod storage.
 
 Example::
 
@@ -161,6 +160,38 @@ Example::
 	    std.log("cookie1 value is: " + cookie.get("cookie1"));
 	}
 
+If ``cookiename`` does not exist, the `NULL` string is returned. Note
+that a `NULL` string is converted to an empty string when assigned to
+a header. This means that the following is correct::
+
+	if (req.http.Cookie) {
+		cookie.parse(req.http.Cookie);
+		set req.http.X-tmp = cookie.get("a_cookie");
+	} else {
+		set req.http.X-tmp = "";
+	}
+	if (req.http.X-tmp != "") {
+		// do something with the X-tmp header...
+	} else {
+		// fallback case
+	}
+
+However, using `cookie.isset()` is often a better way to check if a
+particular cookie is present, like this::
+
+	unset req.http.X-tmp; # unnecessary if no fallback is needed
+	if (req.http.Cookie) {
+		cookie.parse(req.http.Cookie);
+		if (cookie.isset("a_cookie")) {
+			set req.http.X-tmp = cookie.get("a_cookie");
+			# do something with the X-tmp header...
+		}
+	}
+	# if necessary, do something when a_cookie is not there
+	if (!req.http.X-tmp) {
+		# fallback case
+	}
+
 $Function STRING get_re(PRIV_TASK, REGEX expression)
 
 Get the value of the first cookie in internal vmod storage that matches

From phk at FreeBSD.org  Wed Aug 25 07:30:08 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 25 Aug 2021 07:30:08 +0000 (UTC)
Subject: [master] 15014ba96 The NULL string is also returned for the regex
 version of get_header
Message-ID: <20210825073008.75B2F63A68@lists.varnish-cache.org>


commit 15014ba9646f07725339bc024375a3feb639978d
Author: P?l Hermunn Johansen <hermunn at varnish-software.com>
Date:   Wed Jul 14 17:30:40 2021 +0200

    The NULL string is also returned for the regex version of get_header

diff --git a/vmod/vmod_cookie.vcc b/vmod/vmod_cookie.vcc
index 1ae5ad020..913240f1b 100644
--- a/vmod/vmod_cookie.vcc
+++ b/vmod/vmod_cookie.vcc
@@ -194,8 +194,8 @@ particular cookie is present, like this::
 
 $Function STRING get_re(PRIV_TASK, REGEX expression)
 
-Get the value of the first cookie in internal vmod storage that matches
-regular expression ``expression``. If nothing matches, an empty string
+Get the value of the first cookie in internal vmod storage that matches the
+regular expression ``expression``. If nothing matches, the `NULL` string
 is returned.
 
 Example::

From dridi.boukelmoune at gmail.com  Wed Aug 25 10:21:04 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 25 Aug 2021 10:21:04 +0000 (UTC)
Subject: [master] dc3115491 build: Clean up individual sanitizers enablement
Message-ID: <20210825102104.D0A55932EF@lists.varnish-cache.org>


commit dc3115491d1df46ef8fcc849423e0512106fc210
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 24 08:41:33 2021 +0200

    build: Clean up individual sanitizers enablement
    
    Removing unused *SAN_*FLAGS variables and making sure to quote anything
    that may be relevant to quote. Hopefully indenting in more readable way.

diff --git a/configure.ac b/configure.ac
index a2cfa6073..0a84b1873 100644
--- a/configure.ac
+++ b/configure.ac
@@ -272,31 +272,32 @@ CFLAGS="${save_CFLAGS}"
 
 SAN_CFLAGS=
 SAN_LDFLAGS=
-UBSAN_CFLAGS=
-UBSAN_LDFLAGS=
+
 AC_ARG_ENABLE(ubsan,
-	AS_HELP_STRING([--enable-ubsan],[enable undefined behavior sanitizer (default is NO)]),
-	UBSAN_FLAGS="-fsanitize=undefined")
+	AS_HELP_STRING([--enable-ubsan],
+		[enable undefined behavior sanitizer (default is NO)]),
+	[UBSAN_FLAGS=-fsanitize=undefined])
 
-TSAN_CFLAGS=
-TSAN_LDFLAGS=
 AC_ARG_ENABLE(tsan,
-	AS_HELP_STRING([--enable-tsan],[enable thread sanitizer (default is NO)]),
-	TSAN_FLAGS="-fsanitize=thread")
+	AS_HELP_STRING([--enable-tsan],
+		[enable thread sanitizer (default is NO)]),
+	[TSAN_FLAGS=-fsanitize=thread])
 
-ASAN_CFLAGS=
-ASAN_LDFLAGS=
 AC_ARG_ENABLE(asan,
-	AS_HELP_STRING([--enable-asan],[enable address sanitizer (default is NO)]),
-	ASAN_FLAGS="-fsanitize=address"
-	AX_CHECK_COMPILE_FLAG([-fsanitize=address -fsanitize-address-use-after-scope],
-			      [ASAN_FLAGS="${ASAN_FLAGS} -fsanitize-address-use-after-scope"]))
+	AS_HELP_STRING([--enable-asan],
+		[enable address sanitizer (default is NO)]),
+	[ASAN_FLAGS=-fsanitize=address])
+
+if test -n "$ASAN_FLAGS"; then
+	AX_CHECK_COMPILE_FLAG(
+		[$ASAN_FLAGS -fsanitize-address-use-after-scope],
+		[ASAN_FLAGS="$ASAN_FLAGS -fsanitize-address-use-after-scope"])
+fi
 
-MSAN_CFLAGS=
-MSAN_LDFLAGS=
 AC_ARG_ENABLE(msan,
-	AS_HELP_STRING([--enable-msan],[enable memory sanitizer (default is NO)]),
-	MSAN_FLAGS="-fsanitize=memory")
+	AS_HELP_STRING([--enable-msan],
+		[enable memory sanitizer (default is NO)]),
+	[MSAN_FLAGS=-fsanitize=memory])
 
 if test "x$UBSAN_FLAGS$TSAN_FLAGS$ASAN_FLAGS$MSAN_FLAGS" != "x"; then
 	SAN_CFLAGS="-D__SANITIZER=1 ${UBSAN_FLAGS} ${TSAN_FLAGS} ${ASAN_FLAGS} ${MSAN_FLAGS} -fPIC -fPIE -fno-omit-frame-pointer"

From dridi.boukelmoune at gmail.com  Wed Aug 25 10:21:04 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 25 Aug 2021 10:21:04 +0000 (UTC)
Subject: [master] c442a169b build: Clean up the consolidated sanitizer config
Message-ID: <20210825102104.E6979932F3@lists.varnish-cache.org>


commit c442a169baffbd545bc665f9fcaa04155bc4768e
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 24 08:53:09 2021 +0200

    build: Clean up the consolidated sanitizer config
    
    To avoid repetition and very long lines.

diff --git a/configure.ac b/configure.ac
index 0a84b1873..3b0fbd5bf 100644
--- a/configure.ac
+++ b/configure.ac
@@ -270,9 +270,6 @@ if test "$ac_cv_have_viz" = no; then
 fi
 CFLAGS="${save_CFLAGS}"
 
-SAN_CFLAGS=
-SAN_LDFLAGS=
-
 AC_ARG_ENABLE(ubsan,
 	AS_HELP_STRING([--enable-ubsan],
 		[enable undefined behavior sanitizer (default is NO)]),
@@ -299,13 +296,18 @@ AC_ARG_ENABLE(msan,
 		[enable memory sanitizer (default is NO)]),
 	[MSAN_FLAGS=-fsanitize=memory])
 
+SAN_CFLAGS=
+SAN_LDFLAGS=
+
 if test "x$UBSAN_FLAGS$TSAN_FLAGS$ASAN_FLAGS$MSAN_FLAGS" != "x"; then
-	SAN_CFLAGS="-D__SANITIZER=1 ${UBSAN_FLAGS} ${TSAN_FLAGS} ${ASAN_FLAGS} ${MSAN_FLAGS} -fPIC -fPIE -fno-omit-frame-pointer"
-	SAN_LDFLAGS="${UBSAN_FLAGS} ${TSAN_FLAGS} ${ASAN_FLAGS} ${MSAN_FLAGS}"
-	save_CFLAGS="${CFLAGS}"
+	AC_DEFINE([__SANITIZER], [1], [Define to 1 if any sanitizer is enabled.])
+	SAN_FLAGS="$ASAN_FLAGS $UBSAN_FLAGS $TSAN_FLAGS $MSAN_FLAGS"
+	SAN_CFLAGS="$SAN_FLAGS -fPIC -fPIE -fno-omit-frame-pointer"
+	SAN_LDFLAGS=$SAN_FLAGS
+	save_CFLAGS=$CFLAGS
 	CFLAGS="${CFLAGS} -Werror=unused-command-line-argument"
-	AX_CHECK_LINK_FLAG([-pie], [SAN_LDFLAGS="${SAN_LDFLAGS} -pie"])
-	CFLAGS="${save_CFLAGS}"
+	AX_CHECK_LINK_FLAG([-pie], [SAN_LDFLAGS="$SAN_LDFLAGS -pie"])
+	CFLAGS=$save_CFLAGS
 	case $CC in
 	gcc*)
 		SAN_CFLAGS="${SAN_CFLAGS} -fuse-ld=gold"

From dridi.boukelmoune at gmail.com  Wed Aug 25 10:21:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 25 Aug 2021 10:21:05 +0000 (UTC)
Subject: [master] 9bcdac225 build: Fold sanitizer flags into regular flags
Message-ID: <20210825102105.1227C932F8@lists.varnish-cache.org>


commit 9bcdac2255a6616fab838ebf56c40096154a02b0
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 24 09:46:00 2021 +0200

    build: Fold sanitizer flags into regular flags
    
    Instead of referring to them for every single binary we build... While
    at it we shouldn't need to add -fsanitize=stuff to LDFLAGS, and we
    didn't in some cases. That should lead to smaller lines when running
    make with V=1 or when silent rules are disabled.

diff --git a/bin/varnishadm/Makefile.am b/bin/varnishadm/Makefile.am
index 2b4c31369..a86fff4b2 100644
--- a/bin/varnishadm/Makefile.am
+++ b/bin/varnishadm/Makefile.am
@@ -8,11 +8,9 @@ bin_PROGRAMS = varnishadm
 
 varnishadm_SOURCES = varnishadm.c
 
-varnishadm_CFLAGS = @LIBEDIT_CFLAGS@ \
-	@SAN_CFLAGS@
+varnishadm_CFLAGS = @LIBEDIT_CFLAGS@
 
 varnishadm_LDADD = \
 	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
 	$(top_builddir)/lib/libvarnish/libvarnish.la \
-	${PTHREAD_LIBS} ${RT_LIBS} ${NET_LIBS} @LIBEDIT_LIBS@ ${LIBM} \
-	@SAN_LDFLAGS@
+	${PTHREAD_LIBS} ${RT_LIBS} ${NET_LIBS} @LIBEDIT_LIBS@ ${LIBM}
diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 90e8e2b05..ea211f8c3 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -159,7 +159,6 @@ nobase_pkginclude_HEADERS = \
 vcldir=$(datarootdir)/$(PACKAGE)/vcl
 
 varnishd_CFLAGS = \
-	@SAN_CFLAGS@ \
 	-DNOT_IN_A_VMOD \
 	-DVARNISH_STATE_DIR='"${VARNISH_STATE_DIR}"' \
 	-DVARNISH_VMOD_DIR='"${vmoddir}"' \
@@ -171,7 +170,6 @@ varnishd_LDADD = \
 	$(top_builddir)/lib/libvcc/libvcc.a \
 	$(top_builddir)/lib/libvarnish/libvarnish.la \
 	$(top_builddir)/lib/libvgz/libvgz.a \
-	@SAN_LDFLAGS@ \
 	@JEMALLOC_LDADD@ \
 	${DL_LIBS} ${PTHREAD_LIBS} ${NET_LIBS} ${RT_LIBS} ${LIBM}
 
@@ -182,31 +180,25 @@ endif
 
 noinst_PROGRAMS = vhp_gen_hufdec
 vhp_gen_hufdec_SOURCES = hpack/vhp_gen_hufdec.c
-vhp_gen_hufdec_CFLAGS = @SAN_CFLAGS@ \
-			-include config.h
-vhp_gen_hufdec_LDADD = \
-	$(top_builddir)/lib/libvarnish/libvarnish.la
+vhp_gen_hufdec_CFLAGS = -include config.h
+vhp_gen_hufdec_LDADD = $(top_builddir)/lib/libvarnish/libvarnish.la
 
 noinst_PROGRAMS += vhp_table_test
 vhp_table_test_SOURCES = hpack/vhp_table.c
-vhp_table_test_CFLAGS = @SAN_CFLAGS@ \
-			-DTABLE_TEST_DRIVER -include config.h
-vhp_table_test_LDADD = \
-	$(top_builddir)/lib/libvarnish/libvarnish.la
+vhp_table_test_CFLAGS = -DTABLE_TEST_DRIVER -include config.h
+vhp_table_test_LDADD = $(top_builddir)/lib/libvarnish/libvarnish.la
 
 noinst_PROGRAMS += vhp_decode_test
 vhp_decode_test_SOURCES = hpack/vhp_decode.c hpack/vhp_table.c
-vhp_decode_test_CFLAGS = @SAN_CFLAGS@ \
-			 -DDECODE_TEST_DRIVER -include config.h
-vhp_decode_test_LDADD = \
-	$(top_builddir)/lib/libvarnish/libvarnish.la
+vhp_decode_test_CFLAGS = -DDECODE_TEST_DRIVER -include config.h
+vhp_decode_test_LDADD = $(top_builddir)/lib/libvarnish/libvarnish.la
 
 noinst_PROGRAMS += esi_parse_fuzzer
 esi_parse_fuzzer_SOURCES = \
 	cache/cache_esi_parse.c \
 	fuzzers/esi_parse_fuzzer.c
 esi_parse_fuzzer_CFLAGS = \
-	@SAN_CFLAGS@ -DNOT_IN_A_VMOD -DTEST_DRIVER -include config.h
+	-DNOT_IN_A_VMOD -DTEST_DRIVER -include config.h
 esi_parse_fuzzer_LDADD = \
 	$(top_builddir)/lib/libvarnish/libvarnish.la \
 	$(top_builddir)/lib/libvgz/libvgz.a
diff --git a/bin/varnishhist/Makefile.am b/bin/varnishhist/Makefile.am
index 79e681d8c..f30528b11 100644
--- a/bin/varnishhist/Makefile.am
+++ b/bin/varnishhist/Makefile.am
@@ -12,11 +12,6 @@ varnishhist_SOURCES = \
 	varnishhist_options.h \
 	varnishhist_profiles.h
 
-varnishhist_CFLAGS = \
-	@SAN_CFLAGS@
-
 varnishhist_LDADD = \
 	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
-	-lm \
-	@SAN_LDFLAGS@ \
-	@CURSES_LIBS@ ${RT_LIBS} ${PTHREAD_LIBS}
+	-lm @CURSES_LIBS@ ${RT_LIBS} ${PTHREAD_LIBS}
diff --git a/bin/varnishlog/Makefile.am b/bin/varnishlog/Makefile.am
index 9fccdc507..41e8cfe4e 100644
--- a/bin/varnishlog/Makefile.am
+++ b/bin/varnishlog/Makefile.am
@@ -10,10 +10,6 @@ varnishlog_SOURCES = \
 	varnishlog.c \
 	varnishlog_options.h
 
-varnishlog_CFLAGS = \
-	@SAN_CFLAGS@
-
 varnishlog_LDADD = \
 	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
-	@SAN_LDFLAGS@ \
 	${RT_LIBS} ${LIBM} ${PTHREAD_LIBS}
diff --git a/bin/varnishncsa/Makefile.am b/bin/varnishncsa/Makefile.am
index 60345bec7..08c8421e9 100644
--- a/bin/varnishncsa/Makefile.am
+++ b/bin/varnishncsa/Makefile.am
@@ -12,10 +12,6 @@ varnishncsa_SOURCES = \
 	b64.h \
 	b64.c
 
-varnishncsa_CFLAGS = \
-	@SAN_CFLAGS@
-
 varnishncsa_LDADD = \
 	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
-	@SAN_LDFLAGS@ \
 	${RT_LIBS} ${LIBM}
diff --git a/bin/varnishstat/Makefile.am b/bin/varnishstat/Makefile.am
index 94ad656aa..45dc4f34e 100644
--- a/bin/varnishstat/Makefile.am
+++ b/bin/varnishstat/Makefile.am
@@ -28,14 +28,9 @@ varnishstat_curses_help.c: varnishstat_help_gen
 	$(AM_V_GEN) ./varnishstat_help_gen >$@_
 	@mv $@_ $@
 
-varnishstat_CFLAGS = \
-	@SAN_CFLAGS@
-
 varnishstat_LDADD = \
 	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
-	@SAN_LDFLAGS@ \
 	@CURSES_LIBS@ ${RT_LIBS} ${LIBM} ${PTHREAD_LIBS}
 
 varnishstat_help_gen_LDADD = \
-	$(top_builddir)/lib/libvarnish/libvarnish.la \
-	@SAN_LDFLAGS@
+	$(top_builddir)/lib/libvarnish/libvarnish.la
diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am
index 2574fb586..3a79b6fff 100644
--- a/bin/varnishtest/Makefile.am
+++ b/bin/varnishtest/Makefile.am
@@ -55,11 +55,9 @@ varnishtest_LDADD = \
 		$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
 		$(top_builddir)/lib/libvarnish/libvarnish.la \
 		$(top_builddir)/lib/libvgz/libvgz.a \
-		@SAN_LDFLAGS@ \
 		${PTHREAD_LIBS} ${NET_LIBS} ${LIBM}
 
 varnishtest_CFLAGS = \
-		@SAN_CFLAGS@ \
 		-DVTEST_WITH_VTC_LOGEXPECT \
 		-DVTEST_WITH_VTC_VARNISH \
 		-DTOP_BUILDDIR='"${top_builddir}"'
diff --git a/bin/varnishtop/Makefile.am b/bin/varnishtop/Makefile.am
index cdb63b5f8..c68eb6654 100644
--- a/bin/varnishtop/Makefile.am
+++ b/bin/varnishtop/Makefile.am
@@ -11,11 +11,6 @@ varnishtop_SOURCES = \
 	varnishtop.c \
 	varnishtop_options.h
 
-
-varnishtop_CFLAGS = \
-	@SAN_CFLAGS@
-
 varnishtop_LDADD = \
 	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
-	@SAN_LDFLAGS@ \
 	@CURSES_LIBS@ ${RT_LIBS} ${LIBM} ${PTHREAD_LIBS}
diff --git a/configure.ac b/configure.ac
index 3b0fbd5bf..5c4558548 100644
--- a/configure.ac
+++ b/configure.ac
@@ -296,26 +296,23 @@ AC_ARG_ENABLE(msan,
 		[enable memory sanitizer (default is NO)]),
 	[MSAN_FLAGS=-fsanitize=memory])
 
-SAN_CFLAGS=
-SAN_LDFLAGS=
-
 if test "x$UBSAN_FLAGS$TSAN_FLAGS$ASAN_FLAGS$MSAN_FLAGS" != "x"; then
 	AC_DEFINE([__SANITIZER], [1], [Define to 1 if any sanitizer is enabled.])
 	SAN_FLAGS="$ASAN_FLAGS $UBSAN_FLAGS $TSAN_FLAGS $MSAN_FLAGS"
 	SAN_CFLAGS="$SAN_FLAGS -fPIC -fPIE -fno-omit-frame-pointer"
-	SAN_LDFLAGS=$SAN_FLAGS
+	SAN_LDFLAGS=
 	save_CFLAGS=$CFLAGS
 	CFLAGS="${CFLAGS} -Werror=unused-command-line-argument"
-	AX_CHECK_LINK_FLAG([-pie], [SAN_LDFLAGS="$SAN_LDFLAGS -pie"])
+	AX_CHECK_LINK_FLAG([-pie], [SAN_LDFLAGS=-pie])
 	CFLAGS=$save_CFLAGS
 	case $CC in
 	gcc*)
-		SAN_CFLAGS="${SAN_CFLAGS} -fuse-ld=gold"
+		SAN_CFLAGS="$SAN_CFLAGS -fuse-ld=gold"
 		;;
 	esac
+	CFLAGS="$CFLAGS $SAN_CFLAGS"
+	LDFLAGS="$LDFLAGS $SAN_LDFLAGS"
 fi
-AC_SUBST(SAN_CFLAGS)
-AC_SUBST(SAN_LDFLAGS)
 
 # Use jemalloc on Linux
 JEMALLOC_LDADD=
diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am
index eef530795..5471895d8 100644
--- a/lib/libvarnish/Makefile.am
+++ b/lib/libvarnish/Makefile.am
@@ -5,8 +5,8 @@ AM_CPPFLAGS = \
 	-I$(top_builddir)/include \
 	@PCRE2_CFLAGS@
 
-AM_CFLAGS   = $(AM_LT_CFLAGS)  @SAN_CFLAGS@
-AM_LDFLAGS  = $(AM_LT_LDFLAGS) @SAN_LDFLAGS@
+AM_CFLAGS   = $(AM_LT_CFLAGS)
+AM_LDFLAGS  = $(AM_LT_LDFLAGS)
 
 noinst_LTLIBRARIES = libvarnish.la
 
diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index 23e9a485c..264beb1da 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -35,12 +35,11 @@ libvarnishapi_la_SOURCES += daemon.c
 endif
 
 libvarnishapi_la_CFLAGS = \
-	-DVARNISH_STATE_DIR='"${VARNISH_STATE_DIR}"' \
-	@SAN_CFLAGS@
+	-DVARNISH_STATE_DIR='"${VARNISH_STATE_DIR}"'
 
 libvarnishapi_la_LIBADD = \
 	$(top_builddir)/lib/libvarnish/libvarnish.la \
-	@SAN_LDFLAGS@ ${NET_LIBS} ${RT_LIBS} ${LIBM}
+	${NET_LIBS} ${RT_LIBS} ${LIBM}
 
 if HAVE_LD_VERSION_SCRIPT
 libvarnishapi_la_LDFLAGS += -Wl,--version-script=$(srcdir)/libvarnishapi.map
@@ -79,17 +78,15 @@ vxp_test_SOURCES = \
 	vxp_test.c
 vxp_test_CFLAGS = \
 	-DVARNISH_STATE_DIR='"${VARNISH_STATE_DIR}"' \
-	-DVXP_DEBUG \
-	${SAN_CFLAGS}
+	-DVXP_DEBUG
 vxp_test_LDADD = \
 	$(top_builddir)/lib/libvarnish/libvarnish.la \
-	${RT_LIBS} ${LIBM} ${PTHREAD_LIBS} ${SAN_LDFLAGS}
+	${RT_LIBS} ${LIBM} ${PTHREAD_LIBS}
 
 noinst_PROGRAMS += vsl_glob_test
 
 vsl_glob_test_SOURCES = vsl_glob_test.c
-vsl_glob_test_CFLAGS = @SAN_CFLAGS@
-vsl_glob_test_LDADD = libvarnishapi.la @SAN_LDFLAGS@
+vsl_glob_test_LDADD = libvarnishapi.la
 
 dist_noinst_SCRIPTS = vsl_glob_test_coverage.sh vxp_test_coverage.sh
 
diff --git a/lib/libvcc/Makefile.am b/lib/libvcc/Makefile.am
index c97d709c5..b6b8bca3e 100644
--- a/lib/libvcc/Makefile.am
+++ b/lib/libvcc/Makefile.am
@@ -8,9 +8,6 @@ AM_CPPFLAGS = \
 
 noinst_LIBRARIES = libvcc.a
 
-libvcc_a_CFLAGS = \
-	@SAN_CFLAGS@
-
 libvcc_a_SOURCES = \
 	vcc_compile.h \
 	vcc_namespace.h \
diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index e37a75966..0eb77c527 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -56,8 +56,7 @@ vmod_LTLIBRARIES += libvmod_XXX.la
 libvmod_XXX_la_SOURCES = \\
 \tSRC
 
-libvmod_XXX_la_CFLAGS = \\
-\t at SAN_CFLAGS@
+libvmod_XXX_la_CFLAGS =
 
 vmodtoolargs_XXX ?= --strict --boilerplate -o PFX
 vmod_XXX_symbols_regex ?= Vmod_XXX_Data
@@ -65,8 +64,7 @@ vmod_XXX_symbols_regex ?= Vmod_XXX_Data
 libvmod_XXX_la_LDFLAGS = \\
 \t-export-symbols-regex $(vmod_XXX_symbols_regex) \\
 \t$(AM_LDFLAGS) \\
-\t$(VMOD_LDFLAGS) \\
-\t at SAN_LDFLAGS@
+\t$(VMOD_LDFLAGS)
 
 nodist_libvmod_XXX_la_SOURCES = PFX.c PFX.h
 
diff --git a/lib/libvgz/Makefile.am b/lib/libvgz/Makefile.am
index db9a48bdc..5cee560e7 100644
--- a/lib/libvgz/Makefile.am
+++ b/lib/libvgz/Makefile.am
@@ -5,7 +5,7 @@ AM_LDFLAGS  = $(AM_LT_LDFLAGS)
 noinst_LIBRARIES = libvgz.a
 
 libvgz_a_CFLAGS = -D_LARGEFILE64_SOURCE=1 -DZLIB_CONST \
-	$(libvgz_extra_cflags) @SAN_CFLAGS@
+	$(libvgz_extra_cflags)
 
 libvgz_a_SOURCES = \
 	adler32.c \
diff --git a/vmod/automake_boilerplate_blob.am b/vmod/automake_boilerplate_blob.am
index d604614ac..c1934c9b6 100644
--- a/vmod/automake_boilerplate_blob.am
+++ b/vmod/automake_boilerplate_blob.am
@@ -12,8 +12,7 @@ libvmod_blob_la_SOURCES = \
 	vmod_blob_tbl_encodings.h \
 	vmod_blob_url.c
 
-libvmod_blob_la_CFLAGS = \
-	@SAN_CFLAGS@
+libvmod_blob_la_CFLAGS =
 
 vmodtoolargs_blob ?= --strict --boilerplate -o vcc_blob_if
 vmod_blob_symbols_regex ?= Vmod_blob_Data
@@ -21,8 +20,7 @@ vmod_blob_symbols_regex ?= Vmod_blob_Data
 libvmod_blob_la_LDFLAGS = \
 	-export-symbols-regex $(vmod_blob_symbols_regex) \
 	$(AM_LDFLAGS) \
-	$(VMOD_LDFLAGS) \
-	@SAN_LDFLAGS@
+	$(VMOD_LDFLAGS)
 
 nodist_libvmod_blob_la_SOURCES = vcc_blob_if.c vcc_blob_if.h
 
diff --git a/vmod/automake_boilerplate_cookie.am b/vmod/automake_boilerplate_cookie.am
index a1509d920..2ab718f63 100644
--- a/vmod/automake_boilerplate_cookie.am
+++ b/vmod/automake_boilerplate_cookie.am
@@ -5,8 +5,7 @@ vmod_LTLIBRARIES += libvmod_cookie.la
 libvmod_cookie_la_SOURCES = \
 	vmod_cookie.c
 
-libvmod_cookie_la_CFLAGS = \
-	@SAN_CFLAGS@
+libvmod_cookie_la_CFLAGS =
 
 vmodtoolargs_cookie ?= --strict --boilerplate -o vcc_cookie_if
 vmod_cookie_symbols_regex ?= Vmod_cookie_Data
@@ -14,8 +13,7 @@ vmod_cookie_symbols_regex ?= Vmod_cookie_Data
 libvmod_cookie_la_LDFLAGS = \
 	-export-symbols-regex $(vmod_cookie_symbols_regex) \
 	$(AM_LDFLAGS) \
-	$(VMOD_LDFLAGS) \
-	@SAN_LDFLAGS@
+	$(VMOD_LDFLAGS)
 
 nodist_libvmod_cookie_la_SOURCES = vcc_cookie_if.c vcc_cookie_if.h
 
diff --git a/vmod/automake_boilerplate_debug.am b/vmod/automake_boilerplate_debug.am
index 5617c04c9..d546d4587 100644
--- a/vmod/automake_boilerplate_debug.am
+++ b/vmod/automake_boilerplate_debug.am
@@ -8,8 +8,7 @@ libvmod_debug_la_SOURCES = \
 	vmod_debug_dyn.c \
 	vmod_debug_obj.c
 
-libvmod_debug_la_CFLAGS = \
-	@SAN_CFLAGS@
+libvmod_debug_la_CFLAGS =
 
 vmodtoolargs_debug ?= --strict --boilerplate -o vcc_debug_if
 vmod_debug_symbols_regex ?= Vmod_debug_Data
@@ -17,8 +16,7 @@ vmod_debug_symbols_regex ?= Vmod_debug_Data
 libvmod_debug_la_LDFLAGS = \
 	-export-symbols-regex $(vmod_debug_symbols_regex) \
 	$(AM_LDFLAGS) \
-	$(VMOD_LDFLAGS) \
-	@SAN_LDFLAGS@
+	$(VMOD_LDFLAGS)
 
 nodist_libvmod_debug_la_SOURCES = vcc_debug_if.c vcc_debug_if.h
 
diff --git a/vmod/automake_boilerplate_directors.am b/vmod/automake_boilerplate_directors.am
index 2a3618a43..157d06138 100644
--- a/vmod/automake_boilerplate_directors.am
+++ b/vmod/automake_boilerplate_directors.am
@@ -15,8 +15,7 @@ libvmod_directors_la_SOURCES = \
 	vmod_directors_shard_dir.c \
 	vmod_directors_shard_dir.h
 
-libvmod_directors_la_CFLAGS = \
-	@SAN_CFLAGS@
+libvmod_directors_la_CFLAGS =
 
 vmodtoolargs_directors ?= --strict --boilerplate -o vcc_directors_if
 vmod_directors_symbols_regex ?= Vmod_directors_Data
@@ -24,8 +23,7 @@ vmod_directors_symbols_regex ?= Vmod_directors_Data
 libvmod_directors_la_LDFLAGS = \
 	-export-symbols-regex $(vmod_directors_symbols_regex) \
 	$(AM_LDFLAGS) \
-	$(VMOD_LDFLAGS) \
-	@SAN_LDFLAGS@
+	$(VMOD_LDFLAGS)
 
 nodist_libvmod_directors_la_SOURCES = vcc_directors_if.c vcc_directors_if.h
 
diff --git a/vmod/automake_boilerplate_proxy.am b/vmod/automake_boilerplate_proxy.am
index d5873e989..2fa58068d 100644
--- a/vmod/automake_boilerplate_proxy.am
+++ b/vmod/automake_boilerplate_proxy.am
@@ -5,8 +5,7 @@ vmod_LTLIBRARIES += libvmod_proxy.la
 libvmod_proxy_la_SOURCES = \
 	vmod_proxy.c
 
-libvmod_proxy_la_CFLAGS = \
-	@SAN_CFLAGS@
+libvmod_proxy_la_CFLAGS =
 
 vmodtoolargs_proxy ?= --strict --boilerplate -o vcc_proxy_if
 vmod_proxy_symbols_regex ?= Vmod_proxy_Data
@@ -14,8 +13,7 @@ vmod_proxy_symbols_regex ?= Vmod_proxy_Data
 libvmod_proxy_la_LDFLAGS = \
 	-export-symbols-regex $(vmod_proxy_symbols_regex) \
 	$(AM_LDFLAGS) \
-	$(VMOD_LDFLAGS) \
-	@SAN_LDFLAGS@
+	$(VMOD_LDFLAGS)
 
 nodist_libvmod_proxy_la_SOURCES = vcc_proxy_if.c vcc_proxy_if.h
 
diff --git a/vmod/automake_boilerplate_purge.am b/vmod/automake_boilerplate_purge.am
index 2146f4055..35b893abb 100644
--- a/vmod/automake_boilerplate_purge.am
+++ b/vmod/automake_boilerplate_purge.am
@@ -5,8 +5,7 @@ vmod_LTLIBRARIES += libvmod_purge.la
 libvmod_purge_la_SOURCES = \
 	vmod_purge.c
 
-libvmod_purge_la_CFLAGS = \
-	@SAN_CFLAGS@
+libvmod_purge_la_CFLAGS =
 
 vmodtoolargs_purge ?= --strict --boilerplate -o vcc_purge_if
 vmod_purge_symbols_regex ?= Vmod_purge_Data
@@ -14,8 +13,7 @@ vmod_purge_symbols_regex ?= Vmod_purge_Data
 libvmod_purge_la_LDFLAGS = \
 	-export-symbols-regex $(vmod_purge_symbols_regex) \
 	$(AM_LDFLAGS) \
-	$(VMOD_LDFLAGS) \
-	@SAN_LDFLAGS@
+	$(VMOD_LDFLAGS)
 
 nodist_libvmod_purge_la_SOURCES = vcc_purge_if.c vcc_purge_if.h
 
diff --git a/vmod/automake_boilerplate_std.am b/vmod/automake_boilerplate_std.am
index b035ba7b7..57d87f773 100644
--- a/vmod/automake_boilerplate_std.am
+++ b/vmod/automake_boilerplate_std.am
@@ -8,8 +8,7 @@ libvmod_std_la_SOURCES = \
 	vmod_std_fileread.c \
 	vmod_std_querysort.c
 
-libvmod_std_la_CFLAGS = \
-	@SAN_CFLAGS@
+libvmod_std_la_CFLAGS =
 
 vmodtoolargs_std ?= --strict --boilerplate -o vcc_std_if
 vmod_std_symbols_regex ?= Vmod_std_Data
@@ -17,8 +16,7 @@ vmod_std_symbols_regex ?= Vmod_std_Data
 libvmod_std_la_LDFLAGS = \
 	-export-symbols-regex $(vmod_std_symbols_regex) \
 	$(AM_LDFLAGS) \
-	$(VMOD_LDFLAGS) \
-	@SAN_LDFLAGS@
+	$(VMOD_LDFLAGS)
 
 nodist_libvmod_std_la_SOURCES = vcc_std_if.c vcc_std_if.h
 
diff --git a/vmod/automake_boilerplate_unix.am b/vmod/automake_boilerplate_unix.am
index 1b0274e54..be3afe8cf 100644
--- a/vmod/automake_boilerplate_unix.am
+++ b/vmod/automake_boilerplate_unix.am
@@ -6,8 +6,7 @@ libvmod_unix_la_SOURCES = \
 	vmod_unix.c \
 	vmod_unix_cred_compat.h
 
-libvmod_unix_la_CFLAGS = \
-	@SAN_CFLAGS@
+libvmod_unix_la_CFLAGS =
 
 vmodtoolargs_unix ?= --strict --boilerplate -o vcc_unix_if
 vmod_unix_symbols_regex ?= Vmod_unix_Data
@@ -15,8 +14,7 @@ vmod_unix_symbols_regex ?= Vmod_unix_Data
 libvmod_unix_la_LDFLAGS = \
 	-export-symbols-regex $(vmod_unix_symbols_regex) \
 	$(AM_LDFLAGS) \
-	$(VMOD_LDFLAGS) \
-	@SAN_LDFLAGS@
+	$(VMOD_LDFLAGS)
 
 nodist_libvmod_unix_la_SOURCES = vcc_unix_if.c vcc_unix_if.h
 
diff --git a/vmod/automake_boilerplate_vtc.am b/vmod/automake_boilerplate_vtc.am
index efe418b00..2d8d9a666 100644
--- a/vmod/automake_boilerplate_vtc.am
+++ b/vmod/automake_boilerplate_vtc.am
@@ -5,8 +5,7 @@ vmod_LTLIBRARIES += libvmod_vtc.la
 libvmod_vtc_la_SOURCES = \
 	vmod_vtc.c
 
-libvmod_vtc_la_CFLAGS = \
-	@SAN_CFLAGS@
+libvmod_vtc_la_CFLAGS =
 
 vmodtoolargs_vtc ?= --strict --boilerplate -o vcc_vtc_if
 vmod_vtc_symbols_regex ?= Vmod_vtc_Data
@@ -14,8 +13,7 @@ vmod_vtc_symbols_regex ?= Vmod_vtc_Data
 libvmod_vtc_la_LDFLAGS = \
 	-export-symbols-regex $(vmod_vtc_symbols_regex) \
 	$(AM_LDFLAGS) \
-	$(VMOD_LDFLAGS) \
-	@SAN_LDFLAGS@
+	$(VMOD_LDFLAGS)
 
 nodist_libvmod_vtc_la_SOURCES = vcc_vtc_if.c vcc_vtc_if.h
 

From dridi.boukelmoune at gmail.com  Wed Aug 25 10:21:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 25 Aug 2021 10:21:05 +0000 (UTC)
Subject: [master] 9eeb5ffdd build: Don't involve C flags to include config.h
Message-ID: <20210825102105.2EED9932FC@lists.varnish-cache.org>


commit 9eeb5ffdd1bf099ed194cc83d3b969fc1ba918b9
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 24 09:55:25 2021 +0200

    build: Don't involve C flags to include config.h
    
    Spotted this unusual construct while sweeping through the sanitizer flags.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index ea211f8c3..ba666267f 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -180,17 +180,16 @@ endif
 
 noinst_PROGRAMS = vhp_gen_hufdec
 vhp_gen_hufdec_SOURCES = hpack/vhp_gen_hufdec.c
-vhp_gen_hufdec_CFLAGS = -include config.h
 vhp_gen_hufdec_LDADD = $(top_builddir)/lib/libvarnish/libvarnish.la
 
 noinst_PROGRAMS += vhp_table_test
 vhp_table_test_SOURCES = hpack/vhp_table.c
-vhp_table_test_CFLAGS = -DTABLE_TEST_DRIVER -include config.h
+vhp_table_test_CFLAGS = -DTABLE_TEST_DRIVER
 vhp_table_test_LDADD = $(top_builddir)/lib/libvarnish/libvarnish.la
 
 noinst_PROGRAMS += vhp_decode_test
 vhp_decode_test_SOURCES = hpack/vhp_decode.c hpack/vhp_table.c
-vhp_decode_test_CFLAGS = -DDECODE_TEST_DRIVER -include config.h
+vhp_decode_test_CFLAGS = -DDECODE_TEST_DRIVER
 vhp_decode_test_LDADD = $(top_builddir)/lib/libvarnish/libvarnish.la
 
 noinst_PROGRAMS += esi_parse_fuzzer
@@ -198,7 +197,7 @@ esi_parse_fuzzer_SOURCES = \
 	cache/cache_esi_parse.c \
 	fuzzers/esi_parse_fuzzer.c
 esi_parse_fuzzer_CFLAGS = \
-	-DNOT_IN_A_VMOD -DTEST_DRIVER -include config.h
+	-DNOT_IN_A_VMOD -DTEST_DRIVER
 esi_parse_fuzzer_LDADD = \
 	$(top_builddir)/lib/libvarnish/libvarnish.la \
 	$(top_builddir)/lib/libvgz/libvgz.a
diff --git a/bin/varnishd/fuzzers/esi_parse_fuzzer.c b/bin/varnishd/fuzzers/esi_parse_fuzzer.c
index a0ddaad11..2cc659778 100644
--- a/bin/varnishd/fuzzers/esi_parse_fuzzer.c
+++ b/bin/varnishd/fuzzers/esi_parse_fuzzer.c
@@ -30,6 +30,8 @@
  * ESI parser fuzzer.
  */
 
+#include "config.h"
+
 #include <string.h>
 #include <stdlib.h>
 #include <stdio.h>
diff --git a/bin/varnishd/hpack/vhp_gen_hufdec.c b/bin/varnishd/hpack/vhp_gen_hufdec.c
index bfc052816..ae2b812e4 100644
--- a/bin/varnishd/hpack/vhp_gen_hufdec.c
+++ b/bin/varnishd/hpack/vhp_gen_hufdec.c
@@ -28,6 +28,8 @@
  * SUCH DAMAGE.
  */
 
+#include "config.h"
+
 #include <stdlib.h>
 #include <ctype.h>
 #include <stdint.h>

From dridi.boukelmoune at gmail.com  Wed Aug 25 10:21:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 25 Aug 2021 10:21:05 +0000 (UTC)
Subject: [master] db18d8408 build: Split --enable-debugging-symbols setup
Message-ID: <20210825102105.46D8793302@lists.varnish-cache.org>


commit db18d84089f7704fee3ac34ec73cbb2f80ce5978
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 24 10:37:25 2021 +0200

    build: Split --enable-debugging-symbols setup
    
    Similarly to how the ASAN setup was split in two steps.

diff --git a/configure.ac b/configure.ac
index 5c4558548..b2c81109a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -771,12 +771,18 @@ esac
 
 # --enable-debugging-symbols
 AC_ARG_ENABLE(debugging-symbols,
-	AS_HELP_STRING([--enable-debugging-symbols],[enable debugging symbols (default is NO)]),
+	AS_HELP_STRING([--enable-debugging-symbols],
+		[enable debugging symbols (default is NO)]),
+	    [],
+	    [enable_debugging_symbols=no])
+
+if test "$enable_debugging_symbols" != no; then
 	if test "x$SUNCC" = "xyes" ; then
 		CFLAGS="${CFLAGS} -O0 -g"
 	else
 		CFLAGS="${CFLAGS} -O0 -g -fno-inline"
-	fi)
+	fi
+fi
 
 AC_SUBST(AM_LT_LDFLAGS)
 

From dridi.boukelmoune at gmail.com  Wed Aug 25 10:21:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 25 Aug 2021 10:21:05 +0000 (UTC)
Subject: [master] 69c9e7e7f build: New configure --enable-coverage option
Message-ID: <20210825102105.65C3593313@lists.varnish-cache.org>


commit 69c9e7e7f881885f57aee81ed5bb28aee6cf7617
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 24 15:55:20 2021 +0200

    build: New configure --enable-coverage option
    
    Some of the autoconf logic is originally from Asad. In addition, the
    explicit call to __gcov_flush() is now vendored in vdef.h like other
    toolchain abstractions.

diff --git a/.gitignore b/.gitignore
index 4f6a67eaf..8e7e13ed0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -125,6 +125,10 @@ cscope.*out
 /lib/libvarnishapi/vsl_glob_test
 /lib/libvarnishapi/vxp_test
 
+# GCOV droppings
+*.gcda
+*.gcno
+
 # vtc-bisect.sh default vtc
 /bisect.vtc
 
diff --git a/Makefile.am b/Makefile.am
index 1c3bbfff9..d6fbc0139 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -42,6 +42,9 @@ install-data-local:
 	$(install_sh) -d -m 0755 $(DESTDIR)$(localstatedir)/varnish
 
 
+distclean-local:
+	find . '(' -name '*.gcda' -o -name '*.gcda' ')' -exec rm '{}' ';'
+
 distcleancheck_listfiles = \
 	find . -type f -exec sh -c 'test -f $(srcdir)/$$1 || echo $$1' \
 		sh '{}' ';'
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 30235976f..799cf354c 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -65,10 +65,6 @@
  *
  */
 
-#ifdef GCOVING
-    int __gcov_flush(void);
-#endif
-
 static struct vsb pan_vsb_storage, *pan_vsb;
 static pthread_mutex_t panicstr_mtx;
 
@@ -812,9 +808,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond,
 	VSB_cat(pan_vsb, "\n");
 	VSB_putc(pan_vsb, '\0');	/* NUL termination */
 
-#ifdef GCOVING
-	__gcov_flush();
-#endif
+	v_gcov_flush();
 	abort();
 }
 
diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index ab889506f..5692335a5 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -63,10 +63,6 @@ static const struct cli_cmd_desc *cmds[] = {
 #include "tbl/cli_cmds.h"
 };
 
-#ifdef GCOVING
-    int __gcov_flush(void);
-#endif
-
 static const int ncmds = sizeof cmds / sizeof cmds[0];
 
 static int		cli_i = -1, cli_o = -1;
@@ -112,9 +108,7 @@ mcf_panic(struct cli *cli, const char * const *av, void *priv)
 	(void)cli;
 	(void)av;
 	(void)priv;
-#ifdef GCOVING
-	__gcov_flush();
-#endif
+	v_gcov_flush();
 	AZ(strcmp("", "You asked for it"));
 	/* NOTREACHED */
 	abort();
diff --git a/configure.ac b/configure.ac
index b2c81109a..2d2e0416a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -769,6 +769,13 @@ case $CFLAGS in
     ;;
 esac
 
+# --enable-coverage
+AC_ARG_ENABLE(coverage,
+	AS_HELP_STRING([--enable-coverage],
+		[enable coverage (implies debugging symbols, default is NO)]),
+	    [],
+	    [enable_coverage=no])
+
 # --enable-debugging-symbols
 AC_ARG_ENABLE(debugging-symbols,
 	AS_HELP_STRING([--enable-debugging-symbols],
@@ -776,6 +783,25 @@ AC_ARG_ENABLE(debugging-symbols,
 	    [],
 	    [enable_debugging_symbols=no])
 
+if test "$enable_coverage" != no; then
+	AC_DEFINE([GCOVING], [1], [Define to 1 if code coverage is enabled.])
+	save_CFLAGS=$CFLAGS
+	CFLAGS=
+	AX_CHECK_COMPILE_FLAG([--coverage],
+		[COV_FLAGS=--coverage],
+		[AX_CHECK_COMPILE_FLAG([-fprofile-arcs -ftest-coverage],
+			[COV_FLAGS="-fprofile-arcs -ftest-coverage"])])
+	AX_CHECK_COMPILE_FLAG([-fprofile-abs-path],
+		[COV_FLAGS="$COV_FLAGS -fprofile-abs-path"])
+	AX_CHECK_COMPILE_FLAG([-fPIC], [COV_FLAGS="$COV_FLAGS -fPIC"])
+	CFLAGS=$COV_FLAGS
+	AC_CHECK_FUNCS([__gcov_flush])
+	AC_CHECK_FUNCS([__gcov_dump])
+	AC_CHECK_FUNCS([__llvm_gcov_flush])
+	CFLAGS="$save_CFLAGS $COV_FLAGS"
+	enable_debugging_symbols=yes
+fi
+
 if test "$enable_debugging_symbols" != no; then
 	if test "x$SUNCC" = "xyes" ; then
 		CFLAGS="${CFLAGS} -O0 -g"
diff --git a/include/vdef.h b/include/vdef.h
index fb6ce4f7a..ae072a2c3 100644
--- a/include/vdef.h
+++ b/include/vdef.h
@@ -112,6 +112,19 @@
 #  define v_dont_optimize
 #endif
 
+#ifdef HAVE___GCOV_FLUSH
+#  define v_gcov_flush() __gcov_flush()
+int __gcov_flush(void);
+#elif defined HAVE___GCOV_DUMP
+#  define v_gcov_flush() __gcov_dump()
+void __gcov_dump(void);
+#elif defined HAVE___LLVM_GCOV_FLUSH
+#  define v_gcov_flush() __llvm_gcov_flush()
+int __llvm_gcov_flush(void);
+#else
+#  define v_gcov_flush() do { } while (0)
+#endif
+
 /*********************************************************************
  * Fundamental numerical limits
   * These limits track RFC8941

From dridi.boukelmoune at gmail.com  Wed Aug 25 10:21:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 25 Aug 2021 10:21:05 +0000 (UTC)
Subject: [master] 58bdd33af build: Derive DONT_DLCLOSE_VMODS from the
 configuration
Message-ID: <20210825102105.7D57D93318@lists.varnish-cache.org>


commit 58bdd33afa195b49057b4d9aeb8d46c1edd85777
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 24 17:44:07 2021 +0200

    build: Derive DONT_DLCLOSE_VMODS from the configuration

diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index f9236cf0e..e3ed61639 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -514,6 +514,10 @@ void VCL_TaskLeave(VRT_CTX, struct vrt_privs *);
 void VMOD_Init(void);
 void VMOD_Panic(struct vsb *);
 
+#if defined(GCOVING) || defined(__SANITIZER)
+#  define DONT_DLCLOSE_VMODS
+#endif
+
 /* cache_wrk.c */
 void WRK_Init(void);
 void WRK_AddStat(const struct worker *);

From dridi.boukelmoune at gmail.com  Wed Aug 25 10:21:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 25 Aug 2021 10:21:05 +0000 (UTC)
Subject: [master] 842c7e70f vtest.sh: Set up GCOV jobs with configure flags
Message-ID: <20210825102105.C38FA93325@lists.varnish-cache.org>


commit 842c7e70f9774109ade49c7b775f0ee64532fa48
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 24 17:53:08 2021 +0200

    vtest.sh: Set up GCOV jobs with configure flags

diff --git a/tools/vtest.sh b/tools/vtest.sh
index 6b0da37d5..ef99fadde 100755
--- a/tools/vtest.sh
+++ b/tools/vtest.sh
@@ -129,7 +129,7 @@ autogen () (
 	set -e
 	cd "${SRCDIR}"
 	nice make distclean > /dev/null 2>&1 || true
-	nice sh "${SRCDIR}"/autogen.des
+	nice sh "${SRCDIR}"/autogen.des $AUTOGEN_FLAGS
 )
 
 makedistcheck () (
@@ -207,7 +207,7 @@ if $enable_gcov ; then
 	#export CC=gcc6
 	#export CC=clang80
 	export GCOVPROG='llvm-cov gcov'
-	export CFLAGS="-fprofile-arcs -ftest-coverage -fstack-protector -DDONT_DLCLOSE_VMODS -DGCOVING"
+	export AUTOGEN_FLAGS='--enable-coverage --enable-stack-protector'
 	export MAKEFLAGS=-j1
 fi
 

From dridi.boukelmoune at gmail.com  Wed Aug 25 10:21:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 25 Aug 2021 10:21:05 +0000 (UTC)
Subject: [master] f01918f5c build: s/GCOVING/ENABLE_COVERAGE/
Message-ID: <20210825102105.E54149333E@lists.varnish-cache.org>


commit f01918f5c88cac27f3ac77a8e9cba1b0eb0101a0
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 24 17:57:29 2021 +0200

    build: s/GCOVING/ENABLE_COVERAGE/

diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index e3ed61639..4d436cd11 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -514,7 +514,7 @@ void VCL_TaskLeave(VRT_CTX, struct vrt_privs *);
 void VMOD_Init(void);
 void VMOD_Panic(struct vsb *);
 
-#if defined(GCOVING) || defined(__SANITIZER)
+#if defined(ENABLE_COVERAGE) || defined(__SANITIZER)
 #  define DONT_DLCLOSE_VMODS
 #endif
 
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index cdeaa3d18..af70b1b09 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -688,7 +688,7 @@ MCF_InitParams(struct cli *cli)
 	low = sysconf(_SC_THREAD_STACK_MIN);
 	MCF_ParamConf(MCF_MINIMUM, "thread_pool_stack", "%jdb", (intmax_t)low);
 
-#if defined(__SANITIZER) || __has_feature(address_sanitizer) || defined(GCOVING)
+#if defined(__SANITIZER) || __has_feature(address_sanitizer) || defined(ENABLE_COVERAGE)
 	def = 192 * 1024;
 #endif
 
diff --git a/configure.ac b/configure.ac
index 2d2e0416a..ab32052f4 100644
--- a/configure.ac
+++ b/configure.ac
@@ -784,7 +784,7 @@ AC_ARG_ENABLE(debugging-symbols,
 	    [enable_debugging_symbols=no])
 
 if test "$enable_coverage" != no; then
-	AC_DEFINE([GCOVING], [1], [Define to 1 if code coverage is enabled.])
+	AC_DEFINE([ENABLE_COVERAGE], [1], [Define to 1 if code coverage is enabled.])
 	save_CFLAGS=$CFLAGS
 	CFLAGS=
 	AX_CHECK_COMPILE_FLAG([--coverage],

From dridi.boukelmoune at gmail.com  Wed Aug 25 10:21:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 25 Aug 2021 10:21:06 +0000 (UTC)
Subject: [master] c7eaf5d2a varnishtest: New coverage feature check
Message-ID: <20210825102106.0CDBB9334E@lists.varnish-cache.org>


commit c7eaf5d2a0a0a08e31a8a2547dfe00ff20f52e99
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 24 18:00:42 2021 +0200

    varnishtest: New coverage feature check

diff --git a/bin/varnishtest/tests/v00004.vtc b/bin/varnishtest/tests/v00004.vtc
index 805543c55..3e8bca71c 100644
--- a/bin/varnishtest/tests/v00004.vtc
+++ b/bin/varnishtest/tests/v00004.vtc
@@ -2,9 +2,7 @@ varnishtest "test if our default paramers make sense ..."
 
 feature 64bit
 feature !sanitizer
-
-# Skip this test in GCOV mode, 68xx bytes extra per level makes it fail
-feature cmd {test -z "$GCOVPROG"}
+feature !coverage
 
 # ... for our definition of a standard use case:
 # - 2019 header madness
diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c
index eaf9a99a7..f19714c6b 100644
--- a/bin/varnishtest/vtc_misc.c
+++ b/bin/varnishtest/vtc_misc.c
@@ -438,6 +438,8 @@ addr_no_randomize_works(void)
  *        recognized as a macro.
  * persistent_storage
  *        Varnish was built with the deprecated persistent storage.
+ * coverage
+ *        Varnish was built with code coverage enabled.
  * sanitizer
  *        Varnish was built with a sanitizer.
  *
@@ -449,6 +451,12 @@ addr_no_randomize_works(void)
  * run a test with strings of the form "${...}".
  */
 
+#if ENABLE_COVERAGE
+static const unsigned coverage = 1;
+#else
+static const unsigned coverage = 0;
+#endif
+
 #if WITH_PERSISTENT_STORAGE
 static const unsigned with_persistent_storage = 1;
 #else
@@ -513,6 +521,7 @@ cmd_feature(CMD_ARGS)
 		FEATURE("user_vcache", getpwnam("vcache") != NULL);
 		FEATURE("group_varnish", getgrnam("varnish") != NULL);
 		FEATURE("persistent_storage", with_persistent_storage);
+		FEATURE("coverage", coverage);
 		FEATURE("sanitizer", sanitizer);
 		FEATURE("SO_RCVTIMEO_WORKS", so_rcvtimeo_works);
 

From dridi.boukelmoune at gmail.com  Wed Aug 25 10:21:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 25 Aug 2021 10:21:06 +0000 (UTC)
Subject: [master] a4ae320cf build: s/__SANITIZER/ENABLE_SANITIZER/
Message-ID: <20210825102106.566B093370@lists.varnish-cache.org>


commit a4ae320cfdf1cdae065f8c4f584a78737ae69d94
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 24 18:06:53 2021 +0200

    build: s/__SANITIZER/ENABLE_SANITIZER/

diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 799cf354c..190ac90b7 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -614,7 +614,7 @@ pan_backtrace(struct vsb *vsb)
 
 #else /* WITH_UNWIND */
 
-#if __SANITIZER
+#if ENABLE_SANITIZER
 #  define BACKTRACE_LEVELS	20
 #else
 #  define BACKTRACE_LEVELS	10
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index 4d436cd11..df9ca6286 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -514,7 +514,7 @@ void VCL_TaskLeave(VRT_CTX, struct vrt_privs *);
 void VMOD_Init(void);
 void VMOD_Panic(struct vsb *);
 
-#if defined(ENABLE_COVERAGE) || defined(__SANITIZER)
+#if defined(ENABLE_COVERAGE) || defined(ENABLE_SANITIZER)
 #  define DONT_DLCLOSE_VMODS
 #endif
 
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index af70b1b09..d80033269 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -688,7 +688,7 @@ MCF_InitParams(struct cli *cli)
 	low = sysconf(_SC_THREAD_STACK_MIN);
 	MCF_ParamConf(MCF_MINIMUM, "thread_pool_stack", "%jdb", (intmax_t)low);
 
-#if defined(__SANITIZER) || __has_feature(address_sanitizer) || defined(ENABLE_COVERAGE)
+#if defined(ENABLE_SANITIZER) || defined(ENABLE_COVERAGE)
 	def = 192 * 1024;
 #endif
 
diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c
index f19714c6b..35df80fb8 100644
--- a/bin/varnishtest/vtc_misc.c
+++ b/bin/varnishtest/vtc_misc.c
@@ -457,16 +457,16 @@ static const unsigned coverage = 1;
 static const unsigned coverage = 0;
 #endif
 
-#if WITH_PERSISTENT_STORAGE
-static const unsigned with_persistent_storage = 1;
+#if ENABLE_SANITIZER
+static const unsigned sanitizer = 1;
 #else
-static const unsigned with_persistent_storage = 0;
+static const unsigned sanitizer = 0;
 #endif
 
-#if __SANITIZER
-static const unsigned sanitizer = 1;
+#if WITH_PERSISTENT_STORAGE
+static const unsigned with_persistent_storage = 1;
 #else
-static const unsigned sanitizer = 0;
+static const unsigned with_persistent_storage = 0;
 #endif
 
 #ifdef SO_RCVTIMEO_WORKS
diff --git a/configure.ac b/configure.ac
index ab32052f4..d34fdf8c0 100644
--- a/configure.ac
+++ b/configure.ac
@@ -297,7 +297,8 @@ AC_ARG_ENABLE(msan,
 	[MSAN_FLAGS=-fsanitize=memory])
 
 if test "x$UBSAN_FLAGS$TSAN_FLAGS$ASAN_FLAGS$MSAN_FLAGS" != "x"; then
-	AC_DEFINE([__SANITIZER], [1], [Define to 1 if any sanitizer is enabled.])
+	AC_DEFINE([ENABLE_SANITIZER], [1],
+		[Define to 1 if any sanitizer is enabled.])
 	SAN_FLAGS="$ASAN_FLAGS $UBSAN_FLAGS $TSAN_FLAGS $MSAN_FLAGS"
 	SAN_CFLAGS="$SAN_FLAGS -fPIC -fPIE -fno-omit-frame-pointer"
 	SAN_LDFLAGS=
diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index 1dca3c6ce..9e43f5da0 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -626,7 +626,7 @@ VTCP_Check(ssize_t a)
 	if (errno == EINVAL)
 		return (1);
 #endif
-#if (defined(__SANITIZER) || __has_feature(address_sanitizer))
+#if defined(ENABLE_SANITIZER)
 	if (errno == EINTR)
 		return (1);
 #endif

From dridi.boukelmoune at gmail.com  Wed Aug 25 10:21:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 25 Aug 2021 10:21:06 +0000 (UTC)
Subject: [master] e996e01e8 build: Give individual sanitizers their own C
 macros
Message-ID: <20210825102106.7C96793389@lists.varnish-cache.org>


commit e996e01e82177d23288d7eb91e103ad9a9ffb09c
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 24 18:10:05 2021 +0200

    build: Give individual sanitizers their own C macros

diff --git a/configure.ac b/configure.ac
index d34fdf8c0..4bcecd4bf 100644
--- a/configure.ac
+++ b/configure.ac
@@ -273,17 +273,29 @@ CFLAGS="${save_CFLAGS}"
 AC_ARG_ENABLE(ubsan,
 	AS_HELP_STRING([--enable-ubsan],
 		[enable undefined behavior sanitizer (default is NO)]),
-	[UBSAN_FLAGS=-fsanitize=undefined])
+	[
+		AC_DEFINE([ENABLE_UBSAN], [1],
+			[Define to 1 if UBSAN is enabled.])
+		UBSAN_FLAGS=-fsanitize=undefined
+	])
 
 AC_ARG_ENABLE(tsan,
 	AS_HELP_STRING([--enable-tsan],
 		[enable thread sanitizer (default is NO)]),
-	[TSAN_FLAGS=-fsanitize=thread])
+	[
+		AC_DEFINE([ENABLE_TSAN], [1],
+			[Define to 1 if TSAN is enabled.])
+		TSAN_FLAGS=-fsanitize=thread
+	])
 
 AC_ARG_ENABLE(asan,
 	AS_HELP_STRING([--enable-asan],
 		[enable address sanitizer (default is NO)]),
-	[ASAN_FLAGS=-fsanitize=address])
+	[
+		AC_DEFINE([ENABLE_ASAN], [1],
+			[Define to 1 if ASAN sanitizer is enabled.])
+		ASAN_FLAGS=-fsanitize=address
+	])
 
 if test -n "$ASAN_FLAGS"; then
 	AX_CHECK_COMPILE_FLAG(
@@ -294,7 +306,11 @@ fi
 AC_ARG_ENABLE(msan,
 	AS_HELP_STRING([--enable-msan],
 		[enable memory sanitizer (default is NO)]),
-	[MSAN_FLAGS=-fsanitize=memory])
+	[
+		AC_DEFINE([ENABLE_MSAN], [1],
+			[Define to 1 if MSAN is enabled.])
+		MSAN_FLAGS=-fsanitize=memory
+	])
 
 if test "x$UBSAN_FLAGS$TSAN_FLAGS$ASAN_FLAGS$MSAN_FLAGS" != "x"; then
 	AC_DEFINE([ENABLE_SANITIZER], [1],

From dridi.boukelmoune at gmail.com  Wed Aug 25 10:21:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 25 Aug 2021 10:21:06 +0000 (UTC)
Subject: [master] dbc7fa74b varnishtest: Add feature checks per sanitizer
Message-ID: <20210825102106.A3BF093397@lists.varnish-cache.org>


commit dbc7fa74b52976b0ddffa7457936c056d2e01fa8
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 24 18:15:57 2021 +0200

    varnishtest: Add feature checks per sanitizer

diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c
index 35df80fb8..f72963428 100644
--- a/bin/varnishtest/vtc_misc.c
+++ b/bin/varnishtest/vtc_misc.c
@@ -440,6 +440,14 @@ addr_no_randomize_works(void)
  *        Varnish was built with the deprecated persistent storage.
  * coverage
  *        Varnish was built with code coverage enabled.
+ * asan
+ *        Varnish was built with the address sanitizer.
+ * msan
+ *        Varnish was built with the memory sanitizer.
+ * tsan
+ *        Varnish was built with the thread sanitizer.
+ * ubsan
+ *        Varnish was built with the undefined behavior sanitizer.
  * sanitizer
  *        Varnish was built with a sanitizer.
  *
@@ -457,6 +465,30 @@ static const unsigned coverage = 1;
 static const unsigned coverage = 0;
 #endif
 
+#if ENABLE_ASAN
+static const unsigned asan = 1;
+#else
+static const unsigned asan = 0;
+#endif
+
+#if ENABLE_MSAN
+static const unsigned msan = 1;
+#else
+static const unsigned msan = 0;
+#endif
+
+#if ENABLE_TSAN
+static const unsigned tsan = 1;
+#else
+static const unsigned tsan = 0;
+#endif
+
+#if ENABLE_UBSAN
+static const unsigned ubsan = 1;
+#else
+static const unsigned ubsan = 0;
+#endif
+
 #if ENABLE_SANITIZER
 static const unsigned sanitizer = 1;
 #else
@@ -522,6 +554,10 @@ cmd_feature(CMD_ARGS)
 		FEATURE("group_varnish", getgrnam("varnish") != NULL);
 		FEATURE("persistent_storage", with_persistent_storage);
 		FEATURE("coverage", coverage);
+		FEATURE("asan", asan);
+		FEATURE("msan", msan);
+		FEATURE("tsan", tsan);
+		FEATURE("ubsan", ubsan);
 		FEATURE("sanitizer", sanitizer);
 		FEATURE("SO_RCVTIMEO_WORKS", so_rcvtimeo_works);
 

From dridi.boukelmoune at gmail.com  Thu Aug 26 12:53:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 26 Aug 2021 12:53:06 +0000 (UTC)
Subject: [master] dea029c3f git: Properly ignore vrt_test droppings
Message-ID: <20210826125306.85146A03D4@lists.varnish-cache.org>


commit dea029c3fd0e80ce735e5763a1007d555ac47277
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Aug 26 14:50:56 2021 +0200

    git: Properly ignore vrt_test droppings

diff --git a/.gitignore b/.gitignore
index 8e7e13ed0..b36966a18 100644
--- a/.gitignore
+++ b/.gitignore
@@ -113,9 +113,7 @@ cscope.*out
 /bin/varnishd/vhp_decode_test
 /bin/varnishd/vhp_table_test
 /bin/varnishtest/tests/*.log-t
-/include/_vrt.c
-/include/_vrt_test
-/include/vrt_test
+/include/vrt_test*
 /include/vbm_test
 /lib/libvarnish/vav_test
 /lib/libvarnish/vbh_test

From dridi.boukelmoune at gmail.com  Fri Aug 27 15:30:09 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 27 Aug 2021 15:30:09 +0000 (UTC)
Subject: [master] 2bb355c73 vrt: Fix use-after-release
Message-ID: <20210827153009.24932A5D07@lists.varnish-cache.org>


commit 2bb355c73c9f1c15c0f20e0d58324d9b381f2b55
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Aug 25 19:00:21 2021 +0200

    vrt: Fix use-after-release

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index fa62c8957..e0eafe374 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -627,8 +627,8 @@ VRT_SetHdr(VRT_CTX , VCL_HEADER hs, const char *pfx, VCL_STRANDS s)
 		p += l;
 	}
 	if (FEATURE(FEATURE_VALIDATE_HEADERS) && !validhdr(b)) {
-		WS_Release(hp->ws, 0);
 		VRT_fail(ctx, "Bad header %s", b);
+		WS_Release(hp->ws, 0);
 		return;
 	}
 	WS_ReleaseP(hp->ws, strchr(p, '\0') + 1);

From dridi.boukelmoune at gmail.com  Fri Aug 27 15:30:09 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 27 Aug 2021 15:30:09 +0000 (UTC)
Subject: [master] 5b4f0f1ad htc: Defer workspace rollbacks for request tasks
Message-ID: <20210827153009.7153AA5D0C@lists.varnish-cache.org>


commit 5b4f0f1addaab9c4cedb5a2e0cdcc97fe58836d4
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Aug 25 16:17:23 2021 +0200

    htc: Defer workspace rollbacks for request tasks
    
    When we take on a new request on a connection from which something was
    already received, we need to pipeline it and we do so at the beginning
    of the request workspace. There's a high probability that the pipeline
    is coming from the same workspace, which is a form of use-after-free
    only made safe by the workspace implementation details.
    
    To avoid the conceptual use-after-free, we defer req workspace rollbacks
    and perform them during the next HTC_RxInit() call before the pipelining
    operation.
    
    Because HTTP/1 works directly on the session, a worker can safely switch
    back and forth between sess and req tasks. This means that unless the
    session goes idle the same workspace is used from one client request
    to the next, hence the rollback previously happening in Req_Cleanup().
    
    With h2 however there is a disconnect between the session and streams.
    The connection is received in req0's workspace, and then copied into
    a stream's req workspace via the pipelining scheme. Rollbacks can be
    deferred as well, but they need to happen otherwise the session will
    soon overflow. Independent HTC_RxInit() calls happen for req0 in the
    h2 session thread, and for h2 streams in the regular request task code
    path.
    
    PROXY Protocol parsing may result in receiving more than the proxy
    preamble itself and pipelining will happen, whether it is via a req
    for HTTP/1 or req0 for h2.
    
    On the other end of the spectrum when Varnish acts as a client it
    only sends one HTTP/1 request at a time for a given connection, so
    we never expect pipelining to occur in fetch task.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 8c641e5e4..6f6f3d637 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -153,7 +153,6 @@ Bereq_Rollback(VRT_CTX)
 	HTTP_Clone(bo->bereq, bo->bereq0);
 	bo->vfp_filter_list = NULL;
 	bo->err_reason = NULL;
-	WS_Rollback(bo->bereq->ws, bo->ws_bo);
 	WS_Rollback(bo->ws, bo->ws_bo);
 }
 
diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c
index eeb339b62..5a5e752f7 100644
--- a/bin/varnishd/cache/cache_req.c
+++ b/bin/varnishd/cache/cache_req.c
@@ -261,10 +261,6 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 
 	if (WS_Overflowed(req->ws))
 		wrk->stats->ws_client_overflow++;
-
-	/* no snapshot for h2 stream 0 */
-	if (req->ws_req)
-		WS_Rollback(req->ws, req->ws_req);
 }
 
 /*----------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index f8215dc13..d6a711937 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -228,6 +228,12 @@ HTC_RxInit(struct http_conn *htc, struct ws *ws)
 
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
 	htc->ws = ws;
+
+	if (!strcasecmp(ws->id, "req"))
+		WS_Rollback(ws, 0);
+	else
+		AZ(htc->pipeline_b);
+
 	r = WS_ReserveAll(htc->ws);
 	htc->rxbuf_b = htc->rxbuf_e = WS_Reservation(ws);
 	if (htc->pipeline_b != NULL) {
diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c
index 88bc74a95..21b9a0e4e 100644
--- a/bin/varnishd/http2/cache_http2_session.c
+++ b/bin/varnishd/http2/cache_http2_session.c
@@ -368,7 +368,6 @@ h2_new_session(struct worker *wrk, void *arg)
 	}
 	assert(HTC_S_COMPLETE == H2_prism_complete(h2->htc));
 	HTC_RxPipeline(h2->htc, h2->htc->rxbuf_b + sizeof(H2_prism));
-	WS_Rollback(h2->ws, 0);
 	HTC_RxInit(h2->htc, h2->ws);
 	AN(WS_Reservation(h2->ws));
 	VSLb(h2->vsl, SLT_Debug, "H2: Got pu PRISM");
@@ -390,7 +389,6 @@ h2_new_session(struct worker *wrk, void *arg)
 	h2->cond = &wrk->cond;
 
 	while (h2_rxframe(wrk, h2)) {
-		WS_Rollback(h2->ws, 0);
 		HTC_RxInit(h2->htc, h2->ws);
 		if (WS_Overflowed(h2->ws)) {
 			VSLb(h2->vsl, SLT_Debug, "H2: Empty Rx Workspace");
diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c
index 29e806826..296e0e20b 100644
--- a/bin/varnishd/proxy/cache_proxy_proto.c
+++ b/bin/varnishd/proxy/cache_proxy_proto.c
@@ -151,7 +151,6 @@ vpx_proto1(const struct worker *wrk, const struct req *req)
 	VSL(SLT_Proxy, req->sp->vxid, "1 %s %s %s %s",
 	    fld[1], fld[3], fld[2], fld[4]);
 	HTC_RxPipeline(req->htc, q);
-	WS_Rollback(req->htc->ws, 0);
 	return (0);
 }
 
@@ -353,7 +352,6 @@ vpx_proto2(const struct worker *wrk, struct req *req)
 	hdr_len = l + 16L;
 	assert(req->htc->rxbuf_e >= req->htc->rxbuf_b + hdr_len);
 	HTC_RxPipeline(req->htc, req->htc->rxbuf_b + hdr_len);
-	WS_Rollback(req->ws, 0);
 	p = (const void *)req->htc->rxbuf_b;
 	d = req->htc->rxbuf_b + 16L;
 
diff --git a/bin/varnishtest/tests/c00108.vtc b/bin/varnishtest/tests/c00108.vtc
new file mode 100644
index 000000000..9c2014329
--- /dev/null
+++ b/bin/varnishtest/tests/c00108.vtc
@@ -0,0 +1,53 @@
+varnishtest "Session pipelining exceeding available workspace"
+
+server s1 {
+	loop 2 {
+		rxreq
+		expect req.bodylen == 32769
+		txresp
+	}
+
+	rxreq
+	expect req.bodylen == 32768
+	txresp
+} -start
+
+varnish v1 -cliok "param.set workspace_client 24k"
+varnish v1 -cliok "param.set http_req_size 1k"
+varnish v1 -cliok "param.set http_resp_size 1k"
+varnish v1 -cliok "param.set vsl_buffer 1k"
+varnish v1 -vcl+backend { } -start
+
+client c1 {
+	# Multi-line strings aren't escaped, the send argument
+	# below contains actual carriage returns. The extra body
+	# byte on top of the 32kB is a line feed.
+	send {
+POST / HTTP/1.1
+host: ${localhost}
+content-length: 32769
+
+${string,repeat,1024,abcdefghijklmnopqrstuvwxyz012345}
+POST / HTTP/1.1
+host: ${localhost}
+content-length: 32769
+
+${string,repeat,1024,abcdefghijklmnopqrstuvwxyz012345}
+}
+
+	loop 2 {
+		rxresp
+		expect resp.status == 200
+	}
+} -run
+
+varnish v1 -cliok "param.set feature +http2"
+
+client c2 {
+	stream 1 {
+		txreq -method POST -hdr content-length 32768 -nostrend
+		txdata -datalen 16384 -nostrend
+		txdata -datalen 16384
+		rxresp
+	} -run
+} -run

From dridi.boukelmoune at gmail.com  Fri Aug 27 15:30:09 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 27 Aug 2021 15:30:09 +0000 (UTC)
Subject: [master] 17ca56af5 ws: New WS_ReqPipeline()
Message-ID: <20210827153009.ADCC3A5D18@lists.varnish-cache.org>


commit 17ca56af55e11858baa154772668b387391342e4
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Jul 12 14:15:47 2021 +0200

    ws: New WS_ReqPipeline()
    
    When a session has data pipelined we perform a dirty dance to move it at
    the beginning of the workspace. The rollbacks used to occur between
    HTC_RxPipeline() and HTC_RxInit() calls until it was centralized in the
    latter.
    
    With a dedicated WS_ReqPipeline() operation we can capture the semantics
    of initializing an existing connection for its next task with or without
    data fetched from the previous task.
    
    While conceptually there is still a use-after-free since the pipelined
    data may belong to the same workspace, it is fine if that happens within
    the bounds of an atomic workspace operation.

diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index d6a711937..f3bb4da15 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -223,30 +223,16 @@ HTC_Status(enum htc_status_e e)
 void
 HTC_RxInit(struct http_conn *htc, struct ws *ws)
 {
-	unsigned r;
-	ssize_t l;
+	unsigned l;
 
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
 	htc->ws = ws;
 
-	if (!strcasecmp(ws->id, "req"))
-		WS_Rollback(ws, 0);
-	else
-		AZ(htc->pipeline_b);
-
-	r = WS_ReserveAll(htc->ws);
-	htc->rxbuf_b = htc->rxbuf_e = WS_Reservation(ws);
-	if (htc->pipeline_b != NULL) {
-		AN(htc->pipeline_e);
-		// assert(WS_Inside(ws, htc->pipeline_b, htc->pipeline_e));
-		l = htc->pipeline_e - htc->pipeline_b;
-		assert(l > 0);
-		assert(l <= r);
-		memmove(htc->rxbuf_b, htc->pipeline_b, l);
-		htc->rxbuf_e += l;
-		htc->pipeline_b = NULL;
-		htc->pipeline_e = NULL;
-	}
+	l = WS_ReqPipeline(htc->ws, htc->pipeline_b, htc->pipeline_e);
+	htc->rxbuf_b = WS_Reservation(ws);
+	htc->rxbuf_e = htc->rxbuf_b + l;
+	htc->pipeline_b = NULL;
+	htc->pipeline_e = NULL;
 }
 
 void
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index df9ca6286..24746b0f7 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -534,6 +534,7 @@ WS_IsReserved(const struct ws *ws)
 
 void WS_Rollback(struct ws *, uintptr_t);
 void *WS_AtOffset(const struct ws *ws, unsigned off, unsigned len);
+unsigned WS_ReqPipeline(struct ws *, const void *b, const void *e);
 
 static inline unsigned
 WS_ReservationOffset(const struct ws *ws)
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index 091179d47..38f439779 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -174,6 +174,37 @@ WS_Rollback(struct ws *ws, uintptr_t pp)
 	WS_Reset(ws, pp);
 }
 
+/*
+ * Make a reservation and optionally pipeline a memory region that may or
+ * may not originate from the same workspace.
+ */
+
+unsigned
+WS_ReqPipeline(struct ws *ws, const void *b, const void *e)
+{
+	unsigned r, l;
+
+	WS_Assert(ws);
+
+	if (!strcasecmp(ws->id, "req"))
+		WS_Rollback(ws, 0);
+	else
+		AZ(b);
+
+	if (b == NULL) {
+		AZ(e);
+		(void)WS_ReserveAll(ws);
+		return (0);
+	}
+
+	AN(e);
+	r = WS_ReserveAll(ws);
+	l = pdiff(b, e);
+	assert(l <= r);
+	memmove(ws->f, b, l);
+	return (l);
+}
+
 void *
 WS_Alloc(struct ws *ws, unsigned bytes)
 {

From dridi.boukelmoune at gmail.com  Fri Aug 27 15:30:09 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 27 Aug 2021 15:30:09 +0000 (UTC)
Subject: [master] cfa0983df ws_emu: Introduce the workspace emulator
Message-ID: <20210827153009.EC722A5D1D@lists.varnish-cache.org>


commit cfa0983dfe3f8582d8bcd69d061671f2afc4eadd
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jul 13 09:29:47 2021 +0200

    ws_emu: Introduce the workspace emulator
    
    The goal of the workspace emulator is to replicate the regular workspace
    behavior with individual allocations and make it work transparently.
    
    It's the successor of the workspace sanitizer from #3320 with notable
    differences:
    
    - enabled at configure time instead of run time
    - in a separate source file instead of mixed in
    - using sparse allocations instead of built-in red zones
    
    This means that the workspace emulator can be combined with regular
    sanitizer, in particular asan and lsan. If available, asan's public
    interface is used to mitigate the possible overflow of a reservation
    after some of it was released.
    
    Even without sanitizers, the fact that we integrate with jemalloc by
    default and enable its abort and junk options in varnishtest is enough
    to detect a use-after-free in some cases.
    
    With sanitizers though, the workspace emulator can observe #3550.
    
    One drawback is that the logic is split in two files, and some functions
    are identical in the two files. It might be possible to split cache_ws.c
    into something like cache_ws_alloc.c and cache_ws_util.c for example.
    
    Closes #3320
    Refs #3550
    Refs #3600

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index ba666267f..e129b9a8a 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -53,7 +53,6 @@ varnishd_SOURCES = \
 	cache/cache_vrt_vcl.c \
 	cache/cache_vrt_vmod.c \
 	cache/cache_wrk.c \
-	cache/cache_ws.c \
 	common/common_vsc.c \
 	common/common_vsmw.c \
 	hash/hash_classic.c \
@@ -110,6 +109,14 @@ varnishd_SOURCES = \
 	waiter/cache_waiter_ports.c \
 	waiter/mgt_waiter.c
 
+if ENABLE_WORKSPACE_EMULATOR
+varnishd_SOURCES += \
+	cache/cache_ws_emu.c
+else
+varnishd_SOURCES += \
+	cache/cache_ws.c
+endif
+
 if WITH_PERSISTENT_STORAGE
 varnishd_SOURCES += \
 	storage/mgt_storage_persistent.c \
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index 24746b0f7..bf246677a 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -534,16 +534,9 @@ WS_IsReserved(const struct ws *ws)
 
 void WS_Rollback(struct ws *, uintptr_t);
 void *WS_AtOffset(const struct ws *ws, unsigned off, unsigned len);
+unsigned WS_ReservationOffset(const struct ws *ws);
 unsigned WS_ReqPipeline(struct ws *, const void *b, const void *e);
 
-static inline unsigned
-WS_ReservationOffset(const struct ws *ws)
-{
-
-	AN(ws->r);
-	return (ws->f - ws->s);
-}
-
 /* http1/cache_http1_pipe.c */
 void V1P_Init(void);
 
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index 38f439779..1db047a1a 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -386,6 +386,14 @@ WS_AtOffset(const struct ws *ws, unsigned off, unsigned len)
 	return (ptr);
 }
 
+unsigned
+WS_ReservationOffset(const struct ws *ws)
+{
+
+	AN(ws->r);
+	return (ws->f - ws->s);
+}
+
 /*---------------------------------------------------------------------
  * Build a VSB on a workspace.
  * Usage pattern:
diff --git a/bin/varnishd/cache/cache_ws_emu.c b/bin/varnishd/cache/cache_ws_emu.c
new file mode 100644
index 000000000..bc9a4c67e
--- /dev/null
+++ b/bin/varnishd/cache/cache_ws_emu.c
@@ -0,0 +1,724 @@
+/*-
+ * Copyright (c) 2021 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#if HAVE_SANITIZER_ASAN_INTERFACE_H
+#  include <sanitizer/asan_interface.h>
+#endif
+
+#include "cache_varnishd.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+struct ws_alloc {
+	unsigned		magic;
+#define WS_ALLOC_MAGIC		0x22e7fd05
+	unsigned		off;
+	unsigned		len;
+	char			*ptr;
+	VTAILQ_ENTRY(ws_alloc)	list;
+};
+
+VTAILQ_HEAD(ws_alloc_head, ws_alloc);
+
+struct ws_emu {
+	unsigned		magic;
+#define WS_EMU_MAGIC		0x1c89b6ab
+	unsigned		len;
+	struct ws		*ws;
+	struct ws_alloc_head	head;
+};
+
+static const uintptr_t snap_overflowed = (uintptr_t)&snap_overflowed;
+
+static struct ws_emu *
+ws_emu(const struct ws *ws)
+{
+	struct ws_emu *we;
+
+	CAST_OBJ_NOTNULL(we, (void *)ws->s, WS_EMU_MAGIC);
+	return (we);
+}
+
+void
+WS_Assert(const struct ws *ws)
+{
+	struct ws_emu *we;
+	struct ws_alloc *wa, *wa2 = NULL;
+	size_t len;
+
+	CHECK_OBJ_NOTNULL(ws, WS_MAGIC);
+	assert(ws->s != NULL);
+	assert(PAOK(ws->s));
+	assert(ws->e != NULL);
+	assert(PAOK(ws->e));
+
+	we = ws_emu(ws);
+	len = pdiff(ws->s, ws->e);
+	assert(len == we->len);
+
+	len = 0;
+	VTAILQ_FOREACH(wa, &we->head, list) {
+		CHECK_OBJ_NOTNULL(wa, WS_ALLOC_MAGIC);
+		wa2 = wa;
+		assert(len == wa->off);
+		if (wa->ptr == ws->f || wa->ptr == NULL) /* reservation */
+			break;
+		AN(wa->len);
+		len += PRNDUP(wa->len);
+		assert(len <= we->len);
+	}
+
+	if (wa != NULL) {
+		AZ(VTAILQ_NEXT(wa, list));
+		if (wa->ptr == NULL) {
+			AZ(wa->len);
+			assert(ws->f == ws->e);
+			assert(ws->r == ws->e);
+		} else {
+			AN(wa->len);
+			assert(ws->f == wa->ptr);
+			assert(ws->r == ws->f + wa->len);
+		}
+		len += PRNDUP(wa->len);
+		assert(len <= we->len);
+	} else {
+		AZ(ws->f);
+		AZ(ws->r);
+	}
+
+	DSL(DBG_WORKSPACE, 0, "WS(%p) = (%s, %p %zu %zu %zu)",
+	    ws, ws->id, ws->s, wa2 == NULL ? 0 : wa2->off + PRNDUP(wa2->len),
+	    ws->r == NULL ? 0 : pdiff(ws->f, ws->r),
+	    pdiff(ws->s, ws->e));
+}
+
+int
+WS_Allocated(const struct ws *ws, const void *ptr, ssize_t len)
+{
+	struct ws_emu *we;
+	struct ws_alloc *wa;
+	uintptr_t p, pa;
+
+	WS_Assert(ws);
+	AN(ptr);
+	if (len < 0)
+		len = strlen(ptr) + 1;
+	p = (uintptr_t)ptr;
+	we = ws_emu(ws);
+
+	VTAILQ_FOREACH(wa, &we->head, list) {
+		pa = (uintptr_t)wa->ptr;
+		if (p >= (uintptr_t)ws->f && p <= (uintptr_t)ws->r)
+			return (1);
+		/* XXX: clang 12's ubsan triggers a pointer overflow on
+		 * the if statement below. Since the purpose is to check
+		 * that a pointer+length is within bounds of another
+		 * pointer+length it's unclear whether a pointer overflow
+		 * is relevant. Worked around for now with uintptr_t.
+		 */
+		if (p >= pa && p + len <= pa + wa->len)
+			return (1);
+	}
+	return (0);
+}
+
+void
+WS_Init(struct ws *ws, const char *id, void *space, unsigned len)
+{
+	struct ws_emu *we;
+
+	DSL(DBG_WORKSPACE, 0,
+	    "WS_Init(%p, \"%s\", %p, %u)", ws, id, space, len);
+	assert(space != NULL);
+	assert(PAOK(space));
+	assert(len >= sizeof *we);
+
+	len = PRNDDN(len - 1);
+	INIT_OBJ(ws, WS_MAGIC);
+	ws->s = space;
+	ws->e = ws->s + len;
+
+	assert(id[0] & 0x20);		// cheesy islower()
+	bstrcpy(ws->id, id);
+
+	we = space;
+	INIT_OBJ(we, WS_EMU_MAGIC);
+	VTAILQ_INIT(&we->head);
+	we->len = len;
+
+	WS_Assert(ws);
+}
+
+void
+WS_Id(const struct ws *ws, char *id)
+{
+
+	WS_Assert(ws);
+	AN(id);
+	memcpy(id, ws->id, WS_ID_SIZE);
+	id[0] |= 0x20;			// cheesy tolower()
+}
+
+void
+WS_MarkOverflow(struct ws *ws)
+{
+	CHECK_OBJ_NOTNULL(ws, WS_MAGIC);
+
+	ws->id[0] &= ~0x20;		// cheesy toupper()
+}
+
+static void
+ws_ClearOverflow(struct ws *ws)
+{
+	CHECK_OBJ_NOTNULL(ws, WS_MAGIC);
+
+	ws->id[0] |= 0x20;		// cheesy tolower()
+}
+
+static void
+ws_alloc_free(struct ws_emu *we, struct ws_alloc **wap)
+{
+	struct ws_alloc *wa;
+
+	TAKE_OBJ_NOTNULL(wa, wap, WS_ALLOC_MAGIC);
+	AZ(VTAILQ_NEXT(wa, list));
+	VTAILQ_REMOVE(&we->head, wa, list);
+	free(wa->ptr);
+	FREE_OBJ(wa);
+}
+
+void
+WS_Reset(struct ws *ws, uintptr_t pp)
+{
+	struct ws_emu *we;
+	struct ws_alloc *wa;
+	char *p;
+
+	WS_Assert(ws);
+	AN(pp);
+	if (pp == snap_overflowed) {
+		DSL(DBG_WORKSPACE, 0, "WS_Reset(%p, overflowed)", ws);
+		AN(WS_Overflowed(ws));
+		return;
+	}
+	p = (char *)pp;
+	DSL(DBG_WORKSPACE, 0, "WS_Reset(%p, %p)", ws, p);
+	AZ(ws->r);
+
+	we = ws_emu(ws);
+	while ((wa = VTAILQ_LAST(&we->head, ws_alloc_head)) != NULL &&
+	    wa->ptr != p)
+		ws_alloc_free(we, &wa);
+	if (wa == NULL)
+		assert(p == ws->s);
+
+	WS_Assert(ws);
+}
+
+void
+WS_Rollback(struct ws *ws, uintptr_t pp)
+{
+
+	WS_Assert(ws);
+
+	if (pp == 0)
+		pp = (uintptr_t)ws->s;
+	ws_ClearOverflow(ws);
+	WS_Reset(ws, pp);
+}
+
+unsigned
+WS_ReqPipeline(struct ws *ws, const void *b, const void *e)
+{
+	struct ws_emu *we;
+	struct ws_alloc *wa;
+	unsigned l;
+
+	WS_Assert(ws);
+	AZ(ws->f);
+	AZ(ws->r);
+
+	if (strcasecmp(ws->id, "req"))
+		AZ(b);
+
+	if (b == NULL) {
+		AZ(e);
+		if (!strcasecmp(ws->id, "req"))
+			WS_Rollback(ws, 0);
+		(void)WS_ReserveAll(ws);
+		return (0);
+	}
+
+	we = ws_emu(ws);
+	ALLOC_OBJ(wa, WS_ALLOC_MAGIC);
+	AN(wa);
+	wa->len = we->len;
+	wa->ptr = malloc(wa->len);
+	AN(wa->ptr);
+
+	AN(e);
+	l = pdiff(b, e);
+	assert(l <= wa->len);
+	memcpy(wa->ptr, b, l);
+
+	WS_Rollback(ws, 0);
+	ws->f = wa->ptr;
+	ws->r = ws->f + wa->len;
+	VTAILQ_INSERT_TAIL(&we->head, wa, list);
+	WS_Assert(ws);
+	return (l);
+}
+
+static struct ws_alloc *
+ws_emu_alloc(struct ws *ws, unsigned len)
+{
+	struct ws_emu *we;
+	struct ws_alloc *wa;
+	size_t off = 0;
+
+	WS_Assert(ws);
+	AZ(ws->r);
+
+	we = ws_emu(ws);
+	wa = VTAILQ_LAST(&we->head, ws_alloc_head);
+	CHECK_OBJ_ORNULL(wa, WS_ALLOC_MAGIC);
+
+	if (wa != NULL)
+		off = wa->off + PRNDUP(wa->len);
+	if (off + len > we->len) {
+		WS_MarkOverflow(ws);
+		return (NULL);
+	}
+	if (len == 0)
+		len = we->len - off;
+
+	ALLOC_OBJ(wa, WS_ALLOC_MAGIC);
+	AN(wa);
+	wa->off = off;
+	wa->len = len;
+	if (len > 0) {
+		wa->ptr = malloc(len);
+		AN(wa->ptr);
+	}
+	VTAILQ_INSERT_TAIL(&we->head, wa, list);
+	return (wa);
+}
+
+void *
+WS_Alloc(struct ws *ws, unsigned bytes)
+{
+	struct ws_alloc *wa;
+
+	assert(bytes > 0);
+	wa = ws_emu_alloc(ws, bytes);
+	WS_Assert(ws);
+	if (wa != NULL) {
+		AN(wa->ptr);
+		DSL(DBG_WORKSPACE, 0, "WS_Alloc(%p, %u) = %p",
+		    ws, bytes, wa->ptr);
+		return (wa->ptr);
+	}
+	return (NULL);
+}
+
+void *
+WS_Copy(struct ws *ws, const void *str, int len)
+{
+	struct ws_alloc *wa;
+
+	AN(str);
+	if (len == -1)
+		len = strlen(str) + 1;
+	assert(len > 0);
+	wa = ws_emu_alloc(ws, len);
+	WS_Assert(ws);
+	if (wa != NULL) {
+		AN(wa->ptr);
+		memcpy(wa->ptr, str, len);
+		DSL(DBG_WORKSPACE, 0, "WS_Copy(%p, %d) = %p",
+		    ws, len, wa->ptr);
+		return (wa->ptr);
+	}
+	return (NULL);
+}
+
+const char *
+WS_Printf(struct ws *ws, const char *fmt, ...)
+{
+	unsigned u, v;
+	va_list ap;
+	char *p;
+
+	u = WS_ReserveAll(ws);
+	p = ws->f;
+	va_start(ap, fmt);
+	v = vsnprintf(p, u, fmt, ap);
+	va_end(ap);
+	if (v >= u) {
+		WS_Release(ws, 0);
+		WS_MarkOverflow(ws);
+		p = NULL;
+	} else {
+		WS_Release(ws, v + 1);
+	}
+	return (p);
+}
+
+uintptr_t
+WS_Snapshot(struct ws *ws)
+{
+	struct ws_emu *we;
+	struct ws_alloc *wa;
+	void *p;
+
+	WS_Assert(ws);
+	assert(ws->r == NULL);
+	if (WS_Overflowed(ws)) {
+		DSL(DBG_WORKSPACE, 0, "WS_Snapshot(%p) = overflowed", ws);
+		return (snap_overflowed);
+	}
+
+	we = ws_emu(ws);
+	wa = VTAILQ_LAST(&we->head, ws_alloc_head);
+	CHECK_OBJ_ORNULL(wa, WS_ALLOC_MAGIC);
+	p = (wa == NULL ? ws->s : wa->ptr);
+	DSL(DBG_WORKSPACE, 0, "WS_Snapshot(%p) = %p", ws, p);
+	return ((uintptr_t)p);
+}
+
+unsigned
+WS_ReserveAll(struct ws *ws)
+{
+	struct ws_alloc *wa;
+	unsigned b;
+
+	wa = ws_emu_alloc(ws, 0);
+	AN(wa);
+
+	if (wa->ptr != NULL) {
+		AN(wa->len);
+		ws->f = wa->ptr;
+		ws->r = ws->f + wa->len;
+	} else {
+		ws->f = ws->r = ws->e;
+	}
+
+	b = pdiff(ws->f, ws->r);
+	DSL(DBG_WORKSPACE, 0, "WS_ReserveAll(%p) = %u", ws, b);
+	WS_Assert(ws);
+	return (b);
+}
+
+unsigned
+WS_ReserveSize(struct ws *ws, unsigned bytes)
+{
+	struct ws_emu *we;
+	struct ws_alloc *wa;
+
+	assert(bytes > 0);
+	wa = ws_emu_alloc(ws, bytes);
+	if (wa == NULL)
+		return (0);
+
+	AN(wa->ptr);
+	assert(wa->len == bytes);
+	ws->f = wa->ptr;
+	ws->r = ws->f + bytes;
+	we = ws_emu(ws);
+	DSL(DBG_WORKSPACE, 0, "WS_ReserveSize(%p, %u/%u) = %u",
+	    ws, bytes, we->len - wa->off, bytes);
+	WS_Assert(ws);
+	return (bytes);
+}
+
+unsigned
+WS_ReserveLumps(struct ws *ws, size_t sz)
+{
+	return (WS_ReserveAll(ws) / sz);
+}
+
+static void
+ws_release(struct ws *ws, unsigned bytes)
+{
+	struct ws_emu *we;
+	struct ws_alloc *wa;
+
+	WS_Assert(ws);
+	AN(ws->f);
+	AN(ws->r);
+	we = ws_emu(ws);
+	wa = VTAILQ_LAST(&we->head, ws_alloc_head);
+	AN(wa);
+	assert(bytes <= wa->len);
+	ws->f = ws->r = NULL;
+
+	if (bytes == 0) {
+		ws_alloc_free(we, &wa);
+		return;
+	}
+
+	AN(wa->ptr);
+#ifdef ASAN_POISON_MEMORY_REGION
+	ASAN_POISON_MEMORY_REGION(wa->ptr + bytes, wa->len - bytes);
+#endif
+	wa->len = bytes;
+	WS_Assert(ws);
+}
+
+void
+WS_Release(struct ws *ws, unsigned bytes)
+{
+
+	ws_release(ws, bytes);
+	DSL(DBG_WORKSPACE, 0, "WS_Release(%p, %u)", ws, bytes);
+}
+
+void
+WS_ReleaseP(struct ws *ws, const char *ptr)
+{
+	unsigned l;
+
+	WS_Assert(ws);
+	assert(ws->r != NULL);
+	assert(ptr >= ws->f);
+	assert(ptr <= ws->r);
+	l = pdiff(ws->f, ptr);
+	ws_release(ws, l);
+	DSL(DBG_WORKSPACE, 0, "WS_ReleaseP(%p, %p (%u))", ws, ptr, l);
+}
+
+int
+WS_Overflowed(const struct ws *ws)
+{
+	CHECK_OBJ_NOTNULL(ws, WS_MAGIC);
+	AN(ws->id[0]);
+
+	if (ws->id[0] & 0x20)		// cheesy islower()
+		return (0);
+	return (1);
+}
+
+void *
+WS_AtOffset(const struct ws *ws, unsigned off, unsigned len)
+{
+	struct ws_emu *we;
+	struct ws_alloc *wa;
+
+	WS_Assert(ws);
+	we = ws_emu(ws);
+
+	VTAILQ_FOREACH(wa, &we->head, list) {
+		if (wa->off == off) {
+			assert(wa->len >= len);
+			return (wa->ptr);
+		}
+	}
+
+	WRONG("invalid offset");
+	NEEDLESS(return (NULL));
+}
+
+unsigned
+WS_ReservationOffset(const struct ws *ws)
+{
+	struct ws_emu *we;
+	struct ws_alloc *wa;
+
+	WS_Assert(ws);
+	AN(ws->f);
+	AN(ws->r);
+	we = ws_emu(ws);
+	wa = VTAILQ_LAST(&we->head, ws_alloc_head);
+	AN(wa);
+	return (wa->off);
+}
+
+void
+WS_VSB_new(struct vsb *vsb, struct ws *ws)
+{
+	unsigned u;
+	static char bogus[2];	// Smallest possible vsb
+
+	AN(vsb);
+	WS_Assert(ws);
+	u = WS_ReserveAll(ws);
+	if (WS_Overflowed(ws) || u < 2)
+		AN(VSB_init(vsb, bogus, sizeof bogus));
+	else
+		AN(VSB_init(vsb, WS_Reservation(ws), u));
+}
+
+char *
+WS_VSB_finish(struct vsb *vsb, struct ws *ws, size_t *szp)
+{
+	char *p;
+
+	AN(vsb);
+	WS_Assert(ws);
+	if (!VSB_finish(vsb)) {
+		p = VSB_data(vsb);
+		if (p == ws->f) {
+			WS_Release(ws, VSB_len(vsb) + 1);
+			if (szp != NULL)
+				*szp = VSB_len(vsb);
+			VSB_fini(vsb);
+			return (p);
+		}
+	}
+	WS_MarkOverflow(ws);
+	VSB_fini(vsb);
+	WS_Release(ws, 0);
+	if (szp)
+		*szp = 0;
+	return (NULL);
+}
+
+unsigned
+WS_Dump(const struct ws *ws, char where, size_t off, void *buf, size_t len)
+{
+	struct ws_emu *we;
+	struct ws_alloc *wa;
+	unsigned l, r;
+	char *b;
+
+	WS_Assert(ws);
+	AN(buf);
+	AN(len);
+
+	if (strchr("sfr", where) == NULL) {
+		errno = EINVAL;
+		return (0);
+	}
+
+	if (where == 'r' && ws->r == NULL) {
+		errno = EAGAIN;
+		return (0);
+	}
+
+	we = ws_emu(ws);
+	wa = VTAILQ_LAST(&we->head, ws_alloc_head);
+
+	l = we->len;
+	if (where != 's' && wa != NULL) {
+		l -= wa->off;
+		if (where == 'f')
+			l -= wa->len;
+	}
+
+	if (off > l) {
+		errno = EFAULT;
+		return (0);
+	}
+
+	b = buf;
+	r = 0;
+	if (where == 'f' && ws->r != NULL) {
+		if (l > len)
+			l = len;
+		memcpy(b, wa->ptr, l);
+		b += l;
+		r += l;
+		len -= l;
+	}
+
+	if (where == 's') {
+		VTAILQ_FOREACH(wa, &we->head, list) {
+			if (len == 0)
+				break;
+			if (wa->ptr == NULL)
+				break;
+			l = wa->len;
+			if (l > len)
+				l = len;
+			memcpy(b, wa->ptr, l);
+			b += l;
+			r += l;
+			len -= l;
+		}
+	}
+
+	if (len > 0)
+		memset(b, 0xa5, len);
+	return (l);
+}
+
+static void
+ws_emu_panic(struct vsb *vsb, const struct ws *ws)
+{
+	const struct ws_emu *we;
+	const struct ws_alloc *wa;
+
+	we = (void *)ws->s;
+	if (PAN_dump_once(vsb, we, WS_EMU_MAGIC, "ws_emu"))
+		return;
+	VSB_printf(vsb, "len = %u,\n", we->len);
+
+	VTAILQ_FOREACH(wa, &we->head, list) {
+		if (PAN_dump_once_oneline(vsb, wa, WS_ALLOC_MAGIC, "ws_alloc"))
+			break;
+		VSB_printf(vsb, "off, len, ptr} = {%u, %u, %p}\n",
+		    wa->off, wa->len, wa->ptr);
+	}
+
+	VSB_indent(vsb, -2);
+	VSB_cat(vsb, "},\n");
+}
+
+void
+WS_Panic(struct vsb *vsb, const struct ws *ws)
+{
+
+	if (PAN_dump_struct(vsb, ws, WS_MAGIC, "ws"))
+		return;
+	if (ws->id[0] != '\0' && (!(ws->id[0] & 0x20)))	// cheesy islower()
+		VSB_cat(vsb, "OVERFLOWED ");
+	VSB_printf(vsb, "id = \"%s\",\n", ws->id);
+	VSB_printf(vsb, "{s, e} = {%p", ws->s);
+	if (ws->e >= ws->s)
+		VSB_printf(vsb, ", +%ld", (long) (ws->e - ws->s));
+	else
+		VSB_printf(vsb, ", %p", ws->e);
+	VSB_cat(vsb, "},\n");
+	VSB_printf(vsb, "{f, r} = {%p", ws->f);
+	if (ws->r >= ws->f)
+		VSB_printf(vsb, ", +%ld", (long) (ws->r - ws->f));
+	else
+		VSB_printf(vsb, ", %p", ws->r);
+	VSB_cat(vsb, "},\n");
+
+	ws_emu_panic(vsb, ws);
+
+	VSB_indent(vsb, -2);
+	VSB_cat(vsb, "},\n");
+}
diff --git a/bin/varnishtest/tests/b00068.vtc b/bin/varnishtest/tests/b00068.vtc
index dea15e802..0559f393e 100644
--- a/bin/varnishtest/tests/b00068.vtc
+++ b/bin/varnishtest/tests/b00068.vtc
@@ -1,5 +1,7 @@
 varnishtest "Check timeout_linger"
 
+feature !workspace_emulator
+
 # XXX this test exploits the fact that the struct waited is
 # left near the free pointer of the session ws when a session
 # made a tour over the waiter
diff --git a/bin/varnishtest/tests/m00017.vtc b/bin/varnishtest/tests/m00017.vtc
index 4e2e8cdd8..57158b020 100644
--- a/bin/varnishtest/tests/m00017.vtc
+++ b/bin/varnishtest/tests/m00017.vtc
@@ -1,5 +1,8 @@
 varnishtest "Test std.rollback"
 
+# ws_emu triggers #3550
+feature !workspace_emulator
+
 # bug regressions:
 # - #3009
 # - #3083
diff --git a/bin/varnishtest/tests/r03353.vtc b/bin/varnishtest/tests/r03353.vtc
index d5ab5f1a4..864a2357f 100644
--- a/bin/varnishtest/tests/r03353.vtc
+++ b/bin/varnishtest/tests/r03353.vtc
@@ -1,5 +1,8 @@
 varnishtest "Test rollback and retry"
 
+# ws_emu triggers #3550
+feature !workspace_emulator
+
 server s1 {
 	rxreq
 	txresp -nolen -hdr "Content-Length: 3"
diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c
index f72963428..9efe6fd5a 100644
--- a/bin/varnishtest/vtc_misc.c
+++ b/bin/varnishtest/vtc_misc.c
@@ -450,6 +450,8 @@ addr_no_randomize_works(void)
  *        Varnish was built with the undefined behavior sanitizer.
  * sanitizer
  *        Varnish was built with a sanitizer.
+ * workspace_emulator
+ *        Varnish was built with its workspace emulator.
  *
  * A feature name can be prefixed with an exclamation mark (!) to skip a
  * test if a feature is present.
@@ -495,6 +497,12 @@ static const unsigned sanitizer = 1;
 static const unsigned sanitizer = 0;
 #endif
 
+#if ENABLE_WORKSPACE_EMULATOR
+static const unsigned workspace_emulator = 1;
+#else
+static const unsigned workspace_emulator = 0;
+#endif
+
 #if WITH_PERSISTENT_STORAGE
 static const unsigned with_persistent_storage = 1;
 #else
@@ -560,6 +568,7 @@ cmd_feature(CMD_ARGS)
 		FEATURE("ubsan", ubsan);
 		FEATURE("sanitizer", sanitizer);
 		FEATURE("SO_RCVTIMEO_WORKS", so_rcvtimeo_works);
+		FEATURE("workspace_emulator", workspace_emulator);
 
 		if (!strcmp(feat, "cmd")) {
 			good = 1;
diff --git a/configure.ac b/configure.ac
index 4bcecd4bf..ccc5afd8f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -331,6 +331,20 @@ if test "x$UBSAN_FLAGS$TSAN_FLAGS$ASAN_FLAGS$MSAN_FLAGS" != "x"; then
 	LDFLAGS="$LDFLAGS $SAN_LDFLAGS"
 fi
 
+AC_ARG_ENABLE(workspace-emulator,
+	AS_HELP_STRING([--enable-workspace-emulator],
+		[emulate workspace allocations (default is NO)]),
+    [], [enable_workspace_emulator=no])
+
+AM_CONDITIONAL([ENABLE_WORKSPACE_EMULATOR],
+	[test "$enable_workspace_emulator" = yes])
+
+AM_COND_IF([ENABLE_WORKSPACE_EMULATOR], [
+	AC_CHECK_HEADERS([sanitizer/asan_interface.h])
+	AC_DEFINE([ENABLE_WORKSPACE_EMULATOR], [1],
+              [Define to 1 if the workspace emulator is enabled])
+])
+
 # Use jemalloc on Linux
 JEMALLOC_LDADD=
 AC_ARG_WITH([jemalloc],

From dridi.boukelmoune at gmail.com  Fri Aug 27 15:30:10 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 27 Aug 2021 15:30:10 +0000 (UTC)
Subject: [master] 5770abc35 circleci: Workspace emulator coverage
Message-ID: <20210827153010.30BE2A5D21@lists.varnish-cache.org>


commit 5770abc3559b3824fbf083fdfe3f66d2984d3f46
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jul 13 10:28:28 2021 +0200

    circleci: Workspace emulator coverage

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 2a58cbf45..8ed135706 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -326,7 +326,7 @@ workflows:
           name: distcheck_debian_buster
           dist: debian
           release: buster
-          extra_conf: --enable-asan --enable-ubsan
+          extra_conf: --enable-asan --enable-ubsan --enable-workspace-emulator
       - distcheck:
           name: distcheck_ubuntu_xenial
           dist: ubuntu

From dridi.boukelmoune at gmail.com  Fri Aug 27 15:30:10 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 27 Aug 2021 15:30:10 +0000 (UTC)
Subject: [master] f237ed15f travis: Workspace emulator coverage
Message-ID: <20210827153010.4C7D1A5D24@lists.varnish-cache.org>


commit f237ed15fa6fdb87150f71439f68f8cb82ae113e
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jul 13 10:29:59 2021 +0200

    travis: Workspace emulator coverage

diff --git a/.travis.yml b/.travis.yml
index cc553fcbe..3928c4d04 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -68,7 +68,7 @@ jobs:
           export UBSAN_OPTIONS=halt_on_error=1,print_stacktrace=1,use_sigaltstack=0,suppressions=$(pwd)/tools/ubsan.suppr
           export CC=clang-9
         - ./autogen.sh
-        - ./configure --enable-maintainer-mode --with-unwind --enable-debugging-symbols --disable-stack-protector --with-persistent-storage --enable-asan --enable-ubsan
+        - ./configure --enable-maintainer-mode --with-unwind --enable-debugging-symbols --disable-stack-protector --with-persistent-storage --enable-asan --enable-ubsan --enable-workspace-emulator
     - stage: test
       os: osx
       osx_image: xcode12.5

From dridi.boukelmoune at gmail.com  Fri Aug 27 15:35:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 27 Aug 2021 15:35:07 +0000 (UTC)
Subject: [master] 8e36cb707 vrt: Make sure VRT_Rollback() has a snapshot
Message-ID: <20210827153507.A9A80A685A@lists.varnish-cache.org>


commit 8e36cb707c3b003ed7764731cc55b1cf80a2afa2
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Aug 27 17:31:28 2021 +0200

    vrt: Make sure VRT_Rollback() has a snapshot
    
    Otherwise we would entirely unravel the task workspace.

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 6f6f3d637..16ab14558 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -153,6 +153,7 @@ Bereq_Rollback(VRT_CTX)
 	HTTP_Clone(bo->bereq, bo->bereq0);
 	bo->vfp_filter_list = NULL;
 	bo->err_reason = NULL;
+	AN(bo->ws_bo);
 	WS_Rollback(bo->ws, bo->ws_bo);
 }
 
diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c
index 5a5e752f7..cff1d0470 100644
--- a/bin/varnishd/cache/cache_req.c
+++ b/bin/varnishd/cache/cache_req.c
@@ -209,6 +209,7 @@ Req_Rollback(VRT_CTX)
 	req->filter_list = NULL;
 	if (WS_Overflowed(req->ws))
 		req->wrk->stats->ws_client_overflow++;
+	AN(req->ws_req);
 	WS_Rollback(req->ws, req->ws_req);
 }
 

From phk at FreeBSD.org  Sat Aug 28 06:34:07 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 28 Aug 2021 06:34:07 +0000 (UTC)
Subject: [master] 1fa2d2288 Emit valid json from `vcl.list -j` when no VCL is
 loaded.
Message-ID: <20210828063407.A3C7D93222@lists.varnish-cache.org>


commit 1fa2d2288db3fd0007eb5551390180df3e39aa99
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Aug 28 06:32:36 2021 +0000

    Emit valid json from `vcl.list -j` when no VCL is loaded.
    
    Fixes #3677

diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c
index 69cd27ec5..df77dd0c8 100644
--- a/bin/varnishd/mgt/mgt_vcl.c
+++ b/bin/varnishd/mgt/mgt_vcl.c
@@ -870,8 +870,8 @@ mcf_vcl_list_json(struct cli *cli, const char * const *av, void *priv)
 		free(p);
 	} else {
 		VCLI_JSON_begin(cli, 2, av);
-		VCLI_Out(cli, ",\n");
 		VTAILQ_FOREACH(vp, &vclhead, list) {
+			VCLI_Out(cli, ",\n");
 			VCLI_Out(cli, "{\n");
 			VSB_indent(cli->sb, 2);
 			VCLI_Out(cli, "\"status\": \"%s\",\n",
@@ -899,8 +899,6 @@ mcf_vcl_list_json(struct cli *cli, const char * const *av, void *priv)
 			}
 			VSB_indent(cli->sb, -2);
 			VCLI_Out(cli, "\n}");
-			if (VTAILQ_NEXT(vp, list) != NULL)
-				VCLI_Out(cli, ",\n");
 		}
 		VCLI_JSON_end(cli);
 	}
diff --git a/bin/varnishtest/tests/b00032.vtc b/bin/varnishtest/tests/b00032.vtc
index 7fd326289..1222339a3 100644
--- a/bin/varnishtest/tests/b00032.vtc
+++ b/bin/varnishtest/tests/b00032.vtc
@@ -8,6 +8,10 @@ server s1 {
 	txresp
 } -start
 
+varnish v1 
+varnish v1 -cliok vcl.list
+varnish v1 -clijson "vcl.list -j"
+
 varnish v1 -vcl+backend {}
 varnish v1 -vcl+backend {}
 

From phk at FreeBSD.org  Sat Aug 28 08:56:04 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 28 Aug 2021 08:56:04 +0000 (UTC)
Subject: [master] 7d09e0b23 Add `vct_case[len]cmp()` as prep-work for #3650
Message-ID: <20210828085604.D64F797830@lists.varnish-cache.org>


commit 7d09e0b237005cb5d434e6b49cdd768e2dbeb23b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Aug 28 08:54:49 2021 +0000

    Add `vct_case[len]cmp()` as prep-work for #3650

diff --git a/include/vct.h b/include/vct.h
index 7ccbbf233..8d7e58308 100644
--- a/include/vct.h
+++ b/include/vct.h
@@ -54,6 +54,7 @@
 #define VCT_LOWER		(1<<14)
 
 extern const uint16_t vct_typtab[256];
+extern const uint8_t vct_lowertab[256];
 
 const char *VCT_invalid_name(const char *b, const char *e);
 
@@ -106,3 +107,43 @@ vct_skipcrlf(char* p, const char* end)
 {
 	return (p + vct_iscrlf(p, end));
 }
+
+static inline int
+vct_casecmp(const void *a, const void *b)
+{
+	const uint8_t *aa = a;
+	const uint8_t *bb = b;
+
+	while (*aa && vct_lowertab[*aa] == vct_lowertab[*bb]) {
+		aa++;
+		bb++;
+	}
+	if (!*aa && !*bb)
+		return (0);
+	if (!*aa)
+		return (-1);
+	if (!*bb)
+		return (1);
+	return ((int)vct_lowertab[*aa] - (int)vct_lowertab[*bb]);
+}
+
+static inline int
+vct_caselencmp(const void *a, const void *b, ssize_t sz)
+{
+	const uint8_t *aa = a;
+	const uint8_t *bb = b;
+
+	assert(sz >= 0);
+	while (sz > 0 && *aa && vct_lowertab[*aa] == vct_lowertab[*bb]) {
+		aa++;
+		bb++;
+		sz--;
+	}
+	if (!sz || (!*aa && !*bb))
+		return (0);
+	if (!*aa)
+		return (-1);
+	if (!*bb)
+		return (1);
+	return ((int)vct_lowertab[*aa] - (int)vct_lowertab[*bb]);
+}
diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am
index 5471895d8..0559a77cc 100644
--- a/lib/libvarnish/Makefile.am
+++ b/lib/libvarnish/Makefile.am
@@ -46,7 +46,7 @@ libvarnish_la_SOURCES = \
 
 libvarnish_la_LIBADD = @PCRE2_LIBS@
 
-TESTS = vav_test vjsn_test vnum_c_test vbh_test vsb_test
+TESTS = vav_test vbh_test vct_test vjsn_test vnum_c_test vsb_test
 
 noinst_PROGRAMS = ${TESTS}
 
@@ -58,6 +58,10 @@ vbh_test_SOURCES	= vbh.c
 vbh_test_CFLAGS = $(AM_CFLAGS) -DTEST_DRIVER
 vbh_test_LDADD = $(AM_LDFLAGS) libvarnish.la
 
+vct_test_SOURCES	= vct.c
+vct_test_CFLAGS = $(AM_CFLAGS) -DTEST_DRIVER
+vct_test_LDADD = $(AM_LDFLAGS) libvarnish.la
+
 vnum_c_test_SOURCES = vnum.c
 vnum_c_test_CFLAGS = $(AM_CFLAGS) -DNUM_C_TEST
 vnum_c_test_LDADD = $(AM_LDFLAGS) libvarnish.la ${LIBM}
diff --git a/lib/libvarnish/vct.c b/lib/libvarnish/vct.c
index 566f1d9ea..3ad131105 100644
--- a/lib/libvarnish/vct.c
+++ b/lib/libvarnish/vct.c
@@ -240,6 +240,41 @@ const uint16_t vct_typtab[256] = {
 	[0xff]	=	VCT_XMLNAMESTART,
 };
 
+const uint8_t vct_lowertab[256] = {
+	0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,	// 0x00
+	0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,	// 0x08
+	0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17,	// 0x10
+	0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f,	// 0x18
+	0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27,	// 0x20
+	0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f,	// 0x28
+	0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,	// 0x30
+	0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f,	// 0x38
+	0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,	// 0x40
+	0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f,	// 0x48
+	0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,	// 0x50
+	0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f,	// 0x58
+	0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67,	// 0x60
+	0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f,	// 0x68
+	0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77,	// 0x70
+	0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f,	// 0x78
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,	// 0x80
+	0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f,	// 0x88
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,	// 0x90
+	0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f,	// 0x98
+	0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7,	// 0xa0
+	0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf,	// 0xa8
+	0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7,	// 0xb0
+	0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf,	// 0xb8
+	0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7,	// 0xc0
+	0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf,	// 0xc8
+	0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7,	// 0xd0
+	0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf,	// 0xd8
+	0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7,	// 0xe0
+	0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef,	// 0xe8
+	0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7,	// 0xf0
+	0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff,	// 0xf8
+};
+
 const char *
 VCT_invalid_name(const char *b, const char *e)
 {
@@ -258,3 +293,45 @@ VCT_invalid_name(const char *b, const char *e)
 
 	return (NULL);
 }
+
+#ifdef TEST_DRIVER
+
+#include <ctype.h>
+#include <locale.h>
+
+int
+main(int argc, char **argv)
+{
+	int i;
+	(void)argc;
+	(void)argv;
+
+	AN(setlocale(LC_ALL, "C"));
+
+	for (i = 0x20; i < 0x7f; i++)
+		assert(vct_lowertab[i] == tolower(i));
+
+	assert(vct_casecmp("AZaz", "azAZ") == 0);
+	assert(vct_casecmp("AZaz", "azAY") > 0);
+	assert(vct_casecmp("AZay", "azAZ") < 0);
+	assert(vct_casecmp("AZaz_", "azAZ") > 0);
+	assert(vct_casecmp("AZaz", "azAZ_") < 0);
+	assert(vct_casecmp("", "") == 0);
+
+	assert(vct_caselencmp("AZaz_", "azAZ", 4) == 0);
+	assert(vct_caselencmp("AZaz", "azAZ_", 4) == 0);
+
+	assert(vct_caselencmp("AZaz1", "azAZ1", 4) == 0);
+	assert(vct_caselencmp("AZaz1", "azAY1", 4) > 0);
+	assert(vct_caselencmp("AZay1", "azAZ1", 4) < 0);
+
+	assert(vct_caselencmp("AZaz", "azAZ", 5) == 0);
+	assert(vct_caselencmp("AZaz ", "azAY", 5) > 0);
+	assert(vct_caselencmp("AZay ", "azAZ", 5) < 0);
+
+	assert(vct_caselencmp("A", "B", 0) == 0);
+
+	return (0);
+}
+
+#endif /* TEST_DRIVER */

From phk at FreeBSD.org  Sat Aug 28 09:12:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 28 Aug 2021 09:12:05 +0000 (UTC)
Subject: [master] 0c02c6758 Implment strict HTTP comparisons.
Message-ID: <20210828091205.8C63F981B1@lists.varnish-cache.org>


commit 0c02c67586821517aeb43a4ba2e07cb9f5c4a8c2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Aug 28 09:09:30 2021 +0000

    Implment strict HTTP comparisons.
    
    Most of this is taken from #3650 but instead of `str[n]casecmp`
    it uses VCT functionality, in order to avoid LOCALE poisoning.
    
    Closes #3650

diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl
index 89c4c54fa..770081366 100644
--- a/bin/varnishd/builtin.vcl
+++ b/bin/varnishd/builtin.vcl
@@ -53,7 +53,7 @@ sub vcl_req_host {
 	}
 	if (!req.http.host &&
 	    req.esi_level == 0 &&
-	    req.proto ~ "^(?i)HTTP/1.1") {
+	    req.proto == "HTTP/1.1") {
 		# In HTTP/1.1, Host is required.
 		return (synth(400));
 	}
diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 8b00b4fd4..b9e4022cb 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -646,6 +646,36 @@ extern const char H__Status[];
 extern const char H__Proto[];
 extern const char H__Reason[];
 
+// rfc7233,l,1207,1208
+#define http_tok_eq(s1, s2)		(!vct_casecmp(s1, s2))
+#define http_tok_at(s1, s2, l)		(!vct_caselencmp(s1, s2, l))
+#define http_ctok_at(s, cs)		(!vct_caselencmp(s, cs, sizeof(cs) - 1))
+
+// rfc7230,l,1037,1038
+#define http_scheme_at(str, tok)	http_ctok_at(str, #tok "://")
+
+// rfc7230,l,1144,1144
+// rfc7231,l,1156,1158
+#define http_method_eq(str, tok)	(!strcmp(str, #tok))
+
+// rfc7230,l,1222,1222
+// rfc7230,l,2848,2848
+// rfc7231,l,3883,3885
+// rfc7234,l,1339,1340
+// rfc7234,l,1418,1419
+#define http_hdr_eq(s1, s2)		http_tok_eq(s1, s2)
+#define http_hdr_at(s1, s2, l)		http_tok_at(s1, s2, l)
+
+// rfc7230,l,1952,1952
+// rfc7231,l,604,604
+#define http_coding_eq(str, tok)	http_tok_eq(str, #tok)
+
+// rfc7231,l,1864,1864
+#define http_expect_eq(str, tok)	http_tok_eq(str, #tok)
+
+// rfc7233,l,1207,1208
+#define http_range_at(str, tok)		http_ctok_at(str, #tok)
+
 /* cache_main.c */
 #define VXID(u) ((u) & VSL_IDENTMASK)
 uint32_t VXID_Get(const struct worker *, uint32_t marker);
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 466e096a4..7518b215b 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -40,6 +40,7 @@
 #include "cache_filter.h"
 #include "cache_vgz.h"
 
+#include "vct.h"
 #include "vtim.h"
 #include "cache_esi.h"
 #include "vend.h"
@@ -860,7 +861,7 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody)
 		return;
 
 	if (http_GetHdr(req->resp, H_Content_Encoding, &p))
-		i = !strcasecmp(p, "gzip");
+		i = http_coding_eq(p, gzip);
 	if (i)
 		i = ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED);
 
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index dd4843c1c..233dd4f7e 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -153,7 +153,7 @@ http_hdr_flags(const char *b, const char *e)
 	retval = &http_hdrflg[u];
 	if (retval->hdr == NULL)
 		return(NULL);
-	if (strncasecmp(retval->hdr + 1, b, e - b))
+	if (!http_hdr_at(retval->hdr + 1, b, e - b))
 		return(NULL);
 	return(retval);
 }
@@ -433,7 +433,7 @@ http_IsHdr(const txt *hh, hdr_t hdr)
 	assert(l == strlen(hdr + 1));
 	assert(hdr[l] == ':');
 	hdr++;
-	return (!strncasecmp(hdr, hh->b, l));
+	return (http_hdr_at(hdr, hh->b, l));
 }
 
 /*--------------------------------------------------------------------*/
@@ -449,7 +449,7 @@ http_findhdr(const struct http *hp, unsigned l, const char *hdr)
 			continue;
 		if (hp->hd[u].b[l] != ':')
 			continue;
-		if (strncasecmp(hdr, hp->hd[u].b, l))
+		if (!http_hdr_at(hdr, hp->hd[u].b, l))
 			continue;
 		return (u);
 	}
@@ -654,7 +654,7 @@ http_split(const char **src, const char *stop, const char *sep,
  * Comparison rule for tokens:
  *	if target string starts with '"', we use memcmp() and expect closing
  *	double quote as well
- *	otherwise we use strncasecmp()
+ *	otherwise we use http_tok_at()
  *
  * On match we increment *bp past the token name.
  */
@@ -676,7 +676,7 @@ http_istoken(const char **bp, const char *e, const char *token)
 		*bp += fl + 2;
 		return (1);
 	}
-	if (b + fl <= e && !strncasecmp(b, token, fl) &&
+	if (b + fl <= e && http_tok_at(b, token, fl) &&
 	    (b + fl == e || !vct_istchar(b[fl]))) {
 		*bp += fl;
 		return (1);
@@ -876,9 +876,9 @@ http_DoConnection(struct http *hp, enum sess_close sc_close)
 	AN(h);
 	while (http_split(&h, NULL, ",", &b, &e)) {
 		u = pdiff(b, e);
-		if (u == 5 && !strncasecmp(b, "close", u))
+		if (u == 5 && http_hdr_at(b, "close", u))
 			retval = sc_close;
-		if (u == 10 && !strncasecmp(b, "keep-alive", u))
+		if (u == 10 && http_hdr_at(b, "keep-alive", u))
 			retval = SC_NULL;
 
 		/* Refuse removal of well-known-headers if they would pass. */
@@ -892,7 +892,7 @@ http_DoConnection(struct http *hp, enum sess_close sc_close)
 				continue;
 			if (hp->hd[v].b[u] != ':')
 				continue;
-			if (strncasecmp(b, hp->hd[v].b, u))
+			if (!http_hdr_at(b, hp->hd[v].b, u))
 				continue;
 			hp->hdf[v] |= HDF_FILTER;
 		}
@@ -910,7 +910,7 @@ http_HdrIs(const struct http *hp, hdr_t hdr, const char *val)
 	if (!http_GetHdr(hp, hdr, &p))
 		return (0);
 	AN(p);
-	if (!strcasecmp(p, val))
+	if (http_tok_eq(p, val))
 		return (1);
 	return (0);
 }
@@ -989,10 +989,14 @@ http_ForceField(struct http *to, unsigned n, const char *t)
 
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
 	assert(n < HTTP_HDR_FIRST);
+	assert(n == HTTP_HDR_METHOD || n == HTTP_HDR_PROTO);
 	AN(t);
+
+	/* NB: method names and protocol versions are case-sensitive. */
 	if (to->hd[n].b == NULL || strcmp(to->hd[n].b, t)) {
 		i = (HTTP_HDR_UNSET - HTTP_HDR_METHOD);
 		i += to->logtag;
+		/* XXX: this is a dead branch */
 		if (n >= HTTP_HDR_FIRST)
 			VSLbt(to->vsl, (enum VSL_tag_e)i, to->hd[n]);
 		http_SetH(to, n, t);
@@ -1204,6 +1208,7 @@ HTTP_GetHdrPack(struct worker *wrk, struct objcore *oc, hdr_t hdr)
 		AN(ptr);
 		ptr += 4;	/* Skip nhd and status */
 
+		/* XXX: should we also have h2_hdr_eq() ? */
 		if (!strcmp(hdr, ":proto:"))
 			return (ptr);
 		ptr = strchr(ptr, '\0') + 1;
@@ -1216,7 +1221,7 @@ HTTP_GetHdrPack(struct worker *wrk, struct objcore *oc, hdr_t hdr)
 	}
 
 	HTTP_FOREACH_PACK(wrk, oc, ptr) {
-		if (!strncasecmp(ptr, hdr, l)) {
+		if (http_hdr_at(ptr, hdr, l)) {
 			ptr += l;
 			while (vct_islws(*ptr))
 				ptr++;
diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c
index be2340a19..7451c3812 100644
--- a/bin/varnishd/cache/cache_range.c
+++ b/bin/varnishd/cache/cache_range.c
@@ -108,7 +108,7 @@ vrg_dorange(struct req *req, const char *r, void **priv)
 	ssize_t low, high, has_low, has_high, t;
 	struct vrg_priv *vrg_priv;
 
-	if (strncasecmp(r, "bytes=", 6))
+	if (!http_range_at(r, bytes=))
 		return ("Not Bytes");
 	r += 6;
 
@@ -214,6 +214,7 @@ vrg_ifrange(struct req *req)
 			return (0);
 		if ((e[0] == 'W' && e[1] == '/'))	// rfc7232,l,547,548
 			return (0);
+		/* XXX: should we also have http_etag_cmp() ? */
 		return (strcmp(p, e) == 0);		// rfc7232,l,548,548
 	}
 
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 75ea7b27c..9ad2aa28d 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -50,6 +50,7 @@
 #include "storage/storage.h"
 #include "common/heritage.h"
 #include "vcl.h"
+#include "vct.h"
 #include "vsha256.h"
 #include "vtim.h"
 
@@ -93,7 +94,7 @@ cnt_transport(struct worker *wrk, struct req *req)
 	AN(req->req_body_status);
 
 	if (http_GetHdr(req->http, H_Expect, &p)) {
-		if (strcasecmp(p, "100-continue")) {
+		if (!http_expect_eq(p, 100-continue)) {
 			req->doclose = SC_RX_JUNK;
 			(void)req->transport->minimal_response(req, 417);
 			wrk->stats->client_req_417++;
@@ -423,7 +424,7 @@ cnt_transmit(struct worker *wrk, struct req *req)
 	clval = http_GetContentLength(req->resp);
 	/* RFC 7230, 3.3.3 */
 	status = http_GetStatus(req->resp);
-	head = !strcmp(req->http0->hd[HTTP_HDR_METHOD].b, "HEAD");
+	head = http_method_eq(req->http0->hd[HTTP_HDR_METHOD].b, HEAD);
 
 	if (boc != NULL || (req->objcore->flags & (OC_F_FAILED)))
 		req->resp_len = clval;
diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index f00b37ebb..93bc30843 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -265,6 +265,7 @@ rfc2616_strong_compare(const char *p, const char *e)
 	if ((p[0] == 'W' && p[1] == '/') ||
 	    (e[0] == 'W' && e[1] == '/'))
 		return (0);
+	/* XXX: should we also have http_etag_cmp() ? */
 	return (strcmp(p, e) == 0);
 }
 
@@ -276,6 +277,7 @@ rfc2616_weak_compare(const char *p, const char *e)
 		p += 2;
 	if (e[0] == 'W' && e[1] == '/')
 		e += 2;
+	/* XXX: should we also have http_etag_cmp() ? */
 	return (strcmp(p, e) == 0);
 }
 
diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index 1e77730b3..88567674d 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -200,7 +200,7 @@ vry_cmp(const uint8_t *v1, const uint8_t *v2)
 		/* Different header */
 		retval = 1;
 	} else if (cache_param->http_gzip_support &&
-	    !strcasecmp(H_Accept_Encoding, (const char*) v1 + 2)) {
+	    http_hdr_eq(H_Accept_Encoding, (const char*) v1 + 2)) {
 		/*
 		 * If we do gzip processing, we do not vary on Accept-Encoding,
 		 * because we want everybody to get the gzip'ed object, and
diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c
index dfd0555b4..1861d9a12 100644
--- a/bin/varnishd/http1/cache_http1_fetch.c
+++ b/bin/varnishd/http1/cache_http1_fetch.c
@@ -240,7 +240,7 @@ V1F_FetchRespHdr(struct busyobj *bo)
 	 * Figure out how the fetch is supposed to happen, before the
 	 * headers are adultered by VCL
 	 */
-	if (!strcasecmp(http_GetMethod(bo->bereq), "head")) {
+	if (http_method_eq(http_GetMethod(bo->bereq), HEAD)) {
 		/*
 		 * A HEAD request can never have a body in the reply,
 		 * no matter what the headers might say.
diff --git a/bin/varnishd/http1/cache_http1_proto.c b/bin/varnishd/http1/cache_http1_proto.c
index ebf9ad1f1..eb19825cc 100644
--- a/bin/varnishd/http1/cache_http1_proto.c
+++ b/bin/varnishd/http1/cache_http1_proto.c
@@ -315,7 +315,7 @@ http1_body_status(const struct http *hp, struct http_conn *htc, int request)
 	if (cl == -2)
 		return (BS_ERROR);
 	if (http_GetHdr(hp, H_Transfer_Encoding, &b)) {
-		if (strcasecmp(b, "chunked"))
+		if (!http_coding_eq(b, chunked))
 			return (BS_ERROR);
 		if (cl != -1) {
 			/*
@@ -375,10 +375,10 @@ HTTP1_DissectRequest(struct http_conn *htc, struct http *hp)
 		return (400);
 
 	/* RFC2616, section 5.2, point 1 */
-	if (!strncasecmp(hp->hd[HTTP_HDR_URL].b, "http://", 7))
+	if (http_scheme_at(hp->hd[HTTP_HDR_URL].b, http))
 		b = hp->hd[HTTP_HDR_URL].b + 7;
 	else if (FEATURE(FEATURE_HTTPS_SCHEME) &&
-	    !strncasecmp(hp->hd[HTTP_HDR_URL].b, "https://", 8))
+	    http_scheme_at(hp->hd[HTTP_HDR_URL].b, https))
 		b = hp->hd[HTTP_HDR_URL].b + 8;
 	if (b) {
 		e = strchr(b, '/');
@@ -399,13 +399,13 @@ HTTP1_DissectRequest(struct http_conn *htc, struct http *hp)
 	if (htc->body_status == BS_EOF) {
 		assert(hp->protover == 10);
 		/* RFC1945 8.3 p32 and D.1.1 p58 */
-		if (!strcasecmp(p, "post") || !strcasecmp(p, "put"))
+		if (http_method_eq(p, POST) || http_method_eq(p, PUT))
 			return (400);
 		htc->body_status = BS_NONE;
 	}
 
 	/* HEAD with a body is a hard error */
-	if (htc->body_status != BS_NONE && !strcasecmp(p, "head"))
+	if (htc->body_status != BS_NONE && http_method_eq(p, HEAD))
 		return (400);
 
 	return (retval);
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index d4265f4c4..5eb5d6306 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -1201,7 +1201,7 @@ cmd_http_txreq(CMD_ARGS)
 		} else if (!strcmp(*av, "-method") ||
 		    !strcmp(*av, "-req")) {
 			req = av[1];
-			hp->head_method = !strcasecmp(av[1], "HEAD") ;
+			hp->head_method = !strcmp(av[1], "HEAD") ;
 			av++;
 		} else if (!hp->sfd && !strcmp(*av, "-up")) {
 			up = av[1];
@@ -1216,7 +1216,7 @@ cmd_http_txreq(CMD_ARGS)
 				"Upgrade: h2c%s"
 				"HTTP2-Settings: %s%s", nl, nl, up, nl);
 
-	nohost = strcasecmp(proto, "HTTP/1.1") != 0;
+	nohost = strcmp(proto, "HTTP/1.1") != 0;
 	av = http_tx_parse_args(av, vl, hp, NULL, nohost);
 	if (*av != NULL)
 		vtc_fatal(hp->vl, "Unknown http txreq spec: %s\n", *av);

From phk at FreeBSD.org  Sat Aug 28 10:33:04 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 28 Aug 2021 10:33:04 +0000 (UTC)
Subject: [master] 7d2d86dce Test-cover for the last three lines of vct.h
Message-ID: <20210828103304.C1E3B9CAA2@lists.varnish-cache.org>


commit 7d2d86dce385a0ca132e608378b1ba8541adf902
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Aug 28 10:32:00 2021 +0000

    Test-cover for the last three lines of vct.h

diff --git a/lib/libvarnish/vct.c b/lib/libvarnish/vct.c
index 3ad131105..4bf6004a7 100644
--- a/lib/libvarnish/vct.c
+++ b/lib/libvarnish/vct.c
@@ -303,11 +303,16 @@ int
 main(int argc, char **argv)
 {
 	int i;
+	const char *p;
+
 	(void)argc;
 	(void)argv;
 
 	AN(setlocale(LC_ALL, "C"));
 
+	p = "";
+	assert(vct_iscrlf(p, p) == 0);
+
 	for (i = 0x20; i < 0x7f; i++)
 		assert(vct_lowertab[i] == tolower(i));
 
@@ -329,6 +334,10 @@ main(int argc, char **argv)
 	assert(vct_caselencmp("AZaz ", "azAY", 5) > 0);
 	assert(vct_caselencmp("AZay ", "azAZ", 5) < 0);
 
+	assert(vct_caselencmp("AZaz", "azAZ1", 5) < 0);
+	assert(vct_caselencmp("AZaz1", "azAZ", 5) > 0);
+
+
 	assert(vct_caselencmp("A", "B", 0) == 0);
 
 	return (0);

From phk at FreeBSD.org  Sat Aug 28 13:42:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 28 Aug 2021 13:42:05 +0000 (UTC)
Subject: [master] bee9d898c Teach FlexeLint a bit about ws-emu
Message-ID: <20210828134205.310F9A56AE@lists.varnish-cache.org>


commit bee9d898c17dc8fd5296da62b1c8e351134ef0bd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Aug 28 13:41:32 2021 +0000

    Teach FlexeLint a bit about ws-emu

diff --git a/bin/varnishd/cache/cache_ws_emu.c b/bin/varnishd/cache/cache_ws_emu.c
index bc9a4c67e..631045bdd 100644
--- a/bin/varnishd/cache/cache_ws_emu.c
+++ b/bin/varnishd/cache/cache_ws_emu.c
@@ -31,6 +31,8 @@
 
 #include "config.h"
 
+#ifdef ENABLE_WORKSPACE_EMULATOR
+
 #if HAVE_SANITIZER_ASAN_INTERFACE_H
 #  include <sanitizer/asan_interface.h>
 #endif
@@ -722,3 +724,5 @@ WS_Panic(struct vsb *vsb, const struct ws *ws)
 	VSB_indent(vsb, -2);
 	VSB_cat(vsb, "},\n");
 }
+
+#endif /* ENABLE_WORKSPACE_EMULATOR */
diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt
index 0d8f5c5e7..910ace52c 100644
--- a/bin/varnishd/flint.lnt
+++ b/bin/varnishd/flint.lnt
@@ -63,6 +63,12 @@
 -esym(714, Vmod_*_Data)
 -esym(765, Vmod_*_Data)
 
+-esym(755, PAN_dump_once)		// global macro not referenced
+-esym(755, PAN_dump_once_oneline)	// global macro not referenced
+-esym(714, WS_Dump)	// Info 714: sym not referenced
+-esym(759, WS_Dump)	// Info 759: header declaration could be moved to module
+-esym(765, WS_Dump)	// Info 765: external could be static
+
 
 //-sem (pthread_mutex_lock, thread_lock)
 -sem (pthread_mutex_trylock, thread_lock)

From dridi.boukelmoune at gmail.com  Mon Aug 30 05:43:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 05:43:07 +0000 (UTC)
Subject: [master] 66e2ea313 http1: Missing workspace release
Message-ID: <20210830054307.A42D596783@lists.varnish-cache.org>


commit 66e2ea313e7d75530d90d5bd0df78f9609169c6e
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 30 07:38:00 2021 +0200

    http1: Missing workspace release
    
    Working on the workspace sanitizer (ancestor of the workspace emulator)
    final rollbacks were needed to unwind allocations. There was however a
    branch where error handling was missing a workspace release, and it was
    fine before the introduction of the final rollbacks.
    
    To avoid turning the workspace emulator into a DoS vector the rollbacks
    are now only enforced for emulator builds. The specific "insufficient
    workspace" log is amended to ensure future changes to the session
    workspace footprint don't accidentally remove test coverage for that
    branch. The same could be done for other "insufficient workspace" logs
    in the PROXY protocol parsing.
    
    Refs 0632b84693f3 (req: Prevent early rollback)
    Refs ce71896ae353 (sess: Plug conceptual leak)
    Refs 246b1eb1e888 (busyobj: Plug conceptual leak)
    Refs 5b4f0f1addaa (htc: Defer workspace rollbacks for request tasks)
    Refs #3644
    
    Spotted by Alf's single process fuzzing setup that we should eventually
    revisit.
    
    Refs #3152

diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 139112854..f3e41716f 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -181,7 +181,9 @@ VBO_ReleaseBusyObj(struct worker *wrk, struct busyobj **pbo)
 	}
 
 	VCL_Rel(&bo->vcl);
+#ifdef ENABLE_WORKSPACE_EMULATOR
 	WS_Rollback(bo->ws, 0);
+#endif
 
 	memset(&bo->retries, 0,
 	    sizeof *bo - offsetof(struct busyobj, retries));
diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c
index cff1d0470..09addfde4 100644
--- a/bin/varnishd/cache/cache_req.c
+++ b/bin/varnishd/cache/cache_req.c
@@ -172,7 +172,9 @@ Req_Release(struct req *req)
 	AZ(req->vcl);
 	if (req->vsl->wid)
 		VSL_End(req->vsl);
+#ifdef ENABLE_WORKSPACE_EMULATOR
 	WS_Rollback(req->ws, 0);
+#endif
 	TAKE_OBJ_NOTNULL(sp, &req->sp, SESS_MAGIC);
 	pp = sp->pool;
 	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index f3bb4da15..f3a9619c0 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -646,7 +646,9 @@ SES_Rel(struct sess *sp)
 	if (i)
 		return;
 	Lck_Delete(&sp->mtx);
+#ifdef ENABLE_WORKSPACE_EMULATOR
 	WS_Rollback(sp->ws, 0);
+#endif
 	MPL_Free(sp->pool->mpl_sess, sp);
 }
 
diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c
index 393088520..da8c5fea6 100644
--- a/bin/varnishd/http1/cache_http1_fsm.c
+++ b/bin/varnishd/http1/cache_http1_fsm.c
@@ -119,7 +119,9 @@ http1_new_session(struct worker *wrk, void *arg)
 		/* Out of session workspace. Free the req, close the sess,
 		 * and do not set a new task func, which will exit the
 		 * worker thread. */
-		VSL(SLT_Error, req->sp->vxid, "insufficient workspace");
+		VSL(SLT_Error, req->sp->vxid,
+		    "insufficient workspace (proto_priv)");
+		WS_Release(req->ws, 0);
 		Req_Release(req);
 		SES_Delete(sp, SC_RX_JUNK, NAN);
 		return;
diff --git a/bin/varnishtest/tests/o00006.vtc b/bin/varnishtest/tests/o00006.vtc
new file mode 100644
index 000000000..5a6b4e357
--- /dev/null
+++ b/bin/varnishtest/tests/o00006.vtc
@@ -0,0 +1,46 @@
+varnishtest "SES_Reserve_proto_priv() overflow"
+
+feature ipv4
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -arg "-p pool_sess=0,0,0" -proto "PROXY" -vcl+backend {} -start
+
+logexpect l1 -v v1 -g raw {
+	expect 0 1000	Begin		"sess 0 PROXY"
+	expect 0 =	SessOpen
+	expect 0 =	Proxy		"2 217.70.181.33 60822 95.142.168.34 443"
+	expect 0 =	Error		{\Qinsufficient workspace (proto_priv)\E}
+	expect 0 =	SessClose	"RX_JUNK"
+} -start
+
+varnish v1 -cliok "param.set workspace_session 480"
+
+client c1 {
+	# PROXY2 with CRC32C TLV
+	sendhex {
+		0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a
+		21 11 00 65
+		d9 46 b5 21
+		5f 8e a8 22
+		ed 96
+		01 bb
+		03 00 04  95 03 ee 75
+		01 00 02  68 32
+		02 00 0a  68 6f 63 64 65 74 2e 6e 65 74
+		20 00 3d
+		01 00 00 00 00
+		21 00 07  54 4c 53 76 31 2e 33
+		25 00 05  45 43 32 35 36
+		24 00 0a  52 53 41 2d 53 48 41 32 35 36
+		23 00 16  41 45 41 44 2d 41 45 53 31 32 38
+			  2d 47 43 4d 2d 53 48 41 32 35 36
+	}
+	txreq
+	expect_close
+} -run
+
+logexpect l1 -wait

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:06 +0000 (UTC)
Subject: [master] 19cc1e4cc STV temp buffer API
Message-ID: <20210830083106.BF4F2A19D7@lists.varnish-cache.org>


commit 19cc1e4cca5b932299dd88b56e4a6b791eb6673e
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 22 11:48:49 2021 +0200

    STV temp buffer API
    
    This is an API for getting an arbitrary buffer through the
    stevedores. The stevedore in question may then deploy LRU nuking or other
    measures to control resource usage.

diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index bf246677a..c3defbdce 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -553,6 +553,11 @@ void STV_BanExport(const uint8_t *banlist, unsigned len);
 int STV_NewObject(struct worker *, struct objcore *,
     const struct stevedore *, unsigned len);
 
+struct stv_buffer;
+struct stv_buffer *STV_AllocBuf(struct worker *wrk, const struct stevedore *stv,
+    size_t size);
+void STV_FreeBuf(struct worker *wrk, struct stv_buffer **pstvbuf);
+void *STV_GetBufPtr(struct stv_buffer *stvbuf, size_t *psize);
 
 #if WITH_PERSISTENT_STORAGE
 /* storage_persistent.c */
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 9edf85774..498822b69 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -96,6 +96,74 @@ STV_NewObject(struct worker *wrk, struct objcore *oc,
 
 /*-------------------------------------------------------------------*/
 
+struct stv_buffer {
+	unsigned		magic;
+#define STV_BUFFER_MAGIC	0xf39cb6c2
+	const struct stevedore	*stv;
+	size_t			size;
+	uintptr_t		priv;
+};
+
+struct stv_buffer *
+STV_AllocBuf(struct worker *wrk, const struct stevedore *stv, size_t size)
+{
+	struct stv_buffer *stvbuf;
+	uint8_t *buf;
+	uintptr_t priv = 0;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
+
+	if (size == 0)
+		return (NULL);
+	if (stv->allocbuf == NULL)
+		return (NULL);
+
+	buf = stv->allocbuf(wrk, stv, size + PRNDUP(sizeof *stvbuf), &priv);
+	if (buf == NULL)
+		return (NULL);
+
+	assert(PAOK(buf));
+	stvbuf = (void *)buf;
+	INIT_OBJ(stvbuf, STV_BUFFER_MAGIC);
+	stvbuf->stv = stv;
+	stvbuf->priv = priv;
+	stvbuf->size = size;
+
+	return (stvbuf);
+}
+
+void
+STV_FreeBuf(struct worker *wrk, struct stv_buffer **pstvbuf)
+{
+	struct stv_buffer *stvbuf;
+	const struct stevedore *stv;
+	uintptr_t priv;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	TAKE_OBJ_NOTNULL(stvbuf, pstvbuf, STV_BUFFER_MAGIC);
+	CHECK_OBJ_NOTNULL(stvbuf->stv, STEVEDORE_MAGIC);
+
+	stv = stvbuf->stv;
+	priv = stvbuf->priv;
+	CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
+	ZERO_OBJ(stvbuf, sizeof *stvbuf);
+
+	AN(stv->freebuf);
+	stv->freebuf(wrk, stv, priv);
+}
+
+void *
+STV_GetBufPtr(struct stv_buffer *stvbuf, size_t *psize)
+{
+	CHECK_OBJ_NOTNULL(stvbuf, STV_BUFFER_MAGIC);
+	if (psize)
+		*psize = stvbuf->size;
+	return (&stvbuf[1]);
+}
+
+/*-------------------------------------------------------------------*/
+
 void
 STV_open(void)
 {
diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h
index c6cda8998..3ad4ecc55 100644
--- a/bin/varnishd/storage/storage.h
+++ b/bin/varnishd/storage/storage.h
@@ -75,6 +75,10 @@ typedef void storage_banexport_f(const struct stevedore *, const uint8_t *bans,
     unsigned len);
 typedef void storage_panic_f(struct vsb *vsb, const struct objcore *oc);
 
+typedef void *storage_allocbuf_f(struct worker *, const struct stevedore *,
+    size_t size, uintptr_t *ppriv);
+typedef void storage_freebuf_f(struct worker *, const struct stevedore *,
+    uintptr_t priv);
 
 typedef struct object *sml_getobj_f(struct worker *, struct objcore *);
 typedef struct storage *sml_alloc_f(const struct stevedore *, size_t size);
@@ -104,6 +108,8 @@ struct stevedore {
 	storage_baninfo_f	*baninfo;
 	storage_banexport_f	*banexport;
 	storage_panic_f		*panic;
+	storage_allocbuf_f	*allocbuf;
+	storage_freebuf_f	*freebuf;
 
 	/* Only if SML is used */
 	sml_alloc_f		*sml_alloc;

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:06 +0000 (UTC)
Subject: [master] 9588cfe7c Simple STV temp buffer implementation
Message-ID: <20210830083106.DE07BA19DA@lists.varnish-cache.org>


commit 9588cfe7c925081a5eeaa1f8ee6c6c949b92d519
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 22 11:48:55 2021 +0200

    Simple STV temp buffer implementation

diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c
index 7114398d8..bf7fc37b7 100644
--- a/bin/varnishd/storage/storage_simple.c
+++ b/bin/varnishd/storage/storage_simple.c
@@ -47,7 +47,11 @@
 /*-------------------------------------------------------------------*/
 
 static struct storage *
-sml_stv_alloc(const struct stevedore *stv, ssize_t size, int flags)
+objallocwithnuke(struct worker *, const struct stevedore *, ssize_t size,
+    int flags);
+
+static struct storage *
+sml_stv_alloc(const struct stevedore *stv, size_t size, int flags)
 {
 	struct storage *st;
 
@@ -161,6 +165,39 @@ SML_allocobj(struct worker *wrk, const struct stevedore *stv,
 	return (1);
 }
 
+void * v_matchproto_(storage_allocbuf_t)
+SML_AllocBuf(struct worker *wrk, const struct stevedore *stv, size_t size,
+    uintptr_t *ppriv)
+{
+	struct storage *st;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
+	AN(ppriv);
+
+	if (size > UINT_MAX)
+		return (NULL);
+	st = objallocwithnuke(wrk, stv, size, 0);
+	if (st == NULL)
+		return (NULL);
+	assert(st->space >= size);
+	st->len = size;
+	*ppriv = (uintptr_t)st;
+	return (st->ptr);
+}
+
+void v_matchproto_(storage_freebuf_t)
+SML_FreeBuf(struct worker *wrk, const struct stevedore *stv, uintptr_t priv)
+{
+	struct storage *st;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
+
+	CAST_OBJ_NOTNULL(st, (void *)priv, STORAGE_MAGIC);
+	sml_stv_free(stv, st);
+}
+
 /*---------------------------------------------------------------------
  */
 
diff --git a/bin/varnishd/storage/storage_simple.h b/bin/varnishd/storage/storage_simple.h
index 4e63efb9d..3f288aef7 100644
--- a/bin/varnishd/storage/storage_simple.h
+++ b/bin/varnishd/storage/storage_simple.h
@@ -68,6 +68,9 @@ extern const struct obj_methods SML_methods;
 
 struct object *SML_MkObject(const struct stevedore *, struct objcore *,
     void *ptr);
+void *SML_AllocBuf(struct worker *, const struct stevedore *, size_t,
+    uintptr_t *);
+void SML_FreeBuf(struct worker *, const struct stevedore *, uintptr_t);
 
 storage_allocobj_f SML_allocobj;
 storage_panic_f SML_panic;

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:07 +0000 (UTC)
Subject: [master] 3142d4a88 Use SML_AllocBuf with -smalloc
Message-ID: <20210830083107.10E64A19DD@lists.varnish-cache.org>


commit 3142d4a881b39e77d1c11d35401bfb0e9b68fe25
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 22 11:48:58 2021 +0200

    Use SML_AllocBuf with -smalloc

diff --git a/bin/varnishd/storage/storage_malloc.c b/bin/varnishd/storage/storage_malloc.c
index 013d5e58d..6bb789e8b 100644
--- a/bin/varnishd/storage/storage_malloc.c
+++ b/bin/varnishd/storage/storage_malloc.c
@@ -232,4 +232,6 @@ const struct stevedore sma_stevedore = {
 	.methods	=	&SML_methods,
 	.var_free_space =	sma_free_space,
 	.var_used_space =	sma_used_space,
+	.allocbuf	=	SML_AllocBuf,
+	.freebuf	=	SML_FreeBuf,
 };

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:07 +0000 (UTC)
Subject: [master] e10b61da0 Use SML_AllocBuf also for -sfile
Message-ID: <20210830083107.35552A19E5@lists.varnish-cache.org>


commit e10b61da0d3eb314e8f643bed68af86da58933e5
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 22 11:49:03 2021 +0200

    Use SML_AllocBuf also for -sfile

diff --git a/bin/varnishd/storage/storage_file.c b/bin/varnishd/storage/storage_file.c
index 3c61a61c5..ba7009078 100644
--- a/bin/varnishd/storage/storage_file.c
+++ b/bin/varnishd/storage/storage_file.c
@@ -501,6 +501,8 @@ const struct stevedore smf_stevedore = {
 	.allocobj	=	SML_allocobj,
 	.panic		=	SML_panic,
 	.methods	=	&SML_methods,
+	.allocbuf	=	SML_AllocBuf,
+	.freebuf	=	SML_FreeBuf,
 };
 
 #ifdef INCLUDE_TEST_DRIVER

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:07 +0000 (UTC)
Subject: [master] 64c2b6642 Make h2_del_req take a non-const struct h2_req
Message-ID: <20210830083107.58DB5A19EA@lists.varnish-cache.org>


commit 64c2b66421dee8e1c70ed59cdd9285ecb467711d
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 22 11:49:10 2021 +0200

    Make h2_del_req take a non-const struct h2_req

diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h
index ea5eb528d..ee91d6fac 100644
--- a/bin/varnishd/http2/cache_http2.h
+++ b/bin/varnishd/http2/cache_http2.h
@@ -237,7 +237,7 @@ void H2_Send(struct worker *, struct h2_req *, h2_frame type, uint8_t flags,
 struct h2_req * h2_new_req(const struct worker *, struct h2_sess *,
     unsigned stream, struct req *);
 int h2_stream_tmo(struct h2_sess *, const struct h2_req *, vtim_real);
-void h2_del_req(struct worker *, const struct h2_req *);
+void h2_del_req(struct worker *, struct h2_req *);
 void h2_kill_req(struct worker *, struct h2_sess *, struct h2_req *, h2_error);
 int h2_rxframe(struct worker *, struct h2_sess *);
 h2_error h2_set_setting(struct h2_sess *, const uint8_t *);
diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index b668523a7..61c97a894 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -168,7 +168,7 @@ h2_new_req(const struct worker *wrk, struct h2_sess *h2,
 }
 
 void
-h2_del_req(struct worker *wrk, const struct h2_req *r2)
+h2_del_req(struct worker *wrk, struct h2_req *r2)
 {
 	struct h2_sess *h2;
 	struct sess *sp;

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:07 +0000 (UTC)
Subject: [master] 7ab694ac4 Set up 'wrk->vsl' pointer for H/2 session thread
Message-ID: <20210830083107.766CDA19F0@lists.varnish-cache.org>


commit 7ab694ac4ebfb044add8cd402f8a08b8b9faacab
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Aug 11 17:13:59 2021 +0200

    Set up 'wrk->vsl' pointer for H/2 session thread
    
    The H/2 session thread does have a VSL buffer already set up, but the
    'wrk->vsl' pointer was not set. This caused issue for e.g. LRU_NukeOne()
    as it wants to log. Set the buffer for the duration that the worker is
    dedicated as an H/2 session thread.

diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c
index 21b9a0e4e..f8f7ae755 100644
--- a/bin/varnishd/http2/cache_http2_session.c
+++ b/bin/varnishd/http2/cache_http2_session.c
@@ -360,10 +360,14 @@ h2_new_session(struct worker *wrk, void *arg)
 	AZ(h2->htc->priv);
 	h2->htc->priv = h2;
 
+	AZ(wrk->vsl);
+	wrk->vsl = h2->vsl;
+
 	if (req->err_code == H2_OU_MARKER && !h2_ou_session(wrk, h2, req)) {
 		assert(h2->refcnt == 1);
 		h2_del_req(wrk, h2->req0);
 		h2_del_sess(wrk, h2, SC_RX_JUNK);
+		wrk->vsl = NULL;
 		return;
 	}
 	assert(HTC_S_COMPLETE == H2_prism_complete(h2->htc));
@@ -432,6 +436,7 @@ h2_new_session(struct worker *wrk, void *arg)
 	assert(h2->refcnt == 1);
 	h2_del_req(wrk, h2->req0);
 	h2_del_sess(wrk, h2, h2->error->reason);
+	wrk->vsl = NULL;
 }
 
 struct transport H2_transport = {

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:07 +0000 (UTC)
Subject: [master] 91143910a Redo H/2 tx data handling
Message-ID: <20210830083107.97BB4A1A01@lists.varnish-cache.org>


commit 91143910a2ea2a7af7db390e02096786c7a78645
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 22 11:49:24 2021 +0200

    Redo H/2 tx data handling
    
    This implements stream data handling using a buffer between the H/2
    session thread and each stream thread. This is needed to avoid head of
    line blocking on the session socket when a data frame is received for a
    stream thread that is not yet ready to receive it.
    
    The buffer used will have to be as large as the send window the peer
    expects at the time the stream is opened. This will typically be 65535
    unless the h2_initial_window_size parameter has been changed.
    
    Stream window updates will then be issued only once data is removed from
    the buffer by the request body being consumed from the request handling
    thread, limited in size to what space is then available in the buffer.

diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h
index ee91d6fac..a075f6305 100644
--- a/bin/varnishd/http2/cache_http2.h
+++ b/bin/varnishd/http2/cache_http2.h
@@ -115,6 +115,16 @@ enum h2_stream_e {
 #define H2_FRAME_FLAGS(l,u,v)   extern const uint8_t H2FF_##u;
 #include "tbl/h2_frames.h"
 
+struct h2_rxbuf {
+	unsigned			magic;
+#define H2_RXBUF_MAGIC			0x73f9fb27
+	unsigned			size;
+	uint64_t			tail;
+	uint64_t			head;
+	struct stv_buffer		*stvbuf;
+	uint8_t				data[];
+};
+
 struct h2_req {
 	unsigned			magic;
 #define H2_REQ_MAGIC			0x03411584
@@ -134,7 +144,7 @@ struct h2_req {
 	/* Where to wake this stream up */
 	struct worker			*wrk;
 
-	ssize_t				reqbody_bytes;
+	struct h2_rxbuf			*rxbuf;
 
 	VTAILQ_ENTRY(h2_req)		tx_list;
 	h2_error			error;
@@ -147,7 +157,6 @@ struct h2_sess {
 #define H2_SESS_MAGIC			0xa16f7e4b
 
 	pthread_t			rxthr;
-	struct h2_req			*mailcall;
 	pthread_cond_t			*cond;
 	pthread_cond_t			winupd_cond[1];
 
diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 61c97a894..155717fac 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -40,6 +40,7 @@
 #include "cache/cache_filter.h"
 #include "http2/cache_http2.h"
 #include "cache/cache_objhead.h"
+#include "storage/storage.h"
 
 #include "vend.h"
 #include "vtcp.h"
@@ -172,6 +173,7 @@ h2_del_req(struct worker *wrk, struct h2_req *r2)
 {
 	struct h2_sess *h2;
 	struct sess *sp;
+	struct stv_buffer *stvbuf;
 
 	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
 	AZ(r2->scheduled);
@@ -185,7 +187,18 @@ h2_del_req(struct worker *wrk, struct h2_req *r2)
 	/* XXX: PRIORITY reshuffle */
 	VTAILQ_REMOVE(&h2->streams, r2, list);
 	Lck_Unlock(&sp->mtx);
+
 	assert(!WS_IsReserved(r2->req->ws));
+	AZ(r2->req->ws->r);
+
+	CHECK_OBJ_ORNULL(r2->rxbuf, H2_RXBUF_MAGIC);
+	if (r2->rxbuf) {
+		stvbuf = r2->rxbuf->stvbuf;
+		r2->rxbuf = NULL;
+		STV_FreeBuf(wrk, &stvbuf);
+		AZ(stvbuf);
+	}
+
 	Req_Cleanup(sp, wrk, r2->req);
 	Req_Release(r2->req);
 }
@@ -537,10 +550,6 @@ h2_do_req(struct worker *wrk, void *priv)
 		r2->scheduled = 0;
 		r2->state = H2_S_CLOSED;
 		r2->h2sess->do_sweep = 1;
-		if (h2->mailcall == r2) {
-			h2->mailcall = NULL;
-			AZ(pthread_cond_signal(h2->cond));
-		}
 		Lck_Unlock(&h2->sess->mtx);
 	}
 	THR_SetRequest(NULL);
@@ -582,8 +591,16 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2,
 	if (req->req_body_status == NULL) {
 		if (cl == -1)
 			req->req_body_status = BS_EOF;
-		else
+		else {
+			/* Note: If cl==0 here, we still need to have
+			 * req_body_status==BS_LENGTH, so that there will
+			 * be a wait for the stream to reach H2_S_CLOS_REM
+			 * while dealing with the request body. */
 			req->req_body_status = BS_LENGTH;
+		}
+		/* Set req->htc->content_length because this is used as
+		 * the hint in vrb_pull() for how large the storage
+		 * buffers need to be */
 		req->htc->content_length = cl;
 	} else {
 		/* A HEADER frame contained END_STREAM */
@@ -742,78 +759,181 @@ h2_rx_continuation(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
-	int w1 = 0, w2 = 0;
 	char buf[4];
-	unsigned wi;
-	ssize_t cl;
+	uint64_t l, l2, head;
+	const uint8_t *src;
+
+	/* XXX: Shouldn't error handling, setting of r2->error and
+	 * r2->cond signalling be handled more generally at the end of
+	 * procframe()??? */
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
 	CHECK_OBJ_ORNULL(r2, H2_REQ_MAGIC);
 
-	if (r2 == NULL || !r2->scheduled)
+	if (r2 == NULL)
 		return (0);
+
 	if (r2->state >= H2_S_CLOS_REM) {
 		r2->error = H2SE_STREAM_CLOSED;
 		return (H2SE_STREAM_CLOSED); // rfc7540,l,1766,1769
 	}
+
 	Lck_Lock(&h2->sess->mtx);
-	while (h2->mailcall != NULL && h2->error == 0 && r2->error == 0)
-		AZ(Lck_CondWait(h2->cond, &h2->sess->mtx, 0));
+	CHECK_OBJ_ORNULL(r2->rxbuf, H2_RXBUF_MAGIC);
+
 	if (h2->error || r2->error) {
+		if (r2->cond)
+			AZ(pthread_cond_signal(r2->cond));
 		Lck_Unlock(&h2->sess->mtx);
 		return (h2->error ? h2->error : r2->error);
 	}
 
-	r2->reqbody_bytes += h2->rxf_len;
-	if (h2->rxf_flags & H2FF_DATA_END_STREAM)
-		r2->state = H2_S_CLOS_REM;
-	cl = r2->req->htc->content_length;
-	if (cl >= 0 && (r2->reqbody_bytes > cl ||
-	      (r2->state >= H2_S_CLOS_REM && r2->reqbody_bytes != cl))) {
-		VSLb(h2->vsl, SLT_Debug,
-		    "H2: stream %u: Received data and Content-Length"
-		    " mismatch", h2->rxf_stream);
-		r2->error = H2SE_PROTOCOL_ERROR; // rfc7540,l,3150,3163
+	/* Check against the Content-Length header if given */
+	if (r2->req->htc->content_length >= 0) {
+		if (r2->rxbuf)
+			l = r2->rxbuf->head;
+		else
+			l = 0;
+		l += h2->rxf_len;
+		if (l > r2->req->htc->content_length ||
+		    ((h2->rxf_flags & H2FF_DATA_END_STREAM) &&
+		     l != r2->req->htc->content_length)) {
+			VSLb(h2->vsl, SLT_Debug,
+			    "H2: stream %u: Received data and Content-Length"
+			    " mismatch", h2->rxf_stream);
+			r2->error = H2SE_PROTOCOL_ERROR;
+			if (r2->cond)
+				AZ(pthread_cond_signal(r2->cond));
+			Lck_Unlock(&h2->sess->mtx);
+			return (H2SE_PROTOCOL_ERROR);
+		}
+	}
+
+	/* Handle zero size frame before starting to allocate buffers */
+	if (h2->rxf_len == 0) {
+		if (h2->rxf_flags & H2FF_DATA_END_STREAM)
+			r2->state = H2_S_CLOS_REM;
 		if (r2->cond)
 			AZ(pthread_cond_signal(r2->cond));
 		Lck_Unlock(&h2->sess->mtx);
-		return (H2SE_PROTOCOL_ERROR);
+		return (0);
 	}
 
-	AZ(h2->mailcall);
-	h2->mailcall = r2;
+	/* Check and charge connection window */
+	if (h2->rxf_len > h2->req0->r_window) {
+		VSLb(h2->vsl, SLT_Debug,
+		    "H2: stream %u: Exceeded connection receive window",
+		    h2->rxf_stream);
+		r2->error = H2CE_FLOW_CONTROL_ERROR;
+		if (r2->cond)
+			AZ(pthread_cond_signal(r2->cond));
+		Lck_Unlock(&h2->sess->mtx);
+		return (H2CE_FLOW_CONTROL_ERROR);
+	}
 	h2->req0->r_window -= h2->rxf_len;
-	r2->r_window -= h2->rxf_len;
-	// req_bodybytes accounted in CNT code.
-	if (r2->cond)
-		AZ(pthread_cond_signal(r2->cond));
-	while (h2->mailcall != NULL && h2->error == 0 && r2->error == 0)
-		AZ(Lck_CondWait(h2->cond, &h2->sess->mtx, 0));
-	wi = cache_param->h2_rx_window_increment;
 	if (h2->req0->r_window < cache_param->h2_rx_window_low_water) {
-		h2->req0->r_window += wi;
-		w1 = 1;
+		h2->req0->r_window += cache_param->h2_rx_window_increment;
+		vbe32enc(buf, cache_param->h2_rx_window_increment);
+		Lck_Unlock(&h2->sess->mtx);
+		H2_Send_Get(wrk, h2, h2->req0);
+		H2_Send_Frame(wrk, h2, H2_F_WINDOW_UPDATE, 0, 4, 0, buf);
+		H2_Send_Rel(h2, h2->req0);
+		Lck_Lock(&h2->sess->mtx);
 	}
-	if (r2->r_window < cache_param->h2_rx_window_low_water) {
-		r2->r_window += wi;
-		w2 = 1;
+
+	/* Check stream window */
+	if (h2->rxf_len > r2->r_window) {
+		VSLb(h2->vsl, SLT_Debug,
+		    "H2: stream %u: Exceeded stream receive window",
+		    h2->rxf_stream);
+		r2->error = H2SE_FLOW_CONTROL_ERROR;
+		if (r2->cond)
+			AZ(pthread_cond_signal(r2->cond));
+		Lck_Unlock(&h2->sess->mtx);
+		return (H2SE_FLOW_CONTROL_ERROR);
 	}
 
+	/* Make the buffer on demand */
+	if (r2->rxbuf == NULL) {
+		unsigned bufsize;
+		size_t bstest;
+		struct stv_buffer *stvbuf;
+		struct h2_rxbuf *rxbuf;
 
-	Lck_Unlock(&h2->sess->mtx);
+		Lck_Unlock(&h2->sess->mtx);
 
-	if (w1 || w2) {
-		vbe32enc(buf, wi);
-		H2_Send_Get(wrk, h2, h2->req0);
-		if (w1)
-			H2_Send_Frame(wrk, h2, H2_F_WINDOW_UPDATE, 0,
-			    4, 0, buf);
-		if (w2)
-			H2_Send_Frame(wrk, h2, H2_F_WINDOW_UPDATE, 0,
-			    4, r2->stream, buf);
-		H2_Send_Rel(h2, h2->req0);
+		bufsize = r2->r_window;
+		/* This is the first data frame, so r_window will be the
+		 * initial window size. */
+		assert(bufsize > 0);
+		if ((h2->rxf_flags & H2FF_DATA_END_STREAM) &&
+		    bufsize > h2->rxf_len)
+			/* Cap the buffer size when we know this is the
+			 * single data frame. */
+			bufsize = h2->rxf_len;
+		stvbuf = STV_AllocBuf(wrk, stv_transient,
+		    bufsize + sizeof *rxbuf);
+		if (stvbuf == NULL) {
+			VSLb(h2->vsl, SLT_Debug,
+			    "H2: stream %u: Failed to allocate request body"
+			    " buffer",
+			    h2->rxf_stream);
+			Lck_Lock(&h2->sess->mtx);
+			r2->error = H2SE_INTERNAL_ERROR;
+			if (r2->cond)
+				AZ(pthread_cond_signal(r2->cond));
+			Lck_Unlock(&h2->sess->mtx);
+			return (H2SE_INTERNAL_ERROR);
+		}
+		rxbuf = STV_GetBufPtr(stvbuf, &bstest);
+		AN(rxbuf);
+		assert(bstest >= bufsize + sizeof *rxbuf);
+		assert(PAOK(rxbuf));
+		INIT_OBJ(rxbuf, H2_RXBUF_MAGIC);
+		rxbuf->size = bufsize;
+		rxbuf->stvbuf = stvbuf;
+
+		r2->rxbuf = rxbuf;
+
+		Lck_Lock(&h2->sess->mtx);
 	}
+
+	CHECK_OBJ_NOTNULL(r2->rxbuf, H2_RXBUF_MAGIC);
+	assert(r2->rxbuf->tail <= r2->rxbuf->head);
+	l = r2->rxbuf->head - r2->rxbuf->tail;
+	assert(l <= r2->rxbuf->size);
+	l = r2->rxbuf->size - l;
+	assert(h2->rxf_len <= l); /* Stream window handling ensures
+				   * this */
+
+	Lck_Unlock(&h2->sess->mtx);
+
+	src = h2->rxf_data;
+	l = h2->rxf_len;
+	head = r2->rxbuf->head;
+	do {
+		l2 = l;
+		if ((head % r2->rxbuf->size) + l2 > r2->rxbuf->size)
+			l2 = r2->rxbuf->size - (head % r2->rxbuf->size);
+		assert(l2 > 0);
+		memcpy(&r2->rxbuf->data[head % r2->rxbuf->size], src, l2);
+		src += l2;
+		head += l2;
+		l -= l2;
+	} while (l > 0);
+
+	Lck_Lock(&h2->sess->mtx);
+	/* Charge stream window */
+	r2->r_window -= h2->rxf_len;
+	r2->rxbuf->head += h2->rxf_len;
+	assert(r2->rxbuf->tail <= r2->rxbuf->head);
+	if (h2->rxf_flags & H2FF_DATA_END_STREAM)
+		r2->state = H2_S_CLOS_REM;
+	if (r2->cond)
+		AZ(pthread_cond_signal(r2->cond));
+	Lck_Unlock(&h2->sess->mtx);
+
 	return (0);
 }
 
@@ -822,8 +942,10 @@ h2_vfp_body(struct vfp_ctx *vc, struct vfp_entry *vfe, void *ptr, ssize_t *lp)
 {
 	struct h2_req *r2;
 	struct h2_sess *h2;
-	unsigned l;
 	enum vfp_status retval;
+	uint64_t l, l2, tail;
+	uint8_t *dst;
+	char buf[4];
 
 	CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
@@ -832,40 +954,87 @@ h2_vfp_body(struct vfp_ctx *vc, struct vfp_entry *vfe, void *ptr, ssize_t *lp)
 
 	AN(ptr);
 	AN(lp);
-	l = *lp;
-	*lp = 0;
+	assert(*lp >= 0);
 
 	Lck_Lock(&h2->sess->mtx);
+
 	r2->cond = &vc->wrk->cond;
-	while (h2->mailcall != r2 && h2->error == 0 && r2->error == 0)
-		AZ(Lck_CondWait(r2->cond, &h2->sess->mtx, 0));
-	r2->cond = NULL;
-	if (h2->error || r2->error) {
-		retval = VFP_ERROR;
-	} else {
-		assert(h2->mailcall == r2);
-		if (l > h2->rxf_len)
-			l = h2->rxf_len;
-		if (l > 0) {
-			memcpy(ptr, h2->rxf_data, l);
-			h2->rxf_data += l;
-			h2->rxf_len -= l;
-		}
-		*lp = l;
-		if (h2->rxf_len > 0) {
-			/* We ran out of storage: Have VFP call us
-			 * again with a fresh buffer */
-			Lck_Unlock(&h2->sess->mtx);
-			return (VFP_OK);
-		}
-		if (h2->rxf_len == 0 && r2->state >= H2_S_CLOS_REM)
+	while (1) {
+		CHECK_OBJ_ORNULL(r2->rxbuf, H2_RXBUF_MAGIC);
+		if (r2->rxbuf) {
+			assert(r2->rxbuf->tail <= r2->rxbuf->head);
+			l = r2->rxbuf->head - r2->rxbuf->tail;
+		} else
+			l = 0;
+
+		if (h2->error || r2->error)
+			retval = VFP_ERROR;
+		else if (r2->state >= H2_S_CLOS_REM && l <= *lp)
 			retval = VFP_END;
-		else
+		else {
+			if (l > *lp)
+				l = *lp;
 			retval = VFP_OK;
-		h2->mailcall = NULL;
-		AZ(pthread_cond_signal(h2->cond));
+		}
+
+		if (retval != VFP_OK || l > 0)
+			break;
+
+		/* XXX: Timeout */
+		AZ(Lck_CondWait(r2->cond, &h2->sess->mtx, 0));
 	}
+	r2->cond = NULL;
+
+	Lck_Unlock(&h2->sess->mtx);
+
+	if (l == 0 || retval == VFP_ERROR) {
+		*lp = 0;
+		return (retval);
+	}
+
+	*lp = l;
+	dst = ptr;
+	tail = r2->rxbuf->tail;
+	do {
+		l2 = l;
+		if ((tail % r2->rxbuf->size) + l2 > r2->rxbuf->size)
+			l2 = r2->rxbuf->size - (tail % r2->rxbuf->size);
+		assert(l2 > 0);
+		memcpy(dst, &r2->rxbuf->data[tail % r2->rxbuf->size], l2);
+		dst += l2;
+		tail += l2;
+		l -= l2;
+	} while (l > 0);
+
+	Lck_Lock(&h2->sess->mtx);
+
+	CHECK_OBJ_NOTNULL(r2->rxbuf, H2_RXBUF_MAGIC);
+	r2->rxbuf->tail = tail;
+	assert(r2->rxbuf->tail <= r2->rxbuf->head);
+
+	if (r2->r_window < cache_param->h2_rx_window_low_water &&
+	    r2->state < H2_S_CLOS_REM) {
+		/* l is free buffer space */
+		/* l2 is calculated window increment */
+		l = r2->rxbuf->size - (r2->rxbuf->head - r2->rxbuf->tail);
+		assert(r2->r_window <= l);
+		l2 = cache_param->h2_rx_window_increment;
+		if (r2->r_window + l2 > l)
+			l2 = l - r2->r_window;
+		r2->r_window += l2;
+	} else
+		l2 = 0;
+
 	Lck_Unlock(&h2->sess->mtx);
+
+	if (l2 > 0) {
+		vbe32enc(buf, l2);
+		H2_Send_Get(vc->wrk, h2, r2);
+		H2_Send_Frame(vc->wrk, h2, H2_F_WINDOW_UPDATE, 0, 4,
+		    r2->stream, buf);
+		H2_Send_Rel(h2, r2);
+	}
+
 	return (retval);
 }
 
@@ -874,6 +1043,7 @@ h2_vfp_body_fini(struct vfp_ctx *vc, struct vfp_entry *vfe)
 {
 	struct h2_req *r2;
 	struct h2_sess *h2;
+	struct stv_buffer *stvbuf = NULL;
 
 	CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
@@ -889,11 +1059,21 @@ h2_vfp_body_fini(struct vfp_ctx *vc, struct vfp_entry *vfe)
 		H2_Send_Rel(h2, r2);
 		Lck_Lock(&h2->sess->mtx);
 		r2->error = H2SE_REFUSED_STREAM;
-		if (h2->mailcall == r2) {
-			h2->mailcall = NULL;
-			AZ(pthread_cond_signal(h2->cond));
+		Lck_Unlock(&h2->sess->mtx);
+	}
+
+	if (r2->state >= H2_S_CLOS_REM && r2->rxbuf != NULL) {
+		Lck_Lock(&h2->sess->mtx);
+		CHECK_OBJ_ORNULL(r2->rxbuf, H2_RXBUF_MAGIC);
+		if (r2->rxbuf != NULL) {
+			stvbuf = r2->rxbuf->stvbuf;
+			r2->rxbuf = NULL;
 		}
 		Lck_Unlock(&h2->sess->mtx);
+		if (stvbuf != NULL) {
+			STV_FreeBuf(vc->wrk, &stvbuf);
+			AZ(stvbuf);
+		}
 	}
 }
 

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:07 +0000 (UTC)
Subject: [master] 96b196b54 Make f00007.vtc stable
Message-ID: <20210830083107.B6AE2A1A0A@lists.varnish-cache.org>


commit 96b196b54d8ee892eef15398e7747923dc124935
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 22 11:50:19 2021 +0200

    Make f00007.vtc stable

diff --git a/bin/varnishtest/tests/f00007.vtc b/bin/varnishtest/tests/f00007.vtc
index 23a2b7657..e982548a0 100644
--- a/bin/varnishtest/tests/f00007.vtc
+++ b/bin/varnishtest/tests/f00007.vtc
@@ -62,6 +62,7 @@ client c3 {
 	stream 1 {
 		txreq -req POST -url /3 -hdr "content-length" "1" -nostrend
 		txdata -data "A" -nostrend
+		delay 0.5
 		txdata -data "GET /FAIL HTTP/1.1\r\n\r\n"
 		rxrst
 		expect rst.err == PROTOCOL_ERROR

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:07 +0000 (UTC)
Subject: [master] 2776433e6 Use `timeout_idle` as a timeout for request body
Message-ID: <20210830083107.D8745A1A17@lists.varnish-cache.org>


commit 2776433e682f396cf07f9022e5176b82d162ca32
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 22 11:49:28 2021 +0200

    Use `timeout_idle` as a timeout for request body
    
    H2 streams waiting for request body data will timeout after timeout_idle
    seconds if no new data on the stream is being received. This will ensure
    that individual H2 streams can be reaped if there is no data received from
    the peer.

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 155717fac..570ef9209 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -33,6 +33,7 @@
 
 #include "cache/cache_varnishd.h"
 
+#include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 
@@ -946,6 +947,7 @@ h2_vfp_body(struct vfp_ctx *vc, struct vfp_entry *vfe, void *ptr, ssize_t *lp)
 	uint64_t l, l2, tail;
 	uint8_t *dst;
 	char buf[4];
+	int i;
 
 	CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC);
@@ -980,8 +982,12 @@ h2_vfp_body(struct vfp_ctx *vc, struct vfp_entry *vfe, void *ptr, ssize_t *lp)
 		if (retval != VFP_OK || l > 0)
 			break;
 
-		/* XXX: Timeout */
-		AZ(Lck_CondWait(r2->cond, &h2->sess->mtx, 0));
+		i = Lck_CondWait(r2->cond, &h2->sess->mtx,
+		    VTIM_real() + SESS_TMO(h2->sess, timeout_idle));
+		if (i == ETIMEDOUT) {
+			retval = VFP_ERROR;
+			break;
+		}
 	}
 	r2->cond = NULL;
 

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:08 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:08 +0000 (UTC)
Subject: [master] 739c54da7 Adjust the minimum value of h2_initial_window_size
Message-ID: <20210830083108.08A03A1A1C@lists.varnish-cache.org>


commit 739c54da7e9abb1a8a00b02189e46ca66ba25459
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 22 11:49:31 2021 +0200

    Adjust the minimum value of h2_initial_window_size
    
    We have a strict min at the protocol default here. This is because we
    don't have the 'use settings only after peer ack' in place yet. If the
    value is lower than the protocol default, the very first stream could get
    a flow control error.

diff --git a/include/tbl/params.h b/include/tbl/params.h
index 7d383b8d2..ccd5a8f54 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -1195,15 +1195,19 @@ PARAM_SIMPLE(
 	"at the same time for a single HTTP2 connection."
 )
 
+/* We have a strict min at the protocol default here. This is because we
+ * don't have the 'use settings only after peer ack' in place yet. If the
+ * value is lower than the protocol default, the very first stream could
+ * get a flow control error. */
 PARAM_SIMPLE(
 	/* name */	h2_initial_window_size,
 	/* type */	bytes_u,
-	/* min */	"0",
+	/* min */	"65535b",
 	/* max */	"2147483647b",
 	/* def */	"65535b",
 	/* units */	"bytes",
 	/* descr */
-	"HTTP2 initial flow control window size."
+	"HTTP2 initial flow control window size.",
 )
 
 PARAM_SIMPLE(

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:08 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:08 +0000 (UTC)
Subject: [master] 728549101 Make H/2 varnishtest ignore window updates while
 in rxreq/rxresp
Message-ID: <20210830083108.280A4A1A2A@lists.varnish-cache.org>


commit 7285491019712223ca1c33b873d0d21362a35107
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 22 11:49:35 2021 +0200

    Make H/2 varnishtest ignore window updates while in rxreq/rxresp
    
    This makes it easier to not have to know exactly when and how many window
    updates to expect in a test case.

diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c
index be80d32ed..750d6f074 100644
--- a/bin/varnishtest/vtc_http2.c
+++ b/bin/varnishtest/vtc_http2.c
@@ -2302,9 +2302,11 @@ cmd_rxmsg(CMD_ARGS)
 	else
 		ONLY_H2_CLIENT(s->hp, av);
 
-	f = rxstuff(s);
-	if (!f)
-		return;
+	do {
+		f = rxstuff(s);
+		if (!f)
+			return;
+	} while (f->type == TYPE_WINDOW_UPDATE);
 
 	rcv++;
 	CHKFRAME(f->type, TYPE_HEADERS, rcv, *av);

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:08 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:08 +0000 (UTC)
Subject: [master] 34737e22e Stabilise test cases
Message-ID: <20210830083108.46E52A1A2E@lists.varnish-cache.org>


commit 34737e22e43b50379e0ad0bca47de51631c0ef0c
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 22 11:49:39 2021 +0200

    Stabilise test cases
    
    With the new request body data handling, Varnish changes behaviour
    significantly wrt to stream window updates sent to the client. Window
    updates will only be sent once the data is consumed by the client through
    the request body VFP handling. Test cases that rely on receiving a window
    update to sync the H/2 stream needs to be adopted.

diff --git a/bin/varnishtest/tests/r02679.vtc b/bin/varnishtest/tests/r02679.vtc
index d1284e2f5..590dfb264 100644
--- a/bin/varnishtest/tests/r02679.vtc
+++ b/bin/varnishtest/tests/r02679.vtc
@@ -22,7 +22,6 @@ client c1 {
 	stream 1 {
 		txreq -req POST -hdr "content-length" "31469" -nostrend
 		txdata -datalen 1550 -nostrend
-		rxwinup
 		txdata -datalen 16000 -nostrend
 		txdata -datalen 13919
 		rxresp
diff --git a/bin/varnishtest/tests/t02014.vtc b/bin/varnishtest/tests/t02014.vtc
index e73477311..f47eed657 100644
--- a/bin/varnishtest/tests/t02014.vtc
+++ b/bin/varnishtest/tests/t02014.vtc
@@ -2,7 +2,13 @@ varnishtest "Exercise h/2 sender flow control code"
 
 barrier b1 sock 3 -cyclic
 
-server s1 -repeat 2 {
+server s1 {
+	rxreq
+	txresp -bodylen 66300
+} -start
+
+server s2 {
+	non_fatal
 	rxreq
 	txresp -bodylen 66300
 } -start
@@ -10,6 +16,12 @@ server s1 -repeat 2 {
 varnish v1 -vcl+backend {
 	import vtc;
 
+	sub vcl_backend_fetch {
+		if (bereq.method == "POST") {
+			set bereq.backend = s2;
+		}
+	}
+
 	sub vcl_deliver {
 		vtc.barrier_sync("${b1_sock}");
 	}
@@ -47,7 +59,7 @@ client c1 {
 	stream 0 -wait
 } -run
 
-client c1 {
+client c2 {
 	stream 0 {
 		barrier b1 sync
 	} -start
@@ -63,7 +75,7 @@ client c1 {
 	stream 0 -wait
 } -run
 
-client c1 {
+client c3 {
 	stream 0 {
 		barrier b1 sync
 		barrier b1 sync
@@ -79,7 +91,6 @@ client c1 {
 	stream 1 {
 		txreq -req "POST" -nostrend
 		txdata -data "ok"
-		rxwinup
 		txdata -data "fail"
 		rxrst
 		expect rst.err == STREAM_CLOSED

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:08 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:08 +0000 (UTC)
Subject: [master] 843a141b3 Add H/2 stream data head of line blocking test case
Message-ID: <20210830083108.668C5A1A34@lists.varnish-cache.org>


commit 843a141b3ee88fc7a180386a6ea9ecb9d7ccebc5
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 22 11:49:43 2021 +0200

    Add H/2 stream data head of line blocking test case
    
    This is the test case that fails if these changes aren't in tree. Note the
    commented out rxwinup commands that are necessary for the proper fail mode
    when run without the varnishtest window update changes.

diff --git a/bin/varnishtest/tests/t02017.vtc b/bin/varnishtest/tests/t02017.vtc
new file mode 100644
index 000000000..5090e6262
--- /dev/null
+++ b/bin/varnishtest/tests/t02017.vtc
@@ -0,0 +1,46 @@
+varnishtest "H/2 stream data head of line blocking"
+
+barrier b1 cond 2
+barrier b2 cond 2
+barrier b3 cond 2
+barrier b4 cond 2
+
+server s1 {
+	rxreq
+	barrier b4 sync
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		if (req.url == "/2") {
+			return (synth(700));
+		}
+	}
+} -start
+
+varnish v1 -cliok "param.set feature +http2"
+
+client c1 {
+	stream 1 {
+		txreq -req GET -url /1 -hdr "content-length" "1" -nostrend
+		barrier b1 sync
+		barrier b2 sync
+		txdata -data 1
+#		rxwinup
+		barrier b3 sync
+		rxresp
+		expect resp.status == 200
+	} -start
+	stream 3 {
+		barrier b1 sync
+		txreq -req GET -url /2 -hdr "content-length" "1" -nostrend
+		barrier b2 sync
+		barrier b3 sync
+		txdata -data 2
+#		rxwinup
+		rxresp
+		expect resp.status == 700
+		barrier b4 sync
+	} -start
+} -run

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:08 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:08 +0000 (UTC)
Subject: [master] 613036a06 Add test case for large req body buffer use
Message-ID: <20210830083108.8C9F8A1A67@lists.varnish-cache.org>


commit 613036a067c9f70da76c062be9e036dd047d79bd
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 22 11:49:46 2021 +0200

    Add test case for large req body buffer use

diff --git a/bin/varnishtest/tests/t02018.vtc b/bin/varnishtest/tests/t02018.vtc
new file mode 100644
index 000000000..b6ff9eec8
--- /dev/null
+++ b/bin/varnishtest/tests/t02018.vtc
@@ -0,0 +1,36 @@
+varnishtest "H/2 stream multiple buffer exhaustion"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+} -start
+
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.reset h2_initial_window_size"
+varnish v1 -cliok "param.reset h2_rx_window_low_water"
+
+client c1 {
+	stream 1 {
+		txreq -req GET -url /1 -hdr "content-length" "131072" -nostrend
+		txdata -datalen 16384 -nostrend
+		rxwinup
+		txdata -datalen 16384 -nostrend
+		rxwinup
+		txdata -datalen 16384 -nostrend
+		rxwinup
+		txdata -datalen 16384 -nostrend
+		rxwinup
+		txdata -datalen 16384 -nostrend
+		rxwinup
+		txdata -datalen 16384 -nostrend
+		rxwinup
+		txdata -datalen 16384 -nostrend
+		rxwinup
+		txdata -datalen 16384
+		rxresp
+		expect resp.status == 200
+	} -start
+} -run

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:08 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:08 +0000 (UTC)
Subject: [master] 1590acee8 Improve the panic output when triggered on an H2
 session
Message-ID: <20210830083108.CB51EA1A7B@lists.varnish-cache.org>


commit 1590acee878b9bad6073d3ffac09f42caf2b31de
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 22 11:49:53 2021 +0200

    Improve the panic output when triggered on an H2 session

diff --git a/bin/varnishd/http2/cache_http2_panic.c b/bin/varnishd/http2/cache_http2_panic.c
index e2a3e1e0a..dd580fc34 100644
--- a/bin/varnishd/http2/cache_http2_panic.c
+++ b/bin/varnishd/http2/cache_http2_panic.c
@@ -36,6 +36,15 @@
 #include "cache/cache_transport.h"
 #include "http2/cache_http2.h"
 
+static const char *
+h2_panic_error(const struct h2_error_s *e)
+{
+	if (e == NULL)
+		return ("(null)");
+	else
+		return (e->name);
+}
+
 void
 h2_sess_panic(struct vsb *vsb, const struct sess *sp)
 {
@@ -49,21 +58,38 @@ h2_sess_panic(struct vsb *vsb, const struct sess *sp)
 	h2 = (void*)*up;
 	if (PAN_dump_struct(vsb, h2, H2_SESS_MAGIC, "h2_sess"))
 		return;
-	h2 = (void*)*up;
+	VSB_printf(vsb, "refcnt = %d, bogosity = %d, error = %s\n",
+	    h2->refcnt, h2->bogosity, h2_panic_error(h2->error));
+	VSB_printf(vsb,
+	    "open_streams = %u, highest_stream = %u,"
+	    " goaway_last_stream = %u,\n",
+	    h2->open_streams, h2->highest_stream, h2->goaway_last_stream);
+	VSB_printf(vsb,
+	    "{rxf_len, rxf_type, rxf_flags, rxf_stream} ="
+	    " {%u, %u, 0x%x, %u},\n",
+	    h2->rxf_len, h2->rxf_type, h2->rxf_flags, h2->rxf_stream);
 	VTAILQ_FOREACH(r2, &h2->streams, list) {
 		if (PAN_dump_struct(vsb, r2, H2_REQ_MAGIC, "stream"))
 			continue;
-		VSB_printf(vsb, "0x%08x", r2->stream);
+		VSB_printf(vsb, "id = %u, state = ", r2->stream);
 		switch (r2->state) {
-#define H2_STREAM(U,sd,d) case H2_S_##U: VSB_printf(vsb, " %-6s", sd); break;
+#define H2_STREAM(U,sd,d) case H2_S_##U: VSB_printf(vsb, "%s", sd); break;
 #include <tbl/h2_stream.h>
 		default:
-			VSB_printf(vsb, " State %d", r2->state);
+			VSB_printf(vsb, " 0x%x", r2->state);
 			break;
 		}
-		VSB_cat(vsb, "},\n");
+		VSB_cat(vsb, ",\n");
+
+		VSB_printf(vsb, "h2_sess = %p, scheduled = %d, error = %s,\n",
+		    r2->h2sess, r2->scheduled, h2_panic_error(r2->error));
+		VSB_printf(vsb, "t_send = %f, t_winupd = %f,\n",
+		    r2->t_send, r2->t_winupd);
+		VSB_printf(vsb, "t_window = %jd, r_window = %jd,\n",
+		    r2->t_window, r2->r_window);
+
 		VSB_indent(vsb, -2);
+		VSB_cat(vsb, "},\n");
 	}
-	VSB_cat(vsb, "},\n");
 	VSB_indent(vsb, -2);
 }

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:08 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:08 +0000 (UTC)
Subject: [master] 529a35dd4 Panic dump H2 rxbuf
Message-ID: <20210830083109.05EBFA1A81@lists.varnish-cache.org>


commit 529a35dd470660a72907b0fc1cb31ecc53ecf0e5
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 22 11:49:57 2021 +0200

    Panic dump H2 rxbuf

diff --git a/bin/varnishd/http2/cache_http2_panic.c b/bin/varnishd/http2/cache_http2_panic.c
index dd580fc34..1539d69ec 100644
--- a/bin/varnishd/http2/cache_http2_panic.c
+++ b/bin/varnishd/http2/cache_http2_panic.c
@@ -88,6 +88,15 @@ h2_sess_panic(struct vsb *vsb, const struct sess *sp)
 		VSB_printf(vsb, "t_window = %jd, r_window = %jd,\n",
 		    r2->t_window, r2->r_window);
 
+		if (!PAN_dump_struct(vsb, r2->rxbuf, H2_RXBUF_MAGIC, "rxbuf")) {
+			VSB_printf(vsb, "stvbuf = %p,\n", r2->rxbuf->stvbuf);
+			VSB_printf(vsb,
+			    "{size, tail, head} = {%u, %ju, %ju},\n",
+			    r2->rxbuf->size, r2->rxbuf->tail, r2->rxbuf->head);
+			VSB_indent(vsb, -2);
+			VSB_cat(vsb, "},\n");
+		}
+
 		VSB_indent(vsb, -2);
 		VSB_cat(vsb, "},\n");
 	}

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:09 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:09 +0000 (UTC)
Subject: [master] 8a18531a1 Panic H2 local and remote settings
Message-ID: <20210830083109.34464A1A92@lists.varnish-cache.org>


commit 8a18531a166bb35142a86c12ac8b12c85b404e69
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 22 11:50:00 2021 +0200

    Panic H2 local and remote settings

diff --git a/bin/varnishd/http2/cache_http2_panic.c b/bin/varnishd/http2/cache_http2_panic.c
index 1539d69ec..25dda29a0 100644
--- a/bin/varnishd/http2/cache_http2_panic.c
+++ b/bin/varnishd/http2/cache_http2_panic.c
@@ -45,6 +45,20 @@ h2_panic_error(const struct h2_error_s *e)
 		return (e->name);
 }
 
+static void
+h2_panic_settings(struct vsb *vsb, const struct h2_settings *s)
+{
+	int cont = 0;
+
+#define H2_SETTING(U,l,...)			\
+	if (cont)				\
+		VSB_printf(vsb, ", ");		\
+	cont = 1;				\
+	VSB_printf(vsb, "0x%x", s->l);
+#include "tbl/h2_settings.h"
+#undef H2_SETTING
+}
+
 void
 h2_sess_panic(struct vsb *vsb, const struct sess *sp)
 {
@@ -64,6 +78,12 @@ h2_sess_panic(struct vsb *vsb, const struct sess *sp)
 	    "open_streams = %u, highest_stream = %u,"
 	    " goaway_last_stream = %u,\n",
 	    h2->open_streams, h2->highest_stream, h2->goaway_last_stream);
+	VSB_cat(vsb, "local_settings = {");
+	h2_panic_settings(vsb, &h2->local_settings);
+	VSB_cat(vsb, "},\n");
+	VSB_cat(vsb, "remote_settings = {");
+	h2_panic_settings(vsb, &h2->remote_settings);
+	VSB_cat(vsb, "},\n");
 	VSB_printf(vsb,
 	    "{rxf_len, rxf_type, rxf_flags, rxf_stream} ="
 	    " {%u, %u, 0x%x, %u},\n",

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:09 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:09 +0000 (UTC)
Subject: [master] 82f073574 Test case that buffers fully before starting to
 consume the req body
Message-ID: <20210830083109.565F1A1A9A@lists.varnish-cache.org>


commit 82f073574185b8e1f60f5fe1675a2993917f95ff
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 22 11:50:03 2021 +0200

    Test case that buffers fully before starting to consume the req body

diff --git a/bin/varnishtest/tests/t02019.vtc b/bin/varnishtest/tests/t02019.vtc
new file mode 100644
index 000000000..3adcf2603
--- /dev/null
+++ b/bin/varnishtest/tests/t02019.vtc
@@ -0,0 +1,43 @@
+varnishtest "H/2 stream early buffer exhaustion"
+
+barrier b1 sock 2
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import vtc;
+	sub vcl_recv {
+		vtc.barrier_sync("${b1_sock}");
+		vtc.sleep(0.1s);
+	}
+} -start
+
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.reset h2_initial_window_size"
+varnish v1 -cliok "param.reset h2_rx_window_low_water"
+
+client c1 {
+	stream 1 {
+		txreq -req POST -url /1 -hdr "content-length" "131072" -nostrend
+		txdata -datalen 16384 -nostrend
+		txdata -datalen 16384 -nostrend
+		txdata -datalen 16384 -nostrend
+		txdata -datalen 16383 -nostrend
+		barrier b1 sync
+		rxwinup
+		txdata -datalen 16384 -nostrend
+		rxwinup
+		txdata -datalen 16384 -nostrend
+		rxwinup
+		txdata -datalen 16384 -nostrend
+		rxwinup
+		txdata -datalen 16384 -nostrend
+		rxwinup
+		txdata -datalen 1
+		rxresp
+		expect resp.status == 200
+	} -run
+} -run

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:09 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:09 +0000 (UTC)
Subject: [master] 31d5925e4 Varnishtest: Allow padding up to and including 255
 bytes
Message-ID: <20210830083109.766D7A1AA4@lists.varnish-cache.org>


commit 31d5925e45584d36a6133acb6d38d54746e335af
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 22 11:50:07 2021 +0200

    Varnishtest: Allow padding up to and including 255 bytes
    
    According to the spec the padding is an 8-bit field, and fields should be
    treated as unsigned unless otherwise specified, which it is not for any of
    the padding related places. Allow varnishtest to generate padding up to
    255 bytes long.

diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c
index 750d6f074..297045ff9 100644
--- a/bin/varnishtest/vtc_http2.c
+++ b/bin/varnishtest/vtc_http2.c
@@ -1633,8 +1633,8 @@ cmd_tx11obj(CMD_ARGS)
 			exclusive_stream_dependency(s);
 	}
 	if (pad) {
-		if (strlen(pad) >= 128)
-			vtc_fatal(vl, "Padding is limited to 128 bytes");
+		if (strlen(pad) > 255)
+			vtc_fatal(vl, "Padding is limited to 255 bytes");
 		f.flags |= PADDED;
 		assert(f.size + strlen(pad) < BUF_SIZE);
 		memmove(buf + 1, buf, f.size);
@@ -1726,8 +1726,8 @@ cmd_txdata(CMD_ARGS)
 
 	if (pad) {
 		f.flags |= PADDED;
-		if (strlen(pad) >= 128)
-			vtc_fatal(vl, "Padding is limited to 128 bytes");
+		if (strlen(pad) > 255)
+			vtc_fatal(vl, "Padding is limited to 255 bytes");
 		data = malloc( 1 + strlen(body) + strlen(pad));
 		AN(data);
 		*((uint8_t *)data) = strlen(pad);

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:09 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:09 +0000 (UTC)
Subject: [master] bfdfa6a5c Implement handling of padding bytes in received
 data frames
Message-ID: <20210830083109.9D963A1AB9@lists.varnish-cache.org>


commit bfdfa6a5c884a17bbf265649ef9ecf90af6940f7
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 22 11:50:11 2021 +0200

    Implement handling of padding bytes in received data frames
    
    This was found lacking in our H2 implementation. Previously we would have
    included any padding bytes in the request body. Possibly it would have
    caused errors if there also was a C-L present, or more likely just corrupt
    request bodies.
    
    If the client sends nothing but padding bytes and ends up consuming the
    entire stream window with no actual request bytes buffered, the request
    thread side of things would not send any stream window updates. Handle
    this corner case by sending a window update from the session thread.

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 570ef9209..de2ac5965 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -763,6 +763,7 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 	char buf[4];
 	uint64_t l, l2, head;
 	const uint8_t *src;
+	unsigned len;
 
 	/* XXX: Shouldn't error handling, setting of r2->error and
 	 * r2->cond signalling be handled more generally at the end of
@@ -790,13 +791,31 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 		return (h2->error ? h2->error : r2->error);
 	}
 
+	/* Check padding if present */
+	src = h2->rxf_data;
+	len = h2->rxf_len;
+	if (h2->rxf_flags & H2FF_DATA_PADDED) {
+		if (*src >= len) {
+			VSLb(h2->vsl, SLT_Debug,
+			    "H2: stream %u: Padding larger than frame length",
+			    h2->rxf_stream);
+			r2->error = H2CE_PROTOCOL_ERROR;
+			if (r2->cond)
+				AZ(pthread_cond_signal(r2->cond));
+			Lck_Unlock(&h2->sess->mtx);
+			return (H2CE_PROTOCOL_ERROR);
+		}
+		len -= 1 + *src;
+		src += 1;
+	}
+
 	/* Check against the Content-Length header if given */
 	if (r2->req->htc->content_length >= 0) {
 		if (r2->rxbuf)
 			l = r2->rxbuf->head;
 		else
 			l = 0;
-		l += h2->rxf_len;
+		l += len;
 		if (l > r2->req->htc->content_length ||
 		    ((h2->rxf_flags & H2FF_DATA_END_STREAM) &&
 		     l != r2->req->htc->content_length)) {
@@ -811,17 +830,8 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 		}
 	}
 
-	/* Handle zero size frame before starting to allocate buffers */
-	if (h2->rxf_len == 0) {
-		if (h2->rxf_flags & H2FF_DATA_END_STREAM)
-			r2->state = H2_S_CLOS_REM;
-		if (r2->cond)
-			AZ(pthread_cond_signal(r2->cond));
-		Lck_Unlock(&h2->sess->mtx);
-		return (0);
-	}
-
-	/* Check and charge connection window */
+	/* Check and charge connection window. The entire frame including
+	 * padding (h2->rxf_len) counts towards the window. */
 	if (h2->rxf_len > h2->req0->r_window) {
 		VSLb(h2->vsl, SLT_Debug,
 		    "H2: stream %u: Exceeded connection receive window",
@@ -843,7 +853,8 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 		Lck_Lock(&h2->sess->mtx);
 	}
 
-	/* Check stream window */
+	/* Check stream window. The entire frame including padding
+	 * (h2->rxf_len) counts towards the window. */
 	if (h2->rxf_len > r2->r_window) {
 		VSLb(h2->vsl, SLT_Debug,
 		    "H2: stream %u: Exceeded stream receive window",
@@ -855,6 +866,41 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 		return (H2SE_FLOW_CONTROL_ERROR);
 	}
 
+	/* Handle zero size frame before starting to allocate buffers */
+	if (len == 0) {
+		r2->r_window -= h2->rxf_len;
+
+		/* Handle the specific corner case where the entire window
+		 * has been exhausted using nothing but padding
+		 * bytes. Since no bytes have been buffered, no bytes
+		 * would be consumed by the request thread and no stream
+		 * window updates sent. Unpaint ourselves from this corner
+		 * by sending a stream window update here. */
+		CHECK_OBJ_ORNULL(r2->rxbuf, H2_RXBUF_MAGIC);
+		if (r2->r_window == 0 &&
+		    (r2->rxbuf == NULL || r2->rxbuf->tail == r2->rxbuf->head)) {
+			if (r2->rxbuf)
+				l = r2->rxbuf->size;
+			else
+				l = h2->local_settings.initial_window_size;
+			r2->r_window += l;
+			Lck_Unlock(&h2->sess->mtx);
+			vbe32enc(buf, l);
+			H2_Send_Get(wrk, h2, h2->req0);
+			H2_Send_Frame(wrk, h2, H2_F_WINDOW_UPDATE, 0, 4,
+			    r2->stream, buf);
+			H2_Send_Rel(h2, h2->req0);
+			Lck_Lock(&h2->sess->mtx);
+		}
+
+		if (h2->rxf_flags & H2FF_DATA_END_STREAM)
+			r2->state = H2_S_CLOS_REM;
+		if (r2->cond)
+			AZ(pthread_cond_signal(r2->cond));
+		Lck_Unlock(&h2->sess->mtx);
+		return (0);
+	}
+
 	/* Make the buffer on demand */
 	if (r2->rxbuf == NULL) {
 		unsigned bufsize;
@@ -864,15 +910,20 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 
 		Lck_Unlock(&h2->sess->mtx);
 
-		bufsize = r2->r_window;
-		/* This is the first data frame, so r_window will be the
-		 * initial window size. */
+		bufsize = h2->local_settings.initial_window_size;
+		if (bufsize < r2->r_window) {
+			/* This will not happen because we do not have any
+			 * mechanism to change the initial window size on
+			 * a running session. But if we gain that ability,
+			 * this future proofs it. */
+			bufsize = r2->r_window;
+		}
 		assert(bufsize > 0);
 		if ((h2->rxf_flags & H2FF_DATA_END_STREAM) &&
-		    bufsize > h2->rxf_len)
+		    bufsize > len)
 			/* Cap the buffer size when we know this is the
 			 * single data frame. */
-			bufsize = h2->rxf_len;
+			bufsize = len;
 		stvbuf = STV_AllocBuf(wrk, stv_transient,
 		    bufsize + sizeof *rxbuf);
 		if (stvbuf == NULL) {
@@ -905,13 +956,11 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 	l = r2->rxbuf->head - r2->rxbuf->tail;
 	assert(l <= r2->rxbuf->size);
 	l = r2->rxbuf->size - l;
-	assert(h2->rxf_len <= l); /* Stream window handling ensures
-				   * this */
+	assert(len <= l); /* Stream window handling ensures this */
 
 	Lck_Unlock(&h2->sess->mtx);
 
-	src = h2->rxf_data;
-	l = h2->rxf_len;
+	l = len;
 	head = r2->rxbuf->head;
 	do {
 		l2 = l;
@@ -925,9 +974,14 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 	} while (l > 0);
 
 	Lck_Lock(&h2->sess->mtx);
-	/* Charge stream window */
+
+	/* Charge stream window. The entire frame including padding
+	 * (h2->rxf_len) counts towards the window. The used padding
+	 * bytes will be included in the next connection window update
+	 * sent when the buffer bytes are consumed because that is
+	 * calculated against the available buffer space. */
 	r2->r_window -= h2->rxf_len;
-	r2->rxbuf->head += h2->rxf_len;
+	r2->rxbuf->head += len;
 	assert(r2->rxbuf->tail <= r2->rxbuf->head);
 	if (h2->rxf_flags & H2FF_DATA_END_STREAM)
 		r2->state = H2_S_CLOS_REM;
diff --git a/bin/varnishtest/tests/t02020.vtc b/bin/varnishtest/tests/t02020.vtc
new file mode 100644
index 000000000..77738e17f
--- /dev/null
+++ b/bin/varnishtest/tests/t02020.vtc
@@ -0,0 +1,66 @@
+varnishtest "H/2 received data frames with padding"
+
+barrier b1 sock 2
+
+server s1 {
+	rxreq
+	expect req.url == /1
+	expect req.body == abcde
+	txresp
+	rxreq
+	txresp
+	rxreq
+	txresp
+	expect req.body == a
+} -start
+
+varnish v1 -vcl+backend {
+	import vtc;
+	sub vcl_recv {
+		if (req.url == "/3") {
+			vtc.barrier_sync("${b1_sock}");
+		}
+	}
+} -start
+
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.reset h2_initial_window_size"
+varnish v1 -cliok "param.reset h2_rx_window_low_water"
+varnish v1 -cliok "param.set debug +syncvsl"
+
+client c1 {
+	stream 1 {
+		txreq -req POST -url /1 -hdr "content-length" "5" -nostrend
+		txdata -data abcde -padlen 1
+		rxresp
+		expect resp.status == 200
+	} -run
+
+	stream 3 {
+		txreq -req POST -url /3 -hdr "content-length" "131072" -nostrend
+		txdata -datalen 16300 -padlen 83 -nostrend
+		txdata -datalen 16300 -padlen 83 -nostrend
+		txdata -datalen 16300 -padlen 83 -nostrend
+		txdata -datalen 16300 -padlen 82 -nostrend
+		barrier b1 sync
+		rxwinup
+		txdata -datalen 16300 -padlen 83 -nostrend
+		rxwinup
+		txdata -datalen 16300 -padlen 83 -nostrend
+		rxwinup
+		txdata -datalen 16300 -padlen 83 -nostrend
+		rxwinup
+		txdata -datalen 16300 -padlen 83 -nostrend
+		rxwinup
+		txdata -datalen 672
+		rxresp
+		expect resp.status == 200
+	} -run
+
+	stream 5 {
+		txreq -req POST -url /5 -nostrend
+		txdata -data a -padlen 255
+		rxresp
+		expect resp.status == 200
+	} -run
+} -run

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:09 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:09 +0000 (UTC)
Subject: [master] e5a982aa0 Add a test case for exhausting window by padding
Message-ID: <20210830083109.BD492A1AC9@lists.varnish-cache.org>


commit e5a982aa01bd86ec667b3be815a677fcc5654697
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Jun 22 11:50:15 2021 +0200

    Add a test case for exhausting window by padding

diff --git a/bin/varnishtest/tests/t02021.vtc b/bin/varnishtest/tests/t02021.vtc
new file mode 100644
index 000000000..97bbb90ac
--- /dev/null
+++ b/bin/varnishtest/tests/t02021.vtc
@@ -0,0 +1,36 @@
+varnishtest "H/2 data frame padding exhaust window"
+
+server s1 {
+	rxreq
+	expect req.body == abcde
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+} -start
+
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.reset h2_initial_window_size"
+varnish v1 -cliok "param.reset h2_rx_window_low_water"
+
+client c1 {
+	stream 1 {
+		txreq -req POST -url /1 -hdr "content-length" "5" -nostrend
+
+		# Fill 65535 bytes of stream window using padding only
+		# Note that each frame consumes 256 bytes of window (padlen + 1)
+
+		loop 255 {
+			txdata -padlen 255 -nostrend
+		}
+		txdata -padlen 254 -nostrend
+
+		# Here the window have been exhausted, so we should receive
+		# a window update
+		rxwinup
+
+		txdata -data abcde
+		rxresp
+		expect resp.status == 200
+	} -run
+} -run

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:31:09 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:31:09 +0000 (UTC)
Subject: [master] 84e992675 New 'h2_rxbuf_storage' param to set rxbuf stevedore
Message-ID: <20210830083110.016FCA1AD8@lists.varnish-cache.org>


commit 84e992675337ff6587d56d8cece3ae429549400b
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Aug 11 17:16:13 2021 +0200

    New 'h2_rxbuf_storage' param to set rxbuf stevedore
    
    This parameter allows the user to choose which storage backend / stevedore
    that the H/2 receive buffers are allocated from. By default it uses
    Transient.

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index de2ac5965..fe65b32db 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -924,7 +924,8 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 			/* Cap the buffer size when we know this is the
 			 * single data frame. */
 			bufsize = len;
-		stvbuf = STV_AllocBuf(wrk, stv_transient,
+		CHECK_OBJ_NOTNULL(stv_h2_rxbuf, STEVEDORE_MAGIC);
+		stvbuf = STV_AllocBuf(wrk, stv_h2_rxbuf,
 		    bufsize + sizeof *rxbuf);
 		if (stvbuf == NULL) {
 			VSLb(h2->vsl, SLT_Debug,
diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index bf0bacca9..c8db1954e 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -218,6 +218,7 @@ char **MGT_NamedArg(const char *, const char **, const char *);
 
 
 /* stevedore_mgt.c */
+extern const char *mgt_stv_h2_rxbuf;
 void STV_Config(const char *spec);
 void STV_Config_Transient(void);
 
diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h
index 20d82641f..29f472299 100644
--- a/bin/varnishd/mgt/mgt_param.h
+++ b/bin/varnishd/mgt/mgt_param.h
@@ -78,6 +78,7 @@ tweak_t tweak_timeout;
 tweak_t tweak_uint;
 tweak_t tweak_vsl_buffer;
 tweak_t tweak_vsl_reclen;
+tweak_t tweak_h2_rxbuf_storage;
 
 extern struct parspec mgt_parspec[]; /* mgt_param_tbl.c */
 extern struct parspec VSL_parspec[]; /* mgt_param_vsl.c */
diff --git a/bin/varnishd/mgt/mgt_param_tweak.c b/bin/varnishd/mgt/mgt_param_tweak.c
index b3148b857..192cb5259 100644
--- a/bin/varnishd/mgt/mgt_param_tweak.c
+++ b/bin/varnishd/mgt/mgt_param_tweak.c
@@ -42,6 +42,7 @@
 #include "mgt/mgt.h"
 
 #include "mgt/mgt_param.h"
+#include "storage/storage.h"
 #include "vav.h"
 #include "vnum.h"
 
@@ -488,3 +489,43 @@ tweak_thread_pool_max(struct vsb *vsb, const struct parspec *par,
 	    "%u", mgt_param.wthread_max);
 	return (0);
 }
+
+/*--------------------------------------------------------------------
+ * Tweak 'h2_rxbuf_storage'
+ *
+ */
+
+int v_matchproto_(tweak_t)
+tweak_h2_rxbuf_storage(struct vsb *vsb, const struct parspec *par,
+    const char *arg)
+{
+	struct stevedore *stv;
+
+	/* XXX: If we want to remove the MUST_RESTART flag from the
+	 * h2_rxbuf_storage parameter, we could have a mechanism here
+	 * that when the child is running calls out through CLI to change
+	 * the stevedore being used. */
+
+	if (arg == NULL || arg == JSON_FMT)
+		return (tweak_string(vsb, par, arg));
+
+	if (!strcmp(arg, "Transient")) {
+		/* Always allow setting to the special name
+		 * "Transient". There will always be a stevedore with this
+		 * name, but it may not have been configured at the time
+		 * this is called. */
+	} else {
+		/* Only allow setting the value to a known configured
+		 * stevedore */
+		STV_Foreach(stv) {
+			CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
+			if (!strcmp(stv->ident, arg))
+				break;
+		}
+		if (stv == NULL) {
+			VSB_printf(vsb, "unknown storage backend '%s'", arg);
+			return (-1);
+		}
+	}
+	return (tweak_string(vsb, par, arg));
+}
diff --git a/bin/varnishd/storage/mgt_stevedore.c b/bin/varnishd/storage/mgt_stevedore.c
index 24cc27fdb..f96f841ce 100644
--- a/bin/varnishd/storage/mgt_stevedore.c
+++ b/bin/varnishd/storage/mgt_stevedore.c
@@ -54,6 +54,8 @@ static VTAILQ_HEAD(, stevedore) stevedores =
 
 struct stevedore *stv_transient;
 
+const char *mgt_stv_h2_rxbuf;
+
 /*--------------------------------------------------------------------*/
 
 int
@@ -247,7 +249,6 @@ STV_Config(const char *spec)
 void
 STV_Config_Transient(void)
 {
-
 	ASSERT_MGT();
 
 	VCLS_AddFunc(mgt_cls, MCF_AUTH, cli_stv);
diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index 498822b69..ea9c3d073 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -43,6 +43,9 @@
 #include "storage/storage.h"
 #include "vrt_obj.h"
 
+extern const char *mgt_stv_h2_rxbuf;
+struct stevedore *stv_h2_rxbuf = NULL;
+
 static pthread_mutex_t stv_mtx;
 
 /*--------------------------------------------------------------------
@@ -172,13 +175,23 @@ STV_open(void)
 
 	ASSERT_CLI();
 	AZ(pthread_mutex_init(&stv_mtx, NULL));
+
+	/* This string was prepared for us before the fork, and should
+	 * point to a configured stevedore. */
+	AN(mgt_stv_h2_rxbuf);
+
+	stv_h2_rxbuf = NULL;
 	STV_Foreach(stv) {
+		CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC);
 		bprintf(buf, "storage.%s", stv->ident);
 		stv->vclname = strdup(buf);
 		AN(stv->vclname);
 		if (stv->open != NULL)
 			stv->open(stv);
+		if (!strcmp(stv->ident, mgt_stv_h2_rxbuf))
+			stv_h2_rxbuf = stv;
 	}
+	AN(stv_h2_rxbuf);
 }
 
 void
diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h
index 3ad4ecc55..193b273e3 100644
--- a/bin/varnishd/storage/storage.h
+++ b/bin/varnishd/storage/storage.h
@@ -134,6 +134,7 @@ struct stevedore {
 };
 
 extern struct stevedore *stv_transient;
+extern struct stevedore *stv_h2_rxbuf;
 
 /*--------------------------------------------------------------------*/
 
diff --git a/bin/varnishtest/tests/t02022.vtc b/bin/varnishtest/tests/t02022.vtc
new file mode 100644
index 000000000..6a7ea5cc4
--- /dev/null
+++ b/bin/varnishtest/tests/t02022.vtc
@@ -0,0 +1,86 @@
+varnishtest "Test non-transient rxbuf stevedore with LRU nuking"
+
+barrier b1 sock 2 -cyclic
+
+server s1 {
+	rxreq
+	txresp -body asdf
+	rxreq
+	txresp -bodylen 1048000
+	rxreq
+	txresp -body ASDF
+} -start
+
+varnish v1 -arg "-srxbuf=malloc,1m -smain=malloc,1m" -vcl+backend {
+	import vtc;
+	sub vcl_recv {
+		if (req.url == "/1") {
+			vtc.barrier_sync("${b1_sock}");
+		}
+	}
+	sub vcl_backend_response {
+		if (bereq.url == "/2") {
+			set beresp.storage = storage.rxbuf;
+		} else {
+			set beresp.storage = storage.main;
+		}
+	}
+}
+
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.reset h2_initial_window_size"
+varnish v1 -cliok "param.reset h2_rx_window_low_water"
+varnish v1 -cliok "param.set h2_rxbuf_storage rxbuf"
+varnish v1 -cliok "param.set debug +syncvsl"
+
+varnish v1 -start
+
+client c1 {
+	stream 1 {
+		txreq -req POST -url /1 -hdr "content-length" "2048" -nostrend
+		txdata -datalen 2048
+		rxresp
+		expect resp.status == 200
+	} -start
+} -start
+
+varnish v1 -expect SMA.rxbuf.g_bytes >= 2048
+varnish v1 -expect SMA.Transient.g_bytes == 0
+varnish v1 -expect MAIN.n_lru_nuked == 0
+
+barrier b1 sync
+client c1 -wait
+
+varnish v1 -expect SMA.rxbuf.g_bytes == 0
+varnish v1 -expect SMA.Transient.g_bytes == 0
+varnish v1 -expect MAIN.n_lru_nuked == 0
+
+client c2 {
+	txreq -url /2
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 1048000
+} -run
+
+varnish v1 -expect SMA.rxbuf.g_bytes >= 1048000
+varnish v1 -expect MAIN.n_lru_nuked == 0
+
+client c3 {
+	stream 1 {
+		txreq -req POST -url /1 -hdr "content-length" "2048" -nostrend
+		txdata -datalen 2048
+		rxresp
+		expect resp.status == 200
+	} -start
+} -start
+
+varnish v1 -expect SMA.rxbuf.g_bytes >= 2048
+varnish v1 -expect SMA.rxbuf.g_bytes < 3000
+varnish v1 -expect SMA.Transient.g_bytes == 0
+varnish v1 -expect MAIN.n_lru_nuked == 1
+
+barrier b1 sync
+client c3 -wait
+
+varnish v1 -expect SMA.rxbuf.g_bytes == 0
+varnish v1 -expect SMA.Transient.g_bytes == 0
diff --git a/include/tbl/params.h b/include/tbl/params.h
index ccd5a8f54..ade2b944a 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -1657,10 +1657,28 @@ PARAM_PCRE2(
 	" messages."
 )
 
+/*--------------------------------------------------------------------
+ * Custom parameters with separate tweak function
+ */
+
+#  define PARAM_CUSTOM(nm, pv, def, ...) \
+	PARAM(, , nm, tweak_ ## nm, pv, NULL, NULL, def, NULL, __VA_ARGS__)
+
+PARAM_CUSTOM(
+	/* name */	h2_rxbuf_storage,
+	/* priv */	&mgt_stv_h2_rxbuf,
+	/* def */	"Transient",
+	/* descr */
+	"The name of the storage backend that HTTP/2 receive buffers"
+	" should be allocated from.",
+	/* flags */	MUST_RESTART
+)
+
 #  undef PARAM_ALL
 #  undef PARAM_PCRE2
 #  undef PARAM_STRING
 #  undef PARAM_VCC
+#  undef PARAM_CUSTOM
 #endif /* defined(PARAM_ALL) */
 
 #undef PARAM_MEMPOOL

From dridi.boukelmoune at gmail.com  Mon Aug 30 08:46:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 08:46:05 +0000 (UTC)
Subject: [master] e2dbf2148 [ncsa] -j will escape control characters
Message-ID: <20210830084605.B5A7AA43E9@lists.varnish-cache.org>


commit e2dbf214896d0975b6b67d87cf588124d99ad25f
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Mon Jun 21 14:53:15 2021 -0700

    [ncsa] -j will escape control characters

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index d394220be..9c9faf8f6 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -572,7 +572,7 @@ addf_vsl(enum VSL_tag_e tag, long i, const char *prefix)
 
 	ALLOC_OBJ(w, VSL_WATCH_MAGIC);
 	AN(w);
-	if (VSL_tagflags[tag])
+	if (VSL_tagflags[tag] && CTX.quote_how != VSB_QUOTE_JSON)
 		VUT_Error(vut, 1, "Tag %s can contain control characters",
 		    VSL_tags[tag]);
 	w->tag = tag;
@@ -921,7 +921,6 @@ process_vsl(const struct vsl_watch_head *head, enum VSL_tag_e tag,
 {
 	struct vsl_watch *w;
 	const char *p;
-
 	VTAILQ_FOREACH(w, head, list) {
 		CHECK_OBJ_NOTNULL(w, VSL_WATCH_MAGIC);
 		if (tag != w->tag)
@@ -966,13 +965,16 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 		skip = 0;
 		while (skip == 0 && 1 == VSL_Next(t->c)) {
 			tag = VSL_TAG(t->c->rec.ptr);
-			if (VSL_tagflags[tag])
+			if (VSL_tagflags[tag] &&
+			    CTX.quote_how != VSB_QUOTE_JSON)
 				continue;
 
 			b = VSL_CDATA(t->c->rec.ptr);
 			e = b + VSL_LEN(t->c->rec.ptr);
-			while (e > b && e[-1] == '\0')
-				e--;
+			if (!VSL_tagflags[tag]) {
+				while (e > b && e[-1] == '\0')
+					e--;
+			}
 
 			switch (tag) {
 			case SLT_HttpGarbage:
diff --git a/bin/varnishtest/tests/u00017.vtc b/bin/varnishtest/tests/u00017.vtc
new file mode 100644
index 000000000..a7fa82598
--- /dev/null
+++ b/bin/varnishtest/tests/u00017.vtc
@@ -0,0 +1,31 @@
+varnishtest "SLT_Debug can be printed by varnishncsa -j"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import cookie;
+
+	sub vcl_recv {
+		cookie.parse("");
+	}
+} -start
+
+varnish v1 -cliok "param.show vsl_mask +Debug"
+
+client c1 {
+	txreq
+	rxresp
+} -run
+
+# Let's fist create a script to reduce in all the variants below.
+
+shell {
+	varnishncsa -d -n ${v1_name} -F '%{VSL:Debug}x' -j | grep "^cookie: nothing to parse\\\\u0000"
+}
+
+shell -err -expect "Tag Debug can contain control characters" {
+	varnishncsa -d -n ${v1_name} -F '%{VSL:Debug}x'
+}
diff --git a/doc/changes.rst b/doc/changes.rst
index 59ed9eef3..0588c5359 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -57,6 +57,8 @@ Varnish Cache 7.x.x (2021-09-15)
 
 * ACLs default to `pedantic` which is now a per-ACL feature flag.
 
+* `varnishncsa -j` will now accept to print fields with control characters.
+
 ================================
 Varnish Cache 6.6.0 (2021-03-15)
 ================================

From dridi.boukelmoune at gmail.com  Mon Aug 30 09:27:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 09:27:05 +0000 (UTC)
Subject: [master] ac64b5120 vct: Missing ssize_t
Message-ID: <20210830092705.C3D01A59CD@lists.varnish-cache.org>


commit ac64b51206f0ebcae83ebac93d55c8d54fc92b33
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 30 10:57:47 2021 +0200

    vct: Missing ssize_t

diff --git a/lib/libvarnish/vct.c b/lib/libvarnish/vct.c
index 4bf6004a7..6993e22e6 100644
--- a/lib/libvarnish/vct.c
+++ b/lib/libvarnish/vct.c
@@ -32,6 +32,8 @@
 
 #include "config.h"
 
+#include <sys/types.h>
+
 #include <stdlib.h>
 #include <stdint.h>
 #include <string.h>

From dridi.boukelmoune at gmail.com  Mon Aug 30 09:27:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 09:27:05 +0000 (UTC)
Subject: [master] e380fe212 git: Ignore lib/libvarnish/vct_test
Message-ID: <20210830092705.D9905A59D0@lists.varnish-cache.org>


commit e380fe212a01db72d9fd412a018fb149d5efbb6e
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 30 11:25:41 2021 +0200

    git: Ignore lib/libvarnish/vct_test

diff --git a/.gitignore b/.gitignore
index b36966a18..7c5b4eb39 100644
--- a/.gitignore
+++ b/.gitignore
@@ -117,6 +117,7 @@ cscope.*out
 /include/vbm_test
 /lib/libvarnish/vav_test
 /lib/libvarnish/vbh_test
+/lib/libvarnish/vct_test
 /lib/libvarnish/vjsn_test
 /lib/libvarnish/vnum_c_test
 /lib/libvarnish/vsb_test

From phk at FreeBSD.org  Mon Aug 30 09:37:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 30 Aug 2021 09:37:05 +0000 (UTC)
Subject: [master] 6a2db563e This test is 64 bit only
Message-ID: <20210830093705.A3809A615D@lists.varnish-cache.org>


commit 6a2db563e95fd4fb32a4c19085a95ef2c4b79f66
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 30 09:36:26 2021 +0000

    This test is 64 bit only

diff --git a/bin/varnishtest/tests/o00006.vtc b/bin/varnishtest/tests/o00006.vtc
index 5a6b4e357..5f7eb4638 100644
--- a/bin/varnishtest/tests/o00006.vtc
+++ b/bin/varnishtest/tests/o00006.vtc
@@ -1,6 +1,6 @@
 varnishtest "SES_Reserve_proto_priv() overflow"
 
-feature ipv4
+feature 64bit ipv4
 
 server s1 {
 	rxreq

From phk at FreeBSD.org  Mon Aug 30 10:27:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 30 Aug 2021 10:27:05 +0000 (UTC)
Subject: [master] 8caf560c5 Consistently use ssize_t (spotted by FlexeLint)
Message-ID: <20210830102705.29F6CA7B43@lists.varnish-cache.org>


commit 8caf560c59625756890c41c2f0b34a36890c26d1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 30 10:24:28 2021 +0000

    Consistently use ssize_t (spotted by FlexeLint)

diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c
index bf7fc37b7..bcf00c75b 100644
--- a/bin/varnishd/storage/storage_simple.c
+++ b/bin/varnishd/storage/storage_simple.c
@@ -51,7 +51,7 @@ objallocwithnuke(struct worker *, const struct stevedore *, ssize_t size,
     int flags);
 
 static struct storage *
-sml_stv_alloc(const struct stevedore *stv, size_t size, int flags)
+sml_stv_alloc(const struct stevedore *stv, ssize_t size, int flags)
 {
 	struct storage *st;
 

From phk at FreeBSD.org  Mon Aug 30 10:27:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 30 Aug 2021 10:27:05 +0000 (UTC)
Subject: [master] eb56e6df6 Constification (spotted by flexelint)
Message-ID: <20210830102705.43B66A7B46@lists.varnish-cache.org>


commit eb56e6df6496f98ff60b6a057aa79f7bbb920155
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 30 10:25:07 2021 +0000

    Constification (spotted by flexelint)

diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c
index 296e0e20b..db008a230 100644
--- a/bin/varnishd/proxy/cache_proxy_proto.c
+++ b/bin/varnishd/proxy/cache_proxy_proto.c
@@ -323,7 +323,7 @@ VPX_tlv(const struct req *req, int typ, void **dst, int *len)
 }
 
 static int
-vpx_proto2(const struct worker *wrk, struct req *req)
+vpx_proto2(const struct worker *wrk, const struct req *req)
 {
 	uintptr_t *up;
 	uint16_t tlv_len;

From phk at FreeBSD.org  Mon Aug 30 10:27:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 30 Aug 2021 10:27:05 +0000 (UTC)
Subject: [master] 825e5b2d4 Always wrap code-generating macros in do { ... }
 while(0)
Message-ID: <20210830102705.61EFAA7B49@lists.varnish-cache.org>


commit 825e5b2d4d20a7fd5680e2647090b04c61a6b2a4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 30 10:25:26 2021 +0000

    Always wrap code-generating macros in do { ... } while(0)

diff --git a/bin/varnishd/http2/cache_http2_panic.c b/bin/varnishd/http2/cache_http2_panic.c
index 25dda29a0..b9720070a 100644
--- a/bin/varnishd/http2/cache_http2_panic.c
+++ b/bin/varnishd/http2/cache_http2_panic.c
@@ -51,10 +51,12 @@ h2_panic_settings(struct vsb *vsb, const struct h2_settings *s)
 	int cont = 0;
 
 #define H2_SETTING(U,l,...)			\
-	if (cont)				\
-		VSB_printf(vsb, ", ");		\
-	cont = 1;				\
-	VSB_printf(vsb, "0x%x", s->l);
+	do {					\
+		if (cont)			\
+			VSB_printf(vsb, ", ");	\
+		cont = 1;			\
+		VSB_printf(vsb, "0x%x", s->l);	\
+	} while (0);
 #include "tbl/h2_settings.h"
 #undef H2_SETTING
 }

From phk at FreeBSD.org  Mon Aug 30 10:27:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 30 Aug 2021 10:27:05 +0000 (UTC)
Subject: [master] ad152231f Signed/unsigned fix, spotted by FlexeLint
Message-ID: <20210830102705.7FA10A7B55@lists.varnish-cache.org>


commit ad152231fd0477b43f3061e573c1e8ead1af5940
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 30 10:25:53 2021 +0000

    Signed/unsigned fix, spotted by FlexeLint

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index fe65b32db..d77fd0596 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -761,7 +761,8 @@ static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 	char buf[4];
-	uint64_t l, l2, head;
+	ssize_t l;
+	uint64_t l2, head;
 	const uint8_t *src;
 	unsigned len;
 
@@ -999,7 +1000,8 @@ h2_vfp_body(struct vfp_ctx *vc, struct vfp_entry *vfe, void *ptr, ssize_t *lp)
 	struct h2_req *r2;
 	struct h2_sess *h2;
 	enum vfp_status retval;
-	uint64_t l, l2, tail;
+	ssize_t l, l2;
+	uint64_t tail;
 	uint8_t *dst;
 	char buf[4];
 	int i;

From phk at FreeBSD.org  Mon Aug 30 10:49:04 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 30 Aug 2021 10:49:04 +0000 (UTC)
Subject: [master] a0ad961ae Add missing copyright
Message-ID: <20210830104905.1EE9AA8B13@lists.varnish-cache.org>


commit a0ad961ae349e3269856787d3b83f72a77ab13fb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 30 10:37:50 2021 +0000

    Add missing copyright

diff --git a/lib/libvarnish/vsb_test.c b/lib/libvarnish/vsb_test.c
index ef09a14c4..7ca3beec7 100644
--- a/lib/libvarnish/vsb_test.c
+++ b/lib/libvarnish/vsb_test.c
@@ -1,3 +1,30 @@
+/*-
+ * Copyright (c) 2020-2021 Varnish Software AS
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
 
 #ifdef VSB_TEST
 

From phk at FreeBSD.org  Mon Aug 30 10:49:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 30 Aug 2021 10:49:05 +0000 (UTC)
Subject: [master] bb6d88588 Style OCD:  () around return values
Message-ID: <20210830104905.3D17BA8B18@lists.varnish-cache.org>


commit bb6d88588aea0ecd179e9cf21077901e13ff44e1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 30 10:48:39 2021 +0000

    Style OCD:  () around return values

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index e0eafe374..872f9388f 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -203,7 +203,7 @@ VPI_acl_table(VRT_CTX, VCL_IP p, unsigned n, unsigned m, const uint8_t *tbl,
 		sz /= m;
 		if (str != NULL)
 			VPI_acl_log(ctx, str[sz]);
-		return *ptr;
+		return (*ptr);
 	}
 	if (str != NULL)
 		VPI_acl_log(ctx, fin);
diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index d77fd0596..41f18d828 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -75,7 +75,7 @@ h2_framename(enum h2frame h2f)
 {
 
 	switch (h2f) {
-#define H2_FRAME(l,u,t,f,...)	case H2F_##u: return #u;
+#define H2_FRAME(l,u,t,f,...)	case H2F_##u: return (#u);
 #include "tbl/h2_frames.h"
 	default:
 		return (NULL);
diff --git a/include/vbm.h b/include/vbm.h
index 4f0486cf1..1136d19d9 100644
--- a/include/vbm.h
+++ b/include/vbm.h
@@ -102,7 +102,7 @@ vbit_init(void *p, size_t sz)
 	struct vbitmap *vb;
 
 	if (sz < sizeof(*vb))
-		return NULL;
+		return (NULL);
 
 	memset(p, 0, sz);
 	vb = p;
diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c
index 96360b8d3..1982db96a 100644
--- a/lib/libvcc/vcc_acl.c
+++ b/lib/libvcc/vcc_acl.c
@@ -527,7 +527,7 @@ vcc_acl_emit_tables(const struct vcc *tl, unsigned n, const char *name)
 		}
 		Fh(tl, 0, "};\n");
 	}
-	return  (rv);
+	return (rv);
 }
 
 /*********************************************************************

From dridi.boukelmoune at gmail.com  Mon Aug 30 10:59:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 10:59:05 +0000 (UTC)
Subject: [master] b63725c3d vtc_http2: Don't leak ignored window updates
Message-ID: <20210830105905.80CD7A926F@lists.varnish-cache.org>


commit b63725c3d5e39eee67b40f1fb169e73d13a24c28
Author: Asad Sajjad Ahmed <asadsa at varnish-software.com>
Date:   Mon Aug 30 12:49:15 2021 +0200

    vtc_http2: Don't leak ignored window updates
    
    Refs #3442
    Refs #3661

diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c
index 297045ff9..2cdb34a68 100644
--- a/bin/varnishtest/vtc_http2.c
+++ b/bin/varnishtest/vtc_http2.c
@@ -2291,7 +2291,7 @@ static void
 cmd_rxmsg(CMD_ARGS)
 {
 	struct stream *s;
-	struct frame *f;
+	struct frame *f = NULL;
 	int end_stream;
 	int rcv = 0;
 
@@ -2303,8 +2303,9 @@ cmd_rxmsg(CMD_ARGS)
 		ONLY_H2_CLIENT(s->hp, av);
 
 	do {
-		f = rxstuff(s);
-		if (!f)
+		replace_frame(&f, rxstuff(s));
+		CHECK_OBJ_ORNULL(f, FRAME_MAGIC);
+		if (f == NULL)
 			return;
 	} while (f->type == TYPE_WINDOW_UPDATE);
 
@@ -2315,6 +2316,7 @@ cmd_rxmsg(CMD_ARGS)
 
 	while (!(f->flags & END_HEADERS)) {
 		replace_frame(&f, rxstuff(s));
+		CHECK_OBJ_ORNULL(f, FRAME_MAGIC);
 		if (f == NULL)
 			return;
 		rcv++;
@@ -2323,6 +2325,7 @@ cmd_rxmsg(CMD_ARGS)
 
 	while (!end_stream) {
 		replace_frame(&f, rxstuff(s));
+		CHECK_OBJ_ORNULL(f, FRAME_MAGIC);
 		if (f == NULL)
 			break;
 		rcv++;

From dridi.boukelmoune at gmail.com  Mon Aug 30 12:12:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 12:12:06 +0000 (UTC)
Subject: [master] 8bfbc2247 tools: New script to check magic numbers
Message-ID: <20210830121206.DC429AB44B@lists.varnish-cache.org>


commit 8bfbc2247b6ff4e4057339ff8d8bbccfc659f1cd
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Sat Aug 28 21:20:57 2021 +0200

    tools: New script to check magic numbers
    
    Only from a git clone, and only from the root directory, which should
    leave alone distcheck and builds from release archives.

diff --git a/tools/magic_check.sh b/tools/magic_check.sh
new file mode 100755
index 000000000..b5ec57ee0
--- /dev/null
+++ b/tools/magic_check.sh
@@ -0,0 +1,61 @@
+#!/bin/sh
+#
+# Copyright (c) 2021 Varnish Software AS
+# All rights reserved.
+#
+# Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# Check that all magic numbers are declared as:
+#
+# - 0x00112233	(most magic numbers)
+# - 0x00	(where size matters)
+
+set -e
+set -u
+
+ROOT=$(git rev-parse --show-cdup 2>/dev/null) || exit 77
+test -z "$ROOT" || exit 77
+
+git grep -h '^#define \w*_MAGIC' |
+sed 's/\\t/\t/g' |
+awk '{print $3}' |
+tr '[:upper:]' '[:lower:]' |
+sort |
+uniq -c |
+sort |
+awk '$1 != 1 || $2 !~ /^0x[[:xdigit:]]*$/ || length($2) !~ /^(4|10)$/' |
+while read -r COUNT MAGIC
+do
+	if [ $COUNT -eq 1 ]
+	then
+		echo "Invalid magic number:"
+	else
+		echo "Duplicate magic number:"
+	fi
+	git grep -ih '^#define \w*_MAGIC\s*'$MAGIC
+	echo
+	false # propagate non-zero exit status
+done

From dridi.boukelmoune at gmail.com  Mon Aug 30 12:12:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 12:12:06 +0000 (UTC)
Subject: [master] 471156022 vlu: Use 8 hexadecimal digits for LINEUP_MAGIC
Message-ID: <20210830121206.F3540AB44E@lists.varnish-cache.org>


commit 47115602249e16a94ea0578d7fc78de59651afb8
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 30 11:40:04 2021 +0200

    vlu: Use 8 hexadecimal digits for LINEUP_MAGIC

diff --git a/lib/libvarnish/vlu.c b/lib/libvarnish/vlu.c
index 217c50b12..8883a350c 100644
--- a/lib/libvarnish/vlu.c
+++ b/lib/libvarnish/vlu.c
@@ -45,7 +45,7 @@
 
 struct vlu {
 	unsigned	magic;
-#define LINEUP_MAGIC	0x8286661
+#define LINEUP_MAGIC	0x08286661
 	char		*buf;
 	unsigned	bufl;
 	unsigned	bufp;

From dridi.boukelmoune at gmail.com  Mon Aug 30 12:12:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 12:12:07 +0000 (UTC)
Subject: [master] 1a20dcb52 vbh: Remove unsigned suffix from VBH_MAGIC
Message-ID: <20210830121207.1FE4BAB451@lists.varnish-cache.org>


commit 1a20dcb527c90149ca0f878ab0046f0611f49f28
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 30 11:41:47 2021 +0200

    vbh: Remove unsigned suffix from VBH_MAGIC

diff --git a/lib/libvarnish/vbh.c b/lib/libvarnish/vbh.c
index 02afefa44..43fb08de1 100644
--- a/lib/libvarnish/vbh.c
+++ b/lib/libvarnish/vbh.c
@@ -81,7 +81,7 @@
 
 struct vbh {
 	unsigned		magic;
-#define VBH_MAGIC		0xf581581aU	/* from /dev/random */
+#define VBH_MAGIC		0xf581581a	/* from /dev/random */
 	void			*priv;
 	vbh_cmp_t		*cmp;
 	vbh_update_t		*update;

From dridi.boukelmoune at gmail.com  Mon Aug 30 12:12:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 12:12:07 +0000 (UTC)
Subject: [master] 7ea720002 vtc_tunnel: TUNNEL_MAGIC is a duplicate
Message-ID: <20210830121207.3DF2DAB456@lists.varnish-cache.org>


commit 7ea72000254fcc76ca3e17f2a7ba3b2bf37957fd
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 30 11:42:51 2021 +0200

    vtc_tunnel: TUNNEL_MAGIC is a duplicate
    
    My absolute bad on this copy-pasta mistake.

diff --git a/bin/varnishtest/vtc_tunnel.c b/bin/varnishtest/vtc_tunnel.c
index 28fc05ec0..342617c2b 100644
--- a/bin/varnishtest/vtc_tunnel.c
+++ b/bin/varnishtest/vtc_tunnel.c
@@ -121,7 +121,7 @@ struct tunnel_lane {
 
 struct tunnel {
 	unsigned		magic;
-#define TUNNEL_MAGIC		0x55286619
+#define TUNNEL_MAGIC		0x7f59913d
 	char			*name;
 	struct vtclog		*vl;
 	VTAILQ_ENTRY(tunnel)	list;

From dridi.boukelmoune at gmail.com  Mon Aug 30 12:12:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 12:12:07 +0000 (UTC)
Subject: [master] 33fa4953a build: Check magic numbers
Message-ID: <20210830121207.5C591AB45A@lists.varnish-cache.org>


commit 33fa4953a163582ae80d2b1c30de6bbbcae372fe
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 30 11:43:43 2021 +0200

    build: Check magic numbers

diff --git a/Makefile.am b/Makefile.am
index d6fbc0139..3e84a6cf6 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -2,6 +2,8 @@ ACLOCAL_AMFLAGS = -I m4 -I .
 
 SUBDIRS = include lib bin vmod etc doc man
 
+TESTS = tools/magic_check.sh
+
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = varnishapi.pc
 
@@ -16,6 +18,7 @@ CLEANFILES = \
 	witness.svg
 
 EXTRA_DIST = \
+	$(TESTS) \
 	README.rst \
 	README.Packaging \
 	LICENSE \

From dridi.boukelmoune at gmail.com  Mon Aug 30 12:25:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 12:25:07 +0000 (UTC)
Subject: [master] 76f72a395 cache: Move the txt declaration to vdef.h
Message-ID: <20210830122508.04E58AC07B@lists.varnish-cache.org>


commit 76f72a395521c71f38dbfca4b9b60da4d82e5da3
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 3 10:06:48 2021 +0200

    cache: Move the txt declaration to vdef.h
    
    There are other components than the cache process that could benefit
    from it, in particular libvcc. The relationship with vas.h is somewhat
    unfortunate but that also centralize the Tcheck() logic in pdiff() and
    doesn't actually require vas.h unless the macros are used, in which
    case it's almost guaranteed that the calling code already included
    vas.h in the first place.
    
    The benefits should outweigh the drawbacks.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index b9e4022cb..35000cf6d 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -112,13 +112,6 @@ struct worker_priv;
 
 /*--------------------------------------------------------------------*/
 
-typedef struct {
-	const char		*b;
-	const char		*e;
-} txt;
-
-/*--------------------------------------------------------------------*/
-
 struct lock { void *priv; };	// Opaque
 
 /*--------------------------------------------------------------------
@@ -833,24 +826,6 @@ int RFC2616_Do_Cond(const struct req *sp);
 void RFC2616_Weaken_Etag(struct http *hp);
 void RFC2616_Vary_AE(struct http *hp);
 
-#define Tcheck(t) do {						\
-		AN((t).b);					\
-		AN((t).e);					\
-		assert((t).b <= (t).e);				\
-	} while(0)
-
-/*
- * unsigned length of a txt
- */
-
-static inline unsigned
-Tlen(const txt t)
-{
-
-	Tcheck(t);
-	return ((unsigned)(t.e - t.b));
-}
-
 /*
  * We want to cache the most recent timestamp in wrk->lastused to avoid
  * extra timestamps in cache_pool.c.  Hide this detail with a macro
diff --git a/include/vas.h b/include/vas.h
index 302170ae2..534737483 100644
--- a/include/vas.h
+++ b/include/vas.h
@@ -127,6 +127,8 @@ static inline size_t
 pdiff(const void *b, const void *e)
 {
 
+	AN(b);
+	AN(e);
 	assert(b <= e);
 	return ((size_t)((const char *)e - (const char *)b));
 }
diff --git a/include/vdef.h b/include/vdef.h
index ae072a2c3..c4004611f 100644
--- a/include/vdef.h
+++ b/include/vdef.h
@@ -197,3 +197,15 @@ int __llvm_gcov_flush(void);
 typedef double vtim_mono;
 typedef double vtim_real;
 typedef double vtim_dur;
+
+/**********************************************************************
+ * txt (vas.h needed for the macros)
+ */
+
+typedef struct {
+	const char		*b;
+	const char		*e;
+} txt;
+
+#define Tcheck(t)	do { (void)pdiff((t).b, (t).e); } while (0)
+#define Tlen(t)		(pdiff((t).b, (t).e))

From dridi.boukelmoune at gmail.com  Mon Aug 30 12:25:08 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 12:25:08 +0000 (UTC)
Subject: [master] 0eb6916a8 vre: Optionally capture groups in a txt array
Message-ID: <20210830122508.207CFAC07F@lists.varnish-cache.org>


commit 0eb6916a8df50d5910c16a2c5dfe9cf664362b23
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 3 10:57:40 2021 +0200

    vre: Optionally capture groups in a txt array

diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index e2efdcbc8..065563641 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -182,11 +182,14 @@ VRE_export(const vre_t *code, size_t *sz)
 }
 
 static int
-vre_match(const vre_t *code, const char *subject, size_t length, size_t offset,
-    int options, pcre2_match_data **datap)
+vre_capture(const vre_t *code, const char *subject, size_t length,
+    size_t offset, int options, txt *groups, size_t *count,
+    pcre2_match_data **datap)
 {
 	pcre2_match_data *data;
 	pcre2_code *re;
+	PCRE2_SIZE *ovector;
+	size_t nov, g;
 	int matches;
 
 	re = VRE_unpack(code);
@@ -199,9 +202,24 @@ vre_match(const vre_t *code, const char *subject, size_t length, size_t offset,
 		AN(data);
 	}
 
-	matches =  pcre2_match(re, (PCRE2_SPTR)subject, length, offset,
+	matches = pcre2_match(re, (PCRE2_SPTR)subject, length, offset,
 	    options, data, code->re_ctx);
 
+	if (groups != NULL) {
+		AN(count);
+		AN(*count);
+		ovector = pcre2_get_ovector_pointer(data);
+		nov = pcre2_get_ovector_count(data);
+		if (nov > *count)
+			nov = *count;
+		for (g = 0; g < nov; g++) {
+			groups->b = subject + ovector[2 * g];
+			groups->e = subject + ovector[2 * g + 1];
+			groups++;
+		}
+		*count = nov;
+	}
+
 	if (datap != NULL && matches > VRE_ERROR_NOMATCH)
 		*datap = data;
 	else
@@ -221,7 +239,8 @@ VRE_match(const vre_t *code, const char *subject, size_t length,
 	if (length == 0)
 		length = PCRE2_ZERO_TERMINATED;
 	vre_limit(code, lim);
-	return (vre_match(code, subject, length, 0, options, NULL));
+	return (vre_capture(code, subject, length, 0, options,
+	    NULL, NULL, NULL));
 }
 
 int
@@ -242,7 +261,8 @@ VRE_sub(const vre_t *code, const char *subject, const char *replacement,
 	AN(replacement);
 
 	vre_limit(code, lim);
-	i = vre_match(code, subject, PCRE2_ZERO_TERMINATED, offset, 0, &data);
+	i = vre_capture(code, subject, PCRE2_ZERO_TERMINATED, offset, 0,
+	    NULL, NULL, &data);
 
 	if (i <= VRE_ERROR_NOMATCH)
 		return (i);
@@ -274,8 +294,8 @@ VRE_sub(const vre_t *code, const char *subject, const char *replacement,
 		offset = ovector[1];
 		if (!all)
 			break;
-		i = vre_match(code, subject, PCRE2_ZERO_TERMINATED, offset,
-		    PCRE2_NOTEMPTY, &data);
+		i = vre_capture(code, subject, PCRE2_ZERO_TERMINATED, offset,
+		    PCRE2_NOTEMPTY, NULL, NULL, &data);
 		if (i < VRE_ERROR_NOMATCH)
 			return (i);
 	} while (i != VRE_ERROR_NOMATCH);

From dridi.boukelmoune at gmail.com  Mon Aug 30 12:25:08 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 12:25:08 +0000 (UTC)
Subject: [master] 357242a19 vre: Capture groups in a txt array in VRE_sub()
Message-ID: <20210830122508.491A9AC083@lists.varnish-cache.org>


commit 357242a19ca68bb6c7a264a819e3de193d63ed97
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 3 12:08:48 2021 +0200

    vre: Capture groups in a txt array in VRE_sub()
    
    And add a bunch of assertions in addition to the implicit ones from
    pdiff().

diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index 065563641..83077656c 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -248,12 +248,11 @@ VRE_sub(const vre_t *code, const char *subject, const char *replacement,
     struct vsb *vsb, const volatile struct vre_limits *lim, int all)
 {
 	pcre2_match_data *data = NULL;
-	PCRE2_SIZE *ovector;
-	uint32_t nov;
-	int i, l;
+	txt groups[10];
+	size_t count;
+	int i, offset = 0;
 	const char *s;
 	unsigned x;
-	int offset = 0;
 
 	CHECK_OBJ_NOTNULL(code, VRE_MAGIC);
 	CHECK_OBJ_NOTNULL(vsb, VSB_MAGIC);
@@ -261,20 +260,22 @@ VRE_sub(const vre_t *code, const char *subject, const char *replacement,
 	AN(replacement);
 
 	vre_limit(code, lim);
+	count = 10;
 	i = vre_capture(code, subject, PCRE2_ZERO_TERMINATED, offset, 0,
-	    NULL, NULL, &data);
+	    groups, &count, &data);
 
-	if (i <= VRE_ERROR_NOMATCH)
+	if (i <= VRE_ERROR_NOMATCH) {
+		AZ(data);
 		return (i);
+	}
 
 	do {
-		AN(data);
-		ovector = pcre2_get_ovector_pointer(data);
-		nov = pcre2_get_ovector_count(data);
-		AN(ovector);
+		AN(data); /* check reuse across successful captures */
+		AN(count);
 
 		/* Copy prefix to match */
-		VSB_bcat(vsb, subject + offset, ovector[0] - offset);
+		s = subject + offset;
+		VSB_bcat(vsb, s, pdiff(s, groups[0].b));
 		for (s = replacement; *s != '\0'; s++ ) {
 			if (*s != '\\' || s[1] == '\0') {
 				VSB_putc(vsb, *s);
@@ -283,21 +284,24 @@ VRE_sub(const vre_t *code, const char *subject, const char *replacement,
 			s++;
 			if (isdigit(*s)) {
 				x = *s - '0';
-				if (x >= nov)
+				if (x >= count)
 					continue;
-				l = ovector[2*x+1] - ovector[2*x];
-				VSB_bcat(vsb, subject + ovector[2*x], l);
+				VSB_bcat(vsb, groups[x].b, Tlen(groups[x]));
 				continue;
 			}
 			VSB_putc(vsb, *s);
 		}
-		offset = ovector[1];
+		offset = pdiff(subject, groups[0].e);
 		if (!all)
 			break;
+		count = 10;
 		i = vre_capture(code, subject, PCRE2_ZERO_TERMINATED, offset,
-		    PCRE2_NOTEMPTY, NULL, NULL, &data);
-		if (i < VRE_ERROR_NOMATCH)
+		    PCRE2_NOTEMPTY, groups, &count, &data);
+
+		if (i < VRE_ERROR_NOMATCH) {
+			AZ(data);
 			return (i);
+		}
 	} while (i != VRE_ERROR_NOMATCH);
 
 	if (data != NULL) {
@@ -307,7 +311,7 @@ VRE_sub(const vre_t *code, const char *subject, const char *replacement,
 	}
 
 	/* Copy suffix to match */
-	VSB_cat(vsb, subject + offset);
+	VSB_cat(vsb, groups[0].e);
 	return (1);
 }
 

From dridi.boukelmoune at gmail.com  Mon Aug 30 12:25:08 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 12:25:08 +0000 (UTC)
Subject: [master] 32f1b62ce vre: New VRE_capture() function
Message-ID: <20210830122508.73489AC086@lists.varnish-cache.org>


commit 32f1b62ce0d3d572f0ce8bfa7c7b3f7c11d6b7a2
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 3 12:13:34 2021 +0200

    vre: New VRE_capture() function
    
    It's a thin wrapper on top of vre_capture() that returns the number of
    groups captured during a match.
    
    Now that txt appears in vre.h we need vdef.h anywhere the former is
    included. There was one generated file where that wasn't the case.
    
    Closes #3655

diff --git a/include/vre.h b/include/vre.h
index c206078da..2a9450ad5 100644
--- a/include/vre.h
+++ b/include/vre.h
@@ -60,6 +60,8 @@ vre_t *VRE_export(const vre_t *, size_t *);
 int VRE_error(struct vsb *, int err);
 int VRE_match(const vre_t *code, const char *subject, size_t length,
     int options, const volatile struct vre_limits *lim);
+int VRE_capture(vre_t *code, const char *subject, size_t length, int options,
+    txt *groups, size_t count, const volatile struct vre_limits *lim);
 int VRE_sub(const vre_t *code, const char *subject, const char *replacement,
     struct vsb *vsb, const volatile struct vre_limits *lim, int all);
 void VRE_free(vre_t **);
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index 83077656c..959201fa8 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -243,6 +243,29 @@ VRE_match(const vre_t *code, const char *subject, size_t length,
 	    NULL, NULL, NULL));
 }
 
+int
+VRE_capture(vre_t *code, const char *subject, size_t length, int options,
+    txt *groups, size_t count, const volatile struct vre_limits *lim)
+{
+	int i;
+
+	CHECK_OBJ_NOTNULL(code, VRE_MAGIC);
+	AN(subject);
+	AZ(options & (~VRE_MASK_MATCH));
+	AN(groups);
+	AN(count);
+
+	if (length == 0)
+		length = PCRE2_ZERO_TERMINATED;
+	vre_limit(code, lim);
+	i = vre_capture(code, subject, length, 0, options,
+	    groups, &count, NULL);
+
+	if (i <= 0)
+		return (i);
+	return (count);
+}
+
 int
 VRE_sub(const vre_t *code, const char *subject, const char *replacement,
     struct vsb *vsb, const volatile struct vre_limits *lim, int all)
diff --git a/lib/libvarnishapi/generate.py b/lib/libvarnishapi/generate.py
index 3924e4a13..10bef4257 100755
--- a/lib/libvarnishapi/generate.py
+++ b/lib/libvarnishapi/generate.py
@@ -192,6 +192,7 @@ fo.write("""
 #include <ctype.h>
 #include <stdio.h>
 
+#include "vdef.h"
 #include "vqueue.h"
 #include "vre.h"
 

From dridi.boukelmoune at gmail.com  Mon Aug 30 12:28:04 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 12:28:04 +0000 (UTC)
Subject: [master] d3a6a6076 vsl: Log the progress of streaming hits
Message-ID: <20210830122804.DB20DAC831@lists.varnish-cache.org>


commit d3a6a60769a751676df0bf478741573f4efc6281
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Apr 1 08:10:17 2021 +0200

    vsl: Log the progress of streaming hits
    
    The larger the response we hit, the more likely we are to hit an ongoing
    fetch and since we buffer transaction logs it can become really tedious
    to infer whether the backend transaction may have already been overrun
    or not flushed yet. At least, such a heuristic was impossible with a VSL
    query until now.

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 1af3a3793..5d24e0479 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -369,6 +369,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp)
 	vtim_real exp_t_origin;
 	int busy_found;
 	const uint8_t *vary;
+	intmax_t boc_progress;
 	unsigned xid = 0;
 	float dttl = 0.0;
 
@@ -519,7 +520,9 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp)
 			return (HSH_HITMISS);
 		}
 		oc->hits++;
+		boc_progress = oc->boc == NULL ? -1 : oc->boc->len_so_far;
 		AN(hsh_deref_objhead_unlock(wrk, &oh, HSH_RUSH_POLICY));
+		Req_LogHit(wrk, req, oc, boc_progress);
 		return (HSH_HIT);
 	}
 
@@ -539,6 +542,11 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp)
 		return (HSH_HITMISS);
 	}
 
+	if (exp_oc != NULL && exp_oc->boc != NULL)
+		boc_progress = exp_oc->boc->len_so_far;
+	else
+		boc_progress = -1;
+
 	if (!busy_found) {
 		*bocp = hsh_insert_busyobj(wrk, oh);
 
@@ -548,6 +556,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp)
 			if (EXP_Ttl_grace(req, exp_oc) >= req->t_req) {
 				exp_oc->hits++;
 				Lck_Unlock(&oh->mtx);
+				Req_LogHit(wrk, req, exp_oc, boc_progress);
 				return (HSH_GRACE);
 			}
 		}
@@ -562,6 +571,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp)
 		*ocp = exp_oc;
 		exp_oc->hits++;
 		AN(hsh_deref_objhead_unlock(wrk, &oh, 0));
+		Req_LogHit(wrk, req, exp_oc, boc_progress);
 		return (HSH_GRACE);
 	}
 
diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c
index 09addfde4..5b3fe1964 100644
--- a/bin/varnishd/cache/cache_req.c
+++ b/bin/varnishd/cache/cache_req.c
@@ -72,6 +72,33 @@ Req_AcctLogCharge(struct VSC_main_wrk *ds, struct req *req)
 	memset(a, 0, sizeof *a);
 }
 
+void
+Req_LogHit(struct worker *wrk, struct req *req, struct objcore *oc,
+    intmax_t fetch_progress)
+{
+	const char *clen, *sep;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+
+	if (fetch_progress >= 0) {
+		clen = HTTP_GetHdrPack(wrk, oc, H_Content_Length);
+		if (clen == NULL)
+			clen = sep = "";
+		else
+			sep = " ";
+		VSLb(req->vsl, SLT_Hit, "%u %.6f %.6f %.6f %jd%s%s",
+		    ObjGetXID(wrk, oc), EXP_Dttl(req, oc),
+		    oc->grace, oc->keep,
+		    fetch_progress, sep, clen);
+	} else {
+		VSLb(req->vsl, SLT_Hit, "%u %.6f %.6f %.6f",
+		    ObjGetXID(wrk, oc), EXP_Dttl(req, oc),
+		    oc->grace, oc->keep);
+	}
+}
+
 /*--------------------------------------------------------------------
  * Alloc/Free a request
  */
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 9ad2aa28d..6c6484236 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -602,12 +602,6 @@ cnt_lookup(struct worker *wrk, struct req *req)
 	req->objcore = oc;
 	AZ(oc->flags & OC_F_HFM);
 
-	VSLb(req->vsl, SLT_Hit, "%u %.6f %.6f %.6f",
-	    ObjGetXID(wrk, req->objcore),
-	    EXP_Dttl(req, req->objcore),
-	    req->objcore->grace,
-	    req->objcore->keep);
-
 	VCL_hit_method(req->vcl, wrk, req, NULL, NULL);
 
 	switch (wrk->handling) {
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index c3defbdce..653976cdc 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -403,6 +403,7 @@ void Req_Rollback(VRT_CTX);
 void Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req);
 void Req_Fail(struct req *req, enum sess_close reason);
 void Req_AcctLogCharge(struct VSC_main_wrk *, struct req *);
+void Req_LogHit(struct worker *, struct req *, struct objcore *, intmax_t);
 
 /* cache_req_body.c */
 int VRB_Ignore(struct req *);
diff --git a/bin/varnishtest/tests/b00077.vtc b/bin/varnishtest/tests/b00077.vtc
new file mode 100644
index 000000000..d44cc1073
--- /dev/null
+++ b/bin/varnishtest/tests/b00077.vtc
@@ -0,0 +1,73 @@
+varnishtest "SLT_Hit ongoing fetch"
+
+barrier b1 cond 2 -cyclic
+barrier b2 cond 2 -cyclic
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr "Content-Length: 10"
+	send hello
+	barrier b1 sync
+	barrier b2 sync
+	send world
+
+	rxreq
+	txresp -nolen -hdr "Transfer-Encoding: chunked"
+	chunked hello
+	barrier b1 sync
+	barrier b2 sync
+	chunked world
+	chunked ""
+} -start
+
+varnish v1 -cliok "param.set debug +syncvsl"
+varnish v1 -cliok "param.set thread_pools 1"
+varnish v1 -vcl+backend "" -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.body == helloworld
+} -start
+
+barrier b1 sync
+
+logexpect l1 -v v1 -g raw {
+	#                   vxid TTL     grace     keep     fetch length
+	expect * 1004 Hit "^1002 [0-9.]+ 10.000000 0.000000 [0-5] 10$"
+} -start
+
+client c2 {
+	txreq
+	rxresp
+	expect resp.body == helloworld
+} -start
+
+logexpect l1 -wait
+
+barrier b2 sync
+
+client c1 -wait
+client c2 -wait
+
+# Recycle almost everything for the chunked variant
+
+varnish v1 -cliok "ban obj.status != 0"
+
+client c1 -start
+
+barrier b1 sync
+
+logexpect l2 -v v1 -g raw {
+	#                   vxid TTL     grace     keep     fetch
+	expect * 1009 Hit "^1007 [0-9.]+ 10.000000 0.000000 [0-5]$"
+} -start
+
+client c2 -start
+
+logexpect l2 -wait
+
+barrier b2 sync
+
+client c1 -wait
+client c2 -wait
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 2d3d3cf86..0ff7e0301 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -338,12 +338,14 @@ SLTM(ReqStart, 0, "Client request start",
 SLTM(Hit, 0, "Hit object in cache",
 	"Object looked up in cache.\n\n"
 	"The format is::\n\n"
-	"\t%u %f %f %f\n"
-	"\t|  |  |  |\n"
-	"\t|  |  |  +- Keep period\n"
-	"\t|  |  +---- Grace period\n"
-	"\t|  +------- Remaining TTL\n"
-	"\t+---------- VXID of the object\n"
+	"\t%u %f %f %f [%u [%u]]\n"
+	"\t|  |  |  |   |   |\n"
+	"\t|  |  |  |   |   +- Content length\n"
+	"\t|  |  |  |   +----- Fetched so far\n"
+	"\t|  |  |  +--------- Keep period\n"
+	"\t|  |  +------------ Grace period\n"
+	"\t|  +--------------- Remaining TTL\n"
+	"\t+------------------ VXID of the object\n"
 	"\n"
 )
 

From dridi.boukelmoune at gmail.com  Mon Aug 30 12:46:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 12:46:05 +0000 (UTC)
Subject: [master] 9963084be vre: Constify VRE_Capture
Message-ID: <20210830124605.2A4CFAD24B@lists.varnish-cache.org>


commit 9963084be898d86b3129b4894f5a1ec84c448b68
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 30 14:44:57 2021 +0200

    vre: Constify VRE_Capture
    
    Spotted by Flexelint.

diff --git a/include/vre.h b/include/vre.h
index 2a9450ad5..bb8702b22 100644
--- a/include/vre.h
+++ b/include/vre.h
@@ -60,8 +60,9 @@ vre_t *VRE_export(const vre_t *, size_t *);
 int VRE_error(struct vsb *, int err);
 int VRE_match(const vre_t *code, const char *subject, size_t length,
     int options, const volatile struct vre_limits *lim);
-int VRE_capture(vre_t *code, const char *subject, size_t length, int options,
-    txt *groups, size_t count, const volatile struct vre_limits *lim);
+int VRE_capture(const vre_t *code, const char *subject, size_t length,
+    int options, txt *groups, size_t count,
+    const volatile struct vre_limits *lim);
 int VRE_sub(const vre_t *code, const char *subject, const char *replacement,
     struct vsb *vsb, const volatile struct vre_limits *lim, int all);
 void VRE_free(vre_t **);
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index 959201fa8..0cf5c9bfd 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -244,7 +244,7 @@ VRE_match(const vre_t *code, const char *subject, size_t length,
 }
 
 int
-VRE_capture(vre_t *code, const char *subject, size_t length, int options,
+VRE_capture(const vre_t *code, const char *subject, size_t length, int options,
     txt *groups, size_t count, const volatile struct vre_limits *lim)
 {
 	int i;

From dridi.boukelmoune at gmail.com  Mon Aug 30 13:58:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 30 Aug 2021 13:58:06 +0000 (UTC)
Subject: [master] 6fca9764d vtc: Stablize c105
Message-ID: <20210830135806.47F93AF610@lists.varnish-cache.org>


commit 6fca9764d193e963ec18152b570edf7dc3716141
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 30 15:52:59 2021 +0200

    vtc: Stablize c105

diff --git a/bin/varnishtest/tests/c00105.vtc b/bin/varnishtest/tests/c00105.vtc
index a5588b550..ebe558be5 100644
--- a/bin/varnishtest/tests/c00105.vtc
+++ b/bin/varnishtest/tests/c00105.vtc
@@ -17,6 +17,8 @@ server s2 {
 	txresp -status 304 -nolen -hdr {Etag: "abc"} -hdr "Content-Length: 100"
 } -start
 
+varnish v1 -cliok "param.set vsl_mask +ExpKill"
+
 varnish v1 -vcl+backend {
 	import directors;
 	import vtc;
@@ -36,6 +38,10 @@ varnish v1 -vcl+backend {
 	}
 } -start
 
+logexpect l1 -v v1 -g raw -q "ExpKill ~ EXP_expire" {
+	expect 0 0 ExpKill
+} -start
+
 client c1 {
 	txreq -hdr "backend: s1"
 	rxresphdrs
@@ -45,8 +51,7 @@ client c1 {
 
 barrier b1 sync
 
-# ensure stale_oc
-delay 0.01
+logexpect l1 -wait
 
 client c2 {
 	txreq -hdr "backend: s2"

From nils.goroll at uplex.de  Mon Aug 30 17:27:06 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 30 Aug 2021 17:27:06 +0000 (UTC)
Subject: [master] de6815250 Add allocbuf/freebuf to the umem stevedore
Message-ID: <20210830172706.EF2A3417A@lists.varnish-cache.org>


commit de6815250b2441a942eca29249799d6fc1d6f049
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Aug 30 19:25:53 2021 +0200

    Add allocbuf/freebuf to the umem stevedore
    
    Ref #3661

diff --git a/bin/varnishd/storage/storage_umem.c b/bin/varnishd/storage/storage_umem.c
index d4fc52a0d..2535d1171 100644
--- a/bin/varnishd/storage/storage_umem.c
+++ b/bin/varnishd/storage/storage_umem.c
@@ -465,6 +465,8 @@ const struct stevedore smu_stevedore = {
 	.methods	=	&SML_methods,
 	.var_free_space =	smu_free_space,
 	.var_used_space =	smu_used_space,
+	.allocbuf	=	SML_AllocBuf,
+	.freebuf	=	SML_FreeBuf,
 };
 
 #endif /* HAVE_UMEM_H */

From nils.goroll at uplex.de  Mon Aug 30 17:35:05 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 30 Aug 2021 17:35:05 +0000 (UTC)
Subject: [master] b10d0cba9 Make test compatible with malloc and umem stevedore
Message-ID: <20210830173505.9352D4755@lists.varnish-cache.org>


commit b10d0cba9bd409c0d691fab27c5f81e044c793d7
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Aug 30 19:34:10 2021 +0200

    Make test compatible with malloc and umem stevedore

diff --git a/bin/varnishtest/tests/t02022.vtc b/bin/varnishtest/tests/t02022.vtc
index 6a7ea5cc4..4b636be68 100644
--- a/bin/varnishtest/tests/t02022.vtc
+++ b/bin/varnishtest/tests/t02022.vtc
@@ -44,15 +44,15 @@ client c1 {
 	} -start
 } -start
 
-varnish v1 -expect SMA.rxbuf.g_bytes >= 2048
-varnish v1 -expect SMA.Transient.g_bytes == 0
+varnish v1 -expect SM?.rxbuf.g_bytes >= 2048
+varnish v1 -expect SM?.Transient.g_bytes == 0
 varnish v1 -expect MAIN.n_lru_nuked == 0
 
 barrier b1 sync
 client c1 -wait
 
-varnish v1 -expect SMA.rxbuf.g_bytes == 0
-varnish v1 -expect SMA.Transient.g_bytes == 0
+varnish v1 -expect SM?.rxbuf.g_bytes == 0
+varnish v1 -expect SM?.Transient.g_bytes == 0
 varnish v1 -expect MAIN.n_lru_nuked == 0
 
 client c2 {
@@ -62,7 +62,7 @@ client c2 {
 	expect resp.bodylen == 1048000
 } -run
 
-varnish v1 -expect SMA.rxbuf.g_bytes >= 1048000
+varnish v1 -expect SM?.rxbuf.g_bytes >= 1048000
 varnish v1 -expect MAIN.n_lru_nuked == 0
 
 client c3 {
@@ -74,13 +74,13 @@ client c3 {
 	} -start
 } -start
 
-varnish v1 -expect SMA.rxbuf.g_bytes >= 2048
-varnish v1 -expect SMA.rxbuf.g_bytes < 3000
-varnish v1 -expect SMA.Transient.g_bytes == 0
+varnish v1 -expect SM?.rxbuf.g_bytes >= 2048
+varnish v1 -expect SM?.rxbuf.g_bytes < 3000
+varnish v1 -expect SM?.Transient.g_bytes == 0
 varnish v1 -expect MAIN.n_lru_nuked == 1
 
 barrier b1 sync
 client c3 -wait
 
-varnish v1 -expect SMA.rxbuf.g_bytes == 0
-varnish v1 -expect SMA.Transient.g_bytes == 0
+varnish v1 -expect SM?.rxbuf.g_bytes == 0
+varnish v1 -expect SM?.Transient.g_bytes == 0

From phk at FreeBSD.org  Tue Aug 31 06:57:06 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 31 Aug 2021 06:57:06 +0000 (UTC)
Subject: [master] 983645cbe Coverage test of varnishtest
Message-ID: <20210831065706.7A340A4C54@lists.varnish-cache.org>


commit 983645cbeafc353ad926a3c9db8fe198af714f5b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 31 06:55:44 2021 +0000

    Coverage test of varnishtest

diff --git a/bin/varnishtest/tests/u00018.vtc b/bin/varnishtest/tests/u00018.vtc
new file mode 100644
index 000000000..4662fb1f4
--- /dev/null
+++ b/bin/varnishtest/tests/u00018.vtc
@@ -0,0 +1,19 @@
+varnishtest "Varnishtest coverage"
+
+# Cover usage processing
+shell -err -expect {usage: varnishtest} {
+	varnishtest -v -l
+}
+
+# Cover argument processing
+shell {
+	echo 'varnishtest foo' > ${tmpdir}/_.vtc
+	varnishtest -j 4 -n 10 -C -v -b 100k -L -l ${tmpdir}/_.vtc
+	ls -l ${tmpdir}
+}
+
+shell -err -expect {No such file or directory} {
+	rm -f ${tmpdir}/_.vtc
+	env VTEST_DURATION=15 \
+	    varnishtest -j 4 -n 10 -C -v -b 100k -L -l ${tmpdir}/_.vtc
+}

From phk at FreeBSD.org  Tue Aug 31 07:03:04 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 31 Aug 2021 07:03:04 +0000 (UTC)
Subject: [master] 630b58354 I hate autocrap.
Message-ID: <20210831070304.C6CA1A50C1@lists.varnish-cache.org>


commit 630b58354fc045e077f3c5653b81d80d738f6e0b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 31 07:02:07 2021 +0000

    I hate autocrap.

diff --git a/bin/varnishtest/tests/u00018.vtc b/bin/varnishtest/tests/u00018.vtc
index 4662fb1f4..2ad27dfe5 100644
--- a/bin/varnishtest/tests/u00018.vtc
+++ b/bin/varnishtest/tests/u00018.vtc
@@ -1,7 +1,7 @@
 varnishtest "Varnishtest coverage"
 
 # Cover usage processing
-shell -err -expect {usage: varnishtest} {
+shell -err -expect {usage:} {
 	varnishtest -v -l
 }
 

From phk at FreeBSD.org  Tue Aug 31 07:36:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 31 Aug 2021 07:36:05 +0000 (UTC)
Subject: [master] a9af17c9c Stabilize this test on CentOs
Message-ID: <20210831073605.E3581A6299@lists.varnish-cache.org>


commit a9af17c9c715722f51eb0c5461cf4c229c6bade1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 31 07:35:25 2021 +0000

    Stabilize this test on CentOs

diff --git a/bin/varnishtest/tests/c00105.vtc b/bin/varnishtest/tests/c00105.vtc
index ebe558be5..1c6005900 100644
--- a/bin/varnishtest/tests/c00105.vtc
+++ b/bin/varnishtest/tests/c00105.vtc
@@ -1,8 +1,7 @@
 varnishtest "Failed post-streaming revalidation"
 
-barrier b1 cond 2
+barrier b1 cond 3
 barrier b2 sock 2
-barrier b3 sock 2
 
 server s1 {
 	rxreq
@@ -20,7 +19,6 @@ server s2 {
 varnish v1 -cliok "param.set vsl_mask +ExpKill"
 
 varnish v1 -vcl+backend {
-	import directors;
 	import vtc;
 
 	sub vcl_recv {
@@ -32,7 +30,6 @@ varnish v1 -vcl+backend {
 	sub vcl_backend_response {
 		if (beresp.was_304) {
 			vtc.barrier_sync("${b2_sock}");
-			vtc.barrier_sync("${b3_sock}");
 		}
 		set beresp.ttl = 1ms;
 	}
@@ -46,20 +43,21 @@ client c1 {
 	txreq -hdr "backend: s1"
 	rxresphdrs
 	expect resp.status == 200
+	barrier b1 sync
 	expect_close
 } -start
 
-barrier b1 sync
-
 logexpect l1 -wait
 
+varnish v1 -vsl_catchup
+
+barrier b1 sync
+
 client c2 {
 	txreq -hdr "backend: s2"
 	rxresphdrs
 	expect resp.status == 200
 	expect_close
-} -start
+} -run
 
 client c1 -wait
-barrier b3 sync
-client c2 -wait

From dridi.boukelmoune at gmail.com  Tue Aug 31 08:44:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 31 Aug 2021 08:44:05 +0000 (UTC)
Subject: [master] de3b95607 stevedore: Set up nuke_limit for buffer allocations
Message-ID: <20210831084405.9B373A8353@lists.varnish-cache.org>


commit de3b95607de7698aefc80e39a8ec1e814ebb7d79
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 31 07:32:13 2021 +0200

    stevedore: Set up nuke_limit for buffer allocations

diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c
index ea9c3d073..a35e9eb7a 100644
--- a/bin/varnishd/storage/stevedore.c
+++ b/bin/varnishd/storage/stevedore.c
@@ -122,6 +122,7 @@ STV_AllocBuf(struct worker *wrk, const struct stevedore *stv, size_t size)
 	if (stv->allocbuf == NULL)
 		return (NULL);
 
+	wrk->strangelove = cache_param->nuke_limit;
 	buf = stv->allocbuf(wrk, stv, size + PRNDUP(sizeof *stvbuf), &priv);
 	if (buf == NULL)
 		return (NULL);

From dridi.boukelmoune at gmail.com  Tue Aug 31 08:44:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 31 Aug 2021 08:44:05 +0000 (UTC)
Subject: [master] cd76bc492 vtc: Stabilize t02022
Message-ID: <20210831084405.B36C7A8357@lists.varnish-cache.org>


commit cd76bc49243007f9c81e23df9681096c59280fed
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 31 07:33:45 2021 +0200

    vtc: Stabilize t02022

diff --git a/bin/varnishtest/tests/t02022.vtc b/bin/varnishtest/tests/t02022.vtc
index 4b636be68..39fbc4c68 100644
--- a/bin/varnishtest/tests/t02022.vtc
+++ b/bin/varnishtest/tests/t02022.vtc
@@ -6,6 +6,7 @@ server s1 {
 	rxreq
 	txresp -body asdf
 	rxreq
+	expect req.http.X-Varnish == 1005
 	txresp -bodylen 1048000
 	rxreq
 	txresp -body ASDF
@@ -31,6 +32,7 @@ varnish v1 -cliok "param.set feature +http2"
 varnish v1 -cliok "param.reset h2_initial_window_size"
 varnish v1 -cliok "param.reset h2_rx_window_low_water"
 varnish v1 -cliok "param.set h2_rxbuf_storage rxbuf"
+varnish v1 -cliok "param.set vsl_mask +ExpKill"
 varnish v1 -cliok "param.set debug +syncvsl"
 
 varnish v1 -start
@@ -65,6 +67,10 @@ client c2 {
 varnish v1 -expect SM?.rxbuf.g_bytes >= 1048000
 varnish v1 -expect MAIN.n_lru_nuked == 0
 
+logexpect l1 -v v1 -g raw -q "Expkill ~ LRU" {
+	expect * * Expkill x=1005
+} -start
+
 client c3 {
 	stream 1 {
 		txreq -req POST -url /1 -hdr "content-length" "2048" -nostrend
@@ -74,6 +80,8 @@ client c3 {
 	} -start
 } -start
 
+logexpect l1 -wait
+
 varnish v1 -expect SM?.rxbuf.g_bytes >= 2048
 varnish v1 -expect SM?.rxbuf.g_bytes < 3000
 varnish v1 -expect SM?.Transient.g_bytes == 0

From dridi.boukelmoune at gmail.com  Tue Aug 31 10:23:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 31 Aug 2021 10:23:06 +0000 (UTC)
Subject: [master] de952053e vtc: Improve range request checks in v34
Message-ID: <20210831102306.790B0AB1F2@lists.varnish-cache.org>


commit de952053ed5ce10a082839e0319fb497b6bf455d
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 23 07:54:09 2021 +0200

    vtc: Improve range request checks in v34
    
    While at it avoid reusing the same client name twice to help them stand
    out in the test log.

diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc
index c3f904663..a605977c4 100644
--- a/bin/varnishtest/tests/c00034.vtc
+++ b/bin/varnishtest/tests/c00034.vtc
@@ -16,7 +16,7 @@ varnish v1 -cliok "param.set http_range_support off"
 client c1 {
 	txreq -hdr "Range: bytes=0-9"
 	rxresp
-	expect resp.accept-ranges == "resp.accept-ranges"
+	expect resp.http.accept-ranges == <undef>
 	expect resp.status == 200
 	expect resp.bodylen == 100
 } -run
@@ -27,7 +27,7 @@ varnish v1 -vsl_catchup
 
 varnish v1 -cliok "param.set http_range_support on"
 
-client c1 {
+client c2 {
 	# Invalid range requests
 
 	txreq -hdr "Range: bytes =0-9"
@@ -78,7 +78,7 @@ varnish v1 -expect s_resp_bodybytes == 100
 
 varnish v1 -vsl_catchup
 
-client c1 {
+client c3 {
 	# Valid range requests
 
 	txreq -hdr "Range: bytes=0-49"
@@ -132,33 +132,21 @@ varnish v1 -vsl_catchup
 # Test Range streaming with streaming objects with C-L
 
 server s1 {
-	rxreq
-	txresp -nolen -hdr "Content-Length: 100"
-	send "0123456789"
-	send "0123456789"
-	send "0123456789"
-	send "0123456789"
-	send "0123456789"
-	send "0123456789"
-	send "0123456789"
-	send "0123456789"
-	send "0123456789"
-	delay 2
-	send "0123456789"
-
-	rxreq
-	txresp -nolen -hdr "Content-Length: 100"
-	send "0123456789"
-	send "0123456789"
-	send "0123456789"
-	send "0123456789"
-	send "0123456789"
-	send "0123456789"
-	send "0123456789"
-	send "0123456789"
-	send "0123456789"
-	delay 2
-	send "0123456789"
+	loop 2 {
+		rxreq
+		txresp -nolen -hdr "Content-Length: 100"
+		send "0123456789"
+		send "0123456789"
+		send "0123456789"
+		send "0123456789"
+		send "0123456789"
+		send "0123456789"
+		send "0123456789"
+		send "0123456789"
+		send "0123456789"
+		delay 2
+		send "0123456789"
+	}
 } -start
 
 varnish v1 -vcl+backend {
@@ -169,27 +157,31 @@ varnish v1 -vcl+backend {
 	}
 }
 
-client c1 {
+client c4 {
 	# Open ended range to force C-L usage
 	txreq -url /1 \
 		-hdr "Range: bytes=20-" \
 		-hdr "Connection: close"
-	rxresphdrs
+	rxresp
 	expect resp.status == 206
 	expect resp.http.Content-Range == "bytes 20-99/100"
-	recv 80
+	expect resp.http.Content-Length == 80
+	expect resp.bodylen == 80
 	expect_close
 } -run
 
 varnish v1 -vsl_catchup
 
-client c1 {
+client c5 {
 	# Closed C-L because we cannot use C-L
 	txreq -url /2 \
 		-hdr "Range: bytes=2-5" \
 		-hdr "Accept-encoding: gzip"
 	rxresp
 	expect resp.status == 200
+	expect resp.http.Content-Range == <undef>
+	expect resp.http.Content-Length == <undef>
+	expect resp.http.Content-Encoding == gzip
 	gunzip
 	expect resp.bodylen == 100
 } -run
@@ -204,7 +196,7 @@ server s1 {
 varnish v1 -cliok "param.set feature +http2"
 varnish v1 -vcl+backend ""
 
-client c2 {
+client c6 {
 	stream 1 {
 		txreq -url /3 -hdr "range" "bytes=0-1"
 		rxresp
@@ -232,13 +224,13 @@ varnish v1 -vcl+backend {
 	}
 }
 
-client c1 {
+client c7 {
 	txreq
 	rxresp
 	expect resp.http.foobar == ""
-	expect resp.http.accept-ranges == resp.http.accept-ranges
+	expect resp.http.accept-ranges == <undef>
 	txreq
 	rxresp
-	expect resp.http.foobar == "foobar"
+	expect resp.http.foobar == foobar
 	expect resp.http.accept-ranges == foobar
 } -run

From dridi.boukelmoune at gmail.com  Tue Aug 31 10:23:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 31 Aug 2021 10:23:06 +0000 (UTC)
Subject: [master] 723cd7b2a http: Explode http_GetContentLength() for reuse
Message-ID: <20210831102306.93371AB1F5@lists.varnish-cache.org>


commit 723cd7b2a1c596f231d56da07d4faac338669cd0
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Feb 25 08:27:37 2021 +0100

    http: Explode http_GetContentLength() for reuse

diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 233dd4f7e..bc30d4a66 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -821,29 +821,42 @@ http_GetHdrField(const struct http *hp, hdr_t hdr,
 
 /*--------------------------------------------------------------------*/
 
+static ssize_t
+http_parse_uint(const char *b, const char **e)
+{
+	ssize_t u;
+	unsigned n;
+
+	u = 0;
+	if (!vct_isdigit(*b))
+		return (-1);
+	for (; vct_isdigit(*b); b++) {
+		if (u > (SSIZE_MAX / 10))
+			return (-1);
+		u *= 10;
+		n = *b - '0';
+		if (u > (SSIZE_MAX - n))
+			return (-1);
+		u += n;
+	}
+
+	*e = b;
+	return (u);
+}
+
 ssize_t
 http_GetContentLength(const struct http *hp)
 {
 	ssize_t cl;
 	const char *b;
-	unsigned n;
 
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 
 	if (!http_GetHdr(hp, H_Content_Length, &b))
 		return (-1);
-	cl = 0;
-	if (!vct_isdigit(*b))
+	cl = http_parse_uint(b, &b);
+	if (cl < 0)
 		return (-2);
-	for (; vct_isdigit(*b); b++) {
-		if (cl > (SSIZE_MAX / 10))
-			return (-2);
-		cl *= 10;
-		n = *b - '0';
-		if (cl > (SSIZE_MAX - n))
-			return (-2);
-		cl += n;
-	}
 	while (vct_islws(*b))
 		b++;
 	if (*b != '\0')

From dridi.boukelmoune at gmail.com  Tue Aug 31 10:23:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 31 Aug 2021 10:23:06 +0000 (UTC)
Subject: [master] e68610a43 http: New http_GetContentRange() helper
Message-ID: <20210831102306.C32E6AB1F9@lists.varnish-cache.org>


commit e68610a43878505e459f983f2794813cc20902ae
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Feb 25 08:39:17 2021 +0100

    http: New http_GetContentRange() helper

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 35000cf6d..31fdd4d1f 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -606,6 +606,7 @@ int http_GetHdrField(const struct http *hp, hdr_t,
     const char *field, const char **ptr);
 double http_GetHdrQ(const struct http *hp, hdr_t, const char *field);
 ssize_t http_GetContentLength(const struct http *hp);
+ssize_t http_GetContentRange(const struct http *hp, ssize_t *lo, ssize_t *hi);
 uint16_t http_GetStatus(const struct http *hp);
 int http_IsStatus(const struct http *hp, int);
 void http_SetStatus(struct http *to, uint16_t status, const char *reason);
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index bc30d4a66..7819c383d 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -864,6 +864,70 @@ http_GetContentLength(const struct http *hp)
 	return (cl);
 }
 
+ssize_t
+http_GetContentRange(const struct http *hp, ssize_t *lo, ssize_t *hi)
+{
+	ssize_t tmp, cl;
+	const char *b;
+
+	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+
+	if (lo == NULL)
+		lo = &tmp;
+	if (hi == NULL)
+		hi = &tmp;
+
+	*lo = *hi = -1;
+
+	if (!http_GetHdr(hp, H_Content_Range, &b))
+		return (-1);
+
+	if (strncasecmp("bytes", b, strlen("bytes")))
+		return (-1);		// Unknown range unit, ignore
+	b += strlen("bytes");
+
+	if (!vct_islws(*b))
+		return (-1);		// Unknown range unit, ignore
+	while (vct_islws(*b))
+		b++;
+	if (*b == '*') {		// Content-Range: bytes */123
+		*lo = *hi = -1;
+		b++;
+	} else {			// Content-Range: bytes 1-2/3
+		*lo = http_parse_uint(b, &b);
+		if (*lo < 0)
+			return (-2);
+		if (*b != '-')
+			return (-2);
+		*hi = http_parse_uint(b + 1, &b);
+		if (*hi < 0)
+			return (-2);
+	}
+	if (*b != '/')
+		return (-2);
+	if (b[1] == '*') {		// Content-Range: bytes 1-2/*
+		cl = -1;
+		b += 2;
+	} else {
+		cl = http_parse_uint(b + 1, &b);
+		if (cl <= 0)
+			return (-2);
+	}
+	while (vct_islws(*b))
+		b++;
+	if (*b != '\0')
+		return (-2);
+	if (*lo > *hi)
+		return (-2);
+	assert(cl >= -1);
+	if (cl == -1)
+		return (-1);
+	if (*lo >= cl || *hi >= cl)
+		return (-2);
+	AN(cl);
+	return (cl);
+}
+
 /*--------------------------------------------------------------------
  */
 

From dridi.boukelmoune at gmail.com  Tue Aug 31 10:23:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 31 Aug 2021 10:23:06 +0000 (UTC)
Subject: [master] acd2cd05d http: Extract vrg_dorange parsing in
 http_GetRange()
Message-ID: <20210831102306.EAFBDAB1FD@lists.varnish-cache.org>


commit acd2cd05d029ee33f60c1ed2c1a86e8d5ce4b339
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 23 10:36:24 2021 +0200

    http: Extract vrg_dorange parsing in http_GetRange()

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 31fdd4d1f..245b7aa98 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -607,6 +607,7 @@ int http_GetHdrField(const struct http *hp, hdr_t,
 double http_GetHdrQ(const struct http *hp, hdr_t, const char *field);
 ssize_t http_GetContentLength(const struct http *hp);
 ssize_t http_GetContentRange(const struct http *hp, ssize_t *lo, ssize_t *hi);
+const char * http_GetRange(const struct http *hp, ssize_t *lo, ssize_t *hi);
 uint16_t http_GetStatus(const struct http *hp);
 int http_IsStatus(const struct http *hp, int);
 void http_SetStatus(struct http *to, uint16_t status, const char *reason);
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 7819c383d..005a5ac04 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -832,11 +832,11 @@ http_parse_uint(const char *b, const char **e)
 		return (-1);
 	for (; vct_isdigit(*b); b++) {
 		if (u > (SSIZE_MAX / 10))
-			return (-1);
+			return (-2);
 		u *= 10;
 		n = *b - '0';
 		if (u > (SSIZE_MAX - n))
-			return (-1);
+			return (-2);
 		u += n;
 	}
 
@@ -928,6 +928,51 @@ http_GetContentRange(const struct http *hp, ssize_t *lo, ssize_t *hi)
 	return (cl);
 }
 
+const char *
+http_GetRange(const struct http *hp, ssize_t *lo, ssize_t *hi)
+{
+	ssize_t tmp;
+	const char *b;
+
+	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
+
+	if (lo == NULL)
+		lo = &tmp;
+	if (hi == NULL)
+		hi = &tmp;
+
+	*lo = *hi = -1;
+
+	if (!http_GetHdr(hp, H_Range, &b))
+		return (NULL);
+
+	if (!http_range_at(b, bytes=))
+		return ("Not Bytes");
+	b += strlen("bytes=");
+
+	*lo = http_parse_uint(b, &b);
+	if (*lo == -2)
+		return ("Low number too big");
+	if (*b++ != '-')
+		return ("Missing hyphen");
+
+	*hi = http_parse_uint(b, &b);
+	if (*hi == -2)
+		return ("High number too big");
+	if (*lo == -1 && *hi == -1)
+		return ("Neither high nor low");
+	if (*lo == -1 && *hi == 0)
+		return ("No low, high is zero");
+	if (*hi >= 0 && *hi < *lo)
+		return ("high smaller than low");
+
+	while (vct_islws(*b))
+		b++;
+	if (*b != '\0')
+		return ("Trailing stuff");
+	return (NULL);
+}
+
 /*--------------------------------------------------------------------
  */
 
diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c
index 7451c3812..75c434a94 100644
--- a/bin/varnishd/cache/cache_range.c
+++ b/bin/varnishd/cache/cache_range.c
@@ -103,58 +103,30 @@ vrg_range_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv,
 /*--------------------------------------------------------------------*/
 
 static const char *
-vrg_dorange(struct req *req, const char *r, void **priv)
+vrg_dorange(struct req *req, void **priv)
 {
-	ssize_t low, high, has_low, has_high, t;
+	ssize_t low, high;
 	struct vrg_priv *vrg_priv;
+	const char *err;
 
-	if (!http_range_at(r, bytes=))
-		return ("Not Bytes");
-	r += 6;
-
-	/* The low end of range */
-	has_low = low = 0;
-	while (vct_isdigit(*r)) {
-		has_low = 1;
-		t = low;
-		low *= 10;
-		low += *r++ - '0';
-		if (low < t)
-			return ("Low number too big");
-	}
-
-	if (*r++ != '-')
-		return ("Missing hyphen");
-
-	/* The high end of range */
-	has_high = high = 0;
-	while (vct_isdigit(*r)) {
-		has_high = 1;
-		t = high;
-		high *= 10;
-		high += *r++ - '0';
-		if (high < t)
-			return ("High number too big");
-	}
-
-	if (*r != '\0')
-		return ("Trailing stuff");
+	err = http_GetRange(req->http, &low, &high);
+	if (err != NULL)
+		return (err);
 
-	if (has_high + has_low == 0)
-		return ("Neither high nor low");
+	assert(low >= -1);
+	assert(high >= -1);
 
-	if (!has_low) {
+	if (low < 0) {
 		if (req->resp_len < 0)
 			return (NULL);		// Allow 200 response
-		if (high == 0)
-			return ("No low, high is zero");
+		assert(high > 0);
 		low = req->resp_len - high;
 		if (low < 0)
 			low = 0;
 		high = req->resp_len - 1;
-	} else if (req->resp_len >= 0 && (high >= req->resp_len || !has_high))
+	} else if (req->resp_len >= 0 && (high >= req->resp_len || high < 0))
 		high = req->resp_len - 1;
-	else if (!has_high || req->resp_len < 0)
+	else if (high < 0 || req->resp_len < 0)
 		return (NULL);			// Allow 200 response
 	/*
 	 * else (bo != NULL) {
@@ -163,9 +135,6 @@ vrg_dorange(struct req *req, const char *r, void **priv)
 	 * }
 	 */
 
-	if (high < low)
-		return ("high smaller than low");
-
 	if (req->resp_len >= 0 && low >= req->resp_len)
 		return ("low range beyond object");
 
@@ -252,7 +221,6 @@ vrg_ifrange(struct req *req)
 static int v_matchproto_(vdp_init_f)
 vrg_range_init(struct vdp_ctx *vdc, void **priv, struct objcore *oc)
 {
-	const char *r;
 	const char *err;
 	struct req *req;
 
@@ -260,10 +228,10 @@ vrg_range_init(struct vdp_ctx *vdc, void **priv, struct objcore *oc)
 	(void)oc;
 	req = vdc->req;
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	assert(http_GetHdr(req->http, H_Range, &r));
+	assert(http_GetHdr(req->http, H_Range, NULL));
 	if (!vrg_ifrange(req))		// rfc7233,l,455,456
 		return (1);
-	err = vrg_dorange(req, r, priv);
+	err = vrg_dorange(req, priv);
 	if (err == NULL)
 		return (*priv == NULL ? 1 : 0);
 

From dridi.boukelmoune at gmail.com  Tue Aug 31 10:23:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 31 Aug 2021 10:23:07 +0000 (UTC)
Subject: [master] 4ab110047 vrg: Consistency check of backend responses
Message-ID: <20210831102307.3749EAB206@lists.varnish-cache.org>


commit 4ab110047130e3a89936104d74f4729b650676f9
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 23 17:27:36 2021 +0200

    vrg: Consistency check of backend responses
    
    That is, only when http_range_support is on, which is the default.
    
    Refs #3246

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 16ab14558..eb09990ff 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -492,6 +492,11 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	    http_GetHdrField(bo->beresp, H_Connection, "close", NULL))
 		bo->htc->doclose = SC_RESP_CLOSE;
 
+	if (VRG_CheckBo(bo) < 0) {
+		VDI_Finish(bo);
+		return (F_STP_ERROR);
+	}
+
 	if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL ||
 	    wrk->handling == VCL_RET_ERROR) {
 		/* do not count deliberately ending the backend connection as
diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c
index 75c434a94..20f326455 100644
--- a/bin/varnishd/cache/cache_range.c
+++ b/bin/varnishd/cache/cache_range.c
@@ -255,3 +255,73 @@ const struct vdp VDP_range = {
 	.bytes =	vrg_range_bytes,
 	.fini =		vrg_range_fini,
 };
+
+/*--------------------------------------------------------------------*/
+
+int
+VRG_CheckBo(struct busyobj *bo)
+{
+	ssize_t rlo, rhi, crlo, crhi, crlen, clen;
+	const char *err;
+
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+
+	if (!cache_param->http_range_support)
+		return (0);
+
+	err = http_GetRange(bo->bereq0, &rlo, &rhi);
+	clen = http_GetContentLength(bo->beresp);
+	crlen = http_GetContentRange(bo->beresp, &crlo, &crhi);
+
+	if (err != NULL) {
+		VSLb(bo->vsl, SLT_Error, "Invalid range header (%s)", err);
+		return (-1);
+	}
+
+	if (crlen < -1) {
+		VSLb(bo->vsl, SLT_Error, "Invalid content-range header");
+		return (-1);
+	}
+
+	if (clen < -1) {
+		VSLb(bo->vsl, SLT_Error, "Invalid content-length header");
+		return (-1);
+	}
+
+	if (crlo < 0 || crhi < 0) {
+		AZ(http_GetHdr(bo->beresp, H_Content_Range, NULL));
+		return (0);
+	}
+
+	if (rlo < 0 && rhi < 0) {
+		VSLb(bo->vsl, SLT_Error, "Unexpected content-range header");
+		return (-1);
+	}
+
+#define RANGE_CHECK(val, op, crval, what)			\
+	do {							\
+		if (val >= 0 && !(val op crval)) {		\
+			VSLb(bo->vsl, SLT_Error,		\
+			    "Expected " what " %zd, got %zd",	\
+			    crval, val);			\
+			return (-1);				\
+		}						\
+	} while (0)
+
+	crlen = crhi - crlo + 1;
+	RANGE_CHECK(clen, ==, crlen, "content length");
+
+	/* NB: if the client didn't specify a low range the high range
+	 * was adjusted based on the resource length, and a high range
+	 * is allowed to be out of bounds so at this point there is
+	 * nothing left to check.
+	 */
+	if (rlo < 0)
+		return (0);
+
+	RANGE_CHECK(rlo, ==, crlo, "low range");
+	RANGE_CHECK(rhi, >=, crhi, "minimum high range");
+#undef RANGE_CHECK
+
+	return (0);
+}
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index 653976cdc..47213fff6 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -396,6 +396,9 @@ void Pool_PurgeStat(unsigned nobj);
 int Pool_Task_Any(struct pool_task *task, enum task_prio prio);
 void pan_pool(struct vsb *);
 
+/* cache_range.c */
+int VRG_CheckBo(struct busyobj *);
+
 /* cache_req.c */
 struct req *Req_New(const struct worker *, struct sess *);
 void Req_Release(struct req *);
diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc
index a605977c4..44773ffe2 100644
--- a/bin/varnishtest/tests/c00034.vtc
+++ b/bin/varnishtest/tests/c00034.vtc
@@ -190,6 +190,7 @@ client c5 {
 
 server s1 {
 	rxreq
+	expect req.url == "/3"
 	txresp -hdr "Content-length: 3" -body "BLA"
 } -start
 
@@ -234,3 +235,42 @@ client c7 {
 	expect resp.http.foobar == foobar
 	expect resp.http.accept-ranges == foobar
 } -run
+
+server s1 {
+	rxreq
+	expect req.http.range == "bytes=10-19"
+	txresp -status 206 -hdr "content-range: bytes 0-49/100" -bodylen 40
+
+	rxreq
+	expect req.http.range == "bytes=10-19"
+	txresp -status 206 -hdr "content-range: bytes 10-19/100" -bodylen 40
+
+	rxreq
+	expect req.url == "/4"
+	expect req.http.range == <undef>
+	txresp -hdr "content-range: bytes 0-49/100" -bodylen 40
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		if (req.http.return == "pass") {
+			return (pass);
+		}
+	}
+}
+
+client c8 {
+	# range vs content-range vs content-length
+
+	txreq -hdr "range: bytes=10-19" -hdr "return: pass"
+	rxresp
+	expect resp.status == 503
+
+	txreq -hdr "range: bytes=10-19" -hdr "return: pass"
+	rxresp
+	expect resp.status == 503
+
+	txreq -url /4
+	rxresp
+	expect resp.status == 503
+} -run
diff --git a/bin/varnishtest/tests/r00466.vtc b/bin/varnishtest/tests/r00466.vtc
index d530b443c..94c700a22 100644
--- a/bin/varnishtest/tests/r00466.vtc
+++ b/bin/varnishtest/tests/r00466.vtc
@@ -36,10 +36,14 @@ client c1 {
 	expect resp.status == 206
 	expect resp.http.Content-Length == 2
 	expect resp.http.X-Varnish == "1001"
+} -run
+
+varnish v1 -cliok "param.set http_range_support off"
 
+client c2 {
 	# NB: Deliberately bogus Range header
 	txreq -url "/bar" -hdr "Range: 200-300"
 	rxresp
 	expect resp.status == 206
-	expect resp.http.X-Varnish == "1003"
+	expect resp.http.X-Varnish == "1004"
 } -run
diff --git a/bin/varnishtest/tests/r02530.vtc b/bin/varnishtest/tests/r02530.vtc
index c458b6a5b..f664caee4 100644
--- a/bin/varnishtest/tests/r02530.vtc
+++ b/bin/varnishtest/tests/r02530.vtc
@@ -30,6 +30,8 @@ server s1 {
 	sendhex 1f8b
 } -start
 
+varnish v1 -cliok "param.set http_range_support off"
+
 varnish v1 -vcl+backend {
 	sub vcl_recv {
 		if (req.url == "/pass") {

From dridi.boukelmoune at gmail.com  Tue Aug 31 10:23:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 31 Aug 2021 10:23:07 +0000 (UTC)
Subject: [master] b47e278ae range: Don't panic in the absence of a range header
Message-ID: <20210831102307.56BE1AB209@lists.varnish-cache.org>


commit b47e278ae5c9dfc0ff09d773390afadd188f1923
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 30 18:16:47 2021 +0200

    range: Don't panic in the absence of a range header
    
    That's just another case of falling back to regular delivery if the VCL
    mistakenly attempts to set up a range filter.

diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c
index 20f326455..cecc1efdb 100644
--- a/bin/varnishd/cache/cache_range.c
+++ b/bin/varnishd/cache/cache_range.c
@@ -117,7 +117,7 @@ vrg_dorange(struct req *req, void **priv)
 	assert(high >= -1);
 
 	if (low < 0) {
-		if (req->resp_len < 0)
+		if (req->resp_len < 0 || high < 0)
 			return (NULL);		// Allow 200 response
 		assert(high > 0);
 		low = req->resp_len - high;
@@ -228,7 +228,6 @@ vrg_range_init(struct vdp_ctx *vdc, void **priv, struct objcore *oc)
 	(void)oc;
 	req = vdc->req;
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	assert(http_GetHdr(req->http, H_Range, NULL));
 	if (!vrg_ifrange(req))		// rfc7233,l,455,456
 		return (1);
 	err = vrg_dorange(req, priv);
diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc
index 44773ffe2..d34b91cbe 100644
--- a/bin/varnishtest/tests/c00034.vtc
+++ b/bin/varnishtest/tests/c00034.vtc
@@ -274,3 +274,23 @@ client c8 {
 	rxresp
 	expect resp.status == 503
 } -run
+
+# range filter without a range header
+
+server s1 {
+	rxreq
+	txresp -bodylen 256
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_deliver {
+		set resp.filters = "range";
+	}
+}
+
+client c1 {
+	txreq -url /5
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 256
+} -run

From dridi.boukelmoune at gmail.com  Tue Aug 31 10:23:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 31 Aug 2021 10:23:07 +0000 (UTC)
Subject: [master] 3f6a25170 vnum: New VNUM_uint() and VNUM_hex() functions
Message-ID: <20210831102307.85813AB20D@lists.varnish-cache.org>


commit 3f6a25170488bb8bfca5f07a81e94cc1c5c35ade
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 31 11:57:38 2021 +0200

    vnum: New VNUM_uint() and VNUM_hex() functions
    
    With the hex table suggested by fgs.

diff --git a/include/vnum.h b/include/vnum.h
index cd08c3216..fe9e78bd1 100644
--- a/include/vnum.h
+++ b/include/vnum.h
@@ -37,6 +37,8 @@ vtim_dur VNUM_duration(const char *p);
 int64_t VNUM_bytes_unit(double r, const char *b, const char *e, uintmax_t rel,
     const char **errtxt);
 const char *VNUM_2bytes(const char *p, uintmax_t *r, uintmax_t rel);
+ssize_t VNUM_uint(const char *b, const char *e, const char **p);
+ssize_t VNUM_hex(const char *b, const char *e, const char **p);
 
 int64_t SF_Parse_Integer(const char **ipp, const char **errtxt);
 double SF_Parse_Decimal(const char **ipp, int strict, const char **errtxt);
diff --git a/lib/libvarnish/vnum.c b/lib/libvarnish/vnum.c
index c6c59e0ba..fa3a8cc97 100644
--- a/lib/libvarnish/vnum.c
+++ b/lib/libvarnish/vnum.c
@@ -33,6 +33,9 @@
 
 #include "config.h"
 
+#include <sys/types.h>
+
+#include <limits.h>
 #include <math.h>
 #include <stdint.h>
 #include <stdio.h>
@@ -353,6 +356,66 @@ VNUM_2bytes(const char *p, uintmax_t *r, uintmax_t rel)
 	return (NULL);
 }
 
+/**********************************************************************/
+
+static const uint8_t hex_table[] = {
+	0,   1,   2,   3,   4,   5,   6,   7,   8,   9,
+	127, 127, 127, 127, 127, 127, 127, 10,  11,  12,
+	13,  14,  15,  127, 127, 127, 127, 127, 127, 127,
+	127, 127, 127, 127, 127, 127, 127, 127, 127, 127,
+	127, 127, 127, 127, 127, 127, 127, 127, 127, 10,
+	11,  12,  13,  14,  15
+};
+
+static ssize_t
+vnum_uint(const char *b, const char *e, const char **p, unsigned base)
+{
+	ssize_t u;
+	unsigned n;
+
+	AN(b);
+	AN(p);
+	if (e == NULL)
+		e = strchr(b, '\0');
+
+	u = 0;
+	if (!vct_ishex(*b) || hex_table[*b - '0'] >= base) {
+		*p = b;
+		return (-1);
+	}
+
+	for (; b < e && vct_ishex(*b) && hex_table[*b - '0'] < base; b++) {
+		if (u > (SSIZE_MAX / base)) {
+			u = -2;
+			break;
+		}
+		u *= base;
+		n = hex_table[*b - '0'];
+		if (u > (SSIZE_MAX - n)) {
+			u = -2;
+			break;
+		}
+		u += n;
+	}
+
+	*p = b;
+	return (u);
+}
+
+ssize_t
+VNUM_uint(const char *b, const char *e, const char **p)
+{
+
+	return (vnum_uint(b, e, p, 10));
+}
+
+ssize_t
+VNUM_hex(const char *b, const char *e, const char **p)
+{
+
+	return (vnum_uint(b, e, p, 16));
+}
+
 #ifdef NUM_C_TEST
 /*
  * Compile with:

From dridi.boukelmoune at gmail.com  Tue Aug 31 10:23:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 31 Aug 2021 10:23:07 +0000 (UTC)
Subject: [master] 27df9bc70 http: Move parsing to VNUM_uint()
Message-ID: <20210831102307.AE88DAB213@lists.varnish-cache.org>


commit 27df9bc704c77e406a3ea0ddcd6c9be63865e56b
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 31 11:58:14 2021 +0200

    http: Move parsing to VNUM_uint()

diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 005a5ac04..701ff3a81 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -36,8 +36,9 @@
 #include "cache_varnishd.h"
 #include <stdio.h>
 
-#include "vend.h"
 #include "vct.h"
+#include "vend.h"
+#include "vnum.h"
 #include "vtim.h"
 
 #define BODYSTATUS(U, l, n, a, k)				\
@@ -821,29 +822,6 @@ http_GetHdrField(const struct http *hp, hdr_t hdr,
 
 /*--------------------------------------------------------------------*/
 
-static ssize_t
-http_parse_uint(const char *b, const char **e)
-{
-	ssize_t u;
-	unsigned n;
-
-	u = 0;
-	if (!vct_isdigit(*b))
-		return (-1);
-	for (; vct_isdigit(*b); b++) {
-		if (u > (SSIZE_MAX / 10))
-			return (-2);
-		u *= 10;
-		n = *b - '0';
-		if (u > (SSIZE_MAX - n))
-			return (-2);
-		u += n;
-	}
-
-	*e = b;
-	return (u);
-}
-
 ssize_t
 http_GetContentLength(const struct http *hp)
 {
@@ -854,7 +832,7 @@ http_GetContentLength(const struct http *hp)
 
 	if (!http_GetHdr(hp, H_Content_Length, &b))
 		return (-1);
-	cl = http_parse_uint(b, &b);
+	cl = VNUM_uint(b, NULL, &b);
 	if (cl < 0)
 		return (-2);
 	while (vct_islws(*b))
@@ -894,12 +872,12 @@ http_GetContentRange(const struct http *hp, ssize_t *lo, ssize_t *hi)
 		*lo = *hi = -1;
 		b++;
 	} else {			// Content-Range: bytes 1-2/3
-		*lo = http_parse_uint(b, &b);
+		*lo = VNUM_uint(b, NULL, &b);
 		if (*lo < 0)
 			return (-2);
 		if (*b != '-')
 			return (-2);
-		*hi = http_parse_uint(b + 1, &b);
+		*hi = VNUM_uint(b + 1, NULL, &b);
 		if (*hi < 0)
 			return (-2);
 	}
@@ -909,7 +887,7 @@ http_GetContentRange(const struct http *hp, ssize_t *lo, ssize_t *hi)
 		cl = -1;
 		b += 2;
 	} else {
-		cl = http_parse_uint(b + 1, &b);
+		cl = VNUM_uint(b + 1, NULL, &b);
 		if (cl <= 0)
 			return (-2);
 	}
@@ -950,13 +928,13 @@ http_GetRange(const struct http *hp, ssize_t *lo, ssize_t *hi)
 		return ("Not Bytes");
 	b += strlen("bytes=");
 
-	*lo = http_parse_uint(b, &b);
+	*lo = VNUM_uint(b, NULL, &b);
 	if (*lo == -2)
 		return ("Low number too big");
 	if (*b++ != '-')
 		return ("Missing hyphen");
 
-	*hi = http_parse_uint(b, &b);
+	*hi = VNUM_uint(b, NULL, &b);
 	if (*hi == -2)
 		return ("High number too big");
 	if (*lo == -1 && *hi == -1)

From dridi.boukelmoune at gmail.com  Tue Aug 31 10:23:07 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 31 Aug 2021 10:23:07 +0000 (UTC)
Subject: [master] f3b172acd vav: Ditch sscanf() in favor of VNUM_hex()
Message-ID: <20210831102307.C993EAB21F@lists.varnish-cache.org>


commit f3b172acdfb9f0c9022714630f8228eecd6ec618
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 31 11:59:03 2021 +0200

    vav: Ditch sscanf() in favor of VNUM_hex()
    
    With this vav_test needs to link against libm, but it's simpler to tell
    libvarnish to link since it's the one shipping VNUM in the first place.
    
    Fixes #3645

diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am
index 0559a77cc..d19e61409 100644
--- a/lib/libvarnish/Makefile.am
+++ b/lib/libvarnish/Makefile.am
@@ -44,7 +44,7 @@ libvarnish_la_SOURCES = \
 	vtim.c \
 	vus.c
 
-libvarnish_la_LIBADD = @PCRE2_LIBS@
+libvarnish_la_LIBADD = @PCRE2_LIBS@ $(LIBM)
 
 TESTS = vav_test vbh_test vct_test vjsn_test vnum_c_test vsb_test
 
@@ -64,7 +64,7 @@ vct_test_LDADD = $(AM_LDFLAGS) libvarnish.la
 
 vnum_c_test_SOURCES = vnum.c
 vnum_c_test_CFLAGS = $(AM_CFLAGS) -DNUM_C_TEST
-vnum_c_test_LDADD = $(AM_LDFLAGS) libvarnish.la ${LIBM}
+vnum_c_test_LDADD = $(AM_LDFLAGS) libvarnish.la
 
 vjsn_test_SOURCES = vjsn.c
 vjsn_test_CFLAGS = $(AM_CFLAGS) -DVJSN_TEST
diff --git a/lib/libvarnish/vav.c b/lib/libvarnish/vav.c
index ffebc3f49..42ea2bca1 100644
--- a/lib/libvarnish/vav.c
+++ b/lib/libvarnish/vav.c
@@ -43,6 +43,7 @@
 #include "config.h"
 
 #include <ctype.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -51,13 +52,14 @@
 
 #include "vas.h"
 #include "vav.h"
+#include "vnum.h"
 
 static int
 vav_backslash_txt(const char *s, const char *e, char *res)
 {
-	int r, l;
+	int r, l, i;
+	const char *p;
 	char c;
-	unsigned u;
 
 	AN(s);
 	if (e == NULL)
@@ -102,10 +104,10 @@ vav_backslash_txt(const char *s, const char *e, char *res)
 		}
 		break;
 	case 'x':
-		if (l >= 4 && isxdigit(s[2]) && isxdigit(s[3]) &&
-		    sscanf(s + 1, "x%02x", &u) == 1) {
-			AZ(u & ~0xff);
-			c = u;	/*lint !e734 loss of precision */
+		if (l >= 4 && (i = VNUM_hex(s + 2, s + 4, &p)) >= 0 &&
+		    p == s + 4) {
+			AZ(i & ~0xff);
+			c = i;	/*lint !e734 loss of precision */
 			r = 4;
 		}
 		break;

From dridi.boukelmoune at gmail.com  Tue Aug 31 10:48:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 31 Aug 2021 10:48:06 +0000 (UTC)
Subject: [master] 557e99bf6 range: Polish
Message-ID: <20210831104806.7A525AC956@lists.varnish-cache.org>


commit 557e99bf6359af015a801534717e46149da992d9
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 31 12:46:04 2021 +0200

    range: Polish
    
    Spotted by Flexelint.

diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c
index cecc1efdb..b9944aa6e 100644
--- a/bin/varnishd/cache/cache_range.c
+++ b/bin/varnishd/cache/cache_range.c
@@ -307,7 +307,7 @@ VRG_CheckBo(struct busyobj *bo)
 		}						\
 	} while (0)
 
-	crlen = crhi - crlo + 1;
+	crlen = (crhi - crlo) + 1;
 	RANGE_CHECK(clen, ==, crlen, "content length");
 
 	/* NB: if the client didn't specify a low range the high range