From phk at FreeBSD.org  Mon Aug  3 07:58:09 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon,  3 Aug 2020 07:58:09 +0000 (UTC)
Subject: [master] e94bcc969 Move the generated vcc_types.h from include/tbl to
 lib/libvcc
Message-ID: <20200803075809.A8AF910F7AC@lists.varnish-cache.org>


commit e94bcc9694826fab265e2bf2fd055d2324c15be9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 3 07:56:40 2020 +0000

    Move the generated vcc_types.h from include/tbl to lib/libvcc

diff --git a/.gitignore b/.gitignore
index d98e8064e..4d0db9281 100644
--- a/.gitignore
+++ b/.gitignore
@@ -58,11 +58,11 @@ cscope.*out
 /include/vmod_abi.h
 /include/tbl/vcl_returns.h
 /include/tbl/vrt_stv_var.h
-/include/tbl/vcc_types.h
 /include/vcs_version.h
 /lib/libvcc/vcc_fixed_token.c
 /lib/libvcc/vcc_obj.c
 /lib/libvcc/vcc_token_defs.h
+/lib/libvcc/vcc_types.h
 /lib/libvarnishapi/vsl2rst
 /lib/libvarnishapi/vxp_fixed_token.c
 /lib/libvarnishapi/vxp_tokens.h
diff --git a/include/Makefile.am b/include/Makefile.am
index fa6430de7..6172c657e 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -32,7 +32,6 @@ nobase_pkginclude_HEADERS = \
 	tbl/sess_close.h \
 	tbl/steps.h \
 	tbl/symbol_kind.h \
-	tbl/vcc_types.h \
 	tbl/vcl_returns.h \
 	tbl/vcl_states.h \
 	tbl/vhd_fsm.h \
@@ -122,7 +121,6 @@ vcl.h: \
 GEN_H = \
 	tbl/vrt_stv_var.h \
 	tbl/vcl_returns.h \
-	tbl/vcc_types.h \
 	vrt_obj.h
 
 $(GEN_H): vcl.h
diff --git a/lib/libvcc/Makefile.am b/lib/libvcc/Makefile.am
index bf352e019..1e3d6f2d0 100644
--- a/lib/libvcc/Makefile.am
+++ b/lib/libvcc/Makefile.am
@@ -15,6 +15,7 @@ libvcc_a_SOURCES = \
 	vcc_compile.h \
 	vcc_namespace.h \
 	vcc_token_defs.h \
+	vcc_types.h \
 	vcc_acl.c \
 	vcc_action.c \
 	vcc_backend.c \
@@ -56,6 +57,7 @@ vcc_obj.c: \
 GEN_H = \
 	vcc_fixed_token.c \
 	vcc_token_defs.h \
+	vcc_types.h \
 	tbl/vrt_stv_var.h
 
 $(GEN_H): vcc_obj.c
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index c0f4a4543..86112f640 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -786,7 +786,7 @@ fo.write("\n}\n")
 fo.close()
 
 #######################################################################
-ft = open(join(buildroot, "include/tbl/vcc_types.h"), "w")
+ft = open(join(buildroot, "lib/libvcc/vcc_types.h"), "w")
 file_header(ft)
 
 lint_start(ft)
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index b6db44343..173617e84 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -116,7 +116,7 @@ struct type {
 };
 
 #define VCC_TYPE(UC, lc)	extern const struct type UC[1];
-#include "tbl/vcc_types.h"
+#include "vcc_types.h"
 
 /*---------------------------------------------------------------------*/
 
diff --git a/lib/libvcc/vcc_types.c b/lib/libvcc/vcc_types.c
index b49eaa9f6..dace016e9 100644
--- a/lib/libvcc/vcc_types.c
+++ b/lib/libvcc/vcc_types.c
@@ -224,7 +224,7 @@ VCC_Type(const char *p)
 {
 
 #define VCC_TYPE(UC, lc)	if (!strcmp(p, #UC)) return (UC);
-#include "tbl/vcc_types.h"
+#include "vcc_types.h"
 	return (NULL);
 }
 
@@ -291,5 +291,5 @@ vcc_Type_Init(struct vcc *tl)
 {
 
 #define VCC_TYPE(UC, lc)	vcc_type_init(tl, UC);
-#include "tbl/vcc_types.h"
+#include "vcc_types.h"
 }

From phk at FreeBSD.org  Mon Aug  3 09:38:06 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon,  3 Aug 2020 09:38:06 +0000 (UTC)
Subject: [master] 3b0335889 Dont include <stdatomic.h> when running FlexeLint
Message-ID: <20200803093806.F007B111AA1@lists.varnish-cache.org>


commit 3b033588970c0af54eea3dae742c5becafd19916
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 3 09:37:33 2020 +0000

    Dont include <stdatomic.h> when running FlexeLint

diff --git a/include/vmb.h b/include/vmb.h
index 7029e29f6..2e3f52550 100644
--- a/include/vmb.h
+++ b/include/vmb.h
@@ -34,7 +34,7 @@
 #ifndef VMB_H_INCLUDED
 #define VMB_H_INCLUDED
 
-#if defined(HAVE_STDATOMIC_H)
+#if defined(HAVE_STDATOMIC_H) && !defined(__FLEXELINT__)
 
 #  include <stdatomic.h>
 #  define VWMB()	atomic_thread_fence(memory_order_release)
diff --git a/tools/flint_skel.sh b/tools/flint_skel.sh
index 557c7aa59..561a7d05e 100644
--- a/tools/flint_skel.sh
+++ b/tools/flint_skel.sh
@@ -7,6 +7,7 @@ if [ "x$1" = "x-ok" -a -f _.fl ] ; then
 fi
 
 flexelint \
+	-D__FLEXELINT__ \
 	../../flint.lnt \
 	../flint.lnt \
 	flint.lnt \

From phk at FreeBSD.org  Mon Aug  3 13:04:07 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon,  3 Aug 2020 13:04:07 +0000 (UTC)
Subject: [master] e7d8faab2 Reflect that vcc_types.h moved
Message-ID: <20200803130407.A1F7811781D@lists.varnish-cache.org>


commit e7d8faab231522a3c288b7af209db21e38baab18
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 3 09:41:51 2020 +0000

    Reflect that vcc_types.h moved

diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt
index d9175cacd..8d1867f6c 100644
--- a/bin/varnishd/flint.lnt
+++ b/bin/varnishd/flint.lnt
@@ -86,7 +86,6 @@
 //////////////
 -efunc(1791, pdiff)	// return last on line
 //////////////
--efile(451, "vcc_types.h")	// No include guard
 -efile(451, "symbol_kind.h")	// No include guard
 -efile(451, "config.h")	// No include guard
 //////////////
diff --git a/lib/libvcc/flint.lnt b/lib/libvcc/flint.lnt
index f060458b9..d419a731d 100644
--- a/lib/libvcc/flint.lnt
+++ b/lib/libvcc/flint.lnt
@@ -7,3 +7,4 @@
 -esym(768, token)	// FLINTBUG: global struct member 'token' not ref
 
 -efile(451, vcc_namespace.h)	// No include guard
+-efile(451, vcc_types.h)	// No include guard

From fgsch at lodoss.net  Mon Aug  3 14:01:07 2020
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon,  3 Aug 2020 14:01:07 +0000 (UTC)
Subject: [master] 8b7919309 Partially revert b82271e5
Message-ID: <20200803140107.B94EF118CCB@lists.varnish-cache.org>


commit 8b79193094ccc5e910e34f2bb1cbb84f0378868d
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Aug 3 14:55:59 2020 +0100

    Partially revert b82271e5
    
    Automake generates a different libtool invocation if ?= is used.
    More investigation needed but for now this is enough to fix make
    check under macos.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 66d6e67fd..c68b30836 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -66,7 +66,7 @@ vmod_LTLIBRARIES = libvmod_XXX.la
 libvmod_XXX_la_CFLAGS ?= \\
 \t at SAN_CFLAGS@
 
-libvmod_XXX_la_LDFLAGS ?= \\
+libvmod_XXX_la_LDFLAGS = \\
 \t-export-symbols-regex 'Vmod_XXX_Data' \\
 \t$(AM_LDFLAGS) \\
 \t$(VMOD_LDFLAGS) \\
diff --git a/lib/libvmod_blob/automake_boilerplate.am b/lib/libvmod_blob/automake_boilerplate.am
index 7931bbbf0..60e496959 100644
--- a/lib/libvmod_blob/automake_boilerplate.am
+++ b/lib/libvmod_blob/automake_boilerplate.am
@@ -15,7 +15,7 @@ vmod_LTLIBRARIES = libvmod_blob.la
 libvmod_blob_la_CFLAGS ?= \
 	@SAN_CFLAGS@
 
-libvmod_blob_la_LDFLAGS ?= \
+libvmod_blob_la_LDFLAGS = \
 	-export-symbols-regex 'Vmod_blob_Data' \
 	$(AM_LDFLAGS) \
 	$(VMOD_LDFLAGS) \
diff --git a/lib/libvmod_cookie/automake_boilerplate.am b/lib/libvmod_cookie/automake_boilerplate.am
index bf5214708..88d6167a6 100644
--- a/lib/libvmod_cookie/automake_boilerplate.am
+++ b/lib/libvmod_cookie/automake_boilerplate.am
@@ -15,7 +15,7 @@ vmod_LTLIBRARIES = libvmod_cookie.la
 libvmod_cookie_la_CFLAGS ?= \
 	@SAN_CFLAGS@
 
-libvmod_cookie_la_LDFLAGS ?= \
+libvmod_cookie_la_LDFLAGS = \
 	-export-symbols-regex 'Vmod_cookie_Data' \
 	$(AM_LDFLAGS) \
 	$(VMOD_LDFLAGS) \
diff --git a/lib/libvmod_debug/automake_boilerplate.am b/lib/libvmod_debug/automake_boilerplate.am
index b0bc43174..7850094f6 100644
--- a/lib/libvmod_debug/automake_boilerplate.am
+++ b/lib/libvmod_debug/automake_boilerplate.am
@@ -15,7 +15,7 @@ vmod_LTLIBRARIES = libvmod_debug.la
 libvmod_debug_la_CFLAGS ?= \
 	@SAN_CFLAGS@
 
-libvmod_debug_la_LDFLAGS ?= \
+libvmod_debug_la_LDFLAGS = \
 	-export-symbols-regex 'Vmod_debug_Data' \
 	$(AM_LDFLAGS) \
 	$(VMOD_LDFLAGS) \
diff --git a/lib/libvmod_directors/automake_boilerplate.am b/lib/libvmod_directors/automake_boilerplate.am
index f12c3c4e8..d84a44076 100644
--- a/lib/libvmod_directors/automake_boilerplate.am
+++ b/lib/libvmod_directors/automake_boilerplate.am
@@ -15,7 +15,7 @@ vmod_LTLIBRARIES = libvmod_directors.la
 libvmod_directors_la_CFLAGS ?= \
 	@SAN_CFLAGS@
 
-libvmod_directors_la_LDFLAGS ?= \
+libvmod_directors_la_LDFLAGS = \
 	-export-symbols-regex 'Vmod_directors_Data' \
 	$(AM_LDFLAGS) \
 	$(VMOD_LDFLAGS) \
diff --git a/lib/libvmod_proxy/automake_boilerplate.am b/lib/libvmod_proxy/automake_boilerplate.am
index a8a257dab..71cd6dab3 100644
--- a/lib/libvmod_proxy/automake_boilerplate.am
+++ b/lib/libvmod_proxy/automake_boilerplate.am
@@ -15,7 +15,7 @@ vmod_LTLIBRARIES = libvmod_proxy.la
 libvmod_proxy_la_CFLAGS ?= \
 	@SAN_CFLAGS@
 
-libvmod_proxy_la_LDFLAGS ?= \
+libvmod_proxy_la_LDFLAGS = \
 	-export-symbols-regex 'Vmod_proxy_Data' \
 	$(AM_LDFLAGS) \
 	$(VMOD_LDFLAGS) \
diff --git a/lib/libvmod_purge/automake_boilerplate.am b/lib/libvmod_purge/automake_boilerplate.am
index 04904a376..f06b85afe 100644
--- a/lib/libvmod_purge/automake_boilerplate.am
+++ b/lib/libvmod_purge/automake_boilerplate.am
@@ -15,7 +15,7 @@ vmod_LTLIBRARIES = libvmod_purge.la
 libvmod_purge_la_CFLAGS ?= \
 	@SAN_CFLAGS@
 
-libvmod_purge_la_LDFLAGS ?= \
+libvmod_purge_la_LDFLAGS = \
 	-export-symbols-regex 'Vmod_purge_Data' \
 	$(AM_LDFLAGS) \
 	$(VMOD_LDFLAGS) \
diff --git a/lib/libvmod_std/automake_boilerplate.am b/lib/libvmod_std/automake_boilerplate.am
index 6a14779b2..5bd015b9b 100644
--- a/lib/libvmod_std/automake_boilerplate.am
+++ b/lib/libvmod_std/automake_boilerplate.am
@@ -15,7 +15,7 @@ vmod_LTLIBRARIES = libvmod_std.la
 libvmod_std_la_CFLAGS ?= \
 	@SAN_CFLAGS@
 
-libvmod_std_la_LDFLAGS ?= \
+libvmod_std_la_LDFLAGS = \
 	-export-symbols-regex 'Vmod_std_Data' \
 	$(AM_LDFLAGS) \
 	$(VMOD_LDFLAGS) \
diff --git a/lib/libvmod_unix/automake_boilerplate.am b/lib/libvmod_unix/automake_boilerplate.am
index f053524a1..fa0586db3 100644
--- a/lib/libvmod_unix/automake_boilerplate.am
+++ b/lib/libvmod_unix/automake_boilerplate.am
@@ -15,7 +15,7 @@ vmod_LTLIBRARIES = libvmod_unix.la
 libvmod_unix_la_CFLAGS ?= \
 	@SAN_CFLAGS@
 
-libvmod_unix_la_LDFLAGS ?= \
+libvmod_unix_la_LDFLAGS = \
 	-export-symbols-regex 'Vmod_unix_Data' \
 	$(AM_LDFLAGS) \
 	$(VMOD_LDFLAGS) \
diff --git a/lib/libvmod_vtc/automake_boilerplate.am b/lib/libvmod_vtc/automake_boilerplate.am
index 1d4d60cb9..75f8ccb1f 100644
--- a/lib/libvmod_vtc/automake_boilerplate.am
+++ b/lib/libvmod_vtc/automake_boilerplate.am
@@ -15,7 +15,7 @@ vmod_LTLIBRARIES = libvmod_vtc.la
 libvmod_vtc_la_CFLAGS ?= \
 	@SAN_CFLAGS@
 
-libvmod_vtc_la_LDFLAGS ?= \
+libvmod_vtc_la_LDFLAGS = \
 	-export-symbols-regex 'Vmod_vtc_Data' \
 	$(AM_LDFLAGS) \
 	$(VMOD_LDFLAGS) \

From fgsch at lodoss.net  Mon Aug  3 14:38:07 2020
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon,  3 Aug 2020 14:38:07 +0000 (UTC)
Subject: [master] 5e95f6079 Bump image to hopefully speed things up
Message-ID: <20200803143808.08DF8119A41@lists.varnish-cache.org>


commit 5e95f6079f3fdb903d797c5a0f4a73dda5aafe60
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Aug 3 15:18:13 2020 +0100

    Bump image to hopefully speed things up

diff --git a/.travis.yml b/.travis.yml
index af13775e2..93365e41a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -75,7 +75,7 @@ jobs:
         - ./configure --enable-maintainer-mode --with-unwind --enable-developer-warnings --enable-debugging-symbols --disable-stack-protector --with-persistent-storage --enable-asan --enable-ubsan
     - stage: test
       os: osx
-      osx_image: xcode11.4
+      osx_image: xcode12
       compiler: clang
       addons:
         homebrew:

From fgsch at lodoss.net  Mon Aug  3 15:31:07 2020
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon,  3 Aug 2020 15:31:07 +0000 (UTC)
Subject: [master] 6c3ce87fc Don't need to update brew anymore
Message-ID: <20200803153107.7993F11BB1F@lists.varnish-cache.org>


commit 6c3ce87fc182c6bde3a6ca30c7fcf58b0cf58504
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Aug 3 16:04:17 2020 +0100

    Don't need to update brew anymore

diff --git a/.travis.yml b/.travis.yml
index 93365e41a..cf12dcde0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -83,7 +83,6 @@ jobs:
             - docutils
             - nghttp2
             - sphinx-doc
-          update: true
       before_script:
         - export PATH="/usr/local/opt/sphinx-doc/bin:$PATH"
         - ./autogen.sh

From nils.goroll at uplex.de  Tue Aug  4 05:12:03 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  4 Aug 2020 05:12:03 +0000 (UTC)
Subject: [master] 6128783f8 binary heap: use common miniobj macros
Message-ID: <20200804051203.C2512108621@lists.varnish-cache.org>


commit 6128783f8c2fc2fcd5bcdbba9b86104ae03232a6
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jul 28 14:58:51 2020 +0200

    binary heap: use common miniobj macros

diff --git a/lib/libvarnish/binary_heap.c b/lib/libvarnish/binary_heap.c
index f77b63d9c..7d1f0e50b 100644
--- a/lib/libvarnish/binary_heap.c
+++ b/lib/libvarnish/binary_heap.c
@@ -42,6 +42,7 @@
 #include <stdlib.h>
 #include <unistd.h>
 
+#include "miniobj.h"
 #include "vdef.h"
 #include "vas.h"
 #include "binary_heap.h"
@@ -214,7 +215,7 @@ binheap_new(void *priv, binheap_cmp_t *cmp_f, binheap_update_t *update_f)
 	struct binheap *bh;
 	unsigned u;
 
-	bh = calloc(1, sizeof *bh);
+	ALLOC_OBJ(bh, BINHEAP_MAGIC);
 	if (bh == NULL)
 		return (bh);
 	bh->priv = priv;
@@ -246,8 +247,7 @@ binheap_new(void *priv, binheap_cmp_t *cmp_f, binheap_update_t *update_f)
 static void
 binheap_update(const struct binheap *bh, unsigned u)
 {
-	assert(bh != NULL);
-	assert(bh->magic == BINHEAP_MAGIC);
+	CHECK_OBJ_NOTNULL(bh, BINHEAP_MAGIC);
 	assert(u < bh->next);
 	assert(A(bh, u) != NULL);
 	if (bh->update != NULL)
@@ -259,8 +259,7 @@ binhead_swap(const struct binheap *bh, unsigned u, unsigned v)
 {
 	void *p;
 
-	assert(bh != NULL);
-	assert(bh->magic == BINHEAP_MAGIC);
+	CHECK_OBJ_NOTNULL(bh, BINHEAP_MAGIC);
 	assert(u < bh->next);
 	assert(A(bh, u) != NULL);
 	assert(v < bh->next);
@@ -277,7 +276,7 @@ binheap_trickleup(const struct binheap *bh, unsigned u)
 {
 	unsigned v;
 
-	assert(bh != NULL); assert(bh->magic == BINHEAP_MAGIC);
+	CHECK_OBJ_NOTNULL(bh, BINHEAP_MAGIC);
 	assert(u < bh->next);
 	assert(A(bh, u) != NULL);
 
@@ -301,8 +300,7 @@ binheap_trickledown(const struct binheap *bh, unsigned u)
 {
 	unsigned v1, v2;
 
-	assert(bh != NULL);
-	assert(bh->magic == BINHEAP_MAGIC);
+	CHECK_OBJ_NOTNULL(bh, BINHEAP_MAGIC);
 	assert(u < bh->next);
 	assert(A(bh, u) != NULL);
 
@@ -337,8 +335,7 @@ binheap_insert(struct binheap *bh, void *p)
 {
 	unsigned u;
 
-	assert(bh != NULL);
-	assert(bh->magic == BINHEAP_MAGIC);
+	CHECK_OBJ_NOTNULL(bh, BINHEAP_MAGIC);
 	assert(bh->length >= bh->next);
 	if (bh->length == bh->next)
 		binheap_addrow(bh);
@@ -369,8 +366,7 @@ void *
 binheap_root(const struct binheap *bh)
 {
 
-	assert(bh != NULL);
-	assert(bh->magic == BINHEAP_MAGIC);
+	CHECK_OBJ_NOTNULL(bh, BINHEAP_MAGIC);
 #ifdef PARANOIA
 	chk(bh);
 #endif
@@ -405,8 +401,7 @@ void
 binheap_delete(struct binheap *bh, unsigned idx)
 {
 
-	assert(bh != NULL);
-	assert(bh->magic == BINHEAP_MAGIC);
+	CHECK_OBJ_NOTNULL(bh, BINHEAP_MAGIC);
 	assert(bh->next > ROOT_IDX);
 	assert(idx < bh->next);
 	assert(idx > 0);
@@ -448,8 +443,7 @@ void
 binheap_reorder(const struct binheap *bh, unsigned idx)
 {
 
-	assert(bh != NULL);
-	assert(bh->magic == BINHEAP_MAGIC);
+	CHECK_OBJ_NOTNULL(bh, BINHEAP_MAGIC);
 	assert(bh->next > ROOT_IDX);
 	assert(idx < bh->next);
 	assert(idx > 0);
@@ -470,7 +464,6 @@ binheap_reorder(const struct binheap *bh, unsigned idx)
 #include <string.h>
 
 #include "vrnd.h"
-#include "miniobj.h"
 
 /* Test driver -------------------------------------------------------*/
 

From nils.goroll at uplex.de  Tue Aug  4 05:12:03 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  4 Aug 2020 05:12:03 +0000 (UTC)
Subject: [master] ccbb958d6 binary heap: add a destructor and test it
Message-ID: <20200804051203.D69F5108626@lists.varnish-cache.org>


commit ccbb958d6f057315233e84a420d15155988bab81
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jul 28 15:00:13 2020 +0200

    binary heap: add a destructor and test it

diff --git a/include/binary_heap.h b/include/binary_heap.h
index da2b27133..c9e39593c 100644
--- a/include/binary_heap.h
+++ b/include/binary_heap.h
@@ -58,6 +58,11 @@ struct binheap *binheap_new(void *priv, binheap_cmp_t, binheap_update_t);
 	 * 'priv' is passed to cmp and update functions.
 	 */
 
+void binheap_destroy(struct binheap **);
+	/*
+	 * Destroy an empty Binary tree
+	 */
+
 void binheap_insert(struct binheap *, void *);
 	/*
 	 * Insert an item
diff --git a/lib/libvarnish/binary_heap.c b/lib/libvarnish/binary_heap.c
index 7d1f0e50b..f31093f98 100644
--- a/lib/libvarnish/binary_heap.c
+++ b/lib/libvarnish/binary_heap.c
@@ -244,6 +244,21 @@ binheap_new(void *priv, binheap_cmp_t *cmp_f, binheap_update_t *update_f)
 	return (bh);
 }
 
+void
+binheap_destroy(struct binheap **bhp)
+{
+	struct binheap *bh;
+	unsigned u;
+
+	TAKE_OBJ_NOTNULL(bh, bhp, BINHEAP_MAGIC);
+	AZ(binheap_root(bh));
+
+	for (u = 0; u < bh->length; u += ROW_WIDTH)
+		free(ROW(bh, u));
+	free(bh->array);
+	free(bh);
+}
+
 static void
 binheap_update(const struct binheap *bh, unsigned u)
 {
@@ -634,6 +649,10 @@ main(void)
 		}
 		fprintf(stderr, "%d updates OK\n", M);
 	}
+	while ((fp = binheap_root(bh)) != NULL)
+		binheap_delete(bh, fp->idx);
+	binheap_destroy(&bh);
+	AZ(bh);
 	return (0);
 }
 #endif

From nils.goroll at uplex.de  Tue Aug  4 05:12:03 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  4 Aug 2020 05:12:03 +0000 (UTC)
Subject: [master] ab70a5d22 install binary_heap.h for use with vmods
Message-ID: <20200804051204.0063F108629@lists.varnish-cache.org>


commit ab70a5d222fe602e6db5e7b99e103cf847c5bdef
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jul 28 15:01:20 2020 +0200

    install binary_heap.h for use with vmods

diff --git a/include/Makefile.am b/include/Makefile.am
index 6172c657e..c344f593c 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -58,6 +58,7 @@ nobase_pkginclude_HEADERS = \
 
 # Headers for use with vmods
 nobase_pkginclude_HEADERS += \
+	binary_heap.h \
 	miniobj.h \
 	vas.h \
 	vav.h \
@@ -81,7 +82,6 @@ nobase_pkginclude_HEADERS += \
 
 # Private headers
 nobase_noinst_HEADERS = \
-	binary_heap.h \
 	compat/daemon.h \
 	vfl.h \
 	libvcc.h \

From nils.goroll at uplex.de  Tue Aug  4 05:12:04 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  4 Aug 2020 05:12:04 +0000 (UTC)
Subject: [master] c676ca46d additional cleanup for the binheap test
Message-ID: <20200804051204.187E010862E@lists.varnish-cache.org>


commit c676ca46d19ca31aad3e9c5de991516ad04822ac
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jul 29 09:52:54 2020 +0200

    additional cleanup for the binheap test
    
    by Dridi, thank you

diff --git a/lib/libvarnish/binary_heap.c b/lib/libvarnish/binary_heap.c
index f31093f98..0896ed14c 100644
--- a/lib/libvarnish/binary_heap.c
+++ b/lib/libvarnish/binary_heap.c
@@ -538,14 +538,6 @@ vrnd_lock(void)
 {
 }
 
-#if defined(__SANITIZER) || __has_feature(address_sanitizer)
-int __lsan_is_turned_off(void);
-int __lsan_is_turned_off(void)
-{
-	return (1);
-}
-
-#endif
 int
 main(void)
 {
@@ -649,8 +641,10 @@ main(void)
 		}
 		fprintf(stderr, "%d updates OK\n", M);
 	}
-	while ((fp = binheap_root(bh)) != NULL)
+	while ((fp = binheap_root(bh)) != NULL) {
 		binheap_delete(bh, fp->idx);
+		FREE_OBJ(fp);
+	}
 	binheap_destroy(&bh);
 	AZ(bh);
 	return (0);

From nils.goroll at uplex.de  Tue Aug  4 05:12:04 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  4 Aug 2020 05:12:04 +0000 (UTC)
Subject: [master] a016850fb rename binheap / binary_heap -> vbh
Message-ID: <20200804051204.35704108632@lists.varnish-cache.org>


commit a016850fbb8fe32c1f2f326045646126ea4154af
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Aug 4 07:09:55 2020 +0200

    rename binheap / binary_heap -> vbh
    
    with the public api functions called VBH_

diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c
index 4db99c2c1..51fd00754 100644
--- a/bin/varnishd/cache/cache_backend_probe.c
+++ b/bin/varnishd/cache/cache_backend_probe.c
@@ -46,7 +46,7 @@
 
 #include "cache_varnishd.h"
 
-#include "binary_heap.h"
+#include "vbh.h"
 #include "vcli_serve.h"
 #include "vsa.h"
 #include "vtcp.h"
@@ -89,7 +89,7 @@ struct vbp_target {
 
 static struct lock			vbp_mtx;
 static pthread_cond_t			vbp_cond;
-static struct binheap			*vbp_heap;
+static struct vbh			*vbp_heap;
 
 static const unsigned char vbp_proxy_local[] = {
 	0x0d, 0x0a, 0x0d, 0x0a, 0x00, 0x0d, 0x0a, 0x51,
@@ -417,8 +417,8 @@ static void
 vbp_heap_insert(struct vbp_target *vt)
 {
 	// Lck_AssertHeld(&vbp_mtx);
-	binheap_insert(vbp_heap, vt);
-	if (binheap_root(vbp_heap) == vt)
+	VBH_insert(vbp_heap, vt);
+	if (VBH_root(vbp_heap) == vt)
 		AZ(pthread_cond_signal(&vbp_cond));
 }
 
@@ -444,13 +444,13 @@ vbp_task(struct worker *wrk, void *priv)
 
 	Lck_Lock(&vbp_mtx);
 	if (vt->running < 0) {
-		assert(vt->heap_idx == BINHEAP_NOIDX);
+		assert(vt->heap_idx == VBH_NOIDX);
 		vbp_delete(vt);
 	} else {
 		vt->running = 0;
-		if (vt->heap_idx != BINHEAP_NOIDX) {
+		if (vt->heap_idx != VBH_NOIDX) {
 			vt->due = VTIM_real() + vt->interval;
-			binheap_delete(vbp_heap, vt->heap_idx);
+			VBH_delete(vbp_heap, vt->heap_idx);
 			vbp_heap_insert(vt);
 		}
 	}
@@ -472,7 +472,7 @@ vbp_thread(struct worker *wrk, void *priv)
 	Lck_Lock(&vbp_mtx);
 	while (1) {
 		now = VTIM_real();
-		vt = binheap_root(vbp_heap);
+		vt = VBH_root(vbp_heap);
 		if (vt == NULL) {
 			nxt = 8.192 + now;
 			(void)Lck_CondWait(&vbp_cond, &vbp_mtx, nxt);
@@ -481,7 +481,7 @@ vbp_thread(struct worker *wrk, void *priv)
 			vt = NULL;
 			(void)Lck_CondWait(&vbp_cond, &vbp_mtx, nxt);
 		} else {
-			binheap_delete(vbp_heap, vt->heap_idx);
+			VBH_delete(vbp_heap, vt->heap_idx);
 			vt->due = now + vt->interval;
 			if (!vt->running) {
 				vt->running = 1;
@@ -493,7 +493,7 @@ vbp_thread(struct worker *wrk, void *priv)
 				if (r)
 					vt->running = 0;
 			}
-			binheap_insert(vbp_heap, vt);
+			VBH_insert(vbp_heap, vt);
 		}
 	}
 	NEEDLESS(Lck_Unlock(&vbp_mtx));
@@ -654,12 +654,12 @@ VBP_Control(const struct backend *be, int enable)
 
 	Lck_Lock(&vbp_mtx);
 	if (enable) {
-		assert(vt->heap_idx == BINHEAP_NOIDX);
+		assert(vt->heap_idx == VBH_NOIDX);
 		vt->due = VTIM_real();
 		vbp_heap_insert(vt);
 	} else {
-		assert(vt->heap_idx != BINHEAP_NOIDX);
-		binheap_delete(vbp_heap, vt->heap_idx);
+		assert(vt->heap_idx != VBH_NOIDX);
+		VBH_delete(vbp_heap, vt->heap_idx);
 	}
 	Lck_Unlock(&vbp_mtx);
 }
@@ -713,14 +713,14 @@ VBP_Remove(struct backend *be)
 	}
 	Lck_Unlock(&vbp_mtx);
 	if (vt != NULL) {
-		assert(vt->heap_idx == BINHEAP_NOIDX);
+		assert(vt->heap_idx == VBH_NOIDX);
 		vbp_delete(vt);
 	}
 }
 
 /*-------------------------------------------------------------------*/
 
-static int v_matchproto_(binheap_cmp_t)
+static int v_matchproto_(vbh_cmp_t)
 vbp_cmp(void *priv, const void *a, const void *b)
 {
 	const struct vbp_target *aa, *bb;
@@ -739,7 +739,7 @@ vbp_cmp(void *priv, const void *a, const void *b)
 	return (aa->due < bb->due);
 }
 
-static void v_matchproto_(binheap_update_t)
+static void v_matchproto_(vbh_update_t)
 vbp_update(void *priv, void *p, unsigned u)
 {
 	struct vbp_target *vt;
@@ -757,7 +757,7 @@ VBP_Init(void)
 	pthread_t thr;
 
 	Lck_New(&vbp_mtx, lck_probe);
-	vbp_heap = binheap_new(NULL, vbp_cmp, vbp_update);
+	vbp_heap = VBH_new(NULL, vbp_cmp, vbp_update);
 	AN(vbp_heap);
 	AZ(pthread_cond_init(&vbp_cond, NULL));
 	WRK_BgThread(&thr, "backend-poller", vbp_thread, NULL);
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index ac3594629..85ee9ee1f 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -39,7 +39,7 @@
 #include "cache_varnishd.h"
 #include "cache_objhead.h"
 
-#include "binary_heap.h"
+#include "vbh.h"
 #include "vtim.h"
 
 struct exp_priv {
@@ -53,7 +53,7 @@ struct exp_priv {
 	/* owned by exp thread */
 	struct worker			*wrk;
 	struct vsl_log			vsl;
-	struct binheap			*heap;
+	struct vbh			*heap;
 	pthread_t			thread;
 };
 
@@ -272,10 +272,10 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, unsigned flags)
 
 	if (flags & OC_EF_REMOVE) {
 		if (!(flags & OC_EF_INSERT)) {
-			assert(oc->timer_idx != BINHEAP_NOIDX);
-			binheap_delete(ep->heap, oc->timer_idx);
+			assert(oc->timer_idx != VBH_NOIDX);
+			VBH_delete(ep->heap, oc->timer_idx);
 		}
-		assert(oc->timer_idx == BINHEAP_NOIDX);
+		assert(oc->timer_idx == VBH_NOIDX);
 		assert(oc->refcnt > 0);
 		AZ(oc->exp_flags);
 		ObjSendEvent(ep->wrk, oc, OEV_EXPIRE);
@@ -298,13 +298,13 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, unsigned flags)
 	 */
 
 	if (flags & OC_EF_INSERT) {
-		assert(oc->timer_idx == BINHEAP_NOIDX);
-		binheap_insert(exphdl->heap, oc);
-		assert(oc->timer_idx != BINHEAP_NOIDX);
+		assert(oc->timer_idx == VBH_NOIDX);
+		VBH_insert(exphdl->heap, oc);
+		assert(oc->timer_idx != VBH_NOIDX);
 	} else if (flags & OC_EF_MOVE) {
-		assert(oc->timer_idx != BINHEAP_NOIDX);
-		binheap_reorder(exphdl->heap, oc->timer_idx);
-		assert(oc->timer_idx != BINHEAP_NOIDX);
+		assert(oc->timer_idx != VBH_NOIDX);
+		VBH_reorder(exphdl->heap, oc->timer_idx);
+		assert(oc->timer_idx != VBH_NOIDX);
 	} else {
 		WRONG("Objcore state wrong in inbox");
 	}
@@ -321,7 +321,7 @@ exp_expire(struct exp_priv *ep, vtim_real now)
 
 	CHECK_OBJ_NOTNULL(ep, EXP_PRIV_MAGIC);
 
-	oc = binheap_root(ep->heap);
+	oc = VBH_root(ep->heap);
 	if (oc == NULL)
 		return (now + 355./113.);
 	VSLb(&ep->vsl, SLT_ExpKill, "EXP_expire p=%p e=%.6f f=0x%x", oc,
@@ -348,9 +348,9 @@ exp_expire(struct exp_priv *ep, vtim_real now)
 			HSH_Kill(oc);
 
 		/* Remove from binheap */
-		assert(oc->timer_idx != BINHEAP_NOIDX);
-		binheap_delete(ep->heap, oc->timer_idx);
-		assert(oc->timer_idx == BINHEAP_NOIDX);
+		assert(oc->timer_idx != VBH_NOIDX);
+		VBH_delete(ep->heap, oc->timer_idx);
+		assert(oc->timer_idx == VBH_NOIDX);
 
 		CHECK_OBJ_NOTNULL(oc->objhead, OBJHEAD_MAGIC);
 		VSLb(&ep->vsl, SLT_ExpKill, "EXP_Expired x=%u t=%.0f",
@@ -366,7 +366,7 @@ exp_expire(struct exp_priv *ep, vtim_real now)
  * object expires, accounting also for graceability, it is killed.
  */
 
-static int v_matchproto_(binheap_cmp_t)
+static int v_matchproto_(vbh_cmp_t)
 object_cmp(void *priv, const void *a, const void *b)
 {
 	const struct objcore *aa, *bb;
@@ -377,7 +377,7 @@ object_cmp(void *priv, const void *a, const void *b)
 	return (aa->timer_when < bb->timer_when);
 }
 
-static void v_matchproto_(binheap_update_t)
+static void v_matchproto_(vbh_update_t)
 object_update(void *priv, void *p, unsigned u)
 {
 	struct objcore *oc;
@@ -398,7 +398,7 @@ exp_thread(struct worker *wrk, void *priv)
 	CAST_OBJ_NOTNULL(ep, priv, EXP_PRIV_MAGIC);
 	ep->wrk = wrk;
 	VSL_Setup(&ep->vsl, NULL, 0);
-	ep->heap = binheap_new(NULL, object_cmp, object_update);
+	ep->heap = VBH_new(NULL, object_cmp, object_update);
 	AN(ep->heap);
 	while (exp_shutdown == 0) {
 
diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt
index 8d1867f6c..3caf03b01 100644
--- a/bin/varnishd/flint.lnt
+++ b/bin/varnishd/flint.lnt
@@ -151,7 +151,7 @@
 -e441	//  for clause irregularity: loop variable '___' not found in 2nd for expression
 
 // from libvarnish
---emacro((835),BINHEAP_NOIDX)
+--emacro((835),VBH_NOIDX)
 --emacro((835),O_CLOEXEC)
 
 // Review all below this line ///////////////////////////////////////////////
diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c
index 79de2cdef..0957f6ca8 100644
--- a/bin/varnishd/waiter/cache_waiter.c
+++ b/bin/varnishd/waiter/cache_waiter.c
@@ -36,13 +36,13 @@
 
 #include <stdlib.h>
 
-#include "binary_heap.h"
+#include "vbh.h"
 
 #include "waiter/waiter.h"
 #include "waiter/waiter_priv.h"
 #include "waiter/mgt_waiter.h"
 
-static int v_matchproto_(binheap_cmp_t)
+static int v_matchproto_(vbh_cmp_t)
 waited_cmp(void *priv, const void *a, const void *b)
 {
 	const struct waiter *ww;
@@ -55,7 +55,7 @@ waited_cmp(void *priv, const void *a, const void *b)
 	return (Wait_When(aa) < Wait_When(bb));
 }
 
-static void v_matchproto_(binheap_update_t)
+static void v_matchproto_(vbh_update_t)
 waited_update(void *priv, void *p, unsigned u)
 {
 	struct waited *pp;
@@ -74,7 +74,7 @@ Wait_Call(const struct waiter *w, struct waited *wp,
 	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
 	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
 	AN(wp->func);
-	assert(wp->idx == BINHEAP_NOIDX);
+	assert(wp->idx == VBH_NOIDX);
 	wp->func(wp, ev, now);
 }
 
@@ -85,8 +85,8 @@ Wait_HeapInsert(const struct waiter *w, struct waited *wp)
 {
 	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
 	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
-	assert(wp->idx == BINHEAP_NOIDX);
-	binheap_insert(w->heap, wp);
+	assert(wp->idx == VBH_NOIDX);
+	VBH_insert(w->heap, wp);
 }
 
 /*
@@ -102,9 +102,9 @@ Wait_HeapDelete(const struct waiter *w, const struct waited *wp)
 {
 	CHECK_OBJ_NOTNULL(w, WAITER_MAGIC);
 	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
-	if (wp->idx == BINHEAP_NOIDX)
+	if (wp->idx == VBH_NOIDX)
 		return (0);
-	binheap_delete(w->heap, wp->idx);
+	VBH_delete(w->heap, wp->idx);
 	return (1);
 }
 
@@ -113,7 +113,7 @@ Wait_HeapDue(const struct waiter *w, struct waited **wpp)
 {
 	struct waited *wp;
 
-	wp = binheap_root(w->heap);
+	wp = VBH_root(w->heap);
 	CHECK_OBJ_ORNULL(wp, WAITED_MAGIC);
 	if (wp == NULL) {
 		if (wpp != NULL)
@@ -135,7 +135,7 @@ Wait_Enter(const struct waiter *w, struct waited *wp)
 	CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC);
 	assert(wp->fd > 0);			// stdin never comes here
 	AN(wp->func);
-	wp->idx = BINHEAP_NOIDX;
+	wp->idx = VBH_NOIDX;
 	return (w->impl->enter(w->priv, wp));
 }
 
@@ -168,7 +168,7 @@ Waiter_New(void)
 	w->priv = (void*)(w + 1);
 	w->impl = waiter;
 	VTAILQ_INIT(&w->waithead);
-	w->heap = binheap_new(w, waited_cmp, waited_update);
+	w->heap = VBH_new(w, waited_cmp, waited_update);
 
 	waiter->init(w);
 
@@ -182,7 +182,7 @@ Waiter_Destroy(struct waiter **wp)
 
 	TAKE_OBJ_NOTNULL(w, wp, WAITER_MAGIC);
 
-	AZ(binheap_root(w->heap));
+	AZ(VBH_root(w->heap));
 	AN(w->impl->fini);
 	w->impl->fini(w);
 	FREE_OBJ(w);
diff --git a/bin/varnishd/waiter/waiter_priv.h b/bin/varnishd/waiter/waiter_priv.h
index ce8f93b4e..e44c548b9 100644
--- a/bin/varnishd/waiter/waiter_priv.h
+++ b/bin/varnishd/waiter/waiter_priv.h
@@ -32,7 +32,7 @@
  */
 
 struct waited;
-struct binheap;
+struct vbh;
 
 struct waiter {
 	unsigned			magic;
@@ -42,7 +42,7 @@ struct waiter {
 	VTAILQ_HEAD(,waited)		waithead;
 
 	void				*priv;
-	struct binheap			*heap;
+	struct vbh			*heap;
 };
 
 typedef void waiter_init_f(struct waiter *);
diff --git a/include/Makefile.am b/include/Makefile.am
index c344f593c..628993bf0 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -58,7 +58,7 @@ nobase_pkginclude_HEADERS = \
 
 # Headers for use with vmods
 nobase_pkginclude_HEADERS += \
-	binary_heap.h \
+	vbh.h \
 	miniobj.h \
 	vas.h \
 	vav.h \
diff --git a/include/binary_heap.h b/include/vbh.h
similarity index 82%
rename from include/binary_heap.h
rename to include/vbh.h
index c9e39593c..2ac0f8565 100644
--- a/include/binary_heap.h
+++ b/include/vbh.h
@@ -35,16 +35,16 @@
 
 /* Public Interface --------------------------------------------------*/
 
-struct binheap;
+struct vbh;
 
-typedef int binheap_cmp_t(void *priv, const void *a, const void *b);
+typedef int vbh_cmp_t(void *priv, const void *a, const void *b);
 	/*
 	 * Comparison function.
 	 * Should return true if item 'a' should be closer to the root
 	 * than item 'b'
 	 */
 
-typedef void binheap_update_t(void *priv, void *a, unsigned newidx);
+typedef void vbh_update_t(void *priv, void *a, unsigned newidx);
 	/*
 	 * Update function (optional)
 	 * When items move in the tree, this function gets called to
@@ -52,36 +52,36 @@ typedef void binheap_update_t(void *priv, void *a, unsigned newidx);
 	 * Only needed if deleting non-root items.
 	 */
 
-struct binheap *binheap_new(void *priv, binheap_cmp_t, binheap_update_t);
+struct vbh *VBH_new(void *priv, vbh_cmp_t, vbh_update_t);
 	/*
 	 * Create Binary tree
 	 * 'priv' is passed to cmp and update functions.
 	 */
 
-void binheap_destroy(struct binheap **);
+void VBH_destroy(struct vbh **);
 	/*
 	 * Destroy an empty Binary tree
 	 */
 
-void binheap_insert(struct binheap *, void *);
+void VBH_insert(struct vbh *, void *);
 	/*
 	 * Insert an item
 	 */
 
-void binheap_reorder(const struct binheap *, unsigned idx);
+void VBH_reorder(const struct vbh *, unsigned idx);
 	/*
 	 * Move an order after changing its key value.
 	 */
 
-void binheap_delete(struct binheap *, unsigned idx);
+void VBH_delete(struct vbh *, unsigned idx);
 	/*
 	 * Delete an item
 	 * The root item has 'idx' zero
 	 */
 
-void *binheap_root(const struct binheap *);
+void *VBH_root(const struct vbh *);
 	/*
 	 * Return the root item
 	 */
 
-#define BINHEAP_NOIDX	0
+#define VBH_NOIDX	0
diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am
index 298d63400..4318d5734 100644
--- a/lib/libvarnish/Makefile.am
+++ b/lib/libvarnish/Makefile.am
@@ -15,7 +15,7 @@ libvarnish_la_CFLAGS = \
 	$(AM_CFLAGS)
 
 libvarnish_la_SOURCES = \
-	binary_heap.c \
+	vbh.c \
 	vas.c \
 	vav.c \
 	vcli_proto.c \
@@ -44,13 +44,13 @@ libvarnish_la_SOURCES = \
 	vtim.c \
 	vus.c
 
-TESTS = vjsn_test vnum_c_test binheap vsb_test
+TESTS = vjsn_test vnum_c_test vbh_test vsb_test
 
 noinst_PROGRAMS = ${TESTS}
 
-binheap_SOURCES	= binary_heap.c
-binheap_CFLAGS = $(AM_CFLAGS) -DTEST_DRIVER
-binheap_LDADD = $(AM_LDFLAGS) libvarnish.la
+vbh_test_SOURCES	= vbh.c
+vbh_test_CFLAGS = $(AM_CFLAGS) -DTEST_DRIVER
+vbh_test_LDADD = $(AM_LDFLAGS) libvarnish.la
 
 vnum_c_test_SOURCES = vnum.c
 vnum_c_test_CFLAGS = $(AM_CFLAGS) -DNUM_C_TEST
diff --git a/lib/libvarnish/flint.lnt b/lib/libvarnish/flint.lnt
index 775226510..eaf3e61b5 100644
--- a/lib/libvarnish/flint.lnt
+++ b/lib/libvarnish/flint.lnt
@@ -8,5 +8,5 @@
 
 -dVARNISH_STATE_DIR="foo"
 
---emacro((835),BINHEAP_NOIDX)
+--emacro((835),VBH_NOIDX)
 --emacro((835),O_CLOEXEC)
diff --git a/lib/libvarnish/binary_heap.c b/lib/libvarnish/vbh.c
similarity index 83%
rename from lib/libvarnish/binary_heap.c
rename to lib/libvarnish/vbh.c
index 0896ed14c..b8417eb87 100644
--- a/lib/libvarnish/binary_heap.c
+++ b/lib/libvarnish/vbh.c
@@ -45,7 +45,7 @@
 #include "miniobj.h"
 #include "vdef.h"
 #include "vas.h"
-#include "binary_heap.h"
+#include "vbh.h"
 
 #if !defined(__has_feature)
 #define __has_feature(x)	0
@@ -82,12 +82,12 @@
 /*lint -emacro(835, A) 0 left of & */
 #define A(b, n)			ROW(b, n)[(n) & (ROW_WIDTH - 1)]
 
-struct binheap {
+struct vbh {
 	unsigned		magic;
-#define BINHEAP_MAGIC		0xf581581aU	/* from /dev/random */
+#define VBH_MAGIC		0xf581581aU	/* from /dev/random */
 	void			*priv;
-	binheap_cmp_t		*cmp;
-	binheap_update_t	*update;
+	vbh_cmp_t		*cmp;
+	vbh_update_t		*update;
 	void			***array;
 	unsigned		rows;
 	unsigned		length;
@@ -102,7 +102,7 @@ struct binheap {
 #ifdef VM_AWARE
 
 static  unsigned
-parent(const struct binheap *bh, unsigned u)
+parent(const struct vbh *bh, unsigned u)
 {
 	unsigned po;
 	unsigned v;
@@ -123,7 +123,7 @@ parent(const struct binheap *bh, unsigned u)
 }
 
 static void
-child(const struct binheap *bh, unsigned u, unsigned *a, unsigned *b)
+child(const struct vbh *bh, unsigned u, unsigned *a, unsigned *b)
 {
 	uintmax_t uu;
 
@@ -168,7 +168,7 @@ child(const struct binheap *bh, unsigned u, unsigned *a, unsigned *b)
 #else
 
 static unsigned
-parent(const struct binheap *bh, unsigned u)
+parent(const struct vbh *bh, unsigned u)
 {
 
 	(void)bh;
@@ -176,7 +176,7 @@ parent(const struct binheap *bh, unsigned u)
 }
 
 static void
-child(const struct binheap *bh, unsigned u, unsigned *a, unsigned *b)
+child(const struct vbh *bh, unsigned u, unsigned *a, unsigned *b)
 {
 
 	(void)bh;
@@ -189,7 +189,7 @@ child(const struct binheap *bh, unsigned u, unsigned *a, unsigned *b)
 /* Implementation ----------------------------------------------------*/
 
 static void
-binheap_addrow(struct binheap *bh)
+vbh_addrow(struct vbh *bh)
 {
 	unsigned u;
 
@@ -209,13 +209,13 @@ binheap_addrow(struct binheap *bh)
 	bh->length += ROW_WIDTH;
 }
 
-struct binheap *
-binheap_new(void *priv, binheap_cmp_t *cmp_f, binheap_update_t *update_f)
+struct vbh *
+VBH_new(void *priv, vbh_cmp_t *cmp_f, vbh_update_t *update_f)
 {
-	struct binheap *bh;
+	struct vbh *bh;
 	unsigned u;
 
-	ALLOC_OBJ(bh, BINHEAP_MAGIC);
+	ALLOC_OBJ(bh, VBH_MAGIC);
 	if (bh == NULL)
 		return (bh);
 	bh->priv = priv;
@@ -238,20 +238,20 @@ binheap_new(void *priv, binheap_cmp_t *cmp_f, binheap_update_t *update_f)
 #endif
 	bh->array = calloc(bh->rows, sizeof *bh->array);
 	assert(bh->array != NULL);
-	binheap_addrow(bh);
+	vbh_addrow(bh);
 	A(bh, ROOT_IDX) = NULL;
-	bh->magic = BINHEAP_MAGIC;
+	bh->magic = VBH_MAGIC;
 	return (bh);
 }
 
 void
-binheap_destroy(struct binheap **bhp)
+VBH_destroy(struct vbh **bhp)
 {
-	struct binheap *bh;
+	struct vbh *bh;
 	unsigned u;
 
-	TAKE_OBJ_NOTNULL(bh, bhp, BINHEAP_MAGIC);
-	AZ(binheap_root(bh));
+	TAKE_OBJ_NOTNULL(bh, bhp, VBH_MAGIC);
+	AZ(VBH_root(bh));
 
 	for (u = 0; u < bh->length; u += ROW_WIDTH)
 		free(ROW(bh, u));
@@ -260,9 +260,9 @@ binheap_destroy(struct binheap **bhp)
 }
 
 static void
-binheap_update(const struct binheap *bh, unsigned u)
+vbh_update(const struct vbh *bh, unsigned u)
 {
-	CHECK_OBJ_NOTNULL(bh, BINHEAP_MAGIC);
+	CHECK_OBJ_NOTNULL(bh, VBH_MAGIC);
 	assert(u < bh->next);
 	assert(A(bh, u) != NULL);
 	if (bh->update != NULL)
@@ -270,11 +270,11 @@ binheap_update(const struct binheap *bh, unsigned u)
 }
 
 static void
-binhead_swap(const struct binheap *bh, unsigned u, unsigned v)
+binhead_swap(const struct vbh *bh, unsigned u, unsigned v)
 {
 	void *p;
 
-	CHECK_OBJ_NOTNULL(bh, BINHEAP_MAGIC);
+	CHECK_OBJ_NOTNULL(bh, VBH_MAGIC);
 	assert(u < bh->next);
 	assert(A(bh, u) != NULL);
 	assert(v < bh->next);
@@ -282,16 +282,16 @@ binhead_swap(const struct binheap *bh, unsigned u, unsigned v)
 	p = A(bh, u);
 	A(bh, u) = A(bh, v);
 	A(bh, v) = p;
-	binheap_update(bh, u);
-	binheap_update(bh, v);
+	vbh_update(bh, u);
+	vbh_update(bh, v);
 }
 
 static unsigned
-binheap_trickleup(const struct binheap *bh, unsigned u)
+vbh_trickleup(const struct vbh *bh, unsigned u)
 {
 	unsigned v;
 
-	CHECK_OBJ_NOTNULL(bh, BINHEAP_MAGIC);
+	CHECK_OBJ_NOTNULL(bh, VBH_MAGIC);
 	assert(u < bh->next);
 	assert(A(bh, u) != NULL);
 
@@ -311,11 +311,11 @@ binheap_trickleup(const struct binheap *bh, unsigned u)
 }
 
 static unsigned
-binheap_trickledown(const struct binheap *bh, unsigned u)
+vbh_trickledown(const struct vbh *bh, unsigned u)
 {
 	unsigned v1, v2;
 
-	CHECK_OBJ_NOTNULL(bh, BINHEAP_MAGIC);
+	CHECK_OBJ_NOTNULL(bh, VBH_MAGIC);
 	assert(u < bh->next);
 	assert(A(bh, u) != NULL);
 
@@ -346,19 +346,19 @@ binheap_trickledown(const struct binheap *bh, unsigned u)
 }
 
 void
-binheap_insert(struct binheap *bh, void *p)
+VBH_insert(struct vbh *bh, void *p)
 {
 	unsigned u;
 
-	CHECK_OBJ_NOTNULL(bh, BINHEAP_MAGIC);
+	CHECK_OBJ_NOTNULL(bh, VBH_MAGIC);
 	assert(bh->length >= bh->next);
 	if (bh->length == bh->next)
-		binheap_addrow(bh);
+		vbh_addrow(bh);
 	assert(bh->length > bh->next);
 	u = bh->next++;
 	A(bh, u) = p;
-	binheap_update(bh, u);
-	(void)binheap_trickleup(bh, u);
+	vbh_update(bh, u);
+	(void)vbh_trickleup(bh, u);
 	assert(u < bh->next);
 	assert(A(bh, u) != NULL);
 }
@@ -366,7 +366,7 @@ binheap_insert(struct binheap *bh, void *p)
 
 #ifdef PARANOIA
 static void
-chk(const struct binheap *bh)
+chk(const struct vbh *bh)
 {
 	unsigned u, v;
 
@@ -378,10 +378,10 @@ chk(const struct binheap *bh)
 #endif
 
 void *
-binheap_root(const struct binheap *bh)
+VBH_root(const struct vbh *bh)
 {
 
-	CHECK_OBJ_NOTNULL(bh, BINHEAP_MAGIC);
+	CHECK_OBJ_NOTNULL(bh, VBH_MAGIC);
 #ifdef PARANOIA
 	chk(bh);
 #endif
@@ -413,27 +413,27 @@ binheap_root(const struct binheap *bh)
  */
 
 void
-binheap_delete(struct binheap *bh, unsigned idx)
+VBH_delete(struct vbh *bh, unsigned idx)
 {
 
-	CHECK_OBJ_NOTNULL(bh, BINHEAP_MAGIC);
+	CHECK_OBJ_NOTNULL(bh, VBH_MAGIC);
 	assert(bh->next > ROOT_IDX);
 	assert(idx < bh->next);
 	assert(idx > 0);
 	assert(A(bh, idx) != NULL);
-	bh->update(bh->priv, A(bh, idx), BINHEAP_NOIDX);
+	bh->update(bh->priv, A(bh, idx), VBH_NOIDX);
 	if (idx == --bh->next) {
 		A(bh, bh->next) = NULL;
 		return;
 	}
 	A(bh, idx) = A(bh, bh->next);
 	A(bh, bh->next) = NULL;
-	binheap_update(bh, idx);
-	idx = binheap_trickleup(bh, idx);
+	vbh_update(bh, idx);
+	idx = vbh_trickleup(bh, idx);
 	assert(idx < bh->next);
 	assert(idx > 0);
 	assert(A(bh, idx) != NULL);
-	idx = binheap_trickledown(bh, idx);
+	idx = vbh_trickledown(bh, idx);
 	assert(idx < bh->next);
 	assert(idx > 0);
 	assert(A(bh, idx) != NULL);
@@ -455,19 +455,19 @@ binheap_delete(struct binheap *bh, unsigned idx)
  */
 
 void
-binheap_reorder(const struct binheap *bh, unsigned idx)
+VBH_reorder(const struct vbh *bh, unsigned idx)
 {
 
-	CHECK_OBJ_NOTNULL(bh, BINHEAP_MAGIC);
+	CHECK_OBJ_NOTNULL(bh, VBH_MAGIC);
 	assert(bh->next > ROOT_IDX);
 	assert(idx < bh->next);
 	assert(idx > 0);
 	assert(A(bh, idx) != NULL);
-	idx = binheap_trickleup(bh, idx);
+	idx = vbh_trickleup(bh, idx);
 	assert(idx < bh->next);
 	assert(idx > 0);
 	assert(A(bh, idx) != NULL);
-	idx = binheap_trickledown(bh, idx);
+	idx = vbh_trickledown(bh, idx);
 	assert(idx < bh->next);
 	assert(idx > 0);
 	assert(A(bh, idx) != NULL);
@@ -496,7 +496,7 @@ struct foo {
 
 struct foo *ff[N];
 
-static int v_matchproto_(binheap_cmp_t)
+static int v_matchproto_(vbh_cmp_t)
 cmp(void *priv, const void *a, const void *b)
 {
 	const struct foo *fa, *fb;
@@ -507,7 +507,7 @@ cmp(void *priv, const void *a, const void *b)
 	return (fa->key < fb->key);
 }
 
-static void v_matchproto_(binheap_update_t)
+static void v_matchproto_(vbh_update_t)
 update(void *priv, void *a, unsigned u)
 {
 	struct foo *fa;
@@ -519,7 +519,7 @@ update(void *priv, void *a, unsigned u)
 
 #ifdef CHECK2
 static void
-chk2(struct binheap *bh)
+chk2(struct vbh *bh)
 {
 	unsigned u, v;
 	struct foo *fa, *fb;
@@ -541,7 +541,7 @@ vrnd_lock(void)
 int
 main(void)
 {
-	struct binheap *bh;
+	struct vbh *bh;
 	unsigned j, u, v, lr, n;
 	struct foo *fp;
 
@@ -550,7 +550,7 @@ main(void)
 	VRND_Lock = vrnd_lock;
 	VRND_Unlock = vrnd_lock;
 
-	bh = binheap_new(NULL, cmp, update);
+	bh = VBH_new(NULL, cmp, update);
 	for (n = 2; n; n += n) {
 		child(bh, n - 1, &u, &v);
 		child(bh, n, &u, &v);
@@ -567,16 +567,16 @@ main(void)
 			assert(ff[u] != NULL);
 			ff[u]->key = lr;
 			ff[u]->n = u;
-			binheap_insert(bh, ff[u]);
+			VBH_insert(bh, ff[u]);
 
-			fp = binheap_root(bh);
+			fp = VBH_root(bh);
 			assert(fp->idx == 1);
 			assert(fp->key <= lr);
 		}
 		fprintf(stderr, "%d inserts OK\n", N);
 		/* For M cycles, pick the root, insert new */
 		for (u = 0; u < M; u++) {
-			fp = binheap_root(bh);
+			fp = VBH_root(bh);
 			CHECK_OBJ_NOTNULL(fp, FOO_MAGIC);
 			assert(fp->idx == 1);
 
@@ -585,7 +585,7 @@ main(void)
 			 * value we added
 			 */
 			assert(fp->key <= lr);
-			binheap_delete(bh, fp->idx);
+			VBH_delete(bh, fp->idx);
 
 			n = fp->n;
 			ALLOC_OBJ(ff[n], FOO_MAGIC);
@@ -596,18 +596,18 @@ main(void)
 
 			lr = VRND_RandomTestable() % R;
 			fp->key = lr;
-			binheap_insert(bh, fp);
+			VBH_insert(bh, fp);
 		}
 		fprintf(stderr, "%d replacements OK\n", M);
 		/* The remove everything */
 		lr = 0;
 		for (u = 0; u < N; u++) {
-			fp = binheap_root(bh);
+			fp = VBH_root(bh);
 			CHECK_OBJ_NOTNULL(fp, FOO_MAGIC);
 			assert(fp->idx == 1);
 			assert(fp->key >= lr);
 			lr = fp->key;
-			binheap_delete(bh, fp->idx);
+			VBH_delete(bh, fp->idx);
 			ff[fp->n] = NULL;
 			FREE_OBJ(fp);
 		}
@@ -619,19 +619,19 @@ main(void)
 				CHECK_OBJ_NOTNULL(ff[v], FOO_MAGIC);
 				AN(ff[v]->idx);
 				if (ff[v]->key & 1) {
-					binheap_delete(bh, ff[v]->idx);
-					assert(ff[v]->idx == BINHEAP_NOIDX);
+					VBH_delete(bh, ff[v]->idx);
+					assert(ff[v]->idx == VBH_NOIDX);
 					FREE_OBJ(ff[v]);
 					ff[v] = NULL;
 				} else {
 					ff[v]->key = VRND_RandomTestable() % R;
-					binheap_reorder(bh, ff[v]->idx);
+					VBH_reorder(bh, ff[v]->idx);
 				}
 			} else {
 				ALLOC_OBJ(ff[v], FOO_MAGIC);
 				assert(ff[v] != NULL);
 				ff[v]->key = VRND_RandomTestable() % R;
-				binheap_insert(bh, ff[v]);
+				VBH_insert(bh, ff[v]);
 				CHECK_OBJ_NOTNULL(ff[v], FOO_MAGIC);
 				AN(ff[v]->idx);
 			}
@@ -641,11 +641,11 @@ main(void)
 		}
 		fprintf(stderr, "%d updates OK\n", M);
 	}
-	while ((fp = binheap_root(bh)) != NULL) {
-		binheap_delete(bh, fp->idx);
+	while ((fp = VBH_root(bh)) != NULL) {
+		VBH_delete(bh, fp->idx);
 		FREE_OBJ(fp);
 	}
-	binheap_destroy(&bh);
+	VBH_destroy(&bh);
 	AZ(bh);
 	return (0);
 }
diff --git a/lib/libvarnish/vev.c b/lib/libvarnish/vev.c
index ac8fd4d4f..f6359d38f 100644
--- a/lib/libvarnish/vev.c
+++ b/lib/libvarnish/vev.c
@@ -43,7 +43,7 @@
 #include "miniobj.h"
 #include "vas.h"
 
-#include "binary_heap.h"
+#include "vbh.h"
 #include "vev.h"
 #include "vtim.h"
 
@@ -71,7 +71,7 @@ struct vev_root {
 	struct vev		**pev;
 	unsigned		npfd;
 	unsigned		lpfd;
-	struct binheap		*binheap;
+	struct vbh		*binheap;
 	unsigned		psig;
 	pthread_t		thread;
 #ifdef DEBUG_EVENTS
@@ -93,7 +93,7 @@ struct vev_root {
 
 /*--------------------------------------------------------------------*/
 
-static void v_matchproto_(binheap_update_t)
+static void v_matchproto_(vbh_update_t)
 vev_bh_update(void *priv, void *a, unsigned u)
 {
 	struct vev_root *evb;
@@ -103,7 +103,7 @@ vev_bh_update(void *priv, void *a, unsigned u)
 	CAST_OBJ_NOTNULL(e, a, VEV_MAGIC);
 	assert(u < evb->lpfd);
 	e->__binheap_idx = u;
-	if (u != BINHEAP_NOIDX) {
+	if (u != VBH_NOIDX) {
 		evb->pev[u] = e;
 		evb->pfd[u].fd = e->fd;
 		evb->pfd[u].events =
@@ -111,7 +111,7 @@ vev_bh_update(void *priv, void *a, unsigned u)
 	}
 }
 
-static int v_matchproto_(binheap_cmp_t)
+static int v_matchproto_(vbh_cmp_t)
 vev_bh_cmp(void *priv, const void *a, const void *b)
 {
 	struct vev_root *evb;
@@ -196,13 +196,13 @@ VEV_New(void)
 	evb = calloc(1, sizeof *evb);
 	if (evb == NULL)
 		return (evb);
-	evb->lpfd = BINHEAP_NOIDX + 1;
+	evb->lpfd = VBH_NOIDX + 1;
 	if (vev_get_pfd(evb)) {
 		free(evb);
 		return (NULL);
 	}
 	evb->magic = VEV_BASE_MAGIC;
-	evb->binheap = binheap_new(evb, vev_bh_cmp, vev_bh_update);
+	evb->binheap = VBH_new(evb, vev_bh_cmp, vev_bh_update);
 	evb->thread = pthread_self();
 #ifdef DEBUG_EVENTS
 	evb->debug = fopen("/tmp/_.events", "w");
@@ -278,7 +278,7 @@ VEV_Start(struct vev_root *evb, struct vev *e)
 		es = NULL;
 	}
 
-	e->magic = VEV_MAGIC;	/* before binheap_insert() */
+	e->magic = VEV_MAGIC;	/* before VBH_insert() */
 
 	if (e->timeout != 0.0)
 		e->__when += VTIM_mono() + e->timeout;
@@ -286,8 +286,8 @@ VEV_Start(struct vev_root *evb, struct vev *e)
 		e->__when = 9e99;
 
 	evb->lpfd++;
-	binheap_insert(evb->binheap, e);
-	assert(e->__binheap_idx != BINHEAP_NOIDX);
+	VBH_insert(evb->binheap, e);
+	assert(e->__binheap_idx != VBH_NOIDX);
 
 	e->__vevb = evb;
 	e->__privflags = 0;
@@ -314,10 +314,10 @@ VEV_Stop(struct vev_root *evb, struct vev *e)
 	assert(evb->thread == pthread_self());
 	assert(evb->pev[e->__binheap_idx] == e);
 
-	assert(e->__binheap_idx != BINHEAP_NOIDX);
+	assert(e->__binheap_idx != VBH_NOIDX);
 	e->fd = -1;
-	binheap_delete(evb->binheap, e->__binheap_idx);
-	assert(e->__binheap_idx == BINHEAP_NOIDX);
+	VBH_delete(evb->binheap, e->__binheap_idx);
+	assert(e->__binheap_idx == VBH_NOIDX);
 	evb->lpfd--;
 
 	if (e->sig > 0) {
@@ -365,8 +365,8 @@ vev_sched_timeout(struct vev_root *evb, struct vev *e, vtim_mono t)
 		free(e);
 	} else {
 		e->__when = t + e->timeout;
-		binheap_delete(evb->binheap, e->__binheap_idx);
-		binheap_insert(evb->binheap, e);
+		VBH_delete(evb->binheap, e->__binheap_idx);
+		VBH_insert(evb->binheap, e);
 	}
 	return (1);
 }
@@ -413,10 +413,10 @@ VEV_Once(struct vev_root *evb)
 		return (vev_sched_signal(evb));
 
 	tmo = INFTIM;
-	e = binheap_root(evb->binheap);
+	e = VBH_root(evb->binheap);
 	if (e != NULL) {
 		CHECK_OBJ(e, VEV_MAGIC);
-		assert(e->__binheap_idx == BINHEAP_NOIDX + 1);
+		assert(e->__binheap_idx == VBH_NOIDX + 1);
 		t = VTIM_mono();
 		if (e->__when <= t)
 			return (vev_sched_timeout(evb, e, t));
@@ -426,7 +426,7 @@ VEV_Once(struct vev_root *evb)
 			tmo = 1;
 	}
 
-	if (tmo == INFTIM && evb->lpfd == BINHEAP_NOIDX + 1)
+	if (tmo == INFTIM && evb->lpfd == VBH_NOIDX + 1)
 		return (0);
 
 	i = poll(evb->pfd + 1, evb->lpfd - 1, tmo);
@@ -452,7 +452,7 @@ VEV_Once(struct vev_root *evb)
 
 	DBG(evb, "EVENTS %d\n", i);
 	while (i > 0) {
-		for (u = BINHEAP_NOIDX + 1; u < evb->lpfd; u++) {
+		for (u = VBH_NOIDX + 1; u < evb->lpfd; u++) {
 			e = evb->pev[u];
 			if (e->fd_events == 0)
 				continue;
diff --git a/tools/lsan.suppr b/tools/lsan.suppr
index 8ed7048df..f9ca2be93 100644
--- a/tools/lsan.suppr
+++ b/tools/lsan.suppr
@@ -9,7 +9,7 @@ leak:vcc_
 leak:VSL_Setup
 leak:WRK_BgThread
 #
-leak:binheap_new
+leak:VBH_new
 # ev
 leak:mct_callback
 #

From phk at FreeBSD.org  Tue Aug  4 10:54:07 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  4 Aug 2020 10:54:07 +0000 (UTC)
Subject: [master] 39bcda5cb Use VSB_destroy() where applicable.
Message-ID: <20200804105407.B59D8113292@lists.varnish-cache.org>


commit 39bcda5cb29dbe01513fa09f60c3c22ed83f0273
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 4 10:51:14 2020 +0000

    Use VSB_destroy() where applicable.

diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c
index 3b04026fc..4eccf309d 100644
--- a/bin/varnishtest/vtc_barrier.c
+++ b/bin/varnishtest/vtc_barrier.c
@@ -306,7 +306,7 @@ barrier_sock_sync(struct barrier *b, struct vtclog *vl)
 		vtc_fatal(vl, "Barrier(%s) connection failed: %s",
 		    b->name, err);
 
-	VSB_delete(vsb);
+	VSB_destroy(&vsb);
 
 	/* emulate pthread_cond_wait's behavior */
 	AZ(pthread_mutex_unlock(&b->mtx));
diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c
index c0a532e47..6ccfbb8d3 100644
--- a/bin/varnishtest/vtc_client.c
+++ b/bin/varnishtest/vtc_client.c
@@ -225,7 +225,7 @@ client_thread(void *priv)
 	vsb = macro_expand(vl, c->connect);
 	AN(vsb);
 #if !defined(__sun)
-	pthread_cleanup_push((void (*)(void *))VSB_delete, vsb);
+	pthread_cleanup_push((void (*)(void *))VSB_destroy, &vsb);
 #endif
 	c->addr = VSB_data(vsb);
 
@@ -252,7 +252,7 @@ client_thread(void *priv)
 	pthread_cleanup_pop(0);
 #endif
 	pthread_cleanup_pop(0);
-	VSB_delete(vsb);
+	VSB_destroy(&vsb);
 	vtc_logclose(vl);
 	return (NULL);
 }
diff --git a/bin/varnishtest/vtc_proxy.c b/bin/varnishtest/vtc_proxy.c
index 1bee39657..ab7167ee8 100644
--- a/bin/varnishtest/vtc_proxy.c
+++ b/bin/varnishtest/vtc_proxy.c
@@ -129,6 +129,6 @@ vtc_send_proxy(int fd, int version, const struct suckaddr *sac,
 
 	AZ(VSB_finish(vsb));
 	i = VSB_tofile(vsb, fd);
-	VSB_delete(vsb);
+	VSB_destroy(&vsb);
 	return (i);
 }

From phk at FreeBSD.org  Tue Aug  4 10:54:07 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  4 Aug 2020 10:54:07 +0000 (UTC)
Subject: [master] cd1f75af4 Use VSB_destroy() where applicable
Message-ID: <20200804105407.CC586113295@lists.varnish-cache.org>


commit cd1f75af4fbbb7bcd3d5ca4f907d469576134404
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 4 10:53:08 2020 +0000

    Use VSB_destroy() where applicable

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index bbd8700ec..32b1501c2 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -738,7 +738,7 @@ MCF_ParamConf(enum mcf_which_e which, const char * const param,
 	va_end(ap);
 	AZ(VSB_finish(vsb));
 	mcf_dyn_vsb(which, pp, vsb);
-	VSB_delete(vsb);
+	VSB_destroy(&vsb);
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c
index f825273b2..1cc36d47f 100644
--- a/bin/varnishd/proxy/cache_proxy_proto.c
+++ b/bin/varnishd/proxy/cache_proxy_proto.c
@@ -755,7 +755,7 @@ VPX_Send_Proxy(int fd, int version, const struct sess *sp)
 	    version == 2 ? VSB_QUOTE_HEX : 0);
 	AZ(VSB_finish(vsb2));
 	VSL(SLT_Debug, 999, "PROXY_HDR %s", VSB_data(vsb2));
-	VSB_delete(vsb2);
+	VSB_destroy(&vsb2);
 	return (r);
 }
 
diff --git a/lib/libvarnish/vsb_test.c b/lib/libvarnish/vsb_test.c
index 9ac60154c..8ce277c65 100644
--- a/lib/libvarnish/vsb_test.c
+++ b/lib/libvarnish/vsb_test.c
@@ -44,7 +44,7 @@ main(int argc, char *argv[])
 		printf("\n");
 		VSB_clear(vsb);
 	}
-	VSB_delete(vsb);
+	VSB_destroy(&vsb);
 	printf("error is %i\n", err);
 	return (err);
 }
diff --git a/lib/libvmod_vtc/vmod_vtc.c b/lib/libvmod_vtc/vmod_vtc.c
index b2b9afd7d..bc95fcb84 100644
--- a/lib/libvmod_vtc/vmod_vtc.c
+++ b/lib/libvmod_vtc/vmod_vtc.c
@@ -362,7 +362,7 @@ vmod_proxy_header(VRT_CTX, VCL_ENUM venum, VCL_IP client, VCL_IP server,
 	VRT_Format_Proxy(vsb, version, client, server, authority);
 	l = VSB_len(vsb);
 	h = WS_Copy(ctx->ws, VSB_data(vsb), l);
-	VSB_delete(vsb);
+	VSB_destroy(&vsb);
 
 	if (h == NULL) {
 		VRT_fail(ctx, "proxy_header: out of workspace");

From phk at FreeBSD.org  Tue Aug  4 11:34:07 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  4 Aug 2020 11:34:07 +0000 (UTC)
Subject: [master] 36225f2ee Expend a pointer mere in struct vsb,
 in an attempt to make it clear for static analysis tools what goes on.
Message-ID: <20200804113407.04A6E114EF7@lists.varnish-cache.org>


commit 36225f2ee88f6a5bb4c709e27346962a0e085822
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 4 11:32:47 2020 +0000

    Expend a pointer mere in struct vsb, in an attempt to make it
    clear for static analysis tools what goes on.
    
    This commit will be reverted if that doesn't work.

diff --git a/flint.lnt b/flint.lnt
index b47a74bdb..3eb5fbd52 100644
--- a/flint.lnt
+++ b/flint.lnt
@@ -185,7 +185,6 @@
 -esym(765, VSB_*)		// extern could be made static
 -esym(714, VSB_*)		// symb not ref
 -sem(VSB_new, @p == (1p ? 1p : malloc(1)))
--sem(VSB_delete, custodial(1))
 
 // ignore retval
 -esym(534, VSB_cat)
diff --git a/include/vsb.h b/include/vsb.h
index facea9394..4041f4c4f 100644
--- a/include/vsb.h
+++ b/include/vsb.h
@@ -48,9 +48,9 @@ struct vsb {
 #define	VSB_USRFLAGMSK	0x0000ffff	/* mask of flags the user may specify */
 #define	VSB_DYNAMIC	0x00010000	/* s_buf must be freed */
 #define	VSB_FINISHED	0x00020000	/* set by VSB_finish() */
-#define	VSB_DYNSTRUCT	0x00080000	/* vsb must be freed */
 	int		 s_flags;	/* flags */
 	int		 s_indent;	/* Ident level */
+	void		*s_alloced;	/* Ptr to free, if vsb is malloced */
 };
 
 #ifdef __cplusplus
diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c
index ef9f164b0..c47977e90 100644
--- a/lib/libvarnish/vsb.c
+++ b/lib/libvarnish/vsb.c
@@ -51,8 +51,7 @@ __FBSDID("$FreeBSD: head/sys/kern/subr_vsb.c 222004 2011-05-17 06:36:32Z phk $")
 /*
  * Predicates
  */
-#define	VSB_ISDYNAMIC(s)	((s)->s_flags & VSB_DYNAMIC)
-#define	VSB_ISDYNSTRUCT(s)	((s)->s_flags & VSB_DYNSTRUCT)
+#define VSB_ISDYNAMIC(s)	((s)->s_flags & VSB_DYNAMIC)
 #define	VSB_HASROOM(s)		((s)->s_len < (s)->s_size - 1L)
 #define	VSB_FREESPACE(s)	((s)->s_size - ((s)->s_len + 1L))
 #define	VSB_CANEXTEND(s)	((s)->s_flags & VSB_AUTOEXTEND)
@@ -232,7 +231,7 @@ VSB_new(struct vsb *s, char *buf, int length, int flags)
 		SBFREE(s);
 		return (NULL);
 	}
-	VSB_SETFLAG(s, VSB_DYNSTRUCT);
+	s->s_alloced = s;
 	return (s);
 }
 
@@ -480,17 +479,17 @@ VSB_len(const struct vsb *s)
 void
 VSB_delete(struct vsb *s)
 {
-	int isdyn;
+	void *p;
 
 	assert_VSB_integrity(s);
 	/* don't care if it's finished or not */
 
 	if (VSB_ISDYNAMIC(s))
 		SBFREE(s->s_buf);
-	isdyn = VSB_ISDYNSTRUCT(s);
+	p = s->s_alloced;
 	memset(s, 0, sizeof(*s));
-	if (isdyn)
-		SBFREE(s);
+	if (p != NULL)
+		SBFREE(p);
 }
 
 void

From martin at varnish-software.com  Tue Aug  4 11:38:06 2020
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Tue,  4 Aug 2020 11:38:06 +0000 (UTC)
Subject: [master] a390e0803 Attempt again to silence coverity
Message-ID: <20200804113806.99D781154CD@lists.varnish-cache.org>


commit a390e0803475d3132f2dc4d447a02507c4b352b5
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Aug 4 13:37:05 2020 +0200

    Attempt again to silence coverity

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 85ee9ee1f..ae9cb2314 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -208,6 +208,7 @@ EXP_Insert(struct worker *wrk, struct objcore *oc)
 	if (remove_race) {
 		ObjSendEvent(wrk, oc, OEV_EXPIRE);
 		tmpoc = oc;
+		assert(oc->refcnt >= 2); /* Silence coverity */
 		(void)HSH_DerefObjCore(wrk, &tmpoc, 0);
 		AZ(tmpoc);
 		assert(oc->refcnt >= 1); /* Silence coverity */

From nils.goroll at uplex.de  Thu Aug  6 15:16:08 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu,  6 Aug 2020 15:16:08 +0000 (UTC)
Subject: [master] fb044cb20 Fix base64 decoding for length= argument and
 non-padding decoding
Message-ID: <20200806151608.35499113A3A@lists.varnish-cache.org>


commit fb044cb204ca34493b2b33645938075ca64d2c73
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Aug 6 17:10:33 2020 +0200

    Fix base64 decoding for length= argument and non-padding decoding
    
    When decoding only a substring, we naturally see no padding, so we must
    not base tail processing on the number of pad characters seen, but
    rather on the number of characters missing until the end of the current
    block of four.
    
    Fixes #3378

diff --git a/bin/varnishtest/tests/m00042.vtc b/bin/varnishtest/tests/m00042.vtc
index a03e46a3d..1f2db55b8 100644
--- a/bin/varnishtest/tests/m00042.vtc
+++ b/bin/varnishtest/tests/m00042.vtc
@@ -263,7 +263,7 @@ client c1 {
 	expect resp.http.hexmix2hexlc == resp.http.hexmix2hex
 	expect resp.http.hexparam == "0123456789"
 	expect resp.http.b642b64 == "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij"
-	expect resp.http.b64url2b64url == "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefeQ=="
+	expect resp.http.b64url2b64url == "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefgg=="
 	expect resp.http.b64urlnopad2b64urlnopad == "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefgg"
 } -run
 
diff --git a/lib/libvmod_blob/base64.c b/lib/libvmod_blob/base64.c
index 06b4c1df0..909dc11e3 100644
--- a/lib/libvmod_blob/base64.c
+++ b/lib/libvmod_blob/base64.c
@@ -309,7 +309,7 @@ base64_decode(const enum encoding dec, blob_dest_t buf,
 		}
 	}
 	if (n) {
-		if (!alpha->padding)
+		if (n - term != 0)
 			u <<= (6 * (4 - n));
 		if (decode(&dest, buf, buflen, u, n-term) < 0)
 			return (-1);

From phk at FreeBSD.org  Fri Aug  7 08:37:09 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri,  7 Aug 2020 08:37:09 +0000 (UTC)
Subject: [master] cd41b81c9 Add VENC functions to encode & decode base64
Message-ID: <20200807083709.81C84105EDC@lists.varnish-cache.org>


commit cd41b81c93eda7dc24535b99c744906d6b718809
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Aug 7 08:33:30 2020 +0000

    Add VENC functions to encode & decode base64

diff --git a/doc/sphinx/reference/vtla.rst b/doc/sphinx/reference/vtla.rst
index a2f2e3030..82b99ed24 100644
--- a/doc/sphinx/reference/vtla.rst
+++ b/doc/sphinx/reference/vtla.rst
@@ -45,6 +45,12 @@ VDD
     Varnish (Core) Developer Day -- Quarterly invite-only meeting strictly
     for Varnish core (C) developers, packagers and VMOD hackers.
 
+VENC
+    Varnish ENCoding -- base64 functions
+
+VEND
+    Varnish ENDianess -- functions to marshall data in specified endianess
+
 VEV
     Varnish EVent -- library functions to implement a simple event-dispatcher.
 
diff --git a/include/Makefile.am b/include/Makefile.am
index 628993bf0..78ff414f2 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -90,6 +90,7 @@ nobase_noinst_HEADERS = \
 	vcs_version.h \
 	vct.h \
 	vcurses.h \
+	venc.h \
 	vend.h \
 	vev.h \
 	vfil.h \
diff --git a/include/venc.h b/include/venc.h
new file mode 100644
index 000000000..a6d089838
--- /dev/null
+++ b/include/venc.h
@@ -0,0 +1,36 @@
+/*-
+ * Copyright (c) 2020 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+struct vsb;
+
+const char *VENC_Decode_Base64(struct vsb *, const char *, const char *);
+
+void VENC_Encode_Base64(struct vsb*, const void *, size_t);
diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am
index 4318d5734..6fbdfcba5 100644
--- a/lib/libvarnish/Makefile.am
+++ b/lib/libvarnish/Makefile.am
@@ -21,6 +21,7 @@ libvarnish_la_SOURCES = \
 	vcli_proto.c \
 	vcli_serve.c \
 	vct.c \
+	venc.c \
 	verrno.c \
 	version.c \
 	vev.c \
diff --git a/lib/libvarnish/venc.c b/lib/libvarnish/venc.c
new file mode 100644
index 000000000..27f80bf2b
--- /dev/null
+++ b/lib/libvarnish/venc.c
@@ -0,0 +1,227 @@
+/*-
+ * Copyright (c) 2020 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#include <stdint.h>
+#include <string.h>
+#include <limits.h>
+#include <sys/types.h>
+
+#include "vdef.h"
+#include "vas.h"
+#include "vrt.h"
+#include "venc.h"
+
+#include "vsb.h"
+
+/*
+ * Yes: Cumbersome.  But if "..." syntax is used, C adds a NUL
+ */
+
+static const char base64_enc[64] = {
+    'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
+    'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
+    'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
+    'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
+    'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
+    'o', 'p', 'r', 'q', 's', 't', 'u', 'v',
+    'w', 'x', 'y', 'z', '0', '1', '2', '3',
+    '4', '5', '6', '7', '8', '9', '+', '/'
+};
+
+/*
+ * Encode base64 (RFC4648 section 4) into VSB.
+ */
+
+void
+VENC_Encode_Base64(struct vsb *dst, const void *src, size_t len)
+{
+	const uint8_t *p = src;
+	uint32_t u;
+	char w[4];
+
+	AN(dst);
+	AN(src);
+	for (; len >= 3; len -= 3, p += 3) {
+		u = (p[0] << 16) | (p[1] << 8) | p[2];
+		w[3] = base64_enc[u & 0x3f];
+		u >>= 6;
+		w[2] = base64_enc[u & 0x3f];
+		u >>= 6;
+		w[1] = base64_enc[u & 0x3f];
+		u >>= 6;
+		w[0] = base64_enc[u & 0x3f];
+		if (VSB_bcat(dst, w, sizeof w))
+			break;
+	}
+	if (len) {
+		memset(w, '=', sizeof w);
+		u = (p[0] << 16);
+		if (len == 2)
+		    u |= (p[1] << 8);
+		w[0] = base64_enc[(u >> 18) & 0x3f];
+		w[1] = base64_enc[(u >> 12) & 0x3f];
+		if (len == 2)
+			w[2] = base64_enc[(u >> 6) & 0x3f];
+		VSB_bcat(dst, w, sizeof w);
+	}
+}
+
+#define BV 64
+
+static const uint8_t base64_dec[256] = {
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, 62, BV, BV, BV, 63,
+    52, 53, 54, 55, 56, 57, 58, 59, 60, 61, BV, BV, BV,  0, BV, BV,
+    BV,  0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14,
+    15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, BV, BV, BV, BV, BV,
+    BV, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
+    41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, BV, BV, BV, BV, BV,
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV
+};
+
+#if 0 // pending #3376 decision
+static const uint8_t base64url_dec[256] = {
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, 62, BV, BV,
+    52, 53, 54, 55, 56, 57, 58, 59, 60, 61, BV, BV, BV,  0, BV, BV,
+    BV,  0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14,
+    15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, BV, BV, BV, BV, 63,
+    BV, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
+    41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, BV, BV, BV, BV, BV,
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
+    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV
+};
+#endif
+
+struct venc_state {
+	unsigned		n;
+	unsigned		f;
+	uint8_t			w;
+	struct vsb		*vsb;
+	const uint8_t		*tbl;
+};
+
+static const char *
+venc_decode_base64(struct venc_state *ves, const char *b, const char *e)
+{
+	unsigned i;
+
+	AN(ves);
+	AN(ves->vsb);
+	AN(ves->tbl);
+
+	AN(b);
+	if (e == NULL)
+		e = strchr(b, '\0');
+	assert(e >= b);
+
+	for (; b < e; b++) {
+		i = ves->tbl[*(const uint8_t*)b];
+		if (i == BV)
+			return (b);
+		if (*b == '=' && ves->n < 2)
+			return (b);
+		else if (*b == '=')
+			ves->f++;
+		else if (ves->f)
+			return (b - 1);
+		if (ves->f && ves->w)
+			return (b - 1);
+		switch(++ves->n) {
+		case 1:
+			ves->w = i << 2;
+			break;
+		case 2:
+			ves->w |= i >> 4;
+			VSB_putc(ves->vsb, ves->w);
+			ves->w = i << 4;
+			break;
+		case 3:
+			ves->w |= i >> 2;
+			if (!ves->f)
+				VSB_putc(ves->vsb, ves->w);
+			ves->w = i << 6;
+			break;
+		case 4:
+			ves->w |= i;
+			if (!ves->f)
+				VSB_putc(ves->vsb, ves->w);
+			ves->w = 0;
+			ves->n = 0;
+			break;
+		default:
+			WRONG("Wrong turn in venc_decode_base64()");
+		}
+	}
+	return (NULL);
+}
+
+/*
+ * Decode base64 (RFC4648 section 4) into VSB.
+ *
+ * Returns NULL on success.
+ * Returns pointer to offending input character on failure.
+ */
+
+const char *
+VENC_Decode_Base64(struct vsb *dst, const char *b, const char *e)
+{
+	struct venc_state ves;
+	const char *rv;
+
+	memset(&ves, 0, sizeof ves);
+	ves.vsb = dst;
+	ves.tbl = base64_dec;
+
+	rv = venc_decode_base64(&ves, b, e);
+	if (rv)
+		return (rv);
+	if (ves.n)
+		return (e);
+	return (NULL);
+}

From phk at FreeBSD.org  Fri Aug  7 08:37:09 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri,  7 Aug 2020 08:37:09 +0000 (UTC)
Subject: [master] 0b374145f Add  BLOB to STRING folder,
 using Structured Fields (upcoming RFC):
Message-ID: <20200807083709.96121105EDF@lists.varnish-cache.org>


commit 0b374145f5bf8ce9fe83abfd797bba0b6312ed54
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Aug 7 08:34:37 2020 +0000

    Add  BLOB to STRING folder, using Structured Fields (upcoming RFC):
    
            ':' base64(blob) ':'

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 4229d8e53..7da0734ab 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -39,6 +39,7 @@
 #include "vav.h"
 #include "vcl.h"
 #include "vct.h"
+#include "venc.h"
 #include "vend.h"
 #include "vrt_obj.h"
 #include "vsa.h"
@@ -759,6 +760,22 @@ VRT_BOOL_string(VCL_BOOL val)
 	return (val ? "true" : "false");
 }
 
+VCL_STRING v_matchproto_()
+VRT_BLOB_string(VRT_CTX, VCL_BLOB val)
+{
+	struct vsb vsb[1];
+	const char *s;
+
+	if (val == NULL)
+		return (NULL);
+	WS_VSB_new(vsb, ctx->ws);
+	VSB_putc(vsb, ':');
+	VENC_Encode_Base64(vsb, val->blob, val->len);
+	VSB_putc(vsb, ':');
+	s = WS_VSB_finish(vsb, ctx->ws, NULL);
+	return (s);
+}
+
 /*--------------------------------------------------------------------*/
 
 VCL_VOID
diff --git a/lib/libvcc/vcc_types.c b/lib/libvcc/vcc_types.c
index dace016e9..576e55cee 100644
--- a/lib/libvcc/vcc_types.c
+++ b/lib/libvcc/vcc_types.c
@@ -76,6 +76,7 @@ const struct type BACKEND[1] = {{
 const struct type BLOB[1] = {{
 	.magic =		TYPE_MAGIC,
 	.name =			"BLOB",
+	.tostring =		"VRT_BLOB_string(ctx, \v1)",
 }};
 
 const struct type BODY[1] = {{

From phk at FreeBSD.org  Fri Aug  7 08:37:09 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri,  7 Aug 2020 08:37:09 +0000 (UTC)
Subject: [master] d044efa19 Record addition of VRT_BLOB_string()
Message-ID: <20200807083709.CC73C105EE4@lists.varnish-cache.org>


commit d044efa195ed4944e770f089ae2f4c61e8c9e777
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Aug 7 08:35:39 2020 +0000

    Record addition of VRT_BLOB_string()

diff --git a/include/vrt.h b/include/vrt.h
index e38b9bf8f..365f59b1c 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -1,5 +1,4 @@
-/*-
- * Copyright (c) 2006 Verdens Gang AS
+/*- * Copyright (c) 2006 Verdens Gang AS
  * Copyright (c) 2006-2015 Varnish Software AS
  * All rights reserved.
  *
@@ -54,7 +53,8 @@
  * binary/load-time compatible, increment MAJOR version
  *
  * NEXT (2020-09-15)
- *	VRT_DirectorResolve() added
+ *	Added VRT_DirectorResolve()
+ *	Added VCL_STRING VRT_BLOB_string(VRT_CTX, VCL_BLOB)
  * 11.0 (2020-03-16)
  *	Changed type of vsa_suckaddr_len from int to size_t
  *	New prefix_{ptr|len} fields in vrt_backend
@@ -596,6 +596,7 @@ VCL_STRING VRT_UpperLowerStrands(VRT_CTX, VCL_STRANDS s, int up);
 
 VCL_STRING VRT_BACKEND_string(VCL_BACKEND);
 VCL_STRING VRT_BOOL_string(VCL_BOOL);
+VCL_STRING VRT_BLOB_string(VRT_CTX, VCL_BLOB);
 VCL_STRING VRT_CollectString(VRT_CTX, const char *p, ...);
 VCL_STRING VRT_INT_string(VRT_CTX, VCL_INT);
 VCL_STRING VRT_IP_string(VRT_CTX, VCL_IP);

From phk at FreeBSD.org  Fri Aug  7 08:37:10 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri,  7 Aug 2020 08:37:10 +0000 (UTC)
Subject: [master] 4ddb72ca1 Parse Structured Fields (upcoming RFC) sf-binaries
 as static BLOBs in VCL.
Message-ID: <20200807083710.1C574105EE8@lists.varnish-cache.org>


commit 4ddb72ca18e9b090890493504f7cefac735ff0aa
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Aug 7 08:36:02 2020 +0000

    Parse Structured Fields (upcoming RFC) sf-binaries as static BLOBs in VCL.

diff --git a/bin/varnishtest/tests/b00051.vtc b/bin/varnishtest/tests/b00051.vtc
index c6a012509..5b65e7590 100644
--- a/bin/varnishtest/tests/b00051.vtc
+++ b/bin/varnishtest/tests/b00051.vtc
@@ -13,6 +13,7 @@ varnish v1 -vcl+backend {
 	}
 	sub vcl_deliver {
 		set resp.http.req_hash = blob.encode(HEX, blob=req.hash);
+		set resp.http.req_hash-sf = req.hash;
 	}
 } -start
 
@@ -21,4 +22,5 @@ client c1 {
 	rxresp
 	expect resp.http.req_hash ~ "[[:xdigit:]]{64}"
 	expect resp.http.req_hash == resp.http.bereq_hash
+	expect resp.http.req_hash-sf == ":d9gz676uHvX8wU6y/pTPI6fRK/XVgpJuGZtogIxniLA=:"
 } -run
diff --git a/bin/varnishtest/tests/i00000.vtc b/bin/varnishtest/tests/i00000.vtc
index 14849f336..4767205d4 100644
--- a/bin/varnishtest/tests/i00000.vtc
+++ b/bin/varnishtest/tests/i00000.vtc
@@ -1,10 +1,14 @@
-varnishtest "SF-blob parsing in VCL"
+varnishtest "SF-binary/BLOB parsing in VCL"
 
-varnish v1 -errvcl "BLOB must have n*3 base64 characters" { sub vcl_recv { :a: } }
-varnish v2 -errvcl "BLOB must have n*3 base64 characters" { sub vcl_recv { :aa: } }
-varnish v3 -errvcl "Illegal BLOB character:" { sub vcl_recv { :aa?: } }
-varnish v4 -errvcl "BLOB must have n*3 base64 characters" { sub vcl_recv { :aaaa: } }
-varnish v5 -errvcl "Wrong padding ('=') in BLOB" { sub vcl_recv { :aaa=aa: } }
-varnish v6 -errvcl "Wrong padding ('=') in BLOB" { sub vcl_recv { :aaa==a: } }
-varnish v7 -errvcl "Wrong padding ('=') in BLOB" { sub vcl_recv { :aaaa=a: } }
-varnish v8 -errvcl "BLOB is not supported yet" { sub vcl_recv { :aaaa==: } }
+varnish v1 -errvcl "BLOB must have n*4 base64 characters" { sub vcl_recv { :a: } }
+varnish v1 -errvcl "BLOB must have n*4 base64 characters" { sub vcl_recv { :bbbbaa: } }
+varnish v1 -errvcl "BLOB must have n*4 base64 characters" { sub vcl_recv { :bbbbccccaaa: } }
+varnish v1 -errvcl "Illegal BLOB character:" { sub vcl_recv { :aa?: } }
+varnish v1 -errvcl "Wrong padding ('=') in BLOB" { sub vcl_recv { :aaaa=aa: } }
+varnish v1 -errvcl "Wrong padding ('=') in BLOB" { sub vcl_recv { :aaaa==a: } }
+varnish v1 -errvcl "Wrong padding ('=') in BLOB" { sub vcl_recv { :aaaaa=a: } }
+varnish v1 -errvcl "Missing colon at end of BLOB" " sub vcl_recv { :aaaa"
+varnish v1 -errvcl "Missing colon at end of BLOB" " sub vcl_recv { :"
+
+# hint: the 'B' leaves bits in the second output byte
+varnish v1 -errvcl "Illegal BLOB character:" { sub vcl_recv { :AB==: } }
diff --git a/bin/varnishtest/tests/m00012.vtc b/bin/varnishtest/tests/m00012.vtc
deleted file mode 100644
index 33f25c7c6..000000000
--- a/bin/varnishtest/tests/m00012.vtc
+++ /dev/null
@@ -1,21 +0,0 @@
-varnishtest "Test VMOD BLOBS"
-
-varnish v1 -errvcl {BLOBs can only be used as arguments to VMOD functions.} {
-
-	backend b1 {.host = "${bad_backend}";}
-
-	sub vcl_deliver {
-		set resp.http.foo = req.hash;
-	}
-}
-
-varnish v1 -errvcl {Expression has type STRING, expected BLOB} {
-
-	backend b1 {.host = "${bad_backend}";}
-
-	import blob;
-
-	sub vcl_deliver {
-		blob.encode(blob = "blob");
-	}
-}
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 86112f640..12b8c417e 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -73,8 +73,11 @@ tokens = {
     "CNUM":     None,
     "FNUM":     None,
     "CSTR":     None,
-    "EOI":      None,
     "CSRC":     None,
+    "CBLOB":    None,
+
+    # End of token list
+    "EOI":      None,
 }
 
 #######################################################################
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 60a29b585..182cff8e1 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -825,6 +825,15 @@ vcc_expr5(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 		e1->constant = EXPR_CONST;
 		*e = e1;
 		return;
+	case CBLOB:
+		e1 = vcc_new_expr(BLOB);
+		VSB_printf(e1->vsb, "%s", tl->t->dec);
+		AZ(VSB_finish(e1->vsb));
+		e1->constant |= EXPR_STR_CONST;
+		e1->t1 = tl->t;
+		vcc_NextToken(tl);
+		*e = e1;
+		return;
 	default:
 		break;
 	}
diff --git a/lib/libvcc/vcc_token.c b/lib/libvcc/vcc_token.c
index fa2d504cf..00e21b414 100644
--- a/lib/libvcc/vcc_token.c
+++ b/lib/libvcc/vcc_token.c
@@ -36,7 +36,9 @@
 
 #include "vcc_compile.h"
 
+#include "venc.h"
 #include "vct.h"
+#include "vsb.h"
 
 /*--------------------------------------------------------------------*/
 
@@ -390,6 +392,8 @@ vcc_Lexer(struct vcc *tl, const struct source *sp, int eoi)
 {
 	const char *p, *q, *r;
 	unsigned u;
+	struct vsb *vsb;
+	char namebuf[40];
 
 	for (p = sp->b; p < sp->e; ) {
 
@@ -485,45 +489,64 @@ vcc_Lexer(struct vcc *tl, const struct source *sp, int eoi)
 
 		/* Recognize BLOB (= SF-binary) */
 		if (*p == ':') {
-			r = NULL;
-			for (q = p + 1; q < sp->e && vct_isbase64(*q); q++) {
-				if (r == NULL && *q == '=')
-					r = q;
-			}
-			if (q == sp->e || *q != ':') {
+			vsb = VSB_new_auto();
+			AN(vsb);
+			q = sp->e;
+			q -= (q - (p + 1)) % 4;
+			assert(q > p);
+			r = VENC_Decode_Base64(vsb, p + 1, q);
+			if (r == NULL) {
+				vcc_addtoken(tl, CBLOB, sp, p, q + 1);
 				VSB_cat(tl->sb,
-				    "Illegal BLOB character:\n");
-				vcc_addtoken(tl, EOI, sp, q, q+1);
+				    "Missing colon at end of BLOB:\n");
 				vcc_ErrWhere(tl, tl->t);
+				VSB_destroy(&vsb);
 				return;
 			}
-			if ((q - p) % 3 != 1) {
-				u = ((q - 1) - p) / 3;
-				vcc_addtoken(tl, EOI, sp, p + u * 3 + 1, q);
+			vcc_addtoken(tl, CBLOB, sp, p, r + 1);
+			if (*r == ':' && ((r - p) % 4) != 1) {
 				VSB_cat(tl->sb,
-				    "BLOB must have n*3 base64 characters\n");
+				    "BLOB must have n*4 base64 characters\n");
 				vcc_ErrWhere(tl, tl->t);
+				VSB_destroy(&vsb);
 				return;
 			}
-			if (r == NULL) {
-				/* No padding; */
-			} else if (r + 1 == q) {
-				/* One pad char */
-			} else if (r + 2 == q && r[1] == '=') {
-				/* Two (valid) pad chars */
-			} else {
+			if (*r == '=') {
 				VSB_cat(tl->sb,
 				    "Wrong padding ('=') in BLOB:\n");
-				vcc_addtoken(tl, EOI, sp, r, r+1);
 				vcc_ErrWhere(tl, tl->t);
+				VSB_destroy(&vsb);
 				return;
 			}
-			p = q + 1;
-			vcc_addtoken(tl, EOI, sp, p, q);
-			VSB_cat(tl->sb,
-			    "BLOB is not supported yet.\n");
-			vcc_ErrWhere(tl, tl->t);
-			return;
+			if (*r != ':') {
+				VSB_cat(tl->sb, "Illegal BLOB character:\n");
+				vcc_ErrWhere(tl, tl->t);
+				VSB_destroy(&vsb);
+				return;
+			}
+			r++;
+			AZ(VSB_finish(vsb));
+
+			bprintf(namebuf, "blob_%u", tl->unique++);
+			Fh(tl, 0, "\nconst unsigned char %s_data[%zd] = {\n",
+			    namebuf, VSB_len(vsb));
+			for (u = 0; u < VSB_len(vsb); u++) {
+				Fh(tl, 0, "\t0x%02x,", VSB_data(vsb)[u] & 0xff);
+				if ((u & 7) == 7)
+					Fh(tl, 0, "\n");
+			}
+			if ((u & 7) != 7)
+				Fh(tl, 0, "\n");
+			Fh(tl, 0, "};\n");
+			Fh(tl, 0, "\nconst struct vrt_blob %s[1] = {{\n",
+			    namebuf);
+			Fh(tl, 0, "\t.len =\t%zd,\n", VSB_len(vsb));
+			Fh(tl, 0, "\t.blob =\t%s_data,\n", namebuf);
+			Fh(tl, 0, "}};\n");
+			REPLACE(tl->t->dec, namebuf);
+			VSB_destroy(&vsb);
+			p = r;
+			continue;
 		}
 
 		/* Match for the fixed tokens (see generate.py) */

From phk at FreeBSD.org  Fri Aug  7 11:04:08 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri,  7 Aug 2020 11:04:08 +0000 (UTC)
Subject: [master] 10a223484 Make the hash key not depend on server.ip
Message-ID: <20200807110408.1D78110976A@lists.varnish-cache.org>


commit 10a223484d5548e3585a326a8c280ebfa4e0c883
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Aug 7 11:02:07 2020 +0000

    Make the hash key not depend on server.ip

diff --git a/bin/varnishtest/tests/b00051.vtc b/bin/varnishtest/tests/b00051.vtc
index 5b65e7590..3681ad2b3 100644
--- a/bin/varnishtest/tests/b00051.vtc
+++ b/bin/varnishtest/tests/b00051.vtc
@@ -18,9 +18,9 @@ varnish v1 -vcl+backend {
 } -start
 
 client c1 {
-	txreq
+	txreq -hdr "Host: localhost"
 	rxresp
 	expect resp.http.req_hash ~ "[[:xdigit:]]{64}"
 	expect resp.http.req_hash == resp.http.bereq_hash
-	expect resp.http.req_hash-sf == ":d9gz676uHvX8wU6y/pTPI6fRK/XVgpJuGZtogIxniLA=:"
+	expect resp.http.req_hash-sf == ":3k0f0yRKtKt7akzkyNsTGSDOJAZOQowTwKWhu5+kIu0=:"
 } -run

From nils.goroll at uplex.de  Fri Aug  7 11:19:06 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri,  7 Aug 2020 11:19:06 +0000 (UTC)
Subject: [master] b3043ce14 Optimize VRND_RandomCrypto()
Message-ID: <20200807111907.00BA9109E35@lists.varnish-cache.org>


commit b3043ce14065266fe2d94b233357cbe3e74eff3f
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jul 28 07:51:38 2020 +0200

    Optimize VRND_RandomCrypto()
    
    Try to read into the buffer in one go.
    
    closes #3366

diff --git a/lib/libvarnish/vrnd.c b/lib/libvarnish/vrnd.c
index 778c31fb2..891c40f27 100644
--- a/lib/libvarnish/vrnd.c
+++ b/lib/libvarnish/vrnd.c
@@ -168,17 +168,19 @@ int
 VRND_RandomCrypto(void *ptr, size_t len)
 {
 	int fd;
-	char *p;
+	char *p = ptr;
 	ssize_t l;
 
 	AN(ptr);
 	fd = open("/dev/urandom", O_RDONLY);
 	if (fd < 0)
 		return (-1);
-	for (p = ptr; len > 0; len--, p++) {
-		l = read(fd, p, 1);
-		if (l != 1)
+	while (len > 0) {
+		l = read(fd, p, len);
+		if (l < 0)
 			break;
+		p += l;
+		len -= l;
 	}
 	closefd(&fd);
 	return (len == 0 ? 0 : -1);

From phk at FreeBSD.org  Mon Aug 10 09:07:08 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 10 Aug 2020 09:07:08 +0000 (UTC)
Subject: [master] 1a708692a Revert "Expend a pointer mere in struct vsb,
 in an attempt to make it clear for static analysis tools what goes
 on."
Message-ID: <20200810090708.751F311D8B1@lists.varnish-cache.org>


commit 1a708692a30972a60c66580fb2dbe2b5227e87e6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 10 08:48:56 2020 +0000

    Revert "Expend a pointer mere in struct vsb, in an attempt to make it
            clear for static analysis tools what goes on."
    
    This reverts commit 36225f2ee88f6a5bb4c709e27346962a0e085822.

diff --git a/flint.lnt b/flint.lnt
index 3eb5fbd52..b47a74bdb 100644
--- a/flint.lnt
+++ b/flint.lnt
@@ -185,6 +185,7 @@
 -esym(765, VSB_*)		// extern could be made static
 -esym(714, VSB_*)		// symb not ref
 -sem(VSB_new, @p == (1p ? 1p : malloc(1)))
+-sem(VSB_delete, custodial(1))
 
 // ignore retval
 -esym(534, VSB_cat)
diff --git a/include/vsb.h b/include/vsb.h
index 4041f4c4f..facea9394 100644
--- a/include/vsb.h
+++ b/include/vsb.h
@@ -48,9 +48,9 @@ struct vsb {
 #define	VSB_USRFLAGMSK	0x0000ffff	/* mask of flags the user may specify */
 #define	VSB_DYNAMIC	0x00010000	/* s_buf must be freed */
 #define	VSB_FINISHED	0x00020000	/* set by VSB_finish() */
+#define	VSB_DYNSTRUCT	0x00080000	/* vsb must be freed */
 	int		 s_flags;	/* flags */
 	int		 s_indent;	/* Ident level */
-	void		*s_alloced;	/* Ptr to free, if vsb is malloced */
 };
 
 #ifdef __cplusplus
diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c
index c47977e90..ef9f164b0 100644
--- a/lib/libvarnish/vsb.c
+++ b/lib/libvarnish/vsb.c
@@ -51,7 +51,8 @@ __FBSDID("$FreeBSD: head/sys/kern/subr_vsb.c 222004 2011-05-17 06:36:32Z phk $")
 /*
  * Predicates
  */
-#define VSB_ISDYNAMIC(s)	((s)->s_flags & VSB_DYNAMIC)
+#define	VSB_ISDYNAMIC(s)	((s)->s_flags & VSB_DYNAMIC)
+#define	VSB_ISDYNSTRUCT(s)	((s)->s_flags & VSB_DYNSTRUCT)
 #define	VSB_HASROOM(s)		((s)->s_len < (s)->s_size - 1L)
 #define	VSB_FREESPACE(s)	((s)->s_size - ((s)->s_len + 1L))
 #define	VSB_CANEXTEND(s)	((s)->s_flags & VSB_AUTOEXTEND)
@@ -231,7 +232,7 @@ VSB_new(struct vsb *s, char *buf, int length, int flags)
 		SBFREE(s);
 		return (NULL);
 	}
-	s->s_alloced = s;
+	VSB_SETFLAG(s, VSB_DYNSTRUCT);
 	return (s);
 }
 
@@ -479,17 +480,17 @@ VSB_len(const struct vsb *s)
 void
 VSB_delete(struct vsb *s)
 {
-	void *p;
+	int isdyn;
 
 	assert_VSB_integrity(s);
 	/* don't care if it's finished or not */
 
 	if (VSB_ISDYNAMIC(s))
 		SBFREE(s->s_buf);
-	p = s->s_alloced;
+	isdyn = VSB_ISDYNSTRUCT(s);
 	memset(s, 0, sizeof(*s));
-	if (p != NULL)
-		SBFREE(p);
+	if (isdyn)
+		SBFREE(s);
 }
 
 void

From phk at FreeBSD.org  Wed Aug 12 12:07:08 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 12 Aug 2020 12:07:08 +0000 (UTC)
Subject: [master] 73629964c Allocate VSB the canonical way
Message-ID: <20200812120708.853321197C4@lists.varnish-cache.org>


commit 73629964ce15485619b48b9d3550ac449a7f48ef
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 12 08:36:04 2020 +0000

    Allocate VSB the canonical way

diff --git a/bin/varnishstat/varnishstat_help_gen.c b/bin/varnishstat/varnishstat_help_gen.c
index 2b145bed3..1412ea992 100644
--- a/bin/varnishstat/varnishstat_help_gen.c
+++ b/bin/varnishstat/varnishstat_help_gen.c
@@ -45,11 +45,12 @@ static const char help[] = "\n\n"
 int
 main(void)
 {
-	struct vsb vsb[1];
+	struct vsb *vsb;
 	const char *p, *n;
 	unsigned u;
 
-	AN(VSB_new(vsb, NULL, 0, VSB_AUTOEXTEND));
+	vsb = VSB_new_auto();
+	AN(vsb);
 	VSB_cat(vsb,
 	    "/*\n"
 	    " * NB:  This file is machine generated, DO NOT EDIT!\n"
@@ -83,6 +84,6 @@ main(void)
 	    "const int bindings_help_len = %u;\n", u);
 	AZ(VSB_finish(vsb));
 	AZ(VSB_tofile(vsb, STDOUT_FILENO));
-	VSB_delete(vsb);
+	VSB_destroy(&vsb);
 	return (0);
 }

From phk at FreeBSD.org  Wed Aug 12 12:07:08 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 12 Aug 2020 12:07:08 +0000 (UTC)
Subject: [master] 06575ef41 Add missing asserts
Message-ID: <20200812120708.9E5BB1197C7@lists.varnish-cache.org>


commit 06575ef411805fba5988e92b78f533eac0593569
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 12 09:42:43 2020 +0000

    Add missing asserts

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 0c8e221fb..6c7427eb6 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -536,6 +536,7 @@ vcl_print_refs(const struct vcl *vcl)
 
 	CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC);
 	msg = VSB_new_auto();
+	AN(msg);
 
 	VSB_printf(msg, "VCL %s is waiting for:", vcl->loaded_name);
 	Lck_Lock(&vcl_mtx);
@@ -646,6 +647,7 @@ vcl_load(struct cli *cli,
 	AZ(vcl);
 
 	msg = VSB_new_auto();
+	AN(msg);
 	vcl = VCL_Open(fn, msg);
 	AZ(VSB_finish(msg));
 	if (vcl == NULL) {
diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 32b1501c2..df62c2859 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -251,6 +251,7 @@ mcf_param_show(struct cli *cli, const char * const *av, void *priv)
 	struct vsb *vsb;
 
 	vsb = VSB_new_auto();
+	AN(vsb);
 	(void)priv;
 
 	if (av[2] != NULL && !strcmp(av[2], "changed"))
@@ -373,7 +374,9 @@ mcf_param_show_json(struct cli *cli, const char * const *av, void *priv)
 	}
 
 	vsb = VSB_new_auto();
+	AN(vsb);
 	def = VSB_new_auto();
+	AN(def);
 
 	n = 0;
 	VCLI_JSON_begin(cli, 2, av);

From phk at FreeBSD.org  Wed Aug 12 12:07:08 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 12 Aug 2020 12:07:08 +0000 (UTC)
Subject: [master] 660f73519 Add missing assert
Message-ID: <20200812120708.BAD9B1197CB@lists.varnish-cache.org>


commit 660f73519f726449bd854981b2d5d283fa43493f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 12 09:43:11 2020 +0000

    Add missing assert

diff --git a/bin/varnishd/cache/cache_ban_build.c b/bin/varnishd/cache/cache_ban_build.c
index e8c768ee5..f3891805e 100644
--- a/bin/varnishd/cache/cache_ban_build.c
+++ b/bin/varnishd/cache/cache_ban_build.c
@@ -425,9 +425,11 @@ ban_build_arg_operhelp(struct vsb *vsb, int arg)
 
 void
 BAN_Build_Init(void) {
-	struct vsb *vsb = VSB_new_auto();
+	struct vsb *vsb;
 	int i;
 
+	vsb = VSB_new_auto();
+	AN(vsb);
 	for (i = BANS_ARG_OFF_; i < BANS_ARG_LIM; i ++) {
 		VSB_clear(vsb);
 		ban_build_arg_operhelp(vsb, i);
@@ -438,7 +440,6 @@ BAN_Build_Init(void) {
 	}
 	arg_operhelp[BAN_ARGIDX(i)] = NULL;
 	VSB_destroy(&vsb);
-	AZ(vsb);
 }
 
 void

From phk at FreeBSD.org  Wed Aug 12 12:07:08 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 12 Aug 2020 12:07:08 +0000 (UTC)
Subject: [master] 904659a14 Evolve the VSB API,
 to explain things to static checkers.
Message-ID: <20200812120708.D85BB1197CF@lists.varnish-cache.org>


commit 904659a1475a395c9972626670abbf0d2e4222f4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 12 12:04:46 2020 +0000

    Evolve the VSB API, to explain things to static checkers.
    
    For dynamic allocations use:
    
            VSB_new_auto() + VSB_destroy()
    
    For preexisting buffers use:
    
            VSB_init() + VSB_fini()
    
    VSB_new + VSB_delete are deprecated.

diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c
index 51fd00754..020ee1e75 100644
--- a/bin/varnishd/cache/cache_backend_probe.c
+++ b/bin/varnishd/cache/cache_backend_probe.c
@@ -256,7 +256,7 @@ vbp_write_proxy_v1(struct vbp_target *vt, int *sock)
 	socklen_t l;
 
 	VTCP_myname(*sock, addr, sizeof addr, port, sizeof port);
-	AN(VSB_new(&vsb, buf, sizeof buf, VSB_FIXEDLEN));
+	AN(VSB_init(&vsb, buf, sizeof buf));
 
 	l = sizeof ss;
 	AZ(getsockname(*sock, (void *)&ss, &l));
@@ -268,7 +268,8 @@ vbp_write_proxy_v1(struct vbp_target *vt, int *sock)
 		VSB_cat(&vsb, "PROXY UNKNOWN\r\n");
 	AZ(VSB_finish(&vsb));
 
-	return (vbp_write(vt, sock, VSB_data(&vsb), VSB_len(&vsb)));
+	VSB_fini(&vsb);
+	return (vbp_write(vt, sock, buf, strlen(buf)));
 }
 
 static void
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index c59f35c53..19ae14a00 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -853,8 +853,7 @@ PAN_Init(void)
 	pan_vsb = &pan_vsb_storage;
 	AN(heritage.panic_str);
 	AN(heritage.panic_str_len);
-	AN(VSB_new(pan_vsb, heritage.panic_str, heritage.panic_str_len,
-	    VSB_FIXEDLEN));
+	AN(VSB_init(pan_vsb, heritage.panic_str, heritage.panic_str_len));
 	VSB_cat(pan_vsb, "This is a test\n");
 	AZ(VSB_finish(pan_vsb));
 	VSB_clear(pan_vsb);
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index dc1744779..874181436 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -393,12 +393,13 @@ WS_VSB_new(struct vsb *vsb, struct ws *ws)
 	unsigned u;
 	static char bogus[2];	// Smallest possible vsb
 
+	AN(vsb);
 	WS_Assert(ws);
 	u = WS_ReserveAll(ws);
 	if (WS_Overflowed(ws) || u < 2)
-		AN(VSB_new(vsb, bogus, sizeof bogus, VSB_FIXEDLEN));
+		AN(VSB_init(vsb, bogus, sizeof bogus));
 	else
-		AN(VSB_new(vsb, WS_Front(ws), u, VSB_FIXEDLEN));
+		AN(VSB_init(vsb, WS_Front(ws), u));
 }
 
 char *
@@ -406,6 +407,7 @@ WS_VSB_finish(struct vsb *vsb, struct ws *ws, size_t *szp)
 {
 	char *p;
 
+	AN(vsb);
 	WS_Assert(ws);
 	if (!VSB_finish(vsb)) {
 		p = VSB_data(vsb);
@@ -413,12 +415,12 @@ WS_VSB_finish(struct vsb *vsb, struct ws *ws, size_t *szp)
 			WS_Release(ws, VSB_len(vsb) + 1);
 			if (szp != NULL)
 				*szp = VSB_len(vsb);
-			VSB_delete(vsb);
+			VSB_fini(vsb);
 			return (p);
 		}
 	}
 	WS_MarkOverflow(ws);
-	VSB_delete(vsb);
+	VSB_fini(vsb);
 	WS_Release(ws, 0);
 	if (szp)
 		*szp = 0;
diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c
index 1cc36d47f..ff9066e0d 100644
--- a/bin/varnishd/proxy/cache_proxy_proto.c
+++ b/bin/varnishd/proxy/cache_proxy_proto.c
@@ -725,7 +725,7 @@ VPX_Send_Proxy(int fd, int version, const struct sess *sp)
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	assert(version == 1 || version == 2);
-	AN(VSB_new(vsb, buf, sizeof buf, VSB_FIXEDLEN));
+	AN(VSB_init(vsb, buf, sizeof buf));
 
 	AZ(SES_Get_server_addr(sp, &sas));
 	AN(sas);
@@ -756,6 +756,7 @@ VPX_Send_Proxy(int fd, int version, const struct sess *sp)
 	AZ(VSB_finish(vsb2));
 	VSL(SLT_Debug, 999, "PROXY_HDR %s", VSB_data(vsb2));
 	VSB_destroy(&vsb2);
+	VSB_fini(vsb);
 	return (r);
 }
 
diff --git a/include/vsb.h b/include/vsb.h
index facea9394..23d9dad82 100644
--- a/include/vsb.h
+++ b/include/vsb.h
@@ -59,9 +59,9 @@ extern "C" {
 /*
  * API functions
  */
-struct vsb	*VSB_new(struct vsb *, char *, int, int);
-#define		 VSB_new_auto()				\
-	VSB_new(NULL, NULL, 0, VSB_AUTOEXTEND)
+struct vsb	*VSB_new(struct vsb *, char *, int, int);	// DEPRECATED
+struct vsb	*VSB_init(struct vsb *, void *, ssize_t);
+struct vsb	*VSB_new_auto(void);
 void		 VSB_clear(struct vsb *);
 int		 VSB_bcat(struct vsb *, const void *, ssize_t);
 int		 VSB_cat(struct vsb *, const char *);
@@ -76,7 +76,8 @@ int		 VSB_error(const struct vsb *);
 int		 VSB_finish(struct vsb *);
 char		*VSB_data(const struct vsb *);
 ssize_t		 VSB_len(const struct vsb *);
-void		 VSB_delete(struct vsb *);
+void		 VSB_delete(struct vsb *);			// DEPRECATED
+void		 VSB_fini(struct vsb *);
 void		 VSB_destroy(struct vsb **);
 #define VSB_QUOTE_NONL		1
 #define VSB_QUOTE_JSON		2
diff --git a/lib/libvarnish/vin.c b/lib/libvarnish/vin.c
index 8f8d616dd..581bbdba2 100644
--- a/lib/libvarnish/vin.c
+++ b/lib/libvarnish/vin.c
@@ -69,7 +69,7 @@ VIN_n_Arg(const char *n_arg, char **dir)
 
 	/* Second: find the directory name */
 
-	AN(VSB_new(vsb, dn, sizeof dn, VSB_FIXEDLEN));
+	AN(VSB_init(vsb, dn, sizeof dn));
 
 	if (*nm == '/')
 		i = VSB_printf(vsb, "%s/", nm);
@@ -82,11 +82,11 @@ VIN_n_Arg(const char *n_arg, char **dir)
 	}
 
 	AZ(VSB_finish(vsb));
-	VSB_clear(vsb);
 
-	*dir = strdup(dn);
+	*dir = strdup(VSB_data(vsb));
 	if (*dir == NULL)
 		return (-1);
 
+	VSB_fini(vsb);
 	return (0);
 }
diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c
index ef9f164b0..df007c445 100644
--- a/lib/libvarnish/vsb.c
+++ b/lib/libvarnish/vsb.c
@@ -236,6 +236,36 @@ VSB_new(struct vsb *s, char *buf, int length, int flags)
 	return (s);
 }
 
+struct vsb *
+VSB_init(struct vsb *s, void *buf, ssize_t length)
+{
+	AN(s);
+	AN(buf);
+
+	KASSERT(length >= 0,
+	    ("attempt to create an vsb of negative length (%zd)", length));
+	return (VSB_newbuf(s, buf, length, VSB_FIXEDLEN));
+}
+
+/*
+ * Allocate a dynamic vsb
+ */
+struct vsb *
+VSB_new_auto(void)
+{
+	struct vsb *s;
+
+	s = SBMALLOC(sizeof(*s));
+	if (s == NULL)
+		return (NULL);
+	if (VSB_newbuf(s, NULL, 0, VSB_AUTOEXTEND) == NULL) {
+		SBFREE(s);
+		return (NULL);
+	}
+	VSB_SETFLAG(s, VSB_DYNSTRUCT);
+	return (s);
+}
+
 /*
  * Clear an vsb and reset its position.
  */
@@ -493,10 +523,24 @@ VSB_delete(struct vsb *s)
 		SBFREE(s);
 }
 
+void
+VSB_fini(struct vsb *s)
+{
+
+	assert_VSB_integrity(s);
+	assert(!VSB_ISDYNAMIC(s));
+	memset(s, 0, sizeof(*s));
+}
+
 void
 VSB_destroy(struct vsb **s)
 {
-	VSB_delete(*s);
+
+	AN(s);
+	assert_VSB_integrity(*s);
+	assert(VSB_ISDYNAMIC(*s));
+	memset(*s, 0, sizeof(**s));
+	SBFREE(*s);
 	*s = NULL;
 }
 
diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map
index 0ba4c8500..4436a9205 100644
--- a/lib/libvarnishapi/libvarnishapi.map
+++ b/lib/libvarnishapi/libvarnishapi.map
@@ -185,3 +185,13 @@ LIBVARNISHAPI_2.4 {	/* 2020-03-15 release */
     local:
 	*;
 };
+
+LIBVARNISHAPI_2.5 {	/* 2020-09-15 release */
+    global:
+	# vsb.c
+	VSB_init;
+	VSB_fini;
+	VSB_new_auto;
+    local:
+	*;
+};
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 182cff8e1..1447ec2b3 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -1085,13 +1085,14 @@ cmp_regexp(struct vcc *tl, struct expr **e, const struct cmps *cp)
 	*e = vcc_expr_edit(tl, STRING, "\vS", *e, NULL);
 	vcc_NextToken(tl);
 	ExpectErr(tl, CSTR);
-	AN(VSB_new(&vsb, buf, sizeof buf, VSB_FIXEDLEN));
+	AN(VSB_init(&vsb, buf, sizeof buf));
 	VSB_printf(&vsb, "%sVRT_re_match(ctx, \v1, ", cp->emit);
 	vcc_regexp(tl, &vsb);
 	ERRCHK(tl);
 	VSB_cat(&vsb, ")");
 	AZ(VSB_finish(&vsb));
 	*e = vcc_expr_edit(tl, BOOL, VSB_data(&vsb), *e, NULL);
+	VSB_fini(&vsb);
 }
 
 static void v_matchproto_(cmp_f)
@@ -1473,12 +1474,13 @@ vcc_Eval_Regsub(struct vcc *tl, struct expr **e, struct token *t,
 	SkipToken(tl, ',');
 	ExpectErr(tl, CSTR);
 
-	AN(VSB_new(&vsb, buf, sizeof buf, VSB_FIXEDLEN));
+	AN(VSB_init(&vsb, buf, sizeof buf));
 	VSB_printf(&vsb, "VRT_regsub(ctx, %d,\v+\n\v1,\n", all);
 	vcc_regexp(tl, &vsb);
 	ERRCHK(tl);
 	AZ(VSB_finish(&vsb));
 	*e = vcc_expr_edit(tl, STRING, VSB_data(&vsb), e2, NULL);
+	VSB_fini(&vsb);
 	SkipToken(tl, ',');
 	vcc_expr0(tl, &e2, STRING);
 	ERRCHK(tl);

From nils.goroll at uplex.de  Wed Aug 12 12:55:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 12 Aug 2020 12:55:07 +0000 (UTC)
Subject: [master] 83ab25a25 use our v_ attribute for marking the deprecation
Message-ID: <20200812125507.685E111AD64@lists.varnish-cache.org>


commit 83ab25a259374194bda0ae222a5484cb1f2091c4
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Aug 12 14:52:45 2020 +0200

    use our v_ attribute for marking the deprecation
    
    Ref 904659a1475a395c9972626670abbf0d2e4222f4
        1594037cb896970c2cbfdab101d0059ac00201f9

diff --git a/include/vsb.h b/include/vsb.h
index 23d9dad82..06239b6bb 100644
--- a/include/vsb.h
+++ b/include/vsb.h
@@ -59,7 +59,7 @@ extern "C" {
 /*
  * API functions
  */
-struct vsb	*VSB_new(struct vsb *, char *, int, int);	// DEPRECATED
+struct vsb	*VSB_new(struct vsb *, char *, int, int) v_deprecated_;
 struct vsb	*VSB_init(struct vsb *, void *, ssize_t);
 struct vsb	*VSB_new_auto(void);
 void		 VSB_clear(struct vsb *);
@@ -76,7 +76,7 @@ int		 VSB_error(const struct vsb *);
 int		 VSB_finish(struct vsb *);
 char		*VSB_data(const struct vsb *);
 ssize_t		 VSB_len(const struct vsb *);
-void		 VSB_delete(struct vsb *);			// DEPRECATED
+void		 VSB_delete(struct vsb *) v_deprecated_;
 void		 VSB_fini(struct vsb *);
 void		 VSB_destroy(struct vsb **);
 #define VSB_QUOTE_NONL		1

From guillaume at varnish-software.com  Fri Aug 14 14:44:08 2020
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Fri, 14 Aug 2020 14:44:08 +0000 (UTC)
Subject: [6.0] 99950850a [cci] dummy config to silence checks
Message-ID: <20200814144408.7CF86A40B6@lists.varnish-cache.org>


commit 99950850a13d4d1d0be7a9bdc6e77786e8bd49c7
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Fri Aug 14 07:42:59 2020 -0700

    [cci] dummy config to silence checks

diff --git a/.circleci/config.yml b/.circleci/config.yml
new file mode 100644
index 000000000..158e03071
--- /dev/null
+++ b/.circleci/config.yml
@@ -0,0 +1,12 @@
+version: 2.1
+jobs:
+  dummy:
+    docker:
+	    - image: alpine:3
+    steps:
+      - run: "" 
+
+workflows:
+  build:
+    jobs:
+      - dummy

From phk at FreeBSD.org  Sat Aug 15 12:02:08 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 15 Aug 2020 12:02:08 +0000 (UTC)
Subject: [master] 20de7660c Emit the "ipv4+ipv6 and mask" error first.
Message-ID: <20200815120208.75983975B8@lists.varnish-cache.org>


commit 20de7660cb88f251c39351fba3615dc4fee16cbb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Aug 15 12:01:12 2020 +0000

    Emit the "ipv4+ipv6 and mask" error first.

diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c
index 689455926..795a7571d 100644
--- a/lib/libvcc/vcc_acl.c
+++ b/lib/libvcc/vcc_acl.c
@@ -138,17 +138,17 @@ vcc_acl_chk(struct vcc *tl, const struct acl_e *ae, const int l,
 	AN(sa);
 	VTCP_name(sa, h, sizeof h, NULL, 0);
 	bprintf(t, "%s/%d", h, ae->mask);
+	VSB_printf(tl->sb, "Address/Netmask mismatch, ");
 	if (tl->acl_pedantic != 0)
-		VSB_printf(tl->sb, "Address/Netmask mismatch, need be %s\n", t);
+		VSB_printf(tl->sb, "need be %s\n", t);
 	else
-		VSB_printf(tl->sb, "Address/Netmask mismatch, changed to %s\n", t);
+		VSB_printf(tl->sb, "changed to %s\n", t);
 	vcc_ErrWhere(tl, ae->t_addr);
 	if (tl->acl_pedantic == 0)
 		vcc_Warn(tl);
 	return (strdup(t));
 }
 
-
 static void
 vcc_acl_add_entry(struct vcc *tl, const struct acl_e *ae, int l,
     unsigned char *u, int fam)
@@ -256,6 +256,30 @@ vcc_acl_try_getaddrinfo(struct vcc *tl, struct acl_e *ae)
 	}
 
 	i4 = i6 = 0;
+	for (res = res0; res != NULL; res = res->ai_next) {
+		switch (res->ai_family) {
+		case PF_INET:
+			i4++;
+			break;
+		case PF_INET6:
+			i6++;
+			break;
+		default:
+			VSB_printf(tl->sb,
+			    "Ignoring unknown protocol family (%d) for %.*s\n",
+				res->ai_family, PF(ae->t_addr));
+			continue;
+		}
+	}
+
+	if (ae->t_mask != NULL && i4 > 0 && i6 > 0) {
+		VSB_printf(tl->sb,
+		    "Mask (/%u) specified, but string resolves to"
+		    " both IPv4 and IPv6 addresses.\n", ae->mask);
+		vcc_ErrWhere(tl, ae->t_mask);
+		return;
+	}
+
 	for (res = res0; res != NULL; res = res->ai_next) {
 		switch (res->ai_family) {
 		case PF_INET:
@@ -265,7 +289,6 @@ vcc_acl_try_getaddrinfo(struct vcc *tl, struct acl_e *ae)
 			u = (void*)&sin4->sin_addr;
 			if (ae->t_mask == NULL)
 				ae->mask = 32;
-			i4++;
 			vcc_acl_add_entry(tl, ae, 4, u, res->ai_family);
 			break;
 		case PF_INET6:
@@ -275,13 +298,9 @@ vcc_acl_try_getaddrinfo(struct vcc *tl, struct acl_e *ae)
 			u = (void*)&sin6->sin6_addr;
 			if (ae->t_mask == NULL)
 				ae->mask = 128;
-			i6++;
 			vcc_acl_add_entry(tl, ae, 16, u, res->ai_family);
 			break;
 		default:
-			VSB_printf(tl->sb,
-			    "Ignoring unknown protocol family (%d) for %.*s\n",
-				res->ai_family, PF(ae->t_addr));
 			continue;
 		}
 		if (tl->err)
@@ -290,13 +309,6 @@ vcc_acl_try_getaddrinfo(struct vcc *tl, struct acl_e *ae)
 	}
 	freeaddrinfo(res0);
 
-	if (ae->t_mask != NULL && i4 > 0 && i6 > 0) {
-		VSB_printf(tl->sb,
-		    "Mask (/%u) specified, but string resolves to"
-		    " both IPv4 and IPv6 addresses.\n", ae->mask);
-		vcc_ErrWhere(tl, ae->t_mask);
-		return;
-	}
 }
 
 /*--------------------------------------------------------------------

From phk at FreeBSD.org  Mon Aug 17 11:25:07 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 17 Aug 2020 11:25:07 +0000 (UTC)
Subject: [master] 999e7bfe9 Cherry-pick std.blobread out of #3123
Message-ID: <20200817112507.6941B62741@lists.varnish-cache.org>


commit 999e7bfe9802451617cbedb0d2c7a01f3af27a55
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 17 11:24:08 2020 +0000

    Cherry-pick std.blobread out of #3123

diff --git a/bin/varnishtest/tests/r01145.vtc b/bin/varnishtest/tests/r01145.vtc
index ee33bb0c8..c1d33ead1 100644
--- a/bin/varnishtest/tests/r01145.vtc
+++ b/bin/varnishtest/tests/r01145.vtc
@@ -19,6 +19,7 @@ varnish v1 -vcl+backend {
 
 	sub vcl_deliver {
 	    set resp.http.foo = std.fileread("${tmpdir}" + req.url);
+	    set resp.http.fooblob = std.blobread("${tmpdir}" + req.url);
 	}
 } -start
 
@@ -27,17 +28,18 @@ client c1 {
 	txreq -url "/one"
 	rxresp
 	expect resp.http.foo == "File One"
+	expect resp.http.fooblob == :RmlsZSBPbmU=:
 
 	txreq -url "/two"
 	rxresp
 	expect resp.http.foo == "File Two"
+	expect resp.http.fooblob == :RmlsZSBUd28=:
 
 	txreq -url "/three"
 	rxresp
 	expect resp.http.foo == "File Three"
+	expect resp.http.fooblob == :RmlsZSBUaHJlZQ==:
 } -run
 
 client c1 -run
 client c1 -run
-
-
diff --git a/lib/libvmod_std/vmod_std.vcc b/lib/libvmod_std/vmod_std.vcc
index 8f8c7cacf..961e90ae1 100644
--- a/lib/libvmod_std/vmod_std.vcc
+++ b/lib/libvmod_std/vmod_std.vcc
@@ -187,10 +187,13 @@ File(system) functions
 
 $Function STRING fileread(PRIV_CALL, STRING)
 
-Reads a file and returns a string with the content. The result is
-cached indefinitely per filename.
+Reads a text file and returns a string with the content.
 
-This function should not be used for reading binary files.
+The entire file is cached on the first call, and subsequent calls
+will return this cached contents, even if the file has changed in
+the meantime.
+
+For binary files, use std.blobread() instead.
 
 Example::
 
@@ -204,6 +207,14 @@ case, you may need to modify the string, for example with
 
   set beresp.http.served-by = regsub(std.fileread("/etc/hostname"), "\R$", "");
 
+$Function BLOB blobread(PRIV_CALL, STRING)
+
+Reads any file and returns a blob with the content.
+
+The entire file is cached on the first call, and subsequent calls
+will return this cached contents, even if the file has changed in
+the meantime.
+
 $Function BOOL file_exists(STRING path)
 
 Returns ``true`` if path or the file pointed to by path exists,
diff --git a/lib/libvmod_std/vmod_std_fileread.c b/lib/libvmod_std/vmod_std_fileread.c
index 0e9f5cadc..df7a160d1 100644
--- a/lib/libvmod_std/vmod_std_fileread.c
+++ b/lib/libvmod_std/vmod_std_fileread.c
@@ -54,7 +54,8 @@ struct frfile {
 	unsigned			magic;
 #define CACHED_FILE_MAGIC 0xa8e9d87a
 	char				*file_name;
-	char				*contents;
+	void				*contents;
+	struct vrt_blob			blob[1];
 	int				refcount;
 	VTAILQ_ENTRY(frfile)		list;
 };
@@ -82,14 +83,13 @@ free_frfile(void *ptr)
 	}
 }
 
-VCL_STRING v_matchproto_(td_std_fileread)
-vmod_fileread(VRT_CTX, struct vmod_priv *priv,
-    VCL_STRING file_name)
+static struct frfile *
+find_frfile(struct vmod_priv *priv, VCL_STRING file_name)
 {
 	struct frfile *frf = NULL;
 	char *s;
+	ssize_t sz;
 
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	AN(priv);
 
 	if (file_name == NULL)
@@ -98,7 +98,7 @@ vmod_fileread(VRT_CTX, struct vmod_priv *priv,
 	if (priv->priv != NULL) {
 		CAST_OBJ_NOTNULL(frf, priv->priv, CACHED_FILE_MAGIC);
 		if (!strcmp(file_name, frf->file_name))
-			return (frf->contents);
+			return (frf);
 	}
 
 	AZ(pthread_mutex_lock(&frmtx));
@@ -114,22 +114,52 @@ vmod_fileread(VRT_CTX, struct vmod_priv *priv,
 	if (frf != NULL) {
 		priv->free = free_frfile;
 		priv->priv = frf;
-		return (frf->contents);
+		return (frf);
 	}
 
-	s = VFIL_readfile(NULL, file_name, NULL);
+	s = VFIL_readfile(NULL, file_name, &sz);
 	if (s != NULL) {
+		assert(sz > 0);
 		ALLOC_OBJ(frf, CACHED_FILE_MAGIC);
 		AN(frf);
-		frf->file_name = strdup(file_name);
-		AN(frf->file_name);
+		REPLACE(frf->file_name, file_name);
 		frf->refcount = 1;
 		frf->contents = s;
+		frf->blob->blob = s;
+		frf->blob->len = (size_t)sz;
 		priv->free = free_frfile;
 		priv->priv = frf;
 		AZ(pthread_mutex_lock(&frmtx));
 		VTAILQ_INSERT_HEAD(&frlist, frf, list);
 		AZ(pthread_mutex_unlock(&frmtx));
 	}
-	return (s);
+	return (frf);
+}
+
+VCL_STRING v_matchproto_(td_std_fileread)
+vmod_fileread(VRT_CTX, struct vmod_priv *priv, VCL_STRING file_name)
+{
+	struct frfile *frf;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	AN(priv);
+
+	frf = find_frfile(priv, file_name);
+	if (frf == NULL)
+		return (NULL);
+	return (frf->contents);
+}
+
+VCL_BLOB v_matchproto_(td_std_blobread)
+vmod_blobread(VRT_CTX, struct vmod_priv *priv, VCL_STRING file_name)
+{
+	struct frfile *frf;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	AN(priv);
+
+	frf = find_frfile(priv, file_name);
+	if (frf == NULL)
+		return (NULL);
+	return (frf->blob);
 }

From nils.goroll at uplex.de  Mon Aug 17 14:26:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 17 Aug 2020 14:26:07 +0000 (UTC)
Subject: [master] 6a11be319 Do not attempt to remove the vcl dir if we need
 the shared object
Message-ID: <20200817142607.1284D6EE7D@lists.varnish-cache.org>


commit 6a11be31975d6db7cdcd6aff7e45d153e8c023d9
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Aug 17 16:22:51 2020 +0200

    Do not attempt to remove the vcl dir if we need the shared object
    
    This avoids a bogus message
    
            Could not rmdir 'vcl_...': Directory not empty

diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c
index 0b85cbd93..729804b5a 100644
--- a/bin/varnishd/mgt/mgt_vcc.c
+++ b/bin/varnishd/mgt/mgt_vcc.c
@@ -283,10 +283,11 @@ mgt_vcc_fini_vp(struct vcc_priv *vp, int leave_lib)
 	if (!MGT_DO_DEBUG(DBG_VCL_KEEP)) {
 		VJ_unlink(VSB_data(vp->csrcfile));
 		VJ_unlink(VSB_data(vp->symfile));
-		if (!leave_lib)
+		if (!leave_lib) {
 			VJ_unlink(VSB_data(vp->libfile));
+			VJ_rmdir(VSB_data(vp->dir));
+		}
 	}
-	VJ_rmdir(VSB_data(vp->dir));
 	VSB_destroy(&vp->csrcfile);
 	VSB_destroy(&vp->libfile);
 	VSB_destroy(&vp->symfile);

From phk at FreeBSD.org  Tue Aug 18 10:38:07 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 18 Aug 2020 10:38:07 +0000 (UTC)
Subject: [master] 92c4a3987 base64url_dec table not needed.
Message-ID: <20200818103807.A630D4E2E@lists.varnish-cache.org>


commit 92c4a39877b6a86144725628dd6db8b133740705
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 18 09:00:45 2020 +0000

    base64url_dec table not needed.

diff --git a/lib/libvarnish/venc.c b/lib/libvarnish/venc.c
index 27f80bf2b..61916affa 100644
--- a/lib/libvarnish/venc.c
+++ b/lib/libvarnish/venc.c
@@ -117,27 +117,6 @@ static const uint8_t base64_dec[256] = {
     BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV
 };
 
-#if 0 // pending #3376 decision
-static const uint8_t base64url_dec[256] = {
-    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
-    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
-    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, 62, BV, BV,
-    52, 53, 54, 55, 56, 57, 58, 59, 60, 61, BV, BV, BV,  0, BV, BV,
-    BV,  0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14,
-    15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, BV, BV, BV, BV, 63,
-    BV, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40,
-    41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, BV, BV, BV, BV, BV,
-    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
-    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
-    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
-    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
-    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
-    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
-    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV,
-    BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV, BV
-};
-#endif
-
 struct venc_state {
 	unsigned		n;
 	unsigned		f;

From phk at FreeBSD.org  Tue Aug 18 10:38:07 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 18 Aug 2020 10:38:07 +0000 (UTC)
Subject: [master] 8bdfb793f Zero out the "len_so_far" when we abandon a fetch
 object
Message-ID: <20200818103807.C293F4E32@lists.varnish-cache.org>


commit 8bdfb793fe6fb1fe2764a465cdd5ff04f60387fb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 18 10:37:34 2020 +0000

    Zero out the "len_so_far" when we abandon a fetch object

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index c28f1ebeb..ce603e509 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -833,8 +833,10 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 	now = W_TIM_real(wrk);
 	VSLb_ts_busyobj(bo, "Error", now);
 
-	if (oc->stobj->stevedore != NULL)
+	if (oc->stobj->stevedore != NULL) {
+		oc->boc->len_so_far = 0;
 		ObjFreeObj(bo->wrk, oc);
+	}
 
 	if (bo->storage == NULL)
 		bo->storage = STV_next();

From reza at naghibi.com  Tue Aug 18 17:45:07 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 18 Aug 2020 17:45:07 +0000 (UTC)
Subject: [6.0] 346475f45 Revert "Rename the obj flag OF_CHGGZIP to OF_CHGCE,
 for "changed Content-Encoding"."
Message-ID: <20200818174507.843786E490@lists.varnish-cache.org>


commit 346475f4560921efe88c4341b80b21f0b7b8b834
Author: Reza Naghibi <reza at naghibi.com>
Date:   Tue Aug 18 13:33:54 2020 -0400

    Revert "Rename the obj flag OF_CHGGZIP to OF_CHGCE, for "changed Content-Encoding"."
    
    This reverts commit b90509232e5668aa79332749d0593718077f50ae.
    
     Conflicts:
            bin/varnishd/cache/cache_esi_fetch.c
            bin/varnishd/cache/cache_fetch.c

diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
index 575685603..d8383afa7 100644
--- a/bin/varnishd/cache/cache_esi_fetch.c
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -163,7 +163,7 @@ vfp_esi_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe)
 	ALLOC_OBJ(vef, VEF_MAGIC);
 	if (vef == NULL)
 		return (VFP_ERROR);
-	vc->obj_flags |= OF_GZIPED | OF_CHGCE | OF_ESIPROC;
+	vc->obj_flags |= OF_GZIPED | OF_CHGGZIP | OF_ESIPROC;
 	vef->vep = VEP_Init(vc, vc->req, vfp_vep_callback, vef);
 	if (vef->vep == NULL) {
 		FREE_OBJ(vef);
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 840120b06..0d794c836 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -291,7 +291,7 @@ vbf_304_logic(struct busyobj *bo)
 	if (bo->stale_oc != NULL &&
 	    ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND)) {
 		AZ(bo->stale_oc->flags & (OC_F_HFM|OC_F_PRIVATE));
-		if (ObjCheckFlag(bo->wrk, bo->stale_oc, OF_CHGCE)) {
+		if (ObjCheckFlag(bo->wrk, bo->stale_oc, OF_CHGGZIP)) {
 			/*
 			 * If a VFP changed C-E in the stored
 			 * object, then don't overwrite C-E from
diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index 390c9bae9..b325e79c5 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -466,14 +466,14 @@ vfp_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe)
 		if (http_GetHdr(vc->resp, H_Content_Encoding, NULL))
 			return (VFP_NULL);
 		vg = VGZ_NewGzip(vc->wrk->vsl, vfe->vfp->priv1);
-		vc->obj_flags |= OF_GZIPED | OF_CHGCE;
+		vc->obj_flags |= OF_GZIPED | OF_CHGGZIP;
 	} else {
 		if (!http_HdrIs(vc->resp, H_Content_Encoding, "gzip"))
 			return (VFP_NULL);
 		if (vfe->vfp == &VFP_gunzip) {
 			vg = VGZ_NewGunzip(vc->wrk->vsl, vfe->vfp->priv1);
 			vc->obj_flags &= ~OF_GZIPED;
-			vc->obj_flags |= OF_CHGCE;
+			vc->obj_flags |= OF_CHGGZIP;
 		} else {
 			vg = VGZ_NewTestGunzip(vc->wrk->vsl, vfe->vfp->priv1);
 			vc->obj_flags |= OF_GZIPED;
diff --git a/include/tbl/obj_attr.h b/include/tbl/obj_attr.h
index a695c1efa..b0b6773e1 100644
--- a/include/tbl/obj_attr.h
+++ b/include/tbl/obj_attr.h
@@ -55,7 +55,7 @@
 #ifdef OBJ_FLAG
 /* upper, lower, val */
   OBJ_FLAG(GZIPED,	gziped,		(1<<1))
-  OBJ_FLAG(CHGCE,	chgce,		(1<<2))
+  OBJ_FLAG(CHGGZIP,	chggzip,	(1<<2))
   OBJ_FLAG(IMSCAND,	imscand,	(1<<3))
   OBJ_FLAG(ESIPROC,	esiproc,	(1<<4))
   #undef OBJ_FLAG

From nils.goroll at uplex.de  Wed Aug 19 09:18:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 19 Aug 2020 09:18:07 +0000 (UTC)
Subject: [master] 031d3e880 plug VSB leak
Message-ID: <20200819091807.A948CA96A7@lists.varnish-cache.org>


commit 031d3e880d2cf974fb175da439c4db3eb20be2f0
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Tue Aug 18 11:36:19 2020 -0700

    plug VSB leak

diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c
index df007c445..0cbc66305 100644
--- a/lib/libvarnish/vsb.c
+++ b/lib/libvarnish/vsb.c
@@ -539,6 +539,7 @@ VSB_destroy(struct vsb **s)
 	AN(s);
 	assert_VSB_integrity(*s);
 	assert(VSB_ISDYNAMIC(*s));
+	SBFREE((*s)->s_buf);
 	memset(*s, 0, sizeof(**s));
 	SBFREE(*s);
 	*s = NULL;

From reza at naghibi.com  Wed Aug 19 13:12:07 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:12:07 +0000 (UTC)
Subject: [6.0] a45e1f4e4 Fail fetch retries when uncached request body has
 been released
Message-ID: <20200819131207.DFCE85333@lists.varnish-cache.org>


commit a45e1f4e49dbd6949659e9bb9ddfd2cdf9f6a479
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Oct 10 14:30:18 2019 +0200

    Fail fetch retries when uncached request body has been released
    
    Currently we allow fetch retries with body even after we have released the
    request that initiated the fetch, and the request body with it. The
    attached test case demonstrates this, where s2 on the retry attempt gets
    stuck waiting for 3 bytes of body data that is never sent.
    
    Fix this by keeping track of what the initial request body status was, and
    failing the retry attempt if the request was already released
    (BOS_REQ_DONE) and the request body was not cached.
    
    Conflicts:
        bin/varnishd/cache/cache.h

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 9e47c7215..cb4f1ba79 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -390,7 +390,8 @@ struct busyobj {
 	 * All fields from retries and down are zeroed when the busyobj
 	 * is recycled.
 	 */
-	int			retries;
+	unsigned		retries;
+	enum req_body_state_e	initial_req_body_status;
 	struct req		*req;
 	struct sess		*sp;
 	struct worker		*wrk;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 0d794c836..9962e1c9e 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -242,6 +242,7 @@ vbf_stp_mkbereq(struct worker *wrk, struct busyobj *bo)
 	bo->ws_bo = WS_Snapshot(bo->ws);
 	HTTP_Copy(bo->bereq, bo->bereq0);
 
+	bo->initial_req_body_status = bo->req->req_body_status;
 	if (bo->req->req_body_status == REQ_BODY_NONE) {
 		bo->req = NULL;
 		ObjSetState(bo->wrk, bo->fetch_objcore, BOS_REQ_DONE);
@@ -262,6 +263,15 @@ vbf_stp_retry(struct worker *wrk, struct busyobj *bo)
 
 	assert(bo->fetch_objcore->boc->state <= BOS_REQ_DONE);
 
+	if (bo->fetch_objcore->boc->state == BOS_REQ_DONE &&
+	    bo->initial_req_body_status != REQ_BODY_NONE &&
+	    bo->bereq_body == NULL) {
+		/* We have already released the req and there was a
+		 * request body that was not cached. Too late to retry. */
+		VSLb(bo->vsl, SLT_Error, "req.body already consumed");
+		return (F_STP_FAIL);
+	}
+
 	VSLb_ts_busyobj(bo, "Retry", W_TIM_real(wrk));
 
 	/* VDI_Finish (via vbf_cleanup) must have been called before */
diff --git a/bin/varnishtest/tests/r03093.vtc b/bin/varnishtest/tests/r03093.vtc
new file mode 100644
index 000000000..f9aaa2a7c
--- /dev/null
+++ b/bin/varnishtest/tests/r03093.vtc
@@ -0,0 +1,45 @@
+varnishtest "r03093 - fail retry on missing req body"
+
+barrier b1 sock 2
+
+server s1 {
+	rxreq
+	expect req.method == POST
+	expect req.body == foo
+	txresp -nolen -hdr "Content-Length: 3"
+	barrier b1 sync
+} -start
+
+# In this test s2 should not be called. The attempt to retry should fail because
+# the request was already released from the fetch thread in the first attempt.
+server s2 {
+	rxreq
+	expect req.method == POST
+	expect req.body == foo
+	txresp -body bar
+} -start
+
+varnish v1 -arg "-p debug=+syncvsl" -vcl+backend {
+	import vtc;
+	sub vcl_backend_fetch {
+		set bereq.http.retries = bereq.retries;
+		if (bereq.retries == 1) {
+			set bereq.backend = s2;
+		}
+	}
+	sub vcl_backend_response {
+		if (bereq.http.retries == "0") {
+			vtc.barrier_sync("${b1_sock}");
+		}
+		set beresp.do_stream = false;
+	}
+	sub vcl_backend_error {
+		return (retry);
+	}
+} -start
+
+client c1 {
+	txreq -req POST -body foo
+	rxresp
+	expect resp.status == 503
+} -run

From reza at naghibi.com  Wed Aug 19 13:12:08 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:12:08 +0000 (UTC)
Subject: [6.0] bb9d7d6f5 Fetch thread reference count and keep cached request
 bodies
Message-ID: <20200819131208.085395336@lists.varnish-cache.org>


commit bb9d7d6f5c6737998a97d81269341baf7cf96249
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Nov 4 11:32:59 2019 +0100

    Fetch thread reference count and keep cached request bodies
    
    With this patch fetch threads will for completely cached request bodies
    keep a reference to it for the entire duration of the fetch. This extends
    the retry window of backend requests with request body beyond the
    BOS_REQ_DONE point.
    
    Patch by: Poul-Henning Kamp

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index cb4f1ba79..95d6d97b1 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -403,6 +403,7 @@ struct busyobj {
 	struct http		*bereq0;
 	struct http		*bereq;
 	struct http		*beresp;
+	struct objcore		*bereq_body;
 	struct objcore		*stale_oc;
 	struct objcore		*fetch_objcore;
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 9962e1c9e..6381bcfaf 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -246,6 +246,12 @@ vbf_stp_mkbereq(struct worker *wrk, struct busyobj *bo)
 	if (bo->req->req_body_status == REQ_BODY_NONE) {
 		bo->req = NULL;
 		ObjSetState(bo->wrk, bo->fetch_objcore, BOS_REQ_DONE);
+	} else if (bo->req->req_body_status == REQ_BODY_CACHED) {
+		AN(bo->req->body_oc);
+		bo->bereq_body = bo->req->body_oc;
+		HSH_Ref(bo->bereq_body);
+		bo->req = NULL;
+		ObjSetState(bo->wrk, bo->fetch_objcore, BOS_REQ_DONE);
 	}
 	return (F_STP_STARTFETCH);
 }
@@ -1019,6 +1025,8 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 	VCL_TaskLeave(bo->vcl, bo->privs);
 	http_Teardown(bo->bereq);
 	http_Teardown(bo->beresp);
+	if (bo->bereq_body != NULL)
+		HSH_DerefObjCore(bo->wrk, &bo->bereq_body, 0);
 
 	if (bo->fetch_objcore->boc->state == BOS_FINISHED) {
 		AZ(bo->fetch_objcore->flags & OC_F_FAILED);
diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c
index 879115b63..860f5ee4b 100644
--- a/bin/varnishd/http1/cache_http1_fetch.c
+++ b/bin/varnishd/http1/cache_http1_fetch.c
@@ -105,7 +105,14 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes,
 	/* Deal with any message-body the request might (still) have */
 	i = 0;
 
-	if (bo->req != NULL &&
+	if (bo->bereq_body != NULL) {
+		if (do_chunked)
+			V1L_Chunked(wrk);
+		(void)ObjIterate(bo->wrk, bo->bereq_body,
+		    bo, vbf_iter_req_body, 0);
+		if (do_chunked)
+			V1L_EndChunk(wrk);
+	} else if (bo->req != NULL &&
 	    (bo->req->req_body_status == REQ_BODY_CACHED || !onlycached)) {
 		if (do_chunked)
 			V1L_Chunked(wrk);

From reza at naghibi.com  Wed Aug 19 13:12:08 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:12:08 +0000 (UTC)
Subject: [6.0] 0644d58ab A request body now might have up to two references
Message-ID: <20200819131208.1F874533A@lists.varnish-cache.org>


commit 0644d58abe07a18a972ead14e4aa304c80f354e7
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jan 15 18:13:45 2020 +0100

    A request body now might have up to two references
    
    since d4b6228e1e32cda3014eeff0a5cb60cd7a0b5474 the busyobj might also
    gain one, so depending on who deref's first, there may be one or zero
    references left.

diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c
index 40f698be3..f411fc66a 100644
--- a/bin/varnishd/cache/cache_req_body.c
+++ b/bin/varnishd/cache/cache_req_body.c
@@ -263,11 +263,17 @@ VRB_Ignore(struct req *req)
 void
 VRB_Free(struct req *req)
 {
+	int r;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 
-	if (req->body_oc != NULL)
-		AZ(HSH_DerefObjCore(req->wrk, &req->body_oc, 0));
+	if (req->body_oc == NULL)
+		return;
+
+	r = HSH_DerefObjCore(req->wrk, &req->body_oc, 0);
+
+	// a busyobj may have gained a reference
+	assert (r == 0 || r == 1);
 }
 
 /*----------------------------------------------------------------------

From reza at naghibi.com  Wed Aug 19 13:14:06 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:14:06 +0000 (UTC)
Subject: [6.0] 6cb9c7669 Handle unformatable VCL_TIME to string conversion
 failures
Message-ID: <20200819131406.C3CF25954@lists.varnish-cache.org>


commit 6cb9c7669a042fd6614e83298f323d0814b0e51d
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed May 6 11:46:55 2020 +0200

    Handle unformatable VCL_TIME to string conversion failures
    
    For VCL_TIME values that would convert to a year element that can not fit
    in an int, gmtime_r would fail, and VTIM_format() would use random stack
    values when picking weekday and month strings.
    
    This patch changes VTIM_format to return "" when gmtime_r reports
    failures. This way the API is not changed. Callers can test for empty
    string to catch the failure if needed.
    
    VRT_TIME_string is patched to catch the VTIM_format error, and return NULL
    on failure.
    
    Fixes: #3308

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 4f34fba44..8eed99e4c 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -629,8 +629,11 @@ VRT_TIME_string(VRT_CTX, VCL_TIME t)
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	p = WS_Alloc(ctx->ws, VTIM_FORMAT_SIZE);
-	if (p != NULL)
+	if (p != NULL) {
 		VTIM_format(t, p);
+		if (*p == '\0')
+			p = NULL;
+	}
 	return (p);
 }
 
diff --git a/bin/varnishtest/tests/r03308.vtc b/bin/varnishtest/tests/r03308.vtc
new file mode 100644
index 000000000..a97cb0cd5
--- /dev/null
+++ b/bin/varnishtest/tests/r03308.vtc
@@ -0,0 +1,20 @@
+varnishtest "Unformatable VCL_TIME"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import std;
+
+	sub vcl_deliver {
+		set resp.http.ts = std.real2time(std.real("1e+22", 0), now);
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.http.ts == ""
+} -run
diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index 14b9752d6..dc6e01e31 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -165,11 +165,16 @@ VTIM_format(vtim_real t, char *p)
 	struct tm tm;
 	time_t tt;
 
+	AN(p);
 	tt = (time_t) t;
-	(void)gmtime_r(&tt, &tm);
-	AN(snprintf(p, VTIM_FORMAT_SIZE, "%s, %02d %s %4d %02d:%02d:%02d GMT",
-	    weekday_name[tm.tm_wday], tm.tm_mday, month_name[tm.tm_mon],
-	    tm.tm_year + 1900, tm.tm_hour, tm.tm_min, tm.tm_sec));
+	if (gmtime_r(&tt, &tm) != NULL)
+		AN(snprintf(p, VTIM_FORMAT_SIZE,
+			"%s, %02d %s %4d %02d:%02d:%02d GMT",
+			weekday_name[tm.tm_wday],
+			tm.tm_mday, month_name[tm.tm_mon],
+			tm.tm_year + 1900, tm.tm_hour, tm.tm_min, tm.tm_sec));
+	else
+		*p = '\0';
 }
 
 #ifdef TEST_DRIVER

From reza at naghibi.com  Wed Aug 19 13:14:06 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:14:06 +0000 (UTC)
Subject: [6.0] 9bf79d4ec Don't waste workspace on format failure
Message-ID: <20200819131406.DD8D25958@lists.varnish-cache.org>


commit 9bf79d4ec4e961d6100871e0185544f2d6de7fb6
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed May 6 15:00:38 2020 +0200

    Don't waste workspace on format failure

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 8eed99e4c..20d855478 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -626,13 +626,17 @@ VCL_STRING v_matchproto_()
 VRT_TIME_string(VRT_CTX, VCL_TIME t)
 {
 	char *p;
+	uintptr_t snapshot;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	snapshot = WS_Snapshot(ctx->ws);
 	p = WS_Alloc(ctx->ws, VTIM_FORMAT_SIZE);
 	if (p != NULL) {
 		VTIM_format(t, p);
-		if (*p == '\0')
+		if (*p == '\0') {
 			p = NULL;
+			WS_Reset(ctx->ws, snapshot);
+		}
 	}
 	return (p);
 }

From reza at naghibi.com  Wed Aug 19 13:15:07 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:15:07 +0000 (UTC)
Subject: [6.0] 4da4a8e3a Remove extraneous VSB_new()
Message-ID: <20200819131507.875BA5C87@lists.varnish-cache.org>


commit 4da4a8e3afe5ae0c115195ffb8b86dadbdb317dc
Author: Reza Naghibi <reza at naghibi.com>
Date:   Wed Aug 12 11:32:10 2020 -0400

    Remove extraneous VSB_new()
    
    This came from the H2 backport c3902d57ff253eb7bf90f067c98e40f3bc5cad70
    Fixes #3382

diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c
index ee9f40cb2..304dab2de 100644
--- a/bin/varnishd/http2/cache_http2_deliver.c
+++ b/bin/varnishd/http2/cache_http2_deliver.c
@@ -230,7 +230,6 @@ h2_build_headers(struct vsb *resp, struct req *req)
 	ssize_t sz, sz1;
 
 	l = WS_ReserveAll(req->ws);
-	AN(VSB_new(resp, req->ws->f, l, VSB_FIXEDLEN));
 	if (l < 10) {
 		WS_Release(req->ws, 0);
 		return (-1);

From reza at naghibi.com  Wed Aug 19 13:15:07 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:15:07 +0000 (UTC)
Subject: [6.0] 46d4d5941 Revert "Fix WS_Reserve() error handling"
Message-ID: <20200819131507.A574B5C8A@lists.varnish-cache.org>


commit 46d4d594157583cab7886b032aa11ff848ff7c51
Author: Reza Naghibi <reza at naghibi.com>
Date:   Mon Aug 17 10:11:54 2020 -0400

    Revert "Fix WS_Reserve() error handling"
    
    This reverts commit 0e32f1667ea55156fc2a7fa84e3f1080a05bcb2e.
    
     Conflicts:
            bin/varnishd/http2/cache_http2_deliver.c

diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c
index 304dab2de..ceb2b46c3 100644
--- a/bin/varnishd/http2/cache_http2_deliver.c
+++ b/bin/varnishd/http2/cache_http2_deliver.c
@@ -230,10 +230,8 @@ h2_build_headers(struct vsb *resp, struct req *req)
 	ssize_t sz, sz1;
 
 	l = WS_ReserveAll(req->ws);
-	if (l < 10) {
-		WS_Release(req->ws, 0);
+	if (l < 10)
 		return (-1);
-	}
 
 	AN(VSB_new(resp, req->ws->f, l, VSB_FIXEDLEN));
 

From reza at naghibi.com  Wed Aug 19 13:15:07 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:15:07 +0000 (UTC)
Subject: [6.0] 18fa60f75 Add vtc covering H2 out of workspace
Message-ID: <20200819131507.C0A615C8E@lists.varnish-cache.org>


commit 18fa60f759d18517dc07cbc220ce7572135997ac
Author: Reza Naghibi <reza at naghibi.com>
Date:   Tue Aug 18 18:11:03 2020 -0400

    Add vtc covering H2 out of workspace

diff --git a/bin/varnishtest/tests/r03384.vtc b/bin/varnishtest/tests/r03384.vtc
new file mode 100644
index 000000000..1a00c16e0
--- /dev/null
+++ b/bin/varnishtest/tests/r03384.vtc
@@ -0,0 +1,25 @@
+varnishtest "h/2 delivery double WS_Release"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -cliok "param.set feature +http2"
+
+varnish v1 -vcl+backend {
+	import vtc;
+
+	sub vcl_deliver {
+		vtc.workspace_alloc(client, -5);
+	}
+
+} -start
+
+client c1 {
+	stream 1 {
+		txreq
+		rxresp
+		expect resp.status == 500
+	} -run
+} -run

From reza at naghibi.com  Wed Aug 19 13:17:06 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:06 +0000 (UTC)
Subject: [6.0] e966c61cd Flexelinting
Message-ID: <20200819131706.2F2416217@lists.varnish-cache.org>


commit e966c61cd6490492fcf319183f207a1d713896c5
Author: Reza Naghibi <reza at naghibi.com>
Date:   Fri Jun 12 10:15:03 2020 -0400

    Flexelinting
    
    ebbbad96484f2ec6823911f460ad5a1107c8a9c3
    
    Co-authored-by: Poul-Henning Kamp <phk at FreeBSD.org>

diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c
index 54735953d..ab6d70380 100644
--- a/lib/libvmod_directors/shard_cfg.c
+++ b/lib/libvmod_directors/shard_cfg.c
@@ -42,7 +42,7 @@
 
 /*lint -esym(749,  shard_change_task_e::*) */
 enum shard_change_task_e {
-	_INVALID = 0,
+	_SHARD_TASK_E_INVALID = 0,
 	CLEAR,
 	ADD_BE,
 	REMOVE_BE,

From reza at naghibi.com  Wed Aug 19 13:17:06 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:06 +0000 (UTC)
Subject: [6.0] 673ba3a7d Kill goto statements in vmod-shard
Message-ID: <20200819131706.493AE621B@lists.varnish-cache.org>


commit 673ba3a7df1218f629a487b40981f07cf1da2e59
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Feb 25 18:49:06 2019 +0100

    Kill goto statements in vmod-shard
    
    With that, (almost) only libvgz carries goto statements from zlib.
    
    This isn't changing any of the previous semantics, in particular the
    AN(be) assertion from the "ok:" jump is honored by all code paths
    formerly leading to it.
    
    Previously, the bitmap was allocated on the stack prior to the magic
    check of shardd, which is now fixed at the expense of a potential code
    style violation. But more importantly, we currently read the number of
    backends prior to acquiring the read lock, but there is no evidence
    that this was done on purpose and not overlooked, besides moving a
    possibly expensive state initialization outside of the critical
    section.
    
    If that was on purpose, please document it.

diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index a936e08d6..be240eb70 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -331,114 +331,135 @@ sharddir_any_healthy(struct sharddir *shardd, const struct busyobj *bo,
 	sharddir_unlock(shardd);
 	return (retval);
 }
+
 /*
  * core function for the director backend/resolve method
  */
-VCL_BACKEND
-sharddir_pick_be(VRT_CTX, struct sharddir *shardd,
+
+static VCL_BACKEND
+sharddir_pick_be_locked(VRT_CTX, struct sharddir *shardd,
     uint32_t key, VCL_INT alt, VCL_REAL warmup, VCL_BOOL rampup,
-    enum healthy_e healthy)
+    enum healthy_e healthy, struct shard_state *state)
 {
 	VCL_BACKEND be;
-	struct shard_state state;
-	unsigned picklist_sz = VBITMAP_SZ(shardd->n_backend);
-	char picklist_spc[picklist_sz];
 	VCL_DURATION chosen_r, alt_r;
 
 	CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	AN(ctx->vsl);
 
-	memset(&state, 0, sizeof(state));
-	init_state(&state, ctx, shardd, vbit_init(picklist_spc, picklist_sz));
-
-	sharddir_rdlock(shardd);
 	if (shardd->n_backend == 0) {
 		shard_err0(ctx, shardd, "no backends");
-		goto err;
+		return (NULL);
 	}
 
 	assert(shardd->hashcircle);
 
 	validate_alt(ctx, shardd, &alt);
 
-	state.idx = shard_lookup(shardd, key);
-	assert(state.idx >= 0);
+	state->idx = shard_lookup(shardd, key);
+	assert(state->idx >= 0);
 
 	SHDBG(SHDBG_LOOKUP, shardd, "lookup key %x idx %d host %u",
-	    key, state.idx, shardd->hashcircle[state.idx].host);
+	    key, state->idx, shardd->hashcircle[state->idx].host);
 
 	if (alt > 0) {
-		if (shard_next(&state, alt - 1, healthy == ALL ? 1 : 0) == -1) {
-			if (state.previous.hostid != -1) {
+		if (shard_next(state, alt - 1, healthy == ALL ? 1 : 0) == -1) {
+			if (state->previous.hostid != -1) {
 				be = sharddir_backend(shardd,
-				    state.previous.hostid);
-				goto ok;
+				    state->previous.hostid);
+				AN(be);
+				return (be);
 			}
-			goto err;
+			return (NULL);
 		}
 	}
 
-	if (shard_next(&state, 0, healthy == IGNORE ? 0 : 1) == -1) {
-		if (state.previous.hostid != -1) {
-			be = sharddir_backend(shardd, state.previous.hostid);
-			goto ok;
+	if (shard_next(state, 0, healthy == IGNORE ? 0 : 1) == -1) {
+		if (state->previous.hostid != -1) {
+			be = sharddir_backend(shardd, state->previous.hostid);
+			AN(be);
+			return (be);
 		}
-		goto err;
+		return (NULL);
 	}
 
-	be = sharddir_backend(shardd, state.last.hostid);
+	be = sharddir_backend(shardd, state->last.hostid);
+	AN(be);
 
 	if (warmup == -1)
 		warmup = shardd->warmup;
 
 	/* short path for cases we dont want ramup/warmup or can't */
 	if (alt > 0 || healthy == IGNORE || (!rampup && warmup == 0) ||
-	    shard_next(&state, 0, 1) == -1)
-		goto ok;
+	    shard_next(state, 0, 1) == -1)
+		return (be);
 
 	assert(alt == 0);
-	assert(state.previous.hostid >= 0);
-	assert(state.last.hostid >= 0);
-	assert(state.previous.hostid != state.last.hostid);
-	assert(be == sharddir_backend(shardd, state.previous.hostid));
+	assert(state->previous.hostid >= 0);
+	assert(state->last.hostid >= 0);
+	assert(state->previous.hostid != state->last.hostid);
+	assert(be == sharddir_backend(shardd, state->previous.hostid));
 
-	chosen_r = shardcfg_get_rampup(shardd, state.previous.hostid);
-	alt_r = shardcfg_get_rampup(shardd, state.last.hostid);
+	chosen_r = shardcfg_get_rampup(shardd, state->previous.hostid);
+	alt_r = shardcfg_get_rampup(shardd, state->last.hostid);
 
 	SHDBG(SHDBG_RAMPWARM, shardd, "chosen host %d rampup %f changed %f",
-	    state.previous.hostid, chosen_r,
-	    ctx->now - state.previous.changed);
+	    state->previous.hostid, chosen_r,
+	    ctx->now - state->previous.changed);
 	SHDBG(SHDBG_RAMPWARM, shardd, "alt host %d rampup %f changed %f",
-	    state.last.hostid, alt_r,
-	    ctx->now - state.last.changed);
+	    state->last.hostid, alt_r,
+	    ctx->now - state->last.changed);
 
-	if (ctx->now - state.previous.changed < chosen_r) {
+	if (ctx->now - state->previous.changed < chosen_r) {
 		/*
 		 * chosen host is in rampup
 		 * - no change if alternative host is also in rampup or the dice
 		 *   has rolled in favour of the chosen host
 		 */
 		if (!rampup ||
-		    ctx->now - state.last.changed < alt_r ||
+		    ctx->now - state->last.changed < alt_r ||
 		    VRND_RandomTestableDouble() * chosen_r <
-		     (ctx->now - state.previous.changed))
-			goto ok;
+		    (ctx->now - state->previous.changed))
+			return (be);
 	} else {
 		/* chosen host not in rampup - warmup ? */
 		if (warmup == 0 || VRND_RandomTestableDouble() > warmup)
-			goto ok;
+			return (be);
 	}
 
-	be = sharddir_backend(shardd, state.last.hostid);
+	be = sharddir_backend(shardd, state->last.hostid);
+	return (be);
+}
 
-  ok:
-	AN(be);
+VCL_BACKEND
+sharddir_pick_be(VRT_CTX, struct sharddir *shardd,
+    uint32_t key, VCL_INT alt, VCL_REAL warmup, VCL_BOOL rampup,
+    enum healthy_e healthy)
+{
+	VCL_BACKEND be;
+	struct shard_state state[1];
+	unsigned picklist_sz;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC);
+
+	/* NB: Allocate bitmap on the stack
+	 *
+	 * XXX: Should we read n_backend under the lock and check whether
+	 * n_backend is zero before allocating the state?
+	 */
+	picklist_sz = VBITMAP_SZ(shardd->n_backend);
+	char picklist_spc[picklist_sz];
+
+	memset(state, 0, sizeof(state));
+	init_state(state, ctx, shardd, vbit_init(picklist_spc, picklist_sz));
+
+	sharddir_rdlock(shardd);
+	be = sharddir_pick_be_locked(ctx, shardd, key, alt, warmup, rampup,
+	    healthy, state);
 	sharddir_unlock(shardd);
-	vbit_destroy(state.picklist);
+
+	vbit_destroy(state->picklist);
 	return (be);
-  err:
-	sharddir_unlock(shardd);
-	vbit_destroy(state.picklist);
-	return NULL;
 }

From reza at naghibi.com  Wed Aug 19 13:17:06 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:06 +0000 (UTC)
Subject: [6.0] 82297b9cb complement the cleanup started in the previous commit
Message-ID: <20200819131706.6242B621F@lists.varnish-cache.org>


commit 82297b9cbcc85052c1157c9f40713ff83d101705
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Feb 25 19:12:07 2019 +0100

    complement the cleanup started in the previous commit
    
    Thank you, @Dridi
    
    - yes, allocations outside the lock were motivated by minimizing the
      critical section, but n_backend could actually change, so this was
      wrong. As we use an RW lock, doing more work under it should only have
      marginal impact.
    
    - n_backend == 0 is probably best handled as a special case

diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index be240eb70..d17cb6e4c 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -347,11 +347,7 @@ sharddir_pick_be_locked(VRT_CTX, struct sharddir *shardd,
 	CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	AN(ctx->vsl);
-
-	if (shardd->n_backend == 0) {
-		shard_err0(ctx, shardd, "no backends");
-		return (NULL);
-	}
+	assert(shardd->n_backend > 0);
 
 	assert(shardd->hashcircle);
 
@@ -444,18 +440,20 @@ sharddir_pick_be(VRT_CTX, struct sharddir *shardd,
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC);
 
-	/* NB: Allocate bitmap on the stack
-	 *
-	 * XXX: Should we read n_backend under the lock and check whether
-	 * n_backend is zero before allocating the state?
-	 */
+	sharddir_rdlock(shardd);
+
+	if (shardd->n_backend == 0) {
+		shard_err0(ctx, shardd, "no backends");
+		sharddir_unlock(shardd);
+		return (NULL);
+	}
+
 	picklist_sz = VBITMAP_SZ(shardd->n_backend);
 	char picklist_spc[picklist_sz];
 
 	memset(state, 0, sizeof(state));
 	init_state(state, ctx, shardd, vbit_init(picklist_spc, picklist_sz));
 
-	sharddir_rdlock(shardd);
 	be = sharddir_pick_be_locked(ctx, shardd, key, alt, warmup, rampup,
 	    healthy, state);
 	sharddir_unlock(shardd);

From reza at naghibi.com  Wed Aug 19 13:17:06 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:06 +0000 (UTC)
Subject: [6.0] 1e8215ca2 Flexelint constification
Message-ID: <20200819131706.79B986227@lists.varnish-cache.org>


commit 1e8215ca2fdd3e8ae745805f5a4d6a6128eb6d68
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Feb 25 22:06:55 2019 +0000

    Flexelint constification

diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index d17cb6e4c..c9b38ee43 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -337,7 +337,7 @@ sharddir_any_healthy(struct sharddir *shardd, const struct busyobj *bo,
  */
 
 static VCL_BACKEND
-sharddir_pick_be_locked(VRT_CTX, struct sharddir *shardd,
+sharddir_pick_be_locked(VRT_CTX, const struct sharddir *shardd,
     uint32_t key, VCL_INT alt, VCL_REAL warmup, VCL_BOOL rampup,
     enum healthy_e healthy, struct shard_state *state)
 {

From reza at naghibi.com  Wed Aug 19 13:17:06 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:06 +0000 (UTC)
Subject: [6.0] f0065411b Whitespace OCD
Message-ID: <20200819131706.94197622B@lists.varnish-cache.org>


commit f0065411b28d36da9ee294a317231aa6ad9325ce
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Feb 26 10:06:21 2019 +0100

    Whitespace OCD

diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index c9b38ee43..9d4976aab 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -337,9 +337,9 @@ sharddir_any_healthy(struct sharddir *shardd, const struct busyobj *bo,
  */
 
 static VCL_BACKEND
-sharddir_pick_be_locked(VRT_CTX, const struct sharddir *shardd,
-    uint32_t key, VCL_INT alt, VCL_REAL warmup, VCL_BOOL rampup,
-    enum healthy_e healthy, struct shard_state *state)
+sharddir_pick_be_locked(VRT_CTX, const struct sharddir *shardd, uint32_t key,
+    VCL_INT alt, VCL_REAL warmup, VCL_BOOL rampup, enum healthy_e healthy,
+    struct shard_state *state)
 {
 	VCL_BACKEND be;
 	VCL_DURATION chosen_r, alt_r;
@@ -429,9 +429,8 @@ sharddir_pick_be_locked(VRT_CTX, const struct sharddir *shardd,
 }
 
 VCL_BACKEND
-sharddir_pick_be(VRT_CTX, struct sharddir *shardd,
-    uint32_t key, VCL_INT alt, VCL_REAL warmup, VCL_BOOL rampup,
-    enum healthy_e healthy)
+sharddir_pick_be(VRT_CTX, struct sharddir *shardd, uint32_t key, VCL_INT alt,
+    VCL_REAL warmup, VCL_BOOL rampup, enum healthy_e healthy)
 {
 	VCL_BACKEND be;
 	struct shard_state state[1];

From reza at naghibi.com  Wed Aug 19 13:17:06 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:06 +0000 (UTC)
Subject: [6.0] 1e3cfe582 directors.shard: fix hole handling during
 recondiguration
Message-ID: <20200819131706.ABD91622F@lists.varnish-cache.org>


commit 1e3cfe582bbbcf1cee1cbb021eb4486a170e8192
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Mar 7 11:51:57 2019 +0100

    directors.shard: fix hole handling during recondiguration

diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c
index ab6d70380..56127e28b 100644
--- a/lib/libvmod_directors/shard_cfg.c
+++ b/lib/libvmod_directors/shard_cfg.c
@@ -353,10 +353,15 @@ shardcfg_backend_lookup(const struct backend_reconfig *re,
 	unsigned i, max = re->shardd->n_backend + re->hole_n;
 	const struct shard_backend *bb = re->shardd->backend;
 
-	for (i = 0; i < max; i++)
+	if (max > 0)
+		AN(bb);
+
+	for (i = 0; i < max; i++) {
+		if (bb[i].backend == NULL)
+			continue;	// hole
 		if (!shardcfg_backend_cmp(b, &bb[i]))
 			return &bb[i];
-
+	}
 	return NULL;
 }
 
@@ -432,6 +437,8 @@ shardcfg_backend_del(struct backend_reconfig *re,
 	struct shard_backend * const bb = re->shardd->backend;
 
 	for (i = 0; i < max; i++) {
+		if (bb[i].backend == NULL)
+			continue;	// hole
 		if (shardcfg_backend_del_cmp(spec, &bb[i]))
 			continue;
 

From reza at naghibi.com  Wed Aug 19 13:17:06 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:06 +0000 (UTC)
Subject: [6.0] 3808881e1 Style(9) and whitespace OCD
Message-ID: <20200819131706.C5F5A6233@lists.varnish-cache.org>


commit 3808881e1f8fb9e60221ac85a211a1e3fac226c5
Author: Reza Naghibi <reza at naghibi.com>
Date:   Fri Jun 12 10:47:54 2020 -0400

    Style(9) and whitespace OCD
    
    dd28fc739f47d08a95b3f29e39ec2db73d2fe79f
    
    Co-authored-by: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>

diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c
index 56127e28b..b471e61eb 100644
--- a/lib/libvmod_directors/shard_cfg.c
+++ b/lib/libvmod_directors/shard_cfg.c
@@ -97,7 +97,7 @@ shard_change_get(VRT_CTX, struct vmod_priv *priv,
 			shard_err0(ctx, shardd,
 			    "cannot change more than one shard director "
 			    "at a time");
-			return NULL;
+			return (NULL);
 		}
 		return (change);
 	}
@@ -105,7 +105,7 @@ shard_change_get(VRT_CTX, struct vmod_priv *priv,
 	change = WS_Alloc(ctx->ws, sizeof(*change));
 	if (change == NULL) {
 		shard_err0(ctx, shardd, "could not get workspace");
-		return NULL;
+		return (NULL);
 	}
 
 	INIT_OBJ(change, SHARD_CHANGE_MAGIC);
@@ -160,13 +160,13 @@ shard_change_task_backend(VRT_CTX,
 
 	change = shard_change_get(ctx, priv, shardd);
 	if (change == NULL)
-		return 0;
+		return (0);
 
 	b = WS_Alloc(ctx->ws, sizeof(*b));
 	if (b == NULL) {
 		shard_err(ctx, shardd, ".%s_backend() WS_Alloc() failed",
 		    task_e == ADD_BE ? "add" : "remove");
-		return 0;
+		return (0);
 	}
 
 	b->backend = be;
@@ -175,7 +175,7 @@ shard_change_task_backend(VRT_CTX,
 
 	shard_change_task_add(ctx, change, task_e, b);
 
-	return 1;
+	return (1);
 }
 
 /*
@@ -188,16 +188,16 @@ shardcfg_add_backend(VRT_CTX, struct vmod_priv *priv,
     VCL_DURATION rampup)
 {
 	AN(be);
-	return shard_change_task_backend(ctx, priv, shardd, ADD_BE,
-	    be, ident, rampup);
+	return (shard_change_task_backend(ctx, priv, shardd, ADD_BE,
+	    be, ident, rampup));
 }
 
 VCL_BOOL
 shardcfg_remove_backend(VRT_CTX, struct vmod_priv *priv,
     const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident)
 {
-	return shard_change_task_backend(ctx, priv, shardd, REMOVE_BE,
-	    be, ident, 0);
+	return (shard_change_task_backend(ctx, priv, shardd, REMOVE_BE,
+	    be, ident, 0));
 }
 
 VCL_BOOL
@@ -209,11 +209,11 @@ shardcfg_clear(VRT_CTX, struct vmod_priv *priv, const struct sharddir *shardd)
 
 	change = shard_change_get(ctx, priv, shardd);
 	if (change == NULL)
-		return 0;
+		return (0);
 
 	shard_change_task_add(ctx, change, CLEAR, NULL);
 
-	return 1;
+	return (1);
 }
 
 /*
@@ -227,7 +227,7 @@ static int
 circlepoint_compare(const struct shard_circlepoint *a,
     const struct shard_circlepoint *b)
 {
-	return (a->point == b->point) ? 0 : ((a->point > b->point) ? 1 : -1);
+	return ((a->point == b->point) ? 0 : ((a->point > b->point) ? 1 : -1));
 }
 
 static void
@@ -322,7 +322,7 @@ shardcfg_backend_cmp(const struct shard_backend *a,
 
 	/* vcl_names are unique, so we can compare the backend pointers */
 	if (ai == NULL && bi == NULL)
-		return a->backend != b->backend;
+		return (a->backend != b->backend);
 
 	if (ai == NULL)
 		ai = a->backend->vcl_name;
@@ -330,7 +330,7 @@ shardcfg_backend_cmp(const struct shard_backend *a,
 	if (bi == NULL)
 		bi = b->backend->vcl_name;
 
-	return strcmp(ai, bi);
+	return (strcmp(ai, bi));
 }
 
 /* for removal, we delete all instances if the backend matches */
@@ -341,9 +341,9 @@ shardcfg_backend_del_cmp(const struct shard_backend *task,
 	assert(task->backend || task->ident);
 
 	if (task->ident == NULL)
-		return task->backend != b->backend;
+		return (task->backend != b->backend);
 
-	return shardcfg_backend_cmp(task, b);
+	return (shardcfg_backend_cmp(task, b));
 }
 
 static const struct shard_backend *
@@ -360,9 +360,9 @@ shardcfg_backend_lookup(const struct backend_reconfig *re,
 		if (bb[i].backend == NULL)
 			continue;	// hole
 		if (!shardcfg_backend_cmp(b, &bb[i]))
-			return &bb[i];
+			return (&bb[i]);
 	}
-	return NULL;
+	return (NULL);
 }
 
 static void
@@ -585,15 +585,15 @@ shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv,
 	if (replicas <= 0) {
 		shard_err(ctx, shardd,
 		    ".reconfigure() invalid replicas argument %ld", replicas);
-		return 0;
+		return (0);
 	}
 
 	change = shard_change_get(ctx, priv, shardd);
 	if (change == NULL)
-		return 0;
+		return (0);
 
 	if (VSTAILQ_FIRST(&change->tasks) == NULL)
-		return 1;
+		return (1);
 
 	sharddir_wrlock(shardd);
 
@@ -607,7 +607,7 @@ shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv,
 	if (shardd->n_backend == 0) {
 		shard_err0(ctx, shardd, ".reconfigure() no backends");
 		sharddir_unlock(shardd);
-		return 0;
+		return (0);
 	}
 
 	shardcfg_hashcircle(shardd, replicas);
diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index 9d4976aab..f18c2131c 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -156,7 +156,7 @@ shard_lookup(const struct sharddir *shardd, const uint32_t key)
 		low = i;
 	} while (idx == -1);
 
-	return idx;
+	return (idx);
 }
 
 static int
@@ -173,7 +173,7 @@ shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy)
 	CHECK_OBJ_NOTNULL(state->shardd, SHARDDIR_MAGIC);
 
 	if (state->pickcount >= state->shardd->n_backend)
-		return -1;
+		return (-1);
 
 	ringsz = state->shardd->n_backend * state->shardd->replicas;
 
@@ -218,7 +218,7 @@ shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy)
 		if (++(state->idx) == ringsz)
 			state->idx = 0;
 	}
-	return chosen;
+	return (chosen);
 }
 
 void

From reza at naghibi.com  Wed Aug 19 13:17:06 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:06 +0000 (UTC)
Subject: [6.0] aa7ab50e7 It is OK to call realloc(3) with a NULL pointer
Message-ID: <20200819131706.DE5EB6243@lists.varnish-cache.org>


commit aa7ab50e73f9dd48ab5d615ba303d02087e51e7e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Apr 18 08:39:40 2019 +0000

    It is OK to call realloc(3) with a NULL pointer

diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c
index b471e61eb..c2e0a326b 100644
--- a/lib/libvmod_directors/shard_cfg.c
+++ b/lib/libvmod_directors/shard_cfg.c
@@ -380,12 +380,8 @@ shardcfg_backend_expand(const struct backend_reconfig *re)
 	else
 		re->shardd->l_backend <<= 1;
 
-	if (re->shardd->backend)
-		re->shardd->backend = realloc(re->shardd->backend,
-		    re->shardd->l_backend * sizeof *re->shardd->backend);
-	else
-		re->shardd->backend = malloc(
-		    re->shardd->l_backend * sizeof *re->shardd->backend);
+	re->shardd->backend = realloc(re->shardd->backend,
+	    re->shardd->l_backend * sizeof *re->shardd->backend);
 
 	AN(re->shardd->backend);
 }

From reza at naghibi.com  Wed Aug 19 13:17:06 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:06 +0000 (UTC)
Subject: [6.0] b0fb538fe GC unused function
Message-ID: <20200819131707.00C456256@lists.varnish-cache.org>


commit b0fb538fe22f60de83fd3271233b3312155554a8
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Sat Apr 20 16:42:23 2019 +0100

    GC unused function

diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h
index e3a8b3128..1daae58ca 100644
--- a/lib/libvmod_directors/shard_dir.h
+++ b/lib/libvmod_directors/shard_dir.h
@@ -95,14 +95,6 @@ sharddir_backend(const struct sharddir *shardd, int id)
 	return (shardd->backend[id].backend);
 }
 
-static inline const char *
-sharddir_backend_ident(const struct sharddir *shardd, int host)
-{
-	assert(host >= 0);
-	assert(host < shardd->n_backend);
-	return (shardd->backend[host].ident);
-}
-
 #define SHDBG(flag, shardd, ...)					\
 	do {								\
 		if ((shardd)->debug_flags & (flag))			\

From reza at naghibi.com  Wed Aug 19 13:17:07 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:07 +0000 (UTC)
Subject: [6.0] f52e298f1 FlexeLinting: Be more careful about signed/unsigned
Message-ID: <20200819131707.1B4796262@lists.varnish-cache.org>


commit f52e298f107283efbab3881eb6a38fe29b1933d7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 29 07:20:10 2019 +0000

    FlexeLinting: Be more careful about signed/unsigned
    
     Conflicts:
            lib/libvmod_directors/shard_dir.h
            lib/libvmod_directors/vmod_shard.c

diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c
index c2e0a326b..a198f0c85 100644
--- a/lib/libvmod_directors/shard_cfg.c
+++ b/lib/libvmod_directors/shard_cfg.c
@@ -368,7 +368,7 @@ shardcfg_backend_lookup(const struct backend_reconfig *re,
 static void
 shardcfg_backend_expand(const struct backend_reconfig *re)
 {
-	unsigned min = re->hint;
+	int min = re->hint;
 
 	CHECK_OBJ_NOTNULL(re->shardd, SHARDDIR_MAGIC);
 
@@ -378,7 +378,7 @@ shardcfg_backend_expand(const struct backend_reconfig *re)
 	if (re->shardd->l_backend < min)
 		re->shardd->l_backend = min;
 	else
-		re->shardd->l_backend <<= 1;
+		re->shardd->l_backend *= 2;
 
 	re->shardd->backend = realloc(re->shardd->backend,
 	    re->shardd->l_backend * sizeof *re->shardd->backend);
diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index f18c2131c..fffcad572 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -311,7 +311,7 @@ sharddir_any_healthy(struct sharddir *shardd, const struct busyobj *bo,
 {
 	unsigned retval = 0;
 	VCL_BACKEND be;
-	unsigned u;
+	int i;
 	vtim_real c;
 
 	CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC);
@@ -319,8 +319,8 @@ sharddir_any_healthy(struct sharddir *shardd, const struct busyobj *bo,
 	sharddir_rdlock(shardd);
 	if (changed != NULL)
 		*changed = 0;
-	for (u = 0; u < shardd->n_backend; u++) {
-		be = shardd->backend[u].backend;
+	for (i = 0; i < shardd->n_backend; i++) {
+		be = shardd->backend[i].backend;
 		CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC);
 		retval = be->healthy(be, bo, &c);
 		if (changed != NULL && c > *changed)
diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h
index 1daae58ca..384b32cb9 100644
--- a/lib/libvmod_directors/shard_dir.h
+++ b/lib/libvmod_directors/shard_dir.h
@@ -76,8 +76,8 @@ struct sharddir {
 
 	const char				*name;
 
-	unsigned				n_backend;
-	unsigned				l_backend;
+	int					n_backend;
+	int					l_backend;
 	struct shard_backend			*backend;
 
 	struct shard_circlepoint		*hashcircle;
diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c
index 4ea5e5ae7..815b28432 100644
--- a/lib/libvmod_directors/vmod_shard.c
+++ b/lib/libvmod_directors/vmod_shard.c
@@ -466,8 +466,8 @@ static uint32_t
 shard_blob_key(VCL_BLOB key_blob)
 {
 	uint8_t k[4] = { 0 };
-	uint8_t *b;
-	int i, ki;
+	const uint8_t *b;
+	size_t i, ki;
 
 	assert(key_blob);
 	assert(key_blob->len > 0);

From reza at naghibi.com  Wed Aug 19 13:17:07 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:07 +0000 (UTC)
Subject: [6.0] 8b2980143 shard director: add optional weight parameter to
 .add_backend()
Message-ID: <20200819131707.34A086266@lists.varnish-cache.org>


commit 8b29801432fe98dbdd8a518a8c0ced6684563008
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 9 19:02:00 2020 +0200

    shard director: add optional weight parameter to .add_backend()
    
    We implement weights by scaling the number of replicas of each backend.
    The replicas parameter of .reconfigure() remains a minimum.
    
    For existing vtcs, the Debug hashcircle output has been compared
    before/after this change to ensure that behaviour is exactly equivalent.
    
    For for wighted backends, it has been checked that the number of
    instances per host on the hashcircle matches the expectation.
    
    Also refactor and clean up some of the code:
    
    - consistently make the number of ring points a uint32_t
    - some constification
    
    Ref #3276
    
     Conflicts:
            lib/libvmod_directors/shard_cfg.c
            lib/libvmod_directors/shard_dir.h
            lib/libvmod_directors/vmod.vcc

diff --git a/bin/varnishtest/tests/d00041.vtc b/bin/varnishtest/tests/d00041.vtc
new file mode 100644
index 000000000..bd79cfff1
--- /dev/null
+++ b/bin/varnishtest/tests/d00041.vtc
@@ -0,0 +1,224 @@
+varnishtest "d00017.vtc but with weights"
+
+server s1 {
+	rxreq
+	txresp -body "ech3Ooj"
+} -start
+
+server s2 {
+	rxreq
+	txresp -body "ieQu2qua"
+} -start
+
+server s3 {
+	rxreq
+	txresp -body "xiuFi3Pe"
+} -start
+
+varnish v1 -vcl+backend {
+	import std;
+	import directors;
+	import blob;
+
+	sub vcl_init {
+		new vd = directors.shard();
+		vd.debug(3);
+		if (!vd.add_backend(s1)) {
+			std.log("add s1 failed");
+		}
+		if (!vd.add_backend(s2, weight=2)) {
+			std.log("add s2 failed");
+		}
+		if (!vd.add_backend(s3, weight=3)) {
+			std.log("add s3 failed");
+		}
+		if (!vd.reconfigure(replicas=25)) {
+			std.log("reconfigure failed");
+		}
+	}
+
+	sub vcl_recv {
+		set req.backend_hint = vd.backend(by=BLOB,
+		    key_blob=blob.decode(HEX, encoded=
+			regsub(req.url, "^/", "")));
+		return(pass);
+	}
+
+} -start
+
+logexpect l1 -v v1 -g raw -d 1 {
+	expect 0 0    CLI   "^Rd vcl.load"
+
+	expect 0 =    Debug {^shard: hashcircle.* 0. = .point = *238d0ef, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 1. = .point = *321c598, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 2. = .point = *3b6b56a, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 3. = .point = *408ec1e, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 4. = .point = *66986a7, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 5. = .point = *7e41e30, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 6. = .point = *b749e7b, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 7. = .point = *e543430, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 8. = .point = *10136c05, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 9. = .point = *102d847f, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 10. = .point = *1112f910, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 11. = .point = *1119a7c7, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 12. = .point = *14d95c44, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 13. = .point = *150fea1f, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 14. = .point = *1643ecb6, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 15. = .point = *189ff2f2, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 16. = .point = *19cfe9f3, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 17. = .point = *1e1c78c3, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 18. = .point = *1fe0dea0, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 19. = .point = *22464ee9, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 20. = .point = *22b35675, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 21. = .point = *2363bebb, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 22. = .point = *24f827bb, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 23. = .point = *259eeccf, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 24. = .point = *26f0c3e7, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 25. = .point = *271874d4, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 26. = .point = *28340f35, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 27. = .point = *285e8475, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 28. = .point = *28ec7a6f, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 29. = .point = *299c6298, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 30. = .point = *2aedc3f7, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 31. = .point = *2b031742, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 32. = .point = *2da0e37b, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 33. = .point = *310bd2ca, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 34. = .point = *31e5f2df, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 35. = .point = *32d6b3ed, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 36. = .point = *33047373, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 37. = .point = *3392487a, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 38. = .point = *37597c4c, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 39. = .point = *3f6b2b89, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 40. = .point = *43cf6426, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 41. = .point = *46a58f28, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 42. = .point = *4b1f5b22, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 43. = .point = *523723f2, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 44. = .point = *539234db, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 45. = .point = *564ca84f, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 46. = .point = *58501380, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 47. = .point = *58704432, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 48. = .point = *5b1bcbbe, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 49. = .point = *5d2df428, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 50. = .point = *5fa294ee, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 51. = .point = *606fd878, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 52. = .point = *60dded53, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 53. = .point = *616cdb68, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 54. = .point = *6257bc27, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 55. = .point = *64014b25, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 56. = .point = *6918f467, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 57. = .point = *6a08c380, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 58. = .point = *6bfd5a2d, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 59. = .point = *6c0b607a, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 60. = .point = *6c74d296, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 61. = .point = *6e040182, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 62. = .point = *6e3819f7, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 63. = .point = *720ec1a4, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 64. = .point = *7232b381, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 65. = .point = *74c384ad, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 66. = .point = *76d47350, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 67. = .point = *791eb3a3, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 68. = .point = *7a048f20, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 69. = .point = *7f874929, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 70. = .point = *83ce71ce, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 71. = .point = *888b6447, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 72. = .point = *8997c018, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 73. = .point = *89b7d09c, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 74. = .point = *8aa6b5b4, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 75. = .point = *8ae34bde, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 76. = .point = *8b382e03, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 77. = .point = *8b47e6ac, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 78. = .point = *8bc76115, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 79. = .point = *8bc8bc11, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 80. = .point = *8e2d3849, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 81. = .point = *8e7e012c, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 82. = .point = *8f5b4c63, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 83. = .point = *94a94162, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 84. = .point = *99892987, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 85. = .point = *9a6f2f00, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 86. = .point = *9b970b49, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 87. = .point = *9e09a3a7, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 88. = .point = *9ef9125d, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 89. = .point = *9f33cd30, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 90. = .point = *9fc69b51, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 91. = .point = *a19f99eb, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 92. = .point = *a28b9595, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 93. = .point = *a3582038, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 94. = .point = *a4b6a3b9, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 95. = .point = *a66da9cb, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 96. = .point = *a8657c76, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 97. = .point = *a8afe9c4, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 98. = .point = *aa488703, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 99. = .point = *ac7b4454, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 100. = .point = *ad923ad3, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 101. = .point = *ae8946c6, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 102. = .point = *b197e339, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 103. = .point = *b3c305e6, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 104. = .point = *b4dab004, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 105. = .point = *b6bf43ea, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 106. = .point = *b9004d3d, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 107. = .point = *b96b6455, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 108. = .point = *b9a0edb9, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 109. = .point = *b9ec6465, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 110. = .point = *bb8eed4d, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 111. = .point = *bbcc0bad, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 112. = .point = *bcfea141, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 113. = .point = *be300622, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 114. = .point = *bf514d68, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 115. = .point = *c1afc7d2, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 116. = .point = *c2542a5d, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 117. = .point = *c6c43fa7, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 118. = .point = *c945958a, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 119. = .point = *c9f304a4, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 120. = .point = *cb896aa8, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 121. = .point = *cbd9198a, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 122. = .point = *ccd61dad, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 123. = .point = *d07e4431, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 124. = .point = *d21fe35f, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 125. = .point = *d4c93105, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 126. = .point = *d570b815, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 127. = .point = *d7de63b6, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 128. = .point = *d8634aef, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 129. = .point = *d92d916d, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 130. = .point = *d937a7df, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 131. = .point = *dac52229, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 132. = .point = *db7840f0, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 133. = .point = *dd5c6bef, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 134. = .point = *dded5798, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 135. = .point = *dfd5333b, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 136. = .point = *e183345a, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 137. = .point = *e2c71c27, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 138. = .point = *e49bf9d8, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 139. = .point = *e72bc224, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 140. = .point = *e8b27f41, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 141. = .point = *e991584c, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 142. = .point = *ea201c5e, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 143. = .point = *ec8891c5, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 144. = .point = *edcc8dd9, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 145. = .point = *ef6b4ab5, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 146. = .point = *f08ad325, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 147. = .point = *f3325ba2, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 148. = .point = *f6530dd1, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 149. = .point = *fc28e8d2, host =  2.}
+
+	expect 0 =    CLI   Loaded
+
+	expect * =    Debug {^shard: lookup key 564ca84f idx 45 host 0}
+	expect * =    Debug {^shard: lookup key 19cfe9f3 idx 16 host 1}
+	expect * =    Debug {^shard: lookup key 46a58f28 idx 41 host 2}
+} -start
+
+client c1 {
+	txreq -url /564ca84f
+	rxresp
+	expect resp.body == "ech3Ooj"
+
+	txreq -url /19cfe9f3
+	rxresp
+	expect resp.body == "ieQu2qua"
+
+	txreq -url /46a58f28
+	rxresp
+	expect resp.body == "xiuFi3Pe"
+} -run
+
+logexpect l1 -wait
diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c
index a198f0c85..b3496a15b 100644
--- a/lib/libvmod_directors/shard_cfg.c
+++ b/lib/libvmod_directors/shard_cfg.c
@@ -54,6 +54,7 @@ struct shard_change_task {
 #define SHARD_CHANGE_TASK_MAGIC			0x1e1168af
 	enum shard_change_task_e		task;
 	void					*priv;
+	VCL_REAL				weight;
 	VSTAILQ_ENTRY(shard_change_task)	list;
 };
 
@@ -126,7 +127,7 @@ shard_change_finish(struct shard_change *change)
 	VSTAILQ_INIT(&change->tasks);
 }
 
-static void
+static struct shard_change_task *
 shard_change_task_add(VRT_CTX, struct shard_change *change,
     enum shard_change_task_e task_e, void *priv)
 {
@@ -138,15 +139,17 @@ shard_change_task_add(VRT_CTX, struct shard_change *change,
 	if (task == NULL) {
 		shard_err0(ctx, change->shardd,
 		    "could not get workspace for task");
-		return;
+		return (NULL);
 	}
 	INIT_OBJ(task, SHARD_CHANGE_TASK_MAGIC);
 	task->task = task_e;
 	task->priv = priv;
 	VSTAILQ_INSERT_TAIL(&change->tasks, task, list);
+
+	return (task);
 }
 
-static inline VCL_BOOL
+static inline struct shard_change_task *
 shard_change_task_backend(VRT_CTX,
     struct vmod_priv *priv, const struct sharddir *shardd,
     enum shard_change_task_e task_e, VCL_BACKEND be, VCL_STRING ident,
@@ -160,22 +163,20 @@ shard_change_task_backend(VRT_CTX,
 
 	change = shard_change_get(ctx, priv, shardd);
 	if (change == NULL)
-		return (0);
+		return (NULL);
 
 	b = WS_Alloc(ctx->ws, sizeof(*b));
 	if (b == NULL) {
 		shard_err(ctx, shardd, ".%s_backend() WS_Alloc() failed",
 		    task_e == ADD_BE ? "add" : "remove");
-		return (0);
+		return (NULL);
 	}
 
 	b->backend = be;
 	b->ident = ident != NULL && *ident != '\0' ? ident : NULL;
 	b->rampup = rampup;
 
-	shard_change_task_add(ctx, change, task_e, b);
-
-	return (1);
+	return (shard_change_task_add(ctx, change, task_e, b));
 }
 
 /*
@@ -185,11 +186,21 @@ shard_change_task_backend(VRT_CTX,
 VCL_BOOL
 shardcfg_add_backend(VRT_CTX, struct vmod_priv *priv,
     const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident,
-    VCL_DURATION rampup)
+    VCL_DURATION rampup, VCL_REAL weight)
 {
+	struct shard_change_task *task;
+
+	assert (weight >= 1);
 	AN(be);
-	return (shard_change_task_backend(ctx, priv, shardd, ADD_BE,
-	    be, ident, rampup));
+
+	task = shard_change_task_backend(ctx, priv, shardd, ADD_BE,
+	    be, ident, rampup);
+
+	if (task == NULL)
+		return (0);
+
+	task->weight = weight;
+	return (1);
 }
 
 VCL_BOOL
@@ -197,7 +208,7 @@ shardcfg_remove_backend(VRT_CTX, struct vmod_priv *priv,
     const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident)
 {
 	return (shard_change_task_backend(ctx, priv, shardd, REMOVE_BE,
-	    be, ident, 0));
+	    be, ident, 0) != NULL);
 }
 
 VCL_BOOL
@@ -211,9 +222,7 @@ shardcfg_clear(VRT_CTX, struct vmod_priv *priv, const struct sharddir *shardd)
 	if (change == NULL)
 		return (0);
 
-	shard_change_task_add(ctx, change, CLEAR, NULL);
-
-	return (1);
+	return (shard_change_task_add(ctx, change, CLEAR, NULL) != NULL);
 }
 
 /*
@@ -231,9 +240,11 @@ circlepoint_compare(const struct shard_circlepoint *a,
 }
 
 static void
-shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas)
+shardcfg_hashcircle(struct sharddir *shardd)
 {
-	int i, j;
+	const struct shard_backend *backends, *b;
+	int j, h;
+	uint32_t i, n_points, r, rmax;
 	const char *ident;
 	const int len = 12; // log10(UINT32_MAX) + 2;
 	char s[len];
@@ -242,47 +253,59 @@ shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas)
 	AZ(shardd->hashcircle);
 
 	assert(shardd->n_backend > 0);
-	AN(shardd->backend);
-
-	shardd->hashcircle = calloc(shardd->n_backend * replicas,
-		sizeof(struct shard_circlepoint));
-	AN(shardd->hashcircle);
+	backends=shardd->backend;
+	AN(backends);
+
+	n_points = 0;
+	rmax = (UINT32_MAX - 1) / shardd->n_backend;
+	for (b = backends; b < backends + shardd->n_backend; b++) {
+		CHECK_OBJ_NOTNULL(b->backend, DIRECTOR_MAGIC);
+		r = b->replicas;
+		if (r > rmax)
+			r = rmax;
+		n_points += r;
+	}
 
-	shardd->replicas = replicas;
+	assert(n_points < UINT32_MAX);
 
-	for (i = 0; i < shardd->n_backend; i++) {
-		CHECK_OBJ_NOTNULL(shardd->backend[i].backend, DIRECTOR_MAGIC);
+	shardd->n_points = n_points;
+	shardd->hashcircle = calloc(n_points, sizeof(struct shard_circlepoint));
+	AN(shardd->hashcircle);
 
-		ident = shardd->backend[i].ident
-		    ? shardd->backend[i].ident
-		    : shardd->backend[i].backend->vcl_name;
+	i = 0;
+	for (h = 0, b = backends; h < shardd->n_backend; h++, b++) {
+		ident = b->ident ? b->ident : b->backend->vcl_name;
 
 		assert(ident[0] != '\0');
 
-		for (j = 0; j < replicas; j++) {
+		r = b->replicas;
+		if (r > rmax)
+			r = rmax;
+
+		for (j = 0; j < r; j++) {
 			assert(snprintf(s, len, "%d", j) < len);
-			shardd->hashcircle[i * replicas + j].point =
+			assert (i < n_points);
+			shardd->hashcircle[i].point =
 				sharddir_sha256(ident, s, vrt_magic_string_end);
-			shardd->hashcircle[i * replicas + j].host = i;
+			shardd->hashcircle[i].host = h;
+			i++;
 		}
 		/* not used in current interface */
-		shardd->backend[i].canon_point =
-		    shardd->hashcircle[i * replicas].point;
+		shardd->backend[h].canon_point =
+		    shardd->hashcircle[i].point;
 	}
-	qsort( (void *) shardd->hashcircle, shardd->n_backend * replicas,
+	assert (i == n_points);
+	qsort( (void *) shardd->hashcircle, n_points,
 	    sizeof (struct shard_circlepoint), (compar) circlepoint_compare);
 
 	if ((shardd->debug_flags & SHDBG_CIRCLE) == 0)
 		return;
 
-	for (i = 0; i < shardd->n_backend; i++)
-		for (j = 0; j < replicas; j++)
-			SHDBG(SHDBG_CIRCLE, shardd,
-			    "hashcircle[%5jd] = "
-			    "{point = %8x, host = %2u}\n",
-			    (intmax_t)(i * replicas + j),
-			    shardd->hashcircle[i * replicas + j].point,
-			    shardd->hashcircle[i * replicas + j].host);
+	for (i = 0; i < n_points; i++)
+		SHDBG(SHDBG_CIRCLE, shardd,
+		    "hashcircle[%5jd] = {point = %8x, host = %2u}\n",
+		    (intmax_t)i, shardd->hashcircle[i].point,
+		    shardd->hashcircle[i].host);
 }
 
 /*
@@ -388,7 +411,7 @@ shardcfg_backend_expand(const struct backend_reconfig *re)
 
 static void
 shardcfg_backend_add(struct backend_reconfig *re,
-    const struct shard_backend *b)
+    const struct shard_backend *b, uint32_t replicas)
 {
 	unsigned i;
 	struct shard_backend *bb = re->shardd->backend;
@@ -413,6 +436,7 @@ shardcfg_backend_add(struct backend_reconfig *re,
 
 	re->shardd->n_backend++;
 	shardcfg_backend_copyin(&bb[i], b);
+	bb[i].replicas = replicas;
 }
 
 static void
@@ -493,10 +517,11 @@ shardcfg_backend_finalize(struct backend_reconfig *re)
 
 static void
 shardcfg_apply_change(VRT_CTX, struct sharddir *shardd,
-    const struct shard_change *change)
+    const struct shard_change *change, VCL_INT replicas)
 {
 	struct shard_change_task *task, *clear;
 	const struct shard_backend *b;
+	uint32_t b_replicas;
 
 	struct backend_reconfig re = {
 		.shardd = shardd,
@@ -544,7 +569,14 @@ shardcfg_apply_change(VRT_CTX, struct sharddir *shardd,
 			b = shardcfg_backend_lookup(&re, task->priv);
 
 			if (b == NULL) {
-				shardcfg_backend_add(&re, task->priv);
+				assert (task->weight >= 1);
+				if (replicas * task->weight > UINT32_MAX)
+					b_replicas = UINT32_MAX;
+				else
+					b_replicas = replicas * task->weight;
+
+				shardcfg_backend_add(&re, task->priv,
+				    b_replicas);
 				break;
 			}
 
@@ -593,7 +625,7 @@ shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv,
 
 	sharddir_wrlock(shardd);
 
-	shardcfg_apply_change(ctx, shardd, change);
+	shardcfg_apply_change(ctx, shardd, change, replicas);
 	shard_change_finish(change);
 
 	if (shardd->hashcircle)
@@ -606,7 +638,7 @@ shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv,
 		return (0);
 	}
 
-	shardcfg_hashcircle(shardd, replicas);
+	shardcfg_hashcircle(shardd);
 	sharddir_unlock(shardd);
 	return (1);
 }
diff --git a/lib/libvmod_directors/shard_cfg.h b/lib/libvmod_directors/shard_cfg.h
index 2ce1817b8..f1eb92412 100644
--- a/lib/libvmod_directors/shard_cfg.h
+++ b/lib/libvmod_directors/shard_cfg.h
@@ -28,7 +28,7 @@
 
 VCL_BOOL shardcfg_add_backend(VRT_CTX, struct vmod_priv *priv,
     const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident,
-    VCL_DURATION rampup);
+    VCL_DURATION rampup, VCL_REAL weight);
 VCL_BOOL shardcfg_remove_backend(VRT_CTX, struct vmod_priv *priv,
     const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident);
 VCL_BOOL shardcfg_clear(VRT_CTX, struct vmod_priv *priv,
diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index fffcad572..47f7b4927 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -60,7 +60,7 @@ struct shard_be_info {
 struct shard_state {
 	const struct vrt_ctx	*ctx;
 	struct sharddir	*shardd;
-	int			idx;
+	uint32_t		idx;
 
 	struct vbitmap		*picklist;
 	int			pickcount;
@@ -135,8 +135,10 @@ shard_lookup(const struct sharddir *shardd, const uint32_t key)
 {
 	CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC);
 
-	const int n = shardd->n_backend * shardd->replicas;
-	int idx = -1, high = n, low = 0, i;
+	const uint32_t n = shardd->n_points;
+	uint32_t i, idx = UINT32_MAX, high = n, low = 0;
+
+	assert (n < idx);
 
 	do {
 	    i = (high + low) / 2 ;
@@ -154,7 +156,7 @@ shard_lookup(const struct sharddir *shardd, const uint32_t key)
 		    high = i;
 	    else
 		low = i;
-	} while (idx == -1);
+	} while (idx == UINT32_MAX);
 
 	return (idx);
 }
@@ -163,7 +165,6 @@ static int
 shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy)
 {
 	int c, chosen = -1;
-	uint32_t ringsz;
 	VCL_BACKEND be;
 	vtim_real changed;
 	struct shard_be_info *sbe;
@@ -175,8 +176,6 @@ shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy)
 	if (state->pickcount >= state->shardd->n_backend)
 		return (-1);
 
-	ringsz = state->shardd->n_backend * state->shardd->replicas;
-
 	while (state->pickcount < state->shardd->n_backend && skip >= 0) {
 
 		c = state->shardd->hashcircle[state->idx].host;
@@ -215,7 +214,7 @@ shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy)
 				break;
 		}
 
-		if (++(state->idx) == ringsz)
+		if (++(state->idx) == state->shardd->n_points)
 			state->idx = 0;
 	}
 	return (chosen);
diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h
index 384b32cb9..8d1b86b52 100644
--- a/lib/libvmod_directors/shard_dir.h
+++ b/lib/libvmod_directors/shard_dir.h
@@ -61,6 +61,7 @@ struct shard_backend {
 	};
 	VCL_DURATION		rampup;
 	uint32_t		canon_point;
+	uint32_t		replicas;
 };
 
 #define	SHDBG_LOOKUP	 1
@@ -84,7 +85,8 @@ struct sharddir {
 
 	VCL_DURATION				rampup_duration;
 	VCL_REAL				warmup;
-	VCL_INT					replicas;
+
+	uint32_t				n_points;
 };
 
 static inline VCL_BACKEND
diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 54b111fba..de467ee27 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -358,7 +358,7 @@ The association can be changed per backend request using the `param`
 argument of `func_shard.backend`_.
 
 $Method BOOL .add_backend(PRIV_TASK, BACKEND backend,
-	[STRING ident], [DURATION rampup])
+	[STRING ident], [DURATION rampup], [REAL weight])
 
 Add a backend `backend` to the director.
 
@@ -373,6 +373,12 @@ backend name.
 backend. Otherwise, the per-director rampup time is used (see
 :ref:`func_shard.set_rampup`).
 
+*weight*: Optionally specify a weight to scale the
+`shard.reconfigure()` *replicas* parameter. *weight* is limited to
+at least 1. Values above 10 probably do not make much sense. The
+effect of *weight* is also capped such that the total number of
+replicas does not exceed `UINT32_MAX`.
+
 NOTE: Backend changes need to be finalized with `shard.reconfigure()`
 and are only supported on one shard director at a time.
 
diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c
index 815b28432..03b6ace5d 100644
--- a/lib/libvmod_directors/vmod_shard.c
+++ b/lib/libvmod_directors/vmod_shard.c
@@ -340,6 +340,8 @@ VCL_BOOL v_matchproto_(td_directors_shard_add_backend)
 vmod_shard_add_backend(VRT_CTX, struct vmod_directors_shard *vshard,
     struct vmod_shard_add_backend_arg *args)
 {
+	VCL_REAL weight = 1;
+
 	CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC);
 
 	if (args->backend == NULL) {
@@ -348,10 +350,14 @@ vmod_shard_add_backend(VRT_CTX, struct vmod_directors_shard *vshard,
 		return (0);
 	}
 
+	if (args->valid_weight && args->weight > 1)
+		weight = args->weight;
+
 	return shardcfg_add_backend(ctx, args->arg1,
 	    vshard->shardd, args->backend,
 	    args->valid_ident ? args->ident : NULL,
-	    args->valid_rampup ? args->rampup : nan(""));
+	    args->valid_rampup ? args->rampup : nan(""),
+	    weight);
 }
 
 VCL_BOOL v_matchproto_(td_directors_shard_remove_backend)

From reza at naghibi.com  Wed Aug 19 13:17:07 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:07 +0000 (UTC)
Subject: [6.0] 092239ab2 ->idx is unsigned, so cannot be < 0 (and GCC sux).
Message-ID: <20200819131707.4C9CF626A@lists.varnish-cache.org>


commit 092239ab2a148d1e185ba3756f3050e5b61fe034
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 9 20:28:58 2020 +0000

    ->idx is unsigned, so cannot be < 0 (and GCC sux).

diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index 47f7b4927..a07bb1e3f 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -170,7 +170,6 @@ shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy)
 	struct shard_be_info *sbe;
 
 	AN(state);
-	assert(state->idx >= 0);
 	CHECK_OBJ_NOTNULL(state->shardd, SHARDDIR_MAGIC);
 
 	if (state->pickcount >= state->shardd->n_backend)

From reza at naghibi.com  Wed Aug 19 13:17:07 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:07 +0000 (UTC)
Subject: [6.0] 35a99eb67 One more time for VTEST...
Message-ID: <20200819131707.64AA56273@lists.varnish-cache.org>


commit 35a99eb67319055c8c19ff4261c0aa56b2b759d7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 9 20:47:18 2020 +0000

    One more time for VTEST...

diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index a07bb1e3f..b978c8780 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -352,7 +352,6 @@ sharddir_pick_be_locked(VRT_CTX, const struct sharddir *shardd, uint32_t key,
 	validate_alt(ctx, shardd, &alt);
 
 	state->idx = shard_lookup(shardd, key);
-	assert(state->idx >= 0);
 
 	SHDBG(SHDBG_LOOKUP, shardd, "lookup key %x idx %d host %u",
 	    key, state->idx, shardd->hashcircle[state->idx].host);

From reza at naghibi.com  Wed Aug 19 13:17:07 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:07 +0000 (UTC)
Subject: [6.0] a3b50786c signedness improvements
Message-ID: <20200819131707.7BFB66292@lists.varnish-cache.org>


commit a3b50786c6550d49e8e585c79011f7715b48f385
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jun 10 08:59:26 2020 +0200

    signedness improvements
    
    from flexelint review
    
     Conflicts:
            lib/libvmod_directors/shard_dir.h

diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h
index 8d1b86b52..fc59bc8fd 100644
--- a/lib/libvmod_directors/shard_dir.h
+++ b/lib/libvmod_directors/shard_dir.h
@@ -77,8 +77,8 @@ struct sharddir {
 
 	const char				*name;
 
-	int					n_backend;
-	int					l_backend;
+	unsigned				n_backend;
+	unsigned				l_backend;
 	struct shard_backend			*backend;
 
 	struct shard_circlepoint		*hashcircle;

From reza at naghibi.com  Wed Aug 19 13:17:07 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:07 +0000 (UTC)
Subject: [6.0] e6a469148 flexelinting
Message-ID: <20200819131707.975BD629C@lists.varnish-cache.org>


commit e6a469148adc3402ad94f3deb5ffd670f14e6801
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jun 10 09:05:24 2020 +0200

    flexelinting

diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c
index b3496a15b..006c27ca6 100644
--- a/lib/libvmod_directors/shard_cfg.c
+++ b/lib/libvmod_directors/shard_cfg.c
@@ -243,8 +243,8 @@ static void
 shardcfg_hashcircle(struct sharddir *shardd)
 {
 	const struct shard_backend *backends, *b;
-	int j, h;
-	uint32_t i, n_points, r, rmax;
+	int h;
+	uint32_t i, j, n_points, r, rmax;
 	const char *ident;
 	const int len = 12; // log10(UINT32_MAX) + 2;
 	char s[len];
@@ -573,7 +573,8 @@ shardcfg_apply_change(VRT_CTX, struct sharddir *shardd,
 				if (replicas * task->weight > UINT32_MAX)
 					b_replicas = UINT32_MAX;
 				else
-					b_replicas = replicas * task->weight;
+					b_replicas = (uint32_t) // flint
+						(replicas * task->weight);
 
 				shardcfg_backend_add(&re, task->priv,
 				    b_replicas);
diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index b978c8780..875616f68 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -290,7 +290,7 @@ init_state(struct shard_state *state,
 
 	state->ctx = ctx;
 	state->shardd = shardd;
-	state->idx = -1;
+	state->idx = UINT32_MAX;
 	state->picklist = picklist;
 
 	/* healhy and changed only defined for hostid != -1 */
@@ -352,8 +352,9 @@ sharddir_pick_be_locked(VRT_CTX, const struct sharddir *shardd, uint32_t key,
 	validate_alt(ctx, shardd, &alt);
 
 	state->idx = shard_lookup(shardd, key);
+	assert(state->idx < UINT32_MAX);
 
-	SHDBG(SHDBG_LOOKUP, shardd, "lookup key %x idx %d host %u",
+	SHDBG(SHDBG_LOOKUP, shardd, "lookup key %x idx %u host %u",
 	    key, state->idx, shardd->hashcircle[state->idx].host);
 
 	if (alt > 0) {

From reza at naghibi.com  Wed Aug 19 13:17:07 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:07 +0000 (UTC)
Subject: [6.0] 3e8316c21 shard: VRT_fail for out-of-workspace errors
Message-ID: <20200819131707.C54D462B2@lists.varnish-cache.org>


commit 3e8316c2109cc19effc64054ecd6b71178652aa2
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jun 10 09:18:21 2020 +0200

    shard: VRT_fail for out-of-workspace errors

diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c
index 006c27ca6..e08fd8ca3 100644
--- a/lib/libvmod_directors/shard_cfg.c
+++ b/lib/libvmod_directors/shard_cfg.c
@@ -105,7 +105,7 @@ shard_change_get(VRT_CTX, struct vmod_priv *priv,
 
 	change = WS_Alloc(ctx->ws, sizeof(*change));
 	if (change == NULL) {
-		shard_err0(ctx, shardd, "could not get workspace");
+		VRT_fail(ctx, "could not get workspace");
 		return (NULL);
 	}
 
@@ -137,8 +137,7 @@ shard_change_task_add(VRT_CTX, struct shard_change *change,
 
 	task = WS_Alloc(ctx->ws, sizeof(*task));
 	if (task == NULL) {
-		shard_err0(ctx, change->shardd,
-		    "could not get workspace for task");
+		VRT_fail(ctx, "could not get workspace for task");
 		return (NULL);
 	}
 	INIT_OBJ(task, SHARD_CHANGE_TASK_MAGIC);
@@ -167,8 +166,7 @@ shard_change_task_backend(VRT_CTX,
 
 	b = WS_Alloc(ctx->ws, sizeof(*b));
 	if (b == NULL) {
-		shard_err(ctx, shardd, ".%s_backend() WS_Alloc() failed",
-		    task_e == ADD_BE ? "add" : "remove");
+		VRT_fail(ctx, "could not get workspace for change");
 		return (NULL);
 	}
 

From reza at naghibi.com  Wed Aug 19 13:17:07 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:07 +0000 (UTC)
Subject: [6.0] 774b262fe shard: more signedness stir
Message-ID: <20200819131707.EDFA762BF@lists.varnish-cache.org>


commit 774b262fe213ac000e4b24d50d3c50debbf2808a
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jun 10 09:29:34 2020 +0200

    shard: more signedness stir
    
     Conflicts:
            lib/libvmod_directors/shard_dir.c
            lib/libvmod_directors/vmod_shard.c

diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c
index e08fd8ca3..bb0a7aabb 100644
--- a/lib/libvmod_directors/shard_cfg.c
+++ b/lib/libvmod_directors/shard_cfg.c
@@ -241,7 +241,7 @@ static void
 shardcfg_hashcircle(struct sharddir *shardd)
 {
 	const struct shard_backend *backends, *b;
-	int h;
+	unsigned h;
 	uint32_t i, j, n_points, r, rmax;
 	const char *ident;
 	const int len = 12; // log10(UINT32_MAX) + 2;
@@ -389,7 +389,7 @@ shardcfg_backend_lookup(const struct backend_reconfig *re,
 static void
 shardcfg_backend_expand(const struct backend_reconfig *re)
 {
-	int min = re->hint;
+	unsigned min = re->hint;
 
 	CHECK_OBJ_NOTNULL(re->shardd, SHARDDIR_MAGIC);
 
@@ -422,6 +422,7 @@ shardcfg_backend_add(struct backend_reconfig *re,
 		assert(re->shardd->n_backend < re->shardd->l_backend);
 		i = re->shardd->n_backend;
 	} else {
+		assert(re->hole_i != UINT_MAX);
 		do {
 			if (!bb[re->hole_i].backend)
 				break;
@@ -440,7 +441,7 @@ shardcfg_backend_add(struct backend_reconfig *re,
 static void
 shardcfg_backend_clear(struct sharddir *shardd)
 {
-	int i;
+	unsigned i;
 	for (i = 0; i < shardd->n_backend; i++)
 		shardcfg_backend_free(&shardd->backend[i]);
 	shardd->n_backend = 0;
@@ -525,7 +526,7 @@ shardcfg_apply_change(VRT_CTX, struct sharddir *shardd,
 		.shardd = shardd,
 		.hint = shardd->n_backend,
 		.hole_n = 0,
-		.hole_i = INT_MAX
+		.hole_i = UINT_MAX
 	};
 
 	// XXX assert sharddir_locked(shardd)
@@ -651,7 +652,7 @@ shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv,
 void
 shardcfg_delete(const struct sharddir *shardd)
 {
-	int i;
+	unsigned i;
 
 	for (i = 0; i < shardd->n_backend; i++)
 		shardcfg_backend_free(&shardd->backend[i]);
@@ -682,7 +683,7 @@ shardcfg_set_rampup(struct sharddir *shardd, VCL_DURATION duration)
 }
 
 VCL_DURATION
-shardcfg_get_rampup(const struct sharddir *shardd, int host)
+shardcfg_get_rampup(const struct sharddir *shardd, unsigned host)
 {
 	VCL_DURATION r;
 
diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index 875616f68..f2c3db8dd 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -35,6 +35,7 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <time.h>
+#include <limits.h>
 
 #include "cache/cache.h"
 #include "cache/cache_director.h"
@@ -47,7 +48,7 @@
 #include "shard_dir.h"
 
 struct shard_be_info {
-	int		hostid;
+	unsigned	hostid;
 	unsigned	healthy;
 	double		changed;	// when
 };
@@ -63,7 +64,7 @@ struct shard_state {
 	uint32_t		idx;
 
 	struct vbitmap		*picklist;
-	int			pickcount;
+	unsigned		pickcount;
 
 	struct shard_be_info	previous;
 	struct shard_be_info	last;
@@ -200,7 +201,7 @@ shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy)
 				sbe = &state->last;
 			}
 			if (sbe == &state->last &&
-			    state->last.hostid != -1)
+			    state->last.hostid != UINT_MAX)
 				memcpy(&state->previous, &state->last,
 				    sizeof(state->previous));
 
@@ -293,9 +294,9 @@ init_state(struct shard_state *state,
 	state->idx = UINT32_MAX;
 	state->picklist = picklist;
 
-	/* healhy and changed only defined for hostid != -1 */
-	state->previous.hostid = -1;
-	state->last.hostid = -1;
+	/* healhy and changed only defined for valid hostids */
+	state->previous.hostid = UINT_MAX;
+	state->last.hostid = UINT_MAX;
 }
 
 /* basically same as vdir_any_healthy
@@ -359,7 +360,7 @@ sharddir_pick_be_locked(VRT_CTX, const struct sharddir *shardd, uint32_t key,
 
 	if (alt > 0) {
 		if (shard_next(state, alt - 1, healthy == ALL ? 1 : 0) == -1) {
-			if (state->previous.hostid != -1) {
+			if (state->previous.hostid != UINT_MAX) {
 				be = sharddir_backend(shardd,
 				    state->previous.hostid);
 				AN(be);
@@ -370,7 +371,7 @@ sharddir_pick_be_locked(VRT_CTX, const struct sharddir *shardd, uint32_t key,
 	}
 
 	if (shard_next(state, 0, healthy == IGNORE ? 0 : 1) == -1) {
-		if (state->previous.hostid != -1) {
+		if (state->previous.hostid != UINT_MAX) {
 			be = sharddir_backend(shardd, state->previous.hostid);
 			AN(be);
 			return (be);
@@ -390,8 +391,8 @@ sharddir_pick_be_locked(VRT_CTX, const struct sharddir *shardd, uint32_t key,
 		return (be);
 
 	assert(alt == 0);
-	assert(state->previous.hostid >= 0);
-	assert(state->last.hostid >= 0);
+	assert(state->previous.hostid != UINT_MAX);
+	assert(state->last.hostid != UINT_MAX);
 	assert(state->previous.hostid != state->last.hostid);
 	assert(be == sharddir_backend(shardd, state->previous.hostid));
 
diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h
index fc59bc8fd..ac2621a85 100644
--- a/lib/libvmod_directors/shard_dir.h
+++ b/lib/libvmod_directors/shard_dir.h
@@ -90,9 +90,8 @@ struct sharddir {
 };
 
 static inline VCL_BACKEND
-sharddir_backend(const struct sharddir *shardd, int id)
+sharddir_backend(const struct sharddir *shardd, unsigned id)
 {
-	assert(id >= 0);
 	assert(id < shardd->n_backend);
 	return (shardd->backend[id].backend);
 }
@@ -130,4 +129,4 @@ VCL_BACKEND sharddir_pick_be(VRT_CTX, struct sharddir *, uint32_t, VCL_INT,
 
 /* in shard_cfg.c */
 void shardcfg_delete(const struct sharddir *shardd);
-VCL_DURATION shardcfg_get_rampup(const struct sharddir *shardd, int host);
+VCL_DURATION shardcfg_get_rampup(const struct sharddir *shardd, unsigned host);

From reza at naghibi.com  Wed Aug 19 13:17:08 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:08 +0000 (UTC)
Subject: [6.0] 14fad5a0c shard signedness stir: last bits?
Message-ID: <20200819131708.10C3962C5@lists.varnish-cache.org>


commit 14fad5a0c2942f74b261b5cd6f2f4e31041c6b1f
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jun 10 09:52:06 2020 +0200

    shard signedness stir: last bits?
    
     Conflicts:
            lib/libvmod_directors/vmod_shard.c

diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index f2c3db8dd..93ac0fa07 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -308,9 +308,8 @@ unsigned
 sharddir_any_healthy(struct sharddir *shardd, const struct busyobj *bo,
     double *changed)
 {
-	unsigned retval = 0;
+	unsigned i, retval = 0;
 	VCL_BACKEND be;
-	int i;
 	vtim_real c;
 
 	CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC);
@@ -399,10 +398,10 @@ sharddir_pick_be_locked(VRT_CTX, const struct sharddir *shardd, uint32_t key,
 	chosen_r = shardcfg_get_rampup(shardd, state->previous.hostid);
 	alt_r = shardcfg_get_rampup(shardd, state->last.hostid);
 
-	SHDBG(SHDBG_RAMPWARM, shardd, "chosen host %d rampup %f changed %f",
+	SHDBG(SHDBG_RAMPWARM, shardd, "chosen host %u rampup %f changed %f",
 	    state->previous.hostid, chosen_r,
 	    ctx->now - state->previous.changed);
-	SHDBG(SHDBG_RAMPWARM, shardd, "alt host %d rampup %f changed %f",
+	SHDBG(SHDBG_RAMPWARM, shardd, "alt host %u rampup %f changed %f",
 	    state->last.hostid, alt_r,
 	    ctx->now - state->last.changed);
 

From reza at naghibi.com  Wed Aug 19 13:17:08 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:08 +0000 (UTC)
Subject: [6.0] 6f14d2ed4 shard director: explain how the ring is constructed
 with weights
Message-ID: <20200819131708.468F062DC@lists.varnish-cache.org>


commit 6f14d2ed4236096be8eefa411c145f085d92f4f2
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Jul 6 18:07:09 2020 +0200

    shard director: explain how the ring is constructed with weights
    
     Conflicts:
            lib/libvmod_directors/vmod.vcc

diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index de467ee27..f79424e93 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -307,7 +307,9 @@ structure gets built from the last 32 bits of SHA256 hash values of
 `<ident>`\ `<n>` (default `ident` being the backend name) for each
 backend and for a running number `n` from 1 to `replicas`. Hashing
 creates the seemingly random order for placement of backends on the
-consistent hashing ring.
+consistent hashing ring.  When ``.add_backend()`` is called with a
+weight argument, replicas is scaled by that weight to add
+proportionally more copies of the that backend on the ring.
 
 When ``.backend()`` is called, a load balancing key gets generated
 unless provided. The smallest hash value in the circle is looked up

From reza at naghibi.com  Wed Aug 19 13:17:08 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:08 +0000 (UTC)
Subject: [6.0] 478ff9e20 Loosen assertion on ctx->(req|bo),
 fix shard and vcl_pipe
Message-ID: <20200819131708.73EE962EF@lists.varnish-cache.org>


commit 478ff9e207f858d8dde59940c5a96fcdcfd62d8c
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Jul 6 17:15:03 2020 +0200

    Loosen assertion on ctx->(req|bo), fix shard and vcl_pipe
    
    in VRT_priv_task() we asserted that only one of ctx->req and ctx->bo is
    set when not in vcl_pipe {}, but we also need to extend that assertion
    to when ctx->method == 0 after vcl_pipe as finished because
    VRT_priv_task() could be called from director resolution.
    
    Being at it, I also noticed that our behavior in vcl_pipe {} is
    inconsistent as, from the shard director perspective, it is a backend
    method. So now, vcl_pipe {} is handled like vcl_backend_* {}.
    
    We still need to make up our mind about #3329 / #3330 and depending on
    the outcome we might need to touch some places again which were changed
    in this commit.
    
    Fixes #3361
    
     Conflicts:
            doc/changes.rst
            lib/libvmod_directors/vmod.vcc
            lib/libvmod_directors/vmod_shard.c

diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c
index 6657cdfb6..4ba0e6261 100644
--- a/bin/varnishd/cache/cache_vrt_priv.c
+++ b/bin/varnishd/cache/cache_vrt_priv.c
@@ -126,8 +126,15 @@ VRT_priv_task(VRT_CTX, const void *vmod_id)
 	struct vmod_priv *vp;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	/*
+	 * XXX when coming from VRT_DirectorResolve() in pipe mode
+	 * (ctx->method == 0), both req and bo are set.
+	 * see #3329 #3330: we should make up our mind where
+	 * pipe objects live
+	 */
+
 	assert(ctx->req == NULL || ctx->bo == NULL ||
-	    ctx->method == VCL_MET_PIPE);
+	    ctx->method == VCL_MET_PIPE || ctx->method == 0);
 
 	if (ctx->req) {
 		CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
diff --git a/bin/varnishtest/tests/d00029.vtc b/bin/varnishtest/tests/d00029.vtc
index 93a1517c1..daaba5cf1 100644
--- a/bin/varnishtest/tests/d00029.vtc
+++ b/bin/varnishtest/tests/d00029.vtc
@@ -1,16 +1,16 @@
 varnishtest "shard director LAZY - d18.vtc"
 
-server s1 {
+server s1 -repeat 3 {
 	rxreq
 	txresp -body "ech3Ooj"
 } -start
 
-server s2 {
+server s2 -repeat 3 {
 	rxreq
 	txresp -body "ieQu2qua"
 } -start
 
-server s3 {
+server s3 -repeat 3 {
 	rxreq
 	txresp -body "xiuFi3Pe"
 } -start
@@ -45,10 +45,13 @@ varnish v1 -vcl+backend {
 	}
 
 	sub vcl_recv {
+	    if (req.http.pipe) {
+		return (pipe);
+	    }
 	    return(pass);
 	}
 
-	sub vcl_backend_fetch {
+	sub shard_be {
 	    set bereq.backend=vd.backend(resolve=LAZY);
 
 	    if (bereq.url == "/1") {
@@ -64,6 +67,14 @@ varnish v1 -vcl+backend {
 	    }
 	}
 
+	sub vcl_backend_fetch {
+	    call shard_be;
+	}
+
+	sub vcl_pipe {
+	    call shard_be;
+	}
+
 	sub vcl_backend_response {
 	    set beresp.http.backend = bereq.backend;
 	}
@@ -85,4 +96,20 @@ client c1 {
 	rxresp
 	expect resp.body == "xiuFi3Pe"
 	expect resp.http.backend == "vd"
+
+	txreq -url /1 -hdr "pipe: true"
+	rxresp
+	expect resp.body == "ech3Ooj"
+} -run
+
+client c1 {
+	txreq -url /2 -hdr "pipe: true"
+	rxresp
+	expect resp.body == "ieQu2qua"
+} -run
+
+client c1 {
+	txreq -url /3 -hdr "pipe: true"
+	rxresp
+	expect resp.body == "xiuFi3Pe"
 } -run
diff --git a/bin/varnishtest/tests/d00030.vtc b/bin/varnishtest/tests/d00030.vtc
index 497b3c7cf..0bfb724c3 100644
--- a/bin/varnishtest/tests/d00030.vtc
+++ b/bin/varnishtest/tests/d00030.vtc
@@ -31,7 +31,7 @@ logexpect l2 -v v1 -g raw {
     expect * 1001 VCL_Error {shard .backend param invalid}
 } -start
 logexpect l3 -v v1 -g raw {
-    expect * 1003 VCL_Error {shard_param.set.. may only be used in vcl_init and in backend context}
+    expect * 1003 VCL_Error {shard_param.set.. may only be used in vcl_init and in backend/pipe context}
 } -start
 
 client c1 {
@@ -159,7 +159,7 @@ varnish v1 -errvcl {invalid warmup argument 1.1} {
     }
 }
 
-varnish v1 -errvcl {resolve=LAZY with other parameters can only be used in backend context} {
+varnish v1 -errvcl {resolve=LAZY with other parameters can only be used in backend/pipe context} {
     import directors;
     import blob;
 
diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index f79424e93..fec56e155 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -443,10 +443,11 @@ is _not_ the order given when backends are added.
 
   * `HASH`:
 
-    * when called in backend context: Use the varnish hash value as
-      set by `vcl_hash`
+    * when called in backend context and in ``vcl_pipe {}``: Use the
+      varnish hash value as set by ``vcl_hash{}``
 
-    * when called in client context: hash `req.url`
+    * when called in client context other than ``vcl_pipe {}``: hash
+      ``req.url``
 
   * `URL`: hash req.url / bereq.url
 
@@ -537,9 +538,10 @@ is _not_ the order given when backends are added.
     In ``vcl_init{}`` and on the client side, ``LAZY`` mode can not be
     used with any other argument.
 
-    On the backend side, parameters from arguments or an associated
-    parameter set affect the shard director instance for the backend
-    request irrespective of where it is referenced.
+    On the backend side and in ``vcl_pipe {}``, parameters from
+    arguments or an associated parameter set affect the shard director
+    instance for the backend request irrespective of where it is
+    referenced.
 
 * `param`
 
@@ -586,21 +588,22 @@ Parameter sets have two scopes:
 * per backend request scope
 
 The per-VCL scope defines defaults for the per backend scope. Any
-changes to a parameter set in backend context only affect the
-respective backend request.
+changes to a parameter set in backend context and in ``vcl_pipe {}``
+only affect the respective backend request.
 
-Parameter sets can not be used in client context.
+Parameter sets can not be used in client context except for
+``vcl_pipe {}``.
 
 $Method VOID .clear()
 
 Reset the parameter set to default values as documented for
 `func_shard.backend`_.
 
-* in ``vcl_init{}``, resets the parameter set default for this VCL
-* in backend context, resets the parameter set for this backend
-  request to the VCL defaults
+* in ``vcl_init{}``, resets the parameter set default for this VCL in
+* backend context and in ``vcl_pipe {}``, resets the parameter set for
+  this backend request to the VCL defaults
 
-This method may not be used in client context
+This method may not be used in client context other than ``vcl_pipe {}``.
 
 $Method VOID .set(
 	[ ENUM {HASH, URL, KEY, BLOB} by ],
@@ -616,11 +619,11 @@ Change the given parameters of a parameter set as documented for
 
 * in ``vcl_init{}``, changes the parameter set default for this VCL
 
-* in backend context, changes the parameter set for this backend
-  request, keeping the defaults set for this VCL for unspecified
-  arguments.
+* in backend context and in ``vcl_pipe {}``, changes the parameter set
+  for this backend request, keeping the defaults set for this VCL for
+  unspecified arguments.
 
-This method may not be used in client context
+This method may not be used in client context other than ``vcl_pipe {}``.
 
 $Method STRING .get_by()
 
@@ -656,7 +659,7 @@ shard director using this parameter object would use. See
 
 $Method BLOB .use()
 
-This method may only be used in backend context.
+This method may only be used in backend context and in ``vcl_pipe {}``.
 
 For use with the `param` argument of `func_shard.backend`_ to associate
 this shard parameter set with a shard director.
diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c
index 03b6ace5d..846d75a6a 100644
--- a/lib/libvmod_directors/vmod_shard.c
+++ b/lib/libvmod_directors/vmod_shard.c
@@ -165,6 +165,9 @@ vmod_shard_param_read(VRT_CTX, const void *id,
     const struct vmod_directors_shard_param *p,
     struct vmod_directors_shard_param *pstk, const char *who);
 
+// XXX #3329 #3330 revisit - for now, treat pipe like backend
+#define SHARD_VCL_TASK_REQ (VCL_MET_TASK_C & ~VCL_MET_PIPE)
+#define SHARD_VCL_TASK_BEREQ (VCL_MET_TASK_B | VCL_MET_PIPE)
 /* -------------------------------------------------------------------------
  * shard vmod interface
  */
@@ -667,14 +670,14 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard,
 			return (vshard->dir);
 		}
 
-		if ((ctx->method & VCL_MET_TASK_B) == 0) {
+		if ((ctx->method & SHARD_VCL_TASK_BEREQ) == 0) {
 			VRT_fail(ctx, "shard .backend resolve=LAZY with other "
-				 "parameters can only be used in backend "
+				 "parameters can only be used in backend/pipe "
 				 "context");
 			return (NULL);
 		}
 
-		assert(ctx->method & VCL_MET_TASK_B);
+		assert(ctx->method & SHARD_VCL_TASK_BEREQ);
 
 		pp = shard_param_task(ctx, vshard, vshard->param);
 		if (pp == NULL)
@@ -884,11 +887,11 @@ shard_param_prep(VRT_CTX, struct vmod_directors_shard_param *p,
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(p, VMOD_SHARD_SHARD_PARAM_MAGIC);
 
-	if (ctx->method & VCL_MET_TASK_C) {
+	if (ctx->method & SHARD_VCL_TASK_REQ) {
 		VRT_fail(ctx, "%s may only be used "
-			 "in vcl_init and in backend context", who);
+			 "in vcl_init and in backend/pipe context", who);
 		return (NULL);
-	} else if (ctx->method & VCL_MET_TASK_B)
+	} else if (ctx->method & SHARD_VCL_TASK_BEREQ)
 		p = shard_param_task(ctx, p, p);
 	else
 		assert(ctx->method & VCL_MET_TASK_H);
@@ -933,7 +936,7 @@ vmod_shard_param_read(VRT_CTX, const void *id,
 	CHECK_OBJ_NOTNULL(p, VMOD_SHARD_SHARD_PARAM_MAGIC);
 	(void) who; // XXX
 
-	if (ctx->method & VCL_MET_TASK_B)
+	if (ctx->method == 0 || (ctx->method & SHARD_VCL_TASK_BEREQ))
 		p = shard_param_task(ctx, id, p);
 
 	if (p == NULL)

From reza at naghibi.com  Wed Aug 19 13:17:08 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:08 +0000 (UTC)
Subject: [6.0] 9d8417698 Setup bo->privs in vcl_pipe and add magic checks to
 priv destruction
Message-ID: <20200819131708.8C29662F6@lists.varnish-cache.org>


commit 9d8417698bfc51328f854ea843dba18b19c37072
Author: Reza Naghibi <reza at naghibi.com>
Date:   Fri Aug 14 09:44:55 2020 -0400

    Setup bo->privs in vcl_pipe and add magic checks to priv destruction

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 74b9f8998..04470ab59 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -768,6 +768,7 @@ cnt_pipe(struct worker *wrk, struct req *req)
 	THR_SetBusyobj(bo);
 	bo->sp = req->sp;
 	SES_Ref(bo->sp);
+	VCL_TaskEnter(bo->vcl, bo->privs);
 
 	HTTP_Setup(bo->bereq, bo->ws, bo->vsl, SLT_BereqMethod);
 	http_FilterReq(bo->bereq, req->http, 0);	// XXX: 0 ?
@@ -802,6 +803,7 @@ cnt_pipe(struct worker *wrk, struct req *req)
 	}
 	http_Teardown(bo->bereq);
 	SES_Rel(bo->sp);
+	VCL_TaskLeave(bo->vcl, bo->privs);
 	VBO_ReleaseBusyObj(wrk, &bo);
 	THR_SetBusyobj(NULL);
 	return (nxt);
diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c
index 4ba0e6261..05aae3f35 100644
--- a/bin/varnishd/cache/cache_vrt_priv.c
+++ b/bin/varnishd/cache/cache_vrt_priv.c
@@ -204,7 +204,9 @@ VCL_TaskLeave(const struct vcl *vcl, struct vrt_privs *privs)
 	CHECK_OBJ_NOTNULL(privs, VRT_PRIVS_MAGIC);
 	VTAILQ_FOREACH_SAFE(vp, &privs->privs, list, vp1) {
 		VTAILQ_REMOVE(&privs->privs, vp, list);
+		CHECK_OBJ_NOTNULL(vp, VRT_PRIV_MAGIC);
 		VRT_priv_fini(vp->priv);
+		ZERO_OBJ(vp, sizeof *vp);
 	}
 	ZERO_OBJ(privs, sizeof *privs);
 }

From reza at naghibi.com  Wed Aug 19 13:17:08 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:08 +0000 (UTC)
Subject: [6.0] 911d2b8c9 Use req->ws for the ctx when piping
Message-ID: <20200819131708.C97906310@lists.varnish-cache.org>


commit 911d2b8c999b1f1a637aeea71410006cecf8771e
Author: Reza Naghibi <reza at naghibi.com>
Date:   Fri Aug 14 10:01:02 2020 -0400

    Use req->ws for the ctx when piping
    
    Fixes #3329
    Fixes #3385

diff --git a/bin/varnishd/cache/cache_vcl_vrt.c b/bin/varnishd/cache/cache_vcl_vrt.c
index a63161b81..4133f46b7 100644
--- a/bin/varnishd/cache/cache_vcl_vrt.c
+++ b/bin/varnishd/cache/cache_vcl_vrt.c
@@ -353,11 +353,13 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 		VCL_Req2Ctx(&ctx, req);
 	}
 	if (bo != NULL) {
-		if (req)
-			assert(method == VCL_MET_PIPE);
 		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 		CHECK_OBJ_NOTNULL(bo->vcl, VCL_MAGIC);
 		VCL_Bo2Ctx(&ctx, bo);
+		if (req) {
+			assert(method == VCL_MET_PIPE);
+			ctx.ws = req->ws;
+		}
 	}
 	assert(ctx.now != 0);
 	ctx.syntax = ctx.vcl->conf->syntax;
diff --git a/bin/varnishtest/tests/r03329.vtc b/bin/varnishtest/tests/r03329.vtc
new file mode 100644
index 000000000..f3453bb61
--- /dev/null
+++ b/bin/varnishtest/tests/r03329.vtc
@@ -0,0 +1,18 @@
+varnishtest "Lost reason from pipe to synth"
+
+server s1 {} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		return (pipe);
+	}
+	sub vcl_pipe {
+		return (synth(505, req.http.foo + "/bar"));
+	}
+} -start
+
+client c1 {
+	txreq -hdr "foo: foo"
+	rxresp
+	expect resp.reason == "foo/bar"
+} -run
diff --git a/bin/varnishtest/tests/r03385.vtc b/bin/varnishtest/tests/r03385.vtc
new file mode 100644
index 000000000..cee34301f
--- /dev/null
+++ b/bin/varnishtest/tests/r03385.vtc
@@ -0,0 +1,23 @@
+varnishtest "Use a priv in vcl_pipe"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import debug;
+
+	sub vcl_recv {
+		return (pipe);
+	}
+
+	sub vcl_pipe {
+		debug.test_priv_task();
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+} -run

From reza at naghibi.com  Wed Aug 19 13:17:08 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 19 Aug 2020 13:17:08 +0000 (UTC)
Subject: [6.0] 61c193889 Fix shard director in pipe mode
Message-ID: <20200819131708.E5902631E@lists.varnish-cache.org>


commit 61c193889a1d1ba810100e278faaa0f7bc2f9a3a
Author: Reza Naghibi <reza at naghibi.com>
Date:   Mon Aug 17 11:35:18 2020 -0400

    Fix shard director in pipe mode
    
    Switch the ctx and workspace to the req when resolving in pipe mode.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 5a17db15a..0e6e1ab59 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -352,7 +352,7 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo)
 	v1a.req = req->acct.req_hdrbytes;
 	req->acct.req_hdrbytes = 0;
 
-	req->res_mode = RES_PIPE;
+	assert(req->res_mode & RES_PIPE);
 
 	pfd = vbe_dir_getfd(req->wrk, bp, bo, 0);
 
diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c
index 3b6e4b71f..14e569df1 100644
--- a/bin/varnishd/cache/cache_director.c
+++ b/bin/varnishd/cache/cache_director.c
@@ -193,8 +193,11 @@ VDI_Http1Pipe(struct req *req, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
+	req->res_mode = RES_PIPE;
+
 	d = vdi_resolve(req->wrk, bo);
 	if (d == NULL || d->http1pipe == NULL) {
+		req->res_mode = 0;
 		VSLb(bo->vsl, SLT_VCL_Error, "Backend does not support pipe");
 		return (SC_TX_ERROR);
 	}
diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c
index 846d75a6a..f6e8b67d2 100644
--- a/lib/libvmod_directors/vmod_shard.c
+++ b/lib/libvmod_directors/vmod_shard.c
@@ -759,7 +759,13 @@ vmod_shard_resolve(const struct director *dir, struct worker *wrk,
 	ctx->sp = bo->sp;
 	ctx->now = bo->t_prev;
 	ctx->ws = bo->ws;
-	ctx->method	= VCL_MET_BACKEND_FETCH;
+	if (bo->req) {
+		CHECK_OBJ_NOTNULL(bo->req, REQ_MAGIC);
+		if (bo->req->res_mode & RES_PIPE) {
+			ctx->req = bo->req;
+			ctx->ws = ctx->req->ws;
+		}
+	}
 
 	pp = vmod_shard_param_read(ctx, vshard,
 				   vshard->param, pstk, "shard_resolve");

From reza at naghibi.com  Thu Aug 20 17:52:07 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Thu, 20 Aug 2020 17:52:07 +0000 (UTC)
Subject: [6.0] 39abc13e2 Add VCL_TIME fix
Message-ID: <20200820175207.BC3BB61735@lists.varnish-cache.org>


commit 39abc13e2ef06c7fecc2ea267e20fe3b47fe5fdf
Author: Reza Naghibi <reza at naghibi.com>
Date:   Tue Aug 18 12:06:19 2020 -0400

    Add VCL_TIME fix

diff --git a/doc/changes.rst b/doc/changes.rst
index 5b42e3a2a..08ccb7bf0 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -32,6 +32,9 @@ Varnish Cache 6.0.7 (YYYY-MM-DD)
 
 * Add weighted backend to vmod_shard.
 
+* Fix an issue where an undefined value is returned when converting
+  a VCL_TIME to a string. (3308_)
+
 * Fix an issue where the wrong workspace is used during `vcl_pipe`.
   (3329_) (3361_) (3385_)
 
@@ -72,6 +75,7 @@ Varnish Cache 6.0.7 (YYYY-MM-DD)
 
 * Expose the master and worker PIDs via the CLI: `varnishadm pid`. (3171_)
 
+.. _3308: https://github.com/varnishcache/varnish-cache/pull/3308
 .. _3385: https://github.com/varnishcache/varnish-cache/issues/3385
 .. _3361: https://github.com/varnishcache/varnish-cache/issues/3361
 .. _3329: https://github.com/varnishcache/varnish-cache/issues/3329

From reza at naghibi.com  Thu Aug 20 17:52:07 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Thu, 20 Aug 2020 17:52:07 +0000 (UTC)
Subject: [6.0] d6866bc0a 6.0.7 changelog
Message-ID: <20200820175207.B131F61733@lists.varnish-cache.org>


commit d6866bc0a65b1e2359d6d2d294501628454683dd
Author: Reza Naghibi <reza at naghibi.com>
Date:   Tue Aug 18 11:46:15 2020 -0400

    6.0.7 changelog

diff --git a/doc/changes.rst b/doc/changes.rst
index da4adaeff..5b42e3a2a 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -26,6 +26,71 @@ http://varnish-cache.org/docs/trunk/whats-new/index.html and via
 individual releases. These documents are updated as part of the
 release process.
 
+================================
+Varnish Cache 6.0.7 (YYYY-MM-DD)
+================================
+
+* Add weighted backend to vmod_shard.
+
+* Fix an issue where the wrong workspace is used during `vcl_pipe`.
+  (3329_) (3361_) (3385_)
+
+* Fix an assert situation when running out of workspace during H2
+  delivery. (3382_)
+
+* Fix an issue where we can incorrectly re-use a req.body. This
+  introduces a new bereq body attribute: `bo->bereq_body`. (3093_)
+
+* Fix `send_timeout` for certain kinds of slow HTTP1 writes. (3189_)
+
+* Fix std.rollback() when not followed by a restart/retry. (3009_)
+
+* Fix an assert situation when running out of workspace during ESI
+  processing. (3253_)
+
+* Don't override Content-Encoding on 304 responses. (3169_)
+
+* Better thread creation signalling. (2942_)
+
+* Fix an error where when using a certain internal status code will
+  trigger an assert. (3301_)
+
+* Fix a situation where a closed connection is recycled. (3266_)
+
+* Fix an assert situation when doing a conditional fetch. (3273_)
+
+* Fix an issue where `varnishadm` improperly returns an empty 200 response
+  when you overflow. (3038_)
+
+* Add new VCL parameters to warn when too many VCLs are loaded:
+  `max_vcl` and `max_vcl_handling`. (2713_)
+
+* Fix an issue where we can use a streaming object in a conditional
+  fetch too soon. (3089_)
+
+* Fix an issue where the Age is dropped when passing. (3221_)
+
+* Expose the master and worker PIDs via the CLI: `varnishadm pid`. (3171_)
+
+.. _3385: https://github.com/varnishcache/varnish-cache/issues/3385
+.. _3361: https://github.com/varnishcache/varnish-cache/issues/3361
+.. _3329: https://github.com/varnishcache/varnish-cache/issues/3329
+.. _3382: https://github.com/varnishcache/varnish-cache/issues/3382
+.. _3093: https://github.com/varnishcache/varnish-cache/pull/3093
+.. _3189: https://github.com/varnishcache/varnish-cache/issues/3189
+.. _3009: https://github.com/varnishcache/varnish-cache/issues/3009
+.. _3253: https://github.com/varnishcache/varnish-cache/issues/3253
+.. _3169: https://github.com/varnishcache/varnish-cache/issues/3169
+.. _2942: https://github.com/varnishcache/varnish-cache/pull/2942
+.. _3301: https://github.com/varnishcache/varnish-cache/issues/3301
+.. _3266: https://github.com/varnishcache/varnish-cache/issues/3266
+.. _3273: https://github.com/varnishcache/varnish-cache/issues/3273
+.. _3038: https://github.com/varnishcache/varnish-cache/issues/3038
+.. _2713: https://github.com/varnishcache/varnish-cache/issues/2713
+.. _3089: https://github.com/varnishcache/varnish-cache/issues/3089
+.. _3221: https://github.com/varnishcache/varnish-cache/issues/3221
+.. _3171: https://github.com/varnishcache/varnish-cache/pull/3171
+
 ================================
 Varnish Cache 6.0.6 (2020-02-04)
 ================================

From phk at FreeBSD.org  Mon Aug 24 12:30:10 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 24 Aug 2020 12:30:10 +0000 (UTC)
Subject: [master] 54a41e217 Code deduplication,
 and an assert on the wrong side of an #endif
Message-ID: <20200824123010.8C2FA9C3CB@lists.varnish-cache.org>


commit 54a41e2173e396586d97e6d356e5dfb4ebd4d9ad
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 24 09:44:09 2020 +0000

    Code deduplication, and an assert on the wrong side of an #endif

diff --git a/bin/varnishtest/vtc_gzip.c b/bin/varnishtest/vtc_gzip.c
index 2f9043a9c..2e56d41d5 100644
--- a/bin/varnishtest/vtc_gzip.c
+++ b/bin/varnishtest/vtc_gzip.c
@@ -40,6 +40,21 @@
 
 #define OVERHEAD 64L
 
+#ifdef VGZ_EXTENSIONS
+static void
+vtc_report_gz_bits(const struct http *hp, z_stream *vz)
+{
+	vtc_log(hp->vl, 4, "startbit = %ju %ju/%ju",
+	    (uintmax_t)vz->start_bit,
+	    (uintmax_t)vz->start_bit >> 3, (uintmax_t)vz->start_bit & 7);
+	vtc_log(hp->vl, 4, "lastbit = %ju %ju/%ju",
+	    (uintmax_t)vz->last_bit,
+	    (uintmax_t)vz->last_bit >> 3, (uintmax_t)vz->last_bit & 7);
+	vtc_log(hp->vl, 4, "stopbit = %ju %ju/%ju",
+	    (uintmax_t)vz->stop_bit,
+	    (uintmax_t)vz->stop_bit >> 3, (uintmax_t)vz->stop_bit & 7);
+}
+#endif
 
 void
 vtc_gzip(const struct http *hp, const char *input, char **body, long *bodylen)
@@ -72,17 +87,9 @@ vtc_gzip(const struct http *hp, const char *input, char **body, long *bodylen)
 		vtc_log(hp->vl, hp->fatal,
 		    "Wrong gzip residual got %d wanted %d",
 		    i, hp->gzipresidual);
-	vtc_log(hp->vl, 4, "startbit = %ju %ju/%ju",
-	    (uintmax_t)vz.start_bit,
-	    (uintmax_t)vz.start_bit >> 3, (uintmax_t)vz.start_bit & 7);
-	vtc_log(hp->vl, 4, "lastbit = %ju %ju/%ju",
-	    (uintmax_t)vz.last_bit,
-	    (uintmax_t)vz.last_bit >> 3, (uintmax_t)vz.last_bit & 7);
-	vtc_log(hp->vl, 4, "stopbit = %ju %ju/%ju",
-	    (uintmax_t)vz.stop_bit,
-	    (uintmax_t)vz.stop_bit >> 3, (uintmax_t)vz.stop_bit & 7);
-	assert(Z_OK == deflateEnd(&vz));
+	vtc_report_gz_bits(hp, &vz);
 #endif
+	assert(Z_OK == deflateEnd(&vz));
 }
 
 void
@@ -119,15 +126,7 @@ vtc_gunzip(struct http *hp, char *body, long *bodylen)
 	vtc_dump(hp->vl, 4, "body", body, *bodylen);
 	bprintf(hp->bodylen, "%ld", *bodylen);
 #ifdef VGZ_EXTENSIONS
-	vtc_log(hp->vl, 4, "startbit = %ju %ju/%ju",
-	    (uintmax_t)vz.start_bit,
-	    (uintmax_t)vz.start_bit >> 3, (uintmax_t)vz.start_bit & 7);
-	vtc_log(hp->vl, 4, "lastbit = %ju %ju/%ju",
-	    (uintmax_t)vz.last_bit,
-	    (uintmax_t)vz.last_bit >> 3, (uintmax_t)vz.last_bit & 7);
-	vtc_log(hp->vl, 4, "stopbit = %ju %ju/%ju",
-	    (uintmax_t)vz.stop_bit,
-	    (uintmax_t)vz.stop_bit >> 3, (uintmax_t)vz.stop_bit & 7);
+	vtc_report_gz_bits(hp, &vz);
 #endif
 	if (i != Z_STREAM_END)
 		vtc_log(hp->vl, hp->fatal,

From phk at FreeBSD.org  Mon Aug 24 12:30:10 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 24 Aug 2020 12:30:10 +0000 (UTC)
Subject: [master] 438d765ef Rewrite the core of the gzip/gunzip code to use
 VSBs.
Message-ID: <20200824123010.A18029C3CE@lists.varnish-cache.org>


commit 438d765ef1ffb80e6adb59daa270bc317298cc86
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 24 10:37:59 2020 +0000

    Rewrite the core of the gzip/gunzip code to use VSBs.

diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index 2300288d4..cd21edb75 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -132,7 +132,7 @@ void stop_h2(struct http *hp);
 void b64_settings(const struct http *hp, const char *s);
 
 /* vtc_gzip.c */
-void vtc_gzip(const struct http *, const char *, char **, long *);
+void vtc_gzip(struct http *, const char *, char **, long *);
 void vtc_gunzip(struct http *, char *, long *);
 
 /* vtc_subr.c */
diff --git a/bin/varnishtest/vtc_gzip.c b/bin/varnishtest/vtc_gzip.c
index 2e56d41d5..9615efdfd 100644
--- a/bin/varnishtest/vtc_gzip.c
+++ b/bin/varnishtest/vtc_gzip.c
@@ -38,100 +38,149 @@
 #include "vtc_http.h"
 #include "vgz.h"
 
-#define OVERHEAD 64L
-
 #ifdef VGZ_EXTENSIONS
 static void
-vtc_report_gz_bits(const struct http *hp, z_stream *vz)
+vtc_report_gz_bits(struct vtclog *vl, const z_stream *vz)
 {
-	vtc_log(hp->vl, 4, "startbit = %ju %ju/%ju",
+	vtc_log(vl, 4, "startbit = %ju %ju/%ju",
 	    (uintmax_t)vz->start_bit,
 	    (uintmax_t)vz->start_bit >> 3, (uintmax_t)vz->start_bit & 7);
-	vtc_log(hp->vl, 4, "lastbit = %ju %ju/%ju",
+	vtc_log(vl, 4, "lastbit = %ju %ju/%ju",
 	    (uintmax_t)vz->last_bit,
 	    (uintmax_t)vz->last_bit >> 3, (uintmax_t)vz->last_bit & 7);
-	vtc_log(hp->vl, 4, "stopbit = %ju %ju/%ju",
+	vtc_log(vl, 4, "stopbit = %ju %ju/%ju",
 	    (uintmax_t)vz->stop_bit,
 	    (uintmax_t)vz->stop_bit >> 3, (uintmax_t)vz->stop_bit & 7);
 }
 #endif
 
-void
-vtc_gzip(const struct http *hp, const char *input, char **body, long *bodylen)
+static struct vsb *
+vtc_gzip_vsb(struct vtclog *vl, int fatal, int gzip_level, const struct vsb *vin, int *residual)
 {
-	unsigned l;
 	z_stream vz;
-#ifdef VGZ_EXTENSIONS
+	struct vsb *vout;
 	int i;
-#endif
+	char buf[BUFSIZ];
 
+	AN(residual);
 	memset(&vz, 0, sizeof vz);
+	vout = VSB_new_auto();
+	AN(vout);
 
-	l = strlen(input);
-	*body = calloc(1, l + OVERHEAD);
-	AN(*body);
+	vz.next_in = (void*)VSB_data(vin);
+	vz.avail_in = VSB_len(vin);
 
-	vz.next_in = TRUST_ME(input);
-	vz.avail_in = l;
+	assert(Z_OK == deflateInit2(&vz,
+	    gzip_level, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY));
+
+	do {
+		vz.next_out = (void*)buf;
+		vz.avail_out = sizeof buf;
+		i = deflate(&vz, Z_FINISH);
+		if (vz.avail_out != sizeof buf)
+			VSB_bcat(vout, buf, sizeof buf - vz.avail_out);
+	} while (i == Z_OK || i == Z_BUF_ERROR);
+	if (i != Z_STREAM_END)
+		vtc_log(vl, fatal,
+		    "Gzip error = %d (%s) in:%jd out:%jd",
+		    i, vz.msg, (intmax_t)vz.total_in, (intmax_t)vz.total_out);
+	AZ(VSB_finish(vout));
+#ifdef VGZ_EXTENSIONS
+	*residual = vz.stop_bit & 7;
+	vtc_report_gz_bits(vl, &vz);
+#else
+	*residual = 0;
+#endif
+	assert(Z_OK == deflateEnd(&vz));
+	return (vout);
+}
 
-	vz.next_out = TRUST_ME(*body);
-	vz.avail_out = l + OVERHEAD;
+void
+vtc_gzip(struct http *hp, const char *input, char **body, long *bodylen)
+{
+	struct vsb *vin, *vout;
+	int res;
+
+	vin = VSB_new_auto();
+	AN(vin);
+	VSB_bcat(vin, input, strlen(input));
+	AZ(VSB_finish(vin));
+	vout = vtc_gzip_vsb(hp->vl, hp->fatal, hp->gziplevel, vin, &res);
+	VSB_destroy(&vin);
 
-	assert(Z_OK == deflateInit2(&vz,
-	    hp->gziplevel, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY));
-	assert(Z_STREAM_END == deflate(&vz, Z_FINISH));
-	*bodylen = vz.total_out;
 #ifdef VGZ_EXTENSIONS
-	i = vz.stop_bit & 7;
-	if (hp->gzipresidual >= 0 && hp->gzipresidual != i)
+	if (hp->gzipresidual >= 0 && hp->gzipresidual != res)
 		vtc_log(hp->vl, hp->fatal,
 		    "Wrong gzip residual got %d wanted %d",
-		    i, hp->gzipresidual);
-	vtc_report_gz_bits(hp, &vz);
+		    res, hp->gzipresidual);
 #endif
-	assert(Z_OK == deflateEnd(&vz));
+	*body = malloc(VSB_len(vout) + 1);
+	AN(*body);
+	memcpy(*body, VSB_data(vout), VSB_len(vout) + 1);
+	*bodylen = VSB_len(vout);
+	VSB_destroy(&vout);
+	vtc_log(hp->vl, 3, "new bodylen %ld", *bodylen);
+	vtc_dump(hp->vl, 4, "body", *body, *bodylen);
+	bprintf(hp->bodylen, "%ld", *bodylen);
 }
 
-void
-vtc_gunzip(struct http *hp, char *body, long *bodylen)
+static struct vsb *
+vtc_gunzip_vsb(struct vtclog *vl, int fatal, const struct vsb *vin)
 {
 	z_stream vz;
-	char *p;
-	unsigned l;
+	struct vsb *vout;
 	int i;
+	char buf[BUFSIZ];
 
 	memset(&vz, 0, sizeof vz);
+	vout = VSB_new_auto();
+	AN(vout);
+
+	vz.next_in = (void*)VSB_data(vin);
+	vz.avail_in = VSB_len(vin);
+
+	assert(Z_OK == inflateInit2(&vz, 31));
+
+	do {
+		vz.next_out = (void*)buf;
+		vz.avail_out = sizeof buf;
+		i = inflate(&vz, Z_FINISH);
+		if (vz.avail_out != sizeof buf)
+			VSB_bcat(vout, buf, sizeof buf - vz.avail_out);
+	} while (i == Z_OK || i == Z_BUF_ERROR);
+	if (i != Z_STREAM_END)
+		vtc_log(vl, fatal,
+		    "Gunzip error = %d (%s) in:%jd out:%jd",
+		    i, vz.msg, (intmax_t)vz.total_in, (intmax_t)vz.total_out);
+	AZ(VSB_finish(vout));
+#ifdef VGZ_EXTENSIONS
+	vtc_report_gz_bits(vl, &vz);
+#endif
+	assert(Z_OK == inflateEnd(&vz));
+	return (vout);
+}
+
+void
+vtc_gunzip(struct http *hp, char *body, long *bodylen)
+{
+	struct vsb *vin, *vout;
 
 	AN(body);
 	if (body[0] != (char)0x1f || body[1] != (char)0x8b)
 		vtc_log(hp->vl, hp->fatal,
 		    "Gunzip error: body lacks gzip magic");
-	vz.next_in = TRUST_ME(body);
-	vz.avail_in = *bodylen;
-
-	l = *bodylen * 10;
-	p = calloc(1, l);
-	AN(p);
 
-	vz.next_out = TRUST_ME(p);
-	vz.avail_out = l;
+	vin = VSB_new_auto();
+	AN(vin);
+	VSB_bcat(vin, body, *bodylen);
+	AZ(VSB_finish(vin));
+	vout = vtc_gunzip_vsb(hp->vl, hp->fatal, vin);
+	VSB_destroy(&vin);
 
-	assert(Z_OK == inflateInit2(&vz, 31));
-	i = inflate(&vz, Z_FINISH);
-	assert(vz.total_out < l);
-	*bodylen = vz.total_out;
-	memcpy(body, p, *bodylen);
-	free(p);
+	memcpy(body, VSB_data(vout), VSB_len(vout) + 1);
+	*bodylen = VSB_len(vout);
+	VSB_destroy(&vout);
 	vtc_log(hp->vl, 3, "new bodylen %ld", *bodylen);
 	vtc_dump(hp->vl, 4, "body", body, *bodylen);
 	bprintf(hp->bodylen, "%ld", *bodylen);
-#ifdef VGZ_EXTENSIONS
-	vtc_report_gz_bits(hp, &vz);
-#endif
-	if (i != Z_STREAM_END)
-		vtc_log(hp->vl, hp->fatal,
-		    "Gunzip error = %d (%s) in:%jd out:%jd",
-		    i, vz.msg, (intmax_t)vz.total_in, (intmax_t)vz.total_out);
-	assert(Z_OK == inflateEnd(&vz));
-	body[*bodylen] = '\0';
 }

From phk at FreeBSD.org  Mon Aug 24 13:21:06 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 24 Aug 2020 13:21:06 +0000 (UTC)
Subject: [master] b0e252d73 force mv to avoid issues on alpine
Message-ID: <20200824132106.498739DA2C@lists.varnish-cache.org>


commit b0e252d73bae3a9120486f7f829955ca206ecc63
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Wed Aug 19 08:59:41 2020 -0700

    force mv to avoid issues on alpine

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 4ef49bc66..b5cf56dc5 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -76,12 +76,12 @@ distclean-local:
 
 include/cli.rst: $(top_builddir)/bin/varnishd/varnishd
 	$(top_builddir)/bin/varnishd/varnishd -x cli > ${@}_
-	mv ${@}_ ${@}
+	mv -f ${@}_ ${@}
 BUILT_SOURCES = include/cli.rst
 
 include/params.rst: $(top_builddir)/bin/varnishd/varnishd
 	$(top_builddir)/bin/varnishd/varnishd -x parameter > ${@}_
-	mv ${@}_ ${@}
+	mv -f ${@}_ ${@}
 BUILT_SOURCES += include/params.rst
 
 COUNTERS = \
@@ -99,62 +99,62 @@ include/counters.rst: $(top_srcdir)/lib/libvcc/vsctool.py $(COUNTERS)
 	for i in $(COUNTERS); do \
 		$(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -r $$i >> ${@}_ ; \
 	done
-	mv ${@}_ ${@}
+	mv -f ${@}_ ${@}
 
 BUILT_SOURCES += include/counters.rst
 
 include/varnishncsa_options.rst: $(top_builddir)/bin/varnishncsa/varnishncsa
 	$(top_builddir)/bin/varnishncsa/varnishncsa --options > ${@}_
-	mv ${@}_ ${@}
+	mv -f ${@}_ ${@}
 include/varnishncsa_synopsis.rst: $(top_builddir)/bin/varnishncsa/varnishncsa
 	$(top_builddir)/bin/varnishncsa/varnishncsa --synopsis > ${@}_
-	mv ${@}_ ${@}
+	mv -f ${@}_ ${@}
 BUILT_SOURCES += include/varnishncsa_options.rst \
 	include/varnishncsa_synopsis.rst
 
 include/varnishlog_options.rst: $(top_builddir)/bin/varnishlog/varnishlog
 	$(top_builddir)/bin/varnishlog/varnishlog --options > ${@}_
-	mv ${@}_ ${@}
+	mv -f ${@}_ ${@}
 include/varnishlog_synopsis.rst: $(top_builddir)/bin/varnishlog/varnishlog
 	$(top_builddir)/bin/varnishlog/varnishlog --synopsis > ${@}_
-	mv ${@}_ ${@}
+	mv -f ${@}_ ${@}
 BUILT_SOURCES += include/varnishlog_options.rst \
 	include/varnishlog_synopsis.rst
 
 include/varnishtop_options.rst: $(top_builddir)/bin/varnishtop/varnishtop
 	$(top_builddir)/bin/varnishtop/varnishtop --options > ${@}_
-	mv ${@}_ ${@}
+	mv -f ${@}_ ${@}
 include/varnishtop_synopsis.rst: $(top_builddir)/bin/varnishtop/varnishtop
 	$(top_builddir)/bin/varnishtop/varnishtop --synopsis > ${@}_
-	mv ${@}_ ${@}
+	mv -f ${@}_ ${@}
 BUILT_SOURCES += include/varnishtop_options.rst \
 	 include/varnishtop_synopsis.rst
 
 include/varnishhist_options.rst: $(top_builddir)/bin/varnishhist/varnishhist
 	$(top_builddir)/bin/varnishhist/varnishhist --options > ${@}_
-	mv ${@}_ ${@}
+	mv -f ${@}_ ${@}
 include/varnishhist_synopsis.rst: $(top_builddir)/bin/varnishhist/varnishhist
 	$(top_builddir)/bin/varnishhist/varnishhist --synopsis > ${@}_
-	mv ${@}_ ${@}
+	mv -f ${@}_ ${@}
 BUILT_SOURCES += include/varnishhist_options.rst \
 	 include/varnishhist_synopsis.rst
 
 include/varnishstat_options.rst: $(top_builddir)/bin/varnishstat/varnishstat
 	$(top_builddir)/bin/varnishstat/varnishstat --options > ${@}_
-	mv ${@}_ ${@}
+	mv -f ${@}_ ${@}
 include/varnishstat_synopsis.rst: $(top_builddir)/bin/varnishstat/varnishstat
 	$(top_builddir)/bin/varnishstat/varnishstat --synopsis > ${@}_
-	mv ${@}_ ${@}
+	mv -f ${@}_ ${@}
 include/varnishstat_bindings.rst: $(top_builddir)/bin/varnishstat/varnishstat
 	$(top_builddir)/bin/varnishstat/varnishstat --bindings > ${@}_
-	mv ${@}_ ${@}
+	mv -f ${@}_ ${@}
 BUILT_SOURCES += include/varnishstat_options.rst \
 	 include/varnishstat_synopsis.rst \
 	 include/varnishstat_bindings.rst
 
 include/vsl-tags.rst: $(top_builddir)/lib/libvarnishapi/vsl2rst
 	$(top_builddir)/lib/libvarnishapi/vsl2rst > ${@}_
-	mv ${@}_ ${@}
+	mv -f ${@}_ ${@}
 BUILT_SOURCES += include/vsl-tags.rst
 
 VTCSYN_SRC = $(top_srcdir)/bin/varnishtest/vtc.c \
@@ -169,7 +169,7 @@ VTCSYN_SRC = $(top_srcdir)/bin/varnishtest/vtc.c \
 	     $(top_srcdir)/bin/varnishtest/vtc_varnish.c
 include/vtc-syntax.rst: vtc-syntax.py $(VTCSYN_SRC)
 	$(AM_V_GEN) $(PYTHON) $(top_srcdir)/doc/sphinx/vtc-syntax.py $(VTCSYN_SRC) > ${@}_
-	@mv ${@}_ ${@}
+	@mv -f ${@}_ ${@}
 BUILT_SOURCES += include/vtc-syntax.rst
 
 # XXX copy/paste rules need some TLC

From reza at naghibi.com  Wed Aug 26 16:27:06 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 26 Aug 2020 16:27:06 +0000 (UTC)
Subject: [6.0] 18bf1d491 Move cache.h changes to the bottom
Message-ID: <20200826162707.096D8A1E29@lists.varnish-cache.org>


commit 18bf1d491bf857f05b703c29f690d1be33a9bddc
Author: Reza Naghibi <reza at naghibi.com>
Date:   Wed Aug 26 12:25:48 2020 -0400

    Move cache.h changes to the bottom

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 95d6d97b1..5cff79beb 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -390,8 +390,7 @@ struct busyobj {
 	 * All fields from retries and down are zeroed when the busyobj
 	 * is recycled.
 	 */
-	unsigned		retries;
-	enum req_body_state_e	initial_req_body_status;
+	int			retries;
 	struct req		*req;
 	struct sess		*sp;
 	struct worker		*wrk;
@@ -403,7 +402,6 @@ struct busyobj {
 	struct http		*bereq0;
 	struct http		*bereq;
 	struct http		*beresp;
-	struct objcore		*bereq_body;
 	struct objcore		*stale_oc;
 	struct objcore		*fetch_objcore;
 
@@ -439,6 +437,9 @@ struct busyobj {
 
 	uint16_t		err_code;
 	const char		*err_reason;
+
+	enum req_body_state_e	initial_req_body_status;
+	struct objcore		*bereq_body;
 };
 
 

From nils.goroll at uplex.de  Mon Aug 31 13:33:08 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 31 Aug 2020 13:33:08 +0000 (UTC)
Subject: [master] ff73bc4e5 Add counters: beresp_uncacheable, beresp_shortlived
Message-ID: <20200831133308.6E6C9AE71E@lists.varnish-cache.org>


commit ff73bc4e50f34988507c8a8ba89569ecd64895fc
Author: Emanuele Rocca <ema at wikimedia.org>
Date:   Mon Aug 31 10:27:14 2020 +0200

    Add counters: beresp_uncacheable, beresp_shortlived
    
    Uncacheable and shortlived objects end up in transient memory. Add two
    counters to track both type of responses separately.

diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc
index a7cacb7cc..41c2b1228 100644
--- a/bin/varnishd/VSC_main.vsc
+++ b/bin/varnishd/VSC_main.vsc
@@ -137,6 +137,19 @@
 	Count of misses. A cache miss indicates the object was fetched from
 	the backend before delivering it to the client.
 
+.. varnish_vsc:: beresp_uncacheable
+   :group: wrk
+   :oneliner: Uncacheable backend responses
+
+   Count of backend responses considered uncacheable.
+
+.. varnish_vsc:: beresp_shortlived
+   :group: wrk
+   :oneliner: Shortlived objects
+
+   Count of objects created with ttl+grace+keep shorter than the 'shortlived'
+   runtime parameter.
+
 .. varnish_vsc:: backend_conn
 	:oneliner:	Backend conn. success
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index ce603e509..1c71e08b4 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -58,8 +58,14 @@ vbf_allocobj(struct busyobj *bo, unsigned l)
 
 	lifetime = oc->ttl + oc->grace + oc->keep;
 
-	if (bo->uncacheable || lifetime < cache_param->shortlived)
+	if (bo->uncacheable) {
 		stv = stv_transient;
+		bo->wrk->stats->beresp_uncacheable++;
+	}
+	else if (lifetime < cache_param->shortlived) {
+		stv = stv_transient;
+		bo->wrk->stats->beresp_shortlived++;
+	}
 	else
 		stv = bo->storage;
 
diff --git a/bin/varnishtest/tests/b00066.vtc b/bin/varnishtest/tests/b00066.vtc
index 7eb286f4b..d1c224e04 100644
--- a/bin/varnishtest/tests/b00066.vtc
+++ b/bin/varnishtest/tests/b00066.vtc
@@ -100,3 +100,5 @@ client c1 {
 
 
 varnish v1 -expect *.s1.req == 12
+varnish v1 -expect beresp_uncacheable == 12
+varnish v1 -expect beresp_shortlived == 0
diff --git a/bin/varnishtest/tests/s00007.vtc b/bin/varnishtest/tests/s00007.vtc
index 39eecf5b2..0374605ab 100644
--- a/bin/varnishtest/tests/s00007.vtc
+++ b/bin/varnishtest/tests/s00007.vtc
@@ -40,3 +40,4 @@ client c1 {
 	expect resp.body == "foo"
 } -run
 
+varnish v1 -expect beresp_shortlived == 1

From phk at FreeBSD.org  Mon Aug 31 14:25:07 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 31 Aug 2020 14:25:07 +0000 (UTC)
Subject: [master] f658f830d Unify some of the code from client/server using a
 "session" concept, which will become more and more important.
Message-ID: <20200831142507.97712AFC19@lists.varnish-cache.org>


commit f658f830df003e7258cba883c3ccd3a2f2cbfb95
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 31 14:23:06 2020 +0000

    Unify some of the code from client/server using a "session"
    concept, which will become more and more important.

diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am
index 34f11f77e..61c2be73d 100644
--- a/bin/varnishtest/Makefile.am
+++ b/bin/varnishtest/Makefile.am
@@ -47,6 +47,7 @@ varnishtest_SOURCES = \
 		vtc_process.c \
 		vtc_proxy.c \
 		vtc_server.c \
+		vtc_sess.c \
 		vtc_subr.c \
 		vtc_syslog.c \
 		vtc_varnish.c
diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index cd21edb75..23a3a6472 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -88,6 +88,27 @@ extern int ign_unknown_macro;
 void init_server(void);
 void init_syslog(void);
 
+/* Sessions */
+struct vtc_sess *Sess_New(struct vtclog *vl, const char *name);
+void Sess_Destroy(struct vtc_sess **spp);
+int Sess_GetOpt(struct vtc_sess *, char * const **);
+int sess_process(struct vtclog *vl, const struct vtc_sess *,
+    const char *spec, int sock, int *sfd, const char *addr);
+
+typedef int sess_conn_f(void *priv, struct vtclog *);
+typedef void sess_disc_f(void *priv, struct vtclog *, int *fd);
+pthread_t
+Sess_Start_Thread(
+    void *priv,
+    struct vtc_sess *vsp,
+    sess_conn_f *conn,
+    sess_disc_f *disc,
+    const char *listen_addr,
+    int *asocket,
+    const char *spec
+);
+
+
 int http_process(struct vtclog *vl, const char *spec, int sock, int *sfd,
     const char *addr, int rcvbuf);
 
diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c
index 6ccfbb8d3..41cc3a581 100644
--- a/bin/varnishtest/vtc_client.c
+++ b/bin/varnishtest/vtc_client.c
@@ -52,19 +52,16 @@ struct client {
 	char			*name;
 	struct vtclog		*vl;
 	VTAILQ_ENTRY(client)	list;
+	struct vtc_sess		*vsp;
 
 	char			*spec;
 
 	char			connect[256];
-	const char		*addr;
-	int			rcvbuf;
+	char			*addr;
 
 	char			*proxy_spec;
 	int			proxy_version;
 
-	int			repeat;
-	unsigned		keepalive;
-
 	unsigned		running;
 	pthread_t		tp;
 };
@@ -207,54 +204,21 @@ client_connect(struct vtclog *vl, struct client *c)
  * Client thread
  */
 
-static void *
-client_thread(void *priv)
+static int
+client_conn(void *priv, struct vtclog *vl)
 {
 	struct client *c;
-	struct vtclog *vl;
-	int fd;
-	int i;
-	struct vsb *vsb;
 
 	CAST_OBJ_NOTNULL(c, priv, CLIENT_MAGIC);
-	AN(*c->connect);
-
-	vl = vtc_logopen(c->name);
-	pthread_cleanup_push(vtc_logclose, vl);
+	return (client_connect(vl, c));
+}
 
-	vsb = macro_expand(vl, c->connect);
-	AN(vsb);
-#if !defined(__sun)
-	pthread_cleanup_push((void (*)(void *))VSB_destroy, &vsb);
-#endif
-	c->addr = VSB_data(vsb);
-
-	if (c->repeat == 0)
-		c->repeat = 1;
-	if (c->repeat != 1)
-		vtc_log(vl, 2, "Started (%u iterations%s)", c->repeat,
-			c->keepalive ? " using keepalive" : "");
-	for (i = 0; i < c->repeat; i++) {
-		fd = client_connect(vl, c);
-
-		if (! c->keepalive)
-			fd = http_process(vl, c->spec, fd, NULL, c->addr,
-			    c->rcvbuf);
-		else
-			while (fd >= 0 && i++ < c->repeat)
-				fd = http_process(vl, c->spec, fd, NULL,
-				    c->addr, c->rcvbuf);
-		vtc_log(vl, 3, "closing fd %d", fd);
-		VTCP_close(&fd);
-	}
-	vtc_log(vl, 2, "Ending");
-#if !defined(__sun)
-	pthread_cleanup_pop(0);
-#endif
-	pthread_cleanup_pop(0);
-	VSB_destroy(&vsb);
-	vtc_logclose(vl);
-	return (NULL);
+static void
+client_disc(void *priv, struct vtclog *vl, int *fdp)
+{
+	(void)priv;
+	vtc_log(vl, 3, "closing fd %d", *fdp);
+	VTCP_close(fdp);
 }
 
 /**********************************************************************
@@ -271,6 +235,8 @@ client_new(const char *name)
 	REPLACE(c->name, name);
 	c->vl = vtc_logopen(name);
 	AN(c->vl);
+	c->vsp = Sess_New(c->vl, name);
+	AN(c->vsp);
 
 	bprintf(c->connect, "%s", "${v1_sock}");
 	VTAILQ_INSERT_TAIL(&clients, c, list);
@@ -286,9 +252,11 @@ client_delete(struct client *c)
 {
 
 	CHECK_OBJ_NOTNULL(c, CLIENT_MAGIC);
+	Sess_Destroy(&c->vsp);
 	vtc_logclose(c->vl);
 	free(c->spec);
 	free(c->name);
+	free(c->addr);
 	free(c->proxy_spec);
 	/* XXX: MEMLEAK (?)*/
 	FREE_OBJ(c);
@@ -301,11 +269,24 @@ client_delete(struct client *c)
 static void
 client_start(struct client *c)
 {
+	struct vsb *vsb;
 
 	CHECK_OBJ_NOTNULL(c, CLIENT_MAGIC);
 	vtc_log(c->vl, 2, "Starting client");
-	AZ(pthread_create(&c->tp, NULL, client_thread, c));
 	c->running = 1;
+	vsb = macro_expand(c->vl, c->connect);
+	AN(vsb);
+	REPLACE(c->addr, VSB_data(vsb));
+	VSB_destroy(&vsb);
+	c->tp = Sess_Start_Thread(
+	    c,
+	    c->vsp,
+	    client_conn,
+	    client_disc,
+	    c->addr,
+	    NULL,
+	    c->spec
+	);
 }
 
 /**********************************************************************
@@ -386,6 +367,10 @@ cmd_client(CMD_ARGS)
 		if (c->running)
 			client_wait(c);
 
+		AZ(c->running);
+		if (Sess_GetOpt(c->vsp, &av))
+			continue;
+
 		if (!strcmp(*av, "-connect")) {
 			bprintf(c->connect, "%s", av[1]);
 			av++;
@@ -403,20 +388,6 @@ cmd_client(CMD_ARGS)
 			av++;
 			continue;
 		}
-		if (!strcmp(*av, "-repeat")) {
-			c->repeat = atoi(av[1]);
-			av++;
-			continue;
-		}
-		if (!strcmp(*av, "-keepalive")) {
-			c->keepalive = 1;
-			continue;
-		}
-		if (!strcmp(*av, "-rcvbuf")) {
-			c->rcvbuf = atoi(av[1]);
-			av++;
-			continue;
-		}
 		if (!strcmp(*av, "-start")) {
 			client_start(c);
 			continue;
diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index 9ed58d923..4043c27da 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -50,16 +50,14 @@ struct server {
 	char			*name;
 	struct vtclog		*vl;
 	VTAILQ_ENTRY(server)	list;
+	struct vtc_sess		*vsp;
 	char			run;
 
-	int			repeat;
-	unsigned		keepalive;
 	char			*spec;
 
 	int			depth;
 	int			sock;
 	int			fd;
-	int			rcvbuf;
 	char			listen[256];
 	char			aaddr[32];
 	char			aport[32];
@@ -87,9 +85,10 @@ server_new(const char *name, struct vtclog *vl)
 	REPLACE(s->name, name);
 	s->vl = vtc_logopen(s->name);
 	AN(s->vl);
+	s->vsp = Sess_New(s->vl, name);
+	AN(s->vsp);
 
 	bprintf(s->listen, "%s", "127.0.0.1 0");
-	s->repeat = 1;
 	s->depth = 10;
 	s->sock = -1;
 	s->fd = -1;
@@ -108,6 +107,7 @@ server_delete(struct server *s)
 {
 
 	CHECK_OBJ_NOTNULL(s, SERVER_MAGIC);
+	Sess_Destroy(&s->vsp);
 	macro_undef(s->vl, s->name, "addr");
 	macro_undef(s->vl, s->name, "port");
 	macro_undef(s->vl, s->name, "sock");
@@ -219,56 +219,61 @@ server_listen(struct server *s)
  * Server thread
  */
 
-static void *
-server_thread(void *priv)
+static int
+server_conn(void *priv, struct vtclog *vl)
 {
 	struct server *s;
-	struct vtclog *vl;
-	int i, j, fd;
 	struct sockaddr_storage addr_s;
 	struct sockaddr *addr;
-	socklen_t l;
 	char abuf[VTCP_ADDRBUFSIZE];
 	char pbuf[VTCP_PORTBUFSIZE];
+	socklen_t l;
+	int fd;
 
 	CAST_OBJ_NOTNULL(s, priv, SERVER_MAGIC);
-	assert(s->sock >= 0);
 
-	vl = vtc_logopen(s->name);
-	pthread_cleanup_push(vtc_logclose, vl);
+	addr = (void*)&addr_s;
+	l = sizeof addr_s;
+	fd = accept(s->sock, addr, &l);
+	if (fd < 0)
+		vtc_fatal(vl, "Accept failed: %s", strerror(errno));
+	if (*s->listen != '/') {
+		VTCP_hisname(fd, abuf, sizeof abuf, pbuf, sizeof pbuf);
+		vtc_log(vl, 3, "accepted fd %d %s %s", fd, abuf, pbuf);
+	} else
+		vtc_log(vl, 3, "accepted fd %d 0.0.0.0 0", fd);
+	return (fd);
+}
 
-	vtc_log(vl, 2, "Started on %s (%u iterations%s)", s->listen,
-		s->repeat, s->keepalive ? " using keepalive" : "");
-	for (i = 0; i < s->repeat; i++) {
-		addr = (void*)&addr_s;
-		l = sizeof addr_s;
-		fd = accept(s->sock, addr, &l);
-		if (fd < 0)
-			vtc_fatal(vl, "Accept failed: %s", strerror(errno));
-		if (*s->listen != '/') {
-			VTCP_hisname(fd, abuf, sizeof abuf, pbuf, sizeof pbuf);
-			vtc_log(vl, 3, "accepted fd %d %s %s", fd, abuf, pbuf);
-		} else
-			vtc_log(vl, 3, "accepted fd %d 0.0.0.0 0", fd);
-		if (! s->keepalive)
-			fd = http_process(vl, s->spec, fd, &s->sock, s->listen,
-			    s->rcvbuf);
-		else
-			while (fd >= 0 && i++ < s->repeat)
-				fd = http_process(vl, s->spec, fd,
-				    &s->sock, s->listen, s->rcvbuf);
-		vtc_log(vl, 3, "shutting fd %d", fd);
-		j = shutdown(fd, SHUT_WR);
-		if (!VTCP_Check(j))
-			vtc_fatal(vl, "Shutdown failed: %s", strerror(errno));
-		VTCP_close(&fd);
-	}
-	vtc_log(vl, 2, "Ending");
-	pthread_cleanup_pop(0);
-	vtc_logclose(vl);
-	return (NULL);
+static void
+server_disc(void *priv, struct vtclog *vl, int *fdp)
+{
+	int j;
+	struct server *s;
+
+	CAST_OBJ_NOTNULL(s, priv, SERVER_MAGIC);
+	vtc_log(vl, 3, "shutting fd %d", *fdp);
+	j = shutdown(*fdp, SHUT_WR);
+	if (!VTCP_Check(j))
+		vtc_fatal(vl, "Shutdown failed: %s", strerror(errno));
+	VTCP_close(fdp);
 }
 
+static void
+server_start_thread(struct server *s)
+{
+
+	s->run = 1;
+	s->tp = Sess_Start_Thread(
+	    s,
+	    s->vsp,
+	    server_conn,
+	    server_disc,
+	    s->listen,
+	    &s->sock,
+	    s->spec
+	);
+}
 
 /**********************************************************************
  * Start the server thread
@@ -281,8 +286,7 @@ server_start(struct server *s)
 	vtc_log(s->vl, 2, "Starting server");
 	server_listen(s);
 	vtc_log(s->vl, 1, "Listen on %s", s->listen);
-	s->run = 1;
-	AZ(pthread_create(&s->tp, NULL, server_thread, s));
+	server_start_thread(s);
 }
 
 /**********************************************************************
@@ -304,7 +308,7 @@ server_dispatch_wrk(void *priv)
 	fd = s->fd;
 
 	vtc_log(vl, 3, "start with fd %d", fd);
-	fd = http_process(vl, s->spec, fd, &s->sock, s->listen, s->rcvbuf);
+	fd = sess_process(vl, s->vsp, s->spec, fd, &s->sock, s->listen);
 	vtc_log(vl, 3, "shutting fd %d", fd);
 	j = shutdown(fd, SHUT_WR);
 	if (!VTCP_Check(j))
@@ -530,15 +534,10 @@ cmd_server(CMD_ARGS)
 			server_wait(s);
 
 		AZ(s->run);
-		if (!strcmp(*av, "-repeat")) {
-			s->repeat = atoi(av[1]);
-			av++;
-			continue;
-		}
-		if (!strcmp(*av, "-keepalive")) {
-			s->keepalive = 1;
+
+		if (Sess_GetOpt(s->vsp, &av))
 			continue;
-		}
+
 		if (!strcmp(*av, "-listen")) {
 			if (s->sock >= 0)
 				VTCP_close(&s->sock);
@@ -546,11 +545,6 @@ cmd_server(CMD_ARGS)
 			av++;
 			continue;
 		}
-		if (!strcmp(*av, "-rcvbuf")) {
-			s->rcvbuf = atoi(av[1]);
-			av++;
-			continue;
-		}
 		if (!strcmp(*av, "-start")) {
 			server_start(s);
 			continue;

From dridi.boukelmoune at gmail.com  Mon Aug 31 16:00:07 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 16:00:07 +0000 (UTC)
Subject: [master] c80068a8c vapi: Declare VSC_OPT_f
Message-ID: <20200831160008.0C0C753DD@lists.varnish-cache.org>


commit c80068a8c5912d3e861a3912c216c47e6a4eb710
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 25 16:47:58 2020 +0200

    vapi: Declare VSC_OPT_f
    
    It is not limited to varnishstat.

diff --git a/bin/varnishstat/varnishstat_options.h b/bin/varnishstat/varnishstat_options.h
index 73cba2a99..35bdb5c11 100644
--- a/bin/varnishstat/varnishstat_options.h
+++ b/bin/varnishstat/varnishstat_options.h
@@ -38,14 +38,6 @@
 	    "Instead of presenting a continuously updated display,"	\
 	    " print the statistics to stdout."				\
 	)
-#define STAT_OPT_f							\
-	VOPT("f:", "[-f <glob>]", "Field inclusion glob",		\
-	    "Field inclusion glob."					\
-	    " Use backslash to escape characters. If the argument"	\
-	    " starts with '^' it is used as an exclusive glob."		\
-	    " Multiple -f arguments may be given. Inclusive globs"	\
-	    " are accumulative and are run before exclusive ones."	\
-	)
 #define STAT_OPT_j							\
 	VOPT("j", "[-j]", "Print statistics to stdout as JSON",		\
 	    "Print statistics to stdout as JSON."			\
@@ -61,7 +53,7 @@
 	)
 
 STAT_OPT_1
-STAT_OPT_f
+VSC_OPT_f
 VUT_OPT_h
 STAT_OPT_j
 STAT_OPT_l
diff --git a/include/vapi/vapi_options.h b/include/vapi/vapi_options.h
index 419a71302..823c3b673 100644
--- a/include/vapi/vapi_options.h
+++ b/include/vapi/vapi_options.h
@@ -124,3 +124,14 @@
 	    " taglist and regular expression. Applies to any tag if"	\
 	    " taglist is absent. Multiple -X options may be given.\n"	\
 	)
+
+/* VSC options */
+
+#define VSC_OPT_f							\
+	VOPT("f:", "[-f <glob>]", "Field inclusion glob",		\
+	    "Field inclusion glob."					\
+	    " Use backslash to escape characters. If the argument"	\
+	    " starts with '^' it is used as an exclusive glob."		\
+	    " Multiple -f arguments may be given. Inclusive globs"	\
+	    " are accumulative and are run before exclusive ones."	\
+	)

From dridi.boukelmoune at gmail.com  Mon Aug 31 16:00:08 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 16:00:08 +0000 (UTC)
Subject: [master] 6351aac33 varnishstat: Polish
Message-ID: <20200831160008.2648453E2@lists.varnish-cache.org>


commit 6351aac3370ec48cf451d9d8bcdb80a4e61b3e88
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 25 16:50:55 2020 +0200

    varnishstat: Polish

diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index f816184a8..4215783c7 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -352,7 +352,7 @@ main(int argc, char * const *argv)
 	else if (f_list)
 		list_fields(vd, vsc);
 	else
-		assert(0);
+		WRONG("undefined varnishstat mode");
 
 	exit(0);
 }

From dridi.boukelmoune at gmail.com  Mon Aug 31 16:00:08 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 16:00:08 +0000 (UTC)
Subject: [master] b78cb58ab vsc: Fold the include and exclude lists together
Message-ID: <20200831160008.4CC0953E8@lists.varnish-cache.org>


commit b78cb58ab820e30696f5591279e9081a5b3e8cf1
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 25 17:07:44 2020 +0200

    vsc: Fold the include and exclude lists together
    
    Instead we now have a single list that works on a first-match basis,
    slightly changing the previous outcome where all exclusions were
    checked before inclusions as illustrated by the u5 VTC.
    
    To add insult to the injury the precedence of exclusive patterns over
    inclusive patterns was documented backwards.

diff --git a/bin/varnishtest/tests/u00005.vtc b/bin/varnishtest/tests/u00005.vtc
index 415307c45..bc6c79207 100644
--- a/bin/varnishtest/tests/u00005.vtc
+++ b/bin/varnishtest/tests/u00005.vtc
@@ -19,8 +19,8 @@ shell "grep -q vbe ${p1_out}"
 shell "grep -q LCK.mempool.creat ${p1_out}"
 shell -err "grep -q LCK.vbe.destroy ${p1_out}"
 
-process p2 {varnishstat -1 -n ${v1_name} -f *vbe* \
-	-f ^*vbe.destroy -f LCK.mempool.c* | tr '[1-9]' '0'} -run
+process p2 {varnishstat -1 -n ${v1_name} -f ^*vbe.destroy -f *vbe* \
+	-f LCK.mempool.c* | tr '[1-9]' '0'} -run
 
 shell "cmp -s ${p1_out} ${p2_out}"
 
diff --git a/include/vapi/vapi_options.h b/include/vapi/vapi_options.h
index 823c3b673..8febf761b 100644
--- a/include/vapi/vapi_options.h
+++ b/include/vapi/vapi_options.h
@@ -132,6 +132,6 @@
 	    "Field inclusion glob."					\
 	    " Use backslash to escape characters. If the argument"	\
 	    " starts with '^' it is used as an exclusive glob."		\
-	    " Multiple -f arguments may be given. Inclusive globs"	\
-	    " are accumulative and are run before exclusive ones."	\
+	    " Multiple -f arguments may be given. Filtering globs"	\
+	    " are run in order on a first-match basis."			\
 	)
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index 6846e19f3..d0ee9afb2 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -53,12 +53,23 @@
 #include "vapi/vsc.h"
 #include "vapi/vsm.h"
 
+struct vsc_sf_mode {
+	const char		*name;
+	unsigned		include;
+	unsigned		fail;
+};
+
+static struct vsc_sf_mode VSC_SF_INCLUDE[1] = {{"include", 1, 1}};
+static struct vsc_sf_mode VSC_SF_EXCLUDE[1] = {{"exclude", 0, 0}};
+
 struct vsc_sf {
-	unsigned		magic;
-#define VSC_SF_MAGIC		0x558478dd
-	VTAILQ_ENTRY(vsc_sf)	list;
-	char			*pattern;
+	unsigned			magic;
+#define VSC_SF_MAGIC			0x558478dd
+	VTAILQ_ENTRY(vsc_sf)		list;
+	char				*pattern;
+	const struct vsc_sf_mode	*mode;
 };
+
 VTAILQ_HEAD(vsc_sf_head, vsc_sf);
 
 struct vsc_pt {
@@ -85,8 +96,7 @@ struct vsc {
 	unsigned		magic;
 #define VSC_MAGIC		0x3373554a
 
-	struct vsc_sf_head	sf_list_include;
-	struct vsc_sf_head	sf_list_exclude;
+	struct vsc_sf_head	sf_list;
 	VTAILQ_HEAD(,vsc_seg)	segs;
 
 	VSC_new_f		*fnew;
@@ -120,8 +130,7 @@ VSC_New(void)
 	ALLOC_OBJ(vsc, VSC_MAGIC);
 	if (vsc == NULL)
 		return (vsc);
-	VTAILQ_INIT(&vsc->sf_list_include);
-	VTAILQ_INIT(&vsc->sf_list_exclude);
+	VTAILQ_INIT(&vsc->sf_list);
 	VTAILQ_INIT(&vsc->segs);
 	return (vsc);
 }
@@ -132,7 +141,6 @@ static int
 vsc_f_arg(struct vsc *vsc, const char *opt)
 {
 	struct vsc_sf *sf;
-	unsigned exclude = 0;
 
 	AN(opt);
 
@@ -140,18 +148,14 @@ vsc_f_arg(struct vsc *vsc, const char *opt)
 	AN(sf);
 
 	if (opt[0] == '^') {
-		exclude = 1;
+		sf->mode = VSC_SF_EXCLUDE;
 		opt++;
+	} else {
+		sf->mode = VSC_SF_INCLUDE;
 	}
 
-	sf->pattern = strdup(opt);
-	AN(sf->pattern);
-
-	if (exclude)
-		VTAILQ_INSERT_TAIL(&vsc->sf_list_exclude, sf, list);
-	else
-		VTAILQ_INSERT_TAIL(&vsc->sf_list_include, sf, list);
-
+	REPLACE(sf->pattern, opt);
+	VTAILQ_INSERT_TAIL(&vsc->sf_list, sf, list);
 	return (1);
 }
 
@@ -177,17 +181,15 @@ static int
 vsc_filter(const struct vsc *vsc, const char *nm)
 {
 	struct vsc_sf *sf;
+	unsigned res = 0;
 
 	CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);
-	VTAILQ_FOREACH(sf, &vsc->sf_list_exclude, list)
+	VTAILQ_FOREACH(sf, &vsc->sf_list, list) {
 		if (!fnmatch(sf->pattern, nm, 0))
-			return (1);
-	if (VTAILQ_EMPTY(&vsc->sf_list_include))
-		return (0);
-	VTAILQ_FOREACH(sf, &vsc->sf_list_include, list)
-		if (!fnmatch(sf->pattern, nm, 0))
-			return (0);
-	return (1);
+			return (!sf->mode->include);
+		res |= sf->mode->fail;
+	}
+	return (res);
 }
 
 /*--------------------------------------------------------------------
@@ -516,29 +518,21 @@ VSC_ChangeLevel(const struct VSC_level_desc *old, int chg)
 
 /*--------------------------------------------------------------------*/
 
-static void
-vsc_delete_sf_list(struct vsc_sf_head *head)
-{
-	struct vsc_sf *sf;
-
-	while (!VTAILQ_EMPTY(head)) {
-		sf = VTAILQ_FIRST(head);
-		CHECK_OBJ_NOTNULL(sf, VSC_SF_MAGIC);
-		VTAILQ_REMOVE(head, sf, list);
-		free(sf->pattern);
-		FREE_OBJ(sf);
-	}
-}
-
 void
 VSC_Destroy(struct vsc **vscp, struct vsm *vsm)
 {
 	struct vsc *vsc;
+	struct vsc_sf *sf, *sf2;
 	struct vsc_seg *sp, *sp2;
 
 	TAKE_OBJ_NOTNULL(vsc, vscp, VSC_MAGIC);
-	vsc_delete_sf_list(&vsc->sf_list_include);
-	vsc_delete_sf_list(&vsc->sf_list_exclude);
+
+	VTAILQ_FOREACH_SAFE(sf, &vsc->sf_list, list, sf2) {
+		CHECK_OBJ_NOTNULL(sf, VSC_SF_MAGIC);
+		VTAILQ_REMOVE(&vsc->sf_list, sf, list);
+		free(sf->pattern);
+		FREE_OBJ(sf);
+	}
 	VTAILQ_FOREACH_SAFE(sp, &vsc->segs, list, sp2) {
 		VTAILQ_REMOVE(&vsc->segs, sp, list);
 		vsc_expose(vsc, sp, 1);

From dridi.boukelmoune at gmail.com  Mon Aug 31 16:00:08 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 16:00:08 +0000 (UTC)
Subject: [master] 7968d1d2a vsc: Add new 'I' and 'X' arguments
Message-ID: <20200831160008.74DDF53EE@lists.varnish-cache.org>


commit 7968d1d2a3e4f556dfcafc8b78228e66646394a3
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 25 18:01:21 2020 +0200

    vsc: Add new 'I' and 'X' arguments
    
    And simplify the 'f' argument handling.

diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
index b2ba993e9..b4dda6c3c 100644
--- a/include/vapi/vsc.h
+++ b/include/vapi/vsc.h
@@ -114,7 +114,9 @@ void VSC_Destroy(struct vsc **, struct vsm *);
 int VSC_Arg(struct vsc *, char arg, const char *opt);
 	/*
 	 * Handle standard stat-presenter arguments
-	 *	'f' - filter
+	 *	'I' - field inclusion glob
+	 *	'X' - field exclusion glob
+	 *	'f' - legacy field filter glob
 	 *
 	 * Return:
 	 *	-1 error, VSM_Error() returns diagnostic string
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index d0ee9afb2..70ad3d507 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -138,27 +138,34 @@ VSC_New(void)
 /*--------------------------------------------------------------------*/
 
 static int
-vsc_f_arg(struct vsc *vsc, const char *opt)
+vsc_sf_arg(struct vsc *vsc, const char *glob, const struct vsc_sf_mode *mode)
 {
 	struct vsc_sf *sf;
 
-	AN(opt);
+	CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);
+	AN(glob);
+	AN(mode);
 
 	ALLOC_OBJ(sf, VSC_SF_MAGIC);
 	AN(sf);
-
-	if (opt[0] == '^') {
-		sf->mode = VSC_SF_EXCLUDE;
-		opt++;
-	} else {
-		sf->mode = VSC_SF_INCLUDE;
-	}
-
-	REPLACE(sf->pattern, opt);
+	REPLACE(sf->pattern, glob);
+	sf->mode = mode;
 	VTAILQ_INSERT_TAIL(&vsc->sf_list, sf, list);
 	return (1);
 }
 
+static int
+vsc_f_arg(struct vsc *vsc, const char *opt)
+{
+
+	CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);
+	AN(opt);
+
+	if (opt[0] == '^')
+		return (vsc_sf_arg(vsc, opt + 1, VSC_SF_EXCLUDE));
+	return (vsc_sf_arg(vsc, opt, VSC_SF_INCLUDE));
+}
+
 /*--------------------------------------------------------------------*/
 
 int
@@ -168,6 +175,8 @@ VSC_Arg(struct vsc *vsc, char arg, const char *opt)
 	CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);
 
 	switch (arg) {
+	case 'I': return (vsc_sf_arg(vsc, opt, VSC_SF_INCLUDE));
+	case 'X': return (vsc_sf_arg(vsc, opt, VSC_SF_EXCLUDE));
 	case 'f': return (vsc_f_arg(vsc, opt));
 	default: return (0);
 	}

From dridi.boukelmoune at gmail.com  Mon Aug 31 16:00:08 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 16:00:08 +0000 (UTC)
Subject: [master] 40cd5acc5 varnishstat: New -I and -X options
Message-ID: <20200831160008.9F56E53F4@lists.varnish-cache.org>


commit 40cd5acc5d5051cb9c01a537a363a83be46f91f5
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 25 18:14:38 2020 +0200

    varnishstat: New -I and -X options
    
    They map directly to VSC 'I' and 'X' arguments.

diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index 4215783c7..0d308055a 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -309,6 +309,8 @@ main(int argc, char * const *argv)
 		case 'j':
 			json = 1;
 			break;
+		case 'I':
+		case 'X':
 		case 'f':
 			AN(VSC_Arg(vsc, opt, optarg));
 			has_f = 1;
diff --git a/bin/varnishstat/varnishstat_options.h b/bin/varnishstat/varnishstat_options.h
index 35bdb5c11..b98ba11c0 100644
--- a/bin/varnishstat/varnishstat_options.h
+++ b/bin/varnishstat/varnishstat_options.h
@@ -55,9 +55,11 @@
 STAT_OPT_1
 VSC_OPT_f
 VUT_OPT_h
+VSC_OPT_I
 STAT_OPT_j
 STAT_OPT_l
 VUT_OPT_n
 VUT_OPT_t
 VUT_GLOBAL_OPT_V
+VSC_OPT_X
 STAT_OPT_x
diff --git a/bin/varnishtest/tests/u00005.vtc b/bin/varnishtest/tests/u00005.vtc
index bc6c79207..0ffe02465 100644
--- a/bin/varnishtest/tests/u00005.vtc
+++ b/bin/varnishtest/tests/u00005.vtc
@@ -22,7 +22,11 @@ shell -err "grep -q LCK.vbe.destroy ${p1_out}"
 process p2 {varnishstat -1 -n ${v1_name} -f ^*vbe.destroy -f *vbe* \
 	-f LCK.mempool.c* | tr '[1-9]' '0'} -run
 
+process p3 {varnishstat -1 -n ${v1_name} -X *vbe.destroy -I *vbe* \
+	-f LCK.mempool.c* | tr '[1-9]' '0'} -run
+
 shell "cmp -s ${p1_out} ${p2_out}"
+shell "cmp -s ${p1_out} ${p3_out}"
 
 shell -expect "MGT.uptime" \
 	"varnishstat -1 -n ${v1_name} -f ^MAIN*"
diff --git a/include/vapi/vapi_options.h b/include/vapi/vapi_options.h
index 8febf761b..ad5fe86a9 100644
--- a/include/vapi/vapi_options.h
+++ b/include/vapi/vapi_options.h
@@ -128,10 +128,32 @@
 /* VSC options */
 
 #define VSC_OPT_f							\
-	VOPT("f:", "[-f <glob>]", "Field inclusion glob",		\
-	    "Field inclusion glob."					\
+	VOPT("f:", "[-f <glob>]", "Legacy field filtering glob",	\
+	    "Legacy field filtering glob."				\
 	    " Use backslash to escape characters. If the argument"	\
 	    " starts with '^' it is used as an exclusive glob."		\
-	    " Multiple -f arguments may be given. Filtering globs"	\
-	    " are run in order on a first-match basis."			\
+	    " Multiple -f arguments may be given. Legacy filtering"	\
+	    " globs are run along with inclusion globs (-I arguments)"	\
+	    " and exclusion globs (-X arguments) in order on a"		\
+	    " first-match basis."					\
+	)
+
+#define VSC_OPT_I							\
+	VOPT("I:", "[-I <glob>]", "Field inclusion glob",		\
+	    "Field inclusion glob."					\
+	    " Use backslash to escape characters. Multiple -I "		\
+	    " arguments may be given. Exclusion globs are run in"	\
+	    " order along with exclusion globs (-X arguments) and"	\
+	    " legacy filtering globs (-f arguments) on a first-match"	\
+	    " basis."							\
+	)
+
+#define VSC_OPT_X							\
+	VOPT("X:", "[-X <glob>]", "Field exclusion glob",		\
+	    "Field exclusion glob."					\
+	    " Use backslash to escape characters. Multiple -X "		\
+	    " arguments may be given. Exclusion globs are run in"	\
+	    " order along with inclusion globs (-I arguments) and"	\
+	    " legacy filtering globs (-f arguments) on a first-match"	\
+	    " basis."							\
 	)

From dridi.boukelmoune at gmail.com  Mon Aug 31 16:00:08 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 16:00:08 +0000 (UTC)
Subject: [master] 5c2f7acdb vsc: Add an 'R' argument for required fields
Message-ID: <20200831160008.B9D6553F7@lists.varnish-cache.org>


commit 5c2f7acdbe62559479bbdf74a1e715727cbe47f0
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 25 18:53:06 2020 +0200

    vsc: Add an 'R' argument for required fields
    
    Required fields work like inclusion filters, except that not matching
    them doesn't have any consequence if no filter matched. They also take
    precedence over regular filters.
    
    Fixes #3394

diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index 0d308055a..75c88311f 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -338,10 +338,10 @@ main(int argc, char * const *argv)
 
 	if (curses) {
 		if (has_f) {
-			AN(VSC_Arg(vsc, 'f', "MGT.uptime"));
-			AN(VSC_Arg(vsc, 'f', "MAIN.uptime"));
-			AN(VSC_Arg(vsc, 'f', "MAIN.cache_hit"));
-			AN(VSC_Arg(vsc, 'f', "MAIN.cache_miss"));
+			AN(VSC_Arg(vsc, 'R', "MGT.uptime"));
+			AN(VSC_Arg(vsc, 'R', "MAIN.uptime"));
+			AN(VSC_Arg(vsc, 'R', "MAIN.cache_hit"));
+			AN(VSC_Arg(vsc, 'R', "MAIN.cache_miss"));
 		}
 		do_curses(vd, vsc);
 	}
diff --git a/bin/varnishtest/tests/r03394.vtc b/bin/varnishtest/tests/r03394.vtc
new file mode 100644
index 000000000..750d6e414
--- /dev/null
+++ b/bin/varnishtest/tests/r03394.vtc
@@ -0,0 +1,9 @@
+varnishtest "varnishstat curses field exclusion"
+
+server s1 -start
+
+varnish v1 -vcl+backend "" -start
+
+process p1 -dump {varnishstat -n ${v1_name} -f '^VSB.*'} -start
+process p1 -expect-text 0 0 "MAIN.pools"
+process p1 -screen_dump -write q -wait
diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
index b4dda6c3c..ec6c2b92d 100644
--- a/include/vapi/vsc.h
+++ b/include/vapi/vsc.h
@@ -116,6 +116,7 @@ int VSC_Arg(struct vsc *, char arg, const char *opt);
 	 * Handle standard stat-presenter arguments
 	 *	'I' - field inclusion glob
 	 *	'X' - field exclusion glob
+	 *	'R' - required field glob
 	 *	'f' - legacy field filter glob
 	 *
 	 * Return:
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index 70ad3d507..f10dfc387 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -57,10 +57,12 @@ struct vsc_sf_mode {
 	const char		*name;
 	unsigned		include;
 	unsigned		fail;
+	unsigned		append;
 };
 
-static struct vsc_sf_mode VSC_SF_INCLUDE[1] = {{"include", 1, 1}};
-static struct vsc_sf_mode VSC_SF_EXCLUDE[1] = {{"exclude", 0, 0}};
+static struct vsc_sf_mode VSC_SF_INCLUDE[1] = {{"include", 1, 1, 1}};
+static struct vsc_sf_mode VSC_SF_EXCLUDE[1] = {{"exclude", 0, 0, 1}};
+static struct vsc_sf_mode VSC_SF_REQUIRE[1] = {{"require", 1, 0, 0}};
 
 struct vsc_sf {
 	unsigned			magic;
@@ -150,7 +152,10 @@ vsc_sf_arg(struct vsc *vsc, const char *glob, const struct vsc_sf_mode *mode)
 	AN(sf);
 	REPLACE(sf->pattern, glob);
 	sf->mode = mode;
-	VTAILQ_INSERT_TAIL(&vsc->sf_list, sf, list);
+	if (mode->append)
+		VTAILQ_INSERT_TAIL(&vsc->sf_list, sf, list);
+	else
+		VTAILQ_INSERT_HEAD(&vsc->sf_list, sf, list);
 	return (1);
 }
 
@@ -177,6 +182,7 @@ VSC_Arg(struct vsc *vsc, char arg, const char *opt)
 	switch (arg) {
 	case 'I': return (vsc_sf_arg(vsc, opt, VSC_SF_INCLUDE));
 	case 'X': return (vsc_sf_arg(vsc, opt, VSC_SF_EXCLUDE));
+	case 'R': return (vsc_sf_arg(vsc, opt, VSC_SF_REQUIRE));
 	case 'f': return (vsc_f_arg(vsc, opt));
 	default: return (0);
 	}

From dridi at varni.sh  Mon Aug 31 16:22:06 2020
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 16:22:06 +0000
Subject: [master] f658f830d Unify some of the code from client/server
 using a "session" concept, which will become more and more important.
In-Reply-To: <20200831142507.97712AFC19@lists.varnish-cache.org>
References: <20200831142507.97712AFC19@lists.varnish-cache.org>
Message-ID: <CABoVN9BEcb_t2XObK6cOJbXLgTcxtBe6+QF2G6RsTULtQLXmuA@mail.gmail.com>

On Mon, Aug 31, 2020 at 2:25 PM Poul-Henning Kamp <phk at freebsd.org> wrote:
>
>
> commit f658f830df003e7258cba883c3ccd3a2f2cbfb95
> Author: Poul-Henning Kamp <phk at FreeBSD.org>
> Date:   Mon Aug 31 14:23:06 2020 +0000
>
>     Unify some of the code from client/server using a "session"
>     concept, which will become more and more important.
>
> diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am
> index 34f11f77e..61c2be73d 100644
> --- a/bin/varnishtest/Makefile.am
> +++ b/bin/varnishtest/Makefile.am
> @@ -47,6 +47,7 @@ varnishtest_SOURCES = \
>                 vtc_process.c \
>                 vtc_proxy.c \
>                 vtc_server.c \
> +               vtc_sess.c \

vtc_sess.c appears to be missing from this patch!

>                 vtc_subr.c \
>                 vtc_syslog.c \
>                 vtc_varnish.c
> diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
> index cd21edb75..23a3a6472 100644
> --- a/bin/varnishtest/vtc.h
> +++ b/bin/varnishtest/vtc.h
> @@ -88,6 +88,27 @@ extern int ign_unknown_macro;
>  void init_server(void);
>  void init_syslog(void);
>
> +/* Sessions */
> +struct vtc_sess *Sess_New(struct vtclog *vl, const char *name);
> +void Sess_Destroy(struct vtc_sess **spp);
> +int Sess_GetOpt(struct vtc_sess *, char * const **);
> +int sess_process(struct vtclog *vl, const struct vtc_sess *,
> +    const char *spec, int sock, int *sfd, const char *addr);
> +
> +typedef int sess_conn_f(void *priv, struct vtclog *);
> +typedef void sess_disc_f(void *priv, struct vtclog *, int *fd);
> +pthread_t
> +Sess_Start_Thread(
> +    void *priv,
> +    struct vtc_sess *vsp,
> +    sess_conn_f *conn,
> +    sess_disc_f *disc,
> +    const char *listen_addr,
> +    int *asocket,
> +    const char *spec
> +);
> +
> +
>  int http_process(struct vtclog *vl, const char *spec, int sock, int *sfd,
>      const char *addr, int rcvbuf);
>
> diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c
> index 6ccfbb8d3..41cc3a581 100644
> --- a/bin/varnishtest/vtc_client.c
> +++ b/bin/varnishtest/vtc_client.c
> @@ -52,19 +52,16 @@ struct client {
>         char                    *name;
>         struct vtclog           *vl;
>         VTAILQ_ENTRY(client)    list;
> +       struct vtc_sess         *vsp;
>
>         char                    *spec;
>
>         char                    connect[256];
> -       const char              *addr;
> -       int                     rcvbuf;
> +       char                    *addr;
>
>         char                    *proxy_spec;
>         int                     proxy_version;
>
> -       int                     repeat;
> -       unsigned                keepalive;
> -
>         unsigned                running;
>         pthread_t               tp;
>  };
> @@ -207,54 +204,21 @@ client_connect(struct vtclog *vl, struct client *c)
>   * Client thread
>   */
>
> -static void *
> -client_thread(void *priv)
> +static int
> +client_conn(void *priv, struct vtclog *vl)
>  {
>         struct client *c;
> -       struct vtclog *vl;
> -       int fd;
> -       int i;
> -       struct vsb *vsb;
>
>         CAST_OBJ_NOTNULL(c, priv, CLIENT_MAGIC);
> -       AN(*c->connect);
> -
> -       vl = vtc_logopen(c->name);
> -       pthread_cleanup_push(vtc_logclose, vl);
> +       return (client_connect(vl, c));
> +}
>
> -       vsb = macro_expand(vl, c->connect);
> -       AN(vsb);
> -#if !defined(__sun)
> -       pthread_cleanup_push((void (*)(void *))VSB_destroy, &vsb);
> -#endif
> -       c->addr = VSB_data(vsb);
> -
> -       if (c->repeat == 0)
> -               c->repeat = 1;
> -       if (c->repeat != 1)
> -               vtc_log(vl, 2, "Started (%u iterations%s)", c->repeat,
> -                       c->keepalive ? " using keepalive" : "");
> -       for (i = 0; i < c->repeat; i++) {
> -               fd = client_connect(vl, c);
> -
> -               if (! c->keepalive)
> -                       fd = http_process(vl, c->spec, fd, NULL, c->addr,
> -                           c->rcvbuf);
> -               else
> -                       while (fd >= 0 && i++ < c->repeat)
> -                               fd = http_process(vl, c->spec, fd, NULL,
> -                                   c->addr, c->rcvbuf);
> -               vtc_log(vl, 3, "closing fd %d", fd);
> -               VTCP_close(&fd);
> -       }
> -       vtc_log(vl, 2, "Ending");
> -#if !defined(__sun)
> -       pthread_cleanup_pop(0);
> -#endif
> -       pthread_cleanup_pop(0);
> -       VSB_destroy(&vsb);
> -       vtc_logclose(vl);
> -       return (NULL);
> +static void
> +client_disc(void *priv, struct vtclog *vl, int *fdp)
> +{
> +       (void)priv;
> +       vtc_log(vl, 3, "closing fd %d", *fdp);
> +       VTCP_close(fdp);
>  }
>
>  /**********************************************************************
> @@ -271,6 +235,8 @@ client_new(const char *name)
>         REPLACE(c->name, name);
>         c->vl = vtc_logopen(name);
>         AN(c->vl);
> +       c->vsp = Sess_New(c->vl, name);
> +       AN(c->vsp);
>
>         bprintf(c->connect, "%s", "${v1_sock}");
>         VTAILQ_INSERT_TAIL(&clients, c, list);
> @@ -286,9 +252,11 @@ client_delete(struct client *c)
>  {
>
>         CHECK_OBJ_NOTNULL(c, CLIENT_MAGIC);
> +       Sess_Destroy(&c->vsp);
>         vtc_logclose(c->vl);
>         free(c->spec);
>         free(c->name);
> +       free(c->addr);
>         free(c->proxy_spec);
>         /* XXX: MEMLEAK (?)*/
>         FREE_OBJ(c);
> @@ -301,11 +269,24 @@ client_delete(struct client *c)
>  static void
>  client_start(struct client *c)
>  {
> +       struct vsb *vsb;
>
>         CHECK_OBJ_NOTNULL(c, CLIENT_MAGIC);
>         vtc_log(c->vl, 2, "Starting client");
> -       AZ(pthread_create(&c->tp, NULL, client_thread, c));
>         c->running = 1;
> +       vsb = macro_expand(c->vl, c->connect);
> +       AN(vsb);
> +       REPLACE(c->addr, VSB_data(vsb));
> +       VSB_destroy(&vsb);
> +       c->tp = Sess_Start_Thread(
> +           c,
> +           c->vsp,
> +           client_conn,
> +           client_disc,
> +           c->addr,
> +           NULL,
> +           c->spec
> +       );
>  }
>
>  /**********************************************************************
> @@ -386,6 +367,10 @@ cmd_client(CMD_ARGS)
>                 if (c->running)
>                         client_wait(c);
>
> +               AZ(c->running);
> +               if (Sess_GetOpt(c->vsp, &av))
> +                       continue;
> +
>                 if (!strcmp(*av, "-connect")) {
>                         bprintf(c->connect, "%s", av[1]);
>                         av++;
> @@ -403,20 +388,6 @@ cmd_client(CMD_ARGS)
>                         av++;
>                         continue;
>                 }
> -               if (!strcmp(*av, "-repeat")) {
> -                       c->repeat = atoi(av[1]);
> -                       av++;
> -                       continue;
> -               }
> -               if (!strcmp(*av, "-keepalive")) {
> -                       c->keepalive = 1;
> -                       continue;
> -               }
> -               if (!strcmp(*av, "-rcvbuf")) {
> -                       c->rcvbuf = atoi(av[1]);
> -                       av++;
> -                       continue;
> -               }
>                 if (!strcmp(*av, "-start")) {
>                         client_start(c);
>                         continue;
> diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
> index 9ed58d923..4043c27da 100644
> --- a/bin/varnishtest/vtc_server.c
> +++ b/bin/varnishtest/vtc_server.c
> @@ -50,16 +50,14 @@ struct server {
>         char                    *name;
>         struct vtclog           *vl;
>         VTAILQ_ENTRY(server)    list;
> +       struct vtc_sess         *vsp;
>         char                    run;
>
> -       int                     repeat;
> -       unsigned                keepalive;
>         char                    *spec;
>
>         int                     depth;
>         int                     sock;
>         int                     fd;
> -       int                     rcvbuf;
>         char                    listen[256];
>         char                    aaddr[32];
>         char                    aport[32];
> @@ -87,9 +85,10 @@ server_new(const char *name, struct vtclog *vl)
>         REPLACE(s->name, name);
>         s->vl = vtc_logopen(s->name);
>         AN(s->vl);
> +       s->vsp = Sess_New(s->vl, name);
> +       AN(s->vsp);
>
>         bprintf(s->listen, "%s", "127.0.0.1 0");
> -       s->repeat = 1;
>         s->depth = 10;
>         s->sock = -1;
>         s->fd = -1;
> @@ -108,6 +107,7 @@ server_delete(struct server *s)
>  {
>
>         CHECK_OBJ_NOTNULL(s, SERVER_MAGIC);
> +       Sess_Destroy(&s->vsp);
>         macro_undef(s->vl, s->name, "addr");
>         macro_undef(s->vl, s->name, "port");
>         macro_undef(s->vl, s->name, "sock");
> @@ -219,56 +219,61 @@ server_listen(struct server *s)
>   * Server thread
>   */
>
> -static void *
> -server_thread(void *priv)
> +static int
> +server_conn(void *priv, struct vtclog *vl)
>  {
>         struct server *s;
> -       struct vtclog *vl;
> -       int i, j, fd;
>         struct sockaddr_storage addr_s;
>         struct sockaddr *addr;
> -       socklen_t l;
>         char abuf[VTCP_ADDRBUFSIZE];
>         char pbuf[VTCP_PORTBUFSIZE];
> +       socklen_t l;
> +       int fd;
>
>         CAST_OBJ_NOTNULL(s, priv, SERVER_MAGIC);
> -       assert(s->sock >= 0);
>
> -       vl = vtc_logopen(s->name);
> -       pthread_cleanup_push(vtc_logclose, vl);
> +       addr = (void*)&addr_s;
> +       l = sizeof addr_s;
> +       fd = accept(s->sock, addr, &l);
> +       if (fd < 0)
> +               vtc_fatal(vl, "Accept failed: %s", strerror(errno));
> +       if (*s->listen != '/') {
> +               VTCP_hisname(fd, abuf, sizeof abuf, pbuf, sizeof pbuf);
> +               vtc_log(vl, 3, "accepted fd %d %s %s", fd, abuf, pbuf);
> +       } else
> +               vtc_log(vl, 3, "accepted fd %d 0.0.0.0 0", fd);
> +       return (fd);
> +}
>
> -       vtc_log(vl, 2, "Started on %s (%u iterations%s)", s->listen,
> -               s->repeat, s->keepalive ? " using keepalive" : "");
> -       for (i = 0; i < s->repeat; i++) {
> -               addr = (void*)&addr_s;
> -               l = sizeof addr_s;
> -               fd = accept(s->sock, addr, &l);
> -               if (fd < 0)
> -                       vtc_fatal(vl, "Accept failed: %s", strerror(errno));
> -               if (*s->listen != '/') {
> -                       VTCP_hisname(fd, abuf, sizeof abuf, pbuf, sizeof pbuf);
> -                       vtc_log(vl, 3, "accepted fd %d %s %s", fd, abuf, pbuf);
> -               } else
> -                       vtc_log(vl, 3, "accepted fd %d 0.0.0.0 0", fd);
> -               if (! s->keepalive)
> -                       fd = http_process(vl, s->spec, fd, &s->sock, s->listen,
> -                           s->rcvbuf);
> -               else
> -                       while (fd >= 0 && i++ < s->repeat)
> -                               fd = http_process(vl, s->spec, fd,
> -                                   &s->sock, s->listen, s->rcvbuf);
> -               vtc_log(vl, 3, "shutting fd %d", fd);
> -               j = shutdown(fd, SHUT_WR);
> -               if (!VTCP_Check(j))
> -                       vtc_fatal(vl, "Shutdown failed: %s", strerror(errno));
> -               VTCP_close(&fd);
> -       }
> -       vtc_log(vl, 2, "Ending");
> -       pthread_cleanup_pop(0);
> -       vtc_logclose(vl);
> -       return (NULL);
> +static void
> +server_disc(void *priv, struct vtclog *vl, int *fdp)
> +{
> +       int j;
> +       struct server *s;
> +
> +       CAST_OBJ_NOTNULL(s, priv, SERVER_MAGIC);
> +       vtc_log(vl, 3, "shutting fd %d", *fdp);
> +       j = shutdown(*fdp, SHUT_WR);
> +       if (!VTCP_Check(j))
> +               vtc_fatal(vl, "Shutdown failed: %s", strerror(errno));
> +       VTCP_close(fdp);
>  }
>
> +static void
> +server_start_thread(struct server *s)
> +{
> +
> +       s->run = 1;
> +       s->tp = Sess_Start_Thread(
> +           s,
> +           s->vsp,
> +           server_conn,
> +           server_disc,
> +           s->listen,
> +           &s->sock,
> +           s->spec
> +       );
> +}
>
>  /**********************************************************************
>   * Start the server thread
> @@ -281,8 +286,7 @@ server_start(struct server *s)
>         vtc_log(s->vl, 2, "Starting server");
>         server_listen(s);
>         vtc_log(s->vl, 1, "Listen on %s", s->listen);
> -       s->run = 1;
> -       AZ(pthread_create(&s->tp, NULL, server_thread, s));
> +       server_start_thread(s);
>  }
>
>  /**********************************************************************
> @@ -304,7 +308,7 @@ server_dispatch_wrk(void *priv)
>         fd = s->fd;
>
>         vtc_log(vl, 3, "start with fd %d", fd);
> -       fd = http_process(vl, s->spec, fd, &s->sock, s->listen, s->rcvbuf);
> +       fd = sess_process(vl, s->vsp, s->spec, fd, &s->sock, s->listen);
>         vtc_log(vl, 3, "shutting fd %d", fd);
>         j = shutdown(fd, SHUT_WR);
>         if (!VTCP_Check(j))
> @@ -530,15 +534,10 @@ cmd_server(CMD_ARGS)
>                         server_wait(s);
>
>                 AZ(s->run);
> -               if (!strcmp(*av, "-repeat")) {
> -                       s->repeat = atoi(av[1]);
> -                       av++;
> -                       continue;
> -               }
> -               if (!strcmp(*av, "-keepalive")) {
> -                       s->keepalive = 1;
> +
> +               if (Sess_GetOpt(s->vsp, &av))
>                         continue;
> -               }
> +
>                 if (!strcmp(*av, "-listen")) {
>                         if (s->sock >= 0)
>                                 VTCP_close(&s->sock);
> @@ -546,11 +545,6 @@ cmd_server(CMD_ARGS)
>                         av++;
>                         continue;
>                 }
> -               if (!strcmp(*av, "-rcvbuf")) {
> -                       s->rcvbuf = atoi(av[1]);
> -                       av++;
> -                       continue;
> -               }
>                 if (!strcmp(*av, "-start")) {
>                         server_start(s);
>                         continue;
> _______________________________________________
> varnish-commit mailing list
> varnish-commit at varnish-cache.org
> https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit

From phk at FreeBSD.org  Mon Aug 31 16:44:06 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 31 Aug 2020 16:44:06 +0000 (UTC)
Subject: [master] 18ce2e80b I always forget the new file...
Message-ID: <20200831164406.BFA21707F@lists.varnish-cache.org>


commit 18ce2e80b2783ae5765fed7a5d6828b92e09e172
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Aug 31 16:42:42 2020 +0000

    I always forget the new file...

diff --git a/bin/varnishtest/vtc_sess.c b/bin/varnishtest/vtc_sess.c
new file mode 100644
index 000000000..7105308c3
--- /dev/null
+++ b/bin/varnishtest/vtc_sess.c
@@ -0,0 +1,162 @@
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "config.h"
+
+#include "vtc.h"
+
+struct vtc_sess {
+	unsigned		magic;
+#define VTC_SESS_MAGIC		0x932bd565
+	struct vtclog		*vl;
+	char			*name;
+	int			repeat;
+	int			keepalive;
+
+	ssize_t			rcvbuf;
+};
+
+struct thread_arg {
+	unsigned		magic;
+#define THREAD_ARG_MAGIC	0xd5dc5f1c
+	void			*priv;
+	sess_conn_f		*conn_f;
+	sess_disc_f		*disc_f;
+	const char		*listen_addr;
+	struct vtc_sess		*vsp;
+	int			*asocket;
+	const char		*spec;
+};
+
+struct vtc_sess *
+Sess_New(struct vtclog *vl, const char *name)
+{
+	struct vtc_sess *vsp;
+
+	ALLOC_OBJ(vsp, VTC_SESS_MAGIC);
+	AN(vsp);
+	vsp->vl = vl;
+	REPLACE(vsp->name, name);
+	vsp->repeat = 1;
+	return (vsp);
+}
+
+void
+Sess_Destroy(struct vtc_sess **vspp)
+{
+	struct vtc_sess *vsp;
+
+	TAKE_OBJ_NOTNULL(vsp, vspp, VTC_SESS_MAGIC);
+	REPLACE(vsp->name, NULL);
+	FREE_OBJ(vsp);
+}
+
+int
+Sess_GetOpt(struct vtc_sess *vsp, char * const **avp)
+{
+	char * const *av;
+	int rv = 0;
+
+	CHECK_OBJ_NOTNULL(vsp, VTC_SESS_MAGIC);
+	AN(avp);
+	av = *avp;
+	AN(*av);
+	if (!strcmp(*av, "-rcvbuf")) {
+		AN(av[1]);
+		vsp->rcvbuf = atoi(av[1]);
+		av += 1;
+		rv = 1;
+	} else if (!strcmp(*av, "-repeat")) {
+		AN(av[1]);
+		vsp->repeat = atoi(av[1]);
+		av += 1;
+		rv = 1;
+	} else if (!strcmp(*av, "-keepalive")) {
+		vsp->keepalive = 1;
+		rv = 1;
+	}
+	*avp = av;
+	return (rv);
+}
+
+int
+sess_process(struct vtclog *vl, const struct vtc_sess *vsp,
+    const char *spec, int sock, int *sfd, const char *addr)
+{
+	int rv;
+
+	CHECK_OBJ_NOTNULL(vsp, VTC_SESS_MAGIC);
+
+	rv = http_process(vl, spec, sock, sfd, addr, vsp->rcvbuf);
+	return (rv);
+}
+
+static void *
+sess_thread(void *priv)
+{
+	struct vtclog *vl;
+	struct vtc_sess *vsp;
+	struct thread_arg ta, *tap;
+	int i, fd;
+
+	CAST_OBJ_NOTNULL(tap, priv, THREAD_ARG_MAGIC);
+	ta = *tap;
+	FREE_OBJ(tap);
+
+	vsp = ta.vsp;
+	CHECK_OBJ_NOTNULL(vsp, VTC_SESS_MAGIC);
+	vl = vtc_logopen(vsp->name);
+	pthread_cleanup_push(vtc_logclose, vl);
+
+	assert(vsp->repeat > 0);
+	vtc_log(vl, 2, "Started on %s (%u iterations%s)", ta.listen_addr,
+		vsp->repeat, vsp->keepalive ? " using keepalive" : "");
+	for (i = 0; i < vsp->repeat; i++) {
+		fd = ta.conn_f(ta.priv, vl);
+		if (! vsp->keepalive)
+			fd = sess_process(vl, ta.vsp, ta.spec, fd, ta.asocket, ta.listen_addr);
+		else
+			while (fd >= 0 && i++ < vsp->repeat)
+				fd = sess_process(vl, ta.vsp, ta.spec, fd,
+				    ta.asocket, ta.listen_addr);
+		ta.disc_f(ta.priv, vl, &fd);
+	}
+	vtc_log(vl, 2, "Ending");
+	pthread_cleanup_pop(0);
+	vtc_logclose(vl);
+	return (NULL);
+}
+
+pthread_t
+Sess_Start_Thread(
+    void *priv,
+    struct vtc_sess *vsp,
+    sess_conn_f *conn,
+    sess_disc_f *disc,
+    const char *listen_addr,
+    int *asocket,
+    const char *spec
+)
+{
+	struct thread_arg *ta;
+	pthread_t pt;
+
+	AN(priv);
+	CHECK_OBJ_NOTNULL(vsp, VTC_SESS_MAGIC);
+	AN(conn);
+	AN(disc);
+	AN(listen_addr);
+	ALLOC_OBJ(ta, THREAD_ARG_MAGIC);
+	AN(ta);
+	ta->priv = priv;
+	ta->vsp = vsp;
+
+	ta->conn_f = conn;
+	ta->disc_f = disc;
+	ta->listen_addr = listen_addr;
+	ta->asocket = asocket;
+	ta->spec = spec;
+	AZ(pthread_create(&pt, NULL, sess_thread, ta));
+	return (pt);
+}

From nils.goroll at uplex.de  Mon Aug 31 17:34:08 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 31 Aug 2020 17:34:08 +0000 (UTC)
Subject: [master] b648aecf6 changelog tlc
Message-ID: <20200831173408.AEAFF602F0@lists.varnish-cache.org>


commit b648aecf6132955306d81175c19a7f6b47a2f4da
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Aug 31 19:32:39 2020 +0200

    changelog tlc
    
    up to d0a896766fd58fa7f7c298fb92944e108984c1b9

diff --git a/doc/changes.rst b/doc/changes.rst
index cfed269df..93e27a4c5 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -30,6 +30,54 @@ release process.
 NEXT (scheduled 2020-09-15)
 ===========================
 
+* ``varnishstat`` now has a help screen, available via the ``h`` key
+  in curses mode
+
+* The initial ``varnishstat`` verbosity has been changed to ensure any
+  fields specified by the ``-f`` argument are visible (2990_)
+
+* Fixed handling of out-of-workspace conditions after
+  ``vcl_backend_response`` and ``vcl_deliver`` during filter
+  initialization (3253_, 3241_)
+
+* ``PRIV_TOP`` is now thread-safe to support parallel ESI
+  implementations
+
+* ``varnishstat`` JSON format (``-j`` option) has been changed:
+
+  * on the top level, a ``version`` identifier has been introduced,
+    which will be used to mark breaking changes to the JSON
+    formatting. It will not be used to mark changes to the counters
+    themselves.
+
+    The new ``version`` is ``1``.
+
+  * All counters have been moved down one level to the ``counters``
+    object.
+
+* ``VSA_BuildFAP()`` has been added as a convenience function to
+  build a ``struct suckaddr``
+
+* Depending on the setting of the new ``vcc_acl_pedantic`` parameter,
+  VCC now either emits or warning or fails if network numbers used in
+  ACLs do not have an all-zero host part.
+
+  For ``vcc_acl_pedantic`` off, the host part is fixed to all-zero and
+  that fact logged with the ``ACL`` VSL tag.
+
+* Fixed error handling during object creation after
+  ``vcl_backend_response`` (3273_)
+
+* ``obj.can_esi`` has been added to identify if the response can be
+  ESI processed (See 3002_)
+
+* ``resp.filters`` now contains a correct value when the
+  auto-determined filter list is read (3002_)
+
+* It is now a VCL (runtime) error to write to ``resp.do`` and
+  ``beresp.do_`` fields which determine the filter list after setting
+  ``resp.filters`` and ``beresp.filters``, respectively
+
 * The Varnish Jail (least privileges) code for Solaris has been
   largely rewritten. It now reduces privileges even further and thus
   should improve the security of Varnish on Solaris even more.
@@ -41,6 +89,13 @@ NEXT (scheduled 2020-09-15)
 * The shard director and shard director parameter objects should now
   work in ``vcl_pipe {}`` like in ``vcl_backend_* {}`` subs.
 
+.. _2990: https://github.com/varnishcache/varnish-cache/issues/2990
+.. _3253
+.. _3241
+.. _3273
+.. _3002
+
+
 ================================
 Varnish Cache 6.4.0 (2020-03-16)
 ================================

From dridi.boukelmoune at gmail.com  Mon Aug 31 18:41:08 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 18:41:08 +0000 (UTC)
Subject: [master] 4fea42575 ws: Comprehensive workspace magic cheese
Message-ID: <20200831184108.D12DF61DFB@lists.varnish-cache.org>


commit 4fea425753f5857b63e02fda4ed8eed90ccdbe27
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Apr 29 11:35:51 2020 +0200

    ws: Comprehensive workspace magic cheese

diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index 874181436..f3847e5c6 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -107,7 +107,7 @@ WS_Init(struct ws *ws, const char *id, void *space, unsigned len)
 	ws->e = ws->s + len;
 	*ws->e = 0x15;
 	ws->f = ws->s;
-	assert(id[0] & 0x20);
+	assert(id[0] & 0x20);		// cheesy islower()
 	bstrcpy(ws->id, id);
 	WS_Assert(ws);
 }
@@ -368,7 +368,7 @@ WS_Overflowed(const struct ws *ws)
 	CHECK_OBJ_NOTNULL(ws, WS_MAGIC);
 	AN(ws->id[0]);
 
-	if (ws->id[0] & 0x20)
+	if (ws->id[0] & 0x20)		// cheesy islower()
 		return (0);
 	return (1);
 }

From dridi.boukelmoune at gmail.com  Mon Aug 31 18:41:08 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 18:41:08 +0000 (UTC)
Subject: [master] 6e4099e81 ws: Less magic for the workspace end marker
Message-ID: <20200831184108.E83D561DFE@lists.varnish-cache.org>


commit 6e4099e81fa91d0c6a2fb8b63594b75351f97e69
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Apr 29 11:39:37 2020 +0200

    ws: Less magic for the workspace end marker

diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index f3847e5c6..a1e511d30 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -36,6 +36,8 @@
 
 #include <stdio.h>
 
+#define WS_REDZONE_END		'\x15'
+
 static const void * const snap_overflowed = &snap_overflowed;
 
 void
@@ -60,7 +62,7 @@ WS_Assert(const struct ws *ws)
 		assert(ws->r <= ws->e);
 		assert(PAOK(ws->r));
 	}
-	assert(*ws->e == 0x15);
+	assert(*ws->e == WS_REDZONE_END);
 }
 
 int
@@ -105,7 +107,7 @@ WS_Init(struct ws *ws, const char *id, void *space, unsigned len)
 	assert(PAOK(space));
 	len = PRNDDN(len - 1);
 	ws->e = ws->s + len;
-	*ws->e = 0x15;
+	*ws->e = WS_REDZONE_END;
 	ws->f = ws->s;
 	assert(id[0] & 0x20);		// cheesy islower()
 	bstrcpy(ws->id, id);

From dridi.boukelmoune at gmail.com  Mon Aug 31 18:41:09 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 18:41:09 +0000 (UTC)
Subject: [master] 789ccc4fe ws: Retire WS_Reserve()
Message-ID: <20200831184109.2142661E01@lists.varnish-cache.org>


commit 789ccc4fe10490fd39b5d09a5c1c6c199bf7266d
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Apr 29 18:29:18 2020 +0200

    ws: Retire WS_Reserve()

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index c8ad67d1c..c09b5cef6 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -778,8 +778,6 @@ void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func,
 
 void WS_Init(struct ws *ws, const char *id, void *space, unsigned len);
 
-/* WS_Reserve(): Use WS_ReserveSize() or WS_ReserveAll() */
-unsigned WS_Reserve(struct ws *ws, unsigned bytes) v_deprecated_;
 unsigned WS_ReserveSize(struct ws *, unsigned);
 unsigned WS_ReserveAll(struct ws *);
 unsigned WS_ReserveLumps(struct ws *ws, size_t sz);
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index a1e511d30..680983d91 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -307,30 +307,6 @@ WS_ReserveSize(struct ws *ws, unsigned bytes)
 	return (pdiff(ws->f, ws->r));
 }
 
-/* REL_20200915 remove */
-unsigned
-WS_Reserve(struct ws *ws, unsigned bytes)
-{
-	unsigned b2;
-
-	WS_Assert(ws);
-	assert(ws->r == NULL);
-
-	b2 = PRNDDN(ws->e - ws->f);
-	if (bytes != 0 && bytes < b2)
-		b2 = PRNDUP(bytes);
-
-	if (ws->f + b2 > ws->e) {
-		WS_MarkOverflow(ws);
-		return (0);
-	}
-	ws->r = ws->f + b2;
-	DSL(DBG_WORKSPACE, 0, "WS_Reserve(%p, %u/%u) = %u",
-	    ws, b2, bytes, pdiff(ws->f, ws->r));
-	WS_Assert(ws);
-	return (pdiff(ws->f, ws->r));
-}
-
 unsigned
 WS_ReserveLumps(struct ws *ws, size_t sz)
 {
diff --git a/include/vrt.h b/include/vrt.h
index 365f59b1c..8a7e76e16 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -55,6 +55,7 @@
  * NEXT (2020-09-15)
  *	Added VRT_DirectorResolve()
  *	Added VCL_STRING VRT_BLOB_string(VRT_CTX, VCL_BLOB)
+ *	[cache.h] WS_Reserve() removed
  * 11.0 (2020-03-16)
  *	Changed type of vsa_suckaddr_len from int to size_t
  *	New prefix_{ptr|len} fields in vrt_backend

From dridi.boukelmoune at gmail.com  Mon Aug 31 18:41:09 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 18:41:09 +0000 (UTC)
Subject: [master] 3e1ad1a10 ws: Polish workspace snapshot
Message-ID: <20200831184109.3E67461E06@lists.varnish-cache.org>


commit 3e1ad1a10e2b3cc75aa1738b4eebb0c7ee77408b
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon May 4 22:55:40 2020 +0200

    ws: Polish workspace snapshot

diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index 680983d91..da94c2b7d 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -38,7 +38,7 @@
 
 #define WS_REDZONE_END		'\x15'
 
-static const void * const snap_overflowed = &snap_overflowed;
+static const uintptr_t snap_overflowed = (uintptr_t)&snap_overflowed;
 
 void
 WS_Assert(const struct ws *ws)
@@ -146,7 +146,7 @@ WS_Reset(struct ws *ws, uintptr_t pp)
 
 	WS_Assert(ws);
 	AN(pp);
-	if (pp == (uintptr_t)snap_overflowed) {
+	if (pp == snap_overflowed) {
 		DSL(DBG_WORKSPACE, 0, "WS_Reset(%p, overflowed)", ws);
 		AN(WS_Overflowed(ws));
 		return;
@@ -254,7 +254,7 @@ WS_Snapshot(struct ws *ws)
 	assert(ws->r == NULL);
 	if (WS_Overflowed(ws)) {
 		DSL(DBG_WORKSPACE, 0, "WS_Snapshot(%p) = overflowed", ws);
-		return ((uintptr_t) snap_overflowed);
+		return (snap_overflowed);
 	}
 	DSL(DBG_WORKSPACE, 0, "WS_Snapshot(%p) = %p", ws, ws->f);
 	return ((uintptr_t)ws->f);

From dridi.boukelmoune at gmail.com  Mon Aug 31 18:41:09 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 18:41:09 +0000 (UTC)
Subject: [master] e5bf34a0c ws: Constify WS_Printf()
Message-ID: <20200831184109.66F4061E0B@lists.varnish-cache.org>


commit e5bf34a0c828431ae94798f572c786ff712d9f6f
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue May 5 15:33:55 2020 +0200

    ws: Constify WS_Printf()
    
    Unlike WS_Alloc() it fills the allocated space with the result, which
    should be considered constant at this point. And on top of that it's
    a constant string and shouldn't be a void pointer.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index c09b5cef6..63d72b526 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -790,7 +790,7 @@ void *WS_Alloc(struct ws *ws, unsigned bytes);
 void *WS_Copy(struct ws *ws, const void *str, int len);
 uintptr_t WS_Snapshot(struct ws *ws);
 int WS_Overflowed(const struct ws *ws);
-void *WS_Printf(struct ws *ws, const char *fmt, ...) v_printflike_(2, 3);
+const char *WS_Printf(struct ws *ws, const char *fmt, ...) v_printflike_(2, 3);
 int WS_Inside(const struct ws *, const void *, const void *);
 void WS_Assert_Allocated(const struct ws *ws, const void *ptr, ssize_t len);
 
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 9ecb5ce76..c6e8b37d7 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -282,7 +282,7 @@ http_SetH(struct http *to, unsigned n, const char *fm)
 static void
 http_PutField(struct http *to, int field, const char *string)
 {
-	char *p;
+	const char *p;
 
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
 	p = WS_Copy(to->ws, string, -1);
@@ -1182,7 +1182,7 @@ void
 http_CopyHome(const struct http *hp)
 {
 	unsigned u, l;
-	char *p;
+	const char *p;
 
 	for (u = 0; u < hp->nhd; u++) {
 		if (hp->hd[u].b == NULL) {
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index da94c2b7d..8e2127c1f 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -224,7 +224,7 @@ WS_Copy(struct ws *ws, const void *str, int len)
 	return (r);
 }
 
-void *
+const char *
 WS_Printf(struct ws *ws, const char *fmt, ...)
 {
 	unsigned u, v;
diff --git a/include/vrt.h b/include/vrt.h
index 8a7e76e16..2177a45a7 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -56,6 +56,7 @@
  *	Added VRT_DirectorResolve()
  *	Added VCL_STRING VRT_BLOB_string(VRT_CTX, VCL_BLOB)
  *	[cache.h] WS_Reserve() removed
+ *	[cache.h] WS_Printf() changed
  * 11.0 (2020-03-16)
  *	Changed type of vsa_suckaddr_len from int to size_t
  *	New prefix_{ptr|len} fields in vrt_backend
diff --git a/lib/libvmod_cookie/vmod_cookie.c b/lib/libvmod_cookie/vmod_cookie.c
index 5be9178e6..6c8b6a3a4 100644
--- a/lib/libvmod_cookie/vmod_cookie.c
+++ b/lib/libvmod_cookie/vmod_cookie.c
@@ -58,8 +58,8 @@ static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER;
 struct cookie {
 	unsigned		magic;
 #define VMOD_COOKIE_ENTRY_MAGIC	0x3BB41543
-	char			*name;
-	char			*value;
+	const char		*name;
+	const char		*value;
 	VTAILQ_ENTRY(cookie)	list;
 };
 
@@ -163,7 +163,7 @@ vmod_set(VRT_CTX, struct vmod_priv *priv, VCL_STRING name, VCL_STRING value)
 {
 	struct vmod_cookie *vcp = cobj_get(priv);
 	struct cookie *cookie;
-	char *p;
+	const char *p;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->ws, WS_MAGIC);

From dridi.boukelmoune at gmail.com  Mon Aug 31 18:41:09 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 18:41:09 +0000 (UTC)
Subject: [master] dcb1cc800 ws: Enforce that WS_Front() is only for
 reservations
Message-ID: <20200831184109.8ACA461E0E@lists.varnish-cache.org>


commit dcb1cc800bbdf626a51d55bceb96ceb5c313b6ad
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue May 5 15:21:36 2020 +0200

    ws: Enforce that WS_Front() is only for reservations
    
    There is otherwise no valid use case to peek inside the workspace.
    
    Only vmod_blob would grab the front pointer before making its
    reservation, but ultimately grabbing the front pointer because it
    needs it for a reservation, proving the point.
    
    Initially caught by wssan from #3320.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 63d72b526..4d2a7c00e 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -800,6 +800,8 @@ char *WS_VSB_finish(struct vsb *, struct ws *, size_t *);
 static inline char*
 WS_Front(const struct ws *ws)
 {
+
+	AN(ws->r);
 	return ws->f;
 }
 
diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c
index a805fa8e2..84d3675c0 100644
--- a/lib/libvmod_blob/vmod_blob.c
+++ b/lib/libvmod_blob/vmod_blob.c
@@ -340,8 +340,8 @@ vmod_decode(VRT_CTX, VCL_ENUM decs, VCL_INT length, VCL_STRANDS strings)
 	AN(strings);
 	CHECK_OBJ_NOTNULL(ctx->ws, WS_MAGIC);
 
-	buf = WS_Front(ctx->ws);
 	space = WS_ReserveAll(ctx->ws);
+	buf = WS_Front(ctx->ws);
 
 	if (length <= 0)
 		length = -1;
@@ -379,8 +379,8 @@ encode(VRT_CTX, enum encoding enc, enum case_e kase, VCL_BLOB b)
 
 	CHECK_OBJ_NOTNULL(ctx->ws, WS_MAGIC);
 	snap = WS_Snapshot(ctx->ws);
-	buf = WS_Front(ctx->ws);
 	space = WS_ReserveAll(ctx->ws);
+	buf = WS_Front(ctx->ws);
 
 	len = func[enc].encode(enc, kase, buf, space, b->blob, b->len);
 

From dridi.boukelmoune at gmail.com  Mon Aug 31 18:41:09 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 18:41:09 +0000 (UTC)
Subject: [master] 17267bc63 vmod_blob: Remove needless workspace snapshot
Message-ID: <20200831184109.A9DDA61E15@lists.varnish-cache.org>


commit 17267bc63a4c241703720b73ada5c3c153e5ac73
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jun 2 16:10:58 2020 +0200

    vmod_blob: Remove needless workspace snapshot
    
    It may predate the use of a workspace reservation, and is ultimately
    redundant.
    
    Originally from #3320.

diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c
index 84d3675c0..0881798a3 100644
--- a/lib/libvmod_blob/vmod_blob.c
+++ b/lib/libvmod_blob/vmod_blob.c
@@ -369,7 +369,6 @@ encode(VRT_CTX, enum encoding enc, enum case_e kase, VCL_BLOB b)
 {
 	ssize_t len;
 	char *buf;
-	uintptr_t snap;
 	unsigned space;
 
 	AENC(enc);
@@ -378,7 +377,6 @@ encode(VRT_CTX, enum encoding enc, enum case_e kase, VCL_BLOB b)
 		return (NULL);
 
 	CHECK_OBJ_NOTNULL(ctx->ws, WS_MAGIC);
-	snap = WS_Snapshot(ctx->ws);
 	space = WS_ReserveAll(ctx->ws);
 	buf = WS_Front(ctx->ws);
 
@@ -387,12 +385,10 @@ encode(VRT_CTX, enum encoding enc, enum case_e kase, VCL_BLOB b)
 	if (len == -1) {
 		ERRNOMEM(ctx, "cannot encode");
 		WS_Release(ctx->ws, 0);
-		WS_Reset(ctx->ws, snap);
 		return (NULL);
 	}
 	if (len == 0) {
 		WS_Release(ctx->ws, 0);
-		WS_Reset(ctx->ws, snap);
 		return ("");
 	}
 	buf[len] = '\0';

From dridi.boukelmoune at gmail.com  Mon Aug 31 18:41:09 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 18:41:09 +0000 (UTC)
Subject: [master] d3d39f563 std.ip: Honor the workspace snapshot/reset contract
Message-ID: <20200831184109.C53F761E19@lists.varnish-cache.org>


commit d3d39f563ce6d99a6581cf138c2ff7b5c3cbd703
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jun 2 16:12:26 2020 +0200

    std.ip: Honor the workspace snapshot/reset contract
    
    Initially caught by wssan from #3320.

diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index 3f0b74ea1..84ee9e14a 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -195,6 +195,7 @@ vmod_integer(VRT_CTX, struct VARGS(integer) *a)
 VCL_IP
 vmod_ip(VRT_CTX, struct VARGS(ip) *a)
 {
+	uintptr_t sn;
 	void *p;
 	VCL_IP retval = NULL;
 
@@ -202,6 +203,7 @@ vmod_ip(VRT_CTX, struct VARGS(ip) *a)
 	if (a->valid_fallback)
 		assert(VSA_Sane(a->fallback));
 
+	sn = WS_Snapshot(ctx->ws);
 	p = WS_Alloc(ctx->ws, vsa_suckaddr_len);
 	if (p == NULL) {
 		VRT_fail(ctx, "std.ip: insufficient workspace");
@@ -217,7 +219,7 @@ vmod_ip(VRT_CTX, struct VARGS(ip) *a)
 	if (retval != NULL)
 		return (retval);
 
-	WS_Reset(ctx->ws, (uintptr_t)p);
+	WS_Reset(ctx->ws, sn);
 
 	if (a->valid_fallback)
 		return (a->fallback);

From dridi.boukelmoune at gmail.com  Mon Aug 31 18:41:10 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 18:41:10 +0000 (UTC)
Subject: [master] 8a2cde419 ws: Replace WS_Front() with WS_Reservation()
Message-ID: <20200831184110.0F4E161E2E@lists.varnish-cache.org>


commit 8a2cde419e6484617838fb32bb1cbcdc5f57950f
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed May 6 11:57:36 2020 +0200

    ws: Replace WS_Front() with WS_Reservation()
    
    And add a companion function WS_ReservationSize(). This makes it explicit
    that accessing the workspace front pointer is only valid for reservations,
    and also informs you whether you are in the middle of a reservation, which
    would be helpful for assertions.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 4d2a7c00e..fb8e11f0b 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -797,12 +797,20 @@ void WS_Assert_Allocated(const struct ws *ws, const void *ptr, ssize_t len);
 void WS_VSB_new(struct vsb *, struct ws *);
 char *WS_VSB_finish(struct vsb *, struct ws *, size_t *);
 
-static inline char*
-WS_Front(const struct ws *ws)
+static inline void *
+WS_Reservation(const struct ws *ws)
+{
+
+	WS_Assert(ws);
+	return (ws->r != NULL ? ws->f : NULL);
+}
+
+static inline unsigned
+WS_ReservationSize(const struct ws *ws)
 {
 
 	AN(ws->r);
-	return ws->f;
+	return (ws->r - ws->f);
 }
 
 /* cache_rfc2616.c */
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 2dd248558..1b13badae 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -482,7 +482,7 @@ SES_Wait(struct sess *sp, const struct transport *xp)
 		return;
 	}
 
-	wp = (void*)WS_Front(sp->ws);
+	wp = WS_Reservation(sp->ws);
 	INIT_OBJ(wp, WAITED_MAGIC);
 	wp->fd = sp->fd;
 	wp->priv1 = sp;
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 7da0734ab..bdc50c6ec 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -554,7 +554,7 @@ VRT_UpperLowerStrands(VRT_CTX, VCL_STRANDS s, int up)
 	CHECK_OBJ_NOTNULL(ctx->ws, WS_MAGIC);
 	AN(s);
 	u = WS_ReserveAll(ctx->ws);
-	r = b = WS_Front(ctx->ws);
+	r = b = WS_Reservation(ctx->ws);
 	e = b + u;
 	for (i = 0; i < s->n; i++) {
 		if (s->p[i] == NULL || s->p[i][0] == '\0')
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index 8e2127c1f..64defc78b 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -377,7 +377,7 @@ WS_VSB_new(struct vsb *vsb, struct ws *ws)
 	if (WS_Overflowed(ws) || u < 2)
 		AN(VSB_init(vsb, bogus, sizeof bogus));
 	else
-		AN(VSB_init(vsb, WS_Front(ws), u));
+		AN(VSB_init(vsb, WS_Reservation(ws), u));
 }
 
 char *
@@ -389,7 +389,7 @@ WS_VSB_finish(struct vsb *vsb, struct ws *ws, size_t *szp)
 	WS_Assert(ws);
 	if (!VSB_finish(vsb)) {
 		p = VSB_data(vsb);
-		if (p == WS_Front(ws)) {
+		if (p == WS_Reservation(ws)) {
 			WS_Release(ws, VSB_len(vsb) + 1);
 			if (szp != NULL)
 				*szp = VSB_len(vsb);
diff --git a/include/vrt.h b/include/vrt.h
index 2177a45a7..a99798c95 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -57,6 +57,8 @@
  *	Added VCL_STRING VRT_BLOB_string(VRT_CTX, VCL_BLOB)
  *	[cache.h] WS_Reserve() removed
  *	[cache.h] WS_Printf() changed
+ *	[cache.h] WS_ReservationSize() added
+ *	[cache.h] WS_Front() replaced by WS_Reservation()
  * 11.0 (2020-03-16)
  *	Changed type of vsa_suckaddr_len from int to size_t
  *	New prefix_{ptr|len} fields in vrt_backend
diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c
index 0881798a3..c9c3dac2e 100644
--- a/lib/libvmod_blob/vmod_blob.c
+++ b/lib/libvmod_blob/vmod_blob.c
@@ -341,7 +341,7 @@ vmod_decode(VRT_CTX, VCL_ENUM decs, VCL_INT length, VCL_STRANDS strings)
 	CHECK_OBJ_NOTNULL(ctx->ws, WS_MAGIC);
 
 	space = WS_ReserveAll(ctx->ws);
-	buf = WS_Front(ctx->ws);
+	buf = WS_Reservation(ctx->ws);
 
 	if (length <= 0)
 		length = -1;
@@ -378,7 +378,7 @@ encode(VRT_CTX, enum encoding enc, enum case_e kase, VCL_BLOB b)
 
 	CHECK_OBJ_NOTNULL(ctx->ws, WS_MAGIC);
 	space = WS_ReserveAll(ctx->ws);
-	buf = WS_Front(ctx->ws);
+	buf = WS_Reservation(ctx->ws);
 
 	len = func[enc].encode(enc, kase, buf, space, b->blob, b->len);
 

From dridi.boukelmoune at gmail.com  Mon Aug 31 18:41:10 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 18:41:10 +0000 (UTC)
Subject: [master] fbe7e7559 ws: Almost ban direct access to the workspace front
Message-ID: <20200831184110.60B8461E51@lists.varnish-cache.org>


commit fbe7e755990de5ca362c8652cd571052330e8f3e
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed May 6 14:01:49 2020 +0200

    ws: Almost ban direct access to the workspace front
    
    Proving that this always needed for use cases dealing with workspace
    reservations.
    
    Well, almost always, for now vmod_vtc and the panic subsystem are left
    alone.

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index b7c46a8a6..9e80efcb1 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -674,7 +674,7 @@ HSH_Purge(struct worker *wrk, struct objhead *oh, vtim_real ttl_now,
 		more = 0;
 		spc = ospc;
 		nobj = 0;
-		ocp = (void*)wrk->aws->f;
+		ocp = WS_Reservation(wrk->aws);
 		Lck_Lock(&oh->mtx);
 		assert(oh->refcnt > 0);
 		VTAILQ_FOREACH(oc, &oh->objcs, hsh_list) {
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index c6e8b37d7..1c608ff3a 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -403,7 +403,7 @@ http_CollectHdrSep(struct http *hp, const char *hdr, const char *sep)
 		if (b == NULL) {
 			/* Found second header, start our collection */
 			ml = WS_ReserveAll(hp->ws);
-			b = hp->ws->f;
+			b = WS_Reservation(hp->ws);
 			e = b + ml;
 			x = Tlen(hp->hd[f]);
 			if (b + x >= e) {
@@ -444,7 +444,7 @@ http_CollectHdrSep(struct http *hp, const char *hdr, const char *sep)
 	hp->nhd = (uint16_t)d;
 	AN(e);
 	*b = '\0';
-	hp->hd[f].b = hp->ws->f;
+	hp->hd[f].b = WS_Reservation(hp->ws);
 	hp->hd[f].e = b;
 	WS_ReleaseP(hp->ws, b + 1);
 }
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 1b13badae..147934aa0 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -124,7 +124,7 @@ ses_res_attr(struct sess *sp, enum sess_attr a, void **dst, int sz)
 	o = WS_ReserveSize(sp->ws, sz);
 	if (o < sz)
 		return (0);
-	*dst = sp->ws->f;
+	*dst = WS_Reservation(sp->ws);
 	o = sp->ws->f - sp->ws->s;
 	WS_Release(sp->ws, sz);
 	assert(o >= 0 && o <= 0xffff);
@@ -210,19 +210,19 @@ HTC_Status(enum htc_status_e e)
 void
 HTC_RxInit(struct http_conn *htc, struct ws *ws)
 {
+	unsigned r;
 	ssize_t l;
 
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
 	htc->ws = ws;
-	(void)WS_ReserveAll(htc->ws);
-	htc->rxbuf_b = ws->f;
-	htc->rxbuf_e = ws->f;
+	r = WS_ReserveAll(htc->ws);
+	htc->rxbuf_b = htc->rxbuf_e = WS_Reservation(ws);
 	if (htc->pipeline_b != NULL) {
 		AN(htc->pipeline_e);
 		// assert(WS_Inside(ws, htc->pipeline_b, htc->pipeline_e));
 		l = htc->pipeline_e - htc->pipeline_b;
 		assert(l > 0);
-		assert(l <= ws->r - htc->rxbuf_b);
+		assert(l <= r);
 		memmove(htc->rxbuf_b, htc->pipeline_b, l);
 		htc->rxbuf_e += l;
 		htc->pipeline_b = NULL;
@@ -415,7 +415,7 @@ ses_handle(struct waited *wp, enum wait_event ev, vtim_real now)
 	CAST_OBJ_NOTNULL(sp, wp->priv1, SESS_MAGIC);
 	CAST_OBJ_NOTNULL(xp, (const void*)wp->priv2, TRANSPORT_MAGIC);
 	AN(wp->priv2);
-	assert((void *)sp->ws->f == wp);
+	assert(WS_Reservation(sp->ws) == wp);
 	wp->magic = 0;
 	wp = NULL;
 
@@ -434,7 +434,7 @@ ses_handle(struct waited *wp, enum wait_event ev, vtim_real now)
 		CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 		/* SES_Wait() guarantees the next will not assert. */
 		assert(sizeof *tp <= WS_ReserveSize(sp->ws, sizeof *tp));
-		tp = (void*)sp->ws->f;
+		tp = WS_Reservation(sp->ws);
 		tp->func = xp->unwait;
 		tp->priv = sp;
 		if (Pool_Task(pp, tp, TASK_QUEUE_REQ))
diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index 3dc23e33f..82ff49ac8 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -234,10 +234,10 @@ VRY_Prep(struct req *req)
 		AZ(req->vary_e);
 		(void)WS_ReserveAll(req->ws);
 	} else {
-		AN(req->ws->r);
+		AN(WS_Reservation(req->ws));
 	}
-	req->vary_b = (void*)req->ws->f;
-	req->vary_e = (void*)req->ws->r;
+	req->vary_b = WS_Reservation(req->ws);
+	req->vary_e = req->vary_b + WS_ReservationSize(req->ws);
 	if (req->vary_b + 2 < req->vary_e)
 		req->vary_b[2] = '\0';
 }
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index bdc50c6ec..144736ae0 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -366,7 +366,7 @@ VRT_String(struct ws *ws, const char *h, const char *p, va_list ap)
 	va_list aq;
 
 	u = WS_ReserveAll(ws);
-	e = b = ws->f;
+	e = b = WS_Reservation(ws);
 	e += u;
 
 	va_copy(aq, ap);
@@ -422,7 +422,7 @@ VRT_String(struct ws *ws, const char *h, const char *p, va_list ap)
 		return (NULL);
 	}
 	e = b;
-	b = ws->f;
+	b = WS_Reservation(ws);
 	WS_Release(ws, e - b);
 	return (b);
 }
@@ -710,7 +710,7 @@ VRT_IP_string(VRT_CTX, VCL_IP ip)
 		WS_Release(ctx->ws, 0);
 		return (NULL);
 	}
-	p = ctx->ws->f;
+	p = WS_Reservation(ctx->ws);
 	VTCP_name(ip, p, len, NULL, 0);
 	WS_Release(ctx->ws, strlen(p) + 1);
 	return (p);
diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c
index 955fb1c2b..f38515151 100644
--- a/bin/varnishd/cache/cache_vrt_re.c
+++ b/bin/varnishd/cache/cache_vrt_re.c
@@ -130,7 +130,7 @@ VRT_regsub(VRT_CTX, int all, const char *str, void *re,
 	}
 
 	u = WS_ReserveAll(ctx->ws);
-	res_e = res_b = b0 = ctx->ws->f;
+	res_e = res_b = b0 = WS_Reservation(ctx->ws);
 	res_e += u;
 
 	do {
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 13a7cb8ac..39e4b4361 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -246,9 +246,9 @@ Pool_Task_Arg(struct worker *wrk, enum task_prio prio, task_func_t *func,
 	}
 	AZ(wrk2->task->func);
 	assert(arg_len <= WS_ReserveSize(wrk2->aws, arg_len));
-	memcpy(wrk2->aws->f, arg, arg_len);
+	memcpy(WS_Reservation(wrk2->aws), arg, arg_len);
 	wrk2->task->func = func;
-	wrk2->task->priv = wrk2->aws->f;
+	wrk2->task->priv = WS_Reservation(wrk2->aws);
 	Lck_Unlock(&pp->mtx);
 	// see signaling_note at the top for explanation
 	if (retval)
diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c
index fcb6af3ee..da35661af 100644
--- a/bin/varnishd/http1/cache_http1_line.c
+++ b/bin/varnishd/http1/cache_http1_line.c
@@ -112,7 +112,7 @@ V1L_Open(struct worker *wrk, struct ws *ws, int *fd, struct vsl_log *vsl,
 		u = IOV_MAX;
 	if (niov != 0 && u > niov)
 		u = niov;
-	v1l->iov = (void*)ws->f;
+	v1l->iov = WS_Reservation(ws);
 	v1l->siov = u;
 	v1l->ciov = u;
 	v1l->wfd = fd;
diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c
index e00473452..34015ac3c 100644
--- a/bin/varnishd/http2/cache_http2_hpack.c
+++ b/bin/varnishd/http2/cache_http2_hpack.c
@@ -183,7 +183,7 @@ h2h_decode_init(const struct h2_sess *h2)
 	 * space. Require non-zero size.
 	 */
 	XXXAN(d->out_l);
-	d->out = h2->new_req->http->ws->f;
+	d->out = WS_Reservation(h2->new_req->http->ws);
 	d->reset = d->out;
 }
 
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index 244680bfb..948a0972b 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -76,7 +76,7 @@ vmod_updown(VRT_CTX, int up, VCL_STRANDS s)
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	u = WS_ReserveAll(ctx->ws);
-	e = b = ctx->ws->f;
+	e = b = WS_Reservation(ctx->ws);
 	e += u;
 	for (i = 0; i < s->n && b < e; i++) {
 		p = s->p[i];
@@ -96,7 +96,7 @@ vmod_updown(VRT_CTX, int up, VCL_STRANDS s)
 		return (NULL);
 	} else {
 		e = b;
-		b = ctx->ws->f;
+		b = WS_Reservation(ctx->ws);
 		WS_Release(ctx->ws, e - b);
 		return (b);
 	}
diff --git a/lib/libvmod_std/vmod_std_querysort.c b/lib/libvmod_std/vmod_std_querysort.c
index 8602dc6e1..8b0a2704f 100644
--- a/lib/libvmod_std/vmod_std_querysort.c
+++ b/lib/libvmod_std/vmod_std_querysort.c
@@ -81,7 +81,7 @@ vmod_querysort(VRT_CTX, VCL_STRING url)
 		return (url);
 
 	u = WS_ReserveLumps(ctx->ws, sizeof(const char **));
-	pp = (const char**)(void*)(ctx->ws->f);
+	pp = WS_Reservation(ctx->ws);
 	if (u < 4) {
 		WS_Release(ctx->ws, 0);
 		WS_MarkOverflow(ctx->ws);

From dridi.boukelmoune at gmail.com  Mon Aug 31 18:41:10 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 18:41:10 +0000 (UTC)
Subject: [master] 1e776c44e ws: Almost ban direct access to the workspace
 reservation
Message-ID: <20200831184110.9281461E5E@lists.varnish-cache.org>


commit 1e776c44e60730f3fef65a5965ba888a1b5289a4
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed May 6 15:07:10 2020 +0200

    ws: Almost ban direct access to the workspace reservation
    
    With the same exceptions as the front pointer. For the cases where
    actually need the value of the reservation pointer, I decided to
    change the approach and make it a length computation instead, when
    possible.

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index d43144dfc..77dbfaa43 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -373,7 +373,7 @@ vca_make_session(struct worker *wrk, void *arg)
 	VTCP_blocking(wa->acceptsock);
 
 	/* Turn accepted socket into a session */
-	AN(wrk->aws->r);
+	AN(WS_Reservation(wrk->aws));
 	sp = SES_New(wrk->pool);
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	wrk->stats->s_sess++;
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index f6b7e6ad0..6284b9f6b 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -1127,6 +1127,6 @@ CNT_Request(struct req *req)
 		VRB_Free(req);
 		req->wrk = NULL;
 	}
-	assert(nxt == REQ_FSM_DISEMBARK || req->ws->r == NULL);
+	assert(nxt == REQ_FSM_DISEMBARK || WS_Reservation(req->ws) == NULL);
 	return (nxt);
 }
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 147934aa0..5e7994bab 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -270,26 +270,29 @@ HTC_RxStuff(struct http_conn *htc, htc_complete_f *func,
 	vtim_dur tmo;
 	vtim_real now;
 	enum htc_status_e hs;
+	unsigned l, r;
 	ssize_t z;
 
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
 	AN(htc->rfd);
 	assert(*htc->rfd > 0);
-	AN(htc->ws->r);
 	AN(htc->rxbuf_b);
-	assert(htc->rxbuf_b <= htc->rxbuf_e);
-	assert(htc->rxbuf_e <= htc->ws->r);
+	AN(WS_Reservation(htc->ws));
+
+	l = pdiff(htc->rxbuf_b, htc->rxbuf_e);
+	r = WS_ReservationSize(htc->ws);
+	assert(l <= r);
 
 	AZ(isnan(tn) && isnan(td));
 	if (t1 != NULL)
 		assert(isnan(*t1));
 
-	if (htc->rxbuf_e == htc->ws->r) {
+	if (l == r) {
 		/* Can't work with a zero size buffer */
 		WS_ReleaseP(htc->ws, htc->rxbuf_b);
 		return (HTC_S_OVERFLOW);
 	}
-	z = (htc->ws->r - htc->rxbuf_b);
+	z = r;
 	if (z < maxbytes)
 		maxbytes = z;	/* Cap maxbytes at available WS */
 
@@ -297,7 +300,8 @@ HTC_RxStuff(struct http_conn *htc, htc_complete_f *func,
 		now = VTIM_real();
 		AZ(htc->pipeline_b);
 		AZ(htc->pipeline_e);
-		assert(htc->rxbuf_e <= htc->ws->r);
+		l = pdiff(htc->rxbuf_b, htc->rxbuf_e);
+		assert(l <= r);
 
 		hs = func(htc);
 		if (hs == HTC_S_OVERFLOW || hs == HTC_S_JUNK) {
diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c
index ff4aaa72b..3bb6345e0 100644
--- a/bin/varnishd/http1/cache_http1_fsm.c
+++ b/bin/varnishd/http1/cache_http1_fsm.c
@@ -84,7 +84,7 @@ http1_req(struct worker *wrk, void *arg)
 
 	THR_SetRequest(req);
 	req->transport = &HTTP1_transport;
-	AZ(wrk->aws->r);
+	AZ(WS_Reservation(wrk->aws));
 	HTTP1_Session(wrk, req);
 	AZ(wrk->v1l);
 	WS_Assert(wrk->aws);
@@ -311,7 +311,7 @@ HTTP1_Session(struct worker *wrk, struct req *req)
 			assert(isnan(req->t_req));
 			AZ(req->vcl);
 			AZ(req->esi_level);
-			AN(req->htc->ws->r);
+			AN(WS_Reservation(req->htc->ws));
 
 			hs = HTC_RxStuff(req->htc, HTTP1_Complete,
 			    &req->t_first, &req->t_req,
@@ -319,7 +319,7 @@ HTTP1_Session(struct worker *wrk, struct req *req)
 			    sp->t_idle + SESS_TMO(sp, timeout_idle),
 			    NAN,
 			    cache_param->http_req_size);
-			AZ(req->htc->ws->r);
+			AZ(WS_Reservation(req->htc->ws));
 			if (hs < HTC_S_EMPTY) {
 				req->acct.req_hdrbytes +=
 				    req->htc->rxbuf_e - req->htc->rxbuf_b;
@@ -354,8 +354,8 @@ HTTP1_Session(struct worker *wrk, struct req *req)
 			if (H2_prism_complete(req->htc) == HTC_S_COMPLETE) {
 				if (!FEATURE(FEATURE_HTTP2)) {
 					SES_Close(req->sp, SC_REQ_HTTP20);
-					AZ(req->ws->r);
-					AZ(wrk->aws->r);
+					AZ(WS_Reservation(req->ws));
+					AZ(WS_Reservation(wrk->aws));
 					http1_setstate(sp, H1CLEANUP);
 					continue;
 				}
@@ -370,8 +370,8 @@ HTTP1_Session(struct worker *wrk, struct req *req)
 			if (i) {
 				assert(req->doclose > 0);
 				SES_Close(req->sp, req->doclose);
-				AZ(req->ws->r);
-				AZ(wrk->aws->r);
+				AZ(WS_Reservation(req->ws));
+				AZ(WS_Reservation(wrk->aws));
 				http1_setstate(sp, H1CLEANUP);
 				continue;
 			}
@@ -405,13 +405,13 @@ HTTP1_Session(struct worker *wrk, struct req *req)
 			AZ(req->top->vcl0);
 			req->task->func = NULL;
 			req->task->priv = NULL;
-			AZ(req->ws->r);
-			AZ(wrk->aws->r);
+			AZ(WS_Reservation(req->ws));
+			AZ(WS_Reservation(wrk->aws));
 			http1_setstate(sp, H1CLEANUP);
 		} else if (st == H1CLEANUP) {
 
-			AZ(wrk->aws->r);
-			AZ(req->ws->r);
+			AZ(WS_Reservation(wrk->aws));
+			AZ(WS_Reservation(req->ws));
 
 			if (sp->fd >= 0 && req->doclose != SC_NULL)
 				SES_Close(sp, req->doclose);
diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c
index da35661af..7ff71c151 100644
--- a/bin/varnishd/http1/cache_http1_line.c
+++ b/bin/varnishd/http1/cache_http1_line.c
@@ -137,7 +137,7 @@ V1L_Close(struct worker *wrk, uint64_t *cnt)
 	wrk->v1l = NULL;
 	CHECK_OBJ_NOTNULL(v1l, V1L_MAGIC);
 	*cnt = v1l->cnt;
-	if (v1l->ws->r)
+	if (WS_Reservation(v1l->ws))
 		WS_Release(v1l->ws, 0);
 	WS_Rollback(v1l->ws, v1l->res);
 	ZERO_OBJ(v1l, sizeof *v1l);
diff --git a/bin/varnishd/http1/cache_http1_proto.c b/bin/varnishd/http1/cache_http1_proto.c
index f30c02561..0e932d7c9 100644
--- a/bin/varnishd/http1/cache_http1_proto.c
+++ b/bin/varnishd/http1/cache_http1_proto.c
@@ -70,9 +70,8 @@ HTTP1_Complete(struct http_conn *htc)
 	enum htc_status_e retval;
 
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
-
-	assert(htc->rxbuf_e >= htc->rxbuf_b);
-	assert(htc->rxbuf_e <= htc->ws->r);
+	AN(WS_Reservation(htc->ws));
+	assert(pdiff(htc->rxbuf_b, htc->rxbuf_e) <= WS_ReservationSize(htc->ws));
 
 	/* Skip any leading white space */
 	for (p = htc->rxbuf_b ; p < htc->rxbuf_e && vct_islws(*p); p++)
diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c
index 34015ac3c..281fc7fac 100644
--- a/bin/varnishd/http2/cache_http2_hpack.c
+++ b/bin/varnishd/http2/cache_http2_hpack.c
@@ -231,13 +231,16 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l)
 	struct http *hp;
 	struct h2h_decode *d;
 	size_t in_u = 0;
+	const char *r, *e;
 
 	CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC);
 	CHECK_OBJ_NOTNULL(h2->new_req, REQ_MAGIC);
 	hp = h2->new_req->http;
 	CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC);
 	CHECK_OBJ_NOTNULL(hp->ws, WS_MAGIC);
-	AN(hp->ws->r);
+	r = WS_Reservation(hp->ws);
+	AN(r);
+	e = r + WS_ReservationSize(hp->ws);
 	d = h2->decode;
 	CHECK_OBJ_NOTNULL(d, H2H_DECODE_MAGIC);
 
@@ -316,9 +319,9 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l)
 
 		if (d->error == H2SE_ENHANCE_YOUR_CALM) {
 			d->out = d->reset;
-			d->out_l = hp->ws->r - d->out;
+			d->out_l = e - d->out;
 			d->out_u = 0;
-			assert(d->out_u < d->out_l);
+			assert(d->out_l > 0);
 		} else if (d->error)
 			break;
 	}
diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 704b5cc94..edcf64c04 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -184,7 +184,7 @@ h2_del_req(struct worker *wrk, const struct h2_req *r2)
 	/* XXX: PRIORITY reshuffle */
 	VTAILQ_REMOVE(&h2->streams, r2, list);
 	Lck_Unlock(&sp->mtx);
-	AZ(r2->req->ws->r);
+	AZ(WS_Reservation(r2->req->ws));
 	Req_Cleanup(sp, wrk, r2->req);
 	Req_Release(r2->req);
 }
@@ -532,7 +532,7 @@ h2_do_req(struct worker *wrk, void *priv)
 
 	wrk->stats->client_req++;
 	if (CNT_Request(req) != REQ_FSM_DISEMBARK) {
-		AZ(req->ws->r);
+		AZ(WS_Reservation(req->ws));
 		AZ(req->top->vcl0);
 		h2 = r2->h2sess;
 		CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC);
@@ -563,7 +563,7 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2,
 		Lck_Lock(&h2->sess->mtx);
 		VSLb(h2->vsl, SLT_Debug, "HPACK/FINI %s", h2e->name);
 		Lck_Unlock(&h2->sess->mtx);
-		AZ(r2->req->ws->r);
+		AZ(WS_Reservation(r2->req->ws));
 		h2_del_req(wrk, r2);
 		return (h2e);
 	}
@@ -685,7 +685,7 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 		VSLb(h2->vsl, SLT_Debug, "HPACK(hdr) %s", h2e->name);
 		Lck_Unlock(&h2->sess->mtx);
 		(void)h2h_decode_fini(h2);
-		AZ(r2->req->ws->r);
+		AZ(WS_Reservation(r2->req->ws));
 		h2_del_req(wrk, r2);
 		return (h2e);
 	}
@@ -720,7 +720,7 @@ h2_rx_continuation(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 		VSLb(h2->vsl, SLT_Debug, "HPACK(cont) %s", h2e->name);
 		Lck_Unlock(&h2->sess->mtx);
 		(void)h2h_decode_fini(h2);
-		AZ(r2->req->ws->r);
+		AZ(WS_Reservation(r2->req->ws));
 		h2_del_req(wrk, r2);
 		return (h2e);
 	}
diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c
index 571a42843..15dd17b27 100644
--- a/bin/varnishd/http2/cache_http2_session.c
+++ b/bin/varnishd/http2/cache_http2_session.c
@@ -148,7 +148,7 @@ h2_del_sess(struct worker *wrk, struct h2_sess *h2, enum sess_close reason)
 	VHT_Fini(h2->dectbl);
 	AZ(pthread_cond_destroy(h2->winupd_cond));
 	TAKE_OBJ_NOTNULL(req, &h2->srq, REQ_MAGIC);
-	AZ(req->ws->r);
+	AZ(WS_Reservation(req->ws));
 	sp = h2->sess;
 	Req_Cleanup(sp, wrk, req);
 	Req_Release(req);
@@ -367,21 +367,21 @@ h2_new_session(struct worker *wrk, void *arg)
 	HTC_RxPipeline(h2->htc, h2->htc->rxbuf_b + sizeof(H2_prism));
 	WS_Rollback(h2->ws, 0);
 	HTC_RxInit(h2->htc, h2->ws);
-	AN(h2->ws->r);
+	AN(WS_Reservation(h2->ws));
 	VSLb(h2->vsl, SLT_Debug, "H2: Got pu PRISM");
 
 	THR_SetRequest(h2->srq);
-	AN(h2->ws->r);
+	AN(WS_Reservation(h2->ws));
 
 	l = h2_enc_settings(&h2->local_settings, settings, sizeof (settings));
-	AN(h2->ws->r);
+	AN(WS_Reservation(h2->ws));
 	H2_Send_Get(wrk, h2, h2->req0);
-	AN(h2->ws->r);
+	AN(WS_Reservation(h2->ws));
 	H2_Send_Frame(wrk, h2,
 	    H2_F_SETTINGS, H2FF_NONE, l, 0, settings);
-	AN(h2->ws->r);
+	AN(WS_Reservation(h2->ws));
 	H2_Send_Rel(h2, h2->req0);
-	AN(h2->ws->r);
+	AN(WS_Reservation(h2->ws));
 
 	/* and off we go... */
 	h2->cond = &wrk->cond;
@@ -394,7 +394,7 @@ h2_new_session(struct worker *wrk, void *arg)
 			h2->error = H2CE_INTERNAL_ERROR;
 			break;
 		}
-		AN(h2->ws->r);
+		AN(WS_Reservation(h2->ws));
 	}
 
 	AN(h2->error);
diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c
index ff9066e0d..36ffabff2 100644
--- a/bin/varnishd/proxy/cache_proxy_proto.c
+++ b/bin/varnishd/proxy/cache_proxy_proto.c
@@ -480,9 +480,8 @@ vpx_complete(struct http_conn *htc)
 	char *p, *q;
 
 	CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC);
-
-	assert(htc->rxbuf_e >= htc->rxbuf_b);
-	assert(htc->rxbuf_e <= htc->ws->r);
+	AN(WS_Reservation(htc->ws));
+	assert(pdiff(htc->rxbuf_b, htc->rxbuf_e) <= WS_ReservationSize(htc->ws));
 
 	l = htc->rxbuf_e - htc->rxbuf_b;
 	p = htc->rxbuf_b;

From dridi.boukelmoune at gmail.com  Mon Aug 31 18:41:10 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 18:41:10 +0000 (UTC)
Subject: [master] 91f6926e3 ws: Almost ban direct access to the workspace start
Message-ID: <20200831184110.B76A661E69@lists.varnish-cache.org>


commit 91f6926e3af1413664809f42d9ba3af826fa55a6
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed May 6 18:48:29 2020 +0200

    ws: Almost ban direct access to the workspace start
    
    To accomodate the optimization for session attributes new functions are
    added and with them extra checks.

diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 5e7994bab..41fec887e 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -89,10 +89,9 @@ ses_get_attr(const struct sess *sp, enum sess_attr a, void **dst)
 	if (sp->sattr[a] == 0xffff) {
 		*dst = NULL;
 		return (-1);
-	} else {
-		*dst = sp->ws->s + sp->sattr[a];
-		return (0);
 	}
+	*dst = WS_AtOffset(sp->ws, sp->sattr[a], 0);
+	return (0);
 }
 
 static int
@@ -106,7 +105,7 @@ ses_set_attr(const struct sess *sp, enum sess_attr a, const void *src, int sz)
 
 	if (sp->sattr[a] == 0xffff)
 		return (-1);
-	dst = sp->ws->s + sp->sattr[a];
+	dst = WS_AtOffset(sp->ws, sp->sattr[a], sz);
 	AN(dst);
 	memcpy(dst, src, sz);
 	return (0);
@@ -115,19 +114,18 @@ ses_set_attr(const struct sess *sp, enum sess_attr a, const void *src, int sz)
 static int
 ses_res_attr(struct sess *sp, enum sess_attr a, void **dst, int sz)
 {
-	ssize_t o;
+	unsigned o;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	assert(a < SA_LAST);
 	assert(sz >= 0);
 	AN(dst);
-	o = WS_ReserveSize(sp->ws, sz);
-	if (o < sz)
+	if (WS_ReserveSize(sp->ws, sz) == 0)
 		return (0);
 	*dst = WS_Reservation(sp->ws);
-	o = sp->ws->f - sp->ws->s;
+	o = WS_ReservationOffset(sp->ws);
 	WS_Release(sp->ws, sz);
-	assert(o >= 0 && o <= 0xffff);
+	assert(o <= 0xffff);
 	sp->sattr[a] = (uint16_t)o;
 	return (1);
 }
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index 339a89ddd..dca3a5cc2 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -465,6 +465,15 @@ void WRK_Init(void);
 
 /* cache_ws.c */
 void WS_Rollback(struct ws *, uintptr_t);
+void *WS_AtOffset(const struct ws *ws, unsigned off, unsigned len);
+
+static inline unsigned
+WS_ReservationOffset(const struct ws *ws)
+{
+
+	AN(ws->r);
+	return (ws->f - ws->s);
+}
 
 /* http1/cache_http1_pipe.c */
 void V1P_Init(void);
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index 64defc78b..7f51b5379 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -351,6 +351,17 @@ WS_Overflowed(const struct ws *ws)
 	return (1);
 }
 
+void *
+WS_AtOffset(const struct ws *ws, unsigned off, unsigned len)
+{
+	char *ptr;
+
+	WS_Assert(ws);
+	ptr = ws->s + off;
+	WS_Assert_Allocated(ws, ptr, len);
+	return (ptr);
+}
+
 /*---------------------------------------------------------------------
  * Build a VSB on a workspace.
  * Usage pattern:

From dridi.boukelmoune at gmail.com  Mon Aug 31 18:41:10 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 18:41:10 +0000 (UTC)
Subject: [master] 758b4bf4c ws: Move the workspace panic dump to cache_ws.c
Message-ID: <20200831184110.DA7B361E70@lists.varnish-cache.org>


commit 758b4bf4c9d0b2b7afc06b1e548a8592cdfd1ad4
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed May 6 22:30:35 2020 +0200

    ws: Move the workspace panic dump to cache_ws.c
    
    The goal is to avoid direct field access inside struct ws outside of
    cache_ws.c and open the possibility to perform a hexdump of a corrupted
    allocation in the future, when wssan panics while checking red zones.
    
    This effectively completes the ban of direct field access to the front,
    start and reservation pointers outside of cache_ws.c, and since the
    devil is in the details cache_http.c directly touches the id field and
    vmod_vtc also accesses the 3 aforementioned pointers.

diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 19ae14a00..fa1ff00d4 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -130,38 +130,6 @@ PAN_already(struct vsb *vsb, const void *ptr)
 
 /*--------------------------------------------------------------------*/
 
-static void
-pan_ws(struct vsb *vsb, const struct ws *ws)
-{
-
-	VSB_printf(vsb, "ws = %p {\n", ws);
-	if (PAN_already(vsb, ws))
-		return;
-	VSB_indent(vsb, 2);
-	PAN_CheckMagic(vsb, ws, WS_MAGIC);
-	if (ws->id[0] != '\0' && (!(ws->id[0] & 0x20)))
-		VSB_cat(vsb, "OVERFLOWED ");
-	VSB_printf(vsb, "id = \"%s\",\n", ws->id);
-	VSB_printf(vsb, "{s, f, r, e} = {%p", ws->s);
-	if (ws->f >= ws->s)
-		VSB_printf(vsb, ", +%ld", (long) (ws->f - ws->s));
-	else
-		VSB_printf(vsb, ", %p", ws->f);
-	if (ws->r >= ws->s)
-		VSB_printf(vsb, ", +%ld", (long) (ws->r - ws->s));
-	else
-		VSB_printf(vsb, ", %p", ws->r);
-	if (ws->e >= ws->s)
-		VSB_printf(vsb, ", +%ld", (long) (ws->e - ws->s));
-	else
-		VSB_printf(vsb, ", %p", ws->e);
-	VSB_cat(vsb, "},\n");
-	VSB_indent(vsb, -2);
-	VSB_cat(vsb, "},\n");
-}
-
-/*--------------------------------------------------------------------*/
-
 static void
 pan_htc(struct vsb *vsb, const struct http_conn *htc)
 {
@@ -174,7 +142,7 @@ pan_htc(struct vsb *vsb, const struct http_conn *htc)
 	if (htc->rfd != NULL)
 		VSB_printf(vsb, "fd = %d (@%p),\n", *htc->rfd, htc->rfd);
 	VSB_printf(vsb, "doclose = %s,\n", sess_close_2str(htc->doclose, 0));
-	pan_ws(vsb, htc->ws);
+	WS_Panic(htc->ws, vsb);
 	VSB_printf(vsb, "{rxbuf_b, rxbuf_e} = {%p, %p},\n",
 	    htc->rxbuf_b, htc->rxbuf_e);
 	VSB_printf(vsb, "{pipeline_b, pipeline_e} = {%p, %p},\n",
@@ -203,7 +171,7 @@ pan_http(struct vsb *vsb, const char *id, const struct http *h)
 		return;
 	VSB_indent(vsb, 2);
 	PAN_CheckMagic(vsb, h, HTTP_MAGIC);
-	pan_ws(vsb, h->ws);
+	WS_Panic(h->ws, vsb);
 	VSB_cat(vsb, "hdrs {\n");
 	VSB_indent(vsb, 2);
 	for (i = 0; i < h->nhd; ++i) {
@@ -302,7 +270,7 @@ pan_wrk(struct vsb *vsb, const struct worker *wrk)
 		return;
 	VSB_indent(vsb, 2);
 	PAN_CheckMagic(vsb, wrk, WORKER_MAGIC);
-	pan_ws(vsb, wrk->aws);
+	WS_Panic(wrk->aws, vsb);
 
 	m = wrk->cur_method;
 	VSB_cat(vsb, "VCL::method = ");
@@ -424,7 +392,7 @@ pan_busyobj(struct vsb *vsb, const struct busyobj *bo)
 	if (bo->filter_list != NULL)
 		VSB_printf(vsb, "filter_list = \"%s\",\n", bo->filter_list);
 
-	pan_ws(vsb, bo->ws);
+	WS_Panic(bo->ws, vsb);
 	VSB_printf(vsb, "ws_bo = %p,\n", (void *)bo->ws_bo);
 
 	// bereq0 left out
@@ -531,7 +499,7 @@ pan_req(struct vsb *vsb, const struct req *req)
 	if (req->wrk != NULL)
 		pan_wrk(vsb, req->wrk);
 
-	pan_ws(vsb, req->ws);
+	WS_Panic(req->ws, vsb);
 	if (VALID_OBJ(req->htc, HTTP_CONN_MAGIC))
 		pan_htc(vsb, req->htc);
 	pan_http(vsb, "req", req->http);
@@ -582,7 +550,7 @@ pan_sess(struct vsb *vsb, const struct sess *sp)
 	    sp->fd, VXID(sp->vxid));
 	VSB_printf(vsb, "t_open = %f,\n", sp->t_open);
 	VSB_printf(vsb, "t_idle = %f,\n", sp->t_idle);
-	pan_ws(vsb, sp->ws);
+	WS_Panic(sp->ws, vsb);
 	VSB_printf(vsb, "transport = %s",
 	    xp == NULL ? "<none>" : xp->name);
 	if (xp != NULL && xp->sess_panic != NULL) {
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index dca3a5cc2..9d98778d3 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -464,6 +464,8 @@ void VMOD_Panic(struct vsb *);
 void WRK_Init(void);
 
 /* cache_ws.c */
+void WS_Panic(const struct ws *ws, struct vsb *vsb);
+
 void WS_Rollback(struct ws *, uintptr_t);
 void *WS_AtOffset(const struct ws *ws, unsigned off, unsigned len);
 
diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index 82ff49ac8..44610e682 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -219,9 +219,6 @@ vry_cmp(const uint8_t *v1, const uint8_t *v2)
 
 /**********************************************************************
  * Prepare predictive vary string
- *
- * XXX: Strictly speaking vary_b and vary_e could be replaced with
- * XXX: req->ws->{f,r}.   Space in struct req vs. code-readability...
  */
 
 void
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index 7f51b5379..b4dc26435 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -415,3 +415,35 @@ WS_VSB_finish(struct vsb *vsb, struct ws *ws, size_t *szp)
 		*szp = 0;
 	return (NULL);
 }
+
+/*--------------------------------------------------------------------*/
+
+void
+WS_Panic(const struct ws *ws, struct vsb *vsb)
+{
+
+	VSB_printf(vsb, "ws = %p {\n", ws);
+	if (PAN_already(vsb, ws))
+		return;
+	VSB_indent(vsb, 2);
+	PAN_CheckMagic(vsb, ws, WS_MAGIC);
+	if (ws->id[0] != '\0' && (!(ws->id[0] & 0x20)))	// cheesy islower()
+		VSB_cat(vsb, "OVERFLOWED ");
+	VSB_printf(vsb, "id = \"%s\",\n", ws->id);
+	VSB_printf(vsb, "{s, f, r, e} = {%p", ws->s);
+	if (ws->f >= ws->s)
+		VSB_printf(vsb, ", +%ld", (long) (ws->f - ws->s));
+	else
+		VSB_printf(vsb, ", %p", ws->f);
+	if (ws->r >= ws->s)
+		VSB_printf(vsb, ", +%ld", (long) (ws->r - ws->s));
+	else
+		VSB_printf(vsb, ", %p", ws->r);
+	if (ws->e >= ws->s)
+		VSB_printf(vsb, ", +%ld", (long) (ws->e - ws->s));
+	else
+		VSB_printf(vsb, ", %p", ws->e);
+	VSB_cat(vsb, "},\n");
+	VSB_indent(vsb, -2);
+	VSB_cat(vsb, "},\n");
+}

From dridi.boukelmoune at gmail.com  Mon Aug 31 18:41:11 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 18:41:11 +0000 (UTC)
Subject: [master] 23f8e3407 ws: Ban direct access to the workspace id
Message-ID: <20200831184111.0717461E76@lists.varnish-cache.org>


commit 23f8e3407d0ce6dc6a411e33d9ba16a079d06302
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed May 6 22:58:07 2020 +0200

    ws: Ban direct access to the workspace id
    
    From this point on, only cache_ws.c fiddles with struct ws, which needs
    to remain visible in order to be embeddable in other data structures. We
    have an API covering all use cases in tree, except vmod_vtc operations
    that violate the contract to provide a rudimentary dump in VCL for test
    purposes.
    
    Inline functions provided in headers aren't considered a problem here.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index fb8e11f0b..b3ae30c99 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -136,10 +136,12 @@ struct lock { void *priv; };	// Opaque
  * Workspace structure for quick memory allocation.
  */
 
+#define WS_ID_SIZE 4
+
 struct ws {
 	unsigned		magic;
 #define WS_MAGIC		0x35fac554
-	char			id[4];		/* identity */
+	char			id[WS_ID_SIZE];	/* identity */
 	char			*s;		/* (S)tart of buffer */
 	char			*f;		/* (F)ree/front pointer */
 	char			*r;		/* (R)eserved length */
@@ -793,6 +795,7 @@ int WS_Overflowed(const struct ws *ws);
 const char *WS_Printf(struct ws *ws, const char *fmt, ...) v_printflike_(2, 3);
 int WS_Inside(const struct ws *, const void *, const void *);
 void WS_Assert_Allocated(const struct ws *ws, const void *ptr, ssize_t len);
+void WS_Id(const struct ws *ws, char *id);
 
 void WS_VSB_new(struct vsb *, struct ws *);
 char *WS_VSB_finish(struct vsb *, struct ws *, size_t *);
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 1c608ff3a..9fa4c1f64 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -112,10 +112,11 @@ http_VSL_log(const struct http *hp)
 static void
 http_fail(const struct http *hp)
 {
+	char id[WS_ID_SIZE];
 
 	VSC_C_main->losthdr++;
-	hp->ws->id[0] |= 0x20;		// cheesy tolower()
-	VSLb(hp->vsl, SLT_Error, "out of workspace (%s)", hp->ws->id);
+	WS_Id(hp->ws, id);
+	VSLb(hp->vsl, SLT_Error, "out of workspace (%s)", id);
 	WS_MarkOverflow(hp->ws);
 }
 
diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index b4dc26435..c8fb32210 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -114,6 +114,16 @@ WS_Init(struct ws *ws, const char *id, void *space, unsigned len)
 	WS_Assert(ws);
 }
 
+void
+WS_Id(const struct ws *ws, char *id)
+{
+
+	WS_Assert(ws);
+	AN(id);
+	memcpy(id, ws->id, WS_ID_SIZE);
+	id[0] |= 0x20;			// cheesy tolower()
+}
+
 void
 WS_MarkOverflow(struct ws *ws)
 {

From dridi.boukelmoune at gmail.com  Mon Aug 31 18:41:11 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 18:41:11 +0000 (UTC)
Subject: [master] b2ffd1a9e sess: Forbid the reserved "no attribute" offset
Message-ID: <20200831184111.3411A61E7A@lists.varnish-cache.org>


commit b2ffd1a9ed31096c9c1750a506378363e2fd11e5
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu May 7 11:51:55 2020 +0200

    sess: Forbid the reserved "no attribute" offset
    
    The offset must be strictly lower than 0xffff, otherwise subsequent get
    and set operations will ignore it. Instead of panicking, we release the
    workspace and carry on if that happens.
    
    This is only for correctness' sake, the probability to run into this is
    epsilon.

diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 41fec887e..fa6ab0b96 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -79,6 +79,8 @@ SES_SetTransport(struct worker *wrk, struct sess *sp, struct req *req,
 
 /*--------------------------------------------------------------------*/
 
+#define SES_NOATTR_OFFSET 0xffff
+
 static int
 ses_get_attr(const struct sess *sp, enum sess_attr a, void **dst)
 {
@@ -86,7 +88,7 @@ ses_get_attr(const struct sess *sp, enum sess_attr a, void **dst)
 	assert(a < SA_LAST);
 	AN(dst);
 
-	if (sp->sattr[a] == 0xffff) {
+	if (sp->sattr[a] == SES_NOATTR_OFFSET) {
 		*dst = NULL;
 		return (-1);
 	}
@@ -103,7 +105,7 @@ ses_set_attr(const struct sess *sp, enum sess_attr a, const void *src, int sz)
 	AN(src);
 	assert(sz > 0);
 
-	if (sp->sattr[a] == 0xffff)
+	if (sp->sattr[a] == SES_NOATTR_OFFSET)
 		return (-1);
 	dst = WS_AtOffset(sp->ws, sp->sattr[a], sz);
 	AN(dst);
@@ -122,11 +124,14 @@ ses_res_attr(struct sess *sp, enum sess_attr a, void **dst, int sz)
 	AN(dst);
 	if (WS_ReserveSize(sp->ws, sz) == 0)
 		return (0);
-	*dst = WS_Reservation(sp->ws);
 	o = WS_ReservationOffset(sp->ws);
-	WS_Release(sp->ws, sz);
-	assert(o <= 0xffff);
+	if (o >= SES_NOATTR_OFFSET) {
+		WS_Release(sp->ws, 0);
+		return (0);
+	}
+	*dst = WS_Reservation(sp->ws);
 	sp->sattr[a] = (uint16_t)o;
+	WS_Release(sp->ws, sz);
 	return (1);
 }
 

From dridi.boukelmoune at gmail.com  Mon Aug 31 18:41:11 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 18:41:11 +0000 (UTC)
Subject: [master] 4537efefd ws: Put canaries all the way to the end of the
 workspace
Message-ID: <20200831184111.637BF61E83@lists.varnish-cache.org>


commit 4537efefd3eee3778a367503d8da8c674102578b
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu May 7 18:55:40 2020 +0200

    ws: Put canaries all the way to the end of the workspace
    
    It might help catch more buffer overflows, although we still only check
    the first byte.

diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index c8fb32210..4e09b59ae 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -98,6 +98,7 @@ WS_Assert_Allocated(const struct ws *ws, const void *ptr, ssize_t len)
 void
 WS_Init(struct ws *ws, const char *id, void *space, unsigned len)
 {
+	unsigned l;
 
 	DSL(DBG_WORKSPACE, 0,
 	    "WS_Init(%p, \"%s\", %p, %u)", ws, id, space, len);
@@ -105,9 +106,9 @@ WS_Init(struct ws *ws, const char *id, void *space, unsigned len)
 	INIT_OBJ(ws, WS_MAGIC);
 	ws->s = space;
 	assert(PAOK(space));
-	len = PRNDDN(len - 1);
-	ws->e = ws->s + len;
-	*ws->e = WS_REDZONE_END;
+	l = PRNDDN(len - 1);
+	ws->e = ws->s + l;
+	memset(ws->e, WS_REDZONE_END, len - l);
 	ws->f = ws->s;
 	assert(id[0] & 0x20);		// cheesy islower()
 	bstrcpy(ws->id, id);

From dridi.boukelmoune at gmail.com  Mon Aug 31 18:41:11 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 18:41:11 +0000 (UTC)
Subject: [master] e0cb8d325 vary: Prevent a buffer overflow in VRY_Validate()
Message-ID: <20200831184111.A800661E9A@lists.varnish-cache.org>


commit e0cb8d325aeca870f3adb94c96685728ae684ebf
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu May 7 19:00:10 2020 +0200

    vary: Prevent a buffer overflow in VRY_Validate()
    
    We might read past the end of the workspace when no space was available
    at reservation time. This would normally go unnotticed since we used to
    get zeros after the end of workspace marker, and no assertion would
    trigger. It became visible with the previous commit for pointer-aligned
    workspace sizes like the current page-aligned default values.
    
    Initially caught by wssan from #3320.
    
    Fixes #3319

diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index 44610e682..1e77730b3 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -260,6 +260,15 @@ VRY_Finish(struct req *req, enum vry_finish_flag flg)
 {
 	uint8_t *p = NULL;
 
+	if (req->vary_b + 2 >= req->vary_e) {
+		AZ(req->vary_l);
+		req->vary_b = NULL;
+		req->vary_e = NULL;
+		WS_Release(req->ws, 0);
+		WS_MarkOverflow(req->ws);
+		return;
+	}
+
 	(void)VRY_Validate(req->vary_b);
 	if (flg == KEEP && req->vary_l != NULL) {
 		p = malloc(req->vary_l - req->vary_b);
diff --git a/bin/varnishtest/tests/r03319.vtc b/bin/varnishtest/tests/r03319.vtc
new file mode 100644
index 000000000..4f457fb89
--- /dev/null
+++ b/bin/varnishtest/tests/r03319.vtc
@@ -0,0 +1,21 @@
+varnishtest "Vary handling out of workspace"
+
+varnish v1 -vcl {
+	import vtc;
+
+	backend be none;
+
+	sub vcl_recv {
+		vtc.workspace_alloc(client, vtc.workspace_free(client));
+	}
+
+	sub vcl_backend_fetch {
+		return (error(200));
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 500
+} -run

From dridi.boukelmoune at gmail.com  Mon Aug 31 18:41:11 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 18:41:11 +0000 (UTC)
Subject: [master] 3aac3b67f http: Remove redundant WS_MarkOverflow()
Message-ID: <20200831184111.D012461EA3@lists.varnish-cache.org>


commit 3aac3b67f6b89f087957b1e1cab2807101be17b8
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Jul 13 15:02:36 2020 +0200

    http: Remove redundant WS_MarkOverflow()
    
    It used to be relevant before we had WS_Id().

diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 9fa4c1f64..9760f79a1 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -117,7 +117,7 @@ http_fail(const struct http *hp)
 	VSC_C_main->losthdr++;
 	WS_Id(hp->ws, id);
 	VSLb(hp->vsl, SLT_Error, "out of workspace (%s)", id);
-	WS_MarkOverflow(hp->ws);
+	assert(WS_Overflowed(hp->ws));
 }
 
 /*--------------------------------------------------------------------

From dridi.boukelmoune at gmail.com  Mon Aug 31 18:53:08 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 18:53:08 +0000 (UTC)
Subject: [master] c70b159aa changelog: Fix and sort ticket references
Message-ID: <20200831185308.4B49A63D54@lists.varnish-cache.org>


commit c70b159aac881477e958c649f743f3cbe4544c6f
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 31 20:51:58 2020 +0200

    changelog: Fix and sort ticket references
    
    Spotted by @gquintard

diff --git a/doc/changes.rst b/doc/changes.rst
index 93e27a4c5..a6ecf1d96 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -90,10 +90,10 @@ NEXT (scheduled 2020-09-15)
   work in ``vcl_pipe {}`` like in ``vcl_backend_* {}`` subs.
 
 .. _2990: https://github.com/varnishcache/varnish-cache/issues/2990
-.. _3253
-.. _3241
-.. _3273
-.. _3002
+.. _3002: https://github.com/varnishcache/varnish-cache/issues/3002
+.. _3241: https://github.com/varnishcache/varnish-cache/issues/3241
+.. _3253: https://github.com/varnishcache/varnish-cache/issues/3253
+.. _3273: https://github.com/varnishcache/varnish-cache/issues/3273
 
 
 ================================

From dridi.boukelmoune at gmail.com  Mon Aug 31 19:05:07 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 31 Aug 2020 19:05:07 +0000 (UTC)
Subject: [master] 7d3aee03d Missing ssize_t definition
Message-ID: <20200831190507.D7122643CB@lists.varnish-cache.org>


commit 7d3aee03df8e2f7488334237b28ac1db520f52bf
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 31 21:03:41 2020 +0200

    Missing ssize_t definition

diff --git a/bin/varnishtest/vtc_sess.c b/bin/varnishtest/vtc_sess.c
index 7105308c3..ae2284687 100644
--- a/bin/varnishtest/vtc_sess.c
+++ b/bin/varnishtest/vtc_sess.c
@@ -1,4 +1,6 @@
 
+#include <sys/types.h>
+
 #include <stdlib.h>
 #include <string.h>