From nils.goroll at uplex.de  Tue Jun  2 07:07:10 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  2 Jun 2020 07:07:10 +0000 (UTC)
Subject: [master] f80b1910d VJ_make_subdir needs JAIL_MASTER_FILE
Message-ID: <20200602070710.1B8956142@lists.varnish-cache.org>


commit f80b1910ddb5854e51df752718dff4e5afd84bc8
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 2 08:34:52 2020 +0200

    VJ_make_subdir needs JAIL_MASTER_FILE

diff --git a/bin/varnishd/mgt/mgt_jail.c b/bin/varnishd/mgt/mgt_jail.c
index 615f6735d..b65005fc6 100644
--- a/bin/varnishd/mgt/mgt_jail.c
+++ b/bin/varnishd/mgt/mgt_jail.c
@@ -180,6 +180,7 @@ VJ_make_subdir(const char *dname, const char *what, struct vsb *vsb)
 	if (vjt->make_subdir != NULL)
 		return (vjt->make_subdir(dname, what, vsb));
 
+	VJ_master(JAIL_MASTER_FILE);
 	if (mkdir(dname, 0755) < 0 && errno != EEXIST) {
 		e = errno;
 		if (vsb != NULL) {
@@ -193,6 +194,7 @@ VJ_make_subdir(const char *dname, const char *what, struct vsb *vsb)
 		}
 		return (1);
 	}
+	VJ_master(JAIL_MASTER_LOW);
 	return (0);
 }
 

From nils.goroll at uplex.de  Tue Jun  2 07:07:10 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  2 Jun 2020 07:07:10 +0000 (UTC)
Subject: [master] 6f1a0831d we can not rmdir our current working directory (on
 solaris)
Message-ID: <20200602070710.395F66145@lists.varnish-cache.org>


commit 6f1a0831d08c5d78d137d706b4d414fef5ae921f
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 2 09:04:27 2020 +0200

    we can not rmdir our current working directory (on solaris)

diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index cd63ed93b..7eb7669cc 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -252,6 +252,7 @@ mgt_Cflag_atexit(void)
 		return;
 	VJ_rmdir("vmod_cache");
 	VJ_unlink("_.pid");
+	(void)chdir("/");
 	VJ_rmdir(workdir);
 }
 

From nils.goroll at uplex.de  Tue Jun  2 09:22:06 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  2 Jun 2020 09:22:06 +0000 (UTC)
Subject: [master] 19593074d Rework the solaris jail code
Message-ID: <20200602092206.7C4C9618E4@lists.varnish-cache.org>


commit 19593074d901e002be3a09ada50c6d9c40d8d071
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 2 10:48:08 2020 +0200

    Rework the solaris jail code
    
    - simplify definition of privileges in a table file
    - only initialize priv sets once
    - implement the master jails

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 74974525b..3b87ddfe6 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -80,6 +80,7 @@ varnishd_SOURCES = \
 	mgt/mgt_cli.c \
 	mgt/mgt_jail.c \
 	mgt/mgt_jail_solaris.c \
+	mgt/mgt_jail_solaris_tbl.h \
 	mgt/mgt_jail_unix.c \
 	mgt/mgt_main.c \
 	mgt/mgt_param.c \
diff --git a/bin/varnishd/mgt/mgt_jail_solaris.c b/bin/varnishd/mgt/mgt_jail_solaris.c
index 961ce1e8e..ef2b5423a 100644
--- a/bin/varnishd/mgt/mgt_jail_solaris.c
+++ b/bin/varnishd/mgt/mgt_jail_solaris.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2006-2011 Varnish Software AS
- * Copyright (c) 2011-2015 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2011-2020 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
@@ -209,6 +209,7 @@
 
 #ifdef HAVE_SETPPRIV
 
+#include <stdio.h>	// ARG_ERR
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
@@ -221,46 +222,35 @@
 #include <priv.h>
 #endif
 
-/* ============================================================
- * the real thing
- */
+/* renamed from sys/priv_const.h */
+#define VJS_EFFECTIVE		0
+#define VJS_INHERITABLE		1
+#define VJS_PERMITTED		2
+#define VJS_LIMIT		3
 
-// XXX @phk can we merge jail_subproc_e and jail_master_e please?
+#define VJS_NSET		(VJS_LIMIT + 1)
 
-#define JAILG_SHIFT 16
+#define VJS_MASK(x) (1U << (x))
 
-enum jail_gen_e {
-	JAILG_SUBPROC_VCC = JAIL_SUBPROC_VCC,
-	JAILG_SUBPROC_CC = JAIL_SUBPROC_CC,
-	JAILG_SUBPROC_VCLLOAD = JAIL_SUBPROC_VCLLOAD,
-	JAILG_SUBPROC_WORKER = JAIL_SUBPROC_WORKER,
+/* to denote sharing */
+#define JAIL_MASTER_ANY 0
 
-	JAILG_MASTER_LOW = JAIL_MASTER_LOW << JAILG_SHIFT,
-	JAILG_MASTER_STORAGE = JAIL_MASTER_STORAGE << JAILG_SHIFT,
-	JAILG_MASTER_PRIVPORT = JAIL_MASTER_PRIVPORT << JAILG_SHIFT
+const priv_ptype_t vjs_ptype[VJS_NSET] = {
+	[VJS_EFFECTIVE]		= PRIV_EFFECTIVE,
+	[VJS_INHERITABLE]	= PRIV_INHERITABLE,
+	[VJS_PERMITTED]		= PRIV_PERMITTED,
+	[VJS_LIMIT]		= PRIV_LIMIT
 };
 
-static inline enum jail_gen_e
-jail_subproc_gen(enum jail_subproc_e e)
-{
-	assert(e < (1 << JAILG_SHIFT));
-	return ((enum jail_gen_e)e);
-}
+static priv_set_t *vjs_sets[JAIL_LIMIT][VJS_NSET];
+static priv_set_t *vjs_inverse[JAIL_LIMIT][VJS_NSET];
+static priv_set_t *vjs_proc_setid;	// for vjs_setuid
 
-static inline enum jail_gen_e
-jail_master_gen(enum jail_master_e e)
-{
-	return ((enum jail_gen_e)(e << JAILG_SHIFT));
-}
+static void v_matchproto_(jail_master_f)
+	vjs_master(enum jail_master_e jme);
 
-static int v_matchproto_(jail_init_f)
-vjs_init(char **args)
-{
-	(void)args;
-	return 0;
-}
+/*------------------------------------------------------------*/
 
-/* for priv_delset() and priv_addset() */
 static inline int
 priv_setop_check(int a)
 {
@@ -273,13 +263,6 @@ priv_setop_check(int a)
 
 #define priv_setop_assert(a) assert(priv_setop_check(a))
 
-/*
- * we try to add all possible privileges to waive them later.
- *
- * when doing so, we need to expect EPERM
- */
-
-/* for setppriv */
 static inline int
 setppriv_check(int a)
 {
@@ -292,139 +275,163 @@ setppriv_check(int a)
 
 #define setppriv_assert(a) assert(setppriv_check(a))
 
-static void
-vjs_add_inheritable(priv_set_t *pset, enum jail_gen_e jge)
-{
-	switch (jge) {
-	case JAILG_SUBPROC_VCC:
-		break;
-	case JAILG_SUBPROC_CC:
-		priv_setop_assert(priv_addset(pset, PRIV_PROC_EXEC));
-		priv_setop_assert(priv_addset(pset, PRIV_PROC_FORK));
-		priv_setop_assert(priv_addset(pset, "file_read"));
-		priv_setop_assert(priv_addset(pset, "file_write"));
-		break;
-	case JAILG_SUBPROC_VCLLOAD:
-		break;
-	case JAILG_SUBPROC_WORKER:
-		break;
-	default:
-		INCOMPL();
-	}
-}
+/* ------------------------------------------------------------
+ * initialization of privilege sets from mgt_jail_solaris_tbl.h
+ * and implicit rules documented therein
+ */
 
-static void
-vjs_add_effective(priv_set_t *pset, enum jail_gen_e jge)
+static inline void
+vjs_add(priv_set_t *sets[VJS_NSET], unsigned mask, const char *priv)
 {
-	switch (jge) {
-	case JAILG_SUBPROC_VCC:
-		// open vmods
-		priv_setop_assert(priv_addset(pset, "file_read"));
-		// write .c output
-		priv_setop_assert(priv_addset(pset, "file_write"));
-		break;
-	case JAILG_SUBPROC_CC:
-		priv_setop_assert(priv_addset(pset, PRIV_PROC_EXEC));
-		priv_setop_assert(priv_addset(pset, PRIV_PROC_FORK));
-		priv_setop_assert(priv_addset(pset, "file_read"));
-		priv_setop_assert(priv_addset(pset, "file_write"));
-		break;
-	case JAILG_SUBPROC_VCLLOAD:
-		priv_setop_assert(priv_addset(pset, "file_read"));
-		break;
-	case JAILG_SUBPROC_WORKER:
-		priv_setop_assert(priv_addset(pset, "net_access"));
-		priv_setop_assert(priv_addset(pset, "file_read"));
-		priv_setop_assert(priv_addset(pset, "file_write"));
-		break;
-	default:
-		INCOMPL();
-	}
+	int i;
+	for (i = 0; i < VJS_NSET; i++)
+		if (mask & VJS_MASK(i))
+			priv_setop_assert(priv_addset(sets[i], priv));
 }
 
 /*
- * permitted is initialized from effective (see vjs_waive)
- * so only additionally required privileges need to be added here
+ * we reduce the limit set to the union of all jail level limit sets: first we
+ * try to enable all privileges which we possibly need, then we waive the
+ * inverse in vjs_init()
  */
 
-static void
-vjs_add_permitted(priv_set_t *pset, enum jail_gen_e jge)
+static int
+vjs_master_rules(void)
 {
-	(void) pset;
-	switch (jge) {
-	case JAILG_SUBPROC_VCC:
-	case JAILG_SUBPROC_CC:
-	case JAILG_SUBPROC_VCLLOAD:
-		break;
-	case JAILG_SUBPROC_WORKER:
-		/* vmod_unix getpeerucred() */
-		AZ(priv_addset(pset, PRIV_PROC_INFO));
-		break;
-	default:
-		INCOMPL();
+	priv_set_t *punion = priv_allocset();
+	int vs, vj;
+
+	AN(punion);
+
+	for (vs = VJS_INHERITABLE; vs <= VJS_PERMITTED; vs ++) {
+		priv_emptyset(punion);
+		for (vj = JAIL_SUBPROC; vj < JAIL_LIMIT; vj++)
+			priv_union(vjs_sets[vj][vs], punion);
+		priv_union(punion, vjs_sets[JAIL_MASTER_ANY][vs]);
 	}
+
+	priv_freeset(punion);
+
+	return (0);
 }
 
-/*
- * additional privileges needed by vjs_privsep -
- * will get waived in vjs_waive
- */
-static void
-vjs_add_initial(priv_set_t *pset, enum jail_gen_e jge)
+static priv_set_t *
+vjs_alloc(void)
 {
-	(void)jge;
+	priv_set_t *s;
 
-	/* for setgid/setuid */
-	AZ(priv_addset(pset, PRIV_PROC_SETID));
+	s = priv_allocset();
+	AN(s);
+	priv_emptyset(s);
+	return (s);
 }
 
-/*
- * if we are not yet privilege-aware already (ie we have been started
- * not-privilege aware with euid 0), we try to grab any privileges we
- * will need later.
- * We will reduce to least privileges in vjs_waive
- *
- * We need to become privilege-aware to avoid setuid resetting them.
- */
-
-static void
-vjs_setup(enum jail_gen_e jge)
+static int v_matchproto_(jail_init_f)
+vjs_init(char **args)
 {
-	priv_set_t *priv_all;
+	priv_set_t **sets;
+	int vj, vs;
 
-	if (!(priv_all = priv_allocset())) {
-		MGT_Complain(C_SECURITY,
-		    "Solaris Jail warning: "
-		    " vjs_setup - priv_allocset failed: errno=%d (%s)",
-		    errno, vstrerror(errno));
-		return;
+	if (args != NULL && *args != NULL) {
+		ARGV_ERR("-jsolaris takes no arguments.\n");
+		return (0);
 	}
 
-	priv_emptyset(priv_all);
+	/* init privset for vjs_setuid() */
+	vjs_proc_setid = priv_allocset();
+	AN(vjs_proc_setid);
+	priv_emptyset(vjs_proc_setid);
+	priv_setop_assert(priv_addset(vjs_proc_setid, PRIV_PROC_SETID));
 
-	vjs_add_inheritable(priv_all, jge);
-	vjs_add_effective(priv_all, jge);
-	vjs_add_permitted(priv_all, jge);
-	vjs_add_initial(priv_all, jge);
+	assert(JAIL_MASTER_ANY < JAIL_SUBPROC);
+	/* alloc privsets.
+	 * for master, anything but EFFECTIVE is shared
+	 */
+	for (vj = 0; vj < JAIL_SUBPROC; vj++)
+		for (vs = 0; vs < VJS_NSET; vs++) {
+			if (vj == JAIL_MASTER_ANY || vs == VJS_EFFECTIVE) {
+				vjs_sets[vj][vs] = vjs_alloc();
+				vjs_inverse[vj][vs] = vjs_alloc();
+			} else {
+				vjs_sets[vj][vs] =
+					vjs_sets[JAIL_MASTER_ANY][vs];
+				vjs_inverse[vj][vs] =
+					vjs_inverse[JAIL_MASTER_ANY][vs];
+			}
+		}
+
+	for (; vj < JAIL_LIMIT; vj++)
+		for (vs = 0; vs < VJS_NSET; vs++) {
+			vjs_sets[vj][vs] = vjs_alloc();
+			vjs_inverse[vj][vs] = vjs_alloc();
+		}
+
+	/* init from table */
+#define PRIV(name, mask, priv) vjs_add(vjs_sets[JAIL_ ## name], mask, priv);
+#include "mgt_jail_solaris_tbl.h"
+
+	/* SUBPROC implicit rules */
+	for (vj = JAIL_SUBPROC; vj < JAIL_LIMIT; vj++) {
+		sets = vjs_sets[vj];
+		priv_union(sets[VJS_EFFECTIVE], sets[VJS_PERMITTED]);
+		priv_union(sets[VJS_PERMITTED], sets[VJS_LIMIT]);
+		priv_union(sets[VJS_INHERITABLE], sets[VJS_LIMIT]);
+	}
+
+	vjs_master_rules();
+
+	/* MASTER implicit rules */
+	for (vj = 0; vj < JAIL_SUBPROC; vj++) {
+		sets = vjs_sets[vj];
+		priv_union(sets[VJS_EFFECTIVE], sets[VJS_PERMITTED]);
+		priv_union(sets[VJS_PERMITTED], sets[VJS_LIMIT]);
+		priv_union(sets[VJS_INHERITABLE], sets[VJS_LIMIT]);
+	}
+
+	/* attempt to enable privileges */
+	for (vs = VJS_PERMITTED; vs > VJS_EFFECTIVE; vs--)
+		setppriv_assert(setppriv(PRIV_ON, vjs_ptype[vs],
+		    vjs_sets[JAIL_MASTER_ANY][vs]));
 
-	/* try to get all possible privileges, expect EPERM here */
-	setppriv_assert(setppriv(PRIV_ON, PRIV_PERMITTED, priv_all));
-	setppriv_assert(setppriv(PRIV_ON, PRIV_EFFECTIVE, priv_all));
-	setppriv_assert(setppriv(PRIV_ON, PRIV_INHERITABLE, priv_all));
+	/* generate inverse */
+	for (vj = 0; vj < JAIL_LIMIT; vj++)
+		for (vs = 0; vs < VJS_NSET; vs++) {
+			priv_copyset(vjs_sets[vj][vs], vjs_inverse[vj][vs]);
+			priv_inverse(vjs_inverse[vj][vs]);
+		}
 
-	priv_freeset(priv_all);
+	vjs_master(JAIL_MASTER_LOW);
+
+	/* XXX LEAK: no _fini for priv_freeset() */
+	return (0);
 }
 
 static void
-vjs_privsep(enum jail_gen_e jge)
+vjs_waive(int jail)
 {
-	(void)jge;
+	priv_set_t **sets;
+	int i;
+
+	assert(jail >= 0);
+	assert(jail < JAIL_LIMIT);
 
+	sets = vjs_inverse[jail];
+
+	for (i = 0; i < VJS_NSET; i++)
+		AZ(setppriv(PRIV_OFF, vjs_ptype[i], sets[i]));
+}
+
+static void
+vjs_setuid(void)
+{
+	setppriv_assert(setppriv(PRIV_ON, PRIV_EFFECTIVE, vjs_proc_setid));
 	if (priv_ineffect(PRIV_PROC_SETID)) {
 		if (getgid() != mgt_param.gid)
 			XXXAZ(setgid(mgt_param.gid));
 		if (getuid() != mgt_param.uid)
 			XXXAZ(setuid(mgt_param.uid));
+		AZ(setppriv(PRIV_OFF, PRIV_EFFECTIVE, vjs_proc_setid));
+		AZ(setppriv(PRIV_OFF, PRIV_PERMITTED, vjs_proc_setid));
 	} else {
 		MGT_Complain(C_SECURITY,
 		    "Privilege %s missing, will not change uid/gid",
@@ -432,95 +439,27 @@ vjs_privsep(enum jail_gen_e jge)
 	}
 }
 
-/*
- * Waive most privileges in the child
- *
- * as of onnv_151a, we should end up with:
- *
- * > ppriv -v #pid of varnish child
- * PID:  .../varnishd ...
- * flags = PRIV_AWARE
- *      E: file_read,file_write,net_access
- *      I: none
- *      P: file_read,file_write,net_access,sys_resource
- *      L: file_read,file_write,net_access,sys_resource
- *
- * We should keep sys_resource in P in order to adjust our limits if we need to
- */
-
-static void
-vjs_waive(enum jail_gen_e jge)
-{
-	priv_set_t *effective, *inheritable, *permitted, *limited;
-
-	if (!(effective = priv_allocset()) ||
-	    !(inheritable = priv_allocset()) ||
-	    !(permitted = priv_allocset()) ||
-	    !(limited = priv_allocset())) {
-		MGT_Complain(C_SECURITY,
-		    "Solaris Jail warning: "
-		    " vjs_waive - priv_allocset failed: errno=%d (%s)",
-		    errno, vstrerror(errno));
-		return;
-	}
-
-	/*
-	 * inheritable and effective are distinct sets
-	 * effective is a subset of permitted
-	 * limit is the union of all
-	 */
-
-	priv_emptyset(inheritable);
-	vjs_add_inheritable(inheritable, jge);
-
-	priv_emptyset(effective);
-	vjs_add_effective(effective, jge);
-
-	priv_copyset(effective, permitted);
-	vjs_add_permitted(permitted, jge);
-
-	priv_copyset(inheritable, limited);
-	priv_union(permitted, limited);
-	/*
-	 * invert the sets and clear privileges such that setppriv will always
-	 * succeed
-	 */
-	priv_inverse(limited);
-	priv_inverse(permitted);
-	priv_inverse(effective);
-	priv_inverse(inheritable);
-
-	AZ(setppriv(PRIV_OFF, PRIV_LIMIT, limited));
-	AZ(setppriv(PRIV_OFF, PRIV_PERMITTED, permitted));
-	AZ(setppriv(PRIV_OFF, PRIV_EFFECTIVE, effective));
-	AZ(setppriv(PRIV_OFF, PRIV_INHERITABLE, inheritable));
-
-	priv_freeset(limited);
-	priv_freeset(permitted);
-	priv_freeset(effective);
-	priv_freeset(inheritable);
-}
-
 static void v_matchproto_(jail_subproc_f)
 vjs_subproc(enum jail_subproc_e jse)
 {
-	enum jail_gen_e jge = jail_subproc_gen(jse);
-	vjs_setup(jge);
-	vjs_privsep(jge);
-	vjs_waive(jge);
+	vjs_setuid();
+	vjs_waive(jse);
 }
 
 static void v_matchproto_(jail_master_f)
 vjs_master(enum jail_master_e jme)
 {
-	enum jail_gen_e jge = jail_master_gen(jme);
-	(void)jge;
-/*
-	if (jme == JAILG_MASTER_HIGH)
-		AZ(seteuid(0));
-	else
-		AZ(seteuid(vju_uid));
-*/
+	priv_set_t **sets;
+	int i;
+
+	assert(jme < JAIL_SUBPROC);
+
+	sets = vjs_sets[jme];
+
+	i = VJS_EFFECTIVE;
+	setppriv_assert(setppriv(PRIV_ON, vjs_ptype[i], sets[i]));
+
+	vjs_waive(jme);
 }
 
 const struct jail_tech jail_tech_solaris = {
diff --git a/bin/varnishd/mgt/mgt_jail_solaris_tbl.h b/bin/varnishd/mgt/mgt_jail_solaris_tbl.h
new file mode 100644
index 000000000..dfe912094
--- /dev/null
+++ b/bin/varnishd/mgt/mgt_jail_solaris_tbl.h
@@ -0,0 +1,93 @@
+/*-
+ * Copyright 2020 UPLEX - Nils Goroll Systemoptimierung
+ * All rights reserved.
+ *
+ * Author: Nils Goroll <nils.goroll at uplex.de>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * definition of privileges to use for the different Varnish Jail (VJ)
+ * levels
+ */
+#define E VJS_MASK(VJS_EFFECTIVE)	// as-is
+#define I VJS_MASK(VJS_INHERITABLE)	// as-is
+#define P VJS_MASK(VJS_PERMITTED)	// joined with effective
+#define L VJS_MASK(VJS_LIMIT)		// joined with of all the above
+
+/* ------------------------------------------------------------
+ * MASTER
+ * - only MASTER_EFFECTIVE is per JAIL state
+ * - other priv sets are shared across all MASTER_* JAIL states
+ *
+ * MASTER implicit rules (vjs_master_rules())
+ * - INHERITABLE and PERMITTED joined from SUBPROC*
+ * - implicit rules from above
+ */
+PRIV(MASTER_LOW,	E	, PRIV_PROC_EXEC)	// XXX fork
+PRIV(MASTER_LOW,	E	, PRIV_PROC_FORK)	// XXX fork
+PRIV(MASTER_LOW,	E	, "file_write")	// XXX vcl_boot
+PRIV(MASTER_LOW,	E	, "file_read")	// XXX library open
+PRIV(MASTER_LOW,	E	, "net_access")
+
+PRIV(MASTER_FILE,	E	, PRIV_PROC_EXEC)	// XXX rm -rf in shm
+PRIV(MASTER_FILE,	E	, PRIV_PROC_FORK)	// XXX rm -rf in shm
+PRIV(MASTER_FILE,	E	, "file_read")
+PRIV(MASTER_FILE,	E	, "file_write")
+
+PRIV(MASTER_STORAGE,	E	, "file_read")
+PRIV(MASTER_STORAGE,	E	, "file_write")
+
+PRIV(MASTER_PRIVPORT,	E	, "file_write")	// bind(AF_UNIX)
+PRIV(MASTER_PRIVPORT,	E	, "net_access")
+PRIV(MASTER_PRIVPORT,	E	, PRIV_NET_PRIVADDR)
+
+PRIV(MASTER_KILL,	E	, PRIV_PROC_OWNER)
+
+/* ------------------------------------------------------------
+ * SUBPROC
+ */
+PRIV(SUBPROC_VCC,	E	, PRIV_PROC_SETID)	// waived after setuid
+PRIV(SUBPROC_VCC,	E	, "file_read")
+PRIV(SUBPROC_VCC,	E	, "file_write")
+
+PRIV(SUBPROC_CC,	E	, PRIV_PROC_SETID)	// waived after setuid
+PRIV(SUBPROC_CC,	E|I	, PRIV_PROC_EXEC)
+PRIV(SUBPROC_CC,	E|I	, PRIV_PROC_FORK)
+PRIV(SUBPROC_CC,	E|I	, "file_read")
+PRIV(SUBPROC_CC,	E|I	, "file_write")
+
+PRIV(SUBPROC_VCLLOAD,	E	, PRIV_PROC_SETID)	// waived after setuid
+PRIV(SUBPROC_VCLLOAD,	E	, "file_read")
+
+PRIV(SUBPROC_WORKER,	E	, PRIV_PROC_SETID)	// waived after setuid
+PRIV(SUBPROC_WORKER,	E	, "net_access")
+PRIV(SUBPROC_WORKER,	E	, "file_read")
+PRIV(SUBPROC_WORKER,	E	, "file_write")
+PRIV(SUBPROC_WORKER,	P	, PRIV_PROC_INFO)	/* vmod_unix */
+
+#undef E
+#undef I
+#undef P
+#undef L
+#undef PRIV

From dridi.boukelmoune at gmail.com  Tue Jun  2 11:40:07 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue,  2 Jun 2020 11:40:07 +0000 (UTC)
Subject: [master] 0786c5cd4 Revert temporary debugging in u0.vtc
Message-ID: <20200602114007.6E21864DFC@lists.varnish-cache.org>


commit 0786c5cd4f12b4909b4ed25eb2ac7ee682e3741e
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jun 2 13:27:44 2020 +0200

    Revert temporary debugging in u0.vtc
    
    Reverts:
            8eb42d2cc91179ac56d428d6f1b96fe25d8b3135
            f43679b69c9dad8258b7a3f5e5ac58b30fbee042
            6f1563cf9b301677f2b2934d7ef97419cd74a7be

diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc
index 563caca8a..f7c13fb27 100644
--- a/bin/varnishtest/tests/u00000.vtc
+++ b/bin/varnishtest/tests/u00000.vtc
@@ -4,16 +4,11 @@ shell "varnishd -b 127.0.0.1:80 -C 2> ${tmpdir}/_.c"
 
 shell {
 	varnishd -n ${tmpdir}/no_keep -C -b 127.0.0.1:80 2> no_keep.c
-	ls -laR
-	set -x
-	tail no_keep.c || true
 	test -s no_keep.c && ! test -d no_keep
 }
 
 shell {
 	varnishd -n ${tmpdir}/keep -p debug=+vcl_keep -C -b 127.0.0.1:80 2> keep.c
-	ls -l
-	set -x
 	test -s keep.c && test -d keep
 }
 

From dridi.boukelmoune at gmail.com  Tue Jun  2 11:40:07 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue,  2 Jun 2020 11:40:07 +0000 (UTC)
Subject: [master] 9a14e68e9 Tolerate GCOV droppings in u0.vtc
Message-ID: <20200602114007.8CA7564DFF@lists.varnish-cache.org>


commit 9a14e68e91ad59bb67839f87469fa8166c91612c
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jun 2 13:33:52 2020 +0200

    Tolerate GCOV droppings in u0.vtc

diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc
index f7c13fb27..e5797fe8c 100644
--- a/bin/varnishtest/tests/u00000.vtc
+++ b/bin/varnishtest/tests/u00000.vtc
@@ -4,7 +4,7 @@ shell "varnishd -b 127.0.0.1:80 -C 2> ${tmpdir}/_.c"
 
 shell {
 	varnishd -n ${tmpdir}/no_keep -C -b 127.0.0.1:80 2> no_keep.c
-	test -s no_keep.c && ! test -d no_keep
+	test -s no_keep.c && ! test -d no_keep || test -f no_keep/vgc.gcda
 }
 
 shell {

From nils.goroll at uplex.de  Tue Jun  2 12:23:06 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  2 Jun 2020 12:23:06 +0000 (UTC)
Subject: [master] f77eb3b3c fix comment
Message-ID: <20200602122306.EE4336E2FA@lists.varnish-cache.org>


commit f77eb3b3c12615b80e0e1b73f71f11250f86fb41
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 2 13:26:21 2020 +0200

    fix comment

diff --git a/bin/varnishd/mgt/mgt_jail_solaris.c b/bin/varnishd/mgt/mgt_jail_solaris.c
index ef2b5423a..6b303df1c 100644
--- a/bin/varnishd/mgt/mgt_jail_solaris.c
+++ b/bin/varnishd/mgt/mgt_jail_solaris.c
@@ -289,12 +289,7 @@ vjs_add(priv_set_t *sets[VJS_NSET], unsigned mask, const char *priv)
 			priv_setop_assert(priv_addset(sets[i], priv));
 }
 
-/*
- * we reduce the limit set to the union of all jail level limit sets: first we
- * try to enable all privileges which we possibly need, then we waive the
- * inverse in vjs_init()
- */
-
+/* add SUBPROC INHERITABLE and PERMITTED to MASTER */
 static int
 vjs_master_rules(void)
 {

From nils.goroll at uplex.de  Tue Jun  2 12:23:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  2 Jun 2020 12:23:07 +0000 (UTC)
Subject: [master] 6df5b1bbd Solaris jail: mask available privileges
Message-ID: <20200602122307.0E6566E2FE@lists.varnish-cache.org>


commit 6df5b1bbdca28e0a6c3fef956221e3d826229193
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 2 12:34:05 2020 +0200

    Solaris jail: mask available privileges
    
    Avoid setppriv() tolerating EPERM by masking privileges with the
    available upper bound.

diff --git a/bin/varnishd/mgt/mgt_jail_solaris.c b/bin/varnishd/mgt/mgt_jail_solaris.c
index 6b303df1c..34a457bbc 100644
--- a/bin/varnishd/mgt/mgt_jail_solaris.c
+++ b/bin/varnishd/mgt/mgt_jail_solaris.c
@@ -263,18 +263,6 @@ priv_setop_check(int a)
 
 #define priv_setop_assert(a) assert(priv_setop_check(a))
 
-static inline int
-setppriv_check(int a)
-{
-	if (a == 0)
-		return (1);
-	if (errno == EPERM)
-		return (1);
-	return (0);
-}
-
-#define setppriv_assert(a) assert(setppriv_check(a))
-
 /* ------------------------------------------------------------
  * initialization of privilege sets from mgt_jail_solaris_tbl.h
  * and implicit rules documented therein
@@ -324,7 +312,7 @@ vjs_alloc(void)
 static int v_matchproto_(jail_init_f)
 vjs_init(char **args)
 {
-	priv_set_t **sets;
+	priv_set_t **sets, *permitted, *inheritable;
 	int vj, vs;
 
 	if (args != NULL && *args != NULL) {
@@ -332,6 +320,15 @@ vjs_init(char **args)
 		return (0);
 	}
 
+	permitted = vjs_alloc();
+	AN(permitted);
+	AZ(getppriv(PRIV_PERMITTED, permitted));
+
+	inheritable = vjs_alloc();
+	AN(inheritable);
+	AZ(getppriv(PRIV_INHERITABLE, inheritable));
+	priv_union(permitted, inheritable);
+
 	/* init privset for vjs_setuid() */
 	vjs_proc_setid = priv_allocset();
 	AN(vjs_proc_setid);
@@ -365,6 +362,14 @@ vjs_init(char **args)
 #define PRIV(name, mask, priv) vjs_add(vjs_sets[JAIL_ ## name], mask, priv);
 #include "mgt_jail_solaris_tbl.h"
 
+	/* mask by available privs */
+	for (vj = 0; vj < JAIL_LIMIT; vj++) {
+		sets = vjs_sets[vj];
+		priv_intersect(permitted, sets[VJS_EFFECTIVE]);
+		priv_intersect(permitted, sets[VJS_PERMITTED]);
+		priv_intersect(inheritable, sets[VJS_INHERITABLE]);
+	}
+
 	/* SUBPROC implicit rules */
 	for (vj = JAIL_SUBPROC; vj < JAIL_LIMIT; vj++) {
 		sets = vjs_sets[vj];
@@ -383,10 +388,9 @@ vjs_init(char **args)
 		priv_union(sets[VJS_INHERITABLE], sets[VJS_LIMIT]);
 	}
 
-	/* attempt to enable privileges */
-	for (vs = VJS_PERMITTED; vs > VJS_EFFECTIVE; vs--)
-		setppriv_assert(setppriv(PRIV_ON, vjs_ptype[vs],
-		    vjs_sets[JAIL_MASTER_ANY][vs]));
+	/* extend inheritable */
+	vs = VJS_INHERITABLE;
+	AZ(setppriv(PRIV_ON, vjs_ptype[vs], vjs_sets[JAIL_MASTER_ANY][vs]));
 
 	/* generate inverse */
 	for (vj = 0; vj < JAIL_LIMIT; vj++)
@@ -397,6 +401,8 @@ vjs_init(char **args)
 
 	vjs_master(JAIL_MASTER_LOW);
 
+	priv_freeset(permitted);
+	priv_freeset(inheritable);
 	/* XXX LEAK: no _fini for priv_freeset() */
 	return (0);
 }
@@ -419,7 +425,6 @@ vjs_waive(int jail)
 static void
 vjs_setuid(void)
 {
-	setppriv_assert(setppriv(PRIV_ON, PRIV_EFFECTIVE, vjs_proc_setid));
 	if (priv_ineffect(PRIV_PROC_SETID)) {
 		if (getgid() != mgt_param.gid)
 			XXXAZ(setgid(mgt_param.gid));
@@ -437,6 +442,14 @@ vjs_setuid(void)
 static void v_matchproto_(jail_subproc_f)
 vjs_subproc(enum jail_subproc_e jse)
 {
+	priv_set_t **sets;
+	int i;
+
+	sets = vjs_sets[jse];
+
+	i = VJS_EFFECTIVE;
+	AZ(setppriv(PRIV_ON, vjs_ptype[i], sets[i]));
+
 	vjs_setuid();
 	vjs_waive(jse);
 }
@@ -452,7 +465,7 @@ vjs_master(enum jail_master_e jme)
 	sets = vjs_sets[jme];
 
 	i = VJS_EFFECTIVE;
-	setppriv_assert(setppriv(PRIV_ON, vjs_ptype[i], sets[i]));
+	AZ(setppriv(PRIV_ON, vjs_ptype[i], sets[i]));
 
 	vjs_waive(jme);
 }

From nils.goroll at uplex.de  Tue Jun  2 12:23:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  2 Jun 2020 12:23:07 +0000 (UTC)
Subject: [master] 883ad98f1 polish
Message-ID: <20200602122307.2835A6E301@lists.varnish-cache.org>


commit 883ad98f1ac313037929c8833cd5b87fea1e74a0
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 2 12:36:18 2020 +0200

    polish

diff --git a/bin/varnishd/mgt/mgt_jail_solaris.c b/bin/varnishd/mgt/mgt_jail_solaris.c
index 34a457bbc..437021215 100644
--- a/bin/varnishd/mgt/mgt_jail_solaris.c
+++ b/bin/varnishd/mgt/mgt_jail_solaris.c
@@ -330,9 +330,8 @@ vjs_init(char **args)
 	priv_union(permitted, inheritable);
 
 	/* init privset for vjs_setuid() */
-	vjs_proc_setid = priv_allocset();
+	vjs_proc_setid = vjs_alloc();
 	AN(vjs_proc_setid);
-	priv_emptyset(vjs_proc_setid);
 	priv_setop_assert(priv_addset(vjs_proc_setid, PRIV_PROC_SETID));
 
 	assert(JAIL_MASTER_ANY < JAIL_SUBPROC);

From nils.goroll at uplex.de  Tue Jun  2 12:23:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  2 Jun 2020 12:23:07 +0000 (UTC)
Subject: [master] 7a895f7fb Solaris jail: wrap setppriv(PRIV_ON, ...)
Message-ID: <20200602122307.471756E306@lists.varnish-cache.org>


commit 7a895f7fbb40d7bb14164cfb92d74ddee075c28d
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 2 12:52:52 2020 +0200

    Solaris jail: wrap setppriv(PRIV_ON, ...)

diff --git a/bin/varnishd/mgt/mgt_jail_solaris.c b/bin/varnishd/mgt/mgt_jail_solaris.c
index 437021215..ec3e788b9 100644
--- a/bin/varnishd/mgt/mgt_jail_solaris.c
+++ b/bin/varnishd/mgt/mgt_jail_solaris.c
@@ -263,6 +263,17 @@ priv_setop_check(int a)
 
 #define priv_setop_assert(a) assert(priv_setop_check(a))
 
+/*------------------------------------------------------------*/
+
+static int
+vjs_priv_on(int vs, priv_set_t **set)
+{
+	assert(vs >= 0);
+	assert(vs < VJS_NSET);
+
+	return (setppriv(PRIV_ON, vjs_ptype[vs], set[vs]));
+}
+
 /* ------------------------------------------------------------
  * initialization of privilege sets from mgt_jail_solaris_tbl.h
  * and implicit rules documented therein
@@ -388,8 +399,7 @@ vjs_init(char **args)
 	}
 
 	/* extend inheritable */
-	vs = VJS_INHERITABLE;
-	AZ(setppriv(PRIV_ON, vjs_ptype[vs], vjs_sets[JAIL_MASTER_ANY][vs]));
+	AZ(vjs_priv_on(VJS_INHERITABLE, vjs_sets[JAIL_MASTER_ANY]));
 
 	/* generate inverse */
 	for (vj = 0; vj < JAIL_LIMIT; vj++)
@@ -441,13 +451,8 @@ vjs_setuid(void)
 static void v_matchproto_(jail_subproc_f)
 vjs_subproc(enum jail_subproc_e jse)
 {
-	priv_set_t **sets;
-	int i;
-
-	sets = vjs_sets[jse];
 
-	i = VJS_EFFECTIVE;
-	AZ(setppriv(PRIV_ON, vjs_ptype[i], sets[i]));
+	AZ(vjs_priv_on(VJS_EFFECTIVE, vjs_sets[jse]));
 
 	vjs_setuid();
 	vjs_waive(jse);
@@ -456,15 +461,10 @@ vjs_subproc(enum jail_subproc_e jse)
 static void v_matchproto_(jail_master_f)
 vjs_master(enum jail_master_e jme)
 {
-	priv_set_t **sets;
-	int i;
 
 	assert(jme < JAIL_SUBPROC);
 
-	sets = vjs_sets[jme];
-
-	i = VJS_EFFECTIVE;
-	AZ(setppriv(PRIV_ON, vjs_ptype[i], sets[i]));
+	AZ(vjs_priv_on(VJS_EFFECTIVE, vjs_sets[jme]));
 
 	vjs_waive(jme);
 }

From nils.goroll at uplex.de  Tue Jun  2 12:23:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  2 Jun 2020 12:23:07 +0000 (UTC)
Subject: [master] d50da8306 Solaris jail: manage INHERITABLE for JAIL_MASTER
Message-ID: <20200602122307.6FC896E30A@lists.varnish-cache.org>


commit d50da8306fcce5ec5cffdc525aae47698f6f3345
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 2 13:33:33 2020 +0200

    Solaris jail: manage INHERITABLE for JAIL_MASTER
    
    we now dynamically manage the INHERITABLE set also, which has the
    advantage of reducing the privileges available to anything we exec()
    (likely via system()) from master which is not managed through
    JAIL_SUBPROC.
    
    See next commit.

diff --git a/bin/varnishd/mgt/mgt_jail_solaris.c b/bin/varnishd/mgt/mgt_jail_solaris.c
index ec3e788b9..3a50b572e 100644
--- a/bin/varnishd/mgt/mgt_jail_solaris.c
+++ b/bin/varnishd/mgt/mgt_jail_solaris.c
@@ -288,7 +288,7 @@ vjs_add(priv_set_t *sets[VJS_NSET], unsigned mask, const char *priv)
 			priv_setop_assert(priv_addset(sets[i], priv));
 }
 
-/* add SUBPROC INHERITABLE and PERMITTED to MASTER */
+/* add SUBPROC INHERITABLE and PERMITTED to MASTER PERMITTED */
 static int
 vjs_master_rules(void)
 {
@@ -301,7 +301,7 @@ vjs_master_rules(void)
 		priv_emptyset(punion);
 		for (vj = JAIL_SUBPROC; vj < JAIL_LIMIT; vj++)
 			priv_union(vjs_sets[vj][vs], punion);
-		priv_union(punion, vjs_sets[JAIL_MASTER_ANY][vs]);
+		priv_union(punion, vjs_sets[JAIL_MASTER_ANY][VJS_PERMITTED]);
 	}
 
 	priv_freeset(punion);
@@ -347,11 +347,11 @@ vjs_init(char **args)
 
 	assert(JAIL_MASTER_ANY < JAIL_SUBPROC);
 	/* alloc privsets.
-	 * for master, anything but EFFECTIVE is shared
+	 * for master, PERMITTED and LIMIT are shared
 	 */
 	for (vj = 0; vj < JAIL_SUBPROC; vj++)
 		for (vs = 0; vs < VJS_NSET; vs++) {
-			if (vj == JAIL_MASTER_ANY || vs == VJS_EFFECTIVE) {
+			if (vj == JAIL_MASTER_ANY || vs < VJS_PERMITTED) {
 				vjs_sets[vj][vs] = vjs_alloc();
 				vjs_inverse[vj][vs] = vjs_alloc();
 			} else {
@@ -398,9 +398,6 @@ vjs_init(char **args)
 		priv_union(sets[VJS_INHERITABLE], sets[VJS_LIMIT]);
 	}
 
-	/* extend inheritable */
-	AZ(vjs_priv_on(VJS_INHERITABLE, vjs_sets[JAIL_MASTER_ANY]));
-
 	/* generate inverse */
 	for (vj = 0; vj < JAIL_LIMIT; vj++)
 		for (vs = 0; vs < VJS_NSET; vs++) {
@@ -453,6 +450,7 @@ vjs_subproc(enum jail_subproc_e jse)
 {
 
 	AZ(vjs_priv_on(VJS_EFFECTIVE, vjs_sets[jse]));
+	AZ(vjs_priv_on(VJS_INHERITABLE, vjs_sets[jse]));
 
 	vjs_setuid();
 	vjs_waive(jse);
@@ -465,6 +463,7 @@ vjs_master(enum jail_master_e jme)
 	assert(jme < JAIL_SUBPROC);
 
 	AZ(vjs_priv_on(VJS_EFFECTIVE, vjs_sets[jme]));
+	AZ(vjs_priv_on(VJS_INHERITABLE, vjs_sets[jme]));
 
 	vjs_waive(jme);
 }

From nils.goroll at uplex.de  Tue Jun  2 12:23:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  2 Jun 2020 12:23:07 +0000 (UTC)
Subject: [master] 900e9f393 add JAIL_MASTER_SYSTEM for system() calls from
 master
Message-ID: <20200602122307.956806E30F@lists.varnish-cache.org>


commit 900e9f39371639582d2f08ccc2cc9a9fbc5b70ae
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 2 12:36:52 2020 +0200

    add JAIL_MASTER_SYSTEM for system() calls from master
    
    Also (re)used to make fork privileges available when we start a
    subprocess: As we are going to apply the JAIL_SUBPROC privileges to the
    forked process, having slightly eleveated privileges only agross the
    fork() should not cause any harm.
    
            -
    
    This concludes the current series of Solaris jail patches, hopefully.
    With this commit, varnishd started with pfexec ("root privileges") keeps
    the following privileges only (ppriv -v output) on Solaris:
    
    * master::
    
      flags = PRIV_AWARE
            E: file_read,file_write,net_access
            I: none
            P: file_read,file_write,net_access,net_privaddr,proc_exec,proc_fork,proc_info,proc_owner,proc_setid
            L: file_read,file_write,net_access,net_privaddr,proc_exec,proc_fork,proc_info,proc_owner,proc_setid
    
      notes:
    
      E: file_read is required for basic config files like /etc/netconfig
         net_access is required for CLI communication
    
         file_write could potentially be removed if any file write
         operations (e.g. writing vcl files) were wrapped with
         JAIL_MASTER_FILE, but I do not consider this a relevant gain for
         now.
    
         For other master jail states, E will be momentarily expanded.
    
      I: will be momentarily expanded for system()
    
      P: Contains the union of all privileges used anywhere in varnish
    
      L: Could potentially be reduced further, but P already limits
    
    * worker::
    
      flags = PRIV_AWARE
            E: file_read,file_write,net_access
            I: none
            P: file_read,file_write,net_access,proc_info
            L: file_read,file_write,net_access,proc_info,proc_setid
    
      proc_setid is only used when the worker starts and then dropped
    
      proc_info is only used by vmod_unix

diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 25169384d..02a17f88a 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -104,6 +104,7 @@ void mgt_cli_init_cls(void);
 
 enum jail_master_e {
 	JAIL_MASTER_LOW = 0,
+	JAIL_MASTER_SYSTEM,
 	JAIL_MASTER_FILE,
 	JAIL_MASTER_STORAGE,
 	JAIL_MASTER_PRIVPORT,
diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index f2d90e52e..5b1b82b57 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -327,7 +327,9 @@ mgt_launch_child(struct cli *cli)
 
 	AN(heritage.param);
 	AN(heritage.panic_str);
+	VJ_master(JAIL_MASTER_SYSTEM);
 	if ((pid = fork()) < 0) {
+		VJ_master(JAIL_MASTER_LOW);
 		perror("Could not fork child");
 		exit(1);		// XXX Harsh ?
 	}
@@ -389,6 +391,7 @@ mgt_launch_child(struct cli *cli)
 
 		exit(0);
 	}
+	VJ_master(JAIL_MASTER_LOW);
 	assert(pid > 1);
 	MGT_Complain(C_DEBUG, "Child (%jd) Started", (intmax_t)pid);
 	VSC_C_mgt->child_start++;
diff --git a/bin/varnishd/mgt/mgt_jail_solaris_tbl.h b/bin/varnishd/mgt/mgt_jail_solaris_tbl.h
index dfe912094..51dee41b6 100644
--- a/bin/varnishd/mgt/mgt_jail_solaris_tbl.h
+++ b/bin/varnishd/mgt/mgt_jail_solaris_tbl.h
@@ -44,14 +44,15 @@
  * - INHERITABLE and PERMITTED joined from SUBPROC*
  * - implicit rules from above
  */
-PRIV(MASTER_LOW,	E	, PRIV_PROC_EXEC)	// XXX fork
-PRIV(MASTER_LOW,	E	, PRIV_PROC_FORK)	// XXX fork
 PRIV(MASTER_LOW,	E	, "file_write")	// XXX vcl_boot
 PRIV(MASTER_LOW,	E	, "file_read")	// XXX library open
 PRIV(MASTER_LOW,	E	, "net_access")
 
-PRIV(MASTER_FILE,	E	, PRIV_PROC_EXEC)	// XXX rm -rf in shm
-PRIV(MASTER_FILE,	E	, PRIV_PROC_FORK)	// XXX rm -rf in shm
+PRIV(MASTER_SYSTEM,	E|I	, PRIV_PROC_EXEC)
+PRIV(MASTER_SYSTEM,	E|I	, PRIV_PROC_FORK)
+PRIV(MASTER_SYSTEM,	E|I	, "file_read")
+PRIV(MASTER_SYSTEM,	E|I	, "file_write")
+
 PRIV(MASTER_FILE,	E	, "file_read")
 PRIV(MASTER_FILE,	E	, "file_write")
 
diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index 439c9d42c..ca08e2e10 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -78,6 +78,7 @@ mgt_shm_atexit(void)
 	VJ_master(JAIL_MASTER_FILE);
 	VSMW_Destroy(&mgt_vsmw);
 	if (!MGT_DO_DEBUG(DBG_VTC_MODE)) {
+		VJ_master(JAIL_MASTER_SYSTEM);
 		AZ(system("rm -rf " VSM_MGT_DIRNAME));
 		AZ(system("rm -rf " VSM_CHILD_DIRNAME));
 	}
@@ -93,8 +94,9 @@ mgt_SHM_Init(void)
 {
 	int fd;
 
-	VJ_master(JAIL_MASTER_FILE);
+	VJ_master(JAIL_MASTER_SYSTEM);
 	AZ(system("rm -rf " VSM_MGT_DIRNAME));
+	VJ_master(JAIL_MASTER_FILE);
 	AZ(mkdir(VSM_MGT_DIRNAME, 0755));
 	fd = open(VSM_MGT_DIRNAME, O_RDONLY);
 	VJ_fix_fd(fd, JAIL_FIXFD_VSMMGT);
@@ -112,8 +114,9 @@ void
 mgt_SHM_ChildNew(void)
 {
 
-	VJ_master(JAIL_MASTER_FILE);
+	VJ_master(JAIL_MASTER_SYSTEM);
 	AZ(system("rm -rf " VSM_CHILD_DIRNAME));
+	VJ_master(JAIL_MASTER_FILE);
 	AZ(mkdir(VSM_CHILD_DIRNAME, 0750));
 
 	heritage.vsm_fd = open(VSM_CHILD_DIRNAME, O_RDONLY);
@@ -140,7 +143,7 @@ mgt_SHM_ChildDestroy(void)
 
 	closefd(&heritage.vsm_fd);
 	if (!MGT_DO_DEBUG(DBG_VTC_MODE)) {
-		VJ_master(JAIL_MASTER_FILE);
+		VJ_master(JAIL_MASTER_SYSTEM);
 		AZ(system("rm -rf " VSM_CHILD_DIRNAME));
 		VJ_master(JAIL_MASTER_LOW);
 	}
diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c
index 1d1d86d58..0b85cbd93 100644
--- a/bin/varnishd/mgt/mgt_vcc.c
+++ b/bin/varnishd/mgt/mgt_vcc.c
@@ -229,7 +229,9 @@ mgt_vcc_compile(struct vcc_priv *vp, struct vsb *sb, int C_flag)
 	if (mgt_vcc_touchfile(VSB_data(vp->libfile), sb))
 		return (2);
 
+	VJ_master(JAIL_MASTER_SYSTEM);
 	subs = VSUB_run(sb, run_vcc, vp, "VCC-compiler", -1);
+	VJ_master(JAIL_MASTER_LOW);
 	if (subs)
 		return (subs);
 
@@ -247,11 +249,15 @@ mgt_vcc_compile(struct vcc_priv *vp, struct vsb *sb, int C_flag)
 		free(csrc);
 	}
 
+	VJ_master(JAIL_MASTER_SYSTEM);
 	subs = VSUB_run(sb, run_cc, vp, "C-compiler", 10);
+	VJ_master(JAIL_MASTER_LOW);
 	if (subs)
 		return (subs);
 
+	VJ_master(JAIL_MASTER_SYSTEM);
 	subs = VSUB_run(sb, run_dlopen, vp, "dlopen", 10);
+	VJ_master(JAIL_MASTER_LOW);
 	return (subs);
 }
 

From nils.goroll at uplex.de  Tue Jun  2 12:30:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  2 Jun 2020 12:30:07 +0000 (UTC)
Subject: [master] 72a92760c post-push posh polish
Message-ID: <20200602123007.5400F6EFC8@lists.varnish-cache.org>


commit 72a92760cb6dcd19c600d35c99cff7d117f75461
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 2 14:28:59 2020 +0200

    post-push posh polish

diff --git a/bin/varnishd/mgt/mgt_jail_solaris_tbl.h b/bin/varnishd/mgt/mgt_jail_solaris_tbl.h
index 51dee41b6..f13d81331 100644
--- a/bin/varnishd/mgt/mgt_jail_solaris_tbl.h
+++ b/bin/varnishd/mgt/mgt_jail_solaris_tbl.h
@@ -37,11 +37,11 @@
 
 /* ------------------------------------------------------------
  * MASTER
- * - only MASTER_EFFECTIVE is per JAIL state
+ * - only EFFECTIVE & INHERITABLE are per JAIL state
  * - other priv sets are shared across all MASTER_* JAIL states
  *
  * MASTER implicit rules (vjs_master_rules())
- * - INHERITABLE and PERMITTED joined from SUBPROC*
+ * - INHERITABLE and PERMITTED from SUBPROC* joined into PERMITTED
  * - implicit rules from above
  */
 PRIV(MASTER_LOW,	E	, "file_write")	// XXX vcl_boot
@@ -85,7 +85,7 @@ PRIV(SUBPROC_WORKER,	E	, PRIV_PROC_SETID)	// waived after setuid
 PRIV(SUBPROC_WORKER,	E	, "net_access")
 PRIV(SUBPROC_WORKER,	E	, "file_read")
 PRIV(SUBPROC_WORKER,	E	, "file_write")
-PRIV(SUBPROC_WORKER,	P	, PRIV_PROC_INFO)	/* vmod_unix */
+PRIV(SUBPROC_WORKER,	P	, PRIV_PROC_INFO)	// vmod_unix
 
 #undef E
 #undef I

From nils.goroll at uplex.de  Tue Jun  2 13:28:06 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  2 Jun 2020 13:28:06 +0000 (UTC)
Subject: [master] 109201227 Solaris jail: Allow to pass a privilege
 specification for the worker
Message-ID: <20200602132806.E5A93938F2@lists.varnish-cache.org>


commit 10920122713844aa2df46fac1d71900cb7574ad5
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 2 15:26:43 2020 +0200

    Solaris jail: Allow to pass a privilege specification for the worker

diff --git a/bin/varnishd/mgt/mgt_jail_solaris.c b/bin/varnishd/mgt/mgt_jail_solaris.c
index 3a50b572e..fb118cfd6 100644
--- a/bin/varnishd/mgt/mgt_jail_solaris.c
+++ b/bin/varnishd/mgt/mgt_jail_solaris.c
@@ -323,12 +323,24 @@ vjs_alloc(void)
 static int v_matchproto_(jail_init_f)
 vjs_init(char **args)
 {
-	priv_set_t **sets, *permitted, *inheritable;
+	priv_set_t **sets, *permitted, *inheritable, *user = NULL;
+	const char *e;
 	int vj, vs;
 
 	if (args != NULL && *args != NULL) {
-		ARGV_ERR("-jsolaris takes no arguments.\n");
-		return (0);
+		for (;*args != NULL; args++) {
+			if (!strncmp(*args, "worker=", 7)) {
+				user = priv_str_to_set((*args) + 7, ",", &e);
+				if (user == NULL)
+					ARGV_ERR(
+					    "-jsolaris: parsing worker= "
+					    "argument failed near %s.\n",
+					    e);
+				continue;
+			}
+			ARGV_ERR("-jsolrais: unknown sub-argument '%s'\n",
+			    *args);
+		}
 	}
 
 	permitted = vjs_alloc();
@@ -372,6 +384,9 @@ vjs_init(char **args)
 #define PRIV(name, mask, priv) vjs_add(vjs_sets[JAIL_ ## name], mask, priv);
 #include "mgt_jail_solaris_tbl.h"
 
+	if (user != NULL)
+		priv_union(user, vjs_sets[JAIL_SUBPROC_WORKER][VJS_EFFECTIVE]);
+
 	/* mask by available privs */
 	for (vj = 0; vj < JAIL_LIMIT; vj++) {
 		sets = vjs_sets[vj];
diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index dd01fbeba..3c4a1d931 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -392,11 +392,21 @@ Varnish jails are a generalization over various platform specific
 methods to reduce the privileges of varnish processes. They may have
 specific options. Available jails are:
 
--j solaris
+-j <solaris[,worker=`privspec`]>
 
-  Reduce privileges(5) for `varnishd` and sub-process to the minimally
-  required set. Only available on platforms which have the setppriv(2)
-  call.
+  Reduce `privileges(5)` for `varnishd` and sub-process to the
+  minimally required set. Only available on platforms which have the
+  `setppriv(2)` call.
+
+  The optional `worker` argument can be used to pass a
+  privilege-specification (see `ppriv(1)`) by which to extend the
+  effective set of the varnish worker process. While extended
+  privileges may be required by custom vmods, it is always the more
+  secure to *not* use the `worker` option.
+
+  Example to grant basic privileges to the worker process::
+
+    -j solaris,worker=basic
 
 -j <unix[,user=`user`][,ccgroup=`group`][,workuser=`user`]>
 

From nils.goroll at uplex.de  Tue Jun  2 13:33:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  2 Jun 2020 13:33:07 +0000 (UTC)
Subject: [master] f061c8957 changelog solaris jail
Message-ID: <20200602133307.4C98293CE5@lists.varnish-cache.org>


commit f061c89570ecf7d7e83421a98491a882b71201ee
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 2 15:32:00 2020 +0200

    changelog solaris jail

diff --git a/doc/changes.rst b/doc/changes.rst
index 5512de671..ec0aa9acd 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -26,6 +26,17 @@ http://varnish-cache.org/docs/trunk/whats-new/index.html and via
 individual releases. These documents are updated as part of the
 release process.
 
+===========================
+NEXT (scheduled 2020-09-15)
+===========================
+
+* The Varnish Jail (least privileges) code for Solaris has been
+  largely rewritten. It now reduces privileges even further and thus
+  should improve the security of Varnish on Solaris even more.
+
+* The Varnish Jail now accepts an optional ``worker=`` argument which
+  allows to extend the effective privilege set of the worker process.
+
 ================================
 Varnish Cache 6.4.0 (2020-03-16)
 ================================

From nils.goroll at uplex.de  Tue Jun  2 14:34:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  2 Jun 2020 14:34:07 +0000 (UTC)
Subject: [master] 512e39533 names, names, names
Message-ID: <20200602143407.B9F3C955D0@lists.varnish-cache.org>


commit 512e39533d19b610534786271ebbe8eb3158d121
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 2 16:32:55 2020 +0200

    names, names, names

diff --git a/doc/changes.rst b/doc/changes.rst
index ec0aa9acd..b9d7e44ca 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -34,8 +34,9 @@ NEXT (scheduled 2020-09-15)
   largely rewritten. It now reduces privileges even further and thus
   should improve the security of Varnish on Solaris even more.
 
-* The Varnish Jail now accepts an optional ``worker=`` argument which
-  allows to extend the effective privilege set of the worker process.
+* The Varnish Jail for Solaris now accepts an optional ``worker=``
+  argument which allows to extend the effective privilege set of the
+  worker process.
 
 ================================
 Varnish Cache 6.4.0 (2020-03-16)

From reza at naghibi.com  Tue Jun  2 15:06:08 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Tue,  2 Jun 2020 15:06:08 +0000 (UTC)
Subject: [6.0] a8cbe14c3 Handle workspace allocation errors in VEP_Init()
Message-ID: <20200602150608.B913096419@lists.varnish-cache.org>


commit a8cbe14c33e7a9bb050c27e4a940cf1f710b8c1e
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Mar 18 16:57:15 2020 +0100

    Handle workspace allocation errors in VEP_Init()
    
    Turn assertion into VFP error
    
    The vtc is based upon r02645.vtc and reliably reproduces the panic
    without the patch by sweeping through possible amounts of free workspace
    ranging from 4 to 400 bytes.
    
    Fixes #3253

diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
index 16ff6e749..3852dec88 100644
--- a/bin/varnishd/cache/cache_esi_fetch.c
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -164,8 +164,13 @@ vfp_esi_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe)
 	if (vef == NULL)
 		return (VFP_ERROR);
 	vc->obj_flags |= OF_GZIPED | OF_CHGCE | OF_ESIPROC;
-	vef->vgz = VGZ_NewGzip(vc->wrk->vsl, "G F E");
 	vef->vep = VEP_Init(vc, vc->req, vfp_vep_callback, vef);
+	if (vef->vep == NULL) {
+		FREE_OBJ(vef);
+		return (VFP_ERROR);
+	}
+	vef->vgz = VGZ_NewGzip(vc->wrk->vsl, "G F E");
+
 	vef->ibuf_sz = cache_param->gzip_buffer;
 	vef->ibuf = calloc(1L, vef->ibuf_sz);
 	if (vef->ibuf == NULL)
@@ -231,6 +236,7 @@ static enum vfp_status v_matchproto_(vfp_init_f)
 vfp_esi_init(struct vfp_ctx *vc, struct vfp_entry *vfe)
 {
 	struct vef_priv *vef;
+	struct vep_state *vep;
 
 	CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(vc->req, HTTP_MAGIC);
@@ -239,11 +245,14 @@ vfp_esi_init(struct vfp_ctx *vc, struct vfp_entry *vfe)
 		    "Attempted ESI on partial (206) response");
 		return (VFP_ERROR);
 	}
+	vep = VEP_Init(vc, vc->req, NULL, NULL);
+	if (vep == NULL)
+		return (VFP_ERROR);
 	ALLOC_OBJ(vef, VEF_MAGIC);
 	if (vef == NULL)
 		return (VFP_ERROR);
 	vc->obj_flags |= OF_ESIPROC;
-	vef->vep = VEP_Init(vc, vc->req, NULL, NULL);
+	vef->vep = vep;
 	vfe->priv1 = vef;
 	return (VFP_OK);
 }
diff --git a/bin/varnishd/cache/cache_esi_parse.c b/bin/varnishd/cache/cache_esi_parse.c
index 1c1f13a04..302203a98 100644
--- a/bin/varnishd/cache/cache_esi_parse.c
+++ b/bin/varnishd/cache/cache_esi_parse.c
@@ -1040,7 +1040,11 @@ VEP_Init(struct vfp_ctx *vc, const struct http *req, vep_callback_t *cb,
 	CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(req, HTTP_MAGIC);
 	vep = WS_Alloc(vc->resp->ws, sizeof *vep);
-	AN(vep);
+	if (vep == NULL) {
+		VSLb(vc->wrk->vsl, SLT_VCL_Error,
+		     "VEP_Init() workspace overflow");
+		return (NULL);
+	}
 
 	INIT_OBJ(vep, VEP_MAGIC);
 	vep->url = req->hd[HTTP_HDR_URL].b;
diff --git a/bin/varnishtest/tests/r03253.vtc b/bin/varnishtest/tests/r03253.vtc
new file mode 100644
index 000000000..af72560c5
--- /dev/null
+++ b/bin/varnishtest/tests/r03253.vtc
@@ -0,0 +1,26 @@
+varnishtest "ESI: sweep through tight backend workspace conditions"
+
+server s1 -repeat 100 {
+	rxreq
+	txresp -gzipbody "<html>"
+} -start
+
+varnish v1 -vcl+backend {
+	import vtc;
+	import std;
+	sub vcl_recv {
+		return (pass);
+	}
+	sub vcl_backend_response {
+		vtc.workspace_alloc(backend, -4 *
+			(std.integer(bereq.xid, 1002) - 1000) / 2);
+		set beresp.do_esi = true;
+	}
+} -start
+
+client c1 -repeat 100 {
+	txreq -url "/"
+	# some responses will fail (503), some won't. All we care
+	# about here is the fact that we don't panic
+	rxresp
+} -run

From reza at naghibi.com  Tue Jun  2 15:06:08 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Tue,  2 Jun 2020 15:06:08 +0000 (UTC)
Subject: [6.0] 92455b9c5 assert for VGZ_NewGzip() failures
Message-ID: <20200602150608.CEDBA9641F@lists.varnish-cache.org>


commit 92455b9c5782679ba8a2d24e7013ea00f3cd2a4d
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Mar 19 10:52:15 2020 +0100

    assert for VGZ_NewGzip() failures
    
    VGZ_NewGzip will either assert or succeed.

diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c
index 3852dec88..575685603 100644
--- a/bin/varnishd/cache/cache_esi_fetch.c
+++ b/bin/varnishd/cache/cache_esi_fetch.c
@@ -170,6 +170,7 @@ vfp_esi_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe)
 		return (VFP_ERROR);
 	}
 	vef->vgz = VGZ_NewGzip(vc->wrk->vsl, "G F E");
+	AN(vef->vgz);
 
 	vef->ibuf_sz = cache_param->gzip_buffer;
 	vef->ibuf = calloc(1L, vef->ibuf_sz);
diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index 6483a6758..390c9bae9 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -479,8 +479,7 @@ vfp_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe)
 			vc->obj_flags |= OF_GZIPED;
 		}
 	}
-	if (vg == NULL)
-		return (VFP_ERROR);
+	AN(vg);
 	vfe->priv1 = vg;
 	if (vgz_getmbuf(vg))
 		return (VFP_ERROR);

From phk at FreeBSD.org  Wed Jun  3 06:24:07 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed,  3 Jun 2020 06:24:07 +0000 (UTC)
Subject: [master] 08c466957 (SP|TAB) is rightfully called OWS in RFC7230
Message-ID: <20200603062407.E1CC7B207B@lists.varnish-cache.org>


commit 08c466957b12c566d320302ad2e191db1abc46af
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 3 06:19:25 2020 +0000

    (SP|TAB) is rightfully called OWS in RFC7230
    
    This change is binary compatible.

diff --git a/include/vct.h b/include/vct.h
index 34aca2536..9bd3ce3d8 100644
--- a/include/vct.h
+++ b/include/vct.h
@@ -34,9 +34,9 @@
 
 #include "vas.h"
 
-#define VCT_SP			(1<<0)
+#define VCT_OWS			(1<<0)
 #define VCT_CRLF		(1<<1)
-#define VCT_LWS			(VCT_CRLF | VCT_SP)
+#define VCT_LWS			(VCT_CRLF | VCT_OWS)
 #define VCT_CTL			(1<<2)
 #define VCT_ALPHA		(1<<3)
 #define VCT_SEPARATOR		(1<<4)
@@ -64,7 +64,8 @@ vct_is(int x, uint16_t y)
 	return (vct_typtab[x] & (y));
 }
 
-#define vct_issp(x) vct_is(x, VCT_SP)
+#define vct_isows(x) vct_is(x, VCT_OWS)
+#define vct_issp(x) vct_is(x, VCT_OWS)
 #define vct_ishex(x) vct_is(x, VCT_HEX)
 #define vct_islws(x) vct_is(x, VCT_LWS)
 #define vct_isctl(x) vct_is(x, VCT_CTL)
diff --git a/lib/libvarnish/vct.c b/lib/libvarnish/vct.c
index 1c301d37e..7b0d051ae 100644
--- a/lib/libvarnish/vct.c
+++ b/lib/libvarnish/vct.c
@@ -56,7 +56,7 @@ const uint16_t vct_typtab[256] = {
 	[0x06]	=	VCT_CTL,
 	[0x07]	=	VCT_CTL,
 	[0x08]	=	VCT_CTL,
-	[0x09]	=	VCT_CTL | VCT_SP,
+	[0x09]	=	VCT_CTL | VCT_OWS,
 	[0x0a]	=	VCT_CTL | VCT_CRLF,
 	[0x0b]	=	VCT_CTL | VCT_VT,
 	[0x0c]	=	VCT_CTL,
@@ -79,7 +79,7 @@ const uint16_t vct_typtab[256] = {
 	[0x1d]	=	VCT_CTL,
 	[0x1e]	=	VCT_CTL,
 	[0x1f]	=	VCT_CTL,
-	[0x20]	=	VCT_SP,
+	[0x20]	=	VCT_OWS,
 	[0x21]	=	VCT_TCHAR,
 	[0x22]	=	VCT_SEPARATOR,
 	[0x23]	=	VCT_TCHAR,

From phk at FreeBSD.org  Wed Jun  3 06:24:07 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed,  3 Jun 2020 06:24:07 +0000 (UTC)
Subject: [master] 04c42ac39 Make VJSN able to cope with \u0000
Message-ID: <20200603062408.03808B207E@lists.varnish-cache.org>


commit 04c42ac395b30f36a76251de25ef5ab6a4da4b81
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 3 06:20:58 2020 +0000

    Make VJSN able to cope with \u0000

diff --git a/include/vjsn.h b/include/vjsn.h
index 274c23b12..15a9eaa46 100644
--- a/include/vjsn.h
+++ b/include/vjsn.h
@@ -41,9 +41,11 @@ struct vjsn_val {
 #define VJSN_VAL_MAGIC		0x08a06b80
 	const char		*type;
 	const char		*name;
+	const char		*name_e;
 	VTAILQ_ENTRY(vjsn_val)	list;
 	VTAILQ_HEAD(,vjsn_val)	children;
 	char			*value;
+	char			*value_e;
 };
 
 struct vjsn {
diff --git a/lib/libvarnish/vjsn.c b/lib/libvarnish/vjsn.c
index 2a52bd874..8b5444703 100644
--- a/lib/libvarnish/vjsn.c
+++ b/lib/libvarnish/vjsn.c
@@ -201,20 +201,21 @@ vjsn_unicode(struct vjsn *js, char **d)
 }
 
 static char *
-vjsn_string(struct vjsn *js)
+vjsn_string(struct vjsn *js, char **e)
 {
 	char *p, *b;
 
+	AN(e);
 	vjsn_skip_ws(js);
 	VJSN_EXPECT(js, '"', NULL);
 	b = p = js->ptr;
 	while (*js->ptr != '"') {
 		if (*js->ptr == '\0') {
-			js->err = "Unterminate string";
+			js->err = "Unterminated string";
 			return (NULL);
 		}
 		if ((unsigned char)(*js->ptr) <= 0x1f) {
-			js->err = "unescaped control char in string";
+			js->err = "Unescaped control char in string";
 			return (NULL);
 		}
 		if (*js->ptr != '\\') {
@@ -242,6 +243,7 @@ vjsn_string(struct vjsn *js)
 	}
 	VJSN_EXPECT(js, '"', NULL);
 	*p = '\0';
+	*e = p;
 	return (b);
 }
 
@@ -249,7 +251,7 @@ static struct vjsn_val *
 vjsn_object(struct vjsn *js)
 {
 	struct vjsn_val *jsv, *jsve;
-	char *s;
+	char *s, *e;
 
 	VJSN_EXPECT(js, '{', NULL);
 
@@ -259,7 +261,7 @@ vjsn_object(struct vjsn *js)
 	vjsn_skip_ws(js);
 	if (*js->ptr != '}') {
 		while (1) {
-			s = vjsn_string(js);
+			s = vjsn_string(js, &e);
 			if (js->err != NULL)
 				return (jsv);
 			vjsn_skip_ws(js);
@@ -272,6 +274,7 @@ vjsn_object(struct vjsn *js)
 			}
 			CHECK_OBJ_NOTNULL(jsve, VJSN_VAL_MAGIC);
 			jsve->name = s;
+			jsve->name_e = e;
 			VTAILQ_INSERT_TAIL(&jsv->children, jsve, list);
 			vjsn_skip_ws(js);
 			if (*js->ptr == '}')
@@ -372,7 +375,7 @@ vjsn_value(struct vjsn *js)
 		return (vjsn_array(js));
 	if (*js->ptr== '"') {
 		jsv = vjsn_val_new(VJSN_STRING);
-		jsv->value = vjsn_string(js);
+		jsv->value = vjsn_string(js, &jsv->value_e);
 		if (js->err != NULL)
 			return (jsv);
 		AN(jsv->value);

From dridi.boukelmoune at gmail.com  Wed Jun  3 07:21:07 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed,  3 Jun 2020 07:21:07 +0000 (UTC)
Subject: [master] e89652998 Fix the GCOV workaround in u0.vtc
Message-ID: <20200603072107.36499BE6AB@lists.varnish-cache.org>


commit e896529982027a4325dc3bf19f97c8ebbff4ef53
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Jun 3 09:18:13 2020 +0200

    Fix the GCOV workaround in u0.vtc
    
    Refs 9a14e68e91ad

diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc
index e5797fe8c..570c778d3 100644
--- a/bin/varnishtest/tests/u00000.vtc
+++ b/bin/varnishtest/tests/u00000.vtc
@@ -4,7 +4,7 @@ shell "varnishd -b 127.0.0.1:80 -C 2> ${tmpdir}/_.c"
 
 shell {
 	varnishd -n ${tmpdir}/no_keep -C -b 127.0.0.1:80 2> no_keep.c
-	test -s no_keep.c && ! test -d no_keep || test -f no_keep/vgc.gcda
+	test -s no_keep.c && ! test -d no_keep || test -f no_keep/*/vgc.gcda
 }
 
 shell {

From nils.goroll at uplex.de  Wed Jun  3 11:12:19 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 3 Jun 2020 13:12:19 +0200
Subject: [master] 1b04343a0 jail enum assertions
In-Reply-To: <CABoVN9DUR6CE7UxJyyPK4eSCuqn2Um3Ywb6vFBbfyN3UphSxZw@mail.gmail.com>
References: <20200529162306.6EFDC4E89@lists.varnish-cache.org>
 <CABoVN9DUR6CE7UxJyyPK4eSCuqn2Um3Ywb6vFBbfyN3UphSxZw@mail.gmail.com>
Message-ID: <e9d10044-7ff4-d4ca-55fe-1eceb30ad156@uplex.de>

On 29/05/2020 18:38, Dridi Boukelmoune wrote:
> You might as well revert the last two commits and turn the enums into
> structs right off the bat.

FTR, I have now based code on the assumption that the enums work as array
indices and would prefer to keep that for simplicity.

Alternatively, we could have structs with priv pointers to the implementation,
but at this point I think that the complication would have no relevant benefit.
Or would it?

Nils

-- 

** * * UPLEX - Nils Goroll Systemoptimierung

Scheffelstra?e 32
22301 Hamburg

tel +49 40 28805731
mob +49 170 2723133
fax +49 40 42949753

xmpp://slink at jabber.int.uplex.de/

http://uplex.de/

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 488 bytes
Desc: OpenPGP digital signature
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20200603/cf634b21/attachment.bin>

From dridi at varni.sh  Wed Jun  3 11:31:04 2020
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Wed, 3 Jun 2020 11:31:04 +0000
Subject: [master] 1b04343a0 jail enum assertions
In-Reply-To: <e9d10044-7ff4-d4ca-55fe-1eceb30ad156@uplex.de>
References: <20200529162306.6EFDC4E89@lists.varnish-cache.org>
 <CABoVN9DUR6CE7UxJyyPK4eSCuqn2Um3Ywb6vFBbfyN3UphSxZw@mail.gmail.com>
 <e9d10044-7ff4-d4ca-55fe-1eceb30ad156@uplex.de>
Message-ID: <CABoVN9D_rQo3m7VEK3cPkmQip9j3GHumVJCL_y9V91_K3N=iiQ@mail.gmail.com>

On Wed, Jun 3, 2020 at 11:12 AM Nils Goroll <nils.goroll at uplex.de> wrote:
>
> On 29/05/2020 18:38, Dridi Boukelmoune wrote:
> > You might as well revert the last two commits and turn the enums into
> > structs right off the bat.
>
> FTR, I have now based code on the assumption that the enums work as array
> indices and would prefer to keep that for simplicity.

The structs could be generated from an include table and have an index field.

> Alternatively, we could have structs with priv pointers to the implementation,
> but at this point I think that the complication would have no relevant benefit.
> Or would it?

I don't think we could make this work with const struct symbols, so
unless there is a compelling reason we should keep the working enums.

Dridi

From dridi.boukelmoune at gmail.com  Thu Jun  4 09:59:07 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu,  4 Jun 2020 09:59:07 +0000 (UTC)
Subject: [master] 8709deb83 Test coverage for VCL "weak" symbols
Message-ID: <20200604095907.AE9AEB24DF@lists.varnish-cache.org>


commit 8709deb830a99862e956817c750192f24f5bd313
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Jun 4 11:56:52 2020 +0200

    Test coverage for VCL "weak" symbols
    
    Simply make sure we don't allow symbols starting with an underscore
    symbol.

diff --git a/bin/varnishtest/tests/v00019.vtc b/bin/varnishtest/tests/v00019.vtc
index e37751869..fc98616a6 100644
--- a/bin/varnishtest/tests/v00019.vtc
+++ b/bin/varnishtest/tests/v00019.vtc
@@ -77,3 +77,11 @@ varnish v1 -errvcl {Unknown token '--' when looking for INT} {
 		set resp.status = --200;
 	}
 }
+
+varnish v1 -errvcl "Syntax error" {
+	import debug;
+	backend be none;
+	sub vcl_init {
+		new _invalid = debug.obj();
+	}
+}

From dridi.boukelmoune at gmail.com  Thu Jun  4 15:17:07 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu,  4 Jun 2020 15:17:07 +0000 (UTC)
Subject: [master] fa988f5a5 Constify struct symtab
Message-ID: <20200604151707.7FEDF992B@lists.varnish-cache.org>


commit fa988f5a53bd28dd8574827d4338b3f9679ef1db
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Jun 4 17:13:53 2020 +0200

    Constify struct symtab
    
    Children shouldn't be able to mess with their parents, even though in
    practice they do.

diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index bf33292e4..6784ef48b 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -49,7 +49,7 @@ struct symtab {
 #define SYMTAB_MAGIC			0x084d9c8a
 	unsigned			nlen;
 	const char			*name;
-	struct symtab			*parent;
+	const struct symtab		*parent;
 	VTAILQ_ENTRY(symtab)		list;
 	VTAILQ_HEAD(,symtab)		children;
 	VTAILQ_HEAD(,symbol)		symbols;
@@ -196,7 +196,7 @@ static struct symbol *
 vcc_sym_in_tab(struct vcc *tl, struct symtab *st,
     vcc_kind_t kind, int vlo, int vhi)
 {
-	struct symtab *pst;
+	const struct symtab *pst;
 	struct symbol *sym, *psym;
 
 	VTAILQ_FOREACH(sym, &st->symbols, list) {

From dridi.boukelmoune at gmail.com  Fri Jun  5 14:26:07 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri,  5 Jun 2020 14:26:07 +0000 (UTC)
Subject: [master] f989ab248 Stop VCL processing if vcl_recv failed
Message-ID: <20200605142607.0D028BEAF1@lists.varnish-cache.org>


commit f989ab2487c771f1a2a4883461397f74624bde7f
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Jun 5 16:24:27 2020 +0200

    Stop VCL processing if vcl_recv failed
    
    VCL failure should abort execution, but vcl_recv would possibly modify
    req and even continue execution in vcl_hash.
    
    Refs #3303

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 2a1903957..8c075789f 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -884,6 +884,12 @@ cnt_recv(struct worker *wrk, struct req *req)
 	}
 
 	VCL_recv_method(req->vcl, wrk, req, NULL, NULL);
+
+	if (recv_handling == VCL_RET_FAIL) {
+		req->req_step = R_STP_VCLFAIL;
+		return (REQ_FSM_MORE);
+	}
+
 	if (wrk->handling == VCL_RET_VCL && req->restarts == 0) {
 		// Req_Rollback has happened in VPI_vcl_select
 		assert(WS_Snapshot(req->ws) == req->ws_req);

From dridi.boukelmoune at gmail.com  Fri Jun  5 14:29:06 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri,  5 Jun 2020 14:29:06 +0000 (UTC)
Subject: [master] 72fba6f5e Failure coverage for vcl_recv
Message-ID: <20200605142906.CAF5ABED4E@lists.varnish-cache.org>


commit 72fba6f5ebc97493929deecae98b454ec31df209
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Jun 5 16:27:46 2020 +0200

    Failure coverage for vcl_recv
    
    Refs #3303

diff --git a/bin/varnishtest/tests/b00072.vtc b/bin/varnishtest/tests/b00072.vtc
new file mode 100644
index 000000000..55e19b9d9
--- /dev/null
+++ b/bin/varnishtest/tests/b00072.vtc
@@ -0,0 +1,21 @@
+varnishtest "failure in vcl_recv"
+
+varnish v1 -vcl {
+	import vtc;
+
+	backend be none;
+
+	sub vcl_recv {
+		return (fail);
+	}
+
+	sub vcl_hash {
+		vtc.panic("unreachable");
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 503
+} -run

From dridi.boukelmoune at gmail.com  Fri Jun  5 14:31:06 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri,  5 Jun 2020 14:31:06 +0000 (UTC)
Subject: [master] 4f557bf0f Copy-pasta typo
Message-ID: <20200605143106.D4CF2BEF57@lists.varnish-cache.org>


commit 4f557bf0f8a98140f7d756d06d3e2e7c9446872e
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Jun 5 16:29:58 2020 +0200

    Copy-pasta typo

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 8c075789f..f6b7e6ad0 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -885,7 +885,7 @@ cnt_recv(struct worker *wrk, struct req *req)
 
 	VCL_recv_method(req->vcl, wrk, req, NULL, NULL);
 
-	if (recv_handling == VCL_RET_FAIL) {
+	if (wrk->handling == VCL_RET_FAIL) {
 		req->req_step = R_STP_VCLFAIL;
 		return (REQ_FSM_MORE);
 	}

From dridi.boukelmoune at gmail.com  Fri Jun  5 15:21:07 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri,  5 Jun 2020 15:21:07 +0000 (UTC)
Subject: [master] d85f4c38f Fix workspace VSB example
Message-ID: <20200605152107.2D5EBC0056@lists.varnish-cache.org>


commit d85f4c38f566d4f45fba8e4e5c51622655ecce91
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Jun 5 17:19:11 2020 +0200

    Fix workspace VSB example

diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c
index c563c29c6..dc1744779 100644
--- a/bin/varnishd/cache/cache_ws.c
+++ b/bin/varnishd/cache/cache_ws.c
@@ -382,7 +382,7 @@ WS_Overflowed(const struct ws *ws)
  *
  *	WS_VSB_new(vsb, ctx->ws);
  *	VSB_printf(vsb, "blablabla");
- *	p = WS_VSB_finish(vsb, NULL);
+ *	p = WS_VSB_finish(vsb, ctx->ws, NULL);
  *	if (p == NULL)
  *		return (FAILURE);
  */

From phk at FreeBSD.org  Tue Jun  9 07:01:06 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  9 Jun 2020 07:01:06 +0000 (UTC)
Subject: [master] d5c626213 Avoid a race by using syncvsl
Message-ID: <20200609070106.F3FC67580@lists.varnish-cache.org>


commit d5c626213b8979ab9ac4adb48c93c861c6b3fd28
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 9 06:58:35 2020 +0000

    Avoid a race by using syncvsl
    
    Without syncvsl, client N+1 may emitting VSL before client N
    has flushed its VSL.
    
    (In general we should probably always use syncvsl with logexpect)

diff --git a/bin/varnishtest/tests/m00049.vtc b/bin/varnishtest/tests/m00049.vtc
index 01ad05b66..8a3dfd4f5 100644
--- a/bin/varnishtest/tests/m00049.vtc
+++ b/bin/varnishtest/tests/m00049.vtc
@@ -6,6 +6,7 @@ varnishtest "VMOD blob workspace overflow conditions"
 # will not result in a compilation failure.
 
 varnish v1 -cliok "param.set vcc_err_unref off"
+varnish v1 -cliok "param.set debug +syncvsl"
 
 shell {
 	cat >vrt_blob.vcl <<-EOF

From phk at FreeBSD.org  Tue Jun  9 08:59:06 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  9 Jun 2020 08:59:06 +0000 (UTC)
Subject: [master] 64b3e63bb Use setrlimit(2) to disable core dumps.
Message-ID: <20200609085907.0D706626F8@lists.varnish-cache.org>


commit 64b3e63bbb45339abbc03bba9fc4a575f7e527b0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 9 08:13:50 2020 +0000

    Use setrlimit(2) to disable core dumps.
    
    VTC's change because the exit-instead-of-abort-hack goes away.

diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 05d229227..407b8eacf 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -811,10 +811,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond,
 	VSB_cat(pan_vsb, "\n");
 	VSB_putc(pan_vsb, '\0');	/* NUL termination */
 
-	if (FEATURE(FEATURE_NO_COREDUMP))
-		exit(4);
-	else
-		abort();
+	abort();
 }
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 5b1b82b57..9d01f0f03 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -43,6 +43,9 @@
 #include <string.h>
 #include <syslog.h>
 #include <unistd.h>
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/resource.h>
 
 #include "mgt.h"
 
@@ -295,6 +298,7 @@ mgt_launch_child(struct cli *cli)
 	char *p;
 	struct vev *e;
 	int i, cp[2];
+	struct rlimit rl[1];
 
 	if (child_state != CH_STOPPED && child_state != CH_DIED)
 		return;
@@ -335,6 +339,12 @@ mgt_launch_child(struct cli *cli)
 	}
 	if (pid == 0) {
 
+		if (MGT_FEATURE(FEATURE_NO_COREDUMP)) {
+			memset(rl, 0, sizeof *rl);
+			rl->rlim_cur = 0;
+			AZ(setrlimit(RLIMIT_CORE, rl));
+		}
+
 		/* Redirect stdin/out/err */
 		VFIL_null_fd(STDIN_FILENO);
 		assert(dup2(heritage.std_fd, STDOUT_FILENO) == STDOUT_FILENO);
@@ -463,10 +473,7 @@ kill_child(void)
 	int i, error;
 
 	VJ_master(JAIL_MASTER_KILL);
-	if (MGT_FEATURE(FEATURE_NO_COREDUMP))
-		i = kill(child_pid, SIGKILL);
-	else
-		i = kill(child_pid, SIGQUIT);
+	i = kill(child_pid, SIGQUIT);
 	error = errno;
 	VJ_master(JAIL_MASTER_LOW);
 	errno = error;
diff --git a/bin/varnishtest/tests/c00057.vtc b/bin/varnishtest/tests/c00057.vtc
index 65951be6b..280cc3711 100644
--- a/bin/varnishtest/tests/c00057.vtc
+++ b/bin/varnishtest/tests/c00057.vtc
@@ -60,7 +60,7 @@ client c1 {
 	rxresp
 } -run
 
-varnish v1 -expectexit 0x20
+varnish v1 -expectexit 0x40
 
 ####################
 
@@ -91,4 +91,4 @@ varnish v2 -clijson "panic.show -j"
 
 varnish v2 -cliok "panic.clear"
 
-varnish v2 -expectexit 0x20
+varnish v2 -expectexit 0x40
diff --git a/bin/varnishtest/tests/p00007.vtc b/bin/varnishtest/tests/p00007.vtc
index 08c5c27b1..67d00444e 100644
--- a/bin/varnishtest/tests/p00007.vtc
+++ b/bin/varnishtest/tests/p00007.vtc
@@ -80,4 +80,4 @@ client c1 {
 	expect resp.bodylen == 48
 } -run
 
-varnish v1 -expectexit 0x20
+varnish v1 -expectexit 0x40
diff --git a/bin/varnishtest/tests/t02004.vtc b/bin/varnishtest/tests/t02004.vtc
index 709fff70c..2cc33717b 100644
--- a/bin/varnishtest/tests/t02004.vtc
+++ b/bin/varnishtest/tests/t02004.vtc
@@ -5,6 +5,10 @@ server s1 {
 	txresp
 } -start
 
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.set feature +no_coredump"
+varnish v1 -cliok "param.set debug +syncvsl"
+
 varnish v1 -vcl+backend {
 	import vtc;
 
@@ -13,10 +17,6 @@ varnish v1 -vcl+backend {
 	}
 } -start
 
-varnish v1 -cliok "param.set feature +http2"
-varnish v1 -cliok "param.set feature +no_coredump"
-varnish v1 -cliok "param.set debug +syncvsl"
-
 client c1 {
 	stream 1 {
 		txreq -hdr :authority foo.bar -pad cotton
@@ -28,4 +28,4 @@ delay 2
 
 varnish v1 -cliok "panic.clear"
 
-varnish v1 -expectexit 0x20
+varnish v1 -expectexit 0x40
diff --git a/bin/varnishtest/tests/v00010.vtc b/bin/varnishtest/tests/v00010.vtc
index 89d3c298e..bbfa927fb 100644
--- a/bin/varnishtest/tests/v00010.vtc
+++ b/bin/varnishtest/tests/v00010.vtc
@@ -24,7 +24,10 @@ server s1 {
 	txresp -hdr "Foo: foo" -body "abcdef\n"
 } -start
 
-varnish v1 -arg "-sdefault,1m" -vcl+backend {
+varnish v1 -arg "-sdefault,1m"
+varnish v1 -cliok "param.set feature +no_coredump"
+
+varnish v1 -vcl+backend {
 	import vtc;
 	import debug;
 
@@ -49,7 +52,6 @@ varnish v1 -cliok "stop"
 varnish v1 -cliok "start"
 varnish v1 -wait-running
 varnish v1 -expect MGT.child_panic == 0
-varnish v1 -cliok "param.set feature +no_coredump"
 
 client c1 {
 	txreq -url "/"
@@ -92,4 +94,4 @@ client c1 {
 } -run
 
 varnish v1 -cliok "panic.clear -z"
-varnish v1 -expectexit 0x20
+varnish v1 -expectexit 0x40
diff --git a/bin/varnishtest/tests/v00063.vtc b/bin/varnishtest/tests/v00063.vtc
index c36ff1aa4..c5984ccd9 100644
--- a/bin/varnishtest/tests/v00063.vtc
+++ b/bin/varnishtest/tests/v00063.vtc
@@ -3,7 +3,7 @@ varnishtest "Create a backend after a COLD event"
 server s1 -start
 
 varnish v1 -cliok "param.set feature +no_coredump"
-varnish v1 -expectexit 0x20
+varnish v1 -expectexit 0x40
 varnish v1 -vcl+backend {
 	import debug;
 	sub vcl_init {

From phk at FreeBSD.org  Tue Jun  9 08:59:07 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  9 Jun 2020 08:59:07 +0000 (UTC)
Subject: [master] f23d3944b Somewhere along the way we lost the "long
 description" of the feature bits. Merge it into "short description" (which
 can be longer now), reorder features and improve messages.
Message-ID: <20200609085907.2ABA8626FA@lists.varnish-cache.org>


commit f23d3944b78104b4fec6ee743451d297efaf0a93
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 9 08:57:41 2020 +0000

    Somewhere along the way we lost the "long description" of the
    feature bits.  Merge it into "short description" (which can be
    longer now), reorder features and improve messages.

diff --git a/bin/varnishd/common/common_param.h b/bin/varnishd/common/common_param.h
index 6d1630c2e..5896bde0a 100644
--- a/bin/varnishd/common/common_param.h
+++ b/bin/varnishd/common/common_param.h
@@ -53,7 +53,7 @@ COM_DO_DEBUG(const volatile uint8_t *p, enum debug_bits x)
 }
 
 enum feature_bits {
-#define FEATURE_BIT(U, l, d, ld) FEATURE_##U,
+#define FEATURE_BIT(U, l, d) FEATURE_##U,
 #include "tbl/feature_bits.h"
        FEATURE_Reserved
 };
diff --git a/bin/varnishd/mgt/mgt_param_bits.c b/bin/varnishd/mgt/mgt_param_bits.c
index cf39ee027..61edab2e4 100644
--- a/bin/varnishd/mgt/mgt_param_bits.c
+++ b/bin/varnishd/mgt/mgt_param_bits.c
@@ -208,7 +208,7 @@ tweak_debug(struct vsb *vsb, const struct parspec *par, const char *arg)
  */
 
 static const char * const feature_tags[] = {
-#  define FEATURE_BIT(U, l, d, ld) [FEATURE_##U] = #l,
+#  define FEATURE_BIT(U, l, d) [FEATURE_##U] = #l,
 #  include "tbl/feature_bits.h"
        NULL
 };
@@ -275,7 +275,7 @@ struct parspec VSL_parspec[] = {
 		"Enable/Disable various minor features.\n"
 		"\tnone\tDisable all features.\n\n"
 		"Use +/- prefix to enable/disable individual feature:"
-#define FEATURE_BIT(U, l, d, ld) "\n\t" #l "\t" d
+#define FEATURE_BIT(U, l, d) "\n\t" #l "\t" d
 #include "tbl/feature_bits.h"
 #undef FEATURE_BIT
 		},
diff --git a/include/tbl/feature_bits.h b/include/tbl/feature_bits.h
index 042240f3c..2f314cfb2 100644
--- a/include/tbl/feature_bits.h
+++ b/include/tbl/feature_bits.h
@@ -33,56 +33,45 @@
 
 /*lint -save -e525 -e539 */
 
-FEATURE_BIT(SHORT_PANIC,		short_panic,
-    "Short panic message.",
-    "Reduce level of detail for panic messages."
+FEATURE_BIT(HTTP2,		http2,
+    "Enable HTTP/2 protocol support."
 )
 
-FEATURE_BIT(WAIT_SILO,			wait_silo,
-    "Wait for persistent silo.",
-    "Wait for persistent silos to load completely before serving requests."
+FEATURE_BIT(SHORT_PANIC,		short_panic,
+    "Short panic message."
 )
 
 FEATURE_BIT(NO_COREDUMP,		no_coredump,
-    "No coredumps.",
-    "Don't attempt to coredump child process on panics."
+    "No coredumps.  Must be set before child process starts."
 )
 
-FEATURE_BIT(ESI_IGNORE_HTTPS,		esi_ignore_https,
-    "Treat HTTPS as HTTP in ESI:includes",
-    "Convert <esi:include src\"https://... to http://..."
+FEATURE_BIT(HTTPS_SCHEME,		https_scheme,
+    "Extract host from full URI in the HTTP/1 request line, if the scheme is https."
 )
 
-FEATURE_BIT(ESI_DISABLE_XML_CHECK,	esi_disable_xml_check,
-    "Don't check of body looks like XML",
-    "Allow ESI processing on any kind of object"
+FEATURE_BIT(HTTP_DATE_POSTEL,	http_date_postel,
+    "Tolerate non compliant timestamp headers "
+    "like `Date`, `Last-Modified`, `Expires` etc."
 )
 
-FEATURE_BIT(ESI_IGNORE_OTHER_ELEMENTS,	esi_ignore_other_elements,
-    "Ignore non-esi XML-elements",
-    "Allows syntax errors in the XML"
+FEATURE_BIT(ESI_IGNORE_HTTPS,		esi_ignore_https,
+    "Convert `<esi:include src\"https://...` to `http://...`"
 )
 
-FEATURE_BIT(ESI_REMOVE_BOM,		esi_remove_bom,
-    "Remove UTF-8 BOM",
-    "Remove UTF-8 BOM from front of object."
-    "Ignore and remove the UTF-8 BOM (0xeb 0xbb 0xbf) from front of object."
+FEATURE_BIT(ESI_DISABLE_XML_CHECK,	esi_disable_xml_check,
+    "Allow ESI processing on non-XML ESI bodies"
 )
 
-FEATURE_BIT(HTTPS_SCHEME,		https_scheme,
-    "Also split https URIs",
-    "Extract host from full URI in the request line if the scheme is https."
+FEATURE_BIT(ESI_IGNORE_OTHER_ELEMENTS,	esi_ignore_other_elements,
+    "Ignore XML syntax errors in ESI bodies."
 )
 
-FEATURE_BIT(HTTP2,		http2,
-    "Support HTTP/2 protocol",
-    "Enable HTTP/2 protocol support."
+FEATURE_BIT(ESI_REMOVE_BOM,		esi_remove_bom,
+    "Ignore UTF-8 BOM in ESI bodies."
 )
 
-FEATURE_BIT(HTTP_DATE_POSTEL,	http_date_postel,
-    "Relax parsing of timestamps in HTTP headers",
-    "Tolerate non standards conforming variations of timestamp headers"
-    "like Date:, Last-Modified:, Expires: etc."
+FEATURE_BIT(WAIT_SILO,			wait_silo,
+    "Wait for persistent silos to completely load before serving requests."
 )
 
 #undef FEATURE_BIT

From phk at FreeBSD.org  Tue Jun  9 09:31:07 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  9 Jun 2020 09:31:07 +0000 (UTC)
Subject: [master] 20d257981 Don't report coredumps if they were disabled.
Message-ID: <20200609093107.EF577636B4@lists.varnish-cache.org>


commit 20d25798183131f3b4d325c42447fab66fecb7a5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 9 09:27:46 2020 +0000

    Don't report coredumps if they were disabled.
    
    Some operating systems still report core-dumps in process exit-status
    even though the core-dump was eliminated with setrlimit(2).

diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 9d01f0f03..9c3bbd02d 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -554,7 +554,8 @@ mgt_reap_child(void)
 #ifdef WCOREDUMP
 	if (WCOREDUMP(status)) {
 		VSB_cat(vsb, " (core dumped)");
-		exit_status |= 0x80;
+		if (!MGT_FEATURE(FEATURE_NO_COREDUMP))
+			exit_status |= 0x80;
 		VSC_C_mgt->child_dump++;
 	}
 #endif

From nils.goroll at uplex.de  Tue Jun  9 10:25:06 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  9 Jun 2020 10:25:06 +0000 (UTC)
Subject: [master] 6294c0cb2 object/stevedore: document required vs. optional
 callbacks
Message-ID: <20200609102506.E001F64C9B@lists.varnish-cache.org>


commit 6294c0cb29b49081c5eae45a8039f22e98c83211
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 9 12:17:59 2020 +0200

    object/stevedore: document required vs. optional callbacks

diff --git a/bin/varnishd/cache/cache_obj.h b/bin/varnishd/cache/cache_obj.h
index 5208afa0c..1f936a534 100644
--- a/bin/varnishd/cache/cache_obj.h
+++ b/bin/varnishd/cache/cache_obj.h
@@ -51,15 +51,17 @@ typedef void *objsetattr_f(struct worker *, struct objcore *,
 typedef void objtouch_f(struct worker *, struct objcore *, vtim_real now);
 
 struct obj_methods {
+	/* required */
 	objfree_f	*objfree;
 	objiterator_f	*objiterator;
 	objgetspace_f	*objgetspace;
 	objextend_f	*objextend;
+	objgetattr_f	*objgetattr;
+	objsetattr_f	*objsetattr;
+	/* optional */
 	objtrimstore_f	*objtrimstore;
 	objbocdone_f	*objbocdone;
 	objslim_f	*objslim;
-	objgetattr_f	*objgetattr;
-	objsetattr_f	*objsetattr;
 	objtouch_f	*objtouch;
 	objsetstate_f	*objsetstate;
 };
diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h
index 32bd1a2fa..c6cda8998 100644
--- a/bin/varnishd/storage/storage.h
+++ b/bin/varnishd/storage/storage.h
@@ -95,7 +95,9 @@ struct stevedore {
 	/* Called in MGT process */
 	storage_init_f		*init;
 
-	/* Called in cache process */
+	/* Called in cache process
+	 * only allocobj is required, other callbacks are optional
+	 */
 	storage_open_f		*open;
 	storage_close_f		*close;
 	storage_allocobj_f	*allocobj;

From nils.goroll at uplex.de  Tue Jun  9 17:03:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  9 Jun 2020 17:03:07 +0000 (UTC)
Subject: [master] d624ddf62 shard director: retire canon_point
Message-ID: <20200609170307.5EB31A2127@lists.varnish-cache.org>


commit d624ddf62a0db701d138c9552458013d2dc08b88
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 9 18:04:17 2020 +0200

    shard director: retire canon_point
    
    The first hashcircle point of a backend was saves as canon_point, but
    never used for anything but JSON backend.list output.
    
    I assume no-one needs it. Famous Last Words (tm).

diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c
index 7567f0050..6276e1728 100644
--- a/lib/libvmod_directors/shard_cfg.c
+++ b/lib/libvmod_directors/shard_cfg.c
@@ -273,9 +273,6 @@ shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas)
 			    VRT_HashStrands32(ss);
 			shardd->hashcircle[i * replicas + j].host = i;
 		}
-		/* not used in current interface */
-		shardd->backend[i].canon_point =
-		    shardd->hashcircle[i * replicas].point;
 	}
 	qsort( (void *) shardd->hashcircle, shardd->n_backend * replicas,
 	    sizeof (struct shard_circlepoint), (compar) circlepoint_compare);
@@ -313,7 +310,6 @@ shardcfg_backend_copyin(struct shard_backend *dst,
 	dst->backend = src->backend;
 	dst->ident = src->ident ? strdup(src->ident) : NULL;
 	dst->rampup = src->rampup;
-	dst->canon_point = 0xffffffff;
 }
 
 static int
diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h
index 067b4415a..03df1c312 100644
--- a/lib/libvmod_directors/shard_dir.h
+++ b/lib/libvmod_directors/shard_dir.h
@@ -43,7 +43,6 @@ struct shard_backend {
 		void		*freeptr;
 	};
 	VCL_DURATION		rampup;
-	uint32_t		canon_point;
 };
 
 struct vmod_directors_shard_param;
diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c
index 7c56cedb6..c3ec5d837 100644
--- a/lib/libvmod_directors/vmod_shard.c
+++ b/lib/libvmod_directors/vmod_shard.c
@@ -763,8 +763,6 @@ vmod_shard_list(VRT_CTX, VCL_BACKEND dir, struct vsb *vsb, int pflag, int jflag)
 			    sbe->ident ? sbe->ident : be->vcl_name);
 			VSB_printf(vsb, "\"health\": \"%s\",\n",
 			    h ? "healthy" : "sick");
-			VSB_printf(vsb, "\"canon_point\": %u,\n",
-			    sbe->canon_point);
 			VSB_printf(vsb, "\"rampup\": %f,\n", rampup_p);
 			VSB_printf(vsb, "\"rampup_remaining\": %.3f\n",
 			    rampup_d);

From nils.goroll at uplex.de  Tue Jun  9 17:03:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  9 Jun 2020 17:03:07 +0000 (UTC)
Subject: [master] b90b60d0a shard director: add optional weight parameter to
 .add_backend()
Message-ID: <20200609170307.79A6CA212A@lists.varnish-cache.org>


commit b90b60d0a066f1aa7302a70125cb91cdfa605119
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 9 19:02:00 2020 +0200

    shard director: add optional weight parameter to .add_backend()
    
    We implement weights by scaling the number of replicas of each backend.
    The replicas parameter of .reconfigure() remains a minimum.
    
    For existing vtcs, the Debug hashcircle output has been compared
    before/after this change to ensure that behaviour is exactly equivalent.
    
    For for wighted backends, it has been checked that the number of
    instances per host on the hashcircle matches the expectation.
    
    Also refactor and clean up some of the code:
    
    - consistently make the number of ring points a uint32_t
    - some constification
    
    Ref #3276

diff --git a/bin/varnishtest/tests/d00041.vtc b/bin/varnishtest/tests/d00041.vtc
new file mode 100644
index 000000000..bd79cfff1
--- /dev/null
+++ b/bin/varnishtest/tests/d00041.vtc
@@ -0,0 +1,224 @@
+varnishtest "d00017.vtc but with weights"
+
+server s1 {
+	rxreq
+	txresp -body "ech3Ooj"
+} -start
+
+server s2 {
+	rxreq
+	txresp -body "ieQu2qua"
+} -start
+
+server s3 {
+	rxreq
+	txresp -body "xiuFi3Pe"
+} -start
+
+varnish v1 -vcl+backend {
+	import std;
+	import directors;
+	import blob;
+
+	sub vcl_init {
+		new vd = directors.shard();
+		vd.debug(3);
+		if (!vd.add_backend(s1)) {
+			std.log("add s1 failed");
+		}
+		if (!vd.add_backend(s2, weight=2)) {
+			std.log("add s2 failed");
+		}
+		if (!vd.add_backend(s3, weight=3)) {
+			std.log("add s3 failed");
+		}
+		if (!vd.reconfigure(replicas=25)) {
+			std.log("reconfigure failed");
+		}
+	}
+
+	sub vcl_recv {
+		set req.backend_hint = vd.backend(by=BLOB,
+		    key_blob=blob.decode(HEX, encoded=
+			regsub(req.url, "^/", "")));
+		return(pass);
+	}
+
+} -start
+
+logexpect l1 -v v1 -g raw -d 1 {
+	expect 0 0    CLI   "^Rd vcl.load"
+
+	expect 0 =    Debug {^shard: hashcircle.* 0. = .point = *238d0ef, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 1. = .point = *321c598, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 2. = .point = *3b6b56a, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 3. = .point = *408ec1e, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 4. = .point = *66986a7, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 5. = .point = *7e41e30, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 6. = .point = *b749e7b, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 7. = .point = *e543430, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 8. = .point = *10136c05, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 9. = .point = *102d847f, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 10. = .point = *1112f910, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 11. = .point = *1119a7c7, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 12. = .point = *14d95c44, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 13. = .point = *150fea1f, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 14. = .point = *1643ecb6, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 15. = .point = *189ff2f2, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 16. = .point = *19cfe9f3, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 17. = .point = *1e1c78c3, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 18. = .point = *1fe0dea0, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 19. = .point = *22464ee9, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 20. = .point = *22b35675, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 21. = .point = *2363bebb, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 22. = .point = *24f827bb, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 23. = .point = *259eeccf, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 24. = .point = *26f0c3e7, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 25. = .point = *271874d4, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 26. = .point = *28340f35, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 27. = .point = *285e8475, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 28. = .point = *28ec7a6f, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 29. = .point = *299c6298, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 30. = .point = *2aedc3f7, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 31. = .point = *2b031742, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 32. = .point = *2da0e37b, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 33. = .point = *310bd2ca, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 34. = .point = *31e5f2df, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 35. = .point = *32d6b3ed, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 36. = .point = *33047373, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 37. = .point = *3392487a, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 38. = .point = *37597c4c, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 39. = .point = *3f6b2b89, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 40. = .point = *43cf6426, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 41. = .point = *46a58f28, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 42. = .point = *4b1f5b22, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 43. = .point = *523723f2, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 44. = .point = *539234db, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 45. = .point = *564ca84f, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 46. = .point = *58501380, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 47. = .point = *58704432, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 48. = .point = *5b1bcbbe, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 49. = .point = *5d2df428, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 50. = .point = *5fa294ee, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 51. = .point = *606fd878, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 52. = .point = *60dded53, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 53. = .point = *616cdb68, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 54. = .point = *6257bc27, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 55. = .point = *64014b25, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 56. = .point = *6918f467, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 57. = .point = *6a08c380, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 58. = .point = *6bfd5a2d, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 59. = .point = *6c0b607a, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 60. = .point = *6c74d296, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 61. = .point = *6e040182, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 62. = .point = *6e3819f7, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 63. = .point = *720ec1a4, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 64. = .point = *7232b381, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 65. = .point = *74c384ad, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 66. = .point = *76d47350, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 67. = .point = *791eb3a3, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 68. = .point = *7a048f20, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 69. = .point = *7f874929, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 70. = .point = *83ce71ce, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 71. = .point = *888b6447, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 72. = .point = *8997c018, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 73. = .point = *89b7d09c, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 74. = .point = *8aa6b5b4, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 75. = .point = *8ae34bde, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 76. = .point = *8b382e03, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 77. = .point = *8b47e6ac, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 78. = .point = *8bc76115, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 79. = .point = *8bc8bc11, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 80. = .point = *8e2d3849, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 81. = .point = *8e7e012c, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 82. = .point = *8f5b4c63, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 83. = .point = *94a94162, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 84. = .point = *99892987, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 85. = .point = *9a6f2f00, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 86. = .point = *9b970b49, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 87. = .point = *9e09a3a7, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 88. = .point = *9ef9125d, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 89. = .point = *9f33cd30, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 90. = .point = *9fc69b51, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 91. = .point = *a19f99eb, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 92. = .point = *a28b9595, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 93. = .point = *a3582038, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 94. = .point = *a4b6a3b9, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 95. = .point = *a66da9cb, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 96. = .point = *a8657c76, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 97. = .point = *a8afe9c4, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 98. = .point = *aa488703, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 99. = .point = *ac7b4454, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 100. = .point = *ad923ad3, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 101. = .point = *ae8946c6, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 102. = .point = *b197e339, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 103. = .point = *b3c305e6, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 104. = .point = *b4dab004, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 105. = .point = *b6bf43ea, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 106. = .point = *b9004d3d, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 107. = .point = *b96b6455, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 108. = .point = *b9a0edb9, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 109. = .point = *b9ec6465, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 110. = .point = *bb8eed4d, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 111. = .point = *bbcc0bad, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 112. = .point = *bcfea141, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 113. = .point = *be300622, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 114. = .point = *bf514d68, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 115. = .point = *c1afc7d2, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 116. = .point = *c2542a5d, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 117. = .point = *c6c43fa7, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 118. = .point = *c945958a, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 119. = .point = *c9f304a4, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 120. = .point = *cb896aa8, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 121. = .point = *cbd9198a, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 122. = .point = *ccd61dad, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 123. = .point = *d07e4431, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 124. = .point = *d21fe35f, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 125. = .point = *d4c93105, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 126. = .point = *d570b815, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 127. = .point = *d7de63b6, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 128. = .point = *d8634aef, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 129. = .point = *d92d916d, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 130. = .point = *d937a7df, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 131. = .point = *dac52229, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 132. = .point = *db7840f0, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 133. = .point = *dd5c6bef, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 134. = .point = *dded5798, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 135. = .point = *dfd5333b, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 136. = .point = *e183345a, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 137. = .point = *e2c71c27, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 138. = .point = *e49bf9d8, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 139. = .point = *e72bc224, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 140. = .point = *e8b27f41, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 141. = .point = *e991584c, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 142. = .point = *ea201c5e, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 143. = .point = *ec8891c5, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 144. = .point = *edcc8dd9, host =  1.}
+	expect 0 =    Debug {^shard: hashcircle.* 145. = .point = *ef6b4ab5, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 146. = .point = *f08ad325, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 147. = .point = *f3325ba2, host =  2.}
+	expect 0 =    Debug {^shard: hashcircle.* 148. = .point = *f6530dd1, host =  0.}
+	expect 0 =    Debug {^shard: hashcircle.* 149. = .point = *fc28e8d2, host =  2.}
+
+	expect 0 =    CLI   Loaded
+
+	expect * =    Debug {^shard: lookup key 564ca84f idx 45 host 0}
+	expect * =    Debug {^shard: lookup key 19cfe9f3 idx 16 host 1}
+	expect * =    Debug {^shard: lookup key 46a58f28 idx 41 host 2}
+} -start
+
+client c1 {
+	txreq -url /564ca84f
+	rxresp
+	expect resp.body == "ech3Ooj"
+
+	txreq -url /19cfe9f3
+	rxresp
+	expect resp.body == "ieQu2qua"
+
+	txreq -url /46a58f28
+	rxresp
+	expect resp.body == "xiuFi3Pe"
+} -run
+
+logexpect l1 -wait
diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c
index 6276e1728..3f478ecf1 100644
--- a/lib/libvmod_directors/shard_cfg.c
+++ b/lib/libvmod_directors/shard_cfg.c
@@ -55,6 +55,7 @@ struct shard_change_task {
 #define SHARD_CHANGE_TASK_MAGIC			0x1e1168af
 	enum shard_change_task_e		task;
 	void					*priv;
+	VCL_REAL				weight;
 	VSTAILQ_ENTRY(shard_change_task)	list;
 };
 
@@ -127,7 +128,7 @@ shard_change_finish(struct shard_change *change)
 	VSTAILQ_INIT(&change->tasks);
 }
 
-static void
+static struct shard_change_task *
 shard_change_task_add(VRT_CTX, struct shard_change *change,
     enum shard_change_task_e task_e, void *priv)
 {
@@ -139,15 +140,17 @@ shard_change_task_add(VRT_CTX, struct shard_change *change,
 	if (task == NULL) {
 		shard_err0(ctx, change->shardd,
 		    "could not get workspace for task");
-		return;
+		return (NULL);
 	}
 	INIT_OBJ(task, SHARD_CHANGE_TASK_MAGIC);
 	task->task = task_e;
 	task->priv = priv;
 	VSTAILQ_INSERT_TAIL(&change->tasks, task, list);
+
+	return (task);
 }
 
-static inline VCL_BOOL
+static inline struct shard_change_task *
 shard_change_task_backend(VRT_CTX,
     struct vmod_priv *priv, const struct sharddir *shardd,
     enum shard_change_task_e task_e, VCL_BACKEND be, VCL_STRING ident,
@@ -161,22 +164,20 @@ shard_change_task_backend(VRT_CTX,
 
 	change = shard_change_get(ctx, priv, shardd);
 	if (change == NULL)
-		return (0);
+		return (NULL);
 
 	b = WS_Alloc(ctx->ws, sizeof(*b));
 	if (b == NULL) {
 		shard_err(ctx, shardd, ".%s_backend() WS_Alloc() failed",
 		    task_e == ADD_BE ? "add" : "remove");
-		return (0);
+		return (NULL);
 	}
 
 	b->backend = be;
 	b->ident = ident != NULL && *ident != '\0' ? ident : NULL;
 	b->rampup = rampup;
 
-	shard_change_task_add(ctx, change, task_e, b);
-
-	return (1);
+	return (shard_change_task_add(ctx, change, task_e, b));
 }
 
 /*
@@ -186,11 +187,21 @@ shard_change_task_backend(VRT_CTX,
 VCL_BOOL
 shardcfg_add_backend(VRT_CTX, struct vmod_priv *priv,
     const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident,
-    VCL_DURATION rampup)
+    VCL_DURATION rampup, VCL_REAL weight)
 {
+	struct shard_change_task *task;
+
+	assert (weight >= 1);
 	AN(be);
-	return (shard_change_task_backend(ctx, priv, shardd, ADD_BE,
-	    be, ident, rampup));
+
+	task = shard_change_task_backend(ctx, priv, shardd, ADD_BE,
+	    be, ident, rampup);
+
+	if (task == NULL)
+		return (0);
+
+	task->weight = weight;
+	return (1);
 }
 
 VCL_BOOL
@@ -198,7 +209,7 @@ shardcfg_remove_backend(VRT_CTX, struct vmod_priv *priv,
     const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident)
 {
 	return (shard_change_task_backend(ctx, priv, shardd, REMOVE_BE,
-	    be, ident, 0));
+	    be, ident, 0) != NULL);
 }
 
 VCL_BOOL
@@ -212,9 +223,7 @@ shardcfg_clear(VRT_CTX, struct vmod_priv *priv, const struct sharddir *shardd)
 	if (change == NULL)
 		return (0);
 
-	shard_change_task_add(ctx, change, CLEAR, NULL);
-
-	return (1);
+	return (shard_change_task_add(ctx, change, CLEAR, NULL) != NULL);
 }
 
 /*
@@ -232,9 +241,11 @@ circlepoint_compare(const struct shard_circlepoint *a,
 }
 
 static void
-shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas)
+shardcfg_hashcircle(struct sharddir *shardd)
 {
-	int i, j;
+	const struct shard_backend *backends, *b;
+	int j, h;
+	uint32_t i, n_points, r, rmax;
 	const char *ident;
 	const int len = 12; // log10(UINT32_MAX) + 2;
 	char s[len];
@@ -245,49 +256,60 @@ shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas)
 	AZ(shardd->hashcircle);
 
 	assert(shardd->n_backend > 0);
-	AN(shardd->backend);
-
-	shardd->hashcircle = calloc(shardd->n_backend * replicas,
-		sizeof(struct shard_circlepoint));
-	AN(shardd->hashcircle);
+	backends=shardd->backend;
+	AN(backends);
+
+	n_points = 0;
+	rmax = (UINT32_MAX - 1) / shardd->n_backend;
+	for (b = backends; b < backends + shardd->n_backend; b++) {
+		CHECK_OBJ_NOTNULL(b->backend, DIRECTOR_MAGIC);
+		r = b->replicas;
+		if (r > rmax)
+			r = rmax;
+		n_points += r;
+	}
 
-	shardd->replicas = replicas;
+	assert(n_points < UINT32_MAX);
 
-	for (i = 0; i < shardd->n_backend; i++) {
-		CHECK_OBJ_NOTNULL(shardd->backend[i].backend, DIRECTOR_MAGIC);
+	shardd->n_points = n_points;
+	shardd->hashcircle = calloc(n_points, sizeof(struct shard_circlepoint));
+	AN(shardd->hashcircle);
 
-		ident = shardd->backend[i].ident
-		    ? shardd->backend[i].ident
-		    : VRT_BACKEND_string(shardd->backend[i].backend);
+	i = 0;
+	for (h = 0, b = backends; h < shardd->n_backend; h++, b++) {
+		ident = b->ident ? b->ident : VRT_BACKEND_string(b->backend);
 
 		AN(ident);
 		assert(ident[0] != '\0');
 
-		for (j = 0; j < replicas; j++) {
+		r = b->replicas;
+		if (r > rmax)
+			r = rmax;
+
+		for (j = 0; j < r; j++) {
 			assert(snprintf(s, len, "%d", j) < len);
 			ss->n = 2;
 			ssp[0] = ident;
 			ssp[1] = s;
 			ss->p = ssp;
-			shardd->hashcircle[i * replicas + j].point =
-			    VRT_HashStrands32(ss);
-			shardd->hashcircle[i * replicas + j].host = i;
+			assert (i < n_points);
+			shardd->hashcircle[i].point = VRT_HashStrands32(ss);
+			shardd->hashcircle[i].host = h;
+			i++;
 		}
 	}
-	qsort( (void *) shardd->hashcircle, shardd->n_backend * replicas,
+	assert (i == n_points);
+	qsort( (void *) shardd->hashcircle, n_points,
 	    sizeof (struct shard_circlepoint), (compar) circlepoint_compare);
 
 	if ((shardd->debug_flags & SHDBG_CIRCLE) == 0)
 		return;
 
-	for (i = 0; i < shardd->n_backend; i++)
-		for (j = 0; j < replicas; j++)
-			SHDBG(SHDBG_CIRCLE, shardd,
-			    "hashcircle[%5jd] = "
-			    "{point = %8x, host = %2u}\n",
-			    (intmax_t)(i * replicas + j),
-			    shardd->hashcircle[i * replicas + j].point,
-			    shardd->hashcircle[i * replicas + j].host);
+	for (i = 0; i < n_points; i++)
+		SHDBG(SHDBG_CIRCLE, shardd,
+		    "hashcircle[%5jd] = {point = %8x, host = %2u}\n",
+		    (intmax_t)i, shardd->hashcircle[i].point,
+		    shardd->hashcircle[i].host);
 }
 
 /*
@@ -394,7 +416,7 @@ shardcfg_backend_expand(const struct backend_reconfig *re)
 
 static void
 shardcfg_backend_add(struct backend_reconfig *re,
-    const struct shard_backend *b)
+    const struct shard_backend *b, uint32_t replicas)
 {
 	unsigned i;
 	struct shard_backend *bb = re->shardd->backend;
@@ -419,6 +441,7 @@ shardcfg_backend_add(struct backend_reconfig *re,
 
 	re->shardd->n_backend++;
 	shardcfg_backend_copyin(&bb[i], b);
+	bb[i].replicas = replicas;
 }
 
 static void
@@ -499,10 +522,11 @@ shardcfg_backend_finalize(struct backend_reconfig *re)
 
 static void
 shardcfg_apply_change(VRT_CTX, struct sharddir *shardd,
-    const struct shard_change *change)
+    const struct shard_change *change, VCL_INT replicas)
 {
 	struct shard_change_task *task, *clear;
 	const struct shard_backend *b;
+	uint32_t b_replicas;
 
 	struct backend_reconfig re = {
 		.shardd = shardd,
@@ -550,7 +574,14 @@ shardcfg_apply_change(VRT_CTX, struct sharddir *shardd,
 			b = shardcfg_backend_lookup(&re, task->priv);
 
 			if (b == NULL) {
-				shardcfg_backend_add(&re, task->priv);
+				assert (task->weight >= 1);
+				if (replicas * task->weight > UINT32_MAX)
+					b_replicas = UINT32_MAX;
+				else
+					b_replicas = replicas * task->weight;
+
+				shardcfg_backend_add(&re, task->priv,
+				    b_replicas);
 				break;
 			}
 
@@ -599,7 +630,7 @@ shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv,
 
 	sharddir_wrlock(shardd);
 
-	shardcfg_apply_change(ctx, shardd, change);
+	shardcfg_apply_change(ctx, shardd, change, replicas);
 	shard_change_finish(change);
 
 	if (shardd->hashcircle)
@@ -612,7 +643,7 @@ shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv,
 		return (0);
 	}
 
-	shardcfg_hashcircle(shardd, replicas);
+	shardcfg_hashcircle(shardd);
 	sharddir_unlock(shardd);
 	return (1);
 }
diff --git a/lib/libvmod_directors/shard_cfg.h b/lib/libvmod_directors/shard_cfg.h
index 5c6f1e6b6..0b8b8612c 100644
--- a/lib/libvmod_directors/shard_cfg.h
+++ b/lib/libvmod_directors/shard_cfg.h
@@ -30,7 +30,7 @@
 
 VCL_BOOL shardcfg_add_backend(VRT_CTX, struct vmod_priv *priv,
     const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident,
-    VCL_DURATION rampup);
+    VCL_DURATION rampup, VCL_REAL weight);
 VCL_BOOL shardcfg_remove_backend(VRT_CTX, struct vmod_priv *priv,
     const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident);
 VCL_BOOL shardcfg_clear(VRT_CTX, struct vmod_priv *priv,
diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index 850aeebcf..001286011 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -60,7 +60,7 @@ struct shard_be_info {
 struct shard_state {
 	const struct vrt_ctx	*ctx;
 	struct sharddir	*shardd;
-	int			idx;
+	uint32_t		idx;
 
 	struct vbitmap		*picklist;
 	int			pickcount;
@@ -94,8 +94,10 @@ shard_lookup(const struct sharddir *shardd, const uint32_t key)
 {
 	CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC);
 
-	const int n = shardd->n_backend * shardd->replicas;
-	int idx = -1, high = n, low = 0, i;
+	const uint32_t n = shardd->n_points;
+	uint32_t i, idx = UINT32_MAX, high = n, low = 0;
+
+	assert (n < idx);
 
 	do {
 	    i = (high + low) / 2 ;
@@ -113,7 +115,7 @@ shard_lookup(const struct sharddir *shardd, const uint32_t key)
 		    high = i;
 	    else
 		low = i;
-	} while (idx == -1);
+	} while (idx == UINT32_MAX);
 
 	return (idx);
 }
@@ -122,7 +124,6 @@ static int
 shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy)
 {
 	int c, chosen = -1;
-	uint32_t ringsz;
 	VCL_BACKEND be;
 	vtim_real changed;
 	struct shard_be_info *sbe;
@@ -134,8 +135,6 @@ shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy)
 	if (state->pickcount >= state->shardd->n_backend)
 		return (-1);
 
-	ringsz = state->shardd->n_backend * state->shardd->replicas;
-
 	while (state->pickcount < state->shardd->n_backend && skip >= 0) {
 
 		c = state->shardd->hashcircle[state->idx].host;
@@ -174,7 +173,7 @@ shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy)
 				break;
 		}
 
-		if (++(state->idx) == ringsz)
+		if (++(state->idx) == state->shardd->n_points)
 			state->idx = 0;
 	}
 	return (chosen);
diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h
index 03df1c312..3ad305180 100644
--- a/lib/libvmod_directors/shard_dir.h
+++ b/lib/libvmod_directors/shard_dir.h
@@ -43,6 +43,7 @@ struct shard_backend {
 		void		*freeptr;
 	};
 	VCL_DURATION		rampup;
+	uint32_t		replicas;
 };
 
 struct vmod_directors_shard_param;
@@ -68,7 +69,8 @@ struct sharddir {
 
 	VCL_DURATION				rampup_duration;
 	VCL_REAL				warmup;
-	VCL_INT					replicas;
+
+	uint32_t				n_points;
 };
 
 static inline VCL_BACKEND
diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
index 054b6c34e..8ab9f12fa 100644
--- a/lib/libvmod_directors/vmod.vcc
+++ b/lib/libvmod_directors/vmod.vcc
@@ -373,7 +373,7 @@ The association can be changed per backend request using the *param*
 argument of `xshard.backend()`_.
 
 $Method BOOL .add_backend(PRIV_TASK, BACKEND backend,
-	[STRING ident], [DURATION rampup])
+	[STRING ident], [DURATION rampup], [REAL weight])
 
 Add a backend *backend* to the director.
 
@@ -388,6 +388,12 @@ defaults to the backend name.
 backend. Otherwise, the per-director rampup time is used (see
 `xshard.set_rampup()`_).
 
+*weight*: Optionally specify a weight to scale the
+`xshard.reconfigure()`_ *replicas* parameter. *weight* is limited to
+at least 1. Values above 10 probably do not make much sense. The
+effect of *weight* is also capped such that the total number of
+replicas does not exceed `UINT32_MAX`.
+
 NOTE: Backend changes need to be finalized with
 `xshard.reconfigure()`_ and are only supported on one
 shard director at a time.
diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c
index c3ec5d837..8a8bda9a2 100644
--- a/lib/libvmod_directors/vmod_shard.c
+++ b/lib/libvmod_directors/vmod_shard.c
@@ -305,6 +305,8 @@ VCL_BOOL v_matchproto_(td_directors_shard_add_backend)
 vmod_shard_add_backend(VRT_CTX, struct vmod_directors_shard *vshard,
     struct VARGS(shard_add_backend) *args)
 {
+	VCL_REAL weight = 1;
+
 	CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC);
 
 	if (args->backend == NULL) {
@@ -313,10 +315,14 @@ vmod_shard_add_backend(VRT_CTX, struct vmod_directors_shard *vshard,
 		return (0);
 	}
 
+	if (args->valid_weight && args->weight > 1)
+		weight = args->weight;
+
 	return shardcfg_add_backend(ctx, args->arg1,
 	    vshard->shardd, args->backend,
 	    args->valid_ident ? args->ident : NULL,
-	    args->valid_rampup ? args->rampup : nan(""));
+	    args->valid_rampup ? args->rampup : nan(""),
+	    weight);
 }
 
 VCL_BOOL v_matchproto_(td_directors_shard_remove_backend)

From phk at FreeBSD.org  Tue Jun  9 20:30:11 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  9 Jun 2020 20:30:11 +0000 (UTC)
Subject: [master] c20ad2982 ->idx is unsigned, so cannot be < 0 (and GCC sux).
Message-ID: <20200609203011.25AEFA72CA@lists.varnish-cache.org>


commit c20ad298207735c5aeb0cadd96b147a7fa789fd0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 9 20:28:58 2020 +0000

    ->idx is unsigned, so cannot be < 0 (and GCC sux).

diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index 001286011..63f6c22ab 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -129,7 +129,6 @@ shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy)
 	struct shard_be_info *sbe;
 
 	AN(state);
-	assert(state->idx >= 0);
 	CHECK_OBJ_NOTNULL(state->shardd, SHARDDIR_MAGIC);
 
 	if (state->pickcount >= state->shardd->n_backend)

From dridi at varni.sh  Tue Jun  9 20:43:06 2020
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Tue, 9 Jun 2020 20:43:06 +0000
Subject: [master] b90b60d0a shard director: add optional weight parameter
 to .add_backend()
In-Reply-To: <20200609170307.79A6CA212A@lists.varnish-cache.org>
References: <20200609170307.79A6CA212A@lists.varnish-cache.org>
Message-ID: <CABoVN9Cev0bCFgmaFUFqDZ2ixkfq-UhoHQ+9ufi34ZA0C8_caw@mail.gmail.com>

On Tue, Jun 9, 2020 at 5:03 PM Nils Goroll <nils.goroll at uplex.de> wrote:
>
>
> commit b90b60d0a066f1aa7302a70125cb91cdfa605119
> Author: Nils Goroll <nils.goroll at uplex.de>
> Date:   Tue Jun 9 19:02:00 2020 +0200
>
>     shard director: add optional weight parameter to .add_backend()
>
>     We implement weights by scaling the number of replicas of each backend.
>     The replicas parameter of .reconfigure() remains a minimum.
>
>     For existing vtcs, the Debug hashcircle output has been compared
>     before/after this change to ensure that behaviour is exactly equivalent.
>
>     For for wighted backends, it has been checked that the number of
>     instances per host on the hashcircle matches the expectation.
>
>     Also refactor and clean up some of the code:
>
>     - consistently make the number of ring points a uint32_t
>     - some constification
>
>     Ref #3276
>
> diff --git a/bin/varnishtest/tests/d00041.vtc b/bin/varnishtest/tests/d00041.vtc
> new file mode 100644
> index 000000000..bd79cfff1
> --- /dev/null
> +++ b/bin/varnishtest/tests/d00041.vtc
> @@ -0,0 +1,224 @@
> +varnishtest "d00017.vtc but with weights"
> +
> +server s1 {
> +       rxreq
> +       txresp -body "ech3Ooj"
> +} -start
> +
> +server s2 {
> +       rxreq
> +       txresp -body "ieQu2qua"
> +} -start
> +
> +server s3 {
> +       rxreq
> +       txresp -body "xiuFi3Pe"
> +} -start
> +
> +varnish v1 -vcl+backend {
> +       import std;
> +       import directors;
> +       import blob;
> +
> +       sub vcl_init {
> +               new vd = directors.shard();
> +               vd.debug(3);
> +               if (!vd.add_backend(s1)) {
> +                       std.log("add s1 failed");
> +               }
> +               if (!vd.add_backend(s2, weight=2)) {
> +                       std.log("add s2 failed");
> +               }
> +               if (!vd.add_backend(s3, weight=3)) {
> +                       std.log("add s3 failed");
> +               }
> +               if (!vd.reconfigure(replicas=25)) {
> +                       std.log("reconfigure failed");

Any reason not to return(fail("reason")) instead of std.log() calls?

> +               }
> +       }
> +
> +       sub vcl_recv {
> +               set req.backend_hint = vd.backend(by=BLOB,
> +                   key_blob=blob.decode(HEX, encoded=
> +                       regsub(req.url, "^/", "")));
> +               return(pass);
> +       }
> +
> +} -start
> +
> +logexpect l1 -v v1 -g raw -d 1 {
> +       expect 0 0    CLI   "^Rd vcl.load"
> +
> +       expect 0 =    Debug {^shard: hashcircle.* 0. = .point = *238d0ef, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 1. = .point = *321c598, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 2. = .point = *3b6b56a, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 3. = .point = *408ec1e, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 4. = .point = *66986a7, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 5. = .point = *7e41e30, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 6. = .point = *b749e7b, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 7. = .point = *e543430, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 8. = .point = *10136c05, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 9. = .point = *102d847f, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 10. = .point = *1112f910, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 11. = .point = *1119a7c7, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 12. = .point = *14d95c44, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 13. = .point = *150fea1f, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 14. = .point = *1643ecb6, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 15. = .point = *189ff2f2, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 16. = .point = *19cfe9f3, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 17. = .point = *1e1c78c3, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 18. = .point = *1fe0dea0, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 19. = .point = *22464ee9, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 20. = .point = *22b35675, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 21. = .point = *2363bebb, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 22. = .point = *24f827bb, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 23. = .point = *259eeccf, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 24. = .point = *26f0c3e7, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 25. = .point = *271874d4, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 26. = .point = *28340f35, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 27. = .point = *285e8475, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 28. = .point = *28ec7a6f, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 29. = .point = *299c6298, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 30. = .point = *2aedc3f7, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 31. = .point = *2b031742, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 32. = .point = *2da0e37b, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 33. = .point = *310bd2ca, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 34. = .point = *31e5f2df, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 35. = .point = *32d6b3ed, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 36. = .point = *33047373, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 37. = .point = *3392487a, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 38. = .point = *37597c4c, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 39. = .point = *3f6b2b89, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 40. = .point = *43cf6426, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 41. = .point = *46a58f28, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 42. = .point = *4b1f5b22, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 43. = .point = *523723f2, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 44. = .point = *539234db, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 45. = .point = *564ca84f, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 46. = .point = *58501380, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 47. = .point = *58704432, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 48. = .point = *5b1bcbbe, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 49. = .point = *5d2df428, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 50. = .point = *5fa294ee, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 51. = .point = *606fd878, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 52. = .point = *60dded53, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 53. = .point = *616cdb68, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 54. = .point = *6257bc27, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 55. = .point = *64014b25, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 56. = .point = *6918f467, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 57. = .point = *6a08c380, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 58. = .point = *6bfd5a2d, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 59. = .point = *6c0b607a, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 60. = .point = *6c74d296, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 61. = .point = *6e040182, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 62. = .point = *6e3819f7, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 63. = .point = *720ec1a4, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 64. = .point = *7232b381, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 65. = .point = *74c384ad, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 66. = .point = *76d47350, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 67. = .point = *791eb3a3, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 68. = .point = *7a048f20, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 69. = .point = *7f874929, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 70. = .point = *83ce71ce, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 71. = .point = *888b6447, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 72. = .point = *8997c018, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 73. = .point = *89b7d09c, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 74. = .point = *8aa6b5b4, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 75. = .point = *8ae34bde, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 76. = .point = *8b382e03, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 77. = .point = *8b47e6ac, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 78. = .point = *8bc76115, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 79. = .point = *8bc8bc11, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 80. = .point = *8e2d3849, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 81. = .point = *8e7e012c, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 82. = .point = *8f5b4c63, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 83. = .point = *94a94162, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 84. = .point = *99892987, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 85. = .point = *9a6f2f00, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 86. = .point = *9b970b49, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 87. = .point = *9e09a3a7, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 88. = .point = *9ef9125d, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 89. = .point = *9f33cd30, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 90. = .point = *9fc69b51, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 91. = .point = *a19f99eb, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 92. = .point = *a28b9595, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 93. = .point = *a3582038, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 94. = .point = *a4b6a3b9, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 95. = .point = *a66da9cb, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 96. = .point = *a8657c76, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 97. = .point = *a8afe9c4, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 98. = .point = *aa488703, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 99. = .point = *ac7b4454, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 100. = .point = *ad923ad3, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 101. = .point = *ae8946c6, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 102. = .point = *b197e339, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 103. = .point = *b3c305e6, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 104. = .point = *b4dab004, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 105. = .point = *b6bf43ea, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 106. = .point = *b9004d3d, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 107. = .point = *b96b6455, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 108. = .point = *b9a0edb9, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 109. = .point = *b9ec6465, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 110. = .point = *bb8eed4d, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 111. = .point = *bbcc0bad, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 112. = .point = *bcfea141, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 113. = .point = *be300622, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 114. = .point = *bf514d68, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 115. = .point = *c1afc7d2, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 116. = .point = *c2542a5d, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 117. = .point = *c6c43fa7, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 118. = .point = *c945958a, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 119. = .point = *c9f304a4, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 120. = .point = *cb896aa8, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 121. = .point = *cbd9198a, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 122. = .point = *ccd61dad, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 123. = .point = *d07e4431, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 124. = .point = *d21fe35f, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 125. = .point = *d4c93105, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 126. = .point = *d570b815, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 127. = .point = *d7de63b6, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 128. = .point = *d8634aef, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 129. = .point = *d92d916d, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 130. = .point = *d937a7df, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 131. = .point = *dac52229, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 132. = .point = *db7840f0, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 133. = .point = *dd5c6bef, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 134. = .point = *dded5798, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 135. = .point = *dfd5333b, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 136. = .point = *e183345a, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 137. = .point = *e2c71c27, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 138. = .point = *e49bf9d8, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 139. = .point = *e72bc224, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 140. = .point = *e8b27f41, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 141. = .point = *e991584c, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 142. = .point = *ea201c5e, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 143. = .point = *ec8891c5, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 144. = .point = *edcc8dd9, host =  1.}
> +       expect 0 =    Debug {^shard: hashcircle.* 145. = .point = *ef6b4ab5, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 146. = .point = *f08ad325, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 147. = .point = *f3325ba2, host =  2.}
> +       expect 0 =    Debug {^shard: hashcircle.* 148. = .point = *f6530dd1, host =  0.}
> +       expect 0 =    Debug {^shard: hashcircle.* 149. = .point = *fc28e8d2, host =  2.}
> +
> +       expect 0 =    CLI   Loaded
> +
> +       expect * =    Debug {^shard: lookup key 564ca84f idx 45 host 0}
> +       expect * =    Debug {^shard: lookup key 19cfe9f3 idx 16 host 1}
> +       expect * =    Debug {^shard: lookup key 46a58f28 idx 41 host 2}
> +} -start
> +
> +client c1 {
> +       txreq -url /564ca84f
> +       rxresp
> +       expect resp.body == "ech3Ooj"
> +
> +       txreq -url /19cfe9f3
> +       rxresp
> +       expect resp.body == "ieQu2qua"
> +
> +       txreq -url /46a58f28
> +       rxresp
> +       expect resp.body == "xiuFi3Pe"
> +} -run
> +
> +logexpect l1 -wait
> diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c
> index 6276e1728..3f478ecf1 100644
> --- a/lib/libvmod_directors/shard_cfg.c
> +++ b/lib/libvmod_directors/shard_cfg.c
> @@ -55,6 +55,7 @@ struct shard_change_task {
>  #define SHARD_CHANGE_TASK_MAGIC                        0x1e1168af
>         enum shard_change_task_e                task;
>         void                                    *priv;
> +       VCL_REAL                                weight;
>         VSTAILQ_ENTRY(shard_change_task)        list;
>  };
>
> @@ -127,7 +128,7 @@ shard_change_finish(struct shard_change *change)
>         VSTAILQ_INIT(&change->tasks);
>  }
>
> -static void
> +static struct shard_change_task *
>  shard_change_task_add(VRT_CTX, struct shard_change *change,
>      enum shard_change_task_e task_e, void *priv)
>  {
> @@ -139,15 +140,17 @@ shard_change_task_add(VRT_CTX, struct shard_change *change,
>         if (task == NULL) {
>                 shard_err0(ctx, change->shardd,
>                     "could not get workspace for task");
> -               return;
> +               return (NULL);
>         }
>         INIT_OBJ(task, SHARD_CHANGE_TASK_MAGIC);
>         task->task = task_e;
>         task->priv = priv;
>         VSTAILQ_INSERT_TAIL(&change->tasks, task, list);
> +
> +       return (task);
>  }
>
> -static inline VCL_BOOL
> +static inline struct shard_change_task *
>  shard_change_task_backend(VRT_CTX,
>      struct vmod_priv *priv, const struct sharddir *shardd,
>      enum shard_change_task_e task_e, VCL_BACKEND be, VCL_STRING ident,
> @@ -161,22 +164,20 @@ shard_change_task_backend(VRT_CTX,
>
>         change = shard_change_get(ctx, priv, shardd);
>         if (change == NULL)
> -               return (0);
> +               return (NULL);
>
>         b = WS_Alloc(ctx->ws, sizeof(*b));
>         if (b == NULL) {
>                 shard_err(ctx, shardd, ".%s_backend() WS_Alloc() failed",
>                     task_e == ADD_BE ? "add" : "remove");
> -               return (0);
> +               return (NULL);
>         }
>
>         b->backend = be;
>         b->ident = ident != NULL && *ident != '\0' ? ident : NULL;
>         b->rampup = rampup;
>
> -       shard_change_task_add(ctx, change, task_e, b);
> -
> -       return (1);
> +       return (shard_change_task_add(ctx, change, task_e, b));
>  }
>
>  /*
> @@ -186,11 +187,21 @@ shard_change_task_backend(VRT_CTX,
>  VCL_BOOL
>  shardcfg_add_backend(VRT_CTX, struct vmod_priv *priv,
>      const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident,
> -    VCL_DURATION rampup)
> +    VCL_DURATION rampup, VCL_REAL weight)
>  {
> +       struct shard_change_task *task;
> +
> +       assert (weight >= 1);
>         AN(be);
> -       return (shard_change_task_backend(ctx, priv, shardd, ADD_BE,
> -           be, ident, rampup));
> +
> +       task = shard_change_task_backend(ctx, priv, shardd, ADD_BE,
> +           be, ident, rampup);
> +
> +       if (task == NULL)
> +               return (0);
> +
> +       task->weight = weight;
> +       return (1);
>  }
>
>  VCL_BOOL
> @@ -198,7 +209,7 @@ shardcfg_remove_backend(VRT_CTX, struct vmod_priv *priv,
>      const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident)
>  {
>         return (shard_change_task_backend(ctx, priv, shardd, REMOVE_BE,
> -           be, ident, 0));
> +           be, ident, 0) != NULL);
>  }
>
>  VCL_BOOL
> @@ -212,9 +223,7 @@ shardcfg_clear(VRT_CTX, struct vmod_priv *priv, const struct sharddir *shardd)
>         if (change == NULL)
>                 return (0);
>
> -       shard_change_task_add(ctx, change, CLEAR, NULL);
> -
> -       return (1);
> +       return (shard_change_task_add(ctx, change, CLEAR, NULL) != NULL);
>  }
>
>  /*
> @@ -232,9 +241,11 @@ circlepoint_compare(const struct shard_circlepoint *a,
>  }
>
>  static void
> -shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas)
> +shardcfg_hashcircle(struct sharddir *shardd)
>  {
> -       int i, j;
> +       const struct shard_backend *backends, *b;
> +       int j, h;
> +       uint32_t i, n_points, r, rmax;
>         const char *ident;
>         const int len = 12; // log10(UINT32_MAX) + 2;
>         char s[len];
> @@ -245,49 +256,60 @@ shardcfg_hashcircle(struct sharddir *shardd, VCL_INT replicas)
>         AZ(shardd->hashcircle);
>
>         assert(shardd->n_backend > 0);
> -       AN(shardd->backend);
> -
> -       shardd->hashcircle = calloc(shardd->n_backend * replicas,
> -               sizeof(struct shard_circlepoint));
> -       AN(shardd->hashcircle);
> +       backends=shardd->backend;
> +       AN(backends);
> +
> +       n_points = 0;
> +       rmax = (UINT32_MAX - 1) / shardd->n_backend;
> +       for (b = backends; b < backends + shardd->n_backend; b++) {
> +               CHECK_OBJ_NOTNULL(b->backend, DIRECTOR_MAGIC);
> +               r = b->replicas;
> +               if (r > rmax)
> +                       r = rmax;
> +               n_points += r;
> +       }
>
> -       shardd->replicas = replicas;
> +       assert(n_points < UINT32_MAX);
>
> -       for (i = 0; i < shardd->n_backend; i++) {
> -               CHECK_OBJ_NOTNULL(shardd->backend[i].backend, DIRECTOR_MAGIC);
> +       shardd->n_points = n_points;
> +       shardd->hashcircle = calloc(n_points, sizeof(struct shard_circlepoint));
> +       AN(shardd->hashcircle);
>
> -               ident = shardd->backend[i].ident
> -                   ? shardd->backend[i].ident
> -                   : VRT_BACKEND_string(shardd->backend[i].backend);
> +       i = 0;
> +       for (h = 0, b = backends; h < shardd->n_backend; h++, b++) {
> +               ident = b->ident ? b->ident : VRT_BACKEND_string(b->backend);
>
>                 AN(ident);
>                 assert(ident[0] != '\0');
>
> -               for (j = 0; j < replicas; j++) {
> +               r = b->replicas;
> +               if (r > rmax)
> +                       r = rmax;
> +
> +               for (j = 0; j < r; j++) {
>                         assert(snprintf(s, len, "%d", j) < len);
>                         ss->n = 2;
>                         ssp[0] = ident;
>                         ssp[1] = s;
>                         ss->p = ssp;
> -                       shardd->hashcircle[i * replicas + j].point =
> -                           VRT_HashStrands32(ss);
> -                       shardd->hashcircle[i * replicas + j].host = i;
> +                       assert (i < n_points);
> +                       shardd->hashcircle[i].point = VRT_HashStrands32(ss);
> +                       shardd->hashcircle[i].host = h;
> +                       i++;
>                 }
>         }
> -       qsort( (void *) shardd->hashcircle, shardd->n_backend * replicas,
> +       assert (i == n_points);
> +       qsort( (void *) shardd->hashcircle, n_points,
>             sizeof (struct shard_circlepoint), (compar) circlepoint_compare);
>
>         if ((shardd->debug_flags & SHDBG_CIRCLE) == 0)
>                 return;
>
> -       for (i = 0; i < shardd->n_backend; i++)
> -               for (j = 0; j < replicas; j++)
> -                       SHDBG(SHDBG_CIRCLE, shardd,
> -                           "hashcircle[%5jd] = "
> -                           "{point = %8x, host = %2u}\n",
> -                           (intmax_t)(i * replicas + j),
> -                           shardd->hashcircle[i * replicas + j].point,
> -                           shardd->hashcircle[i * replicas + j].host);
> +       for (i = 0; i < n_points; i++)
> +               SHDBG(SHDBG_CIRCLE, shardd,
> +                   "hashcircle[%5jd] = {point = %8x, host = %2u}\n",
> +                   (intmax_t)i, shardd->hashcircle[i].point,
> +                   shardd->hashcircle[i].host);
>  }
>
>  /*
> @@ -394,7 +416,7 @@ shardcfg_backend_expand(const struct backend_reconfig *re)
>
>  static void
>  shardcfg_backend_add(struct backend_reconfig *re,
> -    const struct shard_backend *b)
> +    const struct shard_backend *b, uint32_t replicas)
>  {
>         unsigned i;
>         struct shard_backend *bb = re->shardd->backend;
> @@ -419,6 +441,7 @@ shardcfg_backend_add(struct backend_reconfig *re,
>
>         re->shardd->n_backend++;
>         shardcfg_backend_copyin(&bb[i], b);
> +       bb[i].replicas = replicas;
>  }
>
>  static void
> @@ -499,10 +522,11 @@ shardcfg_backend_finalize(struct backend_reconfig *re)
>
>  static void
>  shardcfg_apply_change(VRT_CTX, struct sharddir *shardd,
> -    const struct shard_change *change)
> +    const struct shard_change *change, VCL_INT replicas)
>  {
>         struct shard_change_task *task, *clear;
>         const struct shard_backend *b;
> +       uint32_t b_replicas;
>
>         struct backend_reconfig re = {
>                 .shardd = shardd,
> @@ -550,7 +574,14 @@ shardcfg_apply_change(VRT_CTX, struct sharddir *shardd,
>                         b = shardcfg_backend_lookup(&re, task->priv);
>
>                         if (b == NULL) {
> -                               shardcfg_backend_add(&re, task->priv);
> +                               assert (task->weight >= 1);
> +                               if (replicas * task->weight > UINT32_MAX)
> +                                       b_replicas = UINT32_MAX;
> +                               else
> +                                       b_replicas = replicas * task->weight;
> +
> +                               shardcfg_backend_add(&re, task->priv,
> +                                   b_replicas);
>                                 break;
>                         }
>
> @@ -599,7 +630,7 @@ shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv,
>
>         sharddir_wrlock(shardd);
>
> -       shardcfg_apply_change(ctx, shardd, change);
> +       shardcfg_apply_change(ctx, shardd, change, replicas);
>         shard_change_finish(change);
>
>         if (shardd->hashcircle)
> @@ -612,7 +643,7 @@ shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv,
>                 return (0);
>         }
>
> -       shardcfg_hashcircle(shardd, replicas);
> +       shardcfg_hashcircle(shardd);
>         sharddir_unlock(shardd);
>         return (1);
>  }
> diff --git a/lib/libvmod_directors/shard_cfg.h b/lib/libvmod_directors/shard_cfg.h
> index 5c6f1e6b6..0b8b8612c 100644
> --- a/lib/libvmod_directors/shard_cfg.h
> +++ b/lib/libvmod_directors/shard_cfg.h
> @@ -30,7 +30,7 @@
>
>  VCL_BOOL shardcfg_add_backend(VRT_CTX, struct vmod_priv *priv,
>      const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident,
> -    VCL_DURATION rampup);
> +    VCL_DURATION rampup, VCL_REAL weight);
>  VCL_BOOL shardcfg_remove_backend(VRT_CTX, struct vmod_priv *priv,
>      const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident);
>  VCL_BOOL shardcfg_clear(VRT_CTX, struct vmod_priv *priv,
> diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
> index 850aeebcf..001286011 100644
> --- a/lib/libvmod_directors/shard_dir.c
> +++ b/lib/libvmod_directors/shard_dir.c
> @@ -60,7 +60,7 @@ struct shard_be_info {
>  struct shard_state {
>         const struct vrt_ctx    *ctx;
>         struct sharddir *shardd;
> -       int                     idx;
> +       uint32_t                idx;
>
>         struct vbitmap          *picklist;
>         int                     pickcount;
> @@ -94,8 +94,10 @@ shard_lookup(const struct sharddir *shardd, const uint32_t key)
>  {
>         CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC);
>
> -       const int n = shardd->n_backend * shardd->replicas;
> -       int idx = -1, high = n, low = 0, i;
> +       const uint32_t n = shardd->n_points;
> +       uint32_t i, idx = UINT32_MAX, high = n, low = 0;
> +
> +       assert (n < idx);
>
>         do {
>             i = (high + low) / 2 ;
> @@ -113,7 +115,7 @@ shard_lookup(const struct sharddir *shardd, const uint32_t key)
>                     high = i;
>             else
>                 low = i;
> -       } while (idx == -1);
> +       } while (idx == UINT32_MAX);
>
>         return (idx);
>  }
> @@ -122,7 +124,6 @@ static int
>  shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy)
>  {
>         int c, chosen = -1;
> -       uint32_t ringsz;
>         VCL_BACKEND be;
>         vtim_real changed;
>         struct shard_be_info *sbe;
> @@ -134,8 +135,6 @@ shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy)
>         if (state->pickcount >= state->shardd->n_backend)
>                 return (-1);
>
> -       ringsz = state->shardd->n_backend * state->shardd->replicas;
> -
>         while (state->pickcount < state->shardd->n_backend && skip >= 0) {
>
>                 c = state->shardd->hashcircle[state->idx].host;
> @@ -174,7 +173,7 @@ shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy)
>                                 break;
>                 }
>
> -               if (++(state->idx) == ringsz)
> +               if (++(state->idx) == state->shardd->n_points)
>                         state->idx = 0;
>         }
>         return (chosen);
> diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h
> index 03df1c312..3ad305180 100644
> --- a/lib/libvmod_directors/shard_dir.h
> +++ b/lib/libvmod_directors/shard_dir.h
> @@ -43,6 +43,7 @@ struct shard_backend {
>                 void            *freeptr;
>         };
>         VCL_DURATION            rampup;
> +       uint32_t                replicas;
>  };
>
>  struct vmod_directors_shard_param;
> @@ -68,7 +69,8 @@ struct sharddir {
>
>         VCL_DURATION                            rampup_duration;
>         VCL_REAL                                warmup;
> -       VCL_INT                                 replicas;
> +
> +       uint32_t                                n_points;
>  };
>
>  static inline VCL_BACKEND
> diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc
> index 054b6c34e..8ab9f12fa 100644
> --- a/lib/libvmod_directors/vmod.vcc
> +++ b/lib/libvmod_directors/vmod.vcc
> @@ -373,7 +373,7 @@ The association can be changed per backend request using the *param*
>  argument of `xshard.backend()`_.
>
>  $Method BOOL .add_backend(PRIV_TASK, BACKEND backend,
> -       [STRING ident], [DURATION rampup])
> +       [STRING ident], [DURATION rampup], [REAL weight])
>
>  Add a backend *backend* to the director.
>
> @@ -388,6 +388,12 @@ defaults to the backend name.
>  backend. Otherwise, the per-director rampup time is used (see
>  `xshard.set_rampup()`_).
>
> +*weight*: Optionally specify a weight to scale the
> +`xshard.reconfigure()`_ *replicas* parameter. *weight* is limited to
> +at least 1. Values above 10 probably do not make much sense. The
> +effect of *weight* is also capped such that the total number of
> +replicas does not exceed `UINT32_MAX`.
> +
>  NOTE: Backend changes need to be finalized with
>  `xshard.reconfigure()`_ and are only supported on one
>  shard director at a time.
> diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c
> index c3ec5d837..8a8bda9a2 100644
> --- a/lib/libvmod_directors/vmod_shard.c
> +++ b/lib/libvmod_directors/vmod_shard.c
> @@ -305,6 +305,8 @@ VCL_BOOL v_matchproto_(td_directors_shard_add_backend)
>  vmod_shard_add_backend(VRT_CTX, struct vmod_directors_shard *vshard,
>      struct VARGS(shard_add_backend) *args)
>  {
> +       VCL_REAL weight = 1;
> +
>         CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC);
>
>         if (args->backend == NULL) {
> @@ -313,10 +315,14 @@ vmod_shard_add_backend(VRT_CTX, struct vmod_directors_shard *vshard,
>                 return (0);
>         }
>
> +       if (args->valid_weight && args->weight > 1)
> +               weight = args->weight;
> +

Shouldn't we VRT_fail() if the supplied weight is not valid?


>         return shardcfg_add_backend(ctx, args->arg1,
>             vshard->shardd, args->backend,
>             args->valid_ident ? args->ident : NULL,
> -           args->valid_rampup ? args->rampup : nan(""));
> +           args->valid_rampup ? args->rampup : nan(""),
> +           weight);
>  }
>
>  VCL_BOOL v_matchproto_(td_directors_shard_remove_backend)
> _______________________________________________
> varnish-commit mailing list
> varnish-commit at varnish-cache.org
> https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit

From phk at FreeBSD.org  Tue Jun  9 20:48:09 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  9 Jun 2020 20:48:09 +0000 (UTC)
Subject: [master] 364932ece One more time for VTEST...
Message-ID: <20200609204809.0F599A7C40@lists.varnish-cache.org>


commit 364932ece2628ba4cc766445dfdf974e341787b2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 9 20:47:18 2020 +0000

    One more time for VTEST...

diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index 63f6c22ab..2c688d949 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -319,7 +319,6 @@ sharddir_pick_be_locked(VRT_CTX, const struct sharddir *shardd, uint32_t key,
 	validate_alt(ctx, shardd, &alt);
 
 	state->idx = shard_lookup(shardd, key);
-	assert(state->idx >= 0);
 
 	SHDBG(SHDBG_LOOKUP, shardd, "lookup key %x idx %d host %u",
 	    key, state->idx, shardd->hashcircle[state->idx].host);

From phk at FreeBSD.org  Tue Jun  9 21:03:07 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  9 Jun 2020 21:03:07 +0000 (UTC)
Subject: [master] 4932ce965 Try to regain gcov for tests which panic.
Message-ID: <20200609210307.36D07A8341@lists.varnish-cache.org>


commit 4932ce96544133b806c751e633de82714f0b78da
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 9 21:01:24 2020 +0000

    Try to regain gcov for tests which panic.

diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 407b8eacf..bfee6a1ad 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -63,6 +63,10 @@
  *
  */
 
+#ifdef GCOVING
+    int __llvm_profile_write_file(void);
+#endif
+
 static struct vsb pan_vsb_storage, *pan_vsb;
 static pthread_mutex_t panicstr_mtx;
 
@@ -811,6 +815,9 @@ pan_ic(const char *func, const char *file, int line, const char *cond,
 	VSB_cat(pan_vsb, "\n");
 	VSB_putc(pan_vsb, '\0');	/* NUL termination */
 
+#ifdef GCOVING
+	__llvm_profile_write_file();
+#endif
 	abort();
 }
 
diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index b046d20c7..e5aeffdef 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -63,6 +63,10 @@ static const struct cli_cmd_desc *cmds[] = {
 #include "tbl/cli_cmds.h"
 };
 
+#ifdef GCOVING
+    int __llvm_profile_write_file(void);
+#endif
+
 static const int ncmds = sizeof cmds / sizeof cmds[0];
 
 static int		cli_i = -1, cli_o = -1;
@@ -108,6 +112,9 @@ mcf_panic(struct cli *cli, const char * const *av, void *priv)
 	(void)cli;
 	(void)av;
 	(void)priv;
+#ifdef GCOVING
+	__llvm_profile_write_file();
+#endif
 	AZ(strcmp("", "You asked for it"));
 	/* NOTREACHED */
 	abort();

From phk at FreeBSD.org  Tue Jun  9 21:07:07 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  9 Jun 2020 21:07:07 +0000 (UTC)
Subject: [master] 4f0f6eb01 Changed GCOV settings
Message-ID: <20200609210707.1AEFCA85FB@lists.varnish-cache.org>


commit 4f0f6eb01f79ffb381456a6c2125abf4134ea4ec
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 9 21:05:38 2020 +0000

    Changed GCOV settings

diff --git a/tools/vtest.sh b/tools/vtest.sh
index 2e451e638..de1464118 100755
--- a/tools/vtest.sh
+++ b/tools/vtest.sh
@@ -202,9 +202,9 @@ failedtests () (
 
 if $enable_gcov ; then
 	#export CC=gcc6
-	export CC=clang80
-	export GCOVPROG='llvm-cov80 gcov'
-	export CFLAGS="-fprofile-arcs -ftest-coverage -fstack-protector -DDONT_DLCLOSE_VMODS"
+	#export CC=clang80
+	export GCOVPROG='llvm-cov gcov'
+	export CFLAGS="-fprofile-arcs -ftest-coverage -fstack-protector -DDONT_DLCLOSE_VMODS -DGCOVING"
 	export MAKEFLAGS=-j1
 fi
 

From phk at FreeBSD.org  Tue Jun  9 22:13:12 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  9 Jun 2020 22:13:12 +0000 (UTC)
Subject: [master] 2f8e7967f Try to force GCOV out another way before we call
 abort()
Message-ID: <20200609221312.8B7DDA9F81@lists.varnish-cache.org>


commit 2f8e7967f454bd694c7eafebbc548fc89db72357
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jun 9 22:12:09 2020 +0000

    Try to force GCOV out another way before we call abort()

diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index bfee6a1ad..2623fa422 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -64,7 +64,7 @@
  */
 
 #ifdef GCOVING
-    int __llvm_profile_write_file(void);
+    int __llvm_gcov_flush(void);
 #endif
 
 static struct vsb pan_vsb_storage, *pan_vsb;
@@ -816,7 +816,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond,
 	VSB_putc(pan_vsb, '\0');	/* NUL termination */
 
 #ifdef GCOVING
-	__llvm_profile_write_file();
+	__llvm_gcov_flush();
 #endif
 	abort();
 }
diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index e5aeffdef..678c43f9e 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -64,7 +64,7 @@ static const struct cli_cmd_desc *cmds[] = {
 };
 
 #ifdef GCOVING
-    int __llvm_profile_write_file(void);
+    int __llvm_gcov_flush(void);
 #endif
 
 static const int ncmds = sizeof cmds / sizeof cmds[0];
@@ -113,7 +113,7 @@ mcf_panic(struct cli *cli, const char * const *av, void *priv)
 	(void)av;
 	(void)priv;
 #ifdef GCOVING
-	__llvm_profile_write_file();
+	__llvm_gcov_flush();
 #endif
 	AZ(strcmp("", "You asked for it"));
 	/* NOTREACHED */

From phk at FreeBSD.org  Wed Jun 10 04:46:07 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 10 Jun 2020 04:46:07 +0000 (UTC)
Subject: [master] 9598facea Closer to getting GCOV back on panics: Now try
 global __gcov_flush()
Message-ID: <20200610044607.E5464B2545@lists.varnish-cache.org>


commit 9598facea5a8d7f59c4a23bcf78a94ac404ae378
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 10 04:44:56 2020 +0000

    Closer to getting GCOV back on panics:  Now try global __gcov_flush()

diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 2623fa422..c59f35c53 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -64,7 +64,7 @@
  */
 
 #ifdef GCOVING
-    int __llvm_gcov_flush(void);
+    int __gcov_flush(void);
 #endif
 
 static struct vsb pan_vsb_storage, *pan_vsb;
@@ -816,7 +816,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond,
 	VSB_putc(pan_vsb, '\0');	/* NUL termination */
 
 #ifdef GCOVING
-	__llvm_gcov_flush();
+	__gcov_flush();
 #endif
 	abort();
 }
diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index 678c43f9e..18b6202b2 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -64,7 +64,7 @@ static const struct cli_cmd_desc *cmds[] = {
 };
 
 #ifdef GCOVING
-    int __llvm_gcov_flush(void);
+    int __gcov_flush(void);
 #endif
 
 static const int ncmds = sizeof cmds / sizeof cmds[0];
@@ -113,7 +113,7 @@ mcf_panic(struct cli *cli, const char * const *av, void *priv)
 	(void)av;
 	(void)priv;
 #ifdef GCOVING
-	__llvm_gcov_flush();
+	__gcov_flush();
 #endif
 	AZ(strcmp("", "You asked for it"));
 	/* NOTREACHED */

From nils.goroll at uplex.de  Wed Jun 10 06:41:10 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 10 Jun 2020 08:41:10 +0200
Subject: [master] b90b60d0a shard director: add optional weight parameter
 to .add_backend()
In-Reply-To: <CABoVN9Cev0bCFgmaFUFqDZ2ixkfq-UhoHQ+9ufi34ZA0C8_caw@mail.gmail.com>
References: <20200609170307.79A6CA212A@lists.varnish-cache.org>
 <CABoVN9Cev0bCFgmaFUFqDZ2ixkfq-UhoHQ+9ufi34ZA0C8_caw@mail.gmail.com>
Message-ID: <6266ceed-7cad-d168-e5c7-8e0e0eae14bd@uplex.de>

Hi Dridi,

thank you for the review :)

> Any reason not to return(fail("reason")) instead of std.log() calls?

The reason is legacy - this was copy-pasta of a long existing test. I'll polish...

>> +       if (args->valid_weight && args->weight > 1)
>> +               weight = args->weight;
>> +
> 
> Shouldn't we VRT_fail() if the supplied weight is not valid?

Yes we should, good point!

Nils

-- 

** * * UPLEX - Nils Goroll Systemoptimierung

Scheffelstra?e 32
22301 Hamburg

tel +49 40 28805731
mob +49 170 2723133
fax +49 40 42949753

xmpp://slink at jabber.int.uplex.de/

http://uplex.de/

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 488 bytes
Desc: OpenPGP digital signature
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20200610/a78ee576/attachment.bin>

From nils.goroll at uplex.de  Wed Jun 10 07:06:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 10 Jun 2020 07:06:07 +0000 (UTC)
Subject: [master] 1d75f29c7 test polish suggested by Dridi, thank you
Message-ID: <20200610070607.36B4EC06A7@lists.varnish-cache.org>


commit 1d75f29c7b49219aa6c2ca75917dd5f9ded1afcc
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jun 10 08:56:08 2020 +0200

    test polish suggested by Dridi, thank you

diff --git a/bin/varnishtest/tests/d00015.vtc b/bin/varnishtest/tests/d00015.vtc
index 371f6bccd..5050cff45 100644
--- a/bin/varnishtest/tests/d00015.vtc
+++ b/bin/varnishtest/tests/d00015.vtc
@@ -22,33 +22,36 @@ varnish v1 -vcl+backend {
 
 		std.log("-- invalid replicas");
 		if (!vd.reconfigure(replicas=0)) {
+			# continue intentionally
 			std.log("reconfigure failed");
 		}
 
 		std.log("-- no changes - no debug output");
 		if (!vd.reconfigure(replicas=25)) {
+			# continue intentionally
 			std.log("reconfigure failed");
 		}
 
 		std.log("-- no backends");
 		if (!vd.clear()) {
-			std.log("clear failed");
+			return(fail("clear failed"));
 		}
 		if (!vd.reconfigure(replicas=25)) {
+			# continue intentionally
 			std.log("reconfigure failed");
 		}
 
 		std.log("-- one backend");
 		if (!vd.add_backend(s1)) {
-			std.log("add s1 failed");
+			return(fail("add s1 failed"));
 		}
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 
 		std.log("-- no change - no output");
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 
 		std.log("-- change, clear, no backends");
@@ -60,6 +63,7 @@ varnish v1 -vcl+backend {
 		vd.add_backend(s2);
 		vd.clear();
 		if (!vd.reconfigure()) {
+			# continue intentionally
 			std.log("reconfigure failed");
 		}
 
@@ -69,7 +73,7 @@ varnish v1 -vcl+backend {
 		vd.add_backend(s2);
 		vd.add_backend(s1);
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 
 		std.log("-- duplicate add with idents");
@@ -82,19 +86,19 @@ varnish v1 -vcl+backend {
 		vd.add_backend(s2, ident="s1_1");
 		vd.add_backend(s2, ident="s1_2");
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 
 		std.log("-- remove s1_2 specifically");
 		vd.remove_backend(ident="s1_2");
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 
 		std.log("-- remove all instances of s1");
 		vd.remove_backend(s1);
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 
 		std.log("-- re-add some - no 2nd director");
@@ -110,19 +114,19 @@ varnish v1 -vcl+backend {
 		vd.add_backend(s3, "8");
 		vd.add_backend(s3, "9");
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 
 		std.log("-- remove second-last");
 		vd.remove_backend(ident="8");
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 
 		std.log("-- remove last");
 		vd.remove_backend(ident="9");
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 
 		std.log("-- END");
@@ -146,7 +150,7 @@ varnish v1 -vcl+backend {
 		vd.add_backend(s1, "0x0e");
 		vd.add_backend(s1, "0x0f");
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 		vd.remove_backend(s1, "0x00");
 		vd.remove_backend(s1, "0x01");
@@ -165,7 +169,7 @@ varnish v1 -vcl+backend {
 		vd.remove_backend(s1, "0x0e");
 		vd.remove_backend(s1, "0x0f");
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 		vd.set_warmup(0.5);
 
@@ -180,7 +184,7 @@ varnish v1 -vcl+backend {
 		vd.add_backend(s3, "4");
 		vd.add_backend(s3, "5");
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 		vd.set_warmup(0);
 	}
diff --git a/bin/varnishtest/tests/d00016.vtc b/bin/varnishtest/tests/d00016.vtc
index b58ea294b..ac0bb7644 100644
--- a/bin/varnishtest/tests/d00016.vtc
+++ b/bin/varnishtest/tests/d00016.vtc
@@ -22,35 +22,39 @@ varnish v1 -vcl+backend {
 	}
 
 	sub vcl_recv {
+
 		std.log("-- invalid replicas");
 		if (!vd.reconfigure(replicas=0)) {
+			# continue intentionally
 			std.log("reconfigure failed");
 		}
 
 		std.log("-- no changes - no debug output");
 		if (!vd.reconfigure(replicas=25)) {
+			# continue intentionally
 			std.log("reconfigure failed");
 		}
 
 		std.log("-- no backends");
 		if (!vd.clear()) {
-			std.log("clear failed");
+			return(fail("clear failed"));
 		}
 		if (!vd.reconfigure(replicas=25)) {
+			# continue intentionally
 			std.log("reconfigure failed");
 		}
 
 		std.log("-- one backend");
 		if (!vd.add_backend(s1)) {
-			std.log("add s1 failed");
+			return(fail("add s1 failed"));
 		}
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 
 		std.log("-- no change - no output");
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 
 		std.log("-- change, clear, no backends");
@@ -62,6 +66,7 @@ varnish v1 -vcl+backend {
 		vd.add_backend(s2);
 		vd.clear();
 		if (!vd.reconfigure()) {
+			# continue intentionally
 			std.log("reconfigure failed");
 		}
 
@@ -71,7 +76,7 @@ varnish v1 -vcl+backend {
 		vd.add_backend(s2);
 		vd.add_backend(s1);
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 
 		std.log("-- duplicate add with idents");
@@ -84,19 +89,19 @@ varnish v1 -vcl+backend {
 		vd.add_backend(s2, ident="s1_1");
 		vd.add_backend(s2, ident="s1_2");
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 
 		std.log("-- remove s1_2 specifically");
 		vd.remove_backend(ident="s1_2");
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 
 		std.log("-- remove all instances of s1");
 		vd.remove_backend(s1);
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 
 		std.log("-- re-add some - no 2nd director");
@@ -112,23 +117,22 @@ varnish v1 -vcl+backend {
 		vd.add_backend(s3, "8");
 		vd.add_backend(s3, "9");
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 
 		std.log("-- remove second-last");
 		vd.remove_backend(ident="8");
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 
 		std.log("-- remove last");
 		vd.remove_backend(ident="9");
 		if (!vd.reconfigure(replicas=1)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 
 		std.log("-- END");
-
 		set req.backend_hint = vd.backend();
 		return(pass);
 	}
diff --git a/bin/varnishtest/tests/d00017.vtc b/bin/varnishtest/tests/d00017.vtc
index a8f33b143..36793a386 100644
--- a/bin/varnishtest/tests/d00017.vtc
+++ b/bin/varnishtest/tests/d00017.vtc
@@ -31,16 +31,16 @@ varnish v1 -vcl+backend {
 		new vd = directors.shard();
 		vd.debug(3);
 		if (!vd.add_backend(s1)) {
-			std.log("add s1 failed");
+			return(fail("add s1 failed"));
 		}
 		if (!vd.add_backend(s2)) {
-			std.log("add s2 failed");
+			return(fail("add s2 failed"));
 		}
 		if (!vd.add_backend(s3)) {
-			std.log("add s3 failed");
+			return(fail("add s3 failed"));
 		}
 		if (!vd.reconfigure(replicas=25)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 	}
 
diff --git a/bin/varnishtest/tests/d00018.vtc b/bin/varnishtest/tests/d00018.vtc
index a3f02e0ae..429bc9014 100644
--- a/bin/varnishtest/tests/d00018.vtc
+++ b/bin/varnishtest/tests/d00018.vtc
@@ -22,16 +22,16 @@ varnish v1 -vcl+backend {
 	sub vcl_init {
 		new vd = directors.shard();
 		if (!vd.add_backend(s1)) {
-			std.log("add s1 failed");
+			return(fail("add s1 failed"));
 		}
 		if (!vd.add_backend(s2)) {
-			std.log("add s2 failed");
+			return(fail("add s2 failed"));
 		}
 		if (!vd.add_backend(s3)) {
-			std.log("add s3 failed");
+			return(fail("add s3 failed"));
 		}
 		if (!vd.reconfigure(replicas=25)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 	}
 
diff --git a/bin/varnishtest/tests/d00021.vtc b/bin/varnishtest/tests/d00021.vtc
index 91930c18b..ee85ace22 100644
--- a/bin/varnishtest/tests/d00021.vtc
+++ b/bin/varnishtest/tests/d00021.vtc
@@ -22,32 +22,32 @@ varnish v1 -vcl+backend {
 	sub vcl_init {
 	    new vd = directors.shard();
 	    if (!vd.add_backend(s1)) {
-		std.log("add s1 failed");
+		return(fail("add s1 failed"));
 	    }
 	    if (!vd.add_backend(s2)) {
-		std.log("add s2 failed");
+		return(fail("add s2 failed"));
 	    }
 	    if (!vd.add_backend(s3)) {
-		std.log("add s3 failed");
+		return(fail("add s3 failed"));
 	    }
 	    if (!vd.reconfigure(replicas=25)) {
-		std.log("reconfigure failed");
+		return(fail("reconfigure failed"));
 	    }
 
 	    new l = directors.shard();
 	    new lp = directors.shard_param();
 	    l.associate(lp.use());
 	    if (!l.add_backend(s1)) {
-		std.log("add s1 failed");
+		return(fail("add s1 failed"));
 	    }
 	    if (!l.add_backend(s2)) {
-		std.log("add s2 failed");
+		return(fail("add s2 failed"));
 	    }
 	    if (!l.add_backend(s3)) {
-		std.log("add s3 failed");
+		return(fail("add s3 failed"));
 	    }
 	    if (!l.reconfigure(replicas=25)) {
-		std.log("reconfigure failed");
+		return(fail("reconfigure failed"));
 	    }
 	    new ll = directors.round_robin();
 	    ll.add_backend(l.backend());
diff --git a/bin/varnishtest/tests/d00029.vtc b/bin/varnishtest/tests/d00029.vtc
index 93a1517c1..698f89e21 100644
--- a/bin/varnishtest/tests/d00029.vtc
+++ b/bin/varnishtest/tests/d00029.vtc
@@ -22,16 +22,16 @@ varnish v1 -vcl+backend {
 	sub vcl_init {
 	    new vd = directors.shard();
 	    if (!vd.add_backend(s1)) {
-		std.log("add s1 failed");
+		return(fail("add s1 failed"));
 	    }
 	    if (!vd.add_backend(s2)) {
-		std.log("add s2 failed");
+		return(fail("add s2 failed"));
 	    }
 	    if (!vd.add_backend(s3)) {
-		std.log("add s3 failed");
+		return(fail("add s3 failed"));
 	    }
 	    if (!vd.reconfigure(replicas=25)) {
-		std.log("reconfigure failed");
+		return(fail("reconfigure failed"));
 	    }
 
 	    vd.debug(1);
diff --git a/bin/varnishtest/tests/d00041.vtc b/bin/varnishtest/tests/d00041.vtc
index bd79cfff1..0f7d5b058 100644
--- a/bin/varnishtest/tests/d00041.vtc
+++ b/bin/varnishtest/tests/d00041.vtc
@@ -24,16 +24,16 @@ varnish v1 -vcl+backend {
 		new vd = directors.shard();
 		vd.debug(3);
 		if (!vd.add_backend(s1)) {
-			std.log("add s1 failed");
+			return(fail("add s1 failed"));
 		}
 		if (!vd.add_backend(s2, weight=2)) {
-			std.log("add s2 failed");
+			return(fail("add s2 failed"));
 		}
 		if (!vd.add_backend(s3, weight=3)) {
-			std.log("add s3 failed");
+			return(fail("add s3 failed"));
 		}
 		if (!vd.reconfigure(replicas=25)) {
-			std.log("reconfigure failed");
+			return(fail("reconfigure failed"));
 		}
 	}
 

From nils.goroll at uplex.de  Wed Jun 10 07:06:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 10 Jun 2020 07:06:07 +0000 (UTC)
Subject: [master] 72bb92f03 signedness improvements
Message-ID: <20200610070607.4B443C06AA@lists.varnish-cache.org>


commit 72bb92f03f64282be653815001558283fbe59770
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jun 10 08:59:26 2020 +0200

    signedness improvements
    
    from flexelint review

diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h
index 3ad305180..3de949943 100644
--- a/lib/libvmod_directors/shard_dir.h
+++ b/lib/libvmod_directors/shard_dir.h
@@ -59,8 +59,8 @@ struct sharddir {
 
 	pthread_rwlock_t			mtx;
 
-	int					n_backend;
-	int					l_backend;
+	unsigned				n_backend;
+	unsigned				l_backend;
 	struct shard_backend			*backend;
 
 	const char				*name;

From nils.goroll at uplex.de  Wed Jun 10 07:06:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 10 Jun 2020 07:06:07 +0000 (UTC)
Subject: [master] e148a5ef6 flexelinting
Message-ID: <20200610070607.66EC0C06AE@lists.varnish-cache.org>


commit e148a5ef6b1248b5e2535e553f2264c2c79f34ef
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jun 10 09:05:24 2020 +0200

    flexelinting

diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c
index 3f478ecf1..ced267c60 100644
--- a/lib/libvmod_directors/shard_cfg.c
+++ b/lib/libvmod_directors/shard_cfg.c
@@ -244,8 +244,8 @@ static void
 shardcfg_hashcircle(struct sharddir *shardd)
 {
 	const struct shard_backend *backends, *b;
-	int j, h;
-	uint32_t i, n_points, r, rmax;
+	int h;
+	uint32_t i, j, n_points, r, rmax;
 	const char *ident;
 	const int len = 12; // log10(UINT32_MAX) + 2;
 	char s[len];
@@ -578,7 +578,8 @@ shardcfg_apply_change(VRT_CTX, struct sharddir *shardd,
 				if (replicas * task->weight > UINT32_MAX)
 					b_replicas = UINT32_MAX;
 				else
-					b_replicas = replicas * task->weight;
+					b_replicas = (uint32_t) // flint
+						(replicas * task->weight);
 
 				shardcfg_backend_add(&re, task->priv,
 				    b_replicas);
diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index 2c688d949..1c5f7d193 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -259,7 +259,7 @@ init_state(struct shard_state *state,
 
 	state->ctx = ctx;
 	state->shardd = shardd;
-	state->idx = -1;
+	state->idx = UINT32_MAX;
 	state->picklist = picklist;
 
 	/* healhy and changed only defined for hostid != -1 */
@@ -319,8 +319,9 @@ sharddir_pick_be_locked(VRT_CTX, const struct sharddir *shardd, uint32_t key,
 	validate_alt(ctx, shardd, &alt);
 
 	state->idx = shard_lookup(shardd, key);
+	assert(state->idx < UINT32_MAX);
 
-	SHDBG(SHDBG_LOOKUP, shardd, "lookup key %x idx %d host %u",
+	SHDBG(SHDBG_LOOKUP, shardd, "lookup key %x idx %u host %u",
 	    key, state->idx, shardd->hashcircle[state->idx].host);
 
 	if (alt > 0) {

From nils.goroll at uplex.de  Wed Jun 10 07:42:06 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 10 Jun 2020 07:42:06 +0000 (UTC)
Subject: [master] 2c47803ec shard: VRT_fail for backend add/remove errors
Message-ID: <20200610074206.7C9CA4839@lists.varnish-cache.org>


commit 2c47803ec0d5eb4a687577881f04a160c62965e1
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jun 10 09:14:49 2020 +0200

    shard: VRT_fail for backend add/remove errors
    
    ... to be more in line with other bundled directors

diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c
index 8a8bda9a2..3f0444f4c 100644
--- a/lib/libvmod_directors/vmod_shard.c
+++ b/lib/libvmod_directors/vmod_shard.c
@@ -310,8 +310,8 @@ vmod_shard_add_backend(VRT_CTX, struct vmod_directors_shard *vshard,
 	CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC);
 
 	if (args->backend == NULL) {
-		shard_err0(ctx, vshard->shardd,
-		    ".backend_add() NULL backend given");
+		VRT_fail(ctx, "%s: NULL backend cannot be added",
+			 vshard->shardd->name);
 		return (0);
 	}
 
@@ -335,10 +335,10 @@ vmod_shard_remove_backend(VRT_CTX, struct vmod_directors_shard *vshard,
 	CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC);
 
 	if (be == NULL && ident == NULL) {
-		shard_err0(ctx, vshard->shardd,
-		    ".backend_remove() at least one of backend "
-		    "and ident must be given");
-		return 0;
+		VRT_fail(ctx, "%s.backend_remove(): "
+		    "either backend or ident are required",
+		    vshard->shardd->name);
+		return (0);
 	}
 
 	return shardcfg_remove_backend(ctx, args->arg1, vshard->shardd,

From nils.goroll at uplex.de  Wed Jun 10 07:42:06 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 10 Jun 2020 07:42:06 +0000 (UTC)
Subject: [master] 7fec5db75 shard: VRT_fail for out-of-workspace errors
Message-ID: <20200610074206.92E06483C@lists.varnish-cache.org>


commit 7fec5db7583d44d8425f93c899153b1b676dbeb1
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jun 10 09:18:21 2020 +0200

    shard: VRT_fail for out-of-workspace errors

diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c
index ced267c60..174e6e67e 100644
--- a/lib/libvmod_directors/shard_cfg.c
+++ b/lib/libvmod_directors/shard_cfg.c
@@ -106,7 +106,7 @@ shard_change_get(VRT_CTX, struct vmod_priv *priv,
 
 	change = WS_Alloc(ctx->ws, sizeof(*change));
 	if (change == NULL) {
-		shard_err0(ctx, shardd, "could not get workspace");
+		VRT_fail(ctx, "could not get workspace");
 		return (NULL);
 	}
 
@@ -138,8 +138,7 @@ shard_change_task_add(VRT_CTX, struct shard_change *change,
 
 	task = WS_Alloc(ctx->ws, sizeof(*task));
 	if (task == NULL) {
-		shard_err0(ctx, change->shardd,
-		    "could not get workspace for task");
+		VRT_fail(ctx, "could not get workspace for task");
 		return (NULL);
 	}
 	INIT_OBJ(task, SHARD_CHANGE_TASK_MAGIC);
@@ -168,8 +167,7 @@ shard_change_task_backend(VRT_CTX,
 
 	b = WS_Alloc(ctx->ws, sizeof(*b));
 	if (b == NULL) {
-		shard_err(ctx, shardd, ".%s_backend() WS_Alloc() failed",
-		    task_e == ADD_BE ? "add" : "remove");
+		VRT_fail(ctx, "could not get workspace for change");
 		return (NULL);
 	}
 

From nils.goroll at uplex.de  Wed Jun 10 07:42:06 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 10 Jun 2020 07:42:06 +0000 (UTC)
Subject: [master] daf48542b shard: more signedness stir
Message-ID: <20200610074206.AE2DB4841@lists.varnish-cache.org>


commit daf48542b0ce50c7e167792a43b6126764d1fbf0
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jun 10 09:29:34 2020 +0200

    shard: more signedness stir

diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c
index 174e6e67e..82c51fd16 100644
--- a/lib/libvmod_directors/shard_cfg.c
+++ b/lib/libvmod_directors/shard_cfg.c
@@ -242,7 +242,7 @@ static void
 shardcfg_hashcircle(struct sharddir *shardd)
 {
 	const struct shard_backend *backends, *b;
-	int h;
+	unsigned h;
 	uint32_t i, j, n_points, r, rmax;
 	const char *ident;
 	const int len = 12; // log10(UINT32_MAX) + 2;
@@ -394,7 +394,7 @@ shardcfg_backend_lookup(const struct backend_reconfig *re,
 static void
 shardcfg_backend_expand(const struct backend_reconfig *re)
 {
-	int min = re->hint;
+	unsigned min = re->hint;
 
 	CHECK_OBJ_NOTNULL(re->shardd, SHARDDIR_MAGIC);
 
@@ -427,6 +427,7 @@ shardcfg_backend_add(struct backend_reconfig *re,
 		assert(re->shardd->n_backend < re->shardd->l_backend);
 		i = re->shardd->n_backend;
 	} else {
+		assert(re->hole_i != UINT_MAX);
 		do {
 			if (!bb[re->hole_i].backend)
 				break;
@@ -445,7 +446,7 @@ shardcfg_backend_add(struct backend_reconfig *re,
 static void
 shardcfg_backend_clear(struct sharddir *shardd)
 {
-	int i;
+	unsigned i;
 	for (i = 0; i < shardd->n_backend; i++)
 		shardcfg_backend_free(&shardd->backend[i]);
 	shardd->n_backend = 0;
@@ -530,7 +531,7 @@ shardcfg_apply_change(VRT_CTX, struct sharddir *shardd,
 		.shardd = shardd,
 		.hint = shardd->n_backend,
 		.hole_n = 0,
-		.hole_i = INT_MAX
+		.hole_i = UINT_MAX
 	};
 
 	// XXX assert sharddir_locked(shardd)
@@ -656,7 +657,7 @@ shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv,
 void
 shardcfg_delete(const struct sharddir *shardd)
 {
-	int i;
+	unsigned i;
 
 	for (i = 0; i < shardd->n_backend; i++)
 		shardcfg_backend_free(&shardd->backend[i]);
@@ -687,7 +688,7 @@ shardcfg_set_rampup(struct sharddir *shardd, VCL_DURATION duration)
 }
 
 VCL_DURATION
-shardcfg_get_rampup(const struct sharddir *shardd, int host)
+shardcfg_get_rampup(const struct sharddir *shardd, unsigned host)
 {
 	VCL_DURATION r;
 
diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index 1c5f7d193..b82364297 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -37,6 +37,7 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <time.h>
+#include <limits.h>
 
 #include "cache/cache.h"
 
@@ -47,7 +48,7 @@
 #include "shard_dir.h"
 
 struct shard_be_info {
-	int		hostid;
+	unsigned	hostid;
 	unsigned	healthy;
 	double		changed;	// when
 };
@@ -63,7 +64,7 @@ struct shard_state {
 	uint32_t		idx;
 
 	struct vbitmap		*picklist;
-	int			pickcount;
+	unsigned		pickcount;
 
 	struct shard_be_info	previous;
 	struct shard_be_info	last;
@@ -159,7 +160,7 @@ shard_next(struct shard_state *state, VCL_INT skip, VCL_BOOL healthy)
 				sbe = &state->last;
 			}
 			if (sbe == &state->last &&
-			    state->last.hostid != -1)
+			    state->last.hostid != UINT_MAX)
 				memcpy(&state->previous, &state->last,
 				    sizeof(state->previous));
 
@@ -262,9 +263,9 @@ init_state(struct shard_state *state,
 	state->idx = UINT32_MAX;
 	state->picklist = picklist;
 
-	/* healhy and changed only defined for hostid != -1 */
-	state->previous.hostid = -1;
-	state->last.hostid = -1;
+	/* healhy and changed only defined for valid hostids */
+	state->previous.hostid = UINT_MAX;
+	state->last.hostid = UINT_MAX;
 }
 
 /* basically same as vdir_any_healthy
@@ -327,7 +328,7 @@ sharddir_pick_be_locked(VRT_CTX, const struct sharddir *shardd, uint32_t key,
 	if (alt > 0) {
 		if (shard_next(state, alt - 1,
 		    healthy == VENUM(ALL) ? 1 : 0) == -1) {
-			if (state->previous.hostid != -1) {
+			if (state->previous.hostid != UINT_MAX) {
 				be = sharddir_backend(shardd,
 				    state->previous.hostid);
 				AN(be);
@@ -338,7 +339,7 @@ sharddir_pick_be_locked(VRT_CTX, const struct sharddir *shardd, uint32_t key,
 	}
 
 	if (shard_next(state, 0, healthy == VENUM(IGNORE) ? 0 : 1) == -1) {
-		if (state->previous.hostid != -1) {
+		if (state->previous.hostid != UINT_MAX) {
 			be = sharddir_backend(shardd, state->previous.hostid);
 			AN(be);
 			return (be);
@@ -358,8 +359,8 @@ sharddir_pick_be_locked(VRT_CTX, const struct sharddir *shardd, uint32_t key,
 		return (be);
 
 	assert(alt == 0);
-	assert(state->previous.hostid >= 0);
-	assert(state->last.hostid >= 0);
+	assert(state->previous.hostid != UINT_MAX);
+	assert(state->last.hostid != UINT_MAX);
 	assert(state->previous.hostid != state->last.hostid);
 	assert(be == sharddir_backend(shardd, state->previous.hostid));
 
diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h
index 3de949943..67b709aac 100644
--- a/lib/libvmod_directors/shard_dir.h
+++ b/lib/libvmod_directors/shard_dir.h
@@ -74,9 +74,8 @@ struct sharddir {
 };
 
 static inline VCL_BACKEND
-sharddir_backend(const struct sharddir *shardd, int id)
+sharddir_backend(const struct sharddir *shardd, unsigned id)
 {
-	assert(id >= 0);
 	assert(id < shardd->n_backend);
 	return (shardd->backend[id].backend);
 }
@@ -115,4 +114,4 @@ VCL_BACKEND sharddir_pick_be(VRT_CTX, struct sharddir *, uint32_t, VCL_INT,
 
 /* in shard_cfg.c */
 void shardcfg_delete(const struct sharddir *shardd);
-VCL_DURATION shardcfg_get_rampup(const struct sharddir *shardd, int host);
+VCL_DURATION shardcfg_get_rampup(const struct sharddir *shardd, unsigned host);
diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c
index 3f0444f4c..6e59044d6 100644
--- a/lib/libvmod_directors/vmod_shard.c
+++ b/lib/libvmod_directors/vmod_shard.c
@@ -707,9 +707,8 @@ vmod_shard_list(VRT_CTX, VCL_BACKEND dir, struct vsb *vsb, int pflag, int jflag)
 	VCL_DURATION rampup_d, d;
 	VCL_BACKEND be;
 	VCL_BOOL h;
-	unsigned nh = 0;
+	unsigned i, nh = 0;
 	double rampup_p;
-	int i;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC);

From nils.goroll at uplex.de  Wed Jun 10 07:53:06 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 10 Jun 2020 07:53:06 +0000 (UTC)
Subject: [master] 90c5be03d shard: add error message for invalid weight
Message-ID: <20200610075306.A402F5064@lists.varnish-cache.org>


commit 90c5be03d9514b45fe26c5ee029857841aef0241
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jun 10 09:44:40 2020 +0200

    shard: add error message for invalid weight
    
    This was brought up by Dridi in an email response to
    b90b60d0a066f1aa7302a70125cb91cdfa605119:
    
    Initially, I also thought that we should VRT_fail() for an invalid
    parameter, but on second thought I realized that, as the shard director
    supports request-time reconfiguration, graceful error handling should be
    possible, so VRT_fail() is too hard.
    
    I think that even returning false for .add_backend() is too harsh, as an
    invalid weight is probably not too much of an issue to abort the
    reconfiguration in the case of caller error handling.
    
    That said, this might all be over the top. But I really do not want to
    run into the (still unfixed) case I saw recently trying to base64 decode
    an invalid input, which is not possible to handle gracefully.

diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c
index 6e59044d6..156479eb2 100644
--- a/lib/libvmod_directors/vmod_shard.c
+++ b/lib/libvmod_directors/vmod_shard.c
@@ -315,8 +315,13 @@ vmod_shard_add_backend(VRT_CTX, struct vmod_directors_shard *vshard,
 		return (0);
 	}
 
-	if (args->valid_weight && args->weight > 1)
-		weight = args->weight;
+	if (args->valid_weight) {
+		if (args->weight >= 1)
+			weight = args->weight;
+		else
+			shard_err(ctx, vshard->shardd,
+			    ".add_backend(weight=%f) ignored", args->weight);
+	}
 
 	return shardcfg_add_backend(ctx, args->arg1,
 	    vshard->shardd, args->backend,

From nils.goroll at uplex.de  Wed Jun 10 07:53:06 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 10 Jun 2020 07:53:06 +0000 (UTC)
Subject: [master] 3c0609bc8 shard signedness stir: last bits?
Message-ID: <20200610075306.BBB335067@lists.varnish-cache.org>


commit 3c0609bc8c5a66beb6ada598e7ba60b62940f242
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jun 10 09:52:06 2020 +0200

    shard signedness stir: last bits?

diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index b82364297..ea3dafad3 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -276,9 +276,8 @@ init_state(struct shard_state *state,
 VCL_BOOL
 sharddir_any_healthy(VRT_CTX, struct sharddir *shardd, VCL_TIME *changed)
 {
-	unsigned retval = 0;
+	unsigned i, retval = 0;
 	VCL_BACKEND be;
-	int i;
 	vtim_real c;
 
 	CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC);
@@ -367,10 +366,10 @@ sharddir_pick_be_locked(VRT_CTX, const struct sharddir *shardd, uint32_t key,
 	chosen_r = shardcfg_get_rampup(shardd, state->previous.hostid);
 	alt_r = shardcfg_get_rampup(shardd, state->last.hostid);
 
-	SHDBG(SHDBG_RAMPWARM, shardd, "chosen host %d rampup %f changed %f",
+	SHDBG(SHDBG_RAMPWARM, shardd, "chosen host %u rampup %f changed %f",
 	    state->previous.hostid, chosen_r,
 	    ctx->now - state->previous.changed);
-	SHDBG(SHDBG_RAMPWARM, shardd, "alt host %d rampup %f changed %f",
+	SHDBG(SHDBG_RAMPWARM, shardd, "alt host %u rampup %f changed %f",
 	    state->last.hostid, alt_r,
 	    ctx->now - state->last.changed);
 
diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c
index 156479eb2..d634f0785 100644
--- a/lib/libvmod_directors/vmod_shard.c
+++ b/lib/libvmod_directors/vmod_shard.c
@@ -800,10 +800,10 @@ vmod_shard_list(VRT_CTX, VCL_BACKEND dir, struct vsb *vsb, int pflag, int jflag)
 		return;
 
 	if (jflag)
-		VSB_printf(vsb, "[%u, %d, \"%s\"]", nh, i,
+		VSB_printf(vsb, "[%u, %u, \"%s\"]", nh, i,
 		    nh ? "healthy" : "sick");
 	else
-		VSB_printf(vsb, "%u/%d\t%s", nh, i, nh ? "healthy" : "sick");
+		VSB_printf(vsb, "%u/%u\t%s", nh, i, nh ? "healthy" : "sick");
 }
 
 VCL_VOID v_matchproto_(td_directors_shard_backend)

From nils.goroll at uplex.de  Wed Jun 10 08:06:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 10 Jun 2020 08:06:07 +0000 (UTC)
Subject: [master] b782c2c6d Revert "Use sphinx source/build facility directly"
Message-ID: <20200610080607.3FE3D5918@lists.varnish-cache.org>


commit b782c2c6d4ec3710dcd7124959c1a6a505a34f70
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri May 8 19:29:38 2020 +0200

    Revert "Use sphinx source/build facility directly"
    
    Partially reverts commit 883fddfea4e185ff417762fb64207b81d4cea971.
    
    Ref #3309

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 7c2a38a37..fa71cacc9 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -5,9 +5,9 @@
 SPHINXOPTS    =
 SPHINXBUILD   = $(SPHINX) -W -q -N
 PAPER         = a4
-BUILDDIR      = $(builddir)/build
+BUILDDIR      = build
 
-ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees -D latex_elements.papersize=$(PAPER) $(SPHINXOPTS) $(srcdir)
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees -D latex_elements.papersize=$(PAPER) $(SPHINXOPTS) $(builddir)
 
 .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest
 
@@ -27,12 +27,22 @@ help:
 clean:
 	-rm -rf $(BUILDDIR)/* $(CLEANFILES)
 
+# use index.rst as an indicator if we have copied already
+.PHONY: link_srcdir
+link_srcdir:
+	if test "x$(srcdir)" != "x$(builddir)" && test ! -f index.rst ; then \
+		d=`pwd`/$(builddir) ; \
+		cd $(srcdir) && find . -type f | cpio -dmp $${d} || true ; \
+	fi
+
 # work around for make html called within doc/sphinx
 .PHONY: graphviz
 graphviz:
 	cd ../graphviz && $(MAKE) html
 
-sphinx_prereq: graphviz conf.py
+sphinx_prereq: link_srcdir graphviz conf.py
+
+all: link_srcdir
 
 html: sphinx_prereq
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html

From nils.goroll at uplex.de  Wed Jun 10 08:06:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 10 Jun 2020 08:06:07 +0000 (UTC)
Subject: [master] 146a28651 symlink documentation source files from $(srcdir)
 to $(builddir)
Message-ID: <20200610080607.593D0591B@lists.varnish-cache.org>


commit 146a28651a140db54b8efd576e971efcd6cc3ae3
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri May 8 19:37:16 2020 +0200

    symlink documentation source files from $(srcdir) to $(builddir)
    
    ... if they differ.
    
    Ref #3309

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index fa71cacc9..ea010ab1a 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -30,9 +30,13 @@ clean:
 # use index.rst as an indicator if we have copied already
 .PHONY: link_srcdir
 link_srcdir:
-	if test "x$(srcdir)" != "x$(builddir)" && test ! -f index.rst ; then \
-		d=`pwd`/$(builddir) ; \
-		cd $(srcdir) && find . -type f | cpio -dmp $${d} || true ; \
+	if test "x$(srcdir)" != "x$(builddir)" && test ! -f index.rst; then \
+		s=`realpath $(srcdir)`; \
+		for f in `cd $$s && find . -type f`; do \
+			d=`dirname $$f`; \
+			test -d $$d || mkdir -p $$d; \
+			test -f $$f || ln -s $$s/$$f $$f; \
+		done \
 	fi
 
 # work around for make html called within doc/sphinx

From nils.goroll at uplex.de  Wed Jun 10 08:06:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 10 Jun 2020 08:06:07 +0000 (UTC)
Subject: [master] 390a39a36 build documentation entirely from the builddir
Message-ID: <20200610080607.773E3591F@lists.varnish-cache.org>


commit 390a39a3697dd2b708f76085f03f1c73c40ca8dc
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri May 8 19:43:49 2020 +0200

    build documentation entirely from the builddir
    
    The previous commit added symlinks for source files.
    
    Fixes #3309

diff --git a/man/Makefile.am b/man/Makefile.am
index 53646cde6..f04a0647c 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -31,69 +31,69 @@ RST2ANY_FLAGS = --halt=2
 BUILD_MAN = $(AM_V_GEN) $(RST2MAN) $(RST2ANY_FLAGS)
 
 varnish-cli.7: $(top_builddir)/doc/sphinx/reference/varnish-cli.rst \
-	$(top_srcdir)/doc/sphinx/include/cli.rst
-	$(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnish-cli.rst $@
+	$(top_builddir)/doc/sphinx/include/cli.rst
+	$(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnish-cli.rst $@
 
 varnish-counters.7: $(top_builddir)/doc/sphinx/reference/varnish-counters.rst \
-	$(top_srcdir)/doc/sphinx/include/counters.rst
-	$(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnish-counters.rst $@
+	$(top_builddir)/doc/sphinx/include/counters.rst
+	$(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnish-counters.rst $@
 
-vcl.7: $(top_srcdir)/doc/sphinx/reference/vcl.rst \
-	$(top_srcdir)/doc/sphinx/reference/vcl_var.rst \
+vcl.7: $(top_builddir)/doc/sphinx/reference/vcl.rst \
+	$(top_builddir)/doc/sphinx/reference/vcl_var.rst \
 	$(top_srcdir)/bin/varnishd/builtin.vcl
-	$(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/vcl.rst $@
+	$(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/vcl.rst $@
 
 vsl.7: $(top_builddir)/doc/sphinx/reference/vsl.rst \
 	$(top_builddir)/doc/sphinx/include/vsl-tags.rst
-	$(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/vsl.rst $@
+	$(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/vsl.rst $@
 
 vsl-query.7: $(top_builddir)/doc/sphinx/reference/vsl-query.rst
-	$(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/vsl-query.rst $@
+	$(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/vsl-query.rst $@
 
 varnishadm.1: $(top_builddir)/doc/sphinx/reference/varnishadm.rst
-	$(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishadm.rst $@
+	$(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishadm.rst $@
 
 varnishd.1: \
 	$(top_builddir)/doc/sphinx/reference/varnishd.rst \
 	$(top_builddir)/doc/sphinx/include/params.rst
-	$(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishd.rst $@
+	$(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishd.rst $@
 
 varnishncsa.1: \
 	$(top_builddir)/doc/sphinx/reference/varnishncsa.rst \
 	$(top_builddir)/doc/sphinx/include/varnishncsa_options.rst \
 	$(top_builddir)/doc/sphinx/include/varnishncsa_synopsis.rst
-	$(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishncsa.rst $@
+	$(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishncsa.rst $@
 
 varnishlog.1: \
 	$(top_builddir)/doc/sphinx/reference/varnishlog.rst \
 	$(top_builddir)/doc/sphinx/include/varnishlog_options.rst \
 	$(top_builddir)/doc/sphinx/include/varnishlog_synopsis.rst
-	$(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishlog.rst $@
+	$(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishlog.rst $@
 
 varnishstat.1: $(top_builddir)/doc/sphinx/reference/varnishstat.rst \
 	$(top_builddir)/doc/sphinx/include/varnishstat_options.rst \
 	$(top_builddir)/doc/sphinx/include/varnishstat_synopsis.rst \
 	$(top_builddir)/doc/sphinx/include/varnishstat_bindings.rst
-	$(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishstat.rst $@
+	$(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishstat.rst $@
 
 varnishtest.1: $(top_builddir)/doc/sphinx/reference/varnishtest.rst
-	$(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishtest.rst $@
+	$(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishtest.rst $@
 
 vtc.7: $(top_builddir)/doc/sphinx/reference/vtc.rst \
 	$(top_builddir)/doc/sphinx/include/vtc-syntax.rst
-	$(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/vtc.rst $@
+	$(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/vtc.rst $@
 
 varnishtop.1: \
 	$(top_builddir)/doc/sphinx/reference/varnishtop.rst \
 	$(top_builddir)/doc/sphinx/include/varnishtop_options.rst \
 	$(top_builddir)/doc/sphinx/include/varnishtop_synopsis.rst
-	$(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishtop.rst $@
+	$(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishtop.rst $@
 
 varnishhist.1: \
 	$(top_builddir)/doc/sphinx/reference/varnishhist.rst \
 	$(top_builddir)/doc/sphinx/include/varnishhist_options.rst \
 	$(top_builddir)/doc/sphinx/include/varnishhist_synopsis.rst
-	$(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishhist.rst $@
+	$(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishhist.rst $@
 
 vmod_cookie.3: $(top_builddir)/lib/libvmod_cookie/vmod_cookie.man.rst
 	$(BUILD_MAN) $? $@

From dridi at varni.sh  Wed Jun 10 08:40:02 2020
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Wed, 10 Jun 2020 08:40:02 +0000
Subject: [master] 146a28651 symlink documentation source files from
 $(srcdir) to $(builddir)
In-Reply-To: <20200610080607.593D0591B@lists.varnish-cache.org>
References: <20200610080607.593D0591B@lists.varnish-cache.org>
Message-ID: <CABoVN9DRYjmHs-HNd6u+ieXHxcQHNK80SaUO9ReK1BMPaDJgSg@mail.gmail.com>

On Wed, Jun 10, 2020 at 8:06 AM Nils Goroll <nils.goroll at uplex.de> wrote:
>
>
> commit 146a28651a140db54b8efd576e971efcd6cc3ae3
> Author: Nils Goroll <nils.goroll at uplex.de>
> Date:   Fri May 8 19:37:16 2020 +0200
>
>     symlink documentation source files from $(srcdir) to $(builddir)
>
>     ... if they differ.
>
>     Ref #3309

There should be a better way to manage this mess. I'm personally
tempted to wrap the sphinx-build execution in a shell script that
merges $(srcdir) and $(builddir) in a temp directory, and specify the
output directory as $(builddir)/html.

This change doesn't pass distcheck.

For rst2man execution I'd need to check all cases to enumerate VPATH quirks.

Question: do we need to keep rules to build all kinds of sphinx output
other than html?

> diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
> index fa71cacc9..ea010ab1a 100644
> --- a/doc/sphinx/Makefile.am
> +++ b/doc/sphinx/Makefile.am
> @@ -30,9 +30,13 @@ clean:
>  # use index.rst as an indicator if we have copied already
>  .PHONY: link_srcdir
>  link_srcdir:
> -       if test "x$(srcdir)" != "x$(builddir)" && test ! -f index.rst ; then \
> -               d=`pwd`/$(builddir) ; \
> -               cd $(srcdir) && find . -type f | cpio -dmp $${d} || true ; \
> +       if test "x$(srcdir)" != "x$(builddir)" && test ! -f index.rst; then \
> +               s=`realpath $(srcdir)`; \
> +               for f in `cd $$s && find . -type f`; do \
> +                       d=`dirname $$f`; \
> +                       test -d $$d || mkdir -p $$d; \
> +                       test -f $$f || ln -s $$s/$$f $$f; \
> +               done \
>         fi
>
>  # work around for make html called within doc/sphinx
> _______________________________________________
> varnish-commit mailing list
> varnish-commit at varnish-cache.org
> https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit

From nils.goroll at uplex.de  Wed Jun 10 11:36:21 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 10 Jun 2020 13:36:21 +0200
Subject: [master] 146a28651 symlink documentation source files from
 $(srcdir) to $(builddir)
In-Reply-To: <CABoVN9DRYjmHs-HNd6u+ieXHxcQHNK80SaUO9ReK1BMPaDJgSg@mail.gmail.com>
References: <20200610080607.593D0591B@lists.varnish-cache.org>
 <CABoVN9DRYjmHs-HNd6u+ieXHxcQHNK80SaUO9ReK1BMPaDJgSg@mail.gmail.com>
Message-ID: <652b2c72-1b95-db98-c026-9436227e8298@uplex.de>

On 10/06/2020 10:40, Dridi Boukelmoune wrote:
> There should be a better way to manage this mess.

I'd be very happy if you found one.

> This change doesn't pass distcheck.

Sorry, it did not happen locally. will look.

-- 

** * * UPLEX - Nils Goroll Systemoptimierung

Scheffelstra?e 32
22301 Hamburg

tel +49 40 28805731
mob +49 170 2723133
fax +49 40 42949753

xmpp://slink at jabber.int.uplex.de/

http://uplex.de/

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 488 bytes
Desc: OpenPGP digital signature
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20200610/24496172/attachment.bin>

From nils.goroll at uplex.de  Wed Jun 10 11:38:29 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 10 Jun 2020 13:38:29 +0200
Subject: [master] 146a28651 symlink documentation source files from
 $(srcdir) to $(builddir)
In-Reply-To: <652b2c72-1b95-db98-c026-9436227e8298@uplex.de>
References: <20200610080607.593D0591B@lists.varnish-cache.org>
 <CABoVN9DRYjmHs-HNd6u+ieXHxcQHNK80SaUO9ReK1BMPaDJgSg@mail.gmail.com>
 <652b2c72-1b95-db98-c026-9436227e8298@uplex.de>
Message-ID: <c7273975-5390-0eb3-b65e-7145e364fbc5@uplex.de>

On 10/06/2020 13:36, Nils Goroll wrote:
>> This change doesn't pass distcheck.
> Sorry, it did not happen locally. will look.

solaris does not have realpath, your Fedora vtest machine does not have cpio.

could you install it?

-- 

** * * UPLEX - Nils Goroll Systemoptimierung

Scheffelstra?e 32
22301 Hamburg

tel +49 40 28805731
mob +49 170 2723133
fax +49 40 42949753

xmpp://slink at jabber.int.uplex.de/

http://uplex.de/

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 488 bytes
Desc: OpenPGP digital signature
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20200610/b828e5e8/attachment.bin>

From dridi at varni.sh  Wed Jun 10 11:40:23 2020
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Wed, 10 Jun 2020 11:40:23 +0000
Subject: [master] 146a28651 symlink documentation source files from
 $(srcdir) to $(builddir)
In-Reply-To: <c7273975-5390-0eb3-b65e-7145e364fbc5@uplex.de>
References: <20200610080607.593D0591B@lists.varnish-cache.org>
 <CABoVN9DRYjmHs-HNd6u+ieXHxcQHNK80SaUO9ReK1BMPaDJgSg@mail.gmail.com>
 <652b2c72-1b95-db98-c026-9436227e8298@uplex.de>
 <c7273975-5390-0eb3-b65e-7145e364fbc5@uplex.de>
Message-ID: <CABoVN9CxLwiWdUCWw3w8OUapUR5UDozQVc4pMeV0eMwwEEa6dw@mail.gmail.com>

On Wed, Jun 10, 2020 at 11:38 AM Nils Goroll <nils.goroll at uplex.de> wrote:
>
> On 10/06/2020 13:36, Nils Goroll wrote:
> >> This change doesn't pass distcheck.
> > Sorry, it did not happen locally. will look.
>
> solaris does not have realpath, your Fedora vtest machine does not have cpio.
>
> could you install it?

Not a machine, but one of those new-fangled containers ;-)

will do

From dridi.boukelmoune at gmail.com  Wed Jun 10 15:50:07 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 10 Jun 2020 15:50:07 +0000 (UTC)
Subject: [master] eeee10ffa Retire unused sphinx builders
Message-ID: <20200610155007.C974A9624F@lists.varnish-cache.org>


commit eeee10ffa7039adee45d6890576bfd7f14897167
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Jun 10 16:53:28 2020 +0200

    Retire unused sphinx builders
    
    One exception could be the changes builder, but if we ever reach the
    point where we can automate the changelog initial creation we can bring
    it back.

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index ea010ab1a..a9a47e432 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -4,25 +4,16 @@
 # You can set these variables from the command line.
 SPHINXOPTS    =
 SPHINXBUILD   = $(SPHINX) -W -q -N
-PAPER         = a4
 BUILDDIR      = build
 
-ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees -D latex_elements.papersize=$(PAPER) $(SPHINXOPTS) $(builddir)
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(SPHINXOPTS) $(builddir)
 
-.PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest
+.PHONY: help clean html linkcheck doctest
 
 help:
 	@echo "Please use \`make <target>' where <target> is one of"
 	@echo "  html      to make standalone HTML files"
-	@echo "  dirhtml   to make HTML files named index.html in directories"
-	@echo "  pickle    to make pickle files"
-	@echo "  json      to make JSON files"
-	@echo "  htmlhelp  to make HTML files and a HTML help project"
-	@echo "  qthelp    to make HTML files and a qthelp project"
-	@echo "  latex     to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
-	@echo "  changes   to make an overview of all changed/added/deprecated items"
 	@echo "  linkcheck to check all external links for integrity"
-	@echo "  doctest   to run all doctests embedded in the documentation (if enabled)"
 
 clean:
 	-rm -rf $(BUILDDIR)/* $(CLEANFILES)
@@ -53,59 +44,12 @@ html: sphinx_prereq
 	@echo
 	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
 
-dirhtml: sphinx_prereq
-	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
-	@echo
-	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
-
-pickle: sphinx_prereq
-	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
-	@echo
-	@echo "Build finished; now you can process the pickle files."
-
-json: sphinx_prereq
-	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
-	@echo
-	@echo "Build finished; now you can process the JSON files."
-
-htmlhelp: sphinx_prereq
-	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
-	@echo
-	@echo "Build finished; now you can run HTML Help Workshop with the" \
-	      ".hhp project file in $(BUILDDIR)/htmlhelp."
-
-qthelp: sphinx_prereq
-	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
-	@echo
-	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
-	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
-	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Varnish.qhcp"
-	@echo "To view the help file:"
-	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Varnish.qhc"
-
-latex: sphinx_prereq
-	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
-	@echo
-	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
-	@echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \
-	      "run these through (pdf)latex."
-
-changes: sphinx_prereq
-	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
-	@echo
-	@echo "The overview file is in $(BUILDDIR)/changes."
-
 linkcheck: sphinx_prereq
 	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
 	@echo
 	@echo "Link check complete; look for any errors in the above output " \
 	      "or in $(BUILDDIR)/linkcheck/output.txt."
 
-doctest: sphinx_prereq
-	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
-	@echo "Testing of doctests in the sources finished, look at the " \
-	      "results in $(BUILDDIR)/doctest/output.txt."
-
 EXTRA_DIST = \
 	conf.py \
 	dev-guide \

From dridi.boukelmoune at gmail.com  Wed Jun 10 15:50:07 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 10 Jun 2020 15:50:07 +0000 (UTC)
Subject: [master] bbc94c1c7 Add missing sphinx-build make dependencies
Message-ID: <20200610155007.DE0BB96252@lists.varnish-cache.org>


commit bbc94c1c7a40d3d82f4d88be8ce39794f1237d9f
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Jun 10 17:17:50 2020 +0200

    Add missing sphinx-build make dependencies
    
    At this point I'm wondering whether we should talk about hacks when we
    really add missing dependencies. In some cases it boils down to RST
    being generated by programs we need to compile, what I call "regular"
    targets here.

diff --git a/Makefile.am b/Makefile.am
index a48e06819..f0e132389 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -56,6 +56,10 @@ LICENSE: all
 # before we try to run tests anywhere in the tree.
 check-recursive: all
 
+# XXX: This is the exact same hack since some parts of the documentation
+# are generated as regular targets but needed by the html special target.
+html-recursive: all
+
 cscope:
 	-rm -f cscope*
 	find . -name '*.[hcS]' > cscope.files

From dridi.boukelmoune at gmail.com  Wed Jun 10 15:50:08 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 10 Jun 2020 15:50:08 +0000 (UTC)
Subject: [master] aeaf698f0 Make sphinx rules a bit more informative
Message-ID: <20200610155008.08E1696256@lists.varnish-cache.org>


commit aeaf698f0105b363bd1cb04e66cd47d5d4cb6f04
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Jun 10 17:31:00 2020 +0200

    Make sphinx rules a bit more informative

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index a9a47e432..4ef49bc66 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -42,13 +42,13 @@ all: link_srcdir
 html: sphinx_prereq
 	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
 	@echo
-	@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+	@echo "Build finished. The HTML pages are in $(subdir)/$(BUILDDIR)/html."
 
 linkcheck: sphinx_prereq
 	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
 	@echo
 	@echo "Link check complete; look for any errors in the above output " \
-	      "or in $(BUILDDIR)/linkcheck/output.txt."
+	      "or in $(subdir)/$(BUILDDIR)/linkcheck/output.txt."
 
 EXTRA_DIST = \
 	conf.py \

From nils.goroll at uplex.de  Wed Jun 10 17:37:48 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 10 Jun 2020 19:37:48 +0200
Subject: [master] 146a28651 symlink documentation source files from
 $(srcdir) to $(builddir)
In-Reply-To: <CABoVN9CxLwiWdUCWw3w8OUapUR5UDozQVc4pMeV0eMwwEEa6dw@mail.gmail.com>
References: <20200610080607.593D0591B@lists.varnish-cache.org>
 <CABoVN9DRYjmHs-HNd6u+ieXHxcQHNK80SaUO9ReK1BMPaDJgSg@mail.gmail.com>
 <652b2c72-1b95-db98-c026-9436227e8298@uplex.de>
 <c7273975-5390-0eb3-b65e-7145e364fbc5@uplex.de>
 <CABoVN9CxLwiWdUCWw3w8OUapUR5UDozQVc4pMeV0eMwwEEa6dw@mail.gmail.com>
Message-ID: <d8530a4c-b37e-7719-a7d6-bfe88b6a5ec2@uplex.de>

FTR:

The reason for the failing sunos builds was

cp: preserving permissions for '../../varnish-trunk/doc/sphinx/dev-guide':
Operation not applicable

triggered by

distdir: $(DISTFILES)
	# ...
	cp -fpR $$d/$$file "$(distdir)$$dir"

in the autocrap-generated Makefile code.

I have not dug deep enough to understand why exactly it happens, but it seems to
happen only on tmpfs.

I have now moved the builds to zfs and, after manual runs looked good, hope that
vtest will become greener again.

Nils

-- 

** * * UPLEX - Nils Goroll Systemoptimierung

Scheffelstra?e 32
22301 Hamburg

tel +49 40 28805731
mob +49 170 2723133
fax +49 40 42949753

xmpp://slink at jabber.int.uplex.de/

http://uplex.de/

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 488 bytes
Desc: OpenPGP digital signature
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20200610/397e9532/attachment.bin>

From dridi.boukelmoune at gmail.com  Wed Jun 10 19:26:10 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 10 Jun 2020 19:26:10 +0000 (UTC)
Subject: [master] 8e45ba9fa circleci: Explicitly install cpio on DPKG systems
Message-ID: <20200610192610.D3323A20CD@lists.varnish-cache.org>


commit 8e45ba9faba203d6516175f521b44569be5dd612
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Jun 10 21:21:43 2020 +0200

    circleci: Explicitly install cpio on DPKG systems
    
    This is understandably no longer present on the buster base docker
    image, but I'm not adding this to installation instructions in the
    documentation because I hope to remove that dependency before the
    next release.

diff --git a/.circleci/config.yml b/.circleci/config.yml
index ee4c63b37..fe106028c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -173,6 +173,7 @@ jobs:
                     automake \
                     build-essential \
                     ca-certificates \
+                    cpio \
                     graphviz \
                     libedit-dev \
                     libjemalloc-dev \

From nils.goroll at uplex.de  Thu Jun 11 07:47:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 11 Jun 2020 07:47:07 +0000 (UTC)
Subject: [master] 6744a6936 Add the debugging aid to Assert Curses return codes
Message-ID: <20200611074707.6618BB2733@lists.varnish-cache.org>


commit 6744a6936f57d054d722c4e0a3dfcfe26d1f7ea3
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Jun 11 09:44:35 2020 +0200

    Add the debugging aid to Assert Curses return codes
    
    taken from varnishtop. NOOP because #if 0'd

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 1e5905093..0ff068390 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -57,6 +57,12 @@
 #include "vtim.h"
 #include "vapi/vsig.h"
 
+#if 0
+#define AC(x) assert((x) != ERR)
+#else
+#define AC(x) x
+#endif
+
 #define HIST_N 2000		/* how far back we remember */
 #define HIST_RES 100		/* bucket resolution */
 
@@ -146,7 +152,7 @@ update(void)
 	int i, j;
 	unsigned k, l;
 
-	erase();
+	AC(erase());
 
 	/* Draw horizontal axis */
 	for (k = 0; k < n; ++k)
@@ -199,7 +205,7 @@ update(void)
 			(void)mvaddch((LINES - 3) - l, k, '|');
 	}
 
-	refresh();
+	AC(refresh());
 }
 
 inline static void
@@ -380,12 +386,12 @@ do_curses(void *arg)
 	(void)arg;
 
 	initscr();
-	raw();
-	noecho();
-	nonl();
-	intrflush(stdscr, FALSE);
-	curs_set(0);
-	erase();
+	AC(raw());
+	AC(noecho());
+	AC(nonl());
+	AC(intrflush(stdscr, FALSE));
+	AC(curs_set(0));
+	AC(erase());
 	while (!VSIG_int && !VSIG_term && !VSIG_hup) {
 		AZ(pthread_mutex_lock(&mtx));
 		update();
@@ -398,16 +404,16 @@ do_curses(void *arg)
 			break;
 #ifdef KEY_RESIZE
 		case KEY_RESIZE:
-			erase();
+			AC(erase());
 			break;
 #endif
 		case '\014':	/* Ctrl-L */
 		case '\024':	/* Ctrl-T */
 			redrawwin(stdscr);
-			refresh();
+			AC(refresh());
 			break;
 		case '\032':	/* Ctrl-Z */
-			endwin();
+			AC(endwin());
 			AZ(raise(SIGTSTP));
 			break;
 		case '\003':	/* Ctrl-C */
@@ -441,7 +447,7 @@ do_curses(void *arg)
 			/* see below */
 			break;
 		default:
-			beep();
+			AC(beep());
 			break;
 		}
 
@@ -459,7 +465,7 @@ do_curses(void *arg)
 			AZ(pthread_mutex_unlock(&mtx));
 		}
 	}
-	endwin();
+	AC(endwin());
 	return (NULL);
 }
 

From dridi.boukelmoune at gmail.com  Thu Jun 11 14:13:07 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 11 Jun 2020 14:13:07 +0000 (UTC)
Subject: [master] cc42bd31a Only build manual pages in maintainer mode
Message-ID: <20200611141307.548D262CA0@lists.varnish-cache.org>


commit cc42bd31af078b8d0a2532d3744d2e662b16e422
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Jun 11 07:12:04 2020 +0200

    Only build manual pages in maintainer mode
    
    When we release a dist archive, it comes with the documentation but
    since some of it is generated by programs we end up it makes no real
    difference, and actually it makes things worse: rebuilding from the
    dist archive ends up with a rebuild of the documentation because some
    sources ended up being rebuilt. In other words, the documentation we
    ship is dead weight in the dist archive and doesn't reduce the number
    of build dependencies downstream.
    
    From now on, rst2man remains mandatory to build our manual pages but
    can safely be omitted by packaging scripts.

diff --git a/configure.ac b/configure.ac
index cbdeb59cf..ac1a3bdcf 100644
--- a/configure.ac
+++ b/configure.ac
@@ -37,6 +37,7 @@ CC="$PTHREAD_CC"
 
 AC_PROG_INSTALL
 
+AM_COND_IF([MAINTAINER_MODE], [dnl
 AC_ARG_WITH([rst2man],
   AS_HELP_STRING([--with-rst2man=PATH], [Location of rst2man (auto)]),
   [RST2MAN="$withval"],
@@ -47,6 +48,7 @@ if test "x$RST2MAN" = "xno"; then
   AC_MSG_ERROR(
     [rst2man is needed to build Varnish, please install python3-docutils.])
 fi
+])dnl AM_COND_IF MAINTAINER_MODE
 
 AC_ARG_WITH([sphinx-build],
   AS_HELP_STRING([--with-sphinx-build=PATH], [Location of sphinx-build (auto)]),
diff --git a/man/Makefile.am b/man/Makefile.am
index f04a0647c..dd4b7771c 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -24,8 +24,9 @@ dist_man_MANS = \
 	vmod_unix.3 \
 	vmod_proxy.3
 
-CLEANFILES = $(dist_man_MANS)
+MAINTAINERCLEANFILES = $(dist_man_MANS)
 
+if MAINTAINER_MODE
 RST2ANY_FLAGS = --halt=2
 
 BUILD_MAN = $(AM_V_GEN) $(RST2MAN) $(RST2ANY_FLAGS)
@@ -120,3 +121,4 @@ vmod_proxy.3: $(top_builddir)/lib/libvmod_proxy/vmod_proxy.man.rst
 	$(BUILD_MAN) $? $@
 
 .NOPATH: $(dist_man_MANS)
+endif # MAINTAINER_MODE

From dridi.boukelmoune at gmail.com  Thu Jun 11 14:34:06 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 11 Jun 2020 14:34:06 +0000 (UTC)
Subject: [master] 0b4a90c07 Soft retirement of autogen.sh
Message-ID: <20200611143406.92939636A8@lists.varnish-cache.org>


commit 0b4a90c07882af3e41d4ce921283c2af8232591d
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Jun 11 16:18:54 2020 +0200

    Soft retirement of autogen.sh
    
    We shouldn't need to care about all the details when autoreconf(1) can
    do that for us. The autoworld has changed over the last decade and this
    only affects Varnish developers since we ship release archives with a
    turnkey configure script. So let's see how much wreckage this change will
    cause.
    
    The script remains until its refcount drops to zero.
    
    In order to get early feedback for MacOS the relevant Travis CI job is
    already updated.

diff --git a/.travis.yml b/.travis.yml
index b3c37356e..0c0318393 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -86,7 +86,7 @@ jobs:
           update: true
       before_script:
         - export PATH="/usr/local/opt/sphinx-doc/bin:$PATH"
-        - ./autogen.sh
+        - autoreconf -i
         - ./configure
       script: *script-common
     - <<: *test-linux
diff --git a/autogen.des b/autogen.des
index 6fe0dcbca..7ef93d968 100755
--- a/autogen.des
+++ b/autogen.des
@@ -23,7 +23,7 @@ else
 fi
 
 rm -f configure
-(cd $SRCDIR && . ./autogen.sh 2>&1 | egrep -v "(subdir-objects|is in a subdirectory)")
+(cd $SRCDIR && autoreconf -i 2>&1 | egrep -v "(subdir-objects|is in a subdirectory)")
 
 # autoconf prior to 2.62 has issues with zsh 4.2 and newer
 CONFIG_SHELL=/bin/sh

From dridi.boukelmoune at gmail.com  Thu Jun 11 14:42:06 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 11 Jun 2020 14:42:06 +0000 (UTC)
Subject: [master] be1191ea9 Revert "Only build manual pages in maintainer mode"
Message-ID: <20200611144206.550C963B2A@lists.varnish-cache.org>


commit be1191ea9d5b0230ebb6c187ec40a51e23844d83
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Jun 11 16:34:49 2020 +0200

    Revert "Only build manual pages in maintainer mode"
    
    This reverts commit cc42bd31af078b8d0a2532d3744d2e662b16e422.
    
    For some reason Travis CI jobs use autogen.sh and don't build with all
    the bells and whistles we expect developers to work with. Despite the
    explicit mention in autogen.des:
    
        # Use this when doing code development
    
    Some of our continuous integration doesn't emulate this properly. At
    least on the VTEST and CircleCI sides we do things as expected. I will
    reintroduce this change later when Travis CI is ready to take it.

diff --git a/configure.ac b/configure.ac
index ac1a3bdcf..cbdeb59cf 100644
--- a/configure.ac
+++ b/configure.ac
@@ -37,7 +37,6 @@ CC="$PTHREAD_CC"
 
 AC_PROG_INSTALL
 
-AM_COND_IF([MAINTAINER_MODE], [dnl
 AC_ARG_WITH([rst2man],
   AS_HELP_STRING([--with-rst2man=PATH], [Location of rst2man (auto)]),
   [RST2MAN="$withval"],
@@ -48,7 +47,6 @@ if test "x$RST2MAN" = "xno"; then
   AC_MSG_ERROR(
     [rst2man is needed to build Varnish, please install python3-docutils.])
 fi
-])dnl AM_COND_IF MAINTAINER_MODE
 
 AC_ARG_WITH([sphinx-build],
   AS_HELP_STRING([--with-sphinx-build=PATH], [Location of sphinx-build (auto)]),
diff --git a/man/Makefile.am b/man/Makefile.am
index dd4b7771c..f04a0647c 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -24,9 +24,8 @@ dist_man_MANS = \
 	vmod_unix.3 \
 	vmod_proxy.3
 
-MAINTAINERCLEANFILES = $(dist_man_MANS)
+CLEANFILES = $(dist_man_MANS)
 
-if MAINTAINER_MODE
 RST2ANY_FLAGS = --halt=2
 
 BUILD_MAN = $(AM_V_GEN) $(RST2MAN) $(RST2ANY_FLAGS)
@@ -121,4 +120,3 @@ vmod_proxy.3: $(top_builddir)/lib/libvmod_proxy/vmod_proxy.man.rst
 	$(BUILD_MAN) $? $@
 
 .NOPATH: $(dist_man_MANS)
-endif # MAINTAINER_MODE

From dridi.boukelmoune at gmail.com  Fri Jun 12 11:53:06 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 12 Jun 2020 11:53:06 +0000 (UTC)
Subject: [master] 319ed9f6c circleci: Explicitly install cpio on Alpine Linux
Message-ID: <20200612115306.B3DF7BF3C5@lists.varnish-cache.org>


commit 319ed9f6c80cd635278258d7d5123b810755b447
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Jun 12 13:51:35 2020 +0200

    circleci: Explicitly install cpio on Alpine Linux
    
    See 8e45ba9faba203d6516175f521b44569be5dd612

diff --git a/.circleci/config.yml b/.circleci/config.yml
index fe106028c..7b600094a 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -191,6 +191,7 @@ jobs:
                     automake \
                     build-base \
                     ca-certificates \
+                    cpio \
                     gzip \
                     libedit-dev \
                     libtool \

From dridi.boukelmoune at gmail.com  Fri Jun 12 13:25:07 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 12 Jun 2020 13:25:07 +0000 (UTC)
Subject: [master] 1638ff0a6 Move the locking calls outside exp_mail_it
Message-ID: <20200612132507.EB6ABC1449@lists.varnish-cache.org>


commit 1638ff0a64a3deabfd1be9416d10e79db7e026c8
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Mar 19 11:08:50 2020 +0100

    Move the locking calls outside exp_mail_it
    
    This enables doing extra handling while holding the mutex specific to
    EXP_Insert/EXP_Remove before/after calling exp_mail_it.

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 2ed544dc2..293ced626 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -106,7 +106,8 @@ exp_mail_it(struct objcore *oc, uint8_t cmds)
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	assert(oc->refcnt > 0);
 
-	Lck_Lock(&exphdl->mtx);
+	Lck_AssertHeld(&exphdl->mtx);
+
 	if ((cmds | oc->exp_flags) & OC_EF_REFD) {
 		if (!(oc->exp_flags & OC_EF_POSTED)) {
 			if (cmds & OC_EF_REMOVE)
@@ -121,7 +122,6 @@ exp_mail_it(struct objcore *oc, uint8_t cmds)
 		VSC_C_main->exp_mailed++;
 		AZ(pthread_cond_signal(&exphdl->condvar));
 	}
-	Lck_Unlock(&exphdl->mtx);
 }
 
 /*--------------------------------------------------------------------
@@ -133,8 +133,11 @@ EXP_Remove(struct objcore *oc)
 {
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	if (oc->exp_flags & OC_EF_REFD)
+	if (oc->exp_flags & OC_EF_REFD) {
+		Lck_Lock(&exphdl->mtx);
 		exp_mail_it(oc, OC_EF_REMOVE);
+		Lck_Unlock(&exphdl->mtx);
+	}
 }
 
 /*--------------------------------------------------------------------
@@ -151,11 +154,13 @@ EXP_Insert(struct worker *wrk, struct objcore *oc)
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	assert(oc->refcnt >= 2);
 
-	AZ(oc->exp_flags & (OC_EF_INSERT | OC_EF_MOVE));
 	AZ(oc->flags & OC_F_DYING);
 
 	ObjSendEvent(wrk, oc, OEV_INSERT);
+	Lck_Lock(&exphdl->mtx);
+	AZ(oc->exp_flags & (OC_EF_INSERT | OC_EF_MOVE));
 	exp_mail_it(oc, OC_EF_INSERT | OC_EF_REFD | OC_EF_MOVE);
+	Lck_Unlock(&exphdl->mtx);
 }
 
 /*--------------------------------------------------------------------
@@ -187,8 +192,11 @@ EXP_Rearm(struct objcore *oc, vtim_real now,
 	VSL(SLT_ExpKill, 0, "EXP_Rearm p=%p E=%.6f e=%.6f f=0x%x", oc,
 	    oc->timer_when, when, oc->flags);
 
-	if (when < oc->t_origin || when < oc->timer_when)
+	if (when < oc->t_origin || when < oc->timer_when) {
+		Lck_Lock(&exphdl->mtx);
 		exp_mail_it(oc, OC_EF_MOVE);
+		Lck_Unlock(&exphdl->mtx);
+	}
 }
 
 /*--------------------------------------------------------------------

From dridi.boukelmoune at gmail.com  Fri Jun 12 13:25:08 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 12 Jun 2020 13:25:08 +0000 (UTC)
Subject: [master] 9210cd770 Only count exp_mailed events when actually posting
Message-ID: <20200612132508.086C3C144C@lists.varnish-cache.org>


commit 9210cd7709b3026d1ff6bc8ed78e1119d3d3eda5
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Mar 19 13:35:28 2020 +0100

    Only count exp_mailed events when actually posting
    
    When posting to the expiry thread, we wrongly counted exp_mailed also if
    the OC in question was already on the mail queue. This could cause a
    discrepency between the exp_mailed and exp_received counters.

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 293ced626..67d5b85ba 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -116,10 +116,10 @@ exp_mail_it(struct objcore *oc, uint8_t cmds)
 			else
 				VSTAILQ_INSERT_TAIL(&exphdl->inbox,
 				    oc, exp_list);
+			VSC_C_main->exp_mailed++;
 		}
 		oc->exp_flags |= cmds | OC_EF_POSTED;
 		AN(oc->exp_flags & OC_EF_REFD);
-		VSC_C_main->exp_mailed++;
 		AZ(pthread_cond_signal(&exphdl->condvar));
 	}
 }

From dridi.boukelmoune at gmail.com  Fri Jun 12 13:25:08 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 12 Jun 2020 13:25:08 +0000 (UTC)
Subject: [master] 7050ccee9 Repurpose OC_EF_REFD flag slightly
Message-ID: <20200612132508.2B047C144F@lists.varnish-cache.org>


commit 7050ccee978b77d94a622406dc3a181d876fbc2d
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Mar 19 15:17:45 2020 +0100

    Repurpose OC_EF_REFD flag slightly
    
    The OC_EF_REFD flag indicates whether expiry has a ref on the
    OC. Previously, the flag was only gained during the call to
    EXP_Insert. With this patch, and the helper function EXP_RefNewObjcore(),
    the flag is gained while holding the objhead mutex during
    HSH_Unbusy(). This enables the expiry functions to test on missing
    OC_EF_REFD and quickly return without having to take the main expiry
    mutex.

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 67d5b85ba..12feff3da 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -105,10 +105,11 @@ exp_mail_it(struct objcore *oc, uint8_t cmds)
 {
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	assert(oc->refcnt > 0);
+	AZ(cmds & OC_EF_REFD);
 
 	Lck_AssertHeld(&exphdl->mtx);
 
-	if ((cmds | oc->exp_flags) & OC_EF_REFD) {
+	if (oc->exp_flags & OC_EF_REFD) {
 		if (!(oc->exp_flags & OC_EF_POSTED)) {
 			if (cmds & OC_EF_REMOVE)
 				VSTAILQ_INSERT_HEAD(&exphdl->inbox,
@@ -119,11 +120,30 @@ exp_mail_it(struct objcore *oc, uint8_t cmds)
 			VSC_C_main->exp_mailed++;
 		}
 		oc->exp_flags |= cmds | OC_EF_POSTED;
-		AN(oc->exp_flags & OC_EF_REFD);
 		AZ(pthread_cond_signal(&exphdl->condvar));
 	}
 }
 
+/*--------------------------------------------------------------------
+ * Setup a new ObjCore for control by expire. Should be called with the
+ * ObjHead locked by HSH_Unbusy(/HSH_Insert) (in private access).
+ */
+
+void
+EXP_RefNewObjcore(struct objcore *oc)
+{
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+
+	Lck_AssertHeld(&oc->objhead->mtx);
+
+	AZ(oc->exp_flags);
+	assert(oc->refcnt >= 1);
+	oc->refcnt++;
+	oc->exp_flags |= OC_EF_REFD;
+}
+
+
+
 /*--------------------------------------------------------------------
  * Call EXP's attention to a an oc
  */
@@ -152,6 +172,10 @@ EXP_Insert(struct worker *wrk, struct objcore *oc)
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+
+	if (!(oc->exp_flags & OC_EF_REFD))
+		return;
+
 	assert(oc->refcnt >= 2);
 
 	AZ(oc->flags & OC_F_DYING);
@@ -159,7 +183,7 @@ EXP_Insert(struct worker *wrk, struct objcore *oc)
 	ObjSendEvent(wrk, oc, OEV_INSERT);
 	Lck_Lock(&exphdl->mtx);
 	AZ(oc->exp_flags & (OC_EF_INSERT | OC_EF_MOVE));
-	exp_mail_it(oc, OC_EF_INSERT | OC_EF_REFD | OC_EF_MOVE);
+	exp_mail_it(oc, OC_EF_INSERT | OC_EF_MOVE);
 	Lck_Unlock(&exphdl->mtx);
 }
 
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 831572956..9fd1b5091 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -305,7 +305,7 @@ HSH_Insert(struct worker *wrk, const void *digest, struct objcore *oc,
 	   objecthead. The new object inherits our objhead reference. */
 	oc->objhead = oh;
 	VTAILQ_INSERT_TAIL(&oh->objcs, oc, hsh_list);
-	oc->refcnt++;				// For EXP_Insert
+	EXP_RefNewObjcore(oc);
 	Lck_Unlock(&oh->mtx);
 
 	BAN_RefBan(oc, ban);
@@ -836,7 +836,7 @@ HSH_Unbusy(struct worker *wrk, struct objcore *oc)
 	assert(oh->refcnt > 0);
 	assert(oc->refcnt > 0);
 	if (!(oc->flags & OC_F_PRIVATE))
-		oc->refcnt++;			// For EXP_Insert
+		EXP_RefNewObjcore(oc); /* Takes a ref for expiry */
 	/* XXX: strictly speaking, we should sort in Date: order. */
 	VTAILQ_REMOVE(&oh->objcs, oc, hsh_list);
 	VTAILQ_INSERT_HEAD(&oh->objcs, oc, hsh_list);
@@ -846,8 +846,8 @@ HSH_Unbusy(struct worker *wrk, struct objcore *oc)
 		hsh_rush1(wrk, oh, &rush, HSH_RUSH_POLICY);
 	}
 	Lck_Unlock(&oh->mtx);
-	if (!(oc->flags & OC_F_PRIVATE))
-		EXP_Insert(wrk, oc);
+	EXP_Insert(wrk, oc); /* Does nothing unless EXP_RefNewObjcore was
+			      * called */
 	hsh_rush2(wrk, &rush);
 }
 
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index b0a6a1930..339a89ddd 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -165,6 +165,7 @@ void VDI_Init(void);
 /* cache_exp.c */
 vtim_real EXP_Ttl(const struct req *, const struct objcore *);
 vtim_real EXP_Ttl_grace(const struct req *, const struct objcore *oc);
+void EXP_RefNewObjcore(struct objcore *);
 void EXP_Insert(struct worker *wrk, struct objcore *oc);
 void EXP_Remove(struct objcore *);
 

From dridi.boukelmoune at gmail.com  Fri Jun 12 13:25:08 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 12 Jun 2020 13:25:08 +0000 (UTC)
Subject: [master] 68e262a7f Execute EXP_Insert after unbusy in HSH_Insert
Message-ID: <20200612132508.464C2C1454@lists.varnish-cache.org>


commit 68e262a7f90389f21297ba622c300a8576803b55
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Mar 23 14:23:02 2020 +0100

    Execute EXP_Insert after unbusy in HSH_Insert
    
    This makes the order of events the same as on real cache insertions.

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 9fd1b5091..b7c46a8a6 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -310,7 +310,6 @@ HSH_Insert(struct worker *wrk, const void *digest, struct objcore *oc,
 
 	BAN_RefBan(oc, ban);
 	AN(oc->ban);
-	EXP_Insert(wrk, oc);
 
 	/* Move the object first in the oh list, unbusy it and run the
 	   waitinglist if necessary */
@@ -322,6 +321,8 @@ HSH_Insert(struct worker *wrk, const void *digest, struct objcore *oc,
 		hsh_rush1(wrk, oh, &rush, HSH_RUSH_POLICY);
 	Lck_Unlock(&oh->mtx);
 	hsh_rush2(wrk, &rush);
+
+	EXP_Insert(wrk, oc);
 }
 
 /*---------------------------------------------------------------------

From dridi.boukelmoune at gmail.com  Fri Jun 12 13:25:08 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 12 Jun 2020 13:25:08 +0000 (UTC)
Subject: [master] 611a48e34 Allow EXP_Remove() to be called before EXP_Insert()
Message-ID: <20200612132508.6A3A0C1458@lists.varnish-cache.org>


commit 611a48e34610c6330f3c7291f2cb89e7cba8c032
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Mar 23 14:25:46 2020 +0100

    Allow EXP_Remove() to be called before EXP_Insert()
    
    Once HSH_Unbusy() has been called there is a possibility for
    EXP_Remove() to be called before the fetch thread has had a chance to call
    EXP_Insert(). By adding a OC_EF_NEW flag on the objects during
    HSH_Unbusy(), that is removed again during EXP_Insert(), we can keep track
    and clean up once EXP_Insert() is called by the inserting thread if
    EXP_Remove() was called in the mean time.
    
    This patch also removes the AZ(OC_F_DYING) in EXP_Insert(), as that is no
    longer a requirement.
    
    Fixes: #2999

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 12feff3da..c6bb36ef5 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -139,7 +139,7 @@ EXP_RefNewObjcore(struct objcore *oc)
 	AZ(oc->exp_flags);
 	assert(oc->refcnt >= 1);
 	oc->refcnt++;
-	oc->exp_flags |= OC_EF_REFD;
+	oc->exp_flags |= OC_EF_REFD | OC_EF_NEW;
 }
 
 
@@ -155,7 +155,14 @@ EXP_Remove(struct objcore *oc)
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	if (oc->exp_flags & OC_EF_REFD) {
 		Lck_Lock(&exphdl->mtx);
-		exp_mail_it(oc, OC_EF_REMOVE);
+		if (oc->exp_flags & OC_EF_NEW) {
+			/* EXP_Insert has not been called for this object
+			 * yet. Mark it for removal, and EXP_Insert will
+			 * clean up once it is called. */
+			AZ(oc->exp_flags & OC_EF_POSTED);
+			oc->exp_flags |= OC_EF_REMOVE;
+		} else
+			exp_mail_it(oc, OC_EF_REMOVE);
 		Lck_Unlock(&exphdl->mtx);
 	}
 }
@@ -169,22 +176,37 @@ EXP_Remove(struct objcore *oc)
 void
 EXP_Insert(struct worker *wrk, struct objcore *oc)
 {
+	unsigned remove_race = 0;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 
+	AZ(oc->flags & OC_F_BUSY);
+
 	if (!(oc->exp_flags & OC_EF_REFD))
 		return;
 
 	assert(oc->refcnt >= 2);
 
-	AZ(oc->flags & OC_F_DYING);
-
 	ObjSendEvent(wrk, oc, OEV_INSERT);
+
 	Lck_Lock(&exphdl->mtx);
-	AZ(oc->exp_flags & (OC_EF_INSERT | OC_EF_MOVE));
-	exp_mail_it(oc, OC_EF_INSERT | OC_EF_MOVE);
+	AN(oc->exp_flags & OC_EF_NEW);
+	oc->exp_flags &= ~OC_EF_NEW;
+	AZ(oc->exp_flags & (OC_EF_INSERT | OC_EF_MOVE | OC_EF_POSTED));
+	if (oc->exp_flags & OC_EF_REMOVE) {
+		/* We raced some other thread executing EXP_Remove */
+		remove_race = 1;
+		oc->exp_flags &= ~(OC_EF_REFD | OC_EF_REMOVE);
+	} else
+		exp_mail_it(oc, OC_EF_INSERT | OC_EF_MOVE);
 	Lck_Unlock(&exphdl->mtx);
+
+	if (remove_race) {
+		ObjSendEvent(wrk, oc, OEV_EXPIRE);
+		(void)HSH_DerefObjCore(wrk, &oc, 0);
+		AZ(oc);
+	}
 }
 
 /*--------------------------------------------------------------------
@@ -218,7 +240,12 @@ EXP_Rearm(struct objcore *oc, vtim_real now,
 
 	if (when < oc->t_origin || when < oc->timer_when) {
 		Lck_Lock(&exphdl->mtx);
-		exp_mail_it(oc, OC_EF_MOVE);
+		if (oc->exp_flags & OC_EF_NEW) {
+			/* EXP_Insert has not been called yet, do nothing
+			 * as the initial insert will execute the move
+			 * operation. */
+		} else
+			exp_mail_it(oc, OC_EF_MOVE);
 		Lck_Unlock(&exphdl->mtx);
 	}
 }
diff --git a/include/tbl/oc_exp_flags.h b/include/tbl/oc_exp_flags.h
index 926ab5d8e..ffa570006 100644
--- a/include/tbl/oc_exp_flags.h
+++ b/include/tbl/oc_exp_flags.h
@@ -35,6 +35,7 @@ OC_EXP_FLAG(REFD,	refd,		(1<<2))
 OC_EXP_FLAG(MOVE,	move,		(1<<3))
 OC_EXP_FLAG(INSERT,	insert,		(1<<4))
 OC_EXP_FLAG(REMOVE,	remove,		(1<<5))
+OC_EXP_FLAG(NEW,	new,		(1<<6))
 #undef OC_EXP_FLAG
 
 /*lint -restore */

From dridi.boukelmoune at gmail.com  Fri Jun 12 15:01:06 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 12 Jun 2020 15:01:06 +0000 (UTC)
Subject: [master] c82e3abab New VRE_quote() function
Message-ID: <20200612150106.DB5CC5F2D@lists.varnish-cache.org>


commit c82e3ababca5d6533f07ec9dd4c064cfbee82f75
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Jun 5 17:09:22 2020 +0200

    New VRE_quote() function
    
    This is a tool for VMOD authors for the use case of building a regular
    expression partially from arbitrary input, where the input is intended
    for an exact match.
    
    For example, one could implement a dispatch feature depending on the
    request's host header, building something like:
    
        "\.?\Q" + req.http.host + "\E$"
    
    A malicious client could however hijack the regular expression with a
    \E sequence in the host header. To get safely to this result you can
    do this instead in pseudo-code before compiling the regex:
    
        VSB_cat(vsb, "\\.?");
        VRE_quote(vsb, req.http.host);
        VSB_putc(vsb, '$');
    
    The input is enclosed with PCRE's \Q and \E escape sequences, ensuring
    that \E sequences in the input string don't allow Little Bobby Tables'
    cousin to mess with your regular expressions.

diff --git a/bin/varnishtest/tests/v00065.vtc b/bin/varnishtest/tests/v00065.vtc
new file mode 100644
index 000000000..b941c19b9
--- /dev/null
+++ b/bin/varnishtest/tests/v00065.vtc
@@ -0,0 +1,33 @@
+varnishtest "VRT_re_quote() coverage"
+
+varnish v1 -vcl {
+	import debug;
+
+	backend be none;
+
+	sub vcl_recv {
+		return (synth(200));
+	}
+
+	sub vcl_synth {
+		set resp.http.sanity = regsub("\Q", "\Q\Q\E", "sane");
+		set resp.http.q0 = debug.re_quote("");
+		set resp.http.q1 = debug.re_quote("hello");
+		set resp.http.q2 = debug.re_quote("hello\E");
+		set resp.http.q3 = debug.re_quote("hello\Eworld");
+		set resp.http.q4 = debug.re_quote("\E");
+		set resp.http.q5 = debug.re_quote("\Q");
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.http.sanity == sane
+	expect resp.http.q0 == {}
+	expect resp.http.q1 == {\Qhello\E}
+	expect resp.http.q2 == {\Qhello\\EE}
+	expect resp.http.q3 == {\Qhello\\EE\Qworld\E}
+	expect resp.http.q4 == {\Q\\EE}
+	expect resp.http.q5 == {\Q\Q\E}
+} -run
diff --git a/include/vre.h b/include/vre.h
index b139c8a06..6dcf0e593 100644
--- a/include/vre.h
+++ b/include/vre.h
@@ -38,6 +38,7 @@
 #define VRE_H_INCLUDED
 
 struct vre;
+struct vsb;
 
 struct vre_limits {
 	unsigned	match;
@@ -59,5 +60,6 @@ int VRE_exec(const vre_t *code, const char *subject, int length,
     int startoffset, int options, int *ovector, int ovecsize,
     const volatile struct vre_limits *lim);
 void VRE_free(vre_t **);
+void VRE_quote(struct vsb *, const char *);
 
 #endif /* VRE_H_INCLUDED */
diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index 7ef62da8d..4b9c6d7b1 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -36,6 +36,7 @@
 #include "vdef.h"
 
 #include "vas.h"	// XXX Flexelint "not used" - but req'ed for assert()
+#include "vsb.h"
 #include "miniobj.h"
 
 #include "vre.h"
@@ -147,3 +148,17 @@ VRE_free(vre_t **vv)
 		pcre_free(v->re);
 	FREE_OBJ(v);
 }
+
+void
+VRE_quote(struct vsb *vsb, const char *src)
+{
+	const char *b, *e;
+
+	CHECK_OBJ_NOTNULL(vsb, VSB_MAGIC);
+	if (src == NULL)
+		return;
+	for (b = src; (e = strstr(b, "\\E")) != NULL; b = e + 2)
+		VSB_printf(vsb, "\\Q%.*s\\\\EE", (int)(e - b), b);
+	if (*b != '\0')
+		VSB_printf(vsb, "\\Q%s\\E", b);
+}
diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc
index 89cc648e4..63c326cf4 100644
--- a/lib/libvmod_debug/vmod.vcc
+++ b/lib/libvmod_debug/vmod.vcc
@@ -300,3 +300,7 @@ fail any rollback before ok_rollback() is called
 $Function VOID ok_rollback()
 
 Allow rollbacks. Must be called before the end of the task.
+
+$Function STRING re_quote(STRING)
+
+Quote an input string to be usable for an exact match in a regular expression.
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index 01013b892..aa54dfeb8 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -39,6 +39,7 @@
 #include "cache/cache_varnishd.h"
 #include "cache/cache_filter.h"
 
+#include "vre.h"
 #include "vsa.h"
 #include "vtim.h"
 #include "vcc_if.h"
@@ -1145,3 +1146,19 @@ xyzzy_ok_rollback(VRT_CTX)
 	p->priv = NULL;
 	p->free = NULL;
 }
+
+VCL_STRING v_matchproto_(td_xyzzy_debug_re_quote)
+xyzzy_re_quote(VRT_CTX, VCL_STRING s)
+{
+	struct vsb vsb[1];
+	char *q;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->ws, WS_MAGIC);
+	WS_VSB_new(vsb, ctx->ws);
+	VRE_quote(vsb, s);
+	q = WS_VSB_finish(vsb, ctx->ws, NULL);
+	if (q == NULL)
+		WS_MarkOverflow(ctx->ws);
+	return (q);
+}

From dridi.boukelmoune at gmail.com  Sun Jun 14 09:09:08 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Sun, 14 Jun 2020 09:09:08 +0000 (UTC)
Subject: [master] 1e1de0987 Wrong function name
Message-ID: <20200614090908.2F0D8A6F99@lists.varnish-cache.org>


commit 1e1de0987a0df4fb31adb7ced6a94180157606ac
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Sun Jun 14 10:57:26 2020 +0200

    Wrong function name
    
    Refs #3345

diff --git a/bin/varnishtest/tests/v00065.vtc b/bin/varnishtest/tests/v00065.vtc
index b941c19b9..5b818765f 100644
--- a/bin/varnishtest/tests/v00065.vtc
+++ b/bin/varnishtest/tests/v00065.vtc
@@ -1,4 +1,4 @@
-varnishtest "VRT_re_quote() coverage"
+varnishtest "VRE_quote() coverage"
 
 varnish v1 -vcl {
 	import debug;

From dridi.boukelmoune at gmail.com  Sun Jun 14 09:09:08 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Sun, 14 Jun 2020 09:09:08 +0000 (UTC)
Subject: [master] 4dc4d4564 We need ssize_t in vre.c
Message-ID: <20200614090908.46144A6F9D@lists.varnish-cache.org>


commit 4dc4d4564c1b6a87dbb311616cc8edc4b104c35f
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Sun Jun 14 10:57:53 2020 +0200

    We need ssize_t in vre.c
    
    Refs #3345

diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c
index 4b9c6d7b1..5f2a81a72 100644
--- a/lib/libvarnish/vre.c
+++ b/lib/libvarnish/vre.c
@@ -32,6 +32,7 @@
 
 #include <pcre.h>
 #include <string.h>
+#include <unistd.h>
 
 #include "vdef.h"
 

From phk at FreeBSD.org  Mon Jun 15 07:46:08 2020
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 15 Jun 2020 07:46:08 +0000 (UTC)
Subject: [master] 0d4966b40 Experiment to see if curses error reporting has
 improved.
Message-ID: <20200615074609.031499C7A4@lists.varnish-cache.org>


commit 0d4966b404e33d7f973b9c15e44561fd71151c3e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jun 15 07:41:58 2020 +0000

    Experiment to see if curses error reporting has improved.
    
    Ref: #3348, #741

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 0ff068390..a2e94cb5d 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -57,7 +57,7 @@
 #include "vtim.h"
 #include "vapi/vsig.h"
 
-#if 0
+#if 1
 #define AC(x) assert((x) != ERR)
 #else
 #define AC(x) x

From nils.goroll at uplex.de  Mon Jun 15 13:02:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 15 Jun 2020 13:02:07 +0000 (UTC)
Subject: [master] a5fb7797a Deprecate BackendReuse in favor of BackendClose
Message-ID: <20200615130207.EBED3A82EE@lists.varnish-cache.org>


commit a5fb7797a3f9739554018d63d47076bebfb54007
Author: Reza Naghibi <reza at naghibi.com>
Date:   Wed Jun 10 08:35:42 2020 -0400

    Deprecate BackendReuse in favor of BackendClose

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 5c1f5b58d..fb289c7a1 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -242,7 +242,7 @@ vbe_dir_finish(VRT_CTX, VCL_BACKEND d)
 		Lck_Lock(&bp->mtx);
 	} else {
 		assert (PFD_State(pfd) == PFD_STATE_USED);
-		VSLb(bo->vsl, SLT_BackendReuse, "%d %s", *PFD_Fd(pfd),
+		VSLb(bo->vsl, SLT_BackendClose, "%d %s", *PFD_Fd(pfd),
 		    VRT_BACKEND_string(bp->director));
 		Lck_Lock(&bp->mtx);
 		VSC_C_main->backend_recycle++;
diff --git a/bin/varnishtest/tests/e00008.vtc b/bin/varnishtest/tests/e00008.vtc
index fce13457d..4c516e003 100644
--- a/bin/varnishtest/tests/e00008.vtc
+++ b/bin/varnishtest/tests/e00008.vtc
@@ -83,19 +83,19 @@ logexpect l1 -v v1 -g vxid {
 	expect 0 = ESI_xmlerror {^ERR after 665 ESI 1.0 </esi:include> illegal end-tag$}
 	expect 0 = ESI_xmlerror {^ERR after 767 XML 1.0 Missing end attribute delimiter$}
 	expect 0 = ESI_xmlerror {^ERR after 843 ESI 1.0 <esi:include> has whitespace in src= attribute$}
-	expect 0 = BackendReuse
+	expect 0 = BackendClose
 } -start
 
 logexpect l2 -v v1 -g vxid {
 	expect * * BereqURL	{^/body$}
 	expect * = ESI_xmlerror {^ERR after 30 VEP ended inside a tag$}
-	expect 0 = BackendReuse
+	expect 0 = BackendClose
 } -start
 
 logexpect l3 -v v1 -g vxid {
 	expect * * BereqURL	{^/body2$}
 	expect * = ESI_xmlerror {^ERR after 39 VEP ended inside a tag$}
-	expect 0 = BackendReuse
+	expect 0 = BackendClose
 } -start
 
 varnish v1 -cliok "param.set debug +esi_chop"
diff --git a/bin/varnishtest/tests/e00019.vtc b/bin/varnishtest/tests/e00019.vtc
index 4c69bd19f..98ed56f3e 100644
--- a/bin/varnishtest/tests/e00019.vtc
+++ b/bin/varnishtest/tests/e00019.vtc
@@ -63,7 +63,7 @@ logexpect l1 -v v1 -g vxid -q "vxid == 1002" {
 	expect 0 = ESI_xmlerror {^WARN after 107 ESI 1.0 <esi:include> lacks final '/'$}
 	expect 0 = ESI_xmlerror {^ERR after 130 ESI 1.0 <esi:bogus> element$}
 	expect 0 = ESI_xmlerror {^ERR after 131837 VEP ended inside a tag$}
-	expect 0 = BackendReuse
+	expect 0 = BackendClose
 } -start
 
 client c1 {
diff --git a/bin/varnishtest/tests/e00020.vtc b/bin/varnishtest/tests/e00020.vtc
index 7c27efa05..75210778f 100644
--- a/bin/varnishtest/tests/e00020.vtc
+++ b/bin/varnishtest/tests/e00020.vtc
@@ -27,7 +27,7 @@ logexpect l1 -v v1 -g vxid {
 	expect 0 = ESI_xmlerror {^ERR after 3 ESI 1.0 <esi:include> element nested in <esi:remove>$}
 	expect 0 = ESI_xmlerror {^ERR after 3 ESI 1.0 Nested <!--esi element in <esi:remove>$}
 	expect 0 = Gzip         {^U}
-	expect 0 = BackendReuse
+	expect 0 = BackendClose
 } -start
 
 client c1 {
diff --git a/bin/varnishtest/tests/e00022.vtc b/bin/varnishtest/tests/e00022.vtc
index 446d2e2e8..2a99c8acd 100644
--- a/bin/varnishtest/tests/e00022.vtc
+++ b/bin/varnishtest/tests/e00022.vtc
@@ -33,7 +33,7 @@ logexpect l1 -v v1 -g vxid {
 	expect 0 = ESI_xmlerror {^ERR after 24 ESI 1.0 Nested <!--esi element in <esi:remove>$}
 	expect 0 = Gzip         {^G}
 	expect 0 = Gzip         {^U}
-	expect 0 = BackendReuse
+	expect 0 = BackendClose
 } -start
 
 client c1 {
diff --git a/bin/varnishtest/tests/r00894.vtc b/bin/varnishtest/tests/r00894.vtc
index 4a815233e..0cb094822 100644
--- a/bin/varnishtest/tests/r00894.vtc
+++ b/bin/varnishtest/tests/r00894.vtc
@@ -14,7 +14,7 @@ varnish v1 -vcl+backend {
 logexpect l1 -v v1 -g raw {
 	expect * * Fetch_Body
 	expect 0 = ESI_xmlerror {^ERR after 5 ESI 1.0 <esi:include> has multiple src= attributes$}
-	expect 0 = BackendReuse
+	expect 0 = BackendClose
 } -start
 
 client c1 {
diff --git a/bin/varnishtest/tests/r01092.vtc b/bin/varnishtest/tests/r01092.vtc
index a53b7ba47..468f6a591 100644
--- a/bin/varnishtest/tests/r01092.vtc
+++ b/bin/varnishtest/tests/r01092.vtc
@@ -27,7 +27,7 @@ varnish v1 -vcl+backend {
 logexpect l1 -v v1 -g raw {
 	expect * * Fetch_Body
 	expect 0 * ESI_xmlerror {^ERR after 66 ESI 1.0 Nested <!--esi element in <esi:remove>$}
-	expect 0 = BackendReuse
+	expect 0 = BackendClose
 } -start
 
 client c1 {
diff --git a/bin/varnishtest/tests/r01355.vtc b/bin/varnishtest/tests/r01355.vtc
index 44b1b39e9..5385b2a31 100644
--- a/bin/varnishtest/tests/r01355.vtc
+++ b/bin/varnishtest/tests/r01355.vtc
@@ -24,11 +24,11 @@ varnish v1 -vcl+backend {
 logexpect l1 -v v1 -g raw {
 	expect * * Fetch_Body
 	expect 0 = ESI_xmlerror {^No ESI processing, first char not '<' but BOM. .See feature esi_remove_bom.$}
-	expect 0 = BackendReuse
+	expect 0 = BackendClose
 # XXX another logexpect weirdness - why can't we catch the second occurrence?
 #	expect * * Fetch_Body
 #	expect 0 = ESI_xmlerror {^No ESI processing, first char not '<' but BOM. .See feature esi_remove_bom.$}
-#	expect 0 = BackendReuse
+#	expect 0 = BackendClose
 } -start
 
 client c1 {
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 31fed3547..bebfa738b 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -126,6 +126,7 @@ SLTM(BackendReuse, 0, "Backend connection put up for reuse",
 	"\t|  +- Backend display name\n"
 	"\t+---- Connection file descriptor\n"
 	"\n"
+	NOSUP_NOTICE
 )
 
 SLTM(BackendClose, 0, "Backend connection closed",

From nils.goroll at uplex.de  Mon Jun 15 13:02:08 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 15 Jun 2020 13:02:08 +0000 (UTC)
Subject: [master] 060175fec Add connection reuse status to BackendOpen and
 BackendClose
Message-ID: <20200615130208.1E68DA82F1@lists.varnish-cache.org>


commit 060175fec8856ea598e8ecf9d48d719ecad97e00
Author: Reza Naghibi <reza at naghibi.com>
Date:   Wed Jun 10 08:52:19 2020 -0400

    Add connection reuse status to BackendOpen and BackendClose

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index fb289c7a1..4eb18b8b9 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -202,8 +202,9 @@ vbe_dir_getfd(VRT_CTX, struct worker *wrk, struct backend *bp,
 
 	PFD_LocalName(pfd, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1);
 	PFD_RemoteName(pfd, abuf2, sizeof abuf2, pbuf2, sizeof pbuf2);
-	VSLb(bo->vsl, SLT_BackendOpen, "%d %s %s %s %s %s",
-	    *fdp, VRT_BACKEND_string(bp->director), abuf2, pbuf2, abuf1, pbuf1);
+	VSLb(bo->vsl, SLT_BackendOpen, "%d %s %s %s %s %s %s",
+	    *fdp, VRT_BACKEND_string(bp->director), abuf2, pbuf2, abuf1, pbuf1,
+	    PFD_State(pfd) == PFD_STATE_STOLEN ? "reuse" : "connect");
 
 	INIT_OBJ(bo->htc, HTTP_CONN_MAGIC);
 	bo->htc->priv = pfd;
@@ -235,14 +236,14 @@ vbe_dir_finish(VRT_CTX, VCL_BACKEND d)
 		assert(bo->htc->doclose == SC_TX_PIPE ||
 		    bo->htc->doclose == SC_RX_TIMEOUT);
 	if (bo->htc->doclose != SC_NULL || bp->proxy_header != 0) {
-		VSLb(bo->vsl, SLT_BackendClose, "%d %s", *PFD_Fd(pfd),
+		VSLb(bo->vsl, SLT_BackendClose, "%d %s close", *PFD_Fd(pfd),
 		    VRT_BACKEND_string(bp->director));
 		VTP_Close(&pfd);
 		AZ(pfd);
 		Lck_Lock(&bp->mtx);
 	} else {
 		assert (PFD_State(pfd) == PFD_STATE_USED);
-		VSLb(bo->vsl, SLT_BackendClose, "%d %s", *PFD_Fd(pfd),
+		VSLb(bo->vsl, SLT_BackendClose, "%d %s recycle", *PFD_Fd(pfd),
 		    VRT_BACKEND_string(bp->director));
 		Lck_Lock(&bp->mtx);
 		VSC_C_main->backend_recycle++;
diff --git a/bin/varnishtest/tests/b00060.vtc b/bin/varnishtest/tests/b00060.vtc
index 959e765d6..a8bfba950 100644
--- a/bin/varnishtest/tests/b00060.vtc
+++ b/bin/varnishtest/tests/b00060.vtc
@@ -24,5 +24,5 @@ logexpect l2 -v v1 -d 1 -g vxid {
 
 logexpect l3 -v v1 -d 1 -g vxid {
 	expect * *	Begin		^bereq
-	expect * =	BackendOpen	"0.0.0.0 0 0.0.0.0 0$"
+	expect * =	BackendOpen	"0.0.0.0 0 0.0.0.0 0 connect$"
 } -run
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index bebfa738b..ceb7ff917 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -106,14 +106,15 @@ SLTM(SessClose, 0, "Client connection closed",
 SLTM(BackendOpen, 0, "Backend connection opened",
 	"Logged when a new backend connection is opened.\n\n"
 	"The format is::\n\n"
-	"\t%d %s %s %s %s %s\n"
-	"\t|  |  |  |  |  |\n"
-	"\t|  |  |  |  |  +- Local port\n"
-	"\t|  |  |  |  +---- Local address\n"
-	"\t|  |  |  +------- Remote port\n"
-	"\t|  |  +---------- Remote address\n"
-	"\t|  +------------- Backend display name\n"
-	"\t+---------------- Connection file descriptor\n"
+	"\t%d %s %s %s %s %s %s\n"
+	"\t|  |  |  |  |  |  |\n"
+	"\t|  |  |  |  |  |  +- \"connect\" or \"reuse\"\n"
+	"\t|  |  |  |  |  +---- Local port\n"
+	"\t|  |  |  |  +------- Local address\n"
+	"\t|  |  |  +---------- Remote port\n"
+	"\t|  |  +------------- Remote address\n"
+	"\t|  +---------------- Backend display name\n"
+	"\t+------------------- Connection file descriptor\n"
 	"\n"
 )
 
@@ -132,11 +133,12 @@ SLTM(BackendReuse, 0, "Backend connection put up for reuse",
 SLTM(BackendClose, 0, "Backend connection closed",
 	"Logged when a backend connection is closed.\n\n"
 	"The format is::\n\n"
-	"\t%d %s [ %s ]\n"
-	"\t|  |    |\n"
-	"\t|  |    +- Optional reason\n"
-	"\t|  +------ Backend display name\n"
-	"\t+--------- Connection file descriptor\n"
+	"\t%d %s %s [ %s ]\n"
+	"\t|  |  |    |\n"
+	"\t|  |  |    +- Optional reason\n"
+	"\t|  |  +------ \"close\" or \"recycle\"\n"
+	"\t|  +--------- Backend display name\n"
+	"\t+------------ Connection file descriptor\n"
 	"\n"
 )
 

From nils.goroll at uplex.de  Mon Jun 15 18:17:09 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 15 Jun 2020 18:17:09 +0000 (UTC)
Subject: [master] 6c0a38b66 fix bereq rollback with retry
Message-ID: <20200615181709.9EC99B2967@lists.varnish-cache.org>


commit 6c0a38b6622a76b0b7e91859f0696831c2fee805
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Jun 15 20:13:56 2020 +0200

    fix bereq rollback with retry
    
    When retrying after a rollback, we might have unread bytes of the
    backend response body. As we need to finalize the director for rollback,
    all we can do in this case is to close the connection.
    
    Spotted by reza, who also provided the test case, thank you.
    
    Closes #3353

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 277a52e4d..c143565f3 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -106,6 +106,11 @@ void Bereq_Rollback(struct busyobj *bo)
 {
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
+	if (bo->htc != NULL &&
+	    bo->htc->body_status != BS_NONE &&
+	    bo->htc->body_status != BS_TAKEN)
+		bo->htc->doclose = SC_RESP_CLOSE;
+
 	vbf_cleanup(bo);
 	VCL_TaskLeave(bo->privs);
 	VCL_TaskEnter(bo->privs);
diff --git a/bin/varnishtest/tests/r03353.vtc b/bin/varnishtest/tests/r03353.vtc
new file mode 100644
index 000000000..d5ab5f1a4
--- /dev/null
+++ b/bin/varnishtest/tests/r03353.vtc
@@ -0,0 +1,33 @@
+varnishtest "Test rollback and retry"
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr "Content-Length: 3"
+	expect_close
+	accept
+
+	rxreq
+	txresp -body xxx
+} -start
+
+varnish v1 -vcl+backend {
+	import std;
+
+	sub vcl_backend_response {
+		if (bereq.retries == 0) {
+			std.rollback(bereq);
+		}
+	}
+
+	sub vcl_backend_error {
+		if (bereq.retries == 0) {
+			return (retry);
+		}
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run

From dridi.boukelmoune at gmail.com  Tue Jun 16 15:48:09 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 16 Jun 2020 15:48:09 +0000 (UTC)
Subject: [master] 9b9a955fa Firm retirement of autogen.sh
Message-ID: <20200616154809.33C88AD7C7@lists.varnish-cache.org>


commit 9b9a955fa8d71303cdd1087242a8979a60adea67
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jun 16 17:33:07 2020 +0200

    Firm retirement of autogen.sh
    
    The script remains until its refcount drops to zero.
    
    Refs 0b4a90c07882

diff --git a/autogen.sh b/autogen.sh
index b7ba7ae5a..f3d47408e 100755
--- a/autogen.sh
+++ b/autogen.sh
@@ -1,51 +1,5 @@
 #!/bin/sh
 #
+# Use autogen.des when doing code development
 
-warn() {
-	echo "WARNING: $@" 1>&2
-}
-
-case `uname -s` in
-Darwin)
-    LIBTOOLIZE=glibtoolize
-    ;;
-FreeBSD)
-    LIBTOOLIZE=libtoolize
-    ;;
-NetBSD)
-    LIBTOOLIZE=libtoolize
-    ;;
-OpenBSD)
-    LIBTOOLIZE=libtoolize
-    ;;
-Linux)
-    LIBTOOLIZE=libtoolize
-    ;;
-SunOS)
-    LIBTOOLIZE=libtoolize
-    ;;
-*)
-    warn "unrecognized platform:" `uname -s`
-    LIBTOOLIZE=libtoolize
-esac
-
-automake_version=`automake --version | tr ' ' '\n' | egrep '^[0-9]\.[0-9a-z.-]+'`
-if [ -z "$automake_version" ] ; then
-    warn "unable to determine automake version"
-else
-    case $automake_version in
-	0.*|1.[0-8]|1.[0-8][.-]*)
-	    warn "automake ($automake_version) detected; 1.9 or newer recommended"
-	    ;;
-	*)
-	    ;;
-    esac
-fi
-
-set -ex
-
-$LIBTOOLIZE --copy --force
-aclocal -I m4 -I .
-autoheader
-automake --add-missing --copy --foreign
-autoconf
+autoreconf -i

From dridi.boukelmoune at gmail.com  Tue Jun 16 15:48:09 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 16 Jun 2020 15:48:09 +0000 (UTC)
Subject: [master] e33d0cf15 Enable maintainer mode after autogen.sh
Message-ID: <20200616154809.48C7EAD7CC@lists.varnish-cache.org>


commit e33d0cf155473052f463f3422916a5b5e83e8a5f
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jun 16 17:41:13 2020 +0200

    Enable maintainer mode after autogen.sh
    
    As documented in autogen.des, this is what we should use for development
    and it should encompass continuous integration. Until autogen.sh goes
    away we need to explicitly enable maintainer mode at configure time.
    
    Two thirds of our CI setup rely on autogen.des today.
    
    Better diff with the --word-diff option.

diff --git a/.travis.yml b/.travis.yml
index 0c0318393..af13775e2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -24,7 +24,7 @@ jobs:
             - libpcre3-dev
       before_script:
         - ./autogen.sh
-        - ./configure --with-unwind
+        - ./configure --enable-maintainer-mode --with-unwind
       script: &script-common
         - |
           if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then
@@ -38,13 +38,13 @@ jobs:
       compiler: gcc
       before_script:
         - ./autogen.sh
-        - ./configure
+        - ./configure --enable-maintainer-mode
     - <<: *test-linux
       compiler: gcc
       arch: arm64
       before_script:
         - ./autogen.sh
-        - ./configure
+        - ./configure --enable-maintainer-mode
     - <<: *test-linux
       env: WITNESS=1
       script: make -j8 witness
@@ -72,7 +72,7 @@ jobs:
           export UBSAN_OPTIONS=halt_on_error=1,print_stacktrace=1,use_sigaltstack=0,suppressions=$(pwd)/tools/ubsan.suppr
           export CC=clang-9
         - ./autogen.sh
-        - ./configure --with-unwind --enable-developer-warnings --enable-debugging-symbols --disable-stack-protector --with-persistent-storage --enable-asan --enable-ubsan
+        - ./configure --enable-maintainer-mode --with-unwind --enable-developer-warnings --enable-debugging-symbols --disable-stack-protector --with-persistent-storage --enable-asan --enable-ubsan
     - stage: test
       os: osx
       osx_image: xcode11.4
@@ -86,8 +86,8 @@ jobs:
           update: true
       before_script:
         - export PATH="/usr/local/opt/sphinx-doc/bin:$PATH"
-        - autoreconf -i
-        - ./configure
+        - ./autogen.sh
+        - ./configure --enable-maintainer-mode
       script: *script-common
     - <<: *test-linux
       stage: coverity
@@ -102,7 +102,7 @@ jobs:
         - export PATH=$PATH:$(echo $(pwd)/cov-analysis-*/bin)
       script:
         - ./autogen.sh
-        - ./configure --with-unwind
+        - ./configure --enable-maintainer-mode --with-unwind
         - cov-build --dir cov-int make
         - tar cfz varnish.tgz cov-int
         - curl --form token="$COVTOKEN"
diff --git a/tools/coverity-run b/tools/coverity-run
index a1d6263cf..4b8a81759 100755
--- a/tools/coverity-run
+++ b/tools/coverity-run
@@ -37,7 +37,7 @@ test "`basename $PWD`" = "devscripts" && cd ..
 
 make distclean || true
 test -f configure || ./autogen.sh
-./configure
+./configure --enable-maintainer-mode
 
 cov-build --dir cov-int make
 

From reza at naghibi.com  Tue Jun 16 15:59:08 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 16 Jun 2020 15:59:08 +0000 (UTC)
Subject: [6.0] 2d666b033 make VFP_Close() idempotent
Message-ID: <20200616155908.D376EADF48@lists.varnish-cache.org>


commit 2d666b03337356a680a4235fd2ce6224af5613ec
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Oct 2 16:42:46 2019 +0200

    make VFP_Close() idempotent
    
    required for #3009

diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c
index 1413f1bbd..9bc59bf2f 100644
--- a/bin/varnishd/cache/cache_fetch_proc.c
+++ b/bin/varnishd/cache/cache_fetch_proc.c
@@ -118,13 +118,14 @@ VFP_Setup(struct vfp_ctx *vc, struct worker *wrk)
 void
 VFP_Close(struct vfp_ctx *vc)
 {
-	struct vfp_entry *vfe;
+	struct vfp_entry *vfe, *tmp;
 
-	VTAILQ_FOREACH(vfe, &vc->vfp, list) {
+	VTAILQ_FOREACH_SAFE(vfe, &vc->vfp, list, tmp) {
 		if (vfe->vfp->fini != NULL)
 			vfe->vfp->fini(vc, vfe);
 		VSLb(vc->wrk->vsl, SLT_VfpAcct, "%s %ju %ju", vfe->vfp->name,
 		    (uintmax_t)vfe->calls, (uintmax_t)vfe->bytes_out);
+		VTAILQ_REMOVE(&vc->vfp, vfe, list);
 	}
 }
 

From reza at naghibi.com  Tue Jun 16 15:59:08 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 16 Jun 2020 15:59:08 +0000 (UTC)
Subject: [6.0] 7b40b80d5 centralize cleanup after fetch errors
Message-ID: <20200616155908.EC08FADF4B@lists.varnish-cache.org>


commit 7b40b80d573caace9c4da76230d4a0c04085d3fd
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Oct 2 16:43:05 2019 +0200

    centralize cleanup after fetch errors
    
    imples the following changes:
    
    * VDI_Finish() is now always conditional on bo->director_state !=
      DIR_S_NULL, making it idempotent
    
    * introduces additional calls to VFP_Close() from startfetch and
      for the filter_list / VCL_StackVFP error in vbf_stp_fetch(),
      but VFP_Close() is idempotent.
    
    * adds VFP_Close() for VFP_Open() failure in vbf_stp_fetch() which
      I think is actually missing (for the case that some VFPs could
      get opened before the open failure)
    
    * calls VDI_Finish() earlier in vbf_stp_fetchend: I checked the
      code and can not see any issue with this.
    
    motivated by #3009
    
     Conflicts:
            bin/varnishd/cache/cache_fetch.c

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index bd02fc822..0c0d79c9d 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -85,6 +85,20 @@ vbf_allocobj(struct busyobj *bo, unsigned l)
 	return (STV_NewObject(bo->wrk, bo->fetch_objcore, stv_transient, l));
 }
 
+static void
+vbf_cleanup(struct busyobj *bo)
+{
+	struct vfp_ctx *vfc;
+
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+	vfc = bo->vfc;
+	CHECK_OBJ_NOTNULL(vfc, VFP_CTX_MAGIC);
+
+	VFP_Close(vfc);
+	if (bo->director_state != DIR_S_NULL)
+		VDI_Finish(bo->wrk, bo);
+}
+
 /*--------------------------------------------------------------------
  * Turn the beresp into a obj
  */
@@ -233,7 +247,7 @@ vbf_stp_retry(struct worker *wrk, struct busyobj *bo)
 
 	VSLb_ts_busyobj(bo, "Retry", W_TIM_real(wrk));
 
-	/* VDI_Finish must have been called before */
+	/* VDI_Finish (via vbf_cleanup) must have been called before */
 	assert(bo->director_state == DIR_S_NULL);
 
 	/* reset other bo attributes - See VBO_GetBusyObj */
@@ -281,7 +295,7 @@ vbf_304_logic(struct busyobj *bo)
 		VSLb(bo->vsl, SLT_Error,
 		    "304 response but not conditional fetch");
 		bo->htc->doclose = SC_RX_BAD;
-		VDI_Finish(bo->wrk, bo);
+		vbf_cleanup(bo);
 		return (-1);
 	}
 	return (1);
@@ -344,7 +358,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	if (bo->htc->body_status == BS_ERROR) {
 		bo->htc->doclose = SC_RX_BODY;
-		VDI_Finish(bo->wrk, bo);
+		vbf_cleanup(bo);
 		VSLb(bo->vsl, SLT_Error, "Body cannot be fetched");
 		assert(bo->director_state == DIR_S_NULL);
 		return (F_STP_ERROR);
@@ -391,8 +405,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL ||
 	    wrk->handling == VCL_RET_ERROR) {
 		bo->htc->doclose = SC_RESP_CLOSE;
-
-		VDI_Finish(bo->wrk, bo);
+		vbf_cleanup(bo);
 		if (wrk->handling == VCL_RET_ERROR)
 			return (F_STP_ERROR);
 		else
@@ -402,8 +415,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	if (wrk->handling == VCL_RET_RETRY) {
 		if (bo->htc->body_status != BS_NONE)
 			bo->htc->doclose = SC_RESP_CLOSE;
-		if (bo->director_state != DIR_S_NULL)
-			VDI_Finish(bo->wrk, bo);
+		vbf_cleanup(bo);
 
 		if (bo->retries++ < cache_param->max_retries)
 			return (F_STP_RETRY);
@@ -493,8 +505,7 @@ vbf_stp_fetchbody(struct worker *wrk, struct busyobj *bo)
 	if (vfc->failed) {
 		(void)VFP_Error(vfc, "Fetch pipeline failed to process");
 		bo->htc->doclose = SC_RX_BODY;
-		VFP_Close(vfc);
-		VDI_Finish(wrk, bo);
+		vbf_cleanup(bo);
 		if (!bo->do_stream) {
 			assert(bo->fetch_objcore->boc->state < BOS_STREAM);
 			// XXX: doclose = ?
@@ -587,7 +598,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	if (vbf_figure_out_vfp(bo)) {
 		(bo)->htc->doclose = SC_OVERLOAD;
-		VDI_Finish((bo)->wrk, bo);
+		vbf_cleanup(bo);
 		return (F_STP_ERROR);
 	}
 
@@ -599,15 +610,14 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	if (VFP_Open(bo->vfc)) {
 		(void)VFP_Error(bo->vfc, "Fetch pipeline failed to open");
 		bo->htc->doclose = SC_RX_BODY;
-		VDI_Finish(bo->wrk, bo);
+		vbf_cleanup(bo);
 		return (F_STP_ERROR);
 	}
 
 	if (vbf_beresp2obj(bo)) {
 		(void)VFP_Error(bo->vfc, "Could not get storage");
 		bo->htc->doclose = SC_RX_BODY;
-		VFP_Close(bo->vfc);
-		VDI_Finish(bo->wrk, bo);
+		vbf_cleanup(bo);
 		return (F_STP_ERROR);
 	}
 
@@ -659,7 +669,10 @@ vbf_stp_fetchend(struct worker *wrk, struct busyobj *bo)
 {
 
 	AZ(bo->vfc->failed);
-	VFP_Close(bo->vfc);
+
+	/* Recycle the backend connection before setting BOS_FINISHED to
+	   give predictable backend reuse behavior for varnishtest */
+	vbf_cleanup(bo);
 
 	AZ(ObjSetU64(wrk, bo->fetch_objcore, OA_LEN,
 	    bo->fetch_objcore->boc->len_so_far));
@@ -672,10 +685,6 @@ vbf_stp_fetchend(struct worker *wrk, struct busyobj *bo)
 		HSH_Unbusy(wrk, bo->fetch_objcore);
 	}
 
-	/* Recycle the backend connection before setting BOS_FINISHED to
-	   give predictable backend reuse behavior for varnishtest */
-	VDI_Finish(bo->wrk, bo);
-
 	ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED);
 	VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk));
 	if (bo->stale_oc != NULL)
@@ -775,7 +784,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 		(void)VFP_Error(bo->vfc, "Template object failed");
 
 	if (bo->vfc->failed) {
-		VDI_Finish(bo->wrk, bo);
+		vbf_cleanup(bo);
 		wrk->stats->fetch_failed++;
 		return (F_STP_FAIL);
 	}

From reza at naghibi.com  Tue Jun 16 15:59:09 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 16 Jun 2020 15:59:09 +0000 (UTC)
Subject: [6.0] f5a875a0f add test case by @rezan
Message-ID: <20200616155909.1368DADF4F@lists.varnish-cache.org>


commit f5a875a0faa5522a42999c2ef95a5f29d69fd328
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Oct 2 16:43:13 2019 +0200

    add test case by @rezan

diff --git a/bin/varnishtest/tests/r03009.vtc b/bin/varnishtest/tests/r03009.vtc
new file mode 100644
index 000000000..406a40d02
--- /dev/null
+++ b/bin/varnishtest/tests/r03009.vtc
@@ -0,0 +1,33 @@
+varnishtest "Rollback without restart/retry is unsafe"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import std;
+
+	sub vcl_recv {
+		set req.http.test = "1";
+	}
+
+	sub vcl_backend_fetch {
+		unset bereq.http.test;
+	}
+
+	sub vcl_backend_response {
+		std.rollback(bereq);
+		set beresp.http.test = bereq.http.test;
+		set beresp.http.workspace = "start overwriting active workspace";
+		set beresp.http.workspace = "0123456789012345678901234567890123456789";
+		# panic...
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.http.test == "1"
+} -run

From reza at naghibi.com  Tue Jun 16 15:59:09 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 16 Jun 2020 15:59:09 +0000 (UTC)
Subject: [6.0] 62b0192a7 fix bereq rollback
Message-ID: <20200616155909.35A21ADF5A@lists.varnish-cache.org>


commit 62b0192a7df674b50017e054f6b7d2daa40cfaeb
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Oct 2 16:49:28 2019 +0200

    fix bereq rollback
    
    by properly cleaning up the busyobj
    
    Also move the relevant code from cache_vrt.c to cache_fetch.c
    
    As we fini the director during cleanup, we now also need to handle the
    backend connection gone missing in vbf_stp_fetch(). The hypothetical
    alternative would be to not fini the director, but I believe this is not
    safe in case it also used some workspace.
    
    Fixes #3009
    
     Conflicts:
            bin/varnishd/cache/cache_fetch.c
            bin/varnishd/cache/cache_varnishd.h
            bin/varnishd/cache/cache_vrt.c

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 0c0d79c9d..47cc8fd79 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -99,6 +99,18 @@ vbf_cleanup(struct busyobj *bo)
 		VDI_Finish(bo->wrk, bo);
 }
 
+void Bereq_Rollback(struct busyobj *bo)
+{
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
+
+	vbf_cleanup(bo);
+	VCL_TaskLeave(bo->vcl, bo->privs);
+	VCL_TaskEnter(bo->vcl, bo->privs);
+	HTTP_Copy(bo->bereq, bo->bereq0);
+	WS_Reset(bo->bereq->ws, bo->ws_bo);
+	WS_Reset(bo->ws, bo->ws_bo);
+}
+
 /*--------------------------------------------------------------------
  * Turn the beresp into a obj
  */
@@ -404,7 +416,8 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL ||
 	    wrk->handling == VCL_RET_ERROR) {
-		bo->htc->doclose = SC_RESP_CLOSE;
+		if (bo->htc)
+			bo->htc->doclose = SC_RESP_CLOSE;
 		vbf_cleanup(bo);
 		if (wrk->handling == VCL_RET_ERROR)
 			return (F_STP_ERROR);
@@ -413,7 +426,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	}
 
 	if (wrk->handling == VCL_RET_RETRY) {
-		if (bo->htc->body_status != BS_NONE)
+		if (bo->htc && bo->htc->body_status != BS_NONE)
 			bo->htc->doclose = SC_RESP_CLOSE;
 		vbf_cleanup(bo);
 
@@ -596,6 +609,12 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 
 	assert(wrk->handling == VCL_RET_DELIVER);
 
+	if (bo->htc == NULL) {
+		(void)VFP_Error(bo->vfc, "No backend connection (rollback?)");
+		vbf_cleanup(bo);
+		return (F_STP_ERROR);
+	}
+
 	if (vbf_figure_out_vfp(bo)) {
 		(bo)->htc->doclose = SC_OVERLOAD;
 		vbf_cleanup(bo);
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index 939a0fe17..4eb713fd3 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -202,6 +202,7 @@ enum vbf_fetch_mode_e {
 };
 void VBF_Fetch(struct worker *wrk, struct req *req,
     struct objcore *oc, struct objcore *oldoc, enum vbf_fetch_mode_e);
+void Bereq_Rollback(struct busyobj *);
 
 /* cache_fetch_proc.c */
 void VFP_Init(void);
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index d0188492e..d103a0616 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -663,12 +663,7 @@ VRT_Rollback(VRT_CTX, VCL_HTTP hp)
 		Req_Rollback(ctx->req);
 	} else if (hp == ctx->http_bereq) {
 		CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
-		// -> VBO_Rollback ?
-		VCL_TaskLeave(ctx->bo->vcl, ctx->bo->privs);
-		VCL_TaskEnter(ctx->bo->vcl, ctx->bo->privs);
-		HTTP_Copy(ctx->bo->bereq, ctx->bo->bereq0);
-		WS_Reset(ctx->bo->bereq->ws, ctx->bo->ws_bo);
-		WS_Reset(ctx->bo->ws, ctx->bo->ws_bo);
+		Bereq_Rollback(ctx->bo);
 	} else
 		WRONG("VRT_Rollback 'hp' invalid");
 }
diff --git a/bin/varnishtest/tests/r03009.vtc b/bin/varnishtest/tests/r03009.vtc
index 406a40d02..1bc44521f 100644
--- a/bin/varnishtest/tests/r03009.vtc
+++ b/bin/varnishtest/tests/r03009.vtc
@@ -28,6 +28,5 @@ varnish v1 -vcl+backend {
 client c1 {
 	txreq
 	rxresp
-	expect resp.status == 200
-	expect resp.http.test == "1"
+	expect resp.status == 503
 } -run

From reza at naghibi.com  Tue Jun 16 15:59:09 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 16 Jun 2020 15:59:09 +0000 (UTC)
Subject: [6.0] e0d24df41 Re-Setup the response after rollback
Message-ID: <20200616155909.509D4ADF60@lists.varnish-cache.org>


commit e0d24df41a9026ec8cc9565f79638aaf8871573e
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 7 18:09:19 2019 +0200

    Re-Setup the response after rollback
    
    Fixes #3083
    
    to get there, also centralize req->resp setup for deliver and synth:
    
    No semantic changes other than some reordering, which also fixes an odd
    log line ordering as shown by the change to c00018.vtc
    
     Conflicts:
            bin/varnishd/cache/cache_req_fsm.c

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 03076960b..74b9f8998 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -107,35 +107,29 @@ cnt_transport(struct worker *wrk, struct req *req)
  * Deliver an object to client
  */
 
-static enum req_fsm_nxt
-cnt_deliver(struct worker *wrk, struct req *req)
+static int
+resp_setup_deliver(struct req *req)
 {
+	struct http *h;
+	struct objcore *oc;
 
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
-	CHECK_OBJ_NOTNULL(req->objcore->objhead, OBJHEAD_MAGIC);
-	AZ(req->stale_oc);
-	AN(req->vcl);
+	oc = req->objcore;
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 
-	assert(req->objcore->refcnt > 0);
+	h = req->resp;
 
-	ObjTouch(req->wrk, req->objcore, req->t_prev);
+	HTTP_Setup(h, req->ws, req->vsl, SLT_RespMethod);
+
+	if (HTTP_Decode(h, ObjGetAttr(req->wrk, oc, OA_HEADERS, NULL)))
+		return (-1);
 
-	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
-	if (HTTP_Decode(req->resp,
-	    ObjGetAttr(req->wrk, req->objcore, OA_HEADERS, NULL))) {
-		(void)HSH_DerefObjCore(wrk, &req->objcore, HSH_RUSH_POLICY);
-		req->err_code = 500;
-		req->req_step = R_STP_SYNTH;
-		return (REQ_FSM_MORE);
-	}
 	http_ForceField(req->resp, HTTP_HDR_PROTO, "HTTP/1.1");
 
 	if (req->is_hit)
 		http_PrintfHeader(req->resp,
 		    "X-Varnish: %u %u", VXID(req->vsl->wid),
-		    ObjGetXID(wrk, req->objcore));
+		    ObjGetXID(req->wrk, req->objcore));
 	else
 		http_PrintfHeader(req->resp,
 		    "X-Varnish: %u", VXID(req->vsl->wid));
@@ -158,6 +152,76 @@ cnt_deliver(struct worker *wrk, struct req *req)
 	    !RFC2616_Req_Gzip(req->http))
 		RFC2616_Weaken_Etag(req->resp);
 
+	return (0);
+}
+
+static int
+resp_setup_synth(struct req *req)
+{
+	struct http *h;
+
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+
+	h = req->resp;
+
+	HTTP_Setup(h, req->ws, req->vsl, SLT_RespMethod);
+
+	AZ(req->objcore);
+	http_PutResponse(h, "HTTP/1.1", req->err_code, req->err_reason);
+
+	http_TimeHeader(h, "Date: ", W_TIM_real(req->wrk));
+	http_SetHeader(h, "Server: Varnish");
+	http_PrintfHeader(h, "X-Varnish: %u",
+	    VXID(req->vsl->wid));
+
+	/*
+	 * For late 100-continue, we suggest to VCL to close the connection to
+	 * neither send a 100-continue nor drain-read the request. But VCL has
+	 * the option to veto by removing Connection: close
+	 */
+	if (req->want100cont)
+		http_SetHeader(h, "Connection: close");
+
+	return (0);
+}
+
+int
+Resp_Setup(struct req *req, unsigned method)
+{
+	switch (method) {
+	case VCL_MET_DELIVER:
+		return (resp_setup_deliver(req));
+	case VCL_MET_SYNTH:
+		return (resp_setup_synth(req));
+	default:
+		WRONG("vcl method");
+	}
+
+	return (0);
+}
+
+static enum req_fsm_nxt
+cnt_deliver(struct worker *wrk, struct req *req)
+{
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
+	CHECK_OBJ_NOTNULL(req->objcore->objhead, OBJHEAD_MAGIC);
+	AZ(req->stale_oc);
+	AN(req->vcl);
+
+	assert(req->objcore->refcnt > 0);
+
+	ObjTouch(req->wrk, req->objcore, req->t_prev);
+
+	if (resp_setup_deliver(req)) {
+		(void)HSH_DerefObjCore(wrk, &req->objcore, HSH_RUSH_POLICY);
+		req->err_code = 500;
+		req->req_step = R_STP_SYNTH;
+		return (REQ_FSM_MORE);
+	}
+
 	VCL_deliver_method(req->vcl, wrk, req, NULL, NULL);
 	VSLb_ts_req(req, "Process", W_TIM_real(wrk));
 
@@ -225,8 +289,6 @@ cnt_vclfail(const struct worker *wrk, struct req *req)
 static enum req_fsm_nxt
 cnt_synth(struct worker *wrk, struct req *req)
 {
-	struct http *h;
-	double now;
 	struct vsb *synth_body;
 	ssize_t sz, szl;
 	uint8_t *ptr;
@@ -239,27 +301,12 @@ cnt_synth(struct worker *wrk, struct req *req)
 
 	wrk->stats->s_synth++;
 
-	now = W_TIM_real(wrk);
-	VSLb_ts_req(req, "Process", now);
+	VSLb_ts_req(req, "Process", W_TIM_real(wrk));
 
 	if (req->err_code < 100)
 		req->err_code = 501;
 
-	HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
-	h = req->resp;
-	http_TimeHeader(h, "Date: ", now);
-	http_SetHeader(h, "Server: Varnish");
-	http_PrintfHeader(req->resp, "X-Varnish: %u", VXID(req->vsl->wid));
-	http_PutResponse(h, "HTTP/1.1", req->err_code, req->err_reason);
-
-	/*
-	 * For late 100-continue, we suggest to VCL to close the connection to
-	 * neither send a 100-continue nor drain-read the request. But VCL has
-	 * the option to veto by removing Connection: close
-	 */
-	if (req->want100cont) {
-		http_SetHeader(h, "Connection: close");
-	}
+	(void) resp_setup_synth(req);
 
 	synth_body = VSB_new_auto();
 	AN(synth_body);
@@ -281,14 +328,14 @@ cnt_synth(struct worker *wrk, struct req *req)
 		 * XXX: Should we reset req->doclose = SC_VCL_FAILURE
 		 * XXX: If so, to what ?
 		 */
-		HTTP_Setup(h, req->ws, req->vsl, SLT_RespMethod);
+		HTTP_Setup(req->resp, req->ws, req->vsl, SLT_RespMethod);
 		VSB_destroy(&synth_body);
 		req->req_step = R_STP_RESTART;
 		return (REQ_FSM_MORE);
 	}
 	assert(wrk->handling == VCL_RET_DELIVER);
 
-	http_Unset(h, H_Content_Length);
+	http_Unset(req->resp, H_Content_Length);
 	http_PrintfHeader(req->resp, "Content-Length: %zd",
 	    VSB_len(synth_body));
 
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index 4eb713fd3..cd97d824a 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -328,6 +328,8 @@ void VRB_Free(struct req *);
 
 /* cache_req_fsm.c [CNT] */
 
+int Resp_Setup(struct req *, unsigned);
+
 enum req_fsm_nxt {
 	REQ_FSM_MORE,
 	REQ_FSM_DONE,
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index d103a0616..4f34fba44 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -661,6 +661,8 @@ VRT_Rollback(VRT_CTX, VCL_HTTP hp)
 	if (hp == ctx->http_req) {
 		CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
 		Req_Rollback(ctx->req);
+		if ((ctx->method & (VCL_MET_DELIVER | VCL_MET_SYNTH)) != 0)
+			Resp_Setup(ctx->req, ctx->method);
 	} else if (hp == ctx->http_bereq) {
 		CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
 		Bereq_Rollback(ctx->bo);
diff --git a/bin/varnishtest/tests/c00018.vtc b/bin/varnishtest/tests/c00018.vtc
index 7a27755ff..10359f758 100644
--- a/bin/varnishtest/tests/c00018.vtc
+++ b/bin/varnishtest/tests/c00018.vtc
@@ -123,11 +123,13 @@ logexpect l1 -v v1 -g raw {
 	expect 0 1011	VCL_call        {^HASH$}
 	expect 0 1011	VCL_return      {^lookup$}
 	expect 0 1011	Timestamp       {^Process:}
-	expect 0 1011	RespHeader      {^Date:}
-	expect 0 1011	RespHeader      {^Server: Varnish$}
-	expect 0 1011	RespHeader      {^X-Varnish: 1011$}
 	expect 0 1011	RespProtocol    {^HTTP/1.1$}
 	expect 0 1011	RespStatus      {^405$}
+	expect 0 1011	RespReason      {^Method Not Allowed$}
+	# XXX dup RespReason
+	expect 1 1011	RespHeader      {^Date:}
+	expect 0 1011	RespHeader      {^Server: Varnish$}
+	expect 0 1011	RespHeader      {^X-Varnish: 1011$}
 
 } -start
 

From reza at naghibi.com  Tue Jun 16 15:59:09 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 16 Jun 2020 15:59:09 +0000 (UTC)
Subject: [6.0] cb5a07473 integrate @rezan 's regression tests into m17.vtc
Message-ID: <20200616155909.6CF63ADF64@lists.varnish-cache.org>


commit cb5a074730ad4d7ae65a7df31b601588650ab7f8
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 7 18:25:01 2019 +0200

    integrate @rezan 's regression tests into m17.vtc
    
    I had previously overlooked that there exists a more complete regression
    test by him in #3010
    
    Integrate that vtc, polished and modified to account for the backend 503
    for no restart after rollback, into m17, making r3009 obsolete again

diff --git a/bin/varnishtest/tests/m00017.vtc b/bin/varnishtest/tests/m00017.vtc
index 8fa38f37a..c31322d50 100644
--- a/bin/varnishtest/tests/m00017.vtc
+++ b/bin/varnishtest/tests/m00017.vtc
@@ -1,5 +1,9 @@
 varnishtest "Test std.rollback"
 
+# bug regressions:
+# - #3009
+# - #3083
+
 server s1 {
 	rxreq
 	expect req.url == "/foo"
@@ -67,3 +71,253 @@ client c1 {
 	txreq -url "/baz" -hdr "foobar: baz"
 	rxresp
 } -run
+
+server s1 -wait
+
+varnish v1 -cliok "param.set workspace_client 12000"
+varnish v1 -cliok "param.set workspace_backend 12000"
+
+# Dont panic
+varnish v1 -vcl+backend {
+	import std;
+	import vtc;
+
+	sub vcl_recv {
+		set req.http.test = "1";
+	}
+
+	sub vcl_backend_fetch {
+		unset bereq.http.test;
+	}
+
+	sub vcl_backend_response {
+		std.rollback(bereq);
+		set beresp.http.test = bereq.http.test;
+		vtc.workspace_alloc(backend, -10);
+	}
+}
+
+server s1 {
+	rxreq
+	expect req.url == "/1"
+	txresp
+} -start
+
+client c1 {
+	txreq -url /1
+	rxresp
+	expect resp.status == 503
+	expect resp.http.test == <undef>
+} -run
+
+server s1 -wait
+
+# Dont run out of workspace
+varnish v1 -vcl+backend {
+	import std;
+	import vtc;
+
+	sub vcl_backend_fetch {
+		vtc.workspace_alloc(backend, 1000);
+	}
+
+	sub vcl_backend_response {
+		if (bereq.retries == 0) {
+			vtc.workspace_alloc(backend, -10);
+			std.rollback(bereq);
+			return (retry);
+		}
+	}
+}
+
+server s1 -repeat 2 {
+	rxreq
+	expect req.url == "/2"
+	txresp
+} -start
+
+client c2 {
+	txreq -url /2
+	rxresp
+	expect resp.status == 200
+} -run
+
+server s1 -wait
+
+# Keep workspace intact (and possibly overflow)
+varnish v1 -vcl+backend {
+	import std;
+	import vtc;
+
+	sub vcl_backend_fetch {
+		set bereq.http.fetch = "Fetch value " + bereq.retries;
+	}
+
+	sub vcl_backend_response {
+		if (bereq.retries == 0) {
+			std.rollback(bereq);
+			set bereq.http.response = "123";
+			set bereq.http.response2 = "Another response";
+			if (bereq.url == "/4") {
+				vtc.workspace_alloc(backend, -10);
+			} else if (bereq.url == "/5") {
+				vtc.workspace_alloc(backend, -10);
+				std.rollback(bereq);
+			}
+			return (retry);
+		}
+		set beresp.http.fetch = bereq.http.fetch;
+		set beresp.http.response = bereq.http.response;
+		set beresp.http.response2 = bereq.http.response2;
+	}
+}
+
+server s1 -repeat 5 {
+	rxreq
+	txresp
+} -start
+
+client c3 {
+	txreq -url /3
+	rxresp
+	expect resp.status == 200
+	expect resp.http.fetch == "Fetch value 1"
+	expect resp.http.response == "123"
+	expect resp.http.response2 == "Another response"
+
+	txreq -url /4
+	rxresp
+	expect resp.status == 503
+
+	txreq -url /5
+	rxresp
+	expect resp.status == 200
+	expect resp.http.fetch == "Fetch value 1"
+	expect resp.http.response == ""
+	expect resp.http.response2 == ""
+} -run
+
+server s1 -wait
+
+# CLIENT
+
+# Dont panic
+varnish v1 -vcl+backend {
+	import std;
+	import vtc;
+
+	sub vcl_recv {
+		unset req.http.test;
+	}
+
+	sub vcl_deliver {
+		std.rollback(req);
+		set resp.http.test = req.http.test;
+		vtc.workspace_alloc(client, -200);
+	}
+}
+
+# XXX server keeps params, so we need to reset previous -repeat 5
+server s1 -repeat 1 {
+	rxreq
+	expect req.url == "/6"
+	txresp
+} -start
+
+client c4 {
+	txreq -url /6 -hdr "test: 1"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.test == "1"
+} -run
+
+server s1 -wait
+
+# Dont run out of workspace
+varnish v1 -vcl+backend {
+	import std;
+	import vtc;
+
+	sub vcl_recv {
+		vtc.workspace_alloc(client, 1000);
+	}
+
+	sub vcl_deliver {
+		if (req.restarts == 0) {
+			vtc.workspace_alloc(client, -10);
+			std.rollback(req);
+			return (restart);
+		}
+	}
+}
+
+server s1 {
+	rxreq
+	expect req.url == "/7"
+	txresp
+} -start
+
+client c5 {
+	txreq -url /7
+	rxresp
+	expect resp.status == 200
+} -run
+
+server s1 -wait
+
+# Keep workspace intact (and possibly overflow)
+varnish v1 -vcl+backend {
+	import std;
+	import vtc;
+
+	sub vcl_recv {
+		set req.http.fetch = "Fetch value " + req.restarts;
+	}
+
+	sub vcl_deliver {
+		if (req.restarts == 0) {
+			std.rollback(req);
+			set req.http.response = "123";
+			set req.http.response2 = "Another response";
+			if (req.url == "/9") {
+				vtc.workspace_alloc(client, -200);
+			} else if (req.url == "/10") {
+				vtc.workspace_alloc(client, -10);
+				std.rollback(req);
+			}
+			return (restart);
+		}
+		set resp.http.fetch = req.http.fetch;
+		set resp.http.response = req.http.response;
+		set resp.http.response2 = req.http.response2;
+	}
+}
+
+server s1 -repeat 3 {
+	rxreq
+	txresp
+} -start
+
+client c6 {
+	txreq -url /8
+	rxresp
+	expect resp.status == 200
+	expect resp.http.fetch == "Fetch value 1"
+	expect resp.http.response == "123"
+	expect resp.http.response2 == "Another response"
+
+	txreq -url /9
+	rxresp
+	expect resp.status == 200
+	expect resp.http.fetch == "Fetch value 1"
+	expect resp.http.response == "123"
+	expect resp.http.response2 == "Another response"
+
+	txreq -url /10
+	rxresp
+	expect resp.status == 200
+	expect resp.http.fetch == "Fetch value 1"
+	expect resp.http.response == ""
+} -run
+
+server s1 -wait
diff --git a/bin/varnishtest/tests/r03009.vtc b/bin/varnishtest/tests/r03009.vtc
deleted file mode 100644
index 1bc44521f..000000000
--- a/bin/varnishtest/tests/r03009.vtc
+++ /dev/null
@@ -1,32 +0,0 @@
-varnishtest "Rollback without restart/retry is unsafe"
-
-server s1 {
-	rxreq
-	txresp
-} -start
-
-varnish v1 -vcl+backend {
-	import std;
-
-	sub vcl_recv {
-		set req.http.test = "1";
-	}
-
-	sub vcl_backend_fetch {
-		unset bereq.http.test;
-	}
-
-	sub vcl_backend_response {
-		std.rollback(bereq);
-		set beresp.http.test = bereq.http.test;
-		set beresp.http.workspace = "start overwriting active workspace";
-		set beresp.http.workspace = "0123456789012345678901234567890123456789";
-		# panic...
-	}
-} -start
-
-client c1 {
-	txreq
-	rxresp
-	expect resp.status == 503
-} -run

From reza at naghibi.com  Tue Jun 16 15:59:09 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 16 Jun 2020 15:59:09 +0000 (UTC)
Subject: [6.0] 0b275eb76 For rollback,
 NULL additional pointers which could point to workspace
Message-ID: <20200616155909.8603EADF68@lists.varnish-cache.org>


commit 0b275eb765978209d061ae725c9809300a18eb56
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Apr 14 11:07:40 2020 +0200

    For rollback, NULL additional pointers which could point to workspace
    
     Conflicts:
            bin/varnishd/cache/cache_fetch.c
            bin/varnishd/cache/cache_req.c

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 47cc8fd79..d151298c1 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -107,6 +107,7 @@ void Bereq_Rollback(struct busyobj *bo)
 	VCL_TaskLeave(bo->vcl, bo->privs);
 	VCL_TaskEnter(bo->vcl, bo->privs);
 	HTTP_Copy(bo->bereq, bo->bereq0);
+	bo->err_reason = NULL;
 	WS_Reset(bo->bereq->ws, bo->ws_bo);
 	WS_Reset(bo->ws, bo->ws_bo);
 }

From reza at naghibi.com  Tue Jun 16 15:59:09 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 16 Jun 2020 15:59:09 +0000 (UTC)
Subject: [6.0] 893d1e696 fix bereq rollback with retry
Message-ID: <20200616155909.A52CCADF7E@lists.varnish-cache.org>


commit 893d1e6960c887d9da5e88cd89091c26e8c74aba
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Jun 15 20:13:56 2020 +0200

    fix bereq rollback with retry
    
    When retrying after a rollback, we might have unread bytes of the
    backend response body. As we need to finalize the director for rollback,
    all we can do in this case is to close the connection.
    
    Spotted by reza, who also provided the test case, thank you.
    
    Closes #3353
    
     Conflicts:
            bin/varnishd/cache/cache_fetch.c

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index d151298c1..840120b06 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -103,6 +103,10 @@ void Bereq_Rollback(struct busyobj *bo)
 {
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
+	if (bo->htc != NULL &&
+	    bo->htc->body_status != BS_NONE)
+		bo->htc->doclose = SC_RESP_CLOSE;
+
 	vbf_cleanup(bo);
 	VCL_TaskLeave(bo->vcl, bo->privs);
 	VCL_TaskEnter(bo->vcl, bo->privs);
diff --git a/bin/varnishtest/tests/r03353.vtc b/bin/varnishtest/tests/r03353.vtc
new file mode 100644
index 000000000..d5ab5f1a4
--- /dev/null
+++ b/bin/varnishtest/tests/r03353.vtc
@@ -0,0 +1,33 @@
+varnishtest "Test rollback and retry"
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr "Content-Length: 3"
+	expect_close
+	accept
+
+	rxreq
+	txresp -body xxx
+} -start
+
+varnish v1 -vcl+backend {
+	import std;
+
+	sub vcl_backend_response {
+		if (bereq.retries == 0) {
+			std.rollback(bereq);
+		}
+	}
+
+	sub vcl_backend_error {
+		if (bereq.retries == 0) {
+			return (retry);
+		}
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run

From reza at naghibi.com  Tue Jun 16 16:00:09 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 16 Jun 2020 16:00:09 +0000 (UTC)
Subject: [6.0] b6a94a282 Handle EWOULDBLOCK as a short write
Message-ID: <20200616160009.27EDEAE96D@lists.varnish-cache.org>


commit b6a94a2820747f02fb798ae15e6bd96ecb3857be
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 29 09:26:27 2019 +0000

    Handle EWOULDBLOCK as a short write

diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c
index 7897cca4c..e7be7c8c2 100644
--- a/bin/varnishd/http1/cache_http1_line.c
+++ b/bin/varnishd/http1/cache_http1_line.c
@@ -204,7 +204,7 @@ V1L_Flush(const struct worker *wrk)
 		i = writev(*v1l->wfd, v1l->iov, v1l->niov);
 		if (i > 0)
 			v1l->cnt += i;
-		while (i != v1l->liov && i > 0) {
+		while (i != v1l->liov && (i > 0 || errno == EWOULDBLOCK)) {
 			/* Remove sent data from start of I/O vector,
 			 * then retry; we hit a timeout, but some data
 			 * was sent.
@@ -226,7 +226,8 @@ V1L_Flush(const struct worker *wrk)
 			    "Hit idle send timeout, wrote = %zd/%zd; retrying",
 			    i, v1l->liov);
 
-			v1l_prune(v1l, i);
+			if (i > 0)
+				v1l_prune(v1l, i);
 			i = writev(*v1l->wfd, v1l->iov, v1l->niov);
 			if (i > 0)
 				v1l->cnt += i;

From reza at naghibi.com  Tue Jun 16 16:00:09 2020
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 16 Jun 2020 16:00:09 +0000 (UTC)
Subject: [6.0] 62db24848 respect `send_timeout` for "dripping" http1 writes
Message-ID: <20200616160009.4FA32AE972@lists.varnish-cache.org>


commit 62db248488e450c8dc9b276ab1b9420201deb958
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Jan 27 14:18:28 2020 +0100

    respect `send_timeout` for "dripping" http1 writes
    
    Previously, we only checked `v1l->deadline` (which gets initialized
    from the `send_timeout` session attribute or parameter) only for short
    writes, so for successful "dripping" http1 writes (streaming from a
    backend busy object with small chunks), we did not respect the
    timeout.
    
    This patch restructures `V1L_Flush()` to always check the deadline
    before a write by turning a `while() { ... }` into a `do { ... }
    while` with the same continuation criteria: `while (i != v1l->liov)`
    is turned into `if (i == v1l->liov) break;` and `while (i > 0 || errno
    == EWOULDBLOCK)` is kept to retry short writes.
    
    This also reduces the `writev()` call sites to one.
    
    Fixes #3189
    
     Conflicts:
            bin/varnishd/http1/cache_http1_line.c
            bin/varnishtest/tests/s00011.vtc

diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c
index e7be7c8c2..be0214afb 100644
--- a/bin/varnishd/http1/cache_http1_line.c
+++ b/bin/varnishd/http1/cache_http1_line.c
@@ -201,18 +201,8 @@ V1L_Flush(const struct worker *wrk)
 			v1l->iov[v1l->ciov].iov_len = 0;
 		}
 
-		i = writev(*v1l->wfd, v1l->iov, v1l->niov);
-		if (i > 0)
-			v1l->cnt += i;
-		while (i != v1l->liov && (i > 0 || errno == EWOULDBLOCK)) {
-			/* Remove sent data from start of I/O vector,
-			 * then retry; we hit a timeout, but some data
-			 * was sent.
-			 *
-			 * XXX: Add a "minimum sent data per timeout
-			 * counter to prevent slowlaris attacks
-			*/
-
+		i = 0;
+		do {
 			if (VTIM_real() - v1l->t0 > cache_param->send_timeout) {
 				VSLb(v1l->vsl, SLT_Debug,
 				    "Hit total send timeout, "
@@ -222,16 +212,28 @@ V1L_Flush(const struct worker *wrk)
 				break;
 			}
 
+			i = writev(*v1l->wfd, v1l->iov, v1l->niov);
+			if (i > 0)
+				v1l->cnt += i;
+
+			if (i == v1l->liov)
+				break;
+
+			/* we hit a timeout, and some data may have been sent:
+			 * Remove sent data from start of I/O vector, then retry
+			 *
+			 * XXX: Add a "minimum sent data per timeout counter to
+			 * prevent slowloris attacks
+			 */
+
 			VSLb(v1l->vsl, SLT_Debug,
 			    "Hit idle send timeout, wrote = %zd/%zd; retrying",
 			    i, v1l->liov);
 
 			if (i > 0)
 				v1l_prune(v1l, i);
-			i = writev(*v1l->wfd, v1l->iov, v1l->niov);
-			if (i > 0)
-				v1l->cnt += i;
-		}
+		} while (i > 0 || errno == EWOULDBLOCK);
+
 		if (i <= 0) {
 			v1l->werr++;
 			VSLb(v1l->vsl, SLT_Debug,
diff --git a/bin/varnishtest/tests/r03189.vtc b/bin/varnishtest/tests/r03189.vtc
new file mode 100644
index 000000000..cf3d466c9
--- /dev/null
+++ b/bin/varnishtest/tests/r03189.vtc
@@ -0,0 +1,28 @@
+varnishtest "h1 send_timeout and streaming of dripping chunks"
+
+barrier b cond 2
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr "Transfer-Encoding: chunked"
+	chunkedlen 1
+	delay 1
+	non_fatal
+	barrier b sync
+	chunkedlen 1
+	delay 1
+	chunkedlen 0
+} -start
+
+varnish v1				\
+	-arg "-p idle_send_timeout=.1"	\
+	-arg "-p send_timeout=.8"	\
+	-vcl+backend { } -start
+
+client c1 {
+	txreq
+	rxresphdrs
+	rxchunk
+	barrier b sync
+	expect_close
+} -run

From dridi.boukelmoune at gmail.com  Wed Jun 17 07:50:08 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 17 Jun 2020 07:50:08 +0000 (UTC)
Subject: [master] 970c59eda Revert "Revert "Only build manual pages in
 maintainer mode""
Message-ID: <20200617075008.EFC0297B84@lists.varnish-cache.org>


commit 970c59edac07ccdb68c1ae13d94516ab8ea8fdfa
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jun 16 18:00:31 2020 +0200

    Revert "Revert "Only build manual pages in maintainer mode""
    
    This brings back commit cc42bd31af078b8d0a2532d3744d2e662b16e422:
    
    When we release a dist archive, it comes with the documentation but
    since some of it is generated by programs it makes no real difference,
    and actually it makes things worse: rebuilding from the dist archive
    ends up with a rebuild of the documentation because some documentation
    sources ended up being rebuilt. In other words, the documentation we
    ship is dead weight in the dist archive and doesn't reduce the number
    of build dependencies downstream.
    
    From now on, rst2man remains mandatory to build our manual pages but
    can safely be omitted by packaging scripts.

diff --git a/configure.ac b/configure.ac
index cbdeb59cf..ac1a3bdcf 100644
--- a/configure.ac
+++ b/configure.ac
@@ -37,6 +37,7 @@ CC="$PTHREAD_CC"
 
 AC_PROG_INSTALL
 
+AM_COND_IF([MAINTAINER_MODE], [dnl
 AC_ARG_WITH([rst2man],
   AS_HELP_STRING([--with-rst2man=PATH], [Location of rst2man (auto)]),
   [RST2MAN="$withval"],
@@ -47,6 +48,7 @@ if test "x$RST2MAN" = "xno"; then
   AC_MSG_ERROR(
     [rst2man is needed to build Varnish, please install python3-docutils.])
 fi
+])dnl AM_COND_IF MAINTAINER_MODE
 
 AC_ARG_WITH([sphinx-build],
   AS_HELP_STRING([--with-sphinx-build=PATH], [Location of sphinx-build (auto)]),
diff --git a/man/Makefile.am b/man/Makefile.am
index f04a0647c..dd4b7771c 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -24,8 +24,9 @@ dist_man_MANS = \
 	vmod_unix.3 \
 	vmod_proxy.3
 
-CLEANFILES = $(dist_man_MANS)
+MAINTAINERCLEANFILES = $(dist_man_MANS)
 
+if MAINTAINER_MODE
 RST2ANY_FLAGS = --halt=2
 
 BUILD_MAN = $(AM_V_GEN) $(RST2MAN) $(RST2ANY_FLAGS)
@@ -120,3 +121,4 @@ vmod_proxy.3: $(top_builddir)/lib/libvmod_proxy/vmod_proxy.man.rst
 	$(BUILD_MAN) $? $@
 
 .NOPATH: $(dist_man_MANS)
+endif # MAINTAINER_MODE

From nils.goroll at uplex.de  Thu Jun 18 12:34:09 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 18 Jun 2020 12:34:09 +0000 (UTC)
Subject: [master] 1d62b6685 Revert "Revert "Revert "Only build manual pages in
 maintainer mode"""
Message-ID: <20200618123409.1ED65A7A28@lists.varnish-cache.org>


commit 1d62b6685727029c23cd665a395185709f62f7f8
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Jun 18 14:32:24 2020 +0200

    Revert "Revert "Revert "Only build manual pages in maintainer mode"""
    
    This reverts commit 970c59edac07ccdb68c1ae13d94516ab8ea8fdfa.
    
    this has been discussed on irc.
    
    I agree that not rebuilding man pages when they are already there makes
    sense.
    
    But not building man pages when we could, does not make any sense to me.
    And enforcing maintainer mode to build from git does not seem to make
    any sense to me either.

diff --git a/configure.ac b/configure.ac
index ac1a3bdcf..cbdeb59cf 100644
--- a/configure.ac
+++ b/configure.ac
@@ -37,7 +37,6 @@ CC="$PTHREAD_CC"
 
 AC_PROG_INSTALL
 
-AM_COND_IF([MAINTAINER_MODE], [dnl
 AC_ARG_WITH([rst2man],
   AS_HELP_STRING([--with-rst2man=PATH], [Location of rst2man (auto)]),
   [RST2MAN="$withval"],
@@ -48,7 +47,6 @@ if test "x$RST2MAN" = "xno"; then
   AC_MSG_ERROR(
     [rst2man is needed to build Varnish, please install python3-docutils.])
 fi
-])dnl AM_COND_IF MAINTAINER_MODE
 
 AC_ARG_WITH([sphinx-build],
   AS_HELP_STRING([--with-sphinx-build=PATH], [Location of sphinx-build (auto)]),
diff --git a/man/Makefile.am b/man/Makefile.am
index dd4b7771c..f04a0647c 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -24,9 +24,8 @@ dist_man_MANS = \
 	vmod_unix.3 \
 	vmod_proxy.3
 
-MAINTAINERCLEANFILES = $(dist_man_MANS)
+CLEANFILES = $(dist_man_MANS)
 
-if MAINTAINER_MODE
 RST2ANY_FLAGS = --halt=2
 
 BUILD_MAN = $(AM_V_GEN) $(RST2MAN) $(RST2ANY_FLAGS)
@@ -121,4 +120,3 @@ vmod_proxy.3: $(top_builddir)/lib/libvmod_proxy/vmod_proxy.man.rst
 	$(BUILD_MAN) $? $@
 
 .NOPATH: $(dist_man_MANS)
-endif # MAINTAINER_MODE

From nils.goroll at uplex.de  Thu Jun 18 14:05:07 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 18 Jun 2020 14:05:07 +0000 (UTC)
Subject: [master] 06c4900c4 Document cpio as a build requirement
Message-ID: <20200618140507.9863CA9EAB@lists.varnish-cache.org>


commit 06c4900c4a61dbaa652483dd03a17894bdac1598
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Jun 18 16:03:49 2020 +0200

    Document cpio as a build requirement
    
    note: on SmartOS, it should be part of the base install
    
    Ref #3309

diff --git a/doc/sphinx/installation/install_source.rst b/doc/sphinx/installation/install_source.rst
index 488fe235c..7a7893926 100644
--- a/doc/sphinx/installation/install_source.rst
+++ b/doc/sphinx/installation/install_source.rst
@@ -26,6 +26,8 @@ you can either::
 
 	pkg install automake pkgconf py36-sphinx py36-docutils pcre libtool
 
+.. XXX does cpio need to be installed on FreeBSD?
+
 And optionally, to be able to run all the testcases::
 
 	pkg install haproxy nghttp2 vttest
@@ -55,7 +57,8 @@ them (replace ``sudo apt-get install`` if needed)::
 	libtool \
 	pkg-config \
 	python3-docutils \
-	python3-sphinx
+	python3-sphinx \
+	cpio
 
 Optionally, to rebuild the svg files::
 
@@ -104,7 +107,8 @@ packages::
 	ncurses-devel \
 	pcre-devel \
 	pkgconfig \
-	python3-docutils
+	python3-docutils \
+	cpio
 
 Optionally, to rebuild the svg files::
 

From dridi.boukelmoune at gmail.com  Thu Jun 18 21:57:07 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 18 Jun 2020 21:57:07 +0000 (UTC)
Subject: [master] 30d149fde varnish.m4: Support generated VCC files
Message-ID: <20200618215707.31CF4BF2D5@lists.varnish-cache.org>


commit 30d149fde44a5de83cdf232af7dbf29d68a35a4b
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue May 19 12:18:26 2020 +0200

    varnish.m4: Support generated VCC files
    
    With a separate autoconf macro to preserve existing behavior. While at
    it, I got rid of some intermediate processing in the generated configure
    script that should result in slightly smaller scripts for third party
    VMODs.

diff --git a/varnish.m4 b/varnish.m4
index 2ede52bfc..f86885a9a 100644
--- a/varnish.m4
+++ b/varnish.m4
@@ -31,7 +31,7 @@
 # OF THE POSSIBILITY OF SUCH DAMAGE.
 
 # varnish.m4 - Macros to build against Varnish.         -*- Autoconf -*-
-# serial 10 (varnish-6.0.0)
+# serial 11 (varnish-7.0.0)
 #
 # This collection of macros helps create VMODs or tools interacting with
 # Varnish Cache using the GNU build system (autotools). In order to work
@@ -186,8 +186,8 @@ AC_DEFUN([_VARNISH_VMOD_CONFIG], [
 	AC_SUBST([VMOD_TEST_PATH], [$VARNISH_TEST_PATH])
 ])
 
-# _VARNISH_VMOD(NAME)
-# -------------------
+# _VARNISH_VMOD(NAME, MODE)
+# -------------------------
 AC_DEFUN([_VARNISH_VMOD], [
 
 	AC_REQUIRE([_VARNISH_VMOD_CONFIG])
@@ -201,7 +201,15 @@ AC_DEFUN([_VARNISH_VMOD], [
 	dnl Define the VCL directory for automake
 	AC_SUBST([vmod_$1_vcldir], [\${vcldir}/$1])
 
-	VMOD_RULES="
+	m4_case([$2],
+		[static],
+		[m4_pushdef([VCC_SRC], [\$(srcdir)/vmod_$1.vcc])],
+		[generated],
+		[m4_pushdef([VCC_SRC],
+			[\$(builddir)/vmod_$1.vcc \$(srcdir)/vmod_$1.vcc])],
+		[AC_MSG_ERROR([Unknown mode $2])])
+
+	AC_SUBST(m4_toupper(BUILD_VMOD_$1), ["
 
 vmod_$1.lo: vcc_$1_if.c vcc_$1_if.h
 
@@ -210,7 +218,7 @@ vmod_$1.lo: \$(nodist_libvmod_$1_la_SOURCES)
 vcc_$1_if.h vmod_$1.rst vmod_$1.man.rst: vcc_$1_if.c
 
 vcc_$1_if.c: vmod_$1.vcc
-	\$(A""M_V_VMODTOOL) \$(PYTHON) \$(VMODTOOL) -o vcc_$1_if \$(srcdir)/vmod_$1.vcc
+	\$(A""M_V_VMODTOOL) \$(PYTHON) \$(VMODTOOL) -o vcc_$1_if VCC_SRC
 
 vmod_$1.3: vmod_$1.man.rst
 	\$(A""M_V_GEN) \$(RST2MAN) vmod_$1.man.rst vmod_$1.3
@@ -223,10 +231,9 @@ clean-vmod-$1:
 	rm -f vcc_$1_if.c vcc_$1_if.h
 	rm -f vmod_$1.rst vmod_$1.man.rst vmod_$1.3
 
-"
-
-	AC_SUBST(m4_toupper(BUILD_VMOD_$1), [$VMOD_RULES])
-	AM_SUBST_NOTMAKE(m4_toupper(BUILD_VMOD_$1))
+"])dnl
+	AM_SUBST_NOTMAKE(m4_toupper(BUILD_VMOD_$1))dnl
+	m4_popdef([VCC_SRC])dnl
 ])
 
 # VARNISH_VMODS(NAMES)
@@ -368,7 +375,27 @@ clean-vmod-$1:
 AC_DEFUN([VARNISH_VMODS], [
 	m4_foreach([_vmod_name],
 		m4_split(m4_normalize([$1])),
-		[_VARNISH_VMOD(_vmod_name)])
+		[_VARNISH_VMOD(_vmod_name, [static])])
+])
+
+# VARNISH_VMODS_GENERATED(NAMES)
+# ------------------------------
+# Since: Varnish 7.0.0
+#
+# Varnish 7 adds the possibility to transparently work with a generated VCC
+# file. The VCC file would then be created in the build directory, which is
+# incompatible with how the VARNISH_VMODS macro operates.
+#
+# If that VCC file only needs to be generated once and is distributed, builds
+# from the dist archive will have the VCC file in the source directory.
+#
+# With Varnish's ability to run VMODTOOL in a VPATH build both scnerarios are
+# taken care of. This macro works otherwise exactly like VARNISH_VMODS.
+#
+AC_DEFUN([VARNISH_VMODS_GENERATED], [
+	m4_foreach([_vmod_name],
+		m4_split(m4_normalize([$1])),
+		[_VARNISH_VMOD(_vmod_name, [generated])])
 ])
 
 # _VARNISH_VSC_CONFIG

From nils.goroll at uplex.de  Tue Jun 23 17:47:03 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 23 Jun 2020 17:47:03 +0000 (UTC)
Subject: [master] 5b79a6580 Diffentiate "no backend" from "none resolved" in
 VDI_Resolve()
Message-ID: <20200623174703.50749C15D5@lists.varnish-cache.org>


commit 5b79a65807d63668dd00f0bca9151dd9651d6271
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed May 13 18:16:59 2020 +0200

    Diffentiate "no backend" from "none resolved" in VDI_Resolve()
    
    recover a minor improvement from #2680:
    
    This commit is to emit slightly different error messages to
    differentiate between the cases "no backend set in vcl" and
    "none returned from the director".
    
    Includes minor polishing
    
    Ref #2860 #3311

diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c
index 3e6902031..1aa4388d5 100644
--- a/bin/varnishd/cache/cache_director.c
+++ b/bin/varnishd/cache/cache_director.c
@@ -103,24 +103,26 @@ VRT_VDI_Resolve(VRT_CTX, VCL_BACKEND d)
 static VCL_BACKEND
 VDI_Resolve(VRT_CTX)
 {
-	const struct director *d;
+	VCL_BACKEND d;
 	struct busyobj *bo;
 
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	bo = ctx->bo;
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	CHECK_OBJ_ORNULL(bo->director_req, DIRECTOR_MAGIC);
-	d = VRT_VDI_Resolve(ctx, bo->director_req);
-	CHECK_OBJ_ORNULL(d, DIRECTOR_MAGIC);
-	if (d == NULL) {
-		VSLb(bo->vsl, SLT_FetchError, "No backend");
-	} else {
-		AN(d->vdir);
 
-		if (d->vdir->admin_health->health == 0)
-			d = NULL;
+	if (bo->director_req == NULL) {
+		VSLb(bo->vsl, SLT_FetchError, "No backend");
+		return (NULL);
 	}
 
-	return (d);
+	CHECK_OBJ(bo->director_req, DIRECTOR_MAGIC);
+	d = VRT_VDI_Resolve(ctx, bo->director_req);
+	if (d != NULL)
+		return (d);
+
+	VSLb(bo->vsl, SLT_FetchError,
+	    "Director %s returned no backend", bo->director_req->vcl_name);
+	return (NULL);
 }
 
 /* Get a set of response headers -------------------------------------*/

From nils.goroll at uplex.de  Tue Jun 23 17:47:03 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 23 Jun 2020 17:47:03 +0000 (UTC)
Subject: [master] fca8c59aa Back to the originally proposed name
 VRT_DirectorResolve()
Message-ID: <20200623174703.6D84BC15D8@lists.varnish-cache.org>


commit fca8c59aa07e9eb6b761dd2a5258855630c950e9
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Jun 23 19:39:32 2020 +0200

    Back to the originally proposed name VRT_DirectorResolve()
    
    Ref
    https://github.com/varnishcache/varnish-cache/pull/3315#discussion_r429796022
    
    OKed by phk on irc.
    
    VRT_DirectorResolve() was the name originally proposed in #2680, which
    was renamed in #3311 and accepted by me to get ahead. Not that Dridi
    prefers the original name and phk goes along, we change it back before
    the next release.

diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c
index 1aa4388d5..c7e701026 100644
--- a/bin/varnishd/cache/cache_director.c
+++ b/bin/varnishd/cache/cache_director.c
@@ -81,7 +81,7 @@ VDI_Ahealth(const struct director *d)
 /* Resolve director --------------------------------------------------*/
 
 VCL_BACKEND
-VRT_VDI_Resolve(VRT_CTX, VCL_BACKEND d)
+VRT_DirectorResolve(VRT_CTX, VCL_BACKEND d)
 {
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 
@@ -116,7 +116,7 @@ VDI_Resolve(VRT_CTX)
 	}
 
 	CHECK_OBJ(bo->director_req, DIRECTOR_MAGIC);
-	d = VRT_VDI_Resolve(ctx, bo->director_req);
+	d = VRT_DirectorResolve(ctx, bo->director_req);
 	if (d != NULL)
 		return (d);
 
diff --git a/include/vrt.h b/include/vrt.h
index 3b015ff6a..e38b9bf8f 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -54,7 +54,7 @@
  * binary/load-time compatible, increment MAJOR version
  *
  * NEXT (2020-09-15)
- *	VRT_VDI_Resolve() added
+ *	VRT_DirectorResolve() added
  * 11.0 (2020-03-16)
  *	Changed type of vsa_suckaddr_len from int to size_t
  *	New prefix_{ptr|len} fields in vrt_backend
@@ -437,7 +437,7 @@ struct vrt_backend_probe {
 	VRT_BACKEND_PROBE_FIELDS(const)
 };
 
-VCL_BACKEND VRT_VDI_Resolve(VRT_CTX, VCL_BACKEND);
+VCL_BACKEND VRT_DirectorResolve(VRT_CTX, VCL_BACKEND);
 
 /***********************************************************************
  * Implementation details of ACLs
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 1b0494e11..a459dd757 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -839,7 +839,7 @@ static const struct vcc_methods {
 
 	{ STRINGS, STRING, "upper", "VRT_UpperLowerStrands(ctx, \vT, 1)", 1 },
 	{ STRINGS, STRING, "lower", "VRT_UpperLowerStrands(ctx, \vT, 0)", 1 },
-	{ BACKEND, BACKEND, "resolve", "VRT_VDI_Resolve(ctx, \v1)", 1 },
+	{ BACKEND, BACKEND, "resolve", "VRT_DirectorResolve(ctx, \v1)", 1 },
 
 	{ NULL, NULL,		NULL,		NULL},
 };

From nils.goroll at uplex.de  Tue Jun 23 17:53:35 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 23 Jun 2020 19:53:35 +0200
Subject: [master] fca8c59aa Back to the originally proposed name
 VRT_DirectorResolve()
In-Reply-To: <20200623174703.6D84BC15D8@lists.varnish-cache.org>
References: <20200623174703.6D84BC15D8@lists.varnish-cache.org>
Message-ID: <2fad84ee-8b74-fc54-ed28-b081ade253c0@uplex.de>

On 23/06/2020 19:47, Nils Goroll wrote:
> Not that Dridi

Typo, sorry: *Now* that Dridi...

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 488 bytes
Desc: OpenPGP digital signature
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20200623/199db48e/attachment.bin>

From nils.goroll at uplex.de  Wed Jun 24 12:33:09 2020
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 24 Jun 2020 12:33:09 +0000 (UTC)
Subject: [master] 043895614 basic macos source build dependency install
 instructions
Message-ID: <20200624123309.25155AA4E5@lists.varnish-cache.org>


commit 0438956141b540f4bb61e9cff712dd29e2a6269c
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jun 24 14:31:03 2020 +0200

    basic macos source build dependency install instructions

diff --git a/doc/sphinx/installation/install_source.rst b/doc/sphinx/installation/install_source.rst
index 7a7893926..8f11d33fe 100644
--- a/doc/sphinx/installation/install_source.rst
+++ b/doc/sphinx/installation/install_source.rst
@@ -37,6 +37,8 @@ Or if you want the built from sources::
 	cd /usr/ports/www/varnish6
 	make depends clean
 
+Then continue `Compiling Varnish`_
+
 Build dependencies on Debian / Ubuntu
 --------------------------------------
 
@@ -72,6 +74,8 @@ Optionally, to pull from a repository::
 
     sudo apt-get install git
 
+Then continue `Compiling Varnish`_
+
 Build dependencies on Red Hat / CentOS
 --------------------------------------
 
@@ -121,6 +125,32 @@ Optionally, to pull from a repository::
 .. XXX autoconf-archive ? is this any helpful on the notoriously
    outdated Redhats?
 
+Then continue `Compiling Varnish`_
+
+Build dependencies on MacOS
+---------------------------
+
+To compile varnish on MacOS, these steps should install the required
+dependencies:
+
+* Install ``xcode`` via the App Store
+
+* Install dependencies via `brew`::
+
+    brew install \
+	autoconf \
+	automake \
+	pkg-config \
+	libtool \
+	docutils \
+	sphinx-doc
+
+* Add sphinx to PATH as advised by the installer::
+
+    PATH="/usr/local/opt/sphinx-doc/bin:$PATH"
+
+Then continue `Compiling Varnish`_
+
 Build dependencies on a SmartOS Zone
 ------------------------------------
 

From dridi.boukelmoune at gmail.com  Tue Jun 30 09:54:09 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 30 Jun 2020 09:54:09 +0000 (UTC)
Subject: [master] 7cd4c2427 build: Rename VMOD sources to avoid collisions
Message-ID: <20200630095409.B144EAC935@lists.varnish-cache.org>


commit 7cd4c242795f3832ade51b254e7ab38f98e778ad
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Jun 29 17:28:36 2020 +0200

    build: Rename VMOD sources to avoid collisions
    
    Instead of calling all VCC files vmod.vcc they were renamed to
    vmod_foo.vcc and instead of generating vcc_if.[ch] we do the same
    thing with vcc_foo_if.[ch].
    
    This a manual mechanical change that will then be picked up by
    vmodtool's automake boilerplate generator.
    
    Better diff with the --word-diff --word-diff-regex=. options.

diff --git a/.gitignore b/.gitignore
index c476a47dc..e261781ba 100644
--- a/.gitignore
+++ b/.gitignore
@@ -73,8 +73,8 @@ cscope.*out
 # Misc. generated files for included vmods.
 /lib/libvmod_*/VSC_*.c
 /lib/libvmod_*/VSC_*.h
-/lib/libvmod_*/vcc_if.c
-/lib/libvmod_*/vcc_if.h
+/lib/libvmod_*/vcc_*_if.c
+/lib/libvmod_*/vcc_*_if.h
 /lib/libvmod_*/vmod_*.rst
 
 # Man-files and binaries
diff --git a/lib/libvmod_blob/automake_boilerplate.am b/lib/libvmod_blob/automake_boilerplate.am
index 0ff3f4f98..3da6b00a1 100644
--- a/lib/libvmod_blob/automake_boilerplate.am
+++ b/lib/libvmod_blob/automake_boilerplate.am
@@ -8,7 +8,7 @@ AM_CPPFLAGS = \
 	-I$(top_builddir)/include
 
 vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
-vmodtoolargs ?= --strict --boilerplate -o vcc_if
+vmodtoolargs ?= --strict --boilerplate -o vcc_blob_if
 
 vmod_LTLIBRARIES = libvmod_blob.la
 
@@ -21,17 +21,17 @@ libvmod_blob_la_LDFLAGS = \
 	$(VMOD_LDFLAGS) \
 	@SAN_LDFLAGS@
 
-nodist_libvmod_blob_la_SOURCES = vcc_if.c vcc_if.h
+nodist_libvmod_blob_la_SOURCES = vcc_blob_if.c vcc_blob_if.h
 
-$(libvmod_blob_la_OBJECTS): vcc_if.h
+$(libvmod_blob_la_OBJECTS): vcc_blob_if.h
 
-vcc_if.h vmod_blob.rst vmod_blob.man.rst: vcc_if.c
+vcc_blob_if.h vmod_blob.rst vmod_blob.man.rst: vcc_blob_if.c
 
-vcc_if.c: $(vmodtool) $(srcdir)/vmod.vcc
-	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod.vcc
+vcc_blob_if.c: $(vmodtool) $(srcdir)/vmod_blob.vcc
+	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_blob.vcc
 
-EXTRA_DIST = $(srcdir)/vmod.vcc automake_boilerplate.am
+EXTRA_DIST = $(srcdir)/vmod_blob.vcc automake_boilerplate.am
 
-CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
+CLEANFILES = $(builddir)/vcc_blob_if.c $(builddir)/vcc_blob_if.h \
 	$(builddir)/vmod_blob.rst \
 	$(builddir)/vmod_blob.man.rst
diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c
index ec4e15aef..a805fa8e2 100644
--- a/lib/libvmod_blob/vmod_blob.c
+++ b/lib/libvmod_blob/vmod_blob.c
@@ -33,7 +33,7 @@
 
 #include "cache/cache.h"
 
-#include "vcc_if.h"
+#include "vcc_blob_if.h"
 #include "vmod_blob.h"
 
 #define VMOD_BLOB_TYPE 0xfade4faa
diff --git a/lib/libvmod_blob/vmod.vcc b/lib/libvmod_blob/vmod_blob.vcc
similarity index 100%
rename from lib/libvmod_blob/vmod.vcc
rename to lib/libvmod_blob/vmod_blob.vcc
diff --git a/lib/libvmod_cookie/automake_boilerplate.am b/lib/libvmod_cookie/automake_boilerplate.am
index f5333060a..c3b52f37b 100644
--- a/lib/libvmod_cookie/automake_boilerplate.am
+++ b/lib/libvmod_cookie/automake_boilerplate.am
@@ -8,7 +8,7 @@ AM_CPPFLAGS = \
 	-I$(top_builddir)/include
 
 vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
-vmodtoolargs ?= --strict --boilerplate -o vcc_if
+vmodtoolargs ?= --strict --boilerplate -o vcc_cookie_if
 
 vmod_LTLIBRARIES = libvmod_cookie.la
 
@@ -21,18 +21,18 @@ libvmod_cookie_la_LDFLAGS = \
 	$(VMOD_LDFLAGS) \
 	@SAN_LDFLAGS@
 
-nodist_libvmod_cookie_la_SOURCES = vcc_if.c vcc_if.h
+nodist_libvmod_cookie_la_SOURCES = vcc_cookie_if.c vcc_cookie_if.h
 
-$(libvmod_cookie_la_OBJECTS): vcc_if.h
+$(libvmod_cookie_la_OBJECTS): vcc_cookie_if.h
 
-vcc_if.h vmod_cookie.rst vmod_cookie.man.rst: vcc_if.c
+vcc_cookie_if.h vmod_cookie.rst vmod_cookie.man.rst: vcc_cookie_if.c
 
-vcc_if.c: $(vmodtool) $(srcdir)/vmod.vcc
-	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod.vcc
+vcc_cookie_if.c: $(vmodtool) $(srcdir)/vmod_cookie.vcc
+	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_cookie.vcc
 
-EXTRA_DIST = $(srcdir)/vmod.vcc automake_boilerplate.am
+EXTRA_DIST = $(srcdir)/vmod_cookie.vcc automake_boilerplate.am
 
-CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
+CLEANFILES = $(builddir)/vcc_cookie_if.c $(builddir)/vcc_cookie_if.h \
 	$(builddir)/vmod_cookie.rst \
 	$(builddir)/vmod_cookie.man.rst
 
@@ -57,7 +57,7 @@ TESTS = \
 EXTRA_DIST += $(TESTS)
 
 vtc-refresh-tests:
-	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod.vcc
+	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_cookie.vcc
 	@cd $(top_builddir) && ./config.status --file=$(subdir)/Makefile
 
 include $(top_srcdir)/vtc.am
diff --git a/lib/libvmod_cookie/vmod_cookie.c b/lib/libvmod_cookie/vmod_cookie.c
index 87a963e33..5be9178e6 100644
--- a/lib/libvmod_cookie/vmod_cookie.c
+++ b/lib/libvmod_cookie/vmod_cookie.c
@@ -44,7 +44,7 @@
 #include <vsb.h>
 #include <vre.h>
 
-#include "vcc_if.h"
+#include "vcc_cookie_if.h"
 
 #define VRE_MAX_GROUPS 8
 
diff --git a/lib/libvmod_cookie/vmod.vcc b/lib/libvmod_cookie/vmod_cookie.vcc
similarity index 100%
rename from lib/libvmod_cookie/vmod.vcc
rename to lib/libvmod_cookie/vmod_cookie.vcc
diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am
index 8c6df5232..3f902065c 100644
--- a/lib/libvmod_debug/Makefile.am
+++ b/lib/libvmod_debug/Makefile.am
@@ -17,7 +17,7 @@ libvmod_debug_la_LDFLAGS = \
 
 
 # not --strict
-vmodtoolargs = --boilerplate
+vmodtoolargs = --boilerplate -o vcc_debug_if
 
 .vsc.c:
 	$(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $<
diff --git a/lib/libvmod_debug/automake_boilerplate.am b/lib/libvmod_debug/automake_boilerplate.am
index f404971ef..fe41b579d 100644
--- a/lib/libvmod_debug/automake_boilerplate.am
+++ b/lib/libvmod_debug/automake_boilerplate.am
@@ -8,7 +8,7 @@ AM_CPPFLAGS = \
 	-I$(top_builddir)/include
 
 vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
-vmodtoolargs ?= --strict --boilerplate -o vcc_if
+vmodtoolargs ?= --strict --boilerplate -o vcc_debug_if
 
 vmod_LTLIBRARIES = libvmod_debug.la
 
@@ -21,17 +21,17 @@ libvmod_debug_la_LDFLAGS = \
 	$(VMOD_LDFLAGS) \
 	@SAN_LDFLAGS@
 
-nodist_libvmod_debug_la_SOURCES = vcc_if.c vcc_if.h
+nodist_libvmod_debug_la_SOURCES = vcc_debug_if.c vcc_debug_if.h
 
-$(libvmod_debug_la_OBJECTS): vcc_if.h
+$(libvmod_debug_la_OBJECTS): vcc_debug_if.h
 
-vcc_if.h vmod_debug.rst vmod_debug.man.rst: vcc_if.c
+vcc_debug_if.h vmod_debug.rst vmod_debug.man.rst: vcc_debug_if.c
 
-vcc_if.c: $(vmodtool) $(srcdir)/vmod.vcc
-	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod.vcc
+vcc_debug_if.c: $(vmodtool) $(srcdir)/vmod_debug.vcc
+	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_debug.vcc
 
-EXTRA_DIST = $(srcdir)/vmod.vcc automake_boilerplate.am
+EXTRA_DIST = $(srcdir)/vmod_debug.vcc automake_boilerplate.am
 
-CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
+CLEANFILES = $(builddir)/vcc_debug_if.c $(builddir)/vcc_debug_if.h \
 	$(builddir)/vmod_debug.rst \
 	$(builddir)/vmod_debug.man.rst
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index aa54dfeb8..4dbc5ab3b 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -42,7 +42,7 @@
 #include "vre.h"
 #include "vsa.h"
 #include "vtim.h"
-#include "vcc_if.h"
+#include "vcc_debug_if.h"
 #include "VSC_debug.h"
 
 struct priv_vcl {
diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod_debug.vcc
similarity index 100%
rename from lib/libvmod_debug/vmod.vcc
rename to lib/libvmod_debug/vmod_debug.vcc
diff --git a/lib/libvmod_debug/vmod_debug_dyn.c b/lib/libvmod_debug/vmod_debug_dyn.c
index b2430a978..3ef74544c 100644
--- a/lib/libvmod_debug/vmod_debug_dyn.c
+++ b/lib/libvmod_debug/vmod_debug_dyn.c
@@ -41,7 +41,7 @@
 
 #include "vsa.h"
 #include "vss.h"
-#include "vcc_if.h"
+#include "vcc_debug_if.h"
 
 struct xyzzy_debug_dyn {
 	unsigned		magic;
diff --git a/lib/libvmod_debug/vmod_debug_obj.c b/lib/libvmod_debug/vmod_debug_obj.c
index 0d32683b1..8efd6ad50 100644
--- a/lib/libvmod_debug/vmod_debug_obj.c
+++ b/lib/libvmod_debug/vmod_debug_obj.c
@@ -35,7 +35,7 @@
 
 #include "cache/cache.h"
 
-#include "vcc_if.h"
+#include "vcc_debug_if.h"
 
 struct xyzzy_debug_obj {
 	unsigned		magic;
diff --git a/lib/libvmod_directors/automake_boilerplate.am b/lib/libvmod_directors/automake_boilerplate.am
index 69287d3d6..c03bd61d8 100644
--- a/lib/libvmod_directors/automake_boilerplate.am
+++ b/lib/libvmod_directors/automake_boilerplate.am
@@ -8,7 +8,7 @@ AM_CPPFLAGS = \
 	-I$(top_builddir)/include
 
 vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
-vmodtoolargs ?= --strict --boilerplate -o vcc_if
+vmodtoolargs ?= --strict --boilerplate -o vcc_directors_if
 
 vmod_LTLIBRARIES = libvmod_directors.la
 
@@ -21,17 +21,17 @@ libvmod_directors_la_LDFLAGS = \
 	$(VMOD_LDFLAGS) \
 	@SAN_LDFLAGS@
 
-nodist_libvmod_directors_la_SOURCES = vcc_if.c vcc_if.h
+nodist_libvmod_directors_la_SOURCES = vcc_directors_if.c vcc_directors_if.h
 
-$(libvmod_directors_la_OBJECTS): vcc_if.h
+$(libvmod_directors_la_OBJECTS): vcc_directors_if.h
 
-vcc_if.h vmod_directors.rst vmod_directors.man.rst: vcc_if.c
+vcc_directors_if.h vmod_directors.rst vmod_directors.man.rst: vcc_directors_if.c
 
-vcc_if.c: $(vmodtool) $(srcdir)/vmod.vcc
-	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod.vcc
+vcc_directors_if.c: $(vmodtool) $(srcdir)/vmod_directors.vcc
+	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_directors.vcc
 
-EXTRA_DIST = $(srcdir)/vmod.vcc automake_boilerplate.am
+EXTRA_DIST = $(srcdir)/vmod_directors.vcc automake_boilerplate.am
 
-CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
+CLEANFILES = $(builddir)/vcc_directors_if.c $(builddir)/vcc_directors_if.h \
 	$(builddir)/vmod_directors.rst \
 	$(builddir)/vmod_directors.man.rst
diff --git a/lib/libvmod_directors/fall_back.c b/lib/libvmod_directors/fall_back.c
index 72001d6de..425c1093a 100644
--- a/lib/libvmod_directors/fall_back.c
+++ b/lib/libvmod_directors/fall_back.c
@@ -35,7 +35,7 @@
 
 #include "cache/cache.h"
 
-#include "vcc_if.h"
+#include "vcc_directors_if.h"
 
 #include "vdir.h"
 #include "vsb.h"
diff --git a/lib/libvmod_directors/hash.c b/lib/libvmod_directors/hash.c
index 314e9a2c6..eeddffca2 100644
--- a/lib/libvmod_directors/hash.c
+++ b/lib/libvmod_directors/hash.c
@@ -37,7 +37,7 @@
 
 #include "vdir.h"
 
-#include "vcc_if.h"
+#include "vcc_directors_if.h"
 
 struct vmod_directors_hash {
 	unsigned				magic;
diff --git a/lib/libvmod_directors/misc.c b/lib/libvmod_directors/misc.c
index d70c8ce55..f43875d61 100644
--- a/lib/libvmod_directors/misc.c
+++ b/lib/libvmod_directors/misc.c
@@ -34,7 +34,7 @@
 #include "vrt.h"
 #include "vcl.h"
 
-#include "vcc_if.h"
+#include "vcc_directors_if.h"
 
 VCL_BACKEND
 VPFX(lookup)(VRT_CTX, VCL_STRING name)
diff --git a/lib/libvmod_directors/random.c b/lib/libvmod_directors/random.c
index fc0e355c8..b18e1f6fd 100644
--- a/lib/libvmod_directors/random.c
+++ b/lib/libvmod_directors/random.c
@@ -39,7 +39,7 @@
 
 #include "vdir.h"
 
-#include "vcc_if.h"
+#include "vcc_directors_if.h"
 
 struct vmod_directors_random {
 	unsigned				magic;
diff --git a/lib/libvmod_directors/round_robin.c b/lib/libvmod_directors/round_robin.c
index bc4e81876..74ec567e0 100644
--- a/lib/libvmod_directors/round_robin.c
+++ b/lib/libvmod_directors/round_robin.c
@@ -35,7 +35,7 @@
 
 #include "cache/cache.h"
 
-#include "vcc_if.h"
+#include "vcc_directors_if.h"
 
 #include "vdir.h"
 
diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c
index ea3dafad3..809fd67a5 100644
--- a/lib/libvmod_directors/shard_dir.c
+++ b/lib/libvmod_directors/shard_dir.c
@@ -44,7 +44,7 @@
 #include "vbm.h"
 #include "vrnd.h"
 
-#include "vcc_if.h"
+#include "vcc_directors_if.h"
 #include "shard_dir.h"
 
 struct shard_be_info {
diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod_directors.vcc
similarity index 100%
rename from lib/libvmod_directors/vmod.vcc
rename to lib/libvmod_directors/vmod_directors.vcc
diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c
index d634f0785..9779ae78d 100644
--- a/lib/libvmod_directors/vmod_shard.c
+++ b/lib/libvmod_directors/vmod_shard.c
@@ -39,7 +39,7 @@
 
 #include "vend.h"
 
-#include "vcc_if.h"
+#include "vcc_directors_if.h"
 #include "shard_dir.h"
 #include "shard_cfg.h"
 #include "vsb.h"
diff --git a/lib/libvmod_proxy/automake_boilerplate.am b/lib/libvmod_proxy/automake_boilerplate.am
index fb295e30c..c4a099974 100644
--- a/lib/libvmod_proxy/automake_boilerplate.am
+++ b/lib/libvmod_proxy/automake_boilerplate.am
@@ -8,7 +8,7 @@ AM_CPPFLAGS = \
 	-I$(top_builddir)/include
 
 vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
-vmodtoolargs ?= --strict --boilerplate -o vcc_if
+vmodtoolargs ?= --strict --boilerplate -o vcc_proxy_if
 
 vmod_LTLIBRARIES = libvmod_proxy.la
 
@@ -21,17 +21,17 @@ libvmod_proxy_la_LDFLAGS = \
 	$(VMOD_LDFLAGS) \
 	@SAN_LDFLAGS@
 
-nodist_libvmod_proxy_la_SOURCES = vcc_if.c vcc_if.h
+nodist_libvmod_proxy_la_SOURCES = vcc_proxy_if.c vcc_proxy_if.h
 
-$(libvmod_proxy_la_OBJECTS): vcc_if.h
+$(libvmod_proxy_la_OBJECTS): vcc_proxy_if.h
 
-vcc_if.h vmod_proxy.rst vmod_proxy.man.rst: vcc_if.c
+vcc_proxy_if.h vmod_proxy.rst vmod_proxy.man.rst: vcc_proxy_if.c
 
-vcc_if.c: $(vmodtool) $(srcdir)/vmod.vcc
-	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod.vcc
+vcc_proxy_if.c: $(vmodtool) $(srcdir)/vmod_proxy.vcc
+	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_proxy.vcc
 
-EXTRA_DIST = $(srcdir)/vmod.vcc automake_boilerplate.am
+EXTRA_DIST = $(srcdir)/vmod_proxy.vcc automake_boilerplate.am
 
-CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
+CLEANFILES = $(builddir)/vcc_proxy_if.c $(builddir)/vcc_proxy_if.h \
 	$(builddir)/vmod_proxy.rst \
 	$(builddir)/vmod_proxy.man.rst
diff --git a/lib/libvmod_proxy/vmod_proxy.c b/lib/libvmod_proxy/vmod_proxy.c
index 499cf24d3..b877d5bc1 100644
--- a/lib/libvmod_proxy/vmod_proxy.c
+++ b/lib/libvmod_proxy/vmod_proxy.c
@@ -40,7 +40,7 @@
 
 #include "proxy/cache_proxy.h"
 
-#include "vcc_if.h"
+#include "vcc_proxy_if.h"
 
 
 struct pp2_tlv_ssl {
diff --git a/lib/libvmod_proxy/vmod.vcc b/lib/libvmod_proxy/vmod_proxy.vcc
similarity index 100%
rename from lib/libvmod_proxy/vmod.vcc
rename to lib/libvmod_proxy/vmod_proxy.vcc
diff --git a/lib/libvmod_purge/automake_boilerplate.am b/lib/libvmod_purge/automake_boilerplate.am
index d6d552864..33e9be06c 100644
--- a/lib/libvmod_purge/automake_boilerplate.am
+++ b/lib/libvmod_purge/automake_boilerplate.am
@@ -8,7 +8,7 @@ AM_CPPFLAGS = \
 	-I$(top_builddir)/include
 
 vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
-vmodtoolargs ?= --strict --boilerplate -o vcc_if
+vmodtoolargs ?= --strict --boilerplate -o vcc_purge_if
 
 vmod_LTLIBRARIES = libvmod_purge.la
 
@@ -21,17 +21,17 @@ libvmod_purge_la_LDFLAGS = \
 	$(VMOD_LDFLAGS) \
 	@SAN_LDFLAGS@
 
-nodist_libvmod_purge_la_SOURCES = vcc_if.c vcc_if.h
+nodist_libvmod_purge_la_SOURCES = vcc_purge_if.c vcc_purge_if.h
 
-$(libvmod_purge_la_OBJECTS): vcc_if.h
+$(libvmod_purge_la_OBJECTS): vcc_purge_if.h
 
-vcc_if.h vmod_purge.rst vmod_purge.man.rst: vcc_if.c
+vcc_purge_if.h vmod_purge.rst vmod_purge.man.rst: vcc_purge_if.c
 
-vcc_if.c: $(vmodtool) $(srcdir)/vmod.vcc
-	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod.vcc
+vcc_purge_if.c: $(vmodtool) $(srcdir)/vmod_purge.vcc
+	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_purge.vcc
 
-EXTRA_DIST = $(srcdir)/vmod.vcc automake_boilerplate.am
+EXTRA_DIST = $(srcdir)/vmod_purge.vcc automake_boilerplate.am
 
-CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
+CLEANFILES = $(builddir)/vcc_purge_if.c $(builddir)/vcc_purge_if.h \
 	$(builddir)/vmod_purge.rst \
 	$(builddir)/vmod_purge.man.rst
diff --git a/lib/libvmod_purge/vmod_purge.c b/lib/libvmod_purge/vmod_purge.c
index fd68b31e4..594895d42 100644
--- a/lib/libvmod_purge/vmod_purge.c
+++ b/lib/libvmod_purge/vmod_purge.c
@@ -38,7 +38,7 @@
 #include "miniobj.h"
 #include "vrt.h"
 
-#include "vcc_if.h"
+#include "vcc_purge_if.h"
 
 VCL_INT v_matchproto_(td_purge_hard)
 vmod_hard(VRT_CTX)
diff --git a/lib/libvmod_purge/vmod.vcc b/lib/libvmod_purge/vmod_purge.vcc
similarity index 100%
rename from lib/libvmod_purge/vmod.vcc
rename to lib/libvmod_purge/vmod_purge.vcc
diff --git a/lib/libvmod_std/automake_boilerplate.am b/lib/libvmod_std/automake_boilerplate.am
index 146a4f1a6..3e84940b8 100644
--- a/lib/libvmod_std/automake_boilerplate.am
+++ b/lib/libvmod_std/automake_boilerplate.am
@@ -8,7 +8,7 @@ AM_CPPFLAGS = \
 	-I$(top_builddir)/include
 
 vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
-vmodtoolargs ?= --strict --boilerplate -o vcc_if
+vmodtoolargs ?= --strict --boilerplate -o vcc_std_if
 
 vmod_LTLIBRARIES = libvmod_std.la
 
@@ -21,18 +21,18 @@ libvmod_std_la_LDFLAGS = \
 	$(VMOD_LDFLAGS) \
 	@SAN_LDFLAGS@
 
-nodist_libvmod_std_la_SOURCES = vcc_if.c vcc_if.h
+nodist_libvmod_std_la_SOURCES = vcc_std_if.c vcc_std_if.h
 
-$(libvmod_std_la_OBJECTS): vcc_if.h
+$(libvmod_std_la_OBJECTS): vcc_std_if.h
 
-vcc_if.h vmod_std.rst vmod_std.man.rst: vcc_if.c
+vcc_std_if.h vmod_std.rst vmod_std.man.rst: vcc_std_if.c
 
-vcc_if.c: $(vmodtool) $(srcdir)/vmod.vcc
-	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod.vcc
+vcc_std_if.c: $(vmodtool) $(srcdir)/vmod_std.vcc
+	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_std.vcc
 
-EXTRA_DIST = $(srcdir)/vmod.vcc automake_boilerplate.am
+EXTRA_DIST = $(srcdir)/vmod_std.vcc automake_boilerplate.am
 
-CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
+CLEANFILES = $(builddir)/vcc_std_if.c $(builddir)/vcc_std_if.h \
 	$(builddir)/vmod_std.rst \
 	$(builddir)/vmod_std.man.rst
 
@@ -42,7 +42,7 @@ TESTS = \
 EXTRA_DIST += $(TESTS)
 
 vtc-refresh-tests:
-	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod.vcc
+	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_std.vcc
 	@cd $(top_builddir) && ./config.status --file=$(subdir)/Makefile
 
 include $(top_srcdir)/vtc.am
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index 51a9b7e16..244680bfb 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -49,7 +49,7 @@
 #include "vtim.h"
 #include "vcl.h"
 
-#include "vcc_if.h"
+#include "vcc_std_if.h"
 
 VCL_VOID v_matchproto_(td_std_set_ip_tos)
 vmod_set_ip_tos(VRT_CTX, VCL_INT tos)
diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod_std.vcc
similarity index 100%
rename from lib/libvmod_std/vmod.vcc
rename to lib/libvmod_std/vmod_std.vcc
diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index e038d785a..3f0b74ea1 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -44,7 +44,7 @@
 #include "vsa.h"
 #include "vss.h"
 #include "vtim.h"
-#include "vcc_if.h"
+#include "vcc_std_if.h"
 
 /*
  * technically, as our VCL_INT is int64_t, its limits are INT64_MIN/INT64_MAX.
diff --git a/lib/libvmod_std/vmod_std_fileread.c b/lib/libvmod_std/vmod_std_fileread.c
index cd9181e2b..0e9f5cadc 100644
--- a/lib/libvmod_std/vmod_std_fileread.c
+++ b/lib/libvmod_std/vmod_std_fileread.c
@@ -48,7 +48,7 @@
 
 #include "vfil.h"
 
-#include "vcc_if.h"
+#include "vcc_std_if.h"
 
 struct frfile {
 	unsigned			magic;
diff --git a/lib/libvmod_std/vmod_std_querysort.c b/lib/libvmod_std/vmod_std_querysort.c
index f5b5e115e..8602dc6e1 100644
--- a/lib/libvmod_std/vmod_std_querysort.c
+++ b/lib/libvmod_std/vmod_std_querysort.c
@@ -35,7 +35,7 @@
 
 #include "cache/cache.h"
 
-#include "vcc_if.h"
+#include "vcc_std_if.h"
 
 static int
 compa(const void *a, const void *b)
diff --git a/lib/libvmod_unix/automake_boilerplate.am b/lib/libvmod_unix/automake_boilerplate.am
index 6eaa08742..913bc01c5 100644
--- a/lib/libvmod_unix/automake_boilerplate.am
+++ b/lib/libvmod_unix/automake_boilerplate.am
@@ -8,7 +8,7 @@ AM_CPPFLAGS = \
 	-I$(top_builddir)/include
 
 vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
-vmodtoolargs ?= --strict --boilerplate -o vcc_if
+vmodtoolargs ?= --strict --boilerplate -o vcc_unix_if
 
 vmod_LTLIBRARIES = libvmod_unix.la
 
@@ -21,17 +21,17 @@ libvmod_unix_la_LDFLAGS = \
 	$(VMOD_LDFLAGS) \
 	@SAN_LDFLAGS@
 
-nodist_libvmod_unix_la_SOURCES = vcc_if.c vcc_if.h
+nodist_libvmod_unix_la_SOURCES = vcc_unix_if.c vcc_unix_if.h
 
-$(libvmod_unix_la_OBJECTS): vcc_if.h
+$(libvmod_unix_la_OBJECTS): vcc_unix_if.h
 
-vcc_if.h vmod_unix.rst vmod_unix.man.rst: vcc_if.c
+vcc_unix_if.h vmod_unix.rst vmod_unix.man.rst: vcc_unix_if.c
 
-vcc_if.c: $(vmodtool) $(srcdir)/vmod.vcc
-	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod.vcc
+vcc_unix_if.c: $(vmodtool) $(srcdir)/vmod_unix.vcc
+	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_unix.vcc
 
-EXTRA_DIST = $(srcdir)/vmod.vcc automake_boilerplate.am
+EXTRA_DIST = $(srcdir)/vmod_unix.vcc automake_boilerplate.am
 
-CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
+CLEANFILES = $(builddir)/vcc_unix_if.c $(builddir)/vcc_unix_if.h \
 	$(builddir)/vmod_unix.rst \
 	$(builddir)/vmod_unix.man.rst
diff --git a/lib/libvmod_unix/vmod_unix.c b/lib/libvmod_unix/vmod_unix.c
index 99159a182..d7b882693 100644
--- a/lib/libvmod_unix/vmod_unix.c
+++ b/lib/libvmod_unix/vmod_unix.c
@@ -38,7 +38,7 @@
 #include "common/heritage.h"
 
 #include "cred_compat.h"
-#include "vcc_if.h"
+#include "vcc_unix_if.h"
 
 #define FAIL(ctx, msg) \
 	VRT_fail((ctx), "vmod unix failure: " msg)
diff --git a/lib/libvmod_unix/vmod.vcc b/lib/libvmod_unix/vmod_unix.vcc
similarity index 100%
rename from lib/libvmod_unix/vmod.vcc
rename to lib/libvmod_unix/vmod_unix.vcc
diff --git a/lib/libvmod_vtc/automake_boilerplate.am b/lib/libvmod_vtc/automake_boilerplate.am
index e330e3821..1099900f9 100644
--- a/lib/libvmod_vtc/automake_boilerplate.am
+++ b/lib/libvmod_vtc/automake_boilerplate.am
@@ -8,7 +8,7 @@ AM_CPPFLAGS = \
 	-I$(top_builddir)/include
 
 vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py
-vmodtoolargs ?= --strict --boilerplate -o vcc_if
+vmodtoolargs ?= --strict --boilerplate -o vcc_vtc_if
 
 vmod_LTLIBRARIES = libvmod_vtc.la
 
@@ -21,17 +21,17 @@ libvmod_vtc_la_LDFLAGS = \
 	$(VMOD_LDFLAGS) \
 	@SAN_LDFLAGS@
 
-nodist_libvmod_vtc_la_SOURCES = vcc_if.c vcc_if.h
+nodist_libvmod_vtc_la_SOURCES = vcc_vtc_if.c vcc_vtc_if.h
 
-$(libvmod_vtc_la_OBJECTS): vcc_if.h
+$(libvmod_vtc_la_OBJECTS): vcc_vtc_if.h
 
-vcc_if.h vmod_vtc.rst vmod_vtc.man.rst: vcc_if.c
+vcc_vtc_if.h vmod_vtc.rst vmod_vtc.man.rst: vcc_vtc_if.c
 
-vcc_if.c: $(vmodtool) $(srcdir)/vmod.vcc
-	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod.vcc
+vcc_vtc_if.c: $(vmodtool) $(srcdir)/vmod_vtc.vcc
+	@PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_vtc.vcc
 
-EXTRA_DIST = $(srcdir)/vmod.vcc automake_boilerplate.am
+EXTRA_DIST = $(srcdir)/vmod_vtc.vcc automake_boilerplate.am
 
-CLEANFILES = $(builddir)/vcc_if.c $(builddir)/vcc_if.h \
+CLEANFILES = $(builddir)/vcc_vtc_if.c $(builddir)/vcc_vtc_if.h \
 	$(builddir)/vmod_vtc.rst \
 	$(builddir)/vmod_vtc.man.rst
diff --git a/lib/libvmod_vtc/vmod_vtc.c b/lib/libvmod_vtc/vmod_vtc.c
index 499b611aa..b2b9afd7d 100644
--- a/lib/libvmod_vtc/vmod_vtc.c
+++ b/lib/libvmod_vtc/vmod_vtc.c
@@ -41,7 +41,7 @@
 #include "vtcp.h"
 #include "vtim.h"
 
-#include "vcc_if.h"
+#include "vcc_vtc_if.h"
 
 VCL_VOID v_matchproto_(td_vtc_barrier_sync)
 vmod_barrier_sync(VRT_CTX, VCL_STRING addr, VCL_DURATION tmo)
diff --git a/lib/libvmod_vtc/vmod.vcc b/lib/libvmod_vtc/vmod_vtc.vcc
similarity index 100%
rename from lib/libvmod_vtc/vmod.vcc
rename to lib/libvmod_vtc/vmod_vtc.vcc

From dridi.boukelmoune at gmail.com  Tue Jun 30 09:54:09 2020
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 30 Jun 2020 09:54:09 +0000 (UTC)
Subject: [master] b82271e52 build: Allow VMOD flags to be overridden
Message-ID: <20200630095409.C7007AC939@lists.varnish-cache.org>


commit b82271e52d4fc946d4d0856d150730d430c30a89
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jun 30 11:02:22 2020 +0200

    build: Allow VMOD flags to be overridden
    
    We override vmod_debug's LDFLAGS, so that warning is gone.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 61a695b70..66d6e67fd 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -63,10 +63,10 @@ vmodtoolargs ?= --strict --boilerplate -o PFX
 
 vmod_LTLIBRARIES = libvmod_XXX.la
 
-libvmod_XXX_la_CFLAGS = \\
+libvmod_XXX_la_CFLAGS ?= \\
 \t at SAN_CFLAGS@
 
-libvmod_XXX_la_LDFLAGS = \\
+libvmod_XXX_la_LDFLAGS ?= \\
 \t-export-symbols-regex 'Vmod_XXX_Data' \\
 \t$(AM_LDFLAGS) \\
 \t$(VMOD_LDFLAGS) \\
diff --git a/lib/libvmod_blob/automake_boilerplate.am b/lib/libvmod_blob/automake_boilerplate.am
index 3da6b00a1..7931bbbf0 100644
--- a/lib/libvmod_blob/automake_boilerplate.am
+++ b/lib/libvmod_blob/automake_boilerplate.am
@@ -12,10 +12,10 @@ vmodtoolargs ?= --strict --boilerplate -o vcc_blob_if
 
 vmod_LTLIBRARIES = libvmod_blob.la
 
-libvmod_blob_la_CFLAGS = \
+libvmod_blob_la_CFLAGS ?= \
 	@SAN_CFLAGS@
 
-libvmod_blob_la_LDFLAGS = \
+libvmod_blob_la_LDFLAGS ?= \
 	-export-symbols-regex 'Vmod_blob_Data' \
 	$(AM_LDFLAGS) \
 	$(VMOD_LDFLAGS) \
diff --git a/lib/libvmod_cookie/automake_boilerplate.am b/lib/libvmod_cookie/automake_boilerplate.am
index c3b52f37b..bf5214708 100644
--- a/lib/libvmod_cookie/automake_boilerplate.am
+++ b/lib/libvmod_cookie/automake_boilerplate.am
@@ -12,10 +12,10 @@ vmodtoolargs ?= --strict --boilerplate -o vcc_cookie_if
 
 vmod_LTLIBRARIES = libvmod_cookie.la
 
-libvmod_cookie_la_CFLAGS = \
+libvmod_cookie_la_CFLAGS ?= \
 	@SAN_CFLAGS@
 
-libvmod_cookie_la_LDFLAGS = \
+libvmod_cookie_la_LDFLAGS ?= \
 	-export-symbols-regex 'Vmod_cookie_Data' \
 	$(AM_LDFLAGS) \
 	$(VMOD_LDFLAGS) \
diff --git a/lib/libvmod_debug/automake_boilerplate.am b/lib/libvmod_debug/automake_boilerplate.am
index fe41b579d..b0bc43174 100644
--- a/lib/libvmod_debug/automake_boilerplate.am
+++ b/lib/libvmod_debug/automake_boilerplate.am
@@ -12,10 +12,10 @@ vmodtoolargs ?= --strict --boilerplate -o vcc_debug_if
 
 vmod_LTLIBRARIES = libvmod_debug.la
 
-libvmod_debug_la_CFLAGS = \
+libvmod_debug_la_CFLAGS ?= \
 	@SAN_CFLAGS@
 
-libvmod_debug_la_LDFLAGS = \
+libvmod_debug_la_LDFLAGS ?= \
 	-export-symbols-regex 'Vmod_debug_Data' \
 	$(AM_LDFLAGS) \
 	$(VMOD_LDFLAGS) \
diff --git a/lib/libvmod_directors/automake_boilerplate.am b/lib/libvmod_directors/automake_boilerplate.am
index c03bd61d8..f12c3c4e8 100644
--- a/lib/libvmod_directors/automake_boilerplate.am
+++ b/lib/libvmod_directors/automake_boilerplate.am
@@ -12,10 +12,10 @@ vmodtoolargs ?= --strict --boilerplate -o vcc_directors_if
 
 vmod_LTLIBRARIES = libvmod_directors.la
 
-libvmod_directors_la_CFLAGS = \
+libvmod_directors_la_CFLAGS ?= \
 	@SAN_CFLAGS@
 
-libvmod_directors_la_LDFLAGS = \
+libvmod_directors_la_LDFLAGS ?= \
 	-export-symbols-regex 'Vmod_directors_Data' \
 	$(AM_LDFLAGS) \
 	$(VMOD_LDFLAGS) \
diff --git a/lib/libvmod_proxy/automake_boilerplate.am b/lib/libvmod_proxy/automake_boilerplate.am
index c4a099974..a8a257dab 100644
--- a/lib/libvmod_proxy/automake_boilerplate.am
+++ b/lib/libvmod_proxy/automake_boilerplate.am
@@ -12,10 +12,10 @@ vmodtoolargs ?= --strict --boilerplate -o vcc_proxy_if
 
 vmod_LTLIBRARIES = libvmod_proxy.la
 
-libvmod_proxy_la_CFLAGS = \
+libvmod_proxy_la_CFLAGS ?= \
 	@SAN_CFLAGS@
 
-libvmod_proxy_la_LDFLAGS = \
+libvmod_proxy_la_LDFLAGS ?= \
 	-export-symbols-regex 'Vmod_proxy_Data' \
 	$(AM_LDFLAGS) \
 	$(VMOD_LDFLAGS) \
diff --git a/lib/libvmod_purge/automake_boilerplate.am b/lib/libvmod_purge/automake_boilerplate.am
index 33e9be06c..04904a376 100644
--- a/lib/libvmod_purge/automake_boilerplate.am
+++ b/lib/libvmod_purge/automake_boilerplate.am
@@ -12,10 +12,10 @@ vmodtoolargs ?= --strict --boilerplate -o vcc_purge_if
 
 vmod_LTLIBRARIES = libvmod_purge.la
 
-libvmod_purge_la_CFLAGS = \
+libvmod_purge_la_CFLAGS ?= \
 	@SAN_CFLAGS@
 
-libvmod_purge_la_LDFLAGS = \
+libvmod_purge_la_LDFLAGS ?= \
 	-export-symbols-regex 'Vmod_purge_Data' \
 	$(AM_LDFLAGS) \
 	$(VMOD_LDFLAGS) \
diff --git a/lib/libvmod_std/automake_boilerplate.am b/lib/libvmod_std/automake_boilerplate.am
index 3e84940b8..6a14779b2 100644
--- a/lib/libvmod_std/automake_boilerplate.am
+++ b/lib/libvmod_std/automake_boilerplate.am
@@ -12,10 +12,10 @@ vmodtoolargs ?= --strict --boilerplate -o vcc_std_if
 
 vmod_LTLIBRARIES = libvmod_std.la
 
-libvmod_std_la_CFLAGS = \
+libvmod_std_la_CFLAGS ?= \
 	@SAN_CFLAGS@
 
-libvmod_std_la_LDFLAGS = \
+libvmod_std_la_LDFLAGS ?= \
 	-export-symbols-regex 'Vmod_std_Data' \
 	$(AM_LDFLAGS) \
 	$(VMOD_LDFLAGS) \
diff --git a/lib/libvmod_unix/automake_boilerplate.am b/lib/libvmod_unix/automake_boilerplate.am
index 913bc01c5..f053524a1 100644
--- a/lib/libvmod_unix/automake_boilerplate.am
+++ b/lib/libvmod_unix/automake_boilerplate.am
@@ -12,10 +12,10 @@ vmodtoolargs ?= --strict --boilerplate -o vcc_unix_if
 
 vmod_LTLIBRARIES = libvmod_unix.la
 
-libvmod_unix_la_CFLAGS = \
+libvmod_unix_la_CFLAGS ?= \
 	@SAN_CFLAGS@
 
-libvmod_unix_la_LDFLAGS = \
+libvmod_unix_la_LDFLAGS ?= \
 	-export-symbols-regex 'Vmod_unix_Data' \
 	$(AM_LDFLAGS) \
 	$(VMOD_LDFLAGS) \
diff --git a/lib/libvmod_vtc/automake_boilerplate.am b/lib/libvmod_vtc/automake_boilerplate.am
index 1099900f9..1d4d60cb9 100644
--- a/lib/libvmod_vtc/automake_boilerplate.am
+++ b/lib/libvmod_vtc/automake_boilerplate.am
@@ -12,10 +12,10 @@ vmodtoolargs ?= --strict --boilerplate -o vcc_vtc_if
 
 vmod_LTLIBRARIES = libvmod_vtc.la
 
-libvmod_vtc_la_CFLAGS = \
+libvmod_vtc_la_CFLAGS ?= \
 	@SAN_CFLAGS@
 
-libvmod_vtc_la_LDFLAGS = \
+libvmod_vtc_la_LDFLAGS ?= \
 	-export-symbols-regex 'Vmod_vtc_Data' \
 	$(AM_LDFLAGS) \
 	$(VMOD_LDFLAGS) \