From guillaume at varnish-software.com  Thu Apr  1 01:08:09 2021
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Thu,  1 Apr 2021 01:08:09 +0000 (UTC)
Subject: [master] b46a77b76 introduce bob
Message-ID: <20210401010809.20290AD29B@lists.varnish-cache.org>


commit b46a77b7624e28c4837d716432c9b66190a28add
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Wed Mar 31 17:50:32 2021 -0700

    introduce bob

diff --git a/.circleci/Dockerfile b/.circleci/Dockerfile
new file mode 100644
index 000000000..7553611e0
--- /dev/null
+++ b/.circleci/Dockerfile
@@ -0,0 +1,15 @@
+FROM centos:7
+
+RUN set -e;\
+	yum install -y epel-release; \
+	yum install -y \
+	    automake \
+	    git \
+	    jemalloc-devel \
+	    libedit-devel \
+	    libtool \
+	    libunwind-devel \
+	    make \
+	    pcre-devel \
+	    python3 \
+	    python-sphinx
diff --git a/.circleci/Dockerfile.alpine b/.circleci/Dockerfile.alpine
new file mode 100644
index 000000000..c09aa9d5a
--- /dev/null
+++ b/.circleci/Dockerfile.alpine
@@ -0,0 +1,20 @@
+FROM alpine
+
+RUN set -e; \
+	apk update; \
+	apk add -q \
+	    autoconf \
+	    automake \
+	    build-base \
+	    ca-certificates \
+	    cpio \
+	    git \
+	    gzip \
+	    libedit-dev \
+	    libtool \
+	    libunwind-dev \
+	    linux-headers \
+	    pcre-dev \
+	    py-docutils \
+	    py3-sphinx \
+	    tar
diff --git a/.circleci/Dockerfile.archlinux b/.circleci/Dockerfile.archlinux
new file mode 100644
index 000000000..1e6344354
--- /dev/null
+++ b/.circleci/Dockerfile.archlinux
@@ -0,0 +1,16 @@
+FROM archlinux
+
+RUN set -e; \
+	pacman -Sy --noconfirm \
+	       base-devel \
+	       ca-certificates \
+	       cpio \
+	       git \
+	       libedit \
+	       libtool \
+	       libunwind \
+	       linux-headers \
+	       pcre \
+	       python-docutils \
+	       python-sphinx \
+	       tar
diff --git a/.circleci/Dockerfile.ubuntu b/.circleci/Dockerfile.ubuntu
new file mode 100644
index 000000000..cc50bd3a0
--- /dev/null
+++ b/.circleci/Dockerfile.ubuntu
@@ -0,0 +1,22 @@
+FROM ubuntu
+
+RUN set -e; \
+	export DEBIAN_FRONTEND=noninteractive; \
+	export DEBCONF_NONINTERACTIVE_SEEN=true; \
+	apt-get update; \
+	apt-get install -y \
+	    autoconf \
+	    automake \
+	    build-essential \
+	    ca-certificates \
+	    cpio \
+	    git \
+	    graphviz \
+	    libedit-dev \
+	    libjemalloc-dev \
+	    libncurses-dev \
+	    libpcre3-dev \
+	    libtool \
+	    libunwind-dev \
+	    pkg-config \
+	    python3-sphinx

From guillaume at varnish-software.com  Fri Apr  2 15:13:05 2021
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Fri,  2 Apr 2021 15:13:05 +0000 (UTC)
Subject: [master] 05780b634 disable -Wcast-qual because of musl
Message-ID: <20210402151305.62CC7A0DA1@lists.varnish-cache.org>


commit 05780b634c40233dd2494a2c76a8dd8ba72d79f8
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Fri Apr 2 08:11:27 2021 -0700

    disable -Wcast-qual because of musl

diff --git a/wflags.py b/wflags.py
index f5dedf7e6..3c5e6d093 100644
--- a/wflags.py
+++ b/wflags.py
@@ -44,7 +44,6 @@ DESIRABLE_OPTIONS = [
 
 DESIRABLE_WFLAGS = [
     "-Wcast-align",
-    "-Wcast-qual",
     "-Wchar-subscripts",
     "-Wempty-body",
     "-Wextra",
@@ -69,6 +68,7 @@ DESIRABLE_WFLAGS = [
 ]
 
 UNDESIRABLE_WFLAGS = [
+    "-Wno-cast-qual", # GCC complains about musl::sched.h
     "-Wno-thread-safety", # Does not understand our mutexs are wrapped
     "-Wno-old-style-definition", # Does not like vgz
     "-Wno-sign-compare", # Fixable

From phk at FreeBSD.org  Tue Apr  6 14:02:04 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  6 Apr 2021 14:02:04 +0000 (UTC)
Subject: [master] 78ccf7bd9 Decrease sleep time when waiting for child to die.
Message-ID: <20210406140204.F105DA5268@lists.varnish-cache.org>


commit 78ccf7bd9c62ef434bc0b11011ae6f63ed2ccf62
Author: Simon <simon at devsn.se>
Date:   Tue Apr 6 13:13:28 2021 +0000

    Decrease sleep time when waiting for child to die.

diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index e55e6ca5d..84f3c182e 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -519,11 +519,11 @@ mgt_reap_child(void)
 	XXXAN(vsb);
 
 	/* Wait for child to die */
-	for (i = 0; i < mgt_param.cli_timeout; i++) {
+	for (i = 0; i < mgt_param.cli_timeout * 10; i++) {
 		r = waitpid(child_pid, &status, WNOHANG);
 		if (r == child_pid)
 			break;
-		(void)sleep(1);
+		(void)usleep(100000);
 	}
 	if (r == 0) {
 		VSB_printf(vsb, "Child (%jd) not dying, killing", (intmax_t)r);

From dridi.boukelmoune at gmail.com  Tue Apr  6 17:26:04 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue,  6 Apr 2021 17:26:04 +0000 (UTC)
Subject: [master] 606977bbf doc: Fix the cookie.keep_re() example
Message-ID: <20210406172604.B3741ABF5B@lists.varnish-cache.org>


commit 606977bbfb624ead38e9c8648beac0b3906a4294
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Apr 6 19:23:28 2021 +0200

    doc: Fix the cookie.keep_re() example
    
    Reported by Justin Lloyd.

diff --git a/vmod/vmod_cookie.vcc b/vmod/vmod_cookie.vcc
index ca13d0c0e..85e6b8519 100644
--- a/vmod/vmod_cookie.vcc
+++ b/vmod/vmod_cookie.vcc
@@ -124,7 +124,7 @@ Example::
 
 	sub vcl_recv {
 	    cookie.parse("cookie1: value1; cookie2: value2; cookie3: value3");
-	    cookie.keep_re("^cookie1,cookie2");
+	    cookie.keep_re("^cookie[12]$");
 	    # get_string() will now yield
 	    # "cookie1: value1; cookie2: value2;";
 	}

From nils.goroll at uplex.de  Sat Apr 10 12:34:06 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Sat, 10 Apr 2021 12:34:06 +0000 (UTC)
Subject: [master] cf9548efe vcc: Track the built-in subs which static calls
 originate from
Message-ID: <20210410123406.88F0DAA1BB@lists.varnish-cache.org>


commit cf9548efe2770e1597c49903f065572aaad96fd6
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Sat Apr 10 14:18:22 2021 +0200

    vcc: Track the built-in subs which static calls originate from
    
    We use another "method" bitmask to record the built-in subs from which
    static calls originate.
    
    This allows us to indentify those subs which are called from
    housekeeping only, allowing us to selectively disable compiler
    optimizations.
    
    VGC diff for the example from
    https://github.com/varnishcache/varnish-cache/issues/3545#issue-824634168
    
    ```diff
    @@ -2242,7 +2278,7 @@
    
     #define END_ if (*ctx->handling) return
    
    -void v_matchproto_(vcl_func_f)
    +void v_dont_optimize v_matchproto_(vcl_func_f)
     VGC_function_a(VRT_CTX, enum vcl_func_call_e call,
         enum vcl_func_fail_e *failp)
     {
    ```
    
    Fixes #3545

diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 906567977..88c8baf5b 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -191,7 +191,9 @@ vcc_EmitProc(struct vcc *tl, struct proc *p)
 		maskcmp = "&";
 	}
 
-	if ((mask & VCL_MET_TASK_H) && (mask & ~VCL_MET_TASK_H) == 0)
+	if (dyn == 0 &&
+	    (p->calledfrom &  VCL_MET_TASK_H) != 0 &&
+	    (p->calledfrom & ~VCL_MET_TASK_H) == 0)
 		cc_adv = "v_dont_optimize ";
 	else
 		cc_adv = "";
@@ -210,6 +212,7 @@ vcc_EmitProc(struct vcc *tl, struct proc *p)
 	Fh(tl, 1, "\t.n\t\t= %d,\n", nsub);
 	Fh(tl, 1, "\t.nref\t\t= %d,\n", p->sym->nref);
 	Fh(tl, 1, "\t.called\t\t= %d\n", p->called);
+	Fh(tl, 1, "\t// calledfrom\t  0x%x\n", p->calledfrom);
 	Fh(tl, 1, "}};\n");
 
 	if (dyn) {
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index b52c53216..1ec423b30 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -213,6 +213,7 @@ struct proc {
 	unsigned		called;
 	unsigned		active;
 	unsigned		okmask;
+	unsigned		calledfrom;
 	struct token		*return_tok[VCL_RET_MAX];
 	struct vsb		*cname;
 	struct vsb		*prologue;
diff --git a/lib/libvcc/vcc_xref.c b/lib/libvcc/vcc_xref.c
index e3351ec48..433f5e7b8 100644
--- a/lib/libvcc/vcc_xref.c
+++ b/lib/libvcc/vcc_xref.c
@@ -207,6 +207,7 @@ vcc_CheckActionRecurse(struct vcc *tl, struct proc *p, unsigned bitmap)
 			vcc_ErrWhere(tl, pc->t);
 			return (1);
 		}
+		pc->sym->proc->calledfrom |= p->calledfrom;
 		pc->sym->proc->called++;
 		pc->sym->nref += u;
 		if (vcc_CheckActionRecurse(tl, pc->sym->proc, bitmap)) {
@@ -233,10 +234,12 @@ vcc_checkaction(struct vcc *tl, const struct symbol *sym)
 	AN(p);
 	AN(p->name);
 
-	if (p->method == NULL)
+	if (p->method == NULL) {
 		bitmap = ~0U;
-	else
+	} else {
 		bitmap = p->method->ret_bitmap;
+		p->calledfrom = p->method->bitval;
+	}
 
 	if (! vcc_CheckActionRecurse(tl, p, bitmap))
 		return;

From nils.goroll at uplex.de  Sat Apr 10 15:18:06 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Sat, 10 Apr 2021 15:18:06 +0000 (UTC)
Subject: [master] 7c3db3fe0 varnishadm: use the -t argument as the timeout
Message-ID: <20210410151806.ABEF4AE87C@lists.varnish-cache.org>


commit 7c3db3fe0fb594dcb22db12a8e16dbbf397e52a6
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Sat Apr 10 17:09:21 2021 +0200

    varnishadm: use the -t argument as the timeout
    
    We used the -t argument only for the VSM attach, not for the actual CLI
    operations.
    
    I do not think the complication of differentiated timeouts is justified,
    so just use the one timeout parameter we have for both. Also I think
    that the documentation is already adequate in its simplicity:
    
           -t timeout
                  Wait no longer than this many seconds for an operation to
                  finish.
    
    Fixes #3542

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index 08c4a4125..ebc1aa2cc 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -74,7 +74,7 @@
 	} while (0)
 
 
-static const double timeout = 5;	// XXX should be settable by arg ?
+static double timeout = 5;
 static int p_arg = 0;
 
 static void
@@ -423,6 +423,21 @@ n_arg_sock(const char *n_arg, const char *t_arg)
 	return (sock);
 }
 
+static int
+t_arg_timeout(const char *t_arg)
+{
+	char *p = NULL;
+
+	AN(t_arg);
+	timeout = strtod(t_arg, &p);
+	if ((p != NULL && *p != '\0') ||
+	    !isfinite(timeout) || timeout < 0) {
+		fprintf(stderr, "-t: Invalid argument: %s", t_arg);
+		return (-1);
+	}
+	return (1);
+}
+
 #define OPTARG "hn:pS:T:t:"
 
 int
@@ -486,6 +501,9 @@ main(int argc, char * const *argv)
 	if (sock < 0)
 		exit(2);
 
+	if (t_arg != NULL && t_arg_timeout(t_arg) < 0)
+		exit(2);
+
 	if (argc > 0) {
 		VSIG_Arm_int();
 		VSIG_Arm_term();

From phk at FreeBSD.org  Sun Apr 11 09:38:09 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sun, 11 Apr 2021 09:38:09 +0000 (UTC)
Subject: [master] cb36d6aac Add -Wno-nullability-completeness which barfs all
 over MacOsX
Message-ID: <20210411093809.5AA4AA5AE5@lists.varnish-cache.org>


commit cb36d6aac1487eac4610456354b2ebf2ac0100aa
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Apr 11 09:36:40 2021 +0000

    Add -Wno-nullability-completeness which barfs all over MacOsX

diff --git a/wflags.py b/wflags.py
index 3c5e6d093..7e4d30fed 100644
--- a/wflags.py
+++ b/wflags.py
@@ -78,6 +78,7 @@ UNDESIRABLE_WFLAGS = [
     "-Wno-redundant-decls", # Complains about centos::stdio.h
     "-Wno-missing-variable-declarations", # Complains about optreset
     "-Wno-parentheses", # GCC complains about musl::endian.h
+    "-Wno-nullability-completeness", # Barfs all over MacOSx
 ]
 
 def main():

From phk at FreeBSD.org  Sun Apr 11 10:16:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sun, 11 Apr 2021 10:16:05 +0000 (UTC)
Subject: [master] a3a5d2f62 Cast %ju argument to uintmax_t
Message-ID: <20210411101605.0D35EA6D06@lists.varnish-cache.org>


commit a3a5d2f62d08dad4e56497e40c2d7c9fbe087e07
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Apr 11 10:11:44 2021 +0000

    Cast %ju argument to uintmax_t
    
    Closes: #3570

diff --git a/vmod/vmod_debug_acl.c b/vmod/vmod_debug_acl.c
index bff2a0ce2..7de562743 100644
--- a/vmod/vmod_debug_acl.c
+++ b/vmod/vmod_debug_acl.c
@@ -245,7 +245,8 @@ xyzzy_time_acl(VRT_CTX, VCL_ACL acl, VCL_IP ip0, VCL_IP ip1,
 	}
 	t1 = VTIM_mono();
 	VSLb(ctx->vsl, SLT_Debug,
-	    "Timed ACL: %.9f -> %.9f = %.9f %.9f/round, %.9f/IP %jd IPs",
-	    t0, t1, t1 - t0, (t1-t0) / turnus, (t1-t0) / asw->count, asw->count);
+	    "Timed ACL: %.9f -> %.9f = %.9f %.9f/round, %.9f/IP %ju IPs",
+	    t0, t1, t1 - t0, (t1-t0) / turnus, (t1-t0) / asw->count,
+	    (uintmax_t)asw->count);
 	return ((t1 - t0) / asw->count);
 }

From dridi.boukelmoune at gmail.com  Mon Apr 12 07:22:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 12 Apr 2021 07:22:06 +0000 (UTC)
Subject: [master] c9b469c5b vcc: Simplify v_dont_optimize condition
Message-ID: <20210412072206.CBD9FA2B5C@lists.varnish-cache.org>


commit c9b469c5b5f9425ce2927855a0f6710167f62c3d
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Apr 12 09:16:46 2021 +0200

    vcc: Simplify v_dont_optimize condition

diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 88c8baf5b..8991e9a30 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -191,9 +191,7 @@ vcc_EmitProc(struct vcc *tl, struct proc *p)
 		maskcmp = "&";
 	}
 
-	if (dyn == 0 &&
-	    (p->calledfrom &  VCL_MET_TASK_H) != 0 &&
-	    (p->calledfrom & ~VCL_MET_TASK_H) == 0)
+	if (dyn == 0 && (p->calledfrom & VCL_MET_TASK_H) == p->calledfrom)
 		cc_adv = "v_dont_optimize ";
 	else
 		cc_adv = "";

From phk at FreeBSD.org  Mon Apr 12 07:48:04 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 12 Apr 2021 07:48:04 +0000 (UTC)
Subject: [master] 108c65d37 Dont attempt to re-release the VCL
Message-ID: <20210412074804.9E294A3927@lists.varnish-cache.org>


commit 108c65d37d1788b3e0f6ccc3e8837a7410ca39bb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 12 07:42:53 2021 +0000

    Dont attempt to re-release the VCL
    
    Fixes #3571

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 090ada6c3..9b5d59c82 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -433,7 +433,7 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk)
 				else
 					tmo =  wrk->lastused+60.;
 				i = Lck_CondWait(&wrk->cond, &pp->mtx, tmo);
-				if (i == ETIMEDOUT)
+				if (i == ETIMEDOUT && wrk->vcl != NULL)
 					VCL_Rel(&wrk->vcl);
 				if (wrk->task->func != NULL) {
 					/* We have been handed a new task */

From dridi at varni.sh  Mon Apr 12 08:15:27 2021
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Mon, 12 Apr 2021 08:15:27 +0000
Subject: [master] 7c3db3fe0 varnishadm: use the -t argument as the timeout
In-Reply-To: <20210410151806.ABEF4AE87C@lists.varnish-cache.org>
References: <20210410151806.ABEF4AE87C@lists.varnish-cache.org>
Message-ID: <CABoVN9Cfv0=dbQuZLA9wT3fExvbHT9JmE+1G0xvs58XWXi-qXA@mail.gmail.com>

On Sat, Apr 10, 2021 at 3:18 PM Nils Goroll <nils.goroll at uplex.de> wrote:
>
>
> commit 7c3db3fe0fb594dcb22db12a8e16dbbf397e52a6
> Author: Nils Goroll <nils.goroll at uplex.de>
> Date:   Sat Apr 10 17:09:21 2021 +0200
>
>     varnishadm: use the -t argument as the timeout
>
>     We used the -t argument only for the VSM attach, not for the actual CLI
>     operations.
>
>     I do not think the complication of differentiated timeouts is justified,

Agreed.

>     so just use the one timeout parameter we have for both. Also I think
>     that the documentation is already adequate in its simplicity:
>
>            -t timeout
>                   Wait no longer than this many seconds for an operation to
>                   finish.

We should however make the default apparent.

>     Fixes #3542
>
> diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
> index 08c4a4125..ebc1aa2cc 100644
> --- a/bin/varnishadm/varnishadm.c
> +++ b/bin/varnishadm/varnishadm.c
> @@ -74,7 +74,7 @@
>         } while (0)
>
>
> -static const double timeout = 5;       // XXX should be settable by arg ?
> +static double timeout = 5;

Should we increase the default timeout to match varnishd?s default
cli_timeout instead of libvarnishapi?s t VSM argument?

>  static int p_arg = 0;
>
>  static void
> @@ -423,6 +423,21 @@ n_arg_sock(const char *n_arg, const char *t_arg)
>         return (sock);
>  }
>
> +static int
> +t_arg_timeout(const char *t_arg)
> +{
> +       char *p = NULL;
> +
> +       AN(t_arg);
> +       timeout = strtod(t_arg, &p);
> +       if ((p != NULL && *p != '\0') ||
> +           !isfinite(timeout) || timeout < 0) {
> +               fprintf(stderr, "-t: Invalid argument: %s", t_arg);
> +               return (-1);
> +       }
> +       return (1);
> +}
> +
>  #define OPTARG "hn:pS:T:t:"
>
>  int
> @@ -486,6 +501,9 @@ main(int argc, char * const *argv)
>         if (sock < 0)
>                 exit(2);
>
> +       if (t_arg != NULL && t_arg_timeout(t_arg) < 0)
> +               exit(2);
> +
>         if (argc > 0) {
>                 VSIG_Arm_int();
>                 VSIG_Arm_term();
> _______________________________________________
> varnish-commit mailing list
> varnish-commit at varnish-cache.org
> https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit

From phk at FreeBSD.org  Mon Apr 12 09:50:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 12 Apr 2021 09:50:05 +0000 (UTC)
Subject: [master] ff20fdbb0 Add a "struct acl" to hold info about ACL's while
 we compile them.
Message-ID: <20210412095005.C57ECA76D0@lists.varnish-cache.org>


commit ff20fdbb0e05d8d50c6697c3b568ea2d2de1efca
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 12 09:06:42 2021 +0000

    Add a "struct acl" to hold info about ACL's while we compile them.

diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c
index 82cd31985..a89d0f022 100644
--- a/lib/libvcc/vcc_acl.c
+++ b/lib/libvcc/vcc_acl.c
@@ -41,10 +41,20 @@
 
 #include "vcc_compile.h"
 #include <vtcp.h>
+#include <vtree.h>
 #include <vsa.h>
 
 #define ACL_MAXADDR	(sizeof(struct in6_addr) + 1)
 
+VRBT_HEAD(acl_tree, acl_e);
+
+struct acl {
+	unsigned		magic;
+#define VCC_ACL_MAGIC		0xb9fb3cd0
+
+	struct acl_tree		acl_tree;
+};
+
 struct acl_e {
 	unsigned		magic;
 #define VCC_ACL_E_MAGIC	0xcac81e23
@@ -175,7 +185,7 @@ vcc_acl_insert_entry(struct vcc *tl, struct acl_e **aenp)
 	struct acl_e *ae2;
 
 	CHECK_OBJ_NOTNULL(*aenp, VCC_ACL_E_MAGIC);
-	ae2 = VRBT_INSERT(acl_tree, &tl->acl_tree, *aenp);
+	ae2 = VRBT_INSERT(acl_tree, &tl->acl->acl_tree, *aenp);
 	if (ae2 != NULL) {
 		if (ae2->not != (*aenp)->not) {
 			VSB_cat(tl->sb, "Conflicting ACL entries:\n");
@@ -461,7 +471,7 @@ vcc_acl_emit_tokens(const struct vcc *tl, const struct acl_e *ae)
  */
 
 static void
-vcc_acl_emit(struct vcc *tl, const char *name, const char *rname)
+vcc_acl_emit(struct vcc *tl, const struct symbol *sym)
 {
 	struct acl_e *ae;
 	int depth, l, m, i;
@@ -472,7 +482,7 @@ vcc_acl_emit(struct vcc *tl, const char *name, const char *rname)
 	func = VSB_new_auto();
 	AN(func);
 	VSB_printf(func, "match_acl_");
-	VCC_PrintCName(func, name, NULL);
+	VCC_PrintCName(func, sym->name, NULL);
 	AZ(VSB_finish(func));
 
 	Fh(tl, 0, "\nstatic int v_matchproto_(acl_match_f)\n");
@@ -483,7 +493,7 @@ vcc_acl_emit(struct vcc *tl, const char *name, const char *rname)
 	Fh(tl, 0, "\n");
 	Fh(tl, 0, "\tfam = VRT_VSA_GetPtr(ctx, p, &a);\n");
 	Fh(tl, 0, "\tif (fam < 0) {\n");
-	Fh(tl, 0, "\t\tVPI_acl_log(ctx, \"NO_FAM %s\");\n", name);
+	Fh(tl, 0, "\t\tVPI_acl_log(ctx, \"NO_FAM %s\");\n", sym->name);
 	Fh(tl, 0, "\t\treturn(0);\n");
 	Fh(tl, 0, "\t}\n\n");
 	if (!tl->err_unref) {
@@ -493,7 +503,7 @@ vcc_acl_emit(struct vcc *tl, const char *name, const char *rname)
 	}
 	depth = -1;
 	at[0] = 256;
-	VRBT_FOREACH(ae, acl_tree, &tl->acl_tree) {
+	VRBT_FOREACH(ae, acl_tree, &tl->acl->acl_tree) {
 
 		/* Find how much common prefix we have */
 		for (l = 0; l <= depth && l * 8 < (int)ae->mask - 7; l++) {
@@ -537,7 +547,7 @@ vcc_acl_emit(struct vcc *tl, const char *name, const char *rname)
 		i = ((int)ae->mask + 7) / 8;
 
 		Fh(tl, 0, "\t%*sVPI_acl_log(ctx, \"%sMATCH %s \" ",
-		    -i, "", ae->not ? "NEG_" : "", name);
+		    -i, "", ae->not ? "NEG_" : "", sym->name);
 		vcc_acl_emit_tokens(tl, ae);
 		Fh(tl, 0, ");\n");
 
@@ -549,18 +559,18 @@ vcc_acl_emit(struct vcc *tl, const char *name, const char *rname)
 		Fh(tl, 0, "\t%*.*s}\n", depth, depth, "");
 
 	/* Deny by default */
-	Fh(tl, 0, "\tVPI_acl_log(ctx, \"NO_MATCH %s\");\n", name);
+	Fh(tl, 0, "\tVPI_acl_log(ctx, \"NO_MATCH %s\");\n", sym->name);
 	Fh(tl, 0, "\treturn (0);\n}\n");
 
 	/* Emit the struct that will be referenced */
-	Fh(tl, 0, "\nstatic const struct vrt_acl %s[] = {{\n", rname);
+	Fh(tl, 0, "\nstatic const struct vrt_acl %s[] = {{\n", sym->rname);
 	Fh(tl, 0, "\t.magic = VRT_ACL_MAGIC,\n");
 	Fh(tl, 0, "\t.match = &%s,\n", VSB_data(func));
-	Fh(tl, 0, "\t.name = \"%s\",\n", name);
+	Fh(tl, 0, "\t.name = \"%s\",\n", sym->name);
 	Fh(tl, 0, "}};\n\n");
 	if (!tl->err_unref) {
 		AN(ifp);
-		VSB_printf(ifp->ini, "\t(void)%s;\n", rname);
+		VSB_printf(ifp->ini, "\t(void)%s;\n", sym->rname);
 	}
 	VSB_destroy(&func);
 }
@@ -569,9 +579,12 @@ void
 vcc_ParseAcl(struct vcc *tl)
 {
 	struct symbol *sym;
+	struct acl acl[1];
 
+	INIT_OBJ(acl, VCC_ACL_MAGIC);
+	tl->acl = acl;
 	vcc_NextToken(tl);
-	VRBT_INIT(&tl->acl_tree);
+	VRBT_INIT(&acl->acl_tree);
 
 	vcc_ExpectVid(tl, "ACL");
 	ERRCHK(tl);
@@ -588,5 +601,5 @@ vcc_ParseAcl(struct vcc *tl)
 	}
 	SkipToken(tl, '}');
 
-	vcc_acl_emit(tl, sym->name, sym->rname);
+	vcc_acl_emit(tl, sym);
 }
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 1ec423b30..231a2cd15 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -39,7 +39,6 @@
 #include "vrt.h"
 #include "vcl.h"
 #include "vqueue.h"
-#include "vtree.h"
 #include "vsb.h"
 
 #include "vcc_token_defs.h"
@@ -57,6 +56,7 @@
 
 /*---------------------------------------------------------------------*/
 
+struct acl;
 struct vsb;
 struct token;
 struct sockaddr_storage;
@@ -235,8 +235,6 @@ struct inifin {
 
 VTAILQ_HEAD(inifinhead, inifin);
 
-VRBT_HEAD(acl_tree, acl_e);
-
 struct vcc {
 	unsigned		magic;
 #define VCC_MAGIC		0x24ad719d
@@ -275,7 +273,7 @@ struct vcc {
 	struct proc		*curproc;
 	VTAILQ_HEAD(, proc)	procs;
 
-	struct acl_tree		acl_tree;
+	struct acl		*acl;
 
 	int			nprobe;
 

From phk at FreeBSD.org  Mon Apr 12 09:50:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 12 Apr 2021 09:50:05 +0000 (UTC)
Subject: [master] 1e45d97c3 Introduce 'acl <name> +log {...}' flag and disable
 VSL logging by default.
Message-ID: <20210412095005.DDE8AA76D3@lists.varnish-cache.org>


commit 1e45d97c32b80232d84f6b2dec9c5e275a94c3f8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 12 09:48:09 2021 +0000

    Introduce 'acl <name> +log {...}' flag and disable VSL logging by default.

diff --git a/bin/varnishtest/tests/c00005.vtc b/bin/varnishtest/tests/c00005.vtc
index 448d6f80c..6e8feaaa8 100644
--- a/bin/varnishtest/tests/c00005.vtc
+++ b/bin/varnishtest/tests/c00005.vtc
@@ -72,7 +72,7 @@ varnish v1 -vcl {
 
 	backend dummy None;
 
-	acl acl1 {
+	acl acl1 +log {
 		# bad notation (confusing)
 		"1.2.3.4"/24;
 		"1.2.3.66"/26;
diff --git a/bin/varnishtest/tests/c00087.vtc b/bin/varnishtest/tests/c00087.vtc
index d84862860..832f63993 100644
--- a/bin/varnishtest/tests/c00087.vtc
+++ b/bin/varnishtest/tests/c00087.vtc
@@ -6,7 +6,7 @@ server s1 -listen "${tmpdir}/s1.sock" {
 } -start
 
 varnish v1 -syntax 4.1 -arg "-a foo=${tmpdir}/v1.sock" -vcl+backend {
-	acl acl1 {
+	acl acl1 +log {
 		"${localhost}";
 	}
 
diff --git a/bin/varnishtest/tests/v00017.vtc b/bin/varnishtest/tests/v00017.vtc
index 6e35d0c23..17f94582d 100644
--- a/bin/varnishtest/tests/v00017.vtc
+++ b/bin/varnishtest/tests/v00017.vtc
@@ -100,3 +100,19 @@ varnish v1 -errvcl {/mask only allowed once} {
 	}
 	sub vcl_recv { if (client.ip ~ a) { return(pass); } }
 }
+
+varnish v1 -errvcl {Expected ACL flag after:} {
+	backend b { .host = "${localhost}"; }
+	acl a + foobar {
+		"10.0.1.0/22" / 22;
+	}
+	sub vcl_recv { if (client.ip ~ a) { return(pass); } }
+}
+
+varnish v1 -errvcl {Unknown ACL flag:} {
+	backend b { .host = "${localhost}"; }
+	acl a +foobar {
+		"10.0.1.0/22" / 22;
+	}
+	sub vcl_recv { if (client.ip ~ a) { return(pass); } }
+}
diff --git a/doc/changes.rst b/doc/changes.rst
index b63db87c8..1fe26459f 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -31,6 +31,13 @@ http://varnish-cache.org/docs/trunk/whats-new/index.html and via
 individual releases. These documents are updated as part of the
 release process.
 
+================================
+Varnish Cache 7.x.x (2021-09-15)
+================================
+
+* ACLs no longer produce VSL `VCL_acl` records by default, this must be
+  explicitly enabled with `vcl <name> +log { ... }`.
+
 ================================
 Varnish Cache 6.6.0 (2021-03-15)
 ================================
diff --git a/doc/sphinx/users-guide/vcl-syntax.rst b/doc/sphinx/users-guide/vcl-syntax.rst
index 2cc6fc6a5..7ef4dd7e7 100644
--- a/doc/sphinx/users-guide/vcl-syntax.rst
+++ b/doc/sphinx/users-guide/vcl-syntax.rst
@@ -59,6 +59,16 @@ To match an IP address against an ACL, simply use the match operator::
          return (pipe);
        }
 
+In Varnish versions before 7.0, ACLs would always emit a `VCL_acl`
+record in the VSL log, from 7.0 and forward, this must be explicitly
+enabled by specifying the `+log` flag::
+
+       acl local +log {
+         "localhost";         // myself
+         "192.0.2.0"/24;      // and everyone on the local network
+         ! "192.0.2.23";      // except for the dialin router
+       }
+
 Operators
 ~~~~~~~~~
 
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index c98646d26..94ed41409 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -287,7 +287,7 @@ SLTM(Fetch_Body, 0, "Body fetched from backend",
 )
 
 SLTM(VCL_acl, 0, "VCL ACL check results",
-	"Logs VCL ACL evaluation results.\n\n"
+	"ACLs with the `+log` flag emits this record with the result.\n\n"
 	"The format is::\n\n"
 	"\t%s [%s [%s [fixed: %s]]]\n"
 	"\t|   |   |          |\n"
diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c
index a89d0f022..08f8a135f 100644
--- a/lib/libvcc/vcc_acl.c
+++ b/lib/libvcc/vcc_acl.c
@@ -52,6 +52,8 @@ struct acl {
 	unsigned		magic;
 #define VCC_ACL_MAGIC		0xb9fb3cd0
 
+	int			flag_log;
+
 	struct acl_tree		acl_tree;
 };
 
@@ -493,7 +495,8 @@ vcc_acl_emit(struct vcc *tl, const struct symbol *sym)
 	Fh(tl, 0, "\n");
 	Fh(tl, 0, "\tfam = VRT_VSA_GetPtr(ctx, p, &a);\n");
 	Fh(tl, 0, "\tif (fam < 0) {\n");
-	Fh(tl, 0, "\t\tVPI_acl_log(ctx, \"NO_FAM %s\");\n", sym->name);
+	if (tl->acl->flag_log)
+		Fh(tl, 0, "\t\tVPI_acl_log(ctx, \"NO_FAM %s\");\n", sym->name);
 	Fh(tl, 0, "\t\treturn(0);\n");
 	Fh(tl, 0, "\t}\n\n");
 	if (!tl->err_unref) {
@@ -546,10 +549,12 @@ vcc_acl_emit(struct vcc *tl, const struct symbol *sym)
 
 		i = ((int)ae->mask + 7) / 8;
 
-		Fh(tl, 0, "\t%*sVPI_acl_log(ctx, \"%sMATCH %s \" ",
-		    -i, "", ae->not ? "NEG_" : "", sym->name);
-		vcc_acl_emit_tokens(tl, ae);
-		Fh(tl, 0, ");\n");
+		if (tl->acl->flag_log) {
+			Fh(tl, 0, "\t%*sVPI_acl_log(ctx, \"%sMATCH %s \" ",
+			    -i, "", ae->not ? "NEG_" : "", sym->name);
+			vcc_acl_emit_tokens(tl, ae);
+			Fh(tl, 0, ");\n");
+		}
 
 		Fh(tl, 0, "\t%*sreturn (%d);\n", -i, "", ae->not ? 0 : 1);
 	}
@@ -559,7 +564,8 @@ vcc_acl_emit(struct vcc *tl, const struct symbol *sym)
 		Fh(tl, 0, "\t%*.*s}\n", depth, depth, "");
 
 	/* Deny by default */
-	Fh(tl, 0, "\tVPI_acl_log(ctx, \"NO_MATCH %s\");\n", sym->name);
+	if (tl->acl->flag_log)
+		Fh(tl, 0, "\tVPI_acl_log(ctx, \"NO_MATCH %s\");\n", sym->name);
 	Fh(tl, 0, "\treturn (0);\n}\n");
 
 	/* Emit the struct that will be referenced */
@@ -579,6 +585,7 @@ void
 vcc_ParseAcl(struct vcc *tl)
 {
 	struct symbol *sym;
+	struct token *sign;
 	struct acl acl[1];
 
 	INIT_OBJ(acl, VCC_ACL_MAGIC);
@@ -592,6 +599,24 @@ vcc_ParseAcl(struct vcc *tl)
 	ERRCHK(tl);
 	AN(sym);
 
+        while (tl->t->tok == '-' || tl->t->tok == '+') {
+		sign = tl->t;
+		vcc_NextToken(tl);
+		if (tl->t->b != sign->e) {
+			VSB_cat(tl->sb, "Expected ACL flag after:\n");
+			vcc_ErrWhere(tl, sign);
+			return;
+		}
+		if (vcc_IdIs(tl->t, "log")) {
+			acl->flag_log = sign->tok == '+';
+			vcc_NextToken(tl);
+		} else {
+			VSB_cat(tl->sb, "Unknown ACL flag:\n");
+			vcc_ErrWhere(tl, tl->t);
+			return;
+		}
+	}
+
 	SkipToken(tl, '{');
 
 	while (tl->t->tok != '}') {

From phk at FreeBSD.org  Mon Apr 12 12:09:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 12 Apr 2021 12:09:05 +0000 (UTC)
Subject: [master] d89674d12 Collapse vcc_acl_add_entry() and
 vcc_acl_insert_entry()
Message-ID: <20210412120905.4BD5BAB67B@lists.varnish-cache.org>


commit d89674d129ef0ed1c495e1169f83ec3203db1b42
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 12 10:49:30 2021 +0000

    Collapse vcc_acl_add_entry() and vcc_acl_insert_entry()

diff --git a/lib/libvcc/flint.lnt b/lib/libvcc/flint.lnt
index 22852628c..4e90fc671 100644
--- a/lib/libvcc/flint.lnt
+++ b/lib/libvcc/flint.lnt
@@ -4,6 +4,7 @@
 
 
 -sem(vcc_new_source, custodial(1))
+-sem(acl_tree_VRBT_INSERT, custodial(2))
 
 -emacro(835, EXPR_VAR)	// Info 835: A zero has been given as right argument to operator '<<'
 -esym(755, VCL_40)	// Not used (right now)
diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c
index 08f8a135f..6f2c2c923 100644
--- a/lib/libvcc/vcc_acl.c
+++ b/lib/libvcc/vcc_acl.c
@@ -181,30 +181,11 @@ vcc_acl_chk(struct vcc *tl, const struct acl_e *ae, const int l,
 	return (strdup(t));
 }
 
-static void
-vcc_acl_insert_entry(struct vcc *tl, struct acl_e **aenp)
-{
-	struct acl_e *ae2;
-
-	CHECK_OBJ_NOTNULL(*aenp, VCC_ACL_E_MAGIC);
-	ae2 = VRBT_INSERT(acl_tree, &tl->acl->acl_tree, *aenp);
-	if (ae2 != NULL) {
-		if (ae2->not != (*aenp)->not) {
-			VSB_cat(tl->sb, "Conflicting ACL entries:\n");
-			vcc_ErrWhere(tl, ae2->t_addr);
-			VSB_cat(tl->sb, "vs:\n");
-			vcc_ErrWhere(tl, (*aenp)->t_addr);
-		}
-		return;
-	}
-	*aenp = NULL;
-}
-
 static void
 vcc_acl_add_entry(struct vcc *tl, const struct acl_e *ae, int l,
     unsigned char *u, int fam)
 {
-	struct acl_e *aen;
+	struct acl_e *aen, *ae2;
 
 	if (fam == PF_INET && ae->mask > 32) {
 		VSB_printf(tl->sb,
@@ -239,9 +220,14 @@ vcc_acl_add_entry(struct vcc *tl, const struct acl_e *ae, int l,
 	assert(l + 1UL <= sizeof aen->data);
 	memcpy(aen->data + 1L, u, l);
 
-	vcc_acl_insert_entry(tl, &aen);
-	if (aen != NULL)
+	ae2 = VRBT_INSERT(acl_tree, &tl->acl->acl_tree, aen);
+	if (ae2 != NULL && ae2->not != aen->not) {
+		VSB_cat(tl->sb, "Conflicting ACL entries:\n");
+		vcc_ErrWhere(tl, ae2->t_addr);
+		VSB_cat(tl->sb, "vs:\n");
+		vcc_ErrWhere(tl, aen->t_addr);
 		vcl_acl_free(&aen);
+	}
 }
 
 static void
@@ -599,7 +585,7 @@ vcc_ParseAcl(struct vcc *tl)
 	ERRCHK(tl);
 	AN(sym);
 
-        while (tl->t->tok == '-' || tl->t->tok == '+') {
+	while (tl->t->tok == '-' || tl->t->tok == '+') {
 		sign = tl->t;
 		vcc_NextToken(tl);
 		if (tl->t->b != sign->e) {

From phk at FreeBSD.org  Mon Apr 12 12:09:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 12 Apr 2021 12:09:05 +0000 (UTC)
Subject: [master] cc885760c Add "+table" flag to ACL's
Message-ID: <20210412120905.6360AAB67E@lists.varnish-cache.org>


commit cc885760c1b2fbd6467084a3437a2c5327f6407d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 12 11:58:00 2021 +0000

    Add "+table" flag to ACL's
    
    The +table flag causes all overlapped ACL entries to be compiled
    as usual, and non-overlapped entries to be emitted as a table of
    bytes.
    
    When testing the ACL the compiled code is run before VPI_acl_table()
    is used to do a binary search on the table.
    
    Even with a binary search, the table is approx 3 times slower than
    the regular compiled ACLs (ie: only "blindingly fast" as oppposed
    to "lighting fast").
    
    The advantage of +table is that C-compilers literally take no time,
    no matter the size of the ACL, where they will take seconds
    or even minutes compiling large ACLs as code.

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 5cdaf2b3f..76b427418 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -138,6 +138,74 @@ VRT_acl_match(VRT_CTX, VCL_ACL acl, VCL_IP ip)
 	return (acl->match(ctx, ip));
 }
 
+static int
+acl_tbl_cmp(int fam, const uint8_t *key, const uint8_t *b)
+{
+	int rv;
+
+	rv = fam - (int)b[3];
+	if (rv == 0 && b[1] > 0)
+		rv = memcmp(key, b + 4, b[1]);
+	if (rv == 0 && b[2])
+		rv = (int)(key[b[1]] & b[2]) - (int)b[4 + b[1]];
+	return (rv);
+}
+
+static const uint8_t *
+bbsearch(int fam, const uint8_t *key, const uint8_t *base0,
+    size_t nmemb, size_t size)
+{
+        const uint8_t *base = base0;
+        size_t lim;
+        int cmp;
+        const uint8_t *p;
+
+        for (lim = nmemb; lim != 0; lim >>= 1) {
+                p = base + (lim >> 1) * size;
+                cmp = acl_tbl_cmp(fam, key, p);
+                if (cmp == 0)
+                        return (p);
+                if (cmp > 0) {
+			/* key > p: move right */
+                        base = p + size;
+                        lim--;
+                } /* else move left */
+        }
+        return (NULL);
+}
+
+int
+VPI_acl_table(VRT_CTX, VCL_IP p, unsigned n, unsigned m, const uint8_t *tbl,
+    const char * const *str, const char *fin)
+{
+	int fam;
+	const uint8_t *key;
+	const uint8_t *ptr;
+	size_t sz;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	AN(p);
+	AN(n);
+	assert(m == 20);
+	AN(tbl);
+	AN(fin);
+
+	fam = VRT_VSA_GetPtr(ctx, p, &key);
+	ptr = bbsearch(fam, key, tbl, n, m);
+
+	if (ptr != NULL) {
+		sz = ptr - tbl;
+		AZ(sz % m);
+		sz /= m;
+		if (str != NULL)
+			VPI_acl_log(ctx, str[sz]);
+		return *ptr;
+	}
+	if (str != NULL)
+		VPI_acl_log(ctx, fin);
+	return (0);
+}
+
 /*--------------------------------------------------------------------*/
 
 VCL_VOID
diff --git a/bin/varnishtest/tests/r01312.vtc b/bin/varnishtest/tests/r01312.vtc
index 583871d84..ded2787e2 100644
--- a/bin/varnishtest/tests/r01312.vtc
+++ b/bin/varnishtest/tests/r01312.vtc
@@ -1,24 +1,6 @@
 varnishtest "acl functional (& historic miscompile)"
 
-server s1 {
-	rxreq
-	txresp
-} -start
-
-varnish v1 -vcl+backend {
-	import std;
-	import debug;
-
-	acl foo {
-		"127.0.0.2";
-		"127.0.1"/19;
-	}
-	acl bar {
-		"127.0.1.2";
-		"127.0.1"/19;
-	}
-
-	acl block {
+shell {echo '
 		# Tests all boundary conditions
 		  "192.168.8.0" / 21;
 		! "192.168.16" / 21;
@@ -46,6 +28,29 @@ varnish v1 -vcl+backend {
 		! "::0100" / 124;
 		  "::0130" / 124;
 		  "::0170" / 124;
+' > ${tmpdir}/_acl.vcl 
+}
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import std;
+	import debug;
+
+	acl foo {
+		"127.0.0.2";
+		"127.0.1"/19;
+	}
+	acl bar {
+		"127.0.1.2";
+		"127.0.1"/19;
+	}
+
+	acl block {
+		include "${tmpdir}/_acl.vcl";
 	}
 
 	sub vcl_deliver {
@@ -100,3 +105,67 @@ client c1 {
 } -run
 
 logexpect l1 -wait
+
+varnish v1 -vcl+backend {
+	import std;
+	import debug;
+
+	acl block +table {
+		include "${tmpdir}/_acl.vcl";
+	}
+
+	sub vcl_deliver {
+		set resp.http.acl4 = debug.sweep_acl(
+		    block,
+		    std.ip("192.168.0.0"),
+		    std.ip("192.168.32.255"),
+		    256
+		);
+		set resp.http.acl6 = debug.sweep_acl(
+		    block,
+		    std.ip("::"),
+		    std.ip("::0200"),
+		    16
+		);
+	}
+}
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.http.acl4 == ":4thASR0O18ZxnoKtc4zd8KuO25rPvwvMQyAvRfilz6o=:"
+	expect resp.http.acl6 == ":NSi+7wpvQe7XJj8DPbESjpYPGnIzvjOsA5QCyCnW3kc=:"
+} -run
+
+varnish v1 -vcl+backend {
+	import std;
+	import debug;
+
+	acl block +log +table {
+		include "${tmpdir}/_acl.vcl";
+	}
+
+	sub vcl_deliver {
+		set resp.http.acl4 = debug.sweep_acl(
+		    block,
+		    std.ip("192.168.0.0"),
+		    std.ip("192.168.32.255"),
+		    256
+		);
+		set resp.http.acl6 = debug.sweep_acl(
+		    block,
+		    std.ip("::"),
+		    std.ip("::0200"),
+		    16
+		);
+	}
+}
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.http.acl4 == ":4thASR0O18ZxnoKtc4zd8KuO25rPvwvMQyAvRfilz6o=:"
+	expect resp.http.acl6 == ":NSi+7wpvQe7XJj8DPbESjpYPGnIzvjOsA5QCyCnW3kc=:"
+} -run
+
+shell {rm -f ${tmpdir}/_acl.vcl}
diff --git a/include/vcc_interface.h b/include/vcc_interface.h
index 7ad670ea3..8c81bddc0 100644
--- a/include/vcc_interface.h
+++ b/include/vcc_interface.h
@@ -61,6 +61,8 @@ int VPI_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len,
 void VPI_Vmod_Unload(VRT_CTX, struct vmod **hdl);
 
 typedef int acl_match_f(VRT_CTX, const VCL_IP);
+int VPI_acl_table(VRT_CTX, VCL_IP, unsigned n, unsigned m, const uint8_t *tbl,
+    const char * const *str, const char *fin);
 
 struct vrt_acl {
 	unsigned        magic;
diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c
index 6f2c2c923..226215fda 100644
--- a/lib/libvcc/vcc_acl.c
+++ b/lib/libvcc/vcc_acl.c
@@ -53,6 +53,7 @@ struct acl {
 #define VCC_ACL_MAGIC		0xb9fb3cd0
 
 	int			flag_log;
+	int			flag_table;
 
 	struct acl_tree		acl_tree;
 };
@@ -65,6 +66,7 @@ struct acl_e {
 	unsigned		mask;
 	unsigned		not;
 	unsigned		para;
+	unsigned		overlapped;
 	char			*addr;
 	const char		*fixed;
 	struct token		*t_addr;
@@ -121,10 +123,38 @@ vcl_acl_cmp(const struct acl_e *ae1, const struct acl_e *ae2)
 	return (0);
 }
 
+static int
+vcl_acl_disjoint(const struct acl_e *ae1, const struct acl_e *ae2)
+{
+	const unsigned char *p1, *p2;
+	unsigned m;
+
+	CHECK_OBJ_NOTNULL(ae1, VCC_ACL_E_MAGIC);
+	CHECK_OBJ_NOTNULL(ae2, VCC_ACL_E_MAGIC);
+
+	p1 = ae1->data;
+	p2 = ae2->data;
+	m = ae1->mask;
+	if (ae2->mask < m)
+		m = ae2->mask;
+	for (; m >= 8; m -= 8) {
+		CMP(*p1, *p2);
+		p1++;
+		p2++;
+	}
+	if (m) {
+		m = 0xff00 >> m;
+		m &= 0xff;
+		CMP(*p1 & m, *p2 & m);
+	}
+	return (0);
+}
+
 VRBT_GENERATE_INSERT_COLOR(acl_tree, acl_e, branch, static)
 VRBT_GENERATE_INSERT(acl_tree, acl_e, branch, vcl_acl_cmp, static)
 VRBT_GENERATE_MINMAX(acl_tree, acl_e, branch, static)
 VRBT_GENERATE_NEXT(acl_tree, acl_e, branch, static)
+VRBT_GENERATE_PREV(acl_tree, acl_e, branch, static)
 
 static char *
 vcc_acl_chk(struct vcc *tl, const struct acl_e *ae, const int l,
@@ -454,6 +484,52 @@ vcc_acl_emit_tokens(const struct vcc *tl, const struct acl_e *ae)
 		Fh(tl, 0, "\" fixed: %s\"", ae->fixed);
 }
 
+/*********************************************************************
+ * Emit ACL on table format
+ */
+
+static unsigned
+vcc_acl_emit_tables(const struct vcc *tl, unsigned n, const char *name)
+{
+	struct acl_e *ae;
+	unsigned rv = sizeof(ae->data) + 3;
+	unsigned nn = 0;
+	size_t sz;
+
+	Fh(tl, 0, "\nstatic unsigned char acl_tbl_%s[%u*%u] = {\n",
+	    name, n, rv);
+	VRBT_FOREACH(ae, acl_tree, &tl->acl->acl_tree) {
+		if (ae->overlapped)
+			continue;
+		Fh(tl, 0, "\t0x%02x,", ae->not ? 0 : 1);
+		Fh(tl, 0, "0x%02x,", (ae->mask >> 3) - 1);
+		Fh(tl, 0, "0x%02x,", (0xff00 >> (ae->mask & 7)) & 0xff);
+		for (sz = 0; sz < sizeof(ae->data); sz++)
+			Fh(tl, 0, "0x%02x,", ae->data[sz]);
+		for (; sz < rv - 3; sz++)
+			Fh(tl, 0, "0,");
+		Fh(tl, 0, "\n");
+		nn++;
+	}
+	assert(n == nn);
+	Fh(tl, 0, "};\n");
+	if (tl->acl->flag_log) {
+		Fh(tl, 0, "\nstatic const char *acl_str_%s[%d] = {\n",
+		    name, n);
+		VRBT_FOREACH(ae, acl_tree, &tl->acl->acl_tree) {
+			if (ae->overlapped)
+				continue;
+			Fh(tl, 0, "\t");
+			Fh(tl, 0, "\"%sMATCH %s \" ",
+			    ae->not ? "NEG_" : "", name);
+			vcc_acl_emit_tokens(tl, ae);
+			Fh(tl, 0, ",\n");
+		}
+		Fh(tl, 0, "};\n");
+	}
+	return  (rv);
+}
+
 /*********************************************************************
  * Emit a function to match the ACL we have collected
  */
@@ -461,11 +537,12 @@ vcc_acl_emit_tokens(const struct vcc *tl, const struct acl_e *ae)
 static void
 vcc_acl_emit(struct vcc *tl, const struct symbol *sym)
 {
-	struct acl_e *ae;
+	struct acl_e *ae, *ae2;
 	int depth, l, m, i;
 	unsigned at[ACL_MAXADDR];
 	struct inifin *ifp = NULL;
 	struct vsb *func;
+	unsigned n, no, nw = 0;
 
 	func = VSB_new_auto();
 	AN(func);
@@ -473,6 +550,32 @@ vcc_acl_emit(struct vcc *tl, const struct symbol *sym)
 	VCC_PrintCName(func, sym->name, NULL);
 	AZ(VSB_finish(func));
 
+	depth = -1;
+	at[0] = 256;
+	ae2 = NULL;
+	n = no = 0;
+	VRBT_FOREACH_REVERSE(ae, acl_tree, &tl->acl->acl_tree) {
+		n++;
+		if (ae2 == NULL) {
+			ae2 = ae;
+		} else if (vcl_acl_disjoint(ae, ae2)) {
+			ae2 = ae;
+		} else {
+			no++;
+			ae->overlapped = 1;
+		}
+	}
+
+	Fh(tl, 0, "/* acl_n_%s n %u no %u */\n", sym->name, n, no);
+	if (n - no < (1<<1))
+		no = n;
+	else if (!tl->acl->flag_table)
+		no = n;
+
+	if (no < n)
+		nw = vcc_acl_emit_tables(tl, n - no, sym->name);
+
+
 	Fh(tl, 0, "\nstatic int v_matchproto_(acl_match_f)\n");
 	Fh(tl, 0, "%s(VRT_CTX, const VCL_IP p)\n", VSB_data(func));
 	Fh(tl, 0, "{\n");
@@ -490,10 +593,12 @@ vcc_acl_emit(struct vcc *tl, const struct symbol *sym)
 		VSB_printf(ifp->ini,
 			"\tif (0) %s(0, 0);\n", VSB_data(func));
 	}
-	depth = -1;
-	at[0] = 256;
+
 	VRBT_FOREACH(ae, acl_tree, &tl->acl->acl_tree) {
 
+		if (no < n && !ae->overlapped)
+			continue;
+
 		/* Find how much common prefix we have */
 		for (l = 0; l <= depth && l * 8 < (int)ae->mask - 7; l++) {
 			assert(l >= 0);
@@ -549,10 +654,24 @@ vcc_acl_emit(struct vcc *tl, const struct symbol *sym)
 	for (; 0 <= depth; depth--)
 		Fh(tl, 0, "\t%*.*s}\n", depth, depth, "");
 
-	/* Deny by default */
-	if (tl->acl->flag_log)
-		Fh(tl, 0, "\tVPI_acl_log(ctx, \"NO_MATCH %s\");\n", sym->name);
-	Fh(tl, 0, "\treturn (0);\n}\n");
+	if (no < n) {
+		Fh(tl, 0, "\treturn(\n\t    VPI_acl_table(ctx,\n");
+		Fh(tl, 0, "\t\tp,\n");
+		Fh(tl, 0, "\t\t%u, %u,\n", n - no, nw);
+		Fh(tl, 0, "\t\tacl_tbl_%s,\n", sym->name);
+		if (tl->acl->flag_log)
+			Fh(tl, 0, "\t\tacl_str_%s,\n", sym->name);
+		else
+			Fh(tl, 0, "\t\tNULL,\n");
+		Fh(tl, 0, "\t\t\"NO MATCH %s\"\n\t    )\n\t);\n", sym->name);
+	} else {
+		/* Deny by default */
+		if (tl->acl->flag_log)
+			Fh(tl, 0, "\tVPI_acl_log(ctx, \"NO_MATCH %s\");\n",
+			    sym->name);
+		Fh(tl, 0, "\treturn(0);\n");
+	}
+	Fh(tl, 0, "}\n");
 
 	/* Emit the struct that will be referenced */
 	Fh(tl, 0, "\nstatic const struct vrt_acl %s[] = {{\n", sym->rname);
@@ -596,6 +715,9 @@ vcc_ParseAcl(struct vcc *tl)
 		if (vcc_IdIs(tl->t, "log")) {
 			acl->flag_log = sign->tok == '+';
 			vcc_NextToken(tl);
+		} else if (vcc_IdIs(tl->t, "table")) {
+			acl->flag_table = sign->tok == '+';
+			vcc_NextToken(tl);
 		} else {
 			VSB_cat(tl->sb, "Unknown ACL flag:\n");
 			vcc_ErrWhere(tl, tl->t);

From dridi.boukelmoune at gmail.com  Mon Apr 12 12:49:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 12 Apr 2021 12:49:05 +0000 (UTC)
Subject: [master] 9a4523dfd vcc: Missing minor error handling
Message-ID: <20210412124905.9597EACA07@lists.varnish-cache.org>


commit 9a4523dfd993c4ffc0a22d92b0693c1f738fc5f4
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Apr 12 14:47:30 2021 +0200

    vcc: Missing minor error handling

diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c
index 226215fda..ac6ca3d87 100644
--- a/lib/libvcc/vcc_acl.c
+++ b/lib/libvcc/vcc_acl.c
@@ -164,6 +164,7 @@ vcc_acl_chk(struct vcc *tl, const struct acl_e *ae, const int l,
 	char h[VTCP_ADDRBUFSIZE];
 	char t[VTCP_ADDRBUFSIZE + 10];
 	char s[vsa_suckaddr_len];
+	char *r = NULL;
 	struct suckaddr *sa;
 	unsigned m;
 	int ll, ret = 0;
@@ -208,7 +209,8 @@ vcc_acl_chk(struct vcc *tl, const struct acl_e *ae, const int l,
 	vcc_ErrWhere(tl, ae->t_addr);
 	if (tl->acl_pedantic == 0)
 		vcc_Warn(tl);
-	return (strdup(t));
+	REPLACE(r, t);
+	return (r);
 }
 
 static void

From nils.goroll at uplex.de  Mon Apr 12 12:56:07 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 12 Apr 2021 14:56:07 +0200
Subject: [master] d89674d12 Collapse vcc_acl_add_entry() and
 vcc_acl_insert_entry()
In-Reply-To: <20210412120905.4BD5BAB67B@lists.varnish-cache.org>
References: <20210412120905.4BD5BAB67B@lists.varnish-cache.org>
Message-ID: <9f3c1a61-573f-2a73-ba1f-c8411f729407@uplex.de>

@phk, the split was intentional, see bbb5db47e02f431598e5a7ff7106a8ac316db8be

Can I revert this to have a separate vcc_acl_insert_entry() again?

-- 

** * * UPLEX - Nils Goroll Systemoptimierung

Scheffelstra?e 32
22301 Hamburg

tel +49 40 28805731
mob +49 170 2723133
fax +49 40 42949753

xmpp://slink at jabber.int.uplex.de/

http://uplex.de/

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 488 bytes
Desc: OpenPGP digital signature
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20210412/377836c8/attachment.bin>

From dridi.boukelmoune at gmail.com  Mon Apr 12 13:14:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 12 Apr 2021 13:14:05 +0000 (UTC)
Subject: [master] f44f16cc4 Revert "disable -Wcast-qual because of musl"
Message-ID: <20210412131405.2D545AD952@lists.varnish-cache.org>


commit f44f16cc4696e9e7f434dfd6d67b4e0447b4e97b
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Apr 12 15:09:13 2021 +0200

    Revert "disable -Wcast-qual because of musl"
    
    This reverts commit 05780b634c40233dd2494a2c76a8dd8ba72d79f8.
    
    If we have to choose between this warning and alpine being part of our
    CI, this warning takes precedence. It's always possible to disable
    developer warnings on CCI if that's a problem with musl then we can
    configure its CCI job to disable developer warnings.
    
    Reopens #3565
    Refs #3568

diff --git a/wflags.py b/wflags.py
index 7e4d30fed..d060d1794 100644
--- a/wflags.py
+++ b/wflags.py
@@ -44,6 +44,7 @@ DESIRABLE_OPTIONS = [
 
 DESIRABLE_WFLAGS = [
     "-Wcast-align",
+    "-Wcast-qual",
     "-Wchar-subscripts",
     "-Wempty-body",
     "-Wextra",
@@ -68,7 +69,6 @@ DESIRABLE_WFLAGS = [
 ]
 
 UNDESIRABLE_WFLAGS = [
-    "-Wno-cast-qual", # GCC complains about musl::sched.h
     "-Wno-thread-safety", # Does not understand our mutexs are wrapped
     "-Wno-old-style-definition", # Does not like vgz
     "-Wno-sign-compare", # Fixable

From guillaume at varnish-software.com  Mon Apr 12 14:20:05 2021
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Mon, 12 Apr 2021 14:20:05 +0000 (UTC)
Subject: [master] 2baff3dc8 [cci] no need to pin glibc anymore
Message-ID: <20210412142005.AFD6CAF956@lists.varnish-cache.org>


commit 2baff3dc8b6e7f574cce53cac9345e0f30f72beb
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Mon Apr 12 07:19:19 2021 -0700

    [cci] no need to pin glibc anymore

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 09d0f2db1..9a10465cc 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -206,10 +206,6 @@ jobs:
                     tar \
                     sudo
             elif [ << parameters.dist >> = archlinux ]; then
-                # XXX: TEMPORARY fix for https://bugs.archlinux.org/task/69563
-                cd /tmp
-                patched_glibc=glibc-linux4-2.33-4-x86_64.pkg.tar.zst && curl -LO "https://repo.archlinuxcn.org/x86_64/$patched_glibc" && bsdtar -C / -xvf "$patched_glibc"
-                cd -
                 pacman -Sy --noconfirm \
                     base-devel \
                     ca-certificates \

From guillaume at varnish-software.com  Mon Apr 12 14:41:04 2021
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Mon, 12 Apr 2021 14:41:04 +0000 (UTC)
Subject: [master] 6ab687ca1 Revert "[cci] no need to pin glibc anymore"
Message-ID: <20210412144104.A7E2EB0411@lists.varnish-cache.org>


commit 6ab687ca123831ee9e75d90db0a42b93dad61186
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Mon Apr 12 07:40:38 2021 -0700

    Revert "[cci] no need to pin glibc anymore"
    
    This reverts commit 2baff3dc8b6e7f574cce53cac9345e0f30f72beb.

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 9a10465cc..09d0f2db1 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -206,6 +206,10 @@ jobs:
                     tar \
                     sudo
             elif [ << parameters.dist >> = archlinux ]; then
+                # XXX: TEMPORARY fix for https://bugs.archlinux.org/task/69563
+                cd /tmp
+                patched_glibc=glibc-linux4-2.33-4-x86_64.pkg.tar.zst && curl -LO "https://repo.archlinuxcn.org/x86_64/$patched_glibc" && bsdtar -C / -xvf "$patched_glibc"
+                cd -
                 pacman -Sy --noconfirm \
                     base-devel \
                     ca-certificates \

From nils.goroll at uplex.de  Mon Apr 12 16:04:05 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 12 Apr 2021 16:04:05 +0000 (UTC)
Subject: [master] c39800d8b Bring back {Req, Resp, Bereq,
 Beresp}Unset to varnishtop
Message-ID: <20210412160405.292E7B29D7@lists.varnish-cache.org>


commit c39800d8b6bc5cf81bb96bc165c99fcd5b22fd3a
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Apr 12 17:56:28 2021 +0200

    Bring back {Req,Resp,Bereq,Beresp}Unset to varnishtop
    
    Fixes #3527

diff --git a/include/tbl/vsl_tags_http.h b/include/tbl/vsl_tags_http.h
index 7a0198da6..29d731300 100644
--- a/include/tbl/vsl_tags_http.h
+++ b/include/tbl/vsl_tags_http.h
@@ -83,7 +83,7 @@ SLTH(Header,	HTTP_HDR_FIRST,		1, 1, "header",
 	HEADER_NOTICE
 )
 
-SLTH(Unset,	HTTP_HDR_UNSET,		0, 0, "unset header",
+SLTH(Unset,	HTTP_HDR_UNSET,		1, 1, "unset header",
 	"HTTP header contents.\n\n"
 	"The format is::\n\n"
 	"\t%s: %s\n"

From guillaume at varnish-software.com  Tue Apr 13 00:46:05 2021
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Tue, 13 Apr 2021 00:46:05 +0000 (UTC)
Subject: [archlinux_tests] 149c0a97e [cci][wip] dumb test
Message-ID: <20210413004605.2EF446E0C1@lists.varnish-cache.org>


commit 149c0a97ef22300677f21a5d30aac66958380f91
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Mon Apr 12 17:45:36 2021 -0700

    [cci][wip] dumb test

diff --git a/.circleci/config.yml b/.circleci/config.yml
index fe8f62e96..6f27d151e 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -126,6 +126,13 @@ jobs:
           root: .
           paths:
             - "packages"
+  arch_test:
+    docker:
+      - image: archlinux:latest
+    steps:
+      - run:
+          command: |
+            pacman -Suy --noconfirm
   distcheck:
     parameters:
       dist:
@@ -274,6 +281,7 @@ workflows:
   version: 2
   commit:
     jobs:
+      - arch_test
       - dist
       - distcheck:
           name: distcheck_centos_7

From guillaume at varnish-software.com  Tue Apr 13 00:52:04 2021
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Tue, 13 Apr 2021 00:52:04 +0000 (UTC)
Subject: [archlinux_tests] 0ea31f68f [cci][wipi] ls debug
Message-ID: <20210413005204.B13B06E522@lists.varnish-cache.org>


commit 0ea31f68fdb43f0413f3ba32d568b8ab3f26cb3f
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Mon Apr 12 17:50:07 2021 -0700

    [cci][wipi] ls debug

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 6f27d151e..0da8a788e 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -132,6 +132,7 @@ jobs:
     steps:
       - run:
           command: |
+            ls /var/lib/pacman/
             pacman -Suy --noconfirm
   distcheck:
     parameters:

From phk at FreeBSD.org  Tue Apr 13 13:20:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 13 Apr 2021 13:20:05 +0000 (UTC)
Subject: [master] 9f88fa613 Split the source/preprocessing stuff into a
 separate file.
Message-ID: <20210413132005.DE104A85C1@lists.varnish-cache.org>


commit 9f88fa61306f218d948327894f0ee4000b6a63ae
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Apr 13 12:35:42 2021 +0000

    Split the source/preprocessing stuff into a separate file.

diff --git a/lib/libvcc/Makefile.am b/lib/libvcc/Makefile.am
index 1e3d6f2d0..c97d709c5 100644
--- a/lib/libvcc/Makefile.am
+++ b/lib/libvcc/Makefile.am
@@ -25,6 +25,7 @@ libvcc_a_SOURCES = \
 	vcc_fixed_token.c \
 	vcc_obj.c \
 	vcc_parse.c \
+	vcc_source.c \
 	vcc_storage.c \
 	vcc_symb.c \
 	vcc_token.c \
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 8991e9a30..1bb0ba32b 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -603,158 +603,6 @@ EmitStruct(const struct vcc *tl)
 	Fc(tl, 0, "};\n");
 }
 
-/*--------------------------------------------------------------------*/
-
-static struct source *
-vcc_new_source(const char *src, const char *name)
-{
-	struct source *sp;
-
-	AN(src);
-	AN(name);
-	sp = calloc(1, sizeof *sp);
-	AN(sp);
-	REPLACE(sp->name, name);
-	sp->b = src;
-	sp->e = strchr(src, '\0');
-	return (sp);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-vcc_destroy_source(struct source **spp)
-{
-	struct source *sp;
-
-	AN(spp);
-	sp = *spp;
-	*spp = NULL;
-
-	AN(sp);
-	free(sp->name);
-	free(sp->freeit);
-	free(sp);
-}
-
-/*--------------------------------------------------------------------*/
-
-static struct source *
-vcc_file_source(const struct vcc *tl, const char *fn)
-{
-	char *f, *fnp;
-	struct source *sp;
-
-	if (!tl->unsafe_path && strchr(fn, '/') != NULL) {
-		VSB_printf(tl->sb, "VCL filename '%s' is unsafe.\n", fn);
-		return (NULL);
-	}
-	f = NULL;
-	if (VFIL_searchpath(tl->vcl_path, NULL, &f, fn, &fnp) || f == NULL) {
-		VSB_printf(tl->sb, "Cannot read file '%s' (%s)\n",
-		    fnp != NULL ? fnp : fn, strerror(errno));
-		free(fnp);
-		return (NULL);
-	}
-	sp = vcc_new_source(f, fnp);
-	free(fnp);
-	sp->freeit = f;
-	return (sp);
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-vcc_resolve_includes(struct vcc *tl)
-{
-	struct token *t, *t1, *t2;
-	struct source *sp;
-	const struct source *sp1;
-	struct vsb *vsb;
-	const char *p;
-
-	VTAILQ_FOREACH(t, &tl->tokens, list) {
-		if (t->tok != ID || !vcc_IdIs(t, "include"))
-			continue;
-
-		t1 = VTAILQ_NEXT(t, list);
-		AN(t1);			/* There's always an EOI */
-		if (t1->tok != CSTR) {
-			VSB_cat(tl->sb,
-			    "include not followed by string constant.\n");
-			vcc_ErrWhere(tl, t1);
-			return;
-		}
-		t2 = VTAILQ_NEXT(t1, list);
-		AN(t2);			/* There's always an EOI */
-
-		if (t2->tok != ';') {
-			VSB_cat(tl->sb,
-			    "include <string> not followed by semicolon.\n");
-			vcc_ErrWhere(tl, t1);
-			return;
-		}
-
-		if (t1->dec[0] == '.' && t1->dec[1] == '/') {
-			/*
-			 * Nested include filenames, starting with "./" are
-			 * resolved relative to the VCL file which contains
-			 * the include directive.
-			 */
-			if (t1->src->name[0] != '/') {
-				VSB_cat(tl->sb,
-				    "include \"./xxxxx\"; needs absolute "
-				    "filename of including file.\n");
-				vcc_ErrWhere(tl, t1);
-				return;
-			}
-			vsb = VSB_new_auto();
-			AN(vsb);
-			p = strrchr(t1->src->name, '/');
-			AN(p);
-			VSB_bcat(vsb, t1->src->name, p - t1->src->name);
-			VSB_cat(vsb, t1->dec + 1);
-			AZ(VSB_finish(vsb));
-			sp = vcc_file_source(tl, VSB_data(vsb));
-			VSB_destroy(&vsb);
-		} else {
-			sp = vcc_file_source(tl, t1->dec);
-		}
-		if (sp == NULL) {
-			vcc_ErrWhere(tl, t1);
-			return;
-		}
-
-		for (sp1 = t->src; sp1 != NULL; sp1 = sp1->parent)
-			if (!strcmp(sp1->name, sp->name))
-				break;
-		if (sp1 != NULL) {
-			VSB_printf(tl->sb,
-			    "Recursive include of \"%s\"\n\n", sp->name);
-			vcc_ErrWhere(tl, t1);
-			for (sp1 = t->src; sp1 != NULL; sp1 = sp1->parent) {
-				if (sp1->parent_tok)
-					vcc_ErrWhere(tl, sp1->parent_tok);
-			}
-			vcc_destroy_source(&sp);
-			return;
-		}
-		sp->parent = t->src;
-		sp->parent_tok = t1;
-		VTAILQ_INSERT_TAIL(&tl->sources, sp, list);
-		sp->idx = tl->nsources++;
-		tl->t = t2;
-		vcc_Lexer(tl, sp, 0);
-
-		VTAILQ_REMOVE(&tl->tokens, t, list);
-		VTAILQ_REMOVE(&tl->tokens, t1, list);
-		VTAILQ_REMOVE(&tl->tokens, t2, list);
-		if (!tl->err)
-			vcc_resolve_includes(tl);
-		return;
-	}
-}
-
 /*--------------------------------------------------------------------
  * Compile the VCL code from the given source and return the C-source
  */
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 231a2cd15..aeca3d5c7 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -362,16 +362,10 @@ void vcc_Parse(struct vcc *);
 void vcc_Parse_Init(struct vcc *);
 sym_act_f vcc_Act_If;
 
-/* vcc_utils.c */
-void vcc_regexp(struct vcc *tl, struct vsb *vgc_name);
-void Resolve_Sockaddr(struct vcc *tl, const char *host, const char *defport,
-    const char **ipv4, const char **ipv4_ascii, const char **ipv6,
-    const char **ipv6_ascii, const char **p_ascii, int maxips,
-    const struct token *t_err, const char *errid);
-double vcc_DurationUnit(struct vcc *);
-void vcc_ByteVal(struct vcc *, double *);
-void vcc_Duration(struct vcc *tl, double *);
-unsigned vcc_UintVal(struct vcc *tl);
+/* vcc_source.c */
+struct source * vcc_new_source(const char *src, const char *name);
+struct source *vcc_file_source(const struct vcc *tl, const char *fn);
+void vcc_resolve_includes(struct vcc *tl);
 
 /* vcc_storage.c */
 void vcc_stevedore(struct vcc *vcc, const char *stv_name);
@@ -426,6 +420,17 @@ vcc_type_t VCC_Type(const char *p);
 const char * VCC_Type_EvalMethod(struct vcc *, const struct symbol *);
 void vcc_Type_Init(struct vcc *tl);
 
+/* vcc_utils.c */
+void vcc_regexp(struct vcc *tl, struct vsb *vgc_name);
+void Resolve_Sockaddr(struct vcc *tl, const char *host, const char *defport,
+    const char **ipv4, const char **ipv4_ascii, const char **ipv6,
+    const char **ipv6_ascii, const char **p_ascii, int maxips,
+    const struct token *t_err, const char *errid);
+double vcc_DurationUnit(struct vcc *);
+void vcc_ByteVal(struct vcc *, double *);
+void vcc_Duration(struct vcc *tl, double *);
+unsigned vcc_UintVal(struct vcc *tl);
+
 /* vcc_var.c */
 sym_wildcard_t vcc_Var_Wildcard;
 
diff --git a/lib/libvcc/vcc_source.c b/lib/libvcc/vcc_source.c
new file mode 100644
index 000000000..61095e778
--- /dev/null
+++ b/lib/libvcc/vcc_source.c
@@ -0,0 +1,190 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2015 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "vcc_compile.h"
+
+#include "vfil.h"
+
+struct source *
+vcc_new_source(const char *src, const char *name)
+{
+	struct source *sp;
+
+	AN(src);
+	AN(name);
+	sp = calloc(1, sizeof *sp);
+	AN(sp);
+	REPLACE(sp->name, name);
+	sp->b = src;
+	sp->e = strchr(src, '\0');
+	return (sp);
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+vcc_destroy_source(struct source **spp)
+{
+	struct source *sp;
+
+	AN(spp);
+	sp = *spp;
+	*spp = NULL;
+
+	AN(sp);
+	free(sp->name);
+	free(sp->freeit);
+	free(sp);
+}
+
+/*--------------------------------------------------------------------*/
+
+struct source *
+vcc_file_source(const struct vcc *tl, const char *fn)
+{
+	char *f, *fnp;
+	struct source *sp;
+
+	if (!tl->unsafe_path && strchr(fn, '/') != NULL) {
+		VSB_printf(tl->sb, "VCL filename '%s' is unsafe.\n", fn);
+		return (NULL);
+	}
+	f = NULL;
+	if (VFIL_searchpath(tl->vcl_path, NULL, &f, fn, &fnp) || f == NULL) {
+		VSB_printf(tl->sb, "Cannot read file '%s' (%s)\n",
+		    fnp != NULL ? fnp : fn, strerror(errno));
+		free(fnp);
+		return (NULL);
+	}
+	sp = vcc_new_source(f, fnp);
+	free(fnp);
+	sp->freeit = f;
+	return (sp);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+vcc_resolve_includes(struct vcc *tl)
+{
+	struct token *t, *t1, *t2;
+	struct source *sp;
+	const struct source *sp1;
+	struct vsb *vsb;
+	const char *p;
+
+	VTAILQ_FOREACH(t, &tl->tokens, list) {
+		if (t->tok != ID || !vcc_IdIs(t, "include"))
+			continue;
+
+		t1 = VTAILQ_NEXT(t, list);
+		AN(t1);			/* There's always an EOI */
+		if (t1->tok != CSTR) {
+			VSB_cat(tl->sb,
+			    "include not followed by string constant.\n");
+			vcc_ErrWhere(tl, t1);
+			return;
+		}
+		t2 = VTAILQ_NEXT(t1, list);
+		AN(t2);			/* There's always an EOI */
+
+		if (t2->tok != ';') {
+			VSB_cat(tl->sb,
+			    "include <string> not followed by semicolon.\n");
+			vcc_ErrWhere(tl, t1);
+			return;
+		}
+
+		if (t1->dec[0] == '.' && t1->dec[1] == '/') {
+			/*
+			 * Nested include filenames, starting with "./" are
+			 * resolved relative to the VCL file which contains
+			 * the include directive.
+			 */
+			if (t1->src->name[0] != '/') {
+				VSB_cat(tl->sb,
+				    "include \"./xxxxx\"; needs absolute "
+				    "filename of including file.\n");
+				vcc_ErrWhere(tl, t1);
+				return;
+			}
+			vsb = VSB_new_auto();
+			AN(vsb);
+			p = strrchr(t1->src->name, '/');
+			AN(p);
+			VSB_bcat(vsb, t1->src->name, p - t1->src->name);
+			VSB_cat(vsb, t1->dec + 1);
+			AZ(VSB_finish(vsb));
+			sp = vcc_file_source(tl, VSB_data(vsb));
+			VSB_destroy(&vsb);
+		} else {
+			sp = vcc_file_source(tl, t1->dec);
+		}
+		if (sp == NULL) {
+			vcc_ErrWhere(tl, t1);
+			return;
+		}
+
+		for (sp1 = t->src; sp1 != NULL; sp1 = sp1->parent)
+			if (!strcmp(sp1->name, sp->name))
+				break;
+		if (sp1 != NULL) {
+			VSB_printf(tl->sb,
+			    "Recursive include of \"%s\"\n\n", sp->name);
+			vcc_ErrWhere(tl, t1);
+			for (sp1 = t->src; sp1 != NULL; sp1 = sp1->parent) {
+				if (sp1->parent_tok)
+					vcc_ErrWhere(tl, sp1->parent_tok);
+			}
+			vcc_destroy_source(&sp);
+			return;
+		}
+		sp->parent = t->src;
+		sp->parent_tok = t1;
+		VTAILQ_INSERT_TAIL(&tl->sources, sp, list);
+		sp->idx = tl->nsources++;
+		tl->t = t2;
+		vcc_Lexer(tl, sp, 0);
+
+		VTAILQ_REMOVE(&tl->tokens, t, list);
+		VTAILQ_REMOVE(&tl->tokens, t1, list);
+		VTAILQ_REMOVE(&tl->tokens, t2, list);
+		if (!tl->err)
+			vcc_resolve_includes(tl);
+		return;
+	}
+}

From phk at FreeBSD.org  Tue Apr 13 13:20:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 13 Apr 2021 13:20:05 +0000 (UTC)
Subject: [master] c8174af68 Refactor "preprocesing" of VCL.
Message-ID: <20210413132005.F3D8DA85C4@lists.varnish-cache.org>


commit c8174af68956206115972e72e75ceddbd758116e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Apr 13 13:19:00 2021 +0000

    Refactor "preprocesing" of VCL.

diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 1bb0ba32b..88d558a93 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -638,7 +638,7 @@ vcc_CompileSource(struct vcc *tl, struct source *sp, const char *jfile)
 		AN(vcc_builtin);
 		VTAILQ_INSERT_TAIL(&tl->sources, sp, list);
 		sp->idx = tl->nsources++;
-		vcc_Lexer(tl, sp, 0);
+		vcc_lex_source(tl, sp, 0);
 		if (tl->err)
 			return (NULL);
 	}
@@ -648,12 +648,11 @@ vcc_CompileSource(struct vcc *tl, struct source *sp, const char *jfile)
 	assert(sp != NULL);
 	VTAILQ_INSERT_TAIL(&tl->sources, sp, list);
 	sp->idx = tl->nsources++;
-	vcc_Lexer(tl, sp, 1);
+	vcc_lex_source(tl, sp, 1);
 	if (tl->err)
 		return (NULL);
 
 	/* Expand and lex any includes in the token string */
-	vcc_resolve_includes(tl);
 	if (tl->err)
 		return (NULL);
 
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index aeca3d5c7..76e923693 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -57,11 +57,18 @@
 /*---------------------------------------------------------------------*/
 
 struct acl;
-struct vsb;
-struct token;
-struct sockaddr_storage;
+struct acl_e;
+struct expr;
 struct method;
+struct proc;
+struct sockaddr_storage;
+struct symbol;
 struct symtab;
+struct token;
+struct vcc;
+struct vjsn_val;
+struct vmod_obj;
+struct vsb;
 
 unsigned vcl_fixed_token(const char *p, const char **q);
 extern const char * const vcl_tnames[256];
@@ -71,14 +78,6 @@ void vcl_output_lang_h(struct vsb *sb);
 
 #define INDENT		2
 
-struct acl_e;
-struct proc;
-struct expr;
-struct vcc;
-struct vjsn_val;
-struct symbol;
-struct vmod_obj;
-
 struct source {
 	VTAILQ_ENTRY(source)	list;
 	char			*name;
@@ -88,6 +87,7 @@ struct source {
 	char			*freeit;
 	const struct source	*parent;
 	const struct token	*parent_tok;
+	VTAILQ_HEAD(, token)	src_tokens;
 };
 
 struct token {
@@ -96,6 +96,7 @@ struct token {
 	const char		*e;
 	const struct source	*src;
 	VTAILQ_ENTRY(token)	list;
+	VTAILQ_ENTRY(token)	src_list;
 	unsigned		cnt;
 	char			*dec;
 };
@@ -365,7 +366,7 @@ sym_act_f vcc_Act_If;
 /* vcc_source.c */
 struct source * vcc_new_source(const char *src, const char *name);
 struct source *vcc_file_source(const struct vcc *tl, const char *fn);
-void vcc_resolve_includes(struct vcc *tl);
+void vcc_lex_source(struct vcc *tl, struct source *sp, int eoi);
 
 /* vcc_storage.c */
 void vcc_stevedore(struct vcc *vcc, const char *stv_name);
@@ -410,7 +411,7 @@ void vcc_Warn(struct vcc *);
 void vcc__Expect(struct vcc *tl, unsigned tok, unsigned line);
 int vcc_IdIs(const struct token *t, const char *p);
 void vcc_ExpectVid(struct vcc *tl, const char *what);
-void vcc_Lexer(struct vcc *tl, const struct source *sp, int eoi);
+void vcc_Lexer(struct vcc *tl, struct source *sp);
 void vcc_NextToken(struct vcc *tl);
 void vcc__ErrInternal(struct vcc *tl, const char *func,
     unsigned line);
diff --git a/lib/libvcc/vcc_source.c b/lib/libvcc/vcc_source.c
index 61095e778..48f50553e 100644
--- a/lib/libvcc/vcc_source.c
+++ b/lib/libvcc/vcc_source.c
@@ -51,6 +51,7 @@ vcc_new_source(const char *src, const char *name)
 	REPLACE(sp->name, name);
 	sp->b = src;
 	sp->e = strchr(src, '\0');
+	VTAILQ_INIT(&sp->src_tokens);
 	return (sp);
 }
 
@@ -98,93 +99,110 @@ vcc_file_source(const struct vcc *tl, const char *fn)
 
 /*--------------------------------------------------------------------*/
 
-void
-vcc_resolve_includes(struct vcc *tl)
+static struct source *
+vcc_include_file(struct vcc *tl, const struct source *src_sp,
+    const char *filename, const struct token *parent_token)
 {
-	struct token *t, *t1, *t2;
 	struct source *sp;
 	const struct source *sp1;
 	struct vsb *vsb;
 	const char *p;
 
-	VTAILQ_FOREACH(t, &tl->tokens, list) {
-		if (t->tok != ID || !vcc_IdIs(t, "include"))
+	if (filename[0] == '.' && filename[1] == '/') {
+		/*
+		 * Nested include filenames, starting with "./" are
+		 * resolved relative to the VCL file which contains
+		 * the include directive.
+		 */
+		if (src_sp->name[0] != '/') {
+			VSB_cat(tl->sb,
+			    "include \"./xxxxx\"; needs absolute "
+			    "filename of including file.\n");
+			return (NULL);
+		}
+		vsb = VSB_new_auto();
+		AN(vsb);
+		p = strrchr(src_sp->name, '/');
+		AN(p);
+		VSB_bcat(vsb, src_sp->name, p - src_sp->name);
+		VSB_cat(vsb, filename + 1);
+		AZ(VSB_finish(vsb));
+		sp = vcc_file_source(tl, VSB_data(vsb));
+		VSB_destroy(&vsb);
+	} else {
+		sp = vcc_file_source(tl, filename);
+	}
+	if (sp == NULL)
+		return (sp);
+
+	for (sp1 = src_sp; sp1 != NULL; sp1 = sp1->parent)
+		if (!strcmp(sp1->name, sp->name))
+			break;
+	if (sp1 != NULL) {
+		VSB_printf(tl->sb,
+		    "Recursive include of \"%s\"\n\n", sp->name);
+		vcc_destroy_source(&sp);
+		return (NULL);
+	}
+	sp->parent = src_sp;
+	VTAILQ_INSERT_TAIL(&tl->sources, sp, list);
+	sp->idx = tl->nsources++;
+	sp->parent_tok = parent_token;
+	vcc_lex_source(tl, sp, 0);
+	return (sp);
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+vcc_lex_source(struct vcc *tl, struct source *src_sp, int eoi)
+{
+	struct token *t, *t1;
+	struct source *sp;
+	const struct source *sp1;
+
+	vcc_Lexer(tl, src_sp);
+	if (tl->err)
+		return;
+	VTAILQ_FOREACH(t, &src_sp->src_tokens, src_list) {
+		if (!eoi && t->tok == EOI)
+			break;
+
+		if (t->tok != ID || !vcc_IdIs(t, "include")) {
+			VTAILQ_INSERT_TAIL(&tl->tokens, t, list);
 			continue;
+		}
 
-		t1 = VTAILQ_NEXT(t, list);
+		t1 = VTAILQ_NEXT(t, src_list);
 		AN(t1);			/* There's always an EOI */
 		if (t1->tok != CSTR) {
 			VSB_cat(tl->sb,
 			    "include not followed by string constant.\n");
+			/* Not vcc_ErrWhere2(): tokens not on final list */
 			vcc_ErrWhere(tl, t1);
 			return;
 		}
-		t2 = VTAILQ_NEXT(t1, list);
-		AN(t2);			/* There's always an EOI */
+		t = VTAILQ_NEXT(t1, src_list);
+		AN(t);			/* There's always an EOI */
 
-		if (t2->tok != ';') {
+		if (t->tok != ';') {
 			VSB_cat(tl->sb,
 			    "include <string> not followed by semicolon.\n");
+			/* Not vcc_ErrWhere2(): tokens not on final list */
 			vcc_ErrWhere(tl, t1);
 			return;
 		}
 
-		if (t1->dec[0] == '.' && t1->dec[1] == '/') {
-			/*
-			 * Nested include filenames, starting with "./" are
-			 * resolved relative to the VCL file which contains
-			 * the include directive.
-			 */
-			if (t1->src->name[0] != '/') {
-				VSB_cat(tl->sb,
-				    "include \"./xxxxx\"; needs absolute "
-				    "filename of including file.\n");
-				vcc_ErrWhere(tl, t1);
-				return;
-			}
-			vsb = VSB_new_auto();
-			AN(vsb);
-			p = strrchr(t1->src->name, '/');
-			AN(p);
-			VSB_bcat(vsb, t1->src->name, p - t1->src->name);
-			VSB_cat(vsb, t1->dec + 1);
-			AZ(VSB_finish(vsb));
-			sp = vcc_file_source(tl, VSB_data(vsb));
-			VSB_destroy(&vsb);
-		} else {
-			sp = vcc_file_source(tl, t1->dec);
-		}
+		sp = vcc_include_file(tl, src_sp, t1->dec, t1);
 		if (sp == NULL) {
 			vcc_ErrWhere(tl, t1);
-			return;
-		}
-
-		for (sp1 = t->src; sp1 != NULL; sp1 = sp1->parent)
-			if (!strcmp(sp1->name, sp->name))
-				break;
-		if (sp1 != NULL) {
-			VSB_printf(tl->sb,
-			    "Recursive include of \"%s\"\n\n", sp->name);
-			vcc_ErrWhere(tl, t1);
-			for (sp1 = t->src; sp1 != NULL; sp1 = sp1->parent) {
-				if (sp1->parent_tok)
-					vcc_ErrWhere(tl, sp1->parent_tok);
+			for (sp1 = src_sp; sp1 != NULL; sp1 = sp1->parent) {
+				if (sp1->parent_tok == NULL)
+					break;
+				vcc_ErrWhere(tl, sp1->parent_tok);
 			}
-			vcc_destroy_source(&sp);
-			return;
 		}
-		sp->parent = t->src;
-		sp->parent_tok = t1;
-		VTAILQ_INSERT_TAIL(&tl->sources, sp, list);
-		sp->idx = tl->nsources++;
-		tl->t = t2;
-		vcc_Lexer(tl, sp, 0);
-
-		VTAILQ_REMOVE(&tl->tokens, t, list);
-		VTAILQ_REMOVE(&tl->tokens, t1, list);
-		VTAILQ_REMOVE(&tl->tokens, t2, list);
-		if (!tl->err)
-			vcc_resolve_includes(tl);
-		return;
+		if (tl->err)
+			return;
 	}
 }
diff --git a/lib/libvcc/vcc_token.c b/lib/libvcc/vcc_token.c
index 9edff6226..cc8392196 100644
--- a/lib/libvcc/vcc_token.c
+++ b/lib/libvcc/vcc_token.c
@@ -365,7 +365,7 @@ vcc_decstr(struct vcc *tl, unsigned sep)
 
 static void
 vcc_addtoken(struct vcc *tl, unsigned tok,
-    const struct source *sp, const char *b, const char *e)
+    struct source *sp, const char *b, const char *e)
 {
 	struct token *t;
 
@@ -375,10 +375,7 @@ vcc_addtoken(struct vcc *tl, unsigned tok,
 	t->b = b;
 	t->e = e;
 	t->src = sp;
-	if (tl->t != NULL)
-		VTAILQ_INSERT_AFTER(&tl->tokens, tl->t, t, list);
-	else
-		VTAILQ_INSERT_TAIL(&tl->tokens, t, list);
+	VTAILQ_INSERT_TAIL(&sp->src_tokens, t, src_list);
 	tl->t = t;
 }
 
@@ -405,7 +402,7 @@ static const struct delim_def {
 };
 
 static unsigned
-vcc_delim_token(struct vcc *tl, const struct source *sp, const char *p,
+vcc_delim_token(struct vcc *tl, struct source *sp, const char *p,
     const char **qp)
 {
 	const struct delim_def *dd;
@@ -445,7 +442,7 @@ vcc_delim_token(struct vcc *tl, const struct source *sp, const char *p,
  */
 
 void
-vcc_Lexer(struct vcc *tl, const struct source *sp, int eoi)
+vcc_Lexer(struct vcc *tl, struct source *sp)
 {
 	const char *p, *q, *r;
 	unsigned u;
@@ -611,6 +608,5 @@ vcc_Lexer(struct vcc *tl, const struct source *sp, int eoi)
 		vcc_ErrWhere(tl, tl->t);
 		return;
 	}
-	if (eoi)
-		vcc_addtoken(tl, EOI, sp, sp->e, sp->e);
+	vcc_addtoken(tl, EOI, sp, sp->e, sp->e);
 }

From reza at naghibi.com  Tue Apr 13 13:45:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 13 Apr 2021 13:45:06 +0000 (UTC)
Subject: [6.0] 765d564a9 [cci] PowerTools -> powertools
Message-ID: <20210413134506.23D1AA92E0@lists.varnish-cache.org>


commit 765d564a9d4d1bccdd760a86132b35c4c10e560f
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Wed Dec 9 15:35:22 2020 -0800

    [cci] PowerTools -> powertools

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 0b8da355f..d830ae1df 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -169,7 +169,7 @@ jobs:
             if [ << parameters.dist >> = centos ]; then
                 if [ << parameters.release >> = 8 ]; then
                     dnf install -y 'dnf-command(config-manager)'
-                    yum config-manager --set-enabled PowerTools
+                    yum config-manager --set-enabled powertools
                     yum install -y diffutils python3-sphinx
                 else
                     yum install -y python-sphinx

From reza at naghibi.com  Tue Apr 13 13:45:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 13 Apr 2021 13:45:06 +0000 (UTC)
Subject: [6.0] 140ffe2a4 rename the PowerTools leftovers
Message-ID: <20210413134506.39D3BA92E4@lists.varnish-cache.org>


commit 140ffe2a46bc09139834fb531ff1c5b219b7878b
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Thu Dec 10 21:52:44 2020 -0800

    rename the PowerTools leftovers
    
     Conflicts:
            doc/sphinx/installation/install_source.rst

diff --git a/.circleci/make-rpm-packages.sh b/.circleci/make-rpm-packages.sh
index 5f7d0af1a..56acdad05 100755
--- a/.circleci/make-rpm-packages.sh
+++ b/.circleci/make-rpm-packages.sh
@@ -18,7 +18,7 @@ yum install -y epel-release
 if [ "$PARAM_DIST" = centos ]; then
   if [ "$PARAM_RELEASE" = 8 ]; then
       dnf install -y 'dnf-command(config-manager)'
-      yum config-manager --set-enabled PowerTools
+      yum config-manager --set-enabled powertools
   fi
 fi
 

From phk at FreeBSD.org  Tue Apr 13 17:23:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 13 Apr 2021 17:23:05 +0000 (UTC)
Subject: [master] a1c936c83 Generalize the VCL flag stuff a little bit.
Message-ID: <20210413172305.CF49873EC@lists.varnish-cache.org>


commit a1c936c83b56d1d5ee211dd550a555830ae44396
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Apr 13 15:09:16 2021 +0000

    Generalize the VCL flag stuff a little bit.

diff --git a/bin/varnishtest/tests/v00017.vtc b/bin/varnishtest/tests/v00017.vtc
index 17f94582d..6ed35abc6 100644
--- a/bin/varnishtest/tests/v00017.vtc
+++ b/bin/varnishtest/tests/v00017.vtc
@@ -101,7 +101,7 @@ varnish v1 -errvcl {/mask only allowed once} {
 	sub vcl_recv { if (client.ip ~ a) { return(pass); } }
 }
 
-varnish v1 -errvcl {Expected ACL flag after:} {
+varnish v1 -errvcl {Expected a flag at:} {
 	backend b { .host = "${localhost}"; }
 	acl a + foobar {
 		"10.0.1.0/22" / 22;
diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c
index ac6ca3d87..6f595f7ad 100644
--- a/lib/libvcc/vcc_acl.c
+++ b/lib/libvcc/vcc_acl.c
@@ -692,7 +692,7 @@ void
 vcc_ParseAcl(struct vcc *tl)
 {
 	struct symbol *sym;
-	struct token *sign;
+	int sign;
 	struct acl acl[1];
 
 	INIT_OBJ(acl, VCC_ACL_MAGIC);
@@ -706,19 +706,20 @@ vcc_ParseAcl(struct vcc *tl)
 	ERRCHK(tl);
 	AN(sym);
 
-	while (tl->t->tok == '-' || tl->t->tok == '+') {
-		sign = tl->t;
-		vcc_NextToken(tl);
-		if (tl->t->b != sign->e) {
-			VSB_cat(tl->sb, "Expected ACL flag after:\n");
-			vcc_ErrWhere(tl, sign);
+	while (1) {
+		sign = vcc_IsFlag(tl);
+		if (tl->err) {
+			VSB_cat(tl->sb,
+			    "Valid ACL flags are `log` and `table`:\n");
 			return;
 		}
+		if (sign < 0)
+			break;
 		if (vcc_IdIs(tl->t, "log")) {
-			acl->flag_log = sign->tok == '+';
+			acl->flag_log = sign;
 			vcc_NextToken(tl);
 		} else if (vcc_IdIs(tl->t, "table")) {
-			acl->flag_table = sign->tok == '+';
+			acl->flag_table = sign;
 			vcc_NextToken(tl);
 		} else {
 			VSB_cat(tl->sb, "Unknown ACL flag:\n");
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 76e923693..995399ab6 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -431,6 +431,7 @@ double vcc_DurationUnit(struct vcc *);
 void vcc_ByteVal(struct vcc *, double *);
 void vcc_Duration(struct vcc *tl, double *);
 unsigned vcc_UintVal(struct vcc *tl);
+int vcc_IsFlag(struct vcc *tl);
 
 /* vcc_var.c */
 sym_wildcard_t vcc_Var_Wildcard;
diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c
index 9cc27c68b..6ecee46fb 100644
--- a/lib/libvcc/vcc_utils.c
+++ b/lib/libvcc/vcc_utils.c
@@ -357,3 +357,22 @@ vcc_ByteVal(struct vcc *tl, double *d)
 	vcc_NextToken(tl);
 	*d = v * sc;
 }
+
+/*--------------------------------------------------------------------*/
+
+int
+vcc_IsFlag(struct vcc *tl)
+{
+	struct token *sign;
+
+	if (tl->t->tok != '-' && tl->t->tok != '+')
+		return (-1);
+	sign = tl->t;
+	vcc_NextToken(tl);
+	if (tl->t->b != sign->e) {
+		VSB_cat(tl->sb, "Expected a flag at:\n");
+		vcc_ErrWhere(tl, sign);
+		return (-1);
+	}
+	return (sign->tok == '+' ? 1 : 0);
+}

From phk at FreeBSD.org  Tue Apr 13 17:23:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 13 Apr 2021 17:23:05 +0000 (UTC)
Subject: [master] 1a02b4744 Add a vcc_IsFlagRaw() for when the tokens are not
 on tl->tokens.
Message-ID: <20210413172305.E3C8D73F0@lists.varnish-cache.org>


commit 1a02b4744f3b3777c800ced3d74c5b21f316abd0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Apr 13 17:20:08 2021 +0000

    Add a vcc_IsFlagRaw() for when the tokens are not on tl->tokens.

diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 995399ab6..7d01ae9c8 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -432,6 +432,7 @@ void vcc_ByteVal(struct vcc *, double *);
 void vcc_Duration(struct vcc *tl, double *);
 unsigned vcc_UintVal(struct vcc *tl);
 int vcc_IsFlag(struct vcc *tl);
+int vcc_IsFlagRaw(struct vcc *, const struct token *, const struct token *);
 
 /* vcc_var.c */
 sym_wildcard_t vcc_Var_Wildcard;
diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c
index 6ecee46fb..d60d932cd 100644
--- a/lib/libvcc/vcc_utils.c
+++ b/lib/libvcc/vcc_utils.c
@@ -361,18 +361,27 @@ vcc_ByteVal(struct vcc *tl, double *d)
 /*--------------------------------------------------------------------*/
 
 int
-vcc_IsFlag(struct vcc *tl)
+vcc_IsFlagRaw(struct vcc *tl, const struct token *t1, const struct token *t2)
 {
-	struct token *sign;
 
-	if (tl->t->tok != '-' && tl->t->tok != '+')
+	if (t1->tok != '-' && t1->tok != '+')
 		return (-1);
-	sign = tl->t;
-	vcc_NextToken(tl);
-	if (tl->t->b != sign->e) {
+	if (t2->b != t1->e) {
 		VSB_cat(tl->sb, "Expected a flag at:\n");
-		vcc_ErrWhere(tl, sign);
+		vcc_ErrWhere(tl, t1);
 		return (-1);
 	}
-	return (sign->tok == '+' ? 1 : 0);
+	return (t1->tok == '+' ? 1 : 0);
+}
+
+int
+vcc_IsFlag(struct vcc *tl)
+{
+	int retval;
+
+
+	retval = vcc_IsFlagRaw(tl, tl->t, VTAILQ_NEXT(tl->t, list));
+	if (retval >= 0)
+		vcc_NextToken(tl);
+	return (retval);
 }

From phk at FreeBSD.org  Tue Apr 13 17:23:06 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 13 Apr 2021 17:23:06 +0000 (UTC)
Subject: [master] 195929e4a Implement `+glob` flag to `include` processing.
Message-ID: <20210413172306.082F373F3@lists.varnish-cache.org>


commit 195929e4aa29f46c49038b3982b842a4d647a1d5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Apr 13 17:21:27 2021 +0000

    Implement `+glob` flag to `include` processing.
    
    Reimplementation of #3193 according to consensus.

diff --git a/lib/libvcc/vcc_source.c b/lib/libvcc/vcc_source.c
index 48f50553e..92d3ea7bd 100644
--- a/lib/libvcc/vcc_source.c
+++ b/lib/libvcc/vcc_source.c
@@ -31,6 +31,7 @@
 
 #include "config.h"
 
+#include <glob.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
@@ -99,41 +100,16 @@ vcc_file_source(const struct vcc *tl, const char *fn)
 
 /*--------------------------------------------------------------------*/
 
-static struct source *
+static int
 vcc_include_file(struct vcc *tl, const struct source *src_sp,
     const char *filename, const struct token *parent_token)
 {
 	struct source *sp;
 	const struct source *sp1;
-	struct vsb *vsb;
-	const char *p;
 
-	if (filename[0] == '.' && filename[1] == '/') {
-		/*
-		 * Nested include filenames, starting with "./" are
-		 * resolved relative to the VCL file which contains
-		 * the include directive.
-		 */
-		if (src_sp->name[0] != '/') {
-			VSB_cat(tl->sb,
-			    "include \"./xxxxx\"; needs absolute "
-			    "filename of including file.\n");
-			return (NULL);
-		}
-		vsb = VSB_new_auto();
-		AN(vsb);
-		p = strrchr(src_sp->name, '/');
-		AN(p);
-		VSB_bcat(vsb, src_sp->name, p - src_sp->name);
-		VSB_cat(vsb, filename + 1);
-		AZ(VSB_finish(vsb));
-		sp = vcc_file_source(tl, VSB_data(vsb));
-		VSB_destroy(&vsb);
-	} else {
-		sp = vcc_file_source(tl, filename);
-	}
+	sp = vcc_file_source(tl, filename);
 	if (sp == NULL)
-		return (sp);
+		return (-1);
 
 	for (sp1 = src_sp; sp1 != NULL; sp1 = sp1->parent)
 		if (!strcmp(sp1->name, sp->name))
@@ -142,24 +118,60 @@ vcc_include_file(struct vcc *tl, const struct source *src_sp,
 		VSB_printf(tl->sb,
 		    "Recursive include of \"%s\"\n\n", sp->name);
 		vcc_destroy_source(&sp);
-		return (NULL);
+		return (-1);
 	}
 	sp->parent = src_sp;
 	VTAILQ_INSERT_TAIL(&tl->sources, sp, list);
 	sp->idx = tl->nsources++;
 	sp->parent_tok = parent_token;
 	vcc_lex_source(tl, sp, 0);
-	return (sp);
+	return (0);
 }
 
 /*--------------------------------------------------------------------*/
 
+static int
+vcc_include_glob_file(struct vcc *tl, const struct source *src_sp,
+    const char *filename, const struct token *parent_token)
+{
+	glob_t	g[1];
+	int rv;
+	unsigned u;
+
+	memset(g, 0, sizeof g);
+	rv = glob(filename, 0, NULL, g);
+	switch (rv) {
+	case 0:
+		for (u = 0; rv == 0 && u < g->gl_pathc; u++) {
+			rv = vcc_include_file(
+			    tl, src_sp, g->gl_pathv[u], parent_token);
+		}
+		break;
+	case GLOB_NOMATCH:
+		VSB_printf(tl->sb, "glob pattern matched no files.\n");
+		break;
+	default:
+		VSB_printf(tl->sb, "glob(3) expansion failed (%d)\n", rv);
+		break;
+	}
+	globfree(g);
+	return (rv);
+}
+
+/*--------------------------------------------------------------------
+ * NB: We cannot use vcc_ErrWhere2() on tokens which are no on the
+ * NB: tl->tokens list.
+ */
+
 void
 vcc_lex_source(struct vcc *tl, struct source *src_sp, int eoi)
 {
 	struct token *t, *t1;
-	struct source *sp;
+	int i, glob_flag = 0;
 	const struct source *sp1;
+	struct vsb *vsb = NULL;
+	const char *filename;
+	const char *p;
 
 	vcc_Lexer(tl, src_sp);
 	if (tl->err)
@@ -174,27 +186,72 @@ vcc_lex_source(struct vcc *tl, struct source *src_sp, int eoi)
 		}
 
 		t1 = VTAILQ_NEXT(t, src_list);
-		AN(t1);			/* There's always an EOI */
+		AN(t1);
+
+		while (1) {
+			t = VTAILQ_NEXT(t1, src_list);
+			AN(t);
+			i = vcc_IsFlagRaw(tl, t1, t);
+			if (i < 0)
+				break;
+			if (vcc_IdIs(t, "glob")) {
+				glob_flag = i;
+			} else {
+				VSB_cat(tl->sb, "Unknown include flag:\n");
+				vcc_ErrWhere(tl, t);
+				return;
+			}
+			t1 = VTAILQ_NEXT(t, src_list);
+			AN(t1);
+		}
+
 		if (t1->tok != CSTR) {
 			VSB_cat(tl->sb,
 			    "include not followed by string constant.\n");
-			/* Not vcc_ErrWhere2(): tokens not on final list */
 			vcc_ErrWhere(tl, t1);
 			return;
 		}
 		t = VTAILQ_NEXT(t1, src_list);
-		AN(t);			/* There's always an EOI */
+		AN(t);
 
 		if (t->tok != ';') {
 			VSB_cat(tl->sb,
 			    "include <string> not followed by semicolon.\n");
-			/* Not vcc_ErrWhere2(): tokens not on final list */
 			vcc_ErrWhere(tl, t1);
 			return;
 		}
 
-		sp = vcc_include_file(tl, src_sp, t1->dec, t1);
-		if (sp == NULL) {
+		filename = t1->dec;
+
+		if (filename[0] == '.' && filename[1] == '/') {
+			/*
+			 * Nested include filenames, starting with "./" are
+			 * resolved relative to the VCL file which contains
+			 * the include directive.
+			 */
+			if (src_sp->name[0] != '/') {
+				VSB_cat(tl->sb,
+				    "include \"./xxxxx\"; needs absolute "
+				    "filename of including file.\n");
+				return;
+			}
+			vsb = VSB_new_auto();
+			AN(vsb);
+			p = strrchr(src_sp->name, '/');
+			AN(p);
+			VSB_bcat(vsb, src_sp->name, p - src_sp->name);
+			VSB_cat(vsb, filename + 1);
+			AZ(VSB_finish(vsb));
+			filename = VSB_data(vsb);
+		}
+
+		if (glob_flag)
+			i = vcc_include_glob_file(tl, src_sp, filename, t1);
+		else
+			i = vcc_include_file(tl, src_sp, filename, t1);
+		if (vsb != NULL)
+			VSB_destroy(&vsb);
+		if (i) {
 			vcc_ErrWhere(tl, t1);
 			for (sp1 = src_sp; sp1 != NULL; sp1 = sp1->parent) {
 				if (sp1->parent_tok == NULL)

From phk at FreeBSD.org  Wed Apr 14 08:19:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 14 Apr 2021 08:19:05 +0000 (UTC)
Subject: [master] 8084cebcb Fix white-space copy&paste-o
Message-ID: <20210414081905.E25C8A5B2D@lists.varnish-cache.org>


commit 8084cebcb539a4e344272b121177552ebfb7b5c7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Apr 14 08:17:55 2021 +0000

    Fix white-space copy&paste-o

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 76b427418..d31065245 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -155,23 +155,23 @@ static const uint8_t *
 bbsearch(int fam, const uint8_t *key, const uint8_t *base0,
     size_t nmemb, size_t size)
 {
-        const uint8_t *base = base0;
-        size_t lim;
-        int cmp;
-        const uint8_t *p;
-
-        for (lim = nmemb; lim != 0; lim >>= 1) {
-                p = base + (lim >> 1) * size;
-                cmp = acl_tbl_cmp(fam, key, p);
-                if (cmp == 0)
-                        return (p);
-                if (cmp > 0) {
+	const uint8_t *base = base0;
+	size_t lim;
+	int cmp;
+	const uint8_t *p;
+
+	for (lim = nmemb; lim != 0; lim >>= 1) {
+		p = base + (lim >> 1) * size;
+		cmp = acl_tbl_cmp(fam, key, p);
+		if (cmp == 0)
+			return (p);
+		if (cmp > 0) {
 			/* key > p: move right */
-                        base = p + size;
-                        lim--;
-                } /* else move left */
-        }
-        return (NULL);
+			base = p + size;
+			lim--;
+		} /* else move left */
+	}
+	return (NULL);
 }
 
 int

From dridi.boukelmoune at gmail.com  Wed Apr 14 09:41:04 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 14 Apr 2021 09:41:04 +0000 (UTC)
Subject: [master] f254fff7e circleci: Disable developer warnings for alpine
Message-ID: <20210414094104.C95A2A8289@lists.varnish-cache.org>


commit f254fff7e9682f4c0bb2e0636a07945a5dd89b83
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Apr 14 11:35:59 2021 +0200

    circleci: Disable developer warnings for alpine
    
    This is a temporary measure until #3565 is solved.
    
    Refs f44f16cc4696e9e7f434dfd6d67b4e0447b4e97b
    Refs #3568

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 09d0f2db1..163de78dd 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -296,7 +296,7 @@ workflows:
           name: distcheck_alpine
           dist: alpine
           release: "latest"
-          #extra_conf: --without-jemalloc
+          extra_conf: --disable-developer-warnings
       - distcheck:
           name: distcheck_archlinux
           dist: archlinux

From dridi.boukelmoune at gmail.com  Thu Apr 15 08:24:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 15 Apr 2021 08:24:05 +0000 (UTC)
Subject: [master] 26e92d158 circleci: Pass configure options to distcheck
Message-ID: <20210415082405.E8BD3AF8D8@lists.varnish-cache.org>


commit 26e92d158080f34347d529b3952d0e6d4dd94f22
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Apr 14 22:46:33 2021 +0200

    circleci: Pass configure options to distcheck
    
    Otherwise any futile attempt at disabling developer warnings (even
    temporarily) will be neutered by the distcheck target.
    
    Refs f254fff7e9682f4c0bb2e0636a07945a5dd89b83
    Closes #3592

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 163de78dd..61e9a4726 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -13,6 +13,14 @@ parameters:
   pkg-commit:
     type: string
     default: "master"
+  configure_args:
+    type: string
+    default: |
+        --with-unwind \
+        --enable-developer-warnings \
+        --enable-debugging-symbols \
+        --disable-stack-protector \
+        --with-persistent-storage \
 jobs:
   dist:
     description: Builds varnish-x.y.z.tar.gz that is used later for the packaging jobs
@@ -247,16 +255,14 @@ jobs:
                 autoreconf -i -v
             sudo -u varnish \
                 ./configure \
-            	--quiet \
-            	--with-unwind \
-            	--enable-developer-warnings \
-            	--enable-debugging-symbols \
-            	--disable-stack-protector \
-            	--with-persistent-storage \
-            	<< parameters.extra_conf >>
+                --quiet \
+                << pipeline.parameters.configure_args >> \
+                << parameters.extra_conf >>
             sudo -u varnish \
-            	--preserve-env=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \
-            	make distcheck VERBOSE=1 -j 12 -k
+                --preserve-env=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \
+                make distcheck VERBOSE=1 -j 12 -k \
+                DISTCHECK_CONFIGURE_FLAGS="<< pipeline.parameters.configure_args >> \
+                << parameters.extra_conf >>"
 
   collect_packages:
     docker:

From dridi.boukelmoune at gmail.com  Thu Apr 15 17:11:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 15 Apr 2021 17:11:06 +0000 (UTC)
Subject: [master] f25da8efb vmod_debug: Plug minor ACL leak
Message-ID: <20210415171106.308B395F2A@lists.varnish-cache.org>


commit f25da8efb8822802774262ac83ac310c0566f2b6
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Apr 15 16:25:49 2021 +0200

    vmod_debug: Plug minor ACL leak
    
    Spotted by lsan.

diff --git a/vmod/vmod_debug_acl.c b/vmod/vmod_debug_acl.c
index 7de562743..7d708f00d 100644
--- a/vmod/vmod_debug_acl.c
+++ b/vmod/vmod_debug_acl.c
@@ -221,6 +221,7 @@ xyzzy_time_acl(VRT_CTX, VCL_ACL acl, VCL_IP ip0, VCL_IP ip1,
 {
 	struct acl_sweep asw[1];
 	vtim_mono t0, t1;
+	vtim_dur d;
 	VCL_INT cnt;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
@@ -244,9 +245,12 @@ xyzzy_time_acl(VRT_CTX, VCL_ACL acl, VCL_IP ip0, VCL_IP ip1,
 		} while (step_sweep(asw) <= 0);
 	}
 	t1 = VTIM_mono();
+	cnt = asw->count;
+	assert(cnt > 0);
+	d = (t1 - t0) / cnt;
 	VSLb(ctx->vsl, SLT_Debug,
 	    "Timed ACL: %.9f -> %.9f = %.9f %.9f/round, %.9f/IP %ju IPs",
-	    t0, t1, t1 - t0, (t1-t0) / turnus, (t1-t0) / asw->count,
-	    (uintmax_t)asw->count);
-	return ((t1 - t0) / asw->count);
+	    t0, t1, t1 - t0, (t1-t0) / turnus, d, (intmax_t)cnt);
+	cleanup_sweep(asw);
+	return (d);
 }

From dridi.boukelmoune at gmail.com  Thu Apr 15 17:11:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 15 Apr 2021 17:11:06 +0000 (UTC)
Subject: [master] 0a6bf60d3 vev: Tear down the list of events
Message-ID: <20210415171106.4899D95F2F@lists.varnish-cache.org>


commit 0a6bf60d39c514263c91839db5beb283ec008443
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Apr 15 18:50:21 2021 +0200

    vev: Tear down the list of events
    
    It was either this or adding a suppression for lsan.

diff --git a/lib/libvarnish/vev.c b/lib/libvarnish/vev.c
index f83832c6f..ab3544899 100644
--- a/lib/libvarnish/vev.c
+++ b/lib/libvarnish/vev.c
@@ -219,12 +219,17 @@ void
 VEV_Destroy(struct vev_root **evbp)
 {
 	struct vev_root *evb;
+	struct vev *e;
 
 	TAKE_OBJ_NOTNULL(evb, evbp, VEV_BASE_MAGIC);
 	assert(pthread_equal(evb->thread, pthread_self()));
+	while ((e = VBH_root(evb->binheap)) != NULL) {
+		VEV_Stop(evb, e);
+		free(e);
+	}
+	VBH_destroy(&evb->binheap);
 	free(evb->pfd);
 	free(evb->pev);
-	/* XXX: destroy evb->binheap */
 	FREE_OBJ(evb);
 }
 

From dridi.boukelmoune at gmail.com  Thu Apr 15 17:11:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 15 Apr 2021 17:11:06 +0000 (UTC)
Subject: [master] 0e7fe4a9c lsan: Ignore leaks attributed to readline...
Message-ID: <20210415171106.66EF195F33@lists.varnish-cache.org>


commit 0e7fe4a9c6a3006257957ef6fad8466bd793bb48
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Apr 15 18:57:50 2021 +0200

    lsan: Ignore leaks attributed to readline...
    
    ... on my machine.

diff --git a/tools/lsan.suppr b/tools/lsan.suppr
index f9ca2be93..cce56c4f7 100644
--- a/tools/lsan.suppr
+++ b/tools/lsan.suppr
@@ -18,3 +18,6 @@ leak:vsmw_newcluster
 leak:smp_thread
 #
 leak:Lck_Witness_Lock
+# libedit
+leak:rl_callback_read_char
+leak:wcsdup

From dridi.boukelmoune at gmail.com  Thu Apr 15 17:11:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 15 Apr 2021 17:11:06 +0000 (UTC)
Subject: [master] 5891cb0b6 vdef: Centralize __has_feature(...) fallback
Message-ID: <20210415171106.8C84895F38@lists.varnish-cache.org>


commit 5891cb0b63562bfa1d7f0022343ea18a100c5d10
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Apr 15 19:07:01 2021 +0200

    vdef: Centralize __has_feature(...) fallback

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 6ead58a5f..24c5bfcda 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -45,10 +45,6 @@
 #include "vav.h"
 #include "vcli_serve.h"
 
-#if !defined(__has_feature)
-#define __has_feature(x)	0
-#endif
-
 struct plist {
 	unsigned			magic;
 #define PLIST_MAGIC			0xbfc3ea16
diff --git a/include/vdef.h b/include/vdef.h
index 5ba66828e..d99c02570 100644
--- a/include/vdef.h
+++ b/include/vdef.h
@@ -79,6 +79,10 @@
 		*(fdp) = -1;			\
 	} while (0)
 
+#if !defined(__has_feature)
+#  define __has_feature(x)	0
+#endif
+
 #ifndef __GNUC_PREREQ__
 # if defined __GNUC__ && defined __GNUC_MINOR__
 #  define __GNUC_PREREQ__(maj, min) \
diff --git a/lib/libvarnish/vbh.c b/lib/libvarnish/vbh.c
index 511e9e7de..ce7bd5fd9 100644
--- a/lib/libvarnish/vbh.c
+++ b/lib/libvarnish/vbh.c
@@ -48,10 +48,6 @@
 #include "vas.h"
 #include "vbh.h"
 
-#if !defined(__has_feature)
-#define __has_feature(x)	0
-#endif
-
 /* Parameters --------------------------------------------------------*/
 
 /*

From dridi.boukelmoune at gmail.com  Thu Apr 15 17:11:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 15 Apr 2021 17:11:06 +0000 (UTC)
Subject: [master] b8b29be5b vtcp: Ignore EINTR with sanitizers
Message-ID: <20210415171106.AA0C495F3C@lists.varnish-cache.org>


commit b8b29be5ba4180ff7afb2a332aeb2fa543fd06dc
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Apr 15 19:07:34 2021 +0200

    vtcp: Ignore EINTR with sanitizers
    
    Technically it can also happen with a debugger attached to a process
    despite SA_RESTART.

diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index 4c99beac1..9de0389f2 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -628,6 +628,10 @@ VTCP_Check(ssize_t a)
 	 */
 	if (errno == EINVAL)
 		return (1);
+#endif
+#if (defined(__SANITIZER) || __has_feature(address_sanitizer))
+	if (errno == EINTR)
+		return (1);
 #endif
 	return (0);
 }

From phk at FreeBSD.org  Mon Apr 19 05:38:07 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 19 Apr 2021 05:38:07 +0000 (UTC)
Subject: [master] 2b1a9a65b Rename a variable from 't1' to 'tok1' to silence a
 false positive in Coverity.
Message-ID: <20210419053807.27923774C@lists.varnish-cache.org>


commit 2b1a9a65b603e2c066385e56d1d21063d83a759c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 19 05:37:16 2021 +0000

    Rename a variable from 't1' to 'tok1' to silence a false positive in Coverity.

diff --git a/lib/libvcc/vcc_source.c b/lib/libvcc/vcc_source.c
index 92d3ea7bd..521b5b51e 100644
--- a/lib/libvcc/vcc_source.c
+++ b/lib/libvcc/vcc_source.c
@@ -166,7 +166,7 @@ vcc_include_glob_file(struct vcc *tl, const struct source *src_sp,
 void
 vcc_lex_source(struct vcc *tl, struct source *src_sp, int eoi)
 {
-	struct token *t, *t1;
+	struct token *t, *tok1;
 	int i, glob_flag = 0;
 	const struct source *sp1;
 	struct vsb *vsb = NULL;
@@ -185,13 +185,13 @@ vcc_lex_source(struct vcc *tl, struct source *src_sp, int eoi)
 			continue;
 		}
 
-		t1 = VTAILQ_NEXT(t, src_list);
-		AN(t1);
+		tok1 = VTAILQ_NEXT(t, src_list);
+		AN(tok1);
 
 		while (1) {
-			t = VTAILQ_NEXT(t1, src_list);
+			t = VTAILQ_NEXT(tok1, src_list);
 			AN(t);
-			i = vcc_IsFlagRaw(tl, t1, t);
+			i = vcc_IsFlagRaw(tl, tok1, t);
 			if (i < 0)
 				break;
 			if (vcc_IdIs(t, "glob")) {
@@ -201,27 +201,27 @@ vcc_lex_source(struct vcc *tl, struct source *src_sp, int eoi)
 				vcc_ErrWhere(tl, t);
 				return;
 			}
-			t1 = VTAILQ_NEXT(t, src_list);
-			AN(t1);
+			tok1 = VTAILQ_NEXT(t, src_list);
+			AN(tok1);
 		}
 
-		if (t1->tok != CSTR) {
+		if (tok1->tok != CSTR) {
 			VSB_cat(tl->sb,
 			    "include not followed by string constant.\n");
-			vcc_ErrWhere(tl, t1);
+			vcc_ErrWhere(tl, tok1);
 			return;
 		}
-		t = VTAILQ_NEXT(t1, src_list);
+		t = VTAILQ_NEXT(tok1, src_list);
 		AN(t);
 
 		if (t->tok != ';') {
 			VSB_cat(tl->sb,
 			    "include <string> not followed by semicolon.\n");
-			vcc_ErrWhere(tl, t1);
+			vcc_ErrWhere(tl, tok1);
 			return;
 		}
 
-		filename = t1->dec;
+		filename = tok1->dec;
 
 		if (filename[0] == '.' && filename[1] == '/') {
 			/*
@@ -246,13 +246,13 @@ vcc_lex_source(struct vcc *tl, struct source *src_sp, int eoi)
 		}
 
 		if (glob_flag)
-			i = vcc_include_glob_file(tl, src_sp, filename, t1);
+			i = vcc_include_glob_file(tl, src_sp, filename, tok1);
 		else
-			i = vcc_include_file(tl, src_sp, filename, t1);
+			i = vcc_include_file(tl, src_sp, filename, tok1);
 		if (vsb != NULL)
 			VSB_destroy(&vsb);
 		if (i) {
-			vcc_ErrWhere(tl, t1);
+			vcc_ErrWhere(tl, tok1);
 			for (sp1 = src_sp; sp1 != NULL; sp1 = sp1->parent) {
 				if (sp1->parent_tok == NULL)
 					break;

From phk at FreeBSD.org  Mon Apr 19 09:25:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 19 Apr 2021 09:25:05 +0000 (UTC)
Subject: [master] f2b9846d2 Refactor to use director argument directly
Message-ID: <20210419092505.EFCE2651D8@lists.varnish-cache.org>


commit f2b9846d212dfd5bfc8b94637feff84741b4f282
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 19 09:23:54 2021 +0000

    Refactor to use director argument directly

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 90c7899a5..b67773f12 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -113,7 +113,7 @@ VBE_Connect_Error(struct VSC_vbe *vsc, int err)
  */
 
 static struct pfd *
-vbe_dir_getfd(VRT_CTX, struct worker *wrk, struct backend *bp,
+vbe_dir_getfd(VRT_CTX, struct worker *wrk, VCL_BACKEND dir, struct backend *bp,
     unsigned force_fresh)
 {
 	struct busyobj *bo;
@@ -129,9 +129,9 @@ vbe_dir_getfd(VRT_CTX, struct worker *wrk, struct backend *bp,
 	CHECK_OBJ_NOTNULL(bp, BACKEND_MAGIC);
 	AN(bp->vsc);
 
-	if (! VRT_Healthy(ctx, bp->director, NULL)) {
+	if (! VRT_Healthy(ctx, dir, NULL)) {
 		VSLb(bo->vsl, SLT_FetchError,
-		     "backend %s: unhealthy", VRT_BACKEND_string(bp->director));
+		     "backend %s: unhealthy", VRT_BACKEND_string(dir));
 		bp->vsc->unhealthy++;
 		VSC_C_main->backend_unhealthy++;
 		return (NULL);
@@ -139,7 +139,7 @@ vbe_dir_getfd(VRT_CTX, struct worker *wrk, struct backend *bp,
 
 	if (bp->max_connections > 0 && bp->n_conn >= bp->max_connections) {
 		VSLb(bo->vsl, SLT_FetchError,
-		     "backend %s: busy", VRT_BACKEND_string(bp->director));
+		     "backend %s: busy", VRT_BACKEND_string(dir));
 		bp->vsc->busy++;
 		VSC_C_main->backend_busy++;
 		return (NULL);
@@ -160,7 +160,7 @@ vbe_dir_getfd(VRT_CTX, struct worker *wrk, struct backend *bp,
 		VBE_Connect_Error(bp->vsc, err);
 		VSLb(bo->vsl, SLT_FetchError,
 		     "backend %s: fail errno %d (%s)",
-		     VRT_BACKEND_string(bp->director), err, vstrerror(err));
+		     VRT_BACKEND_string(dir), err, vstrerror(err));
 		VSC_C_main->backend_fail++;
 		bo->htc = NULL;
 		return (NULL);
@@ -183,7 +183,7 @@ vbe_dir_getfd(VRT_CTX, struct worker *wrk, struct backend *bp,
 	if (err < 0) {
 		VSLb(bo->vsl, SLT_FetchError,
 		     "backend %s: proxy write errno %d (%s)",
-		     VRT_BACKEND_string(bp->director),
+		     VRT_BACKEND_string(dir),
 		     errno, vstrerror(errno));
 		// account as if connect failed - good idea?
 		VSC_C_main->backend_fail++;
@@ -202,7 +202,7 @@ vbe_dir_getfd(VRT_CTX, struct worker *wrk, struct backend *bp,
 	PFD_LocalName(pfd, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1);
 	PFD_RemoteName(pfd, abuf2, sizeof abuf2, pbuf2, sizeof pbuf2);
 	VSLb(bo->vsl, SLT_BackendOpen, "%d %s %s %s %s %s %s",
-	    *fdp, VRT_BACKEND_string(bp->director), abuf2, pbuf2, abuf1, pbuf1,
+	    *fdp, VRT_BACKEND_string(dir), abuf2, pbuf2, abuf1, pbuf1,
 	    PFD_State(pfd) == PFD_STATE_STOLEN ? "reuse" : "connect");
 
 	INIT_OBJ(bo->htc, HTTP_CONN_MAGIC);
@@ -242,14 +242,14 @@ vbe_dir_finish(VRT_CTX, VCL_BACKEND d)
 	}
 	if (bo->htc->doclose != SC_NULL || bp->proxy_header != 0) {
 		VSLb(bo->vsl, SLT_BackendClose, "%d %s close", *PFD_Fd(pfd),
-		    VRT_BACKEND_string(bp->director));
+		    VRT_BACKEND_string(d));
 		VCP_Close(&pfd);
 		AZ(pfd);
 		Lck_Lock(&bp->mtx);
 	} else {
 		assert (PFD_State(pfd) == PFD_STATE_USED);
 		VSLb(bo->vsl, SLT_BackendClose, "%d %s recycle", *PFD_Fd(pfd),
-		    VRT_BACKEND_string(bp->director));
+		    VRT_BACKEND_string(d));
 		Lck_Lock(&bp->mtx);
 		VSC_C_main->backend_recycle++;
 		VCP_Recycle(bo->wrk, &pfd);
@@ -290,7 +290,7 @@ vbe_dir_gethdrs(VRT_CTX, VCL_BACKEND d)
 		http_PrintfHeader(bo->bereq, "Host: %s", bp->hosthdr);
 
 	do {
-		pfd = vbe_dir_getfd(ctx, wrk, bp, extrachance == 0 ? 1 : 0);
+		pfd = vbe_dir_getfd(ctx, wrk, d, bp, extrachance == 0 ? 1 : 0);
 		if (pfd == NULL)
 			return (-1);
 		AN(bo->htc);
@@ -375,7 +375,7 @@ vbe_dir_http1pipe(VRT_CTX, VCL_BACKEND d)
 
 	ctx->req->res_mode = RES_PIPE;
 
-	pfd = vbe_dir_getfd(ctx, ctx->req->wrk, bp, 0);
+	pfd = vbe_dir_getfd(ctx, ctx->req->wrk, d, bp, 0);
 
 	if (pfd == NULL) {
 		retval = SC_TX_ERROR;

From dridi.boukelmoune at gmail.com  Mon Apr 19 17:15:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 19 Apr 2021 17:15:06 +0000 (UTC)
Subject: [master] 2376c076d varnishtest: Increase MAX_HDR to 64
Message-ID: <20210419171506.433D7A0DE1@lists.varnish-cache.org>


commit 2376c076d810bdf2816cb94cfb4bea32c69c544a
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Apr 19 19:11:34 2021 +0200

    varnishtest: Increase MAX_HDR to 64
    
    Closes #3574

diff --git a/bin/varnishtest/vtc_http.h b/bin/varnishtest/vtc_http.h
index 6e5bcbdf5..e45b40469 100644
--- a/bin/varnishtest/vtc_http.h
+++ b/bin/varnishtest/vtc_http.h
@@ -28,7 +28,7 @@
  * SUCH DAMAGE.
  */
 
-#define MAX_HDR		50
+#define MAX_HDR		64
 
 struct vtc_sess {
 	unsigned		magic;

From phk at FreeBSD.org  Mon Apr 19 20:17:10 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 19 Apr 2021 20:17:10 +0000 (UTC)
Subject: [master] ea8a98070 Backends created on demand must have their probe
 init to healthy.
Message-ID: <20210419201710.C1FF8A64E6@lists.varnish-cache.org>


commit ea8a9807039752fe9110a1f0c3dea34084c534dc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 19 19:17:42 2021 +0000

    Backends created on demand must have their probe init to healthy.
    
    Fixes #3596

diff --git a/bin/varnishtest/tests/d00007.vtc b/bin/varnishtest/tests/d00007.vtc
index 93c0a0c7f..2c05693dc 100644
--- a/bin/varnishtest/tests/d00007.vtc
+++ b/bin/varnishtest/tests/d00007.vtc
@@ -14,7 +14,10 @@ varnish v1 -vcl {
 
 	backend dummy { .host = "${bad_backend}"; }
 
-	probe pr {}
+	probe pr {
+		.threshold = 8;
+		.initial = 8;
+	}
 
 	sub vcl_init {
 		new s1 = debug.dyn("0.0.0.0", "0");
diff --git a/doc/sphinx/reference/vcl-probe.rst b/doc/sphinx/reference/vcl-probe.rst
index 569c83566..6d05e48ae 100644
--- a/doc/sphinx/reference/vcl-probe.rst
+++ b/doc/sphinx/reference/vcl-probe.rst
@@ -160,6 +160,11 @@ The default values are:
 
 * ``.initial`` = one less than ``.threshold``
 
+Note that the default ``.initial`` means that the backend will be marked
+unhealthy until the first probe response come back successful.
+This means that for backends created on demand (by vmods) cannot use the
+default value for ``.initial``, as the freshly created backend would very
+likely still be unhealthy when the backend request happens.
 
 SEE ALSO
 ========

From phk at FreeBSD.org  Mon Apr 19 20:17:10 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 19 Apr 2021 20:17:10 +0000 (UTC)
Subject: [master] 9905baa5a Minor polish.
Message-ID: <20210419201710.AEF70A6486@lists.varnish-cache.org>


commit 9905baa5a67c1ab29ae3ceca12c89ee412197573
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Apr 19 10:21:04 2021 +0000

    Minor polish.

diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c
index e4181bcce..d1750ac63 100644
--- a/bin/varnishd/cache/cache_backend_probe.c
+++ b/bin/varnishd/cache/cache_backend_probe.c
@@ -181,9 +181,9 @@ VBP_Update_Backend(struct vbp_target *vt)
 	chg = (i != vt->backend->sick);
 	vt->backend->sick = i;
 
-	AN(dir->vcl_name);
+	AN(vt->backend->vcl_name);
 	VSL(SLT_Backend_health, 0, "%s %s %s %s %u %u %u %.6f %.6f \"%s\"",
-	    dir->vcl_name, chg ? "Went" : "Still",
+	    vt->backend->vcl_name, chg ? "Went" : "Still",
 	    i ? "sick" : "healthy", bits,
 	    vt->good, vt->threshold, vt->window,
 	    vt->last, vt->avg, vt->resp_buf);
diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c
index c6fd40d96..173ad6604 100644
--- a/bin/varnishd/cache/cache_director.c
+++ b/bin/varnishd/cache/cache_director.c
@@ -257,15 +257,11 @@ VRT_Healthy(VRT_CTX, VCL_BACKEND d, VCL_TIME *changed)
 VCL_VOID
 VRT_SetChanged(VCL_BACKEND d, VCL_TIME changed)
 {
-	if (d == NULL)
-		return;
 
-	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-
-	if (changed <= d->vdir->health_changed)
-		return;
+	CHECK_OBJ_ORNULL(d, DIRECTOR_MAGIC);
 
-	d->vdir->health_changed = changed;
+	if (d != NULL && changed > d->vdir->health_changed)
+		d->vdir->health_changed = changed;
 }
 
 /* Send Event ----------------------------------------------------------

From reza at naghibi.com  Tue Apr 20 18:11:07 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:11:07 +0000 (UTC)
Subject: [6.0] 73262b47c Clear the status and response on a retry
Message-ID: <20210420181107.DC252A4964@lists.varnish-cache.org>


commit 73262b47c8e07597fcf9965fdc4bc5cf28e68ec1
Author: Steven <wojcikjsteven at gmail.com>
Date:   Thu Feb 18 12:06:53 2021 -0500

    Clear the status and response on a retry
    
    The bo fields err_code and err_reason need to be reset on a retry otherwise the values are kept.
    
    Fixes #3525

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 6381bcfaf..026b61452 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -288,6 +288,8 @@ vbf_stp_retry(struct worker *wrk, struct busyobj *bo)
 	bo->do_esi = 0;
 	bo->do_stream = 1;
 	bo->was_304 = 0;
+	bo->err_code = 0;
+	bo->err_reason = NULL;
 
 	// XXX: BereqEnd + BereqAcct ?
 	VSL_ChgId(bo->vsl, "bereq", "retry", VXID_Get(wrk, VSL_BACKENDMARKER));
diff --git a/bin/varnishtest/tests/r03525.vtc b/bin/varnishtest/tests/r03525.vtc
new file mode 100644
index 000000000..75011e81e
--- /dev/null
+++ b/bin/varnishtest/tests/r03525.vtc
@@ -0,0 +1,24 @@
+varnishtest "Clear beresp status and reason on a retry"
+
+server s1 {
+	rxreq
+	txresp -status 500 -reason "my reason"
+} -start
+
+varnish v1 -arg "-p first_byte_timeout=0.2" -vcl+backend {
+	sub vcl_backend_response {
+		return (error(beresp.status, beresp.reason));
+	}
+	sub vcl_backend_error {
+		if (bereq.retries == 0) {
+			return (retry);
+		}
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 503
+	expect resp.reason == "Backend fetch failed"
+} -run

From reza at naghibi.com  Tue Apr 20 18:13:05 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:13:05 +0000 (UTC)
Subject: [6.0] 5a4a8ba5f Make sure resp.reason is on workspace before using it
 in vcl_synth
Message-ID: <20210420181305.7FDB9A4BFD@lists.varnish-cache.org>


commit 5a4a8ba5fa137985cddf5dfca25f935ac356034a
Author: Reza Naghibi <reza at naghibi.com>
Date:   Mon Mar 8 11:27:16 2021 -0500

    Make sure resp.reason is on workspace before using it in vcl_synth
    
    We can incorrectly reference resp.reason from other sources when
    jumping into vcl_synth. This also covers passing in a reason in
    vcl_backend_error.

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 20d855478..27c80fb5d 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -76,6 +76,14 @@ VRT_synth(VRT_CTX, VCL_INT code, VCL_STRING reason)
 		return;
 	}
 
+	if (reason && !WS_Inside(ctx->ws, reason, NULL)) {
+		reason = WS_Copy(ctx->ws, reason, -1);
+		if (!reason) {
+			VRT_fail(ctx, "Workspace overflow");
+			return;
+		}
+	}
+
 	if (ctx->req == NULL) {
 		CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
 		ctx->bo->err_code = (uint16_t)code;
diff --git a/bin/varnishtest/tests/r03546.vtc b/bin/varnishtest/tests/r03546.vtc
new file mode 100644
index 000000000..20cf8a85b
--- /dev/null
+++ b/bin/varnishtest/tests/r03546.vtc
@@ -0,0 +1,21 @@
+varnishtest "Synth resp.reason race"
+
+varnish v1 -vcl {
+	backend default none;
+
+	sub vcl_backend_error {
+		set beresp.status = 500;
+		set beresp.reason = "VCL";
+	}
+
+	sub vcl_deliver {
+		return (synth(resp.status, resp.reason));
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 500
+	expect resp.reason == "VCL"
+} -run

From reza at naghibi.com  Tue Apr 20 18:14:05 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:14:05 +0000 (UTC)
Subject: [6.0] a1cedd380 Guard against OA_GZIPBITS streaming race
Message-ID: <20210420181405.5E099A4E09@lists.varnish-cache.org>


commit a1cedd380481f13c3468abcb2d35630372ffc6a8
Author: Reza Naghibi <reza at naghibi.com>
Date:   Wed Feb 13 10:45:40 2019 -0500

    Guard against OA_GZIPBITS streaming race

diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index b325e79c5..732c8db62 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -292,6 +292,10 @@ vdp_gunzip_init(struct req *req, void **priv)
 	ssize_t dl;
 	uint64_t u;
 
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
+	CHECK_OBJ_ORNULL(req->objcore->boc, BOC_MAGIC);
+
 	vg = VGZ_NewGunzip(req->vsl, "U D -");
 	AN(vg);
 	if (vgz_getmbuf(vg)) {
@@ -307,6 +311,10 @@ vdp_gunzip_init(struct req *req, void **priv)
 
 	req->resp_len = -1;
 
+	/* OA_GZIPBITS is not stable yet */
+	if (req->objcore->boc)
+		return (0);
+
 	p = ObjGetAttr(req->wrk, req->objcore, OA_GZIPBITS, &dl);
 	if (p != NULL && dl == 32) {
 		u = vbe64dec(p + 24);

From reza at naghibi.com  Tue Apr 20 18:14:05 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:14:05 +0000 (UTC)
Subject: [6.0] 4fe90267a Skip magic check when on boc when gunzip streaming
Message-ID: <20210420181405.79077A4E0C@lists.varnish-cache.org>


commit 4fe90267ada64b880eac5cd72285c56b66a7ad10
Author: Reza Naghibi <reza at naghibi.com>
Date:   Tue May 14 13:12:22 2019 -0400

    Skip magic check when on boc when gunzip streaming
    
    We do not hold a reference, the magic can be unstable.

diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index 732c8db62..3085a384a 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -294,7 +294,6 @@ vdp_gunzip_init(struct req *req, void **priv)
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
-	CHECK_OBJ_ORNULL(req->objcore->boc, BOC_MAGIC);
 
 	vg = VGZ_NewGunzip(req->vsl, "U D -");
 	AN(vg);

From reza at naghibi.com  Tue Apr 20 18:15:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:15:06 +0000 (UTC)
Subject: [6.0] 69d1e52d8 round-robin director to consider all backends also
 when racing
Message-ID: <20210420181506.B55CDA517E@lists.varnish-cache.org>


commit 69d1e52d87c04b28227c41426d5c8330a05cdee3
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Dec 3 15:15:20 2020 +0100

    round-robin director to consider all backends also when racing
    
    When resolve requests race, we were not guaranteed to consider all
    backends because we updated a shared nxt variable.
    
    Fixes #3474

diff --git a/lib/libvmod_directors/round_robin.c b/lib/libvmod_directors/round_robin.c
index 767fe7258..ac73f9b84 100644
--- a/lib/libvmod_directors/round_robin.c
+++ b/lib/libvmod_directors/round_robin.c
@@ -69,14 +69,16 @@ vmod_rr_resolve(const struct director *dir, struct worker *wrk,
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_ROUND_ROBIN_MAGIC);
 	vdir_rdlock(rr->vd);
+	nxt = rr->nxt;
 	for (u = 0; u < rr->vd->n_backend; u++) {
-		nxt = rr->nxt % rr->vd->n_backend;
-		rr->nxt = nxt + 1;
 		be = rr->vd->backend[nxt];
+		nxt++;
+		nxt %= rr->vd->n_backend;
 		CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC);
 		if (be->healthy(be, bo, NULL))
 			break;
 	}
+	rr->nxt = nxt;
 	vdir_unlock(rr->vd);
 	if (u == rr->vd->n_backend)
 		be = NULL;

From reza at naghibi.com  Tue Apr 20 18:16:05 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:16:05 +0000 (UTC)
Subject: [6.0] bb73128a9 vbf: Prevent pooling of a Connection:close beresp
Message-ID: <20210420181605.C9B88A5390@lists.varnish-cache.org>


commit bb73128a90f69e6b265f2d0ff8d6ca2ec11a0f55
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Sep 8 11:36:52 2020 +0200

    vbf: Prevent pooling of a Connection:close beresp
    
    Whether the header was set by the backend or directly in VCL, it is now
    possible to signal that a backend connection should not be added back to
    the pool after a successful fetch with a Connection:close header.
    
    Pooling such a connection would be counter-productive if closing the
    session was requested by the backend itself, because it would then be
    likely that reusing the connection would result in busting the extra
    chance.
    
    Setting the Connection:close directly in VCL can help mitigating against
    a misbehaving backend.
    
    Refs #3400

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 026b61452..cc39e16cd 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -437,6 +437,10 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 
 	VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL);
 
+	if (bo->htc != NULL && bo->htc->doclose == SC_NULL &&
+	    http_GetHdrField(bo->beresp, H_Connection, "close", NULL))
+		bo->htc->doclose = SC_RESP_CLOSE;
+
 	if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL ||
 	    wrk->handling == VCL_RET_ERROR) {
 		if (bo->htc)
diff --git a/bin/varnishtest/tests/b00073.vtc b/bin/varnishtest/tests/b00073.vtc
new file mode 100644
index 000000000..6cae03450
--- /dev/null
+++ b/bin/varnishtest/tests/b00073.vtc
@@ -0,0 +1,39 @@
+varnishtest "backend connection close"
+
+server s1 {
+	rxreq
+	expect req.http.beresp-connection ~ close
+	txresp
+	expect_close
+
+	accept
+	rxreq
+	expect req.http.beresp-connection !~ close
+	txresp -hdr "connection: close"
+	expect_close
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		return (pass);
+	}
+	sub vcl_backend_response {
+		# NB: this overrides unconditionally on purpose
+		set beresp.http.connection = bereq.http.beresp-connection;
+	}
+} -start
+
+client c1 {
+	txreq -hdr "beresp-connection: close, x-varnish"
+	rxresp
+	expect resp.status == 200
+
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run
+
+server s1 -wait
+
+varnish v1 -expect MAIN.backend_recycle == 0
+varnish v1 -expect VBE.vcl1.s1.conn == 0

From reza at naghibi.com  Tue Apr 20 18:16:05 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:16:05 +0000 (UTC)
Subject: [6.0] 4297c891f doc: Mention the effect of Connection:close in beresp
Message-ID: <20210420181605.E3080A5393@lists.varnish-cache.org>


commit 4297c891f258480c4c7a73e3430a2ae8529c7aae
Author: Steven <wojcikjsteven at gmail.com>
Date:   Mon Apr 12 14:18:32 2021 -0400

    doc: Mention the effect of Connection:close in beresp
    
    Original author: @dridi
    
    This commit is from ba78ebeb28105290bef56008f52bf63e58d5a894 but
    with no analogous file in 6.0, it is added to changes-6.0.rst instead of changes-trunk.rst

diff --git a/doc/sphinx/whats-new/changes-6.0.rst b/doc/sphinx/whats-new/changes-6.0.rst
index 41fd5052e..c8b28086a 100644
--- a/doc/sphinx/whats-new/changes-6.0.rst
+++ b/doc/sphinx/whats-new/changes-6.0.rst
@@ -50,6 +50,31 @@ this should make life simpler for everybody we hope.
 
 And it goes without saying that we have fixed a lot of bugs too.
 
+Changes to VCL
+==============
+
+VCL variables
+~~~~~~~~~~~~~
+
+It is now possible to manually set a ``Connection: close`` header in
+``beresp`` to signal that the backend connection shouldn't be recycled.
+This might help dealing with backends that would under certain circumstances
+have trouble managing their end of the connection, for example for certain
+kinds of resources.
+
+Care should be taken to preserve other headers listed in the connection
+header::
+
+    sub vcl_backend_response {
+        if (beresp.backend == faulty_backend) {
+            if (beresp.http.Connection) {
+                set beresp.http.Connection += ", close";
+            } else {
+                set beresp.http.Connection = "close";
+            }
+        }
+    }
+
 
 Under the hood (mostly for developers)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

From reza at naghibi.com  Tue Apr 20 18:16:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:16:06 +0000 (UTC)
Subject: [6.0] 8b36d0387 vbf: Prevent pooling of a Connection:close bereq
Message-ID: <20210420181606.09C25A5397@lists.varnish-cache.org>


commit 8b36d0387d3a7f99b218cff2ede4198c57487eb8
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Sep 29 08:14:58 2020 +0200

    vbf: Prevent pooling of a Connection:close bereq
    
    Once we ask the backend to close the connection after a given request
    there is no benefit from putting the backend connection back in the
    pool. It's actually a surefire way to force a subsequent backend fetch
    to fail its first attempt and go straight to its extra chance.
    
    Since we try to maximize connection reuse this would have to come from
    VCL and a user asking for the backend to close the connection should
    have a good reason to do so, for example when the backend is known to
    misbehave under certain circumstances.
    
    Closes #3400
    Refs #3405

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index cc39e16cd..63d3564fe 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -435,6 +435,10 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	if (http_IsStatus(bo->beresp, 304) && vbf_304_logic(bo) < 0)
 		return (F_STP_ERROR);
 
+	if (bo->htc->doclose == SC_NULL &&
+	    http_GetHdrField(bo->bereq, H_Connection, "close", NULL))
+		bo->htc->doclose = SC_REQ_CLOSE;
+
 	VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL);
 
 	if (bo->htc != NULL && bo->htc->doclose == SC_NULL &&
diff --git a/bin/varnishtest/tests/b00073.vtc b/bin/varnishtest/tests/b00073.vtc
index 6cae03450..3372ba094 100644
--- a/bin/varnishtest/tests/b00073.vtc
+++ b/bin/varnishtest/tests/b00073.vtc
@@ -2,28 +2,54 @@ varnishtest "backend connection close"
 
 server s1 {
 	rxreq
+	expect req.http.connection ~ close
+	expect req.http.beresp-connection !~ close
+	txresp
+	expect_close
+
+	accept
+	rxreq
+	expect req.http.connection !~ close
 	expect req.http.beresp-connection ~ close
 	txresp
 	expect_close
 
 	accept
 	rxreq
+	expect req.http.connection !~ close
 	expect req.http.beresp-connection !~ close
 	txresp -hdr "connection: close"
 	expect_close
+
+	accept
+	rxreq
+	expect req.http.connection ~ close
+	expect req.http.unset-connection == true
+	txresp
+	expect_close
 } -start
 
 varnish v1 -vcl+backend {
 	sub vcl_recv {
 		return (pass);
 	}
+	sub vcl_backend_fetch {
+		set bereq.http.connection = bereq.http.bereq-connection;
+	}
 	sub vcl_backend_response {
+		if (bereq.http.unset-connection) {
+			unset bereq.http.connection;
+		}
 		# NB: this overrides unconditionally on purpose
 		set beresp.http.connection = bereq.http.beresp-connection;
 	}
 } -start
 
 client c1 {
+	txreq -hdr "bereq-connection: close, x-varnish"
+	rxresp
+	expect resp.status == 200
+
 	txreq -hdr "beresp-connection: close, x-varnish"
 	rxresp
 	expect resp.status == 200
@@ -31,6 +57,10 @@ client c1 {
 	txreq
 	rxresp
 	expect resp.status == 200
+
+	txreq -hdr "bereq-connection: close" -hdr "unset-connection: true"
+	rxresp
+	expect resp.status == 200
 } -run
 
 server s1 -wait

From reza at naghibi.com  Tue Apr 20 18:20:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:20:06 +0000 (UTC)
Subject: [6.0] f2f37448b Fix request body reference count assertion
Message-ID: <20210420182007.08C8AA5A72@lists.varnish-cache.org>


commit f2f37448b045c4fab4eeb930d50508d28e261bf7
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Sat Oct 10 12:58:17 2020 +0200

    Fix request body reference count assertion
    
    Test case by Reza, thank you
    
    Fixes #3433
    Closes #3434

diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c
index f411fc66a..463b75b38 100644
--- a/bin/varnishd/cache/cache_req_body.c
+++ b/bin/varnishd/cache/cache_req_body.c
@@ -272,8 +272,9 @@ VRB_Free(struct req *req)
 
 	r = HSH_DerefObjCore(req->wrk, &req->body_oc, 0);
 
-	// a busyobj may have gained a reference
-	assert (r == 0 || r == 1);
+
+	// each busyobj may have gained a reference
+	assert (r <= req->restarts + 1);
 }
 
 /*----------------------------------------------------------------------
diff --git a/bin/varnishtest/tests/r03433.vtc b/bin/varnishtest/tests/r03433.vtc
new file mode 100644
index 000000000..0ebbb5d9b
--- /dev/null
+++ b/bin/varnishtest/tests/r03433.vtc
@@ -0,0 +1,41 @@
+varnishtest "req.body and restarts"
+
+server s1 -repeat 4 {
+	rxreq
+	txresp -bodylen 1000
+} -start
+
+varnish v1 -vcl+backend {
+	import std;
+	import vtc;
+
+	sub vcl_recv {
+		std.cache_req_body(1MB);
+		return (hash);
+	}
+
+	sub vcl_backend_response {
+		set beresp.ttl = 0.1s;
+	}
+
+	sub vcl_deliver {
+		if (!req.restarts) {
+			set req.url = "/2";
+			return (restart);
+		}
+	}
+} -start
+
+client c1 {
+	txreq -req PUT -url /1 -bodylen 250000
+	rxresp
+	expect resp.status == 200
+
+	delay 0.2
+
+	txreq -req PUT -url /1 -bodylen 250000
+	rxresp
+	expect resp.status == 200
+} -run
+
+delay 0.2

From reza at naghibi.com  Tue Apr 20 18:24:04 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:24:04 +0000 (UTC)
Subject: [6.0] 98391cb8e condfetch: Handle failed stale_oc without a boc
Message-ID: <20210420182404.EEB48A5DE5@lists.varnish-cache.org>


commit 98391cb8e673ef646156f628aec599dc8efb619e
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Mar 22 12:16:39 2021 +0100

    condfetch: Handle failed stale_oc without a boc
    
    The assertion that the stale objcore of a conditional fetch cannot be
    failed unless it was streaming is incorrect. Between the moment when
    we grab the stale objcore in HSH_Lookup and the moment we try to use
    it after vcl_backend_response, the backend fetch may have completed or
    failed.
    
    Instead, we need to treat an ongoing fetch and a failed fetch as
    separate checks since the latter may happen with or without a boc.
    
     Conflicts:
            bin/varnishd/cache/cache_fetch.c

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 63d3564fe..034424c6a 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -799,13 +799,18 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 		HSH_DerefBoc(bo->wrk, bo->stale_oc);
 		stale_boc = NULL;
 		if (stale_state != BOS_FINISHED) {
-			(void)VFP_Error(bo->vfc, "Template object failed");
-			VDI_Finish(bo->wrk, bo);
-			wrk->stats->fetch_failed++;
-			return (F_STP_FAIL);
+			assert(stale_state == BOS_FAILED);
+			AN(bo->stale_oc->flags & OC_F_FAILED);
 		}
 	}
-	AZ(bo->stale_oc->flags & OC_F_FAILED);
+
+	AZ(stale_boc);
+	if (bo->stale_oc->flags & OC_F_FAILED) {
+		(void)VFP_Error(bo->vfc, "Template object failed");
+		vbf_cleanup(bo);
+		wrk->stats->fetch_failed++;
+		return (F_STP_FAIL);
+	}
 
 	if (vbf_beresp2obj(bo)) {
 		(void)VFP_Error(bo->vfc, "Could not get storage");
diff --git a/bin/varnishtest/tests/c00105.vtc b/bin/varnishtest/tests/c00105.vtc
new file mode 100644
index 000000000..a5588b550
--- /dev/null
+++ b/bin/varnishtest/tests/c00105.vtc
@@ -0,0 +1,60 @@
+varnishtest "Failed post-streaming revalidation"
+
+barrier b1 cond 2
+barrier b2 sock 2
+barrier b3 sock 2
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr {Etag: "abc"} -hdr "Content-Length: 100"
+	barrier b1 sync
+	barrier b2 sync
+} -start
+
+server s2 {
+	rxreq
+	expect req.http.If-None-Match == {"abc"}
+	txresp -status 304 -nolen -hdr {Etag: "abc"} -hdr "Content-Length: 100"
+} -start
+
+varnish v1 -vcl+backend {
+	import directors;
+	import vtc;
+
+	sub vcl_recv {
+		if (req.http.backend == "s2") {
+			set req.backend_hint = s2;
+		}
+	}
+
+	sub vcl_backend_response {
+		if (beresp.was_304) {
+			vtc.barrier_sync("${b2_sock}");
+			vtc.barrier_sync("${b3_sock}");
+		}
+		set beresp.ttl = 1ms;
+	}
+} -start
+
+client c1 {
+	txreq -hdr "backend: s1"
+	rxresphdrs
+	expect resp.status == 200
+	expect_close
+} -start
+
+barrier b1 sync
+
+# ensure stale_oc
+delay 0.01
+
+client c2 {
+	txreq -hdr "backend: s2"
+	rxresphdrs
+	expect resp.status == 200
+	expect_close
+} -start
+
+client c1 -wait
+barrier b3 sync
+client c2 -wait

From reza at naghibi.com  Tue Apr 20 18:26:05 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:26:05 +0000 (UTC)
Subject: [6.0] 37b6567bc Decrease sleep time when waiting for child to die.
Message-ID: <20210420182605.7F83CA6072@lists.varnish-cache.org>


commit 37b6567bc01adedc0bd1b5cf70070e4bf32c6f08
Author: Simon <simon at devsn.se>
Date:   Tue Apr 6 13:13:28 2021 +0000

    Decrease sleep time when waiting for child to die.

diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index bc5c00ec2..c152ff9f1 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -511,11 +511,11 @@ mgt_reap_child(void)
 	XXXAN(vsb);
 
 	/* Wait for child to die */
-	for (i = 0; i < mgt_param.cli_timeout; i++) {
+	for (i = 0; i < mgt_param.cli_timeout * 10; i++) {
 		r = waitpid(child_pid, &status, WNOHANG);
 		if (r == child_pid)
 			break;
-		(void)sleep(1);
+		(void)usleep(100000);
 	}
 	if (r == 0) {
 		VSB_printf(vsb, "Child (%jd) not dying, killing", (intmax_t)r);

From reza at naghibi.com  Tue Apr 20 18:27:05 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:27:05 +0000 (UTC)
Subject: [6.0] 1bfc346c2 Require Python >= 3.4 at build time
Message-ID: <20210420182706.02724A62B0@lists.varnish-cache.org>


commit 1bfc346c2d90a660e2c8634525a40c7ec7658247
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Mar 12 10:28:35 2019 +0100

    Require Python >= 3.4 at build time
    
    Until the naked "python" executable refers to python3 (currently it is
    still python2) it now takes lower precedence.

diff --git a/varnish-legacy.m4 b/varnish-legacy.m4
index e5e2821eb..9f67359ea 100644
--- a/varnish-legacy.m4
+++ b/varnish-legacy.m4
@@ -98,9 +98,10 @@ AC_SUBST([VMOD_DIR])
 
 AC_DEFUN([VARNISH_VMODTOOL],
 [
-AC_CHECK_PROGS(PYTHON, [python3 python3.1 python3.2 python2.7 python2.6 python2.5 python2 python], "no")
+AC_CHECK_PROGS(PYTHON, [python3.9 python3.8 python3.7 python3.6 python3.5 dnl
+  python3.4 python3 python, "no"])
 if test "x$PYTHON" = "xno"; then
-  AC_MSG_ERROR([Python is needed to build, please install python.])
+  AC_MSG_ERROR([Python >= 3.4 is needed to build, please install python.])
 fi
 VARNISH_PKG_GET_VAR([VMODTOOL], [vmodtool])
 AC_SUBST([VMODTOOL])
diff --git a/varnish.m4 b/varnish.m4
index 71df96d9c..392c36c60 100644
--- a/varnish.m4
+++ b/varnish.m4
@@ -130,10 +130,10 @@ AC_DEFUN([_VARNISH_CHECK_DEVEL], [
 # ---------------------
 AC_DEFUN([_VARNISH_CHECK_PYTHON], [
 	m4_define_default([_AM_PYTHON_INTERPRETER_LIST],
-[python3.9 python3.8 python3.7 python3.6 python3.5 python3.4 python2.7 dnl
-python python2 python3])
-	AM_PATH_PYTHON([2.7], [], [
-		AC_MSG_ERROR([Python >= 2.7 is required.])
+		[python3.9 python3.8 python3.7 python3.6 python3.5 dnl
+		python3.4 python3 python])
+	AM_PATH_PYTHON([3.4], [], [
+		AC_MSG_ERROR([Python >= 3.4 is required.])
 	])
 
 ])

From reza at naghibi.com  Tue Apr 20 18:27:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:27:06 +0000 (UTC)
Subject: [6.0] e2f4cadc2 Drop Python 2 for our docs dependencies too
Message-ID: <20210420182706.215A0A62B3@lists.varnish-cache.org>


commit e2f4cadc27eded2a1d125481adc030b263c8c28b
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Mar 13 15:14:18 2019 +0100

    Drop Python 2 for our docs dependencies too

diff --git a/configure.ac b/configure.ac
index 65b88bda9..9b305c679 100644
--- a/configure.ac
+++ b/configure.ac
@@ -41,7 +41,7 @@ AC_ARG_WITH([rst2man],
   AS_HELP_STRING([--with-rst2man=PATH], [Location of rst2man (auto)]),
   [RST2MAN="$withval"],
   AC_CHECK_PROGS(RST2MAN,
-    [rst2man rst2man.py rst2man-3.6 rst2man-2.7],
+    [rst2man rst2man.py rst2man-3.6 rst2man-3],
     [no]))
 if test "x$RST2MAN" = "xno"; then
   AC_MSG_ERROR(
@@ -52,7 +52,7 @@ AC_ARG_WITH([sphinx-build],
   AS_HELP_STRING([--with-sphinx-build=PATH], [Location of sphinx-build (auto)]),
   [SPHINX="$withval"],
   AC_CHECK_PROGS(SPHINX,
-    [sphinx-build sphinx-build-3.6 sphinx-build-2.7],
+    [sphinx-build sphinx-build-3.6 sphinx-build-3],
     [no]))
 if test "x$SPHINX" = "xno"; then
   AC_MSG_ERROR(

From reza at naghibi.com  Tue Apr 20 18:27:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:27:06 +0000 (UTC)
Subject: [6.0] 74ec31586 More python3 tweaking
Message-ID: <20210420182706.3D9A4A62B7@lists.varnish-cache.org>


commit 74ec31586b3e9845be8722eecb3c5d838578a08a
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Mar 14 21:04:05 2019 +0000

    More python3 tweaking

diff --git a/configure.ac b/configure.ac
index 9b305c679..0954bf389 100644
--- a/configure.ac
+++ b/configure.ac
@@ -45,7 +45,7 @@ AC_ARG_WITH([rst2man],
     [no]))
 if test "x$RST2MAN" = "xno"; then
   AC_MSG_ERROR(
-    [rst2man is needed to build Varnish, please install python-docutils.])
+    [rst2man is needed to build Varnish, please install python3-docutils.])
 fi
 
 AC_ARG_WITH([sphinx-build],
@@ -56,7 +56,7 @@ AC_ARG_WITH([sphinx-build],
     [no]))
 if test "x$SPHINX" = "xno"; then
   AC_MSG_ERROR(
-    [sphinx-build is needed to build Varnish, please install python-sphinx.])
+    [sphinx-build is needed to build Varnish, please install python3-sphinx.])
 fi
 
 AC_ARG_WITH([rst2html],

From reza at naghibi.com  Tue Apr 20 18:27:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:27:06 +0000 (UTC)
Subject: [6.0] b1937f711 We live in the future now
Message-ID: <20210420182706.640EEA62BB@lists.varnish-cache.org>


commit b1937f71101d831e41b8141c2a85913c9eb52cc8
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Mar 12 10:35:38 2019 +0100

    We live in the future now
    
    For a given definition of "future" or "now".

diff --git a/bin/varnishtest/witness.py b/bin/varnishtest/witness.py
index af4a56e55..f5b516ee0 100644
--- a/bin/varnishtest/witness.py
+++ b/bin/varnishtest/witness.py
@@ -7,8 +7,6 @@
 #       python witness.py
 #       dot -Tpng /tmp/_.dot > /tmp/_.png
 
-from __future__ import print_function
-
 d = dict()
 a = dict()
 
diff --git a/doc/sphinx/vtc-syntax.py b/doc/sphinx/vtc-syntax.py
index 6d4946d32..c0d9435c8 100644
--- a/doc/sphinx/vtc-syntax.py
+++ b/doc/sphinx/vtc-syntax.py
@@ -29,7 +29,6 @@
 # Process various varnishtest C files and output reStructuredText to be
 # included in vtc(7).
 
-from __future__ import print_function
 import sys
 import re
 
diff --git a/include/generate.py b/include/generate.py
index c1ea3bcbb..6d1377661 100755
--- a/include/generate.py
+++ b/include/generate.py
@@ -29,8 +29,6 @@
 #
 # Generate vcs_version.h vmod_abi.h
 
-from __future__ import print_function
-
 import subprocess
 import os
 import sys
diff --git a/include/tbl/style.py b/include/tbl/style.py
index 0e7f657fb..31435887e 100644
--- a/include/tbl/style.py
+++ b/include/tbl/style.py
@@ -2,8 +2,6 @@
 #
 # Very basic style-checker for include/tbl files.
 
-from __future__ import print_function
-
 import glob
 
 def check_file(fn):
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 0118d7956..be3dd3294 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -30,8 +30,6 @@
 # Generate various .c and .h files for the VCL compiler and the interfaces
 # for it.
 
-from __future__ import print_function
-
 #######################################################################
 # These are our tokens
 
diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 14f93b11d..35c19e90f 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -33,9 +33,6 @@ Read the vmod.vcc file (inputvcc) and produce:
     vmod_${name}.rst -- Extracted documentation
 """
 
-# This script should work with both Python 2 and Python 3.
-from __future__ import print_function
-
 import os
 import sys
 import re
diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py
index e65b0072f..d95b66ed5 100644
--- a/lib/libvcc/vsctool.py
+++ b/lib/libvcc/vsctool.py
@@ -35,8 +35,6 @@ the same general syntax as a `.rst` file, but for now we process
 it with this program to get a *real* `.rst` file.
 '''
 
-from __future__ import print_function
-
 import getopt
 import json
 import sys
diff --git a/tools/gcov_digest.py b/tools/gcov_digest.py
index 3fbfdf946..dc375d0f2 100644
--- a/tools/gcov_digest.py
+++ b/tools/gcov_digest.py
@@ -48,8 +48,6 @@ Options:
 
 """
 
-from __future__ import print_function
-
 import os
 import sys
 import getopt
diff --git a/tools/include_wash.py b/tools/include_wash.py
index 94c11ba2c..61a0cae91 100644
--- a/tools/include_wash.py
+++ b/tools/include_wash.py
@@ -1,7 +1,5 @@
 #!/usr/bin/env python3
 
-from __future__ import print_function
-
 import os
 
 def check(fn):

From reza at naghibi.com  Tue Apr 20 18:27:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:27:06 +0000 (UTC)
Subject: [6.0] cd7b69af4 Prefer Python 3 tools
Message-ID: <20210420182706.809C4A62BF@lists.varnish-cache.org>


commit cd7b69af4ef0cbd531e872fc64774f25915868db
Author: Klemens Nanni <kn at openbsd.org>
Date:   Tue May 14 17:43:57 2019 +0200

    Prefer Python 3 tools
    
    The last three commits already made configure recommend installing
    Python 3 packages and look for versioned executables, however with a low
    priority.
    
    This is a problem on systems such as OpenBSD 6.5 with a default Python
    version at 2.7, where 3.7 flavored Python packages get installed with
    a "-3" binary suffix.  That is, when both rst2man and rst2man-3 are
    installed at configure time, the lower version will be picked unless
    explicitly passed through `--with-feature' arguments.
    
    Regardless of this specific case, trying more specificly versioned tool
    names first seems correctly in line with recent development and less
    error prone, so change it accordingly.
    
     Conflicts:
            configure.ac

diff --git a/configure.ac b/configure.ac
index 0954bf389..a3d28c733 100644
--- a/configure.ac
+++ b/configure.ac
@@ -41,7 +41,7 @@ AC_ARG_WITH([rst2man],
   AS_HELP_STRING([--with-rst2man=PATH], [Location of rst2man (auto)]),
   [RST2MAN="$withval"],
   AC_CHECK_PROGS(RST2MAN,
-    [rst2man rst2man.py rst2man-3.6 rst2man-3],
+    [rst2man-3.6 rst2man-3 rst2man rst2man.py],
     [no]))
 if test "x$RST2MAN" = "xno"; then
   AC_MSG_ERROR(
@@ -52,7 +52,7 @@ AC_ARG_WITH([sphinx-build],
   AS_HELP_STRING([--with-sphinx-build=PATH], [Location of sphinx-build (auto)]),
   [SPHINX="$withval"],
   AC_CHECK_PROGS(SPHINX,
-    [sphinx-build sphinx-build-3.6 sphinx-build-3],
+    [sphinx-build-3.6 sphinx-build-3 sphinx-build],
     [no]))
 if test "x$SPHINX" = "xno"; then
   AC_MSG_ERROR(
@@ -60,14 +60,16 @@ if test "x$SPHINX" = "xno"; then
 fi
 
 AC_ARG_WITH([rst2html],
-               AS_HELP_STRING([--with-rst2html=PATH],
-                              [Location of rst2html (auto)]),
-               [RST2HTML="$withval"],
-               [AC_CHECK_PROGS(RST2HTML, [rst2html rst2html.py], "no")
-	        if test "x$RST2HTML" = "xno"; then
-		   AC_MSG_WARN([rst2html not found - not building changelog])
-		fi])
+  AS_HELP_STRING([--with-rst2html=PATH], [Location of rst2html (auto)]),
+  [RST2HTML="$withval"],
+  AC_CHECK_PROGS(RST2HTML,
+     [rst2html-3.6 rst2html-3 rst2html rst2html.py],
+     "no"))
 AM_CONDITIONAL(HAVE_RST2HTML,[test "x$RST2HTML" != "xno"])
+if test "x$RST2HTML" = "xno"; then
+   AC_MSG_ERROR(
+     [rst2html not found - (Weird, we found rst2man?!)])
+fi
 
 AC_ARG_WITH([dot],
   AS_HELP_STRING([--with-dot=PATH],

From reza at naghibi.com  Tue Apr 20 18:27:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:27:06 +0000 (UTC)
Subject: [6.0] 7191ed970 [CCI] Install python3 on make dist
Message-ID: <20210420182706.99E19A62C3@lists.varnish-cache.org>


commit 7191ed970a2689d652534855a706d8e18a9f8554
Author: Steven <wojcikjsteven at gmail.com>
Date:   Tue Apr 13 20:33:08 2021 -0400

    [CCI] Install python3 on make dist

diff --git a/.circleci/config.yml b/.circleci/config.yml
index d830ae1df..e6c8027be 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -36,7 +36,7 @@ jobs:
                 libunwind-devel \
                 make \
                 pcre-devel \
-                python \
+                python3 \
                 python-sphinx \
                 curl
       - checkout

From reza at naghibi.com  Tue Apr 20 18:27:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:27:06 +0000 (UTC)
Subject: [6.0] 0e82401e6 We need UTF8 encoding for Python3
Message-ID: <20210420182706.B3C4AA62C8@lists.varnish-cache.org>


commit 0e82401e659a93a9d9a838e1a34a252cde0a288b
Author: Steven <wojcikjsteven at gmail.com>
Date:   Wed Apr 14 16:51:09 2021 -0400

    We need UTF8 encoding for Python3

diff --git a/doc/sphinx/vtc-syntax.py b/doc/sphinx/vtc-syntax.py
index c0d9435c8..8b15ca7e9 100644
--- a/doc/sphinx/vtc-syntax.py
+++ b/doc/sphinx/vtc-syntax.py
@@ -38,7 +38,7 @@ def parse_file(fn, cl, tl, sl):
     section = ""
     resec = re.compile("[ /]\* SECTION: ")
 
-    f = open(fn, "r")
+    f = open(fn, "r", encoding="UTF-8")
 
     for l in f:
         if "*/" in l:

From reza at naghibi.com  Tue Apr 20 18:29:05 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:29:05 +0000 (UTC)
Subject: [6.0] 48d51ff6d Set be->cooled while holding backends_mtx
Message-ID: <20210420182905.3DD54A6E61@lists.varnish-cache.org>


commit 48d51ff6d13655e0e6646a4d7aceeca2daaa8fdd
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Dec 3 14:53:51 2019 +0100

    Set be->cooled while holding backends_mtx
    
    Several functions (VBE_Poll and vbe_destroy) tests be->cooled == 0 to
    determine which of the two lists backends and cool_backends a specific
    instance currently lives on. If the flag is in the process of being
    changed, then the wrong list head may be used and will result in strange
    bugs.
    
     Conflicts:
            bin/varnishd/cache/cache_backend.c

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 0e6e1ab59..b64ac8521 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -565,9 +565,9 @@ VRT_delete_backend(VRT_CTX, struct director **dp)
 	Lck_Lock(&be->mtx);
 	be->director->admin_health = VDI_AH_DELETED;
 	be->director->health_changed = VTIM_real();
-	be->cooled = VTIM_real() + 60.;
 	Lck_Unlock(&be->mtx);
 	Lck_Lock(&backends_mtx);
+	be->cooled = VTIM_real() + 60.;
 	VTAILQ_REMOVE(&backends, be, list);
 	VTAILQ_INSERT_TAIL(&cool_backends, be, list);
 	Lck_Unlock(&backends_mtx);

From reza at naghibi.com  Tue Apr 20 18:29:05 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:29:05 +0000 (UTC)
Subject: [6.0] 7523f3c12 Fix handling of failed VRT_new_backend_clustered
Message-ID: <20210420182905.5D833A6E64@lists.varnish-cache.org>


commit 7523f3c12da877f4374af977279358d8bba6d8ea
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Dec 3 16:00:27 2019 +0100

    Fix handling of failed VRT_new_backend_clustered
    
    We refuse to accept new dynamic backends while the VCL is cooling, and
    drop adding the attempted backend on the VCL's backend list when that
    condition is found. But that would cause an assert later when it is picked
    off the cool_backends list for destruction. Fix this by directly
    destroying the backend instead of going through the cooling list.
    
    Note that this patch removes the ASSERT_CLI() macro in vbe_destroy().

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index b64ac8521..ba0e649ed 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -405,7 +405,6 @@ vbe_destroy(const struct director *d)
 {
 	struct backend *be;
 
-	ASSERT_CLI();
 	CAST_OBJ_NOTNULL(be, d->priv, BACKEND_MAGIC);
 
 	if (be->probe != NULL)
@@ -537,8 +536,8 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc,
 	if (retval == 0)
 		return (d);
 
-	VRT_delete_backend(ctx, &d);
-	AZ(d);
+	/* Undo the above */
+	d->destroy(d);
 	return (NULL);
 }
 
diff --git a/bin/varnishd/cache/cache_vcl_vrt.c b/bin/varnishd/cache/cache_vcl_vrt.c
index 4133f46b7..5f3bfeec0 100644
--- a/bin/varnishd/cache/cache_vcl_vrt.c
+++ b/bin/varnishd/cache/cache_vcl_vrt.c
@@ -142,6 +142,7 @@ VCL_AddDirector(struct vcl *vcl, struct director *d, const char *vcl_name)
 	AZ(errno=pthread_rwlock_rdlock(&vcl->temp_rwl));
 	if (vcl->temp == VCL_TEMP_COOLING) {
 		AZ(errno=pthread_rwlock_unlock(&vcl->temp_rwl));
+		REPLACE(d->display_name, NULL);
 		return (1);
 	}
 

From reza at naghibi.com  Tue Apr 20 18:29:05 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:29:05 +0000 (UTC)
Subject: [6.0] f3f15136d Add an assert that VRT_delete_backend is only called
 once
Message-ID: <20210420182905.74B52A6E6A@lists.varnish-cache.org>


commit f3f15136d56b11a9e72a1c9500e29b3e62dd8c17
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Dec 4 13:10:07 2019 +0100

    Add an assert that VRT_delete_backend is only called once
    
    VRT_delete_backend() sets be->cooled to non-zero as the only place where
    that is done. Assert that it is zero on entry as a check that
    VRT_delete_backend isn't called multiple times.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index ba0e649ed..70efe08af 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -566,6 +566,7 @@ VRT_delete_backend(VRT_CTX, struct director **dp)
 	be->director->health_changed = VTIM_real();
 	Lck_Unlock(&be->mtx);
 	Lck_Lock(&backends_mtx);
+	AZ(be->cooled);
 	be->cooled = VTIM_real() + 60.;
 	VTAILQ_REMOVE(&backends, be, list);
 	VTAILQ_INSERT_TAIL(&cool_backends, be, list);

From reza at naghibi.com  Tue Apr 20 18:31:05 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:31:05 +0000 (UTC)
Subject: [6.0] f97b73465 Don't change the C-L if we detect a failed stream.
Message-ID: <20210420183105.84731A73AC@lists.varnish-cache.org>


commit f97b7346522ec509d86ffbc3dcd2165eae48e1e4
Author: Reza Naghibi <reza at naghibi.com>
Date:   Wed Mar 24 15:58:05 2021 -0400

    Don't change the C-L if we detect a failed stream.
    
    Previously we would read the response Content-Length from a failed oc,
    which would make the error response valid. Now, if this is detected,
    we don't touch the Content-Length.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 30b6fdc0b..f24033ee1 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -409,7 +409,7 @@ cnt_transmit(struct worker *wrk, struct req *req)
 	head = !strcmp(req->http0->hd[HTTP_HDR_METHOD].b, "HEAD");
 	err = 0;
 
-	if (boc != NULL)
+	if (boc != NULL || (req->objcore->flags & (OC_F_FAILED)))
 		req->resp_len = clval;
 	else
 		req->resp_len = ObjGetLen(req->wrk, req->objcore);
@@ -482,6 +482,11 @@ cnt_transmit(struct worker *wrk, struct req *req)
 		ObjSlim(wrk, req->objcore);
 	}
 
+	if (!req->doclose && (req->objcore->flags & OC_F_FAILED))
+		/* The object we delivered failed due to a streaming error.
+		 * Fail the request. */
+		req->doclose = SC_TX_ERROR;
+
 	if (boc != NULL)
 		HSH_DerefBoc(wrk, req->objcore);
 
diff --git a/bin/varnishtest/tests/r03560.vtc b/bin/varnishtest/tests/r03560.vtc
new file mode 100644
index 000000000..fc76bbe1d
--- /dev/null
+++ b/bin/varnishtest/tests/r03560.vtc
@@ -0,0 +1,29 @@
+varnishtest "A backend connection error gets returned as a valid short response"
+
+barrier b1 sock 2
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr "Content-Length: 10"
+	barrier b1 sync
+	send "12345"
+	# Early connection close error
+} -start
+
+varnish v1 -vcl+backend {
+	import vtc;
+
+	sub vcl_deliver {
+		# Make sure we are streaming and give the backend time to error out
+		vtc.barrier_sync("${b1_sock}");
+		vtc.sleep(1s);
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresphdrs
+	expect resp.http.Content-Length == "10"
+	recv 5
+	expect_close
+} -run

From reza at naghibi.com  Tue Apr 20 18:32:05 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:32:05 +0000 (UTC)
Subject: [6.0] 349abc762 Fix canon_point calculation
Message-ID: <20210420183205.EA36EA75AB@lists.varnish-cache.org>


commit 349abc762336b6ae6890933afe0ba00641ef9586
Author: Reza Naghibi <reza at naghibi.com>
Date:   Thu Apr 15 09:46:56 2021 -0400

    Fix canon_point calculation

diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c
index bb0a7aabb..577e51aba 100644
--- a/lib/libvmod_directors/shard_cfg.c
+++ b/lib/libvmod_directors/shard_cfg.c
@@ -290,7 +290,7 @@ shardcfg_hashcircle(struct sharddir *shardd)
 		}
 		/* not used in current interface */
 		shardd->backend[h].canon_point =
-		    shardd->hashcircle[i].point;
+		    shardd->hashcircle[i - j].point;
 	}
 	assert (i == n_points);
 	qsort( (void *) shardd->hashcircle, n_points,

From reza at naghibi.com  Tue Apr 20 18:32:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:32:06 +0000 (UTC)
Subject: [6.0] 13a03e3a1 Allow shard.reconfigure() to work on empty directors
Message-ID: <20210420183206.1296AA75AF@lists.varnish-cache.org>


commit 13a03e3a105d4b71efee4b0e50e647a4e6d79980
Author: Reza Naghibi <reza at naghibi.com>
Date:   Thu Apr 15 10:58:00 2021 -0400

    Allow shard.reconfigure() to work on empty directors
    
    Also move the lock up to cover more operations.

diff --git a/bin/varnishtest/tests/d06001.vtc b/bin/varnishtest/tests/d06001.vtc
new file mode 100644
index 000000000..592f3da93
--- /dev/null
+++ b/bin/varnishtest/tests/d06001.vtc
@@ -0,0 +1,30 @@
+varnishtest "Empty shard reconfig"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import directors;
+
+	sub vcl_init {
+		new shard1 = directors.shard();
+		new shard2 = directors.shard();
+
+		shard1.reconfigure();
+
+		shard2.add_backend(s1);
+		shard2.reconfigure();
+	}
+
+	sub vcl_recv {
+		set req.backend_hint = shard2.backend();
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run
diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c
index 577e51aba..1c4c797e0 100644
--- a/lib/libvmod_directors/shard_cfg.c
+++ b/lib/libvmod_directors/shard_cfg.c
@@ -616,14 +616,19 @@ shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv,
 		return (0);
 	}
 
+	sharddir_wrlock(shardd);
+
 	change = shard_change_get(ctx, priv, shardd);
-	if (change == NULL)
+	if (change == NULL) {
+		sharddir_unlock(shardd);
 		return (0);
+	}
 
-	if (VSTAILQ_FIRST(&change->tasks) == NULL)
+	if (VSTAILQ_FIRST(&change->tasks) == NULL) {
+		shard_change_finish(change);
+		sharddir_unlock(shardd);
 		return (1);
-
-	sharddir_wrlock(shardd);
+	}
 
 	shardcfg_apply_change(ctx, shardd, change, replicas);
 	shard_change_finish(change);

From reza at naghibi.com  Tue Apr 20 18:35:05 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:35:05 +0000 (UTC)
Subject: [6.0] 85204bcb5 vbe: Generic sanity check of non-recyclable
 connections
Message-ID: <20210420183505.F1593A7A2A@lists.varnish-cache.org>


commit 85204bcb5bc8c42d304325884c45539851a788cc
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Mar 24 17:39:47 2021 +0100

    vbe: Generic sanity check of non-recyclable connections
    
    The reason we expect here can be summarized as: this was a pipe
    transaction or an error occurred. This could be much simpler if
    we replaced enum sess_close with a struct stream_close instead.
    
    Refs dc5bddbd301529b101598b644544b99ccabca12c

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 70efe08af..b5217bd4f 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -215,9 +215,15 @@ vbe_dir_finish(const struct director *d, struct worker *wrk,
 	CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
 	pfd = bo->htc->priv;
 	bo->htc->priv = NULL;
-	if (PFD_State(pfd) != PFD_STATE_USED)
-		assert(bo->htc->doclose == SC_TX_PIPE ||
-		    bo->htc->doclose == SC_RX_TIMEOUT);
+	if (PFD_State(pfd) != PFD_STATE_USED) {
+		AN(bo->htc->doclose);
+		if (bo->htc->doclose != SC_TX_PIPE) {
+#define SESS_CLOSE(U, l, err, desc)				\
+			if (bo->htc->doclose == SC_ ## U)	\
+				AN(err);
+#include "tbl/sess_close.h"
+		}
+	}
 	if (bo->htc->doclose != SC_NULL || bp->proxy_header != 0) {
 		VSLb(bo->vsl, SLT_BackendClose, "%d %s", *PFD_Fd(pfd),
 		    bp->director->display_name);

From reza at naghibi.com  Tue Apr 20 18:35:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Tue, 20 Apr 2021 18:35:06 +0000 (UTC)
Subject: [6.0] 584a1c98f vbe: Check failures to send the request earlier
Message-ID: <20210420183506.1349EA7A32@lists.varnish-cache.org>


commit 584a1c98f290448fc981bc389994d9f20f8a4fb3
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Mar 24 17:50:23 2021 +0100

    vbe: Check failures to send the request earlier
    
    There's no point waiting for the backend to complain if we weren't able
    to properly send the backend request.
    
    Fixes #3556
    
     Conflicts:
            bin/varnishd/cache/cache_backend.c

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index b5217bd4f..99e7358e5 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -282,7 +282,7 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk,
 		i = V1F_SendReq(wrk, bo, &bo->acct.bereq_hdrbytes,
 				&bo->acct.bereq_bodybytes, 0, abuf, pbuf);
 
-		if (PFD_State(pfd) != PFD_STATE_USED) {
+		if (i == 0 && PFD_State(pfd) != PFD_STATE_USED) {
 			if (VTP_Wait(wrk, pfd, VTIM_real() +
 			    bo->htc->first_byte_timeout) != 0) {
 				bo->htc->doclose = SC_RX_TIMEOUT;
diff --git a/bin/varnishtest/tests/r03556.vtc b/bin/varnishtest/tests/r03556.vtc
new file mode 100644
index 000000000..d40afa47f
--- /dev/null
+++ b/bin/varnishtest/tests/r03556.vtc
@@ -0,0 +1,34 @@
+varnishtest "#3556"
+
+server s1 {
+	rxreq
+	txresp
+
+	non_fatal
+	rxreq
+} -start
+
+varnish v1 -cliok "param.set first_byte_timeout 10"
+varnish v1 -vcl+backend {} -start
+
+client c1 {
+	txreq
+	rxresp
+} -run
+
+logexpect l2 -v v1 -q "ReqMethod eq POST" {
+	expect * * End
+} -start
+
+client c2 {
+	txreq -req POST \
+	    -hdr "Content-Length: 10" \
+	    -hdr "Content-Type: text/plain"
+	send incompl
+} -run
+
+logexpect l2 -wait
+
+shell -expect POST {
+	exec varnishncsa -d -n ${v1_name} -q 'Timestamp:Process[2] < 10.0'
+}

From phk at FreeBSD.org  Tue Apr 20 21:10:12 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 20 Apr 2021 21:10:12 +0000 (UTC)
Subject: [master] 6e9b8d6d7 Transplant the backend lock up to the director
 layer and expose it in the public side of the API so the backend layer can
 get at it.
Message-ID: <20210420211012.CA7CDABD74@lists.varnish-cache.org>


commit 6e9b8d6d7ca93d365914c044307fab455d4ec750
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Apr 20 20:32:33 2021 +0000

    Transplant the backend lock up to the director layer and
    expose it in the public side of the API so the backend layer
    can get at it.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index b67773f12..7141e5a89 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -171,11 +171,11 @@ vbe_dir_getfd(VRT_CTX, struct worker *wrk, VCL_BACKEND dir, struct backend *bp,
 	AN(fdp);
 	assert(*fdp >= 0);
 
-	Lck_Lock(&bp->mtx);
+	Lck_Lock(bp->director->mtx);
 	bp->n_conn++;
 	bp->vsc->conn++;
 	bp->vsc->req++;
-	Lck_Unlock(&bp->mtx);
+	Lck_Unlock(bp->director->mtx);
 
 	err = 0;
 	if (bp->proxy_header != 0)
@@ -190,11 +190,11 @@ vbe_dir_getfd(VRT_CTX, struct worker *wrk, VCL_BACKEND dir, struct backend *bp,
 		bo->htc = NULL;
 		VCP_Close(&pfd);
 		AZ(pfd);
-		Lck_Lock(&bp->mtx);
+		Lck_Lock(bp->director->mtx);
 		bp->n_conn--;
 		bp->vsc->conn--;
 		bp->vsc->req--;
-		Lck_Unlock(&bp->mtx);
+		Lck_Unlock(bp->director->mtx);
 		return (NULL);
 	}
 	bo->acct.bereq_hdrbytes += err;
@@ -245,12 +245,12 @@ vbe_dir_finish(VRT_CTX, VCL_BACKEND d)
 		    VRT_BACKEND_string(d));
 		VCP_Close(&pfd);
 		AZ(pfd);
-		Lck_Lock(&bp->mtx);
+		Lck_Lock(bp->director->mtx);
 	} else {
 		assert (PFD_State(pfd) == PFD_STATE_USED);
 		VSLb(bo->vsl, SLT_BackendClose, "%d %s recycle", *PFD_Fd(pfd),
 		    VRT_BACKEND_string(d));
-		Lck_Lock(&bp->mtx);
+		Lck_Lock(bp->director->mtx);
 		VSC_C_main->backend_recycle++;
 		VCP_Recycle(bo->wrk, &pfd);
 	}
@@ -260,7 +260,7 @@ vbe_dir_finish(VRT_CTX, VCL_BACKEND d)
 	bp->vsc->conn--;
 #define ACCT(foo)	bp->vsc->foo += bo->acct.foo;
 #include "tbl/acct_fields_bereq.h"
-	Lck_Unlock(&bp->mtx);
+	Lck_Unlock(bp->director->mtx);
 	bo->htc = NULL;
 }
 
@@ -446,7 +446,7 @@ vbe_free(struct backend *be)
 #undef DN
 	free(be->endpoint);
 
-	Lck_Delete(&be->mtx);
+	Lck_Delete(be->director->mtx);
 	FREE_OBJ(be);
 }
 
@@ -593,7 +593,6 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc,
 	ALLOC_OBJ(be, BACKEND_MAGIC);
 	if (be == NULL)
 		return (NULL);
-	Lck_New(&be->mtx, lck_backend);
 
 	vep = be->endpoint = VRT_Endpoint_Clone(vep);
 #define DA(x)	do { if (vrt->x != NULL) REPLACE((be->x), (vrt->x)); } while (0)
@@ -675,9 +674,9 @@ VRT_delete_backend(VRT_CTX, VCL_BACKEND *dp)
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	TAKE_OBJ_NOTNULL(d, dp, DIRECTOR_MAGIC);
 	CAST_OBJ_NOTNULL(be, d->priv, BACKEND_MAGIC);
-	Lck_Lock(&be->mtx);
+	Lck_Lock(be->director->mtx);
 	VRT_DisableDirector(be->director);
-	Lck_Unlock(&be->mtx);
+	Lck_Unlock(be->director->mtx);
 	Lck_Lock(&backends_mtx);
 	AZ(be->cooled);
 	be->cooled = VTIM_real() + 60.;
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index b2c211d9e..7eabfebc3 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -55,7 +55,6 @@ struct backend {
 	unsigned		n_conn;
 
 	VTAILQ_ENTRY(backend)	list;
-	struct lock		mtx;
 
 	struct vrt_endpoint	*endpoint;
 
diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h
index e164e1678..eefbd9335 100644
--- a/bin/varnishd/cache/cache_director.h
+++ b/bin/varnishd/cache/cache_director.h
@@ -36,6 +36,7 @@
 struct vcldir {
 	unsigned			magic;
 #define VCLDIR_MAGIC			0xbf726c7d
+	struct lock			dlck;
 	struct director			*dir;
 	struct vcl			*vcl;
 	const struct vdi_methods	*methods;
diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c
index 023ba0022..a862163ef 100644
--- a/bin/varnishd/cache/cache_vrt_vcl.c
+++ b/bin/varnishd/cache/cache_vrt_vcl.c
@@ -142,6 +142,8 @@ static void
 vcldir_free(struct vcldir *vdir)
 {
 
+	CHECK_OBJ_NOTNULL(vdir, VCLDIR_MAGIC);
+	Lck_Delete(&vdir->dlck);
 	free(vdir->cli_name);
 	FREE_OBJ(vdir->dir);
 	FREE_OBJ(vdir);
@@ -192,6 +194,9 @@ VRT_AddDirector(VRT_CTX, const struct vdi_methods *m, void *priv,
 	vdir->admin_health = VDI_AH_AUTO;
 	vdir->health_changed = VTIM_real();
 
+	Lck_New(&vdir->dlck, lck_director);
+	vdir->dir->mtx = &vdir->dlck;
+
 	/* NB: at this point we look at the VCL temperature after getting
 	 * through the trouble of creating the director even though it might
 	 * not be legal to do so. Because we change the VCL temperature before
diff --git a/include/tbl/locks.h b/include/tbl/locks.h
index 8240ae2b3..3de8e33e6 100644
--- a/include/tbl/locks.h
+++ b/include/tbl/locks.h
@@ -31,10 +31,10 @@
 
 /*lint -save -e525 -e539 */
 
-LOCK(backend)
 LOCK(ban)
 LOCK(busyobj)
 LOCK(cli)
+LOCK(director)
 LOCK(exp)
 LOCK(hcb)
 LOCK(lru)
diff --git a/include/vrt.h b/include/vrt.h
index f469d2dab..10528a3e5 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -202,6 +202,7 @@
 struct busyobj;
 struct director;
 struct http;
+struct lock;
 struct req;
 struct stevedore;
 struct suckaddr;
@@ -621,6 +622,7 @@ struct director {
 	void				*priv;
 	char				*vcl_name;
 	struct vcldir			*vdir;
+	struct lock			*mtx;
 };
 
 VCL_BOOL VRT_Healthy(VRT_CTX, VCL_BACKEND, VCL_TIME *);

From phk at FreeBSD.org  Tue Apr 20 21:10:12 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 20 Apr 2021 21:10:12 +0000 (UTC)
Subject: [master] 4c6e1eed3 Sort list
Message-ID: <20210420211012.ADD7FABD71@lists.varnish-cache.org>


commit 4c6e1eed3f3adc80e411ad76b2afe798e5ccce9c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Apr 20 19:51:08 2021 +0000

    Sort list

diff --git a/include/vrt.h b/include/vrt.h
index 7d50b3431..f469d2dab 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -208,6 +208,7 @@ struct suckaddr;
 struct vcl;
 struct vcldir;
 struct VCL_conf;
+struct vcl_sub;
 struct vmod;
 struct vmod_priv;
 struct vrt_acl;
@@ -217,7 +218,6 @@ struct vsc_seg;
 struct vsl_log;
 struct vsmw_cluster;
 struct ws;
-struct vcl_sub;
 
 /***********************************************************************
  * VCL_STRANDS:

From phk at FreeBSD.org  Tue Apr 20 21:10:12 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 20 Apr 2021 21:10:12 +0000 (UTC)
Subject: [master] 0fab8549d Slight polish
Message-ID: <20210420211012.E9064ABD78@lists.varnish-cache.org>


commit 0fab8549db0d79ff7a180f98abfc39424e225ac5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Apr 20 20:38:41 2021 +0000

    Slight polish

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 7141e5a89..fc774efff 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -405,17 +405,15 @@ vbe_dir_event(const struct director *d, enum vcl_event_e ev)
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
 	CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC);
 
-	if (ev == VCL_EVENT_WARM)
+	if (ev == VCL_EVENT_WARM) {
 		VRT_VSC_Reveal(bp->vsc_seg);
-
-	if (bp->probe != NULL && ev == VCL_EVENT_WARM)
-		VBP_Control(bp, 1);
-
-	if (bp->probe != NULL && ev == VCL_EVENT_COLD)
-		VBP_Control(bp, 0);
-
-	if (ev == VCL_EVENT_COLD)
+		if (bp->probe != NULL)
+			VBP_Control(bp, 1);
+	} else if (ev == VCL_EVENT_COLD) {
+		if (bp->probe != NULL)
+			VBP_Control(bp, 0);
 		VRT_VSC_Hide(bp->vsc_seg);
+	}
 }
 
 /*---------------------------------------------------------------------*/

From phk at FreeBSD.org  Tue Apr 20 21:10:13 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 20 Apr 2021 21:10:13 +0000 (UTC)
Subject: [master] b301c17cb Lock updates of VSC counters for connection errors.
Message-ID: <20210420211013.1D8BBABD7F@lists.varnish-cache.org>


commit b301c17cba2cc25b18a2307456bb140106de29ef
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Apr 20 20:42:31 2021 +0000

    Lock updates of VSC counters for connection errors.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index fc774efff..6ba15ffd8 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -157,7 +157,9 @@ vbe_dir_getfd(VRT_CTX, struct worker *wrk, VCL_BACKEND dir, struct backend *bp,
 	FIND_TMO(connect_timeout, tmod, bo, bp);
 	pfd = VCP_Get(bp->conn_pool, tmod, wrk, force_fresh, &err);
 	if (pfd == NULL) {
+		Lck_Lock(bp->director->mtx);
 		VBE_Connect_Error(bp->vsc, err);
+		Lck_Unlock(bp->director->mtx);
 		VSLb(bo->vsl, SLT_FetchError,
 		     "backend %s: fail errno %d (%s)",
 		     VRT_BACKEND_string(dir), err, vstrerror(err));

From phk at FreeBSD.org  Tue Apr 20 21:10:13 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 20 Apr 2021 21:10:13 +0000 (UTC)
Subject: [master] f7ca7c1ae Fix a bug in backend health state change
 timestamps.
Message-ID: <20210420211013.3D752ABD82@lists.varnish-cache.org>


commit f7ca7c1aed56bb3e5c6253be6e51c8a766ac1da3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Apr 20 20:50:36 2021 +0000

    Fix a bug in backend health state change timestamps.
    
    When admin commands control health, we report the timestamp of that
    override operation.
    
    When probing controls health, we report the timestamp of last
    probing state change.
    
    Otherwise, we report the timestamp of last admin health command.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 6ba15ffd8..3225e1f6d 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -688,17 +688,6 @@ VRT_delete_backend(VRT_CTX, VCL_BACKEND *dp)
 	// this is why we don't bust the director's magic number.
 }
 
-void
-VBE_SetHappy(const struct backend *be, uint64_t happy)
-{
-
-	CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
-	Lck_Lock(&backends_mtx);
-	if (be->vsc != NULL)
-		be->vsc->happy = happy;
-	Lck_Unlock(&backends_mtx);
-}
-
 /*---------------------------------------------------------------------*/
 
 void
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index 7eabfebc3..536494f7b 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -79,9 +79,6 @@ struct backend {
  * Prototypes
  */
 
-/* cache_backend_cfg.c */
-void VBE_SetHappy(const struct backend *, uint64_t);
-
 /* cache_backend_probe.c */
 void VBP_Update_Backend(struct vbp_target *vt);
 void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p,
diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c
index d1750ac63..18126cef2 100644
--- a/bin/varnishd/cache/cache_backend_probe.c
+++ b/bin/varnishd/cache/cache_backend_probe.c
@@ -54,6 +54,8 @@
 #include "cache_backend.h"
 #include "cache_conn_pool.h"
 
+#include "VSC_vbe.h"
+
 /* Default averaging rate, we want something pretty responsive */
 #define AVG_RATE			4
 
@@ -153,7 +155,6 @@ vbp_has_poked(struct vbp_target *vt)
 void
 VBP_Update_Backend(struct vbp_target *vt)
 {
-	const struct director *dir;
 	unsigned i = 0, chg;
 	char bits[10];
 
@@ -171,12 +172,6 @@ VBP_Update_Backend(struct vbp_target *vt)
 		return;
 	}
 
-	dir = vt->backend->director;
-	if (dir == NULL) {
-		Lck_Unlock(&vbp_mtx);
-		return;
-	}
-
 	i = (vt->good < vt->threshold);
 	chg = (i != vt->backend->sick);
 	vt->backend->sick = i;
@@ -187,12 +182,9 @@ VBP_Update_Backend(struct vbp_target *vt)
 	    i ? "sick" : "healthy", bits,
 	    vt->good, vt->threshold, vt->window,
 	    vt->last, vt->avg, vt->resp_buf);
-	VBE_SetHappy(vt->backend, vt->happy);
-
-	if (chg) {
+	vt->backend->vsc->happy = vt->happy;
+	if (chg)
 		vt->backend->changed = VTIM_real();
-		VRT_SetChanged(dir, vt->backend->changed);
-	}
 	Lck_Unlock(&vbp_mtx);
 }
 

From phk at FreeBSD.org  Tue Apr 20 21:10:13 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 20 Apr 2021 21:10:13 +0000 (UTC)
Subject: [master] 193570e5d Whitespace CDO.
Message-ID: <20210420211013.88DD5ABD88@lists.varnish-cache.org>


commit 193570e5d896ef6ffef29da5b5899f78c39d9a7e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Apr 20 20:56:46 2021 +0000

    Whitespace CDO.

diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c
index 6f595f7ad..f4a53a5aa 100644
--- a/lib/libvcc/vcc_acl.c
+++ b/lib/libvcc/vcc_acl.c
@@ -593,7 +593,7 @@ vcc_acl_emit(struct vcc *tl, const struct symbol *sym)
 	if (!tl->err_unref) {
 		ifp = New_IniFin(tl);
 		VSB_printf(ifp->ini,
-			"\tif (0) %s(0, 0);\n", VSB_data(func));
+			"\t(void)%s;\n", VSB_data(func));
 	}
 
 	VRBT_FOREACH(ae, acl_tree, &tl->acl->acl_tree) {
@@ -683,7 +683,7 @@ vcc_acl_emit(struct vcc *tl, const struct symbol *sym)
 	Fh(tl, 0, "}};\n\n");
 	if (!tl->err_unref) {
 		AN(ifp);
-		VSB_printf(ifp->ini, "\t(void)%s;\n", sym->rname);
+		VSB_printf(ifp->ini, "\t(void)%s;", sym->rname);
 	}
 	VSB_destroy(&func);
 }
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index e776f794c..ac1c9e1f4 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -353,8 +353,8 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 		vcc_NextToken(tl);
 		SkipToken(tl, ';');
 		ifp = New_IniFin(tl);
-		VSB_printf(ifp->ini, "\t(void)%s;\n", vgcname);
-		VSB_printf(ifp->fin, "\t\t(void)%s;\n", vgcname);
+		VSB_printf(ifp->ini, "\t(void)%s;", vgcname);
+		VSB_printf(ifp->fin, "\t\t(void)%s;", vgcname);
 		return;
 	}
 

From phk at FreeBSD.org  Wed Apr 21 10:59:05 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 21 Apr 2021 10:59:05 +0000 (UTC)
Subject: [master] 47586588f Dont double-free the director-lock
Message-ID: <20210421105905.E517B9821A@lists.varnish-cache.org>


commit 47586588fccde1869a8bec76001406fc6ebb0b84
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Apr 21 10:58:30 2021 +0000

    Dont double-free the director-lock

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 3225e1f6d..c86a70a0b 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -446,7 +446,6 @@ vbe_free(struct backend *be)
 #undef DN
 	free(be->endpoint);
 
-	Lck_Delete(be->director->mtx);
 	FREE_OBJ(be);
 }
 

From reza at naghibi.com  Wed Apr 21 14:29:05 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 21 Apr 2021 14:29:05 +0000 (UTC)
Subject: [6.0] 5b1905630 generalize the worker pool reserve to avoid deadlocks
Message-ID: <20210421142905.74907A1E20@lists.varnish-cache.org>


commit 5b19056303a273853842d84bd1376f5cc2dad3f2
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Oct 9 13:16:10 2018 +0200

    generalize the worker pool reserve to avoid deadlocks
    
    Previously, we used a minimum number of idle threads (the reserve) to
    ensure that we do not assign all threads with client requests and no
    threads left over for backend requests.
    
    This was actually only a special case of the more general issue
    exposed by h2: Lower priority tasks depend on higher priority tasks
    (for h2, sessions need streams, which need requests, which may need
    backend requests).
    
    To solve this problem, we divide the reserve by the number of priority
    classes and schedule lower priority tasks only if there are enough
    idle threads to run higher priority tasks eventually.
    
    This change does not guarantee any upper limit on the amount of time
    it can take for a task to be scheduled (e.g. backend requests could be
    blocking on arbitrarily long timeouts), so the thread pool watchdog is
    still warranted. But this change should guarantee that we do make
    progress eventually.
    
    With the reserves, thread_pool_min needs to be no smaller than the
    number of priority classes (TASK_QUEUE__END). Ideally, we should have
    an even higher minimum (@Dridi rightly suggested to make it 2 *
    TASK_QUEUE__END), but that would prevent the very useful test
    t02011.vtc.
    
    For now, the value of TASK_QUEUE__END (5) is hardcoded as such for the
    parameter configuration and documentation because auto-generating it
    would require include/macro dances which I consider over the top for
    now. Instead, the respective places are marked and an assert is in
    place to ensure we do not start a worker with too small a number of
    workers. I dicided against checks in the manager to avoid include
    pollution from the worker (cache.h) into the manager.
    
    Fixes #2418 for real
    
     Conflicts:
            bin/varnishd/cache/cache_wrk.c
            bin/varnishd/mgt/mgt_pool.c

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 5cff79beb..ff85761c4 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -217,22 +217,20 @@ struct pool_task {
 /*
  * tasks are taken off the queues in this order
  *
- * prios up to TASK_QUEUE_RESERVE are run from the reserve
- *
  * TASK_QUEUE_{REQ|STR} are new req's (H1/H2), and subject to queue limit.
  *
- * TASK_QUEUE_RUSH is req's returning from waiting list, they are
- * not subject to TASK_QUEUE_CLIENT because we cannot safely clean
- * them up if scheduling them fails.
+ * TASK_QUEUE_RUSH is req's returning from waiting list
+ *
+ * NOTE: When changing the number of classes, update places marked with
+ * TASK_QUEUE__END in mgt_pool.c
  */
 enum task_prio {
 	TASK_QUEUE_BO,
-#define TASK_QUEUE_RESERVE	TASK_QUEUE_BO
 	TASK_QUEUE_RUSH,
 	TASK_QUEUE_REQ,
 	TASK_QUEUE_STR,
 	TASK_QUEUE_VCA,
-	TASK_QUEUE_END
+	TASK_QUEUE__END
 };
 
 #define TASK_QUEUE_CLIENT(prio) \
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index 35be8ad00..3be42a034 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -148,7 +148,7 @@ pool_mkpool(unsigned pool_no)
 
 	VTAILQ_INIT(&pp->idle_queue);
 	VTAILQ_INIT(&pp->poolsocks);
-	for (i = 0; i < TASK_QUEUE_END; i++)
+	for (i = 0; i < TASK_QUEUE__END; i++)
 		VTAILQ_INIT(&pp->queues[i]);
 	AZ(pthread_cond_init(&pp->herder_cond, NULL));
 	AZ(pthread_create(&pp->herder_thr, NULL, pool_herder, pp));
diff --git a/bin/varnishd/cache/cache_pool.h b/bin/varnishd/cache/cache_pool.h
index 4e668fa0b..b4b5890c5 100644
--- a/bin/varnishd/cache/cache_pool.h
+++ b/bin/varnishd/cache/cache_pool.h
@@ -46,7 +46,7 @@ struct pool {
 	struct lock			mtx;
 	unsigned			nidle;
 	struct taskhead			idle_queue;
-	struct taskhead			queues[TASK_QUEUE_END];
+	struct taskhead			queues[TASK_QUEUE__END];
 	unsigned			nthr;
 	unsigned			lqueue;
 	uintmax_t			sdropped;
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 15255736a..e148cdca2 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -170,12 +170,16 @@ pool_reserve(void)
 {
 	unsigned lim;
 
-	if (cache_param->wthread_reserve == 0)
-		return (cache_param->wthread_min / 20 + 1);
-	lim = cache_param->wthread_min * 950 / 1000;
-	if (cache_param->wthread_reserve > lim)
-		return (lim);
-	return (cache_param->wthread_reserve);
+	if (cache_param->wthread_reserve == 0) {
+		lim = cache_param->wthread_min / 20 + 1;
+	} else {
+		lim = cache_param->wthread_min * 950 / 1000;
+		if (cache_param->wthread_reserve < lim)
+			lim = cache_param->wthread_reserve;
+	}
+	if (lim < TASK_QUEUE__END)
+		return (TASK_QUEUE__END);
+	return (lim);
 }
 
 /*--------------------------------------------------------------------*/
@@ -188,7 +192,7 @@ pool_getidleworker(struct pool *pp, enum task_prio prio)
 
 	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 	Lck_AssertHeld(&pp->mtx);
-	if (prio <= TASK_QUEUE_RESERVE || pp->nidle > pool_reserve()) {
+	if (pp->nidle > (pool_reserve() * prio / TASK_QUEUE__END)) {
 		pt = VTAILQ_FIRST(&pp->idle_queue);
 		if (pt == NULL)
 			AZ(pp->nidle);
@@ -257,7 +261,7 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio prio)
 	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 	AN(task);
 	AN(task->func);
-	assert(prio < TASK_QUEUE_END);
+	assert(prio < TASK_QUEUE__END);
 
 	if (prio == TASK_QUEUE_REQ && reqpoolfail) {
 		retval = reqpoolfail & 1;
@@ -332,7 +336,7 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk)
 	struct pool_task *tp = NULL;
 	struct pool_task tpx, tps;
 	vtim_real tmo;
-	int i, prio_lim;
+	int i, reserve;
 
 	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 	wrk->pool = pp;
@@ -343,12 +347,11 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk)
 		AZ(wrk->vsl);
 
 		Lck_Lock(&pp->mtx);
-		if (pp->nidle < pool_reserve())
-			prio_lim = TASK_QUEUE_RESERVE + 1;
-		else
-			prio_lim = TASK_QUEUE_END;
+		reserve = pool_reserve();
 
-		for (i = 0; i < prio_lim; i++) {
+		for (i = 0; i < TASK_QUEUE__END; i++) {
+			if (pp->nidle < (reserve * i / TASK_QUEUE__END))
+				break;
 			tp = VTAILQ_FIRST(&pp->queues[i]);
 			if (tp != NULL) {
 				pp->lqueue--;
@@ -651,6 +654,6 @@ static struct cli_proto debug_cmds[] = {
 void
 WRK_Init(void)
 {
-
-        CLI_AddFuncs(debug_cmds);
+	assert(cache_param->wthread_min >= TASK_QUEUE__END);
+	CLI_AddFuncs(debug_cmds);
 }
diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index 98ad42c3c..136127cf5 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -57,7 +57,6 @@ static int
 tweak_thread_pool_min(struct vsb *vsb, const struct parspec *par,
     const char *arg)
 {
-
 	if (tweak_generic_uint(vsb, par->priv, arg, par->min, par->max))
 		return (-1);
 	MCF_ParamConf(MCF_MINIMUM, "thread_pool_max",
@@ -110,14 +109,17 @@ struct parspec WRK_parspec[] = {
 		DELAYED_EFFECT,
 		"5000", "threads" },
 	{ "thread_pool_min", tweak_thread_pool_min, &mgt_param.wthread_min,
-		NULL, NULL,
+		"5", // TASK_QUEUE__END
+		NULL,
 		"The minimum number of worker threads in each pool. The "
 		"maximum value depends on thread_pool_max.\n"
 		"\n"
 		"Increasing this may help ramp up faster from low load "
 		"situations or when threads have expired.\n"
 		"\n"
-		"Minimum is 10 threads.",
+		"Technical minimum is 5 threads, " // TASK_QUEUE__END
+		"but this parameter is strongly recommended to be "
+		"at least 10", // 2 * TASK_QUEUE__END
 		DELAYED_EFFECT,
 		"100", "threads" },
 	{ "thread_pool_reserve", tweak_uint, &mgt_param.wthread_reserve,
@@ -126,16 +128,15 @@ struct parspec WRK_parspec[] = {
 		"in each pool.\n"
 		"\n"
 		"Tasks may require other tasks to complete (for example, "
-		"client requests may require backend requests). This reserve "
-		"is to ensure that such tasks still get to run even under high "
-		"load.\n"
-		"\n"
-		"Increasing the reserve may help setups with a high number of "
-		"backend requests at the expense of client performance. "
-		"Setting it too high will waste resources by keeping threads "
-		"unused.\n"
+		"client requests may require backend requests, http2 sessions "
+		"require streams, which require requests). This reserve is to "
+		"ensure that lower priority tasks do not prevent higher "
+		"priority tasks from running even under high load.\n"
 		"\n"
-		"Default is 0 to auto-tune (currently 5% of thread_pool_min).\n"
+		"The effective value is at least 5 (the number of internal "
+		//				 ^ TASK_QUEUE__END
+		"priority classes), irrespective of this parameter.\n"
+		"Default is 0 to auto-tune (5% of thread_pool_min).\n"
 		"Minimum is 1 otherwise, maximum is 95% of thread_pool_min.",
 		DELAYED_EFFECT,
 		"0", "threads" },
diff --git a/bin/varnishtest/tests/r01490.vtc b/bin/varnishtest/tests/r01490.vtc
index 88c2550aa..efa6d2b8f 100644
--- a/bin/varnishtest/tests/r01490.vtc
+++ b/bin/varnishtest/tests/r01490.vtc
@@ -6,8 +6,8 @@ server s1 {
 varnish v1 \
 	-arg "-p debug=+syncvsl" \
 	-arg "-p vsl_mask=+WorkThread" \
-	-arg "-p thread_pool_min=2" \
-	-arg "-p thread_pool_max=3" \
+	-arg "-p thread_pool_min=5" \
+	-arg "-p thread_pool_max=6" \
 	-arg "-p thread_pools=1" \
 	-arg "-p thread_pool_timeout=10" \
 	-vcl+backend {}
@@ -15,27 +15,27 @@ varnish v1 -start
 
 delay 2
 
-varnish v1 -expect threads == 2
+varnish v1 -expect threads == 5
 
 logexpect l1 -v v1 -g raw {
 	expect * 0 WorkThread {^\S+ start$}
 	expect * 0 WorkThread {^\S+ end$}
 } -start
 
-varnish v1 -cliok "param.set thread_pool_min 3"
+varnish v1 -cliok "param.set thread_pool_min 6"
 
 # Have to wait longer than thread_pool_timeout
 delay 11
 
-varnish v1 -expect threads == 3
+varnish v1 -expect threads == 6
 
-varnish v1 -cliok "param.set thread_pool_min 2"
-varnish v1 -cliok "param.set thread_pool_max 2"
+varnish v1 -cliok "param.set thread_pool_min 5"
+varnish v1 -cliok "param.set thread_pool_max 5"
 
 # Have to wait longer than thread_pool_timeout
 delay 11
 
-varnish v1 -expect threads == 2
+varnish v1 -expect threads == 5
 
 # Use logexpect to see that the thread actually exited
 logexpect l1 -wait
diff --git a/bin/varnishtest/tests/t02011.vtc b/bin/varnishtest/tests/t02011.vtc
index 9c2269e74..5c94e3040 100644
--- a/bin/varnishtest/tests/t02011.vtc
+++ b/bin/varnishtest/tests/t02011.vtc
@@ -16,15 +16,12 @@ server s1 {
 # - one for stream 1
 # - one for the backend request
 #
-# To work around the reserve's default logic, an additional thread can be
-# created, but won't be consumed for stream 3 because the pool will reserve
-# it for a backend transaction. Otherwise it could starve the pool and dead
-# lock v1.
+# thread priorities ensure that there is exactly one thread per class
+# at this point, so when we try to get a second stream, we fail.
 
 varnish v1 -cliok "param.set thread_pools 1"
-varnish v1 -cliok "param.set thread_pool_min 4"
-varnish v1 -cliok "param.set thread_pool_max 4"
-varnish v1 -cliok "param.set thread_pool_reserve 1"
+varnish v1 -cliok "param.set thread_pool_min 5"
+varnish v1 -cliok "param.set thread_pool_max 5"
 varnish v1 -cliok "param.set thread_queue_limit 0"
 varnish v1 -cliok "param.set thread_stats_rate 1"
 varnish v1 -cliok "param.set feature +http2"
@@ -69,7 +66,10 @@ client c1 {
 } -run
 
 # trigger an update of the stats
-varnish v1 -cliok "param.set thread_pool_min 3"
+varnish v1 -cliok "param.set thread_pool_max 6"
+varnish v1 -cliok "param.set thread_pool_min 6"
+delay 1
+varnish v1 -cliok "param.set thread_pool_min 5"
 delay 1
 varnish v1 -vsl_catchup
 varnish v1 -expect sess_drop == 0

From reza at naghibi.com  Wed Apr 21 14:29:05 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 21 Apr 2021 14:29:05 +0000 (UTC)
Subject: [6.0] 78c9b146e fix missing initialization
Message-ID: <20210421142905.88F98A1E23@lists.varnish-cache.org>


commit 78c9b146e663a32140a2d40da2e6f7c4858d1c24
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Sat Jan 18 14:19:10 2020 +0100

    fix missing initialization
    
    ... introduced with 3bb8b84cd86a4d2b95c7e2508ace6d868ced412c:
    
    in Pool_Work_Thread(), we could break out of the for (i = 0; i <
    TASK_QUEUE__END; i++) loop with tp set to the value from the previous
    iteration of the top while() loop where if should have been NULL (for no
    task found).
    
    Noticed staring at #3192 - unclear yet if related

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index e148cdca2..2083e4cf7 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -333,7 +333,7 @@ pool_kiss_of_death(struct worker *wrk, void *priv)
 static void
 Pool_Work_Thread(struct pool *pp, struct worker *wrk)
 {
-	struct pool_task *tp = NULL;
+	struct pool_task *tp;
 	struct pool_task tpx, tps;
 	vtim_real tmo;
 	int i, reserve;
@@ -342,6 +342,7 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk)
 	wrk->pool = pp;
 	while (1) {
 		CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+		tp = NULL;
 
 		WS_Reset(wrk->aws, 0);
 		AZ(wrk->vsl);

From reza at naghibi.com  Wed Apr 21 14:29:05 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 21 Apr 2021 14:29:05 +0000 (UTC)
Subject: [6.0] 8adc67319 remove a now pointless vtc
Message-ID: <20210421142905.A5475A1E26@lists.varnish-cache.org>


commit 8adc67319a331b481d91094d1c9f1845c54bf71f
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 28 18:34:53 2019 +0100

    remove a now pointless vtc
    
    This test is to detect a deadlock which does not exist any more. IMHO,
    the only sensible way to test for the lack of it now is to do a load
    test, which is not what we want in vtc.

diff --git a/bin/varnishtest/tests/r02418.vtc b/bin/varnishtest/tests/r02418.vtc
deleted file mode 100644
index a03ca546c..000000000
--- a/bin/varnishtest/tests/r02418.vtc
+++ /dev/null
@@ -1,72 +0,0 @@
-varnishtest "h2 queuing deadlock"
-
-barrier b1 cond 2
-
-# A reserve of 1 thread in a pool of 3 leaves a maximum
-# of 2 running sessions, the streams will be queued (except
-# stream 0 that is part of the h2 session).
-
-varnish v1 -cliok "param.set thread_pools 1"
-varnish v1 -cliok "param.set thread_pool_min 3"
-varnish v1 -cliok "param.set thread_pool_max 3"
-varnish v1 -cliok "param.set thread_pool_reserve 1"
-varnish v1 -cliok "param.set thread_pool_watchdog 5"
-varnish v1 -cliok "param.set feature +http2"
-varnish v1 -cliok "param.set feature +no_coredump"
-
-varnish v1 -vcl {
-	backend b1 { .host = "${bad_ip}"; }
-	sub vcl_recv {
-		return (synth(200));
-	}
-} -start
-
-logexpect l1 -v v1 -g raw {
-	expect * *	Error	"Pool Herder: Queue does not move*"
-} -start
-
-# Starve the pool with h2 sessions
-
-client c1 {
-	txpri
-	stream 0 rxsettings -run
-
-	barrier b1 sync
-
-	stream 1 {
-		txreq
-		# can't be scheduled, don't rx
-	} -run
-} -start
-
-client c2 {
-	txpri
-	stream 0 rxsettings -run
-
-	barrier b1 sync
-
-	stream 1 {
-		txreq
-		# can't be scheduled, don't rx
-	} -run
-} -start
-
-client c1 -wait
-client c2 -wait
-
-varnish v1 -vsl_catchup
-
-# At this point c1 and c2 closed their connections
-
-client c3 {
-	txreq
-	delay 10
-} -run
-
-logexpect l1 -wait
-
-varnish v1 -cliok panic.show
-varnish v1 -cliok panic.clear
-
-varnish v1 -expectexit 0x20
-

From reza at naghibi.com  Wed Apr 21 14:29:05 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 21 Apr 2021 14:29:05 +0000 (UTC)
Subject: [6.0] 42267902c Use the REQ priority for incoming connection tasks by
 the acceptor
Message-ID: <20210421142905.C494AA1E2C@lists.varnish-cache.org>


commit 42267902cb9b4c4c757a78f7b2f8b88db60b9160
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Feb 17 13:39:37 2021 +0100

    Use the REQ priority for incoming connection tasks by the acceptor
    
    When accepting new incoming connections in the acceptor thread, it would
    schedule, they would be registered with the VCA priority. This priority is
    reserved for the acceptor thread itself, and specifically is not included
    in the TASK_QUEUE_CLIENT categorisation. This would interfere with the
    thread reserve pools.
    
    t02011.vtc had to be adjusted to account for the new priority
    categorisation of the initial request.

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 0029a68d3..035540ff6 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -518,7 +518,7 @@ vca_accept_task(struct worker *wrk, void *arg)
 
 		wa.acceptsock = i;
 
-		if (!Pool_Task_Arg(wrk, TASK_QUEUE_VCA,
+		if (!Pool_Task_Arg(wrk, TASK_QUEUE_REQ,
 		    vca_make_session, &wa, sizeof wa)) {
 			/*
 			 * We couldn't get another thread, so we will handle
diff --git a/bin/varnishtest/tests/t02011.vtc b/bin/varnishtest/tests/t02011.vtc
index 5c94e3040..c0b6a6900 100644
--- a/bin/varnishtest/tests/t02011.vtc
+++ b/bin/varnishtest/tests/t02011.vtc
@@ -20,8 +20,8 @@ server s1 {
 # at this point, so when we try to get a second stream, we fail.
 
 varnish v1 -cliok "param.set thread_pools 1"
-varnish v1 -cliok "param.set thread_pool_min 5"
-varnish v1 -cliok "param.set thread_pool_max 5"
+varnish v1 -cliok "param.set thread_pool_min 6"
+varnish v1 -cliok "param.set thread_pool_max 6"
 varnish v1 -cliok "param.set thread_queue_limit 0"
 varnish v1 -cliok "param.set thread_stats_rate 1"
 varnish v1 -cliok "param.set feature +http2"
@@ -66,10 +66,10 @@ client c1 {
 } -run
 
 # trigger an update of the stats
-varnish v1 -cliok "param.set thread_pool_max 6"
-varnish v1 -cliok "param.set thread_pool_min 6"
+varnish v1 -cliok "param.set thread_pool_max 7"
+varnish v1 -cliok "param.set thread_pool_min 7"
 delay 1
-varnish v1 -cliok "param.set thread_pool_min 5"
+varnish v1 -cliok "param.set thread_pool_min 6"
 delay 1
 varnish v1 -vsl_catchup
 varnish v1 -expect sess_drop == 0

From reza at naghibi.com  Wed Apr 21 14:30:07 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 21 Apr 2021 14:30:07 +0000 (UTC)
Subject: [6.0] 444378371 Limit watchdog to highest priority only
Message-ID: <20210421143007.57D4EA24BC@lists.varnish-cache.org>


commit 444378371c222c26eb98e6fdad3cf0e77faa4cd7
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Feb 25 15:58:27 2021 +0100

    Limit watchdog to highest priority only
    
    The watchdog mechanism currently triggers when any queueing is happening,
    regardless of the priority. Strictly speaking it is only the backend
    fetches that are critical to get executed, and this prevents the thread
    limits to be used as limits on the amount of work the Varnish instance
    should handle.
    
    This can be especially important for instances with H/2 enabled, as these
    connections will be holding threads for extended periods of time, possibly
    triggering the watchdog in benign situations.
    
    This patch limits the watchdog to only trigger for no queue development
    on the highest priority queue.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index ff85761c4..db5467a8d 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -233,6 +233,7 @@ enum task_prio {
 	TASK_QUEUE__END
 };
 
+#define TASK_QUEUE_HIGHEST_PRIORITY TASK_QUEUE_BO
 #define TASK_QUEUE_CLIENT(prio) \
 	(prio == TASK_QUEUE_REQ || prio == TASK_QUEUE_STR)
 
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 2083e4cf7..eacf21c04 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -532,7 +532,9 @@ pool_herder(void *priv)
 		 * Instead we implement a watchdog and kill the worker if
 		 * nothing has been dequeued for that long.
 		 */
-		if (pp->lqueue == 0) {
+		if (VTAILQ_EMPTY(&pp->queues[TASK_QUEUE_HIGHEST_PRIORITY])) {
+			/* Watchdog only applies to no movement on the
+			 * highest priority queue (TASK_QUEUE_BO) */
 			dq = pp->ndequeued + 1;
 		} else if (dq != pp->ndequeued) {
 			dq = pp->ndequeued;
diff --git a/bin/varnishtest/tests/c00104.vtc b/bin/varnishtest/tests/c00104.vtc
new file mode 100644
index 000000000..3af498179
--- /dev/null
+++ b/bin/varnishtest/tests/c00104.vtc
@@ -0,0 +1,32 @@
+varnishtest "Test watchdog only active on queue 0"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -cliok "param.set thread_pools 1"
+varnish v1 -cliok "param.set thread_pool_min 5"
+varnish v1 -cliok "param.set thread_pool_max 5"
+varnish v1 -cliok "param.set thread_pool_watchdog 1"
+varnish v1 -cliok "param.set feature +http2"
+
+varnish v1 -vcl+backend {
+} -start
+
+client c1 {
+	txpri
+	delay 2
+} -start
+
+client c2 {
+	txpri
+	delay 2
+} -start
+
+client c3 {
+	txpri
+	delay 2
+} -start
+
+delay 2

From reza at naghibi.com  Wed Apr 21 18:20:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 21 Apr 2021 18:20:06 +0000 (UTC)
Subject: [6.0] 88c2b20ac properly maintain the obans list when pruning the ban
 list tail
Message-ID: <20210421182006.B9FF3A9098@lists.varnish-cache.org>


commit 88c2b20ac781b2b23ca5a73b4a58ac91e2ef87f6
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri May 31 15:51:08 2019 +0200

    properly maintain the obans list when pruning the ban list tail
    
    background: When the ban lurker has finished working the bottom of the
    ban list, conceptually we mark all bans it has evaluated as completed
    and then remove the tail of the ban list which has no references any
    more.
    
    Yet, for efficiency, we first remove the tail and then mark only those
    bans completed, which we did not remove. Doing so depends on knowing
    where in the (obans) list of bans to be completed is the new tail of
    the bans list after pruning.
    
    5dd54f8390739c62c201e62c36eb515b1e03c2ee was intended to solve this,
    but the fix was incomplete (and also unnecessarily complicated): For
    example when a duplicate ban was issued, ban_lurker_test_ban() could
    remove a ban from the obans list which later happens to become the new
    ban tail.
    
    We now - hopefully - solve the problem for real by properly cleaning
    the obans list when we prune the ban list.
    
    Fixes #3006
    Fixes #2779
    Fixes #2556 for real (5dd54f8390739c62c201e62c36eb515b1e03c2ee was
    incomplete)
    
     Conflicts:
            bin/varnishd/cache/cache_ban_lurker.c

diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c
index abfbbacdd..245fb9dea 100644
--- a/bin/varnishd/cache/cache_ban_lurker.c
+++ b/bin/varnishd/cache/cache_ban_lurker.c
@@ -58,19 +58,18 @@ ban_kick_lurker(void)
  * still referenced. For already completed bans, we update statistics
  * accordingly, but otherwise just skip the completion step and remove directly
  *
- * return 1 if we removed the victim, 0 otherwise
+ * if an obans list is passed, we clean its tail as well
  */
 
-static int
-ban_cleantail(const struct ban *victim)
+static void
+ban_cleantail(struct banhead_s *obans)
 {
 	struct ban *b, *bt;
 	struct banhead_s freelist = VTAILQ_HEAD_INITIALIZER(freelist);
-	int r = 0;
 
 	/* handle the zero-length tail unprotected */
 	if (VTAILQ_LAST(&ban_head, banhead_s) == VTAILQ_FIRST(&ban_head))
-		return (r);
+		return;
 
 	Lck_Lock(&ban_mtx);
 	do {
@@ -99,13 +98,27 @@ ban_cleantail(const struct ban *victim)
 
 	Lck_Unlock(&ban_mtx);
 
-	VTAILQ_FOREACH_SAFE(b, &freelist, list, bt) {
-		if (b == victim)
-			r = 1;
-		BAN_Free(b);
+	/* oban order is head to tail, freelist tail to head */
+	if (obans != NULL)
+		bt = VTAILQ_LAST(obans, banhead_s);
+	else
+		bt = NULL;
+
+	if (bt != NULL) {
+		VTAILQ_FOREACH(b, &freelist, list) {
+			if (b != bt)
+				continue;
+			VTAILQ_REMOVE(obans, b, l_list);
+			bt = VTAILQ_LAST(obans, banhead_s);
+			if (bt == NULL)
+				break;
+		}
 	}
 
-	return (r);
+	VTAILQ_FOREACH_SAFE(b, &freelist, list, bt)
+		BAN_Free(b);
+
+	return;
 }
 
 /*--------------------------------------------------------------------
@@ -327,7 +340,7 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
 
 	dt = 49.62;		// Random, non-magic
 	if (cache_param->ban_lurker_sleep == 0) {
-		(void)ban_cleantail(NULL);
+		ban_cleantail(NULL);
 		return (dt);
 	}
 	if (cache_param->ban_cutoff > 0)
@@ -363,42 +376,18 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
 	}
 
 	/*
-	 * conceptually, all obans are now completed. Remove the tail. If it
-	 * containted the first oban, all obans were on the tail and we're
-	 * done.
+	 * conceptually, all obans are now completed. Remove the tail.
+	 * If any bans to be completed remain after the tail is cut,
+	 * mark them completed
 	 */
-	if (ban_cleantail(VTAILQ_FIRST(&obans)))
-		return (dt);
+	ban_cleantail(&obans);
 
 	if (VTAILQ_FIRST(&obans) == NULL)
 		return (dt);
 
-	/*
-	 * Mark remaining bans completed: the tail of the obans list is now
-	 * removed, but iterating over it is safe until we hit the new ban list
-	 * tail
-	 *
-	 * bans at the tail of the list may have been completed by other means
-	 * and, consequently, may have been removed from obans, so we skip all
-	 * already completed bans at the tail.
-	 *
-	 * While traversing the ban list backwards, we check if we pass by the
-	 * first oban, in which case we're done.
-	 */
-	bd = VTAILQ_LAST(&ban_head, banhead_s);
-	while (bd->flags & BANS_FLAG_COMPLETED) {
-		if (bd == VTAILQ_FIRST(&ban_head) ||
-		    bd == VTAILQ_FIRST(&obans))
-			return (dt);
-		bd = VTAILQ_PREV(bd, banhead_s, list);
-	}
-
 	Lck_Lock(&ban_mtx);
-	VTAILQ_FOREACH(b, &obans, l_list) {
+	VTAILQ_FOREACH(b, &obans, l_list)
 		ban_mark_completed(b);
-		if (b == bd)
-			break;
-	}
 	Lck_Unlock(&ban_mtx);
 	return (dt);
 }
diff --git a/bin/varnishtest/tests/r03006.vtc b/bin/varnishtest/tests/r03006.vtc
new file mode 100644
index 000000000..9a48e7536
--- /dev/null
+++ b/bin/varnishtest/tests/r03006.vtc
@@ -0,0 +1,54 @@
+varnishtest "test ban lurker destination being completed by dup ban"
+
+server s1 -repeat 4 -keepalive {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {} -start
+
+varnish v1 -cliok "param.set ban_lurker_age 99"
+
+# this ban becomes the pruned tail
+varnish v1 -cliok "ban obj.http.t == 1"
+client c1 {
+	txreq -url /1
+	rxresp
+} -run
+
+# this ban becomes the new tail
+varnish v1 -cliok "ban obj.http.t == 2"
+client c1 {
+	txreq -url /2
+	rxresp
+} -run
+
+# req ban to define where the tail goes (at t == 2)
+varnish v1 -cliok "ban req.http.barrier == here"
+client c1 {
+	txreq -url /3
+	rxresp
+} -run
+
+# dup ban to trigger #3006
+varnish v1 -cliok "ban obj.http.t == 2"
+client c1 {
+	txreq -url /4
+	rxresp
+} -run
+
+varnish v1 -cliok "ban.list"
+
+varnish v1 -cliok "param.set ban_lurker_age 0.1"
+
+varnish v1 -cliok "ban.list"
+
+delay 2
+
+varnish v1 -expect bans == 3
+varnish v1 -expect bans_completed == 2
+
+varnish v1 -cliok "ban.list"
+
+varnish v1 -expect bans == 3
+varnish v1 -expect bans_completed == 2

From reza at naghibi.com  Wed Apr 21 18:26:05 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 21 Apr 2021 18:26:05 +0000 (UTC)
Subject: [6.0] 3f048b835 Move the locking calls outside exp_mail_it
Message-ID: <20210421182605.C5106A94DA@lists.varnish-cache.org>


commit 3f048b8352b9fc2895fdde36c58daafb070dad02
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Mar 19 11:08:50 2020 +0100

    Move the locking calls outside exp_mail_it
    
    This enables doing extra handling while holding the mutex specific to
    EXP_Insert/EXP_Remove before/after calling exp_mail_it.

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 3f2a11caf..e9f12e7b2 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -102,7 +102,8 @@ exp_mail_it(struct objcore *oc, uint8_t cmds)
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	assert(oc->refcnt > 0);
 
-	Lck_Lock(&exphdl->mtx);
+	Lck_AssertHeld(&exphdl->mtx);
+
 	if ((cmds | oc->exp_flags) & OC_EF_REFD) {
 		if (!(oc->exp_flags & OC_EF_POSTED)) {
 			if (cmds & OC_EF_REMOVE)
@@ -117,7 +118,6 @@ exp_mail_it(struct objcore *oc, uint8_t cmds)
 		VSC_C_main->exp_mailed++;
 		AZ(pthread_cond_signal(&exphdl->condvar));
 	}
-	Lck_Unlock(&exphdl->mtx);
 }
 
 /*--------------------------------------------------------------------
@@ -129,8 +129,11 @@ EXP_Remove(struct objcore *oc)
 {
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-	if (oc->exp_flags & OC_EF_REFD)
+	if (oc->exp_flags & OC_EF_REFD) {
+		Lck_Lock(&exphdl->mtx);
 		exp_mail_it(oc, OC_EF_REMOVE);
+		Lck_Unlock(&exphdl->mtx);
+	}
 }
 
 /*--------------------------------------------------------------------
@@ -147,11 +150,13 @@ EXP_Insert(struct worker *wrk, struct objcore *oc)
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	assert(oc->refcnt >= 2);
 
-	AZ(oc->exp_flags & (OC_EF_INSERT | OC_EF_MOVE));
 	AZ(oc->flags & OC_F_DYING);
 
 	ObjSendEvent(wrk, oc, OEV_INSERT);
+	Lck_Lock(&exphdl->mtx);
+	AZ(oc->exp_flags & (OC_EF_INSERT | OC_EF_MOVE));
 	exp_mail_it(oc, OC_EF_INSERT | OC_EF_REFD | OC_EF_MOVE);
+	Lck_Unlock(&exphdl->mtx);
 }
 
 /*--------------------------------------------------------------------
@@ -182,8 +187,11 @@ EXP_Rearm(struct objcore *oc, double now, double ttl, double grace, double keep)
 	VSL(SLT_ExpKill, 0, "EXP_Rearm p=%p E=%.6f e=%.6f f=0x%x", oc,
 	    oc->timer_when, when, oc->flags);
 
-	if (when < oc->t_origin || when < oc->timer_when)
+	if (when < oc->t_origin || when < oc->timer_when) {
+		Lck_Lock(&exphdl->mtx);
 		exp_mail_it(oc, OC_EF_MOVE);
+		Lck_Unlock(&exphdl->mtx);
+	}
 }
 
 /*--------------------------------------------------------------------

From reza at naghibi.com  Wed Apr 21 18:26:05 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 21 Apr 2021 18:26:05 +0000 (UTC)
Subject: [6.0] a12a65c3b Only count exp_mailed events when actually posting
Message-ID: <20210421182605.D4671A94DD@lists.varnish-cache.org>


commit a12a65c3b20bc145b931c8ad13e085bac2112bd8
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Mar 19 13:35:28 2020 +0100

    Only count exp_mailed events when actually posting
    
    When posting to the expiry thread, we wrongly counted exp_mailed also if
    the OC in question was already on the mail queue. This could cause a
    discrepency between the exp_mailed and exp_received counters.

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index e9f12e7b2..25d0c6e0e 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -112,10 +112,10 @@ exp_mail_it(struct objcore *oc, uint8_t cmds)
 			else
 				VSTAILQ_INSERT_TAIL(&exphdl->inbox,
 				    oc, exp_list);
+			VSC_C_main->exp_mailed++;
 		}
 		oc->exp_flags |= cmds | OC_EF_POSTED;
 		AN(oc->exp_flags & OC_EF_REFD);
-		VSC_C_main->exp_mailed++;
 		AZ(pthread_cond_signal(&exphdl->condvar));
 	}
 }

From reza at naghibi.com  Wed Apr 21 18:26:05 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 21 Apr 2021 18:26:05 +0000 (UTC)
Subject: [6.0] 0988d5f3c Repurpose OC_EF_REFD flag slightly
Message-ID: <20210421182606.02372A94E1@lists.varnish-cache.org>


commit 0988d5f3cfe37f4e5d022338dda1ea90e1b78c73
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Mar 19 15:17:45 2020 +0100

    Repurpose OC_EF_REFD flag slightly
    
    The OC_EF_REFD flag indicates whether expiry has a ref on the
    OC. Previously, the flag was only gained during the call to
    EXP_Insert. With this patch, and the helper function EXP_RefNewObjcore(),
    the flag is gained while holding the objhead mutex during
    HSH_Unbusy(). This enables the expiry functions to test on missing
    OC_EF_REFD and quickly return without having to take the main expiry
    mutex.
    
     Conflicts:
            bin/varnishd/cache/cache_varnishd.h

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 25d0c6e0e..3f81e2981 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -101,10 +101,11 @@ exp_mail_it(struct objcore *oc, uint8_t cmds)
 {
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	assert(oc->refcnt > 0);
+	AZ(cmds & OC_EF_REFD);
 
 	Lck_AssertHeld(&exphdl->mtx);
 
-	if ((cmds | oc->exp_flags) & OC_EF_REFD) {
+	if (oc->exp_flags & OC_EF_REFD) {
 		if (!(oc->exp_flags & OC_EF_POSTED)) {
 			if (cmds & OC_EF_REMOVE)
 				VSTAILQ_INSERT_HEAD(&exphdl->inbox,
@@ -115,11 +116,30 @@ exp_mail_it(struct objcore *oc, uint8_t cmds)
 			VSC_C_main->exp_mailed++;
 		}
 		oc->exp_flags |= cmds | OC_EF_POSTED;
-		AN(oc->exp_flags & OC_EF_REFD);
 		AZ(pthread_cond_signal(&exphdl->condvar));
 	}
 }
 
+/*--------------------------------------------------------------------
+ * Setup a new ObjCore for control by expire. Should be called with the
+ * ObjHead locked by HSH_Unbusy(/HSH_Insert) (in private access).
+ */
+
+void
+EXP_RefNewObjcore(struct objcore *oc)
+{
+	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+
+	Lck_AssertHeld(&oc->objhead->mtx);
+
+	AZ(oc->exp_flags);
+	assert(oc->refcnt >= 1);
+	oc->refcnt++;
+	oc->exp_flags |= OC_EF_REFD;
+}
+
+
+
 /*--------------------------------------------------------------------
  * Call EXP's attention to a an oc
  */
@@ -148,6 +168,10 @@ EXP_Insert(struct worker *wrk, struct objcore *oc)
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+
+	if (!(oc->exp_flags & OC_EF_REFD))
+		return;
+
 	assert(oc->refcnt >= 2);
 
 	AZ(oc->flags & OC_F_DYING);
@@ -155,7 +179,7 @@ EXP_Insert(struct worker *wrk, struct objcore *oc)
 	ObjSendEvent(wrk, oc, OEV_INSERT);
 	Lck_Lock(&exphdl->mtx);
 	AZ(oc->exp_flags & (OC_EF_INSERT | OC_EF_MOVE));
-	exp_mail_it(oc, OC_EF_INSERT | OC_EF_REFD | OC_EF_MOVE);
+	exp_mail_it(oc, OC_EF_INSERT | OC_EF_MOVE);
 	Lck_Unlock(&exphdl->mtx);
 }
 
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 9f1558aa0..38ef503ba 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -294,7 +294,7 @@ HSH_Insert(struct worker *wrk, const void *digest, struct objcore *oc,
 	   objecthead. The new object inherits our objhead reference. */
 	oc->objhead = oh;
 	VTAILQ_INSERT_TAIL(&oh->objcs, oc, hsh_list);
-	oc->refcnt++;				// For EXP_Insert
+	EXP_RefNewObjcore(oc);
 	Lck_Unlock(&oh->mtx);
 
 	BAN_RefBan(oc, ban);
@@ -772,7 +772,7 @@ HSH_Unbusy(struct worker *wrk, struct objcore *oc)
 	assert(oh->refcnt > 0);
 	assert(oc->refcnt > 0);
 	if (!(oc->flags & OC_F_PRIVATE))
-		oc->refcnt++;			// For EXP_Insert
+		EXP_RefNewObjcore(oc); /* Takes a ref for expiry */
 	/* XXX: strictly speaking, we should sort in Date: order. */
 	VTAILQ_REMOVE(&oh->objcs, oc, hsh_list);
 	VTAILQ_INSERT_HEAD(&oh->objcs, oc, hsh_list);
@@ -782,8 +782,8 @@ HSH_Unbusy(struct worker *wrk, struct objcore *oc)
 		hsh_rush1(wrk, oh, &rush, HSH_RUSH_POLICY);
 	}
 	Lck_Unlock(&oh->mtx);
-	if (!(oc->flags & OC_F_PRIVATE))
-		EXP_Insert(wrk, oc);
+	EXP_Insert(wrk, oc); /* Does nothing unless EXP_RefNewObjcore was
+			      * called */
 	hsh_rush2(wrk, &rush);
 }
 
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index cd97d824a..0141cf2ff 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -128,6 +128,7 @@ void VDI_Init(void);
 /* cache_exp.c */
 double EXP_Ttl(const struct req *, const struct objcore *);
 double EXP_Ttl_grace(const struct req *, const struct objcore *oc);
+void EXP_RefNewObjcore(struct objcore *);
 void EXP_Insert(struct worker *wrk, struct objcore *oc);
 void EXP_Remove(struct objcore *);
 

From reza at naghibi.com  Wed Apr 21 18:26:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 21 Apr 2021 18:26:06 +0000 (UTC)
Subject: [6.0] 039f65805 Execute EXP_Insert after unbusy in HSH_Insert
Message-ID: <20210421182606.1BECEA94E5@lists.varnish-cache.org>


commit 039f65805c75a3e8e7a6cb1f2ff7a8e86dc8c437
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Mar 23 14:23:02 2020 +0100

    Execute EXP_Insert after unbusy in HSH_Insert
    
    This makes the order of events the same as on real cache insertions.

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 38ef503ba..97dca05a2 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -299,7 +299,6 @@ HSH_Insert(struct worker *wrk, const void *digest, struct objcore *oc,
 
 	BAN_RefBan(oc, ban);
 	AN(oc->ban);
-	EXP_Insert(wrk, oc);
 
 	/* Move the object first in the oh list, unbusy it and run the
 	   waitinglist if necessary */
@@ -311,6 +310,8 @@ HSH_Insert(struct worker *wrk, const void *digest, struct objcore *oc,
 		hsh_rush1(wrk, oh, &rush, HSH_RUSH_POLICY);
 	Lck_Unlock(&oh->mtx);
 	hsh_rush2(wrk, &rush);
+
+	EXP_Insert(wrk, oc);
 }
 
 /*---------------------------------------------------------------------

From reza at naghibi.com  Wed Apr 21 18:26:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 21 Apr 2021 18:26:06 +0000 (UTC)
Subject: [6.0] c9e52f943 Allow EXP_Remove() to be called before EXP_Insert()
Message-ID: <20210421182606.39873A94E9@lists.varnish-cache.org>


commit c9e52f9434f043f2537afdcc87bd3b6ac8874dd7
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Mar 23 14:25:46 2020 +0100

    Allow EXP_Remove() to be called before EXP_Insert()
    
    Once HSH_Unbusy() has been called there is a possibility for
    EXP_Remove() to be called before the fetch thread has had a chance to call
    EXP_Insert(). By adding a OC_EF_NEW flag on the objects during
    HSH_Unbusy(), that is removed again during EXP_Insert(), we can keep track
    and clean up once EXP_Insert() is called by the inserting thread if
    EXP_Remove() was called in the mean time.
    
    This patch also removes the AZ(OC_F_DYING) in EXP_Insert(), as that is no
    longer a requirement.
    
    Fixes: #2999

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 3f81e2981..2ee61f8bb 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -135,7 +135,7 @@ EXP_RefNewObjcore(struct objcore *oc)
 	AZ(oc->exp_flags);
 	assert(oc->refcnt >= 1);
 	oc->refcnt++;
-	oc->exp_flags |= OC_EF_REFD;
+	oc->exp_flags |= OC_EF_REFD | OC_EF_NEW;
 }
 
 
@@ -151,7 +151,14 @@ EXP_Remove(struct objcore *oc)
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	if (oc->exp_flags & OC_EF_REFD) {
 		Lck_Lock(&exphdl->mtx);
-		exp_mail_it(oc, OC_EF_REMOVE);
+		if (oc->exp_flags & OC_EF_NEW) {
+			/* EXP_Insert has not been called for this object
+			 * yet. Mark it for removal, and EXP_Insert will
+			 * clean up once it is called. */
+			AZ(oc->exp_flags & OC_EF_POSTED);
+			oc->exp_flags |= OC_EF_REMOVE;
+		} else
+			exp_mail_it(oc, OC_EF_REMOVE);
 		Lck_Unlock(&exphdl->mtx);
 	}
 }
@@ -165,22 +172,37 @@ EXP_Remove(struct objcore *oc)
 void
 EXP_Insert(struct worker *wrk, struct objcore *oc)
 {
+	unsigned remove_race = 0;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 
+	AZ(oc->flags & OC_F_BUSY);
+
 	if (!(oc->exp_flags & OC_EF_REFD))
 		return;
 
 	assert(oc->refcnt >= 2);
 
-	AZ(oc->flags & OC_F_DYING);
-
 	ObjSendEvent(wrk, oc, OEV_INSERT);
+
 	Lck_Lock(&exphdl->mtx);
-	AZ(oc->exp_flags & (OC_EF_INSERT | OC_EF_MOVE));
-	exp_mail_it(oc, OC_EF_INSERT | OC_EF_MOVE);
+	AN(oc->exp_flags & OC_EF_NEW);
+	oc->exp_flags &= ~OC_EF_NEW;
+	AZ(oc->exp_flags & (OC_EF_INSERT | OC_EF_MOVE | OC_EF_POSTED));
+	if (oc->exp_flags & OC_EF_REMOVE) {
+		/* We raced some other thread executing EXP_Remove */
+		remove_race = 1;
+		oc->exp_flags &= ~(OC_EF_REFD | OC_EF_REMOVE);
+	} else
+		exp_mail_it(oc, OC_EF_INSERT | OC_EF_MOVE);
 	Lck_Unlock(&exphdl->mtx);
+
+	if (remove_race) {
+		ObjSendEvent(wrk, oc, OEV_EXPIRE);
+		(void)HSH_DerefObjCore(wrk, &oc, 0);
+		AZ(oc);
+	}
 }
 
 /*--------------------------------------------------------------------
@@ -213,7 +235,12 @@ EXP_Rearm(struct objcore *oc, double now, double ttl, double grace, double keep)
 
 	if (when < oc->t_origin || when < oc->timer_when) {
 		Lck_Lock(&exphdl->mtx);
-		exp_mail_it(oc, OC_EF_MOVE);
+		if (oc->exp_flags & OC_EF_NEW) {
+			/* EXP_Insert has not been called yet, do nothing
+			 * as the initial insert will execute the move
+			 * operation. */
+		} else
+			exp_mail_it(oc, OC_EF_MOVE);
 		Lck_Unlock(&exphdl->mtx);
 	}
 }
diff --git a/include/tbl/oc_exp_flags.h b/include/tbl/oc_exp_flags.h
index 83160b064..3f85ea807 100644
--- a/include/tbl/oc_exp_flags.h
+++ b/include/tbl/oc_exp_flags.h
@@ -33,6 +33,7 @@ OC_EXP_FLAG(REFD,	refd,		(1<<2))
 OC_EXP_FLAG(MOVE,	move,		(1<<3))
 OC_EXP_FLAG(INSERT,	insert,		(1<<4))
 OC_EXP_FLAG(REMOVE,	remove,		(1<<5))
+OC_EXP_FLAG(NEW,	new,		(1<<6))
 #undef OC_EXP_FLAG
 
 /*lint -restore */

From reza at naghibi.com  Thu Apr 22 15:55:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Thu, 22 Apr 2021 15:55:06 +0000 (UTC)
Subject: [6.0] 7f661b11a Use VTCP_Assert() also in VTCP_close()
Message-ID: <20210422155506.53FAAA81CE@lists.varnish-cache.org>


commit 7f661b11ad19831211cbf4a3b412ba8dd92613ee
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 7 14:35:23 2020 +0200

    Use VTCP_Assert() also in VTCP_close()
    
    Consistently use VTCP_Assert when asserting on the result of VTCP_Check().

diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index 85bac3979..eaf075c9a 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -342,7 +342,7 @@ VTCP_close(int *s)
 
 	i = close(*s);
 
-	assert(VTCP_Check(i));
+	VTCP_Assert(i);
 	*s = -1;
 }
 

From reza at naghibi.com  Thu Apr 22 15:55:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Thu, 22 Apr 2021 15:55:06 +0000 (UTC)
Subject: [6.0] aafb13257 Make VTCP_Check() accept EAGAIN/EWOULDBLOCK as
 acceptable errno
Message-ID: <20210422155506.75A38A81D3@lists.varnish-cache.org>


commit aafb132571ddb1047b6a59202a26702a8d66f5ff
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 7 14:56:38 2020 +0200

    Make VTCP_Check() accept EAGAIN/EWOULDBLOCK as acceptable errno
    
    When a socket timeout is set on a socket and the timeout expires, read()
    and write() calls on that socket will return (-1) with errno set to
    EAGAIN/EWOULDBLOCK.
    
     Conflicts:
            lib/libvarnish/vtcp.c

diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index eaf075c9a..9f253dd56 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -562,6 +562,14 @@ VTCP_Check(int a)
 		return (1);
 	if (errno == ECONNRESET || errno == ENOTCONN || errno == EPIPE)
 		return (1);
+	/* Accept EAGAIN (and EWOULDBLOCK in case they are not the same)
+	 * as errno values. Even though our sockets are all non-blocking,
+	 * when a SO_{SND|RCV}TIMEO expires, read() or write() on the
+	 * socket will return (-1) and errno set to EAGAIN. (This is not
+	 * documented in the read(2) and write(2) manpages, but is
+	 * described in the socket(7) manpage.) */
+	if (errno == EAGAIN || errno == EWOULDBLOCK)
+		return (1);
 #if (defined (__SVR4) && defined (__sun)) || defined (__NetBSD__)
 	/*
 	 * Solaris returns EINVAL if the other end unexpectedly reset the

From reza at naghibi.com  Thu Apr 22 15:55:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Thu, 22 Apr 2021 15:55:06 +0000 (UTC)
Subject: [6.0] 4db89ddab Change VTCP_Check() to take a ssize_t
Message-ID: <20210422155506.AE374A81D8@lists.varnish-cache.org>


commit 4db89ddab334e0325682966011f6923452ea5555
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 7 15:07:46 2020 +0200

    Change VTCP_Check() to take a ssize_t
    
    Since the input value is sometimes the result of a read()/write() call,
    avoid truncating the ssize_t value on calling it.

diff --git a/include/vtcp.h b/include/vtcp.h
index a5431aba6..226f70165 100644
--- a/include/vtcp.h
+++ b/include/vtcp.h
@@ -35,7 +35,7 @@ struct suckaddr;
 #define VTCP_ADDRBUFSIZE		64
 #define VTCP_PORTBUFSIZE		16
 
-int VTCP_Check(int a);
+int VTCP_Check(ssize_t a);
 #define VTCP_Assert(a) assert(VTCP_Check(a))
 
 struct suckaddr *VTCP_my_suckaddr(int sock);
diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index 9f253dd56..13b1ab9ed 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -556,7 +556,7 @@ VTCP_check_hup(int sock)
  */
 
 int
-VTCP_Check(int a)
+VTCP_Check(ssize_t a)
 {
 	if (a == 0)
 		return (1);

From reza at naghibi.com  Thu Apr 22 15:55:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Thu, 22 Apr 2021 15:55:06 +0000 (UTC)
Subject: [6.0] e8f20395e Enable the use of VTCP_Assert in v1f_read()
Message-ID: <20210422155506.C6447A81DC@lists.varnish-cache.org>


commit e8f20395e243235332ea1be54c1586d8b2a15360
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 7 15:35:13 2020 +0200

    Enable the use of VTCP_Assert in v1f_read()
    
    Now that VTCP_Assert() accepts EAGAIN as a legal errno value for read()
    errors, uncomment this check.

diff --git a/bin/varnishd/http1/cache_http1_vfp.c b/bin/varnishd/http1/cache_http1_vfp.c
index 7b03ca8ce..374bcac6a 100644
--- a/bin/varnishd/http1/cache_http1_vfp.c
+++ b/bin/varnishd/http1/cache_http1_vfp.c
@@ -43,6 +43,7 @@
 #include "cache_http1.h"
 
 #include "vct.h"
+#include "vtcp.h"
 
 /*--------------------------------------------------------------------
  * Read up to len bytes, returning pipelined data first.
@@ -75,7 +76,7 @@ v1f_read(const struct vfp_ctx *vc, struct http_conn *htc, void *d, ssize_t len)
 	if (len > 0) {
 		i = read(*htc->rfd, p, len);
 		if (i < 0) {
-			// XXX: VTCP_Assert(i); // but also: EAGAIN
+			VTCP_Assert(i);
 			VSLb(vc->wrk->vsl, SLT_FetchError,
 			    "%s", strerror(errno));
 			return (i);

From reza at naghibi.com  Thu Apr 22 15:55:06 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Thu, 22 Apr 2021 15:55:06 +0000 (UTC)
Subject: [6.0] d8f5f2fe9 Allow a > 0 in VTCP_Check()
Message-ID: <20210422155506.E0878A81E1@lists.varnish-cache.org>


commit d8f5f2fe92665134cc9a00c784902f9b70b62b7c
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 7 15:58:14 2020 +0200

    Allow a > 0 in VTCP_Check()
    
    When used to check the result of read() and write() calls, it is useful
    that a positive return value is accepted in VTCP_Check().

diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index 13b1ab9ed..a40d3d1e4 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -560,6 +560,8 @@ VTCP_Check(ssize_t a)
 {
 	if (a == 0)
 		return (1);
+	if (a > 0)
+		return (1);
 	if (errno == ECONNRESET || errno == ENOTCONN || errno == EPIPE)
 		return (1);
 	/* Accept EAGAIN (and EWOULDBLOCK in case they are not the same)

From reza at naghibi.com  Thu Apr 22 15:55:07 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Thu, 22 Apr 2021 15:55:07 +0000 (UTC)
Subject: [6.0] be1906dc1 Use VTCP_Assert() in VTCP_read()
Message-ID: <20210422155507.086EDA81E5@lists.varnish-cache.org>


commit be1906dc125dd2efce5f19bccf8d55697e59a24d
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 7 16:04:21 2020 +0200

    Use VTCP_Assert() in VTCP_read()

diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index a40d3d1e4..d1796586f 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -613,5 +613,6 @@ VTCP_read(int fd, void *ptr, size_t len, vtim_dur tmo)
 			return (-2);
 	}
 	i = read(fd, ptr, len);
+	VTCP_Assert(i);
 	return (i < 0 ? -1 : i);
 }

From reza at naghibi.com  Thu Apr 22 15:55:07 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Thu, 22 Apr 2021 15:55:07 +0000 (UTC)
Subject: [6.0] cbfb8db0a Add VTCP_Assert() in various places that call read()
 and write()
Message-ID: <20210422155507.2F891A81EB@lists.varnish-cache.org>


commit cbfb8db0a68ef2def278cf2ecaeeeef900213fe6
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed Oct 7 16:17:44 2020 +0200

    Add VTCP_Assert() in various places that call read() and write()
    
    This adds VTCP_Assert() on the result of read and write calls that deals
    with TCP sockets.
    
     Conflicts:
            bin/varnishd/proxy/cache_proxy_proto.c

diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c
index b488c1525..5259c6a81 100644
--- a/bin/varnishd/cache/cache_backend_probe.c
+++ b/bin/varnishd/cache/cache_backend_probe.c
@@ -228,6 +228,7 @@ vbp_write(struct vbp_target *vt, int *sock, const void *buf, size_t len)
 	int i;
 
 	i = write(*sock, buf, len);
+	VTCP_Assert(i);
 	if (i != len) {
 		if (i < 0) {
 			vt->err_xmit |= 1;
@@ -367,6 +368,7 @@ vbp_poke(struct vbp_target *vt)
 			    sizeof vt->resp_buf - rlen);
 		else
 			i = read(s, buf, sizeof buf);
+		VTCP_Assert(i);
 		if (i <= 0) {
 			if (i < 0)
 				bprintf(vt->resp_buf, "Read error %d (%s)",
diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c
index 12bcaa545..659bcad19 100644
--- a/bin/varnishd/http1/cache_http1_deliver.c
+++ b/bin/varnishd/http1/cache_http1_deliver.c
@@ -33,6 +33,8 @@
 #include "cache/cache_filter.h"
 #include "cache_http1.h"
 
+#include "vtcp.h"
+
 /*--------------------------------------------------------------------*/
 
 static int v_matchproto_(vdp_bytes_f)
@@ -74,7 +76,7 @@ v1d_error(struct req *req, const char *msg)
 	VSLb(req->vsl, SLT_RespReason, "Internal Server Error");
 
 	req->wrk->stats->client_resp_500++;
-	(void)write(req->sp->fd, r_500, sizeof r_500 - 1);
+	VTCP_Assert(write(req->sp->fd, r_500, sizeof r_500 - 1));
 	req->doclose = SC_TX_EOF;
 }
 
diff --git a/bin/varnishd/http1/cache_http1_pipe.c b/bin/varnishd/http1/cache_http1_pipe.c
index 224298b01..ea31f4b55 100644
--- a/bin/varnishd/http1/cache_http1_pipe.c
+++ b/bin/varnishd/http1/cache_http1_pipe.c
@@ -37,6 +37,7 @@
 #include <stdio.h>
 
 #include "cache_http1.h"
+#include "vtcp.h"
 
 #include "VSC_vbe.h"
 
@@ -49,10 +50,12 @@ rdf(int fd0, int fd1, uint64_t *pcnt)
 	char buf[BUFSIZ], *p;
 
 	i = read(fd0, buf, sizeof buf);
+	VTCP_Assert(i);
 	if (i <= 0)
 		return (1);
 	for (p = buf; i > 0; i -= j, p += j) {
 		j = write(fd1, p, i);
+		VTCP_Assert(j);
 		if (j <= 0)
 			return (1);
 		*pcnt += j;
@@ -96,6 +99,7 @@ V1P_Process(const struct req *req, int fd, struct v1p_acct *v1a)
 	if (req->htc->pipeline_b != NULL) {
 		j = write(fd,  req->htc->pipeline_b,
 		    req->htc->pipeline_e - req->htc->pipeline_b);
+		VTCP_Assert(j);
 		if (j < 0)
 			return;
 		req->htc->pipeline_b = NULL;
diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c
index 9d3683845..de1083592 100644
--- a/bin/varnishd/http2/cache_http2_session.c
+++ b/bin/varnishd/http2/cache_http2_session.c
@@ -39,6 +39,7 @@
 
 #include "vend.h"
 #include "vtim.h"
+#include "vtcp.h"
 
 static const char h2_resp_101[] =
 	"HTTP/1.1 101 Switching Protocols\r\n"
@@ -256,6 +257,7 @@ h2_ou_session(struct worker *wrk, struct h2_sess *h2,
 	}
 
 	sz = write(h2->sess->fd, h2_resp_101, strlen(h2_resp_101));
+	VTCP_Assert(sz);
 	if (sz != strlen(h2_resp_101)) {
 		VSLb(h2->vsl, SLT_Debug, "H2: Upgrade: Error writing 101"
 		    " response: %s\n", strerror(errno));
diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c
index 64935686d..09b60295b 100644
--- a/bin/varnishd/proxy/cache_proxy_proto.c
+++ b/bin/varnishd/proxy/cache_proxy_proto.c
@@ -620,7 +620,7 @@ VPX_Send_Proxy(int fd, int version, const struct sess *sp)
 	struct suckaddr *sac, *sas;
 	char ha[VTCP_ADDRBUFSIZE];
 	char pa[VTCP_PORTBUFSIZE];
-	int proto;
+	int proto, r;
 
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	assert(version == 1 || version == 2);
@@ -667,7 +667,9 @@ VPX_Send_Proxy(int fd, int version, const struct sess *sp)
 		WRONG("Wrong proxy version");
 
 	AZ(VSB_finish(vsb));
-	(void)VSB_tofile(vsb, fd);	// XXX: Error handling ?
+	r = write(fd, VSB_data(vsb), VSB_len(vsb));
+	VTCP_Assert(r);
+
 	if (!DO_DEBUG(DBG_PROTOCOL)) {
 		VSB_delete(vsb);
 		return;

From reza at naghibi.com  Fri Apr 23 15:05:07 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Fri, 23 Apr 2021 15:05:07 +0000 (UTC)
Subject: [6.0] a16b969e3 6.0.8 changelog
Message-ID: <20210423150508.01B81A8CC1@lists.varnish-cache.org>


commit a16b969e363942fc8f5589e4958170c3a6ee29fd
Author: Reza Naghibi <reza at naghibi.com>
Date:   Fri Apr 23 11:04:53 2021 -0400

    6.0.8 changelog

diff --git a/doc/changes.rst b/doc/changes.rst
index a386b2a5b..caedeefe3 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -26,6 +26,75 @@ http://varnish-cache.org/docs/trunk/whats-new/index.html and via
 individual releases. These documents are updated as part of the
 release process.
 
+================================
+Varnish Cache 6.0.8 (YYYY-MM-DD)
+================================
+
+* Fix an issue where a backend fetch can stall after a client has
+  disconnected. (3556_)
+
+* Fix an issue in `directors.shard()` where calling `reconfigure()` on
+  an empty director breaks subsequent shard directors. Also changed
+  an internal `canon_point` calculation. (3593_)
+
+* Address and fix various issues with H2 work priorities. (2796_, 3536_,
+  3537_)
+
+* Fix a panic situation in `ban_mark_completed()` when using bans. (2556_)
+
+* Fix an issue where an early backend error can trigger a valid zero
+  length response. (3560_)
+
+* Fix a panic situation when cooling a backend in `VBE_Poll()` and
+  `Lck_Delete()`. (3587_)
+
+* Complete source code migration to `python3`.
+
+* Fix an issue where `varnishd` will sleep for 1s when attempting to
+  shutdown. (3569_)
+
+* Fix a panic situation in `vbf_stp_condfetch()`. (3558_)
+
+* Fix a panic situation in `EXP_Insert()`. (2999_)
+
+* Fix a panic situation in `VRB_Free()` which can be triggered when
+  using `std.cache_req_body()`. (3433_)
+
+* Fix a panic situation in `http1_minimal_response()`. (3415_)
+
+* Fix an issue where a closed connection gets recycled for reuse. (3400_)
+
+* Fix an issue where `directors.round_robin()` can sometimes be empty
+  when a single backend is sick. (3474_)
+
+* Fix an issue where the wrong `Content-Length` is used when doing
+  gunzip on delivery. (3535_)
+
+* Fix an issue where `resp.reason` can race when used in `vcl_synth`.
+  (3546_)
+
+* Fix an issue where the `return(error)` status and reason are kept
+  when doing a backend retry. (3525_)
+
+.. _3556: https://github.com/varnishcache/varnish-cache/issues/3556
+.. _3593: https://github.com/varnishcache/varnish-cache/pull/3593
+.. _3537: https://github.com/varnishcache/varnish-cache/pull/3537
+.. _3536: https://github.com/varnishcache/varnish-cache/pull/3536
+.. _2796: https://github.com/varnishcache/varnish-cache/pull/2796
+.. _2556: https://github.com/varnishcache/varnish-cache/issues/2556
+.. _3560: https://github.com/varnishcache/varnish-cache/issues/3560
+.. _3587: https://github.com/varnishcache/varnish-cache/pull/3587
+.. _3569: https://github.com/varnishcache/varnish-cache/pull/3569
+.. _3558: https://github.com/varnishcache/varnish-cache/pull/3558
+.. _2999: https://github.com/varnishcache/varnish-cache/issues/2999
+.. _3433: https://github.com/varnishcache/varnish-cache/issues/3433
+.. _3415: https://github.com/varnishcache/varnish-cache/issues/3415
+.. _3400: https://github.com/varnishcache/varnish-cache/pull/3400
+.. _3474: https://github.com/varnishcache/varnish-cache/issues/3474
+.. _3535: https://github.com/varnishcache/varnish-cache/issues/3535
+.. _3546: https://github.com/varnishcache/varnish-cache/pull/3546
+.. _3525: https://github.com/varnishcache/varnish-cache/issues/3525
+
 ================================
 Varnish Cache 6.0.7 (2020-11-06)
 ================================

From nils.goroll at uplex.de  Mon Apr 26 09:17:07 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 26 Apr 2021 09:17:07 +0000 (UTC)
Subject: [master] 5c0c27205 Unify UPLEX Copyright statements
Message-ID: <20210426091707.51756A5C26@lists.varnish-cache.org>


commit 5c0c27205b835d0919c2a0a9231cb3866396b883
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Apr 12 18:20:46 2021 +0200

    Unify UPLEX Copyright statements
    
    IIUC, the (c) has no meaning.
    
    Ref: https://www.copyright.gov/circs/circ03.pdf

diff --git a/bin/varnishd/mgt/mgt_jail_solaris_tbl.h b/bin/varnishd/mgt/mgt_jail_solaris_tbl.h
index 7b1b0915f..062f25bb6 100644
--- a/bin/varnishd/mgt/mgt_jail_solaris_tbl.h
+++ b/bin/varnishd/mgt/mgt_jail_solaris_tbl.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2020 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2020 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Author: Nils Goroll <nils.goroll at uplex.de>
diff --git a/bin/varnishd/waiter/cache_waiter_ports.c b/bin/varnishd/waiter/cache_waiter_ports.c
index 4055e1b61..6ae65290c 100644
--- a/bin/varnishd/waiter/cache_waiter_ports.c
+++ b/bin/varnishd/waiter/cache_waiter_ports.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2006 Varnish Software AS
  * Copyright (c) 2007 OmniTI Computer Consulting, Inc.
  * Copyright (c) 2007 Theo Schlossnagle
- * Copyright (c) 2010-2016 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2010-2016 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * SPDX-License-Identifier: BSD-2-Clause
diff --git a/bin/varnishhist/varnishhist_profiles.h b/bin/varnishhist/varnishhist_profiles.h
index 42bf02251..6619f06d1 100644
--- a/bin/varnishhist/varnishhist_profiles.h
+++ b/bin/varnishhist/varnishhist_profiles.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2016 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2016 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Author: Nils Goroll <nils.goroll at uplex.de>
diff --git a/include/tbl/ban_arg_oper.h b/include/tbl/ban_arg_oper.h
index 2589f0266..d139e08a6 100644
--- a/include/tbl/ban_arg_oper.h
+++ b/include/tbl/ban_arg_oper.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2017 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2017 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Author: Nils Goroll <nils.goroll at uplex.de>
diff --git a/include/tbl/ban_oper.h b/include/tbl/ban_oper.h
index 2d244c6f5..cd4103e61 100644
--- a/include/tbl/ban_oper.h
+++ b/include/tbl/ban_oper.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2017 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2017 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Author: Nils Goroll <nils.goroll at uplex.de>
diff --git a/include/vbm_test.c b/include/vbm_test.c
index 587f27d5a..bf55a6a30 100644
--- a/include/vbm_test.c
+++ b/include/vbm_test.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2016 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2016 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Author: Nils Goroll <nils.goroll at uplex.de>
diff --git a/include/vus.h b/include/vus.h
index ce4c53494..157b65102 100644
--- a/include/vus.h
+++ b/include/vus.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2018 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2018 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Author: Geoffrey Simmons <geoffrey.simmons at uplex.de>
diff --git a/lib/libvarnish/vus.c b/lib/libvarnish/vus.c
index d36bb8784..ba803b08d 100644
--- a/lib/libvarnish/vus.c
+++ b/lib/libvarnish/vus.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2018 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2018 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Author: Geoffrey Simmons <geoffrey.simmons at uplex.de>
diff --git a/vmod/vmod_blob.c b/vmod/vmod_blob.c
index 3ebb97edf..8274ea6f3 100644
--- a/vmod/vmod_blob.c
+++ b/vmod/vmod_blob.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015-2017 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2015-2017 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Authors: Nils Goroll <nils.goroll at uplex.de>
diff --git a/vmod/vmod_blob.h b/vmod/vmod_blob.h
index 3c1048b48..90872b61a 100644
--- a/vmod/vmod_blob.h
+++ b/vmod/vmod_blob.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015-2016 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2015-2016 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Authors: Nils Goroll <nils.goroll at uplex.de>
diff --git a/vmod/vmod_blob_base64.c b/vmod/vmod_blob_base64.c
index 876f6e309..74a895754 100644
--- a/vmod/vmod_blob_base64.c
+++ b/vmod/vmod_blob_base64.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015-2016 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2015-2016 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Authors: Nils Goroll <nils.goroll at uplex.de>
diff --git a/vmod/vmod_blob_hex.c b/vmod/vmod_blob_hex.c
index fce4f7839..539ab87a7 100644
--- a/vmod/vmod_blob_hex.c
+++ b/vmod/vmod_blob_hex.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2016 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2016 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Authors: Nils Goroll <nils.goroll at uplex.de>
diff --git a/vmod/vmod_blob_id.c b/vmod/vmod_blob_id.c
index cf5b6ee72..ad6b64607 100644
--- a/vmod/vmod_blob_id.c
+++ b/vmod/vmod_blob_id.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015-2016 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2015-2016 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Authors: Nils Goroll <nils.goroll at uplex.de>
diff --git a/vmod/vmod_blob_url.c b/vmod/vmod_blob_url.c
index aff0669e9..9ce272ed8 100644
--- a/vmod/vmod_blob_url.c
+++ b/vmod/vmod_blob_url.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2015-2016 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2015-2016 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Authors: Nils Goroll <nils.goroll at uplex.de>
diff --git a/vmod/vmod_directors_shard.c b/vmod/vmod_directors_shard.c
index 8e544ce4b..19f48339b 100644
--- a/vmod/vmod_directors_shard.c
+++ b/vmod/vmod_directors_shard.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2009-2018 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2009-2018 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Authors: Julian Wiesener <jw at uplex.de>
diff --git a/vmod/vmod_directors_shard_cfg.c b/vmod/vmod_directors_shard_cfg.c
index 53088a404..6ba55b692 100644
--- a/vmod/vmod_directors_shard_cfg.c
+++ b/vmod/vmod_directors_shard_cfg.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2009-2016 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2009-2016 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Authors: Nils Goroll <nils.goroll at uplex.de>
diff --git a/vmod/vmod_directors_shard_cfg.h b/vmod/vmod_directors_shard_cfg.h
index d2f66a717..87a6f69e4 100644
--- a/vmod/vmod_directors_shard_cfg.h
+++ b/vmod/vmod_directors_shard_cfg.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2016 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2016 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Author: Nils Goroll <nils.goroll at uplex.de>
diff --git a/vmod/vmod_directors_shard_dir.c b/vmod/vmod_directors_shard_dir.c
index 3393a7faf..9b82722d1 100644
--- a/vmod/vmod_directors_shard_dir.c
+++ b/vmod/vmod_directors_shard_dir.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2009-2016 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2009-2016 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Authors: Nils Goroll <nils.goroll at uplex.de>
diff --git a/vmod/vmod_directors_shard_dir.h b/vmod/vmod_directors_shard_dir.h
index fb49bcdfc..47a66b92a 100644
--- a/vmod/vmod_directors_shard_dir.h
+++ b/vmod/vmod_directors_shard_dir.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2009-2016 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2009-2016 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Authors: Julian Wiesener <jw at uplex.de>
diff --git a/vmod/vmod_unix.c b/vmod/vmod_unix.c
index ccd0f1a85..1edfca9d5 100644
--- a/vmod/vmod_unix.c
+++ b/vmod/vmod_unix.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2018 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2018 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Authors: Geoffrey Simmons <geoffrey.simmons at uplex.de>
diff --git a/vmod/vmod_unix_cred_compat.h b/vmod/vmod_unix_cred_compat.h
index 1b741ddde..9ac8438f8 100644
--- a/vmod/vmod_unix_cred_compat.h
+++ b/vmod/vmod_unix_cred_compat.h
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2018 UPLEX - Nils Goroll Systemoptimierung
+ * Copyright 2018 UPLEX - Nils Goroll Systemoptimierung
  * All rights reserved.
  *
  * Authors: Geoffrey Simmons <geoffrey.simmons at uplex.de>

From nils.goroll at uplex.de  Mon Apr 26 09:17:07 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 26 Apr 2021 09:17:07 +0000 (UTC)
Subject: [master] d5d70a657 varnishncsa: Support balanced pairs of braces
 within formats
Message-ID: <20210426091707.69454A5C29@lists.varnish-cache.org>


commit d5d70a6578c8645c9bef703340909869dac94c5e
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Apr 26 10:27:23 2021 +0200

    varnishncsa: Support balanced pairs of braces within formats
    
    We now accept braces as long as they come in balanced pairs. As for now,
    might only be useful with %{X}t.
    
    Also extend testing of %{X}t and %{X}T

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 21d95e952..6c5ea6d23 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -631,6 +631,7 @@ parse_format(const char *format)
 	const char *p, *q;
 	struct vsb *vsb;
 	char buf[256];
+	int b;
 
 	if (format == NULL)
 		format = FORMAT;
@@ -712,9 +713,16 @@ parse_format(const char *format)
 		case '{':
 			p++;
 			q = p;
-			while (*q && *q != '}')
+			b = 1;
+			while (*q) {
+				if (*q == '{')
+					b++;
+				else if (*q == '}')
+					if (--b == 0)
+						break;
 				q++;
-			if (!*q)
+			}
+			if (b > 0)
 				VUT_Error(vut, 1, "Unmatched bracket at: %s",
 				    p - 2);
 			assert((unsigned)(q - p) < sizeof buf - 1);
diff --git a/bin/varnishtest/tests/u00003.vtc b/bin/varnishtest/tests/u00003.vtc
index ae81e664c..ee598b1e6 100644
--- a/bin/varnishtest/tests/u00003.vtc
+++ b/bin/varnishtest/tests/u00003.vtc
@@ -149,6 +149,11 @@ req (\d+) rxreq \5 - - - -$} \
 %{VSL:Begin[2]}x %{VSL:Timestamp:Resp}x \
 %{VSL:Timestamp:Resp[2]}x %{VSL:Timestamp:foo}x %{VSL:ReqURL[2]}x"}
 
+# times
+shell -match {^\{\d{4}-\d{2}-\d{2}\}T\{\d{2}:\d{2}:\d{2}\} \{\{\d{4}-\d{2}-\d{2}\T\d{2}:\d{2}:\d{2}\}\} \d+ \d+ \d+} \
+	{varnishncsa -n ${v1_name} -d -F "%{{%F}T{%T}}t %{{{%FT%T}}}t \
+%{s}T %{ms}T %{us}T"}
+
 process p1 -stop -screen-dump
 process p1 -expect-text 1 0 {/1?foo=bar HTTP/1.1" 200 100 "-" "-"}
 process p1 -expect-text 1 0 { - user [}

From nils.goroll at uplex.de  Mon Apr 26 10:56:05 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 26 Apr 2021 10:56:05 +0000 (UTC)
Subject: [master] 3df4eae44 varnishncsa: Refactor %{x}T processing
Message-ID: <20210426105605.6C7E5A8CA3@lists.varnish-cache.org>


commit 3df4eae44be86fd16c4a5e9d87015e00be6b68b9
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Apr 26 12:50:56 2021 +0200

    varnishncsa: Refactor %{x}T processing
    
    As we can safely regard (struct format).time_type is a private contract
    between addf_time() and format_time(), use it to specify the exact
    format type, which moves strcmp() parsing from execution to setup.

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 6c5ea6d23..ba01a98d5 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -294,23 +294,23 @@ format_time(const struct format *format)
 		(void)localtime_r(&t, &tm);
 		AN(strftime(buf, sizeof buf, format->time_fmt, &tm));
 		AZ(VSB_cat(CTX.vsb, buf));
+		return (1);
+	case 's':
+		l = (long)(t_end - t_start);
 		break;
-	case 'T':
-		AN(format->time_fmt);
-		if (!strcmp(format->time_fmt, "s")) /* same as %T */
-			l = (long)(t_end - t_start);
-		else if (!strcmp(format->time_fmt, "ms"))
-			l = (long)((t_end - t_start) * 1e3);
-		else if (!strcmp(format->time_fmt, "us")) /* same as %D */
-			l = (long)((t_end - t_start) * 1e6);
-		else
-			WRONG("Unreachable branch");
-		AZ(VSB_printf(CTX.vsb, "%ld", l));
+	case 'm':
+		l = (long)((t_end - t_start) * 1e3);
+		break;
+	case 'u':
+		l = (long)((t_end - t_start) * 1e6);
 		break;
 	default:
 		WRONG("Time format specifier");
 	}
 
+	AN(format->time_fmt);
+	AZ(VSB_printf(CTX.vsb, format->time_fmt, l));
+
 	return (1);
 }
 
@@ -449,9 +449,18 @@ addf_time(char type, const char *fmt)
 	f->time_fmt = strdup(fmt);
 	AN(f->time_fmt);
 
-	if (f->time_type == 'T' && strcmp(f->time_fmt, "s") &&
-	    strcmp(f->time_fmt, "ms") && strcmp(f->time_fmt, "us"))
-		VUT_Error(vut, 1, "Unknown specifier: %%{%s}T", f->time_fmt);
+	if (f->time_type == 'T') {
+		if (!strcmp(f->time_fmt, "s"))
+			f->time_type = 's';
+		else if (!strcmp(f->time_fmt, "ms"))
+			f->time_type = 'm';
+		else if (!strcmp(f->time_fmt, "us"))
+			f->time_type = 'u';
+		else
+			VUT_Error(vut, 1, "Unknown specifier: %%{%s}T",
+			    f->time_fmt);
+		REPLACE(f->time_fmt, "%ld");
+	}
 
 	VTAILQ_INSERT_TAIL(&CTX.format, f, list);
 }

From nils.goroll at uplex.de  Mon Apr 26 12:09:05 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 26 Apr 2021 12:09:05 +0000 (UTC)
Subject: [master] aee154f68 VUT_Init(): Allocate vut object only when not
 exiting
Message-ID: <20210426120905.64F2FAAF4A@lists.varnish-cache.org>


commit aee154f6846b5a2fbce9c078797e19f3865c1f93
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Apr 26 14:07:24 2021 +0200

    VUT_Init(): Allocate vut object only when not exiting
    
    Closes #3597

diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c
index 3c458f031..82077270a 100644
--- a/lib/libvarnishapi/vut.c
+++ b/lib/libvarnishapi/vut.c
@@ -246,9 +246,6 @@ VUT_Init(const char *progname, int argc, char * const *argv,
 	VSIG_Arm_term();
 	VSIG_Arm_usr1();
 
-	ALLOC_OBJ(vut, VUT_MAGIC);
-	AN(vut);
-
 	if (argc == 2 && !strcmp(argv[1], "--synopsis"))
 		exit(vut_synopsis(voc));
 	if (argc == 2 && !strcmp(argv[1], "--options"))
@@ -258,6 +255,8 @@ VUT_Init(const char *progname, int argc, char * const *argv,
 		exit(0);
 	}
 
+	ALLOC_OBJ(vut, VUT_MAGIC);
+	AN(vut);
 	vut->progname = progname;
 	vut->g_arg = VSL_g_vxid;
 	vut->k_arg = -1;

From nils.goroll at uplex.de  Tue Apr 27 07:30:09 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 27 Apr 2021 07:30:09 +0000 (UTC)
Subject: [master] f84d3edbc varnishncsa: Add %{sec, msec, usec, msec_frac,
 usec_frac}t specification
Message-ID: <20210427073009.A7486A2EE2@lists.varnish-cache.org>


commit f84d3edbc8476cf847976b6bfe2e7b6bcaa56f25
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Apr 26 13:35:59 2021 +0200

    varnishncsa: Add %{sec,msec,usec,msec_frac,usec_frac}t specification
    
    ... following the example of Apache HTTP Server, see
    http://httpd.apache.org/docs/current/mod/mod_log_config.html

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index ba01a98d5..32a6943dc 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -257,7 +257,7 @@ format_fragment(const struct format *format)
 static int v_matchproto_(format_f)
 format_time(const struct format *format)
 {
-	double t_start, t_end;
+	double t_start, t_end, d;
 	char *p;
 	char buf[64];
 	time_t t;
@@ -295,6 +295,21 @@ format_time(const struct format *format)
 		AN(strftime(buf, sizeof buf, format->time_fmt, &tm));
 		AZ(VSB_cat(CTX.vsb, buf));
 		return (1);
+	case '3':
+		l = (long)(modf(t_start, &d) * 1e3);
+		break;
+	case '6':
+		l = (long)(modf(t_start, &d) * 1e6);
+		break;
+	case 'S':
+		l = (long)t_start;
+		break;
+	case 'M':
+		l = (long)(t_start * 1e3);
+		break;
+	case 'U':
+		l = (long)(t_start * 1e6);
+		break;
 	case 's':
 		l = (long)(t_end - t_start);
 		break;
@@ -450,16 +465,39 @@ addf_time(char type, const char *fmt)
 	AN(f->time_fmt);
 
 	if (f->time_type == 'T') {
-		if (!strcmp(f->time_fmt, "s"))
+		if (!strcmp(fmt, "s"))
 			f->time_type = 's';
-		else if (!strcmp(f->time_fmt, "ms"))
+		else if (!strcmp(fmt, "ms"))
 			f->time_type = 'm';
-		else if (!strcmp(f->time_fmt, "us"))
+		else if (!strcmp(fmt, "us"))
 			f->time_type = 'u';
 		else
 			VUT_Error(vut, 1, "Unknown specifier: %%{%s}T",
-			    f->time_fmt);
+			    fmt);
 		REPLACE(f->time_fmt, "%ld");
+	} else if (f->time_type == 't') {
+		if (!strcmp(fmt, "sec")) {
+			f->time_type = 'S';
+			REPLACE(f->time_fmt, "%ld");
+		} else if (!strncmp(fmt, "msec", 4)) {
+			fmt += 4;
+			if (!strcmp(fmt, "_frac")) {
+				f->time_type = '3';
+				REPLACE(f->time_fmt, "%03ld");
+			} else if (*fmt == '\0') {
+				f->time_type = 'M';
+				REPLACE(f->time_fmt, "%ld");
+			}
+		} else if (!strncmp(fmt, "usec", 4)) {
+			fmt += 4;
+			if (!strcmp(fmt, "_frac")) {
+				f->time_type = '6';
+				REPLACE(f->time_fmt, "%06ld");
+			} else if (*fmt == '\0') {
+				f->time_type = 'U';
+				REPLACE(f->time_fmt, "%ld");
+			}
+		}
 	}
 
 	VTAILQ_INSERT_TAIL(&CTX.format, f, list);
diff --git a/bin/varnishtest/tests/u00003.vtc b/bin/varnishtest/tests/u00003.vtc
index ee598b1e6..3cfa187de 100644
--- a/bin/varnishtest/tests/u00003.vtc
+++ b/bin/varnishtest/tests/u00003.vtc
@@ -150,9 +150,10 @@ req (\d+) rxreq \5 - - - -$} \
 %{VSL:Timestamp:Resp[2]}x %{VSL:Timestamp:foo}x %{VSL:ReqURL[2]}x"}
 
 # times
-shell -match {^\{\d{4}-\d{2}-\d{2}\}T\{\d{2}:\d{2}:\d{2}\} \{\{\d{4}-\d{2}-\d{2}\T\d{2}:\d{2}:\d{2}\}\} \d+ \d+ \d+} \
+shell -match {^\{\d{4}-\d{2}-\d{2}\}T\{\d{2}:\d{2}:\d{2}\} \{\{\d{4}-\d{2}-\d{2}\T\d{2}:\d{2}:\d{2}\}\} \d+ \d+ \d+ \d{10,} \d{13,} \d{16,} \d{3} \d{6} usecx msecy} \
 	{varnishncsa -n ${v1_name} -d -F "%{{%F}T{%T}}t %{{{%FT%T}}}t \
-%{s}T %{ms}T %{us}T"}
+%{s}T %{ms}T %{us}T %{sec}t %{msec}t %{usec}t %{msec_frac}t %{usec_frac}t \
+%{usecx}t %{msecy}t"}
 
 process p1 -stop -screen-dump
 process p1 -expect-text 1 0 {/1?foo=bar HTTP/1.1" 200 100 "-" "-"}
diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index 489690866..ceb115b5e 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -138,7 +138,17 @@ Supported formatters are:
 %{X}t
   In client mode, time when the request was received, in the format
   specified by X.  In backend mode, time when the request was sent.
-  The time specification format is the same as for strftime(3).
+  The time specification format is the same as for strftime(3) with
+  these extensions:
+
+  * ``%{sec}``: number of seconds since the Epoch
+  * ``%{msec}``: number of milliseconds since the Epoch
+  * ``%{usec}``: number of milliseconds since the Epoch
+  * ``%{msec_frac}``: millisecond fraction
+  * ``%{usec_frac}``: microsecond fraction
+
+  The extensions can not be combined with each other or strftime(3) in
+  the same specification. Use multiple ``%{X}t`` specifications instead.
 
 %T
   In client mode, time taken to serve the request, in seconds.  In

From nils.goroll at uplex.de  Tue Apr 27 07:39:05 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 27 Apr 2021 07:39:05 +0000 (UTC)
Subject: [master] cbc7c8a32 varnishncsa: Minor polish after merge
Message-ID: <20210427073905.9D636A350E@lists.varnish-cache.org>


commit cbc7c8a32f5f397a8fcb4f685ebdb8c44e25c759
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Apr 27 09:36:06 2021 +0200

    varnishncsa: Minor polish after merge
    
    Why is it that a git push triggers a fresh look at the code?
    
    Ref #3602

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 32a6943dc..75f788b00 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -287,9 +287,10 @@ format_time(const struct format *format)
 	} else
 		t_end = t_start;
 
+	AN(format->time_fmt);
+
 	switch (format->time_type) {
 	case 't':
-		AN(format->time_fmt);
 		t = (long)floor(t_start);
 		(void)localtime_r(&t, &tm);
 		AN(strftime(buf, sizeof buf, format->time_fmt, &tm));
@@ -323,7 +324,6 @@ format_time(const struct format *format)
 		WRONG("Time format specifier");
 	}
 
-	AN(format->time_fmt);
 	AZ(VSB_printf(CTX.vsb, format->time_fmt, l));
 
 	return (1);
@@ -462,7 +462,6 @@ addf_time(char type, const char *fmt)
 	f->func = format_time;
 	f->time_type = type;
 	f->time_fmt = strdup(fmt);
-	AN(f->time_fmt);
 
 	if (f->time_type == 'T') {
 		if (!strcmp(fmt, "s"))
@@ -500,6 +499,7 @@ addf_time(char type, const char *fmt)
 		}
 	}
 
+	AN(f->time_fmt);
 	VTAILQ_INSERT_TAIL(&CTX.format, f, list);
 }
 

From reza at naghibi.com  Wed Apr 28 18:19:07 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 28 Apr 2021 18:19:07 +0000 (UTC)
Subject: [6.0] 83d4ae010 VTCP_Check() to accept ETIMEDOUT on all platforms
Message-ID: <20210428181907.D2FC160E8E@lists.varnish-cache.org>


commit 83d4ae0106928f971a58fb1c32be4c2ad3b1d2e4
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Feb 22 19:50:33 2021 +0100

    VTCP_Check() to accept ETIMEDOUT on all platforms
    
    it was already accepted on Solaris and NetBSD, now we have seen it on
    Linux and I think it does not make sense to keep the exception for
    Apple.
    
    Fixes #3532 (hopefully)
    
     Conflicts:
            lib/libvarnish/vtcp.c

diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index d1796586f..11e6c05ae 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -572,6 +572,10 @@ VTCP_Check(ssize_t a)
 	 * described in the socket(7) manpage.) */
 	if (errno == EAGAIN || errno == EWOULDBLOCK)
 		return (1);
+	/* tcp(7): The other end didn't acknowledge retransmitted data after
+	 * some time. */
+	if (errno == ETIMEDOUT)
+		return (1);
 #if (defined (__SVR4) && defined (__sun)) || defined (__NetBSD__)
 	/*
 	 * Solaris returns EINVAL if the other end unexpectedly reset the

From reza at naghibi.com  Wed Apr 28 18:19:07 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 28 Apr 2021 18:19:07 +0000 (UTC)
Subject: [6.0] a11dce361 Collapse the MacOS case with Solaris & NetBSD in
 VTCP_Check()
Message-ID: <20210428181907.E6B3B60E92@lists.varnish-cache.org>


commit a11dce3615913d1fbeed6a0f4d28647d54f1868c
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Feb 22 19:53:27 2021 +0100

    Collapse the MacOS case with Solaris & NetBSD in VTCP_Check()
    
     Conflicts:
            lib/libvarnish/vtcp.c

diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index 11e6c05ae..02bbe7320 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -576,18 +576,20 @@ VTCP_Check(ssize_t a)
 	 * some time. */
 	if (errno == ETIMEDOUT)
 		return (1);
-#if (defined (__SVR4) && defined (__sun)) || defined (__NetBSD__)
-	/*
-	 * Solaris returns EINVAL if the other end unexpectedly reset the
-	 * connection.
-	 * This is a bug in Solaris and documented behaviour on NetBSD.
-	 */
-	if (errno == EINVAL || errno == ETIMEDOUT)
+#if (defined (__SVR4) && defined (__sun))
+	if (errno == ECONNREFUSED)	// in r02702.vtc
 		return (1);
-#elif defined (__APPLE__)
+	if (errno == EPROTO)
+		return (1);
+#endif
+#if (defined (__SVR4) && defined (__sun)) ||		\
+    defined (__NetBSD__) ||				\
+    defined (__APPLE__)
 	/*
-	 * MacOS returns EINVAL if the other end unexpectedly reset
+	 * Solaris and MacOS returns EINVAL if the other end unexpectedly reset
 	 * the connection.
+	 *
+	 * On NetBSD it is documented behaviour.
 	 */
 	if (errno == EINVAL)
 		return (1);

From reza at naghibi.com  Wed Apr 28 18:19:08 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 28 Apr 2021 18:19:08 +0000 (UTC)
Subject: [6.0] e89baa60f Accept all ENET* ECONN* EHOST* errnos
Message-ID: <20210428181908.0C95F60E95@lists.varnish-cache.org>


commit e89baa60fa77df5e0608bb105e3f1de3291012be
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Mar 2 13:59:50 2021 +0100

    Accept all ENET* ECONN* EHOST* errnos
    
    ... documented on Linux as POSIX.1
    
    The exception here is ECONNREFUSED which so far we only tolerate for
    Solaris and which seems to make sense for connect() only.
    
    To be discussed in #3539

diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index 02bbe7320..89450c52e 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -576,6 +576,13 @@ VTCP_Check(ssize_t a)
 	 * some time. */
 	if (errno == ETIMEDOUT)
 		return (1);
+	/* #3539 various errnos documented on linux as POSIX.1 */
+	if (errno == ENETDOWN || errno == ENETUNREACH || errno == ENETRESET ||
+	    errno == ECONNABORTED || /* ECONNRESET see above */
+	    errno == EHOSTUNREACH || errno == EHOSTDOWN) {
+		return (1);
+	}
+
 #if (defined (__SVR4) && defined (__sun))
 	if (errno == ECONNREFUSED)	// in r02702.vtc
 		return (1);

From reza at naghibi.com  Wed Apr 28 18:19:08 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 28 Apr 2021 18:19:08 +0000 (UTC)
Subject: [6.0] 0777683d7 vdef: Centralize __has_feature(...) fallback
Message-ID: <20210428181908.2A11260E9A@lists.varnish-cache.org>


commit 0777683d7f6ad648ad81119540365b5b6540650c
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Apr 15 19:07:01 2021 +0200

    vdef: Centralize __has_feature(...) fallback
    
     Conflicts:
            lib/libvarnish/binary_heap.c

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 2bf69a0e5..c87c891a6 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -43,10 +43,6 @@
 #include "vav.h"
 #include "vcli_serve.h"
 
-#if !defined(__has_feature)
-#define __has_feature(x)	0
-#endif
-
 struct plist {
 	unsigned			magic;
 #define PLIST_MAGIC			0xbfc3ea16
diff --git a/include/vdef.h b/include/vdef.h
index f76407f80..60d833c33 100644
--- a/include/vdef.h
+++ b/include/vdef.h
@@ -64,6 +64,10 @@
 		*(fdp) = -1;			\
 	} while (0)
 
+#if !defined(__has_feature)
+#  define __has_feature(x)	0
+#endif
+
 #ifndef __GNUC_PREREQ__
 # if defined __GNUC__ && defined __GNUC_MINOR__
 #  define __GNUC_PREREQ__(maj, min) \

From reza at naghibi.com  Wed Apr 28 18:19:08 2021
From: reza at naghibi.com (Reza Naghibi)
Date: Wed, 28 Apr 2021 18:19:08 +0000 (UTC)
Subject: [6.0] 727b4dedd vtcp: Ignore EINTR with sanitizers
Message-ID: <20210428181908.440A460E9E@lists.varnish-cache.org>


commit 727b4deddf92c36823ab77545d0f8b430f397226
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Apr 15 19:07:34 2021 +0200

    vtcp: Ignore EINTR with sanitizers
    
    Technically it can also happen with a debugger attached to a process
    despite SA_RESTART.

diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index 89450c52e..b55d91341 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -600,6 +600,10 @@ VTCP_Check(ssize_t a)
 	 */
 	if (errno == EINVAL)
 		return (1);
+#endif
+#if (defined(__SANITIZER) || __has_feature(address_sanitizer))
+	if (errno == EINTR)
+		return (1);
 #endif
 	return (0);
 }

From guillaume at varnish-software.com  Thu Apr 29 04:48:07 2021
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Thu, 29 Apr 2021 04:48:07 +0000 (UTC)
Subject: [master] 89b393e76 print usage for the right reason
Message-ID: <20210429044807.59F9DA1F47@lists.varnish-cache.org>


commit 89b393e76cd488d08cb22d68ca33661cdf10b475
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Wed Apr 28 21:42:54 2021 -0700

    print usage for the right reason

diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 9f4c23623..9ad6d8a7c 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -79,7 +79,7 @@ static struct vpf_fh *pfh2 = NULL;
 static struct vfil_path *vcl_path = NULL;
 static VTAILQ_HEAD(,f_arg) f_args = VTAILQ_HEAD_INITIALIZER(f_args);
 
-static const char opt_spec[] = "a:b:Cdf:Fh:i:I:j:l:M:n:P:p:r:S:s:T:t:VW:x:";
+static const char opt_spec[] = "?a:b:Cdf:Fh:i:I:j:l:M:n:P:p:r:S:s:T:t:VW:x:";
 
 int optreset;	// Some has it, some doesn't.  Cheaper than auto*
 

From phk at FreeBSD.org  Thu Apr 29 06:39:04 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 29 Apr 2021 06:39:04 +0000 (UTC)
Subject: [master] cae2cf618 Fix u00003 by using intmax_t instead of long,
 which is too short in 32bit.
Message-ID: <20210429063904.DDB6DA54C3@lists.varnish-cache.org>


commit cae2cf61806927a1da9621feb44c0d771986e89a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Apr 29 06:37:22 2021 +0000

    Fix u00003 by using intmax_t instead of long, which is too short in 32bit.

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 75f788b00..afb455154 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -261,7 +261,7 @@ format_time(const struct format *format)
 	char *p;
 	char buf[64];
 	time_t t;
-	long l;
+	intmax_t l;
 	struct tm tm;
 
 	CHECK_OBJ_NOTNULL(format, FORMAT_MAGIC);
@@ -291,39 +291,40 @@ format_time(const struct format *format)
 
 	switch (format->time_type) {
 	case 't':
-		t = (long)floor(t_start);
+		t = (intmax_t)floor(t_start);
 		(void)localtime_r(&t, &tm);
 		AN(strftime(buf, sizeof buf, format->time_fmt, &tm));
 		AZ(VSB_cat(CTX.vsb, buf));
 		return (1);
 	case '3':
-		l = (long)(modf(t_start, &d) * 1e3);
+		l = (intmax_t)(modf(t_start, &d) * 1e3);
 		break;
 	case '6':
-		l = (long)(modf(t_start, &d) * 1e6);
+		l = (intmax_t)(modf(t_start, &d) * 1e6);
 		break;
 	case 'S':
-		l = (long)t_start;
+		l = (intmax_t)t_start;
 		break;
 	case 'M':
-		l = (long)(t_start * 1e3);
+		l = (intmax_t)(t_start * 1e3);
 		break;
 	case 'U':
-		l = (long)(t_start * 1e6);
+		l = (intmax_t)(t_start * 1e6);
 		break;
 	case 's':
-		l = (long)(t_end - t_start);
+		l = (intmax_t)(t_end - t_start);
 		break;
 	case 'm':
-		l = (long)((t_end - t_start) * 1e3);
+		l = (intmax_t)((t_end - t_start) * 1e3);
 		break;
 	case 'u':
-		l = (long)((t_end - t_start) * 1e6);
+		l = (intmax_t)((t_end - t_start) * 1e6);
 		break;
 	default:
 		WRONG("Time format specifier");
 	}
 
+	assert(fmtcheck(format->time_fmt, "%jd") == format->time_fmt);
 	AZ(VSB_printf(CTX.vsb, format->time_fmt, l));
 
 	return (1);
@@ -473,28 +474,28 @@ addf_time(char type, const char *fmt)
 		else
 			VUT_Error(vut, 1, "Unknown specifier: %%{%s}T",
 			    fmt);
-		REPLACE(f->time_fmt, "%ld");
+		REPLACE(f->time_fmt, "%jd");
 	} else if (f->time_type == 't') {
 		if (!strcmp(fmt, "sec")) {
 			f->time_type = 'S';
-			REPLACE(f->time_fmt, "%ld");
+			REPLACE(f->time_fmt, "%jd");
 		} else if (!strncmp(fmt, "msec", 4)) {
 			fmt += 4;
 			if (!strcmp(fmt, "_frac")) {
 				f->time_type = '3';
-				REPLACE(f->time_fmt, "%03ld");
+				REPLACE(f->time_fmt, "%03jd");
 			} else if (*fmt == '\0') {
 				f->time_type = 'M';
-				REPLACE(f->time_fmt, "%ld");
+				REPLACE(f->time_fmt, "%jd");
 			}
 		} else if (!strncmp(fmt, "usec", 4)) {
 			fmt += 4;
 			if (!strcmp(fmt, "_frac")) {
 				f->time_type = '6';
-				REPLACE(f->time_fmt, "%06ld");
+				REPLACE(f->time_fmt, "%06jd");
 			} else if (*fmt == '\0') {
 				f->time_type = 'U';
-				REPLACE(f->time_fmt, "%ld");
+				REPLACE(f->time_fmt, "%jd");
 			}
 		}
 	}

From phk at FreeBSD.org  Thu Apr 29 06:45:06 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 29 Apr 2021 06:45:06 +0000 (UTC)
Subject: [master] 185b70353 Only assert fmtcheck(3) on sane operating systems.
Message-ID: <20210429064506.85755A594A@lists.varnish-cache.org>


commit 185b703532fa4521f51cf938df1de5378066f9e9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Apr 29 06:44:17 2021 +0000

    Only assert fmtcheck(3) on sane operating systems.

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index afb455154..a9b5fd185 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -324,7 +324,9 @@ format_time(const struct format *format)
 		WRONG("Time format specifier");
 	}
 
+#ifdef __FreeBSD__
 	assert(fmtcheck(format->time_fmt, "%jd") == format->time_fmt);
+#endif
 	AZ(VSB_printf(CTX.vsb, format->time_fmt, l));
 
 	return (1);

From phk at FreeBSD.org  Thu Apr 29 20:56:08 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 29 Apr 2021 20:56:08 +0000 (UTC)
Subject: [master] 6362b7a1e Typo.
Message-ID: <20210429205608.79F6194AA1@lists.varnish-cache.org>


commit 6362b7a1ec64e6bafbee919a123a4dc689ac8f83
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Apr 29 20:54:40 2021 +0000

    Typo.
    
    Spotted by @cperciva

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index f8088b90e..5be41cf23 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -330,7 +330,7 @@ hit-for-pass.
 
 If no ``-s`` options are given, the default is::
 
-	-s malloc=100m
+	-s malloc,100m
 
 If no ``Transient`` storage is defined, the default is an unbound
 ``malloc`` storage as if defined as::

From phk at FreeBSD.org  Fri Apr 30 07:52:06 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 30 Apr 2021 07:52:06 +0000 (UTC)
Subject: [master] 94692c47a Give screen-width a second to change when
 expecting things.
Message-ID: <20210430075206.C2807ABDCE@lists.varnish-cache.org>


commit 94692c47a6fc3c181b4f8bd2aa1fe9067283bd81
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Apr 30 07:38:01 2021 +0000

    Give screen-width a second to change when expecting things.

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index 3ef102ffa..bc243894b 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -301,7 +301,13 @@ term_expect_text(struct process *pp,
 	char *t;
 
 	y = strtoul(lin, NULL, 0);
+	if (y < 0 || y > pp->nlin)
+		vtc_fatal(pp->vl, "YYY %d nlin %d", y, pp->nlin);
 	x = strtoul(col, NULL, 0);
+	for(l = 0; l < 10 && x > pp->ncol; l++)	// wait for screen change
+		usleep(100000);
+	if (x < 0 || x > pp->ncol)
+		vtc_fatal(pp->vl, "XXX %d ncol %d", x, pp->ncol);
 	l = strlen(pat);
 	AZ(pthread_mutex_lock(&pp->mtx));
 	while (!term_match_text(pp, &x, &y, pat)) {
@@ -323,12 +329,18 @@ term_expect_text(struct process *pp,
 static void
 term_expect_cursor(const struct process *pp, const char *lin, const char *col)
 {
-	int x, y;
+	int x, y, l;
 	const teken_pos_t *pos;
 
 	pos = teken_get_cursor(pp->tek);
 	y = strtoul(lin, NULL, 0);
+	if (y < 0 || y > pp->nlin)
+		vtc_fatal(pp->vl, "YYY %d nlin %d", y, pp->nlin);
 	x = strtoul(col, NULL, 0);
+	for(l = 0; l < 10 && x > pp->ncol; l++)	// wait for screen change
+		usleep(100000);
+	if (x < 0 || x > pp->ncol)
+		vtc_fatal(pp->vl, "XXX %d ncol %d", x, pp->ncol);
 	if (y != 0 && (y-1) != pos->tp_row)
 		vtc_fatal(pp->vl, "Cursor on line %d (expected %d)",
 		    pos->tp_row + 1, y);

From phk at FreeBSD.org  Fri Apr 30 07:52:06 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 30 Apr 2021 07:52:06 +0000 (UTC)
Subject: [master] 7ae60b722 Only do range processing when data present.
Message-ID: <20210430075206.F0170ABDD1@lists.varnish-cache.org>


commit 7ae60b722be2f810c1a701c1586605e6f9be1409
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Apr 30 07:51:15 2021 +0000

    Only do range processing when data present.
    
    Spotted by: ubsan

diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c
index bbc40cb46..be2340a19 100644
--- a/bin/varnishd/cache/cache_range.c
+++ b/bin/varnishd/cache/cache_range.c
@@ -68,7 +68,7 @@ vrg_range_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv,
     const void *ptr, ssize_t len)
 {
 	int retval = 0;
-	ssize_t l;
+	ssize_t l = 0;
 	const char *p = ptr;
 	struct vrg_priv *vrg_priv;
 
@@ -76,20 +76,23 @@ vrg_range_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv,
 	AN(priv);
 	CAST_OBJ_NOTNULL(vrg_priv, *priv, VRG_PRIV_MAGIC);
 
-	l = vrg_priv->range_low - vrg_priv->range_off;
-	if (l > 0) {
+	if (ptr != NULL) {
+		l = vrg_priv->range_low - vrg_priv->range_off;
+		if (l > 0) {
+			if (l > len)
+				l = len;
+			vrg_priv->range_off += l;
+			p += l;
+			len -= l;
+		}
+		l = vrg_priv->range_high - vrg_priv->range_off;
 		if (l > len)
 			l = len;
-		vrg_priv->range_off += l;
-		p += l;
-		len -= l;
+		vrg_priv->range_off += len;
+		if (vrg_priv->range_off >= vrg_priv->range_high)
+			act = VDP_END;
 	}
-	l = vrg_priv->range_high - vrg_priv->range_off;
-	if (l > len)
-		l = len;
-	vrg_priv->range_off += len;
-	if (vrg_priv->range_off >= vrg_priv->range_high)
-		act = VDP_END;
+
 	if (l > 0)
 		retval = VDP_bytes(vdx, act, p, l);
 	else if (l == 0 && act > VDP_NULL)

From dridi.boukelmoune at gmail.com  Fri Apr 30 08:45:06 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 30 Apr 2021 08:45:06 +0000 (UTC)
Subject: [master] 3fe065cd0 vtc_process: Mitigate screen refresh race
Message-ID: <20210430084506.81DFFAD7FD@lists.varnish-cache.org>


commit 3fe065cd048d095ced7ba45bd42879f5adf470f5
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Apr 30 10:41:19 2021 +0200

    vtc_process: Mitigate screen refresh race
    
    It's otherwise too tight for for varnishstat's default refresh rate,
    possibly for varnishhist and varnishtop too.

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index bc243894b..7387dfa39 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -304,7 +304,7 @@ term_expect_text(struct process *pp,
 	if (y < 0 || y > pp->nlin)
 		vtc_fatal(pp->vl, "YYY %d nlin %d", y, pp->nlin);
 	x = strtoul(col, NULL, 0);
-	for(l = 0; l < 10 && x > pp->ncol; l++)	// wait for screen change
+	for(l = 0; l <= 10 && x > pp->ncol; l++)	// wait for screen change
 		usleep(100000);
 	if (x < 0 || x > pp->ncol)
 		vtc_fatal(pp->vl, "XXX %d ncol %d", x, pp->ncol);

From dridi.boukelmoune at gmail.com  Fri Apr 30 08:51:05 2021
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 30 Apr 2021 08:51:05 +0000 (UTC)
Subject: [master] 405d00759 vrt: Prevent potential null dereference
Message-ID: <20210430085105.56E47ADC65@lists.varnish-cache.org>


commit 405d0075920f8d63ab6f98a08380fc47ceca302c
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Apr 30 10:45:36 2021 +0200

    vrt: Prevent potential null dereference
    
    I'm leaving ctx alone because it is effectively checked.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index c86a70a0b..d60cc704c 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -655,6 +655,8 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc,
 VCL_BACKEND
 VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt)
 {
+
+	CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC);
 	CHECK_OBJ_NOTNULL(vrt->endpoint, VRT_ENDPOINT_MAGIC);
 	return (VRT_new_backend_clustered(ctx, NULL, vrt));
 }

From phk at FreeBSD.org  Fri Apr 30 14:54:06 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 30 Apr 2021 14:54:06 +0000 (UTC)
Subject: [master] 3a6b639e8 Rename strerror() to VAS_errtxt() for consistency.
Message-ID: <20210430145406.46F7E61474@lists.varnish-cache.org>


commit 3a6b639e8626427e77e4fe8de9227c679b0f72ad
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Apr 30 14:50:03 2021 +0000

    Rename strerror() to VAS_errtxt() for consistency.

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index ae4da2f84..8ee346cba 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -509,7 +509,7 @@ vca_accept_task(struct worker *wrk, void *arg)
 
 			VSL(SLT_SessError, 0, "%s %s %s %d %d \"%s\"",
 			    wa.acceptlsock->name, laddr, lport,
-			    ls->sock, i, vstrerror(i));
+			    ls->sock, i, VAS_errtxt(i));
 			(void)Pool_TrySumstat(wrk);
 			continue;
 		}
@@ -629,18 +629,18 @@ ccf_start(struct cli *cli, const char * const *av, void *priv)
 		    VTCP_fastopen(ls->sock, cache_param->listen_depth))
 			VSL(SLT_Error, 0,
 			    "Kernel TCP Fast Open: sock=%d, errno=%d %s",
-			    ls->sock, errno, vstrerror(errno));
+			    ls->sock, errno, VAS_errtxt(errno));
 		if (listen(ls->sock, cache_param->listen_depth)) {
 			VCLI_SetResult(cli, CLIS_CANT);
 			VCLI_Out(cli, "Listen failed on socket '%s': %s",
-			    ls->endpoint, vstrerror(errno));
+			    ls->endpoint, VAS_errtxt(errno));
 			return;
 		}
 		vca_tcp_opt_set(ls->sock, ls->uds, 1);
 		if (cache_param->accept_filter && VTCP_filter_http(ls->sock))
 			VSL(SLT_Error, 0,
 			    "Kernel filtering: sock=%d, errno=%d %s",
-			    ls->sock, errno, vstrerror(errno));
+			    ls->sock, errno, VAS_errtxt(errno));
 	}
 
 	need_test = 1;
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index d60cc704c..829b56595 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -162,7 +162,7 @@ vbe_dir_getfd(VRT_CTX, struct worker *wrk, VCL_BACKEND dir, struct backend *bp,
 		Lck_Unlock(bp->director->mtx);
 		VSLb(bo->vsl, SLT_FetchError,
 		     "backend %s: fail errno %d (%s)",
-		     VRT_BACKEND_string(dir), err, vstrerror(err));
+		     VRT_BACKEND_string(dir), err, VAS_errtxt(err));
 		VSC_C_main->backend_fail++;
 		bo->htc = NULL;
 		return (NULL);
@@ -186,7 +186,7 @@ vbe_dir_getfd(VRT_CTX, struct worker *wrk, VCL_BACKEND dir, struct backend *bp,
 		VSLb(bo->vsl, SLT_FetchError,
 		     "backend %s: proxy write errno %d (%s)",
 		     VRT_BACKEND_string(dir),
-		     errno, vstrerror(errno));
+		     errno, VAS_errtxt(errno));
 		// account as if connect failed - good idea?
 		VSC_C_main->backend_fail++;
 		bo->htc = NULL;
diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c
index 18126cef2..f56a9fac6 100644
--- a/bin/varnishd/cache/cache_backend_probe.c
+++ b/bin/varnishd/cache/cache_backend_probe.c
@@ -225,11 +225,11 @@ vbp_write(struct vbp_target *vt, int *sock, const void *buf, size_t len)
 		if (i < 0) {
 			vt->err_xmit |= 1;
 			bprintf(vt->resp_buf, "Write error %d (%s)",
-				errno, vstrerror(errno));
+				errno, VAS_errtxt(errno));
 		} else {
 			bprintf(vt->resp_buf,
 				"Short write (%d/%zu) error %d (%s)",
-				i, len, errno, vstrerror(errno));
+				i, len, errno, VAS_errtxt(errno));
 		}
 		VTCP_close(sock);
 		return (-1);
@@ -282,7 +282,7 @@ vbp_poke(struct vbp_target *vt)
 
 	s = VCP_Open(vt->conn_pool, t_end - t_now, &sa, &err);
 	if (s < 0) {
-		bprintf(vt->resp_buf, "Open error %d (%s)", err, vstrerror(err));
+		bprintf(vt->resp_buf, "Open error %d (%s)", err, VAS_errtxt(err));
 		Lck_Lock(&vbp_mtx);
 		if (vt->backend)
 			VBE_Connect_Error(vt->backend->vsc, err);
@@ -357,7 +357,7 @@ vbp_poke(struct vbp_target *vt)
 			if (!i)
 				errno = ETIMEDOUT;
 			bprintf(vt->resp_buf, "Poll error %d (%s)",
-			    errno, vstrerror(errno));
+			    errno, VAS_errtxt(errno));
 			i = -1;
 			break;
 		}
@@ -370,7 +370,7 @@ vbp_poke(struct vbp_target *vt)
 		if (i <= 0) {
 			if (i < 0)
 				bprintf(vt->resp_buf, "Read error %d (%s)",
-					errno, vstrerror(errno));
+					errno, VAS_errtxt(errno));
 			break;
 		}
 		rlen += i;
diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 63f605a05..68309705f 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -769,7 +769,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond,
 	pan_backtrace(pan_vsb);
 
 	if (err)
-		VSB_printf(pan_vsb, "errno = %d (%s)\n", err, vstrerror(err));
+		VSB_printf(pan_vsb, "errno = %d (%s)\n", err, VAS_errtxt(err));
 
 	pan_argv(pan_vsb);
 
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 9b5d59c82..fcdab5d6f 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -532,7 +532,7 @@ pool_breed(struct pool *qp)
 	if (errno) {
 		FREE_OBJ(pi);
 		VSL(SLT_Debug, 0, "Create worker thread failed %d %s",
-		    errno, vstrerror(errno));
+		    errno, VAS_errtxt(errno));
 		Lck_Lock(&pool_mtx);
 		VSC_C_main->threads_failed++;
 		Lck_Unlock(&pool_mtx);
diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c
index 99a4c58d9..dfd0555b4 100644
--- a/bin/varnishd/http1/cache_http1_fetch.c
+++ b/bin/varnishd/http1/cache_http1_fetch.c
@@ -131,7 +131,7 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes,
 			assert(i < 0);
 			VSLb(bo->vsl, SLT_FetchError,
 			    "req.body read error: %d (%s)",
-			    errno, vstrerror(errno));
+			    errno, VAS_errtxt(errno));
 			bo->req->doclose = SC_RX_BODY;
 		}
 		if (do_chunked)
@@ -153,7 +153,7 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes,
 
 	if (sc != SC_NULL) {
 		VSLb(bo->vsl, SLT_FetchError, "backend write error: %d (%s)",
-		    errno, vstrerror(errno));
+		    errno, VAS_errtxt(errno));
 		VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk));
 		htc->doclose = sc;
 		return (-1);
diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c
index 2499c68cb..c275c92ef 100644
--- a/bin/varnishd/http1/cache_http1_line.c
+++ b/bin/varnishd/http1/cache_http1_line.c
@@ -233,7 +233,7 @@ V1L_Flush(const struct worker *wrk)
 		if (i <= 0) {
 			VSLb(v1l->vsl, SLT_Debug,
 			    "Write error, retval = %zd, len = %zd, errno = %s",
-			    i, v1l->liov, vstrerror(errno));
+			    i, v1l->liov, VAS_errtxt(errno));
 			AZ(v1l->werr);
 			if (errno == EPIPE)
 				v1l->werr = SC_REM_CLOSE;
diff --git a/bin/varnishd/http1/cache_http1_vfp.c b/bin/varnishd/http1/cache_http1_vfp.c
index 13e042e6f..835db3599 100644
--- a/bin/varnishd/http1/cache_http1_vfp.c
+++ b/bin/varnishd/http1/cache_http1_vfp.c
@@ -79,7 +79,7 @@ v1f_read(const struct vfp_ctx *vc, struct http_conn *htc, void *d, ssize_t len)
 		if (i < 0) {
 			VTCP_Assert(i);
 			VSLb(vc->wrk->vsl, SLT_FetchError,
-			    "%s", vstrerror(errno));
+			    "%s", VAS_errtxt(errno));
 			return (i);
 		}
 		if (i == 0)
diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c
index 36d4a1c0e..26207a3bc 100644
--- a/bin/varnishd/http2/cache_http2_session.c
+++ b/bin/varnishd/http2/cache_http2_session.c
@@ -260,7 +260,7 @@ h2_ou_session(struct worker *wrk, struct h2_sess *h2,
 	VTCP_Assert(sz);
 	if (sz != strlen(h2_resp_101)) {
 		VSLb(h2->vsl, SLT_Debug, "H2: Upgrade: Error writing 101"
-		    " response: %s\n", vstrerror(errno));
+		    " response: %s\n", VAS_errtxt(errno));
 		return (h2_ou_rel(wrk, req));
 	}
 
diff --git a/bin/varnishd/mgt/mgt_acceptor.c b/bin/varnishd/mgt/mgt_acceptor.c
index 1af17f365..6488c5b03 100644
--- a/bin/varnishd/mgt/mgt_acceptor.c
+++ b/bin/varnishd/mgt/mgt_acceptor.c
@@ -134,7 +134,7 @@ MAC_reopen_sockets(void)
 			fail = err;
 		MGT_Complain(C_ERR,
 		    "Could not reopen listen socket %s: %s",
-		    ls->endpoint, vstrerror(err));
+		    ls->endpoint, VAS_errtxt(err));
 	}
 	return (fail);
 }
@@ -168,7 +168,7 @@ mk_listen_sock(const struct listen_arg *la, const struct suckaddr *sa)
 		FREE_OBJ(ls);
 		if (fail != EAFNOSUPPORT)
 			ARGV_ERR("Could not get socket %s: %s\n",
-			    la->endpoint, vstrerror(fail));
+			    la->endpoint, VAS_errtxt(fail));
 		return (NULL);
 	}
 	return (ls);
@@ -331,7 +331,7 @@ MAC_Arg(const char *spec)
 					 val);
 			if (errno)
 				ARGV_ERR("Cannot parse mode sub-arg %s in -a: "
-					 "%s\n", val, vstrerror(errno));
+					 "%s\n", val, VAS_errtxt(errno));
 			if (m <= 0 || m > 0777)
 				ARGV_ERR("Mode sub-arg %s out of range in -a\n",
 					 val);
diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 84f3c182e..5af419fed 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -629,7 +629,7 @@ MCH_Cli_Fail(void)
 		    " killed it.", (intmax_t)child_pid);
 	else
 		MGT_Complain(C_ERR, "Failed to kill child with PID %jd: %s",
-		    (intmax_t)child_pid, vstrerror(errno));
+		    (intmax_t)child_pid, VAS_errtxt(errno));
 }
 
 /*=====================================================================
diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c
index 11cff7889..9d307f793 100644
--- a/bin/varnishd/mgt/mgt_cli.c
+++ b/bin/varnishd/mgt/mgt_cli.c
@@ -281,7 +281,7 @@ mcf_auth(struct cli *cli, const char *const *av, void *priv)
 	fd = open(secret_file, O_RDONLY);
 	if (fd < 0) {
 		VCLI_Out(cli, "Cannot open secret file (%s)\n",
-		    vstrerror(errno));
+		    VAS_errtxt(errno));
 		VCLI_SetResult(cli, CLIS_CANT);
 		VJ_master(JAIL_MASTER_LOW);
 		return;
@@ -591,7 +591,7 @@ Marg_connect(const struct vev *e, int what)
 	M_fd = VTCP_connected(M_fd);
 	if (M_fd < 0) {
 		MGT_Complain(C_INFO, "Could not connect to CLI-master: %s",
-			vstrerror(errno));
+			VAS_errtxt(errno));
 		ma = VTAILQ_FIRST(&m_addr_list);
 		AN(ma);
 		VTAILQ_REMOVE(&m_addr_list, ma, list);
diff --git a/bin/varnishd/mgt/mgt_jail.c b/bin/varnishd/mgt/mgt_jail.c
index b65005fc6..db5c1275c 100644
--- a/bin/varnishd/mgt/mgt_jail.c
+++ b/bin/varnishd/mgt/mgt_jail.c
@@ -151,18 +151,18 @@ VJ_make_workdir(const char *dname)
 		VJ_master(JAIL_MASTER_FILE);
 		if (mkdir(dname, 0755) < 0 && errno != EEXIST)
 			ARGV_ERR("Cannot create working directory '%s': %s\n",
-				 dname, vstrerror(errno));
+				 dname, VAS_errtxt(errno));
 	}
 
 	if (chdir(dname) < 0)
 		ARGV_ERR("Cannot change to working directory '%s': %s\n",
-		    dname, vstrerror(errno));
+		    dname, VAS_errtxt(errno));
 
 	i = open("_.testfile", O_RDWR|O_CREAT|O_EXCL, 0600);
 	if (i < 0)
 		ARGV_ERR("Cannot create test-file in %s (%s)\n"
 		    "Check permissions (or delete old directory)\n",
-		    dname, vstrerror(errno));
+		    dname, VAS_errtxt(errno));
 	closefd(&i);
 	AZ(unlink("_.testfile"));
 	VJ_master(JAIL_MASTER_LOW);
@@ -186,11 +186,11 @@ VJ_make_subdir(const char *dname, const char *what, struct vsb *vsb)
 		if (vsb != NULL) {
 			VSB_printf(vsb,
 			    "Cannot create %s directory '%s': %s\n",
-			    what, dname, vstrerror(e));
+			    what, dname, VAS_errtxt(e));
 		} else {
 			MGT_Complain(C_ERR,
 			    "Cannot create %s directory '%s': %s",
-			    what, dname, vstrerror(e));
+			    what, dname, VAS_errtxt(e));
 		}
 		return (1);
 	}
diff --git a/bin/varnishd/mgt/mgt_jail_unix.c b/bin/varnishd/mgt/mgt_jail_unix.c
index c0d5c50d9..d979c00f0 100644
--- a/bin/varnishd/mgt/mgt_jail_unix.c
+++ b/bin/varnishd/mgt/mgt_jail_unix.c
@@ -256,11 +256,11 @@ vju_make_subdir(const char *dname, const char *what, struct vsb *vsb)
 		if (vsb != NULL) {
 			VSB_printf(vsb,
 			    "Cannot create %s directory '%s': %s\n",
-			    what, dname, vstrerror(e));
+			    what, dname, VAS_errtxt(e));
 		} else {
 			MGT_Complain(C_ERR,
 			    "Cannot create %s directory '%s': %s",
-			    what, dname, vstrerror(e));
+			    what, dname, VAS_errtxt(e));
 		}
 		return (1);
 	}
@@ -280,7 +280,7 @@ vju_make_workdir(const char *dname, const char *what, struct vsb *vsb)
 
 	if (mkdir(dname, 0755) < 0 && errno != EEXIST) {
 		MGT_Complain(C_ERR, "Cannot create working directory '%s': %s",
-		    dname, vstrerror(errno));
+		    dname, VAS_errtxt(errno));
 		return (1);
 	}
 	//lint -e{570}
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 9ad6d8a7c..f16e13127 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -232,7 +232,7 @@ make_secret(const char *dirname)
 	fdo = open(fn, O_RDWR|O_CREAT|O_TRUNC, 0640);
 	if (fdo < 0)
 		ARGV_ERR("Cannot create secret-file in %s (%s)\n",
-		    dirname, vstrerror(errno));
+		    dirname, VAS_errtxt(errno));
 
 	for (i = 0; i < 256; i++) {
 		AZ(VRND_RandomCrypto(&b, 1));
@@ -326,7 +326,7 @@ mgt_eric(void)
 
 	switch (fork()) {
 	case -1:
-		fprintf(stderr, "Fork() failed: %s\n", vstrerror(errno));
+		fprintf(stderr, "Fork() failed: %s\n", VAS_errtxt(errno));
 		exit(-1);
 	case 0:
 		closefd(&eric_pipes[0]);
@@ -426,7 +426,7 @@ mgt_f_read(const char *fn)
 	VFIL_setpath(&vcl_path, mgt_vcl_path);
 	if (VFIL_searchpath(vcl_path, NULL, &f, fn, &fnp) || f == NULL) {
 		ARGV_ERR("Cannot read -f file '%s' (%s)\n",
-		    fnp != NULL ? fnp : fn, vstrerror(errno));
+		    fnp != NULL ? fnp : fn, VAS_errtxt(errno));
 	}
 	free(fa->farg);
 	fa->farg = fnp;
@@ -454,7 +454,7 @@ create_pid_file(pid_t *ppid, const char *fmt, ...)
 		    (intmax_t)*ppid, VSB_data(vsb));
 	if (pfh == NULL)
 		ARGV_ERR("Could not open pid-file (%s): %s\n",
-		    VSB_data(vsb), vstrerror(errno));
+		    VSB_data(vsb), VAS_errtxt(errno));
 	VJ_master(JAIL_MASTER_LOW);
 	VSB_destroy(&vsb);
 	return (pfh);
@@ -667,7 +667,7 @@ main(int argc, char * const *argv)
 			I_fd = open(optarg, O_RDONLY);
 			if (I_fd < 0)
 				ARGV_ERR("\tCant open %s: %s\n",
-				    optarg, vstrerror(errno));
+				    optarg, VAS_errtxt(errno));
 			VJ_master(JAIL_MASTER_LOW);
 			break;
 		case 'l':
@@ -774,13 +774,13 @@ main(int argc, char * const *argv)
 		o = open(S_arg, O_RDONLY, 0);
 		if (o < 0)
 			ARGV_ERR("Cannot open -S file (%s): %s\n",
-			    S_arg, vstrerror(errno));
+			    S_arg, VAS_errtxt(errno));
 		closefd(&o);
 		VJ_master(JAIL_MASTER_LOW);
 	}
 
 	if (VIN_n_Arg(n_arg, &workdir) != 0)
-		ARGV_ERR("Invalid instance (-n) name: %s\n", vstrerror(errno));
+		ARGV_ERR("Invalid instance (-n) name: %s\n", VAS_errtxt(errno));
 
 	if (i_arg == NULL || *i_arg == '\0')
 		i_arg = mgt_HostName();
@@ -792,7 +792,7 @@ main(int argc, char * const *argv)
 
 	if (VJ_make_workdir(workdir))
 		ARGV_ERR("Cannot create working directory (%s): %s\n",
-		    workdir, vstrerror(errno));
+		    workdir, VAS_errtxt(errno));
 
 	VJ_master(JAIL_MASTER_SYSTEM);
 	AZ(system("rm -rf vmod_cache"));
@@ -801,7 +801,7 @@ main(int argc, char * const *argv)
 	if (VJ_make_subdir("vmod_cache", "VMOD cache", NULL)) {
 		ARGV_ERR(
 		    "Cannot create vmod directory (%s/vmod_cache): %s\n",
-		    workdir, vstrerror(errno));
+		    workdir, VAS_errtxt(errno));
 	}
 
 	if (C_flag)
diff --git a/bin/varnishd/mgt/mgt_symtab.c b/bin/varnishd/mgt/mgt_symtab.c
index 530c89d17..88b9aa844 100644
--- a/bin/varnishd/mgt/mgt_symtab.c
+++ b/bin/varnishd/mgt/mgt_symtab.c
@@ -85,13 +85,13 @@ mgt_vcl_cache_vmod(const char *nm, const char *fm, const char *to)
 		return (0);
 	if (fo < 0) {
 		fprintf(stderr, "While creating copy of vmod %s:\n\t%s: %s\n",
-		    nm, to, vstrerror(errno));
+		    nm, to, VAS_errtxt(errno));
 		return (1);
 	}
 	fi = open(fm, O_RDONLY);
 	if (fi < 0) {
 		fprintf(stderr, "Opening vmod %s from %s: %s\n",
-		    nm, fm, vstrerror(errno));
+		    nm, fm, VAS_errtxt(errno));
 		AZ(unlink(to));
 		closefd(&fo);
 		return (1);
@@ -102,7 +102,7 @@ mgt_vcl_cache_vmod(const char *nm, const char *fm, const char *to)
 			break;
 		if (sz < 0 || sz != write(fo, buf, sz)) {
 			fprintf(stderr, "Copying vmod %s: %s\n",
-			    nm, vstrerror(errno));
+			    nm, VAS_errtxt(errno));
 			AZ(unlink(to));
 			ret = 1;
 			break;
diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c
index 729804b5a..5775a2fb9 100644
--- a/bin/varnishd/mgt/mgt_vcc.c
+++ b/bin/varnishd/mgt/mgt_vcc.c
@@ -201,13 +201,13 @@ mgt_vcc_touchfile(const char *fn, struct vsb *sb)
 
 	i = open(fn, O_WRONLY|O_CREAT|O_TRUNC, 0640);
 	if (i < 0) {
-		VSB_printf(sb, "Failed to create %s: %s", fn, vstrerror(errno));
+		VSB_printf(sb, "Failed to create %s: %s", fn, VAS_errtxt(errno));
 		return (2);
 	}
 	if (fchown(i, mgt_param.uid, mgt_param.gid) != 0)
 		if (geteuid() == 0)
 			VSB_printf(sb, "Failed to change owner on %s: %s\n",
-			    fn, vstrerror(errno));
+			    fn, VAS_errtxt(errno));
 	closefd(&i);
 	return (0);
 }
diff --git a/bin/varnishd/storage/mgt_storage_persistent.c b/bin/varnishd/storage/mgt_storage_persistent.c
index 372dfb953..f8d797f85 100644
--- a/bin/varnishd/storage/mgt_storage_persistent.c
+++ b/bin/varnishd/storage/mgt_storage_persistent.c
@@ -210,7 +210,7 @@ smp_mgt_init(struct stevedore *parent, int ac, char * const *av)
 
 	if (sc->base == MAP_FAILED)
 		ARGV_ERR("(-spersistent) failed to mmap (%s)\n",
-		    vstrerror(errno));
+		    VAS_errtxt(errno));
 	if (target != NULL && sc->base != target)
 		fprintf(stderr, "WARNING: Persistent silo lost to ASLR %s\n",
 		    sc->filename);
diff --git a/bin/varnishd/storage/stevedore_utils.c b/bin/varnishd/storage/stevedore_utils.c
index e46b5485e..ac70b7aaf 100644
--- a/bin/varnishd/storage/stevedore_utils.c
+++ b/bin/varnishd/storage/stevedore_utils.c
@@ -103,7 +103,7 @@ STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx)
 		fd = mkstemp(buf);
 		if (fd < 0)
 			ARGV_ERR("(%s) \"%s\" mkstemp(%s) failed (%s)\n",
-			    ctx, fn, buf, vstrerror(errno));
+			    ctx, fn, buf, VAS_errtxt(errno));
 		AZ(unlink(buf));
 		*fnp = strdup(buf);
 		AN(*fnp);
@@ -112,7 +112,7 @@ STV_GetFile(const char *fn, int *fdp, const char **fnp, const char *ctx)
 		fd = open(fn, O_RDWR | O_LARGEFILE);
 		if (fd < 0)
 			ARGV_ERR("(%s) \"%s\" could not open (%s)\n",
-			    ctx, fn, vstrerror(errno));
+			    ctx, fn, VAS_errtxt(errno));
 		*fnp = fn;
 		retval = 0;
 	} else
diff --git a/bin/varnishd/storage/storage_file.c b/bin/varnishd/storage/storage_file.c
index 8d842365e..3c61a61c5 100644
--- a/bin/varnishd/storage/storage_file.c
+++ b/bin/varnishd/storage/storage_file.c
@@ -167,7 +167,7 @@ smf_init(struct stevedore *parent, int ac, char * const *av)
 	MCH_Fd_Inherit(sc->fd, "storage_file");
 	sc->filesize = STV_FileSize(sc->fd, size, &sc->pagesize, "-sfile");
 	if (VFIL_allocate(sc->fd, (off_t)sc->filesize, 0))
-		ARGV_ERR("(-sfile) allocation error: %s\n", vstrerror(errno));
+		ARGV_ERR("(-sfile) allocation error: %s\n", VAS_errtxt(errno));
 }
 
 /*--------------------------------------------------------------------
diff --git a/include/vas.h b/include/vas.h
index ca93fbf43..302170ae2 100644
--- a/include/vas.h
+++ b/include/vas.h
@@ -44,7 +44,7 @@
 #include <errno.h>
 #include <stddef.h>	// size_t
 
-const char * vstrerror(int e);
+const char * VAS_errtxt(int e);
 
 enum vas_e {
 	VAS_WRONG,
diff --git a/lib/libvarnish/vas.c b/lib/libvarnish/vas.c
index 392303fdd..b64418cc9 100644
--- a/lib/libvarnish/vas.c
+++ b/lib/libvarnish/vas.c
@@ -42,7 +42,7 @@
 #include "vas.h"
 
 const char *
-vstrerror(int e)
+VAS_errtxt(int e)
 {
 	const char *p;
 	int oerrno = errno;
diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c
index 9feb7c584..45a3d2f2a 100644
--- a/lib/libvarnish/vss.c
+++ b/lib/libvarnish/vss.c
@@ -132,7 +132,7 @@ vss_resolve(const char *addr, const char *def_port, int family, int socktype,
 	free(p);
 
 	if (ret == EAI_SYSTEM)
-		*errp = vstrerror(errno);
+		*errp = VAS_errtxt(errno);
 	else if (ret != 0)
 		*errp = gai_strerror(ret);
 
diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index 9de0389f2..d5afe460c 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -76,7 +76,7 @@ vtcp_sa_to_ascii(const void *sa, socklen_t l, char *abuf, unsigned alen,
 		 */
 		fprintf(stderr, "getnameinfo = %d %s\n", i, gai_strerror(i));
 		if (i == EAI_SYSTEM)
-			fprintf(stderr, "errno = %d %s\n", errno, vstrerror(errno));
+			fprintf(stderr, "errno = %d %s\n", errno, VAS_errtxt(errno));
 		if (abuf != NULL)
 			(void)snprintf(abuf, alen, "Conversion");
 		if (pbuf != NULL)

From phk at FreeBSD.org  Fri Apr 30 14:54:06 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 30 Apr 2021 14:54:06 +0000 (UTC)
Subject: [master] 3c32d3543 Export VAS_errtxt() so that all vas.c uses can be
 fulfilled.
Message-ID: <20210430145406.5A0DF6147A@lists.varnish-cache.org>


commit 3c32d3543ccad3963c0f2ae64e7a27a937cd58aa
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Apr 30 14:52:14 2021 +0000

    Export VAS_errtxt() so that all vas.c uses can be fulfilled.

diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map
index 2d8e29a97..b98a2b761 100644
--- a/lib/libvarnishapi/libvarnishapi.map
+++ b/lib/libvarnishapi/libvarnishapi.map
@@ -203,3 +203,11 @@ LIBVARNISHAPI_2.6 {	/* 2020-03-15 release */
     local:
 	*;
 };
+
+LIBVARNISHAPI_2.7 {	/* 2021-09-15 release */
+    global:
+	# vas.c
+	VAS_errtxt;
+    local:
+	*;
+};

From phk at FreeBSD.org  Fri Apr 30 14:54:06 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 30 Apr 2021 14:54:06 +0000 (UTC)
Subject: [master] 27585d899 Link libvarnishapi before libvarnish to only get
 one copy of vas.c
Message-ID: <20210430145406.739FD6147F@lists.varnish-cache.org>


commit 27585d8992a4d411e1ff157b354eae996c6d2079
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Apr 30 14:52:52 2021 +0000

    Link libvarnishapi before libvarnish to only get one copy of vas.c

diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am
index 75f84cb56..30cea03ad 100644
--- a/bin/varnishtest/Makefile.am
+++ b/bin/varnishtest/Makefile.am
@@ -51,8 +51,8 @@ varnishtest_SOURCES = \
 		vtc_varnish.c
 
 varnishtest_LDADD = \
-		$(top_builddir)/lib/libvarnish/libvarnish.la \
 		$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
+		$(top_builddir)/lib/libvarnish/libvarnish.la \
 		$(top_builddir)/lib/libvgz/libvgz.a \
 		@SAN_LDFLAGS@ \
 		@PCRE_LIBS@ \

From phk at FreeBSD.org  Fri Apr 30 16:20:06 2021
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 30 Apr 2021 16:20:06 +0000 (UTC)
Subject: [master] 76fdce8c1 Link libvarnishapi before libvarnish to get a
 consistent vas.c
Message-ID: <20210430162006.805F2641EB@lists.varnish-cache.org>


commit 76fdce8c108ae36396f1e8c3cf9243810cfeb341
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Apr 30 16:18:54 2021 +0000

    Link libvarnishapi before libvarnish to get a consistent vas.c

diff --git a/bin/varnishadm/Makefile.am b/bin/varnishadm/Makefile.am
index 0d0c8e709..2b4c31369 100644
--- a/bin/varnishadm/Makefile.am
+++ b/bin/varnishadm/Makefile.am
@@ -12,7 +12,7 @@ varnishadm_CFLAGS = @LIBEDIT_CFLAGS@ \
 	@SAN_CFLAGS@
 
 varnishadm_LDADD = \
-	$(top_builddir)/lib/libvarnish/libvarnish.la \
 	$(top_builddir)/lib/libvarnishapi/libvarnishapi.la \
+	$(top_builddir)/lib/libvarnish/libvarnish.la \
 	${PTHREAD_LIBS} ${RT_LIBS} ${NET_LIBS} @LIBEDIT_LIBS@ ${LIBM} \
 	@SAN_LDFLAGS@

From nils.goroll at uplex.de  Fri Apr 30 19:14:05 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 30 Apr 2021 19:14:05 +0000 (UTC)
Subject: [master] 300ede262 varnishncsa: Add -j to format for JSON
Message-ID: <20210430191405.AA4739371A@lists.varnish-cache.org>


commit 300ede2629a962df55fceba268933eaaf7846463
Author: madhavi.dintakurthy <madhavi.dintakurthy at target.com>
Date:   Thu Apr 15 10:46:27 2021 -0500

    varnishncsa: Add -j to format for JSON
    
    The new -j flag for varnishncsa allows writing JSON logs in combination
    with -f or -F to specify a custom format. Without -j, the format
    specifiers could be replaced with strings that would make the JSON
    invalid.
    
    One example is headers: without -j, some headers may be replaced with
    C-style \xXX escape sequences, which are not valid JSON. Since request
    headers are controlled by users on the internet, it would be easy for an
    attacker to make a log entry impossible to parse, which might cause it
    to be missed by administrators. Another example is numbers. Without -j,
    format specifiers are replaced with - if the value is empty. A bare - is
    not valid in JSON, so numbers are replaced with null with the -j flag.
    
    In general, -j makes the replacements JSON-compatible for all inputs.
    
    Co-authored-by: Ben Zvan <ben.zvan at target.com>
    Co-authored-by: Jasmine Wang <jasmine.wang at target.com>
    Co-authored-by: Jordan Christiansen <jordan.christiansen at target.com>
    Co-authored-by: Madhavi Dintakurthy <madhavi.dintakurthy at target.com>

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index a9b5fd185..d394220be 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -146,6 +146,9 @@ static struct ctx {
 	struct vsb		*vsb;
 	unsigned		gen;
 	VTAILQ_HEAD(,format)	format;
+	int			quote_how;
+	char			*missing_string;
+	char			*missing_int;
 
 	/* State */
 	struct watch_head	watch_vcl_log;
@@ -199,9 +202,9 @@ vsb_fcat(struct vsb *vsb, const struct fragment *f, const char *dflt)
 {
 	if (f->gen == CTX.gen) {
 		assert(f->b <= f->e);
-		VSB_quote(vsb, f->b, f->e - f->b, VSB_QUOTE_ESCHEX);
+		VSB_quote(vsb, f->b, f->e - f->b, CTX.quote_how);
 	} else if (dflt)
-		VSB_quote(vsb, dflt, -1, VSB_QUOTE_ESCHEX);
+		VSB_quote(vsb, dflt, -1, CTX.quote_how);
 	else
 		return (-1);
 	return (0);
@@ -247,7 +250,7 @@ format_fragment(const struct format *format)
 	if (format->frag->gen != CTX.gen) {
 		if (format->string == NULL)
 			return (-1);
-		VSB_quote(CTX.vsb, format->string, -1, VSB_QUOTE_ESCHEX);
+		VSB_quote(CTX.vsb, format->string, -1, CTX.quote_how);
 		return (0);
 	}
 	AZ(vsb_fcat(CTX.vsb, format->frag, NULL));
@@ -363,13 +366,13 @@ format_auth(const struct format *format)
 	    CTX.frag[F_auth].e)) {
 		if (format->string == NULL)
 			return (-1);
-		VSB_quote(CTX.vsb, format->string, -1, VSB_QUOTE_ESCHEX);
+		VSB_quote(CTX.vsb, format->string, -1, CTX.quote_how);
 		return (0);
 	}
 	q = strchr(buf, ':');
 	if (q != NULL)
 		*q = '\0';
-	VSB_quote(CTX.vsb, buf, -1, VSB_QUOTE_ESCHEX);
+	VSB_quote(CTX.vsb, buf, -1, CTX.quote_how);
 	return (1);
 }
 
@@ -557,7 +560,7 @@ addf_hdr(struct watch_head *head, const char *key)
 	AN(f);
 	f->func = format_fragment;
 	f->frag = &w->frag;
-	f->string = strdup("-");
+	f->string = strdup(CTX.missing_string);
 	AN(f->string);
 	VTAILQ_INSERT_TAIL(&CTX.format, f, list);
 }
@@ -580,7 +583,7 @@ addf_vsl(enum VSL_tag_e tag, long i, const char *prefix)
 		assert(w->prefixlen > 0);
 	}
 	VTAILQ_INSERT_TAIL(&CTX.watch_vsl, w, list);
-	addf_fragment(&w->frag, "-");
+	addf_fragment(&w->frag, CTX.missing_string);
 }
 
 static void
@@ -716,28 +719,28 @@ parse_format(const char *format)
 		p++;
 		switch (*p) {
 		case 'b':	/* Body bytes sent */
-			addf_fragment(&CTX.frag[F_b], "-");
+			addf_fragment(&CTX.frag[F_b], CTX.missing_int);
 			break;
 		case 'D':	/* Float request time */
 			addf_time('T', "us");
 			break;
 		case 'h':	/* Client host name / IP Address */
-			addf_fragment(&CTX.frag[F_h], "-");
+			addf_fragment(&CTX.frag[F_h], CTX.missing_string);
 			break;
 		case 'H':	/* Protocol */
 			addf_fragment(&CTX.frag[F_H], "HTTP/1.0");
 			break;
 		case 'I':	/* Bytes received */
-			addf_fragment(&CTX.frag[F_I], "-");
+			addf_fragment(&CTX.frag[F_I], CTX.missing_int);
 			break;
 		case 'l':	/* Client user ID (identd) always '-' */
 			AZ(VSB_putc(vsb, '-'));
 			break;
 		case 'm':	/* Method */
-			addf_fragment(&CTX.frag[F_m], "-");
+			addf_fragment(&CTX.frag[F_m], CTX.missing_string);
 			break;
 		case 'O':	/* Bytes sent */
-			addf_fragment(&CTX.frag[F_O], "-");
+			addf_fragment(&CTX.frag[F_O], CTX.missing_int);
 			break;
 		case 'q':	/* Query string */
 			addf_fragment(&CTX.frag[F_q], "");
@@ -746,7 +749,7 @@ parse_format(const char *format)
 			addf_requestline();
 			break;
 		case 's':	/* Status code */
-			addf_fragment(&CTX.frag[F_s], "-");
+			addf_fragment(&CTX.frag[F_s], CTX.missing_int);
 			break;
 		case 't':	/* strftime */
 			addf_time(*p, TIME_FMT);
@@ -758,7 +761,7 @@ parse_format(const char *format)
 			addf_auth();
 			break;
 		case 'U':	/* URL */
-			addf_fragment(&CTX.frag[F_U], "-");
+			addf_fragment(&CTX.frag[F_U], CTX.missing_string);
 			break;
 		case '{':
 			p++;
@@ -1153,6 +1156,9 @@ main(int argc, char * const *argv)
 	CTX.vsb = VSB_new_auto();
 	AN(CTX.vsb);
 	VB64_init();
+	CTX.quote_how = VSB_QUOTE_ESCHEX;
+	REPLACE(CTX.missing_string, "-");
+	REPLACE(CTX.missing_int, "-");
 
 	tzset();		// We use localtime_r(3)
 
@@ -1184,6 +1190,11 @@ main(int argc, char * const *argv)
 			/* Usage help */
 			VUT_Usage(vut, &vopt_spec, 0);
 			break;
+		case 'j':
+			REPLACE(CTX.missing_string, "");
+			REPLACE(CTX.missing_int, "0");
+			CTX.quote_how = VSB_QUOTE_JSON;
+			break;
 		case 'w':
 			/* Write to file */
 			REPLACE(CTX.w_arg, optarg);
diff --git a/bin/varnishncsa/varnishncsa_options.h b/bin/varnishncsa/varnishncsa_options.h
index 8c86761ed..fa2dd57d1 100644
--- a/bin/varnishncsa/varnishncsa_options.h
+++ b/bin/varnishncsa/varnishncsa_options.h
@@ -78,6 +78,15 @@
 	VOPT("E", "[-E]", "Show ESI requests",				\
 	    "Show ESI requests, implies client mode."			\
 	)
+#define NCSA_OPT_j							\
+	VOPT("j", "[-j]", "Make output JSON-compatible",		\
+	    "Make format-specifier replacements JSON-compatible. When"	\
+	    " escaping characters, use JSON-style \\\\uXXXX escape"	\
+	    " sequences instead of C-style \\\\xXX sequences. Empty"	\
+	    " strings will be replaced with \"\" instead of \"-\", and"	\
+	    " empty integers will be replaced with null. Use -F or -f"	\
+	    " in combination with -j to write JSON logs."		\
+	)
 
 NCSA_OPT_a
 NCSA_OPT_b
@@ -90,6 +99,7 @@ NCSA_OPT_F
 NCSA_OPT_f
 NCSA_OPT_g
 VUT_OPT_h
+NCSA_OPT_j
 VSL_OPT_L
 VUT_OPT_n
 VUT_GLOBAL_OPT_P
diff --git a/bin/varnishtest/tests/u00016.vtc b/bin/varnishtest/tests/u00016.vtc
new file mode 100644
index 000000000..de46499b4
--- /dev/null
+++ b/bin/varnishtest/tests/u00016.vtc
@@ -0,0 +1,56 @@
+varnishtest "varnishncsa -j json escaping"
+
+feature cmd "python3 -c 'import json'"
+
+server s1 -repeat 3 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {} -start
+
+client c1 {
+	txreq -hdr "x-request-id: YES" -hdr "test:ascii"
+	rxresp
+	txreq -hdr "x-request-id: ?" -hdr "test:unicode"
+	rxresp
+	txreq -hdr "test:null"
+	rxresp
+} -run
+
+delay 1
+
+# ASCII string with -j is valid JSON
+shell -match "^YES$" {
+	varnishncsa -n ${v1_name} -d -q 'ReqHeader:test eq "ascii"' -j \
+		-F '{"xrid": "%{X-REQUEST-ID}i"}' |
+	python3 -c 'import sys, json; print(json.load(sys.stdin)["xrid"])'
+}
+
+# ? without -j is not valid JSON
+shell -err -expect "Invalid \\escape" {
+	varnishncsa -n ${v1_name} -d -q 'ReqHeader:test eq "unicode"' \
+		-F '{"xrid": "%{X-REQUEST-ID}i"}' |
+	python3 -c 'import sys, json; print(json.load(sys.stdin)["xrid"])'
+}
+
+# ? with -j is valid JSON
+shell -match "^?$" {
+	varnishncsa -n ${v1_name} -d -q 'ReqHeader:test eq "unicode"' -j \
+		-F '{"xrid": "%{X-REQUEST-ID}i"}' |
+	python3 -c 'import sys, json; print(json.load(sys.stdin)["xrid"])'
+}
+
+# Empty strings are not replaced with "-"
+shell -match "" {
+	varnishncsa -n ${v1_name} -d -q 'ReqHeader:test eq "null"' -j \
+		-F '{"xrid": "%{X-REQUEST-ID}i"}' |
+	python3 -c 'import sys, json; print(json.load(sys.stdin)["xrid"])'
+}
+
+# Empty VCL_Log entries are not replaced with "-"
+shell -match "" {
+	varnishncsa -n ${v1_name} -d -q 'ReqHeader:test eq "null"' -j \
+		-F '{"xrid": "%{VCL_Log:varnishncsa}x"}' |
+	python3 -c 'import sys, json; print(json.load(sys.stdin)["xrid"])'
+}
diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst
index ceb115b5e..51b9316ab 100644
--- a/doc/sphinx/reference/varnishncsa.rst
+++ b/doc/sphinx/reference/varnishncsa.rst
@@ -246,6 +246,11 @@ Log the entire Timestamp record associated with the processing length::
 
   varnishncsa -F "%{VSL:Timestamp:Process}x"
 
+Log in JSON, using the -j flag to ensure that the output is valid JSON
+for all inputs::
+
+  varnishncsa -j -F '{"size": %b, "time": "%t", "ua": "%{User-Agent}i"}'
+
 SEE ALSO
 ========
 

From nils.goroll at uplex.de  Fri Apr 30 19:16:05 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 30 Apr 2021 19:16:05 +0000 (UTC)
Subject: [master] 127964c48 Handle unformatable VCL_TIME to string conversion
 failures
Message-ID: <20210430191605.0971193981@lists.varnish-cache.org>


commit 127964c48d1c780d036266218926d1b56680277c
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed May 6 11:46:55 2020 +0200

    Handle unformatable VCL_TIME to string conversion failures
    
    For VCL_TIME values that would convert to a year element that can not fit
    in an int, gmtime_r would fail, and VTIM_format() would use random stack
    values when picking weekday and month strings.
    
    This patch changes VTIM_format to return "" when gmtime_r reports
    failures. This way the API is not changed. Callers can test for empty
    string to catch the failure if needed.
    
    VRT_TIME_string is patched to catch the VTIM_format error, and return NULL
    on failure.
    
    Fixes: #3308

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index d31065245..c5a8d1633 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -840,8 +840,11 @@ VRT_TIME_string(VRT_CTX, VCL_TIME t)
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	p = WS_Alloc(ctx->ws, VTIM_FORMAT_SIZE);
-	if (p != NULL)
+	if (p != NULL) {
 		VTIM_format(t, p);
+		if (*p == '\0')
+			p = NULL;
+	}
 	return (p);
 }
 
diff --git a/bin/varnishtest/tests/r03308.vtc b/bin/varnishtest/tests/r03308.vtc
new file mode 100644
index 000000000..a97cb0cd5
--- /dev/null
+++ b/bin/varnishtest/tests/r03308.vtc
@@ -0,0 +1,20 @@
+varnishtest "Unformatable VCL_TIME"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import std;
+
+	sub vcl_deliver {
+		set resp.http.ts = std.real2time(std.real("1e+22", 0), now);
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.http.ts == ""
+} -run
diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index 5b91fc970..676330e1b 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -167,11 +167,16 @@ VTIM_format(vtim_real t, char *p)
 	struct tm tm;
 	time_t tt;
 
+	AN(p);
 	tt = (time_t) t;
-	(void)gmtime_r(&tt, &tm);
-	AN(snprintf(p, VTIM_FORMAT_SIZE, "%s, %02d %s %4d %02d:%02d:%02d GMT",
-	    weekday_name[tm.tm_wday], tm.tm_mday, month_name[tm.tm_mon],
-	    tm.tm_year + 1900, tm.tm_hour, tm.tm_min, tm.tm_sec));
+	if (gmtime_r(&tt, &tm) != NULL)
+		AN(snprintf(p, VTIM_FORMAT_SIZE,
+			"%s, %02d %s %4d %02d:%02d:%02d GMT",
+			weekday_name[tm.tm_wday],
+			tm.tm_mday, month_name[tm.tm_mon],
+			tm.tm_year + 1900, tm.tm_hour, tm.tm_min, tm.tm_sec));
+	else
+		*p = '\0';
 }
 
 #ifdef TEST_DRIVER

From nils.goroll at uplex.de  Fri Apr 30 19:16:05 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 30 Apr 2021 19:16:05 +0000 (UTC)
Subject: [master] ffd88acd8 Don't waste workspace on format failure
Message-ID: <20210430191605.1E69493985@lists.varnish-cache.org>


commit ffd88acd822f51ffa248dbcc24b57e715e67a435
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Wed May 6 15:00:38 2020 +0200

    Don't waste workspace on format failure

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index c5a8d1633..b87dae091 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -837,13 +837,17 @@ VCL_STRING v_matchproto_()
 VRT_TIME_string(VRT_CTX, VCL_TIME t)
 {
 	char *p;
+	uintptr_t snapshot;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	snapshot = WS_Snapshot(ctx->ws);
 	p = WS_Alloc(ctx->ws, VTIM_FORMAT_SIZE);
 	if (p != NULL) {
 		VTIM_format(t, p);
-		if (*p == '\0')
+		if (*p == '\0') {
 			p = NULL;
+			WS_Reset(ctx->ws, snapshot);
+		}
 	}
 	return (p);
 }

From nils.goroll at uplex.de  Fri Apr 30 20:26:10 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 30 Apr 2021 20:26:10 +0000 (UTC)
Subject: [master] 96b7bf526 Make the VTIM_format() test 64bit only
Message-ID: <20210430202610.37DF295E8F@lists.varnish-cache.org>


commit 96b7bf5265fe9a22f920fbe70d034ae57f0ae1c3
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Apr 30 22:19:47 2021 +0200

    Make the VTIM_format() test 64bit only
    
    It is not obvious how we could trigger a gmtime_r() EOVERFLOW on 32bit
    
    Thank you phk for pushing me back onto the right track at a late hour.
    
    Ref #3308

diff --git a/bin/varnishtest/tests/r03308.vtc b/bin/varnishtest/tests/r03308.vtc
index f5cfaa8a1..6c60821c9 100644
--- a/bin/varnishtest/tests/r03308.vtc
+++ b/bin/varnishtest/tests/r03308.vtc
@@ -1,5 +1,7 @@
 varnishtest "Unformatable VCL_TIME"
 
+feature 64bit
+
 server s1 {
 	rxreq
 	txresp

From nils.goroll at uplex.de  Fri Apr 30 20:26:09 2021
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 30 Apr 2021 20:26:09 +0000 (UTC)
Subject: [master] 4e1788a7d VRT_TIME_string to trigger VCL failure for
 unformatable time
Message-ID: <20210430202610.2CD3C95E8E@lists.varnish-cache.org>


commit 4e1788a7d612fd2c6f8c4279c4bcc8dc31b6f041
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Apr 30 21:25:13 2021 +0200

    VRT_TIME_string to trigger VCL failure for unformatable time
    
    as per pow-wow decision
    
    Ref #3308

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index b87dae091..59ebaf697 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -837,17 +837,17 @@ VCL_STRING v_matchproto_()
 VRT_TIME_string(VRT_CTX, VCL_TIME t)
 {
 	char *p;
-	uintptr_t snapshot;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	snapshot = WS_Snapshot(ctx->ws);
 	p = WS_Alloc(ctx->ws, VTIM_FORMAT_SIZE);
-	if (p != NULL) {
-		VTIM_format(t, p);
-		if (*p == '\0') {
-			p = NULL;
-			WS_Reset(ctx->ws, snapshot);
-		}
+	if (p == NULL) {
+		VRT_fail(ctx, "Workspace overflow");
+		return (NULL);
+	}
+	VTIM_format(t, p);
+	if (*p == '\0') {
+		VRT_fail(ctx, "Unformatable VCL_TIME");
+		return (NULL);
 	}
 	return (p);
 }
diff --git a/bin/varnishtest/tests/r03308.vtc b/bin/varnishtest/tests/r03308.vtc
index a97cb0cd5..f5cfaa8a1 100644
--- a/bin/varnishtest/tests/r03308.vtc
+++ b/bin/varnishtest/tests/r03308.vtc
@@ -16,5 +16,5 @@ varnish v1 -vcl+backend {
 client c1 {
 	txreq
 	rxresp
-	expect resp.http.ts == ""
+	expect resp.status == 503
 } -run