From phk at FreeBSD.org  Mon Oct  1 09:59:11 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon,  1 Oct 2018 09:59:11 +0000 (UTC)
Subject: [master] 2d6df8e65 Only do the "feature dns" DNS-lookup if/when
 necessary.
Message-ID: <20181001095911.E8F8565686@lists.varnish-cache.org>


commit 2d6df8e651d42da8457e408c02a1552f35226271
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 1 09:54:57 2018 +0000

    Only do the "feature dns" DNS-lookup if/when necessary.
    
    Spotted by:     Willy Tarreau <w at 1wt.eu>

diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index 182f258e9..c091afe40 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -82,7 +82,6 @@ extern char *vmod_path;
 extern struct vsb *params_vsb;
 extern int leave_temp;
 extern int vtc_witness;
-extern int feature_dns;
 extern int ign_unknown_macro;
 
 void init_server(void);
diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index 4e5cf6b2b..01ae08ba7 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -97,7 +97,6 @@ char *vmod_path = NULL;
 struct vsb *params_vsb = NULL;
 int leave_temp;
 int vtc_witness = 0;
-int feature_dns;
 
 /**********************************************************************
  * Parse a -D option argument into a name/val pair, and insert
@@ -430,39 +429,6 @@ i_mode(void)
 	AZ(putenv(strdup("MALLOC_CONF=abort:true,junk:true")));
 }
 
-/**********************************************************************
- * Most test-cases use only numeric IP#'s but a few requires non-demented
- * DNS services.  This is a basic sanity check for those.
- */
-
-static int v_matchproto_(vss_resolved_f)
-dns_cb(void *priv, const struct suckaddr *sa)
-{
-	char abuf[VTCP_ADDRBUFSIZE];
-	char pbuf[VTCP_PORTBUFSIZE];
-	int *ret = priv;
-
-	VTCP_name(sa, abuf, sizeof abuf, pbuf, sizeof pbuf);
-	if (strcmp(abuf, "192.0.2.255")) {
-		fprintf(stderr, "DNS-test: Wrong response: %s\n", abuf);
-		*ret = -1;
-	} else if (*ret == 0)
-		*ret = 1;
-	return (0);
-}
-
-static int
-dns_works(void)
-{
-	int ret = 0, error;
-	const char *msg;
-
-	error = VSS_resolver("dns-canary.freebsd.dk", NULL, dns_cb, &ret, &msg);
-	if (error || msg != NULL || ret != 1)
-		return (0);
-	return (1);
-}
-
 /**********************************************************************
  * Figure out what IP related magic
  */
@@ -682,7 +648,6 @@ main(int argc, char * const *argv)
 
 	AZ(VSB_finish(params_vsb));
 
-	feature_dns = dns_works();
 	ip_magic();
 
 	if (iflg)
diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c
index 2fcf5e0e2..b55d4be13 100644
--- a/bin/varnishtest/vtc_misc.c
+++ b/bin/varnishtest/vtc_misc.c
@@ -47,6 +47,8 @@
 
 #include "vnum.h"
 #include "vre.h"
+#include "vtcp.h"
+#include "vss.h"
 #include "vtim.h"
 
 /* SECTION: vtest vtest
@@ -338,6 +340,39 @@ cmd_delay(CMD_ARGS)
 	VTIM_sleep(f);
 }
 
+/**********************************************************************
+ * Most test-cases use only numeric IP#'s but a few requires non-demented
+ * DNS services.  This is a basic sanity check for those.
+ */
+
+static int v_matchproto_(vss_resolved_f)
+dns_cb(void *priv, const struct suckaddr *sa)
+{
+	char abuf[VTCP_ADDRBUFSIZE];
+	char pbuf[VTCP_PORTBUFSIZE];
+	int *ret = priv;
+
+	VTCP_name(sa, abuf, sizeof abuf, pbuf, sizeof pbuf);
+	if (strcmp(abuf, "192.0.2.255")) {
+		fprintf(stderr, "DNS-test: Wrong response: %s\n", abuf);
+		*ret = -1;
+	} else if (*ret == 0)
+		*ret = 1;
+	return (0);
+}
+
+static int
+dns_works(void)
+{
+	int ret = 0, error;
+	const char *msg;
+
+	error = VSS_resolver("dns-canary.freebsd.dk", NULL, dns_cb, &ret, &msg);
+	if (error || msg != NULL || ret != 1)
+		return (0);
+	return (1);
+}
+
 /* SECTION: feature feature
  *
  * Test that the required feature(s) for a test are available, and skip
@@ -424,7 +459,7 @@ cmd_feature(CMD_ARGS)
 		}
 		FEATURE("pcre_jit", VRE_has_jit);
 		FEATURE("64bit", sizeof(void*) == 8);
-		FEATURE("dns", feature_dns);
+		FEATURE("dns", dns_works());
 		FEATURE("topbuild", iflg);
 		FEATURE("root", !geteuid());
 		FEATURE("user_varnish", getpwnam("varnish") != NULL);

From phk at FreeBSD.org  Wed Oct  3 07:52:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed,  3 Oct 2018 07:52:10 +0000 (UTC)
Subject: [master] c4308c262 Return a canned minimal 500 if we run out of ws in
 H2-deliver.
Message-ID: <20181003075210.E367AA4291@lists.varnish-cache.org>


commit c4308c26282ef64dc6601ae8dc6fbefd87d07c89
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 3 07:50:29 2018 +0000

    Return a canned minimal 500 if we run out of ws in H2-deliver.
    
    (Same as we do in H1)
    
    Fixes #2589

diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c
index 912eb88bc..8f4b56e1d 100644
--- a/bin/varnishd/http2/cache_http2_deliver.c
+++ b/bin/varnishd/http2/cache_http2_deliver.c
@@ -167,17 +167,35 @@ h2_enc_len(struct vsb *vsb, unsigned bits, unsigned val, uint8_t b0)
 	unsigned mask = (1U << bits) - 1U;
 
 	if (val >= mask) {
-		AZ(VSB_putc(vsb, b0 | (uint8_t)mask));
+		VSB_putc(vsb, b0 | (uint8_t)mask);
 		val -= mask;
 		while (val >= 128) {
-			AZ(VSB_putc(vsb, 0x80 | ((uint8_t)val & 0x7f)));
+			VSB_putc(vsb, 0x80 | ((uint8_t)val & 0x7f));
 			val >>= 7;
 		}
 	}
-	AZ(VSB_putc(vsb, (uint8_t)val));
+	VSB_putc(vsb, (uint8_t)val);
 	return (0);
 }
 
+/*
+ * Hand-crafted-H2-HEADERS-R-Us:
+ *
+ * This is a handbuilt HEADERS frame for when we run out of workspace
+ * during delivery.
+ */
+
+static const uint8_t h2_500_resp[] = {
+	// :status 500
+	0x8e,
+
+	// content-length 0
+	0x1f, 0x0d, 0x01, 0x30,
+
+	// server Varnish
+	0x1f, 0x27, 0x07, 'V', 'a', 'r', 'n', 'i', 's', 'h',
+};
+
 void v_matchproto_(vtr_deliver_f)
 h2_deliver(struct req *req, struct boc *boc, int sendbody)
 {
@@ -203,10 +221,10 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 	AN(VSB_new(&resp, req->ws->f, l, VSB_FIXEDLEN));
 
 	l = h2_status(buf, req->resp->status);
-	AZ(VSB_bcat(&resp, buf, l));
+	VSB_bcat(&resp, buf, l);
 
 	hp = req->resp;
-	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
+	for (u = HTTP_HDR_FIRST; u < hp->nhd && !VSB_error(&resp); u++) {
 		r = strchr(hp->hd[u].b, ':');
 		AN(r);
 
@@ -227,24 +245,32 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 			VSLb(req->vsl, SLT_Debug,
 			    "HP {%d, \"%s\", \"%s\"} <%s>",
 			    hps->idx, hps->name, hps->val, hp->hd[u].b);
-			AZ(h2_enc_len(&resp, 4, hps->idx, 0x10));
+			h2_enc_len(&resp, 4, hps->idx, 0x10);
 		} else {
-			AZ(VSB_putc(&resp, 0x10));
+			VSB_putc(&resp, 0x10);
 			sz--;
-			AZ(h2_enc_len(&resp, 7, sz, 0));
+			h2_enc_len(&resp, 7, sz, 0);
 			for (sz1 = 0; sz1 < sz; sz1++)
-				AZ(VSB_putc(&resp, tolower(hp->hd[u].b[sz1])));
+				VSB_putc(&resp, tolower(hp->hd[u].b[sz1]));
 
 		}
 
 		while (vct_islws(*++r))
 			continue;
 		sz = hp->hd[u].e - r;
-		AZ(h2_enc_len(&resp, 7, sz, 0));
-		AZ(VSB_bcat(&resp, r, sz));
+		h2_enc_len(&resp, 7, sz, 0);
+		VSB_bcat(&resp, r, sz);
+	}
+	if (VSB_finish(&resp)) {
+		// We ran out of workspace, return minimal 500
+		// XXX: VSC counter ?
+		r = (const char*)h2_500_resp;
+		sz = sizeof h2_500_resp;
+		sendbody = 0;
+	} else {
+		sz = VSB_len(&resp);
+		r = req->ws->f;
 	}
-	AZ(VSB_finish(&resp));
-	sz = VSB_len(&resp);
 
 	AZ(req->wrk->v1l);
 
@@ -256,7 +282,7 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 	H2_Send_Get(req->wrk, r2->h2sess, r2);
 	H2_Send(req->wrk, r2, H2_F_HEADERS,
 	    (sendbody ? 0 : H2FF_HEADERS_END_STREAM) | H2FF_HEADERS_END_HEADERS,
-	    sz, req->ws->f);
+	    sz, r);
 	H2_Send_Rel(r2->h2sess, r2);
 	req->acct.resp_hdrbytes += sz;
 
diff --git a/bin/varnishtest/tests/r02589.vtc b/bin/varnishtest/tests/r02589.vtc
new file mode 100644
index 000000000..418cc6833
--- /dev/null
+++ b/bin/varnishtest/tests/r02589.vtc
@@ -0,0 +1,29 @@
+varnishtest "workspace overrun on h/2 delivery"
+
+server s1 {
+	rxreq
+	txresp -hdrlen Foo 500
+} -start
+
+varnish v1 -vcl+backend  {
+	import vtc;
+	sub vcl_deliver {
+		vtc.workspace_alloc(client, -50);
+	}
+} -start
+
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.set debug +syncvsl"
+varnish v1 -cliok "param.set vsl_mask +H2TxHdr"
+varnish v1 -cliok "param.set vsl_mask +H2TxBody"
+
+
+client c1 {
+	stream 1 {
+		txreq
+		rxresp
+		expect resp.status == 500
+		expect resp.http.content-length == 0
+		expect resp.http.server == "Varnish"
+	} -run
+} -run

From phk at FreeBSD.org  Wed Oct  3 10:17:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed,  3 Oct 2018 10:17:10 +0000 (UTC)
Subject: [master] ddac7e41b Add a watchdog to worker pools.
Message-ID: <20181003101710.4E2B1A6D00@lists.varnish-cache.org>


commit ddac7e41b64af09662305d07ef3be0376d6e2021
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 3 10:12:59 2018 +0000

    Add a watchdog to worker pools.
    
    If the worker pool is configured too small, it can deadlock.
    
    Recovering from this would require a lot of complicated code, to
    discover queued but unscheduled tasks which can be cancelled
    (because the client went away or otherwise) and code to do the
    cancelling etc. etc.
    
    But fundamentally either people configured their pools wrong, in
    which case we want them to notice, or they are under DoS, in which
    case recovering gracefully is unlikely be a major improvement over
    a restart.
    
    Instead we implement a per-pool watchdog and kill the child process
    if nothing has been dequeued for too long.
    
    Default value 10 seconds, open to discussion.
    
    Band-aid for:   #2418
    
    Test-case by:   @Dridi

diff --git a/bin/varnishd/cache/cache_pool.h b/bin/varnishd/cache/cache_pool.h
index e7a6a618b..9902e32ad 100644
--- a/bin/varnishd/cache/cache_pool.h
+++ b/bin/varnishd/cache/cache_pool.h
@@ -53,6 +53,7 @@ struct pool {
 	uintmax_t			sdropped;
 	uintmax_t			rdropped;
 	uintmax_t			nqueued;
+	uintmax_t			ndequeued;
 	struct VSC_main_wrk		*a_stat;
 	struct VSC_main_wrk		*b_stat;
 
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 589ce68cb..f7f95cfc5 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -341,6 +341,7 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk)
 			tp = VTAILQ_FIRST(&pp->queues[i]);
 			if (tp != NULL) {
 				pp->lqueue--;
+				pp->ndequeued--;
 				VTAILQ_REMOVE(&pp->queues[i], tp, list);
 				break;
 			}
@@ -487,6 +488,8 @@ pool_herder(void *priv)
 	struct worker *wrk;
 	double delay;
 	int wthread_min;
+	uintmax_t dq = (1ULL << 31);
+	double dqt;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 
@@ -494,6 +497,29 @@ pool_herder(void *priv)
 	THR_Init();
 
 	while (!pp->die || pp->nthr > 0) {
+		/*
+		 * If the worker pool is configured too small, we can
+		 * end up deadlocking it (see #2418 for details).
+		 *
+		 * Recovering from this would require a lot of complicated
+		 * code, and fundamentally, either people configured their
+		 * pools wrong, in which case we want them to notice, or
+		 * they are under DoS, in which case recovering gracefully
+		 * is unlikely be a major improvement.
+		 *
+		 * Instead we implement a watchdog and kill the worker if
+		 * nothing has been dequeued for that long.
+		 */
+		if (dq != pp->ndequeued) {
+			dq = pp->ndequeued;
+			dqt = VTIM_real();
+		} else if (pp->lqueue &&
+		    VTIM_real() - dqt > cache_param->wthread_watchdog) {
+			VSL(SLT_Error, 0,
+			   "Pool Herder: Queue does not move ql=%u dt=%f",
+			   pp->lqueue, VTIM_real() - dqt);
+			WRONG("Worker Pool Queue does not move");
+		}
 		wthread_min = cache_param->wthread_min;
 		if (pp->die)
 			wthread_min = 0;
diff --git a/bin/varnishd/common/common_param.h b/bin/varnishd/common/common_param.h
index 2366527b1..13ce0cb18 100644
--- a/bin/varnishd/common/common_param.h
+++ b/bin/varnishd/common/common_param.h
@@ -105,6 +105,7 @@ struct params {
 	double			wthread_add_delay;
 	double			wthread_fail_delay;
 	double			wthread_destroy_delay;
+	double			wthread_watchdog;
 	unsigned		wthread_stats_rate;
 	ssize_t			wthread_stacksize;
 	unsigned		wthread_queue_limit;
diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index 14bceac9e..25b6efba0 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -148,6 +148,15 @@ struct parspec WRK_parspec[] = {
 		"for at least this long, will be destroyed.",
 		EXPERIMENTAL | DELAYED_EFFECT,
 		"300", "seconds" },
+	{ "thread_pool_watchdog",
+		tweak_timeout, &mgt_param.wthread_watchdog,
+		"0.1", NULL,
+		"Thread queue stuck watchdog.\n"
+		"\n"
+		"If no queued work have been released for this long,"
+		" the worker process panics itself.",
+		EXPERIMENTAL,
+		"10", "seconds" },
 	{ "thread_pool_destroy_delay",
 		tweak_timeout, &mgt_param.wthread_destroy_delay,
 		"0.01", NULL,
diff --git a/bin/varnishtest/tests/r02418.vtc b/bin/varnishtest/tests/r02418.vtc
new file mode 100644
index 000000000..a03ca546c
--- /dev/null
+++ b/bin/varnishtest/tests/r02418.vtc
@@ -0,0 +1,72 @@
+varnishtest "h2 queuing deadlock"
+
+barrier b1 cond 2
+
+# A reserve of 1 thread in a pool of 3 leaves a maximum
+# of 2 running sessions, the streams will be queued (except
+# stream 0 that is part of the h2 session).
+
+varnish v1 -cliok "param.set thread_pools 1"
+varnish v1 -cliok "param.set thread_pool_min 3"
+varnish v1 -cliok "param.set thread_pool_max 3"
+varnish v1 -cliok "param.set thread_pool_reserve 1"
+varnish v1 -cliok "param.set thread_pool_watchdog 5"
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.set feature +no_coredump"
+
+varnish v1 -vcl {
+	backend b1 { .host = "${bad_ip}"; }
+	sub vcl_recv {
+		return (synth(200));
+	}
+} -start
+
+logexpect l1 -v v1 -g raw {
+	expect * *	Error	"Pool Herder: Queue does not move*"
+} -start
+
+# Starve the pool with h2 sessions
+
+client c1 {
+	txpri
+	stream 0 rxsettings -run
+
+	barrier b1 sync
+
+	stream 1 {
+		txreq
+		# can't be scheduled, don't rx
+	} -run
+} -start
+
+client c2 {
+	txpri
+	stream 0 rxsettings -run
+
+	barrier b1 sync
+
+	stream 1 {
+		txreq
+		# can't be scheduled, don't rx
+	} -run
+} -start
+
+client c1 -wait
+client c2 -wait
+
+varnish v1 -vsl_catchup
+
+# At this point c1 and c2 closed their connections
+
+client c3 {
+	txreq
+	delay 10
+} -run
+
+logexpect l1 -wait
+
+varnish v1 -cliok panic.show
+varnish v1 -cliok panic.clear
+
+varnish v1 -expectexit 0x20
+
diff --git a/include/tbl/params.h b/include/tbl/params.h
index e583b3977..7bebdeb4b 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -1195,6 +1195,24 @@ PARAM(
 	/* func */	NULL
 )
 
+/* actual location mgt_pool.c */
+PARAM(
+	/* name */	thread_pool_watchdog,
+	/* typ */	timeout,
+	/* min */	"0.1",
+	/* max */	NULL,
+	/* default */	"10.000",
+	/* units */	"seconds",
+	/* flags */	EXPERIMENTAL,
+	/* s-text */
+	"Thread queue stuck watchdog.\n"
+	"\n"
+	"If no queued work have been released for this long,"
+	" the worker process panics itself.",
+	/* l-text */	"",
+	/* func */	NULL
+)
+
 /* actual location mgt_pool.c */
 PARAM(
 	/* name */	thread_pool_destroy_delay,

From phk at FreeBSD.org  Wed Oct  3 10:17:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed,  3 Oct 2018 10:17:10 +0000 (UTC)
Subject: [master] 4e7e34994 Turn enum h2_stream_e into a const struct which
 knows its own name.
Message-ID: <20181003101710.2C65FA6CFE@lists.varnish-cache.org>


commit 4e7e349949d7305dd8d21539b128b0225f7357b7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 3 08:31:17 2018 +0000

    Turn enum h2_stream_e into a const struct which knows its own name.

diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h
index 70e31d14e..938f0d6a4 100644
--- a/bin/varnishd/http2/cache_http2.h
+++ b/bin/varnishd/http2/cache_http2.h
@@ -104,12 +104,14 @@ struct h2_setting_s {
 
 /**********************************************************************/
 
-enum h2_stream_e {
-	H2_STREAM__DUMMY = -1,
-#define H2_STREAM(U,s,d) H2_S_##U,
-#include "tbl/h2_stream.h"
+struct h2_stream_s {
+	const char	*name;
+	const char	*desc;
 };
 
+#define H2_STREAM(u,s,d) extern const struct h2_stream_s H2_S_##u[1];
+#include "tbl/h2_stream.h"
+
 #define H2_FRAME_FLAGS(l,u,v)   extern const uint8_t H2FF_##u;
 #include "tbl/h2_frames.h"
 
@@ -118,7 +120,7 @@ struct h2_req {
 #define H2_REQ_MAGIC			0x03411584
 	uint32_t			stream;
 	int				scheduled;
-	enum h2_stream_e		state;
+	const struct h2_stream_s	*state;
 	struct h2_sess			*h2sess;
 	struct req			*req;
 	double				t_send;
diff --git a/bin/varnishd/http2/cache_http2_panic.c b/bin/varnishd/http2/cache_http2_panic.c
index 9e80eb67b..640cdbe53 100644
--- a/bin/varnishd/http2/cache_http2_panic.c
+++ b/bin/varnishd/http2/cache_http2_panic.c
@@ -50,13 +50,11 @@ h2_sess_panic(struct vsb *vsb, const struct sess *sp)
 	VTAILQ_FOREACH(r2, &h2->streams, list) {
 		PAN_CheckMagic(vsb, r2, H2_REQ_MAGIC);
 		VSB_printf(vsb, "0x%08x", r2->stream);
-		switch (r2->state) {
-#define H2_STREAM(U,sd,d) case H2_S_##U: VSB_printf(vsb, " %-6s", sd); break;
-#include <tbl/h2_stream.h>
-		default:
-			VSB_printf(vsb, " State %d", r2->state);
-			break;
-		}
+		if (r2->state == NULL)
+			VSB_printf(vsb, " State NULL");
+		else
+			VSB_printf(vsb, " State %s (%s)",
+			    r2->state->name, r2->state->desc);
 		VSB_printf(vsb, "\n");
 	}
 	VSB_indent(vsb, -2);
diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 070bf2cad..4d4034af5 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -60,6 +60,10 @@ static const struct h2_error_s H2NN_ERROR[1] = {{
 	1
 }};
 
+#define H2_STREAM(u,s,d) const struct h2_stream_s H2_S_##u[1] = {{ s, d }};
+#include "tbl/h2_stream.h"
+
+
 enum h2frame {
 #define H2_FRAME(l,u,t,f,...)	H2F_##u = t,
 #include "tbl/h2_frames.h"
@@ -213,7 +217,8 @@ h2_kill_req(struct worker *wrk, const struct h2_sess *h2,
 	ASSERT_RXTHR(h2);
 	AN(h2e);
 	Lck_Lock(&h2->sess->mtx);
-	VSLb(h2->vsl, SLT_Debug, "KILL st=%u state=%d", r2->stream, r2->state);
+	VSLb(h2->vsl, SLT_Debug, "KILL st=%u state=%s",
+	    r2->stream, r2->state ? r2->state->name : "NULL");
 	if (r2->error == NULL)
 		r2->error = h2e;
 	if (r2->scheduled) {
@@ -424,18 +429,14 @@ h2_win_adjust(const struct h2_sess *h2, uint32_t oldval, uint32_t newval)
 		CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
 		if (r2 == h2->req0)
 			continue; // rfc7540,l,2699,2699
-		switch (r2->state) {
-		case H2_S_IDLE:
-		case H2_S_OPEN:
-		case H2_S_CLOS_REM:
+		if (r2->state == H2_S_IDLE ||
+		    r2->state == H2_S_OPEN ||
+		    r2->state == H2_S_CLOS_REM) {
 			/*
 			 * We allow a window to go negative, as per
 			 * rfc7540,l,2676,2680
 			 */
 			r2->t_window += (int64_t)newval - oldval;
-			break;
-		default:
-			break;
 		}
 	}
 }
@@ -1002,38 +1003,28 @@ h2_sweep(struct worker *wrk, struct h2_sess *h2)
 	VTAILQ_FOREACH_SAFE(r2, &h2->streams, list, r22) {
 		if (r2 == h2->req0) {
 			assert (r2->state == H2_S_IDLE);
-			continue;
-		}
-		switch (r2->state) {
-		case H2_S_CLOSED:
+		} else if (r2->state == H2_S_CLOSED) {
 			if (!r2->scheduled)
 				h2_del_req(wrk, r2);
-			break;
-		case H2_S_CLOS_REM:
+		} else if (r2->state == H2_S_CLOS_REM) {
 			if (!r2->scheduled) {
 				h2_tx_rst(wrk, h2, h2->req0, r2->stream,
 				    H2SE_REFUSED_STREAM);
 				h2_del_req(wrk, r2);
-				continue;
-			}
-			/* FALLTHROUGH */
-		case H2_S_CLOS_LOC:
-		case H2_S_OPEN:
-			if (h2_stream_tmo(h2, r2)) {
+			} else if (h2_stream_tmo(h2, r2)) {
 				tmo = 1;
-				continue;
 			}
-			break;
-		case H2_S_IDLE:
+		} else if (r2->state == H2_S_CLOS_LOC ||
+			   r2->state == H2_S_OPEN) {
+			if (h2_stream_tmo(h2, r2))
+				tmo = 1;
+		} else if(r2->state == H2_S_IDLE) {
 			/* This stream was created from receiving a
 			 * PRIORITY frame, and should not be counted
 			 * as an active stream keeping the connection
 			 * open. */
 			AZ(r2->scheduled);
 			nprio++;
-			break;
-		default:
-			break;
 		}
 	}
 	if (tmo)
diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c
index ce114d390..f38ef6bfd 100644
--- a/bin/varnishd/http2/cache_http2_session.c
+++ b/bin/varnishd/http2/cache_http2_session.c
@@ -420,8 +420,8 @@ h2_new_session(struct worker *wrk, void *arg)
 			break;
 		Lck_Lock(&h2->sess->mtx);
 		VTAILQ_FOREACH(r2, &h2->streams, list)
-			VSLb(h2->vsl, SLT_Debug, "ST %u %d",
-			    r2->stream, r2->state);
+			VSLb(h2->vsl, SLT_Debug, "ST %u %s",
+			    r2->stream, r2->state ? r2->state->name : "NULL");
 		(void)Lck_CondWait(h2->cond, &h2->sess->mtx, VTIM_real() + .1);
 		Lck_Unlock(&h2->sess->mtx);
 	}

From dridi at varni.sh  Wed Oct  3 12:37:48 2018
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Wed, 3 Oct 2018 14:37:48 +0200
Subject: [master] ddac7e41b Add a watchdog to worker pools.
In-Reply-To: <20181003101710.4E2B1A6D00@lists.varnish-cache.org>
References: <20181003101710.4E2B1A6D00@lists.varnish-cache.org>
Message-ID: <CABoVN9D2ZjJXuUWNcoagXTs8MLg+5rG-NgnPzuMtQviBewvDyA@mail.gmail.com>

On Wed, Oct 3, 2018 at 12:17 PM Poul-Henning Kamp <phk at freebsd.org> wrote:
>
>
> commit ddac7e41b64af09662305d07ef3be0376d6e2021
> Author: Poul-Henning Kamp <phk at FreeBSD.org>
> Date:   Wed Oct 3 10:12:59 2018 +0000
>
>     Add a watchdog to worker pools.
<snip>
> diff --git a/bin/varnishd/cache/cache_pool.h b/bin/varnishd/cache/cache_pool.h
> index e7a6a618b..9902e32ad 100644
> --- a/bin/varnishd/cache/cache_pool.h
> +++ b/bin/varnishd/cache/cache_pool.h
> @@ -53,6 +53,7 @@ struct pool {
>         uintmax_t                       sdropped;
>         uintmax_t                       rdropped;
>         uintmax_t                       nqueued;
> +       uintmax_t                       ndequeued;
>         struct VSC_main_wrk             *a_stat;
>         struct VSC_main_wrk             *b_stat;
>
> diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
> index 589ce68cb..f7f95cfc5 100644
> --- a/bin/varnishd/cache/cache_wrk.c
> +++ b/bin/varnishd/cache/cache_wrk.c
> @@ -341,6 +341,7 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk)
>                         tp = VTAILQ_FIRST(&pp->queues[i]);
>                         if (tp != NULL) {
>                                 pp->lqueue--;
> +                               pp->ndequeued--;

I thought we'd increment pp->ndequeued above based on the field's name.

>                                 VTAILQ_REMOVE(&pp->queues[i], tp, list);
>                                 break;
>                         }
> @@ -487,6 +488,8 @@ pool_herder(void *priv)
>         struct worker *wrk;
>         double delay;
>         int wthread_min;
> +       uintmax_t dq = (1ULL << 31);
> +       double dqt;

If we increment pp->ndequeued and let dq start at zero, and initialize
dqt here [...]

>         CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
>
> @@ -494,6 +497,29 @@ pool_herder(void *priv)
>         THR_Init();
>
>         while (!pp->die || pp->nthr > 0) {
> +               /*
> +                * If the worker pool is configured too small, we can
> +                * end up deadlocking it (see #2418 for details).
> +                *
> +                * Recovering from this would require a lot of complicated
> +                * code, and fundamentally, either people configured their
> +                * pools wrong, in which case we want them to notice, or
> +                * they are under DoS, in which case recovering gracefully
> +                * is unlikely be a major improvement.
> +                *
> +                * Instead we implement a watchdog and kill the worker if
> +                * nothing has been dequeued for that long.
> +                */
> +               if (dq != pp->ndequeued) {
> +                       dq = pp->ndequeued;
> +                       dqt = VTIM_real();

[...] we could log the dequeue delta (both count and time) to let
users know how to sensibly set the parameter.

> +               } else if (pp->lqueue &&
> +                   VTIM_real() - dqt > cache_param->wthread_watchdog) {
> +                       VSL(SLT_Error, 0,
> +                          "Pool Herder: Queue does not move ql=%u dt=%f",
> +                          pp->lqueue, VTIM_real() - dqt);
> +                       WRONG("Worker Pool Queue does not move");
> +               }
>                 wthread_min = cache_param->wthread_min;
>                 if (pp->die)
>                         wthread_min = 0;
<snip>

Dridi

From phk at FreeBSD.org  Wed Oct  3 13:35:17 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed,  3 Oct 2018 13:35:17 +0000 (UTC)
Subject: [master] 883274844 It's nice to know I'm still smarter than gcc
Message-ID: <20181003133517.39A13AC81A@lists.varnish-cache.org>


commit 88327484407689fba7131ed31a9d400f96d0e3b8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 3 13:34:33 2018 +0000

    It's nice to know I'm still smarter than gcc

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index f7f95cfc5..3d4411cb4 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -489,7 +489,7 @@ pool_herder(void *priv)
 	double delay;
 	int wthread_min;
 	uintmax_t dq = (1ULL << 31);
-	double dqt;
+	double dqt = 0;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 

From phk at FreeBSD.org  Wed Oct  3 13:55:11 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed,  3 Oct 2018 13:55:11 +0000 (UTC)
Subject: [master] ff6c1ce39 Revert "Turn enum h2_stream_e into a const struct
 which knows its own name."
Message-ID: <20181003135511.F18E2ACEE8@lists.varnish-cache.org>


commit ff6c1ce396c8e5da579fe62cf7f030e81cbbc8b3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 3 13:53:47 2018 +0000

    Revert "Turn enum h2_stream_e into a const struct which knows its own name."
    
    This reverts commit 4e7e349949d7305dd8d21539b128b0225f7357b7.

diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h
index 938f0d6a4..70e31d14e 100644
--- a/bin/varnishd/http2/cache_http2.h
+++ b/bin/varnishd/http2/cache_http2.h
@@ -104,13 +104,11 @@ struct h2_setting_s {
 
 /**********************************************************************/
 
-struct h2_stream_s {
-	const char	*name;
-	const char	*desc;
-};
-
-#define H2_STREAM(u,s,d) extern const struct h2_stream_s H2_S_##u[1];
+enum h2_stream_e {
+	H2_STREAM__DUMMY = -1,
+#define H2_STREAM(U,s,d) H2_S_##U,
 #include "tbl/h2_stream.h"
+};
 
 #define H2_FRAME_FLAGS(l,u,v)   extern const uint8_t H2FF_##u;
 #include "tbl/h2_frames.h"
@@ -120,7 +118,7 @@ struct h2_req {
 #define H2_REQ_MAGIC			0x03411584
 	uint32_t			stream;
 	int				scheduled;
-	const struct h2_stream_s	*state;
+	enum h2_stream_e		state;
 	struct h2_sess			*h2sess;
 	struct req			*req;
 	double				t_send;
diff --git a/bin/varnishd/http2/cache_http2_panic.c b/bin/varnishd/http2/cache_http2_panic.c
index 640cdbe53..9e80eb67b 100644
--- a/bin/varnishd/http2/cache_http2_panic.c
+++ b/bin/varnishd/http2/cache_http2_panic.c
@@ -50,11 +50,13 @@ h2_sess_panic(struct vsb *vsb, const struct sess *sp)
 	VTAILQ_FOREACH(r2, &h2->streams, list) {
 		PAN_CheckMagic(vsb, r2, H2_REQ_MAGIC);
 		VSB_printf(vsb, "0x%08x", r2->stream);
-		if (r2->state == NULL)
-			VSB_printf(vsb, " State NULL");
-		else
-			VSB_printf(vsb, " State %s (%s)",
-			    r2->state->name, r2->state->desc);
+		switch (r2->state) {
+#define H2_STREAM(U,sd,d) case H2_S_##U: VSB_printf(vsb, " %-6s", sd); break;
+#include <tbl/h2_stream.h>
+		default:
+			VSB_printf(vsb, " State %d", r2->state);
+			break;
+		}
 		VSB_printf(vsb, "\n");
 	}
 	VSB_indent(vsb, -2);
diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 4d4034af5..070bf2cad 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -60,10 +60,6 @@ static const struct h2_error_s H2NN_ERROR[1] = {{
 	1
 }};
 
-#define H2_STREAM(u,s,d) const struct h2_stream_s H2_S_##u[1] = {{ s, d }};
-#include "tbl/h2_stream.h"
-
-
 enum h2frame {
 #define H2_FRAME(l,u,t,f,...)	H2F_##u = t,
 #include "tbl/h2_frames.h"
@@ -217,8 +213,7 @@ h2_kill_req(struct worker *wrk, const struct h2_sess *h2,
 	ASSERT_RXTHR(h2);
 	AN(h2e);
 	Lck_Lock(&h2->sess->mtx);
-	VSLb(h2->vsl, SLT_Debug, "KILL st=%u state=%s",
-	    r2->stream, r2->state ? r2->state->name : "NULL");
+	VSLb(h2->vsl, SLT_Debug, "KILL st=%u state=%d", r2->stream, r2->state);
 	if (r2->error == NULL)
 		r2->error = h2e;
 	if (r2->scheduled) {
@@ -429,14 +424,18 @@ h2_win_adjust(const struct h2_sess *h2, uint32_t oldval, uint32_t newval)
 		CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
 		if (r2 == h2->req0)
 			continue; // rfc7540,l,2699,2699
-		if (r2->state == H2_S_IDLE ||
-		    r2->state == H2_S_OPEN ||
-		    r2->state == H2_S_CLOS_REM) {
+		switch (r2->state) {
+		case H2_S_IDLE:
+		case H2_S_OPEN:
+		case H2_S_CLOS_REM:
 			/*
 			 * We allow a window to go negative, as per
 			 * rfc7540,l,2676,2680
 			 */
 			r2->t_window += (int64_t)newval - oldval;
+			break;
+		default:
+			break;
 		}
 	}
 }
@@ -1003,28 +1002,38 @@ h2_sweep(struct worker *wrk, struct h2_sess *h2)
 	VTAILQ_FOREACH_SAFE(r2, &h2->streams, list, r22) {
 		if (r2 == h2->req0) {
 			assert (r2->state == H2_S_IDLE);
-		} else if (r2->state == H2_S_CLOSED) {
+			continue;
+		}
+		switch (r2->state) {
+		case H2_S_CLOSED:
 			if (!r2->scheduled)
 				h2_del_req(wrk, r2);
-		} else if (r2->state == H2_S_CLOS_REM) {
+			break;
+		case H2_S_CLOS_REM:
 			if (!r2->scheduled) {
 				h2_tx_rst(wrk, h2, h2->req0, r2->stream,
 				    H2SE_REFUSED_STREAM);
 				h2_del_req(wrk, r2);
-			} else if (h2_stream_tmo(h2, r2)) {
-				tmo = 1;
+				continue;
 			}
-		} else if (r2->state == H2_S_CLOS_LOC ||
-			   r2->state == H2_S_OPEN) {
-			if (h2_stream_tmo(h2, r2))
+			/* FALLTHROUGH */
+		case H2_S_CLOS_LOC:
+		case H2_S_OPEN:
+			if (h2_stream_tmo(h2, r2)) {
 				tmo = 1;
-		} else if(r2->state == H2_S_IDLE) {
+				continue;
+			}
+			break;
+		case H2_S_IDLE:
 			/* This stream was created from receiving a
 			 * PRIORITY frame, and should not be counted
 			 * as an active stream keeping the connection
 			 * open. */
 			AZ(r2->scheduled);
 			nprio++;
+			break;
+		default:
+			break;
 		}
 	}
 	if (tmo)
diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c
index f38ef6bfd..ce114d390 100644
--- a/bin/varnishd/http2/cache_http2_session.c
+++ b/bin/varnishd/http2/cache_http2_session.c
@@ -420,8 +420,8 @@ h2_new_session(struct worker *wrk, void *arg)
 			break;
 		Lck_Lock(&h2->sess->mtx);
 		VTAILQ_FOREACH(r2, &h2->streams, list)
-			VSLb(h2->vsl, SLT_Debug, "ST %u %s",
-			    r2->stream, r2->state ? r2->state->name : "NULL");
+			VSLb(h2->vsl, SLT_Debug, "ST %u %d",
+			    r2->stream, r2->state);
 		(void)Lck_CondWait(h2->cond, &h2->sess->mtx, VTIM_real() + .1);
 		Lck_Unlock(&h2->sess->mtx);
 	}

From nils.goroll at uplex.de  Thu Oct  4 06:01:14 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 4 Oct 2018 08:01:14 +0200
Subject: [master] ddac7e41b Add a watchdog to worker pools.
In-Reply-To: <20181003101710.4E2B1A6D00@lists.varnish-cache.org>
References: <20181003101710.4E2B1A6D00@lists.varnish-cache.org>
Message-ID: <9d359f86-d238-b243-a481-8720e3e0fe8d@uplex.de>

I wonder if/why? we can't solve the root cause along the lines of what we did in
42959df8c23914e7124ad888e5de7d061bbc72e5:

We could reserve a number of threads for requests as we already did for backend
requests. Or, knowing that each h2 session requires n >= 1 request threads, we
could reserve n threads for each session we schedule.

Nils

-- 

** * * UPLEX - Nils Goroll Systemoptimierung

Scheffelstra?e 32
22301 Hamburg

tel +49 40 28805731
mob +49 170 2723133
fax +49 40 42949753

xmpp://slink at jabber.int.uplex.de/

http://uplex.de/

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 488 bytes
Desc: OpenPGP digital signature
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20181004/996f26bb/attachment.bin>

From phk at phk.freebsd.dk  Thu Oct  4 06:35:33 2018
From: phk at phk.freebsd.dk (Poul-Henning Kamp)
Date: Thu, 04 Oct 2018 06:35:33 +0000
Subject: [master] ddac7e41b Add a watchdog to worker pools.
In-Reply-To: <9d359f86-d238-b243-a481-8720e3e0fe8d@uplex.de>
References: <20181003101710.4E2B1A6D00@lists.varnish-cache.org>
 <9d359f86-d238-b243-a481-8720e3e0fe8d@uplex.de>
Message-ID: <62619.1538634933@critter.freebsd.dk>

--------
In message <9d359f86-d238-b243-a481-8720e3e0fe8d at uplex.de>, Nils Goroll writes:

>We could reserve a number of threads for [...]

Adding magic reserve-regiments of threads does not solve the problem:

If you schedule a request from the pool-queue, the only situation
where you can be 100% sure it is not going to sleep the thread for
a long time, is if the clients TCP connection is already dead.

In theory we could add magic code to purge the pool queue of requests
where the client TCP connection is dead, but that is not going to free
up any threads, it just makes the queue shorter.

The only way to truly solve this, is to have (very complex) code
to deduce a similarity between the requests which have tied up
threads and then a) blacklist any future such requests and b) hunt
the ones already in progress down and close their client TCP
connection.

That is not happening, unless very heavy real-world evidence emerges
that it would solve a big operational problem for a lot of users.

-- 
Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
phk at FreeBSD.ORG         | TCP/IP since RFC 956
FreeBSD committer       | BSD since 4.3-tahoe    
Never attribute to malice what can adequately be explained by incompetence.

From phk at FreeBSD.org  Thu Oct  4 07:19:15 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu,  4 Oct 2018 07:19:15 +0000 (UTC)
Subject: [master] 5dc531f73 Don't poll VSM_Status() while there is work to do
 and no interruptions.
Message-ID: <20181004071915.4416B91788@lists.varnish-cache.org>


commit 5dc531f7329016799e5a16bbe1ccc64bcce6e0e0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 4 07:17:58 2018 +0000

    Don't poll VSM_Status() while there is work to do and no interruptions.
    
    Fixes   #2788

diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c
index 03cd9f07a..b02644a81 100644
--- a/lib/libvarnishapi/vut.c
+++ b/lib/libvarnishapi/vut.c
@@ -420,7 +420,10 @@ VUT_Main(struct VUT *vut)
 			AZ(c);
 		}
 
-		i = VSLQ_Dispatch(vut->vslq, vut_dispatch, vut);
+		do
+			i = VSLQ_Dispatch(vut->vslq, vut_dispatch, vut);
+		while (i == vsl_more && !vut->sighup && !vut->sigusr1);
+
 		if (i == vsl_more)
 			continue;
 		else if (i == vsl_end) {

From nils.goroll at uplex.de  Thu Oct  4 08:17:56 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 4 Oct 2018 10:17:56 +0200
Subject: [master] ddac7e41b Add a watchdog to worker pools.
In-Reply-To: <62619.1538634933@critter.freebsd.dk>
References: <20181003101710.4E2B1A6D00@lists.varnish-cache.org>
 <9d359f86-d238-b243-a481-8720e3e0fe8d@uplex.de>
 <62619.1538634933@critter.freebsd.dk>
Message-ID: <96e80c16-edf9-9021-2fd6-74137df9dcb0@uplex.de>

On 04/10/2018 08:35, Poul-Henning Kamp wrote:
> If you schedule a request from the pool-queue, the only situation
> where you can be 100% sure it is not going to sleep the thread for
> a long time

but a long time is different to indefinitely and a request being worked on is
different to one not being scheduled in the first place.

I am not against the emergency break in form of the pool watchdog, but I think
we should make use of the knowledge that a h2 session will require at least one
worker for a request.

Nils

-- 

** * * UPLEX - Nils Goroll Systemoptimierung

Scheffelstra?e 32
22301 Hamburg

tel +49 40 28805731
mob +49 170 2723133
fax +49 40 42949753

xmpp://slink at jabber.int.uplex.de/

http://uplex.de/

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 488 bytes
Desc: OpenPGP digital signature
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20181004/2db1500a/attachment.bin>

From nils.goroll at uplex.de  Thu Oct  4 08:21:16 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 4 Oct 2018 10:21:16 +0200
Subject: [master] ddac7e41b Add a watchdog to worker pools.
In-Reply-To: <96e80c16-edf9-9021-2fd6-74137df9dcb0@uplex.de>
References: <20181003101710.4E2B1A6D00@lists.varnish-cache.org>
 <9d359f86-d238-b243-a481-8720e3e0fe8d@uplex.de>
 <62619.1538634933@critter.freebsd.dk>
 <96e80c16-edf9-9021-2fd6-74137df9dcb0@uplex.de>
Message-ID: <d007b9cd-796f-63e3-ba99-16a55eb29119@uplex.de>

> emergency break
emergency brake

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 488 bytes
Desc: OpenPGP digital signature
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20181004/cb27b157/attachment.bin>

From nils.goroll at uplex.de  Thu Oct  4 14:50:13 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu,  4 Oct 2018 14:50:13 +0000 (UTC)
Subject: [master] 5ca0b5d3e restore vsl binary compatibility
Message-ID: <20181004145014.0544DA0D38@lists.varnish-cache.org>


commit 5ca0b5d3e749be81eec39988d7db7a0fdf13e73b
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Oct 4 16:45:27 2018 +0200

    restore vsl binary compatibility
    
    be69499219db0704a136046369884531ee20bc01 unnecessarily changed the
    values of most vsl tag enums and thus introduced an incompatibility
    with logs written with previous code.
    
    Fixes #2790

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 294da9431..49b27cf9b 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -65,20 +65,6 @@ SLTM(CLI, 0, "CLI communication",
 
 /*---------------------------------------------------------------------*/
 
-SLTM(SessError, 0, "Client connection accept failed",
-	"Accepting a client connection has failed.\n\n"
-	"The format is::\n\n"
-	"\t%s %s %s %d %d %s\n"
-	"\t|  |  |  |  |  |\n"
-	"\t|  |  |  |  |  +- Detailed error message\n"
-	"\t|  |  |  |  +---- Error Number (errno) from accept(2)\n"
-	"\t|  |  |  +------- File descriptor number\n"
-	"\t|  |  +---------- Local TCP port / 0 for UDS\n"
-	"\t|  +------------- Local IPv4/6 address / 0.0.0.0 for UDS\n"
-	"\t+---------------- Socket name (from -a argument)\n"
-	"\n"
-)
-
 SLTM(SessOpen, 0, "Client connection opened",
 	"The first record for a client connection, with the socket-endpoints"
 	" of the connection.\n\n"
@@ -642,6 +628,20 @@ SLTM(Filters, 0, "Body filters",
 	"List of filters applied to the body"
 )
 
+SLTM(SessError, 0, "Client connection accept failed",
+	"Accepting a client connection has failed.\n\n"
+	"The format is::\n\n"
+	"\t%s %s %s %d %d %s\n"
+	"\t|  |  |  |  |  |\n"
+	"\t|  |  |  |  |  +- Detailed error message\n"
+	"\t|  |  |  |  +---- Error Number (errno) from accept(2)\n"
+	"\t|  |  |  +------- File descriptor number\n"
+	"\t|  |  +---------- Local TCP port / 0 for UDS\n"
+	"\t|  +------------- Local IPv4/6 address / 0.0.0.0 for UDS\n"
+	"\t+---------------- Socket name (from -a argument)\n"
+	"\n"
+)
+
 #undef NODEF_NOTICE
 #undef SLTM
 

From nils.goroll at uplex.de  Thu Oct  4 15:32:08 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu,  4 Oct 2018 15:32:08 +0000 (UTC)
Subject: [master] ee6811321 changelog tlc
Message-ID: <20181004153208.6C763A1B5D@lists.varnish-cache.org>


commit ee6811321ca04e5d96a1edeb0a905c84034f309d
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Oct 4 17:28:09 2018 +0200

    changelog tlc

diff --git a/doc/changes.rst b/doc/changes.rst
index 109bc066f..581ee514b 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -30,6 +30,46 @@ release process.
 Varnish Cache trunk (ongoing)
 =============================
 
+* fixed ``varnishhist`` display error (2780_)
+
+* ``libvarnish``: ``VRT_VSA_GetPtr`` renamed to ``VSA_GetPtr``
+
+* Improve accuracy of statistics (VSC)
+
+* In ``Error: out of workspace`` log entries, the workspace name is
+  now reported in lowercase
+
+* Adjust code generator python tools to python 3 and prefer python 3
+  over python 2 where available
+
+* fix some stats metrics (vsc) which were wrongly marked as _gauge_
+
+* fix ``varnishd -I`` (2782_)
+
+* fix ``varnishstat -f`` in curses mode (interactively, without
+  ``-1``, 2787_)
+
+* Handle an out-of-workspace condition in HTTP/2 delivery more
+  gracefully (2589_)
+
+* added a thread pool watchdog which will restart the worker process
+  if scheduling tasks onto worker threads appears stuck. The new
+  parameter ``thread_pool_watchdog`` configures it. (2418_)
+
+* Improved varnish log client performance (2788_)
+
+* Fixed regression introduced just before 6.1.0 release which caused
+  an unnecessary incompatibility with VSL files written by previous
+  versions. (2790_)
+
+.. _2780: https://github.com/varnishcache/varnish-cache/issues/2780
+.. _2782: https://github.com/varnishcache/varnish-cache/issues/2782
+.. _2787: https://github.com/varnishcache/varnish-cache/issues/2787
+.. _2589: https://github.com/varnishcache/varnish-cache/issues/2589
+.. _2418: https://github.com/varnishcache/varnish-cache/issues/2418
+.. _2788: https://github.com/varnishcache/varnish-cache/issues/2788
+.. _2790: https://github.com/varnishcache/varnish-cache/issues/2790
+
 ================================
 Varnish Cache 6.1.0 (2018-09-17)
 ================================

From nils.goroll at uplex.de  Thu Oct  4 18:25:07 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu,  4 Oct 2018 18:25:07 +0000 (UTC)
Subject: [master] 8797fd3df debloat the vtim test
Message-ID: <20181004182507.8515BA50D1@lists.varnish-cache.org>


commit 8797fd3dfe902351f3977f75812d33bc3c55743b
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Oct 4 20:24:10 2018 +0200

    debloat the vtim test

diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index 4bef5e711..a9fbc7cb2 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -529,12 +529,26 @@ bench()
 	    e - s, i, 1e9 * (e - s) / i, t);
 }
 
+void
+parse_check(time_t t, const char *s)
+{
+	double tt;
+	char buf[BUFSIZ];
+
+	tt = VTIM_parse(s);
+	if (tt != t) {
+		VTIM_format(tt, buf);
+		printf("  fm: %12jd <%s>\n", (intmax_t)t, s);
+		printf("  to: %12.0f <%s>\n", tt, buf);
+		exit(2);
+	}
+}
+
 int
 main(int argc, char **argv)
 {
 	time_t t;
 	struct tm tm;
-	double tt;
 	char buf[BUFSIZ];
 	char buf1[BUFSIZ];
 
@@ -553,41 +567,17 @@ main(int argc, char **argv)
 			    buf1, buf, (intmax_t)t);
 			exit(2);
 		}
-		tt = VTIM_parse(buf1);
-		if (tt != t) {
-			VTIM_format(tt, buf);
-			printf("  fm: %12jd <%s>\n", (intmax_t)t, buf1);
-			printf("  to: %12.0f <%s>\n", tt, buf);
-			exit(2);
-		}
+		parse_check(t, buf1);
 
 		strftime(buf1, sizeof buf1, "%a %b %e %T %Y", &tm);
-		tt = VTIM_parse(buf1);
-		if (tt != t) {
-			VTIM_format(tt, buf);
-			printf("  fm: %12jd <%s>\n", (intmax_t)t, buf1);
-			printf("  to: %12.0f <%s>\n", tt, buf);
-			exit(2);
-		}
+		parse_check(t, buf1);
 
 		strftime(buf1, sizeof buf1, "%Y-%m-%dT%T", &tm);
-		tt = VTIM_parse(buf1);
-		if (tt != t) {
-			VTIM_format(tt, buf);
-			printf("  fm: %12jd <%s>\n", (intmax_t)t, buf1);
-			printf("  to: %12.0f <%s>\n", tt, buf);
-			exit(2);
-		}
+		parse_check(t, buf1);
 
 		if (tm.tm_year >= 69 && tm.tm_year < 169) {
 			strftime(buf1, sizeof buf1, "%A, %d-%b-%y %T GMT", &tm);
-			tt = VTIM_parse(buf1);
-			if (tt != t) {
-				VTIM_format(tt, buf);
-				printf("  fm: %12jd <%s>\n", (intmax_t)t, buf1);
-				printf("  to: %12.0f <%s>\n", tt, buf);
-				exit(2);
-			}
+			parse_check(t, buf1);
 		}
 	}
 

From nils.goroll at uplex.de  Fri Oct  5 07:58:11 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri,  5 Oct 2018 07:58:11 +0000 (UTC)
Subject: [master] 222a1f597 trivial vtim printf benchmark
Message-ID: <20181005075811.881F76FB7@lists.varnish-cache.org>


commit 222a1f597a8c85c0a562154ea70a315fef1033d0
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Oct 5 09:56:27 2018 +0200

    trivial vtim printf benchmark

diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index a9fbc7cb2..f9de6244e 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -511,6 +511,7 @@ bench()
 {
 	double s, e, t;
 	int i;
+	char buf[64];
 
 	t = 0;
 	s = VTIM_real();
@@ -527,6 +528,17 @@ bench()
 	e = VTIM_real();
 	printf("mono: %fs / %d = %fns - tst val %f\n",
 	    e - s, i, 1e9 * (e - s) / i, t);
+
+	t = 0;
+	s = VTIM_mono();
+	for (i=0; i<100000; i++) {
+		snprintf(buf, sizeof(buf), "%.6f", s);
+		t += buf[4];
+	}
+	e = VTIM_mono();
+	printf("%s\n", buf);
+	printf("printf: %fs / %d = %fns - tst val %f\n",
+	    e - s, i, 1e9 * (e - s) / i, t);
 }
 
 void

From daghf at varnish-software.com  Fri Oct  5 08:50:16 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Fri,  5 Oct 2018 08:50:16 +0000 (UTC)
Subject: [master] 9dcb4f3f1 Don't use txprio to open streams in tests
Message-ID: <20181005085016.C9F2B900E@lists.varnish-cache.org>


commit 9dcb4f3f1d3dd71d2af7a65d8e0a678bf6ee91a9
Author: Carlo Cannas <carlo94 at gmail.com>
Date:   Sun Sep 23 03:34:02 2018 +0200

    Don't use txprio to open streams in tests
    
    As per spec a client can only send a HEADERS frame to cause a stream to
    transition from the "idle" state to "open". A PRIORITY frame can be sent to an
    "idle" stream, but it will remain in that state.
    
    rfc7540,l,916,940
    https://tools.ietf.org/html/rfc7540#section-5.1
    
    To open a stream the command
    
    txreq -nostrend
    
    can be used. The -nostrend option will ensure that the stream won't transition
    to a "half-closed" state.

diff --git a/bin/varnishtest/tests/t02003.vtc b/bin/varnishtest/tests/t02003.vtc
index b355fd4b8..215171cf6 100644
--- a/bin/varnishtest/tests/t02003.vtc
+++ b/bin/varnishtest/tests/t02003.vtc
@@ -43,10 +43,10 @@ client c1 {
 		expect goaway.err == PROTOCOL_ERROR
 	} -start
 	stream 3 {
-		txprio
+		txreq
 	} -run
 	stream 1 {
-		txprio
+		txreq
 	} -run
 	stream 0 -wait
 } -run
@@ -63,13 +63,13 @@ varnish v1 -expect MEMPOOL.sess1.live == 0
 
 client c1 {
 	stream 1 {
-		txprio
+		txreq -nostrend
 		txwinup -size 0
 		rxrst
 		expect rst.err == PROTOCOL_ERROR
 	} -run
 	stream 3 {
-		txprio
+		txreq -nostrend
 		txwinup -size 0x40000000
 		txwinup -size 0x40000000
 		rxrst
@@ -81,7 +81,7 @@ client c1 {
 		expect goaway.err == FRAME_SIZE_ERROR
 	} -start
 	stream 5 {
-		txprio
+		txreq -nostrend
 		sendhex "000003 08 00 00000005"
 		delay .1
 		sendhex 01
@@ -167,7 +167,7 @@ client c1 {
 		expect goaway.laststream == 1
 	} -start
 	stream 1 {
-		txprio
+		txreq -nostrend
 		sendhex "000008 05 00 00000001 0001020304050607"
 	} -run
 	stream 0 -wait
@@ -200,7 +200,7 @@ client c1 {
 		expect goaway.laststream == 1
 	} -start
 	stream 1 {
-		txprio
+		txreq -nostrend
 		# RST wrong length
 		sendhex "000005 03 00 00000001 0000000800"
 	} -run

From daghf at varnish-software.com  Fri Oct  5 08:50:16 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Fri,  5 Oct 2018 08:50:16 +0000 (UTC)
Subject: [master] bc823a062 Move header block frame sequence check earlier
Message-ID: <20181005085016.D5B749010@lists.varnish-cache.org>


commit bc823a062d23148cb6f76a94a3c8eb27db964939
Author: Carlo Cannas <carlo94 at gmail.com>
Date:   Sun Sep 23 03:34:02 2018 +0200

    Move header block frame sequence check earlier
    
    This moves it before the new stream object creation, so we save ourselves an
    useless allocation and initialization of a stream object which would be never
    used and straight killed.
    
    This also simplifies upcoming commits.

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 070bf2cad..b106f9e36 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -923,6 +923,10 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2, h2_frame h2f)
 		if (r2->stream == h2->rxf_stream)
 			break;
 
+	if (h2->new_req != NULL &&
+	    !(r2 && h2->new_req == r2->req && h2f == H2_F_CONTINUATION))
+		return (H2CE_PROTOCOL_ERROR);	// rfc7540,l,1859,1863
+
 	if (r2 == NULL && h2f->act_sidle == 0) {
 		if (h2->rxf_stream <= h2->highest_stream)
 			return (H2CE_PROTOCOL_ERROR);	// rfc7540,l,1153,1158
@@ -940,10 +944,6 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2, h2_frame h2f)
 		AN(r2);
 	}
 
-	if (h2->new_req != NULL &&
-	    !(r2 && h2->new_req == r2->req && h2f == H2_F_CONTINUATION))
-		return (H2CE_PROTOCOL_ERROR);	// rfc7540,l,1859,1863
-
 	h2e = h2f->rxfunc(wrk, h2, r2);
 	if (h2e == 0)
 		return (0);
diff --git a/bin/varnishtest/tests/r02387.vtc b/bin/varnishtest/tests/r02387.vtc
index 34863a057..d2c9796e7 100644
--- a/bin/varnishtest/tests/r02387.vtc
+++ b/bin/varnishtest/tests/r02387.vtc
@@ -30,7 +30,7 @@ client c1 {
 	} -run
 	stream 0 {
 		rxgoaway
-		expect goaway.laststream == "3"
+		expect goaway.laststream == "1"
 		expect goaway.err == PROTOCOL_ERROR
 	} -run
 } -run

From daghf at varnish-software.com  Fri Oct  5 08:50:17 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Fri,  5 Oct 2018 08:50:17 +0000 (UTC)
Subject: [master] ff9c9e2ed Allow PRIORITY frames on closed streams
Message-ID: <20181005085017.05B469013@lists.varnish-cache.org>


commit ff9c9e2ed1f9b5e974f6a14ef659ae4a027dcde6
Author: Carlo Cannas <carlo94 at gmail.com>
Date:   Sun Sep 23 03:34:02 2018 +0200

    Allow PRIORITY frames on closed streams
    
    Currently Varnish doesn't allow PRIORITY frames to be received on closed
    streams: it treats it as a protocol violation and replies with a GOAWAY.
    
    This is not spec compliant, rfc7540 states:
    
    The PRIORITY frame can be sent for a stream in the "idle" or "closed"
    state.
    rfc7540,l,1947,1948
    
    The PRIORITY frame can be sent on a stream in any state
    rfc7540,l,1938,1938
    
    https://tools.ietf.org/html/rfc7540#section-6.3
    
    This behaviour can be triggered by real-world browsers: Chrome 69 has been
    observed sending PRIORITY frames which are received by Varnish after a stream
    has been closed (and cleaned by h2_sweep). When that happens the connection is
    closed and Chrome aborts the loading of all the resources which started to
    load on that connection.
    
    This commit solves the issue by avoiding all the stream creation code and its
    checks to be performed when a PRIORITY frame is received.
    This moves all the stream creation logic inside h2_rx_headers, the only other
    frame which is allowed on idle streams.
    
    This also fixes the concurrent streams counter and highest_stream: they should
    be updated only when a stream enters the "open" state (or "reserved" if
    Varnish used served push) but currently a PRIORITY frame on an idle stream
    affects them.
    
    https://tools.ietf.org/html/rfc7540#section-5.1.1
    rfc7540,l,1153,1156
    rfc7540,l,1193,1198
    rfc7540,l,1530,1533
    
    Fixes: #2775

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index b106f9e36..250af1806 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -382,7 +382,7 @@ h2_rx_priority(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 
 	(void)wrk;
 	ASSERT_RXTHR(h2);
-	xxxassert(r2->stream & 1);
+	(void)r2;
 	return (0);
 }
 
@@ -616,7 +616,21 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 	size_t l;
 
 	ASSERT_RXTHR(h2);
+
+	if (r2 == NULL) {
+		if (h2->rxf_stream <= h2->highest_stream)
+			return (H2CE_PROTOCOL_ERROR);	// rfc7540,l,1153,1158
+		if (h2->refcnt >= h2->local_settings.max_concurrent_streams) {
+			VSLb(h2->vsl, SLT_Debug,
+			     "H2: stream %u: Hit maximum number of "
+			     "concurrent streams", h2->rxf_stream);
+			return (H2SE_REFUSED_STREAM);	// rfc7540,l,1200,1205
+		}
+		h2->highest_stream = h2->rxf_stream;
+		r2 = h2_new_req(wrk, h2, h2->rxf_stream, NULL);
+	}
 	AN(r2);
+
 	if (r2->state != H2_S_IDLE)
 		return (H2CE_PROTOCOL_ERROR);	// XXX spec ?
 	r2->state = H2_S_OPEN;
@@ -927,23 +941,6 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2, h2_frame h2f)
 	    !(r2 && h2->new_req == r2->req && h2f == H2_F_CONTINUATION))
 		return (H2CE_PROTOCOL_ERROR);	// rfc7540,l,1859,1863
 
-	if (r2 == NULL && h2f->act_sidle == 0) {
-		if (h2->rxf_stream <= h2->highest_stream)
-			return (H2CE_PROTOCOL_ERROR);	// rfc7540,l,1153,1158
-		if (h2->refcnt >= h2->local_settings.max_concurrent_streams) {
-			VSLb(h2->vsl, SLT_Debug,
-			     "H2: stream %u: Hit maximum number of "
-			     "concurrent streams", h2->rxf_stream);
-			// rfc7540,l,1200,1205
-			h2_tx_rst(wrk, h2, h2->req0, h2->rxf_stream,
-				H2SE_REFUSED_STREAM);
-			return (0);
-		}
-		h2->highest_stream = h2->rxf_stream;
-		r2 = h2_new_req(wrk, h2, h2->rxf_stream, NULL);
-		AN(r2);
-	}
-
 	h2e = h2f->rxfunc(wrk, h2, r2);
 	if (h2e == 0)
 		return (0);
@@ -993,7 +990,6 @@ static int
 h2_sweep(struct worker *wrk, struct h2_sess *h2)
 {
 	int tmo = 0;
-	int nprio = 0;
 	struct h2_req *r2, *r22;
 
 	ASSERT_RXTHR(h2);
@@ -1025,20 +1021,18 @@ h2_sweep(struct worker *wrk, struct h2_sess *h2)
 			}
 			break;
 		case H2_S_IDLE:
-			/* This stream was created from receiving a
-			 * PRIORITY frame, and should not be counted
-			 * as an active stream keeping the connection
-			 * open. */
-			AZ(r2->scheduled);
-			nprio++;
-			break;
+			/* Current code make this unreachable: h2_new_req is
+			 * only called inside h2_rx_headers, which immediately
+			 * sets the new stream state to H2_S_OPEN */
+			/* FALLTHROUGH */
 		default:
+			WRONG("Wrong h2 stream state");
 			break;
 		}
 	}
 	if (tmo)
 		return (0);
-	return ((h2->refcnt - nprio) > 1);
+	return (h2->refcnt > 1);
 }
 
 
diff --git a/bin/varnishtest/tests/r02775.vtc b/bin/varnishtest/tests/r02775.vtc
new file mode 100644
index 000000000..de66c10d1
--- /dev/null
+++ b/bin/varnishtest/tests/r02775.vtc
@@ -0,0 +1,23 @@
+varnishtest "Regression test for #2775: allow PRIORITY on closed stream"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {} -start
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.set debug +syncvsl"
+
+client c1 {
+	stream 1 {
+		txreq
+		rxresp
+
+		txprio
+	} -run
+	stream 3 {
+		txreq
+		rxresp
+	} -run
+} -run

From dridi.boukelmoune at gmail.com  Fri Oct  5 09:19:12 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri,  5 Oct 2018 09:19:12 +0000 (UTC)
Subject: [master] aad133446 Hardening of the h2_frame_f callbacks
Message-ID: <20181005091912.968249CA4@lists.varnish-cache.org>


commit aad1334466f8255c7754da023d1a8bcca21fe146
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Oct 5 11:16:59 2018 +0200

    Hardening of the h2_frame_f callbacks
    
    And by the way, they are known as h2_rxframe_f these days!
    
    Refs #2781

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 250af1806..6caa358bf 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -275,11 +275,14 @@ h2_vsl_frame(const struct h2_sess *h2, const void *ptr, size_t len)
 /**********************************************************************
  */
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_ping(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
+	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
+	assert(r2 == h2->req0);
 
 	if (h2->rxf_len != 8)				// rfc7540,l,2364,2366
 		return (H2CE_FRAME_SIZE_ERROR);
@@ -296,26 +299,27 @@ h2_rx_ping(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 /**********************************************************************
  */
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_push_promise(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 
-	(void)wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
+	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); /* XXX: wasteful allocation? */
 	// rfc7540,l,2262,2267
-	(void)r2;
 	return (H2CE_PROTOCOL_ERROR);
 }
 
 /**********************************************************************
  */
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_rst_stream(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 
-	(void)wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
+	CHECK_OBJ_ORNULL(r2, H2_REQ_MAGIC);
 
 	if (h2->rxf_len != 4)			// rfc7540,l,2003,2004
 		return (H2CE_FRAME_SIZE_ERROR);
@@ -328,13 +332,15 @@ h2_rx_rst_stream(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 /**********************************************************************
  */
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_goaway(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 
-	(void)wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
-	(void)r2;
+	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
+	assert(r2 == h2->req0);
+
 	h2->goaway_last_stream = vbe32dec(h2->rxf_data);
 	h2->error = h2_connectionerror(vbe32dec(h2->rxf_data + 4));
 	Lck_Lock(&h2->sess->mtx);
@@ -346,13 +352,15 @@ h2_rx_goaway(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 /**********************************************************************
  */
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_window_update(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 	uint32_t wu;
 
-	(void)wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
+	CHECK_OBJ_ORNULL(r2, H2_REQ_MAGIC);
+
 	if (h2->rxf_len != 4)
 		return (H2CE_FRAME_SIZE_ERROR);
 	wu = vbe32dec(h2->rxf_data) & ~(1LU<<31);
@@ -376,13 +384,13 @@ h2_rx_window_update(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
  * Incoming PRIORITY, possibly an ACK of one we sent.
  */
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_priority(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 
-	(void)wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
-	(void)r2;
+	CHECK_OBJ_ORNULL(r2, H2_REQ_MAGIC);
 	return (0);
 }
 
@@ -478,17 +486,19 @@ h2_set_setting(struct h2_sess *h2, const uint8_t *d)
 	return (0);
 }
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_settings(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 	const uint8_t *p;
 	unsigned l;
 	h2_error retval = 0;
 
-	AN(wrk);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
-	AN(r2);
+	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
+	assert(r2 == h2->req0);
 	AZ(h2->rxf_stream);
+
 	if (h2->rxf_flags == H2FF_SETTINGS_ACK) {
 		if (h2->rxf_len > 0)			// rfc7540,l,2047,2049
 			return (H2CE_FRAME_SIZE_ERROR);
@@ -607,7 +617,7 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2,
 	return (0);
 }
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 	struct req *req;
@@ -615,6 +625,7 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 	const uint8_t *p;
 	size_t l;
 
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
 
 	if (r2 == NULL) {
@@ -629,7 +640,7 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 		h2->highest_stream = h2->rxf_stream;
 		r2 = h2_new_req(wrk, h2, h2->rxf_stream, NULL);
 	}
-	AN(r2);
+	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
 
 	if (r2->state != H2_S_IDLE)
 		return (H2CE_PROTOCOL_ERROR);	// XXX spec ?
@@ -695,13 +706,16 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 
 /**********************************************************************/
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_continuation(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 	struct req *req;
 	h2_error h2e;
 
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
+	CHECK_OBJ_ORNULL(r2, H2_REQ_MAGIC);
+
 	if (r2 == NULL || r2->state != H2_S_OPEN || r2->req != h2->new_req)
 		return (H2CE_PROTOCOL_ERROR);	// XXX spec ?
 	req = r2->req;
@@ -723,15 +737,17 @@ h2_rx_continuation(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 
 /**********************************************************************/
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 	int w1 = 0, w2 = 0;
 	char buf[4];
 	unsigned wi;
 
-	(void)wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
+	CHECK_OBJ_ORNULL(r2, H2_REQ_MAGIC);
+
 	if (r2 == NULL || !r2->scheduled)
 		return (0);
 	if (r2->state >= H2_S_CLOS_REM) {

From dridi.boukelmoune at gmail.com  Fri Oct  5 10:12:10 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri,  5 Oct 2018 10:12:10 +0000 (UTC)
Subject: [master] 0742ab0fb Fix assertion for PUSH_PROMISE frames
Message-ID: <20181005101210.9E39F60ED9@lists.varnish-cache.org>


commit 0742ab0fbb97fab5b23dc86840beebfb6b83a4b9
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Oct 5 12:09:28 2018 +0200

    Fix assertion for PUSH_PROMISE frames
    
    r2 can be either null or not.
    
    Test case by @daghf
    
    Refs #2781

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 6caa358bf..7d040b7d0 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -305,7 +305,7 @@ h2_rx_push_promise(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
-	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); /* XXX: wasteful allocation? */
+	CHECK_OBJ_ORNULL(r2, H2_REQ_MAGIC);
 	// rfc7540,l,2262,2267
 	return (H2CE_PROTOCOL_ERROR);
 }
diff --git a/bin/varnishtest/tests/t02003.vtc b/bin/varnishtest/tests/t02003.vtc
index 215171cf6..1d62ac986 100644
--- a/bin/varnishtest/tests/t02003.vtc
+++ b/bin/varnishtest/tests/t02003.vtc
@@ -173,6 +173,21 @@ client c1 {
 	stream 0 -wait
 } -run
 
+client c1 {
+	stream 0 {
+		rxgoaway
+		expect goaway.err == PROTOCOL_ERROR
+		expect goaway.laststream == 1
+	} -start
+	stream 1 {
+		txreq
+		rxresp
+		delay .1
+		# send a PUSH_PROMISE after a request
+		sendhex "000008 05 00 00000001 0001020304050607"
+	} -start
+} -run
+
 varnish v1 -vsl_catchup
 
 varnish v1 -expect MEMPOOL.req0.live == 0

From nils.goroll at uplex.de  Fri Oct  5 10:31:07 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri,  5 Oct 2018 10:31:07 +0000 (UTC)
Subject: [master] 32e518aae gethrtime() is now slower than clock_gettime() on
 modern Solarisen
Message-ID: <20181005103107.D7E59615B9@lists.varnish-cache.org>


commit 32e518aae2b63db4cccfd3861b2256ca6049959d
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Oct 5 12:25:42 2018 +0200

    gethrtime() is now slower than clock_gettime() on modern Solarisen
    
    Throw out the conventional wisdom and base the decision on a micro
    benchmark.
    
    clock_gettime() is now preferred if it is consistently at least
    double as fast as gethrtime(), which is the case on varnishdev-il,
    the SmartOS vtest machine.
    
    config.log gives details on the performance check, sample output
    below:
    
    configure:22703: ./conftest
    hrtime              45989530 check 16748699083977959327
    clock_gettime        4119385 check 16748701613138517215
    ...
    hrtime              48113108 check 16748749015170035860
    clock_gettime        4020802 check 16748751585081458308
    clock_gettime wins 10/10

diff --git a/configure.ac b/configure.ac
index 1f9a2f8b2..85f377dc9 100644
--- a/configure.ac
+++ b/configure.ac
@@ -355,6 +355,63 @@ AC_CHECK_FUNCS([clock_gettime])
 AC_CHECK_FUNCS([gethrtime])
 LIBS="${save_LIBS}"
 
+if test "x$ac_cv_func_gethrtime" = xyes && \
+   test "x$ac_cv_func_clock_gettime" = xyes ; then
+  AC_MSG_CHECKING(if clock_gettime is faster than gethrtime)
+  AC_RUN_IFELSE(
+    [AC_LANG_PROGRAM([[
+#include <sys/time.h>
+#include <time.h>
+#include <stdio.h>
+
+static hrtime_t cl()
+{
+	struct timespec ts;
+
+	(void) clock_gettime(CLOCK_MONOTONIC, &ts);
+	return (ts.tv_sec * 1e9 + ts.tv_nsec);
+}
+    ]],[[
+	hrtime_t s, c, e, t_hr, t_cl;
+	int i, r, wins;
+
+	wins = 0;
+	for (r = 0; r < 10; r++) {
+		c = 0;
+		s = gethrtime();
+		for (i=0; i<100000; i++)
+			c += gethrtime();
+		e = gethrtime();
+		t_hr = e - s;
+		fprintf(stderr, "hrtime\t\t%12lu check %lu\n",
+		    (unsigned long)t_hr, (unsigned long)c);
+
+		c = 0;
+		s = gethrtime();
+		for (i=0; i<100000; i++)
+			c += cl();
+		e = gethrtime();
+		t_cl = e - s;
+		fprintf(stderr, "clock_gettime\t%12lu check %lu\n",
+		    (unsigned long)t_cl, (unsigned long)c);
+
+		if (t_cl * 2 < t_hr)
+			wins++;
+	}
+	fprintf(stderr, "clock_gettime wins %d/%d\n", wins, r);
+	if (2 * wins >= r)
+		return (0);
+	return (1);
+    ]])],
+    [AC_MSG_RESULT(yes)
+     AC_DEFINE([USE_GETHRTIME], [0], [whether to use gethrtime])
+    ],
+    [AC_MSG_RESULT(no)
+     AC_DEFINE([USE_GETHRTIME], [1], [whether to use gethrtime])
+    ]
+  )
+fi
+
 # --enable-kqueue
 AC_ARG_ENABLE(kqueue,
     AS_HELP_STRING([--enable-kqueue],
diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index f9de6244e..39d5ca342 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -118,15 +118,16 @@ init(void)
 #endif
 
 /*
- * Note on Solaris: for some reason, clock_gettime(CLOCK_MONOTONIC, &ts) is not
- * implemented in assembly, but falls into a syscall, while gethrtime() doesn't,
- * so we save a syscall by using gethrtime() if it is defined.
+ * On older Solaris-incarnations, gethrtime() was faster than
+ * clock_gettime(CLOCK_MONOTONIC). Our configure script prefers
+ * clock_gettime if it is consistently at least twice as fast as
+ * gethrtime(), which is the case on modern Solaris descendents.
  */
 
 double
 VTIM_mono(void)
 {
-#ifdef HAVE_GETHRTIME
+#if defined(HAVE_GETHRTIME) && USE_GETHRTIME
 	return (gethrtime() * 1e-9);
 #elif  HAVE_CLOCK_GETTIME
 	struct timespec ts;

From nils.goroll at uplex.de  Fri Oct  5 10:40:07 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri,  5 Oct 2018 10:40:07 +0000 (UTC)
Subject: [master] 7717b4c85 fix a minor oversight
Message-ID: <20181005104007.EA72A61A4C@lists.varnish-cache.org>


commit 7717b4c85bc4e25ba6730b80d9977fa8580b4db7
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Oct 5 12:38:02 2018 +0200

    fix a minor oversight
    
    I failed to consider the hypothetical case that there is only gethrtime()
    and no clock_gettime(CLOCK_MONOTONIC).

diff --git a/configure.ac b/configure.ac
index 85f377dc9..169325892 100644
--- a/configure.ac
+++ b/configure.ac
@@ -355,6 +355,7 @@ AC_CHECK_FUNCS([clock_gettime])
 AC_CHECK_FUNCS([gethrtime])
 LIBS="${save_LIBS}"
 
+AC_DEFINE([USE_GETHRTIME], [1], [whether to use gethrtime])
 if test "x$ac_cv_func_gethrtime" = xyes && \
    test "x$ac_cv_func_clock_gettime" = xyes ; then
   AC_MSG_CHECKING(if clock_gettime is faster than gethrtime)
@@ -407,7 +408,6 @@ static hrtime_t cl()
      AC_DEFINE([USE_GETHRTIME], [0], [whether to use gethrtime])
     ],
     [AC_MSG_RESULT(no)
-     AC_DEFINE([USE_GETHRTIME], [1], [whether to use gethrtime])
     ]
   )
 fi

From hermunn at varnish-software.com  Fri Oct  5 11:32:09 2018
From: hermunn at varnish-software.com (Pål Hermunn Johansen)
Date: Fri,  5 Oct 2018 11:32:09 +0000 (UTC)
Subject: [4.1] 24f57e048 Fix copy&paste sloppyness in http_resp_size
 documentation
Message-ID: <20181005113209.84011645CC@lists.varnish-cache.org>


commit 24f57e048c6466ecd19e15497f48d772ae1a0082
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jun 6 08:34:37 2018 +0000

    Fix copy&paste sloppyness in http_resp_size documentation
    
    Fixes   #2684
    
    Reported by:    ernestojpg at github

diff --git a/include/tbl/params.h b/include/tbl/params.h
index 9a272a3b8..58d6a8c5c 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -638,10 +638,10 @@ PARAM(
 	/* s-text */
 	"Maximum number of bytes of HTTP backend response we will deal "
 	"with.  This is a limit on all bytes up to the double blank line "
-	"which ends the HTTP request.\n"
-	"The memory for the request is allocated from the backend workspace "
+	"which ends the HTTP response.\n"
+	"The memory for the response is allocated from the backend workspace "
 	"(param: workspace_backend) and this parameter limits how much "
-	"of that the request is allowed to take up.",
+	"of that the response is allowed to take up.",
 	/* l-text */	"",
 	/* func */	NULL
 )

From hermunn at varnish-software.com  Fri Oct  5 13:57:11 2018
From: hermunn at varnish-software.com (Pål Hermunn Johansen)
Date: Fri,  5 Oct 2018 13:57:11 +0000 (UTC)
Subject: [4.1] 2fdd68215 Set the task arguments under the lock
Message-ID: <20181005135711.D1C0E91321@lists.varnish-cache.org>


commit 2fdd68215d673dc396e8bfc65f83d246baab6fdd
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Fri Jul 13 12:02:23 2018 +0100

    Set the task arguments under the lock
    
    I've been torturing varnish with this change for some time and was
    not able to reproduce the problem.
    
    Should fix #2719.

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 2355752ab..3e3fe5dcf 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -222,13 +222,12 @@ Pool_Task_Arg(struct worker *wrk, enum task_prio how, task_func_t *func,
 		wrk2 = wrk;
 		retval = 0;
 	}
-	Lck_Unlock(&pp->mtx);
 	AZ(wrk2->task.func);
-
 	assert(arg_len <= WS_Reserve(wrk2->aws, arg_len));
 	memcpy(wrk2->aws->f, arg, arg_len);
 	wrk2->task.func = func;
 	wrk2->task.priv = wrk2->aws->f;
+	Lck_Unlock(&pp->mtx);
 	if (retval)
 		AZ(pthread_cond_signal(&wrk2->cond));
 	return (retval);

From fgsch at lodoss.net  Sat Oct  6 14:13:10 2018
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Sat,  6 Oct 2018 14:13:10 +0000 (UTC)
Subject: [master] a2d0b594a Polish
Message-ID: <20181006141310.99A4649B9@lists.varnish-cache.org>


commit a2d0b594afd73b6c472a25cf3eb01c88e00cc9b7
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Fri Oct 5 14:07:03 2018 +0100

    Polish

diff --git a/configure.ac b/configure.ac
index 169325892..204519b3b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -355,7 +355,6 @@ AC_CHECK_FUNCS([clock_gettime])
 AC_CHECK_FUNCS([gethrtime])
 LIBS="${save_LIBS}"
 
-AC_DEFINE([USE_GETHRTIME], [1], [whether to use gethrtime])
 if test "x$ac_cv_func_gethrtime" = xyes && \
    test "x$ac_cv_func_clock_gettime" = xyes ; then
   AC_MSG_CHECKING(if clock_gettime is faster than gethrtime)
@@ -405,9 +404,9 @@ static hrtime_t cl()
 	return (1);
     ]])],
     [AC_MSG_RESULT(yes)
-     AC_DEFINE([USE_GETHRTIME], [0], [whether to use gethrtime])
     ],
     [AC_MSG_RESULT(no)
+     AC_DEFINE([USE_GETHRTIME], [1], [Define if gethrtime is preferred])
     ]
   )
 fi
diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index 39d5ca342..5c6503c7e 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -127,14 +127,14 @@ init(void)
 double
 VTIM_mono(void)
 {
-#if defined(HAVE_GETHRTIME) && USE_GETHRTIME
-	return (gethrtime() * 1e-9);
-#elif  HAVE_CLOCK_GETTIME
+#if defined(HAVE_CLOCK_GETTIME) && !defined(USE_GETHRTIME)
 	struct timespec ts;
 
 	AZ(clock_gettime(CLOCK_MONOTONIC, &ts));
 	return (ts.tv_sec + 1e-9 * ts.tv_nsec);
-#elif  defined(__MACH__)
+#elif defined(HAVE_GETHRTIME)
+	return (gethrtime() * 1e-9);
+#elif defined(__MACH__)
 	uint64_t mt = mach_absolute_time() - mt_base;
 
 	return (mt * mt_scale);

From nils.goroll at uplex.de  Sat Oct  6 14:48:39 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Sat, 6 Oct 2018 16:48:39 +0200
Subject: [master] a2d0b594a Polish
In-Reply-To: <20181006141310.99A4649B9@lists.varnish-cache.org>
References: <20181006141310.99A4649B9@lists.varnish-cache.org>
Message-ID: <2bf5e6e2-3161-f7ea-d18e-5b264192b00d@uplex.de>

The previous logic set USE_GETHRTIME to 0 when it was not to be used.

Now the logic does not match the test program any more.

On 06/10/2018 16:13, Federico G. Schwindt wrote:
> 
> commit a2d0b594afd73b6c472a25cf3eb01c88e00cc9b7
> Author: Federico G. Schwindt <fgsch at lodoss.net>
> Date:   Fri Oct 5 14:07:03 2018 +0100
> 
>     Polish
> 
> diff --git a/configure.ac b/configure.ac
> index 169325892..204519b3b 100644
> --- a/configure.ac
> +++ b/configure.ac
> @@ -355,7 +355,6 @@ AC_CHECK_FUNCS([clock_gettime])
>  AC_CHECK_FUNCS([gethrtime])
>  LIBS="${save_LIBS}"
>  
> -AC_DEFINE([USE_GETHRTIME], [1], [whether to use gethrtime])
>  if test "x$ac_cv_func_gethrtime" = xyes && \
>     test "x$ac_cv_func_clock_gettime" = xyes ; then
>    AC_MSG_CHECKING(if clock_gettime is faster than gethrtime)
> @@ -405,9 +404,9 @@ static hrtime_t cl()
>  	return (1);
>      ]])],
>      [AC_MSG_RESULT(yes)
> -     AC_DEFINE([USE_GETHRTIME], [0], [whether to use gethrtime])
>      ],
>      [AC_MSG_RESULT(no)
> +     AC_DEFINE([USE_GETHRTIME], [1], [Define if gethrtime is preferred])
>      ]
>    )
>  fi
> diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
> index 39d5ca342..5c6503c7e 100644
> --- a/lib/libvarnish/vtim.c
> +++ b/lib/libvarnish/vtim.c
> @@ -127,14 +127,14 @@ init(void)
>  double
>  VTIM_mono(void)
>  {
> -#if defined(HAVE_GETHRTIME) && USE_GETHRTIME
> -	return (gethrtime() * 1e-9);
> -#elif  HAVE_CLOCK_GETTIME
> +#if defined(HAVE_CLOCK_GETTIME) && !defined(USE_GETHRTIME)
>  	struct timespec ts;
>  
>  	AZ(clock_gettime(CLOCK_MONOTONIC, &ts));
>  	return (ts.tv_sec + 1e-9 * ts.tv_nsec);
> -#elif  defined(__MACH__)
> +#elif defined(HAVE_GETHRTIME)
> +	return (gethrtime() * 1e-9);
> +#elif defined(__MACH__)
>  	uint64_t mt = mach_absolute_time() - mt_base;
>  
>  	return (mt * mt_scale);
> _______________________________________________
> varnish-commit mailing list
> varnish-commit at varnish-cache.org
> https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit
> 


-- 

** * * UPLEX - Nils Goroll Systemoptimierung

Scheffelstra?e 32
22301 Hamburg

tel +49 40 28805731
mob +49 170 2723133
fax +49 40 42949753

xmpp://slink at jabber.int.uplex.de/

http://uplex.de/

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 488 bytes
Desc: OpenPGP digital signature
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20181006/fba3b9e4/attachment.bin>

From fgsch at lodoss.net  Sat Oct  6 14:52:21 2018
From: fgsch at lodoss.net (Federico Schwindt)
Date: Sat, 6 Oct 2018 15:52:21 +0100
Subject: [master] a2d0b594a Polish
In-Reply-To: <2bf5e6e2-3161-f7ea-d18e-5b264192b00d@uplex.de>
References: <20181006141310.99A4649B9@lists.varnish-cache.org>
 <2bf5e6e2-3161-f7ea-d18e-5b264192b00d@uplex.de>
Message-ID: <CAJV_h0ZDFT2te5fAQFzom5NxXREb54X2GYPvYUZnUv83Bm0VTg@mail.gmail.com>

I have no idea what you are referring to but this should work the same.

On Sat, Oct 6, 2018 at 3:48 PM Nils Goroll <nils.goroll at uplex.de> wrote:

> The previous logic set USE_GETHRTIME to 0 when it was not to be used.
>
> Now the logic does not match the test program any more.
>
> On 06/10/2018 16:13, Federico G. Schwindt wrote:
> >
> > commit a2d0b594afd73b6c472a25cf3eb01c88e00cc9b7
> > Author: Federico G. Schwindt <fgsch at lodoss.net>
> > Date:   Fri Oct 5 14:07:03 2018 +0100
> >
> >     Polish
> >
> > diff --git a/configure.ac b/configure.ac
> > index 169325892..204519b3b 100644
> > --- a/configure.ac
> > +++ b/configure.ac
> > @@ -355,7 +355,6 @@ AC_CHECK_FUNCS([clock_gettime])
> >  AC_CHECK_FUNCS([gethrtime])
> >  LIBS="${save_LIBS}"
> >
> > -AC_DEFINE([USE_GETHRTIME], [1], [whether to use gethrtime])
> >  if test "x$ac_cv_func_gethrtime" = xyes && \
> >     test "x$ac_cv_func_clock_gettime" = xyes ; then
> >    AC_MSG_CHECKING(if clock_gettime is faster than gethrtime)
> > @@ -405,9 +404,9 @@ static hrtime_t cl()
> >       return (1);
> >      ]])],
> >      [AC_MSG_RESULT(yes)
> > -     AC_DEFINE([USE_GETHRTIME], [0], [whether to use gethrtime])
> >      ],
> >      [AC_MSG_RESULT(no)
> > +     AC_DEFINE([USE_GETHRTIME], [1], [Define if gethrtime is preferred])
> >      ]
> >    )
> >  fi
> > diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
> > index 39d5ca342..5c6503c7e 100644
> > --- a/lib/libvarnish/vtim.c
> > +++ b/lib/libvarnish/vtim.c
> > @@ -127,14 +127,14 @@ init(void)
> >  double
> >  VTIM_mono(void)
> >  {
> > -#if defined(HAVE_GETHRTIME) && USE_GETHRTIME
> > -     return (gethrtime() * 1e-9);
> > -#elif  HAVE_CLOCK_GETTIME
> > +#if defined(HAVE_CLOCK_GETTIME) && !defined(USE_GETHRTIME)
> >       struct timespec ts;
> >
> >       AZ(clock_gettime(CLOCK_MONOTONIC, &ts));
> >       return (ts.tv_sec + 1e-9 * ts.tv_nsec);
> > -#elif  defined(__MACH__)
> > +#elif defined(HAVE_GETHRTIME)
> > +     return (gethrtime() * 1e-9);
> > +#elif defined(__MACH__)
> >       uint64_t mt = mach_absolute_time() - mt_base;
> >
> >       return (mt * mt_scale);
> > _______________________________________________
> > varnish-commit mailing list
> > varnish-commit at varnish-cache.org
> > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit
> >
>
>
> --
>
> ** * * UPLEX - Nils Goroll Systemoptimierung
>
> Scheffelstra?e 32
> 22301 Hamburg
>
> tel +49 40 28805731
> mob +49 170 2723133
> fax +49 40 42949753
>
> xmpp://slink at jabber.int.uplex.de/
>
> http://uplex.de/
>
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20181006/68aa8b2e/attachment.html>

From nils.goroll at uplex.de  Sat Oct  6 19:01:10 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Sat, 6 Oct 2018 21:01:10 +0200
Subject: [master] a2d0b594a Polish
In-Reply-To: <CAJV_h0ZDFT2te5fAQFzom5NxXREb54X2GYPvYUZnUv83Bm0VTg@mail.gmail.com>
References: <20181006141310.99A4649B9@lists.varnish-cache.org>
 <2bf5e6e2-3161-f7ea-d18e-5b264192b00d@uplex.de>
 <CAJV_h0ZDFT2te5fAQFzom5NxXREb54X2GYPvYUZnUv83Bm0VTg@mail.gmail.com>
Message-ID: <79aa9a00-debc-42cf-da7e-2a6ce51b1357@uplex.de>

On 06/10/2018 16:52, Federico Schwindt wrote:
> I have no idea what you are referring to but this should work the same.

Just looking at diffs before replying to emails is not a good idea. Sorry, I see
it now, yes, that's cleaner, thank you!

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 488 bytes
Desc: OpenPGP digital signature
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20181006/53389a0c/attachment.bin>

From nils.goroll at uplex.de  Mon Oct  8 08:30:09 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon,  8 Oct 2018 08:30:09 +0000 (UTC)
Subject: [master] a289b6191 add phk's suggestion to the micro-benchmark
Message-ID: <20181008083009.DE8FB6469A@lists.varnish-cache.org>


commit a289b619179135b2779b926378bc4558c7fe9e48
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 8 10:29:29 2018 +0200

    add phk's suggestion to the micro-benchmark

diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index 5c6503c7e..822f522d2 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -537,9 +537,20 @@ bench()
 		t += buf[4];
 	}
 	e = VTIM_mono();
-	printf("%s\n", buf);
-	printf("printf: %fs / %d = %fns - tst val %f\n",
-	    e - s, i, 1e9 * (e - s) / i, t);
+	printf("printf %%.6f: %fs / %d = %fns - tst val %f %s\n",
+	    e - s, i, 1e9 * (e - s) / i, t, buf);
+
+	t = 0;
+	s = VTIM_mono();
+	for (i=0; i<100000; i++) {
+		snprintf(buf, sizeof(buf), "%ju.%06ju",
+			 (uint64_t)floor(s),
+			 (uint64_t)floor((s * 1e6)) % 1000000UL);
+		t += buf[4];
+	}
+	e = VTIM_mono();
+	printf("printf %%ju.%%06ju: %fs / %d = %fns - tst val %f %s\n",
+	    e - s, i, 1e9 * (e - s) / i, t, buf);
 }
 
 void

From nils.goroll at uplex.de  Mon Oct  8 08:43:12 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon,  8 Oct 2018 08:43:12 +0000 (UTC)
Subject: [master] c0a92cab0 style(9)
Message-ID: <20181008084312.DBE8C65603@lists.varnish-cache.org>


commit c0a92cab0e7ee593b6979155fb188bff481345aa
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 8 10:42:15 2018 +0200

    style(9)

diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index 822f522d2..c774e2ab8 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -544,8 +544,8 @@ bench()
 	s = VTIM_mono();
 	for (i=0; i<100000; i++) {
 		snprintf(buf, sizeof(buf), "%ju.%06ju",
-			 (uint64_t)floor(s),
-			 (uint64_t)floor((s * 1e6)) % 1000000UL);
+		    (uint64_t)floor(s),
+		    (uint64_t)floor((s * 1e6)) % 1000000UL);
 		t += buf[4];
 	}
 	e = VTIM_mono();

From phk at phk.freebsd.dk  Mon Oct  8 08:50:19 2018
From: phk at phk.freebsd.dk (Poul-Henning Kamp)
Date: Mon, 08 Oct 2018 08:50:19 +0000
Subject: [master] a289b6191 add phk's suggestion to the micro-benchmark
In-Reply-To: <20181008083009.DE8FB6469A@lists.varnish-cache.org>
References: <20181008083009.DE8FB6469A@lists.varnish-cache.org>
Message-ID: <24232.1538988619@critter.freebsd.dk>

--------
In message <20181008083009.DE8FB6469A at lists.varnish-cache.org>, Nils Goroll wri
tes:

>    add phk's suggestion to the micro-benchmark

That's a decent speedup on my laptop:

	real: 0.002875s / 100000 = 28.746128ns - tst val 153898795965262.718750
	mono: 0.002804s / 100000 = 28.038025ns - tst val 23255208087.509720
	printf %.6f: 0.030551s / 100000 = 305.507500ns - tst val 5300000.000000 232552.082280
	printf %ju.%06ju: 0.022367s / 100000 = 223.666760ns - tst val 5300000.000000 232552.112834

Not so much on the VM which runs varnish-cache.org:

	real: 0.232683s / 100000 = 2326.831818ns - tst val 153898853429232.062500
	mono: 0.224068s / 100000 = 2240.684032ns - tst val 702986774.319679
	printf %.6f: 0.036718s / 100000 = 367.176090ns - tst val 4600000.000000 7029.980350
	printf %ju.%06ju: 0.035709s / 100000 = 357.088150ns - tst val 4600000.000000 703


-- 
Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
phk at FreeBSD.ORG         | TCP/IP since RFC 956
FreeBSD committer       | BSD since 4.3-tahoe    
Never attribute to malice what can adequately be explained by incompetence.

From fgsch at lodoss.net  Mon Oct  8 09:03:46 2018
From: fgsch at lodoss.net (Federico Schwindt)
Date: Mon, 8 Oct 2018 10:03:46 +0100
Subject: [master] a289b6191 add phk's suggestion to the micro-benchmark
In-Reply-To: <24232.1538988619@critter.freebsd.dk>
References: <20181008083009.DE8FB6469A@lists.varnish-cache.org>
 <24232.1538988619@critter.freebsd.dk>
Message-ID: <CAJV_h0br_kO5Nf53-1x93gBRx-Y40uN1W8mtEPX2Ap_kAFDarQ@mail.gmail.com>

macos:

real: 0.004912s / 100000 = 49.118996ns - tst val 153898934846862.750000
mono: 0.009153s / 100000 = 91.528893ns - tst val 41563718859.713966
printf %.6f: 0.025697s / 100000 = 256.970000ns - tst val 5100000.000000
415637.193185
printf %ju.%06ju: 0.017690s / 100000 = 176.900000ns - tst val
5100000.000000 415637.218924

inside docker:

real: 0.832294s / 100000 = 8322.939873ns - tst val 153898925522214.531250
mono: 0.829368s / 100000 = 8293.678761ns - tst val 13489198485.484814
printf %.6f: 0.076031s / 100000 = 760.311000ns - tst val 5700000.000000
134892.398735
printf %ju.%06ju: 0.019017s / 100000 = 190.173000ns - tst val
5700000.000000 134892.474827


On Mon, Oct 8, 2018 at 9:50 AM Poul-Henning Kamp <phk at phk.freebsd.dk> wrote:

> --------
> In message <20181008083009.DE8FB6469A at lists.varnish-cache.org>, Nils
> Goroll wri
> tes:
>
> >    add phk's suggestion to the micro-benchmark
>
> That's a decent speedup on my laptop:
>
>         real: 0.002875s / 100000 = 28.746128ns - tst val
> 153898795965262.718750
>         mono: 0.002804s / 100000 = 28.038025ns - tst val 23255208087.509720
>         printf %.6f: 0.030551s / 100000 = 305.507500ns - tst val
> 5300000.000000 232552.082280
>         printf %ju.%06ju: 0.022367s / 100000 = 223.666760ns - tst val
> 5300000.000000 232552.112834
>
> Not so much on the VM which runs varnish-cache.org:
>
>         real: 0.232683s / 100000 = 2326.831818ns - tst val
> 153898853429232.062500
>         mono: 0.224068s / 100000 = 2240.684032ns - tst val 702986774.319679
>         printf %.6f: 0.036718s / 100000 = 367.176090ns - tst val
> 4600000.000000 7029.980350
>         printf %ju.%06ju: 0.035709s / 100000 = 357.088150ns - tst val
> 4600000.000000 703
>
>
> --
> Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
> phk at FreeBSD.ORG         | TCP/IP since RFC 956
> FreeBSD committer       | BSD since 4.3-tahoe
> Never attribute to malice what can adequately be explained by incompetence.
> _______________________________________________
> varnish-commit mailing list
> varnish-commit at varnish-cache.org
> https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20181008/77b91c18/attachment.html>

From fgsch at lodoss.net  Mon Oct  8 09:32:08 2018
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon,  8 Oct 2018 09:32:08 +0000 (UTC)
Subject: [master] 7e0708375 Check we have space before adding the Date header
Message-ID: <20181008093209.139749172F@lists.varnish-cache.org>


commit 7e07083754527744996e4c2e076ed1c7acc6633d
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Oct 8 10:23:38 2018 +0100

    Check we have space before adding the Date header

diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 30988ab56..23eaa0b18 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -1233,6 +1233,11 @@ http_TimeHeader(struct http *to, const char *fmt, double now)
 	char *p;
 
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	if (to->nhd >= to->shd) {
+		VSLb(to->vsl, SLT_LostHeader, "%s", fmt);
+		http_fail(to);
+		return;
+	}
 	p = WS_Alloc(to->ws, strlen(fmt) + VTIM_FORMAT_SIZE);
 	if (p == NULL) {
 		http_fail(to);

From geoff at uplex.de  Mon Oct  8 11:28:11 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Mon,  8 Oct 2018 11:28:11 +0000 (UTC)
Subject: [master] ee8f4f489 Add JSON support for the CLI "status" command.
Message-ID: <20181008112811.B913193F24@lists.varnish-cache.org>


commit ee8f4f4892d25c6df505aca899f7af464607dee6
Author: Geoff Simmons <geoff at uplex.de>
Date:   Fri Sep 21 17:25:19 2018 +0200

    Add JSON support for the CLI "status" command.

diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index d03ff2106..026bdd8e8 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -679,8 +679,19 @@ mch_cli_server_status(struct cli *cli, const char * const *av, void *priv)
 	VCLI_Out(cli, "Child in state %s", ch_state[child_state]);
 }
 
+static void v_matchproto_(cli_func_t)
+mch_cli_server_status_json(struct cli *cli, const char * const *av, void *priv)
+{
+	(void)priv;
+	VCLI_JSON_begin(cli, 2, av);
+	VCLI_Out(cli, ", ");
+	VCLI_JSON_str(cli, ch_state[child_state]);
+	VCLI_JSON_end(cli);
+}
+
 static struct cli_proto cli_mch[] = {
-	{ CLICMD_SERVER_STATUS,		"", mch_cli_server_status },
+	{ CLICMD_SERVER_STATUS,		"", mch_cli_server_status,
+	  mch_cli_server_status_json },
 	{ CLICMD_SERVER_START,		"", mch_cli_server_start },
 	{ CLICMD_SERVER_STOP,		"", mch_cli_server_stop },
 	{ CLICMD_PANIC_SHOW,		"", mch_cli_panic_show },
diff --git a/bin/varnishtest/tests/b00004.vtc b/bin/varnishtest/tests/b00004.vtc
index ec5cf7661..2ed2c10ff 100644
--- a/bin/varnishtest/tests/b00004.vtc
+++ b/bin/varnishtest/tests/b00004.vtc
@@ -7,7 +7,11 @@ server s1 {
 varnish v1 -vcl+backend { }
 
 varnish v1 -start
+varnish v1 -cliexpect "running" status
+varnish v1 -clijson "status -j"
 varnish v1 -stop
+varnish v1 -cliexpect "stopped" status
+varnish v1 -clijson "status -j"
 
 varnish v1 -start
 varnish v1 -stop

From geoff at uplex.de  Mon Oct  8 11:28:11 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Mon,  8 Oct 2018 11:28:11 +0000 (UTC)
Subject: [master] dcc0eb262 Add JSON support for the CLI "vcl.list" command.
Message-ID: <20181008112811.F40D893F27@lists.varnish-cache.org>


commit dcc0eb262ad46d4161fcabff384ac507e64525c1
Author: Geoff Simmons <geoff at uplex.de>
Date:   Sun Sep 23 22:49:07 2018 +0200

    Add JSON support for the CLI "vcl.list" command.

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 6827b7379..3e1173f21 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -667,6 +667,52 @@ vcl_cli_list(struct cli *cli, const char * const *av, void *priv)
 	}
 }
 
+static void v_matchproto_(cli_func_t)
+vcl_cli_list_json(struct cli *cli, const char * const *av, void *priv)
+{
+	struct vcl *vcl;
+
+	(void)priv;
+	ASSERT_CLI();
+	VCLI_JSON_begin(cli, 2, av);
+	VCLI_Out(cli, ",\n");
+	VTAILQ_FOREACH(vcl, &vcl_head, list) {
+		VCLI_Out(cli, "{\n");
+		VSB_indent(cli->sb, 2);
+		VCLI_Out(cli, "\"status\": ");
+		if (vcl == vcl_active) {
+			VCLI_Out(cli, "\"active\",\n");
+		} else if (vcl->discard) {
+			VCLI_Out(cli, "\"discarded\",\n");
+		} else
+			VCLI_Out(cli, "\"available\",\n");
+		VCLI_Out(cli, "\"state\": \"%s\",\n", vcl->state);
+		VCLI_Out(cli, "\"temperature\": \"%s\",\n", vcl->temp);
+		VCLI_Out(cli, "\"busy\": %u,\n", vcl->busy);
+		VCLI_Out(cli, "\"name\": \"%s\"", vcl->loaded_name);
+		if (vcl->label != NULL) {
+			VCLI_Out(cli, ",\n");
+			VCLI_Out(cli, "\"label\": {\n");
+			VSB_indent(cli->sb, 2);
+				VCLI_Out(cli, "\"name\": \"%s\"",
+					 vcl->label->loaded_name);
+			if (vcl->nrefs)
+				VCLI_Out(cli, ",\n\"refs\": %d", vcl->nrefs);
+			VCLI_Out(cli, "\n");
+			VCLI_Out(cli, "}");
+			VSB_indent(cli->sb, -2);
+		} else if (vcl->nlabels > 0) {
+			VCLI_Out(cli, ",\n");
+			VCLI_Out(cli, "\"labels\": %d", vcl->nlabels);
+		}
+		VSB_indent(cli->sb, -2);
+		VCLI_Out(cli, "\n}");
+		if (VTAILQ_NEXT(vcl, list) != NULL)
+			VCLI_Out(cli, ",\n");
+	}
+	VCLI_JSON_end(cli);
+}
+
 static void v_matchproto_(cli_func_t)
 vcl_cli_load(struct cli *cli, const char * const *av, void *priv)
 {
@@ -826,7 +872,7 @@ vcl_cli_show(struct cli *cli, const char * const *av, void *priv)
 
 static struct cli_proto vcl_cmds[] = {
 	{ CLICMD_VCL_LOAD,		"", vcl_cli_load },
-	{ CLICMD_VCL_LIST,		"", vcl_cli_list },
+	{ CLICMD_VCL_LIST,		"", vcl_cli_list, vcl_cli_list_json },
 	{ CLICMD_VCL_STATE,		"", vcl_cli_state },
 	{ CLICMD_VCL_DISCARD,		"", vcl_cli_discard },
 	{ CLICMD_VCL_USE,		"", vcl_cli_use },
diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c
index 2b0e88cf1..62fe45061 100644
--- a/bin/varnishd/mgt/mgt_vcl.c
+++ b/bin/varnishd/mgt/mgt_vcl.c
@@ -814,6 +814,61 @@ mcf_vcl_list(struct cli *cli, const char * const *av, void *priv)
 	}
 }
 
+static void v_matchproto_(cli_func_t)
+mcf_vcl_list_json(struct cli *cli, const char * const *av, void *priv)
+{
+	unsigned status;
+	char *p;
+	struct vclprog *vp;
+	struct vcldep *vd;
+
+	/* NB: Shall generate same output as vcl_cli_list() */
+
+	(void)priv;
+	if (MCH_Running()) {
+		if (!mgt_cli_askchild(&status, &p, "vcl.list -j\n")) {
+			VCLI_SetResult(cli, status);
+			VCLI_Out(cli, "%s", p);
+		}
+		free(p);
+	} else {
+		VCLI_JSON_begin(cli, 2, av);
+		VCLI_Out(cli, ",\n");
+		VTAILQ_FOREACH(vp, &vclhead, list) {
+			VCLI_Out(cli, "{\n");
+			VSB_indent(cli->sb, 2);
+			VCLI_Out(cli, "\"status\": \"%s\",\n",
+			    vp == active_vcl ? "active" : "available");
+			VCLI_Out(cli, "\"state\": \"%s\",\n", vp->state);
+			VCLI_Out(cli, "\"temperature\": \"%s\",\n",
+			    vp->warm ? VCL_STATE_WARM : VCL_STATE_COLD);
+			VCLI_Out(cli, "\"name\": \"%s\"", vp->name);
+			if (mcf_is_label(vp)) {
+				vd = VTAILQ_FIRST(&vp->dfrom);
+				AN(vd);
+				VCLI_Out(cli, ",\n");
+				VCLI_Out(cli, "\"label\": {\n");
+				VSB_indent(cli->sb, 2);
+				VCLI_Out(cli, "\"name\": \"%s\"", vd->to->name);
+				if (vp->nto > 0)
+					VCLI_Out(cli, ",\n\"refs\": %d",
+						 vp->nto);
+				VSB_indent(cli->sb, -2);
+				VCLI_Out(cli, "\n");
+				VCLI_Out(cli, "}");
+			} else if (vp->nto > 0) {
+				VCLI_Out(cli, ",\n");
+				VCLI_Out(cli, "\"labels\": %d", vp->nto);
+			}
+			VSB_indent(cli->sb, -2);
+			VCLI_Out(cli, "\n}");
+			if (VTAILQ_NEXT(vp, list) != NULL)
+				VCLI_Out(cli, ",\n");
+		}
+		VCLI_JSON_end(cli);
+	}
+}
+
 static void v_matchproto_(cli_func_t)
 mcf_vcl_label(struct cli *cli, const char * const *av, void *priv)
 {
@@ -907,7 +962,7 @@ static struct cli_proto cli_vcl[] = {
 	{ CLICMD_VCL_USE,		"", mcf_vcl_use },
 	{ CLICMD_VCL_STATE,		"", mcf_vcl_state },
 	{ CLICMD_VCL_DISCARD,		"", mcf_vcl_discard },
-	{ CLICMD_VCL_LIST,		"", mcf_vcl_list },
+	{ CLICMD_VCL_LIST,		"", mcf_vcl_list, mcf_vcl_list_json },
 	{ CLICMD_VCL_LABEL,		"", mcf_vcl_label },
 	{ NULL }
 };
diff --git a/bin/varnishtest/tests/b00032.vtc b/bin/varnishtest/tests/b00032.vtc
index 89cb26f52..7aa6cddfa 100644
--- a/bin/varnishtest/tests/b00032.vtc
+++ b/bin/varnishtest/tests/b00032.vtc
@@ -11,9 +11,13 @@ varnish v1 -vcl+backend {}
 varnish v1 -vcl+backend {}
 
 varnish v1 -cliok vcl.list
+varnish v1 -clijson "vcl.list -j"
 
 varnish v1 -cliok start
 
+varnish v1 -cliok vcl.list
+varnish v1 -clijson "vcl.list -j"
+
 varnish v1 -cliok "vcl.use vcl1"
 
 varnish v1 -clierr 300 "vcl.discard vcl1"
diff --git a/bin/varnishtest/tests/s00005.vtc b/bin/varnishtest/tests/s00005.vtc
index 67cd3a95a..4dac2f167 100644
--- a/bin/varnishtest/tests/s00005.vtc
+++ b/bin/varnishtest/tests/s00005.vtc
@@ -39,6 +39,7 @@ varnish v1 -clierr 106 "vcl.label foo vcl0"
 varnish v1 -cliok "vcl.label foo vcl2"
 varnish v1 -cliok "vcl.label bar vcl2"
 varnish v1 -cliok "vcl.list"
+varnish v1 -clijson "vcl.list -j"
 varnish v1 -clierr 300 "vcl.discard vcl2"
 varnish v1 -cliok "vcl.discard bar"
 varnish v1 -cliok "vcl.label foo vcl1"
@@ -100,6 +101,7 @@ varnish v1 -cliok "vcl.label label1 vcl1"
 varnish v1 -cliok "vcl.label label2 vcl1"
 varnish v1 -cliok "vcl.label label3 vcl1"
 varnish v1 -cliok "vcl.list"
+varnish v1 -clijson "vcl.list -j"
 
 varnish v1 -start
 varnish v1 -cliok "vcl.list"
@@ -111,6 +113,7 @@ client c1 -run
 #######################################################################
 
 varnish v1 -cliok vcl.list
+varnish v1 -clijson "vcl.list -j"
 
 varnish v1 -vcl+backend { }
 
diff --git a/bin/varnishtest/tests/v00045.vtc b/bin/varnishtest/tests/v00045.vtc
index a0c511465..0f9cdf5a6 100644
--- a/bin/varnishtest/tests/v00045.vtc
+++ b/bin/varnishtest/tests/v00045.vtc
@@ -18,6 +18,7 @@ varnish v1 -cliok "vcl.state vcl1 cold"
 delay 1
 
 varnish v1 -cliexpect "cold/cooling.*vcl1" vcl.list
+varnish v1 -clijson "vcl.list -j"
 
 # It can't be warmed up yet
 delay 1
@@ -26,6 +27,7 @@ varnish v1 -cliexpect "vmod-debug ref on vcl1" "vcl.state vcl1 warm"
 # It will eventually cool down
 delay 2
 varnish v1 -cliexpect "cold/cold.*vcl1" vcl.list
+varnish v1 -clijson "vcl.list -j"
 
 # At this point it becomes possible to warm up again
 varnish v1 -cliok "vcl.state vcl1 warm"

From geoff at uplex.de  Mon Oct  8 11:28:12 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Mon,  8 Oct 2018 11:28:12 +0000 (UTC)
Subject: [master] db0e25961 Add JSON support for the "param.show" CLI command.
Message-ID: <20181008112812.4C69A93F31@lists.varnish-cache.org>


commit db0e25961e424411c596f642cdbc84919621b685
Author: Geoff Simmons <geoff at uplex.de>
Date:   Mon Sep 24 17:51:01 2018 +0200

    Add JSON support for the "param.show" CLI command.
    
    * Each param is represented as a JSON object listed in the output
      array.
    * The object's "value" field contains the current value, which can
      have different types -- int, float, string or boolean.
    * There is no -l form with param.show -j, each object contains the
      full description.
    * But "param.show -j the_param" or "param.show -j changed" may be
      used to limit the number of objects in the output.
    * Each object may or may not have any of the fields "minimum",
      "maximum", "default" or "units", depending on the param.
    * Params not implemented on the present platform just have
      "implemented":false along with the param name, and no other fields.
    * The values of the "minimum", "maximum" and "default" fields are
      always strings.

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 139814c09..f3faa9cb3 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -335,6 +335,126 @@ mcf_param_show(struct cli *cli, const char * const *av, void *priv)
 	VSB_destroy(&vsb);
 }
 
+static inline void
+mcf_json_key_valstr(struct cli *cli, const char *key, const char *val)
+{
+	VCLI_Out(cli, "\"%s\": ", key);
+	VCLI_JSON_str(cli, val);
+	VCLI_Out(cli, ",\n");
+}
+
+static void v_matchproto_(cli_func_t)
+mcf_param_show_json(struct cli *cli, const char * const *av, void *priv)
+{
+	int n, comma = 0;
+	struct plist *pl;
+	const struct parspec *pp;
+	int chg = 0, flags;
+	struct vsb *vsb, *def;
+	const char *show = NULL;
+
+	vsb = VSB_new_auto();
+	def = VSB_new_auto();
+	(void)priv;
+
+	for (int i = 2; av[i] != NULL; i++) {
+		if (strcmp(av[i], "-l") == 0) {
+			VCLI_SetResult(cli, CLIS_PARAM);
+			VCLI_Out(cli, "-l not permitted with param.show -j");
+			return;
+		}
+		if (strcmp(av[i], "changed") == 0) {
+			chg = 1;
+			continue;
+		}
+		if (strcmp(av[i], "-j") == 0)
+			continue;
+		show = av[i];
+	}
+
+	n = 0;
+	VCLI_JSON_begin(cli, 2, av);
+	VCLI_Out(cli, ",\n");
+	VTAILQ_FOREACH(pl, &phead, list) {
+		pp = pl->spec;
+		if (show != NULL && strcmp(pp->name, show) != 0)
+			continue;
+		n++;
+
+		VSB_clear(vsb);
+		if (pp->func(vsb, pp, JSON_FMT))
+			VCLI_SetResult(cli, CLIS_PARAM);
+		AZ(VSB_finish(vsb));
+		VSB_clear(def);
+		if (pp->func(def, pp, NULL))
+			VCLI_SetResult(cli, CLIS_PARAM);
+		AZ(VSB_finish(def));
+		if (chg && pp->def != NULL && !strcmp(pp->def, VSB_data(def)))
+			continue;
+
+		VCLI_Out(cli, "%s", comma ? ",\n" : "");
+		comma++;
+		VCLI_Out(cli, "{\n");
+		VSB_indent(cli->sb, 2);
+		mcf_json_key_valstr(cli, "name", pp->name);
+		if (pp->flags & NOT_IMPLEMENTED) {
+			VCLI_Out(cli, "\"implemented\": false\n");
+			VSB_indent(cli->sb, -2);
+			VCLI_Out(cli, "}");
+			continue;
+		}
+		VCLI_Out(cli, "\"implemented\": true,\n");
+		VCLI_Out(cli, "\"value\": %s,\n", VSB_data(vsb));
+		if (pp->units != NULL && *pp->units != '\0')
+			mcf_json_key_valstr(cli, "units", pp->units);
+
+		if (pp->def != NULL)
+			mcf_json_key_valstr(cli, "default", pp->def);
+		if (pp->min != NULL)
+			mcf_json_key_valstr(cli, "minimum", pp->min);
+		if (pp->max != NULL)
+			mcf_json_key_valstr(cli, "maximum", pp->max);
+		mcf_json_key_valstr(cli, "description", pp->descr);
+
+		flags = 0;
+		VCLI_Out(cli, "\"flags\": [\n");
+		VSB_indent(cli->sb, 2);
+
+#define flag_out(flag, string) do {					\
+			if (pp->flags & flag) {				\
+				if (flags)				\
+					VCLI_Out(cli, ",\n");		\
+				VCLI_Out(cli, "\"%s\"", #string);	\
+				flags++;				\
+			}						\
+		} while(0)
+
+		flag_out(OBJ_STICKY, obj_sticky);
+		flag_out(DELAYED_EFFECT, delayed_effect);
+		flag_out(EXPERIMENTAL, experimental);
+		flag_out(MUST_RELOAD, must_reload);
+		flag_out(MUST_RESTART, must_restart);
+		flag_out(WIZARD, wizard);
+		flag_out(PROTECTED, protected);
+		flag_out(ONLY_ROOT, only_root);
+
+#undef flag_out
+
+		VSB_indent(cli->sb, -2);
+		VCLI_Out(cli, "\n]");
+		VSB_indent(cli->sb, -2);
+		VCLI_Out(cli, "\n}");
+	}
+	VCLI_JSON_end(cli);
+	if (show != NULL && n == 0) {
+		VSB_clear(cli->sb);
+		VCLI_SetResult(cli, CLIS_PARAM);
+		VCLI_Out(cli, "Unknown parameter \"%s\".", show);
+	}
+	VSB_destroy(&vsb);
+	VSB_destroy(&def);
+}
+
 /*--------------------------------------------------------------------
  * Mark parameters as protected
  */
@@ -471,7 +591,8 @@ mcf_wash_param(struct cli *cli, const struct parspec *pp, const char **val,
 /*--------------------------------------------------------------------*/
 
 static struct cli_proto cli_params[] = {
-	{ CLICMD_PARAM_SHOW,		"", mcf_param_show },
+	{ CLICMD_PARAM_SHOW,		"", mcf_param_show,
+	  mcf_param_show_json },
 	{ CLICMD_PARAM_SET,		"", mcf_param_set },
 	{ NULL }
 };
diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h
index f4362ca96..7188b64e1 100644
--- a/bin/varnishd/mgt/mgt_param.h
+++ b/bin/varnishd/mgt/mgt_param.h
@@ -32,6 +32,9 @@ struct parspec;
 
 typedef int tweak_t(struct vsb *, const struct parspec *, const char *arg);
 
+/* Sentinel for the arg position of tweak_t to ask for JSON formatting. */
+extern const char * const JSON_FMT;
+
 struct parspec {
 	const char	*name;
 	tweak_t		*func;
diff --git a/bin/varnishd/mgt/mgt_param_bits.c b/bin/varnishd/mgt/mgt_param_bits.c
index a6d7b9216..263d8a34d 100644
--- a/bin/varnishd/mgt/mgt_param_bits.c
+++ b/bin/varnishd/mgt/mgt_param_bits.c
@@ -119,7 +119,7 @@ tweak_vsl_mask(struct vsb *vsb, const struct parspec *par, const char *arg)
 	const char *s;
 	(void)par;
 
-	if (arg != NULL) {
+	if (arg != NULL && arg != JSON_FMT) {
 		if (!strcmp(arg, "default")) {
 			memset(mgt_param.vsl_mask,
 			    0, sizeof mgt_param.vsl_mask);
@@ -141,6 +141,8 @@ tweak_vsl_mask(struct vsb *vsb, const struct parspec *par, const char *arg)
 			    "VSL tag", "-"));
 		}
 	} else {
+		if (arg == JSON_FMT)
+			VSB_putc(vsb, '"');
 		s = "";
 		for (j = 0; j < (unsigned)SLT__Reserved; j++) {
 			if (bit(mgt_param.vsl_mask, j, BTST)) {
@@ -150,6 +152,8 @@ tweak_vsl_mask(struct vsb *vsb, const struct parspec *par, const char *arg)
 		}
 		if (*s == '\0')
 			VSB_printf(vsb, "(all enabled)");
+		if (arg == JSON_FMT)
+			VSB_putc(vsb, '"');
 	}
 	return (0);
 }
@@ -171,7 +175,7 @@ tweak_debug(struct vsb *vsb, const struct parspec *par, const char *arg)
 	unsigned j;
 	(void)par;
 
-	if (arg != NULL) {
+	if (arg != NULL && arg != JSON_FMT) {
 		if (!strcmp(arg, "none")) {
 			memset(mgt_param.debug_bits,
 			    0, sizeof mgt_param.debug_bits);
@@ -180,6 +184,8 @@ tweak_debug(struct vsb *vsb, const struct parspec *par, const char *arg)
 			    DBG_Reserved, arg, debug_tags, "debug bit", "+"));
 		}
 	} else {
+		if (arg == JSON_FMT)
+			VSB_putc(vsb, '"');
 		s = "";
 		for (j = 0; j < (unsigned)DBG_Reserved; j++) {
 			if (bit(mgt_param.debug_bits, j, BTST)) {
@@ -189,6 +195,8 @@ tweak_debug(struct vsb *vsb, const struct parspec *par, const char *arg)
 		}
 		if (*s == '\0')
 			VSB_printf(vsb, "none");
+		if (arg == JSON_FMT)
+			VSB_putc(vsb, '"');
 	}
 	return (0);
 }
@@ -210,7 +218,7 @@ tweak_feature(struct vsb *vsb, const struct parspec *par, const char *arg)
 	unsigned j;
 	(void)par;
 
-	if (arg != NULL) {
+	if (arg != NULL && arg != JSON_FMT) {
 		if (!strcmp(arg, "none")) {
 			memset(mgt_param.feature_bits,
 			    0, sizeof mgt_param.feature_bits);
@@ -220,6 +228,8 @@ tweak_feature(struct vsb *vsb, const struct parspec *par, const char *arg)
 			    "feature bit", "+"));
 		}
 	} else {
+		if (arg == JSON_FMT)
+			VSB_putc(vsb, '"');
 		s = "";
 		for (j = 0; j < (unsigned)FEATURE_Reserved; j++) {
 			if (bit(mgt_param.feature_bits, j, BTST)) {
@@ -229,6 +239,8 @@ tweak_feature(struct vsb *vsb, const struct parspec *par, const char *arg)
 		}
 		if (*s == '\0')
 			VSB_printf(vsb, "none");
+		if (arg == JSON_FMT)
+			VSB_putc(vsb, '"');
 	}
 	return (0);
 }
diff --git a/bin/varnishd/mgt/mgt_param_tweak.c b/bin/varnishd/mgt/mgt_param_tweak.c
index 0f95237c2..01cab5f6b 100644
--- a/bin/varnishd/mgt/mgt_param_tweak.c
+++ b/bin/varnishd/mgt/mgt_param_tweak.c
@@ -43,6 +43,8 @@
 #include "vav.h"
 #include "vnum.h"
 
+const char * const JSON_FMT = (const char *)&JSON_FMT;
+
 /*--------------------------------------------------------------------
  * Generic handling of double typed parameters
  */
@@ -53,7 +55,7 @@ tweak_generic_double(struct vsb *vsb, volatile double *dest,
 {
 	volatile double u, minv = 0, maxv = 0;
 
-	if (arg != NULL) {
+	if (arg != NULL && arg != JSON_FMT) {
 		if (min != NULL) {
 			minv = VNUM(min);
 			if (isnan(minv)) {
@@ -123,7 +125,7 @@ tweak_bool(struct vsb *vsb, const struct parspec *par, const char *arg)
 	volatile unsigned *dest;
 
 	dest = par->priv;
-	if (arg != NULL) {
+	if (arg != NULL && arg != JSON_FMT) {
 		if (!strcasecmp(arg, "off"))
 			*dest = 0;
 		else if (!strcasecmp(arg, "disable"))
@@ -144,6 +146,8 @@ tweak_bool(struct vsb *vsb, const struct parspec *par, const char *arg)
 			VSB_printf(vsb, "use \"on\" or \"off\"\n");
 			return (-1);
 		}
+	} else if (arg == JSON_FMT) {
+		VSB_printf(vsb, "%s", *dest ? "true" : "false");
 	} else {
 		VSB_printf(vsb, "%s", *dest ? "on" : "off");
 	}
@@ -159,7 +163,7 @@ tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg,
 	unsigned u, minv = 0, maxv = 0;
 	char *p;
 
-	if (arg != NULL) {
+	if (arg != NULL && arg != JSON_FMT) {
 		if (min != NULL) {
 			p = NULL;
 			minv = strtoul(min, &p, 0);
@@ -195,7 +199,7 @@ tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg,
 			return (-1);
 		}
 		*dest = u;
-	} else if (*dest == UINT_MAX) {
+	} else if (*dest == UINT_MAX && arg != JSON_FMT) {
 		VSB_printf(vsb, "unlimited");
 	} else {
 		VSB_printf(vsb, "%u", *dest);
@@ -246,7 +250,7 @@ tweak_generic_bytes(struct vsb *vsb, volatile ssize_t *dest, const char *arg,
 	uintmax_t r, rmin = 0, rmax = 0;
 	const char *p;
 
-	if (arg != NULL) {
+	if (arg != NULL && arg != JSON_FMT) {
 		if (min != NULL) {
 			p = VNUM_2bytes(min, &rmin, 0);
 			if (p != NULL) {
@@ -285,6 +289,8 @@ tweak_generic_bytes(struct vsb *vsb, volatile ssize_t *dest, const char *arg,
 			return (-1);
 		}
 		*dest = r;
+	} else if (arg == JSON_FMT) {
+		VSB_printf(vsb, "%zd", *dest);
 	} else {
 		fmt_bytes(vsb, *dest);
 	}
@@ -364,6 +370,8 @@ tweak_string(struct vsb *vsb, const struct parspec *par, const char *arg)
 	/* XXX should have tweak_generic_string */
 	if (arg == NULL) {
 		VSB_quote(vsb, *p, -1, 0);
+	} else if (arg == JSON_FMT) {
+		VSB_quote(vsb, *p, -1, VSB_QUOTE_JSON|VSB_QUOTE_CSTR);
 	} else {
 		REPLACE(*p, arg);
 	}
@@ -380,7 +388,15 @@ tweak_poolparam(struct vsb *vsb, const struct parspec *par, const char *arg)
 	int retval = 0;
 
 	pp = par->priv;
-	if (arg == NULL) {
+	if (arg == JSON_FMT) {
+		VSB_printf(vsb, "{\n");
+		VSB_indent(vsb, 8);
+		VSB_printf(vsb, "\"min_pool\": %u,\n", pp->min_pool);
+		VSB_printf(vsb, "\"max_pool\": %u,\n", pp->max_pool);
+		VSB_printf(vsb, "\"max_age\": %g\n", pp->max_age);
+		VSB_indent(vsb, -4);
+		VSB_printf(vsb, "}");
+	} else if (arg == NULL) {
 		VSB_printf(vsb, "%u,%u,%g",
 		    pp->min_pool, pp->max_pool, pp->max_age);
 	} else {
diff --git a/bin/varnishtest/tests/b00042.vtc b/bin/varnishtest/tests/b00042.vtc
index 9fa143e9a..6470a7260 100644
--- a/bin/varnishtest/tests/b00042.vtc
+++ b/bin/varnishtest/tests/b00042.vtc
@@ -33,3 +33,10 @@ varnish v1 -clierr "106" {param.show fofofofo}
 varnish v1 -cliok "param.show changed"
 varnish v1 -cliok "param.show "
 varnish v1 -cliok "param.show -l"
+
+varnish v1 -clijson "param.show -j pool_req"
+varnish v1 -clijson "param.show -j pool_sess"
+varnish v1 -clijson "param.show -j changed"
+varnish v1 -clijson "param.show -j"
+varnish v1 -clierr "106" "param.show -j -l"
+varnish v1 -clierr "106" "param.show -j fofofofo"

From geoff at uplex.de  Mon Oct  8 11:28:12 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Mon,  8 Oct 2018 11:28:12 +0000 (UTC)
Subject: [master] b2818da43 Add JSON support for the "ban.list" CLI command.
Message-ID: <20181008112812.825DD93F3C@lists.varnish-cache.org>


commit b2818da43003054a53c98da090f503bc883fd1cd
Author: Geoff Simmons <geoff at uplex.de>
Date:   Tue Sep 25 09:04:42 2018 +0200

    Add JSON support for the "ban.list" CLI command.

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 6ec3577d2..fab8661cf 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -669,7 +669,7 @@ ccf_ban(struct cli *cli, const char * const *av, void *priv)
 }
 
 static void
-ban_render(struct cli *cli, const uint8_t *bs)
+ban_render(struct cli *cli, const uint8_t *bs, int quote)
 {
 	struct ban_test bt;
 	const uint8_t *be;
@@ -704,27 +704,21 @@ ban_render(struct cli *cli, const uint8_t *bs)
 		default:
 			WRONG("Wrong BANS_OPER");
 		}
-		VCLI_Out(cli, "%s", bt.arg2);
+		if (quote)
+			VCLI_Quote(cli, bt.arg2);
+		else
+			VCLI_Out(cli, "%s", bt.arg2);
 		if (bs < be)
 			VCLI_Out(cli, " && ");
 	}
 }
 
-static void v_matchproto_(cli_func_t)
-ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
+static void
+ban_list(struct cli *cli, struct ban *bl)
 {
-	struct ban *b, *bl;
+	struct ban *b;
 	int64_t o;
 
-	(void)av;
-	(void)priv;
-
-	/* Get a reference so we are safe to traverse the list */
-	Lck_Lock(&ban_mtx);
-	bl = VTAILQ_LAST(&ban_head, banhead_s);
-	bl->refcount++;
-	Lck_Unlock(&ban_mtx);
-
 	VCLI_Out(cli, "Present bans:\n");
 	VTAILQ_FOREACH(b, &ban_head, list) {
 		o = bl == b ? 1 : 0;
@@ -738,7 +732,7 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
 			    b);
 		}
 		VCLI_Out(cli, "  ");
-		ban_render(cli, b->spec);
+		ban_render(cli, b->spec, 0);
 		VCLI_Out(cli, "\n");
 		if (VCLI_Overflow(cli))
 			break;
@@ -750,6 +744,80 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
 			Lck_Unlock(&ban_mtx);
 		}
 	}
+}
+
+static void
+ban_list_json(struct cli *cli, const char * const *av, struct ban *bl)
+{
+	struct ban *b;
+	int64_t o;
+	int n = 0;
+	int ocs;
+
+	VCLI_JSON_begin(cli, 2, av);
+	VCLI_Out(cli, ",\n");
+	VTAILQ_FOREACH(b, &ban_head, list) {
+		o = bl == b ? 1 : 0;
+		VCLI_Out(cli, "%s", n ? ",\n" : "");
+		n++;
+		VCLI_Out(cli, "{\n");
+		VSB_indent(cli->sb, 2);
+		VCLI_Out(cli, "\"time\": %.6f,\n", ban_time(b->spec));
+		VCLI_Out(cli, "\"refs\": %ju,\n", (intmax_t)(b->refcount - o));
+		VCLI_Out(cli, "\"completed\": %s,\n",
+			 b->flags & BANS_FLAG_COMPLETED ? "true" : "false");
+		VCLI_Out(cli, "\"spec\": \"");
+		ban_render(cli, b->spec, 1);
+		VCLI_Out(cli, "\"");
+
+		if (DO_DEBUG(DBG_LURKER)) {
+			VCLI_Out(cli, ",\n");
+			VCLI_Out(cli, "\"req_tests\": %s,\n",
+				 b->flags & BANS_FLAG_REQ ? "true" : "false");
+			VCLI_Out(cli, "\"obj_tests\": %s,\n",
+				 b->flags & BANS_FLAG_OBJ ? "true" : "false");
+			VCLI_Out(cli, "\"pointer\": \"%p\",\n", b);
+			if (VCLI_Overflow(cli))
+				break;
+
+			ocs = 0;
+			VCLI_Out(cli, "\"objcores\": [\n");
+			VSB_indent(cli->sb, 2);
+			Lck_Lock(&ban_mtx);
+			struct objcore *oc;
+			VTAILQ_FOREACH(oc, &b->objcore, ban_list) {
+				if (ocs)
+					VCLI_Out(cli, ",\n");
+				VCLI_Out(cli, "%p", oc);
+				ocs++;
+			}
+			Lck_Unlock(&ban_mtx);
+			VSB_indent(cli->sb, -2);
+			VCLI_Out(cli, "\n]");
+		}
+		VSB_indent(cli->sb, -2);
+		VCLI_Out(cli, "\n}");
+	}
+	VCLI_JSON_end(cli);
+}
+
+static void v_matchproto_(cli_func_t)
+ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
+{
+	struct ban *bl;
+
+	(void)priv;
+
+	/* Get a reference so we are safe to traverse the list */
+	Lck_Lock(&ban_mtx);
+	bl = VTAILQ_LAST(&ban_head, banhead_s);
+	bl->refcount++;
+	Lck_Unlock(&ban_mtx);
+
+	if (av[2] != NULL && strcmp(av[2], "-j") == 0)
+		ban_list_json(cli, av, bl);
+	else
+		ban_list(cli, bl);
 
 	Lck_Lock(&ban_mtx);
 	bl->refcount--;
@@ -759,7 +827,8 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
 
 static struct cli_proto ban_cmds[] = {
 	{ CLICMD_BAN,				"", ccf_ban },
-	{ CLICMD_BAN_LIST,			"", ccf_ban_list },
+	{ CLICMD_BAN_LIST,			"", ccf_ban_list,
+	  ccf_ban_list },
 	{ NULL }
 };
 
diff --git a/bin/varnishtest/tests/c00019.vtc b/bin/varnishtest/tests/c00019.vtc
index 22b5ca1bf..3a4a3b2f8 100644
--- a/bin/varnishtest/tests/c00019.vtc
+++ b/bin/varnishtest/tests/c00019.vtc
@@ -84,7 +84,13 @@ client c1 {
 varnish v1 -expect bans_tested == 2
 varnish v1 -expect bans_tests_tested == 2
 varnish v1 -cliok "ban.list"
+varnish v1 -clijson "ban.list -j"
 
 # Test a bogus regexp
-
 varnish v1 -clierr 106 "ban req.url ~ [[["
+
+# Ban expression with quoting
+varnish v1 -cliok {ban req.url ~ "BAR"}
+shell {varnishadm -n ${tmpdir}/v1 ban 'obj.http.Host ~ \"Foo\"'}
+varnish v1 -cliok "ban.list"
+varnish v1 -clijson "ban.list -j"
diff --git a/bin/varnishtest/tests/c00021.vtc b/bin/varnishtest/tests/c00021.vtc
index c4a5ef5cf..d1aff7875 100644
--- a/bin/varnishtest/tests/c00021.vtc
+++ b/bin/varnishtest/tests/c00021.vtc
@@ -112,6 +112,7 @@ client c1 {
 # header check, no header
 varnish v1 -cliok "ban req.url ~ foo && obj.http.bar == barcheck"
 varnish v1 -cliok "ban.list"
+varnish v1 -clijson "ban.list -j"
 
 client c1 {
 	txreq -url "/foo"
diff --git a/bin/varnishtest/tests/c00022.vtc b/bin/varnishtest/tests/c00022.vtc
index e2fa3e727..79a034fb2 100644
--- a/bin/varnishtest/tests/c00022.vtc
+++ b/bin/varnishtest/tests/c00022.vtc
@@ -149,6 +149,7 @@ client c1 {
 	expect resp.status == 410
 } -run
 varnish v1 -cliok "ban.list"
+varnish v1 -clijson "ban.list -j"
 
 client c1 {
 	txreq -url "/foo"
diff --git a/bin/varnishtest/tests/c00049.vtc b/bin/varnishtest/tests/c00049.vtc
index 9ff05d7f0..a442833fa 100644
--- a/bin/varnishtest/tests/c00049.vtc
+++ b/bin/varnishtest/tests/c00049.vtc
@@ -205,6 +205,7 @@ varnish v1 -cliok "ban.list"
 varnish v1 -cliok "param.set ban_lurker_age .1"
 delay 3
 
+varnish v1 -clijson "ban.list -j"
 varnish v1 -cliok "ban.list"
 
 varnish v1 -expect bans == 1
diff --git a/bin/varnishtest/tests/c00059.vtc b/bin/varnishtest/tests/c00059.vtc
index db0e6d29c..156bff35c 100644
--- a/bin/varnishtest/tests/c00059.vtc
+++ b/bin/varnishtest/tests/c00059.vtc
@@ -18,6 +18,7 @@ client c1 {
 varnish v1 -cliok "ban obj.status == 201"
 varnish v1 -cliok "ban obj.status == 200"
 varnish v1 -cliok "ban.list"
+varnish v1 -clijson "ban.list -j"
 
 client c1 {
 	txreq

From geoff at uplex.de  Mon Oct  8 11:28:12 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Mon,  8 Oct 2018 11:28:12 +0000 (UTC)
Subject: [master] 8d83019a5 Add JSON support for the "storage.list" CLI
 command.
Message-ID: <20181008112812.B107F93F4A@lists.varnish-cache.org>


commit 8d83019a575fe512da725481e458d9212d13c375
Author: Geoff Simmons <geoff at uplex.de>
Date:   Tue Sep 25 09:15:06 2018 +0200

    Add JSON support for the "storage.list" CLI command.

diff --git a/bin/varnishd/storage/mgt_stevedore.c b/bin/varnishd/storage/mgt_stevedore.c
index 3f66c3af3..c204a3e53 100644
--- a/bin/varnishd/storage/mgt_stevedore.c
+++ b/bin/varnishd/storage/mgt_stevedore.c
@@ -82,10 +82,36 @@ stv_cli_list(struct cli *cli, const char * const *av, void *priv)
 		VCLI_Out(cli, "\tstorage.%s = %s\n", stv->ident, stv->name);
 }
 
+static void v_matchproto_(cli_func_t)
+stv_cli_list_json(struct cli *cli, const char * const *av, void *priv)
+{
+	struct stevedore *stv;
+	int n = 0;
+
+	(void)priv;
+	ASSERT_MGT();
+	VCLI_JSON_begin(cli, 2, av);
+	VCLI_Out(cli, ",\n");
+	STV_Foreach(stv) {
+		VCLI_Out(cli, "%s", n ? ",\n" : "");
+		n++;
+		VCLI_Out(cli, "{\n");
+		VSB_indent(cli->sb, 2);
+		VCLI_Out(cli, "\"name\": ");
+		VCLI_JSON_str(cli, stv->ident);
+		VCLI_Out(cli, ",\n");
+		VCLI_Out(cli, "\"storage\": ");
+		VCLI_JSON_str(cli, stv->name);
+		VSB_indent(cli->sb, -2);
+		VCLI_Out(cli, "\n}");
+	}
+	VCLI_JSON_end(cli);
+}
+
 /*--------------------------------------------------------------------*/
 
 static struct cli_proto cli_stv[] = {
-	{ CLICMD_STORAGE_LIST,		"", stv_cli_list },
+	{ CLICMD_STORAGE_LIST,		"", stv_cli_list, stv_cli_list_json },
 	{ NULL}
 };
 
diff --git a/bin/varnishtest/tests/b00032.vtc b/bin/varnishtest/tests/b00032.vtc
index 7aa6cddfa..d395fef69 100644
--- a/bin/varnishtest/tests/b00032.vtc
+++ b/bin/varnishtest/tests/b00032.vtc
@@ -1,6 +1,7 @@
 varnishtest "CLI coverage test"
 
 varnish v1 -cliok storage.list
+varnish v1 -clijson "storage.list -j"
 
 server s1 {
 	rxreq

From geoff at uplex.de  Mon Oct  8 11:28:12 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Mon,  8 Oct 2018 11:28:12 +0000 (UTC)
Subject: [master] 5687c10ed Add JSON support for the "panic.show" CLI command.
Message-ID: <20181008112812.DDCD093F4E@lists.varnish-cache.org>


commit 5687c10edc88758dadbf4b9367b20d0ed862ea69
Author: Geoff Simmons <geoff at uplex.de>
Date:   Tue Sep 25 09:47:57 2018 +0200

    Add JSON support for the "panic.show" CLI command.
    
    The panic string is added as a JSON string in the output array.

diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 026bdd8e8..16050b33a 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -117,12 +117,9 @@ mgt_panic_clear(void)
 	VSB_destroy(&child_panic);
 }
 
-static void v_matchproto_(cli_func_t)
-mch_cli_panic_show(struct cli *cli, const char * const *av, void *priv)
+static void
+cli_panic_show(struct cli *cli, const char * const *av, int json)
 {
-	(void)av;
-	(void)priv;
-
 	if (!child_panic) {
 		VCLI_SetResult(cli, CLIS_CANT);
 		VCLI_Out(cli,
@@ -130,7 +127,29 @@ mch_cli_panic_show(struct cli *cli, const char * const *av, void *priv)
 		return;
 	}
 
-	VCLI_Out(cli, "%s\n", VSB_data(child_panic));
+	if (!json) {
+		VCLI_Out(cli, "%s\n", VSB_data(child_panic));
+		return;
+	}
+
+	VCLI_JSON_begin(cli, 2, av);
+	VCLI_Out(cli, ",\n");
+	VCLI_JSON_str(cli, VSB_data(child_panic));
+	VCLI_JSON_end(cli);
+}
+
+static void v_matchproto_(cli_func_t)
+mch_cli_panic_show(struct cli *cli, const char * const *av, void *priv)
+{
+	(void)priv;
+	cli_panic_show(cli, av, 0);
+}
+
+static void v_matchproto_(cli_func_t)
+mch_cli_panic_show_json(struct cli *cli, const char * const *av, void *priv)
+{
+	(void)priv;
+	cli_panic_show(cli, av, 1);
 }
 
 static void v_matchproto_(cli_func_t)
@@ -694,7 +713,8 @@ static struct cli_proto cli_mch[] = {
 	  mch_cli_server_status_json },
 	{ CLICMD_SERVER_START,		"", mch_cli_server_start },
 	{ CLICMD_SERVER_STOP,		"", mch_cli_server_stop },
-	{ CLICMD_PANIC_SHOW,		"", mch_cli_panic_show },
+	{ CLICMD_PANIC_SHOW,		"", mch_cli_panic_show,
+	  mch_cli_panic_show_json },
 	{ CLICMD_PANIC_CLEAR,		"", mch_cli_panic_clear },
 	{ NULL }
 };
diff --git a/bin/varnishtest/tests/c00057.vtc b/bin/varnishtest/tests/c00057.vtc
index 2b2f38891..5118c79a0 100644
--- a/bin/varnishtest/tests/c00057.vtc
+++ b/bin/varnishtest/tests/c00057.vtc
@@ -44,6 +44,7 @@ client c1 {
 } -run
 
 varnish v1 -cliexpect "STACK OVERFLOW" "panic.show"
+varnish v1 -clijson "panic.show -j"
 
 varnish v1 -cliok "panic.clear"
 
@@ -81,6 +82,7 @@ client c2 -connect ${v2_sock} {
 } -run
 
 varnish v2 -cliexpect "[bB]us error|Segmentation [fF]ault" "panic.show"
+varnish v2 -clijson "panic.show -j"
 
 varnish v2 -cliok "panic.clear"
 
diff --git a/bin/varnishtest/tests/v00010.vtc b/bin/varnishtest/tests/v00010.vtc
index b948d0187..c488f377f 100644
--- a/bin/varnishtest/tests/v00010.vtc
+++ b/bin/varnishtest/tests/v00010.vtc
@@ -54,6 +54,7 @@ client c1 {
 
 varnish v1 -wait-stopped
 varnish v1 -cliok "panic.show"
+varnish v1 -clijson "panic.show -j"
 varnish v1 -cliok "panic.clear"
 varnish v1 -expect MGT.child_panic == 1
 varnish v1 -clierr 300 "panic.clear"
@@ -70,6 +71,7 @@ client c1 {
 
 varnish v1 -wait-stopped
 varnish v1 -cliok "panic.show"
+varnish v1 -clijson "panic.show -j"
 varnish v1 -cliok "panic.clear -z"
 varnish v1 -expect MGT.child_panic == 0
 varnish v1 -clierr 300 "panic.clear"

From geoff at uplex.de  Mon Oct  8 11:28:13 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Mon,  8 Oct 2018 11:28:13 +0000 (UTC)
Subject: [master] 57ac35e5a Document JSON support in the CLI.
Message-ID: <20181008112813.169AA93F61@lists.varnish-cache.org>


commit 57ac35e5a24000dfd9bb7517eb2cd42588b8e9ea
Author: Geoff Simmons <geoff at uplex.de>
Date:   Tue Sep 25 11:17:23 2018 +0200

    Document JSON support in the CLI.

diff --git a/include/tbl/cli_cmds.h b/include/tbl/cli_cmds.h
index 7b3d1de19..25f13d4ac 100644
--- a/include/tbl/cli_cmds.h
+++ b/include/tbl/cli_cmds.h
@@ -49,10 +49,11 @@ CLI_CMD(BAN,
 
 CLI_CMD(BAN_LIST,
 	"ban.list",
-	"ban.list",
+	"ban.list [-j]",
 	"List the active bans.",
 
-	"  The output format is:\n\n"
+	" Unless ``-j`` is specified (for JSON output), "
+	" the output format is:\n\n"
 	"  * Time the ban was issued.\n\n"
 	"  * Objects referencing this ban.\n\n"
 	"  * ``C`` if ban is completed = no further testing against it.\n\n"
@@ -102,9 +103,9 @@ CLI_CMD(VCL_DISCARD,
 
 CLI_CMD(VCL_LIST,
 	"vcl.list",
-	"vcl.list",
+	"vcl.list [-j]",
 	"List all loaded configuration.",
-	"",
+	"``-j`` specifies JSON output.",
 	0, 0
 )
 
@@ -134,12 +135,14 @@ CLI_CMD(VCL_LABEL,
 
 CLI_CMD(PARAM_SHOW,
 	"param.show",
-	"param.show [-l] [<param>|changed]",
+	"param.show [-l|-j] [<param>|changed]",
 	"Show parameters and their values.",
 
 	"The long form with ``-l`` shows additional information, including"
 	" documentation and minimum, maximum and default values, if defined"
-	" for the parameter. If a parameter is specified with ``<param>``,"
+	" for the parameter. JSON output is specified with ``-j``, in which"
+	" the information for the long form is included; only one of ``-l`` or"
+	" ``-j`` is permitted. If a parameter is specified with ``<param>``,"
 	" show only that parameter. If ``changed`` is specified, show only"
 	" those parameters whose values differ from their defaults.",
 	0, 2
@@ -171,17 +174,17 @@ CLI_CMD(SERVER_START,
 
 CLI_CMD(PING,
 	"ping",
-	"ping [<timestamp>]",
+	"ping [-j] [<timestamp>]",
 	"Keep connection alive.",
-	"",
+	"The response is formatted as JSON if ``-j`` is specified.",
 	0, 1
 )
 
 CLI_CMD(HELP,
 	"help",
-	"help [<command>]",
+	"help [-j] [<command>]",
 	"Show command/protocol help.",
-	"",
+	"``-j`` specifies JSON output.",
 	0, 1
 )
 
@@ -195,9 +198,9 @@ CLI_CMD(QUIT,
 
 CLI_CMD(SERVER_STATUS,
 	"status",
-	"status",
+	"status [-j]",
 	"Check status of Varnish cache process.",
-	"",
+	"``-j`` specifies JSON output.",
 	0, 0
 )
 
@@ -219,9 +222,10 @@ CLI_CMD(AUTH,
 
 CLI_CMD(PANIC_SHOW,
 	"panic.show",
-	"panic.show",
+	"panic.show [-j]",
 	"Return the last panic, if any.",
-	"",
+	"``-j`` specifies JSON output -- the panic message is returned as an"
+	" unstructured JSON string.",
 	0, 0
 )
 
@@ -244,9 +248,9 @@ CLI_CMD(DEBUG_LISTEN_ADDRESS,
 
 CLI_CMD(BACKEND_LIST,
 	"backend.list",
-	"backend.list [-p] [<backend_pattern>]",
+	"backend.list [-j] [-p] [<backend_pattern>]",
 	"List backends.  -p also shows probe status.",
-	"",
+	"``-j`` specifies JSON output.",
 	0, 2
 )
 
@@ -320,9 +324,9 @@ CLI_CMD(DEBUG_PERSISTENT,
 
 CLI_CMD(STORAGE_LIST,
 	"storage.list",
-	"storage.list",
+	"storage.list [-j]",
 	"List storage devices.",
-	"",
+	"``-j`` specifies JSON output.",
 	0, 0
 )
 

From geoff at uplex.de  Mon Oct  8 11:28:13 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Mon,  8 Oct 2018 11:28:13 +0000 (UTC)
Subject: [master] 24964525a Add a JSON section to varnish-cli(7).
Message-ID: <20181008112813.425D893F7F@lists.varnish-cache.org>


commit 24964525aba9f05245e80e4182236253d4019be3
Author: Geoff Simmons <geoff at uplex.de>
Date:   Tue Oct 2 14:12:02 2018 +0200

    Add a JSON section to varnish-cli(7).
    
    Closes #2783

diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst
index 53387672a..38364cede 100644
--- a/doc/sphinx/reference/varnish-cli.rst
+++ b/doc/sphinx/reference/varnish-cli.rst
@@ -170,6 +170,38 @@ Other pitfalls include variable expansion of the shell invoking ``varnishadm``
 but this is not directly related to the Varnish CLI. If you get the quoting
 right you should be fine even with complex commands.
 
+JSON
+----
+
+A number of commands with informational responses support a ``-j`` parameter
+for JSON output, as specified below. The top-level structure of the JSON
+response is an array with these first three elements:
+
+* A version number for the JSON format (integer)
+
+* An array of strings that comprise the CLI command just received
+
+* The time at which the response was generated, as a Unix epoch time
+  in seconds with millisecond precision (floating point)
+
+The remaining elements of the array form the data that are specific to
+the CLI command, and their structure and content depend on the
+command.
+
+For example, the response to ``status -j`` just contains a string in
+the top-level array indicating the state of the child process
+(``"running"``, ``"stopped"`` and so forth)::
+
+  [ 2, ["status", "-j"], 1538031732.632, "running"
+  ]
+
+The JSON responses to other commands may have longer lists of
+elements, which may have simple data types or form structured objects.
+
+JSON output is only returned if command execution was successful. The
+output for an error response is always the same as it would have been
+for the command without the ``-j`` parameter.
+
 Commands
 --------
 

From geoff at uplex.de  Mon Oct  8 12:38:13 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Mon,  8 Oct 2018 12:38:13 +0000 (UTC)
Subject: [master] 7b6bd37b2 Document extended CLI JSON support in 'Changes in
 $NEXT_RELEASE'
Message-ID: <20181008123813.C6D2396BF9@lists.varnish-cache.org>


commit 7b6bd37b227a2ae9b8ce5e0617992fcd01ad30d2
Author: Geoff Simmons <geoff at uplex.de>
Date:   Mon Oct 8 14:36:46 2018 +0200

    Document extended CLI JSON support in 'Changes in $NEXT_RELEASE'

diff --git a/doc/sphinx/whats-new/changes-trunk.rst b/doc/sphinx/whats-new/changes-trunk.rst
index fbda9d6b9..7bac84756 100644
--- a/doc/sphinx/whats-new/changes-trunk.rst
+++ b/doc/sphinx/whats-new/changes-trunk.rst
@@ -54,6 +54,16 @@ varnishadm
 
 **XXX changes concerning varnishadm(1) and/or varnish-cli(7)**
 
+JSON responses, requested with the ``-j`` option, are now possible for
+the following commands (see :ref:`varnish-cli(7)`):
+
+* ``status -j``
+* ``vcl.list -j``
+* ``param.show -j``
+* ``ban.list -j``
+* ``storage.list -j``
+* ``panic.show -j``
+
 varnishstat
 ===========
 

From geoff at uplex.de  Mon Oct  8 12:42:07 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Mon,  8 Oct 2018 12:42:07 +0000 (UTC)
Subject: [master] b607c202e Document #2783 (JSON support in the CLI) in
 changes.rst.
Message-ID: <20181008124208.30C2B96ECE@lists.varnish-cache.org>


commit b607c202e4c7f145b4535b128e57297a20d99219
Author: Geoff Simmons <geoff at uplex.de>
Date:   Mon Oct 8 14:40:59 2018 +0200

    Document #2783 (JSON support in the CLI) in changes.rst.

diff --git a/doc/changes.rst b/doc/changes.rst
index 581ee514b..d198e8aa1 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -30,6 +30,8 @@ release process.
 Varnish Cache trunk (ongoing)
 =============================
 
+* Extend JSON support in the CLI (2783_)
+
 * fixed ``varnishhist`` display error (2780_)
 
 * ``libvarnish``: ``VRT_VSA_GetPtr`` renamed to ``VSA_GetPtr``
@@ -62,6 +64,7 @@ Varnish Cache trunk (ongoing)
   an unnecessary incompatibility with VSL files written by previous
   versions. (2790_)
 
+.. _2783: https://github.com/varnishcache/varnish-cache/pull/2783
 .. _2780: https://github.com/varnishcache/varnish-cache/issues/2780
 .. _2782: https://github.com/varnishcache/varnish-cache/issues/2782
 .. _2787: https://github.com/varnishcache/varnish-cache/issues/2787

From nils.goroll at uplex.de  Tue Oct  9 08:05:14 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  9 Oct 2018 08:05:14 +0000 (UTC)
Subject: [master] eeed23f9c typedefs for real/mono time and durations
Message-ID: <20181009080514.2FFF19B3F@lists.varnish-cache.org>


commit eeed23f9c46418083dde94d1ccaccfd12d1c9108
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 8 10:19:50 2018 +0200

    typedefs for real/mono time and durations
    
    We use double for all time representations, yet monotonic time,
    real time and durations are not to be confused.
    
    Also, we might want to change the representation of time in the
    future.
    
    To get an implicit documentation of the semantic type of a time
    value and to facilitate working on the latter, we start off by
    introducing simple typedefs which neither inply any change nor
    offer any type checking. But they pave the way...

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index e62bc9e8d..b63664706 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -252,7 +252,7 @@ struct worker {
 
 	struct pool_task	task;
 
-	double			lastused;
+	vtim_real		lastused;
 	int			strangelove;
 
 	struct v1l		*v1l;
@@ -414,9 +414,9 @@ struct busyobj {
 #include "tbl/bo_flags.h"
 
 	/* Timeouts */
-	double			connect_timeout;
-	double			first_byte_timeout;
-	double			between_bytes_timeout;
+	vtim_dur		connect_timeout;
+	vtim_dur		first_byte_timeout;
+	vtim_dur		between_bytes_timeout;
 
 	/* Timers */
 	double			t_first;	/* First timestamp logged */
@@ -565,9 +565,8 @@ struct sess {
 
 	struct ws		ws[1];
 
-	/* Timestamps, all on TIM_real() timescale */
-	double			t_open;		/* fd accepted */
-	double			t_idle;		/* fd accepted or resp sent */
+	vtim_real		t_open;		/* fd accepted */
+	vtim_real		t_idle;		/* fd accepted or resp sent */
 
 };
 
@@ -663,7 +662,7 @@ int Lck__Owned(const struct lock *lck);
 
 /* public interface: */
 void Lck_Delete(struct lock *lck);
-int Lck_CondWait(pthread_cond_t *cond, struct lock *lck, double);
+int Lck_CondWait(pthread_cond_t *cond, struct lock *lck, vtim_real);
 
 #define Lck_New(a, b) Lck__New(a, b, #b)
 #define Lck_Lock(a) Lck__Lock(a, __func__, __LINE__)
diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 2b920720a..c0d44850f 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -52,7 +52,7 @@
 #include "vtim.h"
 
 static pthread_t	VCA_thread;
-static double vca_pace = 0.0;
+static vtim_dur vca_pace = 0.0;
 static struct lock pace_mtx;
 static unsigned pool_accepting;
 static pthread_mutex_t shut_mtx = PTHREAD_MUTEX_INITIALIZER;
@@ -137,9 +137,9 @@ static unsigned		need_test;
  * into the vca_acct() loop which we are running anyway
  */
 static void
-vca_periodic(double t0)
+vca_periodic(vtim_real t0)
 {
-	double now;
+	vtim_real now;
 
 	now = VTIM_real();
 	VSC_C_main->uptime = (uint64_t)(now - t0);
@@ -270,7 +270,7 @@ vca_tcp_opt_set(const int sock, const unsigned uds, const int force)
 static void
 vca_pace_check(void)
 {
-	double p;
+	vtim_dur p;
 
 	if (vca_pace == 0.0)
 		return;
@@ -598,7 +598,7 @@ static void * v_matchproto_()
 vca_acct(void *arg)
 {
 	struct listen_sock *ls;
-	double t0;
+	vtim_real t0;
 
 	// XXX Actually a mis-nomer now because the accept happens in a pool
 	// thread. Rename to accept-nanny or so?
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 9f08c2385..c714af99f 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -114,7 +114,7 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo,
 {
 	struct pfd *pfd;
 	int *fdp, err;
-	double tmod;
+	vtim_dur tmod;
 	char abuf1[VTCP_ADDRBUFSIZE], abuf2[VTCP_ADDRBUFSIZE];
 	char pbuf1[VTCP_PORTBUFSIZE], pbuf2[VTCP_PORTBUFSIZE];
 
@@ -619,7 +619,7 @@ void
 VBE_Poll(void)
 {
 	struct backend *be, *be2;
-	double now = VTIM_real();
+	vtim_real now = VTIM_real();
 
 	ASSERT_CLI();
 	Lck_Lock(&backends_mtx);
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index f5c016860..796be9b32 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -66,7 +66,7 @@ struct backend {
 
 	VCL_BACKEND		director;
 
-	double			cooled;
+	vtim_real		cooled;
 };
 
 /*---------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c
index 1b1190ebd..899847c81 100644
--- a/bin/varnishd/cache/cache_backend_probe.c
+++ b/bin/varnishd/cache/cache_backend_probe.c
@@ -76,11 +76,11 @@ struct vbp_target {
 #define BITMAP(n, c, t, b)	uintmax_t	n;
 #include "tbl/backend_poll.h"
 
-	double				last;
-	double				avg;
+	vtim_dur			last;
+	vtim_dur			avg;
 	double				rate;
 
-	double				due;
+	vtim_real			due;
 	int				running;
 	int				heap_idx;
 	struct pool_task		task;
@@ -278,7 +278,7 @@ static void
 vbp_poke(struct vbp_target *vt)
 {
 	int s, tmo, i, proxy_header, err;
-	double t_start, t_now, t_end;
+	vtim_real t_start, t_now, t_end;
 	unsigned rlen, resp;
 	char buf[8192], *p;
 	struct pollfd pfda[1], *pfd = pfda;
@@ -454,7 +454,7 @@ vbp_task(struct worker *wrk, void *priv)
 static void * v_matchproto_(bgthread_t)
 vbp_thread(struct worker *wrk, void *priv)
 {
-	double now, nxt;
+	vtim_real now, nxt;
 	struct vbp_target *vt;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c
index 6fa816a68..ad182c4b0 100644
--- a/bin/varnishd/cache/cache_lck.c
+++ b/bin/varnishd/cache/cache_lck.c
@@ -187,7 +187,7 @@ Lck__Owned(const struct lock *lck)
 }
 
 int v_matchproto_()
-Lck_CondWait(pthread_cond_t *cond, struct lock *lck, double when)
+Lck_CondWait(pthread_cond_t *cond, struct lock *lck, vtim_real when)
 {
 	struct ilck *ilck;
 	struct timespec ts;
diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c
index f33668b2f..5a04eb5aa 100644
--- a/bin/varnishd/cache/cache_tcp_pool.c
+++ b/bin/varnishd/cache/cache_tcp_pool.c
@@ -540,7 +540,7 @@ VCP_Get(struct conn_pool *cp, double tmo, struct worker *wrk,
  */
 
 static int
-VCP_Wait(struct worker *wrk, struct pfd *pfd, double tmo)
+VCP_Wait(struct worker *wrk, struct pfd *pfd, vtim_real tmo)
 {
 	struct conn_pool *cp;
 	int r;
@@ -834,7 +834,7 @@ VTP_Close(struct pfd **pfdp)
  */
 
 struct pfd *
-VTP_Get(struct tcp_pool *tp, double tmo, struct worker *wrk,
+VTP_Get(struct tcp_pool *tp, vtim_dur tmo, struct worker *wrk,
 	unsigned force_fresh, int *err)
 {
 
@@ -845,7 +845,7 @@ VTP_Get(struct tcp_pool *tp, double tmo, struct worker *wrk,
  */
 
 int
-VTP_Wait(struct worker *wrk, struct pfd *pfd, double tmo)
+VTP_Wait(struct worker *wrk, struct pfd *pfd, vtim_real tmo)
 {
 	return (VCP_Wait(wrk, pfd, tmo));
 }
diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h
index 7c25afda2..57564ba96 100644
--- a/bin/varnishd/cache/cache_tcp_pool.h
+++ b/bin/varnishd/cache/cache_tcp_pool.h
@@ -88,14 +88,14 @@ void VTP_Recycle(const struct worker *, struct pfd **);
 	 * Recycle an open connection.
 	 */
 
-struct pfd *VTP_Get(struct tcp_pool *, double tmo, struct worker *,
+struct pfd *VTP_Get(struct tcp_pool *, vtim_dur tmo, struct worker *,
     unsigned force_fresh, int *err);
 	/*
 	 * Get a (possibly) recycled connection.
 	 * errno will be stored in err
 	 */
 
-int VTP_Wait(struct worker *, struct pfd *, double tmo);
+int VTP_Wait(struct worker *, struct pfd *, vtim_real tmo);
 	/*
 	 * If the connection was recycled (state != VTP_STATE_USED) call this
 	 * function before attempting to receive on the connection.
diff --git a/include/vdef.h b/include/vdef.h
index 185265794..3a5e40329 100644
--- a/include/vdef.h
+++ b/include/vdef.h
@@ -165,3 +165,8 @@
 #   define ___Static_assert(x, y) \
 		typedef char __assert_## y[(x) ? 1 : -1] v_unused_
 #endif
+
+/* VTIM API overhaul WIP */
+typedef double vtim_mono;
+typedef double vtim_real;
+typedef double vtim_dur;
diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index c774e2ab8..14b9752d6 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -124,7 +124,7 @@ init(void)
  * gethrtime(), which is the case on modern Solaris descendents.
  */
 
-double
+vtim_mono
 VTIM_mono(void)
 {
 #if defined(HAVE_CLOCK_GETTIME) && !defined(USE_GETHRTIME)
@@ -143,7 +143,7 @@ VTIM_mono(void)
 #endif
 }
 
-double
+vtim_real
 VTIM_real(void)
 {
 #ifdef HAVE_CLOCK_GETTIME
@@ -160,7 +160,7 @@ VTIM_real(void)
 }
 
 void
-VTIM_format(double t, char *p)
+VTIM_format(vtim_real t, char *p)
 {
 	struct tm tm;
 	time_t tt;
@@ -236,10 +236,10 @@ VTIM_format(double t, char *p)
 		DIGIT(1, sec);					\
 	} while(0)
 
-double
+vtim_real
 VTIM_parse(const char *p)
 {
-	double t;
+	vtim_real t;
 	int month = 0, year = 0, weekday = -1, mday = 0;
 	int hour = 0, min = 0, sec = 0;
 	int d, leap;
@@ -394,7 +394,7 @@ VTIM_parse(const char *p)
 }
 
 void
-VTIM_sleep(double t)
+VTIM_sleep(vtim_dur t)
 {
 #ifdef HAVE_NANOSLEEP
 	struct timespec ts;
@@ -415,7 +415,7 @@ VTIM_sleep(double t)
 }
 
 struct timeval
-VTIM_timeval(double t)
+VTIM_timeval(vtim_real t)
 {
 	struct timeval tv;
 
@@ -426,7 +426,7 @@ VTIM_timeval(double t)
 }
 
 struct timespec
-VTIM_timespec(double t)
+VTIM_timespec(vtim_real t)
 {
 	struct timespec tv;
 
@@ -464,8 +464,9 @@ tst(const char *s, time_t good)
 	}
 }
 
+/* XXX keep as double for the time being */
 static int
-tst_delta_check(const char *name, double begin, double end, double ref)
+tst_delta_check(const char *name, double begin, double end, vtim_dur ref)
 {
 	const double tol_max = 1.1;
 	const double tol_min = 1;
@@ -487,9 +488,9 @@ tst_delta_check(const char *name, double begin, double end, double ref)
 static void
 tst_delta()
 {
-	double m_begin, m_end;
-	double r_begin, r_end;
-	const double ref = 1;
+	vtim_mono m_begin, m_end;
+	vtim_real r_begin, r_end;
+	const vtim_dur ref = 1;
 	int err = 0;
 
 	r_begin = VTIM_real();
@@ -510,53 +511,56 @@ tst_delta()
 static void
 bench()
 {
-	double s, e, t;
+	vtim_mono s, e;
+	vtim_mono t_m;
+	vtim_real t_r;
+	unsigned long t_i;
 	int i;
 	char buf[64];
 
-	t = 0;
-	s = VTIM_real();
+	t_r = 0;
+	s = VTIM_mono();
 	for (i=0; i<100000; i++)
-		t += VTIM_real();
-	e = VTIM_real();
+		t_r += VTIM_real();
+	e = VTIM_mono();
 	printf("real: %fs / %d = %fns - tst val %f\n",
-	    e - s, i, 1e9 * (e - s) / i, t);
+	    e - s, i, 1e9 * (e - s) / i, t_r);
 
-	t = 0;
-	s = VTIM_real();
+	t_i = 0;
+	s = VTIM_mono();
 	for (i=0; i<100000; i++)
-		t += VTIM_mono();
-	e = VTIM_real();
+		t_m += VTIM_mono();
+	e = VTIM_mono();
 	printf("mono: %fs / %d = %fns - tst val %f\n",
-	    e - s, i, 1e9 * (e - s) / i, t);
+	    e - s, i, 1e9 * (e - s) / i, t_m);
 
-	t = 0;
+	t_i = 0;
 	s = VTIM_mono();
 	for (i=0; i<100000; i++) {
 		snprintf(buf, sizeof(buf), "%.6f", s);
-		t += buf[4];
+		t_i += buf[4];
 	}
 	e = VTIM_mono();
-	printf("printf %%.6f: %fs / %d = %fns - tst val %f %s\n",
-	    e - s, i, 1e9 * (e - s) / i, t, buf);
+	printf("printf %%.6f: %fs / %d = %fns - tst val %lu %s\n",
+	    e - s, i, 1e9 * (e - s) / i, t_i, buf);
 
-	t = 0;
+	t_i = 0;
 	s = VTIM_mono();
 	for (i=0; i<100000; i++) {
 		snprintf(buf, sizeof(buf), "%ju.%06ju",
 		    (uint64_t)floor(s),
 		    (uint64_t)floor((s * 1e6)) % 1000000UL);
-		t += buf[4];
+		t_i += buf[4];
 	}
 	e = VTIM_mono();
-	printf("printf %%ju.%%06ju: %fs / %d = %fns - tst val %f %s\n",
-	    e - s, i, 1e9 * (e - s) / i, t, buf);
+	printf("printf %%ju.%%06ju: %fs / %d = %fns - tst val %lu %s\n",
+	    e - s, i, 1e9 * (e - s) / i, t_i, buf);
 }
 
 void
 parse_check(time_t t, const char *s)
 {
-	double tt;
+	vtim_real tt;
 	char buf[BUFSIZ];
 
 	tt = VTIM_parse(s);

From phk at FreeBSD.org  Tue Oct  9 09:09:13 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  9 Oct 2018 09:09:13 +0000 (UTC)
Subject: [master] 094d11b31 Add python2/3 hack
Message-ID: <20181009090913.45E43610E9@lists.varnish-cache.org>


commit 094d11b31c2ddf4a6fdb1ac8c20e9bfd8793ab00
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 9 08:36:02 2018 +0000

    Add python2/3 hack

diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py
index eb795e076..f1e0d02ab 100644
--- a/lib/libvcc/vsctool.py
+++ b/lib/libvcc/vsctool.py
@@ -41,6 +41,7 @@ import getopt
 import json
 import sys
 import collections
+import codecs
 
 # Parameters of 'varnish_vsc_begin', first element is default
 TYPES = ["counter", "gauge", "bitmap"]
@@ -164,7 +165,12 @@ class CounterSet(object):
         assert self.completed
 
         fon = "VSC_" + self.name + ".h"
-        fo = open(fon, "w")
+        try:
+            # Python3
+            fo = open(fon, "w", encoding="UTF-8")
+        except TypeError:
+            # Python2
+            fo = open(fon, "w")
         genhdr(fo, self.name)
 
         fo.write(self.struct + " {\n")
@@ -288,7 +294,12 @@ class CounterSet(object):
         '''Emit .c file'''
         assert self.completed
         fon = "VSC_" + self.name + ".c"
-        fo = open(fon, "w")
+        try:
+            # Python3
+            fo = open(fon, "w", encoding="UTF-8")
+        except TypeError:
+            # Python2
+            fo = open(fon, "w")
         genhdr(fo, self.name)
         fo.write('#include "config.h"\n')
         fo.write('#include <stdio.h>\n')
@@ -437,10 +448,22 @@ def mainfunc(argv):
     rstfile = None
     for f, v in optlist:
         if f == '-r':
+            try:
+                # Python3
+                sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
+            except AttributeError:
+                # Python2
+                pass
             rstfile = sys.stdout
 
     vscset = []
-    scs = open(args[0]).read().split("\n.. ")
+    try:
+        # Python3
+        f = open(args[0], encoding="UTF-8")
+    except TypeError:
+        # Python2
+        f = open(args[0])
+    scs = f.read().split("\n.. ")
     if rstfile:
         rstfile.write(scs[0])
     for i in scs[1:]:

From phk at FreeBSD.org  Tue Oct  9 09:09:13 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  9 Oct 2018 09:09:13 +0000 (UTC)
Subject: [master] fa33ec523 Add a crude python2/python3 hack to deal with UTF-8
Message-ID: <20181009090913.29B2C610E6@lists.varnish-cache.org>


commit fa33ec52361eddeb03f3bb0129663a7149a2e045
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 9 08:25:36 2018 +0000

    Add a crude python2/python3 hack to deal with UTF-8

diff --git a/doc/sphinx/vtc-syntax.py b/doc/sphinx/vtc-syntax.py
index ba7bd6bd7..d98fc96e5 100644
--- a/doc/sphinx/vtc-syntax.py
+++ b/doc/sphinx/vtc-syntax.py
@@ -39,7 +39,12 @@ def parse_file(fn, cl, tl, sl):
     section = ""
     resec = re.compile("[ /]\* SECTION: ")
 
-    f = open(fn, "r")
+    try:
+        # Python3
+        f = open(fn, "r", encoding="UTF-8")
+    except TypeError:
+        # Python2
+        f = open(fn, "r")
 
     for l in f:
         if "*/" in l:

From phk at FreeBSD.org  Tue Oct  9 09:09:13 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  9 Oct 2018 09:09:13 +0000 (UTC)
Subject: [master] aa893088e Renovate this Makefile
Message-ID: <20181009090913.5F128610EE@lists.varnish-cache.org>


commit aa893088e6ba2648d5b9456bf65bf087d16654cf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 9 09:05:24 2018 +0000

    Renovate this Makefile
    
    When producing files with "foo > file", always use the pattern:
    
            foo > file.tmp
            mv file.tmp file
    
    Otherwise program failures end up generating partial content and
    make will not even rerun the failing program next time you type make.
    
    Actually clean CLEANFILES in the clean target.
    
    The reference dir is not built, but it should be in the distfile

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 54f930caf..cf67a6106 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -25,7 +25,7 @@ help:
 	@echo "  doctest   to run all doctests embedded in the documentation (if enabled)"
 
 clean:
-	-rm -rf $(BUILDDIR)/*
+	-rm -rf $(BUILDDIR)/* $(CLEANFILES)
 
 # work around for make html called within doc/sphinx
 .PHONY: graphviz
@@ -101,6 +101,7 @@ EXTRA_DIST = \
 	installation \
 	phk \
 	tutorial \
+	reference \
 	users-guide \
 	vtc-syntax.py \
 	whats-new
@@ -116,11 +117,13 @@ distclean-local:
 	rm -rf $(BUILDDIR)
 
 include/cli.rst: $(top_builddir)/bin/varnishd/varnishd
-	$(top_builddir)/bin/varnishd/varnishd -x cli > $@
+	$(top_builddir)/bin/varnishd/varnishd -x cli > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES = include/cli.rst
 
 include/params.rst: $(top_builddir)/bin/varnishd/varnishd
-	$(top_builddir)/bin/varnishd/varnishd -x parameter > $@
+	$(top_builddir)/bin/varnishd/varnishd -x parameter > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/params.rst
 
 COUNTERS = \
@@ -134,52 +137,64 @@ COUNTERS = \
 	$(top_srcdir)/bin/varnishd/VSC_lck.vsc
 
 include/counters.rst: $(top_srcdir)/lib/libvcc/vsctool.py $(COUNTERS)
-	echo -n '' > $@
+	echo -n '' > ${@}_
 	for i in $(COUNTERS); do \
-		$(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -r $$i >> $@ ; \
+		$(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -r $$i >> ${@}_ ; \
 	done
+	mv ${@}_ ${@}
 
 BUILT_SOURCES += include/counters.rst
 
 # XXX add varnishstat here when it's been _opt2rst'ed
 
 include/varnishncsa_options.rst: $(top_builddir)/bin/varnishncsa/varnishncsa
-	$(top_builddir)/bin/varnishncsa/varnishncsa --options > $@
+	$(top_builddir)/bin/varnishncsa/varnishncsa --options > ${@}_
+	mv ${@}_ ${@}
 include/varnishncsa_synopsis.rst: $(top_builddir)/bin/varnishncsa/varnishncsa
-	$(top_builddir)/bin/varnishncsa/varnishncsa --synopsis > $@
+	$(top_builddir)/bin/varnishncsa/varnishncsa --synopsis > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/varnishncsa_options.rst \
 	include/varnishncsa_synopsis.rst
 
 include/varnishlog_options.rst: $(top_builddir)/bin/varnishlog/varnishlog
-	$(top_builddir)/bin/varnishlog/varnishlog --options > $@
+	$(top_builddir)/bin/varnishlog/varnishlog --options > ${@}_
+	mv ${@}_ ${@}
 include/varnishlog_synopsis.rst: $(top_builddir)/bin/varnishlog/varnishlog
-	$(top_builddir)/bin/varnishlog/varnishlog --synopsis > $@
+	$(top_builddir)/bin/varnishlog/varnishlog --synopsis > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/varnishlog_options.rst \
 	include/varnishlog_synopsis.rst
 
 include/varnishtop_options.rst: $(top_builddir)/bin/varnishtop/varnishtop
-	$(top_builddir)/bin/varnishtop/varnishtop --options > $@
+	$(top_builddir)/bin/varnishtop/varnishtop --options > ${@}_
+	mv ${@}_ ${@}
 include/varnishtop_synopsis.rst: $(top_builddir)/bin/varnishtop/varnishtop
-	$(top_builddir)/bin/varnishtop/varnishtop --synopsis > $@
+	$(top_builddir)/bin/varnishtop/varnishtop --synopsis > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/varnishtop_options.rst \
 	 include/varnishtop_synopsis.rst
 
 include/varnishhist_options.rst: $(top_builddir)/bin/varnishhist/varnishhist
-	$(top_builddir)/bin/varnishhist/varnishhist --options > $@
+	$(top_builddir)/bin/varnishhist/varnishhist --options > ${@}_
+	mv ${@}_ ${@}
 include/varnishhist_synopsis.rst: $(top_builddir)/bin/varnishhist/varnishhist
-	$(top_builddir)/bin/varnishhist/varnishhist --synopsis > $@
+	$(top_builddir)/bin/varnishhist/varnishhist --synopsis > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/varnishhist_options.rst \
 	 include/varnishhist_synopsis.rst
 
 include/varnishstat_options.rst: $(top_builddir)/bin/varnishstat/varnishstat
-	$(top_builddir)/bin/varnishstat/varnishstat --options > $@
+	$(top_builddir)/bin/varnishstat/varnishstat --options > ${@}_
+	mv ${@}_ ${@}
 include/varnishstat_synopsis.rst: $(top_builddir)/bin/varnishstat/varnishstat
-	$(top_builddir)/bin/varnishstat/varnishstat --synopsis > $@
+	$(top_builddir)/bin/varnishstat/varnishstat --synopsis > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/varnishstat_options.rst \
 	 include/varnishstat_synopsis.rst
 
 include/vsl-tags.rst: $(top_builddir)/lib/libvarnishapi/vsl2rst
-	$(top_builddir)/lib/libvarnishapi/vsl2rst > $@
+	$(top_builddir)/lib/libvarnishapi/vsl2rst > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/vsl-tags.rst
 
 VTCSYN_SRC = $(top_srcdir)/bin/varnishtest/vtc.c \
@@ -193,14 +208,10 @@ VTCSYN_SRC = $(top_srcdir)/bin/varnishtest/vtc.c \
 	     $(top_srcdir)/bin/varnishtest/vtc_syslog.c \
 	     $(top_srcdir)/bin/varnishtest/vtc_varnish.c
 include/vtc-syntax.rst: vtc-syntax.py $(VTCSYN_SRC)
-	$(PYTHON) $(top_srcdir)/doc/sphinx/vtc-syntax.py $(VTCSYN_SRC) > $@
+	$(PYTHON) $(top_srcdir)/doc/sphinx/vtc-syntax.py $(VTCSYN_SRC) > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/vtc-syntax.rst
 
-.PHONY: reference
-reference:
-	test -d $@ || mkdir $@
-BUILT_SOURCES += reference
-
 reference/vmod_std.generated.rst: reference $(top_builddir)/lib/libvmod_std/vmod_std.rst
 	cp $(top_builddir)/lib/libvmod_std/vmod_std.rst $@ || true
 BUILT_SOURCES += reference/vmod_std.generated.rst
@@ -231,5 +242,6 @@ BUILT_SOURCES += reference/vmod_proxy.generated.rst
 
 EXTRA_DIST += $(BUILT_SOURCES)
 MAINTAINERCLEANFILES = $(EXTRA_DIST)
+CLEANFILES = $(BUILT_SOURCES)
 
 .NOPATH: $(BUILT_SOURCES)

From nils.goroll at uplex.de  Tue Oct  9 10:02:11 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  9 Oct 2018 10:02:11 +0000 (UTC)
Subject: [master] 6788ed48a Clarify and test object slimming for hfp+hfm
Message-ID: <20181009100211.27E1F6376B@lists.varnish-cache.org>


commit 6788ed48a04f183916c0b6f1647d4c106ec1af9c
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Sep 7 12:32:18 2018 +0200

    Clarify and test object slimming for hfp+hfm
    
    The previous code was correct already, but we can make it
    clearer that HFP implies OC_F_PASS
    
    Also test explicitly that both HFM and HFP have their objects
    slimmed.
    
    Closes #2768

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index c1eece20d..b6b45dcb7 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -417,7 +417,9 @@ cnt_transmit(struct worker *wrk, struct req *req)
 
 	VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
 
-	if (req->objcore->flags & (OC_F_PRIVATE | OC_F_PASS)) {
+	if (req->objcore->flags & (OC_F_PRIVATE | OC_F_PASS | OC_F_HFP)) {
+		if (req->objcore->flags & OC_F_HFP)
+			AN(req->objcore->flags & OC_F_PASS);
 		if (boc != NULL) {
 			HSH_Abandon(req->objcore);
 			ObjWaitState(req->objcore, BOS_FINISHED);
diff --git a/bin/varnishtest/tests/r01821.vtc b/bin/varnishtest/tests/r01821.vtc
index af6e934cb..b07799325 100644
--- a/bin/varnishtest/tests/r01821.vtc
+++ b/bin/varnishtest/tests/r01821.vtc
@@ -1,4 +1,6 @@
-varnishtest "Slim down hit-for-miss objects"
+varnishtest "Slim down hit-for-miss / hit-for-miss objects"
+
+# see also #2768
 
 server s1 -repeat 2 {
 	rxreq
@@ -7,20 +9,32 @@ server s1 -repeat 2 {
 
 varnish v1 -arg "-s Transient=default" -vcl+backend {
 	sub vcl_backend_response {
-		set beresp.uncacheable = true;
+		if (bereq.url == "/hfm") {
+			set beresp.uncacheable = true;
+		} else if (bereq.url == "/hfp") {
+			return (pass(1m));
+		}
 	}
 } -start
 
-logexpect l1 -v v1 {
+logexpect l1 -v v1 -g raw {
+	expect * *	Storage		"Transient"
 	expect * *	Storage		"Transient"
 } -start
 
 client c1 {
-	txreq
+	txreq -url "/hfm"
+	rxresp
+} -start
+
+client c2 {
+	txreq -url "/hfp"
 	rxresp
 } -run
 
+client c1 -wait
+
 logexpect l1 -wait
 
-varnish v1 -expect SM?.Transient.c_bytes != 0
+varnish v1 -expect SM?.Transient.c_bytes > 131072
 varnish v1 -expect SM?.Transient.g_bytes < 65536

From nils.goroll at uplex.de  Tue Oct  9 10:50:05 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  9 Oct 2018 10:50:05 +0000 (UTC)
Subject: [master] 4dcb8fb92 keep the dog in the kennel unless there's a queue
Message-ID: <20181009105005.BE01164778@lists.varnish-cache.org>


commit 4dcb8fb925dff2597311d069818644d57451c7d6
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Oct 9 12:38:25 2018 +0200

    keep the dog in the kennel unless there's a queue
    
    As long as we are not queuing any threads, there is no queue to
    move. We record the queue marker the first time we notice queuing and
    only then see if it doesn't move.
    
    Fixes #2794

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 3d4411cb4..9b26013cd 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -510,14 +510,15 @@ pool_herder(void *priv)
 		 * Instead we implement a watchdog and kill the worker if
 		 * nothing has been dequeued for that long.
 		 */
-		if (dq != pp->ndequeued) {
+		if (pp->lqueue == 0) {
+			dq = pp->ndequeued + 1;
+		} else if (dq != pp->ndequeued) {
 			dq = pp->ndequeued;
 			dqt = VTIM_real();
-		} else if (pp->lqueue &&
-		    VTIM_real() - dqt > cache_param->wthread_watchdog) {
+		} else if (VTIM_real() - dqt > cache_param->wthread_watchdog) {
 			VSL(SLT_Error, 0,
-			   "Pool Herder: Queue does not move ql=%u dt=%f",
-			   pp->lqueue, VTIM_real() - dqt);
+			    "Pool Herder: Queue does not move ql=%u dt=%f",
+			    pp->lqueue, VTIM_real() - dqt);
 			WRONG("Worker Pool Queue does not move");
 		}
 		wthread_min = cache_param->wthread_min;

From nils.goroll at uplex.de  Tue Oct  9 10:57:07 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue,  9 Oct 2018 10:57:07 +0000 (UTC)
Subject: [master] 827cd226e use monotonic time for the watchdog
Message-ID: <20181009105707.B203D64ACB@lists.varnish-cache.org>


commit 827cd226e42cd1048262b4260ee151eb602486ce
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Oct 9 12:56:41 2018 +0200

    use monotonic time for the watchdog

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 9b26013cd..bfbfae374 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -489,7 +489,7 @@ pool_herder(void *priv)
 	double delay;
 	int wthread_min;
 	uintmax_t dq = (1ULL << 31);
-	double dqt = 0;
+	vtim_mono dqt = 0;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 
@@ -514,11 +514,11 @@ pool_herder(void *priv)
 			dq = pp->ndequeued + 1;
 		} else if (dq != pp->ndequeued) {
 			dq = pp->ndequeued;
-			dqt = VTIM_real();
-		} else if (VTIM_real() - dqt > cache_param->wthread_watchdog) {
+			dqt = VTIM_mono();
+		} else if (VTIM_mono() - dqt > cache_param->wthread_watchdog) {
 			VSL(SLT_Error, 0,
 			    "Pool Herder: Queue does not move ql=%u dt=%f",
-			    pp->lqueue, VTIM_real() - dqt);
+			    pp->lqueue, VTIM_mono() - dqt);
 			WRONG("Worker Pool Queue does not move");
 		}
 		wthread_min = cache_param->wthread_min;

From fgsch at lodoss.net  Tue Oct  9 13:10:15 2018
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue,  9 Oct 2018 13:10:15 +0000 (UTC)
Subject: [master] 42b8179fd Rearrange to avoid a tiny leak
Message-ID: <20181009131015.6A6909153E@lists.varnish-cache.org>


commit 42b8179fd1bf44439094d7c09a2578baa13be1a0
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Oct 9 14:07:50 2018 +0100

    Rearrange to avoid a tiny leak

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index f3faa9cb3..9c6a74685 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -353,8 +353,6 @@ mcf_param_show_json(struct cli *cli, const char * const *av, void *priv)
 	struct vsb *vsb, *def;
 	const char *show = NULL;
 
-	vsb = VSB_new_auto();
-	def = VSB_new_auto();
 	(void)priv;
 
 	for (int i = 2; av[i] != NULL; i++) {
@@ -372,6 +370,9 @@ mcf_param_show_json(struct cli *cli, const char * const *av, void *priv)
 		show = av[i];
 	}
 
+	vsb = VSB_new_auto();
+	def = VSB_new_auto();
+
 	n = 0;
 	VCLI_JSON_begin(cli, 2, av);
 	VCLI_Out(cli, ",\n");

From phk at FreeBSD.org  Wed Oct 10 10:00:25 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 10 Oct 2018 10:00:25 +0000 (UTC)
Subject: [master] d75c2a1f9 Split INIT/FINI into separate VDP methods
Message-ID: <20181010100025.F2F4E40E5@lists.varnish-cache.org>


commit d75c2a1f98b33fef0d3bfcb531e81a6660a4de21
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 10 09:01:11 2018 +0000

    Split INIT/FINI into separate VDP methods

diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c
index e3cdcee5c..6d603ad77 100644
--- a/bin/varnishd/cache/cache_deliver_proc.c
+++ b/bin/varnishd/cache/cache_deliver_proc.c
@@ -66,7 +66,7 @@ VDP_bytes(struct req *req, enum vdp_action act, const void *ptr, ssize_t len)
 
 	assert(act > VDP_NULL || len > 0);
 	/* Call the present layer, while pointing to the next layer down */
-	retval = vdpe->vdp->func(req, act, &vdpe->priv, ptr, len);
+	retval = vdpe->vdp->bytes(req, act, &vdpe->priv, ptr, len);
 	if (retval && (vdc->retval == 0 || retval < vdc->retval))
 		vdc->retval = retval; /* Latch error value */
 	vdc->nxt = vdpe;
@@ -83,7 +83,7 @@ VDP_push(struct req *req, const struct vdp *vdp, void *priv, int bottom)
 	vdc = req->vdc;
 	AN(vdp);
 	AN(vdp->name);
-	AN(vdp->func);
+	AN(vdp->bytes);
 
 	if (vdc->retval)
 		return (vdc->retval);
@@ -107,7 +107,8 @@ VDP_push(struct req *req, const struct vdp *vdp, void *priv, int bottom)
 	vdc->nxt = VTAILQ_FIRST(&vdc->vdp);
 
 	AZ(vdc->retval);
-	vdc->retval = vdpe->vdp->func(req, VDP_INIT, &vdpe->priv, NULL, 0);
+	if (vdpe->vdp->init != NULL)
+		vdc->retval = vdpe->vdp->init(req, &vdpe->priv);
 	return (vdc->retval);
 }
 
@@ -126,8 +127,8 @@ VDP_close(struct req *req)
 		if (vdpe != NULL) {
 			CHECK_OBJ_NOTNULL(vdpe, VDP_ENTRY_MAGIC);
 			VTAILQ_REMOVE(&vdc->vdp, vdpe, list);
-			AZ(vdpe->vdp->func(req, VDP_FINI, &vdpe->priv,
-			    NULL, 0));
+			if (vdpe->vdp->fini != NULL)
+				AZ(vdpe->vdp->fini(req, &vdpe->priv));
 			AZ(vdpe->priv);
 		}
 		vdc->nxt = VTAILQ_FIRST(&vdc->vdp);
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 1e238d854..cbc02c2c4 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -247,8 +247,38 @@ ved_decode_len(struct req *req, const uint8_t **pp)
 /*---------------------------------------------------------------------
  */
 
-static int v_matchproto_(vdp_bytes)
-ved_vdp(struct req *req, enum vdp_action act, void **priv,
+static int v_matchproto_(vdp_init_f)
+ved_vdp_esi_init(struct req *req, void **priv)
+{
+	struct ecx *ecx;
+
+	AZ(*priv);
+	ALLOC_OBJ(ecx, ECX_MAGIC);
+	AN(ecx);
+	assert(sizeof gzip_hdr == 10);
+	ecx->preq = req;
+	*priv = ecx;
+	RFC2616_Weaken_Etag(req->resp);
+	req->res_mode |= RES_ESI;
+	if (req->resp_len != 0)
+		req->resp_len = -1;
+	return (0);
+}
+
+static int v_matchproto_(vdp_fini_f)
+ved_vdp_esi_fini(struct req *req, void **priv)
+{
+	struct ecx *ecx;
+
+	(void)req;
+	CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC);
+	FREE_OBJ(ecx);
+	*priv = NULL;
+	return (0);
+}
+
+static int v_matchproto_(vdp_bytes_f)
+ved_vdp_esi_bytes(struct req *req, enum vdp_action act, void **priv,
     const void *ptr, ssize_t len)
 {
 	uint8_t *q, *r;
@@ -259,25 +289,7 @@ ved_vdp(struct req *req, enum vdp_action act, void **priv,
 	struct ecx *ecx, *pecx = NULL;
 	int retval = 0;
 
-	if (act == VDP_INIT) {
-		AZ(*priv);
-		ALLOC_OBJ(ecx, ECX_MAGIC);
-		AN(ecx);
-		assert(sizeof gzip_hdr == 10);
-		ecx->preq = req;
-		*priv = ecx;
-		RFC2616_Weaken_Etag(req->resp);
-		req->res_mode |= RES_ESI;
-		if (req->resp_len != 0)
-			req->resp_len = -1;
-		return (0);
-	}
 	CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC);
-	if (act == VDP_FINI) {
-		FREE_OBJ(ecx);
-		*priv = NULL;
-		return (0);
-	}
 	pp = ptr;
 
 	if (req->esi_level > 0) {
@@ -431,7 +443,9 @@ ved_vdp(struct req *req, enum vdp_action act, void **priv,
 
 const struct vdp VDP_esi = {
 	.name =		"esi",
-	.func =		ved_vdp,
+	.init =		ved_vdp_esi_init,
+	.bytes =	ved_vdp_esi_bytes,
+	.fini =		ved_vdp_esi_fini,
 };
 
 /*
@@ -464,8 +478,16 @@ ved_bytes(struct req *req, struct req *preq, enum vdp_action act,
  * the stream with a bit more overhead.
  */
 
-static int v_matchproto_(vdp_bytes)
-ved_pretend_gzip(struct req *req, enum vdp_action act, void **priv,
+static int v_matchproto_(vdp_fini_f)
+ved_pretend_gzip_fini(struct req *req, void **priv)
+{
+	(void)req;
+	*priv = NULL;
+	return (0);
+}
+
+static int v_matchproto_(vdp_bytes_f)
+ved_pretend_gzip_bytes(struct req *req, enum vdp_action act, void **priv,
     const void *pv, ssize_t l)
 {
 	uint8_t buf1[5], buf2[5];
@@ -479,12 +501,6 @@ ved_pretend_gzip(struct req *req, enum vdp_action act, void **priv,
 	preq = ecx->preq;
 
 	(void)priv;
-	if (act == VDP_INIT)
-		return (0);
-	if (act == VDP_FINI) {
-		*priv = NULL;
-		return (0);
-	}
 	if (l == 0)
 		return (ved_bytes(req, ecx->preq, act, pv, l));
 
@@ -523,7 +539,8 @@ ved_pretend_gzip(struct req *req, enum vdp_action act, void **priv,
 
 static const struct vdp ved_vdp_pgz = {
 	.name =		"PGZ",
-	.func =		ved_pretend_gzip,
+	.bytes =	ved_pretend_gzip_bytes,
+	.fini =		ved_pretend_gzip_fini,
 };
 
 /*---------------------------------------------------------------------
@@ -765,26 +782,29 @@ ved_stripgzip(struct req *req, const struct boc *boc)
 
 /*--------------------------------------------------------------------*/
 
-static int v_matchproto_(vdp_bytes)
+static int v_matchproto_(vdp_fini_f)
+ved_vdp_fini(struct req *req, void **priv)
+{
+	(void)req;
+	*priv = NULL;
+	return (0);
+}
+
+static int v_matchproto_(vdp_bytes_f)
 ved_vdp_bytes(struct req *req, enum vdp_action act, void **priv,
     const void *ptr, ssize_t len)
 {
 	struct req *preq;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	if (act == VDP_INIT)
-		return (0);
-	if (act == VDP_FINI) {
-		*priv = NULL;
-		return (0);
-	}
 	CAST_OBJ_NOTNULL(preq, *priv, REQ_MAGIC);
 	return (ved_bytes(req, preq, act, ptr, len));
 }
 
 static const struct vdp ved_ved = {
 	.name =		"VED",
-	.func =		ved_vdp_bytes,
+	.bytes =	ved_vdp_bytes,
+	.fini =		ved_vdp_fini,
 };
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h
index b66afb08a..b2f72105c 100644
--- a/bin/varnishd/cache/cache_filter.h
+++ b/bin/varnishd/cache/cache_filter.h
@@ -94,18 +94,20 @@ void VRT_RemoveVFP(VRT_CTX, const struct vfp *);
 /* Deliver processors ------------------------------------------------*/
 
 enum vdp_action {
-	VDP_INIT,		/* Happens on VDP_push() */
-	VDP_FINI,		/* Happens on VDP_pop() */
 	VDP_NULL,		/* Input buffer valid after call */
 	VDP_FLUSH,		/* Input buffer will be invalidated */
 };
 
-typedef int vdp_bytes(struct req *, enum vdp_action, void **priv,
+typedef int vdp_init_f(struct req *, void **priv);
+typedef int vdp_fini_f(struct req *, void **priv);
+typedef int vdp_bytes_f(struct req *, enum vdp_action, void **priv,
     const void *ptr, ssize_t len);
 
 struct vdp {
 	const char		*name;
-	vdp_bytes		*func;
+	vdp_init_f		*init;
+	vdp_bytes_f		*bytes;
+	vdp_fini_f		*fini;
 };
 
 struct vdp_entry {
diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index d5d256d56..bfe2f6279 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -284,62 +284,73 @@ VGZ_Gzip(struct vgz *vg, const void **pptr, ssize_t *plen, enum vgz_flag flags)
  * VDP for gunzip'ing
  */
 
-static int v_matchproto_(vdp_bytes)
-vdp_gunzip(struct req *req, enum vdp_action act, void **priv,
-    const void *ptr, ssize_t len)
+static int v_matchproto_(vdp_init_f)
+vdp_gunzip_init(struct req *req, void **priv)
 {
-	enum vgzret_e vr;
-	ssize_t dl;
-	const void *dp;
-	struct worker *wrk;
 	struct vgz *vg;
 	const char *p;
+	ssize_t dl;
 	uint64_t u;
 
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	wrk = req->wrk;
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-
-	if (act == VDP_INIT) {
-		vg = VGZ_NewGunzip(req->vsl, "U D -");
-		AN(vg);
-		if (vgz_getmbuf(vg)) {
-			(void)VGZ_Destroy(&vg);
-			return (-1);
-		}
+	vg = VGZ_NewGunzip(req->vsl, "U D -");
+	AN(vg);
+	if (vgz_getmbuf(vg)) {
+		(void)VGZ_Destroy(&vg);
+		return (-1);
+	}
 
-		req->res_mode |= RES_GUNZIP;
-		VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
-		*priv = vg;
+	req->res_mode |= RES_GUNZIP;
+	VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
+	*priv = vg;
 
-		http_Unset(req->resp, H_Content_Encoding);
+	http_Unset(req->resp, H_Content_Encoding);
 
-		req->resp_len = -1;
+	req->resp_len = -1;
 
-		p = ObjGetAttr(req->wrk, req->objcore, OA_GZIPBITS, &dl);
-		if (p != NULL && dl == 32) {
-			u = vbe64dec(p + 24);
-			/*
-			 * If the size is non-zero AND we are the top VDP
-			 * (ie: no ESI), we know what size the output will be.
-			 */
-			if (u != 0 &&
-			    VTAILQ_FIRST(&req->vdc->vdp)->vdp == &VDP_gunzip)
-				req->resp_len = u;
-		}
-		return (0);
+	p = ObjGetAttr(req->wrk, req->objcore, OA_GZIPBITS, &dl);
+	if (p != NULL && dl == 32) {
+		u = vbe64dec(p + 24);
+		/*
+		 * If the size is non-zero AND we are the top VDP
+		 * (ie: no ESI), we know what size the output will be.
+		 */
+		if (u != 0 &&
+		    VTAILQ_FIRST(&req->vdc->vdp)->vdp == &VDP_gunzip)
+			req->resp_len = u;
 	}
+	return (0);
+}
+
+static int v_matchproto_(vdp_fini_f)
+vdp_gunzip_fini(struct req *req, void **priv)
+{
+	struct vgz *vg;
 
+	(void)req;
 	CAST_OBJ_NOTNULL(vg, *priv, VGZ_MAGIC);
 	AN(vg->m_buf);
+	(void)VGZ_Destroy(&vg);
+	*priv = NULL;
+	return (0);
+}
 
-	if (act == VDP_FINI) {
-		/* NB: Gunzip'ing may or may not have completed successfully. */
-		AZ(len);
-		(void)VGZ_Destroy(&vg);
-		*priv = NULL;
-		return (0);
-	}
+static int v_matchproto_(vdp_bytes_f)
+vdp_gunzip_bytes(struct req *req, enum vdp_action act, void **priv,
+    const void *ptr, ssize_t len)
+{
+	enum vgzret_e vr;
+	ssize_t dl;
+	const void *dp;
+	struct worker *wrk;
+	struct vgz *vg;
+
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	wrk = req->wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	(void)act;
+
+	CAST_OBJ_NOTNULL(vg, *priv, VGZ_MAGIC);
+	AN(vg->m_buf);
 
 	if (len == 0)
 		return (0);
@@ -363,7 +374,9 @@ vdp_gunzip(struct req *req, enum vdp_action act, void **priv,
 
 const struct vdp VDP_gunzip = {
 	.name =		"gunzip",
-	.func =		vdp_gunzip,
+	.init =		vdp_gunzip_init,
+	.bytes =	vdp_gunzip_bytes,
+	.fini =		vdp_gunzip_fini,
 };
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c
index 373d1f676..c6cf1fd3c 100644
--- a/bin/varnishd/cache/cache_range.c
+++ b/bin/varnishd/cache/cache_range.c
@@ -44,7 +44,20 @@ struct vrg_priv {
 	ssize_t			range_off;
 };
 
-static int v_matchproto_(vdp_bytes)
+static int v_matchproto_(vdp_fini_f)
+vrg_range_fini(struct req *req, void **priv)
+{
+	struct vrg_priv *vrg_priv;
+
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CAST_OBJ_NOTNULL(vrg_priv, *priv, VRG_PRIV_MAGIC);
+	if (vrg_priv->range_off < vrg_priv->range_high)
+		Req_Fail(req, SC_RANGE_SHORT);
+	*priv = NULL;	/* struct on ws, no need to free */
+	return (0);
+}
+
+static int v_matchproto_(vdp_bytes_f)
 vrg_range_bytes(struct req *req, enum vdp_action act, void **priv,
     const void *ptr, ssize_t len)
 {
@@ -54,15 +67,7 @@ vrg_range_bytes(struct req *req, enum vdp_action act, void **priv,
 	struct vrg_priv *vrg_priv;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	if (act == VDP_INIT)
-		return (0);
 	CAST_OBJ_NOTNULL(vrg_priv, *priv, VRG_PRIV_MAGIC);
-	if (act == VDP_FINI) {
-		if (vrg_priv->range_off < vrg_priv->range_high)
-			Req_Fail(req, SC_RANGE_SHORT);
-		*priv = NULL;	/* struct on ws, no need to free */
-		return (0);
-	}
 
 	l = vrg_priv->range_low - vrg_priv->range_off;
 	if (l > 0) {
@@ -86,7 +91,8 @@ vrg_range_bytes(struct req *req, enum vdp_action act, void **priv,
 
 static const struct vdp vrg_vdp = {
 	.name =		"RNG",
-	.func =		vrg_range_bytes,
+	.bytes =	vrg_range_bytes,
+	.fini =		vrg_range_fini,
 };
 
 /*--------------------------------------------------------------------*/
diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c
index fda38dda4..06764c67f 100644
--- a/bin/varnishd/http1/cache_http1_deliver.c
+++ b/bin/varnishd/http1/cache_http1_deliver.c
@@ -35,7 +35,7 @@
 
 /*--------------------------------------------------------------------*/
 
-static int v_matchproto_(vdp_bytes)
+static int v_matchproto_(vdp_bytes_f)
 v1d_bytes(struct req *req, enum vdp_action act, void **priv,
     const void *ptr, ssize_t len)
 {
@@ -43,8 +43,6 @@ v1d_bytes(struct req *req, enum vdp_action act, void **priv,
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	(void)priv;
-	if (act == VDP_INIT || act == VDP_FINI)
-		return (0);
 
 	AZ(req->vdc->nxt);		/* always at the bottom of the pile */
 
@@ -59,7 +57,7 @@ v1d_bytes(struct req *req, enum vdp_action act, void **priv,
 
 static const struct vdp v1d_vdp = {
 	.name =		"V1B",
-	.func =		v1d_bytes,
+	.bytes =	v1d_bytes,
 };
 
 static void
diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c
index 8f4b56e1d..f636e5285 100644
--- a/bin/varnishd/http2/cache_http2_deliver.c
+++ b/bin/varnishd/http2/cache_http2_deliver.c
@@ -72,7 +72,21 @@ V2D_Init(void)
 
 /**********************************************************************/
 
-static int v_matchproto_(vdp_bytes)
+static int v_matchproto_(vdp_fini_f)
+h2_fini(struct req *req, void **priv)
+{
+	struct h2_req *r2;
+
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	(void)priv;
+	CAST_OBJ_NOTNULL(r2, req->transport_priv, H2_REQ_MAGIC);
+	H2_Send_Get(req->wrk, r2->h2sess, r2);
+	H2_Send(req->wrk, r2, H2_F_DATA, H2FF_DATA_END_STREAM, 0, "");
+	H2_Send_Rel(r2->h2sess, r2);
+	return (0);
+}
+
+static int v_matchproto_(vdp_bytes_f)
 h2_bytes(struct req *req, enum vdp_action act, void **priv,
     const void *ptr, ssize_t len)
 {
@@ -80,17 +94,13 @@ h2_bytes(struct req *req, enum vdp_action act, void **priv,
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CAST_OBJ_NOTNULL(r2, req->transport_priv, H2_REQ_MAGIC);
+	(void)act;
 	(void)priv;
 
-	if (act == VDP_INIT)
-		return (0);
-	if ((r2->h2sess->error || r2->error) && act != VDP_FINI)
+	if ((r2->h2sess->error || r2->error))
 		return (-1);
 	H2_Send_Get(req->wrk, r2->h2sess, r2);
-	H2_Send(req->wrk, r2,
-	    H2_F_DATA,
-	    act == VDP_FINI ? H2FF_DATA_END_STREAM : H2FF_NONE,
-	    len, ptr);
+	H2_Send(req->wrk, r2, H2_F_DATA, H2FF_NONE, len, ptr);
 	H2_Send_Rel(r2->h2sess, r2);
 	req->acct.resp_bodybytes += len;
 	return (0);
@@ -98,7 +108,8 @@ h2_bytes(struct req *req, enum vdp_action act, void **priv,
 
 static const struct vdp h2_vdp = {
 	.name =		"H2B",
-	.func =		h2_bytes,
+	.bytes =	h2_bytes,
+	.fini =		h2_fini,
 };
 
 static inline size_t

From phk at FreeBSD.org  Wed Oct 10 10:00:25 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 10 Oct 2018 10:00:25 +0000 (UTC)
Subject: [master] 7fd371e0a Add missing decorators
Message-ID: <20181010100026.111CC40E7@lists.varnish-cache.org>


commit 7fd371e0aef576a6ade084015129ed917f74ec74
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 10 09:12:21 2018 +0000

    Add missing decorators

diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index cbc02c2c4..061d1e83c 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -574,7 +574,7 @@ struct ved_foo {
 	uint8_t tailbuf[8];
 };
 
-static int
+static int v_matchproto_(objiterate_f)
 ved_objiterate(void *priv, int flush, const void *ptr, ssize_t len)
 {
 	struct ved_foo *foo;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 04617f2f7..c1cf95f32 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -701,7 +701,7 @@ vbf_stp_fetchend(struct worker *wrk, struct busyobj *bo)
 /*--------------------------------------------------------------------
  */
 
-static int
+static int v_matchproto_(objiterate_f)
 vbf_objiterator(void *priv, int flush, const void *ptr, ssize_t len)
 {
 	struct busyobj *bo;

From phk at FreeBSD.org  Wed Oct 10 10:00:26 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 10 Oct 2018 10:00:26 +0000 (UTC)
Subject: [master] 46cc55c53 Add an advisory hint to object iterators
 signalling that this is the last segment.
Message-ID: <20181010100026.235A640E9@lists.varnish-cache.org>


commit 46cc55c53b507b8c8181a3d5e9f65829f6ab1ed8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 10 09:56:23 2018 +0000

    Add an advisory hint to object iterators signalling that this is
    the last segment.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index b63664706..b038ad157 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -686,7 +686,10 @@ int ObjHasAttr(struct worker *, struct objcore *, enum obj_attr);
 const void *ObjGetAttr(struct worker *, struct objcore *, enum obj_attr,
     ssize_t *len);
 
-typedef int objiterate_f(void *priv, int flush, const void *ptr, ssize_t len);
+typedef int objiterate_f(void *priv, unsigned flush,
+    const void *ptr, ssize_t len);
+#define OBJ_ITER_FLUSH	0x01
+#define OBJ_ITER_FINAL	0x02
 
 int ObjIterate(struct worker *, struct objcore *,
     void *priv, objiterate_f *func, int final);
diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c
index 6d603ad77..aa6a9a7ca 100644
--- a/bin/varnishd/cache/cache_deliver_proc.c
+++ b/bin/varnishd/cache/cache_deliver_proc.c
@@ -138,8 +138,9 @@ VDP_close(struct req *req)
 /*--------------------------------------------------------------------*/
 
 static int v_matchproto_(objiterate_f)
-vdp_objiterator(void *priv, int flush, const void *ptr, ssize_t len)
+vdp_objiterator(void *priv, unsigned flush, const void *ptr, ssize_t len)
 {
+
 	return (VDP_bytes(priv, flush ? VDP_FLUSH : VDP_NULL, ptr, len));
 }
 
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 061d1e83c..312c2a986 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -575,7 +575,7 @@ struct ved_foo {
 };
 
 static int v_matchproto_(objiterate_f)
-ved_objiterate(void *priv, int flush, const void *ptr, ssize_t len)
+ved_objiterate(void *priv, unsigned flush, const void *ptr, ssize_t len)
 {
 	struct ved_foo *foo;
 	const uint8_t *pp;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index c1cf95f32..1235313e0 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -702,15 +702,15 @@ vbf_stp_fetchend(struct worker *wrk, struct busyobj *bo)
  */
 
 static int v_matchproto_(objiterate_f)
-vbf_objiterator(void *priv, int flush, const void *ptr, ssize_t len)
+vbf_objiterator(void *priv, unsigned flush, const void *ptr, ssize_t len)
 {
 	struct busyobj *bo;
 	ssize_t l;
 	const uint8_t *ps = ptr;
 	uint8_t *pd;
 
-	(void)flush;
 	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
+	(void)flush;
 
 	while (len > 0) {
 		l = len;
diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c
index d170bbce6..86d1dd4bf 100644
--- a/bin/varnishd/cache/cache_req_body.c
+++ b/bin/varnishd/cache/cache_req_body.c
@@ -226,7 +226,7 @@ VRB_Iterate(struct req *req, objiterate_f *func, void *priv)
  */
 
 static int v_matchproto_(objiterate_f)
-httpq_req_body_discard(void *priv, int flush, const void *ptr, ssize_t len)
+httpq_req_body_discard(void *priv, unsigned flush, const void *ptr, ssize_t len)
 {
 
 	(void)priv;
diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c
index 2ddc58142..b8e3e876f 100644
--- a/bin/varnishd/http1/cache_http1_fetch.c
+++ b/bin/varnishd/http1/cache_http1_fetch.c
@@ -46,7 +46,7 @@
  */
 
 static int v_matchproto_(objiterate_f)
-vbf_iter_req_body(void *priv, int flush, const void *ptr, ssize_t l)
+vbf_iter_req_body(void *priv, unsigned flush, const void *ptr, ssize_t l)
 {
 	struct busyobj *bo;
 
diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c
index 776f5f7ff..0606f997e 100644
--- a/bin/varnishd/storage/storage_simple.c
+++ b/bin/varnishd/storage/storage_simple.c
@@ -246,6 +246,7 @@ sml_iterator(struct worker *wrk, struct objcore *oc,
 	ssize_t sl;
 	void *p;
 	ssize_t l;
+	unsigned u;
 
 	obj = sml_getobj(wrk, oc);
 	CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC);
@@ -256,8 +257,13 @@ sml_iterator(struct worker *wrk, struct objcore *oc,
 
 	if (boc == NULL) {
 		VTAILQ_FOREACH_SAFE(st, &obj->list, list, checkpoint) {
+			u = 0;
+			if (VTAILQ_NEXT(st, list) == NULL)
+				u |= OBJ_ITER_FINAL;
+			if (final)
+				u |= OBJ_ITER_FLUSH;
 			if (ret == 0 && st->len > 0)
-				ret = func(priv, 1, st->ptr, st->len);
+				ret = func(priv, u, st->ptr, st->len);
 			if (final) {
 				VTAILQ_REMOVE(&obj->list, st, list);
 				sml_stv_free(stv, st);
@@ -322,7 +328,10 @@ sml_iterator(struct worker *wrk, struct objcore *oc,
 			st = NULL;
 		Lck_Unlock(&boc->mtx);
 		assert(l > 0 || boc->state == BOS_FINISHED);
-		ret = func(priv, st != NULL ? final : 1, p, l);
+		u = 0;
+		if (st == NULL || final)
+			u |= OBJ_ITER_FLUSH;
+		ret = func(priv, u, p, l);
 		if (ret)
 			break;
 	}

From phk at FreeBSD.org  Thu Oct 11 09:10:16 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 11 Oct 2018 09:10:16 +0000 (UTC)
Subject: [master] 77ee21812 OCD rename of filename to match the rest.
Message-ID: <20181011091016.4015FA5A45@lists.varnish-cache.org>


commit 77ee218120999a60ff5672d8449712d86f6d893a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 11 08:48:52 2018 +0000

    OCD rename of filename to match the rest.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index e6e0537e4..ed2b5c6b5 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -44,7 +44,7 @@ varnishd_SOURCES = \
 	cache/cache_tcp_pool.c \
 	cache/cache_vary.c \
 	cache/cache_vcl.c \
-	cache/cache_vcl_vrt.c \
+	cache/cache_vrt_vcl.c \
 	cache/cache_vrt.c \
 	cache/cache_vrt_priv.c \
 	cache/cache_vrt_re.c \
diff --git a/bin/varnishd/cache/cache_vcl_vrt.c b/bin/varnishd/cache/cache_vrt_vcl.c
similarity index 100%
rename from bin/varnishd/cache/cache_vcl_vrt.c
rename to bin/varnishd/cache/cache_vrt_vcl.c

From phk at FreeBSD.org  Thu Oct 11 09:10:16 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 11 Oct 2018 09:10:16 +0000 (UTC)
Subject: [master] 3f533abb9 Collect everything related to beresp.filter_list
 in one source file
Message-ID: <20181011091016.57AEDA5A49@lists.varnish-cache.org>


commit 3f533abb9ba1d0200d70b4d4e10b29ccdf9d2244
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 11 09:09:03 2018 +0000

    Collect everything related to beresp.filter_list in one source file

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 1235313e0..9a08a37c6 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -498,97 +498,6 @@ vbf_stp_fetchbody(struct worker *wrk, struct busyobj *bo)
 	ObjTrimStore(wrk, vfc->oc);
 	return (F_STP_FETCHEND);
 }
-
-/*--------------------------------------------------------------------
- */
-
-static void
-vbf_default_filter_list(const struct busyobj *bo, struct vsb *vsb)
-{
-	const char *p;
-	int do_gzip = bo->do_gzip;
-	int do_gunzip = bo->do_gunzip;
-	int is_gzip = 0, is_gunzip = 0;
-
-	/*
-	 * The VCL variables beresp.do_g[un]zip tells us how we want the
-	 * object processed before it is stored.
-	 *
-	 * The backend Content-Encoding header tells us what we are going
-	 * to receive, which we classify in the following three classes:
-	 *
-	 *	"Content-Encoding: gzip"	--> object is gzip'ed.
-	 *	no Content-Encoding		--> object is not gzip'ed.
-	 *	anything else			--> do nothing wrt gzip
-	 */
-
-	/* No body -> done */
-	if (bo->htc->body_status == BS_NONE || bo->htc->content_length == 0)
-		return;
-
-	if (!cache_param->http_gzip_support)
-		do_gzip = do_gunzip = 0;
-
-	if (http_GetHdr(bo->beresp, H_Content_Encoding, &p))
-		is_gzip = !strcasecmp(p, "gzip");
-	else
-		is_gunzip = 1;
-
-	/* We won't gunzip unless it is gzip'ed */
-	if (do_gunzip && !is_gzip)
-		do_gunzip = 0;
-
-	/* We wont gzip unless if it already is gzip'ed */
-	if (do_gzip && !is_gunzip)
-		do_gzip = 0;
-
-	if (do_gunzip || (is_gzip && bo->do_esi))
-		VSB_cat(vsb, " gunzip");
-
-	if (bo->do_esi && (do_gzip || (is_gzip && !do_gunzip))) {
-		VSB_cat(vsb, " esi_gzip");
-		return;
-	}
-
-	if (bo->do_esi) {
-		VSB_cat(vsb, " esi");
-		return;
-	}
-
-	if (do_gzip)
-		VSB_cat(vsb, " gzip");
-
-	if (is_gzip && !do_gunzip)
-		VSB_cat(vsb, " testgunzip");
-}
-
-const char *
-VBF_Get_Filter_List(struct busyobj *bo)
-{
-	unsigned u;
-	struct vsb vsb[1];
-
-	u = WS_Reserve(bo->ws, 0);
-	if (u == 0) {
-		WS_Release(bo->ws, 0);
-		WS_MarkOverflow(bo->ws);
-		return (NULL);
-	}
-	AN(VSB_new(vsb, bo->ws->f, u, VSB_FIXEDLEN));
-	vbf_default_filter_list(bo, vsb);
-	if (VSB_finish(vsb)) {
-		WS_Release(bo->ws, 0);
-		WS_MarkOverflow(bo->ws);
-		return (NULL);
-	}
-	if (VSB_len(vsb)) {
-		WS_Release(bo->ws, VSB_len(vsb) + 1);
-		return (VSB_data(vsb) + 1);
-	}
-	WS_Release(bo->ws, 0);
-	return ("");
-}
-
 static enum fetch_step
 vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 {
diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c
index 653029da1..9ad6a7620 100644
--- a/bin/varnishd/cache/cache_vrt_filter.c
+++ b/bin/varnishd/cache/cache_vrt_filter.c
@@ -36,8 +36,10 @@
 #include <stdlib.h>
 
 #include "vct.h"
+
 #include "cache_varnishd.h"
 #include "cache_vcl.h"
+#include "vrt_obj.h"
 
 #include "cache_filter.h"
 
@@ -142,3 +144,125 @@ VCL_VRT_Init(void)
 	VRT_AddVFP(NULL, &VFP_esi);
 	VRT_AddVFP(NULL, &VFP_esi_gzip);
 }
+
+/*--------------------------------------------------------------------
+ */
+
+static void
+vbf_default_filter_list(const struct busyobj *bo, struct vsb *vsb)
+{
+	const char *p;
+	int do_gzip = bo->do_gzip;
+	int do_gunzip = bo->do_gunzip;
+	int is_gzip = 0, is_gunzip = 0;
+
+	/*
+	 * The VCL variables beresp.do_g[un]zip tells us how we want the
+	 * object processed before it is stored.
+	 *
+	 * The backend Content-Encoding header tells us what we are going
+	 * to receive, which we classify in the following three classes:
+	 *
+	 *	"Content-Encoding: gzip"	--> object is gzip'ed.
+	 *	no Content-Encoding		--> object is not gzip'ed.
+	 *	anything else			--> do nothing wrt gzip
+	 */
+
+	/* No body -> done */
+	if (bo->htc->body_status == BS_NONE || bo->htc->content_length == 0)
+		return;
+
+	if (!cache_param->http_gzip_support)
+		do_gzip = do_gunzip = 0;
+
+	if (http_GetHdr(bo->beresp, H_Content_Encoding, &p))
+		is_gzip = !strcasecmp(p, "gzip");
+	else
+		is_gunzip = 1;
+
+	/* We won't gunzip unless it is gzip'ed */
+	if (do_gunzip && !is_gzip)
+		do_gunzip = 0;
+
+	/* We wont gzip unless if it already is gzip'ed */
+	if (do_gzip && !is_gunzip)
+		do_gzip = 0;
+
+	if (do_gunzip || (is_gzip && bo->do_esi))
+		VSB_cat(vsb, " gunzip");
+
+	if (bo->do_esi && (do_gzip || (is_gzip && !do_gunzip))) {
+		VSB_cat(vsb, " esi_gzip");
+		return;
+	}
+
+	if (bo->do_esi) {
+		VSB_cat(vsb, " esi");
+		return;
+	}
+
+	if (do_gzip)
+		VSB_cat(vsb, " gzip");
+
+	if (is_gzip && !do_gunzip)
+		VSB_cat(vsb, " testgunzip");
+}
+
+const char *
+VBF_Get_Filter_List(struct busyobj *bo)
+{
+	unsigned u;
+	struct vsb vsb[1];
+
+	u = WS_Reserve(bo->ws, 0);
+	if (u == 0) {
+		WS_Release(bo->ws, 0);
+		WS_MarkOverflow(bo->ws);
+		return (NULL);
+	}
+	AN(VSB_new(vsb, bo->ws->f, u, VSB_FIXEDLEN));
+	vbf_default_filter_list(bo, vsb);
+	if (VSB_finish(vsb)) {
+		WS_Release(bo->ws, 0);
+		WS_MarkOverflow(bo->ws);
+		return (NULL);
+	}
+	if (VSB_len(vsb)) {
+		WS_Release(bo->ws, VSB_len(vsb) + 1);
+		return (VSB_data(vsb) + 1);
+	}
+	WS_Release(bo->ws, 0);
+	return ("");
+}
+
+/*--------------------------------------------------------------------*/
+
+VCL_STRING
+VRT_r_beresp_filters(VRT_CTX)
+{
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
+	if (ctx->bo->filter_list != NULL)
+		return(ctx->bo->filter_list);
+	/* We do not set bo->filter_list yet, things might still change */
+	return (VBF_Get_Filter_List(ctx->bo));
+}
+
+VCL_VOID
+VRT_l_beresp_filters(VRT_CTX, const char *str, ...)
+{
+	va_list ap;
+	const char *b;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
+	va_start(ap, str);
+	b = VRT_String(ctx->bo->ws, NULL, str, ap);
+	va_end(ap);
+	if (b == NULL) {
+		WS_MarkOverflow(ctx->bo->ws);
+		return;
+	}
+	ctx->bo->filter_list = b;
+}
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 7cb3feb17..e9e821f11 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -922,35 +922,3 @@ HTTP_VAR(req)
 HTTP_VAR(resp)
 HTTP_VAR(bereq)
 HTTP_VAR(beresp)
-
-/*--------------------------------------------------------------------*/
-
-VCL_STRING
-VRT_r_beresp_filters(VRT_CTX)
-{
-
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
-	if (ctx->bo->filter_list != NULL)
-		return(ctx->bo->filter_list);
-	/* We do not set bo->filter_list yet, things might still change */
-	return (VBF_Get_Filter_List(ctx->bo));
-}
-
-VCL_VOID
-VRT_l_beresp_filters(VRT_CTX, const char *str, ...)
-{
-	va_list ap;
-	const char *b;
-
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
-	va_start(ap, str);
-	b = VRT_String(ctx->bo->ws, NULL, str, ap);
-	va_end(ap);
-	if (b == NULL) {
-		WS_MarkOverflow(ctx->bo->ws);
-		return;
-	}
-	ctx->bo->filter_list = b;
-}

From phk at FreeBSD.org  Thu Oct 11 09:10:16 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 11 Oct 2018 09:10:16 +0000 (UTC)
Subject: [master] 33ec7d099 Split VFP filter stuff out of vrt_vcl
Message-ID: <20181011091016.4C0A2A5A47@lists.varnish-cache.org>


commit 33ec7d099b380762c05aa93be12ca97dd2cbfba3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 11 09:00:39 2018 +0000

    Split VFP filter stuff out of vrt_vcl

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index ed2b5c6b5..6b2e4753b 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -44,11 +44,12 @@ varnishd_SOURCES = \
 	cache/cache_tcp_pool.c \
 	cache/cache_vary.c \
 	cache/cache_vcl.c \
-	cache/cache_vrt_vcl.c \
 	cache/cache_vrt.c \
+	cache/cache_vrt_filter.c \
 	cache/cache_vrt_priv.c \
 	cache/cache_vrt_re.c \
 	cache/cache_vrt_var.c \
+	cache/cache_vrt_vcl.c \
 	cache/cache_vrt_vmod.c \
 	cache/cache_wrk.c \
 	cache/cache_ws.c \
diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c
new file mode 100644
index 000000000..653029da1
--- /dev/null
+++ b/bin/varnishd/cache/cache_vrt_filter.c
@@ -0,0 +1,144 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2016 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#include "vct.h"
+#include "cache_varnishd.h"
+#include "cache_vcl.h"
+
+#include "cache_filter.h"
+
+/*--------------------------------------------------------------------
+ */
+
+struct vfp_filter {
+	unsigned			magic;
+#define VFP_FILTER_MAGIC		0xd40894e9
+	const struct vfp		*filter;
+	int				nlen;
+	VTAILQ_ENTRY(vfp_filter)	list;
+};
+
+static struct vfp_filter_head vfp_filters =
+    VTAILQ_HEAD_INITIALIZER(vfp_filters);
+
+void
+VRT_AddVFP(VRT_CTX, const struct vfp *filter)
+{
+	struct vfp_filter *vp;
+	struct vfp_filter_head *hd = &vfp_filters;
+
+	VTAILQ_FOREACH(vp, hd, list) {
+		xxxassert(vp->filter != filter);
+		xxxassert(strcasecmp(vp->filter->name, filter->name));
+	}
+	if (ctx != NULL) {
+		hd = &ctx->vcl->vfps;
+		VTAILQ_FOREACH(vp, hd, list) {
+			xxxassert(vp->filter != filter);
+			xxxassert(strcasecmp(vp->filter->name, filter->name));
+		}
+	}
+	ALLOC_OBJ(vp, VFP_FILTER_MAGIC);
+	AN(vp);
+	vp->filter = filter;
+	vp->nlen = strlen(filter->name);
+	VTAILQ_INSERT_TAIL(hd, vp, list);
+}
+
+void
+VRT_RemoveVFP(VRT_CTX, const struct vfp *filter)
+{
+	struct vfp_filter *vp;
+	struct vfp_filter_head *hd = &ctx->vcl->vfps;
+
+	VTAILQ_FOREACH(vp, hd, list) {
+		if (vp->filter == filter)
+			break;
+	}
+	XXXAN(vp);
+	VTAILQ_REMOVE(hd, vp, list);
+	FREE_OBJ(vp);
+}
+
+int
+VCL_StackVFP(struct vfp_ctx *vc, const struct vcl *vcl, const char *fl)
+{
+	const char *p, *q;
+	const struct vfp_filter *vp;
+
+	VSLb(vc->wrk->vsl, SLT_Filters, "%s", fl);
+
+	for (p = fl; *p; p = q) {
+		if (vct_isspace(*p)) {
+			q = p + 1;
+			continue;
+		}
+		for (q = p; *q; q++)
+			if (vct_isspace(*q))
+				break;
+		VTAILQ_FOREACH(vp, &vfp_filters, list) {
+			if (vp->nlen != q - p)
+				continue;
+			if (!memcmp(p, vp->filter->name, vp->nlen))
+				break;
+		}
+		if (vp == NULL) {
+			VTAILQ_FOREACH(vp, &vcl->vfps, list) {
+				if (vp->nlen != q - p)
+					continue;
+				if (!memcmp(p, vp->filter->name, vp->nlen))
+					break;
+			}
+		}
+		if (vp == NULL)
+			return (VFP_Error(vc,
+			    "Filter '%.*s' not found", (int)(q-p), p));
+		if (VFP_Push(vc, vp->filter) == NULL)
+			return (-1);
+	}
+	return (0);
+}
+
+void
+VCL_VRT_Init(void)
+{
+	VRT_AddVFP(NULL, &VFP_testgunzip);
+	VRT_AddVFP(NULL, &VFP_gunzip);
+	VRT_AddVFP(NULL, &VFP_gzip);
+	VRT_AddVFP(NULL, &VFP_esi);
+	VRT_AddVFP(NULL, &VFP_esi_gzip);
+}
diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c
index 06b006b52..7b9d7fed8 100644
--- a/bin/varnishd/cache/cache_vrt_vcl.c
+++ b/bin/varnishd/cache/cache_vrt_vcl.c
@@ -37,12 +37,10 @@
 #include "cache_varnishd.h"
 
 #include "vcl.h"
-#include "vct.h"
 #include "vtim.h"
 
 #include "cache_director.h"
 #include "cache_vcl.h"
-#include "cache_filter.h"
 
 /*--------------------------------------------------------------------*/
 
@@ -462,105 +460,3 @@ VCL_##func##_method(struct vcl *vcl, struct worker *wrk,		\
 }
 
 #include "tbl/vcl_returns.h"
-
-/*--------------------------------------------------------------------
- */
-
-struct vfp_filter {
-	unsigned			magic;
-#define VFP_FILTER_MAGIC		0xd40894e9
-	const struct vfp		*filter;
-	int				nlen;
-	VTAILQ_ENTRY(vfp_filter)	list;
-};
-
-static struct vfp_filter_head vfp_filters =
-    VTAILQ_HEAD_INITIALIZER(vfp_filters);
-
-void
-VRT_AddVFP(VRT_CTX, const struct vfp *filter)
-{
-	struct vfp_filter *vp;
-	struct vfp_filter_head *hd = &vfp_filters;
-
-	VTAILQ_FOREACH(vp, hd, list) {
-		xxxassert(vp->filter != filter);
-		xxxassert(strcasecmp(vp->filter->name, filter->name));
-	}
-	if (ctx != NULL) {
-		hd = &ctx->vcl->vfps;
-		VTAILQ_FOREACH(vp, hd, list) {
-			xxxassert(vp->filter != filter);
-			xxxassert(strcasecmp(vp->filter->name, filter->name));
-		}
-	}
-	ALLOC_OBJ(vp, VFP_FILTER_MAGIC);
-	AN(vp);
-	vp->filter = filter;
-	vp->nlen = strlen(filter->name);
-	VTAILQ_INSERT_TAIL(hd, vp, list);
-}
-
-void
-VRT_RemoveVFP(VRT_CTX, const struct vfp *filter)
-{
-	struct vfp_filter *vp;
-	struct vfp_filter_head *hd = &ctx->vcl->vfps;
-
-	VTAILQ_FOREACH(vp, hd, list) {
-		if (vp->filter == filter)
-			break;
-	}
-	XXXAN(vp);
-	VTAILQ_REMOVE(hd, vp, list);
-	FREE_OBJ(vp);
-}
-
-int
-VCL_StackVFP(struct vfp_ctx *vc, const struct vcl *vcl, const char *fl)
-{
-	const char *p, *q;
-	const struct vfp_filter *vp;
-
-	VSLb(vc->wrk->vsl, SLT_Filters, "%s", fl);
-
-	for (p = fl; *p; p = q) {
-		if (vct_isspace(*p)) {
-			q = p + 1;
-			continue;
-		}
-		for (q = p; *q; q++)
-			if (vct_isspace(*q))
-				break;
-		VTAILQ_FOREACH(vp, &vfp_filters, list) {
-			if (vp->nlen != q - p)
-				continue;
-			if (!memcmp(p, vp->filter->name, vp->nlen))
-				break;
-		}
-		if (vp == NULL) {
-			VTAILQ_FOREACH(vp, &vcl->vfps, list) {
-				if (vp->nlen != q - p)
-					continue;
-				if (!memcmp(p, vp->filter->name, vp->nlen))
-					break;
-			}
-		}
-		if (vp == NULL)
-			return (VFP_Error(vc,
-			    "Filter '%.*s' not found", (int)(q-p), p));
-		if (VFP_Push(vc, vp->filter) == NULL)
-			return (-1);
-	}
-	return (0);
-}
-
-void
-VCL_VRT_Init(void)
-{
-	VRT_AddVFP(NULL, &VFP_testgunzip);
-	VRT_AddVFP(NULL, &VFP_gunzip);
-	VRT_AddVFP(NULL, &VFP_gzip);
-	VRT_AddVFP(NULL, &VFP_esi);
-	VRT_AddVFP(NULL, &VFP_esi_gzip);
-}

From phk at FreeBSD.org  Thu Oct 11 10:14:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 11 Oct 2018 10:14:10 +0000 (UTC)
Subject: [master] 62c75f7db Start registering VDPs
Message-ID: <20181011101410.95B45A717C@lists.varnish-cache.org>


commit 62c75f7db0a8af679fdc8455e03219a2a8d9749f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 11 10:13:04 2018 +0000

    Start registering VDPs

diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h
index b2f72105c..9c317ae95 100644
--- a/bin/varnishd/cache/cache_filter.h
+++ b/bin/varnishd/cache/cache_filter.h
@@ -130,3 +130,5 @@ struct vdp_ctx {
 
 int VDP_bytes(struct req *, enum vdp_action act, const void *ptr, ssize_t len);
 int VDP_push(struct req *, const struct vdp *, void *priv, int bottom);
+void VRT_AddVDP(VRT_CTX, const struct vdp *);
+void VRT_RemoveVDP(VRT_CTX, const struct vdp *);
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 3e1173f21..17179fec0 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -408,6 +408,8 @@ VCL_Close(struct vcl **vclp)
 	CHECK_OBJ_NOTNULL(*vclp, VCL_MAGIC);
 	vcl = *vclp;
 	*vclp = NULL;
+	assert(VTAILQ_EMPTY(&vcl->vfps));
+	assert(VTAILQ_EMPTY(&vcl->vdps));
 	AZ(dlclose(vcl->dlh));
 	AZ(errno=pthread_rwlock_destroy(&vcl->temp_rwl));
 	FREE_OBJ(vcl);
@@ -565,6 +567,7 @@ vcl_load(struct cli *cli, struct vrt_ctx *ctx,
 	VTAILQ_INIT(&vcl->director_list);
 	VTAILQ_INIT(&vcl->ref_list);
 	VTAILQ_INIT(&vcl->vfps);
+	VTAILQ_INIT(&vcl->vdps);
 
 	vcl->temp = VCL_TEMP_INIT;
 
diff --git a/bin/varnishd/cache/cache_vcl.h b/bin/varnishd/cache/cache_vcl.h
index f344c37f8..e9d99a710 100644
--- a/bin/varnishd/cache/cache_vcl.h
+++ b/bin/varnishd/cache/cache_vcl.h
@@ -31,9 +31,9 @@
  *
  */
 
-struct vfp_filter;
+struct vfilter;
 
-VTAILQ_HEAD(vfp_filter_head, vfp_filter);
+VTAILQ_HEAD(vfilter_head, vfilter);
 
 
 struct vcl {
@@ -53,7 +53,8 @@ struct vcl {
 	int			nrefs;
 	struct vcl		*label;
 	int			nlabels;
-	struct vfp_filter_head	vfps;
+	struct vfilter_head	vfps;
+	struct vfilter_head	vdps;
 };
 
 struct vclref {
diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c
index 9ad6a7620..ea951c321 100644
--- a/bin/varnishd/cache/cache_vrt_filter.c
+++ b/bin/varnishd/cache/cache_vrt_filter.c
@@ -46,49 +46,103 @@
 /*--------------------------------------------------------------------
  */
 
-struct vfp_filter {
+struct vfilter {
 	unsigned			magic;
-#define VFP_FILTER_MAGIC		0xd40894e9
-	const struct vfp		*filter;
+#define VFILTER_MAGIC			0xd40894e9
+	const struct vfp		*vfp;
+	const struct vdp		*vdp;
+	const char			*name;
 	int				nlen;
-	VTAILQ_ENTRY(vfp_filter)	list;
+	VTAILQ_ENTRY(vfilter)		list;
 };
 
-static struct vfp_filter_head vfp_filters =
+static struct vfilter_head vfp_filters =
     VTAILQ_HEAD_INITIALIZER(vfp_filters);
 
+static struct vfilter_head vdp_filters =
+    VTAILQ_HEAD_INITIALIZER(vdp_filters);
+
 void
 VRT_AddVFP(VRT_CTX, const struct vfp *filter)
 {
-	struct vfp_filter *vp;
-	struct vfp_filter_head *hd = &vfp_filters;
+	struct vfilter *vp;
+	struct vfilter_head *hd = &vfp_filters;
 
+	CHECK_OBJ_ORNULL(ctx, VRT_CTX_MAGIC);
 	VTAILQ_FOREACH(vp, hd, list) {
-		xxxassert(vp->filter != filter);
-		xxxassert(strcasecmp(vp->filter->name, filter->name));
+		xxxassert(vp->vfp != filter);
+		xxxassert(strcasecmp(vp->name, filter->name));
 	}
 	if (ctx != NULL) {
+		ASSERT_CLI();
 		hd = &ctx->vcl->vfps;
 		VTAILQ_FOREACH(vp, hd, list) {
-			xxxassert(vp->filter != filter);
-			xxxassert(strcasecmp(vp->filter->name, filter->name));
+			xxxassert(vp->vfp != filter);
+			xxxassert(strcasecmp(vp->name, filter->name));
+		}
+	}
+	ALLOC_OBJ(vp, VFILTER_MAGIC);
+	AN(vp);
+	vp->vfp = filter;
+	vp->name = filter->name;
+	vp->nlen = strlen(vp->name);
+	VTAILQ_INSERT_TAIL(hd, vp, list);
+}
+
+void
+VRT_AddVDP(VRT_CTX, const struct vdp *filter)
+{
+	struct vfilter *vp;
+	struct vfilter_head *hd = &vdp_filters;
+
+	CHECK_OBJ_ORNULL(ctx, VRT_CTX_MAGIC);
+	VTAILQ_FOREACH(vp, hd, list) {
+		xxxassert(vp->vdp != filter);
+		xxxassert(strcasecmp(vp->name, filter->name));
+	}
+	if (ctx != NULL) {
+		ASSERT_CLI();
+		hd = &ctx->vcl->vdps;
+		VTAILQ_FOREACH(vp, hd, list) {
+			xxxassert(vp->vdp != filter);
+			xxxassert(strcasecmp(vp->name, filter->name));
 		}
 	}
-	ALLOC_OBJ(vp, VFP_FILTER_MAGIC);
+	ALLOC_OBJ(vp, VFILTER_MAGIC);
 	AN(vp);
-	vp->filter = filter;
-	vp->nlen = strlen(filter->name);
+	vp->vdp = filter;
+	vp->name = filter->name;
+	vp->nlen = strlen(vp->name);
 	VTAILQ_INSERT_TAIL(hd, vp, list);
 }
 
 void
 VRT_RemoveVFP(VRT_CTX, const struct vfp *filter)
 {
-	struct vfp_filter *vp;
-	struct vfp_filter_head *hd = &ctx->vcl->vfps;
+	struct vfilter *vp;
+	struct vfilter_head *hd = &ctx->vcl->vfps;
 
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	ASSERT_CLI();
+	VTAILQ_FOREACH(vp, hd, list) {
+		if (vp->vfp == filter)
+			break;
+	}
+	XXXAN(vp);
+	VTAILQ_REMOVE(hd, vp, list);
+	FREE_OBJ(vp);
+}
+
+void
+VRT_RemoveVDP(VRT_CTX, const struct vdp *filter)
+{
+	struct vfilter *vp;
+	struct vfilter_head *hd = &ctx->vcl->vdps;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	ASSERT_CLI();
 	VTAILQ_FOREACH(vp, hd, list) {
-		if (vp->filter == filter)
+		if (vp->vdp == filter)
 			break;
 	}
 	XXXAN(vp);
@@ -100,7 +154,7 @@ int
 VCL_StackVFP(struct vfp_ctx *vc, const struct vcl *vcl, const char *fl)
 {
 	const char *p, *q;
-	const struct vfp_filter *vp;
+	const struct vfilter *vp;
 
 	VSLb(vc->wrk->vsl, SLT_Filters, "%s", fl);
 
@@ -115,21 +169,21 @@ VCL_StackVFP(struct vfp_ctx *vc, const struct vcl *vcl, const char *fl)
 		VTAILQ_FOREACH(vp, &vfp_filters, list) {
 			if (vp->nlen != q - p)
 				continue;
-			if (!memcmp(p, vp->filter->name, vp->nlen))
+			if (!memcmp(p, vp->name, vp->nlen))
 				break;
 		}
 		if (vp == NULL) {
 			VTAILQ_FOREACH(vp, &vcl->vfps, list) {
 				if (vp->nlen != q - p)
 					continue;
-				if (!memcmp(p, vp->filter->name, vp->nlen))
+				if (!memcmp(p, vp->name, vp->nlen))
 					break;
 			}
 		}
 		if (vp == NULL)
 			return (VFP_Error(vc,
 			    "Filter '%.*s' not found", (int)(q-p), p));
-		if (VFP_Push(vc, vp->filter) == NULL)
+		if (VFP_Push(vc, vp->vfp) == NULL)
 			return (-1);
 	}
 	return (0);
@@ -143,6 +197,8 @@ VCL_VRT_Init(void)
 	VRT_AddVFP(NULL, &VFP_gzip);
 	VRT_AddVFP(NULL, &VFP_esi);
 	VRT_AddVFP(NULL, &VFP_esi_gzip);
+	VRT_AddVDP(NULL, &VDP_esi);
+	VRT_AddVDP(NULL, &VDP_gunzip);
 }
 
 /*--------------------------------------------------------------------

From phk at FreeBSD.org  Thu Oct 11 14:10:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 11 Oct 2018 14:10:10 +0000 (UTC)
Subject: [master] 32c3a06f2 Hunt for py-specific variants of sphinx-build
 program
Message-ID: <20181011141010.5C4E2AD762@lists.varnish-cache.org>


commit 32c3a06f2271d1e63efb988db478cca9f3f7e063
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 11 14:08:51 2018 +0000

    Hunt for py-specific variants of sphinx-build program

diff --git a/configure.ac b/configure.ac
index 204519b3b..916ca80f1 100644
--- a/configure.ac
+++ b/configure.ac
@@ -49,7 +49,9 @@ fi
 AC_ARG_WITH([sphinx-build],
   AS_HELP_STRING([--with-sphinx-build=PATH], [Location of sphinx-build (auto)]),
   [SPHINX="$withval"],
-  AC_CHECK_PROGS(SPHINX, [sphinx-build], [no]))
+  AC_CHECK_PROGS(SPHINX,
+    [sphinx-build sphinx-build-3.6 sphinx-build-2.7],
+    [no]))
 if test "x$SPHINX" = "xno"; then
   AC_MSG_ERROR(
     [sphinx-build is needed to build Varnish, please install python-sphinx.])

From phk at FreeBSD.org  Thu Oct 11 14:27:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 11 Oct 2018 14:27:06 +0000 (UTC)
Subject: [master] e8276b9a7 Also look for versioned rst2man scripts
Message-ID: <20181011142706.3A30CADD70@lists.varnish-cache.org>


commit e8276b9a747e2d222cfa75507fb2440f1162de22
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 11 14:26:02 2018 +0000

    Also look for versioned rst2man scripts

diff --git a/configure.ac b/configure.ac
index 916ca80f1..804da7992 100644
--- a/configure.ac
+++ b/configure.ac
@@ -40,7 +40,9 @@ AC_PROG_INSTALL
 AC_ARG_WITH([rst2man],
   AS_HELP_STRING([--with-rst2man=PATH], [Location of rst2man (auto)]),
   [RST2MAN="$withval"],
-  AC_CHECK_PROGS(RST2MAN, [rst2man rst2man.py], [no]))
+  AC_CHECK_PROGS(RST2MAN,
+    [rst2man rst2man.py rst2man-3.6 rst2man-2.7],
+    [no]))
 if test "x$RST2MAN" = "xno"; then
   AC_MSG_ERROR(
     [rst2man is needed to build Varnish, please install python-docutils.])

From phk at FreeBSD.org  Fri Oct 12 08:22:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 12 Oct 2018 08:22:09 +0000 (UTC)
Subject: [master] ecad74bea Rework the filter-list parsing and test a couple
 of corner cases.
Message-ID: <20181012082210.2AD9A94521@lists.varnish-cache.org>


commit ecad74bead02b693e4f24161d6a14555363bcb8f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Oct 12 08:20:42 2018 +0000

    Rework the filter-list parsing and test a couple of corner cases.

diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c
index ea951c321..ec85ddf55 100644
--- a/bin/varnishd/cache/cache_vrt_filter.c
+++ b/bin/varnishd/cache/cache_vrt_filter.c
@@ -150,43 +150,57 @@ VRT_RemoveVDP(VRT_CTX, const struct vdp *filter)
 	FREE_OBJ(vp);
 }
 
+static const struct vfilter vfilter_error[1];
+
+static const struct vfilter *
+vcl_filter_list_iter(const struct vfilter_head *h1,
+    const struct vfilter_head *h2, const char **flp)
+{
+	const char *fl, *q;
+	const struct vfilter *vp;
+
+	AN(h1);
+	AN(h2);
+	AN(flp);
+
+	fl = *flp;
+	AN(fl);
+
+	while (vct_isspace(*fl))
+		fl++;
+	if (*fl == '\0') {
+		*flp = NULL;
+		return (NULL);
+	}
+	for (q = fl; *q && !vct_isspace(*q); q++)
+		continue;
+	*flp = q;
+	VTAILQ_FOREACH(vp, h1, list)
+		if (vp->nlen == q - fl && !memcmp(fl, vp->name, vp->nlen))
+			return (vp);
+	VTAILQ_FOREACH(vp, h2, list)
+		if (vp->nlen == q - fl && !memcmp(fl, vp->name, vp->nlen))
+			return (vp);
+	*flp = fl;
+	return (vfilter_error);
+}
+
 int
 VCL_StackVFP(struct vfp_ctx *vc, const struct vcl *vcl, const char *fl)
 {
-	const char *p, *q;
 	const struct vfilter *vp;
 
 	VSLb(vc->wrk->vsl, SLT_Filters, "%s", fl);
 
-	for (p = fl; *p; p = q) {
-		if (vct_isspace(*p)) {
-			q = p + 1;
-			continue;
-		}
-		for (q = p; *q; q++)
-			if (vct_isspace(*q))
-				break;
-		VTAILQ_FOREACH(vp, &vfp_filters, list) {
-			if (vp->nlen != q - p)
-				continue;
-			if (!memcmp(p, vp->name, vp->nlen))
-				break;
-		}
-		if (vp == NULL) {
-			VTAILQ_FOREACH(vp, &vcl->vfps, list) {
-				if (vp->nlen != q - p)
-					continue;
-				if (!memcmp(p, vp->name, vp->nlen))
-					break;
-			}
-		}
+	while (1) {
+		vp = vcl_filter_list_iter(&vfp_filters, &vcl->vfps, &fl);
 		if (vp == NULL)
-			return (VFP_Error(vc,
-			    "Filter '%.*s' not found", (int)(q-p), p));
+			return (0);
+		if (vp == vfilter_error)
+			return (VFP_Error(vc, "Filter '...%s' not found", fl));
 		if (VFP_Push(vc, vp->vfp) == NULL)
 			return (-1);
 	}
-	return (0);
 }
 
 void
diff --git a/bin/varnishtest/tests/m00048.vtc b/bin/varnishtest/tests/m00048.vtc
index 9f01bf5c7..8f5518d93 100644
--- a/bin/varnishtest/tests/m00048.vtc
+++ b/bin/varnishtest/tests/m00048.vtc
@@ -3,6 +3,11 @@ varnishtest "VMOD vfp"
 server s1 {
 	rxreq
 	txresp -body "Ponto Facto, Caesar Transit!"
+	rxreq
+	txresp -body "Ponto Facto, Caesar Transit!"
+	accept
+	rxreq
+	txresp -body "Ponto Facto, Caesar Transit!"
 } -start
 
 varnish v1 -vcl+backend {
@@ -14,8 +19,39 @@ varnish v1 -vcl+backend {
 } -start
 
 client c1 {
-	txreq
+	txreq -hdr "Cookie: a"
 	rxresp
 	expect resp.body == "Cbagb Snpgb, Pnrfne Genafvg!"
 } -run
 
+varnish v1 -vsl_catchup
+
+varnish v1 -vcl+backend {
+	import debug;
+
+	sub vcl_backend_response {
+		set beresp.filters = "rot13 rot13a";
+	}
+} 
+
+client c1 {
+	txreq -hdr "Cookie: a"
+	rxresp
+	expect resp.status == 503
+} -run
+
+
+varnish v1 -vcl+backend {
+	import debug;
+
+	sub vcl_backend_response {
+		set beresp.filters = "rot13 rot14";
+	}
+} 
+
+client c1 {
+	txreq -hdr "Cookie: a"
+	rxresp
+	expect resp.status == 503
+} -run
+

From phk at FreeBSD.org  Fri Oct 12 11:47:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 12 Oct 2018 11:47:09 +0000 (UTC)
Subject: [master] 49e42d22c Record an idea for later consideration
Message-ID: <20181012114709.A88CE9B6FB@lists.varnish-cache.org>


commit 49e42d22c583f2fab0c49e7d15b2c30607427136
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Oct 12 09:10:58 2018 +0000

    Record an idea for later consideration

diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c
index ec85ddf55..102b6f710 100644
--- a/bin/varnishd/cache/cache_vrt_filter.c
+++ b/bin/varnishd/cache/cache_vrt_filter.c
@@ -152,6 +152,7 @@ VRT_RemoveVDP(VRT_CTX, const struct vdp *filter)
 
 static const struct vfilter vfilter_error[1];
 
+// XXX: idea(fgs): Allow filters (...) arguments in the list
 static const struct vfilter *
 vcl_filter_list_iter(const struct vfilter_head *h1,
     const struct vfilter_head *h2, const char **flp)

From phk at FreeBSD.org  Fri Oct 12 11:47:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 12 Oct 2018 11:47:09 +0000 (UTC)
Subject: [master] 6405cc218 Remove VDP_push's "bottom" argument,
 we always build VDP stacks from the top down.
Message-ID: <20181012114709.B35749B701@lists.varnish-cache.org>


commit 6405cc218e83fc7e607e398c756abd41e4406b48
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Oct 12 11:45:57 2018 +0000

    Remove VDP_push's "bottom" argument, we always build VDP stacks
    from the top down.

diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c
index aa6a9a7ca..0e7602a10 100644
--- a/bin/varnishd/cache/cache_deliver_proc.c
+++ b/bin/varnishd/cache/cache_deliver_proc.c
@@ -74,7 +74,7 @@ VDP_bytes(struct req *req, enum vdp_action act, const void *ptr, ssize_t len)
 }
 
 int
-VDP_push(struct req *req, const struct vdp *vdp, void *priv, int bottom)
+VDP_Push(struct req *req, const struct vdp *vdp, void *priv)
 {
 	struct vdp_entry *vdpe;
 	struct vdp_ctx *vdc;
@@ -100,10 +100,7 @@ VDP_push(struct req *req, const struct vdp *vdp, void *priv, int bottom)
 	INIT_OBJ(vdpe, VDP_ENTRY_MAGIC);
 	vdpe->vdp = vdp;
 	vdpe->priv = priv;
-	if (bottom)
-		VTAILQ_INSERT_TAIL(&vdc->vdp, vdpe, list);
-	else
-		VTAILQ_INSERT_HEAD(&vdc->vdp, vdpe, list);
+	VTAILQ_INSERT_TAIL(&vdc->vdp, vdpe, list);
 	vdc->nxt = VTAILQ_FIRST(&vdc->vdp);
 
 	AZ(vdc->retval);
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 312c2a986..067e64470 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -833,9 +833,9 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody)
 		ved_stripgzip(req, boc);
 	} else {
 		if (ecx->isgzip && !i)
-			(void)VDP_push(req, &ved_vdp_pgz, ecx, 1);
+			(void)VDP_Push(req, &ved_vdp_pgz, ecx);
 		else
-			(void)VDP_push(req, &ved_ved, ecx->preq, 1);
+			(void)VDP_Push(req, &ved_ved, ecx->preq);
 		(void)VDP_DeliverObj(req);
 		(void)VDP_bytes(req, VDP_FLUSH, NULL, 0);
 	}
diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h
index 9c317ae95..79097f99d 100644
--- a/bin/varnishd/cache/cache_filter.h
+++ b/bin/varnishd/cache/cache_filter.h
@@ -129,6 +129,6 @@ struct vdp_ctx {
 };
 
 int VDP_bytes(struct req *, enum vdp_action act, const void *ptr, ssize_t len);
-int VDP_push(struct req *, const struct vdp *, void *priv, int bottom);
+int VDP_Push(struct req *, const struct vdp *, void *priv);
 void VRT_AddVDP(VRT_CTX, const struct vdp *);
 void VRT_RemoveVDP(VRT_CTX, const struct vdp *);
diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c
index c6cf1fd3c..8bcec96d4 100644
--- a/bin/varnishd/cache/cache_range.c
+++ b/bin/varnishd/cache/cache_range.c
@@ -181,7 +181,7 @@ vrg_dorange(struct req *req, const char *r)
 	vrg_priv->range_off = 0;
 	vrg_priv->range_low = low;
 	vrg_priv->range_high = high + 1;
-	if (VDP_push(req, &vrg_vdp, vrg_priv, 1))
+	if (VDP_Push(req, &vrg_vdp, vrg_priv))
 		return ("WS too small");
 	http_PutResponse(req->resp, "HTTP/1.1", 206, NULL);
 	return (NULL);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index b6b45dcb7..ee589d19e 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -379,13 +379,13 @@ cnt_transmit(struct worker *wrk, struct req *req)
 	if (sendbody >= 0) {
 		if (!req->disable_esi && req->resp_len != 0 &&
 		    ObjHasAttr(wrk, req->objcore, OA_ESIDATA) &&
-		    VDP_push(req, &VDP_esi, NULL, 0) < 0)
+		    VDP_Push(req, &VDP_esi, NULL) < 0)
 			err++;
 
 		if (cache_param->http_gzip_support &&
 		    ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED) &&
 		    !RFC2616_Req_Gzip(req->http) &&
-		    VDP_push(req, &VDP_gunzip, NULL, 1) < 0)
+		    VDP_Push(req, &VDP_gunzip, NULL) < 0)
 			err++;
 
 		if (cache_param->http_range_support &&
diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c
index 06764c67f..c075dd908 100644
--- a/bin/varnishd/http1/cache_http1_deliver.c
+++ b/bin/varnishd/http1/cache_http1_deliver.c
@@ -126,7 +126,7 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody)
 	if (req->resp_len == 0)
 		sendbody = 0;
 
-	if (sendbody && VDP_push(req, &v1d_vdp, NULL, 1)) {
+	if (sendbody && VDP_Push(req, &v1d_vdp, NULL)) {
 		v1d_error(req, "workspace_thread overflow");
 		AZ(req->wrk->v1l);
 		return;
diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c
index f636e5285..446cdaa6e 100644
--- a/bin/varnishd/http2/cache_http2_deliver.c
+++ b/bin/varnishd/http2/cache_http2_deliver.c
@@ -301,7 +301,7 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 
 	/* XXX someone into H2 please add appropriate error handling */
 	if (sendbody) {
-		if (!VDP_push(req, &h2_vdp, NULL, 1))
+		if (!VDP_Push(req, &h2_vdp, NULL))
 			(void)VDP_DeliverObj(req);
 	}
 

From phk at FreeBSD.org  Fri Oct 12 13:28:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 12 Oct 2018 13:28:09 +0000 (UTC)
Subject: [master] 71c443fa5 Refactor delivery a little bit
Message-ID: <20181012132809.B1788A0A8A@lists.varnish-cache.org>


commit 71c443fa516ed1ac5bea8cee50988043997c8029
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Oct 12 13:27:32 2018 +0000

    Refactor delivery a little bit

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index ee589d19e..961a3f953 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -372,8 +372,9 @@ cnt_transmit(struct worker *wrk, struct req *req)
 	} else if (status < 200 || status == 204 || status == 304) {
 		req->resp_len = -1;
 		sendbody = 0;
-	} else
+	} else {
 		sendbody = 1;
+	}
 
 	err = 0;
 	if (sendbody >= 0) {
@@ -408,9 +409,11 @@ cnt_transmit(struct worker *wrk, struct req *req)
 			    "Content-Length: %jd", req->resp_len);
 	}
 
-	if (err == 0)
+	if (err == 0) {
+		if (req->resp_len == 0)
+			sendbody = 0;
 		req->transport->deliver(req, boc, sendbody);
-	else {
+	} else {
 		VSLb(req->vsl, SLT_Error, "Failure to push processors");
 		req->doclose = SC_OVERLOAD;
 	}
diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c
index c075dd908..7c617ebe7 100644
--- a/bin/varnishd/http1/cache_http1_deliver.c
+++ b/bin/varnishd/http1/cache_http1_deliver.c
@@ -91,6 +91,16 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody)
 	CHECK_OBJ_ORNULL(boc, BOC_MAGIC);
 	CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC);
 
+	if (!req->doclose && http_HdrIs(req->resp, H_Connection, "close")) {
+		req->doclose = SC_RESP_CLOSE;
+	} else if (req->doclose) {
+		if (!http_HdrIs(req->resp, H_Connection, "close")) {
+			http_Unset(req->resp, H_Connection);
+			http_SetHeader(req->resp, "Connection: close");
+		}
+	} else if (!http_GetHdr(req->resp, H_Connection, NULL))
+		http_SetHeader(req->resp, "Connection: keep-alive");
+
 	if (sendbody) {
 		if (http_GetHdr(req->resp, H_Content_Length, NULL))
 			req->res_mode |= RES_LEN;
@@ -101,17 +111,12 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody)
 			req->res_mode |= RES_EOF;
 			req->doclose = SC_TX_EOF;
 		}
-	}
-
-	if (!req->doclose && http_HdrIs(req->resp, H_Connection, "close")) {
-		req->doclose = SC_RESP_CLOSE;
-	} else if (req->doclose) {
-		if (!http_HdrIs(req->resp, H_Connection, "close")) {
-			http_Unset(req->resp, H_Connection);
-			http_SetHeader(req->resp, "Connection: close");
+		if (VDP_Push(req, &v1d_vdp, NULL)) {
+			v1d_error(req, "workspace_thread overflow");
+			AZ(req->wrk->v1l);
+			return;
 		}
-	} else if (!http_GetHdr(req->resp, H_Connection, NULL))
-		http_SetHeader(req->resp, "Connection: keep-alive");
+	}
 
 	if (WS_Overflowed(req->ws)) {
 		v1d_error(req, "workspace_client overflow");
@@ -123,15 +128,6 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody)
 		return;
 	}
 
-	if (req->resp_len == 0)
-		sendbody = 0;
-
-	if (sendbody && VDP_Push(req, &v1d_vdp, NULL)) {
-		v1d_error(req, "workspace_thread overflow");
-		AZ(req->wrk->v1l);
-		return;
-	}
-
 	AZ(req->wrk->v1l);
 	V1L_Open(req->wrk, req->wrk->aws,
 		 &req->sp->fd, req->vsl, req->t_prev, 0);
@@ -144,9 +140,6 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody)
 
 	hdrbytes = HTTP1_Write(req->wrk, req->resp, HTTP1_Resp);
 
-	if (DO_DEBUG(DBG_FLUSH_HEAD))
-		(void)V1L_Flush(req->wrk);
-
 	if (!sendbody || req->res_mode & RES_ESI) {
 		if (V1L_Close(req->wrk, &bytes) && req->sp->fd >= 0) {
 			Req_Fail(req, SC_REM_CLOSE);
@@ -157,7 +150,8 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody)
 		 * header-bytes have been attempted sent. */
 		req->acct.resp_hdrbytes += bytes;
 		hdrbytes = 0;
-	}
+	} else if (DO_DEBUG(DBG_FLUSH_HEAD))
+		(void)V1L_Flush(req->wrk);
 
 	if (!sendbody) {
 		AZ(req->wrk->v1l);
diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c
index 446cdaa6e..5ca7ee3c3 100644
--- a/bin/varnishd/http2/cache_http2_deliver.c
+++ b/bin/varnishd/http2/cache_http2_deliver.c
@@ -285,9 +285,6 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 
 	AZ(req->wrk->v1l);
 
-	if (sendbody && req->resp_len == 0)
-		sendbody = 0;
-
 	r2->t_send = req->t_prev;
 
 	H2_Send_Get(req->wrk, r2->h2sess, r2);

From daghf at varnish-software.com  Fri Oct 12 14:52:09 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Fri, 12 Oct 2018 14:52:09 +0000 (UTC)
Subject: [master] ff31e8aff Add client_resp_500 counter
Message-ID: <20181012145209.8B09BA2549@lists.varnish-cache.org>


commit ff31e8aff56057ba92b98726187b858618f52c13
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Wed Oct 10 13:41:28 2018 +0200

    Add client_resp_500 counter
    
    Counts the number of times we failed a response due to running out of
    workspace during delivery.

diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc
index 8ca97887e..8a2fa4610 100644
--- a/bin/varnishd/VSC_main.vsc
+++ b/bin/varnishd/VSC_main.vsc
@@ -586,6 +586,14 @@
 
 	Number of session closes with Error VCL_FAILURE (VCL failure)
 
+.. varnish_vsc:: client_resp_500
+	:level: diag
+	:group: wrk
+	:oneliner:	Delivery failed due to insufficient workspace.
+
+	Number of times we failed a response due to running out of
+	workspace memory during delivery.
+
 .. varnish_vsc:: shm_records
 	:level:	diag
 	:oneliner:	SHM records
diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c
index 7c617ebe7..c0fce16d3 100644
--- a/bin/varnishd/http1/cache_http1_deliver.c
+++ b/bin/varnishd/http1/cache_http1_deliver.c
@@ -73,6 +73,7 @@ v1d_error(struct req *req, const char *msg)
 	VSLb(req->vsl, SLT_RespStatus, "500");
 	VSLb(req->vsl, SLT_RespReason, "Internal Server Error");
 
+	req->wrk->stats->client_resp_500++;
 	(void)write(req->sp->fd, r_500, sizeof r_500 - 1);
 	req->doclose = SC_TX_EOF;
 }
diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c
index 5ca7ee3c3..c7d1ad32e 100644
--- a/bin/varnishd/http2/cache_http2_deliver.c
+++ b/bin/varnishd/http2/cache_http2_deliver.c
@@ -274,7 +274,11 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 	}
 	if (VSB_finish(&resp)) {
 		// We ran out of workspace, return minimal 500
-		// XXX: VSC counter ?
+		VSLb(req->vsl, SLT_Error, "workspace_client overflow");
+		VSLb(req->vsl, SLT_RespStatus, "500");
+		VSLb(req->vsl, SLT_RespReason, "Internal Server Error");
+		req->wrk->stats->client_resp_500++;
+
 		r = (const char*)h2_500_resp;
 		sz = sizeof h2_500_resp;
 		sendbody = 0;
diff --git a/bin/varnishtest/tests/c00070.vtc b/bin/varnishtest/tests/c00070.vtc
index 659356935..4ecf82671 100644
--- a/bin/varnishtest/tests/c00070.vtc
+++ b/bin/varnishtest/tests/c00070.vtc
@@ -67,3 +67,5 @@ client c1 {
 } -run
 
 logexpect l2 -wait
+
+varnish v1 -expect client_resp_500 == 1
diff --git a/bin/varnishtest/tests/c00071.vtc b/bin/varnishtest/tests/c00071.vtc
index c0ebbc294..8ebddc490 100644
--- a/bin/varnishtest/tests/c00071.vtc
+++ b/bin/varnishtest/tests/c00071.vtc
@@ -46,3 +46,5 @@ client c2 {
 	expect resp.http.x-of == <undef>
 } -run
 
+
+varnish v1 -expect client_resp_500 == 2
diff --git a/bin/varnishtest/tests/r02233.vtc b/bin/varnishtest/tests/r02233.vtc
index 67fb20c9c..fe36fccb2 100644
--- a/bin/varnishtest/tests/r02233.vtc
+++ b/bin/varnishtest/tests/r02233.vtc
@@ -28,3 +28,4 @@ client c1 {
 } -run
 
 logexpect l1 -wait
+varnish v1 -expect client_resp_500 == 1
diff --git a/bin/varnishtest/tests/r02589.vtc b/bin/varnishtest/tests/r02589.vtc
index 418cc6833..2c43ab6fc 100644
--- a/bin/varnishtest/tests/r02589.vtc
+++ b/bin/varnishtest/tests/r02589.vtc
@@ -27,3 +27,5 @@ client c1 {
 		expect resp.http.server == "Varnish"
 	} -run
 } -run
+
+varnish v1 -expect client_resp_500 == 1
diff --git a/bin/varnishtest/tests/v00058.vtc b/bin/varnishtest/tests/v00058.vtc
index bb6aa869a..96c3972ef 100644
--- a/bin/varnishtest/tests/v00058.vtc
+++ b/bin/varnishtest/tests/v00058.vtc
@@ -164,3 +164,5 @@ client c1 {
 	expect resp.status >= 500
 	expect resp.status <= 503
 } -run
+
+varnish v1 -expect client_resp_500 == 1

From daghf at varnish-software.com  Fri Oct 12 14:52:09 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Fri, 12 Oct 2018 14:52:09 +0000 (UTC)
Subject: [master] 347254847 Add ws_backend_overflow counter
Message-ID: <20181012145209.BF0DBA254E@lists.varnish-cache.org>


commit 34725484702c144befab1edcf26d79b6280b9bc2
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Wed Oct 10 13:41:56 2018 +0200

    Add ws_backend_overflow counter

diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc
index 8a2fa4610..d73b84102 100644
--- a/bin/varnishd/VSC_main.vsc
+++ b/bin/varnishd/VSC_main.vsc
@@ -594,6 +594,13 @@
 	Number of times we failed a response due to running out of
 	workspace memory during delivery.
 
+.. varnish_vsc:: ws_backend_overflow
+	:level: diag
+	:group: wrk
+	:oneliner: workspace_backend overflows
+
+	Number of times we ran out of space in workspace_backend.
+
 .. varnish_vsc:: shm_records
 	:level:	diag
 	:oneliner:	SHM records
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index e27431042..1f452d5e4 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -167,6 +167,9 @@ VBO_ReleaseBusyObj(struct worker *wrk, struct busyobj **pbo)
 
 	AZ(bo->htc);
 
+	if (WS_Overflowed(bo->ws))
+		wrk->stats->ws_backend_overflow++;
+
 	if (bo->fetch_objcore != NULL) {
 		AN(wrk);
 		(void)HSH_DerefObjCore(wrk, &bo->fetch_objcore,
diff --git a/bin/varnishtest/tests/r01739.vtc b/bin/varnishtest/tests/r01739.vtc
index 843ea6933..6e40741f5 100644
--- a/bin/varnishtest/tests/r01739.vtc
+++ b/bin/varnishtest/tests/r01739.vtc
@@ -27,3 +27,4 @@ client c1 {
 } -run
 
 logexpect l1 -wait
+varnish v1 -expect ws_backend_overflow == 1
diff --git a/bin/varnishtest/tests/r01990.vtc b/bin/varnishtest/tests/r01990.vtc
index d86f61c55..c6a83292c 100644
--- a/bin/varnishtest/tests/r01990.vtc
+++ b/bin/varnishtest/tests/r01990.vtc
@@ -40,3 +40,4 @@ client c1 {
 } -run
 
 logexpect l1 -wait
+varnish v1 -expect ws_backend_overflow == 1

From daghf at varnish-software.com  Fri Oct 12 14:52:09 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Fri, 12 Oct 2018 14:52:09 +0000 (UTC)
Subject: [master] 16df092f2 Add ws_client_overflow counter
Message-ID: <20181012145209.F28C9A2552@lists.varnish-cache.org>


commit 16df092f2691f4c896af8fe20f42df18afada157
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Thu Oct 11 11:26:45 2018 +0200

    Add ws_client_overflow counter

diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc
index d73b84102..e7b7b9dcf 100644
--- a/bin/varnishd/VSC_main.vsc
+++ b/bin/varnishd/VSC_main.vsc
@@ -601,6 +601,13 @@
 
 	Number of times we ran out of space in workspace_backend.
 
+.. varnish_vsc:: ws_client_overflow
+	:level: diag
+	:group: wrk
+	:oneliner: workspace_client overflows
+
+	Number of times we ran out of space in workspace_client.
+
 .. varnish_vsc:: shm_records
 	:level:	diag
 	:oneliner:	SHM records
diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c
index 991487201..5ab13a2cf 100644
--- a/bin/varnishd/cache/cache_req.c
+++ b/bin/varnishd/cache/cache_req.c
@@ -191,6 +191,8 @@ Req_Rollback(struct req *req)
 	VCL_TaskLeave(req->vcl, req->privs);
 	VCL_TaskEnter(req->vcl, req->privs);
 	HTTP_Copy(req->http, req->http0);
+	if (WS_Overflowed(req->ws))
+		req->wrk->stats->ws_client_overflow++;
 	WS_Reset(req->ws, req->ws_req);
 }
 
@@ -242,6 +244,9 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 	req->hash_ignore_busy = 0;
 	req->is_hit = 0;
 
+	if (WS_Overflowed(req->ws))
+		wrk->stats->ws_client_overflow++;
+
 	WS_Reset(req->ws, 0);
 }
 
diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c
index c7d1ad32e..50f9dd760 100644
--- a/bin/varnishd/http2/cache_http2_deliver.c
+++ b/bin/varnishd/http2/cache_http2_deliver.c
@@ -273,6 +273,7 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 		VSB_bcat(&resp, r, sz);
 	}
 	if (VSB_finish(&resp)) {
+		WS_MarkOverflow(req->ws);
 		// We ran out of workspace, return minimal 500
 		VSLb(req->vsl, SLT_Error, "workspace_client overflow");
 		VSLb(req->vsl, SLT_RespStatus, "500");
diff --git a/bin/varnishtest/tests/c00070.vtc b/bin/varnishtest/tests/c00070.vtc
index 4ecf82671..9042fc554 100644
--- a/bin/varnishtest/tests/c00070.vtc
+++ b/bin/varnishtest/tests/c00070.vtc
@@ -69,3 +69,4 @@ client c1 {
 logexpect l2 -wait
 
 varnish v1 -expect client_resp_500 == 1
+varnish v1 -expect ws_client_overflow == 2
diff --git a/bin/varnishtest/tests/c00071.vtc b/bin/varnishtest/tests/c00071.vtc
index 8ebddc490..6cff35d87 100644
--- a/bin/varnishtest/tests/c00071.vtc
+++ b/bin/varnishtest/tests/c00071.vtc
@@ -48,3 +48,4 @@ client c2 {
 
 
 varnish v1 -expect client_resp_500 == 2
+varnish v1 -expect ws_client_overflow == 2
diff --git a/bin/varnishtest/tests/r02233.vtc b/bin/varnishtest/tests/r02233.vtc
index fe36fccb2..2bc591b82 100644
--- a/bin/varnishtest/tests/r02233.vtc
+++ b/bin/varnishtest/tests/r02233.vtc
@@ -29,3 +29,4 @@ client c1 {
 
 logexpect l1 -wait
 varnish v1 -expect client_resp_500 == 1
+varnish v1 -expect ws_client_overflow == 1
diff --git a/bin/varnishtest/tests/r02589.vtc b/bin/varnishtest/tests/r02589.vtc
index 2c43ab6fc..0d39c4fb3 100644
--- a/bin/varnishtest/tests/r02589.vtc
+++ b/bin/varnishtest/tests/r02589.vtc
@@ -29,3 +29,4 @@ client c1 {
 } -run
 
 varnish v1 -expect client_resp_500 == 1
+varnish v1 -expect ws_client_overflow == 1
diff --git a/bin/varnishtest/tests/v00058.vtc b/bin/varnishtest/tests/v00058.vtc
index 96c3972ef..e9869c29e 100644
--- a/bin/varnishtest/tests/v00058.vtc
+++ b/bin/varnishtest/tests/v00058.vtc
@@ -166,3 +166,4 @@ client c1 {
 } -run
 
 varnish v1 -expect client_resp_500 == 1
+varnish v1 -expect ws_client_overflow == 2

From daghf at varnish-software.com  Fri Oct 12 14:52:10 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Fri, 12 Oct 2018 14:52:10 +0000 (UTC)
Subject: [master] dda7185c4 Add ws_thread_overflow counter
Message-ID: <20181012145210.20E13A2556@lists.varnish-cache.org>


commit dda7185c41b11b4b61d2a33cd5f7e67a7ffa8d72
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Thu Oct 11 11:56:01 2018 +0200

    Add ws_thread_overflow counter
    
    See details in r02275.vtc for why there isn't a test case for this.

diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc
index e7b7b9dcf..86c0ceb7a 100644
--- a/bin/varnishd/VSC_main.vsc
+++ b/bin/varnishd/VSC_main.vsc
@@ -608,6 +608,13 @@
 
 	Number of times we ran out of space in workspace_client.
 
+.. varnish_vsc:: ws_thread_overflow
+	:level: diag
+	:group: wrk
+	:oneliner: workspace_thread overflows
+
+	Number of times we ran out of space in workspace_thread.
+
 .. varnish_vsc:: shm_records
 	:level:	diag
 	:oneliner:	SHM records
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index bfbfae374..50839143b 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -394,6 +394,8 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk)
 			tp = &tpx;
 		} while (tp->func != NULL);
 
+		if (WS_Overflowed(wrk->aws))
+			wrk->stats->ws_thread_overflow++;
 		/* cleanup for next task */
 		wrk->seen_methods = 0;
 	}

From daghf at varnish-software.com  Fri Oct 12 14:52:10 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Fri, 12 Oct 2018 14:52:10 +0000 (UTC)
Subject: [master] 267504b81 Add ws_session_overflow counter
Message-ID: <20181012145210.49FFBA255B@lists.varnish-cache.org>


commit 267504b8143bdb8ef96240455a3aa788f96b579b
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Thu Oct 11 14:52:12 2018 +0200

    Add ws_session_overflow counter

diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc
index 86c0ceb7a..0c22fead5 100644
--- a/bin/varnishd/VSC_main.vsc
+++ b/bin/varnishd/VSC_main.vsc
@@ -615,6 +615,12 @@
 
 	Number of times we ran out of space in workspace_thread.
 
+.. varnish_vsc:: ws_session_overflow
+	:level: diag
+	:oneliner: workspace_session overflows
+
+	Number of times we ran out of space in workspace_session.
+
 .. varnish_vsc:: shm_records
 	:level:	diag
 	:oneliner:	SHM records
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 20ada52e3..fede8e0ae 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -545,6 +545,8 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now)
 	VSL(SLT_SessClose, sp->vxid, "%s %.3f",
 	    sess_close_2str(reason, 0), now - sp->t_open);
 	VSL(SLT_End, sp->vxid, "%s", "");
+	if (WS_Overflowed(sp->ws))
+		VSC_C_main->ws_session_overflow++;
 	SES_Rel(sp);
 }
 
diff --git a/bin/varnishtest/tests/o00005.vtc b/bin/varnishtest/tests/o00005.vtc
index 3ec7a2881..3ae1aacfe 100644
--- a/bin/varnishtest/tests/o00005.vtc
+++ b/bin/varnishtest/tests/o00005.vtc
@@ -201,3 +201,46 @@ client c1 {
 varnish v1 -vsl_catchup
 
 logexpect l1 -wait
+
+client c1 {
+	# PROXY2 sp->ws overflow
+	sendhex {
+		0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a
+		21 11 01 23
+		d9 46 b5 21
+		5f 8e a8 22
+		ed 96
+		01 bb
+		03 00 04  c5 1b 5b 2b
+		01 00 02  68 32
+		02 00 c8  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61
+		20 00 3d
+		01 00 00 00 00
+		21 00 07  54 4c 53 76 31 2e 33
+		25 00 05  45 43 32 35 36
+		24 00 0a  52 53 41 2d 53 48 41 32 35 36
+		23 00 16  41 45 41 44 2d 41 45 53 31 32 38
+			  2d 47 43 4d 2d 53 48 41 32 35 36
+	}
+	expect_close
+} -run
+
+varnish v1 -expect ws_session_overflow == 1

From phk at FreeBSD.org  Mon Oct 15 06:59:14 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 15 Oct 2018 06:59:14 +0000 (UTC)
Subject: [master] ef4730d93 Emit a VCL_Error SLT when we exceed ESI depth
Message-ID: <20181015065914.9198A96A99@lists.varnish-cache.org>


commit ef4730d93fee8779a2972e473b62c8c8d9d2ba03
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Oct 13 07:05:53 2018 +0000

    Emit a VCL_Error SLT when we exceed ESI depth

diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 067e64470..468504fcd 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -108,8 +108,12 @@ ved_include(struct req *preq, const char *src, const char *host,
 	CHECK_OBJ_NOTNULL(ecx, ECX_MAGIC);
 	wrk = preq->wrk;
 
-	if (preq->esi_level >= cache_param->max_esi_depth)
+	if (preq->esi_level >= cache_param->max_esi_depth) {
+		VSLb(preq->vsl, SLT_VCL_Error,
+		    "ESI depth limit reach (param max_esi_depth = %u",
+		    cache_param->max_esi_depth);
 		return;
+	}
 
 	req = Req_New(wrk, sp);
 	SES_Ref(sp);

From nils.goroll at uplex.de  Mon Oct 15 12:45:11 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 15 Oct 2018 12:45:11 +0000 (UTC)
Subject: [master] 8cd72571e change nanoseconds precision timestamps into
 microseconds
Message-ID: <20181015124511.102B2A398B@lists.varnish-cache.org>


commit 8cd72571e09ca86c115a21360dc2d6b5b7be1a73
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 15 14:39:53 2018 +0200

    change nanoseconds precision timestamps into microseconds
    
    Ref #2792

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 6aa1ae07c..3f2a11caf 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -179,7 +179,7 @@ EXP_Rearm(struct objcore *oc, double now, double ttl, double grace, double keep)
 
 	when = EXP_WHEN(oc);
 
-	VSL(SLT_ExpKill, 0, "EXP_Rearm p=%p E=%.9f e=%.9f f=0x%x", oc,
+	VSL(SLT_ExpKill, 0, "EXP_Rearm p=%p E=%.6f e=%.6f f=0x%x", oc,
 	    oc->timer_when, when, oc->flags);
 
 	if (when < oc->t_origin || when < oc->timer_when)
@@ -198,7 +198,7 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, unsigned flags)
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	assert(oc->refcnt > 0);
 
-	VSLb(&ep->vsl, SLT_ExpKill, "EXP_Inbox flg=%x p=%p e=%.9f f=0x%x",
+	VSLb(&ep->vsl, SLT_ExpKill, "EXP_Inbox flg=%x p=%p e=%.6f f=0x%x",
 	    flags, oc, oc->timer_when, oc->flags);
 
 	if (flags & OC_EF_REMOVE) {
@@ -219,7 +219,7 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, unsigned flags)
 		ObjSendEvent(ep->wrk, oc, OEV_TTLCHG);
 	}
 
-	VSLb(&ep->vsl, SLT_ExpKill, "EXP_When p=%p e=%.9f f=0x%x", oc,
+	VSLb(&ep->vsl, SLT_ExpKill, "EXP_When p=%p e=%.6f f=0x%x", oc,
 	    oc->timer_when, flags);
 
 	/*
@@ -255,7 +255,7 @@ exp_expire(struct exp_priv *ep, double now)
 	oc = binheap_root(ep->heap);
 	if (oc == NULL)
 		return (now + 355./113.);
-	VSLb(&ep->vsl, SLT_ExpKill, "EXP_expire p=%p e=%.9f f=0x%x", oc,
+	VSLb(&ep->vsl, SLT_ExpKill, "EXP_expire p=%p e=%.6f f=0x%x", oc,
 	    oc->timer_when - now, oc->flags);
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c
index d2d94aab3..9777584f8 100644
--- a/bin/varnishd/mgt/mgt_vcc.c
+++ b/bin/varnishd/mgt/mgt_vcc.c
@@ -312,7 +312,7 @@ mgt_VccCompile(struct cli *cli, struct vclprog *vcl, const char *vclname,
 	 *
 	 * The Best way to reproduce this is to have regexps in the VCL.
 	 */
-	VSB_printf(sb, "vcl_%s.%.9f", vclname, VTIM_real());
+	VSB_printf(sb, "vcl_%s.%.6f", vclname, VTIM_real());
 	AZ(VSB_finish(sb));
 	vp.dir = strdup(VSB_data(sb));
 	AN(vp.dir);
diff --git a/doc/changes.rst b/doc/changes.rst
index d198e8aa1..d817aa794 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -64,6 +64,10 @@ Varnish Cache trunk (ongoing)
   an unnecessary incompatibility with VSL files written by previous
   versions. (2790_)
 
+* Changed ``ExpKill`` log tags to emit microsecond-precision
+  timestamps instead of nanoseconds (2792_)
+
+.. _2792: https://github.com/varnishcache/varnish-cache/pull/2792
 .. _2783: https://github.com/varnishcache/varnish-cache/pull/2783
 .. _2780: https://github.com/varnishcache/varnish-cache/issues/2780
 .. _2782: https://github.com/varnishcache/varnish-cache/issues/2782

From phk at FreeBSD.org  Mon Oct 15 20:18:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 15 Oct 2018 20:18:08 +0000 (UTC)
Subject: [master] da8b6dc0b Make sure wrk arg is always present,
 we need it for stats now.
Message-ID: <20181015201808.EAFAEAE086@lists.varnish-cache.org>


commit da8b6dc0b3991aced815974b9f8682c6649b8c49
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 15 19:07:24 2018 +0000

    Make sure wrk arg is always present, we need it for stats now.

diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 1f452d5e4..6eac91434 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -146,7 +146,7 @@ VBO_ReleaseBusyObj(struct worker *wrk, struct busyobj **pbo)
 {
 	struct busyobj *bo;
 
-	CHECK_OBJ_ORNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	TAKE_OBJ_NOTNULL(bo, pbo, BUSYOBJ_MAGIC);
 	CHECK_OBJ_ORNULL(bo->fetch_objcore, OBJCORE_MAGIC);
 
@@ -171,7 +171,6 @@ VBO_ReleaseBusyObj(struct worker *wrk, struct busyobj **pbo)
 		wrk->stats->ws_backend_overflow++;
 
 	if (bo->fetch_objcore != NULL) {
-		AN(wrk);
 		(void)HSH_DerefObjCore(wrk, &bo->fetch_objcore,
 		    HSH_RUSH_POLICY);
 	}

From phk at FreeBSD.org  Mon Oct 15 20:18:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 15 Oct 2018 20:18:09 +0000 (UTC)
Subject: [master] 7bf932f3a Rename the esi_iovs paramter to http1_iovs and
 default it to 64 which is half the wrk-aws it comes from.
Message-ID: <20181015201809.17955AE089@lists.varnish-cache.org>


commit 7bf932f3a674fa934c1bfd586686686003eec550
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 15 19:16:30 2018 +0000

    Rename the esi_iovs paramter to http1_iovs and default it to 64
    which is half the wrk-aws it comes from.
    
    This saves us from peeking into the VDP stack to spot ESI modules.

diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c
index c0fce16d3..580a23e37 100644
--- a/bin/varnishd/http1/cache_http1_deliver.c
+++ b/bin/varnishd/http1/cache_http1_deliver.c
@@ -84,7 +84,7 @@ v1d_error(struct req *req, const char *msg)
 void v_matchproto_(vtr_deliver_f)
 V1D_Deliver(struct req *req, struct boc *boc, int sendbody)
 {
-	int err;
+	int err = 0;
 	unsigned u;
 	uint64_t hdrbytes, bytes;
 
@@ -131,7 +131,7 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody)
 
 	AZ(req->wrk->v1l);
 	V1L_Open(req->wrk, req->wrk->aws,
-		 &req->sp->fd, req->vsl, req->t_prev, 0);
+		 &req->sp->fd, req->vsl, req->t_prev, cache_param->http1_iovs);
 
 	if (WS_Overflowed(req->wrk->aws)) {
 		v1d_error(req, "workspace_thread overflow");
@@ -141,47 +141,18 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody)
 
 	hdrbytes = HTTP1_Write(req->wrk, req->resp, HTTP1_Resp);
 
-	if (!sendbody || req->res_mode & RES_ESI) {
-		if (V1L_Close(req->wrk, &bytes) && req->sp->fd >= 0) {
-			Req_Fail(req, SC_REM_CLOSE);
-			sendbody = 0;
-		}
-
-		/* Charge bytes sent as reported from V1L_Close. Only
-		 * header-bytes have been attempted sent. */
-		req->acct.resp_hdrbytes += bytes;
-		hdrbytes = 0;
-	} else if (DO_DEBUG(DBG_FLUSH_HEAD))
-		(void)V1L_Flush(req->wrk);
-
-	if (!sendbody) {
-		AZ(req->wrk->v1l);
-		VDP_close(req);
-		return;
-	}
-
-	AN(sendbody);
-	if (req->res_mode & RES_ESI) {
-		AZ(req->wrk->v1l);
-
-		V1L_Open(req->wrk, req->wrk->aws,
-			 &req->sp->fd, req->vsl, req->t_prev,
-			 cache_param->esi_iovs);
-
-		if (WS_Overflowed(req->wrk->aws)) {
-			v1d_error(req, "workspace_thread overflow");
-			AZ(req->wrk->v1l);
-			return;
-		}
+	if (sendbody) {
+		if (DO_DEBUG(DBG_FLUSH_HEAD))
+			(void)V1L_Flush(req->wrk);
+		if (req->res_mode & RES_CHUNKED)
+			V1L_Chunked(req->wrk);
+		err = VDP_DeliverObj(req);
+		if (!err && (req->res_mode & RES_CHUNKED))
+			V1L_EndChunk(req->wrk);
 	}
 
-	if (req->res_mode & RES_CHUNKED)
-		V1L_Chunked(req->wrk);
-	err = VDP_DeliverObj(req);
-	if (!err && (req->res_mode & RES_CHUNKED))
-		V1L_EndChunk(req->wrk);
-
 	u = V1L_Close(req->wrk, &bytes);
+	AZ(req->wrk->v1l);
 
 	/* Bytes accounting */
 	if (bytes < hdrbytes)
@@ -193,6 +164,5 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody)
 
 	if ((u || err) && req->sp->fd >= 0)
 		Req_Fail(req, SC_REM_CLOSE);
-	AZ(req->wrk->v1l);
 	VDP_close(req);
 }
diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst
index 2006a97b1..7dcf0850c 100644
--- a/doc/sphinx/whats-new/upgrading-6.0.rst
+++ b/doc/sphinx/whats-new/upgrading-6.0.rst
@@ -123,7 +123,7 @@ taken from :ref:`ref_param_workspace_client`. If you need to reduce
 memory footprint, consider reducing ``workspace_client`` by the amount
 in ``workspace_thread``.
 
-Added :ref:`ref_param_esi_iovs`. tl;dr: Don't touch it, unless advised
+Added `ref_param_esi_iovs`. tl;dr: Don't touch it, unless advised
 to do so by someone familiar with the innards of Varnish.
 
 Changes to VCL
diff --git a/include/tbl/params.h b/include/tbl/params.h
index 7bebdeb4b..f1703613c 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -498,16 +498,17 @@ PARAM(
 )
 
 PARAM(
-	/* name */	esi_iovs,
+	/* name */	http1_iovs,
 	/* typ */	uint,
-	/* min */	"3",
-	/* max */	"1024",	// XXX stringify IOV_MAX
-	/* default */	"10",		// 5 should suffice, add headroom
-	/* units */	"struct iovec",
+	/* min */	"5",
+	/* max */	"1024",		// XXX stringify IOV_MAX
+	/* default */	"64",
+	/* units */	"struct iovec (=16 bytes)",
 	/* flags */	WIZARD,
 	/* s-text */
-	"Number of io vectors to allocate on the thread workspace for "
-	"ESI requests.",
+	"Number of io vectors to allocate for HTTP1 protocol transmission."
+	"  A HTTP1 header needs 7 + 2 per HTTP header field."
+	"  Allocated from workspace_thread.",
 	/* l-text */	"",
 	/* func */	NULL
 )

From phk at FreeBSD.org  Mon Oct 15 20:18:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 15 Oct 2018 20:18:09 +0000 (UTC)
Subject: [master] 11fc2672b Get rid of the trivial/single-file res_mode flags
Message-ID: <20181015201809.32BBAAE08C@lists.varnish-cache.org>


commit 11fc2672b698a6319ad6bffef7cfef3441180ce7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 15 20:16:55 2018 +0000

    Get rid of the trivial/single-file res_mode flags

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index b038ad157..3a3453c60 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -515,12 +515,7 @@ struct req {
 
 	/* Delivery mode */
 	unsigned		res_mode;
-#define RES_LEN			(1<<1)
-#define RES_EOF			(1<<2)
-#define RES_CHUNKED		(1<<3)
 #define RES_ESI			(1<<4)
-#define RES_ESI_CHILD		(1<<5)
-#define RES_GUNZIP		(1<<6)
 #define RES_PIPE		(1<<7)
 
 	/* Transaction VSL buffer */
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 468504fcd..9d1fd4782 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -831,7 +831,6 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody)
 	if (boc == NULL && ObjGetLen(req->wrk, req->objcore) == 0)
 		return;
 
-	req->res_mode |= RES_ESI_CHILD;
 	i = ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED);
 	if (ecx->isgzip && i && !(req->res_mode & RES_ESI)) {
 		ved_stripgzip(req, boc);
diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c
index bfe2f6279..a06bba614 100644
--- a/bin/varnishd/cache/cache_gzip.c
+++ b/bin/varnishd/cache/cache_gzip.c
@@ -299,7 +299,6 @@ vdp_gunzip_init(struct req *req, void **priv)
 		return (-1);
 	}
 
-	req->res_mode |= RES_GUNZIP;
 	VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
 	*priv = vg;
 
diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c
index 580a23e37..b2e061391 100644
--- a/bin/varnishd/http1/cache_http1_deliver.c
+++ b/bin/varnishd/http1/cache_http1_deliver.c
@@ -84,7 +84,7 @@ v1d_error(struct req *req, const char *msg)
 void v_matchproto_(vtr_deliver_f)
 V1D_Deliver(struct req *req, struct boc *boc, int sendbody)
 {
-	int err = 0;
+	int err = 0, chunked = 0;
 	unsigned u;
 	uint64_t hdrbytes, bytes;
 
@@ -103,14 +103,14 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody)
 		http_SetHeader(req->resp, "Connection: keep-alive");
 
 	if (sendbody) {
-		if (http_GetHdr(req->resp, H_Content_Length, NULL))
-			req->res_mode |= RES_LEN;
-		else if (req->http->protover == 11) {
-			req->res_mode |= RES_CHUNKED;
-			http_SetHeader(req->resp, "Transfer-Encoding: chunked");
-		} else {
-			req->res_mode |= RES_EOF;
-			req->doclose = SC_TX_EOF;
+		if (!http_GetHdr(req->resp, H_Content_Length, NULL)) {
+			if (req->http->protover == 11) {
+				chunked = 1;
+				http_SetHeader(req->resp,
+				    "Transfer-Encoding: chunked");
+			} else {
+				req->doclose = SC_TX_EOF;
+			}
 		}
 		if (VDP_Push(req, &v1d_vdp, NULL)) {
 			v1d_error(req, "workspace_thread overflow");
@@ -144,10 +144,10 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody)
 	if (sendbody) {
 		if (DO_DEBUG(DBG_FLUSH_HEAD))
 			(void)V1L_Flush(req->wrk);
-		if (req->res_mode & RES_CHUNKED)
+		if (chunked)
 			V1L_Chunked(req->wrk);
 		err = VDP_DeliverObj(req);
-		if (!err && (req->res_mode & RES_CHUNKED))
+		if (!err && chunked)
 			V1L_EndChunk(req->wrk);
 	}
 

From phk at FreeBSD.org  Tue Oct 16 07:35:13 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 16 Oct 2018 07:35:13 +0000 (UTC)
Subject: [master] 28e6a04f7 Put all generated .rst files in doc/sphinx/include
Message-ID: <20181016073513.6A51660C3E@lists.varnish-cache.org>


commit 28e6a04f7aec186a83559668001e13a90444f215
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 16 07:34:33 2018 +0000

    Put all generated .rst files in doc/sphinx/include

diff --git a/.gitignore b/.gitignore
index 113da821b..c96ca4b91 100644
--- a/.gitignore
+++ b/.gitignore
@@ -82,7 +82,6 @@ cscope.*out
 /man/*.3
 /man/*.7
 /doc/sphinx/include
-/doc/sphinx/*/*generated.rst
 /bin/varnishadm/varnishadm
 /bin/varnishd/varnishd
 /bin/varnishhist/varnishhist
diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index cf67a6106..d9be207ac 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -212,33 +212,33 @@ include/vtc-syntax.rst: vtc-syntax.py $(VTCSYN_SRC)
 	mv ${@}_ ${@}
 BUILT_SOURCES += include/vtc-syntax.rst
 
-reference/vmod_std.generated.rst: reference $(top_builddir)/lib/libvmod_std/vmod_std.rst
-	cp $(top_builddir)/lib/libvmod_std/vmod_std.rst $@ || true
-BUILT_SOURCES += reference/vmod_std.generated.rst
+include/vmod_std.generated.rst: $(top_builddir)/lib/libvmod_std/vmod_std.rst
+	cp $(top_builddir)/lib/libvmod_std/vmod_std.rst $@
+BUILT_SOURCES += include/vmod_std.generated.rst
 
-reference/vmod_directors.generated.rst: reference $(top_builddir)/lib/libvmod_directors/vmod_directors.rst
-	cp $(top_builddir)/lib/libvmod_directors/vmod_directors.rst $@ || true
-BUILT_SOURCES += reference/vmod_directors.generated.rst
+include/vmod_directors.generated.rst: $(top_builddir)/lib/libvmod_directors/vmod_directors.rst
+	cp $(top_builddir)/lib/libvmod_directors/vmod_directors.rst $@
+BUILT_SOURCES += include/vmod_directors.generated.rst
 
-reference/vmod_purge.generated.rst: reference $(top_builddir)/lib/libvmod_purge/vmod_purge.rst
-	cp $(top_builddir)/lib/libvmod_purge/vmod_purge.rst $@ || true
-BUILT_SOURCES += reference/vmod_purge.generated.rst
+include/vmod_purge.generated.rst: $(top_builddir)/lib/libvmod_purge/vmod_purge.rst
+	cp $(top_builddir)/lib/libvmod_purge/vmod_purge.rst $@
+BUILT_SOURCES += include/vmod_purge.generated.rst
 
-reference/vmod_vtc.generated.rst: reference $(top_builddir)/lib/libvmod_vtc/vmod_vtc.rst
-	cp $(top_builddir)/lib/libvmod_vtc/vmod_vtc.rst $@ || true
-BUILT_SOURCES += reference/vmod_vtc.generated.rst
+include/vmod_vtc.generated.rst: $(top_builddir)/lib/libvmod_vtc/vmod_vtc.rst
+	cp $(top_builddir)/lib/libvmod_vtc/vmod_vtc.rst $@
+BUILT_SOURCES += include/vmod_vtc.generated.rst
 
-reference/vmod_blob.generated.rst: reference $(top_builddir)/lib/libvmod_blob/vmod_blob.rst
-	cp $(top_builddir)/lib/libvmod_blob/vmod_blob.rst $@ || true
-BUILT_SOURCES += reference/vmod_blob.generated.rst
+include/vmod_blob.generated.rst: $(top_builddir)/lib/libvmod_blob/vmod_blob.rst
+	cp $(top_builddir)/lib/libvmod_blob/vmod_blob.rst $@
+BUILT_SOURCES += include/vmod_blob.generated.rst
 
-reference/vmod_unix.generated.rst: reference $(top_builddir)/lib/libvmod_unix/vmod_unix.rst
-	cp $(top_builddir)/lib/libvmod_unix/vmod_unix.rst $@ || true
-BUILT_SOURCES += reference/vmod_unix.generated.rst
+include/vmod_unix.generated.rst: $(top_builddir)/lib/libvmod_unix/vmod_unix.rst
+	cp $(top_builddir)/lib/libvmod_unix/vmod_unix.rst $@
+BUILT_SOURCES += include/vmod_unix.generated.rst
 
-reference/vmod_proxy.generated.rst: reference $(top_builddir)/lib/libvmod_proxy/vmod_proxy.rst
-	cp $(top_builddir)/lib/libvmod_proxy/vmod_proxy.rst $@ || true
-BUILT_SOURCES += reference/vmod_proxy.generated.rst
+include/vmod_proxy.generated.rst: $(top_builddir)/lib/libvmod_proxy/vmod_proxy.rst
+	cp $(top_builddir)/lib/libvmod_proxy/vmod_proxy.rst $@
+BUILT_SOURCES += include/vmod_proxy.generated.rst
 
 EXTRA_DIST += $(BUILT_SOURCES)
 MAINTAINERCLEANFILES = $(EXTRA_DIST)
diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst
index 1bf543536..75ec38e99 100644
--- a/doc/sphinx/reference/index.rst
+++ b/doc/sphinx/reference/index.rst
@@ -20,13 +20,7 @@ The Varnish Reference Manual
 	varnishtop.rst
 	vsm.rst
 	vmod.rst
-	vmod_std.generated.rst
-	vmod_directors.generated.rst
-	vmod_vtc.generated.rst
-	vmod_purge.generated.rst
-	vmod_blob.generated.rst
-	vmod_unix.generated.rst
-	vmod_proxy.generated.rst
+	vmod_generated.rst
 	directors.rst
 	varnish-counters.rst
 	vsl.rst
diff --git a/doc/sphinx/reference/vmod_generated.rst b/doc/sphinx/reference/vmod_generated.rst
new file mode 100644
index 000000000..ec89bb9a9
--- /dev/null
+++ b/doc/sphinx/reference/vmod_generated.rst
@@ -0,0 +1,15 @@
+
+.. include::	../include/vmod_std.generated.rst
+
+.. include::	../include/vmod_directors.generated.rst
+
+.. include::	../include/vmod_vtc.generated.rst
+
+.. include::	../include/vmod_purge.generated.rst
+
+.. include::	../include/vmod_blob.generated.rst
+
+.. include::	../include/vmod_unix.generated.rst
+
+.. include::	../include/vmod_proxy.generated.rst
+

From phk at FreeBSD.org  Tue Oct 16 09:06:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 16 Oct 2018 09:06:10 +0000 (UTC)
Subject: [master] ee916f8b2 Update collection copyright year
Message-ID: <20181016090611.1646B6279D@lists.varnish-cache.org>


commit ee916f8b208029a1428b64fe02d11929456efe3b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 16 07:58:50 2018 +0000

    Update collection copyright year

diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst
index 07077ee48..9a708b593 100644
--- a/doc/sphinx/index.rst
+++ b/doc/sphinx/index.rst
@@ -37,7 +37,7 @@ Longer listings like example command output and VCL look like this::
     $ /opt/varnish/sbin/varnishd -V
     varnishd (varnish-trunk revision 199de9b)
     Copyright (c) 2006 Verdens Gang AS
-    Copyright (c) 2006-2015 Varnish Software AS
+    Copyright (c) 2006-2018 Varnish Software AS
 
 
 .. For maintainers:
diff --git a/lib/libvarnish/version.c b/lib/libvarnish/version.c
index 61711654e..1b9a42196 100644
--- a/lib/libvarnish/version.c
+++ b/lib/libvarnish/version.c
@@ -44,5 +44,5 @@ VCS_Message(const char *progname)
 {
 	fprintf(stderr, "%s (%s)\n", progname, VCS_version);
 	fprintf(stderr, "Copyright (c) 2006 Verdens Gang AS\n");
-	fprintf(stderr, "Copyright (c) 2006-2015 Varnish Software AS\n");
+	fprintf(stderr, "Copyright (c) 2006-2018 Varnish Software AS\n");
 }

From phk at FreeBSD.org  Tue Oct 16 09:06:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 16 Oct 2018 09:06:10 +0000 (UTC)
Subject: [master] c8cea2e37 Copyediting
Message-ID: <20181016090610.DB1D16279B@lists.varnish-cache.org>


commit c8cea2e3751bd62dcfc1f2a7dfe817759f04698f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 16 07:55:41 2018 +0000

    Copyediting

diff --git a/doc/sphinx/dev-guide/index.rst b/doc/sphinx/dev-guide/index.rst
index fe4d75986..f1c818245 100644
--- a/doc/sphinx/dev-guide/index.rst
+++ b/doc/sphinx/dev-guide/index.rst
@@ -41,18 +41,17 @@ Technical stuff
 
 * Our license is BSD 2-clause or looser, no GPL or LGPL.
 
-* We havn't had a major security issue in 10 years, and we're not going
-  to start having them now.
+* It took 11 years for the first major security issue, and that was too soon.
 
 Bugs, issues, feature requests & VIPs
 -------------------------------------
 
 Bugs, issues and feature requests start out as github issues.
 
-We do a "bug-wash" every monday at 13:00 EU time, where new and otherwise
-worthy of discussion issues are pow-wowed.
+Monday at 13:00-14:00 (EU time) we "bug-wash" on IRC to
+decide who and how issues are dealt with.
 
-Tickets we cannot do anything about are closed.
+Issues we cannot do anything about are closed.
 
 If feature-requests make sense, they get moved to a wiki/VIP page until
 somebody implements them.

From phk at FreeBSD.org  Tue Oct 16 10:18:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 16 Oct 2018 10:18:10 +0000 (UTC)
Subject: [master] f56cce5c2 Make worker threads give up their VCL much sooner
 in VTC-environment and make sure the expects in the test are scrupulusly
 correct.
Message-ID: <20181016101810.5433F64094@lists.varnish-cache.org>


commit f56cce5c2b19d9e8c5ed8bee69c5b7c9578bac54
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 16 10:16:00 2018 +0000

    Make worker threads give up their VCL much sooner in VTC-environment
    and make sure the expects in the test are scrupulusly correct.

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 50839143b..4d546ffd3 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -321,6 +321,7 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk)
 {
 	struct pool_task *tp = NULL;
 	struct pool_task tpx, tps;
+	vtim_real tmo;
 	int i, prio_lim;
 
 	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
@@ -370,8 +371,13 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk)
 			pp->nidle++;
 			do {
 				// see signaling_note at the top for explanation
-				i = Lck_CondWait(&wrk->cond, &pp->mtx,
-				    wrk->vcl == NULL ?  0 : wrk->lastused+60.);
+				if (wrk->vcl == NULL)
+					tmo = 0;
+				else if (DO_DEBUG(DBG_VTC_MODE))
+					tmo =  wrk->lastused+1.;
+				else
+					tmo =  wrk->lastused+60.;
+				i = Lck_CondWait(&wrk->cond, &pp->mtx, tmo);
 				if (i == ETIMEDOUT)
 					VCL_Rel(&wrk->vcl);
 			} while (wrk->task.func == NULL);
diff --git a/bin/varnishtest/tests/v00006.vtc b/bin/varnishtest/tests/v00006.vtc
index 9b84076d0..4a187c8a7 100644
--- a/bin/varnishtest/tests/v00006.vtc
+++ b/bin/varnishtest/tests/v00006.vtc
@@ -18,7 +18,9 @@ varnish v1 -arg "-p thread_pools=1" -vcl+backend { } -start
 
 # Give the varnishd a chance to start and create workers etc.
 # NB: This is important for to avoid mis-ordering of the workers.
-delay 1
+# delay 1
+
+varnish v1 -expect MAIN.threads == 10
 
 client c1 {
 	txreq -url "/bar"
@@ -47,6 +49,8 @@ varnish v1 -vcl {
 	}
 }
 
+varnish v1 -vsl_catchup
+
 varnish v1 -expect n_backend == 2
 varnish v1 -expect n_vcl_avail == 2
 varnish v1 -expect n_vcl_discard == 0
@@ -57,13 +61,17 @@ varnish v1 -cli "vcl.list"
 
 varnish v1 -cli "vcl.discard vcl1"
 
+varnish v1 -vsl_catchup
+
 # Give expiry thread a chance to let go
 delay 2
 
-# It won't go away as long as the workthread holds a VCL reference
-varnish v1 -expect n_backend == 2
-varnish v1 -expect n_vcl_avail == 1
-varnish v1 -expect n_vcl_discard == 1
+varnish v1 -vsl_catchup
+
+# It may not go away as long as the workthread holds a VCL reference
+varnish v1 -expect n_backend >= 1
+varnish v1 -expect n_vcl_avail >= 1
+varnish v1 -expect n_vcl_discard >= 0
 
 # Do another request through the new VCL to the new backend
 client c1 {
@@ -72,6 +80,8 @@ client c1 {
 	expect resp.status == 200
 } -run
 
+varnish v1 -vsl_catchup
+
 server s2 -wait
 
 # The workthread should have released its VCL reference now

From phk at FreeBSD.org  Tue Oct 16 10:18:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 16 Oct 2018 10:18:10 +0000 (UTC)
Subject: [master] b9459df18 Report errno on asserts
Message-ID: <20181016101810.48AC96405B@lists.varnish-cache.org>


commit b9459df18281b30c3b0fd60135a44bb4c5e83d6c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 16 09:13:45 2018 +0000

    Report errno on asserts

diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c
index 94bee3d3f..1e6e321e8 100644
--- a/bin/varnishtest/vtc_log.c
+++ b/bin/varnishtest/vtc_log.c
@@ -28,6 +28,7 @@
 
 #include "config.h"
 
+#include <errno.h>
 #include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -273,17 +274,19 @@ vtc_log_VAS_Fail(const char *func, const char *file, int line,
     const char *cond, enum vas_e why)
 {
 	struct vtclog *vl;
+	int e = errno;
 
 	(void)why;
 	vl = pthread_getspecific(log_key);
 	if (vl == NULL || vl->act) {
 		fprintf(stderr,
 		    "Assert error in %s(), %s line %d:\n"
-		    "  Condition(%s) not true.\n",
-		    func, file, line, cond);
+		    "  Condition(%s) not true. (errno=%d %s)\n",
+		    func, file, line, cond, e, strerror(e));
 	} else
 		vtc_fatal(vl, "Assert error in %s(), %s line %d:"
-		    "  Condition(%s) not true.\n", func, file, line, cond);
+		    "  Condition(%s) not true."
+		    "  Errno=%d %s", func, file, line, cond, e, strerror(e));
 	abort();
 }
 

From dridi.boukelmoune at gmail.com  Wed Oct 17 14:35:13 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 17 Oct 2018 14:35:13 +0000 (UTC)
Subject: [master] 75f3aea04 Do the same thing with VCL_* for consistency
Message-ID: <20181017143513.1F6EF636B@lists.varnish-cache.org>


commit 75f3aea049204daa3fc297c202b3bfe1a4c24045
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Oct 17 16:11:16 2018 +0200

    Do the same thing with VCL_* for consistency

diff --git a/include/vrt.h b/include/vrt.h
index c04274a4a..8aabdbf83 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -199,19 +199,19 @@ struct vrt_ctx {
 
 	struct vsb			*msg;	// Only in ...init()
 	struct vsl_log			*vsl;
-	struct vcl			*vcl;
+	VCL_VCL				vcl;
 	struct ws			*ws;
 
 	struct sess			*sp;
 
 	struct req			*req;
-	struct http			*http_req;
-	struct http			*http_req_top;
-	struct http			*http_resp;
+	VCL_HTTP			http_req;
+	VCL_HTTP			http_req_top;
+	VCL_HTTP			http_resp;
 
 	struct busyobj			*bo;
-	struct http			*http_bereq;
-	struct http			*http_beresp;
+	VCL_HTTP			http_bereq;
+	VCL_HTTP			http_beresp;
 
 	double				now;
 
@@ -301,8 +301,8 @@ struct vrt_backend {
 	unsigned			magic;
 #define VRT_BACKEND_MAGIC		0x4799ce6b
 	VRT_BACKEND_FIELDS(const)
-	const struct suckaddr		*ipv4_suckaddr;
-	const struct suckaddr		*ipv6_suckaddr;
+	VCL_IP				ipv4_suckaddr;
+	VCL_IP				ipv6_suckaddr;
 	VCL_PROBE			probe;
 };
 
@@ -388,7 +388,7 @@ struct gethdr_s {
 };
 
 VCL_HTTP VRT_selecthttp(VRT_CTX, enum gethdr_e);
-VCL_STRING VRT_GetHdr(VRT_CTX, const struct gethdr_s *);
+VCL_STRING VRT_GetHdr(VRT_CTX, VCL_HEADER);
 
 /***********************************************************************
  * req related
@@ -404,7 +404,7 @@ VCL_INT VRT_purge(VRT_CTX, VCL_DURATION, VCL_DURATION, VCL_DURATION);
 VCL_VOID VRT_synth(VRT_CTX, VCL_INT, VCL_STRING);
 VCL_VOID VRT_hit_for_pass(VRT_CTX, VCL_DURATION);
 
-VCL_VOID VRT_SetHdr(VRT_CTX, const struct gethdr_s *, const char *, ...);
+VCL_VOID VRT_SetHdr(VRT_CTX, VCL_HEADER, const char *, ...);
 VCL_VOID VRT_handling(VRT_CTX, unsigned hand);
 VCL_VOID VRT_fail(VRT_CTX, const char *fmt, ...) v_printflike_(2,3);
 VCL_VOID VRT_hashdata(VRT_CTX, const char *str, ...);
@@ -478,7 +478,7 @@ void VRT_DisableDirector(VCL_BACKEND);
 void VRT_DelDirector(VCL_BACKEND *);
 
 /* Suckaddr related */
-int VRT_VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst);
+int VRT_VSA_GetPtr(VCL_IP sua, const unsigned char ** dst);
 
 /* VMOD/Modules related */
 int VRT_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len,

From dridi.boukelmoune at gmail.com  Wed Oct 17 14:35:13 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 17 Oct 2018 14:35:13 +0000 (UTC)
Subject: [master] 3e5d51575 Also use the VCL_PROBE type where it makes sense
Message-ID: <20181017143513.17B656369@lists.varnish-cache.org>


commit 3e5d5157550f84c7ecb22e8afe80d862af678c20
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Oct 17 16:07:01 2018 +0200

    Also use the VCL_PROBE type where it makes sense
    
    Refs c9d3b3ff5ff3fc400fe72f1e7d66693f88e1b04f

diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c
index 7b9d7fed8..62f42ff45 100644
--- a/bin/varnishd/cache/cache_vrt_vcl.c
+++ b/bin/varnishd/cache/cache_vrt_vcl.c
@@ -262,7 +262,7 @@ VCL_Name(const struct vcl *vcl)
 	return (vcl->loaded_name);
 }
 
-const struct vrt_backend_probe *
+VCL_PROBE
 VCL_DefaultProbe(const struct vcl *vcl)
 {
 
diff --git a/include/vrt.h b/include/vrt.h
index f7508446e..c04274a4a 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -303,7 +303,7 @@ struct vrt_backend {
 	VRT_BACKEND_FIELDS(const)
 	const struct suckaddr		*ipv4_suckaddr;
 	const struct suckaddr		*ipv6_suckaddr;
-	const struct vrt_backend_probe	*probe;
+	VCL_PROBE			probe;
 };
 
 #define VRT_BACKEND_PROBE_FIELDS(rigid)				\
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 5dba1a37a..763bad5ea 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -652,7 +652,7 @@ struct VCL_conf {
 
     unsigned            syntax;
     VCL_BACKEND             *default_director;
-    const struct vrt_backend_probe  *default_probe;
+    VCL_PROBE               default_probe;
     unsigned            nref;
     const struct vrt_ref        *ref;
 

From dridi.boukelmoune at gmail.com  Wed Oct 17 14:35:13 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 17 Oct 2018 14:35:13 +0000 (UTC)
Subject: [master] 346affe30 Generate include/vcl.h with proper whitespace
Message-ID: <20181017143513.39FFC636E@lists.varnish-cache.org>


commit 346affe301a7bc9e643ce2e1b18e8717bc17e18b
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Oct 17 16:29:32 2018 +0200

    Generate include/vcl.h with proper whitespace

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 763bad5ea..5f7191f11 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -647,26 +647,26 @@ typedef void vcl_fini_f(VRT_CTX);
 typedef void vcl_func_f(VRT_CTX);
 
 struct VCL_conf {
-    unsigned            magic;
-#define VCL_CONF_MAGIC          0x7406c509      /* from /dev/random */
+	unsigned		magic;
+#define VCL_CONF_MAGIC		0x7406c509      /* from /dev/random */
 
-    unsigned            syntax;
-    VCL_BACKEND             *default_director;
-    VCL_PROBE               default_probe;
-    unsigned            nref;
-    const struct vrt_ref        *ref;
+	unsigned		syntax;
+	VCL_BACKEND		*default_director;
+	VCL_PROBE		default_probe;
+	unsigned		nref;
+	const struct vrt_ref	*ref;
 
-    unsigned            nsrc;
-    const char              **srcname;
-    const char              **srcbody;
+	unsigned		nsrc;
+	const char		**srcname;
+	const char		**srcbody;
 
-    unsigned            nvmod;
+	unsigned		nvmod;
 
-    vcl_event_f             *event_vcl;
+	vcl_event_f		*event_vcl;
 """)
 
 for i in returns:
-    fo.write("\tvcl_func_f\t\t\t*" + i[0] + "_func;\n")
+    fo.write("\tvcl_func_f\t\t*" + i[0] + "_func;\n")
 
 fo.write("\n};\n")
 fo.close()

From nils.goroll at uplex.de  Thu Oct 18 13:46:08 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 18 Oct 2018 13:46:08 +0000 (UTC)
Subject: [master] bc5967289 gc duplicate initialization
Message-ID: <20181018134608.740C6A8CF0@lists.varnish-cache.org>


commit bc59672891cff5eae54cd9ccfc6fb557c252d0a5
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Oct 18 15:41:57 2018 +0200

    gc duplicate initialization
    
    we INIT_OBJ the v1l, which includes a memset(0)

diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c
index 7897cca4c..50a38e5cd 100644
--- a/bin/varnishd/http1/cache_http1_line.c
+++ b/bin/varnishd/http1/cache_http1_line.c
@@ -114,9 +114,6 @@ V1L_Open(struct worker *wrk, struct ws *ws, int *fd, struct vsl_log *vsl,
 	v1l->iov = (void*)ws->f;
 	v1l->siov = u;
 	v1l->ciov = u;
-	v1l->werr = 0;
-	v1l->liov = 0;
-	v1l->niov = 0;
 	v1l->wfd = fd;
 	v1l->t0 = t0;
 	v1l->vsl = vsl;

From nils.goroll at uplex.de  Thu Oct 18 17:08:10 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 18 Oct 2018 17:08:10 +0000 (UTC)
Subject: [master] 5c60aaa9a remove a useless volatile
Message-ID: <20181018170810.57984AE6C2@lists.varnish-cache.org>


commit 5c60aaa9af86d4d3fa456057a26f2b92e5de7cf1
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Oct 18 17:59:05 2018 +0200

    remove a useless volatile
    
    ... introduced with 8c113ff64b78554f6219f14774fed418c466318c
    
    Here it seems the intention was to make sure that the access to the
    parameter happens uncached, but even then it would not have helped.

diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c
index febc57ff3..569f7e06e 100644
--- a/bin/varnishd/cache/cache_ban_lurker.c
+++ b/bin/varnishd/cache/cache_ban_lurker.c
@@ -408,7 +408,7 @@ void * v_matchproto_(bgthread_t)
 ban_lurker(struct worker *wrk, void *priv)
 {
 	struct vsl_log vsl;
-	volatile vtim_real d;
+	vtim_real d;
 	vtim_dur dt;
 	unsigned gen = ban_generation + 1;
 

From nils.goroll at uplex.de  Thu Oct 18 17:08:10 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 18 Oct 2018 17:08:10 +0000 (UTC)
Subject: [master] 79687f134 more double->vtim_*
Message-ID: <20181018170810.499E6AE6C0@lists.varnish-cache.org>


commit 79687f1344387a26d5e3b0a3b0454ac917db76de
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Oct 18 17:55:37 2018 +0200

    more double->vtim_*
    
    Reviews appreciated.
    
    I will most likely continue applying these changes in smaller chunks
    because a) this kind of work is boring and b) in the hope that smaller
    chunks will get reviewed better.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 3a3453c60..a829a43d7 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -342,10 +342,10 @@ struct objcore {
 	struct storeobj		stobj[1];
 	struct objhead		*objhead;
 	struct boc		*boc;
-	double			timer_when;
+	vtim_real		timer_when;
 	long			hits;
 
-	double			t_origin;
+	vtim_real		t_origin;
 	float			ttl;
 	float			grace;
 	float			keep;
@@ -357,7 +357,7 @@ struct objcore {
 	uint16_t		oa_present;
 
 	unsigned		timer_idx;	// XXX 4Gobj limit
-	double			last_lru;
+	vtim_real		last_lru;
 	VTAILQ_ENTRY(objcore)	hsh_list;
 	VTAILQ_ENTRY(objcore)	lru_list;
 	VTAILQ_ENTRY(objcore)	ban_list;
@@ -419,8 +419,8 @@ struct busyobj {
 	vtim_dur		between_bytes_timeout;
 
 	/* Timers */
-	double			t_first;	/* First timestamp logged */
-	double			t_prev;		/* Previous timestamp logged */
+	vtim_real		t_first;	/* First timestamp logged */
+	vtim_real		t_prev;		/* Previous timestamp logged */
 
 	/* Acct */
 	struct acct_bereq	acct;
@@ -478,8 +478,8 @@ struct req {
 
 	uint8_t			digest[DIGEST_LEN];
 
-	double			d_ttl;
-	double			d_grace;
+	vtim_dur		d_ttl;
+	vtim_dur		d_grace;
 
 	ssize_t			req_bodybytes;	/* Parsed req bodybytes */
 	const struct stevedore	*storage;
@@ -490,9 +490,9 @@ struct req {
 	uintptr_t		ws_req;		/* WS above request data */
 
 	/* Timestamps */
-	double			t_first;	/* First timestamp logged */
-	double			t_prev;		/* Previous timestamp logged */
-	double			t_req;		/* Headers complete */
+	vtim_real		t_first;	/* First timestamp logged */
+	vtim_real		t_prev;		/* Previous timestamp logged */
+	vtim_real		t_req;		/* Headers complete */
 
 	struct http_conn	*htc;
 	struct vfp_ctx		*vfc;
@@ -595,7 +595,7 @@ int HTTP_Decode(struct http *to, const uint8_t *fm);
 void http_ForceHeader(struct http *to, const char *hdr, const char *val);
 void http_PrintfHeader(struct http *to, const char *fmt, ...)
     v_printflike_(2, 3);
-void http_TimeHeader(struct http *to, const char *fmt, double now);
+void http_TimeHeader(struct http *to, const char *fmt, vtim_real now);
 void http_Proto(struct http *to);
 void http_SetHeader(struct http *to, const char *hdr);
 void http_SetH(struct http *to, unsigned n, const char *fm);
@@ -710,12 +710,12 @@ void VSLbv(struct vsl_log *, enum VSL_tag_e tag, const char *fmt, va_list va);
 void VSLb(struct vsl_log *, enum VSL_tag_e tag, const char *fmt, ...)
     v_printflike_(3, 4);
 void VSLbt(struct vsl_log *, enum VSL_tag_e tag, txt t);
-void VSLb_ts(struct vsl_log *, const char *event, double first, double *pprev,
-    double now);
+void VSLb_ts(struct vsl_log *, const char *event, vtim_real first,
+    vtim_real *pprev, vtim_real now);
 void VSLb_bin(struct vsl_log *, enum VSL_tag_e, ssize_t, const void*);
 
 static inline void
-VSLb_ts_req(struct req *req, const char *event, double now)
+VSLb_ts_req(struct req *req, const char *event, vtim_real now)
 {
 
 	if (isnan(req->t_first) || req->t_first == 0.)
@@ -724,7 +724,7 @@ VSLb_ts_req(struct req *req, const char *event, double now)
 }
 
 static inline void
-VSLb_ts_busyobj(struct busyobj *bo, const char *event, double now)
+VSLb_ts_busyobj(struct busyobj *bo, const char *event, vtim_real now)
 {
 
 	if (isnan(bo->t_first) || bo->t_first == 0.)
@@ -775,7 +775,7 @@ WS_Front(const struct ws *ws)
 }
 
 /* cache_rfc2616.c */
-void RFC2616_Ttl(struct busyobj *, double now, double *t_origin,
+void RFC2616_Ttl(struct busyobj *, vtim_real now, vtim_real *t_origin,
     float *ttl, float *grace, float *keep);
 unsigned RFC2616_Req_Gzip(const struct http *);
 int RFC2616_Do_Cond(const struct req *sp);
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index fab8661cf..a8c9a08c1 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -118,10 +118,10 @@ BAN_Release(void)
  * Extract time and length from ban-spec
  */
 
-double
+vtim_real
 ban_time(const uint8_t *banspec)
 {
-	double t;
+	vtim_real t;
 
 	assert(sizeof t == (BANS_LENGTH - BANS_TIMESTAMP));
 	memcpy(&t, banspec, sizeof t);
@@ -257,10 +257,10 @@ BAN_DestroyObj(struct objcore *oc)
  */
 
 struct ban *
-BAN_FindBan(double t0)
+BAN_FindBan(vtim_real t0)
 {
 	struct ban *b;
-	double t1;
+	vtim_real t1;
 
 	assert(ban_holds > 0);
 	VTAILQ_FOREACH(b, &ban_head, list) {
@@ -358,7 +358,7 @@ ban_reload(const uint8_t *ban, unsigned len)
 {
 	struct ban *b, *b2;
 	int duplicate = 0;
-	double t0, t1, t2 = 9e99;
+	vtim_real t0, t1, t2 = 9e99;
 	ASSERT_CLI();
 	Lck_AssertHeld(&ban_mtx);
 
@@ -440,7 +440,7 @@ BAN_Reload(const uint8_t *ptr, unsigned len)
  * Get a bans timestamp
  */
 
-double
+vtim_real
 BAN_Time(const struct ban *b)
 {
 
diff --git a/bin/varnishd/cache/cache_ban.h b/bin/varnishd/cache/cache_ban.h
index 62fc767bb..1a9420825 100644
--- a/bin/varnishd/cache/cache_ban.h
+++ b/bin/varnishd/cache/cache_ban.h
@@ -116,7 +116,7 @@ void ban_info_drop(const uint8_t *ban, unsigned len);
 
 int ban_evaluate(struct worker *wrk, const uint8_t *bs, struct objcore *oc,
     const struct http *reqhttp, unsigned *tests);
-double ban_time(const uint8_t *banspec);
+vtim_real ban_time(const uint8_t *banspec);
 int ban_equal(const uint8_t *bs1, const uint8_t *bs2);
 void BAN_Free(struct ban *b);
 void ban_kick_lurker(void);
diff --git a/bin/varnishd/cache/cache_ban_build.c b/bin/varnishd/cache/cache_ban_build.c
index 1ff460057..e3b9873a8 100644
--- a/bin/varnishd/cache/cache_ban_build.c
+++ b/bin/varnishd/cache/cache_ban_build.c
@@ -250,7 +250,7 @@ BAN_Commit(struct ban_proto *bp)
 {
 	struct ban  *b, *bi;
 	ssize_t ln;
-	double t0;
+	vtim_real t0;
 
 	CHECK_OBJ_NOTNULL(bp, BAN_PROTO_MAGIC);
 	AN(bp->vsb);
diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c
index 1729e04a9..febc57ff3 100644
--- a/bin/varnishd/cache/cache_ban_lurker.c
+++ b/bin/varnishd/cache/cache_ban_lurker.c
@@ -315,12 +315,13 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
  * completed.
  */
 
-static double
+static vtim_dur
 ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
 {
 	struct ban *b, *bd;
 	struct banhead_s obans;
-	double d, dt, n;
+	vtim_real d;
+	vtim_dur dt, n;
 	unsigned count = 0, cutoff = UINT_MAX;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -407,7 +408,8 @@ void * v_matchproto_(bgthread_t)
 ban_lurker(struct worker *wrk, void *priv)
 {
 	struct vsl_log vsl;
-	volatile double d;
+	volatile vtim_real d;
+	vtim_dur dt;
 	unsigned gen = ban_generation + 1;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -416,10 +418,10 @@ ban_lurker(struct worker *wrk, void *priv)
 	VSL_Setup(&vsl, NULL, 0);
 
 	while (!ban_shutdown) {
-		d = ban_lurker_work(wrk, &vsl);
+		dt = ban_lurker_work(wrk, &vsl);
 		if (DO_DEBUG(DBG_LURKER))
-			VSLb(&vsl, SLT_Debug, "lurker: sleep = %lf", d);
-		d += VTIM_real();
+			VSLb(&vsl, SLT_Debug, "lurker: sleep = %lf", dt);
+		d = VTIM_real() + dt;
 		Lck_Lock(&ban_mtx);
 		if (gen == ban_generation) {
 			Pool_Sumstat(wrk);
diff --git a/bin/varnishd/cache/cache_director.h b/bin/varnishd/cache/cache_director.h
index bf7c37f24..d4553e150 100644
--- a/bin/varnishd/cache/cache_director.h
+++ b/bin/varnishd/cache/cache_director.h
@@ -39,7 +39,7 @@ struct vcldir {
 	const struct vdi_methods	*methods;
 	VTAILQ_ENTRY(vcldir)		list;
 	const struct vdi_ahealth	*admin_health;
-	double				health_changed;
+	vtim_real			health_changed;
 	char				*cli_name;
 };
 
diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 3f2a11caf..85a435741 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -61,10 +61,10 @@ static struct exp_priv *exphdl;
  * if it is available.
  */
 
-double
+vtim_real
 EXP_Ttl(const struct req *req, const struct objcore *oc)
 {
-	double r;
+	vtim_dur r;
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 
@@ -79,10 +79,10 @@ EXP_Ttl(const struct req *req, const struct objcore *oc)
  * account if it is available.
  */
 
-double
+vtim_real
 EXP_Ttl_grace(const struct req *req, const struct objcore *oc)
 {
-	double g;
+	vtim_dur g;
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 
@@ -160,9 +160,10 @@ EXP_Insert(struct worker *wrk, struct objcore *oc)
  */
 
 void
-EXP_Rearm(struct objcore *oc, double now, double ttl, double grace, double keep)
+EXP_Rearm(struct objcore *oc, vtim_real now,
+    vtim_dur ttl, vtim_dur grace, vtim_dur keep)
 {
-	double when;
+	vtim_real when;
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	assert(oc->refcnt > 0);
@@ -245,8 +246,8 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, unsigned flags)
  * Expire stuff from the binheap
  */
 
-static double
-exp_expire(struct exp_priv *ep, double now)
+static vtim_real
+exp_expire(struct exp_priv *ep, vtim_real now)
 {
 	struct objcore *oc;
 
@@ -322,7 +323,7 @@ static void * v_matchproto_(bgthread_t)
 exp_thread(struct worker *wrk, void *priv)
 {
 	struct objcore *oc;
-	double t = 0, tnext = 0;
+	vtim_real t = 0, tnext = 0;
 	struct exp_priv *ep;
 	unsigned flags = 0;
 
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 9a08a37c6..1056eca9f 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -48,7 +48,7 @@ vbf_allocobj(struct busyobj *bo, unsigned l)
 {
 	struct objcore *oc;
 	const struct stevedore *stv;
-	double lifetime;
+	vtim_dur lifetime;
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	oc = bo->fetch_objcore;
@@ -258,7 +258,7 @@ static enum fetch_step
 vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 {
 	int i;
-	double now;
+	vtim_real now;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -679,7 +679,7 @@ static enum fetch_step
 vbf_stp_error(struct worker *wrk, struct busyobj *bo)
 {
 	ssize_t l, ll, o;
-	double now;
+	vtim_real now;
 	uint8_t *ptr;
 	struct vsb *synth_body;
 
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index bb32d4fc2..d8cd7e332 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -344,7 +344,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 	struct objhead *oh;
 	struct objcore *oc;
 	struct objcore *exp_oc;
-	double exp_t_origin;
+	vtim_real exp_t_origin;
 	int busy_found;
 	enum lookup_e retval;
 	const uint8_t *vary;
@@ -613,8 +613,8 @@ hsh_rush2(struct worker *wrk, struct rush *r)
  */
 
 unsigned
-HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl_now, double ttl,
-double grace, double keep)
+HSH_Purge(struct worker *wrk, struct objhead *oh, vtim_real ttl_now,
+    vtim_dur ttl, vtim_dur grace, vtim_dur keep)
 {
 	struct objcore *oc, **ocp;
 	unsigned spc, ospc, nobj, n, n_tot = 0;
diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 23eaa0b18..34cd11a02 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -1228,7 +1228,7 @@ http_PrintfHeader(struct http *to, const char *fmt, ...)
 }
 
 void
-http_TimeHeader(struct http *to, const char *fmt, double now)
+http_TimeHeader(struct http *to, const char *fmt, vtim_real now)
 {
 	char *p;
 
diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c
index ad182c4b0..1e6e689ac 100644
--- a/bin/varnishd/cache/cache_lck.c
+++ b/bin/varnishd/cache/cache_lck.c
@@ -191,7 +191,7 @@ Lck_CondWait(pthread_cond_t *cond, struct lock *lck, vtim_real when)
 {
 	struct ilck *ilck;
 	struct timespec ts;
-	double t;
+	vtim_real t;
 
 	CAST_OBJ_NOTNULL(ilck, lck->priv, ILCK_MAGIC);
 	AN(ilck->held);
diff --git a/bin/varnishd/cache/cache_mempool.c b/bin/varnishd/cache/cache_mempool.c
index 62407f607..aa7adc0c4 100644
--- a/bin/varnishd/cache/cache_mempool.c
+++ b/bin/varnishd/cache/cache_mempool.c
@@ -44,7 +44,7 @@ struct memitem {
 #define MEMITEM_MAGIC			0x42e55401
 	unsigned			size;
 	VTAILQ_ENTRY(memitem)		list;
-	double				touched;
+	vtim_real			touched;	// XXX -> mono?
 };
 
 VTAILQ_HEAD(memhead_s, memitem);
@@ -63,7 +63,7 @@ struct mempool {
 	struct VSC_mempool		*vsc;
 	unsigned			n_pool;
 	pthread_t			thread;
-	double				t_now;
+	vtim_real			t_now;	// XXX -> mono?
 	int				self_destruct;
 };
 
@@ -99,8 +99,8 @@ mpl_guard(void *priv)
 {
 	struct mempool *mpl;
 	struct memitem *mi = NULL;
-	double v_statevariable_(mpl_slp);
-	double last = 0;
+	vtim_dur v_statevariable_(mpl_slp);
+	vtim_real last = 0;
 
 	CAST_OBJ_NOTNULL(mpl, priv, MEMPOOL_MAGIC);
 	THR_SetName(mpl->name);
diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c
index a3f2d182e..77a3771a7 100644
--- a/bin/varnishd/cache/cache_obj.c
+++ b/bin/varnishd/cache/cache_obj.c
@@ -481,7 +481,7 @@ ObjSetAttr(struct worker *wrk, struct objcore *oc, enum obj_attr attr,
  */
 
 void
-ObjTouch(struct worker *wrk, struct objcore *oc, double now)
+ObjTouch(struct worker *wrk, struct objcore *oc, vtim_real now)
 {
 	const struct obj_methods *om = obj_getmethods(oc);
 
diff --git a/bin/varnishd/cache/cache_objhead.h b/bin/varnishd/cache/cache_objhead.h
index 6bcb7b301..81bba4da0 100644
--- a/bin/varnishd/cache/cache_objhead.h
+++ b/bin/varnishd/cache/cache_objhead.h
@@ -72,7 +72,7 @@ enum lookup_e HSH_Lookup(struct req *, struct objcore **, struct objcore **,
     int always_insert);
 void HSH_Ref(struct objcore *o);
 void HSH_AddString(struct req *, void *ctx, const char *str);
-unsigned HSH_Purge(struct worker *, struct objhead *, double ttl_now,
-    double ttl, double grace, double keep);
+unsigned HSH_Purge(struct worker *, struct objhead *, vtim_dur ttl_now,
+    vtim_dur ttl, vtim_dur grace, vtim_dur keep);
 struct objcore *HSH_Private(const struct worker *wrk);
 void HSH_Abandon(struct objcore *oc);
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index 0654b9f6b..791fa30b8 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -107,9 +107,9 @@ void VBP_Init(void);
 void BAN_Hold(void);
 void BAN_Release(void);
 void BAN_Reload(const uint8_t *ban, unsigned len);
-struct ban *BAN_FindBan(double t0);
+struct ban *BAN_FindBan(vtim_real t0);
 void BAN_RefBan(struct objcore *oc, struct ban *);
-double BAN_Time(const struct ban *ban);
+vtim_real BAN_Time(const struct ban *ban);
 
 /* cache_busyobj.c */
 struct busyobj *VBO_GetBusyObj(struct worker *, const struct req *);
@@ -125,8 +125,8 @@ void VDI_Event(const struct director *d, enum vcl_event_e ev);
 void VDI_Init(void);
 
 /* cache_exp.c */
-double EXP_Ttl(const struct req *, const struct objcore *);
-double EXP_Ttl_grace(const struct req *, const struct objcore *oc);
+vtim_real EXP_Ttl(const struct req *, const struct objcore *);
+vtim_real EXP_Ttl_grace(const struct req *, const struct objcore *oc);
 void EXP_Insert(struct worker *wrk, struct objcore *oc);
 void EXP_Remove(struct objcore *);
 
@@ -161,9 +161,8 @@ void EXP_Remove(struct objcore *);
 	((to)->t_origin + (to)->ttl + (to)->grace + (to)->keep)
 
 /* cache_exp.c */
-void EXP_Rearm(struct objcore *, double now, double ttl, double grace,
-    double keep);
-
+void EXP_Rearm(struct objcore *oc, vtim_real now,
+    vtim_dur ttl, vtim_dur grace, vtim_dur keep);
 
 /* From cache_main.c */
 void BAN_Init(void);
diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h
index 212142ceb..0db48e3de 100644
--- a/bin/varnishd/storage/storage.h
+++ b/bin/varnishd/storage/storage.h
@@ -139,10 +139,10 @@ uintmax_t STV_FileSize(int fd, const char *size, unsigned *granularity,
 /*--------------------------------------------------------------------*/
 struct lru *LRU_Alloc(void);
 void LRU_Free(struct lru **);
-void LRU_Add(struct objcore *, double now);
+void LRU_Add(struct objcore *, vtim_real now);
 void LRU_Remove(struct objcore *);
 int LRU_NukeOne(struct worker *, struct lru *);
-void LRU_Touch(struct worker *, struct objcore *, double now);
+void LRU_Touch(struct worker *, struct objcore *, vtim_real now);
 
 /*--------------------------------------------------------------------*/
 extern const struct stevedore smu_stevedore;
diff --git a/bin/varnishd/storage/storage_lru.c b/bin/varnishd/storage/storage_lru.c
index d5dcf6a68..381612bb8 100644
--- a/bin/varnishd/storage/storage_lru.c
+++ b/bin/varnishd/storage/storage_lru.c
@@ -79,7 +79,7 @@ LRU_Free(struct lru **pp)
 }
 
 void
-LRU_Add(struct objcore *oc, double now)
+LRU_Add(struct objcore *oc, vtim_real now)
 {
 	struct lru *lru;
 
@@ -121,7 +121,7 @@ LRU_Remove(struct objcore *oc)
 }
 
 void v_matchproto_(objtouch_f)
-LRU_Touch(struct worker *wrk, struct objcore *oc, double now)
+LRU_Touch(struct worker *wrk, struct objcore *oc, vtim_real now)
 {
 	struct lru *lru;
 

From phk at FreeBSD.org  Mon Oct 22 07:50:19 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 22 Oct 2018 07:50:19 +0000 (UTC)
Subject: [master] 4b480b48d Refactor vut_error() into VUT_Error()
Message-ID: <20181022075019.4521093AAE@lists.varnish-cache.org>


commit 4b480b48d98e76d03ab5b50448467b8928ccfff0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 22 06:25:28 2018 +0000

    Refactor vut_error() into VUT_Error()

diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c
index b02644a81..8b3f6ed0e 100644
--- a/lib/libvarnishapi/vut.c
+++ b/lib/libvarnishapi/vut.c
@@ -110,32 +110,23 @@ vut_dispatch(struct VSL_data *vsl, struct VSL_transaction * const trans[],
 	return (i);
 }
 
-//lint -sem(vut_error, r_no)
-static void v_noreturn_ v_matchproto_(VUT_error_f)
-vut_error(struct VUT *vut, int status, const char *fmt, va_list ap)
-{
-
-	CHECK_OBJ_NOTNULL(vut, VUT_MAGIC);
-	AN(fmt);
-	vfprintf(stderr, fmt, ap);
-	fprintf(stderr, "\n");
-
-	exit(status);
-}
-
 void
 VUT_Error(struct VUT *vut, int status, const char *fmt, ...)
 {
 	va_list ap;
 
 	CHECK_OBJ_NOTNULL(vut, VUT_MAGIC);
-	AN(vut->error_f);
 	AN(status);
 
 	va_start(ap, fmt);
-	vut->error_f(vut, status, fmt, ap);
+	if (vut->error_f != NULL) {
+		vut->error_f(vut, status, fmt, ap);
+	} else {
+		vfprintf(stderr, fmt, ap);
+		fprintf(stderr, "\n");
+	}
 	va_end(ap);
-	exit(2);
+	exit(status);
 }
 
 int
@@ -231,7 +222,6 @@ VUT_Init(const char *progname, int argc, char * const *argv,
 	vut->progname = progname;
 	vut->g_arg = VSL_g_vxid;
 	vut->k_arg = -1;
-	vut->error_f = vut_error;
 	AZ(vut->vsl);
 	vut->vsl = VSL_New();
 	AN(vut->vsl);
@@ -306,7 +296,8 @@ VUT_Setup(struct VUT *vut)
 			VUT_Error(vut, 1, "PID file already created");
 		pfh = VPF_Open(vut->P_arg, 0644, NULL);
 		if (pfh == NULL)
-			VUT_Error(vut, 1, "%s: %s", vut->P_arg, strerror(errno));
+			VUT_Error(vut, 1,
+			    "%s: %s", vut->P_arg, strerror(errno));
 	}
 
 	/* Daemon mode */

From phk at FreeBSD.org  Mon Oct 22 07:50:19 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 22 Oct 2018 07:50:19 +0000 (UTC)
Subject: [master] 820f0bb2a Whitespace OCD
Message-ID: <20181022075019.2F13093AAB@lists.varnish-cache.org>


commit 820f0bb2ac17db2a592b1108c80f81fc949742cb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 22 06:06:57 2018 +0000

    Whitespace OCD

diff --git a/bin/varnishtest/tests/m00048.vtc b/bin/varnishtest/tests/m00048.vtc
index 8f5518d93..bb78fdfae 100644
--- a/bin/varnishtest/tests/m00048.vtc
+++ b/bin/varnishtest/tests/m00048.vtc
@@ -32,7 +32,7 @@ varnish v1 -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.filters = "rot13 rot13a";
 	}
-} 
+}
 
 client c1 {
 	txreq -hdr "Cookie: a"
@@ -47,7 +47,7 @@ varnish v1 -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.filters = "rot13 rot14";
 	}
-} 
+}
 
 client c1 {
 	txreq -hdr "Cookie: a"

From phk at FreeBSD.org  Mon Oct 22 07:50:19 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 22 Oct 2018 07:50:19 +0000 (UTC)
Subject: [master] 1e2a2c7f0 Introduce a VUT_Usage() function for consistency
Message-ID: <20181022075019.6BA9193AB0@lists.varnish-cache.org>


commit 1e2a2c7f08610d624bb92710d63043c691542f75
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 22 07:06:56 2018 +0000

    Introduce a VUT_Usage() function for consistency

diff --git a/include/vut.h b/include/vut.h
index 2528de39b..57eb7c02a 100644
--- a/include/vut.h
+++ b/include/vut.h
@@ -73,6 +73,10 @@ void VUT_Error(struct VUT *, int status, const char *fmt, ...)
 
 int VUT_Arg(struct VUT *, int opt, const char *arg);
 
+//lint -sem(VUT_Usage, r_no)
+void VUT_Usage(const struct VUT *, const struct vopt_spec *,
+    int status) v_noreturn_;
+
 #define VUT_InitProg(argc, argv, spec) VUT_Init(argv[0], argc, argv, spec)
 
 struct VUT * VUT_Init(const char *progname, int argc, char * const *argv,
diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map
index 4cc3b5050..1c8a6ff2b 100644
--- a/lib/libvarnishapi/libvarnishapi.map
+++ b/lib/libvarnishapi/libvarnishapi.map
@@ -140,3 +140,11 @@ LIBVARNISHAPI_2.0 {
     local:
 	*;
 };
+
+LIBVARNISHAPI_2.1 {
+    global:
+	# vut.c
+	VUT_Usage;
+    local:
+	*;
+};
diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c
index 8b3f6ed0e..9c02bf407 100644
--- a/lib/libvarnishapi/vut.c
+++ b/lib/libvarnishapi/vut.c
@@ -452,6 +452,20 @@ VUT_Main(struct VUT *vut)
 
 /**********************************************************************/
 
+void v_noreturn_
+VUT_Usage(const struct VUT *vut, const struct vopt_spec *voc, int status)
+{
+	const char **opt;
+
+	fprintf(stderr, "Usage: %s <options>\n\n", vut->progname);
+	fprintf(stderr, "Options:\n");
+	for (opt = voc->vopt_usage; *opt != NULL; opt += 2)
+		fprintf(stderr, " %-25s %s\n", *opt, *(opt + 1));
+	exit(status);
+}
+
+/**********************************************************************/
+
 
 static void
 print_nobrackets(const char *s)

From phk at FreeBSD.org  Mon Oct 22 07:50:19 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 22 Oct 2018 07:50:19 +0000 (UTC)
Subject: [master] 67f28d0e7 Use VUT_Usage() and (void) return of VUT_Main()
Message-ID: <20181022075019.AEB0393AB4@lists.varnish-cache.org>


commit 67f28d0e74f4d172be5cfa33ca70f26010623446
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 22 07:11:52 2018 +0000

    Use VUT_Usage() and (void) return of VUT_Main()

diff --git a/bin/varnishhist/flint.lnt b/bin/varnishhist/flint.lnt
index 1342b8e4e..6399b0249 100644
--- a/bin/varnishhist/flint.lnt
+++ b/bin/varnishhist/flint.lnt
@@ -1,4 +1,3 @@
 -efile(451, "varnishhist_profiles.h")
 -efile(451, "varnishhist_options.h")
--sem(usage, r_no)
 -sem(profile_error, r_no)
diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index c4f99a04a..21ca64666 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -473,18 +473,6 @@ do_curses(void *arg)
 
 /*--------------------------------------------------------------------*/
 
-static void v_noreturn_
-usage(int status)
-{
-	const char **opt;
-
-	fprintf(stderr, "Usage: %s <options>\n\n", vut->progname);
-	fprintf(stderr, "Options:\n");
-	for (opt = vopt_spec.vopt_usage; *opt != NULL; opt += 2)
-		fprintf(stderr, " %-25s %s\n", *opt, *(opt + 1));
-	exit(status);
-}
-
 static void v_noreturn_
 profile_error(const char *s)
 {
@@ -519,7 +507,7 @@ main(int argc, char **argv)
 		switch (i) {
 		case 'h':
 			/* Usage help */
-			usage(0);
+			VUT_Usage(vut, &vopt_spec, 0);
 		case 'p':
 			ms_delay = lround(1e3 * strtod(optarg, NULL));
 			if (ms_delay <= 0)
@@ -602,12 +590,12 @@ main(int argc, char **argv)
 			break;
 		default:
 			if (!VUT_Arg(vut, i, optarg))
-				usage(1);
+				VUT_Usage(vut, &vopt_spec, 1);
 		}
 	}
 
 	if (optind != argc)
-		usage(1);
+		VUT_Usage(vut, &vopt_spec, 1);
 
 	/* Check for valid grouping mode */
 	assert(vut->g_arg < VSL_g__MAX);
@@ -661,7 +649,7 @@ main(int argc, char **argv)
 	vut->dispatch_f = accumulate;
 	vut->dispatch_priv = NULL;
 	vut->sighup_f = sighup;
-	VUT_Main(vut);
+	(void)VUT_Main(vut);
 	end_of_file = 1;
 	AZ(pthread_join(thr, NULL));
 	VUT_Fini(&vut);
diff --git a/bin/varnishlog/flint.lnt b/bin/varnishlog/flint.lnt
index 204e85028..509f2fe55 100644
--- a/bin/varnishlog/flint.lnt
+++ b/bin/varnishlog/flint.lnt
@@ -10,4 +10,3 @@
 
 -e788	// enum constant '___' not used within defaulted switch
 -e641	// Converting enum '___' to '___'
--sem(usage,r_no)
diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c
index d80986b78..e3e2e4576 100644
--- a/bin/varnishlog/varnishlog.c
+++ b/bin/varnishlog/varnishlog.c
@@ -64,17 +64,6 @@ static struct log {
 	FILE		*fo;
 } LOG;
 
-static void v_noreturn_
-usage(int status)
-{
-	const char **opt;
-	fprintf(stderr, "Usage: %s <options>\n\n", vut->progname);
-	fprintf(stderr, "Options:\n");
-	for (opt = vopt_spec.vopt_usage; *opt != NULL; opt += 2)
-		fprintf(stderr, " %-25s %s\n", *opt, *(opt + 1));
-	exit(status);
-}
-
 static void
 openout(int append)
 {
@@ -97,7 +86,7 @@ rotateout(struct VUT *v)
 	assert(v == vut);
 	AN(LOG.w_arg);
 	AN(LOG.fo);
-	fclose(LOG.fo);
+	(void)fclose(LOG.fo);
 	openout(1);
 	AN(LOG.fo);
 	return (0);
@@ -150,19 +139,19 @@ main(int argc, char * const *argv)
 			break;
 		case 'h':
 			/* Usage help */
-			usage(0);
+			VUT_Usage(vut, &vopt_spec, 0);
 		case 'w':
 			/* Write to file */
 			REPLACE(LOG.w_arg, optarg);
 			break;
 		default:
 			if (!VUT_Arg(vut, opt, optarg))
-				usage(1);
+				VUT_Usage(vut, &vopt_spec, 1);
 		}
 	}
 
 	if (optind != argc)
-		usage(1);
+		VUT_Usage(vut, &vopt_spec, 1);
 
 	if (vut->D_opt && !LOG.w_arg)
 		VUT_Error(vut, 1, "Missing -w option");
@@ -184,7 +173,7 @@ main(int argc, char * const *argv)
 
 	VUT_Signal(vut_sighandler);
 	VUT_Setup(vut);
-	VUT_Main(vut);
+	(void)VUT_Main(vut);
 	VUT_Fini(&vut);
 
 	(void)flushout(NULL);
diff --git a/bin/varnishstat/flint.lnt b/bin/varnishstat/flint.lnt
index 0dbb9d82c..b3160aab6 100644
--- a/bin/varnishstat/flint.lnt
+++ b/bin/varnishstat/flint.lnt
@@ -2,5 +2,3 @@
 +libh mgt_event.h
 
 -efile(451, varnishstat_options.h)
-
--sem(usage, r_no)
diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index a4a77e7c9..afe0dad92 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -324,19 +324,6 @@ dump(void)
 	}
 }
 
-//lint -sem(usage, r_no)
-static void v_noreturn_
-usage(int status)
-{
-	const char **opt;
-
-	fprintf(stderr, "Usage: %s <options>\n\n", vut->progname);
-	fprintf(stderr, "Options:\n");
-	for (opt = vopt_spec.vopt_usage; *opt != NULL; opt +=2)
-		fprintf(stderr, " %-25s %s\n", *opt, *(opt + 1));
-	exit(status);
-}
-
 int
 main(int argc, char **argv)
 {
@@ -358,7 +345,7 @@ main(int argc, char **argv)
 			break;
 		case 'h':
 			/* Usage help */
-			usage(0);
+			VUT_Usage(vut, &vopt_spec, 0);
 		case 'p':
 			errno = 0;
 			e = NULL;
@@ -371,12 +358,12 @@ main(int argc, char **argv)
 			break;
 		default:
 			if (!VUT_Arg(vut, o, optarg))
-				usage(1);
+				VUT_Usage(vut, &vopt_spec, 1);
 		}
 	}
 
 	if (optind != argc)
-		usage(1);
+		VUT_Usage(vut, &vopt_spec, 1);
 
 	VUT_Signal(vut_sighandler);
 	VUT_Setup(vut);

From phk at FreeBSD.org  Mon Oct 22 07:50:20 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 22 Oct 2018 07:50:20 +0000 (UTC)
Subject: [master] 487028f5c Use VUT_Usage(), (void)VUT_Main() and other polish.
Message-ID: <20181022075020.28B0893ABE@lists.varnish-cache.org>


commit 487028f5cda53e8c6a486bc102866427038fef9d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 22 07:26:55 2018 +0000

    Use VUT_Usage(), (void)VUT_Main() and other polish.

diff --git a/bin/varnishncsa/flint.lnt b/bin/varnishncsa/flint.lnt
index f7767d1c4..5270ff22b 100644
--- a/bin/varnishncsa/flint.lnt
+++ b/bin/varnishncsa/flint.lnt
@@ -1,3 +1,2 @@
 
 -efile(451, "varnishncsa_options.h")
--sem(usage, r_no)
diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 50631446f..759519eb5 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -162,18 +162,6 @@ static struct ctx {
 	int32_t			vxid;
 } CTX;
 
-static void v_noreturn_
-usage(int status)
-{
-	const char **opt;
-
-	fprintf(stderr, "Usage: %s <options>\n\n", vut->progname);
-	fprintf(stderr, "Options:\n");
-	for (opt = vopt_spec.vopt_usage; *opt != NULL; opt += 2)
-		fprintf(stderr, " %-25s %s\n", *opt, *(opt + 1));
-	exit(status);
-}
-
 static void
 openout(int append)
 {
@@ -192,7 +180,7 @@ rotateout(struct VUT *v)
 	assert(v == vut);
 	AN(CTX.w_arg);
 	AN(CTX.fo);
-	fclose(CTX.fo);
+	(void)fclose(CTX.fo);
 	openout(1);
 	AN(CTX.fo);
 	return (0);
@@ -354,9 +342,9 @@ format_time(const struct format *format)
 		break;
 	case 't':
 		AN(format->time_fmt);
-		t = t_start;
-		localtime_r(&t, &tm);
-		strftime(buf, sizeof buf, format->time_fmt, &tm);
+		t = (long)floor(t_start);
+		(void)localtime_r(&t, &tm);
+		AN(strftime(buf, sizeof buf, format->time_fmt, &tm));
 		AZ(VSB_cat(CTX.vsb, buf));
 		break;
 	case 'T':
@@ -1149,7 +1137,7 @@ read_format(const char *formatfile)
 		VUT_Error(vut, 1, "Can't read format from file (%s)",
 		    strerror(errno));
 	}
-	fclose(fmtfile);
+	AZ(fclose(fmtfile));
 	if (fmt[fmtlen - 1] == '\n')
 		fmt[fmtlen - 1] = '\0';
 	return (fmt);
@@ -1190,22 +1178,21 @@ main(int argc, char * const *argv)
 			CTX.c_opt = 1;
 			break;
 		case 'F':
-			/* Format string */
 			if (format != NULL)
-				free(format);
-			format = strdup(optarg);
+				VUT_Error(vut, 1, "Format already set");
+			REPLACE(format, optarg);
 			AN(format);
 			break;
 		case 'f':
-			/* Format string from file */
 			if (format != NULL)
-				free(format);
+				VUT_Error(vut, 1, "Format already set");
+			/* Format string from file */
 			format = read_format(optarg);
 			AN(format);
 			break;
 		case 'h':
 			/* Usage help */
-			usage(0);
+			VUT_Usage(vut, &vopt_spec, 0);
 			break;
 		case 'w':
 			/* Write to file */
@@ -1213,7 +1200,7 @@ main(int argc, char * const *argv)
 			break;
 		default:
 			if (!VUT_Arg(vut, opt, optarg))
-				usage(1);
+				VUT_Usage(vut, &vopt_spec, 1);
 		}
 	}
 	/* default is client mode: */
@@ -1221,7 +1208,7 @@ main(int argc, char * const *argv)
 		CTX.c_opt = 1;
 
 	if (optind != argc)
-		usage(1);
+		VUT_Usage(vut, &vopt_spec, 1);
 
 	if (vut->D_opt && !CTX.w_arg)
 		VUT_Error(vut, 1, "Missing -w option");
@@ -1234,8 +1221,7 @@ main(int argc, char * const *argv)
 
 	/* Prepare output format */
 	parse_format(format);
-	free(format);
-	format = NULL;
+	REPLACE(format, NULL);
 
 	/* Setup output */
 	vut->dispatch_f = dispatch_f;
@@ -1252,7 +1238,7 @@ main(int argc, char * const *argv)
 
 	VUT_Signal(vut_sighandler);
 	VUT_Setup(vut);
-	VUT_Main(vut);
+	(void)VUT_Main(vut);
 	VUT_Fini(&vut);
 
 	exit(0);

From phk at FreeBSD.org  Mon Oct 22 07:50:19 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 22 Oct 2018 07:50:19 +0000 (UTC)
Subject: [master] 5a4036423 Explicitly call tzset(3) because localtime_r(3)
 was documented by morons.
Message-ID: <20181022075019.ED79393AB9@lists.varnish-cache.org>


commit 5a4036423d90e140d86aaede412012a0e82963df
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 22 07:15:36 2018 +0000

    Explicitly call tzset(3) because localtime_r(3) was documented by morons.

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index c92b0a317..50631446f 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -1173,6 +1173,8 @@ main(int argc, char * const *argv)
 	AN(CTX.vsb);
 	VB64_init();
 
+	tzset();		// We use localtime_r(3)
+
 	while ((opt = getopt(argc, argv, vopt_spec.vopt_optstring)) != -1) {
 		switch (opt) {
 		case 'a':

From phk at FreeBSD.org  Mon Oct 22 07:50:20 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 22 Oct 2018 07:50:20 +0000 (UTC)
Subject: [master] 977d2705a Add VSB_QUOTE_ESCHEX to prefer \xXX to \oOOO
 quoting
Message-ID: <20181022075020.707BB93AC5@lists.varnish-cache.org>


commit 977d2705a28d38ef3bd4a28f5cd8b4184704ae3a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 22 07:48:32 2018 +0000

    Add VSB_QUOTE_ESCHEX to prefer \xXX to \oOOO quoting
    
    Also quote \v as \v

diff --git a/include/vsb.h b/include/vsb.h
index d51088499..ea1ab4de8 100644
--- a/include/vsb.h
+++ b/include/vsb.h
@@ -81,6 +81,7 @@ void		 VSB_destroy(struct vsb **);
 #define VSB_QUOTE_HEX		4
 #define VSB_QUOTE_CSTR		8
 #define VSB_QUOTE_UNSAFE	16
+#define VSB_QUOTE_ESCHEX	32
 void		 VSB_quote_pfx(struct vsb *, const char*, const void *,
 		     int len, int how);
 void		 VSB_quote(struct vsb *, const void *, int len, int how);
diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c
index b44323854..e1e6d1445 100644
--- a/lib/libvarnish/vsb.c
+++ b/lib/libvarnish/vsb.c
@@ -588,12 +588,17 @@ VSB_quote_pfx(struct vsb *s, const char *pfx, const void *v, int len, int how)
 		case '\t':
 			(void)VSB_cat(s, "\\t");
 			break;
+		case '\v':
+			(void)VSB_cat(s, "\\v");
+			break;
 		default:
 			/* XXX: Implement VSB_QUOTE_JSON */
 			if (isgraph(*q))
 				(void)VSB_putc(s, *q);
+			else if (how & VSB_QUOTE_ESCHEX)
+				(void)VSB_printf(s, "\\x%02x", *q & 0xff);
 			else
-				(void)VSB_printf(s, "\\%o", *q & 0xff);
+				(void)VSB_printf(s, "\\%03o", *q & 0xff);
 			break;
 		}
 	}

From phk at FreeBSD.org  Mon Oct 22 07:50:20 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 22 Oct 2018 07:50:20 +0000 (UTC)
Subject: [master] ba43ad121 Drop private string escape function for VSB_quote()
Message-ID: <20181022075020.88B4C93AC9@lists.varnish-cache.org>


commit ba43ad1217623a31758dd5edb61ac941e8a38f54
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 22 07:49:09 2018 +0000

    Drop private string escape function for VSB_quote()

diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c
index 759519eb5..bcf996f94 100644
--- a/bin/varnishncsa/varnishncsa.c
+++ b/bin/varnishncsa/varnishncsa.c
@@ -197,62 +197,17 @@ flushout(struct VUT *v)
 	return (0);
 }
 
-static int
-vsb_esc_cat(struct vsb *sb, const char *b, const char *e)
-{
-	AN(b);
-
-	for (; b < e; b++) {
-		if (isspace(*b)) {
-			switch (*b) {
-			case '\n':
-				VSB_cat(sb, "\\n");
-				break;
-			case '\t':
-				VSB_cat(sb, "\\t");
-				break;
-			case '\f':
-				VSB_cat(sb, "\\f");
-				break;
-			case '\r':
-				VSB_cat(sb, "\\r");
-				break;
-			case '\v':
-				VSB_cat(sb, "\\v");
-				break;
-			default:
-				VSB_putc(sb, *b);
-				break;
-			}
-		} else if (isprint(*b)) {
-			switch (*b) {
-			case '"':
-				VSB_cat(sb, "\\\"");
-				break;
-			case '\\':
-				VSB_cat(sb, "\\\\");
-				break;
-			default:
-				VSB_putc(sb, *b);
-				break;
-			}
-		} else
-			VSB_printf(sb, "\\x%02hhx", *b);
-	}
-
-	return (VSB_error(sb));
-}
-
 static inline int
 vsb_fcat(struct vsb *vsb, const struct fragment *f, const char *dflt)
 {
 	if (f->gen == CTX.gen) {
 		assert(f->b <= f->e);
-		return (vsb_esc_cat(vsb, f->b, f->e));
-	}
-	if (dflt)
-		return (vsb_esc_cat(vsb, dflt, dflt + strlen(dflt)));
-	return (-1);
+		VSB_quote(vsb, f->b, f->e - f->b, VSB_QUOTE_ESCHEX);
+	} else if (dflt)
+		VSB_quote(vsb, dflt, -1, VSB_QUOTE_ESCHEX);
+	else
+		return (-1);
+	return (0);
 }
 
 static int v_matchproto_(format_f)
@@ -295,8 +250,7 @@ format_fragment(const struct format *format)
 	if (format->frag->gen != CTX.gen) {
 		if (format->string == NULL)
 			return (-1);
-		AZ(vsb_esc_cat(CTX.vsb, format->string,
-		    format->string + strlen(format->string)));
+		VSB_quote(CTX.vsb, format->string, -1, VSB_QUOTE_ESCHEX);
 		return (0);
 	}
 	AZ(vsb_fcat(CTX.vsb, format->frag, NULL));
@@ -388,14 +342,13 @@ format_auth(const struct format *format)
 	    CTX.frag[F_auth].e)) {
 		if (format->string == NULL)
 			return (-1);
-		AZ(vsb_esc_cat(CTX.vsb, format->string,
-		    format->string + strlen(format->string)));
+		VSB_quote(CTX.vsb, format->string, -1, VSB_QUOTE_ESCHEX);
 		return (0);
 	}
 	q = strchr(buf, ':');
 	if (q != NULL)
 		*q = '\0';
-	AZ(vsb_esc_cat(CTX.vsb, buf, buf + strlen(buf)));
+	VSB_quote(CTX.vsb, buf, -1, VSB_QUOTE_ESCHEX);
 	return (1);
 }
 
@@ -409,6 +362,7 @@ print(void)
 	VTAILQ_FOREACH(f, &CTX.format, list) {
 		CHECK_OBJ_NOTNULL(f, FORMAT_MAGIC);
 		i = (f->func)(f);
+		AZ(VSB_error(CTX.vsb));
 		if (r > i)
 			r = i;
 	}

From hermunn at varnish-software.com  Wed Oct 24 09:29:13 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:13 +0000 (UTC)
Subject: [6.1] 9a00f2f10 Fix trouble with 2GB+ VSB's on systems where int is
 32 bit but size_t is 64 bit.
Message-ID: <20181024092913.E21B5AFB5F@lists.varnish-cache.org>


commit 9a00f2f1082d1004f84fdd79e5d79954a22ef535
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 06:38:58 2018 +0000

    Fix trouble with 2GB+ VSB's on systems where int is 32 bit but
    size_t is 64 bit.

diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c
index 0bc0925ab..b44323854 100644
--- a/lib/libvarnish/vsb.c
+++ b/lib/libvarnish/vsb.c
@@ -51,8 +51,8 @@ __FBSDID("$FreeBSD: head/sys/kern/subr_vsb.c 222004 2011-05-17 06:36:32Z phk $")
  */
 #define	VSB_ISDYNAMIC(s)	((s)->s_flags & VSB_DYNAMIC)
 #define	VSB_ISDYNSTRUCT(s)	((s)->s_flags & VSB_DYNSTRUCT)
-#define	VSB_HASROOM(s)		((s)->s_len < (s)->s_size - 1)
-#define	VSB_FREESPACE(s)	((s)->s_size - ((s)->s_len + 1))
+#define	VSB_HASROOM(s)		((s)->s_len < (s)->s_size - 1L)
+#define	VSB_FREESPACE(s)	((s)->s_size - ((s)->s_len + 1L))
 #define	VSB_CANEXTEND(s)	((s)->s_flags & VSB_AUTOEXTEND)
 
 /*
@@ -114,10 +114,10 @@ CTASSERT(powerof2(VSB_MAXEXTENDSIZE));
 CTASSERT(powerof2(VSB_MAXEXTENDINCR));
 #endif
 
-static int
-VSB_extendsize(int size)
+static ssize_t
+VSB_extendsize(ssize_t size)
 {
-	int newsize;
+	ssize_t newsize;
 
 	if (size < (int)VSB_MAXEXTENDSIZE) {
 		newsize = VSB_MINEXTENDSIZE;
@@ -133,11 +133,11 @@ VSB_extendsize(int size)
 /*
  * Extend an vsb.
  */
-static int
-VSB_extend(struct vsb *s, int addlen)
+static ssize_t
+VSB_extend(struct vsb *s, ssize_t addlen)
 {
 	char *newbuf;
-	int newsize;
+	ssize_t newsize;
 
 	if (!VSB_CANEXTEND(s))
 		return (-1);

From hermunn at varnish-software.com  Wed Oct 24 09:29:13 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:13 +0000 (UTC)
Subject: [6.1] 96121b696 I börked varnishhist by calling the lround function the wrong place.
Message-ID: <20181024092913.CD492AFB5D@lists.varnish-cache.org>


commit 96121b696aeef59064cb1899011ceb786cafdb12
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 11:23:07 2018 +0000

    I b?rked varnishhist by calling the lround function the wrong place.
    
    Fixes: #2780

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 48bfce265..c4f99a04a 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -299,7 +299,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 			continue;
 
 		/* select bucket */
-		i = HIST_RES * lround(log(value) / log_ten);
+		i = lround(HIST_RES * log(value) / log_ten);
 		if (i < hist_low * HIST_RES)
 			i = hist_low * HIST_RES;
 		if (i >= hist_high * HIST_RES)

From hermunn at varnish-software.com  Wed Oct 24 09:29:14 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:14 +0000 (UTC)
Subject: [6.1] 0dbcb74a2 Eliminated varnishd from these tests
Message-ID: <20181024092914.1B573AFB62@lists.varnish-cache.org>


commit 0dbcb74a2df98d87d17cbd726740c235a16db692
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 06:47:36 2018 +0000

    Eliminated varnishd from these tests

diff --git a/bin/varnishtest/tests/a00015.vtc b/bin/varnishtest/tests/a00015.vtc
index 843637f0f..f363d66d9 100644
--- a/bin/varnishtest/tests/a00015.vtc
+++ b/bin/varnishtest/tests/a00015.vtc
@@ -12,9 +12,7 @@ server s1 {
 	txresp -hdr "Content-Type: text/plain" -body response
 } -start
 
-varnish v1 -vcl+backend {} -start -cliok "param.set debug +syncvsl"
-
-client c1 {
+client c1 -connect ${s1_sock} {
 	txreq -req POST -hdr "Content-Type: text/plain" -body request
 
 	# First, HTTP checks
diff --git a/bin/varnishtest/tests/a00018.vtc b/bin/varnishtest/tests/a00018.vtc
index 036aa4596..5500d7c6e 100644
--- a/bin/varnishtest/tests/a00018.vtc
+++ b/bin/varnishtest/tests/a00018.vtc
@@ -8,28 +8,9 @@ server s1 {
 	txresp -hdr "Bar: ${bar}"
 } -start
 
-varnish v1 -vcl {
-	backend default {
-		.host = "${s1_addr}";
-		.port = "${s1_port}";
-	}
-
-	sub vcl_deliver {
-		if (resp.http.Bar == "${bar}") {
-			set resp.http.Baz = "${baz}";
-		}
-	}
-} -start
-
-client c1 {
+client c1 -connect ${s1_sock} {
 	txreq -hdr "Foo: ${foo}"
 	rxresp
 	expect resp.status == 200
 	expect resp.http.Bar == "${bar}"
-	expect resp.http.Baz == "${baz}"
 } -run
-
-shell {
-	touch ${tmpdir}/tst
-	rm ${tmpdir}/tst
-}

From hermunn at varnish-software.com  Wed Oct 24 09:29:14 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:14 +0000 (UTC)
Subject: [6.1] 913a92db4 Mark our extension to ZLib and #ifdef the code in
 varnishtest which reports stuff from it for debugging.
Message-ID: <20181024092914.4CE49AFB68@lists.varnish-cache.org>


commit 913a92db43cf8be2dd6c2fb54a07cb1b5f1bc4d6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 06:48:49 2018 +0000

    Mark our extension to ZLib and #ifdef the code in varnishtest which
    reports stuff from it for debugging.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index d89a91a5a..952563b9f 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -771,6 +771,7 @@ cmd_http_gunzip(CMD_ARGS)
 	vtc_log(hp->vl, 3, "new bodylen %u", hp->bodyl);
 	vtc_dump(hp->vl, 4, "body", hp->body, hp->bodyl);
 	bprintf(hp->bodylen, "%u", hp->bodyl);
+#ifdef VGZ_EXTENSIONS
 	vtc_log(hp->vl, 4, "startbit = %ju %ju/%ju",
 	    (uintmax_t)vz.start_bit,
 	    (uintmax_t)vz.start_bit >> 3, (uintmax_t)vz.start_bit & 7);
@@ -780,6 +781,7 @@ cmd_http_gunzip(CMD_ARGS)
 	vtc_log(hp->vl, 4, "stopbit = %ju %ju/%ju",
 	    (uintmax_t)vz.stop_bit,
 	    (uintmax_t)vz.stop_bit >> 3, (uintmax_t)vz.stop_bit & 7);
+#endif
 	if (i != Z_STREAM_END)
 		vtc_log(hp->vl, hp->fatal,
 		    "Gunzip error = %d (%s) in:%jd out:%jd",
@@ -813,12 +815,13 @@ gzip_body(const struct http *hp, const char *txt, char **body, int *bodylen)
 	assert(Z_OK == deflateInit2(&vz,
 	    hp->gziplevel, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY));
 	assert(Z_STREAM_END == deflate(&vz, Z_FINISH));
+	*bodylen = vz.total_out;
+#ifdef VGZ_EXTENSIONS
 	i = vz.stop_bit & 7;
 	if (hp->gzipresidual >= 0 && hp->gzipresidual != i)
 		vtc_log(hp->vl, hp->fatal,
 		    "Wrong gzip residual got %d wanted %d",
 		    i, hp->gzipresidual);
-	*bodylen = vz.total_out;
 	vtc_log(hp->vl, 4, "startbit = %ju %ju/%ju",
 	    (uintmax_t)vz.start_bit,
 	    (uintmax_t)vz.start_bit >> 3, (uintmax_t)vz.start_bit & 7);
@@ -829,6 +832,7 @@ gzip_body(const struct http *hp, const char *txt, char **body, int *bodylen)
 	    (uintmax_t)vz.stop_bit,
 	    (uintmax_t)vz.stop_bit >> 3, (uintmax_t)vz.stop_bit & 7);
 	assert(Z_OK == deflateEnd(&vz));
+#endif
 }
 
 /**********************************************************************
diff --git a/lib/libvgz/vgz.h b/lib/libvgz/vgz.h
index 86831eeed..1bb81009d 100644
--- a/lib/libvgz/vgz.h
+++ b/lib/libvgz/vgz.h
@@ -104,6 +104,7 @@ typedef struct z_stream_s {
     uLong   adler;      /* Adler-32 or CRC-32 value of the uncompressed data */
     uLong   reserved;   /* reserved for future use */
 
+#define VGZ_EXTENSIONS 1
     uLong   start_bit;	/* Bit pos of first deflate block */
     uLong   stop_bit;	/* Bit pos after last deflate block */
     uLong   last_bit;	/* Bit pos of 'last' bit */

From hermunn at varnish-software.com  Wed Oct 24 09:29:14 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:14 +0000 (UTC)
Subject: [6.1] d6a2ecffd Get varnishd out of and varnishtest-tests into a*vtc
Message-ID: <20181024092914.AE064AFB6D@lists.varnish-cache.org>


commit d6a2ecffdcce99ee4692cfc8a8064dfd06be20e6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 07:09:30 2018 +0000

    Get varnishd out of and varnishtest-tests into a*vtc

diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc
index 9316a5cb8..22e9f341b 100644
--- a/bin/varnishtest/tests/a00000.vtc
+++ b/bin/varnishtest/tests/a00000.vtc
@@ -41,96 +41,130 @@ shell -exit 77 -expect {TEST _.vtc skipped} {
 	exec varnishtest -v _.vtc || true
 }
 
-process p1 "ps -lw | grep '[p][s]' ; tty ; echo @" -start
-process p1 -expect-text 0 0 {@} -screen_dump -wait
-
-process p2 "stty -a ; echo '*'" -start
-process p2 -expect-text 0 0 {*} -screen_dump -wait
-
-process p4 -hexdump {stty raw -echo; stty -a ; echo "*" ; cat} -start
-process p4 -expect-text 0 0 "*" -screen_dump
-
-process p4 -write "\x1b[H\x1b[2Jzzzzzzz"
-process p4 -write "\x0c1\x1b[79C2\x08>\x1b[25;1H3\x1b[25;80H"
-process p4 -write "\x1b[H\x1b[2J1\x1b[79C2\x08>\x1b[25;1H3\x1b[25;80H"
-process p4 -write "4\x08>\x1b[A\x1b[Cv\x1b[22A^\x1b[79D^\x1b[;2H<\n\n\n\n"
-process p4 -write "\n\n\n\n\n\n\n\n\x1b[B\x1b[11B\x08<\x1b[24;Hv\x1b[12;1H"
-process p4 -write "111111112222222333333\x0d\x0a111111112"
-process p4 -write "222222333333\x0d\x0a111111112222222333333 UTF8: "
-process p4 -writehex {c2 a2 20}
-process p4 -writehex {e2 82 ac 20}
-process p4 -writehex {f0 90 80 80 20}
-process p4 -writehex {f0 9f 90 b0 20}
-process p4 -writehex {f0 a0 80 80 20}
-process p4 -writehex {f0 b0 80 80 20}
-process p4 -write "\x1b[22;24;25;27;30;47;49;97;107m"
-process p4 -write "\x1b[22;24;25;27;30m"
-process p4 -write "\x1b[47;49;97;107m"
-process p4 -write "\x0d\x0a111111112222222333333\x0d\x0a\x1b[12"
-process p4 -write ";12H\x1b[K\x1b[13;12H\x1b[0K\x1b[14;12H\x1b[1K\x1b"
-process p4 -write "[15;12H\x1b[2K\x1b[3;1Hline3 <\x0d\x0a"
-
-process p4 -need-bytes 310 -expect-text 3 1 "line3 <"
-process p4 -expect-cursor 4 1
-process p4 -expect-cursor 4 0
-process p4 -expect-cursor 0 1
-process p4 -screen-dump
+# Simple process tests
+
+process p1 "cat" -start
+process p1 -writeln "foo"
+process p1 -expect-text 2 1 foo
+process p1 -stop
+process p1 -wait
+shell "grep -q foo ${p1_out}"
+shell "test -f ${p1_err} -a ! -s ${p1_err}"
+
+process p2 -log "cat" -start
+process p2 -writeln "bar"
+process p2 -expect-text 2 1 bar
+process p2 -write "\x04"
+process p2 -wait
+shell "grep -q bar ${p2_out}"
+shell "test -f ${p2_err} -a ! -s ${p2_err}"
+
+process p3 -dump "cat" -start
+process p3 -writeln "baz"
+process p3 -expect-text 2 1 baz
+process p3 -kill KILL
+process p3 -wait
+shell "grep -q baz ${p3_out}"
+shell "test -f ${p3_err} -a ! -s ${p3_err}"
+
+process p4 -hexdump "cat" -start
+process p4 -writeln "b\001z"
+process p4 -expect-text 2 1 "b"
+process p4 -kill TERM
+process p4 -wait -screen_dump
+
+# Curses process tests
+
+process p5 "ps -lw | grep '[p][s]' ; tty ; echo @" -start
+process p5 -expect-text 0 0 {@} -screen_dump -wait
+
+process p6 "stty -a ; echo '*'" -start
+process p6 -expect-text 0 0 {*} -screen_dump -wait
+
+process p7 -hexdump {stty raw -echo; stty -a ; echo "*" ; cat} -start
+process p7 -expect-text 0 0 "*" -screen_dump
+
+process p7 -write "\x1b[H\x1b[2Jzzzzzzz"
+process p7 -write "\x0c1\x1b[79C2\x08>\x1b[25;1H3\x1b[25;80H"
+process p7 -write "\x1b[H\x1b[2J1\x1b[79C2\x08>\x1b[25;1H3\x1b[25;80H"
+process p7 -write "4\x08>\x1b[A\x1b[Cv\x1b[22A^\x1b[79D^\x1b[;2H<\n\n\n\n"
+process p7 -write "\n\n\n\n\n\n\n\n\x1b[B\x1b[11B\x08<\x1b[24;Hv\x1b[12;1H"
+process p7 -write "111111112222222333333\x0d\x0a111111112"
+process p7 -write "222222333333\x0d\x0a111111112222222333333 UTF8: "
+process p7 -writehex {c2 a2 20}
+process p7 -writehex {e2 82 ac 20}
+process p7 -writehex {f0 90 80 80 20}
+process p7 -writehex {f0 9f 90 b0 20}
+process p7 -writehex {f0 a0 80 80 20}
+process p7 -writehex {f0 b0 80 80 20}
+process p7 -write "\x1b[22;24;25;27;30;47;49;97;107m"
+process p7 -write "\x1b[22;24;25;27;30m"
+process p7 -write "\x1b[47;49;97;107m"
+process p7 -write "\x0d\x0a111111112222222333333\x0d\x0a\x1b[12"
+process p7 -write ";12H\x1b[K\x1b[13;12H\x1b[0K\x1b[14;12H\x1b[1K\x1b"
+process p7 -write "[15;12H\x1b[2K\x1b[3;1Hline3 <\x0d\x0a"
+
+process p7 -need-bytes 310 -expect-text 3 1 "line3 <"
+process p7 -expect-cursor 4 1
+process p7 -expect-cursor 4 0
+process p7 -expect-cursor 0 1
+process p7 -screen-dump
 
 # Also exercise CONS25 mode
-process p4 -write "\x1b[=1T"
-process p4 -write "\x1b[=2T"
-process p4 -write "\x1b[8z"
-process p4 -write "\x1b[0x"
-process p4 -write "\x1b[=1A"
-process p4 -write "\x1b[=1;2B"
-process p4 -write "\x1b[=1;2;3C"
-process p4 -write "\x1b[=1;2;3;4C"
-process p4 -write "\x1b[=1F"
-process p4 -write "\x1b[=1G"
-process p4 -write "\x1b[=1S"
-process p4 -writehex {0c 08 40 0d 0a 08}
-
-process p4 -expect-text 1 1 "@"
-process p4 -expect-cursor 1 80
-process p4 -writehex "0c 41 0e 42 0f"
-process p4 -expect-text 1 1 "A"
-process p4 -expect-text 0 0 "B"
-process p4 -write "\x1b[=0T"
-
-process p4 -writehex "0c 0a 0d 43 0a 08 08 0e 44 0f"
-
-process p4 -expect-text 3 1 "C"
-process p4 -expect-text 4 1 "D"
-process p4 -write "\x1b[2T"
-process p4 -expect-text 5 1 "C"
-process p4 -expect-text 6 1 "D"
-process p4 -write "\x1b[3S"
-process p4 -expect-text 3 1 "D"
-
-process p4 -write "\x1b[4;200H%"
-process p4 -expect-text 4 80 "%"
-
-process p4 -write "\x1b[7;7H\x09X\x09Y\x09Z\x1b[2ZW\x1b[2Ew\x1b[F*"
-
-process p4 -expect-text 7 17 "W"
-process p4 -expect-text 9 1 "w"
-process p4 -expect-text 8 1 "*"
-
-process p4 -write "\x1b[10;4HABCDEFGHIJKLMN\x1b(A#$%\x1b)A"
-process p4 -write "\x1b[8G\x1b[2X>"
-process p4 -expect-text 10 8 ">"
-process p4 -screen-dump
+process p7 -write "\x1b[=1T"
+process p7 -write "\x1b[=2T"
+process p7 -write "\x1b[8z"
+process p7 -write "\x1b[0x"
+process p7 -write "\x1b[=1A"
+process p7 -write "\x1b[=1;2B"
+process p7 -write "\x1b[=1;2;3C"
+process p7 -write "\x1b[=1;2;3;4C"
+process p7 -write "\x1b[=1F"
+process p7 -write "\x1b[=1G"
+process p7 -write "\x1b[=1S"
+process p7 -writehex {0c 08 40 0d 0a 08}
+
+process p7 -expect-text 1 1 "@"
+process p7 -expect-cursor 1 80
+process p7 -writehex "0c 41 0e 42 0f"
+process p7 -expect-text 1 1 "A"
+process p7 -expect-text 0 0 "B"
+process p7 -write "\x1b[=0T"
+
+process p7 -writehex "0c 0a 0d 43 0a 08 08 0e 44 0f"
+
+process p7 -expect-text 3 1 "C"
+process p7 -expect-text 4 1 "D"
+process p7 -write "\x1b[2T"
+process p7 -expect-text 5 1 "C"
+process p7 -expect-text 6 1 "D"
+process p7 -write "\x1b[3S"
+process p7 -expect-text 3 1 "D"
+
+process p7 -write "\x1b[4;200H%"
+process p7 -expect-text 4 80 "%"
+
+process p7 -write "\x1b[7;7H\x09X\x09Y\x09Z\x1b[2ZW\x1b[2Ew\x1b[F*"
+
+process p7 -expect-text 7 17 "W"
+process p7 -expect-text 9 1 "w"
+process p7 -expect-text 8 1 "*"
+
+process p7 -write "\x1b[10;4HABCDEFGHIJKLMN\x1b(A#$%\x1b)A"
+process p7 -write "\x1b[8G\x1b[2X>"
+process p7 -expect-text 10 8 ">"
+process p7 -screen-dump
 
 # Test responses
-process p4 -write "\x1b[3;1HA\x1b[5n"
-process p4 -write "\x1b[4;1HB\x1b[6n"
-process p4 -write "\x1b[5;1HC\x1b[15n"
-process p4 -write "\x1b[6;1HD\x1b[25n"
-process p4 -write "\x1b[7;1HE\x1b[26n"
-process p4 -write "\x1b[8;1HF\x1b[?26n"
-process p4 -write "\x1b[9;1HG\x1bPfutfutfut\x01"
-process p4 -write "\x1b[10;1HH\x1b]futfutfut\x01"
-process p4 -write "\x1b[11;1HI\x1b[>c"
-process p4 -write "\x1b[24;1HW"
-process p4 -expect-text 24 1 "W"
-process p4 -screen-dump
+process p7 -write "\x1b[3;1HA\x1b[5n"
+process p7 -write "\x1b[4;1HB\x1b[6n"
+process p7 -write "\x1b[5;1HC\x1b[15n"
+process p7 -write "\x1b[6;1HD\x1b[25n"
+process p7 -write "\x1b[7;1HE\x1b[26n"
+process p7 -write "\x1b[8;1HF\x1b[?26n"
+process p7 -write "\x1b[9;1HG\x1bPfutfutfut\x01"
+process p7 -write "\x1b[10;1HH\x1b]futfutfut\x01"
+process p7 -write "\x1b[11;1HI\x1b[>c"
+process p7 -write "\x1b[24;1HW"
+process p7 -expect-text 24 1 "W"
+process p7 -screen-dump
diff --git a/bin/varnishtest/tests/a00009.vtc b/bin/varnishtest/tests/a00009.vtc
deleted file mode 100644
index 4b82d4c7d..000000000
--- a/bin/varnishtest/tests/a00009.vtc
+++ /dev/null
@@ -1,55 +0,0 @@
-varnishtest "Code coverage of mgt_main, (VCL compiler and RSTdump etc)"
-
-shell "varnishd -b 127.0.0.1:80 -C 2> ${tmpdir}/_.c"
-shell -err -expect {VCL version declaration missing} {
-	echo 'bad vcl' > ${tmpdir}/bad.vcl
-	varnishd -f ${tmpdir}/bad.vcl -n ${tmpdir}
-}
-
-shell -err -expect {-x must be the first argument} "varnishd -d -x foo "
-shell -err -expect {-V must be the first argument} "varnishd -d -V foo "
-
-shell -err -expect {Too many arguments for -x} "varnishd -x foo bar"
-shell -err -expect {Invalid -x argument} "varnishd -x foo "
-
-# This one is tricky, the getopt message on stderr is not standardized.
-shell -err -expect {option} "varnishd -A "
-
-shell -err -expect {Usage: varnishd [options]} "varnishd -? "
-shell -err -expect {Too many arguments} "varnishd foo "
-
-shell "varnishd -x parameter > ${tmpdir}/_.param"
-shell "varnishd -x vsl > ${tmpdir}/_.vsl"
-shell "varnishd -x cli > ${tmpdir}/_.cli"
-shell "varnishd -x builtin > ${tmpdir}/_.builtin"
-
-shell -err -expect {-C needs either -b <backend> or -f <vcl_file>} {
-	varnishd -C
-}
-shell -err -expect {Cannot open -S file} {
-	varnishd -S ${tmpdir}/nonexistent -n ${tmpdir}/v0 -f ''
-}
-shell -err -expect {Unknown jail method "xyz"} "varnishd -jxyz -f '' "
-
-shell -err -expect {Invalid backslash sequence} {
-	varnishd -l 'xyz\kk,xyz\foo' -f ''
-}
-shell -err -expect {Invalid backslash sequence} {
-	varnishd -l 'ab\8cd' -f ''
-}
-shell -err -expect {Too many arguments for -V} "varnishd -V -V"
-shell -expect {Copyright (c) 2006} "varnishd -V"
-
-shell -err -expect {Only one of -d or -F can be specified} "varnishd -d -F "
-shell -err -expect {Only one of -b or -f can be specified} "varnishd -b a -f b "
-shell -err -expect {-d makes no sense with -C} "varnishd -C -b 127.0.0.1:80 -d "
-shell -err -expect {-F makes no sense with -C} "varnishd -C -b 127.0.0.1:80 -F "
-shell -err -expect {Neither -b nor -f given} { varnishd -n ${tmpdir}/v0  }
-
-# This check is kept last because it may be skipped
-
-feature persistent_storage
-
-shell -err -expect {-spersistent has been deprecated} {
-	varnishd -spersistent -f ''
-}
diff --git a/bin/varnishtest/tests/a00016.vtc b/bin/varnishtest/tests/a00016.vtc
deleted file mode 100644
index 4055b7a39..000000000
--- a/bin/varnishtest/tests/a00016.vtc
+++ /dev/null
@@ -1,25 +0,0 @@
-varnishtest "Test -I and -l arguments"
-
-shell -err -expect {Only one -I allowed} {
-	touch foo bar
-	varnishd -f '' -I foo -I bar -n ${tmpdir}/v0 -a :0
-}
-
-shell -err -expect {Error: -I file CLI command failed (104)} {
-	echo "vcl.list" > foo
-	echo "-foobar" >> foo
-	echo "vcl.load" >> foo
-	varnishd -f '' -I foo -n ${tmpdir}/v0 -a :0 -l 2m
-}
-
-shell -err -expect {Error: -l ...: Missing '"'} {
-	varnishd -f '' -I foo -n ${tmpdir}/v0 -a :0 -l '2m,"'
-}
-
-shell -err -expect {Error: Too many sub arguments} {
-	varnishd -f '' -I foo -n ${tmpdir}/v0 -a :0 -l 2m,2m,2m
-}
-
-shell -err -expect {Warning: Ignoring deprecated second subargument} {
-	varnishd -f '' -I foo -n ${tmpdir}/v0 -a :0 -l 2m,2m
-}
diff --git a/bin/varnishtest/tests/a00017.vtc b/bin/varnishtest/tests/a00017.vtc
deleted file mode 100644
index 21b184cfa..000000000
--- a/bin/varnishtest/tests/a00017.vtc
+++ /dev/null
@@ -1,80 +0,0 @@
-varnishtest "Code coverage of mgt_main, (VCL compiler and RSTdump etc)"
-
-# Test -F mode with no VCL loaded
-
-process p1 "exec varnishd -n ${tmpdir}/v1 -F -f '' -a :0 -l2m,1m" -log -start
-
-delay 2
-
-shell {
-	(
-	echo 'vcl 4.0;'
-	echo 'backend default {'
-	echo '  .host="${bad_backend}";'
-	echo '}'
-	) > ${tmpdir}/vcl
-}
-
-shell -expect {VCL compiled.} {
-	varnishadm -n ${tmpdir}/v1 vcl.load vcl1 ${tmpdir}/vcl
-}
-
-shell -expect {active      auto/warm          - vcl1} {
-	varnishadm -n ${tmpdir}/v1 vcl.list
-}
-
-shell {varnishadm -n ${tmpdir}/v1 start}
-
-shell {varnishadm -n ${tmpdir}/v1 debug.listen_address}
-
-shell -exit 1 -expect "Command failed with error code 500" {
-	varnishadm -n ${tmpdir}/v1 quit
-}
-
-shell -exit 1 -expect "Command failed with error code 102" {
-	varnishadm -n ${tmpdir}/v1 debug.panic.master -j
-}
-
-shell -exit 1 -expect "Command failed with error code 101" {
-	varnishadm -n ${tmpdir}/v1 123
-}
-
-shell "varnishadm -n ${tmpdir}/v1 param.set cli_limit 128"
-shell -expect "[response was truncated]" "varnishadm -n ${tmpdir}/v1 help"
-
-process p1 -expect-exit 64 -stop -wait
-
-# Test multiple -f options
-
-shell {
-	cat >${tmpdir}/ok1 <<-EOF
-	vcl 4.0;
-	backend ok1 {
-	  .host="${bad_backend}";
-	}
-	EOF
-
-	cat >${tmpdir}/ok2 <<-EOF
-	vcl 4.0;
-	backend ok2 {
-	  .host="${bad_backend}";
-	}
-	EOF
-}
-
-varnish v2 -arg "-f ${tmpdir}/ok1" -arg "-f ${tmpdir}/ok2" -start
-varnish v2 -cliexpect {available *auto/warm *0 boot0} "vcl.list"
-varnish v2 -cliexpect {active *auto/warm *0 boot} "vcl.list"
-varnish v2 -stop -wait
-
-# Test multiple -f options with a bad VCL
-
-shell -err -expect {Cannot read -f file} {
-	exec varnishd -n ${tmpdir}/v0 -F -a :0 -l2m,3m -f ${tmpdir}/ok1 \
-	    -f ${tmpdir}/ok2 -f ${tmpdir}/bad
-}
-
-shell -err -expect {Cannot read -f file} {
-	exec varnishd -n ${tmpdir}/v0 -F -a :0 -l2m,3m -f ${tmpdir}/ok1 \
-	    -f ${tmpdir}/bad -f ${tmpdir}/ok2
-}
diff --git a/bin/varnishtest/tests/a00019.vtc b/bin/varnishtest/tests/a00019.vtc
deleted file mode 100644
index a6d1317d5..000000000
--- a/bin/varnishtest/tests/a00019.vtc
+++ /dev/null
@@ -1,33 +0,0 @@
-varnishtest "vtc v_* macros when the listen address is UDS"
-
-varnish v1 -arg "-a ${tmpdir}/v1.sock -b '${bad_backend}'" -start
-
-varnish v1 -syntax 4.0 -errvcl {Compiled VCL version (4.0) not supported.} {
-	backend default { .host = "${bad_ip}"; }
-}
-
-varnish v1 -syntax 4.0 -errvcl \
-	{Unix socket backends only supported in VCL4.1 and higher.} \
-	{backend default { .path = "${tmpdir}/v1.sock"; }}
-
-varnish v2 -vcl {
-	backend default { .host = "${bad_ip}"; }
-
-	sub vcl_recv {
-		return(synth(200));
-	}
-
-	sub vcl_synth {
-		set resp.http.addr = "${v1_addr}";
-		set resp.http.port = "${v1_port}";
-		set resp.http.sock = "${v1_sock}";
-	}
-} -start
-
-client c1 -connect ${v2_sock} {
-	txreq
-	rxresp
-	expect resp.http.addr == "${tmpdir}/v1.sock"
-	expect resp.http.port == "-"
-	expect resp.http.sock == "${tmpdir}/v1.sock -"
-} -run
diff --git a/bin/varnishtest/tests/a00020.vtc b/bin/varnishtest/tests/a00020.vtc
deleted file mode 100644
index aba786e2e..000000000
--- a/bin/varnishtest/tests/a00020.vtc
+++ /dev/null
@@ -1,34 +0,0 @@
-varnishtest "vtc remote.ip, remote.port and remote.path"
-
-server s1 {
-	rxreq
-	expect remote.ip == "${localhost}"
-	expect remote.port > 0
-	expect remote.path == <undef>
-	txresp
-} -start
-
-varnish v1 -vcl+backend {} -start
-
-client c1 {
-	txreq
-	rxresp
-	expect remote.ip == "${v1_addr}"
-	expect remote.port == "${v1_port}"
-	expect remote.path == <undef>
-} -run
-
-varnish v1 -stop
-
-server s1 -wait
-server s1 -start
-
-varnish v2 -arg "-a ${tmpdir}/v2.sock" -vcl+backend {} -start
-
-client c1 -connect "${tmpdir}/v2.sock" {
-	txreq
-	rxresp
-	expect remote.ip == "0.0.0.0"
-	expect remote.port == 0
-	expect remote.path == "${tmpdir}/v2.sock"
-} -run
diff --git a/bin/varnishtest/tests/b00053.vtc b/bin/varnishtest/tests/b00053.vtc
index 0acb9daec..adcec5974 100644
--- a/bin/varnishtest/tests/b00053.vtc
+++ b/bin/varnishtest/tests/b00053.vtc
@@ -32,3 +32,37 @@ varnish v1 -expect cache_miss == 1
 varnish v1 -expect s_sess == 1
 varnish v1 -expect s_resp_bodybytes == 7
 varnish v1 -expect s_resp_hdrbytes == 178
+
+# varnishtest "vtc v_* macros when the listen address is UDS" (a00019)
+
+varnish v2 -arg "-a ${tmpdir}/v1.sock -b '${bad_backend}'" -start
+
+varnish v2 -syntax 4.0 -errvcl {Compiled VCL version (4.0) not supported.} {
+	backend default { .host = "${bad_ip}"; }
+}
+
+varnish v2 -syntax 4.0 -errvcl \
+	{Unix socket backends only supported in VCL4.1 and higher.} \
+	{backend default { .path = "${tmpdir}/v1.sock"; }}
+
+varnish v3 -vcl {
+	backend default { .host = "${bad_ip}"; }
+
+	sub vcl_recv {
+		return(synth(200));
+	}
+
+	sub vcl_synth {
+		set resp.http.addr = "${v1_addr}";
+		set resp.http.port = "${v1_port}";
+		set resp.http.sock = "${v1_sock}";
+	}
+} -start
+
+client c1 -connect ${v3_sock} {
+	txreq
+	rxresp
+	expect resp.http.addr == "${tmpdir}/v1.sock"
+	expect resp.http.port == "-"
+	expect resp.http.sock == "${tmpdir}/v1.sock -"
+} -run
diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc
index 7e426ec96..ae91b169f 100644
--- a/bin/varnishtest/tests/u00000.vtc
+++ b/bin/varnishtest/tests/u00000.vtc
@@ -1,32 +1,162 @@
-varnishtest "Simple process tests"
-
-process p1 "cat" -start
-process p1 -writeln "foo"
-process p1 -expect-text 2 1 foo
-process p1 -stop
-process p1 -wait
-shell "grep -q foo ${p1_out}"
-shell "test -f ${p1_err} -a ! -s ${p1_err}"
-
-process p2 -log "cat" -start
-process p2 -writeln "bar"
-process p2 -expect-text 2 1 bar
-process p2 -write "\x04"
-process p2 -wait
-shell "grep -q bar ${p2_out}"
-shell "test -f ${p2_err} -a ! -s ${p2_err}"
-
-process p3 -dump "cat" -start
-process p3 -writeln "baz"
-process p3 -expect-text 2 1 baz
-process p3 -kill KILL
-process p3 -wait
-shell "grep -q baz ${p3_out}"
-shell "test -f ${p3_err} -a ! -s ${p3_err}"
-
-process p4 -hexdump "cat" -start
-process p4 -writeln "b\001z"
-process p4 -expect-text 2 1 "b"
-process p4 -kill TERM
-process p4 -wait -screen_dump
+varnishtest "Code coverage of mgt_main, (VCL compiler and RSTdump etc)"
 
+shell "varnishd -b 127.0.0.1:80 -C 2> ${tmpdir}/_.c"
+shell -err -expect {VCL version declaration missing} {
+	echo 'bad vcl' > ${tmpdir}/bad.vcl
+	varnishd -f ${tmpdir}/bad.vcl -n ${tmpdir}
+}
+
+shell -err -expect {-x must be the first argument} "varnishd -d -x foo "
+shell -err -expect {-V must be the first argument} "varnishd -d -V foo "
+
+shell -err -expect {Too many arguments for -x} "varnishd -x foo bar"
+shell -err -expect {Invalid -x argument} "varnishd -x foo "
+
+# This one is tricky, the getopt message on stderr is not standardized.
+shell -err -expect {option} "varnishd -A "
+
+shell -err -expect {Usage: varnishd [options]} "varnishd -? "
+shell -err -expect {Too many arguments} "varnishd foo "
+
+shell "varnishd -x parameter > ${tmpdir}/_.param"
+shell "varnishd -x vsl > ${tmpdir}/_.vsl"
+shell "varnishd -x cli > ${tmpdir}/_.cli"
+shell "varnishd -x builtin > ${tmpdir}/_.builtin"
+
+shell -err -expect {-C needs either -b <backend> or -f <vcl_file>} {
+	varnishd -C
+}
+shell -err -expect {Cannot open -S file} {
+	varnishd -S ${tmpdir}/nonexistent -n ${tmpdir}/v0 -f ''
+}
+shell -err -expect {Unknown jail method "xyz"} "varnishd -jxyz -f '' "
+
+shell -err -expect {Invalid backslash sequence} {
+	varnishd -l 'xyz\kk,xyz\foo' -f ''
+}
+shell -err -expect {Invalid backslash sequence} {
+	varnishd -l 'ab\8cd' -f ''
+}
+shell -err -expect {Too many arguments for -V} "varnishd -V -V"
+shell -expect {Copyright (c) 2006} "varnishd -V"
+
+shell -err -expect {Only one of -d or -F can be specified} "varnishd -d -F "
+shell -err -expect {Only one of -b or -f can be specified} "varnishd -b a -f b "
+shell -err -expect {-d makes no sense with -C} "varnishd -C -b 127.0.0.1:80 -d "
+shell -err -expect {-F makes no sense with -C} "varnishd -C -b 127.0.0.1:80 -F "
+shell -err -expect {Neither -b nor -f given} { varnishd -n ${tmpdir}/v0  }
+
+# This check is kept last because it may be skipped
+
+feature persistent_storage
+
+shell -err -expect {-spersistent has been deprecated} {
+	varnishd -spersistent -f ''
+}
+
+# Test -I and -l arguments (former a00016)
+
+shell -err -expect {Only one -I allowed} {
+	touch foo bar
+	varnishd -f '' -I foo -I bar -n ${tmpdir}/v0 -a :0
+}
+
+shell -err -expect {Error: -I file CLI command failed (104)} {
+	echo "vcl.list" > foo
+	echo "-foobar" >> foo
+	echo "vcl.load" >> foo
+	varnishd -f '' -I foo -n ${tmpdir}/v0 -a :0 -l 2m
+}
+
+shell -err -expect {Error: -l ...: Missing '"'} {
+	varnishd -f '' -I foo -n ${tmpdir}/v0 -a :0 -l '2m,"'
+}
+
+shell -err -expect {Error: Too many sub arguments} {
+	varnishd -f '' -I foo -n ${tmpdir}/v0 -a :0 -l 2m,2m,2m
+}
+
+shell -err -expect {Warning: Ignoring deprecated second subargument} {
+	varnishd -f '' -I foo -n ${tmpdir}/v0 -a :0 -l 2m,2m
+}
+
+# Code coverage of mgt_main, (VCL compiler and RSTdump etc) (former a00017)
+
+# Test -F mode with no VCL loaded
+
+process p1 "exec varnishd -n ${tmpdir}/v1 -F -f '' -a :0 -l2m,1m" -log -start
+
+delay 2
+
+shell {
+	(
+	echo 'vcl 4.0;'
+	echo 'backend default {'
+	echo '  .host="${bad_backend}";'
+	echo '}'
+	) > ${tmpdir}/vcl
+}
+
+shell -expect {VCL compiled.} {
+	varnishadm -n ${tmpdir}/v1 vcl.load vcl1 ${tmpdir}/vcl
+}
+
+shell -expect {active      auto/warm          - vcl1} {
+	varnishadm -n ${tmpdir}/v1 vcl.list
+}
+
+shell {varnishadm -n ${tmpdir}/v1 start}
+
+shell {varnishadm -n ${tmpdir}/v1 debug.listen_address}
+
+shell -exit 1 -expect "Command failed with error code 500" {
+	varnishadm -n ${tmpdir}/v1 quit
+}
+
+shell -exit 1 -expect "Command failed with error code 102" {
+	varnishadm -n ${tmpdir}/v1 debug.panic.master -j
+}
+
+shell -exit 1 -expect "Command failed with error code 101" {
+	varnishadm -n ${tmpdir}/v1 123
+}
+
+shell "varnishadm -n ${tmpdir}/v1 param.set cli_limit 128"
+shell -expect "[response was truncated]" "varnishadm -n ${tmpdir}/v1 help"
+
+process p1 -expect-exit 64 -stop -wait
+
+# Test multiple -f options
+
+shell {
+	cat >${tmpdir}/ok1 <<-EOF
+	vcl 4.0;
+	backend ok1 {
+	  .host="${bad_backend}";
+	}
+	EOF
+
+	cat >${tmpdir}/ok2 <<-EOF
+	vcl 4.0;
+	backend ok2 {
+	  .host="${bad_backend}";
+	}
+	EOF
+}
+
+varnish v2 -arg "-f ${tmpdir}/ok1" -arg "-f ${tmpdir}/ok2" -start
+varnish v2 -cliexpect {available *auto/warm *0 boot0} "vcl.list"
+varnish v2 -cliexpect {active *auto/warm *0 boot} "vcl.list"
+varnish v2 -stop -wait
+
+# Test multiple -f options with a bad VCL
+
+shell -err -expect {Cannot read -f file} {
+	exec varnishd -n ${tmpdir}/v0 -F -a :0 -l2m,3m -f ${tmpdir}/ok1 \
+	    -f ${tmpdir}/ok2 -f ${tmpdir}/bad
+}
+
+shell -err -expect {Cannot read -f file} {
+	exec varnishd -n ${tmpdir}/v0 -F -a :0 -l2m,3m -f ${tmpdir}/ok1 \
+	    -f ${tmpdir}/bad -f ${tmpdir}/ok2
+}
diff --git a/bin/varnishtest/tests/v00054.vtc b/bin/varnishtest/tests/v00054.vtc
index a230fb867..c92e8cce6 100644
--- a/bin/varnishtest/tests/v00054.vtc
+++ b/bin/varnishtest/tests/v00054.vtc
@@ -1,6 +1,40 @@
 varnishtest "client.identity is 0.0.0.0 if unset & client addr is UDS"
+# varnishtest "vtc remote.ip, remote.port and remote.path" (a00020)
 
-varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl {
+server s1 {
+	rxreq
+	expect remote.ip == "${localhost}"
+	expect remote.port > 0
+	expect remote.path == <undef>
+	txresp
+} -start
+
+varnish v1 -vcl+backend {} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect remote.ip == "${v1_addr}"
+	expect remote.port == "${v1_port}"
+	expect remote.path == <undef>
+} -run
+
+varnish v1 -stop
+
+server s1 -wait
+server s1 -start
+
+varnish v2 -arg "-a ${tmpdir}/v2.sock" -vcl+backend {} -start
+
+client c1 -connect "${tmpdir}/v2.sock" {
+	txreq
+	rxresp
+	expect remote.ip == "0.0.0.0"
+	expect remote.port == 0
+	expect remote.path == "${tmpdir}/v2.sock"
+} -run
+
+varnish v3 -arg "-a ${tmpdir}/v3.sock" -vcl {
 	backend b { .host = "${bad_ip}"; }
 
 	sub vcl_recv {
@@ -16,7 +50,7 @@ varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl {
 	}
 } -start
 
-client c1 -connect "${tmpdir}/v1.sock" {
+client c2 -connect "${tmpdir}/v3.sock" {
 	txreq -url "/nobody"
 	rxresp
 	expect resp.status == 200

From hermunn at varnish-software.com  Wed Oct 24 09:29:15 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:15 +0000 (UTC)
Subject: [6.1] 41f7b093b A train-trip worth of python3 migration and other
 polishing
Message-ID: <20181024092915.7D2DEAFB76@lists.varnish-cache.org>


commit 41f7b093bdccdd23d8113c1e636ff7296f0b68e6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 07:26:11 2018 +0000

    A train-trip worth of python3 migration and other polishing

diff --git a/bin/varnishtest/huffman_gen.py b/bin/varnishtest/huffman_gen.py
index 8da37b2ee..b89bbb43e 100755
--- a/bin/varnishtest/huffman_gen.py
+++ b/bin/varnishtest/huffman_gen.py
@@ -7,50 +7,50 @@ import sys
 regex = re.compile("^HPH\((.{4}), (.{10}), +(.{1,3})\)")
 
 if len(sys.argv) != 2:
-	print("{} takes one and only one argument".format(sys.argv[0]))
-	sys.exit(2)
+    print("{} takes one and only one argument".format(sys.argv[0]))
+    sys.exit(2)
 
 class sym:
-	def __init__(self, bigval, bigvall, chr = 0, esc = None):
-		self.vall = bigvall % 8 if bigvall % 8 else 8
-		self.val = bigval & ((1 << self.vall) - 1)
-		self.pfx = (bigval >> self.vall)# & 0xff
-		self.chr = chr
-		self.esc = esc
+    def __init__(self, bigval, bigvall, chr=0, esc=None):
+        self.vall = bigvall % 8 if bigvall % 8 else 8
+        self.val = bigval & ((1 << self.vall) - 1)
+        self.pfx = (bigval >> self.vall)# & 0xff
+        self.chr = chr
+        self.esc = esc
 
 tbls = {}
 msl = {} # max sym length
 
 f = open(sys.argv[1])
 for l in f:
-	grp = 1
-	match = regex.match(l)
-	if not match:
-		continue
-
-	chr = int(match.group(grp), 16)
-	grp += 1
-
-	val = int(match.group(grp), 16)
-	grp += 1
-
-	vall = int(match.group(grp))
-
-	s = sym(val, vall, chr)
-	if s.pfx not in tbls:
-		tbls[s.pfx] = {}
-
-	if (s.val in tbls[s.pfx]):
-		assert(tbls[s.pfx][s.val].e)
-	tbls[s.pfx][s.val] = s
-
-	# add the escape entry in the "previous" table
-	if s.pfx:
-		pp = s.pfx >> 8
-		pv = s.pfx & 0xff
-		if pp not in tbls:
-			tbls[pp] = {}
-		tbls[pp][pv] = sym(pv, 8, 0, "&tbl_{:x}".format(s.pfx))
+    grp = 1
+    match = regex.match(l)
+    if not match:
+        continue
+
+    char = int(match.group(grp), 16)
+    grp += 1
+
+    val = int(match.group(grp), 16)
+    grp += 1
+
+    vall = int(match.group(grp))
+
+    s = sym(val, vall, char)
+    if s.pfx not in tbls:
+        tbls[s.pfx] = {}
+
+    if s.val in tbls[s.pfx]:
+        assert tbls[s.pfx][s.val].e
+    tbls[s.pfx][s.val] = s
+
+    # add the escape entry in the "previous" table
+    if s.pfx:
+        pp = s.pfx >> 8
+        pv = s.pfx & 0xff
+        if pp not in tbls:
+            tbls[pp] = {}
+        tbls[pp][pv] = sym(pv, 8, 0, "&tbl_{:x}".format(s.pfx))
 f.close()
 
 # add the EOS case
@@ -64,33 +64,33 @@ print('''/* NB:  This file is machine generated, DO NOT EDIT!
 struct stbl;
 
 struct ssym {
-	uint8_t csm;		/* bits consumed */
-	uint8_t	chr;		/* character */
-	struct stbl *nxt;	/* next table */
+    uint8_t csm;        /* bits consumed */
+    uint8_t chr;        /* character */
+    struct stbl *nxt;       /* next table */
 };
 
 struct stbl {
-	int msk;
-	struct ssym *syms;
+    int msk;
+    struct ssym *syms;
 };
 ''')
 
 for pfx in sorted(tbls.keys(), reverse=True):
-	msl = max([ x.vall for x in tbls[pfx].values() ])
-	for s in tbls[pfx].values():
-		s.val = s.val << (msl - s.vall)
-
-	tbl = sorted(tbls[pfx].values(), key= lambda x: x.val)
-	print("\nstatic struct ssym sym_{:x}_array[] = {{".format(pfx))
-	for s in tbl:
-		for j in range(2 ** (msl - s.vall)):
-			print("	{} {{{}, {:3d}, {}}},".format(
-				"\t     " if j else "/* idx {:3d} */".format(s.val + j),
-				s.vall, s.chr % 256,
-				s.esc if s.esc else "NULL"))
-	print('''}};
+    msl = max([x.vall for x in tbls[pfx].values()])
+    for s in tbls[pfx].values():
+        s.val = s.val << (msl - s.vall)
+
+    tbl = sorted(tbls[pfx].values(), key=lambda x: x.val)
+    print("\nstatic struct ssym sym_{:x}_array[] = {{".format(pfx))
+    for s in tbl:
+        for j in range(2 ** (msl - s.vall)):
+            print("{} {{{}, {:3d}, {}}},".format(
+                "\t     " if j else "/* idx {:3d} */".format(s.val + j),
+                s.vall, s.chr % 256,
+                s.esc if s.esc else "NULL"))
+    print('''}};
 
 static struct stbl tbl_{:x} = {{
-	{},
-	sym_{:x}_array
+    {},
+    sym_{:x}_array
 }};'''.format(pfx, msl, pfx))
diff --git a/bin/varnishtest/witness.py b/bin/varnishtest/witness.py
index f57fdf221..a4f0fffa6 100644
--- a/bin/varnishtest/witness.py
+++ b/bin/varnishtest/witness.py
@@ -3,9 +3,9 @@
 # This script is in the public domain
 #
 # Run instructions:
-#	varnishtest -W -iv -j <pick_a_number> > _.w
-#	python witness.py
-#	dot -Tpng /tmp/_.dot > /tmp/_.png
+#       varnishtest -W -iv -j <pick_a_number> > _.w
+#       python witness.py
+#       dot -Tpng /tmp/_.dot > /tmp/_.png
 
 from __future__ import print_function
 
@@ -16,37 +16,37 @@ fi = open("_.w")
 fo = open("/tmp/_.dot", "w")
 
 fo.write('''digraph {
-	#rotate="90"
-	#page="8.2,11.7"
-	size="8.2,11.7"
-	rankdir="LR"
-	node [fontname="Inconsolata", fontsize="10"]
-	edge [fontname="Inconsolata", fontsize="10"]
+    #rotate="90"
+    #page="8.2,11.7"
+    size="8.2,11.7"
+    rankdir="LR"
+    node [fontname="Inconsolata", fontsize="10"]
+    edge [fontname="Inconsolata", fontsize="10"]
 ''')
 
 for i in fi:
-	l = "ROOT"
-	j = i.split()
-	if len(j) < 8:
-		continue
-	if j[1][0] != 'v':
-		continue
-	if j[3] != "vsl|":
-		continue
-	if j[5] != "Witness":
-		continue
-	t = j[7:]
-	tt = str(t)
-	if tt in d:
-		continue
-	d[tt] = True
-	for e in t:
-		f = e.split(",")
-		x = '"%s" -> "%s" [label="%s(%s)"]\n' % (l, f[0], f[1], f[2])
-		if not x in a:
-			a[x] = True
-			fo.write(x)
-		l = f[0]
+    l = "ROOT"
+    j = i.split()
+    if len(j) < 8:
+        continue
+    if j[1][0] != 'v':
+        continue
+    if j[3] != "vsl|":
+        continue
+    if j[5] != "Witness":
+        continue
+    t = j[7:]
+    tt = str(t)
+    if tt in d:
+        continue
+    d[tt] = True
+    for e in t:
+        f = e.split(",")
+        x = '"%s" -> "%s" [label="%s(%s)"]\n' % (l, f[0], f[1], f[2])
+        if not x in a:
+            a[x] = True
+            fo.write(x)
+        l = f[0]
 
 fo.write("}\n")
 
diff --git a/doc/sphinx/vtc-syntax.py b/doc/sphinx/vtc-syntax.py
index 298f4bb01..ba7bd6bd7 100644
--- a/doc/sphinx/vtc-syntax.py
+++ b/doc/sphinx/vtc-syntax.py
@@ -35,50 +35,49 @@ import re
 
 
 def parse_file(fn, cl, tl, sl):
-	p = False
-	section = ""
-	resec = re.compile("[ /]\* SECTION: ")
+    p = False
+    section = ""
+    resec = re.compile("[ /]\* SECTION: ")
 
-	f = open(fn, "r")
+    f = open(fn, "r")
 
-	for l in f:
-		if "*/" in l:
-			p = 0
-		if resec.match(l):
-			a = l.split()
-			section = a[2]
-			sl.append(section)
-			cl[section] = []
-			if len(a) > 3:
-				tl[section] = re.sub(
-					r"^[\t ]*\/?\* SECTION: [^ ]+ +",
-					"", l)
-			else:
-				tl[section] = ""
-			p = 1
-		elif p:
-			cl[section].append(re.sub(r"^ \* ?", "", l))
-	f.close()
+    for l in f:
+        if "*/" in l:
+            p = 0
+        if resec.match(l):
+            a = l.split()
+            section = a[2]
+            sl.append(section)
+            cl[section] = []
+            if len(a) > 3:
+                tl[section] = re.sub(
+                    r"^[\t ]*\/?\* SECTION: [^ ]+ +",
+                    "", l)
+            else:
+                tl[section] = ""
+            p = 1
+        elif p:
+            cl[section].append(re.sub(r"^ \* ?", "", l))
+    f.close()
 
 if __name__ == "__main__":
-	cl = {}
-	tl = {}
-	sl = []
-	for fn in sys.argv[1:]:
-		parse_file(fn, cl, tl, sl)
-	sl.sort()
-	for section in sl:
-		print(tl[section], end="")
-		a = section
-		c = section.count(".")
-		if c == 0:
-			r = "-"
-		elif c == 1:
-			r = "~"
-		elif c == 2:
-			r = "."
-		else:
-			r = "*"
-		print(re.sub(r".", r, tl[section]), end="")
-		print("".join(cl[section]))
-
+    cl = {}
+    tl = {}
+    sl = []
+    for fn in sys.argv[1:]:
+        parse_file(fn, cl, tl, sl)
+    sl.sort()
+    for section in sl:
+        print(tl[section], end="")
+        a = section
+        c = section.count(".")
+        if c == 0:
+            r = "-"
+        elif c == 1:
+            r = "~"
+        elif c == 2:
+            r = "."
+        else:
+            r = "*"
+        print(re.sub(r".", r, tl[section]), end="")
+        print("".join(cl[section]))
diff --git a/include/Makefile.am b/include/Makefile.am
index 27f40ec86..98050fd78 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -108,7 +108,7 @@ vcl.h: \
 	    $(top_srcdir)/include/vrt.h \
 	    $(top_srcdir)/doc/sphinx/reference/vcl_var.rst
 	mkdir -p $(top_builddir)/include/tbl
-	@PYTHON@ $(top_srcdir)/lib/libvcc/generate.py \
+	${PYTHON} $(top_srcdir)/lib/libvcc/generate.py \
 	    $(top_srcdir) $(top_builddir)
 
 GEN_H = \
@@ -128,7 +128,7 @@ vcs_version.h:
 	@if test -e $(top_srcdir)/.git || \
 	  ! test -f $(srcdir)/vmod_abi.h || \
 	  ! test -f $(srcdir)/vcs_version.h ; then \
-	    @PYTHON@ $(srcdir)/generate.py \
+	    ${PYTHON} $(srcdir)/generate.py \
 		$(top_srcdir) $(top_builddir) ; \
 	fi
 
diff --git a/include/generate.py b/include/generate.py
index 0d9d9b32b..e7d547593 100755
--- a/include/generate.py
+++ b/include/generate.py
@@ -32,22 +32,21 @@
 from __future__ import print_function
 
 import subprocess
-import collections
 import os
 import sys
 
 srcroot = "../.."
 buildroot = "../.."
 if len(sys.argv) == 3:
-	srcroot = sys.argv[1]
-	buildroot = sys.argv[2]
+    srcroot = sys.argv[1]
+    buildroot = sys.argv[2]
 elif len(sys.argv) != 1:
-	print("Two arguments or none")
-	exit(2)
+    print("Two arguments or none")
+    exit(2)
 
 #######################################################################
 def file_header(fo):
-	fo.write("""/*
+    fo.write("""/*
  * NB:  This file is machine generated, DO NOT EDIT!
  *
  * Edit and run include/generate.py instead.
@@ -66,25 +65,25 @@ v = subprocess.check_output([
 vcsfn = os.path.join(srcroot, "include", "vcs_version.h")
 
 try:
-	i = open(vcsfn).readline()
+    i = open(vcsfn).readline()
 except IOError:
-	i = ""
+    i = ""
 
 ident = "/* " + v + " */\n"
 
 if i != ident:
-	fo = open(vcsfn, "w")
-	fo.write(ident)
-	file_header(fo)
-	fo.write('#define VCS_Version "%s"\n' % v)
-	fo.close()
+    fo = open(vcsfn, "w")
+    fo.write(ident)
+    file_header(fo)
+    fo.write('#define VCS_Version "%s"\n' % v)
+    fo.close()
 
-	for i in open(os.path.join(buildroot, "Makefile")):
-		if i[:14] == "PACKAGE_STRING":
-			break
-	i = i.split("=")[1].strip()
+    for i in open(os.path.join(buildroot, "Makefile")):
+        if i[:14] == "PACKAGE_STRING":
+            break
+    i = i.split("=")[1].strip()
 
-	fo = open(os.path.join(srcroot, "include", "vmod_abi.h"), "w")
-	file_header(fo)
-	fo.write('#define VMOD_ABI_Version "%s %s"\n' % (i, v))
-	fo.close()
+    fo = open(os.path.join(srcroot, "include", "vmod_abi.h"), "w")
+    file_header(fo)
+    fo.write('#define VMOD_ABI_Version "%s %s"\n' % (i, v))
+    fo.close()
diff --git a/include/tbl/h2_settings.h b/include/tbl/h2_settings.h
index adfb416d4..c382e862d 100644
--- a/include/tbl/h2_settings.h
+++ b/include/tbl/h2_settings.h
@@ -47,6 +47,7 @@ H2_SETTING(					// rfc7540,l,2097,2103
 	0xffffffff,
 	0
 )
+
 #ifndef H2_SETTINGS_PARAM_ONLY
 H2_SETTING(					// rfc7540,l,2105,2114
 	ENABLE_PUSH,
@@ -58,6 +59,7 @@ H2_SETTING(					// rfc7540,l,2105,2114
 	H2CE_PROTOCOL_ERROR
 )
 #endif
+
 H2_SETTING(					// rfc7540,l,2116,2121
 	MAX_CONCURRENT_STREAMS,
 	max_concurrent_streams,
@@ -67,6 +69,7 @@ H2_SETTING(					// rfc7540,l,2116,2121
 	0xffffffff,
 	0
 )
+
 H2_SETTING(					// rfc7540,l,2139,2148
 	INITIAL_WINDOW_SIZE,
 	initial_window_size,
@@ -76,6 +79,7 @@ H2_SETTING(					// rfc7540,l,2139,2148
 	0x7fffffff,
 	H2CE_FLOW_CONTROL_ERROR
 )
+
 H2_SETTING(					// rfc7540,l,2150,2157
 	MAX_FRAME_SIZE,
 	max_frame_size,
@@ -85,6 +89,7 @@ H2_SETTING(					// rfc7540,l,2150,2157
 	0x00ffffff,
 	H2CE_PROTOCOL_ERROR
 )
+
 H2_SETTING(					// rfc7540,l,2159,2167
 	MAX_HEADER_LIST_SIZE,
 	max_header_list_size,
diff --git a/include/tbl/htc.h b/include/tbl/htc.h
index a93c0e520..4e9695491 100644
--- a/include/tbl/htc.h
+++ b/include/tbl/htc.h
@@ -36,10 +36,11 @@ HTC_STATUS(JUNK,	-5,	"junk",		"Received unexpected data")
 HTC_STATUS(CLOSE,	-4,	"close",	"Connection closed") // unused?
 HTC_STATUS(TIMEOUT,	-3,	"timeout",	"Timed out")
 HTC_STATUS(OVERFLOW,	-2,	"overflow",	"Buffer/workspace too small")
-HTC_STATUS(EOF,	-1,	"eof",		"Unexpected end of input")
+HTC_STATUS(EOF,		-1,	"eof",		"Unexpected end of input")
 HTC_STATUS(EMPTY,	 0,	"empty",	"Empty response")
-HTC_STATUS(MORE,	 1,	"more",	"More data required")
+HTC_STATUS(MORE,	 1,	"more",		"More data required")
 HTC_STATUS(COMPLETE,	 2,	"complete",	"Data complete (no error)")
-HTC_STATUS(IDLE,	 3,	"idle",	"Connection was closed while idle")
+HTC_STATUS(IDLE,	 3,	"idle",		"Connection was closed while idle")
 #undef HTC_STATUS
+
 /*lint -restore */
diff --git a/include/tbl/style.py b/include/tbl/style.py
index a29459c0a..a88c475a3 100644
--- a/include/tbl/style.py
+++ b/include/tbl/style.py
@@ -7,66 +7,68 @@ from __future__ import print_function
 import glob
 
 def check_file(fn):
-	s = 0
-	ll = []
-	for l in open(fn):
-		ll.append(l)
+    print("Check", fn)
+    ll = []
+    for l in open(fn):
+        ll.append(l)
 
-	assert ll.pop(0)[:2] == "/*"
+    assert ll.pop(0)[:2] == "/*"
 
-	while ll.pop(0) != " */\n":
-		continue
+    while ll.pop(0) != " */\n":
+        continue
 
-	assert len(ll) > 5
+    assert len(ll) > 5
 
-	assert ll.pop(0) == "\n"
-	assert ll.pop(0) == "/*lint -save -e525 -e539 */\n"
-	assert ll.pop(0) == "\n"
+    assert ll.pop(0) == "\n"
+    i = ll.pop(0)
+    assert i == "/*lint -save -e525 -e539 */\n" or \
+           i == "/*lint -save -e525 -e539 -e835 */\n"
+    assert ll.pop(0) == "\n"
 
-	assert ll.pop(-1) == "/*lint -restore */\n"
-	assert ll.pop(-1) == "\n"
+    assert ll.pop(-1) == "/*lint -restore */\n"
+    assert ll.pop(-1) == "\n"
 
-	for i in range(0, len(ll) -1):
-		assert ll[i] != "\n" or ll[i+1] != "\n"
-		assert ll[i] != ")\n" or ll[i+1] == "\n" or ll[i+1][0] == "#"
+    for i in range(0, len(ll) -1):
+        assert ll[i] != "\n" or ll[i+1] != "\n"
+        assert ll[i] != ")\n" or ll[i+1] == "\n" or ll[i+1][0] == "#"
 
-	m = {}
-	while len(ll) > 0:
-		i = ll.pop(0)
-		if i == "\n":
-			continue
-		l = i.lstrip()
-		if l[0] >= 'A' and l[0] <= 'Z':
-			j = l.split('(')
-			m[j[0]] = "Called"
-			l = l.split('//')[0]
-			l = l.split('/*')[0]
-			l = l.rstrip()
-			if l[-1] != ')':
-				while ll.pop(0) != ')\n':
-					continue
-		elif l[0] == "#":
-			j = l[1:].lstrip().split()
-			# print("#", j[0])
-			if j[0] == "define":
-				m[j[1].split("(")[0].strip()] = "Defined"
-			if j[0] == "undef":
-				m[j[1]] = "Undef"
-			while l[-2:] == "\\\n":
-				l = ll.pop(0)
-		else:
-			pass
-			# print(l)
-	rv = 0
-	for i in m:
-		if m[i] != "Undef":
-			print("ERROR", fn, i, m[i])
-			rv += 1
-	return rv
+    m = {}
+    while ll:
+        i = ll.pop(0)
+        if i == "\n":
+            continue
+        l = i.lstrip()
+        if l[0] >= 'A' and l[0] <= 'Z':
+            j = l.split('(')
+            m[j[0]] = "Called"
+            l = l.split('//')[0]
+            l = l.split('/*')[0]
+            l = l.rstrip()
+            if l[-1] != ')':
+                while ll.pop(0).strip() != ')':
+                    continue
+        elif l[0] == "#":
+            j = l[1:].lstrip().split()
+            # print("#", j[0])
+            if j[0] == "define":
+                m[j[1].split("(")[0].strip()] = "Defined"
+            if j[0] == "undef":
+                m[j[1]] = "Undef"
+            while l[-2:] == "\\\n":
+                l = ll.pop(0)
+        else:
+            pass
+            # print(l)
+    rv = 0
+    for i in m:
+        if m[i] != "Undef":
+            print("ERROR", fn, i, m[i])
+            rv += 1
+    return rv
 
 rv = 0
 for fn in glob.glob("*.h"):
-	rv += check_file(fn)
+    rv += check_file(fn)
 if rv != 0:
-	print(rv, "Errors")
+    print(rv, "Errors")
 exit(rv)
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 0997d72f8..294da9431 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -181,16 +181,17 @@ SLTM(Length, 0, "Size of object body",
 	"Logs the size of a fetch object body.\n\n"
 )
 
-/* XXX generate HTC info from tbl include */
-#if 0
-#include <stdio.h>
-int main(void) {
-#define HTC_STATUS(e, n, s, l) \
-	printf("\t\"\\t* %s (%d): %s\\n\"\n", s, n, l);
-#include "include/tbl/htc.h"
-	return (0);
-}
-#endif
+/*
+ * XXX generate HTC info below from tbl include
+ *
+ * #include <stdio.h>
+ * int main(void) {
+ * #define HTC_STATUS(e, n, s, l) \
+ *	printf("\t\"\\t* %s (%d): %s\\n\"\n", s, n, l);
+ * #include "include/tbl/htc.h"
+ *	return (0);
+ * }
+ */
 
 SLTM(FetchError, 0, "Error while fetching object",
 	"Logs the error message of a failed fetch operation.\n\n"
diff --git a/include/tbl/waiters.h b/include/tbl/waiters.h
index f1f1d4900..f2fcf6446 100644
--- a/include/tbl/waiters.h
+++ b/include/tbl/waiters.h
@@ -44,3 +44,5 @@
 
 WAITER(poll)
 #undef WAITER
+
+/*lint -restore */
diff --git a/lib/libvarnishapi/generate.py b/lib/libvarnishapi/generate.py
index a7b505c74..d1357c985 100755
--- a/lib/libvarnishapi/generate.py
+++ b/lib/libvarnishapi/generate.py
@@ -37,122 +37,121 @@ import copy
 srcroot = "../.."
 buildroot = "../.."
 if len(sys.argv) == 3:
-	srcroot = sys.argv[1]
-	buildroot = sys.argv[2]
+    srcroot = sys.argv[1]
+    buildroot = sys.argv[2]
 
 #######################################################################
 # These are our tokens
 
 tokens = {
-	# Numerical comparisons
-	"T_EQ":         "==",
-	"T_NEQ":        "!=",
-	"T_LEQ":        "<=",
-	"T_GEQ":        ">=",
-
-	# String comparisons
-	"T_SEQ":        "eq",
-	"T_SNEQ":       "ne",
-
-	# Regular expression matching
-	"T_NOMATCH":    "!~",
-
-	# Boolean operators
-	"T_AND":        "and",
-	"T_OR":         "or",
-	"T_NOT":        "not",
-
-	# Miscellaneous
-	None:           "<>~[]{}():,",
-
-	# These have handwritten recognizers
-	"VAL":          None,
-	"EOI":          None,
-
-	# Special
-	"T_TRUE":       None,
-	"VXID":         "vxid",
+    # Numerical comparisons
+    "T_EQ":     "==",
+    "T_NEQ":    "!=",
+    "T_LEQ":    "<=",
+    "T_GEQ":    ">=",
+
+    # String comparisons
+    "T_SEQ":    "eq",
+    "T_SNEQ":       "ne",
+
+    # Regular expression matching
+    "T_NOMATCH":    "!~",
+
+    # Boolean operators
+    "T_AND":    "and",
+    "T_OR":     "or",
+    "T_NOT":    "not",
+
+    # Miscellaneous
+    None:       "<>~[]{}():,",
+
+    # These have handwritten recognizers
+    "VAL":      None,
+    "EOI":      None,
+
+    # Special
+    "T_TRUE":       None,
+    "VXID":     "vxid",
 }
 
 #######################################################################
 # Emit a function to recognize tokens in a string
 
 def emit_vxp_fixed_token(fo, tokens):
-	recog = list()
-	emit = dict()
-	for i in tokens:
-		j = tokens[i]
-		if (j != None):
-			recog.append(j)
-			emit[j] = i
-
-	recog.sort()
-	rrecog = copy.copy(recog)
-	rrecog.sort(key = lambda x: -len(x))
-
-	fo.write("""
+    recog = list()
+    emit = dict()
+    for i in tokens:
+        j = tokens[i]
+        if j is not None:
+            recog.append(j)
+            emit[j] = i
+
+    recog.sort()
+    rrecog = copy.copy(recog)
+    rrecog.sort(key=lambda x: -len(x))
+
+    fo.write("""
 unsigned
 vxp_fixed_token(const char *p, const char **q)
 {
 
 \tswitch (p[0]) {
 """)
-	last_initial = None
-	for i in recog:
-		if (i[0] == last_initial):
-			continue
-		last_initial = i[0]
-		fo.write("\tcase '%s':\n" % last_initial)
-		for j in rrecog:
-			if (j[0] != last_initial):
-				continue
-
-			fo.write("\t\tif (")
-			k = 1
-			l = len(j)
-			while (k < l):
-				fo.write("p[%d] == '%s'" % (k, j[k]))
-				fo.write(" &&\n\t\t    ")
-				k += 1
-			fo.write("(isword(p[%d]) ? !isword(p[%d]) : 1)) {\n" %
-				(l - 1, l))
-			fo.write("\t\t\t*q = p + %d;\n" % l)
-			fo.write("\t\t\treturn (%s);\n" % emit[j])
-			fo.write("\t\t}\n")
-		fo.write("\t\treturn (0);\n")
-
-	fo.write("\tdefault:\n\t\treturn (0);\n\t}\n}\n")
+    last_initial = None
+    for i in recog:
+        if i[0] == last_initial:
+            continue
+        last_initial = i[0]
+        fo.write("\tcase '%s':\n" % last_initial)
+        for j in rrecog:
+            if j[0] != last_initial:
+                continue
+
+            fo.write("\t\tif (")
+            k = 1
+            l = len(j)
+            while k < l:
+                fo.write("p[%d] == '%s'" % (k, j[k]))
+                fo.write(" &&\n\t\t    ")
+                k += 1
+            fo.write("(isword(p[%d]) ? !isword(p[%d]) : 1)) {\n" % (l - 1, l))
+            fo.write("\t\t\t*q = p + %d;\n" % l)
+            fo.write("\t\t\treturn (%s);\n" % emit[j])
+            fo.write("\t\t}\n")
+        fo.write("\t\treturn (0);\n")
+
+    fo.write("\tdefault:\n\t\treturn (0);\n\t}\n}\n")
 
 #######################################################################
 # Emit the vxp_tnames (token->string) conversion array
 
 def emit_vxp_tnames(fo, tokens):
-	fo.write("\nconst char * const vxp_tnames[256] = {\n")
-	l = list(tokens.keys())
-	l.sort()
-	for i in l:
-		j = tokens[i]
-		if j == None:
-			j = i
-		if i[0] == "'":
-			j = i
-		fo.write("\t[%s] = \"%s\",\n" % (i, j))
-	fo.write("};\n")
+    fo.write("\nconst char * const vxp_tnames[256] = {\n")
+    l = list(tokens.keys())
+    l.sort()
+    for i in l:
+        j = tokens[i]
+        if j is None:
+            j = i
+        if i[0] == "'":
+            j = i
+        fo.write("\t[%s] = \"%s\",\n" % (i, j))
+    fo.write("};\n")
 
 #######################################################################
 
 def polish_tokens(tokens):
-	# Expand single char tokens
-	st = tokens[None]
-	del tokens[None]
+    # Expand single char tokens
+    st = tokens[None]
+    del tokens[None]
 
-	for i in st:
-		tokens["'" + i + "'"] = i
+    for i in st:
+        tokens["'" + i + "'"] = i
 
 #######################################################################
 
 def file_header(fo):
-	fo.write("""/*
+    fo.write("""/*
  * NB:  This file is machine generated, DO NOT EDIT!
  *
  * Edit and run lib/libvarnishapi/generate.py instead
@@ -171,11 +170,11 @@ j = 128
 l = list(tokens.keys())
 l.sort()
 for i in l:
-	if i[0] == "'":
-		continue
-	fo.write("#define\t%s %d\n" % (i, j))
-	j += 1
-	assert j < 256
+    if i[0] == "'":
+        continue
+    fo.write("#define\t%s %d\n" % (i, j))
+    j += 1
+    assert j < 256
 
 fo.close()
 
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 4328c7072..5dba1a37a 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -32,10 +32,6 @@
 
 from __future__ import print_function
 
-import subprocess
-import collections
-import os
-
 #######################################################################
 # These are our tokens
 
@@ -51,112 +47,112 @@ from os.path import join
 srcroot = "../.."
 buildroot = "../.."
 if len(sys.argv) == 3:
-	srcroot = sys.argv[1]
-	buildroot = sys.argv[2]
+    srcroot = sys.argv[1]
+    buildroot = sys.argv[2]
 elif len(sys.argv) != 1:
-	print("Two arguments or none")
-	exit(2)
+    print("Two arguments or none")
+    exit(2)
 
 tokens = {
-	"T_INC":	"++",
-	"T_DEC":	"--",
-	"T_CAND":	"&&",
-	"T_COR":	"||",
-	"T_LEQ":	"<=",
-	"T_EQ":		"==",
-	"T_NEQ":	"!=",
-	"T_GEQ":	">=",
-	"T_SHR":	">>",
-	"T_SHL":	"<<",
-	"T_INCR":	"+=",
-	"T_DECR":	"-=",
-	"T_MUL":	"*=",
-	"T_DIV":	"/=",
-	"T_NOMATCH":	"!~",
-
-	# Single char tokens, for convenience on one line
-	None:		"{}()*+-/%><=;!&.|~,",
-
-	# These have handwritten recognizers
-	"ID":		None,
-	"CNUM":		None,
-	"FNUM":		None,
-	"CSTR":		None,
-	"EOI":		None,
-	"CSRC":		None,
+    "T_INC":    "++",
+    "T_DEC":    "--",
+    "T_CAND":       "&&",
+    "T_COR":    "||",
+    "T_LEQ":    "<=",
+    "T_EQ":     "==",
+    "T_NEQ":    "!=",
+    "T_GEQ":    ">=",
+    "T_SHR":    ">>",
+    "T_SHL":    "<<",
+    "T_INCR":       "+=",
+    "T_DECR":       "-=",
+    "T_MUL":    "*=",
+    "T_DIV":    "/=",
+    "T_NOMATCH":    "!~",
+
+    # Single char tokens, for convenience on one line
+    None:       "{}()*+-/%><=;!&.|~,",
+
+    # These have handwritten recognizers
+    "ID":       None,
+    "CNUM":     None,
+    "FNUM":     None,
+    "CSTR":     None,
+    "EOI":      None,
+    "CSRC":     None,
 }
 
 #######################################################################
 # Our methods and actions
 
 returns = (
-	###############################################################
-	# Client side
-
-	('recv',
-		"C",
-		('fail', 'synth', 'restart', 'pass', 'pipe', 'hash', 'purge', 'vcl')
-	),
-	('pipe',
-		"C",
-		('fail', 'synth', 'pipe',)
-	),
-	('pass',
-		"C",
-		('fail', 'synth', 'restart', 'fetch',)
-	),
-	('hash',
-		"C",
-		('fail', 'lookup',)
-	),
-	('purge',
-		"C",
-		('fail', 'synth', 'restart',)
-	),
-	('miss',
-		"C",
-		('fail', 'synth', 'restart', 'pass', 'fetch',)
-	),
-	('hit',
-		"C",
-		('fail', 'synth', 'restart', 'pass', 'miss', 'deliver',)
-	),
-	('deliver',
-		"C",
-		('fail', 'synth', 'restart', 'deliver',)
-	),
-	('synth',
-		"C",
-		('fail', 'restart', 'deliver',)
-	),
-
-	###############################################################
-	# Backend-fetch
-
-	('backend_fetch',
-		"B",
-		('fail', 'fetch', 'abandon')
-	),
-	('backend_response',
-		"B",
-		('fail', 'deliver', 'retry', 'abandon', 'pass')
-	),
-	('backend_error',
-		"B",
-		('fail', 'deliver', 'retry', 'abandon')
-	),
-
-	###############################################################
-	# Housekeeping
-
-	('init',
-		"H",
-		('ok', 'fail')
-	),
-	('fini',
-		"H",
-		('ok',)
-	),
+    ###############################################################
+    # Client side
+
+    ('recv',
+     "C",
+     ('fail', 'synth', 'restart', 'pass', 'pipe', 'hash', 'purge', 'vcl')
+    ),
+    ('pipe',
+     "C",
+     ('fail', 'synth', 'pipe',)
+    ),
+    ('pass',
+     "C",
+     ('fail', 'synth', 'restart', 'fetch',)
+    ),
+    ('hash',
+     "C",
+     ('fail', 'lookup',)
+    ),
+    ('purge',
+     "C",
+     ('fail', 'synth', 'restart',)
+    ),
+    ('miss',
+     "C",
+     ('fail', 'synth', 'restart', 'pass', 'fetch',)
+    ),
+    ('hit',
+     "C",
+     ('fail', 'synth', 'restart', 'pass', 'miss', 'deliver',)
+    ),
+    ('deliver',
+     "C",
+     ('fail', 'synth', 'restart', 'deliver',)
+    ),
+    ('synth',
+     "C",
+     ('fail', 'restart', 'deliver',)
+    ),
+
+    ###############################################################
+    # Backend-fetch
+
+    ('backend_fetch',
+     "B",
+     ('fail', 'fetch', 'abandon')
+    ),
+    ('backend_response',
+     "B",
+     ('fail', 'deliver', 'retry', 'abandon', 'pass')
+    ),
+    ('backend_error',
+     "B",
+     ('fail', 'deliver', 'retry', 'abandon')
+    ),
+
+    ###############################################################
+    # Housekeeping
+
+    ('init',
+     "H",
+     ('ok', 'fail')
+    ),
+    ('fini',
+     "H",
+     ('ok',)
+    ),
 )
 
 #######################################################################
@@ -170,152 +166,151 @@ returns = (
 varprotos = {}
 
 def varproto(s):
-	if not s in varprotos:
-		fh.write(s + ";\n")
-		varprotos[s] = True
+    if not s in varprotos:
+        fh.write(s + ";\n")
+        varprotos[s] = True
 
 class vardef(object):
-	def __init__(self, nam, typ, rd, wr, wu, vlo, vhi):
-		self.nam = nam
-		self.typ = typ
-		self.rd = rd
-		self.wr = wr
-		self.uns = wu
-		self.vlo = vlo
-		self.vhi = vhi
-
-		self.emit()
-
-	def emit(self):
-		fh.write("\n")
-		fo.write("\n")
-		cnam = self.nam.replace(".", "_")
-		ctyp = vcltypes[self.typ]
-
-		# fo.write("\t{ \"%s\", %s,\n" % (nm, self.typ))
-		fo.write("\tsym = VCC_MkSym(tl, \"%s\", " % self.nam)
-		if (self.typ == "HEADER"):
-			fo.write(" SYM_NONE, %d, %d);\n" % (self.vlo, self.vhi))
-			fo.write("\tAN(sym);\n");
-			fo.write("\tsym->wildcard = vcc_Var_Wildcard;\n")
-		else:
-			fo.write(" SYM_VAR, %d, %d);\n" % (self.vlo, self.vhi))
-		fo.write("\tAN(sym);\n")
-		fo.write("\tsym->type = %s;\n" % self.typ)
-		fo.write("\tsym->eval = vcc_Eval_Var;\n")
-
-		if self.typ == "HEADER":
-			fo.write('\tsym->rname = "HDR_')
-			fo.write(self.nam.split(".")[0].upper())
-			fo.write('";\n')
-		elif len(self.rd):
-			fo.write('\tsym->rname = "VRT_r_%s(ctx)";\n' % cnam)
-			varproto("VCL_" + self.typ + " VRT_r_%s(VRT_CTX)" % cnam)
-		fo.write("\tsym->r_methods =\n")
-		restrict(fo, self.rd)
-		fo.write(";\n")
-
-		if self.typ == "HEADER":
-			fo.write('\tsym->lname = "HDR_')
-			fo.write(self.nam.split(".")[0].upper())
-			fo.write('";\n')
-		elif len(self.wr):
-			fo.write('\tsym->lname = "VRT_l_%s(ctx, ";\n' % cnam)
-			s = "void VRT_l_%s(VRT_CTX, " % cnam
-			if self.typ != "STRING" and self.typ != "BODY":
-				s += "VCL_" + self.typ + ")"
-			else:
-				s += ctyp.c + ", ...)"
-			varproto(s);
-		fo.write("\tsym->w_methods =\n")
-		restrict(fo, self.wr)
-		fo.write(";\n")
-
-		if len(self.uns):
-			varproto("void VRT_u_%s(VRT_CTX)" % cnam)
-			fo.write('\tsym->uname = "VRT_u_%s(ctx)";\n' % cnam)
-		fo.write('\tsym->u_methods =\n')
-		restrict(fo, self.uns)
-		fo.write(";\n")
+    def __init__(self, nam, typ, rd, wr, wu, vlo, vhi):
+        self.nam = nam
+        self.typ = typ
+        self.rd = rd
+        self.wr = wr
+        self.uns = wu
+        self.vlo = vlo
+        self.vhi = vhi
+
+        self.emit()
+
+    def emit(self):
+        fh.write("\n")
+        fo.write("\n")
+        cnam = self.nam.replace(".", "_")
+        ctyp = vcltypes[self.typ]
+
+        # fo.write("\t{ \"%s\", %s,\n" % (nm, self.typ))
+        fo.write("\tsym = VCC_MkSym(tl, \"%s\", " % self.nam)
+        if self.typ == "HEADER":
+            fo.write(" SYM_NONE, %d, %d);\n" % (self.vlo, self.vhi))
+            fo.write("\tAN(sym);\n")
+            fo.write("\tsym->wildcard = vcc_Var_Wildcard;\n")
+        else:
+            fo.write(" SYM_VAR, %d, %d);\n" % (self.vlo, self.vhi))
+        fo.write("\tAN(sym);\n")
+        fo.write("\tsym->type = %s;\n" % self.typ)
+        fo.write("\tsym->eval = vcc_Eval_Var;\n")
+
+        if self.typ == "HEADER":
+            fo.write('\tsym->rname = "HDR_')
+            fo.write(self.nam.split(".")[0].upper())
+            fo.write('";\n')
+        elif self.rd:
+            fo.write('\tsym->rname = "VRT_r_%s(ctx)";\n' % cnam)
+            varproto("VCL_" + self.typ + " VRT_r_%s(VRT_CTX)" % cnam)
+        fo.write("\tsym->r_methods =\n")
+        restrict(fo, self.rd)
+        fo.write(";\n")
+
+        if self.typ == "HEADER":
+            fo.write('\tsym->lname = "HDR_')
+            fo.write(self.nam.split(".")[0].upper())
+            fo.write('";\n')
+        elif self.wr:
+            fo.write('\tsym->lname = "VRT_l_%s(ctx, ";\n' % cnam)
+            s = "void VRT_l_%s(VRT_CTX, " % cnam
+            if self.typ != "STRING" and self.typ != "BODY":
+                s += "VCL_" + self.typ + ")"
+            else:
+                s += ctyp.c + ", ...)"
+            varproto(s)
+        fo.write("\tsym->w_methods =\n")
+        restrict(fo, self.wr)
+        fo.write(";\n")
+
+        if self.uns:
+            varproto("void VRT_u_%s(VRT_CTX)" % cnam)
+            fo.write('\tsym->uname = "VRT_u_%s(ctx)";\n' % cnam)
+        fo.write('\tsym->u_methods =\n')
+        restrict(fo, self.uns)
+        fo.write(";\n")
 
 def parse_vcl(x):
-	vlo,vhi = (0,99)
-	x = x.split()
-	if x[0] == "VCL" and x[1] == "<=":
-		vhi = int(float(x[2]) * 10)
-	elif x[0] == "VCL" and x[1] == ">=":
-		vlo = int(float(x[2]) * 10)
-	else:
-		print("Unknown variable version spec")
-		print("XXX", x, vlo, vhi)
-		exit(2)
-	return vlo,vhi
+    vlo, vhi = (0, 99)
+    x = x.split()
+    if x[0] == "VCL" and x[1] == "<=":
+        vhi = int(float(x[2]) * 10)
+    elif x[0] == "VCL" and x[1] == ">=":
+        vlo = int(float(x[2]) * 10)
+    else:
+        print("Unknown variable version spec")
+        print("XXX", x, vlo, vhi)
+        exit(2)
+    return vlo, vhi
 
 def parse_var(ln):
-	l1 = ln.pop(0).split("``")
-	assert len(l1) in (1,3)
-	vn = l1[0].strip()
-	if vn[-1] == '*':
-		vn = vn[:-1]
-	if len(l1) == 3:
-		vlo,vhi = parse_vcl(l1[1])
-	else:
-		vlo,vhi = 0,99
-	vr = []
-	vw = []
-	vu = []
-	while True:
-		l = ln.pop(0)
-		if l == "":
-			continue
-		j = l.split()
-		if j[0] == "Type:":
-			assert len(j) == 2
-			vt = j[1]
-			continue
-		if j[0] == "Readable" and j[1] == "from:":
-			for i in j[2:]:
-				vr.append(i.strip(",."))
-			continue
-		if j[0] == "Writable" and j[1] == "from:":
-			for i in j[2:]:
-				vw.append(i.strip(",."))
-			continue
-		if j[0] == "Unsetable" and j[1] == "from:":
-			for i in j[2:]:
-				vu.append(i.strip(",."))
-			continue
-		break
-	if vn[:8] != "storage.":
-		vardef(vn, vt, vr, vw, vu, vlo, vhi)
+    l1 = ln.pop(0).split("``")
+    assert len(l1) in (1, 3)
+    vn = l1[0].strip()
+    if vn[-1] == '*':
+        vn = vn[:-1]
+    if len(l1) == 3:
+        vlo, vhi = parse_vcl(l1[1])
+    else:
+        vlo, vhi = 0, 99
+    vr = []
+    vw = []
+    vu = []
+    while True:
+        l = ln.pop(0)
+        if l == "":
+            continue
+        j = l.split()
+        if j[0] == "Type:":
+            assert len(j) == 2
+            vt = j[1]
+            continue
+        if j[0] == "Readable" and j[1] == "from:":
+            for i in j[2:]:
+                vr.append(i.strip(",."))
+            continue
+        if j[0] == "Writable" and j[1] == "from:":
+            for i in j[2:]:
+                vw.append(i.strip(",."))
+            continue
+        if j[0] == "Unsetable" and j[1] == "from:":
+            for i in j[2:]:
+                vu.append(i.strip(",."))
+            continue
+        break
+    if vn[:8] != "storage.":
+        vardef(vn, vt, vr, vw, vu, vlo, vhi)
 
 def parse_var_doc(fn):
-	s = 0
-	l = []
-	for i in open(fn):
-		l.append(i.rstrip())
-	for n in range(0, len(l)):
-		j = l[n].split()
-		if len(j) != 2 or j[0] != "Type:" or not l[n][0].isspace():
-			continue
-		m = n
-		while m < len(l) and ( l[m] == "" or l[m][0].isspace() ):
-			m += 1
-		parse_var(l[n-2:m-1])
+    l = []
+    for i in open(fn):
+        l.append(i.rstrip())
+    for n in range(0, len(l)):
+        j = l[n].split()
+        if len(j) != 2 or j[0] != "Type:" or not l[n][0].isspace():
+            continue
+        m = n
+        while m < len(l) and (l[m] == "" or l[m][0].isspace()):
+            m += 1
+        parse_var(l[n-2:m-1])
 
 stv_variables = (
-	('free_space',	'BYTES',	"0.", 'storage.<name>.free_space', """
-	Free space available in the named stevedore. Only available for
-	the malloc stevedore.
-	"""),
-	('used_space',	'BYTES',	"0.", 'storage.<name>.used_space', """
-	Used space in the named stevedore. Only available for the malloc
-	stevedore.
-	"""),
-	('happy',	'BOOL',		"0", 'storage.<name>.happy', """
-	Health status for the named stevedore. Not available in any of the
-	current stevedores.
-	"""),
+    ('free_space', 'BYTES', "0.", 'storage.<name>.free_space', """
+    Free space available in the named stevedore. Only available for
+    the malloc stevedore.
+    """),
+    ('used_space', 'BYTES', "0.", 'storage.<name>.used_space', """
+    Used space in the named stevedore. Only available for the malloc
+    stevedore.
+    """),
+    ('happy', 'BOOL', "0", 'storage.<name>.happy', """
+    Health status for the named stevedore. Not available in any of the
+    current stevedores.
+    """),
 )
 
 #######################################################################
@@ -324,11 +319,11 @@ stv_variables = (
 vcltypes = {}
 
 class vcltype(object):
-	def __init__(self, name, ctype, internal=False):
-		self.name = name
-		self.c = ctype
-		self.internal = internal
-		vcltypes[name] = self
+    def __init__(self, name, ctype, internal=False):
+        self.name = name
+        self.c = ctype
+        self.internal = internal
+        vcltypes[name] = self
 
 
 vcltype("STRINGS", "void", True)
@@ -338,19 +333,19 @@ vcltype("SUB", "void*", True)
 fi = open(join(srcroot, "include/vrt.h"))
 
 for i in fi:
-	j = i.split()
-	if len(j) < 3:
-		continue
-	if j[0] != "typedef":
-		continue
-	if j[-1][-1] != ";":
-		continue
-	if j[-1][-2] == ")":
-		continue
-	if j[-1][:4] != "VCL_":
-		continue
-	d = " ".join(j[1:-1])
-	vcltype(j[-1][4:-1], d)
+    j = i.split()
+    if len(j) < 3:
+        continue
+    if j[0] != "typedef":
+        continue
+    if j[-1][-1] != ";":
+        continue
+    if j[-1][-2] == ")":
+        continue
+    if j[-1][:4] != "VCL_":
+        continue
+    d = " ".join(j[1:-1])
+    vcltype(j[-1][4:-1], d)
 fi.close()
 
 #######################################################################
@@ -360,20 +355,20 @@ fi.close()
 
 #######################################################################
 def emit_vcl_fixed_token(fo, tokens):
-	"Emit a function to recognize tokens in a string"
-	recog = list()
-	emit = dict()
-	for i in tokens:
-		j = tokens[i]
-		if j is not None:
-			recog.append(j)
-			emit[j] = i
-
-	recog.sort()
-	rrecog = copy.copy(recog)
-	rrecog.sort(key=lambda x: -len(x))
-
-	fo.write("""
+    "Emit a function to recognize tokens in a string"
+    recog = list()
+    emit = dict()
+    for i in tokens:
+        j = tokens[i]
+        if j is not None:
+            recog.append(j)
+            emit[j] = i
+
+    recog.sort()
+    rrecog = copy.copy(recog)
+    rrecog.sort(key=lambda x: -len(x))
+
+    fo.write("""
 #define M1()\tdo {*q = p + 1; return (p[0]); } while (0)
 #define M2(c,t)\tdo {if (p[1] == (c)) { *q = p + 2; return (t); }} while (0)
 
@@ -383,131 +378,130 @@ vcl_fixed_token(const char *p, const char **q)
 
 \tswitch (p[0]) {
 """)
-	last_initial = None
-	for i in recog:
-		if (i[0] == last_initial):
-			continue
-		last_initial = i[0]
-		fo.write("\tcase '%s':\n" % last_initial)
-		need_ret = True
-		for j in rrecog:
-			if (j[0] != last_initial):
-				continue
-			if len(j) == 2:
-				fo.write("\t\tM2('%s', %s);\n" %
-				    (j[1], emit[j]))
-			elif len(j) == 1:
-				fo.write("\t\tM1();\n")
-				need_ret = False
-			else:
-				fo.write("\t\tif (")
-				k = 1
-				l = len(j)
-				while (k < l):
-					fo.write("p[%d] == '%s'" % (k, j[k]))
-					fo.write(" &&")
-					if (k % 3) == 0:
-						fo.write("\n\t\t    ")
-					else:
-						fo.write(" ")
-					k += 1
-				fo.write("!isvar(p[%d])) {\n" % l)
-				fo.write("\t\t\t*q = p + %d;\n" % l)
-				fo.write("\t\t\treturn (%s);\n" % emit[j])
-				fo.write("\t\t}\n")
-		if need_ret:
-			fo.write("\t\treturn (0);\n")
-	fo.write("\tdefault:\n\t\treturn (0);\n\t}\n}\n")
+    last_initial = None
+    for i in recog:
+        if i[0] == last_initial:
+            continue
+        last_initial = i[0]
+        fo.write("\tcase '%s':\n" % last_initial)
+        need_ret = True
+        for j in rrecog:
+            if j[0] != last_initial:
+                continue
+            if len(j) == 2:
+                fo.write("\t\tM2('%s', %s);\n" % (j[1], emit[j]))
+            elif len(j) == 1:
+                fo.write("\t\tM1();\n")
+                need_ret = False
+            else:
+                fo.write("\t\tif (")
+                k = 1
+                l = len(j)
+                while k < l:
+                    fo.write("p[%d] == '%s'" % (k, j[k]))
+                    fo.write(" &&")
+                    if (k % 3) == 0:
+                        fo.write("\n\t\t    ")
+                    else:
+                        fo.write(" ")
+                    k += 1
+                fo.write("!isvar(p[%d])) {\n" % l)
+                fo.write("\t\t\t*q = p + %d;\n" % l)
+                fo.write("\t\t\treturn (%s);\n" % emit[j])
+                fo.write("\t\t}\n")
+        if need_ret:
+            fo.write("\t\treturn (0);\n")
+    fo.write("\tdefault:\n\t\treturn (0);\n\t}\n}\n")
 
 
 #######################################################################
 def emit_vcl_tnames(fo, tokens):
-	"Emit the vcl_tnames (token->string) conversion array"
-	fo.write("\nconst char * const vcl_tnames[256] = {\n")
-	l = list(tokens.keys())
-	l.sort()
-	for i in l:
-		j = tokens[i]
-		if j is None:
-			j = i
-		if i[0] == "'":
-			j = i
-		fo.write("\t[%s] = \"%s\",\n" % (i, j))
-	fo.write("};\n")
+    "Emit the vcl_tnames (token->string) conversion array"
+    fo.write("\nconst char * const vcl_tnames[256] = {\n")
+    l = list(tokens.keys())
+    l.sort()
+    for i in l:
+        j = tokens[i]
+        if j is None:
+            j = i
+        if i[0] == "'":
+            j = i
+        fo.write("\t[%s] = \"%s\",\n" % (i, j))
+    fo.write("};\n")
 
 
 #######################################################################
 def emit_file(fo, fd, bn):
-	"Read a C-source file and spit out code that outputs it with VSB_cat()"
-	fn = join(fd, bn)
-
-	fi = open(fn)
-	fc = fi.read()
-	fi.close()
-
-	w = 66		# Width of lines, after white space prefix
-	maxlen = 10240  # Max length of string literal
-
-	x = 0
-	l = 0
-	fo.write("\n\t/* %s */\n\n" % fn)
-	fo.write('\tVSB_cat(sb, "/* ---===### %s ###===--- */\\n\\n");\n' % bn)
-	for c in fc:
-		if l == 0:
-			fo.write("\tVSB_cat(sb, \"")
-			l += 12
-			x += 12
-		if x == 0:
-			fo.write("\t    \"")
-		d = c
-		if c == '\n':
-			d = "\\n"
-		elif c == '\t':
-			d = "\\t"
-		elif c == '"':
-			d = "\\\""
-		elif c == '\\':
-			d = "\\\\"
-
-		if c == '\n' and x > w - 20:
-			fo.write(d + "\"\n")
-			x = 0
-			continue
-		if c.isspace() and x > w - 10:
-			fo.write(d + "\"\n")
-			x = 0
-			continue
-
-		fo.write(d)
-		x += len(d)
-		l += len(d)
-		if l > maxlen:
-			fo.write("\");\n")
-			l = 0
-			x = 0
-		if x > w - 3:
-			fo.write("\"\n")
-			x = 0
-	if x != 0:
-		fo.write("\"\n")
-	if l != 0:
-		fo.write("\t);\n")
-	fo.write('\tVSB_cat(sb, "\\n");\n')
+    "Read a C-source file and spit out code that outputs it with VSB_cat()"
+    fn = join(fd, bn)
+
+    fi = open(fn)
+    fc = fi.read()
+    fi.close()
+
+    w = 66      # Width of lines, after white space prefix
+    maxlen = 10240  # Max length of string literal
+
+    x = 0
+    l = 0
+    fo.write("\n\t/* %s */\n\n" % fn)
+    fo.write('\tVSB_cat(sb, "/* ---===### %s ###===--- */\\n\\n");\n' % bn)
+    for c in fc:
+        if l == 0:
+            fo.write("\tVSB_cat(sb, \"")
+            l += 12
+            x += 12
+        if x == 0:
+            fo.write("\t    \"")
+        d = c
+        if c == '\n':
+            d = "\\n"
+        elif c == '\t':
+            d = "\\t"
+        elif c == '"':
+            d = "\\\""
+        elif c == '\\':
+            d = "\\\\"
+
+        if c == '\n' and x > w - 20:
+            fo.write(d + "\"\n")
+            x = 0
+            continue
+        if c.isspace() and x > w - 10:
+            fo.write(d + "\"\n")
+            x = 0
+            continue
+
+        fo.write(d)
+        x += len(d)
+        l += len(d)
+        if l > maxlen:
+            fo.write("\");\n")
+            l = 0
+            x = 0
+        if x > w - 3:
+            fo.write("\"\n")
+            x = 0
+    if x != 0:
+        fo.write("\"\n")
+    if l != 0:
+        fo.write("\t);\n")
+    fo.write('\tVSB_cat(sb, "\\n");\n')
 
 #######################################################################
 
 
 def polish_tokens(tokens):
-	"Expand single char tokens"
-	st = tokens[None]
-	del tokens[None]
-	for i in st:
-		tokens["'" + i + "'"] = i
+    "Expand single char tokens"
+    st = tokens[None]
+    del tokens[None]
+    for i in st:
+        tokens["'" + i + "'"] = i
 
 
 #######################################################################
 def file_header(fo):
-	fo.write("""/*
+    fo.write("""/*
  * NB:  This file is machine generated, DO NOT EDIT!
  *
  * Edit and run lib/libvcc/generate.py instead.
@@ -516,10 +510,10 @@ def file_header(fo):
 """)
 
 def lint_start(fo):
-	fo.write('/*lint -save -e525 -e539 */\n\n')
+    fo.write('/*lint -save -e525 -e539 */\n\n')
 
 def lint_end(fo):
-	fo.write('\n/*lint -restore */\n')
+    fo.write('\n/*lint -restore */\n')
 
 #######################################################################
 
@@ -531,11 +525,11 @@ file_header(fo)
 
 j = 128
 for i in sorted(tokens.keys()):
-	if i[0] == "'":
-		continue
-	fo.write("#define\t%s %d\n" % (i, j))
-	j += 1
-	assert j < 256
+    if i[0] == "'":
+        continue
+    fo.write("#define\t%s %d\n" % (i, j))
+    j += 1
+    assert j < 256
 
 fo.close()
 
@@ -546,14 +540,14 @@ vcls = list()
 vcls_client = list()
 vcls_backend = list()
 for i in returns:
-	vcls.append(i[0])
-	for j in i[1]:
-		if j == "B":
-			vcls_backend.append(i[0])
-		elif j == "C":
-			vcls_client.append(i[0])
-	for j in i[2]:
-		rets[j] = True
+    vcls.append(i[0])
+    for j in i[1]:
+        if j == "B":
+            vcls_backend.append(i[0])
+        elif j == "C":
+            vcls_client.append(i[0])
+    for j in i[2]:
+        rets[j] = True
 
 #######################################################################
 
@@ -566,25 +560,25 @@ lint_start(fo)
 fo.write("#ifdef VCL_RET_MAC\n")
 ll = sorted(returns)
 for i in sorted(rets.keys()):
-	fo.write("VCL_RET_MAC(%s, %s" % (i.lower(), i.upper()))
-	s = ",\n\t"
-	for j in ll:
-		if i in j[2]:
-			fo.write("%sVCL_MET_%s" % (s, j[0].upper()))
-			s = " |\n\t"
-	fo.write("\n)\n\n")
+    fo.write("VCL_RET_MAC(%s, %s" % (i.lower(), i.upper()))
+    s = ",\n\t"
+    for j in ll:
+        if i in j[2]:
+            fo.write("%sVCL_MET_%s" % (s, j[0].upper()))
+            s = " |\n\t"
+    fo.write("\n)\n\n")
 fo.write("#undef VCL_RET_MAC\n")
 fo.write("#endif\n")
 
 fo.write("\n#ifdef VCL_MET_MAC\n")
 for i in ll:
-	fo.write("VCL_MET_MAC(%s, %s, %s," %
-	    (i[0].lower(), i[0].upper(), i[1]))
-	p = " (\n\t"
-	for j in sorted(i[2]):
-		fo.write("%s(1U << VCL_RET_%s)" % (p, j.upper()))
-		p = " |\n\t"
-	fo.write(")\n)\n\n")
+    fo.write("VCL_MET_MAC(%s, %s, %s," %
+             (i[0].lower(), i[0].upper(), i[1]))
+    p = " (\n\t"
+    for j in sorted(i[2]):
+        fo.write("%s(1U << VCL_RET_%s)" % (p, j.upper()))
+        p = " |\n\t"
+    fo.write(")\n)\n\n")
 fo.write("#undef VCL_MET_MAC\n")
 fo.write("#endif\n")
 lint_end(fo)
@@ -609,40 +603,40 @@ fo.write("""
 
 
 def tbl40(a, b):
-	while len(a.expandtabs()) < 40:
-		a += "\t"
-	return a + b
+    while len(a.expandtabs()) < 40:
+        a += "\t"
+    return a + b
 
 fo.write("\n/* VCL Methods */\n")
 task = {}
 n = 1
 for i in returns:
-	fo.write(tbl40("#define VCL_MET_%s" % i[0].upper(), "(1U << %d)\n" % n))
-	if not i[1] in task:
-		task[i[1]] = []
-	task[i[1]].append("VCL_MET_" + i[0].upper())
-	n += 1
+    fo.write(tbl40("#define VCL_MET_%s" % i[0].upper(), "(1U << %d)\n" % n))
+    if not i[1] in task:
+        task[i[1]] = []
+    task[i[1]].append("VCL_MET_" + i[0].upper())
+    n += 1
 
 fo.write("\n" + tbl40("#define VCL_MET_MAX", "%d\n" % n))
 fo.write("\n" + tbl40("#define VCL_MET_MASK", "0x%x\n" % ((1 << n) - 1)))
 
 fo.write("\n")
 for i in sorted(task.keys()):
-	fo.write(tbl40("#define VCL_MET_TASK_%s" % i.upper(),
-		       "( " + (" | \\\n\t\t\t\t\t  ").join(task[i]) + " )\n"))
+    fo.write(tbl40("#define VCL_MET_TASK_%s" % i.upper(),
+                   "( " + (" | \\\n\t\t\t\t\t  ").join(task[i]) + " )\n"))
 
 
 fo.write("\n/* VCL Returns */\n")
 n = 1
 for i in sorted(rets.keys()):
-	fo.write(tbl40("#define VCL_RET_%s" % i.upper(), "%d\n" % n))
-	n += 1
+    fo.write(tbl40("#define VCL_RET_%s" % i.upper(), "%d\n" % n))
+    n += 1
 
 fo.write("\n" + tbl40("#define VCL_RET_MAX", "%d\n" % n))
 
 fo.write("\n/* VCL Types */\n")
 for vcltype in sorted(vcltypes.keys()):
-	fo.write("extern const struct vrt_type VCL_TYPE_%s[1];\n" % vcltype)
+    fo.write("extern const struct vrt_type VCL_TYPE_%s[1];\n" % vcltype)
 
 
 fo.write("""
@@ -653,26 +647,26 @@ typedef void vcl_fini_f(VRT_CTX);
 typedef void vcl_func_f(VRT_CTX);
 
 struct VCL_conf {
-	unsigned			magic;
-#define VCL_CONF_MAGIC			0x7406c509	/* from /dev/random */
+    unsigned            magic;
+#define VCL_CONF_MAGIC          0x7406c509      /* from /dev/random */
 
-	unsigned			syntax;
-	VCL_BACKEND			*default_director;
-	const struct vrt_backend_probe	*default_probe;
-	unsigned			nref;
-	const struct vrt_ref		*ref;
+    unsigned            syntax;
+    VCL_BACKEND             *default_director;
+    const struct vrt_backend_probe  *default_probe;
+    unsigned            nref;
+    const struct vrt_ref        *ref;
 
-	unsigned			nsrc;
-	const char			**srcname;
-	const char			**srcbody;
+    unsigned            nsrc;
+    const char              **srcname;
+    const char              **srcbody;
 
-	unsigned			nvmod;
+    unsigned            nvmod;
 
-	vcl_event_f			*event_vcl;
+    vcl_event_f             *event_vcl;
 """)
 
 for i in returns:
-	fo.write("\tvcl_func_f\t\t\t*" + i[0] + "_func;\n")
+    fo.write("\tvcl_func_f\t\t\t*" + i[0] + "_func;\n")
 
 fo.write("\n};\n")
 fo.close()
@@ -681,44 +675,44 @@ fo.close()
 
 
 def restrict(fo, spec):
-	d = dict()
-	for j in spec:
-		if j[:4] == "vcl_":
-			j = j[4:]
-		if j == 'all':
-			for i in vcls:
-				d[i] = True
-		elif j == 'backend':
-			for i in vcls_backend:
-				d[i] = True
-		elif j == 'client':
-			for i in vcls_client:
-				d[i] = True
-		elif j == 'both':
-			for i in vcls_client:
-				d[i] = True
-			for i in vcls_backend:
-				d[i] = True
-		else:
-			if not j in vcls:
-				print("JJ", j)
-			assert j in vcls
-			d[j] = True
-	p = ""
-	l = list(d.keys())
-	l.sort()
-	w = 0
-	fo.write("\t\t")
-	for j in l:
-		x = p + "VCL_MET_" + j.upper()
-		if w + len(x) > 60:
-			fo.write("\n\t\t")
-			w = 0
-		fo.write(x)
-		w += len(x)
-		p = " | "
-	if len(d) == 0:
-		fo.write("0")
+    d = dict()
+    for j in spec:
+        if j[:4] == "vcl_":
+            j = j[4:]
+        if j == 'all':
+            for i in vcls:
+                d[i] = True
+        elif j == 'backend':
+            for i in vcls_backend:
+                d[i] = True
+        elif j == 'client':
+            for i in vcls_client:
+                d[i] = True
+        elif j == 'both':
+            for i in vcls_client:
+                d[i] = True
+            for i in vcls_backend:
+                d[i] = True
+        else:
+            if not j in vcls:
+                print("JJ", j)
+            assert j in vcls
+            d[j] = True
+    p = ""
+    l = list(d.keys())
+    l.sort()
+    w = 0
+    fo.write("\t\t")
+    for j in l:
+        x = p + "VCL_MET_" + j.upper()
+        if w + len(x) > 60:
+            fo.write("\n\t\t")
+            w = 0
+        fo.write(x)
+        w += len(x)
+        p = " | "
+    if not d:
+        fo.write("0")
 
 #######################################################################
 
@@ -736,7 +730,7 @@ fo.write("""
 void
 vcc_Var_Init(struct vcc *tl)
 {
-	struct symbol *sym;
+    struct symbol *sym;
 """)
 
 
@@ -744,22 +738,22 @@ parse_var_doc(join(srcroot, "doc/sphinx/reference/vcl_var.rst"))
 fo.write("}\n")
 
 for i in stv_variables:
-	fh.write(vcltypes[i[1]].c + " VRT_Stv_" + i[0] + "(const char *);\n")
+    fh.write(vcltypes[i[1]].c + " VRT_Stv_" + i[0] + "(const char *);\n")
 
 fo.write("\n/* VCL type identifiers */\n")
 
 for vn in sorted(vcltypes.keys()):
-	v = vcltypes[vn]
-	if v.internal:
-		continue
-	fo.write("const struct vrt_type VCL_TYPE_%s[1] = { {\n" % v.name)
-	fo.write("\t.magic = VRT_TYPE_MAGIC,\n")
-	fo.write('\t.lname = "%s",\n' % v.name.lower())
-	fo.write('\t.uname = "%s",\n' % v.name)
-	fo.write('\t.ctype = "%s",\n' % v.c)
-	if v.c != "void":
-		fo.write('\t.szof = sizeof(VCL_%s),\n' % v.name)
-	fo.write("}};\n")
+    v = vcltypes[vn]
+    if v.internal:
+        continue
+    fo.write("const struct vrt_type VCL_TYPE_%s[1] = { {\n" % v.name)
+    fo.write("\t.magic = VRT_TYPE_MAGIC,\n")
+    fo.write('\t.lname = "%s",\n' % v.name.lower())
+    fo.write('\t.uname = "%s",\n' % v.name)
+    fo.write('\t.ctype = "%s",\n' % v.c)
+    if v.c != "void":
+        fo.write('\t.szof = sizeof(VCL_%s),\n' % v.name)
+    fo.write("}};\n")
 
 fo.close()
 fh.close()
@@ -800,7 +794,7 @@ file_header(ft)
 lint_start(ft)
 
 for vcltype in sorted(vcltypes.keys()):
-	ft.write("VCC_TYPE(" + vcltype + ")\n")
+    ft.write("VCC_TYPE(" + vcltype + ")\n")
 ft.write("#undef VCC_TYPE\n")
 lint_end(ft)
 ft.close()
@@ -813,10 +807,10 @@ file_header(fo)
 lint_start(fo)
 
 for i in stv_variables:
-	ct = vcltypes[i[1]]
-	fo.write("VRTSTVVAR(" + i[0] + ",\t" + i[1] + ",\t")
-	fo.write(ct.c + ",\t" + i[2] + ")")
-	fo.write("\n")
+    ct = vcltypes[i[1]]
+    fo.write("VRTSTVVAR(" + i[0] + ",\t" + i[1] + ",\t")
+    fo.write(ct.c + ",\t" + i[2] + ")")
+    fo.write("\n")
 
 fo.write("#undef VRTSTVVAR\n")
 lint_end(fo)
diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 25b8cc621..dd157449f 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -41,7 +41,6 @@ import sys
 import re
 import optparse
 import unittest
-import random
 import copy
 import json
 import hashlib
@@ -254,7 +253,7 @@ class CType(object):
     def json(self, jl):
         jl.append([self.vt])
         while jl[-1][-1] is None:
-                jl[-1].pop(-1)
+            jl[-1].pop(-1)
 
 #######################################################################
 
@@ -290,7 +289,7 @@ class arg(CType):
     def json(self, jl):
         jl.append([self.vt, self.nm, self.defval, self.spec])
         if self.opt:
-                jl[-1].append(True)
+            jl[-1].append(True)
         while jl[-1][-1] is None:
             jl[-1].pop(-1)
 
@@ -366,7 +365,7 @@ class ProtoType(object):
             err("%s(): Illegal C-name\n" % self.cname(), warn=False)
 
         if len(wl) == 2 and wl[0] == '(' and wl[1] == ')':
-                return
+            return
 
         if wl[0] != "(":
             err("Syntax error: Expected '(', got '%s'" % wl[0], warn=False)
@@ -386,17 +385,17 @@ class ProtoType(object):
             if not wl:
                 break
             if wl[0] == '[':
-                    wl.pop(0)
-                    t = arg(wl, names, st.vcc.enums, ']')
-                    if t.nm is None:
-                        err("Optional arguments must have names", warn=False)
-                    t.opt = True
-                    x = wl.pop(0)
-                    if x != ']':
-                        err("Expected ']' found '%s'" % x, warn=False)
-                    self.argstruct = True
+                wl.pop(0)
+                t = arg(wl, names, st.vcc.enums, ']')
+                if t.nm is None:
+                    err("Optional arguments must have names", warn=False)
+                t.opt = True
+                x = wl.pop(0)
+                if x != ']':
+                    err("Expected ']' found '%s'" % x, warn=False)
+                self.argstruct = True
             else:
-                    t = arg(wl, names, st.vcc.enums, ',')
+                t = arg(wl, names, st.vcc.enums, ',')
             if t.nm is None:
                 t.nm2 = "arg%d" % n
             else:
@@ -466,10 +465,10 @@ class ProtoType(object):
         s = self.retval.ct + " " + name + '('
         ll = args
         if self.argstruct:
-                ll.append(self.argstructname() + "*")
+            ll.append(self.argstructname() + "*")
         else:
-                for i in self.args:
-                    ll.append(i.ct)
+            for i in self.args:
+                ll.append(i.ct)
         s += ", ".join(ll)
         return s + ');'
 
@@ -483,31 +482,31 @@ class ProtoType(object):
     def argstructure(self):
         s = "\n" + self.argstructname() + " {\n"
         for i in self.args:
-                if i.opt:
-                        assert i.nm is not None
-                        s += "\tchar\t\t\tvalid_%s;\n" % i.nm
+            if i.opt:
+                assert i.nm is not None
+                s += "\tchar\t\t\tvalid_%s;\n" % i.nm
         for i in self.args:
-                s += "\t" + i.ct
-                if len(i.ct) < 8:
-                        s += "\t"
-                if len(i.ct) < 16:
-                        s += "\t"
-                s += "\t" + i.nm2 + ";\n"
+            s += "\t" + i.ct
+            if len(i.ct) < 8:
+                s += "\t"
+            if len(i.ct) < 16:
+                s += "\t"
+            s += "\t" + i.nm2 + ";\n"
         s += "};\n"
         return s
 
     def cstuff(self, args, where):
         s = ""
         if where == 'h':
-                if self.argstruct:
-                        s += self.argstructure()
-                s += lwrap(self.proto(args, self.cname(True)))
+            if self.argstruct:
+                s += self.argstructure()
+            s += lwrap(self.proto(args, self.cname(True)))
         elif where == 'c':
-                s += lwrap(self.typedef(args))
+            s += lwrap(self.typedef(args))
         elif where == 'o':
-                if self.argstruct:
-                        s += self.argstructure()
-                s += lwrap(self.typedef(args))
+            if self.argstruct:
+                s += self.argstructure()
+            s += lwrap(self.typedef(args))
         else:
             assert False
         return s
@@ -517,9 +516,9 @@ class ProtoType(object):
         self.retval.json(ll)
         ll.append('Vmod_%s_Func.%s' % (self.st.vcc.modname, cfunc))
         if self.argstruct:
-                ll.append(self.argstructname())
+            ll.append(self.argstructname())
         else:
-                ll.append("")
+            ll.append("")
         for i in self.args:
             i.json(ll)
         jl.append(ll)
diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py
index a665dff20..6c96388e5 100644
--- a/lib/libvcc/vsctool.py
+++ b/lib/libvcc/vsctool.py
@@ -27,9 +27,13 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
-"""
-This program compiles a .vsc file to C language constructs.
-"""
+'''
+This program compiles a `.vsc` file to `.c`, `.h` and `.rst` formats.
+
+Note: A `.vsc` file is *almost* a `.rst` file, or at last *almost*
+the same general syntax as a `.rst` file, but for now we process
+it with this program to get a *real* `.rst` file.
+'''
 
 from __future__ import print_function
 
@@ -37,385 +41,386 @@ import getopt
 import json
 import sys
 import collections
-import struct
 
-TYPES = [ "counter", "gauge", "bitmap" ]
-CTYPES = [ "uint64_t" ]
-LEVELS = [ "info", "diag", "debug" ]
-FORMATS = [ "integer", "bytes", "bitmap", "duration" ]
+# Parameters of 'varnish_vsc_begin', first element is default
+TYPES = ["counter", "gauge", "bitmap"]
+CTYPES = ["uint64_t"]
+LEVELS = ["info", "diag", "debug"]
+FORMATS = ["integer", "bytes", "bitmap", "duration"]
 
 PARAMS = {
-	"type":		["counter", TYPES],
-	"ctype":	["uint64_t", CTYPES],
-	"level":	["info", LEVELS],
-	"oneliner":	True,
-	"format":	[ "integer", FORMATS],
+    "type": TYPES,
+    "ctype": CTYPES,
+    "level": LEVELS,
+    "oneliner": None,
+    "format": FORMATS,
 }
 
-# http://python3porting.com/problems.html#bytes-strings-and-unicode
-if sys.version_info < (3,):
-	def b(x):
-		return x
-else:
-	import codecs
-	def b(x):
-		return codecs.latin_1_encode(x)[0]
-
 def genhdr(fo, name):
-	fo.write('/*\n')
-	fo.write(' * NB:  This file is machine generated, DO NOT EDIT!\n')
-	fo.write(' *\n')
-	fo.write(' * Edit ' + name +
-	    '.vsc and run lib/libvcc/vsctool.py instead.\n')
-	fo.write(' */\n')
-	fo.write('\n')
 
-#######################################################################
+    '''Emit .[ch] file boiler-plate warning'''
 
-class vscset(object):
-	def __init__(self, name, m):
-		self.name = name
-		self.struct = "struct VSC_" + name
-		self.mbrs = []
-		self.head = m
-		self.completed = False
-		self.off = 0
-
-	def addmbr(self, m):
-		assert not self.completed
-		self.mbrs.append(m)
-		m.param["index"] = self.off
-		self.off += 8
-
-	def complete(self):
-		self.completed = True
-
-	def emit_json(self, fo):
-		dd = collections.OrderedDict()
-		dd["version"] = "1"
-		dd["name"] = self.name
-		dd["oneliner"] = self.head.param["oneliner"].strip()
-		dd["order"] = int(self.head.param["order"])
-		dd["docs"] = "\n".join(self.head.getdoc())
-		dd["elements"] = len(self.mbrs)
-		el = collections.OrderedDict()
-		dd["elem"] = el
-		for i in self.mbrs:
-			ed = collections.OrderedDict()
-			el[i.arg] = ed
-			for j in PARAMS:
-				if j in i.param:
-					ed[j] = i.param[j]
-			ed["index"] = i.param["index"]
-			ed["name"] = i.arg
-			ed["docs"] = "\n".join(i.getdoc())
-		s=json.dumps(dd, separators=(",",":")) + "\0"
-		fo.write("\nstatic const unsigned char")
-		fo.write(" vsc_%s_json[%d] = {\n" % (self.name, len(s)))
-		bz = bytearray(s, encoding="ascii")
-		t = "\t"
-		for i in bz:
-			t += "%d," % i
-			if len(t) >= 69:
-				fo.write(t + "\n")
-				t = "\t"
-		if len(t) > 1:
-			fo.write(t[:-1])
-		fo.write("\n};\n")
-		s = json.dumps(dd, indent=2, separators=(',', ': '))
-		fo.write("\n")
-		for i in s.split("\n"):
-			j = "// " + i
-			if len(j) > 72:
-				fo.write(j[:72] + "[...]\n")
-			else:
-				fo.write(j + "\n")
-		fo.write("\n")
-
-
-	def emit_h(self):
-		fon="VSC_" + self.name + ".h"
-		fo = open(fon, "w")
-		genhdr(fo, self.name)
-		fo.write(self.struct + " {\n")
-		for i in self.mbrs:
-			fo.write("\tuint64_t\t%s;\n" % i.arg)
-		fo.write("};\n")
-		fo.write("\n")
-
-		fo.write("#define VSC_" + self.name +
-		    "_size PRNDUP(sizeof(" + self.struct + "))\n\n")
-
-		fo.write(self.struct + " *VSC_" + self.name + "_New")
-		fo.write("(struct vsmw_cluster *,\n")
-		fo.write("    struct vsc_seg **, const char *fmt, ...);\n")
-
-		fo.write("void VSC_" + self.name + "_Destroy")
-		fo.write("(struct vsc_seg **);\n")
-
-		if 'sumfunction' in self.head.param:
-			fo.write("void VSC_" + self.name + "_Summ")
-			fo.write("(" + self.struct + " *, ")
-			fo.write("const " + self.struct + " *);\n")
-
-	def emit_c(self):
-		fon="VSC_" + self.name + ".c"
-		fo = open(fon, "w")
-		genhdr(fo, self.name)
-		fo.write('#include "config.h"\n')
-		fo.write('#include <stdio.h>\n')
-		fo.write('#include <stdarg.h>\n')
-		fo.write('#include "vdef.h"\n')
-		fo.write('#include "vas.h"\n')
-		fo.write('#include "vrt.h"\n')
-		fo.write('#include "VSC_%s.h"\n' % self.name)
-
-		fo.write("\n")
-		fo.write('static const char vsc_%s_name[] = "%s";\n' %
-		    (self.name, self.name.upper()))
-
-		fo.write("\n")
-		fo.write("#define PARANOIA(a,n)\t\t\t\t\\\n")
-		fo.write("    _Static_assert(\t\t\t\t\\\n")
-		fo.write("\toffsetof(" + self.struct + ", a) == n,\t\\\n")
-		fo.write("\t\"VSC element '\" #a \"' at wrong offset\")\n\n")
-
-		for i in self.mbrs:
-			fo.write("PARANOIA(" + i.arg)
-			fo.write(", %d);\n" % (i.param["index"]))
-
-		fo.write("#undef PARANOIA\n")
-
-		self.emit_json(fo)
-
-		fo.write("\n")
-		fo.write(self.struct + "*\n")
-		fo.write("VSC_" + self.name + "_New")
-		fo.write("(struct vsmw_cluster *vc,\n")
-		fo.write("    struct vsc_seg **sg, const char *fmt, ...)\n")
-		fo.write("{\n")
-		fo.write("\tva_list ap;\n")
-		fo.write("\t" + self.struct + " *retval;\n")
-		fo.write("\n")
-		fo.write("\tva_start(ap, fmt);\n")
-		fo.write("\tretval = VRT_VSC_Alloc")
-		fo.write("(vc, sg, vsc_" + self.name + "_name, ")
-		fo.write("VSC_" + self.name + "_size,\n")
-		fo.write("\t    vsc_" + self.name + "_json, ")
-		fo.write("sizeof vsc_" + self.name + "_json, fmt, ap);\n")
-		fo.write("\tva_end(ap);\n")
-		fo.write("\treturn(retval);\n")
-		fo.write("}\n")
-
-		fo.write("\n")
-		fo.write("void\n")
-		fo.write("VSC_" + self.name + "_Destroy")
-		fo.write("(struct vsc_seg **sg)\n")
-		fo.write("{\n")
-		fo.write("\tstruct vsc_seg *p;\n")
-		fo.write("\n")
-		fo.write("\tAN(sg);\n")
-		fo.write("\tp = *sg;\n")
-		fo.write("\t*sg = NULL;\n")
-		fo.write('\tVRT_VSC_Destroy(vsc_%s_name, p);\n' % self.name)
-		fo.write("}\n")
-
-		if 'sumfunction' in self.head.param:
-			fo.write("\n")
-			fo.write("void\n")
-			fo.write("VSC_" + self.name + "_Summ")
-			fo.write("(" + self.struct + " *dst, ")
-			fo.write("const " + self.struct + " *src)\n")
-			fo.write("{\n")
-			fo.write("\n")
-			fo.write("\tAN(dst);\n")
-			fo.write("\tAN(src);\n")
-			for i in self.mbrs:
-				s1 = "\tdst->" + i.arg + " +="
-				s2 = "src->" + i.arg + ";"
-				if len((s1 + " " + s2).expandtabs()) < 79:
-					fo.write(s1 + " " + s2 + "\n")
-				else:
-					fo.write(s1 + "\n\t    " + s2 + "\n")
-			fo.write("}\n")
+    fo.write('/*\n')
+    fo.write(' * NB:  This file is machine generated, DO NOT EDIT!\n')
+    fo.write(' *\n')
+    fo.write(' * Edit ' + name +
+             '.vsc and run lib/libvcc/vsctool.py instead.\n')
+    fo.write(' */\n')
+    fo.write('\n')
 
 #######################################################################
 
-class directive(object):
-	def __init__(self, s):
-		ll = s.split("\n")
-		i = ll.pop(0).split("::", 2)
-		self.cmd = i[0]
-		self.arg = i[1].strip()
-		assert len(self.arg.split()) == 1
-
-		self.param = {}
-		while len(ll):
-			j = ll[0].split(":",2)
-			if len(j) != 3 or not j[0].isspace():
-				break
-			self.param[j[1]] = j[2].strip()
-			ll.pop(0)
-		self.ldoc = ll
-
-	def getdoc(self):
-		while len(self.ldoc) and self.ldoc[0].strip() == "":
-			self.ldoc.pop(0)
-		while len(self.ldoc) and self.ldoc[-1].strip() == "":
-			self.ldoc.pop(-1)
-		return self.ldoc
-
-	def moredoc(self, s):
-		self.getdoc()
-		self.ldoc += s.split("\n")
-
-	def emit_rst(self, fo):
-		fo.write("\n.. " + self.cmd + ":: " + self.arg + "\n")
-		self.emit_rst_doc(fo)
-
-	def emit_rst_doc(self, fo):
-		fo.write("\n".join(self.ldoc))
-
-	def emit_h(self, fo):
-		return
-
-class rst_vsc_begin(directive):
-	def __init__(self, s):
-		super(rst_vsc_begin, self).__init__(s)
-
-	def vscset(self, ss):
-		ss.append(vscset(self.arg, self))
-
-	def emit_rst(self, fo):
-		fo.write("\n..\n\t" + self.cmd + ":: " + self.arg + "\n")
-
-		s = self.arg.upper() + " ? " + self.param["oneliner"]
-		fo.write("\n")
-		fo.write(s + "\n")
-		fo.write("=" * len(s) + "\n")
-
-		self.emit_rst_doc(fo)
-
-class rst_vsc(directive):
-	def __init__(self, s):
-		super(rst_vsc, self).__init__(s)
-
-		for i,v in PARAMS.items():
-			if v is not True:
-				self.do_default(i, v[0], v[1])
-
-		for p in self.param:
-			if p in PARAMS:
-				continue
-			sys.stderr.write("Unknown parameter ")
-			sys.stderr.write("'" + p + "'")
-			sys.stderr.write(" on field '" + self.arg + "'\n")
-			exit(2)
-
-	def do_default(self, p, v, s):
-		if p not in self.param:
-			self.param[p] = v
-		if self.param[p] not in s:
-			sys.stderr.write("Wrong " + p + " '" + self.param[p])
-			sys.stderr.write("' on field '" + self.arg + "'\n")
-			exit(2)
-
-
-	def emit_rst(self, fo):
-		fo.write("\n``%s`` ? " % self.arg)
-		fo.write("`%s` - " % self.param["type"])
-		fo.write("%s\n\n" % self.param["level"])
-
-		fo.write("\t" + self.param["oneliner"] + "\n")
-		self.emit_rst_doc(fo)
-
-	def vscset(self, ss):
-		ss[-1].addmbr(self)
-
-
-class rst_vsc_end(directive):
-	def __init__(self, s):
-		super(rst_vsc_end, self).__init__(s)
-
-	def vscset(self, ss):
-		ss[-1].complete()
-
-	def emit_rst(self, fo):
-		fo.write("\n..\n\t" + self.cmd + ":: " + self.arg + "\n")
-		self.emit_rst_doc(fo)
-
-class other(object):
-	def __init__(self, s):
-		self.s = s
-
-	def emit_rst(self, fo):
-		fo.write(self.s)
-
-	def emit_h(self, fo):
-		return
-
-	def vscset(self, ss):
-		return
+class CounterSet(object):
+
+    '''
+        A set of counters
+
+        In the `.vsc` file a CounterSet is everything between a
+
+            .. varnish_vsc_begin::
+
+        and the subsequent
+
+            .. varnish_vsc_end::
+    '''
+
+    def __init__(self, name, m):
+        self.name = name
+        self.struct = "struct VSC_" + name
+        self.mbrs = []
+        self.head = m
+        self.completed = False
+        self.off = 0
+
+    def addmbr(self, m):
+        '''Add a counter'''
+        assert not self.completed
+        self.mbrs.append(m)
+        retval = self.off
+        self.off += 8
+        return retval
+
+    def complete(self, arg):
+        '''Mark set completed'''
+        assert arg == self.name
+        self.completed = True
+
+    def emit_json(self, fo):
+        '''Emit JSON as compact C byte-array and as readable C-comments'''
+        assert self.completed
+        dd = collections.OrderedDict()
+        dd["version"] = "1"
+        dd["name"] = self.name
+        dd["oneliner"] = self.head.param["oneliner"].strip()
+        dd["order"] = int(self.head.param["order"])
+        dd["docs"] = "\n".join(self.head.getdoc())
+        dd["elements"] = len(self.mbrs)
+        el = collections.OrderedDict()
+        dd["elem"] = el
+        for i in self.mbrs:
+            ed = collections.OrderedDict()
+            el[i.arg] = ed
+            for j in PARAMS:
+                if j in i.param:
+                    ed[j] = i.param[j]
+            ed["index"] = i.param["index"]
+            ed["name"] = i.arg
+            ed["docs"] = "\n".join(i.getdoc())
+        s = json.dumps(dd, separators=(",", ":")) + "\0"
+        fo.write("\nstatic const unsigned char")
+        fo.write(" vsc_%s_json[%d] = {\n" % (self.name, len(s)))
+        bz = bytearray(s, encoding="ascii")
+        t = "\t"
+        for i in bz:
+            t += "%d," % i
+            if len(t) >= 69:
+                fo.write(t + "\n")
+                t = "\t"
+        if len(t) > 1:
+            fo.write(t[:-1])
+        fo.write("\n};\n")
+        s = json.dumps(dd, indent=2, separators=(',', ': '))
+        fo.write("\n")
+        for i in s.split("\n"):
+            j = "// " + i
+            if len(j) > 72:
+                fo.write(j[:72] + "[...]\n")
+            else:
+                fo.write(j + "\n")
+        fo.write("\n")
+
+    def emit_h(self):
+        '''Emit .h file'''
+        assert self.completed
+        fon = "VSC_" + self.name + ".h"
+        fo = open(fon, "w")
+        genhdr(fo, self.name)
+        fo.write(self.struct + " {\n")
+        for i in self.mbrs:
+            fo.write("\tuint64_t\t%s;\n" % i.arg)
+        fo.write("};\n")
+        fo.write("\n")
+
+        fo.write("#define VSC_" + self.name +
+                 "_size PRNDUP(sizeof(" + self.struct + "))\n\n")
+
+        fo.write(self.struct + " *VSC_" + self.name + "_New")
+        fo.write("(struct vsmw_cluster *,\n")
+        fo.write("    struct vsc_seg **, const char *fmt, ...);\n")
+
+        fo.write("void VSC_" + self.name + "_Destroy")
+        fo.write("(struct vsc_seg **);\n")
+
+        if 'sumfunction' in self.head.param:
+            fo.write("void VSC_" + self.name + "_Summ")
+            fo.write("(" + self.struct + " *, ")
+            fo.write("const " + self.struct + " *);\n")
+
+    def emit_c_paranoia(self, fo):
+        '''Emit asserts to make sure compiler gets same byte index'''
+        fo.write("#define PARANOIA(a,n)\t\t\t\t\\\n")
+        fo.write("    _Static_assert(\t\t\t\t\\\n")
+        fo.write("\toffsetof(" + self.struct + ", a) == n,\t\\\n")
+        fo.write("\t\"VSC element '\" #a \"' at wrong offset\")\n\n")
+
+        for i in self.mbrs:
+            fo.write("PARANOIA(" + i.arg)
+            fo.write(", %d);\n" % (i.param["index"]))
+
+        fo.write("#undef PARANOIA\n")
+
+    def emit_c_sumfunc(self, fo):
+        '''Emit a function summ up countersets'''
+        fo.write("\n")
+        fo.write("void\n")
+        fo.write("VSC_" + self.name + "_Summ")
+        fo.write("(" + self.struct + " *dst, ")
+        fo.write("const " + self.struct + " *src)\n")
+        fo.write("{\n")
+        fo.write("\n")
+        fo.write("\tAN(dst);\n")
+        fo.write("\tAN(src);\n")
+        for i in self.mbrs:
+            s1 = "\tdst->" + i.arg + " +="
+            s2 = "src->" + i.arg + ";"
+            if len((s1 + " " + s2).expandtabs()) < 79:
+                fo.write(s1 + " " + s2 + "\n")
+            else:
+                fo.write(s1 + "\n\t    " + s2 + "\n")
+        fo.write("}\n")
+
+    def emit_c_newfunc(self, fo):
+        '''Emit New function'''
+        fo.write("\n")
+        fo.write(self.struct + "*\n")
+        fo.write("VSC_" + self.name + "_New")
+        fo.write("(struct vsmw_cluster *vc,\n")
+        fo.write("    struct vsc_seg **sg, const char *fmt, ...)\n")
+        fo.write("{\n")
+        fo.write("\tva_list ap;\n")
+        fo.write("\t" + self.struct + " *retval;\n")
+        fo.write("\n")
+        fo.write("\tva_start(ap, fmt);\n")
+        fo.write("\tretval = VRT_VSC_Alloc")
+        fo.write("(vc, sg, vsc_" + self.name + "_name, ")
+        fo.write("VSC_" + self.name + "_size,\n")
+        fo.write("\t    vsc_" + self.name + "_json, ")
+        fo.write("sizeof vsc_" + self.name + "_json, fmt, ap);\n")
+        fo.write("\tva_end(ap);\n")
+        fo.write("\treturn(retval);\n")
+        fo.write("}\n")
+
+    def emit_c_destroyfunc(self, fo):
+        '''Emit Destroy function'''
+        fo.write("\n")
+        fo.write("void\n")
+        fo.write("VSC_" + self.name + "_Destroy")
+        fo.write("(struct vsc_seg **sg)\n")
+        fo.write("{\n")
+        fo.write("\tstruct vsc_seg *p;\n")
+        fo.write("\n")
+        fo.write("\tAN(sg);\n")
+        fo.write("\tp = *sg;\n")
+        fo.write("\t*sg = NULL;\n")
+        fo.write('\tVRT_VSC_Destroy(vsc_%s_name, p);\n' % self.name)
+        fo.write("}\n")
+
+    def emit_c(self):
+        '''Emit .c file'''
+        assert self.completed
+        fon = "VSC_" + self.name + ".c"
+        fo = open(fon, "w")
+        genhdr(fo, self.name)
+        fo.write('#include "config.h"\n')
+        fo.write('#include <stdio.h>\n')
+        fo.write('#include <stdarg.h>\n')
+        fo.write('#include "vdef.h"\n')
+        fo.write('#include "vas.h"\n')
+        fo.write('#include "vrt.h"\n')
+        fo.write('#include "VSC_%s.h"\n' % self.name)
+
+        fo.write("\n")
+        fo.write('static const char vsc_%s_name[] = "%s";\n' %
+                 (self.name, self.name.upper()))
+
+        fo.write("\n")
+
+        self.emit_c_paranoia(fo)
+        self.emit_json(fo)
+        self.emit_c_newfunc(fo)
+        self.emit_c_destroyfunc(fo)
+        if 'sumfunction' in self.head.param:
+            self.emit_c_sumfunc(fo)
 
 #######################################################################
 
-class vsc_file(object):
-	def __init__(self, fin):
-		self.c = []
-		scs = open(fin).read().split("\n.. ")
-		self.c.append(other(scs[0]))
-		ld = None
-		for i in scs[1:]:
-			j = i.split(None, 1)
-			f = {
-				"varnish_vsc_begin::":	rst_vsc_begin,
-				"varnish_vsc::":	rst_vsc,
-				"varnish_vsc_end::":	rst_vsc_end,
-			}.get(j[0])
-			if f is None:
-				s = "\n.. " + i
-				o = other(s)
-				if ld is not None:
-					ld.moredoc(s)
-			else:
-				o = f(i)
-				ld = o
-			self.c.append(o)
-
-		self.vscset = []
-		for i in self.c:
-			i.vscset(self.vscset)
-
-	def emit_h(self):
-		for i in self.vscset:
-			i.emit_h()
-
-	def emit_c(self):
-		for i in self.vscset:
-			i.emit_c()
-
-	def emit_rst(self):
-		fo = sys.stdout
-		for i in self.c:
-			i.emit_rst(fo)
+class OurDirective(object):
+
+    '''
+        One of our `.. blablabla::` directives in the source file
+    '''
+
+    def __init__(self, s):
+        ll = s.split("\n")
+        i = ll.pop(0).split("::", 2)
+        self.cmd = i[0]
+        self.arg = i[1].strip()
+        assert len(self.arg.split()) == 1
+
+        self.param = {}
+        while ll:
+            j = ll[0].split(":", 2)
+            if len(j) != 3 or not j[0].isspace():
+                break
+            self.param[j[1]] = j[2].strip()
+            ll.pop(0)
+        self.ldoc = ll
+
+    def getdoc(self):
+        '''
+        Get docs for JSON
+
+        Note that these docs end with the first '\n.. ' sequence
+        in the .vsc file, so that we can put a longer and more
+        complex description into the .RST docs than the "long"
+        explanation varnishstat(1) and similar programs provide.
+        '''
+        while self.ldoc and self.ldoc[0].strip() == "":
+            self.ldoc.pop(0)
+        while self.ldoc and self.ldoc[-1].strip() == "":
+            self.ldoc.pop(-1)
+        return self.ldoc
+
+    def emit_rst(self, fo):
+        '''Emit the documentation as .rst'''
+        assert False
+
+class RstVscDirectiveBegin(OurDirective):
+
+    '''
+        `varnish_vsc_begin` directive
+    '''
+
+    def __init__(self, s, vsc_set, fo):
+        super(RstVscDirectiveBegin, self).__init__(s)
+        vsc_set.append(CounterSet(self.arg, self))
+        if fo:
+            fo.write("\n..\n\t" + self.cmd + ":: " + self.arg + "\n")
+
+            s = self.arg.upper() + " ? " + self.param["oneliner"]
+            fo.write("\n")
+            fo.write(s + "\n")
+            fo.write("=" * len(s) + "\n")
+            fo.write("\n".join(self.ldoc))
+
+class RstVscDirective(OurDirective):
+
+    '''
+        `varnish_vsc` directive - one counter
+    '''
+
+    def __init__(self, s, vsc_set, fo):
+        assert not vsc_set or vsc_set[-1].complete
+        super(RstVscDirective, self).__init__(s)
+
+        for i, v in PARAMS.items():
+            if v is not None:
+                if i not in self.param:
+                    self.param[i] = v[0]
+                if self.param[i] not in v:
+                    sys.stderr.write("Wrong " + i + " '" + self.param[i])
+                    sys.stderr.write("' on field '" + self.arg + "'\n")
+                    exit(2)
+
+        for p in self.param:
+            if p in PARAMS:
+                continue
+            sys.stderr.write("Unknown parameter ")
+            sys.stderr.write("'" + p + "'")
+            sys.stderr.write(" on field '" + self.arg + "'\n")
+            exit(2)
+        self.param["index"] = vsc_set[-1].addmbr(self)
+        if fo:
+            fo.write("\n``%s`` ? " % self.arg)
+            fo.write("`%s` - " % self.param["type"])
+            fo.write("%s\n\n" % self.param["level"])
+
+            fo.write("\t" + self.param["oneliner"] + "\n")
+            fo.write("\n".join(self.ldoc))
+
+class RstVscDirectiveEnd(OurDirective):
+
+    '''
+        `varnish_vsc_end` directive
+    '''
+
+    def __init__(self, s, vsc_set, fo):
+        super(RstVscDirectiveEnd, self).__init__(s)
+        vsc_set[-1].complete(self.arg)
+        if fo:
+            fo.write("\n..\n\t" + self.cmd + ":: " + self.arg + "\n")
+            fo.write("\n".join(self.ldoc))
 
 #######################################################################
 
-if __name__ == "__main__":
-
-	optlist, args = getopt.getopt(sys.argv[1:], "chr")
+def mainfunc(argv):
+
+    '''Process a .vsc file'''
+
+    optlist, args = getopt.getopt(argv[1:], "chr")
+
+    if len(args) != 1:
+        sys.stderr.write("Need exactly one filename argument\n")
+        exit(2)
+
+    rstfile = None
+    for f, v in optlist:
+        if f == '-r':
+            rstfile = sys.stdout
+
+    vscset = []
+    scs = open(args[0]).read().split("\n.. ")
+    if rstfile:
+        rstfile.write(scs[0])
+    for i in scs[1:]:
+        j = i.split(None, 1)
+        f = {
+            "varnish_vsc_begin::":  RstVscDirectiveBegin,
+            "varnish_vsc::":        RstVscDirective,
+            "varnish_vsc_end::":    RstVscDirectiveEnd,
+        }.get(j[0])
+        if f is not None:
+            f(i, vscset, rstfile)
+        elif rstfile:
+            rstfile.write("\n.. " + i)
+
+    for i in vscset:
+        for f, v in optlist:
+            if f == '-h':
+                i.emit_h()
+            if f == '-c':
+                i.emit_c()
 
-	fo = sys.stdout
-
-	if len(args) != 1:
-		sys.stderr.write("Need exactly one filename argument\n")
-		exit(2)
+if __name__ == "__main__":
 
-	vf = vsc_file(args[0])
-	for f,v in optlist:
-		if f == '-r':
-			vf.emit_rst()
-		if f == '-h':
-			vf.emit_h()
-		if f == '-c':
-			vf.emit_c()
+    mainfunc(sys.argv)
diff --git a/tools/gcov_digest.py b/tools/gcov_digest.py
index 138f7e8cf..188fb499d 100644
--- a/tools/gcov_digest.py
+++ b/tools/gcov_digest.py
@@ -32,19 +32,19 @@ found in a subdirectory tree.
 
 Options:
 
-	-g gcov-program
-		default:gcov6
+    -g gcov-program
+        default:gcov6
 
-	-o output-filename
-		default: stdout
+    -o output-filename
+        default: stdout
 
-	-x exclude-subdir
-		default ".git" and ".deps"
+    -x exclude-subdir
+        default ".git" and ".deps"
 
  Arguments:
 
-	directories to process.
-	default: .
+    directories to process.
+    default: .
 
 """
 
@@ -61,141 +61,141 @@ lengths = {}
 exclude = [".git", ".deps",]
 
 def process_gcov(fn, sn):
-	""" Sum .gcov file into counts, then delete it """
-	dd = counts.get(sn)
-	if dd is None:
-		dd = {}
-	for ln in open(fn):
-		d = ln.split(":")
-		cnt = d[0].strip()
-		ll = d[1]
-		if cnt == "-":
-			continue
-		if cnt == "#####":
-			cnt = 0
-		else:
-			cnt = int(cnt)
-		lno = int(d[1])
-		if lno not in dd:
-			dd[lno] = 0
-		dd[lno] += cnt
-	counts[sn] = dd
-	pl = lengths.get(sn)
-	ll = ll.strip()
-	if d[2] == "/*EOF*/\n":
-		ll = pl
-	elif pl != ll and not pl is None:
-		print("CONFLICT", fn, ll, pl)
-		ll = "-1"
-	lengths[sn] = ll
-	os.remove(fn)
+    """ Sum .gcov file into counts, then delete it """
+    dd = counts.get(sn)
+    if dd is None:
+        dd = {}
+    for ln in open(fn):
+        d = ln.split(":")
+        cnt = d[0].strip()
+        ll = d[1]
+        if cnt == "-":
+            continue
+        if cnt == "#####":
+            cnt = 0
+        else:
+            cnt = int(cnt)
+        lno = int(d[1])
+        if lno not in dd:
+            dd[lno] = 0
+        dd[lno] += cnt
+    counts[sn] = dd
+    pl = lengths.get(sn)
+    ll = ll.strip()
+    if d[2] == "/*EOF*/\n":
+        ll = pl
+    elif pl != ll and not pl is None:
+        print("CONFLICT", fn, ll, pl)
+        ll = "-1"
+    lengths[sn] = ll
+    os.remove(fn)
 
 def run_gcov(prog, subdir):
-	""" Walk tree, run gcov, process output """
-	for root, dirs, files in os.walk(subdir):
-		for i in exclude:
-			if i in dirs:
-				dirs.remove(i)
-		if " ".join(files).find(".gcda") == -1:
-			continue
-		for fn in files:
-			if fn[-2:] != ".o":
-				continue
-
-			# if we find the .o file in a .../.libs the sources
-			# must be found relative to the parent directory
-
-			if root[-6:] == "/.libs":
-				x = subprocess.check_output(
-				    ["cd " + root + "/.. && " +
-				     "exec " + prog + " -r .libs/" + fn],
-				    stderr=subprocess.STDOUT, shell=True,
-				    universal_newlines=True)
-				pf = ".."
-			else:
-				x = subprocess.check_output(
-				    ["cd " + root + " && " +
-				     "exec " + prog + " -r " + fn],
-				    stderr=subprocess.STDOUT, shell=True,
-				    universal_newlines=True)
-				pf = ""
-
-			for ln in x.split("\n"):
-				ln = ln.split()
-				if len(ln) == 0:
-					continue
-				if ln[0] == "Creating":
-					gn = ln[1].strip("'")
-					assert gn[-5:] == ".gcov"
-					sn = gn[:-5]
-					process_gcov(
-					    os.path.join(root, pf, gn), sn)
+    """ Walk tree, run gcov, process output """
+    for root, dirs, files in os.walk(subdir):
+        for i in exclude:
+            if i in dirs:
+                dirs.remove(i)
+        if " ".join(files).find(".gcda") == -1:
+            continue
+        for fn in files:
+            if fn[-2:] != ".o":
+                continue
+
+            # if we find the .o file in a .../.libs the sources
+            # must be found relative to the parent directory
+
+            if root[-6:] == "/.libs":
+                x = subprocess.check_output(
+                    ["cd " + root + "/.. && " +
+                     "exec " + prog + " -r .libs/" + fn],
+                    stderr=subprocess.STDOUT, shell=True,
+                    universal_newlines=True)
+                pf = ".."
+            else:
+                x = subprocess.check_output(
+                    ["cd " + root + " && " +
+                     "exec " + prog + " -r " + fn],
+                    stderr=subprocess.STDOUT, shell=True,
+                    universal_newlines=True)
+                pf = ""
+
+            for ln in x.split("\n"):
+                ln = ln.split()
+                if not ln:
+                    continue
+                if ln[0] == "Creating":
+                    gn = ln[1].strip("'")
+                    assert gn[-5:] == ".gcov"
+                    sn = gn[:-5]
+                    process_gcov(
+                        os.path.join(root, pf, gn), sn)
 
 def produce_output(fdo):
-	"""
-	Produce compact output file
-
-	Format:
-		linefm [lineto] count
-
-		"+" in linefm means "previous line + 1"
-		"." in count means "same as previous count"
-	"""
-
-	for sn, cnt in counts.items():
-		fdo.write("/" + sn + " " + str(lengths[sn]) + "\n")
-		lnos = list(cnt.items())
-		lnos.sort()
-		pln = -1
-		pcn = -1
-		while len(lnos) > 0:
-			ln, cn = lnos.pop(0)
-			lnl = ln
-			while len(lnos) > 0:
-				lnn, cnn = lnos[0]
-				if lnl + 1 != lnn or cnn != cn:
-					break
-				lnos.pop(0)
-				lnl = lnn
-			if ln == pln + 1:
-				s = "+ "
-			else:
-				s = "%d " % ln
-
-			if ln != lnl:
-				s += "%d " % lnl
-				pln = lnl
-			else:
-				pln = ln
-
-			if cn == pcn:
-				s += "."
-			else:
-				s += "%d" % cn
-				pcn = cn
-			fdo.write(s + "\n")
+    """
+    Produce compact output file
+
+    Format:
+        linefm [lineto] count
+
+        "+" in linefm means "previous line + 1"
+        "." in count means "same as previous count"
+    """
+
+    for sn, cnt in counts.items():
+        fdo.write("/" + sn + " " + str(lengths[sn]) + "\n")
+        lnos = list(cnt.items())
+        lnos.sort()
+        pln = -1
+        pcn = -1
+        while lnos:
+            ln, cn = lnos.pop(0)
+            lnl = ln
+            while lnos:
+                lnn, cnn = lnos[0]
+                if lnl + 1 != lnn or cnn != cn:
+                    break
+                lnos.pop(0)
+                lnl = lnn
+            if ln == pln + 1:
+                s = "+ "
+            else:
+                s = "%d " % ln
+
+            if ln != lnl:
+                s += "%d " % lnl
+                pln = lnl
+            else:
+                pln = ln
+
+            if cn == pcn:
+                s += "."
+            else:
+                s += "%d" % cn
+                pcn = cn
+            fdo.write(s + "\n")
 
 if __name__ == "__main__":
 
-	optlist, args = getopt.getopt(sys.argv[1:], "g:o:x:")
-
-	fo = sys.stdout
-	gcovprog = "gcov6"
-
-	for f, v in optlist:
-		if f == '-o' and v == '-':
-			fo = sys.stdout
-		elif f == '-o':
-			fo = open(v, "w")
-		elif f == '-g':
-			gcovprog = v
-		elif f == '-x':
-			exclude.append(v)
-		else:
-			assert False
-	if len(args) == 0:
-		args = ["."]
-	for dn in args:
-		run_gcov(gcovprog, dn)
-
-	produce_output(fo)
+    optlist, args = getopt.getopt(sys.argv[1:], "g:o:x:")
+
+    fo = sys.stdout
+    gcovprog = "gcov6"
+
+    for f, v in optlist:
+        if f == '-o' and v == '-':
+            fo = sys.stdout
+        elif f == '-o':
+            fo = open(v, "w")
+        elif f == '-g':
+            gcovprog = v
+        elif f == '-x':
+            exclude.append(v)
+        else:
+            assert False
+    if not args:
+        args = ["."]
+    for dn in args:
+        run_gcov(gcovprog, dn)
+
+    produce_output(fo)
diff --git a/tools/include_wash.py b/tools/include_wash.py
index 1284e30e1..4df6eeaf3 100644
--- a/tools/include_wash.py
+++ b/tools/include_wash.py
@@ -5,41 +5,39 @@ from __future__ import print_function
 import os
 
 def check(fn):
-	l = []
-	for i in open(fn):
-		i = i.strip()
-		if len(i) == 0:
-			continue
-		if i[0] != "#":
-			continue
-		if i.find("include") == -1:
-			continue
-		if i.find('"') == -1:
-			l.append(i.split('<')[1].split('>')[0])
-		else:
-			l.append(i.split('"')[1])
-	if "vrt.h" in l:
-		vrt = l.index("vrt.h")
-		if not "vdef.h" in l:
-			print(fn, "vdef.h not included with vrt.h")
-		vdef = l.index("vdef.h")
-		if vdef > vrt:
-			print(fn, "vdef.h included after vrt.h")
-		for i in ("stddef.h", "stdint.h", "cache/cache.h", "cache.h"):
-			if i in l:
-				print(fn, i + " included with vrt.h")
+    l = []
+    for i in open(fn):
+        i = i.strip()
+        if not i:
+            continue
+        if i[0] != "#":
+            continue
+        if i.find("include") == -1:
+            continue
+        if i.find('"') == -1:
+            l.append(i.split('<')[1].split('>')[0])
+        else:
+            l.append(i.split('"')[1])
+    if "vrt.h" in l:
+        vrt = l.index("vrt.h")
+        if "vdef.h" not in l:
+            print(fn, "vdef.h not included with vrt.h")
+        vdef = l.index("vdef.h")
+        if vdef > vrt:
+            print(fn, "vdef.h included after vrt.h")
+        for i in ("stddef.h", "stdint.h", "cache/cache.h", "cache.h"):
+            if i in l:
+                print(fn, i + " included with vrt.h")
 
-	for i in ("cache/cache.h", "cache.h"):
-		if i in l:
-			for i in (
-			    "stddef.h", "stdint.h", "vrt.h",
-			    "math.h", "pthread.h", "stdarg.h", "sys/types.h",
-			    "vdef.h", "miniobj.h", "vas.h", "vqueue.h",
-			):
-				if i in l:
-					print(fn, i + " included with cache.h")
+    for i in ("cache/cache.h", "cache.h"):
+        if i in l:
+            for i in ("stddef.h", "stdint.h", "vrt.h",
+                      "math.h", "pthread.h", "stdarg.h", "sys/types.h",
+                      "vdef.h", "miniobj.h", "vas.h", "vqueue.h"):
+                if i in l:
+                    print(fn, i + " included with cache.h")
 
 for (dir, dns, fns) in os.walk("."):
-	for f in fns:
-		if f[-2:] == ".c":
-			check(dir + "/" + f)
+    for f in fns:
+        if f[-2:] == ".c":
+            check(dir + "/" + f)

From hermunn at varnish-software.com  Wed Oct 24 09:29:15 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:15 +0000 (UTC)
Subject: [6.1] 0723fa9f5 Update Teken from FreeBSD source tree
Message-ID: <20181024092915.CCCB8AFB86@lists.varnish-cache.org>


commit 0723fa9f5d7e8856b05031a33040ef71e95dd101
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 07:31:46 2018 +0000

    Update Teken from FreeBSD source tree

diff --git a/bin/varnishtest/gensequences b/bin/varnishtest/gensequences
index 8c2299a61..e82e56ec7 100644
--- a/bin/varnishtest/gensequences
+++ b/bin/varnishtest/gensequences
@@ -25,7 +25,7 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
-# $FreeBSD: head/sys/teken/gensequences 223574 2011-06-26 18:25:10Z ed $
+# $FreeBSD: head/sys/teken/gensequences 333925 2018-05-20 14:21:20Z dumbbell $
 
 function die(msg) {
 	print msg;
@@ -35,6 +35,15 @@ function die(msg) {
 function cchar(str) {
 	if (str == "^[")
 		return "\\x1B";
+	if (str == "SP")
+		return " ";
+
+	return str;
+}
+
+function csequence(str) {
+	if (str == "SP")
+		return " ";
 
 	return str;
 }
@@ -57,11 +66,11 @@ while (getline > 0) {
 	prefix = "";
 	l_prefix_name[""] = "teken_state_init";
 	for (i = 1; i < nsequences; i++) {
-		n = prefix sequence[i];
+		n = prefix csequence(sequence[i]);
 		l_prefix_parent[n] = prefix;
 		l_prefix_suffix[n] = sequence[i];
 		if (!l_prefix_name[n])
-			l_prefix_name[n] = "teken_state_" "" ++npr;
+			l_prefix_name[n] = "teken_state_" ++npr;
 		prefix = n;
 	}
 
diff --git a/bin/varnishtest/sequences b/bin/varnishtest/sequences
index af92df04b..d8f30306b 100644
--- a/bin/varnishtest/sequences
+++ b/bin/varnishtest/sequences
@@ -23,93 +23,94 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
-# $FreeBSD: head/sys/teken/sequences 322662 2017-08-18 15:40:40Z bde $
+# $FreeBSD: head/sys/teken/sequences 334316 2018-05-29 08:41:44Z dumbbell $
 
 # File format is as follows:
 #	Abbr		Abbreviation of sequence name
 #	Name		Sequence name (will be converted to C function name)
 #	Sequence	Bytes that form the sequence
-#	Arguments	Standard value of arguments passed to this sequence
+#	Args		Standard value of arguments passed to this sequence
 #			- `n' non-zero number (0 gets converted to 1)
 #			- `r' regular numeric argument
 #			- `v' means a variable number of arguments
 
-# Abbr	Name					Sequence	Arguments
-CBT	Cursor Backward Tabulation		^[ [ Z		n
-CHT	Cursor Forward Tabulation		^[ [ I		n
-CNL	Cursor Next Line			^[ [ E		n
-CPL	Cursor Previous Line			^[ [ F		n
-CPR	Cursor Position Report			^[ [ n		r
-CUB	Cursor Backward				^[ [ D		n
-CUD	Cursor Down				^[ [ B		n
-CUD	Cursor Down				^[ [ e		n
-CUF	Cursor Forward				^[ [ C		n
-CUF	Cursor Forward				^[ [ a		n
-CUP	Cursor Position				^[ [ H		n n
-CUP	Cursor Position				^[ [ f		n n
-CUU	Cursor Up				^[ [ A		n
-DA1	Primary Device Attributes		^[ [ c		r
-DA2	Secondary Device Attributes		^[ [ > c	r
-DC	Delete character			^[ [ P		n
-DCS	Device Control String			^[ P
-DECALN	Alignment test				^[ # 8
-DECDHL	Double Height Double Width Line Top	^[ # 3
-DECDHL	Double Height Double Width Line Bottom	^[ # 4
-DECDWL	Single Height Double Width Line		^[ # 6
-DECKPAM	Keypad application mode			^[ =
-DECKPNM	Keypad numeric mode			^[ >
-DECRC	Restore cursor				^[ 8
-DECRC	Restore cursor				^[ [ u
-DECRM	Reset DEC mode				^[ [ ? l	r
-DECSC	Save cursor				^[ 7
-DECSC	Save cursor				^[ [ s
-DECSM	Set DEC mode				^[ [ ? h	r
-DECSTBM	Set top and bottom margins		^[ [ r		r r
-DECSWL	Single Height Single Width Line		^[ # 5
-DL	Delete line				^[ [ M		n
-DSR	Device Status Report			^[ [ ? n	r
-ECH	Erase character				^[ [ X		n
-ED	Erase display				^[ [ J		r
-EL	Erase line				^[ [ K		r
-G0SCS0	G0 SCS Special Graphics			^[ ( 0
-G0SCS1	G0 SCS US ASCII				^[ ( 1
-G0SCS2	G0 SCS Special Graphics			^[ ( 2
-G0SCSA	G0 SCS UK National			^[ ( A
-G0SCSB	G0 SCS US ASCII				^[ ( B
-G1SCS0	G1 SCS Special Graphics			^[ ) 0
-G1SCS1	G1 SCS US ASCII				^[ ) 1
-G1SCS2	G1 SCS Special Graphics			^[ ) 2
-G1SCSA	G1 SCS UK National			^[ ) A
-G1SCSB	G1 SCS US ASCII				^[ ) B
-HPA	Horizontal Position Absolute		^[ [ G		n
-HPA	Horizontal Position Absolute		^[ [ `		n
-HTS	Horizontal Tab Set			^[ H
-ICH	Insert character			^[ [ @		n
-IL	Insert line				^[ [ L		n
-IND	Index					^[ D
-NEL	Next line				^[ E
-OSC	Operating System Command		^[ ]
-RI	Reverse index				^[ M
-RIS	Reset to Initial State			^[ c
-RM	Reset Mode				^[ [ l		r
-SD	Pan Up					^[ [ T		n
-SGR	Set Graphic Rendition			^[ [ m		v
-SM	Set Mode				^[ [ h		r
-ST	String Terminator			^[ \\
-SU	Pan Down				^[ [ S		n
-TBC	Tab Clear				^[ [ g		r
-VPA	Vertical Position Absolute		^[ [ d		n
+# Abbr		Name					Sequence	Args
+CBT		Cursor Backward Tabulation		^[ [ Z		n
+CHT		Cursor Forward Tabulation		^[ [ I		n
+CNL		Cursor Next Line			^[ [ E		n
+CPL		Cursor Previous Line			^[ [ F		n
+CPR		Cursor Position Report			^[ [ n		r
+CUB		Cursor Backward				^[ [ D		n
+CUD		Cursor Down				^[ [ B		n
+CUD		Cursor Down				^[ [ e		n
+CUF		Cursor Forward				^[ [ C		n
+CUF		Cursor Forward				^[ [ a		n
+CUP		Cursor Position				^[ [ H		n n
+CUP		Cursor Position				^[ [ f		n n
+CUU		Cursor Up				^[ [ A		n
+DA1		Primary Device Attributes		^[ [ c		r
+DA2		Secondary Device Attributes		^[ [ > c	r
+DC		Delete character			^[ [ P		n
+DCS		Device Control String			^[ P
+DECALN		Alignment test				^[ # 8
+DECDHL		Double Height Double Width Line Top	^[ # 3
+DECDHL		Double Height Double Width Line Bottom	^[ # 4
+DECDWL		Single Height Double Width Line		^[ # 6
+DECKPAM		Keypad application mode			^[ =
+DECKPNM		Keypad numeric mode			^[ >
+DECRC		Restore cursor				^[ 8
+DECRC		Restore cursor				^[ [ u
+DECRM		Reset DEC mode				^[ [ ? l	r
+DECSC		Save cursor				^[ 7
+DECSC		Save cursor				^[ [ s
+DECSCUSR	Set Cursor Style			^[ [ SP q	r
+DECSM		Set DEC mode				^[ [ ? h	r
+DECSTBM		Set top and bottom margins		^[ [ r		r r
+DECSWL		Single Height Single Width Line		^[ # 5
+DL		Delete line				^[ [ M		n
+DSR		Device Status Report			^[ [ ? n	r
+ECH		Erase character				^[ [ X		n
+ED		Erase display				^[ [ J		r
+EL		Erase line				^[ [ K		r
+G0SCS0		G0 SCS Special Graphics			^[ ( 0
+G0SCS1		G0 SCS US ASCII				^[ ( 1
+G0SCS2		G0 SCS Special Graphics			^[ ( 2
+G0SCSA		G0 SCS UK National			^[ ( A
+G0SCSB		G0 SCS US ASCII				^[ ( B
+G1SCS0		G1 SCS Special Graphics			^[ ) 0
+G1SCS1		G1 SCS US ASCII				^[ ) 1
+G1SCS2		G1 SCS Special Graphics			^[ ) 2
+G1SCSA		G1 SCS UK National			^[ ) A
+G1SCSB		G1 SCS US ASCII				^[ ) B
+HPA		Horizontal Position Absolute		^[ [ G		n
+HPA		Horizontal Position Absolute		^[ [ `		n
+HTS		Horizontal Tab Set			^[ H
+ICH		Insert character			^[ [ @		n
+IL		Insert line				^[ [ L		n
+IND		Index					^[ D
+NEL		Next line				^[ E
+OSC		Operating System Command		^[ ]
+RI		Reverse index				^[ M
+RIS		Reset to Initial State			^[ c
+RM		Reset Mode				^[ [ l		r
+SD		Pan Up					^[ [ T		n
+SGR		Set Graphic Rendition			^[ [ m		v
+SM		Set Mode				^[ [ h		r
+ST		String Terminator			^[ \\
+SU		Pan Down				^[ [ S		n
+TBC		Tab Clear				^[ [ g		r
+VPA		Vertical Position Absolute		^[ [ d		n
 
 # Cons25 compatibility sequences
-C25BLPD	Cons25 set bell pitch duration		^[ [ = B	r r
-C25BORD	Cons25 set border			^[ [ = A	r
-C25DBG	Cons25 set default background		^[ [ = G	r
-C25DFG	Cons25 set default foreground		^[ [ = F	r
-C25GCS	Cons25 set global cursor shape		^[ [ = C	v
-C25LCT	Cons25 set local cursor type		^[ [ = S	r
-C25MODE	Cons25 set terminal mode		^[ [ = T	r
-C25SGR	Cons25 set graphic rendition		^[ [ x		r r
-C25VTSW	Cons25 switch virtual terminal		^[ [ z		r
+C25BLPD		Cons25 set bell pitch duration		^[ [ = B	r r
+C25BORD		Cons25 set border			^[ [ = A	r
+C25DBG		Cons25 set default background		^[ [ = G	r
+C25DFG		Cons25 set default foreground		^[ [ = F	r
+C25GCS		Cons25 set global cursor shape		^[ [ = C	v
+C25LCT		Cons25 set local cursor type		^[ [ = S	r
+C25MODE		Cons25 set terminal mode		^[ [ = T	r
+C25SGR		Cons25 set graphic rendition		^[ [ x		r r
+C25VTSW		Cons25 switch virtual terminal		^[ [ z		r
 
 # VT52 compatibility
-#DECID	VT52 DECID				^[ Z
+#DECID		VT52 DECID				^[ Z
diff --git a/bin/varnishtest/teken.c b/bin/varnishtest/teken.c
index ad06c57c6..21eb71752 100644
--- a/bin/varnishtest/teken.c
+++ b/bin/varnishtest/teken.c
@@ -25,7 +25,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * $FreeBSD: head/sys/teken/teken.c 326272 2017-11-27 15:23:17Z pfg $
+ * $FreeBSD: head/sys/teken/teken.c 333683 2018-05-16 18:12:49Z cem $
  */
 
 #include "config.h"
@@ -35,14 +35,13 @@
 #include <stdint.h>
 #include <stdio.h>
 #include <string.h>
+#define	teken_assert(x)		assert(x)
 
 #include "vdef.h"
 #include "vas.h"
 
-#define	teken_assert(x)		assert(x)
-
 /* debug messages */
-#define	teken_printf(...)
+#define	teken_printf(x,...)
 
 /* Private flags for t_stateflags. */
 #define	TS_FIRSTDIGIT	0x0001	/* First numeric digit in escape sequence. */
@@ -128,20 +127,36 @@ teken_funcs_copy(const teken_t *t, const teken_rect_t *r, const teken_pos_t *p)
 	t->t_funcs->tf_copy(t->t_softc, r, p);
 }
 
+static inline void
+teken_funcs_pre_input(const teken_t *t)
+{
+
+	if (t->t_funcs->tf_pre_input != NULL)
+		t->t_funcs->tf_pre_input(t->t_softc);
+}
+
+static inline void
+teken_funcs_post_input(const teken_t *t)
+{
+
+	if (t->t_funcs->tf_post_input != NULL)
+		t->t_funcs->tf_post_input(t->t_softc);
+}
+
 static inline void
 teken_funcs_param(const teken_t *t, int cmd, unsigned int value)
 {
 
-	if (t->t_funcs->tf_param != NULL)
-		t->t_funcs->tf_param(t->t_softc, cmd, value);
+	teken_assert(t->t_funcs->tf_param != NULL);
+	t->t_funcs->tf_param(t->t_softc, cmd, value);
 }
 
 static inline void
 teken_funcs_respond(const teken_t *t, const void *buf, size_t len)
 {
 
-	if (t->t_funcs->tf_respond != NULL)
-		t->t_funcs->tf_respond(t->t_softc, buf, len);
+	teken_assert(t->t_funcs->tf_respond != NULL);
+	t->t_funcs->tf_respond(t->t_softc, buf, len);
 }
 
 #include "teken_subr.h"
@@ -288,8 +303,10 @@ teken_input(teken_t *t, const void *buf, size_t len)
 {
 	const char *c = buf;
 
+	teken_funcs_pre_input(t);
 	while (len-- > 0)
 		teken_input_byte(t, *c++);
+	teken_funcs_post_input(t);
 }
 
 const teken_pos_t *
diff --git a/bin/varnishtest/teken.h b/bin/varnishtest/teken.h
index 986c3bd36..eb59817d7 100644
--- a/bin/varnishtest/teken.h
+++ b/bin/varnishtest/teken.h
@@ -25,7 +25,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * $FreeBSD: head/sys/teken/teken.h 326272 2017-11-27 15:23:17Z pfg $
+ * $FreeBSD: head/sys/teken/teken.h 333669 2018-05-16 09:01:02Z dumbbell $
  */
 
 #ifndef _TEKEN_H_
@@ -93,6 +93,8 @@ typedef void tf_putchar_t(void *, const teken_pos_t *, teken_char_t,
 typedef void tf_fill_t(void *, const teken_rect_t *, teken_char_t,
     const teken_attr_t *);
 typedef void tf_copy_t(void *, const teken_rect_t *, const teken_pos_t *);
+typedef void tf_pre_input_t(void *);
+typedef void tf_post_input_t(void *);
 typedef void tf_param_t(void *, int, unsigned int);
 #define	TP_SHOWCURSOR	0
 #define	TP_KEYPADAPP	1
@@ -114,6 +116,8 @@ typedef struct {
 	tf_putchar_t	*tf_putchar;
 	tf_fill_t	*tf_fill;
 	tf_copy_t	*tf_copy;
+	tf_pre_input_t	*tf_pre_input;
+	tf_post_input_t	*tf_post_input;
 	tf_param_t	*tf_param;
 	tf_respond_t	*tf_respond;
 } teken_funcs_t;
diff --git a/bin/varnishtest/teken_scs.h b/bin/varnishtest/teken_scs.h
index fd99de15d..719f2a98e 100644
--- a/bin/varnishtest/teken_scs.h
+++ b/bin/varnishtest/teken_scs.h
@@ -25,7 +25,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * $FreeBSD: head/sys/teken/teken_scs.h 326272 2017-11-27 15:23:17Z pfg $
+ * $FreeBSD: head/sys/teken/teken_scs.h 332297 2018-04-08 19:23:50Z phk $
  */
 
 static inline teken_char_t
diff --git a/bin/varnishtest/teken_subr.h b/bin/varnishtest/teken_subr.h
index b67ef5cc0..22d06bb19 100644
--- a/bin/varnishtest/teken_subr.h
+++ b/bin/varnishtest/teken_subr.h
@@ -25,7 +25,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * $FreeBSD: head/sys/teken/teken_subr.h 326272 2017-11-27 15:23:17Z pfg $
+ * $FreeBSD: head/sys/teken/teken_subr.h 333995 2018-05-21 20:35:16Z dumbbell $
  */
 
 static void teken_subr_cursor_up(teken_t *, unsigned int);
@@ -371,6 +371,27 @@ teken_subr_cursor_up(teken_t *t, unsigned int nrows)
 	teken_funcs_cursor(t);
 }
 
+static void
+teken_subr_set_cursor_style(teken_t *t, unsigned int style)
+{
+
+	/* TODO */
+	(void)t;
+	(void)style;
+
+	/*
+	 * CSI Ps SP q
+	 *   Set cursor style (DECSCUSR), VT520.
+	 *     Ps = 0  -> blinking block.
+	 *     Ps = 1  -> blinking block (default).
+	 *     Ps = 2  -> steady block.
+	 *     Ps = 3  -> blinking underline.
+	 *     Ps = 4  -> steady underline.
+	 *     Ps = 5  -> blinking bar (xterm).
+	 *     Ps = 6  -> steady bar (xterm).
+	 */
+}
+
 static void
 teken_subr_delete_character(const teken_t *t, unsigned int ncols)
 {
diff --git a/bin/varnishtest/teken_subr_compat.h b/bin/varnishtest/teken_subr_compat.h
index 9c84f1331..7db4d858d 100644
--- a/bin/varnishtest/teken_subr_compat.h
+++ b/bin/varnishtest/teken_subr_compat.h
@@ -25,7 +25,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * $FreeBSD: head/sys/teken/teken_subr_compat.h 326272 2017-11-27 15:23:17Z pfg $
+ * $FreeBSD: head/sys/teken/teken_subr_compat.h 332297 2018-04-08 19:23:50Z phk $
  */
 
 static void
diff --git a/bin/varnishtest/teken_wcwidth.h b/bin/varnishtest/teken_wcwidth.h
index 73401698f..70d92060f 100644
--- a/bin/varnishtest/teken_wcwidth.h
+++ b/bin/varnishtest/teken_wcwidth.h
@@ -7,7 +7,7 @@
  *
  * Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
  *
- * $FreeBSD: head/sys/teken/teken_wcwidth.h 186681 2009-01-01 13:26:53Z ed $
+ * $FreeBSD: head/sys/teken/teken_wcwidth.h 332297 2018-04-08 19:23:50Z phk $
  */
 
 struct interval {

From hermunn at varnish-software.com  Wed Oct 24 09:29:16 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:16 +0000 (UTC)
Subject: [6.1] 6f02ba5a5 Implement and test ECMA-48 "REP" sequence.
Message-ID: <20181024092916.06923AFB92@lists.varnish-cache.org>


commit 6f02ba5a5ad62ccd51746b763e77b02ed636ad92
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 08:31:37 2018 +0000

    Implement and test ECMA-48 "REP" sequence.
    
    Fixes: #2668

diff --git a/bin/varnishtest/gensequences b/bin/varnishtest/gensequences
index e82e56ec7..4337186b8 100644
--- a/bin/varnishtest/gensequences
+++ b/bin/varnishtest/gensequences
@@ -158,6 +158,7 @@ for (p in l_prefix_name) {
 
 	if (l_prefix_name[p] != "teken_state_init") {
 		print "";
+		print "\tt->t_last = 0;";
 		print "\tteken_state_switch(t, teken_state_init);";
 	}
 	print "}";
diff --git a/bin/varnishtest/sequences b/bin/varnishtest/sequences
index d8f30306b..50f589626 100644
--- a/bin/varnishtest/sequences
+++ b/bin/varnishtest/sequences
@@ -114,3 +114,6 @@ C25VTSW		Cons25 switch virtual terminal		^[ [ z		r
 
 # VT52 compatibility
 #DECID		VT52 DECID				^[ Z
+
+# ECMA-48
+REP		Repeat last graphic char		^[ [ b		n
diff --git a/bin/varnishtest/teken.h b/bin/varnishtest/teken.h
index eb59817d7..c08a2bbb8 100644
--- a/bin/varnishtest/teken.h
+++ b/bin/varnishtest/teken.h
@@ -157,6 +157,7 @@ struct __teken {
 
 	unsigned int	 t_utf8_left;
 	teken_char_t	 t_utf8_partial;
+	teken_char_t	 t_last;
 
 	unsigned int	 t_curscs;
 	teken_scs_t	*t_saved_curscs;
diff --git a/bin/varnishtest/teken_subr.h b/bin/varnishtest/teken_subr.h
index 22d06bb19..644e502f6 100644
--- a/bin/varnishtest/teken_subr.h
+++ b/bin/varnishtest/teken_subr.h
@@ -798,10 +798,11 @@ teken_subr_primary_device_attributes(const teken_t *t, unsigned int request)
 }
 
 static void
-teken_subr_do_putchar(const teken_t *t, const teken_pos_t *tp, teken_char_t c,
+teken_subr_do_putchar(teken_t *t, const teken_pos_t *tp, teken_char_t c,
     int width)
 {
 
+	t->t_last = c;
 	if (t->t_stateflags & TS_INSERT &&
 	    tp->tp_col < t->t_winsize.tp_col - width) {
 		teken_rect_t ctr;
@@ -1334,3 +1335,12 @@ teken_subr_vertical_position_absolute(teken_t *t, unsigned int row)
 	t->t_stateflags &= ~TS_WRAPPED;
 	teken_funcs_cursor(t);
 }
+
+static void
+teken_subr_repeat_last_graphic_char(teken_t *t, unsigned int rpts)
+{
+
+	for (; t->t_last != 0 && rpts > 0; rpts--)
+		teken_subr_regular_character(t, t->t_last);
+}
+
diff --git a/bin/varnishtest/tests/a00001.vtc b/bin/varnishtest/tests/a00001.vtc
index b6628d807..a48861144 100644
--- a/bin/varnishtest/tests/a00001.vtc
+++ b/bin/varnishtest/tests/a00001.vtc
@@ -204,6 +204,27 @@ process p4 -writehex 0d
 process p4 -expect-text 21 11 "Enter choice number (0 - 12):"
 process p4 -screen_dump
 
+# 11. Test non-VT100 (e.g., VT220, XTERM) terminals
+process p4 -writehex "31 31 0d"
+process p4 -expect-text 0 0 "Menu 11: Non-VT100 Tests"
+
+process p4 -writehex "37 0d"
+process p4 -expect-text 0 0 "Menu 11.7: Miscellaneous ISO-6429 (ECMA-48) Tests"
+
+process p4 -writehex "32 0d"
+process p4 -expect-text 0 0 "Push <RETURN>"
+process p4 -screen_dump
+process p4 -expect-text 20 1 "Test Repeat (REP)"
+process p4 -expect-text 1 1 " ++ "
+process p4 -expect-text 2 2 " ++ "
+process p4 -expect-text 17 17 " ++ "
+process p4 -expect-text 18 18 "*++*"
+process p4 -writehex "0d"
+process p4 -expect-text 0 0 "Menu 11.7: Miscellaneous ISO-6429 (ECMA-48) Tests"
+process p4 -writehex "30 0d"
+process p4 -expect-text 0 0 "Menu 11: Non-VT100 Tests"
+process p4 -writehex "30 0d"
+
 # 0. Exit
 process p4 -writehex "30 0d"
 process p4 -expect-text 12 30 "That's all, folks!"

From hermunn at varnish-software.com  Wed Oct 24 09:29:16 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:16 +0000 (UTC)
Subject: [6.1] e245eb9b9 Be Sun-c compiler compatible
Message-ID: <20181024092916.22585AFB96@lists.varnish-cache.org>


commit e245eb9b9637837b37e44ccc346ec90a2801d9ee
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 11:41:42 2018 +0000

    Be Sun-c compiler compatible

diff --git a/bin/varnishtest/teken.c b/bin/varnishtest/teken.c
index 21eb71752..48df7e066 100644
--- a/bin/varnishtest/teken.c
+++ b/bin/varnishtest/teken.c
@@ -41,7 +41,7 @@
 #include "vas.h"
 
 /* debug messages */
-#define	teken_printf(x,...)
+#define	teken_printf(...)
 
 /* Private flags for t_stateflags. */
 #define	TS_FIRSTDIGIT	0x0001	/* First numeric digit in escape sequence. */

From hermunn at varnish-software.com  Wed Oct 24 09:29:16 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:16 +0000 (UTC)
Subject: [6.1] 4997ad66f Fix libvgz build error on gcc7
Message-ID: <20181024092916.41BA3AFB9B@lists.varnish-cache.org>


commit 4997ad66f24e502a8f9257030552cc82cf2ebfb5
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Mon Sep 24 13:27:36 2018 +0200

    Fix libvgz build error on gcc7
    
    This adds -Wno-implicit-fallthrough for libvgz to satisfy gcc7.
    
    The -Wno-unknown-warning-option is there to satisfy older gcc/clang that
    don't recognize -Wno-implicit-fallthrough.

diff --git a/configure.ac b/configure.ac
index cb2d76888..42e863fbd 100644
--- a/configure.ac
+++ b/configure.ac
@@ -261,6 +261,11 @@ if test "$ac_cv_have_viz" = no; then
 fi
 CFLAGS="${save_CFLAGS}"
 
+if test "x$GCC" = "xyes"; then
+	libvgz_extra_cflags="${libvgz_extra_cflags} -Wno-unknown-warning-option -Wno-implicit-fallthrough"
+	AC_SUBST(libvgz_extra_cflags)
+fi
+
 SAN_CFLAGS=
 SAN_LDFLAGS=
 UBSAN_CFLAGS=

From hermunn at varnish-software.com  Wed Oct 24 09:29:16 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:16 +0000 (UTC)
Subject: [6.1] 207b0959c Spelling
Message-ID: <20181024092916.6904AAFBA3@lists.varnish-cache.org>


commit 207b0959c20f630e47333b54944b323e2c788a20
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Sep 24 23:25:46 2018 +0100

    Spelling

diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c
index 764710f35..404b68da0 100644
--- a/bin/varnishtest/vtc_http2.c
+++ b/bin/varnishtest/vtc_http2.c
@@ -1213,7 +1213,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf)
 /* SECTION: stream.spec.frame_sendhex sendhex
  *
  * Push bytes directly on the wire. sendhex takes exactly one argument: a string
- * describing the bytes, in hex notation, will possible whitespaces between
+ * describing the bytes, in hex notation, with possible whitespaces between
  * them. Here's an example::
  *
  *	sendhex "00 00 08 00 0900	8d"
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index 03a39453e..d35d89f6c 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -298,7 +298,7 @@ vmod_syntax(VRT_CTX, VCL_REAL r)
 	assert(ctx->syntax == 40 || ctx->syntax == 41);
 	/*
 	 * We need to be careful because non-integer numbers have imprecise
-	 * IEE754 represenation (4.1 is 0x1.0666666666666p+2 = 4.09999...)
+	 * IEE754 representation (4.1 is 0x1.0666666666666p+2 = 4.09999...)
 	 * By scaling up and rounding, this is taken care of.
 	 */
 	return (round(r * 10) <= ctx->syntax);

From hermunn at varnish-software.com  Wed Oct 24 09:29:16 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:16 +0000 (UTC)
Subject: [6.1] 08c46781a Correct output
Message-ID: <20181024092916.88D38AFBB9@lists.varnish-cache.org>


commit 08c46781a9b120480cbd7ae67b55c51f2dc52c58
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Sep 24 23:27:04 2018 +0100

    Correct output

diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c
index 196b35c65..9fca4be7b 100644
--- a/bin/varnishtest/vtc_misc.c
+++ b/bin/varnishtest/vtc_misc.c
@@ -107,9 +107,10 @@ cmd_varnishtest(CMD_ARGS)
  * \-match REGEXP
  *	Expect regexp to match the stdout+err output.
  */
-/* SECTION: client-server.spec.shell shell
+/* SECTION: client-server.spec.shell
  *
- * Same as for the top-level shell.
+ * shell
+ *	Same as for the top-level shell.
  */
 
 static void

From hermunn at varnish-software.com  Wed Oct 24 09:29:16 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:16 +0000 (UTC)
Subject: [6.1] 715bf4ccb No need to mark the overflow twice
Message-ID: <20181024092916.A587FAFBD5@lists.varnish-cache.org>


commit 715bf4ccb7554f42ab915839a5f98ed56769602b
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Sep 24 23:27:29 2018 +0100

    No need to mark the overflow twice

diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 8e41d37b1..2e61ca7cf 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -379,7 +379,6 @@ http_CollectHdrSep(struct http *hp, const char *hdr, const char *sep)
 			if (b + x >= e) {
 				http_fail(hp);
 				VSLb(hp->vsl, SLT_LostHeader, "%s", hdr + 1);
-				WS_MarkOverflow(hp->ws);
 				WS_Release(hp->ws, 0);
 				return;
 			}

From hermunn at varnish-software.com  Wed Oct 24 09:29:16 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:16 +0000 (UTC)
Subject: [6.1] ca7c2d5c4 More spelling and consistency
Message-ID: <20181024092916.C3D41AFBE8@lists.varnish-cache.org>


commit ca7c2d5c4b5d9583c57967ac6f20534b0e3493fb
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Sep 24 23:32:41 2018 +0100

    More spelling and consistency

diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c
index 9fca4be7b..cc975688c 100644
--- a/bin/varnishtest/vtc_misc.c
+++ b/bin/varnishtest/vtc_misc.c
@@ -79,7 +79,7 @@ cmd_varnishtest(CMD_ARGS)
 /* SECTION: shell shell
  *
  * Pass the string given as argument to a shell. If you have multiple
- * commands to run, you can use curly barces to describe a multi-lines
+ * commands to run, you can use curly brackets to describe a multi-lines
  * script, eg::
  *
  *         shell {
diff --git a/doc/sphinx/users-guide/vcl-syntax.rst b/doc/sphinx/users-guide/vcl-syntax.rst
index f4bc219da..4ab5d7f74 100644
--- a/doc/sphinx/users-guide/vcl-syntax.rst
+++ b/doc/sphinx/users-guide/vcl-syntax.rst
@@ -3,7 +3,7 @@ VCL Syntax
 
 VCL has inherited a lot from C and it reads much like simple C or Perl.
 
-Blocks are delimited by curly braces, statements end with semicolons,
+Blocks are delimited by curly brackets, statements end with semicolons,
 and comments may be written as in C, C++ or Perl according to your own
 preferences.
 

From hermunn at varnish-software.com  Wed Oct 24 09:29:16 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:16 +0000 (UTC)
Subject: [6.1] ac244d067 Always report the ws id in lowercase
Message-ID: <20181024092916.EBA02AFBFF@lists.varnish-cache.org>


commit ac244d067ebe8185608835fdd7465fc7be095ccb
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Sep 24 23:44:13 2018 +0100

    Always report the ws id in lowercase

diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 2e61ca7cf..30988ab56 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -102,6 +102,7 @@ http_fail(const struct http *hp)
 {
 
 	VSC_C_main->losthdr++;
+	hp->ws->id[0] |= 0x20;		// cheesy tolower()
 	VSLb(hp->vsl, SLT_Error, "out of workspace (%s)", hp->ws->id);
 	WS_MarkOverflow(hp->ws);
 }
diff --git a/bin/varnishtest/tests/r01739.vtc b/bin/varnishtest/tests/r01739.vtc
index 7b8d6565d..843ea6933 100644
--- a/bin/varnishtest/tests/r01739.vtc
+++ b/bin/varnishtest/tests/r01739.vtc
@@ -17,7 +17,7 @@ varnish v1 -vcl+backend {
 
 logexpect l1 -v v1 -g raw {
 	expect * 1002 FetchError {^Workspace overflow}
-	expect * =    Error      {^out of workspace [(]Bo[)]}
+	expect * =    Error      {^out of workspace [(]bo[)]}
 } -start
 
 client c1 {
diff --git a/bin/varnishtest/tests/r01990.vtc b/bin/varnishtest/tests/r01990.vtc
index 5ee40bda5..d86f61c55 100644
--- a/bin/varnishtest/tests/r01990.vtc
+++ b/bin/varnishtest/tests/r01990.vtc
@@ -26,9 +26,9 @@ logexpect l1 -v v1 -g raw {
 	expect * 1002 FetchError   {^out of workspace}
 	expect * =    BerespStatus {^503}
 	expect * =    BerespReason {^Backend fetch failed}
-	expect * =    Error        {^out of workspace [(]Bo[)]}
+	expect * =    Error        {^out of workspace [(]bo[)]}
 	expect * =    LostHeader   {^Date:}
-	expect * =    Error        {^out of workspace [(]Bo[)]}
+	expect * =    Error        {^out of workspace [(]bo[)]}
 	expect * =    LostHeader   {^299}
 } -start
 

From hermunn at varnish-software.com  Wed Oct 24 09:29:17 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:17 +0000 (UTC)
Subject: [6.1] 816753906 Document the 'changed' parameter for param.show.
Message-ID: <20181024092917.17EF2AFC0B@lists.varnish-cache.org>


commit 816753906c1118991e8993f1db3acc76b6ee1ba0
Author: Geoff Simmons <geoff at uplex.de>
Date:   Tue Sep 25 10:10:39 2018 +0200

    Document the 'changed' parameter for param.show.

diff --git a/include/tbl/cli_cmds.h b/include/tbl/cli_cmds.h
index d162bada5..7b3d1de19 100644
--- a/include/tbl/cli_cmds.h
+++ b/include/tbl/cli_cmds.h
@@ -134,9 +134,14 @@ CLI_CMD(VCL_LABEL,
 
 CLI_CMD(PARAM_SHOW,
 	"param.show",
-	"param.show [-l] [<param>]",
+	"param.show [-l] [<param>|changed]",
 	"Show parameters and their values.",
-	"",
+
+	"The long form with ``-l`` shows additional information, including"
+	" documentation and minimum, maximum and default values, if defined"
+	" for the parameter. If a parameter is specified with ``<param>``,"
+	" show only that parameter. If ``changed`` is specified, show only"
+	" those parameters whose values differ from their defaults.",
 	0, 2
 )
 

From hermunn at varnish-software.com  Wed Oct 24 09:29:17 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:17 +0000 (UTC)
Subject: [6.1] c27e7f5bf Add missing copyright information
Message-ID: <20181024092917.734BDAFC16@lists.varnish-cache.org>


commit c27e7f5bfd3bde87c44f97555f5abccf649d186c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 25 19:54:50 2018 +0000

    Add missing copyright information

diff --git a/bin/varnishtest/hpack.h b/bin/varnishtest/hpack.h
index 94b734715..a443ee4af 100644
--- a/bin/varnishtest/hpack.h
+++ b/bin/varnishtest/hpack.h
@@ -1,3 +1,31 @@
+/*-
+ * Copyright (c) 2008-2016 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Guillaume Quintard <guillaume.quintard at gmail.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
 #include <stdint.h>
 
 enum hpk_result{
diff --git a/bin/varnishtest/huffman_input b/bin/varnishtest/huffman_input
index bfde068cd..117f87f06 100644
--- a/bin/varnishtest/huffman_input
+++ b/bin/varnishtest/huffman_input
@@ -1,3 +1,4 @@
+# For Copyright information see RFC7541 [BSD3]
        (  0)  |11111111|11000                             1ff8  [13]
        (  1)  |11111111|11111111|1011000                7fffd8  [23]
        (  2)  |11111111|11111111|11111110|0010         fffffe2  [28]
diff --git a/bin/varnishtest/vtc_h2_enctbl.h b/bin/varnishtest/vtc_h2_enctbl.h
index 5e69f3303..186a293d9 100644
--- a/bin/varnishtest/vtc_h2_enctbl.h
+++ b/bin/varnishtest/vtc_h2_enctbl.h
@@ -1,3 +1,7 @@
+/*-
+ * For Copyright information see RFC7541 [BSD3]
+ */
+
 HPACK(0,    0x1ff8,      13)
 HPACK(1,    0x7fffd8,    23)
 HPACK(2,    0xfffffe2,   28)
diff --git a/bin/varnishtest/vtc_h2_stattbl.h b/bin/varnishtest/vtc_h2_stattbl.h
index 1e5486245..e845cf85b 100644
--- a/bin/varnishtest/vtc_h2_stattbl.h
+++ b/bin/varnishtest/vtc_h2_stattbl.h
@@ -1,3 +1,6 @@
+/*-
+ * For Copyright information see RFC7541 [BSD3]
+ */
 STAT_HDRS(1,   ":authority",                   "")
 STAT_HDRS(2,   ":method",                      "GET")
 STAT_HDRS(3,   ":method",                      "POST")
diff --git a/bin/varnishtest/vtc_http.h b/bin/varnishtest/vtc_http.h
index a6c5a95f4..5cd079890 100644
--- a/bin/varnishtest/vtc_http.h
+++ b/bin/varnishtest/vtc_http.h
@@ -1,3 +1,31 @@
+/*-
+ * Copyright (c) 2008-2015 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
 #define MAX_HDR		50
 
 struct http {

From hermunn at varnish-software.com  Wed Oct 24 09:29:17 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:17 +0000 (UTC)
Subject: [6.1] 680d04496 Python 3 takes priority over python 2
Message-ID: <20181024092917.90CD5AFC1F@lists.varnish-cache.org>


commit 680d04496231e064ebcd3540d34f44f5d1171402
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Sep 26 07:03:28 2018 -0600

    Python 3 takes priority over python 2
    
    Take 2. Let's see if this time sticks.

diff --git a/varnish.m4 b/varnish.m4
index 8a92d5f10..71df96d9c 100644
--- a/varnish.m4
+++ b/varnish.m4
@@ -129,7 +129,9 @@ AC_DEFUN([_VARNISH_CHECK_DEVEL], [
 # _VARNISH_CHECK_PYTHON
 # ---------------------
 AC_DEFUN([_VARNISH_CHECK_PYTHON], [
-
+	m4_define_default([_AM_PYTHON_INTERPRETER_LIST],
+[python3.9 python3.8 python3.7 python3.6 python3.5 python3.4 python2.7 dnl
+python python2 python3])
 	AM_PATH_PYTHON([2.7], [], [
 		AC_MSG_ERROR([Python >= 2.7 is required.])
 	])

From hermunn at varnish-software.com  Wed Oct 24 09:29:17 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:17 +0000 (UTC)
Subject: [6.1] d5b71b20a Cleanup travis' osx job and bump image
Message-ID: <20181024092917.AEFBBAFC2C@lists.varnish-cache.org>


commit d5b71b20a4b2ee9ef4bd81ae55ecc01ecf300e4d
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Sep 26 07:19:26 2018 -0600

    Cleanup travis' osx job and bump image

diff --git a/.travis.yml b/.travis.yml
index 4441126b0..1b0751c8f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,7 +16,7 @@ matrix:
     env: CLANG=6.0 SAN_FLAGS="--enable-asan --enable-ubsan"
     sudo: required
   - os: osx
-    osx_image: xcode9.3
+    osx_image: xcode10
     compiler: clang
   allow_failures:
   - os: osx
@@ -36,7 +36,6 @@ before_install:
   - |
     if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
       brew update
-      brew upgrade python
       brew install docutils sphinx-doc nghttp2
       export PATH="/usr/local/opt/sphinx-doc/bin:$PATH"
     elif [[ -n "$CLANG" ]]; then
@@ -59,9 +58,6 @@ before_install:
   - ./configure ${CONFIGURE_ARGS}
 script:
   - |
-    if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-      export PYTHONPATH=`brew --prefix`/lib/python2.7/site-packages
-    fi
     if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then
       make -j3 distcheck
     else

From hermunn at varnish-software.com  Wed Oct 24 09:29:17 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:17 +0000 (UTC)
Subject: [6.1] 0855f4d26 Switch to clang 7 in our travis job
Message-ID: <20181024092917.E2A3CAFC36@lists.varnish-cache.org>


commit 0855f4d26705d9fda8cca00b446767d27aad936d
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Sep 26 09:10:54 2018 -0600

    Switch to clang 7 in our travis job

diff --git a/.travis.yml b/.travis.yml
index 1b0751c8f..da2bcbb8b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -13,14 +13,14 @@ matrix:
   - os: linux
     dist: trusty
     compiler: clang
-    env: CLANG=6.0 SAN_FLAGS="--enable-asan --enable-ubsan"
+    env: CLANG=7 SAN_FLAGS="--enable-asan --enable-ubsan"
     sudo: required
   - os: osx
     osx_image: xcode10
     compiler: clang
   allow_failures:
   - os: osx
-  - env: CLANG=6.0 SAN_FLAGS="--enable-asan --enable-ubsan"
+  - env: CLANG=7 SAN_FLAGS="--enable-asan --enable-ubsan"
 addons:
   apt:
     packages:

From hermunn at varnish-software.com  Wed Oct 24 09:29:18 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:18 +0000 (UTC)
Subject: [6.1] aab8fdd9f Fix varnish_vsc metrics type
Message-ID: <20181024092918.1263FAFC3C@lists.varnish-cache.org>


commit aab8fdd9f2b17b164fe6be048c728167bc011930
Author: Emmanuel Hocdet <manu at gandi.net>
Date:   Wed Sep 26 12:03:07 2018 +0200

    Fix varnish_vsc metrics type

diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc
index 0bd491ca6..84b9296c7 100644
--- a/bin/varnishd/VSC_main.vsc
+++ b/bin/varnishd/VSC_main.vsc
@@ -328,20 +328,17 @@
 	Number of backends known to us.
 
 .. varnish_vsc:: n_expired
-	:type:	gauge
 	:oneliner:	Number of expired objects
 
 	Number of objects that expired from cache because of old age.
 
 .. varnish_vsc:: n_lru_nuked
-	:type:	gauge
 	:oneliner:	Number of LRU nuked objects
 
 	How many objects have been forcefully evicted from storage to make
 	room for a new object.
 
 .. varnish_vsc:: n_lru_moved
-	:type:	gauge
 	:level:	diag
 	:oneliner:	Number of LRU moved objects
 
@@ -715,12 +712,10 @@
 	bans in the persistent ban lists.
 
 .. varnish_vsc:: n_purges
-	:type:	gauge
 	:oneliner:	Number of purge operations executed
 
 
 .. varnish_vsc:: n_obj_purged
-	:type:	gauge
 	:oneliner:	Number of purged objects
 
 

From hermunn at varnish-software.com  Wed Oct 24 09:29:18 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:18 +0000 (UTC)
Subject: [6.1] ece4a4e21 Dup(2) stderr before calling VCLS on -I argument.
Message-ID: <20181024092918.35DA2AFC43@lists.varnish-cache.org>


commit ece4a4e21f833c549f89aea0152f2c142be0331c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 26 19:40:05 2018 +0000

    Dup(2) stderr before calling VCLS on -I argument.
    
    Fixes: #2782

diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 7a427d5bf..d03ff2106 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -278,6 +278,8 @@ mgt_launch_child(struct cli *cli)
 	/* Open pipe for mgt->child CLI */
 	AZ(pipe(cp));
 	heritage.cli_in = cp[0];
+	assert(cp[0] > STDERR_FILENO);	// See #2782
+	assert(cp[1] > STDERR_FILENO);
 	MCH_Fd_Inherit(heritage.cli_in, "cli_in");
 	child_cli_out = cp[1];
 
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index aada3ce2c..237fb0f7f 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -861,7 +861,8 @@ main(int argc, char * const *argv)
 
 	if (I_fd >= 0) {
 		fprintf(stderr, "BEGIN of -I file processing\n");
-		mgt_cli_setup(I_fd, 2, 1, "-I file", mgt_I_close, stderr);
+		/* We must dup stderr, because VCLS closes the output fd */
+		mgt_cli_setup(I_fd, dup(2), 1, "-I file", mgt_I_close, stderr);
 		while (I_fd >= 0) {
 			o = VEV_Once(mgt_evb);
 			if (o != 1)

From hermunn at varnish-software.com  Wed Oct 24 09:29:18 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:18 +0000 (UTC)
Subject: [6.1] 5ab2fa795 Convert this test to UTF-8
Message-ID: <20181024092918.558C0AFC4B@lists.varnish-cache.org>


commit 5ab2fa795a09a38324a188ab3306e80a76a89d9f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 27 05:58:35 2018 +0000

    Convert this test to UTF-8

diff --git a/bin/varnishtest/tests/r00545.vtc b/bin/varnishtest/tests/r00545.vtc
index 0888af11a..e293c926b 100644
--- a/bin/varnishtest/tests/r00545.vtc
+++ b/bin/varnishtest/tests/r00545.vtc
@@ -7,12 +7,12 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_deliver {
-		set resp.http.foo = "???";
+		set resp.http.foo = "??????";
 	}
 } -start
 
 client c1 {
 	txreq
 	rxresp
-	expect resp.http.foo == "???"
+	expect resp.http.foo == "??????"
 } -run

From hermunn at varnish-software.com  Wed Oct 24 09:29:18 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:18 +0000 (UTC)
Subject: [6.1] 206e81918 explain a relevant detail of the worker thread
 signaling
Message-ID: <20181024092918.7C606AFC55@lists.varnish-cache.org>


commit 206e81918a4fcdb80e3a38416fc7fb060823b9f3
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Sep 27 10:27:44 2018 +0200

    explain a relevant detail of the worker thread signaling
    
    Over time, I have repeatedly stared at this code again and again
    wondering if (and why) our cv signaling is correct, just to end up
    with the same insight each time (but first overlooking #2719)
    
    Being fully aware that we do not want to plaster our code with
    outdated comments, I hope this explanation is warranted to save myself
    (and others, hopefully) from wasting precious life time on reiterating
    over the same question.

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 7cd02533d..88a0f0935 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -27,6 +27,27 @@
  * SUCH DAMAGE.
  *
  * Worker thread stuff unrelated to the worker thread pools.
+ *
+ * --
+ * signaling_note:
+ *
+ * note on worker wakeup signaling through the wrk condition variable (cv)
+ *
+ * In the general case, a cv needs to be signaled while holding the
+ * corresponding mutex, otherwise the signal may be posted before the waiting
+ * thread could register itself on the cv and, consequently, the signal may be
+ * missed.
+ *
+ * In our case, any worker thread which we wake up comes from the idle queue,
+ * where it put itself under the mutex, releasing that mutex implicitly via
+ * Lck_CondWait() (which calls some variant of pthread_cond_wait). So we avoid
+ * additional mutex contention knowing that any worker thread on the idle queue
+ * is blocking on the cv.
+ *
+ * Except -- when it isn't, because it woke up for releasing its VCL
+ * Reference. To account for this case, we check if the task function has been
+ * set in the meantime, which in turn requires all of the task preparation to be
+ * done holding the pool mutex. (see also #2719)
  */
 
 #include "config.h"
@@ -220,6 +241,7 @@ Pool_Task_Arg(struct worker *wrk, enum task_prio prio, task_func_t *func,
 	wrk2->task.func = func;
 	wrk2->task.priv = wrk2->aws->f;
 	Lck_Unlock(&pp->mtx);
+	// see signaling_note at the top for explanation
 	if (retval)
 		AZ(pthread_cond_signal(&wrk2->cond));
 	return (retval);
@@ -252,6 +274,7 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio prio)
 		wrk->task.func = task->func;
 		wrk->task.priv = task->priv;
 		Lck_Unlock(&pp->mtx);
+		// see signaling_note at the top for explanation
 		AZ(pthread_cond_signal(&wrk->cond));
 		return (0);
 	}
@@ -346,6 +369,7 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk)
 			VTAILQ_INSERT_HEAD(&pp->idle_queue, &wrk->task, list);
 			pp->nidle++;
 			do {
+				// see signaling_note at the top for explanation
 				i = Lck_CondWait(&wrk->cond, &pp->mtx,
 				    wrk->vcl == NULL ?  0 : wrk->lastused+60.);
 				if (i == ETIMEDOUT)

From hermunn at varnish-software.com  Wed Oct 24 09:29:18 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:18 +0000 (UTC)
Subject: [6.1] 1e3dff2b4 VSC_Arg succeeds with non-zero
Message-ID: <20181024092918.A43F3AFC5E@lists.varnish-cache.org>


commit 1e3dff2b4c3e0c4693ada0bed67e2d978e690d93
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Sep 28 15:37:40 2018 +0200

    VSC_Arg succeeds with non-zero
    
    Fixes #2787

diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index 74a8f5adf..be9e64c79 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -319,10 +319,10 @@ main(int argc, char * const *argv)
 
 	if (curses) {
 		if (has_f) {
-			AZ(VSC_Arg(vsc, 'f', "MGT.uptime"));
-			AZ(VSC_Arg(vsc, 'f', "MAIN.uptime"));
-			AZ(VSC_Arg(vsc, 'f', "MAIN.cache_hit"));
-			AZ(VSC_Arg(vsc, 'f', "MAIN.cache_miss"));
+			AN(VSC_Arg(vsc, 'f', "MGT.uptime"));
+			AN(VSC_Arg(vsc, 'f', "MAIN.uptime"));
+			AN(VSC_Arg(vsc, 'f', "MAIN.cache_hit"));
+			AN(VSC_Arg(vsc, 'f', "MAIN.cache_miss"));
 		}
 		do_curses(vd, vsc, 1.0);
 	}

From hermunn at varnish-software.com  Wed Oct 24 09:29:18 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:18 +0000 (UTC)
Subject: [6.1] 8f250c0ea Plug minor leak when we fail loading a vcl
Message-ID: <20181024092918.C17CDAFC67@lists.varnish-cache.org>


commit 8f250c0ead0db46bd227198d0cc307044205496b
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Sat Sep 29 06:57:50 2018 -0600

    Plug minor leak when we fail loading a vcl

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index dd3267ea2..6827b7379 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -347,6 +347,7 @@ vcl_KillBackends(struct vcl *vcl)
 		REPLACE(vdir->cli_name, NULL);
 		AN(vdir->methods->destroy);
 		vdir->methods->destroy(vdir->dir);
+		FREE_OBJ(vdir->dir);
 		FREE_OBJ(vdir);
 	}
 }

From hermunn at varnish-software.com  Wed Oct 24 09:29:18 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:18 +0000 (UTC)
Subject: [6.1] 39eae44fc Only do the "feature dns" DNS-lookup if/when
 necessary.
Message-ID: <20181024092918.DF428AFC72@lists.varnish-cache.org>


commit 39eae44fc427689d024c5f9b9dd9d5a150c59c6b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 1 09:54:57 2018 +0000

    Only do the "feature dns" DNS-lookup if/when necessary.
    
    Spotted by:     Willy Tarreau <w at 1wt.eu>

diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index 6ebff08a5..7483a53b4 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -81,7 +81,6 @@ extern char *vmod_path;
 extern struct vsb *params_vsb;
 extern int leave_temp;
 extern int vtc_witness;
-extern int feature_dns;
 extern int ign_unknown_macro;
 
 void init_server(void);
diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index d6168a2ae..2bc150322 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -95,7 +95,6 @@ char *vmod_path = NULL;
 struct vsb *params_vsb = NULL;
 int leave_temp;
 int vtc_witness = 0;
-int feature_dns;
 
 /**********************************************************************
  * Parse a -D option argument into a name/val pair, and insert
@@ -428,39 +427,6 @@ i_mode(void)
 	AZ(putenv(strdup("MALLOC_CONF=abort:true,junk:true")));
 }
 
-/**********************************************************************
- * Most test-cases use only numeric IP#'s but a few requires non-demented
- * DNS services.  This is a basic sanity check for those.
- */
-
-static int v_matchproto_(vss_resolved_f)
-dns_cb(void *priv, const struct suckaddr *sa)
-{
-	char abuf[VTCP_ADDRBUFSIZE];
-	char pbuf[VTCP_PORTBUFSIZE];
-	int *ret = priv;
-
-	VTCP_name(sa, abuf, sizeof abuf, pbuf, sizeof pbuf);
-	if (strcmp(abuf, "192.0.2.255")) {
-		fprintf(stderr, "DNS-test: Wrong response: %s\n", abuf);
-		*ret = -1;
-	} else if (*ret == 0)
-		*ret = 1;
-	return (0);
-}
-
-static int
-dns_works(void)
-{
-	int ret = 0, error;
-	const char *msg;
-
-	error = VSS_resolver("dns-canary.freebsd.dk", NULL, dns_cb, &ret, &msg);
-	if (error || msg != NULL || ret != 1)
-		return (0);
-	return (1);
-}
-
 /**********************************************************************
  * Figure out what IP related magic
  */
@@ -670,7 +636,6 @@ main(int argc, char * const *argv)
 
 	AZ(VSB_finish(params_vsb));
 
-	feature_dns = dns_works();
 	ip_magic();
 
 	if (iflg)
diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c
index cc975688c..34f79e045 100644
--- a/bin/varnishtest/vtc_misc.c
+++ b/bin/varnishtest/vtc_misc.c
@@ -47,6 +47,8 @@
 
 #include "vnum.h"
 #include "vre.h"
+#include "vtcp.h"
+#include "vss.h"
 #include "vtim.h"
 
 /* SECTION: varnishtest varnishtest
@@ -315,6 +317,39 @@ cmd_delay(CMD_ARGS)
 	VTIM_sleep(f);
 }
 
+/**********************************************************************
+ * Most test-cases use only numeric IP#'s but a few requires non-demented
+ * DNS services.  This is a basic sanity check for those.
+ */
+
+static int v_matchproto_(vss_resolved_f)
+dns_cb(void *priv, const struct suckaddr *sa)
+{
+	char abuf[VTCP_ADDRBUFSIZE];
+	char pbuf[VTCP_PORTBUFSIZE];
+	int *ret = priv;
+
+	VTCP_name(sa, abuf, sizeof abuf, pbuf, sizeof pbuf);
+	if (strcmp(abuf, "192.0.2.255")) {
+		fprintf(stderr, "DNS-test: Wrong response: %s\n", abuf);
+		*ret = -1;
+	} else if (*ret == 0)
+		*ret = 1;
+	return (0);
+}
+
+static int
+dns_works(void)
+{
+	int ret = 0, error;
+	const char *msg;
+
+	error = VSS_resolver("dns-canary.freebsd.dk", NULL, dns_cb, &ret, &msg);
+	if (error || msg != NULL || ret != 1)
+		return (0);
+	return (1);
+}
+
 /* SECTION: feature feature
  *
  * Test that the required feature(s) for a test are available, and skip
@@ -401,7 +436,7 @@ cmd_feature(CMD_ARGS)
 		}
 		FEATURE("pcre_jit", VRE_has_jit);
 		FEATURE("64bit", sizeof(void*) == 8);
-		FEATURE("dns", feature_dns);
+		FEATURE("dns", dns_works());
 		FEATURE("topbuild", iflg);
 		FEATURE("root", !geteuid());
 		FEATURE("user_varnish", getpwnam("varnish") != NULL);

From hermunn at varnish-software.com  Wed Oct 24 09:29:19 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:19 +0000 (UTC)
Subject: [6.1] 2ce2abaa9 Return a canned minimal 500 if we run out of ws in
 H2-deliver.
Message-ID: <20181024092919.07B34AFC82@lists.varnish-cache.org>


commit 2ce2abaa96230418b03dce317c757c474b3b0177
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 3 07:50:29 2018 +0000

    Return a canned minimal 500 if we run out of ws in H2-deliver.
    
    (Same as we do in H1)
    
    Fixes #2589

diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c
index 912eb88bc..8f4b56e1d 100644
--- a/bin/varnishd/http2/cache_http2_deliver.c
+++ b/bin/varnishd/http2/cache_http2_deliver.c
@@ -167,17 +167,35 @@ h2_enc_len(struct vsb *vsb, unsigned bits, unsigned val, uint8_t b0)
 	unsigned mask = (1U << bits) - 1U;
 
 	if (val >= mask) {
-		AZ(VSB_putc(vsb, b0 | (uint8_t)mask));
+		VSB_putc(vsb, b0 | (uint8_t)mask);
 		val -= mask;
 		while (val >= 128) {
-			AZ(VSB_putc(vsb, 0x80 | ((uint8_t)val & 0x7f)));
+			VSB_putc(vsb, 0x80 | ((uint8_t)val & 0x7f));
 			val >>= 7;
 		}
 	}
-	AZ(VSB_putc(vsb, (uint8_t)val));
+	VSB_putc(vsb, (uint8_t)val);
 	return (0);
 }
 
+/*
+ * Hand-crafted-H2-HEADERS-R-Us:
+ *
+ * This is a handbuilt HEADERS frame for when we run out of workspace
+ * during delivery.
+ */
+
+static const uint8_t h2_500_resp[] = {
+	// :status 500
+	0x8e,
+
+	// content-length 0
+	0x1f, 0x0d, 0x01, 0x30,
+
+	// server Varnish
+	0x1f, 0x27, 0x07, 'V', 'a', 'r', 'n', 'i', 's', 'h',
+};
+
 void v_matchproto_(vtr_deliver_f)
 h2_deliver(struct req *req, struct boc *boc, int sendbody)
 {
@@ -203,10 +221,10 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 	AN(VSB_new(&resp, req->ws->f, l, VSB_FIXEDLEN));
 
 	l = h2_status(buf, req->resp->status);
-	AZ(VSB_bcat(&resp, buf, l));
+	VSB_bcat(&resp, buf, l);
 
 	hp = req->resp;
-	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
+	for (u = HTTP_HDR_FIRST; u < hp->nhd && !VSB_error(&resp); u++) {
 		r = strchr(hp->hd[u].b, ':');
 		AN(r);
 
@@ -227,24 +245,32 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 			VSLb(req->vsl, SLT_Debug,
 			    "HP {%d, \"%s\", \"%s\"} <%s>",
 			    hps->idx, hps->name, hps->val, hp->hd[u].b);
-			AZ(h2_enc_len(&resp, 4, hps->idx, 0x10));
+			h2_enc_len(&resp, 4, hps->idx, 0x10);
 		} else {
-			AZ(VSB_putc(&resp, 0x10));
+			VSB_putc(&resp, 0x10);
 			sz--;
-			AZ(h2_enc_len(&resp, 7, sz, 0));
+			h2_enc_len(&resp, 7, sz, 0);
 			for (sz1 = 0; sz1 < sz; sz1++)
-				AZ(VSB_putc(&resp, tolower(hp->hd[u].b[sz1])));
+				VSB_putc(&resp, tolower(hp->hd[u].b[sz1]));
 
 		}
 
 		while (vct_islws(*++r))
 			continue;
 		sz = hp->hd[u].e - r;
-		AZ(h2_enc_len(&resp, 7, sz, 0));
-		AZ(VSB_bcat(&resp, r, sz));
+		h2_enc_len(&resp, 7, sz, 0);
+		VSB_bcat(&resp, r, sz);
+	}
+	if (VSB_finish(&resp)) {
+		// We ran out of workspace, return minimal 500
+		// XXX: VSC counter ?
+		r = (const char*)h2_500_resp;
+		sz = sizeof h2_500_resp;
+		sendbody = 0;
+	} else {
+		sz = VSB_len(&resp);
+		r = req->ws->f;
 	}
-	AZ(VSB_finish(&resp));
-	sz = VSB_len(&resp);
 
 	AZ(req->wrk->v1l);
 
@@ -256,7 +282,7 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 	H2_Send_Get(req->wrk, r2->h2sess, r2);
 	H2_Send(req->wrk, r2, H2_F_HEADERS,
 	    (sendbody ? 0 : H2FF_HEADERS_END_STREAM) | H2FF_HEADERS_END_HEADERS,
-	    sz, req->ws->f);
+	    sz, r);
 	H2_Send_Rel(r2->h2sess, r2);
 	req->acct.resp_hdrbytes += sz;
 
diff --git a/bin/varnishtest/tests/r02589.vtc b/bin/varnishtest/tests/r02589.vtc
new file mode 100644
index 000000000..418cc6833
--- /dev/null
+++ b/bin/varnishtest/tests/r02589.vtc
@@ -0,0 +1,29 @@
+varnishtest "workspace overrun on h/2 delivery"
+
+server s1 {
+	rxreq
+	txresp -hdrlen Foo 500
+} -start
+
+varnish v1 -vcl+backend  {
+	import vtc;
+	sub vcl_deliver {
+		vtc.workspace_alloc(client, -50);
+	}
+} -start
+
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.set debug +syncvsl"
+varnish v1 -cliok "param.set vsl_mask +H2TxHdr"
+varnish v1 -cliok "param.set vsl_mask +H2TxBody"
+
+
+client c1 {
+	stream 1 {
+		txreq
+		rxresp
+		expect resp.status == 500
+		expect resp.http.content-length == 0
+		expect resp.http.server == "Varnish"
+	} -run
+} -run

From hermunn at varnish-software.com  Wed Oct 24 09:29:19 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:19 +0000 (UTC)
Subject: [6.1] 78dbff5c8 Add a watchdog to worker pools.
Message-ID: <20181024092919.34858AFC9E@lists.varnish-cache.org>


commit 78dbff5c8cb5f25f90721b1e4b2bc185c22d4b69
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 3 10:12:59 2018 +0000

    Add a watchdog to worker pools.
    
    If the worker pool is configured too small, it can deadlock.
    
    Recovering from this would require a lot of complicated code, to
    discover queued but unscheduled tasks which can be cancelled
    (because the client went away or otherwise) and code to do the
    cancelling etc. etc.
    
    But fundamentally either people configured their pools wrong, in
    which case we want them to notice, or they are under DoS, in which
    case recovering gracefully is unlikely be a major improvement over
    a restart.
    
    Instead we implement a per-pool watchdog and kill the child process
    if nothing has been dequeued for too long.
    
    Default value 10 seconds, open to discussion.
    
    Band-aid for:   #2418
    
    Test-case by:   @Dridi
    
    Conflicts:
            bin/varnishd/cache/cache_pool.h

diff --git a/bin/varnishd/cache/cache_pool.h b/bin/varnishd/cache/cache_pool.h
index c954aee8c..a928130a4 100644
--- a/bin/varnishd/cache/cache_pool.h
+++ b/bin/varnishd/cache/cache_pool.h
@@ -53,9 +53,9 @@ struct pool {
 	uintmax_t			sdropped;
 	uintmax_t			rdropped;
 	uintmax_t			nqueued;
+	uintmax_t			ndequeued;
 	struct VSC_main			*a_stat;
 	struct VSC_main			*b_stat;
-
 	struct mempool			*mpl_req;
 	struct mempool			*mpl_sess;
 	struct waiter			*waiter;
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 88a0f0935..74c9bd4ba 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -342,6 +342,7 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk)
 			tp = VTAILQ_FIRST(&pp->queues[i]);
 			if (tp != NULL) {
 				pp->lqueue--;
+				pp->ndequeued--;
 				VTAILQ_REMOVE(&pp->queues[i], tp, list);
 				break;
 			}
@@ -488,6 +489,8 @@ pool_herder(void *priv)
 	struct worker *wrk;
 	double delay;
 	int wthread_min;
+	uintmax_t dq = (1ULL << 31);
+	double dqt;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 
@@ -495,6 +498,29 @@ pool_herder(void *priv)
 	THR_Init();
 
 	while (!pp->die || pp->nthr > 0) {
+		/*
+		 * If the worker pool is configured too small, we can
+		 * end up deadlocking it (see #2418 for details).
+		 *
+		 * Recovering from this would require a lot of complicated
+		 * code, and fundamentally, either people configured their
+		 * pools wrong, in which case we want them to notice, or
+		 * they are under DoS, in which case recovering gracefully
+		 * is unlikely be a major improvement.
+		 *
+		 * Instead we implement a watchdog and kill the worker if
+		 * nothing has been dequeued for that long.
+		 */
+		if (dq != pp->ndequeued) {
+			dq = pp->ndequeued;
+			dqt = VTIM_real();
+		} else if (pp->lqueue &&
+		    VTIM_real() - dqt > cache_param->wthread_watchdog) {
+			VSL(SLT_Error, 0,
+			   "Pool Herder: Queue does not move ql=%u dt=%f",
+			   pp->lqueue, VTIM_real() - dqt);
+			WRONG("Worker Pool Queue does not move");
+		}
 		wthread_min = cache_param->wthread_min;
 		if (pp->die)
 			wthread_min = 0;
diff --git a/bin/varnishd/common/common_param.h b/bin/varnishd/common/common_param.h
index 2366527b1..13ce0cb18 100644
--- a/bin/varnishd/common/common_param.h
+++ b/bin/varnishd/common/common_param.h
@@ -105,6 +105,7 @@ struct params {
 	double			wthread_add_delay;
 	double			wthread_fail_delay;
 	double			wthread_destroy_delay;
+	double			wthread_watchdog;
 	unsigned		wthread_stats_rate;
 	ssize_t			wthread_stacksize;
 	unsigned		wthread_queue_limit;
diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index 14bceac9e..25b6efba0 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -148,6 +148,15 @@ struct parspec WRK_parspec[] = {
 		"for at least this long, will be destroyed.",
 		EXPERIMENTAL | DELAYED_EFFECT,
 		"300", "seconds" },
+	{ "thread_pool_watchdog",
+		tweak_timeout, &mgt_param.wthread_watchdog,
+		"0.1", NULL,
+		"Thread queue stuck watchdog.\n"
+		"\n"
+		"If no queued work have been released for this long,"
+		" the worker process panics itself.",
+		EXPERIMENTAL,
+		"10", "seconds" },
 	{ "thread_pool_destroy_delay",
 		tweak_timeout, &mgt_param.wthread_destroy_delay,
 		"0.01", NULL,
diff --git a/bin/varnishtest/tests/r02418.vtc b/bin/varnishtest/tests/r02418.vtc
new file mode 100644
index 000000000..a03ca546c
--- /dev/null
+++ b/bin/varnishtest/tests/r02418.vtc
@@ -0,0 +1,72 @@
+varnishtest "h2 queuing deadlock"
+
+barrier b1 cond 2
+
+# A reserve of 1 thread in a pool of 3 leaves a maximum
+# of 2 running sessions, the streams will be queued (except
+# stream 0 that is part of the h2 session).
+
+varnish v1 -cliok "param.set thread_pools 1"
+varnish v1 -cliok "param.set thread_pool_min 3"
+varnish v1 -cliok "param.set thread_pool_max 3"
+varnish v1 -cliok "param.set thread_pool_reserve 1"
+varnish v1 -cliok "param.set thread_pool_watchdog 5"
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.set feature +no_coredump"
+
+varnish v1 -vcl {
+	backend b1 { .host = "${bad_ip}"; }
+	sub vcl_recv {
+		return (synth(200));
+	}
+} -start
+
+logexpect l1 -v v1 -g raw {
+	expect * *	Error	"Pool Herder: Queue does not move*"
+} -start
+
+# Starve the pool with h2 sessions
+
+client c1 {
+	txpri
+	stream 0 rxsettings -run
+
+	barrier b1 sync
+
+	stream 1 {
+		txreq
+		# can't be scheduled, don't rx
+	} -run
+} -start
+
+client c2 {
+	txpri
+	stream 0 rxsettings -run
+
+	barrier b1 sync
+
+	stream 1 {
+		txreq
+		# can't be scheduled, don't rx
+	} -run
+} -start
+
+client c1 -wait
+client c2 -wait
+
+varnish v1 -vsl_catchup
+
+# At this point c1 and c2 closed their connections
+
+client c3 {
+	txreq
+	delay 10
+} -run
+
+logexpect l1 -wait
+
+varnish v1 -cliok panic.show
+varnish v1 -cliok panic.clear
+
+varnish v1 -expectexit 0x20
+
diff --git a/include/tbl/params.h b/include/tbl/params.h
index e583b3977..7bebdeb4b 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -1195,6 +1195,24 @@ PARAM(
 	/* func */	NULL
 )
 
+/* actual location mgt_pool.c */
+PARAM(
+	/* name */	thread_pool_watchdog,
+	/* typ */	timeout,
+	/* min */	"0.1",
+	/* max */	NULL,
+	/* default */	"10.000",
+	/* units */	"seconds",
+	/* flags */	EXPERIMENTAL,
+	/* s-text */
+	"Thread queue stuck watchdog.\n"
+	"\n"
+	"If no queued work have been released for this long,"
+	" the worker process panics itself.",
+	/* l-text */	"",
+	/* func */	NULL
+)
+
 /* actual location mgt_pool.c */
 PARAM(
 	/* name */	thread_pool_destroy_delay,

From hermunn at varnish-software.com  Wed Oct 24 09:29:19 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:19 +0000 (UTC)
Subject: [6.1] 3acda6314 It's nice to know I'm still smarter than gcc
Message-ID: <20181024092919.5CDD0AFCB5@lists.varnish-cache.org>


commit 3acda6314c686d1f8feff0ad97a3442a2f0f7dbb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 3 13:34:33 2018 +0000

    It's nice to know I'm still smarter than gcc

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 74c9bd4ba..ad1fe08f6 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -490,7 +490,7 @@ pool_herder(void *priv)
 	double delay;
 	int wthread_min;
 	uintmax_t dq = (1ULL << 31);
-	double dqt;
+	double dqt = 0;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 

From hermunn at varnish-software.com  Wed Oct 24 09:29:19 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:19 +0000 (UTC)
Subject: [6.1] ed1696efc Don't poll VSM_Status() while there is work to do and
 no interruptions.
Message-ID: <20181024092919.7BBEBAFCCD@lists.varnish-cache.org>


commit ed1696efc92cb6a9aa96d2b8e586be8dbbb1736b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 4 07:17:58 2018 +0000

    Don't poll VSM_Status() while there is work to do and no interruptions.
    
    Fixes   #2788

diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c
index 03cd9f07a..b02644a81 100644
--- a/lib/libvarnishapi/vut.c
+++ b/lib/libvarnishapi/vut.c
@@ -420,7 +420,10 @@ VUT_Main(struct VUT *vut)
 			AZ(c);
 		}
 
-		i = VSLQ_Dispatch(vut->vslq, vut_dispatch, vut);
+		do
+			i = VSLQ_Dispatch(vut->vslq, vut_dispatch, vut);
+		while (i == vsl_more && !vut->sighup && !vut->sigusr1);
+
 		if (i == vsl_more)
 			continue;
 		else if (i == vsl_end) {

From hermunn at varnish-software.com  Wed Oct 24 09:29:19 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:19 +0000 (UTC)
Subject: [6.1] 13dacb710 restore vsl binary compatibility
Message-ID: <20181024092919.A0818AFCE3@lists.varnish-cache.org>


commit 13dacb7101714b2f6de0146e49b0a82dd302cf15
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Oct 4 16:45:27 2018 +0200

    restore vsl binary compatibility
    
    be69499219db0704a136046369884531ee20bc01 unnecessarily changed the
    values of most vsl tag enums and thus introduced an incompatibility
    with logs written with previous code.
    
    Fixes #2790

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 294da9431..49b27cf9b 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -65,20 +65,6 @@ SLTM(CLI, 0, "CLI communication",
 
 /*---------------------------------------------------------------------*/
 
-SLTM(SessError, 0, "Client connection accept failed",
-	"Accepting a client connection has failed.\n\n"
-	"The format is::\n\n"
-	"\t%s %s %s %d %d %s\n"
-	"\t|  |  |  |  |  |\n"
-	"\t|  |  |  |  |  +- Detailed error message\n"
-	"\t|  |  |  |  +---- Error Number (errno) from accept(2)\n"
-	"\t|  |  |  +------- File descriptor number\n"
-	"\t|  |  +---------- Local TCP port / 0 for UDS\n"
-	"\t|  +------------- Local IPv4/6 address / 0.0.0.0 for UDS\n"
-	"\t+---------------- Socket name (from -a argument)\n"
-	"\n"
-)
-
 SLTM(SessOpen, 0, "Client connection opened",
 	"The first record for a client connection, with the socket-endpoints"
 	" of the connection.\n\n"
@@ -642,6 +628,20 @@ SLTM(Filters, 0, "Body filters",
 	"List of filters applied to the body"
 )
 
+SLTM(SessError, 0, "Client connection accept failed",
+	"Accepting a client connection has failed.\n\n"
+	"The format is::\n\n"
+	"\t%s %s %s %d %d %s\n"
+	"\t|  |  |  |  |  |\n"
+	"\t|  |  |  |  |  +- Detailed error message\n"
+	"\t|  |  |  |  +---- Error Number (errno) from accept(2)\n"
+	"\t|  |  |  +------- File descriptor number\n"
+	"\t|  |  +---------- Local TCP port / 0 for UDS\n"
+	"\t|  +------------- Local IPv4/6 address / 0.0.0.0 for UDS\n"
+	"\t+---------------- Socket name (from -a argument)\n"
+	"\n"
+)
+
 #undef NODEF_NOTICE
 #undef SLTM
 

From hermunn at varnish-software.com  Wed Oct 24 09:29:19 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:19 +0000 (UTC)
Subject: [6.1] 173ca9977 changelog tlc
Message-ID: <20181024092919.C91AAAFD02@lists.varnish-cache.org>


commit 173ca9977f9ef75a4101d924636efc32bc00b42d
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Oct 4 17:28:09 2018 +0200

    changelog tlc

diff --git a/doc/changes.rst b/doc/changes.rst
index f9a1eb4c1..3786d408d 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -27,6 +27,46 @@ individual releases. These documents are updated as part of the
 release process.
 
 
+* fixed ``varnishhist`` display error (2780_)
+
+* ``libvarnish``: ``VRT_VSA_GetPtr`` renamed to ``VSA_GetPtr``
+
+* Improve accuracy of statistics (VSC)
+
+* In ``Error: out of workspace`` log entries, the workspace name is
+  now reported in lowercase
+
+* Adjust code generator python tools to python 3 and prefer python 3
+  over python 2 where available
+
+* fix some stats metrics (vsc) which were wrongly marked as _gauge_
+
+* fix ``varnishd -I`` (2782_)
+
+* fix ``varnishstat -f`` in curses mode (interactively, without
+  ``-1``, 2787_)
+
+* Handle an out-of-workspace condition in HTTP/2 delivery more
+  gracefully (2589_)
+
+* added a thread pool watchdog which will restart the worker process
+  if scheduling tasks onto worker threads appears stuck. The new
+  parameter ``thread_pool_watchdog`` configures it. (2418_)
+
+* Improved varnish log client performance (2788_)
+
+* Fixed regression introduced just before 6.1.0 release which caused
+  an unnecessary incompatibility with VSL files written by previous
+  versions. (2790_)
+
+.. _2780: https://github.com/varnishcache/varnish-cache/issues/2780
+.. _2782: https://github.com/varnishcache/varnish-cache/issues/2782
+.. _2787: https://github.com/varnishcache/varnish-cache/issues/2787
+.. _2589: https://github.com/varnishcache/varnish-cache/issues/2589
+.. _2418: https://github.com/varnishcache/varnish-cache/issues/2418
+.. _2788: https://github.com/varnishcache/varnish-cache/issues/2788
+.. _2790: https://github.com/varnishcache/varnish-cache/issues/2790
+
 ================================
 Varnish Cache 6.1.0 (2018-09-17)
 ================================

From hermunn at varnish-software.com  Wed Oct 24 09:29:19 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:19 +0000 (UTC)
Subject: [6.1] 183dd3ac1 debloat the vtim test
Message-ID: <20181024092919.E5BD1AFD1E@lists.varnish-cache.org>


commit 183dd3ac14d59354c00de3e480526ca57ebf8677
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Oct 4 20:24:10 2018 +0200

    debloat the vtim test

diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index 4bef5e711..a9fbc7cb2 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -529,12 +529,26 @@ bench()
 	    e - s, i, 1e9 * (e - s) / i, t);
 }
 
+void
+parse_check(time_t t, const char *s)
+{
+	double tt;
+	char buf[BUFSIZ];
+
+	tt = VTIM_parse(s);
+	if (tt != t) {
+		VTIM_format(tt, buf);
+		printf("  fm: %12jd <%s>\n", (intmax_t)t, s);
+		printf("  to: %12.0f <%s>\n", tt, buf);
+		exit(2);
+	}
+}
+
 int
 main(int argc, char **argv)
 {
 	time_t t;
 	struct tm tm;
-	double tt;
 	char buf[BUFSIZ];
 	char buf1[BUFSIZ];
 
@@ -553,41 +567,17 @@ main(int argc, char **argv)
 			    buf1, buf, (intmax_t)t);
 			exit(2);
 		}
-		tt = VTIM_parse(buf1);
-		if (tt != t) {
-			VTIM_format(tt, buf);
-			printf("  fm: %12jd <%s>\n", (intmax_t)t, buf1);
-			printf("  to: %12.0f <%s>\n", tt, buf);
-			exit(2);
-		}
+		parse_check(t, buf1);
 
 		strftime(buf1, sizeof buf1, "%a %b %e %T %Y", &tm);
-		tt = VTIM_parse(buf1);
-		if (tt != t) {
-			VTIM_format(tt, buf);
-			printf("  fm: %12jd <%s>\n", (intmax_t)t, buf1);
-			printf("  to: %12.0f <%s>\n", tt, buf);
-			exit(2);
-		}
+		parse_check(t, buf1);
 
 		strftime(buf1, sizeof buf1, "%Y-%m-%dT%T", &tm);
-		tt = VTIM_parse(buf1);
-		if (tt != t) {
-			VTIM_format(tt, buf);
-			printf("  fm: %12jd <%s>\n", (intmax_t)t, buf1);
-			printf("  to: %12.0f <%s>\n", tt, buf);
-			exit(2);
-		}
+		parse_check(t, buf1);
 
 		if (tm.tm_year >= 69 && tm.tm_year < 169) {
 			strftime(buf1, sizeof buf1, "%A, %d-%b-%y %T GMT", &tm);
-			tt = VTIM_parse(buf1);
-			if (tt != t) {
-				VTIM_format(tt, buf);
-				printf("  fm: %12jd <%s>\n", (intmax_t)t, buf1);
-				printf("  to: %12.0f <%s>\n", tt, buf);
-				exit(2);
-			}
+			parse_check(t, buf1);
 		}
 	}
 

From hermunn at varnish-software.com  Wed Oct 24 09:29:20 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:20 +0000 (UTC)
Subject: [6.1] 208583f38 trivial vtim printf benchmark
Message-ID: <20181024092920.48AA4AFD2C@lists.varnish-cache.org>


commit 208583f389d14de6b15a00a315d26cb66c4c844b
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Oct 5 09:56:27 2018 +0200

    trivial vtim printf benchmark

diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index a9fbc7cb2..f9de6244e 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -511,6 +511,7 @@ bench()
 {
 	double s, e, t;
 	int i;
+	char buf[64];
 
 	t = 0;
 	s = VTIM_real();
@@ -527,6 +528,17 @@ bench()
 	e = VTIM_real();
 	printf("mono: %fs / %d = %fns - tst val %f\n",
 	    e - s, i, 1e9 * (e - s) / i, t);
+
+	t = 0;
+	s = VTIM_mono();
+	for (i=0; i<100000; i++) {
+		snprintf(buf, sizeof(buf), "%.6f", s);
+		t += buf[4];
+	}
+	e = VTIM_mono();
+	printf("%s\n", buf);
+	printf("printf: %fs / %d = %fns - tst val %f\n",
+	    e - s, i, 1e9 * (e - s) / i, t);
 }
 
 void

From hermunn at varnish-software.com  Wed Oct 24 09:29:20 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:20 +0000 (UTC)
Subject: [6.1] 34ad8092b Don't use txprio to open streams in tests
Message-ID: <20181024092920.84EA9AFD37@lists.varnish-cache.org>


commit 34ad8092b2ca6bba8559023da3d853e072d57725
Author: Carlo Cannas <carlo94 at gmail.com>
Date:   Sun Sep 23 03:34:02 2018 +0200

    Don't use txprio to open streams in tests
    
    As per spec a client can only send a HEADERS frame to cause a stream to
    transition from the "idle" state to "open". A PRIORITY frame can be sent to an
    "idle" stream, but it will remain in that state.
    
    rfc7540,l,916,940
    https://tools.ietf.org/html/rfc7540#section-5.1
    
    To open a stream the command
    
    txreq -nostrend
    
    can be used. The -nostrend option will ensure that the stream won't transition
    to a "half-closed" state.

diff --git a/bin/varnishtest/tests/t02003.vtc b/bin/varnishtest/tests/t02003.vtc
index b355fd4b8..215171cf6 100644
--- a/bin/varnishtest/tests/t02003.vtc
+++ b/bin/varnishtest/tests/t02003.vtc
@@ -43,10 +43,10 @@ client c1 {
 		expect goaway.err == PROTOCOL_ERROR
 	} -start
 	stream 3 {
-		txprio
+		txreq
 	} -run
 	stream 1 {
-		txprio
+		txreq
 	} -run
 	stream 0 -wait
 } -run
@@ -63,13 +63,13 @@ varnish v1 -expect MEMPOOL.sess1.live == 0
 
 client c1 {
 	stream 1 {
-		txprio
+		txreq -nostrend
 		txwinup -size 0
 		rxrst
 		expect rst.err == PROTOCOL_ERROR
 	} -run
 	stream 3 {
-		txprio
+		txreq -nostrend
 		txwinup -size 0x40000000
 		txwinup -size 0x40000000
 		rxrst
@@ -81,7 +81,7 @@ client c1 {
 		expect goaway.err == FRAME_SIZE_ERROR
 	} -start
 	stream 5 {
-		txprio
+		txreq -nostrend
 		sendhex "000003 08 00 00000005"
 		delay .1
 		sendhex 01
@@ -167,7 +167,7 @@ client c1 {
 		expect goaway.laststream == 1
 	} -start
 	stream 1 {
-		txprio
+		txreq -nostrend
 		sendhex "000008 05 00 00000001 0001020304050607"
 	} -run
 	stream 0 -wait
@@ -200,7 +200,7 @@ client c1 {
 		expect goaway.laststream == 1
 	} -start
 	stream 1 {
-		txprio
+		txreq -nostrend
 		# RST wrong length
 		sendhex "000005 03 00 00000001 0000000800"
 	} -run

From hermunn at varnish-software.com  Wed Oct 24 09:29:20 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:20 +0000 (UTC)
Subject: [6.1] 360c9601d Move header block frame sequence check earlier
Message-ID: <20181024092920.AB50FAFD4C@lists.varnish-cache.org>


commit 360c9601dd4e9e2af9bf29acd0f0ab90f8e76515
Author: Carlo Cannas <carlo94 at gmail.com>
Date:   Sun Sep 23 03:34:02 2018 +0200

    Move header block frame sequence check earlier
    
    This moves it before the new stream object creation, so we save ourselves an
    useless allocation and initialization of a stream object which would be never
    used and straight killed.
    
    This also simplifies upcoming commits.

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 070bf2cad..b106f9e36 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -923,6 +923,10 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2, h2_frame h2f)
 		if (r2->stream == h2->rxf_stream)
 			break;
 
+	if (h2->new_req != NULL &&
+	    !(r2 && h2->new_req == r2->req && h2f == H2_F_CONTINUATION))
+		return (H2CE_PROTOCOL_ERROR);	// rfc7540,l,1859,1863
+
 	if (r2 == NULL && h2f->act_sidle == 0) {
 		if (h2->rxf_stream <= h2->highest_stream)
 			return (H2CE_PROTOCOL_ERROR);	// rfc7540,l,1153,1158
@@ -940,10 +944,6 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2, h2_frame h2f)
 		AN(r2);
 	}
 
-	if (h2->new_req != NULL &&
-	    !(r2 && h2->new_req == r2->req && h2f == H2_F_CONTINUATION))
-		return (H2CE_PROTOCOL_ERROR);	// rfc7540,l,1859,1863
-
 	h2e = h2f->rxfunc(wrk, h2, r2);
 	if (h2e == 0)
 		return (0);
diff --git a/bin/varnishtest/tests/r02387.vtc b/bin/varnishtest/tests/r02387.vtc
index 34863a057..d2c9796e7 100644
--- a/bin/varnishtest/tests/r02387.vtc
+++ b/bin/varnishtest/tests/r02387.vtc
@@ -30,7 +30,7 @@ client c1 {
 	} -run
 	stream 0 {
 		rxgoaway
-		expect goaway.laststream == "3"
+		expect goaway.laststream == "1"
 		expect goaway.err == PROTOCOL_ERROR
 	} -run
 } -run

From hermunn at varnish-software.com  Wed Oct 24 09:29:20 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:20 +0000 (UTC)
Subject: [6.1] 7a56c831e Allow PRIORITY frames on closed streams
Message-ID: <20181024092920.CDF36AFD58@lists.varnish-cache.org>


commit 7a56c831e21fb0d22d4bea4af1ec0c24503005ea
Author: Carlo Cannas <carlo94 at gmail.com>
Date:   Sun Sep 23 03:34:02 2018 +0200

    Allow PRIORITY frames on closed streams
    
    Currently Varnish doesn't allow PRIORITY frames to be received on closed
    streams: it treats it as a protocol violation and replies with a GOAWAY.
    
    This is not spec compliant, rfc7540 states:
    
    The PRIORITY frame can be sent for a stream in the "idle" or "closed"
    state.
    rfc7540,l,1947,1948
    
    The PRIORITY frame can be sent on a stream in any state
    rfc7540,l,1938,1938
    
    https://tools.ietf.org/html/rfc7540#section-6.3
    
    This behaviour can be triggered by real-world browsers: Chrome 69 has been
    observed sending PRIORITY frames which are received by Varnish after a stream
    has been closed (and cleaned by h2_sweep). When that happens the connection is
    closed and Chrome aborts the loading of all the resources which started to
    load on that connection.
    
    This commit solves the issue by avoiding all the stream creation code and its
    checks to be performed when a PRIORITY frame is received.
    This moves all the stream creation logic inside h2_rx_headers, the only other
    frame which is allowed on idle streams.
    
    This also fixes the concurrent streams counter and highest_stream: they should
    be updated only when a stream enters the "open" state (or "reserved" if
    Varnish used served push) but currently a PRIORITY frame on an idle stream
    affects them.
    
    https://tools.ietf.org/html/rfc7540#section-5.1.1
    rfc7540,l,1153,1156
    rfc7540,l,1193,1198
    rfc7540,l,1530,1533
    
    Fixes: #2775

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index b106f9e36..250af1806 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -382,7 +382,7 @@ h2_rx_priority(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 
 	(void)wrk;
 	ASSERT_RXTHR(h2);
-	xxxassert(r2->stream & 1);
+	(void)r2;
 	return (0);
 }
 
@@ -616,7 +616,21 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 	size_t l;
 
 	ASSERT_RXTHR(h2);
+
+	if (r2 == NULL) {
+		if (h2->rxf_stream <= h2->highest_stream)
+			return (H2CE_PROTOCOL_ERROR);	// rfc7540,l,1153,1158
+		if (h2->refcnt >= h2->local_settings.max_concurrent_streams) {
+			VSLb(h2->vsl, SLT_Debug,
+			     "H2: stream %u: Hit maximum number of "
+			     "concurrent streams", h2->rxf_stream);
+			return (H2SE_REFUSED_STREAM);	// rfc7540,l,1200,1205
+		}
+		h2->highest_stream = h2->rxf_stream;
+		r2 = h2_new_req(wrk, h2, h2->rxf_stream, NULL);
+	}
 	AN(r2);
+
 	if (r2->state != H2_S_IDLE)
 		return (H2CE_PROTOCOL_ERROR);	// XXX spec ?
 	r2->state = H2_S_OPEN;
@@ -927,23 +941,6 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2, h2_frame h2f)
 	    !(r2 && h2->new_req == r2->req && h2f == H2_F_CONTINUATION))
 		return (H2CE_PROTOCOL_ERROR);	// rfc7540,l,1859,1863
 
-	if (r2 == NULL && h2f->act_sidle == 0) {
-		if (h2->rxf_stream <= h2->highest_stream)
-			return (H2CE_PROTOCOL_ERROR);	// rfc7540,l,1153,1158
-		if (h2->refcnt >= h2->local_settings.max_concurrent_streams) {
-			VSLb(h2->vsl, SLT_Debug,
-			     "H2: stream %u: Hit maximum number of "
-			     "concurrent streams", h2->rxf_stream);
-			// rfc7540,l,1200,1205
-			h2_tx_rst(wrk, h2, h2->req0, h2->rxf_stream,
-				H2SE_REFUSED_STREAM);
-			return (0);
-		}
-		h2->highest_stream = h2->rxf_stream;
-		r2 = h2_new_req(wrk, h2, h2->rxf_stream, NULL);
-		AN(r2);
-	}
-
 	h2e = h2f->rxfunc(wrk, h2, r2);
 	if (h2e == 0)
 		return (0);
@@ -993,7 +990,6 @@ static int
 h2_sweep(struct worker *wrk, struct h2_sess *h2)
 {
 	int tmo = 0;
-	int nprio = 0;
 	struct h2_req *r2, *r22;
 
 	ASSERT_RXTHR(h2);
@@ -1025,20 +1021,18 @@ h2_sweep(struct worker *wrk, struct h2_sess *h2)
 			}
 			break;
 		case H2_S_IDLE:
-			/* This stream was created from receiving a
-			 * PRIORITY frame, and should not be counted
-			 * as an active stream keeping the connection
-			 * open. */
-			AZ(r2->scheduled);
-			nprio++;
-			break;
+			/* Current code make this unreachable: h2_new_req is
+			 * only called inside h2_rx_headers, which immediately
+			 * sets the new stream state to H2_S_OPEN */
+			/* FALLTHROUGH */
 		default:
+			WRONG("Wrong h2 stream state");
 			break;
 		}
 	}
 	if (tmo)
 		return (0);
-	return ((h2->refcnt - nprio) > 1);
+	return (h2->refcnt > 1);
 }
 
 
diff --git a/bin/varnishtest/tests/r02775.vtc b/bin/varnishtest/tests/r02775.vtc
new file mode 100644
index 000000000..de66c10d1
--- /dev/null
+++ b/bin/varnishtest/tests/r02775.vtc
@@ -0,0 +1,23 @@
+varnishtest "Regression test for #2775: allow PRIORITY on closed stream"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {} -start
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.set debug +syncvsl"
+
+client c1 {
+	stream 1 {
+		txreq
+		rxresp
+
+		txprio
+	} -run
+	stream 3 {
+		txreq
+		rxresp
+	} -run
+} -run

From hermunn at varnish-software.com  Wed Oct 24 09:29:20 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:20 +0000 (UTC)
Subject: [6.1] 18cecf312 Hardening of the h2_frame_f callbacks
Message-ID: <20181024092920.EA1D5AFD5E@lists.varnish-cache.org>


commit 18cecf312b97f1bdf61e576e054f999b4e393003
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Oct 5 11:16:59 2018 +0200

    Hardening of the h2_frame_f callbacks
    
    And by the way, they are known as h2_rxframe_f these days!
    
    Refs #2781

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 250af1806..6caa358bf 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -275,11 +275,14 @@ h2_vsl_frame(const struct h2_sess *h2, const void *ptr, size_t len)
 /**********************************************************************
  */
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_ping(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
+	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
+	assert(r2 == h2->req0);
 
 	if (h2->rxf_len != 8)				// rfc7540,l,2364,2366
 		return (H2CE_FRAME_SIZE_ERROR);
@@ -296,26 +299,27 @@ h2_rx_ping(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 /**********************************************************************
  */
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_push_promise(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 
-	(void)wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
+	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); /* XXX: wasteful allocation? */
 	// rfc7540,l,2262,2267
-	(void)r2;
 	return (H2CE_PROTOCOL_ERROR);
 }
 
 /**********************************************************************
  */
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_rst_stream(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 
-	(void)wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
+	CHECK_OBJ_ORNULL(r2, H2_REQ_MAGIC);
 
 	if (h2->rxf_len != 4)			// rfc7540,l,2003,2004
 		return (H2CE_FRAME_SIZE_ERROR);
@@ -328,13 +332,15 @@ h2_rx_rst_stream(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 /**********************************************************************
  */
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_goaway(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 
-	(void)wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
-	(void)r2;
+	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
+	assert(r2 == h2->req0);
+
 	h2->goaway_last_stream = vbe32dec(h2->rxf_data);
 	h2->error = h2_connectionerror(vbe32dec(h2->rxf_data + 4));
 	Lck_Lock(&h2->sess->mtx);
@@ -346,13 +352,15 @@ h2_rx_goaway(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 /**********************************************************************
  */
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_window_update(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 	uint32_t wu;
 
-	(void)wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
+	CHECK_OBJ_ORNULL(r2, H2_REQ_MAGIC);
+
 	if (h2->rxf_len != 4)
 		return (H2CE_FRAME_SIZE_ERROR);
 	wu = vbe32dec(h2->rxf_data) & ~(1LU<<31);
@@ -376,13 +384,13 @@ h2_rx_window_update(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
  * Incoming PRIORITY, possibly an ACK of one we sent.
  */
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_priority(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 
-	(void)wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
-	(void)r2;
+	CHECK_OBJ_ORNULL(r2, H2_REQ_MAGIC);
 	return (0);
 }
 
@@ -478,17 +486,19 @@ h2_set_setting(struct h2_sess *h2, const uint8_t *d)
 	return (0);
 }
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_settings(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 	const uint8_t *p;
 	unsigned l;
 	h2_error retval = 0;
 
-	AN(wrk);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
-	AN(r2);
+	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
+	assert(r2 == h2->req0);
 	AZ(h2->rxf_stream);
+
 	if (h2->rxf_flags == H2FF_SETTINGS_ACK) {
 		if (h2->rxf_len > 0)			// rfc7540,l,2047,2049
 			return (H2CE_FRAME_SIZE_ERROR);
@@ -607,7 +617,7 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2,
 	return (0);
 }
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 	struct req *req;
@@ -615,6 +625,7 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 	const uint8_t *p;
 	size_t l;
 
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
 
 	if (r2 == NULL) {
@@ -629,7 +640,7 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 		h2->highest_stream = h2->rxf_stream;
 		r2 = h2_new_req(wrk, h2, h2->rxf_stream, NULL);
 	}
-	AN(r2);
+	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
 
 	if (r2->state != H2_S_IDLE)
 		return (H2CE_PROTOCOL_ERROR);	// XXX spec ?
@@ -695,13 +706,16 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 
 /**********************************************************************/
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_continuation(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 	struct req *req;
 	h2_error h2e;
 
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
+	CHECK_OBJ_ORNULL(r2, H2_REQ_MAGIC);
+
 	if (r2 == NULL || r2->state != H2_S_OPEN || r2->req != h2->new_req)
 		return (H2CE_PROTOCOL_ERROR);	// XXX spec ?
 	req = r2->req;
@@ -723,15 +737,17 @@ h2_rx_continuation(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 
 /**********************************************************************/
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 	int w1 = 0, w2 = 0;
 	char buf[4];
 	unsigned wi;
 
-	(void)wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
+	CHECK_OBJ_ORNULL(r2, H2_REQ_MAGIC);
+
 	if (r2 == NULL || !r2->scheduled)
 		return (0);
 	if (r2->state >= H2_S_CLOS_REM) {

From hermunn at varnish-software.com  Wed Oct 24 09:29:21 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:21 +0000 (UTC)
Subject: [6.1] cc1175028 Fix assertion for PUSH_PROMISE frames
Message-ID: <20181024092921.210DFAFD69@lists.varnish-cache.org>


commit cc11750283b35fe487cccf86bd107192430300d6
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Oct 5 12:09:28 2018 +0200

    Fix assertion for PUSH_PROMISE frames
    
    r2 can be either null or not.
    
    Test case by @daghf
    
    Refs #2781

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 6caa358bf..7d040b7d0 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -305,7 +305,7 @@ h2_rx_push_promise(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
-	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); /* XXX: wasteful allocation? */
+	CHECK_OBJ_ORNULL(r2, H2_REQ_MAGIC);
 	// rfc7540,l,2262,2267
 	return (H2CE_PROTOCOL_ERROR);
 }
diff --git a/bin/varnishtest/tests/t02003.vtc b/bin/varnishtest/tests/t02003.vtc
index 215171cf6..1d62ac986 100644
--- a/bin/varnishtest/tests/t02003.vtc
+++ b/bin/varnishtest/tests/t02003.vtc
@@ -173,6 +173,21 @@ client c1 {
 	stream 0 -wait
 } -run
 
+client c1 {
+	stream 0 {
+		rxgoaway
+		expect goaway.err == PROTOCOL_ERROR
+		expect goaway.laststream == 1
+	} -start
+	stream 1 {
+		txreq
+		rxresp
+		delay .1
+		# send a PUSH_PROMISE after a request
+		sendhex "000008 05 00 00000001 0001020304050607"
+	} -start
+} -run
+
 varnish v1 -vsl_catchup
 
 varnish v1 -expect MEMPOOL.req0.live == 0

From hermunn at varnish-software.com  Wed Oct 24 09:29:21 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:21 +0000 (UTC)
Subject: [6.1] dedbc9d10 gethrtime() is now slower than clock_gettime() on
 modern Solarisen
Message-ID: <20181024092921.4C2FFAFD7B@lists.varnish-cache.org>


commit dedbc9d10841b1c05dc8899b1d66dc2634d13356
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Oct 5 12:25:42 2018 +0200

    gethrtime() is now slower than clock_gettime() on modern Solarisen
    
    Throw out the conventional wisdom and base the decision on a micro
    benchmark.
    
    clock_gettime() is now preferred if it is consistently at least
    double as fast as gethrtime(), which is the case on varnishdev-il,
    the SmartOS vtest machine.
    
    config.log gives details on the performance check, sample output
    below:
    
    configure:22703: ./conftest
    hrtime              45989530 check 16748699083977959327
    clock_gettime        4119385 check 16748701613138517215
    ...
    hrtime              48113108 check 16748749015170035860
    clock_gettime        4020802 check 16748751585081458308
    clock_gettime wins 10/10

diff --git a/configure.ac b/configure.ac
index 42e863fbd..83fcbde78 100644
--- a/configure.ac
+++ b/configure.ac
@@ -355,6 +355,63 @@ AC_CHECK_FUNCS([clock_gettime])
 AC_CHECK_FUNCS([gethrtime])
 LIBS="${save_LIBS}"
 
+if test "x$ac_cv_func_gethrtime" = xyes && \
+   test "x$ac_cv_func_clock_gettime" = xyes ; then
+  AC_MSG_CHECKING(if clock_gettime is faster than gethrtime)
+  AC_RUN_IFELSE(
+    [AC_LANG_PROGRAM([[
+#include <sys/time.h>
+#include <time.h>
+#include <stdio.h>
+
+static hrtime_t cl()
+{
+	struct timespec ts;
+
+	(void) clock_gettime(CLOCK_MONOTONIC, &ts);
+	return (ts.tv_sec * 1e9 + ts.tv_nsec);
+}
+    ]],[[
+	hrtime_t s, c, e, t_hr, t_cl;
+	int i, r, wins;
+
+	wins = 0;
+	for (r = 0; r < 10; r++) {
+		c = 0;
+		s = gethrtime();
+		for (i=0; i<100000; i++)
+			c += gethrtime();
+		e = gethrtime();
+		t_hr = e - s;
+		fprintf(stderr, "hrtime\t\t%12lu check %lu\n",
+		    (unsigned long)t_hr, (unsigned long)c);
+
+		c = 0;
+		s = gethrtime();
+		for (i=0; i<100000; i++)
+			c += cl();
+		e = gethrtime();
+		t_cl = e - s;
+		fprintf(stderr, "clock_gettime\t%12lu check %lu\n",
+		    (unsigned long)t_cl, (unsigned long)c);
+
+		if (t_cl * 2 < t_hr)
+			wins++;
+	}
+	fprintf(stderr, "clock_gettime wins %d/%d\n", wins, r);
+	if (2 * wins >= r)
+		return (0);
+	return (1);
+    ]])],
+    [AC_MSG_RESULT(yes)
+     AC_DEFINE([USE_GETHRTIME], [0], [whether to use gethrtime])
+    ],
+    [AC_MSG_RESULT(no)
+     AC_DEFINE([USE_GETHRTIME], [1], [whether to use gethrtime])
+    ]
+  )
+fi
+
 # --enable-kqueue
 AC_ARG_ENABLE(kqueue,
     AS_HELP_STRING([--enable-kqueue],
diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index f9de6244e..39d5ca342 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -118,15 +118,16 @@ init(void)
 #endif
 
 /*
- * Note on Solaris: for some reason, clock_gettime(CLOCK_MONOTONIC, &ts) is not
- * implemented in assembly, but falls into a syscall, while gethrtime() doesn't,
- * so we save a syscall by using gethrtime() if it is defined.
+ * On older Solaris-incarnations, gethrtime() was faster than
+ * clock_gettime(CLOCK_MONOTONIC). Our configure script prefers
+ * clock_gettime if it is consistently at least twice as fast as
+ * gethrtime(), which is the case on modern Solaris descendents.
  */
 
 double
 VTIM_mono(void)
 {
-#ifdef HAVE_GETHRTIME
+#if defined(HAVE_GETHRTIME) && USE_GETHRTIME
 	return (gethrtime() * 1e-9);
 #elif  HAVE_CLOCK_GETTIME
 	struct timespec ts;

From hermunn at varnish-software.com  Wed Oct 24 09:29:21 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:21 +0000 (UTC)
Subject: [6.1] d158533a5 fix a minor oversight
Message-ID: <20181024092921.7778EAFD87@lists.varnish-cache.org>


commit d158533a5015985930b236f024c249e6eac17ed5
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Oct 5 12:38:02 2018 +0200

    fix a minor oversight
    
    I failed to consider the hypothetical case that there is only gethrtime()
    and no clock_gettime(CLOCK_MONOTONIC).

diff --git a/configure.ac b/configure.ac
index 83fcbde78..36e23d534 100644
--- a/configure.ac
+++ b/configure.ac
@@ -355,6 +355,7 @@ AC_CHECK_FUNCS([clock_gettime])
 AC_CHECK_FUNCS([gethrtime])
 LIBS="${save_LIBS}"
 
+AC_DEFINE([USE_GETHRTIME], [1], [whether to use gethrtime])
 if test "x$ac_cv_func_gethrtime" = xyes && \
    test "x$ac_cv_func_clock_gettime" = xyes ; then
   AC_MSG_CHECKING(if clock_gettime is faster than gethrtime)
@@ -407,7 +408,6 @@ static hrtime_t cl()
      AC_DEFINE([USE_GETHRTIME], [0], [whether to use gethrtime])
     ],
     [AC_MSG_RESULT(no)
-     AC_DEFINE([USE_GETHRTIME], [1], [whether to use gethrtime])
     ]
   )
 fi

From hermunn at varnish-software.com  Wed Oct 24 09:29:21 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:21 +0000 (UTC)
Subject: [6.1] 7e7e91c2d Polish
Message-ID: <20181024092921.A73E9AFDA5@lists.varnish-cache.org>


commit 7e7e91c2d90ead26557f42ebaac49c36168bc56b
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Fri Oct 5 14:07:03 2018 +0100

    Polish

diff --git a/configure.ac b/configure.ac
index 36e23d534..b543949b3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -355,7 +355,6 @@ AC_CHECK_FUNCS([clock_gettime])
 AC_CHECK_FUNCS([gethrtime])
 LIBS="${save_LIBS}"
 
-AC_DEFINE([USE_GETHRTIME], [1], [whether to use gethrtime])
 if test "x$ac_cv_func_gethrtime" = xyes && \
    test "x$ac_cv_func_clock_gettime" = xyes ; then
   AC_MSG_CHECKING(if clock_gettime is faster than gethrtime)
@@ -405,9 +404,9 @@ static hrtime_t cl()
 	return (1);
     ]])],
     [AC_MSG_RESULT(yes)
-     AC_DEFINE([USE_GETHRTIME], [0], [whether to use gethrtime])
     ],
     [AC_MSG_RESULT(no)
+     AC_DEFINE([USE_GETHRTIME], [1], [Define if gethrtime is preferred])
     ]
   )
 fi
diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index 39d5ca342..5c6503c7e 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -127,14 +127,14 @@ init(void)
 double
 VTIM_mono(void)
 {
-#if defined(HAVE_GETHRTIME) && USE_GETHRTIME
-	return (gethrtime() * 1e-9);
-#elif  HAVE_CLOCK_GETTIME
+#if defined(HAVE_CLOCK_GETTIME) && !defined(USE_GETHRTIME)
 	struct timespec ts;
 
 	AZ(clock_gettime(CLOCK_MONOTONIC, &ts));
 	return (ts.tv_sec + 1e-9 * ts.tv_nsec);
-#elif  defined(__MACH__)
+#elif defined(HAVE_GETHRTIME)
+	return (gethrtime() * 1e-9);
+#elif defined(__MACH__)
 	uint64_t mt = mach_absolute_time() - mt_base;
 
 	return (mt * mt_scale);

From hermunn at varnish-software.com  Wed Oct 24 09:29:21 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:21 +0000 (UTC)
Subject: [6.1] 96ed2b8d7 add phk's suggestion to the micro-benchmark
Message-ID: <20181024092921.CD6EBAFDAF@lists.varnish-cache.org>


commit 96ed2b8d7cdbea53317298ce9fb2ec5571cca607
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 8 10:29:29 2018 +0200

    add phk's suggestion to the micro-benchmark

diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index 5c6503c7e..822f522d2 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -537,9 +537,20 @@ bench()
 		t += buf[4];
 	}
 	e = VTIM_mono();
-	printf("%s\n", buf);
-	printf("printf: %fs / %d = %fns - tst val %f\n",
-	    e - s, i, 1e9 * (e - s) / i, t);
+	printf("printf %%.6f: %fs / %d = %fns - tst val %f %s\n",
+	    e - s, i, 1e9 * (e - s) / i, t, buf);
+
+	t = 0;
+	s = VTIM_mono();
+	for (i=0; i<100000; i++) {
+		snprintf(buf, sizeof(buf), "%ju.%06ju",
+			 (uint64_t)floor(s),
+			 (uint64_t)floor((s * 1e6)) % 1000000UL);
+		t += buf[4];
+	}
+	e = VTIM_mono();
+	printf("printf %%ju.%%06ju: %fs / %d = %fns - tst val %f %s\n",
+	    e - s, i, 1e9 * (e - s) / i, t, buf);
 }
 
 void

From hermunn at varnish-software.com  Wed Oct 24 09:29:21 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:21 +0000 (UTC)
Subject: [6.1] 721210c7d style(9)
Message-ID: <20181024092922.00B14AFDBB@lists.varnish-cache.org>


commit 721210c7d5c1940e66d286f60b34d0e0c4e76ddb
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 8 10:42:15 2018 +0200

    style(9)

diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index 822f522d2..c774e2ab8 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -544,8 +544,8 @@ bench()
 	s = VTIM_mono();
 	for (i=0; i<100000; i++) {
 		snprintf(buf, sizeof(buf), "%ju.%06ju",
-			 (uint64_t)floor(s),
-			 (uint64_t)floor((s * 1e6)) % 1000000UL);
+		    (uint64_t)floor(s),
+		    (uint64_t)floor((s * 1e6)) % 1000000UL);
 		t += buf[4];
 	}
 	e = VTIM_mono();

From hermunn at varnish-software.com  Wed Oct 24 09:29:22 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:22 +0000 (UTC)
Subject: [6.1] c335240b8 Check we have space before adding the Date header
Message-ID: <20181024092922.20BB9AFDC5@lists.varnish-cache.org>


commit c335240b86bf805dfbb50041467c74c3916897d9
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Oct 8 10:23:38 2018 +0100

    Check we have space before adding the Date header

diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 30988ab56..23eaa0b18 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -1233,6 +1233,11 @@ http_TimeHeader(struct http *to, const char *fmt, double now)
 	char *p;
 
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	if (to->nhd >= to->shd) {
+		VSLb(to->vsl, SLT_LostHeader, "%s", fmt);
+		http_fail(to);
+		return;
+	}
 	p = WS_Alloc(to->ws, strlen(fmt) + VTIM_FORMAT_SIZE);
 	if (p == NULL) {
 		http_fail(to);

From hermunn at varnish-software.com  Wed Oct 24 09:29:22 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:22 +0000 (UTC)
Subject: [6.1] 7891dbc3d typedefs for real/mono time and durations
Message-ID: <20181024092922.4AD02AFDD0@lists.varnish-cache.org>


commit 7891dbc3dc23af1b48cbbc6ac6defdb39160155b
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 8 10:19:50 2018 +0200

    typedefs for real/mono time and durations
    
    We use double for all time representations, yet monotonic time,
    real time and durations are not to be confused.
    
    Also, we might want to change the representation of time in the
    future.
    
    To get an implicit documentation of the semantic type of a time
    value and to facilitate working on the latter, we start off by
    introducing simple typedefs which neither inply any change nor
    offer any type checking. But they pave the way...

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index e8dc2f3de..ba91f2f0f 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -251,7 +251,7 @@ struct worker {
 
 	struct pool_task	task;
 
-	double			lastused;
+	vtim_real		lastused;
 	int			strangelove;
 
 	struct v1l		*v1l;
@@ -413,9 +413,9 @@ struct busyobj {
 #include "tbl/bo_flags.h"
 
 	/* Timeouts */
-	double			connect_timeout;
-	double			first_byte_timeout;
-	double			between_bytes_timeout;
+	vtim_dur		connect_timeout;
+	vtim_dur		first_byte_timeout;
+	vtim_dur		between_bytes_timeout;
 
 	/* Timers */
 	double			t_first;	/* First timestamp logged */
@@ -564,9 +564,8 @@ struct sess {
 
 	struct ws		ws[1];
 
-	/* Timestamps, all on TIM_real() timescale */
-	double			t_open;		/* fd accepted */
-	double			t_idle;		/* fd accepted or resp sent */
+	vtim_real		t_open;		/* fd accepted */
+	vtim_real		t_idle;		/* fd accepted or resp sent */
 
 };
 
@@ -662,7 +661,7 @@ int Lck__Owned(const struct lock *lck);
 
 /* public interface: */
 void Lck_Delete(struct lock *lck);
-int Lck_CondWait(pthread_cond_t *cond, struct lock *lck, double);
+int Lck_CondWait(pthread_cond_t *cond, struct lock *lck, vtim_real);
 
 #define Lck_New(a, b) Lck__New(a, b, #b)
 #define Lck_Lock(a) Lck__Lock(a, __func__, __LINE__)
diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 2b920720a..c0d44850f 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -52,7 +52,7 @@
 #include "vtim.h"
 
 static pthread_t	VCA_thread;
-static double vca_pace = 0.0;
+static vtim_dur vca_pace = 0.0;
 static struct lock pace_mtx;
 static unsigned pool_accepting;
 static pthread_mutex_t shut_mtx = PTHREAD_MUTEX_INITIALIZER;
@@ -137,9 +137,9 @@ static unsigned		need_test;
  * into the vca_acct() loop which we are running anyway
  */
 static void
-vca_periodic(double t0)
+vca_periodic(vtim_real t0)
 {
-	double now;
+	vtim_real now;
 
 	now = VTIM_real();
 	VSC_C_main->uptime = (uint64_t)(now - t0);
@@ -270,7 +270,7 @@ vca_tcp_opt_set(const int sock, const unsigned uds, const int force)
 static void
 vca_pace_check(void)
 {
-	double p;
+	vtim_dur p;
 
 	if (vca_pace == 0.0)
 		return;
@@ -598,7 +598,7 @@ static void * v_matchproto_()
 vca_acct(void *arg)
 {
 	struct listen_sock *ls;
-	double t0;
+	vtim_real t0;
 
 	// XXX Actually a mis-nomer now because the accept happens in a pool
 	// thread. Rename to accept-nanny or so?
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 9f08c2385..c714af99f 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -114,7 +114,7 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo,
 {
 	struct pfd *pfd;
 	int *fdp, err;
-	double tmod;
+	vtim_dur tmod;
 	char abuf1[VTCP_ADDRBUFSIZE], abuf2[VTCP_ADDRBUFSIZE];
 	char pbuf1[VTCP_PORTBUFSIZE], pbuf2[VTCP_PORTBUFSIZE];
 
@@ -619,7 +619,7 @@ void
 VBE_Poll(void)
 {
 	struct backend *be, *be2;
-	double now = VTIM_real();
+	vtim_real now = VTIM_real();
 
 	ASSERT_CLI();
 	Lck_Lock(&backends_mtx);
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index f5c016860..796be9b32 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -66,7 +66,7 @@ struct backend {
 
 	VCL_BACKEND		director;
 
-	double			cooled;
+	vtim_real		cooled;
 };
 
 /*---------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c
index 1b1190ebd..899847c81 100644
--- a/bin/varnishd/cache/cache_backend_probe.c
+++ b/bin/varnishd/cache/cache_backend_probe.c
@@ -76,11 +76,11 @@ struct vbp_target {
 #define BITMAP(n, c, t, b)	uintmax_t	n;
 #include "tbl/backend_poll.h"
 
-	double				last;
-	double				avg;
+	vtim_dur			last;
+	vtim_dur			avg;
 	double				rate;
 
-	double				due;
+	vtim_real			due;
 	int				running;
 	int				heap_idx;
 	struct pool_task		task;
@@ -278,7 +278,7 @@ static void
 vbp_poke(struct vbp_target *vt)
 {
 	int s, tmo, i, proxy_header, err;
-	double t_start, t_now, t_end;
+	vtim_real t_start, t_now, t_end;
 	unsigned rlen, resp;
 	char buf[8192], *p;
 	struct pollfd pfda[1], *pfd = pfda;
@@ -454,7 +454,7 @@ vbp_task(struct worker *wrk, void *priv)
 static void * v_matchproto_(bgthread_t)
 vbp_thread(struct worker *wrk, void *priv)
 {
-	double now, nxt;
+	vtim_real now, nxt;
 	struct vbp_target *vt;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c
index 6fa816a68..ad182c4b0 100644
--- a/bin/varnishd/cache/cache_lck.c
+++ b/bin/varnishd/cache/cache_lck.c
@@ -187,7 +187,7 @@ Lck__Owned(const struct lock *lck)
 }
 
 int v_matchproto_()
-Lck_CondWait(pthread_cond_t *cond, struct lock *lck, double when)
+Lck_CondWait(pthread_cond_t *cond, struct lock *lck, vtim_real when)
 {
 	struct ilck *ilck;
 	struct timespec ts;
diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c
index f33668b2f..5a04eb5aa 100644
--- a/bin/varnishd/cache/cache_tcp_pool.c
+++ b/bin/varnishd/cache/cache_tcp_pool.c
@@ -540,7 +540,7 @@ VCP_Get(struct conn_pool *cp, double tmo, struct worker *wrk,
  */
 
 static int
-VCP_Wait(struct worker *wrk, struct pfd *pfd, double tmo)
+VCP_Wait(struct worker *wrk, struct pfd *pfd, vtim_real tmo)
 {
 	struct conn_pool *cp;
 	int r;
@@ -834,7 +834,7 @@ VTP_Close(struct pfd **pfdp)
  */
 
 struct pfd *
-VTP_Get(struct tcp_pool *tp, double tmo, struct worker *wrk,
+VTP_Get(struct tcp_pool *tp, vtim_dur tmo, struct worker *wrk,
 	unsigned force_fresh, int *err)
 {
 
@@ -845,7 +845,7 @@ VTP_Get(struct tcp_pool *tp, double tmo, struct worker *wrk,
  */
 
 int
-VTP_Wait(struct worker *wrk, struct pfd *pfd, double tmo)
+VTP_Wait(struct worker *wrk, struct pfd *pfd, vtim_real tmo)
 {
 	return (VCP_Wait(wrk, pfd, tmo));
 }
diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h
index 7c25afda2..57564ba96 100644
--- a/bin/varnishd/cache/cache_tcp_pool.h
+++ b/bin/varnishd/cache/cache_tcp_pool.h
@@ -88,14 +88,14 @@ void VTP_Recycle(const struct worker *, struct pfd **);
 	 * Recycle an open connection.
 	 */
 
-struct pfd *VTP_Get(struct tcp_pool *, double tmo, struct worker *,
+struct pfd *VTP_Get(struct tcp_pool *, vtim_dur tmo, struct worker *,
     unsigned force_fresh, int *err);
 	/*
 	 * Get a (possibly) recycled connection.
 	 * errno will be stored in err
 	 */
 
-int VTP_Wait(struct worker *, struct pfd *, double tmo);
+int VTP_Wait(struct worker *, struct pfd *, vtim_real tmo);
 	/*
 	 * If the connection was recycled (state != VTP_STATE_USED) call this
 	 * function before attempting to receive on the connection.
diff --git a/include/vdef.h b/include/vdef.h
index 185265794..3a5e40329 100644
--- a/include/vdef.h
+++ b/include/vdef.h
@@ -165,3 +165,8 @@
 #   define ___Static_assert(x, y) \
 		typedef char __assert_## y[(x) ? 1 : -1] v_unused_
 #endif
+
+/* VTIM API overhaul WIP */
+typedef double vtim_mono;
+typedef double vtim_real;
+typedef double vtim_dur;
diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index c774e2ab8..14b9752d6 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -124,7 +124,7 @@ init(void)
  * gethrtime(), which is the case on modern Solaris descendents.
  */
 
-double
+vtim_mono
 VTIM_mono(void)
 {
 #if defined(HAVE_CLOCK_GETTIME) && !defined(USE_GETHRTIME)
@@ -143,7 +143,7 @@ VTIM_mono(void)
 #endif
 }
 
-double
+vtim_real
 VTIM_real(void)
 {
 #ifdef HAVE_CLOCK_GETTIME
@@ -160,7 +160,7 @@ VTIM_real(void)
 }
 
 void
-VTIM_format(double t, char *p)
+VTIM_format(vtim_real t, char *p)
 {
 	struct tm tm;
 	time_t tt;
@@ -236,10 +236,10 @@ VTIM_format(double t, char *p)
 		DIGIT(1, sec);					\
 	} while(0)
 
-double
+vtim_real
 VTIM_parse(const char *p)
 {
-	double t;
+	vtim_real t;
 	int month = 0, year = 0, weekday = -1, mday = 0;
 	int hour = 0, min = 0, sec = 0;
 	int d, leap;
@@ -394,7 +394,7 @@ VTIM_parse(const char *p)
 }
 
 void
-VTIM_sleep(double t)
+VTIM_sleep(vtim_dur t)
 {
 #ifdef HAVE_NANOSLEEP
 	struct timespec ts;
@@ -415,7 +415,7 @@ VTIM_sleep(double t)
 }
 
 struct timeval
-VTIM_timeval(double t)
+VTIM_timeval(vtim_real t)
 {
 	struct timeval tv;
 
@@ -426,7 +426,7 @@ VTIM_timeval(double t)
 }
 
 struct timespec
-VTIM_timespec(double t)
+VTIM_timespec(vtim_real t)
 {
 	struct timespec tv;
 
@@ -464,8 +464,9 @@ tst(const char *s, time_t good)
 	}
 }
 
+/* XXX keep as double for the time being */
 static int
-tst_delta_check(const char *name, double begin, double end, double ref)
+tst_delta_check(const char *name, double begin, double end, vtim_dur ref)
 {
 	const double tol_max = 1.1;
 	const double tol_min = 1;
@@ -487,9 +488,9 @@ tst_delta_check(const char *name, double begin, double end, double ref)
 static void
 tst_delta()
 {
-	double m_begin, m_end;
-	double r_begin, r_end;
-	const double ref = 1;
+	vtim_mono m_begin, m_end;
+	vtim_real r_begin, r_end;
+	const vtim_dur ref = 1;
 	int err = 0;
 
 	r_begin = VTIM_real();
@@ -510,53 +511,56 @@ tst_delta()
 static void
 bench()
 {
-	double s, e, t;
+	vtim_mono s, e;
+	vtim_mono t_m;
+	vtim_real t_r;
+	unsigned long t_i;
 	int i;
 	char buf[64];
 
-	t = 0;
-	s = VTIM_real();
+	t_r = 0;
+	s = VTIM_mono();
 	for (i=0; i<100000; i++)
-		t += VTIM_real();
-	e = VTIM_real();
+		t_r += VTIM_real();
+	e = VTIM_mono();
 	printf("real: %fs / %d = %fns - tst val %f\n",
-	    e - s, i, 1e9 * (e - s) / i, t);
+	    e - s, i, 1e9 * (e - s) / i, t_r);
 
-	t = 0;
-	s = VTIM_real();
+	t_i = 0;
+	s = VTIM_mono();
 	for (i=0; i<100000; i++)
-		t += VTIM_mono();
-	e = VTIM_real();
+		t_m += VTIM_mono();
+	e = VTIM_mono();
 	printf("mono: %fs / %d = %fns - tst val %f\n",
-	    e - s, i, 1e9 * (e - s) / i, t);
+	    e - s, i, 1e9 * (e - s) / i, t_m);
 
-	t = 0;
+	t_i = 0;
 	s = VTIM_mono();
 	for (i=0; i<100000; i++) {
 		snprintf(buf, sizeof(buf), "%.6f", s);
-		t += buf[4];
+		t_i += buf[4];
 	}
 	e = VTIM_mono();
-	printf("printf %%.6f: %fs / %d = %fns - tst val %f %s\n",
-	    e - s, i, 1e9 * (e - s) / i, t, buf);
+	printf("printf %%.6f: %fs / %d = %fns - tst val %lu %s\n",
+	    e - s, i, 1e9 * (e - s) / i, t_i, buf);
 
-	t = 0;
+	t_i = 0;
 	s = VTIM_mono();
 	for (i=0; i<100000; i++) {
 		snprintf(buf, sizeof(buf), "%ju.%06ju",
 		    (uint64_t)floor(s),
 		    (uint64_t)floor((s * 1e6)) % 1000000UL);
-		t += buf[4];
+		t_i += buf[4];
 	}
 	e = VTIM_mono();
-	printf("printf %%ju.%%06ju: %fs / %d = %fns - tst val %f %s\n",
-	    e - s, i, 1e9 * (e - s) / i, t, buf);
+	printf("printf %%ju.%%06ju: %fs / %d = %fns - tst val %lu %s\n",
+	    e - s, i, 1e9 * (e - s) / i, t_i, buf);
 }
 
 void
 parse_check(time_t t, const char *s)
 {
-	double tt;
+	vtim_real tt;
 	char buf[BUFSIZ];
 
 	tt = VTIM_parse(s);

From hermunn at varnish-software.com  Wed Oct 24 09:29:22 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:22 +0000 (UTC)
Subject: [6.1] eac1bd580 Add a crude python2/python3 hack to deal with UTF-8
Message-ID: <20181024092922.6C5D8AFDE2@lists.varnish-cache.org>


commit eac1bd580ec08b4e3ef1fae8594ced40e390808d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 9 08:25:36 2018 +0000

    Add a crude python2/python3 hack to deal with UTF-8

diff --git a/doc/sphinx/vtc-syntax.py b/doc/sphinx/vtc-syntax.py
index ba7bd6bd7..d98fc96e5 100644
--- a/doc/sphinx/vtc-syntax.py
+++ b/doc/sphinx/vtc-syntax.py
@@ -39,7 +39,12 @@ def parse_file(fn, cl, tl, sl):
     section = ""
     resec = re.compile("[ /]\* SECTION: ")
 
-    f = open(fn, "r")
+    try:
+        # Python3
+        f = open(fn, "r", encoding="UTF-8")
+    except TypeError:
+        # Python2
+        f = open(fn, "r")
 
     for l in f:
         if "*/" in l:

From hermunn at varnish-software.com  Wed Oct 24 09:29:22 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:22 +0000 (UTC)
Subject: [6.1] 74976accf Add python2/3 hack
Message-ID: <20181024092922.905DEAFDF5@lists.varnish-cache.org>


commit 74976accf9c2194495e6441d4b8a461dbbd69f3f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 9 08:36:02 2018 +0000

    Add python2/3 hack

diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py
index 6c96388e5..bd3e68af3 100644
--- a/lib/libvcc/vsctool.py
+++ b/lib/libvcc/vsctool.py
@@ -41,6 +41,7 @@ import getopt
 import json
 import sys
 import collections
+import codecs
 
 # Parameters of 'varnish_vsc_begin', first element is default
 TYPES = ["counter", "gauge", "bitmap"]
@@ -153,7 +154,12 @@ class CounterSet(object):
         '''Emit .h file'''
         assert self.completed
         fon = "VSC_" + self.name + ".h"
-        fo = open(fon, "w")
+        try:
+            # Python3
+            fo = open(fon, "w", encoding="UTF-8")
+        except TypeError:
+            # Python2
+            fo = open(fon, "w")
         genhdr(fo, self.name)
         fo.write(self.struct + " {\n")
         for i in self.mbrs:
@@ -249,7 +255,12 @@ class CounterSet(object):
         '''Emit .c file'''
         assert self.completed
         fon = "VSC_" + self.name + ".c"
-        fo = open(fon, "w")
+        try:
+            # Python3
+            fo = open(fon, "w", encoding="UTF-8")
+        except TypeError:
+            # Python2
+            fo = open(fon, "w")
         genhdr(fo, self.name)
         fo.write('#include "config.h"\n')
         fo.write('#include <stdio.h>\n')
@@ -396,10 +407,22 @@ def mainfunc(argv):
     rstfile = None
     for f, v in optlist:
         if f == '-r':
+            try:
+                # Python3
+                sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
+            except AttributeError:
+                # Python2
+                pass
             rstfile = sys.stdout
 
     vscset = []
-    scs = open(args[0]).read().split("\n.. ")
+    try:
+        # Python3
+        f = open(args[0], encoding="UTF-8")
+    except TypeError:
+        # Python2
+        f = open(args[0])
+    scs = f.read().split("\n.. ")
     if rstfile:
         rstfile.write(scs[0])
     for i in scs[1:]:

From hermunn at varnish-software.com  Wed Oct 24 09:29:22 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:22 +0000 (UTC)
Subject: [6.1] 3a364bd5b Renovate this Makefile
Message-ID: <20181024092922.DDBF3AFE15@lists.varnish-cache.org>


commit 3a364bd5b9813b9c31f8518a7ba0fd273be3d1fc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 9 09:05:24 2018 +0000

    Renovate this Makefile
    
    When producing files with "foo > file", always use the pattern:
    
            foo > file.tmp
            mv file.tmp file
    
    Otherwise program failures end up generating partial content and
    make will not even rerun the failing program next time you type make.
    
    Actually clean CLEANFILES in the clean target.
    
    The reference dir is not built, but it should be in the distfile

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 54f930caf..cf67a6106 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -25,7 +25,7 @@ help:
 	@echo "  doctest   to run all doctests embedded in the documentation (if enabled)"
 
 clean:
-	-rm -rf $(BUILDDIR)/*
+	-rm -rf $(BUILDDIR)/* $(CLEANFILES)
 
 # work around for make html called within doc/sphinx
 .PHONY: graphviz
@@ -101,6 +101,7 @@ EXTRA_DIST = \
 	installation \
 	phk \
 	tutorial \
+	reference \
 	users-guide \
 	vtc-syntax.py \
 	whats-new
@@ -116,11 +117,13 @@ distclean-local:
 	rm -rf $(BUILDDIR)
 
 include/cli.rst: $(top_builddir)/bin/varnishd/varnishd
-	$(top_builddir)/bin/varnishd/varnishd -x cli > $@
+	$(top_builddir)/bin/varnishd/varnishd -x cli > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES = include/cli.rst
 
 include/params.rst: $(top_builddir)/bin/varnishd/varnishd
-	$(top_builddir)/bin/varnishd/varnishd -x parameter > $@
+	$(top_builddir)/bin/varnishd/varnishd -x parameter > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/params.rst
 
 COUNTERS = \
@@ -134,52 +137,64 @@ COUNTERS = \
 	$(top_srcdir)/bin/varnishd/VSC_lck.vsc
 
 include/counters.rst: $(top_srcdir)/lib/libvcc/vsctool.py $(COUNTERS)
-	echo -n '' > $@
+	echo -n '' > ${@}_
 	for i in $(COUNTERS); do \
-		$(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -r $$i >> $@ ; \
+		$(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -r $$i >> ${@}_ ; \
 	done
+	mv ${@}_ ${@}
 
 BUILT_SOURCES += include/counters.rst
 
 # XXX add varnishstat here when it's been _opt2rst'ed
 
 include/varnishncsa_options.rst: $(top_builddir)/bin/varnishncsa/varnishncsa
-	$(top_builddir)/bin/varnishncsa/varnishncsa --options > $@
+	$(top_builddir)/bin/varnishncsa/varnishncsa --options > ${@}_
+	mv ${@}_ ${@}
 include/varnishncsa_synopsis.rst: $(top_builddir)/bin/varnishncsa/varnishncsa
-	$(top_builddir)/bin/varnishncsa/varnishncsa --synopsis > $@
+	$(top_builddir)/bin/varnishncsa/varnishncsa --synopsis > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/varnishncsa_options.rst \
 	include/varnishncsa_synopsis.rst
 
 include/varnishlog_options.rst: $(top_builddir)/bin/varnishlog/varnishlog
-	$(top_builddir)/bin/varnishlog/varnishlog --options > $@
+	$(top_builddir)/bin/varnishlog/varnishlog --options > ${@}_
+	mv ${@}_ ${@}
 include/varnishlog_synopsis.rst: $(top_builddir)/bin/varnishlog/varnishlog
-	$(top_builddir)/bin/varnishlog/varnishlog --synopsis > $@
+	$(top_builddir)/bin/varnishlog/varnishlog --synopsis > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/varnishlog_options.rst \
 	include/varnishlog_synopsis.rst
 
 include/varnishtop_options.rst: $(top_builddir)/bin/varnishtop/varnishtop
-	$(top_builddir)/bin/varnishtop/varnishtop --options > $@
+	$(top_builddir)/bin/varnishtop/varnishtop --options > ${@}_
+	mv ${@}_ ${@}
 include/varnishtop_synopsis.rst: $(top_builddir)/bin/varnishtop/varnishtop
-	$(top_builddir)/bin/varnishtop/varnishtop --synopsis > $@
+	$(top_builddir)/bin/varnishtop/varnishtop --synopsis > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/varnishtop_options.rst \
 	 include/varnishtop_synopsis.rst
 
 include/varnishhist_options.rst: $(top_builddir)/bin/varnishhist/varnishhist
-	$(top_builddir)/bin/varnishhist/varnishhist --options > $@
+	$(top_builddir)/bin/varnishhist/varnishhist --options > ${@}_
+	mv ${@}_ ${@}
 include/varnishhist_synopsis.rst: $(top_builddir)/bin/varnishhist/varnishhist
-	$(top_builddir)/bin/varnishhist/varnishhist --synopsis > $@
+	$(top_builddir)/bin/varnishhist/varnishhist --synopsis > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/varnishhist_options.rst \
 	 include/varnishhist_synopsis.rst
 
 include/varnishstat_options.rst: $(top_builddir)/bin/varnishstat/varnishstat
-	$(top_builddir)/bin/varnishstat/varnishstat --options > $@
+	$(top_builddir)/bin/varnishstat/varnishstat --options > ${@}_
+	mv ${@}_ ${@}
 include/varnishstat_synopsis.rst: $(top_builddir)/bin/varnishstat/varnishstat
-	$(top_builddir)/bin/varnishstat/varnishstat --synopsis > $@
+	$(top_builddir)/bin/varnishstat/varnishstat --synopsis > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/varnishstat_options.rst \
 	 include/varnishstat_synopsis.rst
 
 include/vsl-tags.rst: $(top_builddir)/lib/libvarnishapi/vsl2rst
-	$(top_builddir)/lib/libvarnishapi/vsl2rst > $@
+	$(top_builddir)/lib/libvarnishapi/vsl2rst > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/vsl-tags.rst
 
 VTCSYN_SRC = $(top_srcdir)/bin/varnishtest/vtc.c \
@@ -193,14 +208,10 @@ VTCSYN_SRC = $(top_srcdir)/bin/varnishtest/vtc.c \
 	     $(top_srcdir)/bin/varnishtest/vtc_syslog.c \
 	     $(top_srcdir)/bin/varnishtest/vtc_varnish.c
 include/vtc-syntax.rst: vtc-syntax.py $(VTCSYN_SRC)
-	$(PYTHON) $(top_srcdir)/doc/sphinx/vtc-syntax.py $(VTCSYN_SRC) > $@
+	$(PYTHON) $(top_srcdir)/doc/sphinx/vtc-syntax.py $(VTCSYN_SRC) > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/vtc-syntax.rst
 
-.PHONY: reference
-reference:
-	test -d $@ || mkdir $@
-BUILT_SOURCES += reference
-
 reference/vmod_std.generated.rst: reference $(top_builddir)/lib/libvmod_std/vmod_std.rst
 	cp $(top_builddir)/lib/libvmod_std/vmod_std.rst $@ || true
 BUILT_SOURCES += reference/vmod_std.generated.rst
@@ -231,5 +242,6 @@ BUILT_SOURCES += reference/vmod_proxy.generated.rst
 
 EXTRA_DIST += $(BUILT_SOURCES)
 MAINTAINERCLEANFILES = $(EXTRA_DIST)
+CLEANFILES = $(BUILT_SOURCES)
 
 .NOPATH: $(BUILT_SOURCES)

From hermunn at varnish-software.com  Wed Oct 24 09:29:23 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:23 +0000 (UTC)
Subject: [6.1] 228085633 Clarify and test object slimming for hfp+hfm
Message-ID: <20181024092923.2142EAFE23@lists.varnish-cache.org>


commit 228085633e41e7edde53ca05900f526c2db22af5
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Sep 7 12:32:18 2018 +0200

    Clarify and test object slimming for hfp+hfm
    
    The previous code was correct already, but we can make it
    clearer that HFP implies OC_F_PASS
    
    Also test explicitly that both HFM and HFP have their objects
    slimmed.
    
    Closes #2768

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index c1eece20d..b6b45dcb7 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -417,7 +417,9 @@ cnt_transmit(struct worker *wrk, struct req *req)
 
 	VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
 
-	if (req->objcore->flags & (OC_F_PRIVATE | OC_F_PASS)) {
+	if (req->objcore->flags & (OC_F_PRIVATE | OC_F_PASS | OC_F_HFP)) {
+		if (req->objcore->flags & OC_F_HFP)
+			AN(req->objcore->flags & OC_F_PASS);
 		if (boc != NULL) {
 			HSH_Abandon(req->objcore);
 			ObjWaitState(req->objcore, BOS_FINISHED);
diff --git a/bin/varnishtest/tests/r01821.vtc b/bin/varnishtest/tests/r01821.vtc
index af6e934cb..b07799325 100644
--- a/bin/varnishtest/tests/r01821.vtc
+++ b/bin/varnishtest/tests/r01821.vtc
@@ -1,4 +1,6 @@
-varnishtest "Slim down hit-for-miss objects"
+varnishtest "Slim down hit-for-miss / hit-for-miss objects"
+
+# see also #2768
 
 server s1 -repeat 2 {
 	rxreq
@@ -7,20 +9,32 @@ server s1 -repeat 2 {
 
 varnish v1 -arg "-s Transient=default" -vcl+backend {
 	sub vcl_backend_response {
-		set beresp.uncacheable = true;
+		if (bereq.url == "/hfm") {
+			set beresp.uncacheable = true;
+		} else if (bereq.url == "/hfp") {
+			return (pass(1m));
+		}
 	}
 } -start
 
-logexpect l1 -v v1 {
+logexpect l1 -v v1 -g raw {
+	expect * *	Storage		"Transient"
 	expect * *	Storage		"Transient"
 } -start
 
 client c1 {
-	txreq
+	txreq -url "/hfm"
+	rxresp
+} -start
+
+client c2 {
+	txreq -url "/hfp"
 	rxresp
 } -run
 
+client c1 -wait
+
 logexpect l1 -wait
 
-varnish v1 -expect SM?.Transient.c_bytes != 0
+varnish v1 -expect SM?.Transient.c_bytes > 131072
 varnish v1 -expect SM?.Transient.g_bytes < 65536

From hermunn at varnish-software.com  Wed Oct 24 09:29:23 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:23 +0000 (UTC)
Subject: [6.1] 2e336c041 keep the dog in the kennel unless there's a queue
Message-ID: <20181024092923.51843AFE34@lists.varnish-cache.org>


commit 2e336c04101fc4ed4f000f9c3395f4e6fdce0622
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Oct 9 12:38:25 2018 +0200

    keep the dog in the kennel unless there's a queue
    
    As long as we are not queuing any threads, there is no queue to
    move. We record the queue marker the first time we notice queuing and
    only then see if it doesn't move.
    
    Fixes #2794

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index ad1fe08f6..011010499 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -511,14 +511,15 @@ pool_herder(void *priv)
 		 * Instead we implement a watchdog and kill the worker if
 		 * nothing has been dequeued for that long.
 		 */
-		if (dq != pp->ndequeued) {
+		if (pp->lqueue == 0) {
+			dq = pp->ndequeued + 1;
+		} else if (dq != pp->ndequeued) {
 			dq = pp->ndequeued;
 			dqt = VTIM_real();
-		} else if (pp->lqueue &&
-		    VTIM_real() - dqt > cache_param->wthread_watchdog) {
+		} else if (VTIM_real() - dqt > cache_param->wthread_watchdog) {
 			VSL(SLT_Error, 0,
-			   "Pool Herder: Queue does not move ql=%u dt=%f",
-			   pp->lqueue, VTIM_real() - dqt);
+			    "Pool Herder: Queue does not move ql=%u dt=%f",
+			    pp->lqueue, VTIM_real() - dqt);
 			WRONG("Worker Pool Queue does not move");
 		}
 		wthread_min = cache_param->wthread_min;

From hermunn at varnish-software.com  Wed Oct 24 09:29:23 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:23 +0000 (UTC)
Subject: [6.1] 3fdf117e6 use monotonic time for the watchdog
Message-ID: <20181024092923.8582BAFE46@lists.varnish-cache.org>


commit 3fdf117e64e673e0031acf82b86dcb352c460620
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Oct 9 12:56:41 2018 +0200

    use monotonic time for the watchdog

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 011010499..c133afe6f 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -490,7 +490,7 @@ pool_herder(void *priv)
 	double delay;
 	int wthread_min;
 	uintmax_t dq = (1ULL << 31);
-	double dqt = 0;
+	vtim_mono dqt = 0;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 
@@ -515,11 +515,11 @@ pool_herder(void *priv)
 			dq = pp->ndequeued + 1;
 		} else if (dq != pp->ndequeued) {
 			dq = pp->ndequeued;
-			dqt = VTIM_real();
-		} else if (VTIM_real() - dqt > cache_param->wthread_watchdog) {
+			dqt = VTIM_mono();
+		} else if (VTIM_mono() - dqt > cache_param->wthread_watchdog) {
 			VSL(SLT_Error, 0,
 			    "Pool Herder: Queue does not move ql=%u dt=%f",
-			    pp->lqueue, VTIM_real() - dqt);
+			    pp->lqueue, VTIM_mono() - dqt);
 			WRONG("Worker Pool Queue does not move");
 		}
 		wthread_min = cache_param->wthread_min;

From hermunn at varnish-software.com  Wed Oct 24 09:29:23 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:23 +0000 (UTC)
Subject: [6.1] 48a3b8d23 Hunt for py-specific variants of sphinx-build program
Message-ID: <20181024092923.B8560AFE62@lists.varnish-cache.org>


commit 48a3b8d234132dd3a15c2968c179878d47348f43
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 11 14:08:51 2018 +0000

    Hunt for py-specific variants of sphinx-build program

diff --git a/configure.ac b/configure.ac
index b543949b3..66bdc3d83 100644
--- a/configure.ac
+++ b/configure.ac
@@ -49,7 +49,9 @@ fi
 AC_ARG_WITH([sphinx-build],
   AS_HELP_STRING([--with-sphinx-build=PATH], [Location of sphinx-build (auto)]),
   [SPHINX="$withval"],
-  AC_CHECK_PROGS(SPHINX, [sphinx-build], [no]))
+  AC_CHECK_PROGS(SPHINX,
+    [sphinx-build sphinx-build-3.6 sphinx-build-2.7],
+    [no]))
 if test "x$SPHINX" = "xno"; then
   AC_MSG_ERROR(
     [sphinx-build is needed to build Varnish, please install python-sphinx.])

From hermunn at varnish-software.com  Wed Oct 24 09:29:23 2018
From: hermunn at varnish-software.com (hermunn)
Date: Wed, 24 Oct 2018 09:29:23 +0000 (UTC)
Subject: [6.1] 391394a75 Also look for versioned rst2man scripts
Message-ID: <20181024092923.D5DD2AFE6D@lists.varnish-cache.org>


commit 391394a753a228b6efcfb115ae87c31a5bb1adb1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 11 14:26:02 2018 +0000

    Also look for versioned rst2man scripts

diff --git a/configure.ac b/configure.ac
index 66bdc3d83..fd85311cd 100644
--- a/configure.ac
+++ b/configure.ac
@@ -40,7 +40,9 @@ AC_PROG_INSTALL
 AC_ARG_WITH([rst2man],
   AS_HELP_STRING([--with-rst2man=PATH], [Location of rst2man (auto)]),
   [RST2MAN="$withval"],
-  AC_CHECK_PROGS(RST2MAN, [rst2man rst2man.py], [no]))
+  AC_CHECK_PROGS(RST2MAN,
+    [rst2man rst2man.py rst2man-3.6 rst2man-2.7],
+    [no]))
 if test "x$RST2MAN" = "xno"; then
   AC_MSG_ERROR(
     [rst2man is needed to build Varnish, please install python-docutils.])

From nils.goroll at uplex.de  Thu Oct 25 12:49:14 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 25 Oct 2018 12:49:14 +0000 (UTC)
Subject: [master] 5522ddd91 document that `now` remains constant during vcl
 subs and test for it
Message-ID: <20181025124914.44F8DAF94A@lists.varnish-cache.org>


commit 5522ddd91c1e53f3d03d64ae2dcf03a7072ac2c9
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Oct 25 14:45:17 2018 +0200

    document that `now` remains constant during vcl subs and test for it

diff --git a/bin/varnishtest/tests/b00030.vtc b/bin/varnishtest/tests/b00030.vtc
index 5962b40d2..626324755 100644
--- a/bin/varnishtest/tests/b00030.vtc
+++ b/bin/varnishtest/tests/b00030.vtc
@@ -1,7 +1,8 @@
-varnishtest "Test formatting of timestamps"
+varnishtest "Test timestamps"
 
-# We can't test the value of x-timestamp, but this should fail
+# We can't test the value of a timestamp, but this should fail
 # if we can't set the header at all.
+# We also test that `now` remains unchanged during a vcl sub
 
 server s1 {
 	rxreq
@@ -9,16 +10,65 @@ server s1 {
 } -start
 
 varnish v1 -vcl+backend {
+	import vtc;
+
+	sub recv_sub {
+		set req.http.recv_sub = now;
+	}
 	sub vcl_recv {
-		return (synth(200));
+		set req.http.recv = now;
+		vtc.sleep(1s);
+		call recv_sub;
+		if (req.http.recv != req.http.recv_sub) {
+			return (fail);
+		}
 	}
 	sub vcl_synth {
-		set resp.http.x-timestamp = now;
+		set resp.http.synth = now;
+	}
+
+	sub vcl_deliver {
+		set resp.http.deliver = now;
+		if (req.http.recv == req.http.deliver) {
+			return (fail);
+		}
+
+		vtc.sleep(1s);
+		return (synth(200));
 	}
+
+	sub bf_sub {
+		set bereq.http.bf_sub = now;
+	}
+	sub vcl_backend_fetch {
+		set bereq.http.bf = now;
+		vtc.sleep(1s);
+		call bf_sub;
+		if (bereq.http.bf != bereq.http.bf_sub) {
+			return (fail);
+		}
+	}
+	sub br_sub {
+		set beresp.http.br_sub = now;
+	}
+	sub vcl_backend_response {
+		set beresp.http.br = now;
+		vtc.sleep(1s);
+		call br_sub;
+		if (beresp.http.br != beresp.http.br_sub) {
+			return (fail);
+		}
+		if (bereq.http.bf == beresp.http.br) {
+			return (fail);
+		}
+	}
+
+
 } -start
 
 client c1 {
 	txreq
 	rxresp
-	expect resp.http.x-timestamp ~ "..., .. ... .... ..:..:.. GMT"
+	expect resp.status == 200
+	expect resp.http.synth ~ "^..., .. ... .... ..:..:.. GMT"
 } -run
diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 7aa434da6..67bf1dd12 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -105,8 +105,11 @@ VCL has time. A duration can be added to a time to make another time.
 In string context they return a formatted string in RFC1123 format,
 e.g. ``Sun, 06 Nov 1994 08:49:37 GMT``.
 
-The keyword ``now`` returns a time representing the current time in seconds
-since the Epoch.
+The keyword ``now`` returns a notion of the current time, which is
+kept cosistent during vcl subroutine invocations, so during the
+execution of a vcl subroutine callback (``vcl_* {}``), including all
+user-defined subroutines beging called, ``now`` always returns the
+same value.
 
 Durations
 ~~~~~~~~~

From nils.goroll at uplex.de  Thu Oct 25 12:51:07 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 25 Oct 2018 12:51:07 +0000 (UTC)
Subject: [master] b7cb9b245 typos :(
Message-ID: <20181025125107.A98B9AFAF4@lists.varnish-cache.org>


commit b7cb9b245be9192e9b75e1f81327df3b97b2a2ed
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Oct 25 14:49:33 2018 +0200

    typos :(

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 67bf1dd12..84aadafb4 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -106,9 +106,9 @@ In string context they return a formatted string in RFC1123 format,
 e.g. ``Sun, 06 Nov 1994 08:49:37 GMT``.
 
 The keyword ``now`` returns a notion of the current time, which is
-kept cosistent during vcl subroutine invocations, so during the
+kept consistent during vcl subroutine invocations, so during the
 execution of a vcl subroutine callback (``vcl_* {}``), including all
-user-defined subroutines beging called, ``now`` always returns the
+user-defined subroutines being called, ``now`` always returns the
 same value.
 
 Durations

From hermunn at varnish-software.com  Thu Oct 25 12:52:08 2018
From: hermunn at varnish-software.com (Pål Hermunn Johansen)
Date: Thu, 25 Oct 2018 12:52:08 +0000 (UTC)
Subject: [6.1] a3e090263 Change log entries for 6.1.1
Message-ID: <20181025125208.B01C4AFCCB@lists.varnish-cache.org>


commit a3e090263adf38aa3ff55e5b16a76e898ca162fa
Author: P?l Hermunn Johansen <hermunn at varnish-software.com>
Date:   Thu Oct 25 14:32:04 2018 +0200

    Change log entries for 6.1.1
    
    This should add changes that were not already added by previous
    commits.

diff --git a/doc/changes.rst b/doc/changes.rst
index 3786d408d..496568db0 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -26,6 +26,9 @@ http://varnish-cache.org/docs/trunk/whats-new/index.html and via
 individual releases. These documents are updated as part of the
 release process.
 
+================================
+Varnish Cache 6.1.1 (unreleased)
+================================
 
 * fixed ``varnishhist`` display error (2780_)
 
@@ -39,6 +42,9 @@ release process.
 * Adjust code generator python tools to python 3 and prefer python 3
   over python 2 where available
 
+* Implement and test ECMA-48 "REP" sequence to fix test case
+  u00008.vtc on some newer platforms. (2668_)
+
 * fix some stats metrics (vsc) which were wrongly marked as _gauge_
 
 * fix ``varnishd -I`` (2782_)
@@ -51,7 +57,13 @@ release process.
 
 * added a thread pool watchdog which will restart the worker process
   if scheduling tasks onto worker threads appears stuck. The new
-  parameter ``thread_pool_watchdog`` configures it. (2418_)
+  parameter ``thread_pool_watchdog`` configures it. (2418_, 2794_)
+
+* Clarify and test object slimming for hfp+hfm. (2768_)
+
+* Allow PRIORITY frames on closed streams (2775_)
+
+* Hardening of the h2_frame_f callbacks (2781_)
 
 * Improved varnish log client performance (2788_)
 
@@ -59,13 +71,18 @@ release process.
   an unnecessary incompatibility with VSL files written by previous
   versions. (2790_)
 
+.. _2418: https://github.com/varnishcache/varnish-cache/issues/2418
+.. _2589: https://github.com/varnishcache/varnish-cache/issues/2589
+.. _2668: https://github.com/varnishcache/varnish-cache/issues/2668
+.. _2768: https://github.com/varnishcache/varnish-cache/issues/2768
+.. _2780: https://github.com/varnishcache/varnish-cache/issues/2775
 .. _2780: https://github.com/varnishcache/varnish-cache/issues/2780
+.. _2780: https://github.com/varnishcache/varnish-cache/issues/2781
 .. _2782: https://github.com/varnishcache/varnish-cache/issues/2782
 .. _2787: https://github.com/varnishcache/varnish-cache/issues/2787
-.. _2589: https://github.com/varnishcache/varnish-cache/issues/2589
-.. _2418: https://github.com/varnishcache/varnish-cache/issues/2418
 .. _2788: https://github.com/varnishcache/varnish-cache/issues/2788
 .. _2790: https://github.com/varnishcache/varnish-cache/issues/2790
+.. _2794: https://github.com/varnishcache/varnish-cache/issues/2794
 
 ================================
 Varnish Cache 6.1.0 (2018-09-17)

From nils.goroll at uplex.de  Thu Oct 25 14:06:09 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 25 Oct 2018 14:06:09 +0000 (UTC)
Subject: [master] ddfda3d7c clarify that vmod return values are immutable
Message-ID: <20181025140609.D7438B12BD@lists.varnish-cache.org>


commit ddfda3d7c996e67a1894fc3955eb546ebd0b721b
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Oct 25 16:03:28 2018 +0200

    clarify that vmod return values are immutable

diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index 8978ca4f2..46ad29d5d 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -268,6 +268,16 @@ language representation.  Here is a description of them.
 All but the PRIV and STRING_LIST types have typedefs: VCL_INT, VCL_REAL,
 etc.
 
+Notice that all the non-native (C pointer) types are ``const``, so
+anything returned by a vmod function/method is assumed to be
+immutable. In other words, a vmod `must not` modify any data which was
+previously returned.
+
+When returning non-native values, the producing function is
+responsible for arranging memory management.  Either by freeing the
+structure later by whatever means available or by using storage
+allocated from the client or backend workspaces.
+
 ACL
 	C-type: ``const struct vrt_acl *``
 
@@ -376,12 +386,6 @@ STRING
 	If there were no "foobar" HTTP header, the vmod_foo()
 	function would be passed a NULL pointer as argument.
 
-	When used as a return value, the producing function is
-	responsible for arranging memory management.  Either by
-	freeing the string later by whatever means available or
-	by using storage allocated from the client or backend
-	workspaces.
-
 STEVEDORE
 	C-type: ``const struct stevedore *``
 

From nils.goroll at uplex.de  Thu Oct 25 14:19:07 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 25 Oct 2018 14:19:07 +0000 (UTC)
Subject: [master] 18d8ab242 clarify previous commit
Message-ID: <20181025141907.46292B17F5@lists.varnish-cache.org>


commit 18d8ab242cf6cddc42ed49f1fdf74d525a423740
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Oct 25 16:17:26 2018 +0200

    clarify previous commit
    
    ... after feedback from @slimhazard aka Geoff

diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index 46ad29d5d..d8cf6e4b0 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -268,8 +268,8 @@ language representation.  Here is a description of them.
 All but the PRIV and STRING_LIST types have typedefs: VCL_INT, VCL_REAL,
 etc.
 
-Notice that all the non-native (C pointer) types are ``const``, so
-anything returned by a vmod function/method is assumed to be
+Notice that most of the non-native (C pointer) types are ``const``,
+which, if returned by a vmod function/method, are assumed to be
 immutable. In other words, a vmod `must not` modify any data which was
 previously returned.
 
@@ -339,7 +339,7 @@ HEADER
 HTTP
 	C-type: ``struct http *``
 
-	TODO
+	A reference to a header object as ``req.http`` or ``bereq.http``.
 
 INT
 	C-type: ``long``

From hermunn at varnish-software.com  Thu Oct 25 16:28:08 2018
From: hermunn at varnish-software.com (Pål Hermunn Johansen)
Date: Thu, 25 Oct 2018 16:28:08 +0000 (UTC)
Subject: [6.1] acefd09bb Fixup previous commit
Message-ID: <20181025162808.C5313BEBD4@lists.varnish-cache.org>


commit acefd09bbcacf64bcb3f883b72637164cf1d49f8
Author: P?l Hermunn Johansen <hermunn at varnish-software.com>
Date:   Thu Oct 25 18:21:33 2018 +0200

    Fixup previous commit

diff --git a/doc/changes.rst b/doc/changes.rst
index 496568db0..0c88c664a 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -75,9 +75,9 @@ Varnish Cache 6.1.1 (unreleased)
 .. _2589: https://github.com/varnishcache/varnish-cache/issues/2589
 .. _2668: https://github.com/varnishcache/varnish-cache/issues/2668
 .. _2768: https://github.com/varnishcache/varnish-cache/issues/2768
-.. _2780: https://github.com/varnishcache/varnish-cache/issues/2775
+.. _2775: https://github.com/varnishcache/varnish-cache/issues/2775
 .. _2780: https://github.com/varnishcache/varnish-cache/issues/2780
-.. _2780: https://github.com/varnishcache/varnish-cache/issues/2781
+.. _2781: https://github.com/varnishcache/varnish-cache/issues/2781
 .. _2782: https://github.com/varnishcache/varnish-cache/issues/2782
 .. _2787: https://github.com/varnishcache/varnish-cache/issues/2787
 .. _2788: https://github.com/varnishcache/varnish-cache/issues/2788

From nils.goroll at uplex.de  Fri Oct 26 10:58:16 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 26 Oct 2018 10:58:16 +0000 (UTC)
Subject: [master] 897f3f7d5 fix vmod object constructor documentation
Message-ID: <20181026105816.7E465A1EE1@lists.varnish-cache.org>


commit 897f3f7d58ce512354e91ca1a7cdac06b7f110ed
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Oct 26 12:34:25 2018 +0200

    fix vmod object constructor documentation
    
    The vmod name got lost in e7c0497404a5b4aadda71aead3b0e7fe946d6993
    
    example before/after diff:
    
    - new xshard_param = shard_param()
    + new xshard_param = directors.shard_param()

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index dd157449f..7db51c668 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -407,11 +407,13 @@ class ProtoType(object):
             pfx += pfx
         s = pfx
         if type(self.st) == s_object:
-            s += "new x" + self.bname + " = "
+            s += "new " + self.obj + " = "
         elif self.retval is not None:
             s += self.retval.vcl() + " "
 
-        if type(self.st) == s_method:
+        if type(self.st) == s_object:
+            s += self.st.vcc.modname + "." + self.name + "("
+        elif type(self.st) == s_method:
             s += self.obj + self.bname + "("
         else:
             s += self.name + "("

From hermunn at varnish-software.com  Fri Oct 26 11:23:11 2018
From: hermunn at varnish-software.com (hermunn)
Date: Fri, 26 Oct 2018 11:23:11 +0000 (UTC)
Subject: [6.1] 93212f74a Prepare for 6.1.1
Message-ID: <20181026112311.B26FFA2859@lists.varnish-cache.org>


commit 93212f74ad64532eb3b63073db9c1fa3e5ccd6ef
Author: P?l Hermunn Johansen <hermunn at varnish-software.com>
Date:   Thu Oct 25 18:02:25 2018 +0200

    Prepare for 6.1.1

diff --git a/configure.ac b/configure.ac
index fd85311cd..786dc56be 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,7 +2,7 @@ AC_PREREQ(2.59)
 AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS
 Copyright (c) 2006-2018 Varnish Software])
 AC_REVISION([$Id$])
-AC_INIT([Varnish], [6.1.0], [varnish-dev at varnish-cache.org])
+AC_INIT([Varnish], [6.1.1], [varnish-dev at varnish-cache.org])
 AC_CONFIG_SRCDIR(include/miniobj.h)
 AC_CONFIG_HEADERS([config.h])
 AC_CONFIG_MACRO_DIR([m4])
diff --git a/doc/changes.rst b/doc/changes.rst
index 0c88c664a..77a6088b2 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -4,16 +4,14 @@ About this document
 
 .. keep this section at the top!
 
-This document contains notes from the Varnish developers about ongoing
-development and past versions:
+This document contains notes from the Varnish developers about
+the 6.1 branch of Varnish Cache.
 
 * Developers will note here changes which they consider particularly
   relevant or otherwise noteworthy
 
-* This document is not necessarily up-to-date with the code
-
-* It serves as a basis for release managers and others involved in
-  release documentation
+* This document is updated before releases and partly during
+  develoment. Between releases the document can be out of sync.
 
 * It is not rendered as part of the official documentation and thus
   only available in ReStructuredText (rst) format in the source
@@ -27,7 +25,7 @@ individual releases. These documents are updated as part of the
 release process.
 
 ================================
-Varnish Cache 6.1.1 (unreleased)
+Varnish Cache 6.1.1 (2018-10-26)
 ================================
 
 * fixed ``varnishhist`` display error (2780_)

From hermunn at varnish-software.com  Fri Oct 26 11:54:13 2018
From: hermunn at varnish-software.com (Pål Hermunn Johansen)
Date: Fri, 26 Oct 2018 11:54:13 +0000 (UTC)
Subject: [6.1] efc2f6c15 fix vmod object constructor documentation
Message-ID: <20181026115413.8C0A1A334D@lists.varnish-cache.org>


commit efc2f6c1536cf2272e471f5cff5f145239b19460
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Oct 26 12:34:25 2018 +0200

    fix vmod object constructor documentation
    
    The vmod name got lost in e7c0497404a5b4aadda71aead3b0e7fe946d6993
    
    example before/after diff:
    
    - new xshard_param = shard_param()
    + new xshard_param = directors.shard_param()

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index dd157449f..7db51c668 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -407,11 +407,13 @@ class ProtoType(object):
             pfx += pfx
         s = pfx
         if type(self.st) == s_object:
-            s += "new x" + self.bname + " = "
+            s += "new " + self.obj + " = "
         elif self.retval is not None:
             s += self.retval.vcl() + " "
 
-        if type(self.st) == s_method:
+        if type(self.st) == s_object:
+            s += self.st.vcc.modname + "." + self.name + "("
+        elif type(self.st) == s_method:
             s += self.obj + self.bname + "("
         else:
             s += self.name + "("

From phk at FreeBSD.org  Mon Oct 29 08:42:11 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 29 Oct 2018 08:42:11 +0000 (UTC)
Subject: [master] e7bb89840 Make it easier to parse the output consistently:
Message-ID: <20181029084211.C6312A8734@lists.varnish-cache.org>


commit e7bb898400a4ba8757ea7184868a6da7414aa3e8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 29 08:40:40 2018 +0000

    Make it easier to parse the output consistently:
    
    Always make "* top %f TEST ${test} starting" the first message.
    
    Tag vtest and varnishtest message with "VTEST" instead of "TEST"

diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 2b98596a8..66a2f97f2 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -503,6 +503,8 @@ exec_file(const char *fn, const char *script, const char *tmpdir,
 	vltop = vtc_logopen("top");
 	AN(vltop);
 
+	vtc_log(vltop, 1, "TEST %s starting", fn);
+
 	init_macro();
 	init_server();
 	init_syslog();
@@ -537,7 +539,6 @@ exec_file(const char *fn, const char *script, const char *tmpdir,
 	AZ(fclose(f));
 
 	vtc_stop = 0;
-	vtc_log(vltop, 1, "TEST %s starting", fn);
 
 	vtc_thread = pthread_self();
 	parse_string(script, cmds, NULL, vltop);
diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c
index b55d4be13..9fac1360a 100644
--- a/bin/varnishtest/vtc_misc.c
+++ b/bin/varnishtest/vtc_misc.c
@@ -74,7 +74,7 @@ cmd_vtest(CMD_ARGS)
 		return;
 	AZ(strcmp(av[0], "vtest"));
 
-	vtc_log(vl, 1, "TEST %s", av[1]);
+	vtc_log(vl, 1, "VTEST %s", av[1]);
 	AZ(av[2]);
 }
 
@@ -96,7 +96,7 @@ cmd_varnishtest(CMD_ARGS)
 		return;
 	AZ(strcmp(av[0], "varnishtest"));
 
-	vtc_log(vl, 1, "TEST %s", av[1]);
+	vtc_log(vl, 1, "VTEST %s", av[1]);
 	AZ(av[2]);
 }
 

From phk at FreeBSD.org  Mon Oct 29 08:50:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 29 Oct 2018 08:50:09 +0000 (UTC)
Subject: [master] 1cc0fa997 Make sure the "screen" is blank when we start.
Message-ID: <20181029085009.4FB8DA8A9B@lists.varnish-cache.org>


commit 1cc0fa99780c615d5ae857b14c1090aa0a21994a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 29 08:49:02 2018 +0000

    Make sure the "screen" is blank when we start.

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index 10d545b38..dd6880e37 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -227,7 +227,7 @@ term_resize(struct process *pp, int lin, int col)
 	vram = calloc(lin, sizeof *pp->vram);
 	AN(vram);
 	for (i = 0; i < lin; i++) {
-		vram[i] = malloc(col + 1L);
+		vram[i] = calloc(col + 1L, 1);
 		AN(vram[i]);
 		memset(vram[i], ' ', col);
 		vram[i][col] = '\0';

From phk at FreeBSD.org  Mon Oct 29 12:01:11 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 29 Oct 2018 12:01:11 +0000 (UTC)
Subject: [master] c78361549 Allow BOOL [!=]= BOOL comparisons
Message-ID: <20181029120111.59250AE3C2@lists.varnish-cache.org>


commit c783615498044430703a910393336a3a8edd30f9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 29 11:59:44 2018 +0000

    Allow BOOL [!=]= BOOL comparisons
    
    Fixes:  #2809

diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 1f0699e19..780befa0c 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -1111,6 +1111,8 @@ static const struct cmps vcc_cmps[] = {
 	IDENT_REL(SUB),
 	IDENT_REL(INSTANCE),
 
+	{BOOL,		T_EQ,		cmp_simple, "((!(\v1)) == (!(\v2)))" },
+	{BOOL,		T_NEQ,		cmp_simple, "((!(\v1)) != (!(\v2)))" },
 	{IP,		T_EQ,		cmp_simple, "!VRT_ipcmp(\v1, \v2)" },
 	{IP,		T_NEQ,		cmp_simple, "VRT_ipcmp(\v1, \v2)" },
 
@@ -1144,9 +1146,6 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 	ERRCHK(tl);
 	tk = tl->t;
 
-	if ((*e)->fmt == BOOL)
-		return;
-
 	for (cp = vcc_cmps; cp->fmt != VOID; cp++) {
 		if ((*e)->fmt == cp->fmt && tl->t->tok == cp->token) {
 			AN(cp->func);

From nils.goroll at uplex.de  Mon Oct 29 12:16:06 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 29 Oct 2018 12:16:06 +0000 (UTC)
Subject: [master] 141627835 Test assigning <bool> == <bool>
Message-ID: <20181029121606.BF198AE9DB@lists.varnish-cache.org>


commit 141627835d56fa86b148ce5fa992bca59a997e6f
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 29 13:15:21 2018 +0100

    Test assigning <bool> == <bool>
    
    Tests #2809

diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc
index e36c720c8..3cbe2678e 100644
--- a/bin/varnishtest/tests/v00020.vtc
+++ b/bin/varnishtest/tests/v00020.vtc
@@ -328,6 +328,8 @@ varnish v1 -vcl+backend {
 		debug.sethdr(resp.http.rst, req.restarts);
 		set resp.http.foo =
 		    (resp.http.foo + resp.http.bar) == ("X" + resp.http.foo);
+		// Ticket 2809
+		set resp.http.bar = true == false;
 	}
 	sub vcl_deliver {
 		set resp.http.p = (0 + 9223372036854775807);
@@ -340,6 +342,7 @@ client c1 {
 	rxresp
 	expect resp.http.rst == "0"
 	expect resp.http.foo == "true"
+	expect resp.http.bar == "false"
 	expect resp.http.p == 9223372036854775807
 	expect resp.http.n == -9223372036854775807
 } -run

From phk at FreeBSD.org  Tue Oct 30 07:39:14 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 30 Oct 2018 07:39:14 +0000 (UTC)
Subject: [master] 1cae1a455 Add the VRT handling of resp.filter (doesn't do
 anything yet)
Message-ID: <20181030073914.C062B96587@lists.varnish-cache.org>


commit 1cae1a455310192f32c72bba8c13a3c1aa8328f5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 30 07:34:04 2018 +0000

    Add the VRT handling of resp.filter (doesn't do anything yet)

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index a829a43d7..ce39e683d 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -512,6 +512,7 @@ struct req {
 
 	/* Deliver pipeline */
 	struct vdp_ctx		*vdc;
+	const char		*filter_list;
 
 	/* Delivery mode */
 	unsigned		res_mode;
diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c
index 102b6f710..410d878be 100644
--- a/bin/varnishd/cache/cache_vrt_filter.c
+++ b/bin/varnishd/cache/cache_vrt_filter.c
@@ -285,6 +285,7 @@ VBF_Get_Filter_List(struct busyobj *bo)
 	unsigned u;
 	struct vsb vsb[1];
 
+	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	u = WS_Reserve(bo->ws, 0);
 	if (u == 0) {
 		WS_Release(bo->ws, 0);
@@ -306,34 +307,41 @@ VBF_Get_Filter_List(struct busyobj *bo)
 	return ("");
 }
 
-/*--------------------------------------------------------------------*/
-
-VCL_STRING
-VRT_r_beresp_filters(VRT_CTX)
+static const char *
+resp_Get_Filter_List(struct req *req)
 {
-
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
-	if (ctx->bo->filter_list != NULL)
-		return(ctx->bo->filter_list);
-	/* We do not set bo->filter_list yet, things might still change */
-	return (VBF_Get_Filter_List(ctx->bo));
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	return ("");
 }
 
-VCL_VOID
-VRT_l_beresp_filters(VRT_CTX, const char *str, ...)
-{
-	va_list ap;
-	const char *b;
+/*--------------------------------------------------------------------*/
 
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
-	va_start(ap, str);
-	b = VRT_String(ctx->bo->ws, NULL, str, ap);
-	va_end(ap);
-	if (b == NULL) {
-		WS_MarkOverflow(ctx->bo->ws);
-		return;
+#define FILTER_VAR(vcl, in, func)					\
+	VCL_STRING							\
+	VRT_r_##vcl##_filters(VRT_CTX)					\
+	{								\
+									\
+		CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);			\
+		if (ctx->in->filter_list != NULL)			\
+			return(ctx->in->filter_list);			\
+		return (func(ctx->in));					\
+	}								\
+									\
+	VCL_VOID							\
+	VRT_l_##vcl##_filters(VRT_CTX, const char *str, ...)		\
+	{								\
+		va_list ap;						\
+		const char *b;						\
+									\
+		CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);			\
+		va_start(ap, str);					\
+		b = VRT_String(ctx->in->ws, NULL, str, ap);		\
+		va_end(ap);						\
+		if (b == NULL)						\
+			WS_MarkOverflow(ctx->in->ws);			\
+		else							\
+			ctx->in->filter_list = b;			\
 	}
-	ctx->bo->filter_list = b;
-}
+
+FILTER_VAR(beresp, bo, VBF_Get_Filter_List)
+FILTER_VAR(resp, req, resp_Get_Filter_List)
diff --git a/bin/varnishtest/tests/g00003.vtc b/bin/varnishtest/tests/g00003.vtc
index 74bfa5814..773917cba 100644
--- a/bin/varnishtest/tests/g00003.vtc
+++ b/bin/varnishtest/tests/g00003.vtc
@@ -46,6 +46,10 @@ varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend {
 			set beresp.filters = "gzip gunzip gzip gunzip gzip";
 		}
 	}
+	sub vcl_deliver {
+		set resp.http.filters = resp.filters;
+		set resp.filters = "";
+	}
 } -start
 
 client c1 {
diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst
index 054cf2ff4..b269aa029 100644
--- a/doc/sphinx/reference/vcl_var.rst
+++ b/doc/sphinx/reference/vcl_var.rst
@@ -1147,6 +1147,15 @@ resp.is_streaming
 	Returns true when the response will be streamed
 	while being fetched from the backend.
 
+resp.filters
+
+	Type: STRING
+
+	Readable from: vcl_deliver, vcl_synth
+
+	Writable from: vcl_deliver, vcl_synth
+
+	List of VFP filters the resp.body will be pulled through.
 
 Special variables
 ~~~~~~~~~~~~~~~~~

From nils.goroll at uplex.de  Tue Oct 30 09:58:07 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 30 Oct 2018 09:58:07 +0000 (UTC)
Subject: [master] 46e30ddc5 comment typo fix
Message-ID: <20181030095808.0065B9C3EB@lists.varnish-cache.org>


commit 46e30ddc5cb20e351138dfa45362006e1ef20d3a
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Oct 30 10:55:35 2018 +0100

    comment typo fix
    
    completely irrelevant historical note:
    
    IIUC, ~20+ years ago, "slowlaris" was referring to Solaris being
    percieved as "much slower than linux" (which was partly true, but
    also related to the fact that linux had less mp support at the time)

diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c
index 50a38e5cd..a7fc994fe 100644
--- a/bin/varnishd/http1/cache_http1_line.c
+++ b/bin/varnishd/http1/cache_http1_line.c
@@ -207,7 +207,7 @@ V1L_Flush(const struct worker *wrk)
 			 * was sent.
 			 *
 			 * XXX: Add a "minimum sent data per timeout
-			 * counter to prevent slowlaris attacks
+			 * counter to prevent slowloris attacks
 			*/
 
 			if (VTIM_real() - v1l->t0 > cache_param->send_timeout) {

From dridi.boukelmoune at gmail.com  Tue Oct 30 10:34:09 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 30 Oct 2018 10:34:09 +0000 (UTC)
Subject: [master] 78325c4fb Indent continuations
Message-ID: <20181030103409.ECEEFA0075@lists.varnish-cache.org>


commit 78325c4fb1149817dc669f0dce06c24a74858f87
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Oct 30 11:33:12 2018 +0100

    Indent continuations

diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst
index 9470ee164..33c1584da 100644
--- a/doc/sphinx/installation/install.rst
+++ b/doc/sphinx/installation/install.rst
@@ -161,8 +161,8 @@ requirements are installed.
 By convention, consider installing Varnish under `/opt/local` using::
 
 	./configure \
-	--prefix=/opt/local \
-	--mandir=/opt/local/man
+	        --prefix=/opt/local \
+	        --mandir=/opt/local/man
 
 Alternatively, building with Solaris Studio 12.4 should work
 considering the following recommendations:
@@ -172,11 +172,11 @@ considering the following recommendations:
   dependencies are installed. Example for `/opt/local`::
 
 	./configure \
-	--prefix=/opt/local \
-	--mandir=/opt/local/man \
-	CPPFLAGS="-I/opt/local/include" \
-	CFLAGS="-m64" \
-	LDFLAGS="-L/opt/local/lib -R/opt/local/lib"
+	        --prefix=/opt/local \
+	        --mandir=/opt/local/man \
+	        CPPFLAGS="-I/opt/local/include" \
+	        CFLAGS="-m64" \
+	        LDFLAGS="-L/opt/local/lib -R/opt/local/lib"
 
 Compiling Varnish
 -----------------

From dridi.boukelmoune at gmail.com  Tue Oct 30 10:34:09 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 30 Oct 2018 10:34:09 +0000 (UTC)
Subject: [master] a2e9b25ba Missing line continuation
Message-ID: <20181030103409.D8194A0072@lists.varnish-cache.org>


commit a2e9b25ba44a63db1e0971f11cbb52cd6efb59ce
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Oct 30 11:32:17 2018 +0100

    Missing line continuation

diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst
index 896d29c65..9470ee164 100644
--- a/doc/sphinx/installation/install.rst
+++ b/doc/sphinx/installation/install.rst
@@ -173,7 +173,7 @@ considering the following recommendations:
 
 	./configure \
 	--prefix=/opt/local \
-	--mandir=/opt/local/man
+	--mandir=/opt/local/man \
 	CPPFLAGS="-I/opt/local/include" \
 	CFLAGS="-m64" \
 	LDFLAGS="-L/opt/local/lib -R/opt/local/lib"

From phk at FreeBSD.org  Tue Oct 30 13:17:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 30 Oct 2018 13:17:07 +0000 (UTC)
Subject: [master] 336e375c4 Reuse the "turn WS into VSB" code.
Message-ID: <20181030131707.A4046A4415@lists.varnish-cache.org>


commit 336e375c40ebf6af1eccd21708130f66f51456d8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 30 13:03:55 2018 +0000

    Reuse the "turn WS into VSB" code.

diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c
index 410d878be..23eb919a3 100644
--- a/bin/varnishd/cache/cache_vrt_filter.c
+++ b/bin/varnishd/cache/cache_vrt_filter.c
@@ -219,13 +219,51 @@ VCL_VRT_Init(void)
 /*--------------------------------------------------------------------
  */
 
-static void
-vbf_default_filter_list(const struct busyobj *bo, struct vsb *vsb)
+typedef void filter_list_t(const void *, struct vsb *vsb);
+
+static const char *
+filter_on_ws(struct ws *ws, filter_list_t *func, const void *arg)
+{
+	unsigned u;
+	struct vsb vsb[1];
+
+	AN(func);
+	AN(arg);
+	u = WS_Reserve(ws, 0);
+	if (u == 0) {
+		WS_Release(ws, 0);
+		WS_MarkOverflow(ws);
+		return (NULL);
+	}
+	AN(VSB_new(vsb, ws->f, u, VSB_FIXEDLEN));
+	func(arg, vsb);
+	if (VSB_finish(vsb)) {
+		WS_Release(ws, 0);
+		WS_MarkOverflow(ws);
+		return (NULL);
+	}
+	if (VSB_len(vsb)) {
+		WS_Release(ws, VSB_len(vsb) + 1);
+		return (VSB_data(vsb) + 1);
+	}
+	WS_Release(ws, 0);
+	return ("");
+}
+
+/*--------------------------------------------------------------------
+ */
+
+static void v_matchproto_(filter_list_t)
+vbf_default_filter_list(const void *arg, struct vsb *vsb)
 {
+	const struct busyobj *bo;
 	const char *p;
-	int do_gzip = bo->do_gzip;
-	int do_gunzip = bo->do_gunzip;
-	int is_gzip = 0, is_gunzip = 0;
+	int do_gzip, do_gunzip, is_gzip = 0, is_gunzip = 0;
+
+	CAST_OBJ_NOTNULL(bo, arg, BUSYOBJ_MAGIC);
+
+	do_gzip = bo->do_gzip;
+	do_gunzip = bo->do_gunzip;
 
 	/*
 	 * The VCL variables beresp.do_g[un]zip tells us how we want the
@@ -282,36 +320,28 @@ vbf_default_filter_list(const struct busyobj *bo, struct vsb *vsb)
 const char *
 VBF_Get_Filter_List(struct busyobj *bo)
 {
-	unsigned u;
-	struct vsb vsb[1];
 
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
-	u = WS_Reserve(bo->ws, 0);
-	if (u == 0) {
-		WS_Release(bo->ws, 0);
-		WS_MarkOverflow(bo->ws);
-		return (NULL);
-	}
-	AN(VSB_new(vsb, bo->ws->f, u, VSB_FIXEDLEN));
-	vbf_default_filter_list(bo, vsb);
-	if (VSB_finish(vsb)) {
-		WS_Release(bo->ws, 0);
-		WS_MarkOverflow(bo->ws);
-		return (NULL);
-	}
-	if (VSB_len(vsb)) {
-		WS_Release(bo->ws, VSB_len(vsb) + 1);
-		return (VSB_data(vsb) + 1);
-	}
-	WS_Release(bo->ws, 0);
-	return ("");
+	return (filter_on_ws(bo->ws, vbf_default_filter_list, bo));
+}
+
+/*--------------------------------------------------------------------
+ */
+
+static void v_matchproto_(filter_list_t)
+resp_default_filter_list(const void *arg, struct vsb *vsb)
+{
+	const struct req *req;
+
+	CAST_OBJ_NOTNULL(req, arg, REQ_MAGIC);
+	(void)vsb;
 }
 
 static const char *
 resp_Get_Filter_List(struct req *req)
 {
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	return ("");
+	return (filter_on_ws(req->ws, resp_default_filter_list, req));
 }
 
 /*--------------------------------------------------------------------*/

From nils.goroll at uplex.de  Tue Oct 30 16:52:07 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 30 Oct 2018 16:52:07 +0000 (UTC)
Subject: [master] f3c42c374 Assert that dynamic privs hang off a single vcl
Message-ID: <20181030165207.F09BAAC542@lists.varnish-cache.org>


commit f3c42c37454dacd61214c5e89f309348faa3d6ab
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Oct 30 17:46:49 2018 +0100

    Assert that dynamic privs hang off a single vcl
    
    we could also retire the vcl member of struct vrt_priv, but it could
    be useful for debugging.

diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c
index e4dd16479..e150ee4ac 100644
--- a/bin/varnishd/cache/cache_vrt_priv.c
+++ b/bin/varnishd/cache/cache_vrt_priv.c
@@ -102,9 +102,10 @@ vrt_priv_dynamic(const struct vcl *vcl, struct ws *ws,
 
 	VTAILQ_FOREACH(vp, &vps->privs, list) {
 		CHECK_OBJ_NOTNULL(vp, VRT_PRIV_MAGIC);
-		if (vp->vcl == vcl && vp->id == id &&
-		    vp->vmod_id == vmod_id)
+		if (vp->id == id && vp->vmod_id == vmod_id) {
+			assert(vp->vcl == vcl);
 			return (vp->priv);
+		}
 	}
 	vp = WS_Alloc(ws, sizeof *vp);
 	if (vp == NULL)

From nils.goroll at uplex.de  Tue Oct 30 17:02:08 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 30 Oct 2018 17:02:08 +0000 (UTC)
Subject: [master] 86af5ce09 actually, the scope must not change either
Message-ID: <20181030170208.A67C4AC9F3@lists.varnish-cache.org>


commit 86af5ce095a6e39e313cf7c6c23a49edba4aace6
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Oct 30 17:58:46 2018 +0100

    actually, the scope must not change either
    
    the priv id is basically equivalent to the head (struct vrt_privs),
    and a vrt privs list thus can not contain privs with another id.
    
    So we could also retire the id member of struct vrt_priv...

diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c
index e150ee4ac..754605146 100644
--- a/bin/varnishd/cache/cache_vrt_priv.c
+++ b/bin/varnishd/cache/cache_vrt_priv.c
@@ -42,7 +42,7 @@ struct vrt_priv {
 	VTAILQ_ENTRY(vrt_priv)		list;
 	struct vmod_priv		priv[1];
 	const struct vcl		*vcl;
-	uintptr_t			id;
+	uintptr_t			id;	// = scope / vrt_privs
 	uintptr_t			vmod_id;
 };
 
@@ -102,8 +102,9 @@ vrt_priv_dynamic(const struct vcl *vcl, struct ws *ws,
 
 	VTAILQ_FOREACH(vp, &vps->privs, list) {
 		CHECK_OBJ_NOTNULL(vp, VRT_PRIV_MAGIC);
-		if (vp->id == id && vp->vmod_id == vmod_id) {
+		if (vp->vmod_id == vmod_id) {
 			assert(vp->vcl == vcl);
+			assert(vp->id == id);
 			return (vp->priv);
 		}
 	}

From nils.goroll at uplex.de  Wed Oct 31 11:51:12 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 31 Oct 2018 11:51:12 +0000 (UTC)
Subject: [master] 91a16493f add a trivial VRT_priv_task benchmark
Message-ID: <20181031115113.0984992216@lists.varnish-cache.org>


commit 91a16493fa9fa2536278ce957a1769bf317e056a
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Oct 31 12:35:55 2018 +0100

    add a trivial VRT_priv_task benchmark

diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc
index a67928ca9..533f7d246 100644
--- a/bin/varnishtest/tests/v00041.vtc
+++ b/bin/varnishtest/tests/v00041.vtc
@@ -10,7 +10,7 @@ server s1 {
 	txresp
 } -start
 
-varnish v1 -arg "-p debug=+vclrel" -vcl+backend {
+varnish v1 -arg "-p debug=+vclrel -p workspace_client=1m" -vcl+backend {
 	import debug;
 	import std;
 
@@ -26,11 +26,24 @@ varnish v1 -arg "-p debug=+vclrel" -vcl+backend {
 	}
 
 	sub vcl_recv {
+		if (req.url == "/perf") {
+			return (synth(200));
+		}
 		debug.test_priv_task(req.url);
 		set req.http.x0 = debug.test_priv_task();
 		debug.test_priv_task("bazz");
 	}
 
+	sub vcl_synth {
+		std.log("discard 100 " + debug.priv_perf(100));
+		std.log("perf    1 " + debug.priv_perf(1));
+		std.log("perf   10 " + debug.priv_perf(10));
+		std.log("perf  100 " + debug.priv_perf(100));
+		## unbearable with the list implementation - ok with rb tree
+		# std.log("perf 1000 " + debug.priv_perf(1000));
+		return (deliver);
+	}
+
 	sub vcl_deliver {
 		set resp.http.x0 = req.http.x0;
 		set resp.http.x1 = debug.test_priv_task();
@@ -109,6 +122,9 @@ client c1 {
 	expect resp.http.bx0 == "b /snafu"
 	expect resp.http.bx1 == "b /snafu"
 	expect resp.http.bo1 == "b /snafu"
+
+	txreq -url /perf
+	rxresp
 } -run
 
 shell "echo 'vcl 4.0; backend foo { .host = \"${s1_addr}\"; .port = \"${s1_port}\"; }' > ${tmpdir}/_b00014.vcl"
diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc
index fecf9b150..ad24840da 100644
--- a/lib/libvmod_debug/vmod.vcc
+++ b/lib/libvmod_debug/vmod.vcc
@@ -201,3 +201,14 @@ Return the string formed by concatenating the given strings.
 $Function VOID sethdr(HEADER, STRANDS)
 
 Set the given header with the concatenation of the given strings.
+
+$Function DURATION priv_perf(INT size, INT rounds=10000)
+
+Benchmark VRT_priv_task() with `size` elements, iterating `rounds`
+times.
+
+Returns the average time taken for each call scaled up from nanoseconds
+to seconds - iow the value given as seconds is actually the duration
+in nanoseconds.
+
+For comparable results, a higher size run should called first and discarded.
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index a3a78975a..3a76f6c7e 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -552,3 +552,42 @@ xyzzy_sethdr(VRT_CTX, VCL_HEADER hs, VCL_STRANDS s)
 		}
 	}
 }
+
+VCL_DURATION
+xyzzy_priv_perf(VRT_CTX, VCL_INT size, VCL_INT rounds)
+{
+	vtim_mono t0, t1;
+	vtim_dur d;
+	struct vmod_priv *p;
+	VCL_INT s, r;
+	uintptr_t check = 0;
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+
+	for (s = 1; s <= size; s++) {
+		p = VRT_priv_task(ctx, (void *)s);
+		if (p == NULL) {
+			VRT_fail(ctx, "no priv task - out of ws?");
+			return (-1.0);
+		}
+		p->priv = NULL;
+	}
+
+	t0 = VTIM_mono();
+	for (r = 0; r < rounds; r++) {
+		for (s = 1; s <= size; s++) {
+			p = VRT_priv_task(ctx, (void *)s);
+			check += (uintptr_t)p->priv;
+			p->priv = (void *)(uintptr_t)(s * rounds + r);
+		}
+	}
+	t1 = VTIM_mono();
+
+	d = (t1 - t0) * 1e9 / size / rounds;
+
+	VSLb(ctx->vsl, SLT_Debug,
+	     "perf size %ld rounds %ld time %.9fns check %ld",
+	     size, rounds, d, check);
+
+	return (d);
+}

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:04 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:04 +0000 (UTC)
Subject: [6.0] 4f672b6e1 Add a note about concurrency in the VSC_Iter
 documentation.
Message-ID: <20181031130804.78F0494CFD@lists.varnish-cache.org>


commit 4f672b6e1f1fa5fe319d80a77b923a825517305f
Author: Geoff Simmons <geoff at uplex.de>
Date:   Mon Aug 20 10:03:56 2018 +0200

    Add a note about concurrency in the VSC_Iter documentation.

diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
index 82fa89fbd..3ae67290f 100644
--- a/include/vapi/vsc.h
+++ b/include/vapi/vsc.h
@@ -135,6 +135,10 @@ int VSC_Iter(struct vsc *, struct vsm *, VSC_iter_f *, void *priv);
 	 *
 	 * The returned points are valid until the next call to VSC_Iter()
 	 *
+	 * Not safe for concurrent reads with the same vsc and vsm
+	 * handles.  For concurrency, initalize and attach separate
+	 * structs vsc and vsm.
+	 *
 	 * Arguments:
 	 *	    vd: The vsm context
 	 *	  func: The callback function

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:04 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:04 +0000 (UTC)
Subject: [6.0] 79bc5ace9 Add a cli send/expect facility to HAproxy support.
Message-ID: <20181031130804.721EB94CFB@lists.varnish-cache.org>


commit 79bc5ace9396e6dba618bbd6e9a316703908ab50
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Aug 16 09:23:42 2018 +0000

    Add a cli send/expect facility to HAproxy support.
    
    Submitted by:   Frederic Lecaille <flecaille at haproxy.com>

diff --git a/bin/varnishtest/tests/h00001.vtc b/bin/varnishtest/tests/h00001.vtc
index 1b86d81a0..f8b98a2ec 100644
--- a/bin/varnishtest/tests/h00001.vtc
+++ b/bin/varnishtest/tests/h00001.vtc
@@ -30,3 +30,9 @@ client c1 -connect ${h1_fe1_sock} {
     expect resp.status == 200
     expect resp.body == "s1 >>> Hello world!"
 } -run
+
+haproxy h1 -cli {
+    send "show info"
+    expect ~ "Name: HAProxy"
+} -wait
+
diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c
index a3e89c713..157d0e67c 100644
--- a/bin/varnishtest/vtc_haproxy.c
+++ b/bin/varnishtest/vtc_haproxy.c
@@ -34,12 +34,15 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/stat.h> /* for MUSL (mode_t) */
+#include <sys/types.h>
+#include <sys/socket.h>
 #include <unistd.h>
 
 #include "vtc.h"
 
 #include "vfil.h"
 #include "vpf.h"
+#include "vre.h"
 #include "vtcp.h"
 #include "vtim.h"
 
@@ -50,6 +53,7 @@
 #define HAPROXY_EXPECT_EXIT	(128 + HAPROXY_SIGNAL)
 #define HAPROXY_GOOD_CONF	"Configuration file is valid"
 
+
 struct haproxy {
 	unsigned		magic;
 #define HAPROXY_MAGIC		0x8a45cf75
@@ -73,7 +77,10 @@ struct haproxy {
 	int			expect_signal;
 	int			its_dead_jim;
 
+	/* UNIX socket CLI. */
 	char			*cli_fn;
+	/* TCP socket CLI. */
+	struct haproxy_cli *cli;
 
 	char			*workdir;
 	struct vsb		*msgs;
@@ -82,6 +89,269 @@ struct haproxy {
 static VTAILQ_HEAD(, haproxy)	haproxies =
     VTAILQ_HEAD_INITIALIZER(haproxies);
 
+struct haproxy_cli {
+	unsigned		magic;
+#define HAPROXY_CLI_MAGIC	0xb09a4ed8
+	struct vtclog		*vl;
+	char			running;
+
+	char			*spec;
+
+	int			sock;
+	char			connect[256];
+
+	pthread_t		tp;
+	size_t			txbuf_sz;
+	char			*txbuf;
+	size_t			rxbuf_sz;
+	char			*rxbuf;
+
+	double			timeout;
+};
+
+/**********************************************************************
+ * Socket connect (same as client_tcp_connect()).
+ */
+
+static int
+haproxy_cli_tcp_connect(struct vtclog *vl, const char *addr, double tmo,
+    const char **errp)
+{
+	int fd;
+	char mabuf[32], mpbuf[32];
+
+	fd = VTCP_open(addr, NULL, tmo, errp);
+	if (fd < 0)
+		return fd;
+	VTCP_myname(fd, mabuf, sizeof mabuf, mpbuf, sizeof mpbuf);
+	vtc_log(vl, 3,
+	    "CLI connected fd %d from %s %s to %s", fd, mabuf, mpbuf, addr);
+	return fd;
+}
+
+/*
+ * SECTION: haproxy.cli haproxy CLI Specification
+ * SECTION: haproxy.cli.send
+ * send STRING
+ *         Push STRING on the CLI connection. STRING will be terminated by an
+ *         end of line character (\n).
+ */
+static void v_matchproto_(cmd_f)
+cmd_haproxy_cli_send(CMD_ARGS)
+{
+	struct vsb *vsb;
+	struct haproxy_cli *hc;
+	ssize_t wr;
+
+	(void)cmd;
+	(void)vl;
+	CAST_OBJ_NOTNULL(hc, priv, HAPROXY_CLI_MAGIC);
+	AZ(strcmp(av[0], "send"));
+	AN(av[1]);
+	AZ(av[2]);
+
+	vsb = VSB_new_auto();
+	AN(vsb);
+	AZ(VSB_cat(vsb, av[1]));
+	AZ(VSB_cat(vsb, "\n"));
+	AZ(VSB_finish(vsb));
+	if (hc->sock == -1) {
+		int fd;
+		const char *err;
+		struct vsb *vsb_connect;
+
+		vsb_connect = macro_expand(hc->vl, hc->connect);
+		AN(vsb_connect);
+		fd = haproxy_cli_tcp_connect(hc->vl,
+		    VSB_data(vsb_connect), 10., &err);
+		if (fd < 0)
+			vtc_fatal(hc->vl,
+			    "CLI failed to open %s: %s", VSB_data(vsb), err);
+		VSB_destroy(&vsb_connect);
+		hc->sock = fd;
+	}
+	vtc_dump(hc->vl, 4, "CLI send", VSB_data(vsb), -1);
+
+	wr = write(hc->sock, VSB_data(vsb), VSB_len(vsb));
+	if (wr != VSB_len(vsb))
+		vtc_fatal(hc->vl,
+		    "CLI fd %d send error %s", hc->sock, strerror(errno));
+
+	VSB_destroy(&vsb);
+}
+
+#define HAPROXY_CLI_RECV_LEN (1 << 14)
+static void
+haproxy_cli_recv(struct haproxy_cli *hc)
+{
+	ssize_t ret;
+	size_t rdz, left, off;
+
+	rdz = ret = off = 0;
+	/* We want to null terminate this buffer. */
+	left = hc->rxbuf_sz - 1;
+	while (!vtc_error && left > 0) {
+		VTCP_set_read_timeout(hc->sock, hc->timeout);
+
+		ret = recv(hc->sock, hc->rxbuf + off, HAPROXY_CLI_RECV_LEN, 0);
+		if (ret < 0) {
+			if (errno == EINTR || errno == EAGAIN)
+				continue;
+
+			vtc_fatal(hc->vl,
+			    "CLI fd %d recv() failed (%s)",
+			    hc->sock, strerror(errno));
+		}
+		/* Connection closed. */
+		if (ret == 0) {
+			if (hc->rxbuf[rdz - 1] != '\n')
+				vtc_fatal(hc->vl,
+				    "CLI rx timeout (fd: %d %.3fs ret: %zd)",
+				    hc->sock, hc->timeout, ret);
+
+			vtc_log(hc->vl, 4, "CLI connection normally closed");
+			vtc_log(hc->vl, 3, "CLI closing fd %d", hc->sock);
+			VTCP_close(&hc->sock);
+			break;
+		}
+
+		rdz += ret;
+		left -= ret;
+		off  += ret;
+	}
+	hc->rxbuf[rdz] = '\0';
+	vtc_dump(hc->vl, 4, "CLI recv", hc->rxbuf, rdz);
+}
+
+/*
+ * SECTION: haproxy.cli.expect
+ * expect OP STRING
+ *         Regex match the CLI reception buffer with STRING
+ *         if OP is ~ or, on the contraty, if OP is !~ check that there is
+ *         no regex match.
+ */
+static void v_matchproto_(cmd_f)
+cmd_haproxy_cli_expect(CMD_ARGS)
+{
+	struct haproxy_cli *hc;
+	vre_t *vre;
+	const char *error;
+	int erroroffset, i, ret;
+	char *cmp, *spec;
+
+	(void)cmd;
+	(void)vl;
+	CAST_OBJ_NOTNULL(hc, priv, HAPROXY_CLI_MAGIC);
+	AZ(strcmp(av[0], "expect"));
+	av++;
+
+	cmp = av[0];
+	spec = av[1];
+	AN(cmp);
+	AN(spec);
+	AZ(av[2]);
+
+	assert(!strcmp(cmp, "~") || !strcmp(cmp, "!~"));
+
+	haproxy_cli_recv(hc);
+
+	vre = VRE_compile(spec, 0, &error, &erroroffset);
+	if (!vre)
+		vtc_fatal(hc->vl, "CLI regexp error: '%s' (@%d) (%s)",
+		    error, erroroffset, spec);
+
+	i = VRE_exec(vre, hc->rxbuf, strlen(hc->rxbuf), 0, 0, NULL, 0, 0);
+
+	VRE_free(&vre);
+
+	ret = (i >= 0 && *cmp == '~') || (i < 0 && *cmp == '!');
+	if (!ret)
+		vtc_fatal(hc->vl, "CLI expect failed %s \"%s\"", cmp, spec);
+	else
+		vtc_log(hc->vl, 4, "CLI expect match %s \"%s\"", cmp, spec);
+}
+
+static const struct cmds haproxy_cli_cmds[] = {
+#define CMD_HAPROXY_CLI(n) { #n, cmd_haproxy_cli_##n },
+	CMD_HAPROXY_CLI(send)
+	CMD_HAPROXY_CLI(expect)
+#undef CMD_HAPROXY_CLI
+};
+
+/**********************************************************************
+ * HAProxy CLI client thread
+ */
+
+static void *
+haproxy_cli_thread(void *priv)
+{
+	struct haproxy_cli *hc;
+	struct vsb *vsb;
+	int fd;
+	const char *err;
+
+	CAST_OBJ_NOTNULL(hc, priv, HAPROXY_CLI_MAGIC);
+	AN(*hc->connect);
+
+	vsb = macro_expand(hc->vl, hc->connect);
+	AN(vsb);
+
+	fd = haproxy_cli_tcp_connect(hc->vl, VSB_data(vsb), 10., &err);
+	if (fd < 0)
+		vtc_fatal(hc->vl,
+		    "CLI failed to open %s: %s", VSB_data(vsb), err);
+	(void)VTCP_blocking(fd);
+	hc->sock = fd;
+	parse_string(hc->spec, haproxy_cli_cmds, hc, hc->vl);
+	vtc_log(hc->vl, 2, "CLI ending");
+	VSB_destroy(&vsb);
+	return (NULL);
+}
+
+/**********************************************************************
+ * Wait for the CLI client thread to stop
+ */
+
+static void
+haproxy_cli_wait(struct haproxy_cli *hc)
+{
+	void *res;
+
+	CHECK_OBJ_NOTNULL(hc, HAPROXY_CLI_MAGIC);
+	vtc_log(hc->vl, 2, "CLI waiting");
+	AZ(pthread_join(hc->tp, &res));
+	if (res != NULL)
+		vtc_fatal(hc->vl, "CLI returned \"%s\"", (char *)res);
+	REPLACE(hc->spec, NULL);
+	hc->tp = 0;
+	hc->running = 0;
+}
+
+/**********************************************************************
+ * Start the CLI client thread
+ */
+
+static void
+haproxy_cli_start(struct haproxy_cli *hc)
+{
+	CHECK_OBJ_NOTNULL(hc, HAPROXY_CLI_MAGIC);
+	vtc_log(hc->vl, 2, "CLI starting");
+	AZ(pthread_create(&hc->tp, NULL, haproxy_cli_thread, hc));
+	hc->running = 1;
+
+}
+
+/**********************************************************************
+ * Run the CLI client thread
+ */
+
+static void
+haproxy_cli_run(struct haproxy_cli *hc)
+{
+	haproxy_cli_start(hc);
+	haproxy_cli_wait(hc);
+}
+
 /**********************************************************************
  *
  */
@@ -131,6 +401,41 @@ haproxy_wait_pidfile(struct haproxy *h)
 		  h->name, buf_err);
 }
 
+/**********************************************************************
+ * Allocate and initialize a CLI client
+ */
+
+static struct haproxy_cli *
+haproxy_cli_new(struct haproxy *h)
+{
+	struct haproxy_cli *hc;
+
+	ALLOC_OBJ(hc, HAPROXY_CLI_MAGIC);
+	AN(hc);
+
+	hc->vl = h->vl;
+	hc->sock = -1;
+	bprintf(hc->connect, "${%s_cli_sock}", h->name);
+
+	hc->txbuf_sz = hc->rxbuf_sz = 2048 * 1024;
+	hc->txbuf = malloc(hc->txbuf_sz);
+	AN(hc->txbuf);
+	hc->rxbuf = malloc(hc->rxbuf_sz);
+	AN(hc->rxbuf);
+
+	return hc;
+}
+
+static void
+haproxy_cli_delete(struct haproxy_cli *hc)
+{
+	CHECK_OBJ_NOTNULL(hc, HAPROXY_CLI_MAGIC);
+	REPLACE(hc->spec, NULL);
+	REPLACE(hc->txbuf, NULL);
+	REPLACE(hc->rxbuf, NULL);
+	FREE_OBJ(hc);
+}
+
 /**********************************************************************
  * Allocate and initialize a haproxy
  */
@@ -170,6 +475,9 @@ haproxy_new(const char *name)
 	h->cfg_fn = strdup(buf);
 	AN(h->cfg_fn);
 
+	h->cli = haproxy_cli_new(h);
+	AN(h->cli);
+
 	bprintf(buf, "rm -rf %s ; mkdir -p %s", h->workdir, h->workdir);
 	AZ(system(buf));
 
@@ -201,6 +509,7 @@ haproxy_delete(struct haproxy *h)
 	free(h->cfg_fn);
 	free(h->pid_fn);
 	VSB_destroy(&h->args);
+	haproxy_cli_delete(h->cli);
 
 	/* XXX: MEMLEAK (?) */
 	FREE_OBJ(h);
@@ -323,6 +632,9 @@ haproxy_wait(struct haproxy *h)
 	if (h->pid < 0)
 		haproxy_start(h);
 
+	if (h->cli->spec)
+		haproxy_cli_run(h->cli);
+
 	closefd(&h->fds[1]);
 
 	sig = SIGINT;
@@ -445,6 +757,7 @@ haproxy_write_conf(const struct haproxy *h, const char *cfg, int auto_be)
 
 	VSB_printf(vsb, "    global\n\tstats socket %s "
 		   "level admin mode 600\n", h->cli_fn);
+	VSB_printf(vsb, "    stats socket \"fd@${cli}\" level admin\n");
 	AZ(VSB_cat(vsb, cfg));
 
 	if (auto_be)
@@ -508,6 +821,9 @@ haproxy_write_conf(const struct haproxy *h, const char *cfg, int auto_be)
  * \-arg STRING
  *         Pass an argument to haproxy, for example "-h simple_list".
  *
+ * \-cli STRING
+ *         Specify the spec to be run by the command line interface (CLI).
+ *
  * \-conf STRING
  *         Specify the configuration to be loaded by this HAProxy instance.
  *
@@ -598,6 +914,13 @@ cmd_haproxy(CMD_ARGS)
 			continue;
 		}
 
+		if (!strcmp(*av, "-cli")) {
+			REPLACE(h->cli->spec, av[1]);
+			if (h->tp)
+				haproxy_cli_run(h->cli);
+			av++;
+			continue;
+		}
 		if (!strcmp(*av, "-conf")) {
 			AN(av[1]);
 			haproxy_write_conf(h, av[1], 0);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:04 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:04 +0000 (UTC)
Subject: [6.0] 06468b0dd Typo
Message-ID: <20181031130804.9407394CFF@lists.varnish-cache.org>


commit 06468b0ddd617aae76f1c8128b541389b786c0a4
Author: Geoff Simmons <geoff at uplex.de>
Date:   Mon Aug 20 12:51:33 2018 +0200

    Typo

diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h
index 3ae67290f..6976c857b 100644
--- a/include/vapi/vsc.h
+++ b/include/vapi/vsc.h
@@ -136,7 +136,7 @@ int VSC_Iter(struct vsc *, struct vsm *, VSC_iter_f *, void *priv);
 	 * The returned points are valid until the next call to VSC_Iter()
 	 *
 	 * Not safe for concurrent reads with the same vsc and vsm
-	 * handles.  For concurrency, initalize and attach separate
+	 * handles.  For concurrency, initialize and attach separate
 	 * structs vsc and vsm.
 	 *
 	 * Arguments:

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:04 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:04 +0000 (UTC)
Subject: [6.0] 3c3e37a22 look for sphinx-build during configure add
 --with-sphinx-build
Message-ID: <20181031130804.C4FE194D04@lists.varnish-cache.org>


commit 3c3e37a22a35c89fe0564ee3b7ee95e0c9bd06f3
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Aug 21 09:19:16 2018 +0200

    look for sphinx-build during configure add --with-sphinx-build
    
    ... argument to specify a custom location
    
    Fixes #2742

diff --git a/configure.ac b/configure.ac
index 3fa117537..0c0266aae 100644
--- a/configure.ac
+++ b/configure.ac
@@ -46,6 +46,15 @@ if test "x$RST2MAN" = "xno"; then
     [rst2man is needed to build Varnish, please install python-docutils.])
 fi
 
+AC_ARG_WITH([sphinx-build],
+  AS_HELP_STRING([--with-sphinx-build=PATH], [Location of sphinx-build (auto)]),
+  [SPHINX="$withval"],
+  AC_CHECK_PROGS(SPHINX, [sphinx-build], [no]))
+if test "x$SPHINX" = "xno"; then
+  AC_MSG_ERROR(
+    [sphinx-build is needed to build Varnish, please install python-sphinx.])
+fi
+
 AC_ARG_WITH([rst2html],
                AS_HELP_STRING([--with-rst2html=PATH],
                               [Location of rst2html (auto)]),
diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index f2d0df765..5eb2a1361 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line.
 SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build -W -q -N
+SPHINXBUILD   = $(SPHINX) -W -q -N
 PAPER         = a4
 BUILDDIR      = $(builddir)/build
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:05 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:05 +0000 (UTC)
Subject: [6.0] 9b64ca3d8 Align handling of STRINGS derived types.
Message-ID: <20181031130805.0593A94D09@lists.varnish-cache.org>


commit 9b64ca3d8e2f9850ddd4635981633ca0a435e05b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 22 06:52:48 2018 +0000

    Align handling of STRINGS derived types.
    
    Fixes:  #2745
    
    Conflicts:
            bin/varnishtest/tests/v00020.vtc

diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 3a6860abe..01f2d92c0 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -103,6 +103,7 @@ struct type {
 	const char		*name;
 	const char		*tostring;
 	vcc_type_t		multype;
+	int			stringform;
 };
 
 #define VCC_TYPE(foo)		extern const struct type foo[1];
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index b420dedf8..4bc8ea5d9 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -73,7 +73,7 @@ vcc_islit(const struct expr *e)
 static const char *
 vcc_utype(vcc_type_t t)
 {
-	if (t == STRINGS || t == STRING_LIST)
+	if (t == STRINGS || t->stringform)
 		t = STRING;
 	return (t->name);
 }
@@ -278,7 +278,7 @@ vcc_expr_tostring(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 	uint8_t	constant = EXPR_VAR;
 
 	CHECK_OBJ_NOTNULL(*e, EXPR_MAGIC);
-	assert(fmt == STRINGS || fmt == STRING_LIST || fmt == STRING);
+	assert(fmt == STRINGS || fmt->stringform);
 	assert(fmt != (*e)->fmt);
 
 	p = (*e)->fmt->tostring;
@@ -361,6 +361,7 @@ vcc_Eval_Handle(struct vcc *tl, struct expr **e, struct token *t,
 	(void)t;
 	(void)tl;
 	AN(sym->rname);
+	AZ(type->stringform);
 
 	if (sym->type != STRING && type == STRINGS) {
 		*e = vcc_mk_expr(STRINGS, "\"%s\"", sym->name);
@@ -1274,22 +1275,26 @@ vcc_expr0(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 	assert(fmt != STRINGS);
 	*e = NULL;
 	t1 = tl->t;
-	if (fmt == STRING_LIST || fmt == STRING)
+	if (fmt->stringform)
 		vcc_expr_cor(tl, e, STRINGS);
 	else
 		vcc_expr_cor(tl, e, fmt);
 	ERRCHK(tl);
-	assert((*e)->fmt != STRING_LIST && (*e)->fmt != STRING);
-
-	if ((*e)->fmt == STRINGS && fmt == STRING_LIST)
-		(*e)->fmt = STRING_LIST;
-	else if ((*e)->fmt == STRINGS && fmt == STRING)
-		*e = vcc_expr_edit(tl, STRING, "\vS", *e, NULL);
-	else if ((*e)->fmt == STRINGS && fmt == STRANDS) {
-		*e = vcc_expr_edit(tl, STRANDS, "\vT", (*e), NULL);
-	} else if ((*e)->fmt != STRINGS &&
-	    (fmt == STRING || fmt == STRING_LIST))
-		vcc_expr_tostring(tl, e, fmt);
+	assert(!(*e)->fmt->stringform);
+
+	if ((*e)->fmt != STRINGS && fmt->stringform)
+		vcc_expr_tostring(tl, e, STRINGS);
+
+	if ((*e)->fmt == STRINGS && fmt->stringform) {
+		if (fmt == STRING_LIST)
+			(*e)->fmt = STRING_LIST;
+		else if (fmt == STRING)
+			*e = vcc_expr_edit(tl, STRING, "\vS", *e, NULL);
+		else if (fmt == STRANDS)
+			*e = vcc_expr_edit(tl, STRANDS, "\vT", (*e), NULL);
+		else
+			WRONG("Unhandled stringform");
+	}
 
 	if ((*e)->fmt == STRING_LIST)
 		*e = vcc_expr_edit(tl, STRING_LIST,
diff --git a/lib/libvcc/vcc_types.c b/lib/libvcc/vcc_types.c
index dbeaacb09..2e5660994 100644
--- a/lib/libvcc/vcc_types.c
+++ b/lib/libvcc/vcc_types.c
@@ -131,11 +131,13 @@ const struct type STEVEDORE[1] = {{
 const struct type STRING[1] = {{
 	.magic =		TYPE_MAGIC,
 	.name =			"STRING",
+	.stringform =		1,
 }};
 
 const struct type STRANDS[1] = {{
 	.magic =		TYPE_MAGIC,
 	.name =			"STRANDS",
+	.stringform =		1,
 }};
 
 const struct type STRINGS[1] = {{
@@ -147,6 +149,7 @@ const struct type STRINGS[1] = {{
 const struct type STRING_LIST[1] = {{
 	.magic =		TYPE_MAGIC,
 	.name =			"STRING_LIST",
+	.stringform =		1,
 }};
 
 const struct type SUB[1] = {{

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:05 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:05 +0000 (UTC)
Subject: [6.0] 1c9aeaae8 We don't need stdlib.h unless we compile the code
Message-ID: <20181031130805.2725094D0C@lists.varnish-cache.org>


commit 1c9aeaae89a66521878b7cceeabbd2d98dd7e2a5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 22 06:59:51 2018 +0000

    We don't need stdlib.h unless we compile the code

diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c
index c48b72396..d0a4c690d 100644
--- a/bin/varnishd/waiter/cache_waiter_epoll.c
+++ b/bin/varnishd/waiter/cache_waiter_epoll.c
@@ -33,10 +33,10 @@
 //lint -e{766}
 #include "config.h"
 
-#include <stdlib.h>
-
 #if defined(HAVE_EPOLL_CTL)
 
+#include <stdlib.h>
+
 #include <sys/epoll.h>
 
 #include <errno.h>

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:05 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:05 +0000 (UTC)
Subject: [6.0] ee7770164 Always AZ VSB_finish()
Message-ID: <20181031130805.4E46694D10@lists.varnish-cache.org>


commit ee777016437fe44913140abca08e312b6577a2f8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 22 07:00:07 2018 +0000

    Always AZ VSB_finish()

diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c
index a36d657c0..aea59e9a8 100644
--- a/bin/varnishd/http2/cache_http2_deliver.c
+++ b/bin/varnishd/http2/cache_http2_deliver.c
@@ -243,7 +243,7 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 		AZ(h2_enc_len(&resp, 7, sz, 0));
 		AZ(VSB_bcat(&resp, r, sz));
 	}
-	VSB_finish(&resp);
+	AZ(VSB_finish(&resp));
 	sz = VSB_len(&resp);
 
 	AZ(req->wrk->v1l);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:05 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:05 +0000 (UTC)
Subject: [6.0] a6ab945eb Add missing break
Message-ID: <20181031130805.7126194D14@lists.varnish-cache.org>


commit a6ab945ebe966f4da96dbc5b32364b60389712c5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 22 07:08:12 2018 +0000

    Add missing break
    
    Conflicts:
            bin/varnishd/cache/cache_vrt.c

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 3851089cd..41a689732 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -437,6 +437,7 @@ h2_win_adjust(const struct h2_sess *h2, uint32_t oldval, uint32_t newval)
 			 * rfc7540,l,2676,2680
 			 */
 			r2->t_window += (int64_t)newval - oldval;
+			break;
 		default:
 			break;
 		}

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:05 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:05 +0000 (UTC)
Subject: [6.0] 3ffc27d53 Slight polish
Message-ID: <20181031130805.8DFA094D18@lists.varnish-cache.org>


commit 3ffc27d53c033086a0621d6f354704375b6d581d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 22 07:11:46 2018 +0000

    Slight polish
    
    Conflicts:
            lib/libvmod_blob/hex.c

diff --git a/lib/libvmod_blob/hex.c b/lib/libvmod_blob/hex.c
index 4124100f4..614404d9e 100644
--- a/lib/libvmod_blob/hex.c
+++ b/lib/libvmod_blob/hex.c
@@ -108,7 +108,7 @@ hex_decode(const enum encoding dec, char *restrict const buf,
 	char *dest = buf;
 	const char *b;
 	unsigned char extranib = 0;
-	ssize_t len = 0;
+	size_t len = 0;
 	va_list ap2;
 
 	AN(buf);
@@ -122,26 +122,20 @@ hex_decode(const enum encoding dec, char *restrict const buf,
 		b = s;
 		while (*s) {
 			if (!isxdigit(*s++)) {
-				len = -1;
-				break;
+				errno = EINVAL;
+				return (-1);
 			}
 		}
-		if (len == -1)
-			break;
 		len += s - b;
 	}
 	va_end(ap2);
 
 	if (len == 0)
 		return 0;
-	if (len == -1) {
-		errno = EINVAL;
-		return -1;
-	}
 	if (n != -1 && len > n)
 		len = n;
 
-	if ((len+1) >> 1 > buflen) {
+	if (((len+1) >> 1) > buflen) {
 		errno = ENOMEM;
 		return -1;
 	}

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:05 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:05 +0000 (UTC)
Subject: [6.0] 55deeaf08 Allow a string argument to return(fail("Because!"));
Message-ID: <20181031130805.BD2E794D21@lists.varnish-cache.org>


commit 55deeaf08fc9ac6e2be88bb46c1051b1a922e2d8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 22 07:46:39 2018 +0000

    Allow a string argument to return(fail("Because!"));
    
    Fixes #2694

diff --git a/bin/varnishtest/tests/v00040.vtc b/bin/varnishtest/tests/v00040.vtc
index 408169ffe..573da73b6 100644
--- a/bin/varnishtest/tests/v00040.vtc
+++ b/bin/varnishtest/tests/v00040.vtc
@@ -14,9 +14,15 @@ client c1 {
 
 varnish v1 -errvcl {VCL "vcl2" Failed initialization} {
 	sub vcl_init {
-		return (fail);
+		return(fail("Do Not Press This Button Again"));
 	}
+	backend b1 { .host = "${s1_addr}"; }
+}
 
+varnish v1 -errvcl {Do Not Press} {
+	sub vcl_init {
+		return(fail("Do Not Press This Button Again"));
+	}
 	backend b1 { .host = "${s1_addr}"; }
 }
 
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 8e61f271e..ded906870 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -217,6 +217,22 @@ vcc_act_return_pass(struct vcc *tl)
 }
 /*--------------------------------------------------------------------*/
 
+static void
+vcc_act_return_fail(struct vcc *tl)
+{
+	ExpectErr(tl, '(');
+	vcc_NextToken(tl);
+	Fb(tl, 1, "VRT_fail(ctx,\n");
+	tl->indent += INDENT;
+	vcc_Expr(tl, STRING);
+	tl->indent -= INDENT;
+	ERRCHK(tl);
+	SkipToken(tl, ')');
+	Fb(tl, 1, ");\n");
+}
+
+/*--------------------------------------------------------------------*/
+
 static void
 vcc_act_return_synth(struct vcc *tl)
 {
@@ -321,6 +337,8 @@ vcc_act_return(struct vcc *tl, struct token *t, struct symbol *sym)
 			vcc_act_return_vcl(tl);
 		else if (hand == VCL_RET_PASS)
 			vcc_act_return_pass(tl);
+		else if (hand == VCL_RET_FAIL)
+			vcc_act_return_fail(tl);
 		else {
 			VSB_printf(tl->sb, "Arguments not allowed.\n");
 			vcc_ErrWhere(tl, tl->t);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:05 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:05 +0000 (UTC)
Subject: [6.0] deeca0fd2 vcc: vmod function named argument clarifications
Message-ID: <20181031130805.DEAB094D31@lists.varnish-cache.org>


commit deeca0fd24117f9aa7b17096a22e440720147572
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Apr 10 17:48:41 2018 +0200

    vcc: vmod function named argument clarifications
    
    * either NULL or properly named
    * only named arguments can be possibly be speficied multiple times
    * only named arguments can be optional
    * positional arguments could be missing

diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 4bc8ea5d9..acb6f03f4 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -524,7 +524,6 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv,
 		if (!memcmp(vvp->value, "PRIV_", 5)) {
 			fa->result = vcc_priv_arg(tl, vvp->value,
 			    sym->name, sym->vmod);
-			fa->name = "";
 			continue;
 		}
 		fa->type = VCC_Type(vvp->value);
@@ -579,6 +578,7 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv,
 			return;
 		}
 		if (fa->result != NULL) {
+			AN(fa->name);
 			VSB_printf(tl->sb, "Argument '%s' already used\n",
 			    fa->name);
 			vcc_ErrWhere(tl, tl->t);
@@ -602,6 +602,7 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv,
 	VTAILQ_FOREACH_SAFE(fa, &head, list, fa2) {
 		n++;
 		if (fa->optional) {
+			AN(fa->name);
 			bprintf(ssa, "\v1.valid_%s = %d,\n",
 				fa->name, fa->avail);
 			e1 = vcc_expr_edit(tl, e1->fmt, ssa, e1, NULL);
@@ -611,7 +612,7 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv,
 		if (fa->result == NULL && fa->val != NULL)
 			fa->result = vcc_mk_expr(fa->type, "%s", fa->val);
 		if (fa->result != NULL && sa != NULL) {
-			if (fa->name && *fa->name != '\0')
+			if (fa->name)
 				bprintf(ssa, "\v1.%s = \v2,\n", fa->name);
 			else
 				bprintf(ssa, "\v1.arg%d = \v2,\n", n);
@@ -620,8 +621,11 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv,
 			e1 = vcc_expr_edit(tl, e1->fmt, "\v1,\n\v2",
 			    e1, fa->result);
 		} else if (!fa->optional) {
-			VSB_printf(tl->sb, "Argument '%s' missing\n",
-			    fa->name);
+			if (fa->name)
+				VSB_printf(tl->sb, "Argument '%s' missing\n",
+					   fa->name);
+			else
+				VSB_printf(tl->sb, "Argument %d missing\n", n);
 			vcc_ErrWhere(tl, tl->t);
 		}
 		free(fa);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:06 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:06 +0000 (UTC)
Subject: [6.0] 2cf6e8023 Improve VCC error messages.
Message-ID: <20181031130806.1411994D45@lists.varnish-cache.org>


commit 2cf6e8023ecfb626e1fc336fef5a72a6d430b163
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 22 08:50:12 2018 +0000

    Improve VCC error messages.
    
    Fixes:  #2696

diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc
index bc152ec01..d0a21eb1d 100644
--- a/bin/varnishtest/tests/v00018.vtc
+++ b/bin/varnishtest/tests/v00018.vtc
@@ -83,7 +83,7 @@ varnish v1 -errvcl {Unknown token '<<' when looking for STRING} {
 	sub vcl_recv { ban (<<); }
 }
 
-varnish v1 -errvcl {Expected an action, 'if', '{' or '}'} {
+varnish v1 -errvcl {Symbol not found} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_recv { ban_hash (if); }
 }
@@ -93,7 +93,7 @@ varnish v1 -vcl {
 	sub vcl_recv { ban ("req.url ~ foo"); }
 }
 
-varnish v1 -errvcl {Expected an action, 'if', '{' or '}'} {
+varnish v1 -errvcl "Symbol not found" {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_recv { kluf ; }
 }
diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc
index 4bebb20ae..9b0578522 100644
--- a/bin/varnishtest/tests/v00020.vtc
+++ b/bin/varnishtest/tests/v00020.vtc
@@ -20,7 +20,7 @@ varnish v1 -errvcl {Found: '0' at} { 0; }
 # VCLs tokenstream.
 # XXX: A better error message would be desirable
 
-varnish v1 -errvcl {Expected an action, 'if', } " sub vcl_recv { { } { "
+varnish v1 -errvcl {Symbol not found} " sub vcl_recv { { } { "
 
 varnish v1 -errvcl {Comparison of different types: INT '!=' STRING} {
 	sub vcl_recv {
diff --git a/bin/varnishtest/tests/v00021.vtc b/bin/varnishtest/tests/v00021.vtc
index 871f8732d..689b3faea 100644
--- a/bin/varnishtest/tests/v00021.vtc
+++ b/bin/varnishtest/tests/v00021.vtc
@@ -12,13 +12,13 @@ varnish v1 -errvcl {Variable is read only.} {
 	sub vcl_recv { call foo ; }
 }
 
-varnish v1 -errvcl "Expected an action, 'if', '{' or '}'" {
+varnish v1 -errvcl "Symbol not found" {
 	backend b { .host = "127.0.0.1"; }
 
 	sub vcl_recv { discard; }
 }
 
-varnish v1 -errvcl "Expected an action, 'if', '{' or '}'" {
+varnish v1 -errvcl "Symbol not found" {
 	backend b { .host = "127.0.0.1"; }
 
 	sub foo { discard; }
diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index a4bbf1182..cc0c34751 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -160,7 +160,7 @@ vcc_Compound(struct vcc *tl)
 			vcc_NextToken(tl);
 			tl->indent -= INDENT;
 			Fb(tl, 1, "}\n");
-			return;
+			break;
 		case CSRC:
 			if (tl->allow_inline_c) {
 				Fb(tl, 1, "%.*s\n",
@@ -177,25 +177,31 @@ vcc_Compound(struct vcc *tl)
 			VSB_printf(tl->sb,
 			    "End of input while in compound statement\n");
 			tl->err = 1;
-			return;
+			break;
 		case ID:
 			sym = VCC_SymbolGet(tl, SYM_NONE, SYMTAB_NOERR,
 			    XREF_NONE);
-			if (sym != NULL && sym->action != NULL) {
+			if (sym == NULL) {
+				VSB_printf(tl->sb, "Symbol not found.\n");
+				vcc_ErrWhere(tl, tl->t);
+			} else if (sym->action == NULL) {
+				VSB_printf(tl->sb,
+				    "Symbol cannot be used here.\n");
+				vcc_ErrWhere(tl, tl->t);
+			} else {
 				if (sym->action_mask != 0)
 					vcc_AddUses(tl, t, NULL,
 					    sym->action_mask,
 					    "Not a valid action");
 				sym->action(tl, t, sym);
-				break;
 			}
-			/* FALLTHROUGH */
+			break;
 		default:
 			/* We deliberately do not mention inline C */
 			VSB_printf(tl->sb,
 			    "Expected an action, 'if', '{' or '}'\n");
 			vcc_ErrWhere(tl, tl->t);
-			return;
+			break;
 		}
 		Fb(tl, 1, "if (*ctx->handling) return;\n");
 	}

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:06 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:06 +0000 (UTC)
Subject: [6.0] 977d246a2 So much for untested "stylistic" changes before
 commit.
Message-ID: <20181031130806.3093494D55@lists.varnish-cache.org>


commit 977d246a2b727f31b6b65635559c92891c4e7c74
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 22 09:29:41 2018 +0000

    So much for untested "stylistic" changes before commit.
    
    In other news:  VTEST works.

diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index cc0c34751..ee8830859 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -160,7 +160,7 @@ vcc_Compound(struct vcc *tl)
 			vcc_NextToken(tl);
 			tl->indent -= INDENT;
 			Fb(tl, 1, "}\n");
-			break;
+			return;
 		case CSRC:
 			if (tl->allow_inline_c) {
 				Fb(tl, 1, "%.*s\n",
@@ -177,31 +177,33 @@ vcc_Compound(struct vcc *tl)
 			VSB_printf(tl->sb,
 			    "End of input while in compound statement\n");
 			tl->err = 1;
-			break;
+			return;
 		case ID:
 			sym = VCC_SymbolGet(tl, SYM_NONE, SYMTAB_NOERR,
 			    XREF_NONE);
 			if (sym == NULL) {
 				VSB_printf(tl->sb, "Symbol not found.\n");
 				vcc_ErrWhere(tl, tl->t);
-			} else if (sym->action == NULL) {
+				return;
+			}
+			if (sym->action == NULL) {
 				VSB_printf(tl->sb,
 				    "Symbol cannot be used here.\n");
 				vcc_ErrWhere(tl, tl->t);
-			} else {
-				if (sym->action_mask != 0)
-					vcc_AddUses(tl, t, NULL,
-					    sym->action_mask,
-					    "Not a valid action");
-				sym->action(tl, t, sym);
+				return;
 			}
+			if (sym->action_mask != 0)
+				vcc_AddUses(tl, t, NULL,
+				    sym->action_mask,
+				    "Not a valid action");
+			sym->action(tl, t, sym);
 			break;
 		default:
 			/* We deliberately do not mention inline C */
 			VSB_printf(tl->sb,
 			    "Expected an action, 'if', '{' or '}'\n");
 			vcc_ErrWhere(tl, tl->t);
-			break;
+			return;
 		}
 		Fb(tl, 1, "if (*ctx->handling) return;\n");
 	}

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:06 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:06 +0000 (UTC)
Subject: [6.0] 65c8a547a style(9)-ification
Message-ID: <20181031130806.53DBB94D63@lists.varnish-cache.org>


commit 65c8a547a2ca6e3291d2fafc4f58d69108c39643
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Aug 22 11:37:32 2018 +0200

    style(9)-ification

diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index acb6f03f4..e016696a5 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -595,7 +595,7 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv,
 
 	if (sa != NULL)
 		e1 = vcc_mk_expr(rfmt, "%s(ctx%s,\v+ &(%s){\n",
-				 cfunc, extra, sa);
+		    cfunc, extra, sa);
 	else
 		e1 = vcc_mk_expr(rfmt, "%s(ctx%s\v+", cfunc, extra);
 	n = 0;
@@ -604,7 +604,7 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv,
 		if (fa->optional) {
 			AN(fa->name);
 			bprintf(ssa, "\v1.valid_%s = %d,\n",
-				fa->name, fa->avail);
+			    fa->name, fa->avail);
 			e1 = vcc_expr_edit(tl, e1->fmt, ssa, e1, NULL);
 		}
 		if (fa->result == NULL && fa->type == ENUM && fa->val != NULL)
@@ -623,7 +623,7 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv,
 		} else if (!fa->optional) {
 			if (fa->name)
 				VSB_printf(tl->sb, "Argument '%s' missing\n",
-					   fa->name);
+				    fa->name);
 			else
 				VSB_printf(tl->sb, "Argument %d missing\n", n);
 			vcc_ErrWhere(tl, tl->t);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:06 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:06 +0000 (UTC)
Subject: [6.0] 9fd55a067 avoid running checks twice for real now, hopefully
Message-ID: <20181031130806.757C594D77@lists.varnish-cache.org>


commit 9fd55a067ed422af6b2cc66ce16a3374df63fe95
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Aug 22 14:26:30 2018 +0200

    avoid running checks twice for real now, hopefully
    
    Ref: c2888919964c0498c511db5e7158244e15771eab
         72bacde7fbffc50c2a62c9988b99d8a830b60d30
         46f1557dd80ffbd75fab924e48ecfd020437b1e0

diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am
index 5c3cfae85..357ef9269 100644
--- a/bin/varnishtest/Makefile.am
+++ b/bin/varnishtest/Makefile.am
@@ -5,7 +5,6 @@ TEST_EXTENSIONS = .vtc
 TESTS = @VTC_TESTS@
 
 # Make sure we run check-local first
-check: check-am check-local
 check-am: check-local
 # See if list of checks have changed, recheck
 check-local:

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:06 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:06 +0000 (UTC)
Subject: [6.0] 177355720 Rework to hopefully fix the travis osx job
Message-ID: <20181031130806.9D0B394D7E@lists.varnish-cache.org>


commit 17735572051c9f97b1a8049feefcbb961586a17c
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Aug 22 15:56:56 2018 +0100

    Rework to hopefully fix the travis osx job

diff --git a/.travis.yml b/.travis.yml
index 021ed0de5..4441126b0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -38,6 +38,7 @@ before_install:
       brew update
       brew upgrade python
       brew install docutils sphinx-doc nghttp2
+      export PATH="/usr/local/opt/sphinx-doc/bin:$PATH"
     elif [[ -n "$CLANG" ]]; then
       wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
       sudo apt-add-repository -y \
@@ -60,7 +61,6 @@ script:
   - |
     if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
       export PYTHONPATH=`brew --prefix`/lib/python2.7/site-packages
-      export PATH="/usr/local/opt/sphinx-doc/bin:$PATH"
     fi
     if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then
       make -j3 distcheck

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:06 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:06 +0000 (UTC)
Subject: [6.0] 2ac6b4dec Don't mess with C-L when responding to HEAD
Message-ID: <20181031130806.CCA3994DA5@lists.varnish-cache.org>


commit 2ac6b4dece07758892b858ccf049b5941add21d5
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Thu Aug 16 19:22:52 2018 -0700

    Don't mess with C-L when responding to HEAD

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 17c9899c7..07447bee6 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -342,7 +342,7 @@ cnt_transmit(struct worker *wrk, struct req *req)
 	struct boc *boc;
 	const char *r;
 	uint16_t status;
-	int err, sendbody;
+	int err, sendbody, head;
 	intmax_t clval;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -364,7 +364,8 @@ cnt_transmit(struct worker *wrk, struct req *req)
 
 	/* RFC 7230, 3.3.3 */
 	status = http_GetStatus(req->resp);
-	if (!strcmp(req->http0->hd[HTTP_HDR_METHOD].b, "HEAD")) {
+	head = !strcmp(req->http0->hd[HTTP_HDR_METHOD].b, "HEAD");
+	if (head) {
 		if (req->objcore->flags & OC_F_PASS)
 			sendbody = -1;
 		else
@@ -396,8 +397,8 @@ cnt_transmit(struct worker *wrk, struct req *req)
 		}
 	}
 
-	if (sendbody < 0) {
-		/* Don't touch pass+HEAD C-L */
+	if (sendbody < 0 || head) {
+		/* Don't touch HEAD C-L */
 		sendbody = 0;
 	} else if (clval >= 0 && clval == req->resp_len) {
 		/* Reuse C-L header */
diff --git a/bin/varnishtest/tests/b00065.vtc b/bin/varnishtest/tests/b00065.vtc
new file mode 100644
index 000000000..c8df453bf
--- /dev/null
+++ b/bin/varnishtest/tests/b00065.vtc
@@ -0,0 +1,20 @@
+varnishtest "Test that C-L on HEAD request aren't nuked"
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr "content-length: 17"
+	expect req.method == "HEAD"
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_fetch {
+		set bereq.method = "HEAD";
+	}
+} -start
+
+client c1 {
+	txreq -req HEAD -hdr "connection: close"
+	rxresphdrs
+	expect resp.http.content-length == 17
+	expect_close
+} -run

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:06 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:06 +0000 (UTC)
Subject: [6.0] 3a9b119b2 Expect right after rxreq
Message-ID: <20181031130806.ED30894DB3@lists.varnish-cache.org>


commit 3a9b119b2540f56233a4ca254004070aefc46680
Author: guillaume quintard <gquintard at users.noreply.github.com>
Date:   Sun Aug 19 11:24:11 2018 -0700

    Expect right after rxreq

diff --git a/bin/varnishtest/tests/b00065.vtc b/bin/varnishtest/tests/b00065.vtc
index c8df453bf..5eb86d8d9 100644
--- a/bin/varnishtest/tests/b00065.vtc
+++ b/bin/varnishtest/tests/b00065.vtc
@@ -2,8 +2,8 @@ varnishtest "Test that C-L on HEAD request aren't nuked"
 
 server s1 {
 	rxreq
-	txresp -nolen -hdr "content-length: 17"
 	expect req.method == "HEAD"
+	txresp -nolen -hdr "content-length: 17"
 } -start
 
 varnish v1 -vcl+backend {

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:07 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:07 +0000 (UTC)
Subject: [6.0] 67ca3d891 Assign VMODS unique numbers,
 and make the count part of the VGC metadata
Message-ID: <20181031130807.2D36B94DC1@lists.varnish-cache.org>


commit 67ca3d891b0229cbf7067380ee4f4e8df42a011c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Aug 23 07:19:09 2018 +0000

    Assign VMODS unique numbers, and make the count part of the VGC metadata

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index ee49f55e4..4fc532138 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -666,6 +666,8 @@ struct VCL_conf {
 	const char			**srcname;
 	const char			**srcbody;
 
+	unsigned			nvmod;
+
 	vcl_event_f			*event_vcl;
 """)
 
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 69a2db9c8..a7dfc98d3 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -424,6 +424,7 @@ EmitStruct(const struct vcc *tl)
 	Fc(tl, 0, "\t.nsrc = VGC_NSRCS,\n");
 	Fc(tl, 0, "\t.srcname = srcname,\n");
 	Fc(tl, 0, "\t.srcbody = srcbody,\n");
+	Fc(tl, 0, "\t.nvmod = %u,\n", tl->vmod_count);
 #define VCL_MET_MAC(l,u,t,b) \
 	Fc(tl, 0, "\t." #l "_func = VGC_function_vcl_" #l ",\n");
 #include "tbl/vcl_returns.h"
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 01f2d92c0..befc15a94 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -252,7 +252,7 @@ struct vcc {
 	const char		*default_probe;
 
 	unsigned		unique;
-
+	unsigned		vmod_count;
 };
 
 struct method {
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index f34b0f3b1..a9cef1d71 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -336,6 +336,7 @@ vcc_ParseImport(struct vcc *tl)
 	vcc_json_always(tl, msym);
 
 	Fh(tl, 0, "\n/* --- BEGIN VMOD %.*s --- */\n\n", PF(mod));
+	Fh(tl, 0, "#define VMOD_NUMBER_%.*s %u\n", PF(mod), tl->vmod_count++);
 	Fh(tl, 0, "static struct vmod *VGC_vmod_%.*s;\n", PF(mod));
 	Fh(tl, 0, "static struct vmod_priv vmod_priv_%.*s;\n", PF(mod));
 	Fh(tl, 0, "\n%s\n", vmd->proto);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:07 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:07 +0000 (UTC)
Subject: [6.0] 9f50fc171 Call VCL_Onboard whenever a req or bo gets a
 particular VCL for the first time.
Message-ID: <20181031130807.5EE2C94DD0@lists.varnish-cache.org>


commit 9f50fc171290a7a17648cb948c3a1b1a19b4ebb9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Aug 23 07:47:55 2018 +0000

    Call VCL_Onboard whenever a req or bo gets a particular VCL for the first time.

diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 3533c665e..5527b588c 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -133,6 +133,7 @@ VBO_GetBusyObj(struct worker *wrk, const struct req *req)
 	bo->director_req = req->director_hint;
 	bo->vcl = req->vcl;
 	VCL_Ref(bo->vcl);
+	VCL_Onboard(NULL, bo);
 
 	bo->t_first = bo->t_prev = NAN;
 
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 1829804d6..516268c5b 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -164,6 +164,7 @@ ved_include(struct req *preq, const char *src, const char *host,
 	AZ(req->vcl);
 	req->vcl = preq->vcl;
 	preq->vcl = NULL;
+	VCL_Onboard(req, NULL);
 
 	req->req_step = R_STP_RECV;
 	req->t_req = preq->t_req;
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index e967f1d7f..92f21d53d 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -388,6 +388,7 @@ void VCL_Poll(void);
 void VCL_Ref(struct vcl *);
 void VCL_Refresh(struct vcl **);
 void VCL_Rel(struct vcl **);
+void VCL_Onboard(struct req *, struct busyobj *);
 const char *VCL_Return_Name(unsigned);
 const char *VCL_Method_Name(unsigned);
 void VCL_Bo2Ctx(struct vrt_ctx *, struct busyobj *);
diff --git a/bin/varnishd/cache/cache_vcl_vrt.c b/bin/varnishd/cache/cache_vcl_vrt.c
index f040f9027..36e34b53a 100644
--- a/bin/varnishd/cache/cache_vcl_vrt.c
+++ b/bin/varnishd/cache/cache_vcl_vrt.c
@@ -73,6 +73,16 @@ VCL_Method_Name(unsigned m)
 
 /*--------------------------------------------------------------------*/
 
+void
+VCL_Onboard(struct req *req, struct busyobj *bo)
+{
+	
+	CHECK_OBJ_ORNULL(req, REQ_MAGIC);
+	CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC);
+	assert(req != NULL || bo != NULL);
+	assert(req == NULL || bo == NULL);
+}
+
 void
 VCL_Refresh(struct vcl **vcc)
 {
@@ -265,6 +275,7 @@ VRT_vcl_select(VRT_CTX, VCL_VCL vcl)
 	vcl_get(&req->vcl, vcl);
 	/* XXX: better logging */
 	VSLb(ctx->req->vsl, SLT_Debug, "Now using %s VCL", vcl->loaded_name);
+	VCL_Onboard(req, NULL);
 }
 
 struct vclref *
diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c
index 650690950..b0dc5f472 100644
--- a/bin/varnishd/http1/cache_http1_fsm.c
+++ b/bin/varnishd/http1/cache_http1_fsm.c
@@ -330,6 +330,7 @@ http1_dissect(struct worker *wrk, struct req *req)
 	VCL_Refresh(&wrk->vcl);
 	req->vcl = wrk->vcl;
 	wrk->vcl = NULL;
+	VCL_Onboard(req, NULL);
 
 	HTTP_Setup(req->http, req->ws, req->vsl, SLT_ReqMethod);
 	req->err_code = HTTP1_DissectRequest(req->htc, req->http);
diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 41a689732..7713636c4 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -643,6 +643,7 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 	VCL_Refresh(&wrk->vcl);
 	req->vcl = wrk->vcl;
 	wrk->vcl = NULL;
+	VCL_Onboard(req, NULL);
 	req->acct.req_hdrbytes += h2->rxf_len;
 
 	HTTP_Setup(req->http, req->ws, req->vsl, SLT_ReqMethod);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:07 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:07 +0000 (UTC)
Subject: [6.0] 240a3d1aa Register the vmods unique (per-vcl) number
Message-ID: <20181031130807.8DB6A94DE7@lists.varnish-cache.org>


commit 240a3d1aa1d61b511f38c2ecb724805b77ffe773
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Aug 23 08:08:17 2018 +0000

    Register the vmods unique (per-vcl) number
    
    Conflicts:
            include/vrt.h
    
    This appears to break the VRT interface, but there are symbols we are
    willing to break even on a stable branch because they are needed by
    VCC-generated code and off limits to inline C and VMODs.
    
    Refs #2800

diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c
index ffd039ebc..25a119da4 100644
--- a/bin/varnishd/cache/cache_vrt_vmod.c
+++ b/bin/varnishd/cache/cache_vrt_vmod.c
@@ -53,6 +53,7 @@ struct vmod {
 	int			ref;
 
 	char			*nm;
+	unsigned		nbr;
 	char			*path;
 	char			*backup;
 	void			*hdl;
@@ -77,8 +78,8 @@ vmod_abi_mismatch(const struct vmod_data *d)
 }
 
 int
-VRT_Vmod_Init(VRT_CTX, struct vmod **hdl, void *ptr, int len, const char *nm,
-    const char *path, const char *file_id, const char *backup)
+VRT_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len,
+    const char *nm, const char *path, const char *file_id, const char *backup)
 {
 	struct vmod *v;
 	const struct vmod_data *d;
@@ -138,6 +139,7 @@ VRT_Vmod_Init(VRT_CTX, struct vmod **hdl, void *ptr, int len, const char *nm,
 			return (1);
 		}
 
+		v->nbr = nbr;
 		v->funclen = d->func_len;
 		v->funcs = d->func;
 		v->abi = d->abi;
diff --git a/include/vrt.h b/include/vrt.h
index 936aa207f..2fc7c60cc 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -423,7 +423,7 @@ int VRT_Healthy(VRT_CTX, VCL_BACKEND);
 int VRT_VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst);
 
 /* VMOD/Modules related */
-int VRT_Vmod_Init(VRT_CTX, struct vmod **hdl, void *ptr, int len,
+int VRT_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len,
     const char *nm, const char *path, const char *file_id, const char *backup);
 void VRT_Vmod_Fini(struct vmod **hdl);
 
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index a9cef1d71..fd6a7acc6 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -305,6 +305,7 @@ vcc_ParseImport(struct vcc *tl)
 
 	VSB_printf(ifp->ini, "\tif (VRT_Vmod_Init(ctx,\n");
 	VSB_printf(ifp->ini, "\t    &VGC_vmod_%.*s,\n", PF(mod));
+	VSB_printf(ifp->ini, "\t    %u,\n", tl->vmod_count++);
 	VSB_printf(ifp->ini, "\t    &Vmod_%.*s_Func,\n", PF(mod));
 	VSB_printf(ifp->ini, "\t    sizeof(Vmod_%.*s_Func),\n", PF(mod));
 	VSB_printf(ifp->ini, "\t    \"%.*s\",\n", PF(mod));
@@ -336,7 +337,6 @@ vcc_ParseImport(struct vcc *tl)
 	vcc_json_always(tl, msym);
 
 	Fh(tl, 0, "\n/* --- BEGIN VMOD %.*s --- */\n\n", PF(mod));
-	Fh(tl, 0, "#define VMOD_NUMBER_%.*s %u\n", PF(mod), tl->vmod_count++);
 	Fh(tl, 0, "static struct vmod *VGC_vmod_%.*s;\n", PF(mod));
 	Fh(tl, 0, "static struct vmod_priv vmod_priv_%.*s;\n", PF(mod));
 	Fh(tl, 0, "\n%s\n", vmd->proto);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:07 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:07 +0000 (UTC)
Subject: [6.0] a2e09ed58 Retry on EAGAIN.
Message-ID: <20181031130807.B0E5A94DFA@lists.varnish-cache.org>


commit a2e09ed5892e09edde59f5b631c827ebe3bff7ed
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Aug 24 19:08:07 2018 +0000

    Retry on EAGAIN.
    
    Submitted by:   Frederic Lecaille <flecaille at haproxy.com>

diff --git a/bin/varnishtest/vtc_syslog.c b/bin/varnishtest/vtc_syslog.c
index 4221b40eb..f91014c9a 100644
--- a/bin/varnishtest/vtc_syslog.c
+++ b/bin/varnishtest/vtc_syslog.c
@@ -306,7 +306,7 @@ syslog_rx(const struct syslog_srv *s, int lvl)
 
 		ret = recv(s->sock, s->rxbuf, s->rxbuf_sz - 1, 0);
 		if (ret < 0) {
-			if (errno == EINTR)
+			if (errno == EINTR || errno == EAGAIN)
 				continue;
 
 			vtc_fatal(s->vl,

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:07 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:07 +0000 (UTC)
Subject: [6.0] 76af3fcae Constify
Message-ID: <20181031130807.D4E8A94E13@lists.varnish-cache.org>


commit 76af3fcae38ddae1e0baf5d33024a62b4a9dae5e
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Sat Aug 25 11:30:15 2018 +0200

    Constify

diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index 92f21d53d..44cacac36 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -388,7 +388,7 @@ void VCL_Poll(void);
 void VCL_Ref(struct vcl *);
 void VCL_Refresh(struct vcl **);
 void VCL_Rel(struct vcl **);
-void VCL_Onboard(struct req *, struct busyobj *);
+void VCL_Onboard(const struct req *, const struct busyobj *);
 const char *VCL_Return_Name(unsigned);
 const char *VCL_Method_Name(unsigned);
 void VCL_Bo2Ctx(struct vrt_ctx *, struct busyobj *);
diff --git a/bin/varnishd/cache/cache_vcl_vrt.c b/bin/varnishd/cache/cache_vcl_vrt.c
index 36e34b53a..28132b1d0 100644
--- a/bin/varnishd/cache/cache_vcl_vrt.c
+++ b/bin/varnishd/cache/cache_vcl_vrt.c
@@ -74,7 +74,7 @@ VCL_Method_Name(unsigned m)
 /*--------------------------------------------------------------------*/
 
 void
-VCL_Onboard(struct req *req, struct busyobj *bo)
+VCL_Onboard(const struct req *req, const struct busyobj *bo)
 {
 	
 	CHECK_OBJ_ORNULL(req, REQ_MAGIC);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:07 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:07 +0000 (UTC)
Subject: [6.0] 4aa9251ed Whitespace OCD
Message-ID: <20181031130808.01AB794E28@lists.varnish-cache.org>


commit 4aa9251ed8e5400017ff882c38f6f0d8b8b155ad
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 27 10:43:33 2018 +0200

    Whitespace OCD

diff --git a/bin/varnishd/cache/cache_vcl_vrt.c b/bin/varnishd/cache/cache_vcl_vrt.c
index 28132b1d0..b183f5dff 100644
--- a/bin/varnishd/cache/cache_vcl_vrt.c
+++ b/bin/varnishd/cache/cache_vcl_vrt.c
@@ -76,7 +76,7 @@ VCL_Method_Name(unsigned m)
 void
 VCL_Onboard(const struct req *req, const struct busyobj *bo)
 {
-	
+
 	CHECK_OBJ_ORNULL(req, REQ_MAGIC);
 	CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC);
 	assert(req != NULL || bo != NULL);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:08 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:08 +0000 (UTC)
Subject: [6.0] 352201fb5 fix obj.hits in vcl_hit
Message-ID: <20181031130808.27F4694E35@lists.varnish-cache.org>


commit 352201fb5fd0e7f1070e4cefc49767bbf6c59981
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Aug 27 14:06:19 2018 +0200

    fix obj.hits in vcl_hit
    
    Fixes #2746

diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index c902bee4c..748fe0a64 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -807,6 +807,8 @@ VRT_r_obj_hits(VRT_CTX)
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req->objcore, OBJCORE_MAGIC);
+	if (ctx->method == VCL_MET_HIT)
+		return (ctx->req->objcore->hits);
 	return (ctx->req->is_hit ? ctx->req->objcore->hits : 0);
 }
 
diff --git a/bin/varnishtest/tests/v00013.vtc b/bin/varnishtest/tests/v00013.vtc
index 2772a2e18..52b81fc54 100644
--- a/bin/varnishtest/tests/v00013.vtc
+++ b/bin/varnishtest/tests/v00013.vtc
@@ -18,8 +18,14 @@ varnish v1 -vcl+backend {
 			return (pass);
 		}
 	}
+	sub vcl_hit {
+		set req.http.hit-hits = obj.hits;
+	}
 	sub vcl_deliver {
-		set resp.http.foo = obj.hits;
+		set resp.http.deliver-hits = obj.hits;
+		if (req.http.hit-hits) {
+			set resp.http.hit-hits = req.http.hit-hits;
+		}
 	}
 } -start
 
@@ -27,26 +33,31 @@ client c1 {
 	txreq
 	rxresp
 	expect resp.status == 200
-	expect resp.http.foo == 0
+	expect resp.http.deliver-hits == 0
+	expect resp.http.hit-hits == <undef>
 
 	txreq
 	rxresp
 	expect resp.status == 200
-	expect resp.http.foo == 1
+	expect resp.http.deliver-hits == 1
+	expect resp.http.hit-hits == 1
 
 	txreq -url /foo
 	rxresp
 	expect resp.status == 200
-	expect resp.http.foo == 0
+	expect resp.http.deliver-hits == 0
+	expect resp.http.hit-hits == <undef>
 	delay .1
 
 	txreq
 	rxresp
 	expect resp.status == 200
-	expect resp.http.foo == 2
+	expect resp.http.deliver-hits == 2
+	expect resp.http.hit-hits == 2
 
 	txreq -url /pass
 	rxresp
 	expect resp.status == 200
-	expect resp.http.foo == 0
+	expect resp.http.deliver-hits == 0
+	expect resp.http.hit-hits == <undef>
 } -run

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:08 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:08 +0000 (UTC)
Subject: [6.0] 270706a7b improve documentation on backend timeout defaults
Message-ID: <20181031130808.52B1F94E46@lists.varnish-cache.org>


commit 270706a7baf34e06b31483f5d04cad71374152d6
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Aug 27 15:06:27 2018 +0200

    improve documentation on backend timeout defaults

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 913775778..d720dbc95 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -201,6 +201,8 @@ Example::
     }
 
 
+.. _ref_backend_def:
+
 Backend definition
 ------------------
 
@@ -239,12 +241,18 @@ parameters. The following attributes are available:
   ``.connect_timeout``
     Timeout for connections.
 
+    Default: :ref:`ref_param_connect_timeout` parameter.
+
   ``.first_byte_timeout``
     Timeout for first byte.
 
+    Default: :ref:`ref_param_first_byte_timeout` parameter.
+
   ``.between_bytes_timeout``
     Timeout between bytes.
 
+    Default: :ref:`ref_param_between_bytes_timeout` parameter.
+
   ``.probe``
     Attach a probe to the backend. See `Probes`_
 
diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst
index 2bb40b214..1d629ae91 100644
--- a/doc/sphinx/reference/vcl_var.rst
+++ b/doc/sphinx/reference/vcl_var.rst
@@ -585,6 +585,12 @@ bereq.connect_timeout
 
 	Writable from: vcl_pipe, backend
 
+	Default:
+
+	* :ref:`ref_backend_def` ``.connect_timeout`` attribute, which
+	   defaults to
+	* :ref:`ref_param_connect_timeout` parameter.
+
 	The time in seconds to wait for a backend connection to be
 	established.
 
@@ -597,6 +603,12 @@ bereq.first_byte_timeout
 
 	Writable from: backend
 
+	Default:
+
+	* :ref:`ref_backend_def` ``.first_byte_timeout`` attribute,
+	   which defaults to
+	* :ref:`ref_param_first_byte_timeout` parameter.
+
 	The time in seconds to wait getting the first byte back
 	from the backend.  Not available in pipe mode.
 
@@ -609,6 +621,12 @@ bereq.between_bytes_timeout
 
 	Writable from: backend
 
+	Default:
+
+	* :ref:`ref_backend_def` ``.between_bytes_timeout`` backend
+	   attribute, which defaults to
+	* :ref:`ref_param_between_bytes_timeout` parameter.
+
 	The time in seconds to wait between each received byte from the
 	backend.  Not available in pipe mode.
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:08 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:08 +0000 (UTC)
Subject: [6.0] d7c55ba79 Ensure VRT_priv_task doesn't get both req and bo
Message-ID: <20181031130808.771A494E50@lists.varnish-cache.org>


commit d7c55ba79410af65ee4b71b1017528f362ec0fc0
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Aug 27 20:19:06 2018 +0200

    Ensure VRT_priv_task doesn't get both req and bo

diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c
index 8d83f176c..09c18c54f 100644
--- a/bin/varnishd/cache/cache_vrt_priv.c
+++ b/bin/varnishd/cache/cache_vrt_priv.c
@@ -141,6 +141,7 @@ VRT_priv_task(VRT_CTX, const void *vmod_id)
 	struct vrt_privs *vps;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	assert(ctx->req == NULL || ctx->bo == NULL);
 	if (ctx->req) {
 		CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
 		id = (uintptr_t)ctx->req;

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:08 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:08 +0000 (UTC)
Subject: [6.0] 8e737d427 Unify the H1/H2 reembark code.
Message-ID: <20181031130808.A362D94E6B@lists.varnish-cache.org>


commit 8e737d427f6c1d3ab16d4fd7840808cb76cb0c69
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 28 06:41:38 2018 +0000

    Unify the H1/H2 reembark code.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 07447bee6..080a47cbe 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -51,6 +51,34 @@
 #include "vsha256.h"
 #include "vtim.h"
 
+/*--------------------------------------------------------------------
+ * Reschedule a request from the waiting list
+ */
+
+int
+CNT_Reembark(struct worker *wrk, struct req *req)
+{
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+
+	if (!DO_DEBUG(DBG_FAILRESCHED) &&
+	    !SES_Reschedule_Req(req, TASK_QUEUE_REQ))
+		return (0);
+
+	/* Couldn't schedule, ditch */
+	wrk->stats->busy_wakeup--;
+	wrk->stats->busy_killed++;
+	VSLb(req->vsl, SLT_Error, "Fail to reschedule req from waiting list");
+
+	AN(req->ws->r);
+	WS_Release(req->ws, 0);
+	AN(req->hash_objhead);
+	(void)HSH_DerefObjHead(wrk, &req->hash_objhead);
+	AZ(req->hash_objhead);
+	return(-1);
+}
+
 /*--------------------------------------------------------------------
  * Handle "Expect:" and "Connection:" on incoming request
  */
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index 44cacac36..63f0a2369 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -335,6 +335,7 @@ enum req_fsm_nxt {
 };
 
 enum req_fsm_nxt CNT_Request(struct worker *, struct req *);
+int CNT_Reembark(struct worker *, struct req *);
 
 /* cache_session.c */
 void SES_NewPool(struct pool *, unsigned pool_no);
diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c
index b0dc5f472..ed3c9c8f4 100644
--- a/bin/varnishd/http1/cache_http1_fsm.c
+++ b/bin/varnishd/http1/cache_http1_fsm.c
@@ -47,7 +47,6 @@
 
 static const char H1NEWREQ[] = "HTTP1::NewReq";
 static const char H1PROC[] = "HTTP1::Proc";
-static const char H1BUSY[] = "HTTP1::Busy";
 static const char H1CLEANUP[] = "HTTP1::Cleanup";
 
 static void HTTP1_Session(struct worker *, struct req *);
@@ -206,42 +205,19 @@ http1_req_cleanup(struct sess *sp, struct worker *wrk, struct req *req)
  * Clean up a req from waiting list which cannot complete
  */
 
-static void
-http1_cleanup_waiting(struct worker *wrk, struct req *req,
-    enum sess_close reason)
-{
-	struct sess *sp;
-
-	sp = req->sp;
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	AN(req->ws->r);
-	WS_Release(req->ws, 0);
-	AN(req->hash_objhead);
-	(void)HSH_DerefObjHead(wrk, &req->hash_objhead);
-	AZ(req->hash_objhead);
-	SES_Close(sp, reason);
-	AN(http1_req_cleanup(sp, wrk, req));
-}
-
 static void v_matchproto_(vtr_reembark_f)
 http1_reembark(struct worker *wrk, struct req *req)
 {
-	struct sess *sp;
 
-	sp = req->sp;
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-
-	http1_setstate(sp, H1BUSY);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	assert(req->transport == &HTTP1_transport);
 
-	if (!DO_DEBUG(DBG_FAILRESCHED) &&
-	    !SES_Reschedule_Req(req, TASK_QUEUE_REQ))
+	if (!CNT_Reembark(wrk, req))
 		return;
 
-	/* Couldn't schedule, ditch */
-	wrk->stats->busy_wakeup--;
-	wrk->stats->busy_killed++;
-	VSLb(req->vsl, SLT_Error, "Fail to reschedule req from waiting list");
-	http1_cleanup_waiting(wrk, req, SC_OVERLOAD);
+	SES_Close(req->sp, SC_OVERLOAD);
+	AN(http1_req_cleanup(req->sp, wrk, req));
 }
 
 static int v_matchproto_(vtr_minimal_response_f)
@@ -489,17 +465,6 @@ HTTP1_Session(struct worker *wrk, struct req *req)
 			}
 			req->req_step = R_STP_TRANSPORT;
 			http1_setstate(sp, H1PROC);
-		} else if (st == H1BUSY) {
-			CHECK_OBJ_NOTNULL(req->transport, TRANSPORT_MAGIC);
-			/*
-			 * Return from waitinglist.
-			 * Check to see if the remote has left.
-			 */
-			if (VTCP_check_hup(sp->fd)) {
-				http1_cleanup_waiting(wrk, req, SC_REM_CLOSE);
-				return;
-			}
-			http1_setstate(sp, H1PROC);
 		} else if (st == H1PROC) {
 			req->task.func = http1_req;
 			req->task.priv = req;
diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h
index 792fd985a..1c28f99ee 100644
--- a/bin/varnishd/http2/cache_http2.h
+++ b/bin/varnishd/http2/cache_http2.h
@@ -233,7 +233,6 @@ void h2_kill_req(struct worker *, const struct h2_sess *,
 int h2_rxframe(struct worker *, struct h2_sess *);
 h2_error h2_set_setting(struct h2_sess *, const uint8_t *);
 void h2_req_body(struct req*);
-void h2_cleanup_waiting(struct worker *wrk, struct h2_req *r2);
 task_func_t h2_do_req;
 #ifdef TRANSPORT_MAGIC
 vtr_req_fail_f h2_req_fail;
diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 7713636c4..644474309 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -1149,21 +1149,3 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2)
 	}
 	return (h2e ? 0 : 1);
 }
-
-void
-h2_cleanup_waiting(struct worker *wrk, struct h2_req *r2)
-{
-	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(r2->req, REQ_MAGIC);
-	CHECK_OBJ_NOTNULL(r2->h2sess, H2_SESS_MAGIC);
-
-	AN(r2->req->ws->r);
-	WS_Release(r2->req->ws, 0);
-	AN(r2->req->hash_objhead);
-	(void)HSH_DerefObjHead(wrk, &r2->req->hash_objhead);
-	AZ(r2->req->hash_objhead);
-	assert(r2->state == H2_S_CLOS_REM);
-	AN(r2->scheduled);
-	r2->scheduled = 0;
-	r2->h2sess->do_sweep = 1;
-}
diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c
index 40a111754..8788dc45a 100644
--- a/bin/varnishd/http2/cache_http2_session.c
+++ b/bin/varnishd/http2/cache_http2_session.c
@@ -437,26 +437,22 @@ h2_new_session(struct worker *wrk, void *arg)
 static void v_matchproto_(vtr_reembark_f)
 h2_reembark(struct worker *wrk, struct req *req)
 {
-	struct sess *sp;
 	struct h2_req *r2;
 
-	sp = req->sp;
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	assert(req->transport == &H2_transport);
 
-	if (!DO_DEBUG(DBG_FAILRESCHED) &&
-	    !SES_Reschedule_Req(req, TASK_QUEUE_STR))
+	if (!CNT_Reembark(wrk, req))
 		return;
 
-	/* Couldn't schedule, ditch */
-	wrk->stats->busy_wakeup--;
-	wrk->stats->busy_killed++;
 	CAST_OBJ_NOTNULL(r2, req->transport_priv, H2_REQ_MAGIC);
-	VSLb(req->vsl, SLT_Error, "Fail to reschedule req from waiting list");
-	h2_cleanup_waiting(wrk, r2);
+	assert(r2->state == H2_S_CLOS_REM);
+	AN(r2->scheduled);
+	r2->scheduled = 0;
+	r2->h2sess->do_sweep = 1;
 }
 
-
 struct transport H2_transport = {
 	.name =			"H2",
 	.magic =		TRANSPORT_MAGIC,

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:08 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:08 +0000 (UTC)
Subject: [6.0] d5da1c08a Whitespace OCD
Message-ID: <20181031130808.C03D194E7B@lists.varnish-cache.org>


commit d5da1c08ad91b04c40400cd49ef37433d37afe23
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Aug 28 12:04:07 2018 +0200

    Whitespace OCD

diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c
index 09c18c54f..b29f12047 100644
--- a/bin/varnishd/cache/cache_vrt_priv.c
+++ b/bin/varnishd/cache/cache_vrt_priv.c
@@ -139,6 +139,7 @@ VRT_priv_task(VRT_CTX, const void *vmod_id)
 {
 	uintptr_t id;
 	struct vrt_privs *vps;
+	struct vmod_priv *vp;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	assert(ctx->req == NULL || ctx->bo == NULL);
@@ -155,8 +156,9 @@ VRT_priv_task(VRT_CTX, const void *vmod_id)
 		id = (uintptr_t)cli_task_privs;
 		CAST_OBJ_NOTNULL(vps, cli_task_privs, VRT_PRIVS_MAGIC);
 	}
-	return (vrt_priv_dynamic(ctx->vcl, ctx->ws,
-				 vps, id, (uintptr_t)vmod_id));
+
+	vp = vrt_priv_dynamic(ctx->vcl, ctx->ws, vps, id, (uintptr_t)vmod_id);
+	return (vp);
 }
 
 struct vmod_priv *

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:08 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:08 +0000 (UTC)
Subject: [6.0] 2174c4038 use references which work both for html and man pages
Message-ID: <20181031130808.E3D3394E86@lists.varnish-cache.org>


commit 2174c4038ed0cbe2f652e37804c590c995c66d74
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Aug 28 17:39:06 2018 +0200

    use references which work both for html and man pages

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index d720dbc95..7aa434da6 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -200,8 +200,7 @@ Example::
         */
     }
 
-
-.. _ref_backend_def:
+.. _backend_definition:
 
 Backend definition
 ------------------
@@ -241,17 +240,17 @@ parameters. The following attributes are available:
   ``.connect_timeout``
     Timeout for connections.
 
-    Default: :ref:`ref_param_connect_timeout` parameter.
+    Default: ``connect_timeout`` parameter, see :ref:`varnishd(1)`
 
   ``.first_byte_timeout``
     Timeout for first byte.
 
-    Default: :ref:`ref_param_first_byte_timeout` parameter.
+    Default: ``first_byte_timeout`` parameter, see :ref:`varnishd(1)`
 
   ``.between_bytes_timeout``
     Timeout between bytes.
 
-    Default: :ref:`ref_param_between_bytes_timeout` parameter.
+    Default: ``between_bytes_timeout`` parameter, see :ref:`varnishd(1)`
 
   ``.probe``
     Attach a probe to the backend. See `Probes`_
diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst
index 1d629ae91..d913b0ceb 100644
--- a/doc/sphinx/reference/vcl_var.rst
+++ b/doc/sphinx/reference/vcl_var.rst
@@ -587,9 +587,9 @@ bereq.connect_timeout
 
 	Default:
 
-	* :ref:`ref_backend_def` ``.connect_timeout`` attribute, which
-	   defaults to
-	* :ref:`ref_param_connect_timeout` parameter.
+	* ``.connect_timeout`` attribute from the
+	  :ref:`backend_definition`, which defaults to the
+	* ``connect_timeout`` parameter, see :ref:`varnishd(1)`
 
 	The time in seconds to wait for a backend connection to be
 	established.
@@ -605,9 +605,9 @@ bereq.first_byte_timeout
 
 	Default:
 
-	* :ref:`ref_backend_def` ``.first_byte_timeout`` attribute,
-	   which defaults to
-	* :ref:`ref_param_first_byte_timeout` parameter.
+	* ``.first_byte_timeout`` attribute from the
+	  :ref:`backend_definition`, which defaults to the
+	* ``first_byte_timeout`` parameter, see :ref:`varnishd(1)`
 
 	The time in seconds to wait getting the first byte back
 	from the backend.  Not available in pipe mode.
@@ -623,9 +623,9 @@ bereq.between_bytes_timeout
 
 	Default:
 
-	* :ref:`ref_backend_def` ``.between_bytes_timeout`` backend
-	   attribute, which defaults to
-	* :ref:`ref_param_between_bytes_timeout` parameter.
+	* ``.between_bytes_timeout`` attribute from the
+	  :ref:`backend_definition`, which defaults to the
+	* ``between_bytes_timeout`` parameter, see :ref:`varnishd(1)`
 
 	The time in seconds to wait between each received byte from the
 	backend.  Not available in pipe mode.

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:09 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:09 +0000 (UTC)
Subject: [6.0] 412460e64 in vmod destructors,
 tolerate failed object initializations
Message-ID: <20181031130809.1FEED94EA8@lists.varnish-cache.org>


commit 412460e64d528a9a1dbc0d984a0116b5fe9746d4
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Aug 28 22:15:14 2018 +0200

    in vmod destructors, tolerate failed object initializations
    
    works around #2297
    
    Conflicts:
            lib/libvmod_directors/fall_back.c
            lib/libvmod_directors/hash.c
            lib/libvmod_directors/random.c
            lib/libvmod_directors/round_robin.c

diff --git a/lib/libvmod_directors/fall_back.c b/lib/libvmod_directors/fall_back.c
index 8ae3ddc30..76561f596 100644
--- a/lib/libvmod_directors/fall_back.c
+++ b/lib/libvmod_directors/fall_back.c
@@ -108,9 +108,11 @@ vmod_fallback__fini(struct vmod_directors_fallback **fbp)
 {
 	struct vmod_directors_fallback *fb;
 
-	fb = *fbp;
-	*fbp = NULL;
-	CHECK_OBJ_NOTNULL(fb, VMOD_DIRECTORS_FALLBACK_MAGIC);
+	// XXX 2297
+	if (*fbp == NULL)
+		return;
+
+	TAKE_OBJ_NOTNULL(fb, fbp, VMOD_DIRECTORS_FALLBACK_MAGIC);
 	vdir_delete(&fb->vd);
 	FREE_OBJ(fb);
 }
diff --git a/lib/libvmod_directors/hash.c b/lib/libvmod_directors/hash.c
index a05308195..be7d40b47 100644
--- a/lib/libvmod_directors/hash.c
+++ b/lib/libvmod_directors/hash.c
@@ -67,9 +67,11 @@ vmod_hash__fini(struct vmod_directors_hash **rrp)
 {
 	struct vmod_directors_hash *rr;
 
-	rr = *rrp;
-	*rrp = NULL;
-	CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_HASH_MAGIC);
+	// XXX 2297
+	if (*rrp == NULL)
+		return;
+
+	TAKE_OBJ_NOTNULL(rr, rrp, VMOD_DIRECTORS_HASH_MAGIC);
 	vdir_delete(&rr->vd);
 	FREE_OBJ(rr);
 }
diff --git a/lib/libvmod_directors/random.c b/lib/libvmod_directors/random.c
index 4ce6328f3..856e6f3c0 100644
--- a/lib/libvmod_directors/random.c
+++ b/lib/libvmod_directors/random.c
@@ -95,9 +95,11 @@ vmod_random__fini(struct vmod_directors_random **rrp)
 {
 	struct vmod_directors_random *rr;
 
-	rr = *rrp;
-	*rrp = NULL;
-	CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_RANDOM_MAGIC);
+	// XXX 2297
+	if (*rrp == NULL)
+		return;
+
+	TAKE_OBJ_NOTNULL(rr, rrp, VMOD_DIRECTORS_RANDOM_MAGIC);
 	vdir_delete(&rr->vd);
 	FREE_OBJ(rr);
 }
diff --git a/lib/libvmod_directors/round_robin.c b/lib/libvmod_directors/round_robin.c
index 7e888f365..767fe7258 100644
--- a/lib/libvmod_directors/round_robin.c
+++ b/lib/libvmod_directors/round_robin.c
@@ -104,9 +104,11 @@ vmod_round_robin__fini(struct vmod_directors_round_robin **rrp)
 {
 	struct vmod_directors_round_robin *rr;
 
-	rr = *rrp;
-	*rrp = NULL;
-	CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_ROUND_ROBIN_MAGIC);
+	// XXX 2297
+	if (*rrp == NULL)
+		return;
+
+	TAKE_OBJ_NOTNULL(rr, rrp, VMOD_DIRECTORS_ROUND_ROBIN_MAGIC);
 	vdir_delete(&rr->vd);
 	FREE_OBJ(rr);
 }
diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c
index 98f9c0321..b85aa43b8 100644
--- a/lib/libvmod_directors/vmod_shard.c
+++ b/lib/libvmod_directors/vmod_shard.c
@@ -265,6 +265,10 @@ vmod_shard__fini(struct vmod_directors_shard **vshardp)
 {
 	struct vmod_directors_shard *vshard;
 
+	// XXX 2297
+	if (*vshardp == NULL)
+		return;
+
 	TAKE_OBJ_NOTNULL(vshard, vshardp, VMOD_SHARD_SHARD_MAGIC);
 	sharddir_delete(&vshard->shardd);
 	free(vshard->dir->vcl_name);
@@ -795,8 +799,10 @@ vmod_shard_param__fini(struct vmod_directors_shard_param **pp)
 {
 	struct vmod_directors_shard_param *p;
 
+	// XXX 2297
 	if (*pp == NULL)
 		return;
+
 	TAKE_OBJ_NOTNULL(p, pp, VMOD_SHARD_SHARD_PARAM_MAGIC);
 	FREE_OBJ(p);
 }

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:09 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:09 +0000 (UTC)
Subject: [6.0] 1deced7c6 Always reschedule requests from the waiting list.
Message-ID: <20181031130809.8CCAC94EC4@lists.varnish-cache.org>


commit 1deced7c635807770d0429f7c9b83d72f1ed285a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 28 20:27:37 2018 +0000

    Always reschedule requests from the waiting list.
    
    Previously we used various heuristics (test TCP connection) to avoid
    the reschedule if the req was abandonned while on the waiting list.
    
    We also subjected the rescheduling to pool-queue limits like any
    new request.
    
    This was all safe because we knew we could clean up the request
    state cheaply, even if it was somewhat cumbersome.
    
    Now vmods can have per-task PRIV's and we have no idea what it will
    cost us (stack, time, etc) to clean them up, so we cannot burden
    J.Random Request who happens to rush the waiting list with the burden.
    
    Fix this by always rescheduling, not subject to pool-queue limits,
    and eliminate all the special-casing for exceeded limits, including
    the debug feature to force a rescheduling failure and two tests
    exercising it.
    
    As a side effect of this, requests on the waiting list gets a
    "business class upgrade" over newly arriving requests when there
    are no worker threads available.  Given that these requests arrived
    earlier, and we already performed work on them, this seems only fair.
    
    Forced to pay proper attention by:      slink

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index adf9e34a9..c69075cb9 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -217,10 +217,17 @@ struct pool_task {
  * tasks are taken off the queues in this order
  *
  * prios up to TASK_QUEUE_RESERVE are run from the reserve
+ *
+ * TASK_QUEUE_{REQ|STR} are new req's (H1/H2), and subject to queue limit.
+ *
+ * TASK_QUEUE_RUSH is req's returning from waiting list, they are
+ * not subject to TASK_QUEUE_CLIENT because we cannot safely clean
+ * them up if scheduling them fails.
  */
 enum task_prio {
 	TASK_QUEUE_BO,
 #define TASK_QUEUE_RESERVE	TASK_QUEUE_BO
+	TASK_QUEUE_RUSH,
 	TASK_QUEUE_REQ,
 	TASK_QUEUE_STR,
 	TASK_QUEUE_VCA,
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 3fbbc702d..c3b652bbe 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -76,6 +76,7 @@ static struct objhead *private_oh;
 static void hsh_rush1(const struct worker *, struct objhead *,
     struct rush *, int);
 static void hsh_rush2(struct worker *, struct rush *);
+static int HSH_DerefObjHead(struct worker *wrk, struct objhead **poh);
 
 /*---------------------------------------------------------------------*/
 
@@ -591,8 +592,18 @@ hsh_rush2(struct worker *wrk, struct rush *r)
 		CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 		VTAILQ_REMOVE(&r->reqs, req, w_list);
 		DSL(DBG_WAITINGLIST, req->vsl->wid, "off waiting list");
-		AN(req->transport->reembark);
-		req->transport->reembark(wrk, req);
+		if (req->transport->reembark != NULL) {
+			// For ESI includes
+			req->transport->reembark(wrk, req);
+		} else {
+			/*
+			 * We ignore the queue limits which apply to new
+			 * requests because if we fail to reschedule there
+			 * may be vmod_privs to cleanup and we need a proper
+			 * workerthread for that.
+			 */
+			AZ(Pool_Task(req->sp->pool, &req->task, TASK_QUEUE_RUSH));
+		}
 	}
 }
 
@@ -939,7 +950,7 @@ HSH_DerefObjCore(struct worker *wrk, struct objcore **ocp, int rushmax)
 	return (0);
 }
 
-int
+static int
 HSH_DerefObjHead(struct worker *wrk, struct objhead **poh)
 {
 	struct objhead *oh;
diff --git a/bin/varnishd/cache/cache_objhead.h b/bin/varnishd/cache/cache_objhead.h
index 6bcfbd9b9..6bcb7b301 100644
--- a/bin/varnishd/cache/cache_objhead.h
+++ b/bin/varnishd/cache/cache_objhead.h
@@ -63,7 +63,6 @@ int HSH_Snipe(const struct worker *, struct objcore *);
 struct boc *HSH_RefBoc(const struct objcore *);
 void HSH_DerefBoc(struct worker *wrk, struct objcore *);
 void HSH_DeleteObjHead(const struct worker *, struct objhead *);
-int HSH_DerefObjHead(struct worker *, struct objhead **);
 
 int HSH_DerefObjCore(struct worker *, struct objcore **, int rushmax);
 #define HSH_RUSH_POLICY -1
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 080a47cbe..07447bee6 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -51,34 +51,6 @@
 #include "vsha256.h"
 #include "vtim.h"
 
-/*--------------------------------------------------------------------
- * Reschedule a request from the waiting list
- */
-
-int
-CNT_Reembark(struct worker *wrk, struct req *req)
-{
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-
-	if (!DO_DEBUG(DBG_FAILRESCHED) &&
-	    !SES_Reschedule_Req(req, TASK_QUEUE_REQ))
-		return (0);
-
-	/* Couldn't schedule, ditch */
-	wrk->stats->busy_wakeup--;
-	wrk->stats->busy_killed++;
-	VSLb(req->vsl, SLT_Error, "Fail to reschedule req from waiting list");
-
-	AN(req->ws->r);
-	WS_Release(req->ws, 0);
-	AN(req->hash_objhead);
-	(void)HSH_DerefObjHead(wrk, &req->hash_objhead);
-	AZ(req->hash_objhead);
-	return(-1);
-}
-
 /*--------------------------------------------------------------------
  * Handle "Expect:" and "Connection:" on incoming request
  */
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 0500bcef2..20ada52e3 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -376,30 +376,6 @@ SES_New(struct pool *pp)
 	return (sp);
 }
 
-/*--------------------------------------------------------------------
- * Reschedule a request on a work-thread from its sessions pool
- *
- * This is used to reschedule requests waiting on busy objects
- */
-
-int
-SES_Reschedule_Req(struct req *req, enum task_prio prio)
-{
-	struct sess *sp;
-	struct pool *pp;
-
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	sp = req->sp;
-	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-	pp = sp->pool;
-	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
-	AN(TASK_QUEUE_CLIENT(prio));
-
-	AN(req->task.func);
-
-	return (Pool_Task(pp, &req->task, prio));
-}
-
 /*--------------------------------------------------------------------
  * Handle a session (from waiter)
  */
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index 63f0a2369..4e78cd750 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -335,7 +335,6 @@ enum req_fsm_nxt {
 };
 
 enum req_fsm_nxt CNT_Request(struct worker *, struct req *);
-int CNT_Reembark(struct worker *, struct req *);
 
 /* cache_session.c */
 void SES_NewPool(struct pool *, unsigned pool_no);
@@ -343,7 +342,6 @@ void SES_DestroyPool(struct pool *);
 void SES_Wait(struct sess *, const struct transport *);
 void SES_Ref(struct sess *sp);
 void SES_Rel(struct sess *sp);
-int SES_Reschedule_Req(struct req *, enum task_prio);
 
 const char * HTC_Status(enum htc_status_e);
 void HTC_RxInit(struct http_conn *htc, struct ws *ws);
diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c
index ed3c9c8f4..30bd3b894 100644
--- a/bin/varnishd/http1/cache_http1_fsm.c
+++ b/bin/varnishd/http1/cache_http1_fsm.c
@@ -201,25 +201,6 @@ http1_req_cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 	return (0);
 }
 
-/*----------------------------------------------------------------------
- * Clean up a req from waiting list which cannot complete
- */
-
-static void v_matchproto_(vtr_reembark_f)
-http1_reembark(struct worker *wrk, struct req *req)
-{
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	assert(req->transport == &HTTP1_transport);
-
-	if (!CNT_Reembark(wrk, req))
-		return;
-
-	SES_Close(req->sp, SC_OVERLOAD);
-	AN(http1_req_cleanup(req->sp, wrk, req));
-}
-
 static int v_matchproto_(vtr_minimal_response_f)
 http1_minimal_response(struct req *req, uint16_t status)
 {
@@ -263,7 +244,6 @@ struct transport HTTP1_transport = {
 	.deliver =		V1D_Deliver,
 	.minimal_response =	http1_minimal_response,
 	.new_session =		http1_new_session,
-	.reembark =		http1_reembark,
 	.req_body =		http1_req_body,
 	.req_fail =		http1_req_fail,
 	.req_panic =		http1_req_panic,
diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c
index 8788dc45a..5985fe472 100644
--- a/bin/varnishd/http2/cache_http2_session.c
+++ b/bin/varnishd/http2/cache_http2_session.c
@@ -434,32 +434,12 @@ h2_new_session(struct worker *wrk, void *arg)
 	h2_del_sess(wrk, h2, SC_RX_JUNK);
 }
 
-static void v_matchproto_(vtr_reembark_f)
-h2_reembark(struct worker *wrk, struct req *req)
-{
-	struct h2_req *r2;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	assert(req->transport == &H2_transport);
-
-	if (!CNT_Reembark(wrk, req))
-		return;
-
-	CAST_OBJ_NOTNULL(r2, req->transport_priv, H2_REQ_MAGIC);
-	assert(r2->state == H2_S_CLOS_REM);
-	AN(r2->scheduled);
-	r2->scheduled = 0;
-	r2->h2sess->do_sweep = 1;
-}
-
 struct transport H2_transport = {
 	.name =			"H2",
 	.magic =		TRANSPORT_MAGIC,
 	.deliver =		h2_deliver,
 	.minimal_response =	h2_minimal_response,
 	.new_session =		h2_new_session,
-	.reembark =		h2_reembark,
 	.req_body =		h2_req_body,
 	.req_fail =		h2_req_fail,
 	.sess_panic =		h2_sess_panic,
diff --git a/bin/varnishtest/tests/c00013.vtc b/bin/varnishtest/tests/c00013.vtc
index 1fcccac4a..ef4cc76f8 100644
--- a/bin/varnishtest/tests/c00013.vtc
+++ b/bin/varnishtest/tests/c00013.vtc
@@ -50,60 +50,3 @@ varnish v1 -vsl_catchup
 varnish v1 -expect busy_sleep >= 1
 varnish v1 -expect busy_wakeup >= 1
 varnish v1 -stop
-
-##################################################
-# Now try again where getting a thread fails
-
-barrier b3 cond 2
-barrier b4 cond 2
-
-server s3 {
-	rxreq
-	expect req.url == "/foo"
-	send "HTTP/1.0 200 OK\r\nConnection: close\r\n\r\n"
-	delay .2
-	barrier b3 sync
-	delay .2
-	send "line1\n"
-	delay .2
-	barrier b4 sync
-	send "line2\n"
-} -start
-
-varnish v3 -vcl+backend {
-	sub vcl_backend_fetch {
-		set bereq.backend = s3;
-	}
-	sub vcl_backend_response {
-		set beresp.do_stream = false;
-	}
-} -start
-
-varnish v3 -cliok "param.set debug +failresched"
-
-varnish v3 -cliok "param.set debug +syncvsl"
-
-client c3 -connect ${v3_sock} {
-	txreq -url "/foo" -hdr "client: c3"
-	rxresp
-	expect resp.status == 200
-	expect resp.bodylen == 12
-	expect resp.http.x-varnish == "1001"
-} -start
-
-barrier b3 sync
-
-client c4 -connect ${v3_sock} {
-	txreq -url "/foo" -hdr "client: c4"
-	delay .2
-	barrier b4 sync
-	expect_close
-} -run
-
-client c3 -wait
-
-varnish v1 -vsl_catchup
-varnish v3 -expect busy_sleep >= 1
-varnish v3 -expect busy_wakeup == 0
-varnish v3 -expect busy_killed == 1
-varnish v3 -expect sc_overload == 1
diff --git a/bin/varnishtest/tests/r02563.vtc b/bin/varnishtest/tests/r02563.vtc
deleted file mode 100644
index bfe3514b9..000000000
--- a/bin/varnishtest/tests/r02563.vtc
+++ /dev/null
@@ -1,64 +0,0 @@
-varnishtest "#2563: Panic after reembark failure"
-
-barrier b1 cond 2
-barrier b2 cond 2
-
-server s1 {
-	rxreq
-	expect req.url == "/foo"
-	expect req.http.client == "c1"
-	send "HTTP/1.0 200 OK\r\nConnection: close\r\n\r\n"
-	delay .2
-	barrier b1 sync
-	delay .2
-	send "line1\n"
-	delay .2
-	barrier b2 sync
-	send "line2\n"
-} -start
-
-varnish v1 -vcl+backend {
-	sub vcl_backend_response {
-		set beresp.do_stream = false;
-	}
-} -start
-
-varnish v1 -cliok "param.set feature +http2"
-varnish v1 -cliok "param.set debug +failresched"
-varnish v1 -cliok "param.set debug +waitinglist"
-varnish v1 -cliok "param.set debug +syncvsl"
-
-client c1 {
-	txreq -url "/foo" -hdr "client: c1"
-	rxresp
-	expect resp.status == 200
-	expect resp.bodylen == 12
-	expect resp.http.x-varnish == "1001"
-} -start
-
-barrier b1 sync
-
-client c2 {
-	stream 1 {
-		txreq -url "/foo"
-		delay .2
-		barrier b2 sync
-		rxrst
-		expect rst.err == REFUSED_STREAM
-	} -start
-
-	stream 3 {
-		delay 1
-		txreq -url "/foo"
-		rxresp
-	} -run
-
-	stream 1 -wait
-} -run
-
-client c1 -wait
-
-varnish v1 -vsl_catchup
-varnish v1 -expect busy_sleep >= 1
-varnish v1 -expect busy_wakeup == 0
-varnish v1 -expect busy_killed == 1
diff --git a/include/tbl/debug_bits.h b/include/tbl/debug_bits.h
index b451f3332..4ac3ef60f 100644
--- a/include/tbl/debug_bits.h
+++ b/include/tbl/debug_bits.h
@@ -50,7 +50,6 @@ DEBUG_BIT(H2_NOCHECK,		h2_nocheck,	"Disable various H2 checks")
 DEBUG_BIT(VMOD_SO_KEEP,		vmod_so_keep,	"Keep copied VMOD libraries")
 DEBUG_BIT(PROCESSORS,		processors,	"Fetch/Deliver processors")
 DEBUG_BIT(PROTOCOL,		protocol,	"Protocol debugging")
-DEBUG_BIT(FAILRESCHED,		failresched,	"Fail from waiting list")
 #undef DEBUG_BIT
 
 /*lint -restore */

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:09 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:09 +0000 (UTC)
Subject: [6.0] 09c1fac72 Whitespace
Message-ID: <20181031130809.A854594ECF@lists.varnish-cache.org>


commit 09c1fac720e91022613f3dc05e96c2241d5aa7f5
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Aug 28 21:52:03 2018 +0100

    Whitespace

diff --git a/bin/varnishtest/huffman_gen.py b/bin/varnishtest/huffman_gen.py
index 00ff16fe6..8da37b2ee 100755
--- a/bin/varnishtest/huffman_gen.py
+++ b/bin/varnishtest/huffman_gen.py
@@ -12,7 +12,7 @@ if len(sys.argv) != 2:
 
 class sym:
 	def __init__(self, bigval, bigvall, chr = 0, esc = None):
-		self.vall = bigvall % 8 if bigvall % 8 else 8 
+		self.vall = bigvall % 8 if bigvall % 8 else 8
 		self.val = bigval & ((1 << self.vall) - 1)
 		self.pfx = (bigval >> self.vall)# & 0xff
 		self.chr = chr

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:09 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:09 +0000 (UTC)
Subject: [6.0] 4ecbafddb Formatting
Message-ID: <20181031130809.C438D94EE8@lists.varnish-cache.org>


commit 4ecbafddb59b32036e4f9941ec983dca6c29faa4
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Aug 28 21:52:28 2018 +0100

    Formatting

diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst
index d913b0ceb..4efbdff39 100644
--- a/doc/sphinx/reference/vcl_var.rst
+++ b/doc/sphinx/reference/vcl_var.rst
@@ -585,11 +585,9 @@ bereq.connect_timeout
 
 	Writable from: vcl_pipe, backend
 
-	Default:
-
-	* ``.connect_timeout`` attribute from the
-	  :ref:`backend_definition`, which defaults to the
-	* ``connect_timeout`` parameter, see :ref:`varnishd(1)`
+	Default: ``.connect_timeout`` attribute from the
+	:ref:`backend_definition`, which defaults to the
+	``connect_timeout`` parameter, see :ref:`varnishd(1)`
 
 	The time in seconds to wait for a backend connection to be
 	established.
@@ -603,11 +601,9 @@ bereq.first_byte_timeout
 
 	Writable from: backend
 
-	Default:
-
-	* ``.first_byte_timeout`` attribute from the
-	  :ref:`backend_definition`, which defaults to the
-	* ``first_byte_timeout`` parameter, see :ref:`varnishd(1)`
+	Default: ``.first_byte_timeout`` attribute from the
+	:ref:`backend_definition`, which defaults to the
+	``first_byte_timeout`` parameter, see :ref:`varnishd(1)`
 
 	The time in seconds to wait getting the first byte back
 	from the backend.  Not available in pipe mode.
@@ -621,11 +617,9 @@ bereq.between_bytes_timeout
 
 	Writable from: backend
 
-	Default:
-
-	* ``.between_bytes_timeout`` attribute from the
-	  :ref:`backend_definition`, which defaults to the
-	* ``between_bytes_timeout`` parameter, see :ref:`varnishd(1)`
+	Default: ``.between_bytes_timeout`` attribute from the
+	:ref:`backend_definition`, which defaults to the
+	``between_bytes_timeout`` parameter, see :ref:`varnishd(1)`
 
 	The time in seconds to wait between each received byte from the
 	backend.  Not available in pipe mode.

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:10 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:10 +0000 (UTC)
Subject: [6.0] 5ab4c3141 Collect and make explicit where we enter and leave
 "tasks" for the purpose of VRT_priv_task()
Message-ID: <20181031130810.08AEB94EFC@lists.varnish-cache.org>


commit 5ab4c31410fda9122fa98b06d8d4bd7821d0f694
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 28 20:52:13 2018 +0000

    Collect and make explicit where we enter and leave "tasks" for
    the purpose of VRT_priv_task()

diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 5527b588c..e27431042 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -133,14 +133,11 @@ VBO_GetBusyObj(struct worker *wrk, const struct req *req)
 	bo->director_req = req->director_hint;
 	bo->vcl = req->vcl;
 	VCL_Ref(bo->vcl);
-	VCL_Onboard(NULL, bo);
 
 	bo->t_first = bo->t_prev = NAN;
 
 	memcpy(bo->digest, req->digest, sizeof bo->digest);
 
-	VRTPRIV_init(bo->privs);
-
 	return (bo);
 }
 
@@ -156,8 +153,7 @@ VBO_ReleaseBusyObj(struct worker *wrk, struct busyobj **pbo)
 	AZ(bo->htc);
 	AZ(bo->stale_oc);
 
-	VRTPRIV_dynamic_kill(bo->privs, (uintptr_t)bo);
-	assert(VTAILQ_EMPTY(&bo->privs->privs));
+	AZ(bo->privs->magic);
 
 	VSLb(bo->vsl, SLT_BereqAcct, "%ju %ju %ju %ju %ju %ju",
 	    (uintmax_t)bo->acct.bereq_hdrbytes,
diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 516268c5b..1e238d854 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -164,7 +164,6 @@ ved_include(struct req *preq, const char *src, const char *host,
 	AZ(req->vcl);
 	req->vcl = preq->vcl;
 	preq->vcl = NULL;
-	VCL_Onboard(req, NULL);
 
 	req->req_step = R_STP_RECV;
 	req->t_req = preq->t_req;
@@ -198,8 +197,6 @@ ved_include(struct req *preq, const char *src, const char *host,
 		AZ(req->wrk);
 	}
 
-	VRTPRIV_dynamic_kill(req->privs, (uintptr_t)req);
-
 	AZ(preq->vcl);
 	preq->vcl = req->vcl;
 	req->vcl = NULL;
diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 552e3e35e..1c221aabf 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -900,6 +900,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 	}
 #endif
 
+	VCL_TaskEnter(bo->vcl, bo->privs);
 	while (stp != F_STP_DONE) {
 		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 		assert(bo->fetch_objcore->boc->refcount >= 1);
@@ -920,6 +921,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv)
 
 	assert(bo->director_state == DIR_S_NULL);
 
+	VCL_TaskLeave(bo->vcl, bo->privs);
 	http_Teardown(bo->bereq);
 	http_Teardown(bo->beresp);
 
diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c
index d741441ca..036056740 100644
--- a/bin/varnishd/cache/cache_req.c
+++ b/bin/varnishd/cache/cache_req.c
@@ -149,8 +149,6 @@ Req_New(const struct worker *wrk, struct sess *sp)
 	req->t_prev = NAN;
 	req->t_req = NAN;
 
-	VRTPRIV_init(req->privs);
-
 	return (req);
 }
 
@@ -181,14 +179,6 @@ Req_Release(struct req *req)
 	MPL_Free(pp->mpl_req, req);
 }
 
-static void
-req_finalize(struct req *req)
-{
-	VRTPRIV_dynamic_kill(req->privs, (uintptr_t)req);
-	VRTPRIV_dynamic_kill(req->privs, (uintptr_t)&req->top);
-	assert(VTAILQ_EMPTY(&req->privs->privs));
-}
-
 /*----------------------------------------------------------------------
  * TODO:
  * - check for code duplication with cnt_recv_prep
@@ -198,7 +188,8 @@ req_finalize(struct req *req)
 void
 Req_Rollback(struct req *req)
 {
-	req_finalize(req);
+	VCL_TaskLeave(req->vcl, req->privs);
+	VCL_TaskEnter(req->vcl, req->privs);
 	HTTP_Copy(req->http, req->http0);
 	WS_Reset(req->ws, req->ws_req);
 }
@@ -220,6 +211,7 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 	req->restarts = 0;
 
 	AZ(req->esi_level);
+	AZ(req->privs->magic);
 	assert(req->top == req);
 
 	if (req->vcl != NULL) {
@@ -229,8 +221,6 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 		req->vcl = NULL;
 	}
 
-	req_finalize(req);
-
 	/* Charge and log byte counters */
 	if (req->vsl->wid) {
 		Req_AcctLogCharge(wrk->stats, req);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 07447bee6..717e2993a 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -1023,6 +1023,7 @@ CNT_Request(struct worker *wrk, struct req *req)
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	AN(req->vcl);
 
 	/*
 	 * Possible entrance states
@@ -1037,6 +1038,8 @@ CNT_Request(struct worker *wrk, struct req *req)
 	/* wrk can have changed for restarts */
 	req->vfc->wrk = req->wrk = wrk;
 	wrk->vsl = req->vsl;
+	if (req->req_step != R_STP_LOOKUP)
+		VCL_TaskEnter(req->vcl, req->privs);
 	for (nxt = REQ_FSM_MORE; nxt == REQ_FSM_MORE; ) {
 		/*
 		 * This is a good place to be paranoid about the various
@@ -1061,6 +1064,7 @@ CNT_Request(struct worker *wrk, struct req *req)
 	}
 	wrk->vsl = NULL;
 	if (nxt == REQ_FSM_DONE) {
+		VCL_TaskLeave(req->vcl, req->privs);
 		AN(req->vsl->wid);
 		VRB_Free(req);
 		req->wrk = NULL;
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index 4e78cd750..2b7b254ad 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -387,7 +387,8 @@ void VCL_Poll(void);
 void VCL_Ref(struct vcl *);
 void VCL_Refresh(struct vcl **);
 void VCL_Rel(struct vcl **);
-void VCL_Onboard(const struct req *, const struct busyobj *);
+void VCL_TaskEnter(struct vcl *, struct vrt_privs *);
+void VCL_TaskLeave(struct vcl *, struct vrt_privs *);
 const char *VCL_Return_Name(unsigned);
 const char *VCL_Method_Name(unsigned);
 void VCL_Bo2Ctx(struct vrt_ctx *, struct busyobj *);
diff --git a/bin/varnishd/cache/cache_vcl_vrt.c b/bin/varnishd/cache/cache_vcl_vrt.c
index b183f5dff..40cb78430 100644
--- a/bin/varnishd/cache/cache_vcl_vrt.c
+++ b/bin/varnishd/cache/cache_vcl_vrt.c
@@ -73,16 +73,6 @@ VCL_Method_Name(unsigned m)
 
 /*--------------------------------------------------------------------*/
 
-void
-VCL_Onboard(const struct req *req, const struct busyobj *bo)
-{
-
-	CHECK_OBJ_ORNULL(req, REQ_MAGIC);
-	CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC);
-	assert(req != NULL || bo != NULL);
-	assert(req == NULL || bo == NULL);
-}
-
 void
 VCL_Refresh(struct vcl **vcc)
 {
@@ -271,11 +261,12 @@ VRT_vcl_select(VRT_CTX, VCL_VCL vcl)
 	struct req *req = ctx->req;
 
 	CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC);
+	VCL_TaskLeave(req->vcl, req->privs);
 	VCL_Rel(&req->vcl);
 	vcl_get(&req->vcl, vcl);
 	/* XXX: better logging */
 	VSLb(ctx->req->vsl, SLT_Debug, "Now using %s VCL", vcl->loaded_name);
-	VCL_Onboard(req, NULL);
+	VCL_TaskEnter(req->vcl, req->privs);
 }
 
 struct vclref *
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index c4c709aa7..ada746692 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -523,7 +523,8 @@ VRT_Rollback(VRT_CTX, VCL_HTTP hp)
 	} else if (hp == ctx->http_bereq) {
 		CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
 		// -> VBO_Rollback ?
-		VRTPRIV_dynamic_kill(ctx->bo->privs, (uintptr_t)ctx->bo);
+		VCL_TaskLeave(ctx->bo->vcl, ctx->bo->privs);
+		VCL_TaskEnter(ctx->bo->vcl, ctx->bo->privs);
 		HTTP_Copy(ctx->bo->bereq, ctx->bo->bereq0);
 		WS_Reset(ctx->bo->bereq->ws, ctx->bo->ws_bo);
 		WS_Reset(ctx->bo->ws, ctx->bo->ws_bo);
diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c
index b29f12047..1d7aff5a2 100644
--- a/bin/varnishd/cache/cache_vrt_priv.c
+++ b/bin/varnishd/cache/cache_vrt_priv.c
@@ -190,3 +190,31 @@ VRT_priv_fini(const struct vmod_priv *p)
 	if (p->priv != NULL && p->free != NULL)
 		p->free(p->priv);
 }
+
+/*--------------------------------------------------------------------*/
+
+void
+VCL_TaskEnter(struct vcl *vcl, struct vrt_privs *privs)
+{
+
+	VSL(SLT_Debug, 0, "%s %p %p %u", __func__, vcl, privs, privs->magic);
+	AN(vcl);
+	AZ(privs->magic);
+	VRTPRIV_init(privs);
+}
+
+void
+VCL_TaskLeave(struct vcl *vcl, struct vrt_privs *privs)
+{
+	struct vrt_priv *vp, *vp1;
+
+	AN(vcl);
+	VSL(SLT_Debug, 0, "%s %p %p %u", __func__, vcl, privs, privs->magic);
+	CHECK_OBJ_NOTNULL(privs, VRT_PRIVS_MAGIC);
+	VTAILQ_FOREACH_SAFE(vp, &privs->privs, list, vp1) {
+		VTAILQ_REMOVE(&privs->privs, vp, list);
+		VRT_priv_fini(vp->priv);
+	}
+	INIT_OBJ(privs, 0);
+}
+
diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c
index 30bd3b894..b339339f5 100644
--- a/bin/varnishd/http1/cache_http1_fsm.c
+++ b/bin/varnishd/http1/cache_http1_fsm.c
@@ -286,7 +286,6 @@ http1_dissect(struct worker *wrk, struct req *req)
 	VCL_Refresh(&wrk->vcl);
 	req->vcl = wrk->vcl;
 	wrk->vcl = NULL;
-	VCL_Onboard(req, NULL);
 
 	HTTP_Setup(req->http, req->ws, req->vsl, SLT_ReqMethod);
 	req->err_code = HTTP1_DissectRequest(req->htc, req->http);
diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 644474309..f6ea07eae 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -643,7 +643,6 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 	VCL_Refresh(&wrk->vcl);
 	req->vcl = wrk->vcl;
 	wrk->vcl = NULL;
-	VCL_Onboard(req, NULL);
 	req->acct.req_hdrbytes += h2->rxf_len;
 
 	HTTP_Setup(req->http, req->ws, req->vsl, SLT_ReqMethod);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:10 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:10 +0000 (UTC)
Subject: [6.0] 028b602da Removing debugging
Message-ID: <20181031130810.2668894F10@lists.varnish-cache.org>


commit 028b602da4cce4e9027b5c0d4db8c03bfac72d2a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 28 21:36:48 2018 +0000

    Removing debugging

diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c
index 1d7aff5a2..3d8606b43 100644
--- a/bin/varnishd/cache/cache_vrt_priv.c
+++ b/bin/varnishd/cache/cache_vrt_priv.c
@@ -197,7 +197,6 @@ void
 VCL_TaskEnter(struct vcl *vcl, struct vrt_privs *privs)
 {
 
-	VSL(SLT_Debug, 0, "%s %p %p %u", __func__, vcl, privs, privs->magic);
 	AN(vcl);
 	AZ(privs->magic);
 	VRTPRIV_init(privs);
@@ -209,7 +208,6 @@ VCL_TaskLeave(struct vcl *vcl, struct vrt_privs *privs)
 	struct vrt_priv *vp, *vp1;
 
 	AN(vcl);
-	VSL(SLT_Debug, 0, "%s %p %p %u", __func__, vcl, privs, privs->magic);
 	CHECK_OBJ_NOTNULL(privs, VRT_PRIVS_MAGIC);
 	VTAILQ_FOREACH_SAFE(vp, &privs->privs, list, vp1) {
 		VTAILQ_REMOVE(&privs->privs, vp, list);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:10 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:10 +0000 (UTC)
Subject: [6.0] 1472ac416 Make each event sent to compiled VCL a separate task
 as far as per-task PRIVs.
Message-ID: <20181031130810.40C8A94F1C@lists.varnish-cache.org>


commit 1472ac4167a414ab5d7976159616128bcca0cbcb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Aug 28 21:42:08 2018 +0000

    Make each event sent to compiled VCL a separate task as far as
    per-task PRIVs.
    
    Fixes: #2749

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 98cc47dec..9bd2dee51 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -121,7 +121,6 @@ vcl_get_ctx(unsigned method, int msg)
 	}
 	ctx_cli.ws = &ws_cli;
 	WS_Assert(ctx_cli.ws);
-	VRTPRIV_init(cli_task_privs);
 	return (&ctx_cli);
 }
 
@@ -138,7 +137,6 @@ vcl_rel_ctx(struct vrt_ctx **ctx)
 	WS_Reset(&ws_cli, ws_snapshot_cli);
 	INIT_OBJ(*ctx, VRT_CTX_MAGIC);
 	*ctx = NULL;
-	VRTPRIV_dynamic_kill(cli_task_privs, (uintptr_t)cli_task_privs);
 }
 
 /*--------------------------------------------------------------------*/
@@ -163,7 +161,9 @@ vcl_send_event(VRT_CTX, enum vcl_event_e ev)
 	if (ev == VCL_EVENT_LOAD || ev == VCL_EVENT_WARM)
 		AN(ctx->msg);
 
+	VCL_TaskEnter(ctx->vcl, cli_task_privs);
 	r = ctx->vcl->conf->event_vcl(ctx, ev);
+	VCL_TaskLeave(ctx->vcl, cli_task_privs);
 
 	if (r && (ev == VCL_EVENT_COLD || ev == VCL_EVENT_DISCARD))
 		WRONG("A VMOD cannot fail COLD or DISCARD events");

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:10 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:10 +0000 (UTC)
Subject: [6.0] 7bec6eab2 Privatize stuff.
Message-ID: <20181031130810.61B9D94F2D@lists.varnish-cache.org>


commit 7bec6eab2ac9516fcef856f1ea2beb42a5b3bd8f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Aug 29 05:13:48 2018 +0000

    Privatize stuff.

diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index 2b7b254ad..9d5d78829 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -387,8 +387,8 @@ void VCL_Poll(void);
 void VCL_Ref(struct vcl *);
 void VCL_Refresh(struct vcl **);
 void VCL_Rel(struct vcl **);
-void VCL_TaskEnter(struct vcl *, struct vrt_privs *);
-void VCL_TaskLeave(struct vcl *, struct vrt_privs *);
+void VCL_TaskEnter(const struct vcl *, struct vrt_privs *);
+void VCL_TaskLeave(const struct vcl *, struct vrt_privs *);
 const char *VCL_Return_Name(unsigned);
 const char *VCL_Method_Name(unsigned);
 void VCL_Bo2Ctx(struct vrt_ctx *, struct busyobj *);
@@ -405,8 +405,6 @@ typedef int vcl_be_func(struct cli *, struct director *, void *);
 int VCL_IterDirector(struct cli *, const char *, vcl_be_func *, void *);
 
 /* cache_vrt.c */
-void VRTPRIV_init(struct vrt_privs *privs);
-void VRTPRIV_dynamic_kill(struct vrt_privs *privs, uintptr_t id);
 void pan_privs(struct vsb *, const struct vrt_privs *);
 
 /* cache_vrt_priv.c */
diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c
index 3d8606b43..e4dd16479 100644
--- a/bin/varnishd/cache/cache_vrt_priv.c
+++ b/bin/varnishd/cache/cache_vrt_priv.c
@@ -83,7 +83,7 @@ pan_privs(struct vsb *vsb, const struct vrt_privs *privs)
 /*--------------------------------------------------------------------
  */
 
-void
+static void
 VRTPRIV_init(struct vrt_privs *privs)
 {
 
@@ -117,23 +117,6 @@ vrt_priv_dynamic(const struct vcl *vcl, struct ws *ws,
 	return (vp->priv);
 }
 
-void
-VRTPRIV_dynamic_kill(struct vrt_privs *privs, uintptr_t id)
-{
-	struct vrt_priv *vp, *vp1;
-
-	CHECK_OBJ_NOTNULL(privs, VRT_PRIVS_MAGIC);
-	AN(id);
-
-	VTAILQ_FOREACH_SAFE(vp, &privs->privs, list, vp1) {
-		CHECK_OBJ_NOTNULL(vp, VRT_PRIV_MAGIC);
-		if (id == vp->id) {
-			VTAILQ_REMOVE(&privs->privs, vp, list);
-			VRT_priv_fini(vp->priv);
-		}
-	}
-}
-
 struct vmod_priv *
 VRT_priv_task(VRT_CTX, const void *vmod_id)
 {
@@ -194,7 +177,7 @@ VRT_priv_fini(const struct vmod_priv *p)
 /*--------------------------------------------------------------------*/
 
 void
-VCL_TaskEnter(struct vcl *vcl, struct vrt_privs *privs)
+VCL_TaskEnter(const struct vcl *vcl, struct vrt_privs *privs)
 {
 
 	AN(vcl);
@@ -203,7 +186,7 @@ VCL_TaskEnter(struct vcl *vcl, struct vrt_privs *privs)
 }
 
 void
-VCL_TaskLeave(struct vcl *vcl, struct vrt_privs *privs)
+VCL_TaskLeave(const struct vcl *vcl, struct vrt_privs *privs)
 {
 	struct vrt_priv *vp, *vp1;
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:10 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:10 +0000 (UTC)
Subject: [6.0] f511d912f initialize PRIV_TASK and PRIV_TOP vmod arguments once
 per subroutine
Message-ID: <20181031130810.8D8A194F42@lists.varnish-cache.org>


commit f511d912fc266b2cce814eccebbac5efa72cf3dd
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Aug 14 14:02:34 2018 +0200

    initialize PRIV_TASK and PRIV_TOP vmod arguments once per subroutine
    
    ... and fail the VCL unless successful.
    
    Providing the PRIVs to vmods is a core function, so error handling
    should happen outside vmods.
    
    Besides being safe, this initialization can be more efficient than
    previous code for PRIVs used frequently within the same subroutine.
    
    An alternative approach would be to initialize all privs once per
    task / top request, but unless all privs are actually used in a VCL,
    this approach could impose significant overhead, both in terms of time
    and memory. By initializing privs once per sub, we impose overhead for
    privs which are referenced but not actually used in a subroutine, but
    not for all of the vcl.
    
    Fixes #2708

diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index a7dfc98d3..1adc26721 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -104,6 +104,8 @@ vcc_NewProc(struct vcc *tl, struct symbol *sym)
 	AN(p);
 	VTAILQ_INIT(&p->calls);
 	VTAILQ_INIT(&p->uses);
+	VTAILQ_INIT(&p->priv_tasks);
+	VTAILQ_INIT(&p->priv_tops);
 	VTAILQ_INSERT_TAIL(&tl->procs, p, list);
 	p->prologue = VSB_new_auto();
 	AN(p->prologue);
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index befc15a94..4e67b538c 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -174,6 +174,7 @@ struct symbol {
 };
 
 VTAILQ_HEAD(tokenhead, token);
+VTAILQ_HEAD(procprivhead, procpriv);
 
 struct proc {
 	unsigned		magic;
@@ -181,6 +182,8 @@ struct proc {
 	const struct method	*method;
 	VTAILQ_HEAD(,proccall)	calls;
 	VTAILQ_HEAD(,procuse)	uses;
+	struct procprivhead	priv_tasks;
+	struct procprivhead	priv_tops;
 	VTAILQ_ENTRY(proc)	list;
 	struct token		*name;
 	unsigned		ret_bitmap;
@@ -389,6 +392,8 @@ int vcc_CheckAction(struct vcc *tl);
 void vcc_AddUses(struct vcc *, const struct token *, const struct token *,
      unsigned mask, const char *use);
 int vcc_CheckUses(struct vcc *tl);
+const char *vcc_MarkPriv(struct vcc *, struct procprivhead *,
+    const char *);
 
 #define ERRCHK(tl)      do { if ((tl)->err) return; } while (0)
 #define ErrInternal(tl) vcc__ErrInternal(tl, __func__, __LINE__)
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index e016696a5..1f0699e19 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -399,31 +399,49 @@ vcc_Eval_Var(struct vcc *tl, struct expr **e, struct token *t,
  */
 
 static struct expr *
-vcc_priv_arg(struct vcc *tl, const char *p, const char *name, const char *vmod)
+vcc_priv_arg(struct vcc *tl, const char *p, const struct symbol *sym)
 {
-	struct expr *e2;
-	char buf[32];
+	char buf[64];
 	struct inifin *ifp;
+	const char *vmod, *f = NULL;
+	struct procprivhead *marklist = NULL;
+
+	AN(sym);
+	AN(sym->vmod);
+	vmod = sym->vmod;
 
-	(void)name;
 	if (!strcmp(p, "PRIV_VCL")) {
-		e2 = vcc_mk_expr(VOID, "&vmod_priv_%s", vmod);
+		return (vcc_mk_expr(VOID, "&vmod_priv_%s", vmod));
 	} else if (!strcmp(p, "PRIV_CALL")) {
 		bprintf(buf, "vmod_priv_%u", tl->unique++);
 		ifp = New_IniFin(tl);
 		Fh(tl, 0, "static struct vmod_priv %s;\n", buf);
 		VSB_printf(ifp->fin, "\tVRT_priv_fini(&%s);", buf);
-		e2 = vcc_mk_expr(VOID, "&%s", buf);
+		return (vcc_mk_expr(VOID, "&%s", buf));
 	} else if (!strcmp(p, "PRIV_TASK")) {
-		e2 = vcc_mk_expr(VOID,
-		    "VRT_priv_task(ctx, &VGC_vmod_%s)", vmod);
+		f = "task";
+		marklist = &tl->curproc->priv_tasks;
 	} else if (!strcmp(p, "PRIV_TOP")) {
-		e2 = vcc_mk_expr(VOID,
-		    "VRT_priv_top(ctx, &VGC_vmod_%s)", vmod);
+		f = "top";
+		marklist = &tl->curproc->priv_tops;
 	} else {
 		WRONG("Wrong PRIV_ type");
 	}
-	return (e2);
+	AN(f);
+	AN(marklist);
+	bprintf(buf, "ARG_priv_%s_%s", f, vmod);
+
+	if (vcc_MarkPriv(tl, marklist, vmod) == NULL)
+		VSB_printf(tl->curproc->prologue,
+			   "  struct vmod_priv *%s = "
+			   "VRT_priv_%s(ctx, &VGC_vmod_%s);\n"
+			   "  if (%s == NULL) {\n"
+			   "    VRT_fail(ctx, \"failed to get %s priv "
+			   "for vmod %s\");\n"
+			   "    return;\n"
+			   "  }\n",
+			   buf, f, vmod, buf, f, vmod);
+	return (vcc_mk_expr(VOID, "%s", buf));
 }
 
 struct func_arg {
@@ -522,8 +540,7 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv,
 
 		vvp = VTAILQ_FIRST(&vv->children);
 		if (!memcmp(vvp->value, "PRIV_", 5)) {
-			fa->result = vcc_priv_arg(tl, vvp->value,
-			    sym->name, sym->vmod);
+			fa->result = vcc_priv_arg(tl, vvp->value, sym);
 			continue;
 		}
 		fa->type = VCC_Type(vvp->value);
diff --git a/lib/libvcc/vcc_xref.c b/lib/libvcc/vcc_xref.c
index 31d3bc6cd..bfa98a6f6 100644
--- a/lib/libvcc/vcc_xref.c
+++ b/lib/libvcc/vcc_xref.c
@@ -39,6 +39,7 @@
 
 #include "config.h"
 
+#include <string.h>
 #include "vcc_compile.h"
 
 /*--------------------------------------------------------------------*/
@@ -59,6 +60,11 @@ struct procuse {
 	struct proc		*fm;
 };
 
+struct procpriv {
+	VTAILQ_ENTRY(procpriv)	list;
+	const char		*vmod;
+};
+
 /*--------------------------------------------------------------------*/
 
 static void
@@ -355,3 +361,28 @@ VCC_XrefTable(struct vcc *tl)
 	VCC_WalkSymbols(tl, vcc_xreftable, SYM_NONE);
 	Fc(tl, 0, "*/\n\n");
 }
+
+/*---------------------------------------------------------------------
+ * mark vmod as referenced, return NULL if not yet marked, vmod if marked
+ */
+
+const char *
+vcc_MarkPriv(struct vcc *tl, struct procprivhead *head,
+    const char *vmod)
+{
+	struct procpriv *pp;
+
+	AN(vmod);
+
+	VTAILQ_FOREACH(pp, head, list) {
+		if (pp->vmod == vmod)
+			return (vmod);
+		AN(strcmp(pp->vmod, vmod));
+	}
+
+	pp = TlAlloc(tl, sizeof *pp);
+	assert(pp != NULL);
+	pp->vmod = vmod;
+	VTAILQ_INSERT_TAIL(head, pp, list);
+	return (NULL);
+}

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:10 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:10 +0000 (UTC)
Subject: [6.0] f8fbc8837 add a test for task_priv allocation in the function
 prologue
Message-ID: <20181031130810.A4D8E94F4B@lists.varnish-cache.org>


commit f8fbc88371a11c0f7fe3490655d67e5457fd8bf2
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Aug 22 14:02:54 2018 +0200

    add a test for task_priv allocation in the function prologue

diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc
index 5c0fa7536..355bb3c56 100644
--- a/bin/varnishtest/tests/m00000.vtc
+++ b/bin/varnishtest/tests/m00000.vtc
@@ -20,7 +20,17 @@ varnish v1 -vcl+backend {
 		set req.http.overwrite = "the workspace " +
 		    "to ensure we notice any unfinished privs";
 	}
+
+	sub priv_task {
+		debug.test_priv_task("foo");
+	}
+
 	sub vcl_recv {
+		if (req.url == "/priv-task-no-mem") {
+			vtc.workspace_alloc(client, -4);
+			call priv_task;
+			return (fail);
+		}
 		if (req.url == "/fail") {
 			debug.test_priv_task("foo");
 			return (fail);
@@ -65,6 +75,12 @@ client c1 {
 	expect resp.status == 503
 } -run
 
+client c1 {
+	txreq -url "/priv-task-no-mem"
+	rxresp
+	expect resp.status == 503
+} -run
+
 varnish v1 -expect DEBUG.count == 1
 
 logexpect l1 -v v1 -g raw -d 1 {

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:10 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:10 +0000 (UTC)
Subject: [6.0] 48c3503bc Only dlclose() Vmods after all "fini" processing.
Message-ID: <20181031130810.D5F5094F69@lists.varnish-cache.org>


commit 48c3503bc808c41fd28cdd4f8945c51eb837b002
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Aug 30 07:53:32 2018 +0000

    Only dlclose() Vmods after all "fini" processing.
    
    Conflicts:
            include/vrt.h
    
    This appears to break the VRT interface, but there are symbols we are
    willing to break even on a stable branch because they are needed by
    VCC-generated code and off limits to inline C and VMODs.
    
    Refs #2800

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 9bd2dee51..e705a86a1 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -532,7 +532,7 @@ vcl_cancel_load(VRT_CTX, struct cli *cli, const char *name, const char *step)
 	if (VSB_len(ctx->msg))
 		VCLI_Out(cli, "\nMessage:\n\t%s", VSB_data(ctx->msg));
 	*ctx->handling = 0;
-	AZ(vcl->conf->event_vcl(ctx, VCL_EVENT_DISCARD));
+	AZ(vcl_send_event(ctx, VCL_EVENT_DISCARD));
 	vcl_KillBackends(vcl);
 	free(vcl->loaded_name);
 	VCL_Close(&vcl);
diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c
index 25a119da4..af49ef40b 100644
--- a/bin/varnishd/cache/cache_vrt_vmod.c
+++ b/bin/varnishd/cache/cache_vrt_vmod.c
@@ -162,7 +162,7 @@ VRT_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len,
 }
 
 void
-VRT_Vmod_Fini(struct vmod **hdl)
+VRT_Vmod_Unload(VRT_CTX, struct vmod **hdl)
 {
 	struct vmod *v;
 
@@ -170,6 +170,9 @@ VRT_Vmod_Fini(struct vmod **hdl)
 
 	TAKE_OBJ_NOTNULL(v, hdl, VMOD_MAGIC);
 
+	VCL_TaskLeave(ctx->vcl, cli_task_privs);
+	VCL_TaskEnter(ctx->vcl, cli_task_privs);
+
 #ifndef DONT_DLCLOSE_VMODS
 	/*
 	 * atexit(3) handlers are not called during dlclose(3).  We don't
diff --git a/include/vrt.h b/include/vrt.h
index 2fc7c60cc..3db7c0438 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -425,7 +425,7 @@ int VRT_VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst);
 /* VMOD/Modules related */
 int VRT_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len,
     const char *nm, const char *path, const char *file_id, const char *backup);
-void VRT_Vmod_Fini(struct vmod **hdl);
+void VRT_Vmod_Unload(VRT_CTX, struct vmod **hdl);
 
 /* VCL program related */
 VCL_VCL VRT_vcl_get(VRT_CTX, const char *);
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 1adc26721..bcfaadcab 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -139,12 +139,16 @@ New_IniFin(struct vcc *tl)
 {
 	struct inifin *p;
 
-	p = TlAlloc(tl, sizeof *p);
+	ALLOC_OBJ(p, INIFIN_MAGIC);
 	AN(p);
-	p->magic = INIFIN_MAGIC;
 	p->ini = VSB_new_auto();
+	AN(p->ini);
 	p->fin = VSB_new_auto();
+	AN(p->fin);
+	p->final = VSB_new_auto();
+	AN(p->final);
 	p->event = VSB_new_auto();
+	AN(p->event);
 	p->n = ++tl->ninifin;
 	VTAILQ_INSERT_TAIL(&tl->inifin, p, list);
 	return (p);
@@ -324,20 +328,31 @@ EmitInitFini(const struct vcc *tl)
 	 */
 	Fc(tl, 0, "\nstatic int\nVGC_Discard(VRT_CTX)\n{\n\n");
 
-	Fc(tl, 0, "\tswitch (vgc_inistep) {\n\n");
+	Fc(tl, 0, "\tswitch (vgc_inistep) {\n");
 	VTAILQ_FOREACH_REVERSE(p, &tl->inifin, inifinhead, list) {
 		AZ(VSB_finish(p->fin));
 		if (q)
 			assert(q->n > p->n);
 		q = p;
-		if (VSB_len(p->fin)) {
-			Fc(tl, 0, "\t\tcase %u :\n", p->n);
+		Fc(tl, 0, "\t\tcase %u:\n", p->n);
+		if (VSB_len(p->fin))
 			Fc(tl, 0, "\t%s\n", VSB_data(p->fin));
-			Fc(tl, 0, "\t\t\t/* FALLTHROUGH */\n");
-		}
+		Fc(tl, 0, "\t\t\t/* FALLTHROUGH */\n");
 		VSB_destroy(&p->fin);
 	}
-	Fc(tl, 0, "\t}\n");
+	Fc(tl, 0, "\t\tdefault:\n\t\t\tbreak;\n");
+	Fc(tl, 0, "\t}\n\n");
+	Fc(tl, 0, "\tswitch (vgc_inistep) {\n");
+	VTAILQ_FOREACH_REVERSE(p, &tl->inifin, inifinhead, list) {
+		AZ(VSB_finish(p->final));
+		Fc(tl, 0, "\t\tcase %u:\n", p->n);
+		if (VSB_len(p->final))
+			Fc(tl, 0, "\t%s\n", VSB_data(p->final));
+		Fc(tl, 0, "\t\t\t/* FALLTHROUGH */\n");
+		VSB_destroy(&p->final);
+	}
+	Fc(tl, 0, "\t\tdefault:\n\t\t\tbreak;\n");
+	Fc(tl, 0, "\t}\n\n");
 
 	Fc(tl, 0, "\treturn (0);\n");
 	Fc(tl, 0, "}\n");
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 4e67b538c..3a98c76e0 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -201,6 +201,7 @@ struct inifin {
 	unsigned		n;
 	struct vsb		*ini;
 	struct vsb		*fin;
+	struct vsb		*final;
 	struct vsb		*event;
 	VTAILQ_ENTRY(inifin)	list;
 };
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index fd6a7acc6..babd70eef 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -105,7 +105,7 @@ vcc_json_always(struct vcc *tl, const struct symbol *msym)
 			    vv2->value, msym->name);
 			VSB_printf(ifp->fin,
 			    "\t\t(void)%s(ctx, &vmod_priv_%s,\n"
-			    "\t\t\t    VCL_EVENT_DISCARD);\n",
+			    "\t\t\t    VCL_EVENT_DISCARD);",
 			    vv2->value, msym->name);
 			VSB_printf(ifp->event, "%s(ctx, &vmod_priv_%s, ev)",
 			    vv2->value, msym->name);
@@ -324,8 +324,9 @@ vcc_ParseImport(struct vcc *tl)
 	    VCC_INFO_PREFIX, fnp, PF(mod), vmd->file_id);
 
 	/* XXX: zero the function pointer structure ?*/
-	VSB_printf(ifp->fin, "\t\tVRT_priv_fini(&vmod_priv_%.*s);\n", PF(mod));
-	VSB_printf(ifp->fin, "\t\t\tVRT_Vmod_Fini(&VGC_vmod_%.*s);", PF(mod));
+	VSB_printf(ifp->fin, "\t\tVRT_priv_fini(&vmod_priv_%.*s);", PF(mod));
+	VSB_printf(ifp->final,
+	    "\t\tVRT_Vmod_Unload(ctx, &VGC_vmod_%.*s);", PF(mod));
 
 	vj = vjsn_parse(vmd->json, &p);
 	XXXAZ(p);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:10 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:10 +0000 (UTC)
Subject: [6.0] 2420f54db Test that vmod code is still present when calling a
 task_priv .fini
Message-ID: <20181031130811.0370394F79@lists.varnish-cache.org>


commit 2420f54dbc8ebfe08c9d60312248d417c066f86d
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Aug 30 12:12:42 2018 +0200

    Test that vmod code is still present when calling a task_priv .fini
    
    Tests 20ab2abc36781b03605785c0a9ae463460cdffef

diff --git a/bin/varnishtest/tests/r02451.vtc b/bin/varnishtest/tests/r02451.vtc
index 2ff3facdc..0dc0a5a4c 100644
--- a/bin/varnishtest/tests/r02451.vtc
+++ b/bin/varnishtest/tests/r02451.vtc
@@ -44,8 +44,8 @@ varnish v1 -vcl+backend {
 
 logexpect l1 -v v1 -g raw -d 1 {
 	expect 0 0    CLI		{^Rd vcl.load}
-	expect 0 =    VCL_Log		{^func something to remember}
-	expect 0 =    VCL_Log		{^obj something to remember}
+	expect 4 =    VCL_Log		{^func something to remember}
+	expect 2 =    VCL_Log		{^obj something to remember}
 } -start
 
 client c1 {
diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc
index 3bac4aa3d..a67928ca9 100644
--- a/bin/varnishtest/tests/v00041.vtc
+++ b/bin/varnishtest/tests/v00041.vtc
@@ -61,7 +61,11 @@ varnish v1 -arg "-p debug=+vclrel" -vcl+backend {
 
 logexpect l1 -v v1 -g raw -d 1 {
 	expect 0 0    CLI		{^Rd vcl.load}
+	expect 0 =    Debug		{^test_priv_task.*new.$}
+	expect 0 =    Debug		{^test_priv_task.*update.$}
+	expect 0 =    Debug		{^test_priv_task.*exists.$}
 	expect 0 =    VCL_Log		{^func something to remember}
+	expect 0 =    Debug		{^test_priv_task.*exists.$}
 	expect 0 =    VCL_Log		{^obj something to remember}
 
 	expect * 1002 Begin		fetch$
@@ -77,8 +81,14 @@ logexpect l1 -v v1 -g raw -d 1 {
 	expect 0 =    VCL_Log		^bar
 
 	expect * 0    Debug		{^vcl1: VCL_EVENT_COLD}
+	expect * 0    CLI		{^Rd vcl.discard}
+	expect 0 =    Debug		{^test_priv_task.*new.$}
+	expect 0 =    Debug		{^test_priv_task.*update.$}
+	expect 0 =    Debug		{^test_priv_task.*exists.$}
 	expect * =    VCL_Log		{^func cleaning up}
+	expect 0 =    Debug		{^test_priv_task.*exists.$}
 	expect 0 =    VCL_Log		{^obj cleaning up}
+	expect 0 =    Debug		{^priv_task_free}
 } -start
 
 client c1 {
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index 0da45d826..61a18c9ed 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -91,16 +91,27 @@ xyzzy_test_priv_call(VRT_CTX, struct vmod_priv *priv)
 	}
 }
 
+static void
+priv_task_free(void *ptr)
+{
+	AN(ptr);
+	VSL(SLT_Debug, 0, "priv_task_free(%p)", ptr);
+	free(ptr);
+}
+
 VCL_STRING v_matchproto_(td_debug_test_priv_task)
 xyzzy_test_priv_task(VRT_CTX, struct vmod_priv *priv, VCL_STRING s)
 {
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	if (s == NULL || *s == '\0') {
-		return priv->priv;
+		VSL(SLT_Debug, 0, "test_priv_task(%p) = %p (exists)",
+		    priv, priv->priv);
 	} else if (priv->priv == NULL) {
 		priv->priv = strdup(s);
-		priv->free = free;
+		priv->free = priv_task_free;
+		VSL(SLT_Debug, 0, "test_priv_task(%p) = %p (new)",
+		    priv, priv->priv);
 	} else {
 		char *n = realloc(priv->priv,
 		    strlen(priv->priv) + strlen(s) + 2);
@@ -109,7 +120,11 @@ xyzzy_test_priv_task(VRT_CTX, struct vmod_priv *priv, VCL_STRING s)
 		strcat(n, " ");
 		strcat(n, s);
 		priv->priv = n;
+		VSL(SLT_Debug, 0, "test_priv_task(%p) = %p (update)",
+		    priv, priv->priv);
 	}
+	if (priv->priv != NULL)
+		assert(priv->free == priv_task_free);
 	return (priv->priv);
 }
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:11 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:11 +0000 (UTC)
Subject: [6.0] 5e9e1cdcb varnishtest: add -keepalive to repeat on a single
 connection
Message-ID: <20181031130811.2C19594F94@lists.varnish-cache.org>


commit 5e9e1cdcb45a10b04e1070edd47fc86d00a94636
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Aug 30 17:44:07 2018 +0200

    varnishtest: add -keepalive to repeat on a single connection
    
    For tests which do not require new connections for repetitions (for
    example because of possible error conditions), this reduces run time
    and the number of required ephemeral ports.
    
    The latter is a real issue when running many vtcs in parallel which
    each run many repetitions of the same test (for example to check for
    possible race conditions). When ephemeral ports are exhausted,
    seemingly unrelated issues like the following can be observed:
    
    ---- c1010 14.0 Failed to open 127.0.0.1 59763: (null)
    
    **** v1    1.8 vsl|          0 CLI             - Wr 300 65 Listen failed
    on socket '127.0.0.1:33328': Address already in use
    **** v1    1.8 vsl|          0 CLI             - EOF on CLI connection,
    worker stops
    
    An argument could be made that UDS does not suffer from the port
    exhaustion issue and thus such tests could be migrated to UDS. Yet
    also for this case the run time point remains, plus deliberately
    testing many iterations on a single connection could have its own
    merits.

diff --git a/bin/varnishtest/tests/m00041.vtc b/bin/varnishtest/tests/m00041.vtc
index 4c543d845..aeda981f9 100644
--- a/bin/varnishtest/tests/m00041.vtc
+++ b/bin/varnishtest/tests/m00041.vtc
@@ -404,7 +404,7 @@ client c1 {
 
 # Decode failures
 
-server s1 -repeat 11 {
+server s1 -repeat 11 -keepalive {
 	rxreq
 	txresp
 } -start
diff --git a/bin/varnishtest/tests/m00042.vtc b/bin/varnishtest/tests/m00042.vtc
index aaac4b917..6e6004f63 100644
--- a/bin/varnishtest/tests/m00042.vtc
+++ b/bin/varnishtest/tests/m00042.vtc
@@ -414,7 +414,7 @@ client c1 {
 
 # Decode failures
 
-server s1 -repeat 11 {
+server s1 -repeat 11 -keepalive {
 	rxreq
 	txresp
 } -start
diff --git a/bin/varnishtest/tests/r01834.vtc b/bin/varnishtest/tests/r01834.vtc
index 827105b8b..33437a9b2 100644
--- a/bin/varnishtest/tests/r01834.vtc
+++ b/bin/varnishtest/tests/r01834.vtc
@@ -4,6 +4,7 @@ varnishtest "#1834 - Buffer overflow in backend workspace"
 # workspace left. If failing it would be because we tripped the canary
 # at the end of the workspace.
 
+
 server s1 -repeat 64 {
 	rxreq
 	txresp
diff --git a/bin/varnishtest/tests/r02372.vtc b/bin/varnishtest/tests/r02372.vtc
index c32d17b96..a01401f16 100644
--- a/bin/varnishtest/tests/r02372.vtc
+++ b/bin/varnishtest/tests/r02372.vtc
@@ -1,6 +1,6 @@
 varnishtest "Count purges when there are many variants"
 
-server s1 -repeat 72 {
+server s1 -repeat 72 -keepalive {
 	rxreq
 	txresp -hdr "Vary: foo"
 } -start
@@ -16,7 +16,7 @@ varnish v1 -arg "-p workspace_thread=512" -vcl+backend {
 	}
 } -start
 
-client c1 -repeat 72 {
+client c1 -repeat 72 -keepalive {
 	txreq
 	rxresp
 } -run
diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c
index d2f72b955..5c714de0b 100644
--- a/bin/varnishtest/vtc_client.c
+++ b/bin/varnishtest/vtc_client.c
@@ -60,6 +60,7 @@ struct client {
 	int			proxy_version;
 
 	unsigned		repeat;
+	unsigned		keepalive;
 
 	unsigned		running;
 	pthread_t		tp;
@@ -215,7 +216,8 @@ client_thread(void *priv)
 	if (c->repeat == 0)
 		c->repeat = 1;
 	if (c->repeat != 1)
-		vtc_log(vl, 2, "Started (%u iterations)", c->repeat);
+		vtc_log(vl, 2, "Started (%u iterations%s)", c->repeat,
+			c->keepalive ? " using keepalive" : "");
 	for (u = 0; u < c->repeat; u++) {
 		char *addr = VSB_data(vsb);
 
@@ -231,7 +233,11 @@ client_thread(void *priv)
 		(void)VTCP_blocking(fd);
 		if (c->proxy_spec != NULL)
 			client_proxy(vl, fd, c->proxy_version, c->proxy_spec);
-		fd = http_process(vl, c->spec, fd, NULL, addr);
+		if (! c->keepalive)
+			fd = http_process(vl, c->spec, fd, NULL, addr);
+		else
+			while (fd >= 0 && u++ < c->repeat)
+				fd = http_process(vl, c->spec, fd, NULL, addr);
 		vtc_log(vl, 3, "closing fd %d", fd);
 		VTCP_close(&fd);
 	}
@@ -393,6 +399,10 @@ cmd_client(CMD_ARGS)
 			av++;
 			continue;
 		}
+		if (!strcmp(*av, "-keepalive")) {
+			c->keepalive = 1;
+			continue;
+		}
 		if (!strcmp(*av, "-start")) {
 			client_start(c);
 			continue;
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 741c5f12a..d89a91a5a 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -95,6 +95,10 @@ extern const struct cmds http_cmds[];
  * \-repeat NUMBER
  *        Instead of processing the specification only once, do it NUMBER times.
  *
+ * \-keepalive
+ *        For repeat, do not open new connections but rather run all
+ *        iterations in the same connection
+ *
  * \-break (server only)
  *        Stop the server.
  *
diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index 4876352f2..d156185d9 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -52,6 +52,7 @@ struct server {
 	char			run;
 
 	unsigned		repeat;
+	unsigned		keepalive;
 	char			*spec;
 
 	int			depth;
@@ -234,10 +235,9 @@ server_thread(void *priv)
 	vl = vtc_logopen(s->name);
 	pthread_cleanup_push(vtc_logclose, vl);
 
-	vtc_log(vl, 2, "Started on %s", s->listen);
+	vtc_log(vl, 2, "Started on %s (%u iterations%s)", s->listen,
+		s->repeat, s->keepalive ? " using keepalive" : "");
 	for (i = 0; i < s->repeat; i++) {
-		if (s->repeat > 1)
-			vtc_log(vl, 3, "Iteration %d", i);
 		addr = (void*)&addr_s;
 		l = sizeof addr_s;
 		fd = accept(s->sock, addr, &l);
@@ -248,7 +248,12 @@ server_thread(void *priv)
 			vtc_log(vl, 3, "accepted fd %d %s %s", fd, abuf, pbuf);
 		} else
 			vtc_log(vl, 3, "accepted fd %d 0.0.0.0 0", fd);
-		fd = http_process(vl, s->spec, fd, &s->sock, s->listen);
+		if (! s->keepalive)
+			fd = http_process(vl, s->spec, fd, &s->sock, s->listen);
+		else
+			while (fd >= 0 && i++ < s->repeat)
+				fd = http_process(vl, s->spec, fd,
+				    &s->sock, s->listen);
 		vtc_log(vl, 3, "shutting fd %d", fd);
 		j = shutdown(fd, SHUT_WR);
 		if (!VTCP_Check(j))
@@ -526,6 +531,10 @@ cmd_server(CMD_ARGS)
 			av++;
 			continue;
 		}
+		if (!strcmp(*av, "-keepalive")) {
+			s->keepalive = 1;
+			continue;
+		}
 		if (!strcmp(*av, "-listen")) {
 			if (s->sock >= 0)
 				VTCP_close(&s->sock);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:11 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:11 +0000 (UTC)
Subject: [6.0] 613ab495c check if this stabilizes the test
Message-ID: <20181031130811.4CA2094F9D@lists.varnish-cache.org>


commit 613ab495c7299e4122a02a053a9b89a31727da3e
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Aug 30 17:59:35 2018 +0200

    check if this stabilizes the test

diff --git a/bin/varnishtest/tests/b00064.vtc b/bin/varnishtest/tests/b00064.vtc
index 6054ec1e6..09daf9b09 100644
--- a/bin/varnishtest/tests/b00064.vtc
+++ b/bin/varnishtest/tests/b00064.vtc
@@ -83,7 +83,7 @@ client c2 {
 	expect resp.http.X-req-grace < 0.
 } -start
 
-delay .1
+delay .2
 
 # c3 asks for graced object, but now we disable grace.
 client c3 {

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:11 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:11 +0000 (UTC)
Subject: [6.0] 83a372ecf Stabilize b000064.vtc for real
Message-ID: <20181031130811.7438594FBF@lists.varnish-cache.org>


commit 83a372ecf806041883d51b5a411445aa7632ce05
Author: P?l Hermunn Johansen <hermunn at varnish-software.com>
Date:   Fri Aug 31 15:29:12 2018 +0200

    Stabilize b000064.vtc for real
    
    Fixes: #2753

diff --git a/bin/varnishtest/tests/b00064.vtc b/bin/varnishtest/tests/b00064.vtc
index 09daf9b09..47271831b 100644
--- a/bin/varnishtest/tests/b00064.vtc
+++ b/bin/varnishtest/tests/b00064.vtc
@@ -38,8 +38,11 @@ varnish v1 -vcl+backend {
 		set req.http.X-grace = obj.grace;
 	}
 	sub vcl_backend_response {
-		set beresp.ttl = 0.1s;
+		set beresp.ttl = 1ms;
 		set beresp.grace = 1m;
+		if (bereq.is_bgfetch) {
+			set beresp.http.X-was-bgfetch = "1";
+		}
 	}
 	sub vcl_deliver {
 		if (req.http.X-grace) {
@@ -65,6 +68,7 @@ client c1 {
 	expect resp.body == "1"
 	expect resp.http.X-grace == <undef>
 	expect resp.http.X-req-grace == <undef>
+	expect resp.http.X-was-bgfetch == <undef>
 } -run
 
 # let the latest object's ttl expire.
@@ -81,11 +85,13 @@ client c2 {
 	expect resp.body == "1"
 	expect resp.http.X-grace == "60.000"
 	expect resp.http.X-req-grace < 0.
-} -start
+	expect resp.http.X-was-bgfetch == <undef>
+} -run
 
-delay .2
+# c3 asks for graced object, but now we disable grace. The c2 client
+# started the background fetch, which will take a long time (until c4
+# has gotten its reply).
 
-# c3 asks for graced object, but now we disable grace.
 client c3 {
 	txreq -hdr "X-no-grace: true"
 	rxresp
@@ -95,6 +101,7 @@ client c3 {
 	expect resp.body == "2"
 	expect resp.http.X-grace == "60.000"
 	expect resp.http.X-req-grace == "0.000"
+	expect resp.http.X-was-bgfetch == "1"
 } -start
 
 delay .1
@@ -112,9 +119,9 @@ client c4 {
 	expect resp.body == "1"
 	expect resp.http.X-grace == "60.000"
 	expect resp.http.X-req-grace < 0.
+	expect resp.http.X-was-bgfetch == <undef>
 } -start
 
-client c2 -wait
 client c3 -wait
 client c4 -wait
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:11 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:11 +0000 (UTC)
Subject: [6.0] 9f6cae739 Make the vmod->file_id a hash over the .vcc file.
Message-ID: <20181031130811.8E66A94FCC@lists.varnish-cache.org>


commit 9f6cae73952dfbf2e6f185be208416af8eaea174
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 3 06:52:32 2018 +0000

    Make the vmod->file_id a hash over the .vcc file.
    
    I belive this makes our builds reproducible.
    
    Fixes:  #2436
    
    Conflicts:
            include/vrt.h

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 669450fb7..673b53102 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -44,6 +44,7 @@ import unittest
 import random
 import copy
 import json
+import hashlib
 
 AMBOILERPLATE = '''
 # Boilerplate generated by vmodtool.py - changes will be overwritten
@@ -858,6 +859,7 @@ class vcc(object):
     def parse(self):
         global inputline
         a = "\n" + open(self.inputfile, "r").read()
+        self.file_id = hashlib.sha256(a).hexdigest()
         s = a.split("\n$")
         self.copyright = s.pop(0).strip()
         while s:
@@ -983,14 +985,7 @@ class vcc(object):
         fo.write('\t.proto =\tVmod_Proto,\n')
         fo.write('\t.json =\t\tVmod_Json,\n')
         fo.write('\t.abi =\t\tVMOD_ABI_Version,\n')
-        # NB: Sort of hackish:
-        # Fill file_id with random stuff, so we can tell if
-        # VCC and VRT_Vmod_Init() dlopens the same file
-        #
-        fo.write("\t.file_id =\t\"")
-        for i in range(32):
-            fo.write("%c" % random.randint(0x40, 0x5a))
-        fo.write("\",\n")
+        fo.write("\t.file_id =\t\"%s\",\n" % self.file_id)
         fo.write("};\n")
 
     def cfile(self):

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:11 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:11 +0000 (UTC)
Subject: [6.0] 2061c2c49 Fix vmodtool with python3
Message-ID: <20181031130811.A9ADD94FD8@lists.varnish-cache.org>


commit 2061c2c492b5b7100bd48ac76742c7950c0dd407
Author: Bernhard M. Wiedemann <bwiedemann at suse.de>
Date:   Mon Sep 3 11:58:14 2018 +0200

    Fix vmodtool with python3
    
    Without this patch, it did
    TypeError: Unicode-objects must be encoded before hashing
    
    Fixes: #2436

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 673b53102..e60cca6e5 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -859,7 +859,7 @@ class vcc(object):
     def parse(self):
         global inputline
         a = "\n" + open(self.inputfile, "r").read()
-        self.file_id = hashlib.sha256(a).hexdigest()
+        self.file_id = hashlib.sha256(a.encode('utf-8')).hexdigest()
         s = a.split("\n$")
         self.copyright = s.pop(0).strip()
         while s:

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:11 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:11 +0000 (UTC)
Subject: [6.0] 32a88b672 Improve long desc a little bit.
Message-ID: <20181031130811.C8C2C94FF1@lists.varnish-cache.org>


commit 32a88b67284e69f7817b3e96bfa36213a7b7b8fb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 3 11:32:23 2018 +0000

    Improve long desc a little bit.

diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc
index 4749eb5c2..56ad787b2 100644
--- a/bin/varnishd/VSC_main.vsc
+++ b/bin/varnishd/VSC_main.vsc
@@ -11,7 +11,7 @@
 	:level:	debug
 	:oneliner:	stat summ operations
 
-	Number of times per-thread statistics were added to the
+	Number of times per-thread statistics were summed into the
 	global counters.
 
 .. varnish_vsc:: uptime

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:11 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:11 +0000 (UTC)
Subject: [6.0] 94de5c6d3 vtc syntax cleanup
Message-ID: <20181031130811.E865C94FFF@lists.varnish-cache.org>


commit 94de5c6d3c1c84851011b5def4d34202d9262e07
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Sep 3 15:04:35 2018 +0200

    vtc syntax cleanup
    
    return(pass) from vcl_recv does not take a duration argument

diff --git a/bin/varnishtest/tests/c00010.vtc b/bin/varnishtest/tests/c00010.vtc
index a128b73ca..5e8b087da 100644
--- a/bin/varnishtest/tests/c00010.vtc
+++ b/bin/varnishtest/tests/c00010.vtc
@@ -17,7 +17,7 @@ varnish v1 -vcl+backend {
 		if (req.url == "/foo") {
 			return(hash);
 		} else {
-			return(pass(10m));
+			return(pass);
 		}
 	}
 	sub vcl_hit {

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:12 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:12 +0000 (UTC)
Subject: [6.0] 3f3e220ca fix logexpect syntax in some vtcs
Message-ID: <20181031130812.2289395027@lists.varnish-cache.org>


commit 3f3e220ca36e67d8523551fcbba824e2e3db0a65
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Sep 3 15:55:38 2018 +0200

    fix logexpect syntax in some vtcs
    
    The syntax is
    
            expect <skip> <vxid> <tag> <regex>
    
    Also remove some bogus .*$ from regexen (waving to @slimhazard)

diff --git a/bin/varnishtest/tests/c00011.vtc b/bin/varnishtest/tests/c00011.vtc
index 24622675f..907355f2f 100644
--- a/bin/varnishtest/tests/c00011.vtc
+++ b/bin/varnishtest/tests/c00011.vtc
@@ -23,7 +23,7 @@ varnish v1 -vcl+backend {
 } -start
 
 logexpect l1 -v v1 -g vxid {
-	expect 1003 *	HitMiss "^1002 119.*$"
+	expect * 1003	HitMiss "^1002 119"
 } -start
 
 client c1 {
diff --git a/bin/varnishtest/tests/c00081.vtc b/bin/varnishtest/tests/c00081.vtc
index 5b7c55166..b08fa3250 100644
--- a/bin/varnishtest/tests/c00081.vtc
+++ b/bin/varnishtest/tests/c00081.vtc
@@ -30,7 +30,7 @@ varnish v1 -vcl+backend {
 } -start
 
 logexpect l1 -v v1 -g vxid {
-	expect 1003 *	HitPass "^1002 1.*$"
+	expect * 1003	HitPass "^1002 1"
 } -start
 
 client c1 {
diff --git a/bin/varnishtest/tests/r01858.vtc b/bin/varnishtest/tests/r01858.vtc
index fe1cbe1de..1fb787f76 100644
--- a/bin/varnishtest/tests/r01858.vtc
+++ b/bin/varnishtest/tests/r01858.vtc
@@ -24,7 +24,7 @@ varnish v1 -vcl+backend {
 
 # Tests logging hit-for-miss on an expired object
 logexpect l1 -v v1 -g vxid {
-	expect 1003 *	HitMiss "^1002 -.*$"
+	expect * 1003	HitMiss "^1002 -"
 } -start
 
 client c1 {

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:12 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:12 +0000 (UTC)
Subject: [6.0] 11905e2b3 fix comment
Message-ID: <20181031130812.4517C95034@lists.varnish-cache.org>


commit 11905e2b336e4a78b0e8cdc9d57ff37b27ac4e17
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Sep 3 18:05:09 2018 +0200

    fix comment

diff --git a/include/tbl/boc_state.h b/include/tbl/boc_state.h
index 5297d8af5..a5c4a61ff 100644
--- a/include/tbl/boc_state.h
+++ b/include/tbl/boc_state.h
@@ -29,7 +29,7 @@
 /*lint -save -e525 -e539 */
 
 BOC_STATE(INVALID,	invalid)	/* don't touch (yet) */
-BOC_STATE(REQ_DONE,	req_done)	/* beresp.* can be examined */
+BOC_STATE(REQ_DONE,	req_done)	/* bereq.* can be examined */
 BOC_STATE(PREP_STREAM,	prep_stream)	/* Prepare for streaming */
 BOC_STATE(STREAM,	stream)		/* beresp.* can be examined */
 BOC_STATE(FINISHED,	finished)	/* object is complete */

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:12 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:12 +0000 (UTC)
Subject: [6.0] 29f5ad61d side effect of staring at code not worth a ticket
Message-ID: <20181031130812.6383995044@lists.varnish-cache.org>


commit 29f5ad61db0517066d8a946ac1aceaf083c2887f
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Sep 3 18:11:42 2018 +0200

    side effect of staring at code not worth a ticket

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index c3b652bbe..1b63d906d 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -407,6 +407,8 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 			continue;
 
 		if (oc->boc != NULL && oc->boc->state < BOS_STREAM) {
+			// XXX does it make sense to have the CHECK_OBJ
+			//     if we access boc->state before it?
 			CHECK_OBJ_ORNULL(oc->boc, BOC_MAGIC);
 
 			if (req->hash_ignore_busy)

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:12 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:12 +0000 (UTC)
Subject: [6.0] fd87ec617 Return the old hit-for-miss object so we can expire
 it.
Message-ID: <20181031130812.8071695057@lists.varnish-cache.org>


commit fd87ec61733597a492e2ac3e41a78b8e2ab86589
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 4 06:26:24 2018 +0000

    Return the old hit-for-miss object so we can expire it.
    
    Partial fix for #2654 and #2754

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 1b63d906d..a87462646 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -447,23 +447,25 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 				VSLb(req->vsl, SLT_HitPass, "%u %.6f",
 				    ObjGetXID(wrk, oc), EXP_Dttl(req, oc));
 				oc = NULL;
+				retval = HSH_MISS;
 			} else if (oc->flags & OC_F_PASS) {
 				wrk->stats->cache_hitmiss++;
 				VSLb(req->vsl, SLT_HitMiss, "%u %.6f",
 				    ObjGetXID(wrk, oc), EXP_Dttl(req, oc));
-				oc = NULL;
 				*bocp = hsh_insert_busyobj(wrk, oh);
+				oc->refcnt++;
+				retval = HSH_MISS;
 			} else {
 				oc->refcnt++;
 				if (oc->hits < LONG_MAX)
 					oc->hits++;
+				retval = HSH_HIT;
 			}
 			Lck_Unlock(&oh->mtx);
-			if (oc == NULL)
-				return (HSH_MISS);
-			assert(HSH_DerefObjHead(wrk, &oh));
 			*ocp = oc;
-			return (HSH_HIT);
+			if (retval == HSH_HIT)
+				assert(HSH_DerefObjHead(wrk, &oh));
+			return (retval);
 		}
 		if (EXP_Ttl(NULL, oc) < req->t_req && /* ignore req.ttl */
 		    oc->t_origin > exp_t_origin) {

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:12 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:12 +0000 (UTC)
Subject: [6.0] 5c8880a22 Move boc check before boc->state is dereferenced
Message-ID: <20181031130812.9DE1B95066@lists.varnish-cache.org>


commit 5c8880a22b62d76c338e6ca270eaeff0dfa51478
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Sep 4 08:44:52 2018 +0200

    Move boc check before boc->state is dereferenced
    
    Refs 9804794a0e876800e4e3ca10e965b568bd37c38e

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index a87462646..182a7a97d 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -406,11 +406,8 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 		if (oc->flags & OC_F_FAILED)
 			continue;
 
+		CHECK_OBJ_ORNULL(oc->boc, BOC_MAGIC);
 		if (oc->boc != NULL && oc->boc->state < BOS_STREAM) {
-			// XXX does it make sense to have the CHECK_OBJ
-			//     if we access boc->state before it?
-			CHECK_OBJ_ORNULL(oc->boc, BOC_MAGIC);
-
 			if (req->hash_ignore_busy)
 				continue;
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:12 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:12 +0000 (UTC)
Subject: [6.0] 0e9b5ec63 Test more of varnishadms error handling
Message-ID: <20181031130812.CABBD95082@lists.varnish-cache.org>


commit 0e9b5ec6395ff68e3db8e0f912ab7a1fa144377a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 4 07:38:27 2018 +0000

    Test more of varnishadms error handling

diff --git a/bin/varnishtest/tests/b00041.vtc b/bin/varnishtest/tests/b00041.vtc
index 2fd2690a4..ca66c4539 100644
--- a/bin/varnishtest/tests/b00041.vtc
+++ b/bin/varnishtest/tests/b00041.vtc
@@ -2,4 +2,94 @@ varnishtest "Test varnishadm and the Telnet CLI"
 
 varnish v1 -vcl {backend foo { .host = "127.0.0.1"; } } -start
 
+shell -err -expect {Usage: varnishadm} \
+	"varnishadm -7"
+
+shell -err -expect {Could not get hold of varnishd, is it running?} \
+	"varnishadm -n ${v1_name}/nonexistent"
+
+shell -err -expect {Connection failed} \
+	"varnishadm -t 4 -T ${bad_ip}:1 -S ${v1_name}/_.secret"
+
+server s1 {
+	send "FOO\n"
+} -start
+
+shell -err -expect {Rejected 400} \
+	{varnishadm -T ${s1_addr}:${s1_port} -S /etc/group}
+
+server s1 {
+	send "107 59      \n"
+	send "qbvnnftpkgubadqpzznkkazoxlyqbcbj\n\n"
+	send "Authentication required.\n"
+	send "\n"
+} -start
+
+shell -err -expect {Authentication required} \
+	{varnishadm -T ${s1_addr}:${s1_port}}
+
+server s1 {
+	send "107 59      \n"
+	send "qbvnnftpkgubadqpzznkkazoxlyqbcbj\n\n"
+	send "Authentication required.\n"
+	send "\n"
+} -start
+
+shell -err -expect {Cannot open } \
+	{varnishadm -T ${s1_addr}:${s1_port} -S ${v1_name}/_.nonexistent}
+
+server s1 {
+	send "107 59      \n"
+	send "qbvnnftpkgubadqpzznkkazoxlyqbcbj\n\n"
+	send "Authentication required.\n"
+	send "\n"
+
+	recv 70
+	send "599 0       \n"
+	send "\n"
+} -start
+
+shell -err -expect {Rejected 599} \
+	{varnishadm -T ${s1_addr}:${s1_port} -S ${v1_name}/_.secret}
+
+server s1 {
+	send "107 59      \n"
+	send "qbvnnftpkgubadqpzznkkazoxlyqbcbj\n\n"
+	send "Authentication required.\n"
+	send "\n"
+
+	recv 70
+	send "200 0       \n"
+	send "\n"
+
+	recv 5
+} -start
+
+shell -err -expect {No pong received from server} \
+	{varnishadm -T ${s1_addr}:${s1_port} -S ${v1_name}/_.secret}
+
+server s1 {
+	send "107 59      \n"
+	send "qbvnnftpkgubadqpzznkkazoxlyqbcbj\n\n"
+	send "Authentication required.\n"
+	send "\n"
+
+	recv 70
+	send "200 0       \n"
+	send "\n"
+
+	recv 5
+	send "200 8       \n"
+	send "PONG 12\n"
+	send "\n"
+
+	recv 5
+	send "200 7       \n"
+	send "Tested\n"
+	send "\n"
+} -start
+
+shell -expect {Tested} \
+	{varnishadm -T ${s1_addr}:${s1_port} -S ${v1_name}/_.secret test}
+
 shell "varnishadm -n ${v1_name} help > /dev/null"
diff --git a/bin/varnishtest/tests/c00081.vtc b/bin/varnishtest/tests/c00081.vtc
index b08fa3250..419036cfe 100644
--- a/bin/varnishtest/tests/c00081.vtc
+++ b/bin/varnishtest/tests/c00081.vtc
@@ -4,7 +4,11 @@ server s1 {
 	rxreq
 	txresp -hdr "foo: 1"
 	rxreq
-	txresp -hdr "foo: 2"
+	txresp -hdr "foo: 2a"
+	rxreq
+	txresp -hdr "foo: 2b"
+	rxreq
+	txresp -hdr "foo: 2c"
 	rxreq
 	txresp -hdr "foo: 3"
 } -start
@@ -29,8 +33,31 @@ varnish v1 -vcl+backend {
 
 } -start
 
+varnish v1 -cliok "param.set debug +refcnt"
+
 logexpect l1 -v v1 -g vxid {
-	expect * 1003	HitPass "^1002 1"
+	expect * 1001	Debug		"^oh.*refcnt 1$"
+	expect 0 =	VCL_call	"^MISS"
+} -start
+
+logexpect l2 -v v1 -g vxid {
+	expect * 1003	Debug		"^oh.*refcnt 2$"
+	expect 0 =	HitPass	"^1002 1"
+} -start
+
+logexpect l3 -v v1 -g vxid {
+	expect * 1005	Debug		"^oh.*refcnt 2$"
+	expect 0 =	HitPass	"^1002 1"
+} -start
+
+logexpect l4 -v v1 -g vxid {
+	expect * 1007	Debug		"^oh.*refcnt 2$"
+	expect 0 =	HitPass	"^1002 1"
+} -start
+
+logexpect l5 -v v1 -g vxid {
+	expect * 1009	Debug		"^oh.*refcnt 1$"
+	expect 0 =	VCL_call	"^MISS"
 } -start
 
 client c1 {
@@ -38,6 +65,12 @@ client c1 {
 	rxresp
 	expect resp.http.miss == True
 
+	txreq
+	rxresp
+	expect resp.http.pass == True
+	txreq
+	rxresp
+	expect resp.http.pass == True
 	txreq
 	rxresp
 	expect resp.http.pass == True
@@ -50,7 +83,11 @@ client c1 {
 } -run
 
 logexpect l1 -wait
+logexpect l2 -wait
+logexpect l3 -wait
+logexpect l4 -wait
+logexpect l5 -wait
 
-varnish v1 -expect MAIN.cache_hitpass == 1
+varnish v1 -expect MAIN.cache_hitpass == 3
 varnish v1 -expect MAIN.cache_miss == 2
 varnish v1 -expect MAIN.cache_hitmiss == 0

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:12 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:12 +0000 (UTC)
Subject: [6.0] b875f4d68 Document -need-bytes and implement an optional '+'
 prefix to require new bytes to be received.
Message-ID: <20181031130812.E582695091@lists.varnish-cache.org>


commit b875f4d68e52e372eb7689d1d3134573a7db5437
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 4 08:01:59 2018 +0000

    Document -need-bytes and implement an optional '+' prefix to
    require new bytes to be received.

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index 95f6c0517..5f5cd2a86 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -894,6 +894,10 @@ process_close(struct process *p)
  * \-writehex HEXSTRING
  *	Same as -write but interpreted as hexadecimal bytes.
  *
+ * \-need-bytes [+]NUMBER
+ *	Wait until at least NUMBER bytes have been received in total.
+ *	If '+' is prefixed, NUMBER new bytes must be received.
+ *
  * \-expect-text LIN COL PAT
  *	Wait for PAT to appear at LIN,COL on the virtual screen.
  *	Lines and columns are numbered 1...N
@@ -998,6 +1002,8 @@ cmd_process(CMD_ARGS)
 		}
 		if (!strcmp(*av, "-need-bytes")) {
 			u = strtoumax(av[1], NULL, 0);
+			if (av[1][0] == '+')
+				u += p->stdout_bytes;
 			av++;
 			do {
 				AZ(pthread_mutex_lock(&p->mtx));

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:13 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:13 +0000 (UTC)
Subject: [6.0] 010a7ebc9 Oops, accidentally commited test-patch from #2760
Message-ID: <20181031130813.0C9FC9509B@lists.varnish-cache.org>


commit 010a7ebc9b75fe0d0b84521e009bcec20ef47f66
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 4 08:04:11 2018 +0000

    Oops, accidentally commited test-patch from #2760

diff --git a/bin/varnishtest/tests/c00081.vtc b/bin/varnishtest/tests/c00081.vtc
index 419036cfe..b08fa3250 100644
--- a/bin/varnishtest/tests/c00081.vtc
+++ b/bin/varnishtest/tests/c00081.vtc
@@ -4,11 +4,7 @@ server s1 {
 	rxreq
 	txresp -hdr "foo: 1"
 	rxreq
-	txresp -hdr "foo: 2a"
-	rxreq
-	txresp -hdr "foo: 2b"
-	rxreq
-	txresp -hdr "foo: 2c"
+	txresp -hdr "foo: 2"
 	rxreq
 	txresp -hdr "foo: 3"
 } -start
@@ -33,31 +29,8 @@ varnish v1 -vcl+backend {
 
 } -start
 
-varnish v1 -cliok "param.set debug +refcnt"
-
 logexpect l1 -v v1 -g vxid {
-	expect * 1001	Debug		"^oh.*refcnt 1$"
-	expect 0 =	VCL_call	"^MISS"
-} -start
-
-logexpect l2 -v v1 -g vxid {
-	expect * 1003	Debug		"^oh.*refcnt 2$"
-	expect 0 =	HitPass	"^1002 1"
-} -start
-
-logexpect l3 -v v1 -g vxid {
-	expect * 1005	Debug		"^oh.*refcnt 2$"
-	expect 0 =	HitPass	"^1002 1"
-} -start
-
-logexpect l4 -v v1 -g vxid {
-	expect * 1007	Debug		"^oh.*refcnt 2$"
-	expect 0 =	HitPass	"^1002 1"
-} -start
-
-logexpect l5 -v v1 -g vxid {
-	expect * 1009	Debug		"^oh.*refcnt 1$"
-	expect 0 =	VCL_call	"^MISS"
+	expect * 1003	HitPass "^1002 1"
 } -start
 
 client c1 {
@@ -65,12 +38,6 @@ client c1 {
 	rxresp
 	expect resp.http.miss == True
 
-	txreq
-	rxresp
-	expect resp.http.pass == True
-	txreq
-	rxresp
-	expect resp.http.pass == True
 	txreq
 	rxresp
 	expect resp.http.pass == True
@@ -83,11 +50,7 @@ client c1 {
 } -run
 
 logexpect l1 -wait
-logexpect l2 -wait
-logexpect l3 -wait
-logexpect l4 -wait
-logexpect l5 -wait
 
-varnish v1 -expect MAIN.cache_hitpass == 3
+varnish v1 -expect MAIN.cache_hitpass == 1
 varnish v1 -expect MAIN.cache_miss == 2
 varnish v1 -expect MAIN.cache_hitmiss == 0

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:13 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:13 +0000 (UTC)
Subject: [6.0] 5711cd1e4 Test more of varnishtop
Message-ID: <20181031130813.2E7E1950AD@lists.varnish-cache.org>


commit 5711cd1e470c6f1f8b5200b2171108dbd8ab0c45
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 4 08:05:30 2018 +0000

    Test more of varnishtop

diff --git a/bin/varnishtest/tests/u00004.vtc b/bin/varnishtest/tests/u00004.vtc
index 5dfaa3310..9d2be8ed4 100644
--- a/bin/varnishtest/tests/u00004.vtc
+++ b/bin/varnishtest/tests/u00004.vtc
@@ -31,7 +31,12 @@ shell -match "1\\.00 RespHeader Date\\n" {
 
 shell -match "Usage: .*varnishtop <options>" \
 	"varnishtop -h"
+
 shell -expect "Copyright (c) 2006 Verdens Gang AS" \
 	"varnishtop -V"
+
 shell -err -match "Usage: .*varnishtop <options>" \
 	"varnishtop extra"
+
+shell -err -match "is not a number" \
+	"varnishtop -p ABC"
diff --git a/bin/varnishtest/tests/u00010.vtc b/bin/varnishtest/tests/u00010.vtc
index 5813db981..dd37a15d0 100644
--- a/bin/varnishtest/tests/u00010.vtc
+++ b/bin/varnishtest/tests/u00010.vtc
@@ -18,5 +18,20 @@ client c1 {
 	rxresp
 } -run
 
-process p1 -expect-text 0 0 {Fetch_Body}
+varnish v1 -vsl_catchup
+
+process p1 -expect-text 1 1 {list length 64}
+
+process p1 -writehex 0c
+
+process p1 -need-bytes +1
+
+process p1 -writehex 0b
+
+process p1 -need-bytes +1
+
+process p1 -winsz 30 80
+
+process p1 -need-bytes +1
+
 process p1 -screen_dump -write {q} -wait

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:13 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:13 +0000 (UTC)
Subject: [6.0] 14c846fcf Test strtoul() result comprehensively.
Message-ID: <20181031130813.4AEE4950B8@lists.varnish-cache.org>


commit 14c846fcfef111c3ebc16827ac7a44700fefe2e3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 4 08:49:16 2018 +0000

    Test strtoul() result comprehensively.

diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 0550f8abc..2247cca0f 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -343,6 +343,7 @@ main(int argc, char **argv)
 {
 	int o, once = 0;
 	pthread_t thr;
+	char *e = NULL;
 
 	vut = VUT_InitProg(argc, argv, &vopt_spec);
 	AN(vut);
@@ -361,8 +362,9 @@ main(int argc, char **argv)
 			usage(0);
 		case 'p':
 			errno = 0;
-			period = strtol(optarg, NULL, 0);
-			if (errno != 0)  {
+			e = NULL;
+			period = strtoul(optarg, &e, 0);
+			if (errno != 0 || e == NULL || *e != '\0') {
 				fprintf(stderr,
 				    "Syntax error, %s is not a number", optarg);
 				exit(1);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:13 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:13 +0000 (UTC)
Subject: [6.0] 09edd1e72 Tweak the -p test slightly
Message-ID: <20181031130813.6CF9D950CC@lists.varnish-cache.org>


commit 09edd1e7232e3b16b2ed238983261731ab62b252
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 4 08:50:37 2018 +0000

    Tweak the -p test slightly

diff --git a/bin/varnishtest/tests/u00004.vtc b/bin/varnishtest/tests/u00004.vtc
index 9d2be8ed4..b6015a061 100644
--- a/bin/varnishtest/tests/u00004.vtc
+++ b/bin/varnishtest/tests/u00004.vtc
@@ -39,4 +39,4 @@ shell -err -match "Usage: .*varnishtop <options>" \
 	"varnishtop extra"
 
 shell -err -match "is not a number" \
-	"varnishtop -p ABC"
+	"varnishtop -p 12ABC"

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:13 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:13 +0000 (UTC)
Subject: [6.0] 70a444b91 Deref the objhead if we don't return a busy object.
Message-ID: <20181031130813.91443950E2@lists.varnish-cache.org>


commit 70a444b9116717f2821a421800fc7182d14cdce9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 4 10:15:52 2018 +0000

    Deref the objhead if we don't return a busy object.
    
    Most of the heavy lifting by:   slink
    
    Fixes: #2760, #2754, #2654

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 182a7a97d..bb32d4fc2 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -460,7 +460,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 			}
 			Lck_Unlock(&oh->mtx);
 			*ocp = oc;
-			if (retval == HSH_HIT)
+			if (*bocp == NULL)
 				assert(HSH_DerefObjHead(wrk, &oh));
 			return (retval);
 		}

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:13 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:13 +0000 (UTC)
Subject: [6.0] e44526eed Polish
Message-ID: <20181031130813.B12E3950FA@lists.varnish-cache.org>


commit e44526eedcaded131a055dacac1023ba32674908
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 4 15:19:29 2018 +0000

    Polish

diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 2247cca0f..78246d6fa 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -302,8 +302,8 @@ do_curses(void *arg)
 		case 'Q':
 		case 'q':
 			AZ(raise(SIGINT));
-			AC(endwin());
-			return (NULL);
+			quit = 1;
+			break;
 		default:
 			AC(beep());
 			break;
@@ -389,18 +389,14 @@ main(int argc, char **argv)
 	vut->dispatch_priv = NULL;
 	vut->sighup_f = sighup;
 	if (once) {
-		VUT_Main(vut);
+		(void)VUT_Main(vut);
 		dump();
 	} else {
-		if (pthread_create(&thr, NULL, do_curses, NULL) != 0) {
-			fprintf(stderr, "pthread_create(): %s\n",
-			    strerror(errno));
-			exit(1);
-		}
-		VUT_Main(vut);
+		AZ(pthread_create(&thr, NULL, do_curses, NULL));
+		(void)VUT_Main(vut);
 		end_of_file = 1;
 		AZ(pthread_join(thr, NULL));
 	}
 	VUT_Fini(&vut);
-	exit(0);
+	return (0);
 }

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:13 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:13 +0000 (UTC)
Subject: [6.0] bb7a4d159 cleanup
Message-ID: <20181031130813.CB23B9510A@lists.varnish-cache.org>


commit bb7a4d159d1870e5b7ac212b0289a15ca9190ceb
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Sep 4 19:55:34 2018 +0200

    cleanup

diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c
index b85aa43b8..4ea5e5ae7 100644
--- a/lib/libvmod_directors/vmod_shard.c
+++ b/lib/libvmod_directors/vmod_shard.c
@@ -850,11 +850,6 @@ shard_param_task(VRT_CTX, const void *id,
 	if (task->priv) {
 		CAST_OBJ_NOTNULL(p, task->priv, VMOD_SHARD_SHARD_PARAM_MAGIC);
 		assert(p->scope == SCOPE_TASK);
-		/* XXX
-		VSL(SLT_Debug, 0,
-		    "shard_param_task(id %p, pa %p) = %p (found, ws=%p)",
-		    id, pa, p, ctx->ws);
-		*/
 		return (p);
 	}
 
@@ -873,11 +868,6 @@ shard_param_task(VRT_CTX, const void *id,
 	else
 		p->defaults = shard_param_task(ctx, pa, pa);
 
-	/* XXX
-	VSL(SLT_Debug, 0,
-	    "shard_param_task(id %p, pa %p) = %p (new, defaults = %p, ws=%p)",
-	    id, pa, p, p->defaults, ctx->ws);
-	*/
 	return (p);
 }
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:13 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:13 +0000 (UTC)
Subject: [6.0] 54e938420 Test a few lines more.
Message-ID: <20181031130813.E9C3F95115@lists.varnish-cache.org>


commit 54e938420817bd9d3188c7720ad4ec541f030002
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 4 18:40:06 2018 +0000

    Test a few lines more.

diff --git a/bin/varnishtest/tests/u00004.vtc b/bin/varnishtest/tests/u00004.vtc
index b6015a061..5d95d7611 100644
--- a/bin/varnishtest/tests/u00004.vtc
+++ b/bin/varnishtest/tests/u00004.vtc
@@ -32,6 +32,9 @@ shell -match "1\\.00 RespHeader Date\\n" {
 shell -match "Usage: .*varnishtop <options>" \
 	"varnishtop -h"
 
+shell -err -match "illegal option -- K" \
+	"varnishtop -K"
+
 shell -expect "Copyright (c) 2006 Verdens Gang AS" \
 	"varnishtop -V"
 
@@ -39,4 +42,4 @@ shell -err -match "Usage: .*varnishtop <options>" \
 	"varnishtop extra"
 
 shell -err -match "is not a number" \
-	"varnishtop -p 12ABC"
+	"varnishtop -p 60 -p 12ABC"

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:14 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:14 +0000 (UTC)
Subject: [6.0] 28b6f334c Rely only on error messages we format ourselves.
Message-ID: <20181031130814.0EC0E9511E@lists.varnish-cache.org>


commit 28b6f334ce9cddd27681e12c48a1ddf195fde2a3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 4 19:36:38 2018 +0000

    Rely only on error messages we format ourselves.

diff --git a/bin/varnishtest/tests/u00004.vtc b/bin/varnishtest/tests/u00004.vtc
index 5d95d7611..a861ae57a 100644
--- a/bin/varnishtest/tests/u00004.vtc
+++ b/bin/varnishtest/tests/u00004.vtc
@@ -32,7 +32,7 @@ shell -match "1\\.00 RespHeader Date\\n" {
 shell -match "Usage: .*varnishtop <options>" \
 	"varnishtop -h"
 
-shell -err -match "illegal option -- K" \
+shell -err -match "Usage: .*varnishtop <options>" \
 	"varnishtop -K"
 
 shell -expect "Copyright (c) 2006 Verdens Gang AS" \

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:14 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:14 +0000 (UTC)
Subject: [6.0] f5b1b9d3c Sanitize H2_Send_Frame error handling.
Message-ID: <20181031130814.2BAFB95130@lists.varnish-cache.org>


commit f5b1b9d3c097bb9bef4d128d42e02ba31d0b9004
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 5 06:40:01 2018 +0000

    Sanitize H2_Send_Frame error handling.
    
    The only way H2_Send_Frame can fail is if the TCP connection is
    dead, so reach out and set the session errored and be done with it,
    rather than pass an inconvenient error status down through the callers.
    
    While here use writev(2) instead of two write(2) calls.

diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h
index 1c28f99ee..70e31d14e 100644
--- a/bin/varnishd/http2/cache_http2.h
+++ b/bin/varnishd/http2/cache_http2.h
@@ -217,7 +217,7 @@ h2_error h2h_decode_bytes(struct h2_sess *h2, const uint8_t *ptr,
 void H2_Send_Get(struct worker *, struct h2_sess *, struct h2_req *);
 void H2_Send_Rel(struct h2_sess *, const struct h2_req *);
 
-h2_error H2_Send_Frame(struct worker *, const struct h2_sess *,
+void H2_Send_Frame(struct worker *, struct h2_sess *,
     h2_frame type, uint8_t flags, uint32_t len, uint32_t stream,
     const void *);
 
diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index f6ea07eae..56aa27552 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -132,11 +132,10 @@ h2_connectionerror(uint32_t u)
 
 /**********************************************************************/
 
-static h2_error
+static void
 h2_tx_rst(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2,
     uint32_t stream, h2_error h2e)
 {
-	h2_error ret;
 	char b[4];
 
 	CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC);
@@ -148,11 +147,8 @@ h2_tx_rst(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2,
 	vbe32enc(b, h2e->val);
 
 	H2_Send_Get(wrk, h2, r2);
-	ret = H2_Send_Frame(wrk, h2, H2_F_RST_STREAM,
-	    0, sizeof b, stream, b);
+	H2_Send_Frame(wrk, h2, H2_F_RST_STREAM, 0, sizeof b, stream, b);
 	H2_Send_Rel(h2, r2);
-
-	return (ret);
 }
 
 /**********************************************************************
@@ -936,8 +932,9 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2,
 			     "H2: stream %u: Hit maximum number of "
 			     "concurrent streams", h2->rxf_stream);
 			// rfc7540,l,1200,1205
-			return (h2_tx_rst(wrk, h2, h2->req0, h2->rxf_stream,
-				H2SE_REFUSED_STREAM));
+			h2_tx_rst(wrk, h2, h2->req0, h2->rxf_stream,
+				H2SE_REFUSED_STREAM);
+			return (0);
 		}
 		h2->highest_stream = h2->rxf_stream;
 		r2 = h2_new_req(wrk, h2, h2->rxf_stream, NULL);
@@ -954,7 +951,8 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2,
 	if (h2->rxf_stream == 0 || h2e->connection)
 		return (h2e);	// Connection errors one level up
 
-	return (h2_tx_rst(wrk, h2, h2->req0, h2->rxf_stream, h2e));
+	h2_tx_rst(wrk, h2, h2->req0, h2->rxf_stream, h2e);
+	return (0);
 }
 
 static int
@@ -1143,8 +1141,8 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2)
 		vbe32enc(b, h2->highest_stream);
 		vbe32enc(b + 4, h2e->val);
 		H2_Send_Get(wrk, h2, h2->req0);
-		(void)H2_Send_Frame(wrk, h2, H2_F_GOAWAY, 0, 8, 0, b);
+		H2_Send_Frame(wrk, h2, H2_F_GOAWAY, 0, 8, 0, b);
 		H2_Send_Rel(h2, h2->req0);
 	}
-	return (h2e ? 0 : 1);
+	return (h2->error ? 0 : 1);
 }
diff --git a/bin/varnishd/http2/cache_http2_send.c b/bin/varnishd/http2/cache_http2_send.c
index 054796e4b..41ca146b6 100644
--- a/bin/varnishd/http2/cache_http2_send.c
+++ b/bin/varnishd/http2/cache_http2_send.c
@@ -29,6 +29,8 @@
 
 #include "config.h"
 
+#include <sys/uio.h>
+
 #include "cache/cache_varnishd.h"
 
 #include "cache/cache_transport.h"
@@ -112,13 +114,14 @@ h2_mk_hdr(uint8_t *hdr, h2_frame ftyp, uint8_t flags,
  * the session mtx must be held.
  */
 
-h2_error
-H2_Send_Frame(struct worker *wrk, const struct h2_sess *h2,
+void
+H2_Send_Frame(struct worker *wrk, struct h2_sess *h2,
     h2_frame ftyp, uint8_t flags,
     uint32_t len, uint32_t stream, const void *ptr)
 {
 	uint8_t hdr[9];
 	ssize_t s;
+	struct iovec iov[2];
 
 	(void)wrk;
 
@@ -137,18 +140,24 @@ H2_Send_Frame(struct worker *wrk, const struct h2_sess *h2,
 		h2->srq->acct.resp_bodybytes += len;
 	Lck_Unlock(&h2->sess->mtx);
 
-	s = write(h2->sess->fd, hdr, sizeof hdr);
-	if (s != sizeof hdr)
-		return (H2CE_PROTOCOL_ERROR);		// XXX Need private ?
-	if (len > 0) {
-		s = write(h2->sess->fd, ptr, len);
-		if (s != len)
-			return (H2CE_PROTOCOL_ERROR);	// XXX Need private ?
+	memset(iov, 0, sizeof iov);
+	iov[0].iov_base = hdr;
+	iov[0].iov_len = sizeof hdr;
+	iov[1].iov_base = TRUST_ME(ptr);
+	iov[1].iov_len = len;
+	s = writev(h2->sess->fd, iov, len == 0 ? 1 : 2);
+	if (s != sizeof hdr + len) {
+		/*
+		 * There is no point in being nice here, we will be unable
+		 * to send a GOAWAY once the code unrolls, so go directly
+		 * to the finale and be done with it.
+		 */
+		h2->error = H2CE_PROTOCOL_ERROR;
+	} else if (len > 0) {
 		Lck_Lock(&h2->sess->mtx);
 		VSLb_bin(h2->vsl, SLT_H2TxBody, len, ptr);
 		Lck_Unlock(&h2->sess->mtx);
 	}
-	return (0);
 }
 
 static int64_t
@@ -251,7 +260,6 @@ void
 H2_Send(struct worker *wrk, struct h2_req *r2,
     h2_frame ftyp, uint8_t flags, uint32_t len, const void *ptr)
 {
-	h2_error retval;
 	struct h2_sess *h2;
 	uint32_t mfs, tf;
 	const char *p;
@@ -289,8 +297,7 @@ H2_Send(struct worker *wrk, struct h2_req *r2,
 		tf = mfs;
 
 	if (len <= tf) {
-		(void)H2_Send_Frame(wrk, h2,
-		    ftyp, flags, len, r2->stream, ptr);
+		H2_Send_Frame(wrk, h2, ftyp, flags, len, r2->stream, ptr);
 	} else {
 		AN(ptr);
 		p = ptr;
@@ -308,19 +315,19 @@ H2_Send(struct worker *wrk, struct h2_req *r2,
 				assert(VTAILQ_FIRST(&h2->txqueue) == r2);
 			}
 			if (tf < len) {
-				retval = H2_Send_Frame(wrk, h2, ftyp,
+				H2_Send_Frame(wrk, h2, ftyp,
 				    flags, tf, r2->stream, p);
 			} else {
 				if (ftyp->respect_window)
 					assert(tf == len);
 				tf = len;
-				retval = H2_Send_Frame(wrk, h2, ftyp,
+				H2_Send_Frame(wrk, h2, ftyp,
 				    final_flags, tf, r2->stream, p);
 				flags = 0;
 			}
 			p += tf;
 			len -= tf;
 			ftyp = ftyp->continuation;
-		} while (len > 0 && retval == 0);
+		} while (!h2->error && len > 0);
 	}
 }

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:14 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:14 +0000 (UTC)
Subject: [6.0] 2a19cfb3c Flexelinting
Message-ID: <20181031130814.6BA2F95145@lists.varnish-cache.org>


commit 2a19cfb3cc4a0605597d075b920015c8d1db3027
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 5 07:26:15 2018 +0000

    Flexelinting

diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 9ae48331b..e4db8158f 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -260,7 +260,7 @@ VSL_Flush(struct vsl_log *vsl, int overflow)
 	memcpy(p + 2, vsl->wlb, l);
 	p[1] = l;
 	VWMB();
-	p[0] = ((((unsigned)SLT__Batch & 0xff) << 24) | 0);
+	p[0] = ((((unsigned)SLT__Batch & 0xff) << 24));
 	vsl->wlp = vsl->wlb;
 	vsl->wlr = 0;
 }
diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c
index 379f7b36f..46d6930fb 100644
--- a/bin/varnishd/cache/cache_vary.c
+++ b/bin/varnishd/cache/cache_vary.c
@@ -335,7 +335,7 @@ VRY_Match(struct req *req, const uint8_t *vary)
 			memcpy(vsp + 2, vary + 2, vary[2] + 2);
 			if (h != NULL)
 				memcpy(vsp + 2 + vsp[2] + 2, h, lh);
-			vsp[ln + 0] = 0xff;
+			vsp[ln] = 0xff;
 			vsp[ln + 1] = 0xff;
 			vsp[ln + 2] = 0;
 			(void)VRY_Validate(vsp);
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 748fe0a64..405bb4310 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -630,6 +630,8 @@ VRT_r_##which##_##fld(VRT_CTX)					\
 	return (d);						\
 }
 
+/*lint -save -e835 */	// Zero right hand arg to '-'
+
 VRT_DO_EXP_R(obj, ctx->req->objcore, ttl,
     ttl_now(ctx) - ctx->req->objcore->t_origin)
 VRT_DO_EXP_R(obj, ctx->req->objcore, grace, 0)
@@ -644,6 +646,8 @@ VRT_DO_EXP_R(beresp, ctx->bo->fetch_objcore, grace, 0)
 VRT_DO_EXP_L(beresp, ctx->bo->fetch_objcore, keep, 0)
 VRT_DO_EXP_R(beresp, ctx->bo->fetch_objcore, keep, 0)
 
+/*lint -restore */
+
 /*--------------------------------------------------------------------
  */
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:14 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:14 +0000 (UTC)
Subject: [6.0] 155a65224 Fail object delivery little earlier if the session-fd
 is dead.
Message-ID: <20181031130814.928CA95156@lists.varnish-cache.org>


commit 155a652244e9957be18535b60957ad32871f5ad9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 5 07:28:13 2018 +0000

    Fail object delivery little earlier if the session-fd is dead.

diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c
index aea59e9a8..912eb88bc 100644
--- a/bin/varnishd/http2/cache_http2_deliver.c
+++ b/bin/varnishd/http2/cache_http2_deliver.c
@@ -84,7 +84,7 @@ h2_bytes(struct req *req, enum vdp_action act, void **priv,
 
 	if (act == VDP_INIT)
 		return (0);
-	if (r2->error && act != VDP_FINI)
+	if ((r2->h2sess->error || r2->error) && act != VDP_FINI)
 		return (-1);
 	H2_Send_Get(req->wrk, r2->h2sess, r2);
 	H2_Send(req->wrk, r2,
@@ -189,7 +189,7 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 	struct sess *sp;
 	struct h2_req *r2;
 	struct vsb resp;
-	int i, err;
+	int i;
 	const struct hpack_static *hps;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -264,9 +264,8 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 
 	/* XXX someone into H2 please add appropriate error handling */
 	if (sendbody) {
-		err = VDP_push(req, &h2_vdp, NULL, 1);
-		if (!err)
-			err = VDP_DeliverObj(req);
+		if (!VDP_push(req, &h2_vdp, NULL, 1))
+			(void)VDP_DeliverObj(req);
 	}
 
 	AZ(req->wrk->v1l);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:14 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:14 +0000 (UTC)
Subject: [6.0] a713c1b5c Polishing
Message-ID: <20181031130814.C0E429515E@lists.varnish-cache.org>


commit a713c1b5cd8b480cd6d7a79f4b27e91d7e6369df
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 5 07:28:36 2018 +0000

    Polishing

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 56aa27552..070bf2cad 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -893,10 +893,9 @@ h2_frame_complete(struct http_conn *htc)
 /**********************************************************************/
 
 static h2_error
-h2_procframe(struct worker *wrk, struct h2_sess *h2,
-    h2_frame h2f)
+h2_procframe(struct worker *wrk, struct h2_sess *h2, h2_frame h2f)
 {
-	struct h2_req *r2 = NULL;
+	struct h2_req *r2;
 	h2_error h2e;
 
 	ASSERT_RXTHR(h2);
diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c
index 5985fe472..ce114d390 100644
--- a/bin/varnishd/http2/cache_http2_session.c
+++ b/bin/varnishd/http2/cache_http2_session.c
@@ -55,18 +55,16 @@ static const char H2_prism[24] = {
 static size_t
 h2_enc_settings(const struct h2_settings *h2s, uint8_t *buf, size_t n)
 {
-	uint8_t *b;
 	size_t len = 0;
 
-	b = buf;
 #define H2_SETTING(U,l,v,d,...)				\
 	if (h2s->l != d) {				\
 		len += 6;				\
 		assert(len <= n);			\
-		vbe16enc(b, v);				\
-		b += 2;					\
-		vbe32enc(b, h2s->l);			\
-		b += 4;					\
+		vbe16enc(buf, v);			\
+		buf += 2;				\
+		vbe32enc(buf, h2s->l);			\
+		buf += 4;				\
 	}
 #include "tbl/h2_settings.h"
 	return (len);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:14 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:14 +0000 (UTC)
Subject: [6.0] f224903c8 Spend a void for GCC's -Werror=pointer-sign
Message-ID: <20181031130814.DE20F95168@lists.varnish-cache.org>


commit f224903c8b5bf456bbcd9323d04ec8d11b8d1680
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 5 07:45:27 2018 +0000

    Spend a void for GCC's -Werror=pointer-sign

diff --git a/bin/varnishd/http2/cache_http2_send.c b/bin/varnishd/http2/cache_http2_send.c
index 41ca146b6..638fea632 100644
--- a/bin/varnishd/http2/cache_http2_send.c
+++ b/bin/varnishd/http2/cache_http2_send.c
@@ -141,7 +141,7 @@ H2_Send_Frame(struct worker *wrk, struct h2_sess *h2,
 	Lck_Unlock(&h2->sess->mtx);
 
 	memset(iov, 0, sizeof iov);
-	iov[0].iov_base = hdr;
+	iov[0].iov_base = (void*)hdr;
 	iov[0].iov_len = sizeof hdr;
 	iov[1].iov_base = TRUST_ME(ptr);
 	iov[1].iov_len = len;

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:15 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:15 +0000 (UTC)
Subject: [6.0] 54da09da9 Use a dedicated short-TTL'ed FQDN for testing if DNS
 works.
Message-ID: <20181031130815.0B6279517A@lists.varnish-cache.org>


commit 54da09da90604e5157517fd1644a6ea878edef79
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 5 08:16:37 2018 +0000

    Use a dedicated short-TTL'ed FQDN for testing if DNS works.

diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index e85ae92b7..d6168a2ae 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -441,7 +441,7 @@ dns_cb(void *priv, const struct suckaddr *sa)
 	int *ret = priv;
 
 	VTCP_name(sa, abuf, sizeof abuf, pbuf, sizeof pbuf);
-	if (strcmp(abuf, "130.225.244.222")) {
+	if (strcmp(abuf, "192.0.2.255")) {
 		fprintf(stderr, "DNS-test: Wrong response: %s\n", abuf);
 		*ret = -1;
 	} else if (*ret == 0)
@@ -455,7 +455,7 @@ dns_works(void)
 	int ret = 0, error;
 	const char *msg;
 
-	error = VSS_resolver("phk.freebsd.dk", NULL, dns_cb, &ret, &msg);
+	error = VSS_resolver("dns-canary.freebsd.dk", NULL, dns_cb, &ret, &msg);
 	if (error || msg != NULL || ret != 1)
 		return (0);
 	return (1);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:15 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:15 +0000 (UTC)
Subject: [6.0] 6527437cd Don't attempt to normalize .vcc file encoding,
 just take the bytes
Message-ID: <20181031130815.31B7995187@lists.varnish-cache.org>


commit 6527437cdfed7beac93ca34e5ab5f9346213e4d6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 5 08:58:10 2018 +0000

    Don't attempt to normalize .vcc file encoding, just take the bytes
    
    Fixes #2761

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index e60cca6e5..f307a1ce7 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -859,7 +859,7 @@ class vcc(object):
     def parse(self):
         global inputline
         a = "\n" + open(self.inputfile, "r").read()
-        self.file_id = hashlib.sha256(a.encode('utf-8')).hexdigest()
+        self.file_id = hashlib.sha256(bytearray(a)).hexdigest()
         s = a.split("\n$")
         self.copyright = s.pop(0).strip()
         while s:
diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc
index 3e6b06108..da535a212 100644
--- a/lib/libvmod_debug/vmod.vcc
+++ b/lib/libvmod_debug/vmod.vcc
@@ -24,6 +24,8 @@
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
+#
+# Test vmodtool.py's UTF-8 handling: ?blefl?sk med koldsk?l eller ?l
 
 $Module debug 3 Development, test and debug
 $ABI strict

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:15 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:15 +0000 (UTC)
Subject: [6.0] 68d79ad4e Tighten an assert,
 and replicate it another place with similar constraints.
Message-ID: <20181031130815.5F6769519C@lists.varnish-cache.org>


commit 68d79ad4e0405360f2729ac6c7a3e4ae2305b363
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 5 18:07:06 2018 +0000

    Tighten an assert, and replicate it another place with similar
    constraints.
    
    Related to: #2763

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 1c221aabf..0aaa2017f 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -196,7 +196,7 @@ vbf_stp_mkbereq(struct worker *wrk, struct busyobj *bo)
 	if (bo->stale_oc != NULL &&
 	    ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND) &&
 	    (bo->stale_oc->boc != NULL || ObjGetLen(wrk, bo->stale_oc) != 0)) {
-		AZ(bo->stale_oc->flags & OC_F_PASS);
+		AZ(bo->stale_oc->flags & (OC_F_PASS|OC_F_PRIVATE));
 		q = HTTP_GetHdrPack(bo->wrk, bo->stale_oc, H_Last_Modified);
 		if (q != NULL)
 			http_PrintfHeader(bo->bereq0,
@@ -355,6 +355,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	if (http_IsStatus(bo->beresp, 304)) {
 		if (bo->stale_oc != NULL &&
 		    ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND)) {
+			AZ(bo->stale_oc->flags & (OC_F_PASS|OC_F_PRIVATE));
 			if (ObjCheckFlag(bo->wrk, bo->stale_oc, OF_CHGGZIP)) {
 				/*
 				 * If we changed the gzip status of the object

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:15 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:15 +0000 (UTC)
Subject: [6.0] 1f365b3f1 Test more of varnishhist
Message-ID: <20181031130815.84AD8951B6@lists.varnish-cache.org>


commit 1f365b3f154053b0978edd12ecdb2075c67eddb2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 6 07:40:16 2018 +0000

    Test more of varnishhist

diff --git a/bin/varnishtest/tests/u00009.vtc b/bin/varnishtest/tests/u00009.vtc
index 5a3bf928b..bd6286e4f 100644
--- a/bin/varnishtest/tests/u00009.vtc
+++ b/bin/varnishtest/tests/u00009.vtc
@@ -8,8 +8,12 @@ server s1 {
 varnish v1 -vcl+backend {} -start
 
 process p1 -dump {varnishhist -n ${v1_name}} -start
+process p2 -dump {varnishhist -n ${v1_name} -P b:BereqAcct::5:1:8} -start
+process p3 -dump {varnishhist -n ${v1_name} -P BerespBodytime} -start
 
 process p1 -expect-text 24 0 {1e2}
+process p2 -expect-text 24 0 {1e2}
+process p3 -expect-text 24 0 {1e2}
 
 delay 1
 
@@ -24,14 +28,14 @@ process p1 -expect-text 3 1 {20_}
 
 process p1 -screen_dump
 
-process p1 -winsz 23 80
+process p1 -winsz 23 80 -need-bytes +10
 
-delay 5
-
-process p1 -write {0>+-<}
-
-delay 1.5
+process p1 -write {0>+-<} -need-bytes +10
 
 process p1 -expect-text 2 1 {20_}
 
+process p1 -writehex 0c -need-bytes +10
+
 process p1 -screen_dump -write {q} -wait
+process p2 -screen_dump -write {q} -wait
+process p3 -screen_dump -write {q} -wait

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:15 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:15 +0000 (UTC)
Subject: [6.0] c5c5eac5c Fix a race exposed on fast 32 bit FreeBSD
Message-ID: <20181031130815.A56AD951C5@lists.varnish-cache.org>


commit c5c5eac5c620927634afdf229c65f4f01c093018
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 6 08:58:25 2018 +0000

    Fix a race exposed on fast 32 bit FreeBSD

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index 5f5cd2a86..be943cfaf 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -916,7 +916,7 @@ void
 cmd_process(CMD_ARGS)
 {
 	struct process *p, *p2;
-	uintmax_t u, v;
+	uintmax_t u, v, bsnap;
 	unsigned lin,col;
 	int spec_set = 0;
 
@@ -952,6 +952,8 @@ cmd_process(CMD_ARGS)
 		p = process_new(av[0]);
 	av++;
 
+	bsnap = p->stdout_bytes;
+
 	for (; *av != NULL; av++) {
 		if (vtc_error)
 			break;
@@ -1003,7 +1005,7 @@ cmd_process(CMD_ARGS)
 		if (!strcmp(*av, "-need-bytes")) {
 			u = strtoumax(av[1], NULL, 0);
 			if (av[1][0] == '+')
-				u += p->stdout_bytes;
+				u += bsnap;
 			av++;
 			do {
 				AZ(pthread_mutex_lock(&p->mtx));

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:15 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:15 +0000 (UTC)
Subject: [6.0] dfd83b399 Sync comment with reality
Message-ID: <20181031130815.C8325951D5@lists.varnish-cache.org>


commit dfd83b399291116cbb26e7ab30bc9b46b305846d
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Sep 6 22:18:14 2018 +0200

    Sync comment with reality
    
    it really is about time we rename OC_F_PASS -> OC_F_HFM, but
    my recent cleanup attempt included too much change before the release

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 0aaa2017f..6e8097c6a 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -451,7 +451,7 @@ vbf_stp_fetchbody(struct worker *wrk, struct busyobj *bo)
 		if (vfc->oc->flags & OC_F_ABANDON) {
 			/*
 			 * A pass object and delivery was terminated
-			 * We don't fail the fetch, in order for hit-for-pass
+			 * We don't fail the fetch, in order for HitMiss
 			 * objects to be created.
 			 */
 			AN(vfc->oc->flags & OC_F_PASS);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:15 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:15 +0000 (UTC)
Subject: [6.0] fa4b81b41 Test varnishhist a little bit more
Message-ID: <20181031130816.00DB9951E9@lists.varnish-cache.org>


commit fa4b81b41f1097d52acb2b8236678ec2b669e332
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 6 21:50:04 2018 +0000

    Test varnishhist a little bit more

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 65f9246be..52bb69a6d 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -422,8 +422,8 @@ do_curses(void *arg)
 		case 'Q':
 		case 'q':
 			AZ(raise(SIGINT));
-			endwin();
-			return (NULL);
+			quit = 1;
+			break;
 		case '0':
 		case '1':
 		case '2':
diff --git a/bin/varnishtest/tests/u00009.vtc b/bin/varnishtest/tests/u00009.vtc
index bd6286e4f..8b962a63d 100644
--- a/bin/varnishtest/tests/u00009.vtc
+++ b/bin/varnishtest/tests/u00009.vtc
@@ -24,6 +24,13 @@ client c1 {
 
 process p1 -expect-text 22 0 {#}
 
+client c1 {
+	txreq
+	rxresp
+} -run
+
+process p1 -expect-text 22 0 { | }
+
 process p1 -expect-text 3 1 {20_}
 
 process p1 -screen_dump

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:16 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:16 +0000 (UTC)
Subject: [6.0] 1bb0f189f Clear the IMS object attribute when copying from a
 stale object
Message-ID: <20181031130816.24066951F5@lists.varnish-cache.org>


commit 1bb0f189f0b85bfc506a8bce0adaa9b1cc8c69ec
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Sep 6 23:10:17 2018 +0200

    Clear the IMS object attribute when copying from a stale object
    
    The fact that the previous object was IMS/INM eligable does not make
    one created from it eligable as well.
    
    Also test for the miss + hfm case with IMS/INM
    
    Fixes #2763

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 6e8097c6a..be6dc162c 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -716,6 +716,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo)
 		    OA_ESIDATA));
 
 	AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_FLAGS));
+	ObjSetFlag(bo->wrk, bo->fetch_objcore, OF_IMSCAND, 0);
 	AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_GZIPBITS));
 
 	if (bo->do_stream) {
diff --git a/bin/varnishtest/tests/r02763.vtc b/bin/varnishtest/tests/r02763.vtc
new file mode 100644
index 000000000..3488d7c95
--- /dev/null
+++ b/bin/varnishtest/tests/r02763.vtc
@@ -0,0 +1,68 @@
+varnishtest "Cacheable IMS replaced by HFM object"
+
+server s1 {
+	rxreq
+	expect req.url == "/etag"
+	txresp -hdr "ETag: foo" -bodylen 7
+
+	rxreq
+	expect req.url == "/etag"
+	expect req.http.If-None-Match == "foo"
+	txresp -status 304 -hdr "ETag: foo"
+
+	rxreq
+	expect req.url == "/etag"
+	txresp -hdr "ETag: foo" -bodylen 7
+} -start
+
+varnish v1 -vcl+backend {
+
+	sub vcl_miss {
+		set req.http.X-Cache = "MISS";
+	}
+
+	sub vcl_pass {
+		set req.http.X-Cache = "PASS";
+	}
+
+	sub vcl_backend_response {
+		if (bereq.http.HFM) {
+			set beresp.uncacheable = true;
+		} else {
+			set beresp.ttl = 0.0001s;
+			set beresp.grace = 0s;
+			set beresp.keep = 1m;
+		}
+		return (deliver);
+	}
+
+	sub vcl_deliver {
+		set resp.http.X-Cache = req.http.X-Cache;
+	}
+
+} -start
+
+client c1 {
+	txreq -url "/etag"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 7
+	expect resp.http.ETag == "foo"
+	expect resp.http.X-Cache == "MISS"
+
+	delay 0.1
+
+	txreq -url "/etag" -hdr "HFM: true"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 7
+	expect resp.http.ETag == "foo"
+	expect resp.http.X-Cache == "MISS"
+
+	txreq -url "/etag"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 7
+	expect resp.http.ETag == "foo"
+	expect resp.http.X-Cache == "MISS"
+} -run

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:16 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:16 +0000 (UTC)
Subject: [6.0] b0b82f39e Ignore values non-positive values.
Message-ID: <20181031130816.463F19520C@lists.varnish-cache.org>


commit b0b82f39e6659118483b9d0022b95a7c39ff00c6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 9 12:41:18 2018 +0000

    Ignore values non-positive values.
    
    Fixes:  #2773

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 52bb69a6d..1ef81f15f 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -295,7 +295,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 			}
 		}
 
-		if (skip || !match)
+		if (skip || !match || value <= 0)
 			continue;
 
 		/* select bucket */

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:16 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:16 +0000 (UTC)
Subject: [6.0] 6813e0b1b Test more distant corners of varnishhist
Message-ID: <20181031130816.6DF3495220@lists.varnish-cache.org>


commit 6813e0b1b6958de6caf06824ce6249641e603ecb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 9 14:54:03 2018 +0000

    Test more distant corners of varnishhist

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 1ef81f15f..48bfce265 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -657,8 +657,7 @@ main(int argc, char **argv)
 		ident = VSM_Dup(vut->vsm, "Arg", "-i");
 	else
 		ident = strdup("");
-	if (pthread_create(&thr, NULL, do_curses, NULL) != 0)
-		VUT_Error(vut, 1, "pthread_create(): %s", strerror(errno));
+	AZ(pthread_create(&thr, NULL, do_curses, NULL));
 	vut->dispatch_f = accumulate;
 	vut->dispatch_priv = NULL;
 	vut->sighup_f = sighup;
diff --git a/bin/varnishtest/tests/u00007.vtc b/bin/varnishtest/tests/u00007.vtc
index 64e9dc7c5..fb3a81a26 100644
--- a/bin/varnishtest/tests/u00007.vtc
+++ b/bin/varnishtest/tests/u00007.vtc
@@ -11,12 +11,14 @@ shell -match "Usage: .*varnishhist <options>" \
 	"varnishhist -h"
 shell -expect "Copyright (c) 2006 Verdens Gang AS" \
 	"varnishhist -V"
+shell -err -match "Usage: .*varnishhist <options>" \
+	"varnishhist -K"
 shell -err -match "Usage: .*varnishhist <options>" \
 	"varnishhist extra"
 shell -err -expect "-p: invalid '0'" \
 	"varnishhist -p 0"
 shell -err -expect "-B: being able to bend time does not mean we can stop it" \
-	"varnishhist -B 0"
+	"varnishhist -p 2.0 -B 0"
 shell -err -expect "-B: being able to bend time does not mean we can make it go backwards" \
 	"varnishhist -B -1"
 shell -err -expect "Invalid grouping mode: raw" \
@@ -25,5 +27,13 @@ shell -err -expect "-P: No such profile 'foo'" \
 	"varnishhist -P foo"
 shell -err -expect "-P: 'Timestamp:' is not a valid profile name or definition" \
 	"varnishhist -P Timestamp::"
-shell -err -expect "-P: 'foo::0:0:0' is not a valid tag name" \
-	"varnishhist -P foo::0:0:0"
+shell -err -expect "-P: 'b:' is not a valid profile name or definition" \
+	"varnishhist -P b:"
+shell -err -expect "-P: 'foo:' is not a valid tag name" \
+	"varnishhist -P foo:"
+shell -err -expect "-P: 'b:Debug:' is an unsafe or binary record" \
+	"varnishhist -P b:Debug:"
+shell -err -expect "-P: 'b:BereqAcct:x' is not a valid profile name or definition" \
+	"varnishhist -P b:BereqAcct:x"
+shell -err -expect "-P: 'b:BereqAcct:x' is not a valid profile name or definition" \
+	"varnishhist -P b:BereqAcct:x:"
diff --git a/bin/varnishtest/tests/u00009.vtc b/bin/varnishtest/tests/u00009.vtc
index 8b962a63d..df3551cef 100644
--- a/bin/varnishtest/tests/u00009.vtc
+++ b/bin/varnishtest/tests/u00009.vtc
@@ -9,7 +9,7 @@ varnish v1 -vcl+backend {} -start
 
 process p1 -dump {varnishhist -n ${v1_name}} -start
 process p2 -dump {varnishhist -n ${v1_name} -P b:BereqAcct::5:1:8} -start
-process p3 -dump {varnishhist -n ${v1_name} -P BerespBodytime} -start
+process p3 -dump {varnishhist -n ${v1_name} -P BerespBodytime -B 2} -start
 
 process p1 -expect-text 24 0 {1e2}
 process p2 -expect-text 24 0 {1e2}
@@ -39,6 +39,8 @@ process p1 -winsz 23 80 -need-bytes +10
 
 process p1 -write {0>+-<} -need-bytes +10
 
+process p3 -write {0>+-<x} -need-bytes +10
+
 process p1 -expect-text 2 1 {20_}
 
 process p1 -writehex 0c -need-bytes +10

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:16 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:16 +0000 (UTC)
Subject: [6.0] 0b5a8941f Test even more obscure corners of varnishhist
Message-ID: <20181031130816.90CED9522C@lists.varnish-cache.org>


commit 0b5a8941fc90825cdb7f473716e88669557cb266
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Sep 9 18:31:44 2018 +0000

    Test even more obscure corners of varnishhist

diff --git a/bin/varnishtest/tests/u00007.vtc b/bin/varnishtest/tests/u00007.vtc
index fb3a81a26..161643130 100644
--- a/bin/varnishtest/tests/u00007.vtc
+++ b/bin/varnishtest/tests/u00007.vtc
@@ -37,3 +37,7 @@ shell -err -expect "-P: 'b:BereqAcct:x' is not a valid profile name or definitio
 	"varnishhist -P b:BereqAcct:x"
 shell -err -expect "-P: 'b:BereqAcct:x' is not a valid profile name or definition" \
 	"varnishhist -P b:BereqAcct:x:"
+shell -err -expect "-P: 'b:BereqAcct:' is not a valid profile name or definition" \
+	"varnishhist -P b:BereqAcct::5:1:a"
+shell -err -expect "-p: invalid '0'" \
+	"varnishhist -P b:BereqAcct::5 -p 0"
diff --git a/bin/varnishtest/tests/u00009.vtc b/bin/varnishtest/tests/u00009.vtc
index df3551cef..3f62e34ab 100644
--- a/bin/varnishtest/tests/u00009.vtc
+++ b/bin/varnishtest/tests/u00009.vtc
@@ -2,17 +2,17 @@ varnishtest	"trivial run of varnishhist in curses mode"
 
 server s1 {
 	rxreq
-	txresp
+	txresp -bodylen 32
 } -start
 
 varnish v1 -vcl+backend {} -start
 
 process p1 -dump {varnishhist -n ${v1_name}} -start
-process p2 -dump {varnishhist -n ${v1_name} -P b:BereqAcct::5:1:8} -start
+process p2 -dump {varnishhist -n ${v1_name} -P b:BereqAcct::5:-1:1} -start
 process p3 -dump {varnishhist -n ${v1_name} -P BerespBodytime -B 2} -start
 
 process p1 -expect-text 24 0 {1e2}
-process p2 -expect-text 24 0 {1e2}
+process p2 -expect-text 24 0 {1e-1}
 process p3 -expect-text 24 0 {1e2}
 
 delay 1
@@ -22,13 +22,18 @@ client c1 {
 	rxresp
 } -run
 
+varnish v1 -vsl_catchup
+
 process p1 -expect-text 22 0 {#}
+process p2 -expect-text 22 80 {#}
 
 client c1 {
 	txreq
 	rxresp
 } -run
 
+varnish v1 -vsl_catchup
+
 process p1 -expect-text 22 0 { | }
 
 process p1 -expect-text 3 1 {20_}
@@ -39,12 +44,12 @@ process p1 -winsz 23 80 -need-bytes +10
 
 process p1 -write {0>+-<} -need-bytes +10
 
-process p3 -write {0>+-<x} -need-bytes +10
+process p3 -write {0++++++++++>+-<x} -need-bytes +10
 
 process p1 -expect-text 2 1 {20_}
 
 process p1 -writehex 0c -need-bytes +10
 
 process p1 -screen_dump -write {q} -wait
-process p2 -screen_dump -write {q} -wait
+process p2 -screen_dump -kill HUP -wait
 process p3 -screen_dump -write {q} -wait

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:16 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:16 +0000 (UTC)
Subject: [6.0] 0b1596efa Optimize updating the index when we delete segments.
Message-ID: <20181031130816.B446D95238@lists.varnish-cache.org>


commit 0b1596efacc0db7128e0e837f812ca8938264019
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 10 06:58:04 2018 +0000

    Optimize updating the index when we delete segments.
    
    Helps: #2764

diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c
index 6d4545fc2..339394b58 100644
--- a/bin/varnishd/common/common_vsmw.c
+++ b/bin/varnishd/common/common_vsmw.c
@@ -127,22 +127,17 @@ vsmw_idx_head(const struct vsmw *vsmw, int fd)
 }
 
 static void
-vsmw_write_index(const struct vsmw *vsmw, int fd, const struct vsmwseg *seg)
+vsmw_fmt_index(const struct vsmw *vsmw, const struct vsmwseg *seg)
 {
-	ssize_t s;
 
 	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
 	CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
-	VSB_clear(vsmw->vsb);
 	VSB_printf(vsmw->vsb, "%s %zu %zu %s %s\n",
 	    seg->cluster->fn,
 	    seg->off,
 	    seg->len,
 	    seg->class,
 	    seg->id);
-	AZ(VSB_finish(vsmw->vsb));
-	s = write(fd, VSB_data(vsmw->vsb), VSB_len(vsmw->vsb));
-	assert(s == VSB_len(vsmw->vsb));
 }
 
 /*--------------------------------------------------------------------*/
@@ -174,11 +169,16 @@ static void
 vsmw_addseg(struct vsmw *vsmw, struct vsmwseg *seg)
 {
 	int fd;
+	ssize_t s;
 
 	VTAILQ_INSERT_TAIL(&vsmw->segs, seg, list);
 	fd = openat(vsmw->vdirfd, vsmw->idx, O_APPEND | O_WRONLY);
 	assert(fd >= 0);
-	vsmw_write_index(vsmw, fd, seg);
+	VSB_clear(vsmw->vsb);
+	vsmw_fmt_index(vsmw, seg);
+	AZ(VSB_finish(vsmw->vsb));
+	s = write(fd, VSB_data(vsmw->vsb), VSB_len(vsmw->vsb));
+	assert(s == VSB_len(vsmw->vsb));
 	AZ(close(fd));
 }
 
@@ -188,6 +188,7 @@ static void
 vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
 {
 	char *t = NULL;
+	ssize_t s;
 	int fd;
 
 	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
@@ -206,8 +207,12 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
 		    t, O_WRONLY|O_CREAT|O_EXCL, vsmw->mode);
 		assert(fd >= 0);
 		vsmw_idx_head(vsmw, fd);
+		VSB_clear(vsmw->vsb);
 		VTAILQ_FOREACH(seg, &vsmw->segs, list)
-			vsmw_write_index(vsmw, fd, seg);
+			vsmw_fmt_index(vsmw, seg);
+		AZ(VSB_finish(vsmw->vsb));
+		s = write(fd, VSB_data(vsmw->vsb), VSB_len(vsmw->vsb));
+		assert(s == VSB_len(vsmw->vsb));
 		AZ(close(fd));
 		AZ(renameat(vsmw->vdirfd, t, vsmw->vdirfd, vsmw->idx));
 		REPLACE(t, NULL);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:16 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:16 +0000 (UTC)
Subject: [6.0] 0dba1788b Back down a little bit on UTF-8 in .vcc files.
Message-ID: <20181031130816.D3DA095244@lists.varnish-cache.org>


commit 0dba1788b8b4852776c87987c271c5f62d5b9383
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 10 11:38:52 2018 +0000

    Back down a little bit on UTF-8 in .vcc files.
    
    Not ready to tackle the Python2/Python3 monster a week before release.

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index f307a1ce7..ac54ceebd 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -858,8 +858,9 @@ class vcc(object):
 
     def parse(self):
         global inputline
-        a = "\n" + open(self.inputfile, "r").read()
-        self.file_id = hashlib.sha256(bytearray(a)).hexdigest()
+        b = open(self.inputfile, "rb").read()
+        a = "\n" + b.decode("utf-8")
+        self.file_id = hashlib.sha256(b).hexdigest()
         s = a.split("\n$")
         self.copyright = s.pop(0).strip()
         while s:
diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc
index da535a212..2b56f2e35 100644
--- a/lib/libvmod_debug/vmod.vcc
+++ b/lib/libvmod_debug/vmod.vcc
@@ -25,7 +25,6 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
-# Test vmodtool.py's UTF-8 handling: ?blefl?sk med koldsk?l eller ?l
 
 $Module debug 3 Development, test and debug
 $ABI strict

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:16 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:16 +0000 (UTC)
Subject: [6.0] 625d855d0 add HTC errors to the documentation
Message-ID: <20181031130817.0326F9524D@lists.varnish-cache.org>


commit 625d855d02cbfabdd809d5178d8aa0907782a392
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Sep 10 20:10:25 2018 +0200

    add HTC errors to the documentation
    
    Conflicts:
            doc/sphinx/whats-new/upgrading-6.1.rst

diff --git a/include/tbl/htc.h b/include/tbl/htc.h
index a46b75e28..a93c0e520 100644
--- a/include/tbl/htc.h
+++ b/include/tbl/htc.h
@@ -35,11 +35,11 @@
 HTC_STATUS(JUNK,	-5,	"junk",		"Received unexpected data")
 HTC_STATUS(CLOSE,	-4,	"close",	"Connection closed") // unused?
 HTC_STATUS(TIMEOUT,	-3,	"timeout",	"Timed out")
-HTC_STATUS(OVERFLOW,	-2,	"overflow",	"Buffer too small")
-HTC_STATUS(EOF,	-1,	"eof",		"EOF received")
+HTC_STATUS(OVERFLOW,	-2,	"overflow",	"Buffer/workspace too small")
+HTC_STATUS(EOF,	-1,	"eof",		"Unexpected end of input")
 HTC_STATUS(EMPTY,	 0,	"empty",	"Empty response")
 HTC_STATUS(MORE,	 1,	"more",	"More data required")
-HTC_STATUS(COMPLETE,	 2,	"complete",	"Data complete")
-HTC_STATUS(IDLE,	 3,	"idle",	"Return to waiter")
+HTC_STATUS(COMPLETE,	 2,	"complete",	"Data complete (no error)")
+HTC_STATUS(IDLE,	 3,	"idle",	"Connection was closed while idle")
 #undef HTC_STATUS
 /*lint -restore */
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index ac93f15b7..6e7b81031 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -165,9 +165,32 @@ SLTM(Length, 0, "Size of object body",
 	"Logs the size of a fetch object body.\n\n"
 )
 
+/* XXX generate HTC info from tbl include */
+#if 0
+#include <stdio.h>
+int main(void) {
+#define HTC_STATUS(e, n, s, l) \
+	printf("\t\"\\t* %s (%d): %s\\n\"\n", s, n, l);
+#include "include/tbl/htc.h"
+	return (0);
+}
+#endif
+
 SLTM(FetchError, 0, "Error while fetching object",
 	"Logs the error message of a failed fetch operation.\n\n"
-)
+	"Error messages should be self-explanatory, yet the http connection"
+	"(HTC) class of errors is reported with these symbols:\n\n"
+	"\t* junk (-5): Received unexpected data\n"
+	"\t* close (-4): Connection closed\n"
+	"\t* timeout (-3): Timed out\n"
+	"\t* overflow (-2): Buffer/workspace too small\n"
+	"\t* eof (-1): Unexpected end of input\n"
+	"\t* empty (0): Empty response\n"
+	"\t* more (1): More data required\n"
+	"\t* complete (2): Data complete (no error)\n"
+	"\t* idle (3): Connection was closed while idle\n"
+	"\nNotice that some HTC errors are never emitted."
+	)
 
 #define SLTH(tag, ind, req, resp, sdesc, ldesc) \
 	SLTM(Req##tag, (req ? 0 : SLT_F_UNUSED), "Client request " sdesc, ldesc)

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:17 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:17 +0000 (UTC)
Subject: [6.0] 62f291a16 vmod optional arguments documentation (not new for
 6.1)
Message-ID: <20181031130817.2896A95259@lists.varnish-cache.org>


commit 62f291a16f9fe718be04f72e538e9146e4081395
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Sep 12 08:53:04 2018 +0200

    vmod optional arguments documentation (not new for 6.1)

diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index 3b63f563f..b9c0fcfc8 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -140,6 +140,44 @@ different to user specified values.
 `Note` that default values have to be given in the native C-type
 syntax, see below. As a special case, ``NULL`` has to be given as ``0``.
 
+Optional arguments
+------------------
+
+The vmod.vcc declaration also allows for optional arguments in square
+brackets like so::
+
+	$Function VOID opt(PRIV_TASK, INT four = 4, [ STRING opt])
+
+With any optional argument present, the C function prototype looks
+completely different:
+
+	* Only the ``VRT_CTX`` and object pointer arguments (only for
+	  methods) remain positional
+
+	* All other arguments get passed in a struct as the last
+	  argument of the C function.
+
+The argument struct is simple, vmod authors should check the
+`vmodtool`-generated ``vcc_if.c`` file for the function and struct
+declarations:
+
+	* for each optional argument, a ``valid_``\ `argument` member
+	  is used to signal the presence of the respective optional
+	  argument.
+
+	  ``valid_`` argstruct members should only be used as truth
+	  values, irrespective of their actual data type.
+
+	* named arguments are passed in argument struct members by the
+	  same name and with the same data type.
+
+	* unnamed (positional) arguments are passed as ``arg``\ `n`
+	  with `n` starting at 1 and incrementing with the argument's
+	  position.
+
+	  Note that in particular also ``PRIV_*`` arguments (which are
+	  unnamed by definition) are passed as ``arg``\ `n`
+
 .. _ref-vmod-vcl-c-types:
 
 VCL and C data types

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:17 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:17 +0000 (UTC)
Subject: [6.0] 5459a027f have a go at some vmod objects dev doc (not new for
 6.1)
Message-ID: <20181031130817.4641995271@lists.varnish-cache.org>


commit 5459a027fca0c297b4cbd77057fcefca7d02fa4f
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Sep 12 10:22:51 2018 +0200

    have a go at some vmod objects dev doc (not new for 6.1)

diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index b9c0fcfc8..657585f82 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -178,6 +178,73 @@ declarations:
 	  Note that in particular also ``PRIV_*`` arguments (which are
 	  unnamed by definition) are passed as ``arg``\ `n`
 
+.. _ref-vmod-vcl-c-objects:
+
+Objects and methods
+-------------------
+
+Varnish also supports a simple object model for vmods. Objects and
+methods are declared in the vcc file as::
+
+	$Object class(`constructor arguments`)
+	$Method .method(`method arguments`)
+
+
+For declared object classes of a vmod, object instances can then be
+created in ``vcl_init { }`` using the ``new`` statement::
+
+	sub vcl_init {
+		new foo = vmod.class(...);
+	}
+
+and have their methods called anywhere (including in ``vcl_init {}``
+after the instantiation)::
+
+	sub somewhere {
+		foo.method(...);
+	}
+
+Object instances are represented as pointers to vmod-implemented C
+structs. Varnish only provides space to store the address of object
+instances and ensures that the right object address gets passed to C
+functions implementing methods.
+
+	* Objects' scope and lifetime are the vcl
+
+	* Objects can only be created in ``vcl_init {}`` and have
+	  their destructors called by varnish after ``vcl_fini {}``
+	  has completed.
+
+vmod authors are advised to understand the prototypes in the
+`vmodtool`\ -generated ``vcc_if.c`` file:
+
+	* For ``$Object`` declarations, a constructor and destructor
+	  function must be implemented
+
+	* The constructor is named by the suffix ``__init``, always is
+	  of ``VOID`` return type and has the following arguments
+	  before the vcc-declared parameters:
+
+	  * ``VRT_CTX`` as usual
+	  * a pointer-pointer to return the address of the created
+	    oject
+	  * a string containing the vcl name of the object instance
+
+	* The destructor is named by the suffix ``__fini``, always is
+	  of ``VOID`` return type and has a single argument, the
+	  pointer-pointer to the address of the object. The destructor
+	  is expected clear the address of the object stored in that
+	  pointer-pointer.
+
+	* Methods gain the pointer to the object as an argument after
+	   the ``VRT_CTX``.
+
+As varnish is in no way involved in managing object instances other
+than passing their addresses, vmods need to implement all aspects of
+managing instances, in particular their memory management. As the
+lifetime of object instances is the vcl, they will usually be
+allocated from the heap.
+
 .. _ref-vmod-vcl-c-types:
 
 VCL and C data types
@@ -439,6 +506,20 @@ malloc would look like this::
 
 The per-call vmod_privs are freed before the per-vcl vmod_priv.
 
+Note on use with objects:
+
+``PRIV_TASK`` and ``PRIV_TOP`` arguments are not per object instance,
+but still per vmod as for ordinary vmod functions. Thus, vmods
+requiring per-task / per top-request state for object instances need
+to implement other means to associate storage with object instances.
+
+Using ``VRT_priv_task()`` to maintin per object instance state is a
+convenient yet inofficial interface which was not originally intended
+for this purpose and will likely be replaced with a more suitable
+interface.
+
+
+
 .. _ref-vmod-event-functions:
 
 Event functions
@@ -511,6 +592,7 @@ In the case where properly releasing resources may take some time, you can
 opt for an asynchronous worker, either by spawning a thread and tracking it, or
 by using Varnish's worker pools.
 
+
 When to lock, and when not to lock
 ==================================
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:17 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:17 +0000 (UTC)
Subject: [6.0] 59e486288 polish
Message-ID: <20181031130817.66A4B9528C@lists.varnish-cache.org>


commit 59e486288fc7a2e72a14ec430762cac59b6fba5c
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Sep 12 10:29:07 2018 +0200

    polish

diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index 657585f82..eb35c1bf9 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -186,8 +186,8 @@ Objects and methods
 Varnish also supports a simple object model for vmods. Objects and
 methods are declared in the vcc file as::
 
-	$Object class(`constructor arguments`)
-	$Method .method(`method arguments`)
+	$Object class(...)
+	$Method .method(...)
 
 
 For declared object classes of a vmod, object instances can then be

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:17 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:17 +0000 (UTC)
Subject: [6.0] d285db34b updating vmods should just work (not new in any way)
Message-ID: <20181031130817.8D28A952A7@lists.varnish-cache.org>


commit d285db34b7b6ff4ca1b3a407dd764f688890af72
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Sep 12 10:29:18 2018 +0200

    updating vmods should just work (not new in any way)
    
    We copy vmods using unique filenames for ages
    
    Ref: 3cf587668219166188e0dce304edb0bf8ab346ab
    
    Original text from 2014: 920541a8fa79b2ce9471571df0f973b1beeb25b8

diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index eb35c1bf9..e91b87db1 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -610,33 +610,3 @@ unless they access VMOD specific global state, shared with other VCLs.
 
 Traffic in other VCLs which also import this VMOD, will be happening
 while housekeeping is going on.
-
-Updating VMODs
-==============
-
-A compiled VMOD is a shared library file which Varnish dlopen(3)'s
-using flags RTLD_NOW | RTLD_LOCAL.
-
-As a general rule, once a file is opened with dlopen(3) you should
-never modify it, but it is safe to rename it and put a new file
-under the name it had, which is how most tools installs and updates
-shared libraries.
-
-However, when you call dlopen(3) with the same filename multiple
-times it will give you the same single copy of the shared library
-file, without checking if it was updated in the meantime.
-
-This is obviously an oversight in the design of the dlopen(3) library
-function, but back in the late 1980s nobody could imagine why a
-program would ever want to have multiple different versions of the
-same shared library mapped at the same time.
-
-Varnish does that, and therefore you must restart the worker process
-before Varnish will discover an updated VMOD.
-
-If you want to test a new version of a VMOD, while being able to
-instantly switch back to the old version, you will have to install
-each version with a distinct filename or in a distinct subdirectory
-and use ``import foo from "...";`` to reference it in your VCL.
-
-We're not happy about this, but have found no sensible workarounds.

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:17 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:17 +0000 (UTC)
Subject: [6.0] ee7d2d43e polish
Message-ID: <20181031130817.AD2AC952BE@lists.varnish-cache.org>


commit ee7d2d43eb2e5f74df058a5cde4eda9df68d2162
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Sep 12 10:41:59 2018 +0200

    polish

diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index e91b87db1..df6f5c7d9 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -46,7 +46,7 @@ The vmod.vcc file
 The interface between your VMOD and the VCL compiler ("VCC") and the
 VCL runtime ("VRT") is defined in the vmod.vcc file which a python
 script called "vmodtool.py" turns into thaumaturgically challenged C
-data structures that does all the hard work.
+data structures that do all the hard work.
 
 The std VMODs vmod.vcc file looks somewhat like this::
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:17 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:17 +0000 (UTC)
Subject: [6.0] aa2d8f524 document vmod $ABI
Message-ID: <20181031130817.CDCC7952D5@lists.varnish-cache.org>


commit aa2d8f524833ba88f9cdcd88056af7c0e94a05cb
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Sep 12 10:50:06 2018 +0200

    document vmod $ABI
    
    Text mostly taken from ecdebdc792942165ba71b58ff6816decb2d89bef

diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index df6f5c7d9..209c68188 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -51,6 +51,7 @@ data structures that do all the hard work.
 The std VMODs vmod.vcc file looks somewhat like this::
 
 	$Module std 3
+	$ABI strict
 	$Event event_function
 	$Function STRING toupper(STRING_LIST)
 	$Function STRING tolower(STRING_LIST)
@@ -59,7 +60,17 @@ The std VMODs vmod.vcc file looks somewhat like this::
 The first line gives the name of the module and the manual section where
 the documentation will reside.
 
-The second line specifies an optional "Event" function, which will be
+The ``$ABI`` line is optional (possible values ``strict`` (default)
+and ``vrt``) and allows to specify that a vmod is integrating with the
+blessed ``vrt`` interface provided by ``varnishd`` or go deeper in the
+stack. As a general rule of thumb you are considered "on your own" if
+your VMOD uses more than the VRT (Varnish RunTime), in which case it
+needs to be built for the exact Varnish version.
+
+``$ABI vrt`` means that a module complies to the VRT and only needs to
+be rebuilt when breaking changes are introduced to the VRT API.
+
+The third line specifies an optional "Event" function, which will be
 called whenever a VCL program which imports this VMOD is loaded or
 transitions to any of the warm, active, cold or discarded states.
 More on this below.

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:17 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:17 +0000 (UTC)
Subject: [6.0] 5c85a0235 document STRANDS
Message-ID: <20181031130818.045D8952F1@lists.varnish-cache.org>


commit 5c85a023580822d3b940eeff549052d6aac2ae81
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Sep 12 11:12:07 2018 +0200

    document STRANDS

diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index 209c68188..96fe6f011 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -422,6 +422,16 @@ STRING_LIST
 	and make sure your workspace_client and workspace_backend params
 	are big enough.
 
+STRANDS
+	C-Type: ``const struct strands *``
+
+	Strands are like STRING_LIST, but without the drawbacks of
+	variable arguments: The list of strings gets passed in a
+	struct with the following members:
+
+	* ``int n``: the number of strings
+	* ``const char p[]``: the array of strings with `n` elements
+
 TIME
 	C-type: ``double``
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:18 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:18 +0000 (UTC)
Subject: [6.0] 5d41fc68d Typo
Message-ID: <20181031130818.27FF995306@lists.varnish-cache.org>


commit 5d41fc68d91e9f7b084ff79587751ec777f5b418
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Sep 12 11:50:35 2018 +0200

    Typo

diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index 96fe6f011..b824a26ec 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -430,7 +430,7 @@ STRANDS
 	struct with the following members:
 
 	* ``int n``: the number of strings
-	* ``const char p[]``: the array of strings with `n` elements
+	* ``const char **p``: the array of strings with `n` elements
 
 TIME
 	C-type: ``double``
@@ -535,7 +535,7 @@ requiring per-task / per top-request state for object instances need
 to implement other means to associate storage with object instances.
 
 Using ``VRT_priv_task()`` to maintin per object instance state is a
-convenient yet inofficial interface which was not originally intended
+convenient yet unofficial interface which was not originally intended
 for this purpose and will likely be replaced with a more suitable
 interface.
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:18 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:18 +0000 (UTC)
Subject: [6.0] 229e1da0b Typo
Message-ID: <20181031130818.470C995318@lists.varnish-cache.org>


commit 229e1da0b384d66aae247ef6dd96dea11983cd31
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed Sep 12 11:53:59 2018 +0200

    Typo

diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index b824a26ec..f314527a6 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -534,7 +534,7 @@ but still per vmod as for ordinary vmod functions. Thus, vmods
 requiring per-task / per top-request state for object instances need
 to implement other means to associate storage with object instances.
 
-Using ``VRT_priv_task()`` to maintin per object instance state is a
+Using ``VRT_priv_task()`` to maintain per object instance state is a
 convenient yet unofficial interface which was not originally intended
 for this purpose and will likely be replaced with a more suitable
 interface.

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:18 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:18 +0000 (UTC)
Subject: [6.0] 9496176d8 s/belive/believe/
Message-ID: <20181031130818.6B67C95338@lists.varnish-cache.org>


commit 9496176d8b340b7e67d288245736b0f788543f5e
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Sep 17 09:41:40 2018 +0200

    s/belive/believe/
    
    Conflicts:
            doc/sphinx/whats-new/upgrading-6.1.rst

diff --git a/doc/sphinx/phk/firstdesign.rst b/doc/sphinx/phk/firstdesign.rst
index 32f27b2a2..fee1943f8 100644
--- a/doc/sphinx/phk/firstdesign.rst
+++ b/doc/sphinx/phk/firstdesign.rst
@@ -176,7 +176,7 @@ where we kept thinking in the old frame of reference (ie: Squid)::
 	to ask are:
 
 		How long time after the expiry can we serve a cached copy
-		of this document while we have reason to belive the backend
+		of this document while we have reason to believe the backend
 		can supply us with an update ?
 
 		How long time after the expiry can we serve a cached copy
diff --git a/doc/sphinx/whats-new/changes-6.0.rst b/doc/sphinx/whats-new/changes-6.0.rst
index 55001dde4..41fd5052e 100644
--- a/doc/sphinx/whats-new/changes-6.0.rst
+++ b/doc/sphinx/whats-new/changes-6.0.rst
@@ -103,7 +103,7 @@ variant symbols, presently symbols which are different in
 ``vcl 4.0`` and ``vcl 4.1``.
 
 The "prototype" information in the VMOD shared library has been
-changed to JSON, (look in your vcc_if.c file if you don't belive
+changed to JSON, (look in your vcc_if.c file if you don't believe
 me), and this can express more detailed information, presently
 the optional arguments.
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:18 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:18 +0000 (UTC)
Subject: [6.0] 39207ceda Fix production of VTC documentation.
Message-ID: <20181031130818.99A7B9535A@lists.varnish-cache.org>


commit 39207cedaaa10100216f0f9552b30436dbb938e0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 17 07:42:57 2018 +0000

    Fix production of VTC documentation.
    
    Fixes: #2777

diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c
index 8ffe69643..196b35c65 100644
--- a/bin/varnishtest/vtc_misc.c
+++ b/bin/varnishtest/vtc_misc.c
@@ -217,6 +217,8 @@ cmd_shell(CMD_ARGS)
 }
 
 /* SECTION: err_shell err_shell
+ *
+ * NOTICE: err_shell is deprecated, use `shell -err -expect` instead.
  *
  * This is very similar to the the ``shell`` command, except it takes a first
  * string as argument before the command::
diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 5eb2a1361..54f930caf 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -188,6 +188,7 @@ VTCSYN_SRC = $(top_srcdir)/bin/varnishtest/vtc.c \
 	     $(top_srcdir)/bin/varnishtest/vtc_http.c \
 	     $(top_srcdir)/bin/varnishtest/vtc_http2.c \
 	     $(top_srcdir)/bin/varnishtest/vtc_logexp.c \
+	     $(top_srcdir)/bin/varnishtest/vtc_misc.c \
 	     $(top_srcdir)/bin/varnishtest/vtc_process.c \
 	     $(top_srcdir)/bin/varnishtest/vtc_syslog.c \
 	     $(top_srcdir)/bin/varnishtest/vtc_varnish.c

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:18 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:18 +0000 (UTC)
Subject: [6.0] 443c4401c Clarify what Debug means
Message-ID: <20181031130818.BAC3F95374@lists.varnish-cache.org>


commit 443c4401c168f4922edde3cabb48619d0fd4e207
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Sep 17 11:03:17 2018 +0200

    Clarify what Debug means

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 6e7b81031..85a480e7b 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -51,6 +51,8 @@ SLTM(Debug, SLT_F_UNSAFE, "Debug messages",
 	"Debug messages can normally be ignored, but are sometimes"
 	" helpful during trouble-shooting.  Most debug messages must"
 	" be explicitly enabled with parameters.\n\n"
+	"Debug messages may be added, changed or removed without"
+	" prior notice and shouldn't be considered stable.\n\n"
 )
 
 SLTM(Error, 0, "Error messages",

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:18 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:18 +0000 (UTC)
Subject: [6.0] b894a8d8b I börked varnishhist by calling the lround function the wrong place.
Message-ID: <20181031130818.E0D999538D@lists.varnish-cache.org>


commit b894a8d8b8610618e1537c0f323061f7089aac15
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 11:23:07 2018 +0000

    I b?rked varnishhist by calling the lround function the wrong place.
    
    Fixes: #2780

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 48bfce265..c4f99a04a 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -299,7 +299,7 @@ accumulate(struct VSL_data *vsl, struct VSL_transaction * const pt[],
 			continue;
 
 		/* select bucket */
-		i = HIST_RES * lround(log(value) / log_ten);
+		i = lround(HIST_RES * log(value) / log_ten);
 		if (i < hist_low * HIST_RES)
 			i = hist_low * HIST_RES;
 		if (i >= hist_high * HIST_RES)

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:19 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:19 +0000 (UTC)
Subject: [6.0] 8511c3729 Fix trouble with 2GB+ VSB's on systems where int is
 32 bit but size_t is 64 bit.
Message-ID: <20181031130819.0F610953A1@lists.varnish-cache.org>


commit 8511c37298a8f6d438308125be8074c1ffc94578
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 06:38:58 2018 +0000

    Fix trouble with 2GB+ VSB's on systems where int is 32 bit but
    size_t is 64 bit.

diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c
index 0bc0925ab..b44323854 100644
--- a/lib/libvarnish/vsb.c
+++ b/lib/libvarnish/vsb.c
@@ -51,8 +51,8 @@ __FBSDID("$FreeBSD: head/sys/kern/subr_vsb.c 222004 2011-05-17 06:36:32Z phk $")
  */
 #define	VSB_ISDYNAMIC(s)	((s)->s_flags & VSB_DYNAMIC)
 #define	VSB_ISDYNSTRUCT(s)	((s)->s_flags & VSB_DYNSTRUCT)
-#define	VSB_HASROOM(s)		((s)->s_len < (s)->s_size - 1)
-#define	VSB_FREESPACE(s)	((s)->s_size - ((s)->s_len + 1))
+#define	VSB_HASROOM(s)		((s)->s_len < (s)->s_size - 1L)
+#define	VSB_FREESPACE(s)	((s)->s_size - ((s)->s_len + 1L))
 #define	VSB_CANEXTEND(s)	((s)->s_flags & VSB_AUTOEXTEND)
 
 /*
@@ -114,10 +114,10 @@ CTASSERT(powerof2(VSB_MAXEXTENDSIZE));
 CTASSERT(powerof2(VSB_MAXEXTENDINCR));
 #endif
 
-static int
-VSB_extendsize(int size)
+static ssize_t
+VSB_extendsize(ssize_t size)
 {
-	int newsize;
+	ssize_t newsize;
 
 	if (size < (int)VSB_MAXEXTENDSIZE) {
 		newsize = VSB_MINEXTENDSIZE;
@@ -133,11 +133,11 @@ VSB_extendsize(int size)
 /*
  * Extend an vsb.
  */
-static int
-VSB_extend(struct vsb *s, int addlen)
+static ssize_t
+VSB_extend(struct vsb *s, ssize_t addlen)
 {
 	char *newbuf;
-	int newsize;
+	ssize_t newsize;
 
 	if (!VSB_CANEXTEND(s))
 		return (-1);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:19 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:19 +0000 (UTC)
Subject: [6.0] 91edb36de Don't implement VRT_* functions in libvarnish
Message-ID: <20181031130819.34BD8953B3@lists.varnish-cache.org>


commit 91edb36defc6a92c0721bd404464ca48a86abff8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 06:43:17 2018 +0000

    Don't implement VRT_* functions in libvarnish

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index ada746692..470f691a7 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -762,3 +762,9 @@ VRT_blob(VRT_CTX, const char *err, const void *src, size_t len)
 	p->priv = d;
 	return (p);
 }
+
+int
+VRT_VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst)
+{
+	return (VSA_GetPtr(sua, dst));
+}
diff --git a/include/vsa.h b/include/vsa.h
index 70e4e502d..e5e342a26 100644
--- a/include/vsa.h
+++ b/include/vsa.h
@@ -55,4 +55,11 @@ struct suckaddr *VSA_Malloc(const void *s, unsigned  sal);
  */
 struct suckaddr *VSA_Build(void *d, const void *s, unsigned sal);
 
+/*
+ * This VRT interface is for the VCC generated ACL code, which needs
+ * to know the address family and a pointer to the actual address.
+ */
+  
+int VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst);
+
 #endif
diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c
index 4e0762f95..40d84fa83 100644
--- a/lib/libvarnish/vsa.c
+++ b/lib/libvarnish/vsa.c
@@ -43,7 +43,6 @@
 #include "vdef.h"
 #include "vas.h"
 #include "vsa.h"
-#include "vrt.h"
 #include "miniobj.h"
 
 /*
@@ -204,7 +203,7 @@ VSA_Init()
  */
 
 int
-VRT_VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst)
+VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst)
 {
 
 	AN(dst);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:19 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:19 +0000 (UTC)
Subject: [6.0] cecb501ad Eliminated varnishd from these tests
Message-ID: <20181031130819.71A5F953D0@lists.varnish-cache.org>


commit cecb501addd682cb525bed41300f285ce281faa6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 06:47:36 2018 +0000

    Eliminated varnishd from these tests

diff --git a/bin/varnishtest/tests/a00015.vtc b/bin/varnishtest/tests/a00015.vtc
index 843637f0f..f363d66d9 100644
--- a/bin/varnishtest/tests/a00015.vtc
+++ b/bin/varnishtest/tests/a00015.vtc
@@ -12,9 +12,7 @@ server s1 {
 	txresp -hdr "Content-Type: text/plain" -body response
 } -start
 
-varnish v1 -vcl+backend {} -start -cliok "param.set debug +syncvsl"
-
-client c1 {
+client c1 -connect ${s1_sock} {
 	txreq -req POST -hdr "Content-Type: text/plain" -body request
 
 	# First, HTTP checks
diff --git a/bin/varnishtest/tests/a00018.vtc b/bin/varnishtest/tests/a00018.vtc
index 036aa4596..5500d7c6e 100644
--- a/bin/varnishtest/tests/a00018.vtc
+++ b/bin/varnishtest/tests/a00018.vtc
@@ -8,28 +8,9 @@ server s1 {
 	txresp -hdr "Bar: ${bar}"
 } -start
 
-varnish v1 -vcl {
-	backend default {
-		.host = "${s1_addr}";
-		.port = "${s1_port}";
-	}
-
-	sub vcl_deliver {
-		if (resp.http.Bar == "${bar}") {
-			set resp.http.Baz = "${baz}";
-		}
-	}
-} -start
-
-client c1 {
+client c1 -connect ${s1_sock} {
 	txreq -hdr "Foo: ${foo}"
 	rxresp
 	expect resp.status == 200
 	expect resp.http.Bar == "${bar}"
-	expect resp.http.Baz == "${baz}"
 } -run
-
-shell {
-	touch ${tmpdir}/tst
-	rm ${tmpdir}/tst
-}

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:19 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:19 +0000 (UTC)
Subject: [6.0] aeb44245d Mark our extension to ZLib and #ifdef the code in
 varnishtest which reports stuff from it for debugging.
Message-ID: <20181031130819.9EB11953E3@lists.varnish-cache.org>


commit aeb44245dca511a5ce23036775f849c3f4b793a9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 06:48:49 2018 +0000

    Mark our extension to ZLib and #ifdef the code in varnishtest which
    reports stuff from it for debugging.

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index d89a91a5a..952563b9f 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -771,6 +771,7 @@ cmd_http_gunzip(CMD_ARGS)
 	vtc_log(hp->vl, 3, "new bodylen %u", hp->bodyl);
 	vtc_dump(hp->vl, 4, "body", hp->body, hp->bodyl);
 	bprintf(hp->bodylen, "%u", hp->bodyl);
+#ifdef VGZ_EXTENSIONS
 	vtc_log(hp->vl, 4, "startbit = %ju %ju/%ju",
 	    (uintmax_t)vz.start_bit,
 	    (uintmax_t)vz.start_bit >> 3, (uintmax_t)vz.start_bit & 7);
@@ -780,6 +781,7 @@ cmd_http_gunzip(CMD_ARGS)
 	vtc_log(hp->vl, 4, "stopbit = %ju %ju/%ju",
 	    (uintmax_t)vz.stop_bit,
 	    (uintmax_t)vz.stop_bit >> 3, (uintmax_t)vz.stop_bit & 7);
+#endif
 	if (i != Z_STREAM_END)
 		vtc_log(hp->vl, hp->fatal,
 		    "Gunzip error = %d (%s) in:%jd out:%jd",
@@ -813,12 +815,13 @@ gzip_body(const struct http *hp, const char *txt, char **body, int *bodylen)
 	assert(Z_OK == deflateInit2(&vz,
 	    hp->gziplevel, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY));
 	assert(Z_STREAM_END == deflate(&vz, Z_FINISH));
+	*bodylen = vz.total_out;
+#ifdef VGZ_EXTENSIONS
 	i = vz.stop_bit & 7;
 	if (hp->gzipresidual >= 0 && hp->gzipresidual != i)
 		vtc_log(hp->vl, hp->fatal,
 		    "Wrong gzip residual got %d wanted %d",
 		    i, hp->gzipresidual);
-	*bodylen = vz.total_out;
 	vtc_log(hp->vl, 4, "startbit = %ju %ju/%ju",
 	    (uintmax_t)vz.start_bit,
 	    (uintmax_t)vz.start_bit >> 3, (uintmax_t)vz.start_bit & 7);
@@ -829,6 +832,7 @@ gzip_body(const struct http *hp, const char *txt, char **body, int *bodylen)
 	    (uintmax_t)vz.stop_bit,
 	    (uintmax_t)vz.stop_bit >> 3, (uintmax_t)vz.stop_bit & 7);
 	assert(Z_OK == deflateEnd(&vz));
+#endif
 }
 
 /**********************************************************************
diff --git a/lib/libvgz/vgz.h b/lib/libvgz/vgz.h
index 86831eeed..1bb81009d 100644
--- a/lib/libvgz/vgz.h
+++ b/lib/libvgz/vgz.h
@@ -104,6 +104,7 @@ typedef struct z_stream_s {
     uLong   adler;      /* Adler-32 or CRC-32 value of the uncompressed data */
     uLong   reserved;   /* reserved for future use */
 
+#define VGZ_EXTENSIONS 1
     uLong   start_bit;	/* Bit pos of first deflate block */
     uLong   stop_bit;	/* Bit pos after last deflate block */
     uLong   last_bit;	/* Bit pos of 'last' bit */

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:19 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:19 +0000 (UTC)
Subject: [6.0] 9e3dda99c Whitespace
Message-ID: <20181031130819.D0AAF953FC@lists.varnish-cache.org>


commit 9e3dda99c593accea65054d50a2c60bcbd6c8731
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Oct 16 17:17:46 2018 +0200

    Whitespace

diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc
index 9bd14bc67..9316a5cb8 100644
--- a/bin/varnishtest/tests/a00000.vtc
+++ b/bin/varnishtest/tests/a00000.vtc
@@ -74,7 +74,7 @@ process p4 -need-bytes 310 -expect-text 3 1 "line3 <"
 process p4 -expect-cursor 4 1
 process p4 -expect-cursor 4 0
 process p4 -expect-cursor 0 1
-process p4 -screen-dump 
+process p4 -screen-dump
 
 # Also exercise CONS25 mode
 process p4 -write "\x1b[=1T"

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:20 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:20 +0000 (UTC)
Subject: [6.0] e51b6d7ea Get varnishd out of and varnishtest-tests into a*vtc
Message-ID: <20181031130820.4BD759541D@lists.varnish-cache.org>


commit e51b6d7ea059ab9bd332c94e3ee4cf9fcf45a375
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 07:09:30 2018 +0000

    Get varnishd out of and varnishtest-tests into a*vtc

diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc
index 9316a5cb8..22e9f341b 100644
--- a/bin/varnishtest/tests/a00000.vtc
+++ b/bin/varnishtest/tests/a00000.vtc
@@ -41,96 +41,130 @@ shell -exit 77 -expect {TEST _.vtc skipped} {
 	exec varnishtest -v _.vtc || true
 }
 
-process p1 "ps -lw | grep '[p][s]' ; tty ; echo @" -start
-process p1 -expect-text 0 0 {@} -screen_dump -wait
-
-process p2 "stty -a ; echo '*'" -start
-process p2 -expect-text 0 0 {*} -screen_dump -wait
-
-process p4 -hexdump {stty raw -echo; stty -a ; echo "*" ; cat} -start
-process p4 -expect-text 0 0 "*" -screen_dump
-
-process p4 -write "\x1b[H\x1b[2Jzzzzzzz"
-process p4 -write "\x0c1\x1b[79C2\x08>\x1b[25;1H3\x1b[25;80H"
-process p4 -write "\x1b[H\x1b[2J1\x1b[79C2\x08>\x1b[25;1H3\x1b[25;80H"
-process p4 -write "4\x08>\x1b[A\x1b[Cv\x1b[22A^\x1b[79D^\x1b[;2H<\n\n\n\n"
-process p4 -write "\n\n\n\n\n\n\n\n\x1b[B\x1b[11B\x08<\x1b[24;Hv\x1b[12;1H"
-process p4 -write "111111112222222333333\x0d\x0a111111112"
-process p4 -write "222222333333\x0d\x0a111111112222222333333 UTF8: "
-process p4 -writehex {c2 a2 20}
-process p4 -writehex {e2 82 ac 20}
-process p4 -writehex {f0 90 80 80 20}
-process p4 -writehex {f0 9f 90 b0 20}
-process p4 -writehex {f0 a0 80 80 20}
-process p4 -writehex {f0 b0 80 80 20}
-process p4 -write "\x1b[22;24;25;27;30;47;49;97;107m"
-process p4 -write "\x1b[22;24;25;27;30m"
-process p4 -write "\x1b[47;49;97;107m"
-process p4 -write "\x0d\x0a111111112222222333333\x0d\x0a\x1b[12"
-process p4 -write ";12H\x1b[K\x1b[13;12H\x1b[0K\x1b[14;12H\x1b[1K\x1b"
-process p4 -write "[15;12H\x1b[2K\x1b[3;1Hline3 <\x0d\x0a"
-
-process p4 -need-bytes 310 -expect-text 3 1 "line3 <"
-process p4 -expect-cursor 4 1
-process p4 -expect-cursor 4 0
-process p4 -expect-cursor 0 1
-process p4 -screen-dump
+# Simple process tests
+
+process p1 "cat" -start
+process p1 -writeln "foo"
+process p1 -expect-text 2 1 foo
+process p1 -stop
+process p1 -wait
+shell "grep -q foo ${p1_out}"
+shell "test -f ${p1_err} -a ! -s ${p1_err}"
+
+process p2 -log "cat" -start
+process p2 -writeln "bar"
+process p2 -expect-text 2 1 bar
+process p2 -write "\x04"
+process p2 -wait
+shell "grep -q bar ${p2_out}"
+shell "test -f ${p2_err} -a ! -s ${p2_err}"
+
+process p3 -dump "cat" -start
+process p3 -writeln "baz"
+process p3 -expect-text 2 1 baz
+process p3 -kill KILL
+process p3 -wait
+shell "grep -q baz ${p3_out}"
+shell "test -f ${p3_err} -a ! -s ${p3_err}"
+
+process p4 -hexdump "cat" -start
+process p4 -writeln "b\001z"
+process p4 -expect-text 2 1 "b"
+process p4 -kill TERM
+process p4 -wait -screen_dump
+
+# Curses process tests
+
+process p5 "ps -lw | grep '[p][s]' ; tty ; echo @" -start
+process p5 -expect-text 0 0 {@} -screen_dump -wait
+
+process p6 "stty -a ; echo '*'" -start
+process p6 -expect-text 0 0 {*} -screen_dump -wait
+
+process p7 -hexdump {stty raw -echo; stty -a ; echo "*" ; cat} -start
+process p7 -expect-text 0 0 "*" -screen_dump
+
+process p7 -write "\x1b[H\x1b[2Jzzzzzzz"
+process p7 -write "\x0c1\x1b[79C2\x08>\x1b[25;1H3\x1b[25;80H"
+process p7 -write "\x1b[H\x1b[2J1\x1b[79C2\x08>\x1b[25;1H3\x1b[25;80H"
+process p7 -write "4\x08>\x1b[A\x1b[Cv\x1b[22A^\x1b[79D^\x1b[;2H<\n\n\n\n"
+process p7 -write "\n\n\n\n\n\n\n\n\x1b[B\x1b[11B\x08<\x1b[24;Hv\x1b[12;1H"
+process p7 -write "111111112222222333333\x0d\x0a111111112"
+process p7 -write "222222333333\x0d\x0a111111112222222333333 UTF8: "
+process p7 -writehex {c2 a2 20}
+process p7 -writehex {e2 82 ac 20}
+process p7 -writehex {f0 90 80 80 20}
+process p7 -writehex {f0 9f 90 b0 20}
+process p7 -writehex {f0 a0 80 80 20}
+process p7 -writehex {f0 b0 80 80 20}
+process p7 -write "\x1b[22;24;25;27;30;47;49;97;107m"
+process p7 -write "\x1b[22;24;25;27;30m"
+process p7 -write "\x1b[47;49;97;107m"
+process p7 -write "\x0d\x0a111111112222222333333\x0d\x0a\x1b[12"
+process p7 -write ";12H\x1b[K\x1b[13;12H\x1b[0K\x1b[14;12H\x1b[1K\x1b"
+process p7 -write "[15;12H\x1b[2K\x1b[3;1Hline3 <\x0d\x0a"
+
+process p7 -need-bytes 310 -expect-text 3 1 "line3 <"
+process p7 -expect-cursor 4 1
+process p7 -expect-cursor 4 0
+process p7 -expect-cursor 0 1
+process p7 -screen-dump
 
 # Also exercise CONS25 mode
-process p4 -write "\x1b[=1T"
-process p4 -write "\x1b[=2T"
-process p4 -write "\x1b[8z"
-process p4 -write "\x1b[0x"
-process p4 -write "\x1b[=1A"
-process p4 -write "\x1b[=1;2B"
-process p4 -write "\x1b[=1;2;3C"
-process p4 -write "\x1b[=1;2;3;4C"
-process p4 -write "\x1b[=1F"
-process p4 -write "\x1b[=1G"
-process p4 -write "\x1b[=1S"
-process p4 -writehex {0c 08 40 0d 0a 08}
-
-process p4 -expect-text 1 1 "@"
-process p4 -expect-cursor 1 80
-process p4 -writehex "0c 41 0e 42 0f"
-process p4 -expect-text 1 1 "A"
-process p4 -expect-text 0 0 "B"
-process p4 -write "\x1b[=0T"
-
-process p4 -writehex "0c 0a 0d 43 0a 08 08 0e 44 0f"
-
-process p4 -expect-text 3 1 "C"
-process p4 -expect-text 4 1 "D"
-process p4 -write "\x1b[2T"
-process p4 -expect-text 5 1 "C"
-process p4 -expect-text 6 1 "D"
-process p4 -write "\x1b[3S"
-process p4 -expect-text 3 1 "D"
-
-process p4 -write "\x1b[4;200H%"
-process p4 -expect-text 4 80 "%"
-
-process p4 -write "\x1b[7;7H\x09X\x09Y\x09Z\x1b[2ZW\x1b[2Ew\x1b[F*"
-
-process p4 -expect-text 7 17 "W"
-process p4 -expect-text 9 1 "w"
-process p4 -expect-text 8 1 "*"
-
-process p4 -write "\x1b[10;4HABCDEFGHIJKLMN\x1b(A#$%\x1b)A"
-process p4 -write "\x1b[8G\x1b[2X>"
-process p4 -expect-text 10 8 ">"
-process p4 -screen-dump
+process p7 -write "\x1b[=1T"
+process p7 -write "\x1b[=2T"
+process p7 -write "\x1b[8z"
+process p7 -write "\x1b[0x"
+process p7 -write "\x1b[=1A"
+process p7 -write "\x1b[=1;2B"
+process p7 -write "\x1b[=1;2;3C"
+process p7 -write "\x1b[=1;2;3;4C"
+process p7 -write "\x1b[=1F"
+process p7 -write "\x1b[=1G"
+process p7 -write "\x1b[=1S"
+process p7 -writehex {0c 08 40 0d 0a 08}
+
+process p7 -expect-text 1 1 "@"
+process p7 -expect-cursor 1 80
+process p7 -writehex "0c 41 0e 42 0f"
+process p7 -expect-text 1 1 "A"
+process p7 -expect-text 0 0 "B"
+process p7 -write "\x1b[=0T"
+
+process p7 -writehex "0c 0a 0d 43 0a 08 08 0e 44 0f"
+
+process p7 -expect-text 3 1 "C"
+process p7 -expect-text 4 1 "D"
+process p7 -write "\x1b[2T"
+process p7 -expect-text 5 1 "C"
+process p7 -expect-text 6 1 "D"
+process p7 -write "\x1b[3S"
+process p7 -expect-text 3 1 "D"
+
+process p7 -write "\x1b[4;200H%"
+process p7 -expect-text 4 80 "%"
+
+process p7 -write "\x1b[7;7H\x09X\x09Y\x09Z\x1b[2ZW\x1b[2Ew\x1b[F*"
+
+process p7 -expect-text 7 17 "W"
+process p7 -expect-text 9 1 "w"
+process p7 -expect-text 8 1 "*"
+
+process p7 -write "\x1b[10;4HABCDEFGHIJKLMN\x1b(A#$%\x1b)A"
+process p7 -write "\x1b[8G\x1b[2X>"
+process p7 -expect-text 10 8 ">"
+process p7 -screen-dump
 
 # Test responses
-process p4 -write "\x1b[3;1HA\x1b[5n"
-process p4 -write "\x1b[4;1HB\x1b[6n"
-process p4 -write "\x1b[5;1HC\x1b[15n"
-process p4 -write "\x1b[6;1HD\x1b[25n"
-process p4 -write "\x1b[7;1HE\x1b[26n"
-process p4 -write "\x1b[8;1HF\x1b[?26n"
-process p4 -write "\x1b[9;1HG\x1bPfutfutfut\x01"
-process p4 -write "\x1b[10;1HH\x1b]futfutfut\x01"
-process p4 -write "\x1b[11;1HI\x1b[>c"
-process p4 -write "\x1b[24;1HW"
-process p4 -expect-text 24 1 "W"
-process p4 -screen-dump
+process p7 -write "\x1b[3;1HA\x1b[5n"
+process p7 -write "\x1b[4;1HB\x1b[6n"
+process p7 -write "\x1b[5;1HC\x1b[15n"
+process p7 -write "\x1b[6;1HD\x1b[25n"
+process p7 -write "\x1b[7;1HE\x1b[26n"
+process p7 -write "\x1b[8;1HF\x1b[?26n"
+process p7 -write "\x1b[9;1HG\x1bPfutfutfut\x01"
+process p7 -write "\x1b[10;1HH\x1b]futfutfut\x01"
+process p7 -write "\x1b[11;1HI\x1b[>c"
+process p7 -write "\x1b[24;1HW"
+process p7 -expect-text 24 1 "W"
+process p7 -screen-dump
diff --git a/bin/varnishtest/tests/a00009.vtc b/bin/varnishtest/tests/a00009.vtc
deleted file mode 100644
index 4b82d4c7d..000000000
--- a/bin/varnishtest/tests/a00009.vtc
+++ /dev/null
@@ -1,55 +0,0 @@
-varnishtest "Code coverage of mgt_main, (VCL compiler and RSTdump etc)"
-
-shell "varnishd -b 127.0.0.1:80 -C 2> ${tmpdir}/_.c"
-shell -err -expect {VCL version declaration missing} {
-	echo 'bad vcl' > ${tmpdir}/bad.vcl
-	varnishd -f ${tmpdir}/bad.vcl -n ${tmpdir}
-}
-
-shell -err -expect {-x must be the first argument} "varnishd -d -x foo "
-shell -err -expect {-V must be the first argument} "varnishd -d -V foo "
-
-shell -err -expect {Too many arguments for -x} "varnishd -x foo bar"
-shell -err -expect {Invalid -x argument} "varnishd -x foo "
-
-# This one is tricky, the getopt message on stderr is not standardized.
-shell -err -expect {option} "varnishd -A "
-
-shell -err -expect {Usage: varnishd [options]} "varnishd -? "
-shell -err -expect {Too many arguments} "varnishd foo "
-
-shell "varnishd -x parameter > ${tmpdir}/_.param"
-shell "varnishd -x vsl > ${tmpdir}/_.vsl"
-shell "varnishd -x cli > ${tmpdir}/_.cli"
-shell "varnishd -x builtin > ${tmpdir}/_.builtin"
-
-shell -err -expect {-C needs either -b <backend> or -f <vcl_file>} {
-	varnishd -C
-}
-shell -err -expect {Cannot open -S file} {
-	varnishd -S ${tmpdir}/nonexistent -n ${tmpdir}/v0 -f ''
-}
-shell -err -expect {Unknown jail method "xyz"} "varnishd -jxyz -f '' "
-
-shell -err -expect {Invalid backslash sequence} {
-	varnishd -l 'xyz\kk,xyz\foo' -f ''
-}
-shell -err -expect {Invalid backslash sequence} {
-	varnishd -l 'ab\8cd' -f ''
-}
-shell -err -expect {Too many arguments for -V} "varnishd -V -V"
-shell -expect {Copyright (c) 2006} "varnishd -V"
-
-shell -err -expect {Only one of -d or -F can be specified} "varnishd -d -F "
-shell -err -expect {Only one of -b or -f can be specified} "varnishd -b a -f b "
-shell -err -expect {-d makes no sense with -C} "varnishd -C -b 127.0.0.1:80 -d "
-shell -err -expect {-F makes no sense with -C} "varnishd -C -b 127.0.0.1:80 -F "
-shell -err -expect {Neither -b nor -f given} { varnishd -n ${tmpdir}/v0  }
-
-# This check is kept last because it may be skipped
-
-feature persistent_storage
-
-shell -err -expect {-spersistent has been deprecated} {
-	varnishd -spersistent -f ''
-}
diff --git a/bin/varnishtest/tests/a00016.vtc b/bin/varnishtest/tests/a00016.vtc
deleted file mode 100644
index 4055b7a39..000000000
--- a/bin/varnishtest/tests/a00016.vtc
+++ /dev/null
@@ -1,25 +0,0 @@
-varnishtest "Test -I and -l arguments"
-
-shell -err -expect {Only one -I allowed} {
-	touch foo bar
-	varnishd -f '' -I foo -I bar -n ${tmpdir}/v0 -a :0
-}
-
-shell -err -expect {Error: -I file CLI command failed (104)} {
-	echo "vcl.list" > foo
-	echo "-foobar" >> foo
-	echo "vcl.load" >> foo
-	varnishd -f '' -I foo -n ${tmpdir}/v0 -a :0 -l 2m
-}
-
-shell -err -expect {Error: -l ...: Missing '"'} {
-	varnishd -f '' -I foo -n ${tmpdir}/v0 -a :0 -l '2m,"'
-}
-
-shell -err -expect {Error: Too many sub arguments} {
-	varnishd -f '' -I foo -n ${tmpdir}/v0 -a :0 -l 2m,2m,2m
-}
-
-shell -err -expect {Warning: Ignoring deprecated second subargument} {
-	varnishd -f '' -I foo -n ${tmpdir}/v0 -a :0 -l 2m,2m
-}
diff --git a/bin/varnishtest/tests/a00017.vtc b/bin/varnishtest/tests/a00017.vtc
deleted file mode 100644
index 21b184cfa..000000000
--- a/bin/varnishtest/tests/a00017.vtc
+++ /dev/null
@@ -1,80 +0,0 @@
-varnishtest "Code coverage of mgt_main, (VCL compiler and RSTdump etc)"
-
-# Test -F mode with no VCL loaded
-
-process p1 "exec varnishd -n ${tmpdir}/v1 -F -f '' -a :0 -l2m,1m" -log -start
-
-delay 2
-
-shell {
-	(
-	echo 'vcl 4.0;'
-	echo 'backend default {'
-	echo '  .host="${bad_backend}";'
-	echo '}'
-	) > ${tmpdir}/vcl
-}
-
-shell -expect {VCL compiled.} {
-	varnishadm -n ${tmpdir}/v1 vcl.load vcl1 ${tmpdir}/vcl
-}
-
-shell -expect {active      auto/warm          - vcl1} {
-	varnishadm -n ${tmpdir}/v1 vcl.list
-}
-
-shell {varnishadm -n ${tmpdir}/v1 start}
-
-shell {varnishadm -n ${tmpdir}/v1 debug.listen_address}
-
-shell -exit 1 -expect "Command failed with error code 500" {
-	varnishadm -n ${tmpdir}/v1 quit
-}
-
-shell -exit 1 -expect "Command failed with error code 102" {
-	varnishadm -n ${tmpdir}/v1 debug.panic.master -j
-}
-
-shell -exit 1 -expect "Command failed with error code 101" {
-	varnishadm -n ${tmpdir}/v1 123
-}
-
-shell "varnishadm -n ${tmpdir}/v1 param.set cli_limit 128"
-shell -expect "[response was truncated]" "varnishadm -n ${tmpdir}/v1 help"
-
-process p1 -expect-exit 64 -stop -wait
-
-# Test multiple -f options
-
-shell {
-	cat >${tmpdir}/ok1 <<-EOF
-	vcl 4.0;
-	backend ok1 {
-	  .host="${bad_backend}";
-	}
-	EOF
-
-	cat >${tmpdir}/ok2 <<-EOF
-	vcl 4.0;
-	backend ok2 {
-	  .host="${bad_backend}";
-	}
-	EOF
-}
-
-varnish v2 -arg "-f ${tmpdir}/ok1" -arg "-f ${tmpdir}/ok2" -start
-varnish v2 -cliexpect {available *auto/warm *0 boot0} "vcl.list"
-varnish v2 -cliexpect {active *auto/warm *0 boot} "vcl.list"
-varnish v2 -stop -wait
-
-# Test multiple -f options with a bad VCL
-
-shell -err -expect {Cannot read -f file} {
-	exec varnishd -n ${tmpdir}/v0 -F -a :0 -l2m,3m -f ${tmpdir}/ok1 \
-	    -f ${tmpdir}/ok2 -f ${tmpdir}/bad
-}
-
-shell -err -expect {Cannot read -f file} {
-	exec varnishd -n ${tmpdir}/v0 -F -a :0 -l2m,3m -f ${tmpdir}/ok1 \
-	    -f ${tmpdir}/bad -f ${tmpdir}/ok2
-}
diff --git a/bin/varnishtest/tests/a00019.vtc b/bin/varnishtest/tests/a00019.vtc
deleted file mode 100644
index a6d1317d5..000000000
--- a/bin/varnishtest/tests/a00019.vtc
+++ /dev/null
@@ -1,33 +0,0 @@
-varnishtest "vtc v_* macros when the listen address is UDS"
-
-varnish v1 -arg "-a ${tmpdir}/v1.sock -b '${bad_backend}'" -start
-
-varnish v1 -syntax 4.0 -errvcl {Compiled VCL version (4.0) not supported.} {
-	backend default { .host = "${bad_ip}"; }
-}
-
-varnish v1 -syntax 4.0 -errvcl \
-	{Unix socket backends only supported in VCL4.1 and higher.} \
-	{backend default { .path = "${tmpdir}/v1.sock"; }}
-
-varnish v2 -vcl {
-	backend default { .host = "${bad_ip}"; }
-
-	sub vcl_recv {
-		return(synth(200));
-	}
-
-	sub vcl_synth {
-		set resp.http.addr = "${v1_addr}";
-		set resp.http.port = "${v1_port}";
-		set resp.http.sock = "${v1_sock}";
-	}
-} -start
-
-client c1 -connect ${v2_sock} {
-	txreq
-	rxresp
-	expect resp.http.addr == "${tmpdir}/v1.sock"
-	expect resp.http.port == "-"
-	expect resp.http.sock == "${tmpdir}/v1.sock -"
-} -run
diff --git a/bin/varnishtest/tests/a00020.vtc b/bin/varnishtest/tests/a00020.vtc
deleted file mode 100644
index aba786e2e..000000000
--- a/bin/varnishtest/tests/a00020.vtc
+++ /dev/null
@@ -1,34 +0,0 @@
-varnishtest "vtc remote.ip, remote.port and remote.path"
-
-server s1 {
-	rxreq
-	expect remote.ip == "${localhost}"
-	expect remote.port > 0
-	expect remote.path == <undef>
-	txresp
-} -start
-
-varnish v1 -vcl+backend {} -start
-
-client c1 {
-	txreq
-	rxresp
-	expect remote.ip == "${v1_addr}"
-	expect remote.port == "${v1_port}"
-	expect remote.path == <undef>
-} -run
-
-varnish v1 -stop
-
-server s1 -wait
-server s1 -start
-
-varnish v2 -arg "-a ${tmpdir}/v2.sock" -vcl+backend {} -start
-
-client c1 -connect "${tmpdir}/v2.sock" {
-	txreq
-	rxresp
-	expect remote.ip == "0.0.0.0"
-	expect remote.port == 0
-	expect remote.path == "${tmpdir}/v2.sock"
-} -run
diff --git a/bin/varnishtest/tests/b00053.vtc b/bin/varnishtest/tests/b00053.vtc
index 0acb9daec..adcec5974 100644
--- a/bin/varnishtest/tests/b00053.vtc
+++ b/bin/varnishtest/tests/b00053.vtc
@@ -32,3 +32,37 @@ varnish v1 -expect cache_miss == 1
 varnish v1 -expect s_sess == 1
 varnish v1 -expect s_resp_bodybytes == 7
 varnish v1 -expect s_resp_hdrbytes == 178
+
+# varnishtest "vtc v_* macros when the listen address is UDS" (a00019)
+
+varnish v2 -arg "-a ${tmpdir}/v1.sock -b '${bad_backend}'" -start
+
+varnish v2 -syntax 4.0 -errvcl {Compiled VCL version (4.0) not supported.} {
+	backend default { .host = "${bad_ip}"; }
+}
+
+varnish v2 -syntax 4.0 -errvcl \
+	{Unix socket backends only supported in VCL4.1 and higher.} \
+	{backend default { .path = "${tmpdir}/v1.sock"; }}
+
+varnish v3 -vcl {
+	backend default { .host = "${bad_ip}"; }
+
+	sub vcl_recv {
+		return(synth(200));
+	}
+
+	sub vcl_synth {
+		set resp.http.addr = "${v1_addr}";
+		set resp.http.port = "${v1_port}";
+		set resp.http.sock = "${v1_sock}";
+	}
+} -start
+
+client c1 -connect ${v3_sock} {
+	txreq
+	rxresp
+	expect resp.http.addr == "${tmpdir}/v1.sock"
+	expect resp.http.port == "-"
+	expect resp.http.sock == "${tmpdir}/v1.sock -"
+} -run
diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc
index 7e426ec96..ae91b169f 100644
--- a/bin/varnishtest/tests/u00000.vtc
+++ b/bin/varnishtest/tests/u00000.vtc
@@ -1,32 +1,162 @@
-varnishtest "Simple process tests"
-
-process p1 "cat" -start
-process p1 -writeln "foo"
-process p1 -expect-text 2 1 foo
-process p1 -stop
-process p1 -wait
-shell "grep -q foo ${p1_out}"
-shell "test -f ${p1_err} -a ! -s ${p1_err}"
-
-process p2 -log "cat" -start
-process p2 -writeln "bar"
-process p2 -expect-text 2 1 bar
-process p2 -write "\x04"
-process p2 -wait
-shell "grep -q bar ${p2_out}"
-shell "test -f ${p2_err} -a ! -s ${p2_err}"
-
-process p3 -dump "cat" -start
-process p3 -writeln "baz"
-process p3 -expect-text 2 1 baz
-process p3 -kill KILL
-process p3 -wait
-shell "grep -q baz ${p3_out}"
-shell "test -f ${p3_err} -a ! -s ${p3_err}"
-
-process p4 -hexdump "cat" -start
-process p4 -writeln "b\001z"
-process p4 -expect-text 2 1 "b"
-process p4 -kill TERM
-process p4 -wait -screen_dump
+varnishtest "Code coverage of mgt_main, (VCL compiler and RSTdump etc)"
 
+shell "varnishd -b 127.0.0.1:80 -C 2> ${tmpdir}/_.c"
+shell -err -expect {VCL version declaration missing} {
+	echo 'bad vcl' > ${tmpdir}/bad.vcl
+	varnishd -f ${tmpdir}/bad.vcl -n ${tmpdir}
+}
+
+shell -err -expect {-x must be the first argument} "varnishd -d -x foo "
+shell -err -expect {-V must be the first argument} "varnishd -d -V foo "
+
+shell -err -expect {Too many arguments for -x} "varnishd -x foo bar"
+shell -err -expect {Invalid -x argument} "varnishd -x foo "
+
+# This one is tricky, the getopt message on stderr is not standardized.
+shell -err -expect {option} "varnishd -A "
+
+shell -err -expect {Usage: varnishd [options]} "varnishd -? "
+shell -err -expect {Too many arguments} "varnishd foo "
+
+shell "varnishd -x parameter > ${tmpdir}/_.param"
+shell "varnishd -x vsl > ${tmpdir}/_.vsl"
+shell "varnishd -x cli > ${tmpdir}/_.cli"
+shell "varnishd -x builtin > ${tmpdir}/_.builtin"
+
+shell -err -expect {-C needs either -b <backend> or -f <vcl_file>} {
+	varnishd -C
+}
+shell -err -expect {Cannot open -S file} {
+	varnishd -S ${tmpdir}/nonexistent -n ${tmpdir}/v0 -f ''
+}
+shell -err -expect {Unknown jail method "xyz"} "varnishd -jxyz -f '' "
+
+shell -err -expect {Invalid backslash sequence} {
+	varnishd -l 'xyz\kk,xyz\foo' -f ''
+}
+shell -err -expect {Invalid backslash sequence} {
+	varnishd -l 'ab\8cd' -f ''
+}
+shell -err -expect {Too many arguments for -V} "varnishd -V -V"
+shell -expect {Copyright (c) 2006} "varnishd -V"
+
+shell -err -expect {Only one of -d or -F can be specified} "varnishd -d -F "
+shell -err -expect {Only one of -b or -f can be specified} "varnishd -b a -f b "
+shell -err -expect {-d makes no sense with -C} "varnishd -C -b 127.0.0.1:80 -d "
+shell -err -expect {-F makes no sense with -C} "varnishd -C -b 127.0.0.1:80 -F "
+shell -err -expect {Neither -b nor -f given} { varnishd -n ${tmpdir}/v0  }
+
+# This check is kept last because it may be skipped
+
+feature persistent_storage
+
+shell -err -expect {-spersistent has been deprecated} {
+	varnishd -spersistent -f ''
+}
+
+# Test -I and -l arguments (former a00016)
+
+shell -err -expect {Only one -I allowed} {
+	touch foo bar
+	varnishd -f '' -I foo -I bar -n ${tmpdir}/v0 -a :0
+}
+
+shell -err -expect {Error: -I file CLI command failed (104)} {
+	echo "vcl.list" > foo
+	echo "-foobar" >> foo
+	echo "vcl.load" >> foo
+	varnishd -f '' -I foo -n ${tmpdir}/v0 -a :0 -l 2m
+}
+
+shell -err -expect {Error: -l ...: Missing '"'} {
+	varnishd -f '' -I foo -n ${tmpdir}/v0 -a :0 -l '2m,"'
+}
+
+shell -err -expect {Error: Too many sub arguments} {
+	varnishd -f '' -I foo -n ${tmpdir}/v0 -a :0 -l 2m,2m,2m
+}
+
+shell -err -expect {Warning: Ignoring deprecated second subargument} {
+	varnishd -f '' -I foo -n ${tmpdir}/v0 -a :0 -l 2m,2m
+}
+
+# Code coverage of mgt_main, (VCL compiler and RSTdump etc) (former a00017)
+
+# Test -F mode with no VCL loaded
+
+process p1 "exec varnishd -n ${tmpdir}/v1 -F -f '' -a :0 -l2m,1m" -log -start
+
+delay 2
+
+shell {
+	(
+	echo 'vcl 4.0;'
+	echo 'backend default {'
+	echo '  .host="${bad_backend}";'
+	echo '}'
+	) > ${tmpdir}/vcl
+}
+
+shell -expect {VCL compiled.} {
+	varnishadm -n ${tmpdir}/v1 vcl.load vcl1 ${tmpdir}/vcl
+}
+
+shell -expect {active      auto/warm          - vcl1} {
+	varnishadm -n ${tmpdir}/v1 vcl.list
+}
+
+shell {varnishadm -n ${tmpdir}/v1 start}
+
+shell {varnishadm -n ${tmpdir}/v1 debug.listen_address}
+
+shell -exit 1 -expect "Command failed with error code 500" {
+	varnishadm -n ${tmpdir}/v1 quit
+}
+
+shell -exit 1 -expect "Command failed with error code 102" {
+	varnishadm -n ${tmpdir}/v1 debug.panic.master -j
+}
+
+shell -exit 1 -expect "Command failed with error code 101" {
+	varnishadm -n ${tmpdir}/v1 123
+}
+
+shell "varnishadm -n ${tmpdir}/v1 param.set cli_limit 128"
+shell -expect "[response was truncated]" "varnishadm -n ${tmpdir}/v1 help"
+
+process p1 -expect-exit 64 -stop -wait
+
+# Test multiple -f options
+
+shell {
+	cat >${tmpdir}/ok1 <<-EOF
+	vcl 4.0;
+	backend ok1 {
+	  .host="${bad_backend}";
+	}
+	EOF
+
+	cat >${tmpdir}/ok2 <<-EOF
+	vcl 4.0;
+	backend ok2 {
+	  .host="${bad_backend}";
+	}
+	EOF
+}
+
+varnish v2 -arg "-f ${tmpdir}/ok1" -arg "-f ${tmpdir}/ok2" -start
+varnish v2 -cliexpect {available *auto/warm *0 boot0} "vcl.list"
+varnish v2 -cliexpect {active *auto/warm *0 boot} "vcl.list"
+varnish v2 -stop -wait
+
+# Test multiple -f options with a bad VCL
+
+shell -err -expect {Cannot read -f file} {
+	exec varnishd -n ${tmpdir}/v0 -F -a :0 -l2m,3m -f ${tmpdir}/ok1 \
+	    -f ${tmpdir}/ok2 -f ${tmpdir}/bad
+}
+
+shell -err -expect {Cannot read -f file} {
+	exec varnishd -n ${tmpdir}/v0 -F -a :0 -l2m,3m -f ${tmpdir}/ok1 \
+	    -f ${tmpdir}/bad -f ${tmpdir}/ok2
+}
diff --git a/bin/varnishtest/tests/v00054.vtc b/bin/varnishtest/tests/v00054.vtc
index a230fb867..c92e8cce6 100644
--- a/bin/varnishtest/tests/v00054.vtc
+++ b/bin/varnishtest/tests/v00054.vtc
@@ -1,6 +1,40 @@
 varnishtest "client.identity is 0.0.0.0 if unset & client addr is UDS"
+# varnishtest "vtc remote.ip, remote.port and remote.path" (a00020)
 
-varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl {
+server s1 {
+	rxreq
+	expect remote.ip == "${localhost}"
+	expect remote.port > 0
+	expect remote.path == <undef>
+	txresp
+} -start
+
+varnish v1 -vcl+backend {} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect remote.ip == "${v1_addr}"
+	expect remote.port == "${v1_port}"
+	expect remote.path == <undef>
+} -run
+
+varnish v1 -stop
+
+server s1 -wait
+server s1 -start
+
+varnish v2 -arg "-a ${tmpdir}/v2.sock" -vcl+backend {} -start
+
+client c1 -connect "${tmpdir}/v2.sock" {
+	txreq
+	rxresp
+	expect remote.ip == "0.0.0.0"
+	expect remote.port == 0
+	expect remote.path == "${tmpdir}/v2.sock"
+} -run
+
+varnish v3 -arg "-a ${tmpdir}/v3.sock" -vcl {
 	backend b { .host = "${bad_ip}"; }
 
 	sub vcl_recv {
@@ -16,7 +50,7 @@ varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl {
 	}
 } -start
 
-client c1 -connect "${tmpdir}/v1.sock" {
+client c2 -connect "${tmpdir}/v3.sock" {
 	txreq -url "/nobody"
 	rxresp
 	expect resp.status == 200

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:20 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:20 +0000 (UTC)
Subject: [6.0] 3679ddc16 Make "vtest" an alias for "varnishtest" in the first
 line.
Message-ID: <20181031130820.7AAF995439@lists.varnish-cache.org>


commit 3679ddc162d6d8cd5aaffa1067d8fe834a522629
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 07:16:10 2018 +0000

    Make "vtest" an alias for "varnishtest" in the first line.

diff --git a/bin/varnishtest/cmds.h b/bin/varnishtest/cmds.h
index e856e6084..d3244c258 100644
--- a/bin/varnishtest/cmds.h
+++ b/bin/varnishtest/cmds.h
@@ -43,6 +43,7 @@ CMD(shell)
 CMD(syslog)
 CMD(varnish)
 CMD(varnishtest)
+CMD(vtest)
 #undef CMD
 
 /*lint -restore */
diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc
index 22e9f341b..8fb9ded32 100644
--- a/bin/varnishtest/tests/a00000.vtc
+++ b/bin/varnishtest/tests/a00000.vtc
@@ -10,7 +10,7 @@ shell {
 	echo 'shell "exit 9"' >> _.vtc
 }
 
-shell -exit 2 -expect {doesn't start with 'varnishtest'} {
+shell -exit 2 -expect {doesn't start with 'vtest' or 'varnishtest'} {
 	varnishtest -v _.vtc
 }
 
diff --git a/bin/varnishtest/tests/a00001.vtc b/bin/varnishtest/tests/a00001.vtc
index b6628d807..ea5b458f3 100644
--- a/bin/varnishtest/tests/a00001.vtc
+++ b/bin/varnishtest/tests/a00001.vtc
@@ -1,4 +1,4 @@
-varnishtest "Test Teken terminal emulator"
+vtest "Test Teken terminal emulator"
 
 feature cmd "vttest --version 2>&1 | grep -q Usage"
 
diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index d6168a2ae..cb0c2f07f 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -547,9 +547,11 @@ read_file(const char *fn, int ntest)
 		return (2);
 	}
 
-	if (strncmp(q, "varnishtest", 11) || !isspace(q[11])) {
+	if ((strncmp(q, "varnishtest", 11) || !isspace(q[11])) &&
+	    (strncmp(q, "vtest", 5) || !isspace(q[5]))) {
 		fprintf(stderr,
-		    "File \"%s\" doesn't start with 'varnishtest'\n", fn);
+		    "File \"%s\" doesn't start with"
+		    " 'vtest' or 'varnishtest'\n", fn);
 		free(p);
 		vtc_skip++;
 		return(2);
diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c
index 196b35c65..0c47fe88a 100644
--- a/bin/varnishtest/vtc_misc.c
+++ b/bin/varnishtest/vtc_misc.c
@@ -49,17 +49,39 @@
 #include "vre.h"
 #include "vtim.h"
 
-/* SECTION: varnishtest varnishtest
+/* SECTION: vtest vtest
  *
  * This should be the first command in your vtc as it will identify the test
  * case with a short yet descriptive sentence. It takes exactly one argument, a
  * string, eg::
  *
- *         varnishtest "Check that varnishtest is actually a valid command"
+ *         vtest "Check that vtest is actually a valid command"
  *
  * It will also print that string in the log.
  */
 
+void v_matchproto_(cmd_f)
+cmd_vtest(CMD_ARGS)
+{
+
+	(void)priv;
+	(void)cmd;
+	(void)vl;
+
+	if (av == NULL)
+		return;
+	AZ(strcmp(av[0], "vtest"));
+
+	vtc_log(vl, 1, "TEST %s", av[1]);
+	AZ(av[2]);
+}
+
+/* SECTION: varnishtest varnishtest
+ *
+ * Alternate name for 'vtest', see above.
+ *
+ */
+
 void v_matchproto_(cmd_f)
 cmd_varnishtest(CMD_ARGS)
 {

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:20 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:20 +0000 (UTC)
Subject: [6.0] f3c3a7bad Whitespace OCD
Message-ID: <20181031130820.A2B7D95451@lists.varnish-cache.org>


commit f3c3a7bad3447e4db7ea26f4a5e3979f76300912
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 07:17:52 2018 +0000

    Whitespace OCD

diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c
index d7ceefe89..94bee3d3f 100644
--- a/bin/varnishtest/vtc_log.c
+++ b/bin/varnishtest/vtc_log.c
@@ -155,7 +155,7 @@ vtc_log_emit(const struct vtclog *vl)
 		return;
 	AZ(pthread_mutex_lock(&vtclog_mtx));
 	assert(vtclog_left > l);
-	memcpy(vtclog_buf,VSB_data(vl->vsb), l);
+	memcpy(vtclog_buf, VSB_data(vl->vsb), l);
 	vtclog_buf += l;
 	*vtclog_buf = '\0';
 	vtclog_left -= l;
diff --git a/include/vsa.h b/include/vsa.h
index e5e342a26..67c3e6266 100644
--- a/include/vsa.h
+++ b/include/vsa.h
@@ -59,7 +59,7 @@ struct suckaddr *VSA_Build(void *d, const void *s, unsigned sal);
  * This VRT interface is for the VCC generated ACL code, which needs
  * to know the address family and a pointer to the actual address.
  */
-  
+
 int VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst);
 
 #endif

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:20 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:20 +0000 (UTC)
Subject: [6.0] c39227a60 A train-trip worth of python3 migration and other
 polishing
Message-ID: <20181031130820.F2B9C95492@lists.varnish-cache.org>


commit c39227a60719c3cac4c8a8557c89bf5565ed3504
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 07:26:11 2018 +0000

    A train-trip worth of python3 migration and other polishing
    
    Conflicts:
            lib/libvcc/generate.py

diff --git a/bin/varnishtest/huffman_gen.py b/bin/varnishtest/huffman_gen.py
index 8da37b2ee..b89bbb43e 100755
--- a/bin/varnishtest/huffman_gen.py
+++ b/bin/varnishtest/huffman_gen.py
@@ -7,50 +7,50 @@ import sys
 regex = re.compile("^HPH\((.{4}), (.{10}), +(.{1,3})\)")
 
 if len(sys.argv) != 2:
-	print("{} takes one and only one argument".format(sys.argv[0]))
-	sys.exit(2)
+    print("{} takes one and only one argument".format(sys.argv[0]))
+    sys.exit(2)
 
 class sym:
-	def __init__(self, bigval, bigvall, chr = 0, esc = None):
-		self.vall = bigvall % 8 if bigvall % 8 else 8
-		self.val = bigval & ((1 << self.vall) - 1)
-		self.pfx = (bigval >> self.vall)# & 0xff
-		self.chr = chr
-		self.esc = esc
+    def __init__(self, bigval, bigvall, chr=0, esc=None):
+        self.vall = bigvall % 8 if bigvall % 8 else 8
+        self.val = bigval & ((1 << self.vall) - 1)
+        self.pfx = (bigval >> self.vall)# & 0xff
+        self.chr = chr
+        self.esc = esc
 
 tbls = {}
 msl = {} # max sym length
 
 f = open(sys.argv[1])
 for l in f:
-	grp = 1
-	match = regex.match(l)
-	if not match:
-		continue
-
-	chr = int(match.group(grp), 16)
-	grp += 1
-
-	val = int(match.group(grp), 16)
-	grp += 1
-
-	vall = int(match.group(grp))
-
-	s = sym(val, vall, chr)
-	if s.pfx not in tbls:
-		tbls[s.pfx] = {}
-
-	if (s.val in tbls[s.pfx]):
-		assert(tbls[s.pfx][s.val].e)
-	tbls[s.pfx][s.val] = s
-
-	# add the escape entry in the "previous" table
-	if s.pfx:
-		pp = s.pfx >> 8
-		pv = s.pfx & 0xff
-		if pp not in tbls:
-			tbls[pp] = {}
-		tbls[pp][pv] = sym(pv, 8, 0, "&tbl_{:x}".format(s.pfx))
+    grp = 1
+    match = regex.match(l)
+    if not match:
+        continue
+
+    char = int(match.group(grp), 16)
+    grp += 1
+
+    val = int(match.group(grp), 16)
+    grp += 1
+
+    vall = int(match.group(grp))
+
+    s = sym(val, vall, char)
+    if s.pfx not in tbls:
+        tbls[s.pfx] = {}
+
+    if s.val in tbls[s.pfx]:
+        assert tbls[s.pfx][s.val].e
+    tbls[s.pfx][s.val] = s
+
+    # add the escape entry in the "previous" table
+    if s.pfx:
+        pp = s.pfx >> 8
+        pv = s.pfx & 0xff
+        if pp not in tbls:
+            tbls[pp] = {}
+        tbls[pp][pv] = sym(pv, 8, 0, "&tbl_{:x}".format(s.pfx))
 f.close()
 
 # add the EOS case
@@ -64,33 +64,33 @@ print('''/* NB:  This file is machine generated, DO NOT EDIT!
 struct stbl;
 
 struct ssym {
-	uint8_t csm;		/* bits consumed */
-	uint8_t	chr;		/* character */
-	struct stbl *nxt;	/* next table */
+    uint8_t csm;        /* bits consumed */
+    uint8_t chr;        /* character */
+    struct stbl *nxt;       /* next table */
 };
 
 struct stbl {
-	int msk;
-	struct ssym *syms;
+    int msk;
+    struct ssym *syms;
 };
 ''')
 
 for pfx in sorted(tbls.keys(), reverse=True):
-	msl = max([ x.vall for x in tbls[pfx].values() ])
-	for s in tbls[pfx].values():
-		s.val = s.val << (msl - s.vall)
-
-	tbl = sorted(tbls[pfx].values(), key= lambda x: x.val)
-	print("\nstatic struct ssym sym_{:x}_array[] = {{".format(pfx))
-	for s in tbl:
-		for j in range(2 ** (msl - s.vall)):
-			print("	{} {{{}, {:3d}, {}}},".format(
-				"\t     " if j else "/* idx {:3d} */".format(s.val + j),
-				s.vall, s.chr % 256,
-				s.esc if s.esc else "NULL"))
-	print('''}};
+    msl = max([x.vall for x in tbls[pfx].values()])
+    for s in tbls[pfx].values():
+        s.val = s.val << (msl - s.vall)
+
+    tbl = sorted(tbls[pfx].values(), key=lambda x: x.val)
+    print("\nstatic struct ssym sym_{:x}_array[] = {{".format(pfx))
+    for s in tbl:
+        for j in range(2 ** (msl - s.vall)):
+            print("{} {{{}, {:3d}, {}}},".format(
+                "\t     " if j else "/* idx {:3d} */".format(s.val + j),
+                s.vall, s.chr % 256,
+                s.esc if s.esc else "NULL"))
+    print('''}};
 
 static struct stbl tbl_{:x} = {{
-	{},
-	sym_{:x}_array
+    {},
+    sym_{:x}_array
 }};'''.format(pfx, msl, pfx))
diff --git a/bin/varnishtest/witness.py b/bin/varnishtest/witness.py
index f57fdf221..a4f0fffa6 100644
--- a/bin/varnishtest/witness.py
+++ b/bin/varnishtest/witness.py
@@ -3,9 +3,9 @@
 # This script is in the public domain
 #
 # Run instructions:
-#	varnishtest -W -iv -j <pick_a_number> > _.w
-#	python witness.py
-#	dot -Tpng /tmp/_.dot > /tmp/_.png
+#       varnishtest -W -iv -j <pick_a_number> > _.w
+#       python witness.py
+#       dot -Tpng /tmp/_.dot > /tmp/_.png
 
 from __future__ import print_function
 
@@ -16,37 +16,37 @@ fi = open("_.w")
 fo = open("/tmp/_.dot", "w")
 
 fo.write('''digraph {
-	#rotate="90"
-	#page="8.2,11.7"
-	size="8.2,11.7"
-	rankdir="LR"
-	node [fontname="Inconsolata", fontsize="10"]
-	edge [fontname="Inconsolata", fontsize="10"]
+    #rotate="90"
+    #page="8.2,11.7"
+    size="8.2,11.7"
+    rankdir="LR"
+    node [fontname="Inconsolata", fontsize="10"]
+    edge [fontname="Inconsolata", fontsize="10"]
 ''')
 
 for i in fi:
-	l = "ROOT"
-	j = i.split()
-	if len(j) < 8:
-		continue
-	if j[1][0] != 'v':
-		continue
-	if j[3] != "vsl|":
-		continue
-	if j[5] != "Witness":
-		continue
-	t = j[7:]
-	tt = str(t)
-	if tt in d:
-		continue
-	d[tt] = True
-	for e in t:
-		f = e.split(",")
-		x = '"%s" -> "%s" [label="%s(%s)"]\n' % (l, f[0], f[1], f[2])
-		if not x in a:
-			a[x] = True
-			fo.write(x)
-		l = f[0]
+    l = "ROOT"
+    j = i.split()
+    if len(j) < 8:
+        continue
+    if j[1][0] != 'v':
+        continue
+    if j[3] != "vsl|":
+        continue
+    if j[5] != "Witness":
+        continue
+    t = j[7:]
+    tt = str(t)
+    if tt in d:
+        continue
+    d[tt] = True
+    for e in t:
+        f = e.split(",")
+        x = '"%s" -> "%s" [label="%s(%s)"]\n' % (l, f[0], f[1], f[2])
+        if not x in a:
+            a[x] = True
+            fo.write(x)
+        l = f[0]
 
 fo.write("}\n")
 
diff --git a/doc/sphinx/vtc-syntax.py b/doc/sphinx/vtc-syntax.py
index 298f4bb01..ba7bd6bd7 100644
--- a/doc/sphinx/vtc-syntax.py
+++ b/doc/sphinx/vtc-syntax.py
@@ -35,50 +35,49 @@ import re
 
 
 def parse_file(fn, cl, tl, sl):
-	p = False
-	section = ""
-	resec = re.compile("[ /]\* SECTION: ")
+    p = False
+    section = ""
+    resec = re.compile("[ /]\* SECTION: ")
 
-	f = open(fn, "r")
+    f = open(fn, "r")
 
-	for l in f:
-		if "*/" in l:
-			p = 0
-		if resec.match(l):
-			a = l.split()
-			section = a[2]
-			sl.append(section)
-			cl[section] = []
-			if len(a) > 3:
-				tl[section] = re.sub(
-					r"^[\t ]*\/?\* SECTION: [^ ]+ +",
-					"", l)
-			else:
-				tl[section] = ""
-			p = 1
-		elif p:
-			cl[section].append(re.sub(r"^ \* ?", "", l))
-	f.close()
+    for l in f:
+        if "*/" in l:
+            p = 0
+        if resec.match(l):
+            a = l.split()
+            section = a[2]
+            sl.append(section)
+            cl[section] = []
+            if len(a) > 3:
+                tl[section] = re.sub(
+                    r"^[\t ]*\/?\* SECTION: [^ ]+ +",
+                    "", l)
+            else:
+                tl[section] = ""
+            p = 1
+        elif p:
+            cl[section].append(re.sub(r"^ \* ?", "", l))
+    f.close()
 
 if __name__ == "__main__":
-	cl = {}
-	tl = {}
-	sl = []
-	for fn in sys.argv[1:]:
-		parse_file(fn, cl, tl, sl)
-	sl.sort()
-	for section in sl:
-		print(tl[section], end="")
-		a = section
-		c = section.count(".")
-		if c == 0:
-			r = "-"
-		elif c == 1:
-			r = "~"
-		elif c == 2:
-			r = "."
-		else:
-			r = "*"
-		print(re.sub(r".", r, tl[section]), end="")
-		print("".join(cl[section]))
-
+    cl = {}
+    tl = {}
+    sl = []
+    for fn in sys.argv[1:]:
+        parse_file(fn, cl, tl, sl)
+    sl.sort()
+    for section in sl:
+        print(tl[section], end="")
+        a = section
+        c = section.count(".")
+        if c == 0:
+            r = "-"
+        elif c == 1:
+            r = "~"
+        elif c == 2:
+            r = "."
+        else:
+            r = "*"
+        print(re.sub(r".", r, tl[section]), end="")
+        print("".join(cl[section]))
diff --git a/include/Makefile.am b/include/Makefile.am
index 27f40ec86..98050fd78 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -108,7 +108,7 @@ vcl.h: \
 	    $(top_srcdir)/include/vrt.h \
 	    $(top_srcdir)/doc/sphinx/reference/vcl_var.rst
 	mkdir -p $(top_builddir)/include/tbl
-	@PYTHON@ $(top_srcdir)/lib/libvcc/generate.py \
+	${PYTHON} $(top_srcdir)/lib/libvcc/generate.py \
 	    $(top_srcdir) $(top_builddir)
 
 GEN_H = \
@@ -128,7 +128,7 @@ vcs_version.h:
 	@if test -e $(top_srcdir)/.git || \
 	  ! test -f $(srcdir)/vmod_abi.h || \
 	  ! test -f $(srcdir)/vcs_version.h ; then \
-	    @PYTHON@ $(srcdir)/generate.py \
+	    ${PYTHON} $(srcdir)/generate.py \
 		$(top_srcdir) $(top_builddir) ; \
 	fi
 
diff --git a/include/generate.py b/include/generate.py
index 0d9d9b32b..e7d547593 100755
--- a/include/generate.py
+++ b/include/generate.py
@@ -32,22 +32,21 @@
 from __future__ import print_function
 
 import subprocess
-import collections
 import os
 import sys
 
 srcroot = "../.."
 buildroot = "../.."
 if len(sys.argv) == 3:
-	srcroot = sys.argv[1]
-	buildroot = sys.argv[2]
+    srcroot = sys.argv[1]
+    buildroot = sys.argv[2]
 elif len(sys.argv) != 1:
-	print("Two arguments or none")
-	exit(2)
+    print("Two arguments or none")
+    exit(2)
 
 #######################################################################
 def file_header(fo):
-	fo.write("""/*
+    fo.write("""/*
  * NB:  This file is machine generated, DO NOT EDIT!
  *
  * Edit and run include/generate.py instead.
@@ -66,25 +65,25 @@ v = subprocess.check_output([
 vcsfn = os.path.join(srcroot, "include", "vcs_version.h")
 
 try:
-	i = open(vcsfn).readline()
+    i = open(vcsfn).readline()
 except IOError:
-	i = ""
+    i = ""
 
 ident = "/* " + v + " */\n"
 
 if i != ident:
-	fo = open(vcsfn, "w")
-	fo.write(ident)
-	file_header(fo)
-	fo.write('#define VCS_Version "%s"\n' % v)
-	fo.close()
+    fo = open(vcsfn, "w")
+    fo.write(ident)
+    file_header(fo)
+    fo.write('#define VCS_Version "%s"\n' % v)
+    fo.close()
 
-	for i in open(os.path.join(buildroot, "Makefile")):
-		if i[:14] == "PACKAGE_STRING":
-			break
-	i = i.split("=")[1].strip()
+    for i in open(os.path.join(buildroot, "Makefile")):
+        if i[:14] == "PACKAGE_STRING":
+            break
+    i = i.split("=")[1].strip()
 
-	fo = open(os.path.join(srcroot, "include", "vmod_abi.h"), "w")
-	file_header(fo)
-	fo.write('#define VMOD_ABI_Version "%s %s"\n' % (i, v))
-	fo.close()
+    fo = open(os.path.join(srcroot, "include", "vmod_abi.h"), "w")
+    file_header(fo)
+    fo.write('#define VMOD_ABI_Version "%s %s"\n' % (i, v))
+    fo.close()
diff --git a/include/tbl/h2_settings.h b/include/tbl/h2_settings.h
index adfb416d4..c382e862d 100644
--- a/include/tbl/h2_settings.h
+++ b/include/tbl/h2_settings.h
@@ -47,6 +47,7 @@ H2_SETTING(					// rfc7540,l,2097,2103
 	0xffffffff,
 	0
 )
+
 #ifndef H2_SETTINGS_PARAM_ONLY
 H2_SETTING(					// rfc7540,l,2105,2114
 	ENABLE_PUSH,
@@ -58,6 +59,7 @@ H2_SETTING(					// rfc7540,l,2105,2114
 	H2CE_PROTOCOL_ERROR
 )
 #endif
+
 H2_SETTING(					// rfc7540,l,2116,2121
 	MAX_CONCURRENT_STREAMS,
 	max_concurrent_streams,
@@ -67,6 +69,7 @@ H2_SETTING(					// rfc7540,l,2116,2121
 	0xffffffff,
 	0
 )
+
 H2_SETTING(					// rfc7540,l,2139,2148
 	INITIAL_WINDOW_SIZE,
 	initial_window_size,
@@ -76,6 +79,7 @@ H2_SETTING(					// rfc7540,l,2139,2148
 	0x7fffffff,
 	H2CE_FLOW_CONTROL_ERROR
 )
+
 H2_SETTING(					// rfc7540,l,2150,2157
 	MAX_FRAME_SIZE,
 	max_frame_size,
@@ -85,6 +89,7 @@ H2_SETTING(					// rfc7540,l,2150,2157
 	0x00ffffff,
 	H2CE_PROTOCOL_ERROR
 )
+
 H2_SETTING(					// rfc7540,l,2159,2167
 	MAX_HEADER_LIST_SIZE,
 	max_header_list_size,
diff --git a/include/tbl/htc.h b/include/tbl/htc.h
index a93c0e520..4e9695491 100644
--- a/include/tbl/htc.h
+++ b/include/tbl/htc.h
@@ -36,10 +36,11 @@ HTC_STATUS(JUNK,	-5,	"junk",		"Received unexpected data")
 HTC_STATUS(CLOSE,	-4,	"close",	"Connection closed") // unused?
 HTC_STATUS(TIMEOUT,	-3,	"timeout",	"Timed out")
 HTC_STATUS(OVERFLOW,	-2,	"overflow",	"Buffer/workspace too small")
-HTC_STATUS(EOF,	-1,	"eof",		"Unexpected end of input")
+HTC_STATUS(EOF,		-1,	"eof",		"Unexpected end of input")
 HTC_STATUS(EMPTY,	 0,	"empty",	"Empty response")
-HTC_STATUS(MORE,	 1,	"more",	"More data required")
+HTC_STATUS(MORE,	 1,	"more",		"More data required")
 HTC_STATUS(COMPLETE,	 2,	"complete",	"Data complete (no error)")
-HTC_STATUS(IDLE,	 3,	"idle",	"Connection was closed while idle")
+HTC_STATUS(IDLE,	 3,	"idle",		"Connection was closed while idle")
 #undef HTC_STATUS
+
 /*lint -restore */
diff --git a/include/tbl/style.py b/include/tbl/style.py
index a29459c0a..a88c475a3 100644
--- a/include/tbl/style.py
+++ b/include/tbl/style.py
@@ -7,66 +7,68 @@ from __future__ import print_function
 import glob
 
 def check_file(fn):
-	s = 0
-	ll = []
-	for l in open(fn):
-		ll.append(l)
+    print("Check", fn)
+    ll = []
+    for l in open(fn):
+        ll.append(l)
 
-	assert ll.pop(0)[:2] == "/*"
+    assert ll.pop(0)[:2] == "/*"
 
-	while ll.pop(0) != " */\n":
-		continue
+    while ll.pop(0) != " */\n":
+        continue
 
-	assert len(ll) > 5
+    assert len(ll) > 5
 
-	assert ll.pop(0) == "\n"
-	assert ll.pop(0) == "/*lint -save -e525 -e539 */\n"
-	assert ll.pop(0) == "\n"
+    assert ll.pop(0) == "\n"
+    i = ll.pop(0)
+    assert i == "/*lint -save -e525 -e539 */\n" or \
+           i == "/*lint -save -e525 -e539 -e835 */\n"
+    assert ll.pop(0) == "\n"
 
-	assert ll.pop(-1) == "/*lint -restore */\n"
-	assert ll.pop(-1) == "\n"
+    assert ll.pop(-1) == "/*lint -restore */\n"
+    assert ll.pop(-1) == "\n"
 
-	for i in range(0, len(ll) -1):
-		assert ll[i] != "\n" or ll[i+1] != "\n"
-		assert ll[i] != ")\n" or ll[i+1] == "\n" or ll[i+1][0] == "#"
+    for i in range(0, len(ll) -1):
+        assert ll[i] != "\n" or ll[i+1] != "\n"
+        assert ll[i] != ")\n" or ll[i+1] == "\n" or ll[i+1][0] == "#"
 
-	m = {}
-	while len(ll) > 0:
-		i = ll.pop(0)
-		if i == "\n":
-			continue
-		l = i.lstrip()
-		if l[0] >= 'A' and l[0] <= 'Z':
-			j = l.split('(')
-			m[j[0]] = "Called"
-			l = l.split('//')[0]
-			l = l.split('/*')[0]
-			l = l.rstrip()
-			if l[-1] != ')':
-				while ll.pop(0) != ')\n':
-					continue
-		elif l[0] == "#":
-			j = l[1:].lstrip().split()
-			# print("#", j[0])
-			if j[0] == "define":
-				m[j[1].split("(")[0].strip()] = "Defined"
-			if j[0] == "undef":
-				m[j[1]] = "Undef"
-			while l[-2:] == "\\\n":
-				l = ll.pop(0)
-		else:
-			pass
-			# print(l)
-	rv = 0
-	for i in m:
-		if m[i] != "Undef":
-			print("ERROR", fn, i, m[i])
-			rv += 1
-	return rv
+    m = {}
+    while ll:
+        i = ll.pop(0)
+        if i == "\n":
+            continue
+        l = i.lstrip()
+        if l[0] >= 'A' and l[0] <= 'Z':
+            j = l.split('(')
+            m[j[0]] = "Called"
+            l = l.split('//')[0]
+            l = l.split('/*')[0]
+            l = l.rstrip()
+            if l[-1] != ')':
+                while ll.pop(0).strip() != ')':
+                    continue
+        elif l[0] == "#":
+            j = l[1:].lstrip().split()
+            # print("#", j[0])
+            if j[0] == "define":
+                m[j[1].split("(")[0].strip()] = "Defined"
+            if j[0] == "undef":
+                m[j[1]] = "Undef"
+            while l[-2:] == "\\\n":
+                l = ll.pop(0)
+        else:
+            pass
+            # print(l)
+    rv = 0
+    for i in m:
+        if m[i] != "Undef":
+            print("ERROR", fn, i, m[i])
+            rv += 1
+    return rv
 
 rv = 0
 for fn in glob.glob("*.h"):
-	rv += check_file(fn)
+    rv += check_file(fn)
 if rv != 0:
-	print(rv, "Errors")
+    print(rv, "Errors")
 exit(rv)
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 85a480e7b..80e57eae4 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -167,16 +167,17 @@ SLTM(Length, 0, "Size of object body",
 	"Logs the size of a fetch object body.\n\n"
 )
 
-/* XXX generate HTC info from tbl include */
-#if 0
-#include <stdio.h>
-int main(void) {
-#define HTC_STATUS(e, n, s, l) \
-	printf("\t\"\\t* %s (%d): %s\\n\"\n", s, n, l);
-#include "include/tbl/htc.h"
-	return (0);
-}
-#endif
+/*
+ * XXX generate HTC info below from tbl include
+ *
+ * #include <stdio.h>
+ * int main(void) {
+ * #define HTC_STATUS(e, n, s, l) \
+ *	printf("\t\"\\t* %s (%d): %s\\n\"\n", s, n, l);
+ * #include "include/tbl/htc.h"
+ *	return (0);
+ * }
+ */
 
 SLTM(FetchError, 0, "Error while fetching object",
 	"Logs the error message of a failed fetch operation.\n\n"
diff --git a/include/tbl/waiters.h b/include/tbl/waiters.h
index f1f1d4900..f2fcf6446 100644
--- a/include/tbl/waiters.h
+++ b/include/tbl/waiters.h
@@ -44,3 +44,5 @@
 
 WAITER(poll)
 #undef WAITER
+
+/*lint -restore */
diff --git a/lib/libvarnishapi/generate.py b/lib/libvarnishapi/generate.py
index a7b505c74..d1357c985 100755
--- a/lib/libvarnishapi/generate.py
+++ b/lib/libvarnishapi/generate.py
@@ -37,122 +37,121 @@ import copy
 srcroot = "../.."
 buildroot = "../.."
 if len(sys.argv) == 3:
-	srcroot = sys.argv[1]
-	buildroot = sys.argv[2]
+    srcroot = sys.argv[1]
+    buildroot = sys.argv[2]
 
 #######################################################################
 # These are our tokens
 
 tokens = {
-	# Numerical comparisons
-	"T_EQ":         "==",
-	"T_NEQ":        "!=",
-	"T_LEQ":        "<=",
-	"T_GEQ":        ">=",
-
-	# String comparisons
-	"T_SEQ":        "eq",
-	"T_SNEQ":       "ne",
-
-	# Regular expression matching
-	"T_NOMATCH":    "!~",
-
-	# Boolean operators
-	"T_AND":        "and",
-	"T_OR":         "or",
-	"T_NOT":        "not",
-
-	# Miscellaneous
-	None:           "<>~[]{}():,",
-
-	# These have handwritten recognizers
-	"VAL":          None,
-	"EOI":          None,
-
-	# Special
-	"T_TRUE":       None,
-	"VXID":         "vxid",
+    # Numerical comparisons
+    "T_EQ":     "==",
+    "T_NEQ":    "!=",
+    "T_LEQ":    "<=",
+    "T_GEQ":    ">=",
+
+    # String comparisons
+    "T_SEQ":    "eq",
+    "T_SNEQ":       "ne",
+
+    # Regular expression matching
+    "T_NOMATCH":    "!~",
+
+    # Boolean operators
+    "T_AND":    "and",
+    "T_OR":     "or",
+    "T_NOT":    "not",
+
+    # Miscellaneous
+    None:       "<>~[]{}():,",
+
+    # These have handwritten recognizers
+    "VAL":      None,
+    "EOI":      None,
+
+    # Special
+    "T_TRUE":       None,
+    "VXID":     "vxid",
 }
 
 #######################################################################
 # Emit a function to recognize tokens in a string
 
 def emit_vxp_fixed_token(fo, tokens):
-	recog = list()
-	emit = dict()
-	for i in tokens:
-		j = tokens[i]
-		if (j != None):
-			recog.append(j)
-			emit[j] = i
-
-	recog.sort()
-	rrecog = copy.copy(recog)
-	rrecog.sort(key = lambda x: -len(x))
-
-	fo.write("""
+    recog = list()
+    emit = dict()
+    for i in tokens:
+        j = tokens[i]
+        if j is not None:
+            recog.append(j)
+            emit[j] = i
+
+    recog.sort()
+    rrecog = copy.copy(recog)
+    rrecog.sort(key=lambda x: -len(x))
+
+    fo.write("""
 unsigned
 vxp_fixed_token(const char *p, const char **q)
 {
 
 \tswitch (p[0]) {
 """)
-	last_initial = None
-	for i in recog:
-		if (i[0] == last_initial):
-			continue
-		last_initial = i[0]
-		fo.write("\tcase '%s':\n" % last_initial)
-		for j in rrecog:
-			if (j[0] != last_initial):
-				continue
-
-			fo.write("\t\tif (")
-			k = 1
-			l = len(j)
-			while (k < l):
-				fo.write("p[%d] == '%s'" % (k, j[k]))
-				fo.write(" &&\n\t\t    ")
-				k += 1
-			fo.write("(isword(p[%d]) ? !isword(p[%d]) : 1)) {\n" %
-				(l - 1, l))
-			fo.write("\t\t\t*q = p + %d;\n" % l)
-			fo.write("\t\t\treturn (%s);\n" % emit[j])
-			fo.write("\t\t}\n")
-		fo.write("\t\treturn (0);\n")
-
-	fo.write("\tdefault:\n\t\treturn (0);\n\t}\n}\n")
+    last_initial = None
+    for i in recog:
+        if i[0] == last_initial:
+            continue
+        last_initial = i[0]
+        fo.write("\tcase '%s':\n" % last_initial)
+        for j in rrecog:
+            if j[0] != last_initial:
+                continue
+
+            fo.write("\t\tif (")
+            k = 1
+            l = len(j)
+            while k < l:
+                fo.write("p[%d] == '%s'" % (k, j[k]))
+                fo.write(" &&\n\t\t    ")
+                k += 1
+            fo.write("(isword(p[%d]) ? !isword(p[%d]) : 1)) {\n" % (l - 1, l))
+            fo.write("\t\t\t*q = p + %d;\n" % l)
+            fo.write("\t\t\treturn (%s);\n" % emit[j])
+            fo.write("\t\t}\n")
+        fo.write("\t\treturn (0);\n")
+
+    fo.write("\tdefault:\n\t\treturn (0);\n\t}\n}\n")
 
 #######################################################################
 # Emit the vxp_tnames (token->string) conversion array
 
 def emit_vxp_tnames(fo, tokens):
-	fo.write("\nconst char * const vxp_tnames[256] = {\n")
-	l = list(tokens.keys())
-	l.sort()
-	for i in l:
-		j = tokens[i]
-		if j == None:
-			j = i
-		if i[0] == "'":
-			j = i
-		fo.write("\t[%s] = \"%s\",\n" % (i, j))
-	fo.write("};\n")
+    fo.write("\nconst char * const vxp_tnames[256] = {\n")
+    l = list(tokens.keys())
+    l.sort()
+    for i in l:
+        j = tokens[i]
+        if j is None:
+            j = i
+        if i[0] == "'":
+            j = i
+        fo.write("\t[%s] = \"%s\",\n" % (i, j))
+    fo.write("};\n")
 
 #######################################################################
 
 def polish_tokens(tokens):
-	# Expand single char tokens
-	st = tokens[None]
-	del tokens[None]
+    # Expand single char tokens
+    st = tokens[None]
+    del tokens[None]
 
-	for i in st:
-		tokens["'" + i + "'"] = i
+    for i in st:
+        tokens["'" + i + "'"] = i
 
 #######################################################################
 
 def file_header(fo):
-	fo.write("""/*
+    fo.write("""/*
  * NB:  This file is machine generated, DO NOT EDIT!
  *
  * Edit and run lib/libvarnishapi/generate.py instead
@@ -171,11 +170,11 @@ j = 128
 l = list(tokens.keys())
 l.sort()
 for i in l:
-	if i[0] == "'":
-		continue
-	fo.write("#define\t%s %d\n" % (i, j))
-	j += 1
-	assert j < 256
+    if i[0] == "'":
+        continue
+    fo.write("#define\t%s %d\n" % (i, j))
+    j += 1
+    assert j < 256
 
 fo.close()
 
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 4fc532138..31edaa4a7 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -32,10 +32,6 @@
 
 from __future__ import print_function
 
-import subprocess
-import collections
-import os
-
 #######################################################################
 # These are our tokens
 
@@ -51,112 +47,112 @@ from os.path import join
 srcroot = "../.."
 buildroot = "../.."
 if len(sys.argv) == 3:
-	srcroot = sys.argv[1]
-	buildroot = sys.argv[2]
+    srcroot = sys.argv[1]
+    buildroot = sys.argv[2]
 elif len(sys.argv) != 1:
-	print("Two arguments or none")
-	exit(2)
+    print("Two arguments or none")
+    exit(2)
 
 tokens = {
-	"T_INC":	"++",
-	"T_DEC":	"--",
-	"T_CAND":	"&&",
-	"T_COR":	"||",
-	"T_LEQ":	"<=",
-	"T_EQ":		"==",
-	"T_NEQ":	"!=",
-	"T_GEQ":	">=",
-	"T_SHR":	">>",
-	"T_SHL":	"<<",
-	"T_INCR":	"+=",
-	"T_DECR":	"-=",
-	"T_MUL":	"*=",
-	"T_DIV":	"/=",
-	"T_NOMATCH":	"!~",
-
-	# Single char tokens, for convenience on one line
-	None:		"{}()*+-/%><=;!&.|~,",
-
-	# These have handwritten recognizers
-	"ID":		None,
-	"CNUM":		None,
-	"FNUM":		None,
-	"CSTR":		None,
-	"EOI":		None,
-	"CSRC":		None,
+    "T_INC":    "++",
+    "T_DEC":    "--",
+    "T_CAND":       "&&",
+    "T_COR":    "||",
+    "T_LEQ":    "<=",
+    "T_EQ":     "==",
+    "T_NEQ":    "!=",
+    "T_GEQ":    ">=",
+    "T_SHR":    ">>",
+    "T_SHL":    "<<",
+    "T_INCR":       "+=",
+    "T_DECR":       "-=",
+    "T_MUL":    "*=",
+    "T_DIV":    "/=",
+    "T_NOMATCH":    "!~",
+
+    # Single char tokens, for convenience on one line
+    None:       "{}()*+-/%><=;!&.|~,",
+
+    # These have handwritten recognizers
+    "ID":       None,
+    "CNUM":     None,
+    "FNUM":     None,
+    "CSTR":     None,
+    "EOI":      None,
+    "CSRC":     None,
 }
 
 #######################################################################
 # Our methods and actions
 
 returns = (
-	###############################################################
-	# Client side
-
-	('recv',
-		"C",
-		('fail', 'synth', 'restart', 'pass', 'pipe', 'hash', 'purge', 'vcl')
-	),
-	('pipe',
-		"C",
-		('fail', 'synth', 'pipe',)
-	),
-	('pass',
-		"C",
-		('fail', 'synth', 'restart', 'fetch',)
-	),
-	('hash',
-		"C",
-		('fail', 'lookup',)
-	),
-	('purge',
-		"C",
-		('fail', 'synth', 'restart',)
-	),
-	('miss',
-		"C",
-		('fail', 'synth', 'restart', 'pass', 'fetch',)
-	),
-	('hit',
-		"C",
-		('fail', 'synth', 'restart', 'pass', 'miss', 'deliver',)
-	),
-	('deliver',
-		"C",
-		('fail', 'synth', 'restart', 'deliver',)
-	),
-	('synth',
-		"C",
-		('fail', 'restart', 'deliver',)
-	),
-
-	###############################################################
-	# Backend-fetch
-
-	('backend_fetch',
-		"B",
-		('fail', 'fetch', 'abandon')
-	),
-	('backend_response',
-		"B",
-		('fail', 'deliver', 'retry', 'abandon', 'pass')
-	),
-	('backend_error',
-		"B",
-		('fail', 'deliver', 'retry', 'abandon')
-	),
-
-	###############################################################
-	# Housekeeping
-
-	('init',
-		"H",
-		('ok', 'fail')
-	),
-	('fini',
-		"H",
-		('ok',)
-	),
+    ###############################################################
+    # Client side
+
+    ('recv',
+     "C",
+     ('fail', 'synth', 'restart', 'pass', 'pipe', 'hash', 'purge', 'vcl')
+    ),
+    ('pipe',
+     "C",
+     ('fail', 'synth', 'pipe',)
+    ),
+    ('pass',
+     "C",
+     ('fail', 'synth', 'restart', 'fetch',)
+    ),
+    ('hash',
+     "C",
+     ('fail', 'lookup',)
+    ),
+    ('purge',
+     "C",
+     ('fail', 'synth', 'restart',)
+    ),
+    ('miss',
+     "C",
+     ('fail', 'synth', 'restart', 'pass', 'fetch',)
+    ),
+    ('hit',
+     "C",
+     ('fail', 'synth', 'restart', 'pass', 'miss', 'deliver',)
+    ),
+    ('deliver',
+     "C",
+     ('fail', 'synth', 'restart', 'deliver',)
+    ),
+    ('synth',
+     "C",
+     ('fail', 'restart', 'deliver',)
+    ),
+
+    ###############################################################
+    # Backend-fetch
+
+    ('backend_fetch',
+     "B",
+     ('fail', 'fetch', 'abandon')
+    ),
+    ('backend_response',
+     "B",
+     ('fail', 'deliver', 'retry', 'abandon', 'pass')
+    ),
+    ('backend_error',
+     "B",
+     ('fail', 'deliver', 'retry', 'abandon')
+    ),
+
+    ###############################################################
+    # Housekeeping
+
+    ('init',
+     "H",
+     ('ok', 'fail')
+    ),
+    ('fini',
+     "H",
+     ('ok',)
+    ),
 )
 
 #######################################################################
@@ -170,152 +166,151 @@ returns = (
 varprotos = {}
 
 def varproto(s):
-	if not s in varprotos:
-		fh.write(s + ";\n")
-		varprotos[s] = True
+    if not s in varprotos:
+        fh.write(s + ";\n")
+        varprotos[s] = True
 
 class vardef(object):
-	def __init__(self, nam, typ, rd, wr, wu, vlo, vhi):
-		self.nam = nam
-		self.typ = typ
-		self.rd = rd
-		self.wr = wr
-		self.uns = wu
-		self.vlo = vlo
-		self.vhi = vhi
-
-		self.emit()
-
-	def emit(self):
-		fh.write("\n")
-		fo.write("\n")
-		cnam = self.nam.replace(".", "_")
-		ctyp = vcltypes[self.typ]
-
-		# fo.write("\t{ \"%s\", %s,\n" % (nm, self.typ))
-		fo.write("\tsym = VCC_MkSym(tl, \"%s\", " % self.nam)
-		if (self.typ == "HEADER"):
-			fo.write(" SYM_NONE, %d, %d);\n" % (self.vlo, self.vhi))
-			fo.write("\tAN(sym);\n");
-			fo.write("\tsym->wildcard = vcc_Var_Wildcard;\n")
-		else:
-			fo.write(" SYM_VAR, %d, %d);\n" % (self.vlo, self.vhi))
-		fo.write("\tAN(sym);\n")
-		fo.write("\tsym->type = %s;\n" % self.typ)
-		fo.write("\tsym->eval = vcc_Eval_Var;\n")
-
-		if self.typ == "HEADER":
-			fo.write('\tsym->rname = "HDR_')
-			fo.write(self.nam.split(".")[0].upper())
-			fo.write('";\n')
-		elif len(self.rd):
-			fo.write('\tsym->rname = "VRT_r_%s(ctx)";\n' % cnam)
-			varproto("VCL_" + self.typ + " VRT_r_%s(VRT_CTX)" % cnam)
-		fo.write("\tsym->r_methods =\n")
-		restrict(fo, self.rd)
-		fo.write(";\n")
-
-		if self.typ == "HEADER":
-			fo.write('\tsym->lname = "HDR_')
-			fo.write(self.nam.split(".")[0].upper())
-			fo.write('";\n')
-		elif len(self.wr):
-			fo.write('\tsym->lname = "VRT_l_%s(ctx, ";\n' % cnam)
-			s = "void VRT_l_%s(VRT_CTX, " % cnam
-			if self.typ != "STRING" and self.typ != "BODY":
-				s += "VCL_" + self.typ + ")"
-			else:
-				s += ctyp.c + ", ...)"
-			varproto(s);
-		fo.write("\tsym->w_methods =\n")
-		restrict(fo, self.wr)
-		fo.write(";\n")
-
-		if len(self.uns):
-			varproto("void VRT_u_%s(VRT_CTX)" % cnam)
-			fo.write('\tsym->uname = "VRT_u_%s(ctx)";\n' % cnam)
-		fo.write('\tsym->u_methods =\n')
-		restrict(fo, self.uns)
-		fo.write(";\n")
+    def __init__(self, nam, typ, rd, wr, wu, vlo, vhi):
+        self.nam = nam
+        self.typ = typ
+        self.rd = rd
+        self.wr = wr
+        self.uns = wu
+        self.vlo = vlo
+        self.vhi = vhi
+
+        self.emit()
+
+    def emit(self):
+        fh.write("\n")
+        fo.write("\n")
+        cnam = self.nam.replace(".", "_")
+        ctyp = vcltypes[self.typ]
+
+        # fo.write("\t{ \"%s\", %s,\n" % (nm, self.typ))
+        fo.write("\tsym = VCC_MkSym(tl, \"%s\", " % self.nam)
+        if self.typ == "HEADER":
+            fo.write(" SYM_NONE, %d, %d);\n" % (self.vlo, self.vhi))
+            fo.write("\tAN(sym);\n")
+            fo.write("\tsym->wildcard = vcc_Var_Wildcard;\n")
+        else:
+            fo.write(" SYM_VAR, %d, %d);\n" % (self.vlo, self.vhi))
+        fo.write("\tAN(sym);\n")
+        fo.write("\tsym->type = %s;\n" % self.typ)
+        fo.write("\tsym->eval = vcc_Eval_Var;\n")
+
+        if self.typ == "HEADER":
+            fo.write('\tsym->rname = "HDR_')
+            fo.write(self.nam.split(".")[0].upper())
+            fo.write('";\n')
+        elif self.rd:
+            fo.write('\tsym->rname = "VRT_r_%s(ctx)";\n' % cnam)
+            varproto("VCL_" + self.typ + " VRT_r_%s(VRT_CTX)" % cnam)
+        fo.write("\tsym->r_methods =\n")
+        restrict(fo, self.rd)
+        fo.write(";\n")
+
+        if self.typ == "HEADER":
+            fo.write('\tsym->lname = "HDR_')
+            fo.write(self.nam.split(".")[0].upper())
+            fo.write('";\n')
+        elif self.wr:
+            fo.write('\tsym->lname = "VRT_l_%s(ctx, ";\n' % cnam)
+            s = "void VRT_l_%s(VRT_CTX, " % cnam
+            if self.typ != "STRING" and self.typ != "BODY":
+                s += "VCL_" + self.typ + ")"
+            else:
+                s += ctyp.c + ", ...)"
+            varproto(s)
+        fo.write("\tsym->w_methods =\n")
+        restrict(fo, self.wr)
+        fo.write(";\n")
+
+        if self.uns:
+            varproto("void VRT_u_%s(VRT_CTX)" % cnam)
+            fo.write('\tsym->uname = "VRT_u_%s(ctx)";\n' % cnam)
+        fo.write('\tsym->u_methods =\n')
+        restrict(fo, self.uns)
+        fo.write(";\n")
 
 def parse_vcl(x):
-	vlo,vhi = (0,99)
-	x = x.split()
-	if x[0] == "VCL" and x[1] == "<=":
-		vhi = int(float(x[2]) * 10)
-	elif x[0] == "VCL" and x[1] == ">=":
-		vlo = int(float(x[2]) * 10)
-	else:
-		print("Unknown variable version spec")
-		print("XXX", x, vlo, vhi)
-		exit(2)
-	return vlo,vhi
+    vlo, vhi = (0, 99)
+    x = x.split()
+    if x[0] == "VCL" and x[1] == "<=":
+        vhi = int(float(x[2]) * 10)
+    elif x[0] == "VCL" and x[1] == ">=":
+        vlo = int(float(x[2]) * 10)
+    else:
+        print("Unknown variable version spec")
+        print("XXX", x, vlo, vhi)
+        exit(2)
+    return vlo, vhi
 
 def parse_var(ln):
-	l1 = ln.pop(0).split("``")
-	assert len(l1) in (1,3)
-	vn = l1[0].strip()
-	if vn[-1] == '*':
-		vn = vn[:-1]
-	if len(l1) == 3:
-		vlo,vhi = parse_vcl(l1[1])
-	else:
-		vlo,vhi = 0,99
-	vr = []
-	vw = []
-	vu = []
-	while True:
-		l = ln.pop(0)
-		if l == "":
-			continue
-		j = l.split()
-		if j[0] == "Type:":
-			assert len(j) == 2
-			vt = j[1]
-			continue
-		if j[0] == "Readable" and j[1] == "from:":
-			for i in j[2:]:
-				vr.append(i.strip(",."))
-			continue
-		if j[0] == "Writable" and j[1] == "from:":
-			for i in j[2:]:
-				vw.append(i.strip(",."))
-			continue
-		if j[0] == "Unsetable" and j[1] == "from:":
-			for i in j[2:]:
-				vu.append(i.strip(",."))
-			continue
-		break
-	if vn[:8] != "storage.":
-		vardef(vn, vt, vr, vw, vu, vlo, vhi)
+    l1 = ln.pop(0).split("``")
+    assert len(l1) in (1, 3)
+    vn = l1[0].strip()
+    if vn[-1] == '*':
+        vn = vn[:-1]
+    if len(l1) == 3:
+        vlo, vhi = parse_vcl(l1[1])
+    else:
+        vlo, vhi = 0, 99
+    vr = []
+    vw = []
+    vu = []
+    while True:
+        l = ln.pop(0)
+        if l == "":
+            continue
+        j = l.split()
+        if j[0] == "Type:":
+            assert len(j) == 2
+            vt = j[1]
+            continue
+        if j[0] == "Readable" and j[1] == "from:":
+            for i in j[2:]:
+                vr.append(i.strip(",."))
+            continue
+        if j[0] == "Writable" and j[1] == "from:":
+            for i in j[2:]:
+                vw.append(i.strip(",."))
+            continue
+        if j[0] == "Unsetable" and j[1] == "from:":
+            for i in j[2:]:
+                vu.append(i.strip(",."))
+            continue
+        break
+    if vn[:8] != "storage.":
+        vardef(vn, vt, vr, vw, vu, vlo, vhi)
 
 def parse_var_doc(fn):
-	s = 0
-	l = []
-	for i in open(fn):
-		l.append(i.rstrip())
-	for n in range(0, len(l)):
-		j = l[n].split()
-		if len(j) != 2 or j[0] != "Type:" or not l[n][0].isspace():
-			continue
-		m = n
-		while m < len(l) and ( l[m] == "" or l[m][0].isspace() ):
-			m += 1
-		parse_var(l[n-2:m-1])
+    l = []
+    for i in open(fn):
+        l.append(i.rstrip())
+    for n in range(0, len(l)):
+        j = l[n].split()
+        if len(j) != 2 or j[0] != "Type:" or not l[n][0].isspace():
+            continue
+        m = n
+        while m < len(l) and (l[m] == "" or l[m][0].isspace()):
+            m += 1
+        parse_var(l[n-2:m-1])
 
 stv_variables = (
-	('free_space',	'BYTES',	"0.", 'storage.<name>.free_space', """
-	Free space available in the named stevedore. Only available for
-	the malloc stevedore.
-	"""),
-	('used_space',	'BYTES',	"0.", 'storage.<name>.used_space', """
-	Used space in the named stevedore. Only available for the malloc
-	stevedore.
-	"""),
-	('happy',	'BOOL',		"0", 'storage.<name>.happy', """
-	Health status for the named stevedore. Not available in any of the
-	current stevedores.
-	"""),
+    ('free_space', 'BYTES', "0.", 'storage.<name>.free_space', """
+    Free space available in the named stevedore. Only available for
+    the malloc stevedore.
+    """),
+    ('used_space', 'BYTES', "0.", 'storage.<name>.used_space', """
+    Used space in the named stevedore. Only available for the malloc
+    stevedore.
+    """),
+    ('happy', 'BOOL', "0", 'storage.<name>.happy', """
+    Health status for the named stevedore. Not available in any of the
+    current stevedores.
+    """),
 )
 
 #######################################################################
@@ -324,11 +319,11 @@ stv_variables = (
 vcltypes = {}
 
 class vcltype(object):
-	def __init__(self, name, ctype, internal=False):
-		self.name = name
-		self.c = ctype
-		self.internal = internal
-		vcltypes[name] = self
+    def __init__(self, name, ctype, internal=False):
+        self.name = name
+        self.c = ctype
+        self.internal = internal
+        vcltypes[name] = self
 
 
 vcltype("STRINGS", "void", True)
@@ -338,19 +333,19 @@ vcltype("SUB", "void*", True)
 fi = open(join(srcroot, "include/vrt.h"))
 
 for i in fi:
-	j = i.split()
-	if len(j) < 3:
-		continue
-	if j[0] != "typedef":
-		continue
-	if j[-1][-1] != ";":
-		continue
-	if j[-1][-2] == ")":
-		continue
-	if j[-1][:4] != "VCL_":
-		continue
-	d = " ".join(j[1:-1])
-	vcltype(j[-1][4:-1], d)
+    j = i.split()
+    if len(j) < 3:
+        continue
+    if j[0] != "typedef":
+        continue
+    if j[-1][-1] != ";":
+        continue
+    if j[-1][-2] == ")":
+        continue
+    if j[-1][:4] != "VCL_":
+        continue
+    d = " ".join(j[1:-1])
+    vcltype(j[-1][4:-1], d)
 fi.close()
 
 #######################################################################
@@ -360,20 +355,20 @@ fi.close()
 
 #######################################################################
 def emit_vcl_fixed_token(fo, tokens):
-	"Emit a function to recognize tokens in a string"
-	recog = list()
-	emit = dict()
-	for i in tokens:
-		j = tokens[i]
-		if j is not None:
-			recog.append(j)
-			emit[j] = i
-
-	recog.sort()
-	rrecog = copy.copy(recog)
-	rrecog.sort(key=lambda x: -len(x))
-
-	fo.write("""
+    "Emit a function to recognize tokens in a string"
+    recog = list()
+    emit = dict()
+    for i in tokens:
+        j = tokens[i]
+        if j is not None:
+            recog.append(j)
+            emit[j] = i
+
+    recog.sort()
+    rrecog = copy.copy(recog)
+    rrecog.sort(key=lambda x: -len(x))
+
+    fo.write("""
 #define M1()\tdo {*q = p + 1; return (p[0]); } while (0)
 #define M2(c,t)\tdo {if (p[1] == (c)) { *q = p + 2; return (t); }} while (0)
 
@@ -383,131 +378,130 @@ vcl_fixed_token(const char *p, const char **q)
 
 \tswitch (p[0]) {
 """)
-	last_initial = None
-	for i in recog:
-		if (i[0] == last_initial):
-			continue
-		last_initial = i[0]
-		fo.write("\tcase '%s':\n" % last_initial)
-		need_ret = True
-		for j in rrecog:
-			if (j[0] != last_initial):
-				continue
-			if len(j) == 2:
-				fo.write("\t\tM2('%s', %s);\n" %
-				    (j[1], emit[j]))
-			elif len(j) == 1:
-				fo.write("\t\tM1();\n")
-				need_ret = False
-			else:
-				fo.write("\t\tif (")
-				k = 1
-				l = len(j)
-				while (k < l):
-					fo.write("p[%d] == '%s'" % (k, j[k]))
-					fo.write(" &&")
-					if (k % 3) == 0:
-						fo.write("\n\t\t    ")
-					else:
-						fo.write(" ")
-					k += 1
-				fo.write("!isvar(p[%d])) {\n" % l)
-				fo.write("\t\t\t*q = p + %d;\n" % l)
-				fo.write("\t\t\treturn (%s);\n" % emit[j])
-				fo.write("\t\t}\n")
-		if need_ret:
-			fo.write("\t\treturn (0);\n")
-	fo.write("\tdefault:\n\t\treturn (0);\n\t}\n}\n")
+    last_initial = None
+    for i in recog:
+        if i[0] == last_initial:
+            continue
+        last_initial = i[0]
+        fo.write("\tcase '%s':\n" % last_initial)
+        need_ret = True
+        for j in rrecog:
+            if j[0] != last_initial:
+                continue
+            if len(j) == 2:
+                fo.write("\t\tM2('%s', %s);\n" % (j[1], emit[j]))
+            elif len(j) == 1:
+                fo.write("\t\tM1();\n")
+                need_ret = False
+            else:
+                fo.write("\t\tif (")
+                k = 1
+                l = len(j)
+                while k < l:
+                    fo.write("p[%d] == '%s'" % (k, j[k]))
+                    fo.write(" &&")
+                    if (k % 3) == 0:
+                        fo.write("\n\t\t    ")
+                    else:
+                        fo.write(" ")
+                    k += 1
+                fo.write("!isvar(p[%d])) {\n" % l)
+                fo.write("\t\t\t*q = p + %d;\n" % l)
+                fo.write("\t\t\treturn (%s);\n" % emit[j])
+                fo.write("\t\t}\n")
+        if need_ret:
+            fo.write("\t\treturn (0);\n")
+    fo.write("\tdefault:\n\t\treturn (0);\n\t}\n}\n")
 
 
 #######################################################################
 def emit_vcl_tnames(fo, tokens):
-	"Emit the vcl_tnames (token->string) conversion array"
-	fo.write("\nconst char * const vcl_tnames[256] = {\n")
-	l = list(tokens.keys())
-	l.sort()
-	for i in l:
-		j = tokens[i]
-		if j is None:
-			j = i
-		if i[0] == "'":
-			j = i
-		fo.write("\t[%s] = \"%s\",\n" % (i, j))
-	fo.write("};\n")
+    "Emit the vcl_tnames (token->string) conversion array"
+    fo.write("\nconst char * const vcl_tnames[256] = {\n")
+    l = list(tokens.keys())
+    l.sort()
+    for i in l:
+        j = tokens[i]
+        if j is None:
+            j = i
+        if i[0] == "'":
+            j = i
+        fo.write("\t[%s] = \"%s\",\n" % (i, j))
+    fo.write("};\n")
 
 
 #######################################################################
 def emit_file(fo, fd, bn):
-	"Read a C-source file and spit out code that outputs it with VSB_cat()"
-	fn = join(fd, bn)
-
-	fi = open(fn)
-	fc = fi.read()
-	fi.close()
-
-	w = 66		# Width of lines, after white space prefix
-	maxlen = 10240  # Max length of string literal
-
-	x = 0
-	l = 0
-	fo.write("\n\t/* %s */\n\n" % fn)
-	fo.write('\tVSB_cat(sb, "/* ---===### %s ###===--- */\\n\\n");\n' % bn)
-	for c in fc:
-		if l == 0:
-			fo.write("\tVSB_cat(sb, \"")
-			l += 12
-			x += 12
-		if x == 0:
-			fo.write("\t    \"")
-		d = c
-		if c == '\n':
-			d = "\\n"
-		elif c == '\t':
-			d = "\\t"
-		elif c == '"':
-			d = "\\\""
-		elif c == '\\':
-			d = "\\\\"
-
-		if c == '\n' and x > w - 20:
-			fo.write(d + "\"\n")
-			x = 0
-			continue
-		if c.isspace() and x > w - 10:
-			fo.write(d + "\"\n")
-			x = 0
-			continue
-
-		fo.write(d)
-		x += len(d)
-		l += len(d)
-		if l > maxlen:
-			fo.write("\");\n")
-			l = 0
-			x = 0
-		if x > w - 3:
-			fo.write("\"\n")
-			x = 0
-	if x != 0:
-		fo.write("\"\n")
-	if l != 0:
-		fo.write("\t);\n")
-	fo.write('\tVSB_cat(sb, "\\n");\n')
+    "Read a C-source file and spit out code that outputs it with VSB_cat()"
+    fn = join(fd, bn)
+
+    fi = open(fn)
+    fc = fi.read()
+    fi.close()
+
+    w = 66      # Width of lines, after white space prefix
+    maxlen = 10240  # Max length of string literal
+
+    x = 0
+    l = 0
+    fo.write("\n\t/* %s */\n\n" % fn)
+    fo.write('\tVSB_cat(sb, "/* ---===### %s ###===--- */\\n\\n");\n' % bn)
+    for c in fc:
+        if l == 0:
+            fo.write("\tVSB_cat(sb, \"")
+            l += 12
+            x += 12
+        if x == 0:
+            fo.write("\t    \"")
+        d = c
+        if c == '\n':
+            d = "\\n"
+        elif c == '\t':
+            d = "\\t"
+        elif c == '"':
+            d = "\\\""
+        elif c == '\\':
+            d = "\\\\"
+
+        if c == '\n' and x > w - 20:
+            fo.write(d + "\"\n")
+            x = 0
+            continue
+        if c.isspace() and x > w - 10:
+            fo.write(d + "\"\n")
+            x = 0
+            continue
+
+        fo.write(d)
+        x += len(d)
+        l += len(d)
+        if l > maxlen:
+            fo.write("\");\n")
+            l = 0
+            x = 0
+        if x > w - 3:
+            fo.write("\"\n")
+            x = 0
+    if x != 0:
+        fo.write("\"\n")
+    if l != 0:
+        fo.write("\t);\n")
+    fo.write('\tVSB_cat(sb, "\\n");\n')
 
 #######################################################################
 
 
 def polish_tokens(tokens):
-	"Expand single char tokens"
-	st = tokens[None]
-	del tokens[None]
-	for i in st:
-		tokens["'" + i + "'"] = i
+    "Expand single char tokens"
+    st = tokens[None]
+    del tokens[None]
+    for i in st:
+        tokens["'" + i + "'"] = i
 
 
 #######################################################################
 def file_header(fo):
-	fo.write("""/*
+    fo.write("""/*
  * NB:  This file is machine generated, DO NOT EDIT!
  *
  * Edit and run lib/libvcc/generate.py instead.
@@ -516,10 +510,10 @@ def file_header(fo):
 """)
 
 def lint_start(fo):
-	fo.write('/*lint -save -e525 -e539 */\n\n')
+    fo.write('/*lint -save -e525 -e539 */\n\n')
 
 def lint_end(fo):
-	fo.write('\n/*lint -restore */\n')
+    fo.write('\n/*lint -restore */\n')
 
 #######################################################################
 
@@ -531,11 +525,11 @@ file_header(fo)
 
 j = 128
 for i in sorted(tokens.keys()):
-	if i[0] == "'":
-		continue
-	fo.write("#define\t%s %d\n" % (i, j))
-	j += 1
-	assert j < 256
+    if i[0] == "'":
+        continue
+    fo.write("#define\t%s %d\n" % (i, j))
+    j += 1
+    assert j < 256
 
 fo.close()
 
@@ -546,14 +540,14 @@ vcls = list()
 vcls_client = list()
 vcls_backend = list()
 for i in returns:
-	vcls.append(i[0])
-	for j in i[1]:
-		if j == "B":
-			vcls_backend.append(i[0])
-		elif j == "C":
-			vcls_client.append(i[0])
-	for j in i[2]:
-		rets[j] = True
+    vcls.append(i[0])
+    for j in i[1]:
+        if j == "B":
+            vcls_backend.append(i[0])
+        elif j == "C":
+            vcls_client.append(i[0])
+    for j in i[2]:
+        rets[j] = True
 
 #######################################################################
 
@@ -566,25 +560,25 @@ lint_start(fo)
 fo.write("#ifdef VCL_RET_MAC\n")
 ll = sorted(returns)
 for i in sorted(rets.keys()):
-	fo.write("VCL_RET_MAC(%s, %s" % (i.lower(), i.upper()))
-	s = ",\n\t"
-	for j in ll:
-		if i in j[2]:
-			fo.write("%sVCL_MET_%s" % (s, j[0].upper()))
-			s = " |\n\t"
-	fo.write("\n)\n\n")
+    fo.write("VCL_RET_MAC(%s, %s" % (i.lower(), i.upper()))
+    s = ",\n\t"
+    for j in ll:
+        if i in j[2]:
+            fo.write("%sVCL_MET_%s" % (s, j[0].upper()))
+            s = " |\n\t"
+    fo.write("\n)\n\n")
 fo.write("#undef VCL_RET_MAC\n")
 fo.write("#endif\n")
 
 fo.write("\n#ifdef VCL_MET_MAC\n")
 for i in ll:
-	fo.write("VCL_MET_MAC(%s, %s, %s," %
-	    (i[0].lower(), i[0].upper(), i[1]))
-	p = " (\n\t"
-	for j in sorted(i[2]):
-		fo.write("%s(1U << VCL_RET_%s)" % (p, j.upper()))
-		p = " |\n\t"
-	fo.write(")\n)\n\n")
+    fo.write("VCL_MET_MAC(%s, %s, %s," %
+             (i[0].lower(), i[0].upper(), i[1]))
+    p = " (\n\t"
+    for j in sorted(i[2]):
+        fo.write("%s(1U << VCL_RET_%s)" % (p, j.upper()))
+        p = " |\n\t"
+    fo.write(")\n)\n\n")
 fo.write("#undef VCL_MET_MAC\n")
 fo.write("#endif\n")
 lint_end(fo)
@@ -609,40 +603,40 @@ fo.write("""
 
 
 def tbl40(a, b):
-	while len(a.expandtabs()) < 40:
-		a += "\t"
-	return a + b
+    while len(a.expandtabs()) < 40:
+        a += "\t"
+    return a + b
 
 fo.write("\n/* VCL Methods */\n")
 task = {}
 n = 1
 for i in returns:
-	fo.write(tbl40("#define VCL_MET_%s" % i[0].upper(), "(1U << %d)\n" % n))
-	if not i[1] in task:
-		task[i[1]] = []
-	task[i[1]].append("VCL_MET_" + i[0].upper())
-	n += 1
+    fo.write(tbl40("#define VCL_MET_%s" % i[0].upper(), "(1U << %d)\n" % n))
+    if not i[1] in task:
+        task[i[1]] = []
+    task[i[1]].append("VCL_MET_" + i[0].upper())
+    n += 1
 
 fo.write("\n" + tbl40("#define VCL_MET_MAX", "%d\n" % n))
 fo.write("\n" + tbl40("#define VCL_MET_MASK", "0x%x\n" % ((1 << n) - 1)))
 
 fo.write("\n")
 for i in sorted(task.keys()):
-	fo.write(tbl40("#define VCL_MET_TASK_%s" % i.upper(),
-		       "( " + (" | \\\n\t\t\t\t\t  ").join(task[i]) + " )\n"))
+    fo.write(tbl40("#define VCL_MET_TASK_%s" % i.upper(),
+                   "( " + (" | \\\n\t\t\t\t\t  ").join(task[i]) + " )\n"))
 
 
 fo.write("\n/* VCL Returns */\n")
 n = 1
 for i in sorted(rets.keys()):
-	fo.write(tbl40("#define VCL_RET_%s" % i.upper(), "%d\n" % n))
-	n += 1
+    fo.write(tbl40("#define VCL_RET_%s" % i.upper(), "%d\n" % n))
+    n += 1
 
 fo.write("\n" + tbl40("#define VCL_RET_MAX", "%d\n" % n))
 
 fo.write("\n/* VCL Types */\n")
 for vcltype in sorted(vcltypes.keys()):
-	fo.write("extern const struct vrt_type VCL_TYPE_%s[1];\n" % vcltype)
+    fo.write("extern const struct vrt_type VCL_TYPE_%s[1];\n" % vcltype)
 
 
 fo.write("""
@@ -653,26 +647,26 @@ typedef void vcl_fini_f(VRT_CTX);
 typedef void vcl_func_f(VRT_CTX);
 
 struct VCL_conf {
-	unsigned			magic;
-#define VCL_CONF_MAGIC			0x7406c509	/* from /dev/random */
+    unsigned            magic;
+#define VCL_CONF_MAGIC          0x7406c509      /* from /dev/random */
 
-	unsigned			syntax;
-	struct director			**default_director;
-	const struct vrt_backend_probe	*default_probe;
-	unsigned			nref;
-	const struct vrt_ref		*ref;
+    unsigned            syntax;
+    struct director         **default_director;
+    const struct vrt_backend_probe  *default_probe;
+    unsigned            nref;
+    const struct vrt_ref        *ref;
 
-	unsigned			nsrc;
-	const char			**srcname;
-	const char			**srcbody;
+    unsigned            nsrc;
+    const char              **srcname;
+    const char              **srcbody;
 
-	unsigned			nvmod;
+    unsigned            nvmod;
 
-	vcl_event_f			*event_vcl;
+    vcl_event_f             *event_vcl;
 """)
 
 for i in returns:
-	fo.write("\tvcl_func_f\t\t\t*" + i[0] + "_func;\n")
+    fo.write("\tvcl_func_f\t\t\t*" + i[0] + "_func;\n")
 
 fo.write("\n};\n")
 fo.close()
@@ -681,44 +675,44 @@ fo.close()
 
 
 def restrict(fo, spec):
-	d = dict()
-	for j in spec:
-		if j[:4] == "vcl_":
-			j = j[4:]
-		if j == 'all':
-			for i in vcls:
-				d[i] = True
-		elif j == 'backend':
-			for i in vcls_backend:
-				d[i] = True
-		elif j == 'client':
-			for i in vcls_client:
-				d[i] = True
-		elif j == 'both':
-			for i in vcls_client:
-				d[i] = True
-			for i in vcls_backend:
-				d[i] = True
-		else:
-			if not j in vcls:
-				print("JJ", j)
-			assert j in vcls
-			d[j] = True
-	p = ""
-	l = list(d.keys())
-	l.sort()
-	w = 0
-	fo.write("\t\t")
-	for j in l:
-		x = p + "VCL_MET_" + j.upper()
-		if w + len(x) > 60:
-			fo.write("\n\t\t")
-			w = 0
-		fo.write(x)
-		w += len(x)
-		p = " | "
-	if len(d) == 0:
-		fo.write("0")
+    d = dict()
+    for j in spec:
+        if j[:4] == "vcl_":
+            j = j[4:]
+        if j == 'all':
+            for i in vcls:
+                d[i] = True
+        elif j == 'backend':
+            for i in vcls_backend:
+                d[i] = True
+        elif j == 'client':
+            for i in vcls_client:
+                d[i] = True
+        elif j == 'both':
+            for i in vcls_client:
+                d[i] = True
+            for i in vcls_backend:
+                d[i] = True
+        else:
+            if not j in vcls:
+                print("JJ", j)
+            assert j in vcls
+            d[j] = True
+    p = ""
+    l = list(d.keys())
+    l.sort()
+    w = 0
+    fo.write("\t\t")
+    for j in l:
+        x = p + "VCL_MET_" + j.upper()
+        if w + len(x) > 60:
+            fo.write("\n\t\t")
+            w = 0
+        fo.write(x)
+        w += len(x)
+        p = " | "
+    if not d:
+        fo.write("0")
 
 #######################################################################
 
@@ -736,7 +730,7 @@ fo.write("""
 void
 vcc_Var_Init(struct vcc *tl)
 {
-	struct symbol *sym;
+    struct symbol *sym;
 """)
 
 
@@ -744,22 +738,22 @@ parse_var_doc(join(srcroot, "doc/sphinx/reference/vcl_var.rst"))
 fo.write("}\n")
 
 for i in stv_variables:
-	fh.write(vcltypes[i[1]].c + " VRT_Stv_" + i[0] + "(const char *);\n")
+    fh.write(vcltypes[i[1]].c + " VRT_Stv_" + i[0] + "(const char *);\n")
 
 fo.write("\n/* VCL type identifiers */\n")
 
 for vn in sorted(vcltypes.keys()):
-	v = vcltypes[vn]
-	if v.internal:
-		continue
-	fo.write("const struct vrt_type VCL_TYPE_%s[1] = { {\n" % v.name)
-	fo.write("\t.magic = VRT_TYPE_MAGIC,\n")
-	fo.write('\t.lname = "%s",\n' % v.name.lower())
-	fo.write('\t.uname = "%s",\n' % v.name)
-	fo.write('\t.ctype = "%s",\n' % v.c)
-	if v.c != "void":
-		fo.write('\t.szof = sizeof(VCL_%s),\n' % v.name)
-	fo.write("}};\n")
+    v = vcltypes[vn]
+    if v.internal:
+        continue
+    fo.write("const struct vrt_type VCL_TYPE_%s[1] = { {\n" % v.name)
+    fo.write("\t.magic = VRT_TYPE_MAGIC,\n")
+    fo.write('\t.lname = "%s",\n' % v.name.lower())
+    fo.write('\t.uname = "%s",\n' % v.name)
+    fo.write('\t.ctype = "%s",\n' % v.c)
+    if v.c != "void":
+        fo.write('\t.szof = sizeof(VCL_%s),\n' % v.name)
+    fo.write("}};\n")
 
 fo.close()
 fh.close()
@@ -800,7 +794,7 @@ file_header(ft)
 lint_start(ft)
 
 for vcltype in sorted(vcltypes.keys()):
-	ft.write("VCC_TYPE(" + vcltype + ")\n")
+    ft.write("VCC_TYPE(" + vcltype + ")\n")
 ft.write("#undef VCC_TYPE\n")
 lint_end(ft)
 ft.close()
@@ -813,10 +807,10 @@ file_header(fo)
 lint_start(fo)
 
 for i in stv_variables:
-	ct = vcltypes[i[1]]
-	fo.write("VRTSTVVAR(" + i[0] + ",\t" + i[1] + ",\t")
-	fo.write(ct.c + ",\t" + i[2] + ")")
-	fo.write("\n")
+    ct = vcltypes[i[1]]
+    fo.write("VRTSTVVAR(" + i[0] + ",\t" + i[1] + ",\t")
+    fo.write(ct.c + ",\t" + i[2] + ")")
+    fo.write("\n")
 
 fo.write("#undef VRTSTVVAR\n")
 lint_end(fo)
diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index ac54ceebd..e58041266 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -41,7 +41,6 @@ import sys
 import re
 import optparse
 import unittest
-import random
 import copy
 import json
 import hashlib
@@ -254,7 +253,7 @@ class CType(object):
     def json(self, jl):
         jl.append([self.vt])
         while jl[-1][-1] is None:
-                jl[-1].pop(-1)
+            jl[-1].pop(-1)
 
 #######################################################################
 
@@ -290,7 +289,7 @@ class arg(CType):
     def json(self, jl):
         jl.append([self.vt, self.nm, self.defval, self.spec])
         if self.opt:
-                jl[-1].append(True)
+            jl[-1].append(True)
         while jl[-1][-1] is None:
             jl[-1].pop(-1)
 
@@ -366,7 +365,7 @@ class ProtoType(object):
             err("%s(): Illegal C-name\n" % self.cname(), warn=False)
 
         if len(wl) == 2 and wl[0] == '(' and wl[1] == ')':
-                return
+            return
 
         if wl[0] != "(":
             err("Syntax error: Expected '(', got '%s'" % wl[0], warn=False)
@@ -386,17 +385,17 @@ class ProtoType(object):
             if not wl:
                 break
             if wl[0] == '[':
-                    wl.pop(0)
-                    t = arg(wl, names, st.vcc.enums, ']')
-                    if t.nm is None:
-                        err("Optional arguments must have names", warn=False)
-                    t.opt = True
-                    x = wl.pop(0)
-                    if x != ']':
-                        err("Expected ']' found '%s'" % x, warn=False)
-                    self.argstruct = True
+                wl.pop(0)
+                t = arg(wl, names, st.vcc.enums, ']')
+                if t.nm is None:
+                    err("Optional arguments must have names", warn=False)
+                t.opt = True
+                x = wl.pop(0)
+                if x != ']':
+                    err("Expected ']' found '%s'" % x, warn=False)
+                self.argstruct = True
             else:
-                    t = arg(wl, names, st.vcc.enums, ',')
+                t = arg(wl, names, st.vcc.enums, ',')
             if t.nm is None:
                 t.nm2 = "arg%d" % n
             else:
@@ -466,10 +465,10 @@ class ProtoType(object):
         s = self.retval.ct + " " + name + '('
         ll = args
         if self.argstruct:
-                ll.append(self.argstructname() + "*")
+            ll.append(self.argstructname() + "*")
         else:
-                for i in self.args:
-                    ll.append(i.ct)
+            for i in self.args:
+                ll.append(i.ct)
         s += ", ".join(ll)
         return s + ');'
 
@@ -483,31 +482,31 @@ class ProtoType(object):
     def argstructure(self):
         s = "\n" + self.argstructname() + " {\n"
         for i in self.args:
-                if i.opt:
-                        assert i.nm is not None
-                        s += "\tchar\t\t\tvalid_%s;\n" % i.nm
+            if i.opt:
+                assert i.nm is not None
+                s += "\tchar\t\t\tvalid_%s;\n" % i.nm
         for i in self.args:
-                s += "\t" + i.ct
-                if len(i.ct) < 8:
-                        s += "\t"
-                if len(i.ct) < 16:
-                        s += "\t"
-                s += "\t" + i.nm2 + ";\n"
+            s += "\t" + i.ct
+            if len(i.ct) < 8:
+                s += "\t"
+            if len(i.ct) < 16:
+                s += "\t"
+            s += "\t" + i.nm2 + ";\n"
         s += "};\n"
         return s
 
     def cstuff(self, args, where):
         s = ""
         if where == 'h':
-                if self.argstruct:
-                        s += self.argstructure()
-                s += lwrap(self.proto(args, self.cname(True)))
+            if self.argstruct:
+                s += self.argstructure()
+            s += lwrap(self.proto(args, self.cname(True)))
         elif where == 'c':
-                s += lwrap(self.typedef(args))
+            s += lwrap(self.typedef(args))
         elif where == 'o':
-                if self.argstruct:
-                        s += self.argstructure()
-                s += lwrap(self.typedef(args))
+            if self.argstruct:
+                s += self.argstructure()
+            s += lwrap(self.typedef(args))
         else:
             assert False
         return s
@@ -517,9 +516,9 @@ class ProtoType(object):
         self.retval.json(ll)
         ll.append('Vmod_%s_Func.%s' % (self.st.vcc.modname, cfunc))
         if self.argstruct:
-                ll.append(self.argstructname())
+            ll.append(self.argstructname())
         else:
-                ll.append("")
+            ll.append("")
         for i in self.args:
             i.json(ll)
         jl.append(ll)
diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py
index a665dff20..6c96388e5 100644
--- a/lib/libvcc/vsctool.py
+++ b/lib/libvcc/vsctool.py
@@ -27,9 +27,13 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 
-"""
-This program compiles a .vsc file to C language constructs.
-"""
+'''
+This program compiles a `.vsc` file to `.c`, `.h` and `.rst` formats.
+
+Note: A `.vsc` file is *almost* a `.rst` file, or at last *almost*
+the same general syntax as a `.rst` file, but for now we process
+it with this program to get a *real* `.rst` file.
+'''
 
 from __future__ import print_function
 
@@ -37,385 +41,386 @@ import getopt
 import json
 import sys
 import collections
-import struct
 
-TYPES = [ "counter", "gauge", "bitmap" ]
-CTYPES = [ "uint64_t" ]
-LEVELS = [ "info", "diag", "debug" ]
-FORMATS = [ "integer", "bytes", "bitmap", "duration" ]
+# Parameters of 'varnish_vsc_begin', first element is default
+TYPES = ["counter", "gauge", "bitmap"]
+CTYPES = ["uint64_t"]
+LEVELS = ["info", "diag", "debug"]
+FORMATS = ["integer", "bytes", "bitmap", "duration"]
 
 PARAMS = {
-	"type":		["counter", TYPES],
-	"ctype":	["uint64_t", CTYPES],
-	"level":	["info", LEVELS],
-	"oneliner":	True,
-	"format":	[ "integer", FORMATS],
+    "type": TYPES,
+    "ctype": CTYPES,
+    "level": LEVELS,
+    "oneliner": None,
+    "format": FORMATS,
 }
 
-# http://python3porting.com/problems.html#bytes-strings-and-unicode
-if sys.version_info < (3,):
-	def b(x):
-		return x
-else:
-	import codecs
-	def b(x):
-		return codecs.latin_1_encode(x)[0]
-
 def genhdr(fo, name):
-	fo.write('/*\n')
-	fo.write(' * NB:  This file is machine generated, DO NOT EDIT!\n')
-	fo.write(' *\n')
-	fo.write(' * Edit ' + name +
-	    '.vsc and run lib/libvcc/vsctool.py instead.\n')
-	fo.write(' */\n')
-	fo.write('\n')
 
-#######################################################################
+    '''Emit .[ch] file boiler-plate warning'''
 
-class vscset(object):
-	def __init__(self, name, m):
-		self.name = name
-		self.struct = "struct VSC_" + name
-		self.mbrs = []
-		self.head = m
-		self.completed = False
-		self.off = 0
-
-	def addmbr(self, m):
-		assert not self.completed
-		self.mbrs.append(m)
-		m.param["index"] = self.off
-		self.off += 8
-
-	def complete(self):
-		self.completed = True
-
-	def emit_json(self, fo):
-		dd = collections.OrderedDict()
-		dd["version"] = "1"
-		dd["name"] = self.name
-		dd["oneliner"] = self.head.param["oneliner"].strip()
-		dd["order"] = int(self.head.param["order"])
-		dd["docs"] = "\n".join(self.head.getdoc())
-		dd["elements"] = len(self.mbrs)
-		el = collections.OrderedDict()
-		dd["elem"] = el
-		for i in self.mbrs:
-			ed = collections.OrderedDict()
-			el[i.arg] = ed
-			for j in PARAMS:
-				if j in i.param:
-					ed[j] = i.param[j]
-			ed["index"] = i.param["index"]
-			ed["name"] = i.arg
-			ed["docs"] = "\n".join(i.getdoc())
-		s=json.dumps(dd, separators=(",",":")) + "\0"
-		fo.write("\nstatic const unsigned char")
-		fo.write(" vsc_%s_json[%d] = {\n" % (self.name, len(s)))
-		bz = bytearray(s, encoding="ascii")
-		t = "\t"
-		for i in bz:
-			t += "%d," % i
-			if len(t) >= 69:
-				fo.write(t + "\n")
-				t = "\t"
-		if len(t) > 1:
-			fo.write(t[:-1])
-		fo.write("\n};\n")
-		s = json.dumps(dd, indent=2, separators=(',', ': '))
-		fo.write("\n")
-		for i in s.split("\n"):
-			j = "// " + i
-			if len(j) > 72:
-				fo.write(j[:72] + "[...]\n")
-			else:
-				fo.write(j + "\n")
-		fo.write("\n")
-
-
-	def emit_h(self):
-		fon="VSC_" + self.name + ".h"
-		fo = open(fon, "w")
-		genhdr(fo, self.name)
-		fo.write(self.struct + " {\n")
-		for i in self.mbrs:
-			fo.write("\tuint64_t\t%s;\n" % i.arg)
-		fo.write("};\n")
-		fo.write("\n")
-
-		fo.write("#define VSC_" + self.name +
-		    "_size PRNDUP(sizeof(" + self.struct + "))\n\n")
-
-		fo.write(self.struct + " *VSC_" + self.name + "_New")
-		fo.write("(struct vsmw_cluster *,\n")
-		fo.write("    struct vsc_seg **, const char *fmt, ...);\n")
-
-		fo.write("void VSC_" + self.name + "_Destroy")
-		fo.write("(struct vsc_seg **);\n")
-
-		if 'sumfunction' in self.head.param:
-			fo.write("void VSC_" + self.name + "_Summ")
-			fo.write("(" + self.struct + " *, ")
-			fo.write("const " + self.struct + " *);\n")
-
-	def emit_c(self):
-		fon="VSC_" + self.name + ".c"
-		fo = open(fon, "w")
-		genhdr(fo, self.name)
-		fo.write('#include "config.h"\n')
-		fo.write('#include <stdio.h>\n')
-		fo.write('#include <stdarg.h>\n')
-		fo.write('#include "vdef.h"\n')
-		fo.write('#include "vas.h"\n')
-		fo.write('#include "vrt.h"\n')
-		fo.write('#include "VSC_%s.h"\n' % self.name)
-
-		fo.write("\n")
-		fo.write('static const char vsc_%s_name[] = "%s";\n' %
-		    (self.name, self.name.upper()))
-
-		fo.write("\n")
-		fo.write("#define PARANOIA(a,n)\t\t\t\t\\\n")
-		fo.write("    _Static_assert(\t\t\t\t\\\n")
-		fo.write("\toffsetof(" + self.struct + ", a) == n,\t\\\n")
-		fo.write("\t\"VSC element '\" #a \"' at wrong offset\")\n\n")
-
-		for i in self.mbrs:
-			fo.write("PARANOIA(" + i.arg)
-			fo.write(", %d);\n" % (i.param["index"]))
-
-		fo.write("#undef PARANOIA\n")
-
-		self.emit_json(fo)
-
-		fo.write("\n")
-		fo.write(self.struct + "*\n")
-		fo.write("VSC_" + self.name + "_New")
-		fo.write("(struct vsmw_cluster *vc,\n")
-		fo.write("    struct vsc_seg **sg, const char *fmt, ...)\n")
-		fo.write("{\n")
-		fo.write("\tva_list ap;\n")
-		fo.write("\t" + self.struct + " *retval;\n")
-		fo.write("\n")
-		fo.write("\tva_start(ap, fmt);\n")
-		fo.write("\tretval = VRT_VSC_Alloc")
-		fo.write("(vc, sg, vsc_" + self.name + "_name, ")
-		fo.write("VSC_" + self.name + "_size,\n")
-		fo.write("\t    vsc_" + self.name + "_json, ")
-		fo.write("sizeof vsc_" + self.name + "_json, fmt, ap);\n")
-		fo.write("\tva_end(ap);\n")
-		fo.write("\treturn(retval);\n")
-		fo.write("}\n")
-
-		fo.write("\n")
-		fo.write("void\n")
-		fo.write("VSC_" + self.name + "_Destroy")
-		fo.write("(struct vsc_seg **sg)\n")
-		fo.write("{\n")
-		fo.write("\tstruct vsc_seg *p;\n")
-		fo.write("\n")
-		fo.write("\tAN(sg);\n")
-		fo.write("\tp = *sg;\n")
-		fo.write("\t*sg = NULL;\n")
-		fo.write('\tVRT_VSC_Destroy(vsc_%s_name, p);\n' % self.name)
-		fo.write("}\n")
-
-		if 'sumfunction' in self.head.param:
-			fo.write("\n")
-			fo.write("void\n")
-			fo.write("VSC_" + self.name + "_Summ")
-			fo.write("(" + self.struct + " *dst, ")
-			fo.write("const " + self.struct + " *src)\n")
-			fo.write("{\n")
-			fo.write("\n")
-			fo.write("\tAN(dst);\n")
-			fo.write("\tAN(src);\n")
-			for i in self.mbrs:
-				s1 = "\tdst->" + i.arg + " +="
-				s2 = "src->" + i.arg + ";"
-				if len((s1 + " " + s2).expandtabs()) < 79:
-					fo.write(s1 + " " + s2 + "\n")
-				else:
-					fo.write(s1 + "\n\t    " + s2 + "\n")
-			fo.write("}\n")
+    fo.write('/*\n')
+    fo.write(' * NB:  This file is machine generated, DO NOT EDIT!\n')
+    fo.write(' *\n')
+    fo.write(' * Edit ' + name +
+             '.vsc and run lib/libvcc/vsctool.py instead.\n')
+    fo.write(' */\n')
+    fo.write('\n')
 
 #######################################################################
 
-class directive(object):
-	def __init__(self, s):
-		ll = s.split("\n")
-		i = ll.pop(0).split("::", 2)
-		self.cmd = i[0]
-		self.arg = i[1].strip()
-		assert len(self.arg.split()) == 1
-
-		self.param = {}
-		while len(ll):
-			j = ll[0].split(":",2)
-			if len(j) != 3 or not j[0].isspace():
-				break
-			self.param[j[1]] = j[2].strip()
-			ll.pop(0)
-		self.ldoc = ll
-
-	def getdoc(self):
-		while len(self.ldoc) and self.ldoc[0].strip() == "":
-			self.ldoc.pop(0)
-		while len(self.ldoc) and self.ldoc[-1].strip() == "":
-			self.ldoc.pop(-1)
-		return self.ldoc
-
-	def moredoc(self, s):
-		self.getdoc()
-		self.ldoc += s.split("\n")
-
-	def emit_rst(self, fo):
-		fo.write("\n.. " + self.cmd + ":: " + self.arg + "\n")
-		self.emit_rst_doc(fo)
-
-	def emit_rst_doc(self, fo):
-		fo.write("\n".join(self.ldoc))
-
-	def emit_h(self, fo):
-		return
-
-class rst_vsc_begin(directive):
-	def __init__(self, s):
-		super(rst_vsc_begin, self).__init__(s)
-
-	def vscset(self, ss):
-		ss.append(vscset(self.arg, self))
-
-	def emit_rst(self, fo):
-		fo.write("\n..\n\t" + self.cmd + ":: " + self.arg + "\n")
-
-		s = self.arg.upper() + " ? " + self.param["oneliner"]
-		fo.write("\n")
-		fo.write(s + "\n")
-		fo.write("=" * len(s) + "\n")
-
-		self.emit_rst_doc(fo)
-
-class rst_vsc(directive):
-	def __init__(self, s):
-		super(rst_vsc, self).__init__(s)
-
-		for i,v in PARAMS.items():
-			if v is not True:
-				self.do_default(i, v[0], v[1])
-
-		for p in self.param:
-			if p in PARAMS:
-				continue
-			sys.stderr.write("Unknown parameter ")
-			sys.stderr.write("'" + p + "'")
-			sys.stderr.write(" on field '" + self.arg + "'\n")
-			exit(2)
-
-	def do_default(self, p, v, s):
-		if p not in self.param:
-			self.param[p] = v
-		if self.param[p] not in s:
-			sys.stderr.write("Wrong " + p + " '" + self.param[p])
-			sys.stderr.write("' on field '" + self.arg + "'\n")
-			exit(2)
-
-
-	def emit_rst(self, fo):
-		fo.write("\n``%s`` ? " % self.arg)
-		fo.write("`%s` - " % self.param["type"])
-		fo.write("%s\n\n" % self.param["level"])
-
-		fo.write("\t" + self.param["oneliner"] + "\n")
-		self.emit_rst_doc(fo)
-
-	def vscset(self, ss):
-		ss[-1].addmbr(self)
-
-
-class rst_vsc_end(directive):
-	def __init__(self, s):
-		super(rst_vsc_end, self).__init__(s)
-
-	def vscset(self, ss):
-		ss[-1].complete()
-
-	def emit_rst(self, fo):
-		fo.write("\n..\n\t" + self.cmd + ":: " + self.arg + "\n")
-		self.emit_rst_doc(fo)
-
-class other(object):
-	def __init__(self, s):
-		self.s = s
-
-	def emit_rst(self, fo):
-		fo.write(self.s)
-
-	def emit_h(self, fo):
-		return
-
-	def vscset(self, ss):
-		return
+class CounterSet(object):
+
+    '''
+        A set of counters
+
+        In the `.vsc` file a CounterSet is everything between a
+
+            .. varnish_vsc_begin::
+
+        and the subsequent
+
+            .. varnish_vsc_end::
+    '''
+
+    def __init__(self, name, m):
+        self.name = name
+        self.struct = "struct VSC_" + name
+        self.mbrs = []
+        self.head = m
+        self.completed = False
+        self.off = 0
+
+    def addmbr(self, m):
+        '''Add a counter'''
+        assert not self.completed
+        self.mbrs.append(m)
+        retval = self.off
+        self.off += 8
+        return retval
+
+    def complete(self, arg):
+        '''Mark set completed'''
+        assert arg == self.name
+        self.completed = True
+
+    def emit_json(self, fo):
+        '''Emit JSON as compact C byte-array and as readable C-comments'''
+        assert self.completed
+        dd = collections.OrderedDict()
+        dd["version"] = "1"
+        dd["name"] = self.name
+        dd["oneliner"] = self.head.param["oneliner"].strip()
+        dd["order"] = int(self.head.param["order"])
+        dd["docs"] = "\n".join(self.head.getdoc())
+        dd["elements"] = len(self.mbrs)
+        el = collections.OrderedDict()
+        dd["elem"] = el
+        for i in self.mbrs:
+            ed = collections.OrderedDict()
+            el[i.arg] = ed
+            for j in PARAMS:
+                if j in i.param:
+                    ed[j] = i.param[j]
+            ed["index"] = i.param["index"]
+            ed["name"] = i.arg
+            ed["docs"] = "\n".join(i.getdoc())
+        s = json.dumps(dd, separators=(",", ":")) + "\0"
+        fo.write("\nstatic const unsigned char")
+        fo.write(" vsc_%s_json[%d] = {\n" % (self.name, len(s)))
+        bz = bytearray(s, encoding="ascii")
+        t = "\t"
+        for i in bz:
+            t += "%d," % i
+            if len(t) >= 69:
+                fo.write(t + "\n")
+                t = "\t"
+        if len(t) > 1:
+            fo.write(t[:-1])
+        fo.write("\n};\n")
+        s = json.dumps(dd, indent=2, separators=(',', ': '))
+        fo.write("\n")
+        for i in s.split("\n"):
+            j = "// " + i
+            if len(j) > 72:
+                fo.write(j[:72] + "[...]\n")
+            else:
+                fo.write(j + "\n")
+        fo.write("\n")
+
+    def emit_h(self):
+        '''Emit .h file'''
+        assert self.completed
+        fon = "VSC_" + self.name + ".h"
+        fo = open(fon, "w")
+        genhdr(fo, self.name)
+        fo.write(self.struct + " {\n")
+        for i in self.mbrs:
+            fo.write("\tuint64_t\t%s;\n" % i.arg)
+        fo.write("};\n")
+        fo.write("\n")
+
+        fo.write("#define VSC_" + self.name +
+                 "_size PRNDUP(sizeof(" + self.struct + "))\n\n")
+
+        fo.write(self.struct + " *VSC_" + self.name + "_New")
+        fo.write("(struct vsmw_cluster *,\n")
+        fo.write("    struct vsc_seg **, const char *fmt, ...);\n")
+
+        fo.write("void VSC_" + self.name + "_Destroy")
+        fo.write("(struct vsc_seg **);\n")
+
+        if 'sumfunction' in self.head.param:
+            fo.write("void VSC_" + self.name + "_Summ")
+            fo.write("(" + self.struct + " *, ")
+            fo.write("const " + self.struct + " *);\n")
+
+    def emit_c_paranoia(self, fo):
+        '''Emit asserts to make sure compiler gets same byte index'''
+        fo.write("#define PARANOIA(a,n)\t\t\t\t\\\n")
+        fo.write("    _Static_assert(\t\t\t\t\\\n")
+        fo.write("\toffsetof(" + self.struct + ", a) == n,\t\\\n")
+        fo.write("\t\"VSC element '\" #a \"' at wrong offset\")\n\n")
+
+        for i in self.mbrs:
+            fo.write("PARANOIA(" + i.arg)
+            fo.write(", %d);\n" % (i.param["index"]))
+
+        fo.write("#undef PARANOIA\n")
+
+    def emit_c_sumfunc(self, fo):
+        '''Emit a function summ up countersets'''
+        fo.write("\n")
+        fo.write("void\n")
+        fo.write("VSC_" + self.name + "_Summ")
+        fo.write("(" + self.struct + " *dst, ")
+        fo.write("const " + self.struct + " *src)\n")
+        fo.write("{\n")
+        fo.write("\n")
+        fo.write("\tAN(dst);\n")
+        fo.write("\tAN(src);\n")
+        for i in self.mbrs:
+            s1 = "\tdst->" + i.arg + " +="
+            s2 = "src->" + i.arg + ";"
+            if len((s1 + " " + s2).expandtabs()) < 79:
+                fo.write(s1 + " " + s2 + "\n")
+            else:
+                fo.write(s1 + "\n\t    " + s2 + "\n")
+        fo.write("}\n")
+
+    def emit_c_newfunc(self, fo):
+        '''Emit New function'''
+        fo.write("\n")
+        fo.write(self.struct + "*\n")
+        fo.write("VSC_" + self.name + "_New")
+        fo.write("(struct vsmw_cluster *vc,\n")
+        fo.write("    struct vsc_seg **sg, const char *fmt, ...)\n")
+        fo.write("{\n")
+        fo.write("\tva_list ap;\n")
+        fo.write("\t" + self.struct + " *retval;\n")
+        fo.write("\n")
+        fo.write("\tva_start(ap, fmt);\n")
+        fo.write("\tretval = VRT_VSC_Alloc")
+        fo.write("(vc, sg, vsc_" + self.name + "_name, ")
+        fo.write("VSC_" + self.name + "_size,\n")
+        fo.write("\t    vsc_" + self.name + "_json, ")
+        fo.write("sizeof vsc_" + self.name + "_json, fmt, ap);\n")
+        fo.write("\tva_end(ap);\n")
+        fo.write("\treturn(retval);\n")
+        fo.write("}\n")
+
+    def emit_c_destroyfunc(self, fo):
+        '''Emit Destroy function'''
+        fo.write("\n")
+        fo.write("void\n")
+        fo.write("VSC_" + self.name + "_Destroy")
+        fo.write("(struct vsc_seg **sg)\n")
+        fo.write("{\n")
+        fo.write("\tstruct vsc_seg *p;\n")
+        fo.write("\n")
+        fo.write("\tAN(sg);\n")
+        fo.write("\tp = *sg;\n")
+        fo.write("\t*sg = NULL;\n")
+        fo.write('\tVRT_VSC_Destroy(vsc_%s_name, p);\n' % self.name)
+        fo.write("}\n")
+
+    def emit_c(self):
+        '''Emit .c file'''
+        assert self.completed
+        fon = "VSC_" + self.name + ".c"
+        fo = open(fon, "w")
+        genhdr(fo, self.name)
+        fo.write('#include "config.h"\n')
+        fo.write('#include <stdio.h>\n')
+        fo.write('#include <stdarg.h>\n')
+        fo.write('#include "vdef.h"\n')
+        fo.write('#include "vas.h"\n')
+        fo.write('#include "vrt.h"\n')
+        fo.write('#include "VSC_%s.h"\n' % self.name)
+
+        fo.write("\n")
+        fo.write('static const char vsc_%s_name[] = "%s";\n' %
+                 (self.name, self.name.upper()))
+
+        fo.write("\n")
+
+        self.emit_c_paranoia(fo)
+        self.emit_json(fo)
+        self.emit_c_newfunc(fo)
+        self.emit_c_destroyfunc(fo)
+        if 'sumfunction' in self.head.param:
+            self.emit_c_sumfunc(fo)
 
 #######################################################################
 
-class vsc_file(object):
-	def __init__(self, fin):
-		self.c = []
-		scs = open(fin).read().split("\n.. ")
-		self.c.append(other(scs[0]))
-		ld = None
-		for i in scs[1:]:
-			j = i.split(None, 1)
-			f = {
-				"varnish_vsc_begin::":	rst_vsc_begin,
-				"varnish_vsc::":	rst_vsc,
-				"varnish_vsc_end::":	rst_vsc_end,
-			}.get(j[0])
-			if f is None:
-				s = "\n.. " + i
-				o = other(s)
-				if ld is not None:
-					ld.moredoc(s)
-			else:
-				o = f(i)
-				ld = o
-			self.c.append(o)
-
-		self.vscset = []
-		for i in self.c:
-			i.vscset(self.vscset)
-
-	def emit_h(self):
-		for i in self.vscset:
-			i.emit_h()
-
-	def emit_c(self):
-		for i in self.vscset:
-			i.emit_c()
-
-	def emit_rst(self):
-		fo = sys.stdout
-		for i in self.c:
-			i.emit_rst(fo)
+class OurDirective(object):
+
+    '''
+        One of our `.. blablabla::` directives in the source file
+    '''
+
+    def __init__(self, s):
+        ll = s.split("\n")
+        i = ll.pop(0).split("::", 2)
+        self.cmd = i[0]
+        self.arg = i[1].strip()
+        assert len(self.arg.split()) == 1
+
+        self.param = {}
+        while ll:
+            j = ll[0].split(":", 2)
+            if len(j) != 3 or not j[0].isspace():
+                break
+            self.param[j[1]] = j[2].strip()
+            ll.pop(0)
+        self.ldoc = ll
+
+    def getdoc(self):
+        '''
+        Get docs for JSON
+
+        Note that these docs end with the first '\n.. ' sequence
+        in the .vsc file, so that we can put a longer and more
+        complex description into the .RST docs than the "long"
+        explanation varnishstat(1) and similar programs provide.
+        '''
+        while self.ldoc and self.ldoc[0].strip() == "":
+            self.ldoc.pop(0)
+        while self.ldoc and self.ldoc[-1].strip() == "":
+            self.ldoc.pop(-1)
+        return self.ldoc
+
+    def emit_rst(self, fo):
+        '''Emit the documentation as .rst'''
+        assert False
+
+class RstVscDirectiveBegin(OurDirective):
+
+    '''
+        `varnish_vsc_begin` directive
+    '''
+
+    def __init__(self, s, vsc_set, fo):
+        super(RstVscDirectiveBegin, self).__init__(s)
+        vsc_set.append(CounterSet(self.arg, self))
+        if fo:
+            fo.write("\n..\n\t" + self.cmd + ":: " + self.arg + "\n")
+
+            s = self.arg.upper() + " ? " + self.param["oneliner"]
+            fo.write("\n")
+            fo.write(s + "\n")
+            fo.write("=" * len(s) + "\n")
+            fo.write("\n".join(self.ldoc))
+
+class RstVscDirective(OurDirective):
+
+    '''
+        `varnish_vsc` directive - one counter
+    '''
+
+    def __init__(self, s, vsc_set, fo):
+        assert not vsc_set or vsc_set[-1].complete
+        super(RstVscDirective, self).__init__(s)
+
+        for i, v in PARAMS.items():
+            if v is not None:
+                if i not in self.param:
+                    self.param[i] = v[0]
+                if self.param[i] not in v:
+                    sys.stderr.write("Wrong " + i + " '" + self.param[i])
+                    sys.stderr.write("' on field '" + self.arg + "'\n")
+                    exit(2)
+
+        for p in self.param:
+            if p in PARAMS:
+                continue
+            sys.stderr.write("Unknown parameter ")
+            sys.stderr.write("'" + p + "'")
+            sys.stderr.write(" on field '" + self.arg + "'\n")
+            exit(2)
+        self.param["index"] = vsc_set[-1].addmbr(self)
+        if fo:
+            fo.write("\n``%s`` ? " % self.arg)
+            fo.write("`%s` - " % self.param["type"])
+            fo.write("%s\n\n" % self.param["level"])
+
+            fo.write("\t" + self.param["oneliner"] + "\n")
+            fo.write("\n".join(self.ldoc))
+
+class RstVscDirectiveEnd(OurDirective):
+
+    '''
+        `varnish_vsc_end` directive
+    '''
+
+    def __init__(self, s, vsc_set, fo):
+        super(RstVscDirectiveEnd, self).__init__(s)
+        vsc_set[-1].complete(self.arg)
+        if fo:
+            fo.write("\n..\n\t" + self.cmd + ":: " + self.arg + "\n")
+            fo.write("\n".join(self.ldoc))
 
 #######################################################################
 
-if __name__ == "__main__":
-
-	optlist, args = getopt.getopt(sys.argv[1:], "chr")
+def mainfunc(argv):
+
+    '''Process a .vsc file'''
+
+    optlist, args = getopt.getopt(argv[1:], "chr")
+
+    if len(args) != 1:
+        sys.stderr.write("Need exactly one filename argument\n")
+        exit(2)
+
+    rstfile = None
+    for f, v in optlist:
+        if f == '-r':
+            rstfile = sys.stdout
+
+    vscset = []
+    scs = open(args[0]).read().split("\n.. ")
+    if rstfile:
+        rstfile.write(scs[0])
+    for i in scs[1:]:
+        j = i.split(None, 1)
+        f = {
+            "varnish_vsc_begin::":  RstVscDirectiveBegin,
+            "varnish_vsc::":        RstVscDirective,
+            "varnish_vsc_end::":    RstVscDirectiveEnd,
+        }.get(j[0])
+        if f is not None:
+            f(i, vscset, rstfile)
+        elif rstfile:
+            rstfile.write("\n.. " + i)
+
+    for i in vscset:
+        for f, v in optlist:
+            if f == '-h':
+                i.emit_h()
+            if f == '-c':
+                i.emit_c()
 
-	fo = sys.stdout
-
-	if len(args) != 1:
-		sys.stderr.write("Need exactly one filename argument\n")
-		exit(2)
+if __name__ == "__main__":
 
-	vf = vsc_file(args[0])
-	for f,v in optlist:
-		if f == '-r':
-			vf.emit_rst()
-		if f == '-h':
-			vf.emit_h()
-		if f == '-c':
-			vf.emit_c()
+    mainfunc(sys.argv)
diff --git a/tools/gcov_digest.py b/tools/gcov_digest.py
index 138f7e8cf..188fb499d 100644
--- a/tools/gcov_digest.py
+++ b/tools/gcov_digest.py
@@ -32,19 +32,19 @@ found in a subdirectory tree.
 
 Options:
 
-	-g gcov-program
-		default:gcov6
+    -g gcov-program
+        default:gcov6
 
-	-o output-filename
-		default: stdout
+    -o output-filename
+        default: stdout
 
-	-x exclude-subdir
-		default ".git" and ".deps"
+    -x exclude-subdir
+        default ".git" and ".deps"
 
  Arguments:
 
-	directories to process.
-	default: .
+    directories to process.
+    default: .
 
 """
 
@@ -61,141 +61,141 @@ lengths = {}
 exclude = [".git", ".deps",]
 
 def process_gcov(fn, sn):
-	""" Sum .gcov file into counts, then delete it """
-	dd = counts.get(sn)
-	if dd is None:
-		dd = {}
-	for ln in open(fn):
-		d = ln.split(":")
-		cnt = d[0].strip()
-		ll = d[1]
-		if cnt == "-":
-			continue
-		if cnt == "#####":
-			cnt = 0
-		else:
-			cnt = int(cnt)
-		lno = int(d[1])
-		if lno not in dd:
-			dd[lno] = 0
-		dd[lno] += cnt
-	counts[sn] = dd
-	pl = lengths.get(sn)
-	ll = ll.strip()
-	if d[2] == "/*EOF*/\n":
-		ll = pl
-	elif pl != ll and not pl is None:
-		print("CONFLICT", fn, ll, pl)
-		ll = "-1"
-	lengths[sn] = ll
-	os.remove(fn)
+    """ Sum .gcov file into counts, then delete it """
+    dd = counts.get(sn)
+    if dd is None:
+        dd = {}
+    for ln in open(fn):
+        d = ln.split(":")
+        cnt = d[0].strip()
+        ll = d[1]
+        if cnt == "-":
+            continue
+        if cnt == "#####":
+            cnt = 0
+        else:
+            cnt = int(cnt)
+        lno = int(d[1])
+        if lno not in dd:
+            dd[lno] = 0
+        dd[lno] += cnt
+    counts[sn] = dd
+    pl = lengths.get(sn)
+    ll = ll.strip()
+    if d[2] == "/*EOF*/\n":
+        ll = pl
+    elif pl != ll and not pl is None:
+        print("CONFLICT", fn, ll, pl)
+        ll = "-1"
+    lengths[sn] = ll
+    os.remove(fn)
 
 def run_gcov(prog, subdir):
-	""" Walk tree, run gcov, process output """
-	for root, dirs, files in os.walk(subdir):
-		for i in exclude:
-			if i in dirs:
-				dirs.remove(i)
-		if " ".join(files).find(".gcda") == -1:
-			continue
-		for fn in files:
-			if fn[-2:] != ".o":
-				continue
-
-			# if we find the .o file in a .../.libs the sources
-			# must be found relative to the parent directory
-
-			if root[-6:] == "/.libs":
-				x = subprocess.check_output(
-				    ["cd " + root + "/.. && " +
-				     "exec " + prog + " -r .libs/" + fn],
-				    stderr=subprocess.STDOUT, shell=True,
-				    universal_newlines=True)
-				pf = ".."
-			else:
-				x = subprocess.check_output(
-				    ["cd " + root + " && " +
-				     "exec " + prog + " -r " + fn],
-				    stderr=subprocess.STDOUT, shell=True,
-				    universal_newlines=True)
-				pf = ""
-
-			for ln in x.split("\n"):
-				ln = ln.split()
-				if len(ln) == 0:
-					continue
-				if ln[0] == "Creating":
-					gn = ln[1].strip("'")
-					assert gn[-5:] == ".gcov"
-					sn = gn[:-5]
-					process_gcov(
-					    os.path.join(root, pf, gn), sn)
+    """ Walk tree, run gcov, process output """
+    for root, dirs, files in os.walk(subdir):
+        for i in exclude:
+            if i in dirs:
+                dirs.remove(i)
+        if " ".join(files).find(".gcda") == -1:
+            continue
+        for fn in files:
+            if fn[-2:] != ".o":
+                continue
+
+            # if we find the .o file in a .../.libs the sources
+            # must be found relative to the parent directory
+
+            if root[-6:] == "/.libs":
+                x = subprocess.check_output(
+                    ["cd " + root + "/.. && " +
+                     "exec " + prog + " -r .libs/" + fn],
+                    stderr=subprocess.STDOUT, shell=True,
+                    universal_newlines=True)
+                pf = ".."
+            else:
+                x = subprocess.check_output(
+                    ["cd " + root + " && " +
+                     "exec " + prog + " -r " + fn],
+                    stderr=subprocess.STDOUT, shell=True,
+                    universal_newlines=True)
+                pf = ""
+
+            for ln in x.split("\n"):
+                ln = ln.split()
+                if not ln:
+                    continue
+                if ln[0] == "Creating":
+                    gn = ln[1].strip("'")
+                    assert gn[-5:] == ".gcov"
+                    sn = gn[:-5]
+                    process_gcov(
+                        os.path.join(root, pf, gn), sn)
 
 def produce_output(fdo):
-	"""
-	Produce compact output file
-
-	Format:
-		linefm [lineto] count
-
-		"+" in linefm means "previous line + 1"
-		"." in count means "same as previous count"
-	"""
-
-	for sn, cnt in counts.items():
-		fdo.write("/" + sn + " " + str(lengths[sn]) + "\n")
-		lnos = list(cnt.items())
-		lnos.sort()
-		pln = -1
-		pcn = -1
-		while len(lnos) > 0:
-			ln, cn = lnos.pop(0)
-			lnl = ln
-			while len(lnos) > 0:
-				lnn, cnn = lnos[0]
-				if lnl + 1 != lnn or cnn != cn:
-					break
-				lnos.pop(0)
-				lnl = lnn
-			if ln == pln + 1:
-				s = "+ "
-			else:
-				s = "%d " % ln
-
-			if ln != lnl:
-				s += "%d " % lnl
-				pln = lnl
-			else:
-				pln = ln
-
-			if cn == pcn:
-				s += "."
-			else:
-				s += "%d" % cn
-				pcn = cn
-			fdo.write(s + "\n")
+    """
+    Produce compact output file
+
+    Format:
+        linefm [lineto] count
+
+        "+" in linefm means "previous line + 1"
+        "." in count means "same as previous count"
+    """
+
+    for sn, cnt in counts.items():
+        fdo.write("/" + sn + " " + str(lengths[sn]) + "\n")
+        lnos = list(cnt.items())
+        lnos.sort()
+        pln = -1
+        pcn = -1
+        while lnos:
+            ln, cn = lnos.pop(0)
+            lnl = ln
+            while lnos:
+                lnn, cnn = lnos[0]
+                if lnl + 1 != lnn or cnn != cn:
+                    break
+                lnos.pop(0)
+                lnl = lnn
+            if ln == pln + 1:
+                s = "+ "
+            else:
+                s = "%d " % ln
+
+            if ln != lnl:
+                s += "%d " % lnl
+                pln = lnl
+            else:
+                pln = ln
+
+            if cn == pcn:
+                s += "."
+            else:
+                s += "%d" % cn
+                pcn = cn
+            fdo.write(s + "\n")
 
 if __name__ == "__main__":
 
-	optlist, args = getopt.getopt(sys.argv[1:], "g:o:x:")
-
-	fo = sys.stdout
-	gcovprog = "gcov6"
-
-	for f, v in optlist:
-		if f == '-o' and v == '-':
-			fo = sys.stdout
-		elif f == '-o':
-			fo = open(v, "w")
-		elif f == '-g':
-			gcovprog = v
-		elif f == '-x':
-			exclude.append(v)
-		else:
-			assert False
-	if len(args) == 0:
-		args = ["."]
-	for dn in args:
-		run_gcov(gcovprog, dn)
-
-	produce_output(fo)
+    optlist, args = getopt.getopt(sys.argv[1:], "g:o:x:")
+
+    fo = sys.stdout
+    gcovprog = "gcov6"
+
+    for f, v in optlist:
+        if f == '-o' and v == '-':
+            fo = sys.stdout
+        elif f == '-o':
+            fo = open(v, "w")
+        elif f == '-g':
+            gcovprog = v
+        elif f == '-x':
+            exclude.append(v)
+        else:
+            assert False
+    if not args:
+        args = ["."]
+    for dn in args:
+        run_gcov(gcovprog, dn)
+
+    produce_output(fo)
diff --git a/tools/include_wash.py b/tools/include_wash.py
index 1284e30e1..4df6eeaf3 100644
--- a/tools/include_wash.py
+++ b/tools/include_wash.py
@@ -5,41 +5,39 @@ from __future__ import print_function
 import os
 
 def check(fn):
-	l = []
-	for i in open(fn):
-		i = i.strip()
-		if len(i) == 0:
-			continue
-		if i[0] != "#":
-			continue
-		if i.find("include") == -1:
-			continue
-		if i.find('"') == -1:
-			l.append(i.split('<')[1].split('>')[0])
-		else:
-			l.append(i.split('"')[1])
-	if "vrt.h" in l:
-		vrt = l.index("vrt.h")
-		if not "vdef.h" in l:
-			print(fn, "vdef.h not included with vrt.h")
-		vdef = l.index("vdef.h")
-		if vdef > vrt:
-			print(fn, "vdef.h included after vrt.h")
-		for i in ("stddef.h", "stdint.h", "cache/cache.h", "cache.h"):
-			if i in l:
-				print(fn, i + " included with vrt.h")
+    l = []
+    for i in open(fn):
+        i = i.strip()
+        if not i:
+            continue
+        if i[0] != "#":
+            continue
+        if i.find("include") == -1:
+            continue
+        if i.find('"') == -1:
+            l.append(i.split('<')[1].split('>')[0])
+        else:
+            l.append(i.split('"')[1])
+    if "vrt.h" in l:
+        vrt = l.index("vrt.h")
+        if "vdef.h" not in l:
+            print(fn, "vdef.h not included with vrt.h")
+        vdef = l.index("vdef.h")
+        if vdef > vrt:
+            print(fn, "vdef.h included after vrt.h")
+        for i in ("stddef.h", "stdint.h", "cache/cache.h", "cache.h"):
+            if i in l:
+                print(fn, i + " included with vrt.h")
 
-	for i in ("cache/cache.h", "cache.h"):
-		if i in l:
-			for i in (
-			    "stddef.h", "stdint.h", "vrt.h",
-			    "math.h", "pthread.h", "stdarg.h", "sys/types.h",
-			    "vdef.h", "miniobj.h", "vas.h", "vqueue.h",
-			):
-				if i in l:
-					print(fn, i + " included with cache.h")
+    for i in ("cache/cache.h", "cache.h"):
+        if i in l:
+            for i in ("stddef.h", "stdint.h", "vrt.h",
+                      "math.h", "pthread.h", "stdarg.h", "sys/types.h",
+                      "vdef.h", "miniobj.h", "vas.h", "vqueue.h"):
+                if i in l:
+                    print(fn, i + " included with cache.h")
 
 for (dir, dns, fns) in os.walk("."):
-	for f in fns:
-		if f[-2:] == ".c":
-			check(dir + "/" + f)
+    for f in fns:
+        if f[-2:] == ".c":
+            check(dir + "/" + f)

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:21 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:21 +0000 (UTC)
Subject: [6.0] c5311ef7a Update Teken from FreeBSD source tree
Message-ID: <20181031130821.31665954AF@lists.varnish-cache.org>


commit c5311ef7ab6ee3dcdb2972b1729b469953c20dd1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 07:31:46 2018 +0000

    Update Teken from FreeBSD source tree

diff --git a/bin/varnishtest/gensequences b/bin/varnishtest/gensequences
index 8c2299a61..e82e56ec7 100644
--- a/bin/varnishtest/gensequences
+++ b/bin/varnishtest/gensequences
@@ -25,7 +25,7 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
-# $FreeBSD: head/sys/teken/gensequences 223574 2011-06-26 18:25:10Z ed $
+# $FreeBSD: head/sys/teken/gensequences 333925 2018-05-20 14:21:20Z dumbbell $
 
 function die(msg) {
 	print msg;
@@ -35,6 +35,15 @@ function die(msg) {
 function cchar(str) {
 	if (str == "^[")
 		return "\\x1B";
+	if (str == "SP")
+		return " ";
+
+	return str;
+}
+
+function csequence(str) {
+	if (str == "SP")
+		return " ";
 
 	return str;
 }
@@ -57,11 +66,11 @@ while (getline > 0) {
 	prefix = "";
 	l_prefix_name[""] = "teken_state_init";
 	for (i = 1; i < nsequences; i++) {
-		n = prefix sequence[i];
+		n = prefix csequence(sequence[i]);
 		l_prefix_parent[n] = prefix;
 		l_prefix_suffix[n] = sequence[i];
 		if (!l_prefix_name[n])
-			l_prefix_name[n] = "teken_state_" "" ++npr;
+			l_prefix_name[n] = "teken_state_" ++npr;
 		prefix = n;
 	}
 
diff --git a/bin/varnishtest/sequences b/bin/varnishtest/sequences
index af92df04b..d8f30306b 100644
--- a/bin/varnishtest/sequences
+++ b/bin/varnishtest/sequences
@@ -23,93 +23,94 @@
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
-# $FreeBSD: head/sys/teken/sequences 322662 2017-08-18 15:40:40Z bde $
+# $FreeBSD: head/sys/teken/sequences 334316 2018-05-29 08:41:44Z dumbbell $
 
 # File format is as follows:
 #	Abbr		Abbreviation of sequence name
 #	Name		Sequence name (will be converted to C function name)
 #	Sequence	Bytes that form the sequence
-#	Arguments	Standard value of arguments passed to this sequence
+#	Args		Standard value of arguments passed to this sequence
 #			- `n' non-zero number (0 gets converted to 1)
 #			- `r' regular numeric argument
 #			- `v' means a variable number of arguments
 
-# Abbr	Name					Sequence	Arguments
-CBT	Cursor Backward Tabulation		^[ [ Z		n
-CHT	Cursor Forward Tabulation		^[ [ I		n
-CNL	Cursor Next Line			^[ [ E		n
-CPL	Cursor Previous Line			^[ [ F		n
-CPR	Cursor Position Report			^[ [ n		r
-CUB	Cursor Backward				^[ [ D		n
-CUD	Cursor Down				^[ [ B		n
-CUD	Cursor Down				^[ [ e		n
-CUF	Cursor Forward				^[ [ C		n
-CUF	Cursor Forward				^[ [ a		n
-CUP	Cursor Position				^[ [ H		n n
-CUP	Cursor Position				^[ [ f		n n
-CUU	Cursor Up				^[ [ A		n
-DA1	Primary Device Attributes		^[ [ c		r
-DA2	Secondary Device Attributes		^[ [ > c	r
-DC	Delete character			^[ [ P		n
-DCS	Device Control String			^[ P
-DECALN	Alignment test				^[ # 8
-DECDHL	Double Height Double Width Line Top	^[ # 3
-DECDHL	Double Height Double Width Line Bottom	^[ # 4
-DECDWL	Single Height Double Width Line		^[ # 6
-DECKPAM	Keypad application mode			^[ =
-DECKPNM	Keypad numeric mode			^[ >
-DECRC	Restore cursor				^[ 8
-DECRC	Restore cursor				^[ [ u
-DECRM	Reset DEC mode				^[ [ ? l	r
-DECSC	Save cursor				^[ 7
-DECSC	Save cursor				^[ [ s
-DECSM	Set DEC mode				^[ [ ? h	r
-DECSTBM	Set top and bottom margins		^[ [ r		r r
-DECSWL	Single Height Single Width Line		^[ # 5
-DL	Delete line				^[ [ M		n
-DSR	Device Status Report			^[ [ ? n	r
-ECH	Erase character				^[ [ X		n
-ED	Erase display				^[ [ J		r
-EL	Erase line				^[ [ K		r
-G0SCS0	G0 SCS Special Graphics			^[ ( 0
-G0SCS1	G0 SCS US ASCII				^[ ( 1
-G0SCS2	G0 SCS Special Graphics			^[ ( 2
-G0SCSA	G0 SCS UK National			^[ ( A
-G0SCSB	G0 SCS US ASCII				^[ ( B
-G1SCS0	G1 SCS Special Graphics			^[ ) 0
-G1SCS1	G1 SCS US ASCII				^[ ) 1
-G1SCS2	G1 SCS Special Graphics			^[ ) 2
-G1SCSA	G1 SCS UK National			^[ ) A
-G1SCSB	G1 SCS US ASCII				^[ ) B
-HPA	Horizontal Position Absolute		^[ [ G		n
-HPA	Horizontal Position Absolute		^[ [ `		n
-HTS	Horizontal Tab Set			^[ H
-ICH	Insert character			^[ [ @		n
-IL	Insert line				^[ [ L		n
-IND	Index					^[ D
-NEL	Next line				^[ E
-OSC	Operating System Command		^[ ]
-RI	Reverse index				^[ M
-RIS	Reset to Initial State			^[ c
-RM	Reset Mode				^[ [ l		r
-SD	Pan Up					^[ [ T		n
-SGR	Set Graphic Rendition			^[ [ m		v
-SM	Set Mode				^[ [ h		r
-ST	String Terminator			^[ \\
-SU	Pan Down				^[ [ S		n
-TBC	Tab Clear				^[ [ g		r
-VPA	Vertical Position Absolute		^[ [ d		n
+# Abbr		Name					Sequence	Args
+CBT		Cursor Backward Tabulation		^[ [ Z		n
+CHT		Cursor Forward Tabulation		^[ [ I		n
+CNL		Cursor Next Line			^[ [ E		n
+CPL		Cursor Previous Line			^[ [ F		n
+CPR		Cursor Position Report			^[ [ n		r
+CUB		Cursor Backward				^[ [ D		n
+CUD		Cursor Down				^[ [ B		n
+CUD		Cursor Down				^[ [ e		n
+CUF		Cursor Forward				^[ [ C		n
+CUF		Cursor Forward				^[ [ a		n
+CUP		Cursor Position				^[ [ H		n n
+CUP		Cursor Position				^[ [ f		n n
+CUU		Cursor Up				^[ [ A		n
+DA1		Primary Device Attributes		^[ [ c		r
+DA2		Secondary Device Attributes		^[ [ > c	r
+DC		Delete character			^[ [ P		n
+DCS		Device Control String			^[ P
+DECALN		Alignment test				^[ # 8
+DECDHL		Double Height Double Width Line Top	^[ # 3
+DECDHL		Double Height Double Width Line Bottom	^[ # 4
+DECDWL		Single Height Double Width Line		^[ # 6
+DECKPAM		Keypad application mode			^[ =
+DECKPNM		Keypad numeric mode			^[ >
+DECRC		Restore cursor				^[ 8
+DECRC		Restore cursor				^[ [ u
+DECRM		Reset DEC mode				^[ [ ? l	r
+DECSC		Save cursor				^[ 7
+DECSC		Save cursor				^[ [ s
+DECSCUSR	Set Cursor Style			^[ [ SP q	r
+DECSM		Set DEC mode				^[ [ ? h	r
+DECSTBM		Set top and bottom margins		^[ [ r		r r
+DECSWL		Single Height Single Width Line		^[ # 5
+DL		Delete line				^[ [ M		n
+DSR		Device Status Report			^[ [ ? n	r
+ECH		Erase character				^[ [ X		n
+ED		Erase display				^[ [ J		r
+EL		Erase line				^[ [ K		r
+G0SCS0		G0 SCS Special Graphics			^[ ( 0
+G0SCS1		G0 SCS US ASCII				^[ ( 1
+G0SCS2		G0 SCS Special Graphics			^[ ( 2
+G0SCSA		G0 SCS UK National			^[ ( A
+G0SCSB		G0 SCS US ASCII				^[ ( B
+G1SCS0		G1 SCS Special Graphics			^[ ) 0
+G1SCS1		G1 SCS US ASCII				^[ ) 1
+G1SCS2		G1 SCS Special Graphics			^[ ) 2
+G1SCSA		G1 SCS UK National			^[ ) A
+G1SCSB		G1 SCS US ASCII				^[ ) B
+HPA		Horizontal Position Absolute		^[ [ G		n
+HPA		Horizontal Position Absolute		^[ [ `		n
+HTS		Horizontal Tab Set			^[ H
+ICH		Insert character			^[ [ @		n
+IL		Insert line				^[ [ L		n
+IND		Index					^[ D
+NEL		Next line				^[ E
+OSC		Operating System Command		^[ ]
+RI		Reverse index				^[ M
+RIS		Reset to Initial State			^[ c
+RM		Reset Mode				^[ [ l		r
+SD		Pan Up					^[ [ T		n
+SGR		Set Graphic Rendition			^[ [ m		v
+SM		Set Mode				^[ [ h		r
+ST		String Terminator			^[ \\
+SU		Pan Down				^[ [ S		n
+TBC		Tab Clear				^[ [ g		r
+VPA		Vertical Position Absolute		^[ [ d		n
 
 # Cons25 compatibility sequences
-C25BLPD	Cons25 set bell pitch duration		^[ [ = B	r r
-C25BORD	Cons25 set border			^[ [ = A	r
-C25DBG	Cons25 set default background		^[ [ = G	r
-C25DFG	Cons25 set default foreground		^[ [ = F	r
-C25GCS	Cons25 set global cursor shape		^[ [ = C	v
-C25LCT	Cons25 set local cursor type		^[ [ = S	r
-C25MODE	Cons25 set terminal mode		^[ [ = T	r
-C25SGR	Cons25 set graphic rendition		^[ [ x		r r
-C25VTSW	Cons25 switch virtual terminal		^[ [ z		r
+C25BLPD		Cons25 set bell pitch duration		^[ [ = B	r r
+C25BORD		Cons25 set border			^[ [ = A	r
+C25DBG		Cons25 set default background		^[ [ = G	r
+C25DFG		Cons25 set default foreground		^[ [ = F	r
+C25GCS		Cons25 set global cursor shape		^[ [ = C	v
+C25LCT		Cons25 set local cursor type		^[ [ = S	r
+C25MODE		Cons25 set terminal mode		^[ [ = T	r
+C25SGR		Cons25 set graphic rendition		^[ [ x		r r
+C25VTSW		Cons25 switch virtual terminal		^[ [ z		r
 
 # VT52 compatibility
-#DECID	VT52 DECID				^[ Z
+#DECID		VT52 DECID				^[ Z
diff --git a/bin/varnishtest/teken.c b/bin/varnishtest/teken.c
index ad06c57c6..21eb71752 100644
--- a/bin/varnishtest/teken.c
+++ b/bin/varnishtest/teken.c
@@ -25,7 +25,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * $FreeBSD: head/sys/teken/teken.c 326272 2017-11-27 15:23:17Z pfg $
+ * $FreeBSD: head/sys/teken/teken.c 333683 2018-05-16 18:12:49Z cem $
  */
 
 #include "config.h"
@@ -35,14 +35,13 @@
 #include <stdint.h>
 #include <stdio.h>
 #include <string.h>
+#define	teken_assert(x)		assert(x)
 
 #include "vdef.h"
 #include "vas.h"
 
-#define	teken_assert(x)		assert(x)
-
 /* debug messages */
-#define	teken_printf(...)
+#define	teken_printf(x,...)
 
 /* Private flags for t_stateflags. */
 #define	TS_FIRSTDIGIT	0x0001	/* First numeric digit in escape sequence. */
@@ -128,20 +127,36 @@ teken_funcs_copy(const teken_t *t, const teken_rect_t *r, const teken_pos_t *p)
 	t->t_funcs->tf_copy(t->t_softc, r, p);
 }
 
+static inline void
+teken_funcs_pre_input(const teken_t *t)
+{
+
+	if (t->t_funcs->tf_pre_input != NULL)
+		t->t_funcs->tf_pre_input(t->t_softc);
+}
+
+static inline void
+teken_funcs_post_input(const teken_t *t)
+{
+
+	if (t->t_funcs->tf_post_input != NULL)
+		t->t_funcs->tf_post_input(t->t_softc);
+}
+
 static inline void
 teken_funcs_param(const teken_t *t, int cmd, unsigned int value)
 {
 
-	if (t->t_funcs->tf_param != NULL)
-		t->t_funcs->tf_param(t->t_softc, cmd, value);
+	teken_assert(t->t_funcs->tf_param != NULL);
+	t->t_funcs->tf_param(t->t_softc, cmd, value);
 }
 
 static inline void
 teken_funcs_respond(const teken_t *t, const void *buf, size_t len)
 {
 
-	if (t->t_funcs->tf_respond != NULL)
-		t->t_funcs->tf_respond(t->t_softc, buf, len);
+	teken_assert(t->t_funcs->tf_respond != NULL);
+	t->t_funcs->tf_respond(t->t_softc, buf, len);
 }
 
 #include "teken_subr.h"
@@ -288,8 +303,10 @@ teken_input(teken_t *t, const void *buf, size_t len)
 {
 	const char *c = buf;
 
+	teken_funcs_pre_input(t);
 	while (len-- > 0)
 		teken_input_byte(t, *c++);
+	teken_funcs_post_input(t);
 }
 
 const teken_pos_t *
diff --git a/bin/varnishtest/teken.h b/bin/varnishtest/teken.h
index 986c3bd36..eb59817d7 100644
--- a/bin/varnishtest/teken.h
+++ b/bin/varnishtest/teken.h
@@ -25,7 +25,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * $FreeBSD: head/sys/teken/teken.h 326272 2017-11-27 15:23:17Z pfg $
+ * $FreeBSD: head/sys/teken/teken.h 333669 2018-05-16 09:01:02Z dumbbell $
  */
 
 #ifndef _TEKEN_H_
@@ -93,6 +93,8 @@ typedef void tf_putchar_t(void *, const teken_pos_t *, teken_char_t,
 typedef void tf_fill_t(void *, const teken_rect_t *, teken_char_t,
     const teken_attr_t *);
 typedef void tf_copy_t(void *, const teken_rect_t *, const teken_pos_t *);
+typedef void tf_pre_input_t(void *);
+typedef void tf_post_input_t(void *);
 typedef void tf_param_t(void *, int, unsigned int);
 #define	TP_SHOWCURSOR	0
 #define	TP_KEYPADAPP	1
@@ -114,6 +116,8 @@ typedef struct {
 	tf_putchar_t	*tf_putchar;
 	tf_fill_t	*tf_fill;
 	tf_copy_t	*tf_copy;
+	tf_pre_input_t	*tf_pre_input;
+	tf_post_input_t	*tf_post_input;
 	tf_param_t	*tf_param;
 	tf_respond_t	*tf_respond;
 } teken_funcs_t;
diff --git a/bin/varnishtest/teken_scs.h b/bin/varnishtest/teken_scs.h
index fd99de15d..719f2a98e 100644
--- a/bin/varnishtest/teken_scs.h
+++ b/bin/varnishtest/teken_scs.h
@@ -25,7 +25,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * $FreeBSD: head/sys/teken/teken_scs.h 326272 2017-11-27 15:23:17Z pfg $
+ * $FreeBSD: head/sys/teken/teken_scs.h 332297 2018-04-08 19:23:50Z phk $
  */
 
 static inline teken_char_t
diff --git a/bin/varnishtest/teken_subr.h b/bin/varnishtest/teken_subr.h
index b67ef5cc0..22d06bb19 100644
--- a/bin/varnishtest/teken_subr.h
+++ b/bin/varnishtest/teken_subr.h
@@ -25,7 +25,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * $FreeBSD: head/sys/teken/teken_subr.h 326272 2017-11-27 15:23:17Z pfg $
+ * $FreeBSD: head/sys/teken/teken_subr.h 333995 2018-05-21 20:35:16Z dumbbell $
  */
 
 static void teken_subr_cursor_up(teken_t *, unsigned int);
@@ -371,6 +371,27 @@ teken_subr_cursor_up(teken_t *t, unsigned int nrows)
 	teken_funcs_cursor(t);
 }
 
+static void
+teken_subr_set_cursor_style(teken_t *t, unsigned int style)
+{
+
+	/* TODO */
+	(void)t;
+	(void)style;
+
+	/*
+	 * CSI Ps SP q
+	 *   Set cursor style (DECSCUSR), VT520.
+	 *     Ps = 0  -> blinking block.
+	 *     Ps = 1  -> blinking block (default).
+	 *     Ps = 2  -> steady block.
+	 *     Ps = 3  -> blinking underline.
+	 *     Ps = 4  -> steady underline.
+	 *     Ps = 5  -> blinking bar (xterm).
+	 *     Ps = 6  -> steady bar (xterm).
+	 */
+}
+
 static void
 teken_subr_delete_character(const teken_t *t, unsigned int ncols)
 {
diff --git a/bin/varnishtest/teken_subr_compat.h b/bin/varnishtest/teken_subr_compat.h
index 9c84f1331..7db4d858d 100644
--- a/bin/varnishtest/teken_subr_compat.h
+++ b/bin/varnishtest/teken_subr_compat.h
@@ -25,7 +25,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * $FreeBSD: head/sys/teken/teken_subr_compat.h 326272 2017-11-27 15:23:17Z pfg $
+ * $FreeBSD: head/sys/teken/teken_subr_compat.h 332297 2018-04-08 19:23:50Z phk $
  */
 
 static void
diff --git a/bin/varnishtest/teken_wcwidth.h b/bin/varnishtest/teken_wcwidth.h
index 73401698f..70d92060f 100644
--- a/bin/varnishtest/teken_wcwidth.h
+++ b/bin/varnishtest/teken_wcwidth.h
@@ -7,7 +7,7 @@
  *
  * Latest version: http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
  *
- * $FreeBSD: head/sys/teken/teken_wcwidth.h 186681 2009-01-01 13:26:53Z ed $
+ * $FreeBSD: head/sys/teken/teken_wcwidth.h 332297 2018-04-08 19:23:50Z phk $
  */
 
 struct interval {

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:21 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:21 +0000 (UTC)
Subject: [6.0] 2d0ee3eb9 Introduce support for 'groups' inside a VSC
Message-ID: <20181031130821.59998954D6@lists.varnish-cache.org>


commit 2d0ee3eb908ac804627955fa78a4c45d9c3be006
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 07:33:39 2018 +0000

    Introduce support for 'groups' inside a VSC

diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py
index 6c96388e5..d4e210deb 100644
--- a/lib/libvcc/vsctool.py
+++ b/lib/libvcc/vsctool.py
@@ -53,6 +53,7 @@ PARAMS = {
     "ctype": CTYPES,
     "level": LEVELS,
     "oneliner": None,
+    "group": None,
     "format": FORMATS,
 }
 
@@ -88,22 +89,30 @@ class CounterSet(object):
         self.name = name
         self.struct = "struct VSC_" + name
         self.mbrs = []
+        self.groups = {}
         self.head = m
         self.completed = False
         self.off = 0
 
-    def addmbr(self, m):
+    def addmbr(self, m, g):
         '''Add a counter'''
         assert not self.completed
         self.mbrs.append(m)
         retval = self.off
         self.off += 8
+        if g is not None:
+            if g not in self.groups:
+                self.groups[g] = []
+            self.groups[g].append(m)
         return retval
 
     def complete(self, arg):
         '''Mark set completed'''
         assert arg == self.name
         self.completed = True
+        self.gnames = list(self.groups.keys())
+        self.gnames.sort()
+
 
     def emit_json(self, fo):
         '''Emit JSON as compact C byte-array and as readable C-comments'''
@@ -152,15 +161,24 @@ class CounterSet(object):
     def emit_h(self):
         '''Emit .h file'''
         assert self.completed
+
         fon = "VSC_" + self.name + ".h"
         fo = open(fon, "w")
         genhdr(fo, self.name)
+
         fo.write(self.struct + " {\n")
         for i in self.mbrs:
             fo.write("\tuint64_t\t%s;\n" % i.arg)
         fo.write("};\n")
         fo.write("\n")
 
+        for i in self.gnames:
+            fo.write(self.struct + "_" + i + " {\n")
+            for j in self.groups[i]:
+                fo.write("\tuint64_t\t%s;\n" % j.arg)
+            fo.write("};\n")
+            fo.write("\n")
+
         fo.write("#define VSC_" + self.name +
                  "_size PRNDUP(sizeof(" + self.struct + "))\n\n")
 
@@ -175,6 +193,10 @@ class CounterSet(object):
             fo.write("void VSC_" + self.name + "_Summ")
             fo.write("(" + self.struct + " *, ")
             fo.write("const " + self.struct + " *);\n")
+            for i in self.gnames:
+                fo.write("void VSC_" + self.name + "_Summ_" + i)
+                fo.write("(" + self.struct + " *, ")
+                fo.write("const " + self.struct + "_" + i + " *);\n")
 
     def emit_c_paranoia(self, fo):
         '''Emit asserts to make sure compiler gets same byte index'''
@@ -189,18 +211,27 @@ class CounterSet(object):
 
         fo.write("#undef PARANOIA\n")
 
-    def emit_c_sumfunc(self, fo):
+    def emit_c_sumfunc(self, fo, g=None):
         '''Emit a function summ up countersets'''
         fo.write("\n")
         fo.write("void\n")
         fo.write("VSC_" + self.name + "_Summ")
+        if g is not None:
+            fo.write("_" + g)
         fo.write("(" + self.struct + " *dst, ")
-        fo.write("const " + self.struct + " *src)\n")
+        fo.write("const " + self.struct)
+        if g is not None:
+            fo.write("_" + g)
+        fo.write(" *src)\n")
         fo.write("{\n")
         fo.write("\n")
         fo.write("\tAN(dst);\n")
         fo.write("\tAN(src);\n")
-        for i in self.mbrs:
+        if g:
+            l = self.groups[g]
+        else:
+            l = self.mbrs
+        for i in l:
             s1 = "\tdst->" + i.arg + " +="
             s2 = "src->" + i.arg + ";"
             if len((s1 + " " + s2).expandtabs()) < 79:
@@ -271,6 +302,8 @@ class CounterSet(object):
         self.emit_c_destroyfunc(fo)
         if 'sumfunction' in self.head.param:
             self.emit_c_sumfunc(fo)
+            for i in self.gnames:
+                self.emit_c_sumfunc(fo, i)
 
 #######################################################################
 
@@ -359,7 +392,7 @@ class RstVscDirective(OurDirective):
             sys.stderr.write("'" + p + "'")
             sys.stderr.write(" on field '" + self.arg + "'\n")
             exit(2)
-        self.param["index"] = vsc_set[-1].addmbr(self)
+        self.param["index"] = vsc_set[-1].addmbr(self, self.param.get("group"))
         if fo:
             fo.write("\n``%s`` ? " % self.arg)
             fo.write("`%s` - " % self.param["type"])

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:21 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:21 +0000 (UTC)
Subject: [6.0] 737d6c2d3 Move all the ban counters into a "ban-mtx" group,
 update them directly in VSC_C_main and protect them with the ban-mtx.
Message-ID: <20181031130821.7E94C954F4@lists.varnish-cache.org>


commit 737d6c2d37be7fefa9e2e4f995b02c5282cea286
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 07:41:49 2018 +0000

    Move all the ban counters into a "ban-mtx" group, update them directly
    in VSC_C_main and protect them with the ban-mtx.

diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc
index 56ad787b2..9ec57517a 100644
--- a/bin/varnishd/VSC_main.vsc
+++ b/bin/varnishd/VSC_main.vsc
@@ -595,6 +595,7 @@
 
 .. varnish_vsc:: bans
 	:type:	gauge
+	:group: ban_mtx
 	:oneliner:	Count of bans
 
 	Number of all bans in system, including bans superseded by newer
@@ -603,6 +604,7 @@
 .. varnish_vsc:: bans_completed
 	:type:	gauge
 	:level:	diag
+	:group: ban_mtx
 	:oneliner:	Number of bans marked 'completed'
 
 	Number of bans which are no longer active, either because they got
@@ -611,6 +613,7 @@
 .. varnish_vsc:: bans_obj
 	:type:	gauge
 	:level:	diag
+	:group: ban_mtx
 	:oneliner:	Number of bans using obj.*
 
 	Number of bans which use obj.* variables.  These bans can possibly
@@ -619,6 +622,7 @@
 .. varnish_vsc:: bans_req
 	:type:	gauge
 	:level:	diag
+	:group: ban_mtx
 	:oneliner:	Number of bans using req.*
 
 	Number of bans which use req.* variables.  These bans can not be
@@ -626,18 +630,21 @@
 
 .. varnish_vsc:: bans_added
 	:level:	diag
+	:group: ban_mtx
 	:oneliner:	Bans added
 
 	Counter of bans added to ban list.
 
 .. varnish_vsc:: bans_deleted
 	:level:	diag
+	:group: ban_mtx
 	:oneliner:	Bans deleted
 
 	Counter of bans deleted from ban list.
 
 .. varnish_vsc:: bans_tested
 	:level:	diag
+	:group: ban_mtx
 	:oneliner:	Bans tested against objects (lookup)
 
 	Count of how many bans and objects have been tested against each
@@ -645,12 +652,14 @@
 
 .. varnish_vsc:: bans_obj_killed
 	:level:	diag
+	:group: ban_mtx
 	:oneliner:	Objects killed by bans (lookup)
 
 	Number of objects killed by bans during object lookup.
 
 .. varnish_vsc:: bans_lurker_tested
 	:level:	diag
+	:group: ban_mtx
 	:oneliner:	Bans tested against objects (lurker)
 
 	Count of how many bans and objects have been tested against each
@@ -658,6 +667,7 @@
 
 .. varnish_vsc:: bans_tests_tested
 	:level:	diag
+	:group: ban_mtx
 	:oneliner:	Ban tests tested against objects (lookup)
 
 	Count of how many tests and objects have been tested against each
@@ -666,6 +676,7 @@
 
 .. varnish_vsc:: bans_lurker_tests_tested
 	:level:	diag
+	:group: ban_mtx
 	:oneliner:	Ban tests tested against objects (lurker)
 
 	Count of how many tests and objects have been tested against each
@@ -674,12 +685,14 @@
 
 .. varnish_vsc:: bans_lurker_obj_killed
 	:level:	diag
+	:group: ban_mtx
 	:oneliner:	Objects killed by bans (lurker)
 
 	Number of objects killed by the ban-lurker.
 
 .. varnish_vsc:: bans_lurker_obj_killed_cutoff
 	:level:	diag
+	:group: ban_mtx
 	:oneliner:	Objects killed by bans for cutoff (lurker)
 
 	Number of objects killed by the ban-lurker to keep the number of
@@ -687,12 +700,14 @@
 
 .. varnish_vsc:: bans_dups
 	:level:	diag
+	:group: ban_mtx
 	:oneliner:	Bans superseded by other bans
 
 	Count of bans replaced by later identical bans.
 
 .. varnish_vsc:: bans_lurker_contention
 	:level:	diag
+	:group: ban_mtx
 	:oneliner:	Lurker gave way for lookup
 
 	Number of times the ban-lurker had to wait for lookups.
@@ -701,6 +716,7 @@
 	:type:	gauge
 	:format:	bytes
 	:level:	diag
+	:group: ban_mtx
 	:oneliner:	Bytes used by the persisted ban lists
 
 	Number of bytes used by the persisted ban lists.
@@ -709,6 +725,7 @@
 	:type:	gauge
 	:format:	bytes
 	:level:	diag
+	:group: ban_mtx
 	:oneliner:	Extra bytes in persisted ban lists due to fragmentation
 
 	Number of extra bytes accumulated through dropped and completed
diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index c69075cb9..aef3f2068 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -578,7 +578,7 @@ struct sess {
 struct ban_proto *BAN_Build(void);
 const char *BAN_AddTest(struct ban_proto *,
     const char *, const char *, const char *);
-const char *BAN_Commit(struct ban_proto *b, struct VSC_main *stats);
+const char *BAN_Commit(struct ban_proto *b);
 void BAN_Abandon(struct ban_proto *b);
 
 /* cache_cli.c [CLI] */
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index b3203a042..5e8712503 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -152,7 +152,7 @@ ban_equal(const uint8_t *bs1, const uint8_t *bs2)
 }
 
 void
-ban_mark_completed(struct ban *b, struct VSC_main *stats)
+ban_mark_completed(struct ban *b)
 {
 	unsigned ln;
 
@@ -166,7 +166,7 @@ ban_mark_completed(struct ban *b, struct VSC_main *stats)
 		b->spec[BANS_FLAGS] |= BANS_FLAG_COMPLETED;
 		VWMB();
 		vbe32enc(b->spec + BANS_LENGTH, BANS_HEAD_LEN);
-		stats->bans_completed++;
+		VSC_C_main->bans_completed++;
 		bans_persisted_fragmentation += ln - ban_len(b->spec);
 		/*
 		 * XXX absolute update of gauges - may be inaccurate for
@@ -367,7 +367,6 @@ ban_reload(const uint8_t *ban, unsigned len)
 	struct ban *b, *b2;
 	int duplicate = 0;
 	double t0, t1, t2 = 9e99;
-	struct VSC_main *stats = VSC_C_main;	// XXX accurate?
 	ASSERT_CLI();
 	Lck_AssertHeld(&ban_mtx);
 
@@ -386,8 +385,8 @@ ban_reload(const uint8_t *ban, unsigned len)
 			duplicate = 1;
 	}
 
-	stats->bans++;
-	stats->bans_added++;
+	VSC_C_main->bans++;
+	VSC_C_main->bans_added++;
 
 	b2 = ban_alloc();
 	AN(b2);
@@ -395,13 +394,13 @@ ban_reload(const uint8_t *ban, unsigned len)
 	AN(b2->spec);
 	memcpy(b2->spec, ban, len);
 	if (ban[BANS_FLAGS] & BANS_FLAG_REQ) {
-		stats->bans_req++;
+		VSC_C_main->bans_req++;
 		b2->flags |= BANS_FLAG_REQ;
 	}
 	if (duplicate)
-		stats->bans_dups++;
+		VSC_C_main->bans_dups++;
 	if (duplicate || (ban[BANS_FLAGS] & BANS_FLAG_COMPLETED))
-		ban_mark_completed(b2, stats);
+		ban_mark_completed(b2);
 	if (b == NULL)
 		VTAILQ_INSERT_TAIL(&ban_head, b2, list);
 	else
@@ -418,8 +417,8 @@ ban_reload(const uint8_t *ban, unsigned len)
 		if (b->flags & BANS_FLAG_COMPLETED)
 			continue;
 		if (ban_equal(b->spec, ban)) {
-			ban_mark_completed(b, stats);
-			stats->bans_dups++;
+			ban_mark_completed(b);
+			VSC_C_main->bans_dups++;
 		}
 	}
 }
@@ -548,7 +547,6 @@ BAN_CheckObject(struct worker *wrk, struct objcore *oc, struct req *req)
 	struct vsl_log *vsl;
 	struct ban *b0, *bn;
 	unsigned tests;
-	struct VSC_main *stats;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
@@ -557,7 +555,6 @@ BAN_CheckObject(struct worker *wrk, struct objcore *oc, struct req *req)
 	assert(oc->refcnt > 0);
 
 	vsl = req->vsl;
-	stats = wrk->stats;
 
 	CHECK_OBJ_NOTNULL(oc->ban, BAN_MAGIC);
 
@@ -600,8 +597,8 @@ BAN_CheckObject(struct worker *wrk, struct objcore *oc, struct req *req)
 
 	Lck_Lock(&ban_mtx);
 	bn->refcount--;
-	stats->bans_tested++;
-	stats->bans_tests_tested += tests;
+	VSC_C_main->bans_tested++;
+	VSC_C_main->bans_tests_tested += tests;
 
 	if (b == bn) {
 		/* not banned */
@@ -612,6 +609,8 @@ BAN_CheckObject(struct worker *wrk, struct objcore *oc, struct req *req)
 		oc->ban = b0;
 		b = NULL;
 	}
+	if (b != NULL)
+		VSC_C_main->bans_obj_killed++;
 
 	if (VTAILQ_LAST(&ban_head, banhead_s)->refcount == 0)
 		ban_kick_lurker();
@@ -624,7 +623,6 @@ BAN_CheckObject(struct worker *wrk, struct objcore *oc, struct req *req)
 		return (0);
 	} else {
 		VSLb(vsl, SLT_ExpBan, "%u banned lookup", ObjGetXID(wrk, oc));
-		stats->bans_obj_killed++;
 		return (1);
 	}
 }
@@ -672,7 +670,7 @@ ccf_ban(struct cli *cli, const char * const *av, void *priv)
 
 	if (err == NULL) {
 		// XXX racy - grab wstat lock?
-		err = BAN_Commit(bp, VSC_C_main);
+		err = BAN_Commit(bp);
 	}
 
 	if (err != NULL) {
@@ -821,9 +819,9 @@ BAN_Init(void)
 	bp = BAN_Build();
 	AN(bp);
 	AZ(pthread_cond_init(&ban_lurker_cond, NULL));
-	AZ(BAN_Commit(bp, VSC_C_main));
+	AZ(BAN_Commit(bp));
 	Lck_Lock(&ban_mtx);
-	ban_mark_completed(VTAILQ_FIRST(&ban_head), VSC_C_main);
+	ban_mark_completed(VTAILQ_FIRST(&ban_head));
 	Lck_Unlock(&ban_mtx);
 }
 
diff --git a/bin/varnishd/cache/cache_ban.h b/bin/varnishd/cache/cache_ban.h
index d534f6e99..62fc767bb 100644
--- a/bin/varnishd/cache/cache_ban.h
+++ b/bin/varnishd/cache/cache_ban.h
@@ -109,7 +109,7 @@ extern pthread_cond_t	ban_lurker_cond;
 extern uint64_t bans_persisted_bytes;
 extern uint64_t bans_persisted_fragmentation;
 
-void ban_mark_completed(struct ban *b, struct VSC_main *stats);
+void ban_mark_completed(struct ban *);
 unsigned ban_len(const uint8_t *banspec);
 void ban_info_new(const uint8_t *ban, unsigned len);
 void ban_info_drop(const uint8_t *ban, unsigned len);
diff --git a/bin/varnishd/cache/cache_ban_build.c b/bin/varnishd/cache/cache_ban_build.c
index 95f82bcc8..1ff460057 100644
--- a/bin/varnishd/cache/cache_ban_build.c
+++ b/bin/varnishd/cache/cache_ban_build.c
@@ -246,7 +246,7 @@ BAN_AddTest(struct ban_proto *bp,
  */
 
 const char *
-BAN_Commit(struct ban_proto *bp, struct VSC_main *stats)
+BAN_Commit(struct ban_proto *bp)
 {
 	struct ban  *b, *bi;
 	ssize_t ln;
@@ -294,8 +294,8 @@ BAN_Commit(struct ban_proto *bp, struct VSC_main *stats)
 	VTAILQ_INSERT_HEAD(&ban_head, b, list);
 	ban_start = b;
 
-	stats->bans++;
-	stats->bans_added++;
+	VSC_C_main->bans++;
+	VSC_C_main->bans_added++;
 	bans_persisted_bytes += ln;
 	/*
 	 * XXX absolute update of gauges - may be inaccurate for Pool_Sumstat
@@ -304,9 +304,9 @@ BAN_Commit(struct ban_proto *bp, struct VSC_main *stats)
 	VSC_C_main->bans_persisted_bytes = bans_persisted_bytes;
 
 	if (b->flags & BANS_FLAG_OBJ)
-		stats->bans_obj++;
+		VSC_C_main->bans_obj++;
 	if (b->flags & BANS_FLAG_REQ)
-		stats->bans_req++;
+		VSC_C_main->bans_req++;
 
 	if (bi != NULL)
 		ban_info_new(b->spec, ln);	/* Notify stevedores */
@@ -317,8 +317,8 @@ BAN_Commit(struct ban_proto *bp, struct VSC_main *stats)
 		    bi = VTAILQ_NEXT(bi, list)) {
 			if (!(bi->flags & BANS_FLAG_COMPLETED) &&
 			    ban_equal(b->spec, bi->spec)) {
-				ban_mark_completed(bi, stats);
-				stats->bans_dups++;
+				ban_mark_completed(bi);
+				VSC_C_main->bans_dups++;
 			}
 		}
 	}
diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c
index 343009dd5..0680941ae 100644
--- a/bin/varnishd/cache/cache_ban_lurker.c
+++ b/bin/varnishd/cache/cache_ban_lurker.c
@@ -62,7 +62,7 @@ ban_kick_lurker(void)
  */
 
 static int
-ban_cleantail(const struct ban *victim, struct VSC_main *stats)
+ban_cleantail(const struct ban *victim)
 {
 	struct ban *b, *bt;
 	struct banhead_s freelist = VTAILQ_HEAD_INITIALIZER(freelist);
@@ -78,13 +78,13 @@ ban_cleantail(const struct ban *victim, struct VSC_main *stats)
 		if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) {
 			assert(VTAILQ_EMPTY(&b->objcore));
 			if (b->flags & BANS_FLAG_COMPLETED)
-				stats->bans_completed--;
+				VSC_C_main->bans_completed--;
 			if (b->flags & BANS_FLAG_OBJ)
-				stats->bans_obj--;
+				VSC_C_main->bans_obj--;
 			if (b->flags & BANS_FLAG_REQ)
-				stats->bans_req--;
-			stats->bans--;
-			stats->bans_deleted++;
+				VSC_C_main->bans_req--;
+			VSC_C_main->bans--;
+			VSC_C_main->bans_deleted++;
 			VTAILQ_REMOVE(&ban_head, b, list);
 			VTAILQ_INSERT_TAIL(&freelist, b, list);
 			bans_persisted_fragmentation +=
@@ -125,7 +125,7 @@ ban_cleantail(const struct ban *victim, struct VSC_main *stats)
  */
 
 static struct objcore *
-ban_lurker_getfirst(struct vsl_log *vsl, struct ban *bt, struct VSC_main *stats)
+ban_lurker_getfirst(struct vsl_log *vsl, struct ban *bt)
 {
 	struct objhead *oh;
 	struct objcore *oc, *noc;
@@ -154,8 +154,8 @@ ban_lurker_getfirst(struct vsl_log *vsl, struct ban *bt, struct VSC_main *stats)
 			assert(move_oc == 0);
 
 			/* hold off to give lookup a chance and reiterate */
+			VSC_C_main->bans_lurker_contention++;
 			Lck_Unlock(&ban_mtx);
-			stats->bans_lurker_contention++;
 			VSL_Flush(vsl, 0);
 			VTIM_sleep(cache_param->ban_lurker_holdoff);
 			Lck_Lock(&ban_mtx);
@@ -209,10 +209,9 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
 	struct objcore *oc;
 	unsigned tests;
 	int i;
-	struct VSC_main *stats;
+	uint64_t tested = 0, tested_tests = 0, lok = 0, lokc = 0;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	stats = wrk->stats;
 
 	/*
 	 * First see if there is anything to do, and if so, insert markers
@@ -232,9 +231,16 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
 			VTIM_sleep(cache_param->ban_lurker_sleep);
 			ban_batch = 0;
 		}
-		oc = ban_lurker_getfirst(vsl, bt, stats);
-		if (oc == NULL)
+		oc = ban_lurker_getfirst(vsl, bt);
+		if (oc == NULL) {
+			Lck_Lock(&ban_mtx);
+			VSC_C_main->bans_lurker_tested += tested;
+			VSC_C_main->bans_lurker_tests_tested += tested_tests;
+			VSC_C_main->bans_lurker_obj_killed += lok;
+			VSC_C_main->bans_lurker_obj_killed_cutoff += lokc;
+			Lck_Unlock(&ban_mtx);
 			return;
+		}
 		i = 0;
 		VTAILQ_FOREACH_REVERSE_SAFE(bl, obans, banhead_s, l_list, bln) {
 			if (oc->ban != bt) {
@@ -256,21 +262,20 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
 				tests = 0;
 				i = ban_evaluate(wrk, bl->spec, oc, NULL,
 				    &tests);
-				stats->bans_lurker_tested++;
-				stats->bans_lurker_tests_tested += tests;
+				tested++;
+				tested_tests += tests;
 			}
 			if (i) {
 				if (kill) {
 					VSLb(vsl, SLT_ExpBan,
 					    "%u killed for lurker cutoff",
 					    ObjGetXID(wrk, oc));
-					stats->
-					    bans_lurker_obj_killed_cutoff++;
+					lokc++;
 				} else {
 					VSLb(vsl, SLT_ExpBan,
 					    "%u banned by lurker",
 					    ObjGetXID(wrk, oc));
-					stats->bans_lurker_obj_killed++;
+					lok++;
 				}
 				HSH_Kill(oc);
 				break;
@@ -278,6 +283,11 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
 		}
 		if (i == 0 && oc->ban == bt) {
 			Lck_Lock(&ban_mtx);
+			VSC_C_main->bans_lurker_tested += tested;
+			VSC_C_main->bans_lurker_tests_tested += tested_tests;
+			VSC_C_main->bans_lurker_obj_killed += lok;
+			VSC_C_main->bans_lurker_obj_killed_cutoff += lokc;
+			tested = tested_tests = lok = lokc = 0;
 			if (oc->ban == bt) {
 				bt->refcount--;
 				VTAILQ_REMOVE(&bt->objcore, oc, ban_list);
@@ -311,14 +321,12 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
 	struct banhead_s obans;
 	double d, dt, n;
 	unsigned count = 0, cutoff = UINT_MAX;
-	struct VSC_main *stats;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	stats = wrk->stats;
 
 	dt = 49.62;		// Random, non-magic
 	if (cache_param->ban_lurker_sleep == 0) {
-		(void)ban_cleantail(NULL, stats);
+		(void)ban_cleantail(NULL);
 		return (dt);
 	}
 	if (cache_param->ban_cutoff > 0)
@@ -358,7 +366,7 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
 	 * containted the first oban, all obans were on the tail and we're
 	 * done.
 	 */
-	if (ban_cleantail(VTAILQ_FIRST(&obans), stats))
+	if (ban_cleantail(VTAILQ_FIRST(&obans)))
 		return (dt);
 
 	if (VTAILQ_FIRST(&obans) == NULL)
@@ -386,7 +394,7 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
 
 	Lck_Lock(&ban_mtx);
 	VTAILQ_FOREACH(b, &obans, l_list) {
-		ban_mark_completed(b, stats);
+		ban_mark_completed(b);
 		if (b == bd)
 			break;
 	}
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 470f691a7..9f838132f 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -559,26 +559,6 @@ VRT_synth_page(VRT_CTX, const char *str, ...)
 
 /*--------------------------------------------------------------------*/
 
-static struct VSC_main *
-vrt_stats(VRT_CTX)
-{
-	struct worker *wrk;
-
-	if (ctx->req) {
-		CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
-		wrk = ctx->req->wrk;
-	} else if (ctx->bo) {
-		CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
-		wrk = ctx->bo->wrk;
-	} else {
-		// XXX
-		return (VSC_C_main);
-	}
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	return (wrk->stats);
-}
-
 VCL_VOID
 VRT_ban_string(VRT_CTX, VCL_STRING str)
 {
@@ -634,7 +614,7 @@ VRT_ban_string(VRT_CTX, VCL_STRING str)
 			break;
 		}
 		if (av[++i] == NULL) {
-			err = BAN_Commit(bp, vrt_stats(ctx));
+			err = BAN_Commit(bp);
 			if (err == NULL)
 				bp = NULL;
 			else

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:21 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:21 +0000 (UTC)
Subject: [6.0] 79101ecc6 Move the wrk+pool summed stats into group 'wrk'.
Message-ID: <20181031130821.A6F8A9550E@lists.varnish-cache.org>


commit 79101ecc60fcb197f5451292d3c69d2894134a44
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 07:46:48 2018 +0000

    Move the wrk+pool summed stats into group 'wrk'.
    
    This eliminates read-add-zero-write on counters outside this group.

diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc
index 9ec57517a..3f42feb1c 100644
--- a/bin/varnishd/VSC_main.vsc
+++ b/bin/varnishd/VSC_main.vsc
@@ -5,10 +5,11 @@
 .. varnish_vsc_begin::	main
 	:oneliner:	Main counters
 	:order:		10
-	:sumfunction:
+	:sumfunction:	wrk_wrk wrk
 
 .. varnish_vsc:: summs
 	:level:	debug
+	:group: wrk
 	:oneliner:	stat summ operations
 
 	Number of times per-thread statistics were summed into the
@@ -21,16 +22,19 @@
 	How long the child process has been running.
 
 .. varnish_vsc:: sess_conn
+	:group: wrk
 	:oneliner:	Sessions accepted
 
 	Count of sessions successfully accepted
 
 .. varnish_vsc:: sess_drop
+	:group: wrk
 	:oneliner:	Sessions dropped
 
 	Count of sessions silently dropped due to lack of worker thread.
 
 .. varnish_vsc:: sess_fail
+	:group: wrk
 	:oneliner:	Session accept failures
 
 	Count of failures to accept TCP connection.
@@ -39,64 +43,75 @@
 	give more detailed information.
 
 .. varnish_vsc:: sess_fail_econnaborted
+	:group: wrk
 	:oneliner:	Session accept failures: connection aborted
 
 	Detailed reason for sess_fail: Connection aborted by the
 	client, usually harmless.
 
 .. varnish_vsc:: sess_fail_eintr
+	:group: wrk
 	:oneliner:	Session accept failures: interrupted system call
 
 	Detailed reason for sess_fail: The accept() call was
 	interrupted, usually harmless
 
 .. varnish_vsc:: sess_fail_emfile
+	:group: wrk
 	:oneliner:	Session accept failures: too many open files
 
 	Detailed reason for sess_fail: No file descriptor was
 	available. Consider raising RLIMIT_NOFILE (see ulimit -n).
 
 .. varnish_vsc:: sess_fail_ebadf
+	:group: wrk
 	:oneliner:	Session accept failures: bad file descriptor
 
 	Detailed reason for sess_fail: The listen socket file
 	descriptor was invalid. Should never happen.
 
 .. varnish_vsc:: sess_fail_enomem
+	:group: wrk
 	:oneliner:	Session accept failures: not enough memory
 
 	Detailed reason for sess_fail: Most likely insufficient
 	socket buffer memory. Should never happen
 
 .. varnish_vsc:: sess_fail_other
+	:group: wrk
 	:oneliner:	Session accept failures: other
 
 	Detailed reason for sess_fail: neither of the above, see
 	Debug log (varnishlog -g raw -I Debug:^Accept).
 
 .. varnish_vsc:: client_req_400
+	:group: wrk
 	:oneliner:	Client requests received, subject to 400 errors
 
 	400 means we couldn't make sense of the request, it was malformed
 	in some drastic way.
 
 .. varnish_vsc:: client_req_417
+	:group: wrk
 	:oneliner:	Client requests received, subject to 417 errors
 
 	417 means that something went wrong with an Expect: header.
 
 .. varnish_vsc:: client_req
+	:group: wrk
 	:oneliner:	Good client requests received
 
 	The count of parseable client requests seen.
 
 .. varnish_vsc:: cache_hit
+	:group: wrk
 	:oneliner:	Cache hits
 
 	Count of cache hits.  A cache hit indicates that an object has been
 	delivered to a client without fetching it from a backend server.
 
 .. varnish_vsc:: cache_hit_grace
+	:group: wrk
 	:oneliner:	Cache grace hits
 
 	Count of cache hits with grace. A cache hit with grace is a cache
@@ -104,6 +119,7 @@
 	included in the cache_hit counter.
 
 .. varnish_vsc:: cache_hitpass
+	:group: wrk
 	:oneliner:	Cache hits for pass.
 
 	Count of hits for pass. A cache hit for pass indicates that Varnish
@@ -112,6 +128,7 @@
 	decision is being used.
 
 .. varnish_vsc:: cache_hitmiss
+	:group: wrk
 	:oneliner:	Cache hits for miss.
 
 	Count of hits for miss. A cache hit for miss indicates that Varnish
@@ -120,6 +137,7 @@
 	cached decision is being used.
 
 .. varnish_vsc:: cache_miss
+	:group: wrk
 	:oneliner:	Cache misses
 
 	Count of misses. A cache miss indicates the object was fetched from
@@ -161,56 +179,67 @@
 
 
 .. varnish_vsc:: fetch_head
+	:group: wrk
 	:oneliner:	Fetch no body (HEAD)
 
 	beresp with no body because the request is HEAD.
 
 .. varnish_vsc:: fetch_length
+	:group: wrk
 	:oneliner:	Fetch with Length
 
 	beresp.body with Content-Length.
 
 .. varnish_vsc:: fetch_chunked
+	:group: wrk
 	:oneliner:	Fetch chunked
 
 	beresp.body with Chunked.
 
 .. varnish_vsc:: fetch_eof
+	:group: wrk
 	:oneliner:	Fetch EOF
 
 	beresp.body with EOF.
 
 .. varnish_vsc:: fetch_bad
+	:group: wrk
 	:oneliner:	Fetch bad T-E
 
 	beresp.body length/fetch could not be determined.
 
 .. varnish_vsc:: fetch_none
+	:group: wrk
 	:oneliner:	Fetch no body
 
 	beresp.body empty
 
 .. varnish_vsc:: fetch_1xx
+	:group: wrk
 	:oneliner:	Fetch no body (1xx)
 
 	beresp with no body because of 1XX response.
 
 .. varnish_vsc:: fetch_204
+	:group: wrk
 	:oneliner:	Fetch no body (204)
 
 	beresp with no body because of 204 response.
 
 .. varnish_vsc:: fetch_304
+	:group: wrk
 	:oneliner:	Fetch no body (304)
 
 	beresp with no body because of 304 response.
 
 .. varnish_vsc:: fetch_failed
+	:group: wrk
 	:oneliner:	Fetch failed (all causes)
 
 	beresp fetch failed.
 
 .. varnish_vsc:: fetch_no_thread
+	:group: wrk
 	:oneliner:	Fetch failed (no thread)
 
 	beresp fetch failed, no thread available.
@@ -259,12 +288,14 @@
 	per second. See also parameter thread_queue_limit.
 
 .. varnish_vsc:: busy_sleep
+	:group: wrk
 	:oneliner:	Number of requests sent to sleep on busy objhdr
 
 	Number of requests sent to sleep without a worker thread because
 	they found a busy object.
 
 .. varnish_vsc:: busy_wakeup
+	:group: wrk
 	:oneliner:	Number of requests woken after sleep on busy objhdr
 
 	Number of requests taken off the busy object sleep list and rescheduled.
@@ -295,6 +326,7 @@
 
 .. varnish_vsc:: n_object
 	:type:	gauge
+	:group: wrk
 	:oneliner:	object structs made
 
 	Approximate number of HTTP objects (headers + body, if present) in
@@ -303,12 +335,14 @@
 .. varnish_vsc:: n_vampireobject
 	:type:	gauge
 	:level:	diag
+	:group: wrk
 	:oneliner:	unresurrected objects
 
 	Number of unresurrected objects
 
 .. varnish_vsc:: n_objectcore
 	:type:	gauge
+	:group: wrk
 	:oneliner:	objectcore structs made
 
 	Approximate number of object metadata elements in the cache. Each
@@ -317,6 +351,7 @@
 
 .. varnish_vsc:: n_objecthead
 	:type:	gauge
+	:group: wrk
 	:oneliner:	objecthead structs made
 
 	Approximate number of different hash entries in the cache.
@@ -358,72 +393,83 @@
 
 
 .. varnish_vsc:: s_sess
+	:group: wrk
 	:oneliner:	Total sessions seen
 
 
 .. varnish_vsc:: s_pipe
+	:group: wrk
 	:oneliner:	Total pipe sessions seen
 
 
 .. varnish_vsc:: s_pass
+	:group: wrk
 	:oneliner:	Total pass-ed requests seen
 
 
 .. varnish_vsc:: s_fetch
+	:group: wrk
 	:oneliner:	Total backend fetches initiated
 
 
 .. varnish_vsc:: s_synth
+	:group: wrk
 	:oneliner:	Total synthetic responses made
 
 
 .. varnish_vsc:: s_req_hdrbytes
-	:oneliner:	Request header bytes
 	:format:	bytes
+	:group:		wrk
+	:oneliner:	Request header bytes
 
 	Total request header bytes received
 
 .. varnish_vsc:: s_req_bodybytes
-	:oneliner:	Request body bytes
 	:format:	bytes
+	:group:		wrk
+	:oneliner:	Request body bytes
 
 	Total request body bytes received
 
 .. varnish_vsc:: s_resp_hdrbytes
-	:oneliner:	Response header bytes
 	:format:	bytes
+	:group:		wrk
+	:oneliner:	Response header bytes
 
 	Total response header bytes transmitted
 
 .. varnish_vsc:: s_resp_bodybytes
-	:oneliner:	Response body bytes
 	:format:	bytes
+	:group:		wrk
+	:oneliner:	Response body bytes
 
 	Total response body bytes transmitted
-	:format:	bytes
 
 .. varnish_vsc:: s_pipe_hdrbytes
-	:oneliner:	Pipe request header bytes
 	:format:	bytes
+	:group:		wrk
+	:oneliner:	Pipe request header bytes
 
 	Total request bytes received for piped sessions
 
 .. varnish_vsc:: s_pipe_in
-	:oneliner:	Piped bytes from client
 	:format:	bytes
+	:group:		wrk
+	:oneliner:	Piped bytes from client
 
 	Total number of bytes forwarded from clients in pipe sessions
 
 .. varnish_vsc:: s_pipe_out
-	:oneliner:	Piped bytes to client
 	:format:	bytes
+	:group:		wrk
+	:oneliner:	Piped bytes to client
 
 	Total number of bytes forwarded to clients in pipe sessions
 
 .. varnish_vsc:: sess_closed
+	:group: wrk
 	:oneliner:	Session Closed
 
-
 .. varnish_vsc:: sess_closed_err
 	:oneliner:	Session Closed with error
 
@@ -431,11 +477,12 @@
 	for detailed breakdown
 
 .. varnish_vsc:: sess_readahead
+	:group: wrk
 	:oneliner:	Session Read Ahead
 
-
 .. varnish_vsc:: sess_herd
 	:level:	diag
+	:group: wrk
 	:oneliner:	Session herd
 
 	Number of times the timeout_linger triggered
@@ -589,6 +636,7 @@
 
 
 .. varnish_vsc:: vcl_fail
+	:group: wrk
 	:oneliner:	VCL failures
 
 	Count of failures which prevented VCL from completing.
@@ -754,6 +802,7 @@
 	Number of objects received by expiry thread for handling.
 
 .. varnish_vsc:: hcb_nolock
+	:group: wrk
 	:level:	debug
 	:oneliner:	HCB Lookups without lock
 
diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index aef3f2068..2f14cefa9 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -86,6 +86,7 @@ enum {
 
 struct VSC_lck;
 struct VSC_main;
+struct VSC_main_wrk;
 struct backend;
 struct ban;
 struct ban_proto;
@@ -246,7 +247,7 @@ struct worker {
 	struct objhead		*nobjhead;
 	struct objcore		*nobjcore;
 	void			*nhashpriv;
-	struct VSC_main		*stats;
+	struct VSC_main_wrk	*stats;
 	struct vsl_log		*vsl;		// borrowed from req/bo
 
 	struct pool_task	task;
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index f2bcc23f7..35be8ad00 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -50,20 +50,12 @@ static VTAILQ_HEAD(,pool)	pools = VTAILQ_HEAD_INITIALIZER(pools);
  * Summing of stats into global stats counters
  */
 
-static void
-pool_sumstat(const struct VSC_main *src)
-{
-
-	Lck_AssertHeld(&wstat_mtx);
-	VSC_main_Summ(VSC_C_main, src);
-}
-
 void
 Pool_Sumstat(const struct worker *wrk)
 {
 
 	Lck_Lock(&wstat_mtx);
-	pool_sumstat(wrk->stats);
+	VSC_main_Summ_wrk(VSC_C_main, wrk->stats);
 	Lck_Unlock(&wstat_mtx);
 	memset(wrk->stats, 0, sizeof *wrk->stats);
 }
@@ -73,7 +65,7 @@ Pool_TrySumstat(const struct worker *wrk)
 {
 	if (Lck_Trylock(&wstat_mtx))
 		return (0);
-	pool_sumstat(wrk->stats);
+	VSC_main_Summ_wrk(VSC_C_main, wrk->stats);
 	Lck_Unlock(&wstat_mtx);
 	memset(wrk->stats, 0, sizeof *wrk->stats);
 	return (1);
@@ -121,14 +113,14 @@ Pool_PurgeStat(unsigned nobj)
 void v_matchproto_(task_func_t)
 pool_stat_summ(struct worker *wrk, void *priv)
 {
-	struct VSC_main *src;
+	struct VSC_main_wrk *src;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(wrk->pool, POOL_MAGIC);
 	AN(priv);
 	src = priv;
 	Lck_Lock(&wstat_mtx);
-	pool_sumstat(src);
+	VSC_main_Summ_wrk(VSC_C_main, src);
 	Lck_Unlock(&wstat_mtx);
 	memset(src, 0, sizeof *src);
 	AZ(wrk->pool->b_stat);
diff --git a/bin/varnishd/cache/cache_pool.h b/bin/varnishd/cache/cache_pool.h
index c954aee8c..e7a6a618b 100644
--- a/bin/varnishd/cache/cache_pool.h
+++ b/bin/varnishd/cache/cache_pool.h
@@ -53,8 +53,8 @@ struct pool {
 	uintmax_t			sdropped;
 	uintmax_t			rdropped;
 	uintmax_t			nqueued;
-	struct VSC_main			*a_stat;
-	struct VSC_main			*b_stat;
+	struct VSC_main_wrk		*a_stat;
+	struct VSC_main_wrk		*b_stat;
 
 	struct mempool			*mpl_req;
 	struct mempool			*mpl_sess;
diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c
index 036056740..991487201 100644
--- a/bin/varnishd/cache/cache_req.c
+++ b/bin/varnishd/cache/cache_req.c
@@ -44,7 +44,7 @@
 #include "vtim.h"
 
 void
-Req_AcctLogCharge(struct VSC_main *ds, struct req *req)
+Req_AcctLogCharge(struct VSC_main_wrk *ds, struct req *req)
 {
 	struct acct_req *a;
 
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index 9d5d78829..011ebafe7 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -318,7 +318,7 @@ void Req_Release(struct req *);
 void Req_Rollback(struct req *req);
 void Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req);
 void Req_Fail(struct req *req, enum sess_close reason);
-void Req_AcctLogCharge(struct VSC_main *, struct req *);
+void Req_AcctLogCharge(struct VSC_main_wrk *, struct req *);
 
 /* cache_req_body.c */
 int VRB_Ignore(struct req *);
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 7cd02533d..be98b4e5e 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -61,7 +61,7 @@ wrk_bgthread(void *arg)
 {
 	struct bgthread *bt;
 	struct worker wrk;
-	struct VSC_main ds;
+	struct VSC_main_wrk ds;
 
 	CAST_OBJ_NOTNULL(bt, arg, BGTHREAD_MAGIC);
 	THR_SetName(bt->name);
@@ -98,7 +98,7 @@ WRK_Thread(struct pool *qp, size_t stacksize, unsigned thread_workspace)
 {
 	// child_signal_handler stack overflow check uses struct worker addr
 	struct worker *w, ww;
-	struct VSC_main ds;
+	struct VSC_main_wrk ds;
 	unsigned char ws[thread_workspace];
 
 	AN(qp);
@@ -133,11 +133,11 @@ WRK_Thread(struct pool *qp, size_t stacksize, unsigned thread_workspace)
  */
 
 static void
-pool_addstat(struct VSC_main *dst, struct VSC_main *src)
+pool_addstat(struct VSC_main_wrk *dst, struct VSC_main_wrk *src)
 {
 
 	dst->summs++;
-	VSC_main_Summ(dst, src);
+	VSC_main_Summ_wrk_wrk(dst, src);
 	memset(src, 0, sizeof *src);
 }
 
diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py
index d4e210deb..eb795e076 100644
--- a/lib/libvcc/vsctool.py
+++ b/lib/libvcc/vsctool.py
@@ -93,6 +93,7 @@ class CounterSet(object):
         self.head = m
         self.completed = False
         self.off = 0
+        self.gnames = None
 
     def addmbr(self, m, g):
         '''Add a counter'''
@@ -168,7 +169,13 @@ class CounterSet(object):
 
         fo.write(self.struct + " {\n")
         for i in self.mbrs:
-            fo.write("\tuint64_t\t%s;\n" % i.arg)
+            s = "\tuint64_t\t%s;" % i.arg
+            g = i.param.get("group")
+            if g is not None:
+                while len(s.expandtabs()) < 64:
+                    s += "\t"
+                s += "/* %s */" % g
+            fo.write(s + "\n")
         fo.write("};\n")
         fo.write("\n")
 
@@ -189,14 +196,19 @@ class CounterSet(object):
         fo.write("void VSC_" + self.name + "_Destroy")
         fo.write("(struct vsc_seg **);\n")
 
-        if 'sumfunction' in self.head.param:
-            fo.write("void VSC_" + self.name + "_Summ")
-            fo.write("(" + self.struct + " *, ")
-            fo.write("const " + self.struct + " *);\n")
-            for i in self.gnames:
-                fo.write("void VSC_" + self.name + "_Summ_" + i)
-                fo.write("(" + self.struct + " *, ")
-                fo.write("const " + self.struct + "_" + i + " *);\n")
+        sf = self.head.param.get('sumfunction')
+        if sf is not None:
+            for i in sf.split():
+                j = i.split("_")
+                assert len(j) <= 2
+                if len(j) == 1:
+                    fo.write("void VSC_" + self.name + "_Summ_" + i)
+                    fo.write("(" + self.struct + " *, ")
+                    fo.write("const " + self.struct + "_" + i + " *);\n")
+                else:
+                    fo.write("void VSC_" + self.name + "_Summ_" + i)
+                    fo.write("(" + self.struct + "_" + j[0] + " *, ")
+                    fo.write("const " + self.struct + "_" + j[1] + " *);\n")
 
     def emit_c_paranoia(self, fo):
         '''Emit asserts to make sure compiler gets same byte index'''
@@ -211,27 +223,23 @@ class CounterSet(object):
 
         fo.write("#undef PARANOIA\n")
 
-    def emit_c_sumfunc(self, fo, g=None):
+    def emit_c_sumfunc(self, fo, tgt):
         '''Emit a function summ up countersets'''
         fo.write("\n")
         fo.write("void\n")
         fo.write("VSC_" + self.name + "_Summ")
-        if g is not None:
-            fo.write("_" + g)
-        fo.write("(" + self.struct + " *dst, ")
-        fo.write("const " + self.struct)
-        if g is not None:
-            fo.write("_" + g)
-        fo.write(" *src)\n")
+        fo.write("_" + tgt[0])
+        if len(tgt) > 1:
+            fo.write("_" + tgt[1])
+            fo.write("(" + self.struct + "_" + tgt[1])
+        else:
+            fo.write("(" + self.struct)
+        fo.write(" *dst, const " + self.struct + "_" + tgt[0] + " *src)\n")
         fo.write("{\n")
         fo.write("\n")
         fo.write("\tAN(dst);\n")
         fo.write("\tAN(src);\n")
-        if g:
-            l = self.groups[g]
-        else:
-            l = self.mbrs
-        for i in l:
+        for i in self.groups[tgt[0]]:
             s1 = "\tdst->" + i.arg + " +="
             s2 = "src->" + i.arg + ";"
             if len((s1 + " " + s2).expandtabs()) < 79:
@@ -300,10 +308,10 @@ class CounterSet(object):
         self.emit_json(fo)
         self.emit_c_newfunc(fo)
         self.emit_c_destroyfunc(fo)
-        if 'sumfunction' in self.head.param:
-            self.emit_c_sumfunc(fo)
-            for i in self.gnames:
-                self.emit_c_sumfunc(fo, i)
+        sf = self.head.param.get('sumfunction')
+        if sf is not None:
+            for i in sf.split():
+                self.emit_c_sumfunc(fo, i.split("_"))
 
 #######################################################################
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:21 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:21 +0000 (UTC)
Subject: [6.0] 86337d321 Implement and test ECMA-48 "REP" sequence.
Message-ID: <20181031130821.D34A995526@lists.varnish-cache.org>


commit 86337d32120cd504198018f7ac2bfba03ad44eaf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 08:31:37 2018 +0000

    Implement and test ECMA-48 "REP" sequence.
    
    Fixes: #2668

diff --git a/bin/varnishtest/gensequences b/bin/varnishtest/gensequences
index e82e56ec7..4337186b8 100644
--- a/bin/varnishtest/gensequences
+++ b/bin/varnishtest/gensequences
@@ -158,6 +158,7 @@ for (p in l_prefix_name) {
 
 	if (l_prefix_name[p] != "teken_state_init") {
 		print "";
+		print "\tt->t_last = 0;";
 		print "\tteken_state_switch(t, teken_state_init);";
 	}
 	print "}";
diff --git a/bin/varnishtest/sequences b/bin/varnishtest/sequences
index d8f30306b..50f589626 100644
--- a/bin/varnishtest/sequences
+++ b/bin/varnishtest/sequences
@@ -114,3 +114,6 @@ C25VTSW		Cons25 switch virtual terminal		^[ [ z		r
 
 # VT52 compatibility
 #DECID		VT52 DECID				^[ Z
+
+# ECMA-48
+REP		Repeat last graphic char		^[ [ b		n
diff --git a/bin/varnishtest/teken.h b/bin/varnishtest/teken.h
index eb59817d7..c08a2bbb8 100644
--- a/bin/varnishtest/teken.h
+++ b/bin/varnishtest/teken.h
@@ -157,6 +157,7 @@ struct __teken {
 
 	unsigned int	 t_utf8_left;
 	teken_char_t	 t_utf8_partial;
+	teken_char_t	 t_last;
 
 	unsigned int	 t_curscs;
 	teken_scs_t	*t_saved_curscs;
diff --git a/bin/varnishtest/teken_subr.h b/bin/varnishtest/teken_subr.h
index 22d06bb19..644e502f6 100644
--- a/bin/varnishtest/teken_subr.h
+++ b/bin/varnishtest/teken_subr.h
@@ -798,10 +798,11 @@ teken_subr_primary_device_attributes(const teken_t *t, unsigned int request)
 }
 
 static void
-teken_subr_do_putchar(const teken_t *t, const teken_pos_t *tp, teken_char_t c,
+teken_subr_do_putchar(teken_t *t, const teken_pos_t *tp, teken_char_t c,
     int width)
 {
 
+	t->t_last = c;
 	if (t->t_stateflags & TS_INSERT &&
 	    tp->tp_col < t->t_winsize.tp_col - width) {
 		teken_rect_t ctr;
@@ -1334,3 +1335,12 @@ teken_subr_vertical_position_absolute(teken_t *t, unsigned int row)
 	t->t_stateflags &= ~TS_WRAPPED;
 	teken_funcs_cursor(t);
 }
+
+static void
+teken_subr_repeat_last_graphic_char(teken_t *t, unsigned int rpts)
+{
+
+	for (; t->t_last != 0 && rpts > 0; rpts--)
+		teken_subr_regular_character(t, t->t_last);
+}
+
diff --git a/bin/varnishtest/tests/a00001.vtc b/bin/varnishtest/tests/a00001.vtc
index ea5b458f3..3647e01e3 100644
--- a/bin/varnishtest/tests/a00001.vtc
+++ b/bin/varnishtest/tests/a00001.vtc
@@ -204,6 +204,27 @@ process p4 -writehex 0d
 process p4 -expect-text 21 11 "Enter choice number (0 - 12):"
 process p4 -screen_dump
 
+# 11. Test non-VT100 (e.g., VT220, XTERM) terminals
+process p4 -writehex "31 31 0d"
+process p4 -expect-text 0 0 "Menu 11: Non-VT100 Tests"
+
+process p4 -writehex "37 0d"
+process p4 -expect-text 0 0 "Menu 11.7: Miscellaneous ISO-6429 (ECMA-48) Tests"
+
+process p4 -writehex "32 0d"
+process p4 -expect-text 0 0 "Push <RETURN>"
+process p4 -screen_dump
+process p4 -expect-text 20 1 "Test Repeat (REP)"
+process p4 -expect-text 1 1 " ++ "
+process p4 -expect-text 2 2 " ++ "
+process p4 -expect-text 17 17 " ++ "
+process p4 -expect-text 18 18 "*++*"
+process p4 -writehex "0d"
+process p4 -expect-text 0 0 "Menu 11.7: Miscellaneous ISO-6429 (ECMA-48) Tests"
+process p4 -writehex "30 0d"
+process p4 -expect-text 0 0 "Menu 11: Non-VT100 Tests"
+process p4 -writehex "30 0d"
+
 # 0. Exit
 process p4 -writehex "30 0d"
 process p4 -expect-text 12 30 "That's all, folks!"

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:21 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:21 +0000 (UTC)
Subject: [6.0] 3ada3052d Be Sun-c compiler compatible
Message-ID: <20181031130822.0044F95535@lists.varnish-cache.org>


commit 3ada3052de5b037f598ffa056ac50b4206abe56a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Sep 24 11:41:42 2018 +0000

    Be Sun-c compiler compatible

diff --git a/bin/varnishtest/teken.c b/bin/varnishtest/teken.c
index 21eb71752..48df7e066 100644
--- a/bin/varnishtest/teken.c
+++ b/bin/varnishtest/teken.c
@@ -41,7 +41,7 @@
 #include "vas.h"
 
 /* debug messages */
-#define	teken_printf(x,...)
+#define	teken_printf(...)
 
 /* Private flags for t_stateflags. */
 #define	TS_FIRSTDIGIT	0x0001	/* First numeric digit in escape sequence. */

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:22 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:22 +0000 (UTC)
Subject: [6.0] 505373add Fix libvgz build error on gcc7
Message-ID: <20181031130822.233F59554C@lists.varnish-cache.org>


commit 505373add3a01e40a256c1223c048e1c0f6f3268
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Mon Sep 24 13:27:36 2018 +0200

    Fix libvgz build error on gcc7
    
    This adds -Wno-implicit-fallthrough for libvgz to satisfy gcc7.
    
    The -Wno-unknown-warning-option is there to satisfy older gcc/clang that
    don't recognize -Wno-implicit-fallthrough.

diff --git a/configure.ac b/configure.ac
index 0c0266aae..00172c7cc 100644
--- a/configure.ac
+++ b/configure.ac
@@ -261,6 +261,11 @@ if test "$ac_cv_have_viz" = no; then
 fi
 CFLAGS="${save_CFLAGS}"
 
+if test "x$GCC" = "xyes"; then
+	libvgz_extra_cflags="${libvgz_extra_cflags} -Wno-unknown-warning-option -Wno-implicit-fallthrough"
+	AC_SUBST(libvgz_extra_cflags)
+fi
+
 SAN_CFLAGS=
 SAN_LDFLAGS=
 UBSAN_CFLAGS=

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:22 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:22 +0000 (UTC)
Subject: [6.0] f32e2bc11 Spelling
Message-ID: <20181031130822.45D8795560@lists.varnish-cache.org>


commit f32e2bc114be4b08135932a3b5937f40d628695e
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Sep 24 23:25:46 2018 +0100

    Spelling

diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c
index 764710f35..404b68da0 100644
--- a/bin/varnishtest/vtc_http2.c
+++ b/bin/varnishtest/vtc_http2.c
@@ -1213,7 +1213,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf)
 /* SECTION: stream.spec.frame_sendhex sendhex
  *
  * Push bytes directly on the wire. sendhex takes exactly one argument: a string
- * describing the bytes, in hex notation, will possible whitespaces between
+ * describing the bytes, in hex notation, with possible whitespaces between
  * them. Here's an example::
  *
  *	sendhex "00 00 08 00 0900	8d"
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index 5d7aa3b8d..ef801f11b 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -300,7 +300,7 @@ vmod_syntax(VRT_CTX, VCL_REAL r)
 	assert(ctx->syntax == 40 || ctx->syntax == 41);
 	/*
 	 * We need to be careful because non-integer numbers have imprecise
-	 * IEE754 represenation (4.1 is 0x1.0666666666666p+2 = 4.09999...)
+	 * IEE754 representation (4.1 is 0x1.0666666666666p+2 = 4.09999...)
 	 * By scaling up and rounding, this is taken care of.
 	 */
 	return (round(r * 10) <= ctx->syntax);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:22 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:22 +0000 (UTC)
Subject: [6.0] 40753112e Correct output
Message-ID: <20181031130822.650269557D@lists.varnish-cache.org>


commit 40753112e6a238c4b0362df356e274b77b442304
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Sep 24 23:27:04 2018 +0100

    Correct output

diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c
index 0c47fe88a..b015f3f8f 100644
--- a/bin/varnishtest/vtc_misc.c
+++ b/bin/varnishtest/vtc_misc.c
@@ -129,9 +129,10 @@ cmd_varnishtest(CMD_ARGS)
  * \-match REGEXP
  *	Expect regexp to match the stdout+err output.
  */
-/* SECTION: client-server.spec.shell shell
+/* SECTION: client-server.spec.shell
  *
- * Same as for the top-level shell.
+ * shell
+ *	Same as for the top-level shell.
  */
 
 static void

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:22 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:22 +0000 (UTC)
Subject: [6.0] 3724b5070 No need to mark the overflow twice
Message-ID: <20181031130822.9108B95598@lists.varnish-cache.org>


commit 3724b50701772ebdf9faaa45e5808605aec21418
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Sep 24 23:27:29 2018 +0100

    No need to mark the overflow twice

diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 8e41d37b1..2e61ca7cf 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -379,7 +379,6 @@ http_CollectHdrSep(struct http *hp, const char *hdr, const char *sep)
 			if (b + x >= e) {
 				http_fail(hp);
 				VSLb(hp->vsl, SLT_LostHeader, "%s", hdr + 1);
-				WS_MarkOverflow(hp->ws);
 				WS_Release(hp->ws, 0);
 				return;
 			}

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:22 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:22 +0000 (UTC)
Subject: [6.0] 5b146b2ef More spelling and consistency
Message-ID: <20181031130822.B2405955AD@lists.varnish-cache.org>


commit 5b146b2effb0e07a51aa985277a62d5ad0102986
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Sep 24 23:32:41 2018 +0100

    More spelling and consistency

diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c
index b015f3f8f..650b659e7 100644
--- a/bin/varnishtest/vtc_misc.c
+++ b/bin/varnishtest/vtc_misc.c
@@ -101,7 +101,7 @@ cmd_varnishtest(CMD_ARGS)
 /* SECTION: shell shell
  *
  * Pass the string given as argument to a shell. If you have multiple
- * commands to run, you can use curly barces to describe a multi-lines
+ * commands to run, you can use curly brackets to describe a multi-lines
  * script, eg::
  *
  *         shell {
diff --git a/doc/sphinx/users-guide/vcl-syntax.rst b/doc/sphinx/users-guide/vcl-syntax.rst
index f4bc219da..4ab5d7f74 100644
--- a/doc/sphinx/users-guide/vcl-syntax.rst
+++ b/doc/sphinx/users-guide/vcl-syntax.rst
@@ -3,7 +3,7 @@ VCL Syntax
 
 VCL has inherited a lot from C and it reads much like simple C or Perl.
 
-Blocks are delimited by curly braces, statements end with semicolons,
+Blocks are delimited by curly brackets, statements end with semicolons,
 and comments may be written as in C, C++ or Perl according to your own
 preferences.
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:22 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:22 +0000 (UTC)
Subject: [6.0] 23414442f Always report the ws id in lowercase
Message-ID: <20181031130822.D88C5955BF@lists.varnish-cache.org>


commit 23414442fd30becaf6db83b59627e55c9ce2ab7f
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Sep 24 23:44:13 2018 +0100

    Always report the ws id in lowercase

diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 2e61ca7cf..30988ab56 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -102,6 +102,7 @@ http_fail(const struct http *hp)
 {
 
 	VSC_C_main->losthdr++;
+	hp->ws->id[0] |= 0x20;		// cheesy tolower()
 	VSLb(hp->vsl, SLT_Error, "out of workspace (%s)", hp->ws->id);
 	WS_MarkOverflow(hp->ws);
 }
diff --git a/bin/varnishtest/tests/r01739.vtc b/bin/varnishtest/tests/r01739.vtc
index 7b8d6565d..843ea6933 100644
--- a/bin/varnishtest/tests/r01739.vtc
+++ b/bin/varnishtest/tests/r01739.vtc
@@ -17,7 +17,7 @@ varnish v1 -vcl+backend {
 
 logexpect l1 -v v1 -g raw {
 	expect * 1002 FetchError {^Workspace overflow}
-	expect * =    Error      {^out of workspace [(]Bo[)]}
+	expect * =    Error      {^out of workspace [(]bo[)]}
 } -start
 
 client c1 {
diff --git a/bin/varnishtest/tests/r01990.vtc b/bin/varnishtest/tests/r01990.vtc
index 5ee40bda5..d86f61c55 100644
--- a/bin/varnishtest/tests/r01990.vtc
+++ b/bin/varnishtest/tests/r01990.vtc
@@ -26,9 +26,9 @@ logexpect l1 -v v1 -g raw {
 	expect * 1002 FetchError   {^out of workspace}
 	expect * =    BerespStatus {^503}
 	expect * =    BerespReason {^Backend fetch failed}
-	expect * =    Error        {^out of workspace [(]Bo[)]}
+	expect * =    Error        {^out of workspace [(]bo[)]}
 	expect * =    LostHeader   {^Date:}
-	expect * =    Error        {^out of workspace [(]Bo[)]}
+	expect * =    Error        {^out of workspace [(]bo[)]}
 	expect * =    LostHeader   {^299}
 } -start
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:23 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:23 +0000 (UTC)
Subject: [6.0] 513ad8968 Document the 'changed' parameter for param.show.
Message-ID: <20181031130823.05172955D2@lists.varnish-cache.org>


commit 513ad8968cec0a0baf41c2e38331617bc66faf64
Author: Geoff Simmons <geoff at uplex.de>
Date:   Tue Sep 25 10:10:39 2018 +0200

    Document the 'changed' parameter for param.show.

diff --git a/include/tbl/cli_cmds.h b/include/tbl/cli_cmds.h
index d162bada5..7b3d1de19 100644
--- a/include/tbl/cli_cmds.h
+++ b/include/tbl/cli_cmds.h
@@ -134,9 +134,14 @@ CLI_CMD(VCL_LABEL,
 
 CLI_CMD(PARAM_SHOW,
 	"param.show",
-	"param.show [-l] [<param>]",
+	"param.show [-l] [<param>|changed]",
 	"Show parameters and their values.",
-	"",
+
+	"The long form with ``-l`` shows additional information, including"
+	" documentation and minimum, maximum and default values, if defined"
+	" for the parameter. If a parameter is specified with ``<param>``,"
+	" show only that parameter. If ``changed`` is specified, show only"
+	" those parameters whose values differ from their defaults.",
 	0, 2
 )
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:23 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:23 +0000 (UTC)
Subject: [6.0] 25cb590f3 Make varnishtest (more) agnostic about its own name.
Message-ID: <20181031130823.285E5955EA@lists.varnish-cache.org>


commit 25cb590f3dc9fc224b20858de9015be8652beea0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 25 09:37:01 2018 +0000

    Make varnishtest (more) agnostic about its own name.

diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc
index 8fb9ded32..3f012b105 100644
--- a/bin/varnishtest/tests/a00000.vtc
+++ b/bin/varnishtest/tests/a00000.vtc
@@ -10,7 +10,7 @@ shell {
 	echo 'shell "exit 9"' >> _.vtc
 }
 
-shell -exit 2 -expect {doesn't start with 'vtest' or 'varnishtest'} {
+shell -exit 2 -expect {doesn't start with 'vtest' or 'varnishtes} {
 	varnishtest -v _.vtc
 }
 
diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index cb0c2f07f..a06f6f08e 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -52,6 +52,8 @@
 #include "vtim.h"
 #include "vct.h"
 
+static const char *argv0;
+
 struct vtc_tst {
 	unsigned		magic;
 #define TST_MAGIC		0x618d8b88
@@ -124,7 +126,7 @@ parse_D_opt(char *arg)
 static void
 usage(void)
 {
-	fprintf(stderr, "usage: varnishtest [options] file ...\n");
+	fprintf(stderr, "usage: %s [options] file ...\n", argv0);
 #define FMT "    %-28s # %s\n"
 	fprintf(stderr, FMT, "-b size",
 	    "Set internal buffer size (default: 1M)");
@@ -577,6 +579,12 @@ main(int argc, char * const *argv)
 	uintmax_t bufsiz;
 	const char *p;
 
+	argv0 = strrchr(argv[0], '/');
+	if (argv0 == NULL)
+		argv0 = argv[0];
+	else
+		argv0++;
+
 	if (getenv("TMPDIR") != NULL)
 		tmppath = strdup(getenv("TMPDIR"));
 	else

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:23 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:23 +0000 (UTC)
Subject: [6.0] 910541954 More 'vtest' vs. 'varnishtest' agnosticism.
Message-ID: <20181031130823.4E12A95603@lists.varnish-cache.org>


commit 9105419546868c9d4e577835fc37eafa4b040ca1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 25 10:05:07 2018 +0000

    More 'vtest' vs. 'varnishtest' agnosticism.

diff --git a/bin/varnishtest/huffman_gen.py b/bin/varnishtest/huffman_gen.py
index b89bbb43e..ab6b30e87 100755
--- a/bin/varnishtest/huffman_gen.py
+++ b/bin/varnishtest/huffman_gen.py
@@ -58,7 +58,7 @@ s = sym(63, 6, 0)
 tbls[0xffffff][63] = s
 
 print('''/* NB:  This file is machine generated, DO NOT EDIT!
- * edit bin/varnishtest/huffman_input instead
+ * edit 'huffman_input' instead
  */
 
 struct stbl;
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 952563b9f..6c4fcc8da 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -76,9 +76,9 @@ extern const struct cmds http_cmds[];
  *
  * Be careful though, servers will by default listen to the 127.0.0.1 IP and
  * will pick a random port, and publish 3 macros: sNAME_addr, sNAME_port and
- * sNAME_sock, but only once they are started. For varnishtest to
- * create the vcl with the correct values, the server must be started when you
- * use -vcl+backend.
+ * sNAME_sock, but only once they are started.
+ * For 'varnish -vcl+backend' to create the vcl with the correct values, the
+ * server must be started first.
  *
  * SECTION: client-server.args Arguments
  *
@@ -1452,7 +1452,7 @@ cmd_http_chunked(CMD_ARGS)
 /* SECTION: client-server.spec.chunkedlen
  *
  * chunkedlen NUMBER
- *         Do as ``chunked`` except that varnishtest will generate the string
+ *         Do as ``chunked`` except that the string will be generated
  *         for you, with a length of NUMBER characters.
  */
 
diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index a06f6f08e..4e5cf6b2b 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -597,7 +597,9 @@ main(int argc, char * const *argv)
 
 	params_vsb = VSB_new_auto();
 	AN(params_vsb);
-	p = getenv("VARNISHTEST_DURATION");
+	p = getenv("VTEST_DURATION");
+	if (p == NULL)
+		p = getenv("VARNISHTEST_DURATION");
 	if (p != NULL)
 		vtc_maxdur = atoi(p);
 
diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c
index 650b659e7..f04003da7 100644
--- a/bin/varnishtest/vtc_misc.c
+++ b/bin/varnishtest/vtc_misc.c
@@ -112,8 +112,8 @@ cmd_varnishtest(CMD_ARGS)
  *
  * By default a zero exit code is expected, otherwise the vtc will fail.
  *
- * Notice that the commandstring is prefixed with "exec 2>&1;" to join
- * stderr and stdout back to the varnishtest process.
+ * Notice that the commandstring is prefixed with "exec 2>&1;" to combine
+ * stderr and stdout back to the test process.
  *
  * Optional arguments:
  *
@@ -352,9 +352,9 @@ cmd_delay(CMD_ARGS)
  * dns
  *        DNS lookups are working
  * topbuild
- *        varnishtest has been started with '-i'
+ *        The test has been started with '-i'
  * root
- *        varnishtest has been invoked by the root user
+ *        The test has been invoked by the root user
  * user_varnish
  *        The varnish user is present
  * user_vcache
diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index be943cfaf..10d545b38 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -856,7 +856,7 @@ process_close(struct process *p)
  *	Shorthand for -start -wait.
  *
  *	In most cases, if you just want to start a process and wait for it
- *	to finish, you can use the varnishtest ``shell`` command instead.
+ *	to finish, you can use the ``shell`` command instead.
  *	The following commands are equivalent::
  *
  *	    shell "do --something"

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:23 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:23 +0000 (UTC)
Subject: [6.0] 223c84fa3 Add missing copyright information
Message-ID: <20181031130823.77C739561C@lists.varnish-cache.org>


commit 223c84fa3f37eaa3c81ccf9a243ff8a5bb71f71a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 25 19:54:50 2018 +0000

    Add missing copyright information

diff --git a/bin/varnishtest/hpack.h b/bin/varnishtest/hpack.h
index 94b734715..a443ee4af 100644
--- a/bin/varnishtest/hpack.h
+++ b/bin/varnishtest/hpack.h
@@ -1,3 +1,31 @@
+/*-
+ * Copyright (c) 2008-2016 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Guillaume Quintard <guillaume.quintard at gmail.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
 #include <stdint.h>
 
 enum hpk_result{
diff --git a/bin/varnishtest/huffman_input b/bin/varnishtest/huffman_input
index bfde068cd..117f87f06 100644
--- a/bin/varnishtest/huffman_input
+++ b/bin/varnishtest/huffman_input
@@ -1,3 +1,4 @@
+# For Copyright information see RFC7541 [BSD3]
        (  0)  |11111111|11000                             1ff8  [13]
        (  1)  |11111111|11111111|1011000                7fffd8  [23]
        (  2)  |11111111|11111111|11111110|0010         fffffe2  [28]
diff --git a/bin/varnishtest/vtc_h2_enctbl.h b/bin/varnishtest/vtc_h2_enctbl.h
index 5e69f3303..186a293d9 100644
--- a/bin/varnishtest/vtc_h2_enctbl.h
+++ b/bin/varnishtest/vtc_h2_enctbl.h
@@ -1,3 +1,7 @@
+/*-
+ * For Copyright information see RFC7541 [BSD3]
+ */
+
 HPACK(0,    0x1ff8,      13)
 HPACK(1,    0x7fffd8,    23)
 HPACK(2,    0xfffffe2,   28)
diff --git a/bin/varnishtest/vtc_h2_stattbl.h b/bin/varnishtest/vtc_h2_stattbl.h
index 1e5486245..e845cf85b 100644
--- a/bin/varnishtest/vtc_h2_stattbl.h
+++ b/bin/varnishtest/vtc_h2_stattbl.h
@@ -1,3 +1,6 @@
+/*-
+ * For Copyright information see RFC7541 [BSD3]
+ */
 STAT_HDRS(1,   ":authority",                   "")
 STAT_HDRS(2,   ":method",                      "GET")
 STAT_HDRS(3,   ":method",                      "POST")
diff --git a/bin/varnishtest/vtc_http.h b/bin/varnishtest/vtc_http.h
index a6c5a95f4..5cd079890 100644
--- a/bin/varnishtest/vtc_http.h
+++ b/bin/varnishtest/vtc_http.h
@@ -1,3 +1,31 @@
+/*-
+ * Copyright (c) 2008-2015 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
 #define MAX_HDR		50
 
 struct http {

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:23 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:23 +0000 (UTC)
Subject: [6.0] 3389483d4 Work around autocrappery screwing up argv[0]
Message-ID: <20181031130823.A2A5595634@lists.varnish-cache.org>


commit 3389483d410f19536b295e497933ad8062696d27
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Sep 25 20:12:44 2018 +0000

    Work around autocrappery screwing up argv[0]

diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc
index 3f012b105..2d3604f24 100644
--- a/bin/varnishtest/tests/a00000.vtc
+++ b/bin/varnishtest/tests/a00000.vtc
@@ -1,6 +1,6 @@
 varnishtest "Test varnishtest itself"
 
-shell -exit 1 -expect {usage: varnishtest} {varnishtest -h}
+shell -exit 1 -expect {varnishtest [options]} {varnishtest -h}
 
 shell -exit 1 -match {-D.*Define macro} {varnishtest -h}
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:23 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:23 +0000 (UTC)
Subject: [6.0] adc8a14e5 remove markers about an issue now solved
Message-ID: <20181031130823.C915D9564D@lists.varnish-cache.org>


commit adc8a14e5d2c08cc7f0640a506e75228fa2444d8
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Sep 26 08:12:38 2018 +0200

    remove markers about an issue now solved

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 5e8712503..6ec3577d2 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -168,10 +168,6 @@ ban_mark_completed(struct ban *b)
 		vbe32enc(b->spec + BANS_LENGTH, BANS_HEAD_LEN);
 		VSC_C_main->bans_completed++;
 		bans_persisted_fragmentation += ln - ban_len(b->spec);
-		/*
-		 * XXX absolute update of gauges - may be inaccurate for
-		 * Pool_Sumstat race
-		 */
 		VSC_C_main->bans_persisted_fragmentation =
 		    bans_persisted_fragmentation;
 	}
@@ -319,10 +315,6 @@ ban_export(void)
 	assert(VSB_len(vsb) == ln);
 	STV_BanExport((const uint8_t *)VSB_data(vsb), VSB_len(vsb));
 	VSB_destroy(&vsb);
-	/*
-	 * XXX absolute update of gauges - may be inaccurate for Pool_Sumstat
-	 * race
-	 */
 	VSC_C_main->bans_persisted_bytes =
 	    bans_persisted_bytes = ln;
 	VSC_C_main->bans_persisted_fragmentation =
@@ -406,10 +398,6 @@ ban_reload(const uint8_t *ban, unsigned len)
 	else
 		VTAILQ_INSERT_BEFORE(b, b2, list);
 	bans_persisted_bytes += len;
-	/*
-	 * XXX absolute update of gauges - may be inaccurate for Pool_Sumstat
-	 * race
-	 */
 	VSC_C_main->bans_persisted_bytes = bans_persisted_bytes;
 
 	/* Hunt down older duplicates */
diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c
index 0680941ae..defe55d91 100644
--- a/bin/varnishd/cache/cache_ban_lurker.c
+++ b/bin/varnishd/cache/cache_ban_lurker.c
@@ -89,10 +89,6 @@ ban_cleantail(const struct ban *victim)
 			VTAILQ_INSERT_TAIL(&freelist, b, list);
 			bans_persisted_fragmentation +=
 			    ban_len(b->spec);
-			/*
-			 * XXX absolute update of gauges - may be inaccurate for
-			 * Pool_Sumstat race
-			 */
 			VSC_C_main->bans_persisted_fragmentation =
 			    bans_persisted_fragmentation;
 			ban_info_drop(b->spec, ban_len(b->spec));

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:23 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:23 +0000 (UTC)
Subject: [6.0] 3164e8d64 avoid holding the ban_mtx unless we need to
Message-ID: <20181031130823.EFD9295672@lists.varnish-cache.org>


commit 3164e8d64b0ccb2ce515b5d276865318c2e6fdb6
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Sep 26 08:21:37 2018 +0200

    avoid holding the ban_mtx unless we need to

diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c
index defe55d91..1729e04a9 100644
--- a/bin/varnishd/cache/cache_ban_lurker.c
+++ b/bin/varnishd/cache/cache_ban_lurker.c
@@ -229,6 +229,11 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
 		}
 		oc = ban_lurker_getfirst(vsl, bt);
 		if (oc == NULL) {
+			if (tested == 0 && lokc == 0) {
+				AZ(tested_tests);
+				AZ(lok);
+				return;
+			}
 			Lck_Lock(&ban_mtx);
 			VSC_C_main->bans_lurker_tested += tested;
 			VSC_C_main->bans_lurker_tests_tested += tested_tests;

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:24 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:24 +0000 (UTC)
Subject: [6.0] cc16b09b7 Move 'delay',
 'shell' and 'barrier' to a new category of "global" commands.
Message-ID: <20181031130824.260CD95689@lists.varnish-cache.org>


commit cc16b09b78a96c3aa2c9a50b775940a3dae6442b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 26 11:10:14 2018 +0000

    Move 'delay', 'shell' and 'barrier' to a new category of "global" commands.

diff --git a/bin/varnishtest/cmds.h b/bin/varnishtest/cmds.h
index d3244c258..fd140efe2 100644
--- a/bin/varnishtest/cmds.h
+++ b/bin/varnishtest/cmds.h
@@ -29,21 +29,29 @@
 
 /*lint -save -e525 -e539 */
 
-CMD(barrier)
-CMD(client)
-CMD(delay)
-CMD(err_shell)
-CMD(feature)
-CMD(haproxy)
-CMD(logexpect)
-CMD(process)
-CMD(server)
-CMD(setenv)
-CMD(shell)
-CMD(syslog)
-CMD(varnish)
-CMD(varnishtest)
-CMD(vtest)
-#undef CMD
+#ifndef CMD_GLOBAL
+  #define CMD_GLOBAL(x)
+#endif
+CMD_GLOBAL(barrier)
+CMD_GLOBAL(delay)
+CMD_GLOBAL(shell)
+#undef CMD_GLOBAL
+
+#ifndef CMD_TOP
+  #define CMD_TOP(x)
+#endif
+CMD_TOP(client)
+CMD_TOP(err_shell)
+CMD_TOP(feature)
+CMD_TOP(haproxy)
+CMD_TOP(logexpect)
+CMD_TOP(process)
+CMD_TOP(server)
+CMD_TOP(setenv)
+CMD_TOP(syslog)
+CMD_TOP(varnish)
+CMD_TOP(varnishtest)
+CMD_TOP(vtest)
+#undef CMD_TOP
 
 /*lint -restore */
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 276566514..2b98596a8 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -290,6 +290,13 @@ macro_expand(struct vtclog *vl, const char *text)
  * Static checkers like Coverity may bitch about this, but we don't care.
  */
 
+static const struct cmds global_cmds[] = {
+#define CMD_GLOBAL(n) { #n, cmd_##n },
+#include "cmds.h"
+	{ NULL, NULL }
+};
+
+
 void
 parse_string(const char *spec, const struct cmds *cmd, void *priv,
     struct vtclog *vl)
@@ -410,6 +417,12 @@ parse_string(const char *spec, const struct cmds *cmd, void *priv,
 			if (!strcmp(token_s[0], cp->name))
 				break;
 
+		if (cp->name == NULL) {
+			for (cp = global_cmds; cp->name != NULL; cp++)
+				if (!strcmp(token_s[0], cp->name))
+					break;
+		}
+
 		if (cp->name == NULL)
 			vtc_fatal(vl, "Unknown command: \"%s\"", token_s[0]);
 
@@ -435,7 +448,7 @@ reset_cmds(const struct cmds *cmd)
  */
 
 static const struct cmds cmds[] = {
-#define CMD(n) { #n, cmd_##n },
+#define CMD_TOP(n) { #n, cmd_##n },
 #include "cmds.h"
 	{ NULL, NULL }
 };
diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index 6ebff08a5..182f258e9 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -69,7 +69,8 @@ void parse_string(const char *spec, const struct cmds *cmd, void *priv,
     struct vtclog *vl);
 int fail_out(void);
 
-#define CMD(n) cmd_f cmd_##n;
+#define CMD_GLOBAL(n) cmd_f cmd_##n;
+#define CMD_TOP(n) cmd_f cmd_##n;
 #include "cmds.h"
 
 extern volatile sig_atomic_t vtc_error; /* Error, bail out */
diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c
index 3f0f69065..7f2f0c1ce 100644
--- a/bin/varnishtest/vtc_barrier.c
+++ b/bin/varnishtest/vtc_barrier.c
@@ -348,8 +348,7 @@ barrier_sync(struct barrier *b, struct vtclog *vl)
 
 /* SECTION: barrier barrier
  *
- * NOTE: this can be used from the top-level as well as from client and server
- * specifications.
+ * NOTE: This command is available everywhere commands are given.
  *
  * Barriers allows you to synchronize different threads to make sure events
  * occur in the right order. It's even possible to use them in VCL.
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 6c4fcc8da..03aa7f7e5 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -1659,17 +1659,6 @@ cmd_http_fatal(CMD_ARGS)
 
 #define cmd_http_non_fatal cmd_http_fatal
 
-/* SECTION: client-server.spec.delay
- *
- * delay
- *	Same as for the top-level delay.
- *
- * SECTION: client-server.spec.barrier
- *
- * barrier
- *	Same as for the top-level barrier
- */
-
 static const char PREFACE[24] = {
 	0x50, 0x52, 0x49, 0x20, 0x2a, 0x20, 0x48, 0x54,
 	0x54, 0x50, 0x2f, 0x32, 0x2e, 0x30, 0x0d, 0x0a,
@@ -1815,7 +1804,6 @@ cmd_http_write_body(CMD_ARGS)
  */
 
 const struct cmds http_cmds[] = {
-#define CMD(n) { #n, cmd_##n },
 #define CMD_HTTP(n) { #n, cmd_http_##n },
 	/* session */
 	CMD_HTTP(accept)
@@ -1864,13 +1852,7 @@ const struct cmds http_cmds[] = {
 	CMD_HTTP(expect)
 	CMD_HTTP(expect_close)
 	CMD_HTTP(expect_pattern)
-
-	/* general purpose */
-	CMD(barrier)
-	CMD(delay)
-	CMD(shell)
 #undef CMD_HTTP
-#undef CMD
 	{ NULL, NULL }
 };
 
diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c
index 404b68da0..ed72ea37a 100644
--- a/bin/varnishtest/vtc_http2.c
+++ b/bin/varnishtest/vtc_http2.c
@@ -2445,7 +2445,6 @@ cmd_write_body(CMD_ARGS)
  * client or a server.
  */
 static const struct cmds stream_cmds[] = {
-#define CMD(n) { #n, cmd_##n },
 #define CMD_STREAM(n) { #n, cmd_##n },
 	/* spec */
 	CMD_STREAM(expect)
@@ -2475,14 +2474,8 @@ static const struct cmds stream_cmds[] = {
 	CMD_STREAM(txsettings)
 	CMD_STREAM(txwinup)
 	CMD_STREAM(write_body)
-
-	/* general purpose */
-	CMD(barrier)
-	CMD(delay)
-	CMD(shell)
 	{ NULL, NULL }
 #undef CMD_STREAM
-#undef CMD
 };
 
 static void *
diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c
index f04003da7..2fcf5e0e2 100644
--- a/bin/varnishtest/vtc_misc.c
+++ b/bin/varnishtest/vtc_misc.c
@@ -99,6 +99,8 @@ cmd_varnishtest(CMD_ARGS)
 }
 
 /* SECTION: shell shell
+ *
+ * NOTE: This command is available everywhere commands are given.
  *
  * Pass the string given as argument to a shell. If you have multiple
  * commands to run, you can use curly brackets to describe a multi-lines
@@ -311,13 +313,12 @@ cmd_setenv(CMD_ARGS)
 }
 
 /* SECTION: delay delay
+ *
+ * NOTE: This command is available everywhere commands are given.
  *
  * Sleep for the number of seconds specified in the argument. The number
  * can include a fractional part, e.g. 1.5.
- */
-/* SECTION: stream.spec.delay delay
  *
- * Same as for the top-level delay.
  */
 void
 cmd_delay(CMD_ARGS)

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:24 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:24 +0000 (UTC)
Subject: [6.0] 0e3471116 Python 3 takes priority over python 2
Message-ID: <20181031130824.44880956A5@lists.varnish-cache.org>


commit 0e3471116ede70e20d01d0ca3694e1990dfe61cb
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Sep 26 07:03:28 2018 -0600

    Python 3 takes priority over python 2
    
    Take 2. Let's see if this time sticks.

diff --git a/varnish.m4 b/varnish.m4
index 8a92d5f10..71df96d9c 100644
--- a/varnish.m4
+++ b/varnish.m4
@@ -129,7 +129,9 @@ AC_DEFUN([_VARNISH_CHECK_DEVEL], [
 # _VARNISH_CHECK_PYTHON
 # ---------------------
 AC_DEFUN([_VARNISH_CHECK_PYTHON], [
-
+	m4_define_default([_AM_PYTHON_INTERPRETER_LIST],
+[python3.9 python3.8 python3.7 python3.6 python3.5 python3.4 python2.7 dnl
+python python2 python3])
 	AM_PATH_PYTHON([2.7], [], [
 		AC_MSG_ERROR([Python >= 2.7 is required.])
 	])

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:24 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:24 +0000 (UTC)
Subject: [6.0] 50c46f386 Cleanup travis' osx job and bump image
Message-ID: <20181031130824.6BBF2956B2@lists.varnish-cache.org>


commit 50c46f3864a0792bb1f6e3835b1b5cf06147fc71
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Sep 26 07:19:26 2018 -0600

    Cleanup travis' osx job and bump image

diff --git a/.travis.yml b/.travis.yml
index 4441126b0..1b0751c8f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,7 +16,7 @@ matrix:
     env: CLANG=6.0 SAN_FLAGS="--enable-asan --enable-ubsan"
     sudo: required
   - os: osx
-    osx_image: xcode9.3
+    osx_image: xcode10
     compiler: clang
   allow_failures:
   - os: osx
@@ -36,7 +36,6 @@ before_install:
   - |
     if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
       brew update
-      brew upgrade python
       brew install docutils sphinx-doc nghttp2
       export PATH="/usr/local/opt/sphinx-doc/bin:$PATH"
     elif [[ -n "$CLANG" ]]; then
@@ -59,9 +58,6 @@ before_install:
   - ./configure ${CONFIGURE_ARGS}
 script:
   - |
-    if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-      export PYTHONPATH=`brew --prefix`/lib/python2.7/site-packages
-    fi
     if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then
       make -j3 distcheck
     else

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:24 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:24 +0000 (UTC)
Subject: [6.0] d40336ee4 Switch to clang 7 in our travis job
Message-ID: <20181031130824.A4930956BF@lists.varnish-cache.org>


commit d40336ee4c8cf003fe23b48ae8c11eff5c71614d
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Sep 26 09:10:54 2018 -0600

    Switch to clang 7 in our travis job

diff --git a/.travis.yml b/.travis.yml
index 1b0751c8f..da2bcbb8b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -13,14 +13,14 @@ matrix:
   - os: linux
     dist: trusty
     compiler: clang
-    env: CLANG=6.0 SAN_FLAGS="--enable-asan --enable-ubsan"
+    env: CLANG=7 SAN_FLAGS="--enable-asan --enable-ubsan"
     sudo: required
   - os: osx
     osx_image: xcode10
     compiler: clang
   allow_failures:
   - os: osx
-  - env: CLANG=6.0 SAN_FLAGS="--enable-asan --enable-ubsan"
+  - env: CLANG=7 SAN_FLAGS="--enable-asan --enable-ubsan"
 addons:
   apt:
     packages:

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:24 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:24 +0000 (UTC)
Subject: [6.0] aea1eafd5 Fix varnish_vsc metrics type
Message-ID: <20181031130824.C6AB1956D8@lists.varnish-cache.org>


commit aea1eafd5b0518ffad4cb94a7b9163a8ec2857d4
Author: Emmanuel Hocdet <manu at gandi.net>
Date:   Wed Sep 26 12:03:07 2018 +0200

    Fix varnish_vsc metrics type

diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc
index 3f42feb1c..88a22bb86 100644
--- a/bin/varnishd/VSC_main.vsc
+++ b/bin/varnishd/VSC_main.vsc
@@ -363,20 +363,17 @@
 	Number of backends known to us.
 
 .. varnish_vsc:: n_expired
-	:type:	gauge
 	:oneliner:	Number of expired objects
 
 	Number of objects that expired from cache because of old age.
 
 .. varnish_vsc:: n_lru_nuked
-	:type:	gauge
 	:oneliner:	Number of LRU nuked objects
 
 	How many objects have been forcefully evicted from storage to make
 	room for a new object.
 
 .. varnish_vsc:: n_lru_moved
-	:type:	gauge
 	:level:	diag
 	:oneliner:	Number of LRU moved objects
 
@@ -780,12 +777,10 @@
 	bans in the persistent ban lists.
 
 .. varnish_vsc:: n_purges
-	:type:	gauge
 	:oneliner:	Number of purge operations executed
 
 
 .. varnish_vsc:: n_obj_purged
-	:type:	gauge
 	:oneliner:	Number of purged objects
 
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:24 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:24 +0000 (UTC)
Subject: [6.0] c58af7edf Dup(2) stderr before calling VCLS on -I argument.
Message-ID: <20181031130824.E7D1E956EB@lists.varnish-cache.org>


commit c58af7edf19f05556415dbe3f849ee0e74684c0b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Sep 26 19:40:05 2018 +0000

    Dup(2) stderr before calling VCLS on -I argument.
    
    Fixes: #2782

diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 7a427d5bf..d03ff2106 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -278,6 +278,8 @@ mgt_launch_child(struct cli *cli)
 	/* Open pipe for mgt->child CLI */
 	AZ(pipe(cp));
 	heritage.cli_in = cp[0];
+	assert(cp[0] > STDERR_FILENO);	// See #2782
+	assert(cp[1] > STDERR_FILENO);
 	MCH_Fd_Inherit(heritage.cli_in, "cli_in");
 	child_cli_out = cp[1];
 
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index aada3ce2c..237fb0f7f 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -861,7 +861,8 @@ main(int argc, char * const *argv)
 
 	if (I_fd >= 0) {
 		fprintf(stderr, "BEGIN of -I file processing\n");
-		mgt_cli_setup(I_fd, 2, 1, "-I file", mgt_I_close, stderr);
+		/* We must dup stderr, because VCLS closes the output fd */
+		mgt_cli_setup(I_fd, dup(2), 1, "-I file", mgt_I_close, stderr);
 		while (I_fd >= 0) {
 			o = VEV_Once(mgt_evb);
 			if (o != 1)

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:25 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:25 +0000 (UTC)
Subject: [6.0] 5d8d2a149 Convert this test to UTF-8
Message-ID: <20181031130825.1B8F1956FC@lists.varnish-cache.org>


commit 5d8d2a1498eb3779d808dc9310da473a5d954bd6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Sep 27 05:58:35 2018 +0000

    Convert this test to UTF-8

diff --git a/bin/varnishtest/tests/r00545.vtc b/bin/varnishtest/tests/r00545.vtc
index 0888af11a..e293c926b 100644
--- a/bin/varnishtest/tests/r00545.vtc
+++ b/bin/varnishtest/tests/r00545.vtc
@@ -7,12 +7,12 @@ server s1 {
 
 varnish v1 -vcl+backend {
 	sub vcl_deliver {
-		set resp.http.foo = "???";
+		set resp.http.foo = "??????";
 	}
 } -start
 
 client c1 {
 	txreq
 	rxresp
-	expect resp.http.foo == "???"
+	expect resp.http.foo == "??????"
 } -run

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:25 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:25 +0000 (UTC)
Subject: [6.0] ae85f72ba explain a relevant detail of the worker thread
 signaling
Message-ID: <20181031130825.4090795706@lists.varnish-cache.org>


commit ae85f72ba6fb089db3bd835040b963ed479ea11c
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Sep 27 10:27:44 2018 +0200

    explain a relevant detail of the worker thread signaling
    
    Over time, I have repeatedly stared at this code again and again
    wondering if (and why) our cv signaling is correct, just to end up
    with the same insight each time (but first overlooking #2719)
    
    Being fully aware that we do not want to plaster our code with
    outdated comments, I hope this explanation is warranted to save myself
    (and others, hopefully) from wasting precious life time on reiterating
    over the same question.

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index be98b4e5e..5b88a8338 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -27,6 +27,27 @@
  * SUCH DAMAGE.
  *
  * Worker thread stuff unrelated to the worker thread pools.
+ *
+ * --
+ * signaling_note:
+ *
+ * note on worker wakeup signaling through the wrk condition variable (cv)
+ *
+ * In the general case, a cv needs to be signaled while holding the
+ * corresponding mutex, otherwise the signal may be posted before the waiting
+ * thread could register itself on the cv and, consequently, the signal may be
+ * missed.
+ *
+ * In our case, any worker thread which we wake up comes from the idle queue,
+ * where it put itself under the mutex, releasing that mutex implicitly via
+ * Lck_CondWait() (which calls some variant of pthread_cond_wait). So we avoid
+ * additional mutex contention knowing that any worker thread on the idle queue
+ * is blocking on the cv.
+ *
+ * Except -- when it isn't, because it woke up for releasing its VCL
+ * Reference. To account for this case, we check if the task function has been
+ * set in the meantime, which in turn requires all of the task preparation to be
+ * done holding the pool mutex. (see also #2719)
  */
 
 #include "config.h"
@@ -220,6 +241,7 @@ Pool_Task_Arg(struct worker *wrk, enum task_prio prio, task_func_t *func,
 	wrk2->task.func = func;
 	wrk2->task.priv = wrk2->aws->f;
 	Lck_Unlock(&pp->mtx);
+	// see signaling_note at the top for explanation
 	if (retval)
 		AZ(pthread_cond_signal(&wrk2->cond));
 	return (retval);
@@ -252,6 +274,7 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio prio)
 		wrk->task.func = task->func;
 		wrk->task.priv = task->priv;
 		Lck_Unlock(&pp->mtx);
+		// see signaling_note at the top for explanation
 		AZ(pthread_cond_signal(&wrk->cond));
 		return (0);
 	}
@@ -346,6 +369,7 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk)
 			VTAILQ_INSERT_HEAD(&pp->idle_queue, &wrk->task, list);
 			pp->nidle++;
 			do {
+				// see signaling_note at the top for explanation
 				i = Lck_CondWait(&wrk->cond, &pp->mtx,
 				    wrk->vcl == NULL ?  0 : wrk->lastused+60.);
 				if (i == ETIMEDOUT)

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:25 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:25 +0000 (UTC)
Subject: [6.0] cc70ce50e shrink the critical section under the pool mutex by a
 bit
Message-ID: <20181031130825.614329570F@lists.varnish-cache.org>


commit cc70ce50e3a2eaa4a26f001235875a342f667577
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Sep 27 11:17:27 2018 +0200

    shrink the critical section under the pool mutex by a bit
    
    Checking and preparing our worker struct does not need to happen
    under the lock.

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 5b88a8338..589ce68cb 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -326,13 +326,12 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk)
 	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
 	wrk->pool = pp;
 	while (1) {
-		Lck_Lock(&pp->mtx);
-
 		CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 
 		WS_Reset(wrk->aws, 0);
 		AZ(wrk->vsl);
 
+		Lck_Lock(&pp->mtx);
 		if (pp->nidle < pool_reserve())
 			prio_lim = TASK_QUEUE_RESERVE + 1;
 		else

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:25 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:25 +0000 (UTC)
Subject: [6.0] a23939499 VSC_Arg succeeds with non-zero
Message-ID: <20181031130825.854F89571C@lists.varnish-cache.org>


commit a239394990c7b72498508f9c0359f716d7834585
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Sep 28 15:37:40 2018 +0200

    VSC_Arg succeeds with non-zero
    
    Fixes #2787

diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index 74a8f5adf..be9e64c79 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -319,10 +319,10 @@ main(int argc, char * const *argv)
 
 	if (curses) {
 		if (has_f) {
-			AZ(VSC_Arg(vsc, 'f', "MGT.uptime"));
-			AZ(VSC_Arg(vsc, 'f', "MAIN.uptime"));
-			AZ(VSC_Arg(vsc, 'f', "MAIN.cache_hit"));
-			AZ(VSC_Arg(vsc, 'f', "MAIN.cache_miss"));
+			AN(VSC_Arg(vsc, 'f', "MGT.uptime"));
+			AN(VSC_Arg(vsc, 'f', "MAIN.uptime"));
+			AN(VSC_Arg(vsc, 'f', "MAIN.cache_hit"));
+			AN(VSC_Arg(vsc, 'f', "MAIN.cache_miss"));
 		}
 		do_curses(vd, vsc, 1.0);
 	}

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:25 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:25 +0000 (UTC)
Subject: [6.0] c07621c94 Only do the "feature dns" DNS-lookup if/when
 necessary.
Message-ID: <20181031130825.AD8D295736@lists.varnish-cache.org>


commit c07621c94f3985c4dafd9e34b9cc519f719abcc3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 1 09:54:57 2018 +0000

    Only do the "feature dns" DNS-lookup if/when necessary.
    
    Spotted by:     Willy Tarreau <w at 1wt.eu>

diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index 182f258e9..c091afe40 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -82,7 +82,6 @@ extern char *vmod_path;
 extern struct vsb *params_vsb;
 extern int leave_temp;
 extern int vtc_witness;
-extern int feature_dns;
 extern int ign_unknown_macro;
 
 void init_server(void);
diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index 4e5cf6b2b..01ae08ba7 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -97,7 +97,6 @@ char *vmod_path = NULL;
 struct vsb *params_vsb = NULL;
 int leave_temp;
 int vtc_witness = 0;
-int feature_dns;
 
 /**********************************************************************
  * Parse a -D option argument into a name/val pair, and insert
@@ -430,39 +429,6 @@ i_mode(void)
 	AZ(putenv(strdup("MALLOC_CONF=abort:true,junk:true")));
 }
 
-/**********************************************************************
- * Most test-cases use only numeric IP#'s but a few requires non-demented
- * DNS services.  This is a basic sanity check for those.
- */
-
-static int v_matchproto_(vss_resolved_f)
-dns_cb(void *priv, const struct suckaddr *sa)
-{
-	char abuf[VTCP_ADDRBUFSIZE];
-	char pbuf[VTCP_PORTBUFSIZE];
-	int *ret = priv;
-
-	VTCP_name(sa, abuf, sizeof abuf, pbuf, sizeof pbuf);
-	if (strcmp(abuf, "192.0.2.255")) {
-		fprintf(stderr, "DNS-test: Wrong response: %s\n", abuf);
-		*ret = -1;
-	} else if (*ret == 0)
-		*ret = 1;
-	return (0);
-}
-
-static int
-dns_works(void)
-{
-	int ret = 0, error;
-	const char *msg;
-
-	error = VSS_resolver("dns-canary.freebsd.dk", NULL, dns_cb, &ret, &msg);
-	if (error || msg != NULL || ret != 1)
-		return (0);
-	return (1);
-}
-
 /**********************************************************************
  * Figure out what IP related magic
  */
@@ -682,7 +648,6 @@ main(int argc, char * const *argv)
 
 	AZ(VSB_finish(params_vsb));
 
-	feature_dns = dns_works();
 	ip_magic();
 
 	if (iflg)
diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c
index 2fcf5e0e2..b55d4be13 100644
--- a/bin/varnishtest/vtc_misc.c
+++ b/bin/varnishtest/vtc_misc.c
@@ -47,6 +47,8 @@
 
 #include "vnum.h"
 #include "vre.h"
+#include "vtcp.h"
+#include "vss.h"
 #include "vtim.h"
 
 /* SECTION: vtest vtest
@@ -338,6 +340,39 @@ cmd_delay(CMD_ARGS)
 	VTIM_sleep(f);
 }
 
+/**********************************************************************
+ * Most test-cases use only numeric IP#'s but a few requires non-demented
+ * DNS services.  This is a basic sanity check for those.
+ */
+
+static int v_matchproto_(vss_resolved_f)
+dns_cb(void *priv, const struct suckaddr *sa)
+{
+	char abuf[VTCP_ADDRBUFSIZE];
+	char pbuf[VTCP_PORTBUFSIZE];
+	int *ret = priv;
+
+	VTCP_name(sa, abuf, sizeof abuf, pbuf, sizeof pbuf);
+	if (strcmp(abuf, "192.0.2.255")) {
+		fprintf(stderr, "DNS-test: Wrong response: %s\n", abuf);
+		*ret = -1;
+	} else if (*ret == 0)
+		*ret = 1;
+	return (0);
+}
+
+static int
+dns_works(void)
+{
+	int ret = 0, error;
+	const char *msg;
+
+	error = VSS_resolver("dns-canary.freebsd.dk", NULL, dns_cb, &ret, &msg);
+	if (error || msg != NULL || ret != 1)
+		return (0);
+	return (1);
+}
+
 /* SECTION: feature feature
  *
  * Test that the required feature(s) for a test are available, and skip
@@ -424,7 +459,7 @@ cmd_feature(CMD_ARGS)
 		}
 		FEATURE("pcre_jit", VRE_has_jit);
 		FEATURE("64bit", sizeof(void*) == 8);
-		FEATURE("dns", feature_dns);
+		FEATURE("dns", dns_works());
 		FEATURE("topbuild", iflg);
 		FEATURE("root", !geteuid());
 		FEATURE("user_varnish", getpwnam("varnish") != NULL);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:25 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:25 +0000 (UTC)
Subject: [6.0] a2357876d Return a canned minimal 500 if we run out of ws in
 H2-deliver.
Message-ID: <20181031130825.D50C295742@lists.varnish-cache.org>


commit a2357876d4e361386569ef5ac755573632a0e324
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 3 07:50:29 2018 +0000

    Return a canned minimal 500 if we run out of ws in H2-deliver.
    
    (Same as we do in H1)
    
    Fixes #2589

diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c
index 912eb88bc..8f4b56e1d 100644
--- a/bin/varnishd/http2/cache_http2_deliver.c
+++ b/bin/varnishd/http2/cache_http2_deliver.c
@@ -167,17 +167,35 @@ h2_enc_len(struct vsb *vsb, unsigned bits, unsigned val, uint8_t b0)
 	unsigned mask = (1U << bits) - 1U;
 
 	if (val >= mask) {
-		AZ(VSB_putc(vsb, b0 | (uint8_t)mask));
+		VSB_putc(vsb, b0 | (uint8_t)mask);
 		val -= mask;
 		while (val >= 128) {
-			AZ(VSB_putc(vsb, 0x80 | ((uint8_t)val & 0x7f)));
+			VSB_putc(vsb, 0x80 | ((uint8_t)val & 0x7f));
 			val >>= 7;
 		}
 	}
-	AZ(VSB_putc(vsb, (uint8_t)val));
+	VSB_putc(vsb, (uint8_t)val);
 	return (0);
 }
 
+/*
+ * Hand-crafted-H2-HEADERS-R-Us:
+ *
+ * This is a handbuilt HEADERS frame for when we run out of workspace
+ * during delivery.
+ */
+
+static const uint8_t h2_500_resp[] = {
+	// :status 500
+	0x8e,
+
+	// content-length 0
+	0x1f, 0x0d, 0x01, 0x30,
+
+	// server Varnish
+	0x1f, 0x27, 0x07, 'V', 'a', 'r', 'n', 'i', 's', 'h',
+};
+
 void v_matchproto_(vtr_deliver_f)
 h2_deliver(struct req *req, struct boc *boc, int sendbody)
 {
@@ -203,10 +221,10 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 	AN(VSB_new(&resp, req->ws->f, l, VSB_FIXEDLEN));
 
 	l = h2_status(buf, req->resp->status);
-	AZ(VSB_bcat(&resp, buf, l));
+	VSB_bcat(&resp, buf, l);
 
 	hp = req->resp;
-	for (u = HTTP_HDR_FIRST; u < hp->nhd; u++) {
+	for (u = HTTP_HDR_FIRST; u < hp->nhd && !VSB_error(&resp); u++) {
 		r = strchr(hp->hd[u].b, ':');
 		AN(r);
 
@@ -227,24 +245,32 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 			VSLb(req->vsl, SLT_Debug,
 			    "HP {%d, \"%s\", \"%s\"} <%s>",
 			    hps->idx, hps->name, hps->val, hp->hd[u].b);
-			AZ(h2_enc_len(&resp, 4, hps->idx, 0x10));
+			h2_enc_len(&resp, 4, hps->idx, 0x10);
 		} else {
-			AZ(VSB_putc(&resp, 0x10));
+			VSB_putc(&resp, 0x10);
 			sz--;
-			AZ(h2_enc_len(&resp, 7, sz, 0));
+			h2_enc_len(&resp, 7, sz, 0);
 			for (sz1 = 0; sz1 < sz; sz1++)
-				AZ(VSB_putc(&resp, tolower(hp->hd[u].b[sz1])));
+				VSB_putc(&resp, tolower(hp->hd[u].b[sz1]));
 
 		}
 
 		while (vct_islws(*++r))
 			continue;
 		sz = hp->hd[u].e - r;
-		AZ(h2_enc_len(&resp, 7, sz, 0));
-		AZ(VSB_bcat(&resp, r, sz));
+		h2_enc_len(&resp, 7, sz, 0);
+		VSB_bcat(&resp, r, sz);
+	}
+	if (VSB_finish(&resp)) {
+		// We ran out of workspace, return minimal 500
+		// XXX: VSC counter ?
+		r = (const char*)h2_500_resp;
+		sz = sizeof h2_500_resp;
+		sendbody = 0;
+	} else {
+		sz = VSB_len(&resp);
+		r = req->ws->f;
 	}
-	AZ(VSB_finish(&resp));
-	sz = VSB_len(&resp);
 
 	AZ(req->wrk->v1l);
 
@@ -256,7 +282,7 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 	H2_Send_Get(req->wrk, r2->h2sess, r2);
 	H2_Send(req->wrk, r2, H2_F_HEADERS,
 	    (sendbody ? 0 : H2FF_HEADERS_END_STREAM) | H2FF_HEADERS_END_HEADERS,
-	    sz, req->ws->f);
+	    sz, r);
 	H2_Send_Rel(r2->h2sess, r2);
 	req->acct.resp_hdrbytes += sz;
 
diff --git a/bin/varnishtest/tests/r02589.vtc b/bin/varnishtest/tests/r02589.vtc
new file mode 100644
index 000000000..418cc6833
--- /dev/null
+++ b/bin/varnishtest/tests/r02589.vtc
@@ -0,0 +1,29 @@
+varnishtest "workspace overrun on h/2 delivery"
+
+server s1 {
+	rxreq
+	txresp -hdrlen Foo 500
+} -start
+
+varnish v1 -vcl+backend  {
+	import vtc;
+	sub vcl_deliver {
+		vtc.workspace_alloc(client, -50);
+	}
+} -start
+
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.set debug +syncvsl"
+varnish v1 -cliok "param.set vsl_mask +H2TxHdr"
+varnish v1 -cliok "param.set vsl_mask +H2TxBody"
+
+
+client c1 {
+	stream 1 {
+		txreq
+		rxresp
+		expect resp.status == 500
+		expect resp.http.content-length == 0
+		expect resp.http.server == "Varnish"
+	} -run
+} -run

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:26 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:26 +0000 (UTC)
Subject: [6.0] 4db967e77 Add a watchdog to worker pools.
Message-ID: <20181031130826.0A2CD95750@lists.varnish-cache.org>


commit 4db967e77847dbeac0acd67dea65518ec9230ea4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 3 10:12:59 2018 +0000

    Add a watchdog to worker pools.
    
    If the worker pool is configured too small, it can deadlock.
    
    Recovering from this would require a lot of complicated code, to
    discover queued but unscheduled tasks which can be cancelled
    (because the client went away or otherwise) and code to do the
    cancelling etc. etc.
    
    But fundamentally either people configured their pools wrong, in
    which case we want them to notice, or they are under DoS, in which
    case recovering gracefully is unlikely be a major improvement over
    a restart.
    
    Instead we implement a per-pool watchdog and kill the child process
    if nothing has been dequeued for too long.
    
    Default value 10 seconds, open to discussion.
    
    Band-aid for:   #2418
    
    Test-case by:   @Dridi

diff --git a/bin/varnishd/cache/cache_pool.h b/bin/varnishd/cache/cache_pool.h
index e7a6a618b..9902e32ad 100644
--- a/bin/varnishd/cache/cache_pool.h
+++ b/bin/varnishd/cache/cache_pool.h
@@ -53,6 +53,7 @@ struct pool {
 	uintmax_t			sdropped;
 	uintmax_t			rdropped;
 	uintmax_t			nqueued;
+	uintmax_t			ndequeued;
 	struct VSC_main_wrk		*a_stat;
 	struct VSC_main_wrk		*b_stat;
 
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 589ce68cb..f7f95cfc5 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -341,6 +341,7 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk)
 			tp = VTAILQ_FIRST(&pp->queues[i]);
 			if (tp != NULL) {
 				pp->lqueue--;
+				pp->ndequeued--;
 				VTAILQ_REMOVE(&pp->queues[i], tp, list);
 				break;
 			}
@@ -487,6 +488,8 @@ pool_herder(void *priv)
 	struct worker *wrk;
 	double delay;
 	int wthread_min;
+	uintmax_t dq = (1ULL << 31);
+	double dqt;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 
@@ -494,6 +497,29 @@ pool_herder(void *priv)
 	THR_Init();
 
 	while (!pp->die || pp->nthr > 0) {
+		/*
+		 * If the worker pool is configured too small, we can
+		 * end up deadlocking it (see #2418 for details).
+		 *
+		 * Recovering from this would require a lot of complicated
+		 * code, and fundamentally, either people configured their
+		 * pools wrong, in which case we want them to notice, or
+		 * they are under DoS, in which case recovering gracefully
+		 * is unlikely be a major improvement.
+		 *
+		 * Instead we implement a watchdog and kill the worker if
+		 * nothing has been dequeued for that long.
+		 */
+		if (dq != pp->ndequeued) {
+			dq = pp->ndequeued;
+			dqt = VTIM_real();
+		} else if (pp->lqueue &&
+		    VTIM_real() - dqt > cache_param->wthread_watchdog) {
+			VSL(SLT_Error, 0,
+			   "Pool Herder: Queue does not move ql=%u dt=%f",
+			   pp->lqueue, VTIM_real() - dqt);
+			WRONG("Worker Pool Queue does not move");
+		}
 		wthread_min = cache_param->wthread_min;
 		if (pp->die)
 			wthread_min = 0;
diff --git a/bin/varnishd/common/common_param.h b/bin/varnishd/common/common_param.h
index 2366527b1..13ce0cb18 100644
--- a/bin/varnishd/common/common_param.h
+++ b/bin/varnishd/common/common_param.h
@@ -105,6 +105,7 @@ struct params {
 	double			wthread_add_delay;
 	double			wthread_fail_delay;
 	double			wthread_destroy_delay;
+	double			wthread_watchdog;
 	unsigned		wthread_stats_rate;
 	ssize_t			wthread_stacksize;
 	unsigned		wthread_queue_limit;
diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c
index 14bceac9e..25b6efba0 100644
--- a/bin/varnishd/mgt/mgt_pool.c
+++ b/bin/varnishd/mgt/mgt_pool.c
@@ -148,6 +148,15 @@ struct parspec WRK_parspec[] = {
 		"for at least this long, will be destroyed.",
 		EXPERIMENTAL | DELAYED_EFFECT,
 		"300", "seconds" },
+	{ "thread_pool_watchdog",
+		tweak_timeout, &mgt_param.wthread_watchdog,
+		"0.1", NULL,
+		"Thread queue stuck watchdog.\n"
+		"\n"
+		"If no queued work have been released for this long,"
+		" the worker process panics itself.",
+		EXPERIMENTAL,
+		"10", "seconds" },
 	{ "thread_pool_destroy_delay",
 		tweak_timeout, &mgt_param.wthread_destroy_delay,
 		"0.01", NULL,
diff --git a/bin/varnishtest/tests/r02418.vtc b/bin/varnishtest/tests/r02418.vtc
new file mode 100644
index 000000000..a03ca546c
--- /dev/null
+++ b/bin/varnishtest/tests/r02418.vtc
@@ -0,0 +1,72 @@
+varnishtest "h2 queuing deadlock"
+
+barrier b1 cond 2
+
+# A reserve of 1 thread in a pool of 3 leaves a maximum
+# of 2 running sessions, the streams will be queued (except
+# stream 0 that is part of the h2 session).
+
+varnish v1 -cliok "param.set thread_pools 1"
+varnish v1 -cliok "param.set thread_pool_min 3"
+varnish v1 -cliok "param.set thread_pool_max 3"
+varnish v1 -cliok "param.set thread_pool_reserve 1"
+varnish v1 -cliok "param.set thread_pool_watchdog 5"
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.set feature +no_coredump"
+
+varnish v1 -vcl {
+	backend b1 { .host = "${bad_ip}"; }
+	sub vcl_recv {
+		return (synth(200));
+	}
+} -start
+
+logexpect l1 -v v1 -g raw {
+	expect * *	Error	"Pool Herder: Queue does not move*"
+} -start
+
+# Starve the pool with h2 sessions
+
+client c1 {
+	txpri
+	stream 0 rxsettings -run
+
+	barrier b1 sync
+
+	stream 1 {
+		txreq
+		# can't be scheduled, don't rx
+	} -run
+} -start
+
+client c2 {
+	txpri
+	stream 0 rxsettings -run
+
+	barrier b1 sync
+
+	stream 1 {
+		txreq
+		# can't be scheduled, don't rx
+	} -run
+} -start
+
+client c1 -wait
+client c2 -wait
+
+varnish v1 -vsl_catchup
+
+# At this point c1 and c2 closed their connections
+
+client c3 {
+	txreq
+	delay 10
+} -run
+
+logexpect l1 -wait
+
+varnish v1 -cliok panic.show
+varnish v1 -cliok panic.clear
+
+varnish v1 -expectexit 0x20
+
diff --git a/include/tbl/params.h b/include/tbl/params.h
index 054b2f449..42083aa5e 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -1195,6 +1195,24 @@ PARAM(
 	/* func */	NULL
 )
 
+/* actual location mgt_pool.c */
+PARAM(
+	/* name */	thread_pool_watchdog,
+	/* typ */	timeout,
+	/* min */	"0.1",
+	/* max */	NULL,
+	/* default */	"10.000",
+	/* units */	"seconds",
+	/* flags */	EXPERIMENTAL,
+	/* s-text */
+	"Thread queue stuck watchdog.\n"
+	"\n"
+	"If no queued work have been released for this long,"
+	" the worker process panics itself.",
+	/* l-text */	"",
+	/* func */	NULL
+)
+
 /* actual location mgt_pool.c */
 PARAM(
 	/* name */	thread_pool_destroy_delay,

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:26 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:26 +0000 (UTC)
Subject: [6.0] 2c9f3d157 It's nice to know I'm still smarter than gcc
Message-ID: <20181031130826.2AB5195762@lists.varnish-cache.org>


commit 2c9f3d157ec1ba8de473d60b4f08b9d7dfbda004
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Oct 3 13:34:33 2018 +0000

    It's nice to know I'm still smarter than gcc

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index f7f95cfc5..3d4411cb4 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -489,7 +489,7 @@ pool_herder(void *priv)
 	double delay;
 	int wthread_min;
 	uintmax_t dq = (1ULL << 31);
-	double dqt;
+	double dqt = 0;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:26 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:26 +0000 (UTC)
Subject: [6.0] f465e905e Don't poll VSM_Status() while there is work to do and
 no interruptions.
Message-ID: <20181031130826.5623395769@lists.varnish-cache.org>


commit f465e905e97e858034171ddfbd5ad4ab3bed4015
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 4 07:17:58 2018 +0000

    Don't poll VSM_Status() while there is work to do and no interruptions.
    
    Fixes   #2788

diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c
index 03cd9f07a..b02644a81 100644
--- a/lib/libvarnishapi/vut.c
+++ b/lib/libvarnishapi/vut.c
@@ -420,7 +420,10 @@ VUT_Main(struct VUT *vut)
 			AZ(c);
 		}
 
-		i = VSLQ_Dispatch(vut->vslq, vut_dispatch, vut);
+		do
+			i = VSLQ_Dispatch(vut->vslq, vut_dispatch, vut);
+		while (i == vsl_more && !vut->sighup && !vut->sigusr1);
+
 		if (i == vsl_more)
 			continue;
 		else if (i == vsl_end) {

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:26 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:26 +0000 (UTC)
Subject: [6.0] debd2c3bd debloat the vtim test
Message-ID: <20181031130826.7FD919576F@lists.varnish-cache.org>


commit debd2c3bde369ab01f876e2caa97b640084fe043
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Oct 4 20:24:10 2018 +0200

    debloat the vtim test

diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index 4bef5e711..a9fbc7cb2 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -529,12 +529,26 @@ bench()
 	    e - s, i, 1e9 * (e - s) / i, t);
 }
 
+void
+parse_check(time_t t, const char *s)
+{
+	double tt;
+	char buf[BUFSIZ];
+
+	tt = VTIM_parse(s);
+	if (tt != t) {
+		VTIM_format(tt, buf);
+		printf("  fm: %12jd <%s>\n", (intmax_t)t, s);
+		printf("  to: %12.0f <%s>\n", tt, buf);
+		exit(2);
+	}
+}
+
 int
 main(int argc, char **argv)
 {
 	time_t t;
 	struct tm tm;
-	double tt;
 	char buf[BUFSIZ];
 	char buf1[BUFSIZ];
 
@@ -553,41 +567,17 @@ main(int argc, char **argv)
 			    buf1, buf, (intmax_t)t);
 			exit(2);
 		}
-		tt = VTIM_parse(buf1);
-		if (tt != t) {
-			VTIM_format(tt, buf);
-			printf("  fm: %12jd <%s>\n", (intmax_t)t, buf1);
-			printf("  to: %12.0f <%s>\n", tt, buf);
-			exit(2);
-		}
+		parse_check(t, buf1);
 
 		strftime(buf1, sizeof buf1, "%a %b %e %T %Y", &tm);
-		tt = VTIM_parse(buf1);
-		if (tt != t) {
-			VTIM_format(tt, buf);
-			printf("  fm: %12jd <%s>\n", (intmax_t)t, buf1);
-			printf("  to: %12.0f <%s>\n", tt, buf);
-			exit(2);
-		}
+		parse_check(t, buf1);
 
 		strftime(buf1, sizeof buf1, "%Y-%m-%dT%T", &tm);
-		tt = VTIM_parse(buf1);
-		if (tt != t) {
-			VTIM_format(tt, buf);
-			printf("  fm: %12jd <%s>\n", (intmax_t)t, buf1);
-			printf("  to: %12.0f <%s>\n", tt, buf);
-			exit(2);
-		}
+		parse_check(t, buf1);
 
 		if (tm.tm_year >= 69 && tm.tm_year < 169) {
 			strftime(buf1, sizeof buf1, "%A, %d-%b-%y %T GMT", &tm);
-			tt = VTIM_parse(buf1);
-			if (tt != t) {
-				VTIM_format(tt, buf);
-				printf("  fm: %12jd <%s>\n", (intmax_t)t, buf1);
-				printf("  to: %12.0f <%s>\n", tt, buf);
-				exit(2);
-			}
+			parse_check(t, buf1);
 		}
 	}
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:26 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:26 +0000 (UTC)
Subject: [6.0] eff757e8b trivial vtim printf benchmark
Message-ID: <20181031130826.AF93C9578A@lists.varnish-cache.org>


commit eff757e8bccfb3612f846c74feaca43b616a264b
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Oct 5 09:56:27 2018 +0200

    trivial vtim printf benchmark

diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index a9fbc7cb2..f9de6244e 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -511,6 +511,7 @@ bench()
 {
 	double s, e, t;
 	int i;
+	char buf[64];
 
 	t = 0;
 	s = VTIM_real();
@@ -527,6 +528,17 @@ bench()
 	e = VTIM_real();
 	printf("mono: %fs / %d = %fns - tst val %f\n",
 	    e - s, i, 1e9 * (e - s) / i, t);
+
+	t = 0;
+	s = VTIM_mono();
+	for (i=0; i<100000; i++) {
+		snprintf(buf, sizeof(buf), "%.6f", s);
+		t += buf[4];
+	}
+	e = VTIM_mono();
+	printf("%s\n", buf);
+	printf("printf: %fs / %d = %fns - tst val %f\n",
+	    e - s, i, 1e9 * (e - s) / i, t);
 }
 
 void

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:26 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:26 +0000 (UTC)
Subject: [6.0] faec2d33b Don't use txprio to open streams in tests
Message-ID: <20181031130826.D9EFC9579A@lists.varnish-cache.org>


commit faec2d33b8a8852f6b74c22f9f603be73193aafb
Author: Carlo Cannas <carlo94 at gmail.com>
Date:   Sun Sep 23 03:34:02 2018 +0200

    Don't use txprio to open streams in tests
    
    As per spec a client can only send a HEADERS frame to cause a stream to
    transition from the "idle" state to "open". A PRIORITY frame can be sent to an
    "idle" stream, but it will remain in that state.
    
    rfc7540,l,916,940
    https://tools.ietf.org/html/rfc7540#section-5.1
    
    To open a stream the command
    
    txreq -nostrend
    
    can be used. The -nostrend option will ensure that the stream won't transition
    to a "half-closed" state.

diff --git a/bin/varnishtest/tests/t02003.vtc b/bin/varnishtest/tests/t02003.vtc
index b355fd4b8..215171cf6 100644
--- a/bin/varnishtest/tests/t02003.vtc
+++ b/bin/varnishtest/tests/t02003.vtc
@@ -43,10 +43,10 @@ client c1 {
 		expect goaway.err == PROTOCOL_ERROR
 	} -start
 	stream 3 {
-		txprio
+		txreq
 	} -run
 	stream 1 {
-		txprio
+		txreq
 	} -run
 	stream 0 -wait
 } -run
@@ -63,13 +63,13 @@ varnish v1 -expect MEMPOOL.sess1.live == 0
 
 client c1 {
 	stream 1 {
-		txprio
+		txreq -nostrend
 		txwinup -size 0
 		rxrst
 		expect rst.err == PROTOCOL_ERROR
 	} -run
 	stream 3 {
-		txprio
+		txreq -nostrend
 		txwinup -size 0x40000000
 		txwinup -size 0x40000000
 		rxrst
@@ -81,7 +81,7 @@ client c1 {
 		expect goaway.err == FRAME_SIZE_ERROR
 	} -start
 	stream 5 {
-		txprio
+		txreq -nostrend
 		sendhex "000003 08 00 00000005"
 		delay .1
 		sendhex 01
@@ -167,7 +167,7 @@ client c1 {
 		expect goaway.laststream == 1
 	} -start
 	stream 1 {
-		txprio
+		txreq -nostrend
 		sendhex "000008 05 00 00000001 0001020304050607"
 	} -run
 	stream 0 -wait
@@ -200,7 +200,7 @@ client c1 {
 		expect goaway.laststream == 1
 	} -start
 	stream 1 {
-		txprio
+		txreq -nostrend
 		# RST wrong length
 		sendhex "000005 03 00 00000001 0000000800"
 	} -run

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:27 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:27 +0000 (UTC)
Subject: [6.0] d653365fa Move header block frame sequence check earlier
Message-ID: <20181031130827.11267957A3@lists.varnish-cache.org>


commit d653365fa5a717a2ae085337a4ef343e24eb8a95
Author: Carlo Cannas <carlo94 at gmail.com>
Date:   Sun Sep 23 03:34:02 2018 +0200

    Move header block frame sequence check earlier
    
    This moves it before the new stream object creation, so we save ourselves an
    useless allocation and initialization of a stream object which would be never
    used and straight killed.
    
    This also simplifies upcoming commits.

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 070bf2cad..b106f9e36 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -923,6 +923,10 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2, h2_frame h2f)
 		if (r2->stream == h2->rxf_stream)
 			break;
 
+	if (h2->new_req != NULL &&
+	    !(r2 && h2->new_req == r2->req && h2f == H2_F_CONTINUATION))
+		return (H2CE_PROTOCOL_ERROR);	// rfc7540,l,1859,1863
+
 	if (r2 == NULL && h2f->act_sidle == 0) {
 		if (h2->rxf_stream <= h2->highest_stream)
 			return (H2CE_PROTOCOL_ERROR);	// rfc7540,l,1153,1158
@@ -940,10 +944,6 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2, h2_frame h2f)
 		AN(r2);
 	}
 
-	if (h2->new_req != NULL &&
-	    !(r2 && h2->new_req == r2->req && h2f == H2_F_CONTINUATION))
-		return (H2CE_PROTOCOL_ERROR);	// rfc7540,l,1859,1863
-
 	h2e = h2f->rxfunc(wrk, h2, r2);
 	if (h2e == 0)
 		return (0);
diff --git a/bin/varnishtest/tests/r02387.vtc b/bin/varnishtest/tests/r02387.vtc
index 34863a057..d2c9796e7 100644
--- a/bin/varnishtest/tests/r02387.vtc
+++ b/bin/varnishtest/tests/r02387.vtc
@@ -30,7 +30,7 @@ client c1 {
 	} -run
 	stream 0 {
 		rxgoaway
-		expect goaway.laststream == "3"
+		expect goaway.laststream == "1"
 		expect goaway.err == PROTOCOL_ERROR
 	} -run
 } -run

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:27 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:27 +0000 (UTC)
Subject: [6.0] 7c7a90fdf Allow PRIORITY frames on closed streams
Message-ID: <20181031130827.38DD3957AD@lists.varnish-cache.org>


commit 7c7a90fdf86d0029ea25090513afacfb348bfab1
Author: Carlo Cannas <carlo94 at gmail.com>
Date:   Sun Sep 23 03:34:02 2018 +0200

    Allow PRIORITY frames on closed streams
    
    Currently Varnish doesn't allow PRIORITY frames to be received on closed
    streams: it treats it as a protocol violation and replies with a GOAWAY.
    
    This is not spec compliant, rfc7540 states:
    
    The PRIORITY frame can be sent for a stream in the "idle" or "closed"
    state.
    rfc7540,l,1947,1948
    
    The PRIORITY frame can be sent on a stream in any state
    rfc7540,l,1938,1938
    
    https://tools.ietf.org/html/rfc7540#section-6.3
    
    This behaviour can be triggered by real-world browsers: Chrome 69 has been
    observed sending PRIORITY frames which are received by Varnish after a stream
    has been closed (and cleaned by h2_sweep). When that happens the connection is
    closed and Chrome aborts the loading of all the resources which started to
    load on that connection.
    
    This commit solves the issue by avoiding all the stream creation code and its
    checks to be performed when a PRIORITY frame is received.
    This moves all the stream creation logic inside h2_rx_headers, the only other
    frame which is allowed on idle streams.
    
    This also fixes the concurrent streams counter and highest_stream: they should
    be updated only when a stream enters the "open" state (or "reserved" if
    Varnish used served push) but currently a PRIORITY frame on an idle stream
    affects them.
    
    https://tools.ietf.org/html/rfc7540#section-5.1.1
    rfc7540,l,1153,1156
    rfc7540,l,1193,1198
    rfc7540,l,1530,1533
    
    Fixes: #2775

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index b106f9e36..250af1806 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -382,7 +382,7 @@ h2_rx_priority(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 
 	(void)wrk;
 	ASSERT_RXTHR(h2);
-	xxxassert(r2->stream & 1);
+	(void)r2;
 	return (0);
 }
 
@@ -616,7 +616,21 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 	size_t l;
 
 	ASSERT_RXTHR(h2);
+
+	if (r2 == NULL) {
+		if (h2->rxf_stream <= h2->highest_stream)
+			return (H2CE_PROTOCOL_ERROR);	// rfc7540,l,1153,1158
+		if (h2->refcnt >= h2->local_settings.max_concurrent_streams) {
+			VSLb(h2->vsl, SLT_Debug,
+			     "H2: stream %u: Hit maximum number of "
+			     "concurrent streams", h2->rxf_stream);
+			return (H2SE_REFUSED_STREAM);	// rfc7540,l,1200,1205
+		}
+		h2->highest_stream = h2->rxf_stream;
+		r2 = h2_new_req(wrk, h2, h2->rxf_stream, NULL);
+	}
 	AN(r2);
+
 	if (r2->state != H2_S_IDLE)
 		return (H2CE_PROTOCOL_ERROR);	// XXX spec ?
 	r2->state = H2_S_OPEN;
@@ -927,23 +941,6 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2, h2_frame h2f)
 	    !(r2 && h2->new_req == r2->req && h2f == H2_F_CONTINUATION))
 		return (H2CE_PROTOCOL_ERROR);	// rfc7540,l,1859,1863
 
-	if (r2 == NULL && h2f->act_sidle == 0) {
-		if (h2->rxf_stream <= h2->highest_stream)
-			return (H2CE_PROTOCOL_ERROR);	// rfc7540,l,1153,1158
-		if (h2->refcnt >= h2->local_settings.max_concurrent_streams) {
-			VSLb(h2->vsl, SLT_Debug,
-			     "H2: stream %u: Hit maximum number of "
-			     "concurrent streams", h2->rxf_stream);
-			// rfc7540,l,1200,1205
-			h2_tx_rst(wrk, h2, h2->req0, h2->rxf_stream,
-				H2SE_REFUSED_STREAM);
-			return (0);
-		}
-		h2->highest_stream = h2->rxf_stream;
-		r2 = h2_new_req(wrk, h2, h2->rxf_stream, NULL);
-		AN(r2);
-	}
-
 	h2e = h2f->rxfunc(wrk, h2, r2);
 	if (h2e == 0)
 		return (0);
@@ -993,7 +990,6 @@ static int
 h2_sweep(struct worker *wrk, struct h2_sess *h2)
 {
 	int tmo = 0;
-	int nprio = 0;
 	struct h2_req *r2, *r22;
 
 	ASSERT_RXTHR(h2);
@@ -1025,20 +1021,18 @@ h2_sweep(struct worker *wrk, struct h2_sess *h2)
 			}
 			break;
 		case H2_S_IDLE:
-			/* This stream was created from receiving a
-			 * PRIORITY frame, and should not be counted
-			 * as an active stream keeping the connection
-			 * open. */
-			AZ(r2->scheduled);
-			nprio++;
-			break;
+			/* Current code make this unreachable: h2_new_req is
+			 * only called inside h2_rx_headers, which immediately
+			 * sets the new stream state to H2_S_OPEN */
+			/* FALLTHROUGH */
 		default:
+			WRONG("Wrong h2 stream state");
 			break;
 		}
 	}
 	if (tmo)
 		return (0);
-	return ((h2->refcnt - nprio) > 1);
+	return (h2->refcnt > 1);
 }
 
 
diff --git a/bin/varnishtest/tests/r02775.vtc b/bin/varnishtest/tests/r02775.vtc
new file mode 100644
index 000000000..de66c10d1
--- /dev/null
+++ b/bin/varnishtest/tests/r02775.vtc
@@ -0,0 +1,23 @@
+varnishtest "Regression test for #2775: allow PRIORITY on closed stream"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {} -start
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.set debug +syncvsl"
+
+client c1 {
+	stream 1 {
+		txreq
+		rxresp
+
+		txprio
+	} -run
+	stream 3 {
+		txreq
+		rxresp
+	} -run
+} -run

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:27 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:27 +0000 (UTC)
Subject: [6.0] 413cbd442 Hardening of the h2_frame_f callbacks
Message-ID: <20181031130827.5E0BD957BD@lists.varnish-cache.org>


commit 413cbd442bab8802133cb4cd62d891c938186d97
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Oct 5 11:16:59 2018 +0200

    Hardening of the h2_frame_f callbacks
    
    And by the way, they are known as h2_rxframe_f these days!
    
    Refs #2781

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 250af1806..6caa358bf 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -275,11 +275,14 @@ h2_vsl_frame(const struct h2_sess *h2, const void *ptr, size_t len)
 /**********************************************************************
  */
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_ping(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
+	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
+	assert(r2 == h2->req0);
 
 	if (h2->rxf_len != 8)				// rfc7540,l,2364,2366
 		return (H2CE_FRAME_SIZE_ERROR);
@@ -296,26 +299,27 @@ h2_rx_ping(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 /**********************************************************************
  */
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_push_promise(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 
-	(void)wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
+	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); /* XXX: wasteful allocation? */
 	// rfc7540,l,2262,2267
-	(void)r2;
 	return (H2CE_PROTOCOL_ERROR);
 }
 
 /**********************************************************************
  */
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_rst_stream(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 
-	(void)wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
+	CHECK_OBJ_ORNULL(r2, H2_REQ_MAGIC);
 
 	if (h2->rxf_len != 4)			// rfc7540,l,2003,2004
 		return (H2CE_FRAME_SIZE_ERROR);
@@ -328,13 +332,15 @@ h2_rx_rst_stream(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 /**********************************************************************
  */
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_goaway(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 
-	(void)wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
-	(void)r2;
+	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
+	assert(r2 == h2->req0);
+
 	h2->goaway_last_stream = vbe32dec(h2->rxf_data);
 	h2->error = h2_connectionerror(vbe32dec(h2->rxf_data + 4));
 	Lck_Lock(&h2->sess->mtx);
@@ -346,13 +352,15 @@ h2_rx_goaway(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 /**********************************************************************
  */
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_window_update(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 	uint32_t wu;
 
-	(void)wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
+	CHECK_OBJ_ORNULL(r2, H2_REQ_MAGIC);
+
 	if (h2->rxf_len != 4)
 		return (H2CE_FRAME_SIZE_ERROR);
 	wu = vbe32dec(h2->rxf_data) & ~(1LU<<31);
@@ -376,13 +384,13 @@ h2_rx_window_update(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
  * Incoming PRIORITY, possibly an ACK of one we sent.
  */
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_priority(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 
-	(void)wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
-	(void)r2;
+	CHECK_OBJ_ORNULL(r2, H2_REQ_MAGIC);
 	return (0);
 }
 
@@ -478,17 +486,19 @@ h2_set_setting(struct h2_sess *h2, const uint8_t *d)
 	return (0);
 }
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_settings(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 	const uint8_t *p;
 	unsigned l;
 	h2_error retval = 0;
 
-	AN(wrk);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
-	AN(r2);
+	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
+	assert(r2 == h2->req0);
 	AZ(h2->rxf_stream);
+
 	if (h2->rxf_flags == H2FF_SETTINGS_ACK) {
 		if (h2->rxf_len > 0)			// rfc7540,l,2047,2049
 			return (H2CE_FRAME_SIZE_ERROR);
@@ -607,7 +617,7 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2,
 	return (0);
 }
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 	struct req *req;
@@ -615,6 +625,7 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 	const uint8_t *p;
 	size_t l;
 
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
 
 	if (r2 == NULL) {
@@ -629,7 +640,7 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 		h2->highest_stream = h2->rxf_stream;
 		r2 = h2_new_req(wrk, h2, h2->rxf_stream, NULL);
 	}
-	AN(r2);
+	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
 
 	if (r2->state != H2_S_IDLE)
 		return (H2CE_PROTOCOL_ERROR);	// XXX spec ?
@@ -695,13 +706,16 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 
 /**********************************************************************/
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_continuation(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 	struct req *req;
 	h2_error h2e;
 
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
+	CHECK_OBJ_ORNULL(r2, H2_REQ_MAGIC);
+
 	if (r2 == NULL || r2->state != H2_S_OPEN || r2->req != h2->new_req)
 		return (H2CE_PROTOCOL_ERROR);	// XXX spec ?
 	req = r2->req;
@@ -723,15 +737,17 @@ h2_rx_continuation(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 
 /**********************************************************************/
 
-static h2_error v_matchproto_(h2_frame_f)
+static h2_error v_matchproto_(h2_rxframe_f)
 h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 {
 	int w1 = 0, w2 = 0;
 	char buf[4];
 	unsigned wi;
 
-	(void)wrk;
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
+	CHECK_OBJ_ORNULL(r2, H2_REQ_MAGIC);
+
 	if (r2 == NULL || !r2->scheduled)
 		return (0);
 	if (r2->state >= H2_S_CLOS_REM) {

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:27 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:27 +0000 (UTC)
Subject: [6.0] ded79885b Fix assertion for PUSH_PROMISE frames
Message-ID: <20181031130827.897FE957CF@lists.varnish-cache.org>


commit ded79885b7aef4da453254240f709d9fe4a872a0
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri Oct 5 12:09:28 2018 +0200

    Fix assertion for PUSH_PROMISE frames
    
    r2 can be either null or not.
    
    Test case by @daghf
    
    Refs #2781

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 6caa358bf..7d040b7d0 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -305,7 +305,7 @@ h2_rx_push_promise(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	ASSERT_RXTHR(h2);
-	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); /* XXX: wasteful allocation? */
+	CHECK_OBJ_ORNULL(r2, H2_REQ_MAGIC);
 	// rfc7540,l,2262,2267
 	return (H2CE_PROTOCOL_ERROR);
 }
diff --git a/bin/varnishtest/tests/t02003.vtc b/bin/varnishtest/tests/t02003.vtc
index 215171cf6..1d62ac986 100644
--- a/bin/varnishtest/tests/t02003.vtc
+++ b/bin/varnishtest/tests/t02003.vtc
@@ -173,6 +173,21 @@ client c1 {
 	stream 0 -wait
 } -run
 
+client c1 {
+	stream 0 {
+		rxgoaway
+		expect goaway.err == PROTOCOL_ERROR
+		expect goaway.laststream == 1
+	} -start
+	stream 1 {
+		txreq
+		rxresp
+		delay .1
+		# send a PUSH_PROMISE after a request
+		sendhex "000008 05 00 00000001 0001020304050607"
+	} -start
+} -run
+
 varnish v1 -vsl_catchup
 
 varnish v1 -expect MEMPOOL.req0.live == 0

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:27 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:27 +0000 (UTC)
Subject: [6.0] f6a34391d gethrtime() is now slower than clock_gettime() on
 modern Solarisen
Message-ID: <20181031130827.B1547957DC@lists.varnish-cache.org>


commit f6a34391d1e2a7529dec4c810b2d2799169e8dcf
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Oct 5 12:25:42 2018 +0200

    gethrtime() is now slower than clock_gettime() on modern Solarisen
    
    Throw out the conventional wisdom and base the decision on a micro
    benchmark.
    
    clock_gettime() is now preferred if it is consistently at least
    double as fast as gethrtime(), which is the case on varnishdev-il,
    the SmartOS vtest machine.
    
    config.log gives details on the performance check, sample output
    below:
    
    configure:22703: ./conftest
    hrtime              45989530 check 16748699083977959327
    clock_gettime        4119385 check 16748701613138517215
    ...
    hrtime              48113108 check 16748749015170035860
    clock_gettime        4020802 check 16748751585081458308
    clock_gettime wins 10/10

diff --git a/configure.ac b/configure.ac
index 00172c7cc..f32b8db3f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -355,6 +355,63 @@ AC_CHECK_FUNCS([clock_gettime])
 AC_CHECK_FUNCS([gethrtime])
 LIBS="${save_LIBS}"
 
+if test "x$ac_cv_func_gethrtime" = xyes && \
+   test "x$ac_cv_func_clock_gettime" = xyes ; then
+  AC_MSG_CHECKING(if clock_gettime is faster than gethrtime)
+  AC_RUN_IFELSE(
+    [AC_LANG_PROGRAM([[
+#include <sys/time.h>
+#include <time.h>
+#include <stdio.h>
+
+static hrtime_t cl()
+{
+	struct timespec ts;
+
+	(void) clock_gettime(CLOCK_MONOTONIC, &ts);
+	return (ts.tv_sec * 1e9 + ts.tv_nsec);
+}
+    ]],[[
+	hrtime_t s, c, e, t_hr, t_cl;
+	int i, r, wins;
+
+	wins = 0;
+	for (r = 0; r < 10; r++) {
+		c = 0;
+		s = gethrtime();
+		for (i=0; i<100000; i++)
+			c += gethrtime();
+		e = gethrtime();
+		t_hr = e - s;
+		fprintf(stderr, "hrtime\t\t%12lu check %lu\n",
+		    (unsigned long)t_hr, (unsigned long)c);
+
+		c = 0;
+		s = gethrtime();
+		for (i=0; i<100000; i++)
+			c += cl();
+		e = gethrtime();
+		t_cl = e - s;
+		fprintf(stderr, "clock_gettime\t%12lu check %lu\n",
+		    (unsigned long)t_cl, (unsigned long)c);
+
+		if (t_cl * 2 < t_hr)
+			wins++;
+	}
+	fprintf(stderr, "clock_gettime wins %d/%d\n", wins, r);
+	if (2 * wins >= r)
+		return (0);
+	return (1);
+    ]])],
+    [AC_MSG_RESULT(yes)
+     AC_DEFINE([USE_GETHRTIME], [0], [whether to use gethrtime])
+    ],
+    [AC_MSG_RESULT(no)
+     AC_DEFINE([USE_GETHRTIME], [1], [whether to use gethrtime])
+    ]
+  )
+fi
+
 # --enable-kqueue
 AC_ARG_ENABLE(kqueue,
     AS_HELP_STRING([--enable-kqueue],
diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index f9de6244e..39d5ca342 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -118,15 +118,16 @@ init(void)
 #endif
 
 /*
- * Note on Solaris: for some reason, clock_gettime(CLOCK_MONOTONIC, &ts) is not
- * implemented in assembly, but falls into a syscall, while gethrtime() doesn't,
- * so we save a syscall by using gethrtime() if it is defined.
+ * On older Solaris-incarnations, gethrtime() was faster than
+ * clock_gettime(CLOCK_MONOTONIC). Our configure script prefers
+ * clock_gettime if it is consistently at least twice as fast as
+ * gethrtime(), which is the case on modern Solaris descendents.
  */
 
 double
 VTIM_mono(void)
 {
-#ifdef HAVE_GETHRTIME
+#if defined(HAVE_GETHRTIME) && USE_GETHRTIME
 	return (gethrtime() * 1e-9);
 #elif  HAVE_CLOCK_GETTIME
 	struct timespec ts;

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:27 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:27 +0000 (UTC)
Subject: [6.0] 477e55066 fix a minor oversight
Message-ID: <20181031130827.D8F81957E7@lists.varnish-cache.org>


commit 477e550662397dda1d2186c298b69dfe95efb47b
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Oct 5 12:38:02 2018 +0200

    fix a minor oversight
    
    I failed to consider the hypothetical case that there is only gethrtime()
    and no clock_gettime(CLOCK_MONOTONIC).

diff --git a/configure.ac b/configure.ac
index f32b8db3f..468e1cfd2 100644
--- a/configure.ac
+++ b/configure.ac
@@ -355,6 +355,7 @@ AC_CHECK_FUNCS([clock_gettime])
 AC_CHECK_FUNCS([gethrtime])
 LIBS="${save_LIBS}"
 
+AC_DEFINE([USE_GETHRTIME], [1], [whether to use gethrtime])
 if test "x$ac_cv_func_gethrtime" = xyes && \
    test "x$ac_cv_func_clock_gettime" = xyes ; then
   AC_MSG_CHECKING(if clock_gettime is faster than gethrtime)
@@ -407,7 +408,6 @@ static hrtime_t cl()
      AC_DEFINE([USE_GETHRTIME], [0], [whether to use gethrtime])
     ],
     [AC_MSG_RESULT(no)
-     AC_DEFINE([USE_GETHRTIME], [1], [whether to use gethrtime])
     ]
   )
 fi

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:28 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:28 +0000 (UTC)
Subject: [6.0] d4a8fd371 Polish
Message-ID: <20181031130828.0C1D4957F8@lists.varnish-cache.org>


commit d4a8fd3716e8f474f2bb7580be72a359fb5ea045
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Fri Oct 5 14:07:03 2018 +0100

    Polish

diff --git a/configure.ac b/configure.ac
index 468e1cfd2..13b54b5fb 100644
--- a/configure.ac
+++ b/configure.ac
@@ -355,7 +355,6 @@ AC_CHECK_FUNCS([clock_gettime])
 AC_CHECK_FUNCS([gethrtime])
 LIBS="${save_LIBS}"
 
-AC_DEFINE([USE_GETHRTIME], [1], [whether to use gethrtime])
 if test "x$ac_cv_func_gethrtime" = xyes && \
    test "x$ac_cv_func_clock_gettime" = xyes ; then
   AC_MSG_CHECKING(if clock_gettime is faster than gethrtime)
@@ -405,9 +404,9 @@ static hrtime_t cl()
 	return (1);
     ]])],
     [AC_MSG_RESULT(yes)
-     AC_DEFINE([USE_GETHRTIME], [0], [whether to use gethrtime])
     ],
     [AC_MSG_RESULT(no)
+     AC_DEFINE([USE_GETHRTIME], [1], [Define if gethrtime is preferred])
     ]
   )
 fi
diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index 39d5ca342..5c6503c7e 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -127,14 +127,14 @@ init(void)
 double
 VTIM_mono(void)
 {
-#if defined(HAVE_GETHRTIME) && USE_GETHRTIME
-	return (gethrtime() * 1e-9);
-#elif  HAVE_CLOCK_GETTIME
+#if defined(HAVE_CLOCK_GETTIME) && !defined(USE_GETHRTIME)
 	struct timespec ts;
 
 	AZ(clock_gettime(CLOCK_MONOTONIC, &ts));
 	return (ts.tv_sec + 1e-9 * ts.tv_nsec);
-#elif  defined(__MACH__)
+#elif defined(HAVE_GETHRTIME)
+	return (gethrtime() * 1e-9);
+#elif defined(__MACH__)
 	uint64_t mt = mach_absolute_time() - mt_base;
 
 	return (mt * mt_scale);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:28 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:28 +0000 (UTC)
Subject: [6.0] f9b5e83e1 add phk's suggestion to the micro-benchmark
Message-ID: <20181031130828.2D5C195801@lists.varnish-cache.org>


commit f9b5e83e19621c4eb5032f4773ab3fbd3e2e6571
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 8 10:29:29 2018 +0200

    add phk's suggestion to the micro-benchmark

diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index 5c6503c7e..822f522d2 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -537,9 +537,20 @@ bench()
 		t += buf[4];
 	}
 	e = VTIM_mono();
-	printf("%s\n", buf);
-	printf("printf: %fs / %d = %fns - tst val %f\n",
-	    e - s, i, 1e9 * (e - s) / i, t);
+	printf("printf %%.6f: %fs / %d = %fns - tst val %f %s\n",
+	    e - s, i, 1e9 * (e - s) / i, t, buf);
+
+	t = 0;
+	s = VTIM_mono();
+	for (i=0; i<100000; i++) {
+		snprintf(buf, sizeof(buf), "%ju.%06ju",
+			 (uint64_t)floor(s),
+			 (uint64_t)floor((s * 1e6)) % 1000000UL);
+		t += buf[4];
+	}
+	e = VTIM_mono();
+	printf("printf %%ju.%%06ju: %fs / %d = %fns - tst val %f %s\n",
+	    e - s, i, 1e9 * (e - s) / i, t, buf);
 }
 
 void

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:28 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:28 +0000 (UTC)
Subject: [6.0] 2b078b9ba style(9)
Message-ID: <20181031130828.53C579580C@lists.varnish-cache.org>


commit 2b078b9ba3942267e4a64fa028638be261d309d3
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 8 10:42:15 2018 +0200

    style(9)

diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index 822f522d2..c774e2ab8 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -544,8 +544,8 @@ bench()
 	s = VTIM_mono();
 	for (i=0; i<100000; i++) {
 		snprintf(buf, sizeof(buf), "%ju.%06ju",
-			 (uint64_t)floor(s),
-			 (uint64_t)floor((s * 1e6)) % 1000000UL);
+		    (uint64_t)floor(s),
+		    (uint64_t)floor((s * 1e6)) % 1000000UL);
 		t += buf[4];
 	}
 	e = VTIM_mono();

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:28 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:28 +0000 (UTC)
Subject: [6.0] 2411ac9f6 Check we have space before adding the Date header
Message-ID: <20181031130828.753FB95820@lists.varnish-cache.org>


commit 2411ac9f647da514529951b93681e148a7061558
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Oct 8 10:23:38 2018 +0100

    Check we have space before adding the Date header

diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c
index 30988ab56..23eaa0b18 100644
--- a/bin/varnishd/cache/cache_http.c
+++ b/bin/varnishd/cache/cache_http.c
@@ -1233,6 +1233,11 @@ http_TimeHeader(struct http *to, const char *fmt, double now)
 	char *p;
 
 	CHECK_OBJ_NOTNULL(to, HTTP_MAGIC);
+	if (to->nhd >= to->shd) {
+		VSLb(to->vsl, SLT_LostHeader, "%s", fmt);
+		http_fail(to);
+		return;
+	}
 	p = WS_Alloc(to->ws, strlen(fmt) + VTIM_FORMAT_SIZE);
 	if (p == NULL) {
 		http_fail(to);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:28 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:28 +0000 (UTC)
Subject: [6.0] 0a89d2c9c Add JSON support for the CLI "status" command.
Message-ID: <20181031130828.97E9195835@lists.varnish-cache.org>


commit 0a89d2c9c89955deb833acf00760607f18581520
Author: Geoff Simmons <geoff at uplex.de>
Date:   Fri Sep 21 17:25:19 2018 +0200

    Add JSON support for the CLI "status" command.

diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index d03ff2106..026bdd8e8 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -679,8 +679,19 @@ mch_cli_server_status(struct cli *cli, const char * const *av, void *priv)
 	VCLI_Out(cli, "Child in state %s", ch_state[child_state]);
 }
 
+static void v_matchproto_(cli_func_t)
+mch_cli_server_status_json(struct cli *cli, const char * const *av, void *priv)
+{
+	(void)priv;
+	VCLI_JSON_begin(cli, 2, av);
+	VCLI_Out(cli, ", ");
+	VCLI_JSON_str(cli, ch_state[child_state]);
+	VCLI_JSON_end(cli);
+}
+
 static struct cli_proto cli_mch[] = {
-	{ CLICMD_SERVER_STATUS,		"", mch_cli_server_status },
+	{ CLICMD_SERVER_STATUS,		"", mch_cli_server_status,
+	  mch_cli_server_status_json },
 	{ CLICMD_SERVER_START,		"", mch_cli_server_start },
 	{ CLICMD_SERVER_STOP,		"", mch_cli_server_stop },
 	{ CLICMD_PANIC_SHOW,		"", mch_cli_panic_show },
diff --git a/bin/varnishtest/tests/b00004.vtc b/bin/varnishtest/tests/b00004.vtc
index ec5cf7661..2ed2c10ff 100644
--- a/bin/varnishtest/tests/b00004.vtc
+++ b/bin/varnishtest/tests/b00004.vtc
@@ -7,7 +7,11 @@ server s1 {
 varnish v1 -vcl+backend { }
 
 varnish v1 -start
+varnish v1 -cliexpect "running" status
+varnish v1 -clijson "status -j"
 varnish v1 -stop
+varnish v1 -cliexpect "stopped" status
+varnish v1 -clijson "status -j"
 
 varnish v1 -start
 varnish v1 -stop

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:28 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:28 +0000 (UTC)
Subject: [6.0] bf1d9979c Add JSON support for the CLI "vcl.list" command.
Message-ID: <20181031130828.CBAAC95865@lists.varnish-cache.org>


commit bf1d9979cfa8fe721805016777ed6de147246d9e
Author: Geoff Simmons <geoff at uplex.de>
Date:   Sun Sep 23 22:49:07 2018 +0200

    Add JSON support for the CLI "vcl.list" command.

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index e705a86a1..ae846e0a6 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -664,6 +664,52 @@ vcl_cli_list(struct cli *cli, const char * const *av, void *priv)
 	}
 }
 
+static void v_matchproto_(cli_func_t)
+vcl_cli_list_json(struct cli *cli, const char * const *av, void *priv)
+{
+	struct vcl *vcl;
+
+	(void)priv;
+	ASSERT_CLI();
+	VCLI_JSON_begin(cli, 2, av);
+	VCLI_Out(cli, ",\n");
+	VTAILQ_FOREACH(vcl, &vcl_head, list) {
+		VCLI_Out(cli, "{\n");
+		VSB_indent(cli->sb, 2);
+		VCLI_Out(cli, "\"status\": ");
+		if (vcl == vcl_active) {
+			VCLI_Out(cli, "\"active\",\n");
+		} else if (vcl->discard) {
+			VCLI_Out(cli, "\"discarded\",\n");
+		} else
+			VCLI_Out(cli, "\"available\",\n");
+		VCLI_Out(cli, "\"state\": \"%s\",\n", vcl->state);
+		VCLI_Out(cli, "\"temperature\": \"%s\",\n", vcl->temp);
+		VCLI_Out(cli, "\"busy\": %u,\n", vcl->busy);
+		VCLI_Out(cli, "\"name\": \"%s\"", vcl->loaded_name);
+		if (vcl->label != NULL) {
+			VCLI_Out(cli, ",\n");
+			VCLI_Out(cli, "\"label\": {\n");
+			VSB_indent(cli->sb, 2);
+				VCLI_Out(cli, "\"name\": \"%s\"",
+					 vcl->label->loaded_name);
+			if (vcl->nrefs)
+				VCLI_Out(cli, ",\n\"refs\": %d", vcl->nrefs);
+			VCLI_Out(cli, "\n");
+			VCLI_Out(cli, "}");
+			VSB_indent(cli->sb, -2);
+		} else if (vcl->nlabels > 0) {
+			VCLI_Out(cli, ",\n");
+			VCLI_Out(cli, "\"labels\": %d", vcl->nlabels);
+		}
+		VSB_indent(cli->sb, -2);
+		VCLI_Out(cli, "\n}");
+		if (VTAILQ_NEXT(vcl, list) != NULL)
+			VCLI_Out(cli, ",\n");
+	}
+	VCLI_JSON_end(cli);
+}
+
 static void v_matchproto_(cli_func_t)
 vcl_cli_load(struct cli *cli, const char * const *av, void *priv)
 {
@@ -823,7 +869,7 @@ vcl_cli_show(struct cli *cli, const char * const *av, void *priv)
 
 static struct cli_proto vcl_cmds[] = {
 	{ CLICMD_VCL_LOAD,		"", vcl_cli_load },
-	{ CLICMD_VCL_LIST,		"", vcl_cli_list },
+	{ CLICMD_VCL_LIST,		"", vcl_cli_list, vcl_cli_list_json },
 	{ CLICMD_VCL_STATE,		"", vcl_cli_state },
 	{ CLICMD_VCL_DISCARD,		"", vcl_cli_discard },
 	{ CLICMD_VCL_USE,		"", vcl_cli_use },
diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c
index 5974b589a..8c05c5215 100644
--- a/bin/varnishd/mgt/mgt_vcl.c
+++ b/bin/varnishd/mgt/mgt_vcl.c
@@ -795,6 +795,61 @@ mcf_vcl_list(struct cli *cli, const char * const *av, void *priv)
 	}
 }
 
+static void v_matchproto_(cli_func_t)
+mcf_vcl_list_json(struct cli *cli, const char * const *av, void *priv)
+{
+	unsigned status;
+	char *p;
+	struct vclprog *vp;
+	struct vcldep *vd;
+
+	/* NB: Shall generate same output as vcl_cli_list() */
+
+	(void)priv;
+	if (MCH_Running()) {
+		if (!mgt_cli_askchild(&status, &p, "vcl.list -j\n")) {
+			VCLI_SetResult(cli, status);
+			VCLI_Out(cli, "%s", p);
+		}
+		free(p);
+	} else {
+		VCLI_JSON_begin(cli, 2, av);
+		VCLI_Out(cli, ",\n");
+		VTAILQ_FOREACH(vp, &vclhead, list) {
+			VCLI_Out(cli, "{\n");
+			VSB_indent(cli->sb, 2);
+			VCLI_Out(cli, "\"status\": \"%s\",\n",
+			    vp == active_vcl ? "active" : "available");
+			VCLI_Out(cli, "\"state\": \"%s\",\n", vp->state);
+			VCLI_Out(cli, "\"temperature\": \"%s\",\n",
+			    vp->warm ? VCL_STATE_WARM : VCL_STATE_COLD);
+			VCLI_Out(cli, "\"name\": \"%s\"", vp->name);
+			if (mcf_is_label(vp)) {
+				vd = VTAILQ_FIRST(&vp->dfrom);
+				AN(vd);
+				VCLI_Out(cli, ",\n");
+				VCLI_Out(cli, "\"label\": {\n");
+				VSB_indent(cli->sb, 2);
+				VCLI_Out(cli, "\"name\": \"%s\"", vd->to->name);
+				if (vp->nto > 0)
+					VCLI_Out(cli, ",\n\"refs\": %d",
+						 vp->nto);
+				VSB_indent(cli->sb, -2);
+				VCLI_Out(cli, "\n");
+				VCLI_Out(cli, "}");
+			} else if (vp->nto > 0) {
+				VCLI_Out(cli, ",\n");
+				VCLI_Out(cli, "\"labels\": %d", vp->nto);
+			}
+			VSB_indent(cli->sb, -2);
+			VCLI_Out(cli, "\n}");
+			if (VTAILQ_NEXT(vp, list) != NULL)
+				VCLI_Out(cli, ",\n");
+		}
+		VCLI_JSON_end(cli);
+	}
+}
+
 static void v_matchproto_(cli_func_t)
 mcf_vcl_label(struct cli *cli, const char * const *av, void *priv)
 {
@@ -888,7 +943,7 @@ static struct cli_proto cli_vcl[] = {
 	{ CLICMD_VCL_USE,		"", mcf_vcl_use },
 	{ CLICMD_VCL_STATE,		"", mcf_vcl_state },
 	{ CLICMD_VCL_DISCARD,		"", mcf_vcl_discard },
-	{ CLICMD_VCL_LIST,		"", mcf_vcl_list },
+	{ CLICMD_VCL_LIST,		"", mcf_vcl_list, mcf_vcl_list_json },
 	{ CLICMD_VCL_LABEL,		"", mcf_vcl_label },
 	{ NULL }
 };
diff --git a/bin/varnishtest/tests/b00032.vtc b/bin/varnishtest/tests/b00032.vtc
index 89cb26f52..7aa6cddfa 100644
--- a/bin/varnishtest/tests/b00032.vtc
+++ b/bin/varnishtest/tests/b00032.vtc
@@ -11,9 +11,13 @@ varnish v1 -vcl+backend {}
 varnish v1 -vcl+backend {}
 
 varnish v1 -cliok vcl.list
+varnish v1 -clijson "vcl.list -j"
 
 varnish v1 -cliok start
 
+varnish v1 -cliok vcl.list
+varnish v1 -clijson "vcl.list -j"
+
 varnish v1 -cliok "vcl.use vcl1"
 
 varnish v1 -clierr 300 "vcl.discard vcl1"
diff --git a/bin/varnishtest/tests/s00005.vtc b/bin/varnishtest/tests/s00005.vtc
index 67cd3a95a..4dac2f167 100644
--- a/bin/varnishtest/tests/s00005.vtc
+++ b/bin/varnishtest/tests/s00005.vtc
@@ -39,6 +39,7 @@ varnish v1 -clierr 106 "vcl.label foo vcl0"
 varnish v1 -cliok "vcl.label foo vcl2"
 varnish v1 -cliok "vcl.label bar vcl2"
 varnish v1 -cliok "vcl.list"
+varnish v1 -clijson "vcl.list -j"
 varnish v1 -clierr 300 "vcl.discard vcl2"
 varnish v1 -cliok "vcl.discard bar"
 varnish v1 -cliok "vcl.label foo vcl1"
@@ -100,6 +101,7 @@ varnish v1 -cliok "vcl.label label1 vcl1"
 varnish v1 -cliok "vcl.label label2 vcl1"
 varnish v1 -cliok "vcl.label label3 vcl1"
 varnish v1 -cliok "vcl.list"
+varnish v1 -clijson "vcl.list -j"
 
 varnish v1 -start
 varnish v1 -cliok "vcl.list"
@@ -111,6 +113,7 @@ client c1 -run
 #######################################################################
 
 varnish v1 -cliok vcl.list
+varnish v1 -clijson "vcl.list -j"
 
 varnish v1 -vcl+backend { }
 
diff --git a/bin/varnishtest/tests/v00045.vtc b/bin/varnishtest/tests/v00045.vtc
index a0c511465..0f9cdf5a6 100644
--- a/bin/varnishtest/tests/v00045.vtc
+++ b/bin/varnishtest/tests/v00045.vtc
@@ -18,6 +18,7 @@ varnish v1 -cliok "vcl.state vcl1 cold"
 delay 1
 
 varnish v1 -cliexpect "cold/cooling.*vcl1" vcl.list
+varnish v1 -clijson "vcl.list -j"
 
 # It can't be warmed up yet
 delay 1
@@ -26,6 +27,7 @@ varnish v1 -cliexpect "vmod-debug ref on vcl1" "vcl.state vcl1 warm"
 # It will eventually cool down
 delay 2
 varnish v1 -cliexpect "cold/cold.*vcl1" vcl.list
+varnish v1 -clijson "vcl.list -j"
 
 # At this point it becomes possible to warm up again
 varnish v1 -cliok "vcl.state vcl1 warm"

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:28 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:28 +0000 (UTC)
Subject: [6.0] 6758bc023 Add JSON support for the "param.show" CLI command.
Message-ID: <20181031130828.E977C9587A@lists.varnish-cache.org>


commit 6758bc0235f955615a700a294e0ab7ecdd34d89f
Author: Geoff Simmons <geoff at uplex.de>
Date:   Mon Sep 24 17:51:01 2018 +0200

    Add JSON support for the "param.show" CLI command.
    
    * Each param is represented as a JSON object listed in the output
      array.
    * The object's "value" field contains the current value, which can
      have different types -- int, float, string or boolean.
    * There is no -l form with param.show -j, each object contains the
      full description.
    * But "param.show -j the_param" or "param.show -j changed" may be
      used to limit the number of objects in the output.
    * Each object may or may not have any of the fields "minimum",
      "maximum", "default" or "units", depending on the param.
    * Params not implemented on the present platform just have
      "implemented":false along with the param name, and no other fields.
    * The values of the "minimum", "maximum" and "default" fields are
      always strings.

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index 139814c09..f3faa9cb3 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -335,6 +335,126 @@ mcf_param_show(struct cli *cli, const char * const *av, void *priv)
 	VSB_destroy(&vsb);
 }
 
+static inline void
+mcf_json_key_valstr(struct cli *cli, const char *key, const char *val)
+{
+	VCLI_Out(cli, "\"%s\": ", key);
+	VCLI_JSON_str(cli, val);
+	VCLI_Out(cli, ",\n");
+}
+
+static void v_matchproto_(cli_func_t)
+mcf_param_show_json(struct cli *cli, const char * const *av, void *priv)
+{
+	int n, comma = 0;
+	struct plist *pl;
+	const struct parspec *pp;
+	int chg = 0, flags;
+	struct vsb *vsb, *def;
+	const char *show = NULL;
+
+	vsb = VSB_new_auto();
+	def = VSB_new_auto();
+	(void)priv;
+
+	for (int i = 2; av[i] != NULL; i++) {
+		if (strcmp(av[i], "-l") == 0) {
+			VCLI_SetResult(cli, CLIS_PARAM);
+			VCLI_Out(cli, "-l not permitted with param.show -j");
+			return;
+		}
+		if (strcmp(av[i], "changed") == 0) {
+			chg = 1;
+			continue;
+		}
+		if (strcmp(av[i], "-j") == 0)
+			continue;
+		show = av[i];
+	}
+
+	n = 0;
+	VCLI_JSON_begin(cli, 2, av);
+	VCLI_Out(cli, ",\n");
+	VTAILQ_FOREACH(pl, &phead, list) {
+		pp = pl->spec;
+		if (show != NULL && strcmp(pp->name, show) != 0)
+			continue;
+		n++;
+
+		VSB_clear(vsb);
+		if (pp->func(vsb, pp, JSON_FMT))
+			VCLI_SetResult(cli, CLIS_PARAM);
+		AZ(VSB_finish(vsb));
+		VSB_clear(def);
+		if (pp->func(def, pp, NULL))
+			VCLI_SetResult(cli, CLIS_PARAM);
+		AZ(VSB_finish(def));
+		if (chg && pp->def != NULL && !strcmp(pp->def, VSB_data(def)))
+			continue;
+
+		VCLI_Out(cli, "%s", comma ? ",\n" : "");
+		comma++;
+		VCLI_Out(cli, "{\n");
+		VSB_indent(cli->sb, 2);
+		mcf_json_key_valstr(cli, "name", pp->name);
+		if (pp->flags & NOT_IMPLEMENTED) {
+			VCLI_Out(cli, "\"implemented\": false\n");
+			VSB_indent(cli->sb, -2);
+			VCLI_Out(cli, "}");
+			continue;
+		}
+		VCLI_Out(cli, "\"implemented\": true,\n");
+		VCLI_Out(cli, "\"value\": %s,\n", VSB_data(vsb));
+		if (pp->units != NULL && *pp->units != '\0')
+			mcf_json_key_valstr(cli, "units", pp->units);
+
+		if (pp->def != NULL)
+			mcf_json_key_valstr(cli, "default", pp->def);
+		if (pp->min != NULL)
+			mcf_json_key_valstr(cli, "minimum", pp->min);
+		if (pp->max != NULL)
+			mcf_json_key_valstr(cli, "maximum", pp->max);
+		mcf_json_key_valstr(cli, "description", pp->descr);
+
+		flags = 0;
+		VCLI_Out(cli, "\"flags\": [\n");
+		VSB_indent(cli->sb, 2);
+
+#define flag_out(flag, string) do {					\
+			if (pp->flags & flag) {				\
+				if (flags)				\
+					VCLI_Out(cli, ",\n");		\
+				VCLI_Out(cli, "\"%s\"", #string);	\
+				flags++;				\
+			}						\
+		} while(0)
+
+		flag_out(OBJ_STICKY, obj_sticky);
+		flag_out(DELAYED_EFFECT, delayed_effect);
+		flag_out(EXPERIMENTAL, experimental);
+		flag_out(MUST_RELOAD, must_reload);
+		flag_out(MUST_RESTART, must_restart);
+		flag_out(WIZARD, wizard);
+		flag_out(PROTECTED, protected);
+		flag_out(ONLY_ROOT, only_root);
+
+#undef flag_out
+
+		VSB_indent(cli->sb, -2);
+		VCLI_Out(cli, "\n]");
+		VSB_indent(cli->sb, -2);
+		VCLI_Out(cli, "\n}");
+	}
+	VCLI_JSON_end(cli);
+	if (show != NULL && n == 0) {
+		VSB_clear(cli->sb);
+		VCLI_SetResult(cli, CLIS_PARAM);
+		VCLI_Out(cli, "Unknown parameter \"%s\".", show);
+	}
+	VSB_destroy(&vsb);
+	VSB_destroy(&def);
+}
+
 /*--------------------------------------------------------------------
  * Mark parameters as protected
  */
@@ -471,7 +591,8 @@ mcf_wash_param(struct cli *cli, const struct parspec *pp, const char **val,
 /*--------------------------------------------------------------------*/
 
 static struct cli_proto cli_params[] = {
-	{ CLICMD_PARAM_SHOW,		"", mcf_param_show },
+	{ CLICMD_PARAM_SHOW,		"", mcf_param_show,
+	  mcf_param_show_json },
 	{ CLICMD_PARAM_SET,		"", mcf_param_set },
 	{ NULL }
 };
diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h
index f4362ca96..7188b64e1 100644
--- a/bin/varnishd/mgt/mgt_param.h
+++ b/bin/varnishd/mgt/mgt_param.h
@@ -32,6 +32,9 @@ struct parspec;
 
 typedef int tweak_t(struct vsb *, const struct parspec *, const char *arg);
 
+/* Sentinel for the arg position of tweak_t to ask for JSON formatting. */
+extern const char * const JSON_FMT;
+
 struct parspec {
 	const char	*name;
 	tweak_t		*func;
diff --git a/bin/varnishd/mgt/mgt_param_bits.c b/bin/varnishd/mgt/mgt_param_bits.c
index a6d7b9216..263d8a34d 100644
--- a/bin/varnishd/mgt/mgt_param_bits.c
+++ b/bin/varnishd/mgt/mgt_param_bits.c
@@ -119,7 +119,7 @@ tweak_vsl_mask(struct vsb *vsb, const struct parspec *par, const char *arg)
 	const char *s;
 	(void)par;
 
-	if (arg != NULL) {
+	if (arg != NULL && arg != JSON_FMT) {
 		if (!strcmp(arg, "default")) {
 			memset(mgt_param.vsl_mask,
 			    0, sizeof mgt_param.vsl_mask);
@@ -141,6 +141,8 @@ tweak_vsl_mask(struct vsb *vsb, const struct parspec *par, const char *arg)
 			    "VSL tag", "-"));
 		}
 	} else {
+		if (arg == JSON_FMT)
+			VSB_putc(vsb, '"');
 		s = "";
 		for (j = 0; j < (unsigned)SLT__Reserved; j++) {
 			if (bit(mgt_param.vsl_mask, j, BTST)) {
@@ -150,6 +152,8 @@ tweak_vsl_mask(struct vsb *vsb, const struct parspec *par, const char *arg)
 		}
 		if (*s == '\0')
 			VSB_printf(vsb, "(all enabled)");
+		if (arg == JSON_FMT)
+			VSB_putc(vsb, '"');
 	}
 	return (0);
 }
@@ -171,7 +175,7 @@ tweak_debug(struct vsb *vsb, const struct parspec *par, const char *arg)
 	unsigned j;
 	(void)par;
 
-	if (arg != NULL) {
+	if (arg != NULL && arg != JSON_FMT) {
 		if (!strcmp(arg, "none")) {
 			memset(mgt_param.debug_bits,
 			    0, sizeof mgt_param.debug_bits);
@@ -180,6 +184,8 @@ tweak_debug(struct vsb *vsb, const struct parspec *par, const char *arg)
 			    DBG_Reserved, arg, debug_tags, "debug bit", "+"));
 		}
 	} else {
+		if (arg == JSON_FMT)
+			VSB_putc(vsb, '"');
 		s = "";
 		for (j = 0; j < (unsigned)DBG_Reserved; j++) {
 			if (bit(mgt_param.debug_bits, j, BTST)) {
@@ -189,6 +195,8 @@ tweak_debug(struct vsb *vsb, const struct parspec *par, const char *arg)
 		}
 		if (*s == '\0')
 			VSB_printf(vsb, "none");
+		if (arg == JSON_FMT)
+			VSB_putc(vsb, '"');
 	}
 	return (0);
 }
@@ -210,7 +218,7 @@ tweak_feature(struct vsb *vsb, const struct parspec *par, const char *arg)
 	unsigned j;
 	(void)par;
 
-	if (arg != NULL) {
+	if (arg != NULL && arg != JSON_FMT) {
 		if (!strcmp(arg, "none")) {
 			memset(mgt_param.feature_bits,
 			    0, sizeof mgt_param.feature_bits);
@@ -220,6 +228,8 @@ tweak_feature(struct vsb *vsb, const struct parspec *par, const char *arg)
 			    "feature bit", "+"));
 		}
 	} else {
+		if (arg == JSON_FMT)
+			VSB_putc(vsb, '"');
 		s = "";
 		for (j = 0; j < (unsigned)FEATURE_Reserved; j++) {
 			if (bit(mgt_param.feature_bits, j, BTST)) {
@@ -229,6 +239,8 @@ tweak_feature(struct vsb *vsb, const struct parspec *par, const char *arg)
 		}
 		if (*s == '\0')
 			VSB_printf(vsb, "none");
+		if (arg == JSON_FMT)
+			VSB_putc(vsb, '"');
 	}
 	return (0);
 }
diff --git a/bin/varnishd/mgt/mgt_param_tweak.c b/bin/varnishd/mgt/mgt_param_tweak.c
index 0f95237c2..01cab5f6b 100644
--- a/bin/varnishd/mgt/mgt_param_tweak.c
+++ b/bin/varnishd/mgt/mgt_param_tweak.c
@@ -43,6 +43,8 @@
 #include "vav.h"
 #include "vnum.h"
 
+const char * const JSON_FMT = (const char *)&JSON_FMT;
+
 /*--------------------------------------------------------------------
  * Generic handling of double typed parameters
  */
@@ -53,7 +55,7 @@ tweak_generic_double(struct vsb *vsb, volatile double *dest,
 {
 	volatile double u, minv = 0, maxv = 0;
 
-	if (arg != NULL) {
+	if (arg != NULL && arg != JSON_FMT) {
 		if (min != NULL) {
 			minv = VNUM(min);
 			if (isnan(minv)) {
@@ -123,7 +125,7 @@ tweak_bool(struct vsb *vsb, const struct parspec *par, const char *arg)
 	volatile unsigned *dest;
 
 	dest = par->priv;
-	if (arg != NULL) {
+	if (arg != NULL && arg != JSON_FMT) {
 		if (!strcasecmp(arg, "off"))
 			*dest = 0;
 		else if (!strcasecmp(arg, "disable"))
@@ -144,6 +146,8 @@ tweak_bool(struct vsb *vsb, const struct parspec *par, const char *arg)
 			VSB_printf(vsb, "use \"on\" or \"off\"\n");
 			return (-1);
 		}
+	} else if (arg == JSON_FMT) {
+		VSB_printf(vsb, "%s", *dest ? "true" : "false");
 	} else {
 		VSB_printf(vsb, "%s", *dest ? "on" : "off");
 	}
@@ -159,7 +163,7 @@ tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg,
 	unsigned u, minv = 0, maxv = 0;
 	char *p;
 
-	if (arg != NULL) {
+	if (arg != NULL && arg != JSON_FMT) {
 		if (min != NULL) {
 			p = NULL;
 			minv = strtoul(min, &p, 0);
@@ -195,7 +199,7 @@ tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg,
 			return (-1);
 		}
 		*dest = u;
-	} else if (*dest == UINT_MAX) {
+	} else if (*dest == UINT_MAX && arg != JSON_FMT) {
 		VSB_printf(vsb, "unlimited");
 	} else {
 		VSB_printf(vsb, "%u", *dest);
@@ -246,7 +250,7 @@ tweak_generic_bytes(struct vsb *vsb, volatile ssize_t *dest, const char *arg,
 	uintmax_t r, rmin = 0, rmax = 0;
 	const char *p;
 
-	if (arg != NULL) {
+	if (arg != NULL && arg != JSON_FMT) {
 		if (min != NULL) {
 			p = VNUM_2bytes(min, &rmin, 0);
 			if (p != NULL) {
@@ -285,6 +289,8 @@ tweak_generic_bytes(struct vsb *vsb, volatile ssize_t *dest, const char *arg,
 			return (-1);
 		}
 		*dest = r;
+	} else if (arg == JSON_FMT) {
+		VSB_printf(vsb, "%zd", *dest);
 	} else {
 		fmt_bytes(vsb, *dest);
 	}
@@ -364,6 +370,8 @@ tweak_string(struct vsb *vsb, const struct parspec *par, const char *arg)
 	/* XXX should have tweak_generic_string */
 	if (arg == NULL) {
 		VSB_quote(vsb, *p, -1, 0);
+	} else if (arg == JSON_FMT) {
+		VSB_quote(vsb, *p, -1, VSB_QUOTE_JSON|VSB_QUOTE_CSTR);
 	} else {
 		REPLACE(*p, arg);
 	}
@@ -380,7 +388,15 @@ tweak_poolparam(struct vsb *vsb, const struct parspec *par, const char *arg)
 	int retval = 0;
 
 	pp = par->priv;
-	if (arg == NULL) {
+	if (arg == JSON_FMT) {
+		VSB_printf(vsb, "{\n");
+		VSB_indent(vsb, 8);
+		VSB_printf(vsb, "\"min_pool\": %u,\n", pp->min_pool);
+		VSB_printf(vsb, "\"max_pool\": %u,\n", pp->max_pool);
+		VSB_printf(vsb, "\"max_age\": %g\n", pp->max_age);
+		VSB_indent(vsb, -4);
+		VSB_printf(vsb, "}");
+	} else if (arg == NULL) {
 		VSB_printf(vsb, "%u,%u,%g",
 		    pp->min_pool, pp->max_pool, pp->max_age);
 	} else {
diff --git a/bin/varnishtest/tests/b00042.vtc b/bin/varnishtest/tests/b00042.vtc
index 9fa143e9a..6470a7260 100644
--- a/bin/varnishtest/tests/b00042.vtc
+++ b/bin/varnishtest/tests/b00042.vtc
@@ -33,3 +33,10 @@ varnish v1 -clierr "106" {param.show fofofofo}
 varnish v1 -cliok "param.show changed"
 varnish v1 -cliok "param.show "
 varnish v1 -cliok "param.show -l"
+
+varnish v1 -clijson "param.show -j pool_req"
+varnish v1 -clijson "param.show -j pool_sess"
+varnish v1 -clijson "param.show -j changed"
+varnish v1 -clijson "param.show -j"
+varnish v1 -clierr "106" "param.show -j -l"
+varnish v1 -clierr "106" "param.show -j fofofofo"

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:29 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:29 +0000 (UTC)
Subject: [6.0] 2776c9c73 Add JSON support for the "ban.list" CLI command.
Message-ID: <20181031130829.1323A9588B@lists.varnish-cache.org>


commit 2776c9c73fda55ebb5ecff6f49dde5cb6ae979c9
Author: Geoff Simmons <geoff at uplex.de>
Date:   Tue Sep 25 09:04:42 2018 +0200

    Add JSON support for the "ban.list" CLI command.

diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index 6ec3577d2..fab8661cf 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -669,7 +669,7 @@ ccf_ban(struct cli *cli, const char * const *av, void *priv)
 }
 
 static void
-ban_render(struct cli *cli, const uint8_t *bs)
+ban_render(struct cli *cli, const uint8_t *bs, int quote)
 {
 	struct ban_test bt;
 	const uint8_t *be;
@@ -704,27 +704,21 @@ ban_render(struct cli *cli, const uint8_t *bs)
 		default:
 			WRONG("Wrong BANS_OPER");
 		}
-		VCLI_Out(cli, "%s", bt.arg2);
+		if (quote)
+			VCLI_Quote(cli, bt.arg2);
+		else
+			VCLI_Out(cli, "%s", bt.arg2);
 		if (bs < be)
 			VCLI_Out(cli, " && ");
 	}
 }
 
-static void v_matchproto_(cli_func_t)
-ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
+static void
+ban_list(struct cli *cli, struct ban *bl)
 {
-	struct ban *b, *bl;
+	struct ban *b;
 	int64_t o;
 
-	(void)av;
-	(void)priv;
-
-	/* Get a reference so we are safe to traverse the list */
-	Lck_Lock(&ban_mtx);
-	bl = VTAILQ_LAST(&ban_head, banhead_s);
-	bl->refcount++;
-	Lck_Unlock(&ban_mtx);
-
 	VCLI_Out(cli, "Present bans:\n");
 	VTAILQ_FOREACH(b, &ban_head, list) {
 		o = bl == b ? 1 : 0;
@@ -738,7 +732,7 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
 			    b);
 		}
 		VCLI_Out(cli, "  ");
-		ban_render(cli, b->spec);
+		ban_render(cli, b->spec, 0);
 		VCLI_Out(cli, "\n");
 		if (VCLI_Overflow(cli))
 			break;
@@ -750,6 +744,80 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
 			Lck_Unlock(&ban_mtx);
 		}
 	}
+}
+
+static void
+ban_list_json(struct cli *cli, const char * const *av, struct ban *bl)
+{
+	struct ban *b;
+	int64_t o;
+	int n = 0;
+	int ocs;
+
+	VCLI_JSON_begin(cli, 2, av);
+	VCLI_Out(cli, ",\n");
+	VTAILQ_FOREACH(b, &ban_head, list) {
+		o = bl == b ? 1 : 0;
+		VCLI_Out(cli, "%s", n ? ",\n" : "");
+		n++;
+		VCLI_Out(cli, "{\n");
+		VSB_indent(cli->sb, 2);
+		VCLI_Out(cli, "\"time\": %.6f,\n", ban_time(b->spec));
+		VCLI_Out(cli, "\"refs\": %ju,\n", (intmax_t)(b->refcount - o));
+		VCLI_Out(cli, "\"completed\": %s,\n",
+			 b->flags & BANS_FLAG_COMPLETED ? "true" : "false");
+		VCLI_Out(cli, "\"spec\": \"");
+		ban_render(cli, b->spec, 1);
+		VCLI_Out(cli, "\"");
+
+		if (DO_DEBUG(DBG_LURKER)) {
+			VCLI_Out(cli, ",\n");
+			VCLI_Out(cli, "\"req_tests\": %s,\n",
+				 b->flags & BANS_FLAG_REQ ? "true" : "false");
+			VCLI_Out(cli, "\"obj_tests\": %s,\n",
+				 b->flags & BANS_FLAG_OBJ ? "true" : "false");
+			VCLI_Out(cli, "\"pointer\": \"%p\",\n", b);
+			if (VCLI_Overflow(cli))
+				break;
+
+			ocs = 0;
+			VCLI_Out(cli, "\"objcores\": [\n");
+			VSB_indent(cli->sb, 2);
+			Lck_Lock(&ban_mtx);
+			struct objcore *oc;
+			VTAILQ_FOREACH(oc, &b->objcore, ban_list) {
+				if (ocs)
+					VCLI_Out(cli, ",\n");
+				VCLI_Out(cli, "%p", oc);
+				ocs++;
+			}
+			Lck_Unlock(&ban_mtx);
+			VSB_indent(cli->sb, -2);
+			VCLI_Out(cli, "\n]");
+		}
+		VSB_indent(cli->sb, -2);
+		VCLI_Out(cli, "\n}");
+	}
+	VCLI_JSON_end(cli);
+}
+
+static void v_matchproto_(cli_func_t)
+ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
+{
+	struct ban *bl;
+
+	(void)priv;
+
+	/* Get a reference so we are safe to traverse the list */
+	Lck_Lock(&ban_mtx);
+	bl = VTAILQ_LAST(&ban_head, banhead_s);
+	bl->refcount++;
+	Lck_Unlock(&ban_mtx);
+
+	if (av[2] != NULL && strcmp(av[2], "-j") == 0)
+		ban_list_json(cli, av, bl);
+	else
+		ban_list(cli, bl);
 
 	Lck_Lock(&ban_mtx);
 	bl->refcount--;
@@ -759,7 +827,8 @@ ccf_ban_list(struct cli *cli, const char * const *av, void *priv)
 
 static struct cli_proto ban_cmds[] = {
 	{ CLICMD_BAN,				"", ccf_ban },
-	{ CLICMD_BAN_LIST,			"", ccf_ban_list },
+	{ CLICMD_BAN_LIST,			"", ccf_ban_list,
+	  ccf_ban_list },
 	{ NULL }
 };
 
diff --git a/bin/varnishtest/tests/c00019.vtc b/bin/varnishtest/tests/c00019.vtc
index 22b5ca1bf..3a4a3b2f8 100644
--- a/bin/varnishtest/tests/c00019.vtc
+++ b/bin/varnishtest/tests/c00019.vtc
@@ -84,7 +84,13 @@ client c1 {
 varnish v1 -expect bans_tested == 2
 varnish v1 -expect bans_tests_tested == 2
 varnish v1 -cliok "ban.list"
+varnish v1 -clijson "ban.list -j"
 
 # Test a bogus regexp
-
 varnish v1 -clierr 106 "ban req.url ~ [[["
+
+# Ban expression with quoting
+varnish v1 -cliok {ban req.url ~ "BAR"}
+shell {varnishadm -n ${tmpdir}/v1 ban 'obj.http.Host ~ \"Foo\"'}
+varnish v1 -cliok "ban.list"
+varnish v1 -clijson "ban.list -j"
diff --git a/bin/varnishtest/tests/c00021.vtc b/bin/varnishtest/tests/c00021.vtc
index c4a5ef5cf..d1aff7875 100644
--- a/bin/varnishtest/tests/c00021.vtc
+++ b/bin/varnishtest/tests/c00021.vtc
@@ -112,6 +112,7 @@ client c1 {
 # header check, no header
 varnish v1 -cliok "ban req.url ~ foo && obj.http.bar == barcheck"
 varnish v1 -cliok "ban.list"
+varnish v1 -clijson "ban.list -j"
 
 client c1 {
 	txreq -url "/foo"
diff --git a/bin/varnishtest/tests/c00022.vtc b/bin/varnishtest/tests/c00022.vtc
index e2fa3e727..79a034fb2 100644
--- a/bin/varnishtest/tests/c00022.vtc
+++ b/bin/varnishtest/tests/c00022.vtc
@@ -149,6 +149,7 @@ client c1 {
 	expect resp.status == 410
 } -run
 varnish v1 -cliok "ban.list"
+varnish v1 -clijson "ban.list -j"
 
 client c1 {
 	txreq -url "/foo"
diff --git a/bin/varnishtest/tests/c00049.vtc b/bin/varnishtest/tests/c00049.vtc
index 9ff05d7f0..a442833fa 100644
--- a/bin/varnishtest/tests/c00049.vtc
+++ b/bin/varnishtest/tests/c00049.vtc
@@ -205,6 +205,7 @@ varnish v1 -cliok "ban.list"
 varnish v1 -cliok "param.set ban_lurker_age .1"
 delay 3
 
+varnish v1 -clijson "ban.list -j"
 varnish v1 -cliok "ban.list"
 
 varnish v1 -expect bans == 1
diff --git a/bin/varnishtest/tests/c00059.vtc b/bin/varnishtest/tests/c00059.vtc
index db0e6d29c..156bff35c 100644
--- a/bin/varnishtest/tests/c00059.vtc
+++ b/bin/varnishtest/tests/c00059.vtc
@@ -18,6 +18,7 @@ client c1 {
 varnish v1 -cliok "ban obj.status == 201"
 varnish v1 -cliok "ban obj.status == 200"
 varnish v1 -cliok "ban.list"
+varnish v1 -clijson "ban.list -j"
 
 client c1 {
 	txreq

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:29 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:29 +0000 (UTC)
Subject: [6.0] 95e9cfe6b Add JSON support for the "storage.list" CLI command.
Message-ID: <20181031130829.34E5A9589F@lists.varnish-cache.org>


commit 95e9cfe6b95bc641e1d0a67613b0c61bc77d462c
Author: Geoff Simmons <geoff at uplex.de>
Date:   Tue Sep 25 09:15:06 2018 +0200

    Add JSON support for the "storage.list" CLI command.

diff --git a/bin/varnishd/storage/mgt_stevedore.c b/bin/varnishd/storage/mgt_stevedore.c
index 3f66c3af3..c204a3e53 100644
--- a/bin/varnishd/storage/mgt_stevedore.c
+++ b/bin/varnishd/storage/mgt_stevedore.c
@@ -82,10 +82,36 @@ stv_cli_list(struct cli *cli, const char * const *av, void *priv)
 		VCLI_Out(cli, "\tstorage.%s = %s\n", stv->ident, stv->name);
 }
 
+static void v_matchproto_(cli_func_t)
+stv_cli_list_json(struct cli *cli, const char * const *av, void *priv)
+{
+	struct stevedore *stv;
+	int n = 0;
+
+	(void)priv;
+	ASSERT_MGT();
+	VCLI_JSON_begin(cli, 2, av);
+	VCLI_Out(cli, ",\n");
+	STV_Foreach(stv) {
+		VCLI_Out(cli, "%s", n ? ",\n" : "");
+		n++;
+		VCLI_Out(cli, "{\n");
+		VSB_indent(cli->sb, 2);
+		VCLI_Out(cli, "\"name\": ");
+		VCLI_JSON_str(cli, stv->ident);
+		VCLI_Out(cli, ",\n");
+		VCLI_Out(cli, "\"storage\": ");
+		VCLI_JSON_str(cli, stv->name);
+		VSB_indent(cli->sb, -2);
+		VCLI_Out(cli, "\n}");
+	}
+	VCLI_JSON_end(cli);
+}
+
 /*--------------------------------------------------------------------*/
 
 static struct cli_proto cli_stv[] = {
-	{ CLICMD_STORAGE_LIST,		"", stv_cli_list },
+	{ CLICMD_STORAGE_LIST,		"", stv_cli_list, stv_cli_list_json },
 	{ NULL}
 };
 
diff --git a/bin/varnishtest/tests/b00032.vtc b/bin/varnishtest/tests/b00032.vtc
index 7aa6cddfa..d395fef69 100644
--- a/bin/varnishtest/tests/b00032.vtc
+++ b/bin/varnishtest/tests/b00032.vtc
@@ -1,6 +1,7 @@
 varnishtest "CLI coverage test"
 
 varnish v1 -cliok storage.list
+varnish v1 -clijson "storage.list -j"
 
 server s1 {
 	rxreq

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:29 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:29 +0000 (UTC)
Subject: [6.0] 128fd4000 Add JSON support for the "panic.show" CLI command.
Message-ID: <20181031130829.63E3F958AE@lists.varnish-cache.org>


commit 128fd40008b1f1ddd9b2b78bc850c9a407dcfdd6
Author: Geoff Simmons <geoff at uplex.de>
Date:   Tue Sep 25 09:47:57 2018 +0200

    Add JSON support for the "panic.show" CLI command.
    
    The panic string is added as a JSON string in the output array.

diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 026bdd8e8..16050b33a 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -117,12 +117,9 @@ mgt_panic_clear(void)
 	VSB_destroy(&child_panic);
 }
 
-static void v_matchproto_(cli_func_t)
-mch_cli_panic_show(struct cli *cli, const char * const *av, void *priv)
+static void
+cli_panic_show(struct cli *cli, const char * const *av, int json)
 {
-	(void)av;
-	(void)priv;
-
 	if (!child_panic) {
 		VCLI_SetResult(cli, CLIS_CANT);
 		VCLI_Out(cli,
@@ -130,7 +127,29 @@ mch_cli_panic_show(struct cli *cli, const char * const *av, void *priv)
 		return;
 	}
 
-	VCLI_Out(cli, "%s\n", VSB_data(child_panic));
+	if (!json) {
+		VCLI_Out(cli, "%s\n", VSB_data(child_panic));
+		return;
+	}
+
+	VCLI_JSON_begin(cli, 2, av);
+	VCLI_Out(cli, ",\n");
+	VCLI_JSON_str(cli, VSB_data(child_panic));
+	VCLI_JSON_end(cli);
+}
+
+static void v_matchproto_(cli_func_t)
+mch_cli_panic_show(struct cli *cli, const char * const *av, void *priv)
+{
+	(void)priv;
+	cli_panic_show(cli, av, 0);
+}
+
+static void v_matchproto_(cli_func_t)
+mch_cli_panic_show_json(struct cli *cli, const char * const *av, void *priv)
+{
+	(void)priv;
+	cli_panic_show(cli, av, 1);
 }
 
 static void v_matchproto_(cli_func_t)
@@ -694,7 +713,8 @@ static struct cli_proto cli_mch[] = {
 	  mch_cli_server_status_json },
 	{ CLICMD_SERVER_START,		"", mch_cli_server_start },
 	{ CLICMD_SERVER_STOP,		"", mch_cli_server_stop },
-	{ CLICMD_PANIC_SHOW,		"", mch_cli_panic_show },
+	{ CLICMD_PANIC_SHOW,		"", mch_cli_panic_show,
+	  mch_cli_panic_show_json },
 	{ CLICMD_PANIC_CLEAR,		"", mch_cli_panic_clear },
 	{ NULL }
 };
diff --git a/bin/varnishtest/tests/c00057.vtc b/bin/varnishtest/tests/c00057.vtc
index 2b2f38891..5118c79a0 100644
--- a/bin/varnishtest/tests/c00057.vtc
+++ b/bin/varnishtest/tests/c00057.vtc
@@ -44,6 +44,7 @@ client c1 {
 } -run
 
 varnish v1 -cliexpect "STACK OVERFLOW" "panic.show"
+varnish v1 -clijson "panic.show -j"
 
 varnish v1 -cliok "panic.clear"
 
@@ -81,6 +82,7 @@ client c2 -connect ${v2_sock} {
 } -run
 
 varnish v2 -cliexpect "[bB]us error|Segmentation [fF]ault" "panic.show"
+varnish v2 -clijson "panic.show -j"
 
 varnish v2 -cliok "panic.clear"
 
diff --git a/bin/varnishtest/tests/v00010.vtc b/bin/varnishtest/tests/v00010.vtc
index b948d0187..c488f377f 100644
--- a/bin/varnishtest/tests/v00010.vtc
+++ b/bin/varnishtest/tests/v00010.vtc
@@ -54,6 +54,7 @@ client c1 {
 
 varnish v1 -wait-stopped
 varnish v1 -cliok "panic.show"
+varnish v1 -clijson "panic.show -j"
 varnish v1 -cliok "panic.clear"
 varnish v1 -expect MGT.child_panic == 1
 varnish v1 -clierr 300 "panic.clear"
@@ -70,6 +71,7 @@ client c1 {
 
 varnish v1 -wait-stopped
 varnish v1 -cliok "panic.show"
+varnish v1 -clijson "panic.show -j"
 varnish v1 -cliok "panic.clear -z"
 varnish v1 -expect MGT.child_panic == 0
 varnish v1 -clierr 300 "panic.clear"

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:29 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:29 +0000 (UTC)
Subject: [6.0] 5d501ce11 Document JSON support in the CLI.
Message-ID: <20181031130829.7F01E958B7@lists.varnish-cache.org>


commit 5d501ce1192d0b58b74ac492fa4fb1df65fd7feb
Author: Geoff Simmons <geoff at uplex.de>
Date:   Tue Sep 25 11:17:23 2018 +0200

    Document JSON support in the CLI.

diff --git a/include/tbl/cli_cmds.h b/include/tbl/cli_cmds.h
index 7b3d1de19..25f13d4ac 100644
--- a/include/tbl/cli_cmds.h
+++ b/include/tbl/cli_cmds.h
@@ -49,10 +49,11 @@ CLI_CMD(BAN,
 
 CLI_CMD(BAN_LIST,
 	"ban.list",
-	"ban.list",
+	"ban.list [-j]",
 	"List the active bans.",
 
-	"  The output format is:\n\n"
+	" Unless ``-j`` is specified (for JSON output), "
+	" the output format is:\n\n"
 	"  * Time the ban was issued.\n\n"
 	"  * Objects referencing this ban.\n\n"
 	"  * ``C`` if ban is completed = no further testing against it.\n\n"
@@ -102,9 +103,9 @@ CLI_CMD(VCL_DISCARD,
 
 CLI_CMD(VCL_LIST,
 	"vcl.list",
-	"vcl.list",
+	"vcl.list [-j]",
 	"List all loaded configuration.",
-	"",
+	"``-j`` specifies JSON output.",
 	0, 0
 )
 
@@ -134,12 +135,14 @@ CLI_CMD(VCL_LABEL,
 
 CLI_CMD(PARAM_SHOW,
 	"param.show",
-	"param.show [-l] [<param>|changed]",
+	"param.show [-l|-j] [<param>|changed]",
 	"Show parameters and their values.",
 
 	"The long form with ``-l`` shows additional information, including"
 	" documentation and minimum, maximum and default values, if defined"
-	" for the parameter. If a parameter is specified with ``<param>``,"
+	" for the parameter. JSON output is specified with ``-j``, in which"
+	" the information for the long form is included; only one of ``-l`` or"
+	" ``-j`` is permitted. If a parameter is specified with ``<param>``,"
 	" show only that parameter. If ``changed`` is specified, show only"
 	" those parameters whose values differ from their defaults.",
 	0, 2
@@ -171,17 +174,17 @@ CLI_CMD(SERVER_START,
 
 CLI_CMD(PING,
 	"ping",
-	"ping [<timestamp>]",
+	"ping [-j] [<timestamp>]",
 	"Keep connection alive.",
-	"",
+	"The response is formatted as JSON if ``-j`` is specified.",
 	0, 1
 )
 
 CLI_CMD(HELP,
 	"help",
-	"help [<command>]",
+	"help [-j] [<command>]",
 	"Show command/protocol help.",
-	"",
+	"``-j`` specifies JSON output.",
 	0, 1
 )
 
@@ -195,9 +198,9 @@ CLI_CMD(QUIT,
 
 CLI_CMD(SERVER_STATUS,
 	"status",
-	"status",
+	"status [-j]",
 	"Check status of Varnish cache process.",
-	"",
+	"``-j`` specifies JSON output.",
 	0, 0
 )
 
@@ -219,9 +222,10 @@ CLI_CMD(AUTH,
 
 CLI_CMD(PANIC_SHOW,
 	"panic.show",
-	"panic.show",
+	"panic.show [-j]",
 	"Return the last panic, if any.",
-	"",
+	"``-j`` specifies JSON output -- the panic message is returned as an"
+	" unstructured JSON string.",
 	0, 0
 )
 
@@ -244,9 +248,9 @@ CLI_CMD(DEBUG_LISTEN_ADDRESS,
 
 CLI_CMD(BACKEND_LIST,
 	"backend.list",
-	"backend.list [-p] [<backend_pattern>]",
+	"backend.list [-j] [-p] [<backend_pattern>]",
 	"List backends.  -p also shows probe status.",
-	"",
+	"``-j`` specifies JSON output.",
 	0, 2
 )
 
@@ -320,9 +324,9 @@ CLI_CMD(DEBUG_PERSISTENT,
 
 CLI_CMD(STORAGE_LIST,
 	"storage.list",
-	"storage.list",
+	"storage.list [-j]",
 	"List storage devices.",
-	"",
+	"``-j`` specifies JSON output.",
 	0, 0
 )
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:29 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:29 +0000 (UTC)
Subject: [6.0] 697462f47 Add a JSON section to varnish-cli(7).
Message-ID: <20181031130829.AF0F1958CB@lists.varnish-cache.org>


commit 697462f47c3c7bd9cea64fb6e62c86515bf54c1e
Author: Geoff Simmons <geoff at uplex.de>
Date:   Tue Oct 2 14:12:02 2018 +0200

    Add a JSON section to varnish-cli(7).
    
    Closes #2783

diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst
index 53387672a..38364cede 100644
--- a/doc/sphinx/reference/varnish-cli.rst
+++ b/doc/sphinx/reference/varnish-cli.rst
@@ -170,6 +170,38 @@ Other pitfalls include variable expansion of the shell invoking ``varnishadm``
 but this is not directly related to the Varnish CLI. If you get the quoting
 right you should be fine even with complex commands.
 
+JSON
+----
+
+A number of commands with informational responses support a ``-j`` parameter
+for JSON output, as specified below. The top-level structure of the JSON
+response is an array with these first three elements:
+
+* A version number for the JSON format (integer)
+
+* An array of strings that comprise the CLI command just received
+
+* The time at which the response was generated, as a Unix epoch time
+  in seconds with millisecond precision (floating point)
+
+The remaining elements of the array form the data that are specific to
+the CLI command, and their structure and content depend on the
+command.
+
+For example, the response to ``status -j`` just contains a string in
+the top-level array indicating the state of the child process
+(``"running"``, ``"stopped"`` and so forth)::
+
+  [ 2, ["status", "-j"], 1538031732.632, "running"
+  ]
+
+The JSON responses to other commands may have longer lists of
+elements, which may have simple data types or form structured objects.
+
+JSON output is only returned if command execution was successful. The
+output for an error response is always the same as it would have been
+for the command without the ``-j`` parameter.
+
 Commands
 --------
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:29 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:29 +0000 (UTC)
Subject: [6.0] 3fec64b6b typedefs for real/mono time and durations
Message-ID: <20181031130829.E23AC958D1@lists.varnish-cache.org>


commit 3fec64b6b458ad491363a88f189a00e92e4aa6eb
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 8 10:19:50 2018 +0200

    typedefs for real/mono time and durations
    
    We use double for all time representations, yet monotonic time,
    real time and durations are not to be confused.
    
    Also, we might want to change the representation of time in the
    future.
    
    To get an implicit documentation of the semantic type of a time
    value and to facilitate working on the latter, we start off by
    introducing simple typedefs which neither inply any change nor
    offer any type checking. But they pave the way...

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 2f14cefa9..a6d4f514c 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -252,7 +252,7 @@ struct worker {
 
 	struct pool_task	task;
 
-	double			lastused;
+	vtim_real		lastused;
 	int			strangelove;
 
 	struct v1l		*v1l;
@@ -413,9 +413,9 @@ struct busyobj {
 #include "tbl/bo_flags.h"
 
 	/* Timeouts */
-	double			connect_timeout;
-	double			first_byte_timeout;
-	double			between_bytes_timeout;
+	vtim_dur		connect_timeout;
+	vtim_dur		first_byte_timeout;
+	vtim_dur		between_bytes_timeout;
 
 	/* Timers */
 	double			t_first;	/* First timestamp logged */
@@ -564,9 +564,8 @@ struct sess {
 
 	struct ws		ws[1];
 
-	/* Timestamps, all on TIM_real() timescale */
-	double			t_open;		/* fd accepted */
-	double			t_idle;		/* fd accepted or resp sent */
+	vtim_real		t_open;		/* fd accepted */
+	vtim_real		t_idle;		/* fd accepted or resp sent */
 
 };
 
@@ -662,7 +661,7 @@ int Lck__Owned(const struct lock *lck);
 
 /* public interface: */
 void Lck_Delete(struct lock *lck);
-int Lck_CondWait(pthread_cond_t *cond, struct lock *lck, double);
+int Lck_CondWait(pthread_cond_t *cond, struct lock *lck, vtim_real);
 
 #define Lck_New(a, b) Lck__New(a, b, #b)
 #define Lck_Lock(a) Lck__Lock(a, __func__, __LINE__)
diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 908613f47..b05a53a59 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -52,7 +52,7 @@
 #include "vtim.h"
 
 static pthread_t	VCA_thread;
-static double vca_pace = 0.0;
+static vtim_dur vca_pace = 0.0;
 static struct lock pace_mtx;
 static unsigned pool_accepting;
 static pthread_mutex_t shut_mtx = PTHREAD_MUTEX_INITIALIZER;
@@ -137,9 +137,9 @@ static unsigned		need_test;
  * into the vca_acct() loop which we are running anyway
  */
 static void
-vca_periodic(double t0)
+vca_periodic(vtim_real t0)
 {
-	double now;
+	vtim_real now;
 
 	now = VTIM_real();
 	VSC_C_main->uptime = (uint64_t)(now - t0);
@@ -270,7 +270,7 @@ vca_tcp_opt_set(const int sock, const unsigned uds, const int force)
 static void
 vca_pace_check(void)
 {
-	double p;
+	vtim_dur p;
 
 	if (vca_pace == 0.0)
 		return;
@@ -584,7 +584,7 @@ static void * v_matchproto_()
 vca_acct(void *arg)
 {
 	struct listen_sock *ls;
-	double t0;
+	vtim_real t0;
 
 	// XXX Actually a mis-nomer now because the accept happens in a pool
 	// thread. Rename to accept-nanny or so?
diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 6ad610f5d..5a17db15a 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -114,7 +114,7 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo,
 {
 	struct pfd *pfd;
 	int *fdp, err;
-	double tmod;
+	vtim_dur tmod;
 	char abuf1[VTCP_ADDRBUFSIZE], abuf2[VTCP_ADDRBUFSIZE];
 	char pbuf1[VTCP_PORTBUFSIZE], pbuf2[VTCP_PORTBUFSIZE];
 
@@ -593,7 +593,7 @@ void
 VBE_Poll(void)
 {
 	struct backend *be, *be2;
-	double now = VTIM_real();
+	vtim_real now = VTIM_real();
 
 	ASSERT_CLI();
 	Lck_Lock(&backends_mtx);
diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h
index 9d88cd2b6..f1e708f43 100644
--- a/bin/varnishd/cache/cache_backend.h
+++ b/bin/varnishd/cache/cache_backend.h
@@ -66,7 +66,7 @@ struct backend {
 
 	struct director		director[1];
 
-	double			cooled;
+	vtim_real		cooled;
 };
 
 /*---------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c
index 31fa31773..f632af865 100644
--- a/bin/varnishd/cache/cache_backend_probe.c
+++ b/bin/varnishd/cache/cache_backend_probe.c
@@ -77,11 +77,11 @@ struct vbp_target {
 #define BITMAP(n, c, t, b)	uintmax_t	n;
 #include "tbl/backend_poll.h"
 
-	double				last;
-	double				avg;
+	vtim_dur			last;
+	vtim_dur			avg;
 	double				rate;
 
-	double				due;
+	vtim_real			due;
 	int				running;
 	int				heap_idx;
 	struct pool_task		task;
@@ -274,7 +274,7 @@ static void
 vbp_poke(struct vbp_target *vt)
 {
 	int s, tmo, i, proxy_header, err;
-	double t_start, t_now, t_end;
+	vtim_real t_start, t_now, t_end;
 	unsigned rlen, resp;
 	char buf[8192], *p;
 	struct pollfd pfda[1], *pfd = pfda;
@@ -450,7 +450,7 @@ vbp_task(struct worker *wrk, void *priv)
 static void * v_matchproto_(bgthread_t)
 vbp_thread(struct worker *wrk, void *priv)
 {
-	double now, nxt;
+	vtim_real now, nxt;
 	struct vbp_target *vt;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c
index 6fa816a68..ad182c4b0 100644
--- a/bin/varnishd/cache/cache_lck.c
+++ b/bin/varnishd/cache/cache_lck.c
@@ -187,7 +187,7 @@ Lck__Owned(const struct lock *lck)
 }
 
 int v_matchproto_()
-Lck_CondWait(pthread_cond_t *cond, struct lock *lck, double when)
+Lck_CondWait(pthread_cond_t *cond, struct lock *lck, vtim_real when)
 {
 	struct ilck *ilck;
 	struct timespec ts;
diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c
index f33668b2f..5a04eb5aa 100644
--- a/bin/varnishd/cache/cache_tcp_pool.c
+++ b/bin/varnishd/cache/cache_tcp_pool.c
@@ -540,7 +540,7 @@ VCP_Get(struct conn_pool *cp, double tmo, struct worker *wrk,
  */
 
 static int
-VCP_Wait(struct worker *wrk, struct pfd *pfd, double tmo)
+VCP_Wait(struct worker *wrk, struct pfd *pfd, vtim_real tmo)
 {
 	struct conn_pool *cp;
 	int r;
@@ -834,7 +834,7 @@ VTP_Close(struct pfd **pfdp)
  */
 
 struct pfd *
-VTP_Get(struct tcp_pool *tp, double tmo, struct worker *wrk,
+VTP_Get(struct tcp_pool *tp, vtim_dur tmo, struct worker *wrk,
 	unsigned force_fresh, int *err)
 {
 
@@ -845,7 +845,7 @@ VTP_Get(struct tcp_pool *tp, double tmo, struct worker *wrk,
  */
 
 int
-VTP_Wait(struct worker *wrk, struct pfd *pfd, double tmo)
+VTP_Wait(struct worker *wrk, struct pfd *pfd, vtim_real tmo)
 {
 	return (VCP_Wait(wrk, pfd, tmo));
 }
diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h
index 7c25afda2..57564ba96 100644
--- a/bin/varnishd/cache/cache_tcp_pool.h
+++ b/bin/varnishd/cache/cache_tcp_pool.h
@@ -88,14 +88,14 @@ void VTP_Recycle(const struct worker *, struct pfd **);
 	 * Recycle an open connection.
 	 */
 
-struct pfd *VTP_Get(struct tcp_pool *, double tmo, struct worker *,
+struct pfd *VTP_Get(struct tcp_pool *, vtim_dur tmo, struct worker *,
     unsigned force_fresh, int *err);
 	/*
 	 * Get a (possibly) recycled connection.
 	 * errno will be stored in err
 	 */
 
-int VTP_Wait(struct worker *, struct pfd *, double tmo);
+int VTP_Wait(struct worker *, struct pfd *, vtim_real tmo);
 	/*
 	 * If the connection was recycled (state != VTP_STATE_USED) call this
 	 * function before attempting to receive on the connection.
diff --git a/include/vdef.h b/include/vdef.h
index 185265794..3a5e40329 100644
--- a/include/vdef.h
+++ b/include/vdef.h
@@ -165,3 +165,8 @@
 #   define ___Static_assert(x, y) \
 		typedef char __assert_## y[(x) ? 1 : -1] v_unused_
 #endif
+
+/* VTIM API overhaul WIP */
+typedef double vtim_mono;
+typedef double vtim_real;
+typedef double vtim_dur;
diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index c774e2ab8..14b9752d6 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -124,7 +124,7 @@ init(void)
  * gethrtime(), which is the case on modern Solaris descendents.
  */
 
-double
+vtim_mono
 VTIM_mono(void)
 {
 #if defined(HAVE_CLOCK_GETTIME) && !defined(USE_GETHRTIME)
@@ -143,7 +143,7 @@ VTIM_mono(void)
 #endif
 }
 
-double
+vtim_real
 VTIM_real(void)
 {
 #ifdef HAVE_CLOCK_GETTIME
@@ -160,7 +160,7 @@ VTIM_real(void)
 }
 
 void
-VTIM_format(double t, char *p)
+VTIM_format(vtim_real t, char *p)
 {
 	struct tm tm;
 	time_t tt;
@@ -236,10 +236,10 @@ VTIM_format(double t, char *p)
 		DIGIT(1, sec);					\
 	} while(0)
 
-double
+vtim_real
 VTIM_parse(const char *p)
 {
-	double t;
+	vtim_real t;
 	int month = 0, year = 0, weekday = -1, mday = 0;
 	int hour = 0, min = 0, sec = 0;
 	int d, leap;
@@ -394,7 +394,7 @@ VTIM_parse(const char *p)
 }
 
 void
-VTIM_sleep(double t)
+VTIM_sleep(vtim_dur t)
 {
 #ifdef HAVE_NANOSLEEP
 	struct timespec ts;
@@ -415,7 +415,7 @@ VTIM_sleep(double t)
 }
 
 struct timeval
-VTIM_timeval(double t)
+VTIM_timeval(vtim_real t)
 {
 	struct timeval tv;
 
@@ -426,7 +426,7 @@ VTIM_timeval(double t)
 }
 
 struct timespec
-VTIM_timespec(double t)
+VTIM_timespec(vtim_real t)
 {
 	struct timespec tv;
 
@@ -464,8 +464,9 @@ tst(const char *s, time_t good)
 	}
 }
 
+/* XXX keep as double for the time being */
 static int
-tst_delta_check(const char *name, double begin, double end, double ref)
+tst_delta_check(const char *name, double begin, double end, vtim_dur ref)
 {
 	const double tol_max = 1.1;
 	const double tol_min = 1;
@@ -487,9 +488,9 @@ tst_delta_check(const char *name, double begin, double end, double ref)
 static void
 tst_delta()
 {
-	double m_begin, m_end;
-	double r_begin, r_end;
-	const double ref = 1;
+	vtim_mono m_begin, m_end;
+	vtim_real r_begin, r_end;
+	const vtim_dur ref = 1;
 	int err = 0;
 
 	r_begin = VTIM_real();
@@ -510,53 +511,56 @@ tst_delta()
 static void
 bench()
 {
-	double s, e, t;
+	vtim_mono s, e;
+	vtim_mono t_m;
+	vtim_real t_r;
+	unsigned long t_i;
 	int i;
 	char buf[64];
 
-	t = 0;
-	s = VTIM_real();
+	t_r = 0;
+	s = VTIM_mono();
 	for (i=0; i<100000; i++)
-		t += VTIM_real();
-	e = VTIM_real();
+		t_r += VTIM_real();
+	e = VTIM_mono();
 	printf("real: %fs / %d = %fns - tst val %f\n",
-	    e - s, i, 1e9 * (e - s) / i, t);
+	    e - s, i, 1e9 * (e - s) / i, t_r);
 
-	t = 0;
-	s = VTIM_real();
+	t_i = 0;
+	s = VTIM_mono();
 	for (i=0; i<100000; i++)
-		t += VTIM_mono();
-	e = VTIM_real();
+		t_m += VTIM_mono();
+	e = VTIM_mono();
 	printf("mono: %fs / %d = %fns - tst val %f\n",
-	    e - s, i, 1e9 * (e - s) / i, t);
+	    e - s, i, 1e9 * (e - s) / i, t_m);
 
-	t = 0;
+	t_i = 0;
 	s = VTIM_mono();
 	for (i=0; i<100000; i++) {
 		snprintf(buf, sizeof(buf), "%.6f", s);
-		t += buf[4];
+		t_i += buf[4];
 	}
 	e = VTIM_mono();
-	printf("printf %%.6f: %fs / %d = %fns - tst val %f %s\n",
-	    e - s, i, 1e9 * (e - s) / i, t, buf);
+	printf("printf %%.6f: %fs / %d = %fns - tst val %lu %s\n",
+	    e - s, i, 1e9 * (e - s) / i, t_i, buf);
 
-	t = 0;
+	t_i = 0;
 	s = VTIM_mono();
 	for (i=0; i<100000; i++) {
 		snprintf(buf, sizeof(buf), "%ju.%06ju",
 		    (uint64_t)floor(s),
 		    (uint64_t)floor((s * 1e6)) % 1000000UL);
-		t += buf[4];
+		t_i += buf[4];
 	}
 	e = VTIM_mono();
-	printf("printf %%ju.%%06ju: %fs / %d = %fns - tst val %f %s\n",
-	    e - s, i, 1e9 * (e - s) / i, t, buf);
+	printf("printf %%ju.%%06ju: %fs / %d = %fns - tst val %lu %s\n",
+	    e - s, i, 1e9 * (e - s) / i, t_i, buf);
 }
 
 void
 parse_check(time_t t, const char *s)
 {
-	double tt;
+	vtim_real tt;
 	char buf[BUFSIZ];
 
 	tt = VTIM_parse(s);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:30 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:30 +0000 (UTC)
Subject: [6.0] 187ce707e Add python2/3 hack
Message-ID: <20181031130830.090CF958E3@lists.varnish-cache.org>


commit 187ce707efa4dbf0ce27973930587dc1c19080a7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 9 08:36:02 2018 +0000

    Add python2/3 hack

diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py
index eb795e076..f1e0d02ab 100644
--- a/lib/libvcc/vsctool.py
+++ b/lib/libvcc/vsctool.py
@@ -41,6 +41,7 @@ import getopt
 import json
 import sys
 import collections
+import codecs
 
 # Parameters of 'varnish_vsc_begin', first element is default
 TYPES = ["counter", "gauge", "bitmap"]
@@ -164,7 +165,12 @@ class CounterSet(object):
         assert self.completed
 
         fon = "VSC_" + self.name + ".h"
-        fo = open(fon, "w")
+        try:
+            # Python3
+            fo = open(fon, "w", encoding="UTF-8")
+        except TypeError:
+            # Python2
+            fo = open(fon, "w")
         genhdr(fo, self.name)
 
         fo.write(self.struct + " {\n")
@@ -288,7 +294,12 @@ class CounterSet(object):
         '''Emit .c file'''
         assert self.completed
         fon = "VSC_" + self.name + ".c"
-        fo = open(fon, "w")
+        try:
+            # Python3
+            fo = open(fon, "w", encoding="UTF-8")
+        except TypeError:
+            # Python2
+            fo = open(fon, "w")
         genhdr(fo, self.name)
         fo.write('#include "config.h"\n')
         fo.write('#include <stdio.h>\n')
@@ -437,10 +448,22 @@ def mainfunc(argv):
     rstfile = None
     for f, v in optlist:
         if f == '-r':
+            try:
+                # Python3
+                sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())
+            except AttributeError:
+                # Python2
+                pass
             rstfile = sys.stdout
 
     vscset = []
-    scs = open(args[0]).read().split("\n.. ")
+    try:
+        # Python3
+        f = open(args[0], encoding="UTF-8")
+    except TypeError:
+        # Python2
+        f = open(args[0])
+    scs = f.read().split("\n.. ")
     if rstfile:
         rstfile.write(scs[0])
     for i in scs[1:]:

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:30 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:30 +0000 (UTC)
Subject: [6.0] 4d1919a1a Renovate this Makefile
Message-ID: <20181031130830.2A220958F3@lists.varnish-cache.org>


commit 4d1919a1a7a4f8ecd7032d915f35420e54aec2f8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 9 09:05:24 2018 +0000

    Renovate this Makefile
    
    When producing files with "foo > file", always use the pattern:
    
            foo > file.tmp
            mv file.tmp file
    
    Otherwise program failures end up generating partial content and
    make will not even rerun the failing program next time you type make.
    
    Actually clean CLEANFILES in the clean target.
    
    The reference dir is not built, but it should be in the distfile

diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index 54f930caf..cf67a6106 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -25,7 +25,7 @@ help:
 	@echo "  doctest   to run all doctests embedded in the documentation (if enabled)"
 
 clean:
-	-rm -rf $(BUILDDIR)/*
+	-rm -rf $(BUILDDIR)/* $(CLEANFILES)
 
 # work around for make html called within doc/sphinx
 .PHONY: graphviz
@@ -101,6 +101,7 @@ EXTRA_DIST = \
 	installation \
 	phk \
 	tutorial \
+	reference \
 	users-guide \
 	vtc-syntax.py \
 	whats-new
@@ -116,11 +117,13 @@ distclean-local:
 	rm -rf $(BUILDDIR)
 
 include/cli.rst: $(top_builddir)/bin/varnishd/varnishd
-	$(top_builddir)/bin/varnishd/varnishd -x cli > $@
+	$(top_builddir)/bin/varnishd/varnishd -x cli > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES = include/cli.rst
 
 include/params.rst: $(top_builddir)/bin/varnishd/varnishd
-	$(top_builddir)/bin/varnishd/varnishd -x parameter > $@
+	$(top_builddir)/bin/varnishd/varnishd -x parameter > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/params.rst
 
 COUNTERS = \
@@ -134,52 +137,64 @@ COUNTERS = \
 	$(top_srcdir)/bin/varnishd/VSC_lck.vsc
 
 include/counters.rst: $(top_srcdir)/lib/libvcc/vsctool.py $(COUNTERS)
-	echo -n '' > $@
+	echo -n '' > ${@}_
 	for i in $(COUNTERS); do \
-		$(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -r $$i >> $@ ; \
+		$(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -r $$i >> ${@}_ ; \
 	done
+	mv ${@}_ ${@}
 
 BUILT_SOURCES += include/counters.rst
 
 # XXX add varnishstat here when it's been _opt2rst'ed
 
 include/varnishncsa_options.rst: $(top_builddir)/bin/varnishncsa/varnishncsa
-	$(top_builddir)/bin/varnishncsa/varnishncsa --options > $@
+	$(top_builddir)/bin/varnishncsa/varnishncsa --options > ${@}_
+	mv ${@}_ ${@}
 include/varnishncsa_synopsis.rst: $(top_builddir)/bin/varnishncsa/varnishncsa
-	$(top_builddir)/bin/varnishncsa/varnishncsa --synopsis > $@
+	$(top_builddir)/bin/varnishncsa/varnishncsa --synopsis > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/varnishncsa_options.rst \
 	include/varnishncsa_synopsis.rst
 
 include/varnishlog_options.rst: $(top_builddir)/bin/varnishlog/varnishlog
-	$(top_builddir)/bin/varnishlog/varnishlog --options > $@
+	$(top_builddir)/bin/varnishlog/varnishlog --options > ${@}_
+	mv ${@}_ ${@}
 include/varnishlog_synopsis.rst: $(top_builddir)/bin/varnishlog/varnishlog
-	$(top_builddir)/bin/varnishlog/varnishlog --synopsis > $@
+	$(top_builddir)/bin/varnishlog/varnishlog --synopsis > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/varnishlog_options.rst \
 	include/varnishlog_synopsis.rst
 
 include/varnishtop_options.rst: $(top_builddir)/bin/varnishtop/varnishtop
-	$(top_builddir)/bin/varnishtop/varnishtop --options > $@
+	$(top_builddir)/bin/varnishtop/varnishtop --options > ${@}_
+	mv ${@}_ ${@}
 include/varnishtop_synopsis.rst: $(top_builddir)/bin/varnishtop/varnishtop
-	$(top_builddir)/bin/varnishtop/varnishtop --synopsis > $@
+	$(top_builddir)/bin/varnishtop/varnishtop --synopsis > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/varnishtop_options.rst \
 	 include/varnishtop_synopsis.rst
 
 include/varnishhist_options.rst: $(top_builddir)/bin/varnishhist/varnishhist
-	$(top_builddir)/bin/varnishhist/varnishhist --options > $@
+	$(top_builddir)/bin/varnishhist/varnishhist --options > ${@}_
+	mv ${@}_ ${@}
 include/varnishhist_synopsis.rst: $(top_builddir)/bin/varnishhist/varnishhist
-	$(top_builddir)/bin/varnishhist/varnishhist --synopsis > $@
+	$(top_builddir)/bin/varnishhist/varnishhist --synopsis > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/varnishhist_options.rst \
 	 include/varnishhist_synopsis.rst
 
 include/varnishstat_options.rst: $(top_builddir)/bin/varnishstat/varnishstat
-	$(top_builddir)/bin/varnishstat/varnishstat --options > $@
+	$(top_builddir)/bin/varnishstat/varnishstat --options > ${@}_
+	mv ${@}_ ${@}
 include/varnishstat_synopsis.rst: $(top_builddir)/bin/varnishstat/varnishstat
-	$(top_builddir)/bin/varnishstat/varnishstat --synopsis > $@
+	$(top_builddir)/bin/varnishstat/varnishstat --synopsis > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/varnishstat_options.rst \
 	 include/varnishstat_synopsis.rst
 
 include/vsl-tags.rst: $(top_builddir)/lib/libvarnishapi/vsl2rst
-	$(top_builddir)/lib/libvarnishapi/vsl2rst > $@
+	$(top_builddir)/lib/libvarnishapi/vsl2rst > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/vsl-tags.rst
 
 VTCSYN_SRC = $(top_srcdir)/bin/varnishtest/vtc.c \
@@ -193,14 +208,10 @@ VTCSYN_SRC = $(top_srcdir)/bin/varnishtest/vtc.c \
 	     $(top_srcdir)/bin/varnishtest/vtc_syslog.c \
 	     $(top_srcdir)/bin/varnishtest/vtc_varnish.c
 include/vtc-syntax.rst: vtc-syntax.py $(VTCSYN_SRC)
-	$(PYTHON) $(top_srcdir)/doc/sphinx/vtc-syntax.py $(VTCSYN_SRC) > $@
+	$(PYTHON) $(top_srcdir)/doc/sphinx/vtc-syntax.py $(VTCSYN_SRC) > ${@}_
+	mv ${@}_ ${@}
 BUILT_SOURCES += include/vtc-syntax.rst
 
-.PHONY: reference
-reference:
-	test -d $@ || mkdir $@
-BUILT_SOURCES += reference
-
 reference/vmod_std.generated.rst: reference $(top_builddir)/lib/libvmod_std/vmod_std.rst
 	cp $(top_builddir)/lib/libvmod_std/vmod_std.rst $@ || true
 BUILT_SOURCES += reference/vmod_std.generated.rst
@@ -231,5 +242,6 @@ BUILT_SOURCES += reference/vmod_proxy.generated.rst
 
 EXTRA_DIST += $(BUILT_SOURCES)
 MAINTAINERCLEANFILES = $(EXTRA_DIST)
+CLEANFILES = $(BUILT_SOURCES)
 
 .NOPATH: $(BUILT_SOURCES)

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:30 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:30 +0000 (UTC)
Subject: [6.0] ad292e356 keep the dog in the kennel unless there's a queue
Message-ID: <20181031130830.615EC95912@lists.varnish-cache.org>


commit ad292e356c4547b3456801b21e4f6ba350761028
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Oct 9 12:38:25 2018 +0200

    keep the dog in the kennel unless there's a queue
    
    As long as we are not queuing any threads, there is no queue to
    move. We record the queue marker the first time we notice queuing and
    only then see if it doesn't move.
    
    Fixes #2794

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 3d4411cb4..9b26013cd 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -510,14 +510,15 @@ pool_herder(void *priv)
 		 * Instead we implement a watchdog and kill the worker if
 		 * nothing has been dequeued for that long.
 		 */
-		if (dq != pp->ndequeued) {
+		if (pp->lqueue == 0) {
+			dq = pp->ndequeued + 1;
+		} else if (dq != pp->ndequeued) {
 			dq = pp->ndequeued;
 			dqt = VTIM_real();
-		} else if (pp->lqueue &&
-		    VTIM_real() - dqt > cache_param->wthread_watchdog) {
+		} else if (VTIM_real() - dqt > cache_param->wthread_watchdog) {
 			VSL(SLT_Error, 0,
-			   "Pool Herder: Queue does not move ql=%u dt=%f",
-			   pp->lqueue, VTIM_real() - dqt);
+			    "Pool Herder: Queue does not move ql=%u dt=%f",
+			    pp->lqueue, VTIM_real() - dqt);
 			WRONG("Worker Pool Queue does not move");
 		}
 		wthread_min = cache_param->wthread_min;

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:30 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:30 +0000 (UTC)
Subject: [6.0] 7ace1499f use monotonic time for the watchdog
Message-ID: <20181031130830.7D25A95925@lists.varnish-cache.org>


commit 7ace1499f50c154367af4c926e8dc45ba092813a
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Oct 9 12:56:41 2018 +0200

    use monotonic time for the watchdog

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 9b26013cd..bfbfae374 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -489,7 +489,7 @@ pool_herder(void *priv)
 	double delay;
 	int wthread_min;
 	uintmax_t dq = (1ULL << 31);
-	double dqt = 0;
+	vtim_mono dqt = 0;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 
@@ -514,11 +514,11 @@ pool_herder(void *priv)
 			dq = pp->ndequeued + 1;
 		} else if (dq != pp->ndequeued) {
 			dq = pp->ndequeued;
-			dqt = VTIM_real();
-		} else if (VTIM_real() - dqt > cache_param->wthread_watchdog) {
+			dqt = VTIM_mono();
+		} else if (VTIM_mono() - dqt > cache_param->wthread_watchdog) {
 			VSL(SLT_Error, 0,
 			    "Pool Herder: Queue does not move ql=%u dt=%f",
-			    pp->lqueue, VTIM_real() - dqt);
+			    pp->lqueue, VTIM_mono() - dqt);
 			WRONG("Worker Pool Queue does not move");
 		}
 		wthread_min = cache_param->wthread_min;

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:30 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:30 +0000 (UTC)
Subject: [6.0] e00cdeeca Rearrange to avoid a tiny leak
Message-ID: <20181031130830.9AE4E95932@lists.varnish-cache.org>


commit e00cdeeca3e8dcfa37415d409181b03521257707
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Oct 9 14:07:50 2018 +0100

    Rearrange to avoid a tiny leak

diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c
index f3faa9cb3..9c6a74685 100644
--- a/bin/varnishd/mgt/mgt_param.c
+++ b/bin/varnishd/mgt/mgt_param.c
@@ -353,8 +353,6 @@ mcf_param_show_json(struct cli *cli, const char * const *av, void *priv)
 	struct vsb *vsb, *def;
 	const char *show = NULL;
 
-	vsb = VSB_new_auto();
-	def = VSB_new_auto();
 	(void)priv;
 
 	for (int i = 2; av[i] != NULL; i++) {
@@ -372,6 +370,9 @@ mcf_param_show_json(struct cli *cli, const char * const *av, void *priv)
 		show = av[i];
 	}
 
+	vsb = VSB_new_auto();
+	def = VSB_new_auto();
+
 	n = 0;
 	VCLI_JSON_begin(cli, 2, av);
 	VCLI_Out(cli, ",\n");

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:30 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:30 +0000 (UTC)
Subject: [6.0] 51629df24 Hunt for py-specific variants of sphinx-build program
Message-ID: <20181031130830.B4F129594D@lists.varnish-cache.org>


commit 51629df2488fa9c77f8a7bd7b91e3bd05d893af8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 11 14:08:51 2018 +0000

    Hunt for py-specific variants of sphinx-build program

diff --git a/configure.ac b/configure.ac
index 13b54b5fb..7db6f51ea 100644
--- a/configure.ac
+++ b/configure.ac
@@ -49,7 +49,9 @@ fi
 AC_ARG_WITH([sphinx-build],
   AS_HELP_STRING([--with-sphinx-build=PATH], [Location of sphinx-build (auto)]),
   [SPHINX="$withval"],
-  AC_CHECK_PROGS(SPHINX, [sphinx-build], [no]))
+  AC_CHECK_PROGS(SPHINX,
+    [sphinx-build sphinx-build-3.6 sphinx-build-2.7],
+    [no]))
 if test "x$SPHINX" = "xno"; then
   AC_MSG_ERROR(
     [sphinx-build is needed to build Varnish, please install python-sphinx.])

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:30 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:30 +0000 (UTC)
Subject: [6.0] 51c0fc62f Also look for versioned rst2man scripts
Message-ID: <20181031130830.D58C295958@lists.varnish-cache.org>


commit 51c0fc62fcd128f806a17aac9b89cd220ff37a86
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 11 14:26:02 2018 +0000

    Also look for versioned rst2man scripts

diff --git a/configure.ac b/configure.ac
index 7db6f51ea..8794dfaee 100644
--- a/configure.ac
+++ b/configure.ac
@@ -40,7 +40,9 @@ AC_PROG_INSTALL
 AC_ARG_WITH([rst2man],
   AS_HELP_STRING([--with-rst2man=PATH], [Location of rst2man (auto)]),
   [RST2MAN="$withval"],
-  AC_CHECK_PROGS(RST2MAN, [rst2man rst2man.py], [no]))
+  AC_CHECK_PROGS(RST2MAN,
+    [rst2man rst2man.py rst2man-3.6 rst2man-2.7],
+    [no]))
 if test "x$RST2MAN" = "xno"; then
   AC_MSG_ERROR(
     [rst2man is needed to build Varnish, please install python-docutils.])

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:30 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:30 +0000 (UTC)
Subject: [6.0] fbc455e7a Add client_resp_500 counter
Message-ID: <20181031130831.0556A95966@lists.varnish-cache.org>


commit fbc455e7a635d8e348983ff45fce5ff5730ff1b3
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Wed Oct 10 13:41:28 2018 +0200

    Add client_resp_500 counter
    
    Counts the number of times we failed a response due to running out of
    workspace during delivery.
    
    Conflicts:
            bin/varnishtest/tests/v00058.vtc

diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc
index 88a22bb86..da1306f36 100644
--- a/bin/varnishd/VSC_main.vsc
+++ b/bin/varnishd/VSC_main.vsc
@@ -586,6 +586,14 @@
 
 	Number of session closes with Error VCL_FAILURE (VCL failure)
 
+.. varnish_vsc:: client_resp_500
+	:level: diag
+	:group: wrk
+	:oneliner:	Delivery failed due to insufficient workspace.
+
+	Number of times we failed a response due to running out of
+	workspace memory during delivery.
+
 .. varnish_vsc:: shm_records
 	:level:	diag
 	:oneliner:	SHM records
diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c
index fda38dda4..88447b5a6 100644
--- a/bin/varnishd/http1/cache_http1_deliver.c
+++ b/bin/varnishd/http1/cache_http1_deliver.c
@@ -75,6 +75,7 @@ v1d_error(struct req *req, const char *msg)
 	VSLb(req->vsl, SLT_RespStatus, "500");
 	VSLb(req->vsl, SLT_RespReason, "Internal Server Error");
 
+	req->wrk->stats->client_resp_500++;
 	(void)write(req->sp->fd, r_500, sizeof r_500 - 1);
 	req->doclose = SC_TX_EOF;
 }
diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c
index 8f4b56e1d..de55887ae 100644
--- a/bin/varnishd/http2/cache_http2_deliver.c
+++ b/bin/varnishd/http2/cache_http2_deliver.c
@@ -263,7 +263,11 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 	}
 	if (VSB_finish(&resp)) {
 		// We ran out of workspace, return minimal 500
-		// XXX: VSC counter ?
+		VSLb(req->vsl, SLT_Error, "workspace_client overflow");
+		VSLb(req->vsl, SLT_RespStatus, "500");
+		VSLb(req->vsl, SLT_RespReason, "Internal Server Error");
+		req->wrk->stats->client_resp_500++;
+
 		r = (const char*)h2_500_resp;
 		sz = sizeof h2_500_resp;
 		sendbody = 0;
diff --git a/bin/varnishtest/tests/c00070.vtc b/bin/varnishtest/tests/c00070.vtc
index 659356935..4ecf82671 100644
--- a/bin/varnishtest/tests/c00070.vtc
+++ b/bin/varnishtest/tests/c00070.vtc
@@ -67,3 +67,5 @@ client c1 {
 } -run
 
 logexpect l2 -wait
+
+varnish v1 -expect client_resp_500 == 1
diff --git a/bin/varnishtest/tests/c00071.vtc b/bin/varnishtest/tests/c00071.vtc
index c0ebbc294..8ebddc490 100644
--- a/bin/varnishtest/tests/c00071.vtc
+++ b/bin/varnishtest/tests/c00071.vtc
@@ -46,3 +46,5 @@ client c2 {
 	expect resp.http.x-of == <undef>
 } -run
 
+
+varnish v1 -expect client_resp_500 == 2
diff --git a/bin/varnishtest/tests/r02233.vtc b/bin/varnishtest/tests/r02233.vtc
index 67fb20c9c..fe36fccb2 100644
--- a/bin/varnishtest/tests/r02233.vtc
+++ b/bin/varnishtest/tests/r02233.vtc
@@ -28,3 +28,4 @@ client c1 {
 } -run
 
 logexpect l1 -wait
+varnish v1 -expect client_resp_500 == 1
diff --git a/bin/varnishtest/tests/r02589.vtc b/bin/varnishtest/tests/r02589.vtc
index 418cc6833..2c43ab6fc 100644
--- a/bin/varnishtest/tests/r02589.vtc
+++ b/bin/varnishtest/tests/r02589.vtc
@@ -27,3 +27,5 @@ client c1 {
 		expect resp.http.server == "Varnish"
 	} -run
 } -run
+
+varnish v1 -expect client_resp_500 == 1

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:30 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:30 +0000 (UTC)
Subject: [6.0] a333decc4 Clarify and test object slimming for hfp+hfm
Message-ID: <20181031130830.44E1B958FC@lists.varnish-cache.org>


commit a333decc4f47dba386f917002f70eec3c0603114
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Sep 7 12:32:18 2018 +0200

    Clarify and test object slimming for hfp+hfm
    
    The previous code was correct already, but we can make it
    clearer that HFP implies OC_F_PASS
    
    Also test explicitly that both HFM and HFP have their objects
    slimmed.
    
    Closes #2768

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 717e2993a..b29f484e1 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -418,7 +418,9 @@ cnt_transmit(struct worker *wrk, struct req *req)
 
 	VSLb_ts_req(req, "Resp", W_TIM_real(wrk));
 
-	if (req->objcore->flags & (OC_F_PRIVATE | OC_F_PASS)) {
+	if (req->objcore->flags & (OC_F_PRIVATE | OC_F_PASS | OC_F_HFP)) {
+		if (req->objcore->flags & OC_F_HFP)
+			AN(req->objcore->flags & OC_F_PASS);
 		if (boc != NULL) {
 			HSH_Abandon(req->objcore);
 			ObjWaitState(req->objcore, BOS_FINISHED);
diff --git a/bin/varnishtest/tests/r01821.vtc b/bin/varnishtest/tests/r01821.vtc
index af6e934cb..b07799325 100644
--- a/bin/varnishtest/tests/r01821.vtc
+++ b/bin/varnishtest/tests/r01821.vtc
@@ -1,4 +1,6 @@
-varnishtest "Slim down hit-for-miss objects"
+varnishtest "Slim down hit-for-miss / hit-for-miss objects"
+
+# see also #2768
 
 server s1 -repeat 2 {
 	rxreq
@@ -7,20 +9,32 @@ server s1 -repeat 2 {
 
 varnish v1 -arg "-s Transient=default" -vcl+backend {
 	sub vcl_backend_response {
-		set beresp.uncacheable = true;
+		if (bereq.url == "/hfm") {
+			set beresp.uncacheable = true;
+		} else if (bereq.url == "/hfp") {
+			return (pass(1m));
+		}
 	}
 } -start
 
-logexpect l1 -v v1 {
+logexpect l1 -v v1 -g raw {
+	expect * *	Storage		"Transient"
 	expect * *	Storage		"Transient"
 } -start
 
 client c1 {
-	txreq
+	txreq -url "/hfm"
+	rxresp
+} -start
+
+client c2 {
+	txreq -url "/hfp"
 	rxresp
 } -run
 
+client c1 -wait
+
 logexpect l1 -wait
 
-varnish v1 -expect SM?.Transient.c_bytes != 0
+varnish v1 -expect SM?.Transient.c_bytes > 131072
 varnish v1 -expect SM?.Transient.g_bytes < 65536

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:31 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:31 +0000 (UTC)
Subject: [6.0] 7d1991720 Add ws_backend_overflow counter
Message-ID: <20181031130831.2323395974@lists.varnish-cache.org>


commit 7d19917208a9c9ab6b412c6a10ae28507c0daf39
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Wed Oct 10 13:41:56 2018 +0200

    Add ws_backend_overflow counter

diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc
index da1306f36..f640f1ff5 100644
--- a/bin/varnishd/VSC_main.vsc
+++ b/bin/varnishd/VSC_main.vsc
@@ -594,6 +594,13 @@
 	Number of times we failed a response due to running out of
 	workspace memory during delivery.
 
+.. varnish_vsc:: ws_backend_overflow
+	:level: diag
+	:group: wrk
+	:oneliner: workspace_backend overflows
+
+	Number of times we ran out of space in workspace_backend.
+
 .. varnish_vsc:: shm_records
 	:level:	diag
 	:oneliner:	SHM records
diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index e27431042..1f452d5e4 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -167,6 +167,9 @@ VBO_ReleaseBusyObj(struct worker *wrk, struct busyobj **pbo)
 
 	AZ(bo->htc);
 
+	if (WS_Overflowed(bo->ws))
+		wrk->stats->ws_backend_overflow++;
+
 	if (bo->fetch_objcore != NULL) {
 		AN(wrk);
 		(void)HSH_DerefObjCore(wrk, &bo->fetch_objcore,
diff --git a/bin/varnishtest/tests/r01739.vtc b/bin/varnishtest/tests/r01739.vtc
index 843ea6933..6e40741f5 100644
--- a/bin/varnishtest/tests/r01739.vtc
+++ b/bin/varnishtest/tests/r01739.vtc
@@ -27,3 +27,4 @@ client c1 {
 } -run
 
 logexpect l1 -wait
+varnish v1 -expect ws_backend_overflow == 1
diff --git a/bin/varnishtest/tests/r01990.vtc b/bin/varnishtest/tests/r01990.vtc
index d86f61c55..c6a83292c 100644
--- a/bin/varnishtest/tests/r01990.vtc
+++ b/bin/varnishtest/tests/r01990.vtc
@@ -40,3 +40,4 @@ client c1 {
 } -run
 
 logexpect l1 -wait
+varnish v1 -expect ws_backend_overflow == 1

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:31 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:31 +0000 (UTC)
Subject: [6.0] 9418297fe Add ws_client_overflow counter
Message-ID: <20181031130831.45CDA95989@lists.varnish-cache.org>


commit 9418297fe07a0ae49d5669328da203419625ffa3
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Thu Oct 11 11:26:45 2018 +0200

    Add ws_client_overflow counter
    
    Conflicts:
            bin/varnishtest/tests/v00058.vtc

diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc
index f640f1ff5..3327eb095 100644
--- a/bin/varnishd/VSC_main.vsc
+++ b/bin/varnishd/VSC_main.vsc
@@ -601,6 +601,13 @@
 
 	Number of times we ran out of space in workspace_backend.
 
+.. varnish_vsc:: ws_client_overflow
+	:level: diag
+	:group: wrk
+	:oneliner: workspace_client overflows
+
+	Number of times we ran out of space in workspace_client.
+
 .. varnish_vsc:: shm_records
 	:level:	diag
 	:oneliner:	SHM records
diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c
index 991487201..5ab13a2cf 100644
--- a/bin/varnishd/cache/cache_req.c
+++ b/bin/varnishd/cache/cache_req.c
@@ -191,6 +191,8 @@ Req_Rollback(struct req *req)
 	VCL_TaskLeave(req->vcl, req->privs);
 	VCL_TaskEnter(req->vcl, req->privs);
 	HTTP_Copy(req->http, req->http0);
+	if (WS_Overflowed(req->ws))
+		req->wrk->stats->ws_client_overflow++;
 	WS_Reset(req->ws, req->ws_req);
 }
 
@@ -242,6 +244,9 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 	req->hash_ignore_busy = 0;
 	req->is_hit = 0;
 
+	if (WS_Overflowed(req->ws))
+		wrk->stats->ws_client_overflow++;
+
 	WS_Reset(req->ws, 0);
 }
 
diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c
index de55887ae..2ec67a60b 100644
--- a/bin/varnishd/http2/cache_http2_deliver.c
+++ b/bin/varnishd/http2/cache_http2_deliver.c
@@ -262,6 +262,7 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 		VSB_bcat(&resp, r, sz);
 	}
 	if (VSB_finish(&resp)) {
+		WS_MarkOverflow(req->ws);
 		// We ran out of workspace, return minimal 500
 		VSLb(req->vsl, SLT_Error, "workspace_client overflow");
 		VSLb(req->vsl, SLT_RespStatus, "500");
diff --git a/bin/varnishtest/tests/c00070.vtc b/bin/varnishtest/tests/c00070.vtc
index 4ecf82671..9042fc554 100644
--- a/bin/varnishtest/tests/c00070.vtc
+++ b/bin/varnishtest/tests/c00070.vtc
@@ -69,3 +69,4 @@ client c1 {
 logexpect l2 -wait
 
 varnish v1 -expect client_resp_500 == 1
+varnish v1 -expect ws_client_overflow == 2
diff --git a/bin/varnishtest/tests/c00071.vtc b/bin/varnishtest/tests/c00071.vtc
index 8ebddc490..6cff35d87 100644
--- a/bin/varnishtest/tests/c00071.vtc
+++ b/bin/varnishtest/tests/c00071.vtc
@@ -48,3 +48,4 @@ client c2 {
 
 
 varnish v1 -expect client_resp_500 == 2
+varnish v1 -expect ws_client_overflow == 2
diff --git a/bin/varnishtest/tests/r02233.vtc b/bin/varnishtest/tests/r02233.vtc
index fe36fccb2..2bc591b82 100644
--- a/bin/varnishtest/tests/r02233.vtc
+++ b/bin/varnishtest/tests/r02233.vtc
@@ -29,3 +29,4 @@ client c1 {
 
 logexpect l1 -wait
 varnish v1 -expect client_resp_500 == 1
+varnish v1 -expect ws_client_overflow == 1
diff --git a/bin/varnishtest/tests/r02589.vtc b/bin/varnishtest/tests/r02589.vtc
index 2c43ab6fc..0d39c4fb3 100644
--- a/bin/varnishtest/tests/r02589.vtc
+++ b/bin/varnishtest/tests/r02589.vtc
@@ -29,3 +29,4 @@ client c1 {
 } -run
 
 varnish v1 -expect client_resp_500 == 1
+varnish v1 -expect ws_client_overflow == 1

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:31 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:31 +0000 (UTC)
Subject: [6.0] 4039a2c72 Add ws_thread_overflow counter
Message-ID: <20181031130831.6978595999@lists.varnish-cache.org>


commit 4039a2c72ef8446888e99ff6d704a274af76c923
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Thu Oct 11 11:56:01 2018 +0200

    Add ws_thread_overflow counter
    
    See details in r02275.vtc for why there isn't a test case for this.

diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc
index 3327eb095..bbc1eb08f 100644
--- a/bin/varnishd/VSC_main.vsc
+++ b/bin/varnishd/VSC_main.vsc
@@ -608,6 +608,13 @@
 
 	Number of times we ran out of space in workspace_client.
 
+.. varnish_vsc:: ws_thread_overflow
+	:level: diag
+	:group: wrk
+	:oneliner: workspace_thread overflows
+
+	Number of times we ran out of space in workspace_thread.
+
 .. varnish_vsc:: shm_records
 	:level:	diag
 	:oneliner:	SHM records
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index bfbfae374..50839143b 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -394,6 +394,8 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk)
 			tp = &tpx;
 		} while (tp->func != NULL);
 
+		if (WS_Overflowed(wrk->aws))
+			wrk->stats->ws_thread_overflow++;
 		/* cleanup for next task */
 		wrk->seen_methods = 0;
 	}

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:31 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:31 +0000 (UTC)
Subject: [6.0] 17fdd6aa5 Add ws_session_overflow counter
Message-ID: <20181031130831.86B6F959AC@lists.varnish-cache.org>


commit 17fdd6aa51abba888ed67e3ee6a2e0e06f9434b4
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Thu Oct 11 14:52:12 2018 +0200

    Add ws_session_overflow counter

diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc
index bbc1eb08f..f6925f30b 100644
--- a/bin/varnishd/VSC_main.vsc
+++ b/bin/varnishd/VSC_main.vsc
@@ -615,6 +615,12 @@
 
 	Number of times we ran out of space in workspace_thread.
 
+.. varnish_vsc:: ws_session_overflow
+	:level: diag
+	:oneliner: workspace_session overflows
+
+	Number of times we ran out of space in workspace_session.
+
 .. varnish_vsc:: shm_records
 	:level:	diag
 	:oneliner:	SHM records
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index 20ada52e3..fede8e0ae 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -545,6 +545,8 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now)
 	VSL(SLT_SessClose, sp->vxid, "%s %.3f",
 	    sess_close_2str(reason, 0), now - sp->t_open);
 	VSL(SLT_End, sp->vxid, "%s", "");
+	if (WS_Overflowed(sp->ws))
+		VSC_C_main->ws_session_overflow++;
 	SES_Rel(sp);
 }
 
diff --git a/bin/varnishtest/tests/o00005.vtc b/bin/varnishtest/tests/o00005.vtc
index 3ec7a2881..3ae1aacfe 100644
--- a/bin/varnishtest/tests/o00005.vtc
+++ b/bin/varnishtest/tests/o00005.vtc
@@ -201,3 +201,46 @@ client c1 {
 varnish v1 -vsl_catchup
 
 logexpect l1 -wait
+
+client c1 {
+	# PROXY2 sp->ws overflow
+	sendhex {
+		0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a
+		21 11 01 23
+		d9 46 b5 21
+		5f 8e a8 22
+		ed 96
+		01 bb
+		03 00 04  c5 1b 5b 2b
+		01 00 02  68 32
+		02 00 c8  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61 61 61 61 61 61 61 61 61 61
+			  61 61
+		20 00 3d
+		01 00 00 00 00
+		21 00 07  54 4c 53 76 31 2e 33
+		25 00 05  45 43 32 35 36
+		24 00 0a  52 53 41 2d 53 48 41 32 35 36
+		23 00 16  41 45 41 44 2d 41 45 53 31 32 38
+			  2d 47 43 4d 2d 53 48 41 32 35 36
+	}
+	expect_close
+} -run
+
+varnish v1 -expect ws_session_overflow == 1

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:31 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:31 +0000 (UTC)
Subject: [6.0] 7f7908008 Emit a VCL_Error SLT when we exceed ESI depth
Message-ID: <20181031130831.A4010959BE@lists.varnish-cache.org>


commit 7f790800813359274102112ba64b173d25646362
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Oct 13 07:05:53 2018 +0000

    Emit a VCL_Error SLT when we exceed ESI depth

diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c
index 1e238d854..1ce7ea4ca 100644
--- a/bin/varnishd/cache/cache_esi_deliver.c
+++ b/bin/varnishd/cache/cache_esi_deliver.c
@@ -108,8 +108,12 @@ ved_include(struct req *preq, const char *src, const char *host,
 	CHECK_OBJ_NOTNULL(ecx, ECX_MAGIC);
 	wrk = preq->wrk;
 
-	if (preq->esi_level >= cache_param->max_esi_depth)
+	if (preq->esi_level >= cache_param->max_esi_depth) {
+		VSLb(preq->vsl, SLT_VCL_Error,
+		    "ESI depth limit reach (param max_esi_depth = %u",
+		    cache_param->max_esi_depth);
 		return;
+	}
 
 	req = Req_New(wrk, sp);
 	SES_Ref(sp);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:31 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:31 +0000 (UTC)
Subject: [6.0] df59de4b3 change nanoseconds precision timestamps into
 microseconds
Message-ID: <20181031130831.C19E3959C6@lists.varnish-cache.org>


commit df59de4b3d48efe0cc2bc932067034eae336c86f
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Oct 15 14:39:53 2018 +0200

    change nanoseconds precision timestamps into microseconds
    
    Ref #2792
    
    Conflicts:
            doc/changes.rst

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 6aa1ae07c..3f2a11caf 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -179,7 +179,7 @@ EXP_Rearm(struct objcore *oc, double now, double ttl, double grace, double keep)
 
 	when = EXP_WHEN(oc);
 
-	VSL(SLT_ExpKill, 0, "EXP_Rearm p=%p E=%.9f e=%.9f f=0x%x", oc,
+	VSL(SLT_ExpKill, 0, "EXP_Rearm p=%p E=%.6f e=%.6f f=0x%x", oc,
 	    oc->timer_when, when, oc->flags);
 
 	if (when < oc->t_origin || when < oc->timer_when)
@@ -198,7 +198,7 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, unsigned flags)
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	assert(oc->refcnt > 0);
 
-	VSLb(&ep->vsl, SLT_ExpKill, "EXP_Inbox flg=%x p=%p e=%.9f f=0x%x",
+	VSLb(&ep->vsl, SLT_ExpKill, "EXP_Inbox flg=%x p=%p e=%.6f f=0x%x",
 	    flags, oc, oc->timer_when, oc->flags);
 
 	if (flags & OC_EF_REMOVE) {
@@ -219,7 +219,7 @@ exp_inbox(struct exp_priv *ep, struct objcore *oc, unsigned flags)
 		ObjSendEvent(ep->wrk, oc, OEV_TTLCHG);
 	}
 
-	VSLb(&ep->vsl, SLT_ExpKill, "EXP_When p=%p e=%.9f f=0x%x", oc,
+	VSLb(&ep->vsl, SLT_ExpKill, "EXP_When p=%p e=%.6f f=0x%x", oc,
 	    oc->timer_when, flags);
 
 	/*
@@ -255,7 +255,7 @@ exp_expire(struct exp_priv *ep, double now)
 	oc = binheap_root(ep->heap);
 	if (oc == NULL)
 		return (now + 355./113.);
-	VSLb(&ep->vsl, SLT_ExpKill, "EXP_expire p=%p e=%.9f f=0x%x", oc,
+	VSLb(&ep->vsl, SLT_ExpKill, "EXP_expire p=%p e=%.6f f=0x%x", oc,
 	    oc->timer_when - now, oc->flags);
 
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c
index d2d94aab3..9777584f8 100644
--- a/bin/varnishd/mgt/mgt_vcc.c
+++ b/bin/varnishd/mgt/mgt_vcc.c
@@ -312,7 +312,7 @@ mgt_VccCompile(struct cli *cli, struct vclprog *vcl, const char *vclname,
 	 *
 	 * The Best way to reproduce this is to have regexps in the VCL.
 	 */
-	VSB_printf(sb, "vcl_%s.%.9f", vclname, VTIM_real());
+	VSB_printf(sb, "vcl_%s.%.6f", vclname, VTIM_real());
 	AZ(VSB_finish(sb));
 	vp.dir = strdup(VSB_data(sb));
 	AN(vp.dir);

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:31 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:31 +0000 (UTC)
Subject: [6.0] 56cd85326 Make sure wrk arg is always present,
 we need it for stats now.
Message-ID: <20181031130831.DE63B959CF@lists.varnish-cache.org>


commit 56cd85326514fbeba4e7e3efaf4f4b7b8727593c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Oct 15 19:07:24 2018 +0000

    Make sure wrk arg is always present, we need it for stats now.

diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c
index 1f452d5e4..6eac91434 100644
--- a/bin/varnishd/cache/cache_busyobj.c
+++ b/bin/varnishd/cache/cache_busyobj.c
@@ -146,7 +146,7 @@ VBO_ReleaseBusyObj(struct worker *wrk, struct busyobj **pbo)
 {
 	struct busyobj *bo;
 
-	CHECK_OBJ_ORNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	TAKE_OBJ_NOTNULL(bo, pbo, BUSYOBJ_MAGIC);
 	CHECK_OBJ_ORNULL(bo->fetch_objcore, OBJCORE_MAGIC);
 
@@ -171,7 +171,6 @@ VBO_ReleaseBusyObj(struct worker *wrk, struct busyobj **pbo)
 		wrk->stats->ws_backend_overflow++;
 
 	if (bo->fetch_objcore != NULL) {
-		AN(wrk);
 		(void)HSH_DerefObjCore(wrk, &bo->fetch_objcore,
 		    HSH_RUSH_POLICY);
 	}

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:32 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:32 +0000 (UTC)
Subject: [6.0] 1dd96ecd8 Put all generated .rst files in doc/sphinx/include
Message-ID: <20181031130832.08B4B959E5@lists.varnish-cache.org>


commit 1dd96ecd8e7814a96c80595cce60846a9bbc3797
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 16 07:34:33 2018 +0000

    Put all generated .rst files in doc/sphinx/include

diff --git a/.gitignore b/.gitignore
index 113da821b..c96ca4b91 100644
--- a/.gitignore
+++ b/.gitignore
@@ -82,7 +82,6 @@ cscope.*out
 /man/*.3
 /man/*.7
 /doc/sphinx/include
-/doc/sphinx/*/*generated.rst
 /bin/varnishadm/varnishadm
 /bin/varnishd/varnishd
 /bin/varnishhist/varnishhist
diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index cf67a6106..d9be207ac 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -212,33 +212,33 @@ include/vtc-syntax.rst: vtc-syntax.py $(VTCSYN_SRC)
 	mv ${@}_ ${@}
 BUILT_SOURCES += include/vtc-syntax.rst
 
-reference/vmod_std.generated.rst: reference $(top_builddir)/lib/libvmod_std/vmod_std.rst
-	cp $(top_builddir)/lib/libvmod_std/vmod_std.rst $@ || true
-BUILT_SOURCES += reference/vmod_std.generated.rst
+include/vmod_std.generated.rst: $(top_builddir)/lib/libvmod_std/vmod_std.rst
+	cp $(top_builddir)/lib/libvmod_std/vmod_std.rst $@
+BUILT_SOURCES += include/vmod_std.generated.rst
 
-reference/vmod_directors.generated.rst: reference $(top_builddir)/lib/libvmod_directors/vmod_directors.rst
-	cp $(top_builddir)/lib/libvmod_directors/vmod_directors.rst $@ || true
-BUILT_SOURCES += reference/vmod_directors.generated.rst
+include/vmod_directors.generated.rst: $(top_builddir)/lib/libvmod_directors/vmod_directors.rst
+	cp $(top_builddir)/lib/libvmod_directors/vmod_directors.rst $@
+BUILT_SOURCES += include/vmod_directors.generated.rst
 
-reference/vmod_purge.generated.rst: reference $(top_builddir)/lib/libvmod_purge/vmod_purge.rst
-	cp $(top_builddir)/lib/libvmod_purge/vmod_purge.rst $@ || true
-BUILT_SOURCES += reference/vmod_purge.generated.rst
+include/vmod_purge.generated.rst: $(top_builddir)/lib/libvmod_purge/vmod_purge.rst
+	cp $(top_builddir)/lib/libvmod_purge/vmod_purge.rst $@
+BUILT_SOURCES += include/vmod_purge.generated.rst
 
-reference/vmod_vtc.generated.rst: reference $(top_builddir)/lib/libvmod_vtc/vmod_vtc.rst
-	cp $(top_builddir)/lib/libvmod_vtc/vmod_vtc.rst $@ || true
-BUILT_SOURCES += reference/vmod_vtc.generated.rst
+include/vmod_vtc.generated.rst: $(top_builddir)/lib/libvmod_vtc/vmod_vtc.rst
+	cp $(top_builddir)/lib/libvmod_vtc/vmod_vtc.rst $@
+BUILT_SOURCES += include/vmod_vtc.generated.rst
 
-reference/vmod_blob.generated.rst: reference $(top_builddir)/lib/libvmod_blob/vmod_blob.rst
-	cp $(top_builddir)/lib/libvmod_blob/vmod_blob.rst $@ || true
-BUILT_SOURCES += reference/vmod_blob.generated.rst
+include/vmod_blob.generated.rst: $(top_builddir)/lib/libvmod_blob/vmod_blob.rst
+	cp $(top_builddir)/lib/libvmod_blob/vmod_blob.rst $@
+BUILT_SOURCES += include/vmod_blob.generated.rst
 
-reference/vmod_unix.generated.rst: reference $(top_builddir)/lib/libvmod_unix/vmod_unix.rst
-	cp $(top_builddir)/lib/libvmod_unix/vmod_unix.rst $@ || true
-BUILT_SOURCES += reference/vmod_unix.generated.rst
+include/vmod_unix.generated.rst: $(top_builddir)/lib/libvmod_unix/vmod_unix.rst
+	cp $(top_builddir)/lib/libvmod_unix/vmod_unix.rst $@
+BUILT_SOURCES += include/vmod_unix.generated.rst
 
-reference/vmod_proxy.generated.rst: reference $(top_builddir)/lib/libvmod_proxy/vmod_proxy.rst
-	cp $(top_builddir)/lib/libvmod_proxy/vmod_proxy.rst $@ || true
-BUILT_SOURCES += reference/vmod_proxy.generated.rst
+include/vmod_proxy.generated.rst: $(top_builddir)/lib/libvmod_proxy/vmod_proxy.rst
+	cp $(top_builddir)/lib/libvmod_proxy/vmod_proxy.rst $@
+BUILT_SOURCES += include/vmod_proxy.generated.rst
 
 EXTRA_DIST += $(BUILT_SOURCES)
 MAINTAINERCLEANFILES = $(EXTRA_DIST)
diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst
index 1bf543536..75ec38e99 100644
--- a/doc/sphinx/reference/index.rst
+++ b/doc/sphinx/reference/index.rst
@@ -20,13 +20,7 @@ The Varnish Reference Manual
 	varnishtop.rst
 	vsm.rst
 	vmod.rst
-	vmod_std.generated.rst
-	vmod_directors.generated.rst
-	vmod_vtc.generated.rst
-	vmod_purge.generated.rst
-	vmod_blob.generated.rst
-	vmod_unix.generated.rst
-	vmod_proxy.generated.rst
+	vmod_generated.rst
 	directors.rst
 	varnish-counters.rst
 	vsl.rst
diff --git a/doc/sphinx/reference/vmod_generated.rst b/doc/sphinx/reference/vmod_generated.rst
new file mode 100644
index 000000000..ec89bb9a9
--- /dev/null
+++ b/doc/sphinx/reference/vmod_generated.rst
@@ -0,0 +1,15 @@
+
+.. include::	../include/vmod_std.generated.rst
+
+.. include::	../include/vmod_directors.generated.rst
+
+.. include::	../include/vmod_vtc.generated.rst
+
+.. include::	../include/vmod_purge.generated.rst
+
+.. include::	../include/vmod_blob.generated.rst
+
+.. include::	../include/vmod_unix.generated.rst
+
+.. include::	../include/vmod_proxy.generated.rst
+

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:32 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:32 +0000 (UTC)
Subject: [6.0] 4ca41e59d Copyediting
Message-ID: <20181031130832.2B291959FE@lists.varnish-cache.org>


commit 4ca41e59d8d1fcc9d30d476180e29e1c10d9b517
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 16 07:55:41 2018 +0000

    Copyediting

diff --git a/doc/sphinx/dev-guide/index.rst b/doc/sphinx/dev-guide/index.rst
index fe4d75986..f1c818245 100644
--- a/doc/sphinx/dev-guide/index.rst
+++ b/doc/sphinx/dev-guide/index.rst
@@ -41,18 +41,17 @@ Technical stuff
 
 * Our license is BSD 2-clause or looser, no GPL or LGPL.
 
-* We havn't had a major security issue in 10 years, and we're not going
-  to start having them now.
+* It took 11 years for the first major security issue, and that was too soon.
 
 Bugs, issues, feature requests & VIPs
 -------------------------------------
 
 Bugs, issues and feature requests start out as github issues.
 
-We do a "bug-wash" every monday at 13:00 EU time, where new and otherwise
-worthy of discussion issues are pow-wowed.
+Monday at 13:00-14:00 (EU time) we "bug-wash" on IRC to
+decide who and how issues are dealt with.
 
-Tickets we cannot do anything about are closed.
+Issues we cannot do anything about are closed.
 
 If feature-requests make sense, they get moved to a wiki/VIP page until
 somebody implements them.

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:32 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:32 +0000 (UTC)
Subject: [6.0] a1b38f30e Update collection copyright year
Message-ID: <20181031130832.50BF095A07@lists.varnish-cache.org>


commit a1b38f30eceadd5626dfa278816ac5f36810248b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 16 07:58:50 2018 +0000

    Update collection copyright year

diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst
index 07077ee48..9a708b593 100644
--- a/doc/sphinx/index.rst
+++ b/doc/sphinx/index.rst
@@ -37,7 +37,7 @@ Longer listings like example command output and VCL look like this::
     $ /opt/varnish/sbin/varnishd -V
     varnishd (varnish-trunk revision 199de9b)
     Copyright (c) 2006 Verdens Gang AS
-    Copyright (c) 2006-2015 Varnish Software AS
+    Copyright (c) 2006-2018 Varnish Software AS
 
 
 .. For maintainers:
diff --git a/lib/libvarnish/version.c b/lib/libvarnish/version.c
index 61711654e..1b9a42196 100644
--- a/lib/libvarnish/version.c
+++ b/lib/libvarnish/version.c
@@ -44,5 +44,5 @@ VCS_Message(const char *progname)
 {
 	fprintf(stderr, "%s (%s)\n", progname, VCS_version);
 	fprintf(stderr, "Copyright (c) 2006 Verdens Gang AS\n");
-	fprintf(stderr, "Copyright (c) 2006-2015 Varnish Software AS\n");
+	fprintf(stderr, "Copyright (c) 2006-2018 Varnish Software AS\n");
 }

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:32 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:32 +0000 (UTC)
Subject: [6.0] d9139f2cd Report errno on asserts
Message-ID: <20181031130832.6E27095A1A@lists.varnish-cache.org>


commit d9139f2cddef8e68383ba995f4f71972cd65b3f4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 16 09:13:45 2018 +0000

    Report errno on asserts

diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c
index 94bee3d3f..1e6e321e8 100644
--- a/bin/varnishtest/vtc_log.c
+++ b/bin/varnishtest/vtc_log.c
@@ -28,6 +28,7 @@
 
 #include "config.h"
 
+#include <errno.h>
 #include <stdarg.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -273,17 +274,19 @@ vtc_log_VAS_Fail(const char *func, const char *file, int line,
     const char *cond, enum vas_e why)
 {
 	struct vtclog *vl;
+	int e = errno;
 
 	(void)why;
 	vl = pthread_getspecific(log_key);
 	if (vl == NULL || vl->act) {
 		fprintf(stderr,
 		    "Assert error in %s(), %s line %d:\n"
-		    "  Condition(%s) not true.\n",
-		    func, file, line, cond);
+		    "  Condition(%s) not true. (errno=%d %s)\n",
+		    func, file, line, cond, e, strerror(e));
 	} else
 		vtc_fatal(vl, "Assert error in %s(), %s line %d:"
-		    "  Condition(%s) not true.\n", func, file, line, cond);
+		    "  Condition(%s) not true."
+		    "  Errno=%d %s", func, file, line, cond, e, strerror(e));
 	abort();
 }
 

From dridi.boukelmoune at gmail.com  Wed Oct 31 13:08:32 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 31 Oct 2018 13:08:32 +0000 (UTC)
Subject: [6.0] 980e51457 Make worker threads give up their VCL much sooner in
 VTC-environment and make sure the expects in the test are scrupulusly
 correct.
Message-ID: <20181031130832.89E2595A22@lists.varnish-cache.org>


commit 980e514574e60efb492e04882cc2e55bb59f0752
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Oct 16 10:16:00 2018 +0000

    Make worker threads give up their VCL much sooner in VTC-environment
    and make sure the expects in the test are scrupulusly correct.

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 50839143b..4d546ffd3 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -321,6 +321,7 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk)
 {
 	struct pool_task *tp = NULL;
 	struct pool_task tpx, tps;
+	vtim_real tmo;
 	int i, prio_lim;
 
 	CHECK_OBJ_NOTNULL(pp, POOL_MAGIC);
@@ -370,8 +371,13 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk)
 			pp->nidle++;
 			do {
 				// see signaling_note at the top for explanation
-				i = Lck_CondWait(&wrk->cond, &pp->mtx,
-				    wrk->vcl == NULL ?  0 : wrk->lastused+60.);
+				if (wrk->vcl == NULL)
+					tmo = 0;
+				else if (DO_DEBUG(DBG_VTC_MODE))
+					tmo =  wrk->lastused+1.;
+				else
+					tmo =  wrk->lastused+60.;
+				i = Lck_CondWait(&wrk->cond, &pp->mtx, tmo);
 				if (i == ETIMEDOUT)
 					VCL_Rel(&wrk->vcl);
 			} while (wrk->task.func == NULL);
diff --git a/bin/varnishtest/tests/v00006.vtc b/bin/varnishtest/tests/v00006.vtc
index 9b84076d0..4a187c8a7 100644
--- a/bin/varnishtest/tests/v00006.vtc
+++ b/bin/varnishtest/tests/v00006.vtc
@@ -18,7 +18,9 @@ varnish v1 -arg "-p thread_pools=1" -vcl+backend { } -start
 
 # Give the varnishd a chance to start and create workers etc.
 # NB: This is important for to avoid mis-ordering of the workers.
-delay 1
+# delay 1
+
+varnish v1 -expect MAIN.threads == 10
 
 client c1 {
 	txreq -url "/bar"
@@ -47,6 +49,8 @@ varnish v1 -vcl {
 	}
 }
 
+varnish v1 -vsl_catchup
+
 varnish v1 -expect n_backend == 2
 varnish v1 -expect n_vcl_avail == 2
 varnish v1 -expect n_vcl_discard == 0
@@ -57,13 +61,17 @@ varnish v1 -cli "vcl.list"
 
 varnish v1 -cli "vcl.discard vcl1"
 
+varnish v1 -vsl_catchup
+
 # Give expiry thread a chance to let go
 delay 2
 
-# It won't go away as long as the workthread holds a VCL reference
-varnish v1 -expect n_backend == 2
-varnish v1 -expect n_vcl_avail == 1
-varnish v1 -expect n_vcl_discard == 1
+varnish v1 -vsl_catchup
+
+# It may not go away as long as the workthread holds a VCL reference
+varnish v1 -expect n_backend >= 1
+varnish v1 -expect n_vcl_avail >= 1
+varnish v1 -expect n_vcl_discard >= 0
 
 # Do another request through the new VCL to the new backend
 client c1 {
@@ -72,6 +80,8 @@ client c1 {
 	expect resp.status == 200
 } -run
 
+varnish v1 -vsl_catchup
+
 server s2 -wait
 
 # The workthread should have released its VCL reference now