From phk at FreeBSD.org  Mon Feb  5 10:26:14 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon,  5 Feb 2018 10:26:14 +0000 (UTC)
Subject: [master] 04b93a7 Make the -syntax setting local to each command.
Message-ID: <20180205102614.6E3CCAF424@lists.varnish-cache.org>


commit 04b93a77f995a05683a8c9aaa65ef75c8696db09
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Feb 5 10:07:42 2018 +0000

    Make the -syntax setting local to each command.
    
    We want it to be very explicit when non-default VCL versions
    are used in test-cases.

diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 1b0934a..0917a49 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -303,8 +303,6 @@ varnish_new(const char *name)
 
 	REPLACE(v->jail, "");
 
-	v->syntax = 4.0;
-
 	v->vl = vtc_logopen(name);
 	AN(v->vl);
 
@@ -1016,7 +1014,7 @@ varnish_expect(const struct varnish *v, char * const *av)
  *         Stop the child process.
  *
  * \-syntax
- *         Set the VCL syntax level (default: 4.0)
+ *         Set the VCL syntax level for this command (default: 4.0)
  *
  * \-wait
  *         Wait for that instance to terminate.
@@ -1089,6 +1087,7 @@ cmd_varnish(CMD_ARGS)
 	if (v == NULL)
 		v = varnish_new(av[0]);
 	av++;
+	v->syntax = 4.0;
 
 	for (; *av != NULL; av++) {
 		if (vtc_error)

From phk at FreeBSD.org  Mon Feb  5 10:26:14 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon,  5 Feb 2018 10:26:14 +0000 (UTC)
Subject: [master] fd8fdc2 Make sure we have no floating point round-off issues
 with vcl version.
Message-ID: <20180205102614.88D84AF427@lists.varnish-cache.org>


commit fd8fdc215d8d7040ec75dcc224bfd05b90f6dbd6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Feb 5 10:10:56 2018 +0000

    Make sure we have no floating point round-off issues with vcl version.

diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 699b9ff..776a52f 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -66,7 +66,7 @@ struct symbol;
 
 struct source {
 	VTAILQ_ENTRY(source)	list;
-	float			syntax;
+	int			syntax;
 	char			*name;
 	const char		*b;
 	const char		*e;
@@ -183,7 +183,7 @@ VTAILQ_HEAD(inifinhead, inifin);
 struct vcc {
 	unsigned		magic;
 #define VCC_MAGIC		0x24ad719d
-	float			syntax;
+	int			syntax;
 
 	char			*builtin_vcl;
 	struct vfil_path	*vcl_path;
diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index 7bcd732..e11a16b 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -282,14 +282,14 @@ vcc_ParseVcl(struct vcc *tl)
 
 	tok1 = tl->t;
 	Expect(tl, CNUM);
-	tok1->src->syntax = *tl->t->b - '0';
+	tok1->src->syntax = (*tl->t->b - '0') * 10;
 	vcc_NextToken(tl);
 	Expect(tl, '.');
 	vcc_NextToken(tl);
 
 	Expect(tl, CNUM);
 	tok2 = tl->t;
-	tok1->src->syntax += .1 * (*tl->t->b - '0');
+	tok1->src->syntax += (*tl->t->b - '0');
 	vcc_NextToken(tl);
 
 	if (tok1->e - tok1->b != 1 || tok2->e - tok2->b != 1) {
@@ -314,7 +314,7 @@ vcc_ParseVcl(struct vcc *tl)
 		VSB_printf(tl->sb,
 		    "VCL version %.1f higher than"
 		    " the top level version %.1f\n",
-		    tok1->src->syntax, tl->syntax);
+		    .1 * tok1->src->syntax, .1 * tl->syntax);
 		vcc_ErrWhere2(tl, tok0, tl->t);
 		ERRCHK(tl);
 	}
@@ -357,7 +357,7 @@ vcc_Parse(struct vcc *tl)
 		VSB_printf(tl->sb,
 		    "VCL version declaration missing\n"
 		    "Update your VCL to Version 4 syntax, and add\n"
-		    "\tvcl 4.0;\n"
+		    "\tvcl 4.1;\n"
 		    "on the first line of the VCL files.\n"
 		);
 		vcc_ErrWhere(tl, tl->t);
@@ -367,7 +367,7 @@ vcc_Parse(struct vcc *tl)
 	vcc_ParseVcl(tl);
 	if (tok->src->syntax != 4.0) {
 		VSB_printf(tl->sb, "VCL version %.1f not supported.\n",
-		    tok->src->syntax);
+		    .1 * tok->src->syntax);
 		vcc_ErrWhere2(tl, tok, tl->t);
 		ERRCHK(tl);
 	}

From phk at FreeBSD.org  Mon Feb  5 10:26:14 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon,  5 Feb 2018 10:26:14 +0000 (UTC)
Subject: [master] 9bb435c Start implementing vcl 4.1.
Message-ID: <20180205102614.AAEA7AF42A@lists.varnish-cache.org>


commit 9bb435c4dae447ab9469af8a034bb335d323cdaf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Feb 5 10:12:53 2018 +0000

    Start implementing vcl 4.1.
    
    In vcl 4.1 symbol names are unique across all types & kinds.

diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 81f4135..093e2b1 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -967,6 +967,7 @@ cmp_acl(struct vcc *tl, struct expr **e, const struct cmps *cp)
 	vcc_NextToken(tl);
 	vcc_ExpectVid(tl, "ACL");
 	sym = VCC_SymbolGet(tl, SYM_ACL, SYMTAB_CREATE, XREF_REF);
+	ERRCHK(tl);
 	AN(sym);
 	VCC_GlobalSymbol(sym, ACL, ACL_SYMBOL_PREFIX);
 	bprintf(buf, "%sVRT_acl_match(ctx, %s, \v1)", cp->emit, sym->rname);
diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index e11a16b..e3e69cb 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -365,7 +365,7 @@ vcc_Parse(struct vcc *tl)
 	}
 	tok = tl->t;
 	vcc_ParseVcl(tl);
-	if (tok->src->syntax != 4.0) {
+	if (tok->src->syntax < 40 || tok->src->syntax > 41) {
 		VSB_printf(tl->sb, "VCL version %.1f not supported.\n",
 		    .1 * tok->src->syntax);
 		vcc_ErrWhere2(tl, tok, tl->t);
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index 3ba2b8c..d2cd1bc 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -146,8 +146,9 @@ VCC_Symbol(struct vcc *tl, struct symbol *parent,
 			continue;
 		if (q < e)
 			break;
-		if ((kind == SYM_NONE && kind == sym->kind) ||
-		    (kind != SYM_NONE && kind != sym->kind))
+		if ((kind == SYM_NONE && kind == sym->kind))
+			continue;
+		if (tl->syntax < 41 && (kind != SYM_NONE && kind != sym->kind))
 			continue;
 		break;
 	}
@@ -203,13 +204,31 @@ VCC_SymbolGet(struct vcc *tl, enum symkind kind, const char *e, const char *x)
 	    e == SYMTAB_CREATE ? 1 : 0);
 	if (sym == NULL && e == SYMTAB_NOERR)
 		return (sym);
-	if (sym == NULL || (kind != SYM_NONE && sym->kind != kind)) {
+	if (sym == NULL) {
 		VSB_printf(tl->sb, "%s: ", e);
 		vcc_ErrToken(tl, tl->t);
 		VSB_cat(tl->sb, "\nAt: ");
 		vcc_ErrWhere(tl, tl->t);
 		return (NULL);
 	}
+	if (kind != SYM_NONE && kind != sym->kind) {
+		VSB_printf(tl->sb, "Symbol ");
+		vcc_ErrToken(tl, tl->t);
+		VSB_printf(tl->sb, " has wrong type (%s): ",
+			VCC_SymKind(tl, sym));
+		VSB_cat(tl->sb, "\nAt: ");
+		vcc_ErrWhere(tl, tl->t);
+		if (sym->def_b != NULL) {
+			VSB_printf(tl->sb, "Symbol was defined here: ");
+			vcc_ErrWhere(tl, sym->def_b);
+		} else if (sym->ref_b != NULL) {
+			VSB_printf(tl->sb, "Symbol was declared here: ");
+			vcc_ErrWhere(tl, sym->ref_b);
+		} else {
+			VSB_printf(tl->sb, "Symbol was builtin\n");
+		}
+		return (NULL);
+	}
 	if (x == XREF_DEF) {
 		if (sym->def_b == NULL)
 			sym->def_b = tl->t;
@@ -325,6 +344,8 @@ VCC_HandleSymbol(struct vcc *tl, vcc_type_t fmt, const char *pfx)
 	}
 	if (sym == NULL)
 		sym = VCC_SymbolGet(tl, kind, SYMTAB_CREATE, XREF_NONE);
+	if (sym == NULL)
+		return (NULL);
 	AN(sym);
 	AZ(sym->ndef);
 	VCC_GlobalSymbol(sym, fmt, pfx);
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 32cd611..02789c5 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -281,6 +281,7 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym)
 	ERRCHK(tl);
 	sy1 = VCC_HandleSymbol(tl, INSTANCE, "vo");
 	ERRCHK(tl);
+	AN(sy1);
 	sy1->noref = 1;
 
 	ExpectErr(tl, '=');

From phk at FreeBSD.org  Mon Feb  5 10:26:14 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon,  5 Feb 2018 10:26:14 +0000 (UTC)
Subject: [master] a4a1b30 Don't overload symbols
Message-ID: <20180205102614.D21F2AF42F@lists.varnish-cache.org>


commit a4a1b30390d8a1657bfd0137a982b1c0b904ce69
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Feb 5 10:24:52 2018 +0000

    Don't overload symbols

diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc
index 14875e6..fc0ecf1 100644
--- a/bin/varnishtest/tests/m00000.vtc
+++ b/bin/varnishtest/tests/m00000.vtc
@@ -12,7 +12,7 @@ varnish v1 -vcl+backend {
 	import vtc;
 
 	sub vcl_init {
-		new obj = debug.obj();
+		new objx = debug.obj();
 	}
 
 	sub vcl_recv {
@@ -28,8 +28,8 @@ varnish v1 -vcl+backend {
 		set resp.http.not = vtc.typesize("*");
 		debug.test_priv_call();
 		debug.test_priv_vcl();
-		obj.test_priv_call();
-		obj.test_priv_vcl();
+		objx.test_priv_call();
+		objx.test_priv_vcl();
 		std.log("VCL" + " initiated " + "log");
 		std.syslog(8 + 7, "Somebody runs varnishtest");
 		debug.rot52(resp);
diff --git a/bin/varnishtest/tests/m00023.vtc b/bin/varnishtest/tests/m00023.vtc
index 919f3cb..7492016 100644
--- a/bin/varnishtest/tests/m00023.vtc
+++ b/bin/varnishtest/tests/m00023.vtc
@@ -12,7 +12,7 @@ varnish v1 -vcl+backend {
 		"127"/24;
 	}
 
-	acl local {
+	acl locals {
 		// We assume c1 and s1 comes from same address
 		"${s1_addr}"/24;
 	}
@@ -24,7 +24,7 @@ varnish v1 -vcl+backend {
 	}
 
 	sub vcl_recv {
-		if (debug.match_acl(ip=client.ip, acl=local)) {
+		if (debug.match_acl(ip=client.ip, acl=locals)) {
 			return (hash);
 		}
 		return (synth(500));
diff --git a/bin/varnishtest/tests/m00027.vtc b/bin/varnishtest/tests/m00027.vtc
index d419b78..d4b8ff7 100644
--- a/bin/varnishtest/tests/m00027.vtc
+++ b/bin/varnishtest/tests/m00027.vtc
@@ -13,7 +13,7 @@ varnish v1 -errvcl "Missing dynamic backend address or port" {
 	}
 
 	sub vcl_init {
-		new obj = debug.dyn("", "");
+		new objx = debug.dyn("", "");
 		std.log("Should not happen");
 	}
 }
diff --git a/bin/varnishtest/tests/r01332.vtc b/bin/varnishtest/tests/r01332.vtc
index 43ca406..1f66167 100644
--- a/bin/varnishtest/tests/r01332.vtc
+++ b/bin/varnishtest/tests/r01332.vtc
@@ -9,12 +9,12 @@ varnish v1 -vcl+backend {
 	import debug;
 
 	sub vcl_init {
-		new obj = debug.obj("don't care");
+		new objx = debug.obj("don't care");
 	}
 
 	sub vcl_deliver {
-		obj.enum(martin);
-		set resp.http.foo = obj.foo("");
+		objx.enum(martin);
+		set resp.http.foo = objx.foo("");
 	}
 } -start
 
diff --git a/bin/varnishtest/tests/r02451.vtc b/bin/varnishtest/tests/r02451.vtc
index ef3c45c..2ff3fac 100644
--- a/bin/varnishtest/tests/r02451.vtc
+++ b/bin/varnishtest/tests/r02451.vtc
@@ -21,7 +21,7 @@ varnish v1 -vcl+backend {
 	import std;
 
 	sub vcl_init {
-		new obj = debug.obj();
+		new objx = debug.obj();
 		new vd = directors.shard();
 		debug.test_priv_task("something");
 		debug.test_priv_task("to remember");
@@ -32,7 +32,7 @@ varnish v1 -vcl+backend {
 		vd.reconfigure(replicas=25);
 
 		std.log("func " + debug.test_priv_task());
-		std.log("obj "	+ obj.test_priv_task());
+		std.log("obj "	+ objx.test_priv_task());
 	}
 
 	sub vcl_recv {
diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc
index e3636e5..3bac4aa 100644
--- a/bin/varnishtest/tests/v00041.vtc
+++ b/bin/varnishtest/tests/v00041.vtc
@@ -15,14 +15,14 @@ varnish v1 -arg "-p debug=+vclrel" -vcl+backend {
 	import std;
 
 	sub vcl_init {
-		new obj = debug.obj();
+		new objx = debug.obj();
 	}
 
 	sub vcl_init {
 		debug.test_priv_task("something");
 		debug.test_priv_task("to remember");
 		std.log("func " + debug.test_priv_task());
-		std.log("obj "  + obj.test_priv_task());
+		std.log("obj "  + objx.test_priv_task());
 	}
 
 	sub vcl_recv {
@@ -34,11 +34,11 @@ varnish v1 -arg "-p debug=+vclrel" -vcl+backend {
 	sub vcl_deliver {
 		set resp.http.x0 = req.http.x0;
 		set resp.http.x1 = debug.test_priv_task();
-		set resp.http.o1 = obj.test_priv_task();
+		set resp.http.o1 = objx.test_priv_task();
 	}
 
 	sub vcl_backend_fetch {
-		obj.test_priv_task("b");
+		objx.test_priv_task("b");
 		std.log("foo");
 		set bereq.http.bx0 = debug.test_priv_task(bereq.url);
 		std.log("bar");
@@ -47,14 +47,14 @@ varnish v1 -arg "-p debug=+vclrel" -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.http.bx0 = bereq.http.bx0;
 		set beresp.http.bx1 = debug.test_priv_task("");
-		set beresp.http.bo1 = obj.test_priv_task("");
+		set beresp.http.bo1 = objx.test_priv_task("");
 	}
 
 	sub vcl_fini {
 		debug.test_priv_task("cleaning");
 		debug.test_priv_task("up");
 		std.log("func " + debug.test_priv_task());
-		std.log("obj "  + obj.test_priv_task());
+		std.log("obj "  + objx.test_priv_task());
 	}
 
 } -start

From phk at FreeBSD.org  Mon Feb  5 10:26:14 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon,  5 Feb 2018 10:26:14 +0000 (UTC)
Subject: [master] de03c2c Switch varnishtest to default to vcl4.1
Message-ID: <20180205102615.00712AF433@lists.varnish-cache.org>


commit de03c2cd88b904c19c2e71eee5baf74a1455ece1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Feb 5 10:25:16 2018 +0000

    Switch varnishtest to default to vcl4.1

diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc
index 937fa83..1d1637f 100644
--- a/bin/varnishtest/tests/v00016.vtc
+++ b/bin/varnishtest/tests/v00016.vtc
@@ -89,7 +89,7 @@ varnish v1 -errvcl {resolves to too many addresses} {
 	}
 }
 
-varnish v1 -errvcl {Undefined acl foo} {
+varnish v1 -errvcl {Symbol 'foo' has wrong type (instance):} {
 	import directors;
 	backend b { .host = "127.0.0.1"; }
 
@@ -103,7 +103,22 @@ varnish v1 -errvcl {Undefined acl foo} {
 	}
 }
 
-varnish v1 -errvcl {Symbol not found: 'foo'} {
+varnish v1 -syntax 4.0 -errvcl {Undefined acl foo} {
+	import directors;
+	backend b { .host = "127.0.0.1"; }
+
+	sub vcl_init {
+		new foo = directors.shard();
+	}
+	sub vcl_recv {
+		if (client.ip ~ foo) {
+			return (synth(200));
+		}
+	}
+}
+
+# 'foo' overloaded
+varnish v1 -syntax 4.0 -errvcl {Symbol not found: 'foo'} {
 	backend b { .host = "127.0.0.1"; }
 
 	acl foo {
diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc
index afba076..bc152ec 100644
--- a/bin/varnishtest/tests/v00018.vtc
+++ b/bin/varnishtest/tests/v00018.vtc
@@ -127,7 +127,13 @@ varnish v1 -errvcl {Not a valid action in method 'vcl_recv'} {
 	}
 }
 
-varnish v1 -errvcl {Not a VCL label:} {
+varnish v1 -errvcl {Symbol 'vcl_recv' has wrong type (sub):} {
+	sub vcl_recv {
+		return (vcl(vcl_recv));
+	}
+}
+
+varnish v1 -syntax 4.0 -errvcl {Not a VCL label:} {
 	sub vcl_recv {
 		return (vcl(vcl_recv));
 	}
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 0917a49..73ca160 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -1014,7 +1014,7 @@ varnish_expect(const struct varnish *v, char * const *av)
  *         Stop the child process.
  *
  * \-syntax
- *         Set the VCL syntax level for this command (default: 4.0)
+ *         Set the VCL syntax level for this command (default: 4.1)
  *
  * \-wait
  *         Wait for that instance to terminate.
@@ -1087,7 +1087,7 @@ cmd_varnish(CMD_ARGS)
 	if (v == NULL)
 		v = varnish_new(av[0]);
 	av++;
-	v->syntax = 4.0;
+	v->syntax = 4.1;
 
 	for (; *av != NULL; av++) {
 		if (vtc_error)

From phk at FreeBSD.org  Tue Feb  6 09:34:13 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  6 Feb 2018 09:34:13 +0000 (UTC)
Subject: [master] 8c2ed01 Propagate the VCL syntax to VRT_CTX.
Message-ID: <20180206093413.471A8A0175@lists.varnish-cache.org>


commit 8c2ed01282a157eaf776e3d1106a93f8d850f5ea
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 6 09:33:09 2018 +0000

    Propagate the VCL syntax to VRT_CTX.
    
    More work on vcl4.1 reserved words etc.

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index aa11668..ef05b9c 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -106,6 +106,8 @@ static uintptr_t ws_snapshot_cli;
 static struct vrt_ctx *
 vcl_get_ctx(unsigned method, int msg)
 {
+
+	ASSERT_CLI();
 	AZ(ctx_cli.handling);
 	INIT_OBJ(&ctx_cli, VRT_CTX_MAGIC);
 	handling_cli = 0;
@@ -124,6 +126,8 @@ vcl_get_ctx(unsigned method, int msg)
 static void
 vcl_rel_ctx(struct vrt_ctx **ctx)
 {
+
+	ASSERT_CLI();
 	assert(*ctx == &ctx_cli);
 	AN((*ctx)->handling);
 	if (ctx_cli.msg)
@@ -877,6 +881,7 @@ VCL_Poll(void)
 		if (vcl->temp == VCL_TEMP_BUSY ||
 		    vcl->temp == VCL_TEMP_COOLING) {
 			ctx->vcl = vcl;
+			ctx->syntax = ctx->vcl->conf->syntax;
 			ctx->method = 0;
 			(void)vcl_set_state(ctx, "0");
 		}
@@ -887,6 +892,7 @@ VCL_Poll(void)
 			VTAILQ_REMOVE(&vcl_head, vcl, list);
 			ctx->method = VCL_MET_FINI;
 			ctx->vcl = vcl;
+			ctx->syntax = ctx->vcl->conf->syntax;
 			AZ(vcl_send_event(ctx, VCL_EVENT_DISCARD));
 			vcl_KillBackends(vcl);
 			free(vcl->loaded_name);
@@ -1132,6 +1138,7 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 		ctx.ws = bo->ws;
 	}
 	assert(ctx.now != 0);
+	ctx.syntax = ctx.vcl->conf->syntax;
 	ctx.vsl = vsl;
 	ctx.specific = specific;
 	ctx.method = method;
diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc
index dbfa07a..39defaa 100644
--- a/bin/varnishtest/tests/v00020.vtc
+++ b/bin/varnishtest/tests/v00020.vtc
@@ -203,7 +203,7 @@ varnish v1 -errvcl {Name of function, 'foo.bar', contains illegal character '.'}
 	}
 }
 
-varnish v1 -errvcl {VCL sub's named 'vcl*' are reserved names.} {
+varnish v1 -errvcl {The names 'vcl*' are reserved for methods.} {
 	sub vcl_bar {
 	}
 	sub vcl_recv {
diff --git a/bin/varnishtest/tests/v00021.vtc b/bin/varnishtest/tests/v00021.vtc
index 7db8afd..8394ce8 100644
--- a/bin/varnishtest/tests/v00021.vtc
+++ b/bin/varnishtest/tests/v00021.vtc
@@ -95,3 +95,29 @@ varnish v1 -errvcl {Unknown variable: 'req.foobar'} {
 	backend foo { .host = "127.0.0.1"; }
 	sub vcl_recv { set req.foobar = 3; }
 }
+
+varnish v1 -errvcl {Symbol 'anacl' has wrong type (acl):} {
+	sub vcl_recv {
+		if (client.ip ~ anacl) { }
+	}
+	sub anacl { }
+}
+
+varnish v1 -errvcl {Symbols named 'vcl_*' are reserved.} {
+	sub vcl_recv {
+		if (client.ip ~ vcl_foo) { }
+	}
+}
+
+varnish v1 -errvcl {Symbol 'true' has wrong type (func):} {
+	sub vcl_recv {
+		if (client.ip ~ true) { }
+	}
+}
+
+varnish v1 -errvcl {Symbol 'default' is a reserved word.} {
+	sub vcl_recv {
+		if (client.ip ~ default) { }
+	}
+}
+
diff --git a/include/vrt.h b/include/vrt.h
index 662399d..c2dab1e 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -162,8 +162,10 @@ struct vrt_ctx {
 	unsigned			magic;
 #define VRT_CTX_MAGIC			0x6bb8f0db
 
+	unsigned			syntax;
 	unsigned			method;
 	unsigned			*handling;
+	unsigned			vclver;
 
 	struct vsb			*msg;	// Only in ...init()
 	struct vsl_log			*vsl;
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 6ae254d..b75ae57 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -1179,6 +1179,7 @@ struct VCL_conf {
 	unsigned			magic;
 #define VCL_CONF_MAGIC			0x7406c509	/* from /dev/random */
 
+	unsigned			syntax;
 	struct director			**default_director;
 	const struct vrt_backend_probe	*default_probe;
 	unsigned			nref;
diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c
index ba45ee2..d3536ab 100644
--- a/lib/libvcc/vcc_acl.c
+++ b/lib/libvcc/vcc_acl.c
@@ -470,7 +470,6 @@ vcc_ParseAcl(struct vcc *tl)
 	vcc_ExpectVid(tl, "ACL");
 	ERRCHK(tl);
 	sym = VCC_HandleSymbol(tl, ACL, ACL_SYMBOL_PREFIX);
-
 	ERRCHK(tl);
 	AN(sym);
 
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 3f9393b..55bcd7c 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -414,6 +414,7 @@ EmitStruct(const struct vcc *tl)
 {
 	Fc(tl, 0, "\nconst struct VCL_conf VCL_conf = {\n");
 	Fc(tl, 0, "\t.magic = VCL_CONF_MAGIC,\n");
+	Fc(tl, 0, "\t.syntax = %u,\n", tl->syntax);
 	Fc(tl, 0, "\t.event_vcl = VGC_Event,\n");
 	Fc(tl, 0, "\t.default_director = &%s,\n", tl->default_director);
 	if (tl->default_probe != NULL)
@@ -561,6 +562,8 @@ vcc_CompileSource(struct vcc *tl, struct source *sp)
 	Fh(tl, 0, "/* ---===### VCC generated .h code ###===---*/\n");
 	Fc(tl, 0, "\n/* ---===### VCC generated .c code ###===---*/\n");
 
+	vcc_Parse_Init(tl);
+
 	vcc_Expr_Init(tl);
 
 	vcc_Action_Init(tl);
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 776a52f..d3f6b7a 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -294,7 +294,8 @@ struct symbol *VCC_HandleSymbol(struct vcc *, vcc_type_t , const char *);
 void vcc_Var_Init(struct vcc *);
 
 /* vcc_parse.c */
-void vcc_Parse(struct vcc *tl);
+void vcc_Parse(struct vcc *);
+void vcc_Parse_Init(struct vcc *);
 sym_act_f vcc_Act_If;
 
 /* vcc_utils.c */
diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index e3e69cb..0cd13cc 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -220,6 +220,7 @@ vcc_ParseFunction(struct vcc *tl)
 
 	t = tl->t;
 	sym = VCC_SymbolGet(tl, SYM_SUB, SYMTAB_CREATE, XREF_DEF);
+	ERRCHK(tl);
 	AN(sym);
 	p = sym->proc;
 	if (p == NULL) {
@@ -227,7 +228,7 @@ vcc_ParseFunction(struct vcc *tl)
 		    (t->b[1] == 'c'|| t->b[1] == 'C') &&
 		    (t->b[2] == 'l'|| t->b[2] == 'L')) {
 			VSB_printf(tl->sb,
-			    "VCL sub's named 'vcl*' are reserved names.\n");
+			    "The names 'vcl*' are reserved for methods.\n");
 			vcc_ErrWhere(tl, t);
 			VSB_printf(tl->sb, "Valid vcl_* methods are:\n");
 			VTAILQ_FOREACH(p, &tl->procs, list) {
@@ -343,6 +344,7 @@ static struct toplev {
 	{ "probe",		vcc_ParseProbe },
 	{ "import",		vcc_ParseImport },
 	{ "vcl",		vcc_ParseVcl },
+	{ "default",		NULL },
 	{ NULL, NULL }
 };
 
@@ -392,6 +394,8 @@ vcc_Parse(struct vcc *tl)
 			break;
 		case ID:
 			for (tp = toplev; tp->name != NULL; tp++) {
+				if (tp->func == NULL)
+					continue;
 				if (!vcc_IdIs(tl->t, tp->name))
 					continue;
 				tp->func(tl);
@@ -419,3 +423,12 @@ vcc_Parse(struct vcc *tl)
 	}
 	AZ(tl->indent);
 }
+
+void
+vcc_Parse_Init(struct vcc *tl)
+{
+	struct toplev *tp;
+
+	for (tp = toplev; tp->name != NULL; tp++)
+		VCC_MkSym(tl, tp->name, SYM_NONE);
+}
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index d2cd1bc..1a39ea7 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -200,6 +200,16 @@ VCC_SymbolGet(struct vcc *tl, enum symkind kind, const char *e, const char *x)
 	struct symbol *sym;
 
 	AN(e);
+	if (tl->syntax >= 41 && e == SYMTAB_CREATE && kind != SYM_SUB &&
+	    (tl->t->b[0] == 'v'|| tl->t->b[0] == 'V') &&
+	    (tl->t->b[1] == 'c'|| tl->t->b[1] == 'C') &&
+	    (tl->t->b[2] == 'l'|| tl->t->b[2] == 'L')) {
+		VSB_printf(tl->sb,
+		    "Symbols named 'vcl_*' are reserved.\nAt:");
+		vcc_ErrWhere(tl, tl->t);
+		return (NULL);
+	}
+
 	sym = VCC_Symbol(tl, NULL, tl->t->b, tl->t->e, kind,
 	    e == SYMTAB_CREATE ? 1 : 0);
 	if (sym == NULL && e == SYMTAB_NOERR)
@@ -214,8 +224,11 @@ VCC_SymbolGet(struct vcc *tl, enum symkind kind, const char *e, const char *x)
 	if (kind != SYM_NONE && kind != sym->kind) {
 		VSB_printf(tl->sb, "Symbol ");
 		vcc_ErrToken(tl, tl->t);
-		VSB_printf(tl->sb, " has wrong type (%s): ",
-			VCC_SymKind(tl, sym));
+		if (sym->kind == SYM_NONE)
+			VSB_printf(tl->sb, " is a reserved word.");
+		else
+			VSB_printf(tl->sb, " has wrong type (%s): ",
+				VCC_SymKind(tl, sym));
 		VSB_cat(tl->sb, "\nAt: ");
 		vcc_ErrWhere(tl, tl->t);
 		if (sym->def_b != NULL) {

From dridi at varni.sh  Tue Feb  6 10:09:17 2018
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Tue, 6 Feb 2018 11:09:17 +0100
Subject: [master] 8c2ed01 Propagate the VCL syntax to VRT_CTX.
In-Reply-To: <20180206093413.471A8A0175@lists.varnish-cache.org>
References: <20180206093413.471A8A0175@lists.varnish-cache.org>
Message-ID: <CABoVN9CWEPhPGHbbzj5ygsZ7D6g+fgVBORUzpmEhUPj_PJH7MA@mail.gmail.com>

On Tue, Feb 6, 2018 at 10:34 AM, Poul-Henning Kamp <phk at freebsd.org> wrote:
>
> commit 8c2ed01282a157eaf776e3d1106a93f8d850f5ea
> Author: Poul-Henning Kamp <phk at FreeBSD.org>
> Date:   Tue Feb 6 09:33:09 2018 +0000
>
>     Propagate the VCL syntax to VRT_CTX.
>
>     More work on vcl4.1 reserved words etc.
>
<snip>
> -varnish v1 -errvcl {VCL sub's named 'vcl*' are reserved names.} {
> +varnish v1 -errvcl {The names 'vcl*' are reserved for methods.} {
<snip>

Hi,

Can we avoid vocabulary overloading here? I know that on the C side we
refer to subs as methods (and maybe we should change that) but in the
documentation we call them subroutines.

We already have other user-facing meanings for methods (VCL's
(be)?req.method and VCC's $Method).

Dridi

From phk at phk.freebsd.dk  Tue Feb  6 10:18:45 2018
From: phk at phk.freebsd.dk (Poul-Henning Kamp)
Date: Tue, 06 Feb 2018 10:18:45 +0000
Subject: [master] 8c2ed01 Propagate the VCL syntax to VRT_CTX.
In-Reply-To: <CABoVN9CWEPhPGHbbzj5ygsZ7D6g+fgVBORUzpmEhUPj_PJH7MA@mail.gmail.com>
References: <20180206093413.471A8A0175@lists.varnish-cache.org>
 <CABoVN9CWEPhPGHbbzj5ygsZ7D6g+fgVBORUzpmEhUPj_PJH7MA@mail.gmail.com>
Message-ID: <31619.1517912325@critter.freebsd.dk>

--------
In message <CABoVN9CWEPhPGHbbzj5ygsZ7D6g+fgVBORUzpmEhUPj_PJH7MA at mail.gmail.com>
, Dridi Boukelmoune writes:
>On Tue, Feb 6, 2018 at 10:34 AM, Poul-Henning Kamp <phk at freebsd.org> wrote:

><snip>
>> -varnish v1 -errvcl {VCL sub's named 'vcl*' are reserved names.} {
>> +varnish v1 -errvcl {The names 'vcl*' are reserved for methods.} {
><snip>
>
>Hi,
>
>Can we avoid vocabulary overloading here? I know that on the C side we
>refer to subs as methods (and maybe we should change that) but in the
>documentation we call them subroutines.
>
>We already have other user-facing meanings for methods (VCL's
>(be)?req.method and VCC's $Method).

By all means just fix stuff like that (but remember the VTCs)

-- 
Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
phk at FreeBSD.ORG         | TCP/IP since RFC 956
FreeBSD committer       | BSD since 4.3-tahoe    
Never attribute to malice what can adequately be explained by incompetence.

From dridi.boukelmoune at gmail.com  Tue Feb  6 10:28:07 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue,  6 Feb 2018 10:28:07 +0000 (UTC)
Subject: [master] 20d8489 Refer to VCL subroutines as such in error messages
Message-ID: <20180206102807.D2FB1A32C6@lists.varnish-cache.org>


commit 20d84897a3b47ccbbf324cdc914efb4a6679e401
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Feb 6 11:26:21 2018 +0100

    Refer to VCL subroutines as such in error messages

diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc
index 39defaa..ee2f2a2 100644
--- a/bin/varnishtest/tests/v00020.vtc
+++ b/bin/varnishtest/tests/v00020.vtc
@@ -203,7 +203,7 @@ varnish v1 -errvcl {Name of function, 'foo.bar', contains illegal character '.'}
 	}
 }
 
-varnish v1 -errvcl {The names 'vcl*' are reserved for methods.} {
+varnish v1 -errvcl {The names 'vcl*' are reserved for subroutines.} {
 	sub vcl_bar {
 	}
 	sub vcl_recv {
diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index 0cd13cc..16ba0b7 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -228,9 +228,9 @@ vcc_ParseFunction(struct vcc *tl)
 		    (t->b[1] == 'c'|| t->b[1] == 'C') &&
 		    (t->b[2] == 'l'|| t->b[2] == 'L')) {
 			VSB_printf(tl->sb,
-			    "The names 'vcl*' are reserved for methods.\n");
+			    "The names 'vcl*' are reserved for subroutines.\n");
 			vcc_ErrWhere(tl, t);
-			VSB_printf(tl->sb, "Valid vcl_* methods are:\n");
+			VSB_printf(tl->sb, "Valid vcl_* subroutines are:\n");
 			VTAILQ_FOREACH(p, &tl->procs, list) {
 				if (p->method != NULL)
 					VSB_printf(tl->sb, "\t%s\n",

From dridi at varni.sh  Tue Feb  6 10:28:59 2018
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Tue, 6 Feb 2018 11:28:59 +0100
Subject: [master] 8c2ed01 Propagate the VCL syntax to VRT_CTX.
In-Reply-To: <31619.1517912325@critter.freebsd.dk>
References: <20180206093413.471A8A0175@lists.varnish-cache.org>
 <CABoVN9CWEPhPGHbbzj5ygsZ7D6g+fgVBORUzpmEhUPj_PJH7MA@mail.gmail.com>
 <31619.1517912325@critter.freebsd.dk>
Message-ID: <CABoVN9BqA1otZ0O_dLm+FrCk8vK1qtPU2_nAhcy+msLqRDshfg@mail.gmail.com>

> By all means just fix stuff like that (but remember the VTCs)

Done!

From phk at phk.freebsd.dk  Tue Feb  6 10:30:03 2018
From: phk at phk.freebsd.dk (Poul-Henning Kamp)
Date: Tue, 06 Feb 2018 10:30:03 +0000
Subject: [master] 8c2ed01 Propagate the VCL syntax to VRT_CTX.
In-Reply-To: <CABoVN9CWEPhPGHbbzj5ygsZ7D6g+fgVBORUzpmEhUPj_PJH7MA@mail.gmail.com>
References: <20180206093413.471A8A0175@lists.varnish-cache.org>
 <CABoVN9CWEPhPGHbbzj5ygsZ7D6g+fgVBORUzpmEhUPj_PJH7MA@mail.gmail.com>
Message-ID: <31659.1517913003@critter.freebsd.dk>

--------
In message <CABoVN9CWEPhPGHbbzj5ygsZ7D6g+fgVBORUzpmEhUPj_PJH7MA at mail.gmail.com>
, Dridi Boukelmoune writes:

>Can we avoid vocabulary overloading here? I know that on the C side we
>refer to subs as methods (and maybe we should change that) but in the
>documentation we call them subroutines.

Uhm, thinking more about it:

so if "sub vcl_recv {}" is a subroutine, what is "sub foobar {}"
and how do we explain that "sub vcl_foobar{}" is not OK ?

-- 
Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
phk at FreeBSD.ORG         | TCP/IP since RFC 956
FreeBSD committer       | BSD since 4.3-tahoe    
Never attribute to malice what can adequately be explained by incompetence.

From dridi at varni.sh  Tue Feb  6 10:35:05 2018
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Tue, 6 Feb 2018 11:35:05 +0100
Subject: [master] 8c2ed01 Propagate the VCL syntax to VRT_CTX.
In-Reply-To: <31659.1517913003@critter.freebsd.dk>
References: <20180206093413.471A8A0175@lists.varnish-cache.org>
 <CABoVN9CWEPhPGHbbzj5ygsZ7D6g+fgVBORUzpmEhUPj_PJH7MA@mail.gmail.com>
 <31659.1517913003@critter.freebsd.dk>
Message-ID: <CABoVN9AumWbpMarsoKNqKbayiDiFBmd=b_ApZdEdJ_a+C2w9eA@mail.gmail.com>

On Tue, Feb 6, 2018 at 11:30 AM, Poul-Henning Kamp <phk at phk.freebsd.dk> wrote:
> --------
> In message <CABoVN9CWEPhPGHbbzj5ygsZ7D6g+fgVBORUzpmEhUPj_PJH7MA at mail.gmail.com>
> , Dridi Boukelmoune writes:
>
>>Can we avoid vocabulary overloading here? I know that on the C side we
>>refer to subs as methods (and maybe we should change that) but in the
>>documentation we call them subroutines.
>
> Uhm, thinking more about it:
>
> so if "sub vcl_recv {}" is a subroutine, what is "sub foobar {}"

Also a subroutine.

> and how do we explain that "sub vcl_foobar{}" is not OK ?

That's what the error message you edited does, then it lists the
reserved subroutines.

Arguably we could do this:

-VSB_printf(tl->sb, "Valid vcl_* subroutines are:\n");
+VSB_printf(tl->sb, "Reserved vcl_* subroutines are:\n");

For "The names 'vcl*' are reserved for subroutines.\n" the previous
incarnation did a better job: "VCL sub's named 'vcl*' are reserved
names."

Dridi

From phk at FreeBSD.org  Wed Feb  7 08:30:15 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed,  7 Feb 2018 08:30:15 +0000 (UTC)
Subject: [master] 7251861 Add std.syntax() to check if we run at a certain VCL
 level
Message-ID: <20180207083015.1E636BBE81@lists.varnish-cache.org>


commit 725186156f262c3d52f5355ae86acd6ee83ed3b9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Feb 7 08:21:26 2018 +0000

    Add std.syntax() to check if we run at a certain VCL level

diff --git a/bin/varnishtest/tests/m00001.vtc b/bin/varnishtest/tests/m00001.vtc
index a32a7de..7c79f89 100644
--- a/bin/varnishtest/tests/m00001.vtc
+++ b/bin/varnishtest/tests/m00001.vtc
@@ -5,12 +5,15 @@ server s1 {
 	txresp -hdr "foo: bAr" -hdr "bar: fOo" -bodylen 4
 } -start
 
-varnish v1 -vcl+backend {
+varnish v1 -syntax 4.1 -vcl+backend {
 	import std;
 
 	sub vcl_deliver {
 		set resp.http.foo = std.toupper(resp.http.foo);
 		set resp.http.bar = std.tolower(resp.http.bar);
+		set resp.http.vcl40 = std.syntax(4.0);
+		set resp.http.vcl41 = std.syntax(4.1);
+		set resp.http.vcl42 = std.syntax(4.2);
 		std.set_ip_tos(32);
 	}
 } -start
@@ -24,9 +27,19 @@ client c1 {
 	expect resp.bodylen == "4"
 	expect resp.http.foo == "BAR"
 	expect resp.http.bar == "foo"
+	expect resp.http.vcl40 == "true"
+	expect resp.http.vcl41 == "true"
+	expect resp.http.vcl42 == "false"
 } -run
 
-varnish v1 -vcl+backend {
+varnish v1 -syntax 4.0 -vcl+backend {
+	import std;
+
+	sub vcl_deliver {
+		set resp.http.vcl40 = std.syntax(4.0);
+		set resp.http.vcl41 = std.syntax(4.1);
+		set resp.http.vcl42 = std.syntax(4.2);
+	}
 }
 
 client c1 {
@@ -36,14 +49,20 @@ client c1 {
 	expect resp.bodylen == "4"
 	expect resp.http.foo == "bAr"
 	expect resp.http.bar == "fOo"
+	expect resp.http.vcl40 == "true"
+	expect resp.http.vcl41 == "false"
+	expect resp.http.vcl42 == "false"
 } -run
 
+varnish v1 -vcl+backend { }
+
 varnish v1 -cliok "debug.vmod"
 varnish v1 -cliok "vcl.list"
 
 varnish v1 -expect vmods == 1
 
 varnish v1 -cliok "vcl.discard vcl1"
+varnish v1 -cliok "vcl.discard vcl2"
 varnish v1 -cliok "vcl.list"
 varnish v1 -cliok "debug.vmod"
 
diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index 7c2ed1d..e7c13a4 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -338,6 +338,11 @@ Example
 	|	...
 	| }
 
+$Function BOOL issyntax(REAL)
+
+Description
+	Returns the true if VCL version is at least REAL.
+
 SEE ALSO
 ========
 
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index 414a521..efd6d84 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -285,3 +285,17 @@ vmod_late_100_continue(VRT_CTX, VCL_BOOL late)
 	if (ctx->req->want100cont)
 		ctx->req->late100cont = late;
 }
+
+VCL_BOOL v_matchproto_(td_std_syntax)
+vmod_syntax(VRT_CTX, VCL_REAL r)
+{
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	assert(ctx->syntax == 40 || ctx->syntax == 41);
+	/*
+	 * We need to be careful because non-integer numbers have imprecise
+	 * IEE754 represenation (4.1 is 0x1.0666666666666p+2 = 4.09999...)
+	 * By scaling up and rounding, this is taken care of.
+	 */
+	return (round(r * 10) <= ctx->syntax);
+}

From phk at FreeBSD.org  Wed Feb  7 09:03:05 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed,  7 Feb 2018 09:03:05 +0000 (UTC)
Subject: [master] 1506c55 Forgot to change my mind all over the place.
Message-ID: <20180207090305.B2801BC74D@lists.varnish-cache.org>


commit 1506c55872ac31dee1a7884e5631bebd505351e0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Feb 7 09:02:18 2018 +0000

    Forgot to change my mind all over the place.

diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index e7c13a4..16dd85f 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -338,7 +338,7 @@ Example
 	|	...
 	| }
 
-$Function BOOL issyntax(REAL)
+$Function BOOL syntax(REAL)
 
 Description
 	Returns the true if VCL version is at least REAL.

From phk at FreeBSD.org  Thu Feb  8 10:12:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu,  8 Feb 2018 10:12:06 +0000 (UTC)
Subject: [master] c2cef8f Make VRT_String() mark the workspace as overflowed
 and VRT_CollectString() fail the VCL on overflows.
Message-ID: <20180208101207.00E74B521D@lists.varnish-cache.org>


commit c2cef8fc51d0c09bcef63aa0660f0dd894f138de
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Feb 8 09:19:05 2018 +0000

    Make VRT_String() mark the workspace as overflowed and
    VRT_CollectString() fail the VCL on overflows.
    
    VRT_StringList() can be used in the rare and probably
    non-existent cases where overflow-marking is undesired.

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index b0f5619..cbb4f88 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -268,10 +268,7 @@ VRT_String(struct ws *ws, const char *h, const char *p, va_list ap)
 	}
 	b = VRT_StringList(b, e > b ? e - b : 0, p, ap);
 	if (b == NULL || b == e) {
-		/*
-		 * NO WS_MarkOverflow here because the caller might have a
-		 * fallback
-		 */
+		WS_MarkOverflow(ws);
 		WS_Release(ws, 0);
 		return (NULL);
 	}
@@ -296,6 +293,8 @@ VRT_CollectString(VRT_CTX, const char *p, ...)
 	va_start(ap, p);
 	b = VRT_String(ctx->ws, NULL, p, ap);
 	va_end(ap);
+	if (b == NULL)
+		VRT_fail(ctx, "Workspace overflow");
 	return (b);
 }
 

From phk at FreeBSD.org  Thu Feb  8 10:12:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu,  8 Feb 2018 10:12:07 +0000 (UTC)
Subject: [master] 5969b8a Optimize VRT_String() to not copy if the result is
 already present in the desired workspace.
Message-ID: <20180208101207.18F68B5220@lists.varnish-cache.org>


commit 5969b8adc213864d5ef3ae241d4d1459250f1db3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Feb 8 10:09:33 2018 +0000

    Optimize VRT_String() to not copy if the result is already
    present in the desired workspace.
    
    I'm a little bit uneasy about the tacit assumption that arguments
    to VRT_String() now belong to VRT_String(), but we'll see how it
    goes.
    
    Fixes: #2132

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index cbb4f88..296fa45 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -252,11 +252,51 @@ const char *
 VRT_String(struct ws *ws, const char *h, const char *p, va_list ap)
 {
 	char *b, *e;
+	const char *q;
 	unsigned u, x;
+	va_list aq;
 
 	u = WS_Reserve(ws, 0);
 	e = b = ws->f;
 	e += u;
+
+	va_copy(aq, ap);
+	do
+		q = va_arg(aq, const char *);
+	while (q == NULL || (q != vrt_magic_string_end && *q == '\0'));
+
+	if (h != NULL && p == NULL && q == vrt_magic_string_end &&
+	    WS_Inside(ws, h, NULL)) {
+		va_end(aq);
+		WS_Release(ws, 0);
+		return (h);
+	}
+
+	if (h == NULL && p != NULL && q == vrt_magic_string_end &&
+	    WS_Inside(ws, p, NULL)) {
+		va_end(aq);
+		WS_Release(ws, 0);
+		return (p);
+	}
+
+	if (h == NULL && p == NULL) {
+		if (q == vrt_magic_string_end) {
+			va_end(aq);
+			WS_Release(ws, 0);
+			return ("");
+		}
+		do
+			p = va_arg(aq, const char *);
+		while (p == NULL || (p != vrt_magic_string_end && *p == '\0'));
+		if (p == vrt_magic_string_end && WS_Inside(ws, q, NULL)) {
+			va_end(aq);
+			WS_Release(ws, 0);
+			return (q);
+		}
+		p = NULL;
+		va_end(aq);
+	}
+
 	if (h != NULL) {
 		x = strlen(h);
 		if (b + x < e)
diff --git a/bin/varnishtest/tests/c00074.vtc b/bin/varnishtest/tests/c00074.vtc
index abf886a..2231ba1 100644
--- a/bin/varnishtest/tests/c00074.vtc
+++ b/bin/varnishtest/tests/c00074.vtc
@@ -16,6 +16,10 @@ varnish v1 -vcl+backend {
 		vtc.workspace_snapshot(session);
 		vtc.workspace_reset(session);
 		set req.http.ws-free3 = vtc.workspace_free(session);
+
+		# Cover an obscure case in VRT_String() while here
+		set req.http.bar = vtc.typesize(
+			req.http.h1 + req.http.h2 + req.http.h3);
 	}
 } -start
 
@@ -26,7 +30,7 @@ logexpect l1 -v v1 -g request {
 } -start
 
 client c1 {
-	txreq -url /foo
+	txreq -url /foo -hdr "h3: dd"
 	rxresp
 	expect resp.status == 200
 } -run
diff --git a/bin/varnishtest/tests/r00781.vtc b/bin/varnishtest/tests/r00781.vtc
index 2c3562a..0660583 100644
--- a/bin/varnishtest/tests/r00781.vtc
+++ b/bin/varnishtest/tests/r00781.vtc
@@ -9,12 +9,24 @@ varnish v1 -vcl+backend {
 	sub vcl_recv {
 		set req.url = req.http.foo;
 	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 503
+	expect resp.reason == "VCL failed"
+} -run
+
+varnish v1 -vcl+backend {
 	sub vcl_backend_fetch {
 		set bereq.url = bereq.http.foo;
 	}
-} -start
+}
 
 client c1 {
 	txreq
 	rxresp
+	expect resp.status == 503
+	expect resp.reason == "Service Unavailable"
 } -run

From geoff at uplex.de  Sat Feb 10 11:10:14 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Sat, 10 Feb 2018 11:10:14 +0000 (UTC)
Subject: [master] f06ed26 Rewrite the example for std.fileread(),
 and verify it with a test.
Message-ID: <20180210111014.87A2F9B2FA@lists.varnish-cache.org>


commit f06ed2639b3b2cc2bf9fbb7572d8df7dd47f3cca
Author: Geoff Simmons <geoff at uplex.de>
Date:   Sat Feb 10 12:07:53 2018 +0100

    Rewrite the example for std.fileread(), and verify it with a test.
    
    After it was reported on #varnish that following the previous
    example may lead to invalid response headers.

diff --git a/bin/varnishtest/tests/m00004.vtc b/bin/varnishtest/tests/m00004.vtc
index bb3ba56..70bf0ff 100644
--- a/bin/varnishtest/tests/m00004.vtc
+++ b/bin/varnishtest/tests/m00004.vtc
@@ -4,10 +4,11 @@ shell {
 	printf "File One" > "${tmpdir}/m00004_file_one"
 	printf "File Two" > "${tmpdir}/m00004_file_two"
 	printf "File Three" > "${tmpdir}/m00004_file_three"
+	printf "File Four\n" > "${tmpdir}/m00004_file_four"
 }
 
 server s1 {
-	loop 3 {
+	loop 4 {
 		rxreq
 		txresp -hdr "foo: bar" -bodylen 4
 	}
@@ -16,6 +17,20 @@ server s1 {
 varnish v1 -vcl+backend {
 	import std;
 
+	sub vcl_recv {
+		if (req.url == "/five") {
+			return(synth(200));
+		}
+	}
+
+	sub vcl_backend_response {
+		if (bereq.url == "/four") {
+			set beresp.http.served-by
+			    = regsub(std.fileread("${tmpdir}/m00004_file_four"),
+			             "\R$", "");
+		}
+	}
+
 	sub vcl_deliver {
 		if (req.url == "/one") {
 			set resp.http.one = std.fileread("${tmpdir}/m00004_file_one");
@@ -25,6 +40,12 @@ varnish v1 -vcl+backend {
 			set resp.http.three = std.fileread("${tmpdir}/m00004_file_three");
 		}
 	}
+
+	sub vcl_synth {
+		synthetic("Response was served by "
+		          + std.fileread("${tmpdir}/m00004_file_four"));
+		return(deliver);
+	}
 } -start
 
 client c1 {
@@ -48,11 +69,37 @@ client c1 {
 	expect resp.bodylen == "4"
 	expect resp.http.foo == "bar"
 	expect resp.http.three == "File Three"
+
+	txreq -url "/four"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == "4"
+	expect resp.http.foo == "bar"
+	expect resp.http.served-by == "File Four"
+
+	txreq -url "/five"
+	rxresp
+	expect resp.status == 200
+	expect resp.body == "Response was served by File Four\n"
 } -run
 
 varnish v1 -vcl+backend {
 	import std;
 
+	sub vcl_recv {
+		if (req.url == "/five") {
+			return(synth(200));
+		}
+	}
+
+	sub vcl_backend_response {
+		if (bereq.url == "/four") {
+			set beresp.http.served-by
+			    = regsub(std.fileread("${tmpdir}/m00004_file_four"),
+			             "\R$", "");
+		}
+	}
+
 	sub vcl_deliver {
 		if (req.url == "/one") {
 			set resp.http.one = std.fileread("${tmpdir}/m00004_file_one");
@@ -62,6 +109,12 @@ varnish v1 -vcl+backend {
 			set resp.http.three = std.fileread("${tmpdir}/m00004_file_three");
 		}
 	}
+
+	sub vcl_synth {
+		synthetic("Response was served by "
+		          + std.fileread("${tmpdir}/m00004_file_four"));
+		return(deliver);
+	}
 }
 
 varnish v1 -cli "vcl.list"
diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index 16dd85f..178b0b7 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -111,7 +111,14 @@ Description
 	function the caching in the function doesn't take this into
 	account. Also, files are not re-read.
 Example
-	set beresp.http.served-by = std.fileread("/etc/hostname");
+	synthetic("Response was served by " + std.fileread("/etc/hostname"));
+
+Consider that the entire contents of the file appear in the string
+that is returned, including newlines that may result in invalid
+headers if ``std.fileread()`` is used to form a header. In that case,
+you may need to modify the string, for example with ``regsub()``::
+
+  set beresp.http.served-by = regsub(std.fileread("/etc/hostname"), "\R$", "");
 
 $Function BOOL file_exists(STRING path)
 

From phk at FreeBSD.org  Mon Feb 12 08:45:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 12 Feb 2018 08:45:08 +0000 (UTC)
Subject: [master] b37ea99 Whitespace ocd
Message-ID: <20180212084508.8A958B1AE5@lists.varnish-cache.org>


commit b37ea9928b3cc7d294b8760fb1eede64c841e477
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Feb 12 08:44:30 2018 +0000

    Whitespace ocd

diff --git a/bin/varnishtest/tests/m00004.vtc b/bin/varnishtest/tests/m00004.vtc
index 70bf0ff..f554a01 100644
--- a/bin/varnishtest/tests/m00004.vtc
+++ b/bin/varnishtest/tests/m00004.vtc
@@ -27,7 +27,7 @@ varnish v1 -vcl+backend {
 		if (bereq.url == "/four") {
 			set beresp.http.served-by
 			    = regsub(std.fileread("${tmpdir}/m00004_file_four"),
-			             "\R$", "");
+				     "\R$", "");
 		}
 	}
 
@@ -43,7 +43,7 @@ varnish v1 -vcl+backend {
 
 	sub vcl_synth {
 		synthetic("Response was served by "
-		          + std.fileread("${tmpdir}/m00004_file_four"));
+			  + std.fileread("${tmpdir}/m00004_file_four"));
 		return(deliver);
 	}
 } -start
@@ -96,7 +96,7 @@ varnish v1 -vcl+backend {
 		if (bereq.url == "/four") {
 			set beresp.http.served-by
 			    = regsub(std.fileread("${tmpdir}/m00004_file_four"),
-			             "\R$", "");
+				     "\R$", "");
 		}
 	}
 
@@ -112,7 +112,7 @@ varnish v1 -vcl+backend {
 
 	sub vcl_synth {
 		synthetic("Response was served by "
-		          + std.fileread("${tmpdir}/m00004_file_four"));
+			  + std.fileread("${tmpdir}/m00004_file_four"));
 		return(deliver);
 	}
 }

From guillaume at varnish-software.com  Mon Feb 12 10:32:05 2018
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Mon, 12 Feb 2018 10:32:05 +0000 (UTC)
Subject: [master] 918c848 Typo
Message-ID: <20180212103205.84E7AB3D3B@lists.varnish-cache.org>


commit 918c848d12d4e9eec3b61196358d6fb5a49690ce
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Mon Feb 12 11:31:38 2018 +0100

    Typo

diff --git a/doc/changes.rst b/doc/changes.rst
index cb9fe31..a82cacc 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -55,7 +55,7 @@ Logging / statistics
 * Turned off PROXY protocol debugging by default, can be enabled with
   the ``protocol`` debug flag
 
-* added ``cache_hit_grace`` statisctics counter
+* added ``cache_hit_grace`` statistics counter
 
 C APIs (for vmod and utility authors)
 -------------------------------------

From phk at FreeBSD.org  Tue Feb 13 08:42:12 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 13 Feb 2018 08:42:12 +0000 (UTC)
Subject: [master] a89e93a Don't allow whitespace in ESI:include src=
 attributes.
Message-ID: <20180213084212.2B7BDA6154@lists.varnish-cache.org>


commit a89e93af056bdff2f29b0c244040a4bdead143fb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 13 08:41:00 2018 +0000

    Don't allow whitespace in ESI:include src= attributes.

diff --git a/bin/varnishd/cache/cache_esi_parse.c b/bin/varnishd/cache/cache_esi_parse.c
index 1fd6e53..ef72eeb 100644
--- a/bin/varnishd/cache/cache_esi_parse.c
+++ b/bin/varnishd/cache/cache_esi_parse.c
@@ -445,6 +445,19 @@ vep_do_include(struct vep_state *vep, enum dowhat what)
 			VSB_destroy(&vep->include_src);
 			return;
 		}
+		for (p = VSB_data(vep->attr_vsb); *p != '\0'; p++)
+			if (vct_islws(*p))
+				break;
+		if (*p != '\0') {
+			vep_error(vep,
+			    "ESI 1.0 <esi:include> "
+			    "has whitespace in src= attribute");
+			vep->state = VEP_TAGERROR;
+			VSB_destroy(&vep->attr_vsb);
+			if (vep->include_src != NULL)
+				VSB_destroy(&vep->include_src);
+			return;
+		}
 		vep->include_src = vep->attr_vsb;
 		vep->attr_vsb = NULL;
 		return;
diff --git a/bin/varnishtest/tests/e00008.vtc b/bin/varnishtest/tests/e00008.vtc
index 71b700a..fce1345 100644
--- a/bin/varnishtest/tests/e00008.vtc
+++ b/bin/varnishtest/tests/e00008.vtc
@@ -41,6 +41,7 @@ server s1 {
 		</![CDATA[ bogo-cdata ]]>		35
 		<esi:include  src="foofof />		36
 		<esi:include  foo=bar src=/body2 />	37
+		<esi:include  src="/body 2" />		38
 	}
 	rxreq
 	expect req.url == "/body"
@@ -81,6 +82,7 @@ logexpect l1 -v v1 -g vxid {
 	expect 0 = ESI_xmlerror {^ERR after 636 XML 1.0 Illegal attribute delimiter$}
 	expect 0 = ESI_xmlerror {^ERR after 665 ESI 1.0 </esi:include> illegal end-tag$}
 	expect 0 = ESI_xmlerror {^ERR after 767 XML 1.0 Missing end attribute delimiter$}
+	expect 0 = ESI_xmlerror {^ERR after 843 ESI 1.0 <esi:include> has whitespace in src= attribute$}
 	expect 0 = BackendReuse
 } -start
 
@@ -104,11 +106,11 @@ client c1 {
 	txreq
 	rxresp
 	expect resp.status == 200
-	expect resp.bodylen == 465
+	expect resp.bodylen == 472
 }
 
 client c1 -run
 logexpect l1 -wait
 logexpect l2 -wait
 logexpect l3 -wait
-varnish v1 -expect esi_errors == 17
+varnish v1 -expect esi_errors == 18

From phk at FreeBSD.org  Tue Feb 13 10:26:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 13 Feb 2018 10:26:09 +0000 (UTC)
Subject: [master] f8cf8c7 Rename sess_attr xport_priv to proto_priv because it
 is H1 & H2 which uses it.
Message-ID: <20180213102609.338A8A8214@lists.varnish-cache.org>


commit f8cf8c77bd0202c54607e5cf1dad513d2cf1c8f2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 13 10:23:00 2018 +0000

    Rename sess_attr xport_priv to proto_priv because it is H1 & H2
    which uses it.
    
    Add a new sess_attr xport_priv which PROXY can use to keep state,
    for instance for TLV attributes (see #2561)

diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c
index c4cda48..9109f9b 100644
--- a/bin/varnishd/http1/cache_http1_fsm.c
+++ b/bin/varnishd/http1/cache_http1_fsm.c
@@ -58,7 +58,7 @@ http1_setstate(const struct sess *sp, const char *s)
 	uintptr_t p;
 
 	p = (uintptr_t)s;
-	AZ(SES_Set_xport_priv(sp, &p));
+	AZ(SES_Set_proto_priv(sp, &p));
 }
 
 static const char *
@@ -66,7 +66,7 @@ http1_getstate(const struct sess *sp)
 {
 	uintptr_t *p;
 
-	AZ(SES_Get_xport_priv(sp, &p));
+	AZ(SES_Get_proto_priv(sp, &p));
 	return (const char *)*p;
 }
 
@@ -114,7 +114,7 @@ http1_new_session(struct worker *wrk, void *arg)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 
 	HTC_RxInit(req->htc, req->ws);
-	SES_Reserve_xport_priv(sp, &u);
+	SES_Reserve_proto_priv(sp, &u);
 	http1_setstate(sp, H1NEWREQ);
 	wrk->task.func = http1_req;
 	wrk->task.priv = req;
diff --git a/bin/varnishd/http2/cache_http2_panic.c b/bin/varnishd/http2/cache_http2_panic.c
index da8c5eb..26bb42f 100644
--- a/bin/varnishd/http2/cache_http2_panic.c
+++ b/bin/varnishd/http2/cache_http2_panic.c
@@ -41,7 +41,7 @@ h2_sess_panic(struct vsb *vsb, const struct sess *sp)
 	struct h2_sess *h2;
 	struct h2_req *r2;
 
-	AZ(SES_Get_xport_priv(sp, &up));
+	AZ(SES_Get_proto_priv(sp, &up));
 
 	h2 = (void*)*up;
 	CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC);
diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c
index 599f908..6fe7bf4 100644
--- a/bin/varnishd/http2/cache_http2_session.c
+++ b/bin/varnishd/http2/cache_http2_session.c
@@ -102,9 +102,9 @@ h2_new_sess(const struct worker *wrk, struct sess *sp, struct req *srq)
 	uintptr_t *up;
 	struct h2_sess *h2;
 
-	if (SES_Get_xport_priv(sp, &up)) {
+	if (SES_Get_proto_priv(sp, &up)) {
 		/* Already reserved if we came via H1 */
-		SES_Reserve_xport_priv(sp, &up);
+		SES_Reserve_proto_priv(sp, &up);
 		*up = 0;
 	}
 	if (*up == 0) {
@@ -131,7 +131,7 @@ h2_new_sess(const struct worker *wrk, struct sess *sp, struct req *srq)
 		AZ(VHT_Init(h2->dectbl,
 			h2->local_settings.header_table_size));
 
-		SES_Reserve_xport_priv(sp, &up);
+		SES_Reserve_proto_priv(sp, &up);
 		*up = (uintptr_t)h2;
 	}
 	AN(up);
diff --git a/include/tbl/sess_attr.h b/include/tbl/sess_attr.h
index 6b4494f..6b83fb3 100644
--- a/include/tbl/sess_attr.h
+++ b/include/tbl/sess_attr.h
@@ -40,6 +40,7 @@ SESS_ATTR(SERVER_ADDR,	  server_addr,	struct suckaddr,    vsa_suckaddr_len)
 SESS_ATTR(CLIENT_IP,	  client_ip,	char,		    -1)
 SESS_ATTR(CLIENT_PORT,	  client_port,	char,		    -1)
 SESS_ATTR(XPORT_PRIV,	  xport_priv,	uintptr_t,	    sizeof(uintptr_t))
+SESS_ATTR(PROTO_PRIV,	  proto_priv,	uintptr_t,	    sizeof(uintptr_t))
 #undef SESS_ATTR
 
 /*lint -restore */

From phk at FreeBSD.org  Tue Feb 13 13:34:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 13 Feb 2018 13:34:08 +0000 (UTC)
Subject: [master] 0d0bac7 Polish
Message-ID: <20180213133408.C5AA5ADB65@lists.varnish-cache.org>


commit 0d0bac740f40d92b78777c2cf99647d7c6ec1f74
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 13 13:33:22 2018 +0000

    Polish

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index e2fecec..bb1f000 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -374,10 +374,7 @@ vca_make_session(struct worker *wrk, void *arg)
 	VSL(SLT_Begin, sp->vxid, "sess 0 %s",
 	    wa->acceptlsock->transport->name);
 	VSL(SLT_SessOpen, sp->vxid, "%s %s %s %s %s %.6f %d",
-	    raddr, rport,
-	    wa->acceptlsock->name != NULL ?
-		wa->acceptlsock->name : wa->acceptlsock->endpoint,
-	    laddr, lport,
+	    raddr, rport, wa->acceptlsock->name, laddr, lport,
 	    sp->t_open, sp->fd);
 
 	WS_Release(wrk->aws, 0);
diff --git a/bin/varnishd/common/heritage.h b/bin/varnishd/common/heritage.h
index 9e26079..e59261e 100644
--- a/bin/varnishd/common/heritage.h
+++ b/bin/varnishd/common/heritage.h
@@ -36,16 +36,6 @@ struct listen_sock;
 struct transport;
 struct VCLS;
 
-struct listen_arg {
-	unsigned			magic;
-#define LISTEN_ARG_MAGIC		0xbb2fc333
-	VTAILQ_ENTRY(listen_arg)	list;
-	const char			*endpoint;
-	const char			*name;
-	VTAILQ_HEAD(,listen_sock)	socks;
-	const struct transport		*transport;
-};
-
 struct listen_sock {
 	unsigned			magic;
 #define LISTEN_SOCK_MAGIC		0x999e4b57
diff --git a/bin/varnishd/mgt/mgt_acceptor.c b/bin/varnishd/mgt/mgt_acceptor.c
index 0006716..af09013 100644
--- a/bin/varnishd/mgt/mgt_acceptor.c
+++ b/bin/varnishd/mgt/mgt_acceptor.c
@@ -48,6 +48,16 @@
 #include "vss.h"
 #include "vtcp.h"
 
+struct listen_arg {
+	unsigned			magic;
+#define LISTEN_ARG_MAGIC		0xbb2fc333
+	VTAILQ_ENTRY(listen_arg)	list;
+	const char			*endpoint;
+	const char			*name;
+	VTAILQ_HEAD(,listen_sock)	socks;
+	const struct transport		*transport;
+};
+
 static VTAILQ_HEAD(,listen_arg) listen_args =
     VTAILQ_HEAD_INITIALIZER(listen_args);
 

From hermunn at varnish-software.com  Wed Feb 14 14:45:17 2018
From: hermunn at varnish-software.com (Pål Hermunn Johansen)
Date: Wed, 14 Feb 2018 14:45:17 +0000 (UTC)
Subject: [4.1] dceb4cf Make sure that oc->last_lru has sufficient precision
Message-ID: <20180214144517.ABFBBA44A9@lists.varnish-cache.org>


commit dceb4cf6379e99255ca38a79c4f65f20230b1174
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Jan 8 16:05:05 2018 +0100

    Make sure that oc->last_lru has sufficient precision
    
    Since the last_lru tracks epoch time, it needs the double precision
    floating point type to accurately track the time.
    
    This is simply the test case from 2261dcfd545e6c9a5d60000cce06e993.

diff --git a/bin/varnishtest/tests/r02527.vtc b/bin/varnishtest/tests/r02527.vtc
new file mode 100644
index 0000000..ae8ed03
--- /dev/null
+++ b/bin/varnishtest/tests/r02527.vtc
@@ -0,0 +1,30 @@
+varnishtest "Test the n_lru_moved counter"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -arg "-p lru_interval=1" -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.do_stream = false;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+} -run
+
+varnish v1 -expect MAIN.n_lru_moved == 0
+
+delay 1
+
+client c1 {
+	txreq
+	rxresp
+	txreq
+	rxresp
+} -run
+
+varnish v1 -expect MAIN.n_lru_moved == 1

From phk at FreeBSD.org  Thu Feb 15 09:29:13 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 15 Feb 2018 09:29:13 +0000 (UTC)
Subject: [master] 05a9507 Call types types instead of formats
Message-ID: <20180215092913.244F9BA011@lists.varnish-cache.org>


commit 05a9507205624434378603b6ddfbcaeccf9ce6eb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Feb 15 08:53:43 2018 +0000

    Call types types instead of formats

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index b75ae57..1eb857e 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -1272,7 +1272,7 @@ def one_var(nm, spec):
 	else:
 		fo.write(" SYM_VAR);\n")
 	fo.write("\tAN(sym);\n")
-	fo.write("\tsym->fmt = %s;\n" % spec.typ)
+	fo.write("\tsym->type = %s;\n" % spec.typ)
 	fo.write("\tsym->eval = vcc_Eval_Var;\n")
 
 	if len(spec.rd) == 0:
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 1a4d755..5447e8c 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -88,7 +88,7 @@ static void v_matchproto_(sym_act_f)
 vcc_act_set(struct vcc *tl, struct token *t, struct symbol *sym)
 {
 	const struct arith *ap;
-	vcc_type_t fmt;
+	vcc_type_t type;
 
 	(void)t;
 	ExpectErr(tl, ID);
@@ -107,28 +107,28 @@ vcc_act_set(struct vcc *tl, struct token *t, struct symbol *sym)
 	vcc_AddUses(tl, t, tl->t, sym->w_methods, "Cannot be set");
 	Fb(tl, 1, "%s\n", sym->lname);
 	tl->indent += INDENT;
-	fmt = sym->fmt;
+	type = sym->type;
 	for (ap = arith; ap->type != VOID; ap++) {
-		if (ap->type != fmt)
+		if (ap->type != type)
 			continue;
 		if (ap->oper != tl->t->tok)
 			continue;
 		if (ap->oper != '=')
 			Fb(tl, 1, "%s %c ", sym->rname, *tl->t->b);
 		vcc_NextToken(tl);
-		fmt = ap->want;
+		type = ap->want;
 		break;
 	}
 	if (ap->type == VOID)
 		SkipToken(tl, ap->oper);
-	if (fmt == HEADER) {
+	if (type == HEADER) {
 		vcc_Expr(tl, STRING_LIST);
-	} else if (fmt == STRING) {
+	} else if (type == STRING) {
 		vcc_Expr(tl, STRING_LIST);
-	} else if (fmt == BODY) {
+	} else if (type == BODY) {
 		vcc_Expr(tl, STRING_LIST);
 	} else {
-		vcc_Expr(tl, fmt);
+		vcc_Expr(tl, type);
 	}
 	ERRCHK(tl);
 	tl->indent -= INDENT;
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 55bcd7c..ea8fdf8 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -811,7 +811,7 @@ vcc_predef_vcl(struct vcc *vcc, const char *name)
 
 	sym = VCC_MkSym(vcc, name, SYM_VCL);
 	AN(sym);
-	sym->fmt = VCL;
+	sym->type = VCL;
 	sym->r_methods = VCL_MET_RECV;
 }
 
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index d3f6b7a..479731f 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -129,7 +129,7 @@ struct symbol {
 
 	const struct token		*def_b, *def_e, *ref_b;
 
-	vcc_type_t			fmt;
+	vcc_type_t			type;
 
 	sym_expr_t			*eval;
 	const void			*eval_priv;
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 093e2b1..1e55b83 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -328,19 +328,19 @@ vcc_Eval_BoolConst(struct vcc *tl, struct expr **e, struct token *t,
 
 void v_matchproto_(sym_expr_t)
 vcc_Eval_Handle(struct vcc *tl, struct expr **e, struct token *t,
-    struct symbol *sym, vcc_type_t fmt)
+    struct symbol *sym, vcc_type_t type)
 {
 
 	(void)t;
 	(void)tl;
 	AN(sym->rname);
 
-	if (sym->fmt != STRING && fmt == STRINGS) {
+	if (sym->type != STRING && type == STRINGS) {
 		*e = vcc_mk_expr(STRINGS, "\"%s\"", sym->name);
 		(*e)->nstr = 1;
 		(*e)->constant |= EXPR_CONST | EXPR_STR_CONST;
 	} else {
-		*e = vcc_mk_expr(sym->fmt, "%s", sym->rname);
+		*e = vcc_mk_expr(sym->type, "%s", sym->rname);
 		(*e)->constant = EXPR_VAR;
 		(*e)->nstr = 1;
 		if ((*e)->fmt == STRING)
@@ -353,14 +353,14 @@ vcc_Eval_Handle(struct vcc *tl, struct expr **e, struct token *t,
 
 void v_matchproto_(sym_expr_t)
 vcc_Eval_Var(struct vcc *tl, struct expr **e, struct token *t,
-    struct symbol *sym, vcc_type_t fmt)
+    struct symbol *sym, vcc_type_t type)
 {
 
-	(void)fmt;
+	(void)type;
 	assert(sym->kind == SYM_VAR);
 	vcc_AddUses(tl, t, NULL, sym->r_methods, "Not available");
 	ERRCHK(tl);
-	*e = vcc_mk_expr(sym->fmt, "%s", sym->rname);
+	*e = vcc_mk_expr(sym->type, "%s", sym->rname);
 	(*e)->constant = EXPR_VAR;
 	(*e)->nstr = 1;
 	if ((*e)->fmt == STRING)
@@ -664,18 +664,12 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 		    XREF_REF);
 		ERRCHK(tl);
 		AN(sym);
-		if (sym->kind == SYM_FUNC && sym->fmt == VOID) {
+		if (sym->kind == SYM_FUNC && sym->type == VOID) {
 			VSB_printf(tl->sb, "Function returns VOID:\n");
 			vcc_ErrWhere(tl, tl->t);
 			return;
 		}
-		switch (sym->kind) {
-		case SYM_VAR:
-		case SYM_FUNC:
-		case SYM_ACL:
-		case SYM_BACKEND:
-		case SYM_STEVEDORE:
-		case SYM_PROBE:
+		if (sym->eval != NULL) {
 			AN(sym->eval);
 			AZ(*e);
 			sym->eval(tl, e, t, sym, fmt);
@@ -686,9 +680,6 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 				ERRCHK(tl);
 			}
 			return;
-		default:
-			AZ(sym->eval);
-			break;
 		}
 		VSB_printf(tl->sb,
 		    "Symbol type (%s) can not be used in expression.\n",
@@ -1291,21 +1282,25 @@ vcc_Expr_Init(struct vcc *tl)
 
 	sym = VCC_MkSym(tl, "regsub", SYM_FUNC);
 	AN(sym);
+	sym->type = STRING;
 	sym->eval = vcc_Eval_Regsub;
 	sym->eval_priv = NULL;
 
 	sym = VCC_MkSym(tl, "regsuball", SYM_FUNC);
 	AN(sym);
+	sym->type = STRING;
 	sym->eval = vcc_Eval_Regsub;
 	sym->eval_priv = sym;
 
 	sym = VCC_MkSym(tl, "true", SYM_FUNC);
 	AN(sym);
+	sym->type = BOOL;
 	sym->eval = vcc_Eval_BoolConst;
 	sym->eval_priv = sym;
 
 	sym = VCC_MkSym(tl, "false", SYM_FUNC);
 	AN(sym);
+	sym->type = BOOL;
 	sym->eval = vcc_Eval_BoolConst;
 	sym->eval_priv = NULL;
 }
diff --git a/lib/libvcc/vcc_storage.c b/lib/libvcc/vcc_storage.c
index 8378234..06af8c7 100644
--- a/lib/libvcc/vcc_storage.c
+++ b/lib/libvcc/vcc_storage.c
@@ -67,7 +67,7 @@
 
 static struct stvars {
 	const char	*name;
-	vcc_type_t	fmt;
+	vcc_type_t	type;
 } stvars[] = {
 #define VRTSTVVAR(nm, vtype, ctype, dval)	{ #nm, vtype },
 #include "tbl/vrt_stv_var.h"
@@ -86,7 +86,7 @@ vcc_stevedore(struct vcc *vcc, const char *stv_name)
 	bprintf(buf, "storage.%s", stv_name);
 	sym = VCC_MkSym(vcc, buf, SYM_VAR);
 	AN(sym);
-	sym->fmt = STEVEDORE;
+	sym->type = STEVEDORE;
 	sym->eval = vcc_Eval_Var;
 	bprintf(buf, "VRT_stevedore(\"%s\")", stv_name);
 	sym->rname = TlDup(vcc, buf);
@@ -96,7 +96,7 @@ vcc_stevedore(struct vcc *vcc, const char *stv_name)
 		bprintf(buf, "storage.%s.%s", stv_name, sv->name);
 		sym = VCC_MkSym(vcc, buf, SYM_VAR);
 		AN(sym);
-		sym->fmt = sv->fmt;
+		sym->type = sv->type;
 		sym->eval = vcc_Eval_Var;
 		bprintf(buf, "VRT_Stv_%s(\"%s\")", sv->name, stv_name);
 		sym->rname = TlDup(vcc, buf);
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index 1a39ea7..6f73b16 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -101,6 +101,7 @@ vcc_new_symbol(struct vcc *tl, const char *b, const char *e)
 	sym->name[e - b] = '\0';
 	sym->nlen = e - b;
 	VTAILQ_INIT(&sym->children);
+	sym->type = VOID;
 	return (sym);
 }
 
@@ -290,7 +291,7 @@ VCC_WalkSymbols(struct vcc *tl, symwalk_f *func, enum symkind kind)
 }
 
 void
-VCC_GlobalSymbol(struct symbol *sym, vcc_type_t fmt, const char *pfx)
+VCC_GlobalSymbol(struct symbol *sym, vcc_type_t type, const char *pfx)
 {
 	struct vsb *vsb;
 
@@ -306,8 +307,8 @@ VCC_GlobalSymbol(struct symbol *sym, vcc_type_t fmt, const char *pfx)
 	AN(sym->rname);
 	VSB_destroy(&vsb);
 
-	sym->fmt = fmt;
-	sym->kind = VCC_HandleKind(sym->fmt);
+	sym->type = type;
+	sym->kind = VCC_HandleKind(sym->type);
 	if (sym->kind != SYM_NONE) {
 		AZ(VCT_invalid_name(sym->rname, NULL));
 		sym->eval = vcc_Eval_Handle;
diff --git a/lib/libvcc/vcc_var.c b/lib/libvcc/vcc_var.c
index 5c8eadf..273563c 100644
--- a/lib/libvcc/vcc_var.c
+++ b/lib/libvcc/vcc_var.c
@@ -41,7 +41,7 @@ vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent, struct symbol *sym)
 {
 	struct vsb *vsb;
 
-	assert(parent->fmt == HEADER);
+	assert(parent->type == HEADER);
 
 	if (sym->nlen >= 127) {
 		VSB_printf(tl->sb, "HTTP header (%.20s..) is too long.\n",
@@ -53,7 +53,7 @@ vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent, struct symbol *sym)
 	AN(sym);
 	sym->noref = 1;
 	sym->kind = SYM_VAR;
-	sym->fmt = parent->fmt;
+	sym->type = parent->type;
 	sym->eval = vcc_Eval_Var;
 	sym->r_methods = parent->r_methods;
 	sym->w_methods = parent->w_methods;
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 02789c5..92ae8df 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -250,8 +250,8 @@ vcc_ParseImport(struct vcc *tl)
 			sym->eval = vcc_Eval_SymFunc;
 			p += strlen(p) + 1;
 			sym->eval_priv = p;
-			sym->fmt = VCC_Type(p);
-			AN(sym->fmt);
+			sym->type = VCC_Type(p);
+			AN(sym->type);
 		} else {
 			VSB_printf(tl->sb, "Internal spec error (%s)\n", p);
 			vcc_ErrWhere(tl, mod);
@@ -337,7 +337,7 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym)
 		sy3->eval = vcc_Eval_SymFunc;
 		p += strlen(p) + 1;
 		sy3->eval_priv = p;
-		sy3->fmt = VCC_Type(p);
+		sy3->type = VCC_Type(p);
 		sy3->extra = TlDup(tl, buf1);
 		sy3->vmod = sy2->vmod;
 		while (p[0] != '\0' || p[1] != '\0' || p[2] != '\0')
diff --git a/lib/libvcc/vcc_xref.c b/lib/libvcc/vcc_xref.c
index a419f8b..ce77c2e 100644
--- a/lib/libvcc/vcc_xref.c
+++ b/lib/libvcc/vcc_xref.c
@@ -336,7 +336,7 @@ vcc_xreftable(struct vcc *tl, const struct symbol *sym)
 {
 
 	Fc(tl, 0, " * %-7s ", VCC_SymKind(tl, sym));
-	Fc(tl, 0, " %-9s ", sym->fmt != NULL ? sym->fmt->name : "");
+	Fc(tl, 0, " %-9s ", sym->type->name);
 	vcc_pnam(tl, sym);
 	if (sym->wildcard != NULL)
 		Fc(tl, 0, "*");

From phk at FreeBSD.org  Thu Feb 15 09:29:13 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 15 Feb 2018 09:29:13 +0000 (UTC)
Subject: [master] cebbb48 Make symbol->kind more OO
Message-ID: <20180215092913.3BC86BA014@lists.varnish-cache.org>


commit cebbb48b042654e75fa17f2ed8ab7cc6ba072f42
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Feb 15 09:15:43 2018 +0000

    Make symbol->kind more OO

diff --git a/include/tbl/symbol_kind.h b/include/tbl/symbol_kind.h
index 5095ae3..bfae2c1 100644
--- a/include/tbl/symbol_kind.h
+++ b/include/tbl/symbol_kind.h
@@ -29,20 +29,20 @@
 
 /*lint -save -e525 -e539 */
 
-VCC_SYMB(NONE,		none)
-VCC_SYMB(ACL,		acl)
-VCC_SYMB(ACTION,	action)
-VCC_SYMB(BACKEND,	backend)
-VCC_SYMB(FUNC,		func)
-VCC_SYMB(INSTANCE,	instance)
-VCC_SYMB(METHOD,	method)
-VCC_SYMB(OBJECT,	object)
-VCC_SYMB(PROBE,		probe)
-VCC_SYMB(STEVEDORE,	stevedore)
-VCC_SYMB(SUB,		sub)
-VCC_SYMB(VAR,		var)
-VCC_SYMB(VCL,		vcl)
-VCC_SYMB(VMOD,		vmod)
-#undef VCC_SYMB
+VCC_KIND(NONE,		none)
+VCC_KIND(ACL,		acl)
+VCC_KIND(ACTION,	action)
+VCC_KIND(BACKEND,	backend)
+VCC_KIND(FUNC,		func)
+VCC_KIND(INSTANCE,	instance)
+VCC_KIND(METHOD,	method)
+VCC_KIND(OBJECT,	object)
+VCC_KIND(PROBE,		probe)
+VCC_KIND(STEVEDORE,	stevedore)
+VCC_KIND(SUB,		sub)
+VCC_KIND(VAR,		var)
+VCC_KIND(VCL,		vcl)
+VCC_KIND(VMOD,		vmod)
+#undef VCC_KIND
 
 /*lint -restore */
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 479731f..dd9a33a 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -84,6 +84,7 @@ struct token {
 	char			*dec;
 };
 
+/*---------------------------------------------------------------------*/
 typedef const struct type	*vcc_type_t;
 
 struct type {
@@ -98,12 +99,21 @@ struct type {
 #define VCC_TYPE(foo)		extern const struct type foo[1];
 #include "tbl/vcc_types.h"
 
+/*---------------------------------------------------------------------*/
+typedef const struct kind	*vcc_kind_t;
 
-enum symkind {
-#define VCC_SYMB(uu, ll)	SYM_##uu,
-#include "tbl/symbol_kind.h"
+struct kind {
+	unsigned		magic;
+#define KIND_MAGIC		0xfad72443
+
+	const char		*name;
 };
 
+#define VCC_KIND(U,l)	extern const struct kind SYM_##U[1];
+#include "tbl/symbol_kind.h"
+
+/*---------------------------------------------------------------------*/
+
 typedef void sym_expr_t(struct vcc *tl, struct expr **,
     struct token *, struct symbol *sym, vcc_type_t);
 typedef void sym_wildcard_t(struct vcc *, struct symbol *, struct symbol *);
@@ -122,7 +132,7 @@ struct symbol {
 	char				*name;
 	unsigned			nlen;
 	sym_wildcard_t			*wildcard;
-	enum symkind			kind;
+	vcc_kind_t			kind;
 
 	sym_act_f			*action;
 	unsigned			action_mask;
@@ -315,17 +325,16 @@ void vcc_stevedore(struct vcc *vcc, const char *stv_name);
 
 /* vcc_symb.c */
 void VCC_PrintCName(struct vsb *vsb, const char *b, const char *e);
-struct symbol *VCC_MkSym(struct vcc *tl, const char *b, enum symkind kind);
+struct symbol *VCC_MkSym(struct vcc *tl, const char *b, vcc_kind_t);
 extern const char XREF_NONE[];
 extern const char XREF_DEF[];
 extern const char XREF_REF[];
 extern const char SYMTAB_NOERR[];
 extern const char SYMTAB_CREATE[];
-struct symbol *VCC_SymbolGet(struct vcc *, enum symkind, const char *,
+struct symbol *VCC_SymbolGet(struct vcc *, vcc_kind_t, const char *,
     const char *);
-const char * VCC_SymKind(struct vcc *tl, const struct symbol *s);
 typedef void symwalk_f(struct vcc *tl, const struct symbol *s);
-void VCC_WalkSymbols(struct vcc *tl, symwalk_f *func, enum symkind kind);
+void VCC_WalkSymbols(struct vcc *tl, symwalk_f *func, vcc_kind_t);
 
 /* vcc_token.c */
 void vcc_Coord(const struct vcc *tl, struct vsb *vsb,
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 1e55b83..520e6ed 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -683,7 +683,7 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 		}
 		VSB_printf(tl->sb,
 		    "Symbol type (%s) can not be used in expression.\n",
-		    VCC_SymKind(tl, sym));
+		    sym->kind->name);
 		vcc_ErrWhere(tl, tl->t);
 		if (sym->def_b != NULL) {
 			VSB_printf(tl->sb, "That symbol was defined here:\n");
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index 6f73b16..b5df60c 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -38,8 +38,11 @@
 #include "vct.h"
 
 /*--------------------------------------------------------------------*/
+#define VCC_KIND(U,l) const struct kind SYM_##U[1] = {{ KIND_MAGIC, #l}};
+#include "tbl/symbol_kind.h"
+/*--------------------------------------------------------------------*/
 
-static enum symkind
+static vcc_kind_t
 VCC_HandleKind(vcc_type_t fmt)
 {
 	if (fmt == ACL)		return(SYM_ACL);
@@ -51,19 +54,6 @@ VCC_HandleKind(vcc_type_t fmt)
 	return(SYM_NONE);
 }
 
-const char *
-VCC_SymKind(struct vcc *tl, const struct symbol *s)
-{
-	switch (s->kind) {
-#define VCC_SYMB(uu, ll)	case SYM_##uu: return(#ll);
-#include "tbl/symbol_kind.h"
-	default:
-		ErrInternal(tl);
-		VSB_printf(tl->sb, "Symbol Kind 0x%x\n", s->kind);
-		return("INTERNALERROR");
-	}
-}
-
 void
 VCC_PrintCName(struct vsb *vsb, const char *b, const char *e)
 {
@@ -101,13 +91,14 @@ vcc_new_symbol(struct vcc *tl, const char *b, const char *e)
 	sym->name[e - b] = '\0';
 	sym->nlen = e - b;
 	VTAILQ_INIT(&sym->children);
+	sym->kind = SYM_NONE;
 	sym->type = VOID;
 	return (sym);
 }
 
 static struct symbol *
 VCC_Symbol(struct vcc *tl, struct symbol *parent,
-    const char *b, const char *e, enum symkind kind, int create)
+    const char *b, const char *e, vcc_kind_t kind, int create)
 {
 	const char *q;
 	struct symbol *sym, *sym2 = NULL;
@@ -196,7 +187,7 @@ const char SYMTAB_NOERR[] = "sym_noerror";
 const char SYMTAB_CREATE[] = "sym_create";
 
 struct symbol *
-VCC_SymbolGet(struct vcc *tl, enum symkind kind, const char *e, const char *x)
+VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x)
 {
 	struct symbol *sym;
 
@@ -229,7 +220,7 @@ VCC_SymbolGet(struct vcc *tl, enum symkind kind, const char *e, const char *x)
 			VSB_printf(tl->sb, " is a reserved word.");
 		else
 			VSB_printf(tl->sb, " has wrong type (%s): ",
-				VCC_SymKind(tl, sym));
+				sym->kind->name);
 		VSB_cat(tl->sb, "\nAt: ");
 		vcc_ErrWhere(tl, tl->t);
 		if (sym->def_b != NULL) {
@@ -259,7 +250,7 @@ VCC_SymbolGet(struct vcc *tl, enum symkind kind, const char *e, const char *x)
 }
 
 struct symbol *
-VCC_MkSym(struct vcc *tl, const char *b, enum symkind kind)
+VCC_MkSym(struct vcc *tl, const char *b, vcc_kind_t kind)
 {
 	struct symbol *sym;
 
@@ -271,7 +262,7 @@ VCC_MkSym(struct vcc *tl, const char *b, enum symkind kind)
 
 static void
 vcc_walksymbols(struct vcc *tl, const struct symbol *root,
-    symwalk_f *func, enum symkind kind)
+    symwalk_f *func, vcc_kind_t kind)
 {
 	struct symbol *sym;
 
@@ -284,7 +275,7 @@ vcc_walksymbols(struct vcc *tl, const struct symbol *root,
 }
 
 void
-VCC_WalkSymbols(struct vcc *tl, symwalk_f *func, enum symkind kind)
+VCC_WalkSymbols(struct vcc *tl, symwalk_f *func, vcc_kind_t kind)
 {
 
 	vcc_walksymbols(tl, tl->symbols, func, kind);
@@ -324,7 +315,7 @@ struct symbol *
 VCC_HandleSymbol(struct vcc *tl, vcc_type_t fmt, const char *pfx)
 {
 	struct symbol *sym;
-	enum symkind kind;
+	vcc_kind_t kind;
 	struct token *t;
 	const char *p;
 
@@ -334,7 +325,7 @@ VCC_HandleSymbol(struct vcc *tl, vcc_type_t fmt, const char *pfx)
 	t = tl->t;
 	sym = VCC_SymbolGet(tl, SYM_NONE, SYMTAB_NOERR, XREF_NONE);
 	if (sym != NULL && sym->def_b != NULL && kind == sym->kind) {
-		p = VCC_SymKind(tl, sym);
+		p = sym->kind->name;
 		VSB_printf(tl->sb, "%c%s '%.*s' redefined.\n",
 		    toupper(*p), p + 1, PF(t));
 		vcc_ErrWhere(tl, t);
@@ -352,7 +343,7 @@ VCC_HandleSymbol(struct vcc *tl, vcc_type_t fmt, const char *pfx)
 	} else if (sym != NULL && sym->kind != kind) {
 		VSB_printf(tl->sb,
 		    "Name %.*s must have type '%s'.\n",
-		    PF(t), VCC_SymKind(tl, sym));
+		    PF(t), sym->kind->name);
 		vcc_ErrWhere(tl, t);
 		return (sym);
 	}
diff --git a/lib/libvcc/vcc_xref.c b/lib/libvcc/vcc_xref.c
index ce77c2e..775becc 100644
--- a/lib/libvcc/vcc_xref.c
+++ b/lib/libvcc/vcc_xref.c
@@ -70,12 +70,12 @@ vcc_checkref(struct vcc *tl, const struct symbol *sym)
 	if (sym->ndef == 0 && sym->nref != 0) {
 		AN(sym->ref_b);
 		VSB_printf(tl->sb, "Undefined %s %.*s, first reference:\n",
-		    VCC_SymKind(tl, sym), PF(sym->ref_b));
+		    sym->kind->name, PF(sym->ref_b));
 		vcc_ErrWhere(tl, sym->ref_b);
 	} else if (sym->ndef != 0 && sym->nref == 0) {
 		AN(sym->def_b);
 		VSB_printf(tl->sb, "Unused %s %.*s, defined:\n",
-		    VCC_SymKind(tl, sym), PF(sym->def_b));
+		    sym->kind->name, PF(sym->def_b));
 		vcc_ErrWhere(tl, sym->def_b);
 		if (!tl->err_unref) {
 			VSB_printf(tl->sb, "(That was just a warning)\n");
@@ -335,7 +335,10 @@ static void v_matchproto_(symwalk_f)
 vcc_xreftable(struct vcc *tl, const struct symbol *sym)
 {
 
-	Fc(tl, 0, " * %-7s ", VCC_SymKind(tl, sym));
+	CHECK_OBJ_NOTNULL(sym, SYMBOL_MAGIC);
+	CHECK_OBJ_NOTNULL(sym->kind, KIND_MAGIC);
+	CHECK_OBJ_NOTNULL(sym->type, TYPE_MAGIC);
+	Fc(tl, 0, " * %-7s ", sym->kind->name);
 	Fc(tl, 0, " %-9s ", sym->type->name);
 	vcc_pnam(tl, sym);
 	if (sym->wildcard != NULL)

From phk at FreeBSD.org  Thu Feb 15 09:32:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 15 Feb 2018 09:32:07 +0000 (UTC)
Subject: [master] e488f77 Reduce number of bytes expected to deal with stuff
 we really don't care about in libreadline.
Message-ID: <20180215093207.9D466BA150@lists.varnish-cache.org>


commit e488f773ea411eaead86537e0127954cdcb8e383
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Feb 15 09:31:29 2018 +0000

    Reduce number of bytes expected to deal with stuff we really
    don't care about in libreadline.
    
    Fixes #2550

diff --git a/bin/varnishtest/tests/u00011.vtc b/bin/varnishtest/tests/u00011.vtc
index ab485bb..90fcfe2 100644
--- a/bin/varnishtest/tests/u00011.vtc
+++ b/bin/varnishtest/tests/u00011.vtc
@@ -30,6 +30,6 @@ process p1 -need-bytes 200
 
 process p1 -write "vcl.s\t\th\tvcl1\r"
 
-process p1 -need-bytes 600
+process p1 -need-bytes 590
 
 process p1 -screen_dump -write "quit\r" -wait

From phk at FreeBSD.org  Thu Feb 15 11:40:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 15 Feb 2018 11:40:07 +0000 (UTC)
Subject: [master] 540c588 Have generate.py pick the VCL variables out of the
 documentation, instead of the other way around.
Message-ID: <20180215114007.F2CECBC86F@lists.varnish-cache.org>


commit 540c588574141f2391ad1b65d20caa99d290e820
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Feb 15 11:38:38 2018 +0000

    Have generate.py pick the VCL variables out of the documentation,
    instead of the other way around.

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 1d04884..566a4c3 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -392,7 +392,7 @@ In VCL you have access to certain variable objects. These contain
 requests and responses currently being worked on. What variables are
 available depends on context.
 
-.. include:: ../include/vcl_var.rst
+.. include:: vcl_var.rst
 
 
 Functions
diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst
new file mode 100644
index 0000000..cbd06e4
--- /dev/null
+++ b/doc/sphinx/reference/vcl_var.rst
@@ -0,0 +1,1064 @@
+
+remote
+~~~~~~
+
+remote.ip
+
+	Type: IP
+
+	Readable from: client, backend
+
+	
+	The IP address of the other end of the TCP connection.
+	This can either be the clients IP, or the outgoing IP
+	of a proxy server.
+	
+
+client
+~~~~~~
+
+client.ip
+
+	Type: IP
+
+	Readable from: client, backend
+
+	
+	The client's IP address.
+	
+
+client.identity
+
+	Type: STRING
+
+	Readable from: client
+
+	Writable from: client
+
+	
+	Identification of the client, used to load balance
+	in the client director. Defaults to the client's IP
+	address.
+	
+
+local
+~~~~~
+
+local.endpoint
+
+	Type: STRING
+
+	Readable from: client
+
+	
+	The transport address of the '-a' socket the session was
+	accepted on.
+	
+
+local.socket
+
+	Type: STRING
+
+	Readable from: client
+
+	
+	The name of the '-a' socket the session was accepted on.
+	
+
+local.ip
+
+	Type: IP
+
+	Readable from: client, backend
+
+	
+	The IP address of the local end of the TCP connection.
+	
+
+server
+~~~~~~
+
+server.ip
+
+	Type: IP
+
+	Readable from: client, backend
+
+	
+	The IP address of the socket on which the client
+	connection was received.
+	
+
+server.hostname
+
+	Type: STRING
+
+	Readable from: all
+
+	
+	The host name of the server.
+	
+
+server.identity
+
+	Type: STRING
+
+	Readable from: all
+
+	
+	The identity of the server, as set by the -i
+	parameter.  If the -i parameter is not passed to varnishd,
+	server.identity will be set to the hostname of the machine.
+	
+
+req
+~~~
+
+req
+
+	Type: HTTP
+
+	Readable from: client
+
+	
+	The entire request HTTP data structure
+	
+
+req.method
+
+	Type: STRING
+
+	Readable from: client
+
+	Writable from: client
+
+	
+	The request type (e.g. "GET", "HEAD").
+	
+
+req.hash
+
+	Type: BLOB
+
+	Readable from: vcl_hit, vcl_miss, vcl_pass, vcl_purge, vcl_deliver
+
+	
+	The hash key of this request.
+	
+
+req.url
+
+	Type: STRING
+
+	Readable from: client
+
+	Writable from: client
+
+	
+	The requested URL.
+	
+
+req.proto
+
+	Type: STRING
+
+	Readable from: client
+
+	Writable from: client
+
+	
+	The HTTP protocol version used by the client.
+	
+
+req.http.
+
+	Type: HEADER
+
+	Readable from: client
+
+	Writable from: client
+
+	Unsetable from: client
+
+	
+	The corresponding HTTP header.
+	
+
+req.restarts
+
+	Type: INT
+
+	Readable from: client
+
+	
+	A count of how many times this request has been restarted.
+	
+
+req.storage
+
+	Type: STEVEDORE
+
+	Readable from: client
+
+	Writable from: client
+
+	
+	The storage backend to use to save this request body.
+	
+
+req.esi_level
+
+	Type: INT
+
+	Readable from: client
+
+	
+	A count of how many levels of ESI requests we're currently at.
+	
+
+req.ttl
+
+	Type: DURATION
+
+	Readable from: client
+
+	Writable from: client
+
+	
+	Upper limit on the object age for cache lookups to return hit.
+	
+	Usage of req.ttl should be replaced with a check on
+	obj.ttl in vcl_hit, returning miss when needed, but
+	this currently hits bug #1799, so an additional
+	workaround is required.
+	
+	Deprecated and scheduled for removal with varnish release 7.
+	
+
+req.xid
+
+	Type: STRING
+
+	Readable from: client
+
+	
+	Unique ID of this request.
+	
+
+req.esi
+
+	Type: BOOL
+
+	Readable from: client
+
+	Writable from: client
+
+	
+	Boolean. Set to false to disable ESI processing
+	regardless of any value in beresp.do_esi. Defaults
+	to true. This variable is subject to change in
+	future versions, you should avoid using it.
+	
+
+req.can_gzip
+
+	Type: BOOL
+
+	Readable from: client
+
+	
+	Does the client accept the gzip transfer encoding.
+	
+
+req.backend_hint
+
+	Type: BACKEND
+
+	Readable from: client
+
+	Writable from: client
+
+	
+	Set bereq.backend to this if we attempt to fetch.
+	When set to a director, reading this variable returns
+	an actual backend if the director has resolved immediately,
+	or the director otherwise.
+	When used in string context, returns the name of the director
+	or backend, respectively.
+	
+
+req.hash_ignore_busy
+
+	Type: BOOL
+
+	Readable from: client
+
+	Writable from: client
+
+	
+	Ignore any busy object during cache lookup. You
+	would want to do this if you have two server looking
+	up content from each other to avoid potential deadlocks.
+	
+
+req.hash_always_miss
+
+	Type: BOOL
+
+	Readable from: client
+
+	Writable from: client
+
+	
+	Force a cache miss for this request. If set to true
+	Varnish will disregard any existing objects and
+	always (re)fetch from the backend.
+	
+
+req_top
+~~~~~~~
+
+req_top.method
+
+	Type: STRING
+
+	Readable from: client
+
+	
+	The request method of the top-level request in a tree
+	of ESI requests. (e.g. "GET", "HEAD").
+	Identical to req.method in non-ESI requests.
+	
+
+req_top.url
+
+	Type: STRING
+
+	Readable from: client
+
+	
+	The requested URL of the top-level request in a tree
+	of ESI requests.
+	Identical to req.url in non-ESI requests.
+	
+
+req_top.http.
+
+	Type: HEADER
+
+	Readable from: client
+
+	
+	HTTP headers of the top-level request in a tree of ESI requests.
+	Identical to req.http. in non-ESI requests.
+	
+
+req_top.proto
+
+	Type: STRING
+
+	Readable from: client
+
+	
+	HTTP protocol version of the top-level request in a tree of
+	ESI requests.
+	Identical to req.proto in non-ESI requests.
+	
+
+bereq
+~~~~~
+
+bereq
+
+	Type: HTTP
+
+	Readable from: backend
+
+	
+	The entire backend request HTTP data structure
+	
+
+bereq.xid
+
+	Type: STRING
+
+	Readable from: backend
+
+	
+	Unique ID of this request.
+	
+
+bereq.retries
+
+	Type: INT
+
+	Readable from: backend
+
+	
+	A count of how many times this request has been retried.
+	
+
+bereq.backend
+
+	Type: BACKEND
+
+	Readable from: vcl_pipe, backend
+
+	Writable from: vcl_pipe, backend
+
+	
+	This is the backend or director we attempt to fetch from.
+	When set to a director, reading this variable returns
+	an actual backend if the director has resolved immediately,
+	or the director otherwise.
+	When used in string context, returns the name of the director
+	or backend, respectively.
+	
+
+bereq.body
+
+	Type: BODY
+
+	Unsetable from: vcl_backend_fetch
+
+	
+	The request body.
+	
+
+bereq.hash
+
+	Type: BLOB
+
+	Readable from: vcl_pipe, backend
+
+	
+	The hash key of this request.
+	
+
+bereq.method
+
+	Type: STRING
+
+	Readable from: vcl_pipe, backend
+
+	Writable from: vcl_pipe, backend
+
+	
+	The request type (e.g. "GET", "HEAD").
+	
+
+bereq.url
+
+	Type: STRING
+
+	Readable from: vcl_pipe, backend
+
+	Writable from: vcl_pipe, backend
+
+	
+	The requested URL.
+	
+
+bereq.proto
+
+	Type: STRING
+
+	Readable from: vcl_pipe, backend
+
+	Writable from: vcl_pipe, backend
+
+	
+	The HTTP protocol version used to talk to the server.
+	
+
+bereq.http.
+
+	Type: HEADER
+
+	Readable from: vcl_pipe, backend
+
+	Writable from: vcl_pipe, backend
+
+	Unsetable from: vcl_pipe, backend
+
+	
+	The corresponding HTTP header.
+	
+
+bereq.uncacheable
+
+	Type: BOOL
+
+	Readable from: backend
+
+	
+	Indicates whether this request is uncacheable due
+	to a pass in the client side or a hit on an hit-for-pass
+	object.
+	
+
+bereq.connect_timeout
+
+	Type: DURATION
+
+	Readable from: vcl_pipe, backend
+
+	Writable from: vcl_pipe, backend
+
+	
+	The time in seconds to wait for a backend connection.
+	
+
+bereq.first_byte_timeout
+
+	Type: DURATION
+
+	Readable from: backend
+
+	Writable from: backend
+
+	
+	The time in seconds to wait for the first byte from
+	the backend.  Not available in pipe mode.
+	
+
+bereq.between_bytes_timeout
+
+	Type: DURATION
+
+	Readable from: backend
+
+	Writable from: backend
+
+	
+	The time in seconds to wait between each received byte from the
+	backend.  Not available in pipe mode.
+	
+
+bereq.is_bgfetch
+
+	Type: BOOL
+
+	Readable from: backend
+
+	
+	True for background fetches.
+	
+
+beresp
+~~~~~~
+
+beresp
+
+	Type: HTTP
+
+	Readable from: vcl_backend_response, vcl_backend_error
+
+	
+	The entire backend response HTTP data structure
+	
+
+beresp.body
+
+	Type: BODY
+
+	Writable from: vcl_backend_error
+
+	
+	The response body.
+	
+
+beresp.proto
+
+	Type: STRING
+
+	Readable from: vcl_backend_response, vcl_backend_error
+
+	Writable from: vcl_backend_response, vcl_backend_error
+
+	
+	The HTTP protocol version used the backend replied with.
+	
+
+beresp.status
+
+	Type: INT
+
+	Readable from: vcl_backend_response, vcl_backend_error
+
+	Writable from: vcl_backend_response, vcl_backend_error
+
+	
+	The HTTP status code returned by the server.
+	
+	Status codes >1000 can be set for vcl-internal
+	purposes and will be taken modulo 1000 on delivery.
+	
+
+beresp.reason
+
+	Type: STRING
+
+	Readable from: vcl_backend_response, vcl_backend_error
+
+	Writable from: vcl_backend_response, vcl_backend_error
+
+	
+	The HTTP status message returned by the server.
+	
+
+beresp.http.
+
+	Type: HEADER
+
+	Readable from: vcl_backend_response, vcl_backend_error
+
+	Writable from: vcl_backend_response, vcl_backend_error
+
+	Unsetable from: vcl_backend_response, vcl_backend_error
+
+	
+	The corresponding HTTP header.
+	
+
+beresp.do_esi
+
+	Type: BOOL
+
+	Readable from: vcl_backend_response, vcl_backend_error
+
+	Writable from: vcl_backend_response, vcl_backend_error
+
+	
+	Boolean. ESI-process the object after fetching it.
+	Defaults to false. Set it to true to parse the
+	object for ESI directives. Will only be honored if
+	req.esi is true.
+	
+
+beresp.do_stream
+
+	Type: BOOL
+
+	Readable from: vcl_backend_response, vcl_backend_error
+
+	Writable from: vcl_backend_response, vcl_backend_error
+
+	
+	Deliver the object to the client while fetching the whole
+	object into varnish. For uncacheable objects, storage for
+	parts of the body which have been sent to the client may
+	get freed early, depending on the storage engine used.
+	
+
+beresp.do_gzip
+
+	Type: BOOL
+
+	Readable from: vcl_backend_response, vcl_backend_error
+
+	Writable from: vcl_backend_response, vcl_backend_error
+
+	
+	Boolean. Gzip the object before storing it. Defaults
+	to false. When http_gzip_support is on Varnish will
+	request already compressed content from the backend
+	and as such compression in Varnish is not needed.
+	
+
+beresp.do_gunzip
+
+	Type: BOOL
+
+	Readable from: vcl_backend_response, vcl_backend_error
+
+	Writable from: vcl_backend_response, vcl_backend_error
+
+	
+	Boolean. Unzip the object before storing it in the
+	cache.  Defaults to false.
+	
+
+beresp.was_304
+
+	Type: BOOL
+
+	Readable from: vcl_backend_response, vcl_backend_error
+
+	
+	Boolean. If this is a successful 304 response to a
+	backend conditional request refreshing an existing
+	cache object.
+	
+
+beresp.uncacheable
+
+	Type: BOOL
+
+	Readable from: vcl_backend_response, vcl_backend_error
+
+	Writable from: vcl_backend_response, vcl_backend_error
+
+	
+	Inherited from bereq.uncacheable, see there.
+	
+	Setting this variable makes the object uncacheable, which may
+	get stored as a hit-for-miss object in the cache.
+	
+	Clearing the variable has no effect and will log the warning
+	"Ignoring attempt to reset beresp.uncacheable".
+	
+
+beresp.ttl
+
+	Type: DURATION
+
+	Readable from: vcl_backend_response, vcl_backend_error
+
+	Writable from: vcl_backend_response, vcl_backend_error
+
+	
+	The object's remaining time to live, in seconds.
+	
+
+beresp.age
+
+	Type: DURATION
+
+	Readable from: vcl_backend_response, vcl_backend_error
+
+	
+	The age of the object.
+	
+
+beresp.grace
+
+	Type: DURATION
+
+	Readable from: vcl_backend_response, vcl_backend_error
+
+	Writable from: vcl_backend_response, vcl_backend_error
+
+	
+	Set to a period to enable grace.
+	
+
+beresp.keep
+
+	Type: DURATION
+
+	Readable from: vcl_backend_response, vcl_backend_error
+
+	Writable from: vcl_backend_response, vcl_backend_error
+
+	
+	Set to a period to enable conditional backend requests.
+	
+	The keep time is cache lifetime in addition to the ttl.
+	
+	Objects with ttl expired but with keep time left may be used
+	to issue conditional (If-Modified-Since / If-None-Match)
+	requests to the backend to refresh them.
+	
+
+beresp.backend
+
+	Type: BACKEND
+
+	Readable from: vcl_backend_response, vcl_backend_error
+
+	
+	This is the backend we fetched from.  If bereq.backend
+	was set to a director, this will be the backend selected
+	by the director.
+	When used in string context, returns its name.
+	
+
+beresp.backend.name
+
+	Type: STRING
+
+	Readable from: vcl_backend_response, vcl_backend_error
+
+	
+	Name of the backend this response was fetched from.
+	Same as beresp.backend.
+	
+
+beresp.backend.ip
+
+	Type: IP
+
+	Readable from: vcl_backend_response
+
+	
+	IP of the backend this response was fetched from.
+	
+
+beresp.storage
+
+	Type: STEVEDORE
+
+	Readable from: vcl_backend_response, vcl_backend_error
+
+	Writable from: vcl_backend_response, vcl_backend_error
+
+	
+	The storage backend to use to save this object.
+	
+
+obj
+~~~
+
+obj.proto
+
+	Type: STRING
+
+	Readable from: vcl_hit
+
+	
+	The HTTP protocol version stored with the object.
+	
+
+obj.status
+
+	Type: INT
+
+	Readable from: vcl_hit
+
+	
+	The HTTP status code stored with the object.
+	
+
+obj.reason
+
+	Type: STRING
+
+	Readable from: vcl_hit
+
+	
+	The HTTP reason phrase stored with the object.
+	
+
+obj.hits
+
+	Type: INT
+
+	Readable from: vcl_hit, vcl_deliver
+
+	
+	The count of cache-hits on this object. A value of 0 indicates a
+	cache miss.
+	
+
+obj.http.
+
+	Type: HEADER
+
+	Readable from: vcl_hit
+
+	
+	The corresponding HTTP header.
+	
+
+obj.ttl
+
+	Type: DURATION
+
+	Readable from: vcl_hit, vcl_deliver
+
+	
+	The object's remaining time to live, in seconds.
+	
+
+obj.age
+
+	Type: DURATION
+
+	Readable from: vcl_hit, vcl_deliver
+
+	
+	The age of the object.
+	
+
+obj.grace
+
+	Type: DURATION
+
+	Readable from: vcl_hit, vcl_deliver
+
+	
+	The object's remaining grace period in seconds.
+	
+
+obj.keep
+
+	Type: DURATION
+
+	Readable from: vcl_hit, vcl_deliver
+
+	
+	The object's remaining keep period in seconds.
+	
+
+obj.uncacheable
+
+	Type: BOOL
+
+	Readable from: vcl_deliver
+
+	
+	Whether the object is uncacheable (pass, hit-for-pass or
+	hit-for-miss).
+	
+
+obj.storage
+
+	Type: STEVEDORE
+
+	Readable from: vcl_hit, vcl_deliver
+
+	
+	The storage backend used to save this object.
+	
+
+resp
+~~~~
+
+resp
+
+	Type: HTTP
+
+	Readable from: vcl_deliver, vcl_synth
+
+	
+	The entire response HTTP data structure.
+	
+
+resp.body
+
+	Type: BODY
+
+	Writable from: vcl_synth
+
+	
+	The response body.
+	
+
+resp.proto
+
+	Type: STRING
+
+	Readable from: vcl_deliver, vcl_synth
+
+	Writable from: vcl_deliver, vcl_synth
+
+	
+	The HTTP protocol version to use for the response.
+	
+
+resp.status
+
+	Type: INT
+
+	Readable from: vcl_deliver, vcl_synth
+
+	Writable from: vcl_deliver, vcl_synth
+
+	
+	The HTTP status code that will be returned.
+	
+	Assigning a HTTP standardized code to resp.status will also
+	set resp.reason to the corresponding status message.
+	
+	resp.status 200 will get changed into 304 by core code after
+	a return(deliver) from vcl_deliver for conditional requests
+	to cached content if validation succeeds.
+	
+
+resp.reason
+
+	Type: STRING
+
+	Readable from: vcl_deliver, vcl_synth
+
+	Writable from: vcl_deliver, vcl_synth
+
+	
+	The HTTP status message that will be returned.
+	
+
+resp.http.
+
+	Type: HEADER
+
+	Readable from: vcl_deliver, vcl_synth
+
+	Writable from: vcl_deliver, vcl_synth
+
+	Unsetable from: vcl_deliver, vcl_synth
+
+	
+	The corresponding HTTP header.
+	
+
+resp.is_streaming
+
+	Type: BOOL
+
+	Readable from: vcl_deliver, vcl_synth
+
+	
+	Returns true when the response will be streamed
+	from the backend.
+	
+
+now
+~~~
+
+now
+
+	Type: TIME
+
+	Readable from: all
+
+	
+	The current time, in seconds since the epoch. When
+	used in string context it returns a formatted string.
+	
+
+storage
+~~~~~~~
+
+storage.<name>.free_space
+
+	Type: BYTES
+
+	Readable from: client, backend
+
+	
+	Free space available in the named stevedore. Only available for
+	the malloc stevedore.
+	
+
+storage.<name>.used_space
+
+	Type: BYTES
+
+	Readable from: client, backend
+
+	
+	Used space in the named stevedore. Only available for the malloc
+	stevedore.
+	
+
+storage.<name>.happy
+
+	Type: BOOL
+
+	Readable from: client, backend
+
+	
+	Health status for the named stevedore. Not available in any of the
+	current stevedores.
+	
diff --git a/include/Makefile.am b/include/Makefile.am
index 27a557a..b98f8d0 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -102,7 +102,8 @@ nobase_noinst_HEADERS = \
 vmod_abi.h: \
 	    $(top_srcdir)/lib/libvcc/generate.py \
 	    $(top_srcdir)/include/vdef.h \
-	    $(top_srcdir)/include/vrt.h
+	    $(top_srcdir)/include/vrt.h \
+	    $(top_srcdir)/doc/sphinx/reference/vcl_var.rst
 	mkdir -p $(top_builddir)/include/tbl
 	@PYTHON@ $(top_srcdir)/lib/libvcc/generate.py \
 	    $(top_srcdir) $(top_builddir)
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 1eb857e..345ee6a 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -169,7 +169,7 @@ returns = (
 sp_variables = collections.OrderedDict()
 
 class vardef(object):
-	def __init__(self, nam, typ, rd, wr, doc, uns = None):
+	def __init__(self, nam, typ, rd, wr, doc, uns):
 		sp_variables[nam] = self
 		self.nam = nam
 		self.typ = typ
@@ -178,662 +178,54 @@ class vardef(object):
 		self.doc = doc
 		self.uns = uns
 
-vardef('remote.ip',
-	'IP',
-	('both',),
-	(), """
-	The IP address of the other end of the TCP connection.
-	This can either be the clients IP, or the outgoing IP
-	of a proxy server.
-	"""
-)
-vardef('client.ip',
-	'IP',
-	('both',),
-	(), """
-	The client's IP address.
-	"""
-)
-vardef('client.identity',
-	'STRING',
-	('client',),
-	('client',), """
-	Identification of the client, used to load balance
-	in the client director. Defaults to the client's IP
-	address.
-	"""
-)
-vardef('local.ip',
-	'IP',
-	('both',),
-	(), """
-	The IP address of the local end of the TCP connection.
-	"""
-)
-vardef('server.ip',
-	'IP',
-	('both',),
-	(), """
-	The IP address of the socket on which the client
-	connection was received.
-	"""
-)
-vardef('server.hostname',
-	'STRING',
-	('all',),
-	(), """
-	The host name of the server.
-	"""
-)
-vardef('server.identity',
-	'STRING',
-	('all',),
-	(), """
-	The identity of the server, as set by the -i
-	parameter.  If the -i parameter is not passed to varnishd,
-	server.identity will be set to the hostname of the machine.
-	"""
-)
-vardef('req',
-	'HTTP',
-	('client',),
-	(), """
-	The entire request HTTP data structure
-	"""
-)
-vardef('req.method',
-	'STRING',
-	('client',),
-	('client',), """
-	The request type (e.g. "GET", "HEAD").
-	"""
-)
-vardef('req.hash',
-	'BLOB',
-	('hit', 'miss', 'pass', 'purge', 'deliver', ),
-	(), """
-	The hash key of this request.
-	"""
-)
-vardef('req.url',
-	'STRING',
-	('client',),
-	('client',), """
-	The requested URL.
-	"""
-)
-vardef('req.proto',
-	'STRING',
-	('client',),
-	('client',), """
-	The HTTP protocol version used by the client.
-	"""
-)
-vardef('req.http.',
-	'HEADER',
-	('client',),
-	('client',), """
-	The corresponding HTTP header.
-	""",
-	uns = ('client',)
-)
-vardef('req.restarts',
-	'INT',
-	('client',),
-	(), """
-	A count of how many times this request has been restarted.
-	"""
-)
-vardef('req.storage',
-	'STEVEDORE',
-	('client',),
-	('client',), """
-	The storage backend to use to save this request body.
-	"""
-)
-vardef('req.esi_level',
-	'INT',
-	('client',),
-	(), """
-	A count of how many levels of ESI requests we're currently at.
-	"""
-)
-vardef('req.ttl',
-	'DURATION',
-	('client',),
-	('client',), """
-	Upper limit on the object age for cache lookups to return hit.
-
-	Usage of req.ttl should be replaced with a check on
-	obj.ttl in vcl_hit, returning miss when needed, but
-	this currently hits bug #1799, so an additional
-	workaround is required.
-
-	Deprecated and scheduled for removal with varnish release 7.
-	"""
-)
-vardef('req.xid',
-	'STRING',
-	('client',),
-	(), """
-	Unique ID of this request.
-	"""
-)
-vardef('req.esi',
-	'BOOL',
-	('client',),
-	('client',), """
-	Boolean. Set to false to disable ESI processing
-	regardless of any value in beresp.do_esi. Defaults
-	to true. This variable is subject to change in
-	future versions, you should avoid using it.
-	"""
-)
-vardef('req.can_gzip',
-	'BOOL',
-	('client',),
-	(), """
-	Does the client accept the gzip transfer encoding.
-	"""
-)
-vardef('req.backend_hint',
-	'BACKEND',
-	('client', ),
-	('client',), """
-	Set bereq.backend to this if we attempt to fetch.
-	When set to a director, reading this variable returns
-	an actual backend if the director has resolved immediately,
-	or the director otherwise.
-	When used in string context, returns the name of the director
-	or backend, respectively.
-	"""
-)
-vardef('req.hash_ignore_busy',
-	'BOOL',
-	('client',),
-	('client',), """
-	Ignore any busy object during cache lookup. You
-	would want to do this if you have two server looking
-	up content from each other to avoid potential deadlocks.
-	"""
-)
-vardef('req.hash_always_miss',
-	'BOOL',
-	('client',),
-	('client',), """
-	Force a cache miss for this request. If set to true
-	Varnish will disregard any existing objects and
-	always (re)fetch from the backend.
-	"""
-)
-vardef('req_top.method',
-	'STRING',
-	('client',),
-	(), """
-	The request method of the top-level request in a tree
-	of ESI requests. (e.g. "GET", "HEAD").
-	Identical to req.method in non-ESI requests.
-	"""
-)
-vardef('req_top.url',
-	'STRING',
-	('client',),
-	(), """
-	The requested URL of the top-level request in a tree
-	of ESI requests.
-	Identical to req.url in non-ESI requests.
-	"""
-)
-vardef('req_top.http.',
-	'HEADER',
-	('client',),
-	(), """
-	HTTP headers of the top-level request in a tree of ESI requests.
-	Identical to req.http. in non-ESI requests.
-	"""
-)
-vardef('req_top.proto',
-	'STRING',
-	('client',),
-	(), """
-	HTTP protocol version of the top-level request in a tree of
-	ESI requests.
-	Identical to req.proto in non-ESI requests.
-	"""
-)
-vardef('bereq',
-	'HTTP',
-	('backend',),
-	(), """
-	The entire backend request HTTP data structure
-	"""
-)
-vardef('bereq.xid',
-	'STRING',
-	('backend',),
-	(), """
-	Unique ID of this request.
-	"""
-)
-vardef('bereq.retries',
-	'INT',
-	('backend',),
-	(), """
-	A count of how many times this request has been retried.
-	"""
-)
-vardef('bereq.backend',
-	'BACKEND',
-	('pipe', 'backend', ),
-	('pipe', 'backend', ), """
-	This is the backend or director we attempt to fetch from.
-	When set to a director, reading this variable returns
-	an actual backend if the director has resolved immediately,
-	or the director otherwise.
-	When used in string context, returns the name of the director
-	or backend, respectively.
-	"""
-)
-vardef('bereq.body',
-	'BODY',
-	(),
-	(), """
-	The request body.
-	""",
-	uns = ('backend_fetch',)
-)
-vardef('bereq.hash',
-	'BLOB',
-	('pipe', 'backend', ),
-	(), """
-	The hash key of this request.
-	"""
-)
-vardef('bereq.method',
-	'STRING',
-	('pipe', 'backend', ),
-	('pipe', 'backend', ), """
-	The request type (e.g. "GET", "HEAD").
-	"""
-)
-vardef('bereq.url',
-	'STRING',
-	('pipe', 'backend', ),
-	('pipe', 'backend', ), """
-	The requested URL.
-	"""
-)
-vardef('bereq.proto',
-	'STRING',
-	('pipe', 'backend', ),
-	('pipe', 'backend', ), """
-	The HTTP protocol version used to talk to the server.
-	"""
-)
-vardef('bereq.http.',
-	'HEADER',
-	('pipe', 'backend', ),
-	('pipe', 'backend', ), """
-	The corresponding HTTP header.
-	""",
-	uns = ('pipe', 'backend', )
-)
-vardef('bereq.uncacheable',
-	'BOOL',
-	('backend', ),
-	(), """
-	Indicates whether this request is uncacheable due
-	to a pass in the client side or a hit on an hit-for-pass
-	object.
-	"""
-)
-vardef('bereq.connect_timeout',
-	'DURATION',
-	('pipe', 'backend', ),
-	('pipe', 'backend', ), """
-	The time in seconds to wait for a backend connection.
-	"""
-)
-vardef('bereq.first_byte_timeout',
-	'DURATION',
-	('backend', ),
-	('backend', ), """
-	The time in seconds to wait for the first byte from
-	the backend.  Not available in pipe mode.
-	"""
-)
-vardef('bereq.between_bytes_timeout',
-	'DURATION',
-	('backend', ),
-	('backend', ), """
-	The time in seconds to wait between each received byte from the
-	backend.  Not available in pipe mode.
-	"""
-)
-vardef('bereq.is_bgfetch',
-	'BOOL',
-	('backend', ),
-	(), """
-	True for background fetches.
-	"""
-)
-vardef('beresp',
-	'HTTP',
-	('backend_response', 'backend_error'),
-	(), """
-	The entire backend response HTTP data structure
-	"""
-)
-vardef('beresp.body',
-	'BODY',
-	(),
-	('backend_error',), """
-	The response body.
-	"""
-)
-vardef('beresp.proto',
-	'STRING',
-	('backend_response', 'backend_error'),
-	('backend_response', 'backend_error'), """
-	The HTTP protocol version used the backend replied with.
-	"""
-)
-vardef('beresp.status',
-	'INT',
-	('backend_response', 'backend_error'),
-	('backend_response', 'backend_error'), """
-	The HTTP status code returned by the server.
-
-	Status codes >1000 can be set for vcl-internal
-	purposes and will be taken modulo 1000 on delivery.
-	"""
-)
-vardef('beresp.reason',
-	'STRING',
-	('backend_response', 'backend_error'),
-	('backend_response', 'backend_error'), """
-	The HTTP status message returned by the server.
-	"""
-)
-vardef('beresp.http.',
-	'HEADER',
-	('backend_response', 'backend_error'),
-	('backend_response', 'backend_error'), """
-	The corresponding HTTP header.
-	""",
-	uns = ('backend_response', 'backend_error')
-)
-vardef('beresp.do_esi',
-	'BOOL',
-	('backend_response', 'backend_error'),
-	('backend_response', 'backend_error'), """
-	Boolean. ESI-process the object after fetching it.
-	Defaults to false. Set it to true to parse the
-	object for ESI directives. Will only be honored if
-	req.esi is true.
-	"""
-)
-vardef('beresp.do_stream',
-	'BOOL',
-	('backend_response', 'backend_error'),
-	('backend_response', 'backend_error'), """
-	Deliver the object to the client while fetching the whole
-	object into varnish. For uncacheable objects, storage for
-	parts of the body which have been sent to the client may
-	get freed early, depending on the storage engine used.
-	"""
-)
-vardef('beresp.do_gzip',
-	'BOOL',
-	('backend_response', 'backend_error'),
-	('backend_response', 'backend_error'), """
-	Boolean. Gzip the object before storing it. Defaults
-	to false. When http_gzip_support is on Varnish will
-	request already compressed content from the backend
-	and as such compression in Varnish is not needed.
-	"""
-)
-vardef('beresp.do_gunzip',
-	'BOOL',
-	('backend_response', 'backend_error'),
-	('backend_response', 'backend_error'), """
-	Boolean. Unzip the object before storing it in the
-	cache.  Defaults to false.
-	"""
-)
-vardef('beresp.was_304',
-	'BOOL',
-	('backend_response', 'backend_error'),
-	(), """
-	Boolean. If this is a successful 304 response to a
-	backend conditional request refreshing an existing
-	cache object.
-	"""
-)
-vardef('beresp.uncacheable',
-	'BOOL',
-	('backend_response', 'backend_error'),
-	('backend_response', 'backend_error'), """
-	Inherited from bereq.uncacheable, see there.
-
-	Setting this variable makes the object uncacheable, which may
-	get stored as a hit-for-miss object in the cache.
-
-	Clearing the variable has no effect and will log the warning
-	"Ignoring attempt to reset beresp.uncacheable".
-	"""
-)
-vardef('beresp.ttl',
-	'DURATION',
-	('backend_response', 'backend_error'),
-	('backend_response', 'backend_error'), """
-	The object's remaining time to live, in seconds.
-	"""
-)
-vardef('beresp.age',
-	'DURATION',
-	('backend_response', 'backend_error'),
-	(), """
-	The age of the object.
-	"""
-)
-vardef('beresp.grace',
-	'DURATION',
-	('backend_response', 'backend_error'),
-	('backend_response', 'backend_error'), """
-	Set to a period to enable grace.
-	"""
-)
-vardef('beresp.keep',
-	'DURATION',
-	('backend_response', 'backend_error'),
-	('backend_response', 'backend_error'), """
-	Set to a period to enable conditional backend requests.
-
-	The keep time is cache lifetime in addition to the ttl.
-
-	Objects with ttl expired but with keep time left may be used
-	to issue conditional (If-Modified-Since / If-None-Match)
-	requests to the backend to refresh them.
-	"""
-)
-vardef('beresp.backend',
-	'BACKEND',
-	('backend_response', 'backend_error'),
-	(), """
-	This is the backend we fetched from.  If bereq.backend
-	was set to a director, this will be the backend selected
-	by the director.
-	When used in string context, returns its name.
-	"""
-)
-vardef('beresp.backend.name',
-	'STRING',
-	('backend_response', 'backend_error'),
-	(), """
-	Name of the backend this response was fetched from.
-	Same as beresp.backend.
-	"""
-)
-vardef('beresp.backend.ip',
-	'IP',
-	('backend_response',),
-	(), """
-	IP of the backend this response was fetched from.
-	"""
-)
-vardef('beresp.storage',
-	'STEVEDORE',
-	('backend_response', 'backend_error'),
-	('backend_response', 'backend_error'), """
-	The storage backend to use to save this object.
-	"""
-)
-vardef('obj.proto',
-	'STRING',
-	('hit',),
-	(), """
-	The HTTP protocol version stored with the object.
-	"""
-)
-vardef('obj.status',
-	'INT',
-	('hit',),
-	(), """
-	The HTTP status code stored with the object.
-	"""
-)
-vardef('obj.reason',
-	'STRING',
-	('hit',),
-	(), """
-	The HTTP reason phrase stored with the object.
-	"""
-)
-vardef('obj.hits',
-	'INT',
-	('hit', 'deliver'),
-	(), """
-	The count of cache-hits on this object. A value of 0 indicates a
-	cache miss.
-	"""
-)
-vardef('obj.http.',
-	'HEADER',
-	('hit',),
-	(), """
-	The corresponding HTTP header.
-	"""
-)
-vardef('obj.ttl',
-	'DURATION',
-	('hit', 'deliver'),
-	(), """
-	The object's remaining time to live, in seconds.
-	"""
-)
-vardef('obj.age',
-	'DURATION',
-	('hit', 'deliver'),
-	(), """
-	The age of the object.
-	"""
-)
-vardef('obj.grace',
-	'DURATION',
-	('hit', 'deliver'),
-	(), """
-	The object's remaining grace period in seconds.
-	"""
-)
-vardef('obj.keep',
-	'DURATION',
-	('hit', 'deliver'),
-	(), """
-	The object's remaining keep period in seconds.
-	"""
-)
-vardef('obj.uncacheable',
-	'BOOL',
-	('deliver',),
-	(), """
-	Whether the object is uncacheable (pass, hit-for-pass or
-	hit-for-miss).
-	"""
-)
-vardef('obj.storage',
-	'STEVEDORE',
-	('hit', 'deliver'),
-	(), """
-	The storage backend used to save this object.
-	"""
-)
-vardef('resp',
-	'HTTP',
-	('deliver', 'synth'),
-	(), """
-	The entire response HTTP data structure.
-	"""
-)
-vardef('resp.body',
-	'BODY',
-	(),
-	('synth',), """
-	The response body.
-	"""
-)
-vardef('resp.proto',
-	'STRING',
-	('deliver', 'synth'),
-	('deliver', 'synth'), """
-	The HTTP protocol version to use for the response.
-	"""
-)
-vardef('resp.status',
-	'INT',
-	('deliver', 'synth'),
-	('deliver', 'synth'), """
-	The HTTP status code that will be returned.
-
-	Assigning a HTTP standardized code to resp.status will also
-	set resp.reason to the corresponding status message.
-
-	resp.status 200 will get changed into 304 by core code after
-	a return(deliver) from vcl_deliver for conditional requests
-	to cached content if validation succeeds.
-	"""
-)
-vardef('resp.reason',
-	'STRING',
-	('deliver', 'synth'),
-	('deliver', 'synth'), """
-	The HTTP status message that will be returned.
-	"""
-)
-vardef('resp.http.',
-	'HEADER',
-	('deliver', 'synth'),
-	('deliver', 'synth'), """
-	The corresponding HTTP header.
-	""",
-	uns = ('deliver', 'synth')
-)
-vardef('resp.is_streaming',
-	'BOOL',
-	('deliver', 'synth'),
-	(), """
-	Returns true when the response will be streamed
-	from the backend.
-	"""
-)
-vardef('now',
-	'TIME',
-	('all',),
-	(), """
-	The current time, in seconds since the epoch. When
-	used in string context it returns a formatted string.
-	"""
-)
+def parse_var(ln):
+	vn = ln.pop(0).split()
+	assert len(vn) == 1
+	vn = vn[0]
+	vr = []
+	vw = []
+	vu = []
+	while True:
+		l = ln.pop(0)
+		if l == "":
+			continue
+		j = l.split()
+		if j[0] == "Type:":
+			assert len(j) == 2
+			vt = j[1]
+			continue
+		if j[0] == "Readable" and j[1] == "from:":
+			for i in j[2:]:
+				vr.append(i.strip(",."))
+			continue
+		if j[0] == "Writable" and j[1] == "from:":
+			for i in j[2:]:
+				vw.append(i.strip(",."))
+			continue
+		if j[0] == "Unsetable" and j[1] == "from:":
+			for i in j[2:]:
+				vu.append(i.strip(",."))
+			continue
+		break
+	doc = "\n" + l + "\n" + "\n".join(ln)
+	if vn[:8] != "storage.":
+		vardef(vn, vt, vr, vw, doc, uns=vu)
+
+def parse_var_doc(fn):
+	s = 0
+	l = []
+	for i in open(fn):
+		l.append(i.rstrip())
+	for n in range(0, len(l)):
+		j = l[n].split()
+		if len(j) != 2 or j[0] != "Type:" or not l[n][0].isspace():
+			continue
+		m = n
+		while m < len(l) and ( l[m] == "" or l[m][0].isspace() ):
+			m += 1
+		parse_var(l[n-2:m-1])
+
+parse_var_doc(join(buildroot, "doc/sphinx/reference/vcl_var.rst"))
 
 # Backwards compatibility:
 aliases = []
@@ -1204,6 +596,8 @@ fo.close()
 def restrict(fo, spec):
 	d = dict()
 	for j in spec:
+		if j[:4] == "vcl_":
+			j = j[4:]
 		if j == 'all':
 			for i in vcls:
 				d[i] = True
@@ -1219,6 +613,8 @@ def restrict(fo, spec):
 			for i in vcls_backend:
 				d[i] = True
 		else:
+			if not j in vcls:
+				print("JJ", j)
 			assert j in vcls
 			d[j] = True
 	p = ""
@@ -1398,59 +794,6 @@ fo.close()
 
 #######################################################################
 
-fp_vclvar = open(join(buildroot, "doc/sphinx/include/vcl_var.rst"), "w")
-
-def rst_where(fo, h, l):
-	ll = list()
-	if len(l) == 0:
-		return
-	fo.write("\t" + h)
-	s = ""
-	for j in l:
-		if j == "both":
-			ll.append("client")
-			ll.append("backend")
-		elif j == "client":
-			ll.append(j)
-		elif j == "backend":
-			ll.append(j)
-		elif j == "all":
-			ll.append(j)
-		else:
-			ll.append("vcl_" + j)
-	for j in ll:
-		fo.write(s + j)
-		s = ", "
-	fo.write("\n\n")
-
-hdr = ""
-for i in sp_variables.values():
-	j = i.nam.split(".")
-	if j[0] != hdr:
-		fp_vclvar.write("\n" + j[0] + "\n")
-		fp_vclvar.write("~" * len(j[0]) + "\n")
-		hdr = j[0]
-	fp_vclvar.write("\n" + i.nam + "\n\n")
-	fp_vclvar.write("\tType: " + i.typ + "\n\n")
-	rst_where(fp_vclvar, "Readable from: ", i.rd)
-	rst_where(fp_vclvar, "Writable from: ", i.wr)
-	for j in i.doc.split("\n"):
-		fp_vclvar.write("\t%s\n" % j.strip())
-
-hdr = "storage"
-fp_vclvar.write("\n" + hdr + "\n")
-fp_vclvar.write("~" * len(hdr) + "\n")
-for i in stv_variables:
-	fp_vclvar.write("\n" + i[3] + "\n\n")
-	fp_vclvar.write("\tType: " + i[1] + "\n\n")
-	fp_vclvar.write("\tReadable from: client, backend\n\n")
-	for j in i[4].split("\n"):
-		fp_vclvar.write("\t%s\n" % j.strip())
-
-fp_vclvar.close()
-
-#######################################################################
-
 if os.path.isdir(os.path.join(srcroot, ".git")):
 	v = subprocess.check_output([
 		"git --git-dir=" + os.path.join(srcroot, ".git") +

From phk at FreeBSD.org  Thu Feb 15 14:50:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 15 Feb 2018 14:50:10 +0000 (UTC)
Subject: [master] c4db73b Revise variable descriptions
Message-ID: <20180215145010.63FE364EFA@lists.varnish-cache.org>


commit c4db73b2692299ab6d02a3109c8881d0b7e758b9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Feb 15 14:49:12 2018 +0000

    Revise variable descriptions

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 566a4c3..976a908 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -385,12 +385,6 @@ The built-in VCL distributed with Varnish will be implicitly concatenated
 when the VCL is compiled.
 
 
-Variables
----------
-
-In VCL you have access to certain variable objects. These contain
-requests and responses currently being worked on. What variables are
-available depends on context.
 
 .. include:: vcl_var.rst
 
diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst
index cbd06e4..dddc764 100644
--- a/doc/sphinx/reference/vcl_var.rst
+++ b/doc/sphinx/reference/vcl_var.rst
@@ -1,58 +1,66 @@
+Variables
+---------
 
-remote
-~~~~~~
+Variables provide read, write and delete access to almost all aspects
+of the work at hand.
 
-remote.ip
+Reading a variable is done simply by using its name in VCL::
 
-	Type: IP
+    if (client.ip ~ bad_guys) {
+	return (synth(400));
+    }
 
-	Readable from: client, backend
+Writing a variable, where this is possible, is done with a `set`
+statement::
 
-	
-	The IP address of the other end of the TCP connection.
-	This can either be the clients IP, or the outgoing IP
-	of a proxy server.
-	
+    set resp.http.never = "Let You Down";
 
-client
-~~~~~~
+Similarly, deleting a variable, for the few variables where this is
+possible, is done with a `unset` statement::
 
-client.ip
+    unset req.http.cookie;
 
-	Type: IP
+Which operations are possible on each variable is described below,
+often with the shorthand "backend" which covers the `vcl_backend_*`
+methods and "client" which covers the rest, except `vcl_init` and
+`vcl_fini`.
 
-	Readable from: client, backend
+When setting a variable, the right hand side of the equalsign
+must have the variables type, you cannot assign a STRING to
+a variable of type NUMBER, even if the string is `"42"`.
+(Explicit conversion functions can be found in
+:ref:`vmod_std(3)` .
 
-	
-	The client's IP address.
-	
+local, server, remote and client
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-client.identity
+These variables describe the network connection between the
+client and varnishd.
 
-	Type: STRING
+Without PROXY protocol::
 
-	Readable from: client
+	     client    server
+	     remote    local
+	       v          v 
+	CLIENT ------------ VARNISHD
 
-	Writable from: client
 
-	
-	Identification of the client, used to load balance
-	in the client director. Defaults to the client's IP
-	address.
-	
+With PROXY protocol::
+
+	     client    server   remote     local
+	       v          v       v          v
+	CLIENT ------------ PROXY ------------ VARNISHD
 
-local
-~~~~~
 
 local.endpoint
 
 	Type: STRING
 
 	Readable from: client
-
 	
-	The transport address of the '-a' socket the session was
-	accepted on.
+	The address of the '-a' socket the session was accepted on.
+
+	If the argument was `-a foo=:81` this would be ":81"
 	
 
 local.socket
@@ -60,10 +68,13 @@ local.socket
 	Type: STRING
 
 	Readable from: client
-
 	
 	The name of the '-a' socket the session was accepted on.
-	
+
+	If the argument was `-a foo=:81` this would be "foo".
+
+	Note that all '-a' gets a default name on the form `a%d`
+	if no name is provided.
 
 local.ip
 
@@ -71,12 +82,51 @@ local.ip
 
 	Readable from: client, backend
 
+	The IP address (and port number) of the local end of the
+	TCP connection, for instance `192.168.1.1:81`
+
+	If the connection is a UNIX domain socket, the value
+	will be `0.0.0.0:0`
+
+remote.ip
+
+	Type: IP
+
+	Readable from: client, backend
+
+	The IP address of the other end of the TCP connection.
+	This can either be the clients IP, or the outgoing IP
+	of a proxy server.
+	
+	If the connection is a UNIX domain socket, the value
+	will be `0.0.0.0:0`
+
+client.ip
+
+	Type: IP
+
+	Readable from: client, backend
+
 	
-	The IP address of the local end of the TCP connection.
+	The client's IP address, either the same as `local.ip`
+	or what the PROXY protocol told us.
+
+client.identity
+
+	Type: STRING
+
+	Readable from: client
+
+	Writable from: client
+
 	
+	Identification of the client, used to load balance
+	in the client director.  Defaults to `client.ip`
+
+	This variable can be overwritten with more precise
+	information, for instance extracted from a `Cookie:`
+	header.
 
-server
-~~~~~~
 
 server.ip
 
@@ -86,7 +136,8 @@ server.ip
 
 	
 	The IP address of the socket on which the client
-	connection was received.
+	connection was received, either the same as `server.ip`
+	or what the PROXY protocol told us.
 	
 
 server.hostname
@@ -94,9 +145,9 @@ server.hostname
 	Type: STRING
 
 	Readable from: all
-
 	
-	The host name of the server.
+	The host name of the server, as returned by the
+	`gethostname(3)` system function.
 	
 
 server.identity
@@ -104,15 +155,18 @@ server.identity
 	Type: STRING
 
 	Readable from: all
-
-	
-	The identity of the server, as set by the -i
-	parameter.  If the -i parameter is not passed to varnishd,
-	server.identity will be set to the hostname of the machine.
 	
+	The identity of the server, as set by the `-i` parameter.
 
-req
-~~~
+	If an `-i` parameter is not passed to varnishd, the return
+	value from `gethostname(3)` system function will be used.
+
+req and req_top
+~~~~~~~~~~~~~~~
+
+These variables describe the present request, and when ESI:include
+requests are being processed, req_top points to the request received
+from the client.
 
 req
 
@@ -121,7 +175,8 @@ req
 	Readable from: client
 
 	
-	The entire request HTTP data structure
+	The entire request HTTP data structure.
+	Mostly useful for passing to VMODs.
 	
 
 req.method
@@ -133,7 +188,7 @@ req.method
 	Writable from: client
 
 	
-	The request type (e.g. "GET", "HEAD").
+	The request method (e.g. "GET", "HEAD", ...)
 	
 
 req.hash
@@ -144,6 +199,8 @@ req.hash
 
 	
 	The hash key of this request.
+	Mostly useful for passing to VMODs, but can also be useful
+	for debugging hit/miss status.
 	
 
 req.url
@@ -155,7 +212,7 @@ req.url
 	Writable from: client
 
 	
-	The requested URL.
+	The requested URL, for instance "/robots.txt".
 	
 
 req.proto
@@ -167,10 +224,11 @@ req.proto
 	Writable from: client
 
 	
-	The HTTP protocol version used by the client.
+	The HTTP protocol version used by the client, usually "HTTP/1.1"
+	or "HTTP/2.0".
 	
 
-req.http.
+req.http.*
 
 	Type: HEADER
 
@@ -181,7 +239,10 @@ req.http.
 	Unsetable from: client
 
 	
-	The corresponding HTTP header.
+	The headers of request, things like `req.http.date`.
+
+	The RFCs allow multipl headers with the same name, and both
+	`set` and `unset` will remove *all* headers with the name given.
 	
 
 req.restarts
@@ -211,10 +272,8 @@ req.esi_level
 	Type: INT
 
 	Readable from: client
-
 	
 	A count of how many levels of ESI requests we're currently at.
-	
 
 req.ttl
 
@@ -241,9 +300,7 @@ req.xid
 
 	Readable from: client
 
-	
 	Unique ID of this request.
-	
 
 req.esi
 
@@ -253,10 +310,9 @@ req.esi
 
 	Writable from: client
 
-	
-	Boolean. Set to false to disable ESI processing
+	Set to `false` to disable ESI processing
 	regardless of any value in beresp.do_esi. Defaults
-	to true. This variable is subject to change in
+	to `true`. This variable is subject to change in
 	future versions, you should avoid using it.
 	
 
@@ -265,9 +321,9 @@ req.can_gzip
 	Type: BOOL
 
 	Readable from: client
-
 	
-	Does the client accept the gzip transfer encoding.
+	True if the client provided `gzip` or `x-gzip` in the
+	`Accept-Encoding` header.
 	
 
 req.backend_hint
@@ -278,7 +334,6 @@ req.backend_hint
 
 	Writable from: client
 
-	
 	Set bereq.backend to this if we attempt to fetch.
 	When set to a director, reading this variable returns
 	an actual backend if the director has resolved immediately,
@@ -295,10 +350,12 @@ req.hash_ignore_busy
 
 	Writable from: client
 
-	
-	Ignore any busy object during cache lookup. You
-	would want to do this if you have two server looking
-	up content from each other to avoid potential deadlocks.
+	Default: `false`
+
+	Ignore any busy object during cache lookup.
+
+	You only want to do this when you have two server looking
+	up content sideways from each other to avoid deadlocks.
 	
 
 req.hash_always_miss
@@ -309,14 +366,14 @@ req.hash_always_miss
 
 	Writable from: client
 
-	
-	Force a cache miss for this request. If set to true
-	Varnish will disregard any existing objects and
-	always (re)fetch from the backend.
-	
+	Default: `false`
 
-req_top
-~~~~~~~
+	Force a cache miss for this request, even if perfectly
+	good matching objects are in the cache.
+
+	This is useful to force-update the cache without invalidating
+	existing entries in case the fetch fails.
+	
 
 req_top.method
 
@@ -342,7 +399,7 @@ req_top.url
 	Identical to req.url in non-ESI requests.
 	
 
-req_top.http.
+req_top.http.*
 
 	Type: HEADER
 
@@ -368,14 +425,21 @@ req_top.proto
 bereq
 ~~~~~
 
+This is the request we send to the backend, it is built from the
+clients `req.*` fields by filtering out "per-hop" fields which
+should not be passed along (`Connection:`, `Range:` and similar).
+
+Slightly more fields are allowed through for `pass` fetches
+than for `miss` fetches, for instance `Range`.
+
 bereq
 
 	Type: HTTP
 
 	Readable from: backend
 
-	
-	The entire backend request HTTP data structure
+	The entire backend request HTTP data structure.
+	Mostly useful as argument to VMODs.
 	
 
 bereq.xid
@@ -471,7 +535,7 @@ bereq.proto
 	The HTTP protocol version used to talk to the server.
 	
 
-bereq.http.
+bereq.http.*
 
 	Type: HEADER
 
@@ -607,7 +671,7 @@ beresp.reason
 	The HTTP status message returned by the server.
 	
 
-beresp.http.
+beresp.http.*
 
 	Type: HEADER
 
@@ -851,7 +915,7 @@ obj.hits
 	cache miss.
 	
 
-obj.http.
+obj.http.*
 
 	Type: HEADER
 
@@ -988,7 +1052,7 @@ resp.reason
 	The HTTP status message that will be returned.
 	
 
-resp.http.
+resp.http.*
 
 	Type: HEADER
 
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 345ee6a..ad353ab 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -182,6 +182,8 @@ def parse_var(ln):
 	vn = ln.pop(0).split()
 	assert len(vn) == 1
 	vn = vn[0]
+	if vn[-1] == '*':
+		vn = vn[:-1]
 	vr = []
 	vw = []
 	vu = []

From phk at FreeBSD.org  Fri Feb 16 10:17:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 16 Feb 2018 10:17:09 +0000 (UTC)
Subject: [master] 9336709 More work on variable documentation.
Message-ID: <20180216101710.061F3B0E08@lists.varnish-cache.org>


commit 933670973050f0e256e865ebcc0595bc0ed34361
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Feb 16 09:43:44 2018 +0000

    More work on variable documentation.

diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst
index dddc764..9dfea7e 100644
--- a/doc/sphinx/reference/vcl_var.rst
+++ b/doc/sphinx/reference/vcl_var.rst
@@ -381,7 +381,6 @@ req_top.method
 
 	Readable from: client
 
-	
 	The request method of the top-level request in a tree
 	of ESI requests. (e.g. "GET", "HEAD").
 	Identical to req.method in non-ESI requests.
@@ -392,7 +391,6 @@ req_top.url
 	Type: STRING
 
 	Readable from: client
-
 	
 	The requested URL of the top-level request in a tree
 	of ESI requests.
@@ -404,7 +402,6 @@ req_top.http.*
 	Type: HEADER
 
 	Readable from: client
-
 	
 	HTTP headers of the top-level request in a tree of ESI requests.
 	Identical to req.http. in non-ESI requests.
@@ -415,7 +412,6 @@ req_top.proto
 	Type: STRING
 
 	Readable from: client
-
 	
 	HTTP protocol version of the top-level request in a tree of
 	ESI requests.
@@ -447,7 +443,6 @@ bereq.xid
 	Type: STRING
 
 	Readable from: backend
-
 	
 	Unique ID of this request.
 	
@@ -457,7 +452,6 @@ bereq.retries
 	Type: INT
 
 	Readable from: backend
-
 	
 	A count of how many times this request has been retried.
 	
@@ -469,7 +463,6 @@ bereq.backend
 	Readable from: vcl_pipe, backend
 
 	Writable from: vcl_pipe, backend
-
 	
 	This is the backend or director we attempt to fetch from.
 	When set to a director, reading this variable returns
@@ -484,19 +477,18 @@ bereq.body
 	Type: BODY
 
 	Unsetable from: vcl_backend_fetch
-
-	
-	The request body.
 	
+	The request body, only present on `pass` requests.
+
+	Unset will also remove `bereq.http.Content-Length`.
 
 bereq.hash
 
 	Type: BLOB
 
 	Readable from: vcl_pipe, backend
-
 	
-	The hash key of this request.
+	The hash key of this request, a copy of `req.hash`.
 	
 
 bereq.method
@@ -506,9 +498,10 @@ bereq.method
 	Readable from: vcl_pipe, backend
 
 	Writable from: vcl_pipe, backend
-
 	
 	The request type (e.g. "GET", "HEAD").
+
+	Regular (non-pipe, non-pass) fetches are always "GET"
 	
 
 bereq.url
@@ -519,8 +512,7 @@ bereq.url
 
 	Writable from: vcl_pipe, backend
 
-	
-	The requested URL.
+	The requested URL, copied from `req.url`
 	
 
 bereq.proto
@@ -530,9 +522,9 @@ bereq.proto
 	Readable from: vcl_pipe, backend
 
 	Writable from: vcl_pipe, backend
-
 	
-	The HTTP protocol version used to talk to the server.
+	The HTTP protocol version, "HTTP/1.1" unless a pass or pipe
+	request has "HTTP/1.0" in `req.proto`
 	
 
 bereq.http.*
@@ -545,8 +537,7 @@ bereq.http.*
 
 	Unsetable from: vcl_pipe, backend
 
-	
-	The corresponding HTTP header.
+	The headers to be sent to the backend.
 	
 
 bereq.uncacheable
@@ -556,9 +547,8 @@ bereq.uncacheable
 	Readable from: backend
 
 	
-	Indicates whether this request is uncacheable due
-	to a pass in the client side or a hit on an hit-for-pass
-	object.
+	Indicates whether this request is uncacheable due to a
+	`pass` in the client side or a hit on an hit-for-pass object.
 	
 
 bereq.connect_timeout
@@ -568,9 +558,9 @@ bereq.connect_timeout
 	Readable from: vcl_pipe, backend
 
 	Writable from: vcl_pipe, backend
-
 	
-	The time in seconds to wait for a backend connection.
+	The time in seconds to wait for a backend connection to be
+	established.
 	
 
 bereq.first_byte_timeout
@@ -580,10 +570,9 @@ bereq.first_byte_timeout
 	Readable from: backend
 
 	Writable from: backend
-
 	
-	The time in seconds to wait for the first byte from
-	the backend.  Not available in pipe mode.
+	The time in seconds to wait getting the first byte back
+	from the backend.  Not available in pipe mode.
 	
 
 bereq.between_bytes_timeout
@@ -593,7 +582,6 @@ bereq.between_bytes_timeout
 	Readable from: backend
 
 	Writable from: backend
-
 	
 	The time in seconds to wait between each received byte from the
 	backend.  Not available in pipe mode.
@@ -604,33 +592,33 @@ bereq.is_bgfetch
 	Type: BOOL
 
 	Readable from: backend
-
-	
-	True for background fetches.
 	
+	True for fetches where the client got a hit on an object in
+	grace, and this fetch was kicked of in the background to get
+	a fresh copy.
 
 beresp
 ~~~~~~
 
+The response received from the backend, one cache misses, the
+store object is built from `beresp`.
+
 beresp
 
 	Type: HTTP
 
 	Readable from: vcl_backend_response, vcl_backend_error
 
-	
-	The entire backend response HTTP data structure
-	
+	The entire backend response HTTP data structure, useful as
+	argument to VMOD functions.
 
 beresp.body
 
 	Type: BODY
 
 	Writable from: vcl_backend_error
-
-	
-	The response body.
 	
+	For producing a synthetic body.
 
 beresp.proto
 
@@ -640,8 +628,7 @@ beresp.proto
 
 	Writable from: vcl_backend_response, vcl_backend_error
 
-	
-	The HTTP protocol version used the backend replied with.
+	The HTTP protocol version the backend replied with.
 	
 
 beresp.status
@@ -652,12 +639,15 @@ beresp.status
 
 	Writable from: vcl_backend_response, vcl_backend_error
 
-	
 	The HTTP status code returned by the server.
 	
-	Status codes >1000 can be set for vcl-internal
-	purposes and will be taken modulo 1000 on delivery.
-	
+	Status codes on the form XXYZZ can be set where
+	XXYZZ is less than 65536 and Y is [1...9].
+	Only YZZ will be sent back to clients.
+
+	XX can be therefore be used to pass information
+	around inside VCL, for instance `return(synth(22404))`
+	from `vcl_recv{}` to `vcl_synth{}`
 
 beresp.reason
 
@@ -667,9 +657,7 @@ beresp.reason
 
 	Writable from: vcl_backend_response, vcl_backend_error
 
-	
 	The HTTP status message returned by the server.
-	
 
 beresp.http.*
 
@@ -681,9 +669,7 @@ beresp.http.*
 
 	Unsetable from: vcl_backend_response, vcl_backend_error
 
-	
-	The corresponding HTTP header.
-	
+	The HTTP headers returned from the server.
 
 beresp.do_esi
 
@@ -693,11 +679,10 @@ beresp.do_esi
 
 	Writable from: vcl_backend_response, vcl_backend_error
 
-	
-	Boolean. ESI-process the object after fetching it.
-	Defaults to false. Set it to true to parse the
-	object for ESI directives. Will only be honored if
-	req.esi is true.
+	Default: false
+
+	Set it to true to parse the object for ESI directives.
+	Will only be honored if req.esi is true.
 	
 
 beresp.do_stream
@@ -708,11 +693,12 @@ beresp.do_stream
 
 	Writable from: vcl_backend_response, vcl_backend_error
 
-	
 	Deliver the object to the client while fetching the whole
-	object into varnish. For uncacheable objects, storage for
-	parts of the body which have been sent to the client may
-	get freed early, depending on the storage engine used.
+	object into varnish.
+
+	For uncacheable objects, storage for parts of the body which
+	have been sent to the client may get freed early, depending
+	on the storage engine used.
 	
 
 beresp.do_gzip
@@ -723,12 +709,9 @@ beresp.do_gzip
 
 	Writable from: vcl_backend_response, vcl_backend_error
 
-	
-	Boolean. Gzip the object before storing it. Defaults
-	to false. When http_gzip_support is on Varnish will
-	request already compressed content from the backend
-	and as such compression in Varnish is not needed.
-	
+	Default: false
+
+	Set to `true` to gzip the object while storing it.
 
 beresp.do_gunzip
 
@@ -738,10 +721,10 @@ beresp.do_gunzip
 
 	Writable from: vcl_backend_response, vcl_backend_error
 
-	
-	Boolean. Unzip the object before storing it in the
-	cache.  Defaults to false.
-	
+	Default: false
+
+	Set to `true` to unzip the object while storing it in the
+	cache.
 
 beresp.was_304
 
@@ -750,10 +733,9 @@ beresp.was_304
 	Readable from: vcl_backend_response, vcl_backend_error
 
 	
-	Boolean. If this is a successful 304 response to a
-	backend conditional request refreshing an existing
-	cache object.
-	
+	When `true` this indicates that we got a 304 response
+	to our conditional fetch from the backend and turned
+	that into `beresp.status = 200`
 
 beresp.uncacheable
 
@@ -763,11 +745,11 @@ beresp.uncacheable
 
 	Writable from: vcl_backend_response, vcl_backend_error
 
-	
 	Inherited from bereq.uncacheable, see there.
 	
-	Setting this variable makes the object uncacheable, which may
-	get stored as a hit-for-miss object in the cache.
+	Setting this variable makes the object uncacheable.
+
+	This may may produce a hit-for-miss object in the cache.
 	
 	Clearing the variable has no effect and will log the warning
 	"Ignoring attempt to reset beresp.uncacheable".
@@ -781,7 +763,6 @@ beresp.ttl
 
 	Writable from: vcl_backend_response, vcl_backend_error
 
-	
 	The object's remaining time to live, in seconds.
 	
 
@@ -791,7 +772,6 @@ beresp.age
 
 	Readable from: vcl_backend_response, vcl_backend_error
 
-	
 	The age of the object.
 	
 
@@ -803,7 +783,6 @@ beresp.grace
 
 	Writable from: vcl_backend_response, vcl_backend_error
 
-	
 	Set to a period to enable grace.
 	
 
@@ -815,7 +794,6 @@ beresp.keep
 
 	Writable from: vcl_backend_response, vcl_backend_error
 
-	
 	Set to a period to enable conditional backend requests.
 	
 	The keep time is cache lifetime in addition to the ttl.
@@ -831,7 +809,6 @@ beresp.backend
 
 	Readable from: vcl_backend_response, vcl_backend_error
 
-	
 	This is the backend we fetched from.  If bereq.backend
 	was set to a director, this will be the backend selected
 	by the director.
@@ -844,7 +821,6 @@ beresp.backend.name
 
 	Readable from: vcl_backend_response, vcl_backend_error
 
-	
 	Name of the backend this response was fetched from.
 	Same as beresp.backend.
 	
@@ -855,9 +831,7 @@ beresp.backend.ip
 
 	Readable from: vcl_backend_response
 
-	
 	IP of the backend this response was fetched from.
-	
 
 beresp.storage
 
@@ -869,11 +843,12 @@ beresp.storage
 
 	
 	The storage backend to use to save this object.
-	
 
 obj
 ~~~
 
+This is the object we found in cache.  It cannot be modified.
+
 obj.proto
 
 	Type: STRING
@@ -881,7 +856,7 @@ obj.proto
 	Readable from: vcl_hit
 
 	
-	The HTTP protocol version stored with the object.
+	The HTTP protocol version stored in the object.
 	
 
 obj.status
@@ -891,7 +866,7 @@ obj.status
 	Readable from: vcl_hit
 
 	
-	The HTTP status code stored with the object.
+	The HTTP status code stored in the object.
 	
 
 obj.reason
@@ -901,7 +876,7 @@ obj.reason
 	Readable from: vcl_hit
 
 	
-	The HTTP reason phrase stored with the object.
+	The HTTP reason phrase stored in the object.
 	
 
 obj.hits
@@ -911,8 +886,9 @@ obj.hits
 	Readable from: vcl_hit, vcl_deliver
 
 	
-	The count of cache-hits on this object. A value of 0 indicates a
-	cache miss.
+	The count of cache-hits on this object.
+
+	In `vcl_deliver` a value of 0 indicates a cache miss.
 	
 
 obj.http.*
@@ -921,8 +897,7 @@ obj.http.*
 
 	Readable from: vcl_hit
 
-	
-	The corresponding HTTP header.
+	The HTTP headers stored in the object.
 	
 
 obj.ttl
@@ -931,7 +906,6 @@ obj.ttl
 
 	Readable from: vcl_hit, vcl_deliver
 
-	
 	The object's remaining time to live, in seconds.
 	
 
@@ -941,7 +915,6 @@ obj.age
 
 	Readable from: vcl_hit, vcl_deliver
 
-	
 	The age of the object.
 	
 
@@ -951,7 +924,6 @@ obj.grace
 
 	Readable from: vcl_hit, vcl_deliver
 
-	
 	The object's remaining grace period in seconds.
 	
 
@@ -961,7 +933,6 @@ obj.keep
 
 	Readable from: vcl_hit, vcl_deliver
 
-	
 	The object's remaining keep period in seconds.
 	
 
@@ -971,7 +942,6 @@ obj.uncacheable
 
 	Readable from: vcl_deliver
 
-	
 	Whether the object is uncacheable (pass, hit-for-pass or
 	hit-for-miss).
 	
@@ -982,22 +952,23 @@ obj.storage
 
 	Readable from: vcl_hit, vcl_deliver
 
-	
-	The storage backend used to save this object.
+	The storage backend where this object is stored.
 	
 
 resp
 ~~~~
 
+This is the response we send to the client, it is built from either
+`beresp` (pass/miss) or `obj` (hits).
+
 resp
 
 	Type: HTTP
 
 	Readable from: vcl_deliver, vcl_synth
 
-	
-	The entire response HTTP data structure.
-	
+	The entire response HTTP data structure, useful as argument
+	to VMODs.
 
 resp.body
 
@@ -1005,9 +976,7 @@ resp.body
 
 	Writable from: vcl_synth
 
-	
-	The response body.
-	
+	To produce a synthetic response body, for instance for errors.
 
 resp.proto
 
@@ -1017,9 +986,7 @@ resp.proto
 
 	Writable from: vcl_deliver, vcl_synth
 
-	
 	The HTTP protocol version to use for the response.
-	
 
 resp.status
 
@@ -1029,7 +996,6 @@ resp.status
 
 	Writable from: vcl_deliver, vcl_synth
 
-	
 	The HTTP status code that will be returned.
 	
 	Assigning a HTTP standardized code to resp.status will also
@@ -1048,7 +1014,6 @@ resp.reason
 
 	Writable from: vcl_deliver, vcl_synth
 
-	
 	The HTTP status message that will be returned.
 	
 
@@ -1063,7 +1028,7 @@ resp.http.*
 	Unsetable from: vcl_deliver, vcl_synth
 
 	
-	The corresponding HTTP header.
+	The HTTP headers that will be returned.
 	
 
 resp.is_streaming
@@ -1072,13 +1037,12 @@ resp.is_streaming
 
 	Readable from: vcl_deliver, vcl_synth
 
-	
 	Returns true when the response will be streamed
-	from the backend.
+	while being fetched from the backend.
 	
 
-now
-~~~
+Special variables
+~~~~~~~~~~~~~~~~~
 
 now
 
@@ -1087,8 +1051,10 @@ now
 	Readable from: all
 
 	
-	The current time, in seconds since the epoch. When
-	used in string context it returns a formatted string.
+	The current time, in seconds since the UNIX epoch.
+
+	When converted to STRING in expressions it returns
+	a formatted timestamp like `Tue, 20 Feb 2018 09:30:31 GMT`
 	
 
 storage

From geoff at uplex.de  Fri Feb 16 11:02:02 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Fri, 16 Feb 2018 11:02:02 +0000 (UTC)
Subject: [master] dcce66a Implement the vtc server and client commands for
 Unix domain sockets.
Message-ID: <20180216110202.B25CAB1AAF@lists.varnish-cache.org>


commit dcce66a62c99cfe9697f196bd24272d8c32db10b
Author: Geoff Simmons <geoff at uplex.de>
Date:   Thu Feb 15 20:02:30 2018 +0100

    Implement the vtc server and client commands for Unix domain sockets.
    
    This adds the VUS interface to libvarnish.
    
    UDS code corresponding to VTCP_open() and VTCP_listen_on() is
    implemented in the vtc code, since it will only ever be used
    there.

diff --git a/bin/varnishtest/tests/a00002.vtc b/bin/varnishtest/tests/a00002.vtc
index 2aa6f64..2d8c755 100644
--- a/bin/varnishtest/tests/a00002.vtc
+++ b/bin/varnishtest/tests/a00002.vtc
@@ -21,3 +21,24 @@ client c1 -connect ${s1_sock} {
 client c1 -run
 
 server s1 -wait
+
+# The same tests with unix domain sockets
+server s2 -listen "${tmpdir}/s2.sock" {
+	rxreq
+	expect req.method == PUT
+	expect req.proto == HTTP/1.0
+	expect req.url == "/foo"
+	txresp -proto HTTP/1.2 -status 201 -reason Foo
+} -start
+
+client c2 -connect ${s2_sock} {
+	txreq -req PUT -proto HTTP/1.0 -url /foo
+	rxresp
+	expect resp.proto == HTTP/1.2
+	expect resp.status == 201
+	expect resp.reason == Foo
+}
+
+client c2 -run
+
+server s2 -wait
diff --git a/bin/varnishtest/tests/a00003.vtc b/bin/varnishtest/tests/a00003.vtc
index 8855dba..7962eef 100644
--- a/bin/varnishtest/tests/a00003.vtc
+++ b/bin/varnishtest/tests/a00003.vtc
@@ -43,3 +43,48 @@ client c2 -wait
 
 server s1 -wait
 server s2 -wait
+
+# The same tests with Unix domain sockets
+server s3 -listen "${tmpdir}/s3.sock" {
+	rxreq
+	expect req.method == PUT
+	expect req.proto == HTTP/1.0
+	expect req.url == "/foo"
+	txresp -proto HTTP/1.2 -status 201 -reason Foo
+}
+
+server s4 -listen "${tmpdir}/s4.sock" {
+	rxreq
+	expect req.method == GET
+	expect req.proto == HTTP/1.1
+	expect req.url == "/"
+	txresp
+}
+
+server s3 -start
+server s4 -start
+
+client c3 -connect ${s3_sock} {
+	txreq -req PUT -proto HTTP/1.0 -url /foo
+	rxresp
+	expect resp.proto == HTTP/1.2
+	expect resp.status == 201
+	expect resp.reason == Foo
+}
+
+client c4 -connect ${s4_sock} {
+	txreq
+	rxresp
+	expect resp.proto == HTTP/1.1
+	expect resp.status == 200
+	expect resp.reason == OK
+}
+
+client c3 -start
+client c4 -start
+
+client c3 -wait
+client c4 -wait
+
+server s3 -wait
+server s4 -wait
diff --git a/bin/varnishtest/tests/a00004.vtc b/bin/varnishtest/tests/a00004.vtc
index be185a6..05474f6 100644
--- a/bin/varnishtest/tests/a00004.vtc
+++ b/bin/varnishtest/tests/a00004.vtc
@@ -33,3 +33,38 @@ client c1 -wait
 client c2 -wait
 
 server s1 -wait
+
+# The same tests with Unix domain sockets
+server s2 -repeat 2 -listen "${tmpdir}/s2.sock" {
+	rxreq
+	expect req.method == PUT
+	expect req.proto == HTTP/1.0
+	expect req.url == "/foo"
+	txresp -proto HTTP/1.2 -status 201 -reason Foo
+}
+
+server s2 -start
+
+client c3 -connect ${s2_sock} {
+	txreq -req PUT -proto HTTP/1.0 -url /foo
+	rxresp
+	expect resp.proto == HTTP/1.2
+	expect resp.status == 201
+	expect resp.reason == Foo
+}
+
+client c4 -connect ${s2_sock} {
+	txreq -req PUT -proto HTTP/1.0 -url /foo
+	rxresp
+	expect resp.proto == HTTP/1.2
+	expect resp.status == 201
+	expect resp.reason == Foo
+}
+
+client c3 -start
+client c4 -start
+
+client c3 -wait
+client c4 -wait
+
+server s2 -wait
diff --git a/bin/varnishtest/tests/a00005.vtc b/bin/varnishtest/tests/a00005.vtc
index 7de7675..fe4885f 100644
--- a/bin/varnishtest/tests/a00005.vtc
+++ b/bin/varnishtest/tests/a00005.vtc
@@ -41,3 +41,46 @@ client c1 -run
 
 server s1 -wait
 server s2 -wait
+
+# The same tests with unix domain sockets
+server s3 -listen "${tmpdir}/s3.sock" {
+	rxreq
+	expect req.method == PUT
+	expect req.proto == HTTP/1.0
+	expect req.url == "/foo"
+	txresp -proto HTTP/1.2 -status 201 -reason Foo
+}
+
+server s4 -listen "${tmpdir}/s4.sock" {
+	rxreq
+	expect req.method == GET
+	expect req.proto == HTTP/1.1
+	expect req.url == "/"
+	txresp
+}
+
+server s3 -start
+server s4 -start
+
+client c2 -connect ${s3_sock} {
+	txreq -req PUT -proto HTTP/1.0 -url /foo
+	rxresp
+	expect resp.proto == HTTP/1.2
+	expect resp.status == 201
+	expect resp.reason == Foo
+}
+
+client c2 -run
+
+client c2 -connect ${s4_sock} {
+	txreq
+	rxresp
+	expect resp.proto == HTTP/1.1
+	expect resp.status == 200
+	expect resp.reason == OK
+}
+
+client c2 -run
+
+server s3 -wait
+server s4 -wait
diff --git a/bin/varnishtest/tests/a00006.vtc b/bin/varnishtest/tests/a00006.vtc
index c01522d..2ca486b 100644
--- a/bin/varnishtest/tests/a00006.vtc
+++ b/bin/varnishtest/tests/a00006.vtc
@@ -23,3 +23,28 @@ client c1 -connect ${s1_sock} {
 client c1 -run
 
 server s1 -wait
+
+# The same tests with Unix domain sockets
+server s2 -listen "${tmpdir}/s2.sock" {
+	rxreq
+	expect req.method == PUT
+	expect req.proto == HTTP/1.0
+	expect req.url == "/foo"
+	txresp -proto HTTP/1.2 -status 201 -reason Foo \
+		-body "987654321\n"
+}
+
+server s2 -start
+
+client c2 -connect ${s2_sock} {
+	txreq -req PUT -proto HTTP/1.0 -url /foo \
+		-body "123456789\n"
+	rxresp
+	expect resp.proto == HTTP/1.2
+	expect resp.status == 201
+	expect resp.reason == Foo
+}
+
+client c2 -run
+
+server s2 -wait
diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index 6e99403..b404552 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -93,7 +93,8 @@ extern int feature_dns;
 
 void init_server(void);
 
-int http_process(struct vtclog *vl, const char *spec, int sock, int *sfd);
+int http_process(struct vtclog *vl, const char *spec, int sock, int *sfd,
+		 const char *addr);
 
 char * synth_body(const char *len, int rnd);
 
diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c
index 167d80c..754d4ec 100644
--- a/bin/varnishtest/vtc_client.c
+++ b/bin/varnishtest/vtc_client.c
@@ -29,17 +29,21 @@
 #include "config.h"
 
 #include <sys/socket.h>
+#include <sys/un.h>
 
 #include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <poll.h>
+#include <unistd.h>
 
 #include "vtc.h"
 
 #include "vsa.h"
 #include "vss.h"
 #include "vtcp.h"
+#include "vus.h"
 
 struct client {
 	unsigned		magic;
@@ -106,6 +110,83 @@ client_proxy(struct vtclog *vl, int fd, int version, const char *spec)
 }
 
 /**********************************************************************
+ * Socket connect.
+ */
+
+static int
+client_tcp_connect(struct vtclog *vl, const char *addr, double tmo,
+		   const char **errp)
+{
+	int fd;
+	char mabuf[32], mpbuf[32];
+
+	fd = VTCP_open(addr, NULL, tmo, errp);
+	if (fd < 0)
+		return fd;
+	VTCP_myname(fd, mabuf, sizeof mabuf, mpbuf, sizeof mpbuf);
+	vtc_log(vl, 3, "connected fd %d from %s %s to %s", fd, mabuf, mpbuf,
+		addr);
+	return fd;
+}
+
+/* cf. VTCP_Open() */
+static int v_matchproto_(vus_resolved_f)
+uds_open(void *priv, const struct sockaddr_un *uds)
+{
+	double *p, tmo;
+	int s, i;
+	struct pollfd fds[1];
+	socklen_t sl = sizeof(*uds);
+
+	AN(priv);
+	AN(uds);
+	p = priv;
+	assert(*p > 0.);
+	tmo = *p * 1e3;
+
+	s = socket(uds->sun_family, SOCK_STREAM, 0);
+	if (s < 0)
+		return (s);
+
+	(void) VTCP_nonblocking(s);
+	i = connect(s, uds, sl);
+	if (i == 0)
+		return(s);
+	if (errno != EINPROGRESS) {
+		closefd(&s);
+		return (-1);
+	}
+
+	fds[0].fd = s;
+	fds[0].events = POLLWRNORM;
+	fds[0].revents = 0;
+	i = poll(fds, 1, tmo);
+
+	if (i == 0) {
+		closefd(&s);
+		errno = ETIMEDOUT;
+		return (-1);
+	}
+
+	return (VTCP_connected(s));
+}
+
+static int
+client_uds_connect(struct vtclog *vl, const char *path, double tmo,
+		   const char **errp)
+{
+	int fd;
+
+	assert(tmo >= 0);
+
+	fd = VUS_resolver(path, uds_open, &tmo, errp);
+	if (fd < 0)
+		return fd;
+	vtc_log(vl, 3, "connected fd %d to %s", fd, path);
+	return fd;
+}
+
+/**********************************************************************
  * Client thread
  */
 
@@ -117,7 +198,6 @@ client_thread(void *priv)
 	int fd;
 	unsigned u;
 	struct vsb *vsb;
-	char mabuf[32], mpbuf[32];
 	const char *err;
 
 	CAST_OBJ_NOTNULL(c, priv, CLIENT_MAGIC);
@@ -133,20 +213,21 @@ client_thread(void *priv)
 	if (c->repeat != 1)
 		vtc_log(vl, 2, "Started (%u iterations)", c->repeat);
 	for (u = 0; u < c->repeat; u++) {
-		vtc_log(vl, 3, "Connect to %s", VSB_data(vsb));
-		fd = VTCP_open(VSB_data(vsb), NULL, 10., &err);
+		char *addr = VSB_data(vsb);
+
+		vtc_log(vl, 3, "Connect to %s", addr);
+		if (*addr == '/')
+			fd = client_uds_connect(vl, addr, 10., &err);
+		else
+			fd = client_tcp_connect(vl, VSB_data(vsb), 10., &err);
 		if (fd < 0)
 			vtc_fatal(c->vl, "Failed to open %s: %s",
 			    VSB_data(vsb), err);
-		assert(fd >= 0);
 		/* VTCP_blocking does its own checks, trust it */
 		(void)VTCP_blocking(fd);
-		VTCP_myname(fd, mabuf, sizeof mabuf, mpbuf, sizeof mpbuf);
-		vtc_log(vl, 3, "connected fd %d from %s %s to %s",
-		    fd, mabuf, mpbuf, VSB_data(vsb));
 		if (c->proxy_spec != NULL)
 			client_proxy(vl, fd, c->proxy_version, c->proxy_spec);
-		fd = http_process(vl, c->spec, fd, NULL);
+		fd = http_process(vl, c->spec, fd, NULL, addr);
 		vtc_log(vl, 3, "closing fd %d", fd);
 		VTCP_close(&fd);
 	}
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index c65139b..62df449 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -1840,7 +1840,8 @@ const struct cmds http_cmds[] = {
 };
 
 int
-http_process(struct vtclog *vl, const char *spec, int sock, int *sfd)
+http_process(struct vtclog *vl, const char *spec, int sock, int *sfd,
+	     const char *addr)
 {
 	struct http *hp;
 	int retval;
@@ -1870,8 +1871,13 @@ http_process(struct vtclog *vl, const char *spec, int sock, int *sfd)
 	hp->gziplevel = 0;
 	hp->gzipresidual = -1;
 
-	VTCP_hisname(sock,
-	    hp->rem_ip, VTCP_ADDRBUFSIZE, hp->rem_port, VTCP_PORTBUFSIZE);
+	if (*addr != '/')
+		VTCP_hisname(sock, hp->rem_ip, VTCP_ADDRBUFSIZE, hp->rem_port,
+			     VTCP_PORTBUFSIZE);
+	else {
+		strcpy(hp->rem_ip, "0.0.0.0");
+		strcpy(hp->rem_port, "0");
+	}
 	parse_string(spec, http_cmds, hp, vl);
 	if (hp->h2)
 		stop_h2(hp);
diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index fb811df..a0e531a 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -29,15 +29,19 @@
 #include "config.h"
 
 #include <sys/socket.h>
+#include <sys/types.h>
+#include <sys/stat.h>
 
 #include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <unistd.h>
 
 #include "vtc.h"
 
 #include "vtcp.h"
+#include "vus.h"
 
 struct server {
 	unsigned		magic;
@@ -114,6 +118,81 @@ server_delete(struct server *s)
  * Server listen
  */
 
+struct helper {
+	int		depth;
+	const char	**errp;
+};
+
+/* cf. VTCP_listen_on() */
+static int v_matchproto_(vus_resolved_f)
+uds_listen(void *priv, const struct sockaddr_un *uds)
+{
+	int sock, e;
+	struct helper *hp = priv;
+
+	sock = VUS_bind(uds, hp->errp);
+	if (sock >= 0)   {
+		if (listen(sock, hp->depth) != 0) {
+			e = errno;
+			closefd(&sock);
+			errno = e;
+			if (hp->errp != NULL)
+				*hp->errp = "listen(2)";
+			return (-1);
+		}
+	}
+	if (sock > 0) {
+		*hp->errp = NULL;
+		return (sock);
+	}
+	AN(*hp->errp);
+	return (0);
+}
+
+static void
+server_listen_uds(struct server *s, const char **errp)
+{
+	mode_t m;
+	struct helper h;
+
+	h.depth = s->depth;
+	h.errp = errp;
+
+	errno = 0;
+	if (unlink(s->listen) != 0 && errno != ENOENT)
+		vtc_fatal(s->vl, "Could not unlink %s before bind: %s",
+			  s->listen, strerror(errno));
+	/*
+	 * Temporarily set the umask to 0 to avoid issues with
+	 * permissions.
+	 */
+	m = umask(0);
+	s->sock = VUS_resolver(s->listen, uds_listen, &h, errp);
+	umask(m);
+	if (*errp != NULL)
+		return;
+	assert(s->sock > 0);
+	macro_def(s->vl, s->name, "addr", "0.0.0.0");
+	macro_def(s->vl, s->name, "port", "0");
+	macro_def(s->vl, s->name, "sock", "%s", s->listen);
+}
+
+static void
+server_listen_tcp(struct server *s, const char **errp)
+{
+	s->sock = VTCP_listen_on(s->listen, "0", s->depth, errp);
+	if (*errp != NULL)
+		return;
+	assert(s->sock > 0);
+	VTCP_myname(s->sock, s->aaddr, sizeof s->aaddr,
+	    s->aport, sizeof s->aport);
+	macro_def(s->vl, s->name, "addr", "%s", s->aaddr);
+	macro_def(s->vl, s->name, "port", "%s", s->aport);
+	macro_def(s->vl, s->name, "sock", "%s %s", s->aaddr, s->aport);
+	/* Record the actual port, and reuse it on subsequent starts */
+	bprintf(s->listen, "%s %s", s->aaddr, s->aport);
+}
+
 static void
 server_listen(struct server *s)
 {
@@ -123,19 +202,14 @@ server_listen(struct server *s)
 
 	if (s->sock >= 0)
 		VTCP_close(&s->sock);
-	s->sock = VTCP_listen_on(s->listen, "0", s->depth, &err);
+	if (*s->listen != '/')
+		server_listen_tcp(s, &err);
+	else
+		server_listen_uds(s, &err);
 	if (err != NULL)
 		vtc_fatal(s->vl,
 		    "Server listen address (%s) cannot be resolved: %s",
 		    s->listen, err);
-	assert(s->sock > 0);
-	VTCP_myname(s->sock, s->aaddr, sizeof s->aaddr,
-	    s->aport, sizeof s->aport);
-	macro_def(s->vl, s->name, "addr", "%s", s->aaddr);
-	macro_def(s->vl, s->name, "port", "%s", s->aport);
-	macro_def(s->vl, s->name, "sock", "%s %s", s->aaddr, s->aport);
-	/* Record the actual port, and reuse it on subsequent starts */
-	bprintf(s->listen, "%s %s", s->aaddr, s->aport);
 }
 
 /**********************************************************************
@@ -170,7 +244,7 @@ server_thread(void *priv)
 			vtc_fatal(vl, "Accept failed: %s", strerror(errno));
 		VTCP_hisname(fd, abuf, sizeof abuf, pbuf, sizeof pbuf);
 		vtc_log(vl, 3, "accepted fd %d %s %s", fd, abuf, pbuf);
-		fd = http_process(vl, s->spec, fd, &s->sock);
+		fd = http_process(vl, s->spec, fd, &s->sock, s->listen);
 		vtc_log(vl, 3, "shutting fd %d", fd);
 		j = shutdown(fd, SHUT_WR);
 		if (!VTCP_Check(j))
@@ -215,7 +289,7 @@ server_dispatch_wrk(void *priv)
 	fd = s->fd;
 
 	vtc_log(vl, 3, "start with fd %d", fd);
-	fd = http_process(vl, s->spec, fd, &s->sock);
+	fd = http_process(vl, s->spec, fd, &s->sock, s->listen);
 	vtc_log(vl, 3, "shutting fd %d", fd);
 	j = shutdown(fd, SHUT_WR);
 	if (!VTCP_Check(j))
diff --git a/include/Makefile.am b/include/Makefile.am
index b98f8d0..afa8864 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -96,7 +96,8 @@ nobase_noinst_HEADERS = \
 	vsub.h \
 	vss.h \
 	vtcp.h \
-	vtree.h
+	vtree.h \
+	vus.h
 
 ## keep in sync with lib/libvcc/Makefile.am
 vmod_abi.h: \
diff --git a/include/vus.h b/include/vus.h
new file mode 100644
index 0000000..3747d92
--- /dev/null
+++ b/include/vus.h
@@ -0,0 +1,35 @@
+/*-
+ * Copyright (c) 2018 UPLEX - Nils Goroll Systemoptimierung
+ * All rights reserved.
+ *
+ * Author: Geoffrey Simmons <geoffrey.simmons at uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+struct sockaddr_un;
+
+typedef int vus_resolved_f(void *priv, const struct sockaddr_un *);
+int VUS_resolver(const char *path, vus_resolved_f *func, void *priv,
+		 const char **err);
+int VUS_bind(const struct sockaddr_un *uds, const char **errp);
diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am
index 1616f0e..231827f 100644
--- a/lib/libvarnish/Makefile.am
+++ b/lib/libvarnish/Makefile.am
@@ -37,7 +37,8 @@ libvarnish_a_SOURCES = \
 	vss.c \
 	vsub.c \
 	vtcp.c \
-	vtim.c
+	vtim.c \
+	vus.c
 
 TESTS = vnum_c_test binheap
 
diff --git a/lib/libvarnish/vus.c b/lib/libvarnish/vus.c
new file mode 100644
index 0000000..ce0f0fd
--- /dev/null
+++ b/lib/libvarnish/vus.c
@@ -0,0 +1,96 @@
+/*-
+ * Copyright (c) 2018 UPLEX - Nils Goroll Systemoptimierung
+ * All rights reserved.
+ *
+ * Author: Geoffrey Simmons <geoffrey.simmons at uplex.de>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#include <unistd.h>
+#include <errno.h>
+
+#include "vdef.h"
+#include "vas.h"
+#include "vus.h"
+
+int
+VUS_resolver(const char *path, vus_resolved_f *func, void *priv,
+	     const char **err)
+{
+	struct sockaddr_un uds;
+	int ret = 0;
+
+	AN(path);
+	assert(*path == '/');
+
+	AN(err);
+	*err = NULL;
+	if (strlen(path) + 1 > sizeof(uds.sun_path)) {
+		*err = "Path too long for a Unix domain socket";
+		return(-1);
+	}
+	strcpy(uds.sun_path, path);
+	uds.sun_family = PF_UNIX;
+	if (func != NULL)
+		ret = func(priv, &uds);
+	return(ret);
+}
+
+int
+VUS_bind(const struct sockaddr_un *uds, const char **errp)
+{
+	int sd, e;
+	socklen_t sl = sizeof(*uds);
+
+	if (errp != NULL)
+		*errp = NULL;
+
+	sd = socket(PF_UNIX, SOCK_STREAM, 0);
+	if (sd < 0) {
+		if (errp != NULL)
+			*errp = "socket(2)";
+		return (-1);
+	}
+
+	if (unlink(uds->sun_path) != 0 && errno != ENOENT) {
+		if (errp != NULL)
+			*errp = "unlink(2)";
+		e = errno;
+		closefd(&sd);
+		errno = e;
+		return (-1);
+	}
+
+	if (bind(sd, (const struct sockaddr *)uds, sl) != 0) {
+		if (errp != NULL)
+			*errp = "bind(2)";
+		e = errno;
+		closefd(&sd);
+		errno = e;
+		return (-1);
+	}
+	return (sd);
+}

From geoff at uplex.de  Fri Feb 16 11:13:05 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Fri, 16 Feb 2018 11:13:05 +0000 (UTC)
Subject: [master] bdc7fd1 Add a missing include.
Message-ID: <20180216111305.4959CB1E64@lists.varnish-cache.org>


commit bdc7fd181a3e4fb89f454c0e6ac8599867d493de
Author: Geoff Simmons <geoff at uplex.de>
Date:   Fri Feb 16 12:12:26 2018 +0100

    Add a missing include.

diff --git a/lib/libvarnish/vus.c b/lib/libvarnish/vus.c
index ce0f0fd..3b4ff20 100644
--- a/lib/libvarnish/vus.c
+++ b/lib/libvarnish/vus.c
@@ -31,6 +31,7 @@
 
 #include <unistd.h>
 #include <errno.h>
+#include <string.h>
 
 #include "vdef.h"
 #include "vas.h"

From geoff at uplex.de  Fri Feb 16 11:36:05 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Fri, 16 Feb 2018 11:36:05 +0000 (UTC)
Subject: [master] 48ebe64 Cast the sockaddr_un correctly for use in connect(2).
Message-ID: <20180216113605.9CEFDB25D3@lists.varnish-cache.org>


commit 48ebe64cf7f96603febf96d95636bdf745936f18
Author: Geoff Simmons <geoff at uplex.de>
Date:   Fri Feb 16 12:30:16 2018 +0100

    Cast the sockaddr_un correctly for use in connect(2).

diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c
index 754d4ec..8db014c 100644
--- a/bin/varnishtest/vtc_client.c
+++ b/bin/varnishtest/vtc_client.c
@@ -149,7 +149,7 @@ uds_open(void *priv, const struct sockaddr_un *uds)
 		return (s);
 
 	(void) VTCP_nonblocking(s);
-	i = connect(s, uds, sl);
+	i = connect(s, (const struct sockaddr *)uds, sl);
 	if (i == 0)
 		return(s);
 	if (errno != EINPROGRESS) {

From dridi at varni.sh  Fri Feb 16 16:01:35 2018
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Fri, 16 Feb 2018 17:01:35 +0100
Subject: [master] dcce66a Implement the vtc server and client commands for
 Unix domain sockets.
In-Reply-To: <20180216110202.B25CAB1AAF@lists.varnish-cache.org>
References: <20180216110202.B25CAB1AAF@lists.varnish-cache.org>
Message-ID: <CABoVN9DoUN4zv0aQHSzv2uxUJCjFeqva9eP636=e5HOWn2Fqeg@mail.gmail.com>

>     UDS code corresponding to VTCP_open() and VTCP_listen_on() is
>     implemented in the vtc code, since it will only ever be used
>     there.

Didn't we consider CLI over UDS too?

Dridi

From nils.goroll at uplex.de  Fri Feb 16 16:29:07 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 16 Feb 2018 16:29:07 +0000 (UTC)
Subject: [master] c46f838 Run the generic access test also with a
 jail-specific make_workdir
Message-ID: <20180216162907.91EF6B7EFA@lists.varnish-cache.org>


commit c46f838dcce721d0b685b7b3a1046a82ec40ef69
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Feb 16 17:22:12 2018 +0100

    Run the generic access test also with a jail-specific make_workdir
    
    Related to #2570

diff --git a/bin/varnishd/mgt/mgt_jail.c b/bin/varnishd/mgt/mgt_jail.c
index 1e26487..60b3228 100644
--- a/bin/varnishd/mgt/mgt_jail.c
+++ b/bin/varnishd/mgt/mgt_jail.c
@@ -137,28 +137,32 @@ VJ_subproc(enum jail_subproc_e jse)
 int
 VJ_make_workdir(const char *dname)
 {
-	int fd;
+	int i;
 
 	AN(dname);
 	CHECK_OBJ_NOTNULL(vjt, JAIL_TECH_MAGIC);
-	if (vjt->make_workdir != NULL)
-		return (vjt->make_workdir(dname));
-
-	VJ_master(JAIL_MASTER_FILE);
-	if (mkdir(dname, 0755) < 0 && errno != EEXIST)
-		ARGV_ERR("Cannot create working directory '%s': %s\n",
-		    dname, strerror(errno));
+	if (vjt->make_workdir != NULL) {
+		i = vjt->make_workdir(dname);
+		if (i)
+			return (i);
+		VJ_master(JAIL_MASTER_FILE);
+	} else {
+		VJ_master(JAIL_MASTER_FILE);
+		if (mkdir(dname, 0755) < 0 && errno != EEXIST)
+			ARGV_ERR("Cannot create working directory '%s': %s\n",
+				 dname, strerror(errno));
+	}
 
 	if (chdir(dname) < 0)
 		ARGV_ERR("Cannot change to working directory '%s': %s\n",
 		    dname, strerror(errno));
 
-	fd = open("_.testfile", O_RDWR|O_CREAT|O_EXCL, 0600);
-	if (fd < 0)
+	i = open("_.testfile", O_RDWR|O_CREAT|O_EXCL, 0600);
+	if (i < 0)
 		ARGV_ERR("Cannot create test-file in %s (%s)\n"
 		    "Check permissions (or delete old directory)\n",
 		    dname, strerror(errno));
-	closefd(&fd);
+	closefd(&i);
 	AZ(unlink("_.testfile"));
 	VJ_master(JAIL_MASTER_LOW);
 	return (0);

From nils.goroll at uplex.de  Fri Feb 16 16:29:07 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 16 Feb 2018 16:29:07 +0000 (UTC)
Subject: [master] f674929 When running with the unix jail,
 the work dir should be owned by the jail group
Message-ID: <20180216162907.A91D9B7EFF@lists.varnish-cache.org>


commit f674929a5ade02e8e84dd6e554c0e555c9cabeed
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Feb 16 17:23:04 2018 +0100

    When running with the unix jail, the work dir should be owned by the jail group
    
    Fixes #2570

diff --git a/bin/varnishd/mgt/mgt_jail_unix.c b/bin/varnishd/mgt/mgt_jail_unix.c
index c4ba4c5..4257347 100644
--- a/bin/varnishd/mgt/mgt_jail_unix.c
+++ b/bin/varnishd/mgt/mgt_jail_unix.c
@@ -254,6 +254,20 @@ vju_make_vcldir(const char *dname)
 	return (0);
 }
 
+static int v_matchproto_(jail_make_dir_f)
+vju_make_workdir(const char *dname)
+{
+	AZ(seteuid(0));
+
+	if (mkdir(dname, 0755) < 0 && errno != EEXIST) {
+		MGT_Complain(C_ERR, "Cannot create working directory '%s': %s",
+		    dname, strerror(errno));
+		return (1);
+	}
+	AZ(chown(dname, -1, vju_gid));
+	AZ(seteuid(vju_uid));
+	return (0);
+}
 
 static void v_matchproto_(jail_fixfd_f)
 vju_fixfd(int fd, enum jail_fixfd_e what)
@@ -284,6 +298,7 @@ const struct jail_tech jail_tech_unix = {
 	.init =		vju_init,
 	.master =	vju_master,
 	.make_vcldir =	vju_make_vcldir,
+	.make_workdir =	vju_make_workdir,
 	.fixfd =	vju_fixfd,
 	.subproc =	vju_subproc,
 };

From geoff at uplex.de  Fri Feb 16 16:42:55 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Fri, 16 Feb 2018 17:42:55 +0100
Subject: [master] dcce66a Implement the vtc server and client commands for
 Unix domain sockets.
In-Reply-To: <CABoVN9DoUN4zv0aQHSzv2uxUJCjFeqva9eP636=e5HOWn2Fqeg@mail.gmail.com>
References: <20180216110202.B25CAB1AAF@lists.varnish-cache.org>
 <CABoVN9DoUN4zv0aQHSzv2uxUJCjFeqva9eP636=e5HOWn2Fqeg@mail.gmail.com>
Message-ID: <1908b27c-cc27-2663-e5ed-909245e01abb@uplex.de>

On 02/16/2018 05:01 PM, Dridi Boukelmoune wrote:
>>     UDS code corresponding to VTCP_open() and VTCP_listen_on() is
>>     implemented in the vtc code, since it will only ever be used
>>     there.
> 
> Didn't we consider CLI over UDS too?

We did, but the fork never got that far, because we needed to decide how
to handle UDS in general.

I for one would like to enable UDS for -T and -M as well, but I don't
know what phk's thoughts are about that. First priority is to get code
pushed that enables UDS for -a and backends.

If we agree to support UDS for the CLI, it can probably be done by the
March release, and we can move that code out to VUS for re-use. But for
now, I don't want to generalize something that's only used once.


Best,
Geoff
-- 
** * * UPLEX - Nils Goroll Systemoptimierung

Scheffelstra?e 32
22301 Hamburg

Tel +49 40 2880 5731
Mob +49 176 636 90917
Fax +49 40 42949753

http://uplex.de

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 833 bytes
Desc: OpenPGP digital signature
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20180216/c789801d/attachment.bin>

From nils.goroll at uplex.de  Fri Feb 16 16:51:07 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 16 Feb 2018 16:51:07 +0000 (UTC)
Subject: [master] a4a6eb9 Improve documentation on the secret file
Message-ID: <20180216165107.587FEB85E4@lists.varnish-cache.org>


commit a4a6eb9e0922aec5471642f687b0ac16007f04f7
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri Feb 16 17:50:28 2018 +0100

    Improve documentation on the secret file

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 463f76c..7b1e3eb 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -86,6 +86,8 @@ Basic options
   could later be accessed remotely, starting `varnishd` requires
   local privileges.
 
+.. _opt_n:
+
 -n name
 
   Specify the name for this instance.  This name is used to construct
@@ -205,8 +207,16 @@ Security options
 -S secret-file
 
   Path to a file containing a secret used for authorizing access to
-  the management port. If not provided a new secret will be drawn
-  from the system PRNG.  To disable authentication use ``none``.
+  the management port. To disable authentication use ``none``.
+
+  If this argument is not provided, a secret drawn from the system
+  PRNG will be written to a file called ``_.secret`` in the working
+  directory (see `opt_n`_) with default ownership and permissions of
+  the user having started varnish.
+
+  Thus, users wishing to delegate control over varnish will probably
+  want to create a custom secret file with appropriate permissions
+  (ie. readable by a unix group to delegate control to).
 
 -j <jail[,jailoptions]>
 

From nils.goroll at uplex.de  Fri Feb 16 17:00:43 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 16 Feb 2018 18:00:43 +0100
Subject: [master] dcce66a Implement the vtc server and client commands for
 Unix domain sockets.
In-Reply-To: <1908b27c-cc27-2663-e5ed-909245e01abb@uplex.de>
References: <20180216110202.B25CAB1AAF@lists.varnish-cache.org>
 <CABoVN9DoUN4zv0aQHSzv2uxUJCjFeqva9eP636=e5HOWn2Fqeg@mail.gmail.com>
 <1908b27c-cc27-2663-e5ed-909245e01abb@uplex.de>
Message-ID: <b17134e3-f31d-ae78-3c92-0d523c7b77f4@uplex.de>

FTR, I actually have a use case for -T UDS (running varnishd and varnishadm in
separate containers), but I agree that the priority is on accept socket & backends.

Nils

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 473 bytes
Desc: OpenPGP digital signature
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20180216/690afcaa/attachment.bin>

From fgsch at lodoss.net  Sat Feb 17 12:13:06 2018
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Sat, 17 Feb 2018 12:13:06 +0000 (UTC)
Subject: [master] 140ae48 Whitespace
Message-ID: <20180217121306.9C1CAA79CE@lists.varnish-cache.org>


commit 140ae488266f14df9103a66e72a07be528324a22
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Sat Feb 17 00:33:00 2018 +0000

    Whitespace

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 6801e92..72d0342 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -540,7 +540,7 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2,
 		VSLb(h2->vsl, SLT_Debug, "Missing :method");
 		return (H2SE_PROTOCOL_ERROR); //rfc7540,l,3087,3090
 	}
-	if(req->http->hd[HTTP_HDR_URL].b == NULL) {
+	if (req->http->hd[HTTP_HDR_URL].b == NULL) {
 		VSLb(h2->vsl, SLT_Debug, "Missing :path");
 		return (H2SE_PROTOCOL_ERROR); //rfc7540,l,3087,3090
 	}
diff --git a/bin/varnishtest/vtc_term.c b/bin/varnishtest/vtc_term.c
index be554dc..ee9426d 100644
--- a/bin/varnishtest/vtc_term.c
+++ b/bin/varnishtest/vtc_term.c
@@ -70,7 +70,7 @@ term_scroll(const struct term *tp)
 	char *l;
 
 	l = tp->vram[0];
-	for(i = 0; i < tp->nlin -1; i++)
+	for (i = 0; i < tp->nlin -1; i++)
 		tp->vram[i] = tp->vram[i + 1];
 	tp->vram[i] = l;
 	memset(l, ' ', tp->ncol);
@@ -90,7 +90,7 @@ term_escape(struct term *tp, int c, int n)
 {
 	int i;
 
-	switch(c) {
+	switch (c) {
 	case 'A':
 		// CUU - Cursor up
 		if (tp->arg[0] == -1) tp->arg[0] = 1;
@@ -142,7 +142,7 @@ term_escape(struct term *tp, int c, int n)
 		break;
 	case 'J':
 		// ED - Erase in Display (0=below, 1=above, 2=all)
-		switch(tp->arg[0]) {
+		switch (tp->arg[0]) {
 		case 2:
 			term_clear(tp->vram, tp->nlin, tp->ncol);
 			break;
@@ -195,7 +195,7 @@ term_char(struct term *tp, char c)
 			tp->col--;
 		break;
 	case '\t':
-		while(++tp->col % 8)
+		while (++tp->col % 8)
 			continue;
 		if (tp->col >= tp->ncol) {
 			tp->col = 0;
diff --git a/lib/libvcc/vcc_xref.c b/lib/libvcc/vcc_xref.c
index 775becc..74f7aad 100644
--- a/lib/libvcc/vcc_xref.c
+++ b/lib/libvcc/vcc_xref.c
@@ -198,7 +198,7 @@ vcc_checkaction1(struct vcc *tl, const struct symbol *sym)
 	p = sym->proc;
 	AN(p);
 	AN(p->name);
-	if(p->method == NULL)
+	if (p->method == NULL)
 		return;
 	if (vcc_CheckActionRecurse(tl, p, p->method->ret_bitmap)) {
 		VSB_printf(tl->sb,

From fgsch at lodoss.net  Sat Feb 17 12:13:06 2018
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Sat, 17 Feb 2018 12:13:06 +0000 (UTC)
Subject: [master] fe0b76d Typos
Message-ID: <20180217121306.B452AA79D1@lists.varnish-cache.org>


commit fe0b76d160cdfeb8f252a791b427cb5ffd375269
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Sat Feb 17 00:33:29 2018 +0000

    Typos

diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst
index 9dfea7e..314fc19 100644
--- a/doc/sphinx/reference/vcl_var.rst
+++ b/doc/sphinx/reference/vcl_var.rst
@@ -25,7 +25,7 @@ often with the shorthand "backend" which covers the `vcl_backend_*`
 methods and "client" which covers the rest, except `vcl_init` and
 `vcl_fini`.
 
-When setting a variable, the right hand side of the equalsign
+When setting a variable, the right hand side of the equal sign
 must have the variables type, you cannot assign a STRING to
 a variable of type NUMBER, even if the string is `"42"`.
 (Explicit conversion functions can be found in
@@ -241,7 +241,7 @@ req.http.*
 	
 	The headers of request, things like `req.http.date`.
 
-	The RFCs allow multipl headers with the same name, and both
+	The RFCs allow multiple headers with the same name, and both
 	`set` and `unset` will remove *all* headers with the name given.
 	
 
@@ -723,7 +723,7 @@ beresp.do_gunzip
 
 	Default: false
 
-	Set to `true` to unzip the object while storing it in the
+	Set to `true` to gunzip the object while storing it in the
 	cache.
 
 beresp.was_304

From fgsch at lodoss.net  Sat Feb 17 12:13:06 2018
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Sat, 17 Feb 2018 12:13:06 +0000 (UTC)
Subject: [master] 23569a2 Grace and keep are fixed values
Message-ID: <20180217121306.CB2B7A79D6@lists.varnish-cache.org>


commit 23569a2323fe6f5ade759f8621f698e6f24c2c9d
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Sat Feb 17 12:00:17 2018 +0000

    Grace and keep are fixed values

diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst
index 314fc19..8aa8983 100644
--- a/doc/sphinx/reference/vcl_var.rst
+++ b/doc/sphinx/reference/vcl_var.rst
@@ -924,7 +924,7 @@ obj.grace
 
 	Readable from: vcl_hit, vcl_deliver
 
-	The object's remaining grace period in seconds.
+	The object's grace period in seconds.
 	
 
 obj.keep
@@ -933,7 +933,7 @@ obj.keep
 
 	Readable from: vcl_hit, vcl_deliver
 
-	The object's remaining keep period in seconds.
+	The object's keep period in seconds.
 	
 
 obj.uncacheable

From hermunn at varnish-software.com  Sun Feb 18 22:47:06 2018
From: hermunn at varnish-software.com (Pål Hermunn Johansen)
Date: Sun, 18 Feb 2018 22:47:06 +0000 (UTC)
Subject: [master] 33143e0 Introduce ttl_now and the new way of calculating
 TTLs in VCL
Message-ID: <20180218224706.AB673A9A1B@lists.varnish-cache.org>


commit 33143e05c0720e5be0df4ac5b20a0fcc8a6874e8
Author: P?l Hermunn Johansen <hermunn at varnish-software.com>
Date:   Sun Feb 18 23:45:08 2018 +0100

    Introduce ttl_now and the new way of calculating TTLs in VCL
    
    A new fucntion, ttl_now(VRT_CTX), defines what "now" is when ttl
    and age are calculated in various VCL subs. To sum up,
    
    * Before a backend fetch on the client side (vcl_recv, vcl_hit,
      vcl_miss) we use t_req from the request. This is the significance
      in this commit, and fixes the bug demonstrated by r02555.vtc.
    * On the backend side, most notably vcl_backend_responce, we keep
      the old "now" by simply using ctx->now.
    * In vcl_deliver we use ctx->now, as before.
    
    It was necessary to make all purges use t_req as their base time.
    Then, to not break c00041.vtc it was necessary to change from ">="
    to ">" in HSH_Lookup.
    
    All VMODs that currently use HSH_purge must change to using
    VRT_purge.

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 2aac129..5a0b544 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -436,7 +436,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 				continue;
 		}
 
-		if (EXP_Ttl(req, oc) >= req->t_req) {
+		if (EXP_Ttl(req, oc) > req->t_req) {
 			/* If still valid, use it */
 			assert(oh->refcnt > 1);
 			assert(oc->objhead == oh);
@@ -590,13 +590,12 @@ hsh_rush2(struct worker *wrk, struct rush *r)
  */
 
 unsigned
-HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace,
-double keep)
+HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl_now, double ttl,
+double grace, double keep)
 {
 	struct objcore *oc, **ocp;
 	unsigned spc, ospc, nobj, n, n_tot = 0;
 	int more = 0;
-	double now;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
@@ -623,7 +622,6 @@ double keep)
 		ocp = (void*)wrk->aws->f;
 		Lck_Lock(&oh->mtx);
 		assert(oh->refcnt > 0);
-		now = VTIM_real();
 		VTAILQ_FOREACH(oc, &oh->objcs, hsh_list) {
 			CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 			assert(oc->objhead == oh);
@@ -663,7 +661,7 @@ double keep)
 		for (n = 0; n < nobj; n++) {
 			oc = ocp[n];
 			CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-			EXP_Rearm(oc, now, ttl, grace, keep);
+			EXP_Rearm(oc, ttl_now, ttl, grace, keep);
 			(void)HSH_DerefObjCore(wrk, &oc, 0);
 		}
 		n_tot += nobj;
diff --git a/bin/varnishd/cache/cache_objhead.h b/bin/varnishd/cache/cache_objhead.h
index f4b7055..6bcfbd9 100644
--- a/bin/varnishd/cache/cache_objhead.h
+++ b/bin/varnishd/cache/cache_objhead.h
@@ -73,7 +73,7 @@ enum lookup_e HSH_Lookup(struct req *, struct objcore **, struct objcore **,
     int always_insert);
 void HSH_Ref(struct objcore *o);
 void HSH_AddString(struct req *, void *ctx, const char *str);
-unsigned HSH_Purge(struct worker *, struct objhead *, double ttl, double grace,
-    double keep);
+unsigned HSH_Purge(struct worker *, struct objhead *, double ttl_now,
+    double ttl, double grace, double keep);
 struct objcore *HSH_Private(const struct worker *wrk);
 void HSH_Abandon(struct objcore *oc);
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 142425c..56bf79e 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -933,7 +933,7 @@ cnt_recv(struct worker *wrk, struct req *req)
 /*--------------------------------------------------------------------
  * Find the objhead, purge it.
  *
- * XXX: We should ask VCL if we should fetch a new copy of the object.
+ * In VCL, a restart is necessary to get a new object
  */
 
 static enum req_fsm_nxt
@@ -958,7 +958,7 @@ cnt_purge(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(boc, OBJCORE_MAGIC);
 	VRY_Finish(req, DISCARD);
 
-	(void)HSH_Purge(wrk, boc->objhead, 0, 0, 0);
+	(void)HSH_Purge(wrk, boc->objhead, req->t_req, 0, 0, 0);
 
 	AZ(HSH_DerefObjCore(wrk, &boc, 1));
 
diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index ef05b9c..7bde5b8 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -1126,6 +1126,8 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo,
 		ctx.ws = req->ws;
 	}
 	if (bo != NULL) {
+		if(req)
+			assert(method == VCL_MET_PIPE);
 		CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 		CHECK_OBJ_NOTNULL(bo->vcl, VCL_MAGIC);
 		vsl = bo->vsl;
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 296fa45..053db5c 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -661,7 +661,7 @@ VRT_purge(VRT_CTX, double ttl, double grace, double keep)
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req->wrk, WORKER_MAGIC);
 	return (HSH_Purge(ctx->req->wrk, ctx->req->objcore->objhead,
-	    ttl, grace, keep));
+	    ctx->req->t_req, ttl, grace, keep));
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index bb82fd6..ef6da50 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -35,6 +35,8 @@
 #include "cache_varnishd.h"
 #include "common/heritage.h"
 
+#include "vcl.h"
+
 #include "cache_director.h"
 #include "vrt_obj.h"
 
@@ -516,10 +518,23 @@ VRT_r_bereq_retries(VRT_CTX)
  *	ttl is relative to t_origin
  *	grace&keep are relative to ttl
  * In VCL:
- *	ttl is relative to now
+ *	ttl is relative to "ttl_now", which is t_req on the client
+ *	side, except in vcl_deliver, where it is ctx->now. On the
+ *	fetch side "ttl_now" is ctx->now (which is bo->t_prev).
  *	grace&keep are relative to ttl
  */
 
+static double ttl_now(VRT_CTX)
+{
+	if (ctx->bo) {
+		return (ctx->now);
+	} else {
+		CHECK_OBJ(ctx->req, REQ_MAGIC);
+		return (ctx->method == VCL_MET_DELIVER
+		    ? ctx->now : ctx->req->t_req);
+	}
+}
+
 #define VRT_DO_EXP_L(which, oc, fld, offset)			\
 								\
 void								\
@@ -551,14 +566,14 @@ VRT_r_##which##_##fld(VRT_CTX)					\
 }
 
 VRT_DO_EXP_R(obj, ctx->req->objcore, ttl,
-    ctx->now - ctx->req->objcore->t_origin)
+    ttl_now(ctx) - ctx->req->objcore->t_origin)
 VRT_DO_EXP_R(obj, ctx->req->objcore, grace, 0)
 VRT_DO_EXP_R(obj, ctx->req->objcore, keep, 0)
-
 VRT_DO_EXP_L(beresp, ctx->bo->fetch_objcore, ttl,
-    ctx->now - ctx->bo->fetch_objcore->t_origin)
+    ttl_now(ctx) - ctx->bo->fetch_objcore->t_origin)
 VRT_DO_EXP_R(beresp, ctx->bo->fetch_objcore, ttl,
-    ctx->now - ctx->bo->fetch_objcore->t_origin)
+    ttl_now(ctx) - ctx->bo->fetch_objcore->t_origin)
+
 VRT_DO_EXP_L(beresp, ctx->bo->fetch_objcore, grace, 0)
 VRT_DO_EXP_R(beresp, ctx->bo->fetch_objcore, grace, 0)
 VRT_DO_EXP_L(beresp, ctx->bo->fetch_objcore, keep, 0)
@@ -574,7 +589,7 @@ VRT_r_##which##_##age(VRT_CTX)					\
 {								\
 								\
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);			\
-	return (ctx->now - oc->t_origin);			\
+	return (ttl_now(ctx) - oc->t_origin);			\
 }
 
 VRT_DO_AGE_R(obj, ctx->req->objcore)
diff --git a/bin/varnishtest/tests/r02555.vtc b/bin/varnishtest/tests/r02555.vtc
new file mode 100644
index 0000000..27182e4
--- /dev/null
+++ b/bin/varnishtest/tests/r02555.vtc
@@ -0,0 +1,68 @@
+varnishtest "Expiry during processing"
+
+# When a object runs out of ttl+grace during processing of a
+# request, we want the calculation in the builtin VCL to match
+# the calculation of hit+grace in the C code.
+
+barrier b1 sock 2
+barrier b2 sock 2
+
+server s1 {
+	rxreq
+	expect req.url == "/1"
+	txresp
+
+	# If we get a second request to /1 here, it will be because a
+	# hit was became a miss in the builtin sub vcl_hit or because
+	# we never got to vcl_hit. We want a hit and a deliver.
+
+	rxreq
+	expect req.url == "/2"
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import std;
+	import vtc;
+
+	sub vcl_recv {
+		if (req.http.Sleep) {
+			# This will make the object expire while inside this VCL sub
+			vtc.barrier_sync("${b1_sock}");
+			vtc.barrier_sync("${b2_sock}");
+		}
+		# Update the last timestamp inside varnish
+		std.timestamp("T");
+	}
+	sub vcl_hit {
+		set req.http.X-was-hit = "true";
+		# no return here. Will the builtin VCL take us to vcl_miss?
+	}
+	sub vcl_miss {
+		set req.http.X-was-miss = "true";
+	}
+	sub vcl_backend_response {
+		# Very little ttl, a lot of keep
+		set beresp.ttl = 1s;
+		set beresp.grace = 0s;
+		set beresp.keep = 1w;
+	}
+} -start
+
+client c1 {
+	delay .5
+	txreq -url "/1"
+	rxresp
+	txreq -url "/1" -hdr "Sleep: true"
+	rxresp
+	# Final request to verify that the right amount of requests got to the backend
+	txreq -url "/2"
+	rxresp
+} -start
+
+# help for sleeping inside of vcl
+barrier b1 sync
+delay 1.5
+barrier b2 sync
+
+client c1 -wait
diff --git a/bin/varnishtest/tests/s00007.vtc b/bin/varnishtest/tests/s00007.vtc
new file mode 100644
index 0000000..39eecf5
--- /dev/null
+++ b/bin/varnishtest/tests/s00007.vtc
@@ -0,0 +1,42 @@
+varnishtest "Effective TTL for for a slow backend"
+
+server s1 {
+	rxreq
+	delay 2
+	txresp -body "foo"
+
+	# The second request is never used, but is here to give a
+	# better error if varnish decides to fetch the object the
+	# second time
+
+	rxreq
+	txresp -body "bar"
+} -start
+
+varnish v1 -arg "-p default_ttl=3 -p default_grace=0" -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.http.X-ttl = beresp.ttl;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.body == "foo"
+	expect resp.http.x-ttl <= 3
+	expect resp.http.x-ttl >= 2
+	delay 2
+
+	# It is now 2 seconds since the first response was received
+	# from the backend, but 4 seconds since the first request was
+	# sent to the backend. Timeout is 3 seconds, and here we
+	# consider the object _not_ expired, and thus do not want a
+	# refetch.
+
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.body == "foo"
+} -run
+
diff --git a/bin/varnishtest/tests/s00008.vtc b/bin/varnishtest/tests/s00008.vtc
new file mode 100644
index 0000000..054c2eb
--- /dev/null
+++ b/bin/varnishtest/tests/s00008.vtc
@@ -0,0 +1,28 @@
+varnishtest "setting ttl in vcl_backend_response for slow backends"
+
+server s1 {
+	rxreq
+	delay 2
+	txresp -body "foo"
+	# The second request is never used
+	rxreq
+	txresp -body "bar"
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.ttl = 10s;
+	}
+	sub vcl_deliver {
+		set resp.http.X-ttl = obj.ttl;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.body == "foo"
+	expect resp.http.X-ttl <= 10
+	expect resp.http.X-ttl >= 9
+} -run
diff --git a/bin/varnishtest/tests/s00009.vtc b/bin/varnishtest/tests/s00009.vtc
new file mode 100644
index 0000000..4f7d28d
--- /dev/null
+++ b/bin/varnishtest/tests/s00009.vtc
@@ -0,0 +1,46 @@
+varnishtest "Correct obj.ttl is when backend and processing are slow"
+
+barrier b1 sock 2
+barrier b2 sock 2
+
+server s1 {
+	rxreq
+	delay 2
+	txresp -body "foo"
+	# The second request is never used
+	rxreq
+	txresp -body "bar"
+} -start
+
+varnish v1 -vcl+backend {
+	import vtc;
+	sub vcl_backend_response {
+		# Simulate processing for 1.5 sec
+		vtc.barrier_sync("${b1_sock}");
+		vtc.barrier_sync("${b2_sock}");
+
+		# Moving this above the processing should not change
+		# anything.
+
+		set beresp.ttl = 10s;
+	}
+	sub vcl_deliver {
+		set resp.http.X-ttl = obj.ttl;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.body == "foo"
+	expect resp.http.X-ttl <= 9
+	expect resp.http.X-ttl >= 8
+} -start
+
+# help for sleeping inside of vcl
+barrier b1 sync
+delay 1.5
+barrier b2 sync
+
+client c1 -wait

From noreply at github.com  Mon Feb 19 08:44:08 2018
From: noreply at github.com (GitHub)
Date: Mon, 19 Feb 2018 08:44:08 +0000 (UTC)
Subject: [master] dc6c652 Merge pull request #2569 from
 xcir/feature/vsc_nuke_limited
Message-ID: <20180219084408.30B75B6536@lists.varnish-cache.org>


commit dc6c6520188fa6ffd5b9262cf1cbb86f8e5df4cc
Merge: 33143e0 b201fc7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Feb 19 08:43:09 2018 +0000

    Merge pull request #2569 from xcir/feature/vsc_nuke_limited
    
    Add n_lru_limited counter

From phk at FreeBSD.org  Mon Feb 19 09:47:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 19 Feb 2018 09:47:06 +0000 (UTC)
Subject: [master] 73bd54c Add support code for living with multiple VCL
 versions at the same time.
Message-ID: <20180219094706.4E175B782C@lists.varnish-cache.org>


commit 73bd54ccaae36f288c0a54b82e171b3d14affce4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Feb 19 09:26:18 2018 +0000

    Add support code for living with multiple VCL versions at the same time.

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index 7bde5b8..12280e1 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -518,6 +518,12 @@ VCL_Open(const char *fn, struct vsb *msg)
 		(void)dlclose(dlh);
 		return (NULL);
 	}
+	if (cnf->syntax < heritage.min_vcl || cnf->syntax > heritage.max_vcl) {
+		VSB_printf(msg, "Compiled VCL version (%.1f) not supported.\n",
+		    .1 * cnf->syntax);
+		(void)dlclose(dlh);
+		return (NULL);
+	}
 	ALLOC_OBJ(vcl, VCL_MAGIC);
 	AN(vcl);
 	AZ(errno=pthread_rwlock_init(&vcl->temp_rwl, NULL));
diff --git a/bin/varnishd/common/heritage.h b/bin/varnishd/common/heritage.h
index e59261e..64131d6 100644
--- a/bin/varnishd/common/heritage.h
+++ b/bin/varnishd/common/heritage.h
@@ -81,6 +81,9 @@ struct heritage {
 	long				mgt_pid;
 
 	struct vsmw			*proc_vsmw;
+
+	unsigned			min_vcl;
+	unsigned			max_vcl;
 };
 
 extern struct heritage heritage;
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index f23bc75..8e968ed 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -46,6 +46,7 @@
 #include "common/heritage.h"
 
 #include "hash/hash_slinger.h"
+#include "libvcc.h"
 #include "vav.h"
 #include "vcli_serve.h"
 #include "vend.h"
@@ -274,6 +275,9 @@ mgt_initialize(struct cli *cli)
 	mgt_cli_init_cls();		// CLI commands can be registered
 
 	MCF_InitParams(cli);
+
+	VCC_VCL_Range(&heritage.min_vcl, &heritage.max_vcl);
+
 	cli_check(cli);
 }
 
diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst
index 8aa8983..f94841a 100644
--- a/doc/sphinx/reference/vcl_var.rst
+++ b/doc/sphinx/reference/vcl_var.rst
@@ -29,7 +29,7 @@ When setting a variable, the right hand side of the equal sign
 must have the variables type, you cannot assign a STRING to
 a variable of type NUMBER, even if the string is `"42"`.
 (Explicit conversion functions can be found in
-:ref:`vmod_std(3)` .
+:ref:`vmod_std(3)`).
 
 local, server, remote and client
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/include/libvcc.h b/include/libvcc.h
index 78bf5fa..31b2a7f 100644
--- a/include/libvcc.h
+++ b/include/libvcc.h
@@ -40,6 +40,7 @@ void VCC_Unsafe_Path(struct vcc *, unsigned);
 void VCC_VCL_path(struct vcc *, const char *);
 void VCC_VMOD_path(struct vcc *, const char *);
 void VCC_Predef(struct vcc *, const char *type, const char *name);
+void VCC_VCL_Range(unsigned *, unsigned *);
 
 struct vsb *VCC_Compile(struct vcc *, struct vsb **,
     const char *vclsrc, const char *vclsrcfile);
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index ea8fdf8..758b7e7 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -670,6 +670,19 @@ vcc_CompileSource(struct vcc *tl, struct source *sp)
 }
 
 /*--------------------------------------------------------------------
+ * Report the range of VCL language we support
+ */
+void
+VCC_VCL_Range(unsigned *lo, unsigned *hi)
+{
+
+	AN(lo);
+	*lo = VCL_LOW;
+	AN(hi);
+	*hi = VCL_HIGH;
+}
+
+/*--------------------------------------------------------------------
  * Compile the VCL code in the argument.  Error messages, if any are
  * formatted into the vsb.
  */
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index dd9a33a..bdbb3a6 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -41,9 +41,18 @@
 
 #include "vcc_token_defs.h"
 
-#ifndef NULL
-#define NULL ((void*)0)
-#endif
+/*---------------------------------------------------------------------
+ * VCL version stuff
+ */
+
+#define VCL_LOW		40		// Lowest VCC supports
+#define VCL_HIGH	41		// Highest VCC supports
+
+// Specific VCL versions
+#define	VCL_40		40
+#define	VCL_41		41
+
+/*---------------------------------------------------------------------*/
 
 struct vsb;
 struct token;
@@ -126,11 +135,15 @@ struct symbol {
 	VTAILQ_ENTRY(symbol)		list;
 	VTAILQ_HEAD(,symbol)		children;
 
+	char				*name;
+	unsigned			nlen;
+
+	unsigned			lorev;
+	unsigned			hirev;
+
 	struct symbol			*parent;
 	const char			*vmod;
 
-	char				*name;
-	unsigned			nlen;
 	sym_wildcard_t			*wildcard;
 	vcc_kind_t			kind;
 
diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index 16ba0b7..bccd27e 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -367,7 +367,7 @@ vcc_Parse(struct vcc *tl)
 	}
 	tok = tl->t;
 	vcc_ParseVcl(tl);
-	if (tok->src->syntax < 40 || tok->src->syntax > 41) {
+	if (tok->src->syntax < VCL_LOW || tok->src->syntax > VCL_HIGH) {
 		VSB_printf(tl->sb, "VCL version %.1f not supported.\n",
 		    .1 * tok->src->syntax);
 		vcc_ErrWhere2(tl, tok, tl->t);
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index b5df60c..a039006 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -93,6 +93,8 @@ vcc_new_symbol(struct vcc *tl, const char *b, const char *e)
 	VTAILQ_INIT(&sym->children);
 	sym->kind = SYM_NONE;
 	sym->type = VOID;
+	sym->lorev = VCL_LOW;
+	sym->hirev = VCL_HIGH;
 	return (sym);
 }
 
@@ -140,7 +142,8 @@ VCC_Symbol(struct vcc *tl, struct symbol *parent,
 			break;
 		if ((kind == SYM_NONE && kind == sym->kind))
 			continue;
-		if (tl->syntax < 41 && (kind != SYM_NONE && kind != sym->kind))
+		if (tl->syntax < VCL_41 &&
+		     (kind != SYM_NONE && kind != sym->kind))
 			continue;
 		break;
 	}
@@ -192,7 +195,7 @@ VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x)
 	struct symbol *sym;
 
 	AN(e);
-	if (tl->syntax >= 41 && e == SYMTAB_CREATE && kind != SYM_SUB &&
+	if (tl->syntax >= VCL_41 && e == SYMTAB_CREATE && kind != SYM_SUB &&
 	    (tl->t->b[0] == 'v'|| tl->t->b[0] == 'V') &&
 	    (tl->t->b[1] == 'c'|| tl->t->b[1] == 'C') &&
 	    (tl->t->b[2] == 'l'|| tl->t->b[2] == 'L')) {
@@ -213,6 +216,20 @@ VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x)
 		vcc_ErrWhere(tl, tl->t);
 		return (NULL);
 	}
+	if (sym->lorev > tl->syntax || sym->hirev < tl->syntax) {
+		VSB_printf(tl->sb, "Symbol ");
+		vcc_ErrToken(tl, tl->t);
+		if (sym->lorev > tl->syntax)
+			VSB_printf(tl->sb, " needs vcl %.1f or higher.",
+			    .1 * sym->lorev);
+		else
+			VSB_printf(tl->sb,
+			    " is discontinued after vcl %.1f.",
+			    .1 * sym->hirev);
+		VSB_cat(tl->sb, "\nAt: ");
+		vcc_ErrWhere(tl, tl->t);
+		return(NULL);
+	}
 	if (kind != SYM_NONE && kind != sym->kind) {
 		VSB_printf(tl->sb, "Symbol ");
 		vcc_ErrToken(tl, tl->t);

From phk at FreeBSD.org  Mon Feb 19 10:35:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 19 Feb 2018 10:35:08 +0000 (UTC)
Subject: [master] 5cc47ea Finally fix #2495
Message-ID: <20180219103508.D091CB8722@lists.varnish-cache.org>


commit 5cc47eaa8a174d6f072c68d996ff17b38ccd16eb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Feb 19 10:30:28 2018 +0000

    Finally fix #2495

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 5a0b544..e62e6c9 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -63,7 +63,6 @@
 #include "hash/hash_slinger.h"
 
 #include "vsha256.h"
-#include "vtim.h"
 
 struct rush {
 	unsigned		magic;
diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c
index 9109f9b..edb9907 100644
--- a/bin/varnishd/http1/cache_http1_fsm.c
+++ b/bin/varnishd/http1/cache_http1_fsm.c
@@ -178,6 +178,51 @@ http1_req_fail(struct req *req, enum sess_close reason)
 		SES_Close(req->sp, reason);
 }
 
+/*----------------------------------------------------------------------
+ */
+
+static int
+http1_req_cleanup(struct sess *sp, struct worker *wrk, struct req *req)
+{
+	AZ(wrk->aws->r);
+	AZ(req->ws->r);
+	Req_Cleanup(sp, wrk, req);
+
+	if (sp->fd >= 0 && req->doclose != SC_NULL)
+		SES_Close(sp, req->doclose);
+
+	if (sp->fd < 0) {
+		wrk->stats->sess_closed++;
+		AZ(req->vcl);
+		Req_Release(req);
+		SES_Delete(sp, SC_NULL, NAN);
+		return (1);
+	}
+
+	return (0);
+}
+
+/*----------------------------------------------------------------------
+ * Clean up a req from waiting list which cannot complete
+ */
+
+static void
+http1_cleanup_waiting(struct worker *wrk, struct req *req,
+    enum sess_close reason)
+{
+	struct sess *sp;
+
+	sp = req->sp;
+	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+	AN(req->ws->r);
+	WS_Release(req->ws, 0);
+	AN(req->hash_objhead);
+	(void)HSH_DerefObjHead(wrk, &req->hash_objhead);
+	AZ(req->hash_objhead);
+	SES_Close(sp, reason);
+	AN(http1_req_cleanup(sp, wrk, req));
+}
+
 static void v_matchproto_(vtr_reembark_f)
 http1_reembark(struct worker *wrk, struct req *req)
 {
@@ -188,19 +233,15 @@ http1_reembark(struct worker *wrk, struct req *req)
 
 	http1_setstate(sp, H1BUSY);
 
-	if (!SES_Reschedule_Req(req, TASK_QUEUE_REQ))
+	if (!DO_DEBUG(DBG_FAILRESCHED) &&
+	    !SES_Reschedule_Req(req, TASK_QUEUE_REQ))
 		return;
 
 	/* Couldn't schedule, ditch */
 	wrk->stats->busy_wakeup--;
 	wrk->stats->busy_killed++;
-	AN (req->vcl);
-	VCL_Rel(&req->vcl);
-	Req_AcctLogCharge(wrk->stats, req);
-	Req_Release(req);
-	SES_Delete(sp, SC_OVERLOAD, NAN);
-	DSL(DBG_WAITINGLIST, req->vsl->wid, "kill from waiting list");
-	usleep(10000);
+	VSLb(req->vsl, SLT_Error, "Fail to reschedule req from waiting list");
+	http1_cleanup_waiting(wrk, req, SC_OVERLOAD);
 }
 
 static int v_matchproto_(vtr_minimal_response_f)
@@ -327,30 +368,6 @@ http1_dissect(struct worker *wrk, struct req *req)
 /*----------------------------------------------------------------------
  */
 
-static int
-http1_req_cleanup(struct sess *sp, struct worker *wrk, struct req *req)
-{
-	AZ(wrk->aws->r);
-	AZ(req->ws->r);
-	Req_Cleanup(sp, wrk, req);
-
-	if (sp->fd >= 0 && req->doclose != SC_NULL)
-		SES_Close(sp, req->doclose);
-
-	if (sp->fd < 0) {
-		wrk->stats->sess_closed++;
-		AZ(req->vcl);
-		Req_Release(req);
-		SES_Delete(sp, SC_NULL, NAN);
-		return (1);
-	}
-
-	return (0);
-}
-
-/*----------------------------------------------------------------------
- */
-
 static void
 HTTP1_Session(struct worker *wrk, struct req *req)
 {
@@ -477,13 +494,7 @@ HTTP1_Session(struct worker *wrk, struct req *req)
 			 * Check to see if the remote has left.
 			 */
 			if (VTCP_check_hup(sp->fd)) {
-				AN(req->ws->r);
-				WS_Release(req->ws, 0);
-				AN(req->hash_objhead);
-				(void)HSH_DerefObjHead(wrk, &req->hash_objhead);
-				AZ(req->hash_objhead);
-				SES_Close(sp, SC_REM_CLOSE);
-				AN(http1_req_cleanup(sp, wrk, req));
+				http1_cleanup_waiting(wrk, req, SC_REM_CLOSE);
 				return;
 			}
 			http1_setstate(sp, H1PROC);
diff --git a/bin/varnishtest/tests/c00013.vtc b/bin/varnishtest/tests/c00013.vtc
index e2bbedb..1fcccac 100644
--- a/bin/varnishtest/tests/c00013.vtc
+++ b/bin/varnishtest/tests/c00013.vtc
@@ -46,5 +46,64 @@ client c2 {
 
 client c1 -wait
 
+varnish v1 -vsl_catchup
 varnish v1 -expect busy_sleep >= 1
 varnish v1 -expect busy_wakeup >= 1
+varnish v1 -stop
+
+##################################################
+# Now try again where getting a thread fails
+
+barrier b3 cond 2
+barrier b4 cond 2
+
+server s3 {
+	rxreq
+	expect req.url == "/foo"
+	send "HTTP/1.0 200 OK\r\nConnection: close\r\n\r\n"
+	delay .2
+	barrier b3 sync
+	delay .2
+	send "line1\n"
+	delay .2
+	barrier b4 sync
+	send "line2\n"
+} -start
+
+varnish v3 -vcl+backend {
+	sub vcl_backend_fetch {
+		set bereq.backend = s3;
+	}
+	sub vcl_backend_response {
+		set beresp.do_stream = false;
+	}
+} -start
+
+varnish v3 -cliok "param.set debug +failresched"
+
+varnish v3 -cliok "param.set debug +syncvsl"
+
+client c3 -connect ${v3_sock} {
+	txreq -url "/foo" -hdr "client: c3"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 12
+	expect resp.http.x-varnish == "1001"
+} -start
+
+barrier b3 sync
+
+client c4 -connect ${v3_sock} {
+	txreq -url "/foo" -hdr "client: c4"
+	delay .2
+	barrier b4 sync
+	expect_close
+} -run
+
+client c3 -wait
+
+varnish v1 -vsl_catchup
+varnish v3 -expect busy_sleep >= 1
+varnish v3 -expect busy_wakeup == 0
+varnish v3 -expect busy_killed == 1
+varnish v3 -expect sc_overload == 1
diff --git a/include/tbl/debug_bits.h b/include/tbl/debug_bits.h
index 4ac3ef6..b451f33 100644
--- a/include/tbl/debug_bits.h
+++ b/include/tbl/debug_bits.h
@@ -50,6 +50,7 @@ DEBUG_BIT(H2_NOCHECK,		h2_nocheck,	"Disable various H2 checks")
 DEBUG_BIT(VMOD_SO_KEEP,		vmod_so_keep,	"Keep copied VMOD libraries")
 DEBUG_BIT(PROCESSORS,		processors,	"Fetch/Deliver processors")
 DEBUG_BIT(PROTOCOL,		protocol,	"Protocol debugging")
+DEBUG_BIT(FAILRESCHED,		failresched,	"Fail from waiting list")
 #undef DEBUG_BIT
 
 /*lint -restore */

From phk at FreeBSD.org  Mon Feb 19 10:56:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 19 Feb 2018 10:56:08 +0000 (UTC)
Subject: [master] acb7c01 Flexelinting
Message-ID: <20180219105608.68990B8D57@lists.varnish-cache.org>


commit acb7c01214bce59de85e754d6f7d0c85cdea21bf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Feb 19 10:55:22 2018 +0000

    Flexelinting

diff --git a/lib/libvcc/flint.lnt b/lib/libvcc/flint.lnt
index 6b931e0..7d7d60d 100644
--- a/lib/libvcc/flint.lnt
+++ b/lib/libvcc/flint.lnt
@@ -1,43 +1,5 @@
--passes=3
 
-// Review all below this line
-
--printf_code( H, void *, unsigned)
--printf_code( ju, long long unsigned)
--printf_code( jx, long long unsigned)
-
-+libh ../../config.h
--header(../../config.h)
--sem(lbv_assert, r_no)
--sem(strchr, 1p, type(1), 2n == 0 ? (@p < 1p) : (@p < 1p || @p == 0 ))
 -sem(vcc_new_source, custodial(1))
 
--emacro((???),va_arg)	// the va_arg() macro can yield 415, 416, 661, 662
-			// 796 and 797 (out-of-bounds errors).
-
-
--emacro(413, offsetof)	// likely null pointer
-
-// -ffc    // No automatic custody
-
--esym(534, vsb_printf)	// Ignoring return value of function
--esym(534, vsb_cat)	// Ignoring return value of function
--esym(534, vsb_bcat)	// Ignoring return value of function
--esym(534, vsb_vprintf)	// Ignoring return value of function
--esym(534, memset)	// Ignoring return value of function
--e788			// enum constant 'HND_Unclass' not used within defaulted switch
--e732			// Loss of sign (arg. no. 2) (int to unsigned int)
-
--emacro(835, EXPR_VAR)	// // Info 835: A zero has been given as right argument to operator '<<'
-
--e763	// Redundant declaration for symbol '...' previously declared
-
-
--e534	// Ignoring return value of function
--e506	// Constant value boolean
--e774	// Boolean within 'if' always evaluates to False
--e713	// Loss of precision (assignment) (unsigned long long to long long)
--e539	// Did not expect positive indentation
--e734	// Loss of precision (assignment) (31 bits to 8 bits)
--e747	// Significant prototype coercion (arg. no. 2) long
--e712	// Loss of precision (assignment) (long long to
+-emacro(835, EXPR_VAR)	// Info 835: A zero has been given as right argument to operator '<<'
+-esym(755, VCL_40)	// Not used (right now)
diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c
index d3536ab..eb6304b 100644
--- a/lib/libvcc/vcc_acl.c
+++ b/lib/libvcc/vcc_acl.c
@@ -391,9 +391,9 @@ vcc_acl_emit(struct vcc *tl, const char *name, const char *rname, int anon)
 			depth--;
 		}
 
-		m = ae->mask;
+		m = (int)ae->mask;
+		assert(m >= l*8);
 		m -= l * 8;
-		assert(m >= 0);
 
 		/* Do whole byte compares */
 		for (i = l; m >= 8; m -= 8, i++) {
@@ -416,7 +416,7 @@ vcc_acl_emit(struct vcc *tl, const char *name, const char *rname, int anon)
 			depth = i;
 		}
 
-		i = (ae->mask + 7) / 8;
+		i = ((int)ae->mask + 7) / 8;
 
 		if (!anon) {
 			Fh(tl, 0, "\t%*sVRT_acl_log(ctx, \"%sMATCH %s \" ",
diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index bccd27e..95b459f 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -430,5 +430,5 @@ vcc_Parse_Init(struct vcc *tl)
 	struct toplev *tp;
 
 	for (tp = toplev; tp->name != NULL; tp++)
-		VCC_MkSym(tl, tp->name, SYM_NONE);
+		AN(VCC_MkSym(tl, tp->name, SYM_NONE));
 }

From phk at FreeBSD.org  Mon Feb 19 10:56:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 19 Feb 2018 10:56:08 +0000 (UTC)
Subject: [master] 2a35476 Flexelinting
Message-ID: <20180219105608.873D4B8D5A@lists.varnish-cache.org>


commit 2a35476217c72718b8b265cc14731bfbe81b2b26
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Feb 19 10:55:41 2018 +0000

    Flexelinting

diff --git a/lib/libvarnish/vus.c b/lib/libvarnish/vus.c
index 3b4ff20..8471d15 100644
--- a/lib/libvarnish/vus.c
+++ b/lib/libvarnish/vus.c
@@ -85,7 +85,7 @@ VUS_bind(const struct sockaddr_un *uds, const char **errp)
 		return (-1);
 	}
 
-	if (bind(sd, (const struct sockaddr *)uds, sl) != 0) {
+	if (bind(sd, (const void*)uds, sl) != 0) {
 		if (errp != NULL)
 			*errp = "bind(2)";
 		e = errno;

From daghf at varnish-software.com  Mon Feb 19 10:58:06 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Mon, 19 Feb 2018 10:58:06 +0000 (UTC)
Subject: [master] 8f4671a Also test DBG_FAILRESCHED bit for h2_reembark
Message-ID: <20180219105806.D4BB6B8EA5@lists.varnish-cache.org>


commit 8f4671af29183c2c15e85fc1d3cdc656f9a44759
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Mon Feb 19 11:56:35 2018 +0100

    Also test DBG_FAILRESCHED bit for h2_reembark

diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c
index 6fe7bf4..f2d83b4 100644
--- a/bin/varnishd/http2/cache_http2_session.c
+++ b/bin/varnishd/http2/cache_http2_session.c
@@ -399,7 +399,8 @@ h2_reembark(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
 	assert(req->transport == &H2_transport);
 
-	if (!SES_Reschedule_Req(req, TASK_QUEUE_STR))
+	if (!DO_DEBUG(DBG_FAILRESCHED) &&
+	    !SES_Reschedule_Req(req, TASK_QUEUE_STR))
 		return;
 
 	/* Couldn't schedule, ditch */

From martin at varnish-software.com  Mon Feb 19 12:51:08 2018
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon, 19 Feb 2018 12:51:08 +0000 (UTC)
Subject: [master] 5117664 Accurate byte counters
Message-ID: <20180219125108.5E981BB080@lists.varnish-cache.org>


commit 51176640603b123a38b49ae271cb9c9de9df1d28
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Tue Feb 6 15:21:39 2018 +0100

    Accurate byte counters
    
    There was a regression from Varnish 4.0 to 4.1, where the response
    bytes was counted as the number of bytes fed to the outgoing write
    vector, rather than the bytes that was actually handed off to the OS'
    socket buffer. This would cause for many cases the complete object
    size counted as transmitted bytes, even though the client hung up the
    connection early.
    
    This patch changes the counters to show the amount of bytes sent as
    reported from the write() system calls rather than the bytes we planned
    and prepared to send. The counters will include any protocol overhead (ie
    chunked encoding in HTTP/1 and the frame headers in HTTP/2).
    
    ESI subrequests will as before in their log transactions report the number
    of bytes it (and any subrequests below it) contributed to the total body
    bytes produced.
    
    Some test cases have been adjusted to account for the new counter behaviour.
    
    Fixes: 2558

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 4d31680..204c3c7 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -233,7 +233,8 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk,
 		if (vtp->state != VTP_STATE_STOLEN)
 			extrachance = 0;
 
-		i = V1F_SendReq(wrk, bo, &bo->acct.bereq_hdrbytes, 0);
+		i = V1F_SendReq(wrk, bo, &bo->acct.bereq_hdrbytes,
+				&bo->acct.bereq_bodybytes, 0);
 
 		if (vtp->state != VTP_STATE_USED) {
 			if (VTP_Wait(wrk, vtp, VTIM_real() +
@@ -317,7 +318,8 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo)
 	if (vtp == NULL) {
 		retval = SC_TX_ERROR;
 	} else {
-		i = V1F_SendReq(req->wrk, bo, &v1a.bereq, 1);
+		CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
+		i = V1F_SendReq(req->wrk, bo, &v1a.bereq, &v1a.out, 1);
 		VSLb_ts_req(req, "Pipe", W_TIM_real(req->wrk));
 		if (i == 0)
 			V1P_Process(req, vtp->fd, &v1a);
diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c
index 8fa77b9..4ced78a 100644
--- a/bin/varnishd/cache/cache_req.c
+++ b/bin/varnishd/cache/cache_req.c
@@ -63,8 +63,10 @@ Req_AcctLogCharge(struct VSC_main *ds, struct req *req)
 		    (uintmax_t)(a->resp_hdrbytes + a->resp_bodybytes));
 	}
 
+	/* Charge to main byte counters (except for ESI subrequests) */
 #define ACCT(foo)			\
-	ds->s_##foo += a->foo;		\
+	if (req->esi_level == 0)	\
+		ds->s_##foo += a->foo;	\
 	a->foo = 0;
 #include "tbl/acct_fields_req.h"
 }
diff --git a/bin/varnishd/http1/cache_http1.h b/bin/varnishd/http1/cache_http1.h
index ba010db..f3b3ed2 100644
--- a/bin/varnishd/http1/cache_http1.h
+++ b/bin/varnishd/http1/cache_http1.h
@@ -30,8 +30,8 @@
 struct VSC_vbe;
 
 /* cache_http1_fetch.c [V1F] */
-int V1F_SendReq(struct worker *, struct busyobj *, uint64_t *ctr,
-    int onlycached);
+int V1F_SendReq(struct worker *, struct busyobj *, uint64_t *ctr_hdrbytes,
+    uint64_t *ctr_bodybytes, int onlycached);
 int V1F_FetchRespHdr(struct busyobj *);
 int V1F_Setup_Fetch(struct vfp_ctx *vfc, struct http_conn *htc);
 
@@ -59,5 +59,5 @@ void V1L_EndChunk(const struct worker *w);
 void V1L_Open(struct worker *, struct ws *, int *fd, struct vsl_log *,
     double t0, unsigned niov);
 unsigned V1L_Flush(const struct worker *w);
-unsigned V1L_Close(struct worker *w);
+unsigned V1L_Close(struct worker *w, uint64_t *cnt);
 size_t V1L_Write(const struct worker *w, const void *ptr, ssize_t len);
diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c
index b1b8627..c7a6226 100644
--- a/bin/varnishd/http1/cache_http1_deliver.c
+++ b/bin/varnishd/http1/cache_http1_deliver.c
@@ -50,7 +50,6 @@ v1d_bytes(struct req *req, enum vdp_action act, void **priv,
 
 	if (len > 0)
 		wl = V1L_Write(req->wrk, ptr, len);
-	req->acct.resp_bodybytes += len;
 	if (act > VDP_NULL && V1L_Flush(req->wrk))
 		return (-1);
 	if (len != wl)
@@ -87,6 +86,8 @@ void v_matchproto_(vtr_deliver_f)
 V1D_Deliver(struct req *req, struct boc *boc, int sendbody)
 {
 	int err;
+	unsigned u;
+	uint64_t hdrbytes, bytes;
 
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
 	CHECK_OBJ_ORNULL(boc, BOC_MAGIC);
@@ -140,17 +141,23 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody)
 		return;
 	}
 
-	req->acct.resp_hdrbytes += HTTP1_Write(req->wrk, req->resp, HTTP1_Resp);
+	hdrbytes = HTTP1_Write(req->wrk, req->resp, HTTP1_Resp);
 
 	if (DO_DEBUG(DBG_FLUSH_HEAD))
 		(void)V1L_Flush(req->wrk);
 
-	if (!sendbody || req->res_mode & RES_ESI)
-		if (V1L_Close(req->wrk) && req->sp->fd >= 0) {
+	if (!sendbody || req->res_mode & RES_ESI) {
+		if (V1L_Close(req->wrk, &bytes) && req->sp->fd >= 0) {
 			Req_Fail(req, SC_REM_CLOSE);
 			sendbody = 0;
 		}
 
+		/* Charge bytes sent as reported from V1L_Close. Only
+		 * header-bytes have been attempted sent. */
+		req->acct.resp_hdrbytes += bytes;
+		hdrbytes = 0;
+	}
+
 	if (!sendbody) {
 		AZ(req->wrk->v1l);
 		VDP_close(req);
@@ -178,7 +185,17 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody)
 	if (!err && (req->res_mode & RES_CHUNKED))
 		V1L_EndChunk(req->wrk);
 
-	if ((V1L_Close(req->wrk) || err) && req->sp->fd >= 0)
+	u = V1L_Close(req->wrk, &bytes);
+
+	/* Bytes accounting */
+	if (bytes < hdrbytes)
+		req->acct.resp_hdrbytes += bytes;
+	else {
+		req->acct.resp_hdrbytes += hdrbytes;
+		req->acct.resp_bodybytes += bytes - hdrbytes;
+	}
+
+	if ((u || err) && req->sp->fd >= 0)
 		Req_Fail(req, SC_REM_CLOSE);
 	AZ(req->wrk->v1l);
 	VDP_close(req);
diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c
index 07a2194..a3b7b4f 100644
--- a/bin/varnishd/http1/cache_http1_fetch.c
+++ b/bin/varnishd/http1/cache_http1_fetch.c
@@ -53,7 +53,7 @@ vbf_iter_req_body(void *priv, int flush, const void *ptr, ssize_t l)
 	CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC);
 
 	if (l > 0) {
-		bo->acct.bereq_bodybytes += V1L_Write(bo->wrk, ptr, l);
+		(void)V1L_Write(bo->wrk, ptr, l);
 		if (flush && V1L_Flush(bo->wrk))
 			return (-1);
 	}
@@ -69,12 +69,13 @@ vbf_iter_req_body(void *priv, int flush, const void *ptr, ssize_t l)
  */
 
 int
-V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr,
-    int onlycached)
+V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes,
+    uint64_t *ctr_bodybytes, int onlycached)
 {
 	struct http *hp;
 	int j;
 	ssize_t i;
+	uint64_t bytes, hdrbytes;
 	struct http_conn *htc;
 	int do_chunked = 0;
 	char abuf[VTCP_ADDRBUFSIZE];
@@ -84,6 +85,8 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr,
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 	CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC);
 	CHECK_OBJ_ORNULL(bo->req, REQ_MAGIC);
+	AN(ctr_hdrbytes);
+	AN(ctr_bodybytes);
 
 	htc = bo->htc;
 	assert(*htc->rfd > 0);
@@ -100,7 +103,7 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr,
 
 	(void)VTCP_blocking(*htc->rfd);	/* XXX: we should timeout instead */
 	V1L_Open(wrk, wrk->aws, htc->rfd, bo->vsl, bo->t_prev, 0);
-	*ctr += HTTP1_Write(wrk, hp, HTTP1_Req);
+	hdrbytes = HTTP1_Write(wrk, hp, HTTP1_Req);
 
 	/* Deal with any message-body the request might (still) have */
 	i = 0;
@@ -122,7 +125,16 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr,
 			V1L_EndChunk(wrk);
 	}
 
-	j = V1L_Close(wrk);
+	j = V1L_Close(wrk, &bytes);
+
+	/* Bytes accounting */
+	if (bytes < hdrbytes)
+		*ctr_hdrbytes += bytes;
+	else {
+		*ctr_hdrbytes += hdrbytes;
+		*ctr_bodybytes += bytes - hdrbytes;
+	}
+
 	if (j != 0 || i < 0) {
 		VSLb(bo->vsl, SLT_FetchError, "backend write error: %d (%s)",
 		    errno, strerror(errno));
diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c
index 2ab0712..7897cca 100644
--- a/bin/varnishd/http1/cache_http1_line.c
+++ b/bin/varnishd/http1/cache_http1_line.c
@@ -127,16 +127,18 @@ V1L_Open(struct worker *wrk, struct ws *ws, int *fd, struct vsl_log *vsl,
 }
 
 unsigned
-V1L_Close(struct worker *wrk)
+V1L_Close(struct worker *wrk, uint64_t *cnt)
 {
 	struct v1l *v1l;
 	unsigned u;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	AN(cnt);
 	u = V1L_Flush(wrk);
 	v1l = wrk->v1l;
 	wrk->v1l = NULL;
 	CHECK_OBJ_NOTNULL(v1l, V1L_MAGIC);
+	*cnt = v1l->cnt;
 	if (v1l->ws->r)
 		WS_Release(v1l->ws, 0);
 	WS_Reset(v1l->ws, v1l->res);
diff --git a/bin/varnishtest/tests/e00003.vtc b/bin/varnishtest/tests/e00003.vtc
index b3e65fa..71c5c63 100644
--- a/bin/varnishtest/tests/e00003.vtc
+++ b/bin/varnishtest/tests/e00003.vtc
@@ -32,7 +32,8 @@ varnish v1 -vcl+backend {
 
 logexpect l1 -v v1 -g request {
 	expect 0 1001   Begin   "^req .* rxreq"
-	expect * =	ReqAcct	"^18 0 18 187 75 262$"
+	# ReqAcct body counts include chunked overhead
+	expect * =	ReqAcct	"^18 0 18 187 104 291$"
 	expect 0 =      End
 } -start
 
@@ -54,7 +55,9 @@ logexpect l4 -v v1 -g request {
 
 logexpect l5 -v v1 -g request {
 	expect * 1005   Begin   "^req .* rxreq"
-	expect * =	ReqAcct	"^18 0 18 192 75 267$"
+	# ReqAcct body counts include chunked overhead
+	# Header bytes is 5 larger than in l1 due to two item X-Varnish hdr
+	expect * =	ReqAcct	"^18 0 18 192 104 296$"
 	expect 0 =      End
 } -start
 
diff --git a/bin/varnishtest/tests/l00003.vtc b/bin/varnishtest/tests/l00003.vtc
index af3675e..6ef90a3 100644
--- a/bin/varnishtest/tests/l00003.vtc
+++ b/bin/varnishtest/tests/l00003.vtc
@@ -35,22 +35,34 @@ varnish v1 -vcl+backend {
 
 # Reponse:
 # HTTP/1.1 200 OK\r\n			17 bytes
+# Accept-Ranges: bytes\r\n		22 bytes
 # Transfer-Encoding: chunked\r\n	28 bytes
 # Connection: keep-alive\r\n		24 bytes
 # \r\n					 2 bytes
-# Total:				71 bytes
+# Total:				93 bytes
 
 # Response body:
-# 123\r\n				 5 bytes
-# abc\r\n				 5 bytes
-# 123\r\n				 5 bytes
-# def\r\n				 5 bytes
-# ghi\r\n				 5 bytes
-# Total:				15 bytes
+# Chunk len				 5 bytes
+# 123					 3 bytes
+# Chunk end				 2 bytes
+# Chunk len				 5 bytes
+# abc					 3 bytes
+# Chunk end				 2 bytes
+# Chunk len				 5 bytes
+# 123					 3 bytes
+# Chunk end				 2 bytes
+# Chunk len				 5 bytes
+# def					 3 bytes
+# Chunk end				 2 bytes
+# Chunk len				 5 bytes
+# ghi					 3 bytes
+# Chunk end				 2 bytes
+# Chunked end				 5 bytes
+# Total:				55 bytes
 
 logexpect l1 -v v1 -g request {
 	expect 0 1001	Begin	"^req .* rxreq"
-	expect * =	ReqAcct		"^18 0 18 93 15 108$"
+	expect * =	ReqAcct		"^18 0 18 93 55 148$"
 	expect 0 =	End
 	expect * 1003	Begin		"^req .* esi"
 	expect * =	ReqAcct		"^0 0 0 0 12 12$"
@@ -75,4 +87,4 @@ logexpect l1 -wait
 varnish v1 -expect s_req_hdrbytes == 18
 varnish v1 -expect s_req_bodybytes == 0
 varnish v1 -expect s_resp_hdrbytes == 93
-varnish v1 -expect s_resp_bodybytes == 33
+varnish v1 -expect s_resp_bodybytes == 55
diff --git a/doc/changes.rst b/doc/changes.rst
index a82cacc..ba00c69 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -57,6 +57,13 @@ Logging / statistics
 
 * added ``cache_hit_grace`` statistics counter
 
+* The byte counters in ReqAcct now show the numbers reported from the
+  operating system rather than what we anticipated to send. This will give
+  more accurate numbers when e.g. the client hung up early without
+  receiving the entire response. Also these counters now show how many
+  bytes was attributed to the body, including any protocol overhead (ie
+  chunked encoding).
+
 C APIs (for vmod and utility authors)
 -------------------------------------
 
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index f7fa3d5..dc57526 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -475,9 +475,10 @@ SLTM(Timestamp, 0, "Timing information",
 
 SLTM(ReqAcct, 0, "Request handling byte counts",
 	"Contains byte counts for the request handling.\n"
-	"ESI sub-request counts are also added to their parent request.\n"
-	"The body bytes count does not include transmission "
-	"(ie: chunked encoding) overhead.\n"
+	"The body bytes count includes transmission overhead"
+	" (ie: chunked encoding).\n"
+	"ESI sub-requests show the body bytes this ESI fragment including"
+	" any subfragments contributed to the top level request.\n"
 	"The format is::\n\n"
 	"\t%d %d %d %d %d %d\n"
 	"\t|  |  |  |  |  |\n"

From fgsch at lodoss.net  Mon Feb 19 12:57:07 2018
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon, 19 Feb 2018 12:57:07 +0000 (UTC)
Subject: [master] 892928d Add changes for 5.2.1
Message-ID: <20180219125707.BB9E8BB296@lists.varnish-cache.org>


commit 892928db79bebb23850e15c46cc5d87a6e1d1e1d
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Feb 19 12:55:49 2018 +0000

    Add changes for 5.2.1
    
    Fixes #2562.

diff --git a/doc/changes.rst b/doc/changes.rst
index ba00c69..d4dff31 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -137,6 +137,19 @@ Fixed bugs
 **TODO**
 
 ================================
+Varnish Cache 5.2.1 (2017-11-14)
+================================
+
+Bugs fixed
+----------
+
+* 2429_ - Avoid buffer read overflow on vcl_backend_error and -sfile
+* 2492_ - Use the idle read timeout only on empty requests.
+
+.. _2429: https://github.com/varnishcache/varnish-cache/pull/2429
+.. _2492: https://github.com/varnishcache/varnish-cache/issues/2492
+
+================================
 Varnish Cache 5.2.0 (2017-09-15)
 ================================
 

From hermunn at varnish-software.com  Mon Feb 19 14:08:09 2018
From: hermunn at varnish-software.com (Pål Hermunn Johansen)
Date: Mon, 19 Feb 2018 14:08:09 +0000 (UTC)
Subject: [master] c12a3e5 Fix issue #1799 for keep
Message-ID: <20180219140809.AD94BBCA58@lists.varnish-cache.org>


commit c12a3e5e2bc978edafaccfe1c586e5d5dab01fcb
Author: P?l Hermunn Johansen <hermunn at varnish-software.com>
Date:   Wed Dec 13 14:06:05 2017 +0100

    Fix issue #1799 for keep
    
    This fixes the long-standing #1799 for "keep" objects, and this commit
    message suggests a way of working around #1799 in the remaining
    cases. The following is a (long) explanation on how grace and keep
    works at the moment, how this relates to #1799, and how this commit
    changes things.
    
    1. How does it work now, before this commit?
    
    Objects in cache can outlive their TTL, and the typical reason for
    this is grace. Objects in cache can also linger because of obj.keep or
    in the (rare but observed) case where the expiry thread have not yet
    evicted an object. Grace and keep are here to minimize backend load,
    but #1799 shows that we are not successful in doing this in some
    important cases.
    
    Whenever sub vcl_recv has ended with return (lookup) (which is the
    default action), we arrive at HSH_Lookup, where varnish sometimes only
    finds an expired object (that match Vary logic, is not banned,
    etc). When this happens, we will initiate a background fetch (by
    adding a "busy object") if and only if there is no busy object on the
    oh already. Then the expired object is returned with HSH_EXP or
    HSH_EXPBUSY, depending on whether a busy object was inserted.
    
    2. What makes us run into #1799?
    
    When we have gotten an expired object, we generally hope that it is in
    grace, and that sub vcl_hit will return(deliver). However, if grace
    has expired, then the default action (ie the action from builtin.vcl)
    is return (miss). It is also possible that the user vcl, for some
    reason, decides that the stale object should not be delivered, and
    does return (miss) explicitly. In these cases it is common that the
    current request is not the one to insert a busy object, and then we
    run into the issue with a message "vcl_hit{} returns miss without busy
    object. Doing pass.".
    
    Note that normally, if a resource is very popular and has a positive
    grace, it is unlikely that #1799 will happen. Then a new version will
    always be available before the grace has run out, and everybody get
    the latest fetched version with no #1799 problems.
    
    However, if a resource is very popular (like a manifest file in a live
    streaming setup) and has 0s grace, and the expiry thread lags a little
    bit behind, then vcl_hit can get an expired object even when obj.keep
    is zero. In these circumstances we can get a surge of requests to the
    backend, and this is especially bad on a very busy server.
    
    Another real world example is where grace is initially set high (48h
    or similar) and vcl_hit considers the health of the backend, and, if
    the backend is healthy, explicitly does a return(miss) ensure that the
    client gets a fresh object. This has been a recommended use of
    vcl_hit, but, because of #1799, can cause considerable load on the
    backend.
    
    Similarly, we can get #1799 if we use "keep" to facilitate IMS
    requests to the backend, and we have a stale object for which several
    requests arrive before the first completes.
    
    3. How do we fix this?
    
    The main idea is to teach varnish to consider grace during lookup.
    
    To be specific, the following changes with this commit: If an expired
    object is found, the ttl+grace has expired and there already is an
    ongoing request for the object (ie. there exists a busy object), then
    the request is put on the waiting list instead of simply returning the
    object ("without a busy object") to vcl_hit. This choice is made
    because we anticipate that vcl_hit will do (the default) return (miss)
    and that it is better to wait for the ongoing request than to initiate
    a new one with "pass" behavior.
    
    The result is that when the ongoing request finishes, we will either
    be able to go to vcl_hit, start a new request (can happen if there was
    a Vary mismatch) by inserting a new "busy object", or we lose the race
    and have to go back to the waiting list (typically unlikely).
    
    When grace is in effect we go to vcl_hit even when we did not insert a
    busy object, anticipating that vcl_hit will return (deliver).
    
    This will will fix the cases where the user does not explicitly do a
    return(miss) in vcl_hit for object where ttl+grace has not
    expired. However, since this is not an uncommon practice, we also have
    to change our recommendation on how to use grace and keep. The new
    recommendation will be:
    
    * Set grace to the "normal value" for a working varnish+backend.
    
    * Set keep to a high value if the backend is not 100% reliable and you
      want to use stale objects as a fallback.
    
    * Do not explicitly return(miss) in sub vcl_hit{}. The exception is
      when this only can happen now and then and you are really sure that
      this is the right thing to do.
    
    * In vcl_hit, check if the backend is sick, and then explicitly
      return(deliver) when appropriate (ie you want an stale object
      delivered instead of an error message).
    
    A test case is included.

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 3b6f097..efe3814 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -57,7 +57,7 @@ struct exp_priv {
 static struct exp_priv *exphdl;
 
 /*--------------------------------------------------------------------
- * Calculate an objects effective ttl time, taking req.ttl into account
+ * Calculate an object's effective ttl time, taking req.ttl into account
  * if it is available.
  */
 
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index e62e6c9..55a451c 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -481,22 +481,33 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 	if (exp_oc != NULL) {
 		assert(oh->refcnt > 1);
 		assert(exp_oc->objhead == oh);
-		exp_oc->refcnt++;
 
 		if (!busy_found) {
 			*bocp = hsh_insert_busyobj(wrk, oh);
 			retval = HSH_EXPBUSY;
 		} else {
 			AZ(req->hash_ignore_busy);
-			retval = HSH_EXP;
+			/*
+			 * here we have a busy object, but if the stale object
+			 * is not under grace we go to the waiting list
+			 * instead of returning the stale object
+			 */
+			if (EXP_Ttl(req, exp_oc) + exp_oc->grace < req->t_req)
+				retval = HSH_BUSY;
+			else
+				retval = HSH_EXP;
+		}
+		if (retval != HSH_BUSY) {
+			exp_oc->refcnt++;
+
+			if (exp_oc->hits < LONG_MAX)
+				exp_oc->hits++;
+			Lck_Unlock(&oh->mtx);
+			if (retval == HSH_EXP)
+				assert(HSH_DerefObjHead(wrk, &oh));
+			*ocp = exp_oc;
+			return (retval);
 		}
-		if (exp_oc->hits < LONG_MAX)
-			exp_oc->hits++;
-		Lck_Unlock(&oh->mtx);
-		if (retval == HSH_EXP)
-			assert(HSH_DerefObjHead(wrk, &oh));
-		*ocp = exp_oc;
-		return (retval);
 	}
 
 	if (!busy_found) {
diff --git a/bin/varnishtest/tests/b00098.vtc b/bin/varnishtest/tests/b00098.vtc
new file mode 100644
index 0000000..511a4b4
--- /dev/null
+++ b/bin/varnishtest/tests/b00098.vtc
@@ -0,0 +1,69 @@
+varnishtest "Test that we properly wait for certain 304 cases"
+
+server s1 {
+	rxreq
+	txresp -hdr "Last-Modified: Wed, 11 Sep 2013 13:36:55 GMT" -body "Geoff Still Rules"
+
+	# The IMS request we will spend some time to process for the sake of
+	# this test.
+	rxreq
+	expect req.http.if-modified-since == "Wed, 11 Sep 2013 13:36:55 GMT"
+	delay 1
+	txresp -status 304
+
+	# Last request, to a different URL to catch it if varnish asks for "/" too many times
+	rxreq
+	expect req.url == "/2"
+	txresp -body "x"
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.ttl = 1s;
+		set beresp.grace = 1s;
+		set beresp.keep = 1m;
+		set beresp.http.was-304 = beresp.was_304;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.body == "Geoff Still Rules"
+} -run
+
+# let the object's ttl and grace expire
+delay 2.1
+
+# first client to ask for kept object - this should start the second request
+client c2 {
+	txreq
+	rxresp
+	# we did not disable grace in the request, so we should get the graced object here
+	expect resp.status == 200
+	expect resp.body == "Geoff Still Rules"
+} -start
+
+delay .1
+
+# second client to ask for the kept object. Here we want to wait until the backend fetch completes, not do a pass.
+client c3 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.body == "Geoff Still Rules"
+} -start
+
+client c2 -wait
+client c3 -wait
+
+# Finally the request to "/2". The expect in the server block makes sure that
+# there were no extra requests to "/" from varnish.
+
+client c4 {
+	txreq -url "/2"
+	rxresp
+	expect resp.status == 200
+	expect resp.body == "x"
+} -run

From nils.goroll at uplex.de  Mon Feb 19 17:29:06 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 19 Feb 2018 17:29:06 +0000 (UTC)
Subject: [master] 01379ca Regression-Test that vcc fails for
 $thing.http.$thing.http.$Header
Message-ID: <20180219172906.27E3F6536D@lists.varnish-cache.org>


commit 01379caf4dab6a4287e3915ddd9d92b6f2da1184
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Feb 19 18:24:41 2018 +0100

    Regression-Test that vcc fails for $thing.http.$thing.http.$Header
    
    Ref: #2573

diff --git a/bin/varnishtest/tests/v00019.vtc b/bin/varnishtest/tests/v00019.vtc
index c4bd725..f579b02 100644
--- a/bin/varnishtest/tests/v00019.vtc
+++ b/bin/varnishtest/tests/v00019.vtc
@@ -63,3 +63,10 @@ varnish v1 -errvcl {Comparison of different types: STRING '==' INT} {
 		}
 	}
 }
+
+varnish v1 -errvcl {Unknown variable: 'req.http.req.http.foo'} {
+	backend b { .host = "127.0.0.1"; }
+	sub vcl_recv {
+		set req.http.req.http.foo = "bar";
+	}
+}

From phk at FreeBSD.org  Tue Feb 20 10:49:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 20 Feb 2018 10:49:09 +0000 (UTC)
Subject: [master] fa80a6a Start marking variables with VCL versions.
Message-ID: <20180220104909.38507AE91F@lists.varnish-cache.org>


commit fa80a6a641b997a84ec71813128d86b47620d79c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 20 09:31:38 2018 +0000

    Start marking variables with VCL versions.
    
    (It doesn't get to the C code yet.)

diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst
index f94841a..9631ed3 100644
--- a/doc/sphinx/reference/vcl_var.rst
+++ b/doc/sphinx/reference/vcl_var.rst
@@ -215,7 +215,7 @@ req.url
 	The requested URL, for instance "/robots.txt".
 	
 
-req.proto
+req.proto	``VCL <= 4.0``
 
 	Type: STRING
 
@@ -223,7 +223,15 @@ req.proto
 
 	Writable from: client
 
+	The HTTP protocol version used by the client, usually "HTTP/1.1"
+	or "HTTP/2.0".
 	
+req.proto	``VCL >= 4.1``
+
+	Type: STRING
+
+	Readable from: client
+
 	The HTTP protocol version used by the client, usually "HTTP/1.1"
 	or "HTTP/2.0".
 	
@@ -302,7 +310,7 @@ req.xid
 
 	Unique ID of this request.
 
-req.esi
+req.esi	``VCL <= 4.0``
 
 	Type: BOOL
 
@@ -312,9 +320,8 @@ req.esi
 
 	Set to `false` to disable ESI processing
 	regardless of any value in beresp.do_esi. Defaults
-	to `true`. This variable is subject to change in
-	future versions, you should avoid using it.
-	
+	to `true`. This variable is replaced by `resp.do_esi`
+	in VCL 4.1.
 
 req.can_gzip
 
@@ -515,7 +522,7 @@ bereq.url
 	The requested URL, copied from `req.url`
 	
 
-bereq.proto
+bereq.proto	``VCL <= 4.0``
 
 	Type: STRING
 
@@ -526,6 +533,15 @@ bereq.proto
 	The HTTP protocol version, "HTTP/1.1" unless a pass or pipe
 	request has "HTTP/1.0" in `req.proto`
 	
+bereq.proto	``VCL >= 4.1``
+
+	Type: STRING
+
+	Readable from: vcl_pipe, backend
+	
+	The HTTP protocol version, "HTTP/1.1" unless a pass or pipe
+	request has "HTTP/1.0" in `req.proto`
+	
 
 bereq.http.*
 
@@ -620,7 +636,7 @@ beresp.body
 	
 	For producing a synthetic body.
 
-beresp.proto
+beresp.proto	``VCL <= 4.0``
 
 	Type: STRING
 
@@ -630,6 +646,14 @@ beresp.proto
 
 	The HTTP protocol version the backend replied with.
 	
+beresp.proto	``VCL >= 4.1``
+
+	Type: STRING
+
+	Readable from: vcl_backend_response, vcl_backend_error
+
+	The HTTP protocol version the backend replied with.
+	
 
 beresp.status
 
@@ -855,7 +879,6 @@ obj.proto
 
 	Readable from: vcl_hit
 
-	
 	The HTTP protocol version stored in the object.
 	
 
@@ -959,7 +982,10 @@ resp
 ~~~~
 
 This is the response we send to the client, it is built from either
-`beresp` (pass/miss) or `obj` (hits).
+`beresp` (pass/miss), `obj` (hits) or created from whole cloth (synth).
+
+With the exception of `resp.body` all `resp.*` variables available
+in both `vcl_deliver{}` and `vcl_synth{}` as a matter of symmetry.
 
 resp
 
@@ -978,7 +1004,17 @@ resp.body
 
 	To produce a synthetic response body, for instance for errors.
 
-resp.proto
+resp.proto	``VCL <= 4.0``
+
+	Type: STRING
+
+	Readable from: vcl_deliver, vcl_synth
+
+	Writable from: vcl_deliver, vcl_synth
+
+	The HTTP protocol version to use for the response.
+
+resp.proto	``VCL >= 4.1``
 
 	Type: STRING
 
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index ad353ab..cb0784c 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -169,21 +169,41 @@ returns = (
 sp_variables = collections.OrderedDict()
 
 class vardef(object):
-	def __init__(self, nam, typ, rd, wr, doc, uns):
+	def __init__(self, nam, typ, rd, wr, wu, vlo, vhi):
+		if nam in sp_variables:
+			return
 		sp_variables[nam] = self
 		self.nam = nam
 		self.typ = typ
 		self.rd = rd
 		self.wr = wr
-		self.doc = doc
-		self.uns = uns
+		self.uns = wu
+		self.vlo = vlo
+		self.vhi = vhi
+
+def parse_vcl(x):
+	vlo,vhi = (0,99)
+	x = x.split()
+	if x[0] == "VCL" and x[1] == "<=":
+		vhi = int(float(x[2]) * 10)
+	elif x[0] == "VCL" and x[1] == ">=":
+		vlo = int(float(x[2]) * 10)
+	else:
+		print("Unknown variable version spec")
+		print("XXX", x, vlo, vhi)
+		exit(2)
+	return vlo,vhi
 
 def parse_var(ln):
-	vn = ln.pop(0).split()
-	assert len(vn) == 1
-	vn = vn[0]
+	l1 = ln.pop(0).split("``")
+	assert len(l1) in (1,3)
+	vn = l1[0].strip()
 	if vn[-1] == '*':
 		vn = vn[:-1]
+	if len(l1) == 3:
+		vlo,vhi = parse_vcl(l1[1])
+	else:
+		vlo,vhi = 0,99
 	vr = []
 	vw = []
 	vu = []
@@ -209,9 +229,8 @@ def parse_var(ln):
 				vu.append(i.strip(",."))
 			continue
 		break
-	doc = "\n" + l + "\n" + "\n".join(ln)
 	if vn[:8] != "storage.":
-		vardef(vn, vt, vr, vw, doc, uns=vu)
+		vardef(vn, vt, vr, vw, vu, vlo, vhi)
 
 def parse_var_doc(fn):
 	s = 0

From phk at FreeBSD.org  Tue Feb 20 10:49:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 20 Feb 2018 10:49:09 +0000 (UTC)
Subject: [master] 250b880 Simplify
Message-ID: <20180220104909.4D01EAE922@lists.varnish-cache.org>


commit 250b880196584727cc873240ed44429dbeca534f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 20 09:46:38 2018 +0000

    Simplify

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index cb0784c..92092bf 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -248,9 +248,6 @@ def parse_var_doc(fn):
 
 parse_var_doc(join(buildroot, "doc/sphinx/reference/vcl_var.rst"))
 
-# Backwards compatibility:
-aliases = []
-
 stv_variables = (
 	('free_space',	'BYTES',	"0.", 'storage.<name>.free_space', """
 	Free space available in the named stevedore. Only available for
@@ -692,33 +689,23 @@ def one_var(nm, spec):
 	fo.write("\tsym->type = %s;\n" % spec.typ)
 	fo.write("\tsym->eval = vcc_Eval_Var;\n")
 
-	if len(spec.rd) == 0:
-		fo.write('\t/* No reads allowed */\n')
-	elif spec.typ == "HEADER":
+	if spec.typ == "HEADER":
 		fo.write('\tsym->rname = "HDR_')
 		fo.write(nm.split(".")[0].upper())
 		fo.write('";\n')
-		fo.write("\tsym->r_methods =\n")
-		restrict(fo, spec.rd)
-		fo.write(";\n")
-	else:
+	elif len(spec.rd):
 		fo.write('\tsym->rname = "VRT_r_%s(ctx)";\n' % cnam)
 		if nm == spec.nam:
 			fh.write("VCL_" + spec.typ + " VRT_r_%s(VRT_CTX);\n" % cnam)
-		fo.write("\tsym->r_methods =\n")
-		restrict(fo, spec.rd)
-		fo.write(";\n")
+	fo.write("\tsym->r_methods =\n")
+	restrict(fo, spec.rd)
+	fo.write(";\n")
 
-	if len(spec.wr) == 0:
-		fo.write('\t/* No writes allowed */\n')
-	elif spec.typ == "HEADER":
+	if spec.typ == "HEADER":
 		fo.write('\tsym->lname = "HDR_')
 		fo.write(nm.split(".")[0].upper())
 		fo.write('";\n')
-		fo.write("\tsym->w_methods =\n")
-		restrict(fo, spec.wr)
-		fo.write(";\n")
-	else:
+	elif len(spec.wr):
 		fo.write('\tsym->lname = "VRT_l_%s(ctx, ";\n' % cnam)
 		if nm == spec.nam:
 			fh.write("void VRT_l_%s(VRT_CTX, " % cnam)
@@ -726,25 +713,19 @@ def one_var(nm, spec):
 				fh.write("VCL_" + spec.typ + ");\n")
 			else:
 				fh.write(ctyp + ", ...);\n")
-		fo.write("\tsym->w_methods =\n")
-		restrict(fo, spec.wr)
-		fo.write(";\n")
+	fo.write("\tsym->w_methods =\n")
+	restrict(fo, spec.wr)
+	fo.write(";\n")
 
-	if spec.uns is None or len(spec.uns) == 0:
-		fo.write('\t/* No unsets allowed */\n')
-	else:
+	if len(spec.uns):
 		fh.write("void VRT_u_%s(VRT_CTX);\n" % cnam)
 		fo.write('\tsym->uname = "VRT_u_%s(ctx)";\n' % cnam)
-		fo.write('\tsym->u_methods =\n')
-		restrict(fo, spec.uns)
-		fo.write(";\n")
+	fo.write('\tsym->u_methods =\n')
+	restrict(fo, spec.uns)
+	fo.write(";\n")
 
-aliases.sort()
 for i in sp_variables:
 	one_var(i, sp_variables[i])
-	for j in aliases:
-		if j[1] == i[0]:
-			one_var(j[0], sp_variables[i])
 
 # fo.write("\t{ NULL }\n};\n\n")
 fo.write("}\n")

From phk at FreeBSD.org  Tue Feb 20 10:49:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 20 Feb 2018 10:49:09 +0000 (UTC)
Subject: [master] c24a143 Simplify more
Message-ID: <20180220104909.6684DAE926@lists.varnish-cache.org>


commit c24a1439f94330e4fbf8b450f03b7e3fcce9bac8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 20 09:54:04 2018 +0000

    Simplify more

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 92092bf..9b2809e 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -181,6 +181,57 @@ class vardef(object):
 		self.vlo = vlo
 		self.vhi = vhi
 
+	def emit(self):
+		fh.write("\n")
+		fo.write("\n")
+		cnam = self.nam.replace(".", "_")
+		ctyp = vcltypes[self.typ]
+
+		# fo.write("\t{ \"%s\", %s,\n" % (nm, self.typ))
+		fo.write("\tsym = VCC_MkSym(tl, \"%s\", " % self.nam)
+		if (self.typ == "HEADER"):
+			fo.write(" SYM_NONE);\n")
+			fo.write("\tAN(sym);\n");
+			fo.write("\tsym->wildcard = vcc_Var_Wildcard;\n")
+		else:
+			fo.write(" SYM_VAR);\n")
+		fo.write("\tAN(sym);\n")
+		fo.write("\tsym->type = %s;\n" % self.typ)
+		fo.write("\tsym->eval = vcc_Eval_Var;\n")
+
+		if self.typ == "HEADER":
+			fo.write('\tsym->rname = "HDR_')
+			fo.write(self.nam.split(".")[0].upper())
+			fo.write('";\n')
+		elif len(self.rd):
+			fo.write('\tsym->rname = "VRT_r_%s(ctx)";\n' % cnam)
+			fh.write("VCL_" + self.typ + " VRT_r_%s(VRT_CTX);\n" % cnam)
+		fo.write("\tsym->r_methods =\n")
+		restrict(fo, self.rd)
+		fo.write(";\n")
+
+		if self.typ == "HEADER":
+			fo.write('\tsym->lname = "HDR_')
+			fo.write(self.nam.split(".")[0].upper())
+			fo.write('";\n')
+		elif len(self.wr):
+			fo.write('\tsym->lname = "VRT_l_%s(ctx, ";\n' % cnam)
+			fh.write("void VRT_l_%s(VRT_CTX, " % cnam)
+			if self.typ != "STRING" and self.typ != "BODY":
+				fh.write("VCL_" + self.typ + ");\n")
+			else:
+				fh.write(ctyp + ", ...);\n")
+		fo.write("\tsym->w_methods =\n")
+		restrict(fo, self.wr)
+		fo.write(";\n")
+
+		if len(self.uns):
+			fh.write("void VRT_u_%s(VRT_CTX);\n" % cnam)
+			fo.write('\tsym->uname = "VRT_u_%s(ctx)";\n' % cnam)
+		fo.write('\tsym->u_methods =\n')
+		restrict(fo, self.uns)
+		fo.write(";\n")
+
 def parse_vcl(x):
 	vlo,vhi = (0,99)
 	x = x.split()
@@ -671,61 +722,9 @@ vcc_Var_Init(struct vcc *tl)
 """)
 
 
-def one_var(nm, spec):
-	fh.write("\n")
-	fo.write("\n")
-	cnam = spec.nam.replace(".", "_")
-	ctyp = vcltypes[spec.typ]
-
-	# fo.write("\t{ \"%s\", %s,\n" % (nm, spec.typ))
-	fo.write("\tsym = VCC_MkSym(tl, \"%s\", " % nm)
-	if (spec.typ == "HEADER"):
-		fo.write(" SYM_NONE);\n")
-		fo.write("\tAN(sym);\n");
-		fo.write("\tsym->wildcard = vcc_Var_Wildcard;\n")
-	else:
-		fo.write(" SYM_VAR);\n")
-	fo.write("\tAN(sym);\n")
-	fo.write("\tsym->type = %s;\n" % spec.typ)
-	fo.write("\tsym->eval = vcc_Eval_Var;\n")
-
-	if spec.typ == "HEADER":
-		fo.write('\tsym->rname = "HDR_')
-		fo.write(nm.split(".")[0].upper())
-		fo.write('";\n')
-	elif len(spec.rd):
-		fo.write('\tsym->rname = "VRT_r_%s(ctx)";\n' % cnam)
-		if nm == spec.nam:
-			fh.write("VCL_" + spec.typ + " VRT_r_%s(VRT_CTX);\n" % cnam)
-	fo.write("\tsym->r_methods =\n")
-	restrict(fo, spec.rd)
-	fo.write(";\n")
-
-	if spec.typ == "HEADER":
-		fo.write('\tsym->lname = "HDR_')
-		fo.write(nm.split(".")[0].upper())
-		fo.write('";\n')
-	elif len(spec.wr):
-		fo.write('\tsym->lname = "VRT_l_%s(ctx, ";\n' % cnam)
-		if nm == spec.nam:
-			fh.write("void VRT_l_%s(VRT_CTX, " % cnam)
-			if spec.typ != "STRING" and spec.typ != "BODY":
-				fh.write("VCL_" + spec.typ + ");\n")
-			else:
-				fh.write(ctyp + ", ...);\n")
-	fo.write("\tsym->w_methods =\n")
-	restrict(fo, spec.wr)
-	fo.write(";\n")
-
-	if len(spec.uns):
-		fh.write("void VRT_u_%s(VRT_CTX);\n" % cnam)
-		fo.write('\tsym->uname = "VRT_u_%s(ctx)";\n' % cnam)
-	fo.write('\tsym->u_methods =\n')
-	restrict(fo, spec.uns)
-	fo.write(";\n")
-
-for i in sp_variables:
-	one_var(i, sp_variables[i])
+
+for i in sp_variables.values():
+	i.emit()
 
 # fo.write("\t{ NULL }\n};\n\n")
 fo.write("}\n")

From phk at FreeBSD.org  Tue Feb 20 10:49:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 20 Feb 2018 10:49:09 +0000 (UTC)
Subject: [master] 0c5569f Use -syntax 4.0 where appropriate
Message-ID: <20180220104909.8622AAE92B@lists.varnish-cache.org>


commit 0c5569f7ebef530b71a7146a25fc01563d7ff122
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 20 10:47:53 2018 +0000

    Use -syntax 4.0 where appropriate

diff --git a/bin/varnishtest/tests/c00009.vtc b/bin/varnishtest/tests/c00009.vtc
index 6ff3d97..b71ce59 100644
--- a/bin/varnishtest/tests/c00009.vtc
+++ b/bin/varnishtest/tests/c00009.vtc
@@ -12,7 +12,7 @@ server s2 {
 	txresp -body "foobar"
 } -start
 
-varnish v1 -arg "-smysteve=malloc,1m" -vcl+backend {
+varnish v1 -syntax 4.0 -arg "-smysteve=malloc,1m" -vcl+backend {
 	sub vcl_recv {
 		if (req.restarts == 0) {
 			set req.url = "/foo";
diff --git a/bin/varnishtest/tests/c00045.vtc b/bin/varnishtest/tests/c00045.vtc
index 896a520..6475033 100644
--- a/bin/varnishtest/tests/c00045.vtc
+++ b/bin/varnishtest/tests/c00045.vtc
@@ -14,6 +14,7 @@ varnish v1 \
 	-arg "-sdefault,1m" \
 	-arg "-sdefault,1m" \
 	-arg "-sTransient=default" \
+	-syntax 4.0 \
 	-vcl+backend {
 	sub vcl_backend_response {
 		set beresp.do_stream = false;
diff --git a/bin/varnishtest/tests/c00046.vtc b/bin/varnishtest/tests/c00046.vtc
index 55832f3..a3ce00f 100644
--- a/bin/varnishtest/tests/c00046.vtc
+++ b/bin/varnishtest/tests/c00046.vtc
@@ -10,6 +10,7 @@ varnish v1 \
 	-arg "-sdefault,1m" \
 	-arg "-sdefault,1m" \
 	-arg "-sTransient=default" \
+	-syntax 4.0 \
 	-vcl+backend {
 	sub vcl_backend_response {
 		set beresp.storage = storage.s0;
diff --git a/bin/varnishtest/tests/c00078.vtc b/bin/varnishtest/tests/c00078.vtc
index d5464ef..f8d234a 100644
--- a/bin/varnishtest/tests/c00078.vtc
+++ b/bin/varnishtest/tests/c00078.vtc
@@ -9,6 +9,7 @@ varnish v1 \
     -arg "-ss1=default,1m" \
     -arg "-ss2=default,1m" \
     -arg "-ss0=default,1m" \
+    -syntax 4.0 \
     -vcl+backend {
 	import vtc;
 	sub vcl_backend_response {
diff --git a/bin/varnishtest/tests/c00084.vtc b/bin/varnishtest/tests/c00084.vtc
index 3a4c77c..37b3be2 100644
--- a/bin/varnishtest/tests/c00084.vtc
+++ b/bin/varnishtest/tests/c00084.vtc
@@ -1,6 +1,9 @@
 varnishtest "legal symbol names"
 
-varnish v1 -arg "-s my-store=default" -vcl {
+varnish v1 \
+	-syntax 4.0 \
+	-arg "-s my-store=default" \
+	-vcl {
 	import directors;
 
 	acl my-acl { "127.0.0.1"; }
diff --git a/bin/varnishtest/tests/e00015.vtc b/bin/varnishtest/tests/e00015.vtc
index daaea75..b394668 100644
--- a/bin/varnishtest/tests/e00015.vtc
+++ b/bin/varnishtest/tests/e00015.vtc
@@ -22,7 +22,7 @@ server s1 {
 	}
 } -start
 
-varnish v1 -vcl+backend {
+varnish v1 -syntax 4.0 -vcl+backend {
 	sub vcl_recv {
 		if (req.url == "/") {
 			set req.esi = false;
diff --git a/bin/varnishtest/tests/e00016.vtc b/bin/varnishtest/tests/e00016.vtc
index 7cf78cc..feb86d6 100644
--- a/bin/varnishtest/tests/e00016.vtc
+++ b/bin/varnishtest/tests/e00016.vtc
@@ -26,7 +26,10 @@ server s1 {
 	}
 } -start
 
-varnish v1 -arg "-p feature=+esi_disable_xml_check" -vcl+backend {
+varnish v1 \
+	-syntax 4.0 \
+	-arg "-p feature=+esi_disable_xml_check" \
+	-vcl+backend {
 	sub vcl_backend_response {
 		set beresp.do_esi = true;
 	}
diff --git a/bin/varnishtest/tests/e00021.vtc b/bin/varnishtest/tests/e00021.vtc
index dc8f1cf..8199937 100644
--- a/bin/varnishtest/tests/e00021.vtc
+++ b/bin/varnishtest/tests/e00021.vtc
@@ -13,7 +13,7 @@ server s1 {
 	}
 } -start
 
-varnish v1 -vcl+backend {
+varnish v1 -syntax 4.0 -vcl+backend {
 	sub vcl_recv {
 		set req.esi = true;
 	}
diff --git a/bin/varnishtest/tests/e00022.vtc b/bin/varnishtest/tests/e00022.vtc
index 65e14d1..446d2e2 100644
--- a/bin/varnishtest/tests/e00022.vtc
+++ b/bin/varnishtest/tests/e00022.vtc
@@ -14,7 +14,7 @@ server s1 {
 	}
 } -start
 
-varnish v1 -arg "-p thread_pool_stack=262144" -vcl+backend {
+varnish v1 -syntax 4.0 -arg "-p thread_pool_stack=262144" -vcl+backend {
 	sub vcl_recv {
 		set req.esi = true;
 	}
diff --git a/bin/varnishtest/tests/p00008.vtc b/bin/varnishtest/tests/p00008.vtc
index a17c17c..c3973b4 100644
--- a/bin/varnishtest/tests/p00008.vtc
+++ b/bin/varnishtest/tests/p00008.vtc
@@ -18,6 +18,7 @@ varnish v1 \
 	-arg "-pban_lurker_sleep=0" \
 	-arg "-sper1=deprecated_persistent,${tmpdir}/_.per1,5m" \
 	-arg "-sper2=deprecated_persistent,${tmpdir}/_.per2,5m" \
+	-syntax 4.0 \
 	-vcl+backend {
 		sub vcl_backend_response {
 			set beresp.storage = storage.per1;
diff --git a/bin/varnishtest/tests/r00940.vtc b/bin/varnishtest/tests/r00940.vtc
index 4e4711d..d70bb91 100644
--- a/bin/varnishtest/tests/r00940.vtc
+++ b/bin/varnishtest/tests/r00940.vtc
@@ -19,7 +19,7 @@ server s1 {
 	txresp -hdr {ETag: "foo4"}  -gzipbody {<H11><esi:remove> foo </esi:remove><H22>}
 } -start
 
-varnish v1 -vcl+backend {
+varnish v1 -syntax 4.0 -vcl+backend {
 	sub vcl_backend_response {
 		if (bereq.url == "/2") {
 			set beresp.do_gzip = true;
diff --git a/bin/varnishtest/tests/r00962.vtc b/bin/varnishtest/tests/r00962.vtc
index bff9498..7f96219 100644
--- a/bin/varnishtest/tests/r00962.vtc
+++ b/bin/varnishtest/tests/r00962.vtc
@@ -18,6 +18,7 @@ varnish v1 \
 	-arg "-pfeature=+wait_silo" \
 	-arg "-sdeprecated_persistent,${tmpdir}/_.per1,5m" \
 	-arg "-sdeprecated_persistent,${tmpdir}/_.per2,5m" \
+	-syntax 4.0 \
 	-vcl+backend {
 	sub vcl_backend_response {
 		set beresp.storage = storage.s0;
diff --git a/bin/varnishtest/tests/r01175.vtc b/bin/varnishtest/tests/r01175.vtc
index 5b7a748..6dd776e 100644
--- a/bin/varnishtest/tests/r01175.vtc
+++ b/bin/varnishtest/tests/r01175.vtc
@@ -5,7 +5,7 @@ server s1 {
 	txresp -nolen -hdr "Content-Length: 1048576"
 } -start
 
-varnish v1 -arg "-s test=default,1M" -vcl+backend {
+varnish v1 -syntax 4.0 -arg "-s test=default,1M" -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.storage = storage.test;
 		set beresp.do_stream = false;
diff --git a/bin/varnishtest/tests/r01284.vtc b/bin/varnishtest/tests/r01284.vtc
index 5612acb..d899fac 100644
--- a/bin/varnishtest/tests/r01284.vtc
+++ b/bin/varnishtest/tests/r01284.vtc
@@ -12,6 +12,7 @@ server s1 {
 varnish v1 \
 	-arg "-p nuke_limit=0" \
 	-arg "-sTransient=default,1m" \
+	-syntax 4.0 \
 	-vcl+backend {
 	sub vcl_backend_response {
 		set beresp.do_stream = false;
diff --git a/bin/varnishtest/tests/r01810.vtc b/bin/varnishtest/tests/r01810.vtc
index 3448314..cb641e9 100644
--- a/bin/varnishtest/tests/r01810.vtc
+++ b/bin/varnishtest/tests/r01810.vtc
@@ -8,7 +8,7 @@ server s1 {
 	delay .4
 } -start
 
-varnish v1 -vcl+backend {
+varnish v1 -syntax 4.0 -vcl+backend {
 	sub vcl_backend_fetch {
 		set bereq.proto = "HTTP/1.0";
 	}
diff --git a/bin/varnishtest/tests/r01914.vtc b/bin/varnishtest/tests/r01914.vtc
index d7f1b20..2414dad 100644
--- a/bin/varnishtest/tests/r01914.vtc
+++ b/bin/varnishtest/tests/r01914.vtc
@@ -13,6 +13,7 @@ varnish v1 \
     -arg "-s default,1MB" \
     -arg "-s default,1MB" \
     -arg "-s Transient=default" \
+    -syntax 4.0 \
     -vcl+backend {
 	import std;
 	sub vcl_recv {
diff --git a/bin/varnishtest/tests/v00001.vtc b/bin/varnishtest/tests/v00001.vtc
index 822c847..0e9729d 100644
--- a/bin/varnishtest/tests/v00001.vtc
+++ b/bin/varnishtest/tests/v00001.vtc
@@ -8,7 +8,7 @@ server s1 {
 
 server s1 -start
 
-varnish v1 -vcl+backend {
+varnish v1 -syntax 4.0 -vcl+backend {
 	sub vcl_recv {
 		set req.http.foobar =
 		    req.url +
diff --git a/bin/varnishtest/tests/v00025.vtc b/bin/varnishtest/tests/v00025.vtc
index aadaa83..544240b 100644
--- a/bin/varnishtest/tests/v00025.vtc
+++ b/bin/varnishtest/tests/v00025.vtc
@@ -5,7 +5,7 @@ server s1 {
 	txresp
 } -start
 
-varnish v1 -arg "-i J.F.Nobody" -vcl+backend {
+varnish v1 -syntax 4.0 -arg "-i J.F.Nobody" -vcl+backend {
 
 	import std;
 	import directors;
diff --git a/bin/varnishtest/tests/v00032.vtc b/bin/varnishtest/tests/v00032.vtc
index 47f1af8..b86c583 100644
--- a/bin/varnishtest/tests/v00032.vtc
+++ b/bin/varnishtest/tests/v00032.vtc
@@ -5,7 +5,7 @@ server s1 {
 	txresp
 } -start
 
-varnish v1 -vcl+backend {
+varnish v1 -syntax 4.0 -vcl+backend {
 	sub vcl_backend_response {
 		set beresp.http.has_s0 = storage.s0;
 		set beresp.http.has_Transient = storage.Transient;
diff --git a/bin/varnishtest/tests/v00033.vtc b/bin/varnishtest/tests/v00033.vtc
index b370dd7..bbe21d1 100644
--- a/bin/varnishtest/tests/v00033.vtc
+++ b/bin/varnishtest/tests/v00033.vtc
@@ -8,7 +8,7 @@ server s1 {
 	txresp -bodylen 5
 } -start
 
-varnish v1 -vcl+backend {
+varnish v1 -syntax 4.0 -vcl+backend {
 
 	sub vcl_backend_response {
 		set beresp.http.foo =
@@ -37,13 +37,13 @@ client c1 {
 	expect resp.http.bar == true
 } -run
 
-varnish v1 -errvcl {Expected BYTES unit (B, KB, MB...) got '"X"'} {
+varnish v1 -syntax 4.0 -errvcl {Expected BYTES unit (B, KB, MB...) got '"X"'} {
 	sub vcl_recv {
 		if (storage.Transient.free_space > 4 "X") {
 		}
 	}
 }
-varnish v1 -errvcl {Unknown BYTES unit 'X'.  Legal are 'B', 'KB', 'MB', 'GB' and 'TB'} {
+varnish v1 -syntax 4.0 -errvcl {Unknown BYTES unit 'X'.  Legal are 'B', 'KB', 'MB', 'GB' and 'TB'} {
 	sub vcl_recv {
 		if (storage.Transient.free_space > 4 X) {
 		}
diff --git a/bin/varnishtest/tests/v00052.vtc b/bin/varnishtest/tests/v00052.vtc
index a014b60..76c0590 100644
--- a/bin/varnishtest/tests/v00052.vtc
+++ b/bin/varnishtest/tests/v00052.vtc
@@ -8,7 +8,7 @@ server s1 {
 	txresp
 } -start
 
-varnish v1 -vcl+backend {
+varnish v1 -syntax 4.0 -vcl+backend {
 	sub vcl_hit {
 		set req.http.Hit-Storage = obj.storage;
 	}

From phk at FreeBSD.org  Tue Feb 20 10:49:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 20 Feb 2018 10:49:09 +0000 (UTC)
Subject: [master] d6ed35e Support versioned symbols.
Message-ID: <20180220104909.A52ADAE92F@lists.varnish-cache.org>


commit d6ed35e9cfed11e4a5e04f1bf43e8563b3f8feba
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 20 10:48:30 2018 +0000

    Support versioned symbols.

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 9b2809e..1f2987f 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -166,13 +166,15 @@ returns = (
 # 'backend' means all methods tagged "B"
 # 'both' means all methods tagged "B" or "C"
 
-sp_variables = collections.OrderedDict()
+varprotos = {}
+
+def varproto(s):
+	if not s in varprotos:
+		fh.write(s + ";\n")
+		varprotos[s] = True
 
 class vardef(object):
 	def __init__(self, nam, typ, rd, wr, wu, vlo, vhi):
-		if nam in sp_variables:
-			return
-		sp_variables[nam] = self
 		self.nam = nam
 		self.typ = typ
 		self.rd = rd
@@ -181,6 +183,8 @@ class vardef(object):
 		self.vlo = vlo
 		self.vhi = vhi
 
+		self.emit()
+
 	def emit(self):
 		fh.write("\n")
 		fo.write("\n")
@@ -190,11 +194,11 @@ class vardef(object):
 		# fo.write("\t{ \"%s\", %s,\n" % (nm, self.typ))
 		fo.write("\tsym = VCC_MkSym(tl, \"%s\", " % self.nam)
 		if (self.typ == "HEADER"):
-			fo.write(" SYM_NONE);\n")
+			fo.write(" SYM_NONE, %d, %d);\n" % (self.vlo, self.vhi))
 			fo.write("\tAN(sym);\n");
 			fo.write("\tsym->wildcard = vcc_Var_Wildcard;\n")
 		else:
-			fo.write(" SYM_VAR);\n")
+			fo.write(" SYM_VAR, %d, %d);\n" % (self.vlo, self.vhi))
 		fo.write("\tAN(sym);\n")
 		fo.write("\tsym->type = %s;\n" % self.typ)
 		fo.write("\tsym->eval = vcc_Eval_Var;\n")
@@ -205,7 +209,7 @@ class vardef(object):
 			fo.write('";\n')
 		elif len(self.rd):
 			fo.write('\tsym->rname = "VRT_r_%s(ctx)";\n' % cnam)
-			fh.write("VCL_" + self.typ + " VRT_r_%s(VRT_CTX);\n" % cnam)
+			varproto("VCL_" + self.typ + " VRT_r_%s(VRT_CTX)" % cnam)
 		fo.write("\tsym->r_methods =\n")
 		restrict(fo, self.rd)
 		fo.write(";\n")
@@ -216,17 +220,18 @@ class vardef(object):
 			fo.write('";\n')
 		elif len(self.wr):
 			fo.write('\tsym->lname = "VRT_l_%s(ctx, ";\n' % cnam)
-			fh.write("void VRT_l_%s(VRT_CTX, " % cnam)
+			s = "void VRT_l_%s(VRT_CTX, " % cnam
 			if self.typ != "STRING" and self.typ != "BODY":
-				fh.write("VCL_" + self.typ + ");\n")
+				s += "VCL_" + self.typ + ")"
 			else:
-				fh.write(ctyp + ", ...);\n")
+				s += ctyp + ", ...)"
+			varproto(s);
 		fo.write("\tsym->w_methods =\n")
 		restrict(fo, self.wr)
 		fo.write(";\n")
 
 		if len(self.uns):
-			fh.write("void VRT_u_%s(VRT_CTX);\n" % cnam)
+			varproto("void VRT_u_%s(VRT_CTX)" % cnam)
 			fo.write('\tsym->uname = "VRT_u_%s(ctx)";\n' % cnam)
 		fo.write('\tsym->u_methods =\n')
 		restrict(fo, self.uns)
@@ -297,8 +302,6 @@ def parse_var_doc(fn):
 			m += 1
 		parse_var(l[n-2:m-1])
 
-parse_var_doc(join(buildroot, "doc/sphinx/reference/vcl_var.rst"))
-
 stv_variables = (
 	('free_space',	'BYTES',	"0.", 'storage.<name>.free_space', """
 	Free space available in the named stevedore. Only available for
@@ -722,11 +725,7 @@ vcc_Var_Init(struct vcc *tl)
 """)
 
 
-
-for i in sp_variables.values():
-	i.emit()
-
-# fo.write("\t{ NULL }\n};\n\n")
+parse_var_doc(join(buildroot, "doc/sphinx/reference/vcl_var.rst"))
 fo.write("}\n")
 
 for i in stv_variables:
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 5447e8c..8e61f27 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -364,7 +364,7 @@ vcc_act_synthetic(struct vcc *tl, struct token *t, struct symbol *sym)
 #define ACT(name, func, mask)						\
 	do {								\
 		const char pp[] = #name;				\
-		sym = VCC_MkSym(tl, pp, SYM_ACTION);			\
+		sym = VCC_MkSym(tl, pp, SYM_ACTION, VCL_LOW, VCL_HIGH);	\
 		AN(sym);						\
 		sym->action = func;					\
 		sym->action_mask = (mask);				\
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 758b7e7..69a2db9 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -739,7 +739,8 @@ VCC_New(void)
 	assert(tl->fh != NULL);
 
 	for (i = 1; i < VCL_MET_MAX; i++) {
-		sym = VCC_MkSym(tl, method_tab[i].name, SYM_SUB);
+		sym = VCC_MkSym(tl, method_tab[i].name,
+		    SYM_SUB, VCL_LOW, VCL_HIGH);
 		p = vcc_NewProc(tl, sym);
 		p->method = &method_tab[i];
 		VSB_printf(p->cname, "VGC_function_%s", p->method->name);
@@ -822,7 +823,7 @@ vcc_predef_vcl(struct vcc *vcc, const char *name)
 {
 	struct symbol *sym;
 
-	sym = VCC_MkSym(vcc, name, SYM_VCL);
+	sym = VCC_MkSym(vcc, name, SYM_VCL, VCL_LOW, VCL_HIGH);
 	AN(sym);
 	sym->type = VCL;
 	sym->r_methods = VCL_MET_RECV;
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index bdbb3a6..7ae776c 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -338,7 +338,8 @@ void vcc_stevedore(struct vcc *vcc, const char *stv_name);
 
 /* vcc_symb.c */
 void VCC_PrintCName(struct vsb *vsb, const char *b, const char *e);
-struct symbol *VCC_MkSym(struct vcc *tl, const char *b, vcc_kind_t);
+struct symbol *VCC_MkSym(struct vcc *tl, const char *b, vcc_kind_t,
+    unsigned, unsigned);
 extern const char XREF_NONE[];
 extern const char XREF_DEF[];
 extern const char XREF_REF[];
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 520e6ed..f8e5d90 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -1280,25 +1280,25 @@ vcc_Expr_Init(struct vcc *tl)
 {
 	struct symbol *sym;
 
-	sym = VCC_MkSym(tl, "regsub", SYM_FUNC);
+	sym = VCC_MkSym(tl, "regsub", SYM_FUNC, VCL_LOW, VCL_HIGH);
 	AN(sym);
 	sym->type = STRING;
 	sym->eval = vcc_Eval_Regsub;
 	sym->eval_priv = NULL;
 
-	sym = VCC_MkSym(tl, "regsuball", SYM_FUNC);
+	sym = VCC_MkSym(tl, "regsuball", SYM_FUNC, VCL_LOW, VCL_HIGH);
 	AN(sym);
 	sym->type = STRING;
 	sym->eval = vcc_Eval_Regsub;
 	sym->eval_priv = sym;
 
-	sym = VCC_MkSym(tl, "true", SYM_FUNC);
+	sym = VCC_MkSym(tl, "true", SYM_FUNC, VCL_LOW, VCL_HIGH);
 	AN(sym);
 	sym->type = BOOL;
 	sym->eval = vcc_Eval_BoolConst;
 	sym->eval_priv = sym;
 
-	sym = VCC_MkSym(tl, "false", SYM_FUNC);
+	sym = VCC_MkSym(tl, "false", SYM_FUNC, VCL_LOW, VCL_HIGH);
 	AN(sym);
 	sym->type = BOOL;
 	sym->eval = vcc_Eval_BoolConst;
diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index 95b459f..0326298 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -430,5 +430,5 @@ vcc_Parse_Init(struct vcc *tl)
 	struct toplev *tp;
 
 	for (tp = toplev; tp->name != NULL; tp++)
-		AN(VCC_MkSym(tl, tp->name, SYM_NONE));
+		AN(VCC_MkSym(tl, tp->name, SYM_NONE, VCL_LOW, VCL_HIGH));
 }
diff --git a/lib/libvcc/vcc_storage.c b/lib/libvcc/vcc_storage.c
index 06af8c7..a979708 100644
--- a/lib/libvcc/vcc_storage.c
+++ b/lib/libvcc/vcc_storage.c
@@ -84,7 +84,7 @@ vcc_stevedore(struct vcc *vcc, const char *stv_name)
 
 	CHECK_OBJ_NOTNULL(vcc, VCC_MAGIC);
 	bprintf(buf, "storage.%s", stv_name);
-	sym = VCC_MkSym(vcc, buf, SYM_VAR);
+	sym = VCC_MkSym(vcc, buf, SYM_VAR, VCL_LOW, VCL_40);
 	AN(sym);
 	sym->type = STEVEDORE;
 	sym->eval = vcc_Eval_Var;
@@ -94,7 +94,7 @@ vcc_stevedore(struct vcc *vcc, const char *stv_name)
 
 	for (sv = stvars; sv->name != NULL; sv++) {
 		bprintf(buf, "storage.%s.%s", stv_name, sv->name);
-		sym = VCC_MkSym(vcc, buf, SYM_VAR);
+		sym = VCC_MkSym(vcc, buf, SYM_VAR, VCL_LOW, VCL_40);
 		AN(sym);
 		sym->type = sv->type;
 		sym->eval = vcc_Eval_Var;
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index a039006..3d66e81 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -100,13 +100,15 @@ vcc_new_symbol(struct vcc *tl, const char *b, const char *e)
 
 static struct symbol *
 VCC_Symbol(struct vcc *tl, struct symbol *parent,
-    const char *b, const char *e, vcc_kind_t kind, int create)
+    const char *b, const char *e, vcc_kind_t kind,
+    int create, unsigned vlo, unsigned vhi)
 {
 	const char *q;
 	struct symbol *sym, *sym2 = NULL;
 	size_t l;
 	int i;
 
+	assert(vlo <= vhi);
 	if (tl->symbols == NULL)
 		tl->symbols = vcc_new_symbol(tl, "<root>", NULL);
 	if (parent == NULL)
@@ -128,6 +130,8 @@ VCC_Symbol(struct vcc *tl, struct symbol *parent,
 	assert(l > 0);
 
 	VTAILQ_FOREACH(sym, &parent->children, list) {
+		if (sym->lorev > vhi || sym->hirev < vlo)
+			continue;
 		i = strncasecmp(sym->name, b, l);
 		if (i < 0)
 			continue;
@@ -159,17 +163,21 @@ VCC_Symbol(struct vcc *tl, struct symbol *parent,
 			if (i > 0 || (i == 0 && l < sym->nlen))
 				break;
 		}
+		sym2->lorev = vlo;
+		sym2->hirev = vhi;
 		if (sym == NULL)
 			VTAILQ_INSERT_TAIL(&parent->children, sym2, list);
 		else
 			VTAILQ_INSERT_BEFORE(sym, sym2, list);
-		return (VCC_Symbol(tl, parent, b, e, kind, -1));
+		return (VCC_Symbol(tl, parent, b, e, kind, -1, vlo, vhi));
 	}
 	if (sym == NULL && create < 1)
 		return (sym);
 	if (sym == NULL) {
 		sym = vcc_new_symbol(tl, b, q);
 		sym->parent = parent;
+		sym->lorev = vlo;
+		sym->hirev = vhi;
 		if (sym2 != NULL)
 			VTAILQ_INSERT_BEFORE(sym2, sym, list);
 		else
@@ -180,7 +188,7 @@ VCC_Symbol(struct vcc *tl, struct symbol *parent,
 	if (q == e)
 		return (sym);
 	assert(*q == '.');
-	return (VCC_Symbol(tl, sym, ++q, e, kind, create));
+	return (VCC_Symbol(tl, sym, ++q, e, kind, create, vlo, vhi));
 }
 
 const char XREF_NONE[] = "xref_none";
@@ -206,7 +214,7 @@ VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x)
 	}
 
 	sym = VCC_Symbol(tl, NULL, tl->t->b, tl->t->e, kind,
-	    e == SYMTAB_CREATE ? 1 : 0);
+	    e == SYMTAB_CREATE ? 1 : 0, tl->syntax, tl->syntax);
 	if (sym == NULL && e == SYMTAB_NOERR)
 		return (sym);
 	if (sym == NULL) {
@@ -267,11 +275,12 @@ VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x)
 }
 
 struct symbol *
-VCC_MkSym(struct vcc *tl, const char *b, vcc_kind_t kind)
+VCC_MkSym(struct vcc *tl, const char *b, vcc_kind_t kind,
+    unsigned vlo, unsigned vhi)
 {
 	struct symbol *sym;
 
-	sym = VCC_Symbol(tl, NULL, b, NULL, kind, 1);
+	sym = VCC_Symbol(tl, NULL, b, NULL, kind, 1, vlo, vhi);
 	sym->noref = 1;
 	return (sym);
 }
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 92ae8df..8b5d1e6 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -94,7 +94,7 @@ vcc_ParseImport(struct vcc *tl)
 	}
 
 	bprintf(fn, "%.*s", PF(mod));
-	msym = VCC_MkSym(tl, fn, SYM_VMOD);
+	msym = VCC_MkSym(tl, fn, SYM_VMOD, VCL_LOW, VCL_HIGH);
 	ERRCHK(tl);
 	AN(msym);
 	msym->def_b = t1;
@@ -223,7 +223,7 @@ vcc_ParseImport(struct vcc *tl)
 		p = *spec;
 		if (!strcmp(p, "$OBJ")) {
 			p += strlen(p) + 1;
-			sym = VCC_MkSym(tl, p, SYM_OBJECT);
+			sym = VCC_MkSym(tl, p, SYM_OBJECT, VCL_LOW, VCL_HIGH);
 			XXXAN(sym);
 			sym->extra = p;
 			sym->vmod = msym->name;
@@ -242,7 +242,7 @@ vcc_ParseImport(struct vcc *tl)
 			    p, PF(mod));
 		} else if (!strcmp(p, "$FUNC")) {
 			p += strlen(p) + 1;
-			sym = VCC_MkSym(tl, p, SYM_FUNC);
+			sym = VCC_MkSym(tl, p, SYM_FUNC, VCL_LOW, VCL_HIGH);
 			ERRCHK(tl);
 			AN(sym);
 			sym->action = vcc_Act_Call;
@@ -331,7 +331,7 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym)
 	while (*p != '\0') {
 		p += strlen(s_obj);
 		bprintf(buf2, "%s%s", sy1->name, p);
-		sy3 = VCC_MkSym(tl, buf2, SYM_FUNC);
+		sy3 = VCC_MkSym(tl, buf2, SYM_FUNC, VCL_LOW, VCL_HIGH);
 		AN(sy3);
 		sy3->action = vcc_Act_Call;
 		sy3->eval = vcc_Eval_SymFunc;
diff --git a/lib/libvcc/vcc_xref.c b/lib/libvcc/vcc_xref.c
index 74f7aad..31d3bc6 100644
--- a/lib/libvcc/vcc_xref.c
+++ b/lib/libvcc/vcc_xref.c
@@ -338,8 +338,9 @@ vcc_xreftable(struct vcc *tl, const struct symbol *sym)
 	CHECK_OBJ_NOTNULL(sym, SYMBOL_MAGIC);
 	CHECK_OBJ_NOTNULL(sym->kind, KIND_MAGIC);
 	CHECK_OBJ_NOTNULL(sym->type, TYPE_MAGIC);
-	Fc(tl, 0, " * %-7s ", sym->kind->name);
+	Fc(tl, 0, " * %-8s ", sym->kind->name);
 	Fc(tl, 0, " %-9s ", sym->type->name);
+	Fc(tl, 0, " %2u %2u ", sym->lorev, sym->hirev);
 	vcc_pnam(tl, sym);
 	if (sym->wildcard != NULL)
 		Fc(tl, 0, "*");

From hermunn at varnish-software.com  Tue Feb 20 16:20:13 2018
From: hermunn at varnish-software.com (Pål Hermunn Johansen)
Date: Tue, 20 Feb 2018 16:20:13 +0000 (UTC)
Subject: [4.1] a02e4f2 Introduce ttl_now and the new way of calculating TTLs
 in VCL
Message-ID: <20180220162013.857F6B59DA@lists.varnish-cache.org>


commit a02e4f277c3ff12c8ef05b692713f87813a85da9
Author: P?l Hermunn Johansen <hermunn at varnish-software.com>
Date:   Tue Feb 20 16:25:26 2018 +0100

    Introduce ttl_now and the new way of calculating TTLs in VCL
    
    This is a back port of 33143e05c0720e5be0df4ac5b20a0fcc8a6874e8 in
    master, and for this reason it is a little strange.
    
    The strangeness is due to the fact that obj.ttl is not available
    in vcl_deliver here, but it is in master. This commit could have
    been much simpler without ttl_now, but the function is taken to
    4.1 regardles. The reason is that introducing obj.ttl in
    vcl_deliver is straightforward, and if someone is to do that in
    the future, the code in ttl_now(VRT_CTX) makes sure that obj.ttl
    will behave as in master, also in vcl_deliver.
    
    If obj.ttl is introduced in vcl_deliver, than also the two test
    cases s00008.vtc and s00009.vtc should be brought in, to make sure
    that obj.ttl works as expected.
    
    The following is the test from the commit in master:
    
    A new fucntion, ttl_now(VRT_CTX), defines what "now" is when ttl
    and age are calculated in various VCL subs. To sum up,
    
    * Before a backend fetch on the client side (vcl_recv, vcl_hit,
      vcl_miss) we use t_req from the request. This is the significance
      in this commit, and fixes the bug demonstrated by r02555.vtc.
    * On the backend side, most notably vcl_backend_responce, we keep
      the old "now" by simply using ctx->now.
    * In vcl_deliver we use ctx->now, as before.
    
    It was necessary to make all purges use t_req as their base time.
    Then, to not break c00041.vtc it was necessary to change from ">="
    to ">" in HSH_Lookup.
    
    All VMODs that currently use HSH_purge must change to using
    VRT_purge.
    
    Conflicts:
    	bin/varnishd/cache/cache_hash.c
    	bin/varnishd/cache/cache_objhead.h
    	bin/varnishd/cache/cache_req_fsm.c
    	bin/varnishd/cache/cache_vrt.c
    	bin/varnishd/cache/cache_vrt_var.c

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 42f2db6..88f50a7 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -429,7 +429,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 		if (vary != NULL && !VRY_Match(req, vary))
 			continue;
 
-		if (EXP_Ttl(req, &oc->exp) >= req->t_req) {
+		if (EXP_Ttl(req, &oc->exp) > req->t_req) {
 			/* If still valid, use it */
 			assert(oh->refcnt > 1);
 			assert(oc->objhead == oh);
@@ -574,13 +574,12 @@ hsh_rush(struct worker *wrk, struct objhead *oh)
  */
 
 void
-HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl, double grace,
+HSH_Purge(struct worker *wrk, struct objhead *oh, double ttl_now, double ttl, double grace,
 double keep)
 {
 	struct objcore *oc, **ocp;
 	unsigned spc, ospc, nobj, n, n_tot = 0;
 	int more = 0;
-	double now;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
@@ -607,7 +606,6 @@ double keep)
 		ocp = (void*)wrk->aws->f;
 		Lck_Lock(&oh->mtx);
 		assert(oh->refcnt > 0);
-		now = VTIM_real();
 		VTAILQ_FOREACH(oc, &oh->objcs, list) {
 			CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 			assert(oc->objhead == oh);
@@ -647,7 +645,7 @@ double keep)
 		for (n = 0; n < nobj; n++) {
 			oc = ocp[n];
 			CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-			EXP_Rearm(oc, now, ttl, grace, keep);
+			EXP_Rearm(oc, ttl_now, ttl, grace, keep);
 			(void)HSH_DerefObjCore(wrk, &oc);
 		}
 		n_tot += nobj;
diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 46e7765..25c548f 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -766,7 +766,7 @@ cnt_recv(struct worker *wrk, struct req *req)
 /*--------------------------------------------------------------------
  * Find the objhead, purge it.
  *
- * XXX: We should ask VCL if we should fetch a new copy of the object.
+ * In VCL, a restart is necessary to get a new object
  */
 
 static enum req_fsm_nxt
@@ -791,7 +791,7 @@ cnt_purge(struct worker *wrk, struct req *req)
 	CHECK_OBJ_NOTNULL(boc, OBJCORE_MAGIC);
 	VRY_Finish(req, DISCARD);
 
-	HSH_Purge(wrk, boc->objhead, 0, 0, 0);
+	HSH_Purge(wrk, boc->objhead, req->t_req, 0, 0, 0);
 
 	AZ(HSH_DerefObjCore(wrk, &boc));
 
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 678d8aa..f1955e7 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -510,14 +510,13 @@ VRT_purge(VRT_CTX, double ttl, double grace, double keep)
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req->wrk, WORKER_MAGIC);
-	if (ctx->method == VCL_MET_HIT)
-		HSH_Purge(ctx->req->wrk, ctx->req->objcore->objhead,
-		    ttl, grace, keep);
-	else if (ctx->method == VCL_MET_MISS)
+
+	if (ctx->method == VCL_MET_HIT || ctx->method == VCL_MET_MISS)
 		HSH_Purge(ctx->req->wrk, ctx->req->objcore->objhead,
-		    ttl, grace, keep);
+		    ctx->req->t_req, ttl, grace, keep);
 }
 
+
 /*--------------------------------------------------------------------
  * Simple stuff
  */
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 94441ed..bd67551 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -34,6 +34,8 @@
 #include "common/heritage.h"
 #include "hash/hash_slinger.h"
 
+#include "vcl.h"
+
 #include "cache_director.h"
 #include "vrt.h"
 #include "vrt_obj.h"
@@ -485,10 +487,23 @@ VRT_r_bereq_retries(VRT_CTX)
  *	ttl is relative to t_origin
  *	grace&keep are relative to ttl
  * In VCL:
- *	ttl is relative to now
+ *	ttl is relative to "ttl_now", which is t_req on the client
+ *	side, except in vcl_deliver, where it is ctx->now. On the
+ *	fetch side "ttl_now" is ctx->now (which is bo->t_prev).
  *	grace&keep are relative to ttl
  */
 
+static double ttl_now(VRT_CTX)
+{
+	if (ctx->bo) {
+		return (ctx->now);
+	} else {
+		CHECK_OBJ(ctx->req, REQ_MAGIC);
+		return (ctx->method == VCL_MET_DELIVER
+		    ? ctx->now : ctx->req->t_req);
+	}
+}
+
 #define VRT_DO_EXP_L(which, sexp, fld, offset)			\
 								\
 void								\
@@ -520,14 +535,14 @@ VRT_r_##which##_##fld(VRT_CTX)		\
 }
 
 VRT_DO_EXP_R(obj, ctx->req->objcore->exp, ttl,
-    ctx->now - ctx->req->objcore->exp.t_origin)
+    ttl_now(ctx) - ctx->req->objcore->exp.t_origin)
 VRT_DO_EXP_R(obj, ctx->req->objcore->exp, grace, 0)
 VRT_DO_EXP_R(obj, ctx->req->objcore->exp, keep, 0)
 
 VRT_DO_EXP_L(beresp, ctx->bo->fetch_objcore->exp, ttl,
-    ctx->now - ctx->bo->fetch_objcore->exp.t_origin)
+    ttl_now(ctx) - ctx->bo->fetch_objcore->exp.t_origin)
 VRT_DO_EXP_R(beresp, ctx->bo->fetch_objcore->exp, ttl,
-    ctx->now - ctx->bo->fetch_objcore->exp.t_origin)
+    ttl_now(ctx) - ctx->bo->fetch_objcore->exp.t_origin)
 VRT_DO_EXP_L(beresp, ctx->bo->fetch_objcore->exp, grace, 0)
 VRT_DO_EXP_R(beresp, ctx->bo->fetch_objcore->exp, grace, 0)
 VRT_DO_EXP_L(beresp, ctx->bo->fetch_objcore->exp, keep, 0)
@@ -543,7 +558,7 @@ VRT_r_##which##_##age(VRT_CTX)		\
 {								\
 								\
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);			\
-	return(ctx->now - sexp.t_origin);			\
+	return(ttl_now(ctx) - sexp.t_origin);			\
 }
 
 VRT_DO_AGE_R(obj, ctx->req->objcore->exp)
diff --git a/bin/varnishd/hash/hash_slinger.h b/bin/varnishd/hash/hash_slinger.h
index 736f1b5..e9cf3d7 100644
--- a/bin/varnishd/hash/hash_slinger.h
+++ b/bin/varnishd/hash/hash_slinger.h
@@ -69,8 +69,8 @@ void HSH_Ref(struct objcore *o);
 void HSH_Init(const struct hash_slinger *slinger);
 void HSH_AddString(struct req *, void *ctx, const char *str);
 void HSH_Insert(struct worker *, const void *hash, struct objcore *);
-void HSH_Purge(struct worker *, struct objhead *, double ttl, double grace,
-    double keep);
+void HSH_Purge(struct worker *, struct objhead *, double ttl_now, double ttl,
+    double grace, double keep);
 void HSH_config(const char *h_arg);
 struct busyobj *HSH_RefBusy(const struct objcore *oc);
 struct objcore *HSH_Private(struct worker *wrk);
diff --git a/bin/varnishtest/tests/r02555.vtc b/bin/varnishtest/tests/r02555.vtc
new file mode 100644
index 0000000..7a0fd8b
--- /dev/null
+++ b/bin/varnishtest/tests/r02555.vtc
@@ -0,0 +1,75 @@
+varnishtest "Expiry during processing"
+
+# When a object runs out of ttl+grace during processing of a
+# request, we want the calculation in the builtin VCL to match
+# the calculation of hit+grace in the C code.
+
+barrier b1 sock 2
+barrier b2 sock 2
+
+server s1 {
+	rxreq
+	expect req.url == "/1"
+	txresp
+
+	# If we get a second request to /1 here, it will be because a
+	# hit was became a miss in the builtin sub vcl_hit or because
+	# we never got to vcl_hit. We want a hit and a deliver.
+
+	rxreq
+	expect req.url == "/2"
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	import ${vmod_std};
+	import ${vmod_debug};
+
+	sub vcl_recv {
+		if (req.http.Sleep) {
+			# This will make the object expire while inside this VCL sub
+			if (!debug.barrier_sync("${b1_sock}")) {
+				return (synth(400));
+			}
+			if (!debug.barrier_sync("${b2_sock}")) {
+				return (synth(400));
+			}
+		}
+		# Update the last timestamp inside varnish
+		std.timestamp("T");
+	}
+	sub vcl_hit {
+		set req.http.X-was-hit = "true";
+		# no return here. Will the builtin VCL take us to vcl_miss?
+	}
+	sub vcl_miss {
+		set req.http.X-was-miss = "true";
+	}
+	sub vcl_backend_response {
+		# Very little ttl, a lot of keep
+		set beresp.ttl = 1s;
+		set beresp.grace = 0s;
+		set beresp.keep = 1w;
+	}
+} -start
+
+client c1 {
+	delay .5
+	txreq -url "/1"
+	rxresp
+	expect resp.status == 200
+	txreq -url "/1" -hdr "Sleep: true"
+	rxresp
+	expect resp.status == 200
+	# Final request to verify that the right amount of requests got to the backend
+	txreq -url "/2"
+	rxresp
+	expect resp.status == 200
+} -start
+
+# help for sleeping inside of vcl
+barrier b1 sync
+delay 1.5
+barrier b2 sync
+
+client c1 -wait
diff --git a/bin/varnishtest/tests/s00007.vtc b/bin/varnishtest/tests/s00007.vtc
new file mode 100644
index 0000000..39eecf5
--- /dev/null
+++ b/bin/varnishtest/tests/s00007.vtc
@@ -0,0 +1,42 @@
+varnishtest "Effective TTL for for a slow backend"
+
+server s1 {
+	rxreq
+	delay 2
+	txresp -body "foo"
+
+	# The second request is never used, but is here to give a
+	# better error if varnish decides to fetch the object the
+	# second time
+
+	rxreq
+	txresp -body "bar"
+} -start
+
+varnish v1 -arg "-p default_ttl=3 -p default_grace=0" -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.http.X-ttl = beresp.ttl;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.body == "foo"
+	expect resp.http.x-ttl <= 3
+	expect resp.http.x-ttl >= 2
+	delay 2
+
+	# It is now 2 seconds since the first response was received
+	# from the backend, but 4 seconds since the first request was
+	# sent to the backend. Timeout is 3 seconds, and here we
+	# consider the object _not_ expired, and thus do not want a
+	# refetch.
+
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.body == "foo"
+} -run
+

From fgsch at lodoss.net  Wed Feb 21 13:07:12 2018
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Wed, 21 Feb 2018 13:07:12 +0000 (UTC)
Subject: [master] 5f5185e Add changes for 4.1.9
Message-ID: <20180221130712.9ABE7A84D4@lists.varnish-cache.org>


commit 5f5185e8acd65f881c690bfdbf30fd5f49e16ad1
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Feb 21 12:50:03 2018 +0000

    Add changes for 4.1.9
    
    Prompted on irc by scn.

diff --git a/doc/changes.rst b/doc/changes.rst
index d4dff31..07791b5 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -523,6 +523,42 @@ News for Vmod Authors
 * PRIV_* now also work for object methods with unchanged scope.
 
 ================================
+Varnish Cache 4.1.9 (2017-11-14)
+================================
+
+Changes since 4.1.8:
+
+* Added ``bereq.is_bgfetch`` which is true for background fetches.
+* Add the vtc feature ignore_unknown_macro.
+* Expose to VCL whether or not a fetch is a background fetch (bgfetch)
+* Ignore req.ttl when keeping track of expired objects (see 2422_)
+* Move a cli buffer to VSB (from stack).
+* Use a separate stack for signals.
+
+.. _2422: https://github.com/varnishcache/varnish-cache/pull/2422
+
+Bugs fixed
+----------
+
+* 2337_ and 2366_ - Both Upgrade and Connection headers are needed for
+  WebSocket now
+* 2372_ - Fix problem with purging and the n_obj_purged counter
+* 2373_ - VSC n_vcl, n_vcl_avail, n_vcl_discard are gauge
+* 2380_ - Correct regexp in examples.
+* 2390_ - Straighten locking wrt vcl_active
+* 2429_ - Avoid buffer read overflow on vcl_backend_error and -sfile
+* 2492_ - Use the idle read timeout only on empty requests
+
+.. _2337: https://github.com/varnishcache/varnish-cache/issues/2337
+.. _2366: https://github.com/varnishcache/varnish-cache/issues/2366
+.. _2372: https://github.com/varnishcache/varnish-cache/pull/2372
+.. _2373: https://github.com/varnishcache/varnish-cache/issues/2373
+.. _2380: https://github.com/varnishcache/varnish-cache/issues/2380
+.. _2390: https://github.com/varnishcache/varnish-cache/issues/2390
+.. _2429: https://github.com/varnishcache/varnish-cache/pull/2429
+.. _2492: https://github.com/varnishcache/varnish-cache/issues/2492
+
+================================
 Varnish Cache 4.1.8 (2017-08-02)
 ================================
 

From hermunn at varnish-software.com  Wed Feb 21 15:27:08 2018
From: hermunn at varnish-software.com (Pål Hermunn Johansen)
Date: Wed, 21 Feb 2018 15:27:08 +0000 (UTC)
Subject: [4.1] 5c24e1f Fix issue #1799 for keep
Message-ID: <20180221152708.20785ACF65@lists.varnish-cache.org>


commit 5c24e1f884b87c630afb39556286c19dbfdb3ef4
Author: P?l Hermunn Johansen <hermunn at varnish-software.com>
Date:   Wed Dec 13 14:06:05 2017 +0100

    Fix issue #1799 for keep
    
    This fixes the long-standing #1799 for "keep" objects, and this commit
    message suggests a way of working around #1799 in the remaining
    cases. The following is a (long) explanation on how grace and keep
    works at the moment, how this relates to #1799, and how this commit
    changes things.
    
    1. How does it work now, before this commit?
    
    Objects in cache can outlive their TTL, and the typical reason for
    this is grace. Objects in cache can also linger because of obj.keep or
    in the (rare but observed) case where the expiry thread have not yet
    evicted an object. Grace and keep are here to minimize backend load,
    but #1799 shows that we are not successful in doing this in some
    important cases.
    
    Whenever sub vcl_recv has ended with return (lookup) (which is the
    default action), we arrive at HSH_Lookup, where varnish sometimes only
    finds an expired object (that match Vary logic, is not banned,
    etc). When this happens, we will initiate a background fetch (by
    adding a "busy object") if and only if there is no busy object on the
    oh already. Then the expired object is returned with HSH_EXP or
    HSH_EXPBUSY, depending on whether a busy object was inserted.
    
    2. What makes us run into #1799?
    
    When we have gotten an expired object, we generally hope that it is in
    grace, and that sub vcl_hit will return(deliver). However, if grace
    has expired, then the default action (ie the action from builtin.vcl)
    is return (miss). It is also possible that the user vcl, for some
    reason, decides that the stale object should not be delivered, and
    does return (miss) explicitly. In these cases it is common that the
    current request is not the one to insert a busy object, and then we
    run into the issue with a message "vcl_hit{} returns miss without busy
    object. Doing pass.".
    
    Note that normally, if a resource is very popular and has a positive
    grace, it is unlikely that #1799 will happen. Then a new version will
    always be available before the grace has run out, and everybody get
    the latest fetched version with no #1799 problems.
    
    However, if a resource is very popular (like a manifest file in a live
    streaming setup) and has 0s grace, and the expiry thread lags a little
    bit behind, then vcl_hit can get an expired object even when obj.keep
    is zero. In these circumstances we can get a surge of requests to the
    backend, and this is especially bad on a very busy server.
    
    Another real world example is where grace is initially set high (48h
    or similar) and vcl_hit considers the health of the backend, and, if
    the backend is healthy, explicitly does a return(miss) ensure that the
    client gets a fresh object. This has been a recommended use of
    vcl_hit, but, because of #1799, can cause considerable load on the
    backend.
    
    Similarly, we can get #1799 if we use "keep" to facilitate IMS
    requests to the backend, and we have a stale object for which several
    requests arrive before the first completes.
    
    3. How do we fix this?
    
    The main idea is to teach varnish to consider grace during lookup.
    
    To be specific, the following changes with this commit: If an expired
    object is found, the ttl+grace has expired and there already is an
    ongoing request for the object (ie. there exists a busy object), then
    the request is put on the waiting list instead of simply returning the
    object ("without a busy object") to vcl_hit. This choice is made
    because we anticipate that vcl_hit will do (the default) return (miss)
    and that it is better to wait for the ongoing request than to initiate
    a new one with "pass" behavior.
    
    The result is that when the ongoing request finishes, we will either
    be able to go to vcl_hit, start a new request (can happen if there was
    a Vary mismatch) by inserting a new "busy object", or we lose the race
    and have to go back to the waiting list (typically unlikely).
    
    When grace is in effect we go to vcl_hit even when we did not insert a
    busy object, anticipating that vcl_hit will return (deliver).
    
    This will will fix the cases where the user does not explicitly do a
    return(miss) in vcl_hit for object where ttl+grace has not
    expired. However, since this is not an uncommon practice, we also have
    to change our recommendation on how to use grace and keep. The new
    recommendation will be:
    
    * Set grace to the "normal value" for a working varnish+backend.
    
    * Set keep to a high value if the backend is not 100% reliable and you
      want to use stale objects as a fallback.
    
    * Do not explicitly return(miss) in sub vcl_hit{}. The exception is
      when this only can happen now and then and you are really sure that
      this is the right thing to do.
    
    * In vcl_hit, check if the backend is sick, and then explicitly
      return(deliver) when appropriate (ie you want an stale object
      delivered instead of an error message).
    
    A test case is included.

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 8a7c46b..4172cea 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -101,7 +101,7 @@ EXP_Clr(struct exp *e)
 }
 
 /*--------------------------------------------------------------------
- * Calculate an objects effective ttl time, taking req.ttl into account
+ * Calculate an object's effective ttl time, taking req.ttl into account
  * if it is available.
  */
 
diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 88f50a7..5f657f9 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -452,22 +452,33 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
 	if (exp_oc != NULL) {
 		assert(oh->refcnt > 1);
 		assert(exp_oc->objhead == oh);
-		exp_oc->refcnt++;
 
 		if (!busy_found) {
 			*bocp = hsh_insert_busyobj(wrk, oh);
 			retval = HSH_EXPBUSY;
 		} else {
 			AZ(req->hash_ignore_busy);
-			retval = HSH_EXP;
+			/*
+			 * here we have a busy object, but if the stale object
+			 * is not under grace we go to the waiting list
+			 * instead of returning the stale object
+			 */
+			if (EXP_Ttl(req, &exp_oc->exp) + exp_oc->exp.grace < req->t_req)
+				retval = HSH_BUSY;
+			else
+				retval = HSH_EXP;
+		}
+		if (retval != HSH_BUSY) {
+			exp_oc->refcnt++;
+
+			if (exp_oc->hits < LONG_MAX)
+				exp_oc->hits++;
+			Lck_Unlock(&oh->mtx);
+			if (retval == HSH_EXP)
+				assert(HSH_DerefObjHead(wrk, &oh));
+			*ocp = exp_oc;
+			return (retval);
 		}
-		if (exp_oc->hits < LONG_MAX)
-			exp_oc->hits++;
-		Lck_Unlock(&oh->mtx);
-		if (retval == HSH_EXP)
-			assert(HSH_DerefObjHead(wrk, &oh));
-		*ocp = exp_oc;
-		return (retval);
 	}
 
 	if (!busy_found) {
diff --git a/bin/varnishtest/tests/b00098.vtc b/bin/varnishtest/tests/b00098.vtc
new file mode 100644
index 0000000..511a4b4
--- /dev/null
+++ b/bin/varnishtest/tests/b00098.vtc
@@ -0,0 +1,69 @@
+varnishtest "Test that we properly wait for certain 304 cases"
+
+server s1 {
+	rxreq
+	txresp -hdr "Last-Modified: Wed, 11 Sep 2013 13:36:55 GMT" -body "Geoff Still Rules"
+
+	# The IMS request we will spend some time to process for the sake of
+	# this test.
+	rxreq
+	expect req.http.if-modified-since == "Wed, 11 Sep 2013 13:36:55 GMT"
+	delay 1
+	txresp -status 304
+
+	# Last request, to a different URL to catch it if varnish asks for "/" too many times
+	rxreq
+	expect req.url == "/2"
+	txresp -body "x"
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.ttl = 1s;
+		set beresp.grace = 1s;
+		set beresp.keep = 1m;
+		set beresp.http.was-304 = beresp.was_304;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.body == "Geoff Still Rules"
+} -run
+
+# let the object's ttl and grace expire
+delay 2.1
+
+# first client to ask for kept object - this should start the second request
+client c2 {
+	txreq
+	rxresp
+	# we did not disable grace in the request, so we should get the graced object here
+	expect resp.status == 200
+	expect resp.body == "Geoff Still Rules"
+} -start
+
+delay .1
+
+# second client to ask for the kept object. Here we want to wait until the backend fetch completes, not do a pass.
+client c3 {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.body == "Geoff Still Rules"
+} -start
+
+client c2 -wait
+client c3 -wait
+
+# Finally the request to "/2". The expect in the server block makes sure that
+# there were no extra requests to "/" from varnish.
+
+client c4 {
+	txreq -url "/2"
+	rxresp
+	expect resp.status == 200
+	expect resp.body == "x"
+} -run

From geoff at uplex.de  Wed Feb 21 16:30:11 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Wed, 21 Feb 2018 16:30:11 +0000 (UTC)
Subject: [master] b9f0551 Start the skeleton release notes for release 6.0.
Message-ID: <20180221163011.4147EAE23D@lists.varnish-cache.org>


commit b9f0551b9e189fdb1fb743901744484d8fd48852
Author: Geoff Simmons <geoff at uplex.de>
Date:   Wed Feb 21 17:28:00 2018 +0100

    Start the skeleton release notes for release 6.0.
    
    Just full of XXX's for now.

diff --git a/doc/sphinx/whats-new/changes-6.0.rst b/doc/sphinx/whats-new/changes-6.0.rst
new file mode 100644
index 0000000..73ea543
--- /dev/null
+++ b/doc/sphinx/whats-new/changes-6.0.rst
@@ -0,0 +1,52 @@
+.. _whatsnew_changes_6.0:
+
+Changes in Varnish 6.0
+======================
+
+XXX: ... intro paragraphs ...
+
+.. _whatsnew_new_subhead_1:
+
+XXX subhead 1
+~~~~~~~~~~~~~
+
+XXX ...
+
+XXX subsubhead 1.1
+------------------
+
+XXX: ...
+
+XXX subsubhead 1.2
+------------------
+
+XXX: ...
+
+XXX subhead 2
+~~~~~~~~~~~~~
+
+XXX ...
+
+XXX subsubhead 2.1
+------------------
+
+XXX: ...
+
+News for authors of VMODs and Varnish API client applications
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. _whatsnew_dev_subhead_1:
+
+XXX dev subhead 1
+-----------------
+
+XXX ...
+
+.. _whatsnew_dev_subhead_2:
+
+XXX dev subhead 2
+-----------------
+
+XXX ...
+
+*eof*
diff --git a/doc/sphinx/whats-new/index.rst b/doc/sphinx/whats-new/index.rst
index 35d47c3..1974cdb 100644
--- a/doc/sphinx/whats-new/index.rst
+++ b/doc/sphinx/whats-new/index.rst
@@ -8,6 +8,15 @@ This section describes the changes and improvements between different
 versions of Varnish, and what upgrading between the different versions
 entail.
 
+Varnish 6.0
+-----------
+
+.. toctree::
+   :maxdepth: 2
+
+   changes-6.0
+   upgrading-6.0
+
 Varnish 5.2
 -----------
 
diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst
new file mode 100644
index 0000000..c5e15be
--- /dev/null
+++ b/doc/sphinx/whats-new/upgrading-6.0.rst
@@ -0,0 +1,72 @@
+.. _whatsnew_upgrading_6.0:
+
+%%%%%%%%%%%%%%%%%%%%%%%%
+Upgrading to Varnish 6.0
+%%%%%%%%%%%%%%%%%%%%%%%%
+
+XXX: Most important change first
+================================
+
+XXX ...
+
+varnishd parameters
+===================
+
+XXX: ... terse overview, use refs to link to details ...
+
+Changes to VCL
+==============
+
+XXX: ... intro paragraph
+
+XXX VCL subhead 1
+~~~~~~~~~~~~~~~~~
+
+XXX: ... etc.
+
+VCL variables
+~~~~~~~~~~~~~
+
+XXX: VCL vars subhead 1
+-----------------------
+
+XXX: ...
+
+XXX: VCL vars subhead 2
+-----------------------
+
+XXX: ...
+
+XXX VCL subhead 2
+~~~~~~~~~~~~~~~~~
+
+XXX: ...
+
+Other changes
+=============
+
+* ``varnishd(1)``:
+
+  * XXX ...
+
+  * XXX ...
+
+* ``varnishstat(1)``:
+
+  * XXX ...
+
+  * XXX ...
+
+* XXX: etc. ...
+
+  * XXX ...
+
+  * XXX ...
+
+* Changes for developers:
+
+  * XXX ...
+
+  * XXX ...
+
+*eof*

From phk at FreeBSD.org  Wed Feb 21 17:01:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 21 Feb 2018 17:01:08 +0000 (UTC)
Subject: [master] 9ee6786 Comment why symbol handling is slightly special
Message-ID: <20180221170108.8CE84AECBA@lists.varnish-cache.org>


commit 9ee6786691d2671e550fd2a8187dbe49b2399af1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Feb 21 11:38:36 2018 +0000

    Comment why symbol handling is slightly special

diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 8b5d1e6..6b6969d 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -65,36 +65,36 @@ vcc_ParseImport(struct vcc *tl)
 	const char * const *spec;
 	struct symbol *sym;
 	struct symbol *msym;
-	const struct symbol *osym;
 	const char *p;
-	// int *modlen;
 	const struct vmod_data *vmd;
 
 	t1 = tl->t;
 	SkipToken(tl, ID);		/* "import" */
 
+
 	ExpectErr(tl, ID);
 	mod = tl->t;
-	osym = VCC_SymbolGet(tl, SYM_NONE, SYMTAB_NOERR, XREF_NONE);
-	vcc_NextToken(tl);
+	msym = VCC_SymbolGet(tl, SYM_NONE, SYMTAB_NOERR, XREF_NONE);
 
-	if (osym != NULL && osym->kind != SYM_VMOD) {
+	if (msym != NULL && msym->kind != SYM_VMOD) {
+		/*
+		 * We need to make sure the entire std.* namespace is empty
+		 */
 		VSB_printf(tl->sb, "Module %.*s conflicts with other symbol.\n",
 		    PF(mod));
 		vcc_ErrWhere2(tl, t1, tl->t);
 		return;
 	}
-	if (osym != NULL) {
+	if (msym != NULL) {
 		VSB_printf(tl->sb, "Module %.*s already imported.\n",
 		    PF(mod));
 		vcc_ErrWhere2(tl, t1, tl->t);
 		VSB_printf(tl->sb, "Previous import was here:\n");
-		vcc_ErrWhere2(tl, osym->def_b, osym->def_e);
+		vcc_ErrWhere2(tl, msym->def_b, msym->def_e);
 		return;
 	}
 
-	bprintf(fn, "%.*s", PF(mod));
-	msym = VCC_MkSym(tl, fn, SYM_VMOD, VCL_LOW, VCL_HIGH);
+	msym = VCC_SymbolGet(tl, SYM_VMOD, SYMTAB_CREATE, XREF_NONE);
 	ERRCHK(tl);
 	AN(msym);
 	msym->def_b = t1;

From phk at FreeBSD.org  Wed Feb 21 17:01:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 21 Feb 2018 17:01:08 +0000 (UTC)
Subject: [master] 1a36149 Move vjsn from libvarnishapi to libvarnish
Message-ID: <20180221170108.AA649AECBD@lists.varnish-cache.org>


commit 1a36149c013f2c820c98bbf4ee2d3850ab217ac9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Feb 21 13:39:40 2018 +0000

    Move vjsn from libvarnishapi to libvarnish

diff --git a/include/Makefile.am b/include/Makefile.am
index afa8864..9ddb077 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -86,6 +86,7 @@ nobase_noinst_HEADERS = \
 	vev.h \
 	vfil.h \
 	vin.h \
+	vjsn.h \
 	vlu.h \
 	vmb.h \
 	vnum.h \
diff --git a/include/vjsn.h b/include/vjsn.h
new file mode 100644
index 0000000..3b2f16b
--- /dev/null
+++ b/include/vjsn.h
@@ -0,0 +1,60 @@
+/*-
+ * Copyright (c) 2017 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+extern const char VJSN_OBJECT[];
+extern const char VJSN_ARRAY[];
+extern const char VJSN_NUMBER[];
+extern const char VJSN_STRING[];
+extern const char VJSN_TRUE[];
+extern const char VJSN_FALSE[];
+extern const char VJSN_NULL[];
+
+struct vjsn_val {
+	unsigned		magic;
+#define VJSN_VAL_MAGIC		0x08a06b80
+	const char		*type;
+	const char		*name;
+	VTAILQ_ENTRY(vjsn_val)	list;
+	VTAILQ_HEAD(,vjsn_val)	children;
+	char			*value;
+};
+
+struct vjsn {
+	unsigned		magic;
+#define VJSN_MAGIC		0x86a7f02b
+
+	char			*raw;
+	char			*ptr;
+	struct vjsn_val		*value;
+	const char		*err;
+};
+
+struct vjsn *vjsn_parse(const char *, const char **);
+void vjsn_delete(struct vjsn **);
+void vjsn_dump(const struct vjsn *js, FILE *fo);
+struct vjsn_val *vjsn_child(const struct vjsn_val *, const char *);
diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am
index 231827f..2f4d3eb 100644
--- a/lib/libvarnish/Makefile.am
+++ b/lib/libvarnish/Makefile.am
@@ -25,6 +25,7 @@ libvarnish_a_SOURCES = \
 	vfil.c \
 	vfl.c \
 	vin.c \
+	vjsn.c \
 	vlu.c \
 	vmb.c \
 	vnum.c \
@@ -40,7 +41,7 @@ libvarnish_a_SOURCES = \
 	vtim.c \
 	vus.c
 
-TESTS = vnum_c_test binheap
+TESTS = vjsn_test vnum_c_test binheap
 
 noinst_PROGRAMS = ${TESTS}
 
@@ -51,5 +52,6 @@ vnum_c_test_SOURCES = vnum.c vas.c
 vnum_c_test_CFLAGS = -DNUM_C_TEST -include config.h
 vnum_c_test_LDADD = ${LIBM}
 
-test: ${TESTS}
-	@for test in ${TESTS} ; do ./$${test} ; done
+vjsn_test_SOURCES = vjsn.c
+vjsn_test_CFLAGS = -DVJSN_TEST @SAN_CFLAGS@
+vjsn_test_LDADD = libvarnish.a @SAN_LDFLAGS@
diff --git a/lib/libvarnish/vjsn.c b/lib/libvarnish/vjsn.c
new file mode 100644
index 0000000..8f08e74
--- /dev/null
+++ b/lib/libvarnish/vjsn.c
@@ -0,0 +1,935 @@
+/*-
+ * Copyright (c) 2017 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "vdef.h"
+
+#include "vas.h"
+#include "miniobj.h"
+#include "vqueue.h"
+#include "vjsn.h"
+
+const char VJSN_OBJECT[] = "object";
+const char VJSN_ARRAY[] = "array";
+const char VJSN_NUMBER[] = "number";
+const char VJSN_STRING[] = "string";
+const char VJSN_TRUE[] = "true";
+const char VJSN_FALSE[] = "false";
+const char VJSN_NULL[] = "null";
+
+#define VJSN_EXPECT(js, xxx, ret)					\
+	do {								\
+		AZ(js->err);						\
+		if (*((js)->ptr) != xxx) {				\
+			js->err = "Expected " #xxx " not found.";	\
+			return (ret);					\
+		} else {						\
+			*js->ptr++ = '\0';				\
+		}							\
+	} while (0)
+
+static struct vjsn_val *vjsn_value(struct vjsn *);
+
+static struct vjsn_val *
+vjsn_val_new(const char *type)
+{
+	struct vjsn_val *jsv;
+
+	ALLOC_OBJ(jsv, VJSN_VAL_MAGIC);
+	AN(jsv);
+	VTAILQ_INIT(&jsv->children);
+	jsv->type = type;
+	return (jsv);
+}
+
+static void
+vjsn_val_delete(struct vjsn_val *jsv)
+{
+	struct vjsn_val *jsve;
+
+	CHECK_OBJ_NOTNULL(jsv, VJSN_VAL_MAGIC);
+	do {
+		jsve = VTAILQ_FIRST(&jsv->children);
+		if (jsve != NULL) {
+			VTAILQ_REMOVE(&jsv->children, jsve, list);
+			vjsn_val_delete(jsve);
+		}
+	} while (jsve != NULL);
+	FREE_OBJ(jsv);
+}
+
+void
+vjsn_delete(struct vjsn **jp)
+{
+	struct vjsn *js;
+
+	AN(jp);
+	js = *jp;
+	*jp = NULL;
+	CHECK_OBJ_NOTNULL(js, VJSN_MAGIC);
+	if (js->value != NULL)
+		vjsn_val_delete(js->value);
+	free(js->raw);
+	FREE_OBJ(js);
+}
+
+static void
+vjsn_skip_ws(struct vjsn *js)
+{
+	char c;
+
+	while (1) {
+		c = js->ptr[0];
+		if (c == 0x09 || c == 0x0a || c == 0x0d || c == 0x20) {
+			*js->ptr++ = '\0';
+			continue;
+		}
+#ifdef VJSN_COMMENTS
+		if (c == '/' && js->ptr[1] == '*') {
+			js->ptr += 2;
+			while (js->ptr[0] != '*' || js->ptr[1] != '/')
+				js->ptr++;
+			js->ptr += 2;
+			continue;
+		}
+#endif
+		return;
+	}
+}
+
+static unsigned
+vjsn_unumber(struct vjsn *js)
+{
+	unsigned u = 0;
+	char c;
+	int i;
+
+	VJSN_EXPECT(js, '\\', 0);
+	VJSN_EXPECT(js, 'u', 0);
+	for (i = 0; i < 4; i++) {
+		u <<= 4;
+		c = *js->ptr;
+		if (c >= '0' && c <= '9')
+			u |= c - '0';			/*lint !e737 */
+		else if (c >= 'A' && c <= 'F')
+			u |= c - '7';			/*lint !e737 */
+		else if (c >= 'a' && c <= 'f')
+			u |= c - 'W';			/*lint !e737 */
+		else {
+			js->err = "Illegal \\uXXXX sequence";
+			return (0);
+		}
+		js->ptr++;
+	}
+	return (u);
+}
+
+static void
+vjsn_unicode(struct vjsn *js, char **d)
+{
+	unsigned u1, u2;
+
+	u1 = vjsn_unumber(js);
+	if (js->err)
+		return;
+
+	if (u1 >= 0xdc00 && u1 <= 0xdfff) {
+		js->err = "Lone second UTF-16 Surrogate";
+		return;
+	}
+	if (u1 >= 0xd800 && u1 <= 0xdc00) {
+		u2 = vjsn_unumber(js);
+		if (u2 < 0xdc00 || u2 > 0xdfff) {
+			js->err = "Bad UTF-16 Surrogate pair";
+			return;
+		}
+		u1 -= 0xd800;
+		u2 -= 0xdc00;
+		u1 <<= 10;
+		u1 |= u2;
+		u1 |= 0x10000;
+	}
+	assert(u1 < 0x110000);
+	/*lint -save -e734 -e713 */
+	if (u1 < 0x80)
+		*(*d)++ = u1;
+	else if (u1 < 0x800) {
+		*(*d)++ = 0xc0 + u1 / 64;
+		*(*d)++ = 0x80 + u1 % 64;
+	} else if (u1 < 0x10000) {
+		*(*d)++ = 0xe0 + u1 / 4096;
+		*(*d)++ = 0x80 + u1 / 64 % 64;
+		*(*d)++ = 0x80 + u1 % 64;
+	} else {
+		*(*d)++ = 0xf0 + u1 / 262144;
+		*(*d)++ = 0x80 + u1 / 4096 % 64;
+		*(*d)++ = 0x80 + u1 / 64 % 64;
+		*(*d)++ = 0x80 + u1 % 64;
+	}
+	/*lint -restore */
+}
+
+static char *
+vjsn_string(struct vjsn *js)
+{
+	char *p, *b;
+
+	vjsn_skip_ws(js);
+	VJSN_EXPECT(js, '"', NULL);
+	b = p = js->ptr;
+	while (*js->ptr != '"') {
+		if (*js->ptr == '\0') {
+			js->err = "Unterminate string";
+			return (NULL);
+		}
+		if ((unsigned char)(*js->ptr) <= 0x1f) {
+			js->err = "unescaped control char in string";
+			return (NULL);
+		}
+		if (*js->ptr != '\\') {
+			*p++ = *js->ptr++;
+			continue;
+		}
+		switch (js->ptr[1]) {
+		case '\\':
+		case '/':
+		case '"': *p++ = js->ptr[1]; js->ptr += 2; break;
+		case 'b': *p++ = 0x08; js->ptr += 2; break;
+		case 'f': *p++ = 0x0c; js->ptr += 2; break;
+		case 't': *p++ = 0x09; js->ptr += 2; break;
+		case 'n': *p++ = 0x0a; js->ptr += 2; break;
+		case 'r': *p++ = 0x0d; js->ptr += 2; break;
+		case 'u':
+			vjsn_unicode(js, &p);
+			if (js->err != NULL)
+				return(NULL);
+			break;
+		default:
+			js->err = "Bad string escape";
+			return (NULL);
+		}
+	}
+	VJSN_EXPECT(js, '"', NULL);
+	*p = '\0';
+	return (b);
+}
+
+static struct vjsn_val *
+vjsn_object(struct vjsn *js)
+{
+	struct vjsn_val *jsv, *jsve;
+	char *s;
+
+	VJSN_EXPECT(js, '{', NULL);
+
+	jsv = vjsn_val_new(VJSN_OBJECT);
+	AN(jsv);
+
+	vjsn_skip_ws(js);
+	if (*js->ptr != '}') {
+		while (1) {
+			s = vjsn_string(js);
+			if (js->err != NULL)
+				return (jsv);
+			vjsn_skip_ws(js);
+			VJSN_EXPECT(js, ':', jsv);
+			jsve = vjsn_value(js);
+			if (js->err != NULL) {
+				if (jsve != NULL)
+					vjsn_val_delete(jsve);
+				return (jsv);
+			}
+			CHECK_OBJ_NOTNULL(jsve, VJSN_VAL_MAGIC);
+			jsve->name = s;
+			VTAILQ_INSERT_TAIL(&jsv->children, jsve, list);
+			vjsn_skip_ws(js);
+			if (*js->ptr == '}')
+				break;
+			VJSN_EXPECT(js, ',', jsv);
+		}
+	}
+	VJSN_EXPECT(js, '}', jsv);
+	return (jsv);
+}
+
+static struct vjsn_val *
+vjsn_array(struct vjsn *js)
+{
+	struct vjsn_val *jsv, *jsve;
+
+	VJSN_EXPECT(js, '[', NULL);
+
+	jsv = vjsn_val_new(VJSN_ARRAY);
+	AN(jsv);
+
+	vjsn_skip_ws(js);
+	if (*js->ptr != ']') {
+		while (1) {
+			jsve = vjsn_value(js);
+			if (js->err != NULL) {
+				if (jsve != NULL)
+					vjsn_val_delete(jsve);
+				return (jsv);
+			}
+			CHECK_OBJ_NOTNULL(jsve, VJSN_VAL_MAGIC);
+			VTAILQ_INSERT_TAIL(&jsv->children, jsve, list);
+			vjsn_skip_ws(js);
+			if (*js->ptr == ']')
+				break;
+			VJSN_EXPECT(js, ',', jsv);
+		}
+	}
+	VJSN_EXPECT(js, ']', jsv);
+	return (jsv);
+}
+
+static struct vjsn_val *
+vjsn_number(struct vjsn *js)
+{
+	struct vjsn_val *jsv;
+
+	jsv = vjsn_val_new(VJSN_NUMBER);
+	AN(jsv);
+
+	jsv->value = js->ptr;
+
+	if (*js->ptr == '-')
+		js->ptr++;
+	if (*js->ptr < '0' || *js->ptr > '9') {
+		js->err = "Bad number";
+		return (jsv);
+	}
+	if (*js->ptr == '0' && js->ptr[1] >= '0' && js->ptr[1] <= '9') {
+		js->err = "Bad number";
+		return (jsv);
+	}
+	while (*js->ptr >= '0' && *js->ptr <= '9')
+		js->ptr++;
+	if (*js->ptr == '.') {
+		js->ptr++;
+		if (*js->ptr < '0' || *js->ptr > '9') {
+			js->err = "Bad number";
+			return (jsv);
+		}
+		while (*js->ptr >= '0' && *js->ptr <= '9')
+			js->ptr++;
+	}
+	if (*js->ptr == 'e' || *js->ptr == 'E') {
+		js->ptr++;
+		if (*js->ptr == '-' || *js->ptr == '+')
+			js->ptr++;
+		if (*js->ptr < '0' || *js->ptr > '9') {
+			js->err = "Bad number";
+			return (jsv);
+		}
+		while (*js->ptr >= '0' && *js->ptr <= '9')
+			js->ptr++;
+	}
+	return (jsv);
+}
+
+static struct vjsn_val *
+vjsn_value(struct vjsn *js)
+{
+	struct vjsn_val *jsv;
+
+	AZ(js->err);
+	vjsn_skip_ws(js);
+	if (*js->ptr == '{')
+		return (vjsn_object(js));
+	if (*js->ptr== '[')
+		return (vjsn_array(js));
+	if (*js->ptr== '"') {
+		jsv = vjsn_val_new(VJSN_STRING);
+		jsv->value = vjsn_string(js);
+		if (js->err != NULL)
+			return (jsv);
+		AN(jsv->value);
+		return (jsv);
+	}
+	if (!strncmp(js->ptr, "true", 4)) {
+		js->ptr += 4;
+		return (vjsn_val_new(VJSN_TRUE));
+	}
+	if (!strncmp(js->ptr, "false", 5)) {
+		js->ptr += 5;
+		return (vjsn_val_new(VJSN_FALSE));
+	}
+	if (!strncmp(js->ptr, "null", 4)) {
+		js->ptr += 4;
+		return (vjsn_val_new(VJSN_NULL));
+	}
+	if (*js->ptr == '-' || (*js->ptr >= '0' && *js->ptr <= '9'))
+		return (vjsn_number(js));
+	js->err = "Unrecognized value";
+	return (NULL);
+}
+
+struct vjsn *
+vjsn_parse(const char *src, const char **err)
+{
+	struct vjsn *js;
+	char *p, *e;
+
+	AN(src);
+
+	AN(err);
+	*err = NULL;
+
+	p = strdup(src);
+	AN(p);
+	e = strchr(p, '\0');
+	AN(e);
+
+	ALLOC_OBJ(js, VJSN_MAGIC);
+	AN(js);
+	js->raw = p;
+	js->ptr = p;
+
+	js->value = vjsn_value(js);
+	if (js->err != NULL) {
+		*err = js->err;
+		vjsn_delete(&js);
+		return (NULL);
+	}
+
+	vjsn_skip_ws(js);
+	if (js->ptr != e) {
+		*err = "Garbage after value";
+		vjsn_delete(&js);
+		return (NULL);
+	}
+	return (js);
+}
+
+struct vjsn_val *
+vjsn_child(const struct vjsn_val *vv, const char *key)
+{
+	struct vjsn_val *vc;
+
+	CHECK_OBJ_NOTNULL(vv, VJSN_VAL_MAGIC);
+	AN(key);
+	VTAILQ_FOREACH(vc, &vv->children, list) {
+		if (vc->name != NULL && !strcmp(vc->name, key))
+			return (vc);
+	}
+	return (NULL);
+}
+
+static void
+vjsn_dump_i(const struct vjsn_val *jsv, FILE *fo, int indent)
+{
+	struct vjsn_val *jsve;
+
+	printf("%*s", indent, "");
+	if (jsv->name != NULL)
+		printf("[\"%s\"]: ", jsv->name);
+	printf("{%s}", jsv->type);
+	if (jsv->value != NULL) {
+		if (strlen(jsv->value) < 20)
+			printf(" <%s", jsv->value);
+		else
+			printf(" <%.10s[...#%zu]",
+			    jsv->value, strlen(jsv->value + 10));
+		printf(">");
+	}
+	printf("\n");
+	VTAILQ_FOREACH(jsve, &jsv->children, list)
+		vjsn_dump_i(jsve, fo, indent + 2);
+}
+
+void
+vjsn_dump(const struct vjsn *js, FILE *fo)
+{
+
+	CHECK_OBJ_NOTNULL(js, VJSN_MAGIC);
+	AN(fo);
+	vjsn_dump_i(js->value, fo, 0);
+}
+
+#ifdef VJSN_TEST
+
+/*
+ * Test-cases by Nicolas Seriot
+ *
+ * See: http://seriot.ch/parsing_json.php
+ *
+ * MIT License
+ *
+ * Copyright (c) 2016 Nicolas Seriot
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * We skip tests containing NUL, because we're lazy (The code will actually
+ * pass these tests if you provide for a way to pass the true length of the
+ * input to it, and we skip really huge tests, because we are only limited
+ * by available memory.
+ *
+ * To produce the C-data-structures:
+ *
+ * Clone https://github.com/nst/JSONTestSuite.git
+ *
+ * And run this python in test_parsing:
+
+	import glob
+
+	def emit(fin):
+		if fin in skip:
+			return
+		x = bytearray(open(fin).read())
+		if 0 in x:
+			return
+		if len(x) > 1000:
+			return
+		t = '\t"'
+		for i in x:
+			t += "\\x%02x" % i
+			if len(t) > 64:
+				print(t + '"')
+				t = '\t"'
+		print(t + '",')
+
+	print("const char *good[] = {")
+	for f in glob.glob("y_*"):
+		emit(f)
+	print("\tNULL")
+	print("};")
+
+	print("const char *bad[] = {")
+	for f in glob.glob("n_*"):
+		emit(f)
+	print("\tNULL")
+	print("};")
+
+*/
+
+static const char *good[] = {
+	"\x5b\x31\x32\x33\x65\x36\x35\x5d",
+	"\x5b\x5b\x5d\x20\x20\x20\x5d",
+	"\x5b\x22\x22\x5d",
+	"\x5b\x5d",
+	"\x5b\x22\x61\x22\x5d",
+	"\x5b\x66\x61\x6c\x73\x65\x5d",
+	"\x5b\x6e\x75\x6c\x6c\x2c\x20\x31\x2c\x20\x22\x31\x22\x2c\x20\x7b"
+	"\x7d\x5d",
+	"\x5b\x6e\x75\x6c\x6c\x5d",
+	"\x5b\x31\x0a\x5d",
+	"\x20\x5b\x31\x5d",
+	"\x5b\x31\x2c\x6e\x75\x6c\x6c\x2c\x6e\x75\x6c\x6c\x2c\x6e\x75\x6c"
+	"\x6c\x2c\x32\x5d",
+	"\x5b\x32\x5d\x20",
+	"\x5b\x30\x65\x2b\x31\x5d",
+	"\x5b\x30\x65\x31\x5d",
+	"\x5b\x20\x34\x5d",
+	"\x5b\x2d\x30\x2e\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30"
+	"\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30"
+	"\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30"
+	"\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30"
+	"\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30"
+	"\x30\x31\x5d\x0a",
+	"\x5b\x32\x30\x65\x31\x5d",
+	"\x5b\x2d\x30\x5d",
+	"\x5b\x2d\x31\x32\x33\x5d",
+	"\x5b\x2d\x31\x5d",
+	"\x5b\x2d\x30\x5d",
+	"\x5b\x31\x45\x32\x32\x5d",
+	"\x5b\x31\x45\x2d\x32\x5d",
+	"\x5b\x31\x45\x2b\x32\x5d",
+	"\x5b\x31\x32\x33\x65\x34\x35\x5d",
+	"\x5b\x31\x32\x33\x2e\x34\x35\x36\x65\x37\x38\x5d",
+	"\x5b\x31\x65\x2d\x32\x5d",
+	"\x5b\x31\x65\x2b\x32\x5d",
+	"\x5b\x31\x32\x33\x5d",
+	"\x5b\x31\x32\x33\x2e\x34\x35\x36\x37\x38\x39\x5d",
+	"\x7b\x22\x61\x73\x64\x22\x3a\x22\x73\x64\x66\x22\x2c\x20\x22\x64"
+	"\x66\x67\x22\x3a\x22\x66\x67\x68\x22\x7d",
+	"\x7b\x22\x61\x73\x64\x22\x3a\x22\x73\x64\x66\x22\x7d",
+	"\x7b\x22\x61\x22\x3a\x22\x62\x22\x2c\x22\x61\x22\x3a\x22\x63\x22"
+	"\x7d",
+	"\x7b\x22\x61\x22\x3a\x22\x62\x22\x2c\x22\x61\x22\x3a\x22\x62\x22"
+	"\x7d",
+	"\x7b\x7d",
+	"\x7b\x22\x22\x3a\x30\x7d",
+	"\x7b\x22\x66\x6f\x6f\x5c\x75\x30\x30\x30\x30\x62\x61\x72\x22\x3a"
+	"\x20\x34\x32\x7d",
+	"\x7b\x20\x22\x6d\x69\x6e\x22\x3a\x20\x2d\x31\x2e\x30\x65\x2b\x32"
+	"\x38\x2c\x20\x22\x6d\x61\x78\x22\x3a\x20\x31\x2e\x30\x65\x2b\x32"
+	"\x38\x20\x7d",
+	"\x7b\x22\x78\x22\x3a\x5b\x7b\x22\x69\x64\x22\x3a\x20\x22\x78\x78"
+	"\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78"
+	"\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78"
+	"\x78\x78\x78\x78\x78\x78\x22\x7d\x5d\x2c\x20\x22\x69\x64\x22\x3a"
+	"\x20\x22\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78"
+	"\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78"
+	"\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x22\x7d",
+	"\x7b\x22\x61\x22\x3a\x5b\x5d\x7d",
+	"\x7b\x22\x74\x69\x74\x6c\x65\x22\x3a\x22\x5c\x75\x30\x34\x31\x66"
+	"\x5c\x75\x30\x34\x33\x65\x5c\x75\x30\x34\x33\x62\x5c\x75\x30\x34"
+	"\x34\x32\x5c\x75\x30\x34\x33\x65\x5c\x75\x30\x34\x34\x30\x5c\x75"
+	"\x30\x34\x33\x30\x20\x5c\x75\x30\x34\x31\x37\x5c\x75\x30\x34\x33"
+	"\x35\x5c\x75\x30\x34\x33\x63\x5c\x75\x30\x34\x33\x62\x5c\x75\x30"
+	"\x34\x33\x35\x5c\x75\x30\x34\x33\x61\x5c\x75\x30\x34\x33\x65\x5c"
+	"\x75\x30\x34\x33\x66\x5c\x75\x30\x34\x33\x30\x22\x20\x7d",
+	"\x7b\x0a\x22\x61\x22\x3a\x20\x22\x62\x22\x0a\x7d",
+	"\x5b\x22\x5c\x75\x44\x38\x30\x31\x5c\x75\x64\x63\x33\x37\x22\x5d"
+	"",
+	"\x5b\x22\x5c\x75\x30\x30\x36\x30\x5c\x75\x30\x31\x32\x61\x5c\x75"
+	"\x31\x32\x41\x42\x22\x5d",
+	"\x5b\x22\x5c\x75\x64\x38\x33\x64\x5c\x75\x64\x65\x33\x39\x5c\x75"
+	"\x64\x38\x33\x64\x5c\x75\x64\x63\x38\x64\x22\x5d",
+	"\x5b\x22\x5c\x22\x5c\x5c\x5c\x2f\x5c\x62\x5c\x66\x5c\x6e\x5c\x72"
+	"\x5c\x74\x22\x5d",
+	"\x5b\x22\x5c\x5c\x75\x30\x30\x30\x30\x22\x5d",
+	"\x5b\x22\x5c\x22\x22\x5d",
+	"\x5b\x22\x61\x2f\x2a\x62\x2a\x2f\x63\x2f\x2a\x64\x2f\x2f\x65\x22"
+	"\x5d",
+	"\x5b\x22\x5c\x5c\x61\x22\x5d",
+	"\x5b\x22\x5c\x5c\x6e\x22\x5d",
+	"\x5b\x22\x5c\x75\x30\x30\x31\x32\x22\x5d",
+	"\x5b\x22\x5c\x75\x46\x46\x46\x46\x22\x5d",
+	"\x5b\x22\x61\x73\x64\x22\x5d",
+	"\x5b\x22\x6e\x65\x77\x5c\x75\x30\x30\x41\x30\x6c\x69\x6e\x65\x22"
+	"\x5d",
+	"\x5b\x20\x22\x61\x73\x64\x22\x5d",
+	"\x5b\x22\x5c\x75\x44\x42\x46\x46\x5c\x75\x44\x46\x46\x46\x22\x5d"
+	"",
+	"\x5b\x22\xf4\x8f\xbf\xbf\x22\x5d",
+	"\x5b\x22\xf0\x9b\xbf\xbf\x22\x5d",
+	"\x5b\x22\xef\xbf\xbf\x22\x5d",
+	"\x5b\x22\x5c\x75\x30\x30\x30\x30\x22\x5d",
+	"\x5b\x22\x5c\x75\x30\x30\x32\x63\x22\x5d",
+	"\x5b\x22\xcf\x80\x22\x5d",
+	"\x5b\x22\x61\x73\x64\x20\x22\x5d",
+	"\x22\x20\x22",
+	"\x5b\x22\x5c\x75\x44\x38\x33\x34\x5c\x75\x44\x64\x31\x65\x22\x5d"
+	"",
+	"\x5b\x22\x5c\x75\x30\x38\x32\x31\x22\x5d",
+	"\x5b\x22\x5c\x75\x30\x31\x32\x33\x22\x5d",
+	"\x5b\x22\xe2\x80\xa8\x22\x5d",
+	"\x5b\x22\xe2\x80\xa9\x22\x5d",
+	"\x5b\x22\x5c\x75\x30\x30\x36\x31\x5c\x75\x33\x30\x61\x66\x5c\x75"
+	"\x33\x30\x45\x41\x5c\x75\x33\x30\x62\x39\x22\x5d",
+	"\x5b\x22\x6e\x65\x77\x5c\x75\x30\x30\x30\x41\x6c\x69\x6e\x65\x22"
+	"\x5d",
+	"\x5b\x22\x7f\x22\x5d",
+	"\x5b\x22\x5c\x75\x41\x36\x36\x44\x22\x5d",
+	"\x5b\x22\x5c\x75\x30\x30\x35\x43\x22\x5d",
+	"\x5b\x22\xe2\x8d\x82\xe3\x88\xb4\xe2\x8d\x82\x22\x5d",
+	"\x5b\x22\x5c\x75\x44\x42\x46\x46\x5c\x75\x44\x46\x46\x45\x22\x5d"
+	"",
+	"\x5b\x22\x5c\x75\x44\x38\x33\x46\x5c\x75\x44\x46\x46\x45\x22\x5d"
+	"",
+	"\x5b\x22\xe2\x82\xac\xf0\x9d\x84\x9e\x22\x5d",
+	"\x5b\x22\x5c\x75\x32\x30\x30\x42\x22\x5d",
+	"\x5b\x22\x5c\x75\x32\x30\x36\x34\x22\x5d",
+	"\x5b\x22\x5c\x75\x46\x44\x44\x30\x22\x5d",
+	"\x5b\x22\x5c\x75\x46\x46\x46\x45\x22\x5d",
+	"\x5b\x22\x5c\x75\x30\x30\x32\x32\x22\x5d",
+	"\x5b\x22\x61\x7f\x61\x22\x5d",
+	"\x66\x61\x6c\x73\x65",
+	"\x34\x32",
+	"\x2d\x30\x2e\x31",
+	"\x6e\x75\x6c\x6c",
+	"\x22\x61\x73\x64\x22",
+	"\x74\x72\x75\x65",
+	"\x22\x22",
+	"\x5b\x22\x61\x22\x5d\x0a",
+	"\x5b\x74\x72\x75\x65\x5d",
+	"\x20\x5b\x5d\x20",
+	NULL
+};
+static const char *bad[] = {
+	"\x5b\x31\x20\x74\x72\x75\x65\x5d",
+	"\x5b\x61\xe5\x5d",
+	"\x5b\x22\x22\x3a\x20\x31\x5d",
+	"\x5b\x22\x22\x5d\x2c",
+	"\x5b\x2c\x31\x5d",
+	"\x5b\x31\x2c\x2c\x32\x5d",
+	"\x5b\x22\x78\x22\x2c\x2c\x5d",
+	"\x5b\x22\x78\x22\x5d\x5d",
+	"\x5b\x22\x22\x2c\x5d",
+	"\x5b\x22\x78\x22",
+	"\x5b\xff\x5d",
+	"\x5b\x78",
+	"\x5b\x33\x5b\x34\x5d\x5d",
+	"\x5b\x31\x3a\x32\x5d",
+	"\x5b\x2c\x5d",
+	"\x5b\x2d\x5d",
+	"\x5b\x20\x20\x20\x2c\x20\x22\x22\x5d",
+	"\x5b\x22\x61\x22\x2c\x0a\x34\x0a\x2c\x31\x2c",
+	"\x5b\x31\x2c\x5d",
+	"\x5b\x31\x2c\x2c\x5d",
+	"\x5b\x22\x0b\x61\x22\x5c\x66\x5d",
+	"\x5b\x2a\x5d",
+	"\x5b\x22\x22",
+	"\x5b\x31\x2c",
+	"\x5b\x31\x2c\x0a\x31\x0a\x2c\x31",
+	"\x5b\x7b\x7d",
+	"\x5b\x66\x61\x6c\x73\x5d",
+	"\x5b\x6e\x75\x6c\x5d",
+	"\x5b\x74\x72\x75\x5d",
+	"\x5b\x2b\x2b\x31\x32\x33\x34\x5d",
+	"\x5b\x2b\x31\x5d",
+	"\x5b\x2b\x49\x6e\x66\x5d",
+	"\x5b\x2d\x30\x31\x5d",
+	"\x5b\x2d\x31\x2e\x30\x2e\x5d",
+	"\x5b\x2d\x32\x2e\x5d",
+	"\x5b\x2d\x4e\x61\x4e\x5d",
+	"\x5b\x2e\x2d\x31\x5d",
+	"\x5b\x2e\x32\x65\x2d\x33\x5d",
+	"\x5b\x30\x2e\x31\x2e\x32\x5d",
+	"\x5b\x30\x2e\x33\x65\x2b\x5d",
+	"\x5b\x30\x2e\x33\x65\x5d",
+	"\x5b\x30\x2e\x65\x31\x5d",
+	"\x5b\x30\x45\x2b\x5d",
+	"\x5b\x30\x45\x5d",
+	"\x5b\x30\x65\x2b\x5d",
+	"\x5b\x30\x65\x5d",
+	"\x5b\x31\x2e\x30\x65\x2b\x5d",
+	"\x5b\x31\x2e\x30\x65\x2d\x5d",
+	"\x5b\x31\x2e\x30\x65\x5d",
+	"\x5b\x31\x20\x30\x30\x30\x2e\x30\x5d",
+	"\x5b\x31\x65\x45\x32\x5d",
+	"\x5b\x32\x2e\x65\x2b\x33\x5d",
+	"\x5b\x32\x2e\x65\x2d\x33\x5d",
+	"\x5b\x32\x2e\x65\x33\x5d",
+	"\x5b\x39\x2e\x65\x2b\x5d",
+	"\x5b\x49\x6e\x66\x5d",
+	"\x5b\x4e\x61\x4e\x5d",
+	"\x5b\xef\xbc\x91\x5d",
+	"\x5b\x31\x2b\x32\x5d",
+	"\x5b\x30\x78\x31\x5d",
+	"\x5b\x30\x78\x34\x32\x5d",
+	"\x5b\x49\x6e\x66\x69\x6e\x69\x74\x79\x5d",
+	"\x5b\x30\x65\x2b\x2d\x31\x5d",
+	"\x5b\x2d\x31\x32\x33\x2e\x31\x32\x33\x66\x6f\x6f\x5d",
+	"\x5b\x31\x32\x33\xe5\x5d",
+	"\x5b\x31\x65\x31\xe5\x5d",
+	"\x5b\x30\xe5\x5d\x0a",
+	"\x5b\x2d\x49\x6e\x66\x69\x6e\x69\x74\x79\x5d",
+	"\x5b\x2d\x66\x6f\x6f\x5d",
+	"\x5b\x2d\x20\x31\x5d",
+	"\x5b\x2d\x30\x31\x32\x5d",
+	"\x5b\x2d\x2e\x31\x32\x33\x5d",
+	"\x5b\x2d\x31\x78\x5d",
+	"\x5b\x31\x65\x61\x5d",
+	"\x5b\x31\x65\xe5\x5d",
+	"\x5b\x31\x2e\x5d",
+	"\x5b\x2e\x31\x32\x33\x5d",
+	"\x5b\x31\x2e\x32\x61\x2d\x33\x5d",
+	"\x5b\x31\x2e\x38\x30\x31\x31\x36\x37\x30\x30\x33\x33\x33\x37\x36"
+	"\x35\x31\x34\x48\x2d\x33\x30\x38\x5d",
+	"\x5b\x30\x31\x32\x5d",
+	"\x5b\x22\x78\x22\x2c\x20\x74\x72\x75\x74\x68\x5d",
+	"\x7b\x5b\x3a\x20\x22\x78\x22\x7d\x0a",
+	"\x7b\x22\x78\x22\x2c\x20\x6e\x75\x6c\x6c\x7d",
+	"\x7b\x22\x78\x22\x3a\x3a\x22\x62\x22\x7d",
+	"\x7b\xf0\x9f\x87\xa8\xf0\x9f\x87\xad\x7d",
+	"\x7b\x22\x61\x22\x3a\x22\x61\x22\x20\x31\x32\x33\x7d",
+	"\x7b\x6b\x65\x79\x3a\x20\x27\x76\x61\x6c\x75\x65\x27\x7d",
+	"\x7b\x22\x61\x22\x20\x62\x7d",
+	"\x7b\x3a\x22\x62\x22\x7d",
+	"\x7b\x22\x61\x22\x20\x22\x62\x22\x7d",
+	"\x7b\x22\x61\x22\x3a",
+	"\x7b\x22\x61\x22",
+	"\x7b\x31\x3a\x31\x7d",
+	"\x7b\x39\x39\x39\x39\x45\x39\x39\x39\x39\x3a\x31\x7d",
+	"\x7b\x22\xb9\x22\x3a\x22\x30\x22\x2c\x7d",
+	"\x7b\x6e\x75\x6c\x6c\x3a\x6e\x75\x6c\x6c\x2c\x6e\x75\x6c\x6c\x3a"
+	"\x6e\x75\x6c\x6c\x7d",
+	"\x7b\x22\x69\x64\x22\x3a\x30\x2c\x2c\x2c\x2c\x2c\x7d",
+	"\x7b\x27\x61\x27\x3a\x30\x7d",
+	"\x7b\x22\x69\x64\x22\x3a\x30\x2c\x7d",
+	"\x7b\x22\x61\x22\x3a\x22\x62\x22\x7d\x2f\x2a\x2a\x2f",
+	"\x7b\x22\x61\x22\x3a\x22\x62\x22\x7d\x2f\x2a\x2a\x2f\x2f",
+	"\x7b\x22\x61\x22\x3a\x22\x62\x22\x7d\x2f\x2f",
+	"\x7b\x22\x61\x22\x3a\x22\x62\x22\x7d\x2f",
+	"\x7b\x22\x61\x22\x3a\x22\x62\x22\x2c\x2c\x22\x63\x22\x3a\x22\x64"
+	"\x22\x7d",
+	"\x7b\x61\x3a\x20\x22\x62\x22\x7d",
+	"\x7b\x22\x61\x22\x3a\x22\x61",
+	"\x7b\x20\x22\x66\x6f\x6f\x22\x20\x3a\x20\x22\x62\x61\x72\x22\x2c"
+	"\x20\x22\x61\x22\x20\x7d",
+	"\x7b\x22\x61\x22\x3a\x22\x62\x22\x7d\x23",
+	"\x20",
+	"\x5b\x22\x5c\x75\x44\x38\x30\x30\x5c\x22\x5d",
+	"\x5b\x22\x5c\x75\x44\x38\x30\x30\x5c\x75\x22\x5d",
+	"\x5b\x22\x5c\x75\x44\x38\x30\x30\x5c\x75\x31\x22\x5d",
+	"\x5b\x22\x5c\x75\x44\x38\x30\x30\x5c\x75\x31\x78\x22\x5d",
+	"\x5b\xc3\xa9\x5d",
+	"\x5b\x22\x5c\x78\x30\x30\x22\x5d",
+	"\x5b\x22\x5c\x5c\x5c\x22\x5d",
+	"\x5b\x22\x5c\x09\x22\x5d",
+	"\x5b\x22\x5c\xf0\x9f\x8c\x80\x22\x5d",
+	"\x5b\x22\x5c\x22\x5d",
+	"\x5b\x22\x5c\x75\x30\x30\x41\x22\x5d",
+	"\x5b\x22\x5c\x75\x44\x38\x33\x34\x5c\x75\x44\x64\x22\x5d",
+	"\x5b\x22\x5c\x75\x44\x38\x30\x30\x5c\x75\x44\x38\x30\x30\x5c\x78"
+	"\x22\x5d",
+	"\x5b\x22\x5c\x75\xe5\x22\x5d",
+	"\x5b\x22\x5c\x61\x22\x5d",
+	"\x5b\x22\x5c\x75\x71\x71\x71\x71\x22\x5d",
+	"\x5b\x22\x5c\xe5\x22\x5d",
+	"\x5b\x5c\x75\x30\x30\x32\x30\x22\x61\x73\x64\x22\x5d",
+	"\x5b\x5c\x6e\x5d",
+	"\x22",
+	"\x5b\x27\x73\x69\x6e\x67\x6c\x65\x20\x71\x75\x6f\x74\x65\x27\x5d"
+	"",
+	"\x61\x62\x63",
+	"\x5b\x22\x5c",
+	"\x5b\x22\x6e\x65\x77\x0a\x6c\x69\x6e\x65\x22\x5d",
+	"\x5b\x22\x09\x22\x5d",
+	"\x22\x5c\x55\x41\x36\x36\x44\x22",
+	"\x22\x22\x78",
+	"\x5b\xe2\x81\xa0\x5d",
+	"\xef\xbb\xbf",
+	"\x3c\x2e\x3e",
+	"\x5b\x3c\x6e\x75\x6c\x6c\x3e\x5d",
+	"\x5b\x31\x5d\x78",
+	"\x5b\x31\x5d\x5d",
+	"\x5b\x22\x61\x73\x64\x5d",
+	"\x61\xc3\xa5",
+	"\x5b\x54\x72\x75\x65\x5d",
+	"\x31\x5d",
+	"\x7b\x22\x78\x22\x3a\x20\x74\x72\x75\x65\x2c",
+	"\x5b\x5d\x5b\x5d",
+	"\x5d",
+	"\xef\xbb\x7b\x7d",
+	"\xe5",
+	"\x5b",
+	"",
+	"\x32\x40",
+	"\x7b\x7d\x7d",
+	"\x7b\x22\x22\x3a",
+	"\x7b\x22\x61\x22\x3a\x2f\x2a\x63\x6f\x6d\x6d\x65\x6e\x74\x2a\x2f"
+	"\x22\x62\x22\x7d",
+	"\x7b\x22\x61\x22\x3a\x20\x74\x72\x75\x65\x7d\x20\x22\x78\x22",
+	"\x5b\x27",
+	"\x5b\x2c",
+	"\x5b\x7b",
+	"\x5b\x22\x61",
+	"\x5b\x22\x61\x22",
+	"\x7b",
+	"\x7b\x5d",
+	"\x7b\x2c",
+	"\x7b\x5b",
+	"\x7b\x22\x61",
+	"\x7b\x27\x61\x27",
+	"\x5b\x22\x5c\x7b\x5b\x22\x5c\x7b\x5b\x22\x5c\x7b\x5b\x22\x5c\x7b"
+	"",
+	"\xe9",
+	"\x2a",
+	"\x7b\x22\x61\x22\x3a\x22\x62\x22\x7d\x23\x7b\x7d",
+	"\x5b\x5c\x75\x30\x30\x30\x41\x22\x22\x5d",
+	"\x5b\x31",
+	"\x5b\x20\x66\x61\x6c\x73\x65\x2c\x20\x6e\x75\x6c",
+	"\x5b\x20\x74\x72\x75\x65\x2c\x20\x66\x61\x6c\x73",
+	"\x5b\x20\x66\x61\x6c\x73\x65\x2c\x20\x74\x72\x75",
+	"\x7b\x22\x61\x73\x64\x22\x3a\x22\x61\x73\x64\x22",
+	"\xc3\xa5",
+	"\x5b\xe2\x81\xa0\x5d",
+	"\x5b\x0c\x5d",
+	NULL
+};
+
+static void
+test_good(const char *j)
+{
+	struct vjsn *js;
+	const char *err;
+
+	js = vjsn_parse(j, &err);
+	if (js == NULL || err != NULL) {
+		fprintf(stderr, "Parse error: %s\n%s\n", err, j);
+		exit(1);
+	}
+	printf("GOOD: %s\n", j);
+	vjsn_dump(js, stdout);
+	vjsn_delete(&js);
+}
+
+static void
+test_bad(const char *j)
+{
+	struct vjsn *js;
+	const char *err;
+
+	js = vjsn_parse(j, &err);
+	if (js != NULL || err == NULL) {
+		fprintf(stderr, "Parse succeeded %s\n", j);
+		exit(1);
+	}
+	printf("BAD: %s %s\n", err, j);
+}
+
+int
+main(int argc, char **argv)
+{
+	const char **s;
+
+	(void)argc;
+	(void)argv;
+	for (s = good; *s != NULL; s++)
+		test_good(*s);
+	for (s = bad; *s != NULL; s++)
+		test_bad(*s);
+
+	/*
+	 * This is part of Nicolas i(ndeterminate) test set, for reasons I
+	 * do not fully grasp, but we want it to test bad.
+	 */
+	test_bad("\"\\uDFAA\"");
+	printf("Tests done\n");
+	return (0);
+}
+
+#endif
diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am
index 7c88c23..e679de3 100644
--- a/lib/libvarnishapi/Makefile.am
+++ b/lib/libvarnishapi/Makefile.am
@@ -12,8 +12,6 @@ lib_LTLIBRARIES = libvarnishapi.la
 libvarnishapi_la_LDFLAGS = $(AM_LDFLAGS) -version-info 1:6:0
 
 libvarnishapi_la_SOURCES = \
-	vjsn.c \
-	vjsn.h \
 	vsl_api.h \
 	vxp.h \
 	vxp_tokens.h \
@@ -25,6 +23,7 @@ libvarnishapi_la_SOURCES = \
 	../libvarnish/vfil.c \
 	../libvarnish/vfl.c \
 	../libvarnish/vin.c \
+	../libvarnish/vjsn.c \
 	../libvarnish/vmb.c \
 	../libvarnish/vpf.c \
 	../libvarnish/vre.c \
@@ -98,14 +97,10 @@ vxp_test_CFLAGS = \
 vxp_test_LDADD = @PCRE_LIBS@ \
 	${RT_LIBS} ${LIBM} ${PTHREAD_LIBS}
 
-TESTS = vjsn_test vsl_glob_test
+TESTS = vsl_glob_test
 
 noinst_PROGRAMS += ${TESTS}
 
 vsl_glob_test_SOURCES = vsl_glob_test.c
 vsl_glob_test_CFLAGS = @SAN_CFLAGS@
 vsl_glob_test_LDADD = libvarnishapi.la @SAN_LDFLAGS@
-
-vjsn_test_SOURCES = vjsn.c
-vjsn_test_CFLAGS = -DVJSN_TEST @SAN_CFLAGS@
-vjsn_test_LDADD = libvarnishapi.la @SAN_LDFLAGS@
diff --git a/lib/libvarnishapi/vjsn.c b/lib/libvarnishapi/vjsn.c
deleted file mode 100644
index 8f08e74..0000000
--- a/lib/libvarnishapi/vjsn.c
+++ /dev/null
@@ -1,935 +0,0 @@
-/*-
- * Copyright (c) 2017 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include "config.h"
-
-#include <errno.h>
-#include <fcntl.h>
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-#include <unistd.h>
-
-#include "vdef.h"
-
-#include "vas.h"
-#include "miniobj.h"
-#include "vqueue.h"
-#include "vjsn.h"
-
-const char VJSN_OBJECT[] = "object";
-const char VJSN_ARRAY[] = "array";
-const char VJSN_NUMBER[] = "number";
-const char VJSN_STRING[] = "string";
-const char VJSN_TRUE[] = "true";
-const char VJSN_FALSE[] = "false";
-const char VJSN_NULL[] = "null";
-
-#define VJSN_EXPECT(js, xxx, ret)					\
-	do {								\
-		AZ(js->err);						\
-		if (*((js)->ptr) != xxx) {				\
-			js->err = "Expected " #xxx " not found.";	\
-			return (ret);					\
-		} else {						\
-			*js->ptr++ = '\0';				\
-		}							\
-	} while (0)
-
-static struct vjsn_val *vjsn_value(struct vjsn *);
-
-static struct vjsn_val *
-vjsn_val_new(const char *type)
-{
-	struct vjsn_val *jsv;
-
-	ALLOC_OBJ(jsv, VJSN_VAL_MAGIC);
-	AN(jsv);
-	VTAILQ_INIT(&jsv->children);
-	jsv->type = type;
-	return (jsv);
-}
-
-static void
-vjsn_val_delete(struct vjsn_val *jsv)
-{
-	struct vjsn_val *jsve;
-
-	CHECK_OBJ_NOTNULL(jsv, VJSN_VAL_MAGIC);
-	do {
-		jsve = VTAILQ_FIRST(&jsv->children);
-		if (jsve != NULL) {
-			VTAILQ_REMOVE(&jsv->children, jsve, list);
-			vjsn_val_delete(jsve);
-		}
-	} while (jsve != NULL);
-	FREE_OBJ(jsv);
-}
-
-void
-vjsn_delete(struct vjsn **jp)
-{
-	struct vjsn *js;
-
-	AN(jp);
-	js = *jp;
-	*jp = NULL;
-	CHECK_OBJ_NOTNULL(js, VJSN_MAGIC);
-	if (js->value != NULL)
-		vjsn_val_delete(js->value);
-	free(js->raw);
-	FREE_OBJ(js);
-}
-
-static void
-vjsn_skip_ws(struct vjsn *js)
-{
-	char c;
-
-	while (1) {
-		c = js->ptr[0];
-		if (c == 0x09 || c == 0x0a || c == 0x0d || c == 0x20) {
-			*js->ptr++ = '\0';
-			continue;
-		}
-#ifdef VJSN_COMMENTS
-		if (c == '/' && js->ptr[1] == '*') {
-			js->ptr += 2;
-			while (js->ptr[0] != '*' || js->ptr[1] != '/')
-				js->ptr++;
-			js->ptr += 2;
-			continue;
-		}
-#endif
-		return;
-	}
-}
-
-static unsigned
-vjsn_unumber(struct vjsn *js)
-{
-	unsigned u = 0;
-	char c;
-	int i;
-
-	VJSN_EXPECT(js, '\\', 0);
-	VJSN_EXPECT(js, 'u', 0);
-	for (i = 0; i < 4; i++) {
-		u <<= 4;
-		c = *js->ptr;
-		if (c >= '0' && c <= '9')
-			u |= c - '0';			/*lint !e737 */
-		else if (c >= 'A' && c <= 'F')
-			u |= c - '7';			/*lint !e737 */
-		else if (c >= 'a' && c <= 'f')
-			u |= c - 'W';			/*lint !e737 */
-		else {
-			js->err = "Illegal \\uXXXX sequence";
-			return (0);
-		}
-		js->ptr++;
-	}
-	return (u);
-}
-
-static void
-vjsn_unicode(struct vjsn *js, char **d)
-{
-	unsigned u1, u2;
-
-	u1 = vjsn_unumber(js);
-	if (js->err)
-		return;
-
-	if (u1 >= 0xdc00 && u1 <= 0xdfff) {
-		js->err = "Lone second UTF-16 Surrogate";
-		return;
-	}
-	if (u1 >= 0xd800 && u1 <= 0xdc00) {
-		u2 = vjsn_unumber(js);
-		if (u2 < 0xdc00 || u2 > 0xdfff) {
-			js->err = "Bad UTF-16 Surrogate pair";
-			return;
-		}
-		u1 -= 0xd800;
-		u2 -= 0xdc00;
-		u1 <<= 10;
-		u1 |= u2;
-		u1 |= 0x10000;
-	}
-	assert(u1 < 0x110000);
-	/*lint -save -e734 -e713 */
-	if (u1 < 0x80)
-		*(*d)++ = u1;
-	else if (u1 < 0x800) {
-		*(*d)++ = 0xc0 + u1 / 64;
-		*(*d)++ = 0x80 + u1 % 64;
-	} else if (u1 < 0x10000) {
-		*(*d)++ = 0xe0 + u1 / 4096;
-		*(*d)++ = 0x80 + u1 / 64 % 64;
-		*(*d)++ = 0x80 + u1 % 64;
-	} else {
-		*(*d)++ = 0xf0 + u1 / 262144;
-		*(*d)++ = 0x80 + u1 / 4096 % 64;
-		*(*d)++ = 0x80 + u1 / 64 % 64;
-		*(*d)++ = 0x80 + u1 % 64;
-	}
-	/*lint -restore */
-}
-
-static char *
-vjsn_string(struct vjsn *js)
-{
-	char *p, *b;
-
-	vjsn_skip_ws(js);
-	VJSN_EXPECT(js, '"', NULL);
-	b = p = js->ptr;
-	while (*js->ptr != '"') {
-		if (*js->ptr == '\0') {
-			js->err = "Unterminate string";
-			return (NULL);
-		}
-		if ((unsigned char)(*js->ptr) <= 0x1f) {
-			js->err = "unescaped control char in string";
-			return (NULL);
-		}
-		if (*js->ptr != '\\') {
-			*p++ = *js->ptr++;
-			continue;
-		}
-		switch (js->ptr[1]) {
-		case '\\':
-		case '/':
-		case '"': *p++ = js->ptr[1]; js->ptr += 2; break;
-		case 'b': *p++ = 0x08; js->ptr += 2; break;
-		case 'f': *p++ = 0x0c; js->ptr += 2; break;
-		case 't': *p++ = 0x09; js->ptr += 2; break;
-		case 'n': *p++ = 0x0a; js->ptr += 2; break;
-		case 'r': *p++ = 0x0d; js->ptr += 2; break;
-		case 'u':
-			vjsn_unicode(js, &p);
-			if (js->err != NULL)
-				return(NULL);
-			break;
-		default:
-			js->err = "Bad string escape";
-			return (NULL);
-		}
-	}
-	VJSN_EXPECT(js, '"', NULL);
-	*p = '\0';
-	return (b);
-}
-
-static struct vjsn_val *
-vjsn_object(struct vjsn *js)
-{
-	struct vjsn_val *jsv, *jsve;
-	char *s;
-
-	VJSN_EXPECT(js, '{', NULL);
-
-	jsv = vjsn_val_new(VJSN_OBJECT);
-	AN(jsv);
-
-	vjsn_skip_ws(js);
-	if (*js->ptr != '}') {
-		while (1) {
-			s = vjsn_string(js);
-			if (js->err != NULL)
-				return (jsv);
-			vjsn_skip_ws(js);
-			VJSN_EXPECT(js, ':', jsv);
-			jsve = vjsn_value(js);
-			if (js->err != NULL) {
-				if (jsve != NULL)
-					vjsn_val_delete(jsve);
-				return (jsv);
-			}
-			CHECK_OBJ_NOTNULL(jsve, VJSN_VAL_MAGIC);
-			jsve->name = s;
-			VTAILQ_INSERT_TAIL(&jsv->children, jsve, list);
-			vjsn_skip_ws(js);
-			if (*js->ptr == '}')
-				break;
-			VJSN_EXPECT(js, ',', jsv);
-		}
-	}
-	VJSN_EXPECT(js, '}', jsv);
-	return (jsv);
-}
-
-static struct vjsn_val *
-vjsn_array(struct vjsn *js)
-{
-	struct vjsn_val *jsv, *jsve;
-
-	VJSN_EXPECT(js, '[', NULL);
-
-	jsv = vjsn_val_new(VJSN_ARRAY);
-	AN(jsv);
-
-	vjsn_skip_ws(js);
-	if (*js->ptr != ']') {
-		while (1) {
-			jsve = vjsn_value(js);
-			if (js->err != NULL) {
-				if (jsve != NULL)
-					vjsn_val_delete(jsve);
-				return (jsv);
-			}
-			CHECK_OBJ_NOTNULL(jsve, VJSN_VAL_MAGIC);
-			VTAILQ_INSERT_TAIL(&jsv->children, jsve, list);
-			vjsn_skip_ws(js);
-			if (*js->ptr == ']')
-				break;
-			VJSN_EXPECT(js, ',', jsv);
-		}
-	}
-	VJSN_EXPECT(js, ']', jsv);
-	return (jsv);
-}
-
-static struct vjsn_val *
-vjsn_number(struct vjsn *js)
-{
-	struct vjsn_val *jsv;
-
-	jsv = vjsn_val_new(VJSN_NUMBER);
-	AN(jsv);
-
-	jsv->value = js->ptr;
-
-	if (*js->ptr == '-')
-		js->ptr++;
-	if (*js->ptr < '0' || *js->ptr > '9') {
-		js->err = "Bad number";
-		return (jsv);
-	}
-	if (*js->ptr == '0' && js->ptr[1] >= '0' && js->ptr[1] <= '9') {
-		js->err = "Bad number";
-		return (jsv);
-	}
-	while (*js->ptr >= '0' && *js->ptr <= '9')
-		js->ptr++;
-	if (*js->ptr == '.') {
-		js->ptr++;
-		if (*js->ptr < '0' || *js->ptr > '9') {
-			js->err = "Bad number";
-			return (jsv);
-		}
-		while (*js->ptr >= '0' && *js->ptr <= '9')
-			js->ptr++;
-	}
-	if (*js->ptr == 'e' || *js->ptr == 'E') {
-		js->ptr++;
-		if (*js->ptr == '-' || *js->ptr == '+')
-			js->ptr++;
-		if (*js->ptr < '0' || *js->ptr > '9') {
-			js->err = "Bad number";
-			return (jsv);
-		}
-		while (*js->ptr >= '0' && *js->ptr <= '9')
-			js->ptr++;
-	}
-	return (jsv);
-}
-
-static struct vjsn_val *
-vjsn_value(struct vjsn *js)
-{
-	struct vjsn_val *jsv;
-
-	AZ(js->err);
-	vjsn_skip_ws(js);
-	if (*js->ptr == '{')
-		return (vjsn_object(js));
-	if (*js->ptr== '[')
-		return (vjsn_array(js));
-	if (*js->ptr== '"') {
-		jsv = vjsn_val_new(VJSN_STRING);
-		jsv->value = vjsn_string(js);
-		if (js->err != NULL)
-			return (jsv);
-		AN(jsv->value);
-		return (jsv);
-	}
-	if (!strncmp(js->ptr, "true", 4)) {
-		js->ptr += 4;
-		return (vjsn_val_new(VJSN_TRUE));
-	}
-	if (!strncmp(js->ptr, "false", 5)) {
-		js->ptr += 5;
-		return (vjsn_val_new(VJSN_FALSE));
-	}
-	if (!strncmp(js->ptr, "null", 4)) {
-		js->ptr += 4;
-		return (vjsn_val_new(VJSN_NULL));
-	}
-	if (*js->ptr == '-' || (*js->ptr >= '0' && *js->ptr <= '9'))
-		return (vjsn_number(js));
-	js->err = "Unrecognized value";
-	return (NULL);
-}
-
-struct vjsn *
-vjsn_parse(const char *src, const char **err)
-{
-	struct vjsn *js;
-	char *p, *e;
-
-	AN(src);
-
-	AN(err);
-	*err = NULL;
-
-	p = strdup(src);
-	AN(p);
-	e = strchr(p, '\0');
-	AN(e);
-
-	ALLOC_OBJ(js, VJSN_MAGIC);
-	AN(js);
-	js->raw = p;
-	js->ptr = p;
-
-	js->value = vjsn_value(js);
-	if (js->err != NULL) {
-		*err = js->err;
-		vjsn_delete(&js);
-		return (NULL);
-	}
-
-	vjsn_skip_ws(js);
-	if (js->ptr != e) {
-		*err = "Garbage after value";
-		vjsn_delete(&js);
-		return (NULL);
-	}
-	return (js);
-}
-
-struct vjsn_val *
-vjsn_child(const struct vjsn_val *vv, const char *key)
-{
-	struct vjsn_val *vc;
-
-	CHECK_OBJ_NOTNULL(vv, VJSN_VAL_MAGIC);
-	AN(key);
-	VTAILQ_FOREACH(vc, &vv->children, list) {
-		if (vc->name != NULL && !strcmp(vc->name, key))
-			return (vc);
-	}
-	return (NULL);
-}
-
-static void
-vjsn_dump_i(const struct vjsn_val *jsv, FILE *fo, int indent)
-{
-	struct vjsn_val *jsve;
-
-	printf("%*s", indent, "");
-	if (jsv->name != NULL)
-		printf("[\"%s\"]: ", jsv->name);
-	printf("{%s}", jsv->type);
-	if (jsv->value != NULL) {
-		if (strlen(jsv->value) < 20)
-			printf(" <%s", jsv->value);
-		else
-			printf(" <%.10s[...#%zu]",
-			    jsv->value, strlen(jsv->value + 10));
-		printf(">");
-	}
-	printf("\n");
-	VTAILQ_FOREACH(jsve, &jsv->children, list)
-		vjsn_dump_i(jsve, fo, indent + 2);
-}
-
-void
-vjsn_dump(const struct vjsn *js, FILE *fo)
-{
-
-	CHECK_OBJ_NOTNULL(js, VJSN_MAGIC);
-	AN(fo);
-	vjsn_dump_i(js->value, fo, 0);
-}
-
-#ifdef VJSN_TEST
-
-/*
- * Test-cases by Nicolas Seriot
- *
- * See: http://seriot.ch/parsing_json.php
- *
- * MIT License
- *
- * Copyright (c) 2016 Nicolas Seriot
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- *
- * We skip tests containing NUL, because we're lazy (The code will actually
- * pass these tests if you provide for a way to pass the true length of the
- * input to it, and we skip really huge tests, because we are only limited
- * by available memory.
- *
- * To produce the C-data-structures:
- *
- * Clone https://github.com/nst/JSONTestSuite.git
- *
- * And run this python in test_parsing:
-
-	import glob
-
-	def emit(fin):
-		if fin in skip:
-			return
-		x = bytearray(open(fin).read())
-		if 0 in x:
-			return
-		if len(x) > 1000:
-			return
-		t = '\t"'
-		for i in x:
-			t += "\\x%02x" % i
-			if len(t) > 64:
-				print(t + '"')
-				t = '\t"'
-		print(t + '",')
-
-	print("const char *good[] = {")
-	for f in glob.glob("y_*"):
-		emit(f)
-	print("\tNULL")
-	print("};")
-
-	print("const char *bad[] = {")
-	for f in glob.glob("n_*"):
-		emit(f)
-	print("\tNULL")
-	print("};")
-
-*/
-
-static const char *good[] = {
-	"\x5b\x31\x32\x33\x65\x36\x35\x5d",
-	"\x5b\x5b\x5d\x20\x20\x20\x5d",
-	"\x5b\x22\x22\x5d",
-	"\x5b\x5d",
-	"\x5b\x22\x61\x22\x5d",
-	"\x5b\x66\x61\x6c\x73\x65\x5d",
-	"\x5b\x6e\x75\x6c\x6c\x2c\x20\x31\x2c\x20\x22\x31\x22\x2c\x20\x7b"
-	"\x7d\x5d",
-	"\x5b\x6e\x75\x6c\x6c\x5d",
-	"\x5b\x31\x0a\x5d",
-	"\x20\x5b\x31\x5d",
-	"\x5b\x31\x2c\x6e\x75\x6c\x6c\x2c\x6e\x75\x6c\x6c\x2c\x6e\x75\x6c"
-	"\x6c\x2c\x32\x5d",
-	"\x5b\x32\x5d\x20",
-	"\x5b\x30\x65\x2b\x31\x5d",
-	"\x5b\x30\x65\x31\x5d",
-	"\x5b\x20\x34\x5d",
-	"\x5b\x2d\x30\x2e\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30"
-	"\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30"
-	"\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30"
-	"\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30"
-	"\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30"
-	"\x30\x31\x5d\x0a",
-	"\x5b\x32\x30\x65\x31\x5d",
-	"\x5b\x2d\x30\x5d",
-	"\x5b\x2d\x31\x32\x33\x5d",
-	"\x5b\x2d\x31\x5d",
-	"\x5b\x2d\x30\x5d",
-	"\x5b\x31\x45\x32\x32\x5d",
-	"\x5b\x31\x45\x2d\x32\x5d",
-	"\x5b\x31\x45\x2b\x32\x5d",
-	"\x5b\x31\x32\x33\x65\x34\x35\x5d",
-	"\x5b\x31\x32\x33\x2e\x34\x35\x36\x65\x37\x38\x5d",
-	"\x5b\x31\x65\x2d\x32\x5d",
-	"\x5b\x31\x65\x2b\x32\x5d",
-	"\x5b\x31\x32\x33\x5d",
-	"\x5b\x31\x32\x33\x2e\x34\x35\x36\x37\x38\x39\x5d",
-	"\x7b\x22\x61\x73\x64\x22\x3a\x22\x73\x64\x66\x22\x2c\x20\x22\x64"
-	"\x66\x67\x22\x3a\x22\x66\x67\x68\x22\x7d",
-	"\x7b\x22\x61\x73\x64\x22\x3a\x22\x73\x64\x66\x22\x7d",
-	"\x7b\x22\x61\x22\x3a\x22\x62\x22\x2c\x22\x61\x22\x3a\x22\x63\x22"
-	"\x7d",
-	"\x7b\x22\x61\x22\x3a\x22\x62\x22\x2c\x22\x61\x22\x3a\x22\x62\x22"
-	"\x7d",
-	"\x7b\x7d",
-	"\x7b\x22\x22\x3a\x30\x7d",
-	"\x7b\x22\x66\x6f\x6f\x5c\x75\x30\x30\x30\x30\x62\x61\x72\x22\x3a"
-	"\x20\x34\x32\x7d",
-	"\x7b\x20\x22\x6d\x69\x6e\x22\x3a\x20\x2d\x31\x2e\x30\x65\x2b\x32"
-	"\x38\x2c\x20\x22\x6d\x61\x78\x22\x3a\x20\x31\x2e\x30\x65\x2b\x32"
-	"\x38\x20\x7d",
-	"\x7b\x22\x78\x22\x3a\x5b\x7b\x22\x69\x64\x22\x3a\x20\x22\x78\x78"
-	"\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78"
-	"\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78"
-	"\x78\x78\x78\x78\x78\x78\x22\x7d\x5d\x2c\x20\x22\x69\x64\x22\x3a"
-	"\x20\x22\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78"
-	"\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78"
-	"\x78\x78\x78\x78\x78\x78\x78\x78\x78\x78\x22\x7d",
-	"\x7b\x22\x61\x22\x3a\x5b\x5d\x7d",
-	"\x7b\x22\x74\x69\x74\x6c\x65\x22\x3a\x22\x5c\x75\x30\x34\x31\x66"
-	"\x5c\x75\x30\x34\x33\x65\x5c\x75\x30\x34\x33\x62\x5c\x75\x30\x34"
-	"\x34\x32\x5c\x75\x30\x34\x33\x65\x5c\x75\x30\x34\x34\x30\x5c\x75"
-	"\x30\x34\x33\x30\x20\x5c\x75\x30\x34\x31\x37\x5c\x75\x30\x34\x33"
-	"\x35\x5c\x75\x30\x34\x33\x63\x5c\x75\x30\x34\x33\x62\x5c\x75\x30"
-	"\x34\x33\x35\x5c\x75\x30\x34\x33\x61\x5c\x75\x30\x34\x33\x65\x5c"
-	"\x75\x30\x34\x33\x66\x5c\x75\x30\x34\x33\x30\x22\x20\x7d",
-	"\x7b\x0a\x22\x61\x22\x3a\x20\x22\x62\x22\x0a\x7d",
-	"\x5b\x22\x5c\x75\x44\x38\x30\x31\x5c\x75\x64\x63\x33\x37\x22\x5d"
-	"",
-	"\x5b\x22\x5c\x75\x30\x30\x36\x30\x5c\x75\x30\x31\x32\x61\x5c\x75"
-	"\x31\x32\x41\x42\x22\x5d",
-	"\x5b\x22\x5c\x75\x64\x38\x33\x64\x5c\x75\x64\x65\x33\x39\x5c\x75"
-	"\x64\x38\x33\x64\x5c\x75\x64\x63\x38\x64\x22\x5d",
-	"\x5b\x22\x5c\x22\x5c\x5c\x5c\x2f\x5c\x62\x5c\x66\x5c\x6e\x5c\x72"
-	"\x5c\x74\x22\x5d",
-	"\x5b\x22\x5c\x5c\x75\x30\x30\x30\x30\x22\x5d",
-	"\x5b\x22\x5c\x22\x22\x5d",
-	"\x5b\x22\x61\x2f\x2a\x62\x2a\x2f\x63\x2f\x2a\x64\x2f\x2f\x65\x22"
-	"\x5d",
-	"\x5b\x22\x5c\x5c\x61\x22\x5d",
-	"\x5b\x22\x5c\x5c\x6e\x22\x5d",
-	"\x5b\x22\x5c\x75\x30\x30\x31\x32\x22\x5d",
-	"\x5b\x22\x5c\x75\x46\x46\x46\x46\x22\x5d",
-	"\x5b\x22\x61\x73\x64\x22\x5d",
-	"\x5b\x22\x6e\x65\x77\x5c\x75\x30\x30\x41\x30\x6c\x69\x6e\x65\x22"
-	"\x5d",
-	"\x5b\x20\x22\x61\x73\x64\x22\x5d",
-	"\x5b\x22\x5c\x75\x44\x42\x46\x46\x5c\x75\x44\x46\x46\x46\x22\x5d"
-	"",
-	"\x5b\x22\xf4\x8f\xbf\xbf\x22\x5d",
-	"\x5b\x22\xf0\x9b\xbf\xbf\x22\x5d",
-	"\x5b\x22\xef\xbf\xbf\x22\x5d",
-	"\x5b\x22\x5c\x75\x30\x30\x30\x30\x22\x5d",
-	"\x5b\x22\x5c\x75\x30\x30\x32\x63\x22\x5d",
-	"\x5b\x22\xcf\x80\x22\x5d",
-	"\x5b\x22\x61\x73\x64\x20\x22\x5d",
-	"\x22\x20\x22",
-	"\x5b\x22\x5c\x75\x44\x38\x33\x34\x5c\x75\x44\x64\x31\x65\x22\x5d"
-	"",
-	"\x5b\x22\x5c\x75\x30\x38\x32\x31\x22\x5d",
-	"\x5b\x22\x5c\x75\x30\x31\x32\x33\x22\x5d",
-	"\x5b\x22\xe2\x80\xa8\x22\x5d",
-	"\x5b\x22\xe2\x80\xa9\x22\x5d",
-	"\x5b\x22\x5c\x75\x30\x30\x36\x31\x5c\x75\x33\x30\x61\x66\x5c\x75"
-	"\x33\x30\x45\x41\x5c\x75\x33\x30\x62\x39\x22\x5d",
-	"\x5b\x22\x6e\x65\x77\x5c\x75\x30\x30\x30\x41\x6c\x69\x6e\x65\x22"
-	"\x5d",
-	"\x5b\x22\x7f\x22\x5d",
-	"\x5b\x22\x5c\x75\x41\x36\x36\x44\x22\x5d",
-	"\x5b\x22\x5c\x75\x30\x30\x35\x43\x22\x5d",
-	"\x5b\x22\xe2\x8d\x82\xe3\x88\xb4\xe2\x8d\x82\x22\x5d",
-	"\x5b\x22\x5c\x75\x44\x42\x46\x46\x5c\x75\x44\x46\x46\x45\x22\x5d"
-	"",
-	"\x5b\x22\x5c\x75\x44\x38\x33\x46\x5c\x75\x44\x46\x46\x45\x22\x5d"
-	"",
-	"\x5b\x22\xe2\x82\xac\xf0\x9d\x84\x9e\x22\x5d",
-	"\x5b\x22\x5c\x75\x32\x30\x30\x42\x22\x5d",
-	"\x5b\x22\x5c\x75\x32\x30\x36\x34\x22\x5d",
-	"\x5b\x22\x5c\x75\x46\x44\x44\x30\x22\x5d",
-	"\x5b\x22\x5c\x75\x46\x46\x46\x45\x22\x5d",
-	"\x5b\x22\x5c\x75\x30\x30\x32\x32\x22\x5d",
-	"\x5b\x22\x61\x7f\x61\x22\x5d",
-	"\x66\x61\x6c\x73\x65",
-	"\x34\x32",
-	"\x2d\x30\x2e\x31",
-	"\x6e\x75\x6c\x6c",
-	"\x22\x61\x73\x64\x22",
-	"\x74\x72\x75\x65",
-	"\x22\x22",
-	"\x5b\x22\x61\x22\x5d\x0a",
-	"\x5b\x74\x72\x75\x65\x5d",
-	"\x20\x5b\x5d\x20",
-	NULL
-};
-static const char *bad[] = {
-	"\x5b\x31\x20\x74\x72\x75\x65\x5d",
-	"\x5b\x61\xe5\x5d",
-	"\x5b\x22\x22\x3a\x20\x31\x5d",
-	"\x5b\x22\x22\x5d\x2c",
-	"\x5b\x2c\x31\x5d",
-	"\x5b\x31\x2c\x2c\x32\x5d",
-	"\x5b\x22\x78\x22\x2c\x2c\x5d",
-	"\x5b\x22\x78\x22\x5d\x5d",
-	"\x5b\x22\x22\x2c\x5d",
-	"\x5b\x22\x78\x22",
-	"\x5b\xff\x5d",
-	"\x5b\x78",
-	"\x5b\x33\x5b\x34\x5d\x5d",
-	"\x5b\x31\x3a\x32\x5d",
-	"\x5b\x2c\x5d",
-	"\x5b\x2d\x5d",
-	"\x5b\x20\x20\x20\x2c\x20\x22\x22\x5d",
-	"\x5b\x22\x61\x22\x2c\x0a\x34\x0a\x2c\x31\x2c",
-	"\x5b\x31\x2c\x5d",
-	"\x5b\x31\x2c\x2c\x5d",
-	"\x5b\x22\x0b\x61\x22\x5c\x66\x5d",
-	"\x5b\x2a\x5d",
-	"\x5b\x22\x22",
-	"\x5b\x31\x2c",
-	"\x5b\x31\x2c\x0a\x31\x0a\x2c\x31",
-	"\x5b\x7b\x7d",
-	"\x5b\x66\x61\x6c\x73\x5d",
-	"\x5b\x6e\x75\x6c\x5d",
-	"\x5b\x74\x72\x75\x5d",
-	"\x5b\x2b\x2b\x31\x32\x33\x34\x5d",
-	"\x5b\x2b\x31\x5d",
-	"\x5b\x2b\x49\x6e\x66\x5d",
-	"\x5b\x2d\x30\x31\x5d",
-	"\x5b\x2d\x31\x2e\x30\x2e\x5d",
-	"\x5b\x2d\x32\x2e\x5d",
-	"\x5b\x2d\x4e\x61\x4e\x5d",
-	"\x5b\x2e\x2d\x31\x5d",
-	"\x5b\x2e\x32\x65\x2d\x33\x5d",
-	"\x5b\x30\x2e\x31\x2e\x32\x5d",
-	"\x5b\x30\x2e\x33\x65\x2b\x5d",
-	"\x5b\x30\x2e\x33\x65\x5d",
-	"\x5b\x30\x2e\x65\x31\x5d",
-	"\x5b\x30\x45\x2b\x5d",
-	"\x5b\x30\x45\x5d",
-	"\x5b\x30\x65\x2b\x5d",
-	"\x5b\x30\x65\x5d",
-	"\x5b\x31\x2e\x30\x65\x2b\x5d",
-	"\x5b\x31\x2e\x30\x65\x2d\x5d",
-	"\x5b\x31\x2e\x30\x65\x5d",
-	"\x5b\x31\x20\x30\x30\x30\x2e\x30\x5d",
-	"\x5b\x31\x65\x45\x32\x5d",
-	"\x5b\x32\x2e\x65\x2b\x33\x5d",
-	"\x5b\x32\x2e\x65\x2d\x33\x5d",
-	"\x5b\x32\x2e\x65\x33\x5d",
-	"\x5b\x39\x2e\x65\x2b\x5d",
-	"\x5b\x49\x6e\x66\x5d",
-	"\x5b\x4e\x61\x4e\x5d",
-	"\x5b\xef\xbc\x91\x5d",
-	"\x5b\x31\x2b\x32\x5d",
-	"\x5b\x30\x78\x31\x5d",
-	"\x5b\x30\x78\x34\x32\x5d",
-	"\x5b\x49\x6e\x66\x69\x6e\x69\x74\x79\x5d",
-	"\x5b\x30\x65\x2b\x2d\x31\x5d",
-	"\x5b\x2d\x31\x32\x33\x2e\x31\x32\x33\x66\x6f\x6f\x5d",
-	"\x5b\x31\x32\x33\xe5\x5d",
-	"\x5b\x31\x65\x31\xe5\x5d",
-	"\x5b\x30\xe5\x5d\x0a",
-	"\x5b\x2d\x49\x6e\x66\x69\x6e\x69\x74\x79\x5d",
-	"\x5b\x2d\x66\x6f\x6f\x5d",
-	"\x5b\x2d\x20\x31\x5d",
-	"\x5b\x2d\x30\x31\x32\x5d",
-	"\x5b\x2d\x2e\x31\x32\x33\x5d",
-	"\x5b\x2d\x31\x78\x5d",
-	"\x5b\x31\x65\x61\x5d",
-	"\x5b\x31\x65\xe5\x5d",
-	"\x5b\x31\x2e\x5d",
-	"\x5b\x2e\x31\x32\x33\x5d",
-	"\x5b\x31\x2e\x32\x61\x2d\x33\x5d",
-	"\x5b\x31\x2e\x38\x30\x31\x31\x36\x37\x30\x30\x33\x33\x33\x37\x36"
-	"\x35\x31\x34\x48\x2d\x33\x30\x38\x5d",
-	"\x5b\x30\x31\x32\x5d",
-	"\x5b\x22\x78\x22\x2c\x20\x74\x72\x75\x74\x68\x5d",
-	"\x7b\x5b\x3a\x20\x22\x78\x22\x7d\x0a",
-	"\x7b\x22\x78\x22\x2c\x20\x6e\x75\x6c\x6c\x7d",
-	"\x7b\x22\x78\x22\x3a\x3a\x22\x62\x22\x7d",
-	"\x7b\xf0\x9f\x87\xa8\xf0\x9f\x87\xad\x7d",
-	"\x7b\x22\x61\x22\x3a\x22\x61\x22\x20\x31\x32\x33\x7d",
-	"\x7b\x6b\x65\x79\x3a\x20\x27\x76\x61\x6c\x75\x65\x27\x7d",
-	"\x7b\x22\x61\x22\x20\x62\x7d",
-	"\x7b\x3a\x22\x62\x22\x7d",
-	"\x7b\x22\x61\x22\x20\x22\x62\x22\x7d",
-	"\x7b\x22\x61\x22\x3a",
-	"\x7b\x22\x61\x22",
-	"\x7b\x31\x3a\x31\x7d",
-	"\x7b\x39\x39\x39\x39\x45\x39\x39\x39\x39\x3a\x31\x7d",
-	"\x7b\x22\xb9\x22\x3a\x22\x30\x22\x2c\x7d",
-	"\x7b\x6e\x75\x6c\x6c\x3a\x6e\x75\x6c\x6c\x2c\x6e\x75\x6c\x6c\x3a"
-	"\x6e\x75\x6c\x6c\x7d",
-	"\x7b\x22\x69\x64\x22\x3a\x30\x2c\x2c\x2c\x2c\x2c\x7d",
-	"\x7b\x27\x61\x27\x3a\x30\x7d",
-	"\x7b\x22\x69\x64\x22\x3a\x30\x2c\x7d",
-	"\x7b\x22\x61\x22\x3a\x22\x62\x22\x7d\x2f\x2a\x2a\x2f",
-	"\x7b\x22\x61\x22\x3a\x22\x62\x22\x7d\x2f\x2a\x2a\x2f\x2f",
-	"\x7b\x22\x61\x22\x3a\x22\x62\x22\x7d\x2f\x2f",
-	"\x7b\x22\x61\x22\x3a\x22\x62\x22\x7d\x2f",
-	"\x7b\x22\x61\x22\x3a\x22\x62\x22\x2c\x2c\x22\x63\x22\x3a\x22\x64"
-	"\x22\x7d",
-	"\x7b\x61\x3a\x20\x22\x62\x22\x7d",
-	"\x7b\x22\x61\x22\x3a\x22\x61",
-	"\x7b\x20\x22\x66\x6f\x6f\x22\x20\x3a\x20\x22\x62\x61\x72\x22\x2c"
-	"\x20\x22\x61\x22\x20\x7d",
-	"\x7b\x22\x61\x22\x3a\x22\x62\x22\x7d\x23",
-	"\x20",
-	"\x5b\x22\x5c\x75\x44\x38\x30\x30\x5c\x22\x5d",
-	"\x5b\x22\x5c\x75\x44\x38\x30\x30\x5c\x75\x22\x5d",
-	"\x5b\x22\x5c\x75\x44\x38\x30\x30\x5c\x75\x31\x22\x5d",
-	"\x5b\x22\x5c\x75\x44\x38\x30\x30\x5c\x75\x31\x78\x22\x5d",
-	"\x5b\xc3\xa9\x5d",
-	"\x5b\x22\x5c\x78\x30\x30\x22\x5d",
-	"\x5b\x22\x5c\x5c\x5c\x22\x5d",
-	"\x5b\x22\x5c\x09\x22\x5d",
-	"\x5b\x22\x5c\xf0\x9f\x8c\x80\x22\x5d",
-	"\x5b\x22\x5c\x22\x5d",
-	"\x5b\x22\x5c\x75\x30\x30\x41\x22\x5d",
-	"\x5b\x22\x5c\x75\x44\x38\x33\x34\x5c\x75\x44\x64\x22\x5d",
-	"\x5b\x22\x5c\x75\x44\x38\x30\x30\x5c\x75\x44\x38\x30\x30\x5c\x78"
-	"\x22\x5d",
-	"\x5b\x22\x5c\x75\xe5\x22\x5d",
-	"\x5b\x22\x5c\x61\x22\x5d",
-	"\x5b\x22\x5c\x75\x71\x71\x71\x71\x22\x5d",
-	"\x5b\x22\x5c\xe5\x22\x5d",
-	"\x5b\x5c\x75\x30\x30\x32\x30\x22\x61\x73\x64\x22\x5d",
-	"\x5b\x5c\x6e\x5d",
-	"\x22",
-	"\x5b\x27\x73\x69\x6e\x67\x6c\x65\x20\x71\x75\x6f\x74\x65\x27\x5d"
-	"",
-	"\x61\x62\x63",
-	"\x5b\x22\x5c",
-	"\x5b\x22\x6e\x65\x77\x0a\x6c\x69\x6e\x65\x22\x5d",
-	"\x5b\x22\x09\x22\x5d",
-	"\x22\x5c\x55\x41\x36\x36\x44\x22",
-	"\x22\x22\x78",
-	"\x5b\xe2\x81\xa0\x5d",
-	"\xef\xbb\xbf",
-	"\x3c\x2e\x3e",
-	"\x5b\x3c\x6e\x75\x6c\x6c\x3e\x5d",
-	"\x5b\x31\x5d\x78",
-	"\x5b\x31\x5d\x5d",
-	"\x5b\x22\x61\x73\x64\x5d",
-	"\x61\xc3\xa5",
-	"\x5b\x54\x72\x75\x65\x5d",
-	"\x31\x5d",
-	"\x7b\x22\x78\x22\x3a\x20\x74\x72\x75\x65\x2c",
-	"\x5b\x5d\x5b\x5d",
-	"\x5d",
-	"\xef\xbb\x7b\x7d",
-	"\xe5",
-	"\x5b",
-	"",
-	"\x32\x40",
-	"\x7b\x7d\x7d",
-	"\x7b\x22\x22\x3a",
-	"\x7b\x22\x61\x22\x3a\x2f\x2a\x63\x6f\x6d\x6d\x65\x6e\x74\x2a\x2f"
-	"\x22\x62\x22\x7d",
-	"\x7b\x22\x61\x22\x3a\x20\x74\x72\x75\x65\x7d\x20\x22\x78\x22",
-	"\x5b\x27",
-	"\x5b\x2c",
-	"\x5b\x7b",
-	"\x5b\x22\x61",
-	"\x5b\x22\x61\x22",
-	"\x7b",
-	"\x7b\x5d",
-	"\x7b\x2c",
-	"\x7b\x5b",
-	"\x7b\x22\x61",
-	"\x7b\x27\x61\x27",
-	"\x5b\x22\x5c\x7b\x5b\x22\x5c\x7b\x5b\x22\x5c\x7b\x5b\x22\x5c\x7b"
-	"",
-	"\xe9",
-	"\x2a",
-	"\x7b\x22\x61\x22\x3a\x22\x62\x22\x7d\x23\x7b\x7d",
-	"\x5b\x5c\x75\x30\x30\x30\x41\x22\x22\x5d",
-	"\x5b\x31",
-	"\x5b\x20\x66\x61\x6c\x73\x65\x2c\x20\x6e\x75\x6c",
-	"\x5b\x20\x74\x72\x75\x65\x2c\x20\x66\x61\x6c\x73",
-	"\x5b\x20\x66\x61\x6c\x73\x65\x2c\x20\x74\x72\x75",
-	"\x7b\x22\x61\x73\x64\x22\x3a\x22\x61\x73\x64\x22",
-	"\xc3\xa5",
-	"\x5b\xe2\x81\xa0\x5d",
-	"\x5b\x0c\x5d",
-	NULL
-};
-
-static void
-test_good(const char *j)
-{
-	struct vjsn *js;
-	const char *err;
-
-	js = vjsn_parse(j, &err);
-	if (js == NULL || err != NULL) {
-		fprintf(stderr, "Parse error: %s\n%s\n", err, j);
-		exit(1);
-	}
-	printf("GOOD: %s\n", j);
-	vjsn_dump(js, stdout);
-	vjsn_delete(&js);
-}
-
-static void
-test_bad(const char *j)
-{
-	struct vjsn *js;
-	const char *err;
-
-	js = vjsn_parse(j, &err);
-	if (js != NULL || err == NULL) {
-		fprintf(stderr, "Parse succeeded %s\n", j);
-		exit(1);
-	}
-	printf("BAD: %s %s\n", err, j);
-}
-
-int
-main(int argc, char **argv)
-{
-	const char **s;
-
-	(void)argc;
-	(void)argv;
-	for (s = good; *s != NULL; s++)
-		test_good(*s);
-	for (s = bad; *s != NULL; s++)
-		test_bad(*s);
-
-	/*
-	 * This is part of Nicolas i(ndeterminate) test set, for reasons I
-	 * do not fully grasp, but we want it to test bad.
-	 */
-	test_bad("\"\\uDFAA\"");
-	printf("Tests done\n");
-	return (0);
-}
-
-#endif
diff --git a/lib/libvarnishapi/vjsn.h b/lib/libvarnishapi/vjsn.h
deleted file mode 100644
index 3b2f16b..0000000
--- a/lib/libvarnishapi/vjsn.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*-
- * Copyright (c) 2017 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-extern const char VJSN_OBJECT[];
-extern const char VJSN_ARRAY[];
-extern const char VJSN_NUMBER[];
-extern const char VJSN_STRING[];
-extern const char VJSN_TRUE[];
-extern const char VJSN_FALSE[];
-extern const char VJSN_NULL[];
-
-struct vjsn_val {
-	unsigned		magic;
-#define VJSN_VAL_MAGIC		0x08a06b80
-	const char		*type;
-	const char		*name;
-	VTAILQ_ENTRY(vjsn_val)	list;
-	VTAILQ_HEAD(,vjsn_val)	children;
-	char			*value;
-};
-
-struct vjsn {
-	unsigned		magic;
-#define VJSN_MAGIC		0x86a7f02b
-
-	char			*raw;
-	char			*ptr;
-	struct vjsn_val		*value;
-	const char		*err;
-};
-
-struct vjsn *vjsn_parse(const char *, const char **);
-void vjsn_delete(struct vjsn **);
-void vjsn_dump(const struct vjsn *js, FILE *fo);
-struct vjsn_val *vjsn_child(const struct vjsn_val *, const char *);

From phk at FreeBSD.org  Wed Feb 21 17:01:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 21 Feb 2018 17:01:08 +0000 (UTC)
Subject: [master] 52c7781 Polish
Message-ID: <20180221170108.C662AAECC1@lists.varnish-cache.org>


commit 52c778132e1a94087079eabaf0236b989c2c1c9c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Feb 21 15:42:07 2018 +0000

    Polish

diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py
index 76c9f25..6777032 100644
--- a/lib/libvcc/vsctool.py
+++ b/lib/libvcc/vsctool.py
@@ -123,7 +123,7 @@ class vscset(object):
 			fo.write(t[:-1])
 		fo.write("\n};\n")
 		s = json.dumps(dd, indent=2, separators=(',', ': '))
-		fo.write("\n// ")
+		fo.write("\n")
 		for i in s.split("\n"):
 			j = "// " + i
 			if len(j) > 72:

From phk at FreeBSD.org  Wed Feb 21 17:01:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 21 Feb 2018 17:01:08 +0000 (UTC)
Subject: [master] f96fcf0 Change library order,
 vcc is allowed to depend on libvarnish
Message-ID: <20180221170108.E0C0CAECC5@lists.varnish-cache.org>


commit f96fcf0690f7566780b3aeb7f353c6e265d34ca6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Feb 21 15:45:29 2018 +0000

    Change library order, vcc is allowed to depend on libvarnish

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 3cf7519..a121c8e 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -157,8 +157,8 @@ varnishd_CFLAGS = \
 varnishd_LDFLAGS = -export-dynamic
 
 varnishd_LDADD = \
-	$(top_builddir)/lib/libvarnish/libvarnish.a \
 	$(top_builddir)/lib/libvcc/libvcc.a \
+	$(top_builddir)/lib/libvarnish/libvarnish.a \
 	$(top_builddir)/lib/libvgz/libvgz.a \
 	@SAN_LDFLAGS@ \
 	@JEMALLOC_LDADD@ \

From phk at FreeBSD.org  Wed Feb 21 17:01:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 21 Feb 2018 17:01:09 +0000 (UTC)
Subject: [master] 3f3ff59 Change the specification format in the compiled VMOD
 from our own home-rolled stuff to VJSN in anticipation of more complex
 specifications in the future.
Message-ID: <20180221170109.06FC2AECC9@lists.varnish-cache.org>


commit 3f3ff599eeaf1c421d90605d6f40ce9d22e49390
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Feb 21 16:58:49 2018 +0000

    Change the specification format in the compiled VMOD from our own
    home-rolled stuff to VJSN in anticipation of more complex
    specifications in the future.

diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c
index 658e3ba..4e11ce5 100644
--- a/bin/varnishd/cache/cache_vrt_vmod.c
+++ b/bin/varnishd/cache/cache_vrt_vmod.c
@@ -128,7 +128,7 @@ VRT_Vmod_Init(VRT_CTX, struct vmod **hdl, void *ptr, int len, const char *nm,
 		    d->func == NULL ||
 		    d->func_len <= 0 ||
 		    d->proto == NULL ||
-		    d->spec == NULL) {
+		    d->json == NULL) {
 			VSB_printf(ctx->msg,
 			    "Loading VMOD %s from %s:\n", nm, path);
 			VSB_printf(ctx->msg, "VMOD data is mangled.\n");
diff --git a/include/vrt.h b/include/vrt.h
index c2dab1e..4822b9f 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -209,7 +209,7 @@ struct vmod_data {
 	const void			*func;
 	int				func_len;
 	const char			*proto;
-	const char			* const *spec;
+	const char			*json;
 	const char			*abi;
 };
 
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 7ae776c..696aa6a 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -71,6 +71,7 @@ struct acl_e;
 struct proc;
 struct expr;
 struct vcc;
+struct vjsn_val;
 struct symbol;
 
 struct source {
@@ -308,8 +309,8 @@ void vcc_Expr_Init(struct vcc *tl);
 sym_expr_t vcc_Eval_Var;
 sym_expr_t vcc_Eval_Handle;
 sym_expr_t vcc_Eval_SymFunc;
-void vcc_Eval_Func(struct vcc *tl, const char *spec,
-    const char *extra, const struct symbol *sym);
+void vcc_Eval_Func(struct vcc *, const struct vjsn_val *,
+    const char *, const struct symbol *);
 void VCC_GlobalSymbol(struct symbol *, vcc_type_t fmt, const char *pfx);
 struct symbol *VCC_HandleSymbol(struct vcc *, vcc_type_t , const char *);
 
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index f8e5d90..2191d4b 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -36,6 +36,7 @@
 #include <string.h>
 
 #include "vcc_compile.h"
+#include "vjsn.h"
 
 struct expr {
 	unsigned	magic;
@@ -400,7 +401,7 @@ vcc_priv_arg(struct vcc *tl, const char *p, const char *name, const char *vmod)
 
 struct func_arg {
 	vcc_type_t		type;
-	const char		*enum_bits;
+	const struct vjsn_val	*enums;
 	const char		*cname;
 	const char		*name;
 	const char		*val;
@@ -423,25 +424,20 @@ vcc_do_enum(struct vcc *tl, struct func_arg *fa, int len, const char *ptr)
 static void
 vcc_do_arg(struct vcc *tl, struct func_arg *fa)
 {
-	const char *p, *r;
 	struct expr *e2;
+	struct vjsn_val *vv;
 
 	if (fa->type == ENUM) {
 		ExpectErr(tl, ID);
 		ERRCHK(tl);
-		r = p = fa->enum_bits;
-		do {
-			if (vcc_IdIs(tl->t, p))
+		VTAILQ_FOREACH(vv, &fa->enums->children, list)
+			if (vcc_IdIs(tl->t, vv->value))
 				break;
-			p += strlen(p) + 1;
-		} while (*p != '\1');
-		if (*p == '\1') {
+		if (vv == NULL) {
 			VSB_printf(tl->sb, "Wrong enum value.");
 			VSB_printf(tl->sb, "  Expected one of:\n");
-			do {
-				VSB_printf(tl->sb, "\t%s\n", r);
-				r += strlen(r) + 1;
-			} while (*r != '\0' && *r != '\1');
+			VTAILQ_FOREACH(vv, &fa->enums->children, list)
+				VSB_printf(tl->sb, "\t%s\n", vv->value);
 			vcc_ErrWhere(tl, tl->t);
 			return;
 		}
@@ -456,60 +452,59 @@ vcc_do_arg(struct vcc *tl, struct func_arg *fa)
 }
 
 static void
-vcc_func(struct vcc *tl, struct expr **e, const char *spec,
+vcc_func(struct vcc *tl, struct expr **e, const void *priv,
     const char *extra, const struct symbol *sym)
 {
 	vcc_type_t rfmt;
-	const char *args;
 	const char *cfunc;
-	const char *p;
 	struct expr *e1;
 	struct func_arg *fa, *fa2;
 	VTAILQ_HEAD(,func_arg) head;
 	struct token *t1;
+	const struct vjsn_val *vv, *vvp;
 
-	rfmt = VCC_Type(spec);
-	spec += strlen(spec) + 1;
-	cfunc = spec;
-	spec += strlen(spec) + 1;
-	args = spec;
+	CAST_OBJ_NOTNULL(vv, priv, VJSN_VAL_MAGIC);
+	assert(vv->type == VJSN_ARRAY);
+	vv = VTAILQ_FIRST(&vv->children);
+	rfmt = VCC_Type(VTAILQ_FIRST(&vv->children)->value);
+	AN(rfmt);
+	vv = VTAILQ_NEXT(vv, list);
+	cfunc = vv->value;
+	vv = VTAILQ_NEXT(vv, list);
 	SkipToken(tl, '(');
-	p = args;
 	if (extra == NULL)
 		extra = "";
-	AN(rfmt);
 	VTAILQ_INIT(&head);
-	while (*p != '\0') {
+	for(;vv != NULL; vv = VTAILQ_NEXT(vv, list)) {
+		assert(vv->type == VJSN_ARRAY);
 		fa = calloc(1, sizeof *fa);
 		AN(fa);
 		fa->cname = cfunc;
 		VTAILQ_INSERT_TAIL(&head, fa, list);
-		if (!memcmp(p, "PRIV_", 5)) {
-			fa->result = vcc_priv_arg(tl, p, sym->name, sym->vmod);
+
+		vvp = VTAILQ_FIRST(&vv->children);
+		if (!memcmp(vvp->value, "PRIV_", 5)) {
+			fa->result = vcc_priv_arg(tl, vvp->value,
+			    sym->name, sym->vmod);
 			fa->name = "";
-			p += strlen(p) + 1;
 			continue;
 		}
-		fa->type = VCC_Type(p);
+		fa->type = VCC_Type(vvp->value);
 		AN(fa->type);
-		p += strlen(p) + 1;
-		if (*p == '\1') {
-			fa->enum_bits = ++p;
-			while (*p != '\1')
-				p += strlen(p) + 1;
-			p++;
-			assert(*p == '\0');
-			p++;
-		}
-		if (*p == '\2') {
-			fa->name = p + 1;
-			p += strlen(p) + 1;
-		}
-		if (*p == '\3') {
-			fa->val = p + 1;
-			p += strlen(p) + 1;
+		vvp = VTAILQ_NEXT(vvp, list);
+		if (vvp != NULL) {
+			fa->name = vvp->value;
+			vvp = VTAILQ_NEXT(vvp, list);
+			if (vvp != NULL) {
+				fa->val = vvp->value;
+				vvp = VTAILQ_NEXT(vvp, list);
+				if (vvp != NULL) {
+					fa->enums = vvp;
+					vvp = VTAILQ_NEXT(vvp, list);
+				}
+			}
 		}
-		assert(*p == 0 || *p > ' ');
+		AZ(vvp);
 	}
 
 	VTAILQ_FOREACH(fa, &head, list) {
@@ -576,11 +571,12 @@ vcc_func(struct vcc *tl, struct expr **e, const char *spec,
 	SkipToken(tl, ')');
 }
 
+
 /*--------------------------------------------------------------------
  */
 
 void
-vcc_Eval_Func(struct vcc *tl, const char *spec,
+vcc_Eval_Func(struct vcc *tl, const struct vjsn_val *spec,
     const char *extra, const struct symbol *sym)
 {
 	struct expr *e = NULL;
@@ -605,7 +601,6 @@ vcc_Eval_SymFunc(struct vcc *tl, struct expr **e, struct token *t,
 	assert(sym->kind == SYM_FUNC);
 	AN(sym->eval_priv);
 
-	// assert(sym->fmt == VCC_Type(sym->eval_priv));
 	vcc_func(tl, e, sym->eval_priv, sym->extra, sym);
 	ERRCHK(tl);
 	if ((*e)->fmt == STRING) {
@@ -1261,7 +1256,7 @@ vcc_Act_Call(struct vcc *tl, struct token *t, struct symbol *sym)
 	struct expr *e;
 
 	e = NULL;
-	vcc_Eval_SymFunc(tl, &e, t, sym, VOID);
+	vcc_func(tl, &e, sym->eval_priv, sym->extra, sym);
 	if (!tl->err) {
 		vcc_expr_fmt(tl->fb, tl->indent, e);
 		SkipToken(tl, ';');
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 6b6969d..fe9253d 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -36,6 +36,7 @@
 
 #include "libvcc.h"
 #include "vfil.h"
+#include "vjsn.h"
 #include "vmod_abi.h"
 
 static int
@@ -54,18 +55,97 @@ vcc_path_dlopen(void *priv, const char *fn)
 	return (0);
 }
 
+static void
+func_sym(struct symbol *sym, const char *vmod, const struct vjsn_val *v)
+{
+
+	assert(v->type == VJSN_ARRAY);
+	sym->action = vcc_Act_Call;
+	sym->vmod = vmod;
+	sym->eval = vcc_Eval_SymFunc;
+	sym->eval_priv = v;
+	v = VTAILQ_FIRST(&v->children);
+	assert(v->type == VJSN_ARRAY);
+	v = VTAILQ_FIRST(&v->children);
+	assert(v->type == VJSN_STRING);
+	sym->type = VCC_Type(v->value);
+	AN(sym->type);
+}
+
+static void
+parse_json(struct vcc * tl, const char *vmod, const char *json)
+{
+	struct inifin *ifp;
+	struct vjsn *vj;
+	struct vjsn_val *vv, *vv2;
+	double vmod_syntax = 0.0;
+	const char *p;
+	struct symbol *sym;
+
+	ifp = NULL;
+
+	vj = vjsn_parse(json, &p);
+	XXXAZ(p);
+	AN(vj);
+
+	VTAILQ_FOREACH(vv, &vj->value->children, list) {
+		assert(vv->type == VJSN_ARRAY);
+		vv2 = VTAILQ_FIRST(&vv->children);
+		assert(vv2->type == VJSN_STRING);
+		if (!strcmp(vv2->value, "$VMOD")) {
+			vmod_syntax =
+			    strtod(VTAILQ_NEXT(vv2, list)->value, NULL);
+			continue;
+		}
+		assert (vmod_syntax == 1.0);
+		if (!strcmp(vv2->value, "$EVENT")) {
+			/* XXX: What about the rest of the events ? */
+			if (ifp == NULL)
+				ifp = New_IniFin(tl);
+			vv2 = VTAILQ_NEXT(vv2, list);
+			VSB_printf(ifp->ini,
+			    "\tif (%s(ctx, &vmod_priv_%s, VCL_EVENT_LOAD))\n"
+			    "\t\treturn(1);",
+			    vv2->value, vmod);
+			VSB_printf(ifp->fin,
+			    "\t\t(void)%s(ctx, &vmod_priv_%s,\n"
+			    "\t\t\t    VCL_EVENT_DISCARD);\n",
+			    vv2->value, vmod);
+			VSB_printf(ifp->event, "%s(ctx, &vmod_priv_%s, ev)",
+			    vv2->value, vmod);
+		} else if (!strcmp(vv2->value, "$FUNC")) {
+			vv2 = VTAILQ_NEXT(vv2, list);
+			sym = VCC_MkSym(tl,
+			    vv2->value, SYM_FUNC, VCL_LOW, VCL_HIGH);
+			ERRCHK(tl);
+			AN(sym);
+			func_sym(sym, vmod, VTAILQ_NEXT(vv2, list));
+		} else if (!strcmp(vv2->value, "$OBJ")) {
+			vv2 = VTAILQ_NEXT(vv2, list);
+			sym = VCC_MkSym(tl, vv2->value,
+			    SYM_OBJECT, VCL_LOW, VCL_HIGH);
+			XXXAN(sym);
+			sym->eval_priv = vv2;
+			sym->vmod = vmod;
+		} else {
+			VTAILQ_FOREACH(vv2, &vv->children, list)
+				fprintf(stderr, "\tt %s n %s v %s\n",
+				    vv2->type, vv2->name, vv2->value);
+			WRONG("Vmod JSON syntax error");
+		}
+	}
+}
+
 void
 vcc_ParseImport(struct vcc *tl)
 {
 	void *hdl;
 	char fn[1024], *fnp, *fnpx;
 	char buf[256];
+	const char *p;
 	struct token *mod, *t1;
 	struct inifin *ifp;
-	const char * const *spec;
-	struct symbol *sym;
 	struct symbol *msym;
-	const char *p;
 	const struct vmod_data *vmd;
 
 	t1 = tl->t;
@@ -216,48 +296,7 @@ vcc_ParseImport(struct vcc *tl)
 	VSB_printf(ifp->fin, "\t\tVRT_priv_fini(&vmod_priv_%.*s);\n", PF(mod));
 	VSB_printf(ifp->fin, "\t\t\tVRT_Vmod_Fini(&VGC_vmod_%.*s);", PF(mod));
 
-	ifp = NULL;
-
-	spec = vmd->spec;
-	for (; *spec != NULL; spec++) {
-		p = *spec;
-		if (!strcmp(p, "$OBJ")) {
-			p += strlen(p) + 1;
-			sym = VCC_MkSym(tl, p, SYM_OBJECT, VCL_LOW, VCL_HIGH);
-			XXXAN(sym);
-			sym->extra = p;
-			sym->vmod = msym->name;
-		} else if (!strcmp(p, "$EVENT")) {
-			p += strlen(p) + 1;
-			if (ifp == NULL)
-				ifp = New_IniFin(tl);
-			VSB_printf(ifp->ini,
-			    "\tif (%s(ctx, &vmod_priv_%.*s, VCL_EVENT_LOAD))\n"
-			    "\t\treturn(1);",
-			    p, PF(mod));
-			VSB_printf(ifp->fin,
-			    "\t\t(void)%s(ctx, &vmod_priv_%.*s,\n"
-			    "\t\t\t    VCL_EVENT_DISCARD);\n", p, PF(mod));
-			VSB_printf(ifp->event, "%s(ctx, &vmod_priv_%.*s, ev)",
-			    p, PF(mod));
-		} else if (!strcmp(p, "$FUNC")) {
-			p += strlen(p) + 1;
-			sym = VCC_MkSym(tl, p, SYM_FUNC, VCL_LOW, VCL_HIGH);
-			ERRCHK(tl);
-			AN(sym);
-			sym->action = vcc_Act_Call;
-			sym->vmod = msym->name;
-			sym->eval = vcc_Eval_SymFunc;
-			p += strlen(p) + 1;
-			sym->eval_priv = p;
-			sym->type = VCC_Type(p);
-			AN(sym->type);
-		} else {
-			VSB_printf(tl->sb, "Internal spec error (%s)\n", p);
-			vcc_ErrWhere(tl, mod);
-			return;
-		}
-	}
+	parse_json(tl, msym->name, vmd->json);
 
 	Fh(tl, 0, "\n/* --- BEGIN VMOD %.*s --- */\n\n", PF(mod));
 	Fh(tl, 0, "static struct vmod *VGC_vmod_%.*s;\n", PF(mod));
@@ -271,9 +310,11 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym)
 {
 	struct symbol *sy1, *sy2, *sy3;
 	struct inifin *ifp;
-	const char *p, *s_obj;
+	const char *s_obj;
 	char buf1[128];
 	char buf2[128];
+	const struct vjsn_val *vv, *vf;
+	const char *p;
 
 	(void)sym;
 	ExpectErr(tl, ID);
@@ -292,7 +333,7 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym)
 	sy2 = VCC_SymbolGet(tl, SYM_OBJECT, "Symbol not found", XREF_NONE);
 	ERRCHK(tl);
 	AN(sy2);
-	if (sy2->extra == NULL) {
+	if (sy2->eval_priv == NULL) {
 		VSB_printf(tl->sb, "Constructor not found: ");
 		vcc_ErrToken(tl, t);
 		VSB_printf(tl->sb, " at ");
@@ -300,49 +341,53 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym)
 		return;
 	}
 
-	p = sy2->extra;
-	AN(p);
+	CAST_OBJ_NOTNULL(vv, sy2->eval_priv, VJSN_VAL_MAGIC);
+
+	s_obj = vv->value;
+	vv = VTAILQ_NEXT(vv, list);
+
+	Fh(tl, 0, "static %s *%s;\n\n", vv->value, sy1->rname);
+	vv = VTAILQ_NEXT(vv, list);
 
-	s_obj = p;
-	p += strlen(p) + 1;
+	vf = VTAILQ_FIRST(&vv->children);
+	vv = VTAILQ_NEXT(vv, list);
+	assert(vf->type == VJSN_STRING);
+	assert(!strcmp(vf->value, "$INIT"));
 
-	Fh(tl, 0, "static %s *%s;\n\n", p, sy1->rname);
-	p += strlen(p) + 1;
+	vf = VTAILQ_NEXT(vf, list);
 
 	bprintf(buf1, ", &%s, \"%s\"", sy1->rname, sy1->name);
-	vcc_Eval_Func(tl, p, buf1, sy2);
+	vcc_Eval_Func(tl, vf, buf1, sy2);
 	ERRCHK(tl);
 	SkipToken(tl, ';');
+	sy1->def_e = tl->t;
 
-	while (p[0] != '\0' || p[1] != '\0' || p[2] != '\0')
-		p++;
-	p += 3;
+	vf = VTAILQ_FIRST(&vv->children);
+	vv = VTAILQ_NEXT(vv, list);
+	assert(vf->type == VJSN_STRING);
+	assert(!strcmp(vf->value, "$FINI"));
 
+	vf = VTAILQ_NEXT(vf, list);
+	vf = VTAILQ_FIRST(&vf->children);
+	vf = VTAILQ_NEXT(vf, list);
 	ifp = New_IniFin(tl);
-	p += strlen(p) + 1;
-	VSB_printf(ifp->fin, "\t\t%s(&%s);", p, sy1->rname);
-
-	while (p[0] != '\0' || p[1] != '\0' || p[2] != '\0')
-		p++;
-	p += 3;
+	VSB_printf(ifp->fin, "\t\t%s(&%s);", vf->value, sy1->rname);
 
 	/* Instantiate symbols for the methods */
 	bprintf(buf1, ", %s", sy1->rname);
-	while (*p != '\0') {
-		p += strlen(s_obj);
-		bprintf(buf2, "%s%s", sy1->name, p);
+	p = TlDup(tl, buf1);
+	while (vv != NULL) {
+		vf = VTAILQ_FIRST(&vv->children);
+		assert(vf->type == VJSN_STRING);
+		assert(!strcmp(vf->value, "$METHOD"));
+		vf = VTAILQ_NEXT(vf, list);
+		assert(vf->type == VJSN_STRING);
+
+		bprintf(buf2, "%s%s", sy1->name, vf->value + strlen(s_obj));
 		sy3 = VCC_MkSym(tl, buf2, SYM_FUNC, VCL_LOW, VCL_HIGH);
 		AN(sy3);
-		sy3->action = vcc_Act_Call;
-		sy3->eval = vcc_Eval_SymFunc;
-		p += strlen(p) + 1;
-		sy3->eval_priv = p;
-		sy3->type = VCC_Type(p);
-		sy3->extra = TlDup(tl, buf1);
-		sy3->vmod = sy2->vmod;
-		while (p[0] != '\0' || p[1] != '\0' || p[2] != '\0')
-			p++;
-		p += 3;
+		func_sym(sy3, sy2->vmod, VTAILQ_NEXT(vf, list));
+		sy3->extra = p;
+		vv = VTAILQ_NEXT(vv, list);
 	}
-	sy1->def_e = tl->t;
 }
diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index dad08a8..b5ea510 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -43,6 +43,7 @@ import optparse
 import unittest
 import random
 import copy
+import json
 
 rstfmt = False
 strict_abi = True
@@ -227,24 +228,10 @@ class ctype(object):
             return self.vt
         return self.vt + " {" + ",".join(self.spec) + "}"
 
-    def specstr(self, fo, p):
-        fo.write(p + '"' + self.vt)
-        fo.write('\\0"\n')
-        p = indent(p, 4)
-        if self.spec is not None:
-            fo.write(p + '"\\1"\n')
-            p = indent(p, 4)
-            for i in self.spec:
-                fo.write(p + '"' + i + '\\0"\n')
-            p = indent(p, -4)
-            # This terminating \1 is necessary to ensure that
-            # a prototype always ends with three \0's
-            fo.write(p + '"\\1\\0"\n')
-        if self.nm is not None:
-            fo.write(p + '"\\2" "' + self.nm + '\\0"\n')
-        if self.defval is not None:
-            fo.write(p + '"\\3" "' + quote(self.defval) + '\\0"\n')
-
+    def json(self, jl):
+        jl.append([self.vt, self.nm, self.defval, self.spec])
+        while jl[-1][-1] is None:
+                jl[-1].pop(-1)
 
 def vtype(txt):
     j = len(txt)
@@ -404,17 +391,13 @@ class prototype(object):
         s += '%s %s(%s);' % (self.c_ret(), fn, self.c_args(args))
         return "\n".join(lwrap(s)) + "\n"
 
-    def specstr(self, fo, cfunc, p):
-        if self.retval is None:
-            fo.write(p + '"VOID\\0"\n')
-        else:
-            self.retval.specstr(fo, p)
-        fo.write(p + '"' + cfunc + '\\0"\n')
-        p = indent(p, 4)
-        if self.args is not None:
-            for i in self.args:
-                i.specstr(fo, p)
-        fo.write(p + '"\\0"\n')
+    def json(self, jl, cfunc):
+        ll = []
+        self.retval.json(ll)
+        ll.append(cfunc)
+        for i in self.args:
+                i.json(ll)
+        jl.append(ll)
 
 #######################################################################
 
@@ -469,7 +452,7 @@ class stanza(object):
     def cstruct_init(self, fo):
         return
 
-    def specstr(self, fo):
+    def json(self, jl):
         return
 
 #######################################################################
@@ -568,10 +551,11 @@ class s_event(stanza):
     def cstruct_init(self, fo):
         fo.write("\t%s,\n" % self.event_func)
 
-    def specstr(self, fo):
-        fo.write('\t"$EVENT\\0"\n\t    "Vmod_%s_Func._event",\n\n' %
-                 self.vcc.modname)
-
+    def json(self, jl):
+        jl.append([
+                "$EVENT",
+                "Vmod_%s_Func._event" % self.vcc.modname
+        ])
 
 class s_function(stanza):
     def parse(self):
@@ -591,12 +575,13 @@ class s_function(stanza):
     def cstruct_init(self, fo):
         fo.write("\t" + self.proto.cname(pfx=True) + ",\n")
 
-    def specstr(self, fo):
-        fo.write('\t"$FUNC\\0"\t"%s.%s\\0"\n\n' %
-                 (self.vcc.modname, self.proto.name))
-        self.proto.specstr(fo, 'Vmod_%s_Func.%s' %
-                           (self.vcc.modname, self.proto.cname()), "\t    ")
-        fo.write('\t    "\\0",\n\n')
+    def json(self,jl):
+        jl.append([
+                "$FUNC",
+                "%s.%s" % (self.vcc.modname, self.proto.name),
+        ])
+        self.proto.json(jl[-1], 'Vmod_%s_Func.%s' %
+                           (self.vcc.modname, self.proto.cname()))
 
 
 class s_object(stanza):
@@ -663,29 +648,29 @@ class s_object(stanza):
             i.cstruct_init(fo)
         fo.write("\n")
 
-    def specstr(self, fo):
+    def json(self, jl):
+        ll = [
+                "$OBJ",
+                self.vcc.modname + "." + self.proto.name,
+                "struct %s%s_%s" %
+                    (self.vcc.sympfx, self.vcc.modname, self.proto.name),
+        ]
 
-        fo.write('\t"$OBJ\\0"\t"%s.%s\\0"\n\n' %
-                 (self.vcc.modname, self.proto.name))
+        l2 = [ "$INIT" ]
+        ll.append(l2)
+        self.init.json(l2,
+            'Vmod_%s_Func.%s' % (self.vcc.modname, self.init.name))
 
-        fo.write('\t    "struct %s%s_%s\\0"\n' %
-                 (self.vcc.sympfx, self.vcc.modname, self.proto.name))
-        fo.write("\n")
-
-        self.proto.specstr(fo, 'Vmod_%s_Func.%s' %
-                           (self.vcc.modname, self.init.name), '\t    ')
-        fo.write('\t    "\\0"\n\n')
-
-        fo.write('\t    "VOID\\0"\n')
-        fo.write('\t    "Vmod_%s_Func.%s\\0"\n' %
-                 (self.vcc.modname, self.fini.name))
-        fo.write('\t\t"\\0"\n')
-        fo.write('\t    "\\0"\n\n')
+        l2 = [ "$FINI" ]
+        ll.append(l2)
+        self.fini.json(l2,
+            'Vmod_%s_Func.%s' % (self.vcc.modname, self.fini.name))
 
         for i in self.methods:
-            i.specstr(fo)
+                i.json(ll)
+
+        jl.append(ll)
 
-        fo.write('\t    "\\0",\n\n')
 
     def dump(self):
         super(s_object, self).dump()
@@ -707,12 +692,14 @@ class s_method(stanza):
     def cstruct_init(self, fo):
         fo.write('\t' + self.proto.cname(pfx=True) + ",\n")
 
-    def specstr(self, fo):
-        fo.write('\t    "%s.%s\\0"\n' %
-                 (self.vcc.modname, self.proto.name))
-        self.proto.specstr(fo, 'Vmod_%s_Func.%s' %
-                           (self.vcc.modname, self.proto.cname()), '\t\t')
-        fo.write('\t\t"\\0"\n\n')
+    def json(self, jl):
+        jl.append([
+                "$METHOD",
+                self.vcc.modname + "." + self.proto.name
+        ])
+        self.proto.json(jl[-1],
+            'Vmod_%s_Func.%s' % (self.vcc.modname, self.proto.cname()))
+
 
 #######################################################################
 
@@ -836,15 +823,29 @@ class vcc(object):
             fo.write("\t&%senum_%s,\n" % (self.sympfx, j))
         fo.write("};\n")
 
-    def specstr(self, fo):
-        fo.write("\n/*lint -save -e786 -e840 */\n")
-        fo.write("static const char * const Vmod_Spec[] = {\n")
-
+    def json(self, fo):
+        jl = [ ["$VMOD", "1.0" ] ]
         for j in self.contents:
-            j.specstr(fo)
-        fo.write("\t0\n")
-        fo.write("};\n")
-        fo.write("/*lint -restore */\n")
+                j.json(jl)
+
+        bz = bytearray(json.dumps(jl, separators=(",",":"))) + "\0"
+        fo.write("\nstatic const char Vmod_Json[%d] = {\n" % len(bz))
+        t = "\t"
+        for i in bz:
+                t += "%d," % i
+                if len(t) >= 69:
+                        fo.write(t + "\n")
+                        t = "\t"
+        if len(t) > 1:
+                fo.write(t[:-1])
+        fo.write("\n};\n\n")
+        for i in json.dumps(jl, indent=2, separators=(',', ': ')).split("\n"):
+                j = "// " + i
+                if len(j) > 72:
+                        fo.write(j[:72] + "[...]\n")
+                else:
+                        fo.write(j + "\n")
+        fo.write("\n")
 
     def api(self, fo):
         for i in (714, 759, 765):
@@ -863,7 +864,7 @@ class vcc(object):
         fo.write('\t.func =\t\t&Vmod_Func,\n')
         fo.write('\t.func_len =\tsizeof(Vmod_Func),\n')
         fo.write('\t.proto =\tVmod_Proto,\n')
-        fo.write('\t.spec =\t\tVmod_Spec,\n')
+        fo.write('\t.json =\t\tVmod_Json,\n')
         fo.write('\t.abi =\t\tVMOD_ABI_Version,\n')
         # NB: Sort of hackish:
         # Fill file_id with random stuff, so we can tell if
@@ -927,7 +928,7 @@ class vcc(object):
 
         os.remove(fn2)
 
-        self.specstr(fo)
+        self.json(fo)
 
         self.api(fo)
 

From fgsch at lodoss.net  Thu Feb 22 01:21:11 2018
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Thu, 22 Feb 2018 01:21:11 +0000 (UTC)
Subject: [master] 8bf0dd5 Sync
Message-ID: <20180222012111.C738AB7B4D@lists.varnish-cache.org>


commit 8bf0dd532ac843a120990b6643b405c489ac15df
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Thu Feb 22 01:13:20 2018 +0000

    Sync

diff --git a/.gitignore b/.gitignore
index 01b60d1..2453bd5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -117,9 +117,9 @@ cscope.*out
 /include/vbm_test
 /lib/libvarnish/*.log
 /lib/libvarnish/binheap
+/lib/libvarnish/vjsn_test
 /lib/libvarnish/vnum_c_test
 /lib/libvarnishapi/*.log
-/lib/libvarnishapi/vjsn_test
 /lib/libvarnishapi/vsl_glob_test
 
 # vtest.sh droppings

From geoff at uplex.de  Thu Feb 22 09:23:09 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Thu, 22 Feb 2018 09:23:09 +0000 (UTC)
Subject: [master] a2682c0 Support Unix domain sockets as -a listen addresses.
Message-ID: <20180222092309.C563ABFBF9@lists.varnish-cache.org>


commit a2682c0951d6d7d441939574d0b7e0a85865119f
Author: Geoff Simmons <geoff at uplex.de>
Date:   Fri Feb 16 19:56:43 2018 +0100

    Support Unix domain sockets as -a listen addresses.
    
    Also adds the user, group and mode sub-args to -a, to set
    permissions on the path created by -a for UDS.
    
    Add the bogo_ip pseudo-VSA, representing IPv4 0.0.0.0:0, to be
    exposed in VCL for non-IP addresses.
    
    Also adding the field listen_sock to struct sess: pointer to the
    struct listen_sock that was created by the acceptor and lives in
    heritage.socks. This makes information like the endpoint name
    (named -a arg) and the UDS path available from an sp.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index ed5602f..a8aadd1 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -99,6 +99,7 @@ struct poolparam;
 struct sess;
 struct transport;
 struct worker;
+struct listen_sock;
 
 #define DIGEST_LEN		32
 
@@ -571,6 +572,7 @@ struct sess {
 #define SESS_MAGIC		0x2c2f9c5a
 
 	uint16_t		sattr[SA_LAST];
+	struct listen_sock	*listen_sock;
 	int			refcnt;
 	int			fd;
 	uint32_t		vxid;
diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index bb1f000..b54c99a 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -37,6 +37,8 @@
 #include <stdlib.h>
 #include <netinet/in.h>
 #include <netinet/tcp.h>
+#include <sys/socket.h>
+#include <sys/un.h>
 
 #include "cache_varnishd.h"
 
@@ -86,25 +88,26 @@ static struct tcp_opt {
 	socklen_t	sz;
 	void		*ptr;
 	int		need;
+	int		iponly;
 } tcp_opts[] = {
-#define TCPO(lvl, nam, sz) { lvl, nam, #nam, sizeof(sz), 0, 0},
+#define TCPO(lvl, nam, sz, ip) { lvl, nam, #nam, sizeof(sz), 0, 0, ip},
 
-	TCPO(SOL_SOCKET, SO_LINGER, struct linger)
-	TCPO(SOL_SOCKET, SO_KEEPALIVE, int)
-	TCPO(IPPROTO_TCP, TCP_NODELAY, int)
+	TCPO(SOL_SOCKET, SO_LINGER, struct linger, 0)
+	TCPO(SOL_SOCKET, SO_KEEPALIVE, int, 0)
+	TCPO(IPPROTO_TCP, TCP_NODELAY, int, 1)
 
 #ifdef SO_SNDTIMEO_WORKS
-	TCPO(SOL_SOCKET, SO_SNDTIMEO, struct timeval)
+	TCPO(SOL_SOCKET, SO_SNDTIMEO, struct timeval, 0)
 #endif
 
 #ifdef SO_RCVTIMEO_WORKS
-	TCPO(SOL_SOCKET, SO_RCVTIMEO, struct timeval)
+	TCPO(SOL_SOCKET, SO_RCVTIMEO, struct timeval, 0)
 #endif
 
 #ifdef HAVE_TCP_KEEP
-	TCPO(IPPROTO_TCP, TCP_KEEPIDLE, int)
-	TCPO(IPPROTO_TCP, TCP_KEEPCNT, int)
-	TCPO(IPPROTO_TCP, TCP_KEEPINTVL, int)
+	TCPO(IPPROTO_TCP, TCP_KEEPIDLE, int, 1)
+	TCPO(IPPROTO_TCP, TCP_KEEPCNT, int, 1)
+	TCPO(IPPROTO_TCP, TCP_KEEPINTVL, int, 1)
 #endif
 
 #undef TCPO
@@ -218,15 +221,18 @@ vca_tcp_opt_init(void)
 }
 
 static void
-vca_tcp_opt_test(int sock)
+vca_tcp_opt_test(struct listen_sock *ls)
 {
-	int i, n;
+	int i, n, sock;
 	struct tcp_opt *to;
 	socklen_t l;
 	void *ptr;
 
+	sock = ls->sock;
 	for (n = 0; n < n_tcp_opts; n++) {
 		to = &tcp_opts[n];
+		if (to->iponly && ls->uds)
+			continue;
 		to->need = 1;
 		ptr = calloc(1, to->sz);
 		AN(ptr);
@@ -241,13 +247,16 @@ vca_tcp_opt_test(int sock)
 }
 
 static void
-vca_tcp_opt_set(int sock, int force)
+vca_tcp_opt_set(struct listen_sock *ls, int force)
 {
-	int n;
+	int n, sock;
 	struct tcp_opt *to;
 
+	sock = ls->sock;
 	for (n = 0; n < n_tcp_opts; n++) {
 		to = &tcp_opts[n];
+		if (to->iponly && ls->uds)
+			continue;
 		if (to->need || force) {
 			VTCP_Assert(setsockopt(sock,
 			    to->level, to->optname, to->ptr, to->sz));
@@ -304,15 +313,57 @@ vca_pace_good(void)
  * Called from assigned worker thread
  */
 
+static void
+vca_mk_tcp(struct wrk_accept *wa, struct sess *sp, char *laddr, char *lport,
+	   char *raddr, char *rport)
+{
+	struct suckaddr *sa;
+	struct sockaddr_storage ss;
+	socklen_t sl;
+
+	SES_Reserve_remote_addr(sp, &sa);
+	AN(VSA_Build(sa, &wa->acceptaddr, wa->acceptaddrlen));
+	sp->sattr[SA_CLIENT_ADDR] = sp->sattr[SA_REMOTE_ADDR];
+
+	VTCP_name(sa, raddr, VTCP_ADDRBUFSIZE, rport, VTCP_PORTBUFSIZE);
+	SES_Set_String_Attr(sp, SA_CLIENT_IP, raddr);
+	SES_Set_String_Attr(sp, SA_CLIENT_PORT, rport);
+
+	sl = sizeof ss;
+	AZ(getsockname(sp->fd, (void*)&ss, &sl));
+	SES_Reserve_local_addr(sp, &sa);
+	AN(VSA_Build(sa, &ss, sl));
+	sp->sattr[SA_SERVER_ADDR] = sp->sattr[SA_LOCAL_ADDR];
+	VTCP_name(sa, laddr, VTCP_ADDRBUFSIZE, lport, VTCP_PORTBUFSIZE);
+}
+
+static void
+vca_mk_uds(struct wrk_accept *wa, struct sess *sp, char *laddr, char *lport,
+	   char *raddr, char *rport)
+{
+	struct suckaddr *sa;
+
+	(void) wa;
+	SES_Reserve_remote_addr(sp, &sa);
+	SES_Set_remote_addr(sp, bogo_ip);
+	sp->sattr[SA_CLIENT_ADDR] = sp->sattr[SA_REMOTE_ADDR];
+	sp->sattr[SA_LOCAL_ADDR] = sp->sattr[SA_REMOTE_ADDR];
+	sp->sattr[SA_SERVER_ADDR] = sp->sattr[SA_REMOTE_ADDR];
+	SES_Set_String_Attr(sp, SA_CLIENT_IP, "0.0.0.0");
+	SES_Set_String_Attr(sp, SA_CLIENT_PORT, "0");
+
+	strcpy(laddr, "0.0.0.0");
+	strcpy(raddr, "0.0.0.0");
+	strcpy(lport, "0");
+	strcpy(rport, "0");
+}
+
 static void v_matchproto_(task_func_t)
 vca_make_session(struct worker *wrk, void *arg)
 {
 	struct sess *sp;
 	struct req *req;
 	struct wrk_accept *wa;
-	struct sockaddr_storage ss;
-	struct suckaddr *sa;
-	socklen_t sl;
 	char laddr[VTCP_ADDRBUFSIZE];
 	char lport[VTCP_PORTBUFSIZE];
 	char raddr[VTCP_ADDRBUFSIZE];
@@ -353,24 +404,16 @@ vca_make_session(struct worker *wrk, void *arg)
 
 	sp->fd = wa->acceptsock;
 	wa->acceptsock = -1;
+	sp->listen_sock = wa->acceptlsock;
 
 	assert(wa->acceptaddrlen <= vsa_suckaddr_len);
-	SES_Reserve_remote_addr(sp, &sa);
-	AN(VSA_Build(sa, &wa->acceptaddr, wa->acceptaddrlen));
-	sp->sattr[SA_CLIENT_ADDR] = sp->sattr[SA_REMOTE_ADDR];
-
-	VTCP_name(sa, raddr, sizeof raddr, rport, sizeof rport);
-	SES_Set_String_Attr(sp, SA_CLIENT_IP, raddr);
-	SES_Set_String_Attr(sp, SA_CLIENT_PORT, rport);
-
-	sl = sizeof ss;
-	AZ(getsockname(sp->fd, (void*)&ss, &sl));
-	SES_Reserve_local_addr(sp, &sa);
-	AN(VSA_Build(sa, &ss, sl));
-	sp->sattr[SA_SERVER_ADDR] = sp->sattr[SA_LOCAL_ADDR];
 
-	VTCP_name(sa, laddr, sizeof laddr, lport, sizeof lport);
+	if (wa->acceptlsock->uds)
+		vca_mk_uds(wa, sp, laddr, lport, raddr, rport);
+	else
+		vca_mk_tcp(wa, sp, laddr, lport, raddr, rport);
 
+	AN(wa->acceptlsock->name);
 	VSL(SLT_Begin, sp->vxid, "sess 0 %s",
 	    wa->acceptlsock->transport->name);
 	VSL(SLT_SessOpen, sp->vxid, "%s %s %s %s %s %.6f %d",
@@ -383,10 +426,10 @@ vca_make_session(struct worker *wrk, void *arg)
 	wrk->stats->sess_conn++;
 
 	if (need_test) {
-		vca_tcp_opt_test(sp->fd);
+		vca_tcp_opt_test(wa->acceptlsock);
 		need_test = 0;
 	}
-	vca_tcp_opt_set(sp->fd, 0);
+	vca_tcp_opt_set(wa->acceptlsock, 0);
 
 	req = Req_New(wrk, sp);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -562,7 +605,7 @@ vca_acct(void *arg)
 				    ls->sock, i, strerror(errno));
 		}
 		AZ(listen(ls->sock, cache_param->listen_depth));
-		vca_tcp_opt_set(ls->sock, 1);
+		vca_tcp_opt_set(ls, 1);
 		if (cache_param->accept_filter) {
 			int i;
 			i = VTCP_filter_http(ls->sock);
@@ -585,7 +628,7 @@ vca_acct(void *arg)
 				if (ls->sock == -2)
 					continue;	// VCA_Shutdown
 				assert (ls->sock > 0);
-				vca_tcp_opt_set(ls->sock, 1);
+				vca_tcp_opt_set(ls, 1);
 			}
 			AZ(pthread_mutex_unlock(&shut_mtx));
 		}
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index ef6da50..2ab2fdb 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -250,7 +250,6 @@ VRT_r_beresp_uncacheable(VRT_CTX)
 const char *
 VRT_r_client_identity(VRT_CTX)
 {
-
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
 	if (ctx->req->client_identity != NULL)
diff --git a/bin/varnishd/common/heritage.h b/bin/varnishd/common/heritage.h
index 64131d6..7903cf6 100644
--- a/bin/varnishd/common/heritage.h
+++ b/bin/varnishd/common/heritage.h
@@ -35,6 +35,7 @@ struct suckaddr;
 struct listen_sock;
 struct transport;
 struct VCLS;
+struct uds_perms;
 
 struct listen_sock {
 	unsigned			magic;
@@ -42,10 +43,12 @@ struct listen_sock {
 	VTAILQ_ENTRY(listen_sock)	list;
 	VTAILQ_ENTRY(listen_sock)	arglist;
 	int				sock;
+	int				uds;
 	char				*endpoint;
 	const char			*name;
 	struct suckaddr			*addr;
 	const struct transport		*transport;
+	const struct uds_perms		*perms;
 };
 
 VTAILQ_HEAD(listen_sock_head, listen_sock);
diff --git a/bin/varnishd/mgt/mgt_acceptor.c b/bin/varnishd/mgt/mgt_acceptor.c
index af09013..64dad07 100644
--- a/bin/varnishd/mgt/mgt_acceptor.c
+++ b/bin/varnishd/mgt/mgt_acceptor.c
@@ -33,11 +33,15 @@
 
 #include <sys/types.h>
 #include <sys/socket.h>
+#include <sys/un.h>
+#include <sys/stat.h>
 #include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
+#include <pwd.h>
+#include <grp.h>
 
 #include "mgt/mgt.h"
 #include "common/heritage.h"
@@ -47,6 +51,7 @@
 #include "vsa.h"
 #include "vss.h"
 #include "vtcp.h"
+#include "vus.h"
 
 struct listen_arg {
 	unsigned			magic;
@@ -56,6 +61,15 @@ struct listen_arg {
 	const char			*name;
 	VTAILQ_HEAD(,listen_sock)	socks;
 	const struct transport		*transport;
+	const struct uds_perms		*perms;
+};
+
+struct uds_perms {
+	unsigned	magic;
+#define UDS_PERMS_MAGIC 0x84fb5635
+	mode_t		mode;
+	uid_t		uid;
+	gid_t		gid;
 };
 
 static VTAILQ_HEAD(,listen_arg) listen_args =
@@ -65,18 +79,35 @@ static int
 mac_opensocket(struct listen_sock *ls)
 {
 	int fail;
+	struct sockaddr_un uds;
 
 	CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC);
 	if (ls->sock > 0) {
 		MCH_Fd_Inherit(ls->sock, NULL);
 		closefd(&ls->sock);
 	}
-	ls->sock = VTCP_bind(ls->addr, NULL);
+	if (!ls->uds)
+		ls->sock = VTCP_bind(ls->addr, NULL);
+	else {
+		uds.sun_family = PF_UNIX;
+		strcpy(uds.sun_path, ls->endpoint);
+		ls->sock = VUS_bind(&uds, NULL);
+	}
 	fail = errno;
 	if (ls->sock < 0) {
 		AN(fail);
 		return (fail);
 	}
+	if (ls->perms != NULL) {
+		CHECK_OBJ(ls->perms, UDS_PERMS_MAGIC);
+		assert(ls->uds);
+		errno = 0;
+		if (ls->perms->mode != 0
+		    && chmod(ls->endpoint, ls->perms->mode) != 0)
+			return errno;
+		if (chown(ls->endpoint, ls->perms->uid, ls->perms->gid) != 0)
+			return errno;
+	}
 	MCH_Fd_Inherit(ls->sock, "sock");
 	return (0);
 }
@@ -109,22 +140,12 @@ MAC_reopen_sockets(void)
 
 /*--------------------------------------------------------------------*/
 
-static int v_matchproto_(vss_resolved_f)
-mac_callback(void *priv, const struct suckaddr *sa)
+static struct listen_sock *
+mk_listen_sock(struct listen_arg *la, const struct suckaddr *sa)
 {
-	struct listen_arg *la;
 	struct listen_sock *ls;
-	char abuf[VTCP_ADDRBUFSIZE], pbuf[VTCP_PORTBUFSIZE];
-	char nbuf[VTCP_ADDRBUFSIZE+VTCP_PORTBUFSIZE+2];
 	int fail;
 
-	CAST_OBJ_NOTNULL(la, priv, LISTEN_ARG_MAGIC);
-
-	VTAILQ_FOREACH(ls, &heritage.socks, list) {
-		if (!VSA_Compare(sa, ls->addr))
-			ARGV_ERR("-a arguments %s and %s have same address\n",
-			    ls->endpoint, la->endpoint);
-	}
 	ALLOC_OBJ(ls, LISTEN_SOCK_MAGIC);
 	AN(ls);
 	ls->sock = -1;
@@ -134,6 +155,9 @@ mac_callback(void *priv, const struct suckaddr *sa)
 	AN(ls->endpoint);
 	ls->name = la->name;
 	ls->transport = la->transport;
+	ls->perms = la->perms;
+	if (*la->endpoint == '/')
+		ls->uds = 1;
 	VJ_master(JAIL_MASTER_PRIVPORT);
 	fail = mac_opensocket(ls);
 	VJ_master(JAIL_MASTER_LOW);
@@ -144,8 +168,30 @@ mac_callback(void *priv, const struct suckaddr *sa)
 		if (fail != EAFNOSUPPORT)
 			ARGV_ERR("Could not get socket %s: %s\n",
 			    la->endpoint, strerror(fail));
-		return(0);
+		return(NULL);
 	}
+	return(ls);
+}
+
+static int v_matchproto_(vss_resolved_f)
+mac_tcp(void *priv, const struct suckaddr *sa)
+{
+	struct listen_arg *la;
+	struct listen_sock *ls;
+	char abuf[VTCP_ADDRBUFSIZE], pbuf[VTCP_PORTBUFSIZE];
+	char nbuf[VTCP_ADDRBUFSIZE+VTCP_PORTBUFSIZE+2];
+
+	CAST_OBJ_NOTNULL(la, priv, LISTEN_ARG_MAGIC);
+
+	VTAILQ_FOREACH(ls, &heritage.socks, list) {
+		if (!ls->uds && !VSA_Compare(sa, ls->addr))
+			ARGV_ERR("-a arguments %s and %s have same address\n",
+			    ls->endpoint, la->endpoint);
+	}
+	ls = mk_listen_sock(la, sa);
+	if (ls == NULL)
+		return(0);
+	AZ(ls->uds);
 	if (VSA_Port(ls->addr) == 0) {
 		/*
 		 * If the argv port number is zero, we adopt whatever
@@ -164,6 +210,28 @@ mac_callback(void *priv, const struct suckaddr *sa)
 	return (0);
 }
 
+static int v_matchproto_(vus_resolved_f)
+mac_uds(void *priv, const struct sockaddr_un *uds)
+{
+	struct listen_arg *la;
+	struct listen_sock *ls;
+
+	CAST_OBJ_NOTNULL(la, priv, LISTEN_ARG_MAGIC);
+
+	VTAILQ_FOREACH(ls, &heritage.socks, list) {
+		if (ls->uds && strcmp(uds->sun_path, ls->endpoint) == 0)
+			ARGV_ERR("-a arguments %s and %s have same address\n",
+			    ls->endpoint, la->endpoint);
+	}
+	ls = mk_listen_sock(la, bogo_ip);
+	if (ls == NULL)
+		return(0);
+	AN(ls->uds);
+	VTAILQ_INSERT_TAIL(&la->socks, ls, arglist);
+	VTAILQ_INSERT_TAIL(&heritage.socks, ls, list);
+	return (0);
+}
+
 void
 MAC_Arg(const char *spec)
 {
@@ -171,10 +239,14 @@ MAC_Arg(const char *spec)
 	struct listen_arg *la;
 	const char *err;
 	int error;
-	const struct transport *xp;
+	const struct transport *xp = NULL;
 	const char *name;
 	char name_buf[8];
 	static unsigned seq = 0;
+	struct passwd *pwd = NULL;
+	struct group *grp = NULL;
+	mode_t mode = 0;
+	struct uds_perms *perms;
 
 	av = MGT_NamedArg(spec, &name, "-a");
 	AN(av);
@@ -192,19 +264,107 @@ MAC_Arg(const char *spec)
 	}
 	la->name = name;
 
-	if (av[2] == NULL) {
-		xp = XPORT_Find("http");
-	} else {
-		xp = XPORT_Find(av[2]);
-		if (xp == NULL)
-			ARGV_ERR("Unknown protocol '%s'\n", av[2]);
-		if (av[3] != NULL)
-			ARGV_ERR("Too many sub-arguments to -a(%s)\n", av[2]);
+	if (*la->endpoint != '/' && strchr(la->endpoint, '/') != NULL)
+		ARGV_ERR("Unix domain socket addresses must be absolute paths "
+			 "in -a (%s)\n", la->endpoint);
+
+	for (int i = 2; av[i] != NULL; i++) {
+		char *eq, *val;
+		int len;
+
+		if ((eq = strchr(av[i], '=')) == NULL) {
+			if (xp != NULL)
+				ARGV_ERR("Too many protocol sub-args in -a "
+					 "(%s)\n", av[i]);
+			xp = XPORT_Find(av[i]);
+			if (xp == NULL)
+				ARGV_ERR("Unknown protocol '%s'\n", av[i]);
+			continue;
+		}
+		if (la->endpoint[0] != '/')
+			ARGV_ERR("Invalid sub-arg %s for IP addresses in -a\n",
+				 av[i]);
+
+		val = eq + 1;
+		len = eq - av[i];
+		assert(len >= 0);
+		if (len == 0)
+			ARGV_ERR("Invalid sub-arg %s in -a\n", av[i]);
+
+		if (strncmp(av[i], "user", len) == 0) {
+			if (pwd != NULL)
+				ARGV_ERR("Too many user sub-args in -a (%s)\n",
+					 av[i]);
+			pwd = getpwnam(val);
+			if (pwd == NULL)
+				ARGV_ERR("Unknown user %s in -a\n", val);
+			continue;
+		}
+
+		if (strncmp(av[i], "group", len) == 0) {
+			if (grp != NULL)
+				ARGV_ERR("Too many group sub-args in -a (%s)\n",
+					 av[i]);
+			grp = getgrnam(val);
+			if (grp == NULL)
+				ARGV_ERR("Unknown group %s in -a\n", val);
+			continue;
+		}
+
+		if (strncmp(av[i], "mode", len) == 0) {
+			long m;
+			char *p;
+
+			if (mode != 0)
+				ARGV_ERR("Too many mode sub-args in -a (%s)\n",
+					 av[i]);
+			if (*val == '\0')
+				ARGV_ERR("Empty mode sub-arg in -a\n");
+			errno = 0;
+			m = strtol(val, &p, 8);
+			if (*p != '\0')
+				ARGV_ERR("Invalid mode sub-arg %s in -a\n",
+					 val);
+			if (errno)
+				ARGV_ERR("Cannot parse mode sub-arg %s in -a: "
+					 "%s\n", val, strerror(errno));
+			if (m <= 0 || m > 0777)
+				ARGV_ERR("Mode sub-arg %s out of range in -a\n",
+					 val);
+			mode = (mode_t) m;
+			continue;
+		}
+
+		ARGV_ERR("Invalid sub-arg %s in -a\n", av[i]);
 	}
+
+	if (xp == NULL)
+		xp = XPORT_Find("http");
 	AN(xp);
 	la->transport = xp;
 
-	error = VSS_resolver(av[1], "80", mac_callback, la, &err);
+	if (pwd != NULL || grp != NULL || mode != 0) {
+		ALLOC_OBJ(perms, UDS_PERMS_MAGIC);
+		AN(perms);
+		if (pwd != NULL)
+			perms->uid = pwd->pw_uid;
+		else
+			perms->uid = (uid_t) -1;
+		if (grp != NULL)
+			perms->gid = grp->gr_gid;
+		else
+			perms->gid = (gid_t) -1;
+		perms->mode = mode;
+		la->perms = perms;
+	}
+	else
+		AZ(la->perms);
+
+	if (*la->endpoint != '/')
+		error = VSS_resolver(av[1], "80", mac_tcp, la, &err);
+	else
+		error = VUS_resolver(av[1], mac_uds, la, &err);
+
 	if (VTAILQ_EMPTY(&la->socks) || error)
 		ARGV_ERR("Got no socket(s) for %s\n", av[1]);
 	VAV_Free(av);
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 8e968ed..275f6fb 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -41,6 +41,7 @@
 #include <syslog.h>
 #include <unistd.h>
 #include <sys/stat.h>
+#include <sys/socket.h>
 
 #include "mgt/mgt.h"
 #include "common/heritage.h"
@@ -59,6 +60,7 @@
 #include "vsub.h"
 #include "vtim.h"
 #include "waiter/mgt_waiter.h"
+#include "vsa.h"
 
 struct heritage		heritage;
 unsigned		d_flag = 0;
@@ -564,6 +566,9 @@ main(int argc, char * const *argv)
 
 	VJ_Init(j_arg);
 
+	/* Initialize the bogo-IP VSA */
+	VSA_Init();
+
 	optind = 1;
 	optreset = 1;
 	while ((o = getopt(argc, argv, opt_spec)) != -1) {
diff --git a/bin/varnishtest/tests/b00053.vtc b/bin/varnishtest/tests/b00053.vtc
new file mode 100644
index 0000000..09e3d20
--- /dev/null
+++ b/bin/varnishtest/tests/b00053.vtc
@@ -0,0 +1,34 @@
+varnishtest "Does anything get through Unix domain sockets at all ?"
+
+server s1 {
+	rxreq
+	txresp -body "012345\n"
+} -start
+
+varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.do_stream = false;
+	}
+} -start
+
+varnish v1 -cliok "param.set debug +workspace"
+varnish v1 -cliok "param.set debug +witness"
+
+varnish v1 -expect n_object == 0
+varnish v1 -expect sess_conn == 0
+varnish v1 -expect client_req == 0
+varnish v1 -expect cache_miss == 0
+
+client c1 -connect "${tmpdir}/v1.sock" {
+	txreq -url "/"
+	rxresp
+	expect resp.status == 200
+} -run
+
+varnish v1 -expect n_object == 1
+varnish v1 -expect sess_conn == 1
+varnish v1 -expect client_req == 1
+varnish v1 -expect cache_miss == 1
+varnish v1 -expect s_sess == 1
+varnish v1 -expect s_resp_bodybytes == 7
+varnish v1 -expect s_resp_hdrbytes == 178
diff --git a/bin/varnishtest/tests/b00054.vtc b/bin/varnishtest/tests/b00054.vtc
new file mode 100644
index 0000000..4850a56
--- /dev/null
+++ b/bin/varnishtest/tests/b00054.vtc
@@ -0,0 +1,18 @@
+varnishtest "Check poll acceptor on a UDS listen address"
+
+server s1 {
+	rxreq
+	txresp -hdr "Connection: close" -body "012345\n"
+} -start
+
+varnish v1 -arg "-a ${tmpdir}/v1.sock -Wpoll" -vcl+backend {} -start
+
+client c1 -connect "${tmpdir}/v1.sock" {
+	txreq -url "/"
+	rxresp
+	expect resp.status == 200
+	delay .1
+	txreq -url "/"
+	rxresp
+	expect resp.status == 200
+} -run
diff --git a/bin/varnishtest/tests/b00055.vtc b/bin/varnishtest/tests/b00055.vtc
new file mode 100644
index 0000000..bb3a36d
--- /dev/null
+++ b/bin/varnishtest/tests/b00055.vtc
@@ -0,0 +1,30 @@
+varnishtest "Check pipelining over a UDS listen address"
+
+server s1 {
+	rxreq
+	expect req.url == "/foo"
+	txresp -body "foo"
+	rxreq
+	expect req.url == "/bar"
+	txresp -body "foobar"
+} -start
+
+varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl+backend {} -start
+
+client c1 -connect "${tmpdir}/v1.sock" {
+	send "GET /foo HTTP/1.1\n\nGET /bar HTTP/1.1\n\nGET /bar HTTP/1.1\n\n"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 3
+	expect resp.http.x-varnish == "1001"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 6
+	expect resp.http.x-varnish == "1003"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 6
+	expect resp.http.x-varnish == "1005 1004"
+} -run
+
+varnish v1 -expect sess_readahead == 2
diff --git a/bin/varnishtest/tests/b00056.vtc b/bin/varnishtest/tests/b00056.vtc
new file mode 100644
index 0000000..b875f0b
--- /dev/null
+++ b/bin/varnishtest/tests/b00056.vtc
@@ -0,0 +1,36 @@
+varnishtest "Check read-head / partial pipelining over a UDS listen address"
+
+server s1 {
+	rxreq
+	expect req.url == "/foo"
+	txresp -body "foo"
+	rxreq
+	expect req.url == "/bar"
+	txresp -body "foobar"
+} -start
+
+varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl+backend {}
+
+# NB: The accept_filter param may not exist.
+varnish v1 -cli "param.set accept_filter false"
+varnish v1 -start
+
+client c1 -connect "${tmpdir}/v1.sock" {
+	send "GET /foo HTTP/1.1\r\n\r\nGET "
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 3
+	expect resp.http.x-varnish == "1001"
+	send "/bar HTTP/1.1\n\nGET /bar "
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 6
+	expect resp.http.x-varnish == "1003"
+	send "HTTP/1.1\n\n"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 6
+	expect resp.http.x-varnish == "1005 1004"
+} -run
+
+varnish v1 -expect sess_readahead == 2
diff --git a/bin/varnishtest/tests/b00057.vtc b/bin/varnishtest/tests/b00057.vtc
new file mode 100644
index 0000000..74c73cc
--- /dev/null
+++ b/bin/varnishtest/tests/b00057.vtc
@@ -0,0 +1,22 @@
+varnishtest "Test orderly connection closure of a UDS listen socket"
+
+
+server s1 {
+	rxreq
+	txresp -nolen -hdr "Transfer-encoding: chunked"
+	delay .2
+	chunkedlen 30000
+	delay .2
+	chunkedlen 100000
+	delay .2
+	chunkedlen 0
+} -start
+
+varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl+backend { } -start
+
+client c1 -connect "${tmpdir}/v1.sock" {
+	txreq -hdr "Connection: close"
+	delay 3
+	rxresp
+	expect resp.bodylen == 130000
+} -run
diff --git a/bin/varnishtest/tests/b00059.vtc b/bin/varnishtest/tests/b00059.vtc
new file mode 100644
index 0000000..00ce94f
--- /dev/null
+++ b/bin/varnishtest/tests/b00059.vtc
@@ -0,0 +1,52 @@
+varnishtest "Run a lot of transactions through Unix domain sockets"
+
+server s0 {
+	loop 10 {
+		rxreq
+		txresp -body "foo1"
+	}
+	rxreq
+	txresp -hdr "Connection: close" -body "foo1"
+	expect_close
+} -dispatch
+
+varnish v1 -arg "-a ${tmpdir}/v1.sock -Wpoll" -vcl+backend {
+	sub vcl_recv {
+		return (pass);
+	}
+	sub vcl_backend_fetch {
+		set bereq.backend = s0;
+	}
+
+} -start
+
+client c1 -connect "${tmpdir}/v1.sock" {
+	loop 20 {
+		txreq -url /c1
+		rxresp
+		expect resp.bodylen == 4
+		expect resp.status == 200
+	}
+} -start
+
+client c2 -connect "${tmpdir}/v1.sock" {
+	loop 20 {
+		txreq -url /c2
+		rxresp
+		expect resp.bodylen == 4
+		expect resp.status == 200
+	}
+} -start
+
+client c3 -connect "${tmpdir}/v1.sock" {
+	loop 20 {
+		txreq -url /c3
+		rxresp
+		expect resp.bodylen == 4
+		expect resp.status == 200
+	}
+} -start
+
+client c1 -wait
+client c2 -wait
+client c3 -wait
diff --git a/bin/varnishtest/tests/b00060.vtc b/bin/varnishtest/tests/b00060.vtc
new file mode 100644
index 0000000..6ecd156
--- /dev/null
+++ b/bin/varnishtest/tests/b00060.vtc
@@ -0,0 +1,22 @@
+varnishtest "VSL tags affected by the use of UDS addresses"
+
+varnish v1 -arg "-a foo=${tmpdir}/v1.sock" -vcl {
+	backend b { .host = "${bad_ip}"; }
+
+	sub vcl_recv { return(synth(200)); }
+} -start
+
+client c1 -connect "${tmpdir}/v1.sock" {
+	txreq
+	rxresp
+} -run
+
+logexpect l1 -v v1 -d 1 -g session {
+	expect 0 1000	Begin
+	expect 0 =	SessOpen	"^0.0.0.0 0 foo 0.0.0.0 0"
+} -run
+
+logexpect l2 -v v1 -d 1 -g vxid {
+	expect 0 1001	Begin
+	expect * =	ReqStart	"^0.0.0.0 0$"
+} -run
diff --git a/bin/varnishtest/tests/c00003.vtc b/bin/varnishtest/tests/c00003.vtc
index eca86dc..9dfb220 100644
--- a/bin/varnishtest/tests/c00003.vtc
+++ b/bin/varnishtest/tests/c00003.vtc
@@ -6,14 +6,108 @@ shell -err -match "have same address|already in use" {
 	varnishd -d -a 127.0.0.1:38484 -a 127.0.0.1:38484 -b localhost:80
 }
 
+shell -err -match "have same address|already in use" {
+	varnishd -d -a ${tmpdir}/vtc.sock -a ${tmpdir}/vtc.sock -b localhost:80
+}
+
 # -a bad protocol specs
-shell -err -expect "Too many sub-arguments" {
+shell -err -expect "Too many protocol sub-args" {
 	varnishd -a 127.0.0.1:80000,PROXY,FOO -d
 }
-shell -err -expect "Too many sub-arguments" {
+shell -err -expect "Too many protocol sub-args" {
 	varnishd -a 127.0.0.1:80000,HTTP,FOO -d
 }
 
+# -a relative path for a UDS address not permitted
+shell -err -expect "Unix domain socket addresses must be absolute paths" {
+	varnishd -a foo/bar.sock -d
+}
+
+# -a args for UDS permissions not permitted with IP addresses
+shell -err -expect "Invalid sub-arg user=u for IP addresses" {
+	varnishd -a 127.0.0.1:80000,user=u -d
+}
+
+shell -err -expect "Invalid sub-arg group=g for IP addresses" {
+	varnishd -a 127.0.0.1:80000,group=g -d
+}
+
+shell -err -expect "Invalid sub-arg mode=660 for IP addresses" {
+	varnishd -a 127.0.0.1:80000,mode=660 -d
+}
+
+# Illegal mode sub-args
+shell -err -expect "Too many mode sub-args" {
+	varnishd -a ${tmpdir}/vtc.sock,mode=660,mode=600 -d
+}
+
+shell -err -expect "Empty mode sub-arg" {
+	varnishd -a ${tmpdir}/vtc.sock,mode= -d
+}
+
+shell -err -expect "Invalid mode sub-arg 666devilish" {
+	varnishd -a ${tmpdir}/vtc.sock,mode=666devilish -d
+}
+
+shell -err -expect "Invalid mode sub-arg devilish" {
+	varnishd -a ${tmpdir}/vtc.sock,mode=devilish -d
+}
+
+shell -err -expect "Invalid mode sub-arg 999" {
+	varnishd -a ${tmpdir}/vtc.sock,mode=999 -d
+}
+
+shell -err -expect "Cannot parse mode sub-arg 7" {
+	varnishd -a ${tmpdir}/vtc.sock,mode=77777777777777777777777777777777 -d
+}
+
+shell -err -expect "Cannot parse mode sub-arg -7" {
+	varnishd -a ${tmpdir}/vtc.sock,mode=-77777777777777777777777777777777 -d
+}
+
+shell -err -expect "Mode sub-arg 1666 out of range" {
+	varnishd -a ${tmpdir}/vtc.sock,mode=1666 -d
+}
+
+shell -err -expect "Mode sub-arg 0 out of range" {
+	varnishd -a ${tmpdir}/vtc.sock,mode=0 -d
+}
+
+shell -err -expect "Mode sub-arg -1 out of range" {
+	varnishd -a ${tmpdir}/vtc.sock,mode=-1 -d
+}
+
+##
+## user and group sub-args tested in c00086.vtc, where the user and group
+## features are enabled.
+##
+
+# Invalid sub-arg
+shell -err -expect "Invalid sub-arg foo=bar" {
+	varnishd -a ${tmpdir}/vtc.sock,foo=bar -d
+}
+
+# A sub-arg without '=' is interpreted as a protocol name.
+shell -err -expect "Unknown protocol" {
+	varnishd -a ${tmpdir}/vtc.sock,foobar -d
+}
+
+shell -err -expect "Invalid sub-arg userfoo=u" {
+	varnishd -a ${tmpdir}/vtc.sock,userfoo=u -d
+}
+
+shell -err -expect "Invalid sub-arg groupfoo=g" {
+	varnishd -a ${tmpdir}/vtc.sock,groupfoo=g -d
+}
+
+shell -err -expect "Invalid sub-arg modefoo=666" {
+	varnishd -a ${tmpdir}/vtc.sock,modefoo=666 -d
+}
+
+shell -err -expect "Invalid sub-arg =foo" {
+	varnishd -a ${tmpdir}/vtc.sock,=foo -d
+}
+
 # This requires non-local binds to be disabled.  If you see this fail
 # and are on Linux, ensure /proc/net/ipv4/ip_nonlocal_bind is set to 0.
 
diff --git a/bin/varnishtest/tests/c00086.vtc b/bin/varnishtest/tests/c00086.vtc
new file mode 100644
index 0000000..e149437
--- /dev/null
+++ b/bin/varnishtest/tests/c00086.vtc
@@ -0,0 +1,53 @@
+varnishtest "-a sub-args user, group and mode"
+
+feature user_vcache
+feature group_varnish
+feature root
+
+shell -err -expect "Too many user sub-args" {
+	varnishd -a ${tmpdir}/vtc.sock,user=vcache,user=vcache -d
+}
+
+shell -err -expect "Too many group sub-args" {
+	varnishd -a ${tmpdir}/vtc.sock,group=varnish,group=varnish -d
+}
+
+# Assuming that empty user and group names always fail getpwnam and getgrnam
+shell -err -expect "Unknown user " {
+	varnishd -a ${tmpdir}/vtc.sock,user= -d
+}
+
+shell -err -expect "Unknown group " {
+	varnishd -a ${tmpdir}/vtc.sock,group= -d
+}
+
+server s1 {} -start
+
+varnish v1 -arg "-a ${tmpdir}/v1.sock,user=vcache,group=varnish,mode=660" \
+	-vcl+backend {}
+
+shell -match "rw-rw----.+vcache.+varnish" { ls -l ${tmpdir}/v1.sock }
+
+varnish v2 -arg "-a ${tmpdir}/v2.sock,user=vcache,mode=600" -vcl+backend {}
+
+shell -match "rw-------.+vcache" { ls -l ${tmpdir}/v2.sock }
+
+varnish v3 -arg "-a ${tmpdir}/v3.sock,group=varnish,mode=660" -vcl+backend {}
+
+shell -match "rw---- .+root.+varnish" { ls -l ${tmpdir}/v3.sock }
+
+varnish v4 -arg "-a ${tmpdir}/v4.sock,mode=666" -vcl+backend {}
+
+shell -match "rw-rw-rw-.+root" { ls -l ${tmpdir}/v4.sock }
+
+varnish v5 -arg "-a ${tmpdir}/v5.sock,user=vcache,group=varnish" -vcl+backend {}
+
+shell -match "vcache.+varnish" { ls -l ${tmpdir}/v5.sock }
+
+varnish v6 -arg "-a ${tmpdir}/v6.sock,user=vcache" -vcl+backend {}
+
+shell -match "vcache" { ls -l ${tmpdir}/v6.sock }
+
+varnish v7 -arg "-a ${tmpdir}/v7.sock,group=varnish" -vcl+backend {}
+
+shell -match "root.+varnish" { ls -l ${tmpdir}/v7.sock }
diff --git a/bin/varnishtest/tests/c00087.vtc b/bin/varnishtest/tests/c00087.vtc
new file mode 100644
index 0000000..5f55cdf
--- /dev/null
+++ b/bin/varnishtest/tests/c00087.vtc
@@ -0,0 +1,116 @@
+varnishtest "VCL *.ip vars as bogo-IPs when -a is UDS, and test ACL matches"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl+backend {
+	acl acl1 {
+		"${localhost}";
+	}
+
+	sub vcl_backend_response {
+		set beresp.http.b-client = client.ip;
+		set beresp.http.b-server = server.ip;
+		set beresp.http.b-local = local.ip;
+		set beresp.http.b-remote = remote.ip;
+		set beresp.http.b-compare = local.ip == remote.ip;
+	}
+
+	sub vcl_deliver {
+		set resp.http.c-client = client.ip;
+		set resp.http.c-server = server.ip;
+		set resp.http.c-local = local.ip;
+		set resp.http.c-remote = remote.ip;
+		set resp.http.c-compare = local.ip == remote.ip;
+		set resp.http.client_acl = client.ip ~ acl1;
+		set resp.http.server_acl = server.ip ~ acl1;
+		set resp.http.local_acl = local.ip ~ acl1;
+		set resp.http.remote_acl = remote.ip ~ acl1;
+	}
+} -start
+
+client c1 -connect "${tmpdir}/v1.sock" {
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.http.c-client == "0.0.0.0"
+	expect resp.http.c-server == "0.0.0.0"
+	expect resp.http.c-local == "0.0.0.0"
+	expect resp.http.c-remote == "0.0.0.0"
+	expect resp.http.c-compare == "true"
+	expect resp.http.b-client == "0.0.0.0"
+	expect resp.http.b-server == "0.0.0.0"
+	expect resp.http.b-local == "0.0.0.0"
+	expect resp.http.b-remote == "0.0.0.0"
+	expect resp.http.b-compare == "true"
+	expect resp.http.client_acl == "false"
+	expect resp.http.server_acl == "false"
+	expect resp.http.local_acl == "false"
+	expect resp.http.remote_acl == "false"
+} -run
+
+logexpect l1 -v v1 -d 1 -g vxid -q "VCL_acl" {
+	expect 0 * Begin req
+	expect * = VCL_acl "^NO_MATCH acl1$"
+	expect * = VCL_acl "^NO_MATCH acl1$"
+	expect * = VCL_acl "^NO_MATCH acl1$"
+	expect * = VCL_acl "^NO_MATCH acl1$"
+	expect * = End
+} -run
+
+varnish v1 -vcl {
+	backend b { .host = "${bad_ip}"; }
+
+	acl acl1 {
+		"0.0.0.0";
+	}
+
+	sub vcl_recv {
+		return(synth(200));
+	}
+
+	sub vcl_synth {
+		set resp.http.client = client.ip ~ acl1;
+		set resp.http.server = server.ip ~ acl1;
+		set resp.http.local = local.ip ~ acl1;
+		set resp.http.remote = remote.ip ~ acl1;
+	}
+}
+
+client c1 -connect "${tmpdir}/v1.sock" {
+	txreq -url "foo"
+	rxresp
+	expect resp.http.client == "true"
+	expect resp.http.server == "true"
+	expect resp.http.local == "true"
+	expect resp.http.remote == "true"
+} -run
+
+varnish v1 -errvcl {.../mask is not numeric.} {
+	backend b { .host = "${bad_ip}"; }
+
+	acl acl1 {
+		"${tmpdir}/v1.sock";
+	}
+}
+
+# client.ip == 0.0.0.0 appears in X-Forwarded-For
+server s1 {
+	rxreq
+	expect req.http.X-Forwarded-For == "0.0.0.0"
+	txresp
+	rxreq
+	expect req.http.X-Forwarded-For == "1.2.3.4, 0.0.0.0"
+	txresp
+} -start
+
+varnish v1 -vcl+backend {}
+
+client c1 -connect "${tmpdir}/v1.sock" {
+	txreq -url /1
+	rxresp
+	txreq -url /2 -hdr "X-forwarded-for: 1.2.3.4"
+	rxresp
+} -run
diff --git a/bin/varnishtest/tests/j00004.vtc b/bin/varnishtest/tests/j00004.vtc
new file mode 100644
index 0000000..a23f7d7
--- /dev/null
+++ b/bin/varnishtest/tests/j00004.vtc
@@ -0,0 +1,42 @@
+varnishtest "Listen at a Unix domain socket while in jail"
+
+feature user_varnish
+feature group_varnish
+feature root
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -arg "-a ${tmpdir}/v1.sock" \
+	-jail "-junix,user=varnish,ccgroup=varnish" \
+	-vcl+backend {
+} -start
+
+# Socket is created as management owner before the child goes to jail
+shell -match "root" { ls -l ${tmpdir}/v1.sock }
+
+client c1 -connect "${tmpdir}/v1.sock" {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v2 -arg "-a ${tmpdir}/v2.sock,user=varnish,group=varnish,mode=666" \
+	-jail "-junix,user=varnish,ccgroup=varnish" \
+	-vcl+backend {
+} -start
+
+shell -match "rw-rw-rw-.+varnish.+varnish" { ls -l ${tmpdir}/v2.sock }
+
+client c1 -connect "${tmpdir}/v2.sock" {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run
diff --git a/bin/varnishtest/tests/v00054.vtc b/bin/varnishtest/tests/v00054.vtc
new file mode 100644
index 0000000..a230fb8
--- /dev/null
+++ b/bin/varnishtest/tests/v00054.vtc
@@ -0,0 +1,28 @@
+varnishtest "client.identity is 0.0.0.0 if unset & client addr is UDS"
+
+varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl {
+	backend b { .host = "${bad_ip}"; }
+
+	sub vcl_recv {
+		if (req.url == "/nobody") {
+			set client.identity = "Samuel B. Nobody";
+		}
+		set req.http.id = client.identity;
+		return(synth(200));
+	}
+
+	sub vcl_synth {
+		set resp.http.id = req.http.id;
+	}
+} -start
+
+client c1 -connect "${tmpdir}/v1.sock" {
+	txreq -url "/nobody"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.id == "Samuel B. Nobody"
+	txreq
+	rxresp
+	expect resp.status == 200
+	expect resp.http.id == "0.0.0.0"
+} -run
diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c
index 8db014c..ea33c5a 100644
--- a/bin/varnishtest/vtc_client.c
+++ b/bin/varnishtest/vtc_client.c
@@ -179,9 +179,12 @@ client_uds_connect(struct vtclog *vl, const char *path, double tmo,
 
 	assert(tmo >= 0);
 
+	errno = 0;
 	fd = VUS_resolver(path, uds_open, &tmo, errp);
-	if (fd < 0)
+	if (fd < 0) {
+		*errp = strerror(errno);
 		return fd;
+	}
 	vtc_log(vl, 3, "connected fd %d to %s", fd, path);
 	return fd;
 }
diff --git a/include/vsa.h b/include/vsa.h
index c28a7e9..70e4e50 100644
--- a/include/vsa.h
+++ b/include/vsa.h
@@ -32,7 +32,9 @@
 
 struct suckaddr;
 extern const int vsa_suckaddr_len;
+extern const struct suckaddr *bogo_ip;
 
+void VSA_Init(void);
 int VSA_Sane(const struct suckaddr *);
 unsigned VSA_Port(const struct suckaddr *);
 int VSA_Compare(const struct suckaddr *, const struct suckaddr *);
diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c
index c2e157b..1fc496f 100644
--- a/lib/libvarnish/vsa.c
+++ b/lib/libvarnish/vsa.c
@@ -36,7 +36,9 @@
 #include <string.h>
 #include <stdlib.h>
 #include <sys/socket.h>
+#include <sys/types.h>
 #include <netinet/in.h>
+#include <netdb.h>
 
 #include "vdef.h"
 #include "vas.h"
@@ -175,6 +177,27 @@ struct suckaddr {
 const int vsa_suckaddr_len = sizeof(struct suckaddr);
 
 /*
+ * Bogus IPv4 address 0.0.0.0:0 to be used for VCL *.ip variables when the
+ * "real" address is not IP (such as UDS addresses).
+ */
+static struct suckaddr bogo_ip_vsa;
+const struct suckaddr *bogo_ip = &bogo_ip_vsa;
+
+void
+VSA_Init()
+{
+	struct addrinfo hints, *res = NULL;
+
+	memset(&hints, 0, sizeof(hints));
+	hints.ai_family = AF_INET;
+	hints.ai_flags = AI_NUMERICHOST | AI_NUMERICSERV;
+	AZ(getaddrinfo("0.0.0.0", "0", &hints, &res));
+	AN(VSA_Build(&bogo_ip_vsa, res->ai_addr, res->ai_addrlen));
+	assert(VSA_Sane(bogo_ip));
+	freeaddrinfo(res);
+}
+
+/*
  * This VRT interface is for the VCC generated ACL code, which needs
  * to know the address family and a pointer to the actual address.
  */

From geoff at uplex.de  Thu Feb 22 09:23:09 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Thu, 22 Feb 2018 09:23:09 +0000 (UTC)
Subject: [master] 234ef63 CLI debug.listen_address emits "/path -" when the
 address is UDS.
Message-ID: <20180222092309.DA64ABFBFB@lists.varnish-cache.org>


commit 234ef63f0b2889d7ad80e0816dfd87633233ccfd
Author: Geoff Simmons <geoff at uplex.de>
Date:   Sat Feb 17 10:26:40 2018 +0100

    CLI debug.listen_address emits "/path -" when the address is UDS.
    
    This determines the values of the vtc macros vN_addr, _port and _sock.

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index b54c99a..44b89de 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -673,8 +673,12 @@ ccf_listen_address(struct cli *cli, const char * const *av, void *priv)
 
 	AZ(pthread_mutex_lock(&shut_mtx));
 	VTAILQ_FOREACH(ls, &heritage.socks, list) {
-		VTCP_myname(ls->sock, h, sizeof h, p, sizeof p);
-		VCLI_Out(cli, "%s %s\n", h, p);
+		if (!ls->uds) {
+			VTCP_myname(ls->sock, h, sizeof h, p, sizeof p);
+			VCLI_Out(cli, "%s %s\n", h, p);
+		}
+		else
+			VCLI_Out(cli, "%s -\n", ls->endpoint);
 	}
 	AZ(pthread_mutex_unlock(&shut_mtx));
 }
diff --git a/bin/varnishtest/tests/a00019.vtc b/bin/varnishtest/tests/a00019.vtc
new file mode 100644
index 0000000..864ab21
--- /dev/null
+++ b/bin/varnishtest/tests/a00019.vtc
@@ -0,0 +1,25 @@
+varnishtest "vtc v_* macros when the listen address is UDS"
+
+varnish v1 -arg "-a ${tmpdir}/v1.sock -b '${bad_backend}'" -start
+
+varnish v2 -vcl {
+	backend default { .host = "${bad_ip}"; }
+
+	sub vcl_recv {
+		return(synth(200));
+	}
+
+	sub vcl_synth {
+		set resp.http.addr = "${v1_addr}";
+		set resp.http.port = "${v1_port}";
+		set resp.http.sock = "${v1_sock}";
+	}
+} -start
+
+client c1 -connect ${v2_sock} {
+	txreq
+	rxresp
+	expect resp.http.addr == "${tmpdir}/v1.sock"
+	expect resp.http.port == "-"
+	expect resp.http.sock == "${tmpdir}/v1.sock -"
+} -run

From geoff at uplex.de  Thu Feb 22 09:23:09 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Thu, 22 Feb 2018 09:23:09 +0000 (UTC)
Subject: [master] 2e9682e std.set_ip_tos() is silently ignored if local.ip is
 the bogo-ip, hence if the listen address is UDS.
Message-ID: <20180222092309.F2D37BFBFF@lists.varnish-cache.org>


commit 2e9682ea2f26a292151867803e9596602e7854fa
Author: Geoff Simmons <geoff at uplex.de>
Date:   Sat Feb 17 11:10:30 2018 +0100

    std.set_ip_tos() is silently ignored if local.ip is the bogo-ip,
    hence if the listen address is UDS.
    
    Otherwise, the setsockopt() call will fail VTCP_Assert().
    
    Also, verify that std.ip() and std.port() don't work with UDS.

diff --git a/bin/varnishtest/tests/m00046.vtc b/bin/varnishtest/tests/m00046.vtc
new file mode 100644
index 0000000..fc0945b
--- /dev/null
+++ b/bin/varnishtest/tests/m00046.vtc
@@ -0,0 +1,47 @@
+varnishtest "std.ip(), .port() and .set_ip_tos() don't work for UDS addresses"
+
+varnish v1 -arg "-a ${tmpdir}/v1.sock -b '${bad_backend}'" -start
+
+varnish v2 -arg "-a ${tmpdir}/v2.sock" -vcl {
+	import std;
+	backend default { .host = "${bad_ip}"; }
+
+	sub vcl_recv {
+		return(synth(200));
+	}
+
+	sub vcl_synth {
+		set resp.http.v1_addr = std.ip("${v1_addr}", client.ip);
+	}
+} -start
+
+client c1 -connect "${tmpdir}/v2.sock" {
+	txreq
+	rxresp
+	expect resp.http.v1_addr == "0.0.0.0"
+} -run
+
+varnish v2 -errvcl {IP constant '"${v1_addr}"' could not be resolved to an IP address:} {
+	import std;
+
+	sub vcl_recv {
+		set resp.http.v1_port = std.port("${v1_addr}");
+	}
+}
+
+varnish v2 -vcl {
+	import std;
+	backend default { .host = "${bad_ip}"; }
+
+	sub vcl_recv {
+	    # Silently ignored for a UDS
+	    std.set_ip_tos(0);
+	    return(synth(200));
+	}
+}
+
+client c1 -connect "${tmpdir}/v2.sock" {
+	txreq
+	rxresp
+	expect resp.status == 200
+} -run
diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c
index efd6d84..fdf1227 100644
--- a/lib/libvmod_std/vmod_std.c
+++ b/lib/libvmod_std/vmod_std.c
@@ -53,9 +53,14 @@
 VCL_VOID v_matchproto_(td_std_set_ip_tos)
 vmod_set_ip_tos(VRT_CTX, VCL_INT tos)
 {
+	struct suckaddr *sa;
 	int itos = tos;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	AZ(SES_Get_local_addr(ctx->req->sp, &sa));
+	/* Silently ignore for non-IP addresses. */
+	if (VSA_Compare(sa, bogo_ip) == 0)
+		return;
 	VTCP_Assert(setsockopt(ctx->req->sp->fd,
 	    IPPROTO_IP, IP_TOS, &itos, sizeof(itos)));
 }

From geoff at uplex.de  Thu Feb 22 09:23:10 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Thu, 22 Feb 2018 09:23:10 +0000 (UTC)
Subject: [master] 5c23234 Update panic output for the session client address
 when it's a UDS.
Message-ID: <20180222092310.17754BFC03@lists.varnish-cache.org>


commit 5c23234b1fca450a15d84894e7ffab3237bc1307
Author: Geoff Simmons <geoff at uplex.de>
Date:   Sun Feb 18 16:41:34 2018 +0100

    Update panic output for the session client address when it's a UDS.

diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index 90050ac..4c194bb 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -519,7 +519,11 @@ pan_sess(struct vsb *vsb, const struct sess *sp)
 	VSB_printf(vsb, "\n");
 	ci = SES_Get_String_Attr(sp, SA_CLIENT_IP);
 	cp = SES_Get_String_Attr(sp, SA_CLIENT_PORT);
-	VSB_printf(vsb, "client = %s %s,\n", ci, cp);
+	if (VALID_OBJ(sp->listen_sock, LISTEN_SOCK_MAGIC))
+		VSB_printf(vsb, "client = %s %s %s,\n", ci, cp,
+			   sp->listen_sock->endpoint);
+	else
+		VSB_printf(vsb, "client = %s %s <unknown>\n", ci, cp);
 
 	VSB_indent(vsb, -2);
 	VSB_printf(vsb, "},\n");

From geoff at uplex.de  Thu Feb 22 09:23:10 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Thu, 22 Feb 2018 09:23:10 +0000 (UTC)
Subject: [master] fbbcf28 Add the vtc variable remote.path -- the path for a
 UDS.
Message-ID: <20180222092310.39905BFC08@lists.varnish-cache.org>


commit fbbcf28b3ff90ca8f804b49d0590801d887c6416
Author: Geoff Simmons <geoff at uplex.de>
Date:   Sun Feb 18 17:08:28 2018 +0100

    Add the vtc variable remote.path -- the path for a UDS.
    
    NULL for an IP address (matches <undef>).
    
    Also verify that remote.ip and remote.port correspond to the
    bogo-IP 0.0.0.0:0 for a UDS connection.

diff --git a/bin/varnishtest/tests/a00020.vtc b/bin/varnishtest/tests/a00020.vtc
new file mode 100644
index 0000000..aba786e
--- /dev/null
+++ b/bin/varnishtest/tests/a00020.vtc
@@ -0,0 +1,34 @@
+varnishtest "vtc remote.ip, remote.port and remote.path"
+
+server s1 {
+	rxreq
+	expect remote.ip == "${localhost}"
+	expect remote.port > 0
+	expect remote.path == <undef>
+	txresp
+} -start
+
+varnish v1 -vcl+backend {} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect remote.ip == "${v1_addr}"
+	expect remote.port == "${v1_port}"
+	expect remote.path == <undef>
+} -run
+
+varnish v1 -stop
+
+server s1 -wait
+server s1 -start
+
+varnish v2 -arg "-a ${tmpdir}/v2.sock" -vcl+backend {} -start
+
+client c1 -connect "${tmpdir}/v2.sock" {
+	txreq
+	rxresp
+	expect remote.ip == "0.0.0.0"
+	expect remote.port == 0
+	expect remote.path == "${tmpdir}/v2.sock"
+} -run
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 62df449..8a9f911 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -277,6 +277,7 @@ http_count_header(char * const *hh, const char *hdr)
  *
  *         - remote.ip
  *         - remote.port
+ *         - remote.path
  *         - req.method
  *         - req.url
  *         - req.proto
@@ -300,6 +301,8 @@ cmd_var_resolve(struct http *hp, char *spec)
 		return(hp->rem_ip);
 	if (!strcmp(spec, "remote.port"))
 		return(hp->rem_port);
+	if (!strcmp(spec, "remote.path"))
+		return(hp->rem_path);
 	if (!strcmp(spec, "req.method"))
 		return(hp->req[0]);
 	if (!strcmp(spec, "req.url"))
@@ -1871,12 +1874,15 @@ http_process(struct vtclog *vl, const char *spec, int sock, int *sfd,
 	hp->gziplevel = 0;
 	hp->gzipresidual = -1;
 
-	if (*addr != '/')
+	if (*addr != '/') {
 		VTCP_hisname(sock, hp->rem_ip, VTCP_ADDRBUFSIZE, hp->rem_port,
 			     VTCP_PORTBUFSIZE);
+		hp->rem_path = NULL;
+	}
 	else {
 		strcpy(hp->rem_ip, "0.0.0.0");
 		strcpy(hp->rem_port, "0");
+		hp->rem_path = strdup(addr);
 	}
 	parse_string(spec, http_cmds, hp, vl);
 	if (hp->h2)
diff --git a/bin/varnishtest/vtc_http.h b/bin/varnishtest/vtc_http.h
index f944f36..a6c5a95 100644
--- a/bin/varnishtest/vtc_http.h
+++ b/bin/varnishtest/vtc_http.h
@@ -14,6 +14,7 @@ struct http {
 	char			*rxbuf;
 	char			*rem_ip;
 	char			*rem_port;
+	char			*rem_path;
 	int			prxbuf;
 	char			*body;
 	unsigned		bodyl;

From geoff at uplex.de  Thu Feb 22 09:23:10 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Thu, 22 Feb 2018 09:23:10 +0000 (UTC)
Subject: [master] b849683 Add a test for HTTP/2 over Unix domain sockets.
Message-ID: <20180222092310.50642BFC0B@lists.varnish-cache.org>


commit b849683397632efadf15f694be765bc4ea673bc4
Author: Geoff Simmons <geoff at uplex.de>
Date:   Sun Feb 18 17:22:25 2018 +0100

    Add a test for HTTP/2 over Unix domain sockets.

diff --git a/bin/varnishtest/tests/t02013.vtc b/bin/varnishtest/tests/t02013.vtc
new file mode 100644
index 0000000..47a226c
--- /dev/null
+++ b/bin/varnishtest/tests/t02013.vtc
@@ -0,0 +1,58 @@
+varnishtest	"Direct H2 start over Unix domain sockets"
+
+server s1 {
+	rxreq
+	expect req.http.host == foo.bar
+	txresp \
+		-hdr "H234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789I: foo" \
+		-hdr "Foo: H234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789I" \
+		-bodylen 10
+} -start
+
+varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl+backend {
+	sub vcl_recv {
+		return (pipe);
+	}
+} -start
+
+varnish v1 -cliok "param.set debug +syncvsl"
+
+varnish v1 -cliok "param.set feature -http2"
+
+client c1 -connect "${tmpdir}/v1.sock" {
+	txpri
+	expect_close
+} -run
+
+varnish v1 -cliok "param.set feature +http2"
+
+client c1 -connect "${tmpdir}/v1.sock" {
+	stream 1 {
+		txprio -weight 10 -stream 0
+	} -run
+	stream 3 {
+		txprio -weight 10 -stream 0
+	} -run
+	stream 5 {
+		txprio -weight 10 -stream 2
+	} -run
+	stream 7 {
+		txreq -dep 3 -hdr :authority foo.bar -pad cotton
+		rxresp
+		expect resp.status == 200
+		delay 1
+		txrst -err 0x1111
+	} -start
+	stream 0 {
+		txping -data "_-__-_-_"
+		rxping
+		expect ping.ack == "true"
+		expect ping.data == "_-__-_-_"
+	} -run
+	stream 7 -wait
+} -run
+
+varnish v1 -expect MEMPOOL.req0.live == 0
+varnish v1 -expect MEMPOOL.req1.live == 0
+varnish v1 -expect MEMPOOL.sess0.live == 0
+varnish v1 -expect MEMPOOL.sess1.live == 0

From geoff at uplex.de  Thu Feb 22 09:23:10 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Thu, 22 Feb 2018 09:23:10 +0000 (UTC)
Subject: [master] 0def0b6 Verify outputs of varnishncsa -c when UDS addresses
 are in use.
Message-ID: <20180222092310.6788FBFC0F@lists.varnish-cache.org>


commit 0def0b6aaf99b70ddf852bd2eb14b14685d8bcf0
Author: Geoff Simmons <geoff at uplex.de>
Date:   Wed Feb 21 15:26:57 2018 +0100

    Verify outputs of varnishncsa -c when UDS addresses are in use.

diff --git a/bin/varnishtest/tests/u00013.vtc b/bin/varnishtest/tests/u00013.vtc
new file mode 100644
index 0000000..1d9b47d
--- /dev/null
+++ b/bin/varnishtest/tests/u00013.vtc
@@ -0,0 +1,24 @@
+varnishtest "varnishncsa outputs when UDS addresses are in use"
+
+# The -c %h formatter gets its value from ReqStart, which now may be
+# 0.0.0.0 for a UDS address.
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl+backend {} -start
+
+client c1 -connect "${tmpdir}/v1.sock" {
+	txreq
+	rxresp
+} -run
+
+shell -expect "0.0.0.0" {
+	varnishncsa -n ${v1_name} -d -c -F "%h"
+}
+
+shell -expect "http://localhost/" {
+	varnishncsa -n ${v1_name} -d -c -F "%r"
+}

From geoff at uplex.de  Thu Feb 22 10:12:07 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Thu, 22 Feb 2018 10:12:07 +0000 (UTC)
Subject: [master] 147dfaf Error messages from Resolve_Sockaddr(UDS) differ on
 different platforms.
Message-ID: <20180222101207.9D75D64B1A@lists.varnish-cache.org>


commit 147dfafdec75ce6983b09c5c56619fd08a507f72
Author: Geoff Simmons <geoff at uplex.de>
Date:   Thu Feb 22 11:07:24 2018 +0100

    Error messages from Resolve_Sockaddr(UDS) differ on different platforms.
    
    So don't make the test for the error depend on the specific message.
    
    In this case, because getaddrinfo(3) may or may not resolve to a
    sockaddr_un for path if it happens to be a socket (it does on FreeBSD,
    doesn't on Linux).

diff --git a/bin/varnishtest/tests/m00046.vtc b/bin/varnishtest/tests/m00046.vtc
index fc0945b..a5de275 100644
--- a/bin/varnishtest/tests/m00046.vtc
+++ b/bin/varnishtest/tests/m00046.vtc
@@ -21,7 +21,7 @@ client c1 -connect "${tmpdir}/v2.sock" {
 	expect resp.http.v1_addr == "0.0.0.0"
 } -run
 
-varnish v2 -errvcl {IP constant '"${v1_addr}"' could not be resolved to an IP address:} {
+varnish v2 -errvcl {IP constant '"${v1_addr}"'} {
 	import std;
 
 	sub vcl_recv {

From daghf at varnish-software.com  Thu Feb 22 12:40:11 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Thu, 22 Feb 2018 12:40:11 +0000 (UTC)
Subject: [4.1] bf752ef Avoid leaking an OH ref on reembark failure
Message-ID: <20180222124011.A59FD969DC@lists.varnish-cache.org>


commit bf752eff529f8b1569a2caa5b4a8af9d9499e49c
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Thu Feb 22 13:29:38 2018 +0100

    Avoid leaking an OH ref on reembark failure
    
    This is a backport of 5cc47eaa8.
    
    With this commit hsh_rush has been split into hsh_rush and
    hsh_rush_clean. The former needs to be called while holding the OH lock,
    and the latter needs to be called without holding the lock.
    
    The reason for this added complexity is that we can't hold the lock
    while calling HSH_DerefObjHead.
    
    Fixes: #2495

diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c
index 5f657f9..30e2a06 100644
--- a/bin/varnishd/cache/cache_hash.c
+++ b/bin/varnishd/cache/cache_hash.c
@@ -65,6 +65,12 @@
 static const struct hash_slinger *hash;
 static struct objhead *private_oh;
 
+struct rush {
+	unsigned		magic;
+#define RUSH_MAGIC		0xa1af5f01
+	VTAILQ_HEAD(,req)	reqs;
+};
+
 /*---------------------------------------------------------------------*/
 
 static struct objcore *
@@ -524,15 +530,16 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp,
  */
 
 static void
-hsh_rush(struct worker *wrk, struct objhead *oh)
+hsh_rush(struct worker *wrk, struct objhead *oh, struct rush *r)
 {
 	unsigned u;
 	struct req *req;
-	struct sess *sp;
 	struct waitinglist *wl;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+	CHECK_OBJ_NOTNULL(r, RUSH_MAGIC);
+	assert(VTAILQ_EMPTY(&r->reqs));
 	Lck_AssertHeld(&oh->mtx);
 	wl = oh->waitinglist;
 	CHECK_OBJ_ORNULL(wl, WAITINGLIST_MAGIC);
@@ -547,21 +554,14 @@ hsh_rush(struct worker *wrk, struct objhead *oh)
 		AZ(req->wrk);
 		VTAILQ_REMOVE(&wl->list, req, w_list);
 		DSL(DBG_WAITINGLIST, req->vsl->wid, "off waiting list");
-		if (SES_Reschedule_Req(req)) {
+		if (DO_DEBUG(DBG_FAILRESCHED) || SES_Reschedule_Req(req)) {
 			/*
 			 * In case of overloads, we ditch the entire
 			 * waiting list.
 			 */
 			wrk->stats->busy_wakeup--;
 			while (1) {
-				wrk->stats->busy_killed++;
-				AN (req->vcl);
-				VCL_Rel(&req->vcl);
-				sp = req->sp;
-				CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
-				CNT_AcctLogCharge(wrk->stats, req);
-				Req_Release(req);
-				SES_Delete(sp, SC_OVERLOAD, NAN);
+				VTAILQ_INSERT_TAIL(&r->reqs, req, w_list);
 				req = VTAILQ_FIRST(&wl->list);
 				if (req == NULL)
 					break;
@@ -580,6 +580,36 @@ hsh_rush(struct worker *wrk, struct objhead *oh)
 	}
 }
 
+static void
+hsh_rush_clean(struct worker *wrk, struct rush *r)
+{
+	struct req *req;
+	struct sess *sp;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	CHECK_OBJ_NOTNULL(r, RUSH_MAGIC);
+
+	while (!VTAILQ_EMPTY(&r->reqs)) {
+		req = VTAILQ_FIRST(&r->reqs);
+		CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+		VTAILQ_REMOVE(&r->reqs, req, w_list);
+		wrk->stats->busy_killed++;
+		AN (req->vcl);
+		VCL_Rel(&req->vcl);
+		sp = req->sp;
+		CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+		CNT_AcctLogCharge(wrk->stats, req);
+		CHECK_OBJ_NOTNULL(req->hash_objhead, OBJHEAD_MAGIC);
+		AN(HSH_DerefObjHead(wrk, &req->hash_objhead));
+		AZ(req->hash_objhead);
+		Req_Release(req);
+		SES_Delete(sp, SC_OVERLOAD, NAN);
+		DSL(DBG_WAITINGLIST, req->vsl->wid,
+		    "kill from waiting list");
+	}
+
+}
+
 /*---------------------------------------------------------------------
  * Purge an entire objhead
  */
@@ -719,11 +749,14 @@ void
 HSH_Unbusy(struct worker *wrk, struct objcore *oc)
 {
 	struct objhead *oh;
+	struct rush rush;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	oh = oc->objhead;
 	CHECK_OBJ(oh, OBJHEAD_MAGIC);
+	INIT_OBJ(&rush, RUSH_MAGIC);
+	VTAILQ_INIT(&rush.reqs);
 
 	AN(oc->stobj->stevedore);
 	AN(oc->flags & OC_F_BUSY);
@@ -744,8 +777,9 @@ HSH_Unbusy(struct worker *wrk, struct objcore *oc)
 	VTAILQ_INSERT_HEAD(&oh->objcs, oc, list);
 	oc->flags &= ~OC_F_BUSY;
 	if (oh->waitinglist != NULL)
-		hsh_rush(wrk, oh);
+		hsh_rush(wrk, oh, &rush);
 	Lck_Unlock(&oh->mtx);
+	hsh_rush_clean(wrk, &rush);
 }
 
 /*---------------------------------------------------------------------
@@ -800,6 +834,7 @@ HSH_DerefObjCore(struct worker *wrk, struct objcore **ocp)
 {
 	struct objcore *oc;
 	struct objhead *oh;
+	struct rush rush;
 	unsigned r;
 
 	AN(ocp);
@@ -812,6 +847,8 @@ HSH_DerefObjCore(struct worker *wrk, struct objcore **ocp)
 
 	oh = oc->objhead;
 	CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+	INIT_OBJ(&rush, RUSH_MAGIC);
+	VTAILQ_INIT(&rush.reqs);
 
 	Lck_Lock(&oh->mtx);
 	assert(oh->refcnt > 0);
@@ -819,8 +856,9 @@ HSH_DerefObjCore(struct worker *wrk, struct objcore **ocp)
 	if (!r)
 		VTAILQ_REMOVE(&oh->objcs, oc, list);
 	if (oh->waitinglist != NULL)
-		hsh_rush(wrk, oh);
+		hsh_rush(wrk, oh, &rush);
 	Lck_Unlock(&oh->mtx);
+	hsh_rush_clean(wrk, &rush);
 	if (r != 0)
 		return (r);
 
@@ -842,6 +880,7 @@ int
 HSH_DerefObjHead(struct worker *wrk, struct objhead **poh)
 {
 	struct objhead *oh;
+	struct rush rush;
 	int r;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -867,12 +906,15 @@ HSH_DerefObjHead(struct worker *wrk, struct objhead **poh)
 	 * just make the hold the same ref's as objcore, that would
 	 * confuse hashers.
 	 */
+	INIT_OBJ(&rush, RUSH_MAGIC);
+	VTAILQ_INIT(&rush.reqs);
 	while (oh->waitinglist != NULL) {
 		Lck_Lock(&oh->mtx);
 		assert(oh->refcnt > 0);
 		r = oh->refcnt;
-		hsh_rush(wrk, oh);
+		hsh_rush(wrk, oh, &rush);
 		Lck_Unlock(&oh->mtx);
+		hsh_rush_clean(wrk, &rush);
 		if (r > 1)
 			break;
 		usleep(100000);
diff --git a/bin/varnishtest/tests/c00013.vtc b/bin/varnishtest/tests/c00013.vtc
index 689e118..d89b198 100644
--- a/bin/varnishtest/tests/c00013.vtc
+++ b/bin/varnishtest/tests/c00013.vtc
@@ -1,4 +1,4 @@
-varnishtest "Test parking second request on backend delay"
+varnishtest "Test parking second request on backend delay (waitinglist)"
 
 server s1 {
 	rxreq
@@ -45,3 +45,60 @@ client c1 -wait
 
 varnish v1 -expect busy_sleep == 1
 varnish v1 -expect busy_wakeup == 1
+
+##################################################
+# Now try again where getting a thread fails
+
+barrier b3 cond 2
+barrier b4 cond 2
+
+server s3 {
+	rxreq
+	expect req.url == "/foo"
+	send "HTTP/1.0 200 OK\r\nConnection: close\r\n\r\n"
+	delay .2
+	barrier b3 sync
+	delay .2
+	send "line1\n"
+	delay .2
+	barrier b4 sync
+	send "line2\n"
+} -start
+
+varnish v3 -vcl+backend {
+	sub vcl_backend_fetch {
+		set bereq.backend = s3;
+	}
+	sub vcl_backend_response {
+		set beresp.do_stream = false;
+	}
+} -start
+
+varnish v3 -cliok "param.set debug +failresched"
+
+varnish v3 -cliok "param.set debug +syncvsl"
+
+client c3 -connect ${v3_sock} {
+	txreq -url "/foo" -hdr "client: c3"
+	rxresp
+	expect resp.status == 200
+	expect resp.bodylen == 12
+	expect resp.http.x-varnish == "1001"
+} -start
+
+barrier b3 sync
+
+client c4 -connect ${v3_sock} {
+	txreq -url "/foo" -hdr "client: c4"
+	delay .2
+	barrier b4 sync
+	expect_close
+} -run
+
+client c3 -wait
+
+varnish v1 -vsl_catchup
+varnish v3 -expect busy_sleep >= 1
+varnish v3 -expect busy_wakeup == 0
+varnish v3 -expect busy_killed == 1
+varnish v3 -expect sc_overload == 1
diff --git a/include/tbl/debug_bits.h b/include/tbl/debug_bits.h
index fa0d32f..4b3d74c 100644
--- a/include/tbl/debug_bits.h
+++ b/include/tbl/debug_bits.h
@@ -43,4 +43,5 @@ DEBUG_BIT(FLUSH_HEAD,		flush_head,	"Flush after http1 head")
 DEBUG_BIT(VTC_MODE,		vtc_mode,	"Varnishtest Mode")
 DEBUG_BIT(WITNESS,		witness,	"Emit WITNESS lock records")
 DEBUG_BIT(VSM_KEEP,		vsm_keep,	"Keep the VSM file on restart")
+DEBUG_BIT(FAILRESCHED,		failresched,    "Fail from waiting list")
 /*lint -restore */

From daghf at varnish-software.com  Thu Feb 22 14:57:07 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Thu, 22 Feb 2018 14:57:07 +0000 (UTC)
Subject: [4.1] df90615 Stabilize r01764.vtc
Message-ID: <20180222145707.6A7909A6A8@lists.varnish-cache.org>


commit df906159f2fe3393f176d095fd3e61f6f6e9e371
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Thu Feb 22 15:54:33 2018 +0100

    Stabilize r01764.vtc
    
    The non-fatal ("non_fatal" in master) was omitted when this was first
    backported.

diff --git a/bin/varnishtest/tests/r01764.vtc b/bin/varnishtest/tests/r01764.vtc
index ba428ef..07bdb32 100644
--- a/bin/varnishtest/tests/r01764.vtc
+++ b/bin/varnishtest/tests/r01764.vtc
@@ -18,6 +18,7 @@ server s1 {
 	expect req.url == /url4
 	txresp -bodylen 200000
 
+	non-fatal
 	rxreq
 	expect req.url == /url5
 	txresp -bodylen 1000000

From geoff at uplex.de  Thu Feb 22 15:11:08 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Thu, 22 Feb 2018 15:11:08 +0000 (UTC)
Subject: [master] b89a495 Set the bogo-IP's socket type to SOCK_STREAM,
 to assuage Solaris.
Message-ID: <20180222151108.D98479ACCF@lists.varnish-cache.org>


commit b89a4959651b66751f9a952da766282a0bdc6180
Author: Geoff Simmons <geoff at uplex.de>
Date:   Thu Feb 22 16:09:38 2018 +0100

    Set the bogo-IP's socket type to SOCK_STREAM, to assuage Solaris.

diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c
index 1fc496f..1c05520 100644
--- a/lib/libvarnish/vsa.c
+++ b/lib/libvarnish/vsa.c
@@ -191,6 +191,7 @@ VSA_Init()
 	memset(&hints, 0, sizeof(hints));
 	hints.ai_family = AF_INET;
 	hints.ai_flags = AI_NUMERICHOST | AI_NUMERICSERV;
+	hints.ai_socktype = SOCK_STREAM;
 	AZ(getaddrinfo("0.0.0.0", "0", &hints, &res));
 	AN(VSA_Build(&bogo_ip_vsa, res->ai_addr, res->ai_addrlen));
 	assert(VSA_Sane(bogo_ip));

From phk at FreeBSD.org  Fri Feb 23 08:18:14 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 23 Feb 2018 08:18:14 +0000 (UTC)
Subject: [master] 561b653 EPIPE is a documented errno in tcp(7) on linux
Message-ID: <20180223081814.CF7DFB4B90@lists.varnish-cache.org>


commit 561b653523ba9361cea4d9c1841853cc647f13bd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Feb 23 07:57:12 2018 +0000

    EPIPE is a documented errno in tcp(7) on linux
    
    Fixes: #2582

diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index fb320ce..93c2fce 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -567,7 +567,7 @@ VTCP_Check(int a)
 {
 	if (a == 0)
 		return (1);
-	if (errno == ECONNRESET || errno == ENOTCONN)
+	if (errno == ECONNRESET || errno == ENOTCONN || errno == EPIPE)
 		return (1);
 #if (defined (__SVR4) && defined (__sun)) || defined (__NetBSD__)
 	/*
@@ -575,7 +575,7 @@ VTCP_Check(int a)
 	 * connection.
 	 * This is a bug in Solaris and documented behaviour on NetBSD.
 	 */
-	if (errno == EINVAL || errno == ETIMEDOUT || errno == EPIPE)
+	if (errno == EINVAL || errno == ETIMEDOUT)
 		return (1);
 #elif defined (__APPLE__)
 	/*

From geoff at uplex.de  Fri Feb 23 13:46:06 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Fri, 23 Feb 2018 13:46:06 +0000 (UTC)
Subject: [master] 76ab7d5 Document the use of UDS for -a addresses in man
 varnishd(1).
Message-ID: <20180223134606.DAC4CBADA9@lists.varnish-cache.org>


commit 76ab7d588f4d739400849c42ea2dbfdad7529909
Author: Geoff Simmons <geoff at uplex.de>
Date:   Fri Feb 23 14:44:41 2018 +0100

    Document the use of UDS for -a addresses in man varnishd(1).

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 7b1e3eb..c2c0c88 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -15,7 +15,7 @@ HTTP accelerator daemon
 SYNOPSIS
 ========
 
-varnishd [-a [name=][address][:port][,PROTO]] [-b host[:port]] [-C] [-d] [-F] [-f config] [-h type[,options]] [-I clifile] [-i identity] [-j jail[,jailoptions]] [-l vsl[,vsm]] [-M address:port] [-n name] [-P file] [-p param=value] [-r param[,param...]] [-S secret-file] [-s [name=]kind[,options]] [-T address[:port]] [-t TTL] [-V] [-W waiter]
+varnishd [-a [name=][address][:port][,PROTO][,user=<user>][,group=<group>][,mode=<mode>]] [-b host[:port]] [-C] [-d] [-F] [-f config] [-h type[,options]] [-I clifile] [-i identity] [-j jail[,jailoptions]] [-l vsl[,vsm]] [-M address:port] [-n name] [-P file] [-p param=value] [-r param[,param...]] [-S secret-file] [-s [name=]kind[,options]] [-T address[:port]] [-t TTL] [-V] [-W waiter]
 
 varnishd [-x parameter|vsl|cli|builtin|optstring]
 
@@ -36,14 +36,22 @@ OPTIONS
 Basic options
 -------------
 
--a <[name=][address][:port][,PROTO]>
+-a <[name=][address][:port][,PROTO][,user=<user>][,group=<group>][,mode=<mode>]>
 
   Listen for client requests on the specified address and port. The
   address can be a host name ("localhost"), an IPv4 dotted-quad
-  ("127.0.0.1"), or an IPv6 address enclosed in square brackets
-  ("[::1]"). If address is not specified, `varnishd` will listen
-  on all available IPv4 and IPv6 interfaces. If port is not specified,
-  port 80 (http) is used. At least one of address or port is required.
+  ("127.0.0.1"), an IPv6 address enclosed in square brackets
+  ("[::1]"), or a path beginning with a '/' for a Unix domain socket
+  ("/path/to/listen.sock"). If address is not specified, `varnishd`
+  will listen on all available IPv4 and IPv6 interfaces. If port is
+  not specified, port 80 (http) is used. At least one of address or
+  port is required.
+
+  If a Unix domain socket is specified as the listen address, then the
+  user, group and mode sub-arguments may be used to specify the
+  permissions of the socket file -- use names for user and group, and
+  a 3-digit octal value for mode. These sub-arguments are not
+  permitted if an IP address is specified.
 
   Name is referenced in logs. If name is not specified, "a0", "a1",
   etc. is used. An additional protocol type can be set for the

From geoff at uplex.de  Fri Feb 23 14:04:08 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Fri, 23 Feb 2018 14:04:08 +0000 (UTC)
Subject: [master] 483648a Document UDS -a addresses in the usage output of
 varnishd.
Message-ID: <20180223140408.9199DBB410@lists.varnish-cache.org>


commit 483648ace33e4630449abc436d1be30abb63ea70
Author: Geoff Simmons <geoff at uplex.de>
Date:   Fri Feb 23 15:02:24 2018 +0100

    Document UDS -a addresses in the usage output of varnishd.

diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 275f6fb..eb1ef4d 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -94,9 +94,12 @@ usage(void)
 
 	printf(FMT, "-a address[:port][,proto]",
 	    "HTTP listen address and port");
-	printf(FMT, "", "Can be specified multiple times.");
-	printf(FMT, "", "  default: \":80,HTTP\"");
+	printf(FMT, "   [,user=<u>][,group=<g>]",
+	    "Can be specified multiple times.");
+	printf(FMT, "   [,mode=<m>]", "  default: \":80,HTTP\"");
 	printf(FMT, "", "Proto can be \"PROXY\" or \"HTTP\" (default)");
+	printf(FMT, "", "user, group and mode set permissions for");
+	printf(FMT, "", "  a Unix domain socket.");
 	printf(FMT, "-b address[:port]", "Backend address and port");
 	printf(FMT, "", "  default: \":80\"");
 	printf(FMT, "-f vclfile", "VCL program");

From dridi.boukelmoune at gmail.com  Fri Feb 23 15:18:09 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 23 Feb 2018 15:18:09 +0000 (UTC)
Subject: [master] 5384df7 Don't test gunzip for partial responses
Message-ID: <20180223151809.3AFDBBCC37@lists.varnish-cache.org>


commit 5384df74abd77a54d03727e847e8610c7af068a8
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jan 9 13:29:17 2018 +0100

    Don't test gunzip for partial responses
    
    Some user agents like Safari may "probe" specific resources like medias
    before getting the full resources usually asking for the first 2 or 11
    bytes, probably to peek at magic numbers to figure early whether a
    potentially large resource may not be supported (read: video).
    
    If the user agent also advertises gzip support, and the transaction is
    known beforehand to not be cacheable, varnishd will forward the Range
    header to the backend:
    
        Accept-Encoding: gzip (when http_gzip_support is on)
        Range: bytes=0-1
    
    If the response happens to be both encoded and partial, the gunzip test
    cannot be performed. Otherwise we systematically end up with a broken
    transaction closed prematuraly:
    
        FetchError b tGunzip failed
        Gzip b u F - 2 0 0 0 0
    
    Refs #2530
    Refs #2554

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index c31d88f..9faf4ee 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -501,6 +501,8 @@ vbf_stp_fetchbody(struct worker *wrk, struct busyobj *bo)
 static int
 vbf_figure_out_vfp(struct busyobj *bo)
 {
+	unsigned is_partial;
+
 	/*
 	 * The VCL variables beresp.do_g[un]zip tells us how we want the
 	 * object processed before it is stored.
@@ -514,7 +516,7 @@ vbf_figure_out_vfp(struct busyobj *bo)
 	 *
 	 */
 
-	/* No body or no GZIP supprt -> done */
+	/* No body or no GZIP support -> done */
 	if (bo->htc->body_status == BS_NONE ||
 	    bo->htc->content_length == 0 ||
 	    !cache_param->http_gzip_support) {
@@ -524,6 +526,7 @@ vbf_figure_out_vfp(struct busyobj *bo)
 		return (0);
 	}
 
+	is_partial = http_GetStatus(bo->beresp) == 206;
 	bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
 	bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
 	assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
@@ -552,7 +555,7 @@ vbf_figure_out_vfp(struct busyobj *bo)
 	if (bo->do_gzip)
 		return (VFP_Push(bo->vfc, &VFP_gzip) == NULL ? -1 : 0);
 
-	if (bo->is_gzip && !bo->do_gunzip)
+	if (bo->is_gzip && !bo->do_gunzip && !is_partial)
 		return (VFP_Push(bo->vfc, &VFP_testgunzip) == NULL ? -1 : 0);
 
 	return (0);
diff --git a/bin/varnishtest/tests/r02530.vtc b/bin/varnishtest/tests/r02530.vtc
new file mode 100644
index 0000000..27b24b2
--- /dev/null
+++ b/bin/varnishtest/tests/r02530.vtc
@@ -0,0 +1,74 @@
+varnishtest "Don't test gunzip for partial responses"
+
+# The use of ETags is only here to ensure they aren't accidentally weakened.
+
+server s1 {
+	# pass'ed range request
+	rxreq
+	expect req.url == "/pass"
+	expect req.http.Accept-Encoding == gzip
+	expect req.http.Range == bytes=0-1
+	txresp -status 206 -nolen \
+		-hdr {ETag: "abc123"} \
+		-hdr "Accept-Ranges: bytes" \
+		-hdr "Content-Encoding: gzip" \
+		-hdr "Content-Length: 2" \
+		-hdr "Content-Range: bytes 0-1/*"
+	sendhex 1f8b
+
+	# unattended partial response
+	rxreq
+	expect req.url == "/miss"
+	expect req.http.Accept-Encoding == gzip
+	expect req.http.Range == <undef>
+	txresp -status 206 -nolen \
+		-hdr {ETag: "123abc"} \
+		-hdr "Accept-Ranges: bytes" \
+		-hdr "Content-Encoding: gzip" \
+		-hdr "Content-Length: 2" \
+		-hdr "Content-Range: bytes 0-1/*"
+	sendhex 1f8b
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		if (req.url == "/pass") {
+			return (pass);
+		}
+	}
+} -start
+
+client c1 {
+	txreq -url "/pass" -hdr "Accept-Encoding: gzip" -hdr "Range: bytes=0-1"
+	rxresp
+	expect resp.status == 206
+	expect resp.http.Etag == {"abc123"}
+	expect resp.http.Accept-Ranges == bytes
+	expect resp.http.Content-Range ~ "^bytes 0-1/"
+	expect resp.http.Content-Length == 2
+	expect resp.bodylen == 2
+} -run
+
+varnish v1 -expect n_gzip == 0
+varnish v1 -expect n_gunzip == 0
+varnish v1 -expect n_test_gunzip == 0
+varnish v1 -expect SMA.s0.c_req == 0
+varnish v1 -expect SMA.Transient.c_req == 2
+
+# Invalid partial response, also in Transient
+client c1 {
+	txreq -url "/miss" -hdr "Accept-Encoding: gzip" -hdr "Range: bytes=0-1"
+	rxresp
+	expect resp.status == 206
+	expect resp.http.Etag == {"123abc"}
+	expect resp.http.Accept-Ranges == bytes
+	expect resp.http.Content-Range ~ "^bytes 0-1/"
+	expect resp.http.Content-Length == 2
+	expect resp.bodylen == 2
+} -run
+
+varnish v1 -expect n_gzip == 0
+varnish v1 -expect n_gunzip == 0
+varnish v1 -expect n_test_gunzip == 0
+varnish v1 -expect SMA.s0.c_req == 0
+varnish v1 -expect SMA.Transient.c_req == 4

From dridi.boukelmoune at gmail.com  Fri Feb 23 15:23:07 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 23 Feb 2018 15:23:07 +0000 (UTC)
Subject: [4.1] 94a7f42 Don't test gunzip for partial responses
Message-ID: <20180223152307.A8426BCDC8@lists.varnish-cache.org>


commit 94a7f427c21d8bd0a91541b29141471c8cab220a
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Jan 9 13:29:17 2018 +0100

    Don't test gunzip for partial responses
    
    Some user agents like Safari may "probe" specific resources like medias
    before getting the full resources usually asking for the first 2 or 11
    bytes, probably to peek at magic numbers to figure early whether a
    potentially large resource may not be supported (read: video).
    
    If the user agent also advertises gzip support, and the transaction is
    known beforehand to not be cacheable, varnishd will forward the Range
    header to the backend:
    
        Accept-Encoding: gzip (when http_gzip_support is on)
        Range: bytes=0-1
    
    If the response happens to be both encoded and partial, the gunzip test
    cannot be performed. Otherwise we systematically end up with a broken
    transaction closed prematuraly:
    
        FetchError b tGunzip failed
        Gzip b u F - 2 0 0 0 0
    
    Refs #2530
    Refs #2554

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 70f953f..b356828 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -546,6 +546,7 @@ static enum fetch_step
 vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 {
 	const char *p;
+	unsigned is_partial;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
@@ -566,6 +567,8 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 	 *
 	 */
 
+	is_partial = http_GetStatus(bo->beresp) == 206;
+
 	/* We do nothing unless the param is set */
 	if (!cache_param->http_gzip_support)
 		bo->do_gzip = bo->do_gunzip = 0;
@@ -608,7 +611,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo)
 			vbf_vfp_push(bo, &vfp_esi, 1);
 		} else if (bo->do_gzip) {
 			vbf_vfp_push(bo, &vfp_gzip, 1);
-		} else if (bo->is_gzip && !bo->do_gunzip) {
+		} else if (bo->is_gzip && !bo->do_gunzip && !is_partial) {
 			vbf_vfp_push(bo, &vfp_testgunzip, 1);
 		}
 	}
diff --git a/bin/varnishtest/tests/r02530.vtc b/bin/varnishtest/tests/r02530.vtc
new file mode 100644
index 0000000..c74391a
--- /dev/null
+++ b/bin/varnishtest/tests/r02530.vtc
@@ -0,0 +1,72 @@
+varnishtest "Don't test gunzip for partial responses"
+
+# The use of ETags is only here to ensure they aren't accidentally weakened.
+
+server s1 {
+	# pass'ed range request
+	rxreq
+	expect req.url == "/pass"
+	expect req.http.Accept-Encoding == gzip
+	expect req.http.Range == bytes=0-1
+	txresp -status 206 -nolen \
+		-hdr {ETag: "abc123"} \
+		-hdr "Accept-Ranges: bytes" \
+		-hdr "Content-Encoding: gzip" \
+		-hdr "Content-Length: 2" \
+		-hdr "Content-Range: bytes 0-1/*"
+	sendhex 1f8b
+
+	# unattended partial response
+	rxreq
+	expect req.url == "/miss"
+	expect req.http.Accept-Encoding == gzip
+	expect req.http.Range == <undef>
+	txresp -status 206 -nolen \
+		-hdr {ETag: "123abc"} \
+		-hdr "Accept-Ranges: bytes" \
+		-hdr "Content-Encoding: gzip" \
+		-hdr "Content-Length: 2" \
+		-hdr "Content-Range: bytes 0-1/*"
+	sendhex 1f8b
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv {
+		if (req.url == "/pass") {
+			return (pass);
+		}
+	}
+} -start
+
+client c1 {
+	txreq -url "/pass" -hdr "Accept-Encoding: gzip" -hdr "Range: bytes=0-1"
+	rxresp
+	expect resp.status == 206
+	expect resp.http.Etag == {"abc123"}
+	expect resp.http.Accept-Ranges == bytes
+	expect resp.http.Content-Range ~ "^bytes 0-1/"
+	expect resp.http.Content-Length == 2
+	expect resp.bodylen == 2
+} -run
+
+varnish v1 -expect n_gzip == 0
+varnish v1 -expect n_gunzip == 0
+varnish v1 -expect SMA.s0.c_req == 0
+varnish v1 -expect SMA.Transient.c_req == 2
+
+# Invalid partial response, also in Transient
+client c1 {
+	txreq -url "/miss" -hdr "Accept-Encoding: gzip" -hdr "Range: bytes=0-1"
+	rxresp
+	expect resp.status == 206
+	expect resp.http.Etag == {"123abc"}
+	expect resp.http.Accept-Ranges == bytes
+	expect resp.http.Content-Range ~ "^bytes 0-1/"
+	expect resp.http.Content-Length == 2
+	expect resp.bodylen == 2
+} -run
+
+varnish v1 -expect n_gzip == 0
+varnish v1 -expect n_gunzip == 0
+varnish v1 -expect SMA.s0.c_req == 0
+varnish v1 -expect SMA.Transient.c_req == 4

From hermunn at varnish-software.com  Fri Feb 23 15:52:07 2018
From: hermunn at varnish-software.com (Pål Hermunn Johansen)
Date: Fri, 23 Feb 2018 15:52:07 +0000 (UTC)
Subject: [4.1] ef43067 Fix crash under MacOS while investigating #2332
Message-ID: <20180223155207.6C9C0BD610@lists.varnish-cache.org>


commit ef430670a5f2f11acb3d6d030844e4c4450370e2
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Sun Jun 4 16:28:16 2017 +0100

    Fix crash under MacOS while investigating #2332
    
    MacOS will return EINVAL under e.g. setsockopt if the connection was
    reset.

diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index d28b99c..79498c4 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -579,6 +579,13 @@ VTCP_Check(int a)
 	 */
 	if (errno == EINVAL || errno == ETIMEDOUT)
 		return (1);
+#elif defined (__APPLE__)
+	/*
+	 * MacOS returns EINVAL if the other end unexpectedly reset
+	 * the connection.
+	 */
+	if (errno == EINVAL)
+		return (1);
 #endif
 	return (0);
 }

From hermunn at varnish-software.com  Fri Feb 23 15:52:07 2018
From: hermunn at varnish-software.com (Pål Hermunn Johansen)
Date: Fri, 23 Feb 2018 15:52:07 +0000 (UTC)
Subject: [4.1] 24ac14d EPIPE is a documented errno in tcp(7) on linux
Message-ID: <20180223155207.8785FBD613@lists.varnish-cache.org>


commit 24ac14da6b7b3c5d91dca68c97c8f7ccd5481748
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Feb 23 07:57:12 2018 +0000

    EPIPE is a documented errno in tcp(7) on linux
    
    Fixes: #2582

diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c
index 79498c4..af8a147 100644
--- a/lib/libvarnish/vtcp.c
+++ b/lib/libvarnish/vtcp.c
@@ -569,7 +569,7 @@ VTCP_Check(int a)
 {
 	if (a == 0)
 		return (1);
-	if (errno == ECONNRESET || errno == ENOTCONN)
+	if (errno == ECONNRESET || errno == ENOTCONN || errno == EPIPE)
 		return (1);
 #if (defined (__SVR4) && defined (__sun)) || defined (__NetBSD__)
 	/*

From phk at FreeBSD.org  Fri Feb 23 21:44:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 23 Feb 2018 21:44:08 +0000 (UTC)
Subject: [master] 275800f Implement resp.do_esi
Message-ID: <20180223214408.CA220968A6@lists.varnish-cache.org>


commit 275800fdce33d13763ce0eb014e48db68ca5d8c3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Feb 23 21:42:02 2018 +0000

    Implement resp.do_esi

diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index 2ab2fdb..1b071c9 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -454,6 +454,7 @@ VRT_l_req_esi(VRT_CTX, unsigned process_esi)
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
+	assert(ctx->syntax <= 40);
 	/*
 	 * Only allow you to turn of esi in the main request
 	 * else everything gets confused
@@ -468,6 +469,7 @@ VRT_r_req_esi(VRT_CTX)
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
+	assert(ctx->syntax <= 40);
 	return (!ctx->req->disable_esi);
 }
 
@@ -728,6 +730,32 @@ VRT_r_resp_is_streaming(VRT_CTX)
 
 /*--------------------------------------------------------------------*/
 
+void
+VRT_l_resp_do_esi(VRT_CTX, unsigned process_esi)
+{
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
+	assert(ctx->syntax >= 41);
+	/*
+	 * Only allow you to turn of esi in the main request
+	 * else everything gets confused
+	 */
+	ctx->req->disable_esi = !process_esi;
+}
+
+unsigned
+VRT_r_resp_do_esi(VRT_CTX)
+{
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
+	assert(ctx->syntax >= 41);
+	return (!ctx->req->disable_esi);
+}
+
+/*--------------------------------------------------------------------*/
+
 #define VRT_BODY_L(which)					\
 void								\
 VRT_l_##which##_body(VRT_CTX, const char *str, ...)		\
diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst
index 9631ed3..3d17078 100644
--- a/doc/sphinx/reference/vcl_var.rst
+++ b/doc/sphinx/reference/vcl_var.rst
@@ -1065,6 +1065,20 @@ resp.http.*
 
 	
 	The HTTP headers that will be returned.
+
+resp.do_esi	``VCL >= 4.1``
+
+	Type: BOOL
+
+	Readable from: vcl_deliver, vcl_synth
+
+	Writable from: vcl_deliver, vcl_synth
+
+	Default: Set if ESI parsing has happened.
+
+	This can be used to selectively disable ESI processing,
+	even though ESI parsing happened during fetch.
+	This is useful when Varnish caches peer with each other.
 	
 
 resp.is_streaming

From phk at FreeBSD.org  Fri Feb 23 21:44:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 23 Feb 2018 21:44:08 +0000 (UTC)
Subject: [master] b042383 Test resp.do_esi
Message-ID: <20180223214408.E194E968AA@lists.varnish-cache.org>


commit b042383fcf81b6bac2071b9fec5e3750ff4338ae
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Feb 23 21:42:30 2018 +0000

    Test resp.do_esi

diff --git a/bin/varnishtest/tests/e00015.vtc b/bin/varnishtest/tests/e00015.vtc
index b394668..e8b47de 100644
--- a/bin/varnishtest/tests/e00015.vtc
+++ b/bin/varnishtest/tests/e00015.vtc
@@ -2,6 +2,7 @@ varnishtest "ESI requests turned off"
 
 server s1 {
 	rxreq
+	expect req.url == /
 	txresp -body {
 		<html>
 		Before include
@@ -42,7 +43,46 @@ client c1 {
 	rxresp
 	expect resp.bodylen == 76
 	expect resp.status == 200
+} -run
+
+varnish v1 -vsl_catchup
+
+varnish v1 -expect esi_errors == 0
+
+server s1 {
+	rxreq
+	expect req.url == /top2
+	txresp -body {
+		<html>
+		Before include
+		<esi:include src="/body"/>
+		After include
+	}
+} -start
+
+varnish v1 -syntax 4.1 -vcl+backend {
+	sub vcl_deliver {
+		set resp.http.was = resp.do_esi;
+		if (req.url == "/top2") {
+			set resp.do_esi = false;
+		}
+	}
+	sub vcl_backend_response {
+		set beresp.do_esi = true;
+	}
 }
 
-client c1 -run
+client c1 {
+	txreq -url /top2
+	rxresp
+	expect resp.bodylen == 73
+	expect resp.status == 200
+	expect resp.http.was == true
+	txreq -url "/esi"
+	rxresp
+	expect resp.bodylen == 76
+	expect resp.status == 200
+	expect resp.http.was == true
+} -run
+
 varnish v1 -expect esi_errors == 0

From phk at FreeBSD.org  Fri Feb 23 21:44:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 23 Feb 2018 21:44:09 +0000 (UTC)
Subject: [master] 5574f03 Make VCC report when symbols require a different
 syntax
Message-ID: <20180223214409.099B9968AE@lists.varnish-cache.org>


commit 5574f03f47c26c5d020385fda29aa90c66626e02
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Feb 23 21:42:38 2018 +0000

    Make VCC report when symbols require a different syntax

diff --git a/bin/varnishtest/tests/v00021.vtc b/bin/varnishtest/tests/v00021.vtc
index 8394ce8..871f873 100644
--- a/bin/varnishtest/tests/v00021.vtc
+++ b/bin/varnishtest/tests/v00021.vtc
@@ -121,3 +121,13 @@ varnish v1 -errvcl {Symbol 'default' is a reserved word.} {
 	}
 }
 
+varnish v1 -syntax 4.1 -errvcl {(Only available when VCL syntax <= 4.0)} {
+	sub vcl_recv {
+		set req.esi = false;
+	}
+}
+varnish v1 -syntax 4.0 -errvcl {(Only available when 4.1 <= VCL syntax)} {
+	sub vcl_deliver {
+		set resp.do_esi = false;
+	}
+}
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index 3d66e81..823a53f 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -220,24 +220,22 @@ VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x)
 	if (sym == NULL) {
 		VSB_printf(tl->sb, "%s: ", e);
 		vcc_ErrToken(tl, tl->t);
+		sym = VCC_Symbol(tl, NULL, tl->t->b, tl->t->e, kind, 0,
+			VCL_LOW, VCL_HIGH);
+		if (sym != NULL) {
+			VSB_printf(tl->sb, " (Only available when");
+			if (sym->lorev >= VCL_LOW)
+				VSB_printf(tl->sb, " %.1f <=", .1 * sym->lorev);
+			VSB_printf(tl->sb, " VCL syntax");
+			if (sym->hirev <= VCL_HIGH)
+				VSB_printf(tl->sb, " <= %.1f", .1 * sym->hirev);
+			VSB_printf(tl->sb, ")");
+		}
 		VSB_cat(tl->sb, "\nAt: ");
 		vcc_ErrWhere(tl, tl->t);
 		return (NULL);
 	}
-	if (sym->lorev > tl->syntax || sym->hirev < tl->syntax) {
-		VSB_printf(tl->sb, "Symbol ");
-		vcc_ErrToken(tl, tl->t);
-		if (sym->lorev > tl->syntax)
-			VSB_printf(tl->sb, " needs vcl %.1f or higher.",
-			    .1 * sym->lorev);
-		else
-			VSB_printf(tl->sb,
-			    " is discontinued after vcl %.1f.",
-			    .1 * sym->hirev);
-		VSB_cat(tl->sb, "\nAt: ");
-		vcc_ErrWhere(tl, tl->t);
-		return(NULL);
-	}
+	assert (sym->lorev <= tl->syntax && sym->hirev >= tl->syntax);
 	if (kind != SYM_NONE && kind != sym->kind) {
 		VSB_printf(tl->sb, "Symbol ");
 		vcc_ErrToken(tl, tl->t);

From phk at FreeBSD.org  Fri Feb 23 23:33:05 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 23 Feb 2018 23:33:05 +0000 (UTC)
Subject: [master] 9b785fd Give VMODs a wildcard toplevel symbol,
 and populate symbols on demand.
Message-ID: <20180223233305.3EE1298C27@lists.varnish-cache.org>


commit 9b785fd2f7e5b1f328f6ef0e17b7be07d0faccc7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Feb 23 23:31:21 2018 +0000

    Give VMODs a wildcard toplevel symbol, and populate symbols on demand.

diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index fe9253d..e2e252e 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -73,20 +73,16 @@ func_sym(struct symbol *sym, const char *vmod, const struct vjsn_val *v)
 }
 
 static void
-parse_json(struct vcc * tl, const char *vmod, const char *json)
+vcc_json_always(struct vcc *tl, const struct symbol *msym)
 {
 	struct inifin *ifp;
-	struct vjsn *vj;
-	struct vjsn_val *vv, *vv2;
+	const struct vjsn *vj;
+	const struct vjsn_val *vv, *vv2;
 	double vmod_syntax = 0.0;
-	const char *p;
-	struct symbol *sym;
 
 	ifp = NULL;
 
-	vj = vjsn_parse(json, &p);
-	XXXAZ(p);
-	AN(vj);
+	CAST_OBJ_NOTNULL(vj, msym->eval_priv, VJSN_MAGIC);
 
 	VTAILQ_FOREACH(vv, &vj->value->children, list) {
 		assert(vv->type == VJSN_ARRAY);
@@ -106,27 +102,15 @@ parse_json(struct vcc * tl, const char *vmod, const char *json)
 			VSB_printf(ifp->ini,
 			    "\tif (%s(ctx, &vmod_priv_%s, VCL_EVENT_LOAD))\n"
 			    "\t\treturn(1);",
-			    vv2->value, vmod);
+			    vv2->value, msym->name);
 			VSB_printf(ifp->fin,
 			    "\t\t(void)%s(ctx, &vmod_priv_%s,\n"
 			    "\t\t\t    VCL_EVENT_DISCARD);\n",
-			    vv2->value, vmod);
+			    vv2->value, msym->name);
 			VSB_printf(ifp->event, "%s(ctx, &vmod_priv_%s, ev)",
-			    vv2->value, vmod);
+			    vv2->value, msym->name);
 		} else if (!strcmp(vv2->value, "$FUNC")) {
-			vv2 = VTAILQ_NEXT(vv2, list);
-			sym = VCC_MkSym(tl,
-			    vv2->value, SYM_FUNC, VCL_LOW, VCL_HIGH);
-			ERRCHK(tl);
-			AN(sym);
-			func_sym(sym, vmod, VTAILQ_NEXT(vv2, list));
 		} else if (!strcmp(vv2->value, "$OBJ")) {
-			vv2 = VTAILQ_NEXT(vv2, list);
-			sym = VCC_MkSym(tl, vv2->value,
-			    SYM_OBJECT, VCL_LOW, VCL_HIGH);
-			XXXAN(sym);
-			sym->eval_priv = vv2;
-			sym->vmod = vmod;
 		} else {
 			VTAILQ_FOREACH(vv2, &vv->children, list)
 				fprintf(stderr, "\tt %s n %s v %s\n",
@@ -136,6 +120,37 @@ parse_json(struct vcc * tl, const char *vmod, const char *json)
 	}
 }
 
+static void v_matchproto_(sym_wildcard_t)
+vcc_json_wildcard(struct vcc *tl, struct symbol *msym, struct symbol *tsym)
+{
+	const struct vjsn *vj;
+	const struct vjsn_val *vv, *vv1, *vv2;
+
+	assert(msym->kind == SYM_VMOD);
+	CAST_OBJ_NOTNULL(vj, msym->eval_priv, VJSN_MAGIC);
+	VTAILQ_FOREACH(vv, &vj->value->children, list) {
+		assert(vv->type == VJSN_ARRAY);
+		vv1 = VTAILQ_FIRST(&vv->children);
+		assert(vv1->type == VJSN_STRING);
+		vv2 = VTAILQ_NEXT(vv1, list);
+		assert(vv2->type == VJSN_STRING);
+		if (!strcmp(vv1->value, "$FUNC") &&
+		    !strcmp(vv2->value, tsym->name)) {
+			tsym->kind = SYM_FUNC;
+			tsym->noref = 1;
+			func_sym(tsym, msym->name, VTAILQ_NEXT(vv2, list));
+			return;
+		} else if (!strcmp(vv1->value, "$OBJ") &&
+			   !strcmp(vv2->value, tsym->name)) {
+			tsym->kind = SYM_OBJECT;
+			tsym->eval_priv = vv2;
+			tsym->vmod = msym->name;
+			return;
+		}
+	}
+	tl->err = 1;
+}
+
 void
 vcc_ParseImport(struct vcc *tl)
 {
@@ -147,6 +162,7 @@ vcc_ParseImport(struct vcc *tl)
 	struct inifin *ifp;
 	struct symbol *msym;
 	const struct vmod_data *vmd;
+	struct vjsn *vj;
 
 	t1 = tl->t;
 	SkipToken(tl, ID);		/* "import" */
@@ -296,7 +312,13 @@ vcc_ParseImport(struct vcc *tl)
 	VSB_printf(ifp->fin, "\t\tVRT_priv_fini(&vmod_priv_%.*s);\n", PF(mod));
 	VSB_printf(ifp->fin, "\t\t\tVRT_Vmod_Fini(&VGC_vmod_%.*s);", PF(mod));
 
-	parse_json(tl, msym->name, vmd->json);
+	vj = vjsn_parse(vmd->json, &p);
+	XXXAZ(p);
+	AN(vj);
+	msym->eval_priv = vj;
+	msym->wildcard = vcc_json_wildcard;
+
+	vcc_json_always(tl, msym);
 
 	Fh(tl, 0, "\n/* --- BEGIN VMOD %.*s --- */\n\n", PF(mod));
 	Fh(tl, 0, "static struct vmod *VGC_vmod_%.*s;\n", PF(mod));
@@ -310,7 +332,6 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym)
 {
 	struct symbol *sy1, *sy2, *sy3;
 	struct inifin *ifp;
-	const char *s_obj;
 	char buf1[128];
 	char buf2[128];
 	const struct vjsn_val *vv, *vf;
@@ -343,7 +364,6 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym)
 
 	CAST_OBJ_NOTNULL(vv, sy2->eval_priv, VJSN_VAL_MAGIC);
 
-	s_obj = vv->value;
 	vv = VTAILQ_NEXT(vv, list);
 
 	Fh(tl, 0, "static %s *%s;\n\n", vv->value, sy1->rname);
@@ -383,7 +403,7 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym)
 		vf = VTAILQ_NEXT(vf, list);
 		assert(vf->type == VJSN_STRING);
 
-		bprintf(buf2, "%s%s", sy1->name, vf->value + strlen(s_obj));
+		bprintf(buf2, "%s.%s", sy1->name, vf->value);
 		sy3 = VCC_MkSym(tl, buf2, SYM_FUNC, VCL_LOW, VCL_HIGH);
 		AN(sy3);
 		func_sym(sy3, sy2->vmod, VTAILQ_NEXT(vf, list));
diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index b5ea510..a52d3e7 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -578,7 +578,7 @@ class s_function(stanza):
     def json(self,jl):
         jl.append([
                 "$FUNC",
-                "%s.%s" % (self.vcc.modname, self.proto.name),
+                "%s" % self.proto.name,
         ])
         self.proto.json(jl[-1], 'Vmod_%s_Func.%s' %
                            (self.vcc.modname, self.proto.cname()))
@@ -651,7 +651,7 @@ class s_object(stanza):
     def json(self, jl):
         ll = [
                 "$OBJ",
-                self.vcc.modname + "." + self.proto.name,
+                self.proto.name,
                 "struct %s%s_%s" %
                     (self.vcc.sympfx, self.vcc.modname, self.proto.name),
         ]
@@ -682,7 +682,8 @@ class s_method(stanza):
     def parse(self):
         p = self.vcc.contents[-1]
         assert type(p) == s_object
-        self.proto = prototype(self, prefix=p.proto.name)
+        self.pfx = p.proto.name
+        self.proto = prototype(self, prefix=self.pfx)
         self.rstlbl = "func_" + self.proto.name
         p.methods.append(self)
 
@@ -695,7 +696,7 @@ class s_method(stanza):
     def json(self, jl):
         jl.append([
                 "$METHOD",
-                self.vcc.modname + "." + self.proto.name
+                self.proto.name[len(self.pfx)+1:]
         ])
         self.proto.json(jl[-1],
             'Vmod_%s_Func.%s' % (self.vcc.modname, self.proto.cname()))

From nils.goroll at uplex.de  Sat Feb 24 10:07:11 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Sat, 24 Feb 2018 10:07:11 +0000 (UTC)
Subject: [master] 79e91c0 mark a tbd: For background fetches from vcl_hit,
 should we go to vcl_miss ?
Message-ID: <20180224100711.46751AE03A@lists.varnish-cache.org>


commit 79e91c03e2849ec5bc9b744a766507260c6801b5
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Sat Feb 24 10:59:09 2018 +0100

    mark a tbd: For background fetches from vcl_hit, should we go to vcl_miss ?
    
    There might be code in vcl_miss changing the request which we don't
    run for bgfetches, which could lead to unexpected behaviour.
    
    On the other hand, what purpose does vcl_miss serve? Is there anything
    we can do in vcl_miss which we can't do in vcl_backend_fetch?

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 56bf79e..385e899 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -529,6 +529,7 @@ cnt_lookup(struct worker *wrk, struct req *req)
 		if (busy != NULL) {
 			AZ(oc->flags & OC_F_PASS);
 			CHECK_OBJ_NOTNULL(busy->boc, BOC_MAGIC);
+			// XXX: shouldn't we go to miss?
 			VBF_Fetch(wrk, req, busy, oc, VBF_BACKGROUND);
 		} else {
 			(void)VRB_Ignore(req);// XXX: handle err

From phk at FreeBSD.org  Sat Feb 24 22:20:13 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 24 Feb 2018 22:20:13 +0000 (UTC)
Subject: [master] 15c3618 Flexelinting
Message-ID: <20180224222013.72849BB59C@lists.varnish-cache.org>


commit 15c3618f6a9e80ae37d1cd3281c1b62594091dc2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Feb 24 21:08:21 2018 +0000

    Flexelinting

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 44b89de..c41194d 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -221,7 +221,7 @@ vca_tcp_opt_init(void)
 }
 
 static void
-vca_tcp_opt_test(struct listen_sock *ls)
+vca_tcp_opt_test(const struct listen_sock *ls)
 {
 	int i, n, sock;
 	struct tcp_opt *to;
@@ -247,7 +247,7 @@ vca_tcp_opt_test(struct listen_sock *ls)
 }
 
 static void
-vca_tcp_opt_set(struct listen_sock *ls, int force)
+vca_tcp_opt_set(const struct listen_sock *ls, int force)
 {
 	int n, sock;
 	struct tcp_opt *to;
@@ -314,8 +314,8 @@ vca_pace_good(void)
  */
 
 static void
-vca_mk_tcp(struct wrk_accept *wa, struct sess *sp, char *laddr, char *lport,
-	   char *raddr, char *rport)
+vca_mk_tcp(const struct wrk_accept *wa,
+    struct sess *sp, char *laddr, char *lport, char *raddr, char *rport)
 {
 	struct suckaddr *sa;
 	struct sockaddr_storage ss;
@@ -345,7 +345,7 @@ vca_mk_uds(struct wrk_accept *wa, struct sess *sp, char *laddr, char *lport,
 
 	(void) wa;
 	SES_Reserve_remote_addr(sp, &sa);
-	SES_Set_remote_addr(sp, bogo_ip);
+	AZ(SES_Set_remote_addr(sp, bogo_ip));
 	sp->sattr[SA_CLIENT_ADDR] = sp->sattr[SA_REMOTE_ADDR];
 	sp->sattr[SA_LOCAL_ADDR] = sp->sattr[SA_REMOTE_ADDR];
 	sp->sattr[SA_SERVER_ADDR] = sp->sattr[SA_REMOTE_ADDR];

From phk at FreeBSD.org  Sat Feb 24 22:20:13 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 24 Feb 2018 22:20:13 +0000 (UTC)
Subject: [master] 68921c0 Flexelinting
Message-ID: <20180224222013.8E3E2BB59F@lists.varnish-cache.org>


commit 68921c021a019aeee89afd6fe953f2a10beaad07
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Feb 24 22:19:30 2018 +0000

    Flexelinting

diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt
index 923b702..0925a2b 100644
--- a/bin/varnishd/flint.lnt
+++ b/bin/varnishd/flint.lnt
@@ -1,5 +1,3 @@
--d__flexelint_v9__=1
-+fan
 // -w4
 
 -printf(3, VSL)
diff --git a/flint.lnt b/flint.lnt
index 94c1239..8b055f8 100644
--- a/flint.lnt
+++ b/flint.lnt
@@ -2,6 +2,9 @@
  * Toplevel control file for FlexeLint
  */
 
+//d__flexelint_v9__=1
++fan
+
 ///////////////////////////////////////////////////////////////////////
 // build/config related
 
diff --git a/lib/libvarnish/flint.lnt b/lib/libvarnish/flint.lnt
index 64d4dd2..7752265 100644
--- a/lib/libvarnish/flint.lnt
+++ b/lib/libvarnish/flint.lnt
@@ -5,9 +5,6 @@
 -efunc(662, VSHA256_Update)	// Possible creation of out-of-bounds pointer
 -efunc(662, be32dec_vect)	// Possible creation of out-of-bounds pointer
 -efunc(670, VSHA256_Update)	// Possible access beyond array for function '___',
-+fan			// Allow anon unions
-
--e713	// Loss of precision (assignment) (unsigned long long to long long)
 
 -dVARNISH_STATE_DIR="foo"
 
diff --git a/lib/libvmod_blob/hex.h b/lib/libvmod_blob/hex.h
index 9c0e344..53cfc89 100644
--- a/lib/libvmod_blob/hex.h
+++ b/lib/libvmod_blob/hex.h
@@ -28,7 +28,7 @@
 
 #include <stdint.h>
 
-#define ILL (0xff)
+#define ILL ((int8_t) 127)
 
 /* These are defined in hex.c */
 
diff --git a/lib/libvmod_blob/wb.c b/lib/libvmod_blob/wb.c
index 304b12e..1b54b3e 100644
--- a/lib/libvmod_blob/wb.c
+++ b/lib/libvmod_blob/wb.c
@@ -26,6 +26,8 @@
  * write buffer: utility functions to append-write on a varnish workspace
  */
 
+#include "config.h"
+
 #include <string.h>
 
 #include "cache/cache.h"

From hermunn at varnish-software.com  Sat Feb 24 22:56:06 2018
From: hermunn at varnish-software.com (Pål Hermunn Johansen)
Date: Sat, 24 Feb 2018 22:56:06 +0000 (UTC)
Subject: [4.1] ace6f49 Add n_lru_limited counter
Message-ID: <20180224225606.C3B3FBBFE9@lists.varnish-cache.org>


commit ace6f49dd458d67c71782752652ef9ef31f50700
Author: Shohei Tanaka(@xcir) <kokoniimasu+git at gmail.com>
Date:   Fri Feb 16 14:47:53 2018 +0900

    Add n_lru_limited counter
    
    This is a back port of the commits submitted in #2569 and merged in
    dc6c6520188fa6ffd5b9262cf1cbb86f8e5df4cc.

diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c
index 4172cea..28c79ab 100644
--- a/bin/varnishd/cache/cache_expire.c
+++ b/bin/varnishd/cache/cache_expire.c
@@ -331,6 +331,7 @@ EXP_NukeOne(struct worker *wrk, struct lru *lru)
 	CHECK_OBJ_NOTNULL(lru, LRU_MAGIC);
 	if (wrk->strangelove-- <= 0) {
 		VSLb(wrk->vsl, SLT_ExpKill, "LRU_Exhausted");
+		VSC_C_main->n_lru_limited++;
 		return (0);
 	}
 	/* Find the first currently unused object on the LRU.  */
diff --git a/bin/varnishtest/tests/r01764.vtc b/bin/varnishtest/tests/r01764.vtc
index 07bdb32..d79d23e 100644
--- a/bin/varnishtest/tests/r01764.vtc
+++ b/bin/varnishtest/tests/r01764.vtc
@@ -52,3 +52,5 @@ client c1 {
 	rxresp
 	expect resp.status == 503
 } -run
+
+varnish v1 -expect n_lru_limited == 1
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index 0da1989..710241f 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -322,6 +322,12 @@ VSC_F(n_lru_moved,		uint64_t, 0, 'g', 'i', diag,
 	"Number of move operations done on the LRU list."
 )
 
+VSC_F(n_lru_limited,		uint64_t, 0, 'c', 'i', info,
+    "Reached nuke_limit",
+	"Number of times more storage space were needed, but limit"
+	" was reached in a nuke_limit. See also parameter nuke_limit."
+)
+
 VSC_F(losthdr,			uint64_t, 0, 'c', 'i', info,
     "HTTP header overflows",
 	""

From geoff at uplex.de  Sun Feb 25 13:48:11 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Sun, 25 Feb 2018 13:48:11 +0000 (UTC)
Subject: [master] c785de1 Explicitly set malloc storage in the regression test
 for #2530.
Message-ID: <20180225134811.D4FE1A4DB4@lists.varnish-cache.org>


commit c785de16cbd96f20668f0b879c54db623f8c42f2
Author: Geoff Simmons <geoff at uplex.de>
Date:   Sun Feb 25 14:44:39 2018 +0100

    Explicitly set malloc storage in the regression test for #2530.
    
    Storage defaults to umem where libumem is available, as is usually
    the case on SunOS. So checking SMA.* stats was causing the test to
    fail on Solaris.

diff --git a/bin/varnishtest/tests/r02530.vtc b/bin/varnishtest/tests/r02530.vtc
index 27b24b2..dbdd4e6 100644
--- a/bin/varnishtest/tests/r02530.vtc
+++ b/bin/varnishtest/tests/r02530.vtc
@@ -30,7 +30,7 @@ server s1 {
 	sendhex 1f8b
 } -start
 
-varnish v1 -vcl+backend {
+varnish v1 -arg "-smalloc -sTransient=malloc" -vcl+backend {
 	sub vcl_recv {
 		if (req.url == "/pass") {
 			return (pass);

From nils.goroll at uplex.de  Sun Feb 25 13:54:06 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Sun, 25 Feb 2018 13:54:06 +0000 (UTC)
Subject: [master] 9aae4c6 that's what the vtc stats glob wildcards are for
Message-ID: <20180225135406.8C869A4FEE@lists.varnish-cache.org>


commit 9aae4c63fcad966d42fc1b936db3512f0c2564e8
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Sun Feb 25 14:52:45 2018 +0100

    that's what the vtc stats glob wildcards are for
    
    Ref c785de16cbd96f20668f0b879c54db623f8c42f2

diff --git a/bin/varnishtest/tests/r02530.vtc b/bin/varnishtest/tests/r02530.vtc
index dbdd4e6..c458b6a 100644
--- a/bin/varnishtest/tests/r02530.vtc
+++ b/bin/varnishtest/tests/r02530.vtc
@@ -30,7 +30,7 @@ server s1 {
 	sendhex 1f8b
 } -start
 
-varnish v1 -arg "-smalloc -sTransient=malloc" -vcl+backend {
+varnish v1 -vcl+backend {
 	sub vcl_recv {
 		if (req.url == "/pass") {
 			return (pass);
@@ -52,8 +52,8 @@ client c1 {
 varnish v1 -expect n_gzip == 0
 varnish v1 -expect n_gunzip == 0
 varnish v1 -expect n_test_gunzip == 0
-varnish v1 -expect SMA.s0.c_req == 0
-varnish v1 -expect SMA.Transient.c_req == 2
+varnish v1 -expect SM?.s0.c_req == 0
+varnish v1 -expect SM?.Transient.c_req == 2
 
 # Invalid partial response, also in Transient
 client c1 {
@@ -70,5 +70,5 @@ client c1 {
 varnish v1 -expect n_gzip == 0
 varnish v1 -expect n_gunzip == 0
 varnish v1 -expect n_test_gunzip == 0
-varnish v1 -expect SMA.s0.c_req == 0
-varnish v1 -expect SMA.Transient.c_req == 4
+varnish v1 -expect SM?.s0.c_req == 0
+varnish v1 -expect SM?.Transient.c_req == 4

From geoff at uplex.de  Sun Feb 25 18:22:08 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Sun, 25 Feb 2018 18:22:08 +0000 (UTC)
Subject: [master] e6b8660 Document the non-effect of TCP-related -p params on
 UDS in Upgrading to 6.0.
Message-ID: <20180225182208.82FFDA9F6F@lists.varnish-cache.org>


commit e6b8660a6e169239c89d19b029de3fcc5a7626d0
Author: Geoff Simmons <geoff at uplex.de>
Date:   Sun Feb 25 19:12:48 2018 +0100

    Document the non-effect of TCP-related -p params on UDS in
    Upgrading to 6.0.

diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst
index c5e15be..576f3da 100644
--- a/doc/sphinx/whats-new/upgrading-6.0.rst
+++ b/doc/sphinx/whats-new/upgrading-6.0.rst
@@ -12,7 +12,18 @@ XXX ...
 varnishd parameters
 ===================
 
-XXX: ... terse overview, use refs to link to details ...
+XXX: ...
+
+The parameters :ref:`ref_param_tcp_keepalive_intvl`,
+:ref:`ref_param_tcp_keepalive_probes` and
+:ref:`ref_param_tcp_keepalive_time` are silently ignored for listen
+address that are Unix domain sockets. The parameters
+:ref:`ref_param_accept_filter` and :ref:`ref_param_tcp_fastopen`
+(which your platform may or may not support in the first place) almost
+certainly have no effect on a UDS. It is not an error to use any of
+these parameters with a UDS; you may get error messages in the log for
+``accept_filter`` or ``tcp_fastopen`` (with the VSL tag ``Error`` in
+raw grouping), but they are harmless.
 
 Changes to VCL
 ==============

From geoff at uplex.de  Sun Feb 25 18:22:08 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Sun, 25 Feb 2018 18:22:08 +0000 (UTC)
Subject: [master] 9b8407b Document that the tcp_keep* params are ignored for
 UDSen.
Message-ID: <20180225182208.99FCFA9F72@lists.varnish-cache.org>


commit 9b8407b60fad8eeee802eb23c198490630a8973b
Author: Geoff Simmons <geoff at uplex.de>
Date:   Sun Feb 25 19:20:57 2018 +0100

    Document that the tcp_keep* params are ignored for UDSen.

diff --git a/include/tbl/params.h b/include/tbl/params.h
index 8aaef7d..829f48f 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -1091,7 +1091,8 @@ PARAM(
 	/* units */	"seconds",
 	/* flags */	XYZZY,
 	/* s-text */
-	"The number of seconds between TCP keep-alive probes.",
+	"The number of seconds between TCP keep-alive probes. "
+	"Ignored for Unix domain sockets.",
 	/* l-text */	"",
 	/* func */	NULL
 )
@@ -1107,7 +1108,7 @@ PARAM(
 	/* s-text */
 	"The maximum number of TCP keep-alive probes to send before giving "
 	"up and killing the connection if no response is obtained from the "
-	"other end.",
+	"other end. Ignored for Unix domain sockets.",
 	/* l-text */	"",
 	/* func */	NULL
 )
@@ -1122,7 +1123,8 @@ PARAM(
 	/* flags */	XYZZY,
 	/* s-text */
 	"The number of seconds a connection needs to be idle before TCP "
-	"begins sending out keep-alive probes.",
+	"begins sending out keep-alive probes. "
+	"Ignored for Unix domain sockets.",
 	/* l-text */	"",
 	/* func */	NULL
 )

From geoff at uplex.de  Sun Feb 25 18:27:05 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Sun, 25 Feb 2018 18:27:05 +0000 (UTC)
Subject: [master] a91ab31 Grammar
Message-ID: <20180225182705.D3798AC1DE@lists.varnish-cache.org>


commit a91ab31658a98caf4b566a47c3013246e3dca45c
Author: Geoff Simmons <geoff at uplex.de>
Date:   Sun Feb 25 19:25:46 2018 +0100

    Grammar

diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst
index 576f3da..0ccac58 100644
--- a/doc/sphinx/whats-new/upgrading-6.0.rst
+++ b/doc/sphinx/whats-new/upgrading-6.0.rst
@@ -17,7 +17,7 @@ XXX: ...
 The parameters :ref:`ref_param_tcp_keepalive_intvl`,
 :ref:`ref_param_tcp_keepalive_probes` and
 :ref:`ref_param_tcp_keepalive_time` are silently ignored for listen
-address that are Unix domain sockets. The parameters
+addresses that are Unix domain sockets. The parameters
 :ref:`ref_param_accept_filter` and :ref:`ref_param_tcp_fastopen`
 (which your platform may or may not support in the first place) almost
 certainly have no effect on a UDS. It is not an error to use any of

From hermunn at varnish-software.com  Sun Feb 25 20:40:07 2018
From: hermunn at varnish-software.com (Pål Hermunn Johansen)
Date: Sun, 25 Feb 2018 20:40:07 +0000 (UTC)
Subject: [4.1] 74b6547 Update changelog
Message-ID: <20180225204007.88AE4AE73E@lists.varnish-cache.org>


commit 74b654799151363708384901d4732b2c664eb207
Author: P?l Hermunn Johansen <hermunn at varnish-software.com>
Date:   Sun Feb 25 21:17:03 2018 +0100

    Update changelog

diff --git a/doc/changes.rst b/doc/changes.rst
index e1985b0..cee7a79 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -6,20 +6,32 @@ Changes since 4.1.9:
 
 * New counter added: cache_hit_grace (see 2455_)
 * Doc: Specify that time_firstbyte is in seconds (see 2456_)
+* Fix issue #1799 for keep (2519_)
+* New way of calculating TTLs in VCL (2555_)
+* Added n_lru_limited counter (see 2569_)
 
 .. _2455: https://github.com/varnishcache/varnish-cache/pull/2455
 .. _2456: https://github.com/varnishcache/varnish-cache/pull/2456
+.. _2519: https://github.com/varnishcache/varnish-cache/pull/2519
+.. _2555: https://github.com/varnishcache/varnish-cache/pull/2555
+.. _2569: https://github.com/varnishcache/varnish-cache/pull/2569
 
 Bugs fixed
 ----------
 
 * 1772_ - Honor first_byte_timeout for recycled backend connections
 * 2135_ - Limit Backend Connection retries to a single retry
+* 2495_ - Avoid leaking an OH ref on reembark failure
 * 2502_ - objcore reference count leak
+* 2530_ - Varnish shouldn't test gunzip for range responses
+* 2582_ - Assert error in http1_minimal_response()
 
 .. _1772: https://github.com/varnishcache/varnish-cache/issues/1772
 .. _2135: https://github.com/varnishcache/varnish-cache/pull/2135
+.. _2495: https://github.com/varnishcache/varnish-cache/issues/2495
 .. _2502: https://github.com/varnishcache/varnish-cache/issues/2502
+.. _2530: https://github.com/varnishcache/varnish-cache/issues/2530
+.. _2582: https://github.com/varnishcache/varnish-cache/issues/2582
 
 ================================
 Varnish Cache 4.1.9 (2017-11-14)

From phk at FreeBSD.org  Mon Feb 26 08:26:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 26 Feb 2018 08:26:08 +0000 (UTC)
Subject: [master] 08bf3c8 Fail if ESI is attempted on partial (206) objects.
 Ignore gzipery on same.
Message-ID: <20180226082608.E9B16BACF2@lists.varnish-cache.org>


commit 08bf3c8cc14495e02d20997fc16e4b5f166172e1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Feb 26 08:24:32 2018 +0000

    Fail if ESI is attempted on partial (206) objects.  Ignore gzipery on same.
    
    Document workaround for ESI (also works for gzipery).
    
    Fixes #2554

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 9faf4ee..d5e2548 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -501,7 +501,6 @@ vbf_stp_fetchbody(struct worker *wrk, struct busyobj *bo)
 static int
 vbf_figure_out_vfp(struct busyobj *bo)
 {
-	unsigned is_partial;
 
 	/*
 	 * The VCL variables beresp.do_g[un]zip tells us how we want the
@@ -514,8 +513,22 @@ vbf_figure_out_vfp(struct busyobj *bo)
 	 *	no Content-Encoding		--> object is not gzip'ed.
 	 *	anything else			--> do nothing wrt gzip
 	 *
+	 * On partial responses (206 on pass), we fail if do_esi is
+	 * requested because it could leak partial esi-directives, and
+	 * ignore gzipery, because it makes no sense.
+	 *
 	 */
 
+	if (http_GetStatus(bo->beresp) == 206) {
+		if (bo->do_esi) {
+			VSLb(bo->vsl, SLT_VCL_Error,
+			    "beresp.do_esi on partial response");
+			return (-1);
+		}
+		bo->do_gzip = bo->do_gunzip = 0;
+		return (0);
+	}
+
 	/* No body or no GZIP support -> done */
 	if (bo->htc->body_status == BS_NONE ||
 	    bo->htc->content_length == 0 ||
@@ -526,7 +539,6 @@ vbf_figure_out_vfp(struct busyobj *bo)
 		return (0);
 	}
 
-	is_partial = http_GetStatus(bo->beresp) == 206;
 	bo->is_gzip = http_HdrIs(bo->beresp, H_Content_Encoding, "gzip");
 	bo->is_gunzip = !http_GetHdr(bo->beresp, H_Content_Encoding, NULL);
 	assert(bo->is_gzip == 0 || bo->is_gunzip == 0);
@@ -555,7 +567,7 @@ vbf_figure_out_vfp(struct busyobj *bo)
 	if (bo->do_gzip)
 		return (VFP_Push(bo->vfc, &VFP_gzip) == NULL ? -1 : 0);
 
-	if (bo->is_gzip && !bo->do_gunzip && !is_partial)
+	if (bo->is_gzip && !bo->do_gunzip)
 		return (VFP_Push(bo->vfc, &VFP_testgunzip) == NULL ? -1 : 0);
 
 	return (0);
diff --git a/bin/varnishtest/tests/r02554.vtc b/bin/varnishtest/tests/r02554.vtc
new file mode 100644
index 0000000..672b766
--- /dev/null
+++ b/bin/varnishtest/tests/r02554.vtc
@@ -0,0 +1,58 @@
+varnishtest "VFP'ing partial responses"
+
+server s1 -repeat 4 {
+	rxreq
+	txresp -status 206 -body "<A><esi:remove>foobar</esi:remove>"
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_recv { return(pass); }
+	sub vcl_backend_response {
+		if (bereq.url == "/a") {
+			set beresp.do_esi = True;
+		}
+		if (bereq.url == "/b") {
+			set beresp.do_gzip = True;
+		}
+		if (bereq.url == "/c") {
+			set beresp.do_esi = True;
+			set beresp.do_gzip = True;
+			set beresp.status = 1206;
+		}
+	}
+} -start
+
+client c1 {
+	txreq -url /a
+	rxresp
+	expect resp.status == 503
+} -run
+
+varnish v1 -vsl_catchup
+
+client c1 {
+	txreq -url /b -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.bodylen == 34
+	expect resp.status == 206
+} -run
+
+varnish v1 -vsl_catchup
+
+client c1 {
+	txreq -url /b
+	rxresp
+	expect resp.bodylen == 34
+	expect resp.status == 206
+} -run
+
+varnish v1 -vsl_catchup
+
+client c1 {
+	txreq -url /c -hdr "Accept-Encoding: gzip"
+	rxresp
+	expect resp.status == 206
+	expect resp.bodylen == 32
+	gunzip
+	expect resp.bodylen == 3
+} -run
diff --git a/doc/sphinx/users-guide/esi.rst b/doc/sphinx/users-guide/esi.rst
index 0f16405..afa9236 100644
--- a/doc/sphinx/users-guide/esi.rst
+++ b/doc/sphinx/users-guide/esi.rst
@@ -4,8 +4,8 @@ Content composition with Edge Side Includes
 -------------------------------------------
 
 Varnish can create web pages by assembling different pages, called `fragments`,
-together into one page. These `fragments` can have individual cache policies. If you
-have a web site with a list showing the five most popular articles on
+together into one page. These `fragments` can have individual cache policies.
+If you have a web site with a list showing the five most popular articles on
 your site, this list can probably be cached as a `fragment` and included
 in all the other pages.
 
@@ -14,8 +14,9 @@ in all the other pages.
 Used properly this strategy can dramatically increase
 your hit rate and reduce the load on your servers.
 
-In Varnish we've only so far implemented a small subset of ESI. As of version 2.1 we
-have three ESI statements::
+In Varnish we've only implemented a small subset of ESI, because most of
+the rest of the ESI specifications facilities are easier and better done
+with VCL::
 
  esi:include
  esi:remove
@@ -81,10 +82,55 @@ For example::
   <esi:include src="http://example.com/LICENSE" />
   -->
 
+Footnotes about ESI
+-------------------
+
 Doing ESI on JSON and other non-XML'ish content
------------------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Varnish will peek at the first byte of an object and if it is not
+a "<" Varnish assumes you didn't really mean to ESI process.
+You can alter this behaviour by::
+
+   param.set feature +esi_disable_xml_check
+
+Ignoring BOM in ESI objects
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you backend spits out a Unicode Byte-Order-Mark as the first
+bytes of the reponse, the "<" check will fail unless you set::
+
+   param.set feature +esi_remove_bom
+
+ESI on invalid XML
+~~~~~~~~~~~~~~~~~~
+
+The ESI parser expects the XML to be reasonably well formed, but
+this may fail if you are ESI including non-XML files.  You can
+make the ESI parser disrecard anything but ESI tags by setting:
+
+   param.set feature +esi_ignore_other_elements
+
+ESI includes with HTTPS protocol
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If ESI:include tags specify HTTPS protocol, it will be ignored
+by default, because varnish has no way to fetch it encryption
+enabled.  If you want to treat HTTPS in ESI:include tags as if
+it were HTTP, set::
+
+   param.set feature +esi_ignore_https
+
+ESI on partial responses (206)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Varnish can ``pass`` range requests but it is ESI processing a partial
+response makes no sense, so the fetch fails if you do ask for that.
+If you really know what you are doing, you can use this workaround::
 
-Please note that Varnish will peek at the included content. If it
-doesn't start with a "<" Varnish assumes you didn't really mean to
-include it and disregard it. You can alter this behaviour by setting
-the 'esi_syntax' parameter (see ref:`ref-varnishd`).
+   sub vcl_backend_response {
+       if (beresp.status == 206 && beresp.http.secret == "swordfish") {
+           set beresp.do_esi = True;
+           set beresp.status = 1206;
+       }
+   }

From daghf at varnish-software.com  Mon Feb 26 13:12:22 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Mon, 26 Feb 2018 13:12:22 +0000 (UTC)
Subject: [master] bf1ea70 Add H/2 binary dumps to the default vsl_mask
Message-ID: <20180226131301.2EA7ABFF2C@lists.varnish-cache.org>


commit bf1ea70ea5471fc3a9fa83936b18ae1f85a88f97
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Mon Feb 26 14:08:45 2018 +0100

    Add H/2 binary dumps to the default vsl_mask
    
    In particular H2TxHdr, H2TxBody, H2RxHdr and H2RxBody will now be masked
    out by default.

diff --git a/bin/varnishd/mgt/mgt_param_bits.c b/bin/varnishd/mgt/mgt_param_bits.c
index b2edd05..ad90a12 100644
--- a/bin/varnishd/mgt/mgt_param_bits.c
+++ b/bin/varnishd/mgt/mgt_param_bits.c
@@ -127,6 +127,10 @@ tweak_vsl_mask(struct vsb *vsb, const struct parspec *par, const char *arg)
 			(void)bit(mgt_param.vsl_mask, SLT_WorkThread, BSET);
 			(void)bit(mgt_param.vsl_mask, SLT_Hash, BSET);
 			(void)bit(mgt_param.vsl_mask, SLT_VfpAcct, BSET);
+			(void)bit(mgt_param.vsl_mask, SLT_H2TxBody, BSET);
+			(void)bit(mgt_param.vsl_mask, SLT_H2TxHdr, BSET);
+			(void)bit(mgt_param.vsl_mask, SLT_H2RxBody, BSET);
+			(void)bit(mgt_param.vsl_mask, SLT_H2RxHdr, BSET);
 		} else {
 			return (bit_tweak(vsb, mgt_param.vsl_mask,
 			    SLT__Reserved, arg, VSL_tags,

From daghf at varnish-software.com  Mon Feb 26 13:19:09 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Mon, 26 Feb 2018 13:19:09 +0000 (UTC)
Subject: [master] ac8609f Add H/2 stream timeouts
Message-ID: <20180226131913.DEA77C0129@lists.varnish-cache.org>


commit ac8609f6ca53338fafd66216d603837d61c096a5
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Mon Feb 26 14:13:40 2018 +0100

    Add H/2 stream timeouts
    
    Avoid keeping H/2 streams alive indefinitely.
    
    This adds timeouts for:
      - How long we allow a stream to wait for a WINDOW_UPDATE, subject
        to idle_send_timeout
      - Total duration after we started transmitting a response,
        subject to send_timeout
    
    We currently do not make any attempt at sending a GOAWAY or a PING for
    client inactivity. This may be something to reconsider at a later point.

diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h
index 2883789..51b175b 100644
--- a/bin/varnishd/http2/cache_http2.h
+++ b/bin/varnishd/http2/cache_http2.h
@@ -121,6 +121,8 @@ struct h2_req {
 	enum h2_stream_e		state;
 	struct h2_sess			*h2sess;
 	struct req			*req;
+	double				t_send;
+	double				t_winupd;
 	pthread_cond_t			*cond;
 	VTAILQ_ENTRY(h2_req)		list;
 	int64_t				t_window;
diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c
index 46b0f38..a9f43e9 100644
--- a/bin/varnishd/http2/cache_http2_deliver.c
+++ b/bin/varnishd/http2/cache_http2_deliver.c
@@ -254,6 +254,8 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 	if (sendbody && req->resp_len == 0)
 		sendbody = 0;
 
+	r2->t_send = req->t_prev;
+
 	H2_Send_Get(req->wrk, r2->h2sess, r2);
 	H2_Send(req->wrk, r2, H2_F_HEADERS,
 	    (sendbody ? 0 : H2FF_HEADERS_END_STREAM) | H2FF_HEADERS_END_HEADERS,
diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 72d0342..f09bed9 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -889,6 +889,37 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2,
 	return (h2_tx_rst(wrk, h2, h2e));
 }
 
+static int
+h2_stream_tmo(struct h2_sess *h2, const struct h2_req *r2)
+{
+	int r = 0;
+
+	CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC);
+	CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC);
+
+	if (r2->t_winupd != 0 || r2->t_send != 0) {
+		Lck_Lock(&h2->sess->mtx);
+		if (r2->t_winupd != 0 &&
+		    h2->sess->t_idle - r2->t_winupd >
+		    cache_param->idle_send_timeout) {
+			VSLb(h2->vsl, SLT_Debug,
+			     "H2: stream %u: Hit idle_send_timeout waiting "
+			     "for WINDOW_UPDATE", r2->stream);
+			r = 1;
+		}
+
+		if (r == 0 && r2->t_send != 0 &&
+		    h2->sess->t_idle - r2->t_send > cache_param->send_timeout) {
+			VSLb(h2->vsl, SLT_Debug,
+			     "H2: stream %u: Hit send_timeout", r2->stream);
+			r = 1;
+		}
+		Lck_Unlock(&h2->sess->mtx);
+	}
+
+	return (r);
+}
+
 /***********************************************************************
  * Called in loop from h2_new_session()
  */
@@ -912,6 +943,7 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2)
 	h2_error h2e;
 	struct h2_req *r2, *r22;
 	char b[8];
+	int abort = 0;
 
 	ASSERT_RXTHR(h2);
 	(void)VTCP_blocking(*h2->htc->rfd);
@@ -925,6 +957,8 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2)
 		break;
 	case HTC_S_TIMEOUT:
 		VTAILQ_FOREACH_SAFE(r2, &h2->streams, list, r22) {
+			if (abort)
+				break;
 			switch (r2->state) {
 			case H2_S_CLOSED:
 				if (!r2->scheduled)
@@ -933,6 +967,10 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2)
 			case H2_S_OPEN:
 			case H2_S_CLOS_REM:
 			case H2_S_CLOS_LOC:
+				if (h2_stream_tmo(h2, r2)) {
+					abort = 1;
+					continue;
+				}
 				return (1);
 			default:
 				break;
diff --git a/bin/varnishd/http2/cache_http2_send.c b/bin/varnishd/http2/cache_http2_send.c
index a369df7..839193f 100644
--- a/bin/varnishd/http2/cache_http2_send.c
+++ b/bin/varnishd/http2/cache_http2_send.c
@@ -35,6 +35,7 @@
 #include "http2/cache_http2.h"
 
 #include "vend.h"
+#include "vtim.h"
 
 static void
 h2_send_get(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2)
@@ -204,15 +205,14 @@ h2_do_window(struct worker *wrk, struct h2_req *r2,
 
 	Lck_Lock(&h2->sess->mtx);
 	if (r2->t_window <= 0 || h2->req0->t_window <= 0) {
+		r2->t_winupd = VTIM_real();
 		h2_send_rel(h2, r2);
 		while (r2->t_window <= 0 && h2_errcheck(r2, h2) == 0) {
 			r2->cond = &wrk->cond;
-			// XXX: timeout handling (subject to send_timeout?)
 			AZ(Lck_CondWait(r2->cond, &h2->sess->mtx, 0));
 			r2->cond = NULL;
 		}
 		while (h2->req0->t_window <= 0 && h2_errcheck(r2, h2) == 0) {
-			// XXX: timeout handling
 			AZ(Lck_CondWait(h2->cond, &h2->sess->mtx, 0));
 		}
 
@@ -235,6 +235,7 @@ h2_do_window(struct worker *wrk, struct h2_req *r2,
 		h2_win_charge(r2, h2, w);
 		assert (w > 0);
 	}
+	r2->t_winupd = 0;
 	Lck_Unlock(&h2->sess->mtx);
 	return (w);
 }

From nils.goroll at uplex.de  Mon Feb 26 13:27:06 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 26 Feb 2018 13:27:06 +0000 (UTC)
Subject: [master] ad0b902 Always use HTTP/1.1 on backend connections for pass
 & fetch
Message-ID: <20180226132706.39A77C03CC@lists.varnish-cache.org>


commit ad0b902e5d02052c11a1ba9c5649a2e807faab65
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Feb 26 14:24:41 2018 +0100

    Always use HTTP/1.1 on backend connections for pass & fetch
    
    pipe remains the only exception.
    
    Ref: https://tools.ietf.org/html/rfc7230#section-2.6
    
       Intermediaries that process HTTP messages (i.e., all intermediaries
       other than those acting as tunnels) MUST send their own HTTP-version
       in forwarded messages.
    
    Closes #2574

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index d5e2548..581a707 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -179,16 +179,14 @@ vbf_stp_mkbereq(struct worker *wrk, struct busyobj *bo)
 	http_FilterReq(bo->bereq0, bo->req->http,
 	    bo->do_pass ? HTTPH_R_PASS : HTTPH_R_FETCH);
 
-	if (!bo->do_pass) {
+	if (bo->do_pass)
+		AZ(bo->stale_oc);
+	else {
 		http_ForceField(bo->bereq0, HTTP_HDR_METHOD, "GET");
-		http_ForceField(bo->bereq0, HTTP_HDR_PROTO, "HTTP/1.1");
 		if (cache_param->http_gzip_support)
 			http_ForceHeader(bo->bereq0, H_Accept_Encoding, "gzip");
-	} else {
-		AZ(bo->stale_oc);
-		if (bo->bereq0->protover > 11)
-			http_ForceField(bo->bereq0, HTTP_HDR_PROTO, "HTTP/1.1");
 	}
+	http_ForceField(bo->bereq0, HTTP_HDR_PROTO, "HTTP/1.1");
 	http_CopyHome(bo->bereq0);
 
 	if (bo->stale_oc != NULL &&
diff --git a/bin/varnishtest/tests/b00001.vtc b/bin/varnishtest/tests/b00001.vtc
index 95dc08d..3a07e5e 100644
--- a/bin/varnishtest/tests/b00001.vtc
+++ b/bin/varnishtest/tests/b00001.vtc
@@ -2,9 +2,11 @@ varnishtest "Check that a pipe transaction works"
 
 server s1 -repeat 1 {
 	rxreq
+	expect req.proto == "HTTP/1.0"
 	expect req.http.connection == "close"
 	txresp
 	rxreq
+	expect req.proto == "nonsense"
 	expect req.http.connection == "keep-alive"
 	txresp
 } -start
@@ -22,10 +24,10 @@ varnish v1 -vcl+backend {
 } -start
 
 client c1 {
-	txreq -url /1 -hdr "Connection: keep-alive"
+	txreq -proto HTTP/1.0 -url /1 -hdr "Connection: keep-alive"
 	rxresp
 	expect resp.status == 200
-	txreq -url /2 -hdr "Connection: keep-alive"
+	txreq -proto nonsense -url /2 -hdr "Connection: keep-alive"
 	rxresp
 	expect resp.status == 200
 } -run
diff --git a/bin/varnishtest/tests/b00002.vtc b/bin/varnishtest/tests/b00002.vtc
index 26591b0..2c978ee 100644
--- a/bin/varnishtest/tests/b00002.vtc
+++ b/bin/varnishtest/tests/b00002.vtc
@@ -2,6 +2,7 @@ varnishtest "Check that a pass transaction works"
 
 server s1 {
 	rxreq
+	expect req.proto == HTTP/1.1
 	txresp -hdr "Cache-Control: max-age=120" -hdr "Connection: close" -body "012345\n"
 } -start
 
@@ -25,9 +26,11 @@ logexpect l1 -v v1 -g request {
 } -start
 
 client c1 {
-	txreq -url "/"
+	txreq -proto HTTP/1.0 -url "/"
 	rxresp
+	expect resp.proto == HTTP/1.1
 	expect resp.status == 200
+	expect resp.http.connection == close
 	expect resp.http.x-ttl == 0.000
 } -run
 
diff --git a/bin/varnishtest/tests/r01662.vtc b/bin/varnishtest/tests/r01662.vtc
index d8eb01c..1a19339 100644
--- a/bin/varnishtest/tests/r01662.vtc
+++ b/bin/varnishtest/tests/r01662.vtc
@@ -14,7 +14,7 @@ logexpect l1 -v v1 -g request {
 	expect * 1001	ReqMethod	"HEAD"
 	expect * =	ReqProtocol	"HTTP/1.0"
 	expect * 1002	BereqMethod	"GET"
-	expect 0 =	BereqProtocol	"HTTP/1.1"
+	expect 1 =	BereqProtocol	"HTTP/1.1"
 } -start
 
 client c1 {

From geoff at uplex.de  Mon Feb 26 14:38:07 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Mon, 26 Feb 2018 14:38:07 +0000 (UTC)
Subject: [master] ddc624d Add the listener endpoint name as a third field to
 ReqStart.
Message-ID: <20180226143807.BB50A648F3@lists.varnish-cache.org>


commit ddc624df775fe9da29bfef34fc3cccd00968ea70
Author: Geoff Simmons <geoff at uplex.de>
Date:   Mon Feb 26 11:08:39 2018 +0100

    Add the listener endpoint name as a third field to ReqStart.
    
    Makes it possible to distinguish listeners in a VSL client
    transaction. Especially if there is more than one UDS listen
    address, since all of them are logged as the bogo-IP 0.0.0.0:0
    where addresses are logged.

diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c
index 385e899..f7d96cc 100644
--- a/bin/varnishd/cache/cache_req_fsm.c
+++ b/bin/varnishd/cache/cache_req_fsm.c
@@ -46,6 +46,7 @@
 
 #include "hash/hash_slinger.h"
 #include "storage/storage.h"
+#include "common/heritage.h"
 #include "vcl.h"
 #include "vsha256.h"
 #include "vtim.h"
@@ -814,7 +815,7 @@ cnt_recv(struct worker *wrk, struct req *req)
 {
 	unsigned recv_handling;
 	struct VSHA256Context sha256ctx;
-	const char *ci, *cp;
+	const char *ci, *cp, *endpname;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
@@ -829,7 +830,10 @@ cnt_recv(struct worker *wrk, struct req *req)
 
 	ci = SES_Get_String_Attr(req->sp, SA_CLIENT_IP);
 	cp = SES_Get_String_Attr(req->sp, SA_CLIENT_PORT);
-	VSLb(req->vsl, SLT_ReqStart, "%s %s", ci, cp);
+	CHECK_OBJ_NOTNULL(req->sp->listen_sock, LISTEN_SOCK_MAGIC);
+	endpname = req->sp->listen_sock->name;
+	AN(endpname);
+	VSLb(req->vsl, SLT_ReqStart, "%s %s %s", ci, cp, endpname);
 
 	http_VSL_log(req->http);
 
diff --git a/bin/varnishtest/tests/b00060.vtc b/bin/varnishtest/tests/b00060.vtc
index 6ecd156..9e2750a 100644
--- a/bin/varnishtest/tests/b00060.vtc
+++ b/bin/varnishtest/tests/b00060.vtc
@@ -1,12 +1,12 @@
 varnishtest "VSL tags affected by the use of UDS addresses"
 
-varnish v1 -arg "-a foo=${tmpdir}/v1.sock" -vcl {
+varnish v1 -arg "-a foo=${tmpdir}/foo.sock -a bar=${tmpdir}/bar.sock" -vcl {
 	backend b { .host = "${bad_ip}"; }
 
 	sub vcl_recv { return(synth(200)); }
 } -start
 
-client c1 -connect "${tmpdir}/v1.sock" {
+client c1 -connect "${tmpdir}/foo.sock" {
 	txreq
 	rxresp
 } -run
@@ -18,5 +18,43 @@ logexpect l1 -v v1 -d 1 -g session {
 
 logexpect l2 -v v1 -d 1 -g vxid {
 	expect 0 1001	Begin
-	expect * =	ReqStart	"^0.0.0.0 0$"
+	expect * =	ReqStart	"^0.0.0.0 0 foo$"
+} -run
+
+logexpect l1 -v v1 -d 0 -g session {
+	expect 0 *	Begin
+	expect 0 =	SessOpen	"^0.0.0.0 0 bar 0.0.0.0 0"
+} -start
+
+logexpect l2 -v v1 -d 0 -g vxid {
+	expect 0 *	Begin
+	expect * =	ReqStart	"^0.0.0.0 0 bar"
+} -start
+
+client c1 -connect "${tmpdir}/bar.sock" {
+	txreq
+	rxresp
+} -run
+
+logexpect l1 -wait
+logexpect l2 -wait
+
+varnish v1 -stop
+
+# For completeness, also test the endpoint name field in ReqStart when
+# Varnish listens at an IP address.
+varnish v2 -vcl {
+	backend b { .host = "${bad_ip}"; }
+
+	sub vcl_recv { return(synth(200)); }
+} -start
+
+client c2 -connect ${v2_sock} {
+	txreq
+	rxresp
+} -run
+
+logexpect l3 -v v2 -d 1 -g vxid {
+	expect 0 1001	Begin
+	expect * =	ReqStart	"^${v2_addr} [0-9]+ a0$"
 } -run

From geoff at uplex.de  Mon Feb 26 15:14:09 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Mon, 26 Feb 2018 15:14:09 +0000 (UTC)
Subject: [master] 0ce9097 Update docs for ReqStart in vsl(7),
 adding the listener name field.
Message-ID: <20180226151409.4311E6545D@lists.varnish-cache.org>


commit 0ce909774a37368e2b8b73f56db845f4928091d6
Author: Geoff Simmons <geoff at uplex.de>
Date:   Mon Feb 26 16:12:20 2018 +0100

    Update docs for ReqStart in vsl(7), adding the listener name field.

diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index dc57526..4b75923 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -269,13 +269,14 @@ SLTM(VCL_return, 0, "VCL method return value",
 )
 
 SLTM(ReqStart, 0, "Client request start",
-	"Start of request processing. Logs the client IP address and port"
-	" number.\n\n"
+	"Start of request processing. Logs the client address, port number "
+	" and listener endpoint name (from the -a command-line argument).\n\n"
 	"The format is::\n\n"
-	"\t%s %s\n"
-	"\t|  |\n"
-	"\t|  +- Client Port number\n"
-	"\t+---- Client IP4/6 address\n"
+	"\t%s %s %s\n"
+	"\t|  |  |\n"
+	"\t|  |  +-- Listener name (from -a)\n"
+	"\t|  +----- Client Port number (0 for Unix domain sockets)\n"
+	"\t+-------- Client IP4/6 address (0.0.0.0 for UDS)\n"
 	"\n"
 )
 

From geoff at uplex.de  Mon Feb 26 15:26:06 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Mon, 26 Feb 2018 15:26:06 +0000 (UTC)
Subject: [master] 0c1c2a7 Mention the change to ReqStart in "Upgrading to 6.0".
Message-ID: <20180226152606.609736581D@lists.varnish-cache.org>


commit 0c1c2a756c64e9305635df4e48b2e06e868a419f
Author: Geoff Simmons <geoff at uplex.de>
Date:   Mon Feb 26 16:25:15 2018 +0100

    Mention the change to ReqStart in "Upgrading to 6.0".

diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst
index 0ccac58..4f5d75a 100644
--- a/doc/sphinx/whats-new/upgrading-6.0.rst
+++ b/doc/sphinx/whats-new/upgrading-6.0.rst
@@ -68,9 +68,11 @@ Other changes
 
   * XXX ...
 
-* XXX: etc. ...
+* ``varnishlog(1)``:
 
-  * XXX ...
+  * Added a third field to the ``ReqStart`` log record that contains the
+    name of the listener address over which the request was received, see
+    :ref:`vsl(7)`.
 
   * XXX ...
 

From nils.goroll at uplex.de  Mon Feb 26 15:39:08 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 26 Feb 2018 15:39:08 +0000 (UTC)
Subject: [master] 7fb5adb ws ocd
Message-ID: <20180226153908.57B3165C41@lists.varnish-cache.org>


commit 7fb5adb42f4330da93721cf3f473085ce6db9559
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Feb 26 15:12:04 2018 +0100

    ws ocd

diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index 4b35558..ac708d8 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -190,8 +190,8 @@ vslc_vsm_reset(const struct VSL_cursor *cursor)
 	c->cursor.rec.ptr = NULL;
 
 	segment_n = c->head->segment_n;
-	VRMB();			/* Make sure offset table is not stale
-				   compared to segment_n */
+	/* Make sure offset table is not stale compared to segment_n */
+	VRMB();
 
 	if (c->options & VSL_COPT_TAIL) {
 		/* Start in the same segment varnishd currently is in and

From nils.goroll at uplex.de  Mon Feb 26 15:39:08 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 26 Feb 2018 15:39:08 +0000 (UTC)
Subject: [master] 5edd7cc Fix varnishd synopsis: -l [,vsm] has been removed
Message-ID: <20180226153908.7BFE665C44@lists.varnish-cache.org>


commit 5edd7cc2dbf1c9fbe4bddbd881341d998b46dd8c
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Feb 26 16:36:16 2018 +0100

    Fix varnishd synopsis: -l [,vsm] has been removed

diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index c2c0c88..a70d2df 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -15,7 +15,7 @@ HTTP accelerator daemon
 SYNOPSIS
 ========
 
-varnishd [-a [name=][address][:port][,PROTO][,user=<user>][,group=<group>][,mode=<mode>]] [-b host[:port]] [-C] [-d] [-F] [-f config] [-h type[,options]] [-I clifile] [-i identity] [-j jail[,jailoptions]] [-l vsl[,vsm]] [-M address:port] [-n name] [-P file] [-p param=value] [-r param[,param...]] [-S secret-file] [-s [name=]kind[,options]] [-T address[:port]] [-t TTL] [-V] [-W waiter]
+varnishd [-a [name=][address][:port][,PROTO][,user=<user>][,group=<group>][,mode=<mode>]] [-b host[:port]] [-C] [-d] [-F] [-f config] [-h type[,options]] [-I clifile] [-i identity] [-j jail[,jailoptions]] [-l vsl] [-M address:port] [-n name] [-P file] [-p param=value] [-r param[,param...]] [-S secret-file] [-s [name=]kind[,options]] [-T address[:port]] [-t TTL] [-V] [-W waiter]
 
 varnishd [-x parameter|vsl|cli|builtin|optstring]
 

From geoff at uplex.de  Mon Feb 26 18:52:10 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Mon, 26 Feb 2018 18:52:10 +0000 (UTC)
Subject: [master] 8737cc2 Document UDS addressing for server -listen and
 client -connect in vtc(7).
Message-ID: <20180226185210.DBB26987F1@lists.varnish-cache.org>


commit 8737cc26402f411ae5b6dbdd12a26b937c8db9fa
Author: Geoff Simmons <geoff at uplex.de>
Date:   Mon Feb 26 19:49:56 2018 +0100

    Document UDS addressing for server -listen and client -connect in vtc(7).

diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 8a9f911..496d7c7 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -99,11 +99,14 @@ extern const struct cmds http_cmds[];
  *
  * \-listen STRING (server only)
  *        Dictate the listening socket for the server. STRING is of the form
- *        "IP PORT".
+ *        "IP PORT", or "/PATH/TO/SOCKET" for a Unix domain socket. In the
+ *        latter case, the path must begin with '/', and the server must be
+ *        able to create it.
  *
  * \-connect STRING (client only)
  *        Indicate the server to connect to. STRING is also of the form
- *        "IP PORT".
+ *        "IP PORT", or "/PATH/TO/SOCKET". As with "server -listen", a
+ *        Unix domain socket is recognized when STRING begins with a '/'.
  *
  * \-dispatch (server only, s0 only)
  *        Normally, to keep things simple, server threads only handle one

From geoff at uplex.de  Mon Feb 26 19:27:10 2018
From: geoff at uplex.de (Geoff Simmons)
Date: Mon, 26 Feb 2018 19:27:10 +0000 (UTC)
Subject: [master] 437171d Document the vtc extensions for UDS in "Upgrading to
 6.0".
Message-ID: <20180226192710.62C599B483@lists.varnish-cache.org>


commit 437171de490251dd6cc10d7660e21a5300c92b0b
Author: Geoff Simmons <geoff at uplex.de>
Date:   Mon Feb 26 20:26:08 2018 +0100

    Document the vtc extensions for UDS in "Upgrading to 6.0".

diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst
index 4f5d75a..d2af1ac 100644
--- a/doc/sphinx/whats-new/upgrading-6.0.rst
+++ b/doc/sphinx/whats-new/upgrading-6.0.rst
@@ -56,23 +56,65 @@ XXX: ...
 Other changes
 =============
 
-* ``varnishd(1)``:
+* ``varnishlog(1)``:
 
-  * XXX ...
+  * Added a third field to the ``ReqStart`` log record that contains the
+    name of the listener address over which the request was received, see
+    :ref:`vsl(7)`.
 
   * XXX ...
 
-* ``varnishstat(1)``:
+* ``varnishtest(1)``:
 
-  * XXX ...
+  * The ``client -connect`` and ``server -listen`` commands in vtc
+    scripts now allow Unix domain sockets as addresses, recognized
+    when the argument begins with a ``/``.
 
-  * XXX ...
+    A client attempts the connection immediately, so the socket file
+    must exist at the given path when the client is started, and the
+    client must be able to access it.
 
-* ``varnishlog(1)``:
+    The ``server -listen`` command must be able to create the socket
+    file when it executes ``bind(2)``. To make it easier for other
+    processes to connect to the socket, the server's umask is
+    temporarily set to 0 before the listen is attempted, to minimize
+    issues with permissions. No further attempted is made to set the
+    socket's permissions.
 
-  * Added a third field to the ``ReqStart`` log record that contains the
-    name of the listener address over which the request was received, see
-    :ref:`vsl(7)`.
+    To test a Varnish instance listening at a UDS, just use the
+    ``varnish -arg`` command with the appropriate settings for the
+    ``-a`` command line argument, see :ref:`varnishd(1)`.
+
+    A convenient location for socket files to be used in a test is the
+    temporary directory created by ``varnishtest`` for each test,
+    whose path is held in the macro ``${tmpdir}``. So this is a common
+    idiom for tests that involve UDSen::
+
+      varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl { ... } -start
+
+      client c1 -connect "${tmpdir}/v1.sock" { ... } -run
+
+    When a Varnish instance in a vtc test is listening at a UDS, then
+    its ``vN_*`` macros are set like this:
+
+    * ``v1_addr``: ``/path/to/socket``
+    * ``v1_port``: ``-`` (hyphen)
+    * ``v1_sock``: ``/path/to/socket -``
+
+    When a server ``s1`` is listening at a UDS:
+
+    * ``s1_addr``: ``0.0.0.0``
+    * ``s1_port``: ``0``
+    * ``s1_sock``: ``/path/to/socket``
+
+    The vtc variables ``remote.ip`` and ``remote.port``, which can be
+    used in ``expect`` expressions for both server and client scripts,
+    are set to ``0.0.0.0`` and ``0``, respectively, when the peer
+    address is a UDS.
+
+    We have added the variable ``remote.path`` as a counterpart to the
+    other two. Its value is the path when the peer address is a UDS,
+    and NULL otherwise (matching ``<undef>`` in the latter case).
 
   * XXX ...
 

From nils.goroll at uplex.de  Mon Feb 26 19:28:06 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 26 Feb 2018 19:28:06 +0000 (UTC)
Subject: [master] e68819f Change VCL_Check signature to return enum vcl_check
Message-ID: <20180226192806.E354B9B542@lists.varnish-cache.org>


commit e68819f444a541064dd74bd7509d65b4f37fa5cc
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Feb 26 17:11:38 2018 +0100

    Change VCL_Check signature to return enum vcl_check

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index caa3443..ad2ae52 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -312,7 +312,15 @@ int VSL_ResetCursor(const struct VSL_cursor *c);
 	 *    -1: Operation not supported
 	 */
 
-int VSL_Check(const struct VSL_cursor *c, const struct VSLC_ptr *ptr);
+enum vsl_check {
+	vsl_check_e_notsupp	= -1,
+	vsl_check_e_inval	=  0,
+	vsl_check_warn		=  1,
+	vsl_check_valid	=  2
+};
+
+enum vsl_check
+VSL_Check(const struct VSL_cursor *c, const struct VSLC_ptr *ptr);
 	/*
 	 * Check if the VSLC_ptr structure points to a value that is still
 	 * valid:
diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h
index 95e4ace..913eb54 100644
--- a/lib/libvarnishapi/vsl_api.h
+++ b/lib/libvarnishapi/vsl_api.h
@@ -39,7 +39,8 @@ void vsl_vbm_bitclr(int bit, void *priv);
 typedef void vslc_delete_f(const struct VSL_cursor *);
 typedef int vslc_next_f(const struct VSL_cursor *);
 typedef int vslc_reset_f(const struct VSL_cursor *);
-typedef int vslc_check_f(const struct VSL_cursor *, const struct VSLC_ptr *);
+typedef enum vsl_check vslc_check_f(const struct VSL_cursor *,
+    const struct VSLC_ptr *);
 
 struct vslc_tbl {
 	unsigned			magic;
diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index ac708d8..56b2983 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -91,7 +91,7 @@ vslc_vsm_delete(const struct VSL_cursor *cursor)
  * written
  */
 
-static int
+static enum vsl_check v_matchproto_(vslc_check_f)
 vslc_vsm_check(const struct VSL_cursor *cursor, const struct VSLC_ptr *ptr)
 {
 	const struct vslc_vsm *c;
@@ -101,25 +101,25 @@ vslc_vsm_check(const struct VSL_cursor *cursor, const struct VSLC_ptr *ptr)
 	assert(&c->cursor == cursor);
 
 	if (ptr->ptr == NULL)
-		return (0);
+		return (vsl_check_e_inval);
 
 	dist = c->head->segment_n - ptr->priv;
 
 	if (dist >= VSL_SEGMENTS - 2)
 		/* Too close to continue */
-		return (0);
+		return (vsl_check_e_inval);
 	if (dist >= VSL_SEGMENTS - 4)
 		/* Warning level */
-		return (1);
+		return (vsl_check_warn);
 	/* Safe */
-	return (2);
+	return (vsl_check_valid);
 }
 
 static int
 vslc_vsm_next(const struct VSL_cursor *cursor)
 {
 	struct vslc_vsm *c;
-	int i;
+	enum vsl_check i;
 	uint32_t t;
 
 	CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_VSM_MAGIC);
@@ -127,7 +127,7 @@ vslc_vsm_next(const struct VSL_cursor *cursor)
 
 	while (1) {
 		i = vslc_vsm_check(&c->cursor, &c->next);
-		if (i <= 0) {
+		if (i < vsl_check_warn) {
 			if (VSM_StillValid(c->vsm, &c->vf) != VSM_valid)
 				return (-2); /* VSL abandoned */
 			else
@@ -489,13 +489,13 @@ VSL_Next(const struct VSL_cursor *cursor)
 	return ((tbl->next)(cursor));
 }
 
-int
+enum vsl_check
 VSL_Check(const struct VSL_cursor *cursor, const struct VSLC_ptr *ptr)
 {
 	const struct vslc_tbl *tbl;
 
 	CAST_OBJ_NOTNULL(tbl, cursor->priv_tbl, VSLC_TBL_MAGIC);
 	if (tbl->check == NULL)
-		return (-1);
+		return (vsl_check_e_notsupp);
 	return ((tbl->check)(cursor, ptr));
 }
diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 67d3cb0..ccfe026 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -435,7 +435,7 @@ vtx_append(struct VSLQ *vslq, struct vtx *vtx, const struct VSLC_ptr *start,
 		return;
 	AN(start);
 
-	if (VSL_Check(vslq->c, start) == 2 &&
+	if (VSL_Check(vslq->c, start) == vsl_check_valid &&
 	    !VTAILQ_EMPTY(&vtx->shmchunks_free)) {
 		/* Shmref it */
 		chunk = VTAILQ_FIRST(&vtx->shmchunks_free);

From nils.goroll at uplex.de  Mon Feb 26 19:28:07 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 26 Feb 2018 19:28:07 +0000 (UTC)
Subject: [master] 8a4e28f in vtx_append(),
 assert that we do not try to buffer an invalid chunk
Message-ID: <20180226192807.0AC6D9B545@lists.varnish-cache.org>


commit 8a4e28f17c2ee39237a2c56fa0694f2f0b73e3f8
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Feb 26 17:18:53 2018 +0100

    in vtx_append(), assert that we do not try to buffer an invalid chunk

diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index ccfe026..9ee85fb 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -429,14 +429,15 @@ vtx_append(struct VSLQ *vslq, struct vtx *vtx, const struct VSLC_ptr *start,
     size_t len)
 {
 	struct chunk *chunk;
+	enum vsl_check i;
 
 	AN(vtx);
 	if (len == 0)
 		return;
 	AN(start);
 
-	if (VSL_Check(vslq->c, start) == vsl_check_valid &&
-	    !VTAILQ_EMPTY(&vtx->shmchunks_free)) {
+	i = VSL_Check(vslq->c, start);
+	if (i == vsl_check_valid && !VTAILQ_EMPTY(&vtx->shmchunks_free)) {
 		/* Shmref it */
 		chunk = VTAILQ_FIRST(&vtx->shmchunks_free);
 		CHECK_OBJ_NOTNULL(chunk, CHUNK_MAGIC);
@@ -450,6 +451,7 @@ vtx_append(struct VSLQ *vslq, struct vtx *vtx, const struct VSLC_ptr *start,
 		/* Append to shmref list */
 		VTAILQ_INSERT_TAIL(&vslq->shmrefs, chunk, shm.shmref);
 	} else {
+		assert(i != vsl_check_e_inval);
 		/* Buffer it */
 		chunk = VTAILQ_LAST(&vtx->chunks, chunkhead);
 		CHECK_OBJ_ORNULL(chunk, CHUNK_MAGIC);

From nils.goroll at uplex.de  Mon Feb 26 19:28:07 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 26 Feb 2018 19:28:07 +0000 (UTC)
Subject: [master] 3a9541e gc unused struct member
Message-ID: <20180226192807.2729D9B54A@lists.varnish-cache.org>


commit 3a9541e5ef73dbcfecf62cf8373895e50d861ed4
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Feb 26 18:07:27 2018 +0100

    gc unused struct member

diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index 56b2983..0f91767 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -298,7 +298,6 @@ struct vslc_file {
 	unsigned			magic;
 #define VSLC_FILE_MAGIC			0x1D65FFEF
 
-	int				error;
 	int				fd;
 	int				close_fd;
 	ssize_t				buflen;
@@ -348,9 +347,6 @@ vslc_file_next(const struct VSL_cursor *cursor)
 	CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_FILE_MAGIC);
 	assert(&c->cursor == cursor);
 
-	if (c->error)
-		return (c->error);
-
 	do {
 		c->cursor.rec.ptr = NULL;
 		assert(c->buflen >= 2);

From nils.goroll at uplex.de  Mon Feb 26 19:28:07 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 26 Feb 2018 19:28:07 +0000 (UTC)
Subject: [master] e9a5539 Change return value of VSL_Next()-ish functions to
 enum vsl_status
Message-ID: <20180226192807.44E7E9B54E@lists.varnish-cache.org>


commit e9a5539cf8f29e1c662835529a42dc654670f0c9
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Feb 26 20:00:29 2018 +0100

    Change return value of VSL_Next()-ish functions to enum vsl_status
    
    This should help readability of the code, hopefully.
    
    The API remains unchanged.
    
    Local variables for vsl_status are called 'r', variables holding
    a VSLQ_dispatch_f return value are called 'i'

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index ad2ae52..9a8d70d 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -138,6 +138,9 @@ typedef int VSLQ_dispatch_f(struct VSL_data *vsl,
 	 * Return value:
 	 *     0: OK - continue
 	 *   !=0: Makes VSLQ_Dispatch return with this return value immediatly
+	 *
+	 * Return values of the callback function should be distinct from the
+	 * values of enum vsl_status except for 0
 	 */
 
 typedef void VSL_tagfind_f(int tag, void *priv);
@@ -301,7 +304,8 @@ void VSL_DeleteCursor(const struct VSL_cursor *c);
 	 * Delete the cursor pointed to by c
 	 */
 
-int VSL_ResetCursor(const struct VSL_cursor *c);
+enum vsl_status
+VSL_ResetCursor(const struct VSL_cursor *c);
 	/*
 	 * Reset the cursor position to the head, so that the next call to
 	 * VSL_Next returns the first record. For VSM cursor, it will
@@ -309,7 +313,9 @@ int VSL_ResetCursor(const struct VSL_cursor *c);
 	 * from the tail.
 	 *
 	 * Return values:
-	 *    -1: Operation not supported
+	 * - vsl_end == success
+	 * - and see enum vsl_status
+	 *
 	 */
 
 enum vsl_check {
@@ -332,17 +338,21 @@ VSL_Check(const struct VSL_cursor *c, const struct VSLC_ptr *ptr);
 	 *     2: Valid
 	 */
 
-int VSL_Next(const struct VSL_cursor *c);
+enum vsl_status {
+	vsl_e_io	= -4,	// I/O read error - see errno
+	vsl_e_overrun	= -3,	// Overrun
+	vsl_e_abandon	= -2,	// Remote abandoned or closed
+	vsl_e_eof	= -1,	// End of file
+	vsl_end		=  0,	// End of log/cursor
+	vsl_more	=  1	// Cursor points to next log record
+};
+
+enum vsl_status
+VSL_Next(const struct VSL_cursor *c);
 	/*
 	 * Return raw pointer to next VSL record.
 	 *
-	 * Return values:
-	 *	1:	Cursor points to next log record
-	 *	0:	End of log
-	 *     -1:	End of file
-	 *     -2:	Remote abandoned or closed
-	 *     -3:	Overrun
-	 *     -4:	I/O read error - see errno
+	 * Return values: see enum vsl_status
 	 */
 
 int VSL_Match(struct VSL_data *vsl, const struct VSL_cursor *c);
@@ -521,7 +531,7 @@ int VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv);
 	 * Return values:
 	 *     1: Call again
 	 *     0: No more log records available
-	 *   !=0: The error code from VSL_Next() or func returned non-zero
+	 *   !=0: func returned non-zero or enum vsl_status
 	 */
 
 int VSLQ_Flush(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv);
diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h
index 913eb54..0e265c0 100644
--- a/lib/libvarnishapi/vsl_api.h
+++ b/lib/libvarnishapi/vsl_api.h
@@ -37,8 +37,8 @@ void vsl_vbm_bitset(int bit, void *priv);
 void vsl_vbm_bitclr(int bit, void *priv);
 
 typedef void vslc_delete_f(const struct VSL_cursor *);
-typedef int vslc_next_f(const struct VSL_cursor *);
-typedef int vslc_reset_f(const struct VSL_cursor *);
+typedef enum vsl_status vslc_next_f(const struct VSL_cursor *);
+typedef enum vsl_status vslc_reset_f(const struct VSL_cursor *);
 typedef enum vsl_check vslc_check_f(const struct VSL_cursor *,
     const struct VSLC_ptr *);
 
diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index 0f91767..8559c07 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -115,7 +115,7 @@ vslc_vsm_check(const struct VSL_cursor *cursor, const struct VSLC_ptr *ptr)
 	return (vsl_check_valid);
 }
 
-static int
+static enum vsl_status v_matchproto_(vslc_next_f)
 vslc_vsm_next(const struct VSL_cursor *cursor)
 {
 	struct vslc_vsm *c;
@@ -129,9 +129,9 @@ vslc_vsm_next(const struct VSL_cursor *cursor)
 		i = vslc_vsm_check(&c->cursor, &c->next);
 		if (i < vsl_check_warn) {
 			if (VSM_StillValid(c->vsm, &c->vf) != VSM_valid)
-				return (-2); /* VSL abandoned */
+				return (vsl_e_abandon);
 			else
-				return (-3); /* Overrun */
+				return (vsl_e_overrun);
 		}
 
 		t = *(volatile const uint32_t *)c->next.ptr;
@@ -148,10 +148,11 @@ vslc_vsm_next(const struct VSL_cursor *cursor)
 
 		if (t == VSL_ENDMARKER) {
 			if (VSM_StillValid(c->vsm, &c->vf) != VSM_valid)
-				return (-2); /* VSL abandoned */
+				return (vsl_e_abandon);
 			if (c->options & VSL_COPT_TAILSTOP)
-				return (-1); /* EOF */
-			return (0);	/* No new records available */
+				return (vsl_e_eof);
+			/* No new records available */
+			return (vsl_end);
 		}
 
 		c->cursor.rec = c->next;
@@ -174,16 +175,16 @@ vslc_vsm_next(const struct VSL_cursor *cursor)
 		assert(c->next.ptr >= c->head->log);
 		assert(c->next.ptr < c->end);
 
-		return (1);
+		return (vsl_more);
 	}
 }
 
-static int
+static enum vsl_status v_matchproto_(vslc_reset_f)
 vslc_vsm_reset(const struct VSL_cursor *cursor)
 {
 	struct vslc_vsm *c;
 	unsigned u, segment_n;
-	int i;
+	enum vsl_status r;
 
 	CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_VSM_MAGIC);
 	assert(&c->cursor == cursor);
@@ -204,12 +205,12 @@ vslc_vsm_reset(const struct VSL_cursor *cursor)
 			if (c->head->segment_n - u > 1) {
 				/* Give up if varnishd is moving faster
 				   than us */
-				return (-3); /* overrun */
+				return (vsl_e_overrun);
 			}
-			i = vslc_vsm_next(&c->cursor);
-		} while (i == 1);
-		if (i)
-			return (i);
+			r = vslc_vsm_next(&c->cursor);
+		} while (r == vsl_more);
+		if (r != vsl_end)
+			return (r);
 	} else {
 		/* Starting (VSL_SEGMENTS - 3) behind varnishd. This way
 		 * even if varnishd advances segment_n immediately, we'll
@@ -229,7 +230,7 @@ vslc_vsm_reset(const struct VSL_cursor *cursor)
 	}
 	assert(c->next.ptr >= c->head->log);
 	assert(c->next.ptr < c->end);
-	return (0);
+	return (vsl_end);
 }
 
 static const struct vslc_tbl vslc_vsm_tbl = {
@@ -246,7 +247,7 @@ VSL_CursorVSM(struct VSL_data *vsl, struct vsm *vsm, unsigned options)
 	struct vslc_vsm *c;
 	struct vsm_fantom vf;
 	struct VSL_head *head;
-	int i;
+	enum vsl_status r;
 
 	CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
 
@@ -284,9 +285,9 @@ VSL_CursorVSM(struct VSL_data *vsl, struct vsm *vsm, unsigned options)
 	c->end = c->head->log + c->head->segsize * VSL_SEGMENTS;
 	assert(c->end <= (const uint32_t *)vf.e);
 
-	i = vslc_vsm_reset(&c->cursor);
-	if (i) {
-		(void)vsl_diag(vsl, "Cursor initialization failure (%d)", i);
+	r = vslc_vsm_reset(&c->cursor);
+	if (r != vsl_end) {
+		(void)vsl_diag(vsl, "Cursor initialization failure (%d)", r);
 		FREE_OBJ(c);
 		return (NULL);
 	}
@@ -337,7 +338,7 @@ vslc_file_readn(int fd, void *buf, size_t n)
 	return (t);
 }
 
-static int
+static enum vsl_status v_matchproto_(vslc_next_f)
 vslc_file_next(const struct VSL_cursor *cursor)
 {
 	struct vslc_file *c;
@@ -352,9 +353,9 @@ vslc_file_next(const struct VSL_cursor *cursor)
 		assert(c->buflen >= 2);
 		i = vslc_file_readn(c->fd, c->buf, VSL_BYTES(2));
 		if (i < 0)
-			return (-4);	/* I/O error */
+			return (vsl_e_io);
 		if (i == 0)
-			return (-1);	/* EOF */
+			return (vsl_e_eof);
 		assert(i == VSL_BYTES(2));
 		l = 2 + VSL_WORDS(VSL_LEN(c->buf));
 		if (c->buflen < l) {
@@ -367,22 +368,22 @@ vslc_file_next(const struct VSL_cursor *cursor)
 			i = vslc_file_readn(c->fd, c->buf + 2,
 			    VSL_BYTES(l - 2));
 			if (i < 0)
-				return (-4);	/* I/O error */
+				return (vsl_e_io);
 			if (i == 0)
-				return (-1);	/* EOF */
+				return (vsl_e_eof);
 			assert(i == VSL_BYTES(l - 2));
 		}
 		c->cursor.rec.ptr = c->buf;
 	} while (VSL_TAG(c->cursor.rec.ptr) == SLT__Batch);
-	return (1);
+	return (vsl_more);
 }
 
-static int
+static enum vsl_status v_matchproto_(vslc_reset_f)
 vslc_file_reset(const struct VSL_cursor *cursor)
 {
 	(void)cursor;
 	/* XXX: Implement me */
-	return (-1);
+	return (vsl_e_eof);
 }
 
 static const struct vslc_tbl vslc_file_tbl = {
@@ -464,18 +465,18 @@ VSL_DeleteCursor(const struct VSL_cursor *cursor)
 	(tbl->delete)(cursor);
 }
 
-int
+enum vsl_status
 VSL_ResetCursor(const struct VSL_cursor *cursor)
 {
 	const struct vslc_tbl *tbl;
 
 	CAST_OBJ_NOTNULL(tbl, cursor->priv_tbl, VSLC_TBL_MAGIC);
 	if (tbl->reset == NULL)
-		return (-1);
+		return (vsl_e_eof);
 	return ((tbl->reset)(cursor));
 }
 
-int
+enum vsl_status
 VSL_Next(const struct VSL_cursor *cursor)
 {
 	const struct vslc_tbl *tbl;
diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c
index 9ee85fb..5c1a534 100644
--- a/lib/libvarnishapi/vsl_dispatch.c
+++ b/lib/libvarnishapi/vsl_dispatch.c
@@ -225,7 +225,7 @@ vtx_keycmp(const struct vtx_key *a, const struct vtx_key *b)
 VRB_PROTOTYPE_STATIC(vtx_tree, vtx_key, entry, vtx_keycmp)
 VRB_GENERATE_STATIC(vtx_tree, vtx_key, entry, vtx_keycmp)
 
-static int
+static enum vsl_status v_matchproto_(vslc_next_f)
 vslc_raw_next(const struct VSL_cursor *cursor)
 {
 	struct vslc_raw *c;
@@ -236,14 +236,14 @@ vslc_raw_next(const struct VSL_cursor *cursor)
 	AN(c->ptr);
 	if (c->cursor.rec.ptr == NULL) {
 		c->cursor.rec.ptr = c->ptr;
-		return (1);
+		return (vsl_more);
 	} else {
 		c->cursor.rec.ptr = NULL;
-		return (0);
+		return (vsl_end);
 	}
 }
 
-static int
+static enum vsl_status v_matchproto_(vslc_reset_f)
 vslc_raw_reset(const struct VSL_cursor *cursor)
 {
 	struct vslc_raw *c;
@@ -254,7 +254,7 @@ vslc_raw_reset(const struct VSL_cursor *cursor)
 	AN(c->ptr);
 	c->cursor.rec.ptr = NULL;
 
-	return (0);
+	return (vsl_end);
 }
 
 static const struct vslc_tbl vslc_raw_tbl = {
@@ -265,7 +265,7 @@ static const struct vslc_tbl vslc_raw_tbl = {
 	.check	= NULL,
 };
 
-static int
+static enum vsl_status v_matchproto_(vslc_next_f)
 vslc_vtx_next(const struct VSL_cursor *cursor)
 {
 	struct vslc_vtx *c;
@@ -285,8 +285,7 @@ vslc_vtx_next(const struct VSL_cursor *cursor)
 		} else {
 			assert(c->offset <= c->vtx->len);
 			if (c->offset == c->vtx->len)
-				/* End of cursor */
-				return (0);
+				return (vsl_end);
 
 			/* Advance chunk pointer */
 			if (c->chunk == NULL) {
@@ -313,10 +312,10 @@ vslc_vtx_next(const struct VSL_cursor *cursor)
 		}
 	} while (VSL_TAG(c->cursor.rec.ptr) == SLT__Batch);
 
-	return (1);
+	return (vsl_more);
 }
 
-static int
+static enum vsl_status v_matchproto_(vslc_reset_f)
 vslc_vtx_reset(const struct VSL_cursor *cursor)
 {
 	struct vslc_vtx *c;
@@ -330,7 +329,7 @@ vslc_vtx_reset(const struct VSL_cursor *cursor)
 	c->offset = 0;
 	c->cursor.rec.ptr = NULL;
 
-	return (0);
+	return (vsl_end);
 }
 
 static const struct vslc_tbl vslc_vtx_tbl = {
@@ -513,7 +512,7 @@ vtx_new(struct VSLQ *vslq)
 	vtx->n_childready = 0;
 	vtx->n_descend = 0;
 	vtx->len = 0;
-	(void)vslc_vtx_reset(&vtx->c.cursor);
+	AN(vslc_vtx_reset(&vtx->c.cursor) == vsl_end);
 
 	return (vtx);
 }
@@ -935,7 +934,7 @@ vslq_callback(const struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func,
 		return (0);
 
 	/* Build transaction array */
-	(void)vslc_vtx_reset(&vtx->c.cursor);
+	AN(vslc_vtx_reset(&vtx->c.cursor) == vsl_end);
 	vtxs[0] = vtx;
 	trans[0].level = 1;
 	trans[0].vxid = vtx->key.vxid;
@@ -948,7 +947,7 @@ vslq_callback(const struct VSLQ *vslq, struct vtx *vtx, VSLQ_dispatch_f *func,
 	while (j < i) {
 		VTAILQ_FOREACH(vtx, &vtxs[j]->child, list_child) {
 			assert(i < n);
-			(void)vslc_vtx_reset(&vtx->c.cursor);
+			AN(vslc_vtx_reset(&vtx->c.cursor) == vsl_end);
 			vtxs[i] = vtx;
 			if (vtx->reason == VSL_r_restart)
 				/* Restarts stay at the same level as parent */
@@ -1158,17 +1157,17 @@ VSLQ_SetCursor(struct VSLQ *vslq, struct VSL_cursor **cp)
 static int
 vslq_raw(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 {
-	int i = 1;
-	int r;
+	enum vsl_status r = vsl_more;
+	int i;
 
 	assert(vslq->grouping == VSL_g_raw);
 
 	assert(vslq->raw.offset <= vslq->raw.len);
 	do {
 		if (vslq->raw.offset == vslq->raw.len) {
-			i = VSL_Next(vslq->c);
-			if (i <= 0)
-				return (i);
+			r = VSL_Next(vslq->c);
+			if (r != vsl_more)
+				return (r);
 			AN(vslq->c->rec.ptr);
 			vslq->raw.start = vslq->c->rec;
 			if (VSL_TAG(vslq->c->rec.ptr) == SLT__Batch)
@@ -1188,67 +1187,70 @@ vslq_raw(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 		vslq->raw.offset += VSL_NEXT(vslq->raw.c.ptr) - vslq->raw.c.ptr;
 	} while (VSL_TAG(vslq->raw.c.ptr) == SLT__Batch);
 
+	assert (r == vsl_more);
+
 	if (func == NULL)
-		return (i);
+		return (r);
 
 	if (vslq->query != NULL &&
 	    !vslq_runquery(vslq->query, vslq->raw.ptrans))
-		return (i);
-
-	r = (func)(vslq->vsl, vslq->raw.ptrans, priv);
-	if (r)
 		return (r);
 
-	return (i);
+	i = (func)(vslq->vsl, vslq->raw.ptrans, priv);
+	if (i)
+		return (i);
+
+	return (r);
 }
 
 /* Check the beginning of the shmref list, and buffer refs that are at
  * warning level.
- *
- * Returns:
- *    0:	OK
- *   -3:	Failure
  */
-static int
+static enum vsl_status
 vslq_shmref_check(struct VSLQ *vslq)
 {
 	struct chunk *chunk;
-	int i;
+	enum vsl_check i;
 
 	while ((chunk = VTAILQ_FIRST(&vslq->shmrefs)) != NULL) {
 		CHECK_OBJ_NOTNULL(chunk, CHUNK_MAGIC);
 		assert(chunk->type == chunk_t_shm);
 		i = VSL_Check(vslq->c, &chunk->shm.start);
-		if (i == 2)
+		switch (i) {
+		case vsl_check_valid:
 			/* First on list is OK, refs behind it must also
 			   be OK */
-			return (0);
-		else if (i == 1)
-			/* Warning level. Buffer this chunk */
+			return (vsl_more);
+		case vsl_check_warn:
+			/* Buffer this chunk */
 			chunk_shm_to_buf(vslq, chunk);
-		else
+			break;
+		default:
 			/* Too late to buffer */
-			return (-3);
+			return (vsl_e_overrun);
+		}
 	}
 
-	return (0);
+	return (vsl_more);
 }
 
 /* Process next input record */
-static int
+static enum vsl_status
 vslq_next(struct VSLQ *vslq)
 {
 	struct VSL_cursor *c;
-	int i;
+	enum vsl_status r;
 	enum VSL_tag_e tag;
 	ssize_t len;
 	unsigned vxid;
 	struct vtx *vtx;
 
 	c = vslq->c;
-	i = VSL_Next(c);
-	if (i != 1)
-		return (i);
+	r = VSL_Next(c);
+	if (r != vsl_more)
+		return (r);
+
+	assert (r == vsl_more);
 
 	tag = (enum VSL_tag_e)VSL_TAG(c->rec.ptr);
 	if (tag == SLT__Batch) {
@@ -1256,7 +1258,7 @@ vslq_next(struct VSLQ *vslq)
 		len = VSL_END(c->rec.ptr, VSL_BATCHLEN(c->rec.ptr)) -
 		    c->rec.ptr;
 		if (len == 0)
-			return (i);
+			return (r);
 		tag = (enum VSL_tag_e)VSL_TAG(VSL_NEXT(c->rec.ptr));
 	} else {
 		vxid = VSL_ID(c->rec.ptr);
@@ -1265,7 +1267,7 @@ vslq_next(struct VSLQ *vslq)
 	assert(len > 0);
 	if (vxid == 0)
 		/* Skip non-transactional records */
-		return (i);
+		return (r);
 
 	vtx = vtx_lookup(vslq, vxid);
 	if (vtx == NULL && tag == SLT_Begin) {
@@ -1277,7 +1279,7 @@ vslq_next(struct VSLQ *vslq)
 		vtx_scan(vslq, vtx);
 	}
 
-	return (i);
+	return (r);
 }
 
 /* Test query and report any ready transactions */
@@ -1310,7 +1312,8 @@ vslq_process_ready(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 int
 VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 {
-	int i, r;
+	enum vsl_status r;
+	int i;
 	double now;
 	struct vtx *vtx;
 
@@ -1318,23 +1321,25 @@ VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 
 	/* Check that we have a cursor */
 	if (vslq->c == NULL)
-		return (-2);
+		return (vsl_e_abandon);
 
 	if (vslq->grouping == VSL_g_raw)
 		return (vslq_raw(vslq, func, priv));
 
 	/* Process next cursor input */
-	i = vslq_next(vslq);
-	if (i <= 0)
+	r = vslq_next(vslq);
+	if (r != vsl_more)
 		/* At end of log or cursor reports error condition */
-		return (i);
+		return (r);
 
 	/* Check shmref list and buffer if necessary */
 	r = vslq_shmref_check(vslq);
-	if (r)
+	if (r != vsl_more)
 		/* Buffering of shm ref failed */
 		return (r);
 
+	assert (r == vsl_more);
+
 	/* Check vtx timeout */
 	now = VTIM_mono();
 	while (!VTAILQ_EMPTY(&vslq->incomplete)) {
@@ -1353,22 +1358,21 @@ VSLQ_Dispatch(struct VSLQ *vslq, VSLQ_dispatch_f *func, void *priv)
 		CHECK_OBJ_NOTNULL(vtx, VTX_MAGIC);
 		vtx_force(vslq, vtx, "store overflow");
 		AN(vtx->flags & VTX_F_COMPLETE);
-		r = vslq_process_ready(vslq, func, priv);
-		if (r)
+		i = vslq_process_ready(vslq, func, priv);
+		if (i)
 			/* User return code */
-			return (r);
+			return (i);
 	}
 
 	/* Check ready list */
 	if (!VTAILQ_EMPTY(&vslq->ready)) {
-		r = vslq_process_ready(vslq, func, priv);
-		if (r)
+		i = vslq_process_ready(vslq, func, priv);
+		if (i)
 			/* User return code */
-			return (r);
+			return (i);
 	}
 
-	/* Return cursor return value */
-	return (i);
+	return (vsl_more);
 }
 
 /* Flush any incomplete vtx held on to. Do callbacks if func != NULL */

From nils.goroll at uplex.de  Mon Feb 26 19:28:07 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 26 Feb 2018 19:28:07 +0000 (UTC)
Subject: [master] bfeb331 reserve the error value from VSL_Write() and friends
Message-ID: <20180226192807.5EDA49B552@lists.varnish-cache.org>


commit bfeb3316e349264dc9526391c483da6915cbd861
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Feb 26 20:20:01 2018 +0100

    reserve the error value from VSL_Write() and friends

diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h
index 9a8d70d..2209ac3 100644
--- a/include/vapi/vsl.h
+++ b/include/vapi/vsl.h
@@ -339,6 +339,7 @@ VSL_Check(const struct VSL_cursor *c, const struct VSLC_ptr *ptr);
 	 */
 
 enum vsl_status {
+	vsl_e_write	= -5,	// Error from VSL_Write etc.
 	vsl_e_io	= -4,	// I/O read error - see errno
 	vsl_e_overrun	= -3,	// Overrun
 	vsl_e_abandon	= -2,	// Remote abandoned or closed

From nils.goroll at uplex.de  Mon Feb 26 20:57:09 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 26 Feb 2018 20:57:09 +0000 (UTC)
Subject: [master] e1a0cd5 use enum vsl_status in VUT_Main()
Message-ID: <20180226205709.F096CA2549@lists.varnish-cache.org>


commit e1a0cd558cd5cae6a12a181a6f56023eae5b38c0
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Feb 26 21:33:49 2018 +0100

    use enum vsl_status in VUT_Main()

diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c
index 6766cb9..86a324b 100644
--- a/lib/libvarnishapi/vut.c
+++ b/lib/libvarnishapi/vut.c
@@ -390,11 +390,9 @@ VUT_Main(struct VUT *vut)
 		}
 
 		i = VSLQ_Dispatch(vut->vslq, vut_dispatch, vut);
-		if (i == 1)
-			/* Call again */
+		if (i == vsl_more)
 			continue;
-		else if (i == 0) {
-			/* Nothing to do but wait */
+		else if (i == vsl_end) {
 			if (vut->idle_f) {
 				i = vut->idle_f(vut);
 				if (i)
@@ -402,10 +400,8 @@ VUT_Main(struct VUT *vut)
 			}
 			VTIM_sleep(0.01);
 			continue;
-		} else if (i == -1) {
-			/* EOF */
+		} else if (i == vsl_e_eof)
 			break;
-		}
 
 		if (vut->vsm == NULL)
 			break;
@@ -414,14 +410,14 @@ VUT_Main(struct VUT *vut)
 
 		(void)VSLQ_Flush(vut->vslq, vut_dispatch, vut);
 
-		if (i == -2) {
-			/* Abandoned */
+		if (i == vsl_e_abandon) {
 			fprintf(stderr, "Log abandoned (vsl)\n");
 			VSLQ_SetCursor(vut->vslq, NULL);
 			hascursor = 0;
-		} else if (i < -2)
-			/* Overrun */
+		} else if (i == vsl_e_overrun)
 			fprintf(stderr, "Log overrun\n");
+		else
+			fprintf(stderr, "Error %d from VSLQ_Dispatch()", i);
 	}
 
 	return (i);

From nils.goroll at uplex.de  Mon Feb 26 20:57:10 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 26 Feb 2018 20:57:10 +0000 (UTC)
Subject: [master] 7b22e04 For a log overrun,
 we need to get a fresh VSL cursor also
Message-ID: <20180226205710.0B215A254B@lists.varnish-cache.org>


commit 7b22e04fd870492a927bfb77550c9becae6a53be
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Feb 26 21:41:27 2018 +0100

    For a log overrun, we need to get a fresh VSL cursor also
    
    Hopefully fixes #2586

diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c
index 86a324b..8da4a85 100644
--- a/lib/libvarnishapi/vut.c
+++ b/lib/libvarnishapi/vut.c
@@ -414,9 +414,11 @@ VUT_Main(struct VUT *vut)
 			fprintf(stderr, "Log abandoned (vsl)\n");
 			VSLQ_SetCursor(vut->vslq, NULL);
 			hascursor = 0;
-		} else if (i == vsl_e_overrun)
+		} else if (i == vsl_e_overrun) {
 			fprintf(stderr, "Log overrun\n");
-		else
+			VSLQ_SetCursor(vut->vslq, NULL);
+			hascursor = 0;
+		} else
 			fprintf(stderr, "Error %d from VSLQ_Dispatch()", i);
 	}
 

From nils.goroll at uplex.de  Mon Feb 26 20:57:10 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 26 Feb 2018 20:57:10 +0000 (UTC)
Subject: [master] 18572b1 drop some insights from working with the code
Message-ID: <20180226205710.2500BA254E@lists.varnish-cache.org>


commit 18572b1d9a55834476e22fd1f01a482dc40ca63b
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Feb 26 21:54:16 2018 +0100

    drop some insights from working with the code

diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c
index 8da4a85..4d4572f 100644
--- a/lib/libvarnishapi/vut.c
+++ b/lib/libvarnishapi/vut.c
@@ -361,8 +361,22 @@ VUT_Main(struct VUT *vut)
 			(void)VSLQ_Flush(vut->vslq, vut_dispatch, vut);
 		}
 
-		// We must repeatedly call VSM_Status() when !hascursor
-		// to make VSM discover our segment.
+		/* We must repeatedly call VSM_Status() when !hascursor
+		 * to make VSM discover our segment.
+		 *
+		 * XXX consider moving the error handling to VSLQ_Dispatch.
+		 * or some other VSL utility function
+		 * Reasons:
+		 *
+		 * - it does not seem to make much sense to call VSM_StillValid
+		 *   in vsl if that can only detect invalid segments after
+		 *   VSM_Status has run, so it appears both should be
+		 *   consolidated
+		 *
+		 * - not all VSL Clients will use VUT, yet the log abandoned/
+		 *   overrun situation will be occur for all of them.
+		 */
+
 		if (vut->vsm != NULL &&
 		    (VSM_Status(vut->vsm) & VSM_WRK_RESTARTED)) {
 			if (hascursor < 1) {

From phk at FreeBSD.org  Mon Feb 26 22:20:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 26 Feb 2018 22:20:10 +0000 (UTC)
Subject: [master] b358d56 Lock the VSMW layer in the worker process to
 stabilize v00003.
Message-ID: <20180226222011.02D05A3F20@lists.varnish-cache.org>


commit b358d56bec6575ba3bd5407fad869883f1d958d8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Feb 26 22:18:55 2018 +0000

    Lock the VSMW layer in the worker process to stabilize v00003.
    
    I don't think it matters anywhere else, but future-proofing is good.

diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index c7b5ed8..754c8f6 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -45,6 +45,7 @@
 
 /* These cannot be struct lock, which depends on vsm/vsl working */
 static pthread_mutex_t vsl_mtx;
+static pthread_mutex_t vsc_mtx;
 static pthread_mutex_t vsm_mtx;
 
 static struct VSL_head		*vsl_head;
@@ -478,15 +479,27 @@ VSL_End(struct vsl_log *vsl)
 	vsl->wid = 0;
 }
 
-static void
+static void v_matchproto_(vsm_lock_f)
 vsm_vsc_lock(void)
 {
-	AZ(pthread_mutex_lock(&vsm_mtx));
+	AZ(pthread_mutex_lock(&vsc_mtx));
 }
 
-static void
+static void v_matchproto_(vsm_lock_f)
 vsm_vsc_unlock(void)
 {
+	AZ(pthread_mutex_unlock(&vsc_mtx));
+}
+
+static void v_matchproto_(vsm_lock_f)
+vsm_vsmw_lock(void)
+{
+	AZ(pthread_mutex_lock(&vsm_mtx));
+}
+
+static void v_matchproto_(vsm_lock_f)
+vsm_vsmw_unlock(void)
+{
 	AZ(pthread_mutex_unlock(&vsm_mtx));
 }
 
@@ -500,10 +513,13 @@ VSM_Init(void)
 	assert(UINT_MAX % VSL_SEGMENTS == VSL_SEGMENTS - 1);
 
 	AZ(pthread_mutex_init(&vsl_mtx, NULL));
+	AZ(pthread_mutex_init(&vsc_mtx, NULL));
 	AZ(pthread_mutex_init(&vsm_mtx, NULL));
 
 	vsc_lock = vsm_vsc_lock;
 	vsc_unlock = vsm_vsc_unlock;
+	vsmw_lock = vsm_vsmw_lock;
+	vsmw_unlock = vsm_vsmw_unlock;
 
 	VSC_C_main = VSC_main_New(NULL, NULL, "");
 	AN(VSC_C_main);
diff --git a/bin/varnishd/common/common_vsc.c b/bin/varnishd/common/common_vsc.c
index 23dd5be..7beb903 100644
--- a/bin/varnishd/common/common_vsc.c
+++ b/bin/varnishd/common/common_vsc.c
@@ -71,8 +71,13 @@ struct vsc_seg {
 static VTAILQ_HEAD(,vsc_seg)	vsc_seglist =
     VTAILQ_HEAD_INITIALIZER(vsc_seglist);
 
-vsc_callback_f *vsc_lock;
-vsc_callback_f *vsc_unlock;
+static void v_matchproto_(vsm_lock_f)
+vsc_dummy_lock(void)
+{
+}
+
+vsm_lock_f *vsc_lock = vsc_dummy_lock;
+vsm_lock_f *vsc_unlock = vsc_dummy_lock;
 
 static const size_t vsc_overhead = PRNDUP(sizeof(struct vsc_head));
 
@@ -137,8 +142,7 @@ VRT_VSC_Alloc(struct vsmw_cluster *vc, struct vsc_seg **sg,
 	char buf[1024];
 	uintptr_t jjp;
 
-	if (vsc_lock != NULL)
-		vsc_lock();
+	vsc_lock();
 
 	jjp = (uintptr_t)jp;
 
@@ -178,8 +182,7 @@ VRT_VSC_Alloc(struct vsmw_cluster *vc, struct vsc_seg **sg,
 	VTAILQ_INSERT_TAIL(&vsc_seglist, vsg, list);
 	VWMB();
 	vsg->head->ready = 1;
-	if (vsc_unlock != NULL)
-		vsc_unlock();
+	vsc_unlock();
 	if (sg != NULL)
 		*sg = vsg;
 	return (vsg->ptr);
@@ -190,8 +193,7 @@ VRT_VSC_Destroy(const char *nm, struct vsc_seg *vsg)
 {
 	struct vsc_seg *dvsg;
 
-	if (vsc_lock != NULL)
-		vsc_lock();
+	vsc_lock();
 
 	AN(heritage.proc_vsmw);
 	CHECK_OBJ_NOTNULL(vsg, VSC_SEG_MAGIC);
@@ -209,6 +211,5 @@ VRT_VSC_Destroy(const char *nm, struct vsc_seg *vsg)
 		VTAILQ_REMOVE(&vsc_seglist, dvsg, list);
 		FREE_OBJ(dvsg);
 	}
-	if (vsc_unlock != NULL)
-		vsc_unlock();
+	vsc_unlock();
 }
diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c
index beb6fbd..8501cc4 100644
--- a/bin/varnishd/common/common_vsmw.c
+++ b/bin/varnishd/common/common_vsmw.c
@@ -55,6 +55,7 @@
 #include "vfil.h"
 #include "vrnd.h"
 
+#include "heritage.h"
 #include "vsmw.h"
 
 #ifndef MAP_HASSEMAPHORE
@@ -65,6 +66,14 @@
 #  define MAP_NOSYNC 0 /* XXX Linux */
 #endif
 
+static void v_matchproto_(vsm_lock_f)
+vsmw_dummy_lock(void)
+{
+}
+
+vsm_lock_f *vsmw_lock = vsmw_dummy_lock;
+vsm_lock_f *vsmw_unlock = vsmw_dummy_lock;
+
 /*--------------------------------------------------------------------*/
 
 struct vsmw_cluster {
@@ -249,6 +258,7 @@ VSMW_NewCluster(struct vsmw *vsmw, size_t len, const char *pfx)
 	struct vsmw_cluster *vc;
 	struct vsmwseg *seg;
 
+	vsmw_lock();
 	vc = vsmw_newcluster(vsmw, len, pfx);
 
 	ALLOC_OBJ(seg, VSMWSEG_MAGIC);
@@ -260,6 +270,7 @@ VSMW_NewCluster(struct vsmw *vsmw, size_t len, const char *pfx)
 	REPLACE(seg->id, "");
 	vsmw_addseg(vsmw, seg);
 
+	vsmw_unlock();
 	return (vc);
 }
 
@@ -268,6 +279,7 @@ VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vcp)
 {
 	struct vsmw_cluster *vc;
 
+	vsmw_lock();
 	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
 	AN(vcp);
 	vc = *vcp;
@@ -283,8 +295,10 @@ VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vcp)
 		 */
 		vsmw_delseg(vsmw, vc->cseg, 1);
 		vc->cseg = NULL;
-		if (vc->refs > 0)
+		if (vc->refs > 0) {
+			vsmw_unlock();
 			return;
+		}
 	}
 	AZ(munmap(vc->ptr, vc->len));
 
@@ -294,6 +308,7 @@ VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vcp)
 		assert (errno == ENOENT);
 	REPLACE(vc->fn, NULL);
 	FREE_OBJ(vc);
+	vsmw_unlock();
 }
 
 /*--------------------------------------------------------------------*/
@@ -305,6 +320,7 @@ VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc,
 {
 	struct vsmwseg *seg;
 
+	vsmw_lock();
 	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
 	(void)vc;
 
@@ -331,6 +347,7 @@ VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc,
 
 	vsmw_addseg(vsmw, seg);
 
+	vsmw_unlock();
 	return (seg->ptr);
 }
 
@@ -355,6 +372,7 @@ VSMW_Free(struct vsmw *vsmw, void **pp)
 	struct vsmwseg *seg;
 	void *p;
 
+	vsmw_lock();
 	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
 	AN(pp);
 	p = *pp;
@@ -365,10 +383,14 @@ VSMW_Free(struct vsmw *vsmw, void **pp)
 			break;
 	AN(seg);
 
-	if (!--seg->cluster->refs && seg->cluster->cseg == NULL)
+	if (!--seg->cluster->refs && seg->cluster->cseg == NULL) {
+		vsmw_unlock();
 		VSMW_DestroyCluster(vsmw, &seg->cluster);
+		vsmw_lock();
+	}
 
 	vsmw_delseg(vsmw, seg, 1);
+	vsmw_unlock();
 }
 
 /*--------------------------------------------------------------------*/
@@ -383,6 +405,7 @@ VSMW_New(int vdirfd, int mode, const char *idxname)
 	assert(mode > 0);
 	AN(idxname);
 
+	vsmw_lock();
 	ALLOC_OBJ(vsmw, VSMW_MAGIC);
 	AN(vsmw);
 
@@ -404,6 +427,7 @@ VSMW_New(int vdirfd, int mode, const char *idxname)
 	vsmw_idx_head(vsmw, fd);
 	AZ(close(fd));
 
+	vsmw_unlock();
 	return (vsmw);
 }
 
@@ -413,6 +437,7 @@ VSMW_Destroy(struct vsmw **pp)
 	struct vsmw *vsmw;
 	struct vsmwseg *seg, *s2;
 
+	vsmw_lock();
 	TAKE_OBJ_NOTNULL(vsmw, pp, VSMW_MAGIC);
 	VTAILQ_FOREACH_SAFE(seg, &vsmw->segs, list, s2)
 		vsmw_delseg(vsmw, seg, 0);
@@ -422,4 +447,5 @@ VSMW_Destroy(struct vsmw **pp)
 	VSB_destroy(&vsmw->vsb);
 	AZ(close(vsmw->vdirfd));
 	FREE_OBJ(vsmw);
+	vsmw_unlock();
 }
diff --git a/bin/varnishd/common/heritage.h b/bin/varnishd/common/heritage.h
index 7903cf6..db1dd37 100644
--- a/bin/varnishd/common/heritage.h
+++ b/bin/varnishd/common/heritage.h
@@ -115,8 +115,10 @@ struct transport;
 void XPORT_Init(void);
 const struct transport *XPORT_Find(const char *name);
 
-/* common/common_vsc.c */
-typedef void vsc_callback_f(void);
-extern vsc_callback_f *vsc_lock;
-extern vsc_callback_f *vsc_unlock;
+/* common/common_vsc.c & common/common_vsmw.c */
+typedef void vsm_lock_f(void);
+extern vsm_lock_f *vsc_lock;
+extern vsm_lock_f *vsc_unlock;
+extern vsm_lock_f *vsmw_lock;
+extern vsm_lock_f *vsmw_unlock;
 

From phk at FreeBSD.org  Tue Feb 27 08:13:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 27 Feb 2018 08:13:06 +0000 (UTC)
Subject: [master] 2637332 Clean out stuff which moved to vmod_vtc
Message-ID: <20180227081306.4F393B1004@lists.varnish-cache.org>


commit 2637332b602f37d270d92bdc370bbffbf6ed744c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 27 08:11:40 2018 +0000

    Clean out stuff which moved to vmod_vtc
    
    Fixes #2386

diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc
index 5933d52..ba5fa94 100644
--- a/lib/libvmod_debug/vmod.vcc
+++ b/lib/libvmod_debug/vmod.vcc
@@ -40,10 +40,6 @@ $Event event_function
 
 You are not supposed to be able to see this text in the .RST
 
-$Function VOID panic(STRING_LIST)
-
-Don't.
-
 $Function STRING author(ENUM { phk, des, kristian, mithrandir } person="phk",
 			ENUM { phk, slink, geoff } someone='phk')
 
@@ -65,14 +61,6 @@ $Function STRING test_priv_top(PRIV_TOP, STRING)
 
 Test function for TOP private pointers
 
-$Function BACKEND no_backend()
-
-Fails at backend selection
-
-$Function STEVEDORE no_stevedore()
-
-Fails at storage selection.
-
 $Object obj(STRING string="default", ENUM { one, two, three } number=one)
 
 Test object
@@ -141,10 +129,6 @@ $Function VOID fail()
 
 Function to fail vcl code.  (See also: RFC748)
 
-$Function VOID sleep(DURATION)
-
-Sleep the current worker thread.
-
 $Object dyn(STRING addr, STRING port)
 
 Dynamically create a single-backend director, addr and port must not be empty.
@@ -157,33 +141,6 @@ $Method VOID .refresh(STRING addr, STRING port)
 
 Dynamically refresh & (always!) replace the backend by a new one.
 
-$Function VOID workspace_allocate(ENUM { client, backend, session, thread },
-	  INT size)
-
-Allocate and zero out SIZE bytes from a workspace.
-If SIZE is negative, all but that many bytes are allocated from the workspace.
-(NB: Beware of the alignment imposed on workspace allocations.)
-
-$Function BOOL workspace_overflowed(ENUM { client, backend, session, thread })
-
-Return if the workspace overflow mark is set or not.
-
-$Function VOID workspace_overflow(ENUM { client, backend, session, thread })
-
-Mark a workspace as overflowed.
-
-$Function INT workspace_free(ENUM { client, backend, session, thread })
-
-Find how much unallocated space there is left in a workspace.
-
-$Function VOID workspace_snap(ENUM { client, backend, session, thread})
-
-Snapshot the named workspace. Only one snapshot may be active at a time.
-
-$Function VOID workspace_reset(ENUM { client, backend, session, thread })
-
-Reset to the previous snapshot of a workspace, taken from debug.workspace_snap.
-
 $Function VOID vcl_release_delay(DURATION)
 
 Hold a reference to the VCL when it goes cold for the given delay.
@@ -192,28 +149,10 @@ $Function BOOL match_acl(ACL acl, IP ip)
 
 Perform an IP match against a named ACL.
 
-$Function BOOL barrier_sync(STRING)
-
-Synchronize with a varnishtest shared barrier.
-
 $Function VOID test_probe(PROBE probe, PROBE same = 0)
 
 Only here to make sure probe definitions are passed properly.
 
-$Function INT typesize(STRING)
-
-Returns the size in bytes of a collection of C-datatypes.
-
-* 'p' = pointer
-* 'i' = int
-* 'd' = double
-* 'f' = float
-* 'l' = long
-* 's' = short
-* 'z' = size_t
-* 'o' = off_t
-* 'j' = intmax_t
-
 $Function VOID vsc_new()
 
 Add a vsc
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index 8e730dc..5e1c4d7 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -69,20 +69,6 @@ static pthread_mutex_t vsc_mtx = PTHREAD_MUTEX_INITIALIZER;
 static struct vsc_seg *vsc_seg;
 static struct VSC_debug *vsc;
 
-VCL_VOID v_matchproto_(td_debug_panic)
-xyzzy_panic(VRT_CTX, const char *str, ...)
-{
-	va_list ap;
-	const char *b;
-
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	WARN_RETIRED();
-	va_start(ap, str);
-	b = VRT_String(ctx->ws, "PANIC: ", str, ap);
-	va_end(ap);
-	VAS_Fail("VCL", "", 0, b, VAS_VCL);
-}
-
 VCL_STRING v_matchproto_(td_debug_author)
 xyzzy_author(VRT_CTX, VCL_ENUM person, VCL_ENUM someone)
 {
@@ -163,24 +149,6 @@ xyzzy_test_priv_vcl(VRT_CTX, struct vmod_priv *priv)
 	assert(!strcmp(priv_vcl->foo, "FOO"));
 }
 
-VCL_BACKEND
-xyzzy_no_backend(VRT_CTX)
-{
-
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	WARN_RETIRED();
-	return (NULL);
-}
-
-VCL_STEVEDORE v_matchproto_(td_debug_no_stevedore)
-xyzzy_no_stevedore(VRT_CTX)
-{
-
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	WARN_RETIRED();
-	return (NULL);
-}
-
 VCL_VOID v_matchproto_(td_debug_rot52)
 xyzzy_rot52(VRT_CTX, VCL_HTTP hp)
 {
@@ -368,129 +336,6 @@ event_function(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e)
 	}
 }
 
-VCL_VOID v_matchproto_(td_debug_sleep)
-xyzzy_sleep(VRT_CTX, VCL_DURATION t)
-{
-
-	CHECK_OBJ_ORNULL(ctx, VRT_CTX_MAGIC);
-	WARN_RETIRED();
-	VTIM_sleep(t);
-}
-
-static struct ws *
-wsfind(VRT_CTX, VCL_ENUM which)
-{
-	if (!strcmp(which, "client"))
-		return (ctx->ws);
-	else if (!strcmp(which, "backend"))
-		return (ctx->bo->ws);
-	else if (!strcmp(which, "session"))
-		return (ctx->req->sp->ws);
-	else if (!strcmp(which, "thread"))
-		return (ctx->req->wrk->aws);
-	else
-		WRONG("No such workspace.");
-}
-
-void
-xyzzy_workspace_allocate(VRT_CTX, VCL_ENUM which, VCL_INT size)
-{
-	struct ws *ws;
-	char *s;
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	WARN_RETIRED();
-
-	ws = wsfind(ctx, which);
-
-	WS_Assert(ws);
-	AZ(ws->r);
-
-	if (size < 0) {
-		size += WS_Reserve(ws, 0);
-		WS_Release(ws, 0);
-	}
-	if (size <= 0) {
-		VRT_fail(ctx, "Attempted negative WS allocation");
-		return;
-	}
-	s = WS_Alloc(ws, size);
-	if (!s)
-		return;
-	memset(s, '\0', size);
-}
-
-VCL_INT
-xyzzy_workspace_free(VRT_CTX, VCL_ENUM which)
-{
-	struct ws *ws;
-	unsigned u;
-
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	WARN_RETIRED();
-
-	ws = wsfind(ctx, which);
-
-	WS_Assert(ws);
-	u = WS_Reserve(ws, 0);
-	WS_Release(ws, 0);
-
-	return (u);
-}
-
-VCL_BOOL
-xyzzy_workspace_overflowed(VRT_CTX, VCL_ENUM which)
-{
-	struct ws *ws;
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	WARN_RETIRED();
-
-	ws = wsfind(ctx, which);
-	WS_Assert(ws);
-
-	return (WS_Overflowed(ws));
-}
-
-static uintptr_t debug_ws_snap;
-
-void
-xyzzy_workspace_snap(VRT_CTX, VCL_ENUM which)
-{
-	struct ws *ws;
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	WARN_RETIRED();
-
-	ws = wsfind(ctx, which);
-	WS_Assert(ws);
-
-	debug_ws_snap = WS_Snapshot(ws);
-}
-
-void
-xyzzy_workspace_reset(VRT_CTX, VCL_ENUM which)
-{
-	struct ws *ws;
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	WARN_RETIRED();
-
-	ws = wsfind(ctx, which);
-	WS_Assert(ws);
-
-	WS_Reset(ws, debug_ws_snap);
-}
-
-void
-xyzzy_workspace_overflow(VRT_CTX, VCL_ENUM which)
-{
-	struct ws *ws;
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	WARN_RETIRED();
-
-	ws = wsfind(ctx, which);
-	WS_Assert(ws);
-
-	WS_MarkOverflow(ws);
-}
-
 VCL_VOID v_matchproto_(td_debug_vcl_release_delay)
 xyzzy_vcl_release_delay(VRT_CTX, VCL_DURATION delay)
 {
@@ -511,39 +356,6 @@ xyzzy_match_acl(VRT_CTX, VCL_ACL acl, VCL_IP ip)
 	return (VRT_acl_match(ctx, acl, ip));
 }
 
-VCL_BOOL
-xyzzy_barrier_sync(VRT_CTX, VCL_STRING addr)
-{
-	const char *err;
-	char buf[32];
-	int sock, i;
-	ssize_t sz;
-
-	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	WARN_RETIRED();
-	AN(addr);
-	AN(*addr);
-
-	VSLb(ctx->vsl, SLT_Debug, "barrier_sync(\"%s\")", addr);
-	sock = VTCP_open(addr, NULL, 0., &err);
-	if (sock < 0) {
-		VSLb(ctx->vsl, SLT_Error, "Barrier connection failed: %s", err);
-		return (0);
-	}
-
-	sz = read(sock, buf, sizeof buf);
-	i = errno;
-	closefd(&sock);
-	if (sz == 0)
-		return (1);
-	if (sz < 0)
-		VSLb(ctx->vsl, SLT_Error,
-		    "Barrier read failed: %s (errno=%d)", strerror(i), i);
-	if (sz > 0)
-		VSLb(ctx->vsl, SLT_Error, "Barrier unexpected data (%zdB)", sz);
-	return (0);
-}
-
 VCL_VOID v_matchproto_(td_debug_test_probe)
 xyzzy_test_probe(VRT_CTX, VCL_PROBE probe, VCL_PROBE same)
 {
@@ -554,31 +366,6 @@ xyzzy_test_probe(VRT_CTX, VCL_PROBE probe, VCL_PROBE same)
 	AZ(same == NULL || probe == same);
 }
 
-VCL_INT
-xyzzy_typesize(VRT_CTX, VCL_STRING s)
-{
-	size_t i = 0;
-	const char *p;
-
-	WARN_RETIRED();
-	(void)ctx;
-	for (p = s; *p; p++) {
-		switch (*p) {
-		case 'p':	i += sizeof(void *); break;
-		case 'i':	i += sizeof(int); break;
-		case 'd':	i += sizeof(double); break;
-		case 'f':	i += sizeof(float); break;
-		case 'l':	i += sizeof(long); break;
-		case 's':	i += sizeof(short); break;
-		case 'z':	i += sizeof(size_t); break;
-		case 'o':	i += sizeof(off_t); break;
-		case 'j':	i += sizeof(intmax_t); break;
-		default:	return(-1);
-		}
-	}
-	return ((VCL_INT)i);
-}
-
 VCL_VOID
 xyzzy_vsc_new(VRT_CTX)
 {

From phk at FreeBSD.org  Tue Feb 27 08:55:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 27 Feb 2018 08:55:10 +0000 (UTC)
Subject: [master] f16eed3 Make sure certain failures from jail functions go
 into CLI responses.
Message-ID: <20180227085510.6BF97B1C65@lists.varnish-cache.org>


commit f16eed37949ab298abb83bc59ea59b444fe4e07f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 27 08:53:02 2018 +0000

    Make sure certain failures from jail functions go into CLI responses.
    
    Fixes 2461

diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 19e4060..81d11f5 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -123,8 +123,8 @@ enum jail_fixfd_e {
 typedef int jail_init_f(char **);
 typedef void jail_master_f(enum jail_master_e);
 typedef void jail_subproc_f(enum jail_subproc_e);
-typedef int jail_make_dir_f(const char *dname);
-typedef void jail_fixfd_f(int fd, enum jail_fixfd_e);
+typedef int jail_make_dir_f(const char *, const char *, struct vsb *);
+typedef void jail_fixfd_f(int, enum jail_fixfd_e);
 
 struct jail_tech {
 	unsigned		magic;
@@ -134,16 +134,16 @@ struct jail_tech {
 	jail_master_f		*master;
 	jail_subproc_f		*subproc;
 	jail_make_dir_f		*make_workdir;
-	jail_make_dir_f		*make_vcldir;
+	jail_make_dir_f		*make_subdir;
 	jail_fixfd_f		*fixfd;
 };
 
-void VJ_Init(const char *j_arg);
-void VJ_master(enum jail_master_e jme);
-void VJ_subproc(enum jail_subproc_e jse);
-int VJ_make_workdir(const char *dname);
-int VJ_make_vcldir(const char *dname);
-void VJ_fix_fd(int fd, enum jail_fixfd_e);
+void VJ_Init(const char *);
+void VJ_master(enum jail_master_e);
+void VJ_subproc(enum jail_subproc_e);
+int VJ_make_workdir(const char *);
+int VJ_make_subdir(const char *, const char *, struct vsb *);
+void VJ_fix_fd(int, enum jail_fixfd_e);
 
 extern const struct jail_tech jail_tech_unix;
 extern const struct jail_tech jail_tech_solaris;
diff --git a/bin/varnishd/mgt/mgt_jail.c b/bin/varnishd/mgt/mgt_jail.c
index 60b3228..d8d7dc9 100644
--- a/bin/varnishd/mgt/mgt_jail.c
+++ b/bin/varnishd/mgt/mgt_jail.c
@@ -142,7 +142,7 @@ VJ_make_workdir(const char *dname)
 	AN(dname);
 	CHECK_OBJ_NOTNULL(vjt, JAIL_TECH_MAGIC);
 	if (vjt->make_workdir != NULL) {
-		i = vjt->make_workdir(dname);
+		i = vjt->make_workdir(dname, NULL, NULL);
 		if (i)
 			return (i);
 		VJ_master(JAIL_MASTER_FILE);
@@ -169,17 +169,27 @@ VJ_make_workdir(const char *dname)
 }
 
 int
-VJ_make_vcldir(const char *dname)
+VJ_make_subdir(const char *dname, const char *what, struct vsb *vsb)
 {
+	int e;
 
 	AN(dname);
+	AN(what);
 	CHECK_OBJ_NOTNULL(vjt, JAIL_TECH_MAGIC);
-	if (vjt->make_vcldir != NULL)
-		return (vjt->make_vcldir(dname));
+	if (vjt->make_subdir != NULL)
+		return (vjt->make_subdir(dname, what, vsb));
 
 	if (mkdir(dname, 0755) < 0 && errno != EEXIST) {
-		MGT_Complain(C_ERR, "Cannot create VCL directory '%s': %s",
-		    dname, strerror(errno));
+		e = errno;
+		if (vsb != NULL) {
+			VSB_printf(vsb,
+			    "Cannot create %s directory '%s': %s\n",
+			    what, dname, strerror(e));
+		} else {
+			MGT_Complain(C_ERR,
+			    "Cannot create %s directory '%s': %s",
+			    what, dname, strerror(e));
+		}
 		return (1);
 	}
 	return (0);
diff --git a/bin/varnishd/mgt/mgt_jail_solaris.c b/bin/varnishd/mgt/mgt_jail_solaris.c
index 338cae3..8925d92 100644
--- a/bin/varnishd/mgt/mgt_jail_solaris.c
+++ b/bin/varnishd/mgt/mgt_jail_solaris.c
@@ -523,8 +523,8 @@ vjs_master(enum jail_master_e jme)
 
 const struct jail_tech jail_tech_solaris = {
 	.magic =	JAIL_TECH_MAGIC,
-	.name =	"solaris",
-	.init =	vjs_init,
+	.name =		"solaris",
+	.init =		vjs_init,
 	.master =	vjs_master,
 //	.make_workdir =	vjs_make_workdir,
 //	.storage_file =	vjs_storage_file,
diff --git a/bin/varnishd/mgt/mgt_jail_unix.c b/bin/varnishd/mgt/mgt_jail_unix.c
index 4257347..f91cf24 100644
--- a/bin/varnishd/mgt/mgt_jail_unix.c
+++ b/bin/varnishd/mgt/mgt_jail_unix.c
@@ -240,13 +240,25 @@ vju_subproc(enum jail_subproc_e jse)
 }
 
 static int v_matchproto_(jail_make_dir_f)
-vju_make_vcldir(const char *dname)
+vju_make_subdir(const char *dname, const char *what, struct vsb *vsb)
 {
+	int e;
+
+	AN(dname);
+	AN(what);
 	AZ(seteuid(0));
 
 	if (mkdir(dname, 0755) < 0 && errno != EEXIST) {
-		MGT_Complain(C_ERR, "Cannot create VCL directory '%s': %s",
-		    dname, strerror(errno));
+		e = errno;
+		if (vsb != NULL) {
+			VSB_printf(vsb,
+			    "Cannot create %s directory '%s': %s\n",
+			    what, dname, strerror(e));
+		} else {
+			MGT_Complain(C_ERR,
+			    "Cannot create %s directory '%s': %s",
+			    what, dname, strerror(e));
+		}
 		return (1);
 	}
 	AZ(chown(dname, vju_uid, vju_gid));
@@ -255,8 +267,12 @@ vju_make_vcldir(const char *dname)
 }
 
 static int v_matchproto_(jail_make_dir_f)
-vju_make_workdir(const char *dname)
+vju_make_workdir(const char *dname, const char *what, struct vsb *vsb)
 {
+
+	AN(dname);
+	AZ(what);
+	AZ(vsb);
 	AZ(seteuid(0));
 
 	if (mkdir(dname, 0755) < 0 && errno != EEXIST) {
@@ -297,7 +313,7 @@ const struct jail_tech jail_tech_unix = {
 	.name =		"unix",
 	.init =		vju_init,
 	.master =	vju_master,
-	.make_vcldir =	vju_make_vcldir,
+	.make_subdir =	vju_make_subdir,
 	.make_workdir =	vju_make_workdir,
 	.fixfd =	vju_fixfd,
 	.subproc =	vju_subproc,
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index eb1ef4d..d6a4141 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -745,7 +745,7 @@ main(int argc, char * const *argv)
 		ARGV_ERR("Cannot create working directory (%s): %s\n",
 		    dirname, strerror(errno));
 
-	if (VJ_make_vcldir("vmod_cache")) {
+	if (VJ_make_subdir("vmod_cache", "VMOD cache", NULL)) {
 		ARGV_ERR(
 		    "Cannot create vmod directory (%s/vmod_cache): %s\n",
 		    dirname, strerror(errno));
diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c
index 3c01059..d2d94aa 100644
--- a/bin/varnishd/mgt/mgt_vcc.c
+++ b/bin/varnishd/mgt/mgt_vcc.c
@@ -317,7 +317,7 @@ mgt_VccCompile(struct cli *cli, struct vclprog *vcl, const char *vclname,
 	vp.dir = strdup(VSB_data(sb));
 	AN(vp.dir);
 
-	if (VJ_make_vcldir(vp.dir)) {
+	if (VJ_make_subdir(vp.dir, "VCL", cli->sb)) {
 		free(vp.dir);
 		VSB_destroy(&sb);
 		VCLI_Out(cli, "VCL compilation failed");

From phk at FreeBSD.org  Tue Feb 27 08:55:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 27 Feb 2018 08:55:10 +0000 (UTC)
Subject: [master] 4d0620d 'abort' is a library symbol, don't overload.
Message-ID: <20180227085510.8D341B1C68@lists.varnish-cache.org>


commit 4d0620d59b17e46ba1b4dea53ca3a3e6ac65bf90
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 27 08:54:02 2018 +0000

    'abort' is a library symbol, don't overload.

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index f09bed9..efe91f2 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -943,7 +943,7 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2)
 	h2_error h2e;
 	struct h2_req *r2, *r22;
 	char b[8];
-	int abort = 0;
+	int abandon = 0;
 
 	ASSERT_RXTHR(h2);
 	(void)VTCP_blocking(*h2->htc->rfd);
@@ -957,7 +957,7 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2)
 		break;
 	case HTC_S_TIMEOUT:
 		VTAILQ_FOREACH_SAFE(r2, &h2->streams, list, r22) {
-			if (abort)
+			if (abandon)
 				break;
 			switch (r2->state) {
 			case H2_S_CLOSED:
@@ -968,7 +968,7 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2)
 			case H2_S_CLOS_REM:
 			case H2_S_CLOS_LOC:
 				if (h2_stream_tmo(h2, r2)) {
-					abort = 1;
+					abandon = 1;
 					continue;
 				}
 				return (1);

From phk at FreeBSD.org  Tue Feb 27 10:41:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 27 Feb 2018 10:41:09 +0000 (UTC)
Subject: [master] 796638a GC the WARN_RETIRED macro
Message-ID: <20180227104109.D6773B3C92@lists.varnish-cache.org>


commit 796638ab1a3b83f75fcf9311c9c18959882100f6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 27 09:05:30 2018 +0000

    GC the WARN_RETIRED macro

diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index 5e1c4d7..56dd7ef 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -46,14 +46,6 @@
 
 #include "common/common_param.h"
 
-#define WARN_RETIRED()							\
-	do {								\
-		VSL(SLT_Error, 0,					\
-		    "debug.%s is deprecated, use vmod-vtc instead.",	\
-		    __func__);						\
-	} while (0)
-
-
 struct priv_vcl {
 	unsigned		magic;
 #define PRIV_VCL_MAGIC		0x8E62FA9D

From phk at FreeBSD.org  Tue Feb 27 10:41:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 27 Feb 2018 10:41:09 +0000 (UTC)
Subject: [master] 68c4724 Flexelinting
Message-ID: <20180227104109.ED74BB3C95@lists.varnish-cache.org>


commit 68c4724113611a52036c11010b40d9bccbe29443
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 27 09:14:24 2018 +0000

    Flexelinting

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index a121c8e..890428a 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -198,6 +198,7 @@ builtin_vcl.c:	builtin.vcl
 	echo ' * Edit builtin.vcl instead and run make' >> $@
 	echo ' *' >> $@
 	echo ' */' >> $@
+	echo '#include "config.h"' >> $@
 	echo '#include "mgt/mgt.h"' >> $@
 	echo '' >> $@
 	echo 'const char * const builtin_vcl =' >> $@
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index 56dd7ef..579d384 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -39,7 +39,6 @@
 
 #include "vsa.h"
 #include "vsb.h"
-#include "vtcp.h"
 #include "vtim.h"
 #include "vcc_if.h"
 #include "VSC_debug.h"

From phk at FreeBSD.org  Tue Feb 27 10:41:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 27 Feb 2018 10:41:10 +0000 (UTC)
Subject: [master] c310f04 Flexelinting
Message-ID: <20180227104110.19C99B3C99@lists.varnish-cache.org>


commit c310f04b64a9606afae6b1e85414033c45f19253
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 27 09:55:31 2018 +0000

    Flexelinting

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index a8aadd1..9f564f1 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -732,7 +732,6 @@ extern const char H__Reason[];
 #define VXID(u) ((u) & VSL_IDENTMASK)
 uint32_t VXID_Get(struct worker *, uint32_t marker);
 extern volatile struct params * cache_param;
-extern volatile struct vre_limits *vparam_vre_limits;
 extern pthread_key_t witness_key;
 
 /* cache_lck.c */
diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index 043e968..f1e3915 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -52,8 +52,7 @@
 
 
 volatile struct params		*cache_param;
-// XXX generic solution?
-volatile struct vre_limits	*vparam_vre_limits;
+static volatile struct vre_limits	*vparam_vre_limits;
 
 /*--------------------------------------------------------------------
  * Per thread storage for the session currently being processed by
diff --git a/bin/varnishd/mgt/mgt_acceptor.c b/bin/varnishd/mgt/mgt_acceptor.c
index 64dad07..bffd461 100644
--- a/bin/varnishd/mgt/mgt_acceptor.c
+++ b/bin/varnishd/mgt/mgt_acceptor.c
@@ -141,7 +141,7 @@ MAC_reopen_sockets(void)
 /*--------------------------------------------------------------------*/
 
 static struct listen_sock *
-mk_listen_sock(struct listen_arg *la, const struct suckaddr *sa)
+mk_listen_sock(const struct listen_arg *la, const struct suckaddr *sa)
 {
 	struct listen_sock *ls;
 	int fail;

From phk at FreeBSD.org  Tue Feb 27 10:41:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 27 Feb 2018 10:41:10 +0000 (UTC)
Subject: [master] b0b5eaa Flexelinting
Message-ID: <20180227104110.36323B3C9D@lists.varnish-cache.org>


commit b0b5eaa1f4fb89a93c043b970a62a30148ed22d1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 27 10:09:04 2018 +0000

    Flexelinting

diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c
index ea33c5a..0a0cb6d 100644
--- a/bin/varnishtest/vtc_client.c
+++ b/bin/varnishtest/vtc_client.c
@@ -133,8 +133,8 @@ client_tcp_connect(struct vtclog *vl, const char *addr, double tmo,
 static int v_matchproto_(vus_resolved_f)
 uds_open(void *priv, const struct sockaddr_un *uds)
 {
-	double *p, tmo;
-	int s, i;
+	double *p;
+	int s, i, tmo;
 	struct pollfd fds[1];
 	socklen_t sl = sizeof(*uds);
 
@@ -142,14 +142,14 @@ uds_open(void *priv, const struct sockaddr_un *uds)
 	AN(uds);
 	p = priv;
 	assert(*p > 0.);
-	tmo = *p * 1e3;
+	tmo = (int)(*p * 1e3);
 
 	s = socket(uds->sun_family, SOCK_STREAM, 0);
 	if (s < 0)
 		return (s);
 
 	(void) VTCP_nonblocking(s);
-	i = connect(s, (const struct sockaddr *)uds, sl);
+	i = connect(s, (const void*)uds, sl);
 	if (i == 0)
 		return(s);
 	if (errno != EINPROGRESS) {
diff --git a/bin/varnishtest/vtc_h2_hpack.c b/bin/varnishtest/vtc_h2_hpack.c
index 9e30bf8..9b6f250 100644
--- a/bin/varnishtest/vtc_h2_hpack.c
+++ b/bin/varnishtest/vtc_h2_hpack.c
@@ -395,7 +395,7 @@ HPK_EncHdr(struct hpk_iter *iter, const struct hpk_hdr *h)
 	switch (h->t) {
 		case hpk_idx:
 			*iter->buf = 0x80;
-			num_encode(iter, 7, h->i);
+			assert(num_encode(iter, 7, h->i) != hpk_err);
 			return (ITER_DONE(iter));
 		case hpk_inc:
 			*iter->buf = 0x40;
diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c
index 496d7c7..f72bbb0 100644
--- a/bin/varnishtest/vtc_http.c
+++ b/bin/varnishtest/vtc_http.c
@@ -836,7 +836,7 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp,
 {
 	int bodylen = 0;
 	char *b, *c;
-	char *nullbody = NULL;
+	char *nullbody;
 	int nolen = 0;
 	int l;
 
@@ -1727,7 +1727,7 @@ cmd_http_settings(CMD_ARGS)
 			if (*p != '\0')
 				vtc_fatal(hp->vl, "-dectbl takes an integer as "
 				    "argument (found %s)", av[1]);
-			HPK_ResizeTbl(hp->decctx, n);
+			assert(HPK_ResizeTbl(hp->decctx, n) != hpk_err);
 			av++;
 		} else
 			vtc_fatal(vl, "Unknown settings spec: %s\n", *av);
diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c
index a779c5a..f4307b6 100644
--- a/bin/varnishtest/vtc_http2.c
+++ b/bin/varnishtest/vtc_http2.c
@@ -829,8 +829,8 @@ receive_frame(void *priv)
 		continue;
 	}
 	AZ(pthread_mutex_unlock(&hp->mtx));
-	if (!vtc_error)
-		AZ(vsb);
+	if (vsb != NULL)
+		VSB_destroy(&vsb);
 	return (NULL);
 }
 
@@ -1238,19 +1238,19 @@ cmd_sendhex(CMD_ARGS)
 	VSB_destroy(&vsb);
 }
 
-#define ENC(hdr, k, v)			\
-{					\
-	AN(k);				\
-	hdr.key.ptr = strdup(k);	\
-	AN(hdr.key.ptr);		\
-	hdr.key.len = strlen(k);	\
-	AN(v);				\
-	hdr.value.ptr = strdup(v);	\
-	AN(hdr.value.ptr);		\
-	hdr.value.len = strlen(v);	\
-	(void)HPK_EncHdr(iter, &hdr);	\
-	free(hdr.key.ptr);		\
-	free(hdr.value.ptr);		\
+#define ENC(hdr, k, v)					\
+{							\
+	AN(k);						\
+	hdr.key.ptr = strdup(k);			\
+	AN(hdr.key.ptr);				\
+	hdr.key.len = strlen(k);			\
+	AN(v);						\
+	hdr.value.ptr = strdup(v);			\
+	AN(hdr.value.ptr);				\
+	hdr.value.len = strlen(v);			\
+	assert(HPK_EncHdr(iter, &hdr) != hpk_err);	\
+	free(hdr.key.ptr);				\
+	free(hdr.value.ptr);				\
 }
 
 #define STR_ENC(av, field, str)						       \
@@ -1452,7 +1452,7 @@ cmd_tx11obj(CMD_ARGS)
 		else if (AV_IS("-idxHdr")) {
 			hdr.t = hpk_idx;
 			STRTOU32_CHECK(hdr.i, av, p, vl, "-idxHdr", 0);
-			HPK_EncHdr(iter, &hdr);
+			assert(HPK_EncHdr(iter, &hdr) != hpk_err);
 		}
 		else if (AV_IS("-litIdxHdr")) {
 			av++;
@@ -1469,7 +1469,7 @@ cmd_tx11obj(CMD_ARGS)
 			hdr.key.ptr = NULL;
 			hdr.key.len = 0;
 			STR_ENC(av, value,   "third -litHdr");
-			HPK_EncHdr(iter, &hdr);
+			assert(HPK_EncHdr(iter, &hdr) != hpk_err);
 		}
 		else if (AV_IS("-litHdr")) {
 			av++;
@@ -1482,7 +1482,7 @@ cmd_tx11obj(CMD_ARGS)
 
 			STR_ENC(av, key,   "second -litHdr");
 			STR_ENC(av, value, "fourth -litHdr");
-			HPK_EncHdr(iter, &hdr);
+			assert(HPK_EncHdr(iter, &hdr) != hpk_err);
 		}
 		else if (AV_IS("-nostrend")) {
 			f.flags &= ~END_STREAM;
@@ -1852,7 +1852,7 @@ cmd_txsettings(CMD_ARGS)
 		}
 		else if (!strcmp(*av, "-hdrtbl")) {
 			PUT_KV(av, vl, hdrtbl, val, 0x1);
-			HPK_ResizeTbl(s->hp->decctx, val);
+			assert(HPK_ResizeTbl(s->hp->decctx, val) != hpk_err);
 		}
 		else if (!strcmp(*av, "-maxstreams"))
 			PUT_KV(av, vl, maxstreams, val, 0x3);
@@ -2726,9 +2726,9 @@ b64_settings(const struct http *hp, const char *s)
 
 		if (v == 1) {
 			if (hp->sfd)
-				HPK_ResizeTbl(hp->encctx, v);
+				assert(HPK_ResizeTbl(hp->encctx, v) != hpk_err);
 			else
-				HPK_ResizeTbl(hp->decctx, v);
+				assert(HPK_ResizeTbl(hp->decctx, v) != hpk_err);
 		}
 
 		vtc_log(hp->vl, 4, "Upgrade: %s (%d): %ju",
diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c
index a0e531a..9c084ba 100644
--- a/bin/varnishtest/vtc_server.c
+++ b/bin/varnishtest/vtc_server.c
@@ -168,7 +168,7 @@ server_listen_uds(struct server *s, const char **errp)
 	 */
 	m = umask(0);
 	s->sock = VUS_resolver(s->listen, uds_listen, &h, errp);
-	umask(m);
+	(void)umask(m);
 	if (*errp != NULL)
 		return;
 	assert(s->sock > 0);
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 73ca160..1ba5e7c 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -65,7 +65,7 @@ struct varnish {
 	int			fds[4];
 	pid_t			pid;
 
-	float			syntax;
+	double			syntax;
 
 	pthread_t		tp;
 	pthread_t		tp_vsl;

From phk at FreeBSD.org  Tue Feb 27 10:41:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 27 Feb 2018 10:41:10 +0000 (UTC)
Subject: [master] 767279e Flexelinting
Message-ID: <20180227104110.4CC8DB3CA1@lists.varnish-cache.org>


commit 767279e7a81b3cc924dedd0150d320bf0fe7f3a7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 27 10:20:25 2018 +0000

    Flexelinting

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 62b174b..c5cf838 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -77,9 +77,9 @@ struct vsm_seg {
 	unsigned		magic;
 #define VSM_SEG_MAGIC		0xeb6c6dfd
 	unsigned		flags;
-#define VSM_FLAG_MARKSCAN	(1U<<0)
-#define VSM_FLAG_STALE		(1U<<1)
-#define VSM_FLAG_CLUSTER	(1U<<2)
+#define VSM_FLAG_MARKSCAN	(1U<<1)
+#define VSM_FLAG_STALE		(1U<<2)
+#define VSM_FLAG_CLUSTER	(1U<<3)
 	VTAILQ_ENTRY(vsm_seg)	list;
 	struct vsm_set		*set;
 	struct vsm_seg		*cluster;

From phk at FreeBSD.org  Tue Feb 27 10:41:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 27 Feb 2018 10:41:10 +0000 (UTC)
Subject: [master] 1c05470 Flexelinting
Message-ID: <20180227104110.65329B3CA5@lists.varnish-cache.org>


commit 1c05470dfce0aae174aa21cadbf89e70658ef8a1
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 27 10:21:30 2018 +0000

    Flexelinting

diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index f47c514..c687376 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -382,22 +382,22 @@ main(int argc, char **argv)
 		ident = VSM_Dup(vut->vsm, "Arg", "-i");
 	else
 		ident = strdup("");
-	if (!once) {
+	vut->dispatch_f = accumulate;
+	vut->dispatch_priv = NULL;
+	vut->sighup_f = sighup;
+	if (once) {
+		VUT_Main(vut);
+		dump();
+	} else {
 		if (pthread_create(&thr, NULL, do_curses, NULL) != 0) {
 			fprintf(stderr, "pthread_create(): %s\n",
 			    strerror(errno));
 			exit(1);
 		}
+		VUT_Main(vut);
+		end_of_file = 1;
+		AZ(pthread_join(thr, NULL));
 	}
-	vut->dispatch_f = accumulate;
-	vut->dispatch_priv = NULL;
-	vut->sighup_f = sighup;
-	VUT_Main(vut);
-	end_of_file = 1;
-	if (once)
-		dump();
-	else
-		pthread_join(thr, NULL);
 	VUT_Fini(&vut);
 	exit(0);
 }

From phk at FreeBSD.org  Tue Feb 27 10:41:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 27 Feb 2018 10:41:10 +0000 (UTC)
Subject: [master] d25fc50 Flexelinting
Message-ID: <20180227104110.7D64EB3CA9@lists.varnish-cache.org>


commit d25fc50c61210ed817d50ab9ff27d2742ec4e3c8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 27 10:39:38 2018 +0000

    Flexelinting

diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c
index 1f37397..6b33e00 100644
--- a/bin/varnishstat/varnishstat.c
+++ b/bin/varnishstat/varnishstat.c
@@ -276,6 +276,7 @@ main(int argc, char * const *argv)
 		case 'h':
 			/* Usage help */
 			usage(0);
+			break;
 		case 'l':
 			f_list = 1;
 			break;
diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index 11d6ffe..d498cca 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -257,7 +257,7 @@ sample_points(void)
 			update_ma(&pt->ma_1000, (int64_t)pt->cur);
 		} else if (pt->vpt->semantics == 'c') {
 			if (main_uptime != NULL && *main_uptime)
-				pt->avg = pt->cur / *main_uptime;
+				pt->avg = pt->cur / (double)*main_uptime;
 			else
 				pt->avg = 0.;
 			if (pt->t_last) {
@@ -471,7 +471,7 @@ draw_bar_t(void)
 	x += 4;
 	mvwprintw(w_bar_t, 0, x, "%.*s", colw_name - 4, "NAME");
 	x += colw_name - 4;
-	col = 0;
+	col = COL_CUR;
 	while (col < COL_LAST) {
 		if (X - x < COLW)
 			break;
@@ -520,7 +520,7 @@ draw_line_default(WINDOW *w, int y, int x, int X, const struct pt *pt)
 	AN(w);
 	AN(pt);
 
-	col = 0;
+	col = COL_CUR;
 	while (col < COL_LAST) {
 		if (X - x < COLW)
 			break;
@@ -608,7 +608,7 @@ draw_line_bytes(WINDOW *w, int y, int x, int X, const struct pt *pt)
 	AN(w);
 	AN(pt);
 
-	col = 0;
+	col = COL_CUR;
 	while (col < COL_LAST) {
 		if (X - x < COLW)
 			break;
@@ -652,7 +652,7 @@ draw_line_bytes(WINDOW *w, int y, int x, int X, const struct pt *pt)
 static void
 draw_line_bitmap(WINDOW *w, int y, int x, int X, const struct pt *pt)
 {
-	int ch;
+	unsigned ch;
 	enum {
 		COL_VAL,
 		COL_MAP,
@@ -663,7 +663,7 @@ draw_line_bitmap(WINDOW *w, int y, int x, int X, const struct pt *pt)
 	AN(pt);
 	assert(pt->vpt->format == 'b');
 
-	col = 0;
+	col = COL_VAL;
 	while (col < COL_LAST) {
 		switch (col) {
 		case COL_VAL:
@@ -703,7 +703,7 @@ draw_line_duration(WINDOW *w, int y, int x, int X, const struct pt *pt)
 	AN(w);
 	AN(pt);
 
-	col = 0;
+	col = COL_DUR;
 	while (col < COL_LAST) {
 		if (X - x < COLW)
 			break;
@@ -1021,7 +1021,7 @@ do_curses(struct vsm *vsm, struct vsc *vsc, double delay)
 		if (redraw)
 			draw_screen();
 
-		t = (t_sample + interval - now) * 1000;
+		t = (long)((t_sample + interval - now) * 1000);
 		wtimeout(w_status, t);
 
 		ch = wgetch(w_status);

From nils.goroll at uplex.de  Tue Feb 27 11:00:13 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 27 Feb 2018 11:00:13 +0000 (UTC)
Subject: [master] 63f051d rip out the vmod_re bandaid
Message-ID: <20180227110013.11C1CB4393@lists.varnish-cache.org>


commit 63f051dbb3770b80034b90e3e0e8f06fb60248f9
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Feb 27 11:58:44 2018 +0100

    rip out the vmod_re bandaid
    
    Ref: aa8875a44d587bc5530554963f93fffba99efadd
    Ref: c310f04b64a9606afae6b1e85414033c45f19253

diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index f1e3915..d1ab577 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -52,7 +52,6 @@
 
 
 volatile struct params		*cache_param;
-static volatile struct vre_limits	*vparam_vre_limits;
 
 /*--------------------------------------------------------------------
  * Per thread storage for the session currently being processed by
@@ -312,8 +311,6 @@ child_main(int sigmagic, size_t altstksz)
 #endif
 
 	cache_param = heritage.param;
-	// XXX TODO - generic solution?
-	vparam_vre_limits = &cache_param->vre_limits;
 
 	AZ(pthread_key_create(&req_key, NULL));
 	AZ(pthread_key_create(&bo_key, NULL));

From dridi at varni.sh  Tue Feb 27 11:08:11 2018
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Tue, 27 Feb 2018 12:08:11 +0100
Subject: [master] 767279e Flexelinting
In-Reply-To: <20180227104110.4CC8DB3CA1@lists.varnish-cache.org>
References: <20180227104110.4CC8DB3CA1@lists.varnish-cache.org>
Message-ID: <CABoVN9AwWCz55CJvrEeCH=7BHdTPMUdT_26m_ctWoHH1y51bYA@mail.gmail.com>

On Tue, Feb 27, 2018 at 11:41 AM, Poul-Henning Kamp <phk at freebsd.org> wrote:
>
> commit 767279e7a81b3cc924dedd0150d320bf0fe7f3a7
> Author: Poul-Henning Kamp <phk at FreeBSD.org>
> Date:   Tue Feb 27 10:20:25 2018 +0000
>
>     Flexelinting
>
> diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
> index 62b174b..c5cf838 100644
> --- a/lib/libvarnishapi/vsm.c
> +++ b/lib/libvarnishapi/vsm.c
> @@ -77,9 +77,9 @@ struct vsm_seg {
>         unsigned                magic;
>  #define VSM_SEG_MAGIC          0xeb6c6dfd
>         unsigned                flags;
> -#define VSM_FLAG_MARKSCAN      (1U<<0)
> -#define VSM_FLAG_STALE         (1U<<1)
> -#define VSM_FLAG_CLUSTER       (1U<<2)
> +#define VSM_FLAG_MARKSCAN      (1U<<1)
> +#define VSM_FLAG_STALE         (1U<<2)
> +#define VSM_FLAG_CLUSTER       (1U<<3)
>         VTAILQ_ENTRY(vsm_seg)   list;
>         struct vsm_set          *set;
>         struct vsm_seg          *cluster;

Asking just in case, this is not breaking the ABI, right?

Dridi

From phk at phk.freebsd.dk  Tue Feb 27 11:12:37 2018
From: phk at phk.freebsd.dk (Poul-Henning Kamp)
Date: Tue, 27 Feb 2018 11:12:37 +0000
Subject: [master] 767279e Flexelinting
In-Reply-To: <CABoVN9AwWCz55CJvrEeCH=7BHdTPMUdT_26m_ctWoHH1y51bYA@mail.gmail.com>
References: <20180227104110.4CC8DB3CA1@lists.varnish-cache.org>
 <CABoVN9AwWCz55CJvrEeCH=7BHdTPMUdT_26m_ctWoHH1y51bYA@mail.gmail.com>
Message-ID: <57324.1519729957@critter.freebsd.dk>

--------
In message <CABoVN9AwWCz55CJvrEeCH=7BHdTPMUdT_26m_ctWoHH1y51bYA at mail.gmail.com>, Dridi Boukelmoune writes:

>> diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
>> index 62b174b..c5cf838 100644
>> --- a/lib/libvarnishapi/vsm.c
>> +++ b/lib/libvarnishapi/vsm.c
>> @@ -77,9 +77,9 @@ struct vsm_seg {
>>         unsigned                magic;
>>  #define VSM_SEG_MAGIC          0xeb6c6dfd
>>         unsigned                flags;
>> -#define VSM_FLAG_MARKSCAN      (1U<<0)
>> -#define VSM_FLAG_STALE         (1U<<1)
>> -#define VSM_FLAG_CLUSTER       (1U<<2)
>> +#define VSM_FLAG_MARKSCAN      (1U<<1)
>> +#define VSM_FLAG_STALE         (1U<<2)
>> +#define VSM_FLAG_CLUSTER       (1U<<3)
>>         VTAILQ_ENTRY(vsm_seg)   list;
>>         struct vsm_set          *set;
>>         struct vsm_seg          *cluster;
>
>Asking just in case, this is not breaking the ABI, right?

No, it's a private struct.

-- 
Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
phk at FreeBSD.ORG         | TCP/IP since RFC 956
FreeBSD committer       | BSD since 4.3-tahoe    
Never attribute to malice what can adequately be explained by incompetence.

From phk at FreeBSD.org  Tue Feb 27 13:40:18 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 27 Feb 2018 13:40:18 +0000 (UTC)
Subject: [master] a5b0329 Pycodestyle-ify
Message-ID: <20180227134018.647A4B748D@lists.varnish-cache.org>


commit a5b0329489f273753e1b91f3d87ef1ba88ab0348
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 27 13:14:39 2018 +0000

    Pycodestyle-ify

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index a52d3e7..e7bfff4 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -154,7 +154,7 @@ def lwrap(s, width=64):
     """
     Wrap a C-prototype like string into a number of lines.
     """
-    l = []
+    ll = []
     p = ""
     while len(s) > width:
         y = s[:width].rfind(',')
@@ -162,12 +162,12 @@ def lwrap(s, width=64):
             y = s[:width].rfind('(')
         if y == -1:
             break
-        l.append(p + s[:y + 1])
+        ll.append(p + s[:y + 1])
         s = s[y + 1:].lstrip()
         p = "    "
     if len(s) > 0:
-        l.append(p + s)
-    return l
+        ll.append(p + s)
+    return ll
 
 
 def quote(s):
@@ -233,6 +233,7 @@ class ctype(object):
         while jl[-1][-1] is None:
                 jl[-1].pop(-1)
 
+
 def vtype(txt):
     j = len(txt)
     for i in (',', ' ', '\n', '\t'):
@@ -298,27 +299,29 @@ def arg(txt):
 
     return a, s
 
+
 def nmlegal(nm):
     return re.match('^[a-zA-Z0-9_]+$', nm)
 
+
 # XXX cant have ( or ) in an argument default value
 class prototype(object):
     def __init__(self, st, retval=True, prefix=""):
-        l = st.line[1]
+        ll = st.line[1]
         while True:
-            a1 = l.count("(")
-            a2 = l.count(")")
+            a1 = ll.count("(")
+            a2 = ll.count(")")
             if a1 > 0 and a1 == a2:
                 break
             n = st.doc.split("\n", 1)
-            l += n[0]
+            ll += n[0]
             st.doc = n[1]
 
         if retval:
-            self.retval, s = vtype(l)
+            self.retval, s = vtype(ll)
         else:
             self.retval = None
-            s = l
+            s = ll
         i = s.find("(")
         assert i > 0
         self.name = prefix + s[:i].strip()
@@ -359,7 +362,7 @@ class prototype(object):
         if self.retval is not None:
             s += self.retval.vcl() + " "
         s += self.name + "("
-        l = []
+        ll = []
         for i in self.args:
             t = i.vcl()
             if t in privs:
@@ -369,18 +372,18 @@ class prototype(object):
                     t += " " + i.nm
                 if i.defval is not None:
                     t += "=" + i.defval
-            l.append(t)
-        s += ", ".join(l) + ")"
+            ll.append(t)
+        s += ", ".join(ll) + ")"
         return s
 
     def c_ret(self):
         return self.retval.ct
 
     def c_args(self, a=[]):
-        l = list(a)
+        ll = list(a)
         for i in self.args:
-            l.append(i.ct)
-        return ", ".join(l)
+            ll.append(i.ct)
+        return ", ".join(ll)
 
     def c_fn(self, args=[], h=False):
         s = fn = ''
@@ -499,19 +502,19 @@ class s_module(stanza):
             fo.write("\n")
             return
 
-        l = []
+        ll = []
         for i in self.vcc.contents[1:]:
             j = i.rstlbl
             if j is not None:
-                l.append([j.split("_", 1)[1], j])
+                ll.append([j.split("_", 1)[1], j])
             if i.methods is None:
                 continue
             for x in i.methods:
                 j = x.rstlbl
-                l.append([j.split("_", 1)[1], j])
+                ll.append([j.split("_", 1)[1], j])
 
-        l.sort()
-        for i in l:
+        ll.sort()
+        for i in ll:
             fo.write("* :ref:`%s`\n" % i[1])
         fo.write("\n")
 
@@ -557,6 +560,7 @@ class s_event(stanza):
                 "Vmod_%s_Func._event" % self.vcc.modname
         ])
 
+
 class s_function(stanza):
     def parse(self):
         self.proto = prototype(self)
@@ -575,13 +579,13 @@ class s_function(stanza):
     def cstruct_init(self, fo):
         fo.write("\t" + self.proto.cname(pfx=True) + ",\n")
 
-    def json(self,jl):
+    def json(self, jl):
         jl.append([
                 "$FUNC",
                 "%s" % self.proto.name,
         ])
         self.proto.json(jl[-1], 'Vmod_%s_Func.%s' %
-                           (self.vcc.modname, self.proto.cname()))
+                        (self.vcc.modname, self.proto.cname()))
 
 
 class s_object(stanza):
@@ -653,25 +657,24 @@ class s_object(stanza):
                 "$OBJ",
                 self.proto.name,
                 "struct %s%s_%s" %
-                    (self.vcc.sympfx, self.vcc.modname, self.proto.name),
+                (self.vcc.sympfx, self.vcc.modname, self.proto.name),
         ]
 
-        l2 = [ "$INIT" ]
+        l2 = ["$INIT"]
         ll.append(l2)
         self.init.json(l2,
-            'Vmod_%s_Func.%s' % (self.vcc.modname, self.init.name))
+                       'Vmod_%s_Func.%s' % (self.vcc.modname, self.init.name))
 
-        l2 = [ "$FINI" ]
+        l2 = ["$FINI"]
         ll.append(l2)
         self.fini.json(l2,
-            'Vmod_%s_Func.%s' % (self.vcc.modname, self.fini.name))
+                       'Vmod_%s_Func.%s' % (self.vcc.modname, self.fini.name))
 
         for i in self.methods:
                 i.json(ll)
 
         jl.append(ll)
 
-
     def dump(self):
         super(s_object, self).dump()
         for i in self.methods:
@@ -699,7 +702,8 @@ class s_method(stanza):
                 self.proto.name[len(self.pfx)+1:]
         ])
         self.proto.json(jl[-1],
-            'Vmod_%s_Func.%s' % (self.vcc.modname, self.proto.cname()))
+                        'Vmod_%s_Func.%s' %
+                        (self.vcc.modname, self.proto.cname()))
 
 
 #######################################################################
@@ -794,8 +798,7 @@ class vcc(object):
         fo.write("#endif\n")
         fo.write("\n")
 
-        l = sorted(enum_values)
-        for j in l:
+        for j in sorted(enum_values):
             fo.write("extern VCL_ENUM %senum_%s;\n" % (self.sympfx, j))
         fo.write("\n")
 
@@ -809,8 +812,7 @@ class vcc(object):
         for j in self.contents:
             j.cstruct(fo)
         fo.write("\n")
-        l = sorted(enum_values)
-        for j in l:
+        for j in sorted(enum_values):
             fo.write("\tVCL_ENUM\t\t\t*enum_%s;\n" % j)
         fo.write("};\n")
 
@@ -819,17 +821,16 @@ class vcc(object):
         for j in self.contents:
             j.cstruct_init(fo)
         fo.write("\n")
-        l = sorted(enum_values)
-        for j in l:
+        for j in sorted(enum_values):
             fo.write("\t&%senum_%s,\n" % (self.sympfx, j))
         fo.write("};\n")
 
     def json(self, fo):
-        jl = [ ["$VMOD", "1.0" ] ]
+        jl = [["$VMOD", "1.0"]]
         for j in self.contents:
                 j.json(jl)
 
-        bz = bytearray(json.dumps(jl, separators=(",",":"))) + "\0"
+        bz = bytearray(json.dumps(jl, separators=(",", ":"))) + "\0"
         fo.write("\nstatic const char Vmod_Json[%d] = {\n" % len(bz))
         t = "\t"
         for i in bz:
@@ -891,8 +892,7 @@ class vcc(object):
 
         fo.write("\n")
 
-        l = sorted(enum_values)
-        for j in l:
+        for j in sorted(enum_values):
             fo.write('VCL_ENUM %senum_%s = "%s";\n' % (self.sympfx, j, j))
         fo.write("\n")
 
@@ -952,6 +952,7 @@ def runmain(inputvcc, rstdir, outputprefix):
 
     v.commit()
 
+
 if __name__ == "__main__":
     usagetext = "Usage: %prog [options] <vmod.vcc>"
     oparser = optparse.OptionParser(usage=usagetext)

From phk at FreeBSD.org  Tue Feb 27 13:40:18 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 27 Feb 2018 13:40:18 +0000 (UTC)
Subject: [master] ae5c7e2 Fix a very strange corner-case of vcl syntax level
 handling.
Message-ID: <20180227134018.760A5B7490@lists.varnish-cache.org>


commit ae5c7e25abb4fd0dcda26ca112d257bf3c705c34
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 27 13:38:52 2018 +0000

    Fix a very strange corner-case of vcl syntax level handling.
    
    Spotted by:	simon

diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 696aa6a..2d9478b 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -76,7 +76,6 @@ struct symbol;
 
 struct source {
 	VTAILQ_ENTRY(source)	list;
-	int			syntax;
 	char			*name;
 	const char		*b;
 	const char		*e;
diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index 0326298..eb82b52 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -276,6 +276,7 @@ static void
 vcc_ParseVcl(struct vcc *tl)
 {
 	struct token *tok0, *tok1, *tok2;
+	int syntax;
 
 	assert(vcc_IdIs(tl->t, "vcl"));
 	tok0 = tl->t;
@@ -283,14 +284,14 @@ vcc_ParseVcl(struct vcc *tl)
 
 	tok1 = tl->t;
 	Expect(tl, CNUM);
-	tok1->src->syntax = (*tl->t->b - '0') * 10;
+	syntax = (*tl->t->b - '0') * 10;
 	vcc_NextToken(tl);
 	Expect(tl, '.');
 	vcc_NextToken(tl);
 
 	Expect(tl, CNUM);
 	tok2 = tl->t;
-	tok1->src->syntax += (*tl->t->b - '0');
+	syntax += (*tl->t->b - '0');
 	vcc_NextToken(tl);
 
 	if (tok1->e - tok1->b != 1 || tok2->e - tok2->b != 1) {
@@ -300,6 +301,13 @@ vcc_ParseVcl(struct vcc *tl)
 		ERRCHK(tl);
 	}
 
+	if (syntax < VCL_LOW || syntax > VCL_HIGH) {
+		VSB_printf(tl->sb, "VCL version %.1f not supported.\n",
+		    .1 * syntax);
+		vcc_ErrWhere2(tl, tok0, tl->t);
+		ERRCHK(tl);
+	}
+
 	if (tl->t->tok != ';') {
 		/* Special handling, because next token might be 'vcl'
 		 * in the built-in VCL, and that would give a very
@@ -311,11 +319,13 @@ vcc_ParseVcl(struct vcc *tl)
 		ERRCHK(tl);
 	}
 	vcc_NextToken(tl);
-	if (tl->syntax != 0.0 && tok1->src->syntax > tl->syntax) {
+	if (tl->syntax == 0)
+		tl->syntax = syntax;
+	if (syntax > tl->syntax) {
 		VSB_printf(tl->sb,
 		    "VCL version %.1f higher than"
 		    " the top level version %.1f\n",
-		    .1 * tok1->src->syntax, .1 * tl->syntax);
+		    .1 * syntax, .1 * tl->syntax);
 		vcc_ErrWhere2(tl, tok0, tl->t);
 		ERRCHK(tl);
 	}
@@ -352,7 +362,6 @@ void
 vcc_Parse(struct vcc *tl)
 {
 	struct toplev *tp;
-	struct token *tok;
 
 	AZ(tl->indent);
 	if (tl->t->tok != ID || !vcc_IdIs(tl->t, "vcl")) {
@@ -365,16 +374,9 @@ vcc_Parse(struct vcc *tl)
 		vcc_ErrWhere(tl, tl->t);
 		ERRCHK(tl);
 	}
-	tok = tl->t;
 	vcc_ParseVcl(tl);
-	if (tok->src->syntax < VCL_LOW || tok->src->syntax > VCL_HIGH) {
-		VSB_printf(tl->sb, "VCL version %.1f not supported.\n",
-		    .1 * tok->src->syntax);
-		vcc_ErrWhere2(tl, tok, tl->t);
-		ERRCHK(tl);
-	}
-	tl->syntax = tl->t->src->syntax;
 	ERRCHK(tl);
+	AN(tl->syntax);
 	while (tl->t->tok != EOI) {
 		ERRCHK(tl);
 		switch (tl->t->tok) {

From phk at FreeBSD.org  Tue Feb 27 13:40:18 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 27 Feb 2018 13:40:18 +0000 (UTC)
Subject: [master] 60b699b Update to new error message
Message-ID: <20180227134018.8F3CAB7494@lists.varnish-cache.org>


commit 60b699b91581e2d2f46bddf83ff4b42f28bac930
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Feb 27 13:39:29 2018 +0000

    Update to new error message

diff --git a/bin/varnishtest/tests/v00049.vtc b/bin/varnishtest/tests/v00049.vtc
index 46cf468..53e2cb7 100644
--- a/bin/varnishtest/tests/v00049.vtc
+++ b/bin/varnishtest/tests/v00049.vtc
@@ -11,7 +11,7 @@ varnish v1 -syntax 4.0 -errvcl "silly buggers" {
 	backend b1 { .host = "127.0.0.1:8080"; }
 }
 
-varnish v1 -syntax 4.0 -errvcl "9.9 higher than the top level version" {
+varnish v1 -syntax 4.0 -errvcl "VCL version 9.9 not supported" {
 	vcl 9.9;
 	backend b1 { .host = "127.0.0.1:8080"; }
 }

From nils.goroll at uplex.de  Tue Feb 27 16:19:08 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 27 Feb 2018 16:19:08 +0000 (UTC)
Subject: [master] 02de5d3 vmodtool: Do not output the dummy return type VOID
 of constructors
Message-ID: <20180227161908.651CDBA2B4@lists.varnish-cache.org>


commit 02de5d368a597742118ea85b3fb67cf902a1c6fa
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue Feb 27 17:16:10 2018 +0100

    vmodtool: Do not output the dummy return type VOID of constructors
    
    generated VCL synopsis:
    
    old:	new OBJ = VOID dyn(STRING addr, STRING port)
    new:	new OBJ = dyn(STRING addr, STRING port)

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index e7bfff4..b24d7d2 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -307,6 +307,7 @@ def nmlegal(nm):
 # XXX cant have ( or ) in an argument default value
 class prototype(object):
     def __init__(self, st, retval=True, prefix=""):
+        self.st = st
         ll = st.line[1]
         while True:
             a1 = ll.count("(")
@@ -359,7 +360,7 @@ class prototype(object):
 
     def vcl_proto(self, short):
         s = ""
-        if self.retval is not None:
+        if self.retval is not None and type(self.st) != s_object:
             s += self.retval.vcl() + " "
         s += self.name + "("
         ll = []

From dridi.boukelmoune at gmail.com  Tue Feb 27 18:22:13 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue, 27 Feb 2018 18:22:13 +0000 (UTC)
Subject: [master] 8fc0ae6 Make vcldir downstream-friendly in etc/Makefile.am
Message-ID: <20180227182214.092E1BC5BC@lists.varnish-cache.org>


commit 8fc0ae67cccb4ed5e3cfc621bf97cd09aae2e8e7
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue Feb 27 19:15:12 2018 +0100

    Make vcldir downstream-friendly in etc/Makefile.am
    
    And sync bin/varnishd/Makefile.am to use the same (and more accurate)
    variable name.
    
    Spotted by Reza.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 890428a..66907db 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -145,14 +145,14 @@ nobase_pkginclude_HEADERS = \
 	cache/cache_filter.h \
 	waiter/waiter.h
 
-pkgdatadir = ${datarootdir}/${PACKAGE}/vcl
+vcldir=$(datarootdir)/$(PACKAGE)/vcl
 
 varnishd_CFLAGS = \
 	@PCRE_CFLAGS@ \
 	@SAN_CFLAGS@ \
 	-DVARNISH_STATE_DIR='"${VARNISH_STATE_DIR}"' \
 	-DVARNISH_VMOD_DIR='"${pkglibdir}/vmods"' \
-	-DVARNISH_VCL_DIR='"${pkgsysconfdir}:${pkgdatadir}"'
+	-DVARNISH_VCL_DIR='"${pkgsysconfdir}:${vcldir}"'
 
 varnishd_LDFLAGS = -export-dynamic
 
diff --git a/etc/Makefile.am b/etc/Makefile.am
index 8fc49b9..8cd999c 100644
--- a/etc/Makefile.am
+++ b/etc/Makefile.am
@@ -11,6 +11,6 @@ New users is recommended to use the example.vcl file as a starting point.\n\n";\
 	sed -n '/vcl_recv/,$$p' $(top_srcdir)/bin/varnishd/builtin.vcl ) | \
 	sed 's/^\(.*\)$$/# \1/' > builtin.vcl
 
-vcldir=$(datarootdir)/varnish/vcl
+vcldir=$(datarootdir)/$(PACKAGE)/vcl
 
 dist_vcl_DATA = devicedetect.vcl

From guillaume at varnish-software.com  Wed Feb 28 14:53:16 2018
From: guillaume at varnish-software.com (Guillaume Quintard)
Date: Wed, 28 Feb 2018 14:53:16 +0000 (UTC)
Subject: [master] 73278d8 vmodtool python3 compatibility
Message-ID: <20180228145316.A818BAE005@lists.varnish-cache.org>


commit 73278d8bbcfeae2a2ffa97fd06c2f8dd1bad8eb0
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Wed Feb 28 15:52:48 2018 +0100

    vmodtool python3 compatibility

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index b24d7d2..86fcdad 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -831,7 +831,8 @@ class vcc(object):
         for j in self.contents:
                 j.json(jl)
 
-        bz = bytearray(json.dumps(jl, separators=(",", ":"))) + "\0"
+        bz = bytearray(json.dumps(jl, separators=(",", ":")),
+                       encoding = "ascii") + b"\0"
         fo.write("\nstatic const char Vmod_Json[%d] = {\n" % len(bz))
         t = "\t"
         for i in bz:

From phk at FreeBSD.org  Wed Feb 28 14:57:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 28 Feb 2018 14:57:08 +0000 (UTC)
Subject: [master] 2530bad Create a global description of VCL data types.
Message-ID: <20180228145708.3F3D6AE16B@lists.varnish-cache.org>


commit 2530bad9099fe0821e2c07c0f50223b5925920ca
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Feb 28 11:12:00 2018 +0000

    Create a global description of VCL data types.

diff --git a/include/vrt.h b/include/vrt.h
index 4822b9f..9d67405 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -153,6 +153,15 @@ typedef double					VCL_TIME;
 typedef struct vcl *				VCL_VCL;
 typedef void					VCL_VOID;
 
+struct vrt_type {
+	unsigned			magic;
+#define VRT_TYPE_MAGIC			0xa943bc32
+	const char			*lname;
+	const char			*uname;
+	const char			*ctype;
+	size_t				szof;
+};
+
 /***********************************************************************
  * This is the composite argument we pass to compiled VCL and VRT
  * functions.
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 1f2987f..d744cf3 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -224,7 +224,7 @@ class vardef(object):
 			if self.typ != "STRING" and self.typ != "BODY":
 				s += "VCL_" + self.typ + ")"
 			else:
-				s += ctyp + ", ...)"
+				s += ctyp.c + ", ...)"
 			varproto(s);
 		fo.write("\tsym->w_methods =\n")
 		restrict(fo, self.wr)
@@ -320,11 +320,19 @@ stv_variables = (
 #######################################################################
 # VCL to C type conversion
 
-vcltypes = {
-	'STRINGS':		"void",
-	'STRING_LIST':		"void*",
-	'SUB':			"void*",
-}
+vcltypes = {}
+
+class vcltype(object):
+	def __init__(self, name, ctype, internal=False):
+		self.name = name
+		self.c = ctype
+		self.internal = internal
+		vcltypes[name] = self
+		
+
+vcltype("STRINGS", "void", True)
+vcltype("STRING_LIST", "void*", True)
+vcltype("SUB", "void*", True)
 
 fi = open(join(srcroot, "include/vrt.h"))
 
@@ -341,7 +349,7 @@ for i in fi:
 	if j[-1][:4] != "VCL_":
 		continue
 	d = " ".join(j[1:-1])
-	vcltypes[j[-1][4:-1]] = d
+	vcltype(j[-1][4:-1], d)
 fi.close()
 
 #######################################################################
@@ -631,9 +639,13 @@ for i in sorted(rets.keys()):
 
 fo.write("\n" + tbl40("#define VCL_RET_MAX", "%d\n" % n))
 
+fo.write("\n/* VCL Types */\n")
+for vcltype in sorted(vcltypes.keys()):
+	fo.write("extern const struct vrt_type VCL_TYPE_%s[1];\n" % vcltype)
 
-fo.write("""
 
+fo.write("""
+/* Compiled VCL Interface */
 typedef int vcl_event_f(VRT_CTX, enum vcl_event_e);
 typedef int vcl_init_f(VRT_CTX);
 typedef void vcl_fini_f(VRT_CTX);
@@ -729,7 +741,22 @@ parse_var_doc(join(buildroot, "doc/sphinx/reference/vcl_var.rst"))
 fo.write("}\n")
 
 for i in stv_variables:
-	fh.write(vcltypes[i[1]] + " VRT_Stv_" + i[0] + "(const char *);\n")
+	fh.write(vcltypes[i[1]].c + " VRT_Stv_" + i[0] + "(const char *);\n")
+
+fo.write("\n/* VCL type identifiers */\n")
+
+for vn in sorted(vcltypes.keys()):
+	v = vcltypes[vn]
+	if v.internal:
+		continue
+	fo.write("const struct vrt_type VCL_TYPE_%s[1] = { {\n" % v.name)
+	fo.write("\t.magic = VRT_TYPE_MAGIC,\n")
+	fo.write('\t.lname = "%s",\n' % v.name.lower())
+	fo.write('\t.uname = "%s",\n' % v.name)
+	fo.write('\t.ctype = "%s",\n' % v.c)
+	if v.c != "void":
+		fo.write('\t.szof = sizeof(VCL_%s),\n' % v.name)
+	fo.write("}};\n")
 
 fo.close()
 fh.close()
@@ -785,7 +812,7 @@ lint_start(fo)
 for i in stv_variables:
 	ct = vcltypes[i[1]]
 	fo.write("VRTSTVVAR(" + i[0] + ",\t" + i[1] + ",\t")
-	fo.write(ct + ",\t" + i[2] + ")")
+	fo.write(ct.c + ",\t" + i[2] + ")")
 	fo.write("\n")
 
 fo.write("#undef VRTSTVVAR\n")

From phk at FreeBSD.org  Wed Feb 28 14:57:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 28 Feb 2018 14:57:08 +0000 (UTC)
Subject: [master] e7c0497 Improve the RST output.
Message-ID: <20180228145708.588E6AE16E@lists.varnish-cache.org>


commit e7c0497404a5b4aadda71aead3b0e7fe946d6993
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Feb 28 14:44:39 2018 +0000

    Improve the RST output.
    
    I came for something else, but one thing led to another...
    
    Emit a proper summary at the top, but elide ENUMs to keep it
    compact.
    
    If the entire VCL prototype fits in the header, put it there.
    else if it fits with ENUMs elided do that or finally resort
    to just the name + "(...)".
    
    If the header didn't hold the full prototype, emit a codeblock
    with it, and split that in one line per argument if it is too
    wide for the manual pages.
    
    Remove the manual summary from the libvmod_blob docs.
    
    otherwise summarize it (ie: elide ENUMs)

diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 86fcdad..fca5b03 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -45,7 +45,6 @@ import random
 import copy
 import json
 
-rstfmt = False
 strict_abi = True
 
 AMBOILERPLATE = '''
@@ -226,7 +225,12 @@ class ctype(object):
             return "STRING"
         if self.spec is None:
             return self.vt
-        return self.vt + " {" + ",".join(self.spec) + "}"
+        return self.vt + " {" + ", ".join(self.spec) + "}"
+
+    def synopsis(self):
+        if self.vt == "STRING_LIST":
+            return "STRING"
+        return self.vt
 
     def json(self, jl):
         jl.append([self.vt, self.nm, self.defval, self.spec])
@@ -308,15 +312,8 @@ def nmlegal(nm):
 class prototype(object):
     def __init__(self, st, retval=True, prefix=""):
         self.st = st
+        self.obj = None
         ll = st.line[1]
-        while True:
-            a1 = ll.count("(")
-            a2 = ll.count(")")
-            if a1 > 0 and a1 == a2:
-                break
-            n = st.doc.split("\n", 1)
-            ll += n[0]
-            st.doc = n[1]
 
         if retval:
             self.retval, s = vtype(ll)
@@ -325,7 +322,9 @@ class prototype(object):
             s = ll
         i = s.find("(")
         assert i > 0
-        self.name = prefix + s[:i].strip()
+        self.prefix = prefix
+        self.bname = s[:i].strip()
+        self.name = self.prefix + self.bname
         self.vcc = st.vcc
         if not nmlegal(self.cname()):
             err("%s(): Illegal name\n" % self.name, warn=False)
@@ -358,14 +357,25 @@ class prototype(object):
             return self.vcc.sympfx + r
         return r
 
-    def vcl_proto(self, short):
-        s = ""
-        if self.retval is not None and type(self.st) != s_object:
+    def vcl_proto(self, short, pfx=""):
+        if type(self.st) == s_method:
+            pfx += pfx
+        s = pfx
+        if type(self.st) == s_object:
+            s += "new x" + self.bname + " = "
+        elif self.retval is not None:
             s += self.retval.vcl() + " "
-        s += self.name + "("
+
+        if type(self.st) == s_method:
+            s += self.obj + self.bname + "("
+        else:
+            s += self.name + "("
         ll = []
         for i in self.args:
-            t = i.vcl()
+            if short:
+                t = i.synopsis()
+            else:
+                t = i.vcl()
             if t in privs:
                 continue
             if not short:
@@ -374,9 +384,30 @@ class prototype(object):
                 if i.defval is not None:
                     t += "=" + i.defval
             ll.append(t)
-        s += ", ".join(ll) + ")"
+        t = ",@".join(ll)
+        if len(s + t) > 68 and not short:
+            s += "\n" + pfx + pfx
+            s += t.replace("@", "\n" + pfx + pfx)
+            s += "\n" + pfx + ")"
+        else:
+            s += t.replace("@", " ") + ")"
         return s
 
+    def rsthead(self, fo):
+        s = self.vcl_proto(False)
+        if len(s) < 60:
+            write_rst_hdr(fo, s, '-')
+        else:
+            s = self.vcl_proto(True)
+            if len(s) > 60:
+                s = self.name + "(...)"
+            write_rst_hdr(fo, s, '-')
+            fo.write("\n::\n\n" + self.vcl_proto(False, pfx="   ") + "\n")
+
+    def synopsis(self, fo, man):
+        fo.write(self.vcl_proto(True, pfx="   ") + "\n")
+        fo.write("  \n")
+
     def c_ret(self):
         return self.retval.ct
 
@@ -400,7 +431,7 @@ class prototype(object):
         self.retval.json(ll)
         ll.append(cfunc)
         for i in self.args:
-                i.json(ll)
+            i.json(ll)
         jl.append(ll)
 
 #######################################################################
@@ -409,10 +440,11 @@ class prototype(object):
 class stanza(object):
     def __init__(self, l0, doc, vcc):
         self.line = l0
-        if len(doc) == 1:
-            self.doc = doc[0]
-        else:
-            self.doc = ""
+        while len(doc) > 0 and doc[0] == '':
+            doc.pop(0)
+        while len(doc) > 0 and doc[-1] == '':
+            doc.pop(-1)
+        self.doc = doc
         self.vcc = vcc
         self.rstlbl = None
         self.methods = None
@@ -427,26 +459,27 @@ class stanza(object):
             fo.write(".. _" + self.rstlbl + ":\n\n")
 
         self.rsthead(fo, man)
+        fo.write("\n")
         self.rstmid(fo, man)
+        fo.write("\n")
         self.rsttail(fo, man)
+        fo.write("\n")
 
     def rsthead(self, fo, man):
         if self.proto is None:
             return
-        if rstfmt:
-            s = self.proto.vcl_proto(short=False)
-            write_rst_hdr(fo, s, '-')
-        else:
-            write_rst_hdr(fo, self.proto.name, '-')
-            s = self.proto.vcl_proto(short=False)
-            fo.write("\n::\n\n\t%s\n" % s)
+        self.proto.rsthead(fo)
 
     def rstmid(self, fo, man):
-        fo.write(self.doc + "\n")
+        fo.write("\n".join(self.doc) + "\n")
 
     def rsttail(self, fo, man):
         return
 
+    def synopsis(self, fo, man):
+        if self.proto is not None:
+            self.proto.synopsis(fo, man)
+
     def hfile(self, fo):
         return
 
@@ -487,22 +520,21 @@ class s_module(stanza):
         fo.write("\n")
         write_rst_hdr(fo, "SYNOPSIS", "=")
         fo.write("\n")
-        fo.write('import %s [from "path"] ;\n' % self.vcc.modname)
+        fo.write("\n::\n\n")
+        fo.write('   import %s [from "path"] ;\n' % self.vcc.modname)
+        fo.write("   \n")
+        for c in self.vcc.contents:
+            c.synopsis(fo, man)
         fo.write("\n")
 
     def rsttail(self, fo, man):
 
-        write_rst_hdr(fo, "CONTENTS", "=")
-        fo.write("\n")
-
         if man:
-            for i in self.vcc.contents[1:]:
-                if i.rstlbl is None:
-                    continue
-                fo.write("* %s\n" % i.proto.vcl_proto(short=True))
-            fo.write("\n")
             return
 
+        write_rst_hdr(fo, "CONTENTS", "=")
+        fo.write("\n")
+
         ll = []
         for i in self.vcc.contents[1:]:
             j = i.rstlbl
@@ -593,6 +625,7 @@ class s_object(stanza):
     def parse(self):
         self.proto = prototype(self, retval=False)
         self.proto.retval = vtype('VOID')[0]
+        self.proto.obj = "x" + self.proto.name
 
         self.init = copy.copy(self.proto)
         self.init.name += '__init'
@@ -606,15 +639,9 @@ class s_object(stanza):
         self.methods = []
 
     def rsthead(self, fo, man):
-        if rstfmt:
-            s = self.proto.vcl_proto(short=False)
-            write_rst_hdr(fo, "new OBJ = " + s, '=')
-        else:
-            write_rst_hdr(fo, self.proto.name, '-')
-            s = "new OBJ = " + self.proto.vcl_proto(short=False)
-            fo.write("\n::\n\n\t%s\n" % s)
+        self.proto.rsthead(fo)
 
-        fo.write(self.doc + "\n")
+        fo.write("\n" + "\n".join(self.doc) + "\n\n")
 
         for i in self.methods:
             i.rstfile(fo, man)
@@ -622,6 +649,11 @@ class s_object(stanza):
     def rstmid(self, fo, man):
         return
 
+    def synopsis(self, fo, man):
+        self.proto.synopsis(fo, man)
+        for i in self.methods:
+            i.proto.synopsis(fo, man)
+
     def chfile(self, fo, h):
         sn = self.vcc.sympfx + self.vcc.modname + "_" + self.proto.name
         fo.write("struct %s;\n" % sn)
@@ -688,6 +720,7 @@ class s_method(stanza):
         assert type(p) == s_object
         self.pfx = p.proto.name
         self.proto = prototype(self, prefix=self.pfx)
+        self.proto.obj = "x" + self.pfx
         self.rstlbl = "func_" + self.proto.name
         p.methods.append(self)
 
@@ -742,16 +775,18 @@ class vcc(object):
         a = "\n" + open(self.inputfile, "r").read()
         s = a.split("\n$")
         self.copyright = s.pop(0).strip()
-        for i in range(len(s)):
-            b = s[i].split("\n", 1)
-            c = b[0].split(None, 1)
-
-            if i == 0 and c[0] != "Module":
-                err("$Module must be first stanze", False)
+        while len(s):
+            ss = s.pop(0)
+            i = ss.find("\n\n")
+            if i > -1:
+                i += 1
+            else:
+                i = len(ss)
+            c = ss[:i].split()
             m = dispatch.get(c[0])
             if m is None:
-                err("Unknown stanze $%s" % c[0])
-            m(c, b[1:], self)
+                err("Unknown stanze $%s" % ss[:i])
+            m([c[0], " ".join(c[1:])], ss[i:].split('\n'), self)
 
     def rst_copyright(self, fo):
         write_rst_hdr(fo, "COPYRIGHT", "=")
diff --git a/lib/libvmod_blob/vmod.vcc b/lib/libvmod_blob/vmod.vcc
index a462f3a..341f3fe 100644
--- a/lib/libvmod_blob/vmod.vcc
+++ b/lib/libvmod_blob/vmod.vcc
@@ -8,25 +8,6 @@
 
 $Module blob 3 utilities for the VCL blob type
 
-::
-
-  # binary-to-text encodings
-  STRING blob.encode([ENUM encoding,] [ENUM case,] BLOB blob)
-  BLOB blob.decode([ENUM decoding,] [INT length,] STRING_LIST encoded)
-  STRING blob.transcode([ENUM decoding,] [ENUM encoding,] [ENUM case,]
-			[INT length,] STRING_LIST encoded)
-
-  # other utilities
-  BOOL blob.same(BLOB, BLOB)
-  BOOL blob.equal(BLOB, BLOB)
-  INT blob.length(BLOB)
-  BLOB blob.sub(BLOB, BYTES length [, BYTES offset])
-
-  # blob object
-  new OBJ = blob.blob([ENUM decoding,] STRING_LIST encoded)
-  BLOB <obj>.get()
-  STRING <obj>.encode([ENUM encoding,] [ENUM case])
-
 $ABI strict
 
 DESCRIPTION

From phk at FreeBSD.org  Wed Feb 28 22:33:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 28 Feb 2018 22:33:09 +0000 (UTC)
Subject: [master] 60f1a5f Try to get this test running on SunOS
Message-ID: <20180228223309.AB260B68AB@lists.varnish-cache.org>


commit 60f1a5f6083408b15087629b6d437a03960f5806
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Feb 28 22:32:01 2018 +0000

    Try to get this test running on SunOS
    
    (Theory: "stty raw" doesn't clear -echo)

diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc
index 054519f..fe5682d 100644
--- a/bin/varnishtest/tests/a00000.vtc
+++ b/bin/varnishtest/tests/a00000.vtc
@@ -45,17 +45,19 @@ process p1 "ps -lw | grep '[p][s]' ; tty ; sleep 1" -run -screen_dump
 
 process p2 "stty -a ; sleep 1" -run -screen_dump
 
-process p3 -hexdump {stty raw; echo "*" ; cat} -start
+process p3 "stty raw -echo ; stty -a ; sleep 1" -run -screen_dump
 
-process p3 -need-bytes 2
+process p4 -hexdump {stty raw -echo; echo "*" ; cat} -start
 
-process p3 -write "\x1b[H\x1b[2J1\x1b[79C2\x08>\x1b[25;1H3\x1b[25;80H"
-process p3 -write "4\x08>\x1b[A\x1b[Cv\x1b[22A^\x1b[79D^\x1b[;2H<\n\n\n\n"
-process p3 -write "\n\n\n\n\n\n\n\n\x1b[B\x1b[11B\x08<\x1b[24;Hv\x1b[12;1H"
-process p3 -write "111111112222222333333\x0d\x0a111111112"
-process p3 -write "222222333333\x0d\x0a111111112222222333"
-process p3 -write "333\x0d\x0a111111112222222333333\x0d\x0a\x1b[12"
-process p3 -write ";12H\x1b[K\x1b[13;12H\x1b[0K\x1b[14;12H\x1b[1K\x1b"
-process p3 -write "[15;12H\x1b[2K\x1b[3;1Hline3 <\x0d\x0a"
+process p4 -need-bytes 2
 
-process p3 -need-bytes 252 -screen_dump -stop
+process p4 -write "\x1b[H\x1b[2J1\x1b[79C2\x08>\x1b[25;1H3\x1b[25;80H"
+process p4 -write "4\x08>\x1b[A\x1b[Cv\x1b[22A^\x1b[79D^\x1b[;2H<\n\n\n\n"
+process p4 -write "\n\n\n\n\n\n\n\n\x1b[B\x1b[11B\x08<\x1b[24;Hv\x1b[12;1H"
+process p4 -write "111111112222222333333\x0d\x0a111111112"
+process p4 -write "222222333333\x0d\x0a111111112222222333"
+process p4 -write "333\x0d\x0a111111112222222333333\x0d\x0a\x1b[12"
+process p4 -write ";12H\x1b[K\x1b[13;12H\x1b[0K\x1b[14;12H\x1b[1K\x1b"
+process p4 -write "[15;12H\x1b[2K\x1b[3;1Hline3 <\x0d\x0a"
+
+process p4 -need-bytes 252 -screen_dump -stop

From kokoniimasu+git at gmail.com  Mon Feb 19 08:44:08 2018
From: kokoniimasu+git at gmail.com (Shohei Tanaka (@xcir))
Date: Mon, 19 Feb 2018 08:44:08 -0000
Subject: [master] 3a1f45d Add nuke_limited counter
Message-ID: <20180219084407.DE7C2B652F@lists.varnish-cache.org>


commit 3a1f45d05dcdc0d22a71148c25ae2dedcf93e6ea
Author: Shohei Tanaka(@xcir) <kokoniimasu+git at gmail.com>
Date:   Fri Feb 16 14:47:53 2018 +0900

    Add nuke_limited counter

diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc
index f303453..601cef7 100644
--- a/bin/varnishd/VSC_main.vsc
+++ b/bin/varnishd/VSC_main.vsc
@@ -253,6 +253,12 @@
 	Number of times an HTTP/2 stream was refused because the queue was
 	too long already. See also parameter thread_queue_limit.
 
+.. varnish_vsc:: nuke_limited
+	:oneliner:	Reached nuke_limit
+
+	Number of times more storage space were needed, but limit was reached in
+	a nuke_limit. See also parameter nuke_limit.
+
 .. varnish_vsc:: n_object
 	:type:	gauge
 	:oneliner:	object structs made
diff --git a/bin/varnishd/storage/storage_lru.c b/bin/varnishd/storage/storage_lru.c
index cc84b97..e58ccf6 100644
--- a/bin/varnishd/storage/storage_lru.c
+++ b/bin/varnishd/storage/storage_lru.c
@@ -172,6 +172,7 @@ LRU_NukeOne(struct worker *wrk, struct lru *lru)
 
 	if (wrk->strangelove-- <= 0) {
 		VSLb(wrk->vsl, SLT_ExpKill, "LRU reached nuke_limit");
+		VSC_C_main->nuke_limited++;
 		return (0);
 	}
 
diff --git a/bin/varnishtest/tests/r01764.vtc b/bin/varnishtest/tests/r01764.vtc
index 9f4e4cf..0dc1cb3 100644
--- a/bin/varnishtest/tests/r01764.vtc
+++ b/bin/varnishtest/tests/r01764.vtc
@@ -52,3 +52,5 @@ client c1 {
 	rxresp
 	expect resp.status == 503
 } -run
+
+varnish v1 -expect nuke_limited == 1

From kokoniimasu+git at gmail.com  Mon Feb 19 08:44:08 2018
From: kokoniimasu+git at gmail.com (Shohei Tanaka (@xcir))
Date: Mon, 19 Feb 2018 08:44:08 -0000
Subject: [master] b201fc7 rename nuke_limited -> n_lru_limited
Message-ID: <20180219084408.09F90B6532@lists.varnish-cache.org>


commit b201fc70d736e45051b481c7bf8000c9c0cf80d8
Author: Shohei Tanaka(@xcir) <kokoniimasu+git at gmail.com>
Date:   Fri Feb 16 20:19:40 2018 +0900

    rename nuke_limited -> n_lru_limited

diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc
index 601cef7..fd9a4e6 100644
--- a/bin/varnishd/VSC_main.vsc
+++ b/bin/varnishd/VSC_main.vsc
@@ -253,12 +253,6 @@
 	Number of times an HTTP/2 stream was refused because the queue was
 	too long already. See also parameter thread_queue_limit.
 
-.. varnish_vsc:: nuke_limited
-	:oneliner:	Reached nuke_limit
-
-	Number of times more storage space were needed, but limit was reached in
-	a nuke_limit. See also parameter nuke_limit.
-
 .. varnish_vsc:: n_object
 	:type:	gauge
 	:oneliner:	object structs made
@@ -313,6 +307,12 @@
 
 	Number of move operations done on the LRU list.
 
+.. varnish_vsc:: n_lru_limited
+	:oneliner:	Reached nuke_limit
+
+	Number of times more storage space were needed, but limit was reached in
+	a nuke_limit. See also parameter nuke_limit.
+
 .. varnish_vsc:: losthdr
 	:oneliner:	HTTP header overflows
 
diff --git a/bin/varnishd/storage/storage_lru.c b/bin/varnishd/storage/storage_lru.c
index e58ccf6..d5dcf6a 100644
--- a/bin/varnishd/storage/storage_lru.c
+++ b/bin/varnishd/storage/storage_lru.c
@@ -172,7 +172,7 @@ LRU_NukeOne(struct worker *wrk, struct lru *lru)
 
 	if (wrk->strangelove-- <= 0) {
 		VSLb(wrk->vsl, SLT_ExpKill, "LRU reached nuke_limit");
-		VSC_C_main->nuke_limited++;
+		VSC_C_main->n_lru_limited++;
 		return (0);
 	}
 
diff --git a/bin/varnishtest/tests/r01764.vtc b/bin/varnishtest/tests/r01764.vtc
index 0dc1cb3..4255063 100644
--- a/bin/varnishtest/tests/r01764.vtc
+++ b/bin/varnishtest/tests/r01764.vtc
@@ -53,4 +53,4 @@ client c1 {
 	expect resp.status == 503
 } -run
 
-varnish v1 -expect nuke_limited == 1
+varnish v1 -expect n_lru_limited == 1

From gquintard at users.noreply.github.com  Mon Feb 12 12:53:06 2018
From: gquintard at users.noreply.github.com (guillaume quintard)
Date: Mon, 12 Feb 2018 12:53:06 -0000
Subject: [master] ce0ccf5 Introduce optional resolve arg to std.ip
Message-ID: <20180212125306.AA6F2B6AEF@lists.varnish-cache.org>


commit ce0ccf578c9c102be831d7d08877b590d0c821e0
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Wed Jan 31 10:35:22 2018 +0100

    Introduce optional resolve arg to std.ip

diff --git a/bin/varnishtest/tests/m00011.vtc b/bin/varnishtest/tests/m00011.vtc
index acf3912..cd49806 100644
--- a/bin/varnishtest/tests/m00011.vtc
+++ b/bin/varnishtest/tests/m00011.vtc
@@ -15,6 +15,8 @@ varnish v1 -vcl+backend {
 		set resp.http.foo3 = std.ip("1.2.3.5", "127.0.0.3");
 		set resp.http.foo4 = std.ip("2001:db8::", "[::1]");
 		set resp.http.foo5 = std.ip("2001::db8::", "[::1]");
+		set resp.http.foo6 = std.ip("localhost", "0.0.0.0", resolve = false);
+		set resp.http.foo7 = std.ip("1.2.3.4", "0.0.0.0", resolve = false);
 	}
 } -start
 
@@ -29,4 +31,6 @@ client c1 {
 	expect resp.http.foo3 == "1.2.3.5"
 	expect resp.http.foo4 == "2001:db8::"
 	expect resp.http.foo5 == "::1"
+	expect resp.http.foo6 == "0.0.0.0"
+	expect resp.http.foo7 == "1.2.3.4"
 } -run
diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc
index 178b0b7..67d41f1 100644
--- a/lib/libvmod_std/vmod.vcc
+++ b/lib/libvmod_std/vmod.vcc
@@ -165,12 +165,17 @@ Example
 	|	...
 	| }
 
-$Function IP ip(STRING s, IP fallback)
+$Function IP ip(STRING s, IP fallback, BOOL resolve = 1)
 
 Description
 	Converts the string *s* to the first IP number returned by
 	the system library function getaddrinfo(3). If conversion
 	fails, *fallback* will be returned.
+
+	If *resolve* is true, getaddrinfo() is called using *AI_NUMERICHOST*
+	to avoid network lookups. This makes "pure" IP strings cheaper to
+	convert.
+
 Example
 	| if (std.ip(req.http.X-forwarded-for, "0.0.0.0") ~ my_acl) {
 	|	...
diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c
index 18b0b47..15cd9d1 100644
--- a/lib/libvmod_std/vmod_std_conversions.c
+++ b/lib/libvmod_std/vmod_std_conversions.c
@@ -77,7 +77,7 @@ vmod_integer(VRT_CTX, VCL_STRING p, VCL_INT i)
 }
 
 VCL_IP
-vmod_ip(VRT_CTX, VCL_STRING s, VCL_IP d)
+vmod_ip(VRT_CTX, VCL_STRING s, VCL_IP d, VCL_BOOL n)
 {
 	struct addrinfo hints, *res0 = NULL;
 	const struct addrinfo *res;
@@ -101,6 +101,8 @@ vmod_ip(VRT_CTX, VCL_STRING s, VCL_IP d)
 		memset(&hints, 0, sizeof(hints));
 		hints.ai_family = PF_UNSPEC;
 		hints.ai_socktype = SOCK_STREAM;
+		if (!n)
+			hints.ai_flags |= AI_NUMERICHOST;
 		error = getaddrinfo(s, "80", &hints, &res0);
 		if (!error) {
 			for (res = res0; res != NULL; res = res->ai_next) {