From fgsch at lodoss.net  Mon Jan  1 16:27:10 2018
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon,  1 Jan 2018 16:27:10 +0000 (UTC)
Subject: [master] fd678d7 Drop trailing semicolon
Message-ID: <20180101162710.F276F9A9B8@lists.varnish-cache.org>


commit fd678d720c3eb4e014df29d18aeff38b19916d11
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Jan 1 13:08:58 2018 +0000

    Drop trailing semicolon

diff --git a/lib/libvarnishapi/generate.py b/lib/libvarnishapi/generate.py
index 447c60f..a7b505c 100755
--- a/lib/libvarnishapi/generate.py
+++ b/lib/libvarnishapi/generate.py
@@ -118,7 +118,7 @@ vxp_fixed_token(const char *p, const char **q)
 				(l - 1, l))
 			fo.write("\t\t\t*q = p + %d;\n" % l)
 			fo.write("\t\t\treturn (%s);\n" % emit[j])
-			fo.write("\t\t}\n");
+			fo.write("\t\t}\n")
 		fo.write("\t\treturn (0);\n")
 
 	fo.write("\tdefault:\n\t\treturn (0);\n\t}\n}\n")
diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py
index 5cbe93b..dad08a8 100755
--- a/lib/libvcc/vmodtool.py
+++ b/lib/libvcc/vmodtool.py
@@ -640,7 +640,7 @@ class s_object(stanza):
             ['VRT_CTX', 'struct %s **' % sn, 'const char *'], h))
         fo.write(self.fini.c_fn(['struct %s **' % sn], h))
         for i in self.methods:
-            fo.write(i.proto.c_fn(['VRT_CTX', 'struct %s *' % sn], h));
+            fo.write(i.proto.c_fn(['VRT_CTX', 'struct %s *' % sn], h))
         fo.write("\n")
 
     def hfile(self, fo):
diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py
index 98283ed..4afeb73 100644
--- a/lib/libvcc/vsctool.py
+++ b/lib/libvcc/vsctool.py
@@ -110,7 +110,7 @@ class vscset(object):
 			ed["name"] = i.arg
 			ed["docs"] = "\n".join(i.getdoc())
 		s=json.dumps(dd, separators=(",",":")) + "\0"
-		fo.write("\nstatic const unsigned char");
+		fo.write("\nstatic const unsigned char")
 		fo.write(" vsc_%s_json[%d] = {\n" % (self.name, len(s)))
 		bz = bytearray(s)
 		t = "\t"
@@ -141,10 +141,10 @@ class vscset(object):
 		for i in self.mbrs:
 			fo.write("\tuint64_t\t%s;\n" % i.arg)
 		fo.write("};\n")
-		fo.write("\n");
+		fo.write("\n")
 
 		fo.write(self.struct + " *VSC_" + self.name + "_New")
-		fo.write("(struct vsc_seg **, const char *fmt, ...);\n");
+		fo.write("(struct vsc_seg **, const char *fmt, ...);\n")
 
 		fo.write("void VSC_" + self.name + "_Destroy")
 		fo.write("(struct vsc_seg **);\n")
@@ -184,9 +184,9 @@ class vscset(object):
 		self.emit_json(fo)
 
 		fo.write("\n")
-		fo.write(self.struct + "*\n");
+		fo.write(self.struct + "*\n")
 		fo.write("VSC_" + self.name + "_New")
-		fo.write("(struct vsc_seg **sg, const char *fmt, ...)\n");
+		fo.write("(struct vsc_seg **sg, const char *fmt, ...)\n")
 		fo.write("{\n")
 		fo.write("\tva_list ap;\n")
 		fo.write("\t" + self.struct + " *retval;\n")

From fgsch at lodoss.net  Mon Jan  1 16:27:11 2018
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon,  1 Jan 2018 16:27:11 +0000 (UTC)
Subject: [master] 4ce9376 Mark this noreturn
Message-ID: <20180101162711.173D29A9BC@lists.varnish-cache.org>


commit 4ce9376705c0116cf62689f139f322aea5df406d
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Jan 1 15:04:07 2018 +0000

    Mark this noreturn

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index f09f2d5..3407c70 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -378,7 +378,7 @@ pass(int sock)
 }
 
 
-static void
+static void v_noreturn_
 usage(int status)
 {
 	fprintf(stderr,

From fgsch at lodoss.net  Tue Jan  2 09:51:08 2018
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Tue,  2 Jan 2018 09:51:08 +0000 (UTC)
Subject: [master] 1001cd1 This should also compile if sigaltstack is missing
Message-ID: <20180102095108.D694FB3BF1@lists.varnish-cache.org>


commit 1001cd10d921db37ca2d1f772b525f825ad16037
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Tue Jan 2 09:48:26 2018 +0000

    This should also compile if sigaltstack is missing

diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index 0f92414..043e968 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -31,8 +31,10 @@
 
 #include "cache_varnishd.h"
 
+#include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
+
 #ifdef HAVE_SIGALTSTACK
 #  include <sys/mman.h>
 #endif
@@ -125,7 +127,6 @@ THR_GetName(void)
  * Generic setup all our threads should call
  */
 #ifdef HAVE_SIGALTSTACK
-#include <signal.h>
 static stack_t altstack;
 #endif
 
@@ -277,6 +278,8 @@ child_sigmagic(size_t altstksz)
 	altstack.ss_size = sz;
 	altstack.ss_flags = 0;
 	sa.sa_flags |= SA_ONSTACK;
+#else
+	(void)altstksz;
 #endif
 
 	THR_Init();

From phk at FreeBSD.org  Wed Jan  3 09:46:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed,  3 Jan 2018 09:46:10 +0000 (UTC)
Subject: [master] 9be1300 Move vsmw from libvarnish to varnishd/common
Message-ID: <20180103094610.E5698A9518@lists.varnish-cache.org>


commit 9be13007ffcb36a14f7780a37ba5f626c123802f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 3 09:44:47 2018 +0000

    Move vsmw from libvarnish to varnishd/common

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index 68a7072..3cf7519 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -52,6 +52,7 @@ varnishd_SOURCES = \
 	cache/cache_wrk.c \
 	cache/cache_ws.c \
 	common/common_vsc.c \
+	common/common_vsmw.c \
 	hash/hash_classic.c \
 	hash/hash_critbit.c \
 	hash/hash_simple_list.c \
@@ -123,6 +124,7 @@ noinst_HEADERS = \
 	cache/cache_vgz.h \
 	common/common_param.h \
 	common/heritage.h \
+	common/vsmw.h \
 	hash/hash_slinger.h \
 	hpack/vhp.h \
 	http1/cache_http1.h \
diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index cbb72a8..c01e01e 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -35,13 +35,13 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-#include "common/heritage.h"
-
 #include "vend.h"
 #include "vgz.h"
 #include "vsl_priv.h"
 #include "vmb.h"
-#include "vsmw.h"
+
+#include "common/heritage.h"
+#include "common/vsmw.h"
 
 /* These cannot be struct lock, which depends on vsm/vsl working */
 static pthread_mutex_t vsl_mtx;
diff --git a/bin/varnishd/common/common_vsc.c b/bin/varnishd/common/common_vsc.c
index 3a719ed..6b8c0c5 100644
--- a/bin/varnishd/common/common_vsc.c
+++ b/bin/varnishd/common/common_vsc.c
@@ -42,10 +42,10 @@
 #include "vas.h"
 #include "vmb.h"
 #include "vsc_priv.h"
-#include "vsmw.h"
 #include "vqueue.h"
 
-#include "common/heritage.h"
+#include "heritage.h"
+#include "vsmw.h"
 
 /*--------------------------------------------------------------------*/
 
diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c
new file mode 100644
index 0000000..a5cee24
--- /dev/null
+++ b/bin/varnishd/common/common_vsmw.c
@@ -0,0 +1,317 @@
+/*-
+ * Copyright (c) 2010-2011 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * VSM stuff common to manager and child.
+ *
+ * Please see comments in <vsm_priv.h> for details of protocols and
+ * data consistency.
+ *
+ */
+
+#include "config.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+
+#include "vdef.h"
+#include "vas.h"
+#include "vsb.h"
+#include "miniobj.h"
+#include "vqueue.h"
+
+#include "vfil.h"
+#include "vrnd.h"
+
+#include "vsmw.h"
+
+#ifndef MAP_HASSEMAPHORE
+#  define MAP_HASSEMAPHORE 0 /* XXX Linux */
+#endif
+
+#ifndef MAP_NOSYNC
+#  define MAP_NOSYNC 0 /* XXX Linux */
+#endif
+
+/*--------------------------------------------------------------------*/
+
+struct vsmwseg {
+	unsigned			magic;
+#define VSMWSEG_MAGIC			0x7e4ccaea
+	VTAILQ_ENTRY(vsmwseg)		list;
+	char				*fn;
+
+	char				*class;
+	size_t				len;
+	char				*id;
+	void				*ptr;
+};
+
+struct vsmw {
+	unsigned			magic;
+#define VSMW_MAGIC			0xc2ca2cd9
+	int				vdirfd;
+	int				mode;
+	char				*idx;
+	VTAILQ_HEAD(, vsmwseg)		segs;
+	struct vsb			*vsb;
+	pid_t				pid;
+	time_t				birth;
+};
+
+/*--------------------------------------------------------------------*/
+
+static void
+vsmw_idx_head(const struct vsmw *vsmw, int fd)
+{
+	char buf[64];
+
+	bprintf(buf, "# %jd %jd\n", (intmax_t)vsmw->pid, (intmax_t)vsmw->birth);
+	assert(write(fd, buf, strlen(buf)) == strlen(buf));
+}
+
+static void
+vsmw_write_index(const struct vsmw *vsmw, int fd, const struct vsmwseg *seg)
+{
+	ssize_t s;
+
+	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
+	CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
+	VSB_clear(vsmw->vsb);
+	VSB_printf(vsmw->vsb, "%s 0 %zu %s %s\n",
+	    seg->fn,
+	    seg->len,
+	    seg->class,
+	    seg->id);
+	AZ(VSB_finish(vsmw->vsb));
+	s = write(fd, VSB_data(vsmw->vsb), VSB_len(vsmw->vsb));
+	assert(s == VSB_len(vsmw->vsb));
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+vsmw_mkent(const struct vsmw *vsmw, const char *pfx)
+{
+	int fd;
+
+	while (1) {
+		VSB_clear(vsmw->vsb);
+		VSB_printf(vsmw->vsb, "_.%s", pfx);
+		VSB_printf(vsmw->vsb, ".%08lx", VRND_RandomTestable());
+		VSB_printf(vsmw->vsb, "%08lx", VRND_RandomTestable());
+		AZ(VSB_finish(vsmw->vsb));
+		fd = openat(vsmw->vdirfd, VSB_data(vsmw->vsb), O_RDONLY);
+		if (fd < 0 && errno == ENOENT)
+			return;
+		if (fd >= 0)
+			AZ(close(fd));
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+void *
+VSMW_Allocv(struct vsmw *vsmw, const char *class, size_t len,
+    const char *fmt, va_list va)
+{
+	struct vsmwseg *seg;
+	int fd;
+	size_t ps;
+
+	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
+
+	ALLOC_OBJ(seg, VSMWSEG_MAGIC);
+	AN(seg);
+	REPLACE(seg->class, class);
+	seg->len = len;
+
+	VSB_clear(vsmw->vsb);
+	VSB_vprintf(vsmw->vsb, fmt, va);
+	AZ(VSB_finish(vsmw->vsb));
+	REPLACE(seg->id, VSB_data(vsmw->vsb));
+
+	vsmw_mkent(vsmw, class);
+	REPLACE(seg->fn, VSB_data(vsmw->vsb));
+
+	ps = getpagesize();
+	len = RUP2(len, ps);
+
+	fd = openat(vsmw->vdirfd, seg->fn,
+	    O_RDWR | O_CREAT | O_EXCL, vsmw->mode);
+	assert(fd >= 0);
+
+	AZ(VFIL_allocate(fd, (off_t)len, 1));
+
+	seg->ptr = (void *)mmap(NULL, len,
+	    PROT_READ|PROT_WRITE,
+	    MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED,
+	    fd, 0);
+
+	AZ(close(fd));
+	assert(seg->ptr != MAP_FAILED);
+	(void)mlock(seg->ptr, len);
+
+	VTAILQ_INSERT_TAIL(&vsmw->segs, seg, list);
+	fd = openat(vsmw->vdirfd, vsmw->idx, O_APPEND | O_WRONLY);
+	assert(fd >= 0);
+	vsmw_write_index(vsmw, fd, seg);
+	AZ(close(fd));
+
+	return (seg->ptr);
+}
+
+void *
+VSMW_Allocf(struct vsmw *vsmw, const char *class, size_t len,
+    const char *fmt, ...)
+{
+	va_list ap;
+	void *p;
+
+	va_start(ap, fmt);
+	p = VSMW_Allocv(vsmw, class, len, fmt, ap);
+	va_end(ap);
+	return (p);
+}
+
+/*--------------------------------------------------------------------*/
+static void
+vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
+{
+	char *t = NULL;
+	int fd;
+	size_t len;
+
+	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
+	CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
+
+	len = getpagesize();
+	len = RUP2(seg->len, len);
+	AZ(munmap(seg->ptr, len));
+
+	VTAILQ_REMOVE(&vsmw->segs, seg, list);
+	if (unlinkat(vsmw->vdirfd, seg->fn, 0))
+		assert (errno == ENOENT);
+	REPLACE(seg->fn, NULL);
+	REPLACE(seg->class, NULL);
+	REPLACE(seg->id, NULL);
+	FREE_OBJ(seg);
+
+	if (fixidx) {
+		vsmw_mkent(vsmw, vsmw->idx);
+		REPLACE(t, VSB_data(vsmw->vsb));
+		AN(t);
+		fd = openat(vsmw->vdirfd, t, O_WRONLY|O_CREAT|O_EXCL, vsmw->mode);
+		assert(fd >= 0);
+		vsmw_idx_head(vsmw, fd);
+		VTAILQ_FOREACH(seg, &vsmw->segs, list)
+			vsmw_write_index(vsmw, fd, seg);
+		AZ(close(fd));
+		AZ(renameat(vsmw->vdirfd, t, vsmw->vdirfd, vsmw->idx));
+		REPLACE(t, NULL);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
+void
+VSMW_Free(struct vsmw *vsmw, void **pp)
+{
+	struct vsmwseg *seg;
+	void *p;
+
+	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
+	AN(pp);
+	p = *pp;
+	AN(p);
+	*pp = NULL;
+	VTAILQ_FOREACH(seg, &vsmw->segs, list)
+		if (seg->ptr == p)
+			break;
+	AN(seg);
+	vsmw_delseg(vsmw, seg, 1);
+}
+
+/*--------------------------------------------------------------------*/
+
+struct vsmw *
+VSMW_New(int vdirfd, int mode, const char *idxname)
+{
+	struct vsmw *vsmw;
+	int fd;
+
+	assert(vdirfd > 0);
+	assert(mode > 0);
+	AN(idxname);
+
+	ALLOC_OBJ(vsmw, VSMW_MAGIC);
+	AN(vsmw);
+
+	VTAILQ_INIT(&vsmw->segs);
+	vsmw->vsb = VSB_new_auto();
+	AN(vsmw->vsb);
+	REPLACE(vsmw->idx, idxname);
+	vsmw->mode = mode;
+	vsmw->vdirfd = vdirfd;
+	vsmw->pid = getpid();
+	vsmw->birth = time(NULL);
+
+	if (unlinkat(vdirfd, vsmw->idx, 0))
+		assert (errno == ENOENT);
+	fd = openat(vdirfd,
+	    vsmw->idx, O_APPEND | O_WRONLY | O_CREAT, vsmw->mode);
+	assert(fd >= 0);
+	vsmw_idx_head(vsmw, fd);
+	AZ(close(fd));
+
+	return (vsmw);
+}
+
+void
+VSMW_Destroy(struct vsmw **pp)
+{
+	struct vsmw *vsmw;
+	struct vsmwseg *seg, *s2;
+
+	TAKE_OBJ_NOTNULL(vsmw, pp, VSMW_MAGIC);
+	if (unlinkat(vsmw->vdirfd, vsmw->idx, 0))
+		assert (errno == ENOENT);
+	REPLACE(vsmw->idx, NULL);
+	VTAILQ_FOREACH_SAFE(seg, &vsmw->segs, list, s2)
+		vsmw_delseg(vsmw, seg, 0);
+	VSB_destroy(&vsmw->vsb);
+	AZ(close(vsmw->vdirfd));
+	FREE_OBJ(vsmw);
+}
diff --git a/bin/varnishd/common/vsmw.h b/bin/varnishd/common/vsmw.h
new file mode 100644
index 0000000..1ba2975
--- /dev/null
+++ b/bin/varnishd/common/vsmw.h
@@ -0,0 +1,41 @@
+/*-
+ * Copyright (c) 2017 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifdef VSMW_H_INCLUDED
+#error "Multiple includes of vsmw.h"
+#endif
+#define VSMW_H_INCLUDED
+
+struct vsmw;
+
+void *VSMW_Allocv(struct vsmw *, const char *, size_t, const char *, va_list);
+void *VSMW_Allocf(struct vsmw *, const char *, size_t, const char *, ...);
+void VSMW_Free(struct vsmw *, void **);
+
+struct vsmw *VSMW_New(int, int, const char *);
+void VSMW_Destroy(struct vsmw **);
diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index c0ce43d..062800e 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -44,8 +44,7 @@
 #include <syslog.h>
 #include <unistd.h>
 
-#include "mgt/mgt.h"
-#include "common/heritage.h"
+#include "mgt.h"
 
 #include "vbm.h"
 #include "vcli_serve.h"
@@ -53,7 +52,9 @@
 #include "vfil.h"
 #include "vlu.h"
 #include "vtim.h"
-#include "vsmw.h"
+
+#include "common/heritage.h"
+#include "common/vsmw.h"
 
 static pid_t		child_pid = -1;
 
diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index 954c6e2..c165ceb 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -41,10 +41,11 @@
 #include <unistd.h>
 
 #include "mgt/mgt.h"
-#include "common/heritage.h"
 
 #include "vsm_priv.h"
-#include "vsmw.h"
+
+#include "common/heritage.h"
+#include "common/vsmw.h"
 
 static struct vsmw *mgt_vsmw;
 
diff --git a/include/Makefile.am b/include/Makefile.am
index 822dcdd..d59b310 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -92,7 +92,6 @@ nobase_noinst_HEADERS = \
 	vpf.h \
 	vsc_priv.h \
 	vsl_priv.h \
-	vsmw.h \
 	vsm_priv.h \
 	vsub.h \
 	vss.h \
diff --git a/include/vsmw.h b/include/vsmw.h
deleted file mode 100644
index 1ba2975..0000000
--- a/include/vsmw.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/*-
- * Copyright (c) 2017 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#ifdef VSMW_H_INCLUDED
-#error "Multiple includes of vsmw.h"
-#endif
-#define VSMW_H_INCLUDED
-
-struct vsmw;
-
-void *VSMW_Allocv(struct vsmw *, const char *, size_t, const char *, va_list);
-void *VSMW_Allocf(struct vsmw *, const char *, size_t, const char *, ...);
-void VSMW_Free(struct vsmw *, void **);
-
-struct vsmw *VSMW_New(int, int, const char *);
-void VSMW_Destroy(struct vsmw **);
diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am
index c772bf5..1616f0e 100644
--- a/lib/libvarnish/Makefile.am
+++ b/lib/libvarnish/Makefile.am
@@ -34,7 +34,6 @@ libvarnish_a_SOURCES = \
 	vsa.c \
 	vsb.c \
 	vsha256.c \
-	vsmw.c \
 	vss.c \
 	vsub.c \
 	vtcp.c \
diff --git a/lib/libvarnish/vsmw.c b/lib/libvarnish/vsmw.c
deleted file mode 100644
index c36e2d8..0000000
--- a/lib/libvarnish/vsmw.c
+++ /dev/null
@@ -1,316 +0,0 @@
-/*-
- * Copyright (c) 2010-2011 Varnish Software AS
- * All rights reserved.
- *
- * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * VSM stuff common to manager and child.
- *
- * Please see comments in <vsm_priv.h> for details of protocols and
- * data consistency.
- *
- */
-
-#include "config.h"
-
-#include <errno.h>
-#include <fcntl.h>
-#include <stdarg.h>
-#include <stdio.h>
-#include <stdint.h>
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
-#include <unistd.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-
-#include "vdef.h"
-#include "vas.h"
-#include "vsb.h"
-#include "miniobj.h"
-#include "vsmw.h"
-#include "vqueue.h"
-
-#include "vfil.h"
-#include "vrnd.h"
-
-#ifndef MAP_HASSEMAPHORE
-#  define MAP_HASSEMAPHORE 0 /* XXX Linux */
-#endif
-
-#ifndef MAP_NOSYNC
-#  define MAP_NOSYNC 0 /* XXX Linux */
-#endif
-
-/*--------------------------------------------------------------------*/
-
-struct vsmwseg {
-	unsigned			magic;
-#define VSMWSEG_MAGIC			0x7e4ccaea
-	VTAILQ_ENTRY(vsmwseg)		list;
-	char				*fn;
-
-	char				*class;
-	size_t				len;
-	char				*id;
-	void				*ptr;
-};
-
-struct vsmw {
-	unsigned			magic;
-#define VSMW_MAGIC			0xc2ca2cd9
-	int				vdirfd;
-	int				mode;
-	char				*idx;
-	VTAILQ_HEAD(, vsmwseg)		segs;
-	struct vsb			*vsb;
-	pid_t				pid;
-	time_t				birth;
-};
-
-/*--------------------------------------------------------------------*/
-
-static void
-vsmw_idx_head(const struct vsmw *vsmw, int fd)
-{
-	char buf[64];
-
-	bprintf(buf, "# %jd %jd\n", (intmax_t)vsmw->pid, (intmax_t)vsmw->birth);
-	assert(write(fd, buf, strlen(buf)) == strlen(buf));
-}
-
-static void
-vsmw_write_index(const struct vsmw *vsmw, int fd, const struct vsmwseg *seg)
-{
-	ssize_t s;
-
-	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
-	CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
-	VSB_clear(vsmw->vsb);
-	VSB_printf(vsmw->vsb, "%s 0 %zu %s %s\n",
-	    seg->fn,
-	    seg->len,
-	    seg->class,
-	    seg->id);
-	AZ(VSB_finish(vsmw->vsb));
-	s = write(fd, VSB_data(vsmw->vsb), VSB_len(vsmw->vsb));
-	assert(s == VSB_len(vsmw->vsb));
-}
-
-/*--------------------------------------------------------------------*/
-
-static void
-vsmw_mkent(const struct vsmw *vsmw, const char *pfx)
-{
-	int fd;
-
-	while (1) {
-		VSB_clear(vsmw->vsb);
-		VSB_printf(vsmw->vsb, "_.%s", pfx);
-		VSB_printf(vsmw->vsb, ".%08lx", VRND_RandomTestable());
-		VSB_printf(vsmw->vsb, "%08lx", VRND_RandomTestable());
-		AZ(VSB_finish(vsmw->vsb));
-		fd = openat(vsmw->vdirfd, VSB_data(vsmw->vsb), O_RDONLY);
-		if (fd < 0 && errno == ENOENT)
-			return;
-		if (fd >= 0)
-			AZ(close(fd));
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-void *
-VSMW_Allocv(struct vsmw *vsmw, const char *class, size_t len,
-    const char *fmt, va_list va)
-{
-	struct vsmwseg *seg;
-	int fd;
-	size_t ps;
-
-	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
-
-	ALLOC_OBJ(seg, VSMWSEG_MAGIC);
-	AN(seg);
-	REPLACE(seg->class, class);
-	seg->len = len;
-
-	VSB_clear(vsmw->vsb);
-	VSB_vprintf(vsmw->vsb, fmt, va);
-	AZ(VSB_finish(vsmw->vsb));
-	REPLACE(seg->id, VSB_data(vsmw->vsb));
-
-	vsmw_mkent(vsmw, class);
-	REPLACE(seg->fn, VSB_data(vsmw->vsb));
-
-	ps = getpagesize();
-	len = RUP2(len, ps);
-
-	fd = openat(vsmw->vdirfd, seg->fn,
-	    O_RDWR | O_CREAT | O_EXCL, vsmw->mode);
-	assert(fd >= 0);
-
-	AZ(VFIL_allocate(fd, (off_t)len, 1));
-
-	seg->ptr = (void *)mmap(NULL, len,
-	    PROT_READ|PROT_WRITE,
-	    MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED,
-	    fd, 0);
-
-	AZ(close(fd));
-	assert(seg->ptr != MAP_FAILED);
-	(void)mlock(seg->ptr, len);
-
-	VTAILQ_INSERT_TAIL(&vsmw->segs, seg, list);
-	fd = openat(vsmw->vdirfd, vsmw->idx, O_APPEND | O_WRONLY);
-	assert(fd >= 0);
-	vsmw_write_index(vsmw, fd, seg);
-	AZ(close(fd));
-
-	return (seg->ptr);
-}
-
-void *
-VSMW_Allocf(struct vsmw *vsmw, const char *class, size_t len,
-    const char *fmt, ...)
-{
-	va_list ap;
-	void *p;
-
-	va_start(ap, fmt);
-	p = VSMW_Allocv(vsmw, class, len, fmt, ap);
-	va_end(ap);
-	return (p);
-}
-
-/*--------------------------------------------------------------------*/
-static void
-vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
-{
-	char *t = NULL;
-	int fd;
-	size_t len;
-
-	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
-	CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
-
-	len = getpagesize();
-	len = RUP2(seg->len, len);
-	AZ(munmap(seg->ptr, len));
-
-	VTAILQ_REMOVE(&vsmw->segs, seg, list);
-	if (unlinkat(vsmw->vdirfd, seg->fn, 0))
-		assert (errno == ENOENT);
-	REPLACE(seg->fn, NULL);
-	REPLACE(seg->class, NULL);
-	REPLACE(seg->id, NULL);
-	FREE_OBJ(seg);
-
-	if (fixidx) {
-		vsmw_mkent(vsmw, vsmw->idx);
-		REPLACE(t, VSB_data(vsmw->vsb));
-		AN(t);
-		fd = openat(vsmw->vdirfd, t, O_WRONLY|O_CREAT|O_EXCL, vsmw->mode);
-		assert(fd >= 0);
-		vsmw_idx_head(vsmw, fd);
-		VTAILQ_FOREACH(seg, &vsmw->segs, list)
-			vsmw_write_index(vsmw, fd, seg);
-		AZ(close(fd));
-		AZ(renameat(vsmw->vdirfd, t, vsmw->vdirfd, vsmw->idx));
-		REPLACE(t, NULL);
-	}
-}
-
-/*--------------------------------------------------------------------*/
-
-void
-VSMW_Free(struct vsmw *vsmw, void **pp)
-{
-	struct vsmwseg *seg;
-	void *p;
-
-	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
-	AN(pp);
-	p = *pp;
-	AN(p);
-	*pp = NULL;
-	VTAILQ_FOREACH(seg, &vsmw->segs, list)
-		if (seg->ptr == p)
-			break;
-	AN(seg);
-	vsmw_delseg(vsmw, seg, 1);
-}
-
-/*--------------------------------------------------------------------*/
-
-struct vsmw *
-VSMW_New(int vdirfd, int mode, const char *idxname)
-{
-	struct vsmw *vsmw;
-	int fd;
-
-	assert(vdirfd > 0);
-	assert(mode > 0);
-	AN(idxname);
-
-	ALLOC_OBJ(vsmw, VSMW_MAGIC);
-	AN(vsmw);
-
-	VTAILQ_INIT(&vsmw->segs);
-	vsmw->vsb = VSB_new_auto();
-	AN(vsmw->vsb);
-	REPLACE(vsmw->idx, idxname);
-	vsmw->mode = mode;
-	vsmw->vdirfd = vdirfd;
-	vsmw->pid = getpid();
-	vsmw->birth = time(NULL);
-
-	if (unlinkat(vdirfd, vsmw->idx, 0))
-		assert (errno == ENOENT);
-	fd = openat(vdirfd,
-	    vsmw->idx, O_APPEND | O_WRONLY | O_CREAT, vsmw->mode);
-	assert(fd >= 0);
-	vsmw_idx_head(vsmw, fd);
-	AZ(close(fd));
-
-	return (vsmw);
-}
-
-void
-VSMW_Destroy(struct vsmw **pp)
-{
-	struct vsmw *vsmw;
-	struct vsmwseg *seg, *s2;
-
-	TAKE_OBJ_NOTNULL(vsmw, pp, VSMW_MAGIC);
-	if (unlinkat(vsmw->vdirfd, vsmw->idx, 0))
-		assert (errno == ENOENT);
-	REPLACE(vsmw->idx, NULL);
-	VTAILQ_FOREACH_SAFE(seg, &vsmw->segs, list, s2)
-		vsmw_delseg(vsmw, seg, 0);
-	VSB_destroy(&vsmw->vsb);
-	AZ(close(vsmw->vdirfd));
-	FREE_OBJ(vsmw);
-}

From phk at FreeBSD.org  Wed Jan  3 10:14:05 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed,  3 Jan 2018 10:14:05 +0000 (UTC)
Subject: [master] 630b045 Bring Lck_DestroyClass() back.
Message-ID: <20180103101405.CDC10A9DCB@lists.varnish-cache.org>


commit 630b0455c7dcbfc1581f5f82f9fc1df93c1d4e34
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 3 09:52:17 2018 +0000

    Bring Lck_DestroyClass() back.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index fdf131f..b02fe00 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -757,7 +757,8 @@ int Lck_CondWait(pthread_cond_t *cond, struct lock *lck, double);
 		assert(Lck__Owned(a));	\
 	} while (0)
 
-struct VSC_lck *Lck_CreateClass(struct vsc_seg **, const char *name);
+struct VSC_lck *Lck_CreateClass(struct vsc_seg **, const char *);
+void Lck_DestroyClass(struct vsc_seg **);
 
 #define LOCK(nam) extern struct VSC_lck *lck_##nam;
 #include "tbl/locks.h"
diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c
index 1509723..9f122e9 100644
--- a/bin/varnishd/cache/cache_lck.c
+++ b/bin/varnishd/cache/cache_lck.c
@@ -250,6 +250,12 @@ Lck_CreateClass(struct vsc_seg **sg, const char *name)
 	return(VSC_lck_New(sg, name));
 }
 
+void
+Lck_DestroyClass(struct vsc_seg **sg)
+{
+	VSC_lck_Destroy(sg);
+}
+
 #define LOCK(nam) struct VSC_lck *lck_##nam;
 #include "tbl/locks.h"
 

From phk at FreeBSD.org  Fri Jan  5 10:36:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri,  5 Jan 2018 10:36:09 +0000 (UTC)
Subject: [master] 849fcd3 Add argument to tell which vsm_cluster a VSC goes
 into.
Message-ID: <20180105103609.73E949BBDC@lists.varnish-cache.org>


commit 849fcd3b659ec70359fd1a5c469fc6790bd08280
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 5 09:23:00 2018 +0000

    Add argument to tell which vsm_cluster a VSC goes into.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 3b67257..601beaf 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -342,7 +342,8 @@ vbe_dir_event(const struct director *d, enum vcl_event_e ev)
 	if (ev == VCL_EVENT_WARM) {
 		AZ(bp->vsc);
 		bp->vsc =
-		    VSC_vbe_New(&bp->vsc_seg, bp->director->display_name);
+		    VSC_vbe_New(NULL, &bp->vsc_seg,
+			bp->director->display_name);
 		AN(bp->vsc);
 	}
 
diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c
index 9f122e9..6fa816a 100644
--- a/bin/varnishd/cache/cache_lck.c
+++ b/bin/varnishd/cache/cache_lck.c
@@ -247,7 +247,7 @@ Lck_Delete(struct lock *lck)
 struct VSC_lck *
 Lck_CreateClass(struct vsc_seg **sg, const char *name)
 {
-	return(VSC_lck_New(sg, name));
+	return(VSC_lck_New(NULL, sg, name));
 }
 
 void
diff --git a/bin/varnishd/cache/cache_mempool.c b/bin/varnishd/cache/cache_mempool.c
index 1a9f040..62407f6 100644
--- a/bin/varnishd/cache/cache_mempool.c
+++ b/bin/varnishd/cache/cache_mempool.c
@@ -239,7 +239,7 @@ MPL_New(const char *name,
 	VTAILQ_INIT(&mpl->surplus);
 	Lck_New(&mpl->mtx, lck_mempool);
 	/* XXX: prealloc min_pool */
-	mpl->vsc = VSC_mempool_New(&mpl->vsc_seg, mpl->name + 4);
+	mpl->vsc = VSC_mempool_New(NULL, &mpl->vsc_seg, mpl->name + 4);
 	AN(mpl->vsc);
 	AZ(pthread_create(&mpl->thread, NULL, mpl_guard, mpl));
 	AZ(pthread_detach(mpl->thread));
diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index c01e01e..01c8c36 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -505,7 +505,7 @@ VSM_Init(void)
 	vsc_lock = vsm_vsc_lock;
 	vsc_unlock = vsm_vsc_unlock;
 
-	VSC_C_main = VSC_main_New(NULL, "");
+	VSC_C_main = VSC_main_New(NULL, NULL, "");
 	AN(VSC_C_main);
 
 	AN(heritage.proc_vsmw);
diff --git a/bin/varnishd/common/common_vsc.c b/bin/varnishd/common/common_vsc.c
index 6b8c0c5..fa361b9 100644
--- a/bin/varnishd/common/common_vsc.c
+++ b/bin/varnishd/common/common_vsc.c
@@ -106,13 +106,15 @@ vrt_vsc_mksegf(const char *class, size_t payload, const char *fmt, ...)
 }
 
 void *
-VRT_VSC_Alloc(struct vsc_seg **sg, const char *nm, size_t sd,
+VRT_VSC_Alloc(struct vsm_cluster *vc, struct vsc_seg **sg,
+    const char *nm, size_t sd,
     const unsigned char *jp, size_t sj, const char *fmt, va_list va)
 {
 	struct vsc_seg *vsg, *dvsg;
 	char buf[1024];
 	uintptr_t jjp;
 
+	AZ(vc);
 	if (vsc_lock != NULL)
 		vsc_lock();
 
diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 46595d2..646b19c 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -49,6 +49,7 @@
 #include "common/common_param.h"
 
 struct vsc_seg;
+struct vsm_cluster;
 #include "VSC_mgt.h"
 
 struct cli;
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index da963b5..f23bc75 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -808,7 +808,7 @@ main(int argc, char * const *argv)
 	mgt_SHM_Init();
 
 	mgt_SHM_static_alloc(i_arg, strlen(i_arg) + 1L, "Arg", "-i");
-	VSC_C_mgt = VSC_mgt_New(NULL, "");
+	VSC_C_mgt = VSC_mgt_New(NULL, NULL, "");
 
 	if (M_arg != NULL)
 		mgt_cli_master(M_arg);
diff --git a/bin/varnishd/storage/storage_file.c b/bin/varnishd/storage/storage_file.c
index af2817f..c2cb479 100644
--- a/bin/varnishd/storage/storage_file.c
+++ b/bin/varnishd/storage/storage_file.c
@@ -413,7 +413,7 @@ smf_open(struct stevedore *st)
 	if (lck_smf == NULL)
 		lck_smf = Lck_CreateClass(NULL, "smf");
 	CAST_OBJ_NOTNULL(sc, st->priv, SMF_SC_MAGIC);
-	sc->stats = VSC_smf_New(NULL, st->ident);
+	sc->stats = VSC_smf_New(NULL, NULL, st->ident);
 	Lck_New(&sc->mtx, lck_smf);
 	Lck_Lock(&sc->mtx);
 	smf_open_chunk(sc, sc->filesize, 0, &fail, &sum);
diff --git a/bin/varnishd/storage/storage_malloc.c b/bin/varnishd/storage/storage_malloc.c
index 4ea548b..b8a168a 100644
--- a/bin/varnishd/storage/storage_malloc.c
+++ b/bin/varnishd/storage/storage_malloc.c
@@ -213,7 +213,7 @@ sma_open(struct stevedore *st)
 		lck_sma = Lck_CreateClass(NULL, "sma");
 	CAST_OBJ_NOTNULL(sma_sc, st->priv, SMA_SC_MAGIC);
 	Lck_New(&sma_sc->sma_mtx, lck_sma);
-	sma_sc->stats = VSC_sma_New(NULL, st->ident);
+	sma_sc->stats = VSC_sma_New(NULL, NULL, st->ident);
 	if (sma_sc->sma_max != SIZE_MAX)
 		sma_sc->stats->g_space = sma_sc->sma_max;
 }
diff --git a/include/vrt.h b/include/vrt.h
index ee440aa..9a01feb 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -113,6 +113,7 @@ struct vmod_priv;
 struct vrt_acl;
 struct vsb;
 struct vsc_seg;
+struct vsm_cluster;
 struct vsl_log;
 struct ws;
 
@@ -444,7 +445,7 @@ VCL_STRING VRT_STRANDS_string(VCL_STRANDS);
 VCL_STRING VRT_TIME_string(VRT_CTX, VCL_TIME);
 
 #ifdef va_start	// XXX: hackish
-void *VRT_VSC_Alloc(struct vsc_seg **, const char *, size_t,
-    const unsigned char *, size_t, const char *, va_list);
+void *VRT_VSC_Alloc(struct vsm_cluster *, struct vsc_seg **,
+    const char *, size_t, const unsigned char *, size_t, const char *, va_list);
 void VRT_VSC_Destroy(const char *, struct vsc_seg *);
 #endif
diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py
index 4afeb73..c0c6d80 100644
--- a/lib/libvcc/vsctool.py
+++ b/lib/libvcc/vsctool.py
@@ -144,7 +144,8 @@ class vscset(object):
 		fo.write("\n")
 
 		fo.write(self.struct + " *VSC_" + self.name + "_New")
-		fo.write("(struct vsc_seg **, const char *fmt, ...);\n")
+		fo.write("(struct vsm_cluster *,\n")
+		fo.write("    struct vsc_seg **, const char *fmt, ...);\n")
 
 		fo.write("void VSC_" + self.name + "_Destroy")
 		fo.write("(struct vsc_seg **);\n")
@@ -186,14 +187,15 @@ class vscset(object):
 		fo.write("\n")
 		fo.write(self.struct + "*\n")
 		fo.write("VSC_" + self.name + "_New")
-		fo.write("(struct vsc_seg **sg, const char *fmt, ...)\n")
+		fo.write("(struct vsm_cluster *vc,\n")
+		fo.write("    struct vsc_seg **sg, const char *fmt, ...)\n")
 		fo.write("{\n")
 		fo.write("\tva_list ap;\n")
 		fo.write("\t" + self.struct + " *retval;\n")
 		fo.write("\n")
 		fo.write("\tva_start(ap, fmt);\n")
 		fo.write("\tretval = VRT_VSC_Alloc")
-		fo.write("(sg, vsc_" + self.name + "_name, ")
+		fo.write("(vc, sg, vsc_" + self.name + "_name, ")
 		fo.write("sizeof(" + self.struct + "),\n\t    ")
 		fo.write("vsc_" + self.name + "_json, ")
 		fo.write("sizeof vsc_" + self.name + "_json, fmt, ap);\n")
diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c
index 2298d10..8e730dc 100644
--- a/lib/libvmod_debug/vmod_debug.c
+++ b/lib/libvmod_debug/vmod_debug.c
@@ -585,7 +585,7 @@ xyzzy_vsc_new(VRT_CTX)
 	(void)ctx;
 	AZ(pthread_mutex_lock(&vsc_mtx));
 	if (vsc == NULL)
-		vsc = VSC_debug_New(&vsc_seg, "");
+		vsc = VSC_debug_New(NULL, &vsc_seg, "");
 	AN(vsc);
 	AZ(pthread_mutex_unlock(&vsc_mtx));
 }

From phk at FreeBSD.org  Fri Jan  5 10:36:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri,  5 Jan 2018 10:36:09 +0000 (UTC)
Subject: [master] cfb9223 Teach VCC how to put backend-VSC's into a
 VSM-cluster.
Message-ID: <20180105103609.884249BBDE@lists.varnish-cache.org>


commit cfb922305af6126c3437a3b5ff8d74c983560ae7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 5 10:19:02 2018 +0000

    Teach VCC how to put backend-VSC's into a VSM-cluster.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 601beaf..9da3fca 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -423,8 +423,16 @@ vbe_panic(const struct director *d, struct vsb *vsb)
  * Create a new static or dynamic director::backend instance.
  */
 
-struct director *
-VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt)
+size_t v_matchproto_()
+VRT_backend_vsm_need(VRT_CTX)
+{
+	(void)ctx;
+	return (VSC_vbe_size);
+}
+
+struct director * v_matchproto_()
+VRT_new_backend_clustered(VRT_CTX, struct vsm_cluster *vc,
+    const struct vrt_backend *vrt)
 {
 	struct backend *be;
 	struct director *d;
@@ -433,6 +441,7 @@ VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt)
 	int retval;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	(void)vc;
 	CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC);
 	assert(vrt->ipv4_suckaddr != NULL || vrt->ipv6_suckaddr != NULL);
 
@@ -495,6 +504,12 @@ VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt)
 	return (NULL);
 }
 
+struct director * v_matchproto_()
+VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt)
+{
+	return (VRT_new_backend_clustered(ctx, NULL, vrt));
+}
+
 /*--------------------------------------------------------------------
  * Delete a dynamic director::backend instance.  Undeleted dynamic and
  * static instances are GC'ed when the VCL is discarded (in cache_vcl.c)
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index 9adc590..e8e212c 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -623,6 +623,16 @@ VRT_purge(VRT_CTX, double ttl, double grace, double keep)
 }
 
 /*--------------------------------------------------------------------
+ */
+
+struct vsm_cluster *
+VRT_VSM_Cluster(size_t sz)
+{
+	AN(sz);
+	return ((void*)VRT_VSM_Cluster);
+}
+
+/*--------------------------------------------------------------------
  * Simple stuff
  */
 
diff --git a/include/vrt.h b/include/vrt.h
index 9a01feb..ff70504 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -387,9 +387,14 @@ void VRT_synth_page(VRT_CTX, const char *, ...);
 
 /* Backend related */
 struct director *VRT_new_backend(VRT_CTX, const struct vrt_backend *);
+struct director *VRT_new_backend_clustered(VRT_CTX,
+    struct vsm_cluster *, const struct vrt_backend *);
+size_t VRT_backend_vsm_need(VRT_CTX);
 void VRT_delete_backend(VRT_CTX, struct director **);
 int VRT_backend_healthy(VRT_CTX, struct director *);
 
+/* VSM related */
+struct vsm_cluster *VRT_VSM_Cluster(size_t);
 
 /* cache_director.c */
 int VRT_Healthy(VRT_CTX, VCL_BACKEND);
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index 54458d8..4b81bf2 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -449,7 +449,8 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 
 	ifp = New_IniFin(tl);
 	VSB_printf(ifp->ini,
-	    "\t%s =\n\t    VRT_new_backend(ctx, &vgc_dir_priv_%s);",
+	    "\t%s =\n\t    VRT_new_backend_clustered(ctx, vc,\n"
+	    "\t\t&vgc_dir_priv_%s);",
 	    vgcname, vgcname);
 }
 
@@ -463,6 +464,7 @@ vcc_ParseBackend(struct vcc *tl)
 	struct token *t_first, *t_be;
 	struct symbol *sym;
 
+	tl->ndirector++;
 	t_first = tl->t;
 	vcc_NextToken(tl);		/* ID: backend */
 
@@ -492,3 +494,16 @@ vcc_ParseBackend(struct vcc *tl)
 		tl->t_default_director = t_be;
 	}
 }
+
+void
+vcc_Backend_Init(struct vcc *tl)
+{
+	struct inifin *ifp;
+
+	ifp = New_IniFin(tl);
+	VSB_printf(ifp->ini, "\tstruct vsm_cluster *vc;\n");
+	VSB_printf(ifp->ini, "\n");
+	VSB_printf(ifp->ini,
+	    "\tvc = VRT_VSM_Cluster(ndirector * VRT_backend_vsm_need(ctx));\n");
+	VSB_printf(ifp->ini, "\tif (vc == 0)\n\t\treturn(1);");
+}
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 28975b0..a079aee 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -298,7 +298,9 @@ EmitInitFini(const struct vcc *tl)
 	 * LOAD
 	 */
 	Fc(tl, 0, "\nstatic int\nVGC_Load(VRT_CTX)\n{\n\n");
-	Fc(tl, 0, "\tvgc_inistep = 0;\n\n");
+	Fc(tl, 0, "\tvgc_inistep = 0;\n");
+	Fc(tl, 0, "\tsize_t ndirector = %dUL;\n", tl->ndirector);
+	Fc(tl, 0, "\n");
 	VTAILQ_FOREACH(p, &tl->inifin, list) {
 		AZ(VSB_finish(p->ini));
 		assert(p->n > 0);
@@ -560,6 +562,8 @@ vcc_CompileSource(struct vcc *tl, struct source *sp)
 
 	vcc_Expr_Init(tl);
 
+	vcc_Backend_Init(tl);
+
 	for (v = vcc_vars; v->name != NULL; v++) {
 		if (v->fmt == HEADER) {
 			sym = VCC_Symbol(tl, NULL, v->name, NULL,
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 47d8525..d0c040c 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -217,6 +217,7 @@ struct vcc {
 
 	int			nprobe;
 
+	int			ndirector;
 	const char		*default_director;
 	struct token		*t_default_director;
 	const char		*default_probe;
@@ -252,6 +253,7 @@ int vcc_ParseAction(struct vcc *tl);
 /* vcc_backend.c */
 struct fld_spec;
 
+void vcc_Backend_Init(struct vcc *tl);
 void vcc_ParseProbe(struct vcc *tl);
 void vcc_ParseBackend(struct vcc *tl);
 struct fld_spec * vcc_FldSpec(struct vcc *tl, const char *first, ...);
diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py
index c0c6d80..e368cf9 100644
--- a/lib/libvcc/vsctool.py
+++ b/lib/libvcc/vsctool.py
@@ -143,6 +143,9 @@ class vscset(object):
 		fo.write("};\n")
 		fo.write("\n")
 
+		fo.write("#define VSC_" + self.name + 
+		    "_size PRNDUP(sizeof(" + self.struct + "))\n\n")
+
 		fo.write(self.struct + " *VSC_" + self.name + "_New")
 		fo.write("(struct vsm_cluster *,\n")
 		fo.write("    struct vsc_seg **, const char *fmt, ...);\n")

From phk at FreeBSD.org  Fri Jan  5 10:55:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri,  5 Jan 2018 10:55:07 +0000 (UTC)
Subject: [master] ab412f2 Forgot (as usual) umem
Message-ID: <20180105105507.B4966A03DC@lists.varnish-cache.org>


commit ab412f2735a4654840d33c981b28b87f0ab0708d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 5 10:53:53 2018 +0000

    Forgot (as usual) umem

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 9da3fca..0ef5ffd 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -441,7 +441,7 @@ VRT_new_backend_clustered(VRT_CTX, struct vsm_cluster *vc,
 	int retval;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	(void)vc;
+	(void)vc
 	CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC);
 	assert(vrt->ipv4_suckaddr != NULL || vrt->ipv6_suckaddr != NULL);
 
diff --git a/bin/varnishd/storage/storage_umem.c b/bin/varnishd/storage/storage_umem.c
index f951f09..afc2771 100644
--- a/bin/varnishd/storage/storage_umem.c
+++ b/bin/varnishd/storage/storage_umem.c
@@ -258,7 +258,7 @@ smu_open(struct stevedore *st)
 		lck_smu = Lck_CreateClass(NULL, "smu");
 	CAST_OBJ_NOTNULL(smu_sc, st->priv, SMU_SC_MAGIC);
 	Lck_New(&smu_sc->smu_mtx, lck_smu);
-	smu_sc->stats = VSC_smu_New(NULL, st->ident);
+	smu_sc->stats = VSC_smu_New(NULL, NULL, st->ident);
 	if (smu_sc->smu_max != SIZE_MAX)
 		smu_sc->stats->g_space = smu_sc->smu_max;
 

From phk at FreeBSD.org  Fri Jan  5 11:19:04 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri,  5 Jan 2018 11:19:04 +0000 (UTC)
Subject: [master] 9616126 I usually know how to quit vi(1)...
Message-ID: <20180105111904.EA9E9A0DEE@lists.varnish-cache.org>


commit 961612687370ea7007d0ac2cbf9f90c26426182f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 5 11:17:34 2018 +0000

    I usually know how to quit vi(1)...

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 0ef5ffd..9da3fca 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -441,7 +441,7 @@ VRT_new_backend_clustered(VRT_CTX, struct vsm_cluster *vc,
 	int retval;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	(void)vc
+	(void)vc;
 	CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC);
 	assert(vrt->ipv4_suckaddr != NULL || vrt->ipv6_suckaddr != NULL);
 

From phk at FreeBSD.org  Sun Jan  7 20:15:12 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sun,  7 Jan 2018 20:15:12 +0000 (UTC)
Subject: [master] 355bf7e Refine the VRT_VSM_Cluster prototypes.
Message-ID: <20180107201512.A0C3A65EBB@lists.varnish-cache.org>


commit 355bf7e7a34fc36b23176d607e68615530800279
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 6 10:12:28 2018 +0000

    Refine the VRT_VSM_Cluster prototypes.

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index e8e212c..f8192c1 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -625,11 +625,23 @@ VRT_purge(VRT_CTX, double ttl, double grace, double keep)
 /*--------------------------------------------------------------------
  */
 
-struct vsm_cluster *
-VRT_VSM_Cluster(size_t sz)
+struct vsm_cluster * v_matchproto_()
+VRT_VSM_Cluster_New(VRT_CTX, size_t sz)
 {
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	AN(sz);
-	return ((void*)VRT_VSM_Cluster);
+	return ((void*)VRT_VSM_Cluster_New);
+}
+
+void v_matchproto_()
+VRT_VSM_Cluster_Destroy(VRT_CTX, struct vsm_cluster **vcp)
+{
+
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	AN(vcp);
+	AN(*vcp);
+	*vcp = NULL;
 }
 
 /*--------------------------------------------------------------------
diff --git a/include/vrt.h b/include/vrt.h
index ff70504..0a66893 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -394,7 +394,8 @@ void VRT_delete_backend(VRT_CTX, struct director **);
 int VRT_backend_healthy(VRT_CTX, struct director *);
 
 /* VSM related */
-struct vsm_cluster *VRT_VSM_Cluster(size_t);
+struct vsm_cluster *VRT_VSM_Cluster_New(VRT_CTX, size_t);
+void VRT_VSM_Cluster_Destroy(VRT_CTX, struct vsm_cluster **);
 
 /* cache_director.c */
 int VRT_Healthy(VRT_CTX, VCL_BACKEND);
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index 4b81bf2..52f72ec 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -449,7 +449,7 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 
 	ifp = New_IniFin(tl);
 	VSB_printf(ifp->ini,
-	    "\t%s =\n\t    VRT_new_backend_clustered(ctx, vc,\n"
+	    "\t%s =\n\t    VRT_new_backend_clustered(ctx, vsc_cluster,\n"
 	    "\t\t&vgc_dir_priv_%s);",
 	    vgcname, vgcname);
 }
@@ -500,10 +500,10 @@ vcc_Backend_Init(struct vcc *tl)
 {
 	struct inifin *ifp;
 
+	Fh(tl, 0, "\nstatic struct vsm_cluster *vsc_cluster;\n");
 	ifp = New_IniFin(tl);
-	VSB_printf(ifp->ini, "\tstruct vsm_cluster *vc;\n");
-	VSB_printf(ifp->ini, "\n");
-	VSB_printf(ifp->ini,
-	    "\tvc = VRT_VSM_Cluster(ndirector * VRT_backend_vsm_need(ctx));\n");
-	VSB_printf(ifp->ini, "\tif (vc == 0)\n\t\treturn(1);");
+	VSB_printf(ifp->ini, "\tvsc_cluster = VRT_VSM_Cluster_New(ctx,\n"
+	    "\t    ndirector * VRT_backend_vsm_need(ctx));\n");
+	VSB_printf(ifp->ini, "\tif (vsc_cluster == 0)\n\t\treturn(1);");
+	VSB_printf(ifp->fin, "\tVRT_VSM_Cluster_Destroy(ctx, &vsc_cluster);");
 }
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index a079aee..0e84062 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -560,6 +560,9 @@ vcc_CompileSource(struct vcc *tl, struct source *sp)
 	struct vsb *vsb;
 	struct inifin *ifp;
 
+	Fh(tl, 0, "/* ---===### VCC generated .h code ###===---*/\n");
+	Fc(tl, 0, "\n/* ---===### VCC generated .c code ###===---*/\n");
+
 	vcc_Expr_Init(tl);
 
 	vcc_Backend_Init(tl);
@@ -581,8 +584,6 @@ vcc_CompileSource(struct vcc *tl, struct source *sp)
 		REPLACE(sym->rname, v->rname);
 	}
 
-	Fh(tl, 0, "/* ---===### VCC generated .h code ###===---*/\n");
-	Fc(tl, 0, "\n/* ---===### VCC generated .c code ###===---*/\n");
 	Fh(tl, 0, "\nextern const struct VCL_conf VCL_conf;\n");
 
 	/* Register and lex the main source */
diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py
index e368cf9..1227b7c 100644
--- a/lib/libvcc/vsctool.py
+++ b/lib/libvcc/vsctool.py
@@ -199,8 +199,8 @@ class vscset(object):
 		fo.write("\tva_start(ap, fmt);\n")
 		fo.write("\tretval = VRT_VSC_Alloc")
 		fo.write("(vc, sg, vsc_" + self.name + "_name, ")
-		fo.write("sizeof(" + self.struct + "),\n\t    ")
-		fo.write("vsc_" + self.name + "_json, ")
+		fo.write("VSC_" + self.name + "_size,\n")
+		fo.write("\t    vsc_" + self.name + "_json, ")
 		fo.write("sizeof vsc_" + self.name + "_json, fmt, ap);\n")
 		fo.write("\tva_end(ap);\n")
 		fo.write("\treturn(retval);\n")

From phk at FreeBSD.org  Sun Jan  7 20:15:12 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sun,  7 Jan 2018 20:15:12 +0000 (UTC)
Subject: [master] ecb58d7 (Almost) separate VSMW_cluster from VSMW_segment.
Message-ID: <20180107201512.B5BAE65EC0@lists.varnish-cache.org>


commit ecb58d71c18071ab585cbc3b3c9e9ac044c861ea
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Jan 7 20:13:47 2018 +0000

    (Almost) separate VSMW_cluster from VSMW_segment.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 9da3fca..826b13d 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -431,7 +431,7 @@ VRT_backend_vsm_need(VRT_CTX)
 }
 
 struct director * v_matchproto_()
-VRT_new_backend_clustered(VRT_CTX, struct vsm_cluster *vc,
+VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc,
     const struct vrt_backend *vrt)
 {
 	struct backend *be;
diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index f8192c1..d8914cb 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -625,7 +625,7 @@ VRT_purge(VRT_CTX, double ttl, double grace, double keep)
 /*--------------------------------------------------------------------
  */
 
-struct vsm_cluster * v_matchproto_()
+struct vsmw_cluster * v_matchproto_()
 VRT_VSM_Cluster_New(VRT_CTX, size_t sz)
 {
 
@@ -635,7 +635,7 @@ VRT_VSM_Cluster_New(VRT_CTX, size_t sz)
 }
 
 void v_matchproto_()
-VRT_VSM_Cluster_Destroy(VRT_CTX, struct vsm_cluster **vcp)
+VRT_VSM_Cluster_Destroy(VRT_CTX, struct vsmw_cluster **vcp)
 {
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
diff --git a/bin/varnishd/common/common_vsc.c b/bin/varnishd/common/common_vsc.c
index fa361b9..388418f 100644
--- a/bin/varnishd/common/common_vsc.c
+++ b/bin/varnishd/common/common_vsc.c
@@ -106,7 +106,7 @@ vrt_vsc_mksegf(const char *class, size_t payload, const char *fmt, ...)
 }
 
 void *
-VRT_VSC_Alloc(struct vsm_cluster *vc, struct vsc_seg **sg,
+VRT_VSC_Alloc(struct vsmw_cluster *vc, struct vsc_seg **sg,
     const char *nm, size_t sd,
     const unsigned char *jp, size_t sj, const char *fmt, va_list va)
 {
diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c
index a5cee24..6611487 100644
--- a/bin/varnishd/common/common_vsmw.c
+++ b/bin/varnishd/common/common_vsmw.c
@@ -67,13 +67,25 @@
 
 /*--------------------------------------------------------------------*/
 
+struct vsmw_cluster {
+	unsigned			magic;
+#define VSMW_CLUSTER_MAGIC		0x28b74c00
+
+	VTAILQ_ENTRY(vsmw_cluster)	list;
+	char				*fn;
+	size_t				len;
+	void				*ptr;
+	int				refs;
+};
+
 struct vsmwseg {
 	unsigned			magic;
 #define VSMWSEG_MAGIC			0x7e4ccaea
 	VTAILQ_ENTRY(vsmwseg)		list;
-	char				*fn;
+	struct vsmw_cluster		*cluster;
 
 	char				*class;
+	size_t				off;
 	size_t				len;
 	char				*id;
 	void				*ptr;
@@ -85,6 +97,7 @@ struct vsmw {
 	int				vdirfd;
 	int				mode;
 	char				*idx;
+	VTAILQ_HEAD(, vsmw_cluster)	clusters;
 	VTAILQ_HEAD(, vsmwseg)		segs;
 	struct vsb			*vsb;
 	pid_t				pid;
@@ -110,8 +123,9 @@ vsmw_write_index(const struct vsmw *vsmw, int fd, const struct vsmwseg *seg)
 	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
 	CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
 	VSB_clear(vsmw->vsb);
-	VSB_printf(vsmw->vsb, "%s 0 %zu %s %s\n",
-	    seg->fn,
+	VSB_printf(vsmw->vsb, "%s %zu %zu %s %s\n",
+	    seg->cluster->fn,
+	    seg->off,
 	    seg->len,
 	    seg->class,
 	    seg->id);
@@ -127,6 +141,7 @@ vsmw_mkent(const struct vsmw *vsmw, const char *pfx)
 {
 	int fd;
 
+	AN(pfx);
 	while (1) {
 		VSB_clear(vsmw->vsb);
 		VSB_printf(vsmw->vsb, "_.%s", pfx);
@@ -143,46 +158,92 @@ vsmw_mkent(const struct vsmw *vsmw, const char *pfx)
 
 /*--------------------------------------------------------------------*/
 
-void *
-VSMW_Allocv(struct vsmw *vsmw, const char *class, size_t len,
-    const char *fmt, va_list va)
+static struct vsmw_cluster *
+vsmw_NewCluster(struct vsmw *vsmw, size_t len, const char *pfx)
 {
-	struct vsmwseg *seg;
+	struct vsmw_cluster *vc;
 	int fd;
 	size_t ps;
 
-	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
+	ALLOC_OBJ(vc, VSMW_CLUSTER_MAGIC);
+	AN(vc);
 
-	ALLOC_OBJ(seg, VSMWSEG_MAGIC);
-	AN(seg);
-	REPLACE(seg->class, class);
-	seg->len = len;
-
-	VSB_clear(vsmw->vsb);
-	VSB_vprintf(vsmw->vsb, fmt, va);
-	AZ(VSB_finish(vsmw->vsb));
-	REPLACE(seg->id, VSB_data(vsmw->vsb));
+	vsmw_mkent(vsmw, pfx);
+	REPLACE(vc->fn, VSB_data(vsmw->vsb));
 
-	vsmw_mkent(vsmw, class);
-	REPLACE(seg->fn, VSB_data(vsmw->vsb));
+	VTAILQ_INSERT_TAIL(&vsmw->clusters, vc, list);
 
 	ps = getpagesize();
 	len = RUP2(len, ps);
+	vc->len = len;
 
-	fd = openat(vsmw->vdirfd, seg->fn,
+	fd = openat(vsmw->vdirfd, vc->fn,
 	    O_RDWR | O_CREAT | O_EXCL, vsmw->mode);
 	assert(fd >= 0);
 
 	AZ(VFIL_allocate(fd, (off_t)len, 1));
 
-	seg->ptr = (void *)mmap(NULL, len,
+	vc->ptr = (void *)mmap(NULL, len,
 	    PROT_READ|PROT_WRITE,
 	    MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED,
 	    fd, 0);
 
 	AZ(close(fd));
-	assert(seg->ptr != MAP_FAILED);
-	(void)mlock(seg->ptr, len);
+	assert(vc->ptr != MAP_FAILED);
+	(void)mlock(vc->ptr, len);
+
+	return (vc);
+}
+
+static void
+vsmw_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vcp)
+{
+	struct vsmw_cluster *vc;
+
+	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
+	AN(vcp);
+	vc = *vcp;
+	*vcp = NULL;
+	CHECK_OBJ_NOTNULL(vc, VSMW_CLUSTER_MAGIC);
+	AZ(vc->refs);
+
+	AZ(munmap(vc->ptr, vc->len));
+
+	VTAILQ_REMOVE(&vsmw->clusters, vc, list);
+	if (unlinkat(vsmw->vdirfd, vc->fn, 0))
+		assert (errno == ENOENT);
+	REPLACE(vc->fn, NULL);
+	FREE_OBJ(vc);
+}
+
+/*--------------------------------------------------------------------*/
+
+void *
+VSMW_Allocv(struct vsmw *vsmw, const char *class, size_t len,
+    const char *fmt, va_list va)
+{
+	struct vsmwseg *seg;
+	struct vsmw_cluster *vc;
+	int fd;
+
+	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
+
+	ALLOC_OBJ(seg, VSMWSEG_MAGIC);
+	AN(seg);
+	REPLACE(seg->class, class);
+	seg->len = len;
+
+	VSB_clear(vsmw->vsb);
+	VSB_vprintf(vsmw->vsb, fmt, va);
+	AZ(VSB_finish(vsmw->vsb));
+	REPLACE(seg->id, VSB_data(vsmw->vsb));
+
+	vc = vsmw_NewCluster(vsmw, len, class);
+	AN(vc);
+	vc->refs++;
+
+	seg->cluster = vc;
+	seg->ptr = vc->ptr;
 
 	VTAILQ_INSERT_TAIL(&vsmw->segs, seg, list);
 	fd = openat(vsmw->vdirfd, vsmw->idx, O_APPEND | O_WRONLY);
@@ -212,19 +273,14 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
 {
 	char *t = NULL;
 	int fd;
-	size_t len;
 
 	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
 	CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
 
-	len = getpagesize();
-	len = RUP2(seg->len, len);
-	AZ(munmap(seg->ptr, len));
+	if (!--seg->cluster->refs)
+		vsmw_DestroyCluster(vsmw, &seg->cluster);
 
 	VTAILQ_REMOVE(&vsmw->segs, seg, list);
-	if (unlinkat(vsmw->vdirfd, seg->fn, 0))
-		assert (errno == ENOENT);
-	REPLACE(seg->fn, NULL);
 	REPLACE(seg->class, NULL);
 	REPLACE(seg->id, NULL);
 	FREE_OBJ(seg);
@@ -233,7 +289,8 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
 		vsmw_mkent(vsmw, vsmw->idx);
 		REPLACE(t, VSB_data(vsmw->vsb));
 		AN(t);
-		fd = openat(vsmw->vdirfd, t, O_WRONLY|O_CREAT|O_EXCL, vsmw->mode);
+		fd = openat(vsmw->vdirfd,
+		    t, O_WRONLY|O_CREAT|O_EXCL, vsmw->mode);
 		assert(fd >= 0);
 		vsmw_idx_head(vsmw, fd);
 		VTAILQ_FOREACH(seg, &vsmw->segs, list)
@@ -280,6 +337,7 @@ VSMW_New(int vdirfd, int mode, const char *idxname)
 	AN(vsmw);
 
 	VTAILQ_INIT(&vsmw->segs);
+	VTAILQ_INIT(&vsmw->clusters);
 	vsmw->vsb = VSB_new_auto();
 	AN(vsmw->vsb);
 	REPLACE(vsmw->idx, idxname);
@@ -306,11 +364,11 @@ VSMW_Destroy(struct vsmw **pp)
 	struct vsmwseg *seg, *s2;
 
 	TAKE_OBJ_NOTNULL(vsmw, pp, VSMW_MAGIC);
+	VTAILQ_FOREACH_SAFE(seg, &vsmw->segs, list, s2)
+		vsmw_delseg(vsmw, seg, 0);
 	if (unlinkat(vsmw->vdirfd, vsmw->idx, 0))
 		assert (errno == ENOENT);
 	REPLACE(vsmw->idx, NULL);
-	VTAILQ_FOREACH_SAFE(seg, &vsmw->segs, list, s2)
-		vsmw_delseg(vsmw, seg, 0);
 	VSB_destroy(&vsmw->vsb);
 	AZ(close(vsmw->vdirfd));
 	FREE_OBJ(vsmw);
diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 646b19c..19e4060 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -49,7 +49,7 @@
 #include "common/common_param.h"
 
 struct vsc_seg;
-struct vsm_cluster;
+struct vsmw_cluster;
 #include "VSC_mgt.h"
 
 struct cli;
diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 062800e..63fc3c3 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -347,6 +347,8 @@ mgt_launch_child(struct cli *cli)
 		child_main(mgt_param.sigsegv_handler,
 		    mgt_param.wthread_stacksize);
 
+		// VSMW_Destroy(&heritage.proc_vsmw);
+
 		exit(0);
 	}
 	assert(pid > 1);
diff --git a/include/vrt.h b/include/vrt.h
index 0a66893..05dce64 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -113,7 +113,7 @@ struct vmod_priv;
 struct vrt_acl;
 struct vsb;
 struct vsc_seg;
-struct vsm_cluster;
+struct vsmw_cluster;
 struct vsl_log;
 struct ws;
 
@@ -388,14 +388,14 @@ void VRT_synth_page(VRT_CTX, const char *, ...);
 /* Backend related */
 struct director *VRT_new_backend(VRT_CTX, const struct vrt_backend *);
 struct director *VRT_new_backend_clustered(VRT_CTX,
-    struct vsm_cluster *, const struct vrt_backend *);
+    struct vsmw_cluster *, const struct vrt_backend *);
 size_t VRT_backend_vsm_need(VRT_CTX);
 void VRT_delete_backend(VRT_CTX, struct director **);
 int VRT_backend_healthy(VRT_CTX, struct director *);
 
 /* VSM related */
-struct vsm_cluster *VRT_VSM_Cluster_New(VRT_CTX, size_t);
-void VRT_VSM_Cluster_Destroy(VRT_CTX, struct vsm_cluster **);
+struct vsmw_cluster *VRT_VSM_Cluster_New(VRT_CTX, size_t);
+void VRT_VSM_Cluster_Destroy(VRT_CTX, struct vsmw_cluster **);
 
 /* cache_director.c */
 int VRT_Healthy(VRT_CTX, VCL_BACKEND);
@@ -451,7 +451,7 @@ VCL_STRING VRT_STRANDS_string(VCL_STRANDS);
 VCL_STRING VRT_TIME_string(VRT_CTX, VCL_TIME);
 
 #ifdef va_start	// XXX: hackish
-void *VRT_VSC_Alloc(struct vsm_cluster *, struct vsc_seg **,
+void *VRT_VSC_Alloc(struct vsmw_cluster *, struct vsc_seg **,
     const char *, size_t, const unsigned char *, size_t, const char *, va_list);
 void VRT_VSC_Destroy(const char *, struct vsc_seg *);
 #endif
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index 52f72ec..b8a76ec 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -500,7 +500,7 @@ vcc_Backend_Init(struct vcc *tl)
 {
 	struct inifin *ifp;
 
-	Fh(tl, 0, "\nstatic struct vsm_cluster *vsc_cluster;\n");
+	Fh(tl, 0, "\nstatic struct vsmw_cluster *vsc_cluster;\n");
 	ifp = New_IniFin(tl);
 	VSB_printf(ifp->ini, "\tvsc_cluster = VRT_VSM_Cluster_New(ctx,\n"
 	    "\t    ndirector * VRT_backend_vsm_need(ctx));\n");

From phk at FreeBSD.org  Sun Jan  7 21:06:05 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sun,  7 Jan 2018 21:06:05 +0000 (UTC)
Subject: [master] ba14fe3 Sigh, there are no "safe" last minute renames.
Message-ID: <20180107210605.1B086946B9@lists.varnish-cache.org>


commit ba14fe3ca6b48d970a61edf07cc97e5d0e3c53fd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Jan 7 21:05:12 2018 +0000

    Sigh, there are no "safe" last minute renames.

diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py
index 1227b7c..d39f6f7 100644
--- a/lib/libvcc/vsctool.py
+++ b/lib/libvcc/vsctool.py
@@ -147,7 +147,7 @@ class vscset(object):
 		    "_size PRNDUP(sizeof(" + self.struct + "))\n\n")
 
 		fo.write(self.struct + " *VSC_" + self.name + "_New")
-		fo.write("(struct vsm_cluster *,\n")
+		fo.write("(struct vsmw_cluster *,\n")
 		fo.write("    struct vsc_seg **, const char *fmt, ...);\n")
 
 		fo.write("void VSC_" + self.name + "_Destroy")
@@ -190,7 +190,7 @@ class vscset(object):
 		fo.write("\n")
 		fo.write(self.struct + "*\n")
 		fo.write("VSC_" + self.name + "_New")
-		fo.write("(struct vsm_cluster *vc,\n")
+		fo.write("(struct vsmw_cluster *vc,\n")
 		fo.write("    struct vsc_seg **sg, const char *fmt, ...)\n")
 		fo.write("{\n")
 		fo.write("\tva_list ap;\n")

From martin at varnish-software.com  Mon Jan  8 15:10:08 2018
From: martin at varnish-software.com (Martin Blix Grydeland)
Date: Mon,  8 Jan 2018 15:10:08 +0000 (UTC)
Subject: [master] 2261dcf Change type of oc->last_lru from float back to double
Message-ID: <20180108151008.A0E82B22C3@lists.varnish-cache.org>


commit 2261dcfd545e6c9a5d60000cce06e9930557d73e
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Mon Jan 8 16:05:05 2018 +0100

    Change type of oc->last_lru from float back to double
    
    Since the last_lru tracks epoch time, it needs the double precision
    floating point type to accurately track the time.
    
    Fixes: #2527

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index b02fe00..ed5602f 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -376,7 +376,7 @@ struct objcore {
 	uint16_t		oa_present;
 
 	unsigned		timer_idx;	// XXX 4Gobj limit
-	float			last_lru;
+	double			last_lru;
 	VTAILQ_ENTRY(objcore)	hsh_list;
 	VTAILQ_ENTRY(objcore)	lru_list;
 	VTAILQ_ENTRY(objcore)	ban_list;
diff --git a/bin/varnishtest/tests/r02527.vtc b/bin/varnishtest/tests/r02527.vtc
new file mode 100644
index 0000000..ae8ed03
--- /dev/null
+++ b/bin/varnishtest/tests/r02527.vtc
@@ -0,0 +1,30 @@
+varnishtest "Test the n_lru_moved counter"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -arg "-p lru_interval=1" -vcl+backend {
+	sub vcl_backend_response {
+		set beresp.do_stream = false;
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+} -run
+
+varnish v1 -expect MAIN.n_lru_moved == 0
+
+delay 1
+
+client c1 {
+	txreq
+	rxresp
+	txreq
+	rxresp
+} -run
+
+varnish v1 -expect MAIN.n_lru_moved == 1

From nils.goroll at uplex.de  Mon Jan  8 15:30:10 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon,  8 Jan 2018 15:30:10 +0000 (UTC)
Subject: [master] bef8f49 comment/reality sync
Message-ID: <20180108153010.0FD6BB2A01@lists.varnish-cache.org>


commit bef8f4907414737c550b9bcc7928b485411c2cf0
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Jan 8 16:29:01 2018 +0100

    comment/reality sync

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index e08f6fb..76eb85f 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -160,7 +160,7 @@ vbf_beresp2obj(struct busyobj *bo)
 }
 
 /*--------------------------------------------------------------------
- * Copy req->bereq and release req if not pass fetch
+ * Copy req->bereq and release req if no body
  */
 
 static enum fetch_step

From nils.goroll at uplex.de  Mon Jan  8 17:21:07 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon,  8 Jan 2018 17:21:07 +0000 (UTC)
Subject: [master] 1a851e0 gc statement with no effect
Message-ID: <20180108172107.49A61B4F77@lists.varnish-cache.org>


commit 1a851e07b7b9e58bcd3941a5f088ec33dc4f844b
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Sat Dec 30 14:11:18 2017 +0100

    gc statement with no effect

diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c
index a0fa1dd..9451952 100644
--- a/bin/varnishd/http2/cache_http2_session.c
+++ b/bin/varnishd/http2/cache_http2_session.c
@@ -226,7 +226,6 @@ h2_ou_session(struct worker *wrk, struct h2_sess *h2,
 
 	/* Start req thread */
 	r2 = h2_new_req(wrk, h2, 1, req);
-	req->req_step = R_STP_RECV;
 	req->transport = &H2_transport;
 	req->req_step = R_STP_TRANSPORT;
 	req->task.func = h2_do_req;

From nils.goroll at uplex.de  Mon Jan  8 17:21:07 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon,  8 Jan 2018 17:21:07 +0000 (UTC)
Subject: [master] 26e8d49 Do not step back (struct sess).t_idle
Message-ID: <20180108172107.67C7EB4F7A@lists.varnish-cache.org>


commit 26e8d49fcfdf678a42276d753a75fbd411d7ee02
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Sat Dec 30 13:48:16 2017 +0100

    Do not step back (struct sess).t_idle
    
    t_prev from the request we are cleaning up is not the most recent
    relevant time stamp, in particular for H2

diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c
index 25c4ce1..8fa77b9 100644
--- a/bin/varnishd/cache/cache_req.c
+++ b/bin/varnishd/cache/cache_req.c
@@ -211,7 +211,7 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req)
 	}
 	req->req_bodybytes = 0;
 
-	if (!isnan(req->t_prev) && req->t_prev > 0.)
+	if (!isnan(req->t_prev) && req->t_prev > 0. && req->t_prev > sp->t_idle)
 		sp->t_idle = req->t_prev;
 	else
 		sp->t_idle = W_TIM_real(wrk);

From phk at FreeBSD.org  Mon Jan  8 23:19:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon,  8 Jan 2018 23:19:08 +0000 (UTC)
Subject: [master] fdc46c5 Explain why we don't clean up VSMW
Message-ID: <20180108231908.27FEABB58E@lists.varnish-cache.org>


commit fdc46c544a6e63c0fc5823e57428850a4b56dec0
Author: Charlie Root <root at critter.freebsd.dk>
Date:   Mon Jan 8 08:20:58 2018 +0000

    Explain why we don't clean up VSMW

diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c
index 63fc3c3..8b70f1a 100644
--- a/bin/varnishd/mgt/mgt_child.c
+++ b/bin/varnishd/mgt/mgt_child.c
@@ -347,6 +347,12 @@ mgt_launch_child(struct cli *cli)
 		child_main(mgt_param.sigsegv_handler,
 		    mgt_param.wthread_stacksize);
 
+		/*
+		 * It would be natural to clean VSMW up here, but it is apt
+		 * to fail in some scenarios because of the fall-back
+		 * "rm -rf" in mgt_SHM_ChildDestroy() which is there to
+		 * catch the cases were we don't get here.
+		 */
 		// VSMW_Destroy(&heritage.proc_vsmw);
 
 		exit(0);

From phk at FreeBSD.org  Mon Jan  8 23:19:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon,  8 Jan 2018 23:19:08 +0000 (UTC)
Subject: [master] 0b6966d Teach VSMAPI about non-zero offsets
Message-ID: <20180108231908.428CFBB591@lists.varnish-cache.org>


commit 0b6966d3b5b939e5b2069874e2c5403a2b83b9b6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 8 19:48:54 2018 +0000

    Teach VSMAPI about non-zero offsets

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 4169a23..9842874 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -82,9 +82,10 @@ struct vsm_seg {
 	struct vsm_set		*set;
 	char			**av;
 	int			refs;
+	void			*s;
+	size_t			sz;
 	void			*b;
 	void			*e;
-	size_t			sz;
 	uintptr_t		serial;
 };
 
@@ -150,18 +151,14 @@ vsm_diag(struct vsm *vd, const char *fmt, ...)
 static void
 vsm_unmapseg(struct vsm_seg *vg)
 {
-	size_t sz, ps, len;
 
 	CHECK_OBJ_NOTNULL(vg, VSM_SEG_MAGIC);
 
 	AN(vg->b);
 	AN(vg->e);
-	sz = strtoul(vg->av[3], NULL, 10);
-	assert(sz > 0);
-	ps = getpagesize();
-	len = RUP2(sz, ps);
-	AZ(munmap(vg->b, len));
-	vg->b = vg->e = NULL;
+	AZ(munmap(vg->s, vg->sz));
+	vg->s = vg->b = vg->e = NULL;
+	vg->sz = 0;
 }
 
 /*--------------------------------------------------------------------*/
@@ -475,7 +472,6 @@ vsm_refresh_set2(struct vsm *vd, struct vsm_set *vs, struct vsb *vsb)
 			VAV_Free(av);
 			break;
 		}
-		xxxassert(!strcmp(av[2], "0"));
 
 		if (vg == NULL) {
 			ALLOC_OBJ(vg2, VSM_SEG_MAGIC);
@@ -692,7 +688,7 @@ int
 VSM_Map(struct vsm *vd, struct vsm_fantom *vf)
 {
 	struct vsm_seg *vg;
-	size_t sz, ps, len;
+	size_t of, off, sz, ps, len;
 	struct vsb *vsb;
 	int fd;
 
@@ -716,10 +712,14 @@ VSM_Map(struct vsm *vd, struct vsm_fantom *vf)
 		return (0);
 	}
 
+	ps = getpagesize();
+	of = strtoul(vg->av[2], NULL, 10);
+	assert(of >= 0);
+	off = RDN2(of, ps);
+
 	sz = strtoul(vg->av[3], NULL, 10);
 	assert(sz > 0);
-	ps = getpagesize();
-	len = RUP2(sz, ps);
+	len = RUP2(of + sz, ps);
 
 	vsb = VSB_new_auto();
 	AN(vsb);
@@ -732,17 +732,18 @@ VSM_Map(struct vsm *vd, struct vsm_fantom *vf)
 		return (vsm_diag(vd, "Could not open segment"));
 	}
 
-	vg->b = (void*)mmap(NULL, len,
+	vg->s = (void*)mmap(NULL, len,
 	    PROT_READ,
 	    MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED,
-	    fd, 0);
+	    fd, off);
 
 	VSB_destroy(&vsb);
 
-
 	closefd(&fd);
-	if (vg->b == MAP_FAILED)
+	if (vg->s == MAP_FAILED)
 		return (vsm_diag(vd, "Could not mmap segment"));
+
+	vg->b = (char*)(vg->s) + of;
 	vg->e = (char *)vg->b + sz;
 	vg->sz = len;
 

From phk at FreeBSD.org  Mon Jan  8 23:19:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon,  8 Jan 2018 23:19:08 +0000 (UTC)
Subject: [master] 89e295c Export the VSMW cluster New/Destroy functions
Message-ID: <20180108231908.5D13CBB595@lists.varnish-cache.org>


commit 89e295c8eace218f2ffff24be0a11e827acfdb7c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 8 20:00:33 2018 +0000

    Export the VSMW cluster New/Destroy functions

diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c
index 6611487..f30c7de 100644
--- a/bin/varnishd/common/common_vsmw.c
+++ b/bin/varnishd/common/common_vsmw.c
@@ -158,8 +158,8 @@ vsmw_mkent(const struct vsmw *vsmw, const char *pfx)
 
 /*--------------------------------------------------------------------*/
 
-static struct vsmw_cluster *
-vsmw_NewCluster(struct vsmw *vsmw, size_t len, const char *pfx)
+struct vsmw_cluster *
+VSMW_NewCluster(struct vsmw *vsmw, size_t len, const char *pfx)
 {
 	struct vsmw_cluster *vc;
 	int fd;
@@ -195,8 +195,8 @@ vsmw_NewCluster(struct vsmw *vsmw, size_t len, const char *pfx)
 	return (vc);
 }
 
-static void
-vsmw_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vcp)
+void
+VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vcp)
 {
 	struct vsmw_cluster *vc;
 
@@ -238,12 +238,13 @@ VSMW_Allocv(struct vsmw *vsmw, const char *class, size_t len,
 	AZ(VSB_finish(vsmw->vsb));
 	REPLACE(seg->id, VSB_data(vsmw->vsb));
 
-	vc = vsmw_NewCluster(vsmw, len, class);
+	vc = VSMW_NewCluster(vsmw, 64 + len, class);
 	AN(vc);
 	vc->refs++;
 
 	seg->cluster = vc;
-	seg->ptr = vc->ptr;
+	seg->off = 64;
+	seg->ptr = seg->off + (char*)vc->ptr;
 
 	VTAILQ_INSERT_TAIL(&vsmw->segs, seg, list);
 	fd = openat(vsmw->vdirfd, vsmw->idx, O_APPEND | O_WRONLY);
@@ -278,7 +279,7 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
 	CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
 
 	if (!--seg->cluster->refs)
-		vsmw_DestroyCluster(vsmw, &seg->cluster);
+		VSMW_DestroyCluster(vsmw, &seg->cluster);
 
 	VTAILQ_REMOVE(&vsmw->segs, seg, list);
 	REPLACE(seg->class, NULL);
diff --git a/bin/varnishd/common/vsmw.h b/bin/varnishd/common/vsmw.h
index 1ba2975..da6c3e3 100644
--- a/bin/varnishd/common/vsmw.h
+++ b/bin/varnishd/common/vsmw.h
@@ -32,6 +32,10 @@
 #define VSMW_H_INCLUDED
 
 struct vsmw;
+struct vsmw_cluster;
+
+struct vsmw_cluster *VSMW_NewCluster(struct vsmw *, size_t , const char *);
+void VSMW_DestroyCluster(struct vsmw *, struct vsmw_cluster **);
 
 void *VSMW_Allocv(struct vsmw *, const char *, size_t, const char *, va_list);
 void *VSMW_Allocf(struct vsmw *, const char *, size_t, const char *, ...);

From phk at FreeBSD.org  Mon Jan  8 23:19:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon,  8 Jan 2018 23:19:08 +0000 (UTC)
Subject: [master] 91d33db Add vsmw_cluster arg to VSMW_Alloc[vf]()
Message-ID: <20180108231908.76CDEBB599@lists.varnish-cache.org>


commit 91d33db54b5045d17ef27269620dd2c0913d7d6a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 8 21:03:09 2018 +0000

    Add vsmw_cluster arg to VSMW_Alloc[vf]()

diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c
index 01c8c36..c7b5ed8 100644
--- a/bin/varnishd/cache/cache_shmlog.c
+++ b/bin/varnishd/cache/cache_shmlog.c
@@ -509,7 +509,7 @@ VSM_Init(void)
 	AN(VSC_C_main);
 
 	AN(heritage.proc_vsmw);
-	vsl_head = VSMW_Allocf(heritage.proc_vsmw, VSL_CLASS,
+	vsl_head = VSMW_Allocf(heritage.proc_vsmw, NULL, VSL_CLASS,
 	    cache_param->vsl_space, VSL_CLASS);
 	AN(vsl_head);
 	vsl_segsize = ((cache_param->vsl_space - sizeof *vsl_head) /
diff --git a/bin/varnishd/common/common_vsc.c b/bin/varnishd/common/common_vsc.c
index 388418f..205a49a 100644
--- a/bin/varnishd/common/common_vsc.c
+++ b/bin/varnishd/common/common_vsc.c
@@ -83,7 +83,7 @@ vrt_vsc_mksegv(const char *class, size_t payload, const char *fmt, va_list va)
 	co = PRNDUP(sizeof(struct vsc_head));
 	ALLOC_OBJ(vsg, VSC_SEG_MAGIC);
 	AN(vsg);
-	vsg->seg = VSMW_Allocv(heritage.proc_vsmw, class,
+	vsg->seg = VSMW_Allocv(heritage.proc_vsmw, NULL, class,
 	    co + PRNDUP(payload), fmt, va);
 	AN(vsg->seg);
 	vsg->vsm = heritage.proc_vsmw;
diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c
index f30c7de..68f854e 100644
--- a/bin/varnishd/common/common_vsmw.c
+++ b/bin/varnishd/common/common_vsmw.c
@@ -219,14 +219,15 @@ VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vcp)
 /*--------------------------------------------------------------------*/
 
 void *
-VSMW_Allocv(struct vsmw *vsmw, const char *class, size_t len,
+VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc,
+    const char *class, size_t len,
     const char *fmt, va_list va)
 {
 	struct vsmwseg *seg;
-	struct vsmw_cluster *vc;
 	int fd;
 
 	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
+	(void)vc;
 
 	ALLOC_OBJ(seg, VSMWSEG_MAGIC);
 	AN(seg);
@@ -256,14 +257,14 @@ VSMW_Allocv(struct vsmw *vsmw, const char *class, size_t len,
 }
 
 void *
-VSMW_Allocf(struct vsmw *vsmw, const char *class, size_t len,
-    const char *fmt, ...)
+VSMW_Allocf(struct vsmw *vsmw, struct vsmw_cluster *vc,
+    const char *class, size_t len, const char *fmt, ...)
 {
 	va_list ap;
 	void *p;
 
 	va_start(ap, fmt);
-	p = VSMW_Allocv(vsmw, class, len, fmt, ap);
+	p = VSMW_Allocv(vsmw, vc, class, len, fmt, ap);
 	va_end(ap);
 	return (p);
 }
diff --git a/bin/varnishd/common/vsmw.h b/bin/varnishd/common/vsmw.h
index da6c3e3..4d4e359 100644
--- a/bin/varnishd/common/vsmw.h
+++ b/bin/varnishd/common/vsmw.h
@@ -37,8 +37,10 @@ struct vsmw_cluster;
 struct vsmw_cluster *VSMW_NewCluster(struct vsmw *, size_t , const char *);
 void VSMW_DestroyCluster(struct vsmw *, struct vsmw_cluster **);
 
-void *VSMW_Allocv(struct vsmw *, const char *, size_t, const char *, va_list);
-void *VSMW_Allocf(struct vsmw *, const char *, size_t, const char *, ...);
+void *VSMW_Allocv(struct vsmw *, struct vsmw_cluster *,
+    const char *, size_t, const char *, va_list);
+void *VSMW_Allocf(struct vsmw *, struct vsmw_cluster *,
+    const char *, size_t, const char *, ...);
 void VSMW_Free(struct vsmw *, void **);
 
 struct vsmw *VSMW_New(int, int, const char *);
diff --git a/bin/varnishd/mgt/mgt_shmem.c b/bin/varnishd/mgt/mgt_shmem.c
index c165ceb..ddfa650 100644
--- a/bin/varnishd/mgt/mgt_shmem.c
+++ b/bin/varnishd/mgt/mgt_shmem.c
@@ -58,7 +58,7 @@ mgt_SHM_static_alloc(const void *ptr, ssize_t size,
 {
 	void *p;
 
-	p = VSMW_Allocf(mgt_vsmw, class, size, "%s", ident);
+	p = VSMW_Allocf(mgt_vsmw, NULL, class, size, "%s", ident);
 	AN(p);
 	memcpy(p, ptr, size);
 }
@@ -122,13 +122,13 @@ mgt_SHM_ChildNew(void)
 
 	MCH_Fd_Inherit(heritage.vsm_fd, "VSMW");
 
-	heritage.param = VSMW_Allocf(mgt_vsmw, VSM_CLASS_PARAM,
+	heritage.param = VSMW_Allocf(mgt_vsmw, NULL, VSM_CLASS_PARAM,
 	    sizeof *heritage.param, "");
 	AN(heritage.param);
 	*heritage.param = mgt_param;
 
 	heritage.panic_str_len = 64 * 1024;
-	heritage.panic_str = VSMW_Allocf(mgt_vsmw, "Panic",
+	heritage.panic_str = VSMW_Allocf(mgt_vsmw, NULL, "Panic",
 	    heritage.panic_str_len, "");
 	AN(heritage.panic_str);
 }

From phk at FreeBSD.org  Mon Jan  8 23:19:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon,  8 Jan 2018 23:19:08 +0000 (UTC)
Subject: [master] b5e41e9 Tell VCC to generate backend-teardown for static
 backends.
Message-ID: <20180108231908.8EB80BB59D@lists.varnish-cache.org>


commit b5e41e95378c5bc55feff65e75e25ef7413eb07d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 8 22:12:48 2018 +0000

    Tell VCC to generate backend-teardown for static backends.

diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index b8a76ec..43f9eea 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -452,6 +452,7 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 	    "\t%s =\n\t    VRT_new_backend_clustered(ctx, vsc_cluster,\n"
 	    "\t\t&vgc_dir_priv_%s);",
 	    vgcname, vgcname);
+	VSB_printf(ifp->fin, "\t\tVRT_delete_backend(ctx, &%s);", vgcname);
 }
 
 /*--------------------------------------------------------------------

From phk at FreeBSD.org  Mon Jan  8 23:19:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon,  8 Jan 2018 23:19:08 +0000 (UTC)
Subject: [master] b6b249a Don't yank/recreate VBE->vsc on COLD/WARM events,
 instead just mark them to be ignored in VSCAPI and do so there.
Message-ID: <20180108231908.A5697BB5AD@lists.varnish-cache.org>


commit b6b249ae42e774fcf8dae0b14584fede5af4798d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 8 23:17:40 2018 +0000

    Don't yank/recreate VBE->vsc on COLD/WARM events, instead just mark them
    to be ignored in VSCAPI and do so there.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 826b13d..44c1d2b 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -196,6 +196,7 @@ vbe_dir_finish(const struct director *d, struct worker *wrk,
 	}
 	assert(bp->n_conn > 0);
 	bp->n_conn--;
+	AN(bp->vsc);
 	bp->vsc->conn--;
 #define ACCT(foo)	bp->vsc->foo += bo->acct.foo;
 #include "tbl/acct_fields_bereq.h"
@@ -339,13 +340,8 @@ vbe_dir_event(const struct director *d, enum vcl_event_e ev)
 	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
 	CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC);
 
-	if (ev == VCL_EVENT_WARM) {
-		AZ(bp->vsc);
-		bp->vsc =
-		    VSC_vbe_New(NULL, &bp->vsc_seg,
-			bp->director->display_name);
-		AN(bp->vsc);
-	}
+	if (ev == VCL_EVENT_WARM)
+		VRT_VSC_Reveal(bp->vsc_seg);
 
 	if (bp->probe != NULL && ev == VCL_EVENT_WARM)
 		VBP_Control(bp, 1);
@@ -353,12 +349,8 @@ vbe_dir_event(const struct director *d, enum vcl_event_e ev)
 	if (bp->probe != NULL && ev == VCL_EVENT_COLD)
 		VBP_Control(bp, 0);
 
-	if (ev == VCL_EVENT_COLD) {
-		Lck_Lock(&backends_mtx);
-		bp->vsc = NULL;
-		Lck_Unlock(&backends_mtx);
-		VSC_vbe_Destroy(&bp->vsc_seg);
-	}
+	if (ev == VCL_EVENT_COLD)
+		VRT_VSC_Hide(bp->vsc_seg);
 }
 
 /*---------------------------------------------------------------------*/
@@ -374,6 +366,7 @@ vbe_destroy(const struct director *d)
 	if (be->probe != NULL)
 		VBP_Remove(be);
 
+	VSC_vbe_Destroy(&be->vsc_seg);
 	Lck_Lock(&backends_mtx);
 	if (be->cooled > 0)
 		VTAILQ_REMOVE(&cool_backends, be, list);
@@ -389,7 +382,6 @@ vbe_destroy(const struct director *d)
 #undef DA
 #undef DN
 
-	AZ(be->vsc);
 	Lck_Delete(&be->mtx);
 	FREE_OBJ(be);
 }
@@ -494,6 +486,10 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc,
 		VBP_Insert(be, vbp, be->tcp_pool);
 	}
 
+	be->vsc = VSC_vbe_New(NULL, &be->vsc_seg,
+	    "%s.%s", VCL_Name(ctx->vcl), vrt->vcl_name);
+	AN(be->vsc);
+
 	retval = VCL_AddDirector(ctx->vcl, d, vrt->vcl_name);
 
 	if (retval == 0)
diff --git a/bin/varnishd/common/common_vsc.c b/bin/varnishd/common/common_vsc.c
index 205a49a..e6caf04 100644
--- a/bin/varnishd/common/common_vsc.c
+++ b/bin/varnishd/common/common_vsc.c
@@ -105,6 +105,22 @@ vrt_vsc_mksegf(const char *class, size_t payload, const char *fmt, ...)
 	return (vsg);
 }
 
+void
+VRT_VSC_Hide(struct vsc_seg *vsg)
+{
+	CHECK_OBJ_NOTNULL(vsg, VSC_SEG_MAGIC);
+	assert(vsg->head->ready > 0);
+	vsg->head->ready = 2;
+}
+
+void
+VRT_VSC_Reveal(struct vsc_seg *vsg)
+{
+	CHECK_OBJ_NOTNULL(vsg, VSC_SEG_MAGIC);
+	assert(vsg->head->ready > 0);
+	vsg->head->ready = 1;
+}
+
 void *
 VRT_VSC_Alloc(struct vsmw_cluster *vc, struct vsc_seg **sg,
     const char *nm, size_t sd,
diff --git a/include/vrt.h b/include/vrt.h
index 05dce64..ca80b4c 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -56,6 +56,8 @@
  *	VRT_Healthy() added
  *	VRT_VSC_Alloc() added
  *	VRT_VSC_Destroy() added
+ *	VRT_VSC_Hide() added
+ *	VRT_VSC_Reveal() added
  *	struct director.event added
  *	struct director.destroy added
  *	VRT_r_beresp_storage_hint() removed - under discussion #2509
@@ -454,4 +456,6 @@ VCL_STRING VRT_TIME_string(VRT_CTX, VCL_TIME);
 void *VRT_VSC_Alloc(struct vsmw_cluster *, struct vsc_seg **,
     const char *, size_t, const unsigned char *, size_t, const char *, va_list);
 void VRT_VSC_Destroy(const char *, struct vsc_seg *);
+void VRT_VSC_Hide(struct vsc_seg *);
+void VRT_VSC_Reveal(struct vsc_seg *);
 #endif
diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index 8f7c2df..8ec77b5 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -431,7 +431,7 @@ VSC_Iter(struct vsc *vsc, struct vsm *vsm, VSC_iter_f *fiter, void *priv)
 		} else {
 			sp2 = VTAILQ_NEXT(sp, list);
 		}
-		if (sp != NULL && fiter != NULL)
+		if (sp != NULL && fiter != NULL && sp->head->ready < 2)
 			i = vsc_iter_seg(vsc, sp, fiter, priv);
 		sp = sp2;
 		if (i)

From phk at FreeBSD.org  Mon Jan  8 23:22:03 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon,  8 Jan 2018 23:22:03 +0000 (UTC)
Subject: [master] b9ea8cd Drop pointless assert.
Message-ID: <20180108232203.B05F1BBDB9@lists.varnish-cache.org>


commit b9ea8cd15a49aca30d9944698ad2703f10a73255
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 8 23:21:38 2018 +0000

    Drop pointless assert.

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 9842874..889a25e 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -714,7 +714,6 @@ VSM_Map(struct vsm *vd, struct vsm_fantom *vf)
 
 	ps = getpagesize();
 	of = strtoul(vg->av[2], NULL, 10);
-	assert(of >= 0);
 	off = RDN2(of, ps);
 
 	sz = strtoul(vg->av[3], NULL, 10);

From phk at FreeBSD.org  Tue Jan  9 09:30:19 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  9 Jan 2018 09:30:19 +0000 (UTC)
Subject: [master] 88e5a96 Constify
Message-ID: <20180109093019.AD6199B7D7@lists.varnish-cache.org>


commit 88e5a96e04d7a2e2fa04b6a0cfe0ffd8ff5a8d06
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 9 07:51:43 2018 +0000

    Constify

diff --git a/bin/varnishd/common/common_vsc.c b/bin/varnishd/common/common_vsc.c
index e6caf04..ae2f0a1 100644
--- a/bin/varnishd/common/common_vsc.c
+++ b/bin/varnishd/common/common_vsc.c
@@ -106,7 +106,7 @@ vrt_vsc_mksegf(const char *class, size_t payload, const char *fmt, ...)
 }
 
 void
-VRT_VSC_Hide(struct vsc_seg *vsg)
+VRT_VSC_Hide(const struct vsc_seg *vsg)
 {
 	CHECK_OBJ_NOTNULL(vsg, VSC_SEG_MAGIC);
 	assert(vsg->head->ready > 0);
@@ -114,7 +114,7 @@ VRT_VSC_Hide(struct vsc_seg *vsg)
 }
 
 void
-VRT_VSC_Reveal(struct vsc_seg *vsg)
+VRT_VSC_Reveal(const struct vsc_seg *vsg)
 {
 	CHECK_OBJ_NOTNULL(vsg, VSC_SEG_MAGIC);
 	assert(vsg->head->ready > 0);
diff --git a/include/vrt.h b/include/vrt.h
index ca80b4c..6835983 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -456,6 +456,6 @@ VCL_STRING VRT_TIME_string(VRT_CTX, VCL_TIME);
 void *VRT_VSC_Alloc(struct vsmw_cluster *, struct vsc_seg **,
     const char *, size_t, const unsigned char *, size_t, const char *, va_list);
 void VRT_VSC_Destroy(const char *, struct vsc_seg *);
-void VRT_VSC_Hide(struct vsc_seg *);
-void VRT_VSC_Reveal(struct vsc_seg *);
+void VRT_VSC_Hide(const struct vsc_seg *);
+void VRT_VSC_Reveal(const struct vsc_seg *);
 #endif

From phk at FreeBSD.org  Tue Jan  9 09:30:19 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  9 Jan 2018 09:30:19 +0000 (UTC)
Subject: [master] 111a42f Pass the vsmw_cluster arg through
Message-ID: <20180109093019.BF0D79B7DA@lists.varnish-cache.org>


commit 111a42f745cf188ec13a0be10b6e15b9d1fd3f15
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 9 09:00:34 2018 +0000

    Pass the vsmw_cluster arg through

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 44c1d2b..0dcfa48 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -433,7 +433,6 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc,
 	int retval;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	(void)vc;
 	CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC);
 	assert(vrt->ipv4_suckaddr != NULL || vrt->ipv6_suckaddr != NULL);
 
@@ -486,7 +485,7 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc,
 		VBP_Insert(be, vbp, be->tcp_pool);
 	}
 
-	be->vsc = VSC_vbe_New(NULL, &be->vsc_seg,
+	be->vsc = VSC_vbe_New(vc, &be->vsc_seg,
 	    "%s.%s", VCL_Name(ctx->vcl), vrt->vcl_name);
 	AN(be->vsc);
 
diff --git a/bin/varnishd/common/common_vsc.c b/bin/varnishd/common/common_vsc.c
index ae2f0a1..103b967 100644
--- a/bin/varnishd/common/common_vsc.c
+++ b/bin/varnishd/common/common_vsc.c
@@ -75,7 +75,8 @@ vsc_callback_f *vsc_lock;
 vsc_callback_f *vsc_unlock;
 
 static struct vsc_seg *
-vrt_vsc_mksegv(const char *class, size_t payload, const char *fmt, va_list va)
+vrt_vsc_mksegv(struct vsmw_cluster *vc, const char *class,
+    size_t payload, const char *fmt, va_list va)
 {
 	struct vsc_seg *vsg;
 	size_t co;
@@ -83,7 +84,7 @@ vrt_vsc_mksegv(const char *class, size_t payload, const char *fmt, va_list va)
 	co = PRNDUP(sizeof(struct vsc_head));
 	ALLOC_OBJ(vsg, VSC_SEG_MAGIC);
 	AN(vsg);
-	vsg->seg = VSMW_Allocv(heritage.proc_vsmw, NULL, class,
+	vsg->seg = VSMW_Allocv(heritage.proc_vsmw, vc, class,
 	    co + PRNDUP(payload), fmt, va);
 	AN(vsg->seg);
 	vsg->vsm = heritage.proc_vsmw;
@@ -100,7 +101,7 @@ vrt_vsc_mksegf(const char *class, size_t payload, const char *fmt, ...)
 	struct vsc_seg *vsg;
 
 	va_start(ap, fmt);
-	vsg = vrt_vsc_mksegv(class, payload, fmt, ap);
+	vsg = vrt_vsc_mksegv(NULL, class, payload, fmt, ap);
 	va_end(ap);
 	return (vsg);
 }
@@ -130,7 +131,6 @@ VRT_VSC_Alloc(struct vsmw_cluster *vc, struct vsc_seg **sg,
 	char buf[1024];
 	uintptr_t jjp;
 
-	AZ(vc);
 	if (vsc_lock != NULL)
 		vsc_lock();
 
@@ -164,7 +164,7 @@ VRT_VSC_Alloc(struct vsmw_cluster *vc, struct vsc_seg **sg,
 
 	AN(heritage.proc_vsmw);
 
-	vsg = vrt_vsc_mksegv(VSC_CLASS, sd, buf, va);
+	vsg = vrt_vsc_mksegv(vc, VSC_CLASS, sd, buf, va);
 	AN(vsg);
 	vsg->nm = nm;
 	vsg->doc = dvsg;

From phk at FreeBSD.org  Tue Jan  9 09:30:19 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue,  9 Jan 2018 09:30:19 +0000 (UTC)
Subject: [master] c180d47 Start sharing vsmw_clusters for static backends in
 same VCL.
Message-ID: <20180109093019.D2EC69B7DC@lists.varnish-cache.org>


commit c180d472a8bcac687bb32ff812b870cf92db8faa
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 9 09:13:10 2018 +0000

    Start sharing vsmw_clusters for static backends in same VCL.

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index d8914cb..e4c0261 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -42,6 +42,9 @@
 #include "vtcp.h"
 #include "vtim.h"
 
+#include "common/heritage.h"
+#include "common/vsmw.h"
+
 const void * const vrt_magic_string_end = &vrt_magic_string_end;
 const void * const vrt_magic_string_unset = &vrt_magic_string_unset;
 
@@ -628,10 +631,17 @@ VRT_purge(VRT_CTX, double ttl, double grace, double keep)
 struct vsmw_cluster * v_matchproto_()
 VRT_VSM_Cluster_New(VRT_CTX, size_t sz)
 {
+	struct vsmw_cluster *vc;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
-	AN(sz);
-	return ((void*)VRT_VSM_Cluster_New);
+	assert(sz > 0);
+	AN(vsc_lock);
+	AN(vsc_unlock);
+	AN(heritage.proc_vsmw);
+	vsc_lock();
+	vc = VSMW_NewCluster(heritage.proc_vsmw, sz, "VSC_cluster");
+	vsc_unlock();
+	return (vc);
 }
 
 void v_matchproto_()
diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c
index 68f854e..ec58bbd 100644
--- a/bin/varnishd/common/common_vsmw.c
+++ b/bin/varnishd/common/common_vsmw.c
@@ -75,6 +75,7 @@ struct vsmw_cluster {
 	char				*fn;
 	size_t				len;
 	void				*ptr;
+	size_t				next;
 	int				refs;
 };
 
@@ -239,12 +240,15 @@ VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc,
 	AZ(VSB_finish(vsmw->vsb));
 	REPLACE(seg->id, VSB_data(vsmw->vsb));
 
-	vc = VSMW_NewCluster(vsmw, 64 + len, class);
+	if (vc == NULL)
+		vc = VSMW_NewCluster(vsmw, len, class);
 	AN(vc);
 	vc->refs++;
 
 	seg->cluster = vc;
-	seg->off = 64;
+	seg->off = vc->next;
+	vc->next += len;
+	assert(vc->next <= vc->len);
 	seg->ptr = seg->off + (char*)vc->ptr;
 
 	VTAILQ_INSERT_TAIL(&vsmw->segs, seg, list);

From phk at FreeBSD.org  Wed Jan 10 12:44:12 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 10 Jan 2018 12:44:12 +0000 (UTC)
Subject: [master] 86e8f72 Handle the case where we have no timeouts
Message-ID: <20180110124412.85CD0A714B@lists.varnish-cache.org>


commit 86e8f7216b2e87e2d83ee0d21d4903b24a9b661b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 10 11:55:14 2018 +0000

    Handle the case where we have no timeouts

diff --git a/lib/libvarnish/vev.c b/lib/libvarnish/vev.c
index 25494a2..0254a9c 100644
--- a/lib/libvarnish/vev.c
+++ b/lib/libvarnish/vev.c
@@ -411,6 +411,7 @@ VEV_Once(struct vev_root *evb)
 	if (evb->psig)
 		return (vev_sched_signal(evb));
 
+	tmo = INFTIM;
 	e = binheap_root(evb->binheap);
 	if (e != NULL) {
 		CHECK_OBJ_NOTNULL(e, VEV_MAGIC);
@@ -418,11 +419,11 @@ VEV_Once(struct vev_root *evb)
 		t = VTIM_mono();
 		if (e->__when <= t)
 			return (vev_sched_timeout(evb, e, t));
-		tmo = (int)((e->__when - t) * 1e3);
+		if (e->__when < 9e99)
+			tmo = (int)((e->__when - t) * 1e3);
 		if (tmo == 0)
 			tmo = 1;
-	} else
-		tmo = INFTIM;
+	}
 
 	if (tmo == INFTIM && evb->lpfd == BINHEAP_NOIDX + 1)
 		return (0);
@@ -430,6 +431,8 @@ VEV_Once(struct vev_root *evb)
 	i = poll(evb->pfd + 1, evb->lpfd - 1, tmo);
 	if (i == -1 && errno == EINTR)
 		return (vev_sched_signal(evb));
+	if (i == -1)
+		return (-1);
 
 	if (i == 0) {
 		assert(e != NULL);

From phk at FreeBSD.org  Wed Jan 10 12:44:12 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 10 Jan 2018 12:44:12 +0000 (UTC)
Subject: [master] c8eedc2 Add macro_expandf() to avoid unhealthy use of local
 buffers.
Message-ID: <20180110124412.771E4A7149@lists.varnish-cache.org>


commit c8eedc29c4458cd088023181d039b78f90aa5b11
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 10 07:05:12 2018 +0000

    Add macro_expandf() to avoid unhealthy use of local buffers.

diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 950b003..02e8da3 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -211,6 +211,23 @@ macro_get(const char *b, const char *e)
 }
 
 struct vsb *
+macro_expandf(struct vtclog *vl, const char *fmt, ...)
+{
+	va_list ap;
+	struct vsb *vsb1, *vsb2;
+
+	vsb1 = VSB_new_auto();
+	AN(vsb1);
+	va_start(ap, fmt);
+	VSB_vprintf(vsb1, fmt, ap);
+	va_end(ap);
+	AZ(VSB_finish(vsb1));
+	vsb2 = macro_expand(vl, VSB_data(vsb1));
+	VSB_destroy(&vsb1);
+	return (vsb2);
+}
+
+struct vsb *
 macro_expand(struct vtclog *vl, const char *text)
 {
 	struct vsb *vsb;
diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index 52e12e5..af89358 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -121,6 +121,8 @@ void macro_def(struct vtclog *vl, const char *instance, const char *name,
     const char *fmt, ...)
     v_printflike_(4, 5);
 struct vsb *macro_expand(struct vtclog *vl, const char *text);
+struct vsb *macro_expandf(struct vtclog *vl, const char *, ...)
+    v_printflike_(2, 3);
 
 void extmacro_def(const char *name, const char *fmt, ...)
     v_printflike_(2, 3);
diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c
index 0280211..f7f68af 100644
--- a/bin/varnishtest/vtc_barrier.c
+++ b/bin/varnishtest/vtc_barrier.c
@@ -299,9 +299,7 @@ barrier_sock_sync(struct barrier *b, struct vtclog *vl)
 	CHECK_OBJ_NOTNULL(b, BARRIER_MAGIC);
 	assert(b->type == BARRIER_SOCK);
 
-	i = snprintf(buf, sizeof buf, "${%s_sock}", b->name);
-	assert(i > 0 && i < sizeof buf);
-	vsb = macro_expand(vl, buf);
+	vsb = macro_expandf(vl, "${%s_sock}", b->name);
 	vtc_log(vl, 4, "Barrier(%s) sync with socket", b->name);
 
 	sock = VTCP_open(VSB_data(vsb), NULL, 0., &err);
diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c
index c4d4ad8..167d80c 100644
--- a/bin/varnishtest/vtc_client.c
+++ b/bin/varnishtest/vtc_client.c
@@ -117,7 +117,6 @@ client_thread(void *priv)
 	int fd;
 	unsigned u;
 	struct vsb *vsb;
-	char *p;
 	char mabuf[32], mpbuf[32];
 	const char *err;
 
@@ -126,9 +125,7 @@ client_thread(void *priv)
 
 	vl = vtc_logopen(c->name);
 
-	p = strdup(c->connect);
-	AN(p);
-	vsb = macro_expand(vl, p);
+	vsb = macro_expand(vl, c->connect);
 	AN(vsb);
 
 	if (c->repeat == 0)
@@ -155,7 +152,6 @@ client_thread(void *priv)
 	}
 	vtc_log(vl, 2, "Ending");
 	VSB_destroy(&vsb);
-	free(p);
 	return (NULL);
 }
 
diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c
index 5b983f1..cbb3c07 100644
--- a/bin/varnishtest/vtc_logexp.c
+++ b/bin/varnishtest/vtc_logexp.c
@@ -200,7 +200,7 @@ static struct logexp *
 logexp_new(const char *name, const char *varg)
 {
 	struct logexp *le;
-	struct vsb *vsb, *n_arg;
+	struct vsb *n_arg;
 
 	ALLOC_OBJ(le, LOGEXP_MAGIC);
 	AN(le);
@@ -219,12 +219,7 @@ logexp_new(const char *name, const char *varg)
 
 	REPLACE(le->vname, varg);
 
-	vsb = VSB_new_auto();
-	AN(vsb);
-	AZ(VSB_printf(vsb, "${tmpdir}/%s", varg));
-	AZ(VSB_finish(vsb));
-	n_arg = macro_expand(le->vl, VSB_data(vsb));
-	VSB_destroy(&vsb);
+	n_arg = macro_expandf(le->vl, "${tmpdir}/%s", varg);
 	if (n_arg == NULL)
 		vtc_fatal(le->vl, "-v argument problems");
 	if (VSM_Arg(le->vsm, 'n', VSB_data(n_arg)) <= 0)
diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index cb168eb..e6d9ad1 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -71,14 +71,13 @@ static VTAILQ_HEAD(, process)	processes =
  * Allocate and initialize a process
  */
 
-#define PROCESS_EXPAND(field, format, ...)		\
-	do {						\
-		bprintf(buf, format, __VA_ARGS__);	\
-		vsb = macro_expand(p->vl, buf);		\
-		AN(vsb);				\
-		p->field = strdup(VSB_data(vsb));	\
-		AN(p->field);				\
-		VSB_destroy(&vsb);			\
+#define PROCESS_EXPAND(field, format, ...)				\
+	do {								\
+		vsb = macro_expandf(p->vl, format, __VA_ARGS__);	\
+		AN(vsb);						\
+		p->field = strdup(VSB_data(vsb));			\
+		AN(p->field);						\
+		VSB_destroy(&vsb);					\
 	} while (0)
 
 static struct process *
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 569ad7b..1b0934a 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -308,8 +308,7 @@ varnish_new(const char *name)
 	v->vl = vtc_logopen(name);
 	AN(v->vl);
 
-	bprintf(buf, "${tmpdir}/%s", name);
-	vsb = macro_expand(v->vl, buf);
+	vsb = macro_expandf(v->vl, "${tmpdir}/%s", name);
 	AN(vsb);
 	v->workdir = strdup(VSB_data(vsb));
 	AN(v->workdir);

From phk at FreeBSD.org  Wed Jan 10 12:44:12 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 10 Jan 2018 12:44:12 +0000 (UTC)
Subject: [master] ccfa03f Reintroduce VLU_Feed()
Message-ID: <20180110124412.A0F55A714E@lists.varnish-cache.org>


commit ccfa03fc5f0e935b897204889ebae2bad09c8c1e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 10 11:55:32 2018 +0000

    Reintroduce VLU_Feed()

diff --git a/include/vlu.h b/include/vlu.h
index f92642e..47cf705 100644
--- a/include/vlu.h
+++ b/include/vlu.h
@@ -37,3 +37,4 @@ struct vlu *VLU_New(vlu_f *, void *, unsigned);
 int VLU_Fd(struct vlu *, int);
 void VLU_Destroy(struct vlu **);
 int VLU_File(int, vlu_f *, void *, unsigned);
+int VLU_Feed(struct vlu *, const char*, int);
diff --git a/lib/libvarnish/vlu.c b/lib/libvarnish/vlu.c
index 2ae654e..bc32df9 100644
--- a/lib/libvarnish/vlu.c
+++ b/lib/libvarnish/vlu.c
@@ -143,3 +143,26 @@ VLU_File(int fd, vlu_f *func, void *priv, unsigned bufsize)
 	VLU_Destroy(&vlu);
 	return (i);
 }
+
+int
+VLU_Feed(struct vlu *l, const char *ptr, int len)
+{
+	int i = 0, ll;
+
+	CHECK_OBJ_NOTNULL(l, LINEUP_MAGIC);
+	AN(ptr);
+	assert(len > 0);
+	while (len > 0) {
+		ll = len;
+		if (ll > l->bufl - l->bufp)
+			ll = l->bufl - l->bufp;
+		memcpy(l->buf + l->bufp, ptr, ll);
+		len -= ll;
+		ptr += ll;
+		l->bufp += ll;
+		i = LineUpProcess(l);
+		if (i)
+			return (i);
+	}
+	return (i);
+}

From phk at FreeBSD.org  Wed Jan 10 12:44:12 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 10 Jan 2018 12:44:12 +0000 (UTC)
Subject: [master] 70c9a26 Restructure 'process' a bit and teach it a few new
 tricks.
Message-ID: <20180110124412.B9570A7153@lists.varnish-cache.org>


commit 70c9a26460d64eec7ffd60ba9fe4391b873a1fbb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 10 12:39:47 2018 +0000

    Restructure 'process' a bit and teach it a few new tricks.

diff --git a/bin/varnishtest/tests/a00017.vtc b/bin/varnishtest/tests/a00017.vtc
index b700305..21b184c 100644
--- a/bin/varnishtest/tests/a00017.vtc
+++ b/bin/varnishtest/tests/a00017.vtc
@@ -42,7 +42,7 @@ shell -exit 1 -expect "Command failed with error code 101" {
 shell "varnishadm -n ${tmpdir}/v1 param.set cli_limit 128"
 shell -expect "[response was truncated]" "varnishadm -n ${tmpdir}/v1 help"
 
-process p1 -stop -wait
+process p1 -expect-exit 64 -stop -wait
 
 # Test multiple -f options
 
diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc
index 3ed2eaf..8a0ab79 100644
--- a/bin/varnishtest/tests/u00000.vtc
+++ b/bin/varnishtest/tests/u00000.vtc
@@ -2,13 +2,15 @@ varnishtest "Simple process tests"
 
 # new & start
 process p1 "cat" -start
-process p2 "cat" -start
-process p3 "cat" -start
+process p2 -log "cat" -start
+process p3 -dump "cat" -start
+process p4 -hexdump "cat" -start
 
 # write
 process p1 -writeln "foo"
 process p2 -writeln "bar"
 process p3 -writeln "baz"
+process p4 -writeln "b\001z"
 
 # give enough time for the writes
 delay 0.5
@@ -17,11 +19,13 @@ delay 0.5
 process p1 -stop
 process p2 -close
 process p3 -kill KILL
+process p4 -kill TERM
 
 # wait
 process p1 -wait
 process p2 -wait
 process p3 -wait
+process p4 -wait
 
 # check stdout
 shell "grep -q foo ${p1_out}"
diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index e6d9ad1..0388f99 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -33,13 +33,17 @@
 
 #include <errno.h>
 #include <fcntl.h>
+#include <poll.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
 
 #include "vtc.h"
+
+#include "vev.h"
 #include "vlu.h"
+#include "vsb.h"
 #include "vsub.h"
 
 struct process {
@@ -53,10 +57,17 @@ struct process {
 	char			*dir;
 	char			*out;
 	char			*err;
-	int			fd_to;
+	int			fd_stdin;
+	int			fd_stdout;
+	int			fd_stderr;
+	int			f_stdout;
+	int			f_stderr;
+	struct vlu		*vlu_stdout;
+	struct vlu		*vlu_stderr;
+	const char		*cur_fd;
 	int			log;
-	int			fd_from;
 	pid_t			pid;
+	int			expect_exit;
 
 	pthread_mutex_t		mtx;
 	pthread_t		tp;
@@ -103,8 +114,8 @@ process_new(const char *name)
 	    p->dir, p->dir, p->out, p->err);
 	AZ(system(buf));
 
-	p->fd_to = -1;
-	p->fd_from = -1;
+	p->fd_stdin = -1;
+	p->fd_stdout = -1;
 
 	VTAILQ_INSERT_TAIL(&processes, p, list);
 	return (p);
@@ -158,7 +169,66 @@ process_vlu_func(void *priv, const char *l)
 	struct process *p;
 
 	CAST_OBJ_NOTNULL(p, priv, PROCESS_MAGIC);
-	vtc_dump(p->vl, 4, "output", l, -1);
+	vtc_dump(p->vl, 4, p->cur_fd, l, -1);
+	return (0);
+}
+
+static int v_matchproto_(vev_cb_f)
+process_stdin(const struct vev *ev, int what)
+{
+	struct process *p;
+	CAST_OBJ_NOTNULL(p, ev->priv, PROCESS_MAGIC);
+	vtc_log(p->vl, 4, "stdin event 0x%x", what);
+	return (1);
+}
+
+static int v_matchproto_(vev_cb_f)
+process_stdout(const struct vev *ev, int what)
+{
+	struct process *p;
+	char buf[BUFSIZ];
+	int i;
+
+	CAST_OBJ_NOTNULL(p, ev->priv, PROCESS_MAGIC);
+	(void)what;
+	p->cur_fd = "stdout";
+	i = read(p->fd_stdout, buf, sizeof buf);
+	if (i <= 0) {
+		vtc_log(p->vl, 4, "stdout read %d", i);
+		return (1);
+	}
+	if (p->log == 1)
+		(void)VLU_Feed(p->vlu_stdout, buf, i);
+	else if (p->log == 2)
+		vtc_dump(p->vl, 4, "stdout", buf, i);
+	else if (p->log == 3)
+		vtc_hexdump(p->vl, 4, "stdout", buf, i);
+	(void)write(p->f_stdout, buf, i);
+	return (0);
+}
+
+static int v_matchproto_(vev_cb_f)
+process_stderr(const struct vev *ev, int what)
+{
+	struct process *p;
+	char buf[BUFSIZ];
+	int i;
+
+	CAST_OBJ_NOTNULL(p, ev->priv, PROCESS_MAGIC);
+	(void)what;
+	p->cur_fd = "stderr";
+	i = read(p->fd_stderr, buf, sizeof buf);
+	if (i <= 0) {
+		vtc_log(p->vl, 4, "stderr read %d", i);
+		return (1);
+	}
+	if (p->log == 1)
+		(void)VLU_Feed(p->vlu_stderr, buf, i);
+	else if (p->log == 2)
+		vtc_dump(p->vl, 4, "stderr", buf, i);
+	else if (p->log == 3)
+		vtc_hexdump(p->vl, 4, "stderr", buf, i);
+	(void)write(p->f_stderr, buf, i);
 	return (0);
 }
 
@@ -167,17 +237,65 @@ process_thread(void *priv)
 {
 	struct process *p;
 	struct rusage ru;
+	struct vev_root *evb;
+	struct vev *ev;
 	int r;
 
 	CAST_OBJ_NOTNULL(p, priv, PROCESS_MAGIC);
-	if (p->fd_from > 0)
-		(void)VLU_File(p->fd_from, process_vlu_func, p, 1024);
+
+	p->f_stdout = open(p->out, O_WRONLY|O_APPEND);
+	assert(p->f_stdout >= 0);
+	p->f_stderr = open(p->err, O_WRONLY|O_APPEND);
+	assert(p->f_stderr >= 0);
+
+	evb = VEV_New();
+	AN(evb);
+
+	ev = VEV_Alloc();
+	AN(ev);
+	ev->fd = p->fd_stdin;
+	ev->fd_flags = VEV__HUP | VEV__ERR;
+	ev->priv = p;
+	ev->callback = process_stdin;
+	AZ(VEV_Start(evb, ev));
+
+	ev = VEV_Alloc();
+	AN(ev);
+	ev->fd = p->fd_stdout;
+	ev->fd_flags = VEV__RD | VEV__HUP | VEV__ERR;
+	ev->callback = process_stdout;
+	ev->priv = p;
+	AZ(VEV_Start(evb, ev));
+
+	ev = VEV_Alloc();
+	AN(ev);
+	ev->fd = p->fd_stderr;
+	ev->fd_flags = VEV__RD | VEV__HUP | VEV__ERR;
+	ev->callback = process_stderr;
+	ev->priv = p;
+	AZ(VEV_Start(evb, ev));
+
+	if (p->log == 1) {
+		p->vlu_stdout = VLU_New(process_vlu_func, p, 1024);
+		AN(p->vlu_stdout);
+		p->vlu_stderr = VLU_New(process_vlu_func, p, 1024);
+		AN(p->vlu_stderr);
+	}
+
+	do {
+		r = VEV_Once(evb);
+	} while (r == 1);
+
+	if (r < 0)
+		vtc_fatal(p->vl, "VEV_Once() = %d, error %s", r,
+		    strerror(errno));
+
 	r = wait4(p->pid, &p->status, 0, &ru);
 
-	AZ(pthread_mutex_lock(&p->mtx));
+	closefd(&p->f_stdout);
+	closefd(&p->f_stderr);
 
-	if (p->fd_to >= 0)
-		closefd(&p->fd_to);
+	AZ(pthread_mutex_lock(&p->mtx));
 
 	/* NB: We keep the other macros around */
 	macro_undef(p->vl, p->name, "pid");
@@ -191,17 +309,23 @@ process_thread(void *priv)
 
 	AZ(pthread_mutex_unlock(&p->mtx));
 
-	if (WIFEXITED(p->status) && WEXITSTATUS(p->status) == 0)
-		return (NULL);
 #ifdef WCOREDUMP
-	vtc_log(p->vl, 2, "Bad exit code: %04x sig %d exit %d core %d",
+	vtc_log(p->vl, 2, "Exit code: %04x sig %d exit %d core %d",
 	    p->status, WTERMSIG(p->status), WEXITSTATUS(p->status),
 	    WCOREDUMP(p->status));
 #else
-	vtc_log(p->vl, 2, "Bad exit code: %04x sig %d exit %d",
+	vtc_log(p->vl, 2, "Exit code: %04x sig %d exit %d",
 	    p->status, WTERMSIG(p->status), WEXITSTATUS(p->status));
 #endif
-
+	if (WEXITSTATUS(p->status) != p->expect_exit)
+		vtc_fatal(p->vl, "Expected exit %d got %d",
+			p->expect_exit, WEXITSTATUS(p->status));
+
+	VEV_Destroy(&evb);
+	if (p->log == 1) {
+		VLU_Destroy(&p->vlu_stdout);
+		VLU_Destroy(&p->vlu_stderr);
+	}
 	return (NULL);
 }
 
@@ -209,9 +333,7 @@ static void
 process_start(struct process *p)
 {
 	struct vsb *cl;
-	int out_fd, err_fd;
-	int fds[2];
-	int fdt[2] = { -1, -1 };
+	int fd0[2], fd1[2], fd2[2];
 
 	CHECK_OBJ_NOTNULL(p, PROCESS_MAGIC);
 	if (p->hasthread)
@@ -221,44 +343,35 @@ process_start(struct process *p)
 
 	cl = macro_expand(p->vl, p->spec);
 	AN(cl);
-	AZ(pipe(fds));
-	if (p->log) {
-		AZ(pipe(fdt));
-		out_fd = fdt[1];
-		err_fd = fdt[1];
-	} else {
-		out_fd = open(p->out, O_WRONLY|O_APPEND);
-		assert(out_fd >= 0);
-		err_fd = open(p->err, O_WRONLY|O_APPEND);
-		assert(err_fd >= 0);
-	}
+
+	AZ(pipe(fd0));
+	AZ(pipe(fd1));
+	AZ(pipe(fd2));
+
 	p->pid = fork();
 	assert(p->pid >= 0);
 	if (p->pid == 0) {
-		assert(dup2(fds[0], 0) == 0);
-		assert(dup2(out_fd, 1) == 1);
-		assert(dup2(err_fd, 2) == 2);
+		assert(dup2(fd0[0], STDIN_FILENO) == STDIN_FILENO);
+		assert(dup2(fd1[1], STDOUT_FILENO) == STDOUT_FILENO);
+		assert(dup2(fd2[1], STDERR_FILENO) == STDERR_FILENO);
 		VSUB_closefrom(STDERR_FILENO + 1);
 		AZ(setpgid(0, 0));
-		AZ(execl("/bin/sh", "/bin/sh", "-c", VSB_data(cl),
-		    (char *)NULL));
+		AZ(execl("/bin/sh", "/bin/sh", "-c", VSB_data(cl), NULL));
 		exit(1);
 	}
 	vtc_log(p->vl, 3, "PID: %ld", (long)p->pid);
+	VSB_destroy(&cl);
+
+	closefd(&fd0[0]);
+	closefd(&fd1[1]);
+	closefd(&fd2[1]);
+	p->fd_stdin = fd0[1];
+	p->fd_stdout = fd1[0];
+	p->fd_stderr = fd2[0];
 	macro_def(p->vl, p->name, "pid", "%ld", (long)p->pid);
 	macro_def(p->vl, p->name, "dir", "%s", p->dir);
 	macro_def(p->vl, p->name, "out", "%s", p->out);
 	macro_def(p->vl, p->name, "err", "%s", p->err);
-	closefd(&fds[0]);
-	p->fd_to = fds[1];
-	if (p->log) {
-		closefd(&fdt[1]);
-		p->fd_from = fdt[0];
-	} else {
-		closefd(&out_fd);
-		closefd(&err_fd);
-	}
-	VSB_destroy(&cl);
 	p->hasthread = 1;
 	AZ(pthread_create(&p->tp, NULL, process_thread, p));
 }
@@ -330,7 +443,7 @@ process_write(const struct process *p, const char *text)
 
 	len = strlen(text);
 	vtc_log(p->vl, 4, "Writing %d bytes", len);
-	r = write(p->fd_to, text, len);
+	r = write(p->fd_stdin, text, len);
 	if (r < 0)
 		vtc_fatal(p->vl, "Failed to write: %s (%d)",
 		    strerror(errno), errno);
@@ -344,8 +457,8 @@ process_close(struct process *p)
 		vtc_fatal(p->vl, "Cannot close a non-running process");
 
 	AZ(pthread_mutex_lock(&p->mtx));
-	if (p->fd_to >= 0)
-		closefd(&p->fd_to);
+	if (p->fd_stdin >= 0)
+		closefd(&p->fd_stdin);
 	AZ(pthread_mutex_unlock(&p->mtx));
 }
 
@@ -354,8 +467,10 @@ process_close(struct process *p)
  * Run a process in the background with stdout and stderr redirected to
  * ${pNAME_out} and ${pNAME_err}, both located in ${pNAME_dir}::
  *
- *	process pNAME SPEC [-log] [-start] [-wait] [-run] [-kill STRING] \
- *		[-stop] [-write STRING] [-writeln STRING] [-close]
+ *	process pNAME SPEC [-log] [-dump] [-hexdump] [-expect-exit N]
+ *		[-start] [-run]
+ *		[-write STRING] [-writeln STRING]
+ *		[-kill STRING] [-stop] [-wait] [-close]
  *
  * pNAME
  *	Name of the process. It must start with 'p'.
@@ -363,12 +478,21 @@ process_close(struct process *p)
  * SPEC
  *	The command(s) to run in this process.
  *
- * \-log
+ * \-hexdump
+ *	Log stdout/stderr with vtc_hexdump(). Must be before -start/-run.
+ *
+ * \-dump
  *	Log stdout/stderr with vtc_dump(). Must be before -start/-run.
  *
+ * \-log
+ *	Log stdout/stderr with VLU/vtc_log(). Must be before -start/-run.
+ *
  * \-start
  *	Start the process.
  *
+ * \-expect-exit N
+ *	Expect exit status N
+ *
  * \-wait
  *	Wait for the process to finish.
  *
@@ -461,6 +585,20 @@ cmd_process(CMD_ARGS)
 			process_start(p);
 			continue;
 		}
+		if (!strcmp(*av, "-hexdump")) {
+			if (p->hasthread)
+				vtc_fatal(p->vl,
+				    "Cannot dump a running process");
+			p->log = 3;
+			continue;
+		}
+		if (!strcmp(*av, "-dump")) {
+			if (p->hasthread)
+				vtc_fatal(p->vl,
+				    "Cannot dump a running process");
+			p->log = 2;
+			continue;
+		}
 		if (!strcmp(*av, "-log")) {
 			if (p->hasthread)
 				vtc_fatal(p->vl,
@@ -468,6 +606,11 @@ cmd_process(CMD_ARGS)
 			p->log = 1;
 			continue;
 		}
+		if (!strcmp(*av, "-expect-exit")) {
+			p->expect_exit = strtoul(av[1], NULL, 0);
+			av++;
+			continue;
+		}
 		if (!strcmp(*av, "-wait")) {
 			process_wait(p);
 			continue;

From phk at FreeBSD.org  Wed Jan 10 14:55:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 10 Jan 2018 14:55:08 +0000 (UTC)
Subject: [master] 77e6d3a Drop the poll on fd_stdin linux doesn't handle
 collateral close.
Message-ID: <20180110145508.4172BA9DC0@lists.varnish-cache.org>


commit 77e6d3ae8e7f8b7e9a48872efa0d52b216ecc4fe
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 10 14:54:07 2018 +0000

    Drop the poll on fd_stdin linux doesn't handle collateral close.

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index 0388f99..42bfdf6 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -174,15 +174,6 @@ process_vlu_func(void *priv, const char *l)
 }
 
 static int v_matchproto_(vev_cb_f)
-process_stdin(const struct vev *ev, int what)
-{
-	struct process *p;
-	CAST_OBJ_NOTNULL(p, ev->priv, PROCESS_MAGIC);
-	vtc_log(p->vl, 4, "stdin event 0x%x", what);
-	return (1);
-}
-
-static int v_matchproto_(vev_cb_f)
 process_stdout(const struct vev *ev, int what)
 {
 	struct process *p;
@@ -253,14 +244,6 @@ process_thread(void *priv)
 
 	ev = VEV_Alloc();
 	AN(ev);
-	ev->fd = p->fd_stdin;
-	ev->fd_flags = VEV__HUP | VEV__ERR;
-	ev->priv = p;
-	ev->callback = process_stdin;
-	AZ(VEV_Start(evb, ev));
-
-	ev = VEV_Alloc();
-	AN(ev);
 	ev->fd = p->fd_stdout;
 	ev->fd_flags = VEV__RD | VEV__HUP | VEV__ERR;
 	ev->callback = process_stdout;

From phk at FreeBSD.org  Wed Jan 10 15:08:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 10 Jan 2018 15:08:06 +0000 (UTC)
Subject: [master] cd9e5db GCC doesn't think NULL is a valid terminator of
 execl(2).
Message-ID: <20180110150806.4E3ECAC2E1@lists.varnish-cache.org>


commit cd9e5db018ceebaa05dbb7810425ba91a1e56dfc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 10 15:07:04 2018 +0000

    GCC doesn't think NULL is a valid terminator of execl(2).
    
    I don't think GCC is a valid compiler for sensible code.

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index 42bfdf6..bde9e98 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -339,7 +339,8 @@ process_start(struct process *p)
 		assert(dup2(fd2[1], STDERR_FILENO) == STDERR_FILENO);
 		VSUB_closefrom(STDERR_FILENO + 1);
 		AZ(setpgid(0, 0));
-		AZ(execl("/bin/sh", "/bin/sh", "-c", VSB_data(cl), NULL));
+		// Not using NULL because GCC is now even more demented...
+		AZ(execl("/bin/sh", "/bin/sh", "-c", VSB_data(cl), (char*)0));
 		exit(1);
 	}
 	vtc_log(p->vl, 3, "PID: %ld", (long)p->pid);

From fgsch at lodoss.net  Wed Jan 10 19:02:06 2018
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Wed, 10 Jan 2018 19:02:06 +0000 (UTC)
Subject: [master] 431456a Update xcode and travis image
Message-ID: <20180110190207.0228DB06CE@lists.varnish-cache.org>


commit 431456a9d5fa0c8b53ae90d943a30eb5bc5d991f
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Jan 10 14:00:00 2018 +0000

    Update xcode and travis image

diff --git a/.travis.yml b/.travis.yml
index 3db4182..ef8a2ee 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,7 +1,6 @@
 ---
 sudo: required
 language: c
-group: deprecated-2017Q3
 matrix:
   include:
   - os: linux
@@ -15,7 +14,7 @@ matrix:
     compiler: clang
     env: CLANG=5.0 SAN_FLAGS="--enable-asan --enable-ubsan"
   - os: osx
-    osx_image: xcode9
+    osx_image: xcode9.2
     compiler: clang
   allow_failures:
   - os: osx

From phk at FreeBSD.org  Wed Jan 10 20:25:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 10 Jan 2018 20:25:06 +0000 (UTC)
Subject: [master] b8924bc Remember to take the VSC metadata structure into
 account when sizing VSMW clusters.
Message-ID: <20180110202506.734C5B1C96@lists.varnish-cache.org>


commit b8924bcd4cb1b8df7a4040236cd84df857c2c442
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 10 20:23:31 2018 +0000

    Remember to take the VSC metadata structure into account when
    sizing VSMW clusters.
    
    Fixes	#2535

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 0dcfa48..fd70cb2 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -419,7 +419,7 @@ size_t v_matchproto_()
 VRT_backend_vsm_need(VRT_CTX)
 {
 	(void)ctx;
-	return (VSC_vbe_size);
+	return (VRT_VSC_Overhead(VSC_vbe_size));
 }
 
 struct director * v_matchproto_()
diff --git a/bin/varnishd/common/common_vsc.c b/bin/varnishd/common/common_vsc.c
index 103b967..23dd5be 100644
--- a/bin/varnishd/common/common_vsc.c
+++ b/bin/varnishd/common/common_vsc.c
@@ -74,23 +74,23 @@ static VTAILQ_HEAD(,vsc_seg)	vsc_seglist =
 vsc_callback_f *vsc_lock;
 vsc_callback_f *vsc_unlock;
 
+static const size_t vsc_overhead = PRNDUP(sizeof(struct vsc_head));
+
 static struct vsc_seg *
 vrt_vsc_mksegv(struct vsmw_cluster *vc, const char *class,
     size_t payload, const char *fmt, va_list va)
 {
 	struct vsc_seg *vsg;
-	size_t co;
 
-	co = PRNDUP(sizeof(struct vsc_head));
 	ALLOC_OBJ(vsg, VSC_SEG_MAGIC);
 	AN(vsg);
 	vsg->seg = VSMW_Allocv(heritage.proc_vsmw, vc, class,
-	    co + PRNDUP(payload), fmt, va);
+	    VRT_VSC_Overhead(payload), fmt, va);
 	AN(vsg->seg);
 	vsg->vsm = heritage.proc_vsmw;
 	vsg->head = (void*)vsg->seg;
-	vsg->head->body_offset = co;
-	vsg->ptr = (char*)vsg->seg + co;
+	vsg->head->body_offset = vsc_overhead;
+	vsg->ptr = (char*)vsg->seg + vsc_overhead;
 	return (vsg);
 }
 
@@ -106,6 +106,12 @@ vrt_vsc_mksegf(const char *class, size_t payload, const char *fmt, ...)
 	return (vsg);
 }
 
+size_t
+VRT_VSC_Overhead(size_t payload)
+{
+	return (vsc_overhead + PRNDUP(payload));
+}
+
 void
 VRT_VSC_Hide(const struct vsc_seg *vsg)
 {
diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c
index ec58bbd..cf34a8f 100644
--- a/bin/varnishd/common/common_vsmw.c
+++ b/bin/varnishd/common/common_vsmw.c
@@ -221,7 +221,7 @@ VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vcp)
 
 void *
 VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc,
-    const char *class, size_t len,
+    const char *class, size_t payload,
     const char *fmt, va_list va)
 {
 	struct vsmwseg *seg;
@@ -233,7 +233,7 @@ VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc,
 	ALLOC_OBJ(seg, VSMWSEG_MAGIC);
 	AN(seg);
 	REPLACE(seg->class, class);
-	seg->len = len;
+	seg->len = PRNDUP(payload);
 
 	VSB_clear(vsmw->vsb);
 	VSB_vprintf(vsmw->vsb, fmt, va);
@@ -241,13 +241,13 @@ VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc,
 	REPLACE(seg->id, VSB_data(vsmw->vsb));
 
 	if (vc == NULL)
-		vc = VSMW_NewCluster(vsmw, len, class);
+		vc = VSMW_NewCluster(vsmw, seg->len, class);
 	AN(vc);
 	vc->refs++;
 
 	seg->cluster = vc;
 	seg->off = vc->next;
-	vc->next += len;
+	vc->next += seg->len;
 	assert(vc->next <= vc->len);
 	seg->ptr = seg->off + (char*)vc->ptr;
 
diff --git a/include/vrt.h b/include/vrt.h
index 6835983..662399d 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -58,6 +58,7 @@
  *	VRT_VSC_Destroy() added
  *	VRT_VSC_Hide() added
  *	VRT_VSC_Reveal() added
+ *	VRT_VSC_Overhead() added
  *	struct director.event added
  *	struct director.destroy added
  *	VRT_r_beresp_storage_hint() removed - under discussion #2509
@@ -455,7 +456,8 @@ VCL_STRING VRT_TIME_string(VRT_CTX, VCL_TIME);
 #ifdef va_start	// XXX: hackish
 void *VRT_VSC_Alloc(struct vsmw_cluster *, struct vsc_seg **,
     const char *, size_t, const unsigned char *, size_t, const char *, va_list);
+#endif
 void VRT_VSC_Destroy(const char *, struct vsc_seg *);
 void VRT_VSC_Hide(const struct vsc_seg *);
 void VRT_VSC_Reveal(const struct vsc_seg *);
-#endif
+size_t VRT_VSC_Overhead(size_t);

From phk at FreeBSD.org  Thu Jan 11 10:17:05 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 11 Jan 2018 10:17:05 +0000 (UTC)
Subject: [master] 59aeb8d Change process to use a pseudo-terminal for
 stdin/stdout.
Message-ID: <20180111101705.E6FC165396@lists.varnish-cache.org>


commit 59aeb8dc76e87bd7dbd0706bcc4b329795bce24b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 11 10:16:28 2018 +0000

    Change process to use a pseudo-terminal for stdin/stdout.
    
    Yes, evil things are afoot.

diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc
index 8a0ab79..b590e67 100644
--- a/bin/varnishtest/tests/u00000.vtc
+++ b/bin/varnishtest/tests/u00000.vtc
@@ -1,38 +1,33 @@
 varnishtest "Simple process tests"
 
-# new & start
 process p1 "cat" -start
-process p2 -log "cat" -start
-process p3 -dump "cat" -start
-process p4 -hexdump "cat" -start
-
-# write
 process p1 -writeln "foo"
-process p2 -writeln "bar"
-process p3 -writeln "baz"
-process p4 -writeln "b\001z"
-
-# give enough time for the writes
 delay 0.5
-
-# stop
 process p1 -stop
-process p2 -close
-process p3 -kill KILL
-process p4 -kill TERM
-
-# wait
 process p1 -wait
-process p2 -wait
-process p3 -wait
-process p4 -wait
-
-# check stdout
 shell "grep -q foo ${p1_out}"
-shell "grep -q bar ${p2_out}"
-shell "grep -q baz ${p3_out}"
-
-# check stderr
 shell "test -f ${p1_err} -a ! -s ${p1_err}"
+
+process p2 -log "cat" -start
+process p2 -writeln "bar"
+delay 0.5
+process p2 -close
+process p2 -wait
+shell "grep -q bar ${p2_out}"
 shell "test -f ${p2_err} -a ! -s ${p2_err}"
+
+process p3 -dump "cat" -start
+process p3 -writeln "baz"
+delay 0.5
+process p3 -kill KILL
+process p3 -wait
+shell "grep -q baz ${p3_out}"
 shell "test -f ${p3_err} -a ! -s ${p3_err}"
+
+process p4 -hexdump "cat" -start
+process p4 -writeln "b\001z"
+delay 0.5
+process p4 -kill TERM
+process p4 -wait
+
+process p5 -log "stty -a ; sleep 1" -run
diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index bde9e98..f6a8bdb 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -28,6 +28,7 @@
 
 #include "config.h"
 
+#include <sys/ioctl.h>		// Linux: struct winsize
 #include <sys/resource.h>
 #include <sys/wait.h>
 
@@ -37,6 +38,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <termios.h>
 #include <unistd.h>
 
 #include "vtc.h"
@@ -57,14 +59,12 @@ struct process {
 	char			*dir;
 	char			*out;
 	char			*err;
-	int			fd_stdin;
-	int			fd_stdout;
+	int			fd_term;
 	int			fd_stderr;
 	int			f_stdout;
 	int			f_stderr;
 	struct vlu		*vlu_stdout;
 	struct vlu		*vlu_stderr;
-	const char		*cur_fd;
 	int			log;
 	pid_t			pid;
 	int			expect_exit;
@@ -107,15 +107,14 @@ process_new(const char *name)
 	AN(p->vl);
 
 	PROCESS_EXPAND(dir, "${tmpdir}/%s", name);
-	PROCESS_EXPAND(out, "${tmpdir}/%s/stdout", name);
+	PROCESS_EXPAND(out, "${tmpdir}/%s/term", name);
 	PROCESS_EXPAND(err, "${tmpdir}/%s/stderr", name);
 
 	bprintf(buf, "rm -rf %s ; mkdir -p %s ; touch %s %s",
 	    p->dir, p->dir, p->out, p->err);
 	AZ(system(buf));
 
-	p->fd_stdin = -1;
-	p->fd_stdout = -1;
+	p->fd_term = -1;
 
 	VTAILQ_INSERT_TAIL(&processes, p, list);
 	return (p);
@@ -169,7 +168,7 @@ process_vlu_func(void *priv, const char *l)
 	struct process *p;
 
 	CAST_OBJ_NOTNULL(p, priv, PROCESS_MAGIC);
-	vtc_dump(p->vl, 4, p->cur_fd, l, -1);
+	vtc_dump(p->vl, 4, "output", l, -1);
 	return (0);
 }
 
@@ -182,8 +181,7 @@ process_stdout(const struct vev *ev, int what)
 
 	CAST_OBJ_NOTNULL(p, ev->priv, PROCESS_MAGIC);
 	(void)what;
-	p->cur_fd = "stdout";
-	i = read(p->fd_stdout, buf, sizeof buf);
+	i = read(p->fd_term, buf, sizeof buf);
 	if (i <= 0) {
 		vtc_log(p->vl, 4, "stdout read %d", i);
 		return (1);
@@ -207,18 +205,12 @@ process_stderr(const struct vev *ev, int what)
 
 	CAST_OBJ_NOTNULL(p, ev->priv, PROCESS_MAGIC);
 	(void)what;
-	p->cur_fd = "stderr";
 	i = read(p->fd_stderr, buf, sizeof buf);
 	if (i <= 0) {
 		vtc_log(p->vl, 4, "stderr read %d", i);
 		return (1);
 	}
-	if (p->log == 1)
-		(void)VLU_Feed(p->vlu_stderr, buf, i);
-	else if (p->log == 2)
-		vtc_dump(p->vl, 4, "stderr", buf, i);
-	else if (p->log == 3)
-		vtc_hexdump(p->vl, 4, "stderr", buf, i);
+	vtc_dump(p->vl, 4, "stderr", buf, i);
 	(void)write(p->f_stderr, buf, i);
 	return (0);
 }
@@ -244,7 +236,7 @@ process_thread(void *priv)
 
 	ev = VEV_Alloc();
 	AN(ev);
-	ev->fd = p->fd_stdout;
+	ev->fd = p->fd_term;
 	ev->fd_flags = VEV__RD | VEV__HUP | VEV__ERR;
 	ev->callback = process_stdout;
 	ev->priv = p;
@@ -313,10 +305,40 @@ process_thread(void *priv)
 }
 
 static void
+process_init_term(int fd)
+{
+	struct winsize ws;
+	struct termios tt;
+
+	memset(&ws, 0, sizeof ws);
+	ws.ws_row = 24;
+	ws.ws_col = 80;
+	AZ(ioctl(fd, TIOCSWINSZ, &ws));
+
+	memset(&tt, 0, sizeof tt);
+	tt.c_cflag = CREAD | CS8 | HUPCL;
+	tt.c_iflag = BRKINT | ICRNL | IMAXBEL | IXON | IXANY;
+	tt.c_lflag = ICANON | ISIG | IEXTEN;
+	tt.c_oflag = OPOST | ONLCR;
+	tt.c_ispeed = B9600;
+	tt.c_ospeed = B9600;
+	tt.c_cc[VEOF] = '\x04';			// CTRL-D
+	tt.c_cc[VERASE] = '\x08';		// CTRL-H (Backspace)
+	tt.c_cc[VKILL] = '\x15';		// CTRL-U
+	tt.c_cc[VINTR] = '\x03';		// CTRL-C
+	tt.c_cc[VQUIT] = '\x1c';		// CTRL-backslash
+	tt.c_cc[VMIN] = 1;
+
+	AZ(tcsetattr(fd, TCSAFLUSH, &tt));
+}
+
+static void
 process_start(struct process *p)
 {
 	struct vsb *cl;
-	int fd0[2], fd1[2], fd2[2];
+	int fd2[2];
+	int master, slave;
+	const char *slavename;
 
 	CHECK_OBJ_NOTNULL(p, PROCESS_MAGIC);
 	if (p->hasthread)
@@ -327,15 +349,25 @@ process_start(struct process *p)
 	cl = macro_expand(p->vl, p->spec);
 	AN(cl);
 
-	AZ(pipe(fd0));
-	AZ(pipe(fd1));
+	master = posix_openpt(O_RDWR|O_NOCTTY);
+	assert(master >= 0);
+	AZ(grantpt(master));
+	AZ(unlockpt(master));
+	slavename = ptsname(master);
+	AN(slavename);
+	slave = open(slavename, O_RDWR);
+	assert(slave >= 0);
+
+	process_init_term(slave);
+
 	AZ(pipe(fd2));
 
 	p->pid = fork();
 	assert(p->pid >= 0);
 	if (p->pid == 0) {
-		assert(dup2(fd0[0], STDIN_FILENO) == STDIN_FILENO);
-		assert(dup2(fd1[1], STDOUT_FILENO) == STDOUT_FILENO);
+		setenv("TERM", "adm3a", 1);
+		assert(dup2(slave, STDIN_FILENO) == STDIN_FILENO);
+		assert(dup2(slave, STDOUT_FILENO) == STDOUT_FILENO);
 		assert(dup2(fd2[1], STDERR_FILENO) == STDERR_FILENO);
 		VSUB_closefrom(STDERR_FILENO + 1);
 		AZ(setpgid(0, 0));
@@ -346,11 +378,9 @@ process_start(struct process *p)
 	vtc_log(p->vl, 3, "PID: %ld", (long)p->pid);
 	VSB_destroy(&cl);
 
-	closefd(&fd0[0]);
-	closefd(&fd1[1]);
+	closefd(&slave);
+	p->fd_term = master;
 	closefd(&fd2[1]);
-	p->fd_stdin = fd0[1];
-	p->fd_stdout = fd1[0];
 	p->fd_stderr = fd2[0];
 	macro_def(p->vl, p->name, "pid", "%ld", (long)p->pid);
 	macro_def(p->vl, p->name, "dir", "%s", p->dir);
@@ -401,6 +431,8 @@ process_kill(struct process *p, const char *sig)
 		j = SIGINT;
 	else if (!strcmp(sig, "KILL"))
 		j = SIGKILL;
+	else if (!strcmp(sig, "HUP"))
+		j = SIGHUP;
 	else if (*sig == '-')
 		j = strtoul(sig + 1, NULL, 10);
 	else
@@ -427,7 +459,7 @@ process_write(const struct process *p, const char *text)
 
 	len = strlen(text);
 	vtc_log(p->vl, 4, "Writing %d bytes", len);
-	r = write(p->fd_stdin, text, len);
+	r = write(p->fd_term, text, len);
 	if (r < 0)
 		vtc_fatal(p->vl, "Failed to write: %s (%d)",
 		    strerror(errno), errno);
@@ -440,16 +472,22 @@ process_close(struct process *p)
 	if (!p->hasthread)
 		vtc_fatal(p->vl, "Cannot close a non-running process");
 
-	AZ(pthread_mutex_lock(&p->mtx));
-	if (p->fd_stdin >= 0)
-		closefd(&p->fd_stdin);
-	AZ(pthread_mutex_unlock(&p->mtx));
+	process_kill(p, "HUP");
 }
 
 /* SECTION: process process
  *
- * Run a process in the background with stdout and stderr redirected to
- * ${pNAME_out} and ${pNAME_err}, both located in ${pNAME_dir}::
+ * Run a process with stdin+stdout on a pseudo-terminal and stderr on a pipe.
+ *
+ * Output from the pseudo-terminal is copied verbatim to ${pNAME_out},
+ * and the -log/-dump/-hexdump flags will also put it in the vtc-log.
+ *
+ * The pseudo-terminal is not in ECHO mode, but if the programs run set
+ * it to ECHO mode ("stty sane") any input sent to the process will also
+ * appear in this stream because of the ECHO.
+ *
+ * Output from the stderr-pipe is copied verbatim to ${pNAME_err}, and
+ * is always included in the vtc_log.
  *
  *	process pNAME SPEC [-log] [-dump] [-hexdump] [-expect-exit N]
  *		[-start] [-run]
@@ -463,13 +501,13 @@ process_close(struct process *p)
  *	The command(s) to run in this process.
  *
  * \-hexdump
- *	Log stdout/stderr with vtc_hexdump(). Must be before -start/-run.
+ *	Log output with vtc_hexdump(). Must be before -start/-run.
  *
  * \-dump
- *	Log stdout/stderr with vtc_dump(). Must be before -start/-run.
+ *	Log output with vtc_dump(). Must be before -start/-run.
  *
  * \-log
- *	Log stdout/stderr with VLU/vtc_log(). Must be before -start/-run.
+ *	Log output with VLU/vtc_log(). Must be before -start/-run.
  *
  * \-start
  *	Start the process.
@@ -520,7 +558,7 @@ process_close(struct process *p)
  *	Same as -write followed by a newline (\\n).
  *
  * \-close
- *	Close the process' stdin.
+ *	Alias for "-kill HUP"
  *
  */
 

From phk at FreeBSD.org  Thu Jan 11 10:25:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 11 Jan 2018 10:25:07 +0000 (UTC)
Subject: [master] ddc1100 Use cfset[io]speed() because of Solaris...
Message-ID: <20180111102507.ECA9E6570C@lists.varnish-cache.org>


commit ddc110031a6c719d3711f587a5d79ceee9742aa2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 11 10:23:56 2018 +0000

    Use cfset[io]speed() because of Solaris...

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index f6a8bdb..d6db686 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -320,8 +320,8 @@ process_init_term(int fd)
 	tt.c_iflag = BRKINT | ICRNL | IMAXBEL | IXON | IXANY;
 	tt.c_lflag = ICANON | ISIG | IEXTEN;
 	tt.c_oflag = OPOST | ONLCR;
-	tt.c_ispeed = B9600;
-	tt.c_ospeed = B9600;
+	cfsetispeed(&tt, B9600);
+	cfsetospeed(&tt, B9600);
 	tt.c_cc[VEOF] = '\x04';			// CTRL-D
 	tt.c_cc[VERASE] = '\x08';		// CTRL-H (Backspace)
 	tt.c_cc[VKILL] = '\x15';		// CTRL-U

From phk at FreeBSD.org  Thu Jan 11 11:08:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 11 Jan 2018 11:08:07 +0000 (UTC)
Subject: [master] 895af0a Add a feature test "term" for having a adm3a
 term{info|cap} entry
Message-ID: <20180111110807.7A8599175D@lists.varnish-cache.org>


commit 895af0a667e694ce49efb85c6d498e179cbefe0f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 11 10:41:16 2018 +0000

    Add a feature test "term" for having a adm3a term{info|cap} entry

diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 02e8da3..11c1ca5 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -712,12 +712,32 @@ cmd_delay(CMD_ARGS)
  * ignore_unknown_macro
  *        Do not fail the test if a string of the form ${...} is not
  *        recognized as a macro.
+ * term
+ *	  Support for ADM3A terminal
  *
  * Be careful with the last feature, because it may cause a test with a
  * misspelled macro to fail silently. You should only need it if you must
  * run a test with strings of the form "${...}".
  */
 
+static int
+test_term(struct vtclog *vl)
+{
+	FILE *p;
+	int a, b, c;
+
+	p = popen("tput -T adm3a clear 2>&1", "r");
+	if (p == NULL)
+		return (0);
+	a = fgetc(p);
+	b = fgetc(p);
+	c = pclose(p);
+	if (a == 0x1a && b == EOF && c == 0)
+		return (1);
+	vtc_log(vl, 3, "No adm3a terminfo entry. (insteall ncurses-term ?)");
+	return (0);
+}
+
 static void
 cmd_feature(CMD_ARGS)
 {
@@ -766,6 +786,7 @@ cmd_feature(CMD_ARGS)
 		FEATURE("user_varnish", getpwnam("varnish") != NULL);
 		FEATURE("user_vcache", getpwnam("vcache") != NULL);
 		FEATURE("group_varnish", getgrnam("varnish") != NULL);
+		FEATURE("term", test_term(vl));
 
 		if (!strcmp(*av, "disable_aslr")) {
 			good = 1;
diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index d6db686..81d18bc 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -320,8 +320,7 @@ process_init_term(int fd)
 	tt.c_iflag = BRKINT | ICRNL | IMAXBEL | IXON | IXANY;
 	tt.c_lflag = ICANON | ISIG | IEXTEN;
 	tt.c_oflag = OPOST | ONLCR;
-	cfsetispeed(&tt, B9600);
-	cfsetospeed(&tt, B9600);
+	AZ(cfsetspeed(&tt, B9600));
 	tt.c_cc[VEOF] = '\x04';			// CTRL-D
 	tt.c_cc[VERASE] = '\x08';		// CTRL-H (Backspace)
 	tt.c_cc[VKILL] = '\x15';		// CTRL-U
@@ -365,7 +364,7 @@ process_start(struct process *p)
 	p->pid = fork();
 	assert(p->pid >= 0);
 	if (p->pid == 0) {
-		setenv("TERM", "adm3a", 1);
+		AZ(setenv("TERM", "adm3a", 1));
 		assert(dup2(slave, STDIN_FILENO) == STDIN_FILENO);
 		assert(dup2(slave, STDOUT_FILENO) == STDOUT_FILENO);
 		assert(dup2(fd2[1], STDERR_FILENO) == STDERR_FILENO);

From phk at FreeBSD.org  Thu Jan 11 11:08:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 11 Jan 2018 11:08:07 +0000 (UTC)
Subject: [master] d0805ed Run the utilities in curses-mode.
Message-ID: <20180111110807.8CAEC9175F@lists.varnish-cache.org>


commit d0805ed9673c9d539e43192aea8267eec7cb6d28
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 11 11:06:34 2018 +0000

    Run the utilities in curses-mode.

diff --git a/bin/varnishtest/tests/u00009.vtc b/bin/varnishtest/tests/u00009.vtc
new file mode 100644
index 0000000..7a5985a
--- /dev/null
+++ b/bin/varnishtest/tests/u00009.vtc
@@ -0,0 +1,25 @@
+varnishtest	"trivial run of varnihist in curses mode"
+
+feature term
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {} -start
+
+process p1 -dump {varnishhist -n ${v1_name}} -start
+
+delay 3
+
+client c1 {
+	txreq
+	rxresp
+} -run
+
+delay 3
+
+process p1 -write {q}
+
+process p1 -wait
diff --git a/bin/varnishtest/tests/u00010.vtc b/bin/varnishtest/tests/u00010.vtc
new file mode 100644
index 0000000..6a5c0c5
--- /dev/null
+++ b/bin/varnishtest/tests/u00010.vtc
@@ -0,0 +1,25 @@
+varnishtest	"trivial run of varnishtop in curses mode"
+
+feature term
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {} -start
+
+process p1 -dump {varnishtop -n ${v1_name}} -start
+
+delay 3
+
+client c1 {
+	txreq
+	rxresp
+} -run
+
+delay 3
+
+process p1 -write {q}
+
+process p1 -wait
diff --git a/bin/varnishtest/tests/u00011.vtc b/bin/varnishtest/tests/u00011.vtc
new file mode 100644
index 0000000..3f0f6dd
--- /dev/null
+++ b/bin/varnishtest/tests/u00011.vtc
@@ -0,0 +1,29 @@
+varnishtest	"trivial run of varnishadm in curses mode"
+
+feature term
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {} -start
+
+process p1 -dump {varnishadm -n ${v1_name}} -start
+
+delay 2
+
+client c1 {
+	txreq
+	rxresp
+} -run
+
+delay 2
+
+process p1 -writeln {panic.show}
+
+delay 2
+
+process p1 -writeln {quit}
+
+process p1 -wait
diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index 81d18bc..86d4708 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -365,6 +365,7 @@ process_start(struct process *p)
 	assert(p->pid >= 0);
 	if (p->pid == 0) {
 		AZ(setenv("TERM", "adm3a", 1));
+		AZ(unsetenv("TERMCAP"));
 		assert(dup2(slave, STDIN_FILENO) == STDIN_FILENO);
 		assert(dup2(slave, STDOUT_FILENO) == STDOUT_FILENO);
 		assert(dup2(fd2[1], STDERR_FILENO) == STDERR_FILENO);

From phk at FreeBSD.org  Thu Jan 11 11:09:05 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 11 Jan 2018 11:09:05 +0000 (UTC)
Subject: [master] 7e07428 neuter assert for now.
Message-ID: <20180111110905.F3BAC91B92@lists.varnish-cache.org>


commit 7e074289d260e64b50bc776ed5bb56a8c9e47257
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 11 11:08:00 2018 +0000

    neuter assert for now.

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index 86d4708..f31bbd6 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -313,7 +313,7 @@ process_init_term(int fd)
 	memset(&ws, 0, sizeof ws);
 	ws.ws_row = 24;
 	ws.ws_col = 80;
-	AZ(ioctl(fd, TIOCSWINSZ, &ws));
+	(void)(ioctl(fd, TIOCSWINSZ, &ws));	// assert fails ?
 
 	memset(&tt, 0, sizeof tt);
 	tt.c_cflag = CREAD | CS8 | HUPCL;

From phk at FreeBSD.org  Thu Jan 11 11:22:04 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 11 Jan 2018 11:22:04 +0000 (UTC)
Subject: [master] c462a96 Solaris again...
Message-ID: <20180111112205.0186E9325A@lists.varnish-cache.org>


commit c462a96b12360c9e51dfdfccad4db59a6e352f30
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 11 11:21:23 2018 +0000

    Solaris again...

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index f31bbd6..9a3cff4 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -320,7 +320,8 @@ process_init_term(int fd)
 	tt.c_iflag = BRKINT | ICRNL | IMAXBEL | IXON | IXANY;
 	tt.c_lflag = ICANON | ISIG | IEXTEN;
 	tt.c_oflag = OPOST | ONLCR;
-	AZ(cfsetspeed(&tt, B9600));
+	AZ(cfsetispeed(&tt, B9600));
+	AZ(cfsetospeed(&tt, B9600));
 	tt.c_cc[VEOF] = '\x04';			// CTRL-D
 	tt.c_cc[VERASE] = '\x08';		// CTRL-H (Backspace)
 	tt.c_cc[VKILL] = '\x15';		// CTRL-U

From phk at FreeBSD.org  Thu Jan 11 12:06:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 11 Jan 2018 12:06:07 +0000 (UTC)
Subject: [master] aab0966 Drop all terminfo related asserts for now,
 report on vtc_log instead.
Message-ID: <20180111120607.1DAE194629@lists.varnish-cache.org>


commit aab09666794f8ab7ca4463a0677a009e6cc8dd2e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 11 12:05:28 2018 +0000

    Drop all terminfo related asserts for now, report on vtc_log instead.

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index 9a3cff4..c11799f 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -305,23 +305,30 @@ process_thread(void *priv)
 }
 
 static void
-process_init_term(int fd)
+process_init_term(struct process *p, int fd)
 {
 	struct winsize ws;
 	struct termios tt;
+	int i;
 
 	memset(&ws, 0, sizeof ws);
 	ws.ws_row = 24;
 	ws.ws_col = 80;
-	(void)(ioctl(fd, TIOCSWINSZ, &ws));	// assert fails ?
+	i = ioctl(fd, TIOCSWINSZ, &ws);
+	if (i)
+		vtc_log(p->vl, 4, "TIOCWINSZ %d %s", i, strerror(errno));
 
 	memset(&tt, 0, sizeof tt);
 	tt.c_cflag = CREAD | CS8 | HUPCL;
 	tt.c_iflag = BRKINT | ICRNL | IMAXBEL | IXON | IXANY;
 	tt.c_lflag = ICANON | ISIG | IEXTEN;
 	tt.c_oflag = OPOST | ONLCR;
-	AZ(cfsetispeed(&tt, B9600));
-	AZ(cfsetospeed(&tt, B9600));
+	i = cfsetispeed(&tt, B9600);
+	if (i)
+		vtc_log(p->vl, 4, "cfsetispeed %d %s", i, strerror(errno));
+	i = cfsetospeed(&tt, B9600);
+	if (i)
+		vtc_log(p->vl, 4, "cfsetospeed %d %s", i, strerror(errno));
 	tt.c_cc[VEOF] = '\x04';			// CTRL-D
 	tt.c_cc[VERASE] = '\x08';		// CTRL-H (Backspace)
 	tt.c_cc[VKILL] = '\x15';		// CTRL-U
@@ -329,7 +336,8 @@ process_init_term(int fd)
 	tt.c_cc[VQUIT] = '\x1c';		// CTRL-backslash
 	tt.c_cc[VMIN] = 1;
 
-	AZ(tcsetattr(fd, TCSAFLUSH, &tt));
+	i = tcsetattr(fd, TCSAFLUSH, &tt);
+		vtc_log(p->vl, 4, "TCSAFLUSH %d %s", i, strerror(errno));
 }
 
 static void
@@ -358,7 +366,7 @@ process_start(struct process *p)
 	slave = open(slavename, O_RDWR);
 	assert(slave >= 0);
 
-	process_init_term(slave);
+	process_init_term(p, slave);
 
 	AZ(pipe(fd2));
 

From phk at FreeBSD.org  Thu Jan 11 12:31:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 11 Jan 2018 12:31:06 +0000 (UTC)
Subject: [master] 45db094 Change subtest 2 to be CTRL-D
Message-ID: <20180111123106.8D37F95044@lists.varnish-cache.org>


commit 45db0940802055092355ee9e43275315e43387df
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 11 12:29:58 2018 +0000

    Change subtest 2 to be CTRL-D

diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc
index b590e67..b6b4df3 100644
--- a/bin/varnishtest/tests/u00000.vtc
+++ b/bin/varnishtest/tests/u00000.vtc
@@ -11,7 +11,7 @@ shell "test -f ${p1_err} -a ! -s ${p1_err}"
 process p2 -log "cat" -start
 process p2 -writeln "bar"
 delay 0.5
-process p2 -close
+process p2 -write "\x04"
 process p2 -wait
 shell "grep -q bar ${p2_out}"
 shell "test -f ${p2_err} -a ! -s ${p2_err}"
diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index c11799f..d7b94e6 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -337,6 +337,7 @@ process_init_term(struct process *p, int fd)
 	tt.c_cc[VMIN] = 1;
 
 	i = tcsetattr(fd, TCSAFLUSH, &tt);
+	if (i)
 		vtc_log(p->vl, 4, "TCSAFLUSH %d %s", i, strerror(errno));
 }
 

From dridi.boukelmoune at gmail.com  Thu Jan 11 12:41:06 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Thu, 11 Jan 2018 12:41:06 +0000 (UTC)
Subject: [master] 0f65eb1 Future-proof comment
Message-ID: <20180111124106.4066F95437@lists.varnish-cache.org>


commit 0f65eb1532a11dcce6a128e35142d0dd64c932c8
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Jan 11 13:39:17 2018 +0100

    Future-proof comment

diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 11c1ca5..2f7ff96 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -715,7 +715,7 @@ cmd_delay(CMD_ARGS)
  * term
  *	  Support for ADM3A terminal
  *
- * Be careful with the last feature, because it may cause a test with a
+ * Be careful with ignore_unknown_macro, because it may cause a test with a
  * misspelled macro to fail silently. You should only need it if you must
  * run a test with strings of the form "${...}".
  */

From nils.goroll at uplex.de  Thu Jan 11 13:33:05 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 11 Jan 2018 13:33:05 +0000 (UTC)
Subject: [master] 0773342 (struct director).admin_health needs initialization
Message-ID: <20180111133305.CA641963FD@lists.varnish-cache.org>


commit 0773342679f5cf4e696162ce1e20069d2a3c84b6
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Jan 11 14:14:46 2018 +0100

    (struct director).admin_health needs initialization
    
    ... otherwise we will hit the WRONG("Wrong admin health") in VDI_Healthy()
    for std.healthy().
    
    Ref: https://github.com/nigoroll/varnish-modules/issues/5

diff --git a/bin/varnishtest/tests/d00000.vtc b/bin/varnishtest/tests/d00000.vtc
index 507e9c7..3ca3528 100644
--- a/bin/varnishtest/tests/d00000.vtc
+++ b/bin/varnishtest/tests/d00000.vtc
@@ -22,6 +22,7 @@ server s4 {
 
 varnish v1 -vcl+backend {
 	import directors;
+	import std;
 
 	sub vcl_init {
 		new rr = directors.round_robin();
@@ -40,6 +41,10 @@ varnish v1 -vcl+backend {
 		}
 	}
 
+	sub vcl_deliver {
+		set resp.http.healthy = std.healthy(rr.backend());
+	}
+
 	sub vcl_backend_fetch {
 		set bereq.backend = rr.backend();
 	}
@@ -50,15 +55,19 @@ client c1 {
 	txreq -url "/foo1"
 	rxresp
 	expect resp.bodylen == 1
+	expect resp.http.healthy == "true"
 	txreq -url "/foo2"
 	rxresp
 	expect resp.bodylen == 2
+	expect resp.http.healthy == "true"
 	txreq -url "/foo3"
 	rxresp
 	expect resp.bodylen == 3
+	expect resp.http.healthy == "true"
 	txreq -url "/foo4"
 	rxresp
 	expect resp.bodylen == 4
+	expect resp.http.healthy == "true"
 } -run
 
 server s1 -start
@@ -69,9 +78,11 @@ client c2 {
 	txreq -url "/foo11"
 	rxresp
 	expect resp.bodylen == 1
+	expect resp.http.healthy == "true"
 	txreq -url "/foo22"
 	rxresp
 	expect resp.bodylen == 2
+	expect resp.http.healthy == "true"
 } -run
 
 server s4 -start
@@ -83,4 +94,5 @@ client c3 {
 	txreq -url "/foo31"
 	rxresp
 	expect resp.bodylen == 4
+	expect resp.http.healthy == "true"
 } -run
diff --git a/doc/changes.rst b/doc/changes.rst
index b09221b..cb9fe31 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -104,6 +104,11 @@ C APIs (for vmod and utility authors)
 * Added ``VRT_blob()`` utility function to create a blob as a copy
   of some chunk of data on the workspace.
 
+* Directors now have their own admin health information and always need to
+  have the ``(struct director).admin_health`` initialized to
+  ``VDI_AH_*`` (usually ``VDI_AH_HEALTHY``).
+
+  XXX improve / more details
 
 Other changes relevant for VMODs
 --------------------------------
diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c
index a334635..6a06412 100644
--- a/lib/libvmod_directors/vdir.c
+++ b/lib/libvmod_directors/vdir.c
@@ -71,6 +71,7 @@ vdir_new(struct vdir **vdp, const char *name, const char *vcl_name,
 	vd->dir->priv = priv;
 	vd->dir->healthy = healthy;
 	vd->dir->resolve = resolve;
+	vd->dir->admin_health = VDI_AH_HEALTHY;
 	vd->vbm = vbit_new(8);
 	AN(vd->vbm);
 }

From nils.goroll at uplex.de  Thu Jan 11 17:09:11 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 11 Jan 2018 17:09:11 +0000 (UTC)
Subject: [master] a11dbaa fix typos
Message-ID: <20180111170911.45DDC9AA99@lists.varnish-cache.org>


commit a11dbaaf2eccddcb05e2347e8147795090684252
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Jan 11 18:01:30 2018 +0100

    fix typos

diff --git a/bin/varnishtest/tests/u00009.vtc b/bin/varnishtest/tests/u00009.vtc
index 7a5985a..e3da901 100644
--- a/bin/varnishtest/tests/u00009.vtc
+++ b/bin/varnishtest/tests/u00009.vtc
@@ -1,4 +1,4 @@
-varnishtest	"trivial run of varnihist in curses mode"
+varnishtest	"trivial run of varnishhist in curses mode"
 
 feature term
 
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 2f7ff96..9b21c04 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -734,7 +734,7 @@ test_term(struct vtclog *vl)
 	c = pclose(p);
 	if (a == 0x1a && b == EOF && c == 0)
 		return (1);
-	vtc_log(vl, 3, "No adm3a terminfo entry. (insteall ncurses-term ?)");
+	vtc_log(vl, 3, "No adm3a terminfo entry. (install ncurses-term ?)");
 	return (0);
 }
 

From phk at FreeBSD.org  Thu Jan 11 21:55:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 11 Jan 2018 21:55:07 +0000 (UTC)
Subject: [master] 65e9652 Reduce noise during teardown
Message-ID: <20180111215507.1D875A520E@lists.varnish-cache.org>


commit 65e9652425ff7e1d573a0581185b202dc2259dc2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 11 13:39:56 2018 +0000

    Reduce noise during teardown

diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 11c1ca5..1ce6d0a 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -172,7 +172,8 @@ macro_undef(struct vtclog *vl, const char *instance, const char *name)
 		if (!strcmp(name, m->name))
 			break;
 	if (m != NULL) {
-		vtc_log(vl, 4, "macro undef %s", name);
+		if (!vtc_stop)
+			vtc_log(vl, 4, "macro undef %s", name);
 		VTAILQ_REMOVE(&macro_list, m, list);
 		free(m->name);
 		free(m->val);

From phk at FreeBSD.org  Thu Jan 11 21:55:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 11 Jan 2018 21:55:07 +0000 (UTC)
Subject: [master] 80d571c Merge branch 'master' of
 github.com:varnishcache/varnish-cache
Message-ID: <20180111215507.31CAEA5211@lists.varnish-cache.org>


commit 80d571c98d68b8d07db31462978ebe51cc308168
Merge: 65e9652 a11dbaa
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 11 21:46:45 2018 +0000

    Merge branch 'master' of github.com:varnishcache/varnish-cache

From phk at FreeBSD.org  Thu Jan 11 21:55:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 11 Jan 2018 21:55:07 +0000 (UTC)
Subject: [master] 2ec9691 Integrate Geoff's Solaris patches from #2537
Message-ID: <20180111215507.498C3A5214@lists.varnish-cache.org>


commit 2ec9691b86dbf37c1148d323f9fefced48eeba09
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 11 21:53:00 2018 +0000

    Integrate Geoff's Solaris patches from #2537
    
    I owe the next round Geoff.

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index d7b94e6..b8573b7 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -38,6 +38,9 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#ifdef _sun
+#include <stropts.h>
+#endif
 #include <termios.h>
 #include <unistd.h>
 
@@ -334,7 +337,6 @@ process_init_term(struct process *p, int fd)
 	tt.c_cc[VKILL] = '\x15';		// CTRL-U
 	tt.c_cc[VINTR] = '\x03';		// CTRL-C
 	tt.c_cc[VQUIT] = '\x1c';		// CTRL-backslash
-	tt.c_cc[VMIN] = 1;
 
 	i = tcsetattr(fd, TCSAFLUSH, &tt);
 	if (i)
@@ -366,6 +368,12 @@ process_start(struct process *p)
 	AN(slavename);
 	slave = open(slavename, O_RDWR);
 	assert(slave >= 0);
+#ifdef __sun
+	if (ioctl(slave, I_PUSH, "ptem"))
+		vtc_log(p->vl, 4, "PUSH ptem: %s", strerror(errno));
+	if (ioctl(slave, I_PUSH, "ldterm"))
+		vtc_log(p->vl, 4, "PUSH ldterm: %s", strerror(errno));
+#endif
 
 	process_init_term(p, slave);
 

From phk at FreeBSD.org  Thu Jan 11 22:09:04 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 11 Jan 2018 22:09:04 +0000 (UTC)
Subject: [master] 84d92b4 Missed an underscore
Message-ID: <20180111220904.C2885A5A54@lists.varnish-cache.org>


commit 84d92b486b9d2d9a74f1c3f49457e44bbe14c674
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 11 22:07:50 2018 +0000

    Missed an underscore

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index b8573b7..84b31cc 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -38,8 +38,8 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#ifdef _sun
-#include <stropts.h>
+#ifdef __sun
+#  include <stropts.h>
 #endif
 #include <termios.h>
 #include <unistd.h>

From phk at FreeBSD.org  Fri Jan 12 09:24:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 12 Jan 2018 09:24:08 +0000 (UTC)
Subject: [master] 086a141 Add a very rudimentary 'ansi' terminal emulation.
Message-ID: <20180112092408.3CF4FB4A28@lists.varnish-cache.org>


commit 086a141b77ce6bb9e064f456f66c15828ca8e838
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 12 09:23:19 2018 +0000

    Add a very rudimentary 'ansi' terminal emulation.

diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc
index b6b4df3..ded88ab 100644
--- a/bin/varnishtest/tests/u00000.vtc
+++ b/bin/varnishtest/tests/u00000.vtc
@@ -30,4 +30,4 @@ delay 0.5
 process p4 -kill TERM
 process p4 -wait
 
-process p5 -log "stty -a ; sleep 1" -run
+process p5 "stty -a ; sleep 1" -run -screen_dump
diff --git a/bin/varnishtest/tests/u00009.vtc b/bin/varnishtest/tests/u00009.vtc
index e3da901..1e540bd 100644
--- a/bin/varnishtest/tests/u00009.vtc
+++ b/bin/varnishtest/tests/u00009.vtc
@@ -18,8 +18,8 @@ client c1 {
 	rxresp
 } -run
 
-delay 3
+delay 10
 
 process p1 -write {q}
 
-process p1 -wait
+process p1 -wait -screen_dump
diff --git a/bin/varnishtest/tests/u00010.vtc b/bin/varnishtest/tests/u00010.vtc
index 6a5c0c5..fbca6d5 100644
--- a/bin/varnishtest/tests/u00010.vtc
+++ b/bin/varnishtest/tests/u00010.vtc
@@ -18,8 +18,8 @@ client c1 {
 	rxresp
 } -run
 
-delay 3
+delay 10
 
 process p1 -write {q}
 
-process p1 -wait
+process p1 -wait -screen_dump
diff --git a/bin/varnishtest/tests/u00011.vtc b/bin/varnishtest/tests/u00011.vtc
index 3f0f6dd..4c29d24 100644
--- a/bin/varnishtest/tests/u00011.vtc
+++ b/bin/varnishtest/tests/u00011.vtc
@@ -22,8 +22,8 @@ delay 2
 
 process p1 -writeln {panic.show}
 
-delay 2
-
 process p1 -writeln {quit}
 
-process p1 -wait
+delay 10
+
+process p1 -wait -screen_dump
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index f507d19..6a2a476 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -725,17 +725,16 @@ static int
 test_term(struct vtclog *vl)
 {
 	FILE *p;
-	int a, b, c;
+	int a, b;
 
-	p = popen("tput -T adm3a clear 2>&1", "r");
+	p = popen("tput -T ansi clear 2>&1", "r");
 	if (p == NULL)
 		return (0);
 	a = fgetc(p);
 	b = fgetc(p);
-	c = pclose(p);
-	if (a == 0x1a && b == EOF && c == 0)
+	if (a == 0x1b && b == '[')
 		return (1);
-	vtc_log(vl, 3, "No adm3a terminfo entry. (install ncurses-term ?)");
+	vtc_log(vl, 3, "No 'ansi' terminfo entry.");
 	return (0);
 }
 
diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index 84b31cc..dde798b 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -24,6 +24,9 @@
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
+ *
+ * XXX:
+ *	-ignore-stderr (otherwise output to stderr is fail)
  */
 
 #include "config.h"
@@ -32,6 +35,7 @@
 #include <sys/resource.h>
 #include <sys/wait.h>
 
+#include <ctype.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <poll.h>
@@ -76,6 +80,15 @@ struct process {
 	pthread_t		tp;
 	unsigned		hasthread;
 	int			status;
+
+	unsigned		term_state;
+#define NTERMARG		10
+	int			term_arg[NTERMARG];
+	int			*term_ap;
+#define NLINES			24
+#define NCOLS			80
+	char			*term_c[NLINES];
+	int			term_x, term_y;
 };
 
 static VTAILQ_HEAD(, process)	processes =
@@ -100,6 +113,7 @@ process_new(const char *name)
 	struct process *p;
 	struct vsb *vsb;
 	char buf[1024];
+	int i;
 
 	ALLOC_OBJ(p, PROCESS_MAGIC);
 	AN(p);
@@ -120,6 +134,12 @@ process_new(const char *name)
 	p->fd_term = -1;
 
 	VTAILQ_INSERT_TAIL(&processes, p, list);
+	for (i = 0; i < NLINES; i++) {
+		p->term_c[i] = malloc(NCOLS + 1);
+		AN(p->term_c[i]);
+		memset(p->term_c[i], ' ', NCOLS);
+		p->term_c[i][NCOLS] = '\0';
+	}
 	return (p);
 }
 
@@ -162,7 +182,193 @@ process_undef(const struct process *p)
 }
 
 /**********************************************************************
- * Start the process thread
+ * Terminal emulation
+ */
+
+static void
+process_term_clear(const struct process *p)
+{
+	int i;
+
+	for (i = 0; i < NLINES; i++) {
+		memset(p->term_c[i], ' ', NCOLS);
+		p->term_c[i][NCOLS] = '\0';
+	}
+}
+
+static void
+process_screen_dump(const struct process *p)
+{
+	int i;
+
+	for (i = 0; i < NLINES; i++)
+		vtc_dump(p->vl, 3, "screen", p->term_c[i], NCOLS);
+}
+
+static void
+process_escape(struct process *p, int c, int n)
+{
+	int i;
+
+	for (i = 0; i < NTERMARG; i++)
+		if (!p->term_arg[i])
+			p->term_arg[i] = 1;
+	switch(c) {
+	case 'h':
+		if (p->term_arg[0] <= NLINES && p->term_arg[1] <= NCOLS) {
+			p->term_y = p->term_arg[0] - 1;
+			p->term_x = p->term_arg[1] - 1;
+		} else {
+			vtc_log(p->vl, 4, "ANSI H %d %d WRONG",
+			    p->term_arg[0], p->term_arg[1]);
+		}
+		break;
+	case 'j':
+		if (p->term_arg[0] == 2) {
+			process_term_clear(p);
+		} else {
+			vtc_log(p->vl, 4, "ANSI J %d", p->term_arg[0]);
+		}
+		break;
+	default:
+		for (i = 0; i < n; i++)
+			vtc_log(p->vl, 4, "ANSI arg %d",
+			    p->term_arg[i]);
+		vtc_log(p->vl, 4, "ANSI unk '%c'", c);
+		break;
+	}
+}
+
+static void
+process_scroll(struct process *p)
+{
+	int i;
+	char *l;
+
+	l = p->term_c[0];
+	for(i = 0; i < NLINES -1; i++)
+		p->term_c[i] = p->term_c[i + 1];
+	p->term_c[i] = l;
+	memset(l, ' ', NCOLS);
+}
+
+static void
+process_char(struct process *p, char c)
+{
+	assert(p->term_x < NCOLS);
+	assert(p->term_y < NLINES);
+	assert(p->term_state <= 3);
+	switch (c) {
+	case 0x00:
+		break;
+	case '\b':
+		if (p->term_x > 0)
+			p->term_x--;
+		break;
+	case '\t':
+		while(++p->term_x % 8)
+			continue;
+		if (p->term_x >= NCOLS) {
+			p->term_x = 0;
+			process_char(p, '\n');
+		}
+		break;
+	case '\n':
+		if (p->term_y == NLINES - 1)
+			process_scroll(p);
+		else
+			p->term_y++;
+		break;
+	case '\r':
+		p->term_x = 0;
+		break;
+	default:
+		if (c < ' ' || c > '~')
+			vtc_log(p->vl, 4, "ANSI CTRL 0x%02x", c);
+		else {
+			p->term_c[p->term_y][p->term_x] = c;
+			if (p->term_x == NCOLS - 1) {
+				p->term_x = 0;
+				process_char(p, '\n');
+			} else {
+				p->term_x++;
+			}
+		}
+	}
+}
+
+static void
+process_ansi(struct process *p, const char *b, const char *e)
+{
+
+	while (b < e) {
+		assert(p->term_x < NCOLS);
+		assert(p->term_y < NLINES);
+		assert(p->term_state <= 3);
+		switch (p->term_state) {
+		case 0:
+			if (*b == '\x1b')
+				p->term_state = 1;
+			else if (*(const uint8_t*)b == 0x9b)
+				p->term_state = 2;
+			else
+				process_char(p, *b);
+			b++;
+			break;
+		case 1:
+			if (*b++ == '[') {
+				p->term_state = 2;
+			} else {
+				vtc_log(p->vl, 4, "ANSI not [ 0x%x", b[-1]);
+				p->term_state = 0;
+			}
+			break;
+		case 2:
+			p->term_ap = p->term_arg;
+			memset(p->term_arg, 0, sizeof p->term_arg);
+			p->term_state = 3;
+			break;
+		case 3:
+			if (p->term_ap - p->term_arg >= NTERMARG) {
+				vtc_log(p->vl, 4, "ANSI too many ;");
+				p->term_state = 0;
+				b++;
+				continue;
+			}
+			if (isdigit(*b)) {
+				*p->term_ap *= 10;
+				*p->term_ap += *b++ - '0';
+				continue;
+			}
+			if (*b == ';') {
+				p->term_ap++;
+				p->term_state = 3;
+				b++;
+				continue;
+			}
+			if (islower(*b)) {
+				process_escape(p, *b++,
+				    p->term_ap -  p->term_arg);
+				p->term_state = 2;
+			} else if (isupper(*b)) {
+				process_escape(p, tolower(*b++),
+				    p->term_ap -  p->term_arg);
+				p->term_ap = p->term_arg;
+				p->term_state = 0;
+			} else {
+				vtc_log(p->vl, 4, "ANSI non-letter %c", *b);
+				p->term_state = 0;
+				b++;
+			}
+			break;
+		default:
+			WRONG("Wrong ansi state");
+		}
+	}
+}
+
+/**********************************************************************
+ * Data stream handling
  */
 
 static int
@@ -196,6 +402,7 @@ process_stdout(const struct vev *ev, int what)
 	else if (p->log == 3)
 		vtc_hexdump(p->vl, 4, "stdout", buf, i);
 	(void)write(p->f_stdout, buf, i);
+	process_ansi(p, buf, buf + i);
 	return (0);
 }
 
@@ -343,6 +550,10 @@ process_init_term(struct process *p, int fd)
 		vtc_log(p->vl, 4, "TCSAFLUSH %d %s", i, strerror(errno));
 }
 
+/**********************************************************************
+ * Start the process thread
+ */
+
 static void
 process_start(struct process *p)
 {
@@ -382,7 +593,7 @@ process_start(struct process *p)
 	p->pid = fork();
 	assert(p->pid >= 0);
 	if (p->pid == 0) {
-		AZ(setenv("TERM", "adm3a", 1));
+		AZ(setenv("TERM", "ansi", 1));
 		AZ(unsetenv("TERMCAP"));
 		assert(dup2(slave, STDIN_FILENO) == STDIN_FILENO);
 		assert(dup2(slave, STDOUT_FILENO) == STDOUT_FILENO);
@@ -578,6 +789,9 @@ process_close(struct process *p)
  * \-close
  *	Alias for "-kill HUP"
  *
+ * \-screen_dump
+ *	Dump the virtual screen into vtc_log
+ *
  */
 
 void
@@ -680,6 +894,10 @@ cmd_process(CMD_ARGS)
 			av++;
 			continue;
 		}
+		if (!strcmp(*av, "-screen_dump")) {
+			process_screen_dump(p);
+			continue;
+		}
 		if (!strcmp(*av, "-close")) {
 			process_close(p);
 			continue;

From phk at FreeBSD.org  Fri Jan 12 16:43:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 12 Jan 2018 16:43:08 +0000 (UTC)
Subject: [master] 909ce71 Split the terminal emulation into its own source
 file.
Message-ID: <20180112164308.1516065BCD@lists.varnish-cache.org>


commit 909ce71834084299f38ef8225c00392e0fd0c359
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 12 16:18:07 2018 +0000

    Split the terminal emulation into its own source file.
    
    Also: Make ansi-sequence problems fatal, and render unknown control
    chars as '?'

diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am
index 5ffbf79..974aaf9 100644
--- a/bin/varnishtest/Makefile.am
+++ b/bin/varnishtest/Makefile.am
@@ -48,6 +48,7 @@ varnishtest_SOURCES = \
 		vtc_proxy.c \
 		vtc_server.c \
 		vtc_subr.c \
+		vtc_term.c \
 		vtc_varnish.c
 
 varnishtest_LDADD = \
diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc
index ded88ab..0b33c36 100644
--- a/bin/varnishtest/tests/u00000.vtc
+++ b/bin/varnishtest/tests/u00000.vtc
@@ -28,6 +28,6 @@ process p4 -hexdump "cat" -start
 process p4 -writeln "b\001z"
 delay 0.5
 process p4 -kill TERM
-process p4 -wait
+process p4 -wait -screen_dump
 
 process p5 "stty -a ; sleep 1" -run -screen_dump
diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index af89358..6efef45 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -137,3 +137,8 @@ void b64_settings(const struct http *hp, const char *s);
 struct vsb *vtc_hex_to_bin(struct vtclog *vl, const char *arg);
 void vtc_expect(struct vtclog *, const char *, const char *, const char *,
     const char *, const char *);
+
+/* vtc_term.c */
+struct term *Term_New(struct vtclog *);
+void Term_Feed(struct term *, const char *, const char *);
+void Term_Dump(const struct term *);
diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index dde798b..6d55975 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -81,14 +81,8 @@ struct process {
 	unsigned		hasthread;
 	int			status;
 
-	unsigned		term_state;
-#define NTERMARG		10
-	int			term_arg[NTERMARG];
-	int			*term_ap;
-#define NLINES			24
-#define NCOLS			80
-	char			*term_c[NLINES];
-	int			term_x, term_y;
+	struct term		*term;
+
 };
 
 static VTAILQ_HEAD(, process)	processes =
@@ -113,7 +107,6 @@ process_new(const char *name)
 	struct process *p;
 	struct vsb *vsb;
 	char buf[1024];
-	int i;
 
 	ALLOC_OBJ(p, PROCESS_MAGIC);
 	AN(p);
@@ -134,12 +127,8 @@ process_new(const char *name)
 	p->fd_term = -1;
 
 	VTAILQ_INSERT_TAIL(&processes, p, list);
-	for (i = 0; i < NLINES; i++) {
-		p->term_c[i] = malloc(NCOLS + 1);
-		AN(p->term_c[i]);
-		memset(p->term_c[i], ' ', NCOLS);
-		p->term_c[i][NCOLS] = '\0';
-	}
+	p->term = Term_New(p->vl);
+	AN(p->term);
 	return (p);
 }
 
@@ -182,192 +171,6 @@ process_undef(const struct process *p)
 }
 
 /**********************************************************************
- * Terminal emulation
- */
-
-static void
-process_term_clear(const struct process *p)
-{
-	int i;
-
-	for (i = 0; i < NLINES; i++) {
-		memset(p->term_c[i], ' ', NCOLS);
-		p->term_c[i][NCOLS] = '\0';
-	}
-}
-
-static void
-process_screen_dump(const struct process *p)
-{
-	int i;
-
-	for (i = 0; i < NLINES; i++)
-		vtc_dump(p->vl, 3, "screen", p->term_c[i], NCOLS);
-}
-
-static void
-process_escape(struct process *p, int c, int n)
-{
-	int i;
-
-	for (i = 0; i < NTERMARG; i++)
-		if (!p->term_arg[i])
-			p->term_arg[i] = 1;
-	switch(c) {
-	case 'h':
-		if (p->term_arg[0] <= NLINES && p->term_arg[1] <= NCOLS) {
-			p->term_y = p->term_arg[0] - 1;
-			p->term_x = p->term_arg[1] - 1;
-		} else {
-			vtc_log(p->vl, 4, "ANSI H %d %d WRONG",
-			    p->term_arg[0], p->term_arg[1]);
-		}
-		break;
-	case 'j':
-		if (p->term_arg[0] == 2) {
-			process_term_clear(p);
-		} else {
-			vtc_log(p->vl, 4, "ANSI J %d", p->term_arg[0]);
-		}
-		break;
-	default:
-		for (i = 0; i < n; i++)
-			vtc_log(p->vl, 4, "ANSI arg %d",
-			    p->term_arg[i]);
-		vtc_log(p->vl, 4, "ANSI unk '%c'", c);
-		break;
-	}
-}
-
-static void
-process_scroll(struct process *p)
-{
-	int i;
-	char *l;
-
-	l = p->term_c[0];
-	for(i = 0; i < NLINES -1; i++)
-		p->term_c[i] = p->term_c[i + 1];
-	p->term_c[i] = l;
-	memset(l, ' ', NCOLS);
-}
-
-static void
-process_char(struct process *p, char c)
-{
-	assert(p->term_x < NCOLS);
-	assert(p->term_y < NLINES);
-	assert(p->term_state <= 3);
-	switch (c) {
-	case 0x00:
-		break;
-	case '\b':
-		if (p->term_x > 0)
-			p->term_x--;
-		break;
-	case '\t':
-		while(++p->term_x % 8)
-			continue;
-		if (p->term_x >= NCOLS) {
-			p->term_x = 0;
-			process_char(p, '\n');
-		}
-		break;
-	case '\n':
-		if (p->term_y == NLINES - 1)
-			process_scroll(p);
-		else
-			p->term_y++;
-		break;
-	case '\r':
-		p->term_x = 0;
-		break;
-	default:
-		if (c < ' ' || c > '~')
-			vtc_log(p->vl, 4, "ANSI CTRL 0x%02x", c);
-		else {
-			p->term_c[p->term_y][p->term_x] = c;
-			if (p->term_x == NCOLS - 1) {
-				p->term_x = 0;
-				process_char(p, '\n');
-			} else {
-				p->term_x++;
-			}
-		}
-	}
-}
-
-static void
-process_ansi(struct process *p, const char *b, const char *e)
-{
-
-	while (b < e) {
-		assert(p->term_x < NCOLS);
-		assert(p->term_y < NLINES);
-		assert(p->term_state <= 3);
-		switch (p->term_state) {
-		case 0:
-			if (*b == '\x1b')
-				p->term_state = 1;
-			else if (*(const uint8_t*)b == 0x9b)
-				p->term_state = 2;
-			else
-				process_char(p, *b);
-			b++;
-			break;
-		case 1:
-			if (*b++ == '[') {
-				p->term_state = 2;
-			} else {
-				vtc_log(p->vl, 4, "ANSI not [ 0x%x", b[-1]);
-				p->term_state = 0;
-			}
-			break;
-		case 2:
-			p->term_ap = p->term_arg;
-			memset(p->term_arg, 0, sizeof p->term_arg);
-			p->term_state = 3;
-			break;
-		case 3:
-			if (p->term_ap - p->term_arg >= NTERMARG) {
-				vtc_log(p->vl, 4, "ANSI too many ;");
-				p->term_state = 0;
-				b++;
-				continue;
-			}
-			if (isdigit(*b)) {
-				*p->term_ap *= 10;
-				*p->term_ap += *b++ - '0';
-				continue;
-			}
-			if (*b == ';') {
-				p->term_ap++;
-				p->term_state = 3;
-				b++;
-				continue;
-			}
-			if (islower(*b)) {
-				process_escape(p, *b++,
-				    p->term_ap -  p->term_arg);
-				p->term_state = 2;
-			} else if (isupper(*b)) {
-				process_escape(p, tolower(*b++),
-				    p->term_ap -  p->term_arg);
-				p->term_ap = p->term_arg;
-				p->term_state = 0;
-			} else {
-				vtc_log(p->vl, 4, "ANSI non-letter %c", *b);
-				p->term_state = 0;
-				b++;
-			}
-			break;
-		default:
-			WRONG("Wrong ansi state");
-		}
-	}
-}
-
-/**********************************************************************
  * Data stream handling
  */
 
@@ -402,7 +205,7 @@ process_stdout(const struct vev *ev, int what)
 	else if (p->log == 3)
 		vtc_hexdump(p->vl, 4, "stdout", buf, i);
 	(void)write(p->f_stdout, buf, i);
-	process_ansi(p, buf, buf + i);
+	Term_Feed(p->term, buf, buf + i);
 	return (0);
 }
 
@@ -895,7 +698,7 @@ cmd_process(CMD_ARGS)
 			continue;
 		}
 		if (!strcmp(*av, "-screen_dump")) {
-			process_screen_dump(p);
+			Term_Dump(p->term);
 			continue;
 		}
 		if (!strcmp(*av, "-close")) {
diff --git a/bin/varnishtest/vtc_term.c b/bin/varnishtest/vtc_term.c
new file mode 100644
index 0000000..34caa30
--- /dev/null
+++ b/bin/varnishtest/vtc_term.c
@@ -0,0 +1,244 @@
+/*-
+ * Copyright (c) 2018 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * A trivial ANSI terminal emulation
+ */
+
+#include "config.h"
+
+#include <ctype.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "vtc.h"
+
+struct term {
+	unsigned		magic;
+#define TERM_MAGIC		0x1c258f0f
+
+	struct vtclog		*vl;
+	unsigned		state;
+#define NTERMARG		10
+	int			arg[NTERMARG];
+	int			*argp;
+	int			nlin;
+	int			ncol;
+	char			**vram;
+	int			col;
+	int			line;
+};
+
+static void
+term_clear(const struct term *tp)
+{
+	int i;
+
+	for (i = 0; i < tp->nlin; i++) {
+		memset(tp->vram[i], ' ', tp->ncol);
+		tp->vram[i][tp->ncol] = '\0';
+	}
+}
+
+void
+Term_Dump(const struct term *tp)
+{
+	int i;
+
+	for (i = 0; i < tp->nlin; i++)
+		vtc_dump(tp->vl, 3, "screen", tp->vram[i], tp->ncol);
+}
+
+static void
+term_escape(struct term *tp, int c, int n)
+{
+	int i;
+
+	c = tolower(c);
+	for (i = 0; i < NTERMARG; i++)
+		if (!tp->arg[i])
+			tp->arg[i] = 1;
+	switch(c) {
+	case 'h':
+		if (tp->arg[0] > tp->nlin || tp->arg[1] > tp->ncol)
+			vtc_fatal(tp->vl, "ANSI H[%d,%d] outside vram",
+			    tp->arg[0], tp->arg[1]);
+		tp->line = tp->arg[0] - 1;
+		tp->col = tp->arg[1] - 1;
+		break;
+	case 'j':
+		if (tp->arg[0] != 2)
+			vtc_fatal(tp->vl, "ANSI J[%d]", tp->arg[0]);
+		term_clear(tp);
+		break;
+	default:
+		for (i = 0; i < n; i++)
+			vtc_log(tp->vl, 4, "ANSI arg %d", tp->arg[i]);
+		vtc_fatal(tp->vl, "ANSI unknown (%c)", c);
+		break;
+	}
+}
+
+static void
+term_scroll(const struct term *tp)
+{
+	int i;
+	char *l;
+
+	l = tp->vram[0];
+	for(i = 0; i < tp->nlin -1; i++)
+		tp->vram[i] = tp->vram[i + 1];
+	tp->vram[i] = l;
+	memset(l, ' ', tp->ncol);
+}
+
+static void
+term_char(struct term *tp, char c)
+{
+	assert(tp->col < tp->ncol);
+	assert(tp->line < tp->nlin);
+	assert(tp->state <= 3);
+	switch (c) {
+	case 0x00:
+		break;
+	case '\b':
+		if (tp->col > 0)
+			tp->col--;
+		break;
+	case '\t':
+		while(++tp->col % 8)
+			continue;
+		if (tp->col >= tp->ncol) {
+			tp->col = 0;
+			term_char(tp, '\n');
+		}
+		break;
+	case '\n':
+		if (tp->line == tp->nlin - 1)
+			term_scroll(tp);
+		else
+			tp->line++;
+		break;
+	case '\r':
+		tp->col = 0;
+		break;
+	default:
+		if (c < ' ' || c > '~')
+			c = '?';
+		tp->vram[tp->line][tp->col] = c;
+		if (tp->col == tp->ncol - 1) {
+			tp->col = 0;
+			term_char(tp, '\n');
+		} else {
+			tp->col++;
+		}
+	}
+}
+
+void
+Term_Feed(struct term *tp, const char *b, const char *e)
+{
+
+	while (b < e) {
+		assert(tp->col < tp->ncol);
+		assert(tp->line < tp->nlin);
+		assert(tp->state <= 3);
+		switch (tp->state) {
+		case 0:
+			if (*b == '\x1b')
+				tp->state = 1;
+			else if (*(const uint8_t*)b == 0x9b)
+				tp->state = 2;
+			else
+				term_char(tp, *b);
+			b++;
+			break;
+		case 1:
+			if (*b++ != '[')
+				vtc_fatal(tp->vl, "ANSI not '[' (0x%x)",
+				    b[-1] & 0xff);
+			tp->state = 2;
+			break;
+		case 2:
+			tp->argp = tp->arg;
+			memset(tp->arg, 0, sizeof tp->arg);
+			tp->state = 3;
+			break;
+		case 3:
+			if (tp->argp - tp->arg >= NTERMARG)
+				vtc_fatal(tp->vl, "ANSI too many args");
+
+			if (isdigit(*b)) {
+				*tp->argp *= 10;
+				*tp->argp += *b++ - '0';
+				continue;
+			}
+			if (*b == ';') {
+				tp->argp++;
+				tp->state = 3;
+				b++;
+				continue;
+			}
+			if (islower(*b)) {
+				term_escape(tp, *b++, tp->argp -  tp->arg);
+				tp->state = 2;
+			} else if (isupper(*b)) {
+				term_escape(tp, *b++, tp->argp -  tp->arg);
+				tp->argp = tp->arg;
+				tp->state = 0;
+			} else {
+				vtc_fatal(tp->vl, "ANSI non-letter (0x%02x)",
+				    *b & 0xff);
+			}
+			break;
+		default:
+			WRONG("Wrong ansi state");
+		}
+	}
+}
+
+struct term *
+Term_New(struct vtclog *vl)
+{
+	struct term *tp;
+	int i;
+
+	ALLOC_OBJ(tp, TERM_MAGIC);
+	AN(tp);
+	tp->vl = vl;
+	tp->nlin = 24;
+	tp->ncol = 80;
+	tp->vram = calloc(tp->nlin, sizeof *tp->vram);
+	AN(tp->vram);
+	for (i = 0; i < tp->nlin; i++) {
+		tp->vram[i] = malloc(tp->ncol + 1L);
+		AN(tp->vram[i]);
+	}
+	term_clear(tp);
+	tp->line = tp->nlin - 1;
+	return (tp);
+}
+

From phk at FreeBSD.org  Fri Jan 12 16:43:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 12 Jan 2018 16:43:08 +0000 (UTC)
Subject: [master] 8678574 Add hack to wait for X bytes on stdout.
Message-ID: <20180112164308.2AAE365BD2@lists.varnish-cache.org>


commit 86785741b3d41dd2be66142fe701149febfcfede
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 12 16:39:06 2018 +0000

    Add hack to wait for X bytes on stdout.

diff --git a/bin/varnishtest/tests/u00008.vtc b/bin/varnishtest/tests/u00008.vtc
new file mode 100644
index 0000000..def8166
--- /dev/null
+++ b/bin/varnishtest/tests/u00008.vtc
@@ -0,0 +1,22 @@
+varnishtest	"trivial run of varnistat in curses mode"
+
+feature term
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {} -start
+
+process p1 -dump {varnishstat -n ${v1_name}} -start
+
+process p1 -need-bytes 1
+
+client c1 {
+	txreq
+	rxresp
+} -run
+
+process p1 -need-bytes 2300 -screen_dump -write {q} -wait
+
diff --git a/bin/varnishtest/tests/u00009.vtc b/bin/varnishtest/tests/u00009.vtc
index 1e540bd..13b931c 100644
--- a/bin/varnishtest/tests/u00009.vtc
+++ b/bin/varnishtest/tests/u00009.vtc
@@ -11,15 +11,11 @@ varnish v1 -vcl+backend {} -start
 
 process p1 -dump {varnishhist -n ${v1_name}} -start
 
-delay 3
+process p1 -need-bytes 1
 
 client c1 {
 	txreq
 	rxresp
 } -run
 
-delay 10
-
-process p1 -write {q}
-
-process p1 -wait -screen_dump
+process p1 -need-bytes 300 -screen_dump -write {q} -wait
diff --git a/bin/varnishtest/tests/u00010.vtc b/bin/varnishtest/tests/u00010.vtc
index fbca6d5..e94526f 100644
--- a/bin/varnishtest/tests/u00010.vtc
+++ b/bin/varnishtest/tests/u00010.vtc
@@ -11,15 +11,11 @@ varnish v1 -vcl+backend {} -start
 
 process p1 -dump {varnishtop -n ${v1_name}} -start
 
-delay 3
+process p1 -need-bytes 1
 
 client c1 {
 	txreq
 	rxresp
 } -run
 
-delay 10
-
-process p1 -write {q}
-
-process p1 -wait -screen_dump
+process p1 -need-bytes 2500 -screen_dump -write {q} -wait
diff --git a/bin/varnishtest/tests/u00011.vtc b/bin/varnishtest/tests/u00011.vtc
index 4c29d24..8870b1a 100644
--- a/bin/varnishtest/tests/u00011.vtc
+++ b/bin/varnishtest/tests/u00011.vtc
@@ -11,19 +11,17 @@ varnish v1 -vcl+backend {} -start
 
 process p1 -dump {varnishadm -n ${v1_name}} -start
 
-delay 2
+process p1 -need-bytes 1
 
 client c1 {
 	txreq
 	rxresp
 } -run
 
-delay 2
-
 process p1 -writeln {panic.show}
 
 process p1 -writeln {quit}
 
-delay 10
+process p1 -need-bytes 400
 
-process p1 -wait -screen_dump
+process p1 -screen_dump -writeln {quit} -wait
diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index 6d55975..24e3e74 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -38,6 +38,7 @@
 #include <ctype.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <inttypes.h>
 #include <poll.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -76,6 +77,9 @@ struct process {
 	pid_t			pid;
 	int			expect_exit;
 
+	uintmax_t		stdout_bytes;
+	uintmax_t		stderr_bytes;
+
 	pthread_mutex_t		mtx;
 	pthread_t		tp;
 	unsigned		hasthread;
@@ -198,6 +202,9 @@ process_stdout(const struct vev *ev, int what)
 		vtc_log(p->vl, 4, "stdout read %d", i);
 		return (1);
 	}
+	AZ(pthread_mutex_lock(&p->mtx));
+	p->stdout_bytes += i;
+	AZ(pthread_mutex_unlock(&p->mtx));
 	if (p->log == 1)
 		(void)VLU_Feed(p->vlu_stdout, buf, i);
 	else if (p->log == 2)
@@ -223,6 +230,9 @@ process_stderr(const struct vev *ev, int what)
 		vtc_log(p->vl, 4, "stderr read %d", i);
 		return (1);
 	}
+	AZ(pthread_mutex_lock(&p->mtx));
+	p->stderr_bytes += i;
+	AZ(pthread_mutex_unlock(&p->mtx));
 	vtc_dump(p->vl, 4, "stderr", buf, i);
 	(void)write(p->f_stderr, buf, i);
 	return (0);
@@ -435,6 +445,8 @@ process_wait(struct process *p)
 		AZ(pthread_join(p->tp, &v));
 		p->hasthread = 0;
 	}
+	vtc_log(p->vl, 4, "stdout %ju bytes, stderr %ju bytes",
+	    p->stdout_bytes, p->stderr_bytes);
 }
 
 /**********************************************************************
@@ -601,6 +613,7 @@ void
 cmd_process(CMD_ARGS)
 {
 	struct process *p, *p2;
+	uintmax_t u, v;
 
 	(void)priv;
 	(void)cmd;
@@ -697,6 +710,17 @@ cmd_process(CMD_ARGS)
 			av++;
 			continue;
 		}
+		if (!strcmp(*av, "-need-bytes")) {
+			u = strtoumax(av[1], NULL, 0);
+			av++;
+			do {
+				usleep(100000);
+				AZ(pthread_mutex_lock(&p->mtx));
+				v = p->stdout_bytes;
+				AZ(pthread_mutex_unlock(&p->mtx));
+			} while(v < u);
+			continue;
+		}
 		if (!strcmp(*av, "-screen_dump")) {
 			Term_Dump(p->term);
 			continue;

From phk at FreeBSD.org  Fri Jan 12 16:49:05 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 12 Jan 2018 16:49:05 +0000 (UTC)
Subject: [master] 93195a4 Require fewer bytes of varnishtop
Message-ID: <20180112164905.3B4B09400D@lists.varnish-cache.org>


commit 93195a4e08eb273eb5e26a296977bed18a10601a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 12 16:47:53 2018 +0000

    Require fewer bytes of varnishtop

diff --git a/bin/varnishtest/tests/u00010.vtc b/bin/varnishtest/tests/u00010.vtc
index e94526f..59af38f 100644
--- a/bin/varnishtest/tests/u00010.vtc
+++ b/bin/varnishtest/tests/u00010.vtc
@@ -18,4 +18,4 @@ client c1 {
 	rxresp
 } -run
 
-process p1 -need-bytes 2500 -screen_dump -write {q} -wait
+process p1 -need-bytes 2000 -screen_dump -write {q} -wait

From phk at FreeBSD.org  Fri Jan 12 17:55:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 12 Jan 2018 17:55:07 +0000 (UTC)
Subject: [master] 945d2e1 Switch to 'ansi.sys' terminal type,
 that seems the lowest common denomintor.
Message-ID: <20180112175507.CA56C956F9@lists.varnish-cache.org>


commit 945d2e1ea50f52fde10f9b31d55031c58ffc88d8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 12 17:54:21 2018 +0000

    Switch to 'ansi.sys' terminal type, that seems the lowest common denomintor.

diff --git a/bin/varnishtest/tests/u00009.vtc b/bin/varnishtest/tests/u00009.vtc
index 13b931c..89e6ea7 100644
--- a/bin/varnishtest/tests/u00009.vtc
+++ b/bin/varnishtest/tests/u00009.vtc
@@ -18,4 +18,4 @@ client c1 {
 	rxresp
 } -run
 
-process p1 -need-bytes 300 -screen_dump -write {q} -wait
+process p1 -need-bytes 200 -screen_dump -write {q} -wait
diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 6a2a476..4b90c2f 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -727,14 +727,14 @@ test_term(struct vtclog *vl)
 	FILE *p;
 	int a, b;
 
-	p = popen("tput -T ansi clear 2>&1", "r");
+	p = popen("tput -T ansi.sys clear 2>&1", "r");
 	if (p == NULL)
 		return (0);
 	a = fgetc(p);
 	b = fgetc(p);
 	if (a == 0x1b && b == '[')
 		return (1);
-	vtc_log(vl, 3, "No 'ansi' terminfo entry.");
+	vtc_log(vl, 3, "No 'ansi.sys' terminfo entry.");
 	return (0);
 }
 
diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index 24e3e74..0325b94 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -406,7 +406,7 @@ process_start(struct process *p)
 	p->pid = fork();
 	assert(p->pid >= 0);
 	if (p->pid == 0) {
-		AZ(setenv("TERM", "ansi", 1));
+		AZ(setenv("TERM", "ansi.sys", 1));
 		AZ(unsetenv("TERMCAP"));
 		assert(dup2(slave, STDIN_FILENO) == STDIN_FILENO);
 		assert(dup2(slave, STDOUT_FILENO) == STDOUT_FILENO);
diff --git a/bin/varnishtest/vtc_term.c b/bin/varnishtest/vtc_term.c
index 34caa30..6b89a03 100644
--- a/bin/varnishtest/vtc_term.c
+++ b/bin/varnishtest/vtc_term.c
@@ -63,6 +63,19 @@ term_clear(const struct term *tp)
 	}
 }
 
+static void
+term_scroll(const struct term *tp)
+{
+	int i;
+	char *l;
+
+	l = tp->vram[0];
+	for(i = 0; i < tp->nlin -1; i++)
+		tp->vram[i] = tp->vram[i + 1];
+	tp->vram[i] = l;
+	memset(l, ' ', tp->ncol);
+}
+
 void
 Term_Dump(const struct term *tp)
 {
@@ -77,23 +90,38 @@ term_escape(struct term *tp, int c, int n)
 {
 	int i;
 
-	c = tolower(c);
 	for (i = 0; i < NTERMARG; i++)
 		if (!tp->arg[i])
 			tp->arg[i] = 1;
 	switch(c) {
+	case 'B':
+		if (tp->arg[0] > tp->nlin)
+			vtc_fatal(tp->vl, "ANSI B[%d] outside vram",
+			    tp->arg[0]);
+		tp->line += tp->arg[0];
+		while (tp->line >= tp->nlin) {
+			term_scroll(tp);
+			tp->line--;
+		}
+		break;
 	case 'h':
+		// Ignore screen mode selection
+		break;
+	case 'H':
 		if (tp->arg[0] > tp->nlin || tp->arg[1] > tp->ncol)
 			vtc_fatal(tp->vl, "ANSI H[%d,%d] outside vram",
 			    tp->arg[0], tp->arg[1]);
 		tp->line = tp->arg[0] - 1;
 		tp->col = tp->arg[1] - 1;
 		break;
-	case 'j':
+	case 'J':
 		if (tp->arg[0] != 2)
 			vtc_fatal(tp->vl, "ANSI J[%d]", tp->arg[0]);
 		term_clear(tp);
 		break;
+	case 'm':
+		// Ignore Graphic Rendition settings
+		break;
 	default:
 		for (i = 0; i < n; i++)
 			vtc_log(tp->vl, 4, "ANSI arg %d", tp->arg[i]);
@@ -103,19 +131,6 @@ term_escape(struct term *tp, int c, int n)
 }
 
 static void
-term_scroll(const struct term *tp)
-{
-	int i;
-	char *l;
-
-	l = tp->vram[0];
-	for(i = 0; i < tp->nlin -1; i++)
-		tp->vram[i] = tp->vram[i + 1];
-	tp->vram[i] = l;
-	memset(l, ' ', tp->ncol);
-}
-
-static void
 term_char(struct term *tp, char c)
 {
 	assert(tp->col < tp->ncol);
@@ -186,6 +201,8 @@ Term_Feed(struct term *tp, const char *b, const char *e)
 			tp->argp = tp->arg;
 			memset(tp->arg, 0, sizeof tp->arg);
 			tp->state = 3;
+			if (*b == '?')
+				b++;
 			break;
 		case 3:
 			if (tp->argp - tp->arg >= NTERMARG)
@@ -202,17 +219,8 @@ Term_Feed(struct term *tp, const char *b, const char *e)
 				b++;
 				continue;
 			}
-			if (islower(*b)) {
-				term_escape(tp, *b++, tp->argp -  tp->arg);
-				tp->state = 2;
-			} else if (isupper(*b)) {
-				term_escape(tp, *b++, tp->argp -  tp->arg);
-				tp->argp = tp->arg;
-				tp->state = 0;
-			} else {
-				vtc_fatal(tp->vl, "ANSI non-letter (0x%02x)",
-				    *b & 0xff);
-			}
+			term_escape(tp, *b++, tp->argp -  tp->arg);
+			tp->state = 0;
 			break;
 		default:
 			WRONG("Wrong ansi state");

From phk at FreeBSD.org  Fri Jan 12 19:09:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 12 Jan 2018 19:09:07 +0000 (UTC)
Subject: [master] dc0f151 Implement CSI-K, tweak tests.
Message-ID: <20180112190907.3745996C96@lists.varnish-cache.org>


commit dc0f151065a3f8a7bd23061b25bbc37685950a3c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 12 19:07:53 2018 +0000

    Implement CSI-K, tweak tests.

diff --git a/bin/varnishtest/tests/u00010.vtc b/bin/varnishtest/tests/u00010.vtc
index 59af38f..4822a78 100644
--- a/bin/varnishtest/tests/u00010.vtc
+++ b/bin/varnishtest/tests/u00010.vtc
@@ -18,4 +18,4 @@ client c1 {
 	rxresp
 } -run
 
-process p1 -need-bytes 2000 -screen_dump -write {q} -wait
+process p1 -need-bytes 1000 -screen_dump -write {q} -wait
diff --git a/bin/varnishtest/tests/u00011.vtc b/bin/varnishtest/tests/u00011.vtc
index 8870b1a..378c22d 100644
--- a/bin/varnishtest/tests/u00011.vtc
+++ b/bin/varnishtest/tests/u00011.vtc
@@ -24,4 +24,4 @@ process p1 -writeln {quit}
 
 process p1 -need-bytes 400
 
-process p1 -screen_dump -writeln {quit} -wait
+process p1 -screen_dump -write "quit\r" -wait
diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index 0325b94..9a06e95 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -714,10 +714,11 @@ cmd_process(CMD_ARGS)
 			u = strtoumax(av[1], NULL, 0);
 			av++;
 			do {
-				usleep(100000);
+				usleep(500000);
 				AZ(pthread_mutex_lock(&p->mtx));
 				v = p->stdout_bytes;
 				AZ(pthread_mutex_unlock(&p->mtx));
+				vtc_log(p->vl, 4, "Have %ju bytes", v);
 			} while(v < u);
 			continue;
 		}
diff --git a/bin/varnishtest/vtc_term.c b/bin/varnishtest/vtc_term.c
index 6b89a03..a6309eb 100644
--- a/bin/varnishtest/vtc_term.c
+++ b/bin/varnishtest/vtc_term.c
@@ -119,6 +119,25 @@ term_escape(struct term *tp, int c, int n)
 			vtc_fatal(tp->vl, "ANSI J[%d]", tp->arg[0]);
 		term_clear(tp);
 		break;
+	case 'K':
+		// erase in line 0=right, 1=left, 2=full line
+		switch (tp->arg[0]) {
+		case 0:
+			for (i = tp->col + 1; i < tp->ncol; i++)
+				tp->vram[tp->line][i] = ' ';
+			break;
+		case 1:
+			for (i = 0; i < tp->col; i++)
+				tp->vram[tp->line][i] = ' ';
+			break;
+		case 2:
+			for (i = 0; i < tp->ncol; i++)
+				tp->vram[tp->line][i] = ' ';
+			break;
+		default:
+			vtc_fatal(tp->vl, "ANSI K[%d]", tp->arg[0]);
+		}
+		break;
 	case 'm':
 		// Ignore Graphic Rendition settings
 		break;

From phk at FreeBSD.org  Fri Jan 12 19:31:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 12 Jan 2018 19:31:06 +0000 (UTC)
Subject: [master] 76c740f More curses test-tweaking
Message-ID: <20180112193107.046B69744B@lists.varnish-cache.org>


commit 76c740f57a2113275abaea9a5a5d425263629cc6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 12 19:30:30 2018 +0000

    More curses test-tweaking

diff --git a/bin/varnishtest/tests/u00010.vtc b/bin/varnishtest/tests/u00010.vtc
index 4822a78..a143c72 100644
--- a/bin/varnishtest/tests/u00010.vtc
+++ b/bin/varnishtest/tests/u00010.vtc
@@ -18,4 +18,4 @@ client c1 {
 	rxresp
 } -run
 
-process p1 -need-bytes 1000 -screen_dump -write {q} -wait
+process p1 -need-bytes 500 -screen_dump -write {q} -wait
diff --git a/bin/varnishtest/tests/u00011.vtc b/bin/varnishtest/tests/u00011.vtc
index 378c22d..38e57f0 100644
--- a/bin/varnishtest/tests/u00011.vtc
+++ b/bin/varnishtest/tests/u00011.vtc
@@ -20,8 +20,6 @@ client c1 {
 
 process p1 -writeln {panic.show}
 
-process p1 -writeln {quit}
-
 process p1 -need-bytes 400
 
 process p1 -screen_dump -write "quit\r" -wait
diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index 9a06e95..180f0d4 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -714,11 +714,11 @@ cmd_process(CMD_ARGS)
 			u = strtoumax(av[1], NULL, 0);
 			av++;
 			do {
-				usleep(500000);
 				AZ(pthread_mutex_lock(&p->mtx));
 				v = p->stdout_bytes;
 				AZ(pthread_mutex_unlock(&p->mtx));
 				vtc_log(p->vl, 4, "Have %ju bytes", v);
+				usleep(500000);
 			} while(v < u);
 			continue;
 		}

From phk at FreeBSD.org  Fri Jan 12 19:50:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 12 Jan 2018 19:50:07 +0000 (UTC)
Subject: [master] 00707a9 more tweaking
Message-ID: <20180112195007.B56EB97AFF@lists.varnish-cache.org>


commit 00707a9a979e29dd0859ad6db73ca72294cb787f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 12 19:49:21 2018 +0000

    more tweaking

diff --git a/bin/varnishtest/tests/u00011.vtc b/bin/varnishtest/tests/u00011.vtc
index 38e57f0..698b702 100644
--- a/bin/varnishtest/tests/u00011.vtc
+++ b/bin/varnishtest/tests/u00011.vtc
@@ -20,6 +20,6 @@ client c1 {
 
 process p1 -writeln {panic.show}
 
-process p1 -need-bytes 400
+process p1 -need-bytes 380
 
 process p1 -screen_dump -write "quit\r" -wait

From phk at FreeBSD.org  Fri Jan 12 21:36:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 12 Jan 2018 21:36:10 +0000 (UTC)
Subject: [master] 28d0ff7 Fix the same basic error in all our curses programs:
 When the stdin-fd is dead, it takes more than "break;
 " to get out of a switch-in-a-loop.
Message-ID: <20180112213610.776CB99F0E@lists.varnish-cache.org>


commit 28d0ff71ea027a673f91fb639fc67f6aece20129
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 12 21:34:23 2018 +0000

    Fix the same basic error in all our curses programs:  When the
    stdin-fd is dead, it takes more than "break;" to get out of a
    switch-in-a-loop.

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 1fed3cb..a051716 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -397,6 +397,7 @@ do_curses(void *arg)
 		timeout(delay * 1000);
 		switch ((ch = getch())) {
 		case ERR:
+			quit = 1;
 			break;
 #ifdef KEY_RESIZE
 		case KEY_RESIZE:
diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index 0dfacd1..9f8c432 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -1030,6 +1030,7 @@ do_curses(struct vsm *vsm, struct vsc *vsc, double delay)
 
 		switch (ch = wgetch(w_status)) {
 		case ERR:
+			keep_running = 0;
 			break;
 #ifdef KEY_RESIZE /* sigh, Solaris lacks this.. */
 		case KEY_RESIZE:
diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 332bc0e..582edbd 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -271,14 +271,15 @@ do_curses(void *arg)
 	AC(intrflush(stdscr, FALSE));
 	(void)curs_set(0);
 	AC(erase());
+	timeout(1000);
 	while (!quit) {
 		AZ(pthread_mutex_lock(&mtx));
 		update(period);
 		AZ(pthread_mutex_unlock(&mtx));
 
-		timeout(1000);
 		switch (getch()) {
 		case ERR:
+			quit = 1;
 			break;
 #ifdef KEY_RESIZE
 		case KEY_RESIZE:

From phk at FreeBSD.org  Fri Jan 12 21:48:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri, 12 Jan 2018 21:48:06 +0000 (UTC)
Subject: [master] 7e3e7a7 Implement CSI-C (CUF)
Message-ID: <20180112214807.013DA9A464@lists.varnish-cache.org>


commit 7e3e7a7d785bf775e6ad6ddaaca64ad3b8b226e2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri Jan 12 21:46:58 2018 +0000

    Implement CSI-C (CUF)

diff --git a/bin/varnishtest/vtc_term.c b/bin/varnishtest/vtc_term.c
index a6309eb..8897a61 100644
--- a/bin/varnishtest/vtc_term.c
+++ b/bin/varnishtest/vtc_term.c
@@ -95,6 +95,7 @@ term_escape(struct term *tp, int c, int n)
 			tp->arg[i] = 1;
 	switch(c) {
 	case 'B':
+		// CUD - Cursor down
 		if (tp->arg[0] > tp->nlin)
 			vtc_fatal(tp->vl, "ANSI B[%d] outside vram",
 			    tp->arg[0]);
@@ -104,10 +105,18 @@ term_escape(struct term *tp, int c, int n)
 			tp->line--;
 		}
 		break;
+	case 'C':
+		// CUF - Cursor forward
+		tp->col += tp->arg[0];
+		if (tp->col >= tp->ncol)
+			vtc_fatal(tp->vl, "ANSI C[%d] outside vram",
+			    tp->arg[0]);
+		break;
 	case 'h':
-		// Ignore screen mode selection
+		// SM - Set Mode (ignored XXX?)
 		break;
 	case 'H':
+		// CUP - Cursor Position
 		if (tp->arg[0] > tp->nlin || tp->arg[1] > tp->ncol)
 			vtc_fatal(tp->vl, "ANSI H[%d,%d] outside vram",
 			    tp->arg[0], tp->arg[1]);
@@ -115,12 +124,17 @@ term_escape(struct term *tp, int c, int n)
 		tp->col = tp->arg[1] - 1;
 		break;
 	case 'J':
-		if (tp->arg[0] != 2)
+		// ED - Erase in Display (0=below, 1=above, 2=all)
+		switch(tp->arg[0]) {
+		case 2:
+			term_clear(tp);
+			break;
+		default:
 			vtc_fatal(tp->vl, "ANSI J[%d]", tp->arg[0]);
-		term_clear(tp);
+		}
 		break;
 	case 'K':
-		// erase in line 0=right, 1=left, 2=full line
+		// EL - Erase in line (0=right, 1=left, 2=full line)
 		switch (tp->arg[0]) {
 		case 0:
 			for (i = tp->col + 1; i < tp->ncol; i++)
@@ -139,7 +153,7 @@ term_escape(struct term *tp, int c, int n)
 		}
 		break;
 	case 'm':
-		// Ignore Graphic Rendition settings
+		// SGG - Character Attributes (ignored)
 		break;
 	default:
 		for (i = 0; i < n; i++)

From phk at FreeBSD.org  Sat Jan 13 00:33:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 13 Jan 2018 00:33:09 +0000 (UTC)
Subject: [master] 586dda0 Use -need-bytes instead of delays
Message-ID: <20180113003309.3C032A38D1@lists.varnish-cache.org>


commit 586dda0535ac393f6c02f0b0f7bde9a4ac2e9951
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 13 00:25:49 2018 +0000

    Use -need-bytes instead of delays

diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc
index 0b33c36..99bf22a 100644
--- a/bin/varnishtest/tests/u00000.vtc
+++ b/bin/varnishtest/tests/u00000.vtc
@@ -2,7 +2,7 @@ varnishtest "Simple process tests"
 
 process p1 "cat" -start
 process p1 -writeln "foo"
-delay 0.5
+process p1 -need-bytes 3
 process p1 -stop
 process p1 -wait
 shell "grep -q foo ${p1_out}"
@@ -10,7 +10,7 @@ shell "test -f ${p1_err} -a ! -s ${p1_err}"
 
 process p2 -log "cat" -start
 process p2 -writeln "bar"
-delay 0.5
+process p2 -need-bytes 3
 process p2 -write "\x04"
 process p2 -wait
 shell "grep -q bar ${p2_out}"
@@ -18,7 +18,7 @@ shell "test -f ${p2_err} -a ! -s ${p2_err}"
 
 process p3 -dump "cat" -start
 process p3 -writeln "baz"
-delay 0.5
+process p3 -need-bytes 3
 process p3 -kill KILL
 process p3 -wait
 shell "grep -q baz ${p3_out}"
@@ -26,7 +26,7 @@ shell "test -f ${p3_err} -a ! -s ${p3_err}"
 
 process p4 -hexdump "cat" -start
 process p4 -writeln "b\001z"
-delay 0.5
+process p4 -need-bytes 3
 process p4 -kill TERM
 process p4 -wait -screen_dump
 

From phk at FreeBSD.org  Sat Jan 13 00:33:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 13 Jan 2018 00:33:09 +0000 (UTC)
Subject: [master] c39ed3c Make pty ctrl-terminal to get SIGHUPs
Message-ID: <20180113003309.52421A38DE@lists.varnish-cache.org>


commit c39ed3c8fd4d0c7ec2f63c248b34923140311d40
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 13 00:31:09 2018 +0000

    Make pty ctrl-terminal to get SIGHUPs

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index 180f0d4..330f403 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -390,29 +390,31 @@ process_start(struct process *p)
 	AZ(unlockpt(master));
 	slavename = ptsname(master);
 	AN(slavename);
-	slave = open(slavename, O_RDWR);
-	assert(slave >= 0);
-#ifdef __sun
-	if (ioctl(slave, I_PUSH, "ptem"))
-		vtc_log(p->vl, 4, "PUSH ptem: %s", strerror(errno));
-	if (ioctl(slave, I_PUSH, "ldterm"))
-		vtc_log(p->vl, 4, "PUSH ldterm: %s", strerror(errno));
-#endif
-
-	process_init_term(p, slave);
 
 	AZ(pipe(fd2));
 
 	p->pid = fork();
 	assert(p->pid >= 0);
 	if (p->pid == 0) {
-		AZ(setenv("TERM", "ansi.sys", 1));
-		AZ(unsetenv("TERMCAP"));
-		assert(dup2(slave, STDIN_FILENO) == STDIN_FILENO);
-		assert(dup2(slave, STDOUT_FILENO) == STDOUT_FILENO);
+		setsid();
 		assert(dup2(fd2[1], STDERR_FILENO) == STDERR_FILENO);
+		close(STDIN_FILENO);
+		slave = open(slavename, O_RDWR);
+		assert(slave == STDIN_FILENO);
+		AZ(ioctl(STDIN_FILENO, TIOCSCTTY, 0 ));
+		close(STDOUT_FILENO);
+		assert(dup2(slave, STDOUT_FILENO) == STDOUT_FILENO);
 		VSUB_closefrom(STDERR_FILENO + 1);
-		AZ(setpgid(0, 0));
+#ifdef __sun
+		if (ioctl(slave, I_PUSH, "ptem"))
+			vtc_log(p->vl, 4, "PUSH ptem: %s", strerror(errno));
+		if (ioctl(slave, I_PUSH, "ldterm"))
+			vtc_log(p->vl, 4, "PUSH ldterm: %s", strerror(errno));
+#endif
+		process_init_term(p, slave);
+
+		AZ(setenv("TERM", "ansi.sys", 1));
+		AZ(unsetenv("TERMCAP"));
 		// Not using NULL because GCC is now even more demented...
 		AZ(execl("/bin/sh", "/bin/sh", "-c", VSB_data(cl), (char*)0));
 		exit(1);
@@ -420,7 +422,6 @@ process_start(struct process *p)
 	vtc_log(p->vl, 3, "PID: %ld", (long)p->pid);
 	VSB_destroy(&cl);
 
-	closefd(&slave);
 	p->fd_term = master;
 	closefd(&fd2[1]);
 	p->fd_stderr = fd2[0];

From phk at FreeBSD.org  Sat Jan 13 00:33:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 13 Jan 2018 00:33:09 +0000 (UTC)
Subject: [master] 3e454cf Now that varnishtest sets up SIGHUP, remove hacks.
Message-ID: <20180113003309.6EAB0A38E8@lists.varnish-cache.org>


commit 3e454cf29a958a9b8b29aa98dc2891896faf9218
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 13 00:32:00 2018 +0000

    Now that varnishtest sets up SIGHUP, remove hacks.

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index a051716..1fed3cb 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -397,7 +397,6 @@ do_curses(void *arg)
 		timeout(delay * 1000);
 		switch ((ch = getch())) {
 		case ERR:
-			quit = 1;
 			break;
 #ifdef KEY_RESIZE
 		case KEY_RESIZE:
diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index 9f8c432..56d442f 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -37,7 +37,6 @@
 #include <unistd.h>
 #include <string.h>
 #include <signal.h>
-#include <poll.h>
 #include <stdint.h>
 #include <math.h>
 
@@ -981,7 +980,6 @@ delpt(void *priv, const struct VSC_point *const vpt)
 void
 do_curses(struct vsm *vsm, struct vsc *vsc, double delay)
 {
-	struct pollfd pollfd;
 	long t;
 	int ch;
 	double now;
@@ -996,9 +994,6 @@ do_curses(struct vsm *vsm, struct vsc *vsc, double delay)
 	nonl();
 	curs_set(0);
 
-	pollfd.fd = STDIN_FILENO;
-	pollfd.events = POLLIN;
-
 	make_windows();
 	doupdate();
 
@@ -1025,12 +1020,10 @@ do_curses(struct vsm *vsm, struct vsc *vsc, double delay)
 			draw_screen();
 
 		t = (t_sample + interval - now) * 1000;
-		if (t > 0)
-			(void)poll(&pollfd, 1, t);
+		timeout(t);
 
 		switch (ch = wgetch(w_status)) {
 		case ERR:
-			keep_running = 0;
 			break;
 #ifdef KEY_RESIZE /* sigh, Solaris lacks this.. */
 		case KEY_RESIZE:
diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index 582edbd..f0fa320 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -279,7 +279,6 @@ do_curses(void *arg)
 
 		switch (getch()) {
 		case ERR:
-			quit = 1;
 			break;
 #ifdef KEY_RESIZE
 		case KEY_RESIZE:

From phk at FreeBSD.org  Sat Jan 13 01:33:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 13 Jan 2018 01:33:06 +0000 (UTC)
Subject: [master] e88a5a3 Make sure the process has TCSAFLUSH'ed before
 sending anything to it.
Message-ID: <20180113013306.3AB77A5C02@lists.varnish-cache.org>


commit e88a5a3c8bf76ed43dbd5f26a1b2b7dae49223cb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 13 01:31:25 2018 +0000

    Make sure the process has TCSAFLUSH'ed before sending anything to it.

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index 330f403..cf0571a 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -374,6 +374,7 @@ process_start(struct process *p)
 	int fd2[2];
 	int master, slave;
 	const char *slavename;
+	char c;
 
 	CHECK_OBJ_NOTNULL(p, PROCESS_MAGIC);
 	if (p->hasthread)
@@ -396,13 +397,13 @@ process_start(struct process *p)
 	p->pid = fork();
 	assert(p->pid >= 0);
 	if (p->pid == 0) {
-		setsid();
+		assert(setsid() == getpid());
 		assert(dup2(fd2[1], STDERR_FILENO) == STDERR_FILENO);
-		close(STDIN_FILENO);
+		AZ(close(STDIN_FILENO));
 		slave = open(slavename, O_RDWR);
 		assert(slave == STDIN_FILENO);
-		AZ(ioctl(STDIN_FILENO, TIOCSCTTY, 0 ));
-		close(STDOUT_FILENO);
+		AZ(ioctl(STDIN_FILENO, TIOCSCTTY, NULL));
+		AZ(close(STDOUT_FILENO));
 		assert(dup2(slave, STDOUT_FILENO) == STDOUT_FILENO);
 		VSUB_closefrom(STDERR_FILENO + 1);
 #ifdef __sun
@@ -416,12 +417,14 @@ process_start(struct process *p)
 		AZ(setenv("TERM", "ansi.sys", 1));
 		AZ(unsetenv("TERMCAP"));
 		// Not using NULL because GCC is now even more demented...
+		assert(write(STDERR_FILENO, "+", 1) == 1);
 		AZ(execl("/bin/sh", "/bin/sh", "-c", VSB_data(cl), (char*)0));
 		exit(1);
 	}
 	vtc_log(p->vl, 3, "PID: %ld", (long)p->pid);
 	VSB_destroy(&cl);
 
+	assert(read(fd2[0], &c, 1) == 1);
 	p->fd_term = master;
 	closefd(&fd2[1]);
 	p->fd_stderr = fd2[0];

From phk at FreeBSD.org  Sat Jan 13 10:41:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 13 Jan 2018 10:41:09 +0000 (UTC)
Subject: [master] 2ed5a02 Try to get Solaris to work again.
Message-ID: <20180113104109.7ADB6B0F26@lists.varnish-cache.org>


commit 2ed5a020121a90bc862e9ea291c4fd6c871568b5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 13 10:40:02 2018 +0000

    Try to get Solaris to work again.

diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc
index 99bf22a..1af8263 100644
--- a/bin/varnishtest/tests/u00000.vtc
+++ b/bin/varnishtest/tests/u00000.vtc
@@ -1,5 +1,9 @@
 varnishtest "Simple process tests"
 
+process p6 "ps -lw | grep '[p][s]' ; tty ; sleep 1" -run -screen_dump
+
+process p5 "stty -a ; sleep 1" -run -screen_dump
+
 process p1 "cat" -start
 process p1 -writeln "foo"
 process p1 -need-bytes 3
@@ -30,4 +34,3 @@ process p4 -need-bytes 3
 process p4 -kill TERM
 process p4 -wait -screen_dump
 
-process p5 "stty -a ; sleep 1" -run -screen_dump
diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index cf0571a..3d5903d 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -402,7 +402,11 @@ process_start(struct process *p)
 		AZ(close(STDIN_FILENO));
 		slave = open(slavename, O_RDWR);
 		assert(slave == STDIN_FILENO);
+#ifdef __sun
+		(void)ioctl(STDIN_FILENO, TIOCSCTTY, NULL);
+#else
 		AZ(ioctl(STDIN_FILENO, TIOCSCTTY, NULL));
+#endif
 		AZ(close(STDOUT_FILENO));
 		assert(dup2(slave, STDOUT_FILENO) == STDOUT_FILENO);
 		VSUB_closefrom(STDERR_FILENO + 1);

From phk at FreeBSD.org  Sat Jan 13 17:17:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 13 Jan 2018 17:17:07 +0000 (UTC)
Subject: [master] 6265345 Enable echo on the pty
Message-ID: <20180113171707.A700DB79FC@lists.varnish-cache.org>


commit 626534577acdcc4607398af0b5af226b26d4179e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 13 17:16:03 2018 +0000

    Enable echo on the pty

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index 3d5903d..37fa4b3 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -344,7 +344,7 @@ process_init_term(struct process *p, int fd)
 	memset(&tt, 0, sizeof tt);
 	tt.c_cflag = CREAD | CS8 | HUPCL;
 	tt.c_iflag = BRKINT | ICRNL | IMAXBEL | IXON | IXANY;
-	tt.c_lflag = ICANON | ISIG | IEXTEN;
+	tt.c_lflag = ICANON | ISIG | IEXTEN | ECHO | ECHOE | ECHOKE | ECHOCTL;
 	tt.c_oflag = OPOST | ONLCR;
 	i = cfsetispeed(&tt, B9600);
 	if (i)
@@ -512,9 +512,9 @@ process_write(const struct process *p, const char *text)
 	len = strlen(text);
 	vtc_log(p->vl, 4, "Writing %d bytes", len);
 	r = write(p->fd_term, text, len);
-	if (r < 0)
-		vtc_fatal(p->vl, "Failed to write: %s (%d)",
-		    strerror(errno), errno);
+	if (r != len)
+		vtc_fatal(p->vl, "Failed to write: len=%d %s (%d)",
+		    len, strerror(errno), errno);
 }
 
 static void

From phk at FreeBSD.org  Sat Jan 13 17:17:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 13 Jan 2018 17:17:07 +0000 (UTC)
Subject: [master] e85d1e6 Exercise command complettion in varnishadm
Message-ID: <20180113171707.BF524B79FF@lists.varnish-cache.org>


commit e85d1e67787f97ca1417bb93419931ff01ed0e20
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 13 17:16:14 2018 +0000

    Exercise command complettion in varnishadm

diff --git a/bin/varnishtest/tests/u00011.vtc b/bin/varnishtest/tests/u00011.vtc
index 698b702..ab485bb 100644
--- a/bin/varnishtest/tests/u00011.vtc
+++ b/bin/varnishtest/tests/u00011.vtc
@@ -9,17 +9,27 @@ server s1 {
 
 varnish v1 -vcl+backend {} -start
 
-process p1 -dump {varnishadm -n ${v1_name}} -start
-
-process p1 -need-bytes 1
-
 client c1 {
 	txreq
 	rxresp
 } -run
 
-process p1 -writeln {panic.show}
+varnish v1 -vsl_catchup
+
+process p1 -log {varnishadm -n ${v1_name}} -start
+
+process p1 -need-bytes 1
+
+process p1 -write "pi\t\r"
+
+process p1 -need-bytes 100
+
+process p1 -write "vcl.li\t\r"
+
+process p1 -need-bytes 200
+
+process p1 -write "vcl.s\t\th\tvcl1\r"
 
-process p1 -need-bytes 380
+process p1 -need-bytes 600
 
 process p1 -screen_dump -write "quit\r" -wait

From phk at FreeBSD.org  Sat Jan 13 21:15:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sat, 13 Jan 2018 21:15:08 +0000 (UTC)
Subject: [master] f224bcf Add a test for varnishadm in pass mode
Message-ID: <20180113211508.A8CDBBBC62@lists.varnish-cache.org>


commit f224bcf183c06a3141a0fe47cf0a4b4ebc9bcdf4
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sat Jan 13 21:14:18 2018 +0000

    Add a test for varnishadm in pass mode

diff --git a/bin/varnishtest/tests/u00012.vtc b/bin/varnishtest/tests/u00012.vtc
new file mode 100644
index 0000000..0660d07
--- /dev/null
+++ b/bin/varnishtest/tests/u00012.vtc
@@ -0,0 +1,33 @@
+varnishtest	"trivial run of varnishadm in pass mode"
+
+feature term
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {} -start
+
+client c1 {
+	txreq
+	rxresp
+} -run
+
+varnish v1 -vsl_catchup
+
+process p1 -log {cat | varnishadm -n ${v1_name}} -start
+
+process p1 -write "ping\r"
+
+process p1 -need-bytes 30
+
+process p1 -write "vcl.list\r"
+
+process p1 -need-bytes 80
+
+process p1 -write "vcl.show vcl1\r"
+
+process p1 -need-bytes 100
+
+process p1 -screen_dump -write "\x04" -wait

From phk at FreeBSD.org  Sun Jan 14 08:50:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sun, 14 Jan 2018 08:50:09 +0000 (UTC)
Subject: [master] e3378e4 Increase fidelity of terminal emulation
Message-ID: <20180114085009.D6313A2828@lists.varnish-cache.org>


commit e3378e4eae22519bc3aefc2e4f700b7b10909f21
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Jan 14 08:47:58 2018 +0000

    Increase fidelity of terminal emulation

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index 37fa4b3..e97880b 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -335,7 +335,7 @@ process_init_term(struct process *p, int fd)
 	int i;
 
 	memset(&ws, 0, sizeof ws);
-	ws.ws_row = 24;
+	ws.ws_row = 25;
 	ws.ws_col = 80;
 	i = ioctl(fd, TIOCSWINSZ, &ws);
 	if (i)
diff --git a/bin/varnishtest/vtc_term.c b/bin/varnishtest/vtc_term.c
index 8897a61..6494ddb 100644
--- a/bin/varnishtest/vtc_term.c
+++ b/bin/varnishtest/vtc_term.c
@@ -90,12 +90,18 @@ term_escape(struct term *tp, int c, int n)
 {
 	int i;
 
-	for (i = 0; i < NTERMARG; i++)
-		if (!tp->arg[i])
-			tp->arg[i] = 1;
 	switch(c) {
+	case 'A':
+		// CUU - Cursor up
+		if (tp->arg[0] == -1) tp->arg[0] = 1;
+		tp->line -= tp->arg[0];
+		if (tp->line < 0)
+			vtc_fatal(tp->vl, "ANSI A[%d] outside vram",
+			    tp->arg[0]);
+		break;
 	case 'B':
 		// CUD - Cursor down
+		if (tp->arg[0] == -1) tp->arg[0] = 1;
 		if (tp->arg[0] > tp->nlin)
 			vtc_fatal(tp->vl, "ANSI B[%d] outside vram",
 			    tp->arg[0]);
@@ -107,6 +113,7 @@ term_escape(struct term *tp, int c, int n)
 		break;
 	case 'C':
 		// CUF - Cursor forward
+		if (tp->arg[0] == -1) tp->arg[0] = 1;
 		tp->col += tp->arg[0];
 		if (tp->col >= tp->ncol)
 			vtc_fatal(tp->vl, "ANSI C[%d] outside vram",
@@ -117,6 +124,8 @@ term_escape(struct term *tp, int c, int n)
 		break;
 	case 'H':
 		// CUP - Cursor Position
+		if (tp->arg[0] == -1) tp->arg[0] = 1;
+		if (tp->arg[1] == -1) tp->arg[1] = 1;
 		if (tp->arg[0] > tp->nlin || tp->arg[1] > tp->ncol)
 			vtc_fatal(tp->vl, "ANSI H[%d,%d] outside vram",
 			    tp->arg[0], tp->arg[1]);
@@ -135,9 +144,10 @@ term_escape(struct term *tp, int c, int n)
 		break;
 	case 'K':
 		// EL - Erase in line (0=right, 1=left, 2=full line)
+		if (tp->arg[0] == -1) tp->arg[0] = 0;
 		switch (tp->arg[0]) {
 		case 0:
-			for (i = tp->col + 1; i < tp->ncol; i++)
+			for (i = tp->col; i < tp->ncol; i++)
 				tp->vram[tp->line][i] = ' ';
 			break;
 		case 1:
@@ -209,6 +219,7 @@ term_char(struct term *tp, char c)
 void
 Term_Feed(struct term *tp, const char *b, const char *e)
 {
+	int i;
 
 	while (b < e) {
 		assert(tp->col < tp->ncol);
@@ -232,7 +243,8 @@ Term_Feed(struct term *tp, const char *b, const char *e)
 			break;
 		case 2:
 			tp->argp = tp->arg;
-			memset(tp->arg, 0, sizeof tp->arg);
+			for (i=0; i < NTERMARG; i++)
+				tp->arg[i] = -1;
 			tp->state = 3;
 			if (*b == '?')
 				b++;
@@ -242,6 +254,8 @@ Term_Feed(struct term *tp, const char *b, const char *e)
 				vtc_fatal(tp->vl, "ANSI too many args");
 
 			if (isdigit(*b)) {
+				if (*tp->argp == -1)
+					*tp->argp = 0;
 				*tp->argp *= 10;
 				*tp->argp += *b++ - '0';
 				continue;
@@ -270,7 +284,7 @@ Term_New(struct vtclog *vl)
 	ALLOC_OBJ(tp, TERM_MAGIC);
 	AN(tp);
 	tp->vl = vl;
-	tp->nlin = 24;
+	tp->nlin = 25;
 	tp->ncol = 80;
 	tp->vram = calloc(tp->nlin, sizeof *tp->vram);
 	AN(tp->vram);

From phk at FreeBSD.org  Sun Jan 14 08:50:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Sun, 14 Jan 2018 08:50:09 +0000 (UTC)
Subject: [master] c082ca6 Exercise more of varnishstat
Message-ID: <20180114085009.EC256A282B@lists.varnish-cache.org>


commit c082ca6823cbf0e4925d7d620b4e7fadd18c02a9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Sun Jan 14 08:48:14 2018 +0000

    Exercise more of varnishstat

diff --git a/bin/varnishtest/tests/u00008.vtc b/bin/varnishtest/tests/u00008.vtc
index def8166..43a69a4 100644
--- a/bin/varnishtest/tests/u00008.vtc
+++ b/bin/varnishtest/tests/u00008.vtc
@@ -2,12 +2,16 @@ varnishtest	"trivial run of varnistat in curses mode"
 
 feature term
 
-server s1 {
+server s1 -repeat 4 {
 	rxreq
 	txresp
 } -start
 
-varnish v1 -vcl+backend {} -start
+varnish v1 -vcl+backend {
+	probe default {
+		.url = "/";
+	}
+} -start
 
 process p1 -dump {varnishstat -n ${v1_name}} -start
 
@@ -18,5 +22,15 @@ client c1 {
 	rxresp
 } -run
 
-process p1 -need-bytes 2300 -screen_dump -write {q} -wait
+process p1 -need-bytes 1000
+
+process p1 -write {vG}
+
+varnish v1 -stop
+
+process p1 -write {dek}
+
+delay 1
+
+process p1 -need-bytes 5000 -screen_dump -write {q} -wait
 

From phk at FreeBSD.org  Mon Jan 15 07:31:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 15 Jan 2018 07:31:10 +0000 (UTC)
Subject: [master] 0d9f041 More terminal fidelity improvements.
Message-ID: <20180115073110.87365BBAB3@lists.varnish-cache.org>


commit 0d9f04186ccd979422fc5f31068e43f79c24eee7
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 15 07:30:36 2018 +0000

    More terminal fidelity improvements.

diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc
index d85d51e..02fca74 100644
--- a/bin/varnishtest/tests/a00000.vtc
+++ b/bin/varnishtest/tests/a00000.vtc
@@ -41,3 +41,14 @@ shell -exit 77 -expect {TEST _.vtc skipped} {
 	exec varnishtest -v _.vtc || true
 }
 
+process p1 "ps -lw | grep '[p][s]' ; tty ; sleep 1" -run -screen_dump
+
+process p2 "stty -a ; sleep 1" -run -screen_dump
+
+process p3 -hexdump {stty raw; echo "*" ; dd bs=254 count=1 | tee /tmp/_1} -start
+
+process p3 -need-bytes 2
+
+process p3 -write "\x1b[H\x1b[2J1\x1b[79C2\x08>\x1b[25;1H3\x1b[25;80H4\x08>\x1b[A\x1b[Cv\x1b[22A^\x1b[79D^\x1b[;2H<\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x08<\x1b[24;Hv\x1b[12;1H111111112222222333333\x0d\x0a111111112222222333333\x0d\x0a111111112222222333333\x0d\x0a111111112222222333333\x0d\x0a\x1b[12;12H\x1b[K\x1b[13;12H\x1b[0K\x1b[14;12H\x1b[1K\x1b[15;12H\x1b[2K\x1b[3;1Hline3 <\x0d\x0a"
+
+process p3 -need-bytes 256 -screen_dump -wait
diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc
index 1af8263..fe5b798 100644
--- a/bin/varnishtest/tests/u00000.vtc
+++ b/bin/varnishtest/tests/u00000.vtc
@@ -1,9 +1,5 @@
 varnishtest "Simple process tests"
 
-process p6 "ps -lw | grep '[p][s]' ; tty ; sleep 1" -run -screen_dump
-
-process p5 "stty -a ; sleep 1" -run -screen_dump
-
 process p1 "cat" -start
 process p1 -writeln "foo"
 process p1 -need-bytes 3
diff --git a/bin/varnishtest/vtc_term.c b/bin/varnishtest/vtc_term.c
index 6494ddb..cc9d862 100644
--- a/bin/varnishtest/vtc_term.c
+++ b/bin/varnishtest/vtc_term.c
@@ -116,8 +116,14 @@ term_escape(struct term *tp, int c, int n)
 		if (tp->arg[0] == -1) tp->arg[0] = 1;
 		tp->col += tp->arg[0];
 		if (tp->col >= tp->ncol)
-			vtc_fatal(tp->vl, "ANSI C[%d] outside vram",
-			    tp->arg[0]);
+			tp->col = tp->ncol - 1;
+		break;
+	case 'D':
+		// CUB - Cursor backward
+		if (tp->arg[0] == -1) tp->arg[0] = 1;
+		tp->col -= tp->arg[0];
+		if (tp->col < 0)
+			tp->col = 0;
 		break;
 	case 'h':
 		// SM - Set Mode (ignored XXX?)
@@ -206,13 +212,9 @@ term_char(struct term *tp, char c)
 	default:
 		if (c < ' ' || c > '~')
 			c = '?';
-		tp->vram[tp->line][tp->col] = c;
-		if (tp->col == tp->ncol - 1) {
-			tp->col = 0;
-			term_char(tp, '\n');
-		} else {
-			tp->col++;
-		}
+		tp->vram[tp->line][tp->col++] = c;
+		if (tp->col >= tp->ncol)
+			tp->col = tp->ncol - 1;
 	}
 }
 

From phk at FreeBSD.org  Mon Jan 15 07:57:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 15 Jan 2018 07:57:06 +0000 (UTC)
Subject: [master] 79ed4fc Try to catch a panic on VCO
Message-ID: <20180115075706.A1ABCBC24F@lists.varnish-cache.org>


commit 79ed4fce4c0ea4ee6d159e9c085b9d49dda512bf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 15 07:55:42 2018 +0000

    Try to catch a panic on VCO

diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c
index 9451952..252b631 100644
--- a/bin/varnishd/http2/cache_http2_session.c
+++ b/bin/varnishd/http2/cache_http2_session.c
@@ -315,6 +315,11 @@ h2_new_session(struct worker *wrk, void *arg)
 	while (h2_rxframe(wrk, h2)) {
 		WS_Reset(h2->ws, wsp);
 		HTC_RxInit(h2->htc, h2->ws);
+		if (WS_Overflowed(h2->ws)) {
+			VSLb(h2->vsl, SLT_Debug, "H2: Empty Rx Workspace");
+			h2->error = H2CE_INTERNAL_ERROR;
+			break;
+		}
 	}
 
 	AN(h2->error);

From phk at FreeBSD.org  Mon Jan 15 09:32:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 15 Jan 2018 09:32:06 +0000 (UTC)
Subject: [master] baa7e69 Improve the error messages for 'vcl N.N;'
Message-ID: <20180115093206.655CF64AFD@lists.varnish-cache.org>


commit baa7e69d743d469287c7e36040aaf944bc23616a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 15 08:58:15 2018 +0000

    Improve the error messages for 'vcl N.N;'
    
    Some specialcasing is warranted to get sensible messages.
    
    Fixes #2532

diff --git a/bin/varnishtest/tests/v00049.vtc b/bin/varnishtest/tests/v00049.vtc
index 35a3efc..46cf468 100644
--- a/bin/varnishtest/tests/v00049.vtc
+++ b/bin/varnishtest/tests/v00049.vtc
@@ -15,3 +15,13 @@ varnish v1 -syntax 4.0 -errvcl "9.9 higher than the top level version" {
 	vcl 9.9;
 	backend b1 { .host = "127.0.0.1:8080"; }
 }
+
+varnish v1 -cliexpect {Don't play silly buggers with VCL version numbers} \
+	{vcl.inline t0 "vcl 4.00 ; backend b { .host = \"localhost\";} "}
+
+varnish v1 -cliexpect {Don't play silly buggers with VCL version numbers} \
+	{vcl.inline t1 "vcl 04.0 ; backend b { .host = \"localhost\";} "}
+
+varnish v1 -cliexpect {Expected 'vcl N.N;' found no semi-colon} \
+	{vcl.inline t2 "vcl 4.0  backend b { .host = \"localhost\";} "}
+
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index d0c040c..cff6c8f 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -310,7 +310,6 @@ void Resolve_Sockaddr(struct vcc *tl, const char *host, const char *defport,
 double vcc_TimeUnit(struct vcc *);
 void vcc_ByteVal(struct vcc *, double *);
 void vcc_NumVal(struct vcc *, double *, int *);
-double vcc_DoubleVal(struct vcc *tl);
 void vcc_Duration(struct vcc *tl, double *);
 unsigned vcc_UintVal(struct vcc *tl);
 
diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index 94e6279..9652077 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -274,29 +274,50 @@ vcc_ParseFunction(struct vcc *tl)
 static void
 vcc_ParseVcl(struct vcc *tl)
 {
-	struct token *tok;
+	struct token *tok0, *tok1, *tok2;
 
 	assert(vcc_IdIs(tl->t, "vcl"));
+	tok0 = tl->t;
 	vcc_NextToken(tl);
-	tok = tl->t;
-	tok->src->syntax = vcc_DoubleVal(tl);
-	ERRCHK(tl);
-	if (tl->t->e - tok->b > 4) {
+
+	tok1 = tl->t;
+	Expect(tl, CNUM);
+	tok1->src->syntax = *tl->t->b - '0';
+	vcc_NextToken(tl);
+	Expect(tl, '.');
+	vcc_NextToken(tl);
+
+	Expect(tl, CNUM);
+	tok2 = tl->t;
+	tok1->src->syntax += .1 * (*tl->t->b - '0');
+	vcc_NextToken(tl);
+
+	if (tok1->e - tok1->b != 1 || tok2->e - tok2->b != 1) {
 		VSB_printf(tl->sb,
 		    "Don't play silly buggers with VCL version numbers\n");
-		vcc_ErrWhere2(tl, tok, tl->t);
+		vcc_ErrWhere2(tl, tok0, tl->t);
+		ERRCHK(tl);
+	}
+
+	if (tl->t->tok != ';') {
+		/* Special handling, because next token might be 'vcl'
+		 * in the built-in VCL, and that would give a very
+		 * confusing error message
+		 */
+		VSB_printf(tl->sb,
+		    "Expected 'vcl N.N;' found no semi-colon\n");
+		vcc_ErrWhere2(tl, tok0, tl->t);
 		ERRCHK(tl);
 	}
-	if (tl->syntax != 0.0 && tok->src->syntax > tl->syntax) {
+	vcc_NextToken(tl);
+	if (tl->syntax != 0.0 && tok1->src->syntax > tl->syntax) {
 		VSB_printf(tl->sb,
 		    "VCL version %.1f higher than"
 		    " the top level version %.1f\n",
-		    tok->src->syntax, tl->syntax);
-		vcc_ErrWhere2(tl, tok, tl->t);
+		    tok1->src->syntax, tl->syntax);
+		vcc_ErrWhere2(tl, tok0, tl->t);
 		ERRCHK(tl);
 	}
-	ExpectErr(tl, ';');
-	vcc_NextToken(tl);
 }
 
 /*--------------------------------------------------------------------
diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c
index a793c62..a2ae14d 100644
--- a/lib/libvcc/vcc_utils.c
+++ b/lib/libvcc/vcc_utils.c
@@ -344,7 +344,7 @@ vcc_NumVal(struct vcc *tl, double *d, int *frac)
 	vcc_NextToken(tl);
 }
 
-double
+static double
 vcc_DoubleVal(struct vcc *tl)
 {
 	double d;

From nils.goroll at uplex.de  Mon Jan 15 18:10:06 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 15 Jan 2018 18:10:06 +0000 (UTC)
Subject: [master] 69695c2 Fix pointer arithmatic in VSM_Map(),
 add a test with 100 backends
Message-ID: <20180115181007.04748A0AFB@lists.varnish-cache.org>


commit 69695c23e53942968fd65cd4a8627d47d4e21a70
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon Jan 15 19:00:32 2018 +0100

    Fix pointer arithmatic in VSM_Map(), add a test with 100 backends
    
    Of a VSM segment, we map the least number of pages from the offset into the file
    covering the segment length, aligned to page boundaries. For calculating
    relative pointers into the mapping and the length, we need to deduce the initial
    page boundary offset.
    
    Fixes #2541

diff --git a/bin/varnishtest/tests/b00025.vtc b/bin/varnishtest/tests/b00025.vtc
new file mode 100644
index 0000000..1d23c71
--- /dev/null
+++ b/bin/varnishtest/tests/b00025.vtc
@@ -0,0 +1,113 @@
+varnishtest "more backends"
+
+varnish v1 -arg "-p vcc_err_unref=off" -vcl {
+	backend d0 { .host = "${bad_backend}"; }
+	backend d1 { .host = "${bad_backend}"; }
+	backend d2 { .host = "${bad_backend}"; }
+	backend d3 { .host = "${bad_backend}"; }
+	backend d4 { .host = "${bad_backend}"; }
+	backend d5 { .host = "${bad_backend}"; }
+	backend d6 { .host = "${bad_backend}"; }
+	backend d7 { .host = "${bad_backend}"; }
+	backend d8 { .host = "${bad_backend}"; }
+	backend d9 { .host = "${bad_backend}"; }
+	backend d10 { .host = "${bad_backend}"; }
+	backend d11 { .host = "${bad_backend}"; }
+	backend d12 { .host = "${bad_backend}"; }
+	backend d13 { .host = "${bad_backend}"; }
+	backend d14 { .host = "${bad_backend}"; }
+	backend d15 { .host = "${bad_backend}"; }
+	backend d16 { .host = "${bad_backend}"; }
+	backend d17 { .host = "${bad_backend}"; }
+	backend d18 { .host = "${bad_backend}"; }
+	backend d19 { .host = "${bad_backend}"; }
+	backend d20 { .host = "${bad_backend}"; }
+	backend d21 { .host = "${bad_backend}"; }
+	backend d22 { .host = "${bad_backend}"; }
+	backend d23 { .host = "${bad_backend}"; }
+	backend d24 { .host = "${bad_backend}"; }
+	backend d25 { .host = "${bad_backend}"; }
+	backend d26 { .host = "${bad_backend}"; }
+	backend d27 { .host = "${bad_backend}"; }
+	backend d28 { .host = "${bad_backend}"; }
+	backend d29 { .host = "${bad_backend}"; }
+	backend d30 { .host = "${bad_backend}"; }
+	backend d31 { .host = "${bad_backend}"; }
+	backend d32 { .host = "${bad_backend}"; }
+	backend d33 { .host = "${bad_backend}"; }
+	backend d34 { .host = "${bad_backend}"; }
+	backend d35 { .host = "${bad_backend}"; }
+	backend d36 { .host = "${bad_backend}"; }
+	backend d37 { .host = "${bad_backend}"; }
+	backend d38 { .host = "${bad_backend}"; }
+	backend d39 { .host = "${bad_backend}"; }
+	backend d40 { .host = "${bad_backend}"; }
+	backend d41 { .host = "${bad_backend}"; }
+	backend d42 { .host = "${bad_backend}"; }
+	backend d43 { .host = "${bad_backend}"; }
+	backend d44 { .host = "${bad_backend}"; }
+	backend d45 { .host = "${bad_backend}"; }
+	backend d46 { .host = "${bad_backend}"; }
+	backend d47 { .host = "${bad_backend}"; }
+	backend d48 { .host = "${bad_backend}"; }
+	backend d49 { .host = "${bad_backend}"; }
+	backend d50 { .host = "${bad_backend}"; }
+	backend d51 { .host = "${bad_backend}"; }
+	backend d52 { .host = "${bad_backend}"; }
+	backend d53 { .host = "${bad_backend}"; }
+	backend d54 { .host = "${bad_backend}"; }
+	backend d55 { .host = "${bad_backend}"; }
+	backend d56 { .host = "${bad_backend}"; }
+	backend d57 { .host = "${bad_backend}"; }
+	backend d58 { .host = "${bad_backend}"; }
+	backend d59 { .host = "${bad_backend}"; }
+	backend d60 { .host = "${bad_backend}"; }
+	backend d61 { .host = "${bad_backend}"; }
+	backend d62 { .host = "${bad_backend}"; }
+	backend d63 { .host = "${bad_backend}"; }
+	backend d64 { .host = "${bad_backend}"; }
+	backend d65 { .host = "${bad_backend}"; }
+	backend d66 { .host = "${bad_backend}"; }
+	backend d67 { .host = "${bad_backend}"; }
+	backend d68 { .host = "${bad_backend}"; }
+	backend d69 { .host = "${bad_backend}"; }
+	backend d70 { .host = "${bad_backend}"; }
+	backend d71 { .host = "${bad_backend}"; }
+	backend d72 { .host = "${bad_backend}"; }
+	backend d73 { .host = "${bad_backend}"; }
+	backend d74 { .host = "${bad_backend}"; }
+	backend d75 { .host = "${bad_backend}"; }
+	backend d76 { .host = "${bad_backend}"; }
+	backend d77 { .host = "${bad_backend}"; }
+	# end of 1st 8k VSMW cluster on 64bit
+	backend d78 { .host = "${bad_backend}"; }
+	backend d79 { .host = "${bad_backend}"; }
+	backend d80 { .host = "${bad_backend}"; }
+	backend d81 { .host = "${bad_backend}"; }
+	backend d82 { .host = "${bad_backend}"; }
+	backend d83 { .host = "${bad_backend}"; }
+	backend d84 { .host = "${bad_backend}"; }
+	backend d85 { .host = "${bad_backend}"; }
+	backend d86 { .host = "${bad_backend}"; }
+	backend d87 { .host = "${bad_backend}"; }
+	backend d88 { .host = "${bad_backend}"; }
+	backend d89 { .host = "${bad_backend}"; }
+	backend d90 { .host = "${bad_backend}"; }
+	backend d91 { .host = "${bad_backend}"; }
+	backend d92 { .host = "${bad_backend}"; }
+	backend d93 { .host = "${bad_backend}"; }
+	backend d94 { .host = "${bad_backend}"; }
+	backend d95 { .host = "${bad_backend}"; }
+	backend d96 { .host = "${bad_backend}"; }
+	backend d97 { .host = "${bad_backend}"; }
+	backend d98 { .host = "${bad_backend}"; }
+	backend d99 { .host = "${bad_backend}"; }
+} -start
+
+client c1 {
+	txreq -url "/"
+	rxresp
+	expect resp.status == 503
+} -run
+
+shell -match "d99" "varnishstat -1 -n ${v1_name}"
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 889a25e..282865b 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -718,7 +718,7 @@ VSM_Map(struct vsm *vd, struct vsm_fantom *vf)
 
 	sz = strtoul(vg->av[3], NULL, 10);
 	assert(sz > 0);
-	len = RUP2(of + sz, ps);
+	len = RUP2(sz, ps);
 
 	vsb = VSB_new_auto();
 	AN(vsb);
@@ -742,7 +742,7 @@ VSM_Map(struct vsm *vd, struct vsm_fantom *vf)
 	if (vg->s == MAP_FAILED)
 		return (vsm_diag(vd, "Could not mmap segment"));
 
-	vg->b = (char*)(vg->s) + of;
+	vg->b = (char*)(vg->s) + of - off;
 	vg->e = (char *)vg->b + sz;
 	vg->sz = len;
 

From phk at FreeBSD.org  Mon Jan 15 22:15:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 15 Jan 2018 22:15:10 +0000 (UTC)
Subject: [master] aa21e4b I may be incompetent at math,
 but I'm not _that_ incompetent at math :-)
Message-ID: <20180115221510.8BE79A6B9B@lists.varnish-cache.org>


commit aa21e4b2f8c3b884c4a9c32ef74ed5625e4c6916
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 15 22:14:14 2018 +0000

    I may be incompetent at math, but I'm not _that_ incompetent at math :-)

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 282865b..cb8118b 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -718,7 +718,7 @@ VSM_Map(struct vsm *vd, struct vsm_fantom *vf)
 
 	sz = strtoul(vg->av[3], NULL, 10);
 	assert(sz > 0);
-	len = RUP2(sz, ps);
+	len = RUP2(of + sz, ps);
 
 	vsb = VSB_new_auto();
 	AN(vsb);

From phk at FreeBSD.org  Tue Jan 16 06:32:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 16 Jan 2018 06:32:08 +0000 (UTC)
Subject: [master] 6f337c4 Dump sp->ws on panic
Message-ID: <20180116063208.2A454B0D1C@lists.varnish-cache.org>


commit 6f337c4148a06d760dbdb4e33f741b5eee5ed7f8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 16 06:31:07 2018 +0000

    Dump sp->ws on panic

diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c
index b13ae48..90050ac 100644
--- a/bin/varnishd/cache/cache_panic.c
+++ b/bin/varnishd/cache/cache_panic.c
@@ -506,6 +506,7 @@ pan_sess(struct vsb *vsb, const struct sess *sp)
 	    sp->fd, VXID(sp->vxid));
 	VSB_printf(vsb, "t_open = %f,\n", sp->t_open);
 	VSB_printf(vsb, "t_idle = %f,\n", sp->t_idle);
+	pan_ws(vsb, sp->ws);
 	VSB_printf(vsb, "transport = %s",
 	    xp == NULL ? "<none>" : xp->name);
 	if (xp != NULL && xp->sess_panic != NULL) {

From phk at FreeBSD.org  Tue Jan 16 06:36:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 16 Jan 2018 06:36:06 +0000 (UTC)
Subject: [master] 77d819c Removing debugging hack
Message-ID: <20180116063606.9327BB0F6E@lists.varnish-cache.org>


commit 77d819c9a11033d300b7a87e0bac95dbe6220291
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 16 06:35:27 2018 +0000

    Removing debugging hack

diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc
index 02fca74..c191742 100644
--- a/bin/varnishtest/tests/a00000.vtc
+++ b/bin/varnishtest/tests/a00000.vtc
@@ -45,7 +45,7 @@ process p1 "ps -lw | grep '[p][s]' ; tty ; sleep 1" -run -screen_dump
 
 process p2 "stty -a ; sleep 1" -run -screen_dump
 
-process p3 -hexdump {stty raw; echo "*" ; dd bs=254 count=1 | tee /tmp/_1} -start
+process p3 -hexdump {stty raw; echo "*" ; dd bs=254 count=1} -start
 
 process p3 -need-bytes 2
 

From daghf at varnish-software.com  Tue Jan 16 08:28:05 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Tue, 16 Jan 2018 08:28:05 +0000 (UTC)
Subject: [master] e2dd109 Expose various H2_SETTINGs as parameters
Message-ID: <20180116082805.D4D94B2CEA@lists.varnish-cache.org>


commit e2dd109f77b3cdd82c76cefdecab88f5d383fd23
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Mon Jan 8 14:36:56 2018 +0100

    Expose various H2_SETTINGs as parameters

diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c
index 252b631..afa77fc 100644
--- a/bin/varnishd/http2/cache_http2_session.c
+++ b/bin/varnishd/http2/cache_http2_session.c
@@ -62,6 +62,16 @@ static const struct h2_settings H2_proto_settings = {
 #include "tbl/h2_settings.h"
 };
 
+static void
+h2_local_settings(struct h2_settings *h2s)
+{
+	*h2s = H2_proto_settings;
+#define H2_SETTINGS_PARAM_ONLY
+#define H2_SETTING(U, l, ...)			\
+	h2s->l = cache_param->h2_##l;
+#include "tbl/h2_settings.h"
+#undef H2_SETTINGS_PARAM_ONLY
+}
 
 /**********************************************************************
  * The h2_sess struct needs many of the same things as a request,
@@ -101,7 +111,7 @@ h2_new_sess(const struct worker *wrk, struct sess *sp, struct req *srq)
 		h2->rxthr = pthread_self();
 		VTAILQ_INIT(&h2->streams);
 		VTAILQ_INIT(&h2->txqueue);
-		h2->local_settings = H2_proto_settings;
+		h2_local_settings(&h2->local_settings);
 		h2->remote_settings = H2_proto_settings;
 
 		AZ(VHT_Init(h2->dectbl,
diff --git a/include/tbl/h2_settings.h b/include/tbl/h2_settings.h
index 7214bef..04c24f3 100644
--- a/include/tbl/h2_settings.h
+++ b/include/tbl/h2_settings.h
@@ -47,6 +47,7 @@ H2_SETTING(					// rfc7540,l,2097,2103
 	0xffffffff,
 	0
 )
+#ifndef H2_SETTINGS_PARAM_ONLY
 H2_SETTING(					// rfc7540,l,2105,2114
 	ENABLE_PUSH,
 	enable_push,
@@ -56,6 +57,7 @@ H2_SETTING(					// rfc7540,l,2105,2114
 	1,
 	H2CE_PROTOCOL_ERROR
 )
+#endif
 H2_SETTING(					// rfc7540,l,2116,2121
 	MAX_CONCURRENT_STREAMS,
 	max_concurrent_streams,
diff --git a/include/tbl/params.h b/include/tbl/params.h
index 454021d..436c4af 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -1732,6 +1732,80 @@ PARAM(
 	/* func */	NULL
 )
 
+PARAM(
+	/* name */      h2_header_table_size,
+	/* typ */       bytes_u,
+	/* min */       "0",
+	/* max */       NULL,
+	/* default */   "4k",
+	/* units */     "bytes",
+	/* flags */     0,
+	/* s-text */
+	"HTTP2 header table size.\n"
+	"This is the size that will be used for the HPACK dynamic\n"
+	"decoding table.",
+	/* l-text */    "",
+	/* func */      NULL
+)
+
+PARAM(
+       /* name */      h2_max_concurrent_streams,
+       /* typ */       uint,
+       /* min */       "0",
+       /* max */       "4294967294",
+       /* default */   "100",
+       /* units */     "streams",
+       /* flags */     0,
+       /* s-text */
+       "HTTP2 Maximum number of concurrent streams.\n"
+       "This is the number of requests that can be active\n"
+       "at the same time for a single HTTP2 connection.",
+       /* l-text */    "",
+       /* func */      NULL
+)
+
+PARAM(
+	/* name */      h2_initial_window_size,
+	/* typ */       bytes_u,
+	/* min */       "0",
+	/* max */       "2147483647",
+	/* default */   "65535",
+	/* units */     "bytes",
+	/* flags */     0,
+	/* s-text */
+	"HTTP2 initial flow control window size.",
+	/* l-text */    "",
+	/* func */      NULL
+)
+
+PARAM(
+	/* name */      h2_max_frame_size,
+	/* typ */       bytes_u,
+	/* min */       "16384",
+	/* max */       "16777215",
+	/* default */   "16384",
+	/* units */     "bytes",
+	/* flags */     0,
+	/* s-text */
+	"HTTP2 maximum per frame payload size we are willing to accept.",
+	/* l-text */    "",
+	/* func */      NULL
+)
+
+PARAM(
+	/* name */      h2_max_header_list_size,
+	/* typ */       bytes_u,
+	/* min */       "0",
+	/* max */       NULL,
+	/* default */   "4294967295",
+	/* units */     "bytes",
+	/* flags */     0,
+	/* s-text */
+	"HTTP2 maximum size of an uncompressed header list.",
+	/* l-text */    "",
+	/* func */      NULL
+)
+
 #undef PARAM
 
 /*lint -restore */

From daghf at varnish-software.com  Tue Jan 16 08:28:05 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Tue, 16 Jan 2018 08:28:05 +0000 (UTC)
Subject: [master] 63688fa H2: Transmit our actual settings set
Message-ID: <20180116082805.EDE32B2CED@lists.varnish-cache.org>


commit 63688fa14cba5a9b638e66a2c2c3fd376691bec7
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Mon Jan 15 10:57:51 2018 +0100

    H2: Transmit our actual settings set
    
    We only need to transmit the ones where we differ from rfc7540's initial
    values.

diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c
index afa77fc..cebcf8e 100644
--- a/bin/varnishd/http2/cache_http2_session.c
+++ b/bin/varnishd/http2/cache_http2_session.c
@@ -36,6 +36,7 @@
 #include "cache/cache_transport.h"
 #include "http2/cache_http2.h"
 
+#include "vend.h"
 #include "vtim.h"
 
 static const char h2_resp_101[] =
@@ -50,12 +51,25 @@ static const char H2_prism[24] = {
 	0x0d, 0x0a, 0x53, 0x4d, 0x0d, 0x0a, 0x0d, 0x0a
 };
 
-static const uint8_t H2_settings[] = {
-	0x00, 0x03,
-	0x00, 0x00, 0x00, 0x64,
-	0x00, 0x04,
-	0x00, 0x00, 0xff, 0xff
-};
+static size_t
+h2_enc_settings(const struct h2_settings *h2s, uint8_t *buf, size_t n)
+{
+	uint8_t *b;
+	size_t len = 0;
+
+	b = buf;
+#define H2_SETTING(U,l,v,d,...)				\
+	if (h2s->l != d) {				\
+		len += 6;					\
+		assert(len <= n);			\
+		vbe16enc(b, v);				\
+		b += 2;					\
+		vbe32enc(b, h2s->l);			\
+		b += 4;					\
+	}
+#include "tbl/h2_settings.h"
+	return (len);
+}
 
 static const struct h2_settings H2_proto_settings = {
 #define H2_SETTING(U,l,v,d,...) . l = d,
@@ -289,6 +303,8 @@ h2_new_session(struct worker *wrk, void *arg)
 	struct h2_req *r2, *r22;
 	uintptr_t wsp;
 	int again;
+	uint8_t settings[48];
+	size_t l;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	CAST_OBJ_NOTNULL(req, arg, REQ_MAGIC);
@@ -314,9 +330,10 @@ h2_new_session(struct worker *wrk, void *arg)
 
 	THR_SetRequest(h2->srq);
 
+	l = h2_enc_settings(&h2->local_settings, settings, sizeof (settings));
 	H2_Send_Get(wrk, h2, h2->req0);
 	H2_Send_Frame(wrk, h2,
-	    H2_F_SETTINGS, H2FF_NONE, sizeof H2_settings, 0, H2_settings);
+	    H2_F_SETTINGS, H2FF_NONE, l, 0, settings);
 	H2_Send_Rel(h2, h2->req0);
 
 	/* and off we go... */
diff --git a/bin/varnishtest/tests/t02005.vtc b/bin/varnishtest/tests/t02005.vtc
index 81ad091..c80432b 100644
--- a/bin/varnishtest/tests/t02005.vtc
+++ b/bin/varnishtest/tests/t02005.vtc
@@ -15,7 +15,7 @@ varnish v1 -cliok "param.set debug +syncvsl"
 
 logexpect l1 -v v1 -g raw {
 	expect	* 1001 ReqAcct	"80 7 87 106 8 114"
-	expect	* 1000 ReqAcct	"45 8 53 72 28 100"
+	expect	* 1000 ReqAcct	"45 8 53 72 22 94"
 } -start
 
 client c1 {

From daghf at varnish-software.com  Tue Jan 16 08:32:06 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Tue, 16 Jan 2018 08:32:06 +0000 (UTC)
Subject: [master] f111f85 Whitespace
Message-ID: <20180116083206.0D7D8B30BB@lists.varnish-cache.org>


commit f111f85b2689551e1634f37ef015f44c1f591d3b
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Tue Jan 16 09:30:59 2018 +0100

    Whitespace

diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c
index cebcf8e..e76b0b2 100644
--- a/bin/varnishd/http2/cache_http2_session.c
+++ b/bin/varnishd/http2/cache_http2_session.c
@@ -60,7 +60,7 @@ h2_enc_settings(const struct h2_settings *h2s, uint8_t *buf, size_t n)
 	b = buf;
 #define H2_SETTING(U,l,v,d,...)				\
 	if (h2s->l != d) {				\
-		len += 6;					\
+		len += 6;				\
 		assert(len <= n);			\
 		vbe16enc(b, v);				\
 		b += 2;					\

From daghf at varnish-software.com  Tue Jan 16 09:29:06 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Tue, 16 Jan 2018 09:29:06 +0000 (UTC)
Subject: [master] 5306a71 h2: Revise some param values to appease vtest's i386
 builds
Message-ID: <20180116092906.8BADDB4166@lists.varnish-cache.org>


commit 5306a71a4148ecf5ecfe19d3304a1f067b5445e0
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Tue Jan 16 10:28:22 2018 +0100

    h2: Revise some param values to appease vtest's i386 builds

diff --git a/include/tbl/h2_settings.h b/include/tbl/h2_settings.h
index 04c24f3..adfb416 100644
--- a/include/tbl/h2_settings.h
+++ b/include/tbl/h2_settings.h
@@ -89,7 +89,7 @@ H2_SETTING(					// rfc7540,l,2159,2167
 	MAX_HEADER_LIST_SIZE,
 	max_header_list_size,
 	0x6,
-	0xffffffff,				// rfc7540,l,4229,4229
+	0x7fffffff,				// rfc7540,l,4229,4229
 	0,
 	0xffffffff,
 	0
diff --git a/include/tbl/params.h b/include/tbl/params.h
index 436c4af..8aaef7d 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -1735,7 +1735,7 @@ PARAM(
 PARAM(
 	/* name */      h2_header_table_size,
 	/* typ */       bytes_u,
-	/* min */       "0",
+	/* min */       "0b",
 	/* max */       NULL,
 	/* default */   "4k",
 	/* units */     "bytes",
@@ -1752,7 +1752,7 @@ PARAM(
        /* name */      h2_max_concurrent_streams,
        /* typ */       uint,
        /* min */       "0",
-       /* max */       "4294967294",
+       /* max */       NULL,
        /* default */   "100",
        /* units */     "streams",
        /* flags */     0,
@@ -1768,8 +1768,8 @@ PARAM(
 	/* name */      h2_initial_window_size,
 	/* typ */       bytes_u,
 	/* min */       "0",
-	/* max */       "2147483647",
-	/* default */   "65535",
+	/* max */       "2147483647b",
+	/* default */   "65535b",
 	/* units */     "bytes",
 	/* flags */     0,
 	/* s-text */
@@ -1781,9 +1781,9 @@ PARAM(
 PARAM(
 	/* name */      h2_max_frame_size,
 	/* typ */       bytes_u,
-	/* min */       "16384",
-	/* max */       "16777215",
-	/* default */   "16384",
+	/* min */       "16k",
+	/* max */       "16777215b",
+	/* default */   "16k",
 	/* units */     "bytes",
 	/* flags */     0,
 	/* s-text */
@@ -1795,9 +1795,9 @@ PARAM(
 PARAM(
 	/* name */      h2_max_header_list_size,
 	/* typ */       bytes_u,
-	/* min */       "0",
+	/* min */       "0b",
 	/* max */       NULL,
-	/* default */   "4294967295",
+	/* default */   "2147483647b",
 	/* units */     "bytes",
 	/* flags */     0,
 	/* s-text */

From phk at FreeBSD.org  Tue Jan 16 09:48:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 16 Jan 2018 09:48:07 +0000 (UTC)
Subject: [master] 9b6115d Set the timeout for the same window we call getch
 for.
Message-ID: <20180116094807.0EA14B47BC@lists.varnish-cache.org>


commit 9b6115d5730a3ee9c722e992853f66480473cf57
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 16 07:49:47 2018 +0000

    Set the timeout for the same window we call getch for.

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index 56d442f..ce07b7f 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -1020,9 +1020,10 @@ do_curses(struct vsm *vsm, struct vsc *vsc, double delay)
 			draw_screen();
 
 		t = (t_sample + interval - now) * 1000;
-		timeout(t);
+		wtimeout(w_status, t);
 
-		switch (ch = wgetch(w_status)) {
+		ch = wgetch(w_status);
+		switch (ch) {
 		case ERR:
 			break;
 #ifdef KEY_RESIZE /* sigh, Solaris lacks this.. */

From phk at FreeBSD.org  Tue Jan 16 09:48:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 16 Jan 2018 09:48:07 +0000 (UTC)
Subject: [master] 5e49114 Flexelinting
Message-ID: <20180116094807.2BE87B47C0@lists.varnish-cache.org>


commit 5e49114dab05220dc00033ac0d8e6f0c46bb1d3f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 16 09:02:16 2018 +0000

    Flexelinting

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index cb8118b..be15a14 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -734,7 +734,7 @@ VSM_Map(struct vsm *vd, struct vsm_fantom *vf)
 	vg->s = (void*)mmap(NULL, len,
 	    PROT_READ,
 	    MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED,
-	    fd, off);
+	    fd, (off_t)off);
 
 	VSB_destroy(&vsb);
 

From phk at FreeBSD.org  Tue Jan 16 09:48:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 16 Jan 2018 09:48:07 +0000 (UTC)
Subject: [master] ec2dc56 Check exit status precisely
Message-ID: <20180116094807.4501FB47C4@lists.varnish-cache.org>


commit ec2dc561fa7cfde3953ff9734b4a6a9f79a53264
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 16 09:46:13 2018 +0000

    Check exit status precisely

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index e97880b..df950cd 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -76,6 +76,8 @@ struct process {
 	int			log;
 	pid_t			pid;
 	int			expect_exit;
+	int			expect_signal;
+	int			allow_core;
 
 	uintmax_t		stdout_bytes;
 	uintmax_t		stderr_bytes;
@@ -245,7 +247,7 @@ process_thread(void *priv)
 	struct rusage ru;
 	struct vev_root *evb;
 	struct vev *ev;
-	int r;
+	int core, sig, ext, r;
 
 	CAST_OBJ_NOTNULL(p, priv, PROCESS_MAGIC);
 
@@ -306,18 +308,28 @@ process_thread(void *priv)
 	);
 
 	AZ(pthread_mutex_unlock(&p->mtx));
-
+	sig = WTERMSIG(p->status);
+	ext = WEXITSTATUS(p->status);
 #ifdef WCOREDUMP
+	core = WCOREDUMP(p->status);
 	vtc_log(p->vl, 2, "Exit code: %04x sig %d exit %d core %d",
-	    p->status, WTERMSIG(p->status), WEXITSTATUS(p->status),
-	    WCOREDUMP(p->status));
+	    p->status, sig, ext, core);
 #else
+	core = 0;
 	vtc_log(p->vl, 2, "Exit code: %04x sig %d exit %d",
-	    p->status, WTERMSIG(p->status), WEXITSTATUS(p->status));
+	    p->status, sig, ext);
 #endif
-	if (WEXITSTATUS(p->status) != p->expect_exit)
+	if (core && !p->allow_core)
+		vtc_fatal(p->vl, "Core dump");
+	if (p->expect_signal >= 0 && sig != p->expect_signal)
+		vtc_fatal(p->vl, "Expected signal %d got %d",
+			p->expect_signal, sig);
+	else if (sig && sig != p->expect_signal)
+		vtc_fatal(p->vl, "Expected signal %d got %d",
+			-p->expect_signal, sig);
+	if (ext != p->expect_exit)
 		vtc_fatal(p->vl, "Expected exit %d got %d",
-			p->expect_exit, WEXITSTATUS(p->status));
+			p->expect_exit, ext);
 
 	VEV_Destroy(&evb);
 	if (p->log == 1) {
@@ -490,11 +502,14 @@ process_kill(struct process *p, const char *sig)
 	else
 		vtc_fatal(p->vl, "Could not grok signal (%s)", sig);
 
+	if (p->expect_signal == 0)
+		p->expect_signal = -j;
 	if (kill(-pid, j) < 0)
 		vtc_fatal(p->vl, "Failed to send signal %d (%s)",
 		    j, strerror(errno));
-	else
+	else {
 		vtc_log(p->vl, 4, "Sent signal %d", j);
+	}
 }
 
 /**********************************************************************
@@ -659,15 +674,12 @@ cmd_process(CMD_ARGS)
 		if (vtc_error)
 			break;
 
-		if (!strcmp(*av, "-start")) {
-			process_start(p);
+		if (!strcmp(*av, "-allow-core")) {
+			p->allow_core = 1;
 			continue;
 		}
-		if (!strcmp(*av, "-hexdump")) {
-			if (p->hasthread)
-				vtc_fatal(p->vl,
-				    "Cannot dump a running process");
-			p->log = 3;
+		if (!strcmp(*av, "-close")) {
+			process_close(p);
 			continue;
 		}
 		if (!strcmp(*av, "-dump")) {
@@ -677,25 +689,21 @@ cmd_process(CMD_ARGS)
 			p->log = 2;
 			continue;
 		}
-		if (!strcmp(*av, "-log")) {
-			if (p->hasthread)
-				vtc_fatal(p->vl,
-				    "Cannot log a running process");
-			p->log = 1;
-			continue;
-		}
 		if (!strcmp(*av, "-expect-exit")) {
 			p->expect_exit = strtoul(av[1], NULL, 0);
 			av++;
 			continue;
 		}
-		if (!strcmp(*av, "-wait")) {
-			process_wait(p);
+		if (!strcmp(*av, "-expect-signal")) {
+			p->expect_signal = strtoul(av[1], NULL, 0);
+			av++;
 			continue;
 		}
-		if (!strcmp(*av, "-run")) {
-			process_start(p);
-			process_wait(p);
+		if (!strcmp(*av, "-hexdump")) {
+			if (p->hasthread)
+				vtc_fatal(p->vl,
+				    "Cannot dump a running process");
+			p->log = 3;
 			continue;
 		}
 		if (!strcmp(*av, "-kill")) {
@@ -703,19 +711,11 @@ cmd_process(CMD_ARGS)
 			av++;
 			continue;
 		}
-		if (!strcmp(*av, "-stop")) {
-			process_kill(p, "TERM");
-			continue;
-		}
-		if (!strcmp(*av, "-write")) {
-			process_write(p, av[1]);
-			av++;
-			continue;
-		}
-		if (!strcmp(*av, "-writeln")) {
-			process_write(p, av[1]);
-			process_write(p, "\n");
-			av++;
+		if (!strcmp(*av, "-log")) {
+			if (p->hasthread)
+				vtc_fatal(p->vl,
+				    "Cannot log a running process");
+			p->log = 1;
 			continue;
 		}
 		if (!strcmp(*av, "-need-bytes")) {
@@ -730,12 +730,36 @@ cmd_process(CMD_ARGS)
 			} while(v < u);
 			continue;
 		}
+		if (!strcmp(*av, "-run")) {
+			process_start(p);
+			process_wait(p);
+			continue;
+		}
 		if (!strcmp(*av, "-screen_dump")) {
 			Term_Dump(p->term);
 			continue;
 		}
-		if (!strcmp(*av, "-close")) {
-			process_close(p);
+		if (!strcmp(*av, "-start")) {
+			process_start(p);
+			continue;
+		}
+		if (!strcmp(*av, "-stop")) {
+			process_kill(p, "TERM");
+			continue;
+		}
+		if (!strcmp(*av, "-wait")) {
+			process_wait(p);
+			continue;
+		}
+		if (!strcmp(*av, "-write")) {
+			process_write(p, av[1]);
+			av++;
+			continue;
+		}
+		if (!strcmp(*av, "-writeln")) {
+			process_write(p, av[1]);
+			process_write(p, "\n");
+			av++;
 			continue;
 		}
 		if (**av == '-')

From phk at FreeBSD.org  Tue Jan 16 09:48:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 16 Jan 2018 09:48:07 +0000 (UTC)
Subject: [master] 4dc546c Handle hidden counters in subscription mode
Message-ID: <20180116094807.5CE64B47C8@lists.varnish-cache.org>


commit 4dc546c93e7eabdec86991671f4e812dbf364c6c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 16 09:46:52 2018 +0000

    Handle hidden counters in subscription mode

diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index 8ec77b5..dc5550e 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -62,6 +62,7 @@ VTAILQ_HEAD(vsc_sf_head, vsc_sf);
 struct vsc_pt {
 	struct VSC_point	point;
 	char			*name;
+	int			exposed;
 };
 
 struct vsc_seg {
@@ -166,17 +167,6 @@ VSC_Arg(struct vsc *vsc, char arg, const char *opt)
 	}
 }
 
-/*--------------------------------------------------------------------
- */
-void
-VSC_State(struct vsc *vsc, VSC_new_f *fn, VSC_destroy_f *fd, void *priv)
-{
-
-	CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);
-	vsc->fnew = fn;
-	vsc->fdestroy = fd;
-	vsc->priv = priv;
-}
 
 /*--------------------------------------------------------------------
  */
@@ -207,7 +197,7 @@ vsc_clean_point(struct vsc_pt *point)
 	REPLACE(point->name, NULL);
 }
 
-static int
+static void
 vsc_fill_point(const struct vsc *vsc, const struct vsc_seg *seg,
     const struct vjsn_val *vv, struct vsb *vsb, struct vsc_pt *point)
 {
@@ -225,7 +215,7 @@ vsc_fill_point(const struct vsc *vsc, const struct vsc_seg *seg,
 	AZ(VSB_finish(vsb));
 
 	if (vsc_filter(vsc, VSB_data(vsb)))
-		return (0);
+		return;
 
 	point->name = strdup(VSB_data(vsb));
 	AN(point->name);
@@ -289,7 +279,6 @@ vsc_fill_point(const struct vsc *vsc, const struct vsc_seg *seg,
 	AN(vt);
 
 	point->point.ptr = (volatile void*)(seg->body + atoi(vt->value));
-	return (1);
 }
 
 static void
@@ -306,11 +295,8 @@ vsc_del_seg(const struct vsc *vsc, struct vsm *vsm, struct vsc_seg *sp)
 		vjsn_delete(&sp->vj);
 	} else {
 		pp = sp->points;
-		for (u = 0; u < sp->npoints; u++, pp++) {
-			if (vsc->fdestroy != NULL)
-				vsc->fdestroy(vsc->priv, &pp->point);
+		for (u = 0; u < sp->npoints; u++, pp++)
 			vsc_clean_point(pp);
-		}
 		free(sp->points);
 	}
 	FREE_OBJ(sp);
@@ -364,10 +350,7 @@ vsc_add_seg(const struct vsc *vsc, struct vsm *vsm, const struct vsm_fantom *fp)
 		AN(vve);
 		pp = sp->points;
 		VTAILQ_FOREACH(vv, &vve->children, list) {
-			if (vsc_fill_point(vsc, sp, vv, vsb, pp) &&
-				vsc->fnew != NULL)
-				pp->point.priv =
-				    vsc->fnew(vsc->priv, &pp->point);
+			vsc_fill_point(vsc, sp, vv, vsb, pp);
 			pp++;
 		}
 		VSB_destroy(&vsb);
@@ -380,6 +363,35 @@ vsc_add_seg(const struct vsc *vsc, struct vsm *vsm, const struct vsm_fantom *fp)
 	return (sp);
 }
 
+/*--------------------------------------------------------------------
+ */
+
+static void
+vsc_expose(const struct vsc *vsc, const struct vsc_seg *sp, int del)
+{
+	struct vsc_pt *pp;
+	unsigned u;
+
+	pp = sp->points;
+	for (u = 0; u < sp->npoints; u++, pp++) {
+		if (pp->name == NULL)
+			continue;
+		if (vsc->fdestroy != NULL && pp->exposed &&
+		    (del || sp->head->ready == 2)) {
+			vsc->fdestroy(vsc->priv, &pp->point);
+			pp->exposed = 0;
+		}
+		if (vsc->fnew != NULL && !pp->exposed &&
+		    !del && sp->head->ready == 1) {
+			pp->point.priv = vsc->fnew(vsc->priv, &pp->point);
+			pp->exposed = 1;
+		}
+	}
+}
+
+/*--------------------------------------------------------------------
+ */
+
 static int
 vsc_iter_seg(const struct vsc *vsc, const struct vsc_seg *sp,
     VSC_iter_f *fiter, void *priv)
@@ -420,6 +432,7 @@ VSC_Iter(struct vsc *vsc, struct vsm *vsm, VSC_iter_f *fiter, void *priv)
 			sp2 = sp;
 			sp = VTAILQ_NEXT(sp, list);
 			VTAILQ_REMOVE(&vsc->segs, sp2, list);
+			vsc_expose(vsc, sp2, 1);
 			vsc_del_seg(vsc, vsm, sp2);
 		}
 		if (sp == NULL) {
@@ -427,8 +440,10 @@ VSC_Iter(struct vsc *vsc, struct vsm *vsm, VSC_iter_f *fiter, void *priv)
 			if (sp != NULL) {
 				VTAILQ_INSERT_TAIL(&vsc->segs, sp, list);
 				sp2 = NULL;
+				vsc_expose(vsc, sp, 0);
 			}
 		} else {
+			vsc_expose(vsc, sp, 0);
 			sp2 = VTAILQ_NEXT(sp, list);
 		}
 		if (sp != NULL && fiter != NULL && sp->head->ready < 2)
@@ -441,11 +456,30 @@ VSC_Iter(struct vsc *vsc, struct vsm *vsm, VSC_iter_f *fiter, void *priv)
 		sp2 = sp;
 		sp = VTAILQ_NEXT(sp, list);
 		VTAILQ_REMOVE(&vsc->segs, sp2, list);
+		vsc_expose(vsc, sp2, 1);
 		vsc_del_seg(vsc, vsm, sp2);
 	}
 	return (i);
 }
 
+/*--------------------------------------------------------------------
+ */
+
+void
+VSC_State(struct vsc *vsc, VSC_new_f *fn, VSC_destroy_f *fd, void *priv)
+{
+	struct vsc_seg *sp;
+
+	CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC);
+	assert((fn == NULL && fd == NULL) || (fn != NULL && fd != NULL));
+	if (fd == NULL) {
+		VTAILQ_FOREACH(sp, &vsc->segs, list)
+			vsc_expose(vsc, sp, 1);
+	}
+	vsc->fnew = fn;
+	vsc->fdestroy = fd;
+	vsc->priv = priv;
+}
 
 /*--------------------------------------------------------------------
  */
@@ -498,6 +532,7 @@ VSC_Destroy(struct vsc **vscp, struct vsm *vsm)
 	vsc_delete_sf_list(&vsc->sf_list_exclude);
 	VTAILQ_FOREACH_SAFE(sp, &vsc->segs, list, sp2) {
 		VTAILQ_REMOVE(&vsc->segs, sp, list);
+		vsc_expose(vsc, sp, 1);
 		vsc_del_seg(vsc, vsm, sp);
 	}
 	FREE_OBJ(vsc);

From phk at FreeBSD.org  Tue Jan 16 09:48:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 16 Jan 2018 09:48:07 +0000 (UTC)
Subject: [master] 5332d21 VSM doesn't know about hidden counters,
 leave it to VSC.
Message-ID: <20180116094807.74354B47CC@lists.varnish-cache.org>


commit 5332d2121c300ee99d3eed417c86a342b67b26f0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 16 09:47:09 2018 +0000

    VSM doesn't know about hidden counters, leave it to VSC.

diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c
index ce07b7f..11d6ffe 100644
--- a/bin/varnishstat/varnishstat_curses.c
+++ b/bin/varnishstat/varnishstat_curses.c
@@ -932,9 +932,10 @@ newpt(void *priv, const struct VSC_point *const vpt)
 {
 	struct pt *pt;
 
+	AZ(priv);
 	ALLOC_OBJ(pt, PT_MAGIC);
+	rebuild |= 1;
 	AN(pt);
-	AZ(priv);
 	pt->vpt = vpt;
 	pt->last = *pt->vpt->ptr;
 	pt->ma_10.nmax = 10;
@@ -964,6 +965,7 @@ delpt(void *priv, const struct VSC_point *const vpt)
 
 	AZ(priv);
 	CAST_OBJ_NOTNULL(pt, vpt->priv, PT_MAGIC);
+	rebuild |= 2;
 	VTAILQ_REMOVE(&ptlist, pt, list);
 	n_ptlist--;
 	FREE_OBJ(pt);
@@ -999,17 +1001,17 @@ do_curses(struct vsm *vsm, struct vsc *vsc, double delay)
 
 	VSC_State(vsc, newpt, delpt, NULL);
 
-	rebuild = VSM_WRK_RESTARTED;
+	(void)VSC_Iter(vsc, vsm, NULL, NULL);
+	build_pt_array();
+	init_hitrate();
+
 	while (keep_running) {
+		(void)VSC_Iter(vsc, vsm, NULL, NULL);
 		vsm_status = VSM_Status(vsm);
-		rebuild |= vsm_status & ~(VSM_MGT_RUNNING|VSM_WRK_RUNNING);
-		if (rebuild) {
-			(void)VSC_Iter(vsc, vsm, NULL, NULL);
-			if (rebuild & (VSM_MGT_RESTARTED|VSM_WRK_RESTARTED))
-				init_hitrate();
+		if (vsm_status & (VSM_MGT_RESTARTED|VSM_WRK_RESTARTED))
+			init_hitrate();
+		if (rebuild)
 			build_pt_array();
-			redraw = 1;
-		}
 
 		now = VTIM_mono();
 		if (now - t_sample > interval)

From phk at FreeBSD.org  Tue Jan 16 10:06:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 16 Jan 2018 10:06:06 +0000 (UTC)
Subject: [master] fd06bc0 Get the sign/logic right
Message-ID: <20180116100606.0DC2DB52B8@lists.varnish-cache.org>


commit fd06bc03296470c4b3736dc6e725a6368ec55547
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 16 10:04:42 2018 +0000

    Get the sign/logic right

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index df950cd..b4bedca 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -324,7 +324,7 @@ process_thread(void *priv)
 	if (p->expect_signal >= 0 && sig != p->expect_signal)
 		vtc_fatal(p->vl, "Expected signal %d got %d",
 			p->expect_signal, sig);
-	else if (sig && sig != p->expect_signal)
+	else if (sig != 0 && sig != -p->expect_signal)
 		vtc_fatal(p->vl, "Expected signal %d got %d",
 			-p->expect_signal, sig);
 	if (ext != p->expect_exit)

From phk at FreeBSD.org  Tue Jan 16 10:28:04 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 16 Jan 2018 10:28:04 +0000 (UTC)
Subject: [master] 94283ee Fix a harmless signextension bug.
Message-ID: <20180116102804.9D4E0B59BB@lists.varnish-cache.org>


commit 94283eed20cfac5b1c7c49b9cc6bdf5c540b156a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 16 10:27:31 2018 +0000

    Fix a harmless signextension bug.

diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c
index dc5550e..7df0858 100644
--- a/lib/libvarnishapi/vsc.c
+++ b/lib/libvarnishapi/vsc.c
@@ -106,7 +106,7 @@ static const struct VSC_level_desc * const levels[] = {
 #undef VSC_LEVEL_F
 };
 
-static const size_t nlevels = sizeof(levels)/sizeof(*levels);
+static const ssize_t nlevels = sizeof(levels)/sizeof(*levels);
 
 /*--------------------------------------------------------------------*/
 
@@ -496,8 +496,8 @@ VSC_ChangeLevel(const struct VSC_level_desc *old, int chg)
 			break;
 	if (i == nlevels)
 		i = 0;
-	else
-		i += chg;
+
+	i += chg;
 	if (i >= nlevels)
 		i = nlevels - 1;
 	if (i < 0)

From nils.goroll at uplex.de  Tue Jan 16 11:04:50 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 16 Jan 2018 12:04:50 +0100
Subject: [master] aa21e4b I may be incompetent at math, but I'm not _that_
 incompetent at math :-)
In-Reply-To: <20180115221510.8BE79A6B9B@lists.varnish-cache.org>
References: <20180115221510.8BE79A6B9B@lists.varnish-cache.org>
Message-ID: <05686d35-db8e-dea7-df83-4bfe93c6eca1@uplex.de>

phk,

I think we are both wrong:

the mapping starts at off = RDN2(of, ps)

The precise length of the mapping is of - off + sz, so if
we round up again to the next page, we get

	len = RUP2(of - off + sz, ps)

RUP2(sz) is too small because (of - off) can make the mapping span another page
boundary.

But RUP2(of + sz, ps) ignores that the mapping starts at off.

Or am I missing anything?


Nils

On 15/01/18 23:15, Poul-Henning Kamp wrote:
> 
> commit aa21e4b2f8c3b884c4a9c32ef74ed5625e4c6916
> Author: Poul-Henning Kamp <phk at FreeBSD.org>
> Date:   Mon Jan 15 22:14:14 2018 +0000
> 
>     I may be incompetent at math, but I'm not _that_ incompetent at math :-)
> 
> diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
> index 282865b..cb8118b 100644
> --- a/lib/libvarnishapi/vsm.c
> +++ b/lib/libvarnishapi/vsm.c
> @@ -718,7 +718,7 @@ VSM_Map(struct vsm *vd, struct vsm_fantom *vf)
>  
>  	sz = strtoul(vg->av[3], NULL, 10);
>  	assert(sz > 0);
> -	len = RUP2(sz, ps);
> +	len = RUP2(of + sz, ps);
>  
>  	vsb = VSB_new_auto();
>  	AN(vsb);
> _______________________________________________
> varnish-commit mailing list
> varnish-commit at varnish-cache.org
> https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit
> 


-- 

** * * UPLEX - Nils Goroll Systemoptimierung

Scheffelstra?e 32
22301 Hamburg

tel +49 40 28805731
mob +49 170 2723133
fax +49 40 42949753

xmpp://slink at jabber.int.uplex.de/

http://uplex.de/

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 473 bytes
Desc: OpenPGP digital signature
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20180116/5859e6cd/attachment.bin>

From nils.goroll at uplex.de  Tue Jan 16 12:54:58 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 16 Jan 2018 13:54:58 +0100
Subject: [master] aa21e4b I may be incompetent at math, but I'm not _that_
 incompetent at math :-)
In-Reply-To: <05686d35-db8e-dea7-df83-4bfe93c6eca1@uplex.de>
References: <20180115221510.8BE79A6B9B@lists.varnish-cache.org>
 <05686d35-db8e-dea7-df83-4bfe93c6eca1@uplex.de>
Message-ID: <4d209e30-c8a8-cee7-4774-3d7306edd62f@uplex.de>

And btw, why don't we map whole files?

-- 

** * * UPLEX - Nils Goroll Systemoptimierung

Scheffelstra?e 32
22301 Hamburg

tel +49 40 28805731
mob +49 170 2723133
fax +49 40 42949753

xmpp://slink at jabber.int.uplex.de/

http://uplex.de/

-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 473 bytes
Desc: OpenPGP digital signature
URL: <https://www.varnish-cache.org/lists/pipermail/varnish-commit/attachments/20180116/5fbaad42/attachment.bin>

From phk at phk.freebsd.dk  Tue Jan 16 14:42:21 2018
From: phk at phk.freebsd.dk (Poul-Henning Kamp)
Date: Tue, 16 Jan 2018 14:42:21 +0000
Subject: [master] aa21e4b I may be incompetent at math,
 but I'm not _that_ incompetent at math :-)
In-Reply-To: <4d209e30-c8a8-cee7-4774-3d7306edd62f@uplex.de>
References: <20180115221510.8BE79A6B9B@lists.varnish-cache.org>
 <05686d35-db8e-dea7-df83-4bfe93c6eca1@uplex.de>
 <4d209e30-c8a8-cee7-4774-3d7306edd62f@uplex.de>
Message-ID: <65412.1516113741@critter.freebsd.dk>

--------
In message <4d209e30-c8a8-cee7-4774-3d7306edd62f at uplex.de>, Nils Goroll writes:
>This is an OpenPGP/MIME signed message (RFC 4880 and 3156)

>And btw, why don't we map whole files?

Because that is an implementation detail that can get sorted out
later.

Right now we don't put the cluster *as such* in the index file
and therefore we don't know the size of the cluster any place
convenient.

We will have to invent a special kind of index entry for clusters
and I punted that to a later optimization.


-- 
Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
phk at FreeBSD.ORG         | TCP/IP since RFC 956
FreeBSD committer       | BSD since 4.3-tahoe    
Never attribute to malice what can adequately be explained by incompetence.

From phk at FreeBSD.org  Tue Jan 16 15:01:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 16 Jan 2018 15:01:07 +0000 (UTC)
Subject: [master] a8a3049 Wrap lines, test CUD.
Message-ID: <20180116150107.EE63164480@lists.varnish-cache.org>


commit a8a3049d71bc4989497a3c3da6010fa60ec04942
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 16 15:00:27 2018 +0000

    Wrap lines, test CUD.

diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc
index c191742..054519f 100644
--- a/bin/varnishtest/tests/a00000.vtc
+++ b/bin/varnishtest/tests/a00000.vtc
@@ -45,10 +45,17 @@ process p1 "ps -lw | grep '[p][s]' ; tty ; sleep 1" -run -screen_dump
 
 process p2 "stty -a ; sleep 1" -run -screen_dump
 
-process p3 -hexdump {stty raw; echo "*" ; dd bs=254 count=1} -start
+process p3 -hexdump {stty raw; echo "*" ; cat} -start
 
 process p3 -need-bytes 2
 
-process p3 -write "\x1b[H\x1b[2J1\x1b[79C2\x08>\x1b[25;1H3\x1b[25;80H4\x08>\x1b[A\x1b[Cv\x1b[22A^\x1b[79D^\x1b[;2H<\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x08<\x1b[24;Hv\x1b[12;1H111111112222222333333\x0d\x0a111111112222222333333\x0d\x0a111111112222222333333\x0d\x0a111111112222222333333\x0d\x0a\x1b[12;12H\x1b[K\x1b[13;12H\x1b[0K\x1b[14;12H\x1b[1K\x1b[15;12H\x1b[2K\x1b[3;1Hline3 <\x0d\x0a"
+process p3 -write "\x1b[H\x1b[2J1\x1b[79C2\x08>\x1b[25;1H3\x1b[25;80H"
+process p3 -write "4\x08>\x1b[A\x1b[Cv\x1b[22A^\x1b[79D^\x1b[;2H<\n\n\n\n"
+process p3 -write "\n\n\n\n\n\n\n\n\x1b[B\x1b[11B\x08<\x1b[24;Hv\x1b[12;1H"
+process p3 -write "111111112222222333333\x0d\x0a111111112"
+process p3 -write "222222333333\x0d\x0a111111112222222333"
+process p3 -write "333\x0d\x0a111111112222222333333\x0d\x0a\x1b[12"
+process p3 -write ";12H\x1b[K\x1b[13;12H\x1b[0K\x1b[14;12H\x1b[1K\x1b"
+process p3 -write "[15;12H\x1b[2K\x1b[3;1Hline3 <\x0d\x0a"
 
-process p3 -need-bytes 256 -screen_dump -wait
+process p3 -need-bytes 252 -screen_dump -stop

From phk at FreeBSD.org  Wed Jan 17 10:04:11 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 17 Jan 2018 10:04:11 +0000 (UTC)
Subject: [master] e4d1354 Implement window resizing in the terminal emulation
Message-ID: <20180117100411.7320CAFB5D@lists.varnish-cache.org>


commit e4d1354e927bb70bc2a0fffb79e4df8fa5972261
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 17 10:02:54 2018 +0000

    Implement window resizing in the terminal emulation

diff --git a/bin/varnishtest/tests/u00008.vtc b/bin/varnishtest/tests/u00008.vtc
index 43a69a4..5b6922b 100644
--- a/bin/varnishtest/tests/u00008.vtc
+++ b/bin/varnishtest/tests/u00008.vtc
@@ -32,5 +32,9 @@ process p1 -write {dek}
 
 delay 1
 
-process p1 -need-bytes 5000 -screen_dump -write {q} -wait
+process p1 -need-bytes 5000 -screen_dump
+
+process p1 -winsz 25 132
+
+process p1 -need-bytes 7000 -screen_dump -write {q} -wait
 
diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h
index 6efef45..6e99403 100644
--- a/bin/varnishtest/vtc.h
+++ b/bin/varnishtest/vtc.h
@@ -139,6 +139,7 @@ void vtc_expect(struct vtclog *, const char *, const char *, const char *,
     const char *, const char *);
 
 /* vtc_term.c */
-struct term *Term_New(struct vtclog *);
+struct term *Term_New(struct vtclog *, int, int);
 void Term_Feed(struct term *, const char *, const char *);
 void Term_Dump(const struct term *);
+void Term_SetSize(struct term *, int, int);
diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index b4bedca..a251c6b 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -88,7 +88,8 @@ struct process {
 	int			status;
 
 	struct term		*term;
-
+	int			lin;
+	int			col;
 };
 
 static VTAILQ_HEAD(, process)	processes =
@@ -133,7 +134,9 @@ process_new(const char *name)
 	p->fd_term = -1;
 
 	VTAILQ_INSERT_TAIL(&processes, p, list);
-	p->term = Term_New(p->vl);
+	p->lin = 25;
+	p->col = 80;
+	p->term = Term_New(p->vl, p->lin, p->col);
 	AN(p->term);
 	return (p);
 }
@@ -340,18 +343,26 @@ process_thread(void *priv)
 }
 
 static void
-process_init_term(struct process *p, int fd)
+process_winsz(struct process *p, int fd, int lin, int col)
 {
 	struct winsize ws;
-	struct termios tt;
 	int i;
 
 	memset(&ws, 0, sizeof ws);
-	ws.ws_row = 25;
-	ws.ws_col = 80;
+	ws.ws_row = (short)lin;
+	ws.ws_col = (short)col;
 	i = ioctl(fd, TIOCSWINSZ, &ws);
 	if (i)
 		vtc_log(p->vl, 4, "TIOCWINSZ %d %s", i, strerror(errno));
+}
+
+static void
+process_init_term(struct process *p, int fd)
+{
+	struct termios tt;
+	int i;
+
+	process_winsz(p, fd, p->lin, p->col);
 
 	memset(&tt, 0, sizeof tt);
 	tt.c_cflag = CREAD | CS8 | HUPCL;
@@ -751,6 +762,15 @@ cmd_process(CMD_ARGS)
 			process_wait(p);
 			continue;
 		}
+		if (!strcmp(*av, "-winsz")) {
+			p->lin = atoi(av[1]);
+			assert(p->lin > 1);
+			p->col = atoi(av[2]);
+			assert(p->col > 1);
+			av += 2;
+			Term_SetSize(p->term, p->lin, p->col);
+			process_winsz(p, p->fd_term, p->lin, p->col);
+		}
 		if (!strcmp(*av, "-write")) {
 			process_write(p, av[1]);
 			av++;
diff --git a/bin/varnishtest/vtc_term.c b/bin/varnishtest/vtc_term.c
index cc9d862..be554dc 100644
--- a/bin/varnishtest/vtc_term.c
+++ b/bin/varnishtest/vtc_term.c
@@ -53,13 +53,13 @@ struct term {
 };
 
 static void
-term_clear(const struct term *tp)
+term_clear(char * const *vram, int lin, int col)
 {
 	int i;
 
-	for (i = 0; i < tp->nlin; i++) {
-		memset(tp->vram[i], ' ', tp->ncol);
-		tp->vram[i][tp->ncol] = '\0';
+	for (i = 0; i < lin; i++) {
+		memset(vram[i], ' ', col);
+		vram[i][col] = '\0';
 	}
 }
 
@@ -126,7 +126,9 @@ term_escape(struct term *tp, int c, int n)
 			tp->col = 0;
 		break;
 	case 'h':
-		// SM - Set Mode (ignored XXX?)
+		// SM - Set Mode (mostly ignored XXX?)
+		tp->col = 0;
+		tp->line = 0;
 		break;
 	case 'H':
 		// CUP - Cursor Position
@@ -142,7 +144,7 @@ term_escape(struct term *tp, int c, int n)
 		// ED - Erase in Display (0=below, 1=above, 2=all)
 		switch(tp->arg[0]) {
 		case 2:
-			term_clear(tp);
+			term_clear(tp->vram, tp->nlin, tp->ncol);
 			break;
 		default:
 			vtc_fatal(tp->vl, "ANSI J[%d]", tp->arg[0]);
@@ -213,8 +215,10 @@ term_char(struct term *tp, char c)
 		if (c < ' ' || c > '~')
 			c = '?';
 		tp->vram[tp->line][tp->col++] = c;
-		if (tp->col >= tp->ncol)
-			tp->col = tp->ncol - 1;
+		if (tp->col >= tp->ncol) {
+			tp->col = 0;
+			term_char(tp, '\n');
+		}
 	}
 }
 
@@ -277,24 +281,46 @@ Term_Feed(struct term *tp, const char *b, const char *e)
 	}
 }
 
+void
+Term_SetSize(struct term *tp, int lin, int col)
+{
+	char **vram;
+	int i, j;
+
+	vram = calloc(lin, sizeof *tp->vram);
+	AN(vram);
+	for (i = 0; i < lin; i++) {
+		vram[i] = malloc(col + 1L);
+		AN(vram[i]);
+	}
+	term_clear(vram, lin, col);
+	if (tp->vram != NULL) {
+		for (i = 0; i < lin; i++) {
+			if (i >= tp->nlin)
+				break;
+			j = col;
+			if (j > tp->ncol)
+				j = tp->ncol;
+			memcpy(vram[i], tp->vram[i], j);
+		}
+		for (i = 0; i < tp->nlin; i++)
+			free(tp->vram[i]);
+		free(tp->vram);
+	}
+	tp->vram = vram;
+	tp->nlin = lin;
+	tp->ncol = col;
+}
+
 struct term *
-Term_New(struct vtclog *vl)
+Term_New(struct vtclog *vl, int lin, int col)
 {
 	struct term *tp;
-	int i;
 
 	ALLOC_OBJ(tp, TERM_MAGIC);
 	AN(tp);
 	tp->vl = vl;
-	tp->nlin = 25;
-	tp->ncol = 80;
-	tp->vram = calloc(tp->nlin, sizeof *tp->vram);
-	AN(tp->vram);
-	for (i = 0; i < tp->nlin; i++) {
-		tp->vram[i] = malloc(tp->ncol + 1L);
-		AN(tp->vram[i]);
-	}
-	term_clear(tp);
+	Term_SetSize(tp, lin, col);
 	tp->line = tp->nlin - 1;
 	return (tp);
 }

From dridi.boukelmoune at gmail.com  Wed Jan 17 10:48:07 2018
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 17 Jan 2018 10:48:07 +0000 (UTC)
Subject: [master] 4732640 Introduce obj.storage for VCL
Message-ID: <20180117104807.98EBAB0994@lists.varnish-cache.org>


commit 47326405e9792778cbb2ad8c51ea1918e04d90e8
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Jan 15 14:27:56 2018 +0100

    Introduce obj.storage for VCL
    
    In addition, instead of always initializing a transaction's storage
    backend using the round-robin selection, pick Transient when we know
    in advance that beresp will be uncacheable.
    
    Closes #2533

diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c
index 76eb85f..c31d88f 100644
--- a/bin/varnishd/cache/cache_fetch.c
+++ b/bin/varnishd/cache/cache_fetch.c
@@ -261,8 +261,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo)
 	CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC);
 
 	AZ(bo->storage);
-
-	bo->storage = STV_next();
+	bo->storage = bo->do_pass ? stv_transient : STV_next();
 
 	if (bo->retries > 0)
 		http_Unset(bo->bereq, "\012X-Varnish:");
diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index b99bd9f..b16c614 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -366,6 +366,16 @@ VRT_l_beresp_storage(VRT_CTX, VCL_STEVEDORE stv)
 
 /*--------------------------------------------------------------------*/
 
+VCL_STEVEDORE
+VRT_r_obj_storage(VRT_CTX)
+{
+	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);
+	return (ctx->req->objcore->stobj->stevedore);
+}
+
+/*--------------------------------------------------------------------*/
+
 #define REQ_VAR_L(nm, elem, type,extra)					\
 									\
 void									\
diff --git a/bin/varnishtest/tests/v00052.vtc b/bin/varnishtest/tests/v00052.vtc
new file mode 100644
index 0000000..a014b60
--- /dev/null
+++ b/bin/varnishtest/tests/v00052.vtc
@@ -0,0 +1,49 @@
+varnishtest "obj.storage coverage"
+
+server s1 {
+	rxreq
+	txresp
+
+	rxreq
+	txresp
+} -start
+
+varnish v1 -vcl+backend {
+	sub vcl_hit {
+		set req.http.Hit-Storage = obj.storage;
+	}
+
+	sub vcl_backend_response {
+		set beresp.http.Default-Storage = beresp.storage;
+		if (bereq.method == "GET") {
+			set beresp.storage = storage.Transient;
+		}
+	}
+
+	sub vcl_deliver {
+		set resp.http.Deliver-Storage = obj.storage;
+		if (req.http.Hit-Storage) {
+			set resp.http.Hit-Storage = req.http.Hit-Storage;
+		}
+	}
+} -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.http.Default-Storage == storage.s0
+	expect resp.http.Deliver-Storage == storage.Transient
+	expect resp.http.Hit-Storage == <undef>
+
+	txreq
+	rxresp
+	expect resp.http.Default-Storage == storage.s0
+	expect resp.http.Deliver-Storage == storage.Transient
+	expect resp.http.Hit-Storage == storage.Transient
+
+	txreq -req POST
+	rxresp
+	expect resp.http.Default-Storage == storage.Transient
+	expect resp.http.Deliver-Storage == storage.Transient
+	expect resp.http.Hit-Storage == <undef>
+} -run
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 40d7556..c9b9e74 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -745,6 +745,13 @@ sp_variables = [
 		hit-for-miss).
 		"""
 	),
+	('obj.storage',
+		'STEVEDORE',
+		('hit', 'deliver'),
+		(), """
+		The storage backend used to save this object.
+		"""
+	),
 	('resp',
 		'HTTP',
 		('deliver', 'synth'),

From nils.goroll at uplex.de  Wed Jan 17 12:59:07 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 17 Jan 2018 12:59:07 +0000 (UTC)
Subject: [master] 062234c Do not clean up files generated by
 lib/libvcc/generate.py
Message-ID: <20180117125907.0B960B3149@lists.varnish-cache.org>


commit 062234c19e62fac492bf6559ca2a296d4cc1020a
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jan 17 13:55:13 2018 +0100

    Do not clean up files generated by lib/libvcc/generate.py
    
    ... for a simple make clean, in particular not vmod_abi.h.
    
    Otherwise, building from a trunk dist tarball will give a NOGIT
    VMOD_ABI_Version after a "make clean", while the correct version
    is in place for a simple make after unpacking the tarball.

diff --git a/include/Makefile.am b/include/Makefile.am
index d59b310..c11ba2b 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -119,10 +119,10 @@ $(GEN_H): vmod_abi.h
 GENERATED_H = vmod_abi.h $(GEN_H)
 
 BUILT_SOURCES = $(GENERATED_H)
-MAINTAINERCLEANFILES = vcs_version.h
+
+MAINTAINERCLEANFILES = $(GENERATED_H)
 
 CLEANFILES = \
-	$(GENERATED_H) \
 	vrt_test \
 	_vrt_test \
 	_vrt.c

From nils.goroll at uplex.de  Wed Jan 17 15:41:05 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 17 Jan 2018 15:41:05 +0000 (UTC)
Subject: [master] ad99f14 libvcc/generate.py creates code all over the place
Message-ID: <20180117154105.CE417B62A4@lists.varnish-cache.org>


commit ad99f144c44f034b54c972c712862afbdfd94280
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jan 17 16:35:19 2018 +0100

    libvcc/generate.py creates code all over the place
    
    so tell make in lib/libvcc how to invoke it.
    
    Exposed by 062234c19e62fac492bf6559ca2a296d4cc1020a: Now code changes
    (as when switching branches) may require a rebuild in lib/libvcc only

diff --git a/include/Makefile.am b/include/Makefile.am
index c11ba2b..27a557a 100644
--- a/include/Makefile.am
+++ b/include/Makefile.am
@@ -98,11 +98,12 @@ nobase_noinst_HEADERS = \
 	vtcp.h \
 	vtree.h
 
+## keep in sync with lib/libvcc/Makefile.am
 vmod_abi.h: \
 	    $(top_srcdir)/lib/libvcc/generate.py \
 	    $(top_srcdir)/include/vdef.h \
 	    $(top_srcdir)/include/vrt.h
-	mkdir -p tbl
+	mkdir -p $(top_builddir)/include/tbl
 	@PYTHON@ $(top_srcdir)/lib/libvcc/generate.py \
 	    $(top_srcdir) $(top_builddir)
 
diff --git a/lib/libvcc/Makefile.am b/lib/libvcc/Makefile.am
index 133dd1b..2b141b5 100644
--- a/lib/libvcc/Makefile.am
+++ b/lib/libvcc/Makefile.am
@@ -39,9 +39,23 @@ dist_pkgdata_SCRIPTS = \
 	vmodtool.py \
 	vsctool.py
 
-vcc_obj.c vcc_fixed_token.c vcc_token_defs.h: \
-	$(top_builddir)/include/vcl.h
+## keep in sync with include/Makefile.am
+vcc_obj.c: \
+	    $(top_srcdir)/lib/libvcc/generate.py \
+	    $(top_srcdir)/include/vdef.h \
+	    $(top_srcdir)/include/vrt.h
+	mkdir -p $(top_builddir)/include/tbl
+	@PYTHON@ $(top_srcdir)/lib/libvcc/generate.py \
+	    $(top_srcdir) $(top_builddir)
+
+GEN_H = \
+	vcc_fixed_token.c \
+	vcc_token_defs.h
+
+$(GEN_H): vcc_obj.c
+
+GENERATED_H = vcc_obj.c $(GEN_H)
+
+BUILT_SOURCES = $(GENERATED_H)
 
-CLEANFILES = $(builddir)/vcc_token_defs.h \
-	$(builddir)/vcc_fixed_token.c \
-	$(builddir)/vcc_obj.c
+MAINTAINERCLEANFILES = $(GENERATED_H)

From nils.goroll at uplex.de  Wed Jan 17 15:59:06 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 17 Jan 2018 15:59:06 +0000 (UTC)
Subject: [master] f544c8f replace package lists with install commands
Message-ID: <20180117155906.67650B68D3@lists.varnish-cache.org>


commit f544c8fea69256363e8badbd400a779abb9e7e8d
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jan 17 16:53:33 2018 +0100

    replace package lists with install commands
    
    No need to waste more precious human life time with stupid editing

diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst
index fba97e1..101db8e 100644
--- a/doc/sphinx/installation/install.rst
+++ b/doc/sphinx/installation/install.rst
@@ -73,50 +73,60 @@ git repository by doing.
 Build dependencies on Debian / Ubuntu
 --------------------------------------
 
+..  grep-dctrl -n -sBuild-Depends -r ^ ../../../../varnish-cache-debian/control | tr -d '\n' | awk -F,\  '{ for (i = 0; ++i <= NF;) { sub (/ .*/, "", $i); print "* `" $i "`"; }}' | egrep -v '(debhelper)'
+
 In order to build Varnish from source you need a number of packages
-installed. On a Debian or Ubuntu system these are:
+installed. On a Debian or Ubuntu system, use this command to install
+them (replace ``sudo apt-get install`` if needed)::
+
+    sudo apt-get install \
+	make \
+	automake \
+	autotools-dev \
+	libedit-dev \
+	libjemalloc-dev \
+	libncurses-dev \
+	libpcre3-dev \
+	libtool \
+	pkg-config \
+	python-docutils \
+	python-sphinx
 
-..  grep-dctrl -n -sBuild-Depends -r ^ ../../../../varnish-cache-debian/control | tr -d '\n' | awk -F,\  '{ for (i = 0; ++i <= NF;) { sub (/ .*/, "", $i); print "* `" $i "`"; }}' | egrep -v '(debhelper)'
+Optionally, to rebuild the svg files::
 
-* `make`
-* `automake`
-* `autotools-dev`
-* `libedit-dev`
-* `libjemalloc-dev`
-* `libncurses-dev`
-* `libpcre3-dev`
-* `libtool`
-* `pkg-config`
-* `python-docutils`
-* `python-sphinx`
+    sudo apt-get install graphviz
 
-Optionally, to rebuild the svg files:
+Recommended, in particular if you plan on building custom vmods::
 
-* `graphviz`
+    sudo apt-get install autoconf-archive
 
 Build dependencies on Red Hat / CentOS
 --------------------------------------
 
-To build Varnish on a Red Hat or CentOS system you need the following
-packages installed:
-
 .. gawk '/^BuildRequires/ {print "* `" $2 "`"}' ../../../redhat/varnish.spec | sort | uniq | egrep -v '(systemd)'
 
-* `make`
-* `autoconf`
-* `automake`
-* `jemalloc-devel`
-* `libedit-devel`
-* `libtool`
-* `ncurses-devel`
-* `pcre-devel`
-* `pkgconfig`
-* `python-docutils`
-* `python-sphinx`
-
-Optionally, to rebuild the svg files:
-
-* `graphviz`
+To build Varnish on a Red Hat or CentOS system, this command should
+install required packages (replace ``sudo yum install`` if needed)::
+
+    sudo yum install \
+	make \
+	autoconf \
+	automake \
+	jemalloc-devel \
+	libedit-devel \
+	libtool \
+	ncurses-devel \
+	pcre-devel \
+	pkgconfig \
+	python-docutils \
+	python-sphinx
+
+Optionally, to rebuild the svg files::
+
+    yum install graphviz
+
+.. XXX autoconf-archive ? is this any helpful on the notoriously
+   outdated Redhats?
 
 Build dependencies on a SmartOS Zone
 ------------------------------------

From phk at FreeBSD.org  Wed Jan 17 17:29:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 17 Jan 2018 17:29:06 +0000 (UTC)
Subject: [master] 2410591 Minor NOP rearrangement.
Message-ID: <20180117172906.616DFB8574@lists.varnish-cache.org>


commit 2410591720cdeb17fbaa4810a76fc6935a010e2f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 17 16:41:06 2018 +0000

    Minor NOP rearrangement.

diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c
index cf34a8f..8e55aeb 100644
--- a/bin/varnishd/common/common_vsmw.c
+++ b/bin/varnishd/common/common_vsmw.c
@@ -159,6 +159,55 @@ vsmw_mkent(const struct vsmw *vsmw, const char *pfx)
 
 /*--------------------------------------------------------------------*/
 
+static void
+vsmw_addseg(struct vsmw *vsmw, struct vsmwseg *seg)
+{
+	int fd;
+
+	VTAILQ_INSERT_TAIL(&vsmw->segs, seg, list);
+	fd = openat(vsmw->vdirfd, vsmw->idx, O_APPEND | O_WRONLY);
+	assert(fd >= 0);
+	vsmw_write_index(vsmw, fd, seg);
+	AZ(close(fd));
+}
+
+/*--------------------------------------------------------------------*/
+
+static void
+vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
+{
+	char *t = NULL;
+	int fd;
+
+	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
+	CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
+
+	if (!--seg->cluster->refs)
+		VSMW_DestroyCluster(vsmw, &seg->cluster);
+
+	VTAILQ_REMOVE(&vsmw->segs, seg, list);
+	REPLACE(seg->class, NULL);
+	REPLACE(seg->id, NULL);
+	FREE_OBJ(seg);
+
+	if (fixidx) {
+		vsmw_mkent(vsmw, vsmw->idx);
+		REPLACE(t, VSB_data(vsmw->vsb));
+		AN(t);
+		fd = openat(vsmw->vdirfd,
+		    t, O_WRONLY|O_CREAT|O_EXCL, vsmw->mode);
+		assert(fd >= 0);
+		vsmw_idx_head(vsmw, fd);
+		VTAILQ_FOREACH(seg, &vsmw->segs, list)
+			vsmw_write_index(vsmw, fd, seg);
+		AZ(close(fd));
+		AZ(renameat(vsmw->vdirfd, t, vsmw->vdirfd, vsmw->idx));
+		REPLACE(t, NULL);
+	}
+}
+
+/*--------------------------------------------------------------------*/
+
 struct vsmw_cluster *
 VSMW_NewCluster(struct vsmw *vsmw, size_t len, const char *pfx)
 {
@@ -225,7 +274,6 @@ VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc,
     const char *fmt, va_list va)
 {
 	struct vsmwseg *seg;
-	int fd;
 
 	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
 	(void)vc;
@@ -251,11 +299,7 @@ VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc,
 	assert(vc->next <= vc->len);
 	seg->ptr = seg->off + (char*)vc->ptr;
 
-	VTAILQ_INSERT_TAIL(&vsmw->segs, seg, list);
-	fd = openat(vsmw->vdirfd, vsmw->idx, O_APPEND | O_WRONLY);
-	assert(fd >= 0);
-	vsmw_write_index(vsmw, fd, seg);
-	AZ(close(fd));
+	vsmw_addseg(vsmw, seg);
 
 	return (seg->ptr);
 }
@@ -274,40 +318,6 @@ VSMW_Allocf(struct vsmw *vsmw, struct vsmw_cluster *vc,
 }
 
 /*--------------------------------------------------------------------*/
-static void
-vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
-{
-	char *t = NULL;
-	int fd;
-
-	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
-	CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
-
-	if (!--seg->cluster->refs)
-		VSMW_DestroyCluster(vsmw, &seg->cluster);
-
-	VTAILQ_REMOVE(&vsmw->segs, seg, list);
-	REPLACE(seg->class, NULL);
-	REPLACE(seg->id, NULL);
-	FREE_OBJ(seg);
-
-	if (fixidx) {
-		vsmw_mkent(vsmw, vsmw->idx);
-		REPLACE(t, VSB_data(vsmw->vsb));
-		AN(t);
-		fd = openat(vsmw->vdirfd,
-		    t, O_WRONLY|O_CREAT|O_EXCL, vsmw->mode);
-		assert(fd >= 0);
-		vsmw_idx_head(vsmw, fd);
-		VTAILQ_FOREACH(seg, &vsmw->segs, list)
-			vsmw_write_index(vsmw, fd, seg);
-		AZ(close(fd));
-		AZ(renameat(vsmw->vdirfd, t, vsmw->vdirfd, vsmw->idx));
-		REPLACE(t, NULL);
-	}
-}
-
-/*--------------------------------------------------------------------*/
 
 void
 VSMW_Free(struct vsmw *vsmw, void **pp)

From phk at FreeBSD.org  Wed Jan 17 17:29:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 17 Jan 2018 17:29:06 +0000 (UTC)
Subject: [master] ae25a64 List the VSM clusters in the index file
Message-ID: <20180117172906.75513B8577@lists.varnish-cache.org>


commit ae25a641bb49ff62619f8d89db5a9c714aeadfed
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 17 17:27:43 2018 +0000

    List the VSM clusters in the index file

diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c
index e4c0261..b0f5619 100644
--- a/bin/varnishd/cache/cache_vrt.c
+++ b/bin/varnishd/cache/cache_vrt.c
@@ -650,8 +650,7 @@ VRT_VSM_Cluster_Destroy(VRT_CTX, struct vsmw_cluster **vcp)
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	AN(vcp);
-	AN(*vcp);
-	*vcp = NULL;
+	VSMW_DestroyCluster(heritage.proc_vsmw, vcp);
 }
 
 /*--------------------------------------------------------------------
diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c
index 8e55aeb..beb6fbd 100644
--- a/bin/varnishd/common/common_vsmw.c
+++ b/bin/varnishd/common/common_vsmw.c
@@ -72,6 +72,7 @@ struct vsmw_cluster {
 #define VSMW_CLUSTER_MAGIC		0x28b74c00
 
 	VTAILQ_ENTRY(vsmw_cluster)	list;
+	struct vsmwseg			*cseg;
 	char				*fn;
 	size_t				len;
 	void				*ptr;
@@ -182,9 +183,6 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
 	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
 	CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
 
-	if (!--seg->cluster->refs)
-		VSMW_DestroyCluster(vsmw, &seg->cluster);
-
 	VTAILQ_REMOVE(&vsmw->segs, seg, list);
 	REPLACE(seg->class, NULL);
 	REPLACE(seg->id, NULL);
@@ -208,8 +206,8 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx)
 
 /*--------------------------------------------------------------------*/
 
-struct vsmw_cluster *
-VSMW_NewCluster(struct vsmw *vsmw, size_t len, const char *pfx)
+static struct vsmw_cluster *
+vsmw_newcluster(struct vsmw *vsmw, size_t len, const char *pfx)
 {
 	struct vsmw_cluster *vc;
 	int fd;
@@ -245,6 +243,26 @@ VSMW_NewCluster(struct vsmw *vsmw, size_t len, const char *pfx)
 	return (vc);
 }
 
+struct vsmw_cluster *
+VSMW_NewCluster(struct vsmw *vsmw, size_t len, const char *pfx)
+{
+	struct vsmw_cluster *vc;
+	struct vsmwseg *seg;
+
+	vc = vsmw_newcluster(vsmw, len, pfx);
+
+	ALLOC_OBJ(seg, VSMWSEG_MAGIC);
+	AN(seg);
+	vc->cseg = seg;
+	seg->len = len;
+	seg->cluster = vc;
+	REPLACE(seg->class, "");
+	REPLACE(seg->id, "");
+	vsmw_addseg(vsmw, seg);
+
+	return (vc);
+}
+
 void
 VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vcp)
 {
@@ -255,10 +273,22 @@ VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vcp)
 	vc = *vcp;
 	*vcp = NULL;
 	CHECK_OBJ_NOTNULL(vc, VSMW_CLUSTER_MAGIC);
-	AZ(vc->refs);
 
+	if (vc->cseg != NULL) {
+		/*
+		 * Backends go on the cool list, so the VGC cluster is
+		 * destroyed before they are.  Solve this by turning the
+		 * cluster into an anonymous cluster which dies with the
+		 * refcount on it.
+		 */
+		vsmw_delseg(vsmw, vc->cseg, 1);
+		vc->cseg = NULL;
+		if (vc->refs > 0)
+			return;
+	}
 	AZ(munmap(vc->ptr, vc->len));
 
+	AZ(vc->refs);
 	VTAILQ_REMOVE(&vsmw->clusters, vc, list);
 	if (unlinkat(vsmw->vdirfd, vc->fn, 0))
 		assert (errno == ENOENT);
@@ -289,7 +319,7 @@ VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc,
 	REPLACE(seg->id, VSB_data(vsmw->vsb));
 
 	if (vc == NULL)
-		vc = VSMW_NewCluster(vsmw, seg->len, class);
+		vc = vsmw_newcluster(vsmw, seg->len, class);
 	AN(vc);
 	vc->refs++;
 
@@ -334,6 +364,10 @@ VSMW_Free(struct vsmw *vsmw, void **pp)
 		if (seg->ptr == p)
 			break;
 	AN(seg);
+
+	if (!--seg->cluster->refs && seg->cluster->cseg == NULL)
+		VSMW_DestroyCluster(vsmw, &seg->cluster);
+
 	vsmw_delseg(vsmw, seg, 1);
 }
 
diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt
index e31a0d5..923b702 100644
--- a/bin/varnishd/flint.lnt
+++ b/bin/varnishd/flint.lnt
@@ -96,6 +96,7 @@
 
 +libh mgt_event.h
 
+-sem(vsmw_addseg, custodial(2))
 -sem(BAN_Free, custodial(1))
 -sem(EXP_Inject, custodial(1))
 -sem(HSH_Insert, custodial(3))
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index be15a14..0766e53 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -418,9 +418,7 @@ vsm_refresh_set2(struct vsm *vd, struct vsm_set *vs, struct vsb *vsb)
 		return (retval|VSM_NUKE_ALL);
 
 	/*
-	 * Examine the ident line
-	 * XXX: for now ignore that one of the ID's is a pid which could
-	 * XXX: be kill(pid,0)'ed for more rapid abandonment detection.
+	 * First line is ident comment
 	 */
 	i = sscanf(VSB_data(vsb), "# %ju %ju\n%n", &id1, &id2, &ac);
 	if (i != 2) {
@@ -445,7 +443,7 @@ vsm_refresh_set2(struct vsm *vd, struct vsm_set *vs, struct vsb *vsb)
 		vg->markscan = 0;
 
 	/*
-	 * Efficient comparison walking the two lists side-by-side is ok because
+	 * Efficient comparison by walking the two lists side-by-side because
 	 * segment inserts always happen at the tail (VSMW_Allocv()). So, as
 	 * soon as vg is exhausted, we only insert.
 	 *
@@ -465,13 +463,17 @@ vsm_refresh_set2(struct vsm *vd, struct vsm_set *vs, struct vsb *vsb)
 		av = VAV_Parse(p, &ac, 0);
 		p = e + 1;
 
-		if (av[0] != NULL || ac < 5 || ac > 6) {
+		if (av[0] != NULL || ac < 4 || ac > 6) {
 			(void)(vsm_diag(vd,
 			    "vsm_refresh_set2: bad index (%d/%s)",
 			    ac, av[0]));
 			VAV_Free(av);
 			break;
 		}
+		if (ac == 4) {
+			VAV_Free(av);
+			continue;
+		}
 
 		if (vg == NULL) {
 			ALLOC_OBJ(vg2, VSM_SEG_MAGIC);
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index 43f9eea..a5a433f 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -506,5 +506,5 @@ vcc_Backend_Init(struct vcc *tl)
 	VSB_printf(ifp->ini, "\tvsc_cluster = VRT_VSM_Cluster_New(ctx,\n"
 	    "\t    ndirector * VRT_backend_vsm_need(ctx));\n");
 	VSB_printf(ifp->ini, "\tif (vsc_cluster == 0)\n\t\treturn(1);");
-	VSB_printf(ifp->fin, "\tVRT_VSM_Cluster_Destroy(ctx, &vsc_cluster);");
+	VSB_printf(ifp->fin, "\t\tVRT_VSM_Cluster_Destroy(ctx, &vsc_cluster);");
 }

From nils.goroll at uplex.de  Wed Jan 17 18:17:05 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 17 Jan 2018 18:17:05 +0000 (UTC)
Subject: [master] f72a7e0 document a possible pitfall with the probe .request
 attribute
Message-ID: <20180117181705.59AA4B94DA@lists.varnish-cache.org>


commit f72a7e0dd010277134fea5fbbacad5821ad50054
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jan 17 19:15:12 2018 +0100

    document a possible pitfall with the probe .request attribute

diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst
index 024b568..1d04884 100644
--- a/doc/sphinx/reference/vcl.rst
+++ b/doc/sphinx/reference/vcl.rst
@@ -274,9 +274,14 @@ There are no mandatory options. These are the options you can set:
 
   ``.request``
     Specify a full HTTP request using multiple strings. ``.request`` will
-    have ``\r\n`` automatically inserted after every string. 
+    have ``\r\n`` automatically inserted after every string.
     Mutually exclusive with ``.url``.
 
+    *Note* that probes require the backend to complete sending the
+    response and close the connection within the specified timeout, so
+    ``.request`` will, for ``HTTP/1.1``, most likely need to contain a
+    ``"Connection: close"`` string.
+
   ``.expected_response``
     The expected HTTP response code. Defaults to ``200``.
 

From phk at FreeBSD.org  Thu Jan 18 09:18:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Thu, 18 Jan 2018 09:18:06 +0000 (UTC)
Subject: [master] fe1ebfe Add a bunch of asserts to try to catch #2539
Message-ID: <20180118091806.E84E7A3296@lists.varnish-cache.org>


commit fe1ebfe42d559f17f798d19b03db95a75fd469c0
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Jan 18 09:16:33 2018 +0000

    Add a bunch of asserts to try to catch #2539

diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c
index e76b0b2..599f908 100644
--- a/bin/varnishd/http2/cache_http2_session.c
+++ b/bin/varnishd/http2/cache_http2_session.c
@@ -326,15 +326,21 @@ h2_new_session(struct worker *wrk, void *arg)
 	assert(HTC_S_COMPLETE == H2_prism_complete(h2->htc));
 	HTC_RxPipeline(h2->htc, h2->htc->rxbuf_b + sizeof(H2_prism));
 	HTC_RxInit(h2->htc, h2->ws);
+	AN(h2->ws->r);
 	VSLb(h2->vsl, SLT_Debug, "H2: Got pu PRISM");
 
 	THR_SetRequest(h2->srq);
+	AN(h2->ws->r);
 
 	l = h2_enc_settings(&h2->local_settings, settings, sizeof (settings));
+	AN(h2->ws->r);
 	H2_Send_Get(wrk, h2, h2->req0);
+	AN(h2->ws->r);
 	H2_Send_Frame(wrk, h2,
 	    H2_F_SETTINGS, H2FF_NONE, l, 0, settings);
+	AN(h2->ws->r);
 	H2_Send_Rel(h2, h2->req0);
+	AN(h2->ws->r);
 
 	/* and off we go... */
 	h2->cond = &wrk->cond;
@@ -347,6 +353,7 @@ h2_new_session(struct worker *wrk, void *arg)
 			h2->error = H2CE_INTERNAL_ERROR;
 			break;
 		}
+		AN(h2->ws->r);
 	}
 
 	AN(h2->error);

From daghf at varnish-software.com  Thu Jan 18 15:40:10 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Thu, 18 Jan 2018 15:40:10 +0000 (UTC)
Subject: [master] 0cfa869 Don't HTC_RxStuff with a non-reserved workspace
Message-ID: <20180118154010.C567DAC985@lists.varnish-cache.org>


commit 0cfa8698eedecb92047316039dbb3e1733239a76
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Thu Jan 18 16:35:38 2018 +0100

    Don't HTC_RxStuff with a non-reserved workspace
    
    This commit partially reverts and reworks b9f7170b0. We now do the "wait
    for active streams" handling via the loop in h2_new_session, to ensure
    the workspace is in a predictable state once HTC_RxStuff is called
    again.
    
    Fixes: #2539

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 89eb620..45ab188 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -891,40 +891,36 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2)
 	enum htc_status_e hs;
 	h2_frame h2f;
 	h2_error h2e;
-	int again;
 	struct h2_req *r2, *r22;
 	char b[8];
 
 	ASSERT_RXTHR(h2);
 	(void)VTCP_blocking(*h2->htc->rfd);
-	while (1) {
-		h2->sess->t_idle = VTIM_real();
-		hs = HTC_RxStuff(h2->htc, h2_frame_complete,
-		    NULL, NULL, NAN,
-		    h2->sess->t_idle + cache_param->timeout_idle,
-		    16384 + 9);		// rfc7540,l,4228,4228
-		if (hs == HTC_S_COMPLETE)
-			break;
-		else if (hs == HTC_S_TIMEOUT) {
-			again = 0;
-			VTAILQ_FOREACH_SAFE(r2, &h2->streams, list, r22) {
-				switch (r2->state) {
-				case H2_S_CLOSED:
-					if (!r2->scheduled)
-						h2_del_req(wrk, r2);
-					break;
-				case H2_S_OPEN:
-				case H2_S_CLOS_REM:
-				case H2_S_CLOS_LOC:
-					again = 1;
-					break;
-				default:
-					break;
-				}
+	h2->sess->t_idle = VTIM_real();
+	hs = HTC_RxStuff(h2->htc, h2_frame_complete,
+	    NULL, NULL, NAN,
+	    h2->sess->t_idle + cache_param->timeout_idle,
+	    16384 + 9);		// rfc7540,l,4228,4228
+	switch (hs) {
+	case HTC_S_COMPLETE:
+		break;
+	case HTC_S_TIMEOUT:
+		VTAILQ_FOREACH_SAFE(r2, &h2->streams, list, r22) {
+			switch (r2->state) {
+			case H2_S_CLOSED:
+				if (!r2->scheduled)
+					h2_del_req(wrk, r2);
+				break;
+			case H2_S_OPEN:
+			case H2_S_CLOS_REM:
+			case H2_S_CLOS_LOC:
+				return (1);
+			default:
+				break;
 			}
-			if (again)
-				continue;
 		}
+		/* FALLTHROUGH */
+	default:
 		Lck_Lock(&h2->sess->mtx);
 		VSLb(h2->vsl, SLT_Debug, "H2: No frame (hs=%d)", hs);
 		h2->error = H2CE_NO_ERROR;
diff --git a/bin/varnishtest/tests/r02539.vtc b/bin/varnishtest/tests/r02539.vtc
new file mode 100644
index 0000000..de7be73
--- /dev/null
+++ b/bin/varnishtest/tests/r02539.vtc
@@ -0,0 +1,31 @@
+varnishtest "2359: H/2 Avoid RxStuff with a non-reserved WS"
+
+barrier b1 sock 2
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.set timeout_idle 1"
+
+varnish v1 -vcl+backend {
+	import vtc;
+
+	sub vcl_deliver {
+		vtc.barrier_sync("${b1_sock}");
+	}
+} -start
+
+client c1 {
+	stream 1 {
+		txreq
+		rxresp
+		expect resp.status == 200
+	} -start
+
+	delay 3
+	barrier b1 sync
+	stream 1 -wait
+} -run

From nils.goroll at uplex.de  Thu Jan 18 17:19:05 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Thu, 18 Jan 2018 17:19:05 +0000 (UTC)
Subject: [master] 06012a1 no vsm, no identity
Message-ID: <20180118171905.CA8F6AE921@lists.varnish-cache.org>


commit 06012a1f44bcb88fa403466db50a13d912e53859
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Jan 18 18:18:00 2018 +0100

    no vsm, no identity
    
    Fixes #2546

diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c
index 1fed3cb..3ba61db 100644
--- a/bin/varnishhist/varnishhist.c
+++ b/bin/varnishhist/varnishhist.c
@@ -621,7 +621,10 @@ main(int argc, char **argv)
 
 	VUT_Signal(vut_sighandler);
 	VUT_Setup(vut);
-	ident = VSM_Dup(vut->vsm, "Arg", "-i");
+	if (vut->vsm)
+		ident = VSM_Dup(vut->vsm, "Arg", "-i");
+	else
+		ident = strdup("");
 	if (pthread_create(&thr, NULL, do_curses, NULL) != 0)
 		VUT_Error(vut, 1, "pthread_create(): %s", strerror(errno));
 	vut->dispatch_f = accumulate;
diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c
index f0fa320..f47c514 100644
--- a/bin/varnishtop/varnishtop.c
+++ b/bin/varnishtop/varnishtop.c
@@ -378,7 +378,10 @@ main(int argc, char **argv)
 
 	VUT_Signal(vut_sighandler);
 	VUT_Setup(vut);
-	ident = VSM_Dup(vut->vsm, "Arg", "-i");
+	if (vut->vsm)
+		ident = VSM_Dup(vut->vsm, "Arg", "-i");
+	else
+		ident = strdup("");
 	if (!once) {
 		if (pthread_create(&thr, NULL, do_curses, NULL) != 0) {
 			fprintf(stderr, "pthread_create(): %s\n",

From nils.goroll at uplex.de  Fri Jan 19 08:48:07 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 19 Jan 2018 08:48:07 +0000 (UTC)
Subject: [master] 9e0543c fix VSM_map mmap length for real
Message-ID: <20180119084807.E6DC4BD5E4@lists.varnish-cache.org>


commit 9e0543c8abca2bb70b2b086e291a2d01d20c593b
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Jan 18 06:47:34 2018 +0100

    fix VSM_map mmap length for real
    
    I got this wrong in 69695c23e53942968fd65cd4a8627d47d4e21a70 for the
    case that the segment spans two pages.
    
    Ref: #2541
    
    mmaps from b25.vtc varnishstat:
    
    before this change:
    
    6598  open("/tmp/vtc.6492.2b3b5875/v1//_.vsm_child/_.VSC_cluster.322833fd50de0860", O_RDONLY) = 21
    6598  mmap(NULL, 4096, PROT_READ, MAP_SHARED, 21, 0) = 0x7fb92529e000
    ...
    6598  open("/tmp/vtc.6492.2b3b5875/v1//_.vsm_child/_.VSC_cluster.322833fd50de0860", O_RDONLY) = 21
    6598  mmap(NULL, 8192, PROT_READ, MAP_SHARED, 21, 0) = 0x7fb925271000
    ...
    6598  open("/tmp/vtc.6492.2b3b5875/v1//_.vsm_child/_.VSC_cluster.322833fd50de0860", O_RDONLY) = 21
    6598  mmap(NULL, 12288, PROT_READ, MAP_SHARED, 21, 0x1000) = 0x7fb925222000
    ...
    6598  open("/tmp/vtc.6492.2b3b5875/v1//_.vsm_child/_.VSC_cluster.322833fd50de0860", O_RDONLY) = 21
    6598  mmap(NULL, 12288, PROT_READ, MAP_SHARED, 21, 0x2000) = 0x7fb92521f000
    
    after this change:
    
    7749  open("/tmp/vtc.7648.2c3869d9/v1//_.vsm_child/_.VSC_cluster.09720dc34dd3559c", O_RDONLY) = 21
    7749  mmap(NULL, 4096, PROT_READ, MAP_SHARED, 21, 0) = 0x7f0938a28000
    ...
    7749  open("/tmp/vtc.7648.2c3869d9/v1//_.vsm_child/_.VSC_cluster.09720dc34dd3559c", O_RDONLY) = 21
    7749  mmap(NULL, 4096, PROT_READ, MAP_SHARED, 21, 0x1000) = 0x7f09389f9000
    ...
    7749  open("/tmp/vtc.7648.2c3869d9/v1//_.vsm_child/_.VSC_cluster.09720dc34dd3559c", O_RDONLY) = 21
    7749  mmap(NULL, 8192, PROT_READ, MAP_SHARED, 21, 0x1000) = 0x7f09389d2000
    ...
    7749  open("/tmp/vtc.7648.2c3869d9/v1//_.vsm_child/_.VSC_cluster.09720dc34dd3559c", O_RDONLY) = 21
    7749  mmap(NULL, 4096, PROT_READ, MAP_SHARED, 21, 0x2000) = 0x7f09389d1000
    
    Notice that we keep the mappings at one or two pages now and do not increase
    the length with the offset.

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 0766e53..b2b853c 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -720,7 +720,8 @@ VSM_Map(struct vsm *vd, struct vsm_fantom *vf)
 
 	sz = strtoul(vg->av[3], NULL, 10);
 	assert(sz > 0);
-	len = RUP2(of + sz, ps);
+	assert(of >= off);
+	len = RUP2(of - off + sz, ps);
 
 	vsb = VSB_new_auto();
 	AN(vsb);

From nils.goroll at uplex.de  Fri Jan 19 08:48:08 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 19 Jan 2018 08:48:08 +0000 (UTC)
Subject: [master] 46561a1 NOP: pull out the actual mmap code from VSM_Map()
Message-ID: <20180119084808.06021BD5E7@lists.varnish-cache.org>


commit 46561a19f25e947cdb7748d14f22113671b1554a
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Jan 18 10:21:25 2018 +0100

    NOP: pull out the actual mmap code from VSM_Map()

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index b2b853c..45aa6ce 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -148,6 +148,53 @@ vsm_diag(struct vsm *vd, const char *fmt, ...)
 
 /*--------------------------------------------------------------------*/
 
+static int
+vsm_mapseg(struct vsm *vd, struct vsm_seg *vg)
+{
+	size_t of, off, sz, ps, len;
+	struct vsb *vsb;
+	int fd;
+
+	CHECK_OBJ_NOTNULL(vg, VSM_SEG_MAGIC);
+
+	ps = getpagesize();
+	of = strtoul(vg->av[2], NULL, 10);
+	off = RDN2(of, ps);
+
+	sz = strtoul(vg->av[3], NULL, 10);
+	assert(sz > 0);
+	assert(of >= off);
+	len = RUP2(of - off + sz, ps);
+
+	vsb = VSB_new_auto();
+	AN(vsb);
+	VSB_printf(vsb, "%s/%s/%s", vd->dname, vg->set->dname, vg->av[1]);
+	AZ(VSB_finish(vsb));
+
+	fd = open(VSB_data(vsb), O_RDONLY);	// XXX: openat
+	if (fd < 0) {
+		VSB_destroy(&vsb);
+		return (vsm_diag(vd, "Could not open segment"));
+	}
+
+	vg->s = (void*)mmap(NULL, len,
+	    PROT_READ,
+	    MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED,
+	    fd, (off_t)off);
+
+	VSB_destroy(&vsb);
+
+	closefd(&fd);
+	if (vg->s == MAP_FAILED)
+		return (vsm_diag(vd, "Could not mmap segment"));
+
+	vg->b = (char*)(vg->s) + of - off;
+	vg->e = (char *)vg->b + sz;
+	vg->sz = len;
+
+	return (0);
+}
+
 static void
 vsm_unmapseg(struct vsm_seg *vg)
 {
@@ -690,9 +737,7 @@ int
 VSM_Map(struct vsm *vd, struct vsm_fantom *vf)
 {
 	struct vsm_seg *vg;
-	size_t of, off, sz, ps, len;
-	struct vsb *vsb;
-	int fd;
+	int r;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	AN(vd->attached);
@@ -714,40 +759,10 @@ VSM_Map(struct vsm *vd, struct vsm_fantom *vf)
 		return (0);
 	}
 
-	ps = getpagesize();
-	of = strtoul(vg->av[2], NULL, 10);
-	off = RDN2(of, ps);
-
-	sz = strtoul(vg->av[3], NULL, 10);
-	assert(sz > 0);
-	assert(of >= off);
-	len = RUP2(of - off + sz, ps);
-
-	vsb = VSB_new_auto();
-	AN(vsb);
-	VSB_printf(vsb, "%s/%s/%s", vd->dname, vg->set->dname, vg->av[1]);
-	AZ(VSB_finish(vsb));
+	r = vsm_mapseg(vd, vg);
 
-	fd = open(VSB_data(vsb), O_RDONLY);	// XXX: openat
-	if (fd < 0) {
-		VSB_destroy(&vsb);
-		return (vsm_diag(vd, "Could not open segment"));
-	}
-
-	vg->s = (void*)mmap(NULL, len,
-	    PROT_READ,
-	    MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED,
-	    fd, (off_t)off);
-
-	VSB_destroy(&vsb);
-
-	closefd(&fd);
-	if (vg->s == MAP_FAILED)
-		return (vsm_diag(vd, "Could not mmap segment"));
-
-	vg->b = (char*)(vg->s) + of - off;
-	vg->e = (char *)vg->b + sz;
-	vg->sz = len;
+	if (r)
+		return (r);
 
 	vf->b = vg->b;
 	vf->e = vg->e;

From nils.goroll at uplex.de  Fri Jan 19 08:48:08 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 19 Jan 2018 08:48:08 +0000 (UTC)
Subject: [master] 7a30f9f introduce a bitfield for vsm client segment flags
Message-ID: <20180119084808.1F8A0BD5EA@lists.varnish-cache.org>


commit 7a30f9f19ea3c15da1c696a94899709f240b2ac4
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Jan 18 07:52:12 2018 +0100

    introduce a bitfield for vsm client segment flags

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 45aa6ce..b63684f 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -76,9 +76,10 @@ struct vsm_set;
 struct vsm_seg {
 	unsigned		magic;
 #define VSM_SEG_MAGIC		0xeb6c6dfd
+	unsigned		flags;
+#define VSM_FLAG_MARKSCAN	(1U<<0)
+#define VSM_FLAG_STALE		(1U<<1)
 	VTAILQ_ENTRY(vsm_seg)	list;
-	int			markscan;
-	int			stale;
 	struct vsm_set		*set;
 	char			**av;
 	int			refs;
@@ -219,7 +220,7 @@ vsm_delseg(struct vsm_seg *vg)
 	if (vg->b != NULL)
 		vsm_unmapseg(vg);
 
-	if (vg->stale)
+	if (vg->flags & VSM_FLAG_STALE)
 		VTAILQ_REMOVE(&vg->set->stale, vg, list);
 	else
 		VTAILQ_REMOVE(&vg->set->segs, vg, list);
@@ -487,7 +488,7 @@ vsm_refresh_set2(struct vsm *vd, struct vsm_set *vs, struct vsb *vsb)
 	p = VSB_data(vsb) + ac;
 
 	VTAILQ_FOREACH(vg, &vs->segs, list)
-		vg->markscan = 0;
+		vg->flags &= ~VSM_FLAG_MARKSCAN;
 
 	/*
 	 * Efficient comparison by walking the two lists side-by-side because
@@ -527,7 +528,7 @@ vsm_refresh_set2(struct vsm *vd, struct vsm_set *vs, struct vsb *vsb)
 			AN(vg2);
 			vg2->av = av;
 			vg2->set = vs;
-			vg2->markscan = 1;
+			vg2->flags = VSM_FLAG_MARKSCAN;
 			vg2->serial = ++vd->serial;
 			VTAILQ_INSERT_TAIL(&vs->segs, vg2, list);
 			continue;
@@ -542,7 +543,7 @@ vsm_refresh_set2(struct vsm *vd, struct vsm_set *vs, struct vsb *vsb)
 			continue;
 
 		/* entry compared equal, so it survives */
-		vg->markscan = 1;
+		vg->flags |= VSM_FLAG_MARKSCAN;
 		vg = VTAILQ_NEXT(vg, list);
 	}
 	return (retval);
@@ -558,10 +559,11 @@ vsm_refresh_set(struct vsm *vd, struct vsm_set *vs, struct vsb *vsb)
 	if (retval & VSM_NUKE_ALL)
 		retval |= VSM_MGT_CHANGED;
 	VTAILQ_FOREACH_SAFE(vg, &vs->segs, list, vg2) {
-		if (!vg->markscan || (retval & VSM_NUKE_ALL)) {
+		if ((vg->flags & VSM_FLAG_MARKSCAN) == 0 ||
+		    (retval & VSM_NUKE_ALL)) {
 			VTAILQ_REMOVE(&vs->segs, vg, list);
 			if (vg->refs) {
-				vg->stale = 1;
+				vg->flags |= VSM_FLAG_STALE;
 				VTAILQ_INSERT_TAIL(&vs->stale, vg, list);
 			} else {
 				VAV_Free(vg->av);
@@ -793,7 +795,7 @@ VSM_Unmap(struct vsm *vd, struct vsm_fantom *vf)
 	if (vg->refs > 0)
 		return(0);
 	vsm_unmapseg(vg);
-	if (vg->stale)
+	if (vg->flags & VSM_FLAG_STALE)
 		vsm_delseg(vg);
 	return (0);
 }
@@ -808,7 +810,7 @@ VSM_StillValid(const struct vsm *vd, const struct vsm_fantom *vf)
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
 	AN(vf);
 	vg = vsm_findseg(vd, vf);
-	if (vg == NULL || vg->stale)
+	if (vg == NULL || vg->flags & VSM_FLAG_STALE)
 		return (VSM_invalid);
 	return (VSM_valid);
 }

From nils.goroll at uplex.de  Fri Jan 19 08:48:08 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 19 Jan 2018 08:48:08 +0000 (UTC)
Subject: [master] 9194477 gc unused includes thanks to Flexelint
Message-ID: <20180119084808.46D82BD5F0@lists.varnish-cache.org>


commit 91944772141cdd9b61ee8e2f0135d197ba3eb46d
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Jan 18 07:55:53 2018 +0100

    gc unused includes thanks to Flexelint

diff --git a/lib/libvarnishapi/vjsn.c b/lib/libvarnishapi/vjsn.c
index 1859673..8f08e74 100644
--- a/lib/libvarnishapi/vjsn.c
+++ b/lib/libvarnishapi/vjsn.c
@@ -33,7 +33,6 @@
 #include <stdio.h>
 #include <string.h>
 #include <stdlib.h>
-#include <sys/stat.h>
 #include <unistd.h>
 
 #include "vdef.h"
diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c
index a028423..b4b7752 100644
--- a/lib/libvarnishapi/vsl.c
+++ b/lib/libvarnishapi/vsl.c
@@ -40,7 +40,6 @@
 #include "miniobj.h"
 
 #include "vbm.h"
-#include "vmb.h"
 #include "vqueue.h"
 #include "vre.h"
 #include "vsb.h"
diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c
index f75e172..4b35558 100644
--- a/lib/libvarnishapi/vsl_cursor.c
+++ b/lib/libvarnishapi/vsl_cursor.c
@@ -37,7 +37,6 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
 
diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c
index f4053f6..6766cb9 100644
--- a/lib/libvarnishapi/vut.c
+++ b/lib/libvarnishapi/vut.c
@@ -40,7 +40,6 @@
 #include <errno.h>
 #include <string.h>
 #include <signal.h>
-#include <math.h>
 
 #include "compat/daemon.h"
 #include "vdef.h"

From nils.goroll at uplex.de  Fri Jan 19 08:48:08 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 19 Jan 2018 08:48:08 +0000 (UTC)
Subject: [master] 2babc68 Map whole VSM clusters instead of their segments
Message-ID: <20180119084808.60547BD5F3@lists.varnish-cache.org>


commit 2babc68abe967715b51aa526f14417f1c081cc05
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Thu Jan 18 12:45:18 2018 +0100

    Map whole VSM clusters instead of their segments
    
    Merges #2545

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index b63684f..f19468e 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -79,8 +79,10 @@ struct vsm_seg {
 	unsigned		flags;
 #define VSM_FLAG_MARKSCAN	(1U<<0)
 #define VSM_FLAG_STALE		(1U<<1)
+#define VSM_FLAG_CLUSTER	(1U<<2)
 	VTAILQ_ENTRY(vsm_seg)	list;
 	struct vsm_set		*set;
+	struct vsm_seg		*cluster;
 	char			**av;
 	int			refs;
 	void			*s;
@@ -96,6 +98,7 @@ struct vsm_set {
 	const char		*dname;
 	VTAILQ_HEAD(,vsm_seg)	segs;
 	VTAILQ_HEAD(,vsm_seg)	stale;
+	VTAILQ_HEAD(,vsm_seg)	clusters;
 
 	int			dfd;
 	struct stat		dst;
@@ -158,10 +161,18 @@ vsm_mapseg(struct vsm *vd, struct vsm_seg *vg)
 
 	CHECK_OBJ_NOTNULL(vg, VSM_SEG_MAGIC);
 
+	if (vg->s != NULL)
+		return (0);
+
 	ps = getpagesize();
+
 	of = strtoul(vg->av[2], NULL, 10);
 	off = RDN2(of, ps);
 
+	if (vg->flags & VSM_FLAG_CLUSTER)
+		assert(of == 0);
+	assert(vg->cluster == NULL);
+
 	sz = strtoul(vg->av[3], NULL, 10);
 	assert(sz > 0);
 	assert(of >= off);
@@ -217,11 +228,13 @@ vsm_delseg(struct vsm_seg *vg)
 
 	CHECK_OBJ_NOTNULL(vg, VSM_SEG_MAGIC);
 
-	if (vg->b != NULL)
+	if (vg->s != NULL)
 		vsm_unmapseg(vg);
 
 	if (vg->flags & VSM_FLAG_STALE)
 		VTAILQ_REMOVE(&vg->set->stale, vg, list);
+	else if (vg->flags & VSM_FLAG_CLUSTER)
+		VTAILQ_REMOVE(&vg->set->clusters, vg, list);
 	else
 		VTAILQ_REMOVE(&vg->set->segs, vg, list);
 	VAV_Free(vg->av);
@@ -239,6 +252,7 @@ vsm_newset(const char *dirname)
 	AN(vs);
 	VTAILQ_INIT(&vs->segs);
 	VTAILQ_INIT(&vs->stale);
+	VTAILQ_INIT(&vs->clusters);
 	vs->dname = dirname;
 	vs->dfd = vs->fd = -1;
 	return (vs);
@@ -260,6 +274,8 @@ vsm_delset(struct vsm_set **p)
 		vsm_delseg(VTAILQ_FIRST(&vs->stale));
 	while (!VTAILQ_EMPTY(&vs->segs))
 		vsm_delseg(VTAILQ_FIRST(&vs->segs));
+	while (!VTAILQ_EMPTY(&vs->clusters))
+		vsm_delseg(VTAILQ_FIRST(&vs->clusters));
 	FREE_OBJ(vs);
 }
 
@@ -387,6 +403,21 @@ vsm_cmp_av(char * const *a1, char * const *a2)
 	}
 }
 
+static struct vsm_seg *
+vsm_findcluster(const struct vsm_seg *vga)
+{
+	const struct vsm_set *vs = vga->set;
+	struct vsm_seg *vg;
+	AN(vs);
+	AN(vga->av[1]);
+	VTAILQ_FOREACH(vg, &vs->clusters, list) {
+		AN(vg->av[1]);
+		if (! strcmp(vga->av[1], vg->av[1]))
+			return (vg);
+	}
+	return (NULL);
+}
+
 static unsigned
 vsm_refresh_set2(struct vsm *vd, struct vsm_set *vs, struct vsb *vsb)
 {
@@ -518,10 +549,6 @@ vsm_refresh_set2(struct vsm *vd, struct vsm_set *vs, struct vsb *vsb)
 			VAV_Free(av);
 			break;
 		}
-		if (ac == 4) {
-			VAV_Free(av);
-			continue;
-		}
 
 		if (vg == NULL) {
 			ALLOC_OBJ(vg2, VSM_SEG_MAGIC);
@@ -530,7 +557,13 @@ vsm_refresh_set2(struct vsm *vd, struct vsm_set *vs, struct vsb *vsb)
 			vg2->set = vs;
 			vg2->flags = VSM_FLAG_MARKSCAN;
 			vg2->serial = ++vd->serial;
-			VTAILQ_INSERT_TAIL(&vs->segs, vg2, list);
+			if (ac == 4) {
+				vg2->flags |= VSM_FLAG_CLUSTER;
+				VTAILQ_INSERT_TAIL(&vs->clusters, vg2, list);
+			} else {
+				VTAILQ_INSERT_TAIL(&vs->segs, vg2, list);
+				vg2->cluster = vsm_findcluster(vg2);
+			}
 			continue;
 		}
 
@@ -738,7 +771,8 @@ VSM__itern(struct vsm *vd, struct vsm_fantom *vf)
 int
 VSM_Map(struct vsm *vd, struct vsm_fantom *vf)
 {
-	struct vsm_seg *vg;
+	struct vsm_seg *vg, *vgc;
+	size_t of, sz;
 	int r;
 
 	CHECK_OBJ_NOTNULL(vd, VSM_MAGIC);
@@ -761,10 +795,39 @@ VSM_Map(struct vsm *vd, struct vsm_fantom *vf)
 		return (0);
 	}
 
-	r = vsm_mapseg(vd, vg);
+	assert(vg->refs == 0);
+
+	vgc = vg->cluster;
 
+	if (vgc == NULL) {
+		r = vsm_mapseg(vd, vg);
+		if (r)
+			return (r);
+		vf->b = vg->b;
+		vf->e = vg->e;
+
+		vg->refs++;
+
+		return (0);
+	}
+
+	assert(vgc->flags & VSM_FLAG_CLUSTER);
+	assert(vg->s == NULL);
+	assert(vg->sz == 0);
+
+	r = vsm_mapseg(vd, vgc);
 	if (r)
 		return (r);
+	vgc->refs++;
+
+	of = strtoul(vg->av[2], NULL, 10);
+	sz = strtoul(vg->av[3], NULL, 10);
+	assert(sz > 0);
+
+	assert(vgc->sz >= of + sz);
+	assert(vgc->s == vgc->b);
+	vg->b = (char *)vgc->b + of;
+	vg->e = (char *)vg->b + sz;
 
 	vf->b = vg->b;
 	vf->e = vg->e;
@@ -794,7 +857,17 @@ VSM_Unmap(struct vsm *vd, struct vsm_fantom *vf)
 	vf->e = NULL;
 	if (vg->refs > 0)
 		return(0);
-	vsm_unmapseg(vg);
+
+	if (vg->cluster) {
+		assert(vg->s == NULL);
+		assert(vg->sz == 0);
+		assert(vg->cluster->refs > 0);
+		if (--vg->cluster->refs == 0)
+			vsm_unmapseg(vg->cluster);
+		vg->b = vg->e = NULL;
+	} else {
+		vsm_unmapseg(vg);
+	}
 	if (vg->flags & VSM_FLAG_STALE)
 		vsm_delseg(vg);
 	return (0);

From fgsch at lodoss.net  Sun Jan 21 16:57:08 2018
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Sun, 21 Jan 2018 16:57:08 +0000 (UTC)
Subject: [master] 3f3213d Whitespace
Message-ID: <20180121165708.84000B2233@lists.varnish-cache.org>


commit 3f3213da07ccfec6a81c13787cd6524522fccf6b
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Sun Jan 21 16:33:04 2018 +0000

    Whitespace

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index f19468e..62b174b 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -412,7 +412,7 @@ vsm_findcluster(const struct vsm_seg *vga)
 	AN(vga->av[1]);
 	VTAILQ_FOREACH(vg, &vs->clusters, list) {
 		AN(vg->av[1]);
-		if (! strcmp(vga->av[1], vg->av[1]))
+		if (!strcmp(vga->av[1], vg->av[1]))
 			return (vg);
 	}
 	return (NULL);
diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py
index 023f0dd..76c9f25 100644
--- a/lib/libvcc/vsctool.py
+++ b/lib/libvcc/vsctool.py
@@ -65,7 +65,7 @@ def genhdr(fo, name):
 	fo.write('/*\n')
 	fo.write(' * NB:  This file is machine generated, DO NOT EDIT!\n')
 	fo.write(' *\n')
-	fo.write(' * Edit ' + name + 
+	fo.write(' * Edit ' + name +
 	    '.vsc and run lib/libvcc/vsctool.py instead.\n')
 	fo.write(' */\n')
 	fo.write('\n')
@@ -143,7 +143,7 @@ class vscset(object):
 		fo.write("};\n")
 		fo.write("\n")
 
-		fo.write("#define VSC_" + self.name + 
+		fo.write("#define VSC_" + self.name +
 		    "_size PRNDUP(sizeof(" + self.struct + "))\n\n")
 
 		fo.write(self.struct + " *VSC_" + self.name + "_New")
@@ -178,7 +178,7 @@ class vscset(object):
 		fo.write("    _Static_assert(\t\t\t\t\\\n")
 		fo.write("\toffsetof(" + self.struct + ", a) == n,\t\\\n")
 		fo.write("\t\"VSC element '\" #a \"' at wrong offset\")\n\n")
-		
+
 		for i in self.mbrs:
 			fo.write("PARANOIA(" + i.arg)
 			fo.write(", %d);\n" % (i.param["index"]))

From fgsch at lodoss.net  Sun Jan 21 16:57:08 2018
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Sun, 21 Jan 2018 16:57:08 +0000 (UTC)
Subject: [master] 3cdb917 Add a delay when TERMinating a process
Message-ID: <20180121165708.9B539B2236@lists.varnish-cache.org>


commit 3cdb9171e70a78a2aefcce4923464db66352e3a2
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Sun Jan 21 16:01:14 2018 +0000

    Add a delay when TERMinating a process
    
    Otherwise we might end up missing the signal and fail later when
    reset the process.  All tests are passing in OSX now.

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index a251c6b..4aeabcd 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -756,6 +756,7 @@ cmd_process(CMD_ARGS)
 		}
 		if (!strcmp(*av, "-stop")) {
 			process_kill(p, "TERM");
+			sleep(1);
 			continue;
 		}
 		if (!strcmp(*av, "-wait")) {

From fgsch at lodoss.net  Mon Jan 22 02:41:07 2018
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon, 22 Jan 2018 02:41:07 +0000 (UTC)
Subject: [master] 67d5be2 Switch to clang 6.0
Message-ID: <20180122024107.A5F40BC026@lists.varnish-cache.org>


commit 67d5be2da562a90f24fc6286c48f7b58f1887f4c
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Jan 22 01:33:44 2018 +0000

    Switch to clang 6.0

diff --git a/.travis.yml b/.travis.yml
index ef8a2ee..4b310be 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,13 +12,13 @@ matrix:
   - os: linux
     dist: trusty
     compiler: clang
-    env: CLANG=5.0 SAN_FLAGS="--enable-asan --enable-ubsan"
+    env: CLANG=6.0 SAN_FLAGS="--enable-asan --enable-ubsan"
   - os: osx
     osx_image: xcode9.2
     compiler: clang
   allow_failures:
   - os: osx
-  - env: CLANG=5.0 SAN_FLAGS="--enable-asan --enable-ubsan"
+  - env: CLANG=6.0 SAN_FLAGS="--enable-asan --enable-ubsan"
 addons:
   apt:
     packages:

From fgsch at lodoss.net  Mon Jan 22 02:41:07 2018
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Mon, 22 Jan 2018 02:41:07 +0000 (UTC)
Subject: [master] f319a34 Plug minor leak when discarding a vcl
Message-ID: <20180122024107.B5A53BC028@lists.varnish-cache.org>


commit f319a3486f9f2a403bbaf75a2b1044470d8f965f
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Mon Jan 22 02:21:14 2018 +0000

    Plug minor leak when discarding a vcl

diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c
index d605dcf..aa11668 100644
--- a/bin/varnishd/cache/cache_vcl.c
+++ b/bin/varnishd/cache/cache_vcl.c
@@ -475,6 +475,7 @@ vcl_KillBackends(struct vcl *vcl)
 			break;
 		VTAILQ_REMOVE(&vcl->director_list, d, vcl_list);
 		AN(d->destroy);
+		REPLACE(d->display_name, NULL);
 		d->destroy(d);
 	}
 }

From daghf at varnish-software.com  Mon Jan 22 09:04:09 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Mon, 22 Jan 2018 09:04:09 +0000 (UTC)
Subject: [master] d56e94a h2: Move RST_STREAM tx handling to its own function
Message-ID: <20180122090409.A4CCC9781E@lists.varnish-cache.org>


commit d56e94a0503c7b8ae333c29381036af5f9587f04
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Wed Jan 17 11:05:21 2018 +0100

    h2: Move RST_STREAM tx handling to its own function

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 45ab188..71ec6a8 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -804,12 +804,34 @@ h2_frame_complete(struct http_conn *htc)
 /**********************************************************************/
 
 static h2_error
+h2_tx_rst(struct worker *wrk, struct h2_sess *h2, h2_error h2e)
+{
+	h2_error ret;
+	char b[4];
+
+	CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC);
+
+	Lck_Lock(&h2->sess->mtx);
+	VSLb(h2->vsl, SLT_Debug, "H2: stream %u: %s", h2->rxf_stream, h2e->txt);
+	Lck_Unlock(&h2->sess->mtx);
+	vbe32enc(b, h2e->val);
+
+	H2_Send_Get(wrk, h2, h2->req0);
+	ret = H2_Send_Frame(wrk, h2, H2_F_RST_STREAM,
+	    0, sizeof b, h2->rxf_stream, b);
+	H2_Send_Rel(h2, h2->req0);
+
+	return (ret);
+}
+
+/**********************************************************************/
+
+static h2_error
 h2_procframe(struct worker *wrk, struct h2_sess *h2,
     h2_frame h2f)
 {
 	struct h2_req *r2 = NULL, *r22;
 	h2_error h2e;
-	char b[4];
 
 	ASSERT_RXTHR(h2);
 	if (h2->rxf_stream == 0 && h2f->act_szero != 0)
@@ -857,17 +879,7 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2,
 	if (h2->rxf_stream == 0 || h2e->connection)
 		return (h2e);	// Connection errors one level up
 
-	Lck_Lock(&h2->sess->mtx);
-	VSLb(h2->vsl, SLT_Debug, "H2: stream %u: %s", h2->rxf_stream, h2e->txt);
-	Lck_Unlock(&h2->sess->mtx);
-	vbe32enc(b, h2e->val);
-
-	H2_Send_Get(wrk, h2, h2->req0);
-	(void)H2_Send_Frame(wrk, h2, H2_F_RST_STREAM,
-	    0, sizeof b, h2->rxf_stream, b);
-	H2_Send_Rel(h2, h2->req0);
-
-	return (0);
+	return (h2_tx_rst(wrk, h2, h2e));
 }
 
 /***********************************************************************

From daghf at varnish-software.com  Mon Jan 22 09:04:09 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Mon, 22 Jan 2018 09:04:09 +0000 (UTC)
Subject: [master] 6f74146 Enforce h2_max_concurrent_streams
Message-ID: <20180122090409.C399397821@lists.varnish-cache.org>


commit 6f74146f2189d24a340c194bb756ce36123cff1b
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Wed Jan 17 13:41:35 2018 +0100

    Enforce h2_max_concurrent_streams

diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c
index 71ec6a8..6801e92 100644
--- a/bin/varnishd/http2/cache_http2_proto.c
+++ b/bin/varnishd/http2/cache_http2_proto.c
@@ -864,6 +864,13 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2,
 	if (r2 == NULL && h2f->act_sidle == 0) {
 		if (h2->rxf_stream <= h2->highest_stream)
 			return (H2CE_PROTOCOL_ERROR);	// rfc7540,l,1153,1158
+		if (h2->refcnt >= h2->local_settings.max_concurrent_streams) {
+			VSLb(h2->vsl, SLT_Debug,
+			     "H2: stream %u: Hit maximum number of "
+			     "concurrent streams", h2->rxf_stream);
+			// rfc7540,l,1200,1205
+			return (h2_tx_rst(wrk, h2, H2SE_REFUSED_STREAM));
+		}
 		h2->highest_stream = h2->rxf_stream;
 		r2 = h2_new_req(wrk, h2, h2->rxf_stream, NULL);
 		AN(r2);
diff --git a/bin/varnishtest/tests/t02012.vtc b/bin/varnishtest/tests/t02012.vtc
new file mode 100644
index 0000000..5319453
--- /dev/null
+++ b/bin/varnishtest/tests/t02012.vtc
@@ -0,0 +1,63 @@
+varnishtest "Test max_concurrent_streams"
+
+barrier b1 sock 5
+barrier b2 sock 3
+barrier b3 cond 2
+barrier b4 cond 2
+
+server s1 {
+       rxreq
+       txresp
+} -start
+
+varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.set debug +syncvsl"
+varnish v1 -cliok "param.set h2_max_concurrent_streams 3"
+
+varnish v1 -vcl+backend {
+	import vtc;
+	sub vcl_deliver {
+		if (req.http.sync) {
+			vtc.barrier_sync("${b1_sock}");
+			vtc.barrier_sync("${b2_sock}");
+		}
+	}
+} -start
+
+client c1 {
+	stream 1 {
+		txreq -hdr "sync" "1"
+		barrier b3 sync
+		barrier b1 sync
+		rxresp
+		expect resp.status == 200
+	} -start
+
+	stream 3 {
+		barrier b3 sync
+		txreq -hdr "sync" "1"
+		barrier b4 sync
+		barrier b1 sync
+		rxresp
+		expect resp.status == 200
+	} -start
+
+	stream 5 {
+		barrier b4 sync
+		barrier b1 sync
+		txreq
+		rxrst
+		expect rst.err == REFUSED_STREAM
+		barrier b2 sync
+	} -run
+
+	stream 1 -wait
+	stream 3 -wait
+
+	stream 7 {
+		txreq
+		rxresp
+		expect resp.status == 200
+	} -run
+
+} -run

From daghf at varnish-software.com  Mon Jan 22 13:33:08 2018
From: daghf at varnish-software.com (Dag Haavi Finstad)
Date: Mon, 22 Jan 2018 13:33:08 +0000 (UTC)
Subject: [master] 1867404 Kill stale comment
Message-ID: <20180122133308.7B3E8A0F9C@lists.varnish-cache.org>


commit 186740452b93861398eb1c5941e3511cd58408d7
Author: Dag Haavi Finstad <daghf at varnish-software.com>
Date:   Mon Jan 22 14:30:46 2018 +0100

    Kill stale comment

diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c
index fc3eeca..46b0f38 100644
--- a/bin/varnishd/http2/cache_http2_deliver.c
+++ b/bin/varnishd/http2/cache_http2_deliver.c
@@ -251,8 +251,6 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody)
 
 	AZ(req->wrk->v1l);
 
-	/* XXX: Optimize !sendbody case */
-
 	if (sendbody && req->resp_len == 0)
 		sendbody = 0;
 

From phk at FreeBSD.org  Wed Jan 24 09:27:13 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jan 2018 09:27:13 +0000 (UTC)
Subject: [master] 280495d Clean up some function arguments
Message-ID: <20180124092713.11BCAB3DF5@lists.varnish-cache.org>


commit 280495dc1c4e43987308a0ffc5f9fdf87156a83c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 24 08:54:00 2018 +0000

    Clean up some function arguments

diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c
index d11f926..ca27856 100644
--- a/lib/libvcc/vcc_acl.c
+++ b/lib/libvcc/vcc_acl.c
@@ -462,7 +462,6 @@ vcc_acl_emit(struct vcc *tl, const char *name, const char *rname, int anon)
 void
 vcc_ParseAcl(struct vcc *tl)
 {
-	struct token *an;
 	struct symbol *sym;
 
 	vcc_NextToken(tl);
@@ -470,10 +469,9 @@ vcc_ParseAcl(struct vcc *tl)
 
 	vcc_ExpectVid(tl, "ACL");
 	ERRCHK(tl);
-	an = tl->t;
+	sym = VCC_HandleSymbol(tl, ACL, ACL_SYMBOL_PREFIX);
 	vcc_NextToken(tl);
 
-	sym = VCC_HandleSymbol(tl, an, ACL, ACL_SYMBOL_PREFIX);
 	ERRCHK(tl);
 	AN(sym);
 
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 69adcec..b030c11 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -90,7 +90,7 @@ parse_set(struct vcc *tl)
 
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
-	sym = vcc_FindVar(tl, tl->t, 1, "cannot be set");
+	sym = vcc_FindVar(tl, 1, "cannot be set");
 	ERRCHK(tl);
 	assert(sym != NULL);
 	if (vcc_IdIs(tl->t, "bereq.body")) {
@@ -138,7 +138,7 @@ parse_unset(struct vcc *tl)
 	/* XXX: Wrong, should use VCC_Expr(HEADER) */
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
-	sym = vcc_FindVar(tl, tl->t, 1, "cannot be unset");
+	sym = vcc_FindVar(tl, 1, "cannot be unset");
 	ERRCHK(tl);
 	assert(sym != NULL);
 	if (sym->fmt != HEADER && !vcc_IdIs(tl->t, "bereq.body")) {
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index a5a433f..ed4aa50 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -258,9 +258,9 @@ vcc_ParseProbe(struct vcc *tl)
 	vcc_ExpectVid(tl, "backend probe");	/* ID: name */
 	ERRCHK(tl);
 	t_probe = tl->t;
+	sym = VCC_HandleSymbol(tl, PROBE, "vgc_probe");
 	vcc_NextToken(tl);
 
-	sym = VCC_HandleSymbol(tl, t_probe, PROBE, "vgc_probe");
 	ERRCHK(tl);
 	AN(sym);
 
@@ -473,9 +473,9 @@ vcc_ParseBackend(struct vcc *tl)
 	ERRCHK(tl);
 
 	t_be = tl->t;
+	sym = VCC_HandleSymbol(tl, BACKEND, "vgc_backend");
 	vcc_NextToken(tl);
 
-	sym = VCC_HandleSymbol(tl, t_be, BACKEND, "vgc_backend");
 	ERRCHK(tl);
 
 	Fh(tl, 0, "\nstatic struct director *%s;\n", sym->rname);
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index cff6c8f..4e98e94 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -292,8 +292,7 @@ void vcc_Eval_Func(struct vcc *tl, const char *spec,
     const char *extra, const struct symbol *sym);
 enum symkind VCC_HandleKind(vcc_type_t fmt);
 void VCC_GlobalSymbol(struct symbol *, vcc_type_t fmt, const char *pfx);
-struct symbol *VCC_HandleSymbol(struct vcc *, const struct token *,
-    vcc_type_t fmt, const char *pfx);
+struct symbol *VCC_HandleSymbol(struct vcc *, vcc_type_t , const char *);
 
 /* vcc_obj.c */
 extern const struct var vcc_vars[];
@@ -348,8 +347,7 @@ vcc_type_t VCC_Type(const char *p);
 
 /* vcc_var.c */
 sym_wildcard_t vcc_Var_Wildcard;
-const struct symbol *vcc_FindVar(struct vcc *tl, const struct token *t,
-    int wr_access, const char *use);
+const struct symbol *vcc_FindVar(struct vcc *, int, const char *);
 
 /* vcc_vmod.c */
 void vcc_ParseImport(struct vcc *tl);
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index 8342a89..ef526f1 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -228,8 +228,7 @@ VCC_GlobalSymbol(struct symbol *sym, vcc_type_t fmt, const char *pfx)
 }
 
 struct symbol *
-VCC_HandleSymbol(struct vcc *tl, const struct token *tk, vcc_type_t fmt,
-    const char *pfx)
+VCC_HandleSymbol(struct vcc *tl, vcc_type_t fmt, const char *pfx)
 {
 	struct symbol *sym;
 	enum symkind kind;
@@ -238,19 +237,19 @@ VCC_HandleSymbol(struct vcc *tl, const struct token *tk, vcc_type_t fmt,
 	kind = VCC_HandleKind(fmt);
 	assert(kind != SYM_NONE);
 
-	sym = VCC_SymbolTok(tl, NULL, tk, SYM_NONE, 0);
+	sym = VCC_SymbolTok(tl, NULL, tl->t, SYM_NONE, 0);
 	if (sym != NULL && sym->def_b != NULL && kind == sym->kind) {
 		p = VCC_SymKind(tl, sym);
 		VSB_printf(tl->sb, "%c%s '%.*s' redefined.\n",
-		    toupper(*p), p + 1, PF(tk));
-		vcc_ErrWhere(tl, tk);
+		    toupper(*p), p + 1, PF(tl->t));
+		vcc_ErrWhere(tl, tl->t);
 		VSB_printf(tl->sb, "First definition:\n");
 		AN(sym->def_b);
 		vcc_ErrWhere(tl, sym->def_b);
 		return (sym);
 	} else if (sym != NULL && sym->def_b != NULL) {
-		VSB_printf(tl->sb, "Name '%.*s' already defined.\n", PF(tk));
-		vcc_ErrWhere(tl, tk);
+		VSB_printf(tl->sb, "Name '%.*s' already defined.\n", PF(tl->t));
+		vcc_ErrWhere(tl, tl->t);
 		VSB_printf(tl->sb, "First definition:\n");
 		AN(sym->def_b);
 		vcc_ErrWhere(tl, sym->def_b);
@@ -258,17 +257,17 @@ VCC_HandleSymbol(struct vcc *tl, const struct token *tk, vcc_type_t fmt,
 	} else if (sym != NULL && sym->kind != kind) {
 		VSB_printf(tl->sb,
 		    "Name %.*s must have type '%s'.\n",
-		    PF(tk), VCC_SymKind(tl, sym));
-		vcc_ErrWhere(tl, tk);
+		    PF(tl->t), VCC_SymKind(tl, sym));
+		vcc_ErrWhere(tl, tl->t);
 		return (sym);
 	}
 	if (sym == NULL)
-		sym = VCC_SymbolTok(tl, NULL, tk, kind, 1);
+		sym = VCC_SymbolTok(tl, NULL, tl->t, kind, 1);
 	AN(sym);
 	AZ(sym->ndef);
 	VCC_GlobalSymbol(sym, fmt, pfx);
 	sym->ndef = 1;
 	if (sym->def_b == NULL)
-		sym->def_b = tk;
+		sym->def_b = tl->t;
 	return (sym);
 }
diff --git a/lib/libvcc/vcc_var.c b/lib/libvcc/vcc_var.c
index 4058b4d..02c4e5e 100644
--- a/lib/libvcc/vcc_var.c
+++ b/lib/libvcc/vcc_var.c
@@ -96,39 +96,38 @@ vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent,
 /*--------------------------------------------------------------------*/
 
 const struct symbol *
-vcc_FindVar(struct vcc *tl, const struct token *t, int wr_access,
-    const char *use)
+vcc_FindVar(struct vcc *tl, int wr_access, const char *use)
 {
 	const struct symbol *sym;
 
-	sym = VCC_SymbolTok(tl, NULL, t, SYM_VAR, 0);
+	sym = VCC_SymbolTok(tl, NULL, tl->t, SYM_VAR, 0);
 	if (tl->err)
 		return (NULL);
 	if (sym != NULL) {
 		if (wr_access && sym->w_methods == 0) {
 			VSB_printf(tl->sb, "Variable ");
-			vcc_ErrToken(tl, t);
+			vcc_ErrToken(tl, tl->t);
 			VSB_printf(tl->sb, " is read only.");
 			VSB_cat(tl->sb, "\nAt: ");
-			vcc_ErrWhere(tl, t);
+			vcc_ErrWhere(tl, tl->t);
 			return (NULL);
 		} else if (wr_access) {
-			vcc_AddUses(tl, t, sym->w_methods, use);
+			vcc_AddUses(tl, tl->t, sym->w_methods, use);
 		} else if (sym->r_methods == 0) {
 			VSB_printf(tl->sb, "Variable ");
-			vcc_ErrToken(tl, t);
+			vcc_ErrToken(tl, tl->t);
 			VSB_printf(tl->sb, " is write only.");
 			VSB_cat(tl->sb, "\nAt: ");
-			vcc_ErrWhere(tl, t);
+			vcc_ErrWhere(tl, tl->t);
 			return (NULL);
 		} else {
-			vcc_AddUses(tl, t, sym->r_methods, use);
+			vcc_AddUses(tl, tl->t, sym->r_methods, use);
 		}
 		return (sym);
 	}
 	VSB_printf(tl->sb, "Unknown variable ");
-	vcc_ErrToken(tl, t);
+	vcc_ErrToken(tl, tl->t);
 	VSB_cat(tl->sb, "\nAt: ");
-	vcc_ErrWhere(tl, t);
+	vcc_ErrWhere(tl, tl->t);
 	return (NULL);
 }
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 9a3344d..0cc0725 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -75,9 +75,9 @@ vcc_ParseImport(struct vcc *tl)
 
 	ExpectErr(tl, ID);
 	mod = tl->t;
+	osym = VCC_SymbolTok(tl, NULL, tl->t, SYM_NONE, 0);
 	vcc_NextToken(tl);
 
-	osym = VCC_SymbolTok(tl, NULL, mod, SYM_NONE, 0);
 	if (osym != NULL && osym->kind != SYM_VMOD) {
 		VSB_printf(tl->sb, "Module %.*s conflicts with other symbol.\n",
 		    PF(mod));
@@ -278,7 +278,7 @@ vcc_ParseNew(struct vcc *tl)
 	ExpectErr(tl, ID);
 	vcc_ExpectVid(tl, "VCL object");
 	ERRCHK(tl);
-	sy1 = VCC_HandleSymbol(tl, tl->t, INSTANCE, "vo");
+	sy1 = VCC_HandleSymbol(tl, INSTANCE, "vo");
 	ERRCHK(tl);
 
 	/* We allow implicit use of VMOD objects:  Pretend it's ref'ed */

From phk at FreeBSD.org  Wed Jan 24 09:27:13 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jan 2018 09:27:13 +0000 (UTC)
Subject: [master] 25b2e84 More function argument cleanup
Message-ID: <20180124092713.314C9B3DF8@lists.varnish-cache.org>


commit 25b2e84c0473449f2e2c79d08cc01ccb5ecb9e88
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 24 09:25:26 2018 +0000

    More function argument cleanup

diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index b030c11..98df6d1 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -46,8 +46,8 @@ parse_call(struct vcc *tl)
 
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
-	vcc_AddCall(tl, tl->t);
-	sym = vcc_AddRef(tl, tl->t, SYM_SUB);
+	vcc_AddCall(tl);
+	sym = vcc_AddRef(tl, SYM_SUB);
 	VCC_GlobalSymbol(sym, SUB, "VGC_function");
 	Fb(tl, 1, "%s(ctx);\n", sym->rname);
 	vcc_NextToken(tl);
@@ -245,7 +245,7 @@ parse_return_vcl(struct vcc *tl)
 	ExpectErr(tl, '(');
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
-	sym = VCC_SymbolTok(tl, NULL, tl->t, SYM_VCL, 0);
+	sym = VCC_SymbolTok(tl, NULL, SYM_VCL, 0);
 	ERRCHK(tl);
 	if (sym == NULL) {
 		VSB_printf(tl->sb, "Not a VCL label:\n");
@@ -396,7 +396,7 @@ vcc_ParseAction(struct vcc *tl)
 			return (1);
 		}
 	}
-	sym = VCC_SymbolTok(tl, NULL, tl->t, SYM_NONE, 0);
+	sym = VCC_SymbolTok(tl, NULL, SYM_NONE, 0);
 	if (sym != NULL && sym->kind == SYM_FUNC) {
 		vcc_Expr_Call(tl, sym);
 		return (1);
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index ed4aa50..a5a693f 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -265,10 +265,8 @@ vcc_ParseProbe(struct vcc *tl)
 	AN(sym);
 
 	vcc_ParseProbeSpec(tl, sym, &p);
-	if (vcc_IdIs(t_probe, "default")) {
-		(void)vcc_AddRef(tl, t_probe, SYM_PROBE);
-		tl->default_probe = p;
-	}
+	if (vcc_IdIs(t_probe, "default"))
+		tl->default_probe = sym;
 }
 
 /*--------------------------------------------------------------------
@@ -392,7 +390,7 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 			Fb(tl, 0, "\t.probe = %s,\n", p);
 			ERRCHK(tl);
 		} else if (vcc_IdIs(t_field, "probe") && tl->t->tok == ID) {
-			pb = VCC_SymbolTok(tl, NULL, tl->t, SYM_PROBE, 0);
+			pb = VCC_SymbolTok(tl, NULL, SYM_PROBE, 0);
 			if (pb == NULL) {
 				VSB_printf(tl->sb, "Probe %.*s not found\n",
 				    PF(tl->t));
@@ -400,7 +398,7 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 				return;
 			}
 			Fb(tl, 0, "\t.probe = %s,\n", pb->rname);
-			(void)vcc_AddRef(tl, tl->t, SYM_PROBE);
+			(void)vcc_AddRef(tl, SYM_PROBE);
 			vcc_NextToken(tl);
 			SkipToken(tl, ';');
 		} else if (vcc_IdIs(t_field, "probe")) {
@@ -490,10 +488,8 @@ vcc_ParseBackend(struct vcc *tl)
 		return;
 	}
 
-	if (tl->default_director == NULL || vcc_IdIs(t_be, "default")) {
-		tl->default_director = sym->rname;
-		tl->t_default_director = t_be;
-	}
+	if (tl->default_director == NULL || vcc_IdIs(t_be, "default"))
+		tl->default_director = sym;
 }
 
 void
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 0e84062..298b7a8 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -415,9 +415,11 @@ EmitStruct(const struct vcc *tl)
 	Fc(tl, 0, "\nconst struct VCL_conf VCL_conf = {\n");
 	Fc(tl, 0, "\t.magic = VCL_CONF_MAGIC,\n");
 	Fc(tl, 0, "\t.event_vcl = VGC_Event,\n");
-	Fc(tl, 0, "\t.default_director = &%s,\n", tl->default_director);
+	Fc(tl, 0, "\t.default_director = &%s,\n",
+	   tl->default_director->rname);
 	if (tl->default_probe != NULL)
-		Fc(tl, 0, "\t.default_probe = %s,\n", tl->default_probe);
+		Fc(tl, 0, "\t.default_probe = %s,\n",
+		    tl->default_probe->rname);
 	Fc(tl, 0, "\t.ref = VGC_ref,\n");
 	Fc(tl, 0, "\t.nref = VGC_NREFS,\n");
 	Fc(tl, 0, "\t.nsrc = VGC_NSRCS,\n");
@@ -627,8 +629,10 @@ vcc_CompileSource(struct vcc *tl, struct source *sp)
 		return (NULL);
 	}
 
-	/* Configure the default director */
-	(void)vcc_AddRef(tl, tl->t_default_director, SYM_BACKEND);
+	/* Refcount the default director & probe*/
+	tl->default_director->nref++;
+	if (tl->default_probe != NULL)
+		tl->default_probe->nref++;
 
 	/* Check for orphans */
 	if (vcc_CheckReferences(tl))
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 4e98e94..74a42a5 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -218,9 +218,8 @@ struct vcc {
 	int			nprobe;
 
 	int			ndirector;
-	const char		*default_director;
-	struct token		*t_default_director;
-	const char		*default_probe;
+	struct symbol		*default_director;
+	struct symbol		*default_probe;
 
 	unsigned		unique;
 
@@ -319,8 +318,8 @@ void vcc_stevedore(struct vcc *vcc, const char *stv_name);
 void VCC_PrintCName(struct vsb *vsb, const char *b, const char *e);
 struct symbol *VCC_Symbol(struct vcc *, struct symbol *,
     const char *, const char *, enum symkind, int);
-#define VCC_SymbolTok(vcc, sym, tok, kind, create) \
-    VCC_Symbol(vcc, sym, (tok)->b, (tok)->e, kind, create)
+#define VCC_SymbolTok(vcc, sym, kind, create) \
+    VCC_Symbol(vcc, sym, (vcc)->t->b, (vcc)->t->e, kind, create)
 const char * VCC_SymKind(struct vcc *tl, const struct symbol *s);
 typedef void symwalk_f(struct vcc *tl, const struct symbol *s);
 void VCC_WalkSymbols(struct vcc *tl, symwalk_f *func, enum symkind kind);
@@ -354,14 +353,12 @@ void vcc_ParseImport(struct vcc *tl);
 void vcc_ParseNew(struct vcc *tl);
 
 /* vcc_xref.c */
-struct symbol *vcc_AddDef(struct vcc *tl, const struct token *t,
-    enum symkind type);
-struct symbol *vcc_AddRef(struct vcc *tl, const struct token *t,
-    enum symkind type);
+struct symbol *vcc_AddDef(struct vcc *, enum symkind);
+struct symbol *vcc_AddRef(struct vcc *, enum symkind);
 int vcc_CheckReferences(struct vcc *tl);
 void VCC_XrefTable(struct vcc *);
 
-void vcc_AddCall(struct vcc *tl, struct token *t);
+void vcc_AddCall(struct vcc *);
 void vcc_ProcAction(struct proc *p, unsigned action, struct token *t);
 int vcc_CheckAction(struct vcc *tl);
 void vcc_AddUses(struct vcc *tl, const struct token *t, unsigned mask,
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index affbe68..dacae1e 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -333,13 +333,13 @@ vcc_Eval_Handle(struct vcc *tl, struct expr **e, const struct symbol *sym,
 	AN(sym->rname);
 
 	if (sym->fmt != STRING && fmt == STRINGS) {
-		(void)vcc_AddRef(tl, tl->t, sym->kind);
+		(void)vcc_AddRef(tl, sym->kind);
 		*e = vcc_mk_expr(STRINGS, "\"%s\"", sym->name);
 		(*e)->nstr = 1;
 		(*e)->constant |= EXPR_CONST | EXPR_STR_CONST;
 	} else {
 		vcc_ExpectVid(tl, "handle");
-		(void)vcc_AddRef(tl, tl->t, sym->kind);
+		(void)vcc_AddRef(tl, sym->kind);
 		*e = vcc_mk_expr(sym->fmt, "%s", sym->rname);
 		(*e)->constant = EXPR_VAR;
 		(*e)->nstr = 1;
@@ -657,9 +657,9 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 		sym = NULL;
 		kind = VCC_HandleKind(fmt);
 		if (kind != SYM_NONE)
-			sym = VCC_SymbolTok(tl, NULL, tl->t, kind, 0);
+			sym = VCC_SymbolTok(tl, NULL, kind, 0);
 		if (sym == NULL)
-			sym = VCC_SymbolTok(tl, NULL, tl->t, SYM_NONE, 0);
+			sym = VCC_SymbolTok(tl, NULL, SYM_NONE, 0);
 		if (sym == NULL || sym->eval == NULL) {
 			VSB_printf(tl->sb, "Symbol not found: ");
 			vcc_ErrToken(tl, tl->t);
@@ -966,7 +966,7 @@ cmp_acl(struct vcc *tl, struct expr **e, const struct cmps *cp)
 
 	vcc_NextToken(tl);
 	vcc_ExpectVid(tl, "ACL");
-	sym = vcc_AddRef(tl, tl->t, SYM_ACL);
+	sym = vcc_AddRef(tl, SYM_ACL);
 	vcc_NextToken(tl);
 	VCC_GlobalSymbol(sym, ACL, ACL_SYMBOL_PREFIX);
 	bprintf(buf, "%sVRT_acl_match(ctx, %s, \v1)", cp->emit, sym->rname);
diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index 9652077..a179d4a 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -215,7 +215,7 @@ vcc_ParseFunction(struct vcc *tl)
 	vcc_ExpectVid(tl, "function");
 	ERRCHK(tl);
 
-	sym = vcc_AddDef(tl, tl->t, SYM_SUB);
+	sym = vcc_AddDef(tl, SYM_SUB);
 	AN(sym);
 	p = sym->proc;
 	if (p == NULL) {
@@ -247,8 +247,8 @@ vcc_ParseFunction(struct vcc *tl)
 		/* Add to VCL sub */
 		AN(p->method);
 		if (p->name == NULL) {
-			(void)vcc_AddDef(tl, tl->t, SYM_SUB);
-			(void)vcc_AddRef(tl, tl->t, SYM_SUB);
+			(void)vcc_AddDef(tl, SYM_SUB);
+			(void)vcc_AddRef(tl, SYM_SUB);
 			p->name = tl->t;
 		}
 	}
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index ef526f1..a485df6 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -237,7 +237,7 @@ VCC_HandleSymbol(struct vcc *tl, vcc_type_t fmt, const char *pfx)
 	kind = VCC_HandleKind(fmt);
 	assert(kind != SYM_NONE);
 
-	sym = VCC_SymbolTok(tl, NULL, tl->t, SYM_NONE, 0);
+	sym = VCC_SymbolTok(tl, NULL, SYM_NONE, 0);
 	if (sym != NULL && sym->def_b != NULL && kind == sym->kind) {
 		p = VCC_SymKind(tl, sym);
 		VSB_printf(tl->sb, "%c%s '%.*s' redefined.\n",
@@ -262,7 +262,7 @@ VCC_HandleSymbol(struct vcc *tl, vcc_type_t fmt, const char *pfx)
 		return (sym);
 	}
 	if (sym == NULL)
-		sym = VCC_SymbolTok(tl, NULL, tl->t, kind, 1);
+		sym = VCC_SymbolTok(tl, NULL, kind, 1);
 	AN(sym);
 	AZ(sym->ndef);
 	VCC_GlobalSymbol(sym, fmt, pfx);
diff --git a/lib/libvcc/vcc_var.c b/lib/libvcc/vcc_var.c
index 02c4e5e..51b9897 100644
--- a/lib/libvcc/vcc_var.c
+++ b/lib/libvcc/vcc_var.c
@@ -100,7 +100,7 @@ vcc_FindVar(struct vcc *tl, int wr_access, const char *use)
 {
 	const struct symbol *sym;
 
-	sym = VCC_SymbolTok(tl, NULL, tl->t, SYM_VAR, 0);
+	sym = VCC_SymbolTok(tl, NULL, SYM_VAR, 0);
 	if (tl->err)
 		return (NULL);
 	if (sym != NULL) {
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 0cc0725..21a7216 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -75,7 +75,7 @@ vcc_ParseImport(struct vcc *tl)
 
 	ExpectErr(tl, ID);
 	mod = tl->t;
-	osym = VCC_SymbolTok(tl, NULL, tl->t, SYM_NONE, 0);
+	osym = VCC_SymbolTok(tl, NULL, SYM_NONE, 0);
 	vcc_NextToken(tl);
 
 	if (osym != NULL && osym->kind != SYM_VMOD) {
@@ -290,7 +290,7 @@ vcc_ParseNew(struct vcc *tl)
 	vcc_NextToken(tl);
 
 	ExpectErr(tl, ID);
-	sy2 = VCC_SymbolTok(tl, NULL, tl->t, SYM_OBJECT, 0);
+	sy2 = VCC_SymbolTok(tl, NULL, SYM_OBJECT, 0);
 	if (sy2 == NULL || sy2->extra == NULL) {
 		if (sy2 == NULL)
 			p = "Symbol";
diff --git a/lib/libvcc/vcc_xref.c b/lib/libvcc/vcc_xref.c
index fff066c..1462145 100644
--- a/lib/libvcc/vcc_xref.c
+++ b/lib/libvcc/vcc_xref.c
@@ -61,26 +61,26 @@ struct procuse {
  */
 
 struct symbol *
-vcc_AddRef(struct vcc *tl, const struct token *t, enum symkind kind)
+vcc_AddRef(struct vcc *tl, enum symkind kind)
 {
 	struct symbol *sym;
 
-	sym = VCC_SymbolTok(tl, NULL, t, kind, 1);
+	sym = VCC_SymbolTok(tl, NULL, kind, 1);
 	if (sym->ref_b == NULL)
-		sym->ref_b = t;
+		sym->ref_b = tl->t;
 	AN(sym);
 	sym->nref++;
 	return (sym);
 }
 
 struct symbol *
-vcc_AddDef(struct vcc *tl, const struct token *t, enum symkind kind)
+vcc_AddDef(struct vcc *tl, enum symkind kind)
 {
 	struct symbol *sym;
 
-	sym = VCC_SymbolTok(tl, NULL, t, kind, 1);
+	sym = VCC_SymbolTok(tl, NULL, kind, 1);
 	if (sym->def_b == NULL)
-		sym->def_b = t;
+		sym->def_b = tl->t;
 	AN(sym);
 	sym->ndef++;
 	return (sym);
@@ -138,15 +138,15 @@ vcc_AddUses(struct vcc *tl, const struct token *t, unsigned mask,
 }
 
 void
-vcc_AddCall(struct vcc *tl, struct token *t)
+vcc_AddCall(struct vcc *tl)
 {
 	struct proccall *pc;
 
 	pc = TlAlloc(tl, sizeof *pc);
 	assert(pc != NULL);
-	pc->sym = VCC_SymbolTok(tl, NULL, t, SYM_SUB, 1);
+	pc->sym = VCC_SymbolTok(tl, NULL, SYM_SUB, 1);
 	AN(pc->sym);
-	pc->t = t;
+	pc->t = tl->t;
 	VTAILQ_INSERT_TAIL(&tl->curproc->calls, pc, list);
 }
 

From nils.goroll at uplex.de  Wed Jan 24 10:03:08 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 24 Jan 2018 10:03:08 +0000 (UTC)
Subject: [master] 3dd8921 log local proxy connections
Message-ID: <20180124100308.D6234B4B13@lists.varnish-cache.org>


commit 3dd89218ba8b38ad6ce2f445adf844fb246f6bc5
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jan 24 10:53:33 2018 +0100

    log local proxy connections
    
    Previously, we would get PROXY vxids without a Proxy line like:
    
        262253 Begin          c sess 0 PROXY
        262253 SessOpen       c w.x.y.z 1234 a0 a.b.c.d 85 1516787007.682753 32
        262253 Link           c req 262254 rxreq

diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c
index fbef883..d165281 100644
--- a/bin/varnishd/proxy/cache_proxy_proto.c
+++ b/bin/varnishd/proxy/cache_proxy_proto.c
@@ -193,7 +193,7 @@ vpx_proto2(const struct worker *wrk, struct req *req)
 	/* Command @12 bottom half */
 	switch (p[12] & 0x0f) {
 	case 0x0:
-		/* Local connection from proxy, ignore addresses */
+		VSL(SLT_Proxy, req->sp->vxid, "2 local local local local");
 		return (0);
 	case 0x1:
 		/* Proxied connection */
diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h
index 6463636..f7fa3d5 100644
--- a/include/tbl/vsl_tags.h
+++ b/include/tbl/vsl_tags.h
@@ -140,6 +140,8 @@ SLTM(Proxy, 0, "PROXY protocol information",
 	"\t|  |  +------- client port\n"
 	"\t|  +---------- client ip\n"
 	"\t+------------- PROXY protocol version\n"
+	"\t\n"
+	"\tAll fields are \"local\" for PROXY local connections (command 0x0)\n"
 	"\n"
 )
 

From phk at FreeBSD.org  Wed Jan 24 10:22:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jan 2018 10:22:06 +0000 (UTC)
Subject: [master] 1a5c5f1 Minor polish
Message-ID: <20180124102206.85986B5204@lists.varnish-cache.org>


commit 1a5c5f18c82321cfa19ac5d9b415b35391381792
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 24 09:38:34 2018 +0000

    Minor polish

diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c
index ca27856..ba45ee2 100644
--- a/lib/libvcc/vcc_acl.c
+++ b/lib/libvcc/vcc_acl.c
@@ -470,7 +470,6 @@ vcc_ParseAcl(struct vcc *tl)
 	vcc_ExpectVid(tl, "ACL");
 	ERRCHK(tl);
 	sym = VCC_HandleSymbol(tl, ACL, ACL_SYMBOL_PREFIX);
-	vcc_NextToken(tl);
 
 	ERRCHK(tl);
 	AN(sym);
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index a5a693f..1655f92 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -259,7 +259,6 @@ vcc_ParseProbe(struct vcc *tl)
 	ERRCHK(tl);
 	t_probe = tl->t;
 	sym = VCC_HandleSymbol(tl, PROBE, "vgc_probe");
-	vcc_NextToken(tl);
 
 	ERRCHK(tl);
 	AN(sym);
@@ -472,7 +471,6 @@ vcc_ParseBackend(struct vcc *tl)
 
 	t_be = tl->t;
 	sym = VCC_HandleSymbol(tl, BACKEND, "vgc_backend");
-	vcc_NextToken(tl);
 
 	ERRCHK(tl);
 
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 74a42a5..6824ec3 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -318,8 +318,7 @@ void vcc_stevedore(struct vcc *vcc, const char *stv_name);
 void VCC_PrintCName(struct vsb *vsb, const char *b, const char *e);
 struct symbol *VCC_Symbol(struct vcc *, struct symbol *,
     const char *, const char *, enum symkind, int);
-#define VCC_SymbolTok(vcc, sym, kind, create) \
-    VCC_Symbol(vcc, sym, (vcc)->t->b, (vcc)->t->e, kind, create)
+struct symbol *VCC_SymbolTok(struct vcc *, struct symbol *, enum symkind, int);
 const char * VCC_SymKind(struct vcc *tl, const struct symbol *s);
 typedef void symwalk_f(struct vcc *tl, const struct symbol *s);
 void VCC_WalkSymbols(struct vcc *tl, symwalk_f *func, enum symkind kind);
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index a485df6..c743767 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -105,6 +105,14 @@ vcc_new_symbol(struct vcc *tl, const char *b, const char *e)
 }
 
 struct symbol *
+VCC_SymbolTok(struct vcc *tl, struct symbol *parent,
+    enum symkind kind, int create)
+{
+
+	return (VCC_Symbol(tl, parent, tl->t->b, tl->t->e, kind, create));
+}
+
+struct symbol *
 VCC_Symbol(struct vcc *tl, struct symbol *parent,
     const char *b, const char *e, enum symkind kind, int create)
 {
@@ -269,5 +277,6 @@ VCC_HandleSymbol(struct vcc *tl, vcc_type_t fmt, const char *pfx)
 	sym->ndef = 1;
 	if (sym->def_b == NULL)
 		sym->def_b = tl->t;
+	vcc_NextToken(tl);
 	return (sym);
 }
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 21a7216..79adcb6 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -284,8 +284,6 @@ vcc_ParseNew(struct vcc *tl)
 	/* We allow implicit use of VMOD objects:  Pretend it's ref'ed */
 	sy1->nref++;
 
-	vcc_NextToken(tl);
-
 	ExpectErr(tl, '=');
 	vcc_NextToken(tl);
 

From phk at FreeBSD.org  Wed Jan 24 10:22:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jan 2018 10:22:06 +0000 (UTC)
Subject: [master] d306cd4 GC unused code
Message-ID: <20180124102206.AAD78B5207@lists.varnish-cache.org>


commit d306cd4647a436b4d45671a007d17ccc64ed1369
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 24 10:20:35 2018 +0000

    GC unused code

diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 98df6d1..1d582e2 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -90,7 +90,7 @@ parse_set(struct vcc *tl)
 
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
-	sym = vcc_FindVar(tl, 1, "cannot be set");
+	sym = vcc_FindVar(tl, "cannot be set");
 	ERRCHK(tl);
 	assert(sym != NULL);
 	if (vcc_IdIs(tl->t, "bereq.body")) {
@@ -138,7 +138,7 @@ parse_unset(struct vcc *tl)
 	/* XXX: Wrong, should use VCC_Expr(HEADER) */
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
-	sym = vcc_FindVar(tl, 1, "cannot be unset");
+	sym = vcc_FindVar(tl, "cannot be unset");
 	ERRCHK(tl);
 	assert(sym != NULL);
 	if (sym->fmt != HEADER && !vcc_IdIs(tl->t, "bereq.body")) {
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 6824ec3..00b55dd 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -345,7 +345,7 @@ vcc_type_t VCC_Type(const char *p);
 
 /* vcc_var.c */
 sym_wildcard_t vcc_Var_Wildcard;
-const struct symbol *vcc_FindVar(struct vcc *, int, const char *);
+const struct symbol *vcc_FindVar(struct vcc *, const char *);
 
 /* vcc_vmod.c */
 void vcc_ParseImport(struct vcc *tl);
diff --git a/lib/libvcc/vcc_var.c b/lib/libvcc/vcc_var.c
index 51b9897..706c5d0 100644
--- a/lib/libvcc/vcc_var.c
+++ b/lib/libvcc/vcc_var.c
@@ -96,7 +96,7 @@ vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent,
 /*--------------------------------------------------------------------*/
 
 const struct symbol *
-vcc_FindVar(struct vcc *tl, int wr_access, const char *use)
+vcc_FindVar(struct vcc *tl, const char *use)
 {
 	const struct symbol *sym;
 
@@ -104,25 +104,15 @@ vcc_FindVar(struct vcc *tl, int wr_access, const char *use)
 	if (tl->err)
 		return (NULL);
 	if (sym != NULL) {
-		if (wr_access && sym->w_methods == 0) {
+		if (sym->w_methods == 0) {
 			VSB_printf(tl->sb, "Variable ");
 			vcc_ErrToken(tl, tl->t);
 			VSB_printf(tl->sb, " is read only.");
 			VSB_cat(tl->sb, "\nAt: ");
 			vcc_ErrWhere(tl, tl->t);
 			return (NULL);
-		} else if (wr_access) {
-			vcc_AddUses(tl, tl->t, sym->w_methods, use);
-		} else if (sym->r_methods == 0) {
-			VSB_printf(tl->sb, "Variable ");
-			vcc_ErrToken(tl, tl->t);
-			VSB_printf(tl->sb, " is write only.");
-			VSB_cat(tl->sb, "\nAt: ");
-			vcc_ErrWhere(tl, tl->t);
-			return (NULL);
-		} else {
-			vcc_AddUses(tl, tl->t, sym->r_methods, use);
 		}
+		vcc_AddUses(tl, tl->t, sym->w_methods, use);
 		return (sym);
 	}
 	VSB_printf(tl->sb, "Unknown variable ");

From phk at FreeBSD.org  Wed Jan 24 11:39:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jan 2018 11:39:07 +0000 (UTC)
Subject: [master] e212c5c Get rid of the interstitial "var" structure.
Message-ID: <20180124113907.9E3F0B6C3A@lists.varnish-cache.org>


commit e212c5c009666a2033525ddae24622aa61427fa3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 24 11:37:24 2018 +0000

    Get rid of the interstitial "var" structure.

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index dfa71b9..5e5c04f 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -1229,7 +1229,6 @@ def restrict(fo, spec):
 		p = " | "
 	if len(d) == 0:
 		fo.write("0")
-	fo.write(",\n")
 
 #######################################################################
 
@@ -1244,46 +1243,68 @@ fo.write("""
 
 #include "vcc_compile.h"
 
-const struct var vcc_vars[] = {
+void
+vcc_Var_Init(struct vcc *tl)
+{
+	struct symbol *sym;
 """)
 
 
 def one_var(nm, spec):
 	fh.write("\n")
+	fo.write("\n")
 	cnam = spec.nam.replace(".", "_")
 	ctyp = vcltypes[spec.typ]
 
-	fo.write("\t{ \"%s\", %s,\n" % (nm, spec.typ))
+	# fo.write("\t{ \"%s\", %s,\n" % (nm, spec.typ))
+	fo.write("\tsym = VCC_Symbol(tl, NULL, \"%s\", NULL," % nm)
+	if (spec.typ == "HEADER"):
+		fo.write(" SYM_NONE, 1);\n")
+		fo.write("\tAN(sym);\n");
+		fo.write("\tsym->wildcard = vcc_Var_Wildcard;\n")
+	else:
+		fo.write(" SYM_VAR, 1);\n")
+	fo.write("\tAN(sym);\n")
+	fo.write("\tsym->fmt = %s;\n" % spec.typ)
+	fo.write("\tsym->eval = vcc_Eval_Var;\n")
 
 	if len(spec.rd) == 0:
-		fo.write('\t    NULL,\t/* No reads allowed */\n')
+		fo.write('\t/* No reads allowed */\n')
 	elif spec.typ == "HEADER":
-		fo.write('\t    "HDR_')
+		fo.write('\tsym->rname = "HDR_')
 		fo.write(nm.split(".")[0].upper())
-		fo.write('",\n')
+		fo.write('";\n')
+		fo.write("\tsym->r_methods =\n")
+		restrict(fo, spec.rd)
+		fo.write(";\n")
 	else:
-		fo.write('\t    "VRT_r_%s(ctx)",\n' % cnam)
+		fo.write('\tsym->rname = "VRT_r_%s(ctx)";\n' % cnam)
 		if nm == spec.nam:
 			fh.write("VCL_" + spec.typ + " VRT_r_%s(VRT_CTX);\n" % cnam)
-	restrict(fo, spec.rd)
+		fo.write("\tsym->r_methods =\n")
+		restrict(fo, spec.rd)
+		fo.write(";\n")
 
 	if len(spec.wr) == 0:
-		fo.write('\t    NULL,\t/* No writes allowed */\n')
+		fo.write('\t/* No writes allowed */\n')
 	elif spec.typ == "HEADER":
-		fo.write('\t    "HDR_')
+		fo.write('\tsym->lname = "HDR_')
 		fo.write(nm.split(".")[0].upper())
-		fo.write('",\n')
+		fo.write('";\n')
+		fo.write("\tsym->w_methods =\n")
+		restrict(fo, spec.wr)
+		fo.write(";\n")
 	else:
-		fo.write('\t    "VRT_l_%s(ctx, ",\n' % cnam)
+		fo.write('\tsym->lname = "VRT_l_%s(ctx, ";\n' % cnam)
 		if nm == spec.nam:
 			fh.write("void VRT_l_%s(VRT_CTX, " % cnam)
 			if spec.typ != "STRING" and spec.typ != "BODY":
 				fh.write("VCL_" + spec.typ + ");\n")
 			else:
 				fh.write(ctyp + ", ...);\n")
-	restrict(fo, spec.wr)
-
-	fo.write("\t},\n")
+		fo.write("\tsym->w_methods =\n")
+		restrict(fo, spec.wr)
+		fo.write(";\n")
 
 aliases.sort()
 for i in sp_variables:
@@ -1292,7 +1313,8 @@ for i in sp_variables:
 		if j[1] == i[0]:
 			one_var(j[0], sp_variables[i])
 
-fo.write("\t{ NULL }\n};\n\n")
+# fo.write("\t{ NULL }\n};\n\n")
+fo.write("}\n")
 
 for i in stv_variables:
 	fh.write(vcltypes[i[1]] + " VRT_Stv_" + i[0] + "(const char *);\n")
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 298b7a8..95e5b76 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -556,9 +556,7 @@ vcc_resolve_includes(struct vcc *tl)
 static struct vsb *
 vcc_CompileSource(struct vcc *tl, struct source *sp)
 {
-	struct symbol *sym;
 	struct proc *p;
-	const struct var *v;
 	struct vsb *vsb;
 	struct inifin *ifp;
 
@@ -569,22 +567,7 @@ vcc_CompileSource(struct vcc *tl, struct source *sp)
 
 	vcc_Backend_Init(tl);
 
-	for (v = vcc_vars; v->name != NULL; v++) {
-		if (v->fmt == HEADER) {
-			sym = VCC_Symbol(tl, NULL, v->name, NULL,
-			    SYM_NONE, 1);
-			sym->wildcard = vcc_Var_Wildcard;
-			sym->wildcard_priv = v;
-		} else {
-			sym = VCC_Symbol(tl, NULL, v->name, NULL, SYM_VAR, 1);
-		}
-		sym->fmt = v->fmt;
-		sym->eval = vcc_Eval_Var;
-		sym->r_methods = v->r_methods;
-		sym->w_methods = v->w_methods;
-		sym->lname = v->lname;
-		REPLACE(sym->rname, v->rname);
-	}
+	vcc_Var_Init(tl);
 
 	Fh(tl, 0, "\nextern const struct VCL_conf VCL_conf;\n");
 
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 00b55dd..e0b0da9 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -121,7 +121,6 @@ struct symbol {
 	char				*name;
 	unsigned			nlen;
 	sym_wildcard_t			*wildcard;
-	const void			*wildcard_priv;
 	enum symkind			kind;
 
 	const struct token		*def_b, *def_e, *ref_b;
@@ -138,7 +137,7 @@ struct symbol {
 	const char			*extra;
 
 	/* SYM_VAR */
-	char				*rname;
+	const char			*rname;
 	unsigned			r_methods;
 	const char			*lname;
 	unsigned			w_methods;
@@ -225,15 +224,6 @@ struct vcc {
 
 };
 
-struct var {
-	const char		*name;
-	vcc_type_t		fmt;
-	const char		*rname;
-	unsigned		r_methods;
-	const char		*lname;
-	unsigned		w_methods;
-};
-
 struct method {
 	const char		*name;
 	unsigned		ret_bitmap;
@@ -294,7 +284,7 @@ void VCC_GlobalSymbol(struct symbol *, vcc_type_t fmt, const char *pfx);
 struct symbol *VCC_HandleSymbol(struct vcc *, vcc_type_t , const char *);
 
 /* vcc_obj.c */
-extern const struct var vcc_vars[];
+void vcc_Var_Init(struct vcc *);
 
 /* vcc_parse.c */
 void vcc_Parse(struct vcc *tl);
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index c743767..cb94ffb 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -220,7 +220,7 @@ VCC_GlobalSymbol(struct symbol *sym, vcc_type_t fmt, const char *pfx)
 	VSB_printf(vsb, "%s_", pfx);
 	VCC_PrintCName(vsb, sym->name, NULL);
 	AZ(VSB_finish(vsb));
-	REPLACE(sym->rname, VSB_data(vsb));
+	sym->rname = strdup(VSB_data(vsb));
 	AN(sym->rname);
 	VSB_destroy(&vsb);
 
diff --git a/lib/libvcc/vcc_var.c b/lib/libvcc/vcc_var.c
index 706c5d0..133455f 100644
--- a/lib/libvcc/vcc_var.c
+++ b/lib/libvcc/vcc_var.c
@@ -41,13 +41,10 @@ vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent,
     const char *b, const char *e)
 {
 	struct symbol *sym;
-	struct var *v;
-	const struct var *vh;
 	struct vsb *vsb;
 	unsigned len;
 
-	vh = parent->wildcard_priv;
-	assert(vh->fmt == HEADER);
+	assert(parent->fmt == HEADER);
 
 	if (b + 127 <= e) {
 		VSB_printf(tl->sb, "HTTP header (%.20s..) is too long.\n", b);
@@ -56,16 +53,17 @@ vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent,
 		return;
 	}
 
-	v = TlAlloc(tl, sizeof *v);
-	AN(v);
-	v->r_methods = vh->r_methods;
-	v->w_methods = vh->w_methods;
-	v->fmt = vh->fmt;
+	sym = VCC_Symbol(tl, parent, b, e, SYM_VAR, 1);
+	AN(sym);
+	sym->fmt = parent->fmt;
+	sym->eval = vcc_Eval_Var;
+	sym->r_methods = parent->r_methods;
+	sym->w_methods = parent->w_methods;
 
 	/* Create a C-name version of the header name */
 	vsb = VSB_new_auto();
 	AN(vsb);
-	VSB_printf(vsb, "&VGC_%s_", vh->rname);
+	VSB_printf(vsb, "&VGC_%s_", parent->rname);
 	VCC_PrintCName(vsb, b, e);
 	AZ(VSB_finish(vsb));
 
@@ -73,24 +71,15 @@ vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent,
 	len = (unsigned)(e - b);
 	Fh(tl, 0, "static const struct gethdr_s %s =\n", VSB_data(vsb) + 1);
 	Fh(tl, 0, "    { %s, \"\\%03o%.*s:\"};\n",
-	    vh->rname, len + 1, len, b);
+	    parent->rname, len + 1, len, b);
 
 	/* Create the symbol r/l values */
-	v->rname = TlDup(tl, VSB_data(vsb));
+	sym->rname = TlDup(tl, VSB_data(vsb));
 	VSB_clear(vsb);
-	VSB_printf(vsb, "VRT_SetHdr(ctx, %s,", v->rname);
+	VSB_printf(vsb, "VRT_SetHdr(ctx, %s,", sym->rname);
 	AZ(VSB_finish(vsb));
-	v->lname = TlDup(tl, VSB_data(vsb));
+	sym->lname = TlDup(tl, VSB_data(vsb));
 	VSB_destroy(&vsb);
-
-	sym = VCC_Symbol(tl, parent, b, e, SYM_VAR, 1);
-	AN(sym);
-	sym->fmt = v->fmt;
-	sym->eval = vcc_Eval_Var;
-	sym->r_methods = v->r_methods;
-	sym->w_methods = v->w_methods;
-	sym->lname = v->lname;
-	REPLACE(sym->rname, v->rname);
 }
 
 /*--------------------------------------------------------------------*/

From phk at FreeBSD.org  Wed Jan 24 11:39:07 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jan 2018 11:39:07 +0000 (UTC)
Subject: [master] f9a79a7 Make the python code a bit more readable.
Message-ID: <20180124113907.8EC37B6C38@lists.varnish-cache.org>


commit f9a79a7470b28758e7b75ca47eebc989494e8a7d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 24 10:59:37 2018 +0000

    Make the python code a bit more readable.

diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index c9b9e74..dfa71b9 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -33,6 +33,7 @@
 from __future__ import print_function
 
 import subprocess
+import collections
 import os
 
 #######################################################################
@@ -165,659 +166,668 @@ returns = (
 # 'backend' means all methods tagged "B"
 # 'both' means all methods tagged "B" or "C"
 
-sp_variables = [
-	('remote.ip',
-		'IP',
-		('both',),
-		(), """
-		The IP address of the other end of the TCP connection.
-		This can either be the clients IP, or the outgoing IP
-		of a proxy server.
-		"""
-	),
-	('client.ip',
-		'IP',
-		('both',),
-		(), """
-		The client's IP address.
-		"""
-	),
-	('client.identity',
-		'STRING',
-		('client',),
-		('client',), """
-		Identification of the client, used to load balance
-		in the client director. Defaults to the client's IP
-		address.
-		"""
-	),
-	('local.ip',
-		'IP',
-		('both',),
-		(), """
-		The IP address of the local end of the TCP connection.
-		"""
-	),
-	('server.ip',
-		'IP',
-		('both',),
-		(), """
-		The IP address of the socket on which the client
-		connection was received.
-		"""
-	),
-	('server.hostname',
-		'STRING',
-		('all',),
-		(), """
-		The host name of the server.
-		"""
-	),
-	('server.identity',
-		'STRING',
-		('all',),
-		(), """
-		The identity of the server, as set by the -i
-		parameter.  If the -i parameter is not passed to varnishd,
-		server.identity will be set to the hostname of the machine.
-		"""
-	),
-	('req',
-		'HTTP',
-		('client',),
-		(), """
-		The entire request HTTP data structure
-		"""
-	),
-	('req.method',
-		'STRING',
-		('client',),
-		('client',), """
-		The request type (e.g. "GET", "HEAD").
-		"""
-	),
-	('req.hash',
-		'BLOB',
-		('hit', 'miss', 'pass', 'purge', 'deliver', ),
-		(), """
-		The hash key of this request.
-		"""
-	),
-	('req.url',
-		'STRING',
-		('client',),
-		('client',), """
-		The requested URL.
-		"""
-	),
-	('req.proto',
-		'STRING',
-		('client',),
-		('client',), """
-		The HTTP protocol version used by the client.
-		"""
-	),
-	('req.http.',
-		'HEADER',
-		('client',),
-		('client',), """
-		The corresponding HTTP header.
-		"""
-	),
-	('req.restarts',
-		'INT',
-		('client',),
-		(), """
-		A count of how many times this request has been restarted.
-		"""
-	),
-	('req.storage',
-		'STEVEDORE',
-		('client',),
-		('client',), """
-		The storage backend to use to save this request body.
-		"""
-	),
-	('req.esi_level',
-		'INT',
-		('client',),
-		(), """
-		A count of how many levels of ESI requests we're currently at.
-		"""
-	),
-	('req.ttl',
-		'DURATION',
-		('client',),
-		('client',), """
-		Upper limit on the object age for cache lookups to return hit.
-
-		Usage of req.ttl should be replaced with a check on
-		obj.ttl in vcl_hit, returning miss when needed, but
-		this currently hits bug #1799, so an additional
-		workaround is required.
-
-		Deprecated and scheduled for removal with varnish release 7.
-		"""
-	),
-	('req.xid',
-		'STRING',
-		('client',),
-		(), """
-		Unique ID of this request.
-		"""
-	),
-	('req.esi',
-		'BOOL',
-		('client',),
-		('client',), """
-		Boolean. Set to false to disable ESI processing
-		regardless of any value in beresp.do_esi. Defaults
-		to true. This variable is subject to change in
-		future versions, you should avoid using it.
-		"""
-	),
-	('req.can_gzip',
-		'BOOL',
-		('client',),
-		(), """
-		Does the client accept the gzip transfer encoding.
-		"""
-	),
-	('req.backend_hint',
-		'BACKEND',
-		('client', ),
-		('client',), """
-		Set bereq.backend to this if we attempt to fetch.
-		When set to a director, reading this variable returns
-		an actual backend if the director has resolved immediately,
-		or the director otherwise.
-		When used in string context, returns the name of the director
-		or backend, respectively.
-		"""
-	),
-	('req.hash_ignore_busy',
-		'BOOL',
-		('client',),
-		('client',), """
-		Ignore any busy object during cache lookup. You
-		would want to do this if you have two server looking
-		up content from each other to avoid potential deadlocks.
-		"""
-	),
-	('req.hash_always_miss',
-		'BOOL',
-		('client',),
-		('client',), """
-		Force a cache miss for this request. If set to true
-		Varnish will disregard any existing objects and
-		always (re)fetch from the backend.
-		"""
-	),
-	('req_top.method',
-		'STRING',
-		('client',),
-		(), """
-		The request method of the top-level request in a tree
-		of ESI requests. (e.g. "GET", "HEAD").
-		Identical to req.method in non-ESI requests.
-		"""
-	),
-	('req_top.url',
-		'STRING',
-		('client',),
-		(), """
-		The requested URL of the top-level request in a tree
-		of ESI requests.
-		Identical to req.url in non-ESI requests.
-		"""
-	),
-	('req_top.http.',
-		'HEADER',
-		('client',),
-		(), """
-		HTTP headers of the top-level request in a tree of ESI requests.
-		Identical to req.http. in non-ESI requests.
-		"""
-	),
-	('req_top.proto',
-		'STRING',
-		('client',),
-		(), """
-		HTTP protocol version of the top-level request in a tree of
-		ESI requests.
-		Identical to req.proto in non-ESI requests.
-		"""
-	),
-	('bereq',
-		'HTTP',
-		('backend',),
-		(), """
-		The entire backend request HTTP data structure
-		"""
-	),
-	('bereq.xid',
-		'STRING',
-		('backend',),
-		(), """
-		Unique ID of this request.
-		"""
-	),
-	('bereq.retries',
-		'INT',
-		('backend',),
-		(), """
-		A count of how many times this request has been retried.
-		"""
-	),
-	('bereq.backend',
-		'BACKEND',
-		('pipe', 'backend', ),
-		('pipe', 'backend', ), """
-		This is the backend or director we attempt to fetch from.
-		When set to a director, reading this variable returns
-		an actual backend if the director has resolved immediately,
-		or the director otherwise.
-		When used in string context, returns the name of the director
-		or backend, respectively.
-		"""
-	),
-	('bereq.body',
-		'BODY',
-		(),
-		('backend_fetch',), """
-		The request body.
-		"""
-	),
-	('bereq.hash',
-		'BLOB',
-		('pipe', 'backend', ),
-		(), """
-		The hash key of this request.
-		"""
-	),
-	('bereq.method',
-		'STRING',
-		('pipe', 'backend', ),
-		('pipe', 'backend', ), """
-		The request type (e.g. "GET", "HEAD").
-		"""
-	),
-	('bereq.url',
-		'STRING',
-		('pipe', 'backend', ),
-		('pipe', 'backend', ), """
-		The requested URL.
-		"""
-	),
-	('bereq.proto',
-		'STRING',
-		('pipe', 'backend', ),
-		('pipe', 'backend', ), """
-		The HTTP protocol version used to talk to the server.
-		"""
-	),
-	('bereq.http.',
-		'HEADER',
-		('pipe', 'backend', ),
-		('pipe', 'backend', ), """
-		The corresponding HTTP header.
-		"""
-	),
-	('bereq.uncacheable',
-		'BOOL',
-		('backend', ),
-		(), """
-		Indicates whether this request is uncacheable due
-		to a pass in the client side or a hit on an hit-for-pass
-		object.
-		"""
-	),
-	('bereq.connect_timeout',
-		'DURATION',
-		('pipe', 'backend', ),
-		('pipe', 'backend', ), """
-		The time in seconds to wait for a backend connection.
-		"""
-	),
-	('bereq.first_byte_timeout',
-		'DURATION',
-		('backend', ),
-		('backend', ), """
-		The time in seconds to wait for the first byte from
-		the backend.  Not available in pipe mode.
-		"""
-	),
-	('bereq.between_bytes_timeout',
-		'DURATION',
-		('backend', ),
-		('backend', ), """
-		The time in seconds to wait between each received byte from the
-		backend.  Not available in pipe mode.
-		"""
-	),
-	('bereq.is_bgfetch',
-		'BOOL',
-		('backend', ),
-		(), """
-		True for background fetches.
-		"""
-	),
-	('beresp',
-		'HTTP',
-		('backend_response', 'backend_error'),
-		(), """
-		The entire backend response HTTP data structure
-		"""
-	),
-	('beresp.body',
-		'BODY',
-		(),
-		('backend_error',), """
-		The response body.
-		"""
-	),
-	('beresp.proto',
-		'STRING',
-		('backend_response', 'backend_error'),
-		('backend_response', 'backend_error'), """
-		The HTTP protocol version used the backend replied with.
-		"""
-	),
-	('beresp.status',
-		'INT',
-		('backend_response', 'backend_error'),
-		('backend_response', 'backend_error'), """
-		The HTTP status code returned by the server.
-
-		Status codes >1000 can be set for vcl-internal
-		purposes and will be taken modulo 1000 on delivery.
-		"""
-	),
-	('beresp.reason',
-		'STRING',
-		('backend_response', 'backend_error'),
-		('backend_response', 'backend_error'), """
-		The HTTP status message returned by the server.
-		"""
-	),
-	('beresp.http.',
-		'HEADER',
-		('backend_response', 'backend_error'),
-		('backend_response', 'backend_error'), """
-		The corresponding HTTP header.
-		"""
-	),
-	('beresp.do_esi',
-		'BOOL',
-		('backend_response', 'backend_error'),
-		('backend_response', 'backend_error'), """
-		Boolean. ESI-process the object after fetching it.
-		Defaults to false. Set it to true to parse the
-		object for ESI directives. Will only be honored if
-		req.esi is true.
-		"""
-	),
-	('beresp.do_stream',
-		'BOOL',
-		('backend_response', 'backend_error'),
-		('backend_response', 'backend_error'), """
-		Deliver the object to the client while fetching the whole
-		object into varnish. For uncacheable objects, storage for
-		parts of the body which have been sent to the client may
-		get freed early, depending on the storage engine used.
-		"""
-	),
-	('beresp.do_gzip',
-		'BOOL',
-		('backend_response', 'backend_error'),
-		('backend_response', 'backend_error'), """
-		Boolean. Gzip the object before storing it. Defaults
-		to false. When http_gzip_support is on Varnish will
-		request already compressed content from the backend
-		and as such compression in Varnish is not needed.
-		"""
-	),
-	('beresp.do_gunzip',
-		'BOOL',
-		('backend_response', 'backend_error'),
-		('backend_response', 'backend_error'), """
-		Boolean. Unzip the object before storing it in the
-		cache.  Defaults to false.
-		"""
-	),
-	('beresp.was_304',
-		'BOOL',
-		('backend_response', 'backend_error'),
-		(), """
-		Boolean. If this is a successful 304 response to a
-		backend conditional request refreshing an existing
-		cache object.
-		"""
-	),
-	('beresp.uncacheable',
-		'BOOL',
-		('backend_response', 'backend_error'),
-		('backend_response', 'backend_error'), """
-		Inherited from bereq.uncacheable, see there.
-
-		Setting this variable makes the object uncacheable, which may
-		get stored as a hit-for-miss object in the cache.
-
-		Clearing the variable has no effect and will log the warning
-		"Ignoring attempt to reset beresp.uncacheable".
-		"""
-	),
-	('beresp.ttl',
-		'DURATION',
-		('backend_response', 'backend_error'),
-		('backend_response', 'backend_error'), """
-		The object's remaining time to live, in seconds.
-		"""
-	),
-	('beresp.age',
-		'DURATION',
-		('backend_response', 'backend_error'),
-		(), """
-		The age of the object.
-		"""
-	),
-	('beresp.grace',
-		'DURATION',
-		('backend_response', 'backend_error'),
-		('backend_response', 'backend_error'), """
-		Set to a period to enable grace.
-		"""
-	),
-	('beresp.keep',
-		'DURATION',
-		('backend_response', 'backend_error'),
-		('backend_response', 'backend_error'), """
-		Set to a period to enable conditional backend requests.
-
-		The keep time is cache lifetime in addition to the ttl.
-
-		Objects with ttl expired but with keep time left may be used
-		to issue conditional (If-Modified-Since / If-None-Match)
-		requests to the backend to refresh them.
-		"""
-	),
-	('beresp.backend',
-		'BACKEND',
-		('backend_response', 'backend_error'),
-		(), """
-		This is the backend we fetched from.  If bereq.backend
-		was set to a director, this will be the backend selected
-		by the director.
-		When used in string context, returns its name.
-		"""
-	),
-	('beresp.backend.name',
-		'STRING',
-		('backend_response', 'backend_error'),
-		(), """
-		Name of the backend this response was fetched from.
-		Same as beresp.backend.
-		"""
-	),
-	('beresp.backend.ip',
-		'IP',
-		('backend_response',),
-		(), """
-		IP of the backend this response was fetched from.
-		"""
-	),
-	('beresp.storage',
-		'STEVEDORE',
-		('backend_response', 'backend_error'),
-		('backend_response', 'backend_error'), """
-		The storage backend to use to save this object.
-		"""
-	),
-	('obj.proto',
-		'STRING',
-		('hit',),
-		(), """
-		The HTTP protocol version stored with the object.
-		"""
-	),
-	('obj.status',
-		'INT',
-		('hit',),
-		(), """
-		The HTTP status code stored with the object.
-		"""
-	),
-	('obj.reason',
-		'STRING',
-		('hit',),
-		(), """
-		The HTTP reason phrase stored with the object.
-		"""
-	),
-	('obj.hits',
-		'INT',
-		('hit', 'deliver'),
-		(), """
-		The count of cache-hits on this object. A value of 0 indicates a
-		cache miss.
-		"""
-	),
-	('obj.http.',
-		'HEADER',
-		('hit',),
-		(), """
-		The corresponding HTTP header.
-		"""
-	),
-	('obj.ttl',
-		'DURATION',
-		('hit', 'deliver'),
-		(), """
-		The object's remaining time to live, in seconds.
-		"""
-	),
-	('obj.age',
-		'DURATION',
-		('hit', 'deliver'),
-		(), """
-		The age of the object.
-		"""
-	),
-	('obj.grace',
-		'DURATION',
-		('hit', 'deliver'),
-		(), """
-		The object's remaining grace period in seconds.
-		"""
-	),
-	('obj.keep',
-		'DURATION',
-		('hit', 'deliver'),
-		(), """
-		The object's remaining keep period in seconds.
-		"""
-	),
-	('obj.uncacheable',
-		'BOOL',
-		('deliver',),
-		(), """
-		Whether the object is uncacheable (pass, hit-for-pass or
-		hit-for-miss).
-		"""
-	),
-	('obj.storage',
-		'STEVEDORE',
-		('hit', 'deliver'),
-		(), """
-		The storage backend used to save this object.
-		"""
-	),
-	('resp',
-		'HTTP',
-		('deliver', 'synth'),
-		(), """
-		The entire response HTTP data structure.
-		"""
-	),
-	('resp.body',
-		'BODY',
-		(),
-		('synth',), """
-		The response body.
-		"""
-	),
-	('resp.proto',
-		'STRING',
-		('deliver', 'synth'),
-		('deliver', 'synth'), """
-		The HTTP protocol version to use for the response.
-		"""
-	),
-	('resp.status',
-		'INT',
-		('deliver', 'synth'),
-		('deliver', 'synth'), """
-		The HTTP status code that will be returned.
-
-		Assigning a HTTP standardized code to resp.status will also
-		set resp.reason to the corresponding status message.
-
-		resp.status 200 will get changed into 304 by core code after
-		a return(deliver) from vcl_deliver for conditional requests
-		to cached content if validation succeeds.
-		"""
-	),
-	('resp.reason',
-		'STRING',
-		('deliver', 'synth'),
-		('deliver', 'synth'), """
-		The HTTP status message that will be returned.
-		"""
-	),
-	('resp.http.',
-		'HEADER',
-		('deliver', 'synth'),
-		('deliver', 'synth'), """
-		The corresponding HTTP header.
-		"""
-	),
-	('resp.is_streaming',
-		'BOOL',
-		('deliver', 'synth'),
-		(), """
-		Returns true when the response will be streamed
-		from the backend.
-		"""
-	),
-	('now',
-		'TIME',
-		('all',),
-		(), """
-		The current time, in seconds since the epoch. When
-		used in string context it returns a formatted string.
-		"""
-	),
-]
+sp_variables = collections.OrderedDict()
+
+class vardef(object):
+	def __init__(self, nam, typ, rd, wr, doc):
+		sp_variables[nam] = self
+		self.nam = nam
+		self.typ = typ
+		self.rd = rd
+		self.wr = wr
+		self.doc = doc
+
+vardef('remote.ip',
+	'IP',
+	('both',),
+	(), """
+	The IP address of the other end of the TCP connection.
+	This can either be the clients IP, or the outgoing IP
+	of a proxy server.
+	"""
+)
+vardef('client.ip',
+	'IP',
+	('both',),
+	(), """
+	The client's IP address.
+	"""
+)
+vardef('client.identity',
+	'STRING',
+	('client',),
+	('client',), """
+	Identification of the client, used to load balance
+	in the client director. Defaults to the client's IP
+	address.
+	"""
+)
+vardef('local.ip',
+	'IP',
+	('both',),
+	(), """
+	The IP address of the local end of the TCP connection.
+	"""
+)
+vardef('server.ip',
+	'IP',
+	('both',),
+	(), """
+	The IP address of the socket on which the client
+	connection was received.
+	"""
+)
+vardef('server.hostname',
+	'STRING',
+	('all',),
+	(), """
+	The host name of the server.
+	"""
+)
+vardef('server.identity',
+	'STRING',
+	('all',),
+	(), """
+	The identity of the server, as set by the -i
+	parameter.  If the -i parameter is not passed to varnishd,
+	server.identity will be set to the hostname of the machine.
+	"""
+)
+vardef('req',
+	'HTTP',
+	('client',),
+	(), """
+	The entire request HTTP data structure
+	"""
+)
+vardef('req.method',
+	'STRING',
+	('client',),
+	('client',), """
+	The request type (e.g. "GET", "HEAD").
+	"""
+)
+vardef('req.hash',
+	'BLOB',
+	('hit', 'miss', 'pass', 'purge', 'deliver', ),
+	(), """
+	The hash key of this request.
+	"""
+)
+vardef('req.url',
+	'STRING',
+	('client',),
+	('client',), """
+	The requested URL.
+	"""
+)
+vardef('req.proto',
+	'STRING',
+	('client',),
+	('client',), """
+	The HTTP protocol version used by the client.
+	"""
+)
+vardef('req.http.',
+	'HEADER',
+	('client',),
+	('client',), """
+	The corresponding HTTP header.
+	"""
+)
+vardef('req.restarts',
+	'INT',
+	('client',),
+	(), """
+	A count of how many times this request has been restarted.
+	"""
+)
+vardef('req.storage',
+	'STEVEDORE',
+	('client',),
+	('client',), """
+	The storage backend to use to save this request body.
+	"""
+)
+vardef('req.esi_level',
+	'INT',
+	('client',),
+	(), """
+	A count of how many levels of ESI requests we're currently at.
+	"""
+)
+vardef('req.ttl',
+	'DURATION',
+	('client',),
+	('client',), """
+	Upper limit on the object age for cache lookups to return hit.
+
+	Usage of req.ttl should be replaced with a check on
+	obj.ttl in vcl_hit, returning miss when needed, but
+	this currently hits bug #1799, so an additional
+	workaround is required.
+
+	Deprecated and scheduled for removal with varnish release 7.
+	"""
+)
+vardef('req.xid',
+	'STRING',
+	('client',),
+	(), """
+	Unique ID of this request.
+	"""
+)
+vardef('req.esi',
+	'BOOL',
+	('client',),
+	('client',), """
+	Boolean. Set to false to disable ESI processing
+	regardless of any value in beresp.do_esi. Defaults
+	to true. This variable is subject to change in
+	future versions, you should avoid using it.
+	"""
+)
+vardef('req.can_gzip',
+	'BOOL',
+	('client',),
+	(), """
+	Does the client accept the gzip transfer encoding.
+	"""
+)
+vardef('req.backend_hint',
+	'BACKEND',
+	('client', ),
+	('client',), """
+	Set bereq.backend to this if we attempt to fetch.
+	When set to a director, reading this variable returns
+	an actual backend if the director has resolved immediately,
+	or the director otherwise.
+	When used in string context, returns the name of the director
+	or backend, respectively.
+	"""
+)
+vardef('req.hash_ignore_busy',
+	'BOOL',
+	('client',),
+	('client',), """
+	Ignore any busy object during cache lookup. You
+	would want to do this if you have two server looking
+	up content from each other to avoid potential deadlocks.
+	"""
+)
+vardef('req.hash_always_miss',
+	'BOOL',
+	('client',),
+	('client',), """
+	Force a cache miss for this request. If set to true
+	Varnish will disregard any existing objects and
+	always (re)fetch from the backend.
+	"""
+)
+vardef('req_top.method',
+	'STRING',
+	('client',),
+	(), """
+	The request method of the top-level request in a tree
+	of ESI requests. (e.g. "GET", "HEAD").
+	Identical to req.method in non-ESI requests.
+	"""
+)
+vardef('req_top.url',
+	'STRING',
+	('client',),
+	(), """
+	The requested URL of the top-level request in a tree
+	of ESI requests.
+	Identical to req.url in non-ESI requests.
+	"""
+)
+vardef('req_top.http.',
+	'HEADER',
+	('client',),
+	(), """
+	HTTP headers of the top-level request in a tree of ESI requests.
+	Identical to req.http. in non-ESI requests.
+	"""
+)
+vardef('req_top.proto',
+	'STRING',
+	('client',),
+	(), """
+	HTTP protocol version of the top-level request in a tree of
+	ESI requests.
+	Identical to req.proto in non-ESI requests.
+	"""
+)
+vardef('bereq',
+	'HTTP',
+	('backend',),
+	(), """
+	The entire backend request HTTP data structure
+	"""
+)
+vardef('bereq.xid',
+	'STRING',
+	('backend',),
+	(), """
+	Unique ID of this request.
+	"""
+)
+vardef('bereq.retries',
+	'INT',
+	('backend',),
+	(), """
+	A count of how many times this request has been retried.
+	"""
+)
+vardef('bereq.backend',
+	'BACKEND',
+	('pipe', 'backend', ),
+	('pipe', 'backend', ), """
+	This is the backend or director we attempt to fetch from.
+	When set to a director, reading this variable returns
+	an actual backend if the director has resolved immediately,
+	or the director otherwise.
+	When used in string context, returns the name of the director
+	or backend, respectively.
+	"""
+)
+vardef('bereq.body',
+	'BODY',
+	(),
+	('backend_fetch',), """
+	The request body.
+	"""
+)
+vardef('bereq.hash',
+	'BLOB',
+	('pipe', 'backend', ),
+	(), """
+	The hash key of this request.
+	"""
+)
+vardef('bereq.method',
+	'STRING',
+	('pipe', 'backend', ),
+	('pipe', 'backend', ), """
+	The request type (e.g. "GET", "HEAD").
+	"""
+)
+vardef('bereq.url',
+	'STRING',
+	('pipe', 'backend', ),
+	('pipe', 'backend', ), """
+	The requested URL.
+	"""
+)
+vardef('bereq.proto',
+	'STRING',
+	('pipe', 'backend', ),
+	('pipe', 'backend', ), """
+	The HTTP protocol version used to talk to the server.
+	"""
+)
+vardef('bereq.http.',
+	'HEADER',
+	('pipe', 'backend', ),
+	('pipe', 'backend', ), """
+	The corresponding HTTP header.
+	"""
+)
+vardef('bereq.uncacheable',
+	'BOOL',
+	('backend', ),
+	(), """
+	Indicates whether this request is uncacheable due
+	to a pass in the client side or a hit on an hit-for-pass
+	object.
+	"""
+)
+vardef('bereq.connect_timeout',
+	'DURATION',
+	('pipe', 'backend', ),
+	('pipe', 'backend', ), """
+	The time in seconds to wait for a backend connection.
+	"""
+)
+vardef('bereq.first_byte_timeout',
+	'DURATION',
+	('backend', ),
+	('backend', ), """
+	The time in seconds to wait for the first byte from
+	the backend.  Not available in pipe mode.
+	"""
+)
+vardef('bereq.between_bytes_timeout',
+	'DURATION',
+	('backend', ),
+	('backend', ), """
+	The time in seconds to wait between each received byte from the
+	backend.  Not available in pipe mode.
+	"""
+)
+vardef('bereq.is_bgfetch',
+	'BOOL',
+	('backend', ),
+	(), """
+	True for background fetches.
+	"""
+)
+vardef('beresp',
+	'HTTP',
+	('backend_response', 'backend_error'),
+	(), """
+	The entire backend response HTTP data structure
+	"""
+)
+vardef('beresp.body',
+	'BODY',
+	(),
+	('backend_error',), """
+	The response body.
+	"""
+)
+vardef('beresp.proto',
+	'STRING',
+	('backend_response', 'backend_error'),
+	('backend_response', 'backend_error'), """
+	The HTTP protocol version used the backend replied with.
+	"""
+)
+vardef('beresp.status',
+	'INT',
+	('backend_response', 'backend_error'),
+	('backend_response', 'backend_error'), """
+	The HTTP status code returned by the server.
+
+	Status codes >1000 can be set for vcl-internal
+	purposes and will be taken modulo 1000 on delivery.
+	"""
+)
+vardef('beresp.reason',
+	'STRING',
+	('backend_response', 'backend_error'),
+	('backend_response', 'backend_error'), """
+	The HTTP status message returned by the server.
+	"""
+)
+vardef('beresp.http.',
+	'HEADER',
+	('backend_response', 'backend_error'),
+	('backend_response', 'backend_error'), """
+	The corresponding HTTP header.
+	"""
+)
+vardef('beresp.do_esi',
+	'BOOL',
+	('backend_response', 'backend_error'),
+	('backend_response', 'backend_error'), """
+	Boolean. ESI-process the object after fetching it.
+	Defaults to false. Set it to true to parse the
+	object for ESI directives. Will only be honored if
+	req.esi is true.
+	"""
+)
+vardef('beresp.do_stream',
+	'BOOL',
+	('backend_response', 'backend_error'),
+	('backend_response', 'backend_error'), """
+	Deliver the object to the client while fetching the whole
+	object into varnish. For uncacheable objects, storage for
+	parts of the body which have been sent to the client may
+	get freed early, depending on the storage engine used.
+	"""
+)
+vardef('beresp.do_gzip',
+	'BOOL',
+	('backend_response', 'backend_error'),
+	('backend_response', 'backend_error'), """
+	Boolean. Gzip the object before storing it. Defaults
+	to false. When http_gzip_support is on Varnish will
+	request already compressed content from the backend
+	and as such compression in Varnish is not needed.
+	"""
+)
+vardef('beresp.do_gunzip',
+	'BOOL',
+	('backend_response', 'backend_error'),
+	('backend_response', 'backend_error'), """
+	Boolean. Unzip the object before storing it in the
+	cache.  Defaults to false.
+	"""
+)
+vardef('beresp.was_304',
+	'BOOL',
+	('backend_response', 'backend_error'),
+	(), """
+	Boolean. If this is a successful 304 response to a
+	backend conditional request refreshing an existing
+	cache object.
+	"""
+)
+vardef('beresp.uncacheable',
+	'BOOL',
+	('backend_response', 'backend_error'),
+	('backend_response', 'backend_error'), """
+	Inherited from bereq.uncacheable, see there.
+
+	Setting this variable makes the object uncacheable, which may
+	get stored as a hit-for-miss object in the cache.
+
+	Clearing the variable has no effect and will log the warning
+	"Ignoring attempt to reset beresp.uncacheable".
+	"""
+)
+vardef('beresp.ttl',
+	'DURATION',
+	('backend_response', 'backend_error'),
+	('backend_response', 'backend_error'), """
+	The object's remaining time to live, in seconds.
+	"""
+)
+vardef('beresp.age',
+	'DURATION',
+	('backend_response', 'backend_error'),
+	(), """
+	The age of the object.
+	"""
+)
+vardef('beresp.grace',
+	'DURATION',
+	('backend_response', 'backend_error'),
+	('backend_response', 'backend_error'), """
+	Set to a period to enable grace.
+	"""
+)
+vardef('beresp.keep',
+	'DURATION',
+	('backend_response', 'backend_error'),
+	('backend_response', 'backend_error'), """
+	Set to a period to enable conditional backend requests.
+
+	The keep time is cache lifetime in addition to the ttl.
+
+	Objects with ttl expired but with keep time left may be used
+	to issue conditional (If-Modified-Since / If-None-Match)
+	requests to the backend to refresh them.
+	"""
+)
+vardef('beresp.backend',
+	'BACKEND',
+	('backend_response', 'backend_error'),
+	(), """
+	This is the backend we fetched from.  If bereq.backend
+	was set to a director, this will be the backend selected
+	by the director.
+	When used in string context, returns its name.
+	"""
+)
+vardef('beresp.backend.name',
+	'STRING',
+	('backend_response', 'backend_error'),
+	(), """
+	Name of the backend this response was fetched from.
+	Same as beresp.backend.
+	"""
+)
+vardef('beresp.backend.ip',
+	'IP',
+	('backend_response',),
+	(), """
+	IP of the backend this response was fetched from.
+	"""
+)
+vardef('beresp.storage',
+	'STEVEDORE',
+	('backend_response', 'backend_error'),
+	('backend_response', 'backend_error'), """
+	The storage backend to use to save this object.
+	"""
+)
+vardef('obj.proto',
+	'STRING',
+	('hit',),
+	(), """
+	The HTTP protocol version stored with the object.
+	"""
+)
+vardef('obj.status',
+	'INT',
+	('hit',),
+	(), """
+	The HTTP status code stored with the object.
+	"""
+)
+vardef('obj.reason',
+	'STRING',
+	('hit',),
+	(), """
+	The HTTP reason phrase stored with the object.
+	"""
+)
+vardef('obj.hits',
+	'INT',
+	('hit', 'deliver'),
+	(), """
+	The count of cache-hits on this object. A value of 0 indicates a
+	cache miss.
+	"""
+)
+vardef('obj.http.',
+	'HEADER',
+	('hit',),
+	(), """
+	The corresponding HTTP header.
+	"""
+)
+vardef('obj.ttl',
+	'DURATION',
+	('hit', 'deliver'),
+	(), """
+	The object's remaining time to live, in seconds.
+	"""
+)
+vardef('obj.age',
+	'DURATION',
+	('hit', 'deliver'),
+	(), """
+	The age of the object.
+	"""
+)
+vardef('obj.grace',
+	'DURATION',
+	('hit', 'deliver'),
+	(), """
+	The object's remaining grace period in seconds.
+	"""
+)
+vardef('obj.keep',
+	'DURATION',
+	('hit', 'deliver'),
+	(), """
+	The object's remaining keep period in seconds.
+	"""
+)
+vardef('obj.uncacheable',
+	'BOOL',
+	('deliver',),
+	(), """
+	Whether the object is uncacheable (pass, hit-for-pass or
+	hit-for-miss).
+	"""
+)
+vardef('obj.storage',
+	'STEVEDORE',
+	('hit', 'deliver'),
+	(), """
+	The storage backend used to save this object.
+	"""
+)
+vardef('resp',
+	'HTTP',
+	('deliver', 'synth'),
+	(), """
+	The entire response HTTP data structure.
+	"""
+)
+vardef('resp.body',
+	'BODY',
+	(),
+	('synth',), """
+	The response body.
+	"""
+)
+vardef('resp.proto',
+	'STRING',
+	('deliver', 'synth'),
+	('deliver', 'synth'), """
+	The HTTP protocol version to use for the response.
+	"""
+)
+vardef('resp.status',
+	'INT',
+	('deliver', 'synth'),
+	('deliver', 'synth'), """
+	The HTTP status code that will be returned.
+
+	Assigning a HTTP standardized code to resp.status will also
+	set resp.reason to the corresponding status message.
+
+	resp.status 200 will get changed into 304 by core code after
+	a return(deliver) from vcl_deliver for conditional requests
+	to cached content if validation succeeds.
+	"""
+)
+vardef('resp.reason',
+	'STRING',
+	('deliver', 'synth'),
+	('deliver', 'synth'), """
+	The HTTP status message that will be returned.
+	"""
+)
+vardef('resp.http.',
+	'HEADER',
+	('deliver', 'synth'),
+	('deliver', 'synth'), """
+	The corresponding HTTP header.
+	"""
+)
+vardef('resp.is_streaming',
+	'BOOL',
+	('deliver', 'synth'),
+	(), """
+	Returns true when the response will be streamed
+	from the backend.
+	"""
+)
+vardef('now',
+	'TIME',
+	('all',),
+	(), """
+	The current time, in seconds since the epoch. When
+	used in string context it returns a formatted string.
+	"""
+)
 
 # Backwards compatibility:
 aliases = []
@@ -1240,50 +1250,47 @@ const struct var vcc_vars[] = {
 
 def one_var(nm, spec):
 	fh.write("\n")
-	typ = spec[1]
-	cnam = i[0].replace(".", "_")
-	ctyp = vcltypes[typ]
+	cnam = spec.nam.replace(".", "_")
+	ctyp = vcltypes[spec.typ]
 
-	fo.write("\t{ \"%s\", %s,\n" % (nm, typ))
+	fo.write("\t{ \"%s\", %s,\n" % (nm, spec.typ))
 
-	if len(spec[2]) == 0:
+	if len(spec.rd) == 0:
 		fo.write('\t    NULL,\t/* No reads allowed */\n')
-	elif typ == "HEADER":
+	elif spec.typ == "HEADER":
 		fo.write('\t    "HDR_')
 		fo.write(nm.split(".")[0].upper())
 		fo.write('",\n')
 	else:
 		fo.write('\t    "VRT_r_%s(ctx)",\n' % cnam)
-		if nm == i[0]:
-			fh.write("VCL_" + typ + " VRT_r_%s(VRT_CTX);\n" % cnam)
-	restrict(fo, spec[2])
+		if nm == spec.nam:
+			fh.write("VCL_" + spec.typ + " VRT_r_%s(VRT_CTX);\n" % cnam)
+	restrict(fo, spec.rd)
 
-	if len(spec[3]) == 0:
+	if len(spec.wr) == 0:
 		fo.write('\t    NULL,\t/* No writes allowed */\n')
-	elif typ == "HEADER":
+	elif spec.typ == "HEADER":
 		fo.write('\t    "HDR_')
 		fo.write(nm.split(".")[0].upper())
 		fo.write('",\n')
 	else:
 		fo.write('\t    "VRT_l_%s(ctx, ",\n' % cnam)
-		if nm == i[0]:
+		if nm == spec.nam:
 			fh.write("void VRT_l_%s(VRT_CTX, " % cnam)
-			if typ != "STRING" and typ != "BODY":
-				fh.write("VCL_" + typ + ");\n")
+			if spec.typ != "STRING" and spec.typ != "BODY":
+				fh.write("VCL_" + spec.typ + ");\n")
 			else:
 				fh.write(ctyp + ", ...);\n")
-	restrict(fo, spec[3])
+	restrict(fo, spec.wr)
 
 	fo.write("\t},\n")
 
-
-sp_variables.sort()
 aliases.sort()
 for i in sp_variables:
-	one_var(i[0], i)
+	one_var(i, sp_variables[i])
 	for j in aliases:
 		if j[1] == i[0]:
-			one_var(j[0], i)
+			one_var(j[0], sp_variables[i])
 
 fo.write("\t{ NULL }\n};\n\n")
 
@@ -1355,9 +1362,6 @@ fo.close()
 
 fp_vclvar = open(join(buildroot, "doc/sphinx/include/vcl_var.rst"), "w")
 
-l = sorted(sp_variables)
-
-
 def rst_where(fo, h, l):
 	ll = list()
 	if len(l) == 0:
@@ -1382,17 +1386,17 @@ def rst_where(fo, h, l):
 	fo.write("\n\n")
 
 hdr = ""
-for i in l:
-	j = i[0].split(".")
+for i in sp_variables.values():
+	j = i.nam.split(".")
 	if j[0] != hdr:
 		fp_vclvar.write("\n" + j[0] + "\n")
 		fp_vclvar.write("~" * len(j[0]) + "\n")
 		hdr = j[0]
-	fp_vclvar.write("\n" + i[0] + "\n\n")
-	fp_vclvar.write("\tType: " + i[1] + "\n\n")
-	rst_where(fp_vclvar, "Readable from: ", i[2])
-	rst_where(fp_vclvar, "Writable from: ", i[3])
-	for j in i[4].split("\n"):
+	fp_vclvar.write("\n" + i.nam + "\n\n")
+	fp_vclvar.write("\tType: " + i.typ + "\n\n")
+	rst_where(fp_vclvar, "Readable from: ", i.rd)
+	rst_where(fp_vclvar, "Writable from: ", i.wr)
+	for j in i.doc.split("\n"):
 		fp_vclvar.write("\t%s\n" % j.strip())
 
 hdr = "storage"

From phk at FreeBSD.org  Wed Jan 24 12:24:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jan 2018 12:24:08 +0000 (UTC)
Subject: [master] f53c783 Don't treat 'unset' as a special case of 'set',
 having bereq.body hardcoded in the compiler is not nice.
Message-ID: <20180124122408.5F7F1B7BF8@lists.varnish-cache.org>


commit f53c7838defcf2799be1501eba3a79cf4f3df2ad
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 24 12:21:18 2018 +0000

    Don't treat 'unset' as a special case of 'set',
    having bereq.body hardcoded in the compiler is not nice.

diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c
index b16c614..bb82fd6 100644
--- a/bin/varnishd/cache/cache_vrt_var.c
+++ b/bin/varnishd/cache/cache_vrt_var.c
@@ -434,11 +434,10 @@ VRT_r_beresp_backend(VRT_CTX)
 /*--------------------------------------------------------------------*/
 
 void
-VRT_l_bereq_body(VRT_CTX, const char *p, ...)
+VRT_u_bereq_body(VRT_CTX)
 {
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC);
-	assert(p == vrt_magic_string_unset);
 	if (ctx->bo->req != NULL) {
 		CHECK_OBJ_NOTNULL(ctx->bo->req, REQ_MAGIC);
 		ctx->bo->req = NULL;
diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc
index 186f7cb..7c14f02 100644
--- a/bin/varnishtest/tests/v00018.vtc
+++ b/bin/varnishtest/tests/v00018.vtc
@@ -68,12 +68,12 @@ varnish v1 -errvcl {Symbol not found: 'mu' (expected type BOOL):} {
 	sub vcl_backend_response { set beresp.do_gzip = mu; }
 }
 
-varnish v1 -errvcl {Only bereq.body and HTTP header variables can be unset.} {
+varnish v1 -errvcl {Variable 'beresp.do_gzip' cannot be unset} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_backend_response { unset beresp.do_gzip; }
 }
 
-varnish v1 -errvcl {bereq.body cannot be set.} {
+varnish v1 -errvcl {Variable 'bereq.body' cannot be set.} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_backend_fetch { set bereq.body = "foo"; }
 }
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 5e5c04f..3b6dda7 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -169,13 +169,14 @@ returns = (
 sp_variables = collections.OrderedDict()
 
 class vardef(object):
-	def __init__(self, nam, typ, rd, wr, doc):
+	def __init__(self, nam, typ, rd, wr, doc, uns = None):
 		sp_variables[nam] = self
 		self.nam = nam
 		self.typ = typ
 		self.rd = rd
 		self.wr = wr
 		self.doc = doc
+		self.uns = uns
 
 vardef('remote.ip',
 	'IP',
@@ -273,7 +274,8 @@ vardef('req.http.',
 	('client',),
 	('client',), """
 	The corresponding HTTP header.
-	"""
+	""",
+	uns = ('client',)
 )
 vardef('req.restarts',
 	'INT',
@@ -435,9 +437,10 @@ vardef('bereq.backend',
 vardef('bereq.body',
 	'BODY',
 	(),
-	('backend_fetch',), """
+	(), """
 	The request body.
-	"""
+	""",
+	uns = ('backend_fetch',)
 )
 vardef('bereq.hash',
 	'BLOB',
@@ -472,7 +475,8 @@ vardef('bereq.http.',
 	('pipe', 'backend', ),
 	('pipe', 'backend', ), """
 	The corresponding HTTP header.
-	"""
+	""",
+	uns = ('pipe', 'backend', )
 )
 vardef('bereq.uncacheable',
 	'BOOL',
@@ -556,7 +560,8 @@ vardef('beresp.http.',
 	('backend_response', 'backend_error'),
 	('backend_response', 'backend_error'), """
 	The corresponding HTTP header.
-	"""
+	""",
+	uns = ('backend_response', 'backend_error')
 )
 vardef('beresp.do_esi',
 	'BOOL',
@@ -810,7 +815,8 @@ vardef('resp.http.',
 	('deliver', 'synth'),
 	('deliver', 'synth'), """
 	The corresponding HTTP header.
-	"""
+	""",
+	uns = ('deliver', 'synth')
 )
 vardef('resp.is_streaming',
 	'BOOL',
@@ -1306,6 +1312,15 @@ def one_var(nm, spec):
 		restrict(fo, spec.wr)
 		fo.write(";\n")
 
+	if spec.uns is None or len(spec.uns) == 0:
+		fo.write('\t/* No unsets allowed */\n')
+	else:
+		fh.write("void VRT_u_%s(VRT_CTX);\n" % cnam)
+		fo.write('\tsym->uname = "VRT_u_%s(ctx)";\n' % cnam)
+		fo.write('\tsym->u_methods =\n')
+		restrict(fo, spec.uns)
+		fo.write(";\n")
+
 aliases.sort()
 for i in sp_variables:
 	one_var(i, sp_variables[i])
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 1d582e2..ee6fe85 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -81,6 +81,9 @@ static const struct arith {
 	{ VOID,		'=',		VOID }
 };
 
+
+/*--------------------------------------------------------------------*/
+
 static void
 parse_set(struct vcc *tl)
 {
@@ -90,14 +93,27 @@ parse_set(struct vcc *tl)
 
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
-	sym = vcc_FindVar(tl, "cannot be set");
+	sym = VCC_SymbolTok(tl, NULL, SYM_VAR, 0);
 	ERRCHK(tl);
-	assert(sym != NULL);
-	if (vcc_IdIs(tl->t, "bereq.body")) {
-		VSB_printf(tl->sb, "bereq.body cannot be set.\n");
+	if (sym == NULL) {
+		VSB_printf(tl->sb, "Unknown variable ");
+		vcc_ErrToken(tl, tl->t);
+		VSB_cat(tl->sb, "\nAt: ");
+		vcc_ErrWhere(tl, tl->t);
+		return;
+	}
+	if (sym->w_methods == 0) {
+		VSB_printf(tl->sb, "Variable ");
+		vcc_ErrToken(tl, tl->t);
+		if (sym->r_methods != 0)
+			VSB_printf(tl->sb, " is read only.");
+		else
+			VSB_printf(tl->sb, " cannot be set.");
+		VSB_cat(tl->sb, "\nAt: ");
 		vcc_ErrWhere(tl, tl->t);
 		return;
 	}
+	vcc_AddUses(tl, tl->t, sym->w_methods, "cannot be set");
 	Fb(tl, 1, "%s\n", sym->lname);
 	tl->indent += INDENT;
 	vcc_NextToken(tl);
@@ -138,18 +154,25 @@ parse_unset(struct vcc *tl)
 	/* XXX: Wrong, should use VCC_Expr(HEADER) */
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
-	sym = vcc_FindVar(tl, "cannot be unset");
+	sym = VCC_SymbolTok(tl, NULL, SYM_VAR, 0);
 	ERRCHK(tl);
-	assert(sym != NULL);
-	if (sym->fmt != HEADER && !vcc_IdIs(tl->t, "bereq.body")) {
-		VSB_printf(tl->sb,
-		    "Only bereq.body and HTTP header variables can"
-		    " be unset.\n");
+	if (sym == NULL) {
+		VSB_printf(tl->sb, "Unknown variable ");
+		vcc_ErrToken(tl, tl->t);
+		VSB_cat(tl->sb, "\nAt: ");
 		vcc_ErrWhere(tl, tl->t);
 		return;
 	}
-	ERRCHK(tl);
-	Fb(tl, 1, "%svrt_magic_string_unset);\n", sym->lname);
+	if (sym->u_methods == 0) {
+		VSB_printf(tl->sb, "Variable ");
+		vcc_ErrToken(tl, tl->t);
+		VSB_printf(tl->sb, " cannot be unset.");
+		VSB_cat(tl->sb, "\nAt: ");
+		vcc_ErrWhere(tl, tl->t);
+		return;
+	}
+	vcc_AddUses(tl, tl->t, sym->u_methods, "cannot be unset.");
+	Fb(tl, 1, "%s;\n", sym->uname);
 	vcc_NextToken(tl);
 }
 
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index e0b0da9..8694c38 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -141,6 +141,8 @@ struct symbol {
 	unsigned			r_methods;
 	const char			*lname;
 	unsigned			w_methods;
+	const char			*uname;
+	unsigned			u_methods;
 };
 
 VTAILQ_HEAD(tokenhead, token);
@@ -335,7 +337,6 @@ vcc_type_t VCC_Type(const char *p);
 
 /* vcc_var.c */
 sym_wildcard_t vcc_Var_Wildcard;
-const struct symbol *vcc_FindVar(struct vcc *, const char *);
 
 /* vcc_vmod.c */
 void vcc_ParseImport(struct vcc *tl);
diff --git a/lib/libvcc/vcc_var.c b/lib/libvcc/vcc_var.c
index 133455f..0c9af8c 100644
--- a/lib/libvcc/vcc_var.c
+++ b/lib/libvcc/vcc_var.c
@@ -59,6 +59,7 @@ vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent,
 	sym->eval = vcc_Eval_Var;
 	sym->r_methods = parent->r_methods;
 	sym->w_methods = parent->w_methods;
+	sym->u_methods = parent->u_methods;
 
 	/* Create a C-name version of the header name */
 	vsb = VSB_new_auto();
@@ -79,34 +80,10 @@ vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent,
 	VSB_printf(vsb, "VRT_SetHdr(ctx, %s,", sym->rname);
 	AZ(VSB_finish(vsb));
 	sym->lname = TlDup(tl, VSB_data(vsb));
+	VSB_clear(vsb);
+	VSB_printf(vsb, "VRT_SetHdr(ctx, %s, vrt_magic_string_unset)",
+	    sym->rname);
+	AZ(VSB_finish(vsb));
+	sym->uname = TlDup(tl, VSB_data(vsb));
 	VSB_destroy(&vsb);
 }
-
-/*--------------------------------------------------------------------*/
-
-const struct symbol *
-vcc_FindVar(struct vcc *tl, const char *use)
-{
-	const struct symbol *sym;
-
-	sym = VCC_SymbolTok(tl, NULL, SYM_VAR, 0);
-	if (tl->err)
-		return (NULL);
-	if (sym != NULL) {
-		if (sym->w_methods == 0) {
-			VSB_printf(tl->sb, "Variable ");
-			vcc_ErrToken(tl, tl->t);
-			VSB_printf(tl->sb, " is read only.");
-			VSB_cat(tl->sb, "\nAt: ");
-			vcc_ErrWhere(tl, tl->t);
-			return (NULL);
-		}
-		vcc_AddUses(tl, tl->t, sym->w_methods, use);
-		return (sym);
-	}
-	VSB_printf(tl->sb, "Unknown variable ");
-	vcc_ErrToken(tl, tl->t);
-	VSB_cat(tl->sb, "\nAt: ");
-	vcc_ErrWhere(tl, tl->t);
-	return (NULL);
-}

From phk at FreeBSD.org  Wed Jan 24 12:36:05 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jan 2018 12:36:05 +0000 (UTC)
Subject: [master] 717e8d5 Eliminate unused argument
Message-ID: <20180124123605.B9998B8051@lists.varnish-cache.org>


commit 717e8d50e6a2f23990376413a2e1ffb40aa40192
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 24 12:34:53 2018 +0000

    Eliminate unused argument

diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index ee6fe85..3126ef1 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -93,7 +93,7 @@ parse_set(struct vcc *tl)
 
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
-	sym = VCC_SymbolTok(tl, NULL, SYM_VAR, 0);
+	sym = VCC_SymbolTok(tl, SYM_VAR, 0);
 	ERRCHK(tl);
 	if (sym == NULL) {
 		VSB_printf(tl->sb, "Unknown variable ");
@@ -154,7 +154,7 @@ parse_unset(struct vcc *tl)
 	/* XXX: Wrong, should use VCC_Expr(HEADER) */
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
-	sym = VCC_SymbolTok(tl, NULL, SYM_VAR, 0);
+	sym = VCC_SymbolTok(tl, SYM_VAR, 0);
 	ERRCHK(tl);
 	if (sym == NULL) {
 		VSB_printf(tl->sb, "Unknown variable ");
@@ -268,7 +268,7 @@ parse_return_vcl(struct vcc *tl)
 	ExpectErr(tl, '(');
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
-	sym = VCC_SymbolTok(tl, NULL, SYM_VCL, 0);
+	sym = VCC_SymbolTok(tl, SYM_VCL, 0);
 	ERRCHK(tl);
 	if (sym == NULL) {
 		VSB_printf(tl->sb, "Not a VCL label:\n");
@@ -419,7 +419,7 @@ vcc_ParseAction(struct vcc *tl)
 			return (1);
 		}
 	}
-	sym = VCC_SymbolTok(tl, NULL, SYM_NONE, 0);
+	sym = VCC_SymbolTok(tl, SYM_NONE, 0);
 	if (sym != NULL && sym->kind == SYM_FUNC) {
 		vcc_Expr_Call(tl, sym);
 		return (1);
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index 1655f92..8a6b8fd 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -389,7 +389,7 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 			Fb(tl, 0, "\t.probe = %s,\n", p);
 			ERRCHK(tl);
 		} else if (vcc_IdIs(t_field, "probe") && tl->t->tok == ID) {
-			pb = VCC_SymbolTok(tl, NULL, SYM_PROBE, 0);
+			pb = VCC_SymbolTok(tl, SYM_PROBE, 0);
 			if (pb == NULL) {
 				VSB_printf(tl->sb, "Probe %.*s not found\n",
 				    PF(tl->t));
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 8694c38..1ff5d1a 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -310,7 +310,7 @@ void vcc_stevedore(struct vcc *vcc, const char *stv_name);
 void VCC_PrintCName(struct vsb *vsb, const char *b, const char *e);
 struct symbol *VCC_Symbol(struct vcc *, struct symbol *,
     const char *, const char *, enum symkind, int);
-struct symbol *VCC_SymbolTok(struct vcc *, struct symbol *, enum symkind, int);
+struct symbol *VCC_SymbolTok(struct vcc *, enum symkind, int);
 const char * VCC_SymKind(struct vcc *tl, const struct symbol *s);
 typedef void symwalk_f(struct vcc *tl, const struct symbol *s);
 void VCC_WalkSymbols(struct vcc *tl, symwalk_f *func, enum symkind kind);
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index dacae1e..0919b7e 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -657,9 +657,9 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 		sym = NULL;
 		kind = VCC_HandleKind(fmt);
 		if (kind != SYM_NONE)
-			sym = VCC_SymbolTok(tl, NULL, kind, 0);
+			sym = VCC_SymbolTok(tl, kind, 0);
 		if (sym == NULL)
-			sym = VCC_SymbolTok(tl, NULL, SYM_NONE, 0);
+			sym = VCC_SymbolTok(tl, SYM_NONE, 0);
 		if (sym == NULL || sym->eval == NULL) {
 			VSB_printf(tl->sb, "Symbol not found: ");
 			vcc_ErrToken(tl, tl->t);
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index cb94ffb..ffb66fc 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -105,11 +105,10 @@ vcc_new_symbol(struct vcc *tl, const char *b, const char *e)
 }
 
 struct symbol *
-VCC_SymbolTok(struct vcc *tl, struct symbol *parent,
-    enum symkind kind, int create)
+VCC_SymbolTok(struct vcc *tl, enum symkind kind, int create)
 {
 
-	return (VCC_Symbol(tl, parent, tl->t->b, tl->t->e, kind, create));
+	return (VCC_Symbol(tl, NULL, tl->t->b, tl->t->e, kind, create));
 }
 
 struct symbol *
@@ -245,7 +244,7 @@ VCC_HandleSymbol(struct vcc *tl, vcc_type_t fmt, const char *pfx)
 	kind = VCC_HandleKind(fmt);
 	assert(kind != SYM_NONE);
 
-	sym = VCC_SymbolTok(tl, NULL, SYM_NONE, 0);
+	sym = VCC_SymbolTok(tl, SYM_NONE, 0);
 	if (sym != NULL && sym->def_b != NULL && kind == sym->kind) {
 		p = VCC_SymKind(tl, sym);
 		VSB_printf(tl->sb, "%c%s '%.*s' redefined.\n",
@@ -270,7 +269,7 @@ VCC_HandleSymbol(struct vcc *tl, vcc_type_t fmt, const char *pfx)
 		return (sym);
 	}
 	if (sym == NULL)
-		sym = VCC_SymbolTok(tl, NULL, kind, 1);
+		sym = VCC_SymbolTok(tl, kind, 1);
 	AN(sym);
 	AZ(sym->ndef);
 	VCC_GlobalSymbol(sym, fmt, pfx);
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 79adcb6..96abdda 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -75,7 +75,7 @@ vcc_ParseImport(struct vcc *tl)
 
 	ExpectErr(tl, ID);
 	mod = tl->t;
-	osym = VCC_SymbolTok(tl, NULL, SYM_NONE, 0);
+	osym = VCC_SymbolTok(tl, SYM_NONE, 0);
 	vcc_NextToken(tl);
 
 	if (osym != NULL && osym->kind != SYM_VMOD) {
@@ -288,7 +288,7 @@ vcc_ParseNew(struct vcc *tl)
 	vcc_NextToken(tl);
 
 	ExpectErr(tl, ID);
-	sy2 = VCC_SymbolTok(tl, NULL, SYM_OBJECT, 0);
+	sy2 = VCC_SymbolTok(tl, SYM_OBJECT, 0);
 	if (sy2 == NULL || sy2->extra == NULL) {
 		if (sy2 == NULL)
 			p = "Symbol";
diff --git a/lib/libvcc/vcc_xref.c b/lib/libvcc/vcc_xref.c
index 1462145..d6d5535 100644
--- a/lib/libvcc/vcc_xref.c
+++ b/lib/libvcc/vcc_xref.c
@@ -65,7 +65,7 @@ vcc_AddRef(struct vcc *tl, enum symkind kind)
 {
 	struct symbol *sym;
 
-	sym = VCC_SymbolTok(tl, NULL, kind, 1);
+	sym = VCC_SymbolTok(tl, kind, 1);
 	if (sym->ref_b == NULL)
 		sym->ref_b = tl->t;
 	AN(sym);
@@ -78,7 +78,7 @@ vcc_AddDef(struct vcc *tl, enum symkind kind)
 {
 	struct symbol *sym;
 
-	sym = VCC_SymbolTok(tl, NULL, kind, 1);
+	sym = VCC_SymbolTok(tl, kind, 1);
 	if (sym->def_b == NULL)
 		sym->def_b = tl->t;
 	AN(sym);
@@ -144,7 +144,7 @@ vcc_AddCall(struct vcc *tl)
 
 	pc = TlAlloc(tl, sizeof *pc);
 	assert(pc != NULL);
-	pc->sym = VCC_SymbolTok(tl, NULL, SYM_SUB, 1);
+	pc->sym = VCC_SymbolTok(tl, SYM_SUB, 1);
 	AN(pc->sym);
 	pc->t = tl->t;
 	VTAILQ_INSERT_TAIL(&tl->curproc->calls, pc, list);

From phk at FreeBSD.org  Wed Jan 24 13:20:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jan 2018 13:20:10 +0000 (UTC)
Subject: [master] cdbcb2e Unify symbol lookup diagnostics
Message-ID: <20180124132010.3E7F6B8E1D@lists.varnish-cache.org>


commit cdbcb2ee03394afae106a03e312b76e71f00cad6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 24 13:19:34 2018 +0000

    Unify symbol lookup diagnostics

diff --git a/bin/varnishtest/tests/r01837.vtc b/bin/varnishtest/tests/r01837.vtc
index 4dc3d68..fbfd797 100644
--- a/bin/varnishtest/tests/r01837.vtc
+++ b/bin/varnishtest/tests/r01837.vtc
@@ -1,6 +1,6 @@
 varnishtest "Test VCC errors out if probe is used before it is defined"
 
-varnish v1 -errvcl "Probe p not found" {
+varnish v1 -errvcl "Probe not found: 'p'" {
 	backend b {
 		.host = "127.0.0.1";
 		.probe = p;
diff --git a/bin/varnishtest/tests/v00021.vtc b/bin/varnishtest/tests/v00021.vtc
index 8dd69ad..dcf25d1 100644
--- a/bin/varnishtest/tests/v00021.vtc
+++ b/bin/varnishtest/tests/v00021.vtc
@@ -91,7 +91,7 @@ varnish v1 -vcl {
 	}
 }
 
-varnish v1 -errvcl {Unknown variable 'req.foobar'} {
+varnish v1 -errvcl {Unknown variable: 'req.foobar'} {
 	backend foo { .host = "127.0.0.1"; }
 	sub vcl_recv { set req.foobar = 3; }
 }
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 3126ef1..e37cff5 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -93,15 +93,9 @@ parse_set(struct vcc *tl)
 
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
-	sym = VCC_SymbolTok(tl, SYM_VAR, 0);
+	sym = VCC_SymbolGet(tl, SYM_VAR, "Unknown variable");
 	ERRCHK(tl);
-	if (sym == NULL) {
-		VSB_printf(tl->sb, "Unknown variable ");
-		vcc_ErrToken(tl, tl->t);
-		VSB_cat(tl->sb, "\nAt: ");
-		vcc_ErrWhere(tl, tl->t);
-		return;
-	}
+	AN(sym);
 	if (sym->w_methods == 0) {
 		VSB_printf(tl->sb, "Variable ");
 		vcc_ErrToken(tl, tl->t);
@@ -154,15 +148,9 @@ parse_unset(struct vcc *tl)
 	/* XXX: Wrong, should use VCC_Expr(HEADER) */
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
-	sym = VCC_SymbolTok(tl, SYM_VAR, 0);
+	sym = VCC_SymbolGet(tl, SYM_VAR, "Unknown variable");
 	ERRCHK(tl);
-	if (sym == NULL) {
-		VSB_printf(tl->sb, "Unknown variable ");
-		vcc_ErrToken(tl, tl->t);
-		VSB_cat(tl->sb, "\nAt: ");
-		vcc_ErrWhere(tl, tl->t);
-		return;
-	}
+	AN(sym);
 	if (sym->u_methods == 0) {
 		VSB_printf(tl->sb, "Variable ");
 		vcc_ErrToken(tl, tl->t);
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index 8a6b8fd..55e8ba9 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -389,13 +389,9 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 			Fb(tl, 0, "\t.probe = %s,\n", p);
 			ERRCHK(tl);
 		} else if (vcc_IdIs(t_field, "probe") && tl->t->tok == ID) {
-			pb = VCC_SymbolTok(tl, SYM_PROBE, 0);
-			if (pb == NULL) {
-				VSB_printf(tl->sb, "Probe %.*s not found\n",
-				    PF(tl->t));
-				vcc_ErrWhere(tl, tl->t);
-				return;
-			}
+			pb = VCC_SymbolGet(tl, SYM_PROBE, "Probe not found");
+			ERRCHK(tl);
+			AN(pb);
 			Fb(tl, 0, "\t.probe = %s,\n", pb->rname);
 			(void)vcc_AddRef(tl, SYM_PROBE);
 			vcc_NextToken(tl);
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 1ff5d1a..c5a496e 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -310,6 +310,7 @@ void vcc_stevedore(struct vcc *vcc, const char *stv_name);
 void VCC_PrintCName(struct vsb *vsb, const char *b, const char *e);
 struct symbol *VCC_Symbol(struct vcc *, struct symbol *,
     const char *, const char *, enum symkind, int);
+struct symbol *VCC_SymbolGet(struct vcc *, enum symkind, const char *);
 struct symbol *VCC_SymbolTok(struct vcc *, enum symkind, int);
 const char * VCC_SymKind(struct vcc *tl, const struct symbol *s);
 typedef void symwalk_f(struct vcc *tl, const struct symbol *s);
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index ffb66fc..17ac8c2 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -105,6 +105,21 @@ vcc_new_symbol(struct vcc *tl, const char *b, const char *e)
 }
 
 struct symbol *
+VCC_SymbolGet(struct vcc *tl, enum symkind kind, const char *err)
+{
+	struct symbol *sym;
+
+	sym = VCC_Symbol(tl, NULL, tl->t->b, tl->t->e, kind, 0);
+	if (sym != NULL && sym->kind == kind)
+		return (sym);
+	VSB_printf(tl->sb, "%s: ", err);
+	vcc_ErrToken(tl, tl->t);
+	VSB_cat(tl->sb, "\nAt: ");
+	vcc_ErrWhere(tl, tl->t);
+	return (NULL);
+}
+
+struct symbol *
 VCC_SymbolTok(struct vcc *tl, enum symkind kind, int create)
 {
 
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 96abdda..b4d307d 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -288,13 +288,11 @@ vcc_ParseNew(struct vcc *tl)
 	vcc_NextToken(tl);
 
 	ExpectErr(tl, ID);
-	sy2 = VCC_SymbolTok(tl, SYM_OBJECT, 0);
-	if (sy2 == NULL || sy2->extra == NULL) {
-		if (sy2 == NULL)
-			p = "Symbol";
-		else
-			p = "Constructor";
-		VSB_printf(tl->sb, "%s not found: ", p);
+	sy2 = VCC_SymbolGet(tl, SYM_OBJECT, "Symbol not found");
+	ERRCHK(tl);
+	AN(sy2);
+	if (sy2->extra == NULL) {
+		VSB_printf(tl->sb, "Constructor not found: ");
 		vcc_ErrToken(tl, tl->t);
 		VSB_printf(tl->sb, " at ");
 		vcc_ErrWhere(tl, tl->t);

From phk at FreeBSD.org  Wed Jan 24 20:11:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jan 2018 20:11:08 +0000 (UTC)
Subject: [master] f94c703 Rework the VCC error reporting code in preparation
 of multi-token idents.
Message-ID: <20180124201108.6390495D29@lists.varnish-cache.org>


commit f94c7034bc576233226664f744de72ca6abd0e1f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 24 20:10:09 2018 +0000

    Rework the VCC error reporting code in preparation of multi-token idents.

diff --git a/bin/varnishtest/tests/e00030.vtc b/bin/varnishtest/tests/e00030.vtc
index eaa998a..f5b5f16 100644
--- a/bin/varnishtest/tests/e00030.vtc
+++ b/bin/varnishtest/tests/e00030.vtc
@@ -1,7 +1,7 @@
 varnishtest "Test req_top.* in an ESI context"
 
 varnish v1 -arg "-p feature=+esi_disable_xml_check" \
-	   -errvcl {Variable 'req_top.url' is read only.} {
+	   -errvcl {Variable is read only.} {
 	sub vcl_recv {
 		set req_top.url = "/foo";
 	}
diff --git a/bin/varnishtest/tests/r00310.vtc b/bin/varnishtest/tests/r00310.vtc
index 9d0c431..c00e4c8 100644
--- a/bin/varnishtest/tests/r00310.vtc
+++ b/bin/varnishtest/tests/r00310.vtc
@@ -1,6 +1,6 @@
 varnishtest "Test obj.http.x-cache in vcl_hit"
 
-varnish v1 -errvcl {Variable 'obj.http.x-cache' is read only.} {
+varnish v1 -errvcl {Variable is read only.} {
 	backend foo { .host = "127.0.0.1"; }
 
 	sub vcl_hit {
diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc
index 0f9a4c8..29d96bd 100644
--- a/bin/varnishtest/tests/v00016.vtc
+++ b/bin/varnishtest/tests/v00016.vtc
@@ -52,14 +52,14 @@ varnish v1 -errvcl {Symbol not found: 'foo.bar'} {
 	}
 }
 
-varnish v1 -errvcl {'bereq.first_byte_timeout': cannot be set} {
+varnish v1 -errvcl {Cannot be set in method 'vcl_pipe'} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_pipe {
 		set bereq.first_byte_timeout = 10s;
 	}
 }
 
-varnish v1 -errvcl {'bereq.between_bytes_timeout': cannot be set} {
+varnish v1 -errvcl {Cannot be set in method 'vcl_pipe'.} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_pipe {
 		set bereq.between_bytes_timeout = 10s;
diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc
index 7c14f02..c9b6b6a 100644
--- a/bin/varnishtest/tests/v00018.vtc
+++ b/bin/varnishtest/tests/v00018.vtc
@@ -7,7 +7,7 @@ varnish v1 -vcl {
 	sub vcl_deliver { return(synth(resp.status, resp.reason)); }
 }
 
-varnish v1 -errvcl {Variable 'now' is read only.} {
+varnish v1 -errvcl {Variable is read only.} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_miss { set now = 1000; }
 }
@@ -68,12 +68,12 @@ varnish v1 -errvcl {Symbol not found: 'mu' (expected type BOOL):} {
 	sub vcl_backend_response { set beresp.do_gzip = mu; }
 }
 
-varnish v1 -errvcl {Variable 'beresp.do_gzip' cannot be unset} {
+varnish v1 -errvcl {Variable cannot be unset} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_backend_response { unset beresp.do_gzip; }
 }
 
-varnish v1 -errvcl {Variable 'bereq.body' cannot be set.} {
+varnish v1 -errvcl {Variable cannot be set.} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_backend_fetch { set bereq.body = "foo"; }
 }
@@ -119,7 +119,7 @@ varnish v1 -errvcl {Expected return action name.} {
 }
 
 # issue #936
-varnish v1 -errvcl {'synthetic': not a valid action in method 'vcl_recv'} {
+varnish v1 -errvcl {Not a valid action in method 'vcl_recv'} {
 	backend foo { .host = "127.0.0.1"; }
 	sub vcl_recv {
 		synthetic("XXX");
diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc
index c720531..05f7c24 100644
--- a/bin/varnishtest/tests/v00020.vtc
+++ b/bin/varnishtest/tests/v00020.vtc
@@ -218,7 +218,7 @@ varnish v1 -errvcl {Function returns VOID} {
 	}
 }
 
-varnish v1 -errvcl {'req.hash': Not available in method 'vcl_recv'.} {
+varnish v1 -errvcl {Not available in method 'vcl_recv'.} {
 	import blob;
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_recv {
@@ -226,7 +226,7 @@ varnish v1 -errvcl {'req.hash': Not available in method 'vcl_recv'.} {
 	}
 }
 
-varnish v1 -errvcl {'req.hash': Not available in method 'vcl_hash'.} {
+varnish v1 -errvcl {Not available in method 'vcl_hash'.} {
 	import blob;
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_hash {
@@ -234,7 +234,7 @@ varnish v1 -errvcl {'req.hash': Not available in method 'vcl_hash'.} {
 	}
 }
 
-varnish v1 -errvcl {'beresp.status': Not available in method 'vcl_recv'.} {
+varnish v1 -errvcl {Not available in method 'vcl_recv'.} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_recv {
 		set req.http.foo = 100 + beresp.status;
diff --git a/bin/varnishtest/tests/v00021.vtc b/bin/varnishtest/tests/v00021.vtc
index dcf25d1..7db8afd 100644
--- a/bin/varnishtest/tests/v00021.vtc
+++ b/bin/varnishtest/tests/v00021.vtc
@@ -1,11 +1,11 @@
 varnishtest "VCL compiler coverage test: vcc_xref.c vcc_var.c vcc_symb.c"
 
-varnish v1 -errvcl {Variable 'obj.ttl' is read only.} {
+varnish v1 -errvcl {Variable is read only.} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_recv { set obj.ttl = 1 w; }
 }
 
-varnish v1 -errvcl {Variable 'obj.ttl' is read only.} {
+varnish v1 -errvcl {Variable is read only.} {
 	backend b { .host = "127.0.0.1"; }
 
 	sub foo { set obj.ttl = 1 w; }
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index e37cff5..8f61bad 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -89,28 +89,27 @@ parse_set(struct vcc *tl)
 {
 	const struct symbol *sym;
 	const struct arith *ap;
+	const struct token *t;
 	vcc_type_t fmt;
 
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
+	t = tl->t;
 	sym = VCC_SymbolGet(tl, SYM_VAR, "Unknown variable");
 	ERRCHK(tl);
+	vcc_NextToken(tl);
 	AN(sym);
 	if (sym->w_methods == 0) {
-		VSB_printf(tl->sb, "Variable ");
-		vcc_ErrToken(tl, tl->t);
+		vcc_ErrWhere2(tl, t, tl->t);
 		if (sym->r_methods != 0)
-			VSB_printf(tl->sb, " is read only.");
+			VSB_printf(tl->sb, "Variable is read only.\n");
 		else
-			VSB_printf(tl->sb, " cannot be set.");
-		VSB_cat(tl->sb, "\nAt: ");
-		vcc_ErrWhere(tl, tl->t);
+			VSB_printf(tl->sb, "Variable cannot be set.\n");
 		return;
 	}
-	vcc_AddUses(tl, tl->t, sym->w_methods, "cannot be set");
+	vcc_AddUses(tl, t, tl->t, sym->w_methods, "Cannot be set");
 	Fb(tl, 1, "%s\n", sym->lname);
 	tl->indent += INDENT;
-	vcc_NextToken(tl);
 	fmt = sym->fmt;
 	for (ap = arith; ap->type != VOID; ap++) {
 		if (ap->type != fmt)
@@ -144,24 +143,23 @@ static void
 parse_unset(struct vcc *tl)
 {
 	const struct symbol *sym;
+	const struct token *t;
 
 	/* XXX: Wrong, should use VCC_Expr(HEADER) */
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
+	t = tl->t;
 	sym = VCC_SymbolGet(tl, SYM_VAR, "Unknown variable");
 	ERRCHK(tl);
+	vcc_NextToken(tl);
 	AN(sym);
 	if (sym->u_methods == 0) {
-		VSB_printf(tl->sb, "Variable ");
-		vcc_ErrToken(tl, tl->t);
-		VSB_printf(tl->sb, " cannot be unset.");
-		VSB_cat(tl->sb, "\nAt: ");
-		vcc_ErrWhere(tl, tl->t);
+		vcc_ErrWhere2(tl, t, tl->t);
+		VSB_printf(tl->sb, "Variable cannot be unset.\n");
 		return;
 	}
-	vcc_AddUses(tl, tl->t, sym->u_methods, "cannot be unset.");
+	vcc_AddUses(tl, t, tl->t, sym->u_methods, "Cannot be unset");
 	Fb(tl, 1, "%s;\n", sym->uname);
-	vcc_NextToken(tl);
 }
 
 /*--------------------------------------------------------------------*/
@@ -401,8 +399,8 @@ vcc_ParseAction(struct vcc *tl)
 	for (atp = action_table; atp->name != NULL; atp++) {
 		if (vcc_IdIs(at, atp->name)) {
 			if (atp->bitmask != 0)
-				vcc_AddUses(tl, at, atp->bitmask,
-				    "not a valid action");
+				vcc_AddUses(tl, at, NULL, atp->bitmask,
+				    "Not a valid action");
 			atp->func(tl);
 			return (1);
 		}
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index c5a496e..cd96916 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -352,8 +352,8 @@ void VCC_XrefTable(struct vcc *);
 void vcc_AddCall(struct vcc *);
 void vcc_ProcAction(struct proc *p, unsigned action, struct token *t);
 int vcc_CheckAction(struct vcc *tl);
-void vcc_AddUses(struct vcc *tl, const struct token *t, unsigned mask,
-    const char *use);
+void vcc_AddUses(struct vcc *, const struct token *, const struct token *,
+     unsigned mask, const char *use);
 int vcc_CheckUses(struct vcc *tl);
 
 #define ERRCHK(tl)      do { if ((tl)->err) return; } while (0)
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 0919b7e..f1666e6 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -359,7 +359,7 @@ vcc_Eval_Var(struct vcc *tl, struct expr **e, const struct symbol *sym,
 
 	(void)fmt;
 	assert(sym->kind == SYM_VAR);
-	vcc_AddUses(tl, tl->t, sym->r_methods, "Not available");
+	vcc_AddUses(tl, tl->t, NULL, sym->r_methods, "Not available");
 	ERRCHK(tl);
 	*e = vcc_mk_expr(sym->fmt, "%s", sym->rname);
 	(*e)->constant = EXPR_VAR;
diff --git a/lib/libvcc/vcc_token.c b/lib/libvcc/vcc_token.c
index 856e550..894397d 100644
--- a/lib/libvcc/vcc_token.c
+++ b/lib/libvcc/vcc_token.c
@@ -104,7 +104,10 @@ vcc_icoord(struct vsb *vsb, const struct token *t, int tail)
 		} else
 			pos++;
 	}
-	VSB_printf(vsb, "('%s' Line %u Pos %u)", t->src->name, lin, pos + 1);
+	VSB_printf(vsb, "(");
+	if (tail < 2)
+		VSB_printf(vsb, "'%s' Line %u ", t->src->name, lin);
+	VSB_printf(vsb, "Pos %u)", pos + 1);
 }
 
 /*--------------------------------------------------------------------*/
@@ -200,7 +203,7 @@ vcc_ErrWhere2(struct vcc *tl, const struct token *t, const struct token *t2)
 	if (l1 == l2) {
 		vcc_icoord(tl->sb, t, 0);
 		VSB_cat(tl->sb, " -- ");
-		vcc_icoord(tl->sb, t2, 1);
+		vcc_icoord(tl->sb, t2, 2);
 		VSB_putc(tl->sb, '\n');
 		/* Two tokens on same line */
 		vcc_quoteline(tl, l1, t->src->e);
diff --git a/lib/libvcc/vcc_xref.c b/lib/libvcc/vcc_xref.c
index d6d5535..9ad4d67 100644
--- a/lib/libvcc/vcc_xref.c
+++ b/lib/libvcc/vcc_xref.c
@@ -47,13 +47,16 @@ struct proccall {
 	VTAILQ_ENTRY(proccall)	list;
 	struct symbol		*sym;
 	struct token		*t;
+	struct proc		*fm;
 };
 
 struct procuse {
 	VTAILQ_ENTRY(procuse)	list;
-	const struct token	*t;
+	const struct token	*t1;
+	const struct token	*t2;
 	unsigned		mask;
 	const char		*use;
+	struct proc		*fm;
 };
 
 /*--------------------------------------------------------------------
@@ -122,8 +125,8 @@ vcc_CheckReferences(struct vcc *tl)
  */
 
 void
-vcc_AddUses(struct vcc *tl, const struct token *t, unsigned mask,
-    const char *use)
+vcc_AddUses(struct vcc *tl, const struct token *t1, const struct token *t2,
+    unsigned mask, const char *use)
 {
 	struct procuse *pu;
 
@@ -131,9 +134,13 @@ vcc_AddUses(struct vcc *tl, const struct token *t, unsigned mask,
 		return;
 	pu = TlAlloc(tl, sizeof *pu);
 	assert(pu != NULL);
-	pu->t = t;
+	pu->t1 = t1;
+	pu->t2 = t2;
+	if (pu->t2 == NULL)
+		pu->t2 = VTAILQ_NEXT(t1, list);
 	pu->mask = mask;
 	pu->use = use;
+	pu->fm = tl->curproc;
 	VTAILQ_INSERT_TAIL(&tl->curproc->uses, pu, list);
 }
 
@@ -147,6 +154,7 @@ vcc_AddCall(struct vcc *tl)
 	pc->sym = VCC_SymbolTok(tl, SYM_SUB, 1);
 	AN(pc->sym);
 	pc->t = tl->t;
+	pc->fm = tl->curproc;
 	VTAILQ_INSERT_TAIL(&tl->curproc->calls, pc, list);
 }
 
@@ -286,19 +294,18 @@ vcc_CheckUseRecurse(struct vcc *tl, const struct proc *p,
 
 	pu = vcc_FindIllegalUse(p, m);
 	if (pu != NULL) {
-		VSB_printf(tl->sb,
-		    "'%.*s': %s from method '%.*s'.\n",
-		    PF(pu->t), pu->use, PF(p->name));
-		vcc_ErrWhere(tl, pu->t);
+		vcc_ErrWhere2(tl, pu->t1, pu->t2);
+		VSB_printf(tl->sb, "%s from method '%s'.\n",
+		    pu->use, m->name);
 		VSB_printf(tl->sb, "\n...in subroutine \"%.*s\"\n",
-		    PF(p->name));
+		    PF(pu->fm->name));
 		vcc_ErrWhere(tl, p->name);
 		return (1);
 	}
 	VTAILQ_FOREACH(pc, &p->calls, list) {
 		if (vcc_CheckUseRecurse(tl, pc->sym->proc, m)) {
-			VSB_printf(tl->sb, "\n...called from \"%s\"\n",
-			    pc->sym->name);
+			VSB_printf(tl->sb, "\n...called from \"%.*s\"\n",
+			    PF(pc->fm->name));
 			vcc_ErrWhere(tl, pc->t);
 			return (1);
 		}
@@ -318,10 +325,10 @@ vcc_checkuses(struct vcc *tl, const struct symbol *sym)
 		return;
 	pu = vcc_FindIllegalUse(p, p->method);
 	if (pu != NULL) {
-		VSB_printf(tl->sb, "'%.*s': %s in method '%.*s'.",
-		    PF(pu->t), pu->use, PF(p->name));
+		vcc_ErrWhere2(tl, pu->t1, pu->t2);
+		VSB_printf(tl->sb, "%s in method '%.*s'.",
+		    pu->use, PF(p->name));
 		VSB_cat(tl->sb, "\nAt: ");
-		vcc_ErrWhere(tl, pu->t);
 		return;
 	}
 	if (vcc_CheckUseRecurse(tl, p, p->method)) {

From fgsch at lodoss.net  Wed Jan 24 21:56:07 2018
From: fgsch at lodoss.net (Federico G. Schwindt)
Date: Wed, 24 Jan 2018 21:56:07 +0000 (UTC)
Subject: [master] 78e7155 Test the proxy related VSL tags
Message-ID: <20180124215607.2E350980E3@lists.varnish-cache.org>


commit 78e715538f44927427e31cc1f1dcd1bdf8f3ea4d
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Jan 24 21:54:15 2018 +0000

    Test the proxy related VSL tags

diff --git a/bin/varnishtest/tests/o00000.vtc b/bin/varnishtest/tests/o00000.vtc
index 5bd0a12..931238e 100644
--- a/bin/varnishtest/tests/o00000.vtc
+++ b/bin/varnishtest/tests/o00000.vtc
@@ -30,10 +30,26 @@ varnish v1 -proto "PROXY" -vcl+backend {
 	}
 } -start
 
+logexpect l1 -v v1 {
+	expect * 1002	ProxyGarbage	"PROXY1: Too few fields"
+	expect * 1003	ProxyGarbage	"PROXY1: Too many fields"
+	expect * 1004	ProxyGarbage	"PROXY1: Wrong TCP\\[46\\] field"
+	expect * 1005	ProxyGarbage	"PROXY1: Cannot resolve source address \\(nodename nor servname provided, or not known\\)"
+	expect * 1007	ProxyGarbage	"PROXY1: Cannot resolve destination address \\(nodename nor servname provided, or not known\\)"
+	expect * 1013	ProxyGarbage	"PROXY1: TCP4 got wrong protocol \\(30\\)"
+	expect * 1014	ProxyGarbage	"PROXY1: TCP6 got wrong protocol \\(2\\)"
+	expect * 1015	Proxy		"1 1.2.3.4 1234 5.6.7.8 5678"
+	expect * 1018	Proxy		"1 1:f::2 1234 5:a::8 5678"
+	expect * 1021	Proxy		"1 1:f::3 1234 5:a::8 5678"
+	expect * 1025	ProxyGarbage	"PROXY1: Too many fields"
+	expect * 1026	ProxyGarbage	"PROXY1: Too many fields"
+} -start
+
 client c1 {
 	send "XYZ\r\n"
 	expect_close
 } -run
+
 delay .1
 
 client c1 {
@@ -41,6 +57,7 @@ client c1 {
 	timeout 8
 	expect_close
 } -run
+
 delay .1
 
 client c1 {
@@ -48,6 +65,7 @@ client c1 {
 	timeout 8
 	expect_close
 } -run
+
 delay .1
 
 client c1 {
@@ -55,6 +73,7 @@ client c1 {
 	timeout 8
 	expect_close
 } -run
+
 delay .1
 
 client c1 {
@@ -62,6 +81,7 @@ client c1 {
 	timeout 8
 	expect_close
 } -run
+
 delay .1
 
 client c1 {
@@ -69,6 +89,7 @@ client c1 {
 	timeout 8
 	expect_close
 } -run
+
 delay .1
 
 client c1 {
@@ -76,6 +97,7 @@ client c1 {
 	timeout 8
 	expect_close
 } -run
+
 delay .1
 
 client c1 {
@@ -83,6 +105,7 @@ client c1 {
 	timeout 8
 	expect_close
 } -run
+
 delay .1
 
 client c1 {
@@ -90,6 +113,7 @@ client c1 {
 	timeout 8
 	expect_close
 } -run
+
 delay .1
 
 client c1 {
@@ -97,6 +121,7 @@ client c1 {
 	timeout 8
 	expect_close
 } -run
+
 delay .1
 
 client c1 {
@@ -104,6 +129,7 @@ client c1 {
 	timeout 8
 	expect_close
 } -run
+
 delay .1
 
 client c1 {
@@ -111,6 +137,7 @@ client c1 {
 	timeout 8
 	expect_close
 } -run
+
 delay .1
 
 client c1 {
@@ -118,6 +145,7 @@ client c1 {
 	timeout 8
 	expect_close
 } -run
+
 delay .1
 
 client c1 {
@@ -125,6 +153,7 @@ client c1 {
 	timeout 8
 	expect_close
 } -run
+
 delay .1
 
 client c1 {
@@ -132,12 +161,8 @@ client c1 {
 	timeout 8
 	expect_close
 } -run
-delay .1
 
-logexpect l1 -v v1 {
-	expect * *	ReqStart	"1.2.3.4 1234"
-	expect * *	ReqStart	"1:f::2 1234"
-} -start
+delay .1
 
 # Finally try something which works...
 client c1 -proxy1 "1.2.3.4:1234 5.6.7.8:5678" {
@@ -153,6 +178,7 @@ client c1 -proxy1 "1.2.3.4:1234 5.6.7.8:5678" {
 	expect resp.http.ri != "1.2.3.4"
 	expect resp.http.rp != "1234"
 } -run
+
 delay .1
 
 client c1 -proxy1 "[1:f::2]:1234 [5:a::8]:5678" {
@@ -168,9 +194,8 @@ client c1 -proxy1 "[1:f::2]:1234 [5:a::8]:5678" {
 	expect resp.http.ri != "1:f::2"
 	expect resp.http.rp != "1234"
 } -run
-delay .1
 
-logexpect l1 -wait
+delay .1
 
 # Try with appended request (See also: #1728)
 client c2 {
@@ -184,6 +209,7 @@ client c2 {
 	send "PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678\n"
 	expect_close
 } -run
+
 delay .1
 
 # Malformed, too long (106)
@@ -192,6 +218,7 @@ client c2 {
 	send "PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678                                                                     \r\n"
 	expect_close
 } -run
+
 delay .1
 
 # Malformed, too long (107)
@@ -200,6 +227,7 @@ client c2 {
 	send "PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678                                                                      \r\n"
 	expect_close
 } -run
+
 delay .1
 
 # Malformed, too long (108)
@@ -208,4 +236,7 @@ client c2 {
 	send "PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678                                                                       \r\n"
 	expect_close
 } -run
+
 delay .1
+
+logexpect l1 -wait
diff --git a/bin/varnishtest/tests/o00001.vtc b/bin/varnishtest/tests/o00001.vtc
index 02ff279..ae6e388 100644
--- a/bin/varnishtest/tests/o00001.vtc
+++ b/bin/varnishtest/tests/o00001.vtc
@@ -40,6 +40,17 @@ varnish v1 -proto "PROXY" -vcl+backend {
 	}
 } -start
 
+logexpect l1 -v v1 -g raw {
+	expect * 1000	Proxy		"2 local local local local"
+	expect * 1003	ProxyGarbage	"PROXY2: bad command \\(2\\)"
+	expect * 1004	ProxyGarbage	"PROXY2: Ignoring UNSPEC\\|UNSPEC addresses"
+	expect * 1006	ProxyGarbage	"PROXY2: Ignoring unsupported protocol \\(0x99\\)"
+	expect * 1008	ProxyGarbage	"PROXY2: Ignoring short IPv4 addresses \\(11\\)"
+	expect * 1010	ProxyGarbage	"PROXY2: Ignoring short IPv6 addresses \\(35\\)"
+	expect * 1012	Proxy		"2 1.2.3.4 2314 5.6.7.8 2828"
+	expect * 1015	Proxy		"2 102:304:506::d0e:f10 2314 8182:8384:8586::8d8e:8f80 2828"
+} -start
+
 client c1 {
 	# LOCAL command
 	sendhex "0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a"
@@ -51,6 +62,7 @@ client c1 {
 	expect resp.http.sp == "${v1_port}"
 	expect resp.http.ci == "${localhost}"
 } -run
+
 delay .1
 
 client c1 {
@@ -60,6 +72,7 @@ client c1 {
 	timeout 8
 	expect_close
 } -run
+
 delay .1
 
 client c1 {
@@ -73,6 +86,7 @@ client c1 {
 	expect resp.http.sp == "${v1_port}"
 	expect resp.http.ci == "${localhost}"
 } -run
+
 delay .1
 
 client c1 {
@@ -86,6 +100,7 @@ client c1 {
 	expect resp.http.sp == "${v1_port}"
 	expect resp.http.ci == "${localhost}"
 } -run
+
 delay .1
 
 client c1 {
@@ -100,6 +115,7 @@ client c1 {
 	expect resp.http.sp == "${v1_port}"
 	expect resp.http.ci == "${localhost}"
 } -run
+
 delay .1
 
 client c1 {
@@ -136,6 +152,7 @@ client c1 -proxy2 "1.2.3.4:2314 5.6.7.8:2828" {
 	expect resp.http.lp == "${v1_port}"
 	expect resp.http.ri != "1.2.3.4"
 } -run
+
 delay .1
 
 # good IPv6
@@ -155,4 +172,7 @@ client c1 \
 	expect resp.http.lp == "${v1_port}"
 	expect resp.http.ri != "102:304:506::d0e:f10"
 } -run
+
 delay .1
+
+logexpect l1 -wait

From phk at FreeBSD.org  Wed Jan 24 22:46:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 24 Jan 2018 22:46:06 +0000 (UTC)
Subject: [master] b6ab994 Don't test on exact resolver error messages,
 they vary per platform because the messages in the reference version
 were horrible.
Message-ID: <20180124224607.09494994B1@lists.varnish-cache.org>


commit b6ab994d678e8e308b926c3218a83f7ad417efcc
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 24 22:44:45 2018 +0000

    Don't test on exact resolver error messages, they vary per platform
    because the messages in the reference version were horrible.
    
    Also don't rely on protocol numbers, they vary per platform.

diff --git a/bin/varnishtest/tests/o00000.vtc b/bin/varnishtest/tests/o00000.vtc
index 931238e..3bb7cd5 100644
--- a/bin/varnishtest/tests/o00000.vtc
+++ b/bin/varnishtest/tests/o00000.vtc
@@ -34,10 +34,10 @@ logexpect l1 -v v1 {
 	expect * 1002	ProxyGarbage	"PROXY1: Too few fields"
 	expect * 1003	ProxyGarbage	"PROXY1: Too many fields"
 	expect * 1004	ProxyGarbage	"PROXY1: Wrong TCP\\[46\\] field"
-	expect * 1005	ProxyGarbage	"PROXY1: Cannot resolve source address \\(nodename nor servname provided, or not known\\)"
-	expect * 1007	ProxyGarbage	"PROXY1: Cannot resolve destination address \\(nodename nor servname provided, or not known\\)"
-	expect * 1013	ProxyGarbage	"PROXY1: TCP4 got wrong protocol \\(30\\)"
-	expect * 1014	ProxyGarbage	"PROXY1: TCP6 got wrong protocol \\(2\\)"
+	expect * 1005	ProxyGarbage	"PROXY1: Cannot resolve source address \\(.*\\)"
+	expect * 1007	ProxyGarbage	"PROXY1: Cannot resolve destination address \\(.*\\)"
+	expect * 1013	ProxyGarbage	"PROXY1: TCP4 got wrong protocol \\([0-9]*\\)"
+	expect * 1014	ProxyGarbage	"PROXY1: TCP6 got wrong protocol \\([0-9]*\\)"
 	expect * 1015	Proxy		"1 1.2.3.4 1234 5.6.7.8 5678"
 	expect * 1018	Proxy		"1 1:f::2 1234 5:a::8 5678"
 	expect * 1021	Proxy		"1 1:f::3 1234 5:a::8 5678"

From phk at FreeBSD.org  Mon Jan 29 08:36:14 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 29 Jan 2018 08:36:14 +0000 (UTC)
Subject: [master] 5166c76 Don't ioctl termios until we have pushed STREAMS
 modules on Solaris
Message-ID: <20180129083614.CCF49ACF80@lists.varnish-cache.org>


commit 5166c7617b7ff3eac2078415fc6ffa2681f4bf1b
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 29 08:35:21 2018 +0000

    Don't ioctl termios until we have pushed STREAMS modules on Solaris

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index 4aeabcd..bad1480 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -426,19 +426,15 @@ process_start(struct process *p)
 		slave = open(slavename, O_RDWR);
 		assert(slave == STDIN_FILENO);
 #ifdef __sun
-		(void)ioctl(STDIN_FILENO, TIOCSCTTY, NULL);
-#else
-		AZ(ioctl(STDIN_FILENO, TIOCSCTTY, NULL));
-#endif
-		AZ(close(STDOUT_FILENO));
-		assert(dup2(slave, STDOUT_FILENO) == STDOUT_FILENO);
-		VSUB_closefrom(STDERR_FILENO + 1);
-#ifdef __sun
 		if (ioctl(slave, I_PUSH, "ptem"))
 			vtc_log(p->vl, 4, "PUSH ptem: %s", strerror(errno));
 		if (ioctl(slave, I_PUSH, "ldterm"))
 			vtc_log(p->vl, 4, "PUSH ldterm: %s", strerror(errno));
 #endif
+		AZ(ioctl(STDIN_FILENO, TIOCSCTTY, NULL));
+		AZ(close(STDOUT_FILENO));
+		assert(dup2(slave, STDOUT_FILENO) == STDOUT_FILENO);
+		VSUB_closefrom(STDERR_FILENO + 1);
 		process_init_term(p, slave);
 
 		AZ(setenv("TERM", "ansi.sys", 1));

From phk at FreeBSD.org  Mon Jan 29 09:13:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 29 Jan 2018 09:13:08 +0000 (UTC)
Subject: [master] a1929ff Not that ioctl, the other ioctl...
Message-ID: <20180129091308.2D7BBADACC@lists.varnish-cache.org>


commit a1929ff4a677f08c2e5fa0bcd94144fd8e44484a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 29 09:12:37 2018 +0000

    Not that ioctl, the other ioctl...

diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c
index bad1480..c2e9c9a 100644
--- a/bin/varnishtest/vtc_process.c
+++ b/bin/varnishtest/vtc_process.c
@@ -430,8 +430,10 @@ process_start(struct process *p)
 			vtc_log(p->vl, 4, "PUSH ptem: %s", strerror(errno));
 		if (ioctl(slave, I_PUSH, "ldterm"))
 			vtc_log(p->vl, 4, "PUSH ldterm: %s", strerror(errno));
-#endif
+		(void)ioctl(STDIN_FILENO, TIOCSCTTY, NULL);
+#else
 		AZ(ioctl(STDIN_FILENO, TIOCSCTTY, NULL));
+#endif
 		AZ(close(STDOUT_FILENO));
 		assert(dup2(slave, STDOUT_FILENO) == STDOUT_FILENO);
 		VSUB_closefrom(STDERR_FILENO + 1);

From phk at FreeBSD.org  Mon Jan 29 23:21:09 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 29 Jan 2018 23:21:09 +0000 (UTC)
Subject: [master] 5ba1c33 Refactoring of symbol table
Message-ID: <20180129232109.E1C2E64169@lists.varnish-cache.org>


commit 5ba1c33ebecffaa20e53e55801cae5ab86a0a2b3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 29 20:46:50 2018 +0000

    Refactoring of symbol table

diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 8f61bad..1e99ee9 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -47,7 +47,8 @@ parse_call(struct vcc *tl)
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
 	vcc_AddCall(tl);
-	sym = vcc_AddRef(tl, SYM_SUB);
+	sym = VCC_SymbolGet(tl, SYM_SUB, NULL, XREF_REF);
+	AN(sym);
 	VCC_GlobalSymbol(sym, SUB, "VGC_function");
 	Fb(tl, 1, "%s(ctx);\n", sym->rname);
 	vcc_NextToken(tl);
@@ -95,7 +96,7 @@ parse_set(struct vcc *tl)
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
 	t = tl->t;
-	sym = VCC_SymbolGet(tl, SYM_VAR, "Unknown variable");
+	sym = VCC_SymbolGet(tl, SYM_VAR, "Unknown variable", XREF_NONE);
 	ERRCHK(tl);
 	vcc_NextToken(tl);
 	AN(sym);
@@ -149,7 +150,7 @@ parse_unset(struct vcc *tl)
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
 	t = tl->t;
-	sym = VCC_SymbolGet(tl, SYM_VAR, "Unknown variable");
+	sym = VCC_SymbolGet(tl, SYM_VAR, "Unknown variable", XREF_NONE);
 	ERRCHK(tl);
 	vcc_NextToken(tl);
 	AN(sym);
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index 55e8ba9..db4d11c 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -389,11 +389,11 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 			Fb(tl, 0, "\t.probe = %s,\n", p);
 			ERRCHK(tl);
 		} else if (vcc_IdIs(t_field, "probe") && tl->t->tok == ID) {
-			pb = VCC_SymbolGet(tl, SYM_PROBE, "Probe not found");
+			pb = VCC_SymbolGet(tl, SYM_PROBE, "Probe not found",
+			    XREF_REF);
 			ERRCHK(tl);
 			AN(pb);
 			Fb(tl, 0, "\t.probe = %s,\n", pb->rname);
-			(void)vcc_AddRef(tl, SYM_PROBE);
 			vcc_NextToken(tl);
 			SkipToken(tl, ';');
 		} else if (vcc_IdIs(t_field, "probe")) {
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index cd96916..f510e68 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -310,7 +310,9 @@ void vcc_stevedore(struct vcc *vcc, const char *stv_name);
 void VCC_PrintCName(struct vsb *vsb, const char *b, const char *e);
 struct symbol *VCC_Symbol(struct vcc *, struct symbol *,
     const char *, const char *, enum symkind, int);
-struct symbol *VCC_SymbolGet(struct vcc *, enum symkind, const char *);
+enum xref_e {XREF_NONE, XREF_DEF, XREF_REF};
+struct symbol *VCC_SymbolGet(struct vcc *, enum symkind, const char *,
+    enum xref_e);
 struct symbol *VCC_SymbolTok(struct vcc *, enum symkind, int);
 const char * VCC_SymKind(struct vcc *tl, const struct symbol *s);
 typedef void symwalk_f(struct vcc *tl, const struct symbol *s);
@@ -344,7 +346,6 @@ void vcc_ParseImport(struct vcc *tl);
 void vcc_ParseNew(struct vcc *tl);
 
 /* vcc_xref.c */
-struct symbol *vcc_AddDef(struct vcc *, enum symkind);
 struct symbol *vcc_AddRef(struct vcc *, enum symkind);
 int vcc_CheckReferences(struct vcc *tl);
 void VCC_XrefTable(struct vcc *);
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index f1666e6..810cb1e 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -966,7 +966,8 @@ cmp_acl(struct vcc *tl, struct expr **e, const struct cmps *cp)
 
 	vcc_NextToken(tl);
 	vcc_ExpectVid(tl, "ACL");
-	sym = vcc_AddRef(tl, SYM_ACL);
+	sym = VCC_SymbolGet(tl, SYM_ACL, NULL, XREF_REF);
+	AN(sym);
 	vcc_NextToken(tl);
 	VCC_GlobalSymbol(sym, ACL, ACL_SYMBOL_PREFIX);
 	bprintf(buf, "%sVRT_acl_match(ctx, %s, \v1)", cp->emit, sym->rname);
diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index a179d4a..569b3d8 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -215,7 +215,7 @@ vcc_ParseFunction(struct vcc *tl)
 	vcc_ExpectVid(tl, "function");
 	ERRCHK(tl);
 
-	sym = vcc_AddDef(tl, SYM_SUB);
+	sym = VCC_SymbolGet(tl, SYM_SUB, NULL, XREF_DEF);
 	AN(sym);
 	p = sym->proc;
 	if (p == NULL) {
@@ -247,7 +247,6 @@ vcc_ParseFunction(struct vcc *tl)
 		/* Add to VCL sub */
 		AN(p->method);
 		if (p->name == NULL) {
-			(void)vcc_AddDef(tl, SYM_SUB);
 			(void)vcc_AddRef(tl, SYM_SUB);
 			p->name = tl->t;
 		}
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index 17ac8c2..376e925 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -105,18 +105,34 @@ vcc_new_symbol(struct vcc *tl, const char *b, const char *e)
 }
 
 struct symbol *
-VCC_SymbolGet(struct vcc *tl, enum symkind kind, const char *err)
+VCC_SymbolGet(struct vcc *tl, enum symkind kind, const char *err,
+    enum xref_e xrf)
 {
 	struct symbol *sym;
 
-	sym = VCC_Symbol(tl, NULL, tl->t->b, tl->t->e, kind, 0);
-	if (sym != NULL && sym->kind == kind)
-		return (sym);
-	VSB_printf(tl->sb, "%s: ", err);
-	vcc_ErrToken(tl, tl->t);
-	VSB_cat(tl->sb, "\nAt: ");
-	vcc_ErrWhere(tl, tl->t);
-	return (NULL);
+	sym = VCC_Symbol(tl, NULL, tl->t->b, tl->t->e, kind, err == NULL);
+	if (sym == NULL || sym->kind != kind) {
+		VSB_printf(tl->sb, "%s: ", err);
+		vcc_ErrToken(tl, tl->t);
+		VSB_cat(tl->sb, "\nAt: ");
+		vcc_ErrWhere(tl, tl->t);
+		return (NULL);
+	}
+	switch (xrf) {
+	case XREF_DEF:
+		if (sym->def_b == NULL)
+			sym->def_b = tl->t;
+		sym->ndef++;
+		break;
+	case XREF_REF:
+		if (sym->ref_b == NULL)
+			sym->ref_b = tl->t;
+		sym->nref++;
+		break;
+	default:
+		break;
+	}
+	return (sym);
 }
 
 struct symbol *
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index b4d307d..2b51c4f 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -270,6 +270,7 @@ vcc_ParseNew(struct vcc *tl)
 {
 	struct symbol *sy1, *sy2, *sy3;
 	struct inifin *ifp;
+	struct token *t;
 	const char *p, *s_obj;
 	char buf1[128];
 	char buf2[128];
@@ -288,17 +289,18 @@ vcc_ParseNew(struct vcc *tl)
 	vcc_NextToken(tl);
 
 	ExpectErr(tl, ID);
-	sy2 = VCC_SymbolGet(tl, SYM_OBJECT, "Symbol not found");
+	t = tl->t;
+	sy2 = VCC_SymbolGet(tl, SYM_OBJECT, "Symbol not found", XREF_NONE);
 	ERRCHK(tl);
+	vcc_NextToken(tl);
 	AN(sy2);
 	if (sy2->extra == NULL) {
 		VSB_printf(tl->sb, "Constructor not found: ");
-		vcc_ErrToken(tl, tl->t);
+		vcc_ErrToken(tl, t);
 		VSB_printf(tl->sb, " at ");
-		vcc_ErrWhere(tl, tl->t);
+		vcc_ErrWhere(tl, t);
 		return;
 	}
-	vcc_NextToken(tl);
 
 	p = sy2->extra;
 	AN(p);
diff --git a/lib/libvcc/vcc_xref.c b/lib/libvcc/vcc_xref.c
index 9ad4d67..39031e8 100644
--- a/lib/libvcc/vcc_xref.c
+++ b/lib/libvcc/vcc_xref.c
@@ -76,19 +76,6 @@ vcc_AddRef(struct vcc *tl, enum symkind kind)
 	return (sym);
 }
 
-struct symbol *
-vcc_AddDef(struct vcc *tl, enum symkind kind)
-{
-	struct symbol *sym;
-
-	sym = VCC_SymbolTok(tl, kind, 1);
-	if (sym->def_b == NULL)
-		sym->def_b = tl->t;
-	AN(sym);
-	sym->ndef++;
-	return (sym);
-}
-
 /*--------------------------------------------------------------------*/
 
 static void

From phk at FreeBSD.org  Mon Jan 29 23:21:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 29 Jan 2018 23:21:10 +0000 (UTC)
Subject: [master] 7d31e25 AddRef no longer need to lookup
Message-ID: <20180129232110.098126416C@lists.varnish-cache.org>


commit 7d31e25a7c2144c42404a4d395a72b365678a15a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 29 21:17:31 2018 +0000

    AddRef no longer need to lookup

diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 1e99ee9..998dbc2 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -393,7 +393,7 @@ vcc_ParseAction(struct vcc *tl)
 {
 	struct token *at;
 	struct action_table *atp;
-	const struct symbol *sym;
+	struct symbol *sym;
 
 	at = tl->t;
 	assert(at->tok == ID);
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index f510e68..ec9026c 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -105,7 +105,7 @@ enum symkind {
 };
 
 typedef void sym_expr_t(struct vcc *tl, struct expr **,
-    const struct symbol *sym, vcc_type_t);
+    struct symbol *sym, vcc_type_t);
 typedef void sym_wildcard_t(struct vcc *, struct symbol *,
     const char *, const char *);
 
@@ -274,7 +274,7 @@ char *TlDup(struct vcc *tl, const char *s);
 
 /* vcc_expr.c */
 void vcc_Expr(struct vcc *tl, vcc_type_t typ);
-void vcc_Expr_Call(struct vcc *tl, const struct symbol *sym);
+void vcc_Expr_Call(struct vcc *tl, struct symbol *sym);
 void vcc_Expr_Init(struct vcc *tl);
 sym_expr_t vcc_Eval_Var;
 sym_expr_t vcc_Eval_Handle;
@@ -346,7 +346,7 @@ void vcc_ParseImport(struct vcc *tl);
 void vcc_ParseNew(struct vcc *tl);
 
 /* vcc_xref.c */
-struct symbol *vcc_AddRef(struct vcc *, enum symkind);
+void vcc_AddRef(struct vcc *, struct symbol *);
 int vcc_CheckReferences(struct vcc *tl);
 void VCC_XrefTable(struct vcc *);
 
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 810cb1e..68c2bc7 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -282,7 +282,7 @@ vcc_expr_tostring(struct vcc *tl, struct expr **e, vcc_type_t fmt)
  */
 
 static void v_matchproto_(sym_expr_t)
-vcc_Eval_Regsub(struct vcc *tl, struct expr **e, const struct symbol *sym,
+vcc_Eval_Regsub(struct vcc *tl, struct expr **e, struct symbol *sym,
     vcc_type_t fmt)
 {
 	struct expr *e2;
@@ -312,7 +312,7 @@ vcc_Eval_Regsub(struct vcc *tl, struct expr **e, const struct symbol *sym,
  */
 
 static void v_matchproto_(sym_expr_t)
-vcc_Eval_BoolConst(struct vcc *tl, struct expr **e, const struct symbol *sym,
+vcc_Eval_BoolConst(struct vcc *tl, struct expr **e, struct symbol *sym,
     vcc_type_t fmt)
 {
 
@@ -326,20 +326,19 @@ vcc_Eval_BoolConst(struct vcc *tl, struct expr **e, const struct symbol *sym,
  */
 
 void v_matchproto_(sym_expr_t)
-vcc_Eval_Handle(struct vcc *tl, struct expr **e, const struct symbol *sym,
+vcc_Eval_Handle(struct vcc *tl, struct expr **e, struct symbol *sym,
     vcc_type_t fmt)
 {
 
 	AN(sym->rname);
 
+	vcc_AddRef(tl, sym);
 	if (sym->fmt != STRING && fmt == STRINGS) {
-		(void)vcc_AddRef(tl, sym->kind);
 		*e = vcc_mk_expr(STRINGS, "\"%s\"", sym->name);
 		(*e)->nstr = 1;
 		(*e)->constant |= EXPR_CONST | EXPR_STR_CONST;
 	} else {
 		vcc_ExpectVid(tl, "handle");
-		(void)vcc_AddRef(tl, sym->kind);
 		*e = vcc_mk_expr(sym->fmt, "%s", sym->rname);
 		(*e)->constant = EXPR_VAR;
 		(*e)->nstr = 1;
@@ -353,7 +352,7 @@ vcc_Eval_Handle(struct vcc *tl, struct expr **e, const struct symbol *sym,
  */
 
 void v_matchproto_(sym_expr_t)
-vcc_Eval_Var(struct vcc *tl, struct expr **e, const struct symbol *sym,
+vcc_Eval_Var(struct vcc *tl, struct expr **e, struct symbol *sym,
     vcc_type_t fmt)
 {
 
@@ -598,7 +597,7 @@ vcc_Eval_Func(struct vcc *tl, const char *spec,
  */
 
 void v_matchproto_(sym_expr_t)
-vcc_Eval_SymFunc(struct vcc *tl, struct expr **e, const struct symbol *sym,
+vcc_Eval_SymFunc(struct vcc *tl, struct expr **e, struct symbol *sym,
     vcc_type_t fmt)
 {
 
@@ -630,7 +629,7 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 {
 	struct expr *e1, *e2;
 	const char *ip, *sign;
-	const struct symbol *sym;
+	struct symbol *sym;
 	enum symkind kind;
 	double d;
 	int i;
@@ -1264,7 +1263,7 @@ vcc_Expr(struct vcc *tl, vcc_type_t fmt)
  */
 
 void
-vcc_Expr_Call(struct vcc *tl, const struct symbol *sym)
+vcc_Expr_Call(struct vcc *tl, struct symbol *sym)
 {
 
 	struct expr *e;
diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index 569b3d8..666b06b 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -247,7 +247,7 @@ vcc_ParseFunction(struct vcc *tl)
 		/* Add to VCL sub */
 		AN(p->method);
 		if (p->name == NULL) {
-			(void)vcc_AddRef(tl, SYM_SUB);
+			vcc_AddRef(tl, sym);
 			p->name = tl->t;
 		}
 	}
diff --git a/lib/libvcc/vcc_xref.c b/lib/libvcc/vcc_xref.c
index 39031e8..1d6b0d7 100644
--- a/lib/libvcc/vcc_xref.c
+++ b/lib/libvcc/vcc_xref.c
@@ -63,17 +63,13 @@ struct procuse {
  * Keep track of definitions and references
  */
 
-struct symbol *
-vcc_AddRef(struct vcc *tl, enum symkind kind)
+void
+vcc_AddRef(struct vcc *tl, struct symbol *sym)
 {
-	struct symbol *sym;
 
-	sym = VCC_SymbolTok(tl, kind, 1);
 	if (sym->ref_b == NULL)
 		sym->ref_b = tl->t;
-	AN(sym);
 	sym->nref++;
-	return (sym);
 }
 
 /*--------------------------------------------------------------------*/

From phk at FreeBSD.org  Mon Jan 29 23:21:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 29 Jan 2018 23:21:10 +0000 (UTC)
Subject: [master] 0fc4e8c More symbol simplification
Message-ID: <20180129232110.2D24464174@lists.varnish-cache.org>


commit 0fc4e8c7021cf7169ca80db43657da12aec4f0cf
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 29 21:53:10 2018 +0000

    More symbol simplification

diff --git a/bin/varnishtest/tests/m00001.vtc b/bin/varnishtest/tests/m00001.vtc
index 5f7979f..a32a7de 100644
--- a/bin/varnishtest/tests/m00001.vtc
+++ b/bin/varnishtest/tests/m00001.vtc
@@ -52,7 +52,7 @@ varnish v1 -errvcl {Module std already imported.} {
 	import std;
 }
 
-varnish v1 -errvcl {Symbol not found: 'std' (expected type BOOL):} {
+varnish v1 -errvcl {Symbol type (vmod) can not be used in expression.} {
 	import std;
 
 	sub vcl_recv {
diff --git a/bin/varnishtest/tests/r00916.vtc b/bin/varnishtest/tests/r00916.vtc
index ac268ca..6c951fb 100644
--- a/bin/varnishtest/tests/r00916.vtc
+++ b/bin/varnishtest/tests/r00916.vtc
@@ -5,7 +5,7 @@ server s1 {
 	txresp -body "FOO"
 } -start
 
-varnish v1 -errvcl {Symbol not found: 's-1' (expected type BACKEND):} {
+varnish v1 -errvcl {Symbol not found: 's-1'} {
 	backend b { .host = "127.0.0.1"; }
 	sub s1 {
 	}
diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc
index 29d96bd..937fa83 100644
--- a/bin/varnishtest/tests/v00016.vtc
+++ b/bin/varnishtest/tests/v00016.vtc
@@ -41,7 +41,7 @@ varnish v1 -errvcl {Operator > not possible on BACKEND} {
 	sub vcl_recv { if (a > b) { } }
 }
 
-varnish v1 -errvcl {Symbol not found: 'req.foo' (expected type BOOL):} {
+varnish v1 -errvcl {Symbol not found: 'req.foo'} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_hash { if (req.foo != "bar") { } }
 }
@@ -66,7 +66,7 @@ varnish v1 -errvcl {Cannot be set in method 'vcl_pipe'.} {
 	}
 }
 
-varnish v1 -errvcl {Symbol not found: 'c' (expected type BACKEND):} {
+varnish v1 -errvcl {Symbol not found: 'c'} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_backend_response {
 		if (beresp.backend == c) {
diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc
index c9b6b6a..653c0a0 100644
--- a/bin/varnishtest/tests/v00018.vtc
+++ b/bin/varnishtest/tests/v00018.vtc
@@ -38,7 +38,7 @@ varnish v1 -errvcl {Expected ';' got 'if'} {
 	sub vcl_recv { set req.url = "foo" if "bar"; }
 }
 
-varnish v1 -errvcl {Symbol not found: 'req.foo' (expected type STRING):} {
+varnish v1 -errvcl {Symbol not found: 'req.foo'} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_hash { hash_data(req.foo); }
 }
@@ -63,7 +63,7 @@ varnish v1 -vcl {
 	sub vcl_backend_response { set beresp.do_gzip = false; }
 }
 
-varnish v1 -errvcl {Symbol not found: 'mu' (expected type BOOL):} {
+varnish v1 -errvcl {Symbol not found: 'mu'} {
 	backend b { .host = "127.0.0.1"; }
 	sub vcl_backend_response { set beresp.do_gzip = mu; }
 }
diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc
index 05f7c24..dbfa07a 100644
--- a/bin/varnishtest/tests/v00020.vtc
+++ b/bin/varnishtest/tests/v00020.vtc
@@ -31,7 +31,7 @@ varnish v1 -errvcl {Comparison of different types: INT '!=' STRING} {
 	}
 }
 
-varnish v1 -errvcl {Symbol not found: 'vcl_recv' (expected type STRING)} {
+varnish v1 -errvcl {Symbol type (sub) can not be used in expression.} {
 	sub vcl_recv {
 		set req.http.foo = vcl_recv;
 	}
@@ -262,7 +262,7 @@ varnish v1 -errvcl {INT * BLOB not possible.} {
 }
 
 # XXX: should spot nonexistent storage
-varnish v1 -errvcl {Symbol not found: 'storage.foo' (expected type STEVEDORE):} {
+varnish v1 -errvcl {Symbol not found: 'storage.foo'} {
 	sub vcl_backend_response {
 		set beresp.storage = storage.foo;
 	}
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index ec9026c..be01f2c 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -281,7 +281,6 @@ sym_expr_t vcc_Eval_Handle;
 sym_expr_t vcc_Eval_SymFunc;
 void vcc_Eval_Func(struct vcc *tl, const char *spec,
     const char *extra, const struct symbol *sym);
-enum symkind VCC_HandleKind(vcc_type_t fmt);
 void VCC_GlobalSymbol(struct symbol *, vcc_type_t fmt, const char *pfx);
 struct symbol *VCC_HandleSymbol(struct vcc *, vcc_type_t , const char *);
 
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 68c2bc7..4bf3b72 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -630,7 +630,6 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 	struct expr *e1, *e2;
 	const char *ip, *sign;
 	struct symbol *sym;
-	enum symkind kind;
 	double d;
 	int i;
 
@@ -649,24 +648,9 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 	}
 	switch (tl->t->tok) {
 	case ID:
-		/*
-		 * XXX: what if var and func/proc had same name ?
-		 * XXX: look for SYM_VAR first for consistency ?
-		 */
-		sym = NULL;
-		kind = VCC_HandleKind(fmt);
-		if (kind != SYM_NONE)
-			sym = VCC_SymbolTok(tl, kind, 0);
-		if (sym == NULL)
-			sym = VCC_SymbolTok(tl, SYM_NONE, 0);
-		if (sym == NULL || sym->eval == NULL) {
-			VSB_printf(tl->sb, "Symbol not found: ");
-			vcc_ErrToken(tl, tl->t);
-			VSB_printf(tl->sb, " (expected type %s):\n",
-			    vcc_utype(fmt));
-			vcc_ErrWhere(tl, tl->t);
-			return;
-		}
+		sym = VCC_SymbolGet(tl, SYM_NONE, "Symbol not found",
+		    XREF_NONE);
+		ERRCHK(tl);
 		AN(sym);
 		if (sym->kind == SYM_FUNC && sym->fmt == VOID) {
 			VSB_printf(tl->sb, "Function returns VOID:\n");
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index 376e925..042268d 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -39,7 +39,7 @@
 
 /*--------------------------------------------------------------------*/
 
-enum symkind
+static enum symkind
 VCC_HandleKind(vcc_type_t fmt)
 {
 	if (fmt == ACL)		return(SYM_ACL);
@@ -111,7 +111,7 @@ VCC_SymbolGet(struct vcc *tl, enum symkind kind, const char *err,
 	struct symbol *sym;
 
 	sym = VCC_Symbol(tl, NULL, tl->t->b, tl->t->e, kind, err == NULL);
-	if (sym == NULL || sym->kind != kind) {
+	if (sym == NULL || (kind != SYM_NONE && sym->kind != kind)) {
 		VSB_printf(tl->sb, "%s: ", err);
 		vcc_ErrToken(tl, tl->t);
 		VSB_cat(tl->sb, "\nAt: ");
diff --git a/lib/libvcc/vcc_xref.c b/lib/libvcc/vcc_xref.c
index 1d6b0d7..e6fc15b 100644
--- a/lib/libvcc/vcc_xref.c
+++ b/lib/libvcc/vcc_xref.c
@@ -67,6 +67,7 @@ void
 vcc_AddRef(struct vcc *tl, struct symbol *sym)
 {
 
+	(void)tl;
 	if (sym->ref_b == NULL)
 		sym->ref_b = tl->t;
 	sym->nref++;

From phk at FreeBSD.org  Mon Jan 29 23:21:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 29 Jan 2018 23:21:10 +0000 (UTC)
Subject: [master] a6234c3 Eliminate another double lookup
Message-ID: <20180129232110.4B19764178@lists.varnish-cache.org>


commit a6234c328aa6b4abf21ac76a579d7f6dafcbd1ec
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 29 22:04:05 2018 +0000

    Eliminate another double lookup

diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 998dbc2..5378746 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -46,9 +46,9 @@ parse_call(struct vcc *tl)
 
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
-	vcc_AddCall(tl);
 	sym = VCC_SymbolGet(tl, SYM_SUB, NULL, XREF_REF);
 	AN(sym);
+	vcc_AddCall(tl, sym);
 	VCC_GlobalSymbol(sym, SUB, "VGC_function");
 	Fb(tl, 1, "%s(ctx);\n", sym->rname);
 	vcc_NextToken(tl);
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index be01f2c..40d9f1b 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -349,7 +349,7 @@ void vcc_AddRef(struct vcc *, struct symbol *);
 int vcc_CheckReferences(struct vcc *tl);
 void VCC_XrefTable(struct vcc *);
 
-void vcc_AddCall(struct vcc *);
+void vcc_AddCall(struct vcc *, struct symbol *);
 void vcc_ProcAction(struct proc *p, unsigned action, struct token *t);
 int vcc_CheckAction(struct vcc *tl);
 void vcc_AddUses(struct vcc *, const struct token *, const struct token *,
diff --git a/lib/libvcc/vcc_xref.c b/lib/libvcc/vcc_xref.c
index e6fc15b..31655dc 100644
--- a/lib/libvcc/vcc_xref.c
+++ b/lib/libvcc/vcc_xref.c
@@ -129,14 +129,14 @@ vcc_AddUses(struct vcc *tl, const struct token *t1, const struct token *t2,
 }
 
 void
-vcc_AddCall(struct vcc *tl)
+vcc_AddCall(struct vcc *tl, struct symbol *sym)
 {
 	struct proccall *pc;
 
+	AN(sym);
 	pc = TlAlloc(tl, sizeof *pc);
 	assert(pc != NULL);
-	pc->sym = VCC_SymbolTok(tl, SYM_SUB, 1);
-	AN(pc->sym);
+	pc->sym = sym;
 	pc->t = tl->t;
 	pc->fm = tl->curproc;
 	VTAILQ_INSERT_TAIL(&tl->curproc->calls, pc, list);

From phk at FreeBSD.org  Mon Jan 29 23:21:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 29 Jan 2018 23:21:10 +0000 (UTC)
Subject: [master] 0e90b4c More symbol table simplification
Message-ID: <20180129232110.65DC064180@lists.varnish-cache.org>


commit 0e90b4c702d88001488a92d27892aea2767fb0aa
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 29 22:40:38 2018 +0000

    More symbol table simplification

diff --git a/bin/varnishtest/tests/m00008.vtc b/bin/varnishtest/tests/m00008.vtc
index 849ba4e..7026f57 100644
--- a/bin/varnishtest/tests/m00008.vtc
+++ b/bin/varnishtest/tests/m00008.vtc
@@ -6,6 +6,10 @@ varnish v1 -errvcl {Module debug conflicts with other symbol.} {
 	backend b { .host = "127.0.0.1"; }
 	sub debug {}
 	import debug;
+
+	sub vcl_recv {
+		call debug;
+	}
 }
 
 server s1 {
diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc
index 653c0a0..afba076 100644
--- a/bin/varnishtest/tests/v00018.vtc
+++ b/bin/varnishtest/tests/v00018.vtc
@@ -126,3 +126,9 @@ varnish v1 -errvcl {Not a valid action in method 'vcl_recv'} {
 		return (synth(503));
 	}
 }
+
+varnish v1 -errvcl {Not a VCL label:} {
+	sub vcl_recv {
+		return (vcl(vcl_recv));
+	}
+}
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 5378746..0472479 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -255,13 +255,9 @@ parse_return_vcl(struct vcc *tl)
 	ExpectErr(tl, '(');
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
-	sym = VCC_SymbolTok(tl, SYM_VCL, 0);
+	sym = VCC_SymbolGet(tl, SYM_VCL, "Not a VCL label", XREF_NONE);
 	ERRCHK(tl);
-	if (sym == NULL) {
-		VSB_printf(tl->sb, "Not a VCL label:\n");
-		vcc_ErrWhere(tl, tl->t);
-		return;
-	}
+	AN(sym);
 	if (sym->eval_priv == NULL) {
 		VSB_printf(tl->fi, "%s VCL %s */\n", VCC_INFO_PREFIX,
 		    sym->name);

From phk at FreeBSD.org  Tue Jan 30 00:13:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 30 Jan 2018 00:13:06 +0000 (UTC)
Subject: [master] adc4d89 Don't let default probes complicate symbol table
Message-ID: <20180130001306.8B4D7650D2@lists.varnish-cache.org>


commit adc4d89ef1a9a124154f4be495779dac9128d7d2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon Jan 29 23:39:42 2018 +0000

    Don't let default probes complicate symbol table

diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index db4d11c..30e816a 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -249,7 +249,6 @@ vcc_ParseProbeSpec(struct vcc *tl, const struct symbol *sym, char **name)
 void
 vcc_ParseProbe(struct vcc *tl)
 {
-	struct token *t_probe;
 	struct symbol *sym;
 	char *p;
 
@@ -257,15 +256,16 @@ vcc_ParseProbe(struct vcc *tl)
 
 	vcc_ExpectVid(tl, "backend probe");	/* ID: name */
 	ERRCHK(tl);
-	t_probe = tl->t;
-	sym = VCC_HandleSymbol(tl, PROBE, "vgc_probe");
-
-	ERRCHK(tl);
-	AN(sym);
-
-	vcc_ParseProbeSpec(tl, sym, &p);
-	if (vcc_IdIs(t_probe, "default"))
-		tl->default_probe = sym;
+	if (vcc_IdIs(tl->t, "default")) {
+		vcc_NextToken(tl);
+		vcc_ParseProbeSpec(tl, NULL, &p);
+		tl->default_probe = p;
+	} else {
+		sym = VCC_HandleSymbol(tl, PROBE, "vgc_probe");
+		ERRCHK(tl);
+		AN(sym);
+		vcc_ParseProbeSpec(tl, sym, &p);
+	}
 }
 
 /*--------------------------------------------------------------------
@@ -389,11 +389,21 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 			Fb(tl, 0, "\t.probe = %s,\n", p);
 			ERRCHK(tl);
 		} else if (vcc_IdIs(t_field, "probe") && tl->t->tok == ID) {
-			pb = VCC_SymbolGet(tl, SYM_PROBE, "Probe not found",
-			    XREF_REF);
-			ERRCHK(tl);
-			AN(pb);
-			Fb(tl, 0, "\t.probe = %s,\n", pb->rname);
+			if (vcc_IdIs(tl->t, "default")) {
+				if (tl->default_probe == NULL) {
+					VSB_printf(tl->sb,
+					    "No default probe defined\n");
+					vcc_ErrToken(tl, tl->t);
+					VSB_printf(tl->sb, " at\n");
+					vcc_ErrWhere(tl, tl->t);
+				}
+			} else {
+				pb = VCC_SymbolGet(tl, SYM_PROBE,
+				    "Probe not found", XREF_REF);
+				ERRCHK(tl);
+				AN(pb);
+				Fb(tl, 0, "\t.probe = %s,\n", pb->rname);
+			}
 			vcc_NextToken(tl);
 			SkipToken(tl, ';');
 		} else if (vcc_IdIs(t_field, "probe")) {
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 95e5b76..1dc52ae 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -418,8 +418,7 @@ EmitStruct(const struct vcc *tl)
 	Fc(tl, 0, "\t.default_director = &%s,\n",
 	   tl->default_director->rname);
 	if (tl->default_probe != NULL)
-		Fc(tl, 0, "\t.default_probe = %s,\n",
-		    tl->default_probe->rname);
+		Fc(tl, 0, "\t.default_probe = %s,\n", tl->default_probe);
 	Fc(tl, 0, "\t.ref = VGC_ref,\n");
 	Fc(tl, 0, "\t.nref = VGC_NREFS,\n");
 	Fc(tl, 0, "\t.nsrc = VGC_NSRCS,\n");
@@ -614,8 +613,6 @@ vcc_CompileSource(struct vcc *tl, struct source *sp)
 
 	/* Refcount the default director & probe*/
 	tl->default_director->nref++;
-	if (tl->default_probe != NULL)
-		tl->default_probe->nref++;
 
 	/* Check for orphans */
 	if (vcc_CheckReferences(tl))
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 40d9f1b..e401fec 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -220,7 +220,7 @@ struct vcc {
 
 	int			ndirector;
 	struct symbol		*default_director;
-	struct symbol		*default_probe;
+	char			*default_probe;
 
 	unsigned		unique;
 

From phk at FreeBSD.org  Tue Jan 30 00:13:06 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 30 Jan 2018 00:13:06 +0000 (UTC)
Subject: [master] a535c1a Make the name 'default' truly magic for both
 backends and probes.
Message-ID: <20180130001306.9FF3F650D5@lists.varnish-cache.org>


commit a535c1aef6f88bce1554f72b90b84dc0817acff9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 30 00:11:56 2018 +0000

    Make the name 'default' truly magic for both backends and probes.

diff --git a/bin/varnishtest/tests/v00038.vtc b/bin/varnishtest/tests/v00038.vtc
index 53ba0ea..9b8cfb0 100644
--- a/bin/varnishtest/tests/v00038.vtc
+++ b/bin/varnishtest/tests/v00038.vtc
@@ -92,3 +92,8 @@ varnish v1 -errvcl "Mandatory field 'host' missing." {
 		.port = "NONE";
 	}
 }
+varnish v1 -errvcl "No default probe defined" {
+	backend b1 {
+		.probe = default;
+	}
+}
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index 30e816a..64f73ac 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -35,6 +35,19 @@
 
 #include "vcc_compile.h"
 
+const char *
+vcc_default_probe(struct vcc *tl)
+{
+
+	if (tl->default_probe != NULL)
+		return (tl->default_probe);
+	VSB_printf(tl->sb, "No default probe defined\n");
+	vcc_ErrToken(tl, tl->t);
+	VSB_printf(tl->sb, " at\n");
+	vcc_ErrWhere(tl, tl->t);
+	return ("");
+}
+
 /*--------------------------------------------------------------------
  * Struct sockaddr is not really designed to be a compile time
  * initialized data structure, so we encode it as a byte-string
@@ -390,13 +403,7 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 			ERRCHK(tl);
 		} else if (vcc_IdIs(t_field, "probe") && tl->t->tok == ID) {
 			if (vcc_IdIs(tl->t, "default")) {
-				if (tl->default_probe == NULL) {
-					VSB_printf(tl->sb,
-					    "No default probe defined\n");
-					vcc_ErrToken(tl, tl->t);
-					VSB_printf(tl->sb, " at\n");
-					vcc_ErrWhere(tl, tl->t);
-				}
+				(void)vcc_default_probe(tl);
 			} else {
 				pb = VCC_SymbolGet(tl, SYM_PROBE,
 				    "Probe not found", XREF_REF);
@@ -467,6 +474,7 @@ vcc_ParseBackend(struct vcc *tl)
 {
 	struct token *t_first, *t_be;
 	struct symbol *sym;
+	const char *dn;
 
 	tl->ndirector++;
 	t_first = tl->t;
@@ -476,24 +484,27 @@ vcc_ParseBackend(struct vcc *tl)
 	ERRCHK(tl);
 
 	t_be = tl->t;
-	sym = VCC_HandleSymbol(tl, BACKEND, "vgc_backend");
-
-	ERRCHK(tl);
-
-	Fh(tl, 0, "\nstatic struct director *%s;\n", sym->rname);
-
-	vcc_ParseHostDef(tl, t_be, sym->rname);
-	ERRCHK(tl);
-
+	if (vcc_IdIs(tl->t, "default")) {
+		vcc_NextToken(tl);
+		dn = "vgc_backend_default";
+		tl->default_director = dn;
+	} else {
+		sym = VCC_HandleSymbol(tl, BACKEND, "vgc_backend");
+		ERRCHK(tl);
+		dn = sym->rname;
+		if (tl->default_director == NULL) {
+			tl->default_director = dn;
+			sym->nref++;
+		}
+	}
+	Fh(tl, 0, "\nstatic struct director *%s;\n", dn);
+	vcc_ParseHostDef(tl, t_be, dn);
 	if (tl->err) {
 		VSB_printf(tl->sb,
 		    "\nIn %.*s specification starting at:\n", PF(t_first));
 		vcc_ErrWhere(tl, t_first);
 		return;
 	}
-
-	if (tl->default_director == NULL || vcc_IdIs(t_be, "default"))
-		tl->default_director = sym;
 }
 
 void
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 1dc52ae..ce2d151 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -415,8 +415,7 @@ EmitStruct(const struct vcc *tl)
 	Fc(tl, 0, "\nconst struct VCL_conf VCL_conf = {\n");
 	Fc(tl, 0, "\t.magic = VCL_CONF_MAGIC,\n");
 	Fc(tl, 0, "\t.event_vcl = VGC_Event,\n");
-	Fc(tl, 0, "\t.default_director = &%s,\n",
-	   tl->default_director->rname);
+	Fc(tl, 0, "\t.default_director = &%s,\n", tl->default_director);
 	if (tl->default_probe != NULL)
 		Fc(tl, 0, "\t.default_probe = %s,\n", tl->default_probe);
 	Fc(tl, 0, "\t.ref = VGC_ref,\n");
@@ -611,9 +610,6 @@ vcc_CompileSource(struct vcc *tl, struct source *sp)
 		return (NULL);
 	}
 
-	/* Refcount the default director & probe*/
-	tl->default_director->nref++;
-
 	/* Check for orphans */
 	if (vcc_CheckReferences(tl))
 		return (NULL);
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index e401fec..8d5a75b 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -219,8 +219,8 @@ struct vcc {
 	int			nprobe;
 
 	int			ndirector;
-	struct symbol		*default_director;
-	char			*default_probe;
+	const char		*default_director;
+	const char		*default_probe;
 
 	unsigned		unique;
 
@@ -244,6 +244,7 @@ int vcc_ParseAction(struct vcc *tl);
 /* vcc_backend.c */
 struct fld_spec;
 
+const char *vcc_default_probe(struct vcc *);
 void vcc_Backend_Init(struct vcc *tl);
 void vcc_ParseProbe(struct vcc *tl);
 void vcc_ParseBackend(struct vcc *tl);
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 4bf3b72..0b1d7b6 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -648,6 +648,17 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 	}
 	switch (tl->t->tok) {
 	case ID:
+		if (vcc_IdIs(tl->t, "default") && fmt == PROBE) {
+			vcc_NextToken(tl);
+			*e = vcc_mk_expr(PROBE, "%s", vcc_default_probe(tl));
+			return;
+		}
+		if (vcc_IdIs(tl->t, "default") && fmt == BACKEND) {
+			vcc_NextToken(tl);
+			*e = vcc_mk_expr(BACKEND,
+			    "*(VCL_conf.default_director)");
+			return;
+		}
 		sym = VCC_SymbolGet(tl, SYM_NONE, "Symbol not found",
 		    XREF_NONE);
 		ERRCHK(tl);

From phk at FreeBSD.org  Tue Jan 30 11:01:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 30 Jan 2018 11:01:08 +0000 (UTC)
Subject: [master] 81f25bd Use the symbol table to dispatch action-parsers
Message-ID: <20180130110108.BC50FA604E@lists.varnish-cache.org>


commit 81f25bdfb987238c473130f442cf7b90abd4153d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 30 09:48:26 2018 +0000

    Use the symbol table to dispatch action-parsers

diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 0472479..d131def 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -52,6 +52,7 @@ parse_call(struct vcc *tl)
 	VCC_GlobalSymbol(sym, SUB, "VGC_function");
 	Fb(tl, 1, "%s(ctx);\n", sym->rname);
 	vcc_NextToken(tl);
+	SkipToken(tl, ';');
 }
 
 /*--------------------------------------------------------------------*/
@@ -136,6 +137,7 @@ parse_set(struct vcc *tl)
 	}
 	tl->indent -= INDENT;
 	Fb(tl, 1, ");\n");
+	SkipToken(tl, ';');
 }
 
 /*--------------------------------------------------------------------*/
@@ -161,6 +163,7 @@ parse_unset(struct vcc *tl)
 	}
 	vcc_AddUses(tl, t, tl->t, sym->u_methods, "Cannot be unset");
 	Fb(tl, 1, "%s;\n", sym->uname);
+	SkipToken(tl, ';');
 }
 
 /*--------------------------------------------------------------------*/
@@ -181,8 +184,8 @@ parse_ban(struct vcc *tl)
 	ERRCHK(tl);
 	Fb(tl, 1, ");\n");
 
-	ExpectErr(tl, ')');
-	vcc_NextToken(tl);
+	SkipToken(tl, ')');
+	SkipToken(tl, ';');
 }
 
 /*--------------------------------------------------------------------*/
@@ -198,6 +201,7 @@ parse_hash_data(struct vcc *tl)
 	ERRCHK(tl);
 	Fb(tl, 1, ");\n");
 	SkipToken(tl, ')');
+	SkipToken(tl, ';');
 }
 
 /*--------------------------------------------------------------------*/
@@ -212,8 +216,7 @@ parse_return_pass(struct vcc *tl)
 	tl->indent += INDENT;
 	vcc_Expr(tl, DURATION);
 	ERRCHK(tl);
-	ExpectErr(tl, ')');
-	vcc_NextToken(tl);
+	SkipToken(tl, ')');
 	Fb(tl, 1, ");\n");
 	tl->indent -= INDENT;
 }
@@ -238,8 +241,7 @@ parse_return_synth(struct vcc *tl)
 		Fb(tl, 1, "(const char*)0\n");
 	}
 	tl->indent -= INDENT;
-	ExpectErr(tl, ')');
-	vcc_NextToken(tl);
+	SkipToken(tl, ')');
 	Fb(tl, 1, ");\n");
 }
 
@@ -279,8 +281,7 @@ parse_return_vcl(struct vcc *tl)
 	Fb(tl, 1, "VRT_vcl_select(ctx, %s);\t/* %s */\n",
 	    (const char*)sym->eval_priv, sym->name);
 	vcc_NextToken(tl);
-	ExpectErr(tl, ')');
-	vcc_NextToken(tl);
+	SkipToken(tl, ')');
 }
 
 /*--------------------------------------------------------------------*/
@@ -294,11 +295,11 @@ parse_return(struct vcc *tl)
 	vcc_NextToken(tl);
 	AN(tl->curproc);
 	if (tl->t->tok == ';' && tl->curproc->method == NULL) {
+		SkipToken(tl, ';');
 		Fb(tl, 1, "return;\n");
 		return;
 	}
-	ExpectErr(tl, '(');
-	vcc_NextToken(tl);
+	SkipToken(tl, '(');
 	ExpectErr(tl, ID);
 
 	hand = VCL_RET_MAX;
@@ -337,8 +338,8 @@ parse_return(struct vcc *tl)
 	}
 	ERRCHK(tl);
 	Fb(tl, 1, "VRT_handling(ctx, VCL_RET_%s);\n", h);
-	ExpectErr(tl, ')');
-	vcc_NextToken(tl);
+	SkipToken(tl, ')');
+	SkipToken(tl, ';');
 }
 
 /*--------------------------------------------------------------------*/
@@ -357,55 +358,37 @@ parse_synthetic(struct vcc *tl)
 	ERRCHK(tl);
 	Fb(tl, 1, ");\n");
 
-	ExpectErr(tl, ')');
-	vcc_NextToken(tl);
-	ERRCHK(tl);
+	SkipToken(tl, ')');
+	SkipToken(tl, ';');
 }
 
 /*--------------------------------------------------------------------*/
 
-typedef void action_f(struct vcc *tl);
-
-static struct action_table {
-	const char		*name;
-	action_f		*func;
-	unsigned		bitmask;
-} action_table[] = {
-	/* Keep list sorted from here */
-	{ "ban",		parse_ban },
-	{ "call",		parse_call },
-	{ "hash_data",		parse_hash_data, VCL_MET_HASH },
-	{ "new",		vcc_ParseNew, VCL_MET_INIT},
-	{ "return",		parse_return },
-	{ "set",		parse_set },
-	{ "synthetic",		parse_synthetic,
-		VCL_MET_SYNTH | VCL_MET_BACKEND_ERROR },
-	{ "unset",		parse_unset },
-	{ NULL,			NULL }
-};
-
-int
-vcc_ParseAction(struct vcc *tl)
+// The pp[] trick is to make the length of #name visible to flexelint.
+#define ACT(name, func, mask)						\
+	do {								\
+		const char pp[] = #name;				\
+		sym = VCC_Symbol(tl, NULL, pp, NULL, SYM_ACTION, 1);	\
+		AN(sym);						\
+		sym->action = func;					\
+		sym->action_mask = (mask);				\
+	} while (0)
+
+void
+vcc_Action_Init(struct vcc *tl)
 {
-	struct token *at;
-	struct action_table *atp;
 	struct symbol *sym;
 
-	at = tl->t;
-	assert(at->tok == ID);
-	for (atp = action_table; atp->name != NULL; atp++) {
-		if (vcc_IdIs(at, atp->name)) {
-			if (atp->bitmask != 0)
-				vcc_AddUses(tl, at, NULL, atp->bitmask,
-				    "Not a valid action");
-			atp->func(tl);
-			return (1);
-		}
-	}
-	sym = VCC_SymbolTok(tl, SYM_NONE, 0);
-	if (sym != NULL && sym->kind == SYM_FUNC) {
-		vcc_Expr_Call(tl, sym);
-		return (1);
-	}
-	return (0);
+	ACT(ban,	parse_ban,	0);
+	ACT(call,	parse_call,	0);
+	ACT(hash_data,	parse_hash_data,
+		VCL_MET_HASH);
+	ACT(if,		vcc_ParseIf,	0);
+	ACT(new,	vcc_ParseNew,
+		VCL_MET_INIT);
+	ACT(return,	parse_return,	0);
+	ACT(set,	parse_set,	0);
+	ACT(synthetic,	parse_synthetic,
+		VCL_MET_SYNTH | VCL_MET_BACKEND_ERROR);
+	ACT(unset,	parse_unset,	0);
 }
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index ce2d151..7161b9d 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -563,6 +563,8 @@ vcc_CompileSource(struct vcc *tl, struct source *sp)
 
 	vcc_Expr_Init(tl);
 
+	vcc_Action_Init(tl);
+
 	vcc_Backend_Init(tl);
 
 	vcc_Var_Init(tl);
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 8d5a75b..f590a8e 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -109,6 +109,8 @@ typedef void sym_expr_t(struct vcc *tl, struct expr **,
 typedef void sym_wildcard_t(struct vcc *, struct symbol *,
     const char *, const char *);
 
+typedef void sym_act_f(struct vcc *tl);
+
 struct symbol {
 	unsigned			magic;
 #define SYMBOL_MAGIC			0x3368c9fb
@@ -123,6 +125,9 @@ struct symbol {
 	sym_wildcard_t			*wildcard;
 	enum symkind			kind;
 
+	sym_act_f			*action;
+	unsigned			action_mask;
+
 	const struct token		*def_b, *def_e, *ref_b;
 
 	vcc_type_t			fmt;
@@ -239,7 +244,7 @@ struct method {
 void vcc_ParseAcl(struct vcc *tl);
 
 /* vcc_action.c */
-int vcc_ParseAction(struct vcc *tl);
+void vcc_Action_Init(struct vcc *);
 
 /* vcc_backend.c */
 struct fld_spec;
@@ -290,6 +295,7 @@ void vcc_Var_Init(struct vcc *);
 
 /* vcc_parse.c */
 void vcc_Parse(struct vcc *tl);
+void vcc_ParseIf(struct vcc *tl);
 
 /* vcc_utils.c */
 const char *vcc_regexp(struct vcc *tl);
diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index 666b06b..1d6fcb1 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -79,8 +79,8 @@ vcc_Conditional(struct vcc *tl)
  *	null
  */
 
-static void
-vcc_IfStmt(struct vcc *tl)
+void
+vcc_ParseIf(struct vcc *tl)
 {
 
 	SkipToken(tl, ID);
@@ -141,7 +141,8 @@ vcc_IfStmt(struct vcc *tl)
 static void
 vcc_Compound(struct vcc *tl)
 {
-	int i;
+	struct symbol *sym;
+	struct token *t;
 
 	SkipToken(tl, '{');
 	Fb(tl, 1, "{\n");
@@ -149,6 +150,7 @@ vcc_Compound(struct vcc *tl)
 	C(tl, ";");
 	while (1) {
 		ERRCHK(tl);
+		t = tl->t;
 		switch (tl->t->tok) {
 		case '{':
 			vcc_Compound(tl);
@@ -176,16 +178,19 @@ vcc_Compound(struct vcc *tl)
 			tl->err = 1;
 			return;
 		case ID:
-			if (vcc_IdIs(tl->t, "if")) {
-				vcc_IfStmt(tl);
+			sym = VCC_SymbolTok(tl, SYM_NONE, 0);
+			if (sym != NULL && sym->action != NULL) {
+				if (sym->action_mask != 0)
+					vcc_AddUses(tl, t, NULL,
+					    sym->action_mask,
+					    "Not a valid action");
+				sym->action(tl);
+				break;
+			}
+			if (sym != NULL && sym->kind == SYM_FUNC) {
+				vcc_Expr_Call(tl, sym);
+				SkipToken(tl, ';');
 				break;
-			} else {
-				i = vcc_ParseAction(tl);
-				ERRCHK(tl);
-				if (i) {
-					SkipToken(tl, ';');
-					break;
-				}
 			}
 			/* FALLTHROUGH */
 		default:
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index 042268d..4078619 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -184,10 +184,8 @@ VCC_Symbol(struct vcc *tl, struct symbol *parent,
 			continue;
 		if (q < e)
 			break;
-		if (kind != SYM_NONE && sym->kind != SYM_NONE &&
-		    kind != sym->kind)
-			continue;
-		if (kind == SYM_NONE && sym->kind == kind)
+		if ((kind == SYM_NONE && kind == sym->kind) ||
+		    (kind != SYM_NONE && kind != sym->kind))
 			continue;
 		break;
 	}
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 2b51c4f..67d82f9 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -314,7 +314,7 @@ vcc_ParseNew(struct vcc *tl)
 	bprintf(buf1, ", &%s, \"%s\"", sy1->rname, sy1->name);
 	vcc_Eval_Func(tl, p, buf1, sy2);
 	ERRCHK(tl);
-	ExpectErr(tl, ';');
+	SkipToken(tl, ';');
 
 	while (p[0] != '\0' || p[1] != '\0' || p[2] != '\0')
 		p++;

From phk at FreeBSD.org  Tue Jan 30 11:01:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 30 Jan 2018 11:01:08 +0000 (UTC)
Subject: [master] 3129769 Try to boil symbol table interface down to only two
 functions.
Message-ID: <20180130110108.D19D9A6051@lists.varnish-cache.org>


commit 3129769768102bd50d770bdaa2722d1b99b862fe
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 30 10:15:58 2018 +0000

    Try to boil symbol table interface down to only two functions.

diff --git a/include/tbl/symbol_kind.h b/include/tbl/symbol_kind.h
index f71228b..5095ae3 100644
--- a/include/tbl/symbol_kind.h
+++ b/include/tbl/symbol_kind.h
@@ -31,6 +31,7 @@
 
 VCC_SYMB(NONE,		none)
 VCC_SYMB(ACL,		acl)
+VCC_SYMB(ACTION,	action)
 VCC_SYMB(BACKEND,	backend)
 VCC_SYMB(FUNC,		func)
 VCC_SYMB(INSTANCE,	instance)
diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py
index 3b6dda7..6ae254d 100755
--- a/lib/libvcc/generate.py
+++ b/lib/libvcc/generate.py
@@ -1263,13 +1263,13 @@ def one_var(nm, spec):
 	ctyp = vcltypes[spec.typ]
 
 	# fo.write("\t{ \"%s\", %s,\n" % (nm, spec.typ))
-	fo.write("\tsym = VCC_Symbol(tl, NULL, \"%s\", NULL," % nm)
+	fo.write("\tsym = VCC_MkSym(tl, \"%s\", " % nm)
 	if (spec.typ == "HEADER"):
-		fo.write(" SYM_NONE, 1);\n")
+		fo.write(" SYM_NONE);\n")
 		fo.write("\tAN(sym);\n");
 		fo.write("\tsym->wildcard = vcc_Var_Wildcard;\n")
 	else:
-		fo.write(" SYM_VAR, 1);\n")
+		fo.write(" SYM_VAR);\n")
 	fo.write("\tAN(sym);\n")
 	fo.write("\tsym->fmt = %s;\n" % spec.typ)
 	fo.write("\tsym->eval = vcc_Eval_Var;\n")
diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index d131def..3724cdc 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -46,7 +46,7 @@ parse_call(struct vcc *tl)
 
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
-	sym = VCC_SymbolGet(tl, SYM_SUB, NULL, XREF_REF);
+	sym = VCC_SymbolGet(tl, SYM_SUB, SYMTAB_CREATE, XREF_REF);
 	AN(sym);
 	vcc_AddCall(tl, sym);
 	VCC_GlobalSymbol(sym, SUB, "VGC_function");
@@ -368,7 +368,7 @@ parse_synthetic(struct vcc *tl)
 #define ACT(name, func, mask)						\
 	do {								\
 		const char pp[] = #name;				\
-		sym = VCC_Symbol(tl, NULL, pp, NULL, SYM_ACTION, 1);	\
+		sym = VCC_MkSym(tl, pp, SYM_ACTION);			\
 		AN(sym);						\
 		sym->action = func;					\
 		sym->action_mask = (mask);				\
diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c
index 7161b9d..3f9393b 100644
--- a/lib/libvcc/vcc_compile.c
+++ b/lib/libvcc/vcc_compile.c
@@ -723,8 +723,7 @@ VCC_New(void)
 	assert(tl->fh != NULL);
 
 	for (i = 1; i < VCL_MET_MAX; i++) {
-		sym = VCC_Symbol(tl, NULL,
-		    method_tab[i].name, NULL, SYM_SUB, 1);
+		sym = VCC_MkSym(tl, method_tab[i].name, SYM_SUB);
 		p = vcc_NewProc(tl, sym);
 		p->method = &method_tab[i];
 		VSB_printf(p->cname, "VGC_function_%s", p->method->name);
@@ -807,7 +806,7 @@ vcc_predef_vcl(struct vcc *vcc, const char *name)
 {
 	struct symbol *sym;
 
-	sym = VCC_Symbol(vcc, NULL, name, NULL, SYM_VCL, 1);
+	sym = VCC_MkSym(vcc, name, SYM_VCL);
 	AN(sym);
 	sym->fmt = VCL;
 	sym->r_methods = VCL_MET_RECV;
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index f590a8e..756a92f 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -314,12 +314,16 @@ void vcc_stevedore(struct vcc *vcc, const char *stv_name);
 
 /* vcc_symb.c */
 void VCC_PrintCName(struct vsb *vsb, const char *b, const char *e);
+struct symbol *VCC_MkSym(struct vcc *tl, const char *b, enum symkind kind);
 struct symbol *VCC_Symbol(struct vcc *, struct symbol *,
     const char *, const char *, enum symkind, int);
-enum xref_e {XREF_NONE, XREF_DEF, XREF_REF};
+extern const char XREF_NONE[];
+extern const char XREF_DEF[];
+extern const char XREF_REF[];
+extern const char SYMTAB_NOERR[];
+extern const char SYMTAB_CREATE[];
 struct symbol *VCC_SymbolGet(struct vcc *, enum symkind, const char *,
-    enum xref_e);
-struct symbol *VCC_SymbolTok(struct vcc *, enum symkind, int);
+    const char *);
 const char * VCC_SymKind(struct vcc *tl, const struct symbol *s);
 typedef void symwalk_f(struct vcc *tl, const struct symbol *s);
 void VCC_WalkSymbols(struct vcc *tl, symwalk_f *func, enum symkind kind);
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 0b1d7b6..2cf80e0 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -960,7 +960,7 @@ cmp_acl(struct vcc *tl, struct expr **e, const struct cmps *cp)
 
 	vcc_NextToken(tl);
 	vcc_ExpectVid(tl, "ACL");
-	sym = VCC_SymbolGet(tl, SYM_ACL, NULL, XREF_REF);
+	sym = VCC_SymbolGet(tl, SYM_ACL, SYMTAB_CREATE, XREF_REF);
 	AN(sym);
 	vcc_NextToken(tl);
 	VCC_GlobalSymbol(sym, ACL, ACL_SYMBOL_PREFIX);
@@ -1284,22 +1284,22 @@ vcc_Expr_Init(struct vcc *tl)
 {
 	struct symbol *sym;
 
-	sym = VCC_Symbol(tl, NULL, "regsub", NULL, SYM_FUNC, 1);
+	sym = VCC_MkSym(tl, "regsub", SYM_FUNC);
 	AN(sym);
 	sym->eval = vcc_Eval_Regsub;
 	sym->eval_priv = NULL;
 
-	sym = VCC_Symbol(tl, NULL, "regsuball", NULL, SYM_FUNC, 1);
+	sym = VCC_MkSym(tl, "regsuball", SYM_FUNC);
 	AN(sym);
 	sym->eval = vcc_Eval_Regsub;
 	sym->eval_priv = sym;
 
-	sym = VCC_Symbol(tl, NULL, "true", NULL, SYM_FUNC, 1);
+	sym = VCC_MkSym(tl, "true", SYM_FUNC);
 	AN(sym);
 	sym->eval = vcc_Eval_BoolConst;
 	sym->eval_priv = sym;
 
-	sym = VCC_Symbol(tl, NULL, "false", NULL, SYM_FUNC, 1);
+	sym = VCC_MkSym(tl, "false", SYM_FUNC);
 	AN(sym);
 	sym->eval = vcc_Eval_BoolConst;
 	sym->eval_priv = NULL;
diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index 1d6fcb1..228a1d4 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -178,7 +178,7 @@ vcc_Compound(struct vcc *tl)
 			tl->err = 1;
 			return;
 		case ID:
-			sym = VCC_SymbolTok(tl, SYM_NONE, 0);
+			sym = VCC_SymbolGet(tl, SYM_NONE, SYMTAB_NOERR, XREF_NONE);
 			if (sym != NULL && sym->action != NULL) {
 				if (sym->action_mask != 0)
 					vcc_AddUses(tl, t, NULL,
@@ -220,7 +220,7 @@ vcc_ParseFunction(struct vcc *tl)
 	vcc_ExpectVid(tl, "function");
 	ERRCHK(tl);
 
-	sym = VCC_SymbolGet(tl, SYM_SUB, NULL, XREF_DEF);
+	sym = VCC_SymbolGet(tl, SYM_SUB, SYMTAB_CREATE, XREF_DEF);
 	AN(sym);
 	p = sym->proc;
 	if (p == NULL) {
diff --git a/lib/libvcc/vcc_storage.c b/lib/libvcc/vcc_storage.c
index ad03b01..8378234 100644
--- a/lib/libvcc/vcc_storage.c
+++ b/lib/libvcc/vcc_storage.c
@@ -84,7 +84,7 @@ vcc_stevedore(struct vcc *vcc, const char *stv_name)
 
 	CHECK_OBJ_NOTNULL(vcc, VCC_MAGIC);
 	bprintf(buf, "storage.%s", stv_name);
-	sym = VCC_Symbol(vcc, NULL, buf, NULL, SYM_VAR, 1);
+	sym = VCC_MkSym(vcc, buf, SYM_VAR);
 	AN(sym);
 	sym->fmt = STEVEDORE;
 	sym->eval = vcc_Eval_Var;
@@ -94,7 +94,7 @@ vcc_stevedore(struct vcc *vcc, const char *stv_name)
 
 	for (sv = stvars; sv->name != NULL; sv++) {
 		bprintf(buf, "storage.%s.%s", stv_name, sv->name);
-		sym = VCC_Symbol(vcc, NULL, buf, NULL, SYM_VAR, 1);
+		sym = VCC_MkSym(vcc, buf, SYM_VAR);
 		AN(sym);
 		sym->fmt = sv->fmt;
 		sym->eval = vcc_Eval_Var;
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index 4078619..b083191 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -104,42 +104,47 @@ vcc_new_symbol(struct vcc *tl, const char *b, const char *e)
 	return (sym);
 }
 
+const char XREF_NONE[] = "xref_none";
+const char XREF_DEF[] = "xref_def";
+const char XREF_REF[] = "xref_ref";
+const char SYMTAB_NOERR[] = "sym_noerror";
+const char SYMTAB_CREATE[] = "sym_create";
+
 struct symbol *
-VCC_SymbolGet(struct vcc *tl, enum symkind kind, const char *err,
-    enum xref_e xrf)
+VCC_SymbolGet(struct vcc *tl, enum symkind kind, const char *e, const char *x)
 {
 	struct symbol *sym;
 
-	sym = VCC_Symbol(tl, NULL, tl->t->b, tl->t->e, kind, err == NULL);
+	AN(e);
+	sym = VCC_Symbol(tl, NULL, tl->t->b, tl->t->e, kind,
+	    e == SYMTAB_CREATE ? 1 : 0);
+	if (sym == NULL && e == SYMTAB_NOERR)
+		return (sym);
 	if (sym == NULL || (kind != SYM_NONE && sym->kind != kind)) {
-		VSB_printf(tl->sb, "%s: ", err);
+		VSB_printf(tl->sb, "%s: ", e);
 		vcc_ErrToken(tl, tl->t);
 		VSB_cat(tl->sb, "\nAt: ");
 		vcc_ErrWhere(tl, tl->t);
 		return (NULL);
 	}
-	switch (xrf) {
-	case XREF_DEF:
+	if (x == XREF_DEF) {
 		if (sym->def_b == NULL)
 			sym->def_b = tl->t;
 		sym->ndef++;
-		break;
-	case XREF_REF:
+	} else if (x == XREF_REF) {
 		if (sym->ref_b == NULL)
 			sym->ref_b = tl->t;
 		sym->nref++;
-		break;
-	default:
-		break;
+	} else {
+		assert (x == XREF_NONE);
 	}
 	return (sym);
 }
 
 struct symbol *
-VCC_SymbolTok(struct vcc *tl, enum symkind kind, int create)
+VCC_MkSym(struct vcc *tl, const char *b, enum symkind kind)
 {
-
-	return (VCC_Symbol(tl, NULL, tl->t->b, tl->t->e, kind, create));
+	return (VCC_Symbol(tl, NULL, b, NULL, kind, 1));
 }
 
 struct symbol *
@@ -273,7 +278,7 @@ VCC_HandleSymbol(struct vcc *tl, vcc_type_t fmt, const char *pfx)
 	kind = VCC_HandleKind(fmt);
 	assert(kind != SYM_NONE);
 
-	sym = VCC_SymbolTok(tl, SYM_NONE, 0);
+	sym = VCC_SymbolGet(tl, SYM_NONE, SYMTAB_NOERR, XREF_NONE);
 	if (sym != NULL && sym->def_b != NULL && kind == sym->kind) {
 		p = VCC_SymKind(tl, sym);
 		VSB_printf(tl->sb, "%c%s '%.*s' redefined.\n",
@@ -298,7 +303,7 @@ VCC_HandleSymbol(struct vcc *tl, vcc_type_t fmt, const char *pfx)
 		return (sym);
 	}
 	if (sym == NULL)
-		sym = VCC_SymbolTok(tl, kind, 1);
+		sym = VCC_SymbolGet(tl, kind, SYMTAB_CREATE, XREF_NONE);
 	AN(sym);
 	AZ(sym->ndef);
 	VCC_GlobalSymbol(sym, fmt, pfx);
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 67d82f9..239aa88 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -75,7 +75,7 @@ vcc_ParseImport(struct vcc *tl)
 
 	ExpectErr(tl, ID);
 	mod = tl->t;
-	osym = VCC_SymbolTok(tl, SYM_NONE, 0);
+	osym = VCC_SymbolGet(tl, SYM_NONE, SYMTAB_NOERR, XREF_NONE);
 	vcc_NextToken(tl);
 
 	if (osym != NULL && osym->kind != SYM_VMOD) {
@@ -94,7 +94,7 @@ vcc_ParseImport(struct vcc *tl)
 	}
 
 	bprintf(fn, "%.*s", PF(mod));
-	msym = VCC_Symbol(tl, NULL, fn, NULL, SYM_VMOD, 1);
+	msym = VCC_MkSym(tl, fn, SYM_VMOD);
 	ERRCHK(tl);
 	AN(msym);
 	msym->def_b = t1;
@@ -223,7 +223,7 @@ vcc_ParseImport(struct vcc *tl)
 		p = *spec;
 		if (!strcmp(p, "$OBJ")) {
 			p += strlen(p) + 1;
-			sym = VCC_Symbol(tl, NULL, p, NULL, SYM_OBJECT, 1);
+			sym = VCC_MkSym(tl, p, SYM_OBJECT);
 			XXXAN(sym);
 			sym->extra = p;
 			sym->vmod = msym->name;
@@ -242,7 +242,7 @@ vcc_ParseImport(struct vcc *tl)
 			    p, PF(mod));
 		} else if (!strcmp(p, "$FUNC")) {
 			p += strlen(p) + 1;
-			sym = VCC_Symbol(tl, NULL, p, NULL, SYM_FUNC, 1);
+			sym = VCC_MkSym(tl, p, SYM_FUNC);
 			ERRCHK(tl);
 			AN(sym);
 			sym->vmod = msym->name;
@@ -333,7 +333,7 @@ vcc_ParseNew(struct vcc *tl)
 	while (*p != '\0') {
 		p += strlen(s_obj);
 		bprintf(buf2, "%s%s", sy1->name, p);
-		sy3 = VCC_Symbol(tl, NULL, buf2, NULL, SYM_FUNC, 1);
+		sy3 = VCC_MkSym(tl, buf2, SYM_FUNC);
 		AN(sy3);
 		sy3->eval = vcc_Eval_SymFunc;
 		p += strlen(p) + 1;

From phk at FreeBSD.org  Tue Jan 30 11:01:08 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 30 Jan 2018 11:01:08 +0000 (UTC)
Subject: [master] 2c8c2d9 Simplify the wildcard symbol-lookups
Message-ID: <20180130110108.EC4A5A6054@lists.varnish-cache.org>


commit 2c8c2d981b34242e95f1075cd05f28c182e0d19d
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue Jan 30 11:00:18 2018 +0000

    Simplify the wildcard symbol-lookups

diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 756a92f..84e849e 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -106,8 +106,7 @@ enum symkind {
 
 typedef void sym_expr_t(struct vcc *tl, struct expr **,
     struct symbol *sym, vcc_type_t);
-typedef void sym_wildcard_t(struct vcc *, struct symbol *,
-    const char *, const char *);
+typedef void sym_wildcard_t(struct vcc *, struct symbol *, struct symbol *);
 
 typedef void sym_act_f(struct vcc *tl);
 
@@ -315,8 +314,6 @@ void vcc_stevedore(struct vcc *vcc, const char *stv_name);
 /* vcc_symb.c */
 void VCC_PrintCName(struct vsb *vsb, const char *b, const char *e);
 struct symbol *VCC_MkSym(struct vcc *tl, const char *b, enum symkind kind);
-struct symbol *VCC_Symbol(struct vcc *, struct symbol *,
-    const char *, const char *, enum symkind, int);
 extern const char XREF_NONE[];
 extern const char XREF_DEF[];
 extern const char XREF_REF[];
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index b083191..0e58c4c 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -104,50 +104,7 @@ vcc_new_symbol(struct vcc *tl, const char *b, const char *e)
 	return (sym);
 }
 
-const char XREF_NONE[] = "xref_none";
-const char XREF_DEF[] = "xref_def";
-const char XREF_REF[] = "xref_ref";
-const char SYMTAB_NOERR[] = "sym_noerror";
-const char SYMTAB_CREATE[] = "sym_create";
-
-struct symbol *
-VCC_SymbolGet(struct vcc *tl, enum symkind kind, const char *e, const char *x)
-{
-	struct symbol *sym;
-
-	AN(e);
-	sym = VCC_Symbol(tl, NULL, tl->t->b, tl->t->e, kind,
-	    e == SYMTAB_CREATE ? 1 : 0);
-	if (sym == NULL && e == SYMTAB_NOERR)
-		return (sym);
-	if (sym == NULL || (kind != SYM_NONE && sym->kind != kind)) {
-		VSB_printf(tl->sb, "%s: ", e);
-		vcc_ErrToken(tl, tl->t);
-		VSB_cat(tl->sb, "\nAt: ");
-		vcc_ErrWhere(tl, tl->t);
-		return (NULL);
-	}
-	if (x == XREF_DEF) {
-		if (sym->def_b == NULL)
-			sym->def_b = tl->t;
-		sym->ndef++;
-	} else if (x == XREF_REF) {
-		if (sym->ref_b == NULL)
-			sym->ref_b = tl->t;
-		sym->nref++;
-	} else {
-		assert (x == XREF_NONE);
-	}
-	return (sym);
-}
-
-struct symbol *
-VCC_MkSym(struct vcc *tl, const char *b, enum symkind kind)
-{
-	return (VCC_Symbol(tl, NULL, b, NULL, kind, 1));
-}
-
-struct symbol *
+static struct symbol *
 VCC_Symbol(struct vcc *tl, struct symbol *parent,
     const char *b, const char *e, enum symkind kind, int create)
 {
@@ -196,9 +153,20 @@ VCC_Symbol(struct vcc *tl, struct symbol *parent,
 	}
 	if (sym == NULL && create == 0 && parent->wildcard != NULL) {
 		AN(parent->wildcard);
-		parent->wildcard(tl, parent, b, e);
+		sym2 = vcc_new_symbol(tl, b, e);
+		sym2->parent = parent;
+		parent->wildcard(tl, parent, sym2);
 		if (tl->err)
 			return (NULL);
+		VTAILQ_FOREACH(sym, &parent->children, list) {
+			i = strncasecmp(sym->name, b, l);
+			if (i > 0 || (i == 0 && l < sym->nlen))
+				break;
+		}
+		if (sym == NULL)
+			VTAILQ_INSERT_TAIL(&parent->children, sym2, list);
+		else
+			VTAILQ_INSERT_BEFORE(sym, sym2, list);
 		return (VCC_Symbol(tl, parent, b, e, kind, -1));
 	}
 	if (sym == NULL && create < 1)
@@ -219,6 +187,50 @@ VCC_Symbol(struct vcc *tl, struct symbol *parent,
 	return (VCC_Symbol(tl, sym, ++q, e, kind, create));
 }
 
+const char XREF_NONE[] = "xref_none";
+const char XREF_DEF[] = "xref_def";
+const char XREF_REF[] = "xref_ref";
+const char SYMTAB_NOERR[] = "sym_noerror";
+const char SYMTAB_CREATE[] = "sym_create";
+
+struct symbol *
+VCC_SymbolGet(struct vcc *tl, enum symkind kind, const char *e, const char *x)
+{
+	struct symbol *sym;
+
+	AN(e);
+	sym = VCC_Symbol(tl, NULL, tl->t->b, tl->t->e, kind,
+	    e == SYMTAB_CREATE ? 1 : 0);
+	if (sym == NULL && e == SYMTAB_NOERR)
+		return (sym);
+	if (sym == NULL || (kind != SYM_NONE && sym->kind != kind)) {
+		VSB_printf(tl->sb, "%s: ", e);
+		vcc_ErrToken(tl, tl->t);
+		VSB_cat(tl->sb, "\nAt: ");
+		vcc_ErrWhere(tl, tl->t);
+		return (NULL);
+	}
+	if (x == XREF_DEF) {
+		if (sym->def_b == NULL)
+			sym->def_b = tl->t;
+		sym->ndef++;
+	} else if (x == XREF_REF) {
+		if (sym->ref_b == NULL)
+			sym->ref_b = tl->t;
+		sym->nref++;
+	} else {
+		assert (x == XREF_NONE);
+	}
+	return (sym);
+}
+
+struct symbol *
+VCC_MkSym(struct vcc *tl, const char *b, enum symkind kind)
+{
+	return (VCC_Symbol(tl, NULL, b, NULL, kind, 1));
+}
+
+
 static void
 vcc_walksymbols(struct vcc *tl, const struct symbol *root,
     symwalk_f *func, enum symkind kind)
diff --git a/lib/libvcc/vcc_var.c b/lib/libvcc/vcc_var.c
index 0c9af8c..ce09c4b 100644
--- a/lib/libvcc/vcc_var.c
+++ b/lib/libvcc/vcc_var.c
@@ -37,24 +37,21 @@
 /*--------------------------------------------------------------------*/
 
 void v_matchproto_(sym_wildcard_t)
-vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent,
-    const char *b, const char *e)
+vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent, struct symbol *sym)
 {
-	struct symbol *sym;
 	struct vsb *vsb;
-	unsigned len;
 
 	assert(parent->fmt == HEADER);
 
-	if (b + 127 <= e) {
-		VSB_printf(tl->sb, "HTTP header (%.20s..) is too long.\n", b);
+	if (sym->nlen >= 127) {
+		VSB_printf(tl->sb, "HTTP header (%.20s..) is too long.\n",
+		    sym->name);
 		VSB_cat(tl->sb, "\nAt: ");
 		vcc_ErrWhere(tl, tl->t);
-		return;
 	}
 
-	sym = VCC_Symbol(tl, parent, b, e, SYM_VAR, 1);
 	AN(sym);
+	sym->kind = SYM_VAR;
 	sym->fmt = parent->fmt;
 	sym->eval = vcc_Eval_Var;
 	sym->r_methods = parent->r_methods;
@@ -65,14 +62,13 @@ vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent,
 	vsb = VSB_new_auto();
 	AN(vsb);
 	VSB_printf(vsb, "&VGC_%s_", parent->rname);
-	VCC_PrintCName(vsb, b, e);
+	VCC_PrintCName(vsb, sym->name, NULL);
 	AZ(VSB_finish(vsb));
 
 	/* Create the static identifier */
-	len = (unsigned)(e - b);
 	Fh(tl, 0, "static const struct gethdr_s %s =\n", VSB_data(vsb) + 1);
-	Fh(tl, 0, "    { %s, \"\\%03o%.*s:\"};\n",
-	    parent->rname, len + 1, len, b);
+	Fh(tl, 0, "    { %s, \"\\%03o%s:\"};\n",
+	    parent->rname, sym->nlen + 1, sym->name);
 
 	/* Create the symbol r/l values */
 	sym->rname = TlDup(tl, VSB_data(vsb));

From dridi at varnish-software.com  Tue Jan 30 13:20:08 2018
From: dridi at varnish-software.com (Dridi Boukelmoune)
Date: Tue, 30 Jan 2018 13:20:08 +0000 (UTC)
Subject: [master] 59a2525 Code style OCD
Message-ID: <20180130132008.0DD15A909E@lists.varnish-cache.org>


commit 59a2525b70deb514955787f0632ddb93f0455b0f
Author: Dridi Boukelmoune <dridi at varnish-software.com>
Date:   Tue Jan 30 11:07:55 2018 +0100

    Code style OCD

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index fd70cb2..4d31680 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -179,7 +179,7 @@ vbe_dir_finish(const struct director *d, struct worker *wrk,
 	bo->htc->priv = NULL;
 	if (vtp->state != VTP_STATE_USED)
 		assert(bo->htc->doclose == SC_TX_PIPE ||
-			bo->htc->doclose == SC_RX_TIMEOUT);
+		    bo->htc->doclose == SC_RX_TIMEOUT);
 	if (bo->htc->doclose != SC_NULL || bp->proxy_header != 0) {
 		VSLb(bo->vsl, SLT_BackendClose, "%d %s", vtp->fd,
 		    bp->director->display_name);
diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h
index d9d3c54..f91cac6 100644
--- a/bin/varnishd/cache/cache_tcp_pool.h
+++ b/bin/varnishd/cache/cache_tcp_pool.h
@@ -53,10 +53,12 @@ struct vtp {
  * Prototypes
  */
 
-struct tcp_pool *VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, const void *id);
+struct tcp_pool *VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6,
+    const void *id);
 	/*
-	 * Get a reference to a TCP pool.  Either ip4 or ip6 arg must be non-NULL.
-	 * If recycling is to be used, the id pointer distinguishes the pool per protocol.
+	 * Get a reference to a TCP pool.  Either ip4 or ip6 arg must be
+	 * non-NULL. If recycling is to be used, the id pointer distinguishes
+	 * the pool per protocol.
 	 */
 
 void VTP_AddRef(struct tcp_pool *);

From dridi at varnish-software.com  Tue Jan 30 13:20:08 2018
From: dridi at varnish-software.com (Dridi Boukelmoune)
Date: Tue, 30 Jan 2018 13:20:08 +0000 (UTC)
Subject: [master] 8b70202 Missing matchproto
Message-ID: <20180130132008.2691BA90A1@lists.varnish-cache.org>


commit 8b7020270a6bb14ed6931d93936c77e75513b1c5
Author: Dridi Boukelmoune <dridi at varnish-software.com>
Date:   Tue Jan 30 11:53:40 2018 +0100

    Missing matchproto

diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c
index b362cec..a6c945d 100644
--- a/bin/varnishd/cache/cache_backend_probe.c
+++ b/bin/varnishd/cache/cache_backend_probe.c
@@ -415,7 +415,7 @@ vbp_task(struct worker *wrk, void *priv)
 /*--------------------------------------------------------------------
  */
 
-static void * v_matchproto_()
+static void * v_matchproto_(bgthread_t)
 vbp_thread(struct worker *wrk, void *priv)
 {
 	double now, nxt;

From phk at FreeBSD.org  Wed Jan 31 10:50:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 31 Jan 2018 10:50:10 +0000 (UTC)
Subject: [master] 63fb2ec Polish an error message
Message-ID: <20180131105010.8F110943F1@lists.varnish-cache.org>


commit 63fb2ecb5231b7123153a379d292722ee99ea3fb
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 31 07:51:01 2018 +0000

    Polish an error message

diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 3724cdc..9c77516 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -135,6 +135,7 @@ parse_set(struct vcc *tl)
 	} else {
 		vcc_Expr(tl, fmt);
 	}
+	ERRCHK(tl);
 	tl->indent -= INDENT;
 	Fb(tl, 1, ");\n");
 	SkipToken(tl, ';');
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 2cf80e0..e4d2d20 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -692,6 +692,10 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 		    "Symbol type (%s) can not be used in expression.\n",
 		    VCC_SymKind(tl, sym));
 		vcc_ErrWhere(tl, tl->t);
+		if (sym->def_b != NULL) {
+			VSB_printf(tl->sb, "That symbol was defined here:\n");
+			vcc_ErrWhere(tl, sym->def_b);
+		}
 		return;
 	case CSTR:
 		assert(fmt != VOID);

From phk at FreeBSD.org  Wed Jan 31 10:50:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 31 Jan 2018 10:50:10 +0000 (UTC)
Subject: [master] 1a5ca1e Dispatch function calls through sym->action
Message-ID: <20180131105010.A1B87943F3@lists.varnish-cache.org>


commit 1a5ca1ef9c82e5e44f9c9013416831fa7440303f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 31 08:37:53 2018 +0000

    Dispatch function calls through sym->action

diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 9c77516..61d4264 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -39,10 +39,9 @@
 
 /*--------------------------------------------------------------------*/
 
-static void
-parse_call(struct vcc *tl)
+static void v_matchproto_(sym_act_f)
+parse_call(struct vcc *tl, struct symbol *sym)
 {
-	struct symbol *sym;
 
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
@@ -86,10 +85,9 @@ static const struct arith {
 
 /*--------------------------------------------------------------------*/
 
-static void
-parse_set(struct vcc *tl)
+static void v_matchproto_(sym_act_f)
+parse_set(struct vcc *tl, struct symbol *sym)
 {
-	const struct symbol *sym;
 	const struct arith *ap;
 	const struct token *t;
 	vcc_type_t fmt;
@@ -143,10 +141,9 @@ parse_set(struct vcc *tl)
 
 /*--------------------------------------------------------------------*/
 
-static void
-parse_unset(struct vcc *tl)
+static void v_matchproto_(sym_act_f)
+parse_unset(struct vcc *tl, struct symbol *sym)
 {
-	const struct symbol *sym;
 	const struct token *t;
 
 	/* XXX: Wrong, should use VCC_Expr(HEADER) */
@@ -169,10 +166,11 @@ parse_unset(struct vcc *tl)
 
 /*--------------------------------------------------------------------*/
 
-static void
-parse_ban(struct vcc *tl)
+static void v_matchproto_(sym_act_f)
+parse_ban(struct vcc *tl, struct symbol *sym)
 {
 
+	(void)sym;
 	vcc_NextToken(tl);
 
 	ExpectErr(tl, '(');
@@ -191,9 +189,11 @@ parse_ban(struct vcc *tl)
 
 /*--------------------------------------------------------------------*/
 
-static void
-parse_hash_data(struct vcc *tl)
+static void v_matchproto_(sym_act_f)
+parse_hash_data(struct vcc *tl, struct symbol *sym)
 {
+
+	(void)sym;
 	vcc_NextToken(tl);
 	SkipToken(tl, '(');
 
@@ -287,12 +287,13 @@ parse_return_vcl(struct vcc *tl)
 
 /*--------------------------------------------------------------------*/
 
-static void
-parse_return(struct vcc *tl)
+static void v_matchproto_(sym_act_f)
+parse_return(struct vcc *tl, struct symbol *sym)
 {
 	unsigned hand;
 	const char *h;
 
+	(void)sym;
 	vcc_NextToken(tl);
 	AN(tl->curproc);
 	if (tl->t->tok == ';' && tl->curproc->method == NULL) {
@@ -345,11 +346,12 @@ parse_return(struct vcc *tl)
 
 /*--------------------------------------------------------------------*/
 
-static void
-parse_synthetic(struct vcc *tl)
+static void v_matchproto_(sym_act_f)
+parse_synthetic(struct vcc *tl, struct symbol *sym)
 {
 	vcc_NextToken(tl);
 
+	(void)sym;
 	ExpectErr(tl, '(');
 	ERRCHK(tl);
 	vcc_NextToken(tl);
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 84e849e..0067365 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -108,7 +108,7 @@ typedef void sym_expr_t(struct vcc *tl, struct expr **,
     struct symbol *sym, vcc_type_t);
 typedef void sym_wildcard_t(struct vcc *, struct symbol *, struct symbol *);
 
-typedef void sym_act_f(struct vcc *tl);
+typedef void sym_act_f(struct vcc *, struct symbol *);
 
 struct symbol {
 	unsigned			magic;
@@ -279,7 +279,7 @@ char *TlDup(struct vcc *tl, const char *s);
 
 /* vcc_expr.c */
 void vcc_Expr(struct vcc *tl, vcc_type_t typ);
-void vcc_Expr_Call(struct vcc *tl, struct symbol *sym);
+sym_act_f vcc_ParseCall;
 void vcc_Expr_Init(struct vcc *tl);
 sym_expr_t vcc_Eval_Var;
 sym_expr_t vcc_Eval_Handle;
@@ -294,7 +294,7 @@ void vcc_Var_Init(struct vcc *);
 
 /* vcc_parse.c */
 void vcc_Parse(struct vcc *tl);
-void vcc_ParseIf(struct vcc *tl);
+sym_act_f vcc_ParseIf;
 
 /* vcc_utils.c */
 const char *vcc_regexp(struct vcc *tl);
@@ -350,7 +350,7 @@ sym_wildcard_t vcc_Var_Wildcard;
 
 /* vcc_vmod.c */
 void vcc_ParseImport(struct vcc *tl);
-void vcc_ParseNew(struct vcc *tl);
+sym_act_f vcc_ParseNew;
 
 /* vcc_xref.c */
 void vcc_AddRef(struct vcc *, struct symbol *);
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index e4d2d20..d43f5e8 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -1261,8 +1261,8 @@ vcc_Expr(struct vcc *tl, vcc_type_t fmt)
 /*--------------------------------------------------------------------
  */
 
-void
-vcc_Expr_Call(struct vcc *tl, struct symbol *sym)
+void v_matchproto_(sym_act_f)
+vcc_ParseCall(struct vcc *tl, struct symbol *sym)
 {
 
 	struct expr *e;
@@ -1273,6 +1273,7 @@ vcc_Expr_Call(struct vcc *tl, struct symbol *sym)
 	vcc_Eval_SymFunc(tl, &e, sym, VOID);
 	if (!tl->err) {
 		vcc_expr_fmt(tl->fb, tl->indent, e);
+		SkipToken(tl, ';');
 		VSB_cat(tl->fb, ";\n");
 	} else if (t1 != tl->t) {
 		vcc_ErrWhere2(tl, t1, tl->t);
@@ -1290,11 +1291,13 @@ vcc_Expr_Init(struct vcc *tl)
 
 	sym = VCC_MkSym(tl, "regsub", SYM_FUNC);
 	AN(sym);
+	sym->action = vcc_ParseCall;
 	sym->eval = vcc_Eval_Regsub;
 	sym->eval_priv = NULL;
 
 	sym = VCC_MkSym(tl, "regsuball", SYM_FUNC);
 	AN(sym);
+	sym->action = vcc_ParseCall;
 	sym->eval = vcc_Eval_Regsub;
 	sym->eval_priv = sym;
 
diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index 228a1d4..133024c 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -79,10 +79,11 @@ vcc_Conditional(struct vcc *tl)
  *	null
  */
 
-void
-vcc_ParseIf(struct vcc *tl)
+void v_matchproto_(sym_act_f)
+vcc_ParseIf(struct vcc *tl, struct symbol *sym)
 {
 
+	(void)sym;
 	SkipToken(tl, ID);
 	Fb(tl, 1, "if ");
 	vcc_Conditional(tl);
@@ -178,18 +179,14 @@ vcc_Compound(struct vcc *tl)
 			tl->err = 1;
 			return;
 		case ID:
-			sym = VCC_SymbolGet(tl, SYM_NONE, SYMTAB_NOERR, XREF_NONE);
+			sym = VCC_SymbolGet(tl, SYM_NONE, SYMTAB_NOERR,
+			    XREF_NONE);
 			if (sym != NULL && sym->action != NULL) {
 				if (sym->action_mask != 0)
 					vcc_AddUses(tl, t, NULL,
 					    sym->action_mask,
 					    "Not a valid action");
-				sym->action(tl);
-				break;
-			}
-			if (sym != NULL && sym->kind == SYM_FUNC) {
-				vcc_Expr_Call(tl, sym);
-				SkipToken(tl, ';');
+				sym->action(tl, sym);
 				break;
 			}
 			/* FALLTHROUGH */
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 239aa88..416ac1d 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -245,6 +245,7 @@ vcc_ParseImport(struct vcc *tl)
 			sym = VCC_MkSym(tl, p, SYM_FUNC);
 			ERRCHK(tl);
 			AN(sym);
+			sym->action = vcc_ParseCall;
 			sym->vmod = msym->name;
 			sym->eval = vcc_Eval_SymFunc;
 			p += strlen(p) + 1;
@@ -265,8 +266,8 @@ vcc_ParseImport(struct vcc *tl)
 	Fh(tl, 0, "\n/* --- END VMOD %.*s --- */\n\n", PF(mod));
 }
 
-void
-vcc_ParseNew(struct vcc *tl)
+void v_matchproto_(sym_act_f)
+vcc_ParseNew(struct vcc *tl, struct symbol *sym)
 {
 	struct symbol *sy1, *sy2, *sy3;
 	struct inifin *ifp;
@@ -275,6 +276,7 @@ vcc_ParseNew(struct vcc *tl)
 	char buf1[128];
 	char buf2[128];
 
+	(void)sym;
 	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
 	vcc_ExpectVid(tl, "VCL object");
@@ -335,6 +337,7 @@ vcc_ParseNew(struct vcc *tl)
 		bprintf(buf2, "%s%s", sy1->name, p);
 		sy3 = VCC_MkSym(tl, buf2, SYM_FUNC);
 		AN(sy3);
+		sy3->action = vcc_ParseCall;
 		sy3->eval = vcc_Eval_SymFunc;
 		p += strlen(p) + 1;
 		sy3->eval_priv = p;

From phk at FreeBSD.org  Wed Jan 31 10:50:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 31 Jan 2018 10:50:10 +0000 (UTC)
Subject: [master] adb65db Dont assume that symbols are a single token (1/many)
Message-ID: <20180131105010.B9924943F6@lists.varnish-cache.org>


commit adb65db53cba2ec2282d49d8f9442507e71e8e36
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 31 09:20:13 2018 +0000

    Dont assume that symbols are a single token (1/many)

diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 0067365..a572847 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -105,7 +105,7 @@ enum symkind {
 };
 
 typedef void sym_expr_t(struct vcc *tl, struct expr **,
-    struct symbol *sym, vcc_type_t);
+    struct token *, struct symbol *sym, vcc_type_t);
 typedef void sym_wildcard_t(struct vcc *, struct symbol *, struct symbol *);
 
 typedef void sym_act_f(struct vcc *, struct symbol *);
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index d43f5e8..5cfc76c 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -282,16 +282,16 @@ vcc_expr_tostring(struct vcc *tl, struct expr **e, vcc_type_t fmt)
  */
 
 static void v_matchproto_(sym_expr_t)
-vcc_Eval_Regsub(struct vcc *tl, struct expr **e, struct symbol *sym,
-    vcc_type_t fmt)
+vcc_Eval_Regsub(struct vcc *tl, struct expr **e, struct token *t,
+    struct symbol *sym, vcc_type_t fmt)
 {
 	struct expr *e2;
 	int all = sym->eval_priv == NULL ? 0 : 1;
 	const char *p;
 	char buf[128];
 
+	(void)t;
 	(void)fmt;
-	SkipToken(tl, ID);
 	SkipToken(tl, '(');
 	vcc_expr0(tl, &e2, STRING);
 	ERRCHK(tl);
@@ -312,12 +312,13 @@ vcc_Eval_Regsub(struct vcc *tl, struct expr **e, struct symbol *sym,
  */
 
 static void v_matchproto_(sym_expr_t)
-vcc_Eval_BoolConst(struct vcc *tl, struct expr **e, struct symbol *sym,
-    vcc_type_t fmt)
+vcc_Eval_BoolConst(struct vcc *tl, struct expr **e, struct token *t,
+    struct symbol *sym, vcc_type_t fmt)
 {
 
+	(void)t;
+	(void)tl;
 	(void)fmt;
-	vcc_NextToken(tl);
 	*e = vcc_mk_expr(BOOL, "(0==%d)", sym->eval_priv == NULL ? 1 : 0);
 	(*e)->constant = EXPR_CONST;
 }
@@ -326,10 +327,11 @@ vcc_Eval_BoolConst(struct vcc *tl, struct expr **e, struct symbol *sym,
  */
 
 void v_matchproto_(sym_expr_t)
-vcc_Eval_Handle(struct vcc *tl, struct expr **e, struct symbol *sym,
-    vcc_type_t fmt)
+vcc_Eval_Handle(struct vcc *tl, struct expr **e, struct token *t,
+    struct symbol *sym, vcc_type_t fmt)
 {
 
+	(void)t;
 	AN(sym->rname);
 
 	vcc_AddRef(tl, sym);
@@ -338,34 +340,31 @@ vcc_Eval_Handle(struct vcc *tl, struct expr **e, struct symbol *sym,
 		(*e)->nstr = 1;
 		(*e)->constant |= EXPR_CONST | EXPR_STR_CONST;
 	} else {
-		vcc_ExpectVid(tl, "handle");
 		*e = vcc_mk_expr(sym->fmt, "%s", sym->rname);
 		(*e)->constant = EXPR_VAR;
 		(*e)->nstr = 1;
 		if ((*e)->fmt == STRING)
 			(*e)->fmt = STRINGS;
 	}
-	vcc_NextToken(tl);
 }
 
 /*--------------------------------------------------------------------
  */
 
 void v_matchproto_(sym_expr_t)
-vcc_Eval_Var(struct vcc *tl, struct expr **e, struct symbol *sym,
-    vcc_type_t fmt)
+vcc_Eval_Var(struct vcc *tl, struct expr **e, struct token *t,
+    struct symbol *sym, vcc_type_t fmt)
 {
 
 	(void)fmt;
 	assert(sym->kind == SYM_VAR);
-	vcc_AddUses(tl, tl->t, NULL, sym->r_methods, "Not available");
+	vcc_AddUses(tl, t, NULL, sym->r_methods, "Not available");
 	ERRCHK(tl);
 	*e = vcc_mk_expr(sym->fmt, "%s", sym->rname);
 	(*e)->constant = EXPR_VAR;
 	(*e)->nstr = 1;
 	if ((*e)->fmt == STRING)
 		(*e)->fmt = STRINGS;
-	vcc_NextToken(tl);
 }
 
 /*--------------------------------------------------------------------
@@ -597,16 +596,16 @@ vcc_Eval_Func(struct vcc *tl, const char *spec,
  */
 
 void v_matchproto_(sym_expr_t)
-vcc_Eval_SymFunc(struct vcc *tl, struct expr **e, struct symbol *sym,
-    vcc_type_t fmt)
+vcc_Eval_SymFunc(struct vcc *tl, struct expr **e, struct token *t,
+    struct symbol *sym, vcc_type_t fmt)
 {
 
+	(void)t;
 	(void)fmt;
 	assert(sym->kind == SYM_FUNC);
 	AN(sym->eval_priv);
 
-	SkipToken(tl, ID);
-	assert(sym->fmt == VCC_Type(sym->eval_priv));
+	// assert(sym->fmt == VCC_Type(sym->eval_priv));
 	vcc_func(tl, e, sym->eval_priv, sym->extra, sym);
 	ERRCHK(tl);
 	if ((*e)->fmt == STRING) {
@@ -629,6 +628,7 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 {
 	struct expr *e1, *e2;
 	const char *ip, *sign;
+	struct token *t;
 	struct symbol *sym;
 	double d;
 	int i;
@@ -677,7 +677,9 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 		case SYM_PROBE:
 			AN(sym->eval);
 			AZ(*e);
-			sym->eval(tl, e, sym, fmt);
+			t = tl->t;
+			vcc_NextToken(tl);
+			sym->eval(tl, e, t, sym, fmt);
 			ERRCHK(tl);
 			/* Unless asked for a HEADER, fold to string here */
 			if (*e && fmt != HEADER && (*e)->fmt == HEADER) {
@@ -1270,7 +1272,8 @@ vcc_ParseCall(struct vcc *tl, struct symbol *sym)
 
 	t1 = tl->t;
 	e = NULL;
-	vcc_Eval_SymFunc(tl, &e, sym, VOID);
+	vcc_NextToken(tl);
+	vcc_Eval_SymFunc(tl, &e, t1, sym, VOID);
 	if (!tl->err) {
 		vcc_expr_fmt(tl->fb, tl->indent, e);
 		SkipToken(tl, ';');
@@ -1291,13 +1294,11 @@ vcc_Expr_Init(struct vcc *tl)
 
 	sym = VCC_MkSym(tl, "regsub", SYM_FUNC);
 	AN(sym);
-	sym->action = vcc_ParseCall;
 	sym->eval = vcc_Eval_Regsub;
 	sym->eval_priv = NULL;
 
 	sym = VCC_MkSym(tl, "regsuball", SYM_FUNC);
 	AN(sym);
-	sym->action = vcc_ParseCall;
 	sym->eval = vcc_Eval_Regsub;
 	sym->eval_priv = sym;
 
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index 0e58c4c..f79068e 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -271,10 +271,12 @@ VCC_GlobalSymbol(struct symbol *sym, vcc_type_t fmt, const char *pfx)
 
 	sym->fmt = fmt;
 	sym->kind = VCC_HandleKind(sym->fmt);
-	if (sym->kind != SYM_NONE)
+	if (sym->kind != SYM_NONE) {
+		AZ(VCT_invalid_name(sym->rname, NULL));
 		sym->eval = vcc_Eval_Handle;
-	else
+	} else {
 		WRONG("Wrong kind of global symbol");
+	}
 
 #define VCL_MET_MAC(l,u,t,b)   sym->r_methods |= VCL_MET_##u;
 #include "tbl/vcl_returns.h"

From phk at FreeBSD.org  Wed Jan 31 10:50:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 31 Jan 2018 10:50:10 +0000 (UTC)
Subject: [master] cc850c4 Don't assume symbols are a single token (2/many)
Message-ID: <20180131105010.D2195943FC@lists.varnish-cache.org>


commit cc850c4c382f6159f09f225606c9b217f615bbd2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 31 09:39:45 2018 +0000

    Don't assume symbols are a single token (2/many)

diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index 61d4264..e248fff 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -40,10 +40,10 @@
 /*--------------------------------------------------------------------*/
 
 static void v_matchproto_(sym_act_f)
-parse_call(struct vcc *tl, struct symbol *sym)
+vcc_act_call(struct vcc *tl, struct token *t, struct symbol *sym)
 {
 
-	vcc_NextToken(tl);
+	(void)t;
 	ExpectErr(tl, ID);
 	sym = VCC_SymbolGet(tl, SYM_SUB, SYMTAB_CREATE, XREF_REF);
 	AN(sym);
@@ -86,13 +86,12 @@ static const struct arith {
 /*--------------------------------------------------------------------*/
 
 static void v_matchproto_(sym_act_f)
-parse_set(struct vcc *tl, struct symbol *sym)
+vcc_act_set(struct vcc *tl, struct token *t, struct symbol *sym)
 {
 	const struct arith *ap;
-	const struct token *t;
 	vcc_type_t fmt;
 
-	vcc_NextToken(tl);
+	(void)t;
 	ExpectErr(tl, ID);
 	t = tl->t;
 	sym = VCC_SymbolGet(tl, SYM_VAR, "Unknown variable", XREF_NONE);
@@ -142,12 +141,10 @@ parse_set(struct vcc *tl, struct symbol *sym)
 /*--------------------------------------------------------------------*/
 
 static void v_matchproto_(sym_act_f)
-parse_unset(struct vcc *tl, struct symbol *sym)
+vcc_act_unset(struct vcc *tl, struct token *t, struct symbol *sym)
 {
-	const struct token *t;
 
 	/* XXX: Wrong, should use VCC_Expr(HEADER) */
-	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
 	t = tl->t;
 	sym = VCC_SymbolGet(tl, SYM_VAR, "Unknown variable", XREF_NONE);
@@ -167,11 +164,11 @@ parse_unset(struct vcc *tl, struct symbol *sym)
 /*--------------------------------------------------------------------*/
 
 static void v_matchproto_(sym_act_f)
-parse_ban(struct vcc *tl, struct symbol *sym)
+vcc_act_ban(struct vcc *tl, struct token *t, struct symbol *sym)
 {
 
+	(void)t;
 	(void)sym;
-	vcc_NextToken(tl);
 
 	ExpectErr(tl, '(');
 	vcc_NextToken(tl);
@@ -190,11 +187,11 @@ parse_ban(struct vcc *tl, struct symbol *sym)
 /*--------------------------------------------------------------------*/
 
 static void v_matchproto_(sym_act_f)
-parse_hash_data(struct vcc *tl, struct symbol *sym)
+vcc_act_hash_data(struct vcc *tl, struct token *t, struct symbol *sym)
 {
 
+	(void)t;
 	(void)sym;
-	vcc_NextToken(tl);
 	SkipToken(tl, '(');
 
 	Fb(tl, 1, "VRT_hashdata(ctx,\n  ");
@@ -208,7 +205,7 @@ parse_hash_data(struct vcc *tl, struct symbol *sym)
 /*--------------------------------------------------------------------*/
 
 static void
-parse_return_pass(struct vcc *tl)
+vcc_act_return_pass(struct vcc *tl)
 {
 
 	ExpectErr(tl, '(');
@@ -224,7 +221,7 @@ parse_return_pass(struct vcc *tl)
 /*--------------------------------------------------------------------*/
 
 static void
-parse_return_synth(struct vcc *tl)
+vcc_act_return_synth(struct vcc *tl)
 {
 
 	ExpectErr(tl, '(');
@@ -249,7 +246,7 @@ parse_return_synth(struct vcc *tl)
 /*--------------------------------------------------------------------*/
 
 static void
-parse_return_vcl(struct vcc *tl)
+vcc_act_return_vcl(struct vcc *tl)
 {
 	struct symbol *sym;
 	struct inifin *p;
@@ -288,13 +285,13 @@ parse_return_vcl(struct vcc *tl)
 /*--------------------------------------------------------------------*/
 
 static void v_matchproto_(sym_act_f)
-parse_return(struct vcc *tl, struct symbol *sym)
+vcc_act_return(struct vcc *tl, struct token *t, struct symbol *sym)
 {
 	unsigned hand;
 	const char *h;
 
+	(void)t;
 	(void)sym;
-	vcc_NextToken(tl);
 	AN(tl->curproc);
 	if (tl->t->tok == ';' && tl->curproc->method == NULL) {
 		SkipToken(tl, ';');
@@ -323,11 +320,11 @@ parse_return(struct vcc *tl, struct symbol *sym)
 	vcc_NextToken(tl);
 	if (tl->t->tok == '(') {
 		if (hand == VCL_RET_SYNTH)
-			parse_return_synth(tl);
+			vcc_act_return_synth(tl);
 		else if (hand == VCL_RET_VCL)
-			parse_return_vcl(tl);
+			vcc_act_return_vcl(tl);
 		else if (hand == VCL_RET_PASS)
-			parse_return_pass(tl);
+			vcc_act_return_pass(tl);
 		else {
 			VSB_printf(tl->sb, "Arguments not allowed.\n");
 			vcc_ErrWhere(tl, tl->t);
@@ -347,10 +344,10 @@ parse_return(struct vcc *tl, struct symbol *sym)
 /*--------------------------------------------------------------------*/
 
 static void v_matchproto_(sym_act_f)
-parse_synthetic(struct vcc *tl, struct symbol *sym)
+vcc_act_synthetic(struct vcc *tl, struct token *t, struct symbol *sym)
 {
-	vcc_NextToken(tl);
 
+	(void)t;
 	(void)sym;
 	ExpectErr(tl, '(');
 	ERRCHK(tl);
@@ -382,16 +379,16 @@ vcc_Action_Init(struct vcc *tl)
 {
 	struct symbol *sym;
 
-	ACT(ban,	parse_ban,	0);
-	ACT(call,	parse_call,	0);
-	ACT(hash_data,	parse_hash_data,
+	ACT(ban,	vcc_act_ban,	0);
+	ACT(call,	vcc_act_call,	0);
+	ACT(hash_data,	vcc_act_hash_data,
 		VCL_MET_HASH);
-	ACT(if,		vcc_ParseIf,	0);
-	ACT(new,	vcc_ParseNew,
+	ACT(if,		vcc_Act_If,	0);
+	ACT(new,	vcc_Act_New,
 		VCL_MET_INIT);
-	ACT(return,	parse_return,	0);
-	ACT(set,	parse_set,	0);
-	ACT(synthetic,	parse_synthetic,
+	ACT(return,	vcc_act_return,	0);
+	ACT(set,	vcc_act_set,	0);
+	ACT(synthetic,	vcc_act_synthetic,
 		VCL_MET_SYNTH | VCL_MET_BACKEND_ERROR);
-	ACT(unset,	parse_unset,	0);
+	ACT(unset,	vcc_act_unset,	0);
 }
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index a572847..0bfed2a 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -108,7 +108,7 @@ typedef void sym_expr_t(struct vcc *tl, struct expr **,
     struct token *, struct symbol *sym, vcc_type_t);
 typedef void sym_wildcard_t(struct vcc *, struct symbol *, struct symbol *);
 
-typedef void sym_act_f(struct vcc *, struct symbol *);
+typedef void sym_act_f(struct vcc *, struct token *, struct symbol *);
 
 struct symbol {
 	unsigned			magic;
@@ -294,7 +294,7 @@ void vcc_Var_Init(struct vcc *);
 
 /* vcc_parse.c */
 void vcc_Parse(struct vcc *tl);
-sym_act_f vcc_ParseIf;
+sym_act_f vcc_Act_If;
 
 /* vcc_utils.c */
 const char *vcc_regexp(struct vcc *tl);
@@ -350,7 +350,7 @@ sym_wildcard_t vcc_Var_Wildcard;
 
 /* vcc_vmod.c */
 void vcc_ParseImport(struct vcc *tl);
-sym_act_f vcc_ParseNew;
+sym_act_f vcc_Act_New;
 
 /* vcc_xref.c */
 void vcc_AddRef(struct vcc *, struct symbol *);
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index 5cfc76c..f874d58 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -1264,22 +1264,19 @@ vcc_Expr(struct vcc *tl, vcc_type_t fmt)
  */
 
 void v_matchproto_(sym_act_f)
-vcc_ParseCall(struct vcc *tl, struct symbol *sym)
+vcc_ParseCall(struct vcc *tl, struct token *t, struct symbol *sym)
 {
 
 	struct expr *e;
-	struct token *t1;
 
-	t1 = tl->t;
 	e = NULL;
-	vcc_NextToken(tl);
-	vcc_Eval_SymFunc(tl, &e, t1, sym, VOID);
+	vcc_Eval_SymFunc(tl, &e, t, sym, VOID);
 	if (!tl->err) {
 		vcc_expr_fmt(tl->fb, tl->indent, e);
 		SkipToken(tl, ';');
 		VSB_cat(tl->fb, ";\n");
-	} else if (t1 != tl->t) {
-		vcc_ErrWhere2(tl, t1, tl->t);
+	} else if (t != tl->t) {
+		vcc_ErrWhere2(tl, t, tl->t);
 	}
 	vcc_delete_expr(e);
 }
diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index 133024c..a64e143 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -80,11 +80,11 @@ vcc_Conditional(struct vcc *tl)
  */
 
 void v_matchproto_(sym_act_f)
-vcc_ParseIf(struct vcc *tl, struct symbol *sym)
+vcc_Act_If(struct vcc *tl, struct token *t, struct symbol *sym)
 {
 
+	(void)t;
 	(void)sym;
-	SkipToken(tl, ID);
 	Fb(tl, 1, "if ");
 	vcc_Conditional(tl);
 	ERRCHK(tl);
@@ -186,7 +186,8 @@ vcc_Compound(struct vcc *tl)
 					vcc_AddUses(tl, t, NULL,
 					    sym->action_mask,
 					    "Not a valid action");
-				sym->action(tl, sym);
+				vcc_NextToken(tl);
+				sym->action(tl, t, sym);
 				break;
 			}
 			/* FALLTHROUGH */
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 416ac1d..e0c0fdd 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -267,17 +267,15 @@ vcc_ParseImport(struct vcc *tl)
 }
 
 void v_matchproto_(sym_act_f)
-vcc_ParseNew(struct vcc *tl, struct symbol *sym)
+vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym)
 {
 	struct symbol *sy1, *sy2, *sy3;
 	struct inifin *ifp;
-	struct token *t;
 	const char *p, *s_obj;
 	char buf1[128];
 	char buf2[128];
 
 	(void)sym;
-	vcc_NextToken(tl);
 	ExpectErr(tl, ID);
 	vcc_ExpectVid(tl, "VCL object");
 	ERRCHK(tl);

From phk at FreeBSD.org  Wed Jan 31 10:50:10 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 31 Jan 2018 10:50:10 +0000 (UTC)
Subject: [master] ad3455c Don't assume symbols are a single token
 (xref-edition) (3/many)
Message-ID: <20180131105010.E9D9C94402@lists.varnish-cache.org>


commit ad3455c7b815419b19896d43c98c052b5e65426a
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 31 10:11:42 2018 +0000

    Don't assume symbols are a single token (xref-edition) (3/many)

diff --git a/bin/varnishtest/tests/v00038.vtc b/bin/varnishtest/tests/v00038.vtc
index 9b8cfb0..f4a8c88 100644
--- a/bin/varnishtest/tests/v00038.vtc
+++ b/bin/varnishtest/tests/v00038.vtc
@@ -92,8 +92,20 @@ varnish v1 -errvcl "Mandatory field 'host' missing." {
 		.port = "NONE";
 	}
 }
+
 varnish v1 -errvcl "No default probe defined" {
 	backend b1 {
 		.probe = default;
 	}
 }
+
+varnish v1 -errvcl "Only one default director possible." {
+	backend b1 { .host = "127.0.0.1"; }
+	backend default { .host = "127.0.0.1"; }
+	backend default { .host = "127.0.0.1"; }
+}
+
+varnish v1 -errvcl "Unused backend b1, defined:" {
+	backend b1 { .host = "127.0.0.1"; }
+	backend default { .host = "127.0.0.1"; }
+}
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index 64f73ac..28d05c9 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -485,6 +485,17 @@ vcc_ParseBackend(struct vcc *tl)
 
 	t_be = tl->t;
 	if (vcc_IdIs(tl->t, "default")) {
+		if (tl->first_director != NULL) {
+			tl->first_director->noref = 0;
+			tl->first_director = NULL;
+			tl->default_director = NULL;
+		}
+		if (tl->default_director != NULL) {
+			VSB_printf(tl->sb,
+			    "Only one default director possible.\n");
+			vcc_ErrWhere(tl, t_first);
+			return;
+		}
 		vcc_NextToken(tl);
 		dn = "vgc_backend_default";
 		tl->default_director = dn;
@@ -493,8 +504,9 @@ vcc_ParseBackend(struct vcc *tl)
 		ERRCHK(tl);
 		dn = sym->rname;
 		if (tl->default_director == NULL) {
+			tl->first_director = sym;
 			tl->default_director = dn;
-			sym->nref++;
+			sym->noref = 1;
 		}
 	}
 	Fh(tl, 0, "\nstatic struct director *%s;\n", dn);
diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h
index 0bfed2a..699b9ff 100644
--- a/lib/libvcc/vcc_compile.h
+++ b/lib/libvcc/vcc_compile.h
@@ -136,7 +136,7 @@ struct symbol {
 
 	/* xref.c */
 	struct proc			*proc;
-	unsigned			nref, ndef;
+	unsigned			noref, nref, ndef;
 
 	const char			*extra;
 
@@ -223,6 +223,7 @@ struct vcc {
 	int			nprobe;
 
 	int			ndirector;
+	struct symbol		*first_director;
 	const char		*default_director;
 	const char		*default_probe;
 
@@ -279,7 +280,7 @@ char *TlDup(struct vcc *tl, const char *s);
 
 /* vcc_expr.c */
 void vcc_Expr(struct vcc *tl, vcc_type_t typ);
-sym_act_f vcc_ParseCall;
+sym_act_f vcc_Act_Call;
 void vcc_Expr_Init(struct vcc *tl);
 sym_expr_t vcc_Eval_Var;
 sym_expr_t vcc_Eval_Handle;
@@ -353,7 +354,6 @@ void vcc_ParseImport(struct vcc *tl);
 sym_act_f vcc_Act_New;
 
 /* vcc_xref.c */
-void vcc_AddRef(struct vcc *, struct symbol *);
 int vcc_CheckReferences(struct vcc *tl);
 void VCC_XrefTable(struct vcc *);
 
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index f874d58..cdbfa61 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -332,9 +332,9 @@ vcc_Eval_Handle(struct vcc *tl, struct expr **e, struct token *t,
 {
 
 	(void)t;
+	(void)tl;
 	AN(sym->rname);
 
-	vcc_AddRef(tl, sym);
 	if (sym->fmt != STRING && fmt == STRINGS) {
 		*e = vcc_mk_expr(STRINGS, "\"%s\"", sym->name);
 		(*e)->nstr = 1;
@@ -660,7 +660,7 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 			return;
 		}
 		sym = VCC_SymbolGet(tl, SYM_NONE, "Symbol not found",
-		    XREF_NONE);
+		    XREF_REF);
 		ERRCHK(tl);
 		AN(sym);
 		if (sym->kind == SYM_FUNC && sym->fmt == VOID) {
@@ -688,6 +688,7 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 			}
 			return;
 		default:
+			AZ(sym->eval);
 			break;
 		}
 		VSB_printf(tl->sb,
@@ -1264,7 +1265,7 @@ vcc_Expr(struct vcc *tl, vcc_type_t fmt)
  */
 
 void v_matchproto_(sym_act_f)
-vcc_ParseCall(struct vcc *tl, struct token *t, struct symbol *sym)
+vcc_Act_Call(struct vcc *tl, struct token *t, struct symbol *sym)
 {
 
 	struct expr *e;
diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index a64e143..b94e0a0 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -212,12 +212,14 @@ static void
 vcc_ParseFunction(struct vcc *tl)
 {
 	struct symbol *sym;
+	struct token *t;
 	struct proc *p;
 
 	vcc_NextToken(tl);
 	vcc_ExpectVid(tl, "function");
 	ERRCHK(tl);
 
+	t = tl->t;
 	sym = VCC_SymbolGet(tl, SYM_SUB, SYMTAB_CREATE, XREF_DEF);
 	AN(sym);
 	p = sym->proc;
@@ -249,10 +251,8 @@ vcc_ParseFunction(struct vcc *tl)
 	} else {
 		/* Add to VCL sub */
 		AN(p->method);
-		if (p->name == NULL) {
-			vcc_AddRef(tl, sym);
-			p->name = tl->t;
-		}
+		if (p->name == NULL)
+			p->name = t;
 	}
 	CHECK_OBJ_NOTNULL(p, PROC_MAGIC);
 	tl->fb = p->body;
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index f79068e..51354eb 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -227,7 +227,11 @@ VCC_SymbolGet(struct vcc *tl, enum symkind kind, const char *e, const char *x)
 struct symbol *
 VCC_MkSym(struct vcc *tl, const char *b, enum symkind kind)
 {
-	return (VCC_Symbol(tl, NULL, b, NULL, kind, 1));
+	struct symbol *sym;
+
+	sym = VCC_Symbol(tl, NULL, b, NULL, kind, 1);
+	sym->noref = 1;
+	return (sym);
 }
 
 
diff --git a/lib/libvcc/vcc_var.c b/lib/libvcc/vcc_var.c
index ce09c4b..5c8eadf 100644
--- a/lib/libvcc/vcc_var.c
+++ b/lib/libvcc/vcc_var.c
@@ -51,6 +51,7 @@ vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent, struct symbol *sym)
 	}
 
 	AN(sym);
+	sym->noref = 1;
 	sym->kind = SYM_VAR;
 	sym->fmt = parent->fmt;
 	sym->eval = vcc_Eval_Var;
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index e0c0fdd..168ca04 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -245,7 +245,7 @@ vcc_ParseImport(struct vcc *tl)
 			sym = VCC_MkSym(tl, p, SYM_FUNC);
 			ERRCHK(tl);
 			AN(sym);
-			sym->action = vcc_ParseCall;
+			sym->action = vcc_Act_Call;
 			sym->vmod = msym->name;
 			sym->eval = vcc_Eval_SymFunc;
 			p += strlen(p) + 1;
@@ -281,9 +281,7 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym)
 	ERRCHK(tl);
 	sy1 = VCC_HandleSymbol(tl, INSTANCE, "vo");
 	ERRCHK(tl);
-
-	/* We allow implicit use of VMOD objects:  Pretend it's ref'ed */
-	sy1->nref++;
+	sy1->noref = 1;
 
 	ExpectErr(tl, '=');
 	vcc_NextToken(tl);
@@ -335,7 +333,7 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym)
 		bprintf(buf2, "%s%s", sy1->name, p);
 		sy3 = VCC_MkSym(tl, buf2, SYM_FUNC);
 		AN(sy3);
-		sy3->action = vcc_ParseCall;
+		sy3->action = vcc_Act_Call;
 		sy3->eval = vcc_Eval_SymFunc;
 		p += strlen(p) + 1;
 		sy3->eval_priv = p;
diff --git a/lib/libvcc/vcc_xref.c b/lib/libvcc/vcc_xref.c
index 31655dc..a419f8b 100644
--- a/lib/libvcc/vcc_xref.c
+++ b/lib/libvcc/vcc_xref.c
@@ -59,26 +59,14 @@ struct procuse {
 	struct proc		*fm;
 };
 
-/*--------------------------------------------------------------------
- * Keep track of definitions and references
- */
-
-void
-vcc_AddRef(struct vcc *tl, struct symbol *sym)
-{
-
-	(void)tl;
-	if (sym->ref_b == NULL)
-		sym->ref_b = tl->t;
-	sym->nref++;
-}
-
 /*--------------------------------------------------------------------*/
 
 static void
 vcc_checkref(struct vcc *tl, const struct symbol *sym)
 {
 
+	if (sym->noref)
+		return;
 	if (sym->ndef == 0 && sym->nref != 0) {
 		AN(sym->ref_b);
 		VSB_printf(tl->sb, "Undefined %s %.*s, first reference:\n",

From phk at FreeBSD.org  Wed Jan 31 10:50:11 2018
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed, 31 Jan 2018 10:50:11 +0000 (UTC)
Subject: [master] bda99dc Don't assume symbols are a single token (4/near the
 end now)
Message-ID: <20180131105011.1735E9440C@lists.varnish-cache.org>


commit bda99dceb8aa866b8a5bc1798d72f07e1013df97
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed Jan 31 10:49:18 2018 +0000

    Don't assume symbols are a single token (4/near the end now)

diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c
index e248fff..1a4d755 100644
--- a/lib/libvcc/vcc_action.c
+++ b/lib/libvcc/vcc_action.c
@@ -50,7 +50,6 @@ vcc_act_call(struct vcc *tl, struct token *t, struct symbol *sym)
 	vcc_AddCall(tl, sym);
 	VCC_GlobalSymbol(sym, SUB, "VGC_function");
 	Fb(tl, 1, "%s(ctx);\n", sym->rname);
-	vcc_NextToken(tl);
 	SkipToken(tl, ';');
 }
 
@@ -96,7 +95,6 @@ vcc_act_set(struct vcc *tl, struct token *t, struct symbol *sym)
 	t = tl->t;
 	sym = VCC_SymbolGet(tl, SYM_VAR, "Unknown variable", XREF_NONE);
 	ERRCHK(tl);
-	vcc_NextToken(tl);
 	AN(sym);
 	if (sym->w_methods == 0) {
 		vcc_ErrWhere2(tl, t, tl->t);
@@ -149,7 +147,6 @@ vcc_act_unset(struct vcc *tl, struct token *t, struct symbol *sym)
 	t = tl->t;
 	sym = VCC_SymbolGet(tl, SYM_VAR, "Unknown variable", XREF_NONE);
 	ERRCHK(tl);
-	vcc_NextToken(tl);
 	AN(sym);
 	if (sym->u_methods == 0) {
 		vcc_ErrWhere2(tl, t, tl->t);
@@ -278,7 +275,6 @@ vcc_act_return_vcl(struct vcc *tl)
 	}
 	Fb(tl, 1, "VRT_vcl_select(ctx, %s);\t/* %s */\n",
 	    (const char*)sym->eval_priv, sym->name);
-	vcc_NextToken(tl);
 	SkipToken(tl, ')');
 }
 
diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c
index 28d05c9..db5ce71 100644
--- a/lib/libvcc/vcc_backend.c
+++ b/lib/libvcc/vcc_backend.c
@@ -403,6 +403,7 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 			ERRCHK(tl);
 		} else if (vcc_IdIs(t_field, "probe") && tl->t->tok == ID) {
 			if (vcc_IdIs(tl->t, "default")) {
+				vcc_NextToken(tl);
 				(void)vcc_default_probe(tl);
 			} else {
 				pb = VCC_SymbolGet(tl, SYM_PROBE,
@@ -411,7 +412,6 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname)
 				AN(pb);
 				Fb(tl, 0, "\t.probe = %s,\n", pb->rname);
 			}
-			vcc_NextToken(tl);
 			SkipToken(tl, ';');
 		} else if (vcc_IdIs(t_field, "probe")) {
 			VSB_printf(tl->sb,
@@ -478,7 +478,7 @@ vcc_ParseBackend(struct vcc *tl)
 
 	tl->ndirector++;
 	t_first = tl->t;
-	vcc_NextToken(tl);		/* ID: backend */
+	SkipToken(tl, ID);		/* ID: backend */
 
 	vcc_ExpectVid(tl, "backend");	/* ID: name */
 	ERRCHK(tl);
diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c
index cdbfa61..81f4135 100644
--- a/lib/libvcc/vcc_expr.c
+++ b/lib/libvcc/vcc_expr.c
@@ -659,6 +659,7 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 			    "*(VCL_conf.default_director)");
 			return;
 		}
+		t = tl->t;
 		sym = VCC_SymbolGet(tl, SYM_NONE, "Symbol not found",
 		    XREF_REF);
 		ERRCHK(tl);
@@ -677,8 +678,6 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt)
 		case SYM_PROBE:
 			AN(sym->eval);
 			AZ(*e);
-			t = tl->t;
-			vcc_NextToken(tl);
 			sym->eval(tl, e, t, sym, fmt);
 			ERRCHK(tl);
 			/* Unless asked for a HEADER, fold to string here */
@@ -969,7 +968,6 @@ cmp_acl(struct vcc *tl, struct expr **e, const struct cmps *cp)
 	vcc_ExpectVid(tl, "ACL");
 	sym = VCC_SymbolGet(tl, SYM_ACL, SYMTAB_CREATE, XREF_REF);
 	AN(sym);
-	vcc_NextToken(tl);
 	VCC_GlobalSymbol(sym, ACL, ACL_SYMBOL_PREFIX);
 	bprintf(buf, "%sVRT_acl_match(ctx, %s, \v1)", cp->emit, sym->rname);
 	*e = vcc_expr_edit(tl, BOOL, buf, *e, NULL);
diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c
index b94e0a0..7bcd732 100644
--- a/lib/libvcc/vcc_parse.c
+++ b/lib/libvcc/vcc_parse.c
@@ -186,7 +186,6 @@ vcc_Compound(struct vcc *tl)
 					vcc_AddUses(tl, t, NULL,
 					    sym->action_mask,
 					    "Not a valid action");
-				vcc_NextToken(tl);
 				sym->action(tl, t, sym);
 				break;
 			}
@@ -224,12 +223,12 @@ vcc_ParseFunction(struct vcc *tl)
 	AN(sym);
 	p = sym->proc;
 	if (p == NULL) {
-		if ((tl->t->b[0] == 'v'|| tl->t->b[0] == 'V') &&
-		    (tl->t->b[1] == 'c'|| tl->t->b[1] == 'C') &&
-		    (tl->t->b[2] == 'l'|| tl->t->b[2] == 'L')) {
+		if ((t->b[0] == 'v'|| t->b[0] == 'V') &&
+		    (t->b[1] == 'c'|| t->b[1] == 'C') &&
+		    (t->b[2] == 'l'|| t->b[2] == 'L')) {
 			VSB_printf(tl->sb,
 			    "VCL sub's named 'vcl*' are reserved names.\n");
-			vcc_ErrWhere(tl, tl->t);
+			vcc_ErrWhere(tl, t);
 			VSB_printf(tl->sb, "Valid vcl_* methods are:\n");
 			VTAILQ_FOREACH(p, &tl->procs, list) {
 				if (p->method != NULL)
@@ -240,11 +239,11 @@ vcc_ParseFunction(struct vcc *tl)
 		}
 		VCC_GlobalSymbol(sym, SUB, "VGC_function");
 		p = vcc_NewProc(tl, sym);
-		p->name = tl->t;
+		p->name = t;
 		VSB_printf(p->cname, "%s", sym->rname);
 	} else if (p->method == NULL) {
 		VSB_printf(tl->sb, "Function '%s' redefined\n", sym->name);
-		vcc_ErrWhere(tl, tl->t);
+		vcc_ErrWhere(tl, t);
 		VSB_printf(tl->sb, "Previously defined here:\n");
 		vcc_ErrWhere(tl, p->name);
 		return;
@@ -260,7 +259,6 @@ vcc_ParseFunction(struct vcc *tl)
 	vcc_Coord(tl, p->body, NULL);
 	Fb(tl, 0, " */\n");
 	tl->curproc = p;
-	vcc_NextToken(tl);
 	tl->indent += INDENT;
 	Fb(tl, 1, "{\n");
 	L(tl, vcc_Compound(tl));
diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c
index 51354eb..3ba2b8c 100644
--- a/lib/libvcc/vcc_symb.c
+++ b/lib/libvcc/vcc_symb.c
@@ -221,6 +221,7 @@ VCC_SymbolGet(struct vcc *tl, enum symkind kind, const char *e, const char *x)
 	} else {
 		assert (x == XREF_NONE);
 	}
+	vcc_NextToken(tl);
 	return (sym);
 }
 
@@ -291,24 +292,26 @@ VCC_HandleSymbol(struct vcc *tl, vcc_type_t fmt, const char *pfx)
 {
 	struct symbol *sym;
 	enum symkind kind;
+	struct token *t;
 	const char *p;
 
 	kind = VCC_HandleKind(fmt);
 	assert(kind != SYM_NONE);
 
+	t = tl->t;
 	sym = VCC_SymbolGet(tl, SYM_NONE, SYMTAB_NOERR, XREF_NONE);
 	if (sym != NULL && sym->def_b != NULL && kind == sym->kind) {
 		p = VCC_SymKind(tl, sym);
 		VSB_printf(tl->sb, "%c%s '%.*s' redefined.\n",
-		    toupper(*p), p + 1, PF(tl->t));
-		vcc_ErrWhere(tl, tl->t);
+		    toupper(*p), p + 1, PF(t));
+		vcc_ErrWhere(tl, t);
 		VSB_printf(tl->sb, "First definition:\n");
 		AN(sym->def_b);
 		vcc_ErrWhere(tl, sym->def_b);
 		return (sym);
 	} else if (sym != NULL && sym->def_b != NULL) {
-		VSB_printf(tl->sb, "Name '%.*s' already defined.\n", PF(tl->t));
-		vcc_ErrWhere(tl, tl->t);
+		VSB_printf(tl->sb, "Name '%.*s' already defined.\n", PF(t));
+		vcc_ErrWhere(tl, t);
 		VSB_printf(tl->sb, "First definition:\n");
 		AN(sym->def_b);
 		vcc_ErrWhere(tl, sym->def_b);
@@ -316,8 +319,8 @@ VCC_HandleSymbol(struct vcc *tl, vcc_type_t fmt, const char *pfx)
 	} else if (sym != NULL && sym->kind != kind) {
 		VSB_printf(tl->sb,
 		    "Name %.*s must have type '%s'.\n",
-		    PF(tl->t), VCC_SymKind(tl, sym));
-		vcc_ErrWhere(tl, tl->t);
+		    PF(t), VCC_SymKind(tl, sym));
+		vcc_ErrWhere(tl, t);
 		return (sym);
 	}
 	if (sym == NULL)
@@ -327,7 +330,6 @@ VCC_HandleSymbol(struct vcc *tl, vcc_type_t fmt, const char *pfx)
 	VCC_GlobalSymbol(sym, fmt, pfx);
 	sym->ndef = 1;
 	if (sym->def_b == NULL)
-		sym->def_b = tl->t;
-	vcc_NextToken(tl);
+		sym->def_b = t;
 	return (sym);
 }
diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c
index 168ca04..32cd611 100644
--- a/lib/libvcc/vcc_vmod.c
+++ b/lib/libvcc/vcc_vmod.c
@@ -290,7 +290,6 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym)
 	t = tl->t;
 	sy2 = VCC_SymbolGet(tl, SYM_OBJECT, "Symbol not found", XREF_NONE);
 	ERRCHK(tl);
-	vcc_NextToken(tl);
 	AN(sy2);
 	if (sy2->extra == NULL) {
 		VSB_printf(tl->sb, "Constructor not found: ");

From nils.goroll at uplex.de  Wed Jan 31 18:05:10 2018
From: nils.goroll at uplex.de (Nils Goroll)
Date: Wed, 31 Jan 2018 18:05:10 +0000 (UTC)
Subject: [master] 4756c13 add http_date_postel feature bit for relaxed http
 date parsing
Message-ID: <20180131180510.74A21A3329@lists.varnish-cache.org>


commit 4756c13fd6ae0608b2f89c569c6689d8ece9d015
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Wed Jan 31 18:53:39 2018 +0100

    add http_date_postel feature bit for relaxed http date parsing
    
    For now, the only case this adds is a missing leading zero in the
    day of the month of RFC822/RFC1123 - style http dates as in
    
    		Fri, 2 Mar 2018 14:26:02 GMT (bad)
    instead of	Fri, 02 Mar 2018 14:26:02 GMT (correct)
    
    This is clearly invalid according to
    https://tools.ietf.org/html/rfc7231#section-7.1.1.2
    yet accepted by other webserver software.

diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index 18f02ca..e2fecec 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -130,6 +130,21 @@ static const struct linger linger = {
 static unsigned		need_test;
 
 /*--------------------------------------------------------------------
+ * lacking a better place, we put some generic periodic updates
+ * into the vca_acct() loop which we are running anyway
+ */
+static void
+vca_periodic(double t0)
+{
+	double now;
+
+	now = VTIM_real();
+	VSC_C_main->uptime = (uint64_t)(now - t0);
+
+	VTIM_postel = FEATURE(FEATURE_HTTP_DATE_POSTEL);
+}
+
+/*--------------------------------------------------------------------
  * Some kernels have bugs/limitations with respect to which options are
  * inherited from the accept/listen socket, so we have to keep track of
  * which, if any, sockopts we have to set on the accepted socket.
@@ -522,7 +537,7 @@ static void * v_matchproto_()
 vca_acct(void *arg)
 {
 	struct listen_sock *ls;
-	double t0, now;
+	double t0;
 
 	// XXX Actually a mis-nomer now because the accept happens in a pool
 	// thread. Rename to accept-nanny or so?
@@ -530,6 +545,9 @@ vca_acct(void *arg)
 	THR_Init();
 	(void)arg;
 
+	t0 = VTIM_real();
+
+	vca_periodic(t0);
 	(void)vca_tcp_opt_init();
 
 	AZ(pthread_mutex_lock(&shut_mtx));
@@ -562,7 +580,6 @@ vca_acct(void *arg)
 	need_test = 1;
 	pool_accepting = 1;
 
-	t0 = VTIM_real();
 	while (1) {
 		(void)sleep(1);
 		if (vca_tcp_opt_init()) {
@@ -575,8 +592,7 @@ vca_acct(void *arg)
 			}
 			AZ(pthread_mutex_unlock(&shut_mtx));
 		}
-		now = VTIM_real();
-		VSC_C_main->uptime = (uint64_t)(now - t0);
+		vca_periodic(t0);
 	}
 	NEEDLESS(return NULL);
 }
diff --git a/bin/varnishtest/tests/c00085.vtc b/bin/varnishtest/tests/c00085.vtc
new file mode 100644
index 0000000..ba305f5
--- /dev/null
+++ b/bin/varnishtest/tests/c00085.vtc
@@ -0,0 +1,25 @@
+varnishtest "relaxed date parsing"
+
+server s1 {
+	rxreq
+	txresp \
+	    -hdr "Date: Wed, 31 Jan 2018 14:26:02 GMT" \
+	    -hdr "Last-Modified: Thu, 4 Jan 2018 06:26:29 GMT" \
+	    -hdr "Expires: Fri, 2 Mar 2018 14:26:02 GMT"
+} -start
+
+varnish v1 -arg "-p feature=+http_date_postel" -vcl+backend {
+	import std;
+
+	sub vcl_backend_response {
+		set beresp.http.ttl = beresp.ttl;
+		set beresp.http.lm = std.time(beresp.http.last-modified, now);
+	}
+ } -start
+
+client c1 {
+	txreq
+	rxresp
+	expect resp.http.ttl == "2592000.000"
+	expect resp.http.lm == "Thu, 04 Jan 2018 06:26:29 GMT"
+} -run
diff --git a/include/tbl/feature_bits.h b/include/tbl/feature_bits.h
index 96500da..23f1b01 100644
--- a/include/tbl/feature_bits.h
+++ b/include/tbl/feature_bits.h
@@ -77,6 +77,12 @@ FEATURE_BIT(HTTP2,		http2,
     "Enable HTTP/2 protocol support."
 )
 
+FEATURE_BIT(HTTP_DATE_POSTEL,	http_date_postel,
+    "Relax parsing of timestamps in HTTP headers",
+    "Tolerate non standards conforming variations of timestamp headers"
+    "like Date:, Last-Modified:, Expires: etc."
+)
+
 #undef FEATURE_BIT
 
 /*lint -restore */
diff --git a/include/vtim.h b/include/vtim.h
index 8db4fbe..9c2e6cc 100644
--- a/include/vtim.h
+++ b/include/vtim.h
@@ -29,6 +29,7 @@
  */
 
 /* from libvarnish/vtim.c */
+extern unsigned VTIM_postel;
 #define VTIM_FORMAT_SIZE 30
 void VTIM_format(double t, char *p);
 double VTIM_parse(const char *p);
diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index 4ef9058..9772ac8 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -70,6 +70,9 @@
 #include "vas.h"
 #include "vtim.h"
 
+/* relax vtim parsing */
+unsigned VTIM_postel = 0;
+
 static const char * const weekday_name[] = {
 	"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"
 };
@@ -264,8 +267,12 @@ VTIM_parse(const char *p)
 			/* RFC822 & RFC1123 - "Sun, 06 Nov 1994 08:49:37 GMT" */
 			p++;
 			MUSTBE(' ');
-			DIGIT(10, mday);
-			DIGIT(1, mday);
+			if (VTIM_postel && *p && p[1] == ' ')
+				DIGIT(1, mday);
+			else {
+				DIGIT(10, mday);
+				DIGIT(1, mday);
+			}
 			MUSTBE(' ');
 			MONTH();
 			MUSTBE(' ');

From gquintard at users.noreply.github.com  Mon Jan 15 12:32:11 2018
From: gquintard at users.noreply.github.com (guillaume quintard)
Date: Mon, 15 Jan 2018 12:32:11 -0000
Subject: [master] a36dfe0 bytearray needs an encoding in python3
Message-ID: <20180115123211.8752A96164@lists.varnish-cache.org>


commit a36dfe03a64cdc267a97fda3253238a7cf89b81d
Author: Guillaume Quintard <guillaume at varnish-software.com>
Date:   Wed Jan 10 14:43:29 2018 +0100

    bytearray needs an encoding in python3

diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py
index d39f6f7..023f0dd 100644
--- a/lib/libvcc/vsctool.py
+++ b/lib/libvcc/vsctool.py
@@ -112,7 +112,7 @@ class vscset(object):
 		s=json.dumps(dd, separators=(",",":")) + "\0"
 		fo.write("\nstatic const unsigned char")
 		fo.write(" vsc_%s_json[%d] = {\n" % (self.name, len(s)))
-		bz = bytearray(s)
+		bz = bytearray(s, encoding="ascii")
 		t = "\t"
 		for i in bz:
 			t += "%d," % i