From phk at FreeBSD.org  Wed May  1 08:27:07 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed,  1 May 2024 08:27:07 +0000 (UTC)
Subject: [master] 2626eb27a Use bare `uname` instead of `uname -o`
Message-ID: <20240501082707.B2F86124AF3@lists.varnish-cache.org>


commit 2626eb27ae968ccd19cc77a5d325fc0096dbe19c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 1 06:40:34 2024 +0000

    Use bare `uname` instead of `uname -o`

diff --git a/autogen.des b/autogen.des
index 606c3b62a..3c4a1e04b 100755
--- a/autogen.des
+++ b/autogen.des
@@ -16,7 +16,7 @@ fi
 
 if [ "x$DST" != "x" ] ; then
 	:
-elif [ "x`uname -o`" = "xFreeBSD" ] ; then
+elif [ "x`uname`" = "xFreeBSD" ] ; then
 	DST="--prefix=/usr/local --mandir=/usr/local/man"
 else
 	DST="--prefix=/opt/varnish --mandir=/opt/varnish/man"

From phk at FreeBSD.org  Wed May  1 08:27:07 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed,  1 May 2024 08:27:07 +0000 (UTC)
Subject: [master] 1c9c578ba Have vtest report config.h to the mothership
Message-ID: <20240501082707.C8626124AF6@lists.varnish-cache.org>


commit 1c9c578ba1b1029463185bcc78bbfa4e16a2b4f2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 1 08:26:41 2024 +0000

    Have vtest report config.h to the mothership

diff --git a/tools/vtest.sh b/tools/vtest.sh
index 1d82f65a3..1f060ca7c 100755
--- a/tools/vtest.sh
+++ b/tools/vtest.sh
@@ -55,7 +55,7 @@ MAXRUNS="${MAXRUNS:-0}"
 
 enable_gcov=false
 
-SSH_DST="-p 203 vtest at varnish-cache.org"
+: ${SSH_DST:="-p 203 vtest at varnish-cache.org"}
 
 # make sure we use our own key
 unset SSH_AUTH_SOCK
@@ -283,6 +283,8 @@ do
 		echo "MANIFEST _autogen" >> "${VTEST_REPORT}"
 	else
 		echo "AUTOGEN GOOD" >> "${VTEST_REPORT}"
+		cp ${SRCDIR}/config.h "${REPORTDIR}"/_configh
+		echo "MANIFEST _configh" >> "${VTEST_REPORT}"
 		if $enable_gcov ; then
 			if makegcov >> "${REPORTDIR}"/_makegcov 2>&1 ; then
 				mv ${SRCDIR}/_gcov "${REPORTDIR}"/

From phk at FreeBSD.org  Wed May  1 12:22:04 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed,  1 May 2024 12:22:04 +0000 (UTC)
Subject: [master] d2c49b00b All sensible platforms have <sys/statvfs.h> these
 days.
Message-ID: <20240501122204.D7C9D10458F@lists.varnish-cache.org>


commit d2c49b00b7d1b8283b7425b39f070cb637c2079e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 1 11:34:18 2024 +0000

    All sensible platforms have <sys/statvfs.h> these days.

diff --git a/configure.ac b/configure.ac
index aa1e3f40c..ac237ee97 100644
--- a/configure.ac
+++ b/configure.ac
@@ -214,10 +214,7 @@ CFLAGS="${save_CFLAGS}"
 # Checks for header files.
 AC_CHECK_HEADERS([sys/endian.h])
 AC_CHECK_HEADERS([sys/filio.h])
-AC_CHECK_HEADERS([sys/mount.h], [], [], [#include <sys/param.h>])
 AC_CHECK_HEADERS([sys/personality.h])
-AC_CHECK_HEADERS([sys/statvfs.h])
-AC_CHECK_HEADERS([sys/vfs.h])
 AC_CHECK_HEADERS([endian.h])
 AC_CHECK_HEADERS([pthread_np.h], [], [], [#include <pthread.h>])
 AC_CHECK_HEADERS([priv.h])
diff --git a/lib/libvarnish/vfil.c b/lib/libvarnish/vfil.c
index 8d1f3ab60..df3211343 100644
--- a/lib/libvarnish/vfil.c
+++ b/lib/libvarnish/vfil.c
@@ -41,16 +41,7 @@
 #include <unistd.h>
 #include <stdint.h>
 #include <string.h>
-#ifdef HAVE_SYS_MOUNT_H
-#  include <sys/param.h>
-#  include <sys/mount.h>
-#endif
-#ifdef HAVE_SYS_STATVFS_H
-#  include <sys/statvfs.h>
-#endif
-#ifdef HAVE_SYS_VFS_H
-#  include <sys/vfs.h>
-#endif
+#include <sys/statvfs.h>
 #ifdef HAVE_FALLOCATE
 #  include <linux/magic.h>
 #endif
@@ -188,7 +179,6 @@ VFIL_fsinfo(int fd, unsigned *pbs, uintmax_t *psize, uintmax_t *pspace)
 {
 	unsigned bs;
 	uintmax_t size, space;
-#if defined(HAVE_SYS_STATVFS_H)
 	struct statvfs fsst;
 
 	if (fstatvfs(fd, &fsst))
@@ -196,17 +186,6 @@ VFIL_fsinfo(int fd, unsigned *pbs, uintmax_t *psize, uintmax_t *pspace)
 	bs = fsst.f_frsize;
 	size = fsst.f_blocks * fsst.f_frsize;
 	space = fsst.f_bavail * fsst.f_frsize;
-#elif defined(HAVE_SYS_MOUNT_H) || defined(HAVE_SYS_VFS_H)
-	struct statfs fsst;
-
-	if (fstatfs(fd, &fsst))
-		return (-1);
-	bs = fsst.f_bsize;
-	size = fsst.f_blocks * fsst.f_bsize;
-	space = fsst.f_bavail * fsst.f_bsize;
-#else
-#error no struct statfs / struct statvfs
-#endif
 
 	if (pbs)
 		*pbs = bs;

From phk at FreeBSD.org  Wed May  1 12:22:04 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Wed,  1 May 2024 12:22:04 +0000 (UTC)
Subject: [master] b6153d8b3 Fix #includes to match optimistic use of fallocate
 on EXT4 on linux
Message-ID: <20240501122204.EECB3104592@lists.varnish-cache.org>


commit b6153d8b353d15c30c84b3cc0c51453dc97a4b80
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Wed May 1 11:46:57 2024 +0000

    Fix #includes to match optimistic use of fallocate on EXT4 on linux

diff --git a/lib/libvarnish/vfil.c b/lib/libvarnish/vfil.c
index df3211343..71debaeee 100644
--- a/lib/libvarnish/vfil.c
+++ b/lib/libvarnish/vfil.c
@@ -42,8 +42,9 @@
 #include <stdint.h>
 #include <string.h>
 #include <sys/statvfs.h>
-#ifdef HAVE_FALLOCATE
+#if defined(__linux__) && defined(HAVE_FALLOCATE)
 #  include <linux/magic.h>
+#  include <sys/vfs.h>
 #endif
 
 #include "vdef.h"

From phk at FreeBSD.org  Fri May  3 07:26:06 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri,  3 May 2024 07:26:06 +0000 (UTC)
Subject: [master] 464309df7 Everybody have sigaltstack(2)
Message-ID: <20240503072606.19F6E10ED14@lists.varnish-cache.org>


commit 464309df79c3c12a1db0389b2203da7677f3e0b2
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 3 06:37:17 2024 +0000

    Everybody have sigaltstack(2)

diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index 7b433c596..13d209895 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -37,9 +37,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 
-#ifdef HAVE_SIGALTSTACK
-#  include <sys/mman.h>
-#endif
+#include <sys/mman.h>
 
 #ifdef HAVE_PTHREAD_NP_H
 #  include <pthread_np.h>
@@ -159,17 +157,13 @@ THR_GetName(void)
 /*--------------------------------------------------------------------
  * Generic setup all our threads should call
  */
-#ifdef HAVE_SIGALTSTACK
 static stack_t altstack;
-#endif
 
 void
 THR_Init(void)
 {
-#ifdef HAVE_SIGALTSTACK
 	if (altstack.ss_sp != NULL)
 		AZ(sigaltstack(&altstack, NULL));
-#endif
 }
 
 /*--------------------------------------------------------------------
@@ -341,7 +335,6 @@ child_sigmagic(size_t altstksz)
 
 	memset(&sa, 0, sizeof sa);
 
-#ifdef HAVE_SIGALTSTACK
 	size_t sz = vmax_t(size_t, SIGSTKSZ + 4096, altstksz);
 	altstack.ss_sp = mmap(NULL, sz,  PROT_READ | PROT_WRITE,
 			      MAP_PRIVATE | MAP_ANONYMOUS,
@@ -351,9 +344,6 @@ child_sigmagic(size_t altstksz)
 	altstack.ss_size = sz;
 	altstack.ss_flags = 0;
 	sa.sa_flags |= SA_ONSTACK;
-#else
-	(void)altstksz;
-#endif
 
 	THR_Init();
 
diff --git a/configure.ac b/configure.ac
index ac237ee97..92adda09c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -225,7 +225,6 @@ AC_CHECK_FUNCS([nanosleep])
 AC_CHECK_FUNCS([setppriv])
 AC_CHECK_FUNCS([fallocate])
 AC_CHECK_FUNCS([closefrom])
-AC_CHECK_FUNCS([sigaltstack])
 AC_CHECK_FUNCS([getpeereid])
 AC_CHECK_FUNCS([getpeerucred])
 AC_CHECK_FUNCS([fnmatch], [], [AC_MSG_ERROR([fnmatch(3) is required])])

From phk at FreeBSD.org  Fri May  3 14:15:08 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri,  3 May 2024 14:15:08 +0000 (UTC)
Subject: [master] 41cdd270f Deautocrap: everybody has a pthread_setname_np(2)
 now.
Message-ID: <20240503141509.0A7FC11BB82@lists.varnish-cache.org>


commit 41cdd270f72a6972e050c38a0fabb05793044c29
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 3 13:12:13 2024 +0000

    Deautocrap: everybody has a pthread_setname_np(2) now.

diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c
index 13d209895..d29d3d3ae 100644
--- a/bin/varnishd/cache/cache_main.c
+++ b/bin/varnishd/cache/cache_main.c
@@ -134,9 +134,6 @@ THR_SetName(const char *name)
 {
 
 	PTOK(pthread_setspecific(name_key, name));
-#if defined(HAVE_PTHREAD_SET_NAME_NP)
-	pthread_set_name_np(pthread_self(), name);
-#elif defined(HAVE_PTHREAD_SETNAME_NP)
 #if defined(__APPLE__)
 	(void)pthread_setname_np(name);
 #elif defined(__NetBSD__)
@@ -144,7 +141,6 @@ THR_SetName(const char *name)
 #else
 	(void)pthread_setname_np(pthread_self(), name);
 #endif
-#endif
 }
 
 const char *
diff --git a/configure.ac b/configure.ac
index 92adda09c..2bd5b2800 100644
--- a/configure.ac
+++ b/configure.ac
@@ -231,8 +231,6 @@ AC_CHECK_FUNCS([fnmatch], [], [AC_MSG_ERROR([fnmatch(3) is required])])
 
 save_LIBS="${LIBS}"
 LIBS="${PTHREAD_LIBS}"
-AC_CHECK_FUNCS([pthread_set_name_np])
-AC_CHECK_FUNCS([pthread_setname_np])
 AC_CHECK_FUNCS([pthread_mutex_isowned_np])
 AC_CHECK_FUNCS([pthread_getattr_np])
 LIBS="${save_LIBS}"

From phk at FreeBSD.org  Fri May  3 14:15:09 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri,  3 May 2024 14:15:09 +0000 (UTC)
Subject: [master] afcc3386f deautocrap: everybody have nanosleep(2)
Message-ID: <20240503141509.257BE11BB84@lists.varnish-cache.org>


commit afcc3386f6c6f9af8be45e140ca637a6b1e6b964
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 3 13:17:40 2024 +0000

    deautocrap: everybody have nanosleep(2)

diff --git a/configure.ac b/configure.ac
index 2bd5b2800..23f5f3b6a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -221,7 +221,6 @@ AC_CHECK_HEADERS([priv.h])
 AC_CHECK_HEADERS([fnmatch.h], [], [AC_MSG_ERROR([fnmatch.h is required])])
 
 # Checks for library functions.
-AC_CHECK_FUNCS([nanosleep])
 AC_CHECK_FUNCS([setppriv])
 AC_CHECK_FUNCS([fallocate])
 AC_CHECK_FUNCS([closefrom])
diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index 7485acb2e..05557efab 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -409,22 +409,11 @@ VTIM_parse(const char *p)
 void
 VTIM_sleep(vtim_dur t)
 {
-#ifdef HAVE_NANOSLEEP
 	struct timespec ts;
 
 	ts = VTIM_timespec(t);
 
 	(void)nanosleep(&ts, NULL);
-#else
-	if (t >= 1.) {
-		(void)sleep(floor(t));
-		t -= floor(t);
-	}
-	/* XXX: usleep() is not mandated to be thread safe */
-	t *= 1e6;
-	if (t > 0)
-		(void)usleep(floor(t));
-#endif
 }
 
 /*

From phk at FreeBSD.org  Fri May  3 14:15:09 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Fri,  3 May 2024 14:15:09 +0000 (UTC)
Subject: [master] 20c74f723 deautocrap: Everybody have clock_gettime(2)
Message-ID: <20240503141509.4777111BB87@lists.varnish-cache.org>


commit 20c74f72344c0f87a07242f9d645a1c6af1ebc3f
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Fri May 3 13:24:48 2024 +0000

    deautocrap: Everybody have clock_gettime(2)

diff --git a/configure.ac b/configure.ac
index 23f5f3b6a..e52189329 100644
--- a/configure.ac
+++ b/configure.ac
@@ -93,7 +93,6 @@ _VARNISH_CHECK_PYTHON
 
 # Check for libraries.
 _VARNISH_SEARCH_LIBS(pthread, pthread_create, [thr pthread c_r])
-_VARNISH_CHECK_LIB(rt, clock_gettime)
 _VARNISH_CHECK_LIB(dl, dlopen)
 _VARNISH_CHECK_LIB(socket, socket)
 _VARNISH_CHECK_LIB(nsl, getaddrinfo)
diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c
index 05557efab..6e61ff393 100644
--- a/lib/libvarnish/vtim.c
+++ b/lib/libvarnish/vtim.c
@@ -130,19 +130,13 @@ init(void)
 vtim_mono
 VTIM_mono(void)
 {
-#if defined(HAVE_CLOCK_GETTIME) && !defined(USE_GETHRTIME)
+#if defined(HAVE_GETHRTIME) && defined(USE_GETHRTIME)
+	return (gethrtime() * 1e-9);
+#else
 	struct timespec ts;
 
 	AZ(clock_gettime(CLOCK_MONOTONIC, &ts));
 	return (ts.tv_sec + 1e-9 * ts.tv_nsec);
-#elif defined(HAVE_GETHRTIME)
-	return (gethrtime() * 1e-9);
-#elif defined(__MACH__)
-	uint64_t mt = mach_absolute_time() - mt_base;
-
-	return (mt * mt_scale);
-#else
-#error Varnish needs some monotonic time source
 #endif
 }
 

From dridi.boukelmoune at gmail.com  Mon May  6 13:03:06 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon,  6 May 2024 13:03:06 +0000 (UTC)
Subject: [master] 7f04ec513 vtc: Stabilize r02310.vtc
Message-ID: <20240506130306.9B710104F8B@lists.varnish-cache.org>


commit 7f04ec5134e0d104962c7fbe2fc52606205e5348
Author: Walid Boudebouda <walid.boudebouda at gmail.com>
Date:   Tue Apr 30 13:53:54 2024 +0200

    vtc: Stabilize r02310.vtc
    
    The test is racy, we should always perform the backend fetch to reach
    the "barrier b1 sync" in the server body, otherwise the test will be
    stuck on the second barrier sync until timeout.
    
    Fixes: #4098

diff --git a/bin/varnishtest/tests/r02310.vtc b/bin/varnishtest/tests/r02310.vtc
index 2f303690c..677d2250b 100644
--- a/bin/varnishtest/tests/r02310.vtc
+++ b/bin/varnishtest/tests/r02310.vtc
@@ -11,6 +11,7 @@ server s1 {
 varnish v1 -vcl+backend {} -start
 
 varnish v1 -cliok "param.set feature +http2"
+varnish v1 -cliok "param.set feature -vcl_req_reset"
 
 client c1 {
 	send "GET / HTTP/1.1\r\n"

From dridi.boukelmoune at gmail.com  Tue May  7 13:31:05 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue,  7 May 2024 13:31:05 +0000 (UTC)
Subject: [master] a367c0150 vte: Preserve EOL when left-justified follows
Message-ID: <20240507133105.DC5C810F887@lists.varnish-cache.org>


commit a367c01504996406442dc8b95f8cc71297fbe0c5
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue May 7 15:16:02 2024 +0200

    vte: Preserve EOL when left-justified follows

diff --git a/lib/libvarnish/vte.c b/lib/libvarnish/vte.c
index f27598feb..db15bd94d 100644
--- a/lib/libvarnish/vte.c
+++ b/lib/libvarnish/vte.c
@@ -263,7 +263,7 @@ VTE_format(const struct vte *vte, VTE_format_f *func, void *priv)
 		if (*p == '\v') {
 			if (p > q) {
 				VTE_FORMAT(func, priv, "%.*s%s",
-				    (int)(p - q), q, sep);
+				    (int)((p - 1) - q), q, sep);
 			}
 			q = ++p;
 			just_left = 1;
@@ -310,6 +310,7 @@ static const char *test_vte =
     "foo\t\v1\tthe foo\n"
     "bar\t\v10\tthe bars\n"
     "baz\t\v0\t\n"
+    "\v0\t\v0\t\n"
     "qux\t\v-1\tno eol";
 
 static const char *test_fmt =
@@ -317,6 +318,7 @@ static const char *test_fmt =
     "foo     1  the foo\n"
     "bar    10  the bars\n"
     "baz     0  \n"
+    "   0    0  \n"
     "qux    -1  no eol";
 
 static int

From dridi.boukelmoune at gmail.com  Tue May  7 13:31:05 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue,  7 May 2024 13:31:05 +0000 (UTC)
Subject: [master] ae65ef430 vte: New VTE_dump() utility
Message-ID: <20240507133106.02F7310F88A@lists.varnish-cache.org>


commit ae65ef43095226f6b0ed47fecbcade2e3054878b
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Tue May 7 12:07:31 2024 +0200

    vte: New VTE_dump() utility

diff --git a/include/vte.h b/include/vte.h
index fcd13fb2b..28e26871e 100644
--- a/include/vte.h
+++ b/include/vte.h
@@ -47,4 +47,5 @@ int VTE_cat(struct vte *, const char *);
 int VTE_printf(struct vte *, const char *, ...) v_printflike_(2, 3);
 int VTE_finish(struct vte *);
 int VTE_format(const struct vte *, VTE_format_f *func, void *priv);
+int VTE_dump(const struct vte *, VTE_format_f *func, void *priv);
 void VTE_destroy(struct vte **);
diff --git a/lib/libvarnish/vte.c b/lib/libvarnish/vte.c
index db15bd94d..cad557b67 100644
--- a/lib/libvarnish/vte.c
+++ b/lib/libvarnish/vte.c
@@ -239,6 +239,23 @@ VTE_finish(struct vte *vte)
 			return (-1);			\
 	} while (0)
 
+int
+VTE_dump(const struct vte *vte, VTE_format_f *func, void *priv)
+{
+	const char *p;
+
+	CHECK_OBJ_NOTNULL(vte, VTE_MAGIC);
+	AN(func);
+
+	if (vte->o_sep <= 0)
+		return (-1);
+
+	p = VSB_data(vte->vsb);
+	AN(p);
+	VTE_FORMAT(func, priv, "%s", p);
+	return (0);
+}
+
 int
 VTE_format(const struct vte *vte, VTE_format_f *func, void *priv)
 {

From dridi.boukelmoune at gmail.com  Tue May  7 13:48:05 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Tue,  7 May 2024 13:48:05 +0000 (UTC)
Subject: [master] 531745096 vtc_varnish: Use a longer buffer during launch
Message-ID: <20240507134805.9446411044C@lists.varnish-cache.org>


commit 531745096505f1c4c98ddfbdaf1de459cd1fabf0
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Apr 29 23:00:05 2024 +0200

    vtc_varnish: Use a longer buffer during launch
    
    When TMPDIR is much longer than /tmp, it becomes easy to run out of
    abuf. Since abuf and pbuf are initially used to hold an IP address
    and a port number, it becomes explicit in their declaration. Likewise
    the new lbuf may hold a file name and safely be reused for CLI auth
    and other purposes.

diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 59941b3db..bc42a6dde 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -386,7 +386,9 @@ varnish_launch(struct varnish *v)
 {
 	struct vsb *vsb, *vsb1;
 	int i, nfd, asock;
-	char abuf[128], pbuf[128];
+	char abuf[VTCP_ADDRBUFSIZE];
+	char pbuf[VTCP_PORTBUFSIZE];
+	char lbuf[PATH_MAX];
 	struct pollfd fd[3];
 	enum VCLI_status_e u;
 	const char *err;
@@ -506,19 +508,19 @@ varnish_launch(struct varnish *v)
 	if (u != CLIS_AUTH)
 		vtc_fatal(v->vl, "CLI auth demand expected: %u %s", u, r);
 
-	bprintf(abuf, "%s/_.secret", v->workdir);
-	nfd = open(abuf, O_RDONLY);
+	bprintf(lbuf, "%s/_.secret", v->workdir);
+	nfd = open(lbuf, O_RDONLY);
 	assert(nfd >= 0);
 
-	assert(sizeof abuf >= CLI_AUTH_RESPONSE_LEN + 7);
-	bstrcpy(abuf, "auth ");
-	VCLI_AuthResponse(nfd, r, abuf + 5);
+	assert(sizeof lbuf >= CLI_AUTH_RESPONSE_LEN + 7);
+	bstrcpy(lbuf, "auth ");
+	VCLI_AuthResponse(nfd, r, lbuf + 5);
 	closefd(&nfd);
 	free(r);
 	r = NULL;
-	strcat(abuf, "\n");
+	strcat(lbuf, "\n");
 
-	u = varnish_ask_cli(v, abuf, &r);
+	u = varnish_ask_cli(v, lbuf, &r);
 	if (vtc_error)
 		return;
 	if (u != CLIS_OK)

From phk at FreeBSD.org  Mon May 13 07:03:09 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 13 May 2024 07:03:09 +0000 (UTC)
Subject: [master] f3764e5e9 Also return config.log to mothership
Message-ID: <20240513070309.BB1BD10351C@lists.varnish-cache.org>


commit f3764e5e9a90bc03d70eedce1e91f3b9cc504798
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 13 06:35:45 2024 +0000

    Also return config.log to mothership

diff --git a/tools/vtest.sh b/tools/vtest.sh
index 1f060ca7c..1b3bd60c5 100755
--- a/tools/vtest.sh
+++ b/tools/vtest.sh
@@ -284,7 +284,9 @@ do
 	else
 		echo "AUTOGEN GOOD" >> "${VTEST_REPORT}"
 		cp ${SRCDIR}/config.h "${REPORTDIR}"/_configh
+		cp ${SRCDIR}/config.log "${REPORTDIR}"/_configlog
 		echo "MANIFEST _configh" >> "${VTEST_REPORT}"
+		echo "MANIFEST _configlog" >> "${VTEST_REPORT}"
 		if $enable_gcov ; then
 			if makegcov >> "${REPORTDIR}"/_makegcov 2>&1 ; then
 				mv ${SRCDIR}/_gcov "${REPORTDIR}"/

From phk at FreeBSD.org  Mon May 13 07:32:04 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 13 May 2024 07:32:04 +0000 (UTC)
Subject: [master] c29596ded Try platform default "python3" before looking for
 specific versions
Message-ID: <20240513073204.D8E621046AD@lists.varnish-cache.org>


commit c29596ded371cf3fecb2c330166351be77c1dba3
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 13 07:31:10 2024 +0000

    Try platform default "python3" before looking for specific versions

diff --git a/varnish.m4 b/varnish.m4
index 83d80f2c9..a816db451 100644
--- a/varnish.m4
+++ b/varnish.m4
@@ -125,8 +125,8 @@ AC_DEFUN([_VARNISH_CHECK_DEVEL], [
 # ---------------------
 AC_DEFUN([_VARNISH_CHECK_PYTHON], [
 	m4_define_default([_AM_PYTHON_INTERPRETER_LIST],
-		[python3.10 python3.9 python3.8 python3.7 python3.6 dnl
-		python3.5 python3.4 python3 python])
+		[python3 python3.10 python3.9 python3.8 python3.7 python3.6 dnl
+		python3.5 python3.4 python])
 	AM_PATH_PYTHON([3.4], [], [
 		AC_MSG_ERROR([Python >= 3.4 is required.])
 	])

From phk at FreeBSD.org  Mon May 13 07:49:05 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 13 May 2024 07:49:05 +0000 (UTC)
Subject: [master] 0aaa5566a EXP_Rearm() is explicitly exported
Message-ID: <20240513074905.2757A105216@lists.varnish-cache.org>


commit 0aaa5566aabd16dc67aabfa7374abfb203f316e5
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 13 07:44:48 2024 +0000

    EXP_Rearm() is explicitly exported

diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt
index d6ea93a47..6e3051b88 100644
--- a/bin/varnishd/flint.lnt
+++ b/bin/varnishd/flint.lnt
@@ -173,6 +173,10 @@
 --emacro((835),VBH_NOIDX)
 --emacro((835),O_CLOEXEC)
 
+// Deliberately exported, despite no in-tree users
+-esym(759, EXP_Rearm)
+-esym(765, EXP_Rearm)
+
 // Review all below this line ///////////////////////////////////////////////
 
 -e713	// 42 Loss of precision (___) (___ to ___)

From phk at FreeBSD.org  Mon May 13 07:49:05 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 13 May 2024 07:49:05 +0000 (UTC)
Subject: [master] 86df12b6c Sync up with github/madler/zlib
Message-ID: <20240513074905.6C38A10521A@lists.varnish-cache.org>


commit 86df12b6c1ad8208899ea353fdcbea227356fcf8
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 13 07:48:03 2024 +0000

    Sync up with github/madler/zlib

diff --git a/lib/libvgz/deflate.c b/lib/libvgz/deflate.c
index bd9d5f82a..a2b341d98 100644
--- a/lib/libvgz/deflate.c
+++ b/lib/libvgz/deflate.c
@@ -1,5 +1,5 @@
 /* deflate.c -- compress data using the deflation algorithm
- * Copyright (C) 1995-2023 Jean-loup Gailly and Mark Adler
+ * Copyright (C) 1995-2024 Jean-loup Gailly and Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
@@ -53,7 +53,7 @@
 
 extern const char deflate_copyright[];
 const char deflate_copyright[] =
-   " deflate 1.3 Copyright 1995-2023 Jean-loup Gailly and Mark Adler ";
+   " deflate 1.3.1.1 Copyright 1995-2024 Jean-loup Gailly and Mark Adler ";
 /*
   If you use the zlib library in a product, an acknowledgment is welcome
   in the documentation of your product. If for some reason you cannot
@@ -370,7 +370,6 @@ local void fill_window(deflate_state *s) {
            "not enough room for search");
 }
 
-
 #ifdef NOVGZ
 
 /* ========================================================================= */
@@ -501,8 +500,8 @@ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method,
      * symbols from which it is being constructed.
      */
 
-    s->pending_buf = (uchf *) ZALLOC(strm, s->lit_bufsize, 4);
-    s->pending_buf_size = (ulg)s->lit_bufsize * 4;
+    s->pending_buf = (uchf *) ZALLOC(strm, s->lit_bufsize, LIT_BUFS);
+    s->pending_buf_size = (ulg)s->lit_bufsize * LIT_BUFS;
 
     if (s->window == Z_NULL || s->prev == Z_NULL || s->head == Z_NULL ||
         s->pending_buf == Z_NULL) {
@@ -511,8 +510,14 @@ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method,
         deflateEnd (strm);
         return Z_MEM_ERROR;
     }
+#ifdef LIT_MEM
+    s->d_buf = (ushf *)(s->pending_buf + (s->lit_bufsize << 1));
+    s->l_buf = s->pending_buf + (s->lit_bufsize << 2);
+    s->sym_end = s->lit_bufsize - 1;
+#else
     s->sym_buf = s->pending_buf + s->lit_bufsize;
     s->sym_end = (s->lit_bufsize - 1) * 3;
+#endif
     /* We avoid equality with lit_bufsize*3 because of wraparound at 64K
      * on 16 bit machines and because stored blocks are restricted to
      * 64K-1 bytes.
@@ -737,9 +742,15 @@ int ZEXPORT deflatePrime(z_streamp strm, int bits, int value) {
 
     if (deflateStateCheck(strm)) return Z_STREAM_ERROR;
     s = strm->state;
+#ifdef LIT_MEM
+    if (bits < 0 || bits > 16 ||
+        (uchf *)s->d_buf < s->pending_out + ((Buf_size + 7) >> 3))
+        return Z_BUF_ERROR;
+#else
     if (bits < 0 || bits > 16 ||
         s->sym_buf < s->pending_out + ((Buf_size + 7) >> 3))
         return Z_BUF_ERROR;
+#endif
     do {
         put = Buf_size - s->bi_valid;
         if (put > bits)
@@ -851,13 +862,13 @@ uLong ZEXPORT deflateBound(z_streamp strm, uLong sourceLen) {
     storelen = sourceLen + (sourceLen >> 5) + (sourceLen >> 7) +
                (sourceLen >> 11) + 7;
 
-    /* if can't get parameters, return larger bound plus a zlib wrapper */
+    /* if can't get parameters, return larger bound plus a wrapper */
     if (deflateStateCheck(strm))
-        return (fixedlen > storelen ? fixedlen : storelen) + 6;
+        return (fixedlen > storelen ? fixedlen : storelen) + 18;
 
     /* compute wrapper length */
     s = strm->state;
-    switch (s->wrap) {
+    switch (s->wrap < 0 ? -s->wrap : s->wrap) {
     case 0:                                 /* raw deflate */
         wraplen = 0;
         break;
@@ -887,7 +898,7 @@ uLong ZEXPORT deflateBound(z_streamp strm, uLong sourceLen) {
         break;
 #endif
     default:                                /* for compiler happiness */
-        wraplen = 6;
+        wraplen = 18;
     }
 
     /* if not default parameters, return one of the conservative bounds */
@@ -1328,7 +1339,7 @@ int ZEXPORT deflateCopy(z_streamp dest, z_streamp source) {
     ds->window = (Bytef *) ZALLOC(dest, ds->w_size, 2*sizeof(Byte));
     ds->prev   = (Posf *)  ZALLOC(dest, ds->w_size, sizeof(Pos));
     ds->head   = (Posf *)  ZALLOC(dest, ds->hash_size, sizeof(Pos));
-    ds->pending_buf = (uchf *) ZALLOC(dest, ds->lit_bufsize, 4);
+    ds->pending_buf = (uchf *) ZALLOC(dest, ds->lit_bufsize, LIT_BUFS);
 
     if (ds->window == Z_NULL || ds->prev == Z_NULL || ds->head == Z_NULL ||
         ds->pending_buf == Z_NULL) {
@@ -1339,10 +1350,15 @@ int ZEXPORT deflateCopy(z_streamp dest, z_streamp source) {
     zmemcpy(ds->window, ss->window, ds->w_size * 2 * sizeof(Byte));
     zmemcpy((voidpf)ds->prev, (voidpf)ss->prev, ds->w_size * sizeof(Pos));
     zmemcpy((voidpf)ds->head, (voidpf)ss->head, ds->hash_size * sizeof(Pos));
-    zmemcpy(ds->pending_buf, ss->pending_buf, (uInt)ds->pending_buf_size);
+    zmemcpy(ds->pending_buf, ss->pending_buf, ds->lit_bufsize * LIT_BUFS);
 
     ds->pending_out = ds->pending_buf + (ss->pending_out - ss->pending_buf);
-    ds->sym_buf = ds->pending_buf + ds->lit_bufsize;
+#ifdef LIT_MEM
+    ds->d_buf = (ushf *)(ds->pending_buf + (ds->lit_bufsize << 1));
+    ds->l_buf = ds->pending_buf + (ds->lit_bufsize << 2);
+#else
+     ds->sym_buf = ds->pending_buf + ds->lit_bufsize;
+#endif
 
     ds->l_desc.dyn_tree = ds->dyn_ltree;
     ds->d_desc.dyn_tree = ds->dyn_dtree;
@@ -1354,8 +1370,6 @@ int ZEXPORT deflateCopy(z_streamp dest, z_streamp source) {
 
 #endif /* NOVGZ */
 
-
-
 #ifndef FASTEST
 /* ===========================================================================
  * Set match_start to the longest match starting at the given string and
@@ -1577,13 +1591,21 @@ local uInt longest_match(deflate_state *s, IPos cur_match) {
  */
 local void check_match(deflate_state *s, IPos start, IPos match, int length) {
     /* check that the match is indeed a match */
-    if (zmemcmp(s->window + match,
-                s->window + start, length) != EQUAL) {
-        fprintf(stderr, " start %u, match %u, length %d\n",
-                start, match, length);
+    Bytef *back = s->window + (int)match, *here = s->window + start;
+    IPos len = length;
+    if (match == (IPos)-1) {
+        /* match starts one byte before the current window -- just compare the
+           subsequent length-1 bytes */
+        back++;
+        here++;
+        len--;
+    }
+    if (zmemcmp(back, here, len) != EQUAL) {
+        fprintf(stderr, " start %u, match %d, length %d\n",
+                start, (int)match, length);
         do {
-            fprintf(stderr, "%c%c", s->window[match++], s->window[start++]);
-        } while (--length != 0);
+            fprintf(stderr, "(%02x %02x)", *back++, *here++);
+        } while (--len != 0);
         z_error("invalid match");
     }
     if (z_verbose > 1) {
@@ -1619,10 +1641,8 @@ local void check_match(deflate_state *s, IPos start, IPos match, int length) {
 /* Maximum stored block length in deflate format (not including header). */
 #define MAX_STORED 65535
 
-#if !defined(MIN)
 /* Minimum of a and b. */
 #define MIN(a, b) ((a) > (b) ? (b) : (a))
-#endif
 
 /* ===========================================================================
  * Copy without compression as much as possible from the input stream, return
@@ -1650,7 +1670,8 @@ local block_state deflate_stored(deflate_state *s, int flush) {
      * possible. If flushing, copy the remaining available input to next_out as
      * stored blocks, if there is enough space.
      */
-    unsigned len, left, have, last = 0;
+    int last = 0;
+    unsigned len, left, have;
     unsigned used = s->strm->avail_in;
     do {
         /* Set len to the maximum size block that we can copy directly with the
@@ -1686,10 +1707,10 @@ local block_state deflate_stored(deflate_state *s, int flush) {
         _tr_stored_block(s, (char *)0, 0L, last);
 
         /* Replace the lengths in the dummy stored block with len. */
-        s->pending_buf[s->pending - 4] = len;
-        s->pending_buf[s->pending - 3] = len >> 8;
-        s->pending_buf[s->pending - 2] = ~len;
-        s->pending_buf[s->pending - 1] = ~len >> 8;
+        s->pending_buf[s->pending - 4] = (Bytef)len;
+        s->pending_buf[s->pending - 3] = (Bytef)(len >> 8);
+        s->pending_buf[s->pending - 2] = (Bytef)~len;
+        s->pending_buf[s->pending - 1] = (Bytef)(~len >> 8);
 
         /* Write the stored block header bytes. */
         flush_pending(s->strm);
diff --git a/lib/libvgz/deflate.h b/lib/libvgz/deflate.h
index 6d9f6945e..720dca7e4 100644
--- a/lib/libvgz/deflate.h
+++ b/lib/libvgz/deflate.h
@@ -1,5 +1,5 @@
 /* deflate.h -- internal compression state
- * Copyright (C) 1995-2018 Jean-loup Gailly
+ * Copyright (C) 1995-2024 Jean-loup Gailly
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
@@ -23,6 +23,10 @@
 #  define GZIP
 #endif
 
+/* define LIT_MEM to slightly increase the speed of deflate (order 1% to 2%) at
+   the cost of a larger memory footprint */
+/* #define LIT_MEM */
+
 /* ===========================================================================
  * Internal compression state.
  */
@@ -217,7 +221,14 @@ typedef struct internal_state {
     /* Depth of each subtree used as tie breaker for trees of equal frequency
      */
 
+#ifdef LIT_MEM
+#   define LIT_BUFS 5
+    ushf *d_buf;          /* buffer for distances */
+    uchf *l_buf;          /* buffer for literals/lengths */
+#else
+#   define LIT_BUFS 4
     uchf *sym_buf;        /* buffer for distances and literals/lengths */
+#endif
 
     uInt  lit_bufsize;
     /* Size of match buffer for literals/lengths.  There are 4 reasons for
@@ -239,7 +250,7 @@ typedef struct internal_state {
      *   - I can't count above 4
      */
 
-    uInt sym_next;      /* running index in sym_buf */
+    uInt sym_next;      /* running index in symbol buffer */
     uInt sym_end;       /* symbol table full when sym_next reaches this */
 
     ulg opt_len;        /* bit length of current block with optimal trees */
@@ -318,6 +329,25 @@ void ZLIB_INTERNAL _tr_stored_block (deflate_state *s, charf *buf,
   extern const uch ZLIB_INTERNAL _dist_code[];
 #endif
 
+#ifdef LIT_MEM
+# define _tr_tally_lit(s, c, flush) \
+  { uch cc = (c); \
+    s->d_buf[s->sym_next] = 0; \
+    s->l_buf[s->sym_next++] = cc; \
+    s->dyn_ltree[cc].Freq++; \
+    flush = (s->sym_next == s->sym_end); \
+   }
+# define _tr_tally_dist(s, distance, length, flush) \
+  { uch len = (uch)(length); \
+    ush dist = (ush)(distance); \
+    s->d_buf[s->sym_next] = dist; \
+    s->l_buf[s->sym_next++] = len; \
+    dist--; \
+    s->dyn_ltree[_length_code[len]+LITERALS+1].Freq++; \
+    s->dyn_dtree[d_code(dist)].Freq++; \
+    flush = (s->sym_next == s->sym_end); \
+  }
+#else
 # define _tr_tally_lit(s, c, flush) \
   { uch cc = (c); \
     s->sym_buf[s->sym_next++] = 0; \
@@ -337,6 +367,7 @@ void ZLIB_INTERNAL _tr_stored_block (deflate_state *s, charf *buf,
     s->dyn_dtree[d_code(dist)].Freq++; \
     flush = (s->sym_next == s->sym_end); \
   }
+#endif
 #else
 # define _tr_tally_lit(s, c, flush) flush = _tr_tally(s, 0, c)
 # define _tr_tally_dist(s, distance, length, flush) \
diff --git a/lib/libvgz/gzguts.h b/lib/libvgz/gzguts.h
index cfde69df5..6c238b52d 100644
--- a/lib/libvgz/gzguts.h
+++ b/lib/libvgz/gzguts.h
@@ -1,5 +1,5 @@
 /* gzguts.h -- zlib internal header definitions for gz* operations
- * Copyright (C) 2004-2019 Mark Adler
+ * Copyright (C) 2004-2024 Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
@@ -17,6 +17,18 @@
 #  define ZLIB_INTERNAL
 #endif
 
+#if defined(_WIN32)
+#  ifndef WIN32_LEAN_AND_MEAN
+#    define WIN32_LEAN_AND_MEAN
+#  endif
+#  ifndef _CRT_SECURE_NO_WARNINGS
+#    define _CRT_SECURE_NO_WARNINGS
+#  endif
+#  ifndef _CRT_NONSTDC_NO_DEPRECATE
+#    define _CRT_NONSTDC_NO_DEPRECATE
+#  endif
+#endif
+
 #include <stdio.h>
 #include "vgz.h"
 #ifdef STDC
@@ -25,8 +37,8 @@
 #  include <limits.h>
 #endif
 
-#ifndef _POSIX_SOURCE
-#  define _POSIX_SOURCE
+#ifndef _POSIX_C_SOURCE
+#  define _POSIX_C_SOURCE 200112L
 #endif
 #include <fcntl.h>
 
@@ -36,19 +48,13 @@
 
 #if defined(__TURBOC__) || defined(_MSC_VER) || defined(_WIN32)
 #  include <io.h>
+#  include <sys/stat.h>
 #endif
 
-#if defined(_WIN32)
+#if defined(_WIN32) && !defined(WIDECHAR)
 #  define WIDECHAR
 #endif
 
-#ifdef WINAPI_FAMILY
-#  define open _open
-#  define read _read
-#  define write _write
-#  define close _close
-#endif
-
 #ifdef NO_DEFLATE       /* for compatibility with old definition */
 #  define NO_GZCOMPRESS
 #endif
@@ -72,33 +78,28 @@
 #endif
 
 #ifndef HAVE_VSNPRINTF
-#  ifdef MSDOS
+#  if !defined(NO_vsnprintf) && \
+      (defined(MSDOS) || defined(__TURBOC__) || defined(__SASC) || \
+       defined(VMS) || defined(__OS400) || defined(__MVS__))
 /* vsnprintf may exist on some MS-DOS compilers (DJGPP?),
    but for now we just assume it doesn't. */
 #    define NO_vsnprintf
 #  endif
-#  ifdef __TURBOC__
-#    define NO_vsnprintf
-#  endif
 #  ifdef WIN32
 /* In Win32, vsnprintf is available as the "non-ANSI" _vsnprintf. */
-#    if !defined(vsnprintf) && !defined(NO_vsnprintf)
-#      if !defined(_MSC_VER) || ( defined(_MSC_VER) && _MSC_VER < 1500 )
-#         define vsnprintf _vsnprintf
+#    if !defined(_MSC_VER) || ( defined(_MSC_VER) && _MSC_VER < 1500 )
+#      ifndef vsnprintf
+#        define vsnprintf _vsnprintf
 #      endif
 #    endif
-#  endif
-#  ifdef __SASC
-#    define NO_vsnprintf
-#  endif
-#  ifdef VMS
-#    define NO_vsnprintf
-#  endif
-#  ifdef __OS400__
-#    define NO_vsnprintf
-#  endif
-#  ifdef __MVS__
-#    define NO_vsnprintf
+#  elif !defined(__STDC_VERSION__) || __STDC_VERSION__-0 < 199901L
+/* Otherwise if C89/90, assume no C99 snprintf() or vsnprintf() */
+#    ifndef NO_snprintf
+#      define NO_snprintf
+#    endif
+#    ifndef NO_vsnprintf
+#      define NO_vsnprintf
+#    endif
 #  endif
 #endif
 
@@ -210,9 +211,5 @@ char ZLIB_INTERNAL *gz_strwinerror (DWORD error);
 /* GT_OFF(x), where x is an unsigned value, is true if x > maximum z_off64_t
    value -- needed when comparing unsigned to z_off64_t, which is signed
    (possible z_off64_t types off_t, off64_t, and long are all signed) */
-#ifdef INT_MAX
-#  define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > INT_MAX)
-#else
 unsigned ZLIB_INTERNAL gz_intmax (void);
 #  define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > gz_intmax())
-#endif
diff --git a/lib/libvgz/inflate.c b/lib/libvgz/inflate.c
index deae8231f..07a5f36c9 100644
--- a/lib/libvgz/inflate.c
+++ b/lib/libvgz/inflate.c
@@ -1399,7 +1399,7 @@ int ZEXPORT inflateSync(z_streamp strm) {
     /* if first time, start search in bit buffer */
     if (state->mode != SYNC) {
         state->mode = SYNC;
-        state->hold <<= state->bits & 7;
+        state->hold >>= state->bits & 7;
         state->bits -= state->bits & 7;
         len = 0;
         while (state->bits >= 8) {
diff --git a/lib/libvgz/inflate.h b/lib/libvgz/inflate.h
index f127b6b1f..f758e0dcc 100644
--- a/lib/libvgz/inflate.h
+++ b/lib/libvgz/inflate.h
@@ -100,7 +100,7 @@ struct inflate_state {
     unsigned char FAR *window;  /* allocated sliding window, if needed */
         /* bit accumulator */
     unsigned long hold;         /* input bit accumulator */
-    unsigned bits;              /* number of bits in "in" */
+    unsigned bits;              /* number of bits in hold */
         /* for string and stored block copying */
     unsigned length;            /* literal or length of data to copy */
     unsigned offset;            /* distance back to copy string from */
diff --git a/lib/libvgz/inftrees.c b/lib/libvgz/inftrees.c
index a6a8a9edc..fc15f4676 100644
--- a/lib/libvgz/inftrees.c
+++ b/lib/libvgz/inftrees.c
@@ -1,5 +1,5 @@
 /* inftrees.c -- generate Huffman trees for efficient decoding
- * Copyright (C) 1995-2023 Mark Adler
+ * Copyright (C) 1995-2024 Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
@@ -10,7 +10,7 @@
 
 extern const char inflate_copyright[];
 const char inflate_copyright[] =
-   " inflate 1.3 Copyright 1995-2023 Mark Adler ";
+   " inflate 1.3.1.1 Copyright 1995-2024 Mark Adler ";
 /*
   If you use the zlib library in a product, an acknowledgment is welcome
   in the documentation of your product. If for some reason you cannot
@@ -58,7 +58,7 @@ int ZLIB_INTERNAL inflate_table(codetype type, unsigned short FAR *lens,
         35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0};
     static const unsigned short lext[31] = { /* Length codes 257..285 extra */
         16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18,
-        19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 198, 203};
+        19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 73, 200};
     static const unsigned short dbase[32] = { /* Distance codes 0..29 base */
         1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193,
         257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145,
diff --git a/lib/libvgz/inftrees.h b/lib/libvgz/inftrees.h
index c96d3b62a..4664eaee9 100644
--- a/lib/libvgz/inftrees.h
+++ b/lib/libvgz/inftrees.h
@@ -41,8 +41,8 @@ typedef struct {
    examples/enough.c found in the zlib distribution.  The arguments to that
    program are the number of symbols, the initial root table size, and the
    maximum bit length of a code.  "enough 286 9 15" for literal/length codes
-   returns returns 852, and "enough 30 6 15" for distance codes returns 592.
-   The initial root table size (9 or 6) is found in the fifth argument of the
+   returns 852, and "enough 30 6 15" for distance codes returns 592. The
+   initial root table size (9 or 6) is found in the fifth argument of the
    inflate_table() calls in inflate.c and infback.c.  If the root table size is
    changed, then these maximum sizes would be need to be recalculated and
    updated. */
diff --git a/lib/libvgz/trees.c b/lib/libvgz/trees.c
index 51fdadba8..bbaa24870 100644
--- a/lib/libvgz/trees.c
+++ b/lib/libvgz/trees.c
@@ -1,5 +1,5 @@
 /* trees.c -- output deflated data using Huffman coding
- * Copyright (C) 1995-2021 Jean-loup Gailly
+ * Copyright (C) 1995-2024 Jean-loup Gailly
  * detect_data_type() function provided freely by Cosmin Truta, 2006
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
@@ -724,7 +724,7 @@ local void scan_tree(deflate_state *s, ct_data *tree, int max_code) {
         if (++count < max_count && curlen == nextlen) {
             continue;
         } else if (count < min_count) {
-            s->bl_tree[curlen].Freq += count;
+            s->bl_tree[curlen].Freq += (ush)count;
         } else if (curlen != 0) {
             if (curlen != prevlen) s->bl_tree[curlen].Freq++;
             s->bl_tree[REP_3_6].Freq++;
@@ -906,14 +906,19 @@ local void compress_block(deflate_state *s, const ct_data *ltree,
                           const ct_data *dtree) {
     unsigned dist;      /* distance of matched string */
     int lc;             /* match length or unmatched char (if dist == 0) */
-    unsigned sx = 0;    /* running index in sym_buf */
+    unsigned sx = 0;    /* running index in symbol buffers */
     unsigned code;      /* the code to send */
     int extra;          /* number of extra bits to send */
 
     if (s->sym_next != 0) do {
+#ifdef LIT_MEM
+        dist = s->d_buf[sx];
+        lc = s->l_buf[sx++];
+#else
         dist = s->sym_buf[sx++] & 0xff;
         dist += (unsigned)(s->sym_buf[sx++] & 0xff) << 8;
         lc = s->sym_buf[sx++];
+#endif
         if (dist == 0) {
             send_code(s, lc, ltree); /* send a literal byte */
             Tracecv(isgraph(lc), (stderr," '%c' ", lc));
@@ -938,8 +943,12 @@ local void compress_block(deflate_state *s, const ct_data *ltree,
             }
         } /* literal or match pair ? */
 
-        /* Check that the overlay between pending_buf and sym_buf is ok: */
+        /* Check for no overlay of pending_buf on needed symbols */
+#ifdef LIT_MEM
+        Assert(s->pending < 2 * (s->lit_bufsize + sx), "pendingBuf overflow");
+#else
         Assert(s->pending < s->lit_bufsize + sx, "pendingBuf overflow");
+#endif
 
     } while (sx < s->sym_next);
 
@@ -1095,9 +1104,14 @@ void ZLIB_INTERNAL _tr_flush_block(deflate_state *s, charf *buf,
  * the current block must be flushed.
  */
 int ZLIB_INTERNAL _tr_tally(deflate_state *s, unsigned dist, unsigned lc) {
+#ifdef LIT_MEM
+    s->d_buf[s->sym_next] = (ush)dist;
+    s->l_buf[s->sym_next++] = (uch)lc;
+#else
     s->sym_buf[s->sym_next++] = (uch)dist;
     s->sym_buf[s->sym_next++] = (uch)(dist >> 8);
     s->sym_buf[s->sym_next++] = (uch)lc;
+#endif
     if (dist == 0) {
         /* lc is the unmatched char */
         s->dyn_ltree[lc].Freq++;
diff --git a/lib/libvgz/vgz.h b/lib/libvgz/vgz.h
index b423e9efd..97b140531 100644
--- a/lib/libvgz/vgz.h
+++ b/lib/libvgz/vgz.h
@@ -1,7 +1,7 @@
 /* zlib.h -- interface of the 'zlib' general purpose compression library
-  version 1.3, August 18th, 2023
+  version 1.3.1.1, January xxth, 2024
 
-  Copyright (C) 1995-2023 Jean-loup Gailly and Mark Adler
+  Copyright (C) 1995-2024 Jean-loup Gailly and Mark Adler
 
   This software is provided 'as-is', without any express or implied
   warranty.  In no event will the authors be held liable for any damages
@@ -37,12 +37,12 @@
 extern "C" {
 #endif
 
-#define ZLIB_VERSION "1.3"
-#define ZLIB_VERNUM 0x1300
+#define ZLIB_VERSION "1.3.1.1-motley"
+#define ZLIB_VERNUM 0x1311
 #define ZLIB_VER_MAJOR 1
 #define ZLIB_VER_MINOR 3
-#define ZLIB_VER_REVISION 0
-#define ZLIB_VER_SUBREVISION 0
+#define ZLIB_VER_REVISION 1
+#define ZLIB_VER_SUBREVISION 1
 
 /*
     The 'zlib' compression library provides in-memory compression and
@@ -592,18 +592,21 @@ ZEXTERN int ZEXPORT deflateInit2 (z_streamp strm,
 
      The strategy parameter is used to tune the compression algorithm.  Use the
    value Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data produced by a
-   filter (or predictor), Z_HUFFMAN_ONLY to force Huffman encoding only (no
-   string match), or Z_RLE to limit match distances to one (run-length
-   encoding).  Filtered data consists mostly of small values with a somewhat
-   random distribution.  In this case, the compression algorithm is tuned to
-   compress them better.  The effect of Z_FILTERED is to force more Huffman
-   coding and less string matching; it is somewhat intermediate between
-   Z_DEFAULT_STRATEGY and Z_HUFFMAN_ONLY.  Z_RLE is designed to be almost as
-   fast as Z_HUFFMAN_ONLY, but give better compression for PNG image data.  The
-   strategy parameter only affects the compression ratio but not the
-   correctness of the compressed output even if it is not set appropriately.
-   Z_FIXED prevents the use of dynamic Huffman codes, allowing for a simpler
-   decoder for special applications.
+   filter (or predictor), Z_RLE to limit match distances to one (run-length
+   encoding), or Z_HUFFMAN_ONLY to force Huffman encoding only (no string
+   matching).  Filtered data consists mostly of small values with a somewhat
+   random distribution, as produced by the PNG filters.  In this case, the
+   compression algorithm is tuned to compress them better.  The effect of
+   Z_FILTERED is to force more Huffman coding and less string matching than the
+   default; it is intermediate between Z_DEFAULT_STRATEGY and Z_HUFFMAN_ONLY.
+   Z_RLE is almost as fast as Z_HUFFMAN_ONLY, but should give better
+   compression for PNG image data than Huffman only.  The degree of string
+   matching from most to none is: Z_DEFAULT_STRATEGY, Z_FILTERED, Z_RLE, then
+   Z_HUFFMAN. The strategy parameter affects the compression ratio but never
+   the correctness of the compressed output, even if it is not set optimally
+   for the given data.  Z_FIXED uses the default string matching, but prevents
+   the use of dynamic Huffman codes, allowing for a simpler decoder for special
+   applications.
 
      deflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough
    memory, Z_STREAM_ERROR if any parameter is invalid (such as an invalid
@@ -941,10 +944,10 @@ ZEXTERN int ZEXPORT inflateSync (z_streamp strm);
      inflateSync returns Z_OK if a possible full flush point has been found,
    Z_BUF_ERROR if no more input was provided, Z_DATA_ERROR if no flush point
    has been found, or Z_STREAM_ERROR if the stream structure was inconsistent.
-   In the success case, the application may save the current current value of
-   total_in which indicates where valid compressed data was found.  In the
-   error case, the application may repeatedly call inflateSync, providing more
-   input each time, until success or end of the input data.
+   In the success case, the application may save the current value of total_in
+   which indicates where valid compressed data was found.  In the error case,
+   the application may repeatedly call inflateSync, providing more input each
+   time, until success or end of the input data.
 */
 
 ZEXTERN int ZEXPORT inflateCopy (z_streamp dest,
@@ -1763,14 +1766,14 @@ ZEXTERN uLong ZEXPORT crc32_combine (uLong crc1, uLong crc2, z_off_t len2);
    seq1 and seq2 with lengths len1 and len2, CRC-32 check values were
    calculated for each, crc1 and crc2.  crc32_combine() returns the CRC-32
    check value of seq1 and seq2 concatenated, requiring only crc1, crc2, and
-   len2.
+   len2. len2 must be non-negative.
 */
 
 /*
 ZEXTERN uLong ZEXPORT crc32_combine_gen (z_off_t len2);
 
      Return the operator corresponding to length len2, to be used with
-   crc32_combine_op().
+   crc32_combine_op(). len2 must be non-negative.
 */
 
 ZEXTERN uLong ZEXPORT crc32_combine_op (uLong crc1, uLong crc2, uLong op);
@@ -1893,9 +1896,9 @@ ZEXTERN int ZEXPORT gzgetc_ (gzFile file);  /* backward compatibility */
      ZEXTERN z_off_t ZEXPORT gzseek64 (gzFile, z_off_t, int);
      ZEXTERN z_off_t ZEXPORT gztell64 (gzFile);
      ZEXTERN z_off_t ZEXPORT gzoffset64 (gzFile);
-     ZEXTERN uLong ZEXPORT adler32_combine64 (uLong, uLong, z_off_t);
-     ZEXTERN uLong ZEXPORT crc32_combine64 (uLong, uLong, z_off_t);
-     ZEXTERN uLong ZEXPORT crc32_combine_gen64 (z_off_t);
+     ZEXTERN uLong ZEXPORT adler32_combine64 (uLong, uLong, z_off64_t);
+     ZEXTERN uLong ZEXPORT crc32_combine64 (uLong, uLong, z_off64_t);
+     ZEXTERN uLong ZEXPORT crc32_combine_gen64 (z_off64_t);
 #  endif
 #else
    ZEXTERN gzFile ZEXPORT gzopen (const char *, const char *);
diff --git a/lib/libvgz/zconf.h b/lib/libvgz/zconf.h
index fb76ffe31..ea3b6d685 100644
--- a/lib/libvgz/zconf.h
+++ b/lib/libvgz/zconf.h
@@ -1,5 +1,5 @@
 /* zconf.h -- configuration of the zlib compression library
- * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler
+ * Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
@@ -300,14 +300,6 @@
 #  endif
 #endif
 
-#ifndef Z_ARG /* function prototypes for stdarg */
-#  if defined(STDC) || defined(Z_HAVE_STDARG_H)
-#    define Z_ARG(args)  args
-#  else
-#    define Z_ARG(args)  ()
-#  endif
-#endif
-
 /* The following definitions for FAR are needed only for MSDOS mixed
  * model programming (small or medium model with some far allocations).
  * This was tested only with MSC; for other MSDOS compilers you may have
@@ -478,12 +470,8 @@ typedef uLong FAR uLongf;
 #endif
 
 #ifndef Z_HAVE_UNISTD_H
-#  ifdef __WATCOMC__
-#    define Z_HAVE_UNISTD_H
-#  endif
-#endif
-#ifndef Z_HAVE_UNISTD_H
-#  if defined(_LARGEFILE64_SOURCE) && !defined(_WIN32)
+#  if defined(__WATCOMC__) || defined(__GO32__) || \
+      (defined(_LARGEFILE64_SOURCE) && !defined(_WIN32))
 #    define Z_HAVE_UNISTD_H
 #  endif
 #endif
@@ -523,12 +511,14 @@ typedef uLong FAR uLongf;
 
 #if !defined(_WIN32) && defined(Z_LARGE64)
 #  define z_off64_t off64_t
+#elif defined(__MINGW32__)
+#  define z_off64_t long long
+#elif defined(_WIN32) && !defined(__GNUC__)
+#  define z_off64_t __int64
+#elif defined(__GO32__)
+#  define z_off64_t offset_t
 #else
-#  if defined(_WIN32) && !defined(__GNUC__)
-#    define z_off64_t __int64
-#  else
-#    define z_off64_t z_off_t
-#  endif
+#  define z_off64_t z_off_t
 #endif
 
 /* MVS linker does not support external names larger than 8 bytes */
diff --git a/lib/libvgz/zutil.h b/lib/libvgz/zutil.h
index 1ae739f15..a74201f02 100644
--- a/lib/libvgz/zutil.h
+++ b/lib/libvgz/zutil.h
@@ -1,5 +1,5 @@
 /* zutil.h -- internal interface and configuration of the compression library
- * Copyright (C) 1995-2022 Jean-loup Gailly, Mark Adler
+ * Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler
  * For conditions of distribution and use, see copyright notice in zlib.h
  */
 
@@ -56,7 +56,7 @@ typedef unsigned long  ulg;
 extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */
 /* (size given to avoid silly warnings with Visual C++) */
 
-#define ERR_MSG(err) z_errmsg[Z_NEED_DICT-(err)]
+#define ERR_MSG(err) z_errmsg[(err) < -6 || (err) > 2 ? 9 : 2 - (err)]
 
 #define ERR_RETURN(strm,err) \
   return (strm->msg = ERR_MSG(err), (err))
@@ -137,20 +137,11 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */
 #  endif
 #endif
 
-#if defined(MACOS) || defined(TARGET_OS_MAC)
+#if defined(MACOS)
 #  define OS_CODE  7
-#  ifndef Z_SOLO
-#    if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os
-#      include <unix.h> /* for fdopen */
-#    else
-#      ifndef fdopen
-#        define fdopen(fd,mode) NULL /* No fdopen() */
-#      endif
-#    endif
-#  endif
 #endif
 
-#ifdef __acorn
+#if defined(__acorn) || defined(__riscos)
 #  define OS_CODE 13
 #endif
 
@@ -170,18 +161,6 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */
 #  define OS_CODE 19
 #endif
 
-#if defined(_BEOS_) || defined(RISCOS)
-#  define fdopen(fd,mode) NULL /* No fdopen() */
-#endif
-
-#if (defined(_MSC_VER) && (_MSC_VER > 600)) && !defined __INTERIX
-#  if defined(_WIN32_WCE)
-#    define fdopen(fd,mode) NULL /* No fdopen() */
-#  else
-#    define fdopen(fd,type)  _fdopen(fd,type)
-#  endif
-#endif
-
 #if defined(__BORLANDC__) && !defined(MSDOS)
   #pragma warn -8004
   #pragma warn -8008
@@ -189,11 +168,10 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */
 #endif
 
 /* provide prototypes for these when building zlib without LFS */
-#if !defined(_WIN32) && \
-    (!defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0)
-    ZEXTERN uLong ZEXPORT adler32_combine64 (uLong, uLong, z_off_t);
-    ZEXTERN uLong ZEXPORT crc32_combine64 (uLong, uLong, z_off_t);
-    ZEXTERN uLong ZEXPORT crc32_combine_gen64 (z_off_t);
+#ifndef Z_LARGE64
+    ZEXTERN uLong ZEXPORT adler32_combine64 (uLong, uLong, z_off64_t);
+    ZEXTERN uLong ZEXPORT crc32_combine64 (uLong, uLong, z_off64_t);
+    ZEXTERN uLong ZEXPORT crc32_combine_gen64 (z_off64_t);
 #endif
 
         /* common defaults */

From phk at FreeBSD.org  Mon May 13 12:23:07 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 13 May 2024 12:23:07 +0000 (UTC)
Subject: [master] cc4a46bb2 zlib/vgz no longer uses K&R function arguments
Message-ID: <20240513122307.9B16C111BD9@lists.varnish-cache.org>


commit cc4a46bb2cb23ae2b69b23f732a88e77d3308011
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 13 10:30:19 2024 +0000

    zlib/vgz no longer uses K&R function arguments

diff --git a/wflags.py b/wflags.py
index 10cfd68fd..f9a5bd570 100644
--- a/wflags.py
+++ b/wflags.py
@@ -71,7 +71,6 @@ DESIRABLE_WFLAGS = [
 UNDESIRABLE_WFLAGS = [
     "-Wno-system-headers", # Outside of our control
     "-Wno-thread-safety", # Does not understand our mutexs are wrapped
-    "-Wno-old-style-definition", # Does not like vgz
     "-Wno-sign-compare", # Fixable
     "-Wno-implicit-fallthrough", # Probably Fixable
     "-Wno-missing-variable-declarations", # Complains about optreset

From phk at FreeBSD.org  Mon May 13 12:23:07 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 13 May 2024 12:23:07 +0000 (UTC)
Subject: [master] 05a3d5ccf Spell "FALLTHROUGH" consistently and enable
 compiler warning for it.
Message-ID: <20240513122307.AF771111BDD@lists.varnish-cache.org>


commit 05a3d5ccf559ee1d2d9049bfda7db92ecb06f507
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 13 11:18:05 2024 +0000

    Spell "FALLTHROUGH" consistently and enable compiler warning for it.

diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c
index 90d588e7b..a91c9f862 100644
--- a/bin/varnishd/cache/cache_rfc2616.c
+++ b/bin/varnishd/cache/cache_rfc2616.c
@@ -151,7 +151,7 @@ RFC2616_Ttl(struct busyobj *bo, vtim_real now, vtim_real *t_origin,
 		 * or Expires are present. Uncacheable otherwise.
 		 */
 		*ttl = -1.;
-		/* FALL-THROUGH */
+		/* FALLTHROUGH */
 	case 200: /* OK */
 	case 203: /* Non-Authoritative Information */
 	case 204: /* No Content */
diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c
index 5f6f34cf7..c49f9d2b9 100644
--- a/bin/varnishd/http2/cache_http2_hpack.c
+++ b/bin/varnishd/http2/cache_http2_hpack.c
@@ -80,7 +80,7 @@ h2h_checkhdr(struct vsl_log *vsl, txt nm, txt val)
 			state = FLD_NAME;
 			if (*p == ':')
 				break;
-			/* FALL_THROUGH */
+			/* FALLTHROUGH */
 		case FLD_NAME:
 			if (isupper(*p)) {
 				VSLb(vsl, SLT_BogoHeader,
@@ -110,7 +110,7 @@ h2h_checkhdr(struct vsl_log *vsl, txt nm, txt val)
 				return (H2SE_PROTOCOL_ERROR);
 			}
 			state = FLD_VALUE;
-			/* FALL_THROUGH */
+			/* FALLTHROUGH */
 		case FLD_VALUE:
 			if (!vct_ishdrval(*p)) {
 				VSLb(vsl, SLT_BogoHeader,
diff --git a/wflags.py b/wflags.py
index f9a5bd570..6eb8d112d 100644
--- a/wflags.py
+++ b/wflags.py
@@ -72,7 +72,6 @@ UNDESIRABLE_WFLAGS = [
     "-Wno-system-headers", # Outside of our control
     "-Wno-thread-safety", # Does not understand our mutexs are wrapped
     "-Wno-sign-compare", # Fixable
-    "-Wno-implicit-fallthrough", # Probably Fixable
     "-Wno-missing-variable-declarations", # Complains about optreset
     "-Wno-nullability-completeness", # Barfs all over MacOSx
 ]

From phk at FreeBSD.org  Mon May 13 12:23:07 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 13 May 2024 12:23:07 +0000 (UTC)
Subject: [master] 21e67bd31 Remove OBE -Wno- options
Message-ID: <20240513122307.CC721111BE1@lists.varnish-cache.org>


commit 21e67bd31c9c683d3df213e89c6ae0bc02cc930e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 13 11:27:26 2024 +0000

    Remove OBE -Wno- options

diff --git a/wflags.py b/wflags.py
index 6eb8d112d..85598719d 100644
--- a/wflags.py
+++ b/wflags.py
@@ -72,8 +72,6 @@ UNDESIRABLE_WFLAGS = [
     "-Wno-system-headers", # Outside of our control
     "-Wno-thread-safety", # Does not understand our mutexs are wrapped
     "-Wno-sign-compare", # Fixable
-    "-Wno-missing-variable-declarations", # Complains about optreset
-    "-Wno-nullability-completeness", # Barfs all over MacOSx
 ]
 
 

From phk at FreeBSD.org  Tue May 14 08:18:06 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Tue, 14 May 2024 08:18:06 +0000 (UTC)
Subject: [master] e4a0dbcc2 Typo
Message-ID: <20240514081806.DD4A5110B15@lists.varnish-cache.org>


commit e4a0dbcc2b387457c2baba129ac5560da07105e6
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Tue May 14 08:17:17 2024 +0000

    Typo

diff --git a/bin/varnishd/cache/cache_lck.c b/bin/varnishd/cache/cache_lck.c
index 5c7ab42ce..029ecc8c9 100644
--- a/bin/varnishd/cache/cache_lck.c
+++ b/bin/varnishd/cache/cache_lck.c
@@ -257,7 +257,7 @@ Lck_CondWaitUntil(pthread_cond_t *cond, struct lock *lck, vtim_real when)
 		 * So far I have yet to see a failure if the exact same
 		 * call is repeated after a very short sleep.
 		 *
-		 * Calling pthread_yield_np() instead of sleaping /mostly/
+		 * Calling pthread_yield_np() instead of sleeping /mostly/
 		 * works as well, but still fails sometimes.
 		 *
 		 * Env:

From dridi.boukelmoune at gmail.com  Fri May 17 09:20:12 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 17 May 2024 09:20:12 +0000 (UTC)
Subject: [master] 328f3ab34 cocci: Match miniobj assertions in panic code
Message-ID: <20240517092012.81E426F9A@lists.varnish-cache.org>


commit 328f3ab3465ab7609ace8e2dcff267b8d769c27d
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri May 17 11:04:27 2024 +0200

    cocci: Match miniobj assertions in panic code

diff --git a/tools/coccinelle/panic_assert.cocci b/tools/coccinelle/panic_assert.cocci
new file mode 100644
index 000000000..760c54a0c
--- /dev/null
+++ b/tools/coccinelle/panic_assert.cocci
@@ -0,0 +1,42 @@
+/*
+ * This patch replaces a non-exhaustive list of statements designed to trigger
+ * assertions with something more appropriate for panic code.
+ *
+ * The heuristics for panic functions include:
+ * - something panic-related in the name
+ * - taking a VSB
+ * - returning void
+ */
+
+ at panic@
+identifier sb, func =~ "(^PAN_|^pan_|_panic$)";
+@@
+
+void func(..., struct vsb *sb, ...) { ... }
+
+@@
+identifier panic.sb, panic.func;
+expression obj, magicval;
+@@
+
+func(...)
+{
+...
+- CHECK_OBJ_NOTNULL(obj, magicval);
++ PAN_CheckMagic(sb, obj, magicval);
+...
+}
+
+@@
+identifier panic.sb, panic.func;
+expression obj, from, magicval;
+@@
+
+func(...)
+{
+...
+- CAST_OBJ_NOTNULL(obj, from, magicval);
++ obj = from;
++ PAN_CheckMagic(sb, obj, magicval);
+...
+}

From dridi.boukelmoune at gmail.com  Fri May 17 09:20:12 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 17 May 2024 09:20:12 +0000 (UTC)
Subject: [master] 7e4442d92 vbe: Do not assert in panic code
Message-ID: <20240517092012.9A62B6F9C@lists.varnish-cache.org>


commit 7e4442d926818db5c8e34c912df42de822802c23
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu May 16 11:55:02 2024 +0200

    vbe: Do not assert in panic code

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 935527015..978d50c5d 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -474,8 +474,9 @@ vbe_panic(const struct director *d, struct vsb *vsb)
 {
 	struct backend *bp;
 
-	CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC);
-	CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC);
+	PAN_CheckMagic(vsb, d, DIRECTOR_MAGIC);
+	bp = d->priv;
+	PAN_CheckMagic(vsb, bp, BACKEND_MAGIC);
 
 	VCP_Panic(vsb, bp->conn_pool);
 	VSB_printf(vsb, "hosthdr = %s,\n", bp->hosthdr);

From dridi.boukelmoune at gmail.com  Fri May 17 09:20:12 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 17 May 2024 09:20:12 +0000 (UTC)
Subject: [master] b6118c3b8 sml: Do not assert in panic code
Message-ID: <20240517092012.BA5806F9F@lists.varnish-cache.org>


commit b6118c3b8cacdcaec0f1bc3209d934a04154b5b0
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri May 17 07:39:38 2024 +0200

    sml: Do not assert in panic code

diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c
index c92304593..ba658a8f7 100644
--- a/bin/varnishd/storage/storage_simple.c
+++ b/bin/varnishd/storage/storage_simple.c
@@ -748,7 +748,8 @@ SML_panic(struct vsb *vsb, const struct objcore *oc)
 	VSB_printf(vsb, "Simple = %p,\n", oc->stobj->priv);
 	if (oc->stobj->priv == NULL)
 		return;
-	CAST_OBJ_NOTNULL(o, oc->stobj->priv, OBJECT_MAGIC);
+	o = oc->stobj->priv;
+	PAN_CheckMagic(vsb, o, OBJECT_MAGIC);
 	sml_panic_st(vsb, "Obj", o->objstore);
 
 #define OBJ_FIXATTR(U, l, sz) \

From dridi.boukelmoune at gmail.com  Fri May 17 09:33:05 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 17 May 2024 09:33:05 +0000 (UTC)
Subject: [master] 648a08570 panic: Bring back PAN_CheckMagic()
Message-ID: <20240517093305.79A9C7BAE@lists.varnish-cache.org>


commit 648a08570f335faf55bd41d5f88bd161707ef5bc
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri May 17 11:30:43 2024 +0200

    panic: Bring back PAN_CheckMagic()
    
    It turns out we still need it in a couple places.

diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index 61d3145c9..add300e76 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -375,6 +375,14 @@ int PAN__DumpStruct(struct vsb *vsb, int block, int track, const void *ptr,
     const char *smagic, unsigned magic, const char *fmt, ...)
     v_printflike_(7,8);
 
+#define PAN_CheckMagic(vsb, ptr, exp)					\
+	do {								\
+		if ((ptr)->magic != (exp))				\
+			VSB_printf((vsb),				\
+			    "MAGIC at %p is 0x%08x (Should be: %s/0x%08x)\n", \
+			    ptr, (ptr)->magic, #exp, exp);		\
+	} while(0)
+
 #define PAN_dump_struct(vsb, ptr, magic, ...)		\
     PAN__DumpStruct(vsb, 1, 1, ptr, #magic, magic, __VA_ARGS__)
 

From phk at FreeBSD.org  Mon May 20 05:45:11 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 20 May 2024 05:45:11 +0000 (UTC)
Subject: [master] 5a60b36fe Always use <endian.h> in vsha256
Message-ID: <20240520054511.688DA112268@lists.varnish-cache.org>


commit 5a60b36fe9b45b68244596d93db2b922d53e7b5c
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 20 05:44:35 2024 +0000

    Always use <endian.h> in vsha256

diff --git a/configure.ac b/configure.ac
index e52189329..dee2d9fbd 100644
--- a/configure.ac
+++ b/configure.ac
@@ -211,7 +211,6 @@ AC_CHECK_HEADERS([ncursesw/curses.h ncursesw.h ncurses/curses.h ncurses.h curses
 CFLAGS="${save_CFLAGS}"
 
 # Checks for header files.
-AC_CHECK_HEADERS([sys/endian.h])
 AC_CHECK_HEADERS([sys/filio.h])
 AC_CHECK_HEADERS([sys/personality.h])
 AC_CHECK_HEADERS([endian.h])
diff --git a/lib/libvarnish/vsha256.c b/lib/libvarnish/vsha256.c
index 5f16fe342..e4405f718 100644
--- a/lib/libvarnish/vsha256.c
+++ b/lib/libvarnish/vsha256.c
@@ -31,16 +31,9 @@
 
 #include "config.h"
 
-#if defined(HAVE_SYS_ENDIAN_H)
-#  include <sys/types.h>
-#  include <sys/endian.h>
-#  define VBYTE_ORDER	_BYTE_ORDER
-#  define VBIG_ENDIAN	_BIG_ENDIAN
-#elif defined(HAVE_ENDIAN_H)
-#  include <endian.h>
-#  define VBYTE_ORDER	__BYTE_ORDER
-#  define VBIG_ENDIAN	__BIG_ENDIAN
-#endif
+#include <endian.h>
+#define VBYTE_ORDER	__BYTE_ORDER
+#define VBIG_ENDIAN	__BIG_ENDIAN
 
 #include <stdint.h>
 #include <string.h>

From phk at FreeBSD.org  Mon May 20 06:21:06 2024
From: phk at FreeBSD.org (Poul-Henning Kamp)
Date: Mon, 20 May 2024 06:21:06 +0000 (UTC)
Subject: [master] 4eb20126a On OS/X we get endianess from <sys/types.h>
Message-ID: <20240520062106.B5FAE1135CF@lists.varnish-cache.org>


commit 4eb20126a5824cc479abb51ebf60ae4199674c4e
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Mon May 20 06:20:25 2024 +0000

    On OS/X we get endianess from <sys/types.h>

diff --git a/configure.ac b/configure.ac
index dee2d9fbd..21845879d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -213,7 +213,6 @@ CFLAGS="${save_CFLAGS}"
 # Checks for header files.
 AC_CHECK_HEADERS([sys/filio.h])
 AC_CHECK_HEADERS([sys/personality.h])
-AC_CHECK_HEADERS([endian.h])
 AC_CHECK_HEADERS([pthread_np.h], [], [], [#include <pthread.h>])
 AC_CHECK_HEADERS([priv.h])
 AC_CHECK_HEADERS([fnmatch.h], [], [AC_MSG_ERROR([fnmatch.h is required])])
diff --git a/lib/libvarnish/vsha256.c b/lib/libvarnish/vsha256.c
index e4405f718..8d92abc8c 100644
--- a/lib/libvarnish/vsha256.c
+++ b/lib/libvarnish/vsha256.c
@@ -31,9 +31,16 @@
 
 #include "config.h"
 
-#include <endian.h>
-#define VBYTE_ORDER	__BYTE_ORDER
-#define VBIG_ENDIAN	__BIG_ENDIAN
+#include <sys/types.h>
+
+#ifndef __DARWIN_BYTE_ORDER
+#  include <endian.h>
+#  define VBYTE_ORDER	__BYTE_ORDER
+#  define VBIG_ENDIAN	__BIG_ENDIAN
+#else
+#  define VBYTE_ORDER	__DARWIN_BYTE_ORDER
+#  define VBIG_ENDIAN	__DARWIN_BIG_ENDIAN
+#endif
 
 #include <stdint.h>
 #include <string.h>

From nils.goroll at uplex.de  Tue May 21 12:32:06 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 21 May 2024 12:32:06 +0000 (UTC)
Subject: [master] 7bccded24 Document a pitfall with ObjWaitExtend()
Message-ID: <20240521123206.4FFAE1259E8@lists.varnish-cache.org>


commit 7bccded24df0efd9200f44be111b8eacd486e5de
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue May 21 14:27:03 2024 +0200

    Document a pitfall with ObjWaitExtend()
    
    Maybe it should gain a boc_state_e * argument to return a consistent
    state value?

diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c
index ba658a8f7..6626d3a0d 100644
--- a/bin/varnishd/storage/storage_simple.c
+++ b/bin/varnishd/storage/storage_simple.c
@@ -368,6 +368,13 @@ sml_iterator(struct worker *wrk, struct objcore *oc,
 			break;
 		}
 		if (nl == ol) {
+			/*
+			 * note: the unguarded boc->state read could be
+			 * outdated, in which case we call ObjWaitExtend() again
+			 * for error handling but otherwise cause no harm. When
+			 * using this code as an example, DO NOT rely on
+			 * boc->state to be consistent
+			 */
 			if (boc->state == BOS_FINISHED)
 				break;
 			continue;

From nils.goroll at uplex.de  Fri May 24 07:18:07 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 24 May 2024 07:18:07 +0000 (UTC)
Subject: [master] 725c9d56d Start NEXT sections in vrt.h and changes.rst
Message-ID: <20240524071807.9ACD9111621@lists.varnish-cache.org>


commit 725c9d56d42f23306b993663563821cd07b09fdf
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri May 24 09:16:37 2024 +0200

    Start NEXT sections in vrt.h and changes.rst
    
    to anchor patches

diff --git a/doc/changes.rst b/doc/changes.rst
index aeda844ff..e41815083 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -34,13 +34,17 @@ http://varnish-cache.org/docs/trunk/whats-new/index.html and via
 individual releases. These documents are updated as part of the
 release process.
 
-================================
-Varnish Cache 7.5.0 (2024-03-18)
-================================
+===============================
+Varnish Cache NEXT (2024-09-15)
+===============================
 
 .. PLEASE keep this roughly in commit order as shown by git-log / tig
    (new to old)
 
+================================
+Varnish Cache 7.5.0 (2024-03-18)
+================================
+
 * Add ``h2_window_timeout`` paramater to mitigate CVE-2023-43622 (VSV00014_).
 
 * The parameters ``idle_send_timeout`` and ``timeout_idle`` are now
diff --git a/include/vrt.h b/include/vrt.h
index 0f1df8984..a7ed22bab 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -57,6 +57,7 @@
  * Whenever something is deleted or changed in a way which is not
  * binary/load-time compatible, increment MAJOR version
  *
+ * NEXT (2024-09-15)
  * 19.0 (2024-03-18)
  *	[cache.h] (struct req).filter_list renamed to vdp_filter_list
  *	order of vcl/vmod and director COLD events reversed to directors first

From nils.goroll at uplex.de  Fri May 24 09:17:11 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 24 May 2024 09:17:11 +0000 (UTC)
Subject: [master] d0cd96d20 Generate documentation of the varnishd -n option
Message-ID: <20240524091711.2B5FE11568F@lists.varnish-cache.org>


commit d0cd96d20a04c839f29dfd433ed039441de19419
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri May 24 11:14:22 2024 +0200

    Generate documentation of the varnishd -n option
    
    with greetings from the rabbit-hole opened by #4105

diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index dc2bce56b..6df16b285 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -79,6 +79,19 @@ static const char opt_spec[] = "?a:b:CdE:f:Fh:i:I:j:l:M:n:P:p:r:S:s:T:t:VW:x:";
 
 /*--------------------------------------------------------------------*/
 
+// Basic options documentation with compile time dependencies
+static void
+mgt_DumpOptions(void)
+{
+	printf(".. _opt_n:\n\n");
+	printf("-n workdir\n\n");
+	printf("  Runtime directory for the shared memory, "
+	       "compiled VCLs etc.\n\n");
+	printf("  In performance critical applications, this directory "
+	       "should be on a RAM backed filesystem.\n\n");
+	VIN_DumpDefaults();
+}
+
 static void
 usage(void)
 {
@@ -121,6 +134,7 @@ usage(void)
 	printf(FMT, "-x cli", "CLI command documentation");
 	printf(FMT, "-x builtin", "Builtin VCL program");
 	printf(FMT, "-x optstring", "List of getopt options");
+	printf(FMT, "-x options", "Dynamic options documentation");
 
 	printf("\nOperations options:\n");
 
@@ -359,6 +373,8 @@ mgt_x_arg(const char *x_arg)
 		mgt_DumpBuiltin();
 	else if (!strcmp(x_arg, "optstring"))
 		(void)printf("%s\n", opt_spec);
+	else if (!strcmp(x_arg, "options"))
+		mgt_DumpOptions();
 	else
 		ARGV_ERR("Invalid -x argument\n");
 }
diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am
index b5d34fb8d..9ceb63456 100644
--- a/doc/sphinx/Makefile.am
+++ b/doc/sphinx/Makefile.am
@@ -85,6 +85,11 @@ include/params.rst: $(top_builddir)/bin/varnishd/varnishd
 	mv -f ${@}_ ${@}
 BUILT_SOURCES += include/params.rst
 
+include/options.rst: $(top_builddir)/bin/varnishd/varnishd
+	$(top_builddir)/bin/varnishd/varnishd -x options > ${@}_
+	mv -f ${@}_ ${@}
+BUILT_SOURCES += include/options.rst
+
 include/counters.rst:
 	ln -s $(abs_top_builddir)/lib/libvsc/counters.rst $@
 BUILT_SOURCES += include/counters.rst
diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst
index 3f7dfcb1b..7db93dcf2 100644
--- a/doc/sphinx/reference/varnishd.rst
+++ b/doc/sphinx/reference/varnishd.rst
@@ -142,19 +142,7 @@ Basic options
   could later be accessed remotely, starting `varnishd` requires
   local privileges.
 
-.. _opt_n:
-
--n workdir
-
-  Runtime directory for the shared memory, compiled VCLs etc.
-
-  In performance critical applications, this directory should be
-  on a RAM backed filesystem.
-
-  Relative paths will be appended to `/var/run/` (NB: Binary packages
-  of Varnish may have adjusted this to the platform.)
-
-  The default value is `/var/run/varnishd` (NB: as above.)
+.. include:: ../include/options.rst
 
 Documentation options
 ---------------------
diff --git a/include/vin.h b/include/vin.h
index e7dc3eb8d..2d63cf6c0 100644
--- a/include/vin.h
+++ b/include/vin.h
@@ -33,6 +33,7 @@
 #ifndef VIN_H_INCLUDED
 #define VIN_H_INCLUDED
 
-/* This function lives in both libvarnish and libvarnishapi */
+/* These functions live in both libvarnish and libvarnishapi */
 char *VIN_n_Arg(const char *n_arg);
+void VIN_DumpDefaults(void);
 #endif
diff --git a/lib/libvarnish/vin.c b/lib/libvarnish/vin.c
index 89f132f4e..9e1028f34 100644
--- a/lib/libvarnish/vin.c
+++ b/lib/libvarnish/vin.c
@@ -42,6 +42,7 @@
 #include "vin.h"
 #include "vsb.h"
 
+#define VARNISH_DEFAULT_REL_NAME "varnishd"
 char *
 VIN_n_Arg(const char *n_arg)
 {
@@ -52,7 +53,7 @@ VIN_n_Arg(const char *n_arg)
 	AN(vsb);
 	if (n_arg == NULL || n_arg[0] == '\0') {
 		VSB_cat(vsb, VARNISH_STATE_DIR);
-		VSB_cat(vsb, "/varnishd");
+		VSB_cat(vsb, "/" VARNISH_DEFAULT_REL_NAME);
 	} else if (n_arg[0] == '/') {
 		VSB_cat(vsb, n_arg);
 	} else {
@@ -66,3 +67,13 @@ VIN_n_Arg(const char *n_arg)
 	VSB_destroy(&vsb);
 	return (retval);
 }
+
+void
+VIN_DumpDefaults(void)
+{
+	printf("  Relative paths will be appended to ``%s``.\n\n",
+	       VARNISH_STATE_DIR);
+	printf("  The default value is ``%s``.\n\n",
+	       VARNISH_STATE_DIR "/" VARNISH_DEFAULT_REL_NAME);
+	printf("  Note: These defaults may be distribution specific.\n\n");
+}
diff --git a/man/Makefile.am b/man/Makefile.am
index 560ec8d55..436d7d255 100644
--- a/man/Makefile.am
+++ b/man/Makefile.am
@@ -78,7 +78,8 @@ varnishadm.1: $(top_builddir)/doc/sphinx/reference/varnishadm.rst
 
 varnishd.1: \
 	$(top_builddir)/doc/sphinx/reference/varnishd.rst \
-	$(top_builddir)/doc/sphinx/include/params.rst
+	$(top_builddir)/doc/sphinx/include/params.rst \
+	$(top_builddir)/doc/sphinx/include/options.rst
 	$(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishd.rst $@
 
 varnishncsa.1: \

From nils.goroll at uplex.de  Fri May 24 10:05:07 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 24 May 2024 10:05:07 +0000 (UTC)
Subject: [master] f3bad4216 Unify -n argument documentation
Message-ID: <20240524100507.AF9EF117015@lists.varnish-cache.org>


commit f3bad421683148c45d820580ef78e47184867ebd
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri May 24 11:41:56 2024 +0200

    Unify -n argument documentation
    
    The host name default referenced by VUT_OPT_n is long gone since
    879c9b37ca108e365d4eb16ad391e28a8cfccd46
    
    Refer to varnishd(1) to avoid the documentation getting out of sync.
    
    Ref #4105

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index d08da8278..3f84afcc2 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -388,7 +388,7 @@ static void v_noreturn_
 usage(int status)
 {
 	fprintf(stderr,
-	    "Usage: varnishadm [-h] [-n ident] [-p] [-S secretfile] "
+	    "Usage: varnishadm [-h] [-n workdir] [-p] [-S secretfile] "
 	    "[-T [address]:port] [-t timeout] [command [...]]\n");
 	fprintf(stderr, "\t-n is mutually exclusive with -S and -T\n");
 	exit(status);
diff --git a/doc/sphinx/reference/varnishadm.rst b/doc/sphinx/reference/varnishadm.rst
index 8cfdc6983..9d4aee907 100644
--- a/doc/sphinx/reference/varnishadm.rst
+++ b/doc/sphinx/reference/varnishadm.rst
@@ -19,17 +19,17 @@ Control a running Varnish instance
 SYNOPSIS
 ========
 
-varnishadm [-h] [-n ident] [-p] [-S secretfile] [-T [address]:port] [-t timeout] [command [...]]
+varnishadm [-h] [-n workdir] [-p] [-S secretfile] [-T [address]:port] [-t timeout] [command [...]]
 
 
 DESCRIPTION
 ===========
 
 The `varnishadm` utility establishes a CLI connection to varnishd either
-using -n *name* or using the -T and -S arguments. If -n *name* is
-given the location of the secret file and the address:port is looked
-up in shared memory. If neither is given `varnishadm` will look for an
-instance without a given name.
+using -n *workdir* or using the -T and -S arguments. If -n *workdir* is
+given, the location of the secret file and the address:port are looked
+up in shared memory. If neither is given, `varnishadm` uses the -n
+defaults documented for :ref:`varnishd(1)`.
 
 If a command is given, the command and arguments are sent over the CLI
 connection and the result returned on stdout.
@@ -43,8 +43,11 @@ OPTIONS
 -h
     Print program usage and exit.
 
--n ident
-    Connect to the instance of `varnishd` with this name.
+-n workdir
+
+    Specify the varnish working directory of the instance to attach
+    to. See :ref:`varnishd(1)` ``-n`` option documentation for
+    additional information and defaults.
 
 -p
     Force `pass` mode and make the output follow the VCLI protocol.
diff --git a/doc/sphinx/reference/vsm.rst b/doc/sphinx/reference/vsm.rst
index 13671045e..ad2da1740 100644
--- a/doc/sphinx/reference/vsm.rst
+++ b/doc/sphinx/reference/vsm.rst
@@ -70,7 +70,7 @@ The Varnish way:
 
 When varnishd starts, it opens locked shared memory files, advising to
 use different -n arguments if an attempt is made to run multiple
-varnishd instances with the same name.
+varnishd instances on the same working directory.
 
 Child processes each use their own shared memory files, since a worker
 process restart marks a clean break in operation anyway.
diff --git a/doc/sphinx/users-guide/troubleshooting.rst b/doc/sphinx/users-guide/troubleshooting.rst
index 938cbf5c7..9ef176081 100644
--- a/doc/sphinx/users-guide/troubleshooting.rst
+++ b/doc/sphinx/users-guide/troubleshooting.rst
@@ -129,11 +129,9 @@ data.
    in the service's ``[Service]]`` section of the unit file.
 
 Once you have the core, ``cd`` into varnish's working directory (as
-given by the ``-n`` parameter, whose default is
-``$PREFIX/var/varnish/$HOSTNAME`` with ``$PREFIX`` being the
-installation prefix, usually ``/usr/local``, open the core with
-``gdb`` and issue the command ``bt`` to get a stack trace of the
-thread that caused the segfault.
+given by the ``-n`` parameter (see :ref:`varnishd(1)` for defaults),
+open the core with ``gdb`` and issue the command ``bt`` to get a stack
+trace of the thread that caused the segfault.
 
 A basic debug session for varnish installed under ``/usr/local`` could look
 like this::
diff --git a/include/vut_options.h b/include/vut_options.h
index 14eb0699e..534ff0cc1 100644
--- a/include/vut_options.h
+++ b/include/vut_options.h
@@ -69,10 +69,10 @@
 	)
 
 #define VUT_OPT_n							\
-	VOPT("n:", "[-n <dir>]", "varnishd working directory",		\
-	    "Specify the varnishd working directory (also known as"	\
-	    " instance name) to get logs from. If -n is not specified,"	\
-	    " the host name is used."					\
+	VOPT("n:", "[-n <workdir>]", "varnish working directory",	\
+	     "Specify the varnish working directory of the instance "	\
+	     "to attach to. See :ref:`varnishd(1)` ``-n`` option "	\
+	     "documentation for additional information and defaults."	\
 	)
 
 #define VUT_OPT_Q							\

From nils.goroll at uplex.de  Fri May 24 10:54:08 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 24 May 2024 10:54:08 +0000 (UTC)
Subject: [master] 8f8d26c0c Add VARNISH_DEFAULT_N environment variable as -n
 argument default
Message-ID: <20240524105408.B8E6A118A28@lists.varnish-cache.org>


commit 8f8d26c0c1fec91aec9a7f79380c93eb4d266b7e
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri May 24 12:51:17 2024 +0200

    Add VARNISH_DEFAULT_N environment variable as -n argument default

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index 3f84afcc2..f855d2124 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -467,6 +467,7 @@ main(int argc, char * const *argv)
 		printf(OPTARG "\n");
 		exit(0);
 	}
+	n_arg = getenv("VARNISH_DEFAULT_N");
 	/*
 	 * By default linux::getopt(3) mangles the argv order, such that
 	 *	varnishadm -n bla param.set foo -bar
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 6df16b285..0db5c6559 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -614,7 +614,7 @@ main(int argc, char * const *argv)
 	const char *i_arg = NULL;
 	const char *j_arg = NULL;
 	const char *h_arg = "critbit";
-	const char *n_arg = NULL;
+	const char *n_arg = getenv("VARNISH_DEFAULT_N");
 	const char *S_arg = NULL;
 	const char *s_arg = "default,100m";
 	const char *W_arg = NULL;
diff --git a/bin/varnishtest/tests/b00041.vtc b/bin/varnishtest/tests/b00041.vtc
index b829efc17..8f3107424 100644
--- a/bin/varnishtest/tests/b00041.vtc
+++ b/bin/varnishtest/tests/b00041.vtc
@@ -93,3 +93,5 @@ shell -expect {Tested} \
 	{varnishadm -T ${s1_sock} -S ${v1_name}/_.secret test}
 
 shell "varnishadm -n ${v1_name} help > /dev/null"
+
+shell {VARNISH_DEFAULT_N="${v1_name}" varnishadm help > /dev/null}
diff --git a/bin/varnishtest/tests/b00045.vtc b/bin/varnishtest/tests/b00045.vtc
index c687f2d49..ec8c5cb51 100644
--- a/bin/varnishtest/tests/b00045.vtc
+++ b/bin/varnishtest/tests/b00045.vtc
@@ -25,3 +25,7 @@ shell -err -expect {Error: Varnishd is already running} {
 shell -err -expect {Error: Varnishd is already running} {
 	varnishd -b None -a:0 -n ${tmpdir}/v1 -F
 }
+
+shell -err -expect {Error: Varnishd is already running} {
+	VARNISH_DEFAULT_N="${tmpdir}/v1" varnishd -b None -a:0 -F
+}
diff --git a/bin/varnishtest/tests/u00003.vtc b/bin/varnishtest/tests/u00003.vtc
index 2f6cc7569..343c77e0f 100644
--- a/bin/varnishtest/tests/u00003.vtc
+++ b/bin/varnishtest/tests/u00003.vtc
@@ -118,6 +118,10 @@ shell -err -match "Usage: .*varnishncsa <options>" \
 shell -match {^${localhost} 100
 ${localhost} 0$} \
 	{varnishncsa -n ${v1_name} -b -d -F "%{Host}i %b"}
+# -b with VARNISH_DEFAULT_N
+shell -match {^${localhost} 100
+${localhost} 0$} \
+	{VARNISH_DEFAULT_N="${v1_name}" varnishncsa -b -d -F "%{Host}i %b"}
 # -c
 shell -match {^${localhost} 100
 ${localhost} \d+
diff --git a/bin/varnishtest/tests/u00004.vtc b/bin/varnishtest/tests/u00004.vtc
index a861ae57a..eed2cfa6b 100644
--- a/bin/varnishtest/tests/u00004.vtc
+++ b/bin/varnishtest/tests/u00004.vtc
@@ -13,6 +13,7 @@ client c1 {
 } -run
 
 shell -expect "fetch" "varnishtop -n ${v1_name} -1"
+shell -expect "fetch" {VARNISH_DEFAULT_N="${v1_name}" varnishtop  -1}
 
 process p1 "varnishtop -n ${v1_name} -d" -start
 delay 1
diff --git a/bin/varnishtest/tests/u00005.vtc b/bin/varnishtest/tests/u00005.vtc
index 7e22b22e3..6f85abb90 100644
--- a/bin/varnishtest/tests/u00005.vtc
+++ b/bin/varnishtest/tests/u00005.vtc
@@ -51,8 +51,12 @@ shell -err -expect "-t: Invalid argument: foo" \
 	"varnishstat -t foo"
 shell -err -expect "Could not get hold of varnishd" \
 	"varnishstat -n /nonexistent -t 1"
+shell -err -expect "Could not get hold of varnishd" \
+	{VARNISH_DEFAULT_N="/nonexistent" varnishstat -t 1}
 shell -expect "MAIN.uptime" \
 	"varnishstat -n ${v1_name} -1"
+shell -expect "MAIN.uptime" \
+	{VARNISH_DEFAULT_N="${v1_name}" varnishstat -1}
 shell -expect "<name>MAIN.uptime</name>" \
 	"varnishstat -n ${v1_name} -x"
 shell -match {"MAIN.uptime":} \
diff --git a/bin/varnishtest/tests/u00006.vtc b/bin/varnishtest/tests/u00006.vtc
index 7c4fcf706..3282ebabb 100644
--- a/bin/varnishtest/tests/u00006.vtc
+++ b/bin/varnishtest/tests/u00006.vtc
@@ -131,6 +131,9 @@ process p1 -stop
 shell -match "0 CLI[ ]+- Wr 200 [0-9]+ PONG" \
 	{varnishlog -n ${v1_name} -d -g raw -X "Wr 200 [0-9]+ [^P]"}
 
+shell -match "0 CLI[ ]+- Wr 200 [0-9]+ PONG" \
+	{VARNISH_DEFAULT_N="${v1_name}" varnishlog -d -g raw -X "Wr 200 [0-9]+ [^P]"}
+
 client c1 {
 	txreq -url /foo
 	rxresp
diff --git a/bin/varnishtest/tests/u00009.vtc b/bin/varnishtest/tests/u00009.vtc
index 05c14b046..3abdcdc28 100644
--- a/bin/varnishtest/tests/u00009.vtc
+++ b/bin/varnishtest/tests/u00009.vtc
@@ -9,7 +9,7 @@ varnish v1 -vcl+backend {} -start
 
 process p1 -dump {varnishhist -n ${v1_name}} -start
 process p2 -dump {varnishhist -n ${v1_name} -P b:BereqAcct::5:-1:1} -start
-process p3 -dump {varnishhist -n ${v1_name} -P BerespBodytime -B 2} -start
+process p3 -dump {VARNISH_DEFAULT_N="${v1_name}" varnishhist -P BerespBodytime -B 2} -start
 
 process p1 -expect-text 24 0 {1e2}
 process p2 -expect-text 24 0 {1e-1}
diff --git a/doc/changes.rst b/doc/changes.rst
index e41815083..dbe31e3ab 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -41,6 +41,11 @@ Varnish Cache NEXT (2024-09-15)
 .. PLEASE keep this roughly in commit order as shown by git-log / tig
    (new to old)
 
+* The environment variable ``VARNISH_DEFAULT_N`` now provides the
+  default "varnish name" / "workdir" as otherwise specified by he
+  ``-n`` argument to ``varnishd`` and ``varnish*`` utilities except
+  ``varnishtest``.
+
 ================================
 Varnish Cache 7.5.0 (2024-03-18)
 ================================
diff --git a/lib/libvarnish/vin.c b/lib/libvarnish/vin.c
index 9e1028f34..37fe8a568 100644
--- a/lib/libvarnish/vin.c
+++ b/lib/libvarnish/vin.c
@@ -71,9 +71,12 @@ VIN_n_Arg(const char *n_arg)
 void
 VIN_DumpDefaults(void)
 {
+	printf("  The default is taken from the ``VARNISH_DEFAULT_N`` "
+	       "environment variable.\n\n");
 	printf("  Relative paths will be appended to ``%s``.\n\n",
 	       VARNISH_STATE_DIR);
-	printf("  The default value is ``%s``.\n\n",
+	printf("  If neither ``VARNISH_DEFAULT_N`` nor ``-n`` are "
+	       "present, the value is ``%s``.\n\n",
 	       VARNISH_STATE_DIR "/" VARNISH_DEFAULT_REL_NAME);
 	printf("  Note: These defaults may be distribution specific.\n\n");
 }
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 6ada5dfc0..7da5aa171 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -346,6 +346,8 @@ VSM_New(void)
 	ALLOC_OBJ(vd, VSM_MAGIC);
 	AN(vd);
 
+	REPLACE(vd->wdname, getenv("VARNISH_DEFAULT_N"));
+
 	vd->mgt = vsm_newset(VSM_MGT_DIRNAME);
 	vd->mgt->flag_running = VSM_MGT_RUNNING;
 	vd->mgt->flag_changed = VSM_MGT_CHANGED;
diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c
index 42839da0c..315aee7a5 100644
--- a/lib/libvarnishapi/vut.c
+++ b/lib/libvarnishapi/vut.c
@@ -263,6 +263,7 @@ VUT_Init(const char *progname, int argc, char * const *argv,
 	AZ(vut->vsl);
 	vut->vsl = VSL_New();
 	AN(vut->vsl);
+	REPLACE(vut->n_arg, getenv("VARNISH_DEFAULT_N"));
 	return (vut);
 }
 

From nils.goroll at uplex.de  Fri May 24 11:21:05 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 24 May 2024 11:21:05 +0000 (UTC)
Subject: [master] 4d253f9d7 Polish error message for vext_cache
Message-ID: <20240524112105.DB562119A15@lists.varnish-cache.org>


commit 4d253f9d7aedcf141c50b07c9c9eecb5f8bdaf99
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri May 24 13:20:51 2024 +0200

    Polish error message for vext_cache

diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 0db5c6559..5164992e2 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -876,7 +876,7 @@ main(int argc, char * const *argv)
 	}
 
 	if (arg_list_count("E") &&
-	    VJ_make_subdir("vext_cache", "VMOD cache", NULL)) {
+	    VJ_make_subdir("vext_cache", "VEXT cache", NULL)) {
 		ARGV_ERR(
 		    "Cannot create vmod directory (%s/vext_cache): %s\n",
 		    workdir, VAS_errtxt(errno));

From nils.goroll at uplex.de  Fri May 24 11:23:05 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 24 May 2024 11:23:05 +0000 (UTC)
Subject: [master] 39a29a520 Consolidate system("rm ...")
Message-ID: <20240524112305.35B7C119C80@lists.varnish-cache.org>


commit 39a29a520eb1a02042072ffbe5e74e4460aa3585
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Fri May 24 13:22:07 2024 +0200

    Consolidate system("rm ...")

diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 5164992e2..192ae202a 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -865,8 +865,7 @@ main(int argc, char * const *argv)
 		    workdir, VAS_errtxt(errno));
 
 	VJ_master(JAIL_MASTER_SYSTEM);
-	AZ(system("rm -rf vmod_cache"));
-	AZ(system("rm -rf vext_cache"));
+	AZ(system("rm -rf vmod_cache vext_cache"));
 	VJ_master(JAIL_MASTER_LOW);
 
 	if (VJ_make_subdir("vmod_cache", "VMOD cache", NULL)) {

From dridi.boukelmoune at gmail.com  Fri May 24 12:27:05 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 24 May 2024 12:27:05 +0000 (UTC)
Subject: [master] 1d7cdadda vsm: Centralize VARNISH_DEFAULT_N fallback
Message-ID: <20240524122705.B738111BE21@lists.varnish-cache.org>


commit 1d7cdaddab67d156f6a99b2d2a1089b3a1913275
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri May 24 14:24:17 2024 +0200

    vsm: Centralize VARNISH_DEFAULT_N fallback
    
    Putting the new fallback next to the historical fallback.
    
    Refs #4105

diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c
index f855d2124..3f84afcc2 100644
--- a/bin/varnishadm/varnishadm.c
+++ b/bin/varnishadm/varnishadm.c
@@ -467,7 +467,6 @@ main(int argc, char * const *argv)
 		printf(OPTARG "\n");
 		exit(0);
 	}
-	n_arg = getenv("VARNISH_DEFAULT_N");
 	/*
 	 * By default linux::getopt(3) mangles the argv order, such that
 	 *	varnishadm -n bla param.set foo -bar
diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 7da5aa171..2561d1a1d 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -346,8 +346,6 @@ VSM_New(void)
 	ALLOC_OBJ(vd, VSM_MAGIC);
 	AN(vd);
 
-	REPLACE(vd->wdname, getenv("VARNISH_DEFAULT_N"));
-
 	vd->mgt = vsm_newset(VSM_MGT_DIRNAME);
 	vd->mgt->flag_running = VSM_MGT_RUNNING;
 	vd->mgt->flag_changed = VSM_MGT_CHANGED;
@@ -756,6 +754,9 @@ VSM_Attach(struct vsm *vd, int progress)
 	else
 		t0 = VTIM_mono() + vd->patience;
 
+	if (vd->wdname == NULL)
+		REPLACE(vd->wdname, getenv("VARNISH_DEFAULT_N"));
+
 	if (vd->wdname == NULL) {
 		/* Use default (hostname) */
 		i = VSM_Arg(vd, 'n', "");
diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c
index 315aee7a5..42839da0c 100644
--- a/lib/libvarnishapi/vut.c
+++ b/lib/libvarnishapi/vut.c
@@ -263,7 +263,6 @@ VUT_Init(const char *progname, int argc, char * const *argv,
 	AZ(vut->vsl);
 	vut->vsl = VSL_New();
 	AN(vut->vsl);
-	REPLACE(vut->n_arg, getenv("VARNISH_DEFAULT_N"));
 	return (vut);
 }
 

From dridi.boukelmoune at gmail.com  Fri May 24 12:58:05 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Fri, 24 May 2024 12:58:05 +0000 (UTC)
Subject: [master] daf0cc0bb vin: Extra blank line
Message-ID: <20240524125805.B4F6C11CE78@lists.varnish-cache.org>


commit daf0cc0bb068825d8604ef5e0d168cb2bd177c39
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri May 24 14:56:37 2024 +0200

    vin: Extra blank line

diff --git a/lib/libvarnish/vin.c b/lib/libvarnish/vin.c
index 37fe8a568..496684844 100644
--- a/lib/libvarnish/vin.c
+++ b/lib/libvarnish/vin.c
@@ -43,6 +43,7 @@
 #include "vsb.h"
 
 #define VARNISH_DEFAULT_REL_NAME "varnishd"
+
 char *
 VIN_n_Arg(const char *n_arg)
 {

From nils.goroll at uplex.de  Fri May 24 13:40:28 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Fri, 24 May 2024 15:40:28 +0200
Subject: [master] 1d7cdadda vsm: Centralize VARNISH_DEFAULT_N fallback
In-Reply-To: <20240524122705.B738111BE21@lists.varnish-cache.org>
References: <20240524122705.B738111BE21@lists.varnish-cache.org>
Message-ID: <8f2a4ae3-0c95-43f6-b65d-958f3b31777a@uplex.de>

Dridi, I doubt this is a good idea. vut->n_arg should have the right value.

-- 

Nils Goroll (he/him)

** * * UPLEX - Nils Goroll Systemoptimierung

Scheffelstra?e 32
22301 Hamburg

tel +49 40 28805731
mob +49 170 2723133
fax +49 40 42949753

xmpp://slink at jabber.int.uplex.de/

http://uplex.de/


From dridi.boukelmoune at gmail.com  Sat May 25 05:02:05 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Sat, 25 May 2024 05:02:05 +0000 (UTC)
Subject: [master] 32574ab46 vsm: Pass VARNISH_DEFAULT_N to VSM_Arg()
Message-ID: <20240525050205.A6B1CA387A@lists.varnish-cache.org>


commit 32574ab4666173afc30fc0a072df7ae48c822f5d
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Sat May 25 06:53:23 2024 +0200

    vsm: Pass VARNISH_DEFAULT_N to VSM_Arg()
    
    It should be resolved like the argument of a -n option.
    
    Refs 1d7cdaddab67d156f6a99b2d2a1089b3a1913275

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 2561d1a1d..10faa5d74 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -743,6 +743,7 @@ VSM_Status(struct vsm *vd)
 int
 VSM_Attach(struct vsm *vd, int progress)
 {
+	const char *def;
 	double t0;
 	unsigned u;
 	int i, n = 0;
@@ -754,12 +755,11 @@ VSM_Attach(struct vsm *vd, int progress)
 	else
 		t0 = VTIM_mono() + vd->patience;
 
-	if (vd->wdname == NULL)
-		REPLACE(vd->wdname, getenv("VARNISH_DEFAULT_N"));
-
 	if (vd->wdname == NULL) {
-		/* Use default (hostname) */
-		i = VSM_Arg(vd, 'n', "");
+		def = getenv("VARNISH_DEFAULT_N");
+		if (def == NULL)
+			def = ""; /* Use default (hostname) */
+		i = VSM_Arg(vd, 'n', def);
 		if (i < 0)
 			return (i);
 		AN(vd->wdname);

From nils.goroll at uplex.de  Sun May 26 11:10:51 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Sun, 26 May 2024 13:10:51 +0200
Subject: [master] 32574ab46 vsm: Pass VARNISH_DEFAULT_N to VSM_Arg()
In-Reply-To: <20240525050205.A6B1CA387A@lists.varnish-cache.org>
References: <20240525050205.A6B1CA387A@lists.varnish-cache.org>
Message-ID: <488d5f63-b6e2-4818-8d40-53406d60bf0e@uplex.de>

I still don't get it.

1d7cdadda changed this in, I think, a wrong way, and now we add another wrong 
place. The default should be set for vut->n_arg. The default should be set for 
wd->wdname in VSM_Init(). What is the problem with that?

Nils

From dridi at varni.sh  Sun May 26 18:12:57 2024
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Sun, 26 May 2024 18:12:57 +0000
Subject: [master] 32574ab46 vsm: Pass VARNISH_DEFAULT_N to VSM_Arg()
In-Reply-To: <488d5f63-b6e2-4818-8d40-53406d60bf0e@uplex.de>
References: <20240525050205.A6B1CA387A@lists.varnish-cache.org>
 <488d5f63-b6e2-4818-8d40-53406d60bf0e@uplex.de>
Message-ID: <CABoVN9Ctb7XPcvvfhHRVthCznuC5ZvtZZ1hKR1B4RGfNJNFSyg@mail.gmail.com>

On Sun, May 26, 2024 at 11:11?AM Nils Goroll <nils.goroll at uplex.de> wrote:
>
> I still don't get it.
>
> 1d7cdadda changed this in, I think, a wrong way, and now we add another wrong
> place. The default should be set for vut->n_arg. The default should be set for
> wd->wdname in VSM_Init(). What is the problem with that?

In 8f8d26c0c1fec91aec9a7f79380c93eb4d266b7e you introduced a fallback
to the environment variable in 4 locations:

- varnishadm
- varnishd
- VSM_New in libvarnishapi
- VUT_Init in libvarnishapi

By the way, thank you for your efforts on the docs aspect of this
change. You may also want to amend the "Invalid instance name" error
message to say "Invalid working directory" instead.

To me there should be only 2 locations needed:

- varnishd where we produce working directories
- libvarnishapi where we consume them

Neither the VUT API nor non-VUT programs like varnishadm should
*duplicate* this logic, they both ultimately converge to attaching to
a VSM before doing what they need to do (for the most part to find
what they need to operate). This is done by the VIN code, not exposed
by libvarnishapi.

In 1d7cdaddab67d156f6a99b2d2a1089b3a1913275 I placed the new fallback
next to the existing fallback that exists in VSM_Attach (not in
VSM_New) and I indeed got it slightly wrong and this is partly because
I lazily reused your initial REPLACE logic, partly because I didn't
pay enough attention. So eventually I realized that I wasn't washing
VARNISH_DEFAULT_N through VSM_Arg like the ultimate empty string
fallback.

And if you are now thinking the rationale should have been expanded in
the terse commit message, you are correct.

Dridi

From dridi.boukelmoune at gmail.com  Mon May 27 10:09:05 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 27 May 2024 10:09:05 +0000 (UTC)
Subject: [master] ef400bba4 vsm: Extract vsm_set_findseg() function
Message-ID: <20240527100905.58E5F62A17@lists.varnish-cache.org>


commit ef400bba443f2819b9ad1d227113eebddb6d0cc8
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri May 24 15:57:07 2024 +0200

    vsm: Extract vsm_set_findseg() function

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 10faa5d74..a2c3f7e89 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -790,10 +790,25 @@ VSM_Attach(struct vsm *vd, int progress)
 
 /*--------------------------------------------------------------------*/
 
+static struct vsm_seg *
+vsm_set_findseg(const struct vsm_set *vs, uintptr_t serial)
+{
+	struct vsm_seg *vg;
+
+	VTAILQ_FOREACH(vg, &vs->segs, list) {
+		if (vg->serial == serial)
+			return (vg);
+	}
+	VTAILQ_FOREACH(vg, &vs->stale, list) {
+		if (vg->serial == serial)
+			return (vg);
+	}
+	return (NULL);
+}
+
 static struct vsm_seg *
 vsm_findseg(const struct vsm *vd, const struct vsm_fantom *vf)
 {
-	struct vsm_set *vs;
 	struct vsm_seg *vg;
 	uint64_t x;
 
@@ -807,26 +822,9 @@ vsm_findseg(const struct vsm *vd, const struct vsm_fantom *vf)
 	}
 
 	x = VSM_PRIV_LOW(vf->priv);
-	vs = vd->mgt;
-	VTAILQ_FOREACH(vg, &vs->segs, list)
-		if (vg->serial == x)
-			break;
-	if (vg == NULL) {
-		VTAILQ_FOREACH(vg, &vs->stale, list)
-			if (vg->serial == x)
-				break;
-	}
-	vs = vd->child;
-	if (vg == NULL) {
-		VTAILQ_FOREACH(vg, &vs->segs, list)
-			if (vg->serial == x)
-				break;
-	}
-	if (vg == NULL) {
-		VTAILQ_FOREACH(vg, &vs->stale, list)
-			if (vg->serial == x)
-				break;
-	}
+	vg = vsm_set_findseg(vd->mgt, x);
+	if (vg == NULL)
+		vg = vsm_set_findseg(vd->child, x);
 	if (vg == NULL)
 		return (NULL);
 

From dridi.boukelmoune at gmail.com  Mon May 27 10:09:05 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Mon, 27 May 2024 10:09:05 +0000 (UTC)
Subject: [master] ba7bf7ff6 vsm: Do not store MAP_FAILED in a VSM segment
Message-ID: <20240527100905.8541C62A1B@lists.varnish-cache.org>


commit ba7bf7ff6dba835e2cba9ad0a07131819272fb8e
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Fri May 24 15:29:42 2024 +0200

    vsm: Do not store MAP_FAILED in a VSM segment
    
    The problem is that a MAP_FAILED segment ends up in its vsm_set::segs
    list, at which point NULL checks will determine what to do with the
    segment, and MAP_FAILED != NULL.
    
    This can happen when winning (or losing?) the race with a VSC segment
    that goes away after being found in the index, but before being mapped.
    
    One surefire result is a crash in vsm_unmapseg() tripping on assertions
    that other fields are also not NULL. There may be other unidentified ill
    effects.

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index a2c3f7e89..ad5438428 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -181,6 +181,7 @@ vsm_mapseg(struct vsm *vd, struct vsm_seg *vg)
 {
 	size_t of, off, sz, ps, len;
 	struct vsb *vsb;
+	void *s;
 	int fd;
 
 	CHECK_OBJ_NOTNULL(vg, VSM_SEG_MAGIC);
@@ -213,7 +214,7 @@ vsm_mapseg(struct vsm *vd, struct vsm_seg *vg)
 		return (vsm_diag(vd, "Could not open segment"));
 	}
 
-	vg->s = (void*)mmap(NULL, len,
+	s = (void*)mmap(NULL, len,
 	    PROT_READ,
 	    MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED,
 	    fd, (off_t)off);
@@ -221,9 +222,10 @@ vsm_mapseg(struct vsm *vd, struct vsm_seg *vg)
 	VSB_destroy(&vsb);
 
 	closefd(&fd);
-	if (vg->s == MAP_FAILED)
+	if (s == MAP_FAILED)
 		return (vsm_diag(vd, "Could not mmap segment"));
 
+	vg->s = s;
 	vg->b = (char*)(vg->s) + of - off;
 	vg->e = (char *)vg->b + sz;
 	vg->sz = len;

From nils.goroll at uplex.de  Mon May 27 10:30:54 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 27 May 2024 12:30:54 +0200
Subject: [master] 32574ab46 vsm: Pass VARNISH_DEFAULT_N to VSM_Arg()
In-Reply-To: <CABoVN9Ctb7XPcvvfhHRVthCznuC5ZvtZZ1hKR1B4RGfNJNFSyg@mail.gmail.com>
References: <20240525050205.A6B1CA387A@lists.varnish-cache.org>
 <488d5f63-b6e2-4818-8d40-53406d60bf0e@uplex.de>
 <CABoVN9Ctb7XPcvvfhHRVthCznuC5ZvtZZ1hKR1B4RGfNJNFSyg@mail.gmail.com>
Message-ID: <6257e218-bb9d-4a93-b8ad-adf05e71aefa@uplex.de>

On 26.05.24 20:12, Dridi Boukelmoune wrote:
> Neither the VUT API nor non-VUT programs like varnishadm should
> *duplicate*  this logic

Yes. I see this as the task here: Converge to VUT.

If I consolidate my and your patches into a single diff, this is what I see:

diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c
index 15ab45799..ad5438428 100644
--- a/lib/libvarnishapi/vsm.c
+++ b/lib/libvarnishapi/vsm.c
@@ -745,6 +745,7 @@ VSM_Status(struct vsm *vd)
  int
  VSM_Attach(struct vsm *vd, int progress)
  {
+       const char *def;
         double t0;
         unsigned u;
         int i, n = 0;
@@ -757,8 +758,10 @@ VSM_Attach(struct vsm *vd, int progress)
                 t0 = VTIM_mono() + vd->patience;

         if (vd->wdname == NULL) {
-               /* Use default (hostname) */
-               i = VSM_Arg(vd, 'n', "");
+               def = getenv("VARNISH_DEFAULT_N");
+               if (def == NULL)
+                       def = ""; /* Use default (hostname) */
+               i = VSM_Arg(vd, 'n', def);
                 if (i < 0)
                         return (i);
                 AN(vd->wdname);

To summarize, compared to my original change 
8f8d26c0c1fec91aec9a7f79380c93eb4d266b7e

* this removes *one* line from varnishadm.c

* this does not properly initialize the vut->n_arg

* this does not set the default in VSM_New()

Sorry, you really lost me here.

When you write "there should be only 2 locations needed", I agree that two 
locations are better than four, but no matter how much I appreciate your desire 
to minimize code changes, I do not agree that this is now handled at the right 
place for the reasons given.

Nils



-- 

Nils Goroll (he/him)

** * * UPLEX - Nils Goroll Systemoptimierung

Scheffelstra?e 32
22301 Hamburg

tel +49 40 28805731
mob +49 170 2723133
fax +49 40 42949753

xmpp://slink at jabber.int.uplex.de/

http://uplex.de/


From dridi at varni.sh  Mon May 27 11:19:43 2024
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Mon, 27 May 2024 11:19:43 +0000
Subject: [master] 32574ab46 vsm: Pass VARNISH_DEFAULT_N to VSM_Arg()
In-Reply-To: <6257e218-bb9d-4a93-b8ad-adf05e71aefa@uplex.de>
References: <20240525050205.A6B1CA387A@lists.varnish-cache.org>
 <488d5f63-b6e2-4818-8d40-53406d60bf0e@uplex.de>
 <CABoVN9Ctb7XPcvvfhHRVthCznuC5ZvtZZ1hKR1B4RGfNJNFSyg@mail.gmail.com>
 <6257e218-bb9d-4a93-b8ad-adf05e71aefa@uplex.de>
Message-ID: <CABoVN9D9XWEm_b_0c1ueyMFVKQ0ZnqOPOAY2pAi4pJwoctOzUA@mail.gmail.com>

> * this does not properly initialize the vut->n_arg

Yes, there was no -n option to grab an optarg.

> * this does not set the default in VSM_New()

What you did in VSM_New is as wrong as what I initially did (taking
your code at face value) in 1d7cdaddab67d156f6a99b2d2a1089b3a1913275
because as soon as the environment variable is present, it never goes
through VSM_Arg and relative working directories are never turned into
their absolute counterparts.

Again, I insist, we already had a fallback handling in VSM_Attach in
the absence of a -n option, and this is where the intermediate
VARNISH_DEFAULT_N should be introduced.

> When you write "there should be only 2 locations needed", I agree that two
> locations are better than four, but no matter how much I appreciate your desire
> to minimize code changes, I do not agree that this is now handled at the right
> place for the reasons given.

This not about minimizing the change for the sake of minimizing. I
wanted to avoid spreading the handling of the environment variable in
more places than needed, increasing the risk of doing so incorrectly.

And the risk existed since we both got it wrong initially.

Dridi

From nils.goroll at uplex.de  Mon May 27 11:51:57 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 27 May 2024 13:51:57 +0200
Subject: [master] 32574ab46 vsm: Pass VARNISH_DEFAULT_N to VSM_Arg()
In-Reply-To: <CABoVN9D9XWEm_b_0c1ueyMFVKQ0ZnqOPOAY2pAi4pJwoctOzUA@mail.gmail.com>
References: <20240525050205.A6B1CA387A@lists.varnish-cache.org>
 <488d5f63-b6e2-4818-8d40-53406d60bf0e@uplex.de>
 <CABoVN9Ctb7XPcvvfhHRVthCznuC5ZvtZZ1hKR1B4RGfNJNFSyg@mail.gmail.com>
 <6257e218-bb9d-4a93-b8ad-adf05e71aefa@uplex.de>
 <CABoVN9D9XWEm_b_0c1ueyMFVKQ0ZnqOPOAY2pAi4pJwoctOzUA@mail.gmail.com>
Message-ID: <b696ef7b-6249-48d2-8c18-0f168757f00b@uplex.de>

On 27.05.24 13:19, Dridi Boukelmoune wrote:
> as soon as the environment variable is present, it never goes
> through VSM_Arg and relative working directories are never turned into
> their absolute counterparts.

OK, sorry, it took me until now to see this.

Thank you, Nils

-- 

Nils Goroll (he/him)

** * * UPLEX - Nils Goroll Systemoptimierung

Scheffelstra?e 32
22301 Hamburg

tel +49 40 28805731
mob +49 170 2723133
fax +49 40 42949753

xmpp://slink at jabber.int.uplex.de/

http://uplex.de/


From dridi at varni.sh  Mon May 27 12:54:30 2024
From: dridi at varni.sh (Dridi Boukelmoune)
Date: Mon, 27 May 2024 12:54:30 +0000
Subject: [master] 32574ab46 vsm: Pass VARNISH_DEFAULT_N to VSM_Arg()
In-Reply-To: <b696ef7b-6249-48d2-8c18-0f168757f00b@uplex.de>
References: <20240525050205.A6B1CA387A@lists.varnish-cache.org>
 <488d5f63-b6e2-4818-8d40-53406d60bf0e@uplex.de>
 <CABoVN9Ctb7XPcvvfhHRVthCznuC5ZvtZZ1hKR1B4RGfNJNFSyg@mail.gmail.com>
 <6257e218-bb9d-4a93-b8ad-adf05e71aefa@uplex.de>
 <CABoVN9D9XWEm_b_0c1ueyMFVKQ0ZnqOPOAY2pAi4pJwoctOzUA@mail.gmail.com>
 <b696ef7b-6249-48d2-8c18-0f168757f00b@uplex.de>
Message-ID: <CABoVN9ARE68YCH6T53CHxgbSdqoXTQ6USn-NvHbMtY37b2RYig@mail.gmail.com>

On Mon, May 27, 2024 at 11:51?AM Nils Goroll <nils.goroll at uplex.de> wrote:
>
> On 27.05.24 13:19, Dridi Boukelmoune wrote:
> > as soon as the environment variable is present, it never goes
> > through VSM_Arg and relative working directories are never turned into
> > their absolute counterparts.
>
> OK, sorry, it took me until now to see this.
>
> Thank you, Nils

Thank you for insisting, I believe I had a good understanding of your
arguments and you forced me to clarify the point I was struggling to
make.

Dridi

From nils.goroll at uplex.de  Mon May 27 15:46:02 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 27 May 2024 15:46:02 +0000 (UTC)
Subject: [master] 97537cb17 Create a directory owned by the worker process for
 use as a TMPDIR
Message-ID: <20240527154602.BC6099C0BE@lists.varnish-cache.org>


commit 97537cb1784f69a5862acf6d4e78713a2cb1e033
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon May 27 17:43:22 2024 +0200

    Create a directory owned by the worker process for use as a TMPDIR
    
    Processes created by the worker might require a TMPDIR for temporary
    files (think: filters). Create a dedicated directory for this purpose,
    which is otherwise hard to achieve in a universal manner.

diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 192ae202a..1b2ddbc2f 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -319,6 +319,7 @@ mgt_Cflag_atexit(void)
 		VJ_rmdir("vext_cache");
 	}
 	VJ_rmdir("vmod_cache");
+	VJ_rmdir("worker_tmpdir");
 	(void)chdir("/");
 	VJ_rmdir(workdir);
 }
@@ -865,7 +866,7 @@ main(int argc, char * const *argv)
 		    workdir, VAS_errtxt(errno));
 
 	VJ_master(JAIL_MASTER_SYSTEM);
-	AZ(system("rm -rf vmod_cache vext_cache"));
+	AZ(system("rm -rf vmod_cache vext_cache worker_tmpdir"));
 	VJ_master(JAIL_MASTER_LOW);
 
 	if (VJ_make_subdir("vmod_cache", "VMOD cache", NULL)) {
@@ -881,6 +882,13 @@ main(int argc, char * const *argv)
 		    workdir, VAS_errtxt(errno));
 	}
 
+	if (VJ_make_subdir("worker_tmpdir",
+	    "TMPDIR for the worker process", NULL)) {
+		ARGV_ERR(
+		    "Cannot create vmod directory (%s/worker_tmpdir): %s\n",
+		    workdir, VAS_errtxt(errno));
+	}
+
 	if (C_flag)
 		AZ(atexit(mgt_Cflag_atexit));
 
diff --git a/doc/changes.rst b/doc/changes.rst
index dbe31e3ab..ba74f7c13 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -41,11 +41,17 @@ Varnish Cache NEXT (2024-09-15)
 .. PLEASE keep this roughly in commit order as shown by git-log / tig
    (new to old)
 
+* ``varnishd`` now creates a ``worker_tmpdir`` which can be used by
+  VMODs for temporary files. The `VMOD deleveloper documentation`_ has
+  details.
+
 * The environment variable ``VARNISH_DEFAULT_N`` now provides the
   default "varnish name" / "workdir" as otherwise specified by he
   ``-n`` argument to ``varnishd`` and ``varnish*`` utilities except
   ``varnishtest``.
 
+.. _VMOD deleveloper documentation: doc/sphinx/reference/vmod.rst
+
 ================================
 Varnish Cache 7.5.0 (2024-03-18)
 ================================
diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst
index b8a524ce6..193b9060d 100644
--- a/doc/sphinx/reference/vmod.rst
+++ b/doc/sphinx/reference/vmod.rst
@@ -858,3 +858,47 @@ You should call ``VSC_*_New()`` when your VMOD is loaded and
 ``VSC_*_Destroy()`` when it is unloaded. See the generated
 ``VSC_*.h`` file for the full details about the structure that contains
 your counters.
+
+Temporary Files
+===============
+
+``varnishd`` creates a directroy named ``worker_tmpdir`` under the
+varnish working directory (see ``varnishd -n`` argument) for
+read/write access by the worker process.
+
+From the perspective of VMODs, the relative path is always
+``worker_tmpdir``.
+
+This directory is intended (though not limited) to provide a place for
+VMODs to create temporary files using ``mkstemp()`` and related libc
+functions. VMODs are responsible for cleaning up files which are no
+longer required, and they will ultimately be removed when the
+``varnishd`` worker process restarts. There is no isolation between
+VMODs (as is the case anyway).
+
+A simple example for how to use it::
+
+	#include <stdlib.h>
+	#include <unistd.h>
+
+	#include "vdef.h"
+	#include "vas.h"
+
+	static void
+	tmpfile_example(void) {
+	    int fd;
+	    char name[] = "worker_tmpdir/myvmod.XXXXXX";
+
+	    fd = mkstemp(name);
+	    if (fd < 0) {
+		// handle error
+		return;
+	    }
+
+	    // hide file
+	    AZ(unlink(name));
+
+	    // use fd
+
+	    AZ(close(fd));
+	}
diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c
index 8f3f5636e..e2e42ee51 100644
--- a/vmod/vmod_debug.c
+++ b/vmod/vmod_debug.c
@@ -55,6 +55,7 @@ struct priv_vcl {
 	VCL_BACKEND		be;
 	unsigned		cold_be;
 	unsigned		cooling_be;
+	int			tmpf;
 };
 
 
@@ -401,12 +402,18 @@ VCL_VOID v_matchproto_(td_debug_test_priv_vcl)
 xyzzy_test_priv_vcl(VRT_CTX, struct vmod_priv *priv)
 {
 	struct priv_vcl *priv_vcl;
+	char t[PATH_MAX];
+	ssize_t l;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	AN(priv);
 	CAST_OBJ_NOTNULL(priv_vcl, priv->priv, PRIV_VCL_MAGIC);
+
+	l = pread(priv_vcl->tmpf, t, sizeof t, 0);
+	assert(l > 0);
+
 	AN(priv_vcl->foo);
-	assert(!strcmp(priv_vcl->foo, "FOO"));
+	assert(!strncmp(priv_vcl->foo, t, l));
 }
 
 VCL_VOID v_matchproto_(td_debug_rot104)
@@ -501,7 +508,9 @@ priv_vcl_fini(VRT_CTX, void *priv)
 	struct priv_vcl *priv_vcl;
 
 	CAST_OBJ_NOTNULL(priv_vcl, priv, PRIV_VCL_MAGIC);
+	AZ(close(priv_vcl->tmpf));
 	AN(priv_vcl->foo);
+	AZ(unlink(priv_vcl->foo));
 	free(priv_vcl->foo);
 	if (priv_vcl->obj_cb != 0) {
 		ObjUnsubscribeEvents(&priv_vcl->obj_cb);
@@ -534,8 +543,11 @@ event_load(VRT_CTX, struct vmod_priv *priv)
 
 	ALLOC_OBJ(priv_vcl, PRIV_VCL_MAGIC);
 	AN(priv_vcl);
-	priv_vcl->foo = strdup("FOO");
+	priv_vcl->foo = strdup("worker_tmpdir/vmod_debug.XXXXXX");
 	AN(priv_vcl->foo);
+	priv_vcl->tmpf = mkstemp(priv_vcl->foo);
+	assert(priv_vcl->tmpf >= 0);
+	AN(write(priv_vcl->tmpf, priv_vcl->foo, strlen(priv_vcl->foo)));
 	priv->priv = priv_vcl;
 	priv->methods = priv_vcl_methods;
 

From nils.goroll at uplex.de  Mon May 27 15:54:04 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 27 May 2024 15:54:04 +0000 (UTC)
Subject: [master] 6d1ea035e Copy timeouts from a via backend if undefined for
 the destination
Message-ID: <20240527155405.137859C710@lists.varnish-cache.org>


commit 6d1ea035e2c90f58dd740e4720a9ac7020886854
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue May 14 15:54:39 2024 +0200

    Copy timeouts from a via backend if undefined for the destination
    
    note: use of isnan() is for forward compatibility, at the moment, we use
    -1 to denone NAN for backends.

diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c
index 978d50c5d..5c63a591e 100644
--- a/bin/varnishd/cache/cache_backend.c
+++ b/bin/varnishd/cache/cache_backend.c
@@ -694,6 +694,18 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc,
 #undef DA
 #undef DN
 
+#define CPTMO(a, b, x) do {				\
+		if ((a)->x < 0.0 || isnan((a)->x))	\
+			(a)->x = (b)->x;		\
+	} while(0)
+
+	if (viabe != NULL) {
+		CPTMO(be, viabe, connect_timeout);
+		CPTMO(be, viabe, first_byte_timeout);
+		CPTMO(be, viabe, between_bytes_timeout);
+	}
+#undef CPTMO
+
 	if (viabe || be->hosthdr == NULL) {
 		if (vrt->endpoint->uds_path != NULL)
 			sa = bogo_ip;
diff --git a/doc/changes.rst b/doc/changes.rst
index ba74f7c13..139d4c93d 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -41,6 +41,11 @@ Varnish Cache NEXT (2024-09-15)
 .. PLEASE keep this roughly in commit order as shown by git-log / tig
    (new to old)
 
+* for backends using the ``.via`` attribute to connect through a
+  *proxy*, the ``connect_timeout``, ``first_byte_timeout`` and
+  ``between_bytes_timeout`` attributes are now inherited from *proxy*
+  unless explicitly given.
+
 * ``varnishd`` now creates a ``worker_tmpdir`` which can be used by
   VMODs for temporary files. The `VMOD deleveloper documentation`_ has
   details.
diff --git a/doc/sphinx/reference/vcl-backend.rst b/doc/sphinx/reference/vcl-backend.rst
index ae1a994ec..97bb681c3 100644
--- a/doc/sphinx/reference/vcl-backend.rst
+++ b/doc/sphinx/reference/vcl-backend.rst
@@ -169,6 +169,10 @@ supported.
 The ``.via`` attribute is unrelated to ``.proxy_header``. If both are
 used, a second header is sent as per ``.proxy_header`` specification.
 
+Unless specified for *destination*, the ``connect_timeout``,
+``first_byte_timeout`` and ``between_bytes_timeout`` attributes are
+copied from *proxy* to *destination*.
+
 As of this release, the *proxy* backend used with ``.via`` can not be
 a director, it can not itself use ``.via`` (error: *Can not stack .via
 backends*) and the protocol is fixed to `PROXY2`_.

From nils.goroll at uplex.de  Mon May 27 15:59:02 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 27 May 2024 15:59:02 +0000 (UTC)
Subject: [master] 77110d7bd Return a consistent boc state from ObjWaitExtend()
Message-ID: <20240527155902.AED699CB7C@lists.varnish-cache.org>


commit 77110d7bda11e7feddd89a58b5aaa0e0c77159b5
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue May 21 17:16:05 2024 +0200

    Return a consistent boc state from ObjWaitExtend()
    
    Clients to the Object API need to know not only the current extension
    (new length) of streaming objects, but also the streaming state - in
    particular BOS_FINISHED and BOS_FAILED. The latter for obvious
    reasons, and the former to call the delivery function with
    OBJ_ITER_END, which then likely results in VDP_END sent down the
    delivery pipeline.
    
    Background:
    
    It is important for efficient delivery to not receive an additional
    VDP_END with a null buffer, but rather combined with the last chunk of
    data, so, consequently, it is important to reliably send OBJ_INTER_END
    also with the last chunk of data.
    
    Consequent to all of this, ObjWaitExtend() callers need to know when
    BOS_FINISHED has been reached for some extension.
    
    The current API, however, does not provide a consistent view of the
    streaming state, which is only available from within the critical region
    of ObjWaitExtend().
    
    Thus, we add the streaming state as an optional return value.
    
    With this commit, we also remove a superfluous line to set rv again:
    Because boc->fetched_so_far must only be updated while holding the boc
    mutex, reading the value again provides no benefit.

diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c
index ec819bf8c..21c3d8f91 100644
--- a/bin/varnishd/cache/cache_obj.c
+++ b/bin/varnishd/cache/cache_obj.c
@@ -265,8 +265,10 @@ ObjExtend(struct worker *wrk, struct objcore *oc, ssize_t l, int final)
  */
 
 uint64_t
-ObjWaitExtend(const struct worker *wrk, const struct objcore *oc, uint64_t l)
+ObjWaitExtend(const struct worker *wrk, const struct objcore *oc, uint64_t l,
+    enum boc_state_e *statep)
 {
+	enum boc_state_e state;
 	uint64_t rv;
 
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
@@ -282,15 +284,16 @@ ObjWaitExtend(const struct worker *wrk, const struct objcore *oc, uint64_t l)
 			oc->boc->delivered_so_far = l;
 			PTOK(pthread_cond_signal(&oc->boc->cond));
 		}
-		if (rv > l || oc->boc->state >= BOS_FINISHED)
+		state = oc->boc->state;
+		if (rv > l || state >= BOS_FINISHED)
 			break;
 		(void)Lck_CondWait(&oc->boc->cond, &oc->boc->mtx);
 	}
-	rv = oc->boc->fetched_so_far;
 	Lck_Unlock(&oc->boc->mtx);
+	if (statep != NULL)
+		*statep = state;
 	return (rv);
 }
-
 /*====================================================================
  */
 
diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h
index add300e76..9bd5e2490 100644
--- a/bin/varnishd/cache/cache_varnishd.h
+++ b/bin/varnishd/cache/cache_varnishd.h
@@ -335,7 +335,7 @@ void ObjDestroy(const struct worker *, struct objcore **);
 int ObjGetSpace(struct worker *, struct objcore *, ssize_t *sz, uint8_t **ptr);
 void ObjExtend(struct worker *, struct objcore *, ssize_t l, int final);
 uint64_t ObjWaitExtend(const struct worker *, const struct objcore *,
-    uint64_t l);
+    uint64_t l, enum boc_state_e *statep);
 void ObjSetState(struct worker *, const struct objcore *,
     enum boc_state_e next);
 void ObjWaitState(const struct objcore *, enum boc_state_e want);
diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c
index 6626d3a0d..68c780552 100644
--- a/bin/varnishd/storage/storage_simple.c
+++ b/bin/varnishd/storage/storage_simple.c
@@ -362,23 +362,16 @@ sml_iterator(struct worker *wrk, struct objcore *oc,
 	}
 	while (1) {
 		ol = len;
-		nl = ObjWaitExtend(wrk, oc, ol);
-		if (boc->state == BOS_FAILED) {
+		nl = ObjWaitExtend(wrk, oc, ol, &state);
+		if (state == BOS_FAILED) {
 			ret = -1;
 			break;
 		}
 		if (nl == ol) {
-			/*
-			 * note: the unguarded boc->state read could be
-			 * outdated, in which case we call ObjWaitExtend() again
-			 * for error handling but otherwise cause no harm. When
-			 * using this code as an example, DO NOT rely on
-			 * boc->state to be consistent
-			 */
-			if (boc->state == BOS_FINISHED)
-				break;
-			continue;
+			assert(state == BOS_FINISHED);
+			break;
 		}
+		assert(nl > ol);
 		Lck_Lock(&boc->mtx);
 		AZ(VTAILQ_EMPTY(&obj->list));
 		if (checkpoint == NULL) {
@@ -417,7 +410,6 @@ sml_iterator(struct worker *wrk, struct objcore *oc,
 		st = VTAILQ_PREV(st, storagehead, list);
 		if (st != NULL && st->len == 0)
 			st = NULL;
-		state = boc->state;
 		Lck_Unlock(&boc->mtx);
 		assert(l > 0 || state == BOS_FINISHED);
 		u = 0;
diff --git a/doc/changes.rst b/doc/changes.rst
index 139d4c93d..29a409b6c 100644
--- a/doc/changes.rst
+++ b/doc/changes.rst
@@ -41,6 +41,11 @@ Varnish Cache NEXT (2024-09-15)
 .. PLEASE keep this roughly in commit order as shown by git-log / tig
    (new to old)
 
+* The ObjWaitExtend() Object API function gained a ``statep`` argument
+  to optionally return the busy object state consistent with the
+  current extension. A ``NULL`` value may be passed if the caller does
+  not require it.
+
 * for backends using the ``.via`` attribute to connect through a
   *proxy*, the ``connect_timeout``, ``first_byte_timeout`` and
   ``between_bytes_timeout`` attributes are now inherited from *proxy*
diff --git a/include/vrt.h b/include/vrt.h
index a7ed22bab..707035497 100644
--- a/include/vrt.h
+++ b/include/vrt.h
@@ -48,7 +48,7 @@
 
 #define VRT_MAJOR_VERSION	19U
 
-#define VRT_MINOR_VERSION	0U
+#define VRT_MINOR_VERSION	1U
 
 /***********************************************************************
  * Major and minor VRT API versions.
@@ -58,6 +58,8 @@
  * binary/load-time compatible, increment MAJOR version
  *
  * NEXT (2024-09-15)
+ * 19.1 (2024-05-27)
+ *	[cache_varnishd.h] ObjWaitExtend() gained statep argument
  * 19.0 (2024-03-18)
  *	[cache.h] (struct req).filter_list renamed to vdp_filter_list
  *	order of vcl/vmod and director COLD events reversed to directors first

From nils.goroll at uplex.de  Mon May 27 17:31:04 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Mon, 27 May 2024 17:31:04 +0000 (UTC)
Subject: [master] fec25970c Fix worker_tmpdir when the unix jail is active
Message-ID: <20240527173104.CAB25A2F33@lists.varnish-cache.org>


commit fec25970c8addc28fb90d4dcd8d4fea592a03b13
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Mon May 27 19:29:40 2024 +0200

    Fix worker_tmpdir when the unix jail is active
    
    I misread the code and did not try running varnishtest as root :|

diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h
index 5b22eb85c..5b75a2da3 100644
--- a/bin/varnishd/mgt/mgt.h
+++ b/bin/varnishd/mgt/mgt.h
@@ -131,6 +131,7 @@ enum jail_fixfd_e {
 	JAIL_FIXFD_FILE,
 	JAIL_FIXFD_VSMMGT,
 	JAIL_FIXFD_VSMWRK,
+	JAIL_FIXFD_WRKTMP,
 };
 
 typedef int jail_init_f(char **);
diff --git a/bin/varnishd/mgt/mgt_jail_unix.c b/bin/varnishd/mgt/mgt_jail_unix.c
index f84d63c2e..ecb51ffac 100644
--- a/bin/varnishd/mgt/mgt_jail_unix.c
+++ b/bin/varnishd/mgt/mgt_jail_unix.c
@@ -304,6 +304,7 @@ vju_fixfd(int fd, enum jail_fixfd_e what)
 		AZ(fchown(fd, vju_uid, vju_gid));
 		break;
 	case JAIL_FIXFD_VSMWRK:
+	case JAIL_FIXFD_WRKTMP:
 		AZ(fchmod(fd, 0750));
 		AZ(fchown(fd, vju_wrkuid, vju_wrkgid));
 		break;
diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c
index 1b2ddbc2f..d6fe6bef3 100644
--- a/bin/varnishd/mgt/mgt_main.c
+++ b/bin/varnishd/mgt/mgt_main.c
@@ -889,6 +889,12 @@ main(int argc, char * const *argv)
 		    workdir, VAS_errtxt(errno));
 	}
 
+	o = open("worker_tmpdir", O_RDONLY);
+	VJ_master(JAIL_MASTER_SYSTEM);
+	VJ_fix_fd(o, JAIL_FIXFD_WRKTMP);
+	VJ_master(JAIL_MASTER_LOW);
+	closefd(&o);
+
 	if (C_flag)
 		AZ(atexit(mgt_Cflag_atexit));
 

From nils.goroll at uplex.de  Tue May 28 08:41:05 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 28 May 2024 08:41:05 +0000 (UTC)
Subject: [master] 8fc1b2c3a vtest: Add "option timeout" to adjust the timeout
 per test case
Message-ID: <20240528084105.1781592BDE@lists.varnish-cache.org>


commit 8fc1b2c3af3eea0aa0a424fdc6bb76788a9f5661
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue May 28 10:38:03 2024 +0200

    vtest: Add "option timeout" to adjust the timeout per test case

diff --git a/bin/varnishtest/cmds.h b/bin/varnishtest/cmds.h
index 3b8a9e9fd..257372b11 100644
--- a/bin/varnishtest/cmds.h
+++ b/bin/varnishtest/cmds.h
@@ -45,6 +45,7 @@ CMD_GLOBAL(include)
 #endif
 CMD_TOP(client)
 CMD_TOP(feature)
+CMD_TOP(option)
 CMD_TOP(filewrite)
 CMD_TOP(haproxy)
 #ifdef VTEST_WITH_VTC_LOGEXPECT
diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc
index 2d3604f24..423e8e23c 100644
--- a/bin/varnishtest/tests/a00000.vtc
+++ b/bin/varnishtest/tests/a00000.vtc
@@ -1,5 +1,8 @@
 varnishtest "Test varnishtest itself"
 
+option timeout 60s
+option timeout x1.1
+
 shell -exit 1 -expect {varnishtest [options]} {varnishtest -h}
 
 shell -exit 1 -match {-D.*Define macro} {varnishtest -h}
diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c
index 64d7b506d..52ec902b3 100644
--- a/bin/varnishtest/vtc_misc.c
+++ b/bin/varnishtest/vtc_misc.c
@@ -660,3 +660,52 @@ cmd_feature(CMD_ARGS)
 		return;
 	}
 }
+
+static void
+set_timeout(struct vtclog *vl, const char *val)
+{
+	double num;
+	char *s = NULL;
+
+	if (val == NULL || *val == '\0') {
+		vtc_fatal(vl, "FAIL test, missing \"option timeout\" value");
+		return;
+	}
+	if (*val == 'x') {
+		num = strtod(val + 1, &s);
+		num *= vtc_maxdur;
+	} else
+		num = VNUM_duration(val);
+
+	if (isnan(num) || (s != NULL && *s != '\0')) {
+		vtc_fatal(vl, "FAIL test, invalid \"option timeout\": %s", val);
+		return;
+	}
+	vtc_maxdur = num;
+	vtc_log(vl, 4, "new timeout %.2f", vtc_maxdur);
+}
+
+/* SECTION: option <option> <value>
+ *
+ * Set varnishtest options
+ *
+ * timeout <duration>
+ *         Override -t argument
+ * timeout x<factor>
+ *         Scale timeout by a <factor>
+ */
+
+void v_matchproto_(cmd_f)
+cmd_option(CMD_ARGS)
+{
+
+	(void)priv;
+
+	if (av == NULL)
+		return;
+	av++;
+	if (!strcmp(av[0], "timeout"))
+		set_timeout(vl, av[1]);
+	else
+		vtc_fatal(vl, "FAIL test, unknown option: %s", av[0]);
+}

From nils.goroll at uplex.de  Tue May 28 08:41:05 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 28 May 2024 08:41:05 +0000 (UTC)
Subject: [master] 9b76b9a4b Give v51.vtc more time
Message-ID: <20240528084105.3524D92BE5@lists.varnish-cache.org>


commit 9b76b9a4baa4fedd2fb3a904b0dceafd263d0ddc
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue May 28 10:40:14 2024 +0200

    Give v51.vtc more time
    
    it repeatedly timed out on arm

diff --git a/bin/varnishtest/tests/v00051.vtc b/bin/varnishtest/tests/v00051.vtc
index dbdb26de2..56069362d 100644
--- a/bin/varnishtest/tests/v00051.vtc
+++ b/bin/varnishtest/tests/v00051.vtc
@@ -1,5 +1,7 @@
 varnishtest "Test VCL failures"
 
+option timeout x2
+
 server s1 {
 	rxreq
 	expect req.url == /hit

From phk at phk.freebsd.dk  Tue May 28 08:46:35 2024
From: phk at phk.freebsd.dk (Poul-Henning Kamp)
Date: Tue, 28 May 2024 08:46:35 +0000
Subject: [master] 8fc1b2c3a vtest: Add "option timeout" to adjust the
 timeout per test case
In-Reply-To: <20240528084105.1781592BDE@lists.varnish-cache.org>
References: <20240528084105.1781592BDE@lists.varnish-cache.org>
Message-ID: <202405280846.44S8kZLn054066@critter.freebsd.dk>

--------
Nils Goroll writes:

> commit 8fc1b2c3af3eea0aa0a424fdc6bb76788a9f5661
> Author: Nils Goroll <nils.goroll at uplex.de>
> Date:   Tue May 28 10:38:03 2024 +0200
>
>     vtest: Add "option timeout" to adjust the timeout per test case

I /really/ do not like this.

First:  It overrides constraints which the person running the tests
have laid down.  That is just not OK.  If you could only use it
to shorten timeouts, it would be OK-ish.

Second: It makes assumptions about the platform the test runs on.

-- 
Poul-Henning Kamp       | UNIX since Zilog Zeus 3.20
phk at FreeBSD.ORG         | TCP/IP since RFC 956
FreeBSD committer       | BSD since 4.3-tahoe    
Never attribute to malice what can adequately be explained by incompetence.

From nils.goroll at uplex.de  Tue May 28 09:04:04 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 28 May 2024 09:04:04 +0000 (UTC)
Subject: [master] d6aef69a6 vtest: Polish option code
Message-ID: <20240528090404.C1ADD93D8E@lists.varnish-cache.org>


commit d6aef69a6b1da8fc8e392d2de159a0861418d7ce
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue May 28 11:02:54 2024 +0200

    vtest: Polish option code
    
    Code after vtc_fatal() is unreachable

diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c
index 52ec902b3..686fd191e 100644
--- a/bin/varnishtest/vtc_misc.c
+++ b/bin/varnishtest/vtc_misc.c
@@ -667,20 +667,16 @@ set_timeout(struct vtclog *vl, const char *val)
 	double num;
 	char *s = NULL;
 
-	if (val == NULL || *val == '\0') {
+	if (val == NULL || *val == '\0')
 		vtc_fatal(vl, "FAIL test, missing \"option timeout\" value");
-		return;
-	}
 	if (*val == 'x') {
 		num = strtod(val + 1, &s);
 		num *= vtc_maxdur;
 	} else
 		num = VNUM_duration(val);
 
-	if (isnan(num) || (s != NULL && *s != '\0')) {
+	if (isnan(num) || (s != NULL && *s != '\0'))
 		vtc_fatal(vl, "FAIL test, invalid \"option timeout\": %s", val);
-		return;
-	}
 	vtc_maxdur = num;
 	vtc_log(vl, 4, "new timeout %.2f", vtc_maxdur);
 }

From nils.goroll at uplex.de  Tue May 28 09:13:04 2024
From: nils.goroll at uplex.de (Nils Goroll)
Date: Tue, 28 May 2024 09:13:04 +0000 (UTC)
Subject: [master] 165921542 Revert vtest: Add "option timeout"
Message-ID: <20240528091304.F099F94453@lists.varnish-cache.org>


commit 16592154287145f5bebe77e29128dc3ec77d133a
Author: Nils Goroll <nils.goroll at uplex.de>
Date:   Tue May 28 11:10:54 2024 +0200

    Revert vtest: Add "option timeout"
    
    veto'ed by phk.
    
    It was my intention to make a helpful improvement along the way, but I have
    no capacity at the moment for a longer process.
    
    This reverts commit d6aef69a6b1da8fc8e392d2de159a0861418d7ce.
    This reverts commit 9b76b9a4baa4fedd2fb3a904b0dceafd263d0ddc.
    This reverts commit 8fc1b2c3af3eea0aa0a424fdc6bb76788a9f5661.

diff --git a/bin/varnishtest/cmds.h b/bin/varnishtest/cmds.h
index 257372b11..3b8a9e9fd 100644
--- a/bin/varnishtest/cmds.h
+++ b/bin/varnishtest/cmds.h
@@ -45,7 +45,6 @@ CMD_GLOBAL(include)
 #endif
 CMD_TOP(client)
 CMD_TOP(feature)
-CMD_TOP(option)
 CMD_TOP(filewrite)
 CMD_TOP(haproxy)
 #ifdef VTEST_WITH_VTC_LOGEXPECT
diff --git a/bin/varnishtest/tests/a00000.vtc b/bin/varnishtest/tests/a00000.vtc
index 423e8e23c..2d3604f24 100644
--- a/bin/varnishtest/tests/a00000.vtc
+++ b/bin/varnishtest/tests/a00000.vtc
@@ -1,8 +1,5 @@
 varnishtest "Test varnishtest itself"
 
-option timeout 60s
-option timeout x1.1
-
 shell -exit 1 -expect {varnishtest [options]} {varnishtest -h}
 
 shell -exit 1 -match {-D.*Define macro} {varnishtest -h}
diff --git a/bin/varnishtest/tests/v00051.vtc b/bin/varnishtest/tests/v00051.vtc
index 56069362d..dbdb26de2 100644
--- a/bin/varnishtest/tests/v00051.vtc
+++ b/bin/varnishtest/tests/v00051.vtc
@@ -1,7 +1,5 @@
 varnishtest "Test VCL failures"
 
-option timeout x2
-
 server s1 {
 	rxreq
 	expect req.url == /hit
diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c
index 686fd191e..64d7b506d 100644
--- a/bin/varnishtest/vtc_misc.c
+++ b/bin/varnishtest/vtc_misc.c
@@ -660,48 +660,3 @@ cmd_feature(CMD_ARGS)
 		return;
 	}
 }
-
-static void
-set_timeout(struct vtclog *vl, const char *val)
-{
-	double num;
-	char *s = NULL;
-
-	if (val == NULL || *val == '\0')
-		vtc_fatal(vl, "FAIL test, missing \"option timeout\" value");
-	if (*val == 'x') {
-		num = strtod(val + 1, &s);
-		num *= vtc_maxdur;
-	} else
-		num = VNUM_duration(val);
-
-	if (isnan(num) || (s != NULL && *s != '\0'))
-		vtc_fatal(vl, "FAIL test, invalid \"option timeout\": %s", val);
-	vtc_maxdur = num;
-	vtc_log(vl, 4, "new timeout %.2f", vtc_maxdur);
-}
-
-/* SECTION: option <option> <value>
- *
- * Set varnishtest options
- *
- * timeout <duration>
- *         Override -t argument
- * timeout x<factor>
- *         Scale timeout by a <factor>
- */
-
-void v_matchproto_(cmd_f)
-cmd_option(CMD_ARGS)
-{
-
-	(void)priv;
-
-	if (av == NULL)
-		return;
-	av++;
-	if (!strcmp(av[0], "timeout"))
-		set_timeout(vl, av[1]);
-	else
-		vtc_fatal(vl, "FAIL test, unknown option: %s", av[0]);
-}

From dridi.boukelmoune at gmail.com  Wed May 29 08:18:06 2024
From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune)
Date: Wed, 29 May 2024 08:18:06 +0000 (UTC)
Subject: [master] bb57b07fc vsm: s/fanton/fantom/
Message-ID: <20240529081806.314A6A41A5@lists.varnish-cache.org>


commit bb57b07fc03e8db49753203b61d381e215d8e6cf
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Wed May 29 10:16:51 2024 +0200

    vsm: s/fanton/fantom/

diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h
index 065161787..cbc12941c 100644
--- a/include/vapi/vsm.h
+++ b/include/vapi/vsm.h
@@ -182,7 +182,7 @@ int VSM_Get(struct vsm *, struct vsm_fantom *vf,
 
 char *VSM_Dup(struct vsm*, const char *category, const char *ident);
 	/*
-	 * Returns a malloc'ed copy of the fanton.
+	 * Returns a malloc'ed copy of the fantom.
 	 *
 	 * Return:
 	 *   NULL = Failure