From dridi.boukelmoune at gmail.com Mon Jun 3 08:46:05 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 3 Jun 2024 08:46:05 +0000 (UTC) Subject: [6.0] 255896748 vrt: Perform IP to STRING conversion on the stack Message-ID: <20240603084605.5D091A013B@lists.varnish-cache.org> commit 2558967480a41bff781369e7dad0c0fcc1237897 Author: Dridi Boukelmoune Date: Mon Jan 3 18:36:16 2022 +0100 vrt: Perform IP to STRING conversion on the stack Performing the conversion on the stack could lead to a buffer too small to store the string representation of the IP address. There is no test case because the error handling is output to stderr. Refs #3765 Conflicts: bin/varnishd/cache/cache_vrt.c diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 27c80fb5d..d52ad0bb5 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -597,21 +597,16 @@ VRT_r_now(VRT_CTX) VCL_STRING v_matchproto_() VRT_IP_string(VRT_CTX, VCL_IP ip) { - char *p; - unsigned len; + char buf[VTCP_ADDRBUFSIZE]; + struct vsb vsb[1]; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if (ip == NULL) return (NULL); - len = WS_ReserveAll(ctx->ws); - if (len == 0) { - WS_Release(ctx->ws, 0); - return (NULL); - } - p = ctx->ws->f; - VTCP_name(ip, p, len, NULL, 0); - WS_Release(ctx->ws, strlen(p) + 1); - return (p); + VTCP_name(ip, buf, sizeof buf, NULL, 0); + WS_VSB_new(vsb, ctx->ws); + VSB_cat(vsb, buf); + return (WS_VSB_finish(vsb, ctx->ws, NULL)); } VCL_STRING v_matchproto_() From dridi.boukelmoune at gmail.com Mon Jun 3 08:46:05 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 3 Jun 2024 08:46:05 +0000 (UTC) Subject: [6.0] ec8f05464 vrt: Skip the workspace reservation in VRT_IP_string() Message-ID: <20240603084605.707C0A013E@lists.varnish-cache.org> commit ec8f05464fbed40565103beb4a1d834a1f5b24a1 Author: Dridi Boukelmoune Date: Mon Jan 16 12:33:20 2023 +0100 vrt: Skip the workspace reservation in VRT_IP_string() diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index d52ad0bb5..71ecd1d73 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -598,15 +598,12 @@ VCL_STRING v_matchproto_() VRT_IP_string(VRT_CTX, VCL_IP ip) { char buf[VTCP_ADDRBUFSIZE]; - struct vsb vsb[1]; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if (ip == NULL) return (NULL); VTCP_name(ip, buf, sizeof buf, NULL, 0); - WS_VSB_new(vsb, ctx->ws); - VSB_cat(vsb, buf); - return (WS_VSB_finish(vsb, ctx->ws, NULL)); + return (WS_Copy(ctx->ws, buf, -1)); } VCL_STRING v_matchproto_() From dridi.boukelmoune at gmail.com Mon Jun 3 09:07:05 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 3 Jun 2024 09:07:05 +0000 (UTC) Subject: [master] d24e0c2e9 circleci: Switch centos image to stream9 tag Message-ID: <20240603090705.8946AA0FF7@lists.varnish-cache.org> commit d24e0c2e9547a0368449da664f2647c2eb68c838 Author: Dridi Boukelmoune Date: Mon Jun 3 11:03:03 2024 +0200 circleci: Switch centos image to stream9 tag The plain "stream" image has not been updated for a year, and instead of being the equivalent of a "latest" tag it just appears to be frozen. Since there isn't an automatic centos-stream upgrade path, let's target version 9 explicitly for now. diff --git a/.circleci/config.yml b/.circleci/config.yml index 171165f5c..b178e8a27 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -391,7 +391,7 @@ workflows: name: build_centos_stream prefix: quay.io/centos/ dist: centos - release: stream + release: stream9 - build: name: build_almalinux_8 dist: almalinux From dridi.boukelmoune at gmail.com Mon Jun 3 09:25:05 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 3 Jun 2024 09:25:05 +0000 (UTC) Subject: [master] 67188be2a circleci: Missing nines for centos:stream9 Message-ID: <20240603092505.AB7C7A1B2B@lists.varnish-cache.org> commit 67188be2a282ee8b29d524f17a7ffd05d0cbfe7a Author: Dridi Boukelmoune Date: Mon Jun 3 11:21:15 2024 +0200 circleci: Missing nines for centos:stream9 I don't understand why they didn't maintain the unversioned centos:stream tag as a kind of latest tag after dropping the actual centos:latest tag. diff --git a/.circleci/config.yml b/.circleci/config.yml index b178e8a27..409da8da8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -174,7 +174,7 @@ jobs: esac case "<< parameters.platform >>" in - centos:stream) REPO=quay.io/centos/ ;; + centos:stream9) REPO=quay.io/centos/ ;; *) REPO= ;; esac @@ -239,13 +239,13 @@ jobs: centos|almalinux|fedora) yum groupinstall -y "Development Tools" case "<< parameters.dist >>:<< parameters.release >>" in - almalinux:9) + centos:stream9|almalinux:9) dnf install -y "dnf-command(config-manager)" yum config-manager --set-enabled crb yum install -y diffutils yum install -y epel-release ;; - centos:stream|almalinux:8) + almalinux:8) dnf install -y "dnf-command(config-manager)" yum config-manager --set-enabled powertools yum install -y diffutils @@ -388,7 +388,7 @@ workflows: dist: centos release: "7" - build: - name: build_centos_stream + name: build_centos_stream_9 prefix: quay.io/centos/ dist: centos release: stream9 @@ -465,7 +465,7 @@ workflows: - debian:bullseye - debian:bookworm - centos:7 - - centos:stream + - centos:stream9 - almalinux:8 - almalinux:9 - fedora:latest From dridi.boukelmoune at gmail.com Mon Jun 3 10:56:05 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 3 Jun 2024 10:56:05 +0000 (UTC) Subject: [master] 5b515149f cache_vcl: Better dlopen error reporting Message-ID: <20240603105605.521C9A4E9B@lists.varnish-cache.org> commit 5b515149fd67cdf9d8a3d81c16ab1bf7d13d71e0 Author: Dridi Boukelmoune Date: Wed May 22 19:42:42 2024 +0200 cache_vcl: Better dlopen error reporting diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 90e41ece8..10cb70f8f 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -474,6 +474,8 @@ VCL_Open(const char *fn, struct vsb *msg) struct vcl *vcl; void *dlh; struct VCL_conf const *cnf; + const char *dlerr; + int err; AN(fn); AN(msg); @@ -485,8 +487,15 @@ VCL_Open(const char *fn, struct vsb *msg) #endif dlh = dlopen(fn, RTLD_NOW | RTLD_LOCAL); if (dlh == NULL) { + err = errno; + dlerr = dlerror(); VSB_cat(msg, "Could not load compiled VCL.\n"); - VSB_printf(msg, "\tdlopen() = %s\n", dlerror()); + if (dlerr != NULL) + VSB_printf(msg, "\tdlopen() = %s\n", dlerr); + if (err) { + VSB_printf(msg, "\terror = %s (%d)\n", + strerror(err), err); + } VSB_cat(msg, "\thint: check for \"noexec\" mount\n"); return (NULL); } From dridi.boukelmoune at gmail.com Mon Jun 3 10:56:05 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 3 Jun 2024 10:56:05 +0000 (UTC) Subject: [master] ae66f67e4 cache_vcl: Also hint at vmod_path Message-ID: <20240603105605.74D81A4E9E@lists.varnish-cache.org> commit ae66f67e4aa2e637dc4fccfe87337e986049d023 Author: Dridi Boukelmoune Date: Thu May 23 10:36:55 2024 +0200 cache_vcl: Also hint at vmod_path diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 10cb70f8f..897563b09 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -497,6 +497,7 @@ VCL_Open(const char *fn, struct vsb *msg) strerror(err), err); } VSB_cat(msg, "\thint: check for \"noexec\" mount\n"); + VSB_cat(msg, "\thint: check \"vmod_path\" parameter\n"); return (NULL); } cnf = dlsym(dlh, "VCL_conf"); From nils.goroll at uplex.de Mon Jun 3 13:15:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 3 Jun 2024 13:15:06 +0000 (UTC) Subject: [master] 7392333a9 vbt: Format back-traces from libvarnish Message-ID: <20240603131506.8AB98A9952@lists.varnish-cache.org> commit 7392333a9cf5eac64aa28c36007e3e8831afc306 Author: Dridi Boukelmoune Date: Tue Feb 14 13:51:08 2023 +0100 vbt: Format back-traces from libvarnish This is done in a VSB to match what the panic code is currently doing. When libunwind is used but fails to produce a back-trace, there is now a fallback to the original libexecinfo-based implementation (suggested by @asadsa92). diff --git a/include/Makefile.am b/include/Makefile.am index 1a68ab4a3..ad4bc2555 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -88,8 +88,8 @@ nobase_pkginclude_HEADERS += \ # Private headers nobase_noinst_HEADERS = \ compat/daemon.h \ - vfl.h \ libvcc.h \ + vbt.h \ vcc_interface.h \ vcli_serve.h \ vcs_version.h \ @@ -99,6 +99,7 @@ nobase_noinst_HEADERS = \ vend.h \ vev.h \ vfil.h \ + vfl.h \ vin.h \ vjsn.h \ vlu.h \ diff --git a/include/vbt.h b/include/vbt.h new file mode 100644 index 000000000..bd3a384b9 --- /dev/null +++ b/include/vbt.h @@ -0,0 +1,33 @@ +/*- + * Copyright (c) 2022 Varnish Software AS + * All rights reserved. + * + * Author: Dridi Boukelmoune + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +struct vsb; + +void VBT_format(struct vsb *); diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am index d06f1b5b2..221091110 100644 --- a/lib/libvarnish/Makefile.am +++ b/lib/libvarnish/Makefile.am @@ -18,6 +18,7 @@ libvarnish_la_SOURCES = \ vbh.c \ vas.c \ vav.c \ + vbt.c \ vcli_proto.c \ vcli_serve.c \ vct.c \ @@ -46,6 +47,11 @@ libvarnish_la_SOURCES = \ libvarnish_la_LIBADD = @PCRE2_LIBS@ $(LIBM) +if WITH_UNWIND +libvarnish_la_CFLAGS += -DUNW_LOCAL_ONLY $(LIBUNWIND_CFLAGS) +libvarnish_la_LIBADD += $(LIBUNWIND_LIBS) +endif + TESTS = \ vav_test \ vbh_test \ diff --git a/lib/libvarnish/vbt.c b/lib/libvarnish/vbt.c new file mode 100644 index 000000000..4a2da2c27 --- /dev/null +++ b/lib/libvarnish/vbt.c @@ -0,0 +1,143 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2022 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * Author: Guillaume Quintard + * Author: Dridi Boukelmoune + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include +#include + +#ifdef WITH_UNWIND +# include +#endif + +#include + +#include "vdef.h" +#include "vas.h" +#include "vbt.h" +#include "vsb.h" + +#ifdef WITH_UNWIND +static int +vbt_unwind(struct vsb *vsb) +{ + unw_cursor_t cursor; unw_context_t uc; + unw_word_t ip, sp; + unw_word_t offp; + char fname[1024]; + const char *sep; + int ret; + + ret = unw_getcontext(&uc); + if (ret != 0) { + VSB_printf(vsb, "Backtrace not available " + "(unw_getcontext returned %d)\n", ret); + return (-1); + } + ret = unw_init_local(&cursor, &uc); + if (ret != 0) { + VSB_printf(vsb, "Backtrace not available " + "(unw_init_local returned %d)\n", ret); + return (-1); + } + while (unw_step(&cursor) > 0) { + fname[0] = '\0'; + sep = ""; + if (!unw_get_reg(&cursor, UNW_REG_IP, &ip)) { + VSB_printf(vsb, "ip=0x%lx", (long) ip); + sep = " "; + } + if (!unw_get_reg(&cursor, UNW_REG_SP, &sp)) { + VSB_printf(vsb, "%ssp=0x%lx", sep, (long) sp); + sep = " "; + } + if (!unw_get_proc_name(&cursor, fname, sizeof(fname), &offp)) { + VSB_printf(vsb, "%s<%s+0x%lx>", + sep, fname[0] ? fname : "", (long)offp); + } + VSB_putc(vsb, '\n'); + } + + return (0); +} +#endif + +#define BACKTRACE_LEVELS 20 + +static void +vbt_execinfo(struct vsb *vsb) +{ + void *array[BACKTRACE_LEVELS]; + size_t size; + size_t i; + char **strings; + char *p; + char buf[32]; + + size = backtrace (array, BACKTRACE_LEVELS); + if (size > BACKTRACE_LEVELS) { + VSB_printf(vsb, "Backtrace not available (ret=%zu)\n", size); + return; + } + for (i = 0; i < size; i++) { + bprintf(buf, "%p", array[i]); + VSB_printf(vsb, "%s: ", buf); + strings = backtrace_symbols(&array[i], 1); + if (strings == NULL || strings[0] == NULL) { + VSB_cat(vsb, "(?)"); + } else { + p = strings[0]; + if (!memcmp(buf, p, strlen(buf))) { + p += strlen(buf); + if (*p == ':') + p++; + while (*p == ' ') + p++; + } + VSB_cat(vsb, p); + } + VSB_cat(vsb, "\n"); + free(strings); + } +} + +void +VBT_format(struct vsb *vsb) +{ + +#ifdef WITH_UNWIND + if (!vbt_unwind(vsb)) + return; + VSB_cat(vsb, "Falling back to execinfo backtrace\n"); +#endif + vbt_execinfo(vsb); +} From nils.goroll at uplex.de Mon Jun 3 13:15:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 3 Jun 2024 13:15:06 +0000 (UTC) Subject: [master] a681edf09 panic: Use VBT_printf() to print the back-trace Message-ID: <20240603131506.9F7FEA9956@lists.varnish-cache.org> commit a681edf097ac23703c65a2b95d521c0c77437d3a Author: Dridi Boukelmoune Date: Tue Feb 14 14:33:39 2023 +0100 panic: Use VBT_printf() to print the back-trace diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 7da01e308..c6aa6f564 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -31,12 +31,6 @@ #include "config.h" -#ifdef WITH_UNWIND -# include -#else -# include -#endif - #include #include #include @@ -51,6 +45,7 @@ #include "storage/storage.h" #include "vcli_serve.h" #include "vtim.h" +#include "vbt.h" #include "vcs.h" #include "vtcp.h" #include "vsa.h" @@ -604,93 +599,16 @@ pan_sess(struct vsb *vsb, const struct sess *sp) /*--------------------------------------------------------------------*/ -#ifdef WITH_UNWIND - static void pan_backtrace(struct vsb *vsb) { - unw_cursor_t cursor; unw_context_t uc; - unw_word_t ip, sp; - unw_word_t offp; - char fname[1024]; - int ret; VSB_cat(vsb, "Backtrace:\n"); VSB_indent(vsb, 2); - - ret = unw_getcontext(&uc); - if (ret != 0) { - VSB_printf(vsb, "Backtrace not available " - "(unw_getcontext returned %d)\n", ret); - return; - } - ret = unw_init_local(&cursor, &uc); - if (ret != 0) { - VSB_printf(vsb, "Backtrace not available " - "(unw_init_local returned %d)\n", ret); - return; - } - while (unw_step(&cursor) > 0) { - fname[0] = '\0'; - if (!unw_get_reg(&cursor, UNW_REG_IP, &ip)) - VSB_printf(vsb, "ip=0x%lx", (long) ip); - if (!unw_get_reg(&cursor, UNW_REG_SP, &sp)) - VSB_printf(vsb, " sp=0x%lx", (long) sp); - if (!unw_get_proc_name(&cursor, fname, sizeof(fname), &offp)) - VSB_printf(vsb, " <%s+0x%lx>", - fname[0] ? fname : "", (long)offp); - VSB_putc(vsb, '\n'); - } - + VBT_format(vsb); VSB_indent(vsb, -2); } -#else /* WITH_UNWIND */ - -#define BACKTRACE_LEVELS 20 - -static void -pan_backtrace(struct vsb *vsb) -{ - void *array[BACKTRACE_LEVELS]; - size_t size; - size_t i; - char **strings; - char *p; - char buf[32]; - - size = backtrace (array, BACKTRACE_LEVELS); - if (size > BACKTRACE_LEVELS) { - VSB_printf(vsb, "Backtrace not available (ret=%zu)\n", size); - return; - } - VSB_cat(vsb, "Backtrace:\n"); - VSB_indent(vsb, 2); - for (i = 0; i < size; i++) { - bprintf(buf, "%p", array[i]); - VSB_printf(vsb, "%s: ", buf); - strings = backtrace_symbols(&array[i], 1); - if (strings == NULL || strings[0] == NULL) { - VSB_cat(vsb, "(?)"); - } else { - p = strings[0]; - if (!memcmp(buf, p, strlen(buf))) { - p += strlen(buf); - if (*p == ':') - p++; - while (*p == ' ') - p++; - } - VSB_printf(vsb, "%s", p); - } - VSB_cat(vsb, "\n"); - free(strings); - } - VSB_indent(vsb, -2); -} - -#endif /* WITH_UNWIND */ - #ifdef HAVE_PTHREAD_GETATTR_NP static void pan_threadattr(struct vsb *vsb) From nils.goroll at uplex.de Mon Jun 3 13:20:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 3 Jun 2024 13:20:05 +0000 (UTC) Subject: [master] 355351f34 vsmw: introduce an optional prefix to segment ids in VSMW_Allocv() Message-ID: <20240603132005.A9D9BA9EF1@lists.varnish-cache.org> commit 355351f34c050487fa3eb2922fdb7f63904b2252 Author: Stephane Cance Date: Mon May 13 10:55:07 2024 +0200 vsmw: introduce an optional prefix to segment ids in VSMW_Allocv() This facility allows intermediate layers to set a prefix without meddling with the provided format string. diff --git a/bin/varnishd/common/common_vsc.c b/bin/varnishd/common/common_vsc.c index 7df98b836..98b420b70 100644 --- a/bin/varnishd/common/common_vsc.c +++ b/bin/varnishd/common/common_vsc.c @@ -91,7 +91,7 @@ vrt_vsc_mksegv(struct vsmw_cluster *vc, const char *category, ALLOC_OBJ(vsg, VSC_SEG_MAGIC); AN(vsg); vsg->seg = VSMW_Allocv(heritage.proc_vsmw, vc, category, - VRT_VSC_Overhead(payload), fmt, va); + VRT_VSC_Overhead(payload), NULL, fmt, va); AN(vsg->seg); vsg->vsm = heritage.proc_vsmw; vsg->head = (void*)vsg->seg; diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c index 275d8f05c..2ca388b06 100644 --- a/bin/varnishd/common/common_vsmw.c +++ b/bin/varnishd/common/common_vsmw.c @@ -377,7 +377,7 @@ VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vsmcp) void * VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc, - const char *category, size_t payload, + const char *category, size_t payload, const char *prefix, const char *fmt, va_list va) { struct vsmwseg *seg; @@ -391,6 +391,10 @@ VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc, seg->len = PRNDUP(payload); VSB_clear(vsmw->vsb); + if (prefix != NULL) + VSB_cat(vsmw->vsb, prefix); + if (prefix != NULL && fmt[0] != '\0') + VSB_cat(vsmw->vsb, "."); VSB_vprintf(vsmw->vsb, fmt, va); AZ(VSB_finish(vsmw->vsb)); REPLACE(seg->id, VSB_data(vsmw->vsb)); @@ -420,7 +424,7 @@ VSMW_Allocf(struct vsmw *vsmw, struct vsmw_cluster *vc, void *p; va_start(ap, fmt); - p = VSMW_Allocv(vsmw, vc, category, len, fmt, ap); + p = VSMW_Allocv(vsmw, vc, category, len, NULL, fmt, ap); va_end(ap); return (p); } diff --git a/bin/varnishd/common/vsmw.h b/bin/varnishd/common/vsmw.h index 07d46b19e..e879edaf4 100644 --- a/bin/varnishd/common/vsmw.h +++ b/bin/varnishd/common/vsmw.h @@ -40,7 +40,7 @@ struct vsmw_cluster *VSMW_NewCluster(struct vsmw *, size_t , const char *); void VSMW_DestroyCluster(struct vsmw *, struct vsmw_cluster **); void *VSMW_Allocv(struct vsmw *, struct vsmw_cluster *, - const char *, size_t, const char *, va_list); + const char *, size_t, const char *, const char *, va_list); void *VSMW_Allocf(struct vsmw *, struct vsmw_cluster *, const char *, size_t, const char *, ...); void VSMW_Free(struct vsmw *, void **); From nils.goroll at uplex.de Mon Jun 3 13:20:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 3 Jun 2024 13:20:05 +0000 (UTC) Subject: [master] 427f5f77a vsc: Avoid name interpretation as a format string in VRT_VSC_Alloc() Message-ID: <20240603132005.BF333A9EF4@lists.varnish-cache.org> commit 427f5f77a389ddd3d5b3605ec78fc0c14146ad2c Author: Stephane Cance Date: Mon May 13 14:59:39 2024 +0200 vsc: Avoid name interpretation as a format string in VRT_VSC_Alloc() diff --git a/bin/varnishd/common/common_vsc.c b/bin/varnishd/common/common_vsc.c index 98b420b70..8455e4d95 100644 --- a/bin/varnishd/common/common_vsc.c +++ b/bin/varnishd/common/common_vsc.c @@ -84,14 +84,14 @@ static const size_t vsc_overhead = PRNDUP(sizeof(struct vsc_head)); static struct vsc_seg * vrt_vsc_mksegv(struct vsmw_cluster *vc, const char *category, - size_t payload, const char *fmt, va_list va) + size_t payload, const char *prefix, const char *fmt, va_list va) { struct vsc_seg *vsg; ALLOC_OBJ(vsg, VSC_SEG_MAGIC); AN(vsg); vsg->seg = VSMW_Allocv(heritage.proc_vsmw, vc, category, - VRT_VSC_Overhead(payload), NULL, fmt, va); + VRT_VSC_Overhead(payload), prefix, fmt, va); AN(vsg->seg); vsg->vsm = heritage.proc_vsmw; vsg->head = (void*)vsg->seg; @@ -107,7 +107,7 @@ vrt_vsc_mksegf(const char *category, size_t payload, const char *fmt, ...) struct vsc_seg *vsg; va_start(ap, fmt); - vsg = vrt_vsc_mksegv(NULL, category, payload, fmt, ap); + vsg = vrt_vsc_mksegv(NULL, category, payload, NULL, fmt, ap); va_end(ap); return (vsg); } @@ -140,7 +140,6 @@ VRT_VSC_Alloc(struct vsmw_cluster *vc, struct vsc_seg **sg, const unsigned char *jp, size_t sj, const char *fmt, va_list va) { struct vsc_seg *vsg, *dvsg; - char buf[1024]; uintptr_t jjp; vsc_lock(); @@ -168,14 +167,9 @@ VRT_VSC_Alloc(struct vsmw_cluster *vc, struct vsc_seg **sg, AN(dvsg); dvsg->refs++; - if (*fmt == '\0') - bprintf(buf, "%s", nm); - else - bprintf(buf, "%s.%s", nm, fmt); - AN(heritage.proc_vsmw); - vsg = vrt_vsc_mksegv(vc, VSC_CLASS, sd, buf, va); + vsg = vrt_vsc_mksegv(vc, VSC_CLASS, sd, nm, fmt, va); AN(vsg); vsg->nm = nm; vsg->doc = dvsg; From nils.goroll at uplex.de Mon Jun 3 13:20:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 3 Jun 2024 13:20:05 +0000 (UTC) Subject: [master] 9d40af77f vsmw: make segment ids input validation stricter Message-ID: <20240603132005.D9CDDA9EF8@lists.varnish-cache.org> commit 9d40af77fb153489e0f8c95b61de08a0daf15fb7 Author: Stephane Cance Date: Tue May 14 15:41:12 2024 +0200 vsmw: make segment ids input validation stricter diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c index 2ca388b06..5ca509e34 100644 --- a/bin/varnishd/common/common_vsmw.c +++ b/bin/varnishd/common/common_vsmw.c @@ -381,6 +381,7 @@ VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc, const char *fmt, va_list va) { struct vsmwseg *seg; + ssize_t l; vsmw_do_lock(); CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC); @@ -391,13 +392,20 @@ VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc, seg->len = PRNDUP(payload); VSB_clear(vsmw->vsb); - if (prefix != NULL) + if (prefix != NULL) { + assert(prefix[0] != '\0'); VSB_cat(vsmw->vsb, prefix); - if (prefix != NULL && fmt[0] != '\0') - VSB_cat(vsmw->vsb, "."); + if (fmt[0] != '\0') + VSB_cat(vsmw->vsb, "."); + } + l = VSB_len(vsmw->vsb); + assert(l >= 0); VSB_vprintf(vsmw->vsb, fmt, va); AZ(VSB_finish(vsmw->vsb)); + assert(fmt[0] == '\0' || l < VSB_len(vsmw->vsb)); + REPLACE(seg->id, VSB_data(vsmw->vsb)); + AN(seg->id); if (vc == NULL) vc = vsmw_newcluster(vsmw, seg->len, category); From dridi.boukelmoune at gmail.com Mon Jun 3 13:55:06 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 3 Jun 2024 13:55:06 +0000 (UTC) Subject: [master] 7875dcbe4 build: Always look for libexecinfo Message-ID: <20240603135506.58FA4AB394@lists.varnish-cache.org> commit 7875dcbe462bb62c6e3d6ac247ffa11e4af1c3bd Author: Dridi Boukelmoune Date: Mon Jun 3 15:51:46 2024 +0200 build: Always look for libexecinfo But it is not required when building with libunwind support. diff --git a/configure.ac b/configure.ac index 21845879d..25959ebaf 100644 --- a/configure.ac +++ b/configure.ac @@ -329,6 +329,8 @@ AC_SUBST(JEMALLOC_LDADD) AC_CHECK_FUNCS([setproctitle]) +AC_SEARCH_LIBS(backtrace, [execinfo], [AC_CHECK_HEADERS([[execinfo.h]])]) + # if the default libexecinfo on alpine causes issues, you can use libunwind AC_ARG_WITH([unwind], [AS_HELP_STRING([--with-unwind], @@ -354,10 +356,10 @@ fi if test "$have_unwind" = yes; then AC_DEFINE([WITH_UNWIND], [1], [Define to 1 to use libunwind instead of libexecinfo]) -else - AC_SEARCH_LIBS(backtrace, [execinfo], [], [ - AC_MSG_ERROR([Could not find backtrace() support]) - ]) +elif test "$ac_cv_search_backtrace" = no; then + AC_MSG_ERROR([Could not find backtrace() support]) +elif test "$ac_cv_header_execinfo_h" = no; then + AC_MSG_ERROR([Could not find execinfo.h]) fi AM_CONDITIONAL([WITH_UNWIND], [test "$have_unwind" = yes]) From dridi.boukelmoune at gmail.com Mon Jun 3 13:55:06 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 3 Jun 2024 13:55:06 +0000 (UTC) Subject: [master] 41a1a42dd vbt: Do not assume libexecinfo availability Message-ID: <20240603135506.6C380AB396@lists.varnish-cache.org> commit 41a1a42dd9e4390d400fcc205bf1d3eac7970191 Author: Dridi Boukelmoune Date: Mon Jun 3 15:52:49 2024 +0200 vbt: Do not assume libexecinfo availability This is not the case for example on Alpine Linux. diff --git a/lib/libvarnish/vbt.c b/lib/libvarnish/vbt.c index 4a2da2c27..f68906e51 100644 --- a/lib/libvarnish/vbt.c +++ b/lib/libvarnish/vbt.c @@ -39,7 +39,9 @@ # include #endif -#include +#ifdef HAVE_EXECINFO_H +# include +#endif #include "vdef.h" #include "vas.h" @@ -91,7 +93,8 @@ vbt_unwind(struct vsb *vsb) } #endif -#define BACKTRACE_LEVELS 20 +#ifdef HAVE_EXECINFO_H +# define BACKTRACE_LEVELS 20 static void vbt_execinfo(struct vsb *vsb) @@ -129,6 +132,7 @@ vbt_execinfo(struct vsb *vsb) free(strings); } } +#endif void VBT_format(struct vsb *vsb) @@ -137,7 +141,12 @@ VBT_format(struct vsb *vsb) #ifdef WITH_UNWIND if (!vbt_unwind(vsb)) return; +# ifdef HAVE_EXECINFO_H VSB_cat(vsb, "Falling back to execinfo backtrace\n"); +# endif #endif + +#ifdef HAVE_EXECINFO_H vbt_execinfo(vsb); +#endif } From nils.goroll at uplex.de Mon Jun 3 15:05:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 3 Jun 2024 15:05:06 +0000 (UTC) Subject: [master] 0dfa3b8c9 vxp: Use strtod/strtoll consistently for number parsing for vsl (queries) Message-ID: <20240603150506.0ED1DAD9BF@lists.varnish-cache.org> commit 0dfa3b8c95f67054989a0e9259069e2f2433497a Author: Nils Goroll Date: Mon Jun 3 14:24:52 2024 +0200 vxp: Use strtod/strtoll consistently for number parsing for vsl (queries) Fixes #4088 diff --git a/bin/varnishtest/tests/r03463.vtc b/bin/varnishtest/tests/r03463.vtc index 3d14ea9d0..efaaa644d 100644 --- a/bin/varnishtest/tests/r03463.vtc +++ b/bin/varnishtest/tests/r03463.vtc @@ -1,11 +1,11 @@ -varnishtest "VSL query lenient int comparisons" +varnishtest "VSL query lenient int #3463 and float precision comparisons #4088" varnish v1 -vcl { import std; backend be none; sub vcl_recv { if (req.http.skip != "log") { - std.log("float1: 123.456"); + std.log("float1: 123.4567"); std.log("float2: 123."); std.log("float3: .456"); std.log("float4: 123"); @@ -37,6 +37,12 @@ logexpect l5 -v v1 -q "VCL_Log:float5 != 42 or ReqHeader:skip eq log" { fail clear } -start +#4088 +logexpect l6 -v v1 -q "VCL_Log:float1 > 123.456" { + expect 0 1001 Begin rxreq +} -start + + client c1 { txreq rxresp @@ -50,3 +56,4 @@ logexpect l2 -wait logexpect l3 -wait logexpect l4 -wait logexpect l5 -wait +logexpect l6 -wait diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c index adcb77957..61f9f325c 100644 --- a/lib/libvarnishapi/vsl_query.c +++ b/lib/libvarnishapi/vsl_query.c @@ -122,8 +122,9 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec) const struct vex_rhs *rhs; long long lhs_int = 0; double lhs_float = 0.; - const char *b, *e, *q; + const char *b, *e; int i, dq; + char *q = NULL; AN(vex); AN(rec); @@ -205,20 +206,24 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec) if (*b == '\0') /* Empty string doesn't match */ return (0); + errno = 0; switch (rhs->type) { case VEX_INT: - lhs_int = (long long)SF_Parse_Number(&b, 0, &q); - if (errno) + lhs_int = strtoll(b, &q, 0); + AN(q); + if (q != e && *q != '.') return (0); break; case VEX_FLOAT: - lhs_float = SF_Parse_Decimal(&b, 0, &q); - if (errno) + lhs_float = strtod(b, &q); + if (q != e) return (0); break; default: WRONG("Wrong RHS type"); } + if (errno != 0) + return (0); break; default: break; diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c index 94896837a..bfd162fa8 100644 --- a/lib/libvarnishapi/vxp_parse.c +++ b/lib/libvarnishapi/vxp_parse.c @@ -208,24 +208,21 @@ vxp_expr_num(struct vxp *vxp, struct vex_rhs **prhs, unsigned vxid) AN(vxp->t->dec); ALLOC_OBJ(*prhs, VEX_RHS_MAGIC); AN(*prhs); + endptr = NULL; if (strchr(vxp->t->dec, '.')) { (*prhs)->type = VEX_FLOAT; - (*prhs)->val_float = VNUM(vxp->t->dec); - if (isnan((*prhs)->val_float)) { - VSB_cat(vxp->sb, "Floating point parse error "); - vxp_ErrWhere(vxp, vxp->t, -1); - return; - } + (*prhs)->val_float = strtod(vxp->t->dec, &endptr); } else { (*prhs)->type = VEX_INT; (*prhs)->val_int = strtoll(vxp->t->dec, &endptr, 0); - while (isspace(*endptr)) - endptr++; - if (*endptr != '\0') { - VSB_cat(vxp->sb, "Integer parse error "); - vxp_ErrWhere(vxp, vxp->t, -1); - return; - } + } + while (isspace(*endptr)) + endptr++; + if (*endptr != '\0') { + VSB_printf(vxp->sb, "%s parse error ", + (*prhs)->type == VEX_FLOAT ? "Floating point" : "Integer"); + vxp_ErrWhere(vxp, vxp->t, -1); + return; } if (vxid && (*prhs)->type != VEX_INT) { VSB_printf(vxp->sb, "Expected integer got '%.*s' ", From dridi.boukelmoune at gmail.com Mon Jun 3 16:05:06 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 3 Jun 2024 16:05:06 +0000 (UTC) Subject: [master] 218f3aa60 vsmw: remove spurious assert after REPLACE() Message-ID: <20240603160506.3C676AF9E1@lists.varnish-cache.org> commit 218f3aa606d206913463958b0a173dc924f351ed Author: Stephane Cance Date: Mon Jun 3 17:43:40 2024 +0200 vsmw: remove spurious assert after REPLACE() diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c index 5ca509e34..1e0710996 100644 --- a/bin/varnishd/common/common_vsmw.c +++ b/bin/varnishd/common/common_vsmw.c @@ -405,7 +405,6 @@ VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc, assert(fmt[0] == '\0' || l < VSB_len(vsmw->vsb)); REPLACE(seg->id, VSB_data(vsmw->vsb)); - AN(seg->id); if (vc == NULL) vc = vsmw_newcluster(vsmw, seg->len, category); From dridi at varni.sh Wed Jun 5 07:52:03 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 5 Jun 2024 07:52:03 +0000 Subject: [master] 0dfa3b8c9 vxp: Use strtod/strtoll consistently for number parsing for vsl (queries) In-Reply-To: <20240603150506.0ED1DAD9BF@lists.varnish-cache.org> References: <20240603150506.0ED1DAD9BF@lists.varnish-cache.org> Message-ID: On Mon, Jun 3, 2024 at 3:05?PM Nils Goroll wrote: > > > commit 0dfa3b8c95f67054989a0e9259069e2f2433497a > Author: Nils Goroll > Date: Mon Jun 3 14:24:52 2024 +0200 > > vxp: Use strtod/strtoll consistently for number parsing for vsl (queries) > > Fixes #4088 It was brought to my attention that this commit breaks FreeBSD builds, failing r03463.vtc systematically. I have not figured out why and I don't have more time to spend on this. Dridi From nils.goroll at uplex.de Wed Jun 5 08:09:08 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 5 Jun 2024 10:09:08 +0200 Subject: [master] 0dfa3b8c9 vxp: Use strtod/strtoll consistently for number parsing for vsl (queries) In-Reply-To: References: <20240603150506.0ED1DAD9BF@lists.varnish-cache.org> Message-ID: <1cca8903-360d-4224-a012-6fec6357e6ca@uplex.de> On 05.06.24 09:52, Dridi Boukelmoune wrote: > It was brought to my attention that this commit breaks FreeBSD builds thank you, I am taking care of this From nils.goroll at uplex.de Wed Jun 5 08:26:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 5 Jun 2024 08:26:06 +0000 (UTC) Subject: [master] 0ca8d1771 vxp: bring back fallback to trunc(strtod()) for VEX_INT Message-ID: <20240605082606.3209CB07C3@lists.varnish-cache.org> commit 0ca8d177178b94622a3a8a939f383d6abc1952c7 Author: Nils Goroll Date: Wed Jun 5 10:13:28 2024 +0200 vxp: bring back fallback to trunc(strtod()) for VEX_INT It seems I wrongly assumed strtoll(".", ...) would return 0, so bring back the fallack to float parsing similar to what we had before 05e10eee3076e4c416ebd460cfa70f4fa8fcdb67. Should fix regression on FreeBSD from 0dfa3b8c95f67054989a0e9259069e2f2433497a Also clean up now unneeded include (thank you, Flexelint). Fixes #4088 I promise, if this is still wrong, I will install FreeBSD ;) diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c index 61f9f325c..d6496043e 100644 --- a/lib/libvarnishapi/vsl_query.c +++ b/lib/libvarnishapi/vsl_query.c @@ -41,7 +41,6 @@ #include "miniobj.h" #include "vbm.h" -#include "vnum.h" #include "vqueue.h" #include "vre.h" #include "vsb.h" @@ -211,18 +210,19 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec) case VEX_INT: lhs_int = strtoll(b, &q, 0); AN(q); - if (q != e && *q != '.') - return (0); + if (q != e && (*q == '.' || *q == 'e')) { + lhs_float = strtod(b, &q); + lhs_int = trunc(lhs_float); + lhs_float = 0.; + } break; case VEX_FLOAT: lhs_float = strtod(b, &q); - if (q != e) - return (0); break; default: WRONG("Wrong RHS type"); } - if (errno != 0) + if (q != e || errno != 0) return (0); break; default: From nils.goroll at uplex.de Wed Jun 5 08:26:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 5 Jun 2024 08:26:06 +0000 (UTC) Subject: [master] b7452d038 Move flexelint silencing to the right place Message-ID: <20240605082606.458F1B07C6@lists.varnish-cache.org> commit b7452d0380873a3a4b562726b756b7d6e42a1666 Author: Nils Goroll Date: Wed Jun 5 10:25:02 2024 +0200 Move flexelint silencing to the right place Ref 7392333a9cf5eac64aa28c36007e3e8831afc306 diff --git a/bin/flint.lnt b/bin/flint.lnt index 3bb395da5..e6cee798a 100644 --- a/bin/flint.lnt +++ b/bin/flint.lnt @@ -39,9 +39,6 @@ -elib(659) // no tokens after struct def. -elib(123) // macro def. with arg at, (just warn) -+libh(/usr/include/libunwind.h) --elib(849) - -emacro(702, WEXITSTATUS) // signed shift right -e825 // control flows into case/default without -fallthrough comment diff --git a/lib/libvarnish/flint.lnt b/lib/libvarnish/flint.lnt index 4c41e5fee..83e75c3cc 100644 --- a/lib/libvarnish/flint.lnt +++ b/lib/libvarnish/flint.lnt @@ -14,3 +14,6 @@ --emacro((835),VBH_NOIDX) --emacro((835),O_CLOEXEC) + ++libh(/usr/include/libunwind.h) +-elib(849) From nils.goroll at uplex.de Wed Jun 5 08:40:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 5 Jun 2024 08:40:05 +0000 (UTC) Subject: [master] 4e11608ba More flexelinting Message-ID: <20240605084005.2E1BEB11CA@lists.varnish-cache.org> commit 4e11608baffa34af86f98de529cacbf7499623c9 Author: Nils Goroll Date: Wed Jun 5 10:39:21 2024 +0200 More flexelinting include cleanup and same change as cccb8c1d44daff940e89649be5f69d7b8797ad85 diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c index d6496043e..73824345b 100644 --- a/lib/libvarnishapi/vsl_query.c +++ b/lib/libvarnishapi/vsl_query.c @@ -212,7 +212,7 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec) AN(q); if (q != e && (*q == '.' || *q == 'e')) { lhs_float = strtod(b, &q); - lhs_int = trunc(lhs_float); + lhs_int = (long long)lhs_float; lhs_float = 0.; } break; diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c index bfd162fa8..7297d21a1 100644 --- a/lib/libvarnishapi/vxp_parse.c +++ b/lib/libvarnishapi/vxp_parse.c @@ -41,7 +41,6 @@ #include "miniobj.h" #include "vbm.h" -#include "vnum.h" #include "vqueue.h" #include "vre.h" #include "vsb.h" From nils.goroll at uplex.de Wed Jun 5 09:22:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 5 Jun 2024 09:22:05 +0000 (UTC) Subject: [master] 8b4c7a1eb doc: freebsd install update Message-ID: <20240605092205.5092FB27DF@lists.varnish-cache.org> commit 8b4c7a1eb44bbc3259bf7b95f19f0e3d6b1aefcb Author: Nils Goroll Date: Wed Jun 5 11:16:49 2024 +0200 doc: freebsd install update as of 14.1-RELEASE diff --git a/doc/sphinx/installation/install_source.rst b/doc/sphinx/installation/install_source.rst index 0415513b2..83c65c321 100644 --- a/doc/sphinx/installation/install_source.rst +++ b/doc/sphinx/installation/install_source.rst @@ -29,7 +29,7 @@ Build dependencies on FreeBSD To get the dependencies required to build varnish from source you can either:: - pkg install automake pkgconf py36-sphinx py36-docutils pcre2 libtool + pkg install git automake pkgconf py39-sphinx py39-docutils pcre2 libtool .. XXX does cpio need to be installed on FreeBSD? From nils.goroll at uplex.de Wed Jun 5 09:22:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 5 Jun 2024 09:22:05 +0000 (UTC) Subject: [master] d3db14608 vxp: fix vslq_test_rec() for FreeBSD Message-ID: <20240605092205.63D67B27E4@lists.varnish-cache.org> commit d3db146082d722d06efe74b681c161f2fdc5903d Author: Nils Goroll Date: Wed Jun 5 11:17:38 2024 +0200 vxp: fix vslq_test_rec() for FreeBSD the actual issue was that strtoll(".456", ...) sets errno 22, so when we fall back to strtod(), we need to clear it. Ref 0ca8d177178b94622a3a8a939f383d6abc1952c7 Ref 0dfa3b8c95f67054989a0e9259069e2f2433497a Fixes #4088 Yes, I did install FreeBSD, and it's been a nice refreshing no-frills experience so far. diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c index 73824345b..48874e2d6 100644 --- a/lib/libvarnishapi/vsl_query.c +++ b/lib/libvarnishapi/vsl_query.c @@ -211,6 +211,7 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec) lhs_int = strtoll(b, &q, 0); AN(q); if (q != e && (*q == '.' || *q == 'e')) { + errno = 0; lhs_float = strtod(b, &q); lhs_int = (long long)lhs_float; lhs_float = 0.; From dridi.boukelmoune at gmail.com Wed Jun 5 13:53:04 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 5 Jun 2024 13:53:04 +0000 (UTC) Subject: [master] e444efae5 Partially revert "tree-wide: add missing magic checks" Message-ID: <20240605135304.9448962E61@lists.varnish-cache.org> commit e444efae595634d48d257c963fdf1eb63c53ccc5 Author: Dridi Boukelmoune Date: Wed Jun 5 15:40:15 2024 +0200 Partially revert "tree-wide: add missing magic checks" This reverts commit 328540c6ca58b8009d739ea542442a9d24372ce2 in the lib/libvarnishapi/vsc.c file where no magic check could be added to replace AN(vsm) occurrences. We should probably look for more potential net loss of assertions from that commit. diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index fc1f06467..7bda424ce 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -319,6 +319,7 @@ vsc_del_seg(const struct vsc *vsc, struct vsm *vsm, struct vsc_seg **spp) struct vsc_seg *sp; CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC); + AN(vsm); TAKE_OBJ_NOTNULL(sp, spp, VSC_SEG_MAGIC); AZ(VSM_Unmap(vsm, sp->fantom)); if (sp->vj != NULL) { @@ -342,6 +343,7 @@ vsc_add_seg(const struct vsc *vsc, struct vsm *vsm, const struct vsm_fantom *fp) struct vsc_pt *pp; CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC); + AN(vsm); ALLOC_OBJ(sp, VSC_SEG_MAGIC); AN(sp); From dridi.boukelmoune at gmail.com Thu Jun 6 09:34:04 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 6 Jun 2024 09:34:04 +0000 (UTC) Subject: [master] bc89e8a4e circleci: Run sanitizers on a larger system Message-ID: <20240606093404.794BE61DDC@lists.varnish-cache.org> commit bc89e8a4ea6a690c2ae3078fcbb158cfcd71aae1 Author: Dridi Boukelmoune Date: Thu Jun 6 11:10:13 2024 +0200 circleci: Run sanitizers on a larger system This is an attempt at speeding up this job that usually takes three time as long as other distcheck jobs. diff --git a/.circleci/config.yml b/.circleci/config.yml index 409da8da8..54f1ad5f4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -127,7 +127,7 @@ jobs: description: the Linux distribution, with release, e.g. debian:buster, centos:7 type: string rclass: - description: the resource class to use, usuall arm.medium or medium + description: the resource class to use, usually arm.medium or medium type: string machine: image: ubuntu-2004:202111-02 @@ -220,8 +220,13 @@ jobs: description: platform-specific configure arguments default: "" type: string + rclass: + description: job-specific resource class + default: medium + type: string docker: - image: centos:7 + resource_class: << parameters.rclass >> working_directory: /workspace steps: - setup_remote_docker @@ -433,8 +438,9 @@ workflows: name: build_ubuntu_noble dist: ubuntu release: noble + rclass: large extra_conf: --enable-asan --enable-ubsan --enable-workspace-emulator - make_target: check -j2 + make_target: check - build: name: build_alpine dist: alpine From dridi.boukelmoune at gmail.com Mon Jun 10 12:15:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 10 Jun 2024 12:15:07 +0000 (UTC) Subject: [master] dd8606491 vbt: Check vsb before using it Message-ID: <20240610121507.72EF8449A@lists.varnish-cache.org> commit dd860649179011cd9ac2e48c7af4e1e9385a15ad Author: Dridi Boukelmoune Date: Thu Jun 6 13:42:20 2024 +0200 vbt: Check vsb before using it diff --git a/lib/libvarnish/vbt.c b/lib/libvarnish/vbt.c index f68906e51..c38227a88 100644 --- a/lib/libvarnish/vbt.c +++ b/lib/libvarnish/vbt.c @@ -27,6 +27,9 @@ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. + * + * We tend to print back-traces when there is a fatal error, so the VBT code + * should avoid assertions. */ #include "config.h" @@ -48,6 +51,8 @@ #include "vbt.h" #include "vsb.h" +#include "miniobj.h" + #ifdef WITH_UNWIND static int vbt_unwind(struct vsb *vsb) @@ -138,6 +143,8 @@ void VBT_format(struct vsb *vsb) { + if (!VALID_OBJ(vsb, VSB_MAGIC)) + return; #ifdef WITH_UNWIND if (!vbt_unwind(vsb)) return; From dridi.boukelmoune at gmail.com Mon Jun 10 13:42:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 10 Jun 2024 13:42:07 +0000 (UTC) Subject: [master] 0c45870ee Revert "Retire the kill(SIGNULL) test in VSM" Message-ID: <20240610134207.1EC1576BA@lists.varnish-cache.org> commit 0c45870ee7e476a0a9786f849734989e97d13d7d Author: Dridi Boukelmoune Date: Sat May 25 07:21:46 2024 +0200 Revert "Retire the kill(SIGNULL) test in VSM" This reverts commit b845c2789cae4f165e30a3d0f3af1bfbbeca98ee. The liveness check is brought back with its deficiencies that will be solved separately. diff --git a/bin/varnishtest/tests/u00008.vtc b/bin/varnishtest/tests/u00008.vtc index 59eafd517..61de29818 100644 --- a/bin/varnishtest/tests/u00008.vtc +++ b/bin/varnishtest/tests/u00008.vtc @@ -39,6 +39,10 @@ process p1 -expect-text 0 0 "VBE.vcl1.s1.req" process p1 -expect-text 0 0 "DIAG" process p1 -screen_dump +varnish v1 -stop +process p1 -expect-text 2 1 "Uptime child: Not Running" +process p1 -screen_dump + process p1 -write {dek} process p1 -expect-text 0 1 "Concurrent connections used:" process p1 -screen_dump diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c index ad5438428..5226bf828 100644 --- a/lib/libvarnishapi/vsm.c +++ b/lib/libvarnishapi/vsm.c @@ -151,6 +151,8 @@ struct vsm { int attached; double patience; + + int couldkill; }; /*--------------------------------------------------------------------*/ @@ -362,6 +364,8 @@ VSM_New(void) vd->child->vsm = vd; vd->wdfd = -1; vd->patience = 5; + if (getenv("VSM_NOPID") != NULL) + vd->couldkill = -1; return (vd); } @@ -488,13 +492,17 @@ vsm_vlu_hash(struct vsm *vd, struct vsm_set *vs, const char *line) int i; uintmax_t id1, id2; - (void)vd; - i = sscanf(line, "# %ju %ju", &id1, &id2); if (i != 2) { vs->retval |= VSM_MGT_RESTARTED | VSM_MGT_CHANGED; return (0); } + if (vd->couldkill >= 0 && !kill(id1, 0)) { + vd->couldkill = 1; + } else if (vd->couldkill > 0 && errno == ESRCH) { + vs->retval |= VSM_MGT_RESTARTED | VSM_MGT_CHANGED; + return (0); + } vs->retval |= VSM_MGT_RUNNING; if (id1 != vs->id1 || id2 != vs->id2) { vs->retval |= VSM_MGT_RESTARTED | VSM_MGT_CHANGED; @@ -693,7 +701,8 @@ vsm_refresh_set(struct vsm *vd, struct vsm_set *vs) vs->fst.st_size = lseek(vs->fd, 0L, SEEK_CUR); - vs->retval |= vs->flag_running; + if (vd->couldkill < 1 || !kill(vs->id1, 0)) + vs->retval |= vs->flag_running; return (vs->retval); } From dridi.boukelmoune at gmail.com Mon Jun 10 13:42:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 10 Jun 2024 13:42:07 +0000 (UTC) Subject: [master] b63b5d2ce Revert "Retire also the reference to VSM_NOPID in docs" Message-ID: <20240610134207.28CE476BD@lists.varnish-cache.org> commit b63b5d2ce94c31d2a0ddb350ffdca4adcc5542a0 Author: Dridi Boukelmoune Date: Sat May 25 08:01:54 2024 +0200 Revert "Retire also the reference to VSM_NOPID in docs" This reverts commit a3593245f5a9f149da4890dbc678dcf9142929dd. diff --git a/doc/sphinx/reference/vsm.rst b/doc/sphinx/reference/vsm.rst index ad2da1740..9d09ce77a 100644 --- a/doc/sphinx/reference/vsm.rst +++ b/doc/sphinx/reference/vsm.rst @@ -102,3 +102,22 @@ a chance to discover the deallocation. The include file provides the supported API for accessing VSM files. + +VSM and Containers +------------------ + +The varnish way works great with single purpose containers. By sharing +the varnish working directory read-only, vsm readers can be run in +containers separate from those running varnishd instances on the same +host. + +When running varnishd and vsm readers in the same process namespace, +pid information can be used by vsm readers to determine if varnishd +processes are alive. + +But, when running varnishd and vsm readers in different containers, +the pid information has no relevance and may even be ambiguous across +name spaces. + +Thus, with such setups, the environment variable VSM_NOPID needs to be +set for vsm readers to disable use of pid information. From dridi.boukelmoune at gmail.com Mon Jun 10 13:42:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 10 Jun 2024 13:42:07 +0000 (UTC) Subject: [master] 1f245940a vtc_vsm: New facility to test the VSM API Message-ID: <20240610134207.5C26B76C3@lists.varnish-cache.org> commit 1f245940acb9aa698a55a0717e24675d6f8775a8 Author: Dridi Boukelmoune Date: Tue Jul 4 14:59:52 2023 +0200 vtc_vsm: New facility to test the VSM API diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am index 78d448a44..653d0e2bc 100644 --- a/bin/varnishtest/Makefile.am +++ b/bin/varnishtest/Makefile.am @@ -49,7 +49,8 @@ varnishtest_SOURCES = \ vtc_subr.c \ vtc_syslog.c \ vtc_tunnel.c \ - vtc_varnish.c + vtc_varnish.c \ + vtc_vsm.c varnishtest_LDADD = \ $(top_builddir)/lib/libvarnishapi/libvarnishapi.la \ @@ -60,6 +61,7 @@ varnishtest_LDADD = \ varnishtest_CFLAGS = \ -DVTEST_WITH_VTC_LOGEXPECT \ -DVTEST_WITH_VTC_VARNISH \ + -DVTEST_WITH_VTC_VSM \ -DTOP_BUILDDIR='"${top_builddir}"' EXTRA_DIST = $(top_srcdir)/bin/varnishtest/tests/*.vtc \ diff --git a/bin/varnishtest/cmds.h b/bin/varnishtest/cmds.h index 3b8a9e9fd..519ca8982 100644 --- a/bin/varnishtest/cmds.h +++ b/bin/varnishtest/cmds.h @@ -59,6 +59,9 @@ CMD_TOP(tunnel) CMD_TOP(varnish) #endif CMD_TOP(varnishtest) +#ifdef VTEST_WITH_VTC_VSM +CMD_TOP(vsm) +#endif CMD_TOP(vtest) #undef CMD_TOP diff --git a/bin/varnishtest/tests/a00025.vtc b/bin/varnishtest/tests/a00025.vtc new file mode 100644 index 000000000..5f45be349 --- /dev/null +++ b/bin/varnishtest/tests/a00025.vtc @@ -0,0 +1,30 @@ +varnishtest "Basic VSM status check" + +# just launch varnishd +varnish v1 -cliok ping +# POLA? vsm m1 -expect-status mgt-running +vsm m1 -expect-status mgt-running,wrk-restarted + +varnish v1 -vcl "backend be none;" +# POLA? vsm m1 -expect-status mgt-running +vsm m1 -expect-status mgt-running,wrk-restarted + +varnish v1 -start +# POLA? vsm m1 -expect-status mgt-running,mgt-changed,wrk-running +vsm m1 -expect-status mgt-running,mgt-changed,mgt-restarted,wrk-running + +varnish v1 -vcl "backend be none;" +# POLA? vsm m1 -expect-status mgt-running,wrk-running,wrk-changed +vsm m1 -expect-status mgt-running,wrk-running + +# The order of status flags does not matter, the next one is equivalent. +varnish v1 -vcl "backend be none;" +vsm m1 -expect-status wrk-running,mgt-running + +varnish v1 -stop +# POLA? vsm m1 -expect-status mgt-running,mgt-changed +vsm m1 -expect-status mgt-running + +varnish v1 -start +# POLA? vsm m1 -expect-status mgt-running,mgt-changed,wrk-running,wrk-restarted +vsm m1 -expect-status mgt-running,mgt-changed,mgt-restarted,wrk-running diff --git a/bin/varnishtest/vtc_vsm.c b/bin/varnishtest/vtc_vsm.c new file mode 100644 index 000000000..3a9c2cfac --- /dev/null +++ b/bin/varnishtest/vtc_vsm.c @@ -0,0 +1,296 @@ +/*- + * Copyright (c) 2023 Varnish Software AS + * All rights reserved. + * + * Author: Dridi Boukelmoune + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifdef VTEST_WITH_VTC_VSM + +#include "config.h" + +#include + +#include +#include +#include + +#include "vapi/vsm.h" + +#include "vtc.h" +#include "vav.h" + +struct vtc_vsm { + unsigned magic; +#define VTC_VSM_MAGIC 0x5ca77a36 + VTAILQ_ENTRY(vtc_vsm) list; + + char *name; + char *n_arg; + struct vtclog *vl; + struct vsm *vsm; +}; + +static VTAILQ_HEAD(, vtc_vsm) vsms = VTAILQ_HEAD_INITIALIZER(vsms); + +static struct vtc_vsm * +vsm_new(const char *name) +{ + struct vtc_vsm *m; + + ALLOC_OBJ(m, VTC_VSM_MAGIC); + AN(m); + REPLACE(m->name, name); + REPLACE(m->n_arg, "${v1_name}"); + m->vl = vtc_logopen("%s", name); + + VTAILQ_INSERT_TAIL(&vsms, m, list); + return (m); +} + +static void +vsm_detach(struct vtc_vsm *m) +{ + + CHECK_OBJ_NOTNULL(m, VTC_VSM_MAGIC); + if (m->vsm == NULL) + vtc_fatal(m->vl, "Cannot detach unattached VSM"); + vtc_log(m->vl, 3, "Detaching from VSM"); + VSM_Destroy(&m->vsm); + AZ(m->vsm); +} + +static void +vsm_attach(struct vtc_vsm *m) +{ + struct vsb *n_arg; + + CHECK_OBJ_NOTNULL(m, VTC_VSM_MAGIC); + if (m->vsm != NULL) + vsm_detach(m); + + n_arg = macro_expandf(m->vl, "%s", m->n_arg); + if (n_arg == NULL) + vtc_fatal(m->vl, "Could not expand -n argument"); + vtc_log(m->vl, 3, "Attaching to VSM: %s", VSB_data(n_arg)); + + m->vsm = VSM_New(); + AN(m->vsm); + + if (VSM_Arg(m->vsm, 'n', VSB_data(n_arg)) <= 0) + vtc_fatal(m->vl, "-n argument error: %s", VSM_Error(m->vsm)); + if (VSM_Attach(m->vsm, -1)) + vtc_fatal(m->vl, "VSM_Attach: %s", VSM_Error(m->vsm)); + + VSB_destroy(&n_arg); +} + +static void +vsm_delete(struct vtc_vsm *m) +{ + + CHECK_OBJ_NOTNULL(m, VTC_VSM_MAGIC); + if (m->vsm != NULL) + vsm_detach(m); + REPLACE(m->name, NULL); + REPLACE(m->n_arg, NULL); + vtc_logclose(m->vl); + FREE_OBJ(m); +} + +#define STATUS_BITS() \ + STATUS_BIT(VSM_MGT_RUNNING, mgt-running); \ + STATUS_BIT(VSM_MGT_CHANGED, mgt-changed); \ + STATUS_BIT(VSM_MGT_RESTARTED, mgt-restarted); \ + STATUS_BIT(VSM_WRK_RUNNING, wrk-running); \ + STATUS_BIT(VSM_WRK_CHANGED, wrk-changed); \ + STATUS_BIT(VSM_WRK_RESTARTED, wrk-restarted); + +static void +vsm_expect_status(struct vtc_vsm *m, const char *exp) +{ + struct vsb *stat; + const char *sep; + char **av; + unsigned bstat, bexp, bfound; + int argc, i; + + CHECK_OBJ_NOTNULL(m, VTC_VSM_MAGIC); + if (exp == NULL) + vtc_fatal(m->vl, "Missing expected status"); + + if (m->vsm == NULL) + vsm_attach(m); + + av = VAV_Parse(exp, &argc, ARGV_COMMA|ARGV_NOESC); + AN(av); + + bexp = 0; + for (i = 1; i < argc; i++) { +#define STATUS_BIT(b, s) \ + if (!strcasecmp(#s, av[i])) { \ + bexp |= b; \ + continue; \ + } + STATUS_BITS() +#undef STATUS_BIT + vtc_fatal(m->vl, "Unknown status bit: %s", av[i]); + } + VAV_Free(av); + + bfound = 0; + bstat = VSM_Status(m->vsm); + stat = VSB_new_auto(); + AN(stat); + sep = ""; +#define STATUS_BIT(b, s) \ + if (bstat & b) { \ + VSB_cat(stat, sep); \ + VSB_cat(stat, #s); \ + bfound |= b; \ + sep = ","; \ + } + STATUS_BITS(); +#undef STATUS_BIT + + if (bstat != bfound) { + vtc_fatal(m->vl, "VSM status bits not handled: %x", + bstat & ~bfound); + } + + if (bstat != bexp) { + AZ(VSB_finish(stat)); + vtc_fatal(m->vl, "Expected VSM status '%s' got '%s'", + exp, VSB_data(stat)); + } + + VSB_destroy(&stat); + vtc_log(m->vl, 4, "Found expected VSM status"); +} + +/* SECTION: vsm vsm + * + * Interact with the shared memory of a varnish instance. + * + * To define a VSM consumer, use this syntax:: + * + * vsm mNAME [-n STRING] + * + * Arguments: + * + * mNAME + * Identify the VSM consumer, it must starts with 'm'. + * + * \-n STRING + * Choose the working directory of the varnish instance. By default + * a VSM consumer connects to ``${v1_name}``. + * + * \-attach + * Attach to a new varnish instance. Implicitly detach from the + * current varnish instance if applicable. + * + * \-detach + * Detach from the current varnish instance. + * + * \-expect-status STRING + * Check that the status of VSM matches the list of status flags from + * STRING. The expected status is represented as a comma-separated + * list of flags. The list of flags in STRING is not sensitive to the + * order of flags. + * + * The available flags are: + * + * - ``mgt-running`` + * - ``mgt-changed`` + * - ``mgt-restarted`` + * - ``wrk-running`` + * - ``wrk-changed`` + * - ``wrk-restarted`` + * + * Expecting a status automatically attaches to the varnish instance + * if that was not already the case. + */ + +void +cmd_vsm(CMD_ARGS) +{ + struct vtc_vsm *m, *m2; + + (void)priv; + + if (av == NULL) { + /* Reset and free */ + VTAILQ_FOREACH_SAFE(m, &vsms, list, m2) { + CHECK_OBJ_NOTNULL(m, VTC_VSM_MAGIC); + VTAILQ_REMOVE(&vsms, m, list); + vsm_delete(m); + } + return; + } + + AZ(strcmp(av[0], "vsm")); + av++; + + VTC_CHECK_NAME(vl, av[0], "VSM", 'm'); + VTAILQ_FOREACH(m, &vsms, list) { + if (!strcmp(m->name, av[0])) + break; + } + + if (m == NULL) { + m = vsm_new(*av); + AN(m); + } + av++; + + for (; *av != NULL; av++) { + if (vtc_error) + break; + if (!strcmp(*av, "-attach")) { + vsm_attach(m); + continue; + } + if (!strcmp(*av, "-detach")) { + vsm_detach(m); + continue; + } + if (!strcmp(*av, "-expect-status")) { + vsm_expect_status(m, av[1]); + av++; + continue; + } + if (!strcmp(*av, "-n")) { + if (av[1] == NULL) + vtc_fatal(m->vl, "Missing -n argument"); + REPLACE(m->n_arg, av[1]); + av++; + continue; + } + vtc_fatal(vl, "Unknown VSM argument: %s", *av); + } +} + +#endif /* VTEST_WITH_VTC_VSM */ diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index 9ceb63456..58e680ff0 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -159,7 +159,8 @@ VTCSYN_SRC = \ $(top_srcdir)/bin/varnishtest/vtc_process.c \ $(top_srcdir)/bin/varnishtest/vtc_syslog.c \ $(top_srcdir)/bin/varnishtest/vtc_tunnel.c \ - $(top_srcdir)/bin/varnishtest/vtc_varnish.c + $(top_srcdir)/bin/varnishtest/vtc_varnish.c \ + $(top_srcdir)/bin/varnishtest/vtc_vsm.c include/vtc-syntax.rst: $(srcdir)/vtc-syntax.py $(VTCSYN_SRC) $(AM_V_GEN) $(PYTHON) $(srcdir)/vtc-syntax.py $(VTCSYN_SRC) > ${@}_ @mv -f ${@}_ ${@} From dridi.boukelmoune at gmail.com Mon Jun 10 13:42:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 10 Jun 2024 13:42:07 +0000 (UTC) Subject: [master] 53b078013 vsm: Missing VSM_WRK_CHANGED flag Message-ID: <20240610134207.75D2F76C9@lists.varnish-cache.org> commit 53b078013437343bfca66c5f4c6647ce35274ac1 Author: Dridi Boukelmoune Date: Fri Jun 7 16:01:13 2024 +0200 vsm: Missing VSM_WRK_CHANGED flag This was probably a copy-pasta accident of the previous line. It cannot be tested with an instrumented "varnish v1" type of varnishd instance, because it's not supposed to ever stop the MGT process once started. diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c index 5226bf828..88426e538 100644 --- a/lib/libvarnishapi/vsm.c +++ b/lib/libvarnishapi/vsm.c @@ -732,7 +732,7 @@ VSM_Status(struct vsm *vd) /* Open workdir */ if (vd->wdfd < 0) { retval |= VSM_MGT_RESTARTED | VSM_MGT_CHANGED; - retval |= VSM_WRK_RESTARTED | VSM_MGT_CHANGED; + retval |= VSM_WRK_RESTARTED | VSM_WRK_CHANGED; vd->wdfd = open(vd->wdname, O_RDONLY); if (vd->wdfd < 0) (void)vsm_diag(vd, From dridi.boukelmoune at gmail.com Mon Jun 10 13:42:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 10 Jun 2024 13:42:07 +0000 (UTC) Subject: [master] 728b92442 vsm: Drastically refine status checks Message-ID: <20240610134207.8E63576CE@lists.varnish-cache.org> commit 728b924420bf8eb454d66ed7dd4c238ec470f940 Author: Dridi Boukelmoune Date: Tue Jul 4 16:39:55 2023 +0200 vsm: Drastically refine status checks Summarily: - index hash change means restarted - index new plus or minus means changed - reopening the directory means restarted diff --git a/bin/varnishtest/tests/a00025.vtc b/bin/varnishtest/tests/a00025.vtc index 5f45be349..21c513f5c 100644 --- a/bin/varnishtest/tests/a00025.vtc +++ b/bin/varnishtest/tests/a00025.vtc @@ -2,29 +2,25 @@ varnishtest "Basic VSM status check" # just launch varnishd varnish v1 -cliok ping -# POLA? vsm m1 -expect-status mgt-running -vsm m1 -expect-status mgt-running,wrk-restarted +vsm m1 -expect-status mgt-running varnish v1 -vcl "backend be none;" -# POLA? vsm m1 -expect-status mgt-running -vsm m1 -expect-status mgt-running,wrk-restarted +vsm m1 -expect-status mgt-running varnish v1 -start -# POLA? vsm m1 -expect-status mgt-running,mgt-changed,wrk-running -vsm m1 -expect-status mgt-running,mgt-changed,mgt-restarted,wrk-running +vsm m1 -expect-status \ + mgt-running,mgt-changed,wrk-running,wrk-changed,wrk-restarted varnish v1 -vcl "backend be none;" -# POLA? vsm m1 -expect-status mgt-running,wrk-running,wrk-changed -vsm m1 -expect-status mgt-running,wrk-running +vsm m1 -expect-status mgt-running,wrk-running,wrk-changed # The order of status flags does not matter, the next one is equivalent. varnish v1 -vcl "backend be none;" -vsm m1 -expect-status wrk-running,mgt-running +vsm m1 -expect-status wrk-changed,wrk-running,mgt-running varnish v1 -stop -# POLA? vsm m1 -expect-status mgt-running,mgt-changed -vsm m1 -expect-status mgt-running +vsm m1 -expect-status mgt-running,mgt-changed varnish v1 -start -# POLA? vsm m1 -expect-status mgt-running,mgt-changed,wrk-running,wrk-restarted -vsm m1 -expect-status mgt-running,mgt-changed,mgt-restarted,wrk-running +vsm m1 -expect-status \ + mgt-running,mgt-changed,wrk-running,wrk-changed,wrk-restarted diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c index 88426e538..06632d591 100644 --- a/lib/libvarnishapi/vsm.c +++ b/lib/libvarnishapi/vsm.c @@ -494,18 +494,18 @@ vsm_vlu_hash(struct vsm *vd, struct vsm_set *vs, const char *line) i = sscanf(line, "# %ju %ju", &id1, &id2); if (i != 2) { - vs->retval |= VSM_MGT_RESTARTED | VSM_MGT_CHANGED; + vs->retval |= vs->flag_restarted; return (0); } if (vd->couldkill >= 0 && !kill(id1, 0)) { vd->couldkill = 1; } else if (vd->couldkill > 0 && errno == ESRCH) { - vs->retval |= VSM_MGT_RESTARTED | VSM_MGT_CHANGED; + vs->retval |= vs->flag_restarted | VSM_MGT_CHANGED; return (0); } vs->retval |= VSM_MGT_RUNNING; if (id1 != vs->id1 || id2 != vs->id2) { - vs->retval |= VSM_MGT_RESTARTED | VSM_MGT_CHANGED; + vs->retval |= vs->flag_restarted; vs->id1 = id1; vs->id2 = id2; } @@ -556,12 +556,13 @@ vsm_vlu_plus(struct vsm *vd, struct vsm_set *vs, const char *line) vg->cluster = vsm_findcluster(vs, vg->av[1]); CHECK_OBJ_NOTNULL(vg->cluster, VSM_SEG_MAGIC); } + vs->retval |= vs->flag_changed; } return (0); } static int -vsm_vlu_minus(struct vsm *vd, const struct vsm_set *vs, const char *line) +vsm_vlu_minus(struct vsm *vd, struct vsm_set *vs, const char *line) { char **av; int ac; @@ -584,6 +585,7 @@ vsm_vlu_minus(struct vsm *vd, const struct vsm_set *vs, const char *line) for (;vg != NULL; vg = VTAILQ_NEXT(vg, list)) { if (!vsm_cmp_av(&vg->av[1], &av[1])) { + vs->retval |= vs->flag_changed; vsm_delseg(vg, 1); break; } @@ -645,6 +647,7 @@ vsm_readlines(struct vsm_set *vs) static unsigned vsm_refresh_set(struct vsm *vd, struct vsm_set *vs) { + unsigned restarted = 0; struct stat st; CHECK_OBJ_NOTNULL(vd, VSM_MAGIC); @@ -657,6 +660,7 @@ vsm_refresh_set(struct vsm *vd, struct vsm_set *vs) st.st_mode != vs->dst.st_mode || st.st_nlink == 0)) { closefd(&vs->dfd); + restarted = vs->flag_restarted; } if (vs->dfd < 0) { @@ -668,7 +672,7 @@ vsm_refresh_set(struct vsm *vd, struct vsm_set *vs) if (vs->dfd < 0) { vs->id1 = vs->id2 = 0; vsm_wash_set(vs, 1); - return (vs->retval | vs->flag_restarted); + return (vs->retval | restarted); } AZ(fstat(vs->dfd, &vs->dst)); @@ -681,6 +685,7 @@ vsm_refresh_set(struct vsm *vd, struct vsm_set *vs) st.st_size < vs->fst.st_size || st.st_nlink < 1)) { closefd(&vs->fd); + vs->retval |= vs->flag_changed; } if (vs->fd >= 0) { @@ -692,7 +697,7 @@ vsm_refresh_set(struct vsm *vd, struct vsm_set *vs) vs->vg = VTAILQ_FIRST(&vs->segs); vs->fd = openat(vs->dfd, "_.index", O_RDONLY); if (vs->fd < 0) - return (vs->retval | vs->flag_restarted); + return (vs->retval | restarted); VLU_Reset(vs->vlu); AZ(fstat(vs->fd, &vs->fst)); vsm_readlines(vs); From dridi.boukelmoune at gmail.com Mon Jun 10 13:42:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 10 Jun 2024 13:42:07 +0000 (UTC) Subject: [master] 328247007 vsm: Assume a running process for EPERM Message-ID: <20240610134207.A5E5676D3@lists.varnish-cache.org> commit 328247007b7958e794a0ca0f0c8260b3a67fd57c Author: Dridi Boukelmoune Date: Sat May 25 08:32:48 2024 +0200 vsm: Assume a running process for EPERM Since there are two processes with possibly different credentials, track the ability to kill them at the vsm_set granularity. When a restart is established, stop assuming the ability to send signals to the process. If a live VSM advertises a process that yields something other than ESRCH, assuming that the PID is not recycled too soon, we can conclude that EPERM is as good as a successful kill. diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c index 06632d591..39b601de4 100644 --- a/lib/libvarnishapi/vsm.c +++ b/lib/libvarnishapi/vsm.c @@ -133,6 +133,8 @@ struct vsm_set { unsigned flag_running; unsigned flag_changed; unsigned flag_restarted; + + int couldkill; }; struct vsm { @@ -151,8 +153,6 @@ struct vsm { int attached; double patience; - - int couldkill; }; /*--------------------------------------------------------------------*/ @@ -301,6 +301,8 @@ vsm_newset(const char *dirname) vs->dfd = vs->fd = -1; vs->vlu = VLU_New(vsm_vlu_func, vs, 0); AN(vs->vlu); + if (getenv("VSM_NOPID") != NULL) + vs->couldkill = -1; return (vs); } @@ -364,8 +366,6 @@ VSM_New(void) vd->child->vsm = vd; vd->wdfd = -1; vd->patience = 5; - if (getenv("VSM_NOPID") != NULL) - vd->couldkill = -1; return (vd); } @@ -486,8 +486,28 @@ vsm_findcluster(const struct vsm_set *vs, const char *cnam) return (NULL); } +static unsigned +vsm_running(struct vsm_set *vs, pid_t pid) +{ + + AN(vs); + + if (pid == 0) + return (0); + assert(pid > 1); + + if (kill(pid, 0) == 0) { + vs->couldkill = 1; + return (1); + } + if (errno == EPERM) /* a process exists, assume running */ + return (1); + assert(errno != EINVAL); + return (0); +} + static int -vsm_vlu_hash(struct vsm *vd, struct vsm_set *vs, const char *line) +vsm_vlu_hash(struct vsm_set *vs, const char *line) { int i; uintmax_t id1, id2; @@ -497,9 +517,9 @@ vsm_vlu_hash(struct vsm *vd, struct vsm_set *vs, const char *line) vs->retval |= vs->flag_restarted; return (0); } - if (vd->couldkill >= 0 && !kill(id1, 0)) { - vd->couldkill = 1; - } else if (vd->couldkill > 0 && errno == ESRCH) { + if (vs->couldkill >= 0 && vsm_running(vs, id1)) { + /* nothing to do */ + } else if (vs->couldkill > 0 && errno == ESRCH) { vs->retval |= vs->flag_restarted | VSM_MGT_CHANGED; return (0); } @@ -614,7 +634,7 @@ vsm_vlu_func(void *priv, const char *line) switch (line[0]) { case '#': - i = vsm_vlu_hash(vd, vs, line); + i = vsm_vlu_hash(vs, line); VTAILQ_FOREACH(vs->vg, &vs->segs, list) vs->vg->flags &= ~VSM_FLAG_MARKSCAN; if (!(vs->retval & vs->flag_restarted)) @@ -706,7 +726,7 @@ vsm_refresh_set(struct vsm *vd, struct vsm_set *vs) vs->fst.st_size = lseek(vs->fd, 0L, SEEK_CUR); - if (vd->couldkill < 1 || !kill(vs->id1, 0)) + if (vs->couldkill < 0 || vsm_running(vs, vs->id1)) vs->retval |= vs->flag_running; return (vs->retval); } @@ -748,8 +768,12 @@ VSM_Status(struct vsm *vd) if (vd->wdfd >= 0) { retval |= vsm_refresh_set(vd, vd->mgt); + if (vd->mgt->couldkill > 0 && (retval & VSM_MGT_RESTARTED)) + vd->mgt->couldkill = 0; if (retval & VSM_MGT_RUNNING) retval |= vsm_refresh_set(vd, vd->child); + if (vd->child->couldkill > 0 && (retval & VSM_WRK_RESTARTED)) + vd->child->couldkill = 0; } return (retval); } From dridi.boukelmoune at gmail.com Mon Jun 10 13:42:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 10 Jun 2024 13:42:07 +0000 (UTC) Subject: [master] 98be1dbda vsm: New macros for status bit masks Message-ID: <20240610134207.C4A5E76D7@lists.varnish-cache.org> commit 98be1dbda88475c1771ba05bd75614a22a92f5a3 Author: Dridi Boukelmoune Date: Tue Jul 4 18:16:34 2023 +0200 vsm: New macros for status bit masks diff --git a/include/vapi/vsm.h b/include/vapi/vsm.h index cbc12941c..f41ca86b9 100644 --- a/include/vapi/vsm.h +++ b/include/vapi/vsm.h @@ -130,6 +130,9 @@ int VSM_Attach(struct vsm *, int progress); #define VSM_WRK_CHANGED (1U<<10) #define VSM_WRK_RESTARTED (1U<<11) +#define VSM_MGT_MASK 0x00ff +#define VSM_WRK_MASK 0xff00 + unsigned VSM_Status(struct vsm *); /* * Returns a bitmap of the current status and changes in it From dridi.boukelmoune at gmail.com Mon Jun 10 13:42:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 10 Jun 2024 13:42:07 +0000 (UTC) Subject: [master] 40373a39f vtc_varnish: Proper VSL status check Message-ID: <20240610134207.DD89D76DB@lists.varnish-cache.org> commit 40373a39f7556ae57db5195ff033f7b37dda4733 Author: Dridi Boukelmoune Date: Tue Jul 4 18:17:57 2023 +0200 vtc_varnish: Proper VSL status check diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index bc42a6dde..6f8223859 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -204,7 +204,7 @@ varnishlog_thread(void *priv) struct VSL_cursor *c; enum VSL_tag_e tag; uint64_t vxid; - unsigned len; + unsigned len, vs; const char *tagname, *data; int type, i, opt; struct vsb *vsb = NULL; @@ -271,7 +271,8 @@ varnishlog_thread(void *priv) if (i == 0) { /* Nothing to do but wait */ v->vsl_idle++; - if (!(VSM_Status(vsm) & VSM_WRK_RUNNING)) { + vs = VSM_Status(vsm) & VSM_WRK_MASK; + if ((vs & ~VSM_WRK_CHANGED) != VSM_WRK_RUNNING) { /* Abandoned - try reconnect */ VSL_DeleteCursor(c); c = NULL; From dridi.boukelmoune at gmail.com Mon Jun 10 13:42:08 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 10 Jun 2024 13:42:08 +0000 (UTC) Subject: [master] 8c8807f0e vtc: Proper -start and -stop in c123 Message-ID: <20240610134208.0488576DF@lists.varnish-cache.org> commit 8c8807f0ea9d8cba490f70ed6f1bfb0bcf28a0c9 Author: Dridi Boukelmoune Date: Tue Jul 4 18:26:54 2023 +0200 vtc: Proper -start and -stop in c123 And finally, the test logs contain the complete VSL output for all four transactions. It was an attempt at following the VSL in this test case that led me to finding where the VSM status was lacking or incorrect. diff --git a/bin/varnishtest/tests/c00123.vtc b/bin/varnishtest/tests/c00123.vtc index c72c55267..2875e2cd3 100644 --- a/bin/varnishtest/tests/c00123.vtc +++ b/bin/varnishtest/tests/c00123.vtc @@ -31,10 +31,11 @@ client c1 { # in practice a little over fetch_chunksize is allocated varnish v1 -expect SM?.s0.c_bytes < 20000 +varnish v1 -vsl_catchup # reset s0 counters -varnish v1 -cliok stop -varnish v1 -cliok start +varnish v1 -stop +varnish v1 -start varnish v1 -expect SM?.s0.c_bytes == 0 # content-length req.body streaming also needs one chunk @@ -45,10 +46,11 @@ client c2 { } -run varnish v1 -expect SM?.s0.c_bytes < 20000 +varnish v1 -vsl_catchup # reset s0 counters -varnish v1 -cliok stop -varnish v1 -cliok start +varnish v1 -stop +varnish v1 -start # chunked req.body caching allocates storage for the entire body client c3 { @@ -60,10 +62,11 @@ client c3 { } -run varnish v1 -expect SM?.s0.c_bytes > 100000 +varnish v1 -vsl_catchup # reset s0 counters -varnish v1 -cliok stop -varnish v1 -cliok start +varnish v1 -stop +varnish v1 -start # content-length req.body caching allocates storage for the entire body client c4 { @@ -73,3 +76,4 @@ client c4 { } -run varnish v1 -expect SM?.s0.c_bytes > 100000 +varnish v1 -vsl_catchup From dridi.boukelmoune at gmail.com Mon Jun 10 13:42:08 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 10 Jun 2024 13:42:08 +0000 (UTC) Subject: [master] 954e59b00 doc: Revisit documentation on VSM_NOPID Message-ID: <20240610134208.2336476E4@lists.varnish-cache.org> commit 954e59b00fc3189c5a5ac27de73616ce6c51e6f8 Author: Dridi Boukelmoune Date: Fri Jun 7 19:33:35 2024 +0200 doc: Revisit documentation on VSM_NOPID diff --git a/doc/sphinx/reference/vsm.rst b/doc/sphinx/reference/vsm.rst index 9d09ce77a..22d4f34ef 100644 --- a/doc/sphinx/reference/vsm.rst +++ b/doc/sphinx/reference/vsm.rst @@ -106,18 +106,18 @@ VSM files. VSM and Containers ------------------ -The varnish way works great with single purpose containers. By sharing -the varnish working directory read-only, vsm readers can be run in -containers separate from those running varnishd instances on the same -host. +The Varnish Shared Memory model works well in single-purpose containers. +By sharing the Varnish working directory read-only, VSM readers can run +in individual containers separate from those running varnishd instances on +the same host. -When running varnishd and vsm readers in the same process namespace, -pid information can be used by vsm readers to determine if varnishd -processes are alive. +On Linux, if varnishd and VSM readers run in the same process namespace, the +VSM readers can rely on the PID advertised by varnishd to determine whether +the manager and cache processes are alive. -But, when running varnishd and vsm readers in different containers, -the pid information has no relevance and may even be ambiguous across -name spaces. +However, if they live in different containers, exposing the Varnish working +directory as a volume to containers running VSM readers, the PIDs exposed by +varnishd are no longer relevant across namespaces. -Thus, with such setups, the environment variable VSM_NOPID needs to be -set for vsm readers to disable use of pid information. +To disable liveness checks based on PIDs, the variable ``VSM_NOPID`` needs to +be present in the environment of VSM readers. From dridi.boukelmoune at gmail.com Mon Jun 10 17:08:06 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 10 Jun 2024 17:08:06 +0000 (UTC) Subject: [master] f1698fcbc vfp: New VFP_DEBUG(ctx, fmt, ...) macro Message-ID: <20240610170806.799CE6E5CA@lists.varnish-cache.org> commit f1698fcbc96ebc3d1c5a3d44d1fac67455bbb939 Author: Dridi Boukelmoune Date: Fri May 24 11:50:41 2024 +0200 vfp: New VFP_DEBUG(ctx, fmt, ...) macro It needs the debug::processors flag and Debug records to be unmasked to become useful. diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index 18241ff63..d06540092 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -36,6 +36,14 @@ struct vdp_ctx; /* Fetch processors --------------------------------------------------*/ +#define VFP_DEBUG(ctx, fmt, ...) \ + do { \ + if (!DO_DEBUG(DBG_PROCESSORS)) \ + break; \ + VSLb((ctx)->wrk->vsl, SLT_Debug, "VFP:%s:%d: " fmt, \ + __func__, __LINE__, __VA_ARGS__); \ + } while (0) + enum vfp_status { VFP_ERROR = -1, VFP_OK = 0, From dridi.boukelmoune at gmail.com Mon Jun 10 17:08:06 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 10 Jun 2024 17:08:06 +0000 (UTC) Subject: [master] 50686f61e vfp: First stab at centralized debug logging Message-ID: <20240610170806.848FF6E5CC@lists.varnish-cache.org> commit 50686f61ec7e7b257cb38fdbcadf1b4147ec42d9 Author: Dridi Boukelmoune Date: Fri May 24 11:52:04 2024 +0200 vfp: First stab at centralized debug logging It was instrumental in the understanding of a filter bug that was previously difficult to troubleshoot. diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c index 704262b60..3336cda25 100644 --- a/bin/varnishd/cache/cache_fetch_proc.c +++ b/bin/varnishd/cache/cache_fetch_proc.c @@ -170,20 +170,30 @@ enum vfp_status VFP_Suck(struct vfp_ctx *vc, void *p, ssize_t *lp) { enum vfp_status vp; - struct vfp_entry *vfe; + struct vfp_entry *vfe, *vfe_prev; + const char *prev_name = ""; + ssize_t limit; CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); AN(p); AN(lp); + limit = *lp; vfe = vc->vfp_nxt; CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC); vc->vfp_nxt = VTAILQ_NEXT(vfe, list); + vfe_prev = VTAILQ_PREV(vfe, vfp_entry_s, list); + if (vfe_prev != NULL) + prev_name = vfe_prev->vfp->name; + if (vfe->closed == VFP_NULL) { /* Layer asked to be bypassed when opened */ vp = VFP_Suck(vc, p, lp); + VFP_DEBUG(vc, "bypassing %s vp=%d", vfe->vfp->name, vp); } else if (vfe->closed == VFP_OK) { vp = vfe->vfp->pull(vc, vfe, p, lp); + VFP_DEBUG(vc, "%s pulled %zdB/%zdB from %s vp=%d", + prev_name, *lp, limit, vfe->vfp->name, vp); if (vp != VFP_OK && vp != VFP_END && vp != VFP_ERROR) vp = VFP_Error(vc, "Fetch filter %s returned %d", vfe->vfp->name, vp); @@ -195,6 +205,7 @@ VFP_Suck(struct vfp_ctx *vc, void *p, ssize_t *lp) /* Already closed filter */ *lp = 0; vp = vfe->closed; + VFP_DEBUG(vc, "ignoring %s vp=%d", vfe->vfp->name, vp); } vc->vfp_nxt = vfe; assert(vp != VFP_NULL); From dridi.boukelmoune at gmail.com Tue Jun 11 15:44:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 11 Jun 2024 15:44:07 +0000 (UTC) Subject: [cci_eol] a2f5034be circleci: Retire Debian Buster jobs Message-ID: <20240611154407.47FD09CBA2@lists.varnish-cache.org> commit a2f5034be474a0a14921a0cd5cac749b2c28d04e Author: Dridi Boukelmoune Date: Tue Jun 11 17:36:52 2024 +0200 circleci: Retire Debian Buster jobs It reached EOL almost 2 years ago, but the architectures we cover in CI had 2 extra years of long term support that is coming to an end this month. diff --git a/.circleci/config.yml b/.circleci/config.yml index 09822cf9d..c8cbec3f8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -123,7 +123,7 @@ jobs: package: parameters: platform: - description: the Linux distribution, with release, e.g. debian:buster, fedora:latest + description: the Linux distribution, with release, e.g. debian:bookworm, fedora:latest type: string rclass: description: the resource class to use, usually arm.medium or medium @@ -203,7 +203,7 @@ jobs: description: the Linux distribution (debian|ubuntu) type: string release: - description: the release name (buster|bullseye|bookworm|focal|jammy|noble) + description: the release name (bullseye|bookworm|focal|jammy|noble) type: string make_target: description: the make target to execute during the build @@ -393,15 +393,11 @@ workflows: release: latest make_target: witness.dot # oldest debian goes 32bit - - build: - name: build_debian_buster - dist: debian - release: buster - prefix: i386/ - build: name: build_debian_bullseye dist: debian release: bullseye + prefix: i386/ - build: name: build_debian_bookworm dist: debian @@ -448,7 +444,6 @@ workflows: - ubuntu:focal - ubuntu:jammy - ubuntu:noble - - debian:buster - debian:bullseye - debian:bookworm - almalinux:8 From dridi.boukelmoune at gmail.com Tue Jun 11 15:44:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 11 Jun 2024 15:44:07 +0000 (UTC) Subject: [cci_eol] cbaf8d19f circleci: Align Alpine jobs to alpine:latest Message-ID: <20240611154407.682609CBA5@lists.varnish-cache.org> commit cbaf8d19fadd65de4b8bf36ee4864fbb11100b41 Author: Dridi Boukelmoune Date: Tue Jun 11 17:39:06 2024 +0200 circleci: Align Alpine jobs to alpine:latest diff --git a/.circleci/config.yml b/.circleci/config.yml index c8cbec3f8..f0c2978f4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -421,7 +421,7 @@ workflows: - build: name: build_alpine dist: alpine - release: "latest" + release: latest extra_conf: --without-contrib make_target: check - build: @@ -449,7 +449,7 @@ workflows: - almalinux:8 - almalinux:9 - fedora:latest - - alpine:3 + - alpine:latest rclass: - arm.medium - medium From dridi.boukelmoune at gmail.com Tue Jun 11 15:44:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 11 Jun 2024 15:44:07 +0000 (UTC) Subject: [cci_eol] abd9f0ad9 REMOVEME: attempt at always running packaging jobs Message-ID: <20240611154407.9FDCA9CBA9@lists.varnish-cache.org> commit abd9f0ad9246a40b5c06aba042c4b05289eed27d Author: Dridi Boukelmoune Date: Tue Jun 11 17:42:25 2024 +0200 REMOVEME: attempt at always running packaging jobs diff --git a/.circleci/config.yml b/.circleci/config.yml index f0c2978f4..ebaa18701 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -373,10 +373,6 @@ jobs: workflows: version: 2 commit: - unless: &packaging_cond - or: - - << pipeline.parameters.build-pkgs >> - - << pipeline.parameters.dist-url >> jobs: - build: name: build_almalinux_8 @@ -429,7 +425,6 @@ workflows: dist: archlinux release: base-devel packaging: - when: *packaging_cond jobs: &packaging_jobs - dist - tar_pkg_tools From dridi at varni.sh Tue Jun 11 15:58:33 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 11 Jun 2024 15:58:33 +0000 Subject: [cci_eol] abd9f0ad9 REMOVEME: attempt at always running packaging jobs In-Reply-To: <20240611154407.9FDCA9CBA9@lists.varnish-cache.org> References: <20240611154407.9FDCA9CBA9@lists.varnish-cache.org> Message-ID: Apologies for the noise, I meant to create this branch in my repository. It will hopefully go away soon as CI looks almost all green. Dridi On Tue, Jun 11, 2024 at 3:44?PM Dridi Boukelmoune wrote: > > > commit abd9f0ad9246a40b5c06aba042c4b05289eed27d > Author: Dridi Boukelmoune > Date: Tue Jun 11 17:42:25 2024 +0200 > > REMOVEME: attempt at always running packaging jobs > > diff --git a/.circleci/config.yml b/.circleci/config.yml > index f0c2978f4..ebaa18701 100644 > --- a/.circleci/config.yml > +++ b/.circleci/config.yml > @@ -373,10 +373,6 @@ jobs: > workflows: > version: 2 > commit: > - unless: &packaging_cond > - or: > - - << pipeline.parameters.build-pkgs >> > - - << pipeline.parameters.dist-url >> > jobs: > - build: > name: build_almalinux_8 > @@ -429,7 +425,6 @@ workflows: > dist: archlinux > release: base-devel > packaging: > - when: *packaging_cond > jobs: &packaging_jobs > - dist > - tar_pkg_tools > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi.boukelmoune at gmail.com Fri Jun 14 07:20:08 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 14 Jun 2024 07:20:08 +0000 (UTC) Subject: [cci_eol] 89ff683bc circleci: Use podman instead of docker in build jobs Message-ID: <20240614072008.922B21137E4@lists.varnish-cache.org> commit 89ff683bc3cc6b6f6ad1915424b68a841e087491 Author: Dridi Boukelmoune Date: Fri Jun 14 09:18:26 2024 +0200 circleci: Use podman instead of docker in build jobs diff --git a/.circleci/config.yml b/.circleci/config.yml index f0c2978f4..3bbe8b8c9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -224,15 +224,15 @@ jobs: steps: - setup_remote_docker - run: - name: Install docker - command: dnf -y install moby-engine + name: Install podman + command: dnf -y install podman - checkout - run: name: Extract and build command: | - docker create --name workspace -v /workspace << parameters.prefix >><< parameters.dist >>:<< parameters.release >> /bin/true - docker cp /workspace workspace:/ - docker run --volumes-from workspace -w /workspace << parameters.prefix >><< parameters.dist >>:<< parameters.release >> sh -c ' + podman create --name workspace -v /workspace docker://<< parameters.prefix >><< parameters.dist >>:<< parameters.release >> /bin/true + podman cp /workspace workspace:/ + podman run --volumes-from workspace -w /workspace << parameters.prefix >><< parameters.dist >>:<< parameters.release >> sh -c ' case "<< parameters.dist >>" in almalinux|fedora) dnf -y groupinstall "Development Tools" From dridi.boukelmoune at gmail.com Fri Jun 14 07:28:05 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 14 Jun 2024 07:28:05 +0000 (UTC) Subject: [cci_eol] baf63dc3d SQUASHME: do not create workspace in / Message-ID: <20240614072805.A052B113D72@lists.varnish-cache.org> commit baf63dc3d10e6b3f35104cfda48b38c635043cae Author: Dridi Boukelmoune Date: Fri Jun 14 09:26:11 2024 +0200 SQUASHME: do not create workspace in / diff --git a/.circleci/config.yml b/.circleci/config.yml index 3bbe8b8c9..79602fec4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -230,9 +230,9 @@ jobs: - run: name: Extract and build command: | - podman create --name workspace -v /workspace docker://<< parameters.prefix >><< parameters.dist >>:<< parameters.release >> /bin/true - podman cp /workspace workspace:/ - podman run --volumes-from workspace -w /workspace << parameters.prefix >><< parameters.dist >>:<< parameters.release >> sh -c ' + podman create --name workspace -v /root/workspace docker://<< parameters.prefix >><< parameters.dist >>:<< parameters.release >> /bin/true + podman cp /workspace workspace:/root + podman run --volumes-from workspace -w /root/workspace << parameters.prefix >><< parameters.dist >>:<< parameters.release >> sh -c ' case "<< parameters.dist >>" in almalinux|fedora) dnf -y groupinstall "Development Tools" From dridi.boukelmoune at gmail.com Fri Jun 14 10:07:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 14 Jun 2024 10:07:07 +0000 (UTC) Subject: [master] 346416503 req: Reset connection body status during cleanup Message-ID: <20240614100707.B199A119181@lists.varnish-cache.org> commit 34641650379b576f3b76719fb1025c4c696ce153 Author: Dridi Boukelmoune Date: Tue May 28 09:31:14 2024 +0200 req: Reset connection body status during cleanup Otherwise we can't rely on the body status to make decisions when more than one request is sent to an HTTP/1 connection. diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index bd457ff19..a9c20250d 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -308,6 +308,7 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req) req->vcf = NULL; req->doclose = SC_NULL; req->htc->doclose = SC_NULL; + req->htc->body_status = NULL; if (WS_Overflowed(req->ws)) wrk->stats->ws_client_overflow++; From dridi.boukelmoune at gmail.com Fri Jun 14 15:45:08 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 14 Jun 2024 15:45:08 +0000 (UTC) Subject: [cci_eol] f1add0403 SQUASHME: try rshared mount Message-ID: <20240614154508.64AE762ADC@lists.varnish-cache.org> commit f1add0403986acddfaba7222284795e055ca8c91 Author: Dridi Boukelmoune Date: Fri Jun 14 17:43:37 2024 +0200 SQUASHME: try rshared mount diff --git a/.circleci/config.yml b/.circleci/config.yml index 79602fec4..c9d557d3e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -230,6 +230,7 @@ jobs: - run: name: Extract and build command: | + mount --make-rshared / podman create --name workspace -v /root/workspace docker://<< parameters.prefix >><< parameters.dist >>:<< parameters.release >> /bin/true podman cp /workspace workspace:/root podman run --volumes-from workspace -w /root/workspace << parameters.prefix >><< parameters.dist >>:<< parameters.release >> sh -c ' From dridi.boukelmoune at gmail.com Fri Jun 14 15:47:04 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 14 Jun 2024 15:47:04 +0000 (UTC) Subject: [cci_eol] 02dec850c SQUASHME: try again with privileges? Message-ID: <20240614154704.C1ECE62DB3@lists.varnish-cache.org> commit 02dec850c5f9ab9f69a92971110e54681d11ab46 Author: Dridi Boukelmoune Date: Fri Jun 14 17:45:56 2024 +0200 SQUASHME: try again with privileges? diff --git a/.circleci/config.yml b/.circleci/config.yml index c9d557d3e..31143da42 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -230,7 +230,7 @@ jobs: - run: name: Extract and build command: | - mount --make-rshared / + sudo mount --make-rshared / podman create --name workspace -v /root/workspace docker://<< parameters.prefix >><< parameters.dist >>:<< parameters.release >> /bin/true podman cp /workspace workspace:/root podman run --volumes-from workspace -w /root/workspace << parameters.prefix >><< parameters.dist >>:<< parameters.release >> sh -c ' From dridi.boukelmoune at gmail.com Wed Jun 19 07:36:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 19 Jun 2024 07:36:07 +0000 (UTC) Subject: [cci_eol] b6bc32a65 SQUASHME: circleci: Remove nested docker for build jobs Message-ID: <20240619073607.1AFA257C7@lists.varnish-cache.org> commit b6bc32a65bcc0333693841d981f8fa0ac8694dce Author: Dridi Boukelmoune Date: Wed Jun 19 09:34:10 2024 +0200 SQUASHME: circleci: Remove nested docker for build jobs Better diff with the --ignore-all-space option. diff --git a/.circleci/config.yml b/.circleci/config.yml index 31143da42..bbcb42847 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -218,22 +218,14 @@ jobs: default: medium type: string docker: - - image: fedora:latest + - image: << parameters.prefix >><< parameters.dist >>:<< parameters.release >> resource_class: << parameters.rclass >> working_directory: /workspace steps: - - setup_remote_docker - - run: - name: Install podman - command: dnf -y install podman - checkout - run: name: Extract and build command: | - sudo mount --make-rshared / - podman create --name workspace -v /root/workspace docker://<< parameters.prefix >><< parameters.dist >>:<< parameters.release >> /bin/true - podman cp /workspace workspace:/root - podman run --volumes-from workspace -w /root/workspace << parameters.prefix >><< parameters.dist >>:<< parameters.release >> sh -c ' case "<< parameters.dist >>" in almalinux|fedora) dnf -y groupinstall "Development Tools" @@ -337,21 +329,17 @@ jobs: chown -R varnish:varnish . - export ASAN_OPTIONS=abort_on_error=1,detect_odr_violation=1,detect_leaks=1,detect_stack_use_after_return=1,detect_invalid_pointer_pairs=1,handle_segv=0,handle_sigbus=0,use_sigaltstack=0,disable_coredump=0 - export LSAN_OPTIONS=abort_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/lsan.suppr - export TSAN_OPTIONS=abort_on_error=1,halt_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/tsan.suppr - export UBSAN_OPTIONS=halt_on_error=1,print_stacktrace=1,use_sigaltstack=0,suppressions=$(pwd)/tools/ubsan.suppr - - sudo -u varnish \ + sudo -u varnish sh -c ' + export ASAN_OPTIONS=abort_on_error=1,detect_odr_violation=1,detect_leaks=1,detect_stack_use_after_return=1,detect_invalid_pointer_pairs=1,handle_segv=0,handle_sigbus=0,use_sigaltstack=0,disable_coredump=0 + export LSAN_OPTIONS=abort_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/lsan.suppr + export TSAN_OPTIONS=abort_on_error=1,halt_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/tsan.suppr + export UBSAN_OPTIONS=halt_on_error=1,print_stacktrace=1,use_sigaltstack=0,suppressions=$(pwd)/tools/ubsan.suppr autoreconf -i -v - sudo -u varnish \ ./configure \ - << pipeline.parameters.configure_args >> \ - << parameters.extra_conf >> - sudo -u varnish \ - --preserve-env=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \ + << pipeline.parameters.configure_args >> \ + << parameters.extra_conf >> make -j 4 -k << parameters.make_target >> VERBOSE=1 \ - DISTCHECK_CONFIGURE_FLAGS="<< pipeline.parameters.configure_args >> \ + DISTCHECK_CONFIGURE_FLAGS="<< pipeline.parameters.configure_args >> \ << parameters.extra_conf >>" ' From dridi.boukelmoune at gmail.com Wed Jun 19 08:00:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 19 Jun 2024 08:00:07 +0000 (UTC) Subject: [cci_eol] c955cb3e0 vtc: Do not resolve localhost in test cases Message-ID: <20240619080007.58B34674F@lists.varnish-cache.org> commit c955cb3e07bef551113f25a4b68fc0dfdda7dbcc Author: Dridi Boukelmoune Date: Wed Jun 19 09:58:50 2024 +0200 vtc: Do not resolve localhost in test cases diff --git a/bin/varnishtest/tests/b00058.vtc b/bin/varnishtest/tests/b00058.vtc index ba09916af..0b9afb73e 100644 --- a/bin/varnishtest/tests/b00058.vtc +++ b/bin/varnishtest/tests/b00058.vtc @@ -5,8 +5,8 @@ feature cmd {getent hosts localhost && getent services http} varnish v1 -arg "-p vcc_feature=-err_unref" -vcl { backend b1 {.host = "127.0.0.1";} backend b2 {.host = "[::1]:8080";} - backend b3 {.host = "localhost:8081";} - backend b4 {.host = "localhost:http";} + backend b3 {.host = "${localhost}:8081";} + backend b4 {.host = "${localhost}:http";} backend b5 {.host = "127.0.0.1";.port = "8081";} backend b6 {.host = "127.0.0.1";.port = "http";} } diff --git a/bin/varnishtest/tests/p00000.vtc b/bin/varnishtest/tests/p00000.vtc index e7a4219b4..d31177f73 100644 --- a/bin/varnishtest/tests/p00000.vtc +++ b/bin/varnishtest/tests/p00000.vtc @@ -3,7 +3,7 @@ varnishtest "Test Basic persistence" feature persistent_storage process p1 -dump { - varnishd -spersistent -b localhost -n ${tmpdir} -a :0 -d 2>&1 + varnishd -spersistent -b ${localhost} -n ${tmpdir} -a :0 -d 2>&1 } -start -expect-exit 1 process p1 -expect-text 0 0 {to launch} diff --git a/bin/varnishtest/tests/s00003.vtc b/bin/varnishtest/tests/s00003.vtc index f186d8956..284f0e8d6 100644 --- a/bin/varnishtest/tests/s00003.vtc +++ b/bin/varnishtest/tests/s00003.vtc @@ -46,14 +46,14 @@ varnish v1 -vsl_catchup varnish v1 -cliok "ban obj.http.date ~ ." process p1 { - varnishd -sTransient=file,${tmpdir}/foo,xxx -blocalhost -a:0 -n ${tmpdir} 2>&1 + varnishd -sTransient=file,${tmpdir}/foo,xxx -b${locahost} -a:0 -n ${tmpdir} 2>&1 } -expect-exit 0x2 -dump -start -expect-text 0 0 "Invalid number" -wait -screen_dump process p1 { - varnishd -sTransient=file,${tmpdir}/foo,10M,xxx -blocalhost -a:0 -n ${tmpdir} 2>&1 + varnishd -sTransient=file,${tmpdir}/foo,10M,xxx -b${locahost} -a:0 -n ${tmpdir} 2>&1 } -expect-exit 0x2 -dump -start -expect-text 0 0 "granularity" -wait -screen_dump process p1 { - varnishd -sTransient=file,${tmpdir}/foo,10m,,foo -blocalhost -a:0 -n ${tmpdir} 2>&1 + varnishd -sTransient=file,${tmpdir}/foo,10m,,foo -b${locahost} -a:0 -n ${tmpdir} 2>&1 } -expect-exit 0x2 -dump -start -expect-text 0 0 "invalid advice" -wait From phk at phk.freebsd.dk Wed Jun 19 08:04:14 2024 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Wed, 19 Jun 2024 08:04:14 +0000 Subject: [cci_eol] c955cb3e0 vtc: Do not resolve localhost in test cases In-Reply-To: <20240619080007.58B34674F@lists.varnish-cache.org> References: <20240619080007.58B34674F@lists.varnish-cache.org> Message-ID: <202406190804.45J84EaW075892@critter.freebsd.dk> Dridi Boukelmoune writes: All these should also go into trunk, no ? > - backend b3 {.host = "localhost:8081";} > - backend b4 {.host = "localhost:http";} > + backend b3 {.host = "${localhost}:8081";} > + backend b4 {.host = "${localhost}:http";} [...] -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From dridi.boukelmoune at gmail.com Wed Jun 19 08:21:05 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 19 Jun 2024 08:21:05 +0000 (UTC) Subject: [cci_eol] 14297da08 SQUASHME: unfortunate typo Message-ID: <20240619082105.81CF0783A@lists.varnish-cache.org> commit 14297da0839c284403930d66b460007e926692a7 Author: Dridi Boukelmoune Date: Wed Jun 19 10:19:58 2024 +0200 SQUASHME: unfortunate typo diff --git a/bin/varnishtest/tests/s00003.vtc b/bin/varnishtest/tests/s00003.vtc index 284f0e8d6..b15de012c 100644 --- a/bin/varnishtest/tests/s00003.vtc +++ b/bin/varnishtest/tests/s00003.vtc @@ -46,14 +46,14 @@ varnish v1 -vsl_catchup varnish v1 -cliok "ban obj.http.date ~ ." process p1 { - varnishd -sTransient=file,${tmpdir}/foo,xxx -b${locahost} -a:0 -n ${tmpdir} 2>&1 + varnishd -sTransient=file,${tmpdir}/foo,xxx -b${localhost} -a:0 -n ${tmpdir} 2>&1 } -expect-exit 0x2 -dump -start -expect-text 0 0 "Invalid number" -wait -screen_dump process p1 { - varnishd -sTransient=file,${tmpdir}/foo,10M,xxx -b${locahost} -a:0 -n ${tmpdir} 2>&1 + varnishd -sTransient=file,${tmpdir}/foo,10M,xxx -b${localhost} -a:0 -n ${tmpdir} 2>&1 } -expect-exit 0x2 -dump -start -expect-text 0 0 "granularity" -wait -screen_dump process p1 { - varnishd -sTransient=file,${tmpdir}/foo,10m,,foo -b${locahost} -a:0 -n ${tmpdir} 2>&1 + varnishd -sTransient=file,${tmpdir}/foo,10m,,foo -b${localhost} -a:0 -n ${tmpdir} 2>&1 } -expect-exit 0x2 -dump -start -expect-text 0 0 "invalid advice" -wait From dridi at varni.sh Wed Jun 19 08:21:20 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 19 Jun 2024 08:21:20 +0000 Subject: [cci_eol] c955cb3e0 vtc: Do not resolve localhost in test cases In-Reply-To: <202406190804.45J84EaW075892@critter.freebsd.dk> References: <20240619080007.58B34674F@lists.varnish-cache.org> <202406190804.45J84EaW075892@critter.freebsd.dk> Message-ID: On Wed, Jun 19, 2024 at 8:04?AM Poul-Henning Kamp wrote: > > Dridi Boukelmoune writes: > > All these should also go into trunk, no ? Eventually once https://github.com/varnishcache/varnish-cache/pull/4119 is merged. This is the branch backing the pull request. From dridi.boukelmoune at gmail.com Wed Jun 19 15:52:04 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 19 Jun 2024 15:52:04 +0000 (UTC) Subject: [master] ed9ce1be6 build: Clean more generated includes Message-ID: <20240619155204.AF4F110973F@lists.varnish-cache.org> commit ed9ce1be6c9994fda04e4185b72e999dff7ecd8d Author: Dridi Boukelmoune Date: Wed Jun 19 15:33:21 2024 +0200 build: Clean more generated includes We don't need to dial it up to 11 for the case where `make clean` runs in a source tree coming from a release archive instead of a git clone. Running `make distclean` should be enough to clear build artifacts that may not be known by older branches before switching to them. diff --git a/include/Makefile.am b/include/Makefile.am index ad4bc2555..347724d70 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -125,8 +125,9 @@ vcl.h: \ $(top_srcdir) $(top_builddir) GEN_H = \ - tbl/vrt_stv_var.h \ + tbl/vcl_context.h \ tbl/vcl_returns.h \ + tbl/vrt_stv_var.h \ vrt_obj.h $(GEN_H): vcl.h @@ -157,7 +158,7 @@ BUILT_SOURCES = \ vmod_abi.h \ vrt_test.c -MAINTAINERCLEANFILES = $(GENERATED_H) +DISTCLEANFILES = $(GENERATED_H) CLEANFILES = vrt_test.c diff --git a/lib/libvcc/Makefile.am b/lib/libvcc/Makefile.am index 117f38a3c..9e50c1922 100644 --- a/lib/libvcc/Makefile.am +++ b/lib/libvcc/Makefile.am @@ -65,4 +65,4 @@ GENERATED_H = vcc_obj.c $(GEN_H) BUILT_SOURCES = $(GENERATED_H) -MAINTAINERCLEANFILES = $(GENERATED_H) +DISTCLEANFILES = $(GENERATED_H) From dridi.boukelmoune at gmail.com Thu Jun 20 16:15:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 20 Jun 2024 16:15:07 +0000 (UTC) Subject: [master] 28ce8f181 travisci: Remove defunct Travis CI configuration Message-ID: <20240620161507.6265011748C@lists.varnish-cache.org> commit 28ce8f181b6a9e51f7a6b03a884a9056f2bcdfce Author: Dridi Boukelmoune Date: Thu Jun 20 18:13:34 2024 +0200 travisci: Remove defunct Travis CI configuration diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 3928c4d04..000000000 --- a/.travis.yml +++ /dev/null @@ -1,128 +0,0 @@ ---- -language: c - -jobs: - allow_failures: - - os: osx - - stage: sanitizers - - arch: arm64 - fast_finish: true - include: - - &test-linux - stage: test - os: linux - dist: bionic - arch: amd64 - compiler: clang - addons: - apt: - packages: - - nghttp2 - - python3-docutils - - python3-sphinx - - libunwind-dev - - libpcre2-dev - before_script: - - ./autogen.sh - - ./configure --enable-maintainer-mode --with-unwind - script: &script-common - - make -j16 check VERBOSE=1 - - <<: *test-linux - arch: arm64 - - <<: *test-linux - compiler: gcc - before_script: - - ./autogen.sh - - ./configure --enable-maintainer-mode - - <<: *test-linux - compiler: gcc - arch: arm64 - before_script: - - ./autogen.sh - - ./configure --enable-maintainer-mode - - <<: *test-linux - env: WITNESS=1 - script: make -j16 witness VERBOSE=1 - - <<: *test-linux - stage: sanitizers - addons: - apt: - sources: - - sourceline: 'deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main' - key_url: https://apt.llvm.org/llvm-snapshot.gpg.key - - ubuntu-toolchain-r-test - packages: - - clang-9 - - libunwind-dev - - llvm-9 - - nghttp2 - - python3-docutils - - python3-sphinx - - libpcre2-dev - env: ASAN=1 UBSAN=1 - before_script: - - | - export ASAN_OPTIONS=abort_on_error=1,detect_odr_violation=1,detect_leaks=1,detect_stack_use_after_return=1,detect_invalid_pointer_pairs=1,handle_segv=0,handle_sigbus=0,use_sigaltstack=0,disable_coredump=0 - export LSAN_OPTIONS=abort_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/lsan.suppr - export TSAN_OPTIONS=abort_on_error=1,halt_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/tsan.suppr - export UBSAN_OPTIONS=halt_on_error=1,print_stacktrace=1,use_sigaltstack=0,suppressions=$(pwd)/tools/ubsan.suppr - export CC=clang-9 - - ./autogen.sh - - ./configure --enable-maintainer-mode --with-unwind --enable-debugging-symbols --disable-stack-protector --with-persistent-storage --enable-asan --enable-ubsan --enable-workspace-emulator - - stage: test - os: osx - osx_image: xcode12.5 - compiler: clang - addons: - homebrew: - packages: - - docutils - - nghttp2 - - sphinx-doc - before_script: - - export PATH="/usr/local/opt/sphinx-doc/bin:$PATH" - - ./autogen.sh - - ./configure --enable-maintainer-mode - script: *script-common - - <<: *test-linux - stage: coverity - env: - - secure: "TndnHrwJk9FRSuVQWUk+ZrRc0jcNc0PW3TnvbRicIIwvYSLkMV5Y1tCQ5Jq/P98DA48/N/gf9DCAiFkxrNSKVeOY70FKgHYWlS130GhTv7r0c8zd+CdEXNORclcbBNV5F3Pli/LxZ+RUImjOfwcIcWV4eYv54Xv7aNFDAaDt4G9QlkSwXykLlZkoWLJQXFbhDBFioT1F1mucD9q9izEEeE+kqO1QH/IfobAq9v7/WrcS38sYI+0WvB1S0ajWuZJgRYqy1bocDNcQd05Vbr9NfAdJ9y+4VTuluZtTUyLxu3/0Tw8mAjHkcpOeNU26r3LnpdRk+5JuOFej/MrCmYRRawVfyvNGtu9RwcMkv8jl48TTs5kTf6UwFqJhe85QSlSi7IszfrE8HfB7B6u8eRr67rqjTr9k/BwEQyoBdK4JElQDj4A1GYHClomxgzmMZnVLvStnAm+IjdNlee4SfY0jj2KfPBd/v6Ms+LGVqNV9NDDKRQdOQD+H52MkIWs5Xu9fU5VaWP+xjFomA9aXex3r5FCssgyQ2P+HtWPdjNEtrkNezzfZ5b+VBVP87RdxfSqkZaRxi6gof0AgeTHWoi7GN1scseiKLxxCI7C0dfQgKrXTN7mZdcED1MMYdiaSI9mlSYQDDUHMQGeY1n3a9D6bUcC/TcmYo524PoTFBZgbbYM=" - before_script: - - curl - --data "token=$COVTOKEN&project=varnish" - --insecure - -o coverity_tool.tgz https://scan.coverity.com/download/cxx/linux64 - - tar xfz coverity_tool.tgz - - export PATH=$PATH:$(echo $(pwd)/cov-analysis-*/bin) - script: - - ./autogen.sh - - ./configure --enable-maintainer-mode --with-unwind - - cov-build --dir cov-int make - - tar cfz varnish.tgz cov-int - - curl --form token="$COVTOKEN" - --form email=varnish-dev at varnish-cache.org - --form file=@varnish.tgz - --form version="$TRAVIS_COMMIT" - --form description="$TRAVIS_BRANCH" - --insecure - 'https://scan.coverity.com/builds?project=varnish' - -stages: - - name: test - if: type != cron - - name: sanitizers - if: type != cron AND - type != pull_request - - name: coverity - if: type = cron - -notifications: - irc: - if: branch = master AND - repo = varnishcache/varnish-cache - channels: - - "irc.linpro.no#varnish-hacking" - on_success: change - use_notice: true - From dridi.boukelmoune at gmail.com Thu Jun 20 17:03:05 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 20 Jun 2024 17:03:05 +0000 (UTC) Subject: [cci_eol] da9918340 circleci: Avoid dashes in package version numbers Message-ID: <20240620170305.75A7C1190D6@lists.varnish-cache.org> commit da99183401713af8ecdde3c930a3943d705458a0 Author: Dridi Boukelmoune Date: Thu Jun 20 19:01:12 2024 +0200 circleci: Avoid dashes in package version numbers This wouldn't trigger on the master branch on account of having the version trunk. diff --git a/.circleci/config.yml b/.circleci/config.yml index bbcb42847..185e76fe2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -83,7 +83,7 @@ jobs: if grep 'AC_INIT.*trunk.*' ./configure.ac; then sed -i -e "s/AC_INIT(\[\(.*\)\],\s*\[\(.*\)\],\s*\[\(.*\)\])/AC_INIT([\1],[$(date +%Y%m%d)],[\3])/" ./configure.ac else - sed -i -e "s/AC_INIT(\[\(.*\)\],\s*\[\(.*\)\],\s*\[\(.*\)\])/AC_INIT([\1],[\2-$(date +%Y%m%d)],[\3])/" ./configure.ac + sed -i -e "s/AC_INIT(\[\(.*\)\],\s*\[\(.*\)\],\s*\[\(.*\)\])/AC_INIT([\1],[\2.$(date +%Y%m%d)],[\3])/" ./configure.ac fi ./autogen.des make dist -j 16 From dridi.boukelmoune at gmail.com Fri Jun 21 13:42:14 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:42:14 +0000 (UTC) Subject: [7.4] 553d9752d circleci: Remove Fedora Rawhide job Message-ID: <20240621134214.1FB25120034@lists.varnish-cache.org> commit 553d9752d54825a8f83b8238b21aa6350e150a09 Author: Dridi Boukelmoune Date: Tue Jan 2 11:34:45 2024 +0100 circleci: Remove Fedora Rawhide job Rawhide is not stable and we already have a job for the latest stable branch. diff --git a/.circleci/config.yml b/.circleci/config.yml index 88df75a90..8053fe7e4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -406,10 +406,6 @@ workflows: dist: fedora release: latest make_target: witness.dot - - build: - name: build_fedora_rawhide - dist: fedora - release: rawhide # latest debian uses sanitizers - build: name: build_debian_buster From dridi.boukelmoune at gmail.com Fri Jun 21 13:42:13 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:42:13 +0000 (UTC) Subject: [7.4] 594f01645 Fix .circleci/config.yml to match changes in configure.ac Message-ID: <20240621134214.114E6120032@lists.varnish-cache.org> commit 594f0164544f00cf0e6b9146ac4a47a1e76d6da1 Author: Simon Stridsberg Date: Fri Oct 6 09:01:41 2023 +0200 Fix .circleci/config.yml to match changes in configure.ac diff --git a/.circleci/config.yml b/.circleci/config.yml index 937a79b93..88df75a90 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,9 +82,9 @@ jobs: touch .is_weekly # If version is "trunk", override version to add date if grep 'AC_INIT.*trunk.*' ./configure.ac; then - sed -i -e "s/AC_INIT(\[\(.*\)\], \[\(.*\)\], \[\(.*\)\])/AC_INIT([\1], [$(date +%Y%m%d)], [\3])/" ./configure.ac + sed -i -e "s/AC_INIT(\[\(.*\)\],\[\(.*\)\],\[\(.*\)\])/AC_INIT([\1],[$(date +%Y%m%d)],[\3])/" ./configure.ac else - sed -i -e "s/AC_INIT(\[\(.*\)\], \[\(.*\)\], \[\(.*\)\])/AC_INIT([\1], [\2-$(date +%Y%m%d)], [\3])/" ./configure.ac + sed -i -e "s/AC_INIT(\[\(.*\)\],\[\(.*\)\],\[\(.*\)\])/AC_INIT([\1],[\2-$(date +%Y%m%d)],[\3])/" ./configure.ac fi ./autogen.des make dist -j 16 From dridi.boukelmoune at gmail.com Fri Jun 21 13:42:14 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:42:14 +0000 (UTC) Subject: [7.4] 667ad7e3e circleci: Add ubuntu:noble jobs Message-ID: <20240621134214.5DE8112003A@lists.varnish-cache.org> commit 667ad7e3eb44493b2c187e24b7e1ddedf841c5b9 Author: Guillaume Quintard Date: Fri Feb 23 22:09:31 2024 +0100 circleci: Add ubuntu:noble jobs diff --git a/.circleci/config.yml b/.circleci/config.yml index fa99f4b3e..3203a3018 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -433,6 +433,10 @@ workflows: name: build_ubuntu_jammy dist: ubuntu release: jammy + - build: + name: build_ubuntu_noble + dist: ubuntu + release: noble - build: name: build_alpine dist: alpine @@ -458,6 +462,7 @@ workflows: - ubuntu:bionic - ubuntu:focal - ubuntu:jammy + - ubuntu:noble - debian:buster - debian:bullseye - debian:bookworm From dridi.boukelmoune at gmail.com Fri Jun 21 13:42:14 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:42:14 +0000 (UTC) Subject: [7.4] 49f817ec9 circleci: Update setup_remote_docker version Message-ID: <20240621134214.3D5BB120037@lists.varnish-cache.org> commit 49f817ec9e04017d8d573b83154f091f913a3a9f Author: Guillaume Quintard Date: Sat Feb 24 15:52:39 2024 +0100 circleci: Update setup_remote_docker version https://discuss.circleci.com/t/remote-docker-image-deprecations-and-eol-for-2024/50176 diff --git a/.circleci/config.yml b/.circleci/config.yml index 8053fe7e4..fa99f4b3e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -223,8 +223,7 @@ jobs: - image: centos:7 working_directory: /workspace steps: - - setup_remote_docker: - version: 20.10.11 + - setup_remote_docker - run: name: Install docker command: yum install -y docker From dridi.boukelmoune at gmail.com Fri Jun 21 13:42:14 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:42:14 +0000 (UTC) Subject: [7.4] 2f5cb225e circleci: Swap 32bit/sanitizer responsibilities Message-ID: <20240621134214.7FB9B12003F@lists.varnish-cache.org> commit 2f5cb225e8edeff3674e3fa7cdb5cc658303583c Author: Guillaume Quintard Date: Wed Feb 28 10:42:28 2024 -0800 circleci: Swap 32bit/sanitizer responsibilities The current debian:latest image still ships a 32bit variant, whereas ubuntu:focal, next in line after ubuntu:bionic, no longer does. diff --git a/.circleci/config.yml b/.circleci/config.yml index 3203a3018..d0ed65f21 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -405,11 +405,12 @@ workflows: dist: fedora release: latest make_target: witness.dot - # latest debian uses sanitizers + # oldest debian goes 32bit - build: name: build_debian_buster dist: debian release: buster + prefix: i386/ - build: name: build_debian_bullseye dist: debian @@ -418,11 +419,9 @@ workflows: name: build_debian_bookworm dist: debian release: bookworm - extra_conf: --enable-asan --enable-ubsan --enable-workspace-emulator - # oldest ubuntu goes 32bit + # latest ubuntu uses sanitizers - build: name: build_ubuntu_bionic - prefix: i386/ dist: ubuntu release: bionic - build: @@ -437,6 +436,7 @@ workflows: name: build_ubuntu_noble dist: ubuntu release: noble + extra_conf: --enable-asan --enable-ubsan --enable-workspace-emulator - build: name: build_alpine dist: alpine From dridi.boukelmoune at gmail.com Fri Jun 21 13:42:14 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:42:14 +0000 (UTC) Subject: [7.4] dac257f47 circleci: Retire ubuntu:bionic jobs Message-ID: <20240621134214.9F492120044@lists.varnish-cache.org> commit dac257f47fdb408dc5cf968dd60dcc01969c79d2 Author: Guillaume Quintard Date: Thu Feb 29 09:56:57 2024 -0800 circleci: Retire ubuntu:bionic jobs diff --git a/.circleci/config.yml b/.circleci/config.yml index d0ed65f21..97a22c3bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -209,7 +209,7 @@ jobs: description: the Linux distribution (debian|ubuntu) type: string release: - description: the release name (buster|bullseye|bionic|focal|jammy) + description: the release name (buster|bullseye|bookworm|focal|jammy|noble) type: string make_target: description: the make target to execute during the build @@ -419,11 +419,6 @@ workflows: name: build_debian_bookworm dist: debian release: bookworm - # latest ubuntu uses sanitizers - - build: - name: build_ubuntu_bionic - dist: ubuntu - release: bionic - build: name: build_ubuntu_focal dist: ubuntu @@ -459,7 +454,6 @@ workflows: matrix: parameters: platform: - - ubuntu:bionic - ubuntu:focal - ubuntu:jammy - ubuntu:noble From dridi.boukelmoune at gmail.com Fri Jun 21 13:42:14 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:42:14 +0000 (UTC) Subject: [7.4] 1a0c63571 circleci: Reduce concurrency of ubuntu:noble jobs Message-ID: <20240621134214.BF1D7120047@lists.varnish-cache.org> commit 1a0c635711286ce0d2892c0ab7d1d63f6a7f2341 Author: Dridi Boukelmoune Date: Mon Mar 4 18:22:49 2024 +0100 circleci: Reduce concurrency of ubuntu:noble jobs In an attempt to avoid EAGAIN failures on pthread creations with ASAN and UBSAN sanitizers enabled. diff --git a/.circleci/config.yml b/.circleci/config.yml index 97a22c3bd..a59eb7a1e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -353,7 +353,7 @@ jobs: << parameters.extra_conf >> sudo -u varnish \ --preserve-env=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \ - make << parameters.make_target >> VERBOSE=1 -j 4 -k \ + make -j 4 -k << parameters.make_target >> VERBOSE=1 \ DISTCHECK_CONFIGURE_FLAGS="<< pipeline.parameters.configure_args >> \ << parameters.extra_conf >>" ' @@ -419,6 +419,7 @@ workflows: name: build_debian_bookworm dist: debian release: bookworm + # latest ubuntu uses sanitizers - build: name: build_ubuntu_focal dist: ubuntu @@ -432,6 +433,7 @@ workflows: dist: ubuntu release: noble extra_conf: --enable-asan --enable-ubsan --enable-workspace-emulator + make_target: check -j2 - build: name: build_alpine dist: alpine From dridi.boukelmoune at gmail.com Fri Jun 21 13:42:14 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:42:14 +0000 (UTC) Subject: [7.4] 458b45825 circleci: Only run the test suite on alpine Message-ID: <20240621134214.DC42712004C@lists.varnish-cache.org> commit 458b45825711c3929599afcb0d272e8cfd3cd404 Author: Dridi Boukelmoune Date: Tue Mar 5 07:47:47 2024 +0100 circleci: Only run the test suite on alpine There appears to be a race condition with distcheck where test-suite.log is removed and then another attempt at removing it fails: [...] test -z "test-suite.log" || rm -f test-suite.log rm -f libtool config.lt find . '(' -name '*.gcda' -o -name '*.gcda' ')' -exec rm '{}' ';' rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags rm -f cscope.out cscope.in.out cscope.po.out cscope.files find: ./test-suite.log: No such file or directory [...] It is unclear which find command is choking on test-suite.log, there doesn't seem to be a make rule running find and targeting this file. Could it be the find command listed in the output collecting file names like test-suite.log and then failing to test it against the -name filters after it got removed? That would be infortunate for a find implementation. Since alpine is not a platform we officially support, and after failing numerous times to reproduce the link outside of the CI environment, a simple make check is good enough and it will remove a lot of noise. diff --git a/.circleci/config.yml b/.circleci/config.yml index a59eb7a1e..3af5f35c5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -439,6 +439,7 @@ workflows: dist: alpine release: "latest" extra_conf: --without-contrib + make_target: check - build: name: build_archlinux dist: archlinux From dridi.boukelmoune at gmail.com Fri Jun 21 13:42:15 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:42:15 +0000 (UTC) Subject: [7.4] c3b25cfb3 Run docker without seccomp rules. Message-ID: <20240621134215.06FBA120050@lists.varnish-cache.org> commit c3b25cfb34db4c223b6fcb34e2ae49890b2b618a Author: Simon Stridsberg Date: Fri Mar 15 08:08:31 2024 +0100 Run docker without seccomp rules. Ubuntu noble tries to use `fchmodat2` (new syscall) and gets permission denied instead of ENOSYS. This is a small security risk but it's running inside of circleci containers anyway so i think its acceptable. diff --git a/.circleci/config.yml b/.circleci/config.yml index 3af5f35c5..e72147f5d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -186,6 +186,7 @@ jobs: docker run \ --rm \ -it \ + --security-opt seccomp=unconfined \ -e PARAM_DIST=$(echo "<< parameters.platform >>" | cut -d: -f1) \ -e PARAM_RELEASE=$(echo "<< parameters.platform >>" | cut -d: -f2) \ -v$(pwd):/varnish-cache \ From dridi.boukelmoune at gmail.com Fri Jun 21 13:42:15 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:42:15 +0000 (UTC) Subject: [7.4] fc92e9cb3 circleci: Tolerate spaces in AC_INIT() Message-ID: <20240621134215.25490120055@lists.varnish-cache.org> commit fc92e9cb3b5d26494ba57b98536af3374053c5ac Author: Dridi Boukelmoune Date: Tue Mar 19 10:19:58 2024 +0100 circleci: Tolerate spaces in AC_INIT() Better diff with the --word-diff --word-diff-regex=. options. diff --git a/.circleci/config.yml b/.circleci/config.yml index e72147f5d..94e434e55 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,9 +82,9 @@ jobs: touch .is_weekly # If version is "trunk", override version to add date if grep 'AC_INIT.*trunk.*' ./configure.ac; then - sed -i -e "s/AC_INIT(\[\(.*\)\],\[\(.*\)\],\[\(.*\)\])/AC_INIT([\1],[$(date +%Y%m%d)],[\3])/" ./configure.ac + sed -i -e "s/AC_INIT(\[\(.*\)\],\s*\[\(.*\)\],\s*\[\(.*\)\])/AC_INIT([\1],[$(date +%Y%m%d)],[\3])/" ./configure.ac else - sed -i -e "s/AC_INIT(\[\(.*\)\],\[\(.*\)\],\[\(.*\)\])/AC_INIT([\1],[\2-$(date +%Y%m%d)],[\3])/" ./configure.ac + sed -i -e "s/AC_INIT(\[\(.*\)\],\s*\[\(.*\)\],\s*\[\(.*\)\])/AC_INIT([\1],[\2-$(date +%Y%m%d)],[\3])/" ./configure.ac fi ./autogen.des make dist -j 16 From dridi.boukelmoune at gmail.com Fri Jun 21 13:42:15 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:42:15 +0000 (UTC) Subject: [7.4] b34287567 circleci: Switch centos image to stream9 tag Message-ID: <20240621134215.430AB120059@lists.varnish-cache.org> commit b34287567c56041a6fabc7d9f7e5a885a84b2959 Author: Dridi Boukelmoune Date: Mon Jun 3 11:03:03 2024 +0200 circleci: Switch centos image to stream9 tag The plain "stream" image has not been updated for a year, and instead of being the equivalent of a "latest" tag it just appears to be frozen. Since there isn't an automatic centos-stream upgrade path, let's target version 9 explicitly for now. diff --git a/.circleci/config.yml b/.circleci/config.yml index 94e434e55..e718074fd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -391,7 +391,7 @@ workflows: name: build_centos_stream prefix: quay.io/centos/ dist: centos - release: stream + release: stream9 - build: name: build_almalinux_8 dist: almalinux From dridi.boukelmoune at gmail.com Fri Jun 21 13:42:15 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:42:15 +0000 (UTC) Subject: [7.4] a1e7148c4 circleci: Missing nines for centos:stream9 Message-ID: <20240621134215.6035E12005E@lists.varnish-cache.org> commit a1e7148c4d06bddcce1e7cb40a0fda899a9e4980 Author: Dridi Boukelmoune Date: Mon Jun 3 11:21:15 2024 +0200 circleci: Missing nines for centos:stream9 I don't understand why they didn't maintain the unversioned centos:stream tag as a kind of latest tag after dropping the actual centos:latest tag. diff --git a/.circleci/config.yml b/.circleci/config.yml index e718074fd..38b4f7ec9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -174,7 +174,7 @@ jobs: esac case "<< parameters.platform >>" in - centos:stream) REPO=quay.io/centos/ ;; + centos:stream9) REPO=quay.io/centos/ ;; *) REPO= ;; esac @@ -239,13 +239,13 @@ jobs: centos|almalinux|fedora) yum groupinstall -y "Development Tools" case "<< parameters.dist >>:<< parameters.release >>" in - almalinux:9) + centos:stream9|almalinux:9) dnf install -y "dnf-command(config-manager)" yum config-manager --set-enabled crb yum install -y diffutils yum install -y epel-release ;; - centos:stream|almalinux:8) + almalinux:8) dnf install -y "dnf-command(config-manager)" yum config-manager --set-enabled powertools yum install -y diffutils @@ -388,7 +388,7 @@ workflows: dist: centos release: "7" - build: - name: build_centos_stream + name: build_centos_stream_9 prefix: quay.io/centos/ dist: centos release: stream9 @@ -465,7 +465,7 @@ workflows: - debian:bullseye - debian:bookworm - centos:7 - - centos:stream + - centos:stream9 - almalinux:8 - almalinux:9 - fedora:latest From dridi.boukelmoune at gmail.com Fri Jun 21 13:42:15 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:42:15 +0000 (UTC) Subject: [7.4] 1a08425ab circleci: Run sanitizers on a larger system Message-ID: <20240621134215.80B0D120067@lists.varnish-cache.org> commit 1a08425ab2094f3ade17537cf11f7de595e9fb1e Author: Dridi Boukelmoune Date: Thu Jun 6 11:10:13 2024 +0200 circleci: Run sanitizers on a larger system This is an attempt at speeding up this job that usually takes three time as long as other distcheck jobs. diff --git a/.circleci/config.yml b/.circleci/config.yml index 38b4f7ec9..387743847 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -127,7 +127,7 @@ jobs: description: the Linux distribution, with release, e.g. debian:buster, centos:7 type: string rclass: - description: the resource class to use, usuall arm.medium or medium + description: the resource class to use, usually arm.medium or medium type: string machine: image: ubuntu-2004:202111-02 @@ -220,8 +220,13 @@ jobs: description: platform-specific configure arguments default: "" type: string + rclass: + description: job-specific resource class + default: medium + type: string docker: - image: centos:7 + resource_class: << parameters.rclass >> working_directory: /workspace steps: - setup_remote_docker @@ -433,8 +438,9 @@ workflows: name: build_ubuntu_noble dist: ubuntu release: noble + rclass: large extra_conf: --enable-asan --enable-ubsan --enable-workspace-emulator - make_target: check -j2 + make_target: check - build: name: build_alpine dist: alpine From dridi.boukelmoune at gmail.com Fri Jun 21 13:42:15 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:42:15 +0000 (UTC) Subject: [7.4] acdfe3d7e circleci: Retire CentOS jobs Message-ID: <20240621134215.A209F120077@lists.varnish-cache.org> commit acdfe3d7e4fcd1257025de2ebecfa2f78e0e3b48 Author: Dridi Boukelmoune Date: Tue Jun 11 16:34:25 2024 +0200 circleci: Retire CentOS jobs With CentOS reaching EOL at the end of the month and CentOS Stream 9 flaky packaging jobs, we don't need to keep them around. For el9 builds we rely on the almalinux:9 image. diff --git a/.circleci/config.yml b/.circleci/config.yml index 387743847..0cf89f30d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -124,7 +124,7 @@ jobs: package: parameters: platform: - description: the Linux distribution, with release, e.g. debian:buster, centos:7 + description: the Linux distribution, with release, e.g. debian:buster, fedora:latest type: string rclass: description: the resource class to use, usually arm.medium or medium @@ -164,20 +164,14 @@ jobs: command: | mkdir -p packages case "<< parameters.platform >>" in - debian:*|ubuntu:*) EXT=deb ;; - centos:*|fedora:*) EXT=rpm ;; - almalinux:*) EXT=rpm ;; - alpine:*) EXT=apk ;; + debian:*|ubuntu:*) EXT=deb ;; + almalinux:*|fedora:*) EXT=rpm ;; + alpine:*) EXT=apk ;; *) echo "unrecognized platform: << parameters.platform >>" exit 1 esac - case "<< parameters.platform >>" in - centos:stream9) REPO=quay.io/centos/ ;; - *) REPO= ;; - esac - case "<< parameters.rclass >>" in arm.*) ARCH=arm64 ;; *) ARCH=amd64 ;; @@ -191,7 +185,7 @@ jobs: -e PARAM_RELEASE=$(echo "<< parameters.platform >>" | cut -d: -f2) \ -v$(pwd):/varnish-cache \ --platform linux/$ARCH \ - ${REPO}<< parameters.platform >> \ + << parameters.platform >> \ /varnish-cache/.circleci/make-$EXT-packages.sh - run: name: List created packages @@ -241,26 +235,23 @@ jobs: docker cp /workspace workspace:/ docker run --volumes-from workspace -w /workspace << parameters.prefix >><< parameters.dist >>:<< parameters.release >> sh -c ' case "<< parameters.dist >>" in - centos|almalinux|fedora) - yum groupinstall -y "Development Tools" + almalinux|fedora) + dnf -y groupinstall "Development Tools" case "<< parameters.dist >>:<< parameters.release >>" in - centos:stream9|almalinux:9) - dnf install -y "dnf-command(config-manager)" - yum config-manager --set-enabled crb - yum install -y diffutils - yum install -y epel-release + almalinux:9) + dnf -y install "dnf-command(config-manager)" + dnf config-manager --set-enabled crb + dnf -y install diffutils + dnf -y install epel-release ;; almalinux:8) - dnf install -y "dnf-command(config-manager)" - yum config-manager --set-enabled powertools - yum install -y diffutils - yum install -y epel-release - ;; - centos:7) - yum install -y epel-release + dnf -y install "dnf-command(config-manager)" + dnf config-manager --set-enabled powertools + dnf -y install diffutils + dnf -y install epel-release ;; esac - yum install -y \ + dnf -y install \ cpio \ automake \ git \ @@ -336,7 +327,7 @@ jobs: archlinux) useradd varnish ;; - centos|almalinux|fedora) + almalinux|fedora) adduser varnish ;; *) @@ -388,15 +379,6 @@ workflows: - << pipeline.parameters.build-pkgs >> - << pipeline.parameters.dist-url >> jobs: - - build: - name: build_centos_7 - dist: centos - release: "7" - - build: - name: build_centos_stream_9 - prefix: quay.io/centos/ - dist: centos - release: stream9 - build: name: build_almalinux_8 dist: almalinux @@ -470,8 +452,6 @@ workflows: - debian:buster - debian:bullseye - debian:bookworm - - centos:7 - - centos:stream9 - almalinux:8 - almalinux:9 - fedora:latest diff --git a/.circleci/make-rpm-packages.sh b/.circleci/make-rpm-packages.sh index 848084bc4..4a9f8a142 100755 --- a/.circleci/make-rpm-packages.sh +++ b/.circleci/make-rpm-packages.sh @@ -6,30 +6,27 @@ echo "PARAM_RELEASE: $PARAM_RELEASE" echo "PARAM_DIST: $PARAM_DIST" if [ -z "$PARAM_RELEASE" ]; then - echo "Env variable PARAM_RELEASE is not set! For example PARAM_RELEASE=stream, for CentOS stream" + echo "Env variable PARAM_RELEASE is not set! For example PARAM_RELEASE=9, for almalinux" exit 1 elif [ -z "$PARAM_DIST" ]; then - echo "Env variable PARAM_DIST is not set! For example PARAM_DIST=centos" + echo "Env variable PARAM_DIST is not set! For example PARAM_DIST=fedora" exit 1 fi case "$PARAM_DIST:$PARAM_RELEASE" in almalinux:9) - dnf install -y 'dnf-command(config-manager)' - yum config-manager --set-enabled crb - yum install -y epel-release + dnf -y install 'dnf-command(config-manager)' + dnf config-manager --set-enabled crb + dnf -y install epel-release ;; - centos:stream|almalinux:8) - dnf install -y 'dnf-command(config-manager)' - yum config-manager --set-enabled powertools - yum install -y epel-release - ;; - centos:7) - yum install -y epel-release + almalinux:8) + dnf -y install 'dnf-command(config-manager)' + dnf config-manager --set-enabled powertools + dnf -y install epel-release ;; esac -yum install -y rpm-build yum-utils +dnf -y install rpm-build dnf-utils export DIST_DIR=build @@ -73,7 +70,7 @@ rpmbuild() { "$@" } -yum-builddep -y "$DIST_DIR"/redhat/varnish.spec +dnf builddep -y "$DIST_DIR"/redhat/varnish.spec rpmbuild -bs "$DIST_DIR"/redhat/varnish.spec rpmbuild --rebuild "$RESULT_DIR"/varnish-*.src.rpm From dridi.boukelmoune at gmail.com Fri Jun 21 13:42:15 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:42:15 +0000 (UTC) Subject: [7.4] 0f3722c21 circleci: Retire Debian Buster jobs Message-ID: <20240621134215.BF7FD120081@lists.varnish-cache.org> commit 0f3722c2192610c0dd697f2a72283794b0697915 Author: Dridi Boukelmoune Date: Tue Jun 11 17:36:52 2024 +0200 circleci: Retire Debian Buster jobs It reached EOL almost 2 years ago, but the architectures we cover in CI had 2 extra years of long term support that is coming to an end this month. diff --git a/.circleci/config.yml b/.circleci/config.yml index 0cf89f30d..5bcb72e6b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -124,7 +124,7 @@ jobs: package: parameters: platform: - description: the Linux distribution, with release, e.g. debian:buster, fedora:latest + description: the Linux distribution, with release, e.g. debian:bookworm, fedora:latest type: string rclass: description: the resource class to use, usually arm.medium or medium @@ -204,7 +204,7 @@ jobs: description: the Linux distribution (debian|ubuntu) type: string release: - description: the release name (buster|bullseye|bookworm|focal|jammy|noble) + description: the release name (bullseye|bookworm|focal|jammy|noble) type: string make_target: description: the make target to execute during the build @@ -394,15 +394,11 @@ workflows: release: latest make_target: witness.dot # oldest debian goes 32bit - - build: - name: build_debian_buster - dist: debian - release: buster - prefix: i386/ - build: name: build_debian_bullseye dist: debian release: bullseye + prefix: i386/ - build: name: build_debian_bookworm dist: debian @@ -449,7 +445,6 @@ workflows: - ubuntu:focal - ubuntu:jammy - ubuntu:noble - - debian:buster - debian:bullseye - debian:bookworm - almalinux:8 From dridi.boukelmoune at gmail.com Fri Jun 21 13:42:15 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:42:15 +0000 (UTC) Subject: [7.4] e6d06f71e circleci: Align Alpine jobs to alpine:latest Message-ID: <20240621134215.D7285120096@lists.varnish-cache.org> commit e6d06f71ea7d6a2e0387122bf1c5008b4d0117c7 Author: Dridi Boukelmoune Date: Tue Jun 11 17:39:06 2024 +0200 circleci: Align Alpine jobs to alpine:latest diff --git a/.circleci/config.yml b/.circleci/config.yml index 5bcb72e6b..0d5632560 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -422,7 +422,7 @@ workflows: - build: name: build_alpine dist: alpine - release: "latest" + release: latest extra_conf: --without-contrib make_target: check - build: @@ -450,7 +450,7 @@ workflows: - almalinux:8 - almalinux:9 - fedora:latest - - alpine:3 + - alpine:latest rclass: - arm.medium - medium From dridi.boukelmoune at gmail.com Fri Jun 21 13:42:15 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:42:15 +0000 (UTC) Subject: [7.4] 71abe2ed6 circleci: Move away from centos:7 as a base image Message-ID: <20240621134216.044EC1200A6@lists.varnish-cache.org> commit 71abe2ed63786a5db68813f8917bdc2bb8c39ce2 Author: Dridi Boukelmoune Date: Tue Jun 11 16:28:49 2024 +0200 circleci: Move away from centos:7 as a base image For the build jobs we can directly use the target images and avoid the nested docker invocation. For other jobs or Dockerfiles where centos:7 is used we can switch to a self-updating fedora:latest image. Better diff with the --ignore-all-space option. diff --git a/.circleci/Dockerfile b/.circleci/Dockerfile index 71bbce5be..9ef231e0f 100644 --- a/.circleci/Dockerfile +++ b/.circleci/Dockerfile @@ -1,8 +1,7 @@ -FROM centos:7 +FROM fedora-latest -RUN set -e;\ - yum install -y epel-release; \ - yum install -y \ +RUN set -e; \ + dnf -y install \ automake \ git \ jemalloc-devel \ @@ -12,4 +11,4 @@ RUN set -e;\ make \ pcre2-devel \ python3 \ - python-sphinx + python3-sphinx diff --git a/.circleci/config.yml b/.circleci/config.yml index 0d5632560..a6eeb3f73 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,13 +30,12 @@ jobs: dist: description: Build or download varnish-x.y.z.tar.gz that is used later for the packaging jobs docker: - - image: centos:7 + - image: fedora:latest steps: - run: name: Install deps command: | - yum install -y epel-release - yum install -y \ + dnf -y install \ automake \ jemalloc-devel \ git \ @@ -46,7 +45,7 @@ jobs: make \ pcre2-devel \ python3 \ - python-sphinx + python3-sphinx - checkout - when: condition: << pipeline.parameters.dist-url >> @@ -98,7 +97,7 @@ jobs: tar_pkg_tools: description: Builds archives with the packaging tools from https://github.com/varnishcache/pkg-varnish-cache docker: - - image: centos:7 + - image: fedora:latest steps: - add_ssh_keys: fingerprints: @@ -106,7 +105,7 @@ jobs: - run: name: Grab the pkg repo command: | - yum install -y git + dnf -y install git mkdir -p ~/.ssh ssh-keyscan -H github.com >> ~/.ssh/known_hosts echo ${CIRCLE_REPOSITORY_URL} @@ -219,21 +218,14 @@ jobs: default: medium type: string docker: - - image: centos:7 + - image: << parameters.prefix >><< parameters.dist >>:<< parameters.release >> resource_class: << parameters.rclass >> working_directory: /workspace steps: - - setup_remote_docker - - run: - name: Install docker - command: yum install -y docker - checkout - run: name: Extract and build command: | - docker create --name workspace -v /workspace << parameters.prefix >><< parameters.dist >>:<< parameters.release >> /bin/true - docker cp /workspace workspace:/ - docker run --volumes-from workspace -w /workspace << parameters.prefix >><< parameters.dist >>:<< parameters.release >> sh -c ' case "<< parameters.dist >>" in almalinux|fedora) dnf -y groupinstall "Development Tools" @@ -337,27 +329,23 @@ jobs: chown -R varnish:varnish . - export ASAN_OPTIONS=abort_on_error=1,detect_odr_violation=1,detect_leaks=1,detect_stack_use_after_return=1,detect_invalid_pointer_pairs=1,handle_segv=0,handle_sigbus=0,use_sigaltstack=0,disable_coredump=0 - export LSAN_OPTIONS=abort_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/lsan.suppr - export TSAN_OPTIONS=abort_on_error=1,halt_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/tsan.suppr - export UBSAN_OPTIONS=halt_on_error=1,print_stacktrace=1,use_sigaltstack=0,suppressions=$(pwd)/tools/ubsan.suppr - - sudo -u varnish \ + sudo -u varnish sh -c ' + export ASAN_OPTIONS=abort_on_error=1,detect_odr_violation=1,detect_leaks=1,detect_stack_use_after_return=1,detect_invalid_pointer_pairs=1,handle_segv=0,handle_sigbus=0,use_sigaltstack=0,disable_coredump=0 + export LSAN_OPTIONS=abort_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/lsan.suppr + export TSAN_OPTIONS=abort_on_error=1,halt_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/tsan.suppr + export UBSAN_OPTIONS=halt_on_error=1,print_stacktrace=1,use_sigaltstack=0,suppressions=$(pwd)/tools/ubsan.suppr autoreconf -i -v - sudo -u varnish \ ./configure \ - << pipeline.parameters.configure_args >> \ - << parameters.extra_conf >> - sudo -u varnish \ - --preserve-env=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \ + << pipeline.parameters.configure_args >> \ + << parameters.extra_conf >> make -j 4 -k << parameters.make_target >> VERBOSE=1 \ - DISTCHECK_CONFIGURE_FLAGS="<< pipeline.parameters.configure_args >> \ + DISTCHECK_CONFIGURE_FLAGS="<< pipeline.parameters.configure_args >> \ << parameters.extra_conf >>" ' collect_packages: docker: - - image: centos:7 + - image: fedora:latest steps: - attach_workspace: at: ~/project From dridi.boukelmoune at gmail.com Fri Jun 21 13:42:16 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:42:16 +0000 (UTC) Subject: [7.4] cd4cfc47d circleci: Avoid dashes in package version numbers Message-ID: <20240621134216.267211200D8@lists.varnish-cache.org> commit cd4cfc47d3fa55e8906214f4a894216f814805ab Author: Dridi Boukelmoune Date: Thu Jun 20 19:01:12 2024 +0200 circleci: Avoid dashes in package version numbers This wouldn't trigger on the master branch on account of having the version trunk. diff --git a/.circleci/config.yml b/.circleci/config.yml index a6eeb3f73..185ebe99e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -83,7 +83,7 @@ jobs: if grep 'AC_INIT.*trunk.*' ./configure.ac; then sed -i -e "s/AC_INIT(\[\(.*\)\],\s*\[\(.*\)\],\s*\[\(.*\)\])/AC_INIT([\1],[$(date +%Y%m%d)],[\3])/" ./configure.ac else - sed -i -e "s/AC_INIT(\[\(.*\)\],\s*\[\(.*\)\],\s*\[\(.*\)\])/AC_INIT([\1],[\2-$(date +%Y%m%d)],[\3])/" ./configure.ac + sed -i -e "s/AC_INIT(\[\(.*\)\],\s*\[\(.*\)\],\s*\[\(.*\)\])/AC_INIT([\1],[\2.$(date +%Y%m%d)],[\3])/" ./configure.ac fi ./autogen.des make dist -j 16 From dridi.boukelmoune at gmail.com Fri Jun 21 13:42:16 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:42:16 +0000 (UTC) Subject: [7.4] 93f2e7516 vtc: Do not resolve localhost in test cases Message-ID: <20240621134216.4DB381200EE@lists.varnish-cache.org> commit 93f2e7516bdc055d122d5141642268e36a3759b0 Author: Dridi Boukelmoune Date: Wed Jun 19 09:58:50 2024 +0200 vtc: Do not resolve localhost in test cases diff --git a/bin/varnishtest/tests/b00058.vtc b/bin/varnishtest/tests/b00058.vtc index ba09916af..0b9afb73e 100644 --- a/bin/varnishtest/tests/b00058.vtc +++ b/bin/varnishtest/tests/b00058.vtc @@ -5,8 +5,8 @@ feature cmd {getent hosts localhost && getent services http} varnish v1 -arg "-p vcc_feature=-err_unref" -vcl { backend b1 {.host = "127.0.0.1";} backend b2 {.host = "[::1]:8080";} - backend b3 {.host = "localhost:8081";} - backend b4 {.host = "localhost:http";} + backend b3 {.host = "${localhost}:8081";} + backend b4 {.host = "${localhost}:http";} backend b5 {.host = "127.0.0.1";.port = "8081";} backend b6 {.host = "127.0.0.1";.port = "http";} } diff --git a/bin/varnishtest/tests/p00000.vtc b/bin/varnishtest/tests/p00000.vtc index e7a4219b4..d31177f73 100644 --- a/bin/varnishtest/tests/p00000.vtc +++ b/bin/varnishtest/tests/p00000.vtc @@ -3,7 +3,7 @@ varnishtest "Test Basic persistence" feature persistent_storage process p1 -dump { - varnishd -spersistent -b localhost -n ${tmpdir} -a :0 -d 2>&1 + varnishd -spersistent -b ${localhost} -n ${tmpdir} -a :0 -d 2>&1 } -start -expect-exit 1 process p1 -expect-text 0 0 {to launch} diff --git a/bin/varnishtest/tests/s00003.vtc b/bin/varnishtest/tests/s00003.vtc index f186d8956..b15de012c 100644 --- a/bin/varnishtest/tests/s00003.vtc +++ b/bin/varnishtest/tests/s00003.vtc @@ -46,14 +46,14 @@ varnish v1 -vsl_catchup varnish v1 -cliok "ban obj.http.date ~ ." process p1 { - varnishd -sTransient=file,${tmpdir}/foo,xxx -blocalhost -a:0 -n ${tmpdir} 2>&1 + varnishd -sTransient=file,${tmpdir}/foo,xxx -b${localhost} -a:0 -n ${tmpdir} 2>&1 } -expect-exit 0x2 -dump -start -expect-text 0 0 "Invalid number" -wait -screen_dump process p1 { - varnishd -sTransient=file,${tmpdir}/foo,10M,xxx -blocalhost -a:0 -n ${tmpdir} 2>&1 + varnishd -sTransient=file,${tmpdir}/foo,10M,xxx -b${localhost} -a:0 -n ${tmpdir} 2>&1 } -expect-exit 0x2 -dump -start -expect-text 0 0 "granularity" -wait -screen_dump process p1 { - varnishd -sTransient=file,${tmpdir}/foo,10m,,foo -blocalhost -a:0 -n ${tmpdir} 2>&1 + varnishd -sTransient=file,${tmpdir}/foo,10m,,foo -b${localhost} -a:0 -n ${tmpdir} 2>&1 } -expect-exit 0x2 -dump -start -expect-text 0 0 "invalid advice" -wait From dridi.boukelmoune at gmail.com Fri Jun 21 13:42:16 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:42:16 +0000 (UTC) Subject: [7.4] 15bcca0f4 vtc: Bump stack for ESI tests tight on 32bit systems Message-ID: <20240621134216.87CA3120109@lists.varnish-cache.org> commit 15bcca0f4fc97b9d4d083344e45cf9ba88d2752d Author: Dridi Boukelmoune Date: Fri Mar 1 12:34:09 2024 +0100 vtc: Bump stack for ESI tests tight on 32bit systems Refs #4063 diff --git a/bin/varnishtest/tests/e00016.vtc b/bin/varnishtest/tests/e00016.vtc index 142032366..180f1a769 100644 --- a/bin/varnishtest/tests/e00016.vtc +++ b/bin/varnishtest/tests/e00016.vtc @@ -26,10 +26,11 @@ server s1 { } } -start -varnish v1 \ - -syntax 4.0 \ - -arg "-p feature=+esi_disable_xml_check" \ - -vcl+backend { +# give enough stack to 32bit systems +varnish v1 -cliok "param.set thread_pool_stack 80k" +varnish v1 -cliok "param.set feature +esi_disable_xml_check" + +varnish v1 -syntax 4.0 -vcl+backend { sub vcl_backend_response { set beresp.do_esi = true; } diff --git a/bin/varnishtest/tests/e00030.vtc b/bin/varnishtest/tests/e00030.vtc index ff2016fb9..424207020 100644 --- a/bin/varnishtest/tests/e00030.vtc +++ b/bin/varnishtest/tests/e00030.vtc @@ -1,7 +1,6 @@ varnishtest "Test req_top.* in an ESI context" -varnish v1 -arg "-p feature=+esi_disable_xml_check" \ - -errvcl {Variable is read only.} { +varnish v1 -errvcl {Variable is read only.} { backend foo None; sub vcl_recv { @@ -52,6 +51,10 @@ server s1 { txresp } -start +# give enough stack to 32bit systems +varnish v1 -cliok "param.set thread_pool_stack 80k" +varnish v1 -cliok "param.set feature +esi_disable_xml_check" + varnish v1 -vcl+backend { sub vcl_recv { if (req.esi_level > 0) { diff --git a/bin/varnishtest/tests/e00034.vtc b/bin/varnishtest/tests/e00034.vtc index 2749b3df3..b0e93ba8e 100644 --- a/bin/varnishtest/tests/e00034.vtc +++ b/bin/varnishtest/tests/e00034.vtc @@ -72,6 +72,9 @@ server s1 { } } -start +# give enough stack to 32bit systems +varnish v1 -cliok "param.set thread_pool_stack 80k" + varnish v1 -syntax 4.1 -vcl+backend { import debug; diff --git a/bin/varnishtest/tests/l00003.vtc b/bin/varnishtest/tests/l00003.vtc index 1e94845df..5ecef3faa 100644 --- a/bin/varnishtest/tests/l00003.vtc +++ b/bin/varnishtest/tests/l00003.vtc @@ -14,6 +14,9 @@ server s1 { txresp -body {123} } -start +# give enough stack to 32bit systems +varnish v1 -cliok "param.set thread_pool_stack 80k" + varnish v1 -vcl+backend { sub vcl_backend_response { if (bereq.url != "/2") { diff --git a/bin/varnishtest/tests/r01737.vtc b/bin/varnishtest/tests/r01737.vtc index 845b1de3a..454bd96e8 100644 --- a/bin/varnishtest/tests/r01737.vtc +++ b/bin/varnishtest/tests/r01737.vtc @@ -23,7 +23,11 @@ server s1 { rxresp } -start -varnish v1 -arg "-p feature=+esi_disable_xml_check" -vcl+backend { +# give enough stack to 32bit systems +varnish v1 -cliok "param.set thread_pool_stack 80k" +varnish v1 -cliok "param.set feature +esi_disable_xml_check" + +varnish v1 -vcl+backend { sub vcl_backend_response { set beresp.do_esi = true; } diff --git a/bin/varnishtest/tests/r01781.vtc b/bin/varnishtest/tests/r01781.vtc index 4801d0613..31e21bf9a 100644 --- a/bin/varnishtest/tests/r01781.vtc +++ b/bin/varnishtest/tests/r01781.vtc @@ -13,7 +13,11 @@ server s1 { txresp -body {Foo} } -start -varnish v1 -arg "-p feature=+esi_disable_xml_check" -vcl+backend { +# give enough stack to 32bit systems +varnish v1 -cliok "param.set thread_pool_stack 80k" +varnish v1 -cliok "param.set feature +esi_disable_xml_check" + +varnish v1 -vcl+backend { sub vcl_backend_response { set beresp.do_gzip = true; set beresp.do_esi = true; diff --git a/bin/varnishtest/tests/r01878.vtc b/bin/varnishtest/tests/r01878.vtc index 14831d35b..2d8a0f1fa 100644 --- a/bin/varnishtest/tests/r01878.vtc +++ b/bin/varnishtest/tests/r01878.vtc @@ -14,6 +14,9 @@ server s1 { txresp -hdr "id: bar" -body "<3>bar" } -start +# give enough stack to 32bit systems +varnish v1 -cliok "param.set thread_pool_stack 80k" + varnish v1 -vcl+backend { sub vcl_backend_response { if (bereq.url != "/bar") { diff --git a/bin/varnishtest/tests/r02849.vtc b/bin/varnishtest/tests/r02849.vtc index 3022bcf78..f7860ab35 100644 --- a/bin/varnishtest/tests/r02849.vtc +++ b/bin/varnishtest/tests/r02849.vtc @@ -22,6 +22,9 @@ server s1 { txresp -body {} } -start +# give enough stack to 32bit systems +varnish v1 -cliok "param.set thread_pool_stack 80k" + varnish v1 -vcl+backend { sub vcl_recv { if (req.url == "/l3") { diff --git a/bin/varnishtest/tests/v00042.vtc b/bin/varnishtest/tests/v00042.vtc index b378a2731..3d77175ea 100644 --- a/bin/varnishtest/tests/v00042.vtc +++ b/bin/varnishtest/tests/v00042.vtc @@ -33,6 +33,9 @@ server s1 { txresp } -start +# give enough stack to 32bit systems +varnish v1 -cliok "param.set thread_pool_stack 80k" + varnish v1 -vcl+backend { import debug; diff --git a/bin/varnishtest/tests/v00043.vtc b/bin/varnishtest/tests/v00043.vtc index 49edc6b1a..0d9e6cb93 100644 --- a/bin/varnishtest/tests/v00043.vtc +++ b/bin/varnishtest/tests/v00043.vtc @@ -71,7 +71,11 @@ varnish v1 -errvcl "Not available in subroutine 'vcl_init'" { } } -varnish v1 -cliok "param.set debug +syncvsl" -vcl+backend { +# give enough stack to 32bit systems +varnish v1 -cliok "param.set thread_pool_stack 80k" +varnish v1 -cliok "param.set debug +syncvsl" + +varnish v1 -vcl+backend { import debug; sub vcl_init { From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:08 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:08 +0000 (UTC) Subject: [6.0] 2c214ad55 Make witness mode a first-class citizen Message-ID: <20240621134308.C4E8512096C@lists.varnish-cache.org> commit 2c214ad5573726a73a8a6d04970ec1c65759d30f Author: Dridi Boukelmoune Date: Mon Sep 30 08:02:20 2019 +0200 Make witness mode a first-class citizen This change introduces a top-level make witness target that builds a dot graph and if graphviz is available, an SVG file as well. A shell script replaces the previous python script that no longer works. Instead of fixing witness.py, which is probably trivial, the shell script does an intermediate pass and programmatically looks for cycles using tsort(1). Checking lock dependencies becomes actionable in a CI context. The script also takes explicit test directories on purpose, to have the ability to aggregate test results from multiple executions. For example when the test suite is run on various operating systems or with varying privileges to cover feature-conditional tests. Conflicts: tools/witness.sh The 6.0 branch does not contain the dT change in the vtc output, so columns collected by the shell script were off by one. diff --git a/.gitignore b/.gitignore index f6870aa1b..2bac013d4 100644 --- a/.gitignore +++ b/.gitignore @@ -139,6 +139,10 @@ cscope.*out /cov-int /myproject.tgz +# Witness droppings +witness.dot +witness.svg + # Flexelint droppings _.fl _.fl.old diff --git a/Makefile.am b/Makefile.am index f019082a2..781d413ce 100644 --- a/Makefile.am +++ b/Makefile.am @@ -8,7 +8,13 @@ pkgconfig_DATA = varnishapi.pc m4dir = $(datadir)/aclocal m4_DATA = varnish.m4 varnish-legacy.m4 -CLEANFILES = cscope.in.out cscope.out cscope.po.out +CLEANFILES = \ + cscope.in.out \ + cscope.out \ + cscope.po.out \ + witness.dot \ + witness.svg + EXTRA_DIST = \ README.rst \ README.Packaging \ @@ -48,4 +54,24 @@ cscope: gcov_digest: ${PYTHON} tools/gcov_digest.py -o _gcov -.PHONY: cscope +witness.dot: all + $(MAKE) -C bin/varnishtest check AM_VTC_LOG_FLAGS=-pdebug=+witness + $(AM_V_GEN) $(srcdir)/tools/witness.sh witness.dot bin/varnishtest + +.dot.svg: +if ! HAVE_DOT + @echo ================================================== + @echo You need graphviz installed to generate svg output + @echo ================================================== + @false +else + $(AM_V_GEN) $(DOT) -Tsvg $< >$@ +endif + +if HAVE_DOT +witness: witness.svg +else +witness: witness.dot +endif + +.PHONY: cscope witness.dot diff --git a/bin/varnishtest/witness.py b/bin/varnishtest/witness.py deleted file mode 100644 index f5b516ee0..000000000 --- a/bin/varnishtest/witness.py +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python3 -# -# This script is in the public domain -# -# Run instructions: -# varnishtest -W -iv -j > _.w -# python witness.py -# dot -Tpng /tmp/_.dot > /tmp/_.png - -d = dict() -a = dict() - -fi = open("_.w") -fo = open("/tmp/_.dot", "w") - -fo.write('''digraph { - #rotate="90" - #page="8.2,11.7" - size="8.2,11.7" - rankdir="LR" - node [fontname="Inconsolata", fontsize="10"] - edge [fontname="Inconsolata", fontsize="10"] -''') - -for i in fi: - l = "ROOT" - j = i.split() - if len(j) < 8: - continue - if j[1][0] != 'v': - continue - if j[3] != "vsl|": - continue - if j[5] != "Witness": - continue - t = j[7:] - tt = str(t) - if tt in d: - continue - d[tt] = True - for e in t: - f = e.split(",") - x = '"%s" -> "%s" [label="%s(%s)"]\n' % (l, f[0], f[1], f[2]) - if not x in a: - a[x] = True - fo.write(x) - l = f[0] - -fo.write("}\n") - diff --git a/tools/witness.sh b/tools/witness.sh new file mode 100755 index 000000000..814fae4a0 --- /dev/null +++ b/tools/witness.sh @@ -0,0 +1,82 @@ +#!/bin/sh + +set -e +set -u + +readonly work_dir=$(mktemp -d) +trap 'rm -rf "$work_dir"' EXIT + +witness_full_paths() { + find "$@" -name '*.log' -print0 | + xargs -0 awk '$4 == "vsl|" && $6 == "Witness" { + printf "%s", "ROOT" + for (i = 8; i <= NF; i++) { + printf " %s", $i + } + printf "\n" + }' | + sort | + uniq +} + +witness_edges() { + awk '{ + for (i = 1; i < NF; i++) { + printf "%s %s\n", $i, $(i + 1) + } + }' | + sort | + uniq +} + +witness_cycles() { + ! awk -F '[ ,]' '{print $1 " " $(NF - 2)}' | + tsort >/dev/null 2>&1 +} + +witness_graph() { + cat <<-EOF + digraph { + size="8.2,11.7" + rankdir="LR" + node [fontname="Inconsolata", fontsize="10"] + edge [fontname="Inconsolata", fontsize="10"] + EOF + + awk -F '[ ,]' '{ + printf " \"%s\" -> \"%s\" [label=\"%s(%s)\"]\n", + $1, $(NF - 2), $(NF - 1), $NF + }' | + sort | + uniq + + echo '}' +} + +if [ $# -lt 2 ] +then + cat >&2 <<-EOF + usage: $0 dot_file test_dirs... + EOF + exit 1 +fi + +dest_file=$1 +shift + +witness_full_paths "$@" | +witness_edges >"$work_dir/witness-edges.txt" + +tsort_err= + +if witness_cycles <"$work_dir/witness-edges.txt" +then + echo "Error: lock cycle witnessed" >&2 + tsort_err=1 +fi + +witness_graph <"$work_dir/witness-edges.txt" >"$work_dir/witness.dot" + +mv "$work_dir/witness.dot" "$dest_file" + +exit $tsort_err From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:08 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:08 +0000 (UTC) Subject: [6.0] da278133d Fix failing tests in witness mode Message-ID: <20240621134308.F19DD120972@lists.varnish-cache.org> commit da278133d18d6998fd50b96e9fec47c7e054e86a Author: Dridi Boukelmoune Date: Mon Sep 30 07:53:29 2019 +0200 Fix failing tests in witness mode The simple fact that Witness records might show up in the log might break logexpect commands. There's no reason why we'd want to expect Witness records since their purpose is to be checked after the test finishes. diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 11f046f95..7a10ff8ce 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -281,7 +281,7 @@ logexp_dispatch(struct VSL_data *vsl, struct VSL_transaction * const pt[], data = VSL_CDATA(t->c->rec.ptr); len = VSL_LEN(t->c->rec.ptr) - 1; - if (tag == SLT__Batch) + if (tag == SLT__Batch || tag == SLT_Witness) continue; ok = 1; From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:09 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:09 +0000 (UTC) Subject: [6.0] 5ce486945 build: Enable automake's subdir-objects option Message-ID: <20240621134309.2597E12097D@lists.varnish-cache.org> commit 5ce48694594709bb1faa967f3922a4841777cbc3 Author: Dridi Boukelmoune Date: Wed Jul 1 14:00:03 2020 +0200 build: Enable automake's subdir-objects option We can get rid of autogen.des' egrep command used to silence this warning too. In fact we can even remove the subshell and simply call autoreconf directly. The problem with egrep is the loss of a meaningful exit status for the autoreconf invocation. I also enabled autoreconf's verbose output. Conflicts: autogen.des diff --git a/autogen.des b/autogen.des index 0cc52dc42..c066ee8d3 100755 --- a/autogen.des +++ b/autogen.des @@ -22,8 +22,8 @@ else DST="--prefix=/opt/varnish --mandir=/opt/varnish/man" fi -rm -f configure -(cd $SRCDIR && . ./autogen.sh 2>&1 | egrep -v "(subdir-objects|is in a subdirectory)") +rm -f $SRCDIR/configure +(cd $SRCDIR && . ./autogen.sh 2>&1) # autoconf prior to 2.62 has issues with zsh 4.2 and newer export CONFIG_SHELL=/bin/sh diff --git a/configure.ac b/configure.ac index 1fc8d677c..f3024becd 100644 --- a/configure.ac +++ b/configure.ac @@ -19,7 +19,7 @@ AC_CANONICAL_SYSTEM AC_LANG(C) AM_MAINTAINER_MODE([disable]) -AM_INIT_AUTOMAKE([1.11 foreign color-tests parallel-tests]) +AM_INIT_AUTOMAKE([1.11 foreign color-tests parallel-tests subdir-objects]) AM_SILENT_RULES([yes]) AC_DISABLE_STATIC AC_PROG_LIBTOOL From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:09 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:09 +0000 (UTC) Subject: [6.0] a54a8f1a8 build: Ignore directory stamps Message-ID: <20240621134309.4706A12098D@lists.varnish-cache.org> commit a54a8f1a8806c7ef2434224810ca8ec7ae120078 Author: Dridi Boukelmoune Date: Thu Jul 30 11:34:18 2020 +0200 build: Ignore directory stamps I believe we get those when we have "subdir-objects", which we do now. diff --git a/.gitignore b/.gitignore index 2bac013d4..6f5ccaf37 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ Makefile.in TAGS tags cscope.*out +.dirstamp # Default vcl made from bin/varnishd/default.vcl /bin/varnishd/builtin_vcl.c From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:09 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:09 +0000 (UTC) Subject: [6.0] e5238678c Flexelinting: Avoid stuff which looks like trigraphs Message-ID: <20240621134309.680BF12099E@lists.varnish-cache.org> commit e5238678c3e6d6dffbf632a60365234510e008f5 Author: Poul-Henning Kamp Date: Thu Oct 3 09:04:12 2019 +0000 Flexelinting: Avoid stuff which looks like trigraphs diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 3a37ea65b..4626eaaf2 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -612,7 +612,7 @@ pan_backtrace(struct vsb *vsb) unw_get_reg(&cursor, UNW_REG_SP, &sp); unw_get_proc_name(&cursor, fname, sizeof(fname), &offp); VSB_printf(vsb, "ip=0x%lx, sp=0x%lx <%s+0x%lx>\n", (long) ip, - (long) sp, fname[0] ? fname : "???", offp); + (long) sp, fname[0] ? fname : "", offp); } VSB_indent(vsb, -2); From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:09 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:09 +0000 (UTC) Subject: [6.0] c3d43744a Make unwind work under macos Message-ID: <20240621134309.856C11209A4@lists.varnish-cache.org> commit c3d43744ac4b86600003395965ddd6c0cec6de1f Author: Federico G. Schwindt Date: Thu Oct 31 11:07:30 2019 +0900 Make unwind work under macos diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 4626eaaf2..eefacba6a 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -612,7 +612,7 @@ pan_backtrace(struct vsb *vsb) unw_get_reg(&cursor, UNW_REG_SP, &sp); unw_get_proc_name(&cursor, fname, sizeof(fname), &offp); VSB_printf(vsb, "ip=0x%lx, sp=0x%lx <%s+0x%lx>\n", (long) ip, - (long) sp, fname[0] ? fname : "", offp); + (long) sp, fname[0] ? fname : "", (long)offp); } VSB_indent(vsb, -2); diff --git a/configure.ac b/configure.ac index f3024becd..9cc8a213b 100644 --- a/configure.ac +++ b/configure.ac @@ -341,7 +341,14 @@ AC_ARG_WITH([unwind], [use libunwind to print stacktraces (use libexecinfo otherwise). Recommended on alpine linux. Defaults to no.])]) if test "$with_unwind" = yes; then - PKG_CHECK_MODULES([LIBUNWIND], [libunwind]) + case $target in + *-*-darwin*) + # Always present but .pc is not installed + ;; + *) + PKG_CHECK_MODULES([LIBUNWIND], [libunwind]) + ;; + esac AC_DEFINE([WITH_UNWIND], [1], [Define to 1 to use libunwind instead of libexecinfo]) else From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:09 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:09 +0000 (UTC) Subject: [6.0] f154a99c3 Make the VTIM_format() test 64bit only Message-ID: <20240621134309.A26571209AA@lists.varnish-cache.org> commit f154a99c309aed86a42143f98cbed0b66ce398cf Author: Nils Goroll Date: Fri Apr 30 22:19:47 2021 +0200 Make the VTIM_format() test 64bit only It is not obvious how we could trigger a gmtime_r() EOVERFLOW on 32bit Thank you phk for pushing me back onto the right track at a late hour. Ref #3308 diff --git a/bin/varnishtest/tests/r03308.vtc b/bin/varnishtest/tests/r03308.vtc index a97cb0cd5..79cc250e2 100644 --- a/bin/varnishtest/tests/r03308.vtc +++ b/bin/varnishtest/tests/r03308.vtc @@ -1,5 +1,7 @@ varnishtest "Unformatable VCL_TIME" +feature 64bit + server s1 { rxreq txresp From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:09 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:09 +0000 (UTC) Subject: [6.0] 13100b75d circleci: Catch up with 7.4 Message-ID: <20240621134309.CC1CC1209B5@lists.varnish-cache.org> commit 13100b75dbefd0d1da1d11bae9c786081fd130a7 Author: Dridi Boukelmoune Date: Wed Jun 19 14:51:43 2024 +0200 circleci: Catch up with 7.4 diff --git a/.circleci/Dockerfile b/.circleci/Dockerfile new file mode 100644 index 000000000..7553611e0 --- /dev/null +++ b/.circleci/Dockerfile @@ -0,0 +1,15 @@ +FROM centos:7 + +RUN set -e;\ + yum install -y epel-release; \ + yum install -y \ + automake \ + git \ + jemalloc-devel \ + libedit-devel \ + libtool \ + libunwind-devel \ + make \ + pcre-devel \ + python3 \ + python-sphinx diff --git a/.circleci/Dockerfile.alpine b/.circleci/Dockerfile.alpine new file mode 100644 index 000000000..c09aa9d5a --- /dev/null +++ b/.circleci/Dockerfile.alpine @@ -0,0 +1,20 @@ +FROM alpine + +RUN set -e; \ + apk update; \ + apk add -q \ + autoconf \ + automake \ + build-base \ + ca-certificates \ + cpio \ + git \ + gzip \ + libedit-dev \ + libtool \ + libunwind-dev \ + linux-headers \ + pcre-dev \ + py-docutils \ + py3-sphinx \ + tar diff --git a/.circleci/Dockerfile.archlinux b/.circleci/Dockerfile.archlinux new file mode 100644 index 000000000..2b9201792 --- /dev/null +++ b/.circleci/Dockerfile.archlinux @@ -0,0 +1,15 @@ +FROM archlinux:base-devel + +RUN set -e; \ + pacman -Sy --noconfirm \ + ca-certificates \ + cpio \ + git \ + libedit \ + libtool \ + libunwind \ + linux-headers \ + pcre \ + python-docutils \ + python-sphinx \ + tar diff --git a/.circleci/Dockerfile.ubuntu b/.circleci/Dockerfile.ubuntu new file mode 100644 index 000000000..cc50bd3a0 --- /dev/null +++ b/.circleci/Dockerfile.ubuntu @@ -0,0 +1,22 @@ +FROM ubuntu + +RUN set -e; \ + export DEBIAN_FRONTEND=noninteractive; \ + export DEBCONF_NONINTERACTIVE_SEEN=true; \ + apt-get update; \ + apt-get install -y \ + autoconf \ + automake \ + build-essential \ + ca-certificates \ + cpio \ + git \ + graphviz \ + libedit-dev \ + libjemalloc-dev \ + libncurses-dev \ + libpcre3-dev \ + libtool \ + libunwind-dev \ + pkg-config \ + python3-sphinx diff --git a/.circleci/README.rst b/.circleci/README.rst index 0c52c4235..e19a6485e 100644 --- a/.circleci/README.rst +++ b/.circleci/README.rst @@ -1,3 +1,8 @@ +.. + Copyright (c) 2020 Varnish Software AS + SPDX-License-Identifier: BSD-2-Clause + See LICENSE file for full text of license + Multiarch building, testing & packaging ======================================= diff --git a/.circleci/config.yml b/.circleci/config.yml index 97e0ac01a..056e597ee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,8 +45,7 @@ jobs: make \ pcre-devel \ python3 \ - python-sphinx \ - curl + python-sphinx - checkout - when: condition: << pipeline.parameters.dist-url >> @@ -234,7 +233,7 @@ jobs: command: | docker create --name workspace -v /workspace << parameters.prefix >><< parameters.dist >>:<< parameters.release >> /bin/true docker cp /workspace workspace:/ - docker run --volumes-from workspace -w /workspace --network host << parameters.prefix >><< parameters.dist >>:<< parameters.release >> sh -c ' + docker run --volumes-from workspace -w /workspace << parameters.prefix >><< parameters.dist >>:<< parameters.release >> sh -c ' case "<< parameters.dist >>" in centos|almalinux|fedora) yum groupinstall -y "Development Tools" @@ -387,6 +386,11 @@ workflows: name: build_centos_7 dist: centos release: "7" + - build: + name: build_centos_stream + prefix: quay.io/centos/ + dist: centos + release: stream - build: name: build_almalinux_8 dist: almalinux @@ -395,6 +399,17 @@ workflows: name: build_almalinux_9 dist: almalinux release: "9" + # fedora is our witness + - build: + name: build_fedora_latest + dist: fedora + release: latest + make_target: witness.dot + - build: + name: build_fedora_rawhide + dist: fedora + release: rawhide + # latest debian uses sanitizers - build: name: build_debian_buster dist: debian @@ -407,8 +422,11 @@ workflows: name: build_debian_bookworm dist: debian release: bookworm + extra_conf: --enable-asan --enable-ubsan --enable-workspace-emulator + # oldest ubuntu goes 32bit - build: name: build_ubuntu_bionic + prefix: i386/ dist: ubuntu release: bionic - build: @@ -419,6 +437,15 @@ workflows: name: build_ubuntu_jammy dist: ubuntu release: jammy + - build: + name: build_alpine + dist: alpine + release: "latest" + extra_conf: --without-contrib + - build: + name: build_archlinux + dist: archlinux + release: base-devel packaging: when: *packaging_cond jobs: &packaging_jobs @@ -432,7 +459,6 @@ workflows: matrix: parameters: platform: - - ubuntu:xenial - ubuntu:bionic - ubuntu:focal - ubuntu:jammy @@ -440,6 +466,7 @@ workflows: - debian:bullseye - debian:bookworm - centos:7 + - centos:stream - almalinux:8 - almalinux:9 - fedora:latest From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:09 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:09 +0000 (UTC) Subject: [6.0] 978f453dc Fix .circleci/config.yml to match changes in configure.ac Message-ID: <20240621134309.EA6F51209B8@lists.varnish-cache.org> commit 978f453dcda1de0496c93306cab678b7e96266c6 Author: Simon Stridsberg Date: Fri Oct 6 09:01:41 2023 +0200 Fix .circleci/config.yml to match changes in configure.ac diff --git a/.circleci/config.yml b/.circleci/config.yml index 056e597ee..870f91569 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -81,9 +81,9 @@ jobs: touch .is_weekly # If version is "trunk", override version to add date if grep 'AC_INIT.*trunk.*' ./configure.ac; then - sed -i -e "s/AC_INIT(\[\(.*\)\], \[\(.*\)\], \[\(.*\)\])/AC_INIT([\1], [$(date +%Y%m%d)], [\3])/" ./configure.ac + sed -i -e "s/AC_INIT(\[\(.*\)\],\[\(.*\)\],\[\(.*\)\])/AC_INIT([\1],[$(date +%Y%m%d)],[\3])/" ./configure.ac else - sed -i -e "s/AC_INIT(\[\(.*\)\], \[\(.*\)\], \[\(.*\)\])/AC_INIT([\1], [\2-$(date +%Y%m%d)], [\3])/" ./configure.ac + sed -i -e "s/AC_INIT(\[\(.*\)\],\[\(.*\)\],\[\(.*\)\])/AC_INIT([\1],[\2-$(date +%Y%m%d)],[\3])/" ./configure.ac fi ./autogen.des make dist -j 16 From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:10 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:10 +0000 (UTC) Subject: [6.0] a39968a59 circleci: Remove Fedora Rawhide job Message-ID: <20240621134310.39C051209C0@lists.varnish-cache.org> commit a39968a596354f57328351a5c2df3639251ef176 Author: Dridi Boukelmoune Date: Tue Jan 2 11:34:45 2024 +0100 circleci: Remove Fedora Rawhide job Rawhide is not stable and we already have a job for the latest stable branch. diff --git a/.circleci/config.yml b/.circleci/config.yml index 870f91569..209b7f88d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -405,10 +405,6 @@ workflows: dist: fedora release: latest make_target: witness.dot - - build: - name: build_fedora_rawhide - dist: fedora - release: rawhide # latest debian uses sanitizers - build: name: build_debian_buster From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:10 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:10 +0000 (UTC) Subject: [6.0] efff5c5aa circleci: Update setup_remote_docker version Message-ID: <20240621134310.730841209CF@lists.varnish-cache.org> commit efff5c5aa04865d9a3ebef66788a13e83628a19f Author: Guillaume Quintard Date: Sat Feb 24 15:52:39 2024 +0100 circleci: Update setup_remote_docker version https://discuss.circleci.com/t/remote-docker-image-deprecations-and-eol-for-2024/50176 diff --git a/.circleci/config.yml b/.circleci/config.yml index 209b7f88d..519dadd0b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -222,8 +222,7 @@ jobs: - image: centos:7 working_directory: /workspace steps: - - setup_remote_docker: - version: 20.10.11 + - setup_remote_docker - run: name: Install docker command: yum install -y docker From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:10 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:10 +0000 (UTC) Subject: [6.0] aaea0a48f circleci: Add ubuntu:noble jobs Message-ID: <20240621134310.B1BAC1209E6@lists.varnish-cache.org> commit aaea0a48f712f27ff2bd4b64ba12685873e2f965 Author: Guillaume Quintard Date: Fri Feb 23 22:09:31 2024 +0100 circleci: Add ubuntu:noble jobs diff --git a/.circleci/config.yml b/.circleci/config.yml index 519dadd0b..027120a83 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -432,6 +432,10 @@ workflows: name: build_ubuntu_jammy dist: ubuntu release: jammy + - build: + name: build_ubuntu_noble + dist: ubuntu + release: noble - build: name: build_alpine dist: alpine @@ -457,6 +461,7 @@ workflows: - ubuntu:bionic - ubuntu:focal - ubuntu:jammy + - ubuntu:noble - debian:buster - debian:bullseye - debian:bookworm From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:10 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:10 +0000 (UTC) Subject: [6.0] f3668a3a7 circleci: Swap 32bit/sanitizer responsibilities Message-ID: <20240621134310.E90431209F6@lists.varnish-cache.org> commit f3668a3a77ce07d515545c1c5efea0a47b360420 Author: Guillaume Quintard Date: Wed Feb 28 10:42:28 2024 -0800 circleci: Swap 32bit/sanitizer responsibilities The current debian:latest image still ships a 32bit variant, whereas ubuntu:focal, next in line after ubuntu:bionic, no longer does. diff --git a/.circleci/config.yml b/.circleci/config.yml index 027120a83..36149e80b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -404,11 +404,12 @@ workflows: dist: fedora release: latest make_target: witness.dot - # latest debian uses sanitizers + # oldest debian goes 32bit - build: name: build_debian_buster dist: debian release: buster + prefix: i386/ - build: name: build_debian_bullseye dist: debian @@ -417,11 +418,9 @@ workflows: name: build_debian_bookworm dist: debian release: bookworm - extra_conf: --enable-asan --enable-ubsan --enable-workspace-emulator - # oldest ubuntu goes 32bit + # latest ubuntu uses sanitizers - build: name: build_ubuntu_bionic - prefix: i386/ dist: ubuntu release: bionic - build: @@ -436,6 +435,7 @@ workflows: name: build_ubuntu_noble dist: ubuntu release: noble + extra_conf: --enable-asan --enable-ubsan --enable-workspace-emulator - build: name: build_alpine dist: alpine From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:11 +0000 (UTC) Subject: [6.0] eba586066 circleci: Retire ubuntu:bionic jobs Message-ID: <20240621134311.218AF120A10@lists.varnish-cache.org> commit eba5860669d66dc3150149b7a888939a3ee589f4 Author: Guillaume Quintard Date: Thu Feb 29 09:56:57 2024 -0800 circleci: Retire ubuntu:bionic jobs diff --git a/.circleci/config.yml b/.circleci/config.yml index 36149e80b..cb6587ce7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -208,7 +208,7 @@ jobs: description: the Linux distribution (debian|ubuntu) type: string release: - description: the release name (buster|bullseye|bionic|focal|jammy) + description: the release name (buster|bullseye|bookworm|focal|jammy|noble) type: string make_target: description: the make target to execute during the build @@ -418,11 +418,6 @@ workflows: name: build_debian_bookworm dist: debian release: bookworm - # latest ubuntu uses sanitizers - - build: - name: build_ubuntu_bionic - dist: ubuntu - release: bionic - build: name: build_ubuntu_focal dist: ubuntu @@ -458,7 +453,6 @@ workflows: matrix: parameters: platform: - - ubuntu:bionic - ubuntu:focal - ubuntu:jammy - ubuntu:noble From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:11 +0000 (UTC) Subject: [6.0] 92f44e295 circleci: Reduce concurrency of ubuntu:noble jobs Message-ID: <20240621134311.54C72120A22@lists.varnish-cache.org> commit 92f44e295c5f7692ea00610975cff2f44a79d844 Author: Dridi Boukelmoune Date: Mon Mar 4 18:22:49 2024 +0100 circleci: Reduce concurrency of ubuntu:noble jobs In an attempt to avoid EAGAIN failures on pthread creations with ASAN and UBSAN sanitizers enabled. diff --git a/.circleci/config.yml b/.circleci/config.yml index cb6587ce7..e5f30daa1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -352,7 +352,7 @@ jobs: << parameters.extra_conf >> sudo -u varnish \ --preserve-env=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \ - make << parameters.make_target >> VERBOSE=1 -j 4 -k \ + make -j 4 -k << parameters.make_target >> VERBOSE=1 \ DISTCHECK_CONFIGURE_FLAGS="<< pipeline.parameters.configure_args >> \ << parameters.extra_conf >>" ' @@ -418,6 +418,7 @@ workflows: name: build_debian_bookworm dist: debian release: bookworm + # latest ubuntu uses sanitizers - build: name: build_ubuntu_focal dist: ubuntu @@ -431,6 +432,7 @@ workflows: dist: ubuntu release: noble extra_conf: --enable-asan --enable-ubsan --enable-workspace-emulator + make_target: check -j2 - build: name: build_alpine dist: alpine From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:11 +0000 (UTC) Subject: [6.0] 7f9c9a61f circleci: Only run the test suite on alpine Message-ID: <20240621134311.8AFC5120A29@lists.varnish-cache.org> commit 7f9c9a61fcbfa4c814a46a25905e00bc1577a7bc Author: Dridi Boukelmoune Date: Tue Mar 5 07:47:47 2024 +0100 circleci: Only run the test suite on alpine There appears to be a race condition with distcheck where test-suite.log is removed and then another attempt at removing it fails: [...] test -z "test-suite.log" || rm -f test-suite.log rm -f libtool config.lt find . '(' -name '*.gcda' -o -name '*.gcda' ')' -exec rm '{}' ';' rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags rm -f cscope.out cscope.in.out cscope.po.out cscope.files find: ./test-suite.log: No such file or directory [...] It is unclear which find command is choking on test-suite.log, there doesn't seem to be a make rule running find and targeting this file. Could it be the find command listed in the output collecting file names like test-suite.log and then failing to test it against the -name filters after it got removed? That would be infortunate for a find implementation. Since alpine is not a platform we officially support, and after failing numerous times to reproduce the link outside of the CI environment, a simple make check is good enough and it will remove a lot of noise. diff --git a/.circleci/config.yml b/.circleci/config.yml index e5f30daa1..088c2aca5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -438,6 +438,7 @@ workflows: dist: alpine release: "latest" extra_conf: --without-contrib + make_target: check - build: name: build_archlinux dist: archlinux From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:11 +0000 (UTC) Subject: [6.0] 31c4e87e3 Run docker without seccomp rules. Message-ID: <20240621134311.B4C31120A3D@lists.varnish-cache.org> commit 31c4e87e356aa47f2259217486327bb993cc6038 Author: Simon Stridsberg Date: Fri Mar 15 08:08:31 2024 +0100 Run docker without seccomp rules. Ubuntu noble tries to use `fchmodat2` (new syscall) and gets permission denied instead of ENOSYS. This is a small security risk but it's running inside of circleci containers anyway so i think its acceptable. diff --git a/.circleci/config.yml b/.circleci/config.yml index 088c2aca5..a9b6c5cc4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -185,6 +185,7 @@ jobs: docker run \ --rm \ -it \ + --security-opt seccomp=unconfined \ -e PARAM_DIST=$(echo "<< parameters.platform >>" | cut -d: -f1) \ -e PARAM_RELEASE=$(echo "<< parameters.platform >>" | cut -d: -f2) \ -v$(pwd):/varnish-cache \ From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:11 +0000 (UTC) Subject: [6.0] 396fed02a circleci: Tolerate spaces in AC_INIT() Message-ID: <20240621134312.0DD24120A55@lists.varnish-cache.org> commit 396fed02a35860539646ebd82e664eb9a587867f Author: Dridi Boukelmoune Date: Tue Mar 19 10:19:58 2024 +0100 circleci: Tolerate spaces in AC_INIT() Better diff with the --word-diff --word-diff-regex=. options. diff --git a/.circleci/config.yml b/.circleci/config.yml index a9b6c5cc4..4c4d8e8d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -81,9 +81,9 @@ jobs: touch .is_weekly # If version is "trunk", override version to add date if grep 'AC_INIT.*trunk.*' ./configure.ac; then - sed -i -e "s/AC_INIT(\[\(.*\)\],\[\(.*\)\],\[\(.*\)\])/AC_INIT([\1],[$(date +%Y%m%d)],[\3])/" ./configure.ac + sed -i -e "s/AC_INIT(\[\(.*\)\],\s*\[\(.*\)\],\s*\[\(.*\)\])/AC_INIT([\1],[$(date +%Y%m%d)],[\3])/" ./configure.ac else - sed -i -e "s/AC_INIT(\[\(.*\)\],\[\(.*\)\],\[\(.*\)\])/AC_INIT([\1],[\2-$(date +%Y%m%d)],[\3])/" ./configure.ac + sed -i -e "s/AC_INIT(\[\(.*\)\],\s*\[\(.*\)\],\s*\[\(.*\)\])/AC_INIT([\1],[\2-$(date +%Y%m%d)],[\3])/" ./configure.ac fi ./autogen.des make dist -j 16 From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:12 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:12 +0000 (UTC) Subject: [6.0] fcabd46b6 circleci: Switch centos image to stream9 tag Message-ID: <20240621134312.2DA29120A64@lists.varnish-cache.org> commit fcabd46b69b10e4aeed26c9dba1fc516e09e323d Author: Dridi Boukelmoune Date: Mon Jun 3 11:03:03 2024 +0200 circleci: Switch centos image to stream9 tag The plain "stream" image has not been updated for a year, and instead of being the equivalent of a "latest" tag it just appears to be frozen. Since there isn't an automatic centos-stream upgrade path, let's target version 9 explicitly for now. diff --git a/.circleci/config.yml b/.circleci/config.yml index 4c4d8e8d7..1366bfb22 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -390,7 +390,7 @@ workflows: name: build_centos_stream prefix: quay.io/centos/ dist: centos - release: stream + release: stream9 - build: name: build_almalinux_8 dist: almalinux From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:12 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:12 +0000 (UTC) Subject: [6.0] 6171eb303 circleci: Missing nines for centos:stream9 Message-ID: <20240621134312.5BD15120A72@lists.varnish-cache.org> commit 6171eb303cd0b0cc81770c4a4e6f3f9437af9918 Author: Dridi Boukelmoune Date: Mon Jun 3 11:21:15 2024 +0200 circleci: Missing nines for centos:stream9 I don't understand why they didn't maintain the unversioned centos:stream tag as a kind of latest tag after dropping the actual centos:latest tag. diff --git a/.circleci/config.yml b/.circleci/config.yml index 1366bfb22..32fb5faa4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -173,7 +173,7 @@ jobs: esac case "<< parameters.platform >>" in - centos:stream) REPO=quay.io/centos/ ;; + centos:stream9) REPO=quay.io/centos/ ;; *) REPO= ;; esac @@ -238,13 +238,13 @@ jobs: centos|almalinux|fedora) yum groupinstall -y "Development Tools" case "<< parameters.dist >>:<< parameters.release >>" in - almalinux:9) + centos:stream9|almalinux:9) dnf install -y "dnf-command(config-manager)" yum config-manager --set-enabled crb yum install -y diffutils yum install -y epel-release ;; - centos:stream|almalinux:8) + almalinux:8) dnf install -y "dnf-command(config-manager)" yum config-manager --set-enabled powertools yum install -y diffutils @@ -387,7 +387,7 @@ workflows: dist: centos release: "7" - build: - name: build_centos_stream + name: build_centos_stream_9 prefix: quay.io/centos/ dist: centos release: stream9 @@ -464,7 +464,7 @@ workflows: - debian:bullseye - debian:bookworm - centos:7 - - centos:stream + - centos:stream9 - almalinux:8 - almalinux:9 - fedora:latest From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:12 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:12 +0000 (UTC) Subject: [6.0] e73d61637 circleci: Run sanitizers on a larger system Message-ID: <20240621134312.89831120A86@lists.varnish-cache.org> commit e73d61637a3222aee9f1edae137ab43d1e6a2884 Author: Dridi Boukelmoune Date: Thu Jun 6 11:10:13 2024 +0200 circleci: Run sanitizers on a larger system This is an attempt at speeding up this job that usually takes three time as long as other distcheck jobs. diff --git a/.circleci/config.yml b/.circleci/config.yml index 32fb5faa4..c7db7a47b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -126,7 +126,7 @@ jobs: description: the Linux distribution, with release, e.g. debian:buster, centos:7 type: string rclass: - description: the resource class to use, usuall arm.medium or medium + description: the resource class to use, usually arm.medium or medium type: string machine: image: ubuntu-2004:202111-02 @@ -219,8 +219,13 @@ jobs: description: platform-specific configure arguments default: "" type: string + rclass: + description: job-specific resource class + default: medium + type: string docker: - image: centos:7 + resource_class: << parameters.rclass >> working_directory: /workspace steps: - setup_remote_docker @@ -432,8 +437,9 @@ workflows: name: build_ubuntu_noble dist: ubuntu release: noble + rclass: large extra_conf: --enable-asan --enable-ubsan --enable-workspace-emulator - make_target: check -j2 + make_target: check - build: name: build_alpine dist: alpine From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:12 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:12 +0000 (UTC) Subject: [6.0] 8698c3355 circleci: Retire CentOS jobs Message-ID: <20240621134312.A5C0A120A9B@lists.varnish-cache.org> commit 8698c33550a3efe941435354aff29bf70df0be1d Author: Dridi Boukelmoune Date: Tue Jun 11 16:34:25 2024 +0200 circleci: Retire CentOS jobs With CentOS reaching EOL at the end of the month and CentOS Stream 9 flaky packaging jobs, we don't need to keep them around. For el9 builds we rely on the almalinux:9 image. diff --git a/.circleci/config.yml b/.circleci/config.yml index c7db7a47b..f3ffa3e70 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -123,7 +123,7 @@ jobs: package: parameters: platform: - description: the Linux distribution, with release, e.g. debian:buster, centos:7 + description: the Linux distribution, with release, e.g. debian:buster, fedora:latest type: string rclass: description: the resource class to use, usually arm.medium or medium @@ -163,20 +163,14 @@ jobs: command: | mkdir -p packages case "<< parameters.platform >>" in - debian:*|ubuntu:*) EXT=deb ;; - centos:*|fedora:*) EXT=rpm ;; - almalinux:*) EXT=rpm ;; - alpine:*) EXT=apk ;; + debian:*|ubuntu:*) EXT=deb ;; + almalinux:*|fedora:*) EXT=rpm ;; + alpine:*) EXT=apk ;; *) echo "unrecognized platform: << parameters.platform >>" exit 1 esac - case "<< parameters.platform >>" in - centos:stream9) REPO=quay.io/centos/ ;; - *) REPO= ;; - esac - case "<< parameters.rclass >>" in arm.*) ARCH=arm64 ;; *) ARCH=amd64 ;; @@ -190,7 +184,7 @@ jobs: -e PARAM_RELEASE=$(echo "<< parameters.platform >>" | cut -d: -f2) \ -v$(pwd):/varnish-cache \ --platform linux/$ARCH \ - ${REPO}<< parameters.platform >> \ + << parameters.platform >> \ /varnish-cache/.circleci/make-$EXT-packages.sh - run: name: List created packages @@ -240,26 +234,23 @@ jobs: docker cp /workspace workspace:/ docker run --volumes-from workspace -w /workspace << parameters.prefix >><< parameters.dist >>:<< parameters.release >> sh -c ' case "<< parameters.dist >>" in - centos|almalinux|fedora) - yum groupinstall -y "Development Tools" + almalinux|fedora) + dnf -y groupinstall "Development Tools" case "<< parameters.dist >>:<< parameters.release >>" in - centos:stream9|almalinux:9) - dnf install -y "dnf-command(config-manager)" - yum config-manager --set-enabled crb - yum install -y diffutils - yum install -y epel-release + almalinux:9) + dnf -y install "dnf-command(config-manager)" + dnf config-manager --set-enabled crb + dnf -y install diffutils + dnf -y install epel-release ;; almalinux:8) - dnf install -y "dnf-command(config-manager)" - yum config-manager --set-enabled powertools - yum install -y diffutils - yum install -y epel-release - ;; - centos:7) - yum install -y epel-release + dnf -y install "dnf-command(config-manager)" + dnf config-manager --set-enabled powertools + dnf -y install diffutils + dnf -y install epel-release ;; esac - yum install -y \ + dnf -y install \ cpio \ automake \ git \ @@ -335,7 +326,7 @@ jobs: archlinux) useradd varnish ;; - centos|almalinux|fedora) + almalinux|fedora) adduser varnish ;; *) @@ -387,15 +378,6 @@ workflows: - << pipeline.parameters.build-pkgs >> - << pipeline.parameters.dist-url >> jobs: - - build: - name: build_centos_7 - dist: centos - release: "7" - - build: - name: build_centos_stream_9 - prefix: quay.io/centos/ - dist: centos - release: stream9 - build: name: build_almalinux_8 dist: almalinux @@ -469,8 +451,6 @@ workflows: - debian:buster - debian:bullseye - debian:bookworm - - centos:7 - - centos:stream9 - almalinux:8 - almalinux:9 - fedora:latest diff --git a/.circleci/make-rpm-packages.sh b/.circleci/make-rpm-packages.sh index 848084bc4..4a9f8a142 100755 --- a/.circleci/make-rpm-packages.sh +++ b/.circleci/make-rpm-packages.sh @@ -6,30 +6,27 @@ echo "PARAM_RELEASE: $PARAM_RELEASE" echo "PARAM_DIST: $PARAM_DIST" if [ -z "$PARAM_RELEASE" ]; then - echo "Env variable PARAM_RELEASE is not set! For example PARAM_RELEASE=stream, for CentOS stream" + echo "Env variable PARAM_RELEASE is not set! For example PARAM_RELEASE=9, for almalinux" exit 1 elif [ -z "$PARAM_DIST" ]; then - echo "Env variable PARAM_DIST is not set! For example PARAM_DIST=centos" + echo "Env variable PARAM_DIST is not set! For example PARAM_DIST=fedora" exit 1 fi case "$PARAM_DIST:$PARAM_RELEASE" in almalinux:9) - dnf install -y 'dnf-command(config-manager)' - yum config-manager --set-enabled crb - yum install -y epel-release + dnf -y install 'dnf-command(config-manager)' + dnf config-manager --set-enabled crb + dnf -y install epel-release ;; - centos:stream|almalinux:8) - dnf install -y 'dnf-command(config-manager)' - yum config-manager --set-enabled powertools - yum install -y epel-release - ;; - centos:7) - yum install -y epel-release + almalinux:8) + dnf -y install 'dnf-command(config-manager)' + dnf config-manager --set-enabled powertools + dnf -y install epel-release ;; esac -yum install -y rpm-build yum-utils +dnf -y install rpm-build dnf-utils export DIST_DIR=build @@ -73,7 +70,7 @@ rpmbuild() { "$@" } -yum-builddep -y "$DIST_DIR"/redhat/varnish.spec +dnf builddep -y "$DIST_DIR"/redhat/varnish.spec rpmbuild -bs "$DIST_DIR"/redhat/varnish.spec rpmbuild --rebuild "$RESULT_DIR"/varnish-*.src.rpm From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:12 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:12 +0000 (UTC) Subject: [6.0] e5c4aab32 circleci: Retire Debian Buster jobs Message-ID: <20240621134312.C4EE6120AA8@lists.varnish-cache.org> commit e5c4aab32083e28e02be637c0b7ac9cc83720354 Author: Dridi Boukelmoune Date: Tue Jun 11 17:36:52 2024 +0200 circleci: Retire Debian Buster jobs It reached EOL almost 2 years ago, but the architectures we cover in CI had 2 extra years of long term support that is coming to an end this month. diff --git a/.circleci/config.yml b/.circleci/config.yml index f3ffa3e70..39d1f3290 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -123,7 +123,7 @@ jobs: package: parameters: platform: - description: the Linux distribution, with release, e.g. debian:buster, fedora:latest + description: the Linux distribution, with release, e.g. debian:bookworm, fedora:latest type: string rclass: description: the resource class to use, usually arm.medium or medium @@ -203,7 +203,7 @@ jobs: description: the Linux distribution (debian|ubuntu) type: string release: - description: the release name (buster|bullseye|bookworm|focal|jammy|noble) + description: the release name (bullseye|bookworm|focal|jammy|noble) type: string make_target: description: the make target to execute during the build @@ -393,15 +393,11 @@ workflows: release: latest make_target: witness.dot # oldest debian goes 32bit - - build: - name: build_debian_buster - dist: debian - release: buster - prefix: i386/ - build: name: build_debian_bullseye dist: debian release: bullseye + prefix: i386/ - build: name: build_debian_bookworm dist: debian @@ -448,7 +444,6 @@ workflows: - ubuntu:focal - ubuntu:jammy - ubuntu:noble - - debian:buster - debian:bullseye - debian:bookworm - almalinux:8 From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:12 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:12 +0000 (UTC) Subject: [6.0] 20be89e72 circleci: Align Alpine jobs to alpine:latest Message-ID: <20240621134313.0803B120ABA@lists.varnish-cache.org> commit 20be89e726e88c2481bebbdb72e79079406eb137 Author: Dridi Boukelmoune Date: Tue Jun 11 17:39:06 2024 +0200 circleci: Align Alpine jobs to alpine:latest diff --git a/.circleci/config.yml b/.circleci/config.yml index 39d1f3290..8870b2d68 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -421,7 +421,7 @@ workflows: - build: name: build_alpine dist: alpine - release: "latest" + release: latest extra_conf: --without-contrib make_target: check - build: @@ -449,7 +449,7 @@ workflows: - almalinux:8 - almalinux:9 - fedora:latest - - alpine:3 + - alpine:latest rclass: - arm.medium - medium From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:13 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:13 +0000 (UTC) Subject: [6.0] 8484cbc22 circleci: Move away from centos:7 as a base image Message-ID: <20240621134313.41F10120AD6@lists.varnish-cache.org> commit 8484cbc22ec6a7cbc563a7923a8dc710d80e0b19 Author: Dridi Boukelmoune Date: Tue Jun 11 16:28:49 2024 +0200 circleci: Move away from centos:7 as a base image For the build jobs we can directly use the target images and avoid the nested docker invocation. For other jobs or Dockerfiles where centos:7 is used we can switch to a self-updating fedora:latest image. Better diff with the --ignore-all-space option. diff --git a/.circleci/Dockerfile b/.circleci/Dockerfile index 7553611e0..c8cd03af5 100644 --- a/.circleci/Dockerfile +++ b/.circleci/Dockerfile @@ -1,8 +1,7 @@ -FROM centos:7 +FROM fedora-latest -RUN set -e;\ - yum install -y epel-release; \ - yum install -y \ +RUN set -e; \ + dnf -y install \ automake \ git \ jemalloc-devel \ @@ -12,4 +11,4 @@ RUN set -e;\ make \ pcre-devel \ python3 \ - python-sphinx + python3-sphinx diff --git a/.circleci/config.yml b/.circleci/config.yml index 8870b2d68..e473336f3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,13 +29,12 @@ jobs: dist: description: Build or download varnish-x.y.z.tar.gz that is used later for the packaging jobs docker: - - image: centos:7 + - image: fedora:latest steps: - run: name: Install deps command: | - yum install -y epel-release - yum install -y \ + dnf -y install \ automake \ jemalloc-devel \ git \ @@ -45,7 +44,7 @@ jobs: make \ pcre-devel \ python3 \ - python-sphinx + python3-sphinx - checkout - when: condition: << pipeline.parameters.dist-url >> @@ -97,7 +96,7 @@ jobs: tar_pkg_tools: description: Builds archives with the packaging tools from https://github.com/varnishcache/pkg-varnish-cache docker: - - image: centos:7 + - image: fedora:latest steps: - add_ssh_keys: fingerprints: @@ -105,7 +104,7 @@ jobs: - run: name: Grab the pkg repo command: | - yum install -y git + dnf -y install git mkdir -p ~/.ssh ssh-keyscan -H github.com >> ~/.ssh/known_hosts echo ${CIRCLE_REPOSITORY_URL} @@ -218,21 +217,14 @@ jobs: default: medium type: string docker: - - image: centos:7 + - image: << parameters.prefix >><< parameters.dist >>:<< parameters.release >> resource_class: << parameters.rclass >> working_directory: /workspace steps: - - setup_remote_docker - - run: - name: Install docker - command: yum install -y docker - checkout - run: name: Extract and build command: | - docker create --name workspace -v /workspace << parameters.prefix >><< parameters.dist >>:<< parameters.release >> /bin/true - docker cp /workspace workspace:/ - docker run --volumes-from workspace -w /workspace << parameters.prefix >><< parameters.dist >>:<< parameters.release >> sh -c ' case "<< parameters.dist >>" in almalinux|fedora) dnf -y groupinstall "Development Tools" @@ -336,27 +328,23 @@ jobs: chown -R varnish:varnish . - export ASAN_OPTIONS=abort_on_error=1,detect_odr_violation=1,detect_leaks=1,detect_stack_use_after_return=1,detect_invalid_pointer_pairs=1,handle_segv=0,handle_sigbus=0,use_sigaltstack=0,disable_coredump=0 - export LSAN_OPTIONS=abort_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/lsan.suppr - export TSAN_OPTIONS=abort_on_error=1,halt_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/tsan.suppr - export UBSAN_OPTIONS=halt_on_error=1,print_stacktrace=1,use_sigaltstack=0,suppressions=$(pwd)/tools/ubsan.suppr - - sudo -u varnish \ + sudo -u varnish sh -c ' + export ASAN_OPTIONS=abort_on_error=1,detect_odr_violation=1,detect_leaks=1,detect_stack_use_after_return=1,detect_invalid_pointer_pairs=1,handle_segv=0,handle_sigbus=0,use_sigaltstack=0,disable_coredump=0 + export LSAN_OPTIONS=abort_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/lsan.suppr + export TSAN_OPTIONS=abort_on_error=1,halt_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/tsan.suppr + export UBSAN_OPTIONS=halt_on_error=1,print_stacktrace=1,use_sigaltstack=0,suppressions=$(pwd)/tools/ubsan.suppr autoreconf -i -v - sudo -u varnish \ ./configure \ - << pipeline.parameters.configure_args >> \ - << parameters.extra_conf >> - sudo -u varnish \ - --preserve-env=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \ + << pipeline.parameters.configure_args >> \ + << parameters.extra_conf >> make -j 4 -k << parameters.make_target >> VERBOSE=1 \ - DISTCHECK_CONFIGURE_FLAGS="<< pipeline.parameters.configure_args >> \ + DISTCHECK_CONFIGURE_FLAGS="<< pipeline.parameters.configure_args >> \ << parameters.extra_conf >>" ' collect_packages: docker: - - image: centos:7 + - image: fedora:latest steps: - attach_workspace: at: ~/project From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:13 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:13 +0000 (UTC) Subject: [6.0] 62fdc0986 circleci: Avoid dashes in package version numbers Message-ID: <20240621134313.5F6D0120AE4@lists.varnish-cache.org> commit 62fdc09865e1c87503ceccd3fa34cc61298e2265 Author: Dridi Boukelmoune Date: Thu Jun 20 19:01:12 2024 +0200 circleci: Avoid dashes in package version numbers This wouldn't trigger on the master branch on account of having the version trunk. diff --git a/.circleci/config.yml b/.circleci/config.yml index e473336f3..9476ca00d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,7 +82,7 @@ jobs: if grep 'AC_INIT.*trunk.*' ./configure.ac; then sed -i -e "s/AC_INIT(\[\(.*\)\],\s*\[\(.*\)\],\s*\[\(.*\)\])/AC_INIT([\1],[$(date +%Y%m%d)],[\3])/" ./configure.ac else - sed -i -e "s/AC_INIT(\[\(.*\)\],\s*\[\(.*\)\],\s*\[\(.*\)\])/AC_INIT([\1],[\2-$(date +%Y%m%d)],[\3])/" ./configure.ac + sed -i -e "s/AC_INIT(\[\(.*\)\],\s*\[\(.*\)\],\s*\[\(.*\)\])/AC_INIT([\1],[\2.$(date +%Y%m%d)],[\3])/" ./configure.ac fi ./autogen.des make dist -j 16 From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:13 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:13 +0000 (UTC) Subject: [6.0] f0d0ab75f circleci: Disable sanitizer coverage Message-ID: <20240621134313.820B5120AEB@lists.varnish-cache.org> commit f0d0ab75fa876a570ca1c157fdbb113a40e868cf Author: Dridi Boukelmoune Date: Wed Jun 19 19:38:49 2024 +0200 circleci: Disable sanitizer coverage It turns out there was no sanitizer coverage before catching up with the circleci changes from #4119 and there are leaks that may legitimately fail the build. Until we get a chance to work on those leaks, we adopt status quo and return to the previous (lack of) coverage. diff --git a/.circleci/config.yml b/.circleci/config.yml index 9476ca00d..d6424cac5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -403,8 +403,8 @@ workflows: name: build_ubuntu_noble dist: ubuntu release: noble - rclass: large - extra_conf: --enable-asan --enable-ubsan --enable-workspace-emulator + # not actionable: rclass: large + # not actionable: extra_conf: --enable-asan --enable-ubsan make_target: check - build: name: build_alpine From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:13 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:13 +0000 (UTC) Subject: [6.0] f051dddd0 vtc: Do not resolve localhost in test cases Message-ID: <20240621134313.BF52F120AFD@lists.varnish-cache.org> commit f051dddd06ed75bb8471116cd5d633f84f3e63f2 Author: Dridi Boukelmoune Date: Wed Jun 19 09:58:50 2024 +0200 vtc: Do not resolve localhost in test cases diff --git a/bin/varnishtest/tests/c00003.vtc b/bin/varnishtest/tests/c00003.vtc index 1c369bb07..5028313ab 100644 --- a/bin/varnishtest/tests/c00003.vtc +++ b/bin/varnishtest/tests/c00003.vtc @@ -3,11 +3,11 @@ varnishtest "Check that we fail to start with erroneous -a/-b arguments" # Duplicate -a arguments # XXX: this cannot be tested reliably, we tolerate port collision shell -err -match "have same address|already in use" { - varnishd -d -a 127.0.0.1:38484 -a 127.0.0.1:38484 -b localhost:80 + varnishd -d -a 127.0.0.1:38484 -a 127.0.0.1:38484 -b ${localhost}:80 } shell -err -match "have same address|already in use" { - varnishd -d -a ${tmpdir}/vtc.sock -a ${tmpdir}/vtc.sock -b localhost:80 + varnishd -d -a ${tmpdir}/vtc.sock -a ${tmpdir}/vtc.sock -b ${localhost}:80 } # -a bad protocol specs diff --git a/bin/varnishtest/tests/s00005.vtc b/bin/varnishtest/tests/s00005.vtc index 4dac2f167..37f56a2bc 100644 --- a/bin/varnishtest/tests/s00005.vtc +++ b/bin/varnishtest/tests/s00005.vtc @@ -8,9 +8,9 @@ server s1 { varnish v1 -vcl+backend {} # VCL name must be C-names -varnish v1 -clierr 106 {vcl.inline 0000 "vcl 4.0; backend b { .host = \"localhost\";} "} -varnish v1 -clierr 106 {vcl.inline a00/ "vcl 4.0; backend b { .host = \"localhost\";} "} -varnish v1 -clierr 106 {vcl.inline a00? "vcl 4.0; backend b { .host = \"localhost\";} "} +varnish v1 -clierr 106 {vcl.inline 0000 "vcl 4.0; backend b { .host = \"${localhost}\";} "} +varnish v1 -clierr 106 {vcl.inline a00/ "vcl 4.0; backend b { .host = \"${localhost}\";} "} +varnish v1 -clierr 106 {vcl.inline a00? "vcl 4.0; backend b { .host = \"${localhost}\";} "} varnish v1 -vcl+backend { sub vcl_recv { diff --git a/bin/varnishtest/tests/v00049.vtc b/bin/varnishtest/tests/v00049.vtc index 53e2cb705..c2d497109 100644 --- a/bin/varnishtest/tests/v00049.vtc +++ b/bin/varnishtest/tests/v00049.vtc @@ -17,11 +17,11 @@ varnish v1 -syntax 4.0 -errvcl "VCL version 9.9 not supported" { } varnish v1 -cliexpect {Don't play silly buggers with VCL version numbers} \ - {vcl.inline t0 "vcl 4.00 ; backend b { .host = \"localhost\";} "} + {vcl.inline t0 "vcl 4.00 ; backend b { .host = \"${localhost}\";} "} varnish v1 -cliexpect {Don't play silly buggers with VCL version numbers} \ - {vcl.inline t1 "vcl 04.0 ; backend b { .host = \"localhost\";} "} + {vcl.inline t1 "vcl 04.0 ; backend b { .host = \"${localhost}\";} "} varnish v1 -cliexpect {Expected 'vcl N.N;' found no semi-colon} \ - {vcl.inline t2 "vcl 4.0 backend b { .host = \"localhost\";} "} + {vcl.inline t2 "vcl 4.0 backend b { .host = \"${localhost}\";} "} From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:13 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:13 +0000 (UTC) Subject: [6.0] 16ce13b69 Accommodate the musl libc in c3.vtc Message-ID: <20240621134313.DD1CA120B05@lists.varnish-cache.org> commit 16ce13b6987a3152efb30b654b3c4e1b8d816680 Author: Dridi Boukelmoune Date: Mon May 18 22:21:11 2020 +0200 Accommodate the musl libc in c3.vtc Conflicts: bin/varnishtest/tests/c00003.vtc diff --git a/bin/varnishtest/tests/c00003.vtc b/bin/varnishtest/tests/c00003.vtc index 5028313ab..759b5e3a8 100644 --- a/bin/varnishtest/tests/c00003.vtc +++ b/bin/varnishtest/tests/c00003.vtc @@ -2,11 +2,11 @@ varnishtest "Check that we fail to start with erroneous -a/-b arguments" # Duplicate -a arguments # XXX: this cannot be tested reliably, we tolerate port collision -shell -err -match "have same address|already in use" { +shell -err -match "have same address|already in use|Address in use" { varnishd -d -a 127.0.0.1:38484 -a 127.0.0.1:38484 -b ${localhost}:80 } -shell -err -match "have same address|already in use" { +shell -err -match "have same address|already in use|Address in use" { varnishd -d -a ${tmpdir}/vtc.sock -a ${tmpdir}/vtc.sock -b ${localhost}:80 } From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:14 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:14 +0000 (UTC) Subject: [6.0] 95fecbede stabilize test: do not expect exact byte counts Message-ID: <20240621134314.09A23120B16@lists.varnish-cache.org> commit 95fecbedec516bffbc0c84cadf635acce33f8c31 Author: Nils Goroll Date: Sat Apr 4 13:58:12 2020 +0200 stabilize test: do not expect exact byte counts Across the platforms we support, the overhead of stevedore allocations varies slightly due to different sizes of our structs. diff --git a/bin/varnishtest/tests/v00064.vtc b/bin/varnishtest/tests/v00064.vtc index 8a608e38e..080029b4d 100644 --- a/bin/varnishtest/tests/v00064.vtc +++ b/bin/varnishtest/tests/v00064.vtc @@ -44,7 +44,7 @@ client c1 { delay .1 varnish v1 -expect SM?.Transient.g_bytes > 1048000 -varnish v1 -expect SM?.Transient.g_space == 8 +varnish v1 -expect SM?.Transient.g_space < 50 client c1 { # Fill malloc @@ -56,7 +56,7 @@ client c1 { } -run varnish v1 -expect SM?.s0.g_bytes > 1048000 -varnish v1 -expect SM?.s0.g_space == 4 +varnish v1 -expect SM?.s0.g_space < 50 client c1 { # Check that Varnish is still alive From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:14 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:14 +0000 (UTC) Subject: [6.0] 2a28faac3 vtc: Skip u12 on Alpine Linux Message-ID: <20240621134314.34FA8120B2F@lists.varnish-cache.org> commit 2a28faac393cd77a562e74cda354159387d4b680 Author: Dridi Boukelmoune Date: Wed Jun 19 17:45:04 2024 +0200 vtc: Skip u12 on Alpine Linux I'm not sure why this test is suddenly failing on circleci, but it used to pass when build jobs were running docker in docker, and it passes for me in an alpine:latest container. diff --git a/bin/varnishtest/tests/u00012.vtc b/bin/varnishtest/tests/u00012.vtc index e9822213a..6c467c887 100644 --- a/bin/varnishtest/tests/u00012.vtc +++ b/bin/varnishtest/tests/u00012.vtc @@ -1,5 +1,7 @@ varnishtest "trivial run of varnishadm in pass mode" +feature cmd "! grep -q ID=alpine /etc/os-release" + server s1 { rxreq txresp From dridi.boukelmoune at gmail.com Fri Jun 21 13:43:14 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 21 Jun 2024 13:43:14 +0000 (UTC) Subject: [6.0] 4833a4f18 vtc: Work around 32bit workspace alignment in v58 Message-ID: <20240621134314.69452120B3C@lists.varnish-cache.org> commit 4833a4f1829e6c8d5bcb81ba97fe7543279c5e40 Author: Dridi Boukelmoune Date: Thu Jun 20 12:32:20 2024 +0200 vtc: Work around 32bit workspace alignment in v58 diff --git a/bin/varnishtest/tests/v00058.vtc b/bin/varnishtest/tests/v00058.vtc index 1a3f9cbd8..748c48df0 100644 --- a/bin/varnishtest/tests/v00058.vtc +++ b/bin/varnishtest/tests/v00058.vtc @@ -139,7 +139,7 @@ varnish v1 -vcl+backend { set req.http.Baz = "baz"; set req.http.Quux = "quux"; - vtc.workspace_alloc(client, -12); + vtc.workspace_alloc(client, -11); if (req.url == "/1") { # VRT_StrandsWS() marks the WS as overflowed, From dridi.boukelmoune at gmail.com Mon Jun 24 09:49:06 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 24 Jun 2024 09:49:06 +0000 (UTC) Subject: [master] ef5aa1028 doc: Adapt build from source instructions for el9 Message-ID: <20240624094906.43511AA822@lists.varnish-cache.org> commit ef5aa1028894d8198515f7bc2adee5adb6e86448 Author: Dridi Boukelmoune Date: Mon Jun 24 11:28:41 2024 +0200 doc: Adapt build from source instructions for el9 The libunwind and jemalloc packages appear to be gone, and the infamous PowerTools repository is also gone, but we find Sphinx in the new CRB repository instead. Probably because this landscape needed a little more complications. diff --git a/doc/sphinx/installation/install_source.rst b/doc/sphinx/installation/install_source.rst index 83c65c321..82bec51a9 100644 --- a/doc/sphinx/installation/install_source.rst +++ b/doc/sphinx/installation/install_source.rst @@ -90,29 +90,45 @@ in the following shell commands, replace ``sudo yum install`` if needed. Install sphinx -* On Red Hat / CentOS 8, sphinx is not included in the default - repositories, so execute these steps to include it from the - powertools repository:: +* On Red Hat / CentOS 9, some packages are not included in the + default repositories, so execute these steps to include them + from the CRB repository:: sudo dnf install -y 'dnf-command(config-manager)' - sudo yum config-manager --set-enabled powertools - sudo yum install -y diffutils python3-sphinx + sudo dnf config-manager --set-enabled crb + sudo dnf install -y \ + diffutils \ + python3-sphinx -* On Red Hat / CentOS <= 7, install sphinx:: +* On Red Hat / CentOS 8, some packages are not included in the + default repositories, so execute these steps to include them + from the powertools repository:: - sudo yum install -y python-sphinx + sudo dnf install -y 'dnf-command(config-manager)' + sudo dnf config-manager --set-enabled powertools + sudo dnf install -y \ + diffutils \ + jemalloc-devel \ + libunwind-devel \ + python3-sphinx + +* On Red Hat / CentOS <= 7, install the packages from the default + repository:: + + sudo yum install -y \ + jemalloc-devel \ + libunwind-devel \ + python-sphinx -The following step should conclude installation of the required -packages:: +The following step should conclude installation of the remaining +package dependencies:: yum install -y \ make \ autoconf \ automake \ - jemalloc-devel \ libedit-devel \ libtool \ - libunwind-devel \ ncurses-devel \ pcre2-devel \ pkgconfig \ From dridi.boukelmoune at gmail.com Mon Jun 24 15:29:05 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 24 Jun 2024 15:29:05 +0000 (UTC) Subject: [master] 23ec1fc3f vtc: Use a space as the host:port separator in b58 Message-ID: <20240624152905.99FADC00BC@lists.varnish-cache.org> commit 23ec1fc3fb1198486350e1aedceb5b363d6e429c Author: Dridi Boukelmoune Date: Mon Jun 24 17:27:06 2024 +0200 vtc: Use a space as the host:port separator in b58 If ${localhost} resolves to an IPv6 address the colon separator becomes ambiguous. diff --git a/bin/varnishtest/tests/b00058.vtc b/bin/varnishtest/tests/b00058.vtc index 0b9afb73e..034727395 100644 --- a/bin/varnishtest/tests/b00058.vtc +++ b/bin/varnishtest/tests/b00058.vtc @@ -5,8 +5,8 @@ feature cmd {getent hosts localhost && getent services http} varnish v1 -arg "-p vcc_feature=-err_unref" -vcl { backend b1 {.host = "127.0.0.1";} backend b2 {.host = "[::1]:8080";} - backend b3 {.host = "${localhost}:8081";} - backend b4 {.host = "${localhost}:http";} + backend b3 {.host = "${localhost} 8081";} + backend b4 {.host = "${localhost} http";} backend b5 {.host = "127.0.0.1";.port = "8081";} backend b6 {.host = "127.0.0.1";.port = "http";} } From phk at FreeBSD.org Tue Jun 25 05:19:05 2024 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 25 Jun 2024 05:19:05 +0000 (UTC) Subject: [master] b083216c4 Fix documentation error. Message-ID: <20240625051905.F3AD6112F7C@lists.varnish-cache.org> commit b083216c485597e8873a85bc4b7d857212a73039 Author: Poul-Henning Kamp Date: Tue Jun 25 05:18:28 2024 +0000 Fix documentation error. Spotted by: swojcik at irc diff --git a/doc/sphinx/reference/vcl-backend.rst b/doc/sphinx/reference/vcl-backend.rst index 97bb681c3..de174fb79 100644 --- a/doc/sphinx/reference/vcl-backend.rst +++ b/doc/sphinx/reference/vcl-backend.rst @@ -93,7 +93,7 @@ Attribute ``.host_header`` A host header to add to probes and regular backend requests if they have no such header:: - .host_header = "Host: example.com"; + .host_header = "example.com"; Timeout Attributes ------------------ From phk at FreeBSD.org Tue Jun 25 06:26:05 2024 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 25 Jun 2024 06:26:05 +0000 (UTC) Subject: [master] b37f7e922 More correct typing for Coverity. Message-ID: <20240625062605.24E8F115737@lists.varnish-cache.org> commit b37f7e92297bbeb5a55b0404479046101b993261 Author: Poul-Henning Kamp Date: Tue Jun 25 06:23:34 2024 +0000 More correct typing for Coverity. diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index d54e86e10..fa6d78084 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -169,7 +169,7 @@ http_write(const struct http *hp, int lvl, } static int -get_bytes(const struct http *hp, char *buf, int n) +get_bytes(const struct http *hp, char *buf, size_t n) { int i; struct pollfd pfd[1]; @@ -197,7 +197,7 @@ get_bytes(const struct http *hp, char *buf, int n) i = read(hp->sess->fd, buf, n); if (!(pfd[0].revents & POLLIN)) vtc_log(hp->vl, 4, - "HTTP2 rx poll (fd:%d revents: %x n=%d, i=%d)", + "HTTP2 rx poll (fd:%d revents: %x n=%zu, i=%d)", hp->sess->fd, pfd[0].revents, n, i); if (i == 0) vtc_log(hp->vl, 3, From phk at FreeBSD.org Tue Jun 25 06:26:05 2024 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 25 Jun 2024 06:26:05 +0000 (UTC) Subject: [master] f6e7eb7a0 Guard against underflow in diagnostics Message-ID: <20240625062605.3893C11573A@lists.varnish-cache.org> commit f6e7eb7a0c1ace1874559e52ec566fa19bd0db8f Author: Poul-Henning Kamp Date: Tue Jun 25 06:25:14 2024 +0000 Guard against underflow in diagnostics diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index d2d9d5daa..b3e047dd4 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -267,7 +267,7 @@ haproxy_cli_recv(struct haproxy_cli *hc) } /* Connection closed. */ if (ret == 0) { - if (hc->rxbuf[rdz - 1] != '\n') + if (rdz > 0 && hc->rxbuf[rdz - 1] != '\n') vtc_fatal(hc->vl, "CLI rx timeout (fd: %d %.3fs ret: %zd)", hc->sock, hc->timeout, ret); From dridi.boukelmoune at gmail.com Wed Jun 26 09:25:06 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 26 Jun 2024 09:25:06 +0000 (UTC) Subject: [master] 2d598f31c gzip: Proper double quoting Message-ID: <20240626092506.E5F61617B0@lists.varnish-cache.org> commit 2d598f31cbcc3e0a88b9cd52922a12427d3463f4 Author: Dridi Boukelmoune Date: Tue Nov 29 11:26:13 2022 +0100 gzip: Proper double quoting diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 476c261f5..b5e608db4 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -30,7 +30,7 @@ * Interaction with the linvgz (zlib) library. * * The zlib library pollutes namespace a LOT when you include the "vgz.h" - * (aka (zlib.h") file so we contain the damage by vectoring all access + * (aka "zlib.h") file so we contain the damage by vectoring all access * to libz through this source file. * * The API defined by this file, will also insulate the rest of the code, From dridi.boukelmoune at gmail.com Wed Jun 26 09:25:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 26 Jun 2024 09:25:07 +0000 (UTC) Subject: [master] b37b3c3d3 gzip: Retire enum vgz_ua_e Message-ID: <20240626092507.08EDF617B5@lists.varnish-cache.org> commit b37b3c3d33d063560105be47bed71cf84beccc55 Author: Dridi Boukelmoune Date: Tue Nov 29 11:34:49 2022 +0100 gzip: Retire enum vgz_ua_e We already have everything we need with enum vgzret_e and vgz::dir. diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c index c52b7ff84..e80fee3d0 100644 --- a/bin/varnishd/cache/cache_esi_fetch.c +++ b/bin/varnishd/cache/cache_esi_fetch.c @@ -97,7 +97,7 @@ vfp_vep_callback(struct vfp_ctx *vc, void *priv, ssize_t l, enum vgz_flag flg) } VGZ_Obuf(vef->vgz, ptr, dl); i = VGZ_Gzip(vef->vgz, &dp, &dl, flg); - VGZ_UpdateObj(vc, vef->vgz, VUA_UPDATE); + VGZ_UpdateObj(vc, vef->vgz, i); if (dl > 0) { vef->tot += dl; VFP_Extend(vc, dl, VFP_OK); @@ -145,7 +145,7 @@ vfp_esi_end(struct vfp_ctx *vc, struct vef_priv *vef, if (vef->vgz != NULL) { if (retval == VFP_END) - VGZ_UpdateObj(vc, vef->vgz, VUA_END_GZIP); + VGZ_UpdateObj(vc, vef->vgz, VGZ_END); if (VGZ_Destroy(&vef->vgz) != VGZ_END) retval = VFP_Error(vc, "ESI+Gzip Failed at the very end"); diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index b5e608db4..650aed8c3 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -406,14 +406,16 @@ const struct vdp VDP_gunzip = { /*--------------------------------------------------------------------*/ void -VGZ_UpdateObj(const struct vfp_ctx *vc, struct vgz *vg, enum vgz_ua_e e) +VGZ_UpdateObj(const struct vfp_ctx *vc, struct vgz *vg, enum vgzret_e e) { char *p; intmax_t ii; CHECK_OBJ_NOTNULL(vg, VGZ_MAGIC); + if (e < VGZ_OK) + return; ii = vg->vz.start_bit + vg->vz.last_bit + vg->vz.stop_bit; - if (e == VUA_UPDATE && ii == vg->bits) + if (e != VGZ_END && ii == vg->bits) return; vg->bits = ii; p = ObjSetAttr(vc->wrk, vc->oc, OA_GZIPBITS, 32, NULL); @@ -421,9 +423,11 @@ VGZ_UpdateObj(const struct vfp_ctx *vc, struct vgz *vg, enum vgz_ua_e e) vbe64enc(p, vg->vz.start_bit); vbe64enc(p + 8, vg->vz.last_bit); vbe64enc(p + 16, vg->vz.stop_bit); - if (e == VUA_END_GZIP) + if (e != VGZ_END) + return; + if (vg->dir == VGZ_GZ) vbe64enc(p + 24, vg->vz.total_in); - if (e == VUA_END_GUNZIP) + if (vg->dir == VGZ_UN) vbe64enc(p + 24, vg->vz.total_out); } @@ -620,7 +624,7 @@ vfp_gzip_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p, if (vr < VGZ_OK) return (VFP_Error(vc, "Gzip failed")); if (dl > 0) { - VGZ_UpdateObj(vc, vg, VUA_UPDATE); + VGZ_UpdateObj(vc, vg, vr); *lp = dl; assert(dp == p); if (vr != VGZ_END || !VGZ_IbufEmpty(vg)) @@ -632,7 +636,7 @@ vfp_gzip_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p, if (vr != VGZ_END) return (VFP_Error(vc, "Gzip failed")); - VGZ_UpdateObj(vc, vg, VUA_END_GZIP); + VGZ_UpdateObj(vc, vg, VGZ_END); return (VFP_END); } @@ -673,11 +677,10 @@ vfp_testgunzip_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p, "Invalid Gzip data: %s", vgz_msg(vg))); } while (!VGZ_IbufEmpty(vg)); } - VGZ_UpdateObj(vc, vg, VUA_UPDATE); + VGZ_UpdateObj(vc, vg, vr); if (vp == VFP_END) { if (vr != VGZ_END) return (VFP_Error(vc, "tGunzip failed")); - VGZ_UpdateObj(vc, vg, VUA_END_GUNZIP); } return (vp); } diff --git a/bin/varnishd/cache/cache_vgz.h b/bin/varnishd/cache/cache_vgz.h index af5e08933..0464bc7f1 100644 --- a/bin/varnishd/cache/cache_vgz.h +++ b/bin/varnishd/cache/cache_vgz.h @@ -55,10 +55,4 @@ enum vgzret_e VGZ_Gzip(struct vgz *, const void **, ssize_t *len, // enum vgzret_e VGZ_Gunzip(struct vgz *, const void **, ssize_t *len); enum vgzret_e VGZ_Destroy(struct vgz **); -enum vgz_ua_e { - VUA_UPDATE, // Update start/stop/last bits if changed - VUA_END_GZIP, // Record uncompressed size - VUA_END_GUNZIP, // Record uncompressed size -}; - -void VGZ_UpdateObj(const struct vfp_ctx *, struct vgz*, enum vgz_ua_e); +void VGZ_UpdateObj(const struct vfp_ctx *, struct vgz*, enum vgzret_e); From dridi.boukelmoune at gmail.com Wed Jun 26 09:25:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 26 Jun 2024 09:25:07 +0000 (UTC) Subject: [master] dfa562d67 gzip: Fold if statements together Message-ID: <20240626092507.28125617B9@lists.varnish-cache.org> commit dfa562d671fcad09797180c4230bc3a2b8bd4c54 Author: Dridi Boukelmoune Date: Tue Nov 29 11:37:29 2022 +0100 gzip: Fold if statements together diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 650aed8c3..0b64c87f4 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -678,10 +678,8 @@ vfp_testgunzip_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p, } while (!VGZ_IbufEmpty(vg)); } VGZ_UpdateObj(vc, vg, vr); - if (vp == VFP_END) { - if (vr != VGZ_END) - return (VFP_Error(vc, "tGunzip failed")); - } + if (vp == VFP_END && vr != VGZ_END) + return (VFP_Error(vc, "tGunzip failed")); return (vp); } From dridi.boukelmoune at gmail.com Wed Jun 26 09:25:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 26 Jun 2024 09:25:07 +0000 (UTC) Subject: [master] 82328d2e4 gzip: Wrong incomplete claim Message-ID: <20240626092507.45B6A617C0@lists.varnish-cache.org> commit 82328d2e4a7987e58cbd6f4bfc8645c8af29cd7f Author: Dridi Boukelmoune Date: Tue Nov 29 11:41:00 2022 +0100 gzip: Wrong incomplete claim diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 0b64c87f4..cce6332b4 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -264,7 +264,7 @@ VGZ_Gzip(struct vgz *vg, const void **pptr, ssize_t *plen, enum vgz_flag flags) case VGZ_ALIGN: zflg = Z_SYNC_FLUSH; break; case VGZ_RESET: zflg = Z_FULL_FLUSH; break; case VGZ_FINISH: zflg = Z_FINISH; break; - default: INCOMPL(); + default: WRONG("Invalid VGZ flag"); } i = deflate(&vg->vz, zflg); if (i == Z_OK || i == Z_STREAM_END) { From dridi.boukelmoune at gmail.com Wed Jun 26 09:46:05 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 26 Jun 2024 09:46:05 +0000 (UTC) Subject: [master] 612faef42 gzip: s/linvgz/libvgz/ Message-ID: <20240626094605.266EF639D2@lists.varnish-cache.org> commit 612faef42c7bdf4095c4988cd3cdd3a5f15c1e37 Author: Dridi Boukelmoune Date: Wed Jun 26 11:44:10 2024 +0200 gzip: s/linvgz/libvgz/ diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index cce6332b4..472ac2e5b 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -27,7 +27,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * Interaction with the linvgz (zlib) library. + * Interaction with the libvgz (zlib) library. * * The zlib library pollutes namespace a LOT when you include the "vgz.h" * (aka "zlib.h") file so we contain the damage by vectoring all access