From dridi.boukelmoune at gmail.com Wed Dec 2 10:43:10 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 2 Dec 2020 10:43:10 +0000 (UTC) Subject: [master] 2a0de6d8b vsc: Introduce MAIN.s_bgfetch Message-ID: <20201202104310.55F8BB0522@lists.varnish-cache.org> commit 2a0de6d8bb040624d80e5c16ce5f93b856921e2c Author: Dridi Boukelmoune Date: Fri Nov 27 16:04:06 2020 +0100 vsc: Introduce MAIN.s_bgfetch diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc index 4bede0a9a..b99ca8c2a 100644 --- a/bin/varnishd/VSC_main.vsc +++ b/bin/varnishd/VSC_main.vsc @@ -434,6 +434,12 @@ :group: wrk :oneliner: Total backend fetches initiated + Total backend fetches initiated, including background fetches. + +.. varnish_vsc:: s_bgfetch + :group: wrk + :oneliner: Total backend background fetches initiated + .. varnish_vsc:: s_synth :group: wrk diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index a80d9ea30..7c4e7c036 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -597,6 +597,8 @@ cnt_lookup(struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(busy->boc, BOC_MAGIC); // XXX: shouldn't we go to miss? VBF_Fetch(wrk, req, busy, oc, VBF_BACKGROUND); + wrk->stats->s_fetch++; + wrk->stats->s_bgfetch++; } else { (void)VRB_Ignore(req);// XXX: handle err } From dridi.boukelmoune at gmail.com Wed Dec 2 10:43:10 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 2 Dec 2020 10:43:10 +0000 (UTC) Subject: [master] 0a2c6dad4 vtc: varnish vNAME -expect PATTERN OP PATTERN Message-ID: <20201202104310.6EDB0B0526@lists.varnish-cache.org> commit 0a2c6dad4db82008e8bca67f8132589725c8a3e6 Author: Dridi Boukelmoune Date: Mon Nov 30 16:06:38 2020 +0100 vtc: varnish vNAME -expect PATTERN OP PATTERN Make it possible to compare VSCs, to create a synchronization point based on internal events (especially with custom VSCs). For example: # block until baz catches up varnish v1 -expect FOO.bar == FOO.baz This is still subject to roughly 5s before timing out. diff --git a/bin/varnishtest/tests/b00052.vtc b/bin/varnishtest/tests/b00052.vtc index 23cfcf5de..90cba6988 100644 --- a/bin/varnishtest/tests/b00052.vtc +++ b/bin/varnishtest/tests/b00052.vtc @@ -1,23 +1,27 @@ varnishtest "The cache_hit_grace counter" server s1 { - # normal fetch rxreq expect req.url == "/1" + expect req.http.bgfetch == false txresp -hdr "Age: 1" -hdr "Cache-Control: max-age=2" -body "1" - # background fetch: rxreq expect req.url == "/1" + expect req.http.bgfetch == true txresp -body "2" - # normal fetch rxreq expect req.url == "/2" + expect req.http.bgfetch == false txresp } -start -varnish v1 -vcl+backend { } -start +varnish v1 -vcl+backend { + sub vcl_backend_fetch { + set bereq.http.bgfetch = bereq.is_bgfetch; + } +} -start client c1 { txreq -url "/1" @@ -35,6 +39,8 @@ client c2 { expect resp.body == "1" } -run +varnish v1 -expect cache_hit >= cache_hit_grace + delay 2 client c3 { @@ -45,6 +51,8 @@ client c3 { expect resp.body == "2" } -run +varnish v1 -expect cache_hit >= cache_hit_grace + # Check that counters are correct: varnish v1 -expect cache_hit == 2 diff --git a/bin/varnishtest/tests/r02946.vtc b/bin/varnishtest/tests/r02946.vtc index ecda00483..71872679a 100644 --- a/bin/varnishtest/tests/r02946.vtc +++ b/bin/varnishtest/tests/r02946.vtc @@ -44,4 +44,5 @@ client c1 { delay 1 +varnish v1 -expect s_fetch > s_bgfetch varnish v1 -expect n_objectcore == 1 diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 9912da43b..d1f962919 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -910,12 +910,29 @@ varnish_vsc(const struct varnish *v, const char *arg) * Check statistics */ +struct stat_arg { + const char *pattern; + uintmax_t val; + unsigned good; +}; + struct stat_priv { - char target_pattern[256]; - uintmax_t val; - const struct varnish *v; + struct stat_arg lhs; + struct stat_arg rhs; }; +static int +stat_match(const char *pattern, const char *name) +{ + + if (strchr(pattern, '.') == NULL) { + if (fnmatch("MAIN.*", name, 0)) + return (FNM_NOMATCH); + name += 5; + } + return (fnmatch(pattern, name, 0)); +} + static int do_expect_cb(void *priv, const struct VSC_point * const pt) { @@ -924,13 +941,24 @@ do_expect_cb(void *priv, const struct VSC_point * const pt) if (pt == NULL) return (0); - if (fnmatch(sp->target_pattern, pt->name, 0)) - return (0); + if (!sp->lhs.good && stat_match(sp->lhs.pattern, pt->name) == 0) { + AZ(strcmp(pt->ctype, "uint64_t")); + AN(pt->ptr); + sp->lhs.val = *pt->ptr; + sp->lhs.good = 1; + } - AZ(strcmp(pt->ctype, "uint64_t")); - AN(pt->ptr); - sp->val = *pt->ptr; - return (1); + if (sp->rhs.pattern == NULL) { + sp->rhs.good = 1; + } else if (!sp->rhs.good && + stat_match(sp->rhs.pattern, pt->name) == 0) { + AZ(strcmp(pt->ctype, "uint64_t")); + AN(pt->ptr); + sp->rhs.val = *pt->ptr; + sp->rhs.good = 1; + } + + return (sp->lhs.good && sp->rhs.good); } /********************************************************************** @@ -939,55 +967,49 @@ do_expect_cb(void *priv, const struct VSC_point * const pt) static void varnish_expect(const struct varnish *v, char * const *av) { - uint64_t ref; - int good; - char *r; - char *p; - int i, not = 0; struct stat_priv sp; - - r = av[0]; - if (r[0] == '!') { - not = 1; - r++; + int good, i, not; + uintmax_t u; + char *l, *p; + + ZERO_OBJ(&sp, sizeof sp); + l = av[0]; + not = (*l == '!'); + if (not) { + l++; AZ(av[1]); } else { AN(av[1]); AN(av[2]); - } - p = strrchr(r, '.'); - if (p == NULL) { - bprintf(sp.target_pattern, "MAIN.%s", r); - } else { - bprintf(sp.target_pattern, "%s", r); + u = strtoumax(av[2], &p, 0); + if (u != UINTMAX_MAX && *p == '\0') + sp.rhs.val = u; + else + sp.rhs.pattern = av[2]; } - sp.val = 0; - sp.v = v; - ref = 0; - good = 0; + sp.lhs.pattern = l; + for (i = 0; i < 50; i++, (void)usleep(100000)) { (void)VSM_Status(v->vsm_vsc); + sp.lhs.good = sp.rhs.good = 0; good = VSC_Iter(v->vsc, v->vsm_vsc, do_expect_cb, &sp); - if (!good) { + if (!good) good = -2; + if (good < 0) continue; - } if (not) - vtc_fatal(v->vl, "Found (not expected): %s", av[0]+1); - - good = 0; - ref = strtoumax(av[2], &p, 0); - if (ref == UINTMAX_MAX || *p) - vtc_fatal(v->vl, "Syntax error in number (%s)", av[2]); - if (!strcmp(av[1], "==")) { if (sp.val == ref) good = 1; } - else if (!strcmp(av[1], "!=")) { if (sp.val != ref) good = 1; } - else if (!strcmp(av[1], ">")) { if (sp.val > ref) good = 1; } - else if (!strcmp(av[1], "<")) { if (sp.val < ref) good = 1; } - else if (!strcmp(av[1], ">=")) { if (sp.val >= ref) good = 1; } - else if (!strcmp(av[1], "<=")) { if (sp.val <= ref) good = 1; } - else + vtc_fatal(v->vl, "Found (not expected): %s", l); + + good = -1; + if (!strcmp(av[1], "==")) good = (sp.lhs.val == sp.rhs.val); + if (!strcmp(av[1], "!=")) good = (sp.lhs.val != sp.rhs.val); + if (!strcmp(av[1], ">" )) good = (sp.lhs.val > sp.rhs.val); + if (!strcmp(av[1], "<" )) good = (sp.lhs.val < sp.rhs.val); + if (!strcmp(av[1], ">=")) good = (sp.lhs.val >= sp.rhs.val); + if (!strcmp(av[1], "<=")) good = (sp.lhs.val <= sp.rhs.val); + if (good == -1) vtc_fatal(v->vl, "comparison %s unknown", av[1]); if (good) break; @@ -997,19 +1019,19 @@ varnish_expect(const struct varnish *v, char * const *av) } if (good == -2) { if (not) { - vtc_log(v->vl, 2, "not found (as expected): %s", - av[0] + 1); + vtc_log(v->vl, 2, "not found (as expected): %s", l); return; } - vtc_fatal(v->vl, "stats field %s unknown", av[0]); + vtc_fatal(v->vl, "stats field %s unknown", + sp.lhs.good ? sp.rhs.pattern : sp.lhs.pattern); } if (good == 1) { - vtc_log(v->vl, 2, "as expected: %s (%ju) %s %s", - av[0], sp.val, av[1], av[2]); + vtc_log(v->vl, 2, "as expected: %s (%ju) %s %s (%ju)", + av[0], sp.lhs.val, av[1], av[2], sp.rhs.val); } else { vtc_fatal(v->vl, "Not true: %s (%ju) %s %s (%ju)", - av[0], (uintmax_t)sp.val, av[1], av[2], (uintmax_t)ref); + av[0], sp.lhs.val, av[1], av[2], sp.rhs.val); } } @@ -1091,11 +1113,14 @@ varnish_expect(const struct varnish *v, char * const *av) * Once Varnish is stopped, clean everything after it. This is only used * in very few tests and you should never need it. * + * \-expectexit NUMBER + * Expect varnishd to exit(3) with this value + * * Once Varnish is started, you can talk to it (as you would through * ``varnishadm``) with these additional switches:: * * varnish vNAME [-cli STRING] [-cliok STRING] [-clierr STRING] - * [-clijson STRING] [-expect STRING OP NUMBER] + * [-clijson STRING] * * \-cli STRING|-cliok STRING|-clierr STATUS STRING|-cliexpect REGEXP STRING * All four of these will send STRING to the CLI, the only difference @@ -1107,14 +1132,22 @@ varnish_expect(const struct varnish *v, char * const *av) * Send STRING to the CLI, expect success (CLIS_OK/200) and check * that the response is parsable JSON. * - * \-expect PATTERN OP NUMBER + * It is also possible to interact with its shared memory (as you would + * through tools like ``varnishstat``) with additional switches: + * + * \-expect \!PATTERN|PATTERN OP NUMBER|PATTERN OP PATTERN * Look into the VSM and make sure the first VSC counter identified by * PATTERN has a correct value. OP can be ==, >, >=, <, <=. For * example:: * * varnish v1 -expect SM?.s1.g_space > 1000000 - * \-expectexit NUMBER - * Expect varnishd to exit(3) with this value + * varnish v1 -expect cache_hit >= cache_hit_grace + * + * In the \! form the test fails if a counter matches PATTERN. + * + * The ``MAIN.`` namespace can be omitted from PATTERN. + * + * The test takes up to 5 seconds before timing out. * * \-vsc PATTERN * Dump VSC counters matching PATTERN. From dridi.boukelmoune at gmail.com Wed Dec 2 10:43:10 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 2 Dec 2020 10:43:10 +0000 (UTC) Subject: [master] 76a580dbe vtc: Move vsl_catchup() down Message-ID: <20201202104310.920BDB0529@lists.varnish-cache.org> commit 76a580dbe5ba5795028cf41102f73821a8e832c0 Author: Dridi Boukelmoune Date: Mon Nov 30 16:12:48 2020 +0100 vtc: Move vsl_catchup() down It's more an action than housekeeping. diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index d1f962919..0586737fa 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -195,16 +195,6 @@ wait_running(const struct varnish *v) * Varnishlog gatherer thread */ -static void -vsl_catchup(const struct varnish *v) -{ - int vsl_idle; - - vsl_idle = v->vsl_idle; - while (!vtc_error && vsl_idle == v->vsl_idle) - VTIM_sleep(0.1); -} - static void * varnishlog_thread(void *priv) { @@ -1035,6 +1025,16 @@ varnish_expect(const struct varnish *v, char * const *av) } } +static void +vsl_catchup(const struct varnish *v) +{ + int vsl_idle; + + vsl_idle = v->vsl_idle; + while (!vtc_error && vsl_idle == v->vsl_idle) + VTIM_sleep(0.1); +} + /* SECTION: varnish varnish * * Define and interact with varnish instances. From dridi.boukelmoune at gmail.com Wed Dec 2 10:43:10 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 2 Dec 2020 10:43:10 +0000 (UTC) Subject: [master] 61a5d17fb vtc: Make all relevant commands start varnishd Message-ID: <20201202104310.AFC2AB0532@lists.varnish-cache.org> commit 61a5d17fbae296a80aab70a543a461311cf27849 Author: Dridi Boukelmoune Date: Mon Nov 30 16:20:41 2020 +0100 vtc: Make all relevant commands start varnishd And wrap the logic in a macro to reduce duplication. diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 0586737fa..1a4a3e88a 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -530,6 +530,15 @@ varnish_launch(struct varnish *v) AZ(pthread_create(&v->tp_vsl, NULL, varnishlog_thread, v)); } +#define VARNISH_LAUNCH(v) \ + do { \ + CHECK_OBJ_NOTNULL(v, VARNISH_MAGIC); \ + if (v->cli_fd < 0) \ + varnish_launch(v); \ + if (vtc_error) \ + return; \ + } while (0) + /********************************************************************** * Start a Varnish */ @@ -598,10 +607,7 @@ varnish_start(struct varnish *v) enum VCLI_status_e u; char *resp = NULL; - if (v->cli_fd < 0) - varnish_launch(v); - if (vtc_error) - return; + VARNISH_LAUNCH(v); vtc_log(v->vl, 2, "Start"); u = varnish_ask_cli(v, "start", &resp); if (vtc_error) @@ -639,10 +645,8 @@ varnish_start(struct varnish *v) static void varnish_stop(struct varnish *v) { - if (v->cli_fd < 0) - varnish_launch(v); - if (vtc_error) - return; + + VARNISH_LAUNCH(v); vtc_log(v->vl, 2, "Stop"); (void)varnish_ask_cli(v, "stop", NULL); wait_stopped(v); @@ -713,10 +717,7 @@ varnish_cli_json(struct varnish *v, const char *cli) const char *errptr; struct vjsn *vj; - if (v->cli_fd < 0) - varnish_launch(v); - if (vtc_error) - return; + VARNISH_LAUNCH(v); u = varnish_ask_cli(v, cli, &resp); vtc_log(v->vl, 2, "CLI %03u <%s>", u, cli); if (u != CLIS_OK) @@ -742,18 +743,12 @@ varnish_cli(struct varnish *v, const char *cli, unsigned exp, const char *re) const char *errptr; int err; + VARNISH_LAUNCH(v); if (re != NULL) { vre = VRE_compile(re, 0, &errptr, &err); if (vre == NULL) vtc_fatal(v->vl, "Illegal regexp"); } - if (v->cli_fd < 0) - varnish_launch(v); - if (vtc_error) { - if (vre != NULL) - VRE_free(&vre); - return; - } u = varnish_ask_cli(v, cli, &resp); vtc_log(v->vl, 2, "CLI %03u <%s>", u, cli); if (exp != 0 && exp != (unsigned)u) @@ -777,10 +772,7 @@ varnish_vcl(struct varnish *v, const char *vcl, int fail, char **resp) struct vsb *vsb; enum VCLI_status_e u; - if (v->cli_fd < 0) - varnish_launch(v); - if (vtc_error) - return; + VARNISH_LAUNCH(v); vsb = VSB_new_auto(); AN(vsb); @@ -816,10 +808,7 @@ varnish_vclbackend(struct varnish *v, const char *vcl) struct vsb *vsb, *vsb2; enum VCLI_status_e u; - if (v->cli_fd < 0) - varnish_launch(v); - if (vtc_error) - return; + VARNISH_LAUNCH(v); vsb = VSB_new_auto(); AN(vsb); @@ -885,10 +874,11 @@ do_stat_dump_cb(void *priv, const struct VSC_point * const pt) } static void -varnish_vsc(const struct varnish *v, const char *arg) +varnish_vsc(struct varnish *v, const char *arg) { struct dump_priv dp; + VARNISH_LAUNCH(v); memset(&dp, 0, sizeof dp); dp.v = v; dp.arg = arg; @@ -955,13 +945,14 @@ do_expect_cb(void *priv, const struct VSC_point * const pt) */ static void -varnish_expect(const struct varnish *v, char * const *av) +varnish_expect(struct varnish *v, char * const *av) { struct stat_priv sp; int good, i, not; uintmax_t u; char *l, *p; + VARNISH_LAUNCH(v); ZERO_OBJ(&sp, sizeof sp); l = av[0]; not = (*l == '!'); @@ -1026,10 +1017,11 @@ varnish_expect(const struct varnish *v, char * const *av) } static void -vsl_catchup(const struct varnish *v) +vsl_catchup(struct varnish *v) { int vsl_idle; + VARNISH_LAUNCH(v); vsl_idle = v->vsl_idle; while (!vtc_error && vsl_idle == v->vsl_idle) VTIM_sleep(0.1); From nils.goroll at uplex.de Wed Dec 2 15:34:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 2 Dec 2020 15:34:06 +0000 (UTC) Subject: [master] 036f13d5b typo Message-ID: <20201202153406.A7364782E@lists.varnish-cache.org> commit 036f13d5b4b8d3fecced855c937d05f03db168e4 Author: Nils Goroll Date: Wed Dec 2 16:33:27 2020 +0100 typo diff --git a/bin/varnishd/mgt/mgt_jail_solaris.c b/bin/varnishd/mgt/mgt_jail_solaris.c index fb118cfd6..516f12fc7 100644 --- a/bin/varnishd/mgt/mgt_jail_solaris.c +++ b/bin/varnishd/mgt/mgt_jail_solaris.c @@ -338,7 +338,7 @@ vjs_init(char **args) e); continue; } - ARGV_ERR("-jsolrais: unknown sub-argument '%s'\n", + ARGV_ERR("-jsolaris: unknown sub-argument '%s'\n", *args); } } From phk at FreeBSD.org Thu Dec 3 14:50:07 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 3 Dec 2020 14:50:07 +0000 (UTC) Subject: [master] f77fc42a4 Merger from VTEST: Message-ID: <20201203145007.DE8D44F35@lists.varnish-cache.org> commit f77fc42a448c837d95ff01c595c458f584b27e27 Author: Poul-Henning Kamp Date: Mon Nov 23 10:41:12 2020 +0000 Merger from VTEST: vtc_haproxy: VTCP_listen_on 127.0.0.1 instead of localhost Always bind vtc_haproxy with 127.0.0.1 instead of binding on ::1 or 127.0.0.1 depending on the environment. This is related to bug https://github.com/haproxy/haproxy/issues/937 here the ipv6 and ipv4 are randomly mixed which can't work with socks4. diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index b7f467cb7..46b4001da 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -520,7 +520,7 @@ haproxy_create_mcli(struct haproxy *h) const char *err; char buf[128], addr[128], port[128]; - sock = VTCP_listen_on("localhost:0", NULL, 100, &err); + sock = VTCP_listen_on("127.0.0.1:0", NULL, 100, &err); if (err != NULL) vtc_fatal(h->vl, "Create listen socket failed: %s", err); @@ -592,7 +592,7 @@ haproxy_new(const char *name) * May be useful to simulate an unreachable server. */ bprintf(h->closed_sock, "%s_closed", h->name); - closed_sock = VTCP_listen_on("localhost:0", NULL, 100, &err); + closed_sock = VTCP_listen_on("127.0.0.1:0", NULL, 100, &err); if (err != NULL) vtc_fatal(h->vl, "Create listen socket failed: %s", err); @@ -848,7 +848,7 @@ haproxy_build_backends(struct haproxy *h, const char *vsb_data) break; *q++ = '\0'; - sock = VTCP_listen_on("localhost:0", NULL, 100, &err); + sock = VTCP_listen_on("127.0.0.1:0", NULL, 100, &err); if (err != NULL) vtc_fatal(h->vl, "Create listen socket failed: %s", err); From phk at FreeBSD.org Thu Dec 3 14:50:07 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 3 Dec 2020 14:50:07 +0000 (UTC) Subject: [master] 55dffc358 Refactor the panic structure formatting code Message-ID: <20201203145008.02BBA4F3B@lists.varnish-cache.org> commit 55dffc3585685f2a67e3228094aea9ca9d986798 Author: Poul-Henning Kamp Date: Thu Dec 3 14:48:43 2020 +0000 Refactor the panic structure formatting code This introduces a `PAN_dump_struct` function to do the standard up-front checks for NULL, bad magic & already dumped. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 10bd3079b..c7825f8fa 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -848,10 +848,9 @@ Tlen(const txt t) */ #define W_TIM_real(w) ((w)->lastused = VTIM_real()) -#define PAN_CheckMagic(vsb, ptr, exp) \ - do { \ - if ((ptr)->magic != (exp)) \ - VSB_printf((vsb), \ - "MAGIC at %p is 0x%08x (Should be: %s/0x%08x)\n", \ - ptr, (ptr)->magic, #exp, exp); \ - } while(0) +int PAN_dump_struct2(struct vsb *vsb, const void *ptr, + const char *smagic, unsigned magic, const char *fmt, ...) + v_printflike_(5,6); + +#define PAN_dump_struct(vsb, ptr, magic, ...) \ + PAN_dump_struct2(vsb, ptr, #magic, magic, __VA_ARGS__) diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 85d294b2d..7dbe03a0a 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -39,9 +39,8 @@ VDP_Panic(struct vsb *vsb, const struct vdp_ctx *vdc) { struct vdp_entry *vde; - VSB_printf(vsb, "vdc = %p {\n", vdc); - VSB_indent(vsb, 2); - PAN_CheckMagic(vsb, vdc, VDP_CTX_MAGIC); + if (PAN_dump_struct(vsb, vdc, VDP_CTX_MAGIC, "vdc")) + return; VSB_printf(vsb, "nxt = %p,\n", vdc->nxt); VSB_printf(vsb, "retval = %d,\n", vdc->retval); diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 84b4a77c8..e20e07f80 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -110,23 +110,39 @@ static const void *already_list[N_ALREADY]; static int already_idx; int -PAN_already(struct vsb *vsb, const void *ptr) +PAN_dump_struct2(struct vsb *vsb, const void *ptr, + const char *smagic, unsigned magic, const char *fmt, ...) { + va_list ap; + const unsigned *uptr; int i; + AN(vsb); + va_start(ap, fmt); + VSB_vprintf(vsb, fmt, ap); + va_end(ap); if (ptr == NULL) { - VSB_cat(vsb, "},\n"); - return (1); + VSB_printf(vsb, " = NULL\n"); + return (-1); } + VSB_printf(vsb, " = %p {\n", ptr); for (i = 0; i < already_idx; i++) { if (already_list[i] == ptr) { VSB_cat(vsb, " [Already dumped, see above]\n"); VSB_cat(vsb, "},\n"); - return (1); + return (-2); } } if (already_idx < N_ALREADY) already_list[already_idx++] = ptr; + uptr = ptr; + if (*uptr != magic) { + VSB_printf(vsb, " .magic = 0x%08x", *uptr); + VSB_printf(vsb, " EXPECTED: %s=0x%08x\n", smagic, magic); + VSB_printf(vsb, "}\n"); + return (-3); + } + VSB_indent(vsb, 2); return (0); } @@ -136,15 +152,12 @@ static void pan_htc(struct vsb *vsb, const struct http_conn *htc) { - VSB_printf(vsb, "http_conn = %p {\n", htc); - if (PAN_already(vsb, htc)) + if (PAN_dump_struct(vsb, htc, HTTP_CONN_MAGIC, "http_conn")) return; - VSB_indent(vsb, 2); - PAN_CheckMagic(vsb, htc, HTTP_CONN_MAGIC); if (htc->rfd != NULL) VSB_printf(vsb, "fd = %d (@%p),\n", *htc->rfd, htc->rfd); VSB_printf(vsb, "doclose = %s,\n", sess_close_2str(htc->doclose, 0)); - WS_Panic(htc->ws, vsb); + WS_Panic(vsb, htc->ws); VSB_printf(vsb, "{rxbuf_b, rxbuf_e} = {%p, %p},\n", htc->rxbuf_b, htc->rxbuf_e); VSB_printf(vsb, "{pipeline_b, pipeline_e} = {%p, %p},\n", @@ -168,12 +181,9 @@ pan_http(struct vsb *vsb, const char *id, const struct http *h) { int i; - VSB_printf(vsb, "http[%s] = %p {\n", id, h); - if (PAN_already(vsb, h)) + if (PAN_dump_struct(vsb, h, HTTP_MAGIC, "http[%s]", id)) return; - VSB_indent(vsb, 2); - PAN_CheckMagic(vsb, h, HTTP_MAGIC); - WS_Panic(h->ws, vsb); + WS_Panic(vsb, h->ws); VSB_cat(vsb, "hdrs {\n"); VSB_indent(vsb, 2); for (i = 0; i < h->nhd; ++i) { @@ -189,14 +199,12 @@ pan_http(struct vsb *vsb, const char *id, const struct http *h) } /*--------------------------------------------------------------------*/ + static void pan_boc(struct vsb *vsb, const struct boc *boc) { - VSB_printf(vsb, "boc = %p {\n", boc); - if (PAN_already(vsb, boc)) + if (PAN_dump_struct(vsb, boc, BOC_MAGIC, "boc")) return; - VSB_indent(vsb, 2); - PAN_CheckMagic(vsb, boc, BOC_MAGIC); VSB_printf(vsb, "refcnt = %u,\n", boc->refcount); VSB_printf(vsb, "state = %s,\n", boc_state_2str(boc->state)); VSB_printf(vsb, "vary = %p,\n", boc->vary); @@ -212,11 +220,8 @@ pan_objcore(struct vsb *vsb, const char *typ, const struct objcore *oc) { const char *p; - VSB_printf(vsb, "objcore[%s] = %p {\n", typ, oc); - if (PAN_already(vsb, oc)) + if (PAN_dump_struct(vsb, oc, OBJCORE_MAGIC, "objcore[%s]", typ)) return; - VSB_indent(vsb, 2); - PAN_CheckMagic(vsb, oc, OBJCORE_MAGIC); VSB_printf(vsb, "refcnt = %d,\n", oc->refcnt); VSB_cat(vsb, "flags = {"); @@ -267,12 +272,9 @@ pan_wrk(struct vsb *vsb, const struct worker *wrk) unsigned m, u; const char *p; - VSB_printf(vsb, "worker = %p {\n", wrk); - if (PAN_already(vsb, wrk)) + if (PAN_dump_struct(vsb, wrk, WORKER_MAGIC, "worker")) return; - VSB_indent(vsb, 2); - PAN_CheckMagic(vsb, wrk, WORKER_MAGIC); - WS_Panic(wrk->aws, vsb); + WS_Panic(vsb, wrk->aws); m = wrk->cur_method; VSB_cat(vsb, "VCL::method = "); @@ -314,9 +316,8 @@ pan_vfp(struct vsb *vsb, const struct vfp_ctx *vfc) { struct vfp_entry *vfe; - VSB_printf(vsb, "vfc = %p {\n", vfc); - VSB_indent(vsb, 2); - PAN_CheckMagic(vsb, vfc, VFP_CTX_MAGIC); + if (PAN_dump_struct(vsb, vfc, VFP_CTX_MAGIC, "vfc")) + return; VSB_printf(vsb, "failed = %d,\n", vfc->failed); VSB_printf(vsb, "req = %p,\n", vfc->req); VSB_printf(vsb, "resp = %p,\n", vfc->resp); @@ -349,11 +350,8 @@ pan_busyobj(struct vsb *vsb, const struct busyobj *bo) { const char *p; - VSB_printf(vsb, "busyobj = %p {\n", bo); - if (PAN_already(vsb, bo)) + if (PAN_dump_struct(vsb, bo, BUSYOBJ_MAGIC, "busyobj")) return; - VSB_indent(vsb, 2); - PAN_CheckMagic(vsb, bo, BUSYOBJ_MAGIC); VSB_printf(vsb, "end = %p,\n", bo->end); VSB_printf(vsb, "retries = %u,\n", bo->retries); @@ -369,7 +367,7 @@ pan_busyobj(struct vsb *vsb, const struct busyobj *bo) if (bo->vfp_filter_list != NULL) VSB_printf(vsb, "filter_list = \"%s\",\n", bo->vfp_filter_list); - WS_Panic(bo->ws, vsb); + WS_Panic(vsb, bo->ws); VSB_printf(vsb, "ws_bo = %p,\n", (void *)bo->ws_bo); // bereq0 left out @@ -413,10 +411,8 @@ pan_busyobj(struct vsb *vsb, const struct busyobj *bo) static void pan_top(struct vsb *vsb, const struct reqtop *top) { - VSB_printf(vsb, "top = %p {\n", top); - if (PAN_already(vsb, top)) + if (PAN_dump_struct(vsb, top, REQ_MAGIC, "top")) return; - VSB_indent(vsb, 2); pan_req(vsb, top->topreq); pan_privs(vsb, top->privs); VCL_Panic(vsb, "vcl0", top->vcl0); @@ -431,11 +427,8 @@ pan_req(struct vsb *vsb, const struct req *req) { const struct transport *xp; - VSB_printf(vsb, "req = %p {\n", req); - if (PAN_already(vsb, req)) + if (PAN_dump_struct(vsb, req, REQ_MAGIC, "req")) return; - VSB_indent(vsb, 2); - PAN_CheckMagic(vsb, req, REQ_MAGIC); xp = req->transport; VSB_printf(vsb, "vxid = %u, transport = %s", VXID(req->vsl->wid), xp == NULL ? "NULL" : xp->name); @@ -470,7 +463,7 @@ pan_req(struct vsb *vsb, const struct req *req) if (req->wrk != NULL) pan_wrk(vsb, req->wrk); - WS_Panic(req->ws, vsb); + WS_Panic(vsb, req->ws); if (VALID_OBJ(req->htc, HTTP_CONN_MAGIC)) pan_htc(vsb, req->htc); pan_http(vsb, "req", req->http); @@ -523,11 +516,8 @@ pan_sess(struct vsb *vsb, const struct sess *sp) const char *cp; const struct transport *xp; - VSB_printf(vsb, "sp = %p {\n", sp); - if (PAN_already(vsb, sp)) + if (PAN_dump_struct(vsb, sp, SESS_MAGIC, "sess")) return; - VSB_indent(vsb, 2); - PAN_CheckMagic(vsb, sp, SESS_MAGIC); VSB_printf(vsb, "fd = %d, vxid = %u,\n", sp->fd, VXID(sp->vxid)); VSB_printf(vsb, "t_open = %f,\n", sp->t_open); @@ -539,7 +529,7 @@ pan_sess(struct vsb *vsb, const struct sess *sp) return; } - WS_Panic(sp->ws, vsb); + WS_Panic(vsb, sp->ws); xp = XPORT_ByNumber(sp->sattr[SA_TRANSPORT]); VSB_printf(vsb, "transport = %s", xp == NULL ? "" : xp->name); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 1fc5e41f7..84316cee8 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -350,7 +350,6 @@ void ObjUnsubscribeEvents(uintptr_t *); /* cache_panic.c */ void PAN_Init(void); -int PAN_already(struct vsb *, const void *); const char *sess_close_2str(enum sess_close sc, int want_desc); /* cache_pool.c */ @@ -481,7 +480,7 @@ void WRK_Init(void); /* cache_ws.c */ void WS_Id(const struct ws *ws, char *id); -void WS_Panic(const struct ws *ws, struct vsb *vsb); +void WS_Panic(struct vsb *, const struct ws *); static inline int WS_IsReserved(const struct ws *ws) { diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 6c7427eb6..f0b035b7b 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -251,48 +251,48 @@ vcl_find(const char *name) /*--------------------------------------------------------------------*/ +static void +vcl_panic_conf(struct vsb *vsb, const struct VCL_conf *conf) +{ + int i; + const struct vpi_ii *ii; + + if (PAN_dump_struct(vsb, conf, VCL_CONF_MAGIC, "conf")) + return; + VSB_printf(vsb, "syntax = \"%u\",\n", conf->syntax); + VSB_cat(vsb, "srcname = {\n"); + VSB_indent(vsb, 2); + for (i = 0; i < conf->nsrc; ++i) + VSB_printf(vsb, "\"%s\",\n", conf->srcname[i]); + VSB_indent(vsb, -2); + VSB_cat(vsb, "},\n"); + VSB_cat(vsb, "instances = {\n"); + VSB_indent(vsb, 2); + ii = conf->instance_info; + while (ii != NULL && ii->p != NULL) { + VSB_printf(vsb, "\"%s\" = %p,\n", ii->name, + (const void *)*(const uintptr_t *)ii->p); + ii++; + } + VSB_indent(vsb, -2); + VSB_cat(vsb, "},\n"); + VSB_indent(vsb, -2); + VSB_cat(vsb, "},\n"); +} + void VCL_Panic(struct vsb *vsb, const char *nm, const struct vcl *vcl) { - const struct vpi_ii *ii; - int i; AN(vsb); - if (vcl == NULL) + if (PAN_dump_struct(vsb, vcl, VCL_MAGIC, "vcl[%s]", nm)) return; - VSB_printf(vsb, "%s = {\n", nm); - VSB_indent(vsb, 2); - PAN_CheckMagic(vsb, vcl, VCL_MAGIC); VSB_printf(vsb, "name = \"%s\",\n", vcl->loaded_name); VSB_printf(vsb, "busy = %u,\n", vcl->busy); VSB_printf(vsb, "discard = %u,\n", vcl->discard); VSB_printf(vsb, "state = %s,\n", vcl->state); VSB_printf(vsb, "temp = %s,\n", vcl->temp ? vcl->temp->name : "(null)"); - VSB_cat(vsb, "conf = {\n"); - VSB_indent(vsb, 2); - if (vcl->conf == NULL) { - VSB_cat(vsb, "conf = NULL\n"); - } else { - PAN_CheckMagic(vsb, vcl->conf, VCL_CONF_MAGIC); - VSB_printf(vsb, "syntax = \"%u\",\n", vcl->conf->syntax); - VSB_cat(vsb, "srcname = {\n"); - VSB_indent(vsb, 2); - for (i = 0; i < vcl->conf->nsrc; ++i) - VSB_printf(vsb, "\"%s\",\n", vcl->conf->srcname[i]); - VSB_indent(vsb, -2); - VSB_cat(vsb, "instances = {\n"); - VSB_indent(vsb, 2); - ii = vcl->conf->instance_info; - while (ii != NULL && ii->p != NULL) { - VSB_printf(vsb, "\"%s\" = %p,\n", ii->name, - (const void *)*(const uintptr_t *)ii->p); - ii++; - } - VSB_indent(vsb, -2); - VSB_cat(vsb, "},\n"); - } - VSB_indent(vsb, -2); - VSB_cat(vsb, "},\n"); + vcl_panic_conf(vsb, vcl->conf); VSB_indent(vsb, -2); VSB_cat(vsb, "},\n"); } diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index 53183489f..fa53089f0 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -64,35 +64,35 @@ pan_privs(struct vsb *vsb, const struct vrt_privs *privs) const struct vmod_priv *p; const struct vmod_priv_methods *m; - VSB_printf(vsb, "privs = %p {\n", privs); - if (PAN_already(vsb, privs)) + if (privs == NULL) { + VSB_printf(vsb, "privs = NULL\n"); return; + } + VSB_printf(vsb, "privs = %p {\n", privs); VSB_indent(vsb, 2); - if (privs != NULL) { - VRBT_FOREACH(vp, vrt_privs, privs) { - PAN_CheckMagic(vsb, vp, VRT_PRIV_MAGIC); - p = vp->priv; - //lint -e{774} - if (p == NULL) { - // should never happen - VSB_printf(vsb, "priv NULL vmod %jx\n", - (uintmax_t)vp->vmod_id); - continue; - } - m = p->methods; - VSB_printf(vsb, - "priv {p %p l %ld m %p t \"%s\"} vmod %jx\n", - p->priv, p->len, m, - m != NULL ? m->type : "", - (uintmax_t)vp->vmod_id - ); + VRBT_FOREACH(vp, vrt_privs, privs) { + if (PAN_dump_struct(vsb, vp, VRT_PRIV_MAGIC, "priv")) + continue; + p = vp->priv; + //lint -e{774} + if (p == NULL) { + // should never happen + VSB_printf(vsb, "priv NULL vmod %jx\n", + (uintmax_t)vp->vmod_id); + continue; } + m = p->methods; + VSB_printf(vsb, + "priv {p %p l %ld m %p t \"%s\"} vmod %jx\n", + p->priv, p->len, m, + m != NULL ? m->type : "", + (uintmax_t)vp->vmod_id + ); } VSB_indent(vsb, -2); VSB_cat(vsb, "},\n"); } - /*-------------------------------------------------------------------- */ diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index 36edfd68e..d1f54a2fd 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -430,14 +430,11 @@ WS_VSB_finish(struct vsb *vsb, struct ws *ws, size_t *szp) /*--------------------------------------------------------------------*/ void -WS_Panic(const struct ws *ws, struct vsb *vsb) +WS_Panic(struct vsb *vsb, const struct ws *ws) { - VSB_printf(vsb, "ws = %p {\n", ws); - if (PAN_already(vsb, ws)) + if (PAN_dump_struct(vsb, ws, WS_MAGIC, "ws")) return; - VSB_indent(vsb, 2); - PAN_CheckMagic(vsb, ws, WS_MAGIC); if (ws->id[0] != '\0' && (!(ws->id[0] & 0x20))) // cheesy islower() VSB_cat(vsb, "OVERFLOWED "); VSB_printf(vsb, "id = \"%s\",\n", ws->id); diff --git a/bin/varnishd/http2/cache_http2_panic.c b/bin/varnishd/http2/cache_http2_panic.c index ddc4eb761..e2a3e1e0a 100644 --- a/bin/varnishd/http2/cache_http2_panic.c +++ b/bin/varnishd/http2/cache_http2_panic.c @@ -45,12 +45,14 @@ h2_sess_panic(struct vsb *vsb, const struct sess *sp) AZ(SES_Get_proto_priv(sp, &up)); + AN(up); + h2 = (void*)*up; + if (PAN_dump_struct(vsb, h2, H2_SESS_MAGIC, "h2_sess")) + return; h2 = (void*)*up; - CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); - VSB_cat(vsb, "streams {\n"); - VSB_indent(vsb, 2); VTAILQ_FOREACH(r2, &h2->streams, list) { - PAN_CheckMagic(vsb, r2, H2_REQ_MAGIC); + if (PAN_dump_struct(vsb, r2, H2_REQ_MAGIC, "stream")) + continue; VSB_printf(vsb, "0x%08x", r2->stream); switch (r2->state) { #define H2_STREAM(U,sd,d) case H2_S_##U: VSB_printf(vsb, " %-6s", sd); break; @@ -59,8 +61,9 @@ h2_sess_panic(struct vsb *vsb, const struct sess *sp) VSB_printf(vsb, " State %d", r2->state); break; } - VSB_cat(vsb, "\n"); + VSB_cat(vsb, "},\n"); + VSB_indent(vsb, -2); } + VSB_cat(vsb, "},\n"); VSB_indent(vsb, -2); - VSB_cat(vsb, "}\n"); } From nils.goroll at uplex.de Thu Dec 3 16:03:08 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 3 Dec 2020 16:03:08 +0000 (UTC) Subject: [master] 6ba312289 update smartos build instructions Message-ID: <20201203160308.200027421@lists.varnish-cache.org> commit 6ba3122894989a378379e3e58234463899371b5e Author: Nils Goroll Date: Thu Dec 3 16:18:29 2020 +0100 update smartos build instructions diff --git a/doc/sphinx/installation/install_source.rst b/doc/sphinx/installation/install_source.rst index 8f11d33fe..f290dac66 100644 --- a/doc/sphinx/installation/install_source.rst +++ b/doc/sphinx/installation/install_source.rst @@ -154,10 +154,13 @@ Then continue `Compiling Varnish`_ Build dependencies on a SmartOS Zone ------------------------------------ -As of SmartOS pkgsrc 2017Q1, install the following packages:: +As of SmartOS pkgsrc 2019Q4, install the following packages:: - pkgin in autoconf automake libedit libtool ncurses \ - pcre py27-sphinx python27 gmake gcc49 pkg-config + pkgin in autoconf automake editline libtool ncurses \ + pcre python37 py37-sphinx py37-docutils gmake gcc8 pkg-config + +*Note:* you will probably need to add ``/opt/local/gcc8/bin`` to +``PATH`` in order to have ``gcc`` available. Optionally, to rebuild the svg files:: From nils.goroll at uplex.de Thu Dec 3 16:03:08 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 3 Dec 2020 16:03:08 +0000 (UTC) Subject: [master] e479ee0da fix libedit autoconf when configured via pkg-config Message-ID: <20201203160308.3A8C17424@lists.varnish-cache.org> commit e479ee0da0b77bc9a51d15fd60e1d6945f1a9f83 Author: Nils Goroll Date: Thu Dec 3 16:21:21 2020 +0100 fix libedit autoconf when configured via pkg-config diff --git a/configure.ac b/configure.ac index b63840512..3f7d43744 100644 --- a/configure.ac +++ b/configure.ac @@ -179,10 +179,17 @@ AC_CHECK_HEADERS([edit/readline/readline.h], LIBEDIT_LIBS="-ledit"], [PKG_CHECK_MODULES([LIBEDIT], [libedit], + [ # having the module does not imply having the header - [AC_CHECK_HEADERS([editline/readline.h], + AC_SUBST(LIBEDIT_CFLAGS) + AC_SUBST(LIBEDIT_LIBS) + save_CFLAGS="${CFLAGS}" + CFLAGS="${CFLAGS} ${LIBEDIT_CFLAGS}" + AC_CHECK_HEADERS([editline/readline.h], [AC_DEFINE([HAVE_LIBEDIT], [1], [Define if we have libedit])], - [AC_MSG_ERROR([Found libedit, but header file is missing. Hint: Install dev package?])])], + [AC_MSG_ERROR([Found libedit, but header file is missing. Hint: Install dev package?])]) + CFLAGS="${save_CFLAGS}" + ], [ # AX_LIB_READLINE overwrites LIBS which leads to every binary getting # linked against libreadline uselessly. So we re-use LIBEDIT_LIBS which From nils.goroll at uplex.de Thu Dec 3 16:03:08 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 3 Dec 2020 16:03:08 +0000 (UTC) Subject: [master] daf9b1979 fix curses autoconf when configured via pkg-config Message-ID: <20201203160308.55E867429@lists.varnish-cache.org> commit daf9b1979a3844f3a5be3841bb5342f3372748bc Author: Nils Goroll Date: Thu Dec 3 16:55:50 2020 +0100 fix curses autoconf when configured via pkg-config diff --git a/bin/varnishhist/Makefile.am b/bin/varnishhist/Makefile.am index 9fa07e569..79e681d8c 100644 --- a/bin/varnishhist/Makefile.am +++ b/bin/varnishhist/Makefile.am @@ -2,7 +2,8 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/include \ - -I$(top_builddir)/include + -I$(top_builddir)/include \ + @CURSES_CFLAGS@ bin_PROGRAMS = varnishhist diff --git a/bin/varnishstat/Makefile.am b/bin/varnishstat/Makefile.am index a6bc3eed9..94ad656aa 100644 --- a/bin/varnishstat/Makefile.am +++ b/bin/varnishstat/Makefile.am @@ -2,7 +2,8 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/include \ - -I$(top_builddir)/include + -I$(top_builddir)/include \ + @CURSES_CFLAGS@ bin_PROGRAMS = varnishstat varnishstat_help_gen diff --git a/bin/varnishtop/Makefile.am b/bin/varnishtop/Makefile.am index d26fd038f..cdb63b5f8 100644 --- a/bin/varnishtop/Makefile.am +++ b/bin/varnishtop/Makefile.am @@ -2,7 +2,8 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/include \ - -I$(top_builddir)/include + -I$(top_builddir)/include \ + @CURSES_CFLAGS@ bin_PROGRAMS = varnishtop diff --git a/configure.ac b/configure.ac index 3f7d43744..2417549ac 100644 --- a/configure.ac +++ b/configure.ac @@ -218,9 +218,13 @@ CURSES_LIBS="$CURSES_LIB" ]) ]) ]) +AC_SUBST([CURSES_CFLAGS]) AC_SUBST([CURSES_LIBS]) +save_CFLAGS="${CFLAGS}" +CFLAGS="${CFLAGS} ${CURSES_CFLAGS}" AC_CHECK_HEADERS([ncursesw/curses.h ncursesw.h ncurses/curses.h ncurses.h curses.h]) +CFLAGS="${save_CFLAGS}" # Checks for header files. AC_HEADER_STDC From nils.goroll at uplex.de Mon Dec 7 13:42:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 7 Dec 2020 13:42:07 +0000 (UTC) Subject: [master] 27b951bbc count all bans towards ban_cutoff Message-ID: <20201207134207.57316AE738@lists.varnish-cache.org> commit 27b951bbc4a724903b4d46a7516e915ef05b6407 Author: Nils Goroll Date: Mon Dec 7 14:14:04 2020 +0100 count all bans towards ban_cutoff ... also (C)ompleted bans. The previous code made a lot of sense in light of the performance issue due to too many bans, but would not work to prune a long ban list due to (R)equest bans at the tail, as in this example: Present bans: 1607346873.538879 354304 C 1607346873.532313 0 C ... lots of (C)ompleted bans 1607083561.980118 0 - req.http.Host ~ foo 1607083561.972629 15 C The documentation does not mention the previous behavior (that only "active" bans are being counted), so this changes aligns code to documentation. diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c index 480d5adfc..25bfa604c 100644 --- a/bin/varnishd/cache/cache_ban_lurker.c +++ b/bin/varnishd/cache/cache_ban_lurker.c @@ -371,7 +371,7 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl) d = VTIM_real() - cache_param->ban_lurker_age; bd = NULL; VTAILQ_INIT(&obans); - for (; b != NULL; b = VTAILQ_NEXT(b, list)) { + for (; b != NULL; b = VTAILQ_NEXT(b, list), count++) { if (bd != NULL) ban_lurker_test_ban(wrk, vsl, b, &obans, bd, count > cutoff ? 1 : 0); @@ -383,7 +383,6 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl) bd = VTAILQ_NEXT(b, list); continue; } - count++; n = ban_time(b->spec) - d; if (n < 0) { VTAILQ_INSERT_TAIL(&obans, b, l_list); From nils.goroll at uplex.de Mon Dec 7 13:42:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 7 Dec 2020 13:42:07 +0000 (UTC) Subject: [master] 32ffaad05 cstyle Message-ID: <20201207134207.6A30DAE73B@lists.varnish-cache.org> commit 32ffaad05afc03715e4e124ea5140b12e66e9d6a Author: Nils Goroll Date: Mon Dec 7 14:37:16 2020 +0100 cstyle diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c index 25bfa604c..d0990f410 100644 --- a/bin/varnishd/cache/cache_ban_lurker.c +++ b/bin/varnishd/cache/cache_ban_lurker.c @@ -377,8 +377,7 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl) count > cutoff ? 1 : 0); if (b->flags & BANS_FLAG_COMPLETED) continue; - if (b->flags & BANS_FLAG_REQ && - count <= cutoff) { + if (b->flags & BANS_FLAG_REQ && count <= cutoff) { if (bd != NULL) bd = VTAILQ_NEXT(b, list); continue; From nils.goroll at uplex.de Mon Dec 7 14:31:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 7 Dec 2020 14:31:07 +0000 (UTC) Subject: [master] a469b3de5 round-robin director to consider all backends also when racing Message-ID: <20201207143107.6CC97AFF24@lists.varnish-cache.org> commit a469b3de517579d6da5f190cd3e853fa6a7d79e8 Author: Nils Goroll Date: Thu Dec 3 15:15:20 2020 +0100 round-robin director to consider all backends also when racing When resolve requests race, we were not guaranteed to consider all backends because we updated a shared nxt variable. Fixes #3474 diff --git a/lib/libvmod_directors/round_robin.c b/lib/libvmod_directors/round_robin.c index 74ec567e0..230bcb02e 100644 --- a/lib/libvmod_directors/round_robin.c +++ b/lib/libvmod_directors/round_robin.c @@ -80,14 +80,16 @@ vmod_rr_resolve(VRT_CTX, VCL_BACKEND dir) CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(rr, dir->priv, VMOD_DIRECTORS_ROUND_ROBIN_MAGIC); vdir_rdlock(rr->vd); + nxt = rr->nxt; for (u = 0; u < rr->vd->n_backend; u++) { - nxt = rr->nxt % rr->vd->n_backend; - rr->nxt = nxt + 1; be = rr->vd->backend[nxt]; + nxt++; + nxt %= rr->vd->n_backend; CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC); if (VRT_Healthy(ctx, be, NULL)) break; } + rr->nxt = nxt; vdir_unlock(rr->vd); if (u == rr->vd->n_backend) be = NULL; From nils.goroll at uplex.de Mon Dec 7 16:44:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 7 Dec 2020 16:44:07 +0000 (UTC) Subject: [master] 95dd0bd41 Improve "None" backend documentation Message-ID: <20201207164407.40DDABE5C5@lists.varnish-cache.org> commit 95dd0bd413fa2cf42f4af974aa81ac3199b1c83e Author: Nils Goroll Date: Mon Dec 7 17:39:00 2020 +0100 Improve "None" backend documentation Fixes #3478 diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst index 5a640712f..311c6fd02 100644 --- a/doc/sphinx/users-guide/vcl-backends.rst +++ b/doc/sphinx/users-guide/vcl-backends.rst @@ -31,9 +31,41 @@ Varnish can have several backends defined you can even join several backends together into clusters of backends for load balancing purposes. -backends can also be empty or 'none' with the following syntax.:: +The "None" backend +------------------ + +Backends can also be declared as ``None`` with the following syntax::: + + backend default None; + +``None`` backends are special: + +* All backends declared ``None`` compare equal:: + + backend a None; + backend b None; - backend default none; + sub vcl_recv { + set req.backend_hint = a; + if (req.backend_hint == b) { + return (synth(200, "this is true")); + } + } + +* The ``None`` backend evaluates to ``false`` when used in a boolean + context:: + + backend nil None; + + sub vcl_recv { + set req.backend_hint = nil; + if (! req.backend_hint) { + return (synth(200, "We get here")); + } + } + +* When directors find no healthy backend, they typically return the + ``None`` backend Multiple backends ----------------- From nils.goroll at uplex.de Mon Dec 7 17:43:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 7 Dec 2020 17:43:07 +0000 (UTC) Subject: [master] d81611cc6 test in more detail how a pipe transaction looks like Message-ID: <20201207174307.B95A5513D@lists.varnish-cache.org> commit d81611cc62f576105524185fcfcef261cf69a1ca Author: Nils Goroll Date: Mon Dec 7 18:41:28 2020 +0100 test in more detail how a pipe transaction looks like Ref #3470 diff --git a/bin/varnishtest/tests/v00051.vtc b/bin/varnishtest/tests/v00051.vtc index a12d23ade..dca168794 100644 --- a/bin/varnishtest/tests/v00051.vtc +++ b/bin/varnishtest/tests/v00051.vtc @@ -148,10 +148,21 @@ varnish v1 -vcl+backend { } -logexpect l1 -v v1 -g raw { - expect * 1012 VCL_call "PIPE" - expect 0 1012 VCL_Error "Forced failure" - expect 0 1012 VCL_return "fail" +logexpect l2 -v v1 -g vxid -q "vxid == 1012" { + expect 0 1012 Begin {^bereq 1011 pipe} + expect 0 = BereqMethod {^GET} + expect 0 = BereqURL {^/} + expect 0 = BereqProtocol {^HTTP/1.1} + expect 0 = BereqHeader {^foo: pipe} + expect 0 = BereqHeader {^Host: } + expect 0 = BereqHeader {^X-Forwarded-For: } + expect 0 = BereqHeader {^X-Varnish: 1011} + expect 0 = BereqHeader {^Connection: close} + expect 0 = VCL_call {^PIPE} + expect 0 = VCL_Error {^Forced failure} + expect 0 = VCL_return {^fail} + expect 0 = BereqAcct {^0 0 0 0 0 0} + expect 0 = End } -start client c1 { @@ -164,7 +175,7 @@ client c1 { varnish v1 -expect vcl_fail == 5 varnish v1 -expect sc_vcl_failure == 5 -logexpect l1 -wait +logexpect l2 -wait ####################################################################### # Fail in vcl_pass, no handling in vcl_synth From nils.goroll at uplex.de Mon Dec 7 18:01:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 7 Dec 2020 18:01:07 +0000 (UTC) Subject: [master] 570608996 also test the client side of a pipe transaction Message-ID: <20201207180107.B715A5B44@lists.varnish-cache.org> commit 57060899672b21490b4dc4f5228278cad954717a Author: Nils Goroll Date: Mon Dec 7 18:59:45 2020 +0100 also test the client side of a pipe transaction Ref #3470 diff --git a/bin/varnishtest/tests/v00051.vtc b/bin/varnishtest/tests/v00051.vtc index dca168794..42c12ed3e 100644 --- a/bin/varnishtest/tests/v00051.vtc +++ b/bin/varnishtest/tests/v00051.vtc @@ -165,6 +165,28 @@ logexpect l2 -v v1 -g vxid -q "vxid == 1012" { expect 0 = End } -start +logexpect l3 -v v1 -g vxid -q "vxid == 1011" { + expect 0 1011 Begin {^req 1010 rxreq} + expect 0 = Timestamp {^Start: } + expect 0 = Timestamp {^Req: } + expect 0 = VCL_use {^vcl4} + expect 0 = ReqStart + expect 0 = ReqMethod {^GET} + expect 0 = ReqURL {^/} + expect 0 = ReqProtocol {^HTTP/1.1} + expect 0 = ReqHeader {^foo: pipe} + expect 0 = ReqHeader {^Host: } + expect 0 = ReqHeader {^X-Forwarded-For: } + expect 0 = VCL_call {^RECV} + expect 0 = VCL_return {^pipe} + expect 0 = VCL_call {^HASH} + expect 0 = VCL_return {^lookup} + expect 0 = Link {^bereq 1012 pipe} + expect 0 = RespProtocol {^HTTP/1.1} + expect 0 = RespStatus {^503} + expect 0 = RespReason {^VCL failed} +} -start + client c1 { txreq -hdr "foo: pipe" rxresp @@ -176,6 +198,7 @@ varnish v1 -expect vcl_fail == 5 varnish v1 -expect sc_vcl_failure == 5 logexpect l2 -wait +logexpect l3 -wait ####################################################################### # Fail in vcl_pass, no handling in vcl_synth From dridi.boukelmoune at gmail.com Wed Dec 9 13:29:09 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 9 Dec 2020 13:29:09 +0000 (UTC) Subject: [master] 3458d0ded doc: Polish the none backend section Message-ID: <20201209132909.2790AAE7D5@lists.varnish-cache.org> commit 3458d0ded07d5aeb558470905d35619504f19407 Author: Dridi Boukelmoune Date: Wed Dec 9 14:23:48 2020 +0100 doc: Polish the none backend section Going back to lowercase, and aligning code blocks to 4 spaces to match the rest of the code snippets. diff --git a/doc/sphinx/users-guide/vcl-backends.rst b/doc/sphinx/users-guide/vcl-backends.rst index 311c6fd02..65105f2af 100644 --- a/doc/sphinx/users-guide/vcl-backends.rst +++ b/doc/sphinx/users-guide/vcl-backends.rst @@ -31,41 +31,41 @@ Varnish can have several backends defined you can even join several backends together into clusters of backends for load balancing purposes. -The "None" backend +The "none" backend ------------------ -Backends can also be declared as ``None`` with the following syntax::: +Backends can also be declared as ``none`` with the following syntax::: - backend default None; + backend default none; -``None`` backends are special: +``none`` backends are special: -* All backends declared ``None`` compare equal:: +* All backends declared ``none`` compare equal:: - backend a None; - backend b None; + backend a none; + backend b none; sub vcl_recv { - set req.backend_hint = a; - if (req.backend_hint == b) { - return (synth(200, "this is true")); - } - } + set req.backend_hint = a; + if (req.backend_hint == b) { + return (synth(200, "this is true")); + } + } -* The ``None`` backend evaluates to ``false`` when used in a boolean +* The ``none`` backend evaluates to ``false`` when used in a boolean context:: - backend nil None; + backend nil none; sub vcl_recv { - set req.backend_hint = nil; - if (! req.backend_hint) { - return (synth(200, "We get here")); - } - } + set req.backend_hint = nil; + if (! req.backend_hint) { + return (synth(200, "We get here")); + } + } * When directors find no healthy backend, they typically return the - ``None`` backend + ``none`` backend Multiple backends ----------------- From nils.goroll at uplex.de Wed Dec 9 17:09:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 9 Dec 2020 17:09:07 +0000 (UTC) Subject: [master] 5fb3b4036 need stddef.h for size_t Message-ID: <20201209170907.4B265BF268@lists.varnish-cache.org> commit 5fb3b40362385aeaf75c54dcda4f494af0085d62 Author: Nils Goroll Date: Wed Dec 9 18:07:39 2020 +0100 need stddef.h for size_t Ref 001485e6705976bd1029279cb639a394c4dee59c diff --git a/include/vas.h b/include/vas.h index 32a092241..ca93fbf43 100644 --- a/include/vas.h +++ b/include/vas.h @@ -42,6 +42,7 @@ #include +#include // size_t const char * vstrerror(int e); From guillaume at varnish-software.com Wed Dec 9 23:36:09 2020 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Wed, 9 Dec 2020 23:36:09 +0000 (UTC) Subject: [master] f20d16429 [cci] PowerTools -> powertools Message-ID: <20201209233609.3D8836E001@lists.varnish-cache.org> commit f20d16429b7a6d165230ffc86246c841e86f7cf6 Author: Guillaume Quintard Date: Wed Dec 9 15:35:22 2020 -0800 [cci] PowerTools -> powertools diff --git a/.circleci/config.yml b/.circleci/config.yml index ea34dc7ab..71f8bb03a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -148,7 +148,7 @@ jobs: if [ << parameters.dist >> = centos ]; then if [ << parameters.release >> = 8 ]; then dnf install -y 'dnf-command(config-manager)' - yum config-manager --set-enabled PowerTools + yum config-manager --set-enabled powertools yum install -y diffutils python3-sphinx else yum install -y python-sphinx From nils.goroll at uplex.de Thu Dec 10 16:19:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 10 Dec 2020 16:19:07 +0000 (UTC) Subject: [master] 15f94d33c vsb: Speed VSB_cat() up using VSB_bcat() Message-ID: <20201210161907.C98E0B2A7C@lists.varnish-cache.org> commit 15f94d33c9ad7955bde94386e953d19bbf6fb4de Author: Dridi Boukelmoune Date: Tue Dec 8 16:26:59 2020 +0100 vsb: Speed VSB_cat() up using VSB_bcat() We don't need to assert the integrity of the VSB after every single byte, it can become very expensive depending on the workload. diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c index 0a185c9d6..0eb63ec8d 100644 --- a/lib/libvarnish/vsb.c +++ b/lib/libvarnish/vsb.c @@ -338,6 +338,8 @@ VSB_bcat(struct vsb *s, const void *buf, ssize_t len) int VSB_cat(struct vsb *s, const char *str) { + const char *nl; + size_t l; assert_VSB_integrity(s); assert_VSB_state(s, 0); @@ -347,12 +349,15 @@ VSB_cat(struct vsb *s, const char *str) if (s->s_error != 0) return (-1); - while (*str != '\0') { - VSB_put_byte(s, *str++); - if (s->s_error != 0) + while (s->s_indent > 0 && (nl = strchr(str, '\n')) != NULL) { + l = nl - str + 1; + if (VSB_bcat(s, str, l) < 0) return (-1); + str += l; } - return (0); + + l = strlen(str); + return (VSB_bcat(s, str, l)); } /* From nils.goroll at uplex.de Thu Dec 10 18:43:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 10 Dec 2020 18:43:06 +0000 (UTC) Subject: [master] b1f812182 gc duplicate call to VBP_Update_Backend() when creating backends Message-ID: <20201210184306.B933943B0@lists.varnish-cache.org> commit b1f812182bb03cb8ffbb5451e1220a729f8afc57 Author: Nils Goroll Date: Thu Dec 10 19:08:28 2020 +0100 gc duplicate call to VBP_Update_Backend() when creating backends VRT_new_backend_clustered() is the only caller of VBP_Insert(). There, VBP_Update_Backend() needs to be called after the director is created in order to update the director status for a cold VCL. For a warm VCL, VBP_Update_Backend() is called via VRT_AddDirector() -> VDI_Event() -> vbe_dir_event() -> VBP_Control() Thus, the additonal call before the director is added to the backend, does not do anything but update the poll bits. Somehow related to #3362 diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 8432c8582..8d7a9c9b0 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -695,7 +695,6 @@ VBP_Insert(struct backend *b, const struct vrt_backend_probe *vp, vbp_build_req(vt, vp, b); vbp_reset(vt); - VBP_Update_Backend(vt); } void From nils.goroll at uplex.de Thu Dec 10 18:43:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 10 Dec 2020 18:43:06 +0000 (UTC) Subject: [master] 17c148c12 flexelint Message-ID: <20201210184306.CC4AA43B3@lists.varnish-cache.org> commit 17c148c124a5bbdb50913274a23a461756c6853d Author: Nils Goroll Date: Thu Dec 10 19:40:21 2020 +0100 flexelint ... thinks this statement is confusing. Ref: 15f94d33c9ad7955bde94386e953d19bbf6fb4de diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c index 0eb63ec8d..6238d8349 100644 --- a/lib/libvarnish/vsb.c +++ b/lib/libvarnish/vsb.c @@ -350,7 +350,7 @@ VSB_cat(struct vsb *s, const char *str) return (-1); while (s->s_indent > 0 && (nl = strchr(str, '\n')) != NULL) { - l = nl - str + 1; + l = (nl - str) + 1; if (VSB_bcat(s, str, l) < 0) return (-1); str += l; From guillaume at varnish-software.com Fri Dec 11 05:53:08 2020 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Fri, 11 Dec 2020 05:53:08 +0000 (UTC) Subject: [master] c1ae437c4 rename the PowerTools leftovers Message-ID: <20201211055308.4DC7796442@lists.varnish-cache.org> commit c1ae437c4a5972b00070104321bbef81d98d70a3 Author: Guillaume Quintard Date: Thu Dec 10 21:52:44 2020 -0800 rename the PowerTools leftovers diff --git a/.circleci/make-rpm-packages.sh b/.circleci/make-rpm-packages.sh index 5f7d0af1a..56acdad05 100755 --- a/.circleci/make-rpm-packages.sh +++ b/.circleci/make-rpm-packages.sh @@ -18,7 +18,7 @@ yum install -y epel-release if [ "$PARAM_DIST" = centos ]; then if [ "$PARAM_RELEASE" = 8 ]; then dnf install -y 'dnf-command(config-manager)' - yum config-manager --set-enabled PowerTools + yum config-manager --set-enabled powertools fi fi diff --git a/doc/sphinx/installation/install_source.rst b/doc/sphinx/installation/install_source.rst index f290dac66..084ac44d3 100644 --- a/doc/sphinx/installation/install_source.rst +++ b/doc/sphinx/installation/install_source.rst @@ -87,10 +87,10 @@ Install sphinx * On Red Hat / CentOS 8, sphinx is not included in the default repositories, so execute these steps to include it from the - PowerTools repository:: + powertools repository:: sudo dnf install -y 'dnf-command(config-manager)' - sudo yum config-manager --set-enabled PowerTools + sudo yum config-manager --set-enabled powertools sudo yum install -y diffutils python3-sphinx * On Red Hat / CentOS <= 7, install sphinx:: From nils.goroll at uplex.de Fri Dec 11 16:43:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 11 Dec 2020 16:43:07 +0000 (UTC) Subject: [master] 16e1ea32b lower footprint of v3.vtc Message-ID: <20201211164307.44C85ABB7A@lists.varnish-cache.org> commit 16e1ea32b774f8a04222330553eb826d31883cac Author: Nils Goroll Date: Fri Dec 11 17:02:35 2020 +0100 lower footprint of v3.vtc it does not need many worker threads and runs for >30s. This helps being able to run this test with more concurrency to test #3362 diff --git a/bin/varnishtest/tests/v00003.vtc b/bin/varnishtest/tests/v00003.vtc index 4a8d15898..21fa9607e 100644 --- a/bin/varnishtest/tests/v00003.vtc +++ b/bin/varnishtest/tests/v00003.vtc @@ -7,7 +7,10 @@ server s1 -repeat 20 { } -start # The debug vmod logs temperature vcl events -varnish v1 -arg "-p vcl_cooldown=1" -vcl { +varnish v1 -arg "-p vcl_cooldown=1" \ + -arg "-p thread_pool_min=5" \ + -arg "-p thread_pool_max=5" \ + -vcl { import debug; backend default { .host = "${s1_addr}"; From nils.goroll at uplex.de Fri Dec 11 16:43:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 11 Dec 2020 16:43:07 +0000 (UTC) Subject: [master] a494a1e27 close a race between VBP_Control() and vbp_thread() wrt VBH index Message-ID: <20201211164307.5C997ABB7C@lists.varnish-cache.org> commit a494a1e2706f5fb642d77352b5285d8e3ea51075 Author: Nils Goroll Date: Fri Dec 11 17:04:59 2020 +0100 close a race between VBP_Control() and vbp_thread() wrt VBH index VBP_Control() asserts that, when the vcl is warm, probes are scheduled to run (that is, exist on the probe scheduling binheap) and not so when the VCL is cold. This is correct because any VCL events are guaranteed to be serialized by mgt. vbp_thread(), however, momentarily left probes removed from the binheap while scheduling a probe not holding vbp_mtx, which left a window for VBP_Control() to find an active probe not on the binheap. Both vbp_thread() and vbp_task() re-schedule the probe at hand. The former in case scheduling the task fails, and the latter to reschedule it at the right interval relative to the probe having finished. We now re-schedule the probe in vbp_thread() before adding a task to run it, which does not change anything about the above, and, in particular, keeps vbp_mtx free while adding the probe task. Yet it closes the race and thus fixes #3362 survived varnishtest -t 90 -n 10000 -j 200 -i tests/v00003.vtc diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 8d7a9c9b0..5aca2b4bb 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -487,6 +487,7 @@ vbp_thread(struct worker *wrk, void *priv) } else { VBH_delete(vbp_heap, vt->heap_idx); vt->due = now + vt->interval; + VBH_insert(vbp_heap, vt); if (!vt->running) { vt->running = 1; vt->task->func = vbp_task; @@ -497,7 +498,6 @@ vbp_thread(struct worker *wrk, void *priv) if (r) vt->running = 0; } - VBH_insert(vbp_heap, vt); } } NEEDLESS(Lck_Unlock(&vbp_mtx)); From nils.goroll at uplex.de Fri Dec 11 17:13:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 11 Dec 2020 17:13:06 +0000 (UTC) Subject: [master] b609a5376 for backends created on a cold VCL, hide counters Message-ID: <20201211171306.ABED5AC9BA@lists.varnish-cache.org> commit b609a537660def17237a0249db75c241ad760376 Author: Nils Goroll Date: Fri Dec 11 18:07:47 2020 +0100 for backends created on a cold VCL, hide counters they are revealed by vbe_dir_event() on a warm event Fixes #3358 diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 826ca94df..2fe784892 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -635,6 +635,8 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, be->vsc = VSC_vbe_New(vc, &be->vsc_seg, "%s.%s", VCL_Name(ctx->vcl), vrt->vcl_name); AN(be->vsc); + if (! vcl->temp->is_warm) + VRT_VSC_Hide(be->vsc_seg); be->tcp_pool = VTP_Ref(vep->ipv4, vep->ipv6, vep->uds_path, vrt_hash_be(vrt->endpoint)); From nils.goroll at uplex.de Tue Dec 15 09:05:09 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 15 Dec 2020 09:05:09 +0000 (UTC) Subject: [master] 43757b25c varnishncsa: handle inexistent fields Message-ID: <20201215090509.392EC97802@lists.varnish-cache.org> commit 43757b25cf1f0a3f005471c182165de099c6bbf3 Author: Nils Goroll Date: Tue Dec 15 09:57:53 2020 +0100 varnishncsa: handle inexistent fields fixes #3485 diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 520ea6dfb..9e2fc22d4 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -797,7 +797,8 @@ frag_fields(int force, const char *b, const char *e, ...) n++; } assert(p != NULL && q != NULL); - assert(p < e && q > p); + if (p >= e || q <= p) + continue; if (frag->gen != CTX.gen || force) { /* We only grab the same matching field once */ frag->gen = CTX.gen; diff --git a/bin/varnishtest/tests/u00003.vtc b/bin/varnishtest/tests/u00003.vtc index e6dac92f1..680b1fa7a 100644 --- a/bin/varnishtest/tests/u00003.vtc +++ b/bin/varnishtest/tests/u00003.vtc @@ -142,12 +142,12 @@ shell -match {^\d+.\d{6} miss miss c 1001 quuz %{Varnish:vxid}x %{VCL_Log:quux}x"} # %{VSL:..}x -shell -match {^req (\d+) rxreq \1 \d{10}\.\d{6} (\d+\.\d{6}) \d+\.\d{6} \2 - -req (\d+) rxreq \3 \d{10}\.\d{6} (\d+\.\d{6}) \d+\.\d{6} \4 - -req (\d+) rxreq \5 - - -$} \ +shell -match {^req (\d+) rxreq \1 \d{10}\.\d{6} (\d+\.\d{6}) \d+\.\d{6} \2 - - +req (\d+) rxreq \3 \d{10}\.\d{6} (\d+\.\d{6}) \d+\.\d{6} \4 - - +req (\d+) rxreq \5 - - - -$} \ {varnishncsa -n ${v1_name} -d -F "%{VSL:Begin}x \ %{VSL:Begin[2]}x %{VSL:Timestamp:Resp}x \ -%{VSL:Timestamp:Resp[2]}x %{VSL:Timestamp:foo}x"} +%{VSL:Timestamp:Resp[2]}x %{VSL:Timestamp:foo}x %{VSL:ReqURL[2]}x"} process p1 -stop -screen-dump process p1 -expect-text 1 0 {/1?foo=bar HTTP/1.1" 200 100 "-" "-"} From nils.goroll at uplex.de Wed Dec 16 16:58:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 16 Dec 2020 16:58:07 +0000 (UTC) Subject: [master] 249886ea0 make m49.vtc agnostic to log ordering Message-ID: <20201216165807.E68B2A9150@lists.varnish-cache.org> commit 249886ea030e44798941b47e24f83341d52a48f3 Author: Nils Goroll Date: Wed Dec 16 17:51:20 2020 +0100 make m49.vtc agnostic to log ordering false negatives reported by vtest: ** top === logexpect l1 -wait ** l1 Waiting for logexp **** l1 match| 1005 VCL_use c vcl1 *** l1 expecting| expect 0 = ReqURL struct **** l1 err | 1005 ReqURL c /encode ---- l1 bad| expectation failed diff --git a/bin/varnishtest/tests/m00049.vtc b/bin/varnishtest/tests/m00049.vtc index 8a3dfd4f5..cc6ba8b08 100644 --- a/bin/varnishtest/tests/m00049.vtc +++ b/bin/varnishtest/tests/m00049.vtc @@ -93,7 +93,10 @@ client c-sub { # their first argument. Meaning we make a comprehensive check of all the log # records that aren't filtered out. -logexpect l1 -v v1 -i ReqURL,VCL_Error,VCL_Log,VCL_use +logexpect l1 -v v1 -i ReqURL,VCL_Error,VCL_Log,VCL_use -q "ReqURL ~ decode" +logexpect l2 -v v1 -i ReqURL,VCL_Error,VCL_Log,VCL_use -q "ReqURL ~ struct" +logexpect l3 -v v1 -i ReqURL,VCL_Error,VCL_Log,VCL_use -q "ReqURL ~ encode" +logexpect l4 -v v1 -i ReqURL,VCL_Error,VCL_Log,VCL_use -q "ReqURL ~ req.hash" # IDENTITY codec @@ -128,10 +131,14 @@ logexpect l1 { expect 0 = ReqURL decode expect 0 = VCL_Log shrink expect 0 = VCL_Error "cannot decode, out of space" +} -start +logexpect l2 { expect 0 * VCL_use vcl1 expect 0 = ReqURL struct expect 0 = VCL_Log shrink expect 0 = VCL_Error "Workspace overflow .blob.decode." +} -start +logexpect l3 { expect 0 * VCL_use vcl1 expect 0 = ReqURL encode expect 0 = VCL_Log shrink @@ -143,6 +150,8 @@ client c-struct -run client c-encode -run logexpect l1 -wait +logexpect l2 -wait +logexpect l3 -wait # BASE64 codec @@ -178,10 +187,14 @@ logexpect l1 { expect 0 = ReqURL decode expect 0 = VCL_Log shrink expect 0 = VCL_Error "cannot decode, out of space" +} -start +logexpect l2 { expect 0 * VCL_use vcl2 expect 0 = ReqURL struct expect 0 = VCL_Log shrink expect 0 = VCL_Error "Workspace overflow .blob.decode." +} -start +logexpect l3 { expect 0 * VCL_use vcl2 expect 0 = ReqURL encode expect 0 = VCL_Log shrink @@ -193,6 +206,8 @@ client c-struct -run client c-encode -run logexpect l1 -wait +logexpect l2 -wait +logexpect l3 -wait # URL codec @@ -227,10 +242,14 @@ logexpect l1 { expect 0 = ReqURL decode expect 0 = VCL_Log shrink expect 0 = VCL_Error "cannot decode, out of space" +} -start +logexpect l2 { expect 0 * VCL_use vcl3 expect 0 = ReqURL struct expect 0 = VCL_Log shrink expect 0 = VCL_Error "Workspace overflow .blob.decode." +} -start +logexpect l3 { expect 0 * VCL_use vcl3 expect 0 = ReqURL encode expect 0 = VCL_Log shrink @@ -242,6 +261,8 @@ client c-struct -run client c-encode -run logexpect l1 -wait +logexpect l2 -wait +logexpect l3 -wait # HEX codec @@ -277,10 +298,14 @@ logexpect l1 { expect 0 = ReqURL decode expect 0 = VCL_Log shrink expect 0 = VCL_Error "cannot decode, out of space" +} -start +logexpect l2 { expect 0 * VCL_use vcl4 expect 0 = ReqURL struct expect 0 = VCL_Log shrink expect 0 = VCL_Error "Workspace overflow .blob.decode." +} -start +logexpect l3 { expect 0 * VCL_use vcl4 expect 0 = ReqURL encode expect 0 = VCL_Log shrink @@ -292,6 +317,8 @@ client c-struct -run client c-encode -run logexpect l1 -wait +logexpect l2 -wait +logexpect l3 -wait # blob.sub() function @@ -313,11 +340,13 @@ varnish v1 -vcl { } } -logexpect l1 { +logexpect l4 { expect 0 * VCL_use vcl5 expect 0 = ReqURL req.hash expect 0 = VCL_Log shrink expect 0 = VCL_Error "Workspace overflow .req.hash." +} -start +logexpect l2 { expect 0 * VCL_use vcl5 expect 0 = ReqURL struct expect 0 = VCL_Log shrink @@ -327,4 +356,5 @@ logexpect l1 { client c-req-hash -run client c-struct -run -logexpect l1 -wait +logexpect l4 -wait +logexpect l2 -wait From nils.goroll at uplex.de Wed Dec 16 16:58:08 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 16 Dec 2020 16:58:08 +0000 (UTC) Subject: [master] 8cf192d5b try to stabilize m17.vtc Message-ID: <20201216165808.081ABA9153@lists.varnish-cache.org> commit 8cf192d5bfae1242949f9b89094596ffa84a18e4 Author: Nils Goroll Date: Wed Dec 16 17:54:57 2020 +0100 try to stabilize m17.vtc seen in vtest: ** top === server s1 -repeat 5 { ** s1 Starting server ---- s1 Server listen address (127.0.0.1 58150) cannot be resolved: bind(2) NB: I am not sure about this attempt diff --git a/bin/varnishtest/tests/m00017.vtc b/bin/varnishtest/tests/m00017.vtc index 0f1ddba47..78a47e49f 100644 --- a/bin/varnishtest/tests/m00017.vtc +++ b/bin/varnishtest/tests/m00017.vtc @@ -130,7 +130,7 @@ varnish v1 -vcl+backend { } } -server s1 -repeat 2 { +server s1 -repeat 2 -keepalive { rxreq expect req.url == "/2" txresp @@ -172,7 +172,7 @@ varnish v1 -vcl+backend { } } -server s1 -repeat 5 { +server s1 -repeat 5 -keepalive { rxreq txresp } -start @@ -225,7 +225,7 @@ varnish v1 -vcl+backend { } # XXX server keeps params, so we need to reset previous -repeat 5 -server s1 -repeat 3 { +server s1 -repeat 3 -keepalive { rxreq expect req.url ~ "^/6" txresp @@ -307,7 +307,7 @@ varnish v1 -vcl+backend { } } -server s1 -repeat 3 { +server s1 -repeat 3 -keepalive { rxreq txresp } -start From nils.goroll at uplex.de Wed Dec 16 17:09:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 16 Dec 2020 17:09:07 +0000 (UTC) Subject: [master] bffb264d9 stabilize b40.vtc Message-ID: <20201216170907.180EFA98A3@lists.varnish-cache.org> commit bffb264d95beec4b8e44793e9571604b7b5fd08f Author: Nils Goroll Date: Wed Dec 16 18:04:49 2020 +0100 stabilize b40.vtc seen in vtest: ** l1 Waiting for logexp **** dT 1.477 **** l1 match| 1016 BogoHeader c Missing header name: : Header *** l1 expecting| expect * 1018 VCL_Error Bad header foo: /9 overtook /8 diff --git a/bin/varnishtest/tests/b00040.vtc b/bin/varnishtest/tests/b00040.vtc index 902bcc8ca..7edce6509 100644 --- a/bin/varnishtest/tests/b00040.vtc +++ b/bin/varnishtest/tests/b00040.vtc @@ -99,6 +99,7 @@ client c1 { rxresp expect resp.status == 400 } -run +delay .1 client c1 { txreq -url /9 From nils.goroll at uplex.de Mon Dec 21 13:47:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 21 Dec 2020 13:47:07 +0000 (UTC) Subject: [master] 6cf1138fd refactor VRT_ban_string() error reporting Message-ID: <20201221134707.B44AB92C10@lists.varnish-cache.org> commit 6cf1138fd0c397bb59ad4761a31c17087f32a292 Author: Nils Goroll Date: Mon Dec 21 14:44:36 2020 +0100 refactor VRT_ban_string() error reporting more re-use coming up in a PR diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index f5fa84ea3..3ceb7b775 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -829,6 +829,16 @@ VRT_synth_page(VRT_CTX, VCL_STRANDS s) /*--------------------------------------------------------------------*/ +static VCL_VOID +ban_error(VRT_CTX, VCL_STRING err) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(ctx->vsl); + AN(err); + + VSLb(ctx->vsl, SLT_VCL_Error, "ban(): %s", err); +} + VCL_VOID VRT_ban_string(VRT_CTX, VCL_STRING str) { @@ -839,22 +849,21 @@ VRT_ban_string(VRT_CTX, VCL_STRING str) int i; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - AN(ctx->vsl); if (str == NULL) { - VSLb(ctx->vsl, SLT_VCL_Error, "ban(): Null argument"); + ban_error(ctx, "Null argument"); return; } bp = BAN_Build(); if (bp == NULL) { - VSLb(ctx->vsl, SLT_VCL_Error, "ban(): Out of Memory"); + ban_error(ctx, "Out of Memory"); return; } av = VAV_Parse(str, NULL, ARGV_NOESC); AN(av); if (av[0] != NULL) { - VSLb(ctx->vsl, SLT_VCL_Error, "ban(): %s", av[0]); + ban_error(ctx, av[0]); VAV_Free(av); BAN_Abandon(bp); return; @@ -862,25 +871,22 @@ VRT_ban_string(VRT_CTX, VCL_STRING str) for (i = 0; ;) { a1 = av[++i]; if (a1 == NULL) { - VSLb(ctx->vsl, SLT_VCL_Error, - "ban(): No ban conditions found."); + ban_error(ctx, "No ban conditions found."); break; } a2 = av[++i]; if (a2 == NULL) { - VSLb(ctx->vsl, SLT_VCL_Error, - "ban(): Expected comparison operator."); + ban_error(ctx, "Expected comparison operator."); break; } a3 = av[++i]; if (a3 == NULL) { - VSLb(ctx->vsl, SLT_VCL_Error, - "ban(): Expected second operand."); + ban_error(ctx, "Expected second operand."); break; } err = BAN_AddTest(bp, a1, a2, a3); - if (err) { - VSLb(ctx->vsl, SLT_VCL_Error, "ban(): %s", err); + if (err != NULL) { + ban_error(ctx, err); break; } if (av[++i] == NULL) { @@ -888,10 +894,11 @@ VRT_ban_string(VRT_CTX, VCL_STRING str) if (err == NULL) bp = NULL; else - VSLb(ctx->vsl, SLT_VCL_Error, "ban(): %s", err); + ban_error(ctx, err); break; } if (strcmp(av[i], "&&")) { + // XXX refactoring pending via PR VSLb(ctx->vsl, SLT_VCL_Error, "ban(): Expected && between conditions," " found \"%s\"", av[i]); From nils.goroll at uplex.de Wed Dec 23 09:46:08 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 23 Dec 2020 09:46:08 +0000 (UTC) Subject: [master] cd0484548 Panic: dump basic pool info Message-ID: <20201223094608.EBE5B612EF@lists.varnish-cache.org> commit cd048454894466e979bc96c022bff9be9d968786 Author: Nils Goroll Date: Wed Dec 23 10:35:38 2020 +0100 Panic: dump basic pool info This is to chase a new case of the watchdog off the leash diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index e20e07f80..91268deaa 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -788,6 +788,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond, if (bo != NULL) VSL_Flush(bo->vsl, 0); VMOD_Panic(pan_vsb); + pan_pool(pan_vsb); } else { VSB_cat(pan_vsb, "Feature short panic suppressed details.\n"); } diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index bfdddccb0..7cc42b0cd 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -239,6 +239,27 @@ pool_poolherder(void *priv) NEEDLESS(return (NULL)); } +/*--------------------------------------------------------------------*/ +void +pan_pool(struct vsb *vsb) +{ + struct pool *pp; + + VSB_printf(vsb, "pools = {\n"); + VSB_indent(vsb, 2); + VTAILQ_FOREACH(pp, &pools, list) { + if (PAN_dump_struct(vsb, pp, POOL_MAGIC, "pool")) + continue; + VSB_printf(vsb, "nidle = %u,\n", pp->nidle); + VSB_printf(vsb, "nthr = %u,\n", pp->nthr); + VSB_printf(vsb, "lqueue = %u\n", pp->lqueue); + VSB_indent(vsb, -2); + VSB_printf(vsb, "},\n"); + } + VSB_indent(vsb, -2); + VSB_printf(vsb, "},\n"); +} + /*--------------------------------------------------------------------*/ void diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 84316cee8..d74f6ed5b 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -361,6 +361,7 @@ void Pool_Sumstat(const struct worker *w); int Pool_TrySumstat(const struct worker *wrk); void Pool_PurgeStat(unsigned nobj); int Pool_Task_Any(struct pool_task *task, enum task_prio prio); +void pan_pool(struct vsb *); /* cache_req.c */ struct req *Req_New(const struct worker *, struct sess *); From nils.goroll at uplex.de Wed Dec 23 11:56:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 23 Dec 2020 11:56:06 +0000 (UTC) Subject: [master] b94e12a58 plug insignificant vcc leak Message-ID: <20201223115606.9D98B64C13@lists.varnish-cache.org> commit b94e12a58a248f9f976398aeaa04720a70267c40 Author: Nils Goroll Date: Wed Dec 23 12:54:54 2020 +0100 plug insignificant vcc leak reported by coverity diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index 795a7571d..47c0b4ef2 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -277,6 +277,7 @@ vcc_acl_try_getaddrinfo(struct vcc *tl, struct acl_e *ae) "Mask (/%u) specified, but string resolves to" " both IPv4 and IPv6 addresses.\n", ae->mask); vcc_ErrWhere(tl, ae->t_mask); + freeaddrinfo(res0); return; } From nils.goroll at uplex.de Mon Dec 28 15:22:08 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 28 Dec 2020 15:22:08 +0000 (UTC) Subject: [master] 98f55dae5 vep->state == VEP_MATCH implies vep->match != NULL Message-ID: <20201228152208.5D21D62E05@lists.varnish-cache.org> commit 98f55dae5f3e38358a5a06ecd6c492b897203fac Author: Nils Goroll Date: Mon Dec 28 15:47:35 2020 +0100 vep->state == VEP_MATCH implies vep->match != NULL Coverity CID 1099620 diff --git a/bin/varnishd/cache/cache_esi_parse.c b/bin/varnishd/cache/cache_esi_parse.c index 995087aa5..0bda28c84 100644 --- a/bin/varnishd/cache/cache_esi_parse.c +++ b/bin/varnishd/cache/cache_esi_parse.c @@ -225,6 +225,7 @@ vep_match(const struct vep_state *vep, const char *b, const char *e) struct vep_match *vm; const char *q, *r; + AN(vep->match); for (vm = vep->match; vm->match != NULL; vm++) { assert(strlen(vm->match) <= sizeof (vep->tag)); r = b; From nils.goroll at uplex.de Mon Dec 28 18:03:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 28 Dec 2020 18:03:06 +0000 (UTC) Subject: [master] 2ce736790 Centralize delete session by htc status for reuse Message-ID: <20201228180306.8C1CD915BD@lists.varnish-cache.org> commit 2ce736790d413a90ec18da04d2a190b20d26176f Author: Nils Goroll Date: Mon Dec 28 18:09:00 2020 +0100 Centralize delete session by htc status for reuse adds handling of HTC_S_JUNK diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index e9ca3f215..a50e1c295 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -586,6 +586,34 @@ SES_Delete(struct sess *sp, enum sess_close reason, vtim_real now) SES_Rel(sp); } +void +SES_DeleteHS(struct sess *sp, enum htc_status_e hs, vtim_real now) +{ + enum sess_close reason; + + switch (hs) { + case HTC_S_JUNK: + reason = SC_RX_JUNK; + break; + case HTC_S_CLOSE: + reason = SC_REM_CLOSE; + break; + case HTC_S_TIMEOUT: + reason = SC_RX_TIMEOUT; + break; + case HTC_S_OVERFLOW: + reason = SC_RX_OVERFLOW; + break; + case HTC_S_EOF: + reason = SC_REM_CLOSE; + break; + default: + WRONG("htc_status (bad)"); + } + SES_Delete(sp, reason, now); +} + + /*-------------------------------------------------------------------- */ diff --git a/bin/varnishd/cache/cache_transport.h b/bin/varnishd/cache/cache_transport.h index 9d8f9dfc5..36502911a 100644 --- a/bin/varnishd/cache/cache_transport.h +++ b/bin/varnishd/cache/cache_transport.h @@ -82,6 +82,7 @@ int VPX_Send_Proxy(int fd, int version, const struct sess *); /* cache_session.c */ struct sess *SES_New(struct pool *); void SES_Delete(struct sess *, enum sess_close reason, vtim_real now); +void SES_DeleteHS(struct sess *, enum htc_status_e hs, vtim_real now); void SES_Close(struct sess *, enum sess_close reason); void SES_SetTransport(struct worker *, struct sess *, struct req *, const struct transport *); diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c index de3662f1e..1d1e7c2e6 100644 --- a/bin/varnishd/http1/cache_http1_fsm.c +++ b/bin/varnishd/http1/cache_http1_fsm.c @@ -325,22 +325,8 @@ HTTP1_Session(struct worker *wrk, struct req *req) req->htc->rxbuf_e - req->htc->rxbuf_b; Req_AcctLogCharge(wrk->stats, req); Req_Release(req); - switch (hs) { - case HTC_S_CLOSE: - SES_Delete(sp, SC_REM_CLOSE, NAN); - return; - case HTC_S_TIMEOUT: - SES_Delete(sp, SC_RX_TIMEOUT, NAN); - return; - case HTC_S_OVERFLOW: - SES_Delete(sp, SC_RX_OVERFLOW, NAN); - return; - case HTC_S_EOF: - SES_Delete(sp, SC_REM_CLOSE, NAN); - return; - default: - WRONG("htc_status (bad)"); - } + SES_DeleteHS(sp, hs, NAN); + return; } if (hs == HTC_S_IDLE) { wrk->stats->sess_herd++; From nils.goroll at uplex.de Mon Dec 28 18:03:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 28 Dec 2020 18:03:06 +0000 (UTC) Subject: [master] 5764e6c53 be more specific about the reason for a failed PROXYv2 session Message-ID: <20201228180306.A1D1D915C0@lists.varnish-cache.org> commit 5764e6c5353880fee9968b10f34653672e41a63f Author: Nils Goroll Date: Mon Dec 28 18:20:42 2020 +0100 be more specific about the reason for a failed PROXYv2 session We reported any PROXY connection failure as JUNK. Now we report the same reasons as for failing HTTP1 connections. Implmentation: SES_DeleteHS() asserts that the hs argument be negative (one of JUNK, CLOSE, TIMEOUT, OVERFLOW, EOF). HTC_RxStuff() may return OVERFLOW, JUNK, COMPLETE, EOF or TIMEOUT (or, instead of TIMEOUT, IDLE if the callback returned EMPTY). Because vpx_complete() only returns JUNK, OVERFLOW or MORE, using the HTC_RxStuff() return value for SES_DeleteHS() is safe if different from COMPLETE. diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index e9a2ce164..c31fb6131 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -537,7 +537,7 @@ vpx_new_session(struct worker *wrk, void *arg) 1024); // XXX ? if (hs != HTC_S_COMPLETE) { Req_Release(req); - SES_Delete(sp, SC_RX_JUNK, NAN); + SES_DeleteHS(sp, hs, NAN); return; } p = req->htc->rxbuf_b; diff --git a/bin/varnishtest/tests/o00001.vtc b/bin/varnishtest/tests/o00001.vtc index ef78b2d95..bc07ac3e4 100644 --- a/bin/varnishtest/tests/o00001.vtc +++ b/bin/varnishtest/tests/o00001.vtc @@ -73,6 +73,8 @@ logexpect l1 -v v1 -g raw { expect * 1013 ProxyGarbage "PROXY2: Ignoring short IPv6 addresses \\(35\\)" expect * 1016 Proxy "2 1.2.3.4 2314 5.6.7.8 2828" expect * 1019 Proxy "2 102:304:506::d0e:f10 2314 8182:8384:8586::8d8e:8f80 2828" + expect * 1022 Begin "^sess 0 PROXY" + expect 1 1022 SessClose "^RX_OVERFLOW" } -start client c1 { @@ -197,4 +199,17 @@ client c1 \ delay .1 +client c2 { + # max length with garbage + sendhex "0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a" + # annouce 1025 bytes + sendhex "20 00 04 01" + # 1024 bytes implicit proxy hdr limit + send_n 64 "0123456789abcdef" + timeout 8 + expect_close +} -run + +delay .1 + logexpect l1 -wait From nils.goroll at uplex.de Mon Dec 28 18:03:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 28 Dec 2020 18:03:06 +0000 (UTC) Subject: [master] 19785d6bb make the max PROXY payload length more explicit Message-ID: <20201228180306.BB76F915C4@lists.varnish-cache.org> commit 19785d6bb8a467bb979b8a3ebe7dd804afe4c0ca Author: Nils Goroll Date: Mon Dec 28 18:39:11 2020 +0100 make the max PROXY payload length more explicit diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index c31fb6131..870c84130 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -45,6 +45,9 @@ #include "vss.h" #include "vtcp.h" +// max. PROXY payload length (excl. sig) - XXX parameter? +#define VPX_MAX_LEN 1024 + struct vpx_tlv { unsigned magic; #define VPX_TLV_MAGIC 0xdeb9a4a5 @@ -532,9 +535,8 @@ vpx_new_session(struct worker *wrk, void *arg) assert(sizeof vpx2_sig == 12); HTC_RxInit(req->htc, req->ws); - hs = HTC_RxStuff(req->htc, vpx_complete, - NULL, NULL, NAN, sp->t_idle + cache_param->timeout_idle, NAN, - 1024); // XXX ? + hs = HTC_RxStuff(req->htc, vpx_complete, NULL, NULL, NAN, + sp->t_idle + cache_param->timeout_idle, NAN, VPX_MAX_LEN); if (hs != HTC_S_COMPLETE) { Req_Release(req); SES_DeleteHS(sp, hs, NAN); From nils.goroll at uplex.de Mon Dec 28 18:03:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 28 Dec 2020 18:03:06 +0000 (UTC) Subject: [master] 3a5af9721 scrutinize PROXY header length Message-ID: <20201228180306.D5BA7915C9@lists.varnish-cache.org> commit 3a5af972189b12bb7e16f529e651fb34834c4ceb Author: Nils Goroll Date: Mon Dec 28 18:46:29 2020 +0100 scrutinize PROXY header length ref. coverity CID 1430125 diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index 870c84130..b468c09d8 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -335,7 +335,8 @@ vpx_proto2(const struct worker *wrk, struct req *req) char pb[VTCP_PORTBUFSIZE]; struct vpx_tlv_iter vpi[1], vpi2[1]; struct vpx_tlv *tlv; - unsigned l, hdr_len, flen, alen; + uint16_t l; + unsigned hdr_len, flen, alen; unsigned const plen = 2, aoff = 16; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -344,6 +345,7 @@ vpx_proto2(const struct worker *wrk, struct req *req) assert(req->htc->rxbuf_e - req->htc->rxbuf_b >= 16L); l = vbe16dec(req->htc->rxbuf_b + 14); + assert(l <= VPX_MAX_LEN); // vpx_complete() hdr_len = l + 16L; assert(req->htc->rxbuf_e >= req->htc->rxbuf_b + hdr_len); HTC_RxPipeline(req->htc, req->htc->rxbuf_b + hdr_len); @@ -479,7 +481,7 @@ static enum htc_status_e v_matchproto_(htc_complete_f) vpx_complete(struct http_conn *htc) { size_t z, l; - unsigned j; + uint16_t j; char *p, *q; CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); @@ -508,6 +510,8 @@ vpx_complete(struct http_conn *htc) if (l < 16) return (HTC_S_MORE); j = vbe16dec(p + 14); + if (j > VPX_MAX_LEN) + return (HTC_S_OVERFLOW); if (l < 16L + j) return (HTC_S_MORE); return (HTC_S_COMPLETE); diff --git a/bin/varnishtest/tests/o00001.vtc b/bin/varnishtest/tests/o00001.vtc index bc07ac3e4..5a6b35913 100644 --- a/bin/varnishtest/tests/o00001.vtc +++ b/bin/varnishtest/tests/o00001.vtc @@ -202,11 +202,8 @@ delay .1 client c2 { # max length with garbage sendhex "0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a" - # annouce 1025 bytes + # annouce 1025 bytes > 1024 implicit limit sendhex "20 00 04 01" - # 1024 bytes implicit proxy hdr limit - send_n 64 "0123456789abcdef" - timeout 8 expect_close } -run From nils.goroll at uplex.de Tue Dec 29 10:07:08 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 29 Dec 2020 10:07:08 +0000 (UTC) Subject: [master] 4c987be0d Declare variable length of struct vpx_tlv Message-ID: <20201229100708.7DCD3B1B9A@lists.varnish-cache.org> commit 4c987be0d564d8c1f464f02ec4b60a91aec6b135 Author: Nils Goroll Date: Tue Dec 29 10:59:56 2020 +0100 Declare variable length of struct vpx_tlv 3a5af972189b12bb7e16f529e651fb34834c4ceb enabled flexelint to complain about copying many bytes into a struct member declared as a single byte array (which it never was). warning addressed: proxy/cache_proxy_proto.c 469 Warning 669: Possible data overrun for function 'memcpy(void *, const void *, unsigned long)', argument 3 (size=988) exceeds argument 1 (size=1) [Reference: file proxy/cache_proxy_proto.c: lines 340, 348, 392, 401, 410, 439, 469] diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index b468c09d8..7d6cffe00 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -52,7 +52,7 @@ struct vpx_tlv { unsigned magic; #define VPX_TLV_MAGIC 0xdeb9a4a5 unsigned len; - char tlv[1]; + char tlv[]; }; static inline int From nils.goroll at uplex.de Tue Dec 29 10:48:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 29 Dec 2020 10:48:07 +0000 (UTC) Subject: [master] a35554803 Try to clarify for coverity how vsl_delseg() works Message-ID: <20201229104807.52E01B2D1C@lists.varnish-cache.org> commit a355548039182446ac94ff82ec9f311ba72005bb Author: Nils Goroll Date: Tue Dec 29 11:29:57 2020 +0100 Try to clarify for coverity how vsl_delseg() works Ref CID 1430127, CID 1430118 diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c index 6ec9fd536..d9e67ef32 100644 --- a/lib/libvarnishapi/vsm.c +++ b/lib/libvarnishapi/vsm.c @@ -304,6 +304,7 @@ static void vsm_delset(struct vsm_set **p) { struct vsm_set *vs; + struct vsm_seg *vg; AN(p); vs = *p; @@ -312,10 +313,14 @@ vsm_delset(struct vsm_set **p) closefd(&vs->fd); if (vs->dfd >= 0) closefd(&vs->dfd); - while (!VTAILQ_EMPTY(&vs->stale)) - vsm_delseg(VTAILQ_FIRST(&vs->stale), 0); - while (!VTAILQ_EMPTY(&vs->segs)) - vsm_delseg(VTAILQ_FIRST(&vs->segs), 0); + while ((vg = VTAILQ_FIRST(&vs->stale)) != NULL) { + AN(vg->flags & VSM_FLAG_STALE); + vsm_delseg(vg, 0); + } + while ((vg = VTAILQ_FIRST(&vs->segs)) != NULL) { + AZ(vg->flags & VSM_FLAG_STALE); + vsm_delseg(vg, 0); + } assert(VTAILQ_EMPTY(&vs->clusters)); VLU_Destroy(&vs->vlu); FREE_OBJ(vs); From nils.goroll at uplex.de Tue Dec 29 10:48:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 29 Dec 2020 10:48:07 +0000 (UTC) Subject: [master] 776e8c933 Try to clarify for coverity how mgt_vcl_dep_del() works Message-ID: <20201229104807.7BEC2B2D1F@lists.varnish-cache.org> commit 776e8c9334317f784bff65cb95d6a7ffde84449b Author: Nils Goroll Date: Tue Dec 29 11:44:07 2020 +0100 Try to clarify for coverity how mgt_vcl_dep_del() works Ref CID 1400491 diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index ab635d7fc..5b8a57115 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -226,14 +226,17 @@ mgt_vcl_del(struct vclprog *vp) char *p; struct vmoddep *vd; struct vmodfile *vf; + struct vcldep *dep; CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC); assert(VTAILQ_EMPTY(&vp->dto)); mgt_vcl_symtab_clean(vp); - while (!VTAILQ_EMPTY(&vp->dfrom)) - mgt_vcl_dep_del(VTAILQ_FIRST(&vp->dfrom)); + while ((dep = VTAILQ_FIRST(&vp->dfrom)) != NULL) { + assert(dep->from == vp); + mgt_vcl_dep_del(dep); + } VTAILQ_REMOVE(&vclhead, vp, list); if (vp->state != VCL_STATE_LABEL) From fgsch at lodoss.net Wed Dec 30 00:23:09 2020 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 30 Dec 2020 00:23:09 +0000 (UTC) Subject: [master] 9d6e0b759 Workaround failure under macos Message-ID: <20201230002309.ECB819CEF6@lists.varnish-cache.org> commit 9d6e0b7597d8a3787eacb442f6bca979bd782e0d Author: Federico G. Schwindt Date: Wed Dec 30 00:17:09 2020 +0000 Workaround failure under macos Apple's clang calls dsymutil(1) when -g is used, creating a .dSYM directory which breaks this test. As I haven't found a way to disable it just ignore this check if the directory is present. diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc index 570c778d3..29e602253 100644 --- a/bin/varnishtest/tests/u00000.vtc +++ b/bin/varnishtest/tests/u00000.vtc @@ -4,7 +4,8 @@ shell "varnishd -b 127.0.0.1:80 -C 2> ${tmpdir}/_.c" shell { varnishd -n ${tmpdir}/no_keep -C -b 127.0.0.1:80 2> no_keep.c - test -s no_keep.c && ! test -d no_keep || test -f no_keep/*/vgc.gcda + test -s no_keep.c && ! test -d no_keep || \ + (test -f no_keep/*/vgc.gcda || test -d no_keep/*/vgc.so.dSYM) } shell {