From phk at FreeBSD.org Mon Aug 1 11:07:08 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 1 Aug 2022 11:07:08 +0000 (UTC) Subject: [master] 9843d650a Copy VEXT's in earlier so -C can also see them. Message-ID: <20220801110708.888C011469C@lists.varnish-cache.org> commit 9843d650abb478c4f43b0f4c9c9b4dcb76e73c4a Author: Poul-Henning Kamp Date: Mon Aug 1 11:04:31 2022 +0000 Copy VEXT's in earlier so -C can also see them. diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index c0bd55914..a335ce645 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -856,6 +856,8 @@ main(int argc, char * const *argv) mgt_vcl_init(); + vext_copyin(vident); + u = 0; VTAILQ_FOREACH(alp, &arglist, list) { if (!strcmp(alp->arg, "f") && alp->priv != NULL) @@ -896,8 +898,6 @@ main(int argc, char * const *argv) VPF_Write(alp->priv); } - vext_copyin(vident); - AZ(VSB_finish(vident)); if (S_arg == NULL) From phk at FreeBSD.org Mon Aug 1 11:07:08 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 1 Aug 2022 11:07:08 +0000 (UTC) Subject: [master] 7afd74d2c Create a list of imports, and complain (harmlessly) if the same vmod is imported in multiple versions. Message-ID: <20220801110708.9DFCF11469F@lists.varnish-cache.org> commit 7afd74d2c75e164dd089b782e3a9ac970316978d Author: Poul-Henning Kamp Date: Mon Aug 1 11:05:44 2022 +0000 Create a list of imports, and complain (harmlessly) if the same vmod is imported in multiple versions. diff --git a/bin/varnishtest/tests/m00003.vtc b/bin/varnishtest/tests/m00003.vtc index c02d80071..1660b43b3 100644 --- a/bin/varnishtest/tests/m00003.vtc +++ b/bin/varnishtest/tests/m00003.vtc @@ -158,4 +158,37 @@ filewrite -a ${tmpdir}/libvmod_wrong.so { filewrite -a ${tmpdir}/libvmod_wrong.so "\x03" varnish v1 -errvcl {Bad vmod stanza} { import wrong; } -shell "rm -f ${tmpdir}/libvmod_wrong.so" +filewrite ${tmpdir}/libvmod_wrong.so "VMOD_JSON_SPEC\x02" +filewrite -a ${tmpdir}/libvmod_wrong.so { + [ + [ + "$VMOD", + "1.0", + "std", + "Vmod_vmod_std_Func", + "0000000000000000000000000000000000000000000000000000000000000000", + "0000000000000000000000000000000000000000000000000000000000000000", + "15", + "0" + ], [ + "$CPROTO", "/* blabla */" + ] + ] +} +filewrite -a ${tmpdir}/libvmod_wrong.so "\x03" + +varnish v1 -cliok "param.set vmod_path ${topbuild}/vmod/.libs/" +varnish v1 -cliok "stop" + +filewrite ${tmpdir}/wrong.vcl { + vcl 4.1; + import std; + import std as foo from "${tmpdir}/libvmod_wrong.so"; + backend default none; +} + +varnish v1 -cliexpect {Different version of VMOD std already loaded} { + vcl.load dupvmod ${tmpdir}/wrong.vcl +} + +shell "rm -f ${tmpdir}/libvmod_wrong.so ${tmpdir}wrong.vcl" diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index 3de227b21..a5e3990c4 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -47,51 +47,51 @@ #include "vcc_vmod.h" struct vmod_import { - unsigned magic; -#define VMOD_IMPORT_MAGIC 0x31803a5d - const char *err; - struct vsb *json; - char *path; + unsigned magic; +#define VMOD_IMPORT_MAGIC 0x31803a5d + const char *err; + struct vsb *json; + char *path; + VTAILQ_ENTRY(vmod_import) list; // From $VMOD - double vmod_syntax; - char *name; - char *func_name; - char *file_id; - char *abi; - unsigned major; - unsigned minor; - - struct symbol *sym; - const struct token *t_mod; - struct vjsn *vj; -#define STANZA(UU, ll, ss) int n_##ll; + double vmod_syntax; + char *name; + char *func_name; + char *file_id; + char *abi; + unsigned major; + unsigned minor; + + struct symbol *sym; + const struct token *t_mod; + struct vjsn *vj; +#define STANZA(UU, ll, ss) int n_##ll; STANZA_TBL #undef STANZA }; +static VTAILQ_HEAD(,vmod_import) imports = VTAILQ_HEAD_INITIALIZER(imports); + typedef void vcc_do_stanza_f(struct vcc *tl, const struct vmod_import *vim, const struct vjsn_val *vv); static int -vcc_path_open(void *priv, const char *fn) +vcc_Extract_JSON(struct vmod_import *vim, const char *filename) { - struct vmod_import *vim; const char *magic = "VMOD_JSON_SPEC\x02", *p; int c; FILE *f; - CAST_OBJ_NOTNULL(vim, priv, VMOD_IMPORT_MAGIC); - AN(fn); - - vim->json = VSB_new_auto(); - AN(vim->json); + CHECK_OBJ_NOTNULL(vim, VMOD_IMPORT_MAGIC); + AN(filename); - f = fopen(fn, "rb"); + f = fopen(filename, "rb"); if (f == NULL) { vim->err = strerror(errno); return (-1); } + p = magic; vim->err = "No VMOD JSON found"; while (1) { @@ -110,11 +110,15 @@ vcc_path_open(void *priv, const char *fn) break; } + vim->json = VSB_new_auto(); + AN(vim->json); + while (1) { c = getc(f); if (c == EOF) { AZ(fclose(f)); vim->err = "Truncated VMOD JSON"; + VSB_destroy(&vim->json); return (-1); } if (c == '\x03') @@ -243,9 +247,10 @@ vcc_ParseJSON(const struct vcc *tl, const char *jsn, struct vmod_import *vim) */ static int -vcc_VmodLoad(const struct vcc *tl, struct vmod_import *vim) +vcc_VmodLoad(struct vcc *tl, struct vmod_import *vim) { static const char *err; + struct vmod_import *vim2; CHECK_OBJ_NOTNULL(vim, VMOD_IMPORT_MAGIC); @@ -260,6 +265,24 @@ vcc_VmodLoad(const struct vcc *tl, struct vmod_import *vim) if (err != NULL) return (-1); + VTAILQ_FOREACH(vim2, &imports, list) { + if (strcmp(vim->name, vim2->name)) + continue; + if (!strcmp(vim->file_id, vim2->file_id)) { + // (Truly) duplicate imports are OK + return (0); + } + VSB_printf(tl->sb, + "Different version of VMOD %.*s already loaded\n", + PF(vim->t_mod)); + vcc_ErrWhere(tl, vim->t_mod); + VSB_printf(tl->sb, "Previous import at:\n"); + vcc_ErrWhere(tl, vim2->t_mod); + vcc_Warn(tl); + break; + } + VTAILQ_INSERT_TAIL(&imports, vim, list); + return (0); } @@ -376,6 +399,17 @@ vcc_vim_destroy(struct vmod_import **vimp) FREE_OBJ(vim); } +static int +vcc_path_open(void *priv, const char *fn) +{ + struct vmod_import *vim; + + CAST_OBJ_NOTNULL(vim, priv, VMOD_IMPORT_MAGIC); + AN(fn); + + return (vcc_Extract_JSON(vim, fn)); +} + void vcc_ParseImport(struct vcc *tl) { diff --git a/lib/libvcc/vcc_vmod_sym.c b/lib/libvcc/vcc_vmod_sym.c index 65f7f4d6f..ee30197b9 100644 --- a/lib/libvcc/vcc_vmod_sym.c +++ b/lib/libvcc/vcc_vmod_sym.c @@ -177,8 +177,10 @@ vcc_VmodSymbols(struct vcc *tl, const struct symbol *sym) if (!vjsn_is_array(vv)) continue; vv1 = VTAILQ_FIRST(&vv->children); + AN(vv1); assert(vjsn_is_string(vv1)); vv2 = VTAILQ_NEXT(vv1, list); + AN(vv2); if (!vjsn_is_string(vv2)) continue; From phk at FreeBSD.org Mon Aug 1 12:28:05 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 1 Aug 2022 12:28:05 +0000 (UTC) Subject: [master] e78e58ef1 Tell VCC about VEXTs. Message-ID: <20220801122805.8CEEE116EE3@lists.varnish-cache.org> commit e78e58ef1f9bc26d7a27856654c4417f91a5b79c Author: Poul-Henning Kamp Date: Mon Aug 1 11:23:23 2022 +0000 Tell VCC about VEXTs. diff --git a/bin/varnishd/common/common_vext.c b/bin/varnishd/common/common_vext.c index fffd9a8d5..d655fc4f5 100644 --- a/bin/varnishd/common/common_vext.c +++ b/bin/varnishd/common/common_vext.c @@ -85,6 +85,15 @@ vext_argument(const char *arg) vp->argv[1], strerror(errno)); } +void +vext_iter(vext_iter_f *func, void *priv) +{ + struct vext *vp; + + VTAILQ_FOREACH(vp, &vext_list, list) + func(VSB_data(vp->vsb), priv); +} + void vext_copyin(struct vsb *vident) { diff --git a/bin/varnishd/common/heritage.h b/bin/varnishd/common/heritage.h index 7ef4b04ba..48b0e243d 100644 --- a/bin/varnishd/common/heritage.h +++ b/bin/varnishd/common/heritage.h @@ -134,3 +134,5 @@ void vext_argument(const char *); void vext_copyin(struct vsb *); void vext_load(void); void vext_cleanup(void); +typedef void vext_iter_f(const char *, void *); +void vext_iter(vext_iter_f *func, void *); diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index f55031a1c..d029ab057 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -85,6 +85,13 @@ mgt_DumpBuiltin(void) * Invoke system VCC compiler in a sub-process */ +static void +vcc_vext_iter_func(const char *filename, void *priv) +{ + + VCC_VEXT(priv, filename); +} + static void v_noreturn_ v_matchproto_(vsub_func_f) run_vcc(void *priv) { @@ -110,6 +117,8 @@ run_vcc(void *priv) VCC_Opt_ ## l(vcc, MGT_VCC_FEATURE(VCC_FEATURE_ ## U)); #include "tbl/vcc_feature_bits.h" + vext_iter(vcc_vext_iter_func, vcc); + STV_Foreach(stv) VCC_Predef(vcc, "VCL_STEVEDORE", stv->ident); VTAILQ_FOREACH(vpg, &vclhead, list) diff --git a/include/libvcc.h b/include/libvcc.h index 3bf4e15c8..852fe91b0 100644 --- a/include/libvcc.h +++ b/include/libvcc.h @@ -38,6 +38,7 @@ void VCC_VCL_path(struct vcc *, const char *); void VCC_VMOD_path(struct vcc *, const char *); void VCC_Predef(struct vcc *, const char *type, const char *name); void VCC_VCL_Range(unsigned *, unsigned *); +void VCC_VEXT(struct vcc *, const char *); #define VCC_FEATURE_BIT(U, l, d) \ void VCC_Opt_ ## l(struct vcc *, unsigned); diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 4d04328bf..a23791dca 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -956,3 +956,10 @@ VCC_Predef(struct vcc *vcc, const char *type, const char *name) else WRONG("Unknown VCC predef type"); } + +void +VCC_VEXT(struct vcc *vcc, const char *filename) +{ + CHECK_OBJ_NOTNULL(vcc, VCC_MAGIC); + (void)filename; +} From phk at FreeBSD.org Mon Aug 1 12:28:05 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 1 Aug 2022 12:28:05 +0000 (UTC) Subject: [master] 2e9560056 Prototypical code to make VEXTs containing VMODs work. Message-ID: <20220801122805.A8E42116EE6@lists.varnish-cache.org> commit 2e9560056977c3bea957a2c396ebd2ca66f8f2e1 Author: Poul-Henning Kamp Date: Mon Aug 1 12:27:46 2022 +0000 Prototypical code to make VEXTs containing VMODs work. diff --git a/bin/varnishd/mgt/mgt_symtab.c b/bin/varnishd/mgt/mgt_symtab.c index 754a6d528..7b9dec65f 100644 --- a/bin/varnishd/mgt/mgt_symtab.c +++ b/bin/varnishd/mgt/mgt_symtab.c @@ -122,10 +122,15 @@ mgt_vcl_import_vmod(struct vclprog *vp, const struct vjsn_val *vv) const char *v_name; const char *v_file; const char *v_dst; + const struct vjsn_val *jv; CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC); AN(vv); + jv = vjsn_child(vv, "vext"); + if (vjsn_is_true(jv)) + return; + AN(jv); v_name = mgt_vcl_symtab_val(vv, "name"); v_file = mgt_vcl_symtab_val(vv, "file"); v_dst = mgt_vcl_symtab_val(vv, "dst"); diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index d029ab057..0fcef8a72 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -88,8 +88,16 @@ mgt_DumpBuiltin(void) static void vcc_vext_iter_func(const char *filename, void *priv) { + struct vsb *sb; - VCC_VEXT(priv, filename); + /* VCC runs in the per-VCL subdir */ + sb = VSB_new_auto(); + AN(sb); + VSB_cat(sb, "../"); + VSB_cat(sb, filename); + AZ(VSB_finish(sb)); + VCC_VEXT(priv, VSB_data(sb)); + VSB_destroy(&sb); } static void v_noreturn_ v_matchproto_(vsub_func_f) diff --git a/bin/varnishtest/tests/README b/bin/varnishtest/tests/README index 0f78207b7..a39d117d1 100644 --- a/bin/varnishtest/tests/README +++ b/bin/varnishtest/tests/README @@ -34,5 +34,6 @@ Naming scheme id ~ ^t02 --> HTTP2 id ~ ^u --> Utilities and background processes id ~ ^v --> VCL tests: execute VRT functions + id ~ ^x --> VEXT tests Coverage for individual VMODs is in "${top_srcdir}vmod/tests". diff --git a/bin/varnishtest/tests/x00000.vtc b/bin/varnishtest/tests/x00000.vtc new file mode 100644 index 000000000..7069f4c2c --- /dev/null +++ b/bin/varnishtest/tests/x00000.vtc @@ -0,0 +1,25 @@ +varnishtest "Test VMOD import from VEXT" + +feature topbuild + +server s1 { + rxreq + txresp +} -start + +varnish v1 \ + -arg "-pvmod_path=/nonexistent" \ + -arg "-E${topbuild}/vmod/.libs/libvmod_std.so" \ + -vcl+backend { + import std; + + sub vcl_deliver { + set resp.http.foobar = std.random(10,99); + } + } -start + +client c1 { + txreq + rxresp + expect resp.http.foobar ~ [0-9] +} -run diff --git a/lib/libvcc/flint.lnt b/lib/libvcc/flint.lnt index 4e90fc671..4705d651c 100644 --- a/lib/libvcc/flint.lnt +++ b/lib/libvcc/flint.lnt @@ -4,6 +4,7 @@ -sem(vcc_new_source, custodial(1)) +-sem(vcc_VmodLoad, custodial(2)) -sem(acl_tree_VRBT_INSERT, custodial(2)) -emacro(835, EXPR_VAR) // Info 835: A zero has been given as right argument to operator '<<' diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index a23791dca..e0b065a8a 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -961,5 +961,5 @@ void VCC_VEXT(struct vcc *vcc, const char *filename) { CHECK_OBJ_NOTNULL(vcc, VCC_MAGIC); - (void)filename; + vcc_ImportVext(vcc, filename); } diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 6f8b71f3b..a3e5d51e0 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -458,6 +458,7 @@ sym_wildcard_t vcc_Var_Wildcard; /* vcc_vmod.c */ void vcc_ParseImport(struct vcc *tl); +void vcc_ImportVext(struct vcc *tl, const char *filename); sym_act_f vcc_Act_New; /* vcc_xref.c */ diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index a5e3990c4..cfa99d1c6 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -53,6 +53,8 @@ struct vmod_import { struct vsb *json; char *path; VTAILQ_ENTRY(vmod_import) list; + int from_vext; + int unimported_vext; // From $VMOD double vmod_syntax; @@ -195,13 +197,6 @@ vcc_ParseJSON(const struct vcc *tl, const char *jsn, struct vmod_import *vim) vim->minor = strtoul(vv3->value, &p, 10); assert(p == NULL || *p == '\0' || *p == 'U'); - if (!vcc_IdIs(vim->t_mod, vim->name)) { - VSB_printf(tl->sb, "Wrong file for VMOD %.*s\n", - PF(vim->t_mod)); - VSB_printf(tl->sb, "\tFile name: %s\n", vim->path); - VSB_printf(tl->sb, "\tContains vmod \"%s\"\n", vim->name); - return (""); - } if (vim->major == 0 && vim->minor == 0 && strcmp(vim->abi, VMOD_ABI_Version)) { @@ -353,8 +348,14 @@ vcc_emit_setup(struct vcc *tl, const struct vmod_import *vim) VSB_cat(ifp->ini, ",\n"); AN(vim->file_id); VSB_printf(ifp->ini, "\t \"%s\",\n", vim->file_id); - VSB_printf(ifp->ini, "\t \"./vmod_cache/_vmod_%.*s.%s\"\n", - PF(mod), vim->file_id); + if (vim->from_vext) { + VSB_cat(ifp->ini, "\t "); + VSB_quote(ifp->ini, vim->path, -1, VSB_QUOTE_CSTR); + VSB_cat(ifp->ini, "\n"); + } else { + VSB_printf(ifp->ini, "\t \"./vmod_cache/_vmod_%.*s.%s\"\n", + PF(mod), vim->file_id); + } VSB_cat(ifp->ini, "\t ))\n"); VSB_cat(ifp->ini, "\t\treturn(1);"); @@ -362,6 +363,10 @@ vcc_emit_setup(struct vcc *tl, const struct vmod_import *vim) VSB_cat(tl->symtab, "\t\"dir\": \"import\",\n"); VSB_cat(tl->symtab, "\t\"type\": \"$VMOD\",\n"); VSB_printf(tl->symtab, "\t\"name\": \"%.*s\",\n", PF(mod)); + if (vim->from_vext) + VSB_printf(tl->symtab, "\t\"vext\": true,\n"); + else + VSB_printf(tl->symtab, "\t\"vext\": false,\n"); VSB_printf(tl->symtab, "\t\"file\": \"%s\",\n", vim->path); VSB_printf(tl->symtab, "\t\"dst\": \"./vmod_cache/_vmod_%.*s.%s\"\n", PF(mod), vim->file_id); @@ -417,7 +422,7 @@ vcc_ParseImport(struct vcc *tl) const char *p; struct token *mod, *tmod, *t1; struct symbol *msym, *vsym; - struct vmod_import *vim; + struct vmod_import *vim = NULL; const struct vmod_import *vimold; t1 = tl->t; @@ -438,6 +443,7 @@ vcc_ParseImport(struct vcc *tl) ERRCHK(tl); AN(msym); + bprintf(fn, "libvmod_%.*s.so", PF(mod)); if (tl->t->tok == ID) { if (!vcc_IdIs(tl->t, "from")) { VSB_cat(tl->sb, "Expected 'from path ...'\n"); @@ -460,34 +466,56 @@ vcc_ParseImport(struct vcc *tl) bprintf(fn, "%s", tl->t->dec); vcc_NextToken(tl); } else { - bprintf(fn, "libvmod_%.*s.so", PF(mod)); + VTAILQ_FOREACH(vim, &imports, list) { + if (!vcc_IdIs(mod, vim->name)) + continue; + if (!vim->unimported_vext) + continue; + fprintf(stderr, "IMPORT %s from VEXT\n", vim->name); + vim->unimported_vext = 0; + vim->t_mod = mod; + vim->sym = msym; + break; + } } SkipToken(tl, ';'); - ALLOC_OBJ(vim, VMOD_IMPORT_MAGIC); - AN(vim); - vim->t_mod = mod; - vim->sym = msym; + if (vim == NULL) { + ALLOC_OBJ(vim, VMOD_IMPORT_MAGIC); + AN(vim); + vim->t_mod = mod; + vim->sym = msym; + + if (VFIL_searchpath(tl->vmod_path, vcc_path_open, vim, fn, &vim->path)) { + if (vim->err == NULL) { + VSB_printf(tl->sb, + "Could not find VMOD %.*s\n", PF(mod)); + } else { + VSB_printf(tl->sb, + "Could not open VMOD %.*s\n", PF(mod)); + VSB_printf(tl->sb, "\tFile name: %s\n", + vim->path != NULL ? vim->path : fn); + VSB_printf(tl->sb, "\tError: %s\n", vim->err); + } + vcc_ErrWhere(tl, mod); + vcc_vim_destroy(&vim); + return; + } - if (VFIL_searchpath(tl->vmod_path, vcc_path_open, vim, fn, &vim->path)) { - if (vim->err == NULL) { - VSB_printf(tl->sb, - "Could not find VMOD %.*s\n", PF(mod)); - } else { - VSB_printf(tl->sb, - "Could not open VMOD %.*s\n", PF(mod)); - VSB_printf(tl->sb, "\tFile name: %s\n", - vim->path != NULL ? vim->path : fn); - VSB_printf(tl->sb, "\tError: %s\n", vim->err); + if (vcc_VmodLoad(tl, vim) < 0 || tl->err) { + vcc_ErrWhere(tl, vim->t_mod); + vcc_vim_destroy(&vim); + return; } - vcc_ErrWhere(tl, mod); - vcc_vim_destroy(&vim); - return; } - if (vcc_VmodLoad(tl, vim) < 0 || tl->err) { + if (!vcc_IdIs(vim->t_mod, vim->name)) { vcc_ErrWhere(tl, vim->t_mod); + VSB_printf(tl->sb, "Wrong file for VMOD %.*s\n", + PF(vim->t_mod)); + VSB_printf(tl->sb, "\tFile name: %s\n", vim->path); + VSB_printf(tl->sb, "\tContains vmod \"%s\"\n", vim->name); vcc_vim_destroy(&vim); return; } @@ -534,3 +562,29 @@ vcc_ParseImport(struct vcc *tl) vcc_emit_setup(tl, vim); } + +void +vcc_ImportVext(struct vcc *tl, const char *filename) +{ + struct vmod_import *vim; + + ALLOC_OBJ(vim, VMOD_IMPORT_MAGIC); + AN(vim); + + if (vcc_Extract_JSON(vim, filename)) { + FREE_OBJ(vim); + return; + } + fprintf(stderr, "FOUND VMOD in VEXT %s\n", filename); + if (vcc_VmodLoad(tl, vim) < 0 || tl->err) { + // vcc_ErrWhere(tl, vim->t_mod); + vcc_vim_destroy(&vim); + return; + } + vim->from_vext = 1; + vim->unimported_vext = 1; + vim->path = strdup(filename); + vim->path += 1; + AN(vim->path); + fprintf(stderr, "GOOD VMOD %s in VEXT %s\n", vim->name, filename); +} From phk at FreeBSD.org Mon Aug 1 13:25:06 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 1 Aug 2022 13:25:06 +0000 (UTC) Subject: [master] 9bbfc0047 Make sure a jailed worker process can open the cached VEXT file Message-ID: <20220801132506.A1BBF118BB0@lists.varnish-cache.org> commit 9bbfc0047614c24b3b7952e56d4ec31a424c3005 Author: Poul-Henning Kamp Date: Mon Aug 1 13:24:01 2022 +0000 Make sure a jailed worker process can open the cached VEXT file diff --git a/bin/varnishd/common/common_vext.c b/bin/varnishd/common/common_vext.c index d655fc4f5..3a4523cc3 100644 --- a/bin/varnishd/common/common_vext.c +++ b/bin/varnishd/common/common_vext.c @@ -125,7 +125,7 @@ vext_copyin(struct vsb *vident) VSB_cat(vp->vsb, ".so"); AZ(VSB_finish(vp->vsb)); fprintf(stderr, "ee2 %s\n", VSB_data(vp->vsb)); - fdo = open(VSB_data(vp->vsb), O_WRONLY|O_CREAT|O_EXCL, 0700); + fdo = open(VSB_data(vp->vsb), O_WRONLY|O_CREAT|O_EXCL, 0755); xxxassert(fdo >= 0); AZ(lseek(vp->fd, 0, SEEK_SET)); do { From nils.goroll at uplex.de Tue Aug 2 10:49:06 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 2 Aug 2022 10:49:06 +0000 (UTC) Subject: [master] 66d72f358 Keep vext .so files with -p debug=+vmod_so_keep Message-ID: <20220802104907.02746117512@lists.varnish-cache.org> commit 66d72f3583ab224de1f1ccd62509a1db95b2b765 Author: Nils Goroll Date: Tue Aug 2 12:45:15 2022 +0200 Keep vext .so files with -p debug=+vmod_so_keep Required for seamless debugging. Alternatives considered: - add vext_so_keep debug flag -> overkill To decide: - rename to libso_keep / "Keep copied VMOD/VEXT libraries" ? (libso to disambiguate from so for socket option) diff --git a/bin/varnishd/common/common_vext.c b/bin/varnishd/common/common_vext.c index 3a4523cc3..fba153627 100644 --- a/bin/varnishd/common/common_vext.c +++ b/bin/varnishd/common/common_vext.c @@ -158,14 +158,15 @@ vext_load(void) } void -vext_cleanup(void) +vext_cleanup(int do_unlink) { struct vext *vp; VTAILQ_FOREACH(vp, &vext_list, list) { fprintf(stderr, "ee3 %s\n", VSB_data(vp->vsb)); if (vp->vsb != NULL && VSB_len(vp->vsb) > 0) { - XXXAZ(unlink(VSB_data(vp->vsb))); + if (do_unlink) + XXXAZ(unlink(VSB_data(vp->vsb))); VSB_clear(vp->vsb); } } diff --git a/bin/varnishd/common/heritage.h b/bin/varnishd/common/heritage.h index 48b0e243d..06dbe177d 100644 --- a/bin/varnishd/common/heritage.h +++ b/bin/varnishd/common/heritage.h @@ -133,6 +133,6 @@ extern vsm_lock_f *vsmw_unlock; void vext_argument(const char *); void vext_copyin(struct vsb *); void vext_load(void); -void vext_cleanup(void); +void vext_cleanup(int); typedef void vext_iter_f(const char *, void *); void vext_iter(vext_iter_f *func, void *); diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index a335ce645..6cc52a124 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -285,7 +285,7 @@ mgt_Cflag_atexit(void) /* Only master process */ if (getpid() != heritage.mgt_pid) return; - vext_cleanup(); + vext_cleanup(1); VJ_rmdir("vmod_cache"); VJ_rmdir("vext_cache"); (void)chdir("/"); @@ -986,7 +986,7 @@ main(int argc, char * const *argv) mgt_cli_close_all(); VEV_Destroy(&mgt_evb); VJ_master(JAIL_MASTER_SYSTEM); - vext_cleanup(); + vext_cleanup(! MGT_DO_DEBUG(DBG_VMOD_SO_KEEP)); (void)rmdir("vext_cache"); VJ_master(JAIL_MASTER_LOW); VTAILQ_FOREACH(alp, &arglist, list) { From nils.goroll at uplex.de Thu Aug 4 10:53:05 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 4 Aug 2022 10:53:05 +0000 (UTC) Subject: [master] 77e401adb Improve clarity of INSTALL Message-ID: <20220804105305.A0D9611E275@lists.varnish-cache.org> commit 77e401adbe73161a794a716a3821da789d3c7c7f Author: Nils Goroll Date: Thu Aug 4 12:48:44 2022 +0200 Improve clarity of INSTALL Closes #3831 diff --git a/INSTALL b/INSTALL index 8f6e796d4..0613f5343 100644 --- a/INSTALL +++ b/INSTALL @@ -1,12 +1,21 @@ Installation Instructions -Varnish uses the GNU autotools. To build and install Varnish, simply -run the 'configure' script in the top-level directory, then run 'make' -and 'make install'. On Linux, you need to run 'ldconfig' as root -afterwards in order to update the shared library cache. +See https://varnish-cache.org/docs/7.1/installation/install_source.html +for complete and up to date install instructions. -If you obtained the sources directly from the Git repository, you will -need to run autogen.sh first to create the configure script. +This file only mentions the basic steps: + +* Install prerequesites + +* When building from the source repository, run + + sh autogen.sh + +* To build and install Varnish, run + + sh configure + make + make install Varnish will store run-time state in /var/run/varnish; you may want to tune this using configure's --localstatedir parameter. From nils.goroll at uplex.de Thu Aug 4 10:53:05 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 4 Aug 2022 10:53:05 +0000 (UTC) Subject: [master] 0859ded5d The run dir is /var/run/varnishd Message-ID: <20220804105305.B3DEF11E278@lists.varnish-cache.org> commit 0859ded5d9f7f9fbe8c3ef9c409bc74c1ecfaad8 Author: Nils Goroll Date: Thu Aug 4 12:51:16 2022 +0200 The run dir is /var/run/varnishd diff --git a/INSTALL b/INSTALL index 0613f5343..f161f6fa3 100644 --- a/INSTALL +++ b/INSTALL @@ -17,7 +17,7 @@ This file only mentions the basic steps: make make install -Varnish will store run-time state in /var/run/varnish; you may +Varnish will store run-time state in /var/run/varnishd; you may want to tune this using configure's --localstatedir parameter. Additional configure options of interest: From martin at varnish-software.com Thu Aug 4 13:34:05 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 4 Aug 2022 13:34:05 +0000 (UTC) Subject: [7.0] d478c23b2 Follow redirects when downloading dist Message-ID: <20220804133405.92B3796C8@lists.varnish-cache.org> commit d478c23b241a7cbfc8fe76eb6221599e96b9d010 Author: Martin Blix Grydeland Date: Thu Aug 4 15:33:26 2022 +0200 Follow redirects when downloading dist diff --git a/.circleci/config.yml b/.circleci/config.yml index f6d0cbc91..796e0c23f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,7 +49,7 @@ jobs: - run: name: Download the dist tarball command: | - curl -s << pipeline.parameters.dist-url >> -o varnish-dist.tar.gz + curl -Ls '<< pipeline.parameters.dist-url >>' -o varnish-dist.tar.gz - when: condition: << pipeline.parameters.dist-url-sha256 >> steps: From martin at varnish-software.com Thu Aug 4 13:46:05 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 4 Aug 2022 13:46:05 +0000 (UTC) Subject: [7.1] 3e22c7ab2 Follow redirects when downloading dist Message-ID: <20220804134605.B17079E5B@lists.varnish-cache.org> commit 3e22c7ab28ed2bd756445a1e595c90ec82156293 Author: Martin Blix Grydeland Date: Thu Aug 4 15:33:26 2022 +0200 Follow redirects when downloading dist diff --git a/.circleci/config.yml b/.circleci/config.yml index 5b7d4e41b..231618727 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,7 +49,7 @@ jobs: - run: name: Download the dist tarball command: | - curl -s << pipeline.parameters.dist-url >> -o varnish-dist.tar.gz + curl -Ls '<< pipeline.parameters.dist-url >>' -o varnish-dist.tar.gz - when: condition: << pipeline.parameters.dist-url-sha256 >> steps: From martin at varnish-software.com Thu Aug 4 15:37:05 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 4 Aug 2022 15:37:05 +0000 (UTC) Subject: [7.0] 6d90d31e3 Replace centos:8 with almalinux:8 Message-ID: <20220804153705.9530E100861@lists.varnish-cache.org> commit 6d90d31e39e6c2cb2d96cce3a0aeb9a535ae6af9 Author: Martin Blix Grydeland Date: Thu Aug 4 17:36:05 2022 +0200 Replace centos:8 with almalinux:8 diff --git a/.circleci/config.yml b/.circleci/config.yml index 796e0c23f..61983077f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -155,6 +155,7 @@ jobs: case "<< parameters.platform >>" in debian:*|ubuntu:*) EXT=deb ;; centos:*) EXT=rpm ;; + almalinux:*) EXT=rpm ;; alpine:*) EXT=apk ;; *) echo "unrecognized platform: << parameters.platform >>" @@ -213,7 +214,7 @@ jobs: docker create --name workspace -v /workspace << parameters.dist >>:<< parameters.release >> /bin/true docker cp /workspace workspace:/ docker run --volumes-from workspace -w /workspace << parameters.dist >>:<< parameters.release >> sh -c ' - if [ << parameters.dist >> = centos ]; then + if [ << parameters.dist >> = centos -o << parameters.dist >> = almalinux ]; then if [ << parameters.release >> = 8 ]; then dnf install -y "dnf-command(config-manager)" yum config-manager --set-enabled powertools @@ -295,7 +296,7 @@ jobs: if [ << parameters.dist >> = archlinux ]; then useradd varnish - elif [ << parameters.dist >> = centos ]; then + elif [ << parameters.dist >> = centos -o << parameters.dist >> = almalinux ]; then adduser varnish else adduser --disabled-password --gecos "" varnish @@ -358,7 +359,7 @@ workflows: - debian:buster - debian:stretch - centos:7 - - centos:8 + - almalinux:8 - alpine:3 rclass: - arm.medium From martin at varnish-software.com Thu Aug 4 16:33:04 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 4 Aug 2022 16:33:04 +0000 (UTC) Subject: [7.0] 814926f05 Hopefully get sphinx installed Message-ID: <20220804163304.E84461021B9@lists.varnish-cache.org> commit 814926f056555d0e317b4b65c3ed6c71e8ad5915 Author: Martin Blix Grydeland Date: Thu Aug 4 18:32:06 2022 +0200 Hopefully get sphinx installed diff --git a/.circleci/config.yml b/.circleci/config.yml index 61983077f..f11c949fb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -218,7 +218,7 @@ jobs: if [ << parameters.release >> = 8 ]; then dnf install -y "dnf-command(config-manager)" yum config-manager --set-enabled powertools - yum install -y diffutils python3-sphinx + yum install -y diffutils else yum install -y python-sphinx fi @@ -233,6 +233,7 @@ jobs: make \ pcre2-devel \ python3 \ + /usr/bin/sphinx-build \ sudo elif [ << parameters.dist >> = debian -o << parameters.dist >> = ubuntu ]; then export DEBIAN_FRONTEND=noninteractive From martin at varnish-software.com Fri Aug 5 08:34:10 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Fri, 5 Aug 2022 08:34:10 +0000 (UTC) Subject: [7.0] 3f7f20c87 Enable powertools in make-rpm-packages.sh on almalinux Message-ID: <20220805083410.7C85D11E6F1@lists.varnish-cache.org> commit 3f7f20c87c466c1d3e8e2a34424050260c738432 Author: Martin Blix Grydeland Date: Fri Aug 5 10:33:27 2022 +0200 Enable powertools in make-rpm-packages.sh on almalinux diff --git a/.circleci/make-rpm-packages.sh b/.circleci/make-rpm-packages.sh index 56acdad05..df86f4f22 100755 --- a/.circleci/make-rpm-packages.sh +++ b/.circleci/make-rpm-packages.sh @@ -15,7 +15,7 @@ fi yum install -y epel-release -if [ "$PARAM_DIST" = centos ]; then +if [ "$PARAM_DIST" = centos -o "$PARAM_DIST" = almalinux ]; then if [ "$PARAM_RELEASE" = 8 ]; then dnf install -y 'dnf-command(config-manager)' yum config-manager --set-enabled powertools From dridi.boukelmoune at gmail.com Fri Aug 5 09:15:09 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 5 Aug 2022 09:15:09 +0000 (UTC) Subject: [master] 523f19297 build: Add -mno-omit-leaf-frame-pointer to sanitizer CFLAGS Message-ID: <20220805091510.02EB711F9B1@lists.varnish-cache.org> commit 523f1929706101f0a62120adf755aab814598fb7 Author: Dridi Boukelmoune Date: Fri Jul 15 12:32:04 2022 +0200 build: Add -mno-omit-leaf-frame-pointer to sanitizer CFLAGS According to the GCC manual on -fno-omit-frame-pointer: > Note that -fno-omit-frame-pointer doesn't guarantee the frame pointer > is used in all functions. Several targets always omit the frame > pointer in leaf functions. And what is says about -m[no-]omit-leaf-frame-pointer: > -momit-leaf-frame-pointer > -mno-omit-leaf-frame-pointer > > Omit or keep the frame pointer in leaf functions. The former behavior > is the default. So we should probably prevent both. diff --git a/configure.ac b/configure.ac index f3096aa34..b8175d4d6 100644 --- a/configure.ac +++ b/configure.ac @@ -316,7 +316,7 @@ if test "x$UBSAN_FLAGS$TSAN_FLAGS$ASAN_FLAGS$MSAN_FLAGS" != "x"; then AC_DEFINE([ENABLE_SANITIZER], [1], [Define to 1 if any sanitizer is enabled.]) SAN_FLAGS="$ASAN_FLAGS $UBSAN_FLAGS $TSAN_FLAGS $MSAN_FLAGS" - SAN_CFLAGS="$SAN_FLAGS -fPIC -fPIE -fno-omit-frame-pointer" + SAN_CFLAGS="$SAN_FLAGS -fPIC -fPIE -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer" SAN_LDFLAGS= save_CFLAGS=$CFLAGS CFLAGS="${CFLAGS} -Werror=unused-command-line-argument" From dridi.boukelmoune at gmail.com Fri Aug 5 09:16:05 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 5 Aug 2022 09:16:05 +0000 (UTC) Subject: [master] 8bc97703d vsc: Add a long description to MAIN.shm_* counters Message-ID: <20220805091605.2F96C11FB8C@lists.varnish-cache.org> commit 8bc97703d03e5e277b5d0cbbaa800512f3a44675 Author: Dridi Boukelmoune Date: Mon Jul 18 08:26:32 2022 +0200 vsc: Add a long description to MAIN.shm_* counters diff --git a/lib/libvsc/VSC_main.vsc b/lib/libvsc/VSC_main.vsc index 7b32584c9..87bf4933b 100644 --- a/lib/libvsc/VSC_main.vsc +++ b/lib/libvsc/VSC_main.vsc @@ -670,25 +670,37 @@ :level: diag :oneliner: SHM records + Number of log records written to the shared memory log. + .. varnish_vsc:: shm_writes :level: diag :oneliner: SHM writes + Number of individual writes to the shared memory log. A single + write may batch multiple records for bufferred tasks. + .. varnish_vsc:: shm_flushes :level: diag :oneliner: SHM flushes due to overflow + Number of writes performed before the end of a bufferred task + because adding a record to a batch would exceed vsl_buffer. .. varnish_vsc:: shm_cont :level: diag - :oneliner: SHM MTX contention + :oneliner: SHM lock contention + + Number of times a write had to wait for the lock. .. varnish_vsc:: shm_cycles :level: diag - :oneliner: SHM cycles through buffer + :oneliner: SHM cycles through VSL space + + Number of times a write of log records would reach past the end + of the shared memory log, cycling back to the beginning. .. varnish_vsc:: backend_req From dridi.boukelmoune at gmail.com Fri Aug 5 09:16:05 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 5 Aug 2022 09:16:05 +0000 (UTC) Subject: [master] 284758389 vsc: New MAIN.shm_bytes counters Message-ID: <20220805091605.5812411FB8F@lists.varnish-cache.org> commit 284758389fed2fccf2e9dc719506fa80750594aa Author: Darryl Rodden Date: Tue Jul 5 14:40:48 2022 -0700 vsc: New MAIN.shm_bytes counters We have various stat counters for the shared memory log (shmlog), but nothing to give us the number of bytes used (consumed). The new shm_bytes counter remedies that. Signed-off-by: Dridi Boukelmoune diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 2e67bbd05..e43097ce7 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -193,6 +193,7 @@ vsl_get(unsigned len, unsigned records, unsigned flushes) VSC_C_main->shm_writes++; VSC_C_main->shm_flushes += flushes; VSC_C_main->shm_records += records; + VSC_C_main->shm_bytes += VSL_BYTES(VSL_OVERHEAD + VSL_WORDS(len)); /* Wrap if necessary */ if (VSL_END(vsl_ptr, len) >= vsl_end) diff --git a/bin/varnishtest/tests/l00006.vtc b/bin/varnishtest/tests/l00006.vtc new file mode 100644 index 000000000..93fe5935a --- /dev/null +++ b/bin/varnishtest/tests/l00006.vtc @@ -0,0 +1,18 @@ +varnishtest "Check shmlog stats" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend "" -start + +client c1 { + txreq + rxresp +} -run + +varnish v1 -vsl_catchup +varnish v1 -expect shm_writes > 0 +varnish v1 -expect shm_records > 0 +varnish v1 -expect shm_bytes > 0 diff --git a/lib/libvsc/VSC_main.vsc b/lib/libvsc/VSC_main.vsc index 87bf4933b..c895f7a1c 100644 --- a/lib/libvsc/VSC_main.vsc +++ b/lib/libvsc/VSC_main.vsc @@ -703,6 +703,14 @@ of the shared memory log, cycling back to the beginning. +.. varnish_vsc:: shm_bytes + :level: diag + :format: bytes + :oneliner: SHM bytes + + Number of bytes written to the shared memory log. + + .. varnish_vsc:: backend_req :oneliner: Backend requests made From dridi.boukelmoune at gmail.com Fri Aug 5 09:21:05 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 5 Aug 2022 09:21:05 +0000 (UTC) Subject: [master] f7b11f03a doc: Link to (hopefully) future-proof trunk docs Message-ID: <20220805092105.A47CE40D3@lists.varnish-cache.org> commit f7b11f03a66cd04a9e8793973c9c7d24d068a0a1 Author: Dridi Boukelmoune Date: Fri Aug 5 11:20:03 2022 +0200 doc: Link to (hopefully) future-proof trunk docs Refs #3831 diff --git a/INSTALL b/INSTALL index f161f6fa3..6122bcb55 100644 --- a/INSTALL +++ b/INSTALL @@ -1,6 +1,6 @@ Installation Instructions -See https://varnish-cache.org/docs/7.1/installation/install_source.html +See https://varnish-cache.org/docs/trunk/installation/install_source.html for complete and up to date install instructions. This file only mentions the basic steps: From dridi.boukelmoune at gmail.com Fri Aug 5 09:24:07 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 5 Aug 2022 09:24:07 +0000 (UTC) Subject: [master] b7375650e build: Add a PACKAGE_BRANCH substitution Message-ID: <20220805092407.5C7474521@lists.varnish-cache.org> commit b7375650e9e7a665ca2870e8e852f3f993a1524c Author: Dridi Boukelmoune Date: Tue Jul 26 09:42:28 2022 +0200 build: Add a PACKAGE_BRANCH substitution It works both with trunk or x.y.z versions, resulting in trunk for the former and x.y for the latter. diff --git a/configure.ac b/configure.ac index b8175d4d6..ca2bf0b01 100644 --- a/configure.ac +++ b/configure.ac @@ -9,6 +9,11 @@ AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_AUX_DIR([build-aux]) AC_USE_SYSTEM_EXTENSIONS +PACKAGE_BRANCH=${PACKAGE_VERSION%.*} +AC_SUBST([PACKAGE_BRANCH]) +AC_DEFINE_UNQUOTED([PACKAGE_BRANCH], ["$PACKAGE_BRANCH"], + [Define the branch of this package.]) + # save command line CFLAGS for use in VCC_CC (to pass through things like -m64) # and make distcheck configure OCFLAGS="$CFLAGS" From dridi.boukelmoune at gmail.com Fri Aug 5 09:24:07 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 5 Aug 2022 09:24:07 +0000 (UTC) Subject: [master] 81b486a9f vcs: Add a 'B' string for the package branch Message-ID: <20220805092407.740084524@lists.varnish-cache.org> commit 81b486a9fdf75f938300e014eb80f779f83792b7 Author: Dridi Boukelmoune Date: Tue Jul 26 09:56:12 2022 +0200 vcs: Add a 'B' string for the package branch diff --git a/lib/libvarnish/version.c b/lib/libvarnish/version.c index 99bbf2c42..a18f3bbe7 100644 --- a/lib/libvarnish/version.c +++ b/lib/libvarnish/version.c @@ -57,6 +57,8 @@ VCS_String(const char *which) return (PACKAGE_TARNAME); case 'P': return (PACKAGE_VERSION); + case 'B': + return (PACKAGE_BRANCH); case 'R': return (VCS_Version); case 'V': From dridi.boukelmoune at gmail.com Fri Aug 5 09:24:07 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 5 Aug 2022 09:24:07 +0000 (UTC) Subject: [master] 61d43f58a resp: Don't hard-code the package branch in Via header Message-ID: <20220805092407.9F2B04529@lists.varnish-cache.org> commit 61d43f58a73ec02e0770389e92ddd0152b262c56 Author: Dridi Boukelmoune Date: Tue Jul 26 10:02:31 2022 +0200 resp: Don't hard-code the package branch in Via header This should remove one error-prone step in the release process. Better diff with the --word-diff --word-diff-regex='\w+' options. Refs #3794 diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 857b29cf8..0b6141bbf 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -167,7 +167,7 @@ Resp_Setup_Deliver(struct req *req) http_PrintfHeader(h, "Age: %.0f", floor(fmax(0., req->t_prev - oc->t_origin))); - http_SetHeader(h, "Via: 1.1 varnish (Varnish/7.1)"); + http_SetHeader(h, "Via: 1.1 varnish (Varnish/" PACKAGE_BRANCH ")"); if (cache_param->http_gzip_support && ObjCheckFlag(req->wrk, oc, OF_GZIPED) && diff --git a/bin/varnishtest/tests/b00000.vtc b/bin/varnishtest/tests/b00000.vtc index 59cb4c119..958bdc22c 100644 --- a/bin/varnishtest/tests/b00000.vtc +++ b/bin/varnishtest/tests/b00000.vtc @@ -24,6 +24,10 @@ varnish v1 -vcl+backend { sub vcl_backend_response { set beresp.do_stream = false; } + sub vcl_deliver { + # make s_resp_hdrbytes deterministic + unset resp.http.via; + } } -start varnish v1 -cliok "param.set debug +workspace" @@ -48,7 +52,7 @@ varnish v1 -expect client_req == 1 varnish v1 -expect cache_miss == 1 varnish v1 -expect s_sess == 1 varnish v1 -expect s_resp_bodybytes == 7 -varnish v1 -expect s_resp_hdrbytes == 178 +varnish v1 -expect s_resp_hdrbytes == 146 client c1 { txreq -url "/2" @@ -63,4 +67,4 @@ varnish v1 -expect client_req == 2 varnish v1 -expect cache_miss == 2 varnish v1 -expect s_sess == 2 varnish v1 -expect s_resp_bodybytes == 14 -varnish v1 -expect s_resp_hdrbytes == 356 +varnish v1 -expect s_resp_hdrbytes == 292 diff --git a/bin/varnishtest/tests/b00053.vtc b/bin/varnishtest/tests/b00053.vtc index 1c9bf152b..0759d10b7 100644 --- a/bin/varnishtest/tests/b00053.vtc +++ b/bin/varnishtest/tests/b00053.vtc @@ -9,6 +9,10 @@ varnish v1 -arg "-a ${tmpdir}/v1.sock -a ${listen_addr}" -vcl+backend { sub vcl_backend_response { set beresp.do_stream = false; } + sub vcl_deliver { + # make s_resp_hdrbytes deterministic + unset resp.http.via; + } } -start varnish v1 -cliok "param.set debug +workspace" @@ -31,7 +35,7 @@ varnish v1 -expect client_req == 1 varnish v1 -expect cache_miss == 1 varnish v1 -expect s_sess == 1 varnish v1 -expect s_resp_bodybytes == 7 -varnish v1 -expect s_resp_hdrbytes == 178 +varnish v1 -expect s_resp_hdrbytes == 146 # varnishtest "vtc v_* macros when the listen address is UDS" (a00019) diff --git a/bin/varnishtest/tests/e00003.vtc b/bin/varnishtest/tests/e00003.vtc index e9906163e..c03d9f487 100644 --- a/bin/varnishtest/tests/e00003.vtc +++ b/bin/varnishtest/tests/e00003.vtc @@ -35,12 +35,15 @@ varnish v1 -vcl+backend { } sub vcl_deliver { set resp.http.can_esi = obj.can_esi; + + # make ReqAcct deterministic + unset resp.http.via; } } -start logexpect l1 -v v1 -g request { expect 0 1001 Begin "^req .* rxreq" - expect * = ReqAcct "^29 0 29 202 75 277$" + expect * = ReqAcct "^29 0 29 170 75 245$" expect 0 = End } -start @@ -63,7 +66,7 @@ logexpect l4 -v v1 -g request { logexpect l5 -v v1 -g request { expect * 1005 Begin "^req .* rxreq" # Header bytes is 5 larger than in l1 due to two item X-Varnish hdr - expect * = ReqAcct "^29 0 29 207 75 282$" + expect * = ReqAcct "^29 0 29 175 75 250$" expect 0 = End } -start diff --git a/bin/varnishtest/tests/t02005.vtc b/bin/varnishtest/tests/t02005.vtc index 6412f23c6..19f862ac9 100644 --- a/bin/varnishtest/tests/t02005.vtc +++ b/bin/varnishtest/tests/t02005.vtc @@ -22,11 +22,16 @@ varnish v1 -vcl+backend { return (fail); } } + + sub vcl_deliver { + # make ReqAcct deterministic + unset resp.http.via; + } } -cliok "param.set feature +http2" -start varnish v1 -cliok "param.set debug +syncvsl" logexpect l1 -v v1 -g raw { - expect * 1001 ReqAcct "80 7 87 106 8 114" + expect * 1001 ReqAcct "80 7 87 78 8 86" expect * 1000 ReqAcct "45 8 53 54 20 74" } -start diff --git a/bin/varnishtest/tests/t02015.vtc b/bin/varnishtest/tests/t02015.vtc index 829873acd..6190d8d9a 100644 --- a/bin/varnishtest/tests/t02015.vtc +++ b/bin/varnishtest/tests/t02015.vtc @@ -6,11 +6,16 @@ server s1 { } -start varnish v1 -cliok "param.set feature +http2" -varnish v1 -vcl+backend "" -start +varnish v1 -vcl+backend { + sub vcl_deliver { + # make ReqAcct deterministic + unset resp.http.via; + } +} -start logexpect l1 -v v1 -g raw -q ReqAcct { - expect ? 1001 ReqAcct "46 0 46 97 12345 12442" - expect ? 1003 ReqAcct "46 0 46 102 1000 1102" + expect ? 1001 ReqAcct "46 0 46 69 12345 12414" + expect ? 1003 ReqAcct "46 0 46 74 1000 1074" } -start client c1 { From dridi.boukelmoune at gmail.com Fri Aug 5 09:24:07 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 5 Aug 2022 09:24:07 +0000 (UTC) Subject: [master] c62f60216 http: Add a function to append a value to a header Message-ID: <20220805092407.BA53A452D@lists.varnish-cache.org> commit c62f60216f5b88b04db85f44cb0b8a529f890d66 Author: Dridi Boukelmoune Date: Tue Jul 26 10:33:37 2022 +0200 http: Add a function to append a value to a header diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index cb69f2c2c..3ef0512bf 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -601,6 +601,7 @@ void http_FilterReq(struct http *to, const struct http *fm, unsigned how); void HTTP_Encode(const struct http *fm, uint8_t *, unsigned len, unsigned how); int HTTP_Decode(struct http *to, const uint8_t *fm); void http_ForceHeader(struct http *to, hdr_t, const char *val); +void http_AppendHeader(struct http *to, hdr_t, const char *val); void http_PrintfHeader(struct http *to, const char *fmt, ...) v_printflike_(2, 3); void http_TimeHeader(struct http *to, const char *fmt, vtim_real now); diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 7c6a127f6..e2f8ba726 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -1478,6 +1478,20 @@ http_ForceHeader(struct http *to, hdr_t hdr, const char *val) http_PrintfHeader(to, "%s %s", hdr + 1, val); } +void +http_AppendHeader(struct http *to, hdr_t hdr, const char *val) +{ + const char *old; + + http_CollectHdr(to, hdr); + if (http_GetHdr(to, hdr, &old)) { + http_Unset(to, hdr); + http_PrintfHeader(to, "%s %s, %s", hdr + 1, old, val); + } else { + http_PrintfHeader(to, "%s %s", hdr + 1, val); + } +} + void http_PrintfHeader(struct http *to, const char *fmt, ...) { From dridi.boukelmoune at gmail.com Fri Aug 5 09:24:07 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 5 Aug 2022 09:24:07 +0000 (UTC) Subject: [master] da08a4c26 req: Build X-Forwarded-For with http_AppendHeader() Message-ID: <20220805092407.D42AB4531@lists.varnish-cache.org> commit da08a4c26db8314b1c04bde5e105b11c617052c6 Author: Dridi Boukelmoune Date: Tue Jul 26 10:35:44 2022 +0200 req: Build X-Forwarded-For with http_AppendHeader() diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 0b6141bbf..c23270073 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -866,21 +866,13 @@ cnt_restart(struct worker *wrk, struct req *req) static void v_matchproto_(req_state_f) cnt_recv_prep(struct req *req, const char *ci) { - const char *xff; if (req->restarts == 0) { /* * This really should be done earlier, but we want to capture * it in the VSL log. */ - http_CollectHdr(req->http, H_X_Forwarded_For); - if (http_GetHdr(req->http, H_X_Forwarded_For, &xff)) { - http_Unset(req->http, H_X_Forwarded_For); - http_PrintfHeader(req->http, "X-Forwarded-For: %s, %s", - xff, ci); - } else { - http_PrintfHeader(req->http, "X-Forwarded-For: %s", ci); - } + http_AppendHeader(req->http, H_X_Forwarded_For, ci); http_CollectHdr(req->http, H_Cache_Control); /* By default we use the first backend */ From dridi.boukelmoune at gmail.com Fri Aug 5 09:24:07 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 5 Aug 2022 09:24:07 +0000 (UTC) Subject: [master] c9c9862bc varnishtest: New pkg_{version,branch} VTC macros Message-ID: <20220805092407.F33034535@lists.varnish-cache.org> commit c9c9862bc1f9864bf4cf3931d34469261c535bb9 Author: Dridi Boukelmoune Date: Tue Jul 26 10:51:21 2022 +0200 varnishtest: New pkg_{version,branch} VTC macros diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index dc05e2eaa..9080ab3b3 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -808,6 +808,9 @@ main(int argc, char * const *argv) else tmppath = strdup("/tmp"); + extmacro_def("pkg_version", NULL, PACKAGE_VERSION); + extmacro_def("pkg_branch", NULL, PACKAGE_BRANCH); + cwd = getcwd(buf, sizeof buf); extmacro_def("pwd", NULL, "%s", cwd); From dridi.boukelmoune at gmail.com Fri Aug 5 09:24:08 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 5 Aug 2022 09:24:08 +0000 (UTC) Subject: [master] cbaf1c3f2 resp: Don't overwrite Via headers Message-ID: <20220805092408.2753E453B@lists.varnish-cache.org> commit cbaf1c3f270b8d2e1ae0356d6c3a17fcccd97b0d Author: Dridi Boukelmoune Date: Tue Jul 26 11:04:41 2022 +0200 resp: Don't overwrite Via headers The order for Via entries follows the response return path, from the furthest proxy to the closest. Refs #3794 diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index c23270073..89b585306 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -167,7 +167,7 @@ Resp_Setup_Deliver(struct req *req) http_PrintfHeader(h, "Age: %.0f", floor(fmax(0., req->t_prev - oc->t_origin))); - http_SetHeader(h, "Via: 1.1 varnish (Varnish/" PACKAGE_BRANCH ")"); + http_AppendHeader(h, H_Via, "1.1 varnish (Varnish/" PACKAGE_BRANCH ")"); if (cache_param->http_gzip_support && ObjCheckFlag(req->wrk, oc, OF_GZIPED) && diff --git a/bin/varnishtest/tests/r03794.vtc b/bin/varnishtest/tests/r03794.vtc new file mode 100644 index 000000000..7535b0593 --- /dev/null +++ b/bin/varnishtest/tests/r03794.vtc @@ -0,0 +1,21 @@ +varnishtest "Append Via header" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend "" -start + +varnish v2 -vcl { + backend v1 { + .host = "${v1_sock}"; + } +} -start + +client c1 -connect ${v2_sock} { + txreq + rxresp + expect resp.http.via == \ + "1.1 varnish (Varnish/${pkg_branch}), 1.1 varnish (Varnish/${pkg_branch})" +} -run From dridi.boukelmoune at gmail.com Fri Aug 5 09:24:08 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 5 Aug 2022 09:24:08 +0000 (UTC) Subject: [master] 441cf2328 varnishtest: Predictable identity for Varnish instances Message-ID: <20220805092408.47997453F@lists.varnish-cache.org> commit 441cf2328de36cb7490dcd80cee4ff7c8b3328b8 Author: Dridi Boukelmoune Date: Tue Jul 26 11:14:39 2022 +0200 varnishtest: Predictable identity for Varnish instances It can still be overridden with `varnish vNAME -arg`, as usual. diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index c1739c45b..62e3deec0 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -402,8 +402,8 @@ varnish_launch(struct varnish *v) vsb = VSB_new_auto(); AN(vsb); VSB_cat(vsb, "cd ${pwd} &&"); - VSB_printf(vsb, " exec varnishd %s -d -n %s", - v->jail, v->workdir); + VSB_printf(vsb, " exec varnishd %s -d -n %s -i %s", + v->jail, v->workdir, v->name); VSB_cat(vsb, VSB_data(params_vsb)); if (leave_temp) { VSB_cat(vsb, " -p debug=+vcl_keep"); From dridi.boukelmoune at gmail.com Fri Aug 5 09:24:08 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 5 Aug 2022 09:24:08 +0000 (UTC) Subject: [master] 58b155152 http: Configure the Via header with server identity Message-ID: <20220805092408.6AB4B4545@lists.varnish-cache.org> commit 58b1551528ccf4a67b78f2bd71b54000d4a85c38 Author: Dridi Boukelmoune Date: Tue Jul 26 11:24:16 2022 +0200 http: Configure the Via header with server identity Change made to the Via header: -Via: 1.1 varnish (Varnish/x.y) +Via: 1.1 ${i_opt} (Varnish/x.y) It should actually better serve its purpose of informing about the hops a request (or in this case a response) went through. This is how Via is defined in HTTP: Via = #( received-protocol RWS received-by [ RWS comment ] ) Using the server identity (which defaults to the server host name) is more accurate for the received-by field. We still advertise Varnish in the optional comment field, and hopefully that's what analytics services rely on to recognize a server as a Varnish instance. A bit of trivia, a proxy MAY add a Via entry to the response and MUST add one to a forwarded request: Varnish only adds the optional one. Refs rfc9110/7.6.3. Fixes #3794 diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 3ef0512bf..e74104388 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -605,6 +605,7 @@ void http_AppendHeader(struct http *to, hdr_t, const char *val); void http_PrintfHeader(struct http *to, const char *fmt, ...) v_printflike_(2, 3); void http_TimeHeader(struct http *to, const char *fmt, vtim_real now); +const char * http_ViaHeader(void); void http_Proto(struct http *to); void http_SetHeader(struct http *to, const char *header); void http_SetH(struct http *to, unsigned n, const char *header); diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index e2f8ba726..b68c18638 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -35,6 +35,9 @@ #include "cache_varnishd.h" #include +#include + +#include "common/heritage.h" #include "vct.h" #include "vend.h" @@ -58,6 +61,8 @@ const char H__Status[] = "\010:status:"; const char H__Proto[] = "\007:proto:"; const char H__Reason[] = "\010:reason:"; +static char * via_hdr; + /*-------------------------------------------------------------------- * Perfect hash to rapidly recognize headers from tbl/http_headers.h * which have non-zero flags. @@ -178,9 +183,17 @@ http_init_hdr(char *hdr, int flg) void HTTP_Init(void) { + struct vsb *vsb; #define HTTPH(a, b, c) http_init_hdr(b, c); #include "tbl/http_headers.h" + + vsb = VSB_new_auto(); + VSB_printf(vsb, "1.1 %s (Varnish/" PACKAGE_BRANCH ")", + heritage.identity); + AZ(VSB_finish(vsb)); + REPLACE(via_hdr, VSB_data(vsb)); + VSB_destroy(&vsb); } /*-------------------------------------------------------------------- @@ -1541,6 +1554,13 @@ http_TimeHeader(struct http *to, const char *fmt, vtim_real now) http_SetH(to, to->nhd++, p); } +const char * +http_ViaHeader(void) +{ + + return (via_hdr); +} + /*--------------------------------------------------------------------*/ void diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 89b585306..9afba45b5 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -167,7 +167,7 @@ Resp_Setup_Deliver(struct req *req) http_PrintfHeader(h, "Age: %.0f", floor(fmax(0., req->t_prev - oc->t_origin))); - http_AppendHeader(h, H_Via, "1.1 varnish (Varnish/" PACKAGE_BRANCH ")"); + http_AppendHeader(h, H_Via, http_ViaHeader()); if (cache_param->http_gzip_support && ObjCheckFlag(req->wrk, oc, OF_GZIPED) && diff --git a/bin/varnishtest/tests/r03794.vtc b/bin/varnishtest/tests/r03794.vtc index 7535b0593..f9901a682 100644 --- a/bin/varnishtest/tests/r03794.vtc +++ b/bin/varnishtest/tests/r03794.vtc @@ -1,4 +1,4 @@ -varnishtest "Append Via header" +varnishtest "Append configurable Via header" server s1 { rxreq @@ -17,5 +17,5 @@ client c1 -connect ${v2_sock} { txreq rxresp expect resp.http.via == \ - "1.1 varnish (Varnish/${pkg_branch}), 1.1 varnish (Varnish/${pkg_branch})" + "1.1 v1 (Varnish/${pkg_branch}), 1.1 v2 (Varnish/${pkg_branch})" } -run From dridi.boukelmoune at gmail.com Fri Aug 5 09:24:08 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 5 Aug 2022 09:24:08 +0000 (UTC) Subject: [master] 0b297f643 req: Add a Via header entry Message-ID: <20220805092408.A9602455C@lists.varnish-cache.org> commit 0b297f6431b324e77078d225eda24cfbf20e4ba9 Author: Dridi Boukelmoune Date: Tue Jul 26 13:07:51 2022 +0200 req: Add a Via header entry > A proxy MUST send an appropriate Via header field, as described > below, in each message that it forwards. See rfc9110/7.6.3. diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 9afba45b5..56668026d 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -873,6 +873,7 @@ cnt_recv_prep(struct req *req, const char *ci) * it in the VSL log. */ http_AppendHeader(req->http, H_X_Forwarded_For, ci); + http_AppendHeader(req->http, H_Via, http_ViaHeader()); http_CollectHdr(req->http, H_Cache_Control); /* By default we use the first backend */ diff --git a/bin/varnishtest/tests/l00004.vtc b/bin/varnishtest/tests/l00004.vtc index 95ad36111..5978af8b2 100644 --- a/bin/varnishtest/tests/l00004.vtc +++ b/bin/varnishtest/tests/l00004.vtc @@ -9,6 +9,8 @@ server s1 { varnish v1 -vcl+backend { sub vcl_recv { + # make PipeAcct deterministic + unset req.http.via; return (pipe); } diff --git a/bin/varnishtest/tests/l00005.vtc b/bin/varnishtest/tests/l00005.vtc index 19459348a..51667672a 100644 --- a/bin/varnishtest/tests/l00005.vtc +++ b/bin/varnishtest/tests/l00005.vtc @@ -11,6 +11,10 @@ server s1 { } -start varnish v1 -vcl+backend { + sub vcl_recv { + # make BereqAcct deterministic + unset req.http.via; + } sub vcl_backend_fetch { unset bereq.http.x-forwarded-for; unset bereq.http.x-varnish; diff --git a/bin/varnishtest/tests/r03794.vtc b/bin/varnishtest/tests/r03794.vtc index f9901a682..6597ebbd4 100644 --- a/bin/varnishtest/tests/r03794.vtc +++ b/bin/varnishtest/tests/r03794.vtc @@ -2,6 +2,8 @@ varnishtest "Append configurable Via header" server s1 { rxreq + expect req.http.via == \ + "1.1 v2 (Varnish/${pkg_branch}), 1.1 v1 (Varnish/${pkg_branch})" txresp } -start diff --git a/bin/varnishtest/tests/v00051.vtc b/bin/varnishtest/tests/v00051.vtc index e13dae04b..db7d2e887 100644 --- a/bin/varnishtest/tests/v00051.vtc +++ b/bin/varnishtest/tests/v00051.vtc @@ -189,6 +189,7 @@ logexpect l1012 -v v1 -g vxid -q "vxid == 1012" { expect 0 = BereqHeader {^foo: pipe} expect 0 = BereqHeader {^Host: } expect 0 = BereqHeader {^X-Forwarded-For: } + expect 0 = BereqHeader {^Via: } expect 0 = BereqHeader {^X-Varnish: 1011} expect 0 = BereqHeader {^Connection: close} expect 0 = BereqAcct {^0 0 0 0 0 0} @@ -207,6 +208,7 @@ logexpect l1011 -v v1 -g vxid -q "vxid == 1011" { expect 0 = ReqHeader {^foo: pipe} expect 0 = ReqHeader {^Host: } expect 0 = ReqHeader {^X-Forwarded-For: } + expect 0 = ReqHeader {^Via: } expect 0 = VCL_call {^RECV} expect 0 = VCL_return {^pipe} expect 0 = VCL_call {^HASH} From dridi.boukelmoune at gmail.com Fri Aug 5 09:24:08 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 5 Aug 2022 09:24:08 +0000 (UTC) Subject: [master] a45d8afcf doc: Remove stale reference to VSM_Name() Message-ID: <20220805092408.EF2E84566@lists.varnish-cache.org> commit a45d8afcf937c58dd00ce1abc4bd51af34bebbcc Author: Dridi Boukelmoune Date: Thu Aug 4 08:48:46 2022 +0200 doc: Remove stale reference to VSM_Name() It disappeared from libvarnishapi during the 2.0 soname bump. Refs 74328de8eeb2703591fe59190f73fcabf057adbe Refs 07d4e752952eee1151257a71a7ad310bc3b1a3e0 diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index cdb08d40e..b97a6f374 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -216,8 +216,8 @@ Operations options -i identity Specify the identity of the Varnish server. This can be accessed - using ``server.identity`` from VCL and with VSM_Name() from - utilities. If not specified the output of gethostname(3) is used. + using ``server.identity`` from VCL. If not specified the output of + gethostname(3) is used. -I clifile From dridi.boukelmoune at gmail.com Fri Aug 5 09:24:09 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 5 Aug 2022 09:24:09 +0000 (UTC) Subject: [master] b0f98e363 varnishd: Restrict the -i argument to HTTP token syntax Message-ID: <20220805092409.3402C4577@lists.varnish-cache.org> commit b0f98e363ac177b10cefbf7187adc317093192a2 Author: Dridi Boukelmoune Date: Thu Aug 4 08:55:09 2022 +0200 varnishd: Restrict the -i argument to HTTP token syntax The reason to blindly trust gethostname(3) is that a system returning a broken host name should not prevent Varnish from starting when there is no explicit -i option in the command line. diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 6cc52a124..2bc920f65 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -51,6 +51,7 @@ #include "hash/hash_slinger.h" #include "libvcc.h" #include "vcli_serve.h" +#include "vct.h" #include "vend.h" #include "vev.h" #include "vfil.h" @@ -585,6 +586,7 @@ main(int argc, char * const *argv) const char *S_arg = NULL; const char *s_arg = "default,100m"; const char *W_arg = NULL; + const char *c; char *p; struct cli cli[1]; const char *err; @@ -816,6 +818,10 @@ main(int argc, char * const *argv) if (i_arg == NULL || *i_arg == '\0') i_arg = mgt_HostName(); + else for (c = i_arg; *c != '\0'; c++) { + if (!vct_istchar(*c)) + ARGV_ERR("Invalid character '%c' for -i\n", *c); + } heritage.identity = i_arg; mgt_ProcTitle("Mgt"); diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index b97a6f374..144910516 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -216,8 +216,14 @@ Operations options -i identity Specify the identity of the Varnish server. This can be accessed - using ``server.identity`` from VCL. If not specified the output of - gethostname(3) is used. + using ``server.identity`` from VCL. + + The server identity is used for the ``received-by`` field of ``Via`` + headers generated by Varnish. For this reason, it must be a valid + token as defined by the HTTP grammar. + + If not specified the output of ``gethostname(3)`` is used, in which + case the syntax is assumed to be correct. -I clifile From dridi.boukelmoune at gmail.com Fri Aug 5 11:42:06 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 5 Aug 2022 11:42:06 +0000 (UTC) Subject: [master] 189b60f6d vtc: Stabilize r1941 Message-ID: <20220805114207.0142E6325C@lists.varnish-cache.org> commit 189b60f6d3e5aa070d8c05ca4bd05e48c0e20be1 Author: Dridi Boukelmoune Date: Fri Aug 5 13:35:24 2022 +0200 vtc: Stabilize r1941 Under load, the backend connection may not be back in the pool by the time the ESI fetch starts. Also check the expected storage allocation failure happened. diff --git a/bin/varnishtest/tests/r01941.vtc b/bin/varnishtest/tests/r01941.vtc index 2561bf1fc..74b8b0507 100644 --- a/bin/varnishtest/tests/r01941.vtc +++ b/bin/varnishtest/tests/r01941.vtc @@ -9,6 +9,9 @@ server s1 { rxreq expect req.url == /big txresp -bodylen 1037480 + + expect_close + accept rxreq txresp -nolen -hdr "Content-Length: 6545" -body "" loop 92 { @@ -17,6 +20,9 @@ server s1 { send { />} } send "" + + expect_close + accept rxreq expect req.url == /long1234567890123456789012345678901234567890.txt txresp -body {fo} @@ -25,6 +31,9 @@ server s1 { varnish v1 -arg "-pfetch_chunksize=4k" \ -arg "-p feature=+esi_disable_xml_check" \ -arg "-sdefault,1m" -vcl+backend { + sub vcl_backend_fetch { + set bereq.http.connection = "close"; + } sub vcl_backend_response { set beresp.do_esi = true; } @@ -34,6 +43,9 @@ client c1 { txreq -url /big rxresp expect resp.bodylen == 1037480 - txreq -url / + + txreq rxresp } -run + +varnish v1 -expect SM?.s0.c_fail == 1 From phk at FreeBSD.org Fri Aug 5 13:33:06 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 5 Aug 2022 13:33:06 +0000 (UTC) Subject: [master] d0acf1191 Add missing assert to calm down flexelint. Message-ID: <20220805133307.0D2EF1018E3@lists.varnish-cache.org> commit d0acf119167149e1296d5e297449d010600e51c2 Author: Poul-Henning Kamp Date: Fri Aug 5 13:32:19 2022 +0000 Add missing assert to calm down flexelint. diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index b68c18638..ed15e07f9 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -189,6 +189,7 @@ HTTP_Init(void) #include "tbl/http_headers.h" vsb = VSB_new_auto(); + AN(vsb); VSB_printf(vsb, "1.1 %s (Varnish/" PACKAGE_BRANCH ")", heritage.identity); AZ(VSB_finish(vsb)); From phk at FreeBSD.org Fri Aug 5 14:02:05 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 5 Aug 2022 14:02:05 +0000 (UTC) Subject: [master] 759f38227 Cast to 64 bit to silence FlexeLint Message-ID: <20220805140205.3C18C1027E1@lists.varnish-cache.org> commit 759f38227d26e2bd47a6a0cf9879e19b505911d0 Author: Poul-Henning Kamp Date: Fri Aug 5 14:01:32 2022 +0000 Cast to 64 bit to silence FlexeLint diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index e43097ce7..ca7fc7d1d 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -193,7 +193,8 @@ vsl_get(unsigned len, unsigned records, unsigned flushes) VSC_C_main->shm_writes++; VSC_C_main->shm_flushes += flushes; VSC_C_main->shm_records += records; - VSC_C_main->shm_bytes += VSL_BYTES(VSL_OVERHEAD + VSL_WORDS(len)); + VSC_C_main->shm_bytes += + VSL_BYTES(VSL_OVERHEAD + VSL_WORDS((uint64_t)len)); /* Wrap if necessary */ if (VSL_END(vsl_ptr, len) >= vsl_end) From martin at varnish-software.com Fri Aug 5 15:04:06 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Fri, 5 Aug 2022 15:04:06 +0000 (UTC) Subject: [master] 57d5835dd Follow redirects when downloading dist Message-ID: <20220805150406.4D8B5104480@lists.varnish-cache.org> commit 57d5835dd6208771f9fb3b7ec2fd411c9e08196e Author: Martin Blix Grydeland Date: Thu Aug 4 15:33:26 2022 +0200 Follow redirects when downloading dist diff --git a/.circleci/config.yml b/.circleci/config.yml index 0a32ce358..8b20b9016 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -53,7 +53,7 @@ jobs: - run: name: Download the dist tarball command: | - curl -s << pipeline.parameters.dist-url >> -o varnish-dist.tar.gz + curl -Ls '<< pipeline.parameters.dist-url >>' -o varnish-dist.tar.gz - when: condition: << pipeline.parameters.dist-url-sha256 >> steps: From nils.goroll at uplex.de Mon Aug 8 14:02:06 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 8 Aug 2022 14:02:06 +0000 (UTC) Subject: [master] 4ef50851d Install vnum.h for vmod use Message-ID: <20220808140206.B31AA10E0D2@lists.varnish-cache.org> commit 4ef50851ddf700fe4fb6807375347b4d52d4ca47 Author: Nils Goroll Date: Wed Aug 3 17:13:33 2022 +0200 Install vnum.h for vmod use The vnum symbols are not exported yet, we will decide on that once we have a case for it. diff --git a/include/Makefile.am b/include/Makefile.am index 2fdb7a580..a7a77310d 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -66,6 +66,7 @@ nobase_pkginclude_HEADERS += \ vcl.h \ vcs.h \ vmod_abi.h \ + vnum.h \ vqueue.h \ vre.h \ vre_pcre2.h \ @@ -98,7 +99,6 @@ nobase_noinst_HEADERS = \ vjsn.h \ vlu.h \ vmb.h \ - vnum.h \ vpf.h \ vsc_priv.h \ vsl_priv.h \ From martin at varnish-software.com Tue Aug 9 08:55:06 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 9 Aug 2022 08:55:06 +0000 (UTC) Subject: [master] c5fd097e5 Do not call http_hdr_flags() on pseudo-headers Message-ID: <20220809085506.86700106FDF@lists.varnish-cache.org> commit c5fd097e5cce8b461c6443af02b3448baef2491d Author: Martin Blix Grydeland Date: Thu Aug 4 10:59:33 2022 +0200 Do not call http_hdr_flags() on pseudo-headers In http_EstimateWS(), all headers are passed to the http_isfiltered() function to calculate how many bytes is needed to serialize the entire struct http. http_isfiltered() will check the headers for whether they are going to be filtered out later and if so skip them. However http_isfiltered() would attempt to treat all elements of struct http as regular headers with an implicit structure. That does not hold for the first three pseudo-header entries, which would lead to asserts in later steps. This patch skips the filter step for pseudo-headers. Fixes: #3830 diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index ed15e07f9..d48c0bb36 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -1147,6 +1147,8 @@ http_isfiltered(const struct http *fm, unsigned u, unsigned how) if (fm->hdf[u] & HDF_FILTER) return (1); + if (u < HTTP_HDR_FIRST) + return (0); e = strchr(fm->hd[u].b, ':'); if (e == NULL) return (0); diff --git a/bin/varnishtest/tests/r03830.vtc b/bin/varnishtest/tests/r03830.vtc new file mode 100644 index 000000000..515598192 --- /dev/null +++ b/bin/varnishtest/tests/r03830.vtc @@ -0,0 +1,29 @@ +varnishtest "3830: Do not call http_hdr_flags() on pseudo-headers" + +server s1 { + rxreq + txresp -reason ":x" + + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + return (hash); + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run + +client c2 { + txreq -url :x -method :x + rxresp + expect resp.status == 200 +} -run + +varnish v1 -vsl_catchup From martin at varnish-software.com Tue Aug 9 08:55:06 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 9 Aug 2022 08:55:06 +0000 (UTC) Subject: [master] 3056e30aa Clean up assertions in http_hdr_flags() Message-ID: <20220809085506.9FC12106FE2@lists.varnish-cache.org> commit 3056e30aa4879653d2697167440b21f125b9fa7e Author: Martin Blix Grydeland Date: Thu Aug 4 11:04:37 2022 +0200 Clean up assertions in http_hdr_flags() The input argument assertions and checks in http_hdr_flags() were misleading and lacking. With this patch it returns (NULL) on either input being NULL, and also when called with an empty string instead of asserting. diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index d48c0bb36..6c9f1da6c 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -145,9 +145,9 @@ http_hdr_flags(const char *b, const char *e) unsigned u; struct http_hdrflg *retval; - if (e == NULL) + if (b == NULL || e == NULL) return (NULL); - assert(e > b); + assert(b <= e); u = (unsigned)(e - b); assert(b + u == e); if (u < 2 || u > 19) // MIN_WORD_LENGTH & MAX_WORD_LENGTH From martin at varnish-software.com Tue Aug 9 08:57:05 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 9 Aug 2022 08:57:05 +0000 (UTC) Subject: [7.0] 782f42fb1 Do not call http_hdr_flags() on pseudo-headers Message-ID: <20220809085705.2CE0D1074C3@lists.varnish-cache.org> commit 782f42fb14012d6464aead9ece5aa0a0727b1808 Author: Martin Blix Grydeland Date: Thu Aug 4 10:59:33 2022 +0200 Do not call http_hdr_flags() on pseudo-headers In http_EstimateWS(), all headers are passed to the http_isfiltered() function to calculate how many bytes is needed to serialize the entire struct http. http_isfiltered() will check the headers for whether they are going to be filtered out later and if so skip them. However http_isfiltered() would attempt to treat all elements of struct http as regular headers with an implicit structure. That does not hold for the first three pseudo-header entries, which would lead to asserts in later steps. This patch skips the filter step for pseudo-headers. Fixes: #3830 diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index c30d21ee6..b4eaf911c 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -1131,6 +1131,8 @@ http_isfiltered(const struct http *fm, unsigned u, unsigned how) if (fm->hdf[u] & HDF_FILTER) return (1); + if (u < HTTP_HDR_FIRST) + return (0); e = strchr(fm->hd[u].b, ':'); if (e == NULL) return (0); diff --git a/bin/varnishtest/tests/r03830.vtc b/bin/varnishtest/tests/r03830.vtc new file mode 100644 index 000000000..515598192 --- /dev/null +++ b/bin/varnishtest/tests/r03830.vtc @@ -0,0 +1,29 @@ +varnishtest "3830: Do not call http_hdr_flags() on pseudo-headers" + +server s1 { + rxreq + txresp -reason ":x" + + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + return (hash); + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run + +client c2 { + txreq -url :x -method :x + rxresp + expect resp.status == 200 +} -run + +varnish v1 -vsl_catchup From martin at varnish-software.com Tue Aug 9 08:57:05 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 9 Aug 2022 08:57:05 +0000 (UTC) Subject: [7.0] 6c784d0b3 Clean up assertions in http_hdr_flags() Message-ID: <20220809085705.469481074C6@lists.varnish-cache.org> commit 6c784d0b331d771030bcd1cd331fe6632982c784 Author: Martin Blix Grydeland Date: Thu Aug 4 11:04:37 2022 +0200 Clean up assertions in http_hdr_flags() The input argument assertions and checks in http_hdr_flags() were misleading and lacking. With this patch it returns (NULL) on either input being NULL, and also when called with an empty string instead of asserting. diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index b4eaf911c..1d0a87811 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -140,9 +140,9 @@ http_hdr_flags(const char *b, const char *e) unsigned u; struct http_hdrflg *retval; - if (e == NULL) - return(NULL); - assert(e > b); + if (b == NULL || e == NULL) + return (NULL); + assert(b <= e); u = (unsigned)(e - b); assert(b + u == e); if (u < 2 || u > 19) // MIN_WORD_LENGTH & MAX_WORD_LENGTH From martin at varnish-software.com Tue Aug 9 08:57:05 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 9 Aug 2022 08:57:05 +0000 (UTC) Subject: [7.0] 898456c01 Add #3830 to the changelog Message-ID: <20220809085705.611921074CB@lists.varnish-cache.org> commit 898456c0170a62faf1cf7278efb36abb84c9d3cf Author: Martin Blix Grydeland Date: Thu Aug 4 14:05:32 2022 +0200 Add #3830 to the changelog diff --git a/doc/changes.rst b/doc/changes.rst index f2efae99d..6f2108277 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -31,6 +31,15 @@ http://varnish-cache.org/docs/trunk/whats-new/index.html and via individual releases. These documents are updated as part of the release process. +================================ +Varnish Cache 7.0.3 (unreleased) +================================ + +* Do not filter pseudo-headers as regular headers (VSV00009_ / 3830_). + +.. _VSV00009: https://varnish-cache.org/security/VSV00009.html +.. _3830: https://github.com/varnishcache/varnish-cache/issues/3830 + ================================ Varnish Cache 7.0.2 (2022-01-25) ================================ From martin at varnish-software.com Tue Aug 9 08:57:05 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 9 Aug 2022 08:57:05 +0000 (UTC) Subject: [7.0] 6a4c6a5c7 Prepare for 7.0.3 Message-ID: <20220809085705.83A6A1074CF@lists.varnish-cache.org> commit 6a4c6a5c7e66a664b140278c209f0b18c544cab8 Author: Martin Blix Grydeland Date: Thu Aug 4 14:17:09 2022 +0200 Prepare for 7.0.3 diff --git a/configure.ac b/configure.ac index 0f2406011..bd8c68e7d 100644 --- a/configure.ac +++ b/configure.ac @@ -1,8 +1,8 @@ AC_PREREQ(2.69) AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS -Copyright (c) 2006-2021 Varnish Software]) +Copyright (c) 2006-2022 Varnish Software]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [7.0.2], [varnish-dev at varnish-cache.org]) +AC_INIT([Varnish], [7.0.3], [varnish-dev at varnish-cache.org]) AC_CONFIG_SRCDIR(include/miniobj.h) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/changes.rst b/doc/changes.rst index 6f2108277..84734b6c4 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -32,7 +32,7 @@ individual releases. These documents are updated as part of the release process. ================================ -Varnish Cache 7.0.3 (unreleased) +Varnish Cache 7.0.3 (2022-08-09) ================================ * Do not filter pseudo-headers as regular headers (VSV00009_ / 3830_). diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst index 16af4e23e..f3f31a36d 100644 --- a/doc/sphinx/index.rst +++ b/doc/sphinx/index.rst @@ -42,7 +42,7 @@ Longer listings like example command output and VCL look like this:: $ /opt/varnish/sbin/varnishd -V varnishd (varnish-trunk revision 1234567) Copyright (c) 2006 Verdens Gang AS - Copyright (c) 2006-2021 Varnish Software + Copyright (c) 2006-2022 Varnish Software .. For maintainers: diff --git a/lib/libvarnish/version.c b/lib/libvarnish/version.c index af4fd8835..99bbf2c42 100644 --- a/lib/libvarnish/version.c +++ b/lib/libvarnish/version.c @@ -74,7 +74,7 @@ VCS_String(const char *which) ")" "\n" "Copyright (c) 2006 Verdens Gang AS\n" - "Copyright (c) 2006-2021 Varnish Software\n" + "Copyright (c) 2006-2022 Varnish Software\n" ); default: WRONG("Wrong argument to VCS_String"); From martin at varnish-software.com Tue Aug 9 08:58:05 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 9 Aug 2022 08:58:05 +0000 (UTC) Subject: [7.1] 19544fdc6 Do not call http_hdr_flags() on pseudo-headers Message-ID: <20220809085805.8E1B5107B25@lists.varnish-cache.org> commit 19544fdc6649bd294f25314d9f609b4979b1fe48 Author: Martin Blix Grydeland Date: Thu Aug 4 10:59:33 2022 +0200 Do not call http_hdr_flags() on pseudo-headers In http_EstimateWS(), all headers are passed to the http_isfiltered() function to calculate how many bytes is needed to serialize the entire struct http. http_isfiltered() will check the headers for whether they are going to be filtered out later and if so skip them. However http_isfiltered() would attempt to treat all elements of struct http as regular headers with an implicit structure. That does not hold for the first three pseudo-header entries, which would lead to asserts in later steps. This patch skips the filter step for pseudo-headers. Fixes: #3830 diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 61849ab04..3f5ee0384 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -1132,6 +1132,8 @@ http_isfiltered(const struct http *fm, unsigned u, unsigned how) if (fm->hdf[u] & HDF_FILTER) return (1); + if (u < HTTP_HDR_FIRST) + return (0); e = strchr(fm->hd[u].b, ':'); if (e == NULL) return (0); diff --git a/bin/varnishtest/tests/r03830.vtc b/bin/varnishtest/tests/r03830.vtc new file mode 100644 index 000000000..515598192 --- /dev/null +++ b/bin/varnishtest/tests/r03830.vtc @@ -0,0 +1,29 @@ +varnishtest "3830: Do not call http_hdr_flags() on pseudo-headers" + +server s1 { + rxreq + txresp -reason ":x" + + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + return (hash); + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run + +client c2 { + txreq -url :x -method :x + rxresp + expect resp.status == 200 +} -run + +varnish v1 -vsl_catchup From martin at varnish-software.com Tue Aug 9 08:58:05 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 9 Aug 2022 08:58:05 +0000 (UTC) Subject: [7.1] 0fb3baff7 Clean up assertions in http_hdr_flags() Message-ID: <20220809085805.A7340107B29@lists.varnish-cache.org> commit 0fb3baff7963604a55be0ed6ebdf1e4654ead219 Author: Martin Blix Grydeland Date: Thu Aug 4 11:04:37 2022 +0200 Clean up assertions in http_hdr_flags() The input argument assertions and checks in http_hdr_flags() were misleading and lacking. With this patch it returns (NULL) on either input being NULL, and also when called with an empty string instead of asserting. diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 3f5ee0384..194055c3c 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -140,9 +140,9 @@ http_hdr_flags(const char *b, const char *e) unsigned u; struct http_hdrflg *retval; - if (e == NULL) - return(NULL); - assert(e > b); + if (b == NULL || e == NULL) + return (NULL); + assert(b <= e); u = (unsigned)(e - b); assert(b + u == e); if (u < 2 || u > 19) // MIN_WORD_LENGTH & MAX_WORD_LENGTH From martin at varnish-software.com Tue Aug 9 08:58:05 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 9 Aug 2022 08:58:05 +0000 (UTC) Subject: [7.1] 137d9814d Add #3830 to the changelog Message-ID: <20220809085805.C140B107B2D@lists.varnish-cache.org> commit 137d9814d7b214971c3ddcb04a6d68ca479f0a10 Author: Martin Blix Grydeland Date: Thu Aug 4 14:14:19 2022 +0200 Add #3830 to the changelog diff --git a/doc/changes.rst b/doc/changes.rst index 4b2371646..5ada6fadd 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -31,6 +31,15 @@ http://varnish-cache.org/docs/trunk/whats-new/index.html and via individual releases. These documents are updated as part of the release process. +================================ +Varnish Cache 7.1.1 (unreleased) +================================ + +* Do not filter pseudo-headers as regular headers (VSV00009_ / 3830_). + +.. _VSV00009: https://varnish-cache.org/security/VSV00009.html +.. _3830: https://github.com/varnishcache/varnish-cache/issues/3830 + ================================ Varnish Cache 7.1.0 (2022-03-15) ================================ From martin at varnish-software.com Tue Aug 9 08:58:05 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 9 Aug 2022 08:58:05 +0000 (UTC) Subject: [7.1] 7cee1c581 Prepare for 7.1.1 Message-ID: <20220809085805.DA991107B31@lists.varnish-cache.org> commit 7cee1c581bead20e88d101ab3d72afb29f14d87a Author: Martin Blix Grydeland Date: Thu Aug 4 14:18:08 2022 +0200 Prepare for 7.1.1 diff --git a/configure.ac b/configure.ac index 89619a96d..dc23e5b4a 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ AC_PREREQ(2.69) AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS Copyright (c) 2006-2022 Varnish Software]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [7.1.0], [varnish-dev at varnish-cache.org]) +AC_INIT([Varnish], [7.1.1], [varnish-dev at varnish-cache.org]) AC_CONFIG_SRCDIR(include/miniobj.h) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/changes.rst b/doc/changes.rst index 5ada6fadd..95ff54150 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -32,7 +32,7 @@ individual releases. These documents are updated as part of the release process. ================================ -Varnish Cache 7.1.1 (unreleased) +Varnish Cache 7.1.1 (2022-08-09) ================================ * Do not filter pseudo-headers as regular headers (VSV00009_ / 3830_). From nils.goroll at uplex.de Tue Aug 9 16:08:07 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 9 Aug 2022 16:08:07 +0000 (UTC) Subject: [master] 3a22f5f0d allow bgthreads to terminate for vmod use Message-ID: <20220809160807.BC2D511AF1C@lists.varnish-cache.org> commit 3a22f5f0ded31f8c7340ac00a382ae9106c20a8c Author: Nils Goroll Date: Tue Aug 9 17:25:01 2022 +0200 allow bgthreads to terminate for vmod use diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index e8e6f6154..9671413b6 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -89,7 +89,6 @@ wrk_bgthread(void *arg) struct worker wrk; struct worker_priv wpriv[1]; struct VSC_main_wrk ds; - void *r; CAST_OBJ_NOTNULL(bt, arg, BGTHREAD_MAGIC); THR_SetName(bt->name); @@ -100,12 +99,7 @@ wrk_bgthread(void *arg) memset(&ds, 0, sizeof ds); wrk.stats = &ds; - r = bt->func(&wrk, bt->priv); - - if (! cache_shutdown) - WRONG("BgThread terminated"); - - return (r); + return (bt->func(&wrk, bt->priv)); } void From phk at FreeBSD.org Wed Aug 10 08:01:11 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 10 Aug 2022 08:01:11 +0000 (UTC) Subject: [master] b92430cc9 Eliminate now unused `cache_shutdown` variable Message-ID: <20220810080111.DBE0E1104CD@lists.varnish-cache.org> commit b92430cc958bbd09260cf65d40834af434814e13 Author: Poul-Henning Kamp Date: Wed Aug 10 08:00:59 2022 +0000 Eliminate now unused `cache_shutdown` variable diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index ddcc8d8cd..b8a8174e1 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -54,8 +54,6 @@ #include "hash/hash_slinger.h" -int cache_shutdown = 0; - volatile struct params *cache_param; static pthread_mutex_t cache_vrnd_mtx; static vtim_dur shutdown_delay = 0; @@ -453,8 +451,6 @@ child_main(int sigmagic, size_t altstksz) CLI_Run(); - cache_shutdown = 1; - if (shutdown_delay > 0) VTIM_sleep(shutdown_delay); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index f5c377448..b9d60fc65 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -243,7 +243,6 @@ void EXP_Rearm(struct objcore *oc, vtim_real now, vtim_dur ttl, vtim_dur grace, vtim_dur keep); /* From cache_main.c */ -extern int cache_shutdown; void BAN_Init(void); void BAN_Compile(void); void BAN_Shutdown(void); From phk at FreeBSD.org Wed Aug 10 08:49:06 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 10 Aug 2022 08:49:06 +0000 (UTC) Subject: [master] 82f2f25d2 About trusting backends a bit less Message-ID: <20220810084906.0AFAC111B32@lists.varnish-cache.org> commit 82f2f25d289c4737adc72f0d34846af91f1b64e0 Author: Poul-Henning Kamp Date: Wed Aug 10 08:48:18 2022 +0000 About trusting backends a bit less diff --git a/doc/sphinx/phk/barriers.rst b/doc/sphinx/phk/barriers.rst index cc2728b92..794e57e36 100644 --- a/doc/sphinx/phk/barriers.rst +++ b/doc/sphinx/phk/barriers.rst @@ -14,7 +14,9 @@ if you find yourself thinking "Why did he do _that_ ? the answer has to do with security. The Varnish security model is based on some very crude but easy to understand -barriers between the various components:: +barriers between the various components: + +.. code-block:: text .-->- provides ->---------------------------------------. | | | diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst index c0374116c..f7ccde00d 100644 --- a/doc/sphinx/phk/index.rst +++ b/doc/sphinx/phk/index.rst @@ -13,6 +13,7 @@ You may or may not want to know what Poul-Henning thinks. .. toctree:: :maxdepth: 1 + routine.rst 503aroundtheworld.rst legacy.rst ip_address.rst diff --git a/doc/sphinx/phk/routine.rst b/doc/sphinx/phk/routine.rst new file mode 100644 index 000000000..8cd776ce0 --- /dev/null +++ b/doc/sphinx/phk/routine.rst @@ -0,0 +1,42 @@ +.. + Copyright (c) 2022 Varnish Software AS + SPDX-License-Identifier: BSD-2-Clause + See LICENSE file for full text of license + +.. _phk_routine: + +======================== +Getting into the routine +======================== + +Yesterday we released `VSV00009 `_, a pretty +harmless DoS from the backend side, which could trivially be mitigated +in VCL. + +By now handling security issues seem to have become routine for the +project, which is good, because that is the world we live in, and +bad, because we live in a world where that is a necessary skill. + +From the very start of the project, we have treated backends +as "trusted", in the sense that a lot of nasty stuff we try to handle +from clients got "dont do that then" treatment from the backend. + +That was back when "cloud" were called "mainframes" and "containers" +were called "jails", way back when CDNs were only for companies +with more money than skill. + +Part of the reasoning was also maximizing compatibility. + +Backends were a lot more - let us call it "heterogenous" - back +then. Some of them were literally kludges nailed to the side of +legacy newspaper production systems, and sometimes it was obvious +that they had not heard about RFCs. + +For the problem we fixed yesterday, one line of VCL took care of +the problem, but that is not guaranteed to always be the case. + +These days the "web" is a lot more regimented, and expecting +standards-compliance from backends makes sense, so we will +tighten the screws in that department as an ongoing activity. + +Poul-Henning, 2022-08-05 From nils.goroll at uplex.de Wed Aug 10 09:05:06 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 10 Aug 2022 09:05:06 +0000 (UTC) Subject: [master] 4a50fa61d Silence Flexelint Message-ID: <20220810090506.C360A1124C2@lists.varnish-cache.org> commit 4a50fa61d64835798f4899450cb4a1bae427ae0d Author: Nils Goroll Date: Wed Aug 10 11:03:40 2022 +0200 Silence Flexelint I was tempted to declare the parameter _Bool or even include stdbool, but did not dare to move after having read doc/sphinx/phk/thetoolsweworkwith.rst again ;) diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 2bc920f65..9965ea656 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -992,6 +992,7 @@ main(int argc, char * const *argv) mgt_cli_close_all(); VEV_Destroy(&mgt_evb); VJ_master(JAIL_MASTER_SYSTEM); + /*lint -e(730)*/ vext_cleanup(! MGT_DO_DEBUG(DBG_VMOD_SO_KEEP)); (void)rmdir("vext_cache"); VJ_master(JAIL_MASTER_LOW); From nils.goroll at uplex.de Wed Aug 10 09:13:05 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 10 Aug 2022 09:13:05 +0000 (UTC) Subject: [master] e9b41f869 Flexelinting Message-ID: <20220810091305.8CAD2112A4E@lists.varnish-cache.org> commit e9b41f869f6dad11636376608d3af84f1726c528 Author: Nils Goroll Date: Wed Aug 10 11:09:11 2022 +0200 Flexelinting based on my local installation, which is a bit more picky than the v-c.o one. diff --git a/bin/varnishd/cache/cache_conn_pool.c b/bin/varnishd/cache/cache_conn_pool.c index 2a2a76ca8..f5bdbb015 100644 --- a/bin/varnishd/cache/cache_conn_pool.c +++ b/bin/varnishd/cache/cache_conn_pool.c @@ -37,7 +37,6 @@ #include "cache_varnishd.h" -#include "vend.h" #include "vsa.h" #include "vsha256.h" #include "vtcp.h" diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 31b98c9c5..658f927b0 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -616,7 +616,7 @@ pan_backtrace(struct vsb *vsb) #else /* WITH_UNWIND */ -#if ENABLE_SANITIZER +#if defined(ENABLE_SANITIZER) # define BACKTRACE_LEVELS 20 #else # define BACKTRACE_LEVELS 10 diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 9965ea656..544359db9 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -52,7 +52,6 @@ #include "libvcc.h" #include "vcli_serve.h" #include "vct.h" -#include "vend.h" #include "vev.h" #include "vfil.h" #include "vin.h" From dridi.boukelmoune at gmail.com Fri Aug 12 08:14:14 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 12 Aug 2022 08:14:14 +0000 (UTC) Subject: [master] def50e60c ws: Polish debug logs to make them easier to follow Message-ID: <20220812081414.41A9E118424@lists.varnish-cache.org> commit def50e60ca07892d8b8d31ac3820f4f78dcb5696 Author: Dridi Boukelmoune Date: Fri Aug 12 10:02:26 2022 +0200 ws: Polish debug logs to make them easier to follow diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index 4f7d5f4f2..45295fcb3 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -44,8 +44,8 @@ WS_Assert(const struct ws *ws) { CHECK_OBJ_NOTNULL(ws, WS_MAGIC); - DSLb(DBG_WORKSPACE, "WS(%p) = (%s, %p %zu %zu %zu)", - ws, ws->id, ws->s, pdiff(ws->s, ws->f), + DSLb(DBG_WORKSPACE, "WS(%s, %p) = {%p %zu %zu %zu}", + ws->id, ws, ws->s, pdiff(ws->s, ws->f), ws->r == NULL ? 0 : pdiff(ws->f, ws->r), pdiff(ws->s, ws->e)); assert(ws->s != NULL); @@ -85,7 +85,7 @@ WS_Init(struct ws *ws, const char *id, void *space, unsigned len) unsigned l; DSLb(DBG_WORKSPACE, - "WS_Init(%p, \"%s\", %p, %u)", ws, id, space, len); + "WS_Init(%s, %p, %p, %u)", id, ws, space, len); assert(space != NULL); INIT_OBJ(ws, WS_MAGIC); ws->s = space; @@ -116,12 +116,12 @@ WS_Reset(struct ws *ws, uintptr_t pp) WS_Assert(ws); AN(pp); if (pp == snap_overflowed) { - DSLb(DBG_WORKSPACE, "WS_Reset(%p, overflowed)", ws); + DSLb(DBG_WORKSPACE, "WS_Reset(%s, %p, overflowed)", ws->id, ws); AN(WS_Overflowed(ws)); return; } p = (char *)pp; - DSLb(DBG_WORKSPACE, "WS_Reset(%p, %p)", ws, p); + DSLb(DBG_WORKSPACE, "WS_Reset(%s, %p, %p)", ws->id, ws, p); assert(ws->r == NULL); assert(p >= ws->s); assert(p <= ws->e); @@ -176,7 +176,7 @@ WS_Alloc(struct ws *ws, unsigned bytes) } r = ws->f; ws->f += bytes; - DSLb(DBG_WORKSPACE, "WS_Alloc(%p, %u) = %p", ws, bytes, r); + DSLb(DBG_WORKSPACE, "WS_Alloc(%s, %p, %u) = %p", ws->id, ws, bytes, r); WS_Assert(ws); return (r); } @@ -202,7 +202,7 @@ WS_Copy(struct ws *ws, const void *str, int len) r = ws->f; ws->f += bytes; memcpy(r, str, len); - DSLb(DBG_WORKSPACE, "WS_Copy(%p, %d) = %p", ws, len, r); + DSLb(DBG_WORKSPACE, "WS_Copy(%s, %p, %d) = %p", ws->id, ws, len, r); WS_Assert(ws); return (r); } @@ -214,10 +214,11 @@ WS_Snapshot(struct ws *ws) WS_Assert(ws); assert(ws->r == NULL); if (WS_Overflowed(ws)) { - DSLb(DBG_WORKSPACE, "WS_Snapshot(%p) = overflowed", ws); + DSLb(DBG_WORKSPACE, "WS_Snapshot(%s, %p) = overflowed", + ws->id, ws); return (snap_overflowed); } - DSLb(DBG_WORKSPACE, "WS_Snapshot(%p) = %p", ws, ws->f); + DSLb(DBG_WORKSPACE, "WS_Snapshot(%s, %p) = %p", ws->id, ws, ws->f); return ((uintptr_t)ws->f); } @@ -236,7 +237,7 @@ WS_ReserveAll(struct ws *ws) b = pdiff(ws->f, ws->r); WS_Assert(ws); - DSLb(DBG_WORKSPACE, "WS_ReserveAll(%p) = %u", ws, b); + DSLb(DBG_WORKSPACE, "WS_ReserveAll(%s, %p) = %u", ws->id, ws, b); return (b); } @@ -259,7 +260,7 @@ WS_ReserveSize(struct ws *ws, unsigned bytes) return (0); } ws->r = ws->f + bytes; - DSLb(DBG_WORKSPACE, "WS_ReserveSize(%p, %u/%u) = %u", + DSLb(DBG_WORKSPACE, "WS_ReserveSize(%s, %p, %u/%u) = %u", ws->id, ws, bytes, l, bytes); WS_Assert(ws); return (bytes); @@ -270,7 +271,7 @@ WS_Release(struct ws *ws, unsigned bytes) { WS_Assert(ws); assert(bytes <= ws->e - ws->f); - DSLb(DBG_WORKSPACE, "WS_Release(%p, %u)", ws, bytes); + DSLb(DBG_WORKSPACE, "WS_Release(%s, %p, %u)", ws->id, ws, bytes); assert(ws->r != NULL); assert(ws->f + bytes <= ws->r); ws->f += PRNDUP(bytes); @@ -282,7 +283,8 @@ void WS_ReleaseP(struct ws *ws, const char *ptr) { WS_Assert(ws); - DSLb(DBG_WORKSPACE, "WS_ReleaseP(%p, %p (%zd))", ws, ptr, ptr - ws->f); + DSLb(DBG_WORKSPACE, "WS_ReleaseP(%s, %p, %p (%zd))", ws->id, ws, ptr, + ptr - ws->f); assert(ws->r != NULL); assert(ptr >= ws->f); assert(ptr <= ws->r); From phk at FreeBSD.org Mon Aug 15 08:13:10 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Aug 2022 08:13:10 +0000 (UTC) Subject: [master] 2aa70ca53 Eliminate an unwarranted microoptimization. Message-ID: <20220815081310.9587B4B42@lists.varnish-cache.org> commit 2aa70ca53438a5aaa915ed1986a37de82dc3485a Author: Poul-Henning Kamp Date: Mon Aug 15 08:12:31 2022 +0000 Eliminate an unwarranted microoptimization. diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 9414c324d..619729e0d 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -523,7 +523,7 @@ static int v_matchproto_(vdp_bytes_f) ved_pretend_gzip_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, const void *pv, ssize_t l) { - uint8_t buf1[5], buf2[5]; + uint8_t buf[5]; const uint8_t *p; uint16_t lx; struct ecx *ecx; @@ -541,24 +541,17 @@ ved_pretend_gzip_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, ecx->crc = crc32(ecx->crc, p, l); ecx->l_crc += l; - lx = 65535; - buf1[0] = 0; - vle16enc(buf1 + 1, lx); - vle16enc(buf1 + 3, ~lx); - while (l > 0) { - if (l >= 65535) { + if (l >= 65535) lx = 65535; - if (ved_bytes(ecx, VDP_NULL, buf1, sizeof buf1)) - return (-1); - } else { + else lx = (uint16_t)l; - buf2[0] = 0; - vle16enc(buf2 + 1, lx); - vle16enc(buf2 + 3, ~lx); - if (ved_bytes(ecx, VDP_NULL, buf2, sizeof buf2)) - return (-1); - } + lx = (uint16_t)l; + buf[0] = 0; + vle16enc(buf + 1, lx); + vle16enc(buf + 3, ~lx); + if (ved_bytes(ecx, VDP_NULL, buf, sizeof buf)) + return (-1); if (ved_bytes(ecx, VDP_NULL, p, lx)) return (-1); l -= lx; From phk at FreeBSD.org Mon Aug 15 09:05:06 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Aug 2022 09:05:06 +0000 (UTC) Subject: [master] 7c287eddb Duh! Monday morning typo. Message-ID: <20220815090506.2887379AD@lists.varnish-cache.org> commit 7c287eddb311b4f2f0c84042cd1ae2cb9b119e94 Author: Poul-Henning Kamp Date: Mon Aug 15 09:04:50 2022 +0000 Duh! Monday morning typo. diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 619729e0d..7f7e95ac3 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -546,7 +546,6 @@ ved_pretend_gzip_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, lx = 65535; else lx = (uint16_t)l; - lx = (uint16_t)l; buf[0] = 0; vle16enc(buf + 1, lx); vle16enc(buf + 3, ~lx); From phk at FreeBSD.org Mon Aug 15 09:26:05 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Aug 2022 09:26:05 +0000 (UTC) Subject: [master] 91e130fe8 Revert last two changes, I totally misunderstood why there are two buffers. Message-ID: <20220815092605.298E296F7@lists.varnish-cache.org> commit 91e130fe8f095406782c5142637b8f4b367461f2 Author: Poul-Henning Kamp Date: Mon Aug 15 09:24:37 2022 +0000 Revert last two changes, I totally misunderstood why there are two buffers. Add a comment so nobody makes that mistake again. Also: Dont trust brain to work in heat-wave. diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 7f7e95ac3..245fcd87c 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -523,7 +523,7 @@ static int v_matchproto_(vdp_bytes_f) ved_pretend_gzip_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, const void *pv, ssize_t l) { - uint8_t buf[5]; + uint8_t buf1[5], buf2[5]; const uint8_t *p; uint16_t lx; struct ecx *ecx; @@ -541,22 +541,34 @@ ved_pretend_gzip_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, ecx->crc = crc32(ecx->crc, p, l); ecx->l_crc += l; + /* + * buf1 can safely be emitted multiple times for objects longer + * than 64K-1 bytes. + */ + lx = 65535; + buf1[0] = 0; + vle16enc(buf1 + 1, lx); + vle16enc(buf1 + 3, ~lx); + while (l > 0) { - if (l >= 65535) + if (l >= 65535) { lx = 65535; - else + if (ved_bytes(ecx, VDP_NULL, buf1, sizeof buf1)) + return (-1); + } else { lx = (uint16_t)l; - buf[0] = 0; - vle16enc(buf + 1, lx); - vle16enc(buf + 3, ~lx); - if (ved_bytes(ecx, VDP_NULL, buf, sizeof buf)) - return (-1); + buf2[0] = 0; + vle16enc(buf2 + 1, lx); + vle16enc(buf2 + 3, ~lx); + if (ved_bytes(ecx, VDP_NULL, buf2, sizeof buf2)) + return (-1); + } if (ved_bytes(ecx, VDP_NULL, p, lx)) return (-1); l -= lx; p += lx; } - /* buf1 & buf2 is local, have to flush */ + /* buf1 & buf2 are local, so we have to flush */ return (ved_bytes(ecx, VDP_FLUSH, NULL, 0)); } From nils.goroll at uplex.de Mon Aug 15 13:38:05 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Aug 2022 13:38:05 +0000 (UTC) Subject: [master] a8483cf5c Give VPI its private struct in the worker Message-ID: <20220815133805.C077C96C45@lists.varnish-cache.org> commit a8483cf5c864a87f59b2cc26315f37e9edda2d97 Author: Nils Goroll Date: Mon Feb 1 17:27:12 2021 +0100 Give VPI its private struct in the worker (struct vrt_ctx).handling is a special animal: As seen by VGC, struct vrt_ctx is const (and rightly so), yet VCL needs to have a way to communicate up to core code the return value from subs. So, handling was a (non-const) pointer into struct worker. This is all good and well, with the minor drawback that VMODs would get to see it, while we should make it very clear that they have to go through VRT_handling() / VRT_handled() and know the rules (no reset of handling). As we now want to move also the vpi count into "the ctx", before adding another unsigned pointer, we introduce struct wrk_vpi, provided by the worker, but owned by vpi. With this change, we thus formalize the vpi count requirement and also remove the handling variable from vmods' direct line of sight. This commit comprises the manual changes which either could not be or were not worth being automated. The next two commits contain a coccinelle script to do the rest of the work, and the reult of it. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index e74104388..af090b6fe 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -244,7 +244,8 @@ struct worker { unsigned cur_method; unsigned seen_methods; - unsigned handling; + + struct wrk_vpi *vpi; }; /* Stored object ----------------------------------------------------- diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 4dcf0d8fe..11b7e5044 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -38,6 +38,7 @@ #include "storage/storage.h" #include "vcl.h" #include "vtim.h" +#include "vcc_interface.h" #define FETCH_STEPS \ FETCH_STEP(mkbereq, MKBEREQ) \ diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 658f927b0..dd288c1f6 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -43,6 +43,7 @@ #include "cache_varnishd.h" #include "cache_transport.h" +#include "vcc_interface.h" #include "cache_filter.h" #include "common/heritage.h" diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 56668026d..48f5d5c7b 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -44,6 +44,7 @@ #include "cache_filter.h" #include "cache_objhead.h" #include "cache_transport.h" +#include "vcc_interface.h" #include "hash/hash_slinger.h" #include "http1/cache_http1.h" diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index b9d60fc65..eb3981da0 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -524,6 +524,10 @@ void WRK_Init(void); void WRK_AddStat(const struct worker *); void WRK_Log(enum VSL_tag_e, const char *, ...); +/* cache_vpi.c */ +extern const size_t vpi_wrk_len; +void VPI_wrk_init(struct worker *, void *, size_t); + /* cache_ws.c */ void WS_Panic(struct vsb *, const struct ws *); static inline int diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 293c199ba..97490920d 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -68,7 +68,7 @@ struct lock vcl_mtx; struct vcl *vcl_active; /* protected by vcl_mtx */ static struct vrt_ctx ctx_cli; -static unsigned handling_cli; +static struct wrk_vpi wrk_vpi_cli; static struct ws ws_cli; static uintptr_t ws_snapshot_cli; static struct vsl_log vsl_cli; @@ -125,10 +125,9 @@ VCL_Get_CliCtx(int msg) { ASSERT_CLI(); - AZ(ctx_cli.handling); INIT_OBJ(&ctx_cli, VRT_CTX_MAGIC); - handling_cli = 0; - ctx_cli.handling = &handling_cli; + INIT_OBJ(&wrk_vpi_cli, WRK_VPI_MAGIC); + ctx_cli.vpi = &wrk_vpi_cli; ctx_cli.now = VTIM_real(); if (msg) { ctx_cli.msg = VSB_new_auto(); @@ -156,7 +155,7 @@ VCL_Rel_CliCtx(struct vrt_ctx **ctx) ASSERT_CLI(); assert(*ctx == &ctx_cli); - AN((*ctx)->handling); + AN((*ctx)->vpi); if (ctx_cli.msg) { TAKE_OBJ_NOTNULL(r, &ctx_cli.msg, VSB_MAGIC); AZ(VSB_finish(r)); diff --git a/bin/varnishd/cache/cache_vpi.c b/bin/varnishd/cache/cache_vpi.c index f6e09dd86..d2b149701 100644 --- a/bin/varnishd/cache/cache_vpi.c +++ b/bin/varnishd/cache/cache_vpi.c @@ -45,6 +45,20 @@ * Private & exclusive interfaces between VCC and varnishd */ +const size_t vpi_wrk_len = sizeof(struct wrk_vpi); + +void +VPI_wrk_init(struct worker *wrk, void *p, size_t spc) +{ + struct wrk_vpi *vpi = p; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + AN(vpi); + assert(spc >= sizeof *vpi); + INIT_OBJ(vpi, WRK_VPI_MAGIC); + wrk->vpi = vpi; +} + void VPI_count(VRT_CTX, unsigned u) { diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index 03e684834..685bb451c 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -38,6 +38,7 @@ #include "cache_varnishd.h" #include "vcl.h" +#include "vcc_interface.h" struct vrt_priv { unsigned magic; diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c index a71edc9a3..976930e4e 100644 --- a/bin/varnishd/cache/cache_vrt_vcl.c +++ b/bin/varnishd/cache/cache_vrt_vcl.c @@ -583,7 +583,7 @@ VCL_##func##_method(struct vcl *vcl, struct worker *wrk, \ CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); \ vcl_call_method(wrk, req, bo, specific, \ VCL_MET_ ## upper, vcl->conf->func##_func, vcl->conf->nsub);\ - AN((1U << wrk->handling) & bitmap); \ + AN((1U << wrk->vpi->handling) & bitmap); \ } #include "tbl/vcl_returns.h" diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 9671413b6..3a5c4bddd 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -96,6 +96,7 @@ wrk_bgthread(void *arg) INIT_OBJ(&wrk, WORKER_MAGIC); INIT_OBJ(wpriv, WORKER_PRIV_MAGIC); wrk.wpriv = wpriv; + // bgthreads do not have a vpi member memset(&ds, 0, sizeof ds); wrk.stats = &ds; @@ -126,6 +127,7 @@ WRK_Thread(struct pool *qp, size_t stacksize, unsigned thread_workspace) struct VSC_main_wrk ds; unsigned char ws[thread_workspace]; struct worker_priv wpriv[1]; + unsigned char vpi[vpi_wrk_len]; AN(qp); AN(stacksize); @@ -143,6 +145,8 @@ WRK_Thread(struct pool *qp, size_t stacksize, unsigned thread_workspace) AZ(pthread_cond_init(&w->cond, NULL)); WS_Init(w->aws, "wrk", ws, thread_workspace); + VPI_wrk_init(w, vpi, sizeof vpi); + AN(w->vpi); VSL(SLT_WorkThread, 0, "%p start", w); diff --git a/include/vcc_interface.h b/include/vcc_interface.h index c1a74fc37..4a51868cd 100644 --- a/include/vcc_interface.h +++ b/include/vcc_interface.h @@ -53,6 +53,14 @@ struct vpi_ref { const char *token; }; +/* VPI's private part of the worker */ +struct wrk_vpi { + unsigned magic; +#define WRK_VPI_MAGIC 0xaa3d3df3 + unsigned handling; +}; + + void VPI_count(VRT_CTX, unsigned); void VPI_vcl_fini(VRT_CTX); diff --git a/include/vrt.h b/include/vrt.h index c4afdc6b6..e5e5647bf 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -279,6 +279,7 @@ struct VSC_main; struct vsc_seg; struct vsl_log; struct vsmw_cluster; +struct wrk_vpi; struct ws; typedef const struct stream_close *stream_close_t; @@ -385,7 +386,8 @@ struct vrt_ctx { unsigned syntax; unsigned vclver; unsigned method; - unsigned *handling; + + struct wrk_vpi *vpi; /* * msg is for error messages and exists only for diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index e0b065a8a..442f4f9ca 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -451,7 +451,7 @@ EmitInitFini(const struct vcc *tl) if (VSB_len(p->ini)) Fc(tl, 0, "\t/* %u */\n%s\n", p->n, VSB_data(p->ini)); if (p->ignore_errors == 0) { - Fc(tl, 0, "\tif (*ctx->handling == VCL_RET_FAIL)\n"); + Fc(tl, 0, "\tif (ctx->vpi->handling == VCL_RET_FAIL)\n"); Fc(tl, 0, "\t\treturn(1);\n"); } Fc(tl, 0, "\tvgc_inistep = %u;\n\n", p->n); @@ -464,9 +464,9 @@ EmitInitFini(const struct vcc *tl) /* Handle failures from vcl_init */ Fc(tl, 0, "\n"); - Fc(tl, 0, "\tif (*ctx->handling != VCL_RET_OK)\n"); + Fc(tl, 0, "\tif (ctx->vpi->handling != VCL_RET_OK)\n"); Fc(tl, 0, "\t\treturn(1);\n"); - Fc(tl, 0, "\t*ctx->handling = 0;\n"); + Fc(tl, 0, "\tctx->vpi->handling = 0;\n"); VTAILQ_FOREACH(sy, &tl->sym_objects, sideways) { Fc(tl, 0, "\tif (!%s) {\n", sy->rname); @@ -620,7 +620,7 @@ vcc_CompileSource(struct vcc *tl, struct source *sp, const char *jfile) Fh(tl, 0, "/* ---===### VCC generated .h code ###===---*/\n"); Fc(tl, 0, "\n/* ---===### VCC generated .c code ###===---*/\n"); - Fc(tl, 0, "\n#define END_ if (*ctx->handling) return\n"); + Fc(tl, 0, "\n#define END_ if (ctx->vpi->handling) return\n"); vcc_Parse_Init(tl); diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index d34d1f227..f8c603204 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -38,6 +38,8 @@ #include "cache/cache_varnishd.h" #include "cache/cache_filter.h" +#include "vcc_interface.h" // struct wrk_vpi + #include "vsa.h" #include "vtim.h" @@ -458,7 +460,7 @@ event_load(VRT_CTX, struct vmod_priv *priv) // This should fail AN(VRT_AddFilter(ctx, &xyzzy_vfp_rot13, &xyzzy_vdp_rot13)); // Reset the error, we know what we're doing. - *ctx->handling = 0; + ctx->vpi->handling = 0; VRT_RemoveFilter(ctx, &xyzzy_vfp_rot13, &xyzzy_vdp_rot13); AZ(VRT_AddFilter(ctx, &xyzzy_vfp_rot13, &xyzzy_vdp_rot13)); diff --git a/vmod/vmod_debug_dyn.c b/vmod/vmod_debug_dyn.c index 8ab467085..6b562cfc7 100644 --- a/vmod/vmod_debug_dyn.c +++ b/vmod/vmod_debug_dyn.c @@ -120,8 +120,7 @@ xyzzy_dyn__init(VRT_CTX, struct xyzzy_debug_dyn **dynp, AN(vcl_name); if (*addr == '\0' || *port == '\0') { - AN(ctx->handling); - AZ(*ctx->handling); + AZ(VRT_handled(ctx)); VRT_fail(ctx, "Missing dynamic backend address or port"); return; } From nils.goroll at uplex.de Mon Aug 15 13:38:05 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Aug 2022 13:38:05 +0000 (UTC) Subject: [master] f5b0b201f Coccinelle script to do remaining work to conclude the previous commit Message-ID: <20220815133805.D5E0C96C48@lists.varnish-cache.org> commit f5b0b201fe29999d5ebc1458b354a7459eed9eac Author: Nils Goroll Date: Mon Feb 1 18:11:17 2021 +0100 Coccinelle script to do remaining work to conclude the previous commit diff --git a/tools/coccinelle/archive/wrk_vpi.cocci b/tools/coccinelle/archive/wrk_vpi.cocci new file mode 100644 index 000000000..bea1e285b --- /dev/null +++ b/tools/coccinelle/archive/wrk_vpi.cocci @@ -0,0 +1,55 @@ +/* + * This patch was used to introduce struct wrk_vpi once + * + * Retained for reference only + */ +using "varnish.iso" + +@@ +idexpression struct worker *wrk; +@@ + +- wrk->handling ++ wrk->vpi->handling + +@@ +idexpression struct vrt_ctx *ctx; +expression reqbo; +@@ + +- ctx->handling = &reqbo->wrk->handling; ++ ctx->vpi = reqbo->wrk->vpi; + +@@ +idexpression struct vrt_ctx *ctx; +expression X; +@@ + +- *ctx->handling = X ++ ctx->vpi->handling = X + +/* + * not using idexpression for ctx behind this point + * because I failed to teach coccinelle VRT_CTX + * (which is not just an iso, but rather a macro + * also used in the argument list + */ +@@ +identifier F; +@@ + +- F(ctx->handling) ++ F(ctx->vpi) + +@@ +identifier F; +@@ + +- F(*ctx->handling) ++ F(ctx->vpi->handling) + +@@ +@@ + +- *ctx->handling ++ ctx->vpi->handling From nils.goroll at uplex.de Mon Aug 15 13:38:05 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Aug 2022 13:38:05 +0000 (UTC) Subject: [master] 503e231ba Result of coccinelle script from previous commit Message-ID: <20220815133805.F168A96C4D@lists.varnish-cache.org> commit 503e231baa6f1282256ef7ea133528c532725c5f Author: Nils Goroll Date: Mon Aug 15 15:24:27 2022 +0200 Result of coccinelle script from previous commit diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 11b7e5044..56fef17c6 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -408,10 +408,12 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) VCL_backend_fetch_method(bo->vcl, wrk, NULL, bo, NULL); - if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL) + if (wrk->vpi->handling == VCL_RET_ABANDON || + wrk->vpi->handling == VCL_RET_FAIL) return (F_STP_FAIL); - assert (wrk->handling == VCL_RET_FETCH || wrk->handling == VCL_RET_ERROR); + assert (wrk->vpi->handling == VCL_RET_FETCH || + wrk->vpi->handling == VCL_RET_ERROR); HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod); @@ -424,7 +426,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) bo->vfc->resp = bo->beresp; bo->vfc->req = bo->bereq; - if (wrk->handling == VCL_RET_ERROR) + if (wrk->vpi->handling == VCL_RET_ERROR) return (F_STP_ERROR); VSLb_ts_busyobj(bo, "Fetch", W_TIM_real(wrk)); @@ -501,24 +503,25 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) return (F_STP_ERROR); } - if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL || - wrk->handling == VCL_RET_ERROR) { + if (wrk->vpi->handling == VCL_RET_ABANDON || + wrk->vpi->handling == VCL_RET_FAIL || + wrk->vpi->handling == VCL_RET_ERROR) { /* do not count deliberately ending the backend connection as * fetch failure */ - handling = wrk->handling; + handling = wrk->vpi->handling; if (bo->htc) bo->htc->doclose = SC_RESP_CLOSE; vbf_cleanup(bo); - wrk->handling = handling; + wrk->vpi->handling = handling; - if (wrk->handling == VCL_RET_ERROR) + if (wrk->vpi->handling == VCL_RET_ERROR) return (F_STP_ERROR); else return (F_STP_FAIL); } - if (wrk->handling == VCL_RET_RETRY) { + if (wrk->vpi->handling == VCL_RET_RETRY) { if (bo->htc && bo->htc->body_status != BS_NONE) bo->htc->doclose = SC_RESP_CLOSE; vbf_cleanup(bo); @@ -541,15 +544,15 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) if (bo->do_esi) bo->do_stream = 0; - if (wrk->handling == VCL_RET_PASS) { + if (wrk->vpi->handling == VCL_RET_PASS) { oc->flags |= OC_F_HFP; bo->uncacheable = 1; - wrk->handling = VCL_RET_DELIVER; + wrk->vpi->handling = VCL_RET_DELIVER; } if (bo->uncacheable) oc->flags |= OC_F_HFM; - assert(wrk->handling == VCL_RET_DELIVER); + assert(wrk->vpi->handling == VCL_RET_DELIVER); return (bo->was_304 ? F_STP_CONDFETCH : F_STP_FETCH); } @@ -639,7 +642,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) oc = bo->fetch_objcore; CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - assert(wrk->handling == VCL_RET_DELIVER); + assert(wrk->vpi->handling == VCL_RET_DELIVER); if (bo->htc == NULL) { (void)VFP_Error(bo->vfc, "No backend connection (rollback?)"); @@ -888,7 +891,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) AN(oc->flags & OC_F_BUSY); assert(bo->director_state == DIR_S_NULL); - if (wrk->handling != VCL_RET_ERROR) + if (wrk->vpi->handling != VCL_RET_ERROR) wrk->stats->fetch_failed++; now = W_TIM_real(wrk); @@ -942,12 +945,12 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) AZ(VSB_finish(synth_body)); - if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL) { + if (wrk->vpi->handling == VCL_RET_ABANDON || wrk->vpi->handling == VCL_RET_FAIL) { VSB_destroy(&synth_body); return (F_STP_FAIL); } - if (wrk->handling == VCL_RET_RETRY) { + if (wrk->vpi->handling == VCL_RET_RETRY) { VSB_destroy(&synth_body); if (bo->retries++ < cache_param->max_retries) return (F_STP_RETRY); @@ -955,7 +958,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) return (F_STP_FAIL); } - assert(wrk->handling == VCL_RET_DELIVER); + assert(wrk->vpi->handling == VCL_RET_DELIVER); assert(bo->vfc->wrk == bo->wrk); assert(bo->vfc->oc == oc); diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index dd288c1f6..8b5e33a75 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -297,11 +297,11 @@ pan_wrk(struct vsb *vsb, const struct worker *wrk) else VSB_printf(vsb, "0x%x,\n", m); - hand = VCL_Return_Name(wrk->handling); + hand = VCL_Return_Name(wrk->vpi->handling); if (hand != NULL) VSB_printf(vsb, "VCL::return = %s,\n", hand); else - VSB_printf(vsb, "VCL::return = 0x%x,\n", wrk->handling); + VSB_printf(vsb, "VCL::return = 0x%x,\n", wrk->vpi->handling); VSB_cat(vsb, "VCL::methods = {"); m = wrk->seen_methods; p = ""; diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 48f5d5c7b..95c108789 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -239,12 +239,12 @@ cnt_deliver(struct worker *wrk, struct req *req) assert(req->restarts <= cache_param->max_restarts); - if (wrk->handling != VCL_RET_DELIVER) { + if (wrk->vpi->handling != VCL_RET_DELIVER) { HSH_Cancel(wrk, req->objcore, NULL); (void)HSH_DerefObjCore(wrk, &req->objcore, HSH_RUSH_POLICY); http_Teardown(req->resp); - switch (wrk->handling) { + switch (wrk->vpi->handling) { case VCL_RET_RESTART: req->req_step = R_STP_RESTART; break; @@ -261,7 +261,7 @@ cnt_deliver(struct worker *wrk, struct req *req) return (REQ_FSM_MORE); } - assert(wrk->handling == VCL_RET_DELIVER); + assert(wrk->vpi->handling == VCL_RET_DELIVER); if (IS_TOPREQ(req) && RFC2616_Do_Cond(req)) http_PutResponse(req->resp, "HTTP/1.1", 304, NULL); @@ -334,7 +334,7 @@ cnt_synth(struct worker *wrk, struct req *req) VSLb_ts_req(req, "Process", W_TIM_real(wrk)); - if (wrk->handling == VCL_RET_FAIL) { + if (wrk->vpi->handling == VCL_RET_FAIL) { VSB_destroy(&synth_body); (void)VRB_Ignore(req); (void)req->transport->minimal_response(req, 500); @@ -344,11 +344,11 @@ cnt_synth(struct worker *wrk, struct req *req) return (REQ_FSM_DONE); } - if (wrk->handling == VCL_RET_RESTART && + if (wrk->vpi->handling == VCL_RET_RESTART && req->restarts > cache_param->max_restarts) - wrk->handling = VCL_RET_DELIVER; + wrk->vpi->handling = VCL_RET_DELIVER; - if (wrk->handling == VCL_RET_RESTART) { + if (wrk->vpi->handling == VCL_RET_RESTART) { /* * XXX: Should we reset req->doclose = SC_VCL_FAILURE * XXX: If so, to what ? @@ -358,7 +358,7 @@ cnt_synth(struct worker *wrk, struct req *req) req->req_step = R_STP_RESTART; return (REQ_FSM_MORE); } - assert(wrk->handling == VCL_RET_DELIVER); + assert(wrk->vpi->handling == VCL_RET_DELIVER); http_Unset(req->resp, H_Content_Length); http_PrintfHeader(req->resp, "Content-Length: %zd", @@ -618,7 +618,7 @@ cnt_lookup(struct worker *wrk, struct req *req) VCL_hit_method(req->vcl, wrk, req, NULL, NULL); - switch (wrk->handling) { + switch (wrk->vpi->handling) { case VCL_RET_DELIVER: if (busy != NULL) { AZ(oc->flags & OC_F_HFM); @@ -680,7 +680,7 @@ cnt_miss(struct worker *wrk, struct req *req) CHECK_OBJ_ORNULL(req->stale_oc, OBJCORE_MAGIC); VCL_miss_method(req->vcl, wrk, req, NULL, NULL); - switch (wrk->handling) { + switch (wrk->vpi->handling) { case VCL_RET_FETCH: wrk->stats->cache_miss++; VBF_Fetch(wrk, req, req->objcore, req->stale_oc, VBF_NORMAL); @@ -725,7 +725,7 @@ cnt_pass(struct worker *wrk, struct req *req) AZ(req->stale_oc); VCL_pass_method(req->vcl, wrk, req, NULL, NULL); - switch (wrk->handling) { + switch (wrk->vpi->handling) { case VCL_RET_FAIL: req->req_step = R_STP_VCLFAIL; break; @@ -786,11 +786,11 @@ cnt_pipe(struct worker *wrk, struct req *req) bo->wrk = wrk; if (WS_Overflowed(req->ws)) - wrk->handling = VCL_RET_FAIL; + wrk->vpi->handling = VCL_RET_FAIL; else VCL_pipe_method(req->vcl, wrk, req, bo, NULL); - switch (wrk->handling) { + switch (wrk->vpi->handling) { case VCL_RET_SYNTH: req->req_step = R_STP_SYNTH; nxt = REQ_FSM_MORE; @@ -954,12 +954,12 @@ cnt_recv(struct worker *wrk, struct req *req) VCL_recv_method(req->vcl, wrk, req, NULL, NULL); - if (wrk->handling == VCL_RET_FAIL) { + if (wrk->vpi->handling == VCL_RET_FAIL) { req->req_step = R_STP_VCLFAIL; return (REQ_FSM_MORE); } - if (wrk->handling == VCL_RET_VCL && req->restarts == 0) { + if (wrk->vpi->handling == VCL_RET_VCL && req->restarts == 0) { // Req_Rollback has happened in VPI_vcl_select assert(WS_Snapshot(req->ws) == req->ws_req); cnt_recv_prep(req, ci); @@ -980,7 +980,7 @@ cnt_recv(struct worker *wrk, struct req *req) return (REQ_FSM_DONE); } - recv_handling = wrk->handling; + recv_handling = wrk->vpi->handling; /* We wash the A-E header here for the sake of VRY */ if (cache_param->http_gzip_support && @@ -995,10 +995,10 @@ cnt_recv(struct worker *wrk, struct req *req) VSHA256_Init(&sha256ctx); VCL_hash_method(req->vcl, wrk, req, NULL, &sha256ctx); - if (wrk->handling == VCL_RET_FAIL) - recv_handling = wrk->handling; + if (wrk->vpi->handling == VCL_RET_FAIL) + recv_handling = wrk->vpi->handling; else - assert(wrk->handling == VCL_RET_LOOKUP); + assert(wrk->vpi->handling == VCL_RET_LOOKUP); VSHA256_Final(req->digest, &sha256ctx); switch (recv_handling) { @@ -1083,7 +1083,7 @@ cnt_purge(struct worker *wrk, struct req *req) AZ(HSH_DerefObjCore(wrk, &boc, 1)); VCL_purge_method(req->vcl, wrk, req, NULL, NULL); - switch (wrk->handling) { + switch (wrk->vpi->handling) { case VCL_RET_RESTART: req->req_step = R_STP_RESTART; break; diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 97490920d..8251e149a 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -91,8 +91,8 @@ VCL_Bo2Ctx(struct vrt_ctx *ctx, struct busyobj *bo) ctx->sp = bo->sp; ctx->now = bo->t_prev; ctx->ws = bo->ws; - ctx->handling = &bo->wrk->handling; - *ctx->handling = 0; + ctx->vpi = bo->wrk->vpi; + ctx->vpi->handling = 0; } void @@ -114,8 +114,8 @@ VCL_Req2Ctx(struct vrt_ctx *ctx, struct req *req) ctx->sp = req->sp; ctx->now = req->t_prev; ctx->ws = req->ws; - ctx->handling = &req->wrk->handling; - *ctx->handling = 0; + ctx->vpi = req->wrk->vpi; + ctx->vpi->handling = 0; } /*--------------------------------------------------------------------*/ @@ -186,10 +186,10 @@ vcl_event_handling(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - if (*ctx->handling == 0) + if (ctx->vpi->handling == 0) return (0); - assert(*ctx->handling == VCL_RET_FAIL); + assert(ctx->vpi->handling == VCL_RET_FAIL); if (ctx->method == VCL_MET_INIT) return (1); @@ -200,7 +200,7 @@ vcl_event_handling(VRT_CTX) */ assert(ctx->method == VCL_MET_FINI); - *ctx->handling = 0; + ctx->vpi->handling = 0; VRT_fail(ctx, "VRT_fail() from vcl_fini{} has no effect"); return (0); } @@ -239,8 +239,8 @@ vcl_send_event(struct vcl *vcl, enum vcl_event_e ev, struct vsb **msg) ctx = VCL_Get_CliCtx(havemsg); - AN(ctx->handling); - AZ(*ctx->handling); + AN(ctx->vpi); + AZ(ctx->vpi->handling); AN(ctx->ws); ctx->vcl = vcl; diff --git a/bin/varnishd/cache/cache_vpi.c b/bin/varnishd/cache/cache_vpi.c index d2b149701..302523c39 100644 --- a/bin/varnishd/cache/cache_vpi.c +++ b/bin/varnishd/cache/cache_vpi.c @@ -89,12 +89,12 @@ void VPI_vcl_fini(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - AN(ctx->handling); + AN(ctx->vpi); - if (*ctx->handling == VCL_RET_FAIL) + if (ctx->vpi->handling == VCL_RET_FAIL) return; - assert(*ctx->handling == VCL_RET_OK); - *ctx->handling = 0; + assert(ctx->vpi->handling == VCL_RET_OK); + ctx->vpi->handling = 0; } VCL_VCL diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index dff0692af..f6c427ad1 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -643,19 +643,19 @@ VRT_handling(VRT_CTX, unsigned hand) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); assert(hand != VCL_RET_FAIL); - AN(ctx->handling); - AZ(*ctx->handling); + AN(ctx->vpi); + AZ(ctx->vpi->handling); assert(hand > 0); assert(hand < VCL_RET_MAX); - *ctx->handling = hand; + ctx->vpi->handling = hand; } unsigned VRT_handled(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - AN(ctx->handling); - return (*ctx->handling); + AN(ctx->vpi); + return (ctx->vpi->handling); } /*--------------------------------------------------------------------*/ @@ -667,10 +667,10 @@ VRT_fail(VRT_CTX, const char *fmt, ...) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); assert(ctx->vsl != NULL || ctx->msg != NULL); - AN(ctx->handling); - if (*ctx->handling == VCL_RET_FAIL) + AN(ctx->vpi); + if (ctx->vpi->handling == VCL_RET_FAIL) return; - AZ(*ctx->handling); + AZ(ctx->vpi->handling); AN(fmt); AZ(strchr(fmt, '\n')); va_start(ap, fmt); @@ -682,7 +682,7 @@ VRT_fail(VRT_CTX, const char *fmt, ...) VSB_putc(ctx->msg, '\n'); } va_end(ap); - *ctx->handling = VCL_RET_FAIL; + ctx->vpi->handling = VCL_RET_FAIL; } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index 685bb451c..fd4e84fbc 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -280,7 +280,7 @@ VRT_priv_fini(VRT_CTX, const struct vmod_priv *p) VRT_CTX_Assert(ctx); m->fini(ctx, p->priv); - assert(*ctx->handling == 0 || *ctx->handling == VCL_RET_FAIL); + assert(ctx->vpi->handling == 0 || ctx->vpi->handling == VCL_RET_FAIL); } /*--------------------------------------------------------------------*/ @@ -298,8 +298,8 @@ VCL_TaskLeave(VRT_CTX, struct vrt_privs *privs) struct vrt_priv *vp, *vp1; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - AN(ctx->handling); - assert(*ctx->handling == 0 || *ctx->handling == VCL_RET_FAIL); + AN(ctx->vpi); + assert(ctx->vpi->handling == 0 || ctx->vpi->handling == VCL_RET_FAIL); /* * NB: We don't bother removing entries as we finish them because it's diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c index 976930e4e..56ad0340e 100644 --- a/bin/varnishd/cache/cache_vrt_vcl.c +++ b/bin/varnishd/cache/cache_vrt_vcl.c @@ -558,9 +558,9 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo, VSLbs(ctx.vsl, SLT_VCL_call, TOSTRAND(VCL_Method_Name(method))); func(&ctx, VSUB_STATIC, NULL); VSLbs(ctx.vsl, SLT_VCL_return, - TOSTRAND(VCL_Return_Name(wrk->handling))); + TOSTRAND(VCL_Return_Name(wrk->vpi->handling))); wrk->cur_method |= 1; // Magic marker - if (wrk->handling == VCL_RET_FAIL) + if (wrk->vpi->handling == VCL_RET_FAIL) wrk->stats->vcl_fail++; /* From nils.goroll at uplex.de Mon Aug 15 13:38:06 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Aug 2022 13:38:06 +0000 (UTC) Subject: [master] 21eb6a170 Save VPI_count() info to struct wrk_vpi Message-ID: <20220815133806.188A396C51@lists.varnish-cache.org> commit 21eb6a1700ea0c7a5726d27b14c2908aee714b62 Author: Nils Goroll Date: Tue Feb 9 12:51:49 2021 +0100 Save VPI_count() info to struct wrk_vpi So we have it available for Panics in a follow-up commit diff --git a/bin/varnishd/cache/cache_vpi.c b/bin/varnishd/cache/cache_vpi.c index 302523c39..e22cc6377 100644 --- a/bin/varnishd/cache/cache_vpi.c +++ b/bin/varnishd/cache/cache_vpi.c @@ -67,6 +67,7 @@ VPI_count(VRT_CTX, unsigned u) CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); CHECK_OBJ_NOTNULL(ctx->vcl->conf, VCL_CONF_MAGIC); assert(u < ctx->vcl->conf->nref); + ctx->vpi->ref = u; if (ctx->vsl != NULL) VSLb(ctx->vsl, SLT_VCL_trace, "%s %u %u.%u.%u", ctx->vcl->loaded_name, u, ctx->vcl->conf->ref[u].source, diff --git a/include/vcc_interface.h b/include/vcc_interface.h index 4a51868cd..394f308a0 100644 --- a/include/vcc_interface.h +++ b/include/vcc_interface.h @@ -58,6 +58,7 @@ struct wrk_vpi { unsigned magic; #define WRK_VPI_MAGIC 0xaa3d3df3 unsigned handling; + unsigned ref; // index into (struct vpi_ref)[] }; From nils.goroll at uplex.de Mon Aug 15 13:38:06 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Aug 2022 13:38:06 +0000 (UTC) Subject: [master] 1ad310205 Give struct vpi_ref a magic Message-ID: <20220815133806.368F796C5D@lists.varnish-cache.org> commit 1ad3102055d9df206d8ee0da611950b2d3c97fa3 Author: Nils Goroll Date: Fri May 7 17:48:52 2021 +0200 Give struct vpi_ref a magic to be able to use our standard miniobj facilities on it, PAN_dump_struct in this case. As vpi_ref is static const in VGC, there is no real reason for magic checks other than that. diff --git a/include/vcc_interface.h b/include/vcc_interface.h index 394f308a0..83ef7f096 100644 --- a/include/vcc_interface.h +++ b/include/vcc_interface.h @@ -46,6 +46,8 @@ void v_noreturn_ VPI_Fail(const char *func, const char *file, int line, */ struct vpi_ref { + unsigned magic; +#define VPI_REF_MAGIC 0xd955f567 unsigned source; unsigned offset; unsigned line; diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 442f4f9ca..f3f9a2d70 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -410,7 +410,7 @@ EmitCoordinates(const struct vcc *tl, struct vsb *vsb) pos++; } - VSB_printf(vsb, " [%3u] = { %u, %8tu, %4u, %3u, ", + VSB_printf(vsb, " [%3u] = { VPI_REF_MAGIC, %u, %8tu, %4u, %3u, ", t->cnt, sp->idx, t->b - sp->b, lin, pos + 1); if (t->tok == CSRC) VSB_cat(vsb, " \"C{\"},\n"); From nils.goroll at uplex.de Mon Aug 15 13:38:06 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Aug 2022 13:38:06 +0000 (UTC) Subject: [master] 4f2852623 Add VCL source information to panics Message-ID: <20220815133806.56C9E96C74@lists.varnish-cache.org> commit 4f2852623d62b1d7a5f712cb2b8695fb22f6e93f Author: Nils Goroll Date: Tue Feb 9 13:51:03 2021 +0100 Add VCL source information to panics Now that we have the vpi ref as set by VPI_Count() available via struct wrk_vpi, we can dump our position in VCL at the time of the panic. Additional changes: * move handling info from pan_wrk() to the new VPI_Panic() * output the source index of vcl sources with the name in VCL_Panic() The additional panic section looks like this: vpi = 0x7f5478be93b0 { handling (VCL::return) = 0x0 (none), ref = 6, vpi_ref = 0x7f54753fb530 { source = 0 (\"\"), offset = 365, line = 20, pos = 25, src = \"vtc.panic(\\\"Had Panic header: \\\" + resp.ht[...]\" }, }, The source line is truncated (and truncation denoted by [...]) if longer than 40 characters. If the in-memory vcl data source string looks truncated, tok = "..." may be output instead of src "...". diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 8b5e33a75..779079f8d 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -297,11 +297,6 @@ pan_wrk(struct vsb *vsb, const struct worker *wrk) else VSB_printf(vsb, "0x%x,\n", m); - hand = VCL_Return_Name(wrk->vpi->handling); - if (hand != NULL) - VSB_printf(vsb, "VCL::return = %s,\n", hand); - else - VSB_printf(vsb, "VCL::return = 0x%x,\n", wrk->vpi->handling); VSB_cat(vsb, "VCL::methods = {"); m = wrk->seen_methods; p = ""; @@ -355,6 +350,7 @@ static void pan_busyobj(struct vsb *vsb, const struct busyobj *bo) { const char *p; + const struct worker *wrk; if (PAN_dump_struct(vsb, bo, BUSYOBJ_MAGIC, "busyobj")) return; @@ -365,8 +361,9 @@ pan_busyobj(struct vsb *vsb, const struct busyobj *bo) pan_req(vsb, bo->req); if (bo->sp != NULL) pan_sess(vsb, bo->sp); - if (bo->wrk != NULL) - pan_wrk(vsb, bo->wrk); + wrk = bo->wrk; + if (wrk != NULL) + pan_wrk(vsb, wrk); if (bo->vfc != NULL) pan_vfp(vsb, bo->vfc); @@ -408,6 +405,9 @@ pan_busyobj(struct vsb *vsb, const struct busyobj *bo) else VDI_Panic(bo->director_resp, vsb, "director_resp"); VCL_Panic(vsb, "vcl", bo->vcl); + if (wrk != NULL) + VPI_Panic(vsb, wrk->vpi, bo->vcl); + VSB_indent(vsb, -2); VSB_cat(vsb, "},\n"); } @@ -432,6 +432,7 @@ static void pan_req(struct vsb *vsb, const struct req *req) { const struct transport *xp; + const struct worker *wrk; if (PAN_dump_struct(vsb, req, REQ_MAGIC, "req")) return; @@ -466,8 +467,9 @@ pan_req(struct vsb *vsb, const struct req *req) if (req->sp != NULL) pan_sess(vsb, req->sp); - if (req->wrk != NULL) - pan_wrk(vsb, req->wrk); + wrk = req->wrk; + if (wrk != NULL) + pan_wrk(vsb, wrk); WS_Panic(vsb, req->ws); if (VALID_OBJ(req->htc, HTTP_CONN_MAGIC)) @@ -479,6 +481,8 @@ pan_req(struct vsb *vsb, const struct req *req) VDP_Panic(vsb, req->vdc); VCL_Panic(vsb, "vcl", req->vcl); + if (wrk != NULL) + VPI_Panic(vsb, wrk->vpi, req->vcl); if (req->body_oc != NULL) pan_objcore(vsb, "BODY", req->body_oc); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index eb3981da0..6d5adea5d 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -527,6 +527,7 @@ void WRK_Log(enum VSL_tag_e, const char *, ...); /* cache_vpi.c */ extern const size_t vpi_wrk_len; void VPI_wrk_init(struct worker *, void *, size_t); +void VPI_Panic(struct vsb *, const struct wrk_vpi *, const struct vcl *); /* cache_ws.c */ void WS_Panic(struct vsb *, const struct ws *); diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 8251e149a..d7a58a5d9 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -291,7 +291,7 @@ vcl_panic_conf(struct vsb *vsb, const struct VCL_conf *conf) VSB_cat(vsb, "srcname = {\n"); VSB_indent(vsb, 2); for (i = 0; i < conf->nsrc; ++i) - VSB_printf(vsb, "\"%s\",\n", conf->srcname[i]); + VSB_printf(vsb, "[%d] = \"%s\",\n", i, conf->srcname[i]); VSB_indent(vsb, -2); VSB_cat(vsb, "},\n"); VSB_cat(vsb, "instances = {\n"); diff --git a/bin/varnishd/cache/cache_vpi.c b/bin/varnishd/cache/cache_vpi.c index e22cc6377..375b4ea4d 100644 --- a/bin/varnishd/cache/cache_vpi.c +++ b/bin/varnishd/cache/cache_vpi.c @@ -32,6 +32,8 @@ #include "config.h" +#include + #include "cache_varnishd.h" #include "vcl.h" @@ -78,6 +80,105 @@ VPI_count(VRT_CTX, unsigned u) ctx->vcl->conf->ref[u].line, ctx->vcl->conf->ref[u].pos); } +static void +vpi_ref_panic(struct vsb *vsb, unsigned n, const struct vcl *vcl) +{ + const struct VCL_conf *conf = NULL; + const struct vpi_ref *ref; + const char *p, *src = NULL; + const int lim = 40; + const char *abbstr = "[...]"; + char buf[lim + sizeof(abbstr)]; + int w = 0; + + AN(vsb); + + if (vcl != NULL) + conf = vcl->conf; + if (conf != NULL && conf->magic != VCL_CONF_MAGIC) + conf = NULL; + + if (conf == NULL) { + VSB_printf(vsb, "ref = %u, nref = ?,\n", n); + return; + } + if (n >= conf->nref) { + VSB_printf(vsb, "ref = %u *invalid*, nref = %u\n", + n, conf->nref); + return; + } + + VSB_printf(vsb, "ref = %u,\n", n); + + ref = &conf->ref[n]; + if (PAN_dump_struct(vsb, ref, VPI_REF_MAGIC, "vpi_ref")) + return; + + if (ref->source < conf->nsrc) { + VSB_printf(vsb, "source = %u (\"%s\"),\n", ref->source, + conf->srcname[ref->source]); + src = conf->srcbody[ref->source]; + } else { + VSB_printf(vsb, "source = %u *invalid*,\n", ref->source); + } + + if (src != NULL) { + w = strlen(src); + if (ref->offset >= w) + src = NULL; + } + if (src != NULL) { + src += ref->offset; + p = strchr(src, '\n'); + if (p != NULL) + w = p - src; + else + w -= ref->offset; + if (w > lim) { + w = snprintf(buf, sizeof buf, "%.*s%s", + lim, src, abbstr); + src = buf; + } + } + + VSB_printf(vsb, "offset = %u,\n", ref->offset); + VSB_printf(vsb, "line = %u,\n", ref->line); + VSB_printf(vsb, "pos = %u,\n", ref->pos); + if (src != NULL) { + VSB_cat(vsb, "src = "); + VSB_quote(vsb, src, w, VSB_QUOTE_CSTR); + VSB_putc(vsb, '\n'); + } else { + VSB_printf(vsb, "token = \"%s\"\n", ref->token); + } + VSB_indent(vsb, -2); + VSB_cat(vsb, "},\n"); + +} +void +VPI_Panic(struct vsb *vsb, const struct wrk_vpi *vpi, const struct vcl *vcl) +{ + const char *hand; + + AN(vsb); + if (PAN_dump_struct(vsb, vpi, WRK_VPI_MAGIC, "vpi")) + return; + + hand = VCL_Return_Name(vpi->handling); + if (vpi->handling == 0) + hand = "none"; + else if (hand == NULL) + hand = "*invalid*"; + + VSB_printf(vsb, "handling (VCL::return) = 0x%x (%s),\n", + vpi->handling, hand); + + vpi_ref_panic(vsb, vpi->ref, vcl); + + VSB_indent(vsb, -2); + VSB_cat(vsb, "},\n"); +} + /* * After vcl_fini {} == VGC_function_vcl_fini() is called from VGC_Discard(), * handling must either be OK from VCL "return (ok)" or FAIL from VRT_fail(). From nils.goroll at uplex.de Mon Aug 15 18:30:07 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Aug 2022 18:30:07 +0000 (UTC) Subject: [master] 373caaf28 Flexelinting Message-ID: <20220815183007.97E9FA7BA5@lists.varnish-cache.org> commit 373caaf28bdd266bd5eb643b2ca2a6ed9a609708 Author: Nils Goroll Date: Mon Aug 15 20:22:09 2022 +0200 Flexelinting After #3515 merge I do not understand Warning 552: Symbol 'vpi_wrk_len' (line 50, file cache/cache_vpi.c) not accessed It is used for array sizing, so I have silenced the warning. diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 779079f8d..b93fdb7b1 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -43,7 +43,6 @@ #include "cache_varnishd.h" #include "cache_transport.h" -#include "vcc_interface.h" #include "cache_filter.h" #include "common/heritage.h" diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index d7a58a5d9..0f3245c8c 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -282,7 +282,7 @@ vcl_find(const char *name) static void vcl_panic_conf(struct vsb *vsb, const struct VCL_conf *conf) { - int i; + unsigned u; const struct vpi_ii *ii; if (PAN_dump_struct(vsb, conf, VCL_CONF_MAGIC, "conf")) @@ -290,8 +290,8 @@ vcl_panic_conf(struct vsb *vsb, const struct VCL_conf *conf) VSB_printf(vsb, "syntax = \"%u\",\n", conf->syntax); VSB_cat(vsb, "srcname = {\n"); VSB_indent(vsb, 2); - for (i = 0; i < conf->nsrc; ++i) - VSB_printf(vsb, "[%d] = \"%s\",\n", i, conf->srcname[i]); + for (u = 0; u < conf->nsrc; ++u) + VSB_printf(vsb, "[%u] = \"%s\",\n", u, conf->srcname[u]); VSB_indent(vsb, -2); VSB_cat(vsb, "},\n"); VSB_cat(vsb, "instances = {\n"); @@ -947,6 +947,7 @@ vcl_cli_show(struct cli *cli, const char * const *av, void *priv) struct vcl *vcl; int verbose = 0; int i = 2; + unsigned u; ASSERT_CLI(); ASSERT_VCL_ACTIVE(); @@ -984,11 +985,11 @@ vcl_cli_show(struct cli *cli, const char * const *av, void *priv) } CHECK_OBJ_NOTNULL(vcl->conf, VCL_CONF_MAGIC); if (verbose) { - for (i = 0; i < vcl->conf->nsrc; i++) - VCLI_Out(cli, "// VCL.SHOW %d %zd %s\n%s\n", - i, strlen(vcl->conf->srcbody[i]), - vcl->conf->srcname[i], - vcl->conf->srcbody[i]); + for (u = 0; u < vcl->conf->nsrc; u++) + VCLI_Out(cli, "// VCL.SHOW %u %zd %s\n%s\n", + u, strlen(vcl->conf->srcbody[u]), + vcl->conf->srcname[u], + vcl->conf->srcbody[u]); } else { VCLI_Out(cli, "%s", vcl->conf->srcbody[0]); } diff --git a/bin/varnishd/cache/cache_vpi.c b/bin/varnishd/cache/cache_vpi.c index 375b4ea4d..3b21037fb 100644 --- a/bin/varnishd/cache/cache_vpi.c +++ b/bin/varnishd/cache/cache_vpi.c @@ -124,7 +124,8 @@ vpi_ref_panic(struct vsb *vsb, unsigned n, const struct vcl *vcl) if (src != NULL) { w = strlen(src); - if (ref->offset >= w) + assert(w > 0); + if (ref->offset >= (unsigned)w) src = NULL; } if (src != NULL) { diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt index 65e77d09e..c40188132 100644 --- a/bin/varnishd/flint.lnt +++ b/bin/varnishd/flint.lnt @@ -164,6 +164,9 @@ -e441 // for clause irregularity: loop variable '___' not found in 2nd for expression +// cache_vpi.c +-esym(552, vpi_wrk_len) + // from libvarnish --emacro((835),VBH_NOIDX) --emacro((835),O_CLOEXEC) diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 94b0252fe..2199f6a07 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -689,7 +689,7 @@ struct VCL_conf { unsigned nref; const struct vpi_ref *ref; - int nsrc; + unsigned nsrc; unsigned nsub; const char **srcname; const char **srcbody; From dridi.boukelmoune at gmail.com Tue Aug 16 06:31:07 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 16 Aug 2022 06:31:07 +0000 (UTC) Subject: [master] 495586aa2 vsl: Optional Begin[4] field for req sub-level Message-ID: <20220816063107.F06C49860@lists.varnish-cache.org> commit 495586aa235f17ea1c68488b32adf7cfb29b0010 Author: Dridi Boukelmoune Date: Fri Jul 22 14:46:52 2022 +0200 vsl: Optional Begin[4] field for req sub-level Same thing for Link tags since they share the same parsing code. diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 245fcd87c..33827b916 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -136,14 +136,16 @@ ved_include(struct req *preq, const char *src, const char *host, AZ(req->vsl->wid); req->vsl->wid = VXID_Get(wrk, VSL_CLIENTMARKER); - VSLb(req->vsl, SLT_Begin, "req %u esi", VXID(preq->vsl->wid)); - VSLb(preq->vsl, SLT_Link, "req %u esi", VXID(req->vsl->wid)); - - VSLb_ts_req(req, "Start", W_TIM_real(wrk)); - wrk->stats->esi_req++; req->esi_level = preq->esi_level + 1; + VSLb(req->vsl, SLT_Begin, "req %u esi %u", VXID(preq->vsl->wid), + req->esi_level); + VSLb(preq->vsl, SLT_Link, "req %u esi %u", VXID(req->vsl->wid), + req->esi_level); + + VSLb_ts_req(req, "Start", W_TIM_real(wrk)); + memset(req->top, 0, sizeof *req->top); req->top = preq->top; diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 47414b990..9e28d92e0 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -506,22 +506,24 @@ SLTM(Gzip, 0, "G(un)zip performed on object", SLTM(Link, 0, "Links to a child VXID", "Links this VXID to any child VXID it initiates.\n\n" "The format is::\n\n" - "\t%s %d %s\n" - "\t| | |\n" - "\t| | +- Reason\n" - "\t| +---- Child vxid\n" - "\t+------- Child type (\"req\" or \"bereq\")\n" + "\t%s %d %s [%u]\n" + "\t| | | |\n" + "\t| | | +- Child task sub-level\n" + "\t| | +----- Reason\n" + "\t| +-------- Child vxid\n" + "\t+----------- Child type (\"sess\", \"req\" or \"bereq\")\n" "\n" ) SLTM(Begin, 0, "Marks the start of a VXID", "The first record of a VXID transaction.\n\n" "The format is::\n\n" - "\t%s %d %s\n" - "\t| | |\n" - "\t| | +- Reason\n" - "\t| +---- Parent vxid\n" - "\t+------- Type (\"sess\", \"req\" or \"bereq\")\n" + "\t%s %d %s [%u]\n" + "\t| | | |\n" + "\t| | | +- Task sub-level\n" + "\t| | +----- Reason\n" + "\t| +-------- Parent vxid\n" + "\t+----------- Type (\"sess\", \"req\" or \"bereq\")\n" "\n" ) From dridi.boukelmoune at gmail.com Tue Aug 16 06:31:08 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 16 Aug 2022 06:31:08 +0000 (UTC) Subject: [master] 07a785ebf vsl: Apply VSL_OPT_E to the Begin[4] field Message-ID: <20220816063108.17D959864@lists.varnish-cache.org> commit 07a785ebfae9b942c4ce88fd9ba74d1cfbca655c Author: Dridi Boukelmoune Date: Fri Jul 22 15:17:54 2022 +0200 vsl: Apply VSL_OPT_E to the Begin[4] field As I was relying on the lack of -E option to discard another type of non-ESI sub-requests I recently misled myself with this assumption. It took me a while to realize that I was also seeing transactions I was not interested in, corrupting the statistics I was deriving from a log dump. Now that vmod_pesi [1] sets the precedent, we should expect new types of sub-requests to materialize and expecting the reason for the transaction to be "esi" is no longer appropriate. We could also track this field as vtx::sub_level for example, to make it available as VSL query LHS operand. The focus here was to adjust the behavior of an existing feature, not add a new one. [1] https://code.uplex.de/uplex-varnish/libvdp-pesi diff --git a/bin/varnishtest/tests/l00007.vtc b/bin/varnishtest/tests/l00007.vtc new file mode 100644 index 000000000..0416dd9e6 --- /dev/null +++ b/bin/varnishtest/tests/l00007.vtc @@ -0,0 +1,125 @@ +varnishtest "Begin[4] and VSL_OPT_E" + +varnish v1 -vcl { + import vtc; + + backend be none; + + sub vcl_init { + # make up a replay with -i Begin,Link,End + vtc.vsl_replay({" +**** v1 vsl| 1000 Begin c sess 0 HTTP/1 +**** v1 vsl| 1000 Link c req 1001 rxreq +**** v1 vsl| 1003 Begin b bereq 1002 fetch +**** v1 vsl| 1003 End b +**** v1 vsl| 1002 Begin c req 1001 vmod_foo:subreq 1 +**** v1 vsl| 1002 End c +**** v1 vsl| 1001 Begin c req 1000 rxreq +**** v1 vsl| 1001 Link c req 1002 vmod_foo:subreq 1 +**** v1 vsl| 1001 End c +**** v1 vsl| 1000 End c +**** v1 vsl| 1004 Begin c sess 0 HTTP/1 +**** v1 vsl| 1004 End c + "}); + } +} -start + +varnish v1 -vsl_catchup + +logexpect l1 -v v1 -d 1 { + expect 0 1003 Begin "bereq 1002 fetch" + expect 0 = End + expect 0 1001 Begin "req 1000 rxreq" + expect 0 = Link "req 1002 vmod_foo:subreq" + expect 0 = End + expect 0 1000 Begin "sess 0 HTTP/1" + expect 0 = Link "req 1001 rxreq" + expect 0 = End + expect 0 1004 Begin "sess 0 HTTP/1" + expect 0 = End +} -run + +logexpect l1_bis -v v1 -d 1 -b 1 -c 1 { + expect 0 1003 Begin "bereq 1002 fetch" + expect 0 = End + expect 0 1001 Begin "req 1000 rxreq" + expect 0 = Link "req 1002 vmod_foo:subreq" + expect 0 = End + expect 0 1000 Begin "sess 0 HTTP/1" + expect 0 = Link "req 1001 rxreq" + expect 0 = End + expect 0 1004 Begin "sess 0 HTTP/1" + expect 0 = End +} -run + +logexpect l2 -v v1 -d 1 -b 1 { + expect 0 1003 Begin "bereq 1002 fetch" + expect 0 = End +} -run + +logexpect l3 -v v1 -d 1 -c 1 { + expect 0 1001 Begin "req 1000 rxreq" + expect 0 = Link "req 1002 vmod_foo:subreq" + expect 0 = End + expect 0 1000 Begin "sess 0 HTTP/1" + expect 0 = Link "req 1001 rxreq" + expect 0 = End + expect 0 1004 Begin "sess 0 HTTP/1" + expect 0 = End +} -run + +logexpect l4 -v v1 -d 1 -E 1 { + expect 0 1002 Begin "req 1001 vmod_foo:subreq" + expect 0 = End + expect 0 1001 Begin "req 1000 rxreq" + expect 0 = Link "req 1002 vmod_foo:subreq" + expect 0 = End + expect 0 1000 Begin "sess 0 HTTP/1" + expect 0 = Link "req 1001 rxreq" + expect 0 = End + expect 0 1004 Begin "sess 0 HTTP/1" + expect 0 = End +} -run + +logexpect l4_bis -v v1 -d 1 -c 1 -E 1 { + expect 0 1002 Begin "req 1001 vmod_foo:subreq" + expect 0 = End + expect 0 1001 Begin "req 1000 rxreq" + expect 0 = Link "req 1002 vmod_foo:subreq" + expect 0 = End + expect 0 1000 Begin "sess 0 HTTP/1" + expect 0 = Link "req 1001 rxreq" + expect 0 = End + expect 0 1004 Begin "sess 0 HTTP/1" + expect 0 = End +} -run + +logexpect l5 -v v1 -d 1 -b 1 -E 1 { + expect 0 1003 Begin "bereq 1002 fetch" + expect 0 = End + expect 0 1002 Begin "req 1001 vmod_foo:subreq" + expect 0 = End + expect 0 1001 Begin "req 1000 rxreq" + expect 0 = Link "req 1002 vmod_foo:subreq" + expect 0 = End + expect 0 1000 Begin "sess 0 HTTP/1" + expect 0 = Link "req 1001 rxreq" + expect 0 = End + expect 0 1004 Begin "sess 0 HTTP/1" + expect 0 = End +} -run + +logexpect l5_bis -v v1 -d 1 -b 1 -c 1 -E 1 { + expect 0 1003 Begin "bereq 1002 fetch" + expect 0 = End + expect 0 1002 Begin "req 1001 vmod_foo:subreq" + expect 0 = End + expect 0 1001 Begin "req 1000 rxreq" + expect 0 = Link "req 1002 vmod_foo:subreq" + expect 0 = End + expect 0 1000 Begin "sess 0 HTTP/1" + expect 0 = Link "req 1001 rxreq" + expect 0 = End + expect 0 1004 Begin "sess 0 HTTP/1" + expect 0 = End +} -run diff --git a/include/vapi/vapi_options.h b/include/vapi/vapi_options.h index 498ed08ed..3cff3eb47 100644 --- a/include/vapi/vapi_options.h +++ b/include/vapi/vapi_options.h @@ -54,7 +54,9 @@ #define VSL_OPT_E \ VOPT("E", "[-E]", "Display ESI transactions", \ - "Display ESI transactions and other client transactions." \ + "Display ESI transactions and other types of sub-requests." \ + " This implies the -c option and includes other client" \ + " transactions." \ ) #define VSL_OPT_i \ diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index e5f2ddaf1..0a7b2cbf1 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -190,10 +190,12 @@ VSL_Match(struct VSL_data *vsl, const struct VSL_cursor *c) tag = VSL_TAG(c->rec.ptr); if (tag <= SLT__Bogus || tag >= SLT__Reserved) return (0); - if (vsl->c_opt && !VSL_CLIENT(c->rec.ptr)) - return (0); - if (vsl->b_opt && !VSL_BACKEND(c->rec.ptr)) - return (0); + if (!vsl->c_opt || !vsl->b_opt) { + if (vsl->c_opt && !VSL_CLIENT(c->rec.ptr)) + return (0); + if (vsl->b_opt && !VSL_BACKEND(c->rec.ptr)) + return (0); + } if (!VTAILQ_EMPTY(&vsl->vslf_select) && vsl_match_IX(vsl, &vsl->vslf_select, c)) return (1); diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c index db3c30ee1..8c24a7521 100644 --- a/lib/libvarnishapi/vsl_dispatch.c +++ b/lib/libvarnishapi/vsl_dispatch.c @@ -689,10 +689,10 @@ vtx_set_parent(struct vtx *parent, struct vtx *child) successfully parsed. */ static int vtx_parse_link(const char *str, enum VSL_transaction_e *ptype, - unsigned *pvxid, enum VSL_reason_e *preason) + unsigned *pvxid, enum VSL_reason_e *preason, unsigned *psub) { char type[16], reason[16]; - unsigned vxid; + unsigned vxid, sub; int i; enum VSL_transaction_e et; enum VSL_reason_e er; @@ -702,7 +702,7 @@ vtx_parse_link(const char *str, enum VSL_transaction_e *ptype, AN(pvxid); AN(preason); - i = sscanf(str, "%15s %u %15s", type, &vxid, reason); + i = sscanf(str, "%15s %u %15s %u", type, &vxid, reason, &sub); if (i < 1) return (0); @@ -729,7 +729,13 @@ vtx_parse_link(const char *str, enum VSL_transaction_e *ptype, if (er >= VSL_r__MAX) er = VSL_r_unknown; *preason = er; - return (3); + if (i == 3) + return (3); + + /* request sub-level */ + if (psub != NULL) + *psub = sub; + return (4); } /* Parse and process a begin record */ @@ -746,8 +752,8 @@ vtx_scan_begin(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr) AZ(vtx->flags & VTX_F_READY); - i = vtx_parse_link(VSL_CDATA(ptr), &type, &p_vxid, &reason); - if (i != 3) + i = vtx_parse_link(VSL_CDATA(ptr), &type, &p_vxid, &reason, NULL); + if (i < 3) return (vtx_diag_tag(vtx, ptr, "parse error")); if (type == VSL_t_unknown) (void)vtx_diag_tag(vtx, ptr, "unknown vxid type"); @@ -813,8 +819,8 @@ vtx_scan_link(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr) AZ(vtx->flags & VTX_F_READY); - i = vtx_parse_link(VSL_CDATA(ptr), &c_type, &c_vxid, &c_reason); - if (i != 3) + i = vtx_parse_link(VSL_CDATA(ptr), &c_type, &c_vxid, &c_reason, NULL); + if (i < 3) return (vtx_diag_tag(vtx, ptr, "parse error")); if (c_type == VSL_t_unknown) (void)vtx_diag_tag(vtx, ptr, "unknown vxid type"); @@ -1288,7 +1294,7 @@ vslq_candidate(struct VSLQ *vslq, const uint32_t *ptr) enum VSL_reason_e reason; struct VSL_data *vsl; enum VSL_tag_e tag; - unsigned p_vxid; + unsigned p_vxid, sub; int i; CHECK_OBJ_NOTNULL(vslq, VSLQ_MAGIC); @@ -1302,24 +1308,24 @@ vslq_candidate(struct VSLQ *vslq, const uint32_t *ptr) CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC); if (vslq->grouping == VSL_g_vxid) { if (!vsl->c_opt && !vsl->b_opt) - return (1); /* Implies also !vsl->E_opt */ - if (!vsl->b_opt && !VSL_CLIENT(ptr)) + AZ(vsl->E_opt); + else if (!vsl->b_opt && !VSL_CLIENT(ptr)) return (0); - if (!vsl->c_opt && !VSL_BACKEND(ptr)) + else if (!vsl->c_opt && !VSL_BACKEND(ptr)) return (0); /* Need to parse the Begin tag - fallthrough to below */ } tag = VSL_TAG(ptr); assert(tag == SLT_Begin); - i = vtx_parse_link(VSL_CDATA(ptr), &type, &p_vxid, &reason); - if (i != 3 || type == VSL_t_unknown) + i = vtx_parse_link(VSL_CDATA(ptr), &type, &p_vxid, &reason, &sub); + if (i < 3 || type == VSL_t_unknown) return (0); - if (type == VSL_t_sess) + if (vslq->grouping == VSL_g_request && type == VSL_t_sess) return (0); - if (vslq->grouping == VSL_g_vxid && reason == VSL_r_esi && !vsl->E_opt) + if (vslq->grouping == VSL_g_vxid && i > 3 && sub > 0 && !vsl->E_opt) return (0); return (1); From dridi.boukelmoune at gmail.com Tue Aug 16 06:31:08 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 16 Aug 2022 06:31:08 +0000 (UTC) Subject: [master] bcdbc5b0a vtc: Rename test vtc_b0 to vtc_l0 Message-ID: <20220816063108.3827A9869@lists.varnish-cache.org> commit bcdbc5b0a8b2b4fa4d1326f151fce74d74603a77 Author: Dridi Boukelmoune Date: Tue Aug 9 10:55:58 2022 +0200 vtc: Rename test vtc_b0 to vtc_l0 diff --git a/vmod/tests/vtc_b00000.vtc b/vmod/tests/vtc_l00000.vtc similarity index 100% rename from vmod/tests/vtc_b00000.vtc rename to vmod/tests/vtc_l00000.vtc From dridi.boukelmoune at gmail.com Tue Aug 16 06:31:08 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 16 Aug 2022 06:31:08 +0000 (UTC) Subject: [master] 727cdcbb3 doc: Mention logexpect -run Message-ID: <20220816063108.54948986F@lists.varnish-cache.org> commit 727cdcbb3417d51c8cb531e54b387510bb8e19b5 Author: Dridi Boukelmoune Date: Tue Aug 9 10:57:51 2022 +0200 doc: Mention logexpect -run diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index c542619b1..07024e23d 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -47,7 +47,7 @@ * fail clear * abort * ... - * } [-start|-wait] + * } [-start|-wait|-run] * * And once declared, you can start them, or wait on them:: * @@ -86,6 +86,9 @@ * \-wait * Wait for the logexpect thread to finish * + * \-run + * Equivalent to "-start -wait". + * * VSL arguments (similar to the varnishlog options): * * \-C From phk at FreeBSD.org Tue Aug 23 10:32:08 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Aug 2022 10:32:08 +0000 (UTC) Subject: [master] f10f433d7 Minor issues found by Coverity Message-ID: <20220823103208.C817811E437@lists.varnish-cache.org> commit f10f433d77297cc884056db19235bf3cc6de618b Author: Poul-Henning Kamp Date: Tue Aug 23 10:30:59 2022 +0000 Minor issues found by Coverity diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 56fef17c6..40b304fa7 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -444,7 +444,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) http_VSL_log(bo->beresp); - if (bo->htc->body_status == BS_ERROR) { + if (bo->htc != NULL && bo->htc->body_status == BS_ERROR) { bo->htc->doclose = SC_RX_BODY; vbf_cleanup(bo); VSLb(bo->vsl, SLT_Error, "Body cannot be fetched"); diff --git a/bin/varnishd/common/common_vext.c b/bin/varnishd/common/common_vext.c index fba153627..169cb13ee 100644 --- a/bin/varnishd/common/common_vext.c +++ b/bin/varnishd/common/common_vext.c @@ -163,7 +163,6 @@ vext_cleanup(int do_unlink) struct vext *vp; VTAILQ_FOREACH(vp, &vext_list, list) { - fprintf(stderr, "ee3 %s\n", VSB_data(vp->vsb)); if (vp->vsb != NULL && VSB_len(vp->vsb) > 0) { if (do_unlink) XXXAZ(unlink(VSB_data(vp->vsb))); diff --git a/bin/varnishd/http2/cache_http2_send.c b/bin/varnishd/http2/cache_http2_send.c index 2b66615ca..791cfcbe2 100644 --- a/bin/varnishd/http2/cache_http2_send.c +++ b/bin/varnishd/http2/cache_http2_send.c @@ -409,7 +409,7 @@ void H2_Send(struct worker *wrk, struct h2_req *r2, h2_frame ftyp, uint8_t flags, uint32_t len, const void *ptr, uint64_t *counter) { - uint64_t dummy_counter; + uint64_t dummy_counter = 0; if (counter == NULL) counter = &dummy_counter; From phk at FreeBSD.org Tue Aug 23 10:36:04 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Aug 2022 10:36:04 +0000 (UTC) Subject: [master] 5a39e0cd1 Free the CLI answer, also in terminal circumstances, to silence Coverity Message-ID: <20220823103604.AEC1211E78C@lists.varnish-cache.org> commit 5a39e0cd16e833030f39b53c22a998ec82dee698 Author: Poul-Henning Kamp Date: Tue Aug 23 10:34:43 2022 +0000 Free the CLI answer, also in terminal circumstances, to silence Coverity diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c index d978dcdaa..c09d0b46d 100644 --- a/bin/varnishadm/varnishadm.c +++ b/bin/varnishadm/varnishadm.c @@ -124,6 +124,7 @@ cli_sock(const char *T_arg, const char *S_arg) if (status == CLIS_AUTH) { if (S_arg == NULL) { fprintf(stderr, "Authentication required\n"); + free(answer); closefd(&sock); return (-1); } From dridi.boukelmoune at gmail.com Fri Aug 26 13:01:12 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 26 Aug 2022 13:01:12 +0000 (UTC) Subject: [master] 077ed037c cocci: Help spatch operate on VMODs Message-ID: <20220826130112.0AA0D10E8E6@lists.varnish-cache.org> commit 077ed037ceeab5cdde3a465f8a7f4185bd71bdca Author: Dridi Boukelmoune Date: Fri Aug 26 14:54:14 2022 +0200 cocci: Help spatch operate on VMODs Successfully tested with a dummy semantic patch: @@ idexpression ctx, caller; @@ -CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); +AN(ctx); CHECK_OBJ_NOTNULL(caller, DEBUG_CALLER_MAGIC); -AN(caller->sub); +assert(caller->sub); I decided to cram all definitions in the existing vdef.h file to keep our relatively simple setup. We can add more macros as we learn what coccinelle trips over. Refs f5b0b201fe29999d5ebc1458b354a7459eed9eac diff --git a/tools/coccinelle/vdef.h b/tools/coccinelle/vdef.h index 3cd515d79..ba0248a30 100644 --- a/tools/coccinelle/vdef.h +++ b/tools/coccinelle/vdef.h @@ -1,6 +1,13 @@ +/* vdef.h */ #define v_printflike_(f,a) #define v_deprecated_ #define v_dont_optimize #define v_matchproto_(xxx) #define v_statevariable_(varname) #define v_unused_ + +/* vrt.h */ +#define VRT_CTX const struct vrt_ctx *ctx + +/* vcc_if.h */ +#define VPFX(a) vmod_##a From dridi.boukelmoune at gmail.com Fri Aug 26 14:56:06 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 26 Aug 2022 14:56:06 +0000 (UTC) Subject: [master] b43d539c9 cocci: Apply printf_nofmt.cocci again Message-ID: <20220826145606.9983A111CC0@lists.varnish-cache.org> commit b43d539c9737e3a878ec3a9d9a1b4e03354fe5a4 Author: Dridi Boukelmoune Date: Fri Aug 26 15:26:26 2022 +0200 cocci: Apply printf_nofmt.cocci again diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index cfa99d1c6..b4804cfff 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -271,7 +271,7 @@ vcc_VmodLoad(struct vcc *tl, struct vmod_import *vim) "Different version of VMOD %.*s already loaded\n", PF(vim->t_mod)); vcc_ErrWhere(tl, vim->t_mod); - VSB_printf(tl->sb, "Previous import at:\n"); + VSB_cat(tl->sb, "Previous import at:\n"); vcc_ErrWhere(tl, vim2->t_mod); vcc_Warn(tl); break; @@ -364,9 +364,9 @@ vcc_emit_setup(struct vcc *tl, const struct vmod_import *vim) VSB_cat(tl->symtab, "\t\"type\": \"$VMOD\",\n"); VSB_printf(tl->symtab, "\t\"name\": \"%.*s\",\n", PF(mod)); if (vim->from_vext) - VSB_printf(tl->symtab, "\t\"vext\": true,\n"); + VSB_cat(tl->symtab, "\t\"vext\": true,\n"); else - VSB_printf(tl->symtab, "\t\"vext\": false,\n"); + VSB_cat(tl->symtab, "\t\"vext\": false,\n"); VSB_printf(tl->symtab, "\t\"file\": \"%s\",\n", vim->path); VSB_printf(tl->symtab, "\t\"dst\": \"./vmod_cache/_vmod_%.*s.%s\"\n", PF(mod), vim->file_id); From dridi.boukelmoune at gmail.com Fri Aug 26 14:56:06 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 26 Aug 2022 14:56:06 +0000 (UTC) Subject: [master] eb834f8e2 cocci: Remove VRT_CTX isomorphism attempt Message-ID: <20220826145606.B640B111CC3@lists.varnish-cache.org> commit eb834f8e273609c293b6480fdd1e6dcd577eaf4c Author: Dridi Boukelmoune Date: Fri Aug 26 16:26:45 2022 +0200 cocci: Remove VRT_CTX isomorphism attempt It does not work, and it is inaccurate since it doesn't include the ctx identifier that is normally part of the definition (it can't anyway). Refs f5b0b201fe29999d5ebc1458b354a7459eed9eac diff --git a/tools/coccinelle/varnish.iso b/tools/coccinelle/varnish.iso index 8bbc11c53..3c9434185 100644 --- a/tools/coccinelle/varnish.iso +++ b/tools/coccinelle/varnish.iso @@ -13,13 +13,6 @@ /* This section contains VCL types used by VMODs */ -Type -@ vrt_ctx @ -type VRT_CTX; -@@ - -VRT_CTX <=> const struct vrt_ctx * - Type @ vcl_void @ type VCL_VOID; From dridi.boukelmoune at gmail.com Fri Aug 26 17:23:08 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 26 Aug 2022 17:23:08 +0000 (UTC) Subject: [master] 2af77a57b Revert "Swap isomorphism subjects" Message-ID: <20220826172308.2D0871160EB@lists.varnish-cache.org> commit 2af77a57b033c8d0e288341e7fdd22ecf8eb64fd Author: Dridi Boukelmoune Date: Fri Aug 26 16:42:36 2022 +0200 Revert "Swap isomorphism subjects" This reverts commit c722eb774e4fd5412de94a9cc01a49d9f0f3f76d. It didn't change anything and it certainly didn't fix anything either. diff --git a/tools/coccinelle/varnish.iso b/tools/coccinelle/varnish.iso index 3c9434185..e7034b296 100644 --- a/tools/coccinelle/varnish.iso +++ b/tools/coccinelle/varnish.iso @@ -18,11 +18,11 @@ Type type VCL_VOID; @@ -void <=> VCL_VOID +VCL_VOID <=> void Type @ vcl_bool @ type VCL_BOOL; @@ -unsigned <=> VCL_BOOL +VCL_BOOL <=> unsigned From dridi.boukelmoune at gmail.com Fri Aug 26 17:23:08 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 26 Aug 2022 17:23:08 +0000 (UTC) Subject: [master] dab8e4ac6 cocci: Make isomorphisms actually work Message-ID: <20220826172308.40AE61160ED@lists.varnish-cache.org> commit dab8e4ac6f594fc097148df0993136030d23983d Author: Dridi Boukelmoune Date: Fri Aug 26 16:47:36 2022 +0200 cocci: Make isomorphisms actually work It turns out declaring typedefs in coccinelle works better with the typedef keyword. Reusing the same keyword for the same purposes seems so obvious in hindsight. What happened before was that a generic type would be created, making VCL_BOOL or VCL_VOID match virtually any type, completely defeating the isomorphism purpose. With this out of the way, we can actually resume the effort of adding more typedefs. To summarize: - we should declare a typedef as an isomorphism - we should define macros as such I'm considering renaming tools/coccinelle/vdef.h to varnish.h to match the varnish.iso file name and wrapping the README instructions in a new shell script. diff --git a/tools/coccinelle/README.rst b/tools/coccinelle/README.rst index 99831344a..7582a6c7a 100644 --- a/tools/coccinelle/README.rst +++ b/tools/coccinelle/README.rst @@ -15,6 +15,13 @@ Unless noted otherwise, all patches should work when invoked as:: -I include/ -I bin/varnishd/ --dir . --in-place \ --sp-file $COCCI +To expand a patch and see the implicit rules that will be taken into account, +it is possible to parse the file:: + + spatch --macro-file tools/coccinelle/vdef.h \ + -I include/ -I bin/varnishd/ --parse-cocci + --sp-file $COCCI + The ``archive/`` directory contains patches which we used once and should not need again, but want to retain for reference. diff --git a/tools/coccinelle/varnish.iso b/tools/coccinelle/varnish.iso index e7034b296..c47b97f69 100644 --- a/tools/coccinelle/varnish.iso +++ b/tools/coccinelle/varnish.iso @@ -5,24 +5,26 @@ * It can be used directly by semantic patches from the same directory * with the following syntax: * - * using "varnish.iso" + * @using "varnish.iso"@ + * + * @@ + * + * * * XXX: way incomplete. - * XXX: consider autogeneration. + * XXX: consider autogeneration? */ /* This section contains VCL types used by VMODs */ Type @ vcl_void @ -type VCL_VOID; +typedef VCL_VOID; @@ - VCL_VOID <=> void Type @ vcl_bool @ -type VCL_BOOL; +typedef VCL_BOOL; @@ - VCL_BOOL <=> unsigned From dridi.boukelmoune at gmail.com Mon Aug 29 08:36:07 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 29 Aug 2022 08:36:07 +0000 (UTC) Subject: [master] 91ccf1d9f cocci: Start a vcocci.sh convenience script Message-ID: <20220829083607.5C00810E88E@lists.varnish-cache.org> commit 91ccf1d9f1e04a6cabf15127877bcb1489ad1057 Author: Dridi Boukelmoune Date: Mon Aug 29 10:29:52 2022 +0200 cocci: Start a vcocci.sh convenience script It doesn't do half of the things I have in mind, but it's good enough as a starting point to remove error-prone manual copy-pasta of commands from the README. Eventually it should also become usable for out-of-tree code. diff --git a/tools/coccinelle/README.rst b/tools/coccinelle/README.rst index 7582a6c7a..8b24b0583 100644 --- a/tools/coccinelle/README.rst +++ b/tools/coccinelle/README.rst @@ -1,5 +1,5 @@ .. - Copyright (c) 2019-2021 Varnish Software AS + Copyright (c) 2019-2022 Varnish Software AS SPDX-License-Identifier: BSD-2-Clause See LICENSE file for full text of license @@ -9,6 +9,8 @@ maintenance. Each patch should, in a comment section, explain its purpose. They may be fit for both the in-tree code style or out-of-tree VMOD and VUT development. +For in-tree usage, see the ``vcocci.sh`` script for convenience. + Unless noted otherwise, all patches should work when invoked as:: spatch --macro-file tools/coccinelle/vdef.h \ diff --git a/tools/coccinelle/vcocci.sh b/tools/coccinelle/vcocci.sh new file mode 100755 index 000000000..087fe62fa --- /dev/null +++ b/tools/coccinelle/vcocci.sh @@ -0,0 +1,129 @@ +#!/bin/sh +# +# Copyright (c) 2022 Varnish Software AS +# All rights reserved. +# +# Author: Dridi Boukelmoune +# +# SPDX-License-Identifier: BSD-2-Clause +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +set -e +set -u + +readonly SCRIPT=$(basename $0) +readonly TMP=$(mktemp -d) +trap 'rm -rf $TMP' EXIT + +usage() { + ERR=${1:-1} + test $ERR == 0 || + exec >&2 + + cat <<-EOF + Usage: $SCRIPT [...] + $SCRIPT help + + Operate Coccinnelle semantic patches. + + Available commands: + + apply Apply a patch to the source tree + parse Parse and expand a patch + mkiso Generate a varnish.iso file + help Show this help and exit + + This script operates directly on the Varnish Cache git repository. + EOF + exit $ERR +} + +errorf() { + printf >&2 'Error: ' + printf >&2 "$@" + printf >&2 '\n' +} + +exec_spatch() { + exec spatch \ + --macro-file "$SRCDIR/tools/coccinelle/vdef.h" \ + -I "$SRCDIR/include/" \ + -I "$SRCDIR/bin/varnishd/" \ + "$@" +} + +cmd_apply() { + if test $# != 1 + then + errorf 'invalid arguments' + usage + fi + exec_spatch --in-place --dir "$SRCDIR" --sp-file "$1" +} + +cmd_parse() { + if test $# != 1 + then + errorf 'invalid arguments' + usage + fi + exec_spatch --parse-cocci --sp-file "$1" +} + +cmd_mkiso() { + errorf 'not implemented' + exit 1 +} + +git rev-parse --show-toplevel >/dev/null || +usage + +if test $# = 0 +then + errorf 'missing command' + usage +fi + +readonly SRCDIR=$(git rev-parse --show-toplevel) + +CMD=$1 +shift + +case $CMD in + apply) + cmd_apply "$@" + ;; + parse) + cmd_parse "$@" + ;; + mkiso) + cmd_mkiso "$@" + ;; + help) + usage 0 + ;; + *) + errorf "unknown command '%s'" "$CMD" + usage + ;; +esac From phk at FreeBSD.org Mon Aug 29 10:08:05 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 29 Aug 2022 10:08:05 +0000 (UTC) Subject: [master] 96a320430 Try to quench the last two (insignificant) Coverity issues. Message-ID: <20220829100806.05262111E53@lists.varnish-cache.org> commit 96a32043050a7fa5cc1b74d5d6e04cb9aa9b0a36 Author: Poul-Henning Kamp Date: Mon Aug 29 10:06:48 2022 +0000 Try to quench the last two (insignificant) Coverity issues. diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c index c09d0b46d..e9d280f52 100644 --- a/bin/varnishadm/varnishadm.c +++ b/bin/varnishadm/varnishadm.c @@ -133,6 +133,7 @@ cli_sock(const char *T_arg, const char *S_arg) fprintf(stderr, "Cannot open \"%s\": %s\n", S_arg, strerror(errno)); closefd(&sock); + free(answer); return (-1); } VCLI_AuthResponse(fd, answer, buf); diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c index c75c5d894..5aa84a117 100644 --- a/bin/varnishd/cache/cache_expire.c +++ b/bin/varnishd/cache/cache_expire.c @@ -209,9 +209,9 @@ EXP_Insert(struct worker *wrk, struct objcore *oc) ObjSendEvent(wrk, oc, OEV_EXPIRE); tmpoc = oc; assert(oc->refcnt >= 2); /* Silence coverity */ - (void)HSH_DerefObjCore(wrk, &tmpoc, 0); - AZ(tmpoc); - assert(oc->refcnt >= 1); /* Silence coverity */ + (void)HSH_DerefObjCore(wrk, &oc, 0); + AZ(oc); + assert(tmpoc->refcnt >= 1); /* Silence coverity */ } } From dridi.boukelmoune at gmail.com Mon Aug 29 13:42:07 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 29 Aug 2022 13:42:07 +0000 (UTC) Subject: [master] a44e0a875 task: Rename TASK_QUEUE_CLIENT to TASK_QUEUE_LIMITED Message-ID: <20220829134207.3166E118174@lists.varnish-cache.org> commit a44e0a87526dfb327db222e3760c2effa531d0d9 Author: Dridi Boukelmoune Date: Mon Jul 18 17:08:06 2022 +0200 task: Rename TASK_QUEUE_CLIENT to TASK_QUEUE_LIMITED This better conveys the role of this macro, and leaves the door open to non-client tasks that would be subject to thread_queue_limit. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index af090b6fe..78e4ecee1 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -218,7 +218,7 @@ enum task_prio { }; #define TASK_QUEUE_HIGHEST_PRIORITY TASK_QUEUE_BO -#define TASK_QUEUE_CLIENT(prio) \ +#define TASK_QUEUE_LIMITED(prio) \ (prio == TASK_QUEUE_REQ || prio == TASK_QUEUE_STR) /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 3a5c4bddd..ca09c4f73 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -331,11 +331,9 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio prio) return (0); } - /* - * queue limits only apply to client threads - all other - * work is vital and needs do be done at the earliest + /* Vital work is always queued. */ - if (!TASK_QUEUE_CLIENT(prio) || + if (!TASK_QUEUE_LIMITED(prio) || pp->lqueue + pp->nthr < cache_param->wthread_max + cache_param->wthread_queue_limit) { pp->nqueued++; From dridi.boukelmoune at gmail.com Mon Aug 29 13:42:07 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 29 Aug 2022 13:42:07 +0000 (UTC) Subject: [master] f44514a24 task: Add a TASK_QUEUE_RESERVE macro Message-ID: <20220829134207.57220118177@lists.varnish-cache.org> commit f44514a2490433f3e2ea403405730418dc5e2bb0 Author: Dridi Boukelmoune Date: Mon Jul 18 17:11:04 2022 +0200 task: Add a TASK_QUEUE_RESERVE macro The new macro is added to better reflect operations that cater to the thread reserve. It also enables the allocation the right number of reserve heads in struct pool, instead of having one for each priority. It grants the possibility of priority classes so low that they wouldn't be eligible to queuing, without wasting space with needless queue heads. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 78e4ecee1..805612d3e 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -206,7 +206,7 @@ struct pool_task { * TASK_QUEUE_RUSH is req's returning from waiting list * * NOTE: When changing the number of classes, update places marked with - * TASK_QUEUE__END in mgt_pool.c + * TASK_QUEUE_RESERVE in params.h */ enum task_prio { TASK_QUEUE_BO, @@ -218,6 +218,7 @@ enum task_prio { }; #define TASK_QUEUE_HIGHEST_PRIORITY TASK_QUEUE_BO +#define TASK_QUEUE_RESERVE TASK_QUEUE__END #define TASK_QUEUE_LIMITED(prio) \ (prio == TASK_QUEUE_REQ || prio == TASK_QUEUE_STR) diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index 6d854c127..a71e0a770 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -150,7 +150,7 @@ pool_mkpool(unsigned pool_no) VTAILQ_INIT(&pp->idle_queue); VTAILQ_INIT(&pp->poolsocks); - for (i = 0; i < TASK_QUEUE__END; i++) + for (i = 0; i < TASK_QUEUE_RESERVE; i++) VTAILQ_INIT(&pp->queues[i]); AZ(pthread_cond_init(&pp->herder_cond, NULL)); AZ(pthread_create(&pp->herder_thr, NULL, pool_herder, pp)); diff --git a/bin/varnishd/cache/cache_pool.h b/bin/varnishd/cache/cache_pool.h index 04a026e61..207e275b1 100644 --- a/bin/varnishd/cache/cache_pool.h +++ b/bin/varnishd/cache/cache_pool.h @@ -48,7 +48,7 @@ struct pool { struct lock mtx; unsigned nidle; struct taskhead idle_queue; - struct taskhead queues[TASK_QUEUE__END]; + struct taskhead queues[TASK_QUEUE_RESERVE]; unsigned nthr; unsigned lqueue; uintmax_t sdropped; diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index ca09c4f73..9bbd8f3f7 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -216,8 +216,8 @@ pool_reserve(void) if (cache_param->wthread_reserve < lim) lim = cache_param->wthread_reserve; } - if (lim < TASK_QUEUE__END) - return (TASK_QUEUE__END); + if (lim < TASK_QUEUE_RESERVE) + return (TASK_QUEUE_RESERVE); return (lim); } @@ -231,7 +231,7 @@ pool_getidleworker(struct pool *pp, enum task_prio prio) CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); Lck_AssertHeld(&pp->mtx); - if (pp->nidle > (pool_reserve() * prio / TASK_QUEUE__END)) { + if (pp->nidle > (pool_reserve() * prio / TASK_QUEUE_RESERVE)) { pt = VTAILQ_FIRST(&pp->idle_queue); if (pt == NULL) AZ(pp->nidle); @@ -387,8 +387,8 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk) Lck_Lock(&pp->mtx); reserve = pool_reserve(); - for (i = 0; i < TASK_QUEUE__END; i++) { - if (pp->nidle < (reserve * i / TASK_QUEUE__END)) + for (i = 0; i < TASK_QUEUE_RESERVE; i++) { + if (pp->nidle < (reserve * i / TASK_QUEUE_RESERVE)) break; tp = VTAILQ_FIRST(&pp->queues[i]); if (tp != NULL) { @@ -760,6 +760,6 @@ WRK_Log(enum VSL_tag_e tag, const char *fmt, ...) void WRK_Init(void) { - assert(cache_param->wthread_min >= TASK_QUEUE__END); + assert(cache_param->wthread_min >= TASK_QUEUE_RESERVE); CLI_AddFuncs(debug_cmds); } diff --git a/include/tbl/params.h b/include/tbl/params.h index 9cf5de581..1d3416daa 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1312,7 +1312,7 @@ PARAM_THREAD( /* name */ thread_pool_min, /* field */ min, /* type */ thread_pool_min, - /* min */ "5" /* TASK_QUEUE__END */, + /* min */ "5" /* TASK_QUEUE_RESERVE */, /* max */ NULL, /* def */ "100", /* units */ "threads", @@ -1323,9 +1323,9 @@ PARAM_THREAD( "situations or when threads have expired.\n" "\n" "Technical minimum is 5 threads, but this parameter is " - /* ^ TASK_QUEUE__END */ + /* ^ TASK_QUEUE_RESERVE */ "strongly recommended to be at least 10", - /* 2 * TASK_QUEUE__END ^^ */ + /* 2 * TASK_QUEUE_RESERVE ^^ */ /* flags */ DELAYED_EFFECT, /* dyn_min_reason */ NULL, /* dyn_max_reason */ "thread_pool_max" @@ -1350,7 +1350,7 @@ PARAM_THREAD( "priority tasks from running even under high load.\n" "\n" "The effective value is at least 5 (the number of internal " - /* ^ TASK_QUEUE__END */ + /* ^ TASK_QUEUE_RESERVE */ "priority classes), irrespective of this parameter.", /* flags */ DELAYED_EFFECT, /* dyn_min_reason */ NULL, From dridi.boukelmoune at gmail.com Mon Aug 29 13:42:07 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 29 Aug 2022 13:42:07 +0000 (UTC) Subject: [master] 6e6421360 vsc: Rename fetch_no_thread to bgfetch_no_thread Message-ID: <20220829134207.72B3111817B@lists.varnish-cache.org> commit 6e642136082018545089a627063a8c213ecf46dd Author: Dridi Boukelmoune Date: Mon Jul 18 17:23:39 2022 +0200 vsc: Rename fetch_no_thread to bgfetch_no_thread A fetch transaction is always successfully scheduled or queued, so the fetch_no_thread counter can only ever stay at zero. The plan is to relax this for bgfetch tasks. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 40b304fa7..1edb65576 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -1173,7 +1173,7 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, bo->fetch_task->func = vbf_fetch_thread; if (Pool_Task(wrk->pool, bo->fetch_task, TASK_QUEUE_BO)) { - wrk->stats->fetch_no_thread++; + wrk->stats->bgfetch_no_thread++; (void)vbf_stp_fail(req->wrk, bo); if (bo->stale_oc != NULL) (void)HSH_DerefObjCore(wrk, &bo->stale_oc, 0); diff --git a/lib/libvsc/VSC_main.vsc b/lib/libvsc/VSC_main.vsc index c895f7a1c..047218ad8 100644 --- a/lib/libvsc/VSC_main.vsc +++ b/lib/libvsc/VSC_main.vsc @@ -256,11 +256,11 @@ beresp fetch failed. -.. varnish_vsc:: fetch_no_thread +.. varnish_vsc:: bgfetch_no_thread :group: wrk - :oneliner: Fetch failed (no thread) + :oneliner: Background fetch failed (no thread) - beresp fetch failed, no thread available. + A bgfetch triggered by a grace hit failed, no thread available. .. varnish_vsc:: pools :type: gauge From dridi.boukelmoune at gmail.com Mon Aug 29 13:42:07 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 29 Aug 2022 13:42:07 +0000 (UTC) Subject: [master] 18c6a9a1a wrk: Only queue tasks that fit in the reserve Message-ID: <20220829134207.8DB84118186@lists.varnish-cache.org> commit 18c6a9a1acc8c97af865aba5a5c301dcb97e54c7 Author: Dridi Boukelmoune Date: Mon Jul 18 17:31:44 2022 +0200 wrk: Only queue tasks that fit in the reserve diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 9bbd8f3f7..0b8bd087e 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -331,9 +331,12 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio prio) return (0); } - /* Vital work is always queued. + /* Vital work is always queued. Only priority classes that can + * fit under the reserve capacity are eligible to queuing. */ - if (!TASK_QUEUE_LIMITED(prio) || + if (prio >= TASK_QUEUE_RESERVE) { + retval = -1; + } else if (!TASK_QUEUE_LIMITED(prio) || pp->lqueue + pp->nthr < cache_param->wthread_max + cache_param->wthread_queue_limit) { pp->nqueued++; From dridi.boukelmoune at gmail.com Mon Aug 29 13:42:07 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 29 Aug 2022 13:42:07 +0000 (UTC) Subject: [master] d5b75354d wrk: Give bgfetch tasks the lowest priority Message-ID: <20220829134207.B395611818D@lists.varnish-cache.org> commit d5b75354dcccd077a889bf1a84bfc83b4b575145 Author: Dridi Boukelmoune Date: Mon Jul 18 17:33:15 2022 +0200 wrk: Give bgfetch tasks the lowest priority The goal is to prevent grace mode from adding load to a saturated Varnish server. A background fetch entering the queue will block the client task that triggerred it until it starts its execution, and reaches the point where it no longer needs to hold onto its req. On a saturated system this can result in significant client latency despite a grace hit. A stale object can be served until the end of its grace period, at which point a regular fetch would be attempted, and eligible to queuing if that is still necessary. This turns the Pool_Task() failure dead branch from VBF_Fetch() into a reachable one. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 805612d3e..424067abb 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -214,11 +214,12 @@ enum task_prio { TASK_QUEUE_REQ, TASK_QUEUE_STR, TASK_QUEUE_VCA, + TASK_QUEUE_BG, TASK_QUEUE__END }; #define TASK_QUEUE_HIGHEST_PRIORITY TASK_QUEUE_BO -#define TASK_QUEUE_RESERVE TASK_QUEUE__END +#define TASK_QUEUE_RESERVE TASK_QUEUE_BG #define TASK_QUEUE_LIMITED(prio) \ (prio == TASK_QUEUE_REQ || prio == TASK_QUEUE_STR) diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 1edb65576..251f1676a 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -1109,6 +1109,7 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, { struct boc *boc; struct busyobj *bo; + enum task_prio prio; const char *how; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -1126,13 +1127,16 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, switch (mode) { case VBF_PASS: + prio = TASK_QUEUE_BO; how = "pass"; bo->uncacheable = 1; break; case VBF_NORMAL: + prio = TASK_QUEUE_BO; how = "fetch"; break; case VBF_BACKGROUND: + prio = TASK_QUEUE_BG; how = "bgfetch"; bo->is_bgfetch = 1; break; @@ -1172,7 +1176,7 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, bo->fetch_task->priv = bo; bo->fetch_task->func = vbf_fetch_thread; - if (Pool_Task(wrk->pool, bo->fetch_task, TASK_QUEUE_BO)) { + if (Pool_Task(wrk->pool, bo->fetch_task, prio)) { wrk->stats->bgfetch_no_thread++; (void)vbf_stp_fail(req->wrk, bo); if (bo->stale_oc != NULL) diff --git a/bin/varnishtest/tests/c00120.vtc b/bin/varnishtest/tests/c00120.vtc new file mode 100644 index 000000000..b340a48a9 --- /dev/null +++ b/bin/varnishtest/tests/c00120.vtc @@ -0,0 +1,32 @@ +varnishtest "bgfetch_no_thread" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -cliok "param.set thread_pools 1" +varnish v1 -cliok "param.set thread_pool_min 5" +varnish v1 -cliok "param.set thread_pool_max 5" +varnish v1 -vcl+backend { + sub vcl_backend_response { + set beresp.ttl = 1ms; + set beresp.grace = 1h; + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 + + delay 0.1 + + # At this point the thread reserve is too low to + # allow a low-priority task like a bgfetch. + txreq + rxresp + expect resp.status == 200 +} -start + +varnish v1 -expect MAIN.bgfetch_no_thread == 1 From nils.goroll at uplex.de Mon Aug 29 16:40:07 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 29 Aug 2022 16:40:07 +0000 (UTC) Subject: [master] 4b1aa9acb Macro housekeeping Message-ID: <20220829164007.8A8BB11D86A@lists.varnish-cache.org> commit 4b1aa9acb5b2ed030fcec84250aba961a55b4716 Author: Nils Goroll Date: Mon Aug 29 17:22:13 2022 +0200 Macro housekeeping diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 8d2da90ca..f508dc709 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -253,6 +253,9 @@ VRT_r_bereq_##field(VRT_CTX) \ #undef VBERESPRF1 #undef VBERESPR0 #undef VBERESPR1 + +#undef VBEREQR0 +#undef VBEREQR1 /*--------------------------------------------------------------------*/ VCL_BOOL From nils.goroll at uplex.de Mon Aug 29 16:40:07 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 29 Aug 2022 16:40:07 +0000 (UTC) Subject: [master] 9f996c1fe Refactor bereq/beresp flag tables Message-ID: <20220829164007.ADD6911D86D@lists.varnish-cache.org> commit 9f996c1fee4c736feb2f15ff07d7d3e7bf403f06 Author: Nils Goroll Date: Mon Aug 29 17:38:53 2022 +0200 Refactor bereq/beresp flag tables The filter check does not make sense in the context of bereq and the macros became overloaded with two different cases. Motivated by and in preparation of #3826 diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 424067abb..7b4d8f9a7 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -394,8 +394,10 @@ struct busyobj { struct pool_task fetch_task[1]; -#define BO_FLAG(l, r, rr, rw, f, d) unsigned l:1; -#include "tbl/bo_flags.h" +#define BERESP_FLAG(l, r, w, f, d) unsigned l:1; +#define BEREQ_FLAG(l, r, w, d) BERESP_FLAG(l, r, w, 0, d) +#include "tbl/bereq_flags.h" +#include "tbl/beresp_flags.h" /* Timeouts */ vtim_dur connect_timeout; diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index b93fdb7b1..45384a0bb 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -390,9 +390,11 @@ pan_busyobj(struct vsb *vsb, const struct busyobj *bo) VSB_cat(vsb, "flags = {"); p = ""; /*lint -save -esym(438,p) -e539 */ -#define BO_FLAG(l, r, rr, rw, f, d) \ +#define BERESP_FLAG(l, r, w, f, d) \ if (bo->l) { VSB_printf(vsb, "%s" #l, p); p = ", "; } -#include "tbl/bo_flags.h" +#define BEREQ_FLAG(l, r, w, d) BERESP_FLAG(l, r, w, 0, d) +#include "tbl/bereq_flags.h" +#include "tbl/beresp_flags.h" /*lint -restore */ VSB_cat(vsb, "},\n"); diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index f508dc709..7a12df235 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -181,7 +181,7 @@ VRT_r_obj_reason(VRT_CTX) } /*-------------------------------------------------------------------- - * bool-fields (.do_*) + * beresp bool-fields */ static inline int @@ -228,21 +228,10 @@ VRT_r_beresp_##field(VRT_CTX) \ return (ctx->bo->field); \ } -#define VBEREQR0(field, str, fltchk) -#define VBEREQR1(field, str, fltchk) \ -VCL_BOOL \ -VRT_r_bereq_##field(VRT_CTX) \ -{ \ - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ - CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); \ - return (ctx->bo->field); \ -} - -#define BO_FLAG(l, r, rr, rw, f, d) \ - VBEREQR##r(l, #l, f) \ - VBERESPR##rr(l, #l, f) \ - VBERESPW##rw(l, #l, f) -#include "tbl/bo_flags.h" +#define BERESP_FLAG(l, r, w, f, d) \ + VBERESPR##r(l, #l, f) \ + VBERESPW##w(l, #l, f) +#include "tbl/beresp_flags.h" #undef VBERESPWF0 #undef VBERESPWF1 @@ -254,6 +243,27 @@ VRT_r_bereq_##field(VRT_CTX) \ #undef VBERESPR0 #undef VBERESPR1 +/*-------------------------------------------------------------------- + * bereq bool-fields + */ + +#define VBEREQR0(field, str) +#define VBEREQR1(field, str) \ +VCL_BOOL \ +VRT_r_bereq_##field(VRT_CTX) \ +{ \ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); \ + return (ctx->bo->field); \ +} + +// w is unused +#define VBEREQW0(ctx, str) (void) 0 + +#define BEREQ_FLAG(l, r, w, d) \ + VBEREQR##r(l, #l) +#include "tbl/bereq_flags.h" + #undef VBEREQR0 #undef VBEREQR1 /*--------------------------------------------------------------------*/ diff --git a/include/Makefile.am b/include/Makefile.am index a7a77310d..81d05d96e 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -8,8 +8,8 @@ nobase_pkginclude_HEADERS = \ tbl/ban_arg_oper.h \ tbl/ban_oper.h \ tbl/ban_vars.h \ - tbl/bo_flags.h \ - tbl/boc_state.h \ + tbl/bereq_flags.h \ + tbl/beresp_flags.h \ tbl/body_status.h \ tbl/cli_cmds.h \ tbl/debug_bits.h \ diff --git a/include/tbl/bereq_flags.h b/include/tbl/bereq_flags.h new file mode 100644 index 000000000..c7831fec4 --- /dev/null +++ b/include/tbl/bereq_flags.h @@ -0,0 +1,41 @@ +/*- + * Copyright (c) 2014-2015 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +/*lint -save -e525 -e539 */ + +/* lower, vcl_r, vcl_w, doc */ +BEREQ_FLAG(uncacheable, 0, 0, "") // also beresp +BEREQ_FLAG(is_bgfetch, 1, 0, "") +BEREQ_FLAG(is_hitmiss, 1, 0, "") +BEREQ_FLAG(is_hitpass, 1, 0, "") +#undef BEREQ_FLAG + +/*lint -restore */ diff --git a/include/tbl/bo_flags.h b/include/tbl/beresp_flags.h similarity index 80% rename from include/tbl/bo_flags.h rename to include/tbl/beresp_flags.h index 4a4f1dd4b..b4eb789b3 100644 --- a/include/tbl/bo_flags.h +++ b/include/tbl/beresp_flags.h @@ -34,16 +34,12 @@ /* * filters: whether this flag determines beresp.filters default * - * lower, vcl_r, vcl_beresp_r, vcl_beresp_w, filters, doc */ -BO_FLAG(do_esi, 0, 1, 1, 1, "") -BO_FLAG(do_gzip, 0, 1, 1, 1, "") -BO_FLAG(do_gunzip, 0, 1, 1, 1, "") -BO_FLAG(do_stream, 0, 1, 1, 0, "") -BO_FLAG(uncacheable, 0, 0, 0, 0, "") -BO_FLAG(was_304, 0, 1, 0, 0, "") -BO_FLAG(is_bgfetch, 1, 0, 0, 0, "") -BO_FLAG(is_hitmiss, 1, 0, 0, 0, "") -BO_FLAG(is_hitpass, 1, 0, 0, 0, "") -#undef BO_FLAG + * lower, vcl_r, vcl_w, filters, doc */ +BERESP_FLAG(do_esi, 1, 1, 1, "") +BERESP_FLAG(do_gzip, 1, 1, 1, "") +BERESP_FLAG(do_gunzip, 1, 1, 1, "") +BERESP_FLAG(do_stream, 1, 1, 0, "") +BERESP_FLAG(was_304, 1, 0, 0, "") +#undef BERESP_FLAG /*lint -restore */ From nils.goroll at uplex.de Mon Aug 29 16:40:07 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 29 Aug 2022 16:40:07 +0000 (UTC) Subject: [master] f47db4177 Formalize flags which exist for req and bereq Message-ID: <20220829164007.CDE5A11D875@lists.varnish-cache.org> commit f47db4177de6f2e89e50ee87561a54297f64b190 Author: Nils Goroll Date: Mon Aug 29 18:33:32 2022 +0200 Formalize flags which exist for req and bereq Motivated by and in preparation of #3826 diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 251f1676a..b9be6f4a1 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -1144,8 +1144,8 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, WRONG("Wrong fetch mode"); } - bo->is_hitpass = req->is_hitpass; - bo->is_hitmiss = req->is_hitmiss; +#define REQ_BEREQ_FLAG(l, r, w, d) bo->l = req->l; +#include "tbl/req_bereq_flags.h" VSLb(bo->vsl, SLT_Begin, "bereq %u %s", VXID(req->vsl->wid), how); VSLbs(bo->vsl, SLT_VCL_use, TOSTRAND(VCL_Name(bo->vcl))); diff --git a/include/Makefile.am b/include/Makefile.am index 81d05d96e..76574fb20 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -27,6 +27,7 @@ nobase_pkginclude_HEADERS = \ tbl/oc_exp_flags.h \ tbl/oc_flags.h \ tbl/params.h \ + tbl/req_bereq_flags.h \ tbl/req_flags.h \ tbl/sess_attr.h \ tbl/sess_close.h \ diff --git a/include/tbl/bereq_flags.h b/include/tbl/bereq_flags.h index c7831fec4..31660035c 100644 --- a/include/tbl/bereq_flags.h +++ b/include/tbl/bereq_flags.h @@ -34,8 +34,9 @@ /* lower, vcl_r, vcl_w, doc */ BEREQ_FLAG(uncacheable, 0, 0, "") // also beresp BEREQ_FLAG(is_bgfetch, 1, 0, "") -BEREQ_FLAG(is_hitmiss, 1, 0, "") -BEREQ_FLAG(is_hitpass, 1, 0, "") +#define REQ_BEREQ_FLAG(lower, vcl_r, vcl_w, doc) \ + BEREQ_FLAG(lower, vcl_r, vcl_w, doc) +#include "tbl/req_bereq_flags.h" #undef BEREQ_FLAG /*lint -restore */ diff --git a/include/tbl/req_bereq_flags.h b/include/tbl/req_bereq_flags.h new file mode 100644 index 000000000..36f868d2a --- /dev/null +++ b/include/tbl/req_bereq_flags.h @@ -0,0 +1,39 @@ +/*- + * Copyright (c) 2015 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +/*lint -save -e525 -e539 */ + +/* lower, vcl_r, vcl_w, doc */ +REQ_BEREQ_FLAG(is_hitmiss, 1, 0, "") +REQ_BEREQ_FLAG(is_hitpass, 1, 0, "") +#undef REQ_BEREQ_FLAG + +/*lint -restore */ diff --git a/include/tbl/req_flags.h b/include/tbl/req_flags.h index 0c49540db..88eaff639 100644 --- a/include/tbl/req_flags.h +++ b/include/tbl/req_flags.h @@ -37,11 +37,12 @@ REQ_FLAG(hash_ignore_busy, 1, 1, "") REQ_FLAG(hash_ignore_vary, 1, 1, "") REQ_FLAG(hash_always_miss, 1, 1, "") REQ_FLAG(is_hit, 0, 0, "") -REQ_FLAG(is_hitmiss, 1, 0, "") -REQ_FLAG(is_hitpass, 1, 0, "") REQ_FLAG(waitinglist, 0, 0, "") REQ_FLAG(want100cont, 0, 0, "") REQ_FLAG(late100cont, 0, 0, "") +#define REQ_BEREQ_FLAG(lower, vcl_r, vcl_w, doc) \ + REQ_FLAG(lower, vcl_r, vcl_w, doc) +#include "tbl/req_bereq_flags.h" #undef REQ_FLAG /*lint -restore */ From nils.goroll at uplex.de Mon Aug 29 18:59:04 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 29 Aug 2022 18:59:04 +0000 (UTC) Subject: [master] aecd1383f Bitmap checks to return boolean values Message-ID: <20220829185904.C32A54961@lists.varnish-cache.org> commit aecd1383f1fba7bcb3009bfe97e917b898c7fd39 Author: Nils Goroll Date: Mon Aug 29 20:46:00 2022 +0200 Bitmap checks to return boolean values Stumpled over (unsigned:1)var = FEATURE(...) not being true when it should have been... Related: We might want to remove some redundancy from common_param.h... diff --git a/bin/varnishd/common/common_param.h b/bin/varnishd/common/common_param.h index 14debcca7..4c64c4696 100644 --- a/bin/varnishd/common/common_param.h +++ b/bin/varnishd/common/common_param.h @@ -49,7 +49,7 @@ enum debug_bits { static inline int COM_DO_DEBUG(const volatile uint8_t *p, enum debug_bits x) { - return (p[(unsigned)x>>3] & (0x80U >> ((unsigned)x & 7))); + return ((p[(unsigned)x>>3] & (0x80U >> ((unsigned)x & 7))) != 0); } enum experimental_bits { @@ -61,7 +61,7 @@ enum experimental_bits { static inline int COM_EXPERIMENT(const volatile uint8_t *p, enum experimental_bits x) { - return (p[(unsigned)x>>3] & (0x80U >> ((unsigned)x & 7))); + return ((p[(unsigned)x>>3] & (0x80U >> ((unsigned)x & 7))) != 0); } enum feature_bits { @@ -73,7 +73,7 @@ enum feature_bits { static inline int COM_FEATURE(const volatile uint8_t *p, enum feature_bits x) { - return (p[(unsigned)x>>3] & (0x80U >> ((unsigned)x & 7))); + return ((p[(unsigned)x>>3] & (0x80U >> ((unsigned)x & 7))) != 0); } enum vcc_feature_bits { @@ -85,7 +85,7 @@ enum vcc_feature_bits { static inline int COM_VCC_FEATURE(const volatile uint8_t *p, enum vcc_feature_bits x) { - return (p[(unsigned)x>>3] & (0x80U >> ((unsigned)x & 7))); + return ((p[(unsigned)x>>3] & (0x80U >> ((unsigned)x & 7))) != 0); } struct poolparam { From nils.goroll at uplex.de Tue Aug 30 06:14:05 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 30 Aug 2022 06:14:05 +0000 (UTC) Subject: [master] 886ab87c8 Bring back a lost line Message-ID: <20220830061405.A05EF10FC5F@lists.varnish-cache.org> commit 886ab87c87bc7990be3c316feea1c1a117c5365d Author: Nils Goroll Date: Tue Aug 30 08:13:12 2022 +0200 Bring back a lost line I accidentally broke the installation in 9f996c1fee4c736feb2f15ff07d7d3e7bf403f06 diff --git a/include/Makefile.am b/include/Makefile.am index 76574fb20..0db210f5b 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -10,6 +10,7 @@ nobase_pkginclude_HEADERS = \ tbl/ban_vars.h \ tbl/bereq_flags.h \ tbl/beresp_flags.h \ + tbl/boc_state.h \ tbl/body_status.h \ tbl/cli_cmds.h \ tbl/debug_bits.h \ From dridi.boukelmoune at gmail.com Tue Aug 30 15:18:05 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 30 Aug 2022 15:18:05 +0000 (UTC) Subject: [master] 34a2c5049 build: Introduce a new contrib/ directory Message-ID: <20220830151805.CD7EF11F108@lists.varnish-cache.org> commit 34a2c5049bc731f7dd97d62023494e9f23144141 Author: Dridi Boukelmoune Date: Mon Jul 11 11:20:11 2022 +0200 build: Introduce a new contrib/ directory diff --git a/.circleci/config.yml b/.circleci/config.yml index 8b20b9016..7dc228ff0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,6 +16,7 @@ parameters: configure_args: type: string default: | + --with-contrib \ --with-unwind \ --enable-developer-warnings \ --enable-debugging-symbols \ diff --git a/Makefile.am b/Makefile.am index 3e84a6cf6..d33f79a70 100644 --- a/Makefile.am +++ b/Makefile.am @@ -2,6 +2,10 @@ ACLOCAL_AMFLAGS = -I m4 -I . SUBDIRS = include lib bin vmod etc doc man +if WITH_CONTRIB +SUBDIRS += contrib +endif + TESTS = tools/magic_check.sh pkgconfigdir = $(libdir)/pkgconfig @@ -35,6 +39,7 @@ AM_DISTCHECK_CONFIGURE_FLAGS = \ --enable-developer-warnings \ --enable-debugging-symbols \ --enable-dependency-tracking \ + --with-contrib \ CFLAGS="$(EXTCFLAGS)" if WITH_UNWIND diff --git a/autogen.des b/autogen.des index 294053b87..d256a9bcb 100755 --- a/autogen.des +++ b/autogen.des @@ -35,4 +35,5 @@ $SRCDIR/configure \ --enable-debugging-symbols \ --enable-dependency-tracking \ --with-persistent-storage \ + --with-contrib \ "$@" diff --git a/configure.ac b/configure.ac index ca2bf0b01..57f444a92 100644 --- a/configure.ac +++ b/configure.ac @@ -924,4 +924,18 @@ AC_CONFIG_FILES([ varnishapi-uninstalled.pc vmod/Makefile ]) + +AC_ARG_WITH([contrib], + [AS_HELP_STRING([--with-contrib], + [Build Varnish with external contributions.])]) + +AM_CONDITIONAL([WITH_CONTRIB], [test "$with_contrib" = yes]) + +AM_COND_IF([WITH_CONTRIB], [ + AC_DEFINE([WITH_CONTRIB], [1], + [Define to 1 when Varnish is built with contributions.]) + + AC_CONFIG_FILES([contrib/Makefile]) +]) + AC_OUTPUT diff --git a/contrib/Makefile.am b/contrib/Makefile.am new file mode 100644 index 000000000..2c2a87cbf --- /dev/null +++ b/contrib/Makefile.am @@ -0,0 +1,3 @@ +# + + From dridi.boukelmoune at gmail.com Tue Aug 30 15:18:05 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 30 Aug 2022 15:18:05 +0000 (UTC) Subject: [master] d4dafd5d9 build: Remove stale comment about varnishtest Message-ID: <20220830151805.E034111F10E@lists.varnish-cache.org> commit d4dafd5d956ddd8901fc3af60dc49d8722080e46 Author: Dridi Boukelmoune Date: Mon Jul 11 11:20:52 2022 +0200 build: Remove stale comment about varnishtest In the top makefile we make the check target depend on the all target, so we ensure that everything is built before we start using varnishtest. diff --git a/bin/Makefile.am b/bin/Makefile.am index f103fe94a..2d4185c48 100644 --- a/bin/Makefile.am +++ b/bin/Makefile.am @@ -1,7 +1,5 @@ # -# XXX: varnishtest MUST always be built last - SUBDIRS = \ varnishadm \ varnishd \ From dridi.boukelmoune at gmail.com Tue Aug 30 15:18:06 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 30 Aug 2022 15:18:06 +0000 (UTC) Subject: [master] df4ab5b13 varnishtest: Add a ${topsrc} macro in -i mode Message-ID: <20220830151806.078C111F116@lists.varnish-cache.org> commit df4ab5b13548e94b7138c47d2bb2fb90b6143bde Author: Dridi Boukelmoune Date: Thu Jun 30 10:51:14 2022 +0200 varnishtest: Add a ${topsrc} macro in -i mode This removes the ability to run in -i mode without finding a makefile to derive ${topbuild} from, which shouldn't exist in the first place. diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 9080ab3b3..f1f155c60 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -467,6 +467,40 @@ start_test(void) * */ +static char * +top_dir(const char *makefile, const char *top_var) +{ + const char *b, *e; + char *var; + + AN(makefile); + AN(top_var); + assert(*top_var == '\n'); + + b = strstr(makefile, top_var); + top_var++; + + if (b == NULL) { + fprintf(stderr, "could not find '%s' in Makefile\n", top_var); + return (NULL); + } + + e = strchr(b + 1, '\n'); + if (e == NULL) { + fprintf(stderr, "No NL after '%s' in Makefile\n", top_var); + return (NULL); + } + + b = memchr(b, '/', e - b); + if (b == NULL) { + fprintf(stderr, "No '/' after '%s' in Makefile\n", top_var); + return (NULL); + } + var = strndup(b, e - b); + AN(var); + return (var); +} + static void build_path(const char *topbuilddir, const char *subdir, const char *pfx, const char *sfx, struct vsb *vsb) @@ -501,8 +535,7 @@ static void i_mode(void) { struct vsb *vsb; - char *p, *q; - char *topbuild; + char *p, *topbuild, *topsrc; /* * This code has a rather intimate knowledge of auto* generated @@ -512,39 +545,22 @@ i_mode(void) vsb = VSB_new_auto(); AN(vsb); - q = p = VFIL_readfile(NULL, "Makefile", NULL); + p = VFIL_readfile(NULL, "Makefile", NULL); if (p == NULL) { fprintf(stderr, "No Makefile to search for -i flag.\n"); - VSB_printf(vsb, "%s/../..", cwd); - AZ(VSB_finish(vsb)); - topbuild = strdup(VSB_data(vsb)); - VSB_clear(vsb); - } else { - p = strstr(p, "\nabs_top_builddir"); - if (p == NULL) { - fprintf(stderr, - "could not find 'abs_top_builddir' in Makefile\n"); - exit(2); - } - topbuild = strchr(p + 1, '\n'); - if (topbuild == NULL) { - fprintf(stderr, - "No NL after 'abs_top_builddir' in Makefile\n"); - exit(2); - } - *topbuild = '\0'; - topbuild = strchr(p, '/'); - if (topbuild == NULL) { - fprintf(stderr, - "No '/' after 'abs_top_builddir' in Makefile\n"); - exit(2); - } - topbuild = strdup(topbuild); - free(q); + exit(2); + } + topbuild = top_dir(p, "\nabs_top_builddir"); + topsrc = top_dir(p, "\nabs_top_srcdir"); + free(p); + if (topbuild == NULL || topsrc == NULL) { + free(topbuild); + free(topsrc); + exit(2); } - AN(topbuild); extmacro_def("topbuild", NULL, "%s", topbuild); + extmacro_def("topsrc", NULL, "%s", topsrc); /* * Build $PATH which can find all programs in the build tree @@ -567,6 +583,7 @@ i_mode(void) AN(vmod_path); free(topbuild); + free(topsrc); VSB_destroy(&vsb); /* From dridi.boukelmoune at gmail.com Tue Aug 30 15:18:06 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 30 Aug 2022 15:18:06 +0000 (UTC) Subject: [master] 5b06c8458 varnishtest: Add ${topsrc}/contrib to $PATH in -i mode Message-ID: <20220830151806.276D611F123@lists.varnish-cache.org> commit 5b06c84584476f3311a14c6f36117ee1950498dd Author: Dridi Boukelmoune Date: Thu Jun 30 10:56:38 2022 +0200 varnishtest: Add ${topsrc}/contrib to $PATH in -i mode This will add scripts we ship to the PATH, so they remain reachable for VPATH builds. diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index f1f155c60..d020b37ec 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -502,16 +502,18 @@ top_dir(const char *makefile, const char *top_var) } static void -build_path(const char *topbuilddir, const char *subdir, +build_path(const char *topdir, const char *subdir, const char *pfx, const char *sfx, struct vsb *vsb) { char buf[PATH_MAX]; DIR *dir; struct dirent *de; struct stat st; - const char *sep = ""; + const char *topsep = "", *sep = ""; - bprintf(buf, "%s/%s/", topbuilddir, subdir); + if (*subdir != '\0') + topsep = "/"; + bprintf(buf, "%s%s%s/", topdir, topsep, subdir); dir = opendir(buf); XXXAN(dir); while (1) { @@ -520,7 +522,7 @@ build_path(const char *topbuilddir, const char *subdir, break; if (strncmp(de->d_name, pfx, strlen(pfx))) continue; - bprintf(buf, "%s/%s/%s", topbuilddir, subdir, de->d_name); + bprintf(buf, "%s%s%s/%s", topdir, topsep, subdir, de->d_name); if (!stat(buf, &st) && S_ISDIR(st.st_mode)) { VSB_cat(vsb, sep); VSB_cat(vsb, buf); @@ -568,6 +570,10 @@ i_mode(void) VSB_clear(vsb); VSB_cat(vsb, "PATH="); build_path(topbuild, "bin", "varnish", "", vsb); +#ifdef WITH_CONTRIB + VSB_putc(vsb, ':'); + build_path(topsrc, "", "contrib", "", vsb); +#endif VSB_printf(vsb, ":%s", getenv("PATH")); AZ(VSB_finish(vsb)); AZ(putenv(strdup(VSB_data(vsb)))); From dridi.boukelmoune at gmail.com Tue Aug 30 15:18:06 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 30 Aug 2022 15:18:06 +0000 (UTC) Subject: [master] 3d0a72c86 varnishstatdiff: New utility to compare metrics Message-ID: <20220830151806.4D81211F12D@lists.varnish-cache.org> commit 3d0a72c86a8036544d4bde2d846e355377036a9c Author: Dridi Boukelmoune Date: Mon Jul 11 12:09:15 2022 +0200 varnishstatdiff: New utility to compare metrics As I was comparing the output of two varnishstat executions that were captured after a fresh start of varnishd followed by the workloads to compare, I realized diff(1) was giving me a hard time, and git-diff(1) barely improved the situation. Looking for generic command line utilities to compare metrics I wasn't able to find anything. So instead I came up with the output format I thought would help me spot interesting differences and came up with a format inspired by the unified diff, with a twist. I wanted metrics to be vertically aligned to easily see differences in orders of magnitude and reduce the noise to a minimum. The result is that taking this detour to script varnishstatdiff sped my research up ultimately. This should hopefully be portable to POSIX systems. diff --git a/configure.ac b/configure.ac index 57f444a92..a76725e30 100644 --- a/configure.ac +++ b/configure.ac @@ -932,6 +932,10 @@ AC_ARG_WITH([contrib], AM_CONDITIONAL([WITH_CONTRIB], [test "$with_contrib" = yes]) AM_COND_IF([WITH_CONTRIB], [ + CONTRIB_TESTS="$(cd $srcdir/contrib && echo tests/*.vtc)" + AC_SUBST(CONTRIB_TESTS) + AM_SUBST_NOTMAKE(CONTRIB_TESTS) + AC_DEFINE([WITH_CONTRIB], [1], [Define to 1 when Varnish is built with contributions.]) diff --git a/contrib/Makefile.am b/contrib/Makefile.am index 2c2a87cbf..54482b52b 100644 --- a/contrib/Makefile.am +++ b/contrib/Makefile.am @@ -1,3 +1,10 @@ # +dist_bin_SCRIPTS = \ + varnishstatdiff +TESTS = @CONTRIB_TESTS@ + +include $(top_srcdir)/vtc.am + +EXTRA_DIST = $(TESTS) diff --git a/contrib/tests/statdiff_b00000.vtc b/contrib/tests/statdiff_b00000.vtc new file mode 100644 index 000000000..c99edfece --- /dev/null +++ b/contrib/tests/statdiff_b00000.vtc @@ -0,0 +1,54 @@ +varnishtest "varnishstatdiff coverage" + +feature cmd "command -v column" +feature cmd "command -v diff" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend "" -start + +shell { + varnishstat -n ${v1_name} -1 \ + -I MAIN.n_object -I MAIN.cache_* -I MAIN.client_req | + tee stat1.txt +} + +client c1 { + txreq + rxresp +} -start + +varnish v1 -vsl_catchup + +shell { + varnishstat -n ${v1_name} -1 \ + -I MAIN.n_object -I MAIN.cache_* -I MAIN.esi_req | + tee stat2.txt +} + +shell -expect Usage: {varnishstatdiff -h} +shell -expect "Error: not enough arguments" -err {varnishstatdiff} +shell -expect "Error: not enough arguments" -err {varnishstatdiff a} +shell -expect "Error: too many arguments" -err {varnishstatdiff a b c} + +shell { + varnishstatdiff stat1.txt stat2.txt | tee diff.txt +} + +shell { + sed 's/@/ /' >expected.txt <<-EOF + --- stat1.txt + +++ stat2.txt + @MAIN.cache_miss -0 -0.00 Cache misses + @ +1 +0.00 + -MAIN.client_req 0 0.00 Good client requests received + +MAIN.esi_req 0 0.00 ESI subrequests + @MAIN.n_object -0 . object structs made + @ +1 . + EOF + + diff -u expected.txt diff.txt +} diff --git a/contrib/varnishstatdiff b/contrib/varnishstatdiff new file mode 100755 index 000000000..8ff28d86d --- /dev/null +++ b/contrib/varnishstatdiff @@ -0,0 +1,175 @@ +#!/bin/sh +# +# Copyright (c) 2022 Varnish Software AS +# All rights reserved. +# +# Author: Dridi Boukelmoune +# +# SPDX-License-Identifier: BSD-2-Clause +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +set -e +set -u + +readonly SCRIPT=$0 +readonly TMP=$(mktemp -d) +trap 'rm -rf $TMP' EXIT + +usage() { + test $# -eq 1 && + printf 'Error: %s.\n\n' "$1" + + sed 's:@: :' <<-EOF + Usage: $SCRIPT + $SCRIPT -h + + Show the differences between two sets of varnish metrics extracted + with 'varnishstat -1'. + + Available options: + -h : show this help and exit + + Considering the following metrics in : + + FOO.counter 123 12 Only in file 1 + BAR.counter 456 45 Counter present in both files + BAR.gauge 999 . Gauge present in both files + + And the following metrics in : + + BAR.counter 789 79 Counter present in both files + BAR.gauge 555 . Gauge present in both files + BAZ.gauge 0 . Only in file 2 + + The output is sorted by metric name and looks like this: + + --- + +++ + @BAR.counter -456 -45 Counter present in both files + @ +789 +79 + @BAR.gauge -999 . Gauge present in both files + @ +555 . + +BAZ.gauge 0 . Only in file 2 + -FOO.counter 123 12 Only in file 1 + + The output looks like a unified diff except that when metrics are + present in both files, the diff is rendered as such only in the + metrics columns. + EOF + exit $# +} + +join_prepare() { + # NB: the metrics need to be sorted to later be joined, and since + # the metrics descriptions contain spaces, using a delimiter other + # than space solves the problem. Hopefully @ never shows up in the + # varnishstat -1 output. + sort -k 1b,1 "$1" | + sed 's: *:@: ; s::@: ; s::@:' +} + +join_render() { + # The resulting columns are: + # 1: metric name + # 2: value in file 1 + # 3: rate in file 1 + # 4: value in file 2 + # 5: rate in file 2 + # 6: description in file 1 + # 7: description in file 2 + join -a1 -a2 -t@ -o '0 1.2 1.3 2.2 2.3 1.4 2.4' -- "$1" "$2" +} + +diff_preamble() { + printf "%s %s\n+++ %s\n" --- "$1" "$2" +} + +diff_measure() { + awk -F@ ' + BEGIN { + max[1] = 0 + max[2] = 0 + max[3] = 0 + max[4] = 0 + max[5] = 0 + } + $2 != $4 || $3 != $5 { + for (i in max) { + if (max[i] < length($i)) + max[i] = length($i) + } + } + END { + if (max[2] < max[4]) + max[2] = max[4] + if (max[3] < max[5]) + max[3] = max[5] + printf "%d %d %d\n", max[1] + 2, max[2] + 2, max[3] + 2 + } + ' +} + +diff_render() { + read l1 l2 l3 + awk -F@ -v l1="$l1" -v l2="$l2" -v l3="$l3" ' + $2 != "" && $4 != "" && ($2 != $4 || $3 != $5) { # present in both + sgn = "-" + if ($3 == ".") + sgn = " " + printf " %-*s-%-*s%s%-*s%s\n", l1, $1, l2, $2, sgn, l3, $3, $6 + + sgn = "+" + if ($5 == ".") + sgn = " " + printf " %-*s+%-*s%s%s\n", l1, "", l2, $4, sgn, $5 + } + $2 != "" && $4 == "" { # only in file 1 + printf "-%-*s %-*s %-*s%s\n", l1, $1, l2, $2, l3, $3, $6 + } + $2 == "" && $4 != "" { # only in file 2 + printf "+%-*s %-*s %-*s%s\n", l1, $1, l2, $4, l3, $5, $7 + } + ' <"$1" +} + +while getopts h OPT +do + case $OPT in + h) usage ;; + *) usage "wrong usage" >&2 ;; + esac +done + +shift $((OPTIND - 1)) + +test $# -lt 2 && usage "not enough arguments" >&2 +test $# -gt 2 && usage "too many arguments" >&2 + +export LC_ALL=C.utf-8 + +join_prepare "$1" >"$TMP"/1 +join_prepare "$2" >"$TMP"/2 +join_render "$TMP"/1 "$TMP"/2 >"$TMP"/join +diff_preamble "$1" "$2" +diff_measure <"$TMP"/join | +diff_render "$TMP"/join From dridi.boukelmoune at gmail.com Tue Aug 30 18:55:07 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 30 Aug 2022 18:55:07 +0000 (UTC) Subject: [master] df761f432 vtc: Remove stale feature check Message-ID: <20220830185507.3192A641C1@lists.varnish-cache.org> commit df761f4328a9783dbeca92e5c6883e3eb8b51ecb Author: Dridi Boukelmoune Date: Tue Aug 30 20:51:51 2022 +0200 vtc: Remove stale feature check I was initially going to align the output only if the nonstandard column(1) command was present. diff --git a/contrib/tests/statdiff_b00000.vtc b/contrib/tests/statdiff_b00000.vtc index c99edfece..e12a5bf52 100644 --- a/contrib/tests/statdiff_b00000.vtc +++ b/contrib/tests/statdiff_b00000.vtc @@ -1,6 +1,5 @@ varnishtest "varnishstatdiff coverage" -feature cmd "command -v column" feature cmd "command -v diff" server s1 { From nils.goroll at uplex.de Wed Aug 31 09:04:07 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 31 Aug 2022 09:04:07 +0000 (UTC) Subject: [master] 30fec48b3 Remove unused macros found by Flexelint Message-ID: <20220831090407.ED73911A3C7@lists.varnish-cache.org> commit 30fec48b3577ff64d1c4ee9519e1e05c122dd8e4 Author: Nils Goroll Date: Wed Aug 31 11:02:52 2022 +0200 Remove unused macros found by Flexelint diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 7a12df235..cbe280d11 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -217,7 +217,6 @@ VRT_l_beresp_##field(VRT_CTX, VCL_BOOL a) \ return (0); \ } while(0) -#define VBERESPR0(field, str, fltchk) #define VBERESPR1(field, str, fltchk) \ VCL_BOOL \ VRT_r_beresp_##field(VRT_CTX) \ @@ -240,7 +239,6 @@ VRT_r_beresp_##field(VRT_CTX) \ #undef VBERESPRF0 #undef VBERESPRF1 -#undef VBERESPR0 #undef VBERESPR1 /*-------------------------------------------------------------------- @@ -257,9 +255,6 @@ VRT_r_bereq_##field(VRT_CTX) \ return (ctx->bo->field); \ } -// w is unused -#define VBEREQW0(ctx, str) (void) 0 - #define BEREQ_FLAG(l, r, w, d) \ VBEREQR##r(l, #l) #include "tbl/bereq_flags.h"