cache_fetch

From phk at FreeBSD.org Mon Jun 3 07:23:11 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 3 Jun 2019 07:23:11 +0000 (UTC) Subject: [master] c3610d36e Monday morning flexelinting Message-ID: <20190603072311.B22E210153A@lists.varnish-cache.org> commit c3610d36ef906c869e8e94f3a54b09d1e859d3b6 Author: Poul-Henning Kamp Date: Mon Jun 3 06:00:18 2019 +0000 Monday morning flexelinting diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c index 2d7f7f03f..4d0453807 100644 --- a/bin/varnishd/cache/cache_ban_lurker.c +++ b/bin/varnishd/cache/cache_ban_lurker.c @@ -105,6 +105,7 @@ ban_cleantail(struct banhead_s *obans) bt = NULL; if (bt != NULL) { + AN(obans); VTAILQ_FOREACH(b, &freelist, list) { if (b != bt) continue; diff --git a/bin/varnishd/mgt/mgt_symtab.c b/bin/varnishd/mgt/mgt_symtab.c index be80ec593..b8ef07d3a 100644 --- a/bin/varnishd/mgt/mgt_symtab.c +++ b/bin/varnishd/mgt/mgt_symtab.c @@ -41,7 +41,6 @@ #include "mgt/mgt_vcl.h" #include "vcli_serve.h" -#include "libvcc.h" #include "vjsn.h" /*--------------------------------------------------------------------*/ From phk at FreeBSD.org Mon Jun 3 07:23:11 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 3 Jun 2019 07:23:11 +0000 (UTC) Subject: [master] a80f4ef9a Unify bespoke error messages to "Symbol not found" before things get too confusing. Message-ID: <20190603072311.D22F910153D@lists.varnish-cache.org> commit a80f4ef9a1763f78eea6acf106e03cb12bdf89b1 Author: Poul-Henning Kamp Date: Mon Jun 3 06:36:20 2019 +0000 Unify bespoke error messages to "Symbol not found" before things get too confusing. diff --git a/bin/varnishtest/tests/r01837.vtc b/bin/varnishtest/tests/r01837.vtc index fbfd797c0..e96a44cf7 100644 --- a/bin/varnishtest/tests/r01837.vtc +++ b/bin/varnishtest/tests/r01837.vtc @@ -1,6 +1,6 @@ varnishtest "Test VCC errors out if probe is used before it is defined" -varnish v1 -errvcl "Probe not found: 'p'" { +varnish v1 -errvcl "Symbol not found: 'p'" { backend b { .host = "127.0.0.1"; .probe = p; diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc index d0a21eb1d..fd519c981 100644 --- a/bin/varnishtest/tests/v00018.vtc +++ b/bin/varnishtest/tests/v00018.vtc @@ -133,7 +133,7 @@ varnish v1 -errvcl {Symbol 'vcl_recv' has wrong type (sub):} { } } -varnish v1 -syntax 4.0 -errvcl {Not a VCL label:} { +varnish v1 -syntax 4.0 -errvcl {Symbol not found:} { sub vcl_recv { return (vcl(vcl_recv)); } diff --git a/bin/varnishtest/tests/v00019.vtc b/bin/varnishtest/tests/v00019.vtc index f579b024a..d81455a15 100644 --- a/bin/varnishtest/tests/v00019.vtc +++ b/bin/varnishtest/tests/v00019.vtc @@ -64,7 +64,7 @@ varnish v1 -errvcl {Comparison of different types: STRING '==' INT} { } } -varnish v1 -errvcl {Unknown variable: 'req.http.req.http.foo'} { +varnish v1 -errvcl {Symbol not found: 'req.http.req.http.foo'} { backend b { .host = "127.0.0.1"; } sub vcl_recv { set req.http.req.http.foo = "bar"; diff --git a/bin/varnishtest/tests/v00021.vtc b/bin/varnishtest/tests/v00021.vtc index 44c4c0f3d..80ef0a93e 100644 --- a/bin/varnishtest/tests/v00021.vtc +++ b/bin/varnishtest/tests/v00021.vtc @@ -91,7 +91,7 @@ varnish v1 -vcl { } } -varnish v1 -errvcl {Unknown variable: 'req.foobar'} { +varnish v1 -errvcl {Symbol not found: 'req.foobar'} { backend foo { .host = "127.0.0.1"; } sub vcl_recv { set req.foobar = 3; } } diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c index 4e547a2ab..1e218cb4f 100644 --- a/lib/libvcc/vcc_action.c +++ b/lib/libvcc/vcc_action.c @@ -94,7 +94,7 @@ vcc_act_set(struct vcc *tl, struct token *t, struct symbol *sym) (void)t; ExpectErr(tl, ID); t = tl->t; - sym = VCC_SymbolGet(tl, SYM_VAR, "Unknown variable", XREF_NONE); + sym = VCC_SymbolGet(tl, SYM_VAR, SYMTAB_EXISTING, XREF_NONE); ERRCHK(tl); AN(sym); if (sym->w_methods == 0) { @@ -146,7 +146,7 @@ vcc_act_unset(struct vcc *tl, struct token *t, struct symbol *sym) /* XXX: Wrong, should use VCC_Expr(HEADER) */ ExpectErr(tl, ID); t = tl->t; - sym = VCC_SymbolGet(tl, SYM_VAR, "Unknown variable", XREF_NONE); + sym = VCC_SymbolGet(tl, SYM_VAR, SYMTAB_EXISTING, XREF_NONE); ERRCHK(tl); AN(sym); if (sym->u_methods == 0) { @@ -269,7 +269,7 @@ vcc_act_return_vcl(struct vcc *tl) ExpectErr(tl, '('); vcc_NextToken(tl); ExpectErr(tl, ID); - sym = VCC_SymbolGet(tl, SYM_VCL, "Not a VCL label", XREF_NONE); + sym = VCC_SymbolGet(tl, SYM_VCL, SYMTAB_EXISTING, XREF_NONE); ERRCHK(tl); AN(sym); if (sym->eval_priv == NULL) { diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c index 4243b9da6..3200b1f84 100644 --- a/lib/libvcc/vcc_backend.c +++ b/lib/libvcc/vcc_backend.c @@ -432,7 +432,7 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname) (void)vcc_default_probe(tl); } else { pb = VCC_SymbolGet(tl, SYM_PROBE, - "Probe not found", XREF_REF); + SYMTAB_EXISTING, XREF_REF); ERRCHK(tl); AN(pb); Fb(tl, 0, "\t.probe = %s,\n", pb->rname); @@ -540,6 +540,7 @@ vcc_ParseBackend(struct vcc *tl) } else { sym = VCC_HandleSymbol(tl, BACKEND, "vgc_backend"); ERRCHK(tl); + AN(sym); dn = sym->rname; if (tl->default_director == NULL) { tl->first_director = sym; diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 3a2116a9f..4d0fb0574 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -350,13 +350,19 @@ void vcc_stevedore(struct vcc *vcc, const char *stv_name); /* vcc_symb.c */ void VCC_PrintCName(struct vsb *vsb, const char *b, const char *e); struct symbol *VCC_MkSym(struct vcc *tl, const char *b, vcc_kind_t, int, int); -extern const char XREF_NONE[]; -extern const char XREF_DEF[]; -extern const char XREF_REF[]; -extern const char SYMTAB_NOERR[]; -extern const char SYMTAB_CREATE[]; -struct symbol *VCC_SymbolGet(struct vcc *, vcc_kind_t, const char *, - const char *); + +struct symxref { const char *name; }; +extern const struct symxref XREF_NONE[1]; +extern const struct symxref XREF_DEF[1]; +extern const struct symxref XREF_REF[1]; + +struct symmode { const char *name; }; +extern const struct symmode SYMTAB_NOERR[1]; +extern const struct symmode SYMTAB_CREATE[1]; +extern const struct symmode SYMTAB_EXISTING[1]; + +struct symbol *VCC_SymbolGet(struct vcc *, vcc_kind_t, + const struct symmode *, const struct symxref *); typedef void symwalk_f(struct vcc *tl, const struct symbol *s); void VCC_WalkSymbols(struct vcc *tl, symwalk_f *func, vcc_kind_t); diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index dae6fe2f2..9c263e0b7 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -688,8 +688,7 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt) switch (tl->t->tok) { case ID: t = tl->t; - sym = VCC_SymbolGet(tl, SYM_NONE, "Symbol not found", - XREF_REF); + sym = VCC_SymbolGet(tl, SYM_NONE, SYMTAB_EXISTING, XREF_REF); ERRCHK(tl); AN(sym); if (sym->kind == SYM_FUNC && sym->type == VOID) { diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index 9883dede4..18bfbe6b3 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -226,19 +226,25 @@ vcc_sym_in_tab(struct vcc *tl, struct symtab *st, } -const char XREF_NONE[] = "xref_none"; -const char XREF_DEF[] = "xref_def"; -const char XREF_REF[] = "xref_ref"; -const char SYMTAB_NOERR[] = "sym_noerror"; -const char SYMTAB_CREATE[] = "sym_create"; +const struct symxref XREF_NONE[1] = {{"xref_none"}}; +const struct symxref XREF_DEF[1] = {{"xref_def"}}; +const struct symxref XREF_REF[1] = {{"xref_ref"}}; +const struct symmode SYMTAB_NOERR[1] = {{"sym_noerror"}}; +const struct symmode SYMTAB_CREATE[1] = {{"sym_create"}}; +const struct symmode SYMTAB_EXISTING[1] = {{"Symbol not found"}}; struct symbol * -VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x) +VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, + const struct symmode *e, const struct symxref *x) { struct symtab *st; struct symbol *sym; struct token *t0, *tn, *tn1; + AN(tl); + AN(e); + AN(x); + AN(x->name); if (tl->syntax >= VCL_41 && e == SYMTAB_CREATE && kind != SYM_SUB && (tl->t->b[0] == 'v'|| tl->t->b[0] == 'V') && (tl->t->b[1] == 'c'|| tl->t->b[1] == 'C') && @@ -270,7 +276,7 @@ VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x) return (sym); tl->t = VTAILQ_NEXT(tn, list); if (sym == NULL) { - VSB_printf(tl->sb, "%s: '", e); + VSB_printf(tl->sb, "%s: '", e->name); for (tn1 = t0; tn1 != tl->t; tn1 = VTAILQ_NEXT(tn1, list)) VSB_printf(tl->sb, "%.*s", PF(tn1)); VSB_printf(tl->sb, "'"); diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index c98e261f0..1a5c13453 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -402,7 +402,7 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym) ExpectErr(tl, ID); t = tl->t; - sy2 = VCC_SymbolGet(tl, SYM_OBJECT, "Symbol not found", XREF_NONE); + sy2 = VCC_SymbolGet(tl, SYM_OBJECT, SYMTAB_EXISTING, XREF_NONE); ERRCHK(tl); AN(sy2); if (sy2->eval_priv == NULL) { From phk at FreeBSD.org Mon Jun 3 07:23:11 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 3 Jun 2019 07:23:11 +0000 (UTC) Subject: [master] ddb1358cb Introduce partial symbol lookups (ie: finding foo.bar in foo.bar.barf) Message-ID: <20190603072311.EAFCE101540@lists.varnish-cache.org> commit ddb1358cb55357700f55cb61cb961d32267b8b12 Author: Poul-Henning Kamp Date: Mon Jun 3 06:47:44 2019 +0000 Introduce partial symbol lookups (ie: finding foo.bar in foo.bar.barf) diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 4d0fb0574..74871e421 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -360,6 +360,7 @@ struct symmode { const char *name; }; extern const struct symmode SYMTAB_NOERR[1]; extern const struct symmode SYMTAB_CREATE[1]; extern const struct symmode SYMTAB_EXISTING[1]; +extern const struct symmode SYMTAB_PARTIAL[1]; struct symbol *VCC_SymbolGet(struct vcc *, vcc_kind_t, const struct symmode *, const struct symxref *); diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index 18bfbe6b3..d33a2a564 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -232,14 +232,15 @@ const struct symxref XREF_REF[1] = {{"xref_ref"}}; const struct symmode SYMTAB_NOERR[1] = {{"sym_noerror"}}; const struct symmode SYMTAB_CREATE[1] = {{"sym_create"}}; const struct symmode SYMTAB_EXISTING[1] = {{"Symbol not found"}}; +const struct symmode SYMTAB_PARTIAL[1] = {{"Symbol not found"}}; struct symbol * VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const struct symmode *e, const struct symxref *x) { - struct symtab *st; - struct symbol *sym; - struct token *t0, *tn, *tn1; + struct symtab *st, *st2; + struct symbol *sym = NULL, *sym2 = NULL; + struct token *t0, *tn, *tn1, *tn2 = NULL; AN(tl); AN(e); @@ -261,6 +262,12 @@ VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, tn = tl->t; while (1) { st = vcc_symtab_str(st, tn->b, tn->e); + sym2 = vcc_sym_in_tab(tl, st, kind, tl->syntax, tl->syntax); + if (sym2 != NULL) { + sym = sym2; + st2 = st; + tn2 = tn; + } tn1 = VTAILQ_NEXT(tn, list); if (tn1->tok != '.') break; @@ -269,11 +276,18 @@ VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, break; tn = tn1; } - sym = vcc_sym_in_tab(tl, st, kind, tl->syntax, tl->syntax); - if (sym == NULL && e == SYMTAB_CREATE) - sym = vcc_new_symbol(tl, st, kind, tl->syntax, tl->syntax); + if (sym != NULL && e == SYMTAB_PARTIAL) { + st = st2; + tn = tn2; + } else if (st != st2) { + sym = NULL; + } if (sym == NULL && e == SYMTAB_NOERR) return (sym); + AN(st); + AN(tn); + if (sym == NULL && e == SYMTAB_CREATE) + sym = vcc_new_symbol(tl, st, kind, tl->syntax, tl->syntax); tl->t = VTAILQ_NEXT(tn, list); if (sym == NULL) { VSB_printf(tl->sb, "%s: '", e->name); From phk at FreeBSD.org Mon Jun 3 07:23:12 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 3 Jun 2019 07:23:12 +0000 (UTC) Subject: [master] 513bd412f Insert a no-op level in expression parser to make diffs more readable Message-ID: <20190603072312.116ED101545@lists.varnish-cache.org> commit 513bd412f05904720579dccf3082ce8cf3f308d5 Author: Poul-Henning Kamp Date: Mon Jun 3 06:48:20 2019 +0000 Insert a no-op level in expression parser to make diffs more readable diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 9c263e0b7..49970f2bc 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -655,7 +655,7 @@ vcc_Eval_SymFunc(struct vcc *tl, struct expr **e, struct token *t, /*-------------------------------------------------------------------- * SYNTAX: - * Expr4: + * Expr5: * '(' ExprCor ')' * symbol * CNUM @@ -664,7 +664,7 @@ vcc_Eval_SymFunc(struct vcc *tl, struct expr **e, struct token *t, */ static void -vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt) +vcc_expr5(struct vcc *tl, struct expr **e, vcc_type_t fmt) { struct expr *e1, *e2; const char *ip, *sign; @@ -799,6 +799,21 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt) vcc_ErrWhere(tl, tl->t); } +/*-------------------------------------------------------------------- + * SYNTAX: + * Expr4: + * Expr5 + */ + +static void +vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt) +{ + + *e = NULL; + vcc_expr5(tl, e, fmt); + return; +} + /*-------------------------------------------------------------------- * SYNTAX: * ExprMul: From phk at FreeBSD.org Mon Jun 3 07:23:12 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 3 Jun 2019 07:23:12 +0000 (UTC) Subject: [master] 340abd045 Recognize type-based methods and fix error messages accordingly. Message-ID: <20190603072312.2D7C6101548@lists.varnish-cache.org> commit 340abd0456e4d018fd1589f406287057ab6740ff Author: Poul-Henning Kamp Date: Mon Jun 3 07:22:10 2019 +0000 Recognize type-based methods and fix error messages accordingly. No methods are implemented yet. diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc index 0b8b7b474..7da6152ea 100644 --- a/bin/varnishtest/tests/v00016.vtc +++ b/bin/varnishtest/tests/v00016.vtc @@ -41,7 +41,7 @@ varnish v1 -errvcl {Operator > not possible on BACKEND} { sub vcl_recv { if (a > b) { } } } -varnish v1 -errvcl {Symbol not found: 'req.foo'} { +varnish v1 -errvcl {Unknown property 'foo' for type HTTP} { backend b { .host = "127.0.0.1"; } sub vcl_hash { if (req.foo != "bar") { } } } diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc index fd519c981..3ce6bd051 100644 --- a/bin/varnishtest/tests/v00018.vtc +++ b/bin/varnishtest/tests/v00018.vtc @@ -38,7 +38,7 @@ varnish v1 -errvcl {Expected ';' got 'if'} { sub vcl_recv { set req.url = "foo" if "bar"; } } -varnish v1 -errvcl {Symbol not found: 'req.foo'} { +varnish v1 -errvcl {Unknown property 'foo' for type HTTP} { backend b { .host = "127.0.0.1"; } sub vcl_hash { hash_data(req.foo); } } diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 49970f2bc..9517bcca1 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -688,7 +688,7 @@ vcc_expr5(struct vcc *tl, struct expr **e, vcc_type_t fmt) switch (tl->t->tok) { case ID: t = tl->t; - sym = VCC_SymbolGet(tl, SYM_NONE, SYMTAB_EXISTING, XREF_REF); + sym = VCC_SymbolGet(tl, SYM_NONE, SYMTAB_PARTIAL, XREF_REF); ERRCHK(tl); AN(sym); if (sym->kind == SYM_FUNC && sym->type == VOID) { @@ -811,7 +811,16 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt) *e = NULL; vcc_expr5(tl, e, fmt); - return; + while (tl->t->tok == '.') { + vcc_NextToken(tl); + ExpectErr(tl, ID); + VSB_printf(tl->sb, "Unknown property "); + vcc_ErrToken(tl, tl->t); + VSB_printf(tl->sb, + " for type %s\n", (*e)->fmt->name); + vcc_ErrWhere(tl, tl->t); + return; + } } /*-------------------------------------------------------------------- diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index d33a2a564..6da0505e8 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -238,7 +238,7 @@ struct symbol * VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const struct symmode *e, const struct symxref *x) { - struct symtab *st, *st2; + struct symtab *st, *st2 = NULL; struct symbol *sym = NULL, *sym2 = NULL; struct token *t0, *tn, *tn1, *tn2 = NULL; From phk at FreeBSD.org Mon Jun 3 13:01:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 3 Jun 2019 13:01:09 +0000 (UTC) Subject: [master] f8bc0ea37 Add an empty lookup table for type based methods, while I find out how vcc_expr actually works... Message-ID: <20190603130109.7843C107BA9@lists.varnish-cache.org> commit f8bc0ea370fe2fcbb9912531a1dd08f8ff1ad6da Author: Poul-Henning Kamp Date: Mon Jun 3 10:38:30 2019 +0000 Add an empty lookup table for type based methods, while I find out how vcc_expr actually works... diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 9517bcca1..1221c4e9e 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -802,24 +802,50 @@ vcc_expr5(struct vcc *tl, struct expr **e, vcc_type_t fmt) /*-------------------------------------------------------------------- * SYNTAX: * Expr4: - * Expr5 + * Expr5 */ +static const struct vcc_methods { + vcc_type_t type_from; + vcc_type_t type_to; + const char *method; + const char *impl; +} vcc_methods[] = { + //{ BACKEND, BOOL, "healthy", "VRT_Healthy(ctx, \v1, 0)" }, + { NULL, NULL, NULL, NULL}, // XXX: For Flexelint + { NULL, NULL, NULL, NULL}, +}; + static void vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt) { + const struct vcc_methods *vm; *e = NULL; vcc_expr5(tl, e, fmt); + ERRCHK(tl); + AN(*e); while (tl->t->tok == '.') { vcc_NextToken(tl); ExpectErr(tl, ID); - VSB_printf(tl->sb, "Unknown property "); - vcc_ErrToken(tl, tl->t); - VSB_printf(tl->sb, - " for type %s\n", (*e)->fmt->name); - vcc_ErrWhere(tl, tl->t); - return; + + for(vm = vcc_methods; vm->type_from != NULL; vm++) { + + if (vm->type_from == (*e)->fmt && + vcc_IdIs(tl->t, vm->method)) + break; + } + + if (vm->type_from == NULL) { + VSB_printf(tl->sb, "Unknown property "); + vcc_ErrToken(tl, tl->t); + VSB_printf(tl->sb, + " for type %s\n", (*e)->fmt->name); + vcc_ErrWhere(tl, tl->t); + return; + } + vcc_NextToken(tl); + *e = vcc_expr_edit(tl, vm->type_to, vm->impl, *e, NULL); } } From phk at FreeBSD.org Mon Jun 3 13:01:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 3 Jun 2019 13:01:09 +0000 (UTC) Subject: [master] c878d750f Move the layout of compiled ACLs to VPI space Message-ID: <20190603130109.8CE20107BAC@lists.varnish-cache.org> commit c878d750fa31310e71df7b1590e60e9148545ee0 Author: Poul-Henning Kamp Date: Mon Jun 3 12:05:47 2019 +0000 Move the layout of compiled ACLs to VPI space diff --git a/include/vcc_interface.h b/include/vcc_interface.h index 347c20077..10df9d49a 100644 --- a/include/vcc_interface.h +++ b/include/vcc_interface.h @@ -55,3 +55,11 @@ int VPI_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len, const char *nm, const char *path, const char *file_id, const char *backup); void VPI_Vmod_Unload(VRT_CTX, struct vmod **hdl); +typedef int acl_match_f(VRT_CTX, const VCL_IP); + +struct vrt_acl { + unsigned magic; +#define VRT_ACL_MAGIC 0x78329d96 + acl_match_f *match; +}; + diff --git a/include/vrt.h b/include/vrt.h index 7a227ebc9..4cb78e002 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -387,14 +387,6 @@ struct vrt_backend_probe { * Implementation details of ACLs */ -typedef int acl_match_f(VRT_CTX, const VCL_IP); - -struct vrt_acl { - unsigned magic; -#define VRT_ACL_MAGIC 0x78329d96 - acl_match_f *match; -}; - void VRT_acl_log(VRT_CTX, const char *); int VRT_acl_match(VRT_CTX, VCL_ACL, VCL_IP); diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 9e1463101..8ded76669 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -513,7 +513,7 @@ xyzzy_match_acl(VRT_CTX, VCL_ACL acl, VCL_IP ip) { CHECK_OBJ_ORNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_ORNULL(acl, VRT_ACL_MAGIC); + AN(acl); assert(VSA_Sane(ip)); return (VRT_acl_match(ctx, acl, ip)); From phk at FreeBSD.org Mon Jun 3 13:01:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 3 Jun 2019 13:01:09 +0000 (UTC) Subject: [master] c1990ad9f Necessary #include for previous commit. Message-ID: <20190603130109.A6A44107BAF@lists.varnish-cache.org> commit c1990ad9f9bee5a8532ddd88765fb823ebf3f439 Author: Poul-Henning Kamp Date: Mon Jun 3 12:06:18 2019 +0000 Necessary #include for previous commit. diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 10da99437..854a690f2 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -40,6 +40,7 @@ #include "vsa.h" #include "vtcp.h" #include "vtim.h" +#include "vcc_interface.h" #include "common/heritage.h" #include "common/vsmw.h" From phk at FreeBSD.org Mon Jun 3 13:01:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 3 Jun 2019 13:01:09 +0000 (UTC) Subject: [master] 3ea061c99 Teach ACLs their name Message-ID: <20190603130109.BED8A107BB4@lists.varnish-cache.org> commit 3ea061c9955d5a9875eb2686090689ed86820878 Author: Poul-Henning Kamp Date: Mon Jun 3 13:00:14 2019 +0000 Teach ACLs their name diff --git a/include/vcc_interface.h b/include/vcc_interface.h index 10df9d49a..682767265 100644 --- a/include/vcc_interface.h +++ b/include/vcc_interface.h @@ -61,5 +61,6 @@ struct vrt_acl { unsigned magic; #define VRT_ACL_MAGIC 0x78329d96 acl_match_f *match; + const char *name; }; diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index 43a5b43a1..023a73d52 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -454,6 +454,7 @@ vcc_acl_emit(struct vcc *tl, const char *name, const char *rname, int anon) Fh(tl, 0, "\nconst struct vrt_acl %s[] = {{\n", rname); Fh(tl, 0, "\t.magic = VRT_ACL_MAGIC,\n"); Fh(tl, 0, "\t.match = &%s,\n", VSB_data(func)); + Fh(tl, 0, "\t.name = \"%s\",\n", name); Fh(tl, 0, "}};\n\n"); } VSB_destroy(&func); From phk at FreeBSD.org Mon Jun 3 18:08:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 3 Jun 2019 18:08:10 +0000 (UTC) Subject: [master] f72c22352 Do not preemptively convert handles to STRING if we can do so later Message-ID: <20190603180810.A0D6110E9F1@lists.varnish-cache.org> commit f72c22352467512dca79a8f53f484f1f93f21213 Author: Poul-Henning Kamp Date: Mon Jun 3 13:55:37 2019 +0000 Do not preemptively convert handles to STRING if we can do so later diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 1221c4e9e..13931d13c 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -315,7 +315,8 @@ vcc_Eval_Handle(struct vcc *tl, struct expr **e, struct token *t, AN(sym->rname); AZ(type->stringform); - if (sym->type != STRING && type == STRINGS) { + if (sym->type->tostring == NULL && + sym->type != STRING && type == STRINGS) { *e = vcc_mk_expr(STRINGS, "\"%s\"", sym->name); (*e)->nstr = 1; (*e)->constant |= EXPR_CONST | EXPR_STR_CONST; From phk at FreeBSD.org Mon Jun 3 18:08:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 3 Jun 2019 18:08:10 +0000 (UTC) Subject: [master] 4515993f4 Teach ACL's how to find their own name. Message-ID: <20190603180810.BC90D10E9F4@lists.varnish-cache.org> commit 4515993f430c6dd4f7343e8b88ec58771ac7ddee Author: Poul-Henning Kamp Date: Mon Jun 3 13:56:46 2019 +0000 Teach ACL's how to find their own name. diff --git a/lib/libvcc/vcc_types.c b/lib/libvcc/vcc_types.c index 3611dcef5..196efa539 100644 --- a/lib/libvcc/vcc_types.c +++ b/lib/libvcc/vcc_types.c @@ -37,6 +37,7 @@ const struct type ACL[1] = {{ .magic = TYPE_MAGIC, .name = "ACL", + .tostring = "((\v1)->name)", }}; const struct type BACKEND[1] = {{ From phk at FreeBSD.org Tue Jun 4 06:39:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 4 Jun 2019 06:39:10 +0000 (UTC) Subject: [master] 2962375b4 Reimplement the storage.*.* properties as type driven properties Message-ID: <20190604063910.7118A11B89A@lists.varnish-cache.org> commit 2962375b4ed80e2197a318f92f2fb9f6cfcd12f8 Author: Poul-Henning Kamp Date: Tue Jun 4 06:37:28 2019 +0000 Reimplement the storage.*.* properties as type driven properties diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index 49835eba4..0f33bd656 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -41,7 +41,6 @@ #include "storage/storage.h" #include "vrt_obj.h" - static pthread_mutex_t stv_mtx; /*-------------------------------------------------------------------- @@ -214,18 +213,13 @@ VRT_stevedore(const char *nm) return (stv_find(nm)); } -#define VRTSTVVAR(nm, vtype, ctype, dval) \ -ctype \ -VRT_Stv_##nm(const char *nm) \ -{ \ - const struct stevedore *stv; \ - \ - stv = stv_find(nm); \ - if (stv == NULL) \ - return (dval); \ - if (stv->var_##nm == NULL) \ - return (dval); \ - return (stv->var_##nm(stv)); \ +#define VRTSTVVAR(nm, vtype, ctype, dval) \ +ctype \ +VRT_stevedore_##nm(VCL_STEVEDORE stv) \ +{ \ + if (stv == NULL) \ + return (0); \ + CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); \ + return (stv->var_##nm(stv)); \ } - #include "tbl/vrt_stv_var.h" diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 892dfff5b..fb464bc39 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -731,12 +731,11 @@ vcc_Var_Init(struct vcc *tl) struct symbol *sym; """) - parse_var_doc(join(srcroot, "doc/sphinx/reference/vcl_var.rst")) fo.write("}\n") for i in stv_variables: - fh.write(vcltypes[i[1]].c + " VRT_Stv_" + i[0] + "(const char *);\n") + fh.write(vcltypes[i[1]].c + " VRT_stevedore_" + i[0] + "(VCL_STEVEDORE);\n") fo.write("\n/* VCL type identifiers */\n") diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 13931d13c..6301cb722 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -813,7 +813,11 @@ static const struct vcc_methods { const char *impl; } vcc_methods[] = { //{ BACKEND, BOOL, "healthy", "VRT_Healthy(ctx, \v1, 0)" }, - { NULL, NULL, NULL, NULL}, // XXX: For Flexelint + +#define VRTSTVVAR(nm, vtype, ctype, dval) \ + { STEVEDORE, vtype, #nm, "VRT_stevedore_" #nm "(\v1)"}, +#include "tbl/vrt_stv_var.h" + { NULL, NULL, NULL, NULL}, }; diff --git a/lib/libvcc/vcc_storage.c b/lib/libvcc/vcc_storage.c index aa83a32dc..ac4fbeb7d 100644 --- a/lib/libvcc/vcc_storage.c +++ b/lib/libvcc/vcc_storage.c @@ -59,27 +59,11 @@ #include #include "vcc_compile.h" -#include "libvcc.h" // VCC_Stevedore() proto - -/*-------------------------------------------------------------------- - * - */ - -static struct stvars { - const char *name; - vcc_type_t type; -} stvars[] = { -#define VRTSTVVAR(nm, vtype, ctype, dval) { #nm, vtype }, -#include "tbl/vrt_stv_var.h" -#undef VRTSTVVAR - { NULL, BOOL } -}; void vcc_stevedore(struct vcc *vcc, const char *stv_name) { struct symbol *sym; - struct stvars *sv; char buf[1024]; CHECK_OBJ_NOTNULL(vcc, VCC_MAGIC); @@ -91,15 +75,4 @@ vcc_stevedore(struct vcc *vcc, const char *stv_name) bprintf(buf, "VRT_stevedore(\"%s\")", stv_name); sym->rname = TlDup(vcc, buf); sym->r_methods = ~0; - - for (sv = stvars; sv->name != NULL; sv++) { - bprintf(buf, "storage.%s.%s", stv_name, sv->name); - sym = VCC_MkSym(vcc, buf, SYM_VAR, VCL_LOW, VCL_41); - AN(sym); - sym->type = sv->type; - sym->eval = vcc_Eval_Var; - bprintf(buf, "VRT_Stv_%s(\"%s\")", sv->name, stv_name); - sym->rname = TlDup(vcc, buf); - sym->r_methods = ~0; - } } From dridi.boukelmoune at gmail.com Wed Jun 12 04:58:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 12 Jun 2019 04:58:10 +0000 (UTC) Subject: [master] 21efe76d2 Appropriate VSL group checking Message-ID: <20190612045810.BB744AE051@lists.varnish-cache.org> commit 21efe76d23ed81e5423c59055e55c2c5a7f29dbd Author: Dridi Boukelmoune Date: Thu May 16 15:02:54 2019 +0200 Appropriate VSL group checking We have a dedicated enum entry for that, and nothing prevents the caller from passing a negative value. diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c index fb284a58a..b27646113 100644 --- a/lib/libvarnishapi/vsl_dispatch.c +++ b/lib/libvarnishapi/vsl_dispatch.c @@ -1088,7 +1088,7 @@ VSLQ_New(struct VSL_data *vsl, struct VSL_cursor **cp, struct VSLQ *vslq; CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC); - if (grouping > VSL_g_session) { + if (grouping < 0 || grouping >= VSL_g__MAX) { (void)vsl_diag(vsl, "Illegal query grouping"); return (NULL); } From dridi.boukelmoune at gmail.com Wed Jun 12 04:58:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 12 Jun 2019 04:58:10 +0000 (UTC) Subject: [master] 5ce86e76b Align columns Message-ID: <20190612045810.D0E9BAE054@lists.varnish-cache.org> commit 5ce86e76bbf73040ade4cea380f03e2d8ba438bf Author: Dridi Boukelmoune Date: Thu May 16 16:40:00 2019 +0200 Align columns diff --git a/lib/libvarnishapi/generate.py b/lib/libvarnishapi/generate.py index ea4ebd02a..e411957d6 100755 --- a/lib/libvarnishapi/generate.py +++ b/lib/libvarnishapi/generate.py @@ -45,33 +45,33 @@ if len(sys.argv) == 3: tokens = { # Numerical comparisons - "T_EQ": "==", - "T_NEQ": "!=", - "T_LEQ": "<=", - "T_GEQ": ">=", + "T_EQ": "==", + "T_NEQ": "!=", + "T_LEQ": "<=", + "T_GEQ": ">=", # String comparisons - "T_SEQ": "eq", + "T_SEQ": "eq", "T_SNEQ": "ne", # Regular expression matching "T_NOMATCH": "!~", # Boolean operators - "T_AND": "and", - "T_OR": "or", - "T_NOT": "not", + "T_AND": "and", + "T_OR": "or", + "T_NOT": "not", # Miscellaneous - None: "<>~[]{}():,", + None: "<>~[]{}():,", # These have handwritten recognizers - "VAL": None, - "EOI": None, + "VAL": None, + "EOI": None, # Special "T_TRUE": None, - "VXID": "vxid", + "VXID": "vxid", } ####################################################################### From dridi.boukelmoune at gmail.com Wed Jun 12 04:58:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 12 Jun 2019 04:58:10 +0000 (UTC) Subject: [master] 406ec859a Turn newline into the end of a VSL query Message-ID: <20190612045810.ED56FAE058@lists.varnish-cache.org> commit 406ec859a6275086166a8f4dd4ad0028a062f2ca Author: Dridi Boukelmoune Date: Thu May 16 16:40:47 2019 +0200 Turn newline into the end of a VSL query Instead of treating it as a mere whitespace separator. As a result the lexer output looks like this: TOK_1, ..., TOK_N, EOI[, TOK_1, ..., TOK_N, EOI[...]] At this point though, only the first series of tokens is considered, in a single query mode that was already what we've had so far. It will also be the parser's job to ignore consecutive EOI tokens for "empty" queries. diff --git a/lib/libvarnishapi/vxp_lexer.c b/lib/libvarnishapi/vxp_lexer.c index b004c55f8..ff5221e41 100644 --- a/lib/libvarnishapi/vxp_lexer.c +++ b/lib/libvarnishapi/vxp_lexer.c @@ -103,8 +103,8 @@ vxp_Lexer(struct vxp *vxp) for (p = vxp->b; p < vxp->e; ) { - /* Skip any whitespace */ - if (isspace(*p)) { + /* Skip any space or tab */ + if (isblank(*p)) { p++; continue; } @@ -126,6 +126,8 @@ vxp_Lexer(struct vxp *vxp) q++; if (q == vxp->e) break; + } else if (*q == '\n') { + break; } else if (*q == quote) { q++; quote = '\0'; @@ -157,6 +159,13 @@ vxp_Lexer(struct vxp *vxp) continue; } + /* On to the next query */ + if (*p == '\n') { + vxp_add_token(vxp, EOI, p, p + 1); + p++; + continue; + } + /* Error */ vxp_add_token(vxp, EOI, p, p + 1); VSB_printf(vxp->sb, "Syntax error "); From dridi.boukelmoune at gmail.com Wed Jun 12 04:58:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 12 Jun 2019 04:58:11 +0000 (UTC) Subject: [master] 94ca452d7 Allow comments in a VSL query Message-ID: <20190612045811.154C1AE05D@lists.varnish-cache.org> commit 94ca452d73cd09b4e8fb858797224281c12c1978 Author: Dridi Boukelmoune Date: Fri May 17 09:16:31 2019 +0200 Allow comments in a VSL query Now that we can have multiple lines in a query, it may come in handy to add a description of a query or its purpose. diff --git a/lib/libvarnishapi/vxp_lexer.c b/lib/libvarnishapi/vxp_lexer.c index ff5221e41..e6ce575aa 100644 --- a/lib/libvarnishapi/vxp_lexer.c +++ b/lib/libvarnishapi/vxp_lexer.c @@ -109,6 +109,13 @@ vxp_Lexer(struct vxp *vxp) continue; } + /* Skip comments */ + if (*p == '#') { + while (p < vxp->e && *p != '\n') + p++; + continue; + } + /* Match for the fixed tokens */ u = vxp_fixed_token(p, &q); if (u != 0) { From dridi.boukelmoune at gmail.com Wed Jun 12 04:58:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 12 Jun 2019 04:58:11 +0000 (UTC) Subject: [master] a3ab11ec7 Rename vxp_add_token to vxp_append_token Message-ID: <20190612045811.34983AE060@lists.varnish-cache.org> commit a3ab11ec72ce8bef27c5fa0405585b4f353854ce Author: Dridi Boukelmoune Date: Fri May 17 09:24:38 2019 +0200 Rename vxp_add_token to vxp_append_token We always add tokens in order, we don't need to check whether there was one present in the first place. diff --git a/lib/libvarnishapi/vxp_lexer.c b/lib/libvarnishapi/vxp_lexer.c index e6ce575aa..57f0c971f 100644 --- a/lib/libvarnishapi/vxp_lexer.c +++ b/lib/libvarnishapi/vxp_lexer.c @@ -45,7 +45,7 @@ #include "vxp.h" static void -vxp_add_token(struct vxp *vxp, unsigned tok, const char *b, const char *e) +vxp_append_token(struct vxp *vxp, unsigned tok, const char *b, const char *e) { struct token *t; @@ -54,10 +54,7 @@ vxp_add_token(struct vxp *vxp, unsigned tok, const char *b, const char *e) t->tok = tok; t->b = b; t->e = e; - if (vxp->t != NULL) - VTAILQ_INSERT_AFTER(&vxp->tokens, vxp->t, t, list); - else - VTAILQ_INSERT_TAIL(&vxp->tokens, t, list); + VTAILQ_INSERT_TAIL(&vxp->tokens, t, list); vxp->t = t; } @@ -120,7 +117,7 @@ vxp_Lexer(struct vxp *vxp) u = vxp_fixed_token(p, &q); if (u != 0) { AN(q); - vxp_add_token(vxp, u, p, q); + vxp_append_token(vxp, u, p, q); p = q; continue; } @@ -141,7 +138,7 @@ vxp_Lexer(struct vxp *vxp) break; } } - vxp_add_token(vxp, VAL, p, q); + vxp_append_token(vxp, VAL, p, q); if (quote != '\0') { VSB_printf(vxp->sb, "Unterminated string "); vxp_ErrWhere(vxp, vxp->t, q - p); @@ -157,7 +154,7 @@ vxp_Lexer(struct vxp *vxp) for (q = p; q < vxp->e; q++) if (!isword(*q)) break; - vxp_add_token(vxp, VAL, p, q); + vxp_append_token(vxp, VAL, p, q); vxp->t->dec = vxp_Alloc(vxp, (q - p) + 1); AN(vxp->t->dec); memcpy(vxp->t->dec, p, q - p); @@ -168,20 +165,20 @@ vxp_Lexer(struct vxp *vxp) /* On to the next query */ if (*p == '\n') { - vxp_add_token(vxp, EOI, p, p + 1); + vxp_append_token(vxp, EOI, p, p + 1); p++; continue; } /* Error */ - vxp_add_token(vxp, EOI, p, p + 1); + vxp_append_token(vxp, EOI, p, p + 1); VSB_printf(vxp->sb, "Syntax error "); vxp_ErrWhere(vxp, vxp->t, q - p); return; } /* Finished */ - vxp_add_token(vxp, EOI, vxp->e, vxp->e); + vxp_append_token(vxp, EOI, vxp->e, vxp->e); } #ifdef VXP_DEBUG From dridi.boukelmoune at gmail.com Wed Jun 12 04:58:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 12 Jun 2019 04:58:11 +0000 (UTC) Subject: [master] c061b8eb3 Treat multiline VSL queries as if they were OR'ed Message-ID: <20190612045811.58B88AE065@lists.varnish-cache.org> commit c061b8eb3f70017d343dd680a9886188b22890b9 Author: Dridi Boukelmoune Date: Fri May 17 10:06:05 2019 +0200 Treat multiline VSL queries as if they were OR'ed The two following queries now become equivalent: vut -g request -q ' # catch varnish errors *Error # catch backend errors BerespStatus >= 500 ' vut -g request -q '(*Error) or (BerespStatus >= 500)' It becomes interesting when we wish to capture transactions for different scenarios but would like to decompose them cleanly. Especially when the query of an individual scenario is rather complex and OR'ing everything manually would become cumbersome. This diff is better viewed with the --ignore-all-space flag. diff --git a/bin/varnishtest/tests/u00014.vtc b/bin/varnishtest/tests/u00014.vtc new file mode 100644 index 000000000..648bc45e6 --- /dev/null +++ b/bin/varnishtest/tests/u00014.vtc @@ -0,0 +1,91 @@ +varnishtest "VSL compound queries" + +# This test case compares individual query scenarios to their +# compounded queries counterparts. + +server s1 { + rxreq + txresp -status 500 + + rxreq + txresp -status 503 +} -start + +varnish v1 -vcl+backend "" -start + +client c1 { + txreq + rxresp + expect resp.status == 500 + + txreq + rxresp + expect resp.status == 503 +} -run + +# Let's fist create a script to reduce in all the variants below. + +shell { + cat >ncsa.sh <<-EOF + #!/bin/sh + + varnishncsa -d -n ${v1_name} -F '%s' "\$@" | + tr '\n' ' ' + # ' + EOF + chmod +x ncsa.sh +} + +shell -match "^500 503 $" { + # no query + ./ncsa.sh +} + +shell -err -expect "Query expression error" { + # empty query + varnishncsa -d -n ${v1_name} -q '' +} + +shell -err -expect "Query expression error" { + varnishncsa -d -n ${v1_name} -q ' + # empty multiline query + ' +} + +shell -err -expect "Query expression error" { + varnishncsa -d -n ${v1_name} -q ' + * ~ "Incomplete quoted string + ' +} + +shell -match "^500 $" { + # single query 1 + ./ncsa.sh -q 'RespStatus == 500' +} + +shell -match "^503 $" { + # single query 2 + ./ncsa.sh -q 'RespStatus == 503' +} + +shell -match "^500 503 $" { + # query 1 OR query 2 + ./ncsa.sh -q '(RespStatus == 500) or (RespStatus == 503)' +} + +shell -match "^500 503 $" { + ./ncsa.sh -q ' + # query 1 + RespStatus == 500 + + # query 2 + RespStatus == 503 + ' +} + +shell -match "^500 503 $" { + ./ncsa.sh -q ' + RespStatus == 500 # query 1 + RespStatus == 503 # query 2 + ' +} diff --git a/doc/sphinx/reference/vsl-query.rst b/doc/sphinx/reference/vsl-query.rst index 92b97c9e7..67cf2f02d 100644 --- a/doc/sphinx/reference/vsl-query.rst +++ b/doc/sphinx/reference/vsl-query.rst @@ -131,6 +131,26 @@ whose id can be obtained from an ``X-Varnish`` HTTP header, the default "guru meditation" error page, or ``Begin`` and ``Link`` log records. +A query must fit on a single line, but it is possible to pass multiple +queries at once, one query per line. Empty lines are ignored, and the +list of queries is treated as if the 'or' operator was used to combine +them. + +For example this list of queries:: + + # catch varnish errors + *Error + + # catch backend errors + BerespStatus >= 500 + +is identical to this query:: + + (*Error) or (BerespStatus >= 500) + +Comments can be used and will be ignored, they start with the ``'#'`` +character. + Record selection criteria ------------------------- diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c index e6db240c5..16f65f42f 100644 --- a/lib/libvarnishapi/vxp_parse.c +++ b/lib/libvarnishapi/vxp_parse.c @@ -503,15 +503,30 @@ vxp_expr_or(struct vxp *vxp, struct vex **pvex) /* * SYNTAX: * expr: - * expr_or EOI + * expr_or EOI { 'or' expr_or EOI }? */ static void vxp_expr(struct vxp *vxp, struct vex **pvex) { - vxp_expr_or(vxp, pvex); + struct vex *a = NULL, *or; + + if (*pvex == NULL) { + vxp_expr_or(vxp, pvex); + ERRCHK(vxp); + ExpectErr(vxp, EOI); + return; + } + + vxp_expr(vxp, &a); ERRCHK(vxp); - ExpectErr(vxp, EOI); + + or = vex_alloc(vxp); + AN(or); + or->tok = T_OR; + or->b = *pvex; + or->a = a; + *pvex = or; } /* @@ -523,17 +538,27 @@ vxp_Parse(struct vxp *vxp) { struct vex *vex = NULL; + AZ(vxp->err); vxp->t = VTAILQ_FIRST(&vxp->tokens); - if (vxp->t == NULL) - return (NULL); - vxp_expr(vxp, &vex); + while (vxp->t != NULL) { + /* Ignore empty queries */ + while (vxp->t != NULL && vxp->t->tok == EOI) + vxp->t = VTAILQ_NEXT(vxp->t, list); + + if (vxp->t == NULL) + break; + + vxp_expr(vxp, &vex); + + if (vxp->err) { + if (vex) + vex_Free(&vex); + AZ(vex); + return (NULL); + } - if (vxp->err) { - if (vex) - vex_Free(&vex); - AZ(vex); - return (NULL); + vxp->t = VTAILQ_NEXT(vxp->t, list); } return (vex); From dridi.boukelmoune at gmail.com Wed Jun 12 04:58:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 12 Jun 2019 04:58:11 +0000 (UTC) Subject: [master] b7de3a9da Allow multiple -q options for VUTs Message-ID: <20190612045811.78B87AE069@lists.varnish-cache.org> commit b7de3a9da43bb45bb93c71a02b912ef627bfd7fa Author: Dridi Boukelmoune Date: Fri May 17 12:05:56 2019 +0200 Allow multiple -q options for VUTs This makes the following queries equivalent: vut -g request -q '*Error' -q 'BerespStatus >= 500' vut -g request -q ' *Error BerespStatus >= 500 ' vut -g request -q '(*Error) or (BerespStatus >= 500)' We now have two ways to express "compound" VSL queries, although this ultimately relies on multiline queries. diff --git a/bin/varnishtest/tests/u00014.vtc b/bin/varnishtest/tests/u00014.vtc index 648bc45e6..244dcc08f 100644 --- a/bin/varnishtest/tests/u00014.vtc +++ b/bin/varnishtest/tests/u00014.vtc @@ -89,3 +89,8 @@ shell -match "^500 503 $" { RespStatus == 503 # query 2 ' } + +shell -match "^500 503 $" { + # multiple -q options + ./ncsa.sh -q 'RespStatus == 500' -q 'RespStatus == 503' +} diff --git a/include/vut_options.h b/include/vut_options.h index d6941e1fe..f2c47deb1 100644 --- a/include/vut_options.h +++ b/include/vut_options.h @@ -75,7 +75,9 @@ #define VUT_OPT_q \ VOPT("q:", "[-q ]", "VSL query", \ - "Specifies the VSL query to use." \ + "Specifies the VSL query to use. When multiple -q" \ + " options are specified, all queries are considered" \ + " as if the 'or' operator was used to combine them." \ ) #define VUT_OPT_r \ diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c index 91be51925..bb46128a9 100644 --- a/lib/libvarnishapi/vut.c +++ b/lib/libvarnishapi/vut.c @@ -50,6 +50,7 @@ #include "vas.h" #include "miniobj.h" #include "vcs.h" +#include "vsb.h" #include "vut.h" @@ -126,6 +127,30 @@ VUT_Error(struct VUT *vut, int status, const char *fmt, ...) exit(status); } +static void +vut_arg_q(struct VUT *vut, const char *arg) +{ + struct vsb *vsb; + char *s; + + AN(arg); + if (vut->q_arg == NULL) { + REPLACE(vut->q_arg, arg); + return; + } + + vsb = VSB_new_auto(); + AN(vsb); + AZ(VSB_printf(vsb, "%s\n%s", vut->q_arg, arg)); + AZ(VSB_finish(vsb)); + + s = strdup(VSB_data(vsb)); + REPLACE(vut->q_arg, s); + + VSB_clear(vsb); + VSB_destroy(&vsb); +} + int VUT_Arg(struct VUT *vut, int opt, const char *arg) { @@ -169,8 +194,7 @@ VUT_Arg(struct VUT *vut, int opt, const char *arg) return (1); case 'q': /* Query to use */ - AN(arg); - REPLACE(vut->q_arg, arg); + vut_arg_q(vut, arg); return (1); case 'r': /* Binary file input */ From dridi.boukelmoune at gmail.com Wed Jun 12 04:58:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 12 Jun 2019 04:58:11 +0000 (UTC) Subject: [master] 69976ebfa Assert Message-ID: <20190612045811.9BC89AE06E@lists.varnish-cache.org> commit 69976ebfa0e32abc1ff430d0debcc999bcdd27e4 Author: Dridi Boukelmoune Date: Fri May 17 13:37:42 2019 +0200 Assert diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c index bb46128a9..0fdce6634 100644 --- a/lib/libvarnishapi/vut.c +++ b/lib/libvarnishapi/vut.c @@ -157,6 +157,9 @@ VUT_Arg(struct VUT *vut, int opt, const char *arg) int i; char *p; + CHECK_OBJ_NOTNULL(vut, VUT_MAGIC); + AN(opt); + switch (opt) { case 'd': /* Head */ From dridi.boukelmoune at gmail.com Wed Jun 12 04:58:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 12 Jun 2019 04:58:11 +0000 (UTC) Subject: [master] d704203e1 Whitespace OCD Message-ID: <20190612045811.BE0D8AE072@lists.varnish-cache.org> commit d704203e121eb4cf52bd9829fc0e3b96a28d0c2f Author: Dridi Boukelmoune Date: Fri May 17 13:38:19 2019 +0200 Whitespace OCD diff --git a/include/vut_options.h b/include/vut_options.h index f2c47deb1..6b4675c18 100644 --- a/include/vut_options.h +++ b/include/vut_options.h @@ -36,7 +36,7 @@ #define VUT_GLOBAL_OPT_P \ VOPT("P:", "[-P ]", "PID file", \ - "Write the process' PID to the specified file." \ + "Write the process' PID to the specified file." \ ) #define VUT_GLOBAL_OPT_V \ From dridi.boukelmoune at gmail.com Wed Jun 12 04:58:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 12 Jun 2019 04:58:11 +0000 (UTC) Subject: [master] 8fd8366fd Add VUT_OPT_Q to get a VSL query from a file Message-ID: <20190612045812.06B9BAE079@lists.varnish-cache.org> commit 8fd8366fd55a7818fd66bff09bca3340de2b28c4 Author: Dridi Boukelmoune Date: Fri May 17 13:55:43 2019 +0200 Add VUT_OPT_Q to get a VSL query from a file diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h index 1bf1de7c7..44d710da4 100644 --- a/bin/varnishhist/varnishhist_options.h +++ b/bin/varnishhist/varnishhist_options.h @@ -91,6 +91,7 @@ HIS_OPT_p #undef HIS_CLIENT #undef HIS_PROF HIS_OPT_P +VUT_OPT_Q VUT_OPT_q VUT_OPT_r VUT_OPT_t diff --git a/bin/varnishlog/varnishlog_options.h b/bin/varnishlog/varnishlog_options.h index ea0b4c698..8c9fd1db7 100644 --- a/bin/varnishlog/varnishlog_options.h +++ b/bin/varnishlog/varnishlog_options.h @@ -68,6 +68,7 @@ VUT_OPT_k VSL_OPT_L VUT_OPT_n VUT_GLOBAL_OPT_P +VUT_OPT_Q VUT_OPT_q VUT_OPT_r VSL_OPT_R diff --git a/bin/varnishncsa/varnishncsa_options.h b/bin/varnishncsa/varnishncsa_options.h index 73765083f..8987a9615 100644 --- a/bin/varnishncsa/varnishncsa_options.h +++ b/bin/varnishncsa/varnishncsa_options.h @@ -86,6 +86,7 @@ VUT_OPT_h VSL_OPT_L VUT_OPT_n VUT_GLOBAL_OPT_P +VUT_OPT_Q VUT_OPT_q VUT_OPT_r VSL_OPT_R diff --git a/bin/varnishtest/tests/u00014.vtc b/bin/varnishtest/tests/u00014.vtc index 244dcc08f..dc3138ae3 100644 --- a/bin/varnishtest/tests/u00014.vtc +++ b/bin/varnishtest/tests/u00014.vtc @@ -58,6 +58,15 @@ shell -err -expect "Query expression error" { ' } +shell -err -expect "Query expression error" { + echo '# empty query file' >empty.vslq + varnishncsa -d -n ${v1_name} -Q empty.vslq +} + +shell -err -expect "-Q missing.vslq: No such file or directory" { + varnishncsa -d -n ${v1_name} -Q missing.vslq +} + shell -match "^500 $" { # single query 1 ./ncsa.sh -q 'RespStatus == 500' @@ -90,7 +99,40 @@ shell -match "^500 503 $" { ' } +shell -match "^500 503 $" { + cat >query.vslq <<-EOF + # query 1 + RespStatus == 500 + + # query 2 + RespStatus == 503 + EOF + ./ncsa.sh -Q query.vslq +} + shell -match "^500 503 $" { # multiple -q options ./ncsa.sh -q 'RespStatus == 500' -q 'RespStatus == 503' } + +shell -match "^500 503 $" { + # multiple -Q options + + cat >query1.vslq <<-EOF + # query 1 + RespStatus == 500 + EOF + + cat >query2.vslq <<-EOF + # query 2 + RespStatus == 503 + EOF + + ./ncsa.sh -Q query1.vslq -Q query2.vslq +} + +shell -match "^500 503 $" { + # mix -Q and -q options + + ./ncsa.sh -Q query1.vslq -q 'RespStatus == 503' +} diff --git a/bin/varnishtop/varnishtop_options.h b/bin/varnishtop/varnishtop_options.h index 6605d463a..2a6c312e7 100644 --- a/bin/varnishtop/varnishtop_options.h +++ b/bin/varnishtop/varnishtop_options.h @@ -73,6 +73,7 @@ VSL_OPT_I VSL_OPT_L VUT_OPT_n TOP_OPT_p +VUT_OPT_Q VUT_OPT_q VUT_OPT_r VUT_OPT_t diff --git a/doc/sphinx/reference/vsl-query.rst b/doc/sphinx/reference/vsl-query.rst index 67cf2f02d..072747f4c 100644 --- a/doc/sphinx/reference/vsl-query.rst +++ b/doc/sphinx/reference/vsl-query.rst @@ -149,7 +149,7 @@ is identical to this query:: (*Error) or (BerespStatus >= 500) Comments can be used and will be ignored, they start with the ``'#'`` -character. +character, which may be more useful when the query is read from a file. Record selection criteria ------------------------- diff --git a/include/vut_options.h b/include/vut_options.h index 6b4675c18..7aacaaaec 100644 --- a/include/vut_options.h +++ b/include/vut_options.h @@ -73,10 +73,18 @@ " the host name is used." \ ) +#define VUT_OPT_Q \ + VOPT("Q:", "[-Q ]", "VSL query file", \ + "Specifies the file containing the VSL query to use." \ + " When multiple -Q or -q options are specified, all" \ + " queries are considered as if the 'or' operator was" \ + " used to combine them." \ + ) + #define VUT_OPT_q \ VOPT("q:", "[-q ]", "VSL query", \ - "Specifies the VSL query to use. When multiple -q" \ - " options are specified, all queries are considered" \ + "Specifies the VSL query to use. When multiple -q or" \ + " -Q options are specified, all queries are considered" \ " as if the 'or' operator was used to combine them." \ ) diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c index 0fdce6634..db7f97116 100644 --- a/lib/libvarnishapi/vut.c +++ b/lib/libvarnishapi/vut.c @@ -51,6 +51,7 @@ #include "miniobj.h" #include "vcs.h" #include "vsb.h" +#include "vfil.h" #include "vut.h" @@ -195,8 +196,19 @@ VUT_Arg(struct VUT *vut, int opt, const char *arg) AN(arg); REPLACE(vut->P_arg, arg); return (1); + case 'Q': + AN(arg); + p = VFIL_readfile(NULL, arg, NULL); + if (p == NULL) { + VUT_Error(vut, 1, "-Q %s: %s", arg, strerror(errno)); + return (-1); + } + vut_arg_q(vut, p); + free(p); + return (1); case 'q': /* Query to use */ + AN(arg); vut_arg_q(vut, arg); return (1); case 'r': From dridi.boukelmoune at gmail.com Wed Jun 12 04:58:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 12 Jun 2019 04:58:12 +0000 (UTC) Subject: [master] de44cc3c2 Test coverage for a query on a multiline VSL record Message-ID: <20190612045812.3032CAE089@lists.varnish-cache.org> commit de44cc3c23a1cbf3dc95da7f274461ed8f2223fd Author: Dridi Boukelmoune Date: Mon May 27 16:29:43 2019 +0200 Test coverage for a query on a multiline VSL record Not a multiline VSL query, it's the record containing multiple lines. diff --git a/bin/varnishtest/tests/u00014.vtc b/bin/varnishtest/tests/u00014.vtc index dc3138ae3..9667d85c6 100644 --- a/bin/varnishtest/tests/u00014.vtc +++ b/bin/varnishtest/tests/u00014.vtc @@ -136,3 +136,13 @@ shell -match "^500 503 $" { ./ncsa.sh -Q query1.vslq -q 'RespStatus == 503' } + +# Make v1 log a multiline VSL record and query it + +logexpect l1 -v v1 -g raw -q {CLI ~ "\\nvcl1.s1"} { + expect 0 0 CLI "vcl1.s1 +healthy" +} -start + +varnish v1 -cliok backend.list + +logexpect l1 -wait From dridi.boukelmoune at gmail.com Wed Jun 12 04:58:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 12 Jun 2019 04:58:12 +0000 (UTC) Subject: [master] 11b8a7ebd Reinstitute multi-line VSL queries Message-ID: <20190612045812.61408AE091@lists.varnish-cache.org> commit 11b8a7ebdb30516b4181c8ae61e9b0ce150f08de Author: Dridi Boukelmoune Date: Wed Jun 5 14:12:39 2019 +0200 Reinstitute multi-line VSL queries Forcing one line per query can lead to very long lines for cases where we can't decompose OR-able individual queries. Where newline previously acted as a mere space between tokens, the backslash-newline sequence now fills that void. diff --git a/bin/varnishtest/tests/u00014.vtc b/bin/varnishtest/tests/u00014.vtc index 9667d85c6..488357e1b 100644 --- a/bin/varnishtest/tests/u00014.vtc +++ b/bin/varnishtest/tests/u00014.vtc @@ -54,7 +54,8 @@ shell -err -expect "Query expression error" { shell -err -expect "Query expression error" { varnishncsa -d -n ${v1_name} -q ' - * ~ "Incomplete quoted string + * ~ "Incomplete quoted + string" ' } @@ -137,7 +138,32 @@ shell -match "^500 503 $" { ./ncsa.sh -Q query1.vslq -q 'RespStatus == 503' } -# Make v1 log a multiline VSL record and query it +shell -match "^500 $" { + set -e + + tee query1_cont.vslq <<-EOF | wc -l | grep -q 4 # ensure 4 lines + # line continuation + RespStatus \\ + == \\ + 500 + EOF + + ./ncsa.sh -Q query1_cont.vslq +} + +shell -err -expect "Query expression error" { + set -e + + tee string_cont.vslq <<-EOF | wc -l | grep -q 3 # ensure 3 lines + # quoted string continuation + * ~ "very long \\ + string" + EOF + + varnishncsa -d -n ${v1_name} -Q string_cont.vslq +} + +# Make v1 log a multiline VSL record and query it (hardly useful) logexpect l1 -v v1 -g raw -q {CLI ~ "\\nvcl1.s1"} { expect 0 0 CLI "vcl1.s1 +healthy" diff --git a/doc/sphinx/reference/vsl-query.rst b/doc/sphinx/reference/vsl-query.rst index 072747f4c..71534a4b0 100644 --- a/doc/sphinx/reference/vsl-query.rst +++ b/doc/sphinx/reference/vsl-query.rst @@ -151,6 +151,23 @@ is identical to this query:: Comments can be used and will be ignored, they start with the ``'#'`` character, which may be more useful when the query is read from a file. +For very long queries that couldn't easily be split into multiple queries +it is possible to break them into multiple lines with a backslash preceding +an end of line. + +For example this query:: + + BerespStatus >= 500 + +is identical to this query:: + + BerespStatus \ + >= \ + 500 + +A backslash-newline sequence doesn't continue a comment on the next line +and isn't allowed in a quoted string. + Record selection criteria ------------------------- diff --git a/lib/libvarnishapi/vxp_lexer.c b/lib/libvarnishapi/vxp_lexer.c index 57f0c971f..b1e952a12 100644 --- a/lib/libvarnishapi/vxp_lexer.c +++ b/lib/libvarnishapi/vxp_lexer.c @@ -106,6 +106,11 @@ vxp_Lexer(struct vxp *vxp) continue; } + if (*p == '\\' && p[1] == '\n') { + p += 2; + continue; + } + /* Skip comments */ if (*p == '#') { while (p < vxp->e && *p != '\n') @@ -128,7 +133,7 @@ vxp_Lexer(struct vxp *vxp) for (q = p + 1; q < vxp->e; q++) { if (*q == '\\') { q++; - if (q == vxp->e) + if (q == vxp->e || *q == '\n') break; } else if (*q == '\n') { break; From dridi.boukelmoune at gmail.com Wed Jun 12 04:58:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 12 Jun 2019 04:58:12 +0000 (UTC) Subject: [master] 9689c5399 Start skeleton release notes for the next version. Message-ID: <20190612045812.87C62AE09C@lists.varnish-cache.org> commit 9689c5399e87f487b7da9b413942f3dc091d9729 Author: Geoff Simmons Date: Tue Sep 25 16:31:17 2018 +0200 Start skeleton release notes for the next version. Restructured so that: * 'Upgrading' is limited to work that has to be done to upgrade from a current deployment to the new version. * 'Changes' is a comprehensive, user-level description of changes and new features. Conflicts: doc/sphinx/whats-new/index.rst diff --git a/doc/sphinx/whats-new/changes-trunk.rst b/doc/sphinx/whats-new/changes-trunk.rst new file mode 100644 index 000000000..fbda9d6b9 --- /dev/null +++ b/doc/sphinx/whats-new/changes-trunk.rst @@ -0,0 +1,73 @@ +**Note: This is a working document for a future release, with running +updates for changes in the development branch. For changes in the +released versions of Varnish, see:** :ref:`whats-new-index` + +.. _whatsnew_changes_CURRENT: + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +Changes in Varnish **$NEXT_RELEASE** +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +For information about updating your current Varnish deployment to the +new version, see :ref:`whatsnew_upgrading_CURRENT`. + +A more detailed and technical account of changes in Varnish, with +links to issues that have been fixed and pull requests that have been +merged, may be found in the `change log`_. + +.. _change log: https://github.com/varnishcache/varnish-cache/blob/master/doc/changes.rst + +varnishd +======== + +Parameters +~~~~~~~~~~ + +**XXX changes in -p parameters** + +Other changes in varnishd +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Changes to VCL +============== + +VCL variables +~~~~~~~~~~~~~ + +**XXX new, deprecated or removed variables, or changed semantics** + +Other changes to VCL +~~~~~~~~~~~~~~~~~~~~ + +VMODs +===== + +**XXX changes in the bundled VMODs** + +varnishlog +========== + +**XXX changes concerning varnishlog(1) and/or vsl(7)** + +varnishadm +========== + +**XXX changes concerning varnishadm(1) and/or varnish-cli(7)** + +varnishstat +=========== + +**XXX changes concerning varnishstat(1) and/or varnish-counters(7)** + +varnishtest +=========== + +**XXX changes concerning varnishtest(1) and/or vtc(7)** + +Changes for developers and VMOD authors +======================================= + +**XXX changes concerning VRT, the public APIs, source code organization, +builds etc.** + +*eof* diff --git a/doc/sphinx/whats-new/index.rst b/doc/sphinx/whats-new/index.rst index 424c6685d..510027c08 100644 --- a/doc/sphinx/whats-new/index.rst +++ b/doc/sphinx/whats-new/index.rst @@ -8,6 +8,20 @@ This section describes the changes and improvements between different versions of Varnish, and what upgrading between the different versions entail. +Varnish **$NEXT_RELEASE** +------------------------- + +**Note: These are working documents for a future release, with running +updates for changes in the development branch. For changes in the +released versions of Varnish, see the chapters listed below.** + + +.. toctree:: + :maxdepth: 2 + + changes-trunk + upgrading-trunk + Varnish 6.2 ----------- diff --git a/doc/sphinx/whats-new/upgrading-trunk.rst b/doc/sphinx/whats-new/upgrading-trunk.rst new file mode 100644 index 000000000..6143fde99 --- /dev/null +++ b/doc/sphinx/whats-new/upgrading-trunk.rst @@ -0,0 +1,33 @@ +**Note: This is a working document for a future release, with running +updates for changes in the development branch. For changes in the +released versions of Varnish, see:** :ref:`whats-new-index` + +.. _whatsnew_upgrading_CURRENT: + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +Upgrading to Varnish **$NEXT_RELEASE** +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +**XXX: how to upgrade from previous deployments to this +version. Limited to work that has to be done for an upgrade, new +features are listed in "Changes". Explicitly mention what does *not* +have to be changed, especially in VCL. May include, but is not limited +to:** + +* Elements of VCL that have been removed or are deprecated, or whose + semantics have changed. + +* -p parameters that have been removed or are deprecated, or whose + semantics have changed. + +* Changes in the CLI. + +* Changes in the output or interpretation of stats or the log, including + changes affecting varnishncsa/-hist/-top. + +* Changes that may be necessary in VTCs or in the use of varnishtest. + +* Changes in public APIs that may require changes in VMODs or VAPI/VUT + clients. + +*eof* From dridi.boukelmoune at gmail.com Wed Jun 12 04:58:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 12 Jun 2019 04:58:12 +0000 (UTC) Subject: [master] 316dfbc84 Mention VSL query syntax in 2019-09 release notes Message-ID: <20190612045812.AD1B2AE0A6@lists.varnish-cache.org> commit 316dfbc84d0cf294848a20d059af09833c5d74db Author: Dridi Boukelmoune Date: Thu Jun 6 13:49:23 2019 +0200 Mention VSL query syntax in 2019-09 release notes diff --git a/doc/sphinx/whats-new/changes-trunk.rst b/doc/sphinx/whats-new/changes-trunk.rst index fbda9d6b9..224b81b9f 100644 --- a/doc/sphinx/whats-new/changes-trunk.rst +++ b/doc/sphinx/whats-new/changes-trunk.rst @@ -44,10 +44,81 @@ VMODs **XXX changes in the bundled VMODs** -varnishlog -========== +vsl-query(7) +============ + +The syntax for VSL queries, mainly available via the ``-q`` option with +:ref:`varnishlog(1)` and similar tools, has slightly changed. Previously +and end of line in a query would be treated as a simple token separator +so in a script you could for example write this:: + + varnishlog -q ' + tag operator operand or + tag operator operand or + tag operator operand + ' -g request ... + +From now on, a query ends at the end of the line, but multiple queries +can be specified in which case it acts as if the ``or`` operator was used +to join all the queries. + +With this change in the syntax, the following query:: + + varnishlog -q ' + query1 + query2 + ' + +is equivalent to:: + + varnishlog -q '(query1) or (query2)' + +In other words, if you are using a Varnish utility to process transactions +for several independent reasons, you can decompose complex queries into +simpler ones by breaking them into separate lines, and for the most complex +queries possibly getting rid of parenthesis you would have needed in a +single query. + +If your query is complex and long, but cannot appropriately be broken down +into multiple queries, you can still break it down into multiple lines by +using a backslash-newline sequence:: + + tag operator operand and \ + tag operator operand and \ + tag operator operand + +See :ref:`vsl-query(7)` for more information about this change. + +With this new meaning for an end of line in a query it is now possible to +add comments in a query. If you run into the situation where again you need +to capture transactions for multiple reasons, you may document it directly +in the query:: + + varnishlog -q ' + # catch varnish errors + *Error + + # catch client errors + BerespStatus >= 400 and BerespStatus < 500 + + # catch backend errors + BerespStatus >= 500 + ' -g request + +This way when you later revisit a complex query, comments may help you +maintain an understanding of each individual query. This can become even +more convenient when the query is stored in a file. + +varnishlog(1), varnishncsa(1) and others +======================================== + +Our collection of log-processing tools gained the ability to specify +multiple ``-q`` options. While previously only the last ``-q`` option +would prevail you may now pass multiple individual queries and filtering +will operate as if the ``or`` operator was used to join all the queries. -**XXX changes concerning varnishlog(1) and/or vsl(7)** +A new ``-Q`` option allows you to read the query from a file instead. It +can also be used multiple times and adds up to any ``-q`` option specified. varnishadm ========== From dridi.boukelmoune at gmail.com Wed Jun 12 12:21:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 12 Jun 2019 12:21:09 +0000 (UTC) Subject: [master] f1584fd8d Partial revert of "Appropriate VSL group checking" Message-ID: <20190612122109.376B49A6A@lists.varnish-cache.org> commit f1584fd8d8e28308ef0c37af7b33aad947fcf50f Author: Dridi Boukelmoune Date: Wed Jun 12 14:17:14 2019 +0200 Partial revert of "Appropriate VSL group checking" This would trigger the tautological-compare warning on a recent-enough Clang, and SunCC appears to complain as well. Reported by @daghf. diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c index b27646113..4bef6b233 100644 --- a/lib/libvarnishapi/vsl_dispatch.c +++ b/lib/libvarnishapi/vsl_dispatch.c @@ -1088,7 +1088,7 @@ VSLQ_New(struct VSL_data *vsl, struct VSL_cursor **cp, struct VSLQ *vslq; CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC); - if (grouping < 0 || grouping >= VSL_g__MAX) { + if (grouping >= VSL_g__MAX) { (void)vsl_diag(vsl, "Illegal query grouping"); return (NULL); } From dridi.boukelmoune at gmail.com Thu Jun 13 05:49:07 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 13 Jun 2019 05:49:07 +0000 (UTC) Subject: [master] e4efa422c Kill dead statement Message-ID: <20190613054907.811DBA3D4F@lists.varnish-cache.org> commit e4efa422cec26bf285deab9ebc4f024186e27c88 Author: Dridi Boukelmoune Date: Thu Jun 13 07:47:37 2019 +0200 Kill dead statement Spotted by VTEST. diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c index db7f97116..fe4f22cc7 100644 --- a/lib/libvarnishapi/vut.c +++ b/lib/libvarnishapi/vut.c @@ -199,10 +199,8 @@ VUT_Arg(struct VUT *vut, int opt, const char *arg) case 'Q': AN(arg); p = VFIL_readfile(NULL, arg, NULL); - if (p == NULL) { + if (p == NULL) VUT_Error(vut, 1, "-Q %s: %s", arg, strerror(errno)); - return (-1); - } vut_arg_q(vut, p); free(p); return (1); From dridi.boukelmoune at gmail.com Thu Jun 13 08:13:08 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 13 Jun 2019 08:13:08 +0000 (UTC) Subject: [master] 2e5807745 Give up on one u14 test Message-ID: <20190613081308.1458CA6BAF@lists.varnish-cache.org> commit 2e5807745cde363f45600116477c417fcf5d91cc Author: Dridi Boukelmoune Date: Thu Jun 13 10:12:00 2019 +0200 Give up on one u14 test That libtool bug on SunOS... diff --git a/bin/varnishtest/tests/u00014.vtc b/bin/varnishtest/tests/u00014.vtc index 488357e1b..05f458798 100644 --- a/bin/varnishtest/tests/u00014.vtc +++ b/bin/varnishtest/tests/u00014.vtc @@ -41,11 +41,6 @@ shell -match "^500 503 $" { ./ncsa.sh } -shell -err -expect "Query expression error" { - # empty query - varnishncsa -d -n ${v1_name} -q '' -} - shell -err -expect "Query expression error" { varnishncsa -d -n ${v1_name} -q ' # empty multiline query From fgsch at lodoss.net Thu Jun 13 17:34:08 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 13 Jun 2019 17:34:08 +0000 (UTC) Subject: [master] 96e81ce2e Plug minor leak and cleanup Message-ID: <20190613173408.F0F0C65B4@lists.varnish-cache.org> commit 96e81ce2e2248e4e1c5918544f425e75b36fda40 Author: Federico G. Schwindt Date: Thu Jun 13 18:20:16 2019 +0100 Plug minor leak and cleanup diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c index fe4f22cc7..5c9d6a509 100644 --- a/lib/libvarnishapi/vut.c +++ b/lib/libvarnishapi/vut.c @@ -132,7 +132,6 @@ static void vut_arg_q(struct VUT *vut, const char *arg) { struct vsb *vsb; - char *s; AN(arg); if (vut->q_arg == NULL) { @@ -145,10 +144,8 @@ vut_arg_q(struct VUT *vut, const char *arg) AZ(VSB_printf(vsb, "%s\n%s", vut->q_arg, arg)); AZ(VSB_finish(vsb)); - s = strdup(VSB_data(vsb)); - REPLACE(vut->q_arg, s); + REPLACE(vut->q_arg, VSB_data(vsb)); - VSB_clear(vsb); VSB_destroy(&vsb); } From phk at FreeBSD.org Mon Jun 17 07:00:16 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 17 Jun 2019 07:00:16 +0000 (UTC) Subject: [master] 4600cf986 Simplify vtc_record() Message-ID: <20190617070016.C468CA33A1@lists.varnish-cache.org> commit 4600cf986c4645feb18e939ae6a40bbe211cb350 Author: Poul-Henning Kamp Date: Mon Jun 17 06:03:08 2019 +0000 Simplify vtc_record() diff --git a/bin/varnishtest/vtc_subr.c b/bin/varnishtest/vtc_subr.c index ffcc51ef7..3cf36d586 100644 --- a/bin/varnishtest/vtc_subr.c +++ b/bin/varnishtest/vtc_subr.c @@ -30,7 +30,7 @@ #include #include -#include +#include #include #include #include @@ -42,7 +42,6 @@ #include "vct.h" #include "vnum.h" #include "vre.h" -#include "vtcp.h" struct vsb * vtc_hex_to_bin(struct vtclog *vl, const char *arg) @@ -188,29 +187,24 @@ vtc_wait4(struct vtclog *vl, long pid, void * vtc_record(struct vtclog *vl, int fd, struct vsb *vsb) { - char buf[65536]; - struct pollfd fds[1]; + char buf[BUFSIZ]; int i; - VTCP_nonblocking(fd); while (1) { - memset(fds, 0, sizeof fds); - fds->fd = fd; - fds->events = POLLIN; - i = poll(fds, 1, 10000); - if (i == 0) - continue; - if (fds->revents & POLLIN) { - i = read(fd, buf, sizeof buf - 1); - if (i > 0) { - if (vsb != NULL) - VSB_bcat(vsb, buf, i); - buf[i] = '\0'; - vtc_dump(vl, 3, "debug", buf, -2); - } - } - if (fds->revents & (POLLERR|POLLHUP)) { - vtc_log(vl, 4, "STDOUT poll 0x%x", fds->revents); + errno = 0; + i = read(fd, buf, sizeof buf - 1); + if (i > 0) { + if (vsb != NULL) + VSB_bcat(vsb, buf, i); + buf[i] = '\0'; + vtc_dump(vl, 3, "debug", buf, -2); + } else if (i == 0 && errno == 0) { + vtc_log(vl, 4, "STDOUT EOF"); + break; + } else { + vtc_log(vl, 4, + "STDOUT read failed with %d - %s.", + errno, strerror(errno)); break; } } From phk at FreeBSD.org Mon Jun 17 07:00:17 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 17 Jun 2019 07:00:17 +0000 (UTC) Subject: [master] 18f63d489 Add "error" state to backend fetch and response Message-ID: <20190617070017.15E9BA33AF@lists.varnish-cache.org> commit 18f63d489704db53d6b3f25ab13858f1762c37fd Author: Andrew Wiik Date: Tue Jun 4 11:01:04 2019 -0400 Add "error" state to backend fetch and response diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index a6ade1d39..a204200c1 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -412,6 +412,9 @@ struct busyobj { struct http_conn *htc; + uint16_t err_code; + const char *err_reason; + struct pool_task fetch_task; #define BO_FLAG(l, r, w, d) unsigned l:1; diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 88ee91cd8..c01de9387 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -277,7 +277,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL) return (F_STP_FAIL); - assert (wrk->handling == VCL_RET_FETCH); + assert (wrk->handling == VCL_RET_FETCH || wrk->handling == VCL_RET_ERROR); HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod); @@ -290,6 +290,9 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) bo->vfc->resp = bo->beresp; bo->vfc->req = bo->bereq; + if (wrk->handling == VCL_RET_ERROR) + return (F_STP_ERROR); + i = VDI_GetHdr(bo); now = W_TIM_real(wrk); @@ -384,10 +387,14 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL); - if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL) { + if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL || + wrk->handling == VCL_RET_ERROR) { bo->htc->doclose = SC_RESP_CLOSE; VDI_Finish(bo); - return (F_STP_FAIL); + if (wrk->handling == VCL_RET_ERROR) + return (F_STP_ERROR); + else + return (F_STP_FAIL); } if (wrk->handling == VCL_RET_RETRY) { @@ -705,7 +712,8 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) AN(bo->fetch_objcore->flags & OC_F_BUSY); assert(bo->director_state == DIR_S_NULL); - wrk->stats->fetch_failed++; + if (wrk->handling != VCL_RET_ERROR) + wrk->stats->fetch_failed++; now = W_TIM_real(wrk); VSLb_ts_busyobj(bo, "Error", now); @@ -719,7 +727,11 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) // XXX: reset all beresp flags ? HTTP_Setup(bo->beresp, bo->ws, bo->vsl, SLT_BerespMethod); - http_PutResponse(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed"); + if (bo->err_code > 0) + http_PutResponse(bo->beresp, "HTTP/1.1", bo->err_code, bo->err_reason); + else + http_PutResponse(bo->beresp, "HTTP/1.1", 503, "Backend fetch failed"); + http_TimeHeader(bo->beresp, "Date: ", now); http_SetHeader(bo->beresp, "Server: Varnish"); diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 854a690f2..81cc12b09 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -55,9 +55,18 @@ VRT_synth(VRT_CTX, VCL_INT code, VCL_STRING reason) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); + assert(ctx->req != NULL || ctx->bo != NULL); if (code < 100 || code > 65535) code = 503; + + if (ctx->req == NULL) { + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + ctx->bo->err_code = (uint16_t)code; + ctx->bo->err_reason = reason ? reason + : http_Status2Reason(ctx->bo->err_code % 1000, NULL); + return; + } + ctx->req->err_code = (uint16_t)code; ctx->req->err_reason = reason ? reason : http_Status2Reason(ctx->req->err_code % 1000, NULL); diff --git a/bin/varnishtest/tests/v00062.vtc b/bin/varnishtest/tests/v00062.vtc new file mode 100644 index 000000000..365b949d6 --- /dev/null +++ b/bin/varnishtest/tests/v00062.vtc @@ -0,0 +1,78 @@ +varnishtest "backend_(fetch|response) error state with custom status/reason" + +server s1 -repeat 3 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + sub vcl_backend_fetch { + if (bereq.url == "/errorsynth") { + return (error(403)); + } + + if (bereq.url == "/errorsynthbody") { + return (error(403, "Steven has no cookies")); + } + + if (bereq.url == "/error") { + return (error); + } + } + + sub vcl_backend_response { + if (bereq.url == "/resperrorsynth") { + return (error(403)); + } + + if (bereq.url == "/resperrorsynthbody") { + return (error(403, "Steven has no cookies")); + } + + if (bereq.url == "/resperror") { + return (error); + } + } + + sub vcl_backend_error { + set beresp.http.error = "visited"; + } +} -start + +client c1 { + txreq -url /error + rxresp + expect resp.status == 503 + expect resp.http.error == visited + + txreq -url /errorsynth + rxresp + expect resp.status == 403 + expect resp.http.error == visited + + txreq -url /errorsynthbody + rxresp + expect resp.status == 403 + expect resp.reason == "Steven has no cookies" + expect resp.http.error == visited + + txreq -url /resperror + rxresp + expect resp.status == 503 + expect resp.http.error == visited + + txreq -url /resperrorsynth + rxresp + expect resp.status == 403 + expect resp.http.error == visited + + txreq -url /resperrorsynthbody + rxresp + expect resp.status == 403 + expect resp.reason == "Steven has no cookies" + expect resp.http.error == visited +} -run + +# Make sure we don't increment the failed fetch counter + +varnish v1 -expect fetch_failed == 0 diff --git a/doc/graphviz/cache_fetch.dot b/doc/graphviz/cache_fetch.dot index eb68f72a3..96c0967a0 100644 --- a/doc/graphviz/cache_fetch.dot +++ b/doc/graphviz/cache_fetch.dot @@ -39,16 +39,18 @@ digraph cache_fetch { /* vbf_stp_startfetch() */ v_b_f [ shape=record - label="{vbf_stp_startfetch:|{vcl_backend_fetch\{\}|bereq.*}|{fail|abandon|fetch}}" + label="{vbf_stp_startfetch:|{vcl_backend_fetch\{\}|bereq.*}|{error|fail|abandon|fetch}}" ] + v_b_f:error:s -> v_b_e v_b_f:fetch:s -> v_b_hdrs [style=bold] v_b_hdrs [ label="send bereq,\nread beresp (headers)"] v_b_hdrs -> v_b_r [style=bold] v_b_hdrs -> v_b_e v_b_r [ shape=record - label="{vbf_stp_startfetch:|{vcl_backend_response\{\}|{bereq.*|beresp.*}}|{fail|{retry|{max?|ok?}}|abandon|{deliver or pass|{304?|other?}}}}" + label="{vbf_stp_startfetch:|{vcl_backend_response\{\}|{bereq.*|beresp.*}}|{error|fail|{retry|{max?|ok?}}|abandon|{deliver or pass|{304?|other?}}}}" ] + v_b_r:error:s -> v_b_e v_b_r:retry -> v_b_r_retry [color=purple] v_b_r:max -> v_b_e v_b_r:fetch_304:s -> vbf_stp_condfetch @@ -90,8 +92,8 @@ digraph cache_fetch { ] vbf_stp_condfetch:ok:s -> vbf_stp_fetchend - error [shape=plaintext] - error -> FETCH_FAIL + fail [shape=plaintext] + fail -> FETCH_FAIL /* vbf_stp_error */ v_b_e [ diff --git a/doc/graphviz/cache_fetch.svg b/doc/graphviz/cache_fetch.svg index aeb38a95b..fa09e056e 100644 --- a/doc/graphviz/cache_fetch.svg +++ b/doc/graphviz/cache_fetch.svg @@ -1,283 +1,338 @@ - -