From phk at FreeBSD.org Tue Jun 1 09:15:09 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 1 Jun 2021 09:15:09 +0000 (UTC) Subject: [master] c50ff59c1 Switch std_conversion onto SF_number functions Message-ID: <20210601091509.7CA4A9B0F@lists.varnish-cache.org> commit c50ff59c1d0fa9a7369e9f994e60bd0f431499ce Author: Poul-Henning Kamp Date: Tue Jun 1 09:13:50 2021 +0000 Switch std_conversion onto SF_number functions diff --git a/bin/varnishtest/tests/m00015.vtc b/bin/varnishtest/tests/m00015.vtc index 3aebc7e4d..2df533331 100644 --- a/bin/varnishtest/tests/m00015.vtc +++ b/bin/varnishtest/tests/m00015.vtc @@ -98,7 +98,7 @@ client c1 { txreq -hdr "foo: 999999999999.999" \ -hdr "bytes: 999999999999b" \ -hdr "duration: 999999999999.999s" \ - -hdr "integer: 999999999999.999" \ + -hdr "integer: 999999999999.000" \ -hdr "time: 999999999999.999" rxresp expect resp.http.converted == 999999999999.999 diff --git a/vmod/vmod_std_conversions.c b/vmod/vmod_std_conversions.c index ab7f61cb4..9aaf9a8cf 100644 --- a/vmod/vmod_std_conversions.c +++ b/vmod/vmod_std_conversions.c @@ -147,8 +147,8 @@ vmod_bytes(VRT_CTX, struct VARGS(bytes) *a) VCL_INT v_matchproto_(td_std_integer) vmod_integer(VRT_CTX, struct VARGS(integer) *a) { - const char *e; - double r; + const char *p, *errtxt = NULL; + double r, tmp; int nargs; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -167,9 +167,11 @@ vmod_integer(VRT_CTX, struct VARGS(integer) *a) return (a->bytes); if (a->valid_s && a->s != NULL) { - r = VNUMpfx(a->s, &e); - if (e != NULL) - r = NAN; + p = a->s; + r = SF_Parse_Number(&p, &errtxt); + if (!errno && *p == '\0' && modf(r, &tmp) == 0.0) + return (r); + r = NAN; } if (a->valid_duration) @@ -190,7 +192,10 @@ vmod_integer(VRT_CTX, struct VARGS(integer) *a) if (a->valid_fallback) return (a->fallback); - VRT_fail(ctx, "std.integer: conversion failed"); + if (errtxt != NULL) + VRT_fail(ctx, "std.integer: conversion failed: %s", errtxt); + else + VRT_fail(ctx, "std.integer: conversion failed"); return (0); } @@ -234,6 +239,7 @@ VCL_REAL v_matchproto_(td_std_real) vmod_real(VRT_CTX, struct VARGS(real) *a) { VCL_REAL r; + const char *p, *errtxt = NULL; int nargs; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -260,15 +266,19 @@ vmod_real(VRT_CTX, struct VARGS(real) *a) return ((VCL_REAL)a->time); if (a->valid_s && a->s != NULL) { - r = VNUM(a->s); - if (!isnan(r)) + p = a->s; + r = SF_Parse_Decimal(&p, &errtxt); + if (!errno && *p == '\0') return (r); } if (a->valid_fallback) return (a->fallback); - VRT_fail(ctx, "std.real: conversion failed"); + if (errtxt != NULL) + VRT_fail(ctx, "std.real: conversion failed: %s", errtxt); + else + VRT_fail(ctx, "std.real: conversion failed"); return (0); } From dridi.boukelmoune at gmail.com Tue Jun 1 13:50:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 1 Jun 2021 13:50:07 +0000 (UTC) Subject: [master] 3a214e62a vcc: Centralize the global prefix in the type Message-ID: <20210601135007.0A992103169@lists.varnish-cache.org> commit 3a214e62abc544768134e8ed7ff7dd2d50eeb48c Author: Dridi Boukelmoune Date: Wed May 12 19:32:58 2021 +0200 vcc: Centralize the global prefix in the type This is as far declaring symbols goes, because we can still find hard-coded prefixes in other places. Having them in the types could actually help centralize them for good, but this is a minimal change for the purpose of referencing global symbols before they are defined. Refs #3555 diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index 64dcc03ab..4ae7bb6a5 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -701,7 +701,7 @@ vcc_ParseAcl(struct vcc *tl) vcc_ExpectVid(tl, "ACL"); ERRCHK(tl); - sym = VCC_HandleSymbol(tl, ACL, ACL_SYMBOL_PREFIX); + sym = VCC_HandleSymbol(tl, ACL); ERRCHK(tl); AN(sym); diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c index 60fbdfa7b..70e805504 100644 --- a/lib/libvcc/vcc_action.c +++ b/lib/libvcc/vcc_action.c @@ -61,7 +61,7 @@ vcc_act_call(struct vcc *tl, struct token *t, struct symbol *sym) if (sym->kind == SYM_SUB) { vcc_AddCall(tl, t0, sym); - VCC_GlobalSymbol(sym, SUB, "VGC_function"); + VCC_GlobalSymbol(sym, SUB); Fb(tl, 1, "%s(ctx, VSUB_STATIC, NULL);\n", sym->lname); SkipToken(tl, ';'); diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c index ac1c9e1f4..4fac23c20 100644 --- a/lib/libvcc/vcc_backend.c +++ b/lib/libvcc/vcc_backend.c @@ -316,7 +316,7 @@ vcc_ParseProbe(struct vcc *tl) vcc_ParseProbeSpec(tl, NULL, &p); tl->default_probe = p; } else { - sym = VCC_HandleSymbol(tl, PROBE, "vgc_probe"); + sym = VCC_HandleSymbol(tl, PROBE); ERRCHK(tl); AN(sym); vcc_ParseProbeSpec(tl, sym, NULL); @@ -612,7 +612,7 @@ vcc_ParseBackend(struct vcc *tl) dn = "vgc_backend_default"; tl->default_director = dn; } else { - sym = VCC_HandleSymbol(tl, BACKEND, "vgc_backend"); + sym = VCC_HandleSymbol(tl, BACKEND); ERRCHK(tl); AN(sym); dn = sym->rname; diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 517dab5a6..c01e01094 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -115,6 +115,7 @@ struct type { const char *name; const struct vcc_method *methods; + const char *global_pfx; const char *tostring; vcc_type_t multype; int stringform; @@ -354,8 +355,8 @@ sym_expr_t vcc_Eval_SymFunc; sym_expr_t vcc_Eval_TypeMethod; void vcc_Eval_Func(struct vcc *, const struct vjsn_val *, const char *, struct symbol *); -void VCC_GlobalSymbol(struct symbol *, vcc_type_t fmt, const char *pfx); -struct symbol *VCC_HandleSymbol(struct vcc *, vcc_type_t , const char *); +void VCC_GlobalSymbol(struct symbol *, vcc_type_t fmt); +struct symbol *VCC_HandleSymbol(struct vcc *, vcc_type_t); void VCC_SymName(struct vsb *, const struct symbol *); /* vcc_obj.c */ @@ -476,5 +477,3 @@ const char *vcc_MarkPriv(struct vcc *, struct procprivhead *, do { vcc__Expect(a, b, __LINE__); ERRCHK(a);} while (0) #define SkipToken(a, b) \ do { vcc__Expect(a, b, __LINE__); ERRCHK(a); vcc_NextToken(a); } while (0) - -#define ACL_SYMBOL_PREFIX "vrt_acl" diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 7d8d82926..4a5728bf6 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -1117,7 +1117,7 @@ cmp_acl(struct vcc *tl, struct expr **e, const struct cmps *cp) sym = VCC_SymbolGet(tl, SYM_MAIN, SYM_ACL, SYMTAB_CREATE, XREF_REF); ERRCHK(tl); AN(sym); - VCC_GlobalSymbol(sym, ACL, ACL_SYMBOL_PREFIX); + VCC_GlobalSymbol(sym, ACL); bprintf(buf, "%sVRT_acl_match(ctx, %s, \v1)", cp->emit, sym->rname); *e = vcc_expr_edit(tl, BOOL, buf, *e, NULL); } diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c index 432d5681b..cd9832569 100644 --- a/lib/libvcc/vcc_parse.c +++ b/lib/libvcc/vcc_parse.c @@ -256,7 +256,7 @@ vcc_ParseFunction(struct vcc *tl) } return; } - VCC_GlobalSymbol(sym, SUB, "VGC_function"); + VCC_GlobalSymbol(sym, SUB); p = vcc_NewProc(tl, sym); p->name = t; VSB_printf(p->cname, "%s", sym->lname); diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index e2cda32b3..8ace651a8 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -85,6 +85,7 @@ VCC_HandleKind(vcc_type_t fmt) if (fmt == STEVEDORE) return (SYM_STEVEDORE); if (fmt == SUB) return (SYM_SUB); if (fmt == INSTANCE) return (SYM_INSTANCE); + AZ(fmt->global_pfx); return (SYM_NONE); } @@ -460,16 +461,17 @@ VCC_WalkSymbols(struct vcc *tl, symwalk_f *func, vcc_ns_t ns, vcc_kind_t kind) } void -VCC_GlobalSymbol(struct symbol *sym, vcc_type_t type, const char *pfx) +VCC_GlobalSymbol(struct symbol *sym, vcc_type_t type) { struct vsb *vsb; CHECK_OBJ_NOTNULL(sym, SYMBOL_MAGIC); - AN(pfx); + AN(type); + AN(type->global_pfx); vsb = VSB_new_auto(); AN(vsb); - VSB_printf(vsb, "%s_", pfx); + VSB_printf(vsb, "%s_", type->global_pfx); VCC_PrintCName(vsb, sym->name, NULL); AZ(VSB_finish(vsb)); sym->lname = strdup(VSB_data(vsb)); @@ -499,7 +501,7 @@ VCC_GlobalSymbol(struct symbol *sym, vcc_type_t type, const char *pfx) } struct symbol * -VCC_HandleSymbol(struct vcc *tl, vcc_type_t fmt, const char *pfx) +VCC_HandleSymbol(struct vcc *tl, vcc_type_t fmt) { struct symbol *sym; vcc_kind_t kind; @@ -541,7 +543,7 @@ VCC_HandleSymbol(struct vcc *tl, vcc_type_t fmt, const char *pfx) return (NULL); AN(sym); AZ(sym->ndef); - VCC_GlobalSymbol(sym, fmt, pfx); + VCC_GlobalSymbol(sym, fmt); sym->ndef = 1; if (sym->def_b == NULL) sym->def_b = t; diff --git a/lib/libvcc/vcc_types.c b/lib/libvcc/vcc_types.c index fc3c01cd6..7557e520e 100644 --- a/lib/libvcc/vcc_types.c +++ b/lib/libvcc/vcc_types.c @@ -57,6 +57,7 @@ struct vcc_method { const struct type ACL[1] = {{ .magic = TYPE_MAGIC, .name = "ACL", + .global_pfx = "vrt_acl", .tostring = "((\v1)->name)", }}; @@ -70,6 +71,7 @@ const struct type BACKEND[1] = {{ .magic = TYPE_MAGIC, .name = "BACKEND", .methods = backend_methods, + .global_pfx = "vgc_backend", .tostring = "VRT_BACKEND_string(\v1)", }}; @@ -124,6 +126,7 @@ const struct type HTTP[1] = {{ const struct type INSTANCE[1] = {{ .magic = TYPE_MAGIC, .name = "INSTANCE", + .global_pfx = "vo", }}; const struct type INT[1] = {{ @@ -142,6 +145,7 @@ const struct type IP[1] = {{ const struct type PROBE[1] = {{ .magic = TYPE_MAGIC, .name = "PROBE", + .global_pfx = "vgc_probe", }}; const struct type REAL[1] = {{ @@ -207,6 +211,7 @@ const struct type STRING_LIST[1] = {{ const struct type SUB[1] = {{ .magic = TYPE_MAGIC, .name = "SUB", + .global_pfx = "VGC_function", }}; const struct type TIME[1] = {{ diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index 361357b32..8c676096f 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -474,7 +474,7 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym) ExpectErr(tl, ID); vcc_ExpectVid(tl, "VCL object"); ERRCHK(tl); - isym = VCC_HandleSymbol(tl, INSTANCE, "vo"); + isym = VCC_HandleSymbol(tl, INSTANCE); ERRCHK(tl); AN(isym); isym->noref = 1; From dridi.boukelmoune at gmail.com Tue Jun 1 13:50:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 1 Jun 2021 13:50:07 +0000 (UTC) Subject: [master] 433b6b1d1 vcc: New functions to take a peek at tokens Message-ID: <20210601135007.230AC10316C@lists.varnish-cache.org> commit 433b6b1d120415333bd78033119441c525ceeea5 Author: Dridi Boukelmoune Date: Tue May 25 11:05:07 2021 +0200 vcc: New functions to take a peek at tokens Without advancing to the next token, and leaving the door open to synthetic tokens that would otherwise interfere with direct access. diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index c01e01094..a48d1a902 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -418,6 +418,8 @@ int vcc_IdIs(const struct token *t, const char *p); void vcc_ExpectVid(struct vcc *tl, const char *what); void vcc_Lexer(struct vcc *tl, struct source *sp); void vcc_NextToken(struct vcc *tl); +struct token * vcc_PeekToken(struct vcc *tl); +struct token * vcc_PeekTokenFrom(struct vcc *tl, const struct token *t); void vcc__ErrInternal(struct vcc *tl, const char *func, unsigned line); diff --git a/lib/libvcc/vcc_token.c b/lib/libvcc/vcc_token.c index 35cc6421b..f8d14a86e 100644 --- a/lib/libvcc/vcc_token.c +++ b/lib/libvcc/vcc_token.c @@ -263,18 +263,37 @@ vcc_ErrWhere(struct vcc *tl, const struct token *t) /*--------------------------------------------------------------------*/ -void -vcc_NextToken(struct vcc *tl) +struct token * +vcc_PeekTokenFrom(struct vcc *tl, const struct token *t) { + struct token *tn; - tl->t = VTAILQ_NEXT(tl->t, list); - if (tl->t == NULL) { + CHECK_OBJ_NOTNULL(tl, VCC_MAGIC); + AN(t); + tn = VTAILQ_NEXT(t, list); + if (tn == NULL) { VSB_cat(tl->sb, "Ran out of input, something is missing or" " maybe unbalanced (...) or {...}\n"); tl->err = 1; - return; } + return (tn); +} + +struct token * +vcc_PeekToken(struct vcc *tl) +{ + + CHECK_OBJ_NOTNULL(tl, VCC_MAGIC); + return (vcc_PeekTokenFrom(tl, tl->t)); +} + +void +vcc_NextToken(struct vcc *tl) +{ + + CHECK_OBJ_NOTNULL(tl, VCC_MAGIC); + tl->t = vcc_PeekTokenFrom(tl, tl->t); } void From dridi.boukelmoune at gmail.com Tue Jun 1 13:50:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 1 Jun 2021 13:50:07 +0000 (UTC) Subject: [master] 3c02e5586 vcc: New function to print a series of tokens Message-ID: <20210601135007.4263C103170@lists.varnish-cache.org> commit 3c02e5586113d0a85cfed36f9f58d1c52ea82bb7 Author: Dridi Boukelmoune Date: Tue May 25 12:34:53 2021 +0200 vcc: New function to print a series of tokens The same pattern occurred 3 times while reviewing potential usage for the new vcc_PeekToken*() functions. diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index a48d1a902..40c46ca5d 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -415,6 +415,8 @@ void vcc_Warn(struct vcc *); void vcc__Expect(struct vcc *tl, unsigned tok, unsigned line); int vcc_IdIs(const struct token *t, const char *p); +void vcc_PrintTokens(struct vcc *tl, const struct token *tb, + const struct token *te); void vcc_ExpectVid(struct vcc *tl, const char *what); void vcc_Lexer(struct vcc *tl, struct source *sp); void vcc_NextToken(struct vcc *tl); diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index 8ace651a8..0d98ed084 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -318,8 +318,7 @@ VCC_SymbolGet(struct vcc *tl, vcc_ns_t ns, vcc_kind_t kind, tl->t = VTAILQ_NEXT(tn, list); if (sym == NULL) { VSB_printf(tl->sb, "%s: '", e->name); - for (tn1 = t0; tn1 != tl->t; tn1 = VTAILQ_NEXT(tn1, list)) - VSB_printf(tl->sb, "%.*s", PF(tn1)); + vcc_PrintTokens(tl, t0, tl->t); VSB_cat(tl->sb, "'"); sym = vcc_sym_in_tab(tl, st, kind, VCL_LOW, VCL_HIGH); if (sym != NULL && sym->kind != SYM_OBJECT && @@ -338,8 +337,7 @@ VCC_SymbolGet(struct vcc *tl, vcc_ns_t ns, vcc_kind_t kind, } if (kind != SYM_NONE && kind != sym->kind) { VSB_cat(tl->sb, "Symbol '"); - for (tn1 = t0; tn1 != tl->t; tn1 = VTAILQ_NEXT(tn1, list)) - VSB_printf(tl->sb, "%.*s", PF(tn1)); + vcc_PrintTokens(tl, t0, tl->t); VSB_printf(tl->sb, "' has wrong type (%s), expected %s:", sym->kind->name, kind->name); VSB_cat(tl->sb, "\nAt: "); diff --git a/lib/libvcc/vcc_token.c b/lib/libvcc/vcc_token.c index f8d14a86e..b965c298d 100644 --- a/lib/libvcc/vcc_token.c +++ b/lib/libvcc/vcc_token.c @@ -329,29 +329,45 @@ vcc_IdIs(const struct token *t, const char *p) * Check that we have a Varnish identifier */ +void +vcc_PrintTokens(struct vcc *tl, const struct token *tb, const struct token *te) +{ + + CHECK_OBJ_NOTNULL(tl, VCC_MAGIC); + AN(tb); + AN(te); + while (tb != te) { + VSB_printf(tl->sb, "%.*s", PF(tb)); + tb = vcc_PeekTokenFrom(tl, tb); + AN(tb); + } +} + void vcc_ExpectVid(struct vcc *tl, const char *what) { const char *bad = NULL; - struct token *t2, *t3; + struct token *t2; ExpectErr(tl, ID); ERRCHK(tl); - t2 = VTAILQ_NEXT(tl->t, list); + t2 = vcc_PeekToken(tl); + ERRCHK(tl); while (t2->tok == '.') { bad = "."; - t2 = VTAILQ_NEXT(t2, list); + t2 = vcc_PeekTokenFrom(tl, t2); + ERRCHK(tl); if (t2->tok != ID) break; - t2 = VTAILQ_NEXT(t2, list); + t2 = vcc_PeekTokenFrom(tl, t2); + ERRCHK(tl); } if (bad == NULL) bad = VCT_invalid_name(tl->t->b, tl->t->e); if (bad != NULL) { VSB_printf(tl->sb, "Name of %s, '", what); - for (t3 = tl->t; t3 != t2; t3 = VTAILQ_NEXT(t3, list)) - VSB_printf(tl->sb, "%.*s", PF(t3)); + vcc_PrintTokens(tl, tl->t, t2); VSB_printf(tl->sb, "', contains illegal character '%c'\n", *bad); vcc_ErrWhere2(tl, tl->t, t2); From dridi.boukelmoune at gmail.com Tue Jun 1 13:50:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 1 Jun 2021 13:50:07 +0000 (UTC) Subject: [master] eabe58bc6 vcc: Avoid direct usage of the token list Message-ID: <20210601135007.6E91C103175@lists.varnish-cache.org> commit eabe58bc6633012dd01ba8f2a131f4e8cd351b2e Author: Dridi Boukelmoune Date: Tue May 25 12:39:38 2021 +0200 vcc: Avoid direct usage of the token list This migrates the remaining candidates for vcc_Peek*() functions using a coccinelle semantic patch. When vcc_PeekToken*() is used in an assignment statement it's straightforward to patch. In an expression a careful review is needed, that's why the generated code is meant not to compile on purpose. That's how the pattern for vcc_PrintTokens() was noticed. diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index 4ae7bb6a5..d7e4a63b3 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -457,7 +457,7 @@ vcc_acl_entry(struct vcc *tl) */ static void -vcc_acl_emit_tokens(const struct vcc *tl, const struct acl_e *ae) +vcc_acl_emit_tokens(struct vcc *tl, const struct acl_e *ae) { struct token *t; const char *sep = ""; @@ -476,7 +476,8 @@ vcc_acl_emit_tokens(const struct vcc *tl, const struct acl_e *ae) } if (t == ae->t_mask) break; - t = VTAILQ_NEXT(t, list); + t = vcc_PeekTokenFrom(tl, t); + ERRCHK(tl); AN(t); sep = " "; } while (ae->t_mask != NULL); @@ -489,7 +490,7 @@ vcc_acl_emit_tokens(const struct vcc *tl, const struct acl_e *ae) */ static unsigned -vcc_acl_emit_tables(const struct vcc *tl, unsigned n, const char *name) +vcc_acl_emit_tables(struct vcc *tl, unsigned n, const char *name) { struct acl_e *ae; unsigned rv = sizeof(ae->data) + 3; diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index 0d98ed084..b41334b79 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -293,11 +293,11 @@ VCC_SymbolGet(struct vcc *tl, vcc_ns_t ns, vcc_kind_t kind, st2 = st; tn2 = tn; } - tn1 = VTAILQ_NEXT(tn, list); - if (tn1->tok != '.') + tn1 = vcc_PeekTokenFrom(tl, tn); + if (tn1 == NULL || tn1->tok != '.') break; - tn1 = VTAILQ_NEXT(tn1, list); - if (tn1->tok != ID) + tn1 = vcc_PeekTokenFrom(tl, tn1); + if (tn1 == NULL || tn1->tok != ID) break; tn = tn1; } @@ -309,13 +309,15 @@ VCC_SymbolGet(struct vcc *tl, vcc_ns_t ns, vcc_kind_t kind, } else if (st != st2) { sym = NULL; } - if (sym == NULL && e->noerr) + if (tl->err || (sym == NULL && e->noerr)) return (sym); AN(st); AN(tn); if (sym == NULL && e == SYMTAB_CREATE) sym = vcc_new_symbol(tl, st, kind, tl->syntax, tl->syntax); - tl->t = VTAILQ_NEXT(tn, list); + tl->t = vcc_PeekTokenFrom(tl, tn); + if (tl->err) + return (NULL); if (sym == NULL) { VSB_printf(tl->sb, "%s: '", e->name); vcc_PrintTokens(tl, t0, tl->t); diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c index 7ced4edc1..de064bea0 100644 --- a/lib/libvcc/vcc_utils.c +++ b/lib/libvcc/vcc_utils.c @@ -380,10 +380,13 @@ vcc_IsFlagRaw(struct vcc *tl, const struct token *t1, const struct token *t2) int vcc_IsFlag(struct vcc *tl) { + struct token *t; int retval; - - retval = vcc_IsFlagRaw(tl, tl->t, VTAILQ_NEXT(tl->t, list)); + t = vcc_PeekToken(tl); + if (t == NULL) + return (-1); + retval = vcc_IsFlagRaw(tl, tl->t, t); if (retval >= 0) vcc_NextToken(tl); return (retval); diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index 8c676096f..d0719ec47 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -315,7 +315,8 @@ vcc_ParseImport(struct vcc *tl) ExpectErr(tl, ID); /* "vmod_name" */ mod = tl->t; - tmod = VTAILQ_NEXT(mod, list); + tmod = vcc_PeekTokenFrom(tl, mod); + ERRCHK(tl); if (tmod->tok == ID && vcc_IdIs(tmod, "as")) { vcc_NextToken(tl); /* "vmod_name" */ vcc_NextToken(tl); /* "as" */ diff --git a/lib/libvcc/vcc_xref.c b/lib/libvcc/vcc_xref.c index 433f5e7b8..cf2be5532 100644 --- a/lib/libvcc/vcc_xref.c +++ b/lib/libvcc/vcc_xref.c @@ -122,8 +122,10 @@ vcc_AddUses(struct vcc *tl, const struct token *t1, const struct token *t2, AN(use->name); pu->t1 = t1; pu->t2 = t2; - if (pu->t2 == NULL) - pu->t2 = VTAILQ_NEXT(t1, list); + if (pu->t2 == NULL) { + pu->t2 = vcc_PeekTokenFrom(tl, t1); + ERRCHK(tl); + } pu->sym = sym; pu->use = use; pu->fm = tl->curproc; diff --git a/tools/coccinelle/archive/vcc_peek.cocci b/tools/coccinelle/archive/vcc_peek.cocci new file mode 100644 index 000000000..ab0726d22 --- /dev/null +++ b/tools/coccinelle/archive/vcc_peek.cocci @@ -0,0 +1,35 @@ +/* + * Provide an API to avoid direct next token access. + */ + +@@ +struct vcc *tl; +expression e; +@@ + +-e = VTAILQ_NEXT(tl->t, list); ++e = vcc_PeekToken(tl); ++ERRCHK(tl); + +@@ +struct token *t; +expression e; +@@ + +-e = VTAILQ_NEXT(t, list); ++e = vcc_PeekTokenFrom(tl, t); ++ERRCHK(tl); + +@@ +struct vcc *tl; +@@ + +-VTAILQ_NEXT(tl->t, list) ++TODO_vcc_PeekToken_with_ERRCHK(tl) + +@@ +struct token *t; +@@ + +-VTAILQ_NEXT(t, list) ++TODO_vcc_Peek_with_ERRCHK(tl, t) From dridi.boukelmoune at gmail.com Tue Jun 1 13:50:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 1 Jun 2021 13:50:07 +0000 (UTC) Subject: [master] cf3963845 vcc: Allow global symbols to be defined after use Message-ID: <20210601135007.912DD103178@lists.varnish-cache.org> commit cf396384595ef47cc66f1eedcf0b6730d5b1b659 Author: Dridi Boukelmoune Date: Wed May 12 19:50:46 2021 +0200 vcc: Allow global symbols to be defined after use A side effect is slightly more accurate error messages. Closes #3555 diff --git a/bin/varnishtest/tests/r00916.vtc b/bin/varnishtest/tests/r00916.vtc index e73ef10cd..92900c5b9 100644 --- a/bin/varnishtest/tests/r00916.vtc +++ b/bin/varnishtest/tests/r00916.vtc @@ -5,7 +5,7 @@ server s1 { txresp -body "FOO" } -start -varnish v1 -errvcl {Symbol not found: 's-1'} { +varnish v1 -errvcl {Undefined backend s-1, first reference:} { backend b { .host = "${localhost}"; } sub s1 { } diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc index f5145bf37..d26295fdc 100644 --- a/bin/varnishtest/tests/v00016.vtc +++ b/bin/varnishtest/tests/v00016.vtc @@ -66,7 +66,7 @@ varnish v1 -errvcl {Cannot be set in subroutine 'vcl_pipe'.} { } } -varnish v1 -errvcl {Symbol not found: 'c'} { +varnish v1 -errvcl {Undefined backend c, first reference:} { backend b { .host = "${localhost}"; } sub vcl_backend_response { if (beresp.backend == c) { diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc index e29e4ff4f..47b899585 100644 --- a/bin/varnishtest/tests/v00020.vtc +++ b/bin/varnishtest/tests/v00020.vtc @@ -69,6 +69,10 @@ varnish v1 -errvcl {Operator % only possible on INT} { } varnish v1 -vcl { + sub vcl_recv { + # 3555 (define after use) + set req.backend_hint = b; + } backend b { .host = "${localhost}"; } sub vcl_recv { set req.http.foo = "foo" + "bar"; @@ -220,6 +224,27 @@ varnish v1 -errvcl {Name of subroutine, 'foo.bar', contains illegal character '. } } +# 3555 (define after use) +varnish v1 -vcl { + import debug; + backend be none; + sub caller { + debug.call(foo); + } + sub foo { } + sub vcl_recv { + call caller; + call foo; + } +} + +# 3555 (don't launder partial symbols) +varnish v1 -errvcl {Symbol not found: 'foo.bar'} { + sub vcl_recv { + set req.backend_hint = foo.bar; + } +} + varnish v1 -errvcl {The names 'vcl_*' are reserved for subroutines.} { sub vcl_bar { } diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 4a5728bf6..55d311ea8 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -719,7 +719,7 @@ vcc_expr5(struct vcc *tl, struct expr **e, vcc_type_t fmt) { struct expr *e1, *e2; const char *ip, *sign; - struct token *t; + struct token *t, *t1; struct symbol *sym; sign = ""; @@ -738,8 +738,19 @@ vcc_expr5(struct vcc *tl, struct expr **e, vcc_type_t fmt) switch (tl->t->tok) { case ID: t = tl->t; - sym = VCC_SymbolGet(tl, SYM_MAIN, SYM_NONE, SYMTAB_PARTIAL, - XREF_REF); + t1 = vcc_PeekToken(tl); + ERRCHK(tl); + sym = VCC_SymbolGet(tl, SYM_MAIN, SYM_NONE, + SYMTAB_PARTIAL_NOERR, XREF_REF); + if (sym == NULL && fmt->global_pfx != NULL && t1->tok != '.') { + sym = VCC_SymbolGet(tl, SYM_MAIN, SYM_NONE, + SYMTAB_CREATE, XREF_REF); + AN(sym); + VCC_GlobalSymbol(sym, fmt); + } + if (sym == NULL) + AZ(VCC_SymbolGet(tl, SYM_MAIN, SYM_NONE, + SYMTAB_PARTIAL, XREF_REF)); ERRCHK(tl); AN(sym); if (sym->kind == SYM_INSTANCE) { From dridi.boukelmoune at gmail.com Tue Jun 1 15:04:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 1 Jun 2021 15:04:05 +0000 (UTC) Subject: [master] 184dc5eb3 vtc: Mitigate v41 timeout Message-ID: <20210601150405.AB69F105B16@lists.varnish-cache.org> commit 184dc5eb3ecf1aaa04980bd1b523d8337615d1aa Author: Dridi Boukelmoune Date: Tue Jun 1 16:56:47 2021 +0200 vtc: Mitigate v41 timeout With sanitizers it becomes too easy to trigger a timeout because of the debug.priv_perf() invocations. Note for later, I'm planning to change the feature syntax to this: feature [!] [...] This way we can use `feature !sanitizer` to express the same check. diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc index 5995d82a4..6c83a9f4d 100644 --- a/bin/varnishtest/tests/v00041.vtc +++ b/bin/varnishtest/tests/v00041.vtc @@ -1,5 +1,7 @@ varnishtest "Test priv_task" +feature no_sanitizer + server s1 { rxreq txresp From dridi.boukelmoune at gmail.com Wed Jun 2 06:30:09 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 2 Jun 2021 06:30:09 +0000 (UTC) Subject: [master] 127e5b79a varnishtest: Support feature [!] [] Message-ID: <20210602063009.CC0987BD5@lists.varnish-cache.org> commit 127e5b79acc9692d47690e44ba3c7ed262d4226a Author: Dridi Boukelmoune Date: Wed Jun 2 07:58:46 2021 +0200 varnishtest: Support feature [!] [] diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index 63cddfdbf..b962bedc9 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -423,6 +423,9 @@ ipvx_works(const char *target) * no_sanitizer * Varnish was not built with a sanitizer. * + * A feature name can be prepended with an exclamation mark (!) to skip a + * test if a feature is present. + * * Be careful with ignore_unknown_macro, because it may cause a test with a * misspelled macro to fail silently. You should only need it if you must * run a test with strings of the form "${...}". @@ -449,8 +452,8 @@ static const unsigned so_rcvtimeo_works = 0; void v_matchproto_(cmd_f) cmd_feature(CMD_ARGS) { - int r; - int good; + const char *feat; + int r, good, skip, neg; (void)priv; @@ -459,17 +462,27 @@ cmd_feature(CMD_ARGS) #define FEATURE(nm, tst) \ do { \ - if (!strcmp(*av, nm)) { \ + if (!strcmp(feat, nm)) { \ + good = 1; \ if (tst) { \ - good = 1; \ + skip = neg; \ } else { \ - vtc_stop = 2; \ + skip = !neg; \ } \ } \ } while (0) + skip = 0; + for (av++; *av != NULL; av++) { good = 0; + neg = 0; + feat = *av; + + if (feat[0] == '!') { + neg = 1; + feat++; + } FEATURE("ipv4", ipvx_works("127.0.0.1")); FEATURE("ipv6", ipvx_works("[::1]")); @@ -485,37 +498,41 @@ cmd_feature(CMD_ARGS) FEATURE("no_sanitizer", no_sanitizer); FEATURE("SO_RCVTIMEO_WORKS", so_rcvtimeo_works); - if (!strcmp(*av, "disable_aslr")) { + if (!strcmp(feat, "disable_aslr")) { good = 1; + skip = neg; #ifdef HAVE_SYS_PERSONALITY_H r = personality(0xffffffff); r = personality(r | ADDR_NO_RANDOMIZE); - if (r < 0) { - good = 0; - vtc_stop = 2; - } + if (r < 0) + skip = !neg; #endif - } else if (!strcmp(*av, "cmd")) { + } else if (!strcmp(feat, "cmd")) { + good = 1; + skip = neg; av++; if (*av == NULL) vtc_fatal(vl, "Missing the command-line"); r = system(*av); - if (WEXITSTATUS(r) == 0) - good = 1; - else - vtc_stop = 2; - } else if (!strcmp(*av, "ignore_unknown_macro")) { + if (WEXITSTATUS(r) != 0) + skip = !neg; + } else if (!strcmp(feat, "ignore_unknown_macro")) { ign_unknown_macro = 1; good = 1; } - if (good) + if (!good) + vtc_fatal(vl, "FAIL test, unknown feature: %s", feat); + + if (!skip) continue; - if (!vtc_stop) - vtc_fatal(vl, "FAIL test, unknown feature: %s", *av); + vtc_stop = 2; + if (neg) + vtc_log(vl, 1, + "SKIPPING test, conflicting feature: %s", feat); else vtc_log(vl, 1, - "SKIPPING test, lacking feature: %s", *av); + "SKIPPING test, lacking feature: %s", feat); return; } } From dridi.boukelmoune at gmail.com Wed Jun 2 06:30:09 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 2 Jun 2021 06:30:09 +0000 (UTC) Subject: [master] 8e0c6264a varnishtest: Rename the no_sanitizer feature Message-ID: <20210602063009.E61207BD8@lists.varnish-cache.org> commit 8e0c6264a27acc8715415c8af90c1bc0dbefde94 Author: Dridi Boukelmoune Date: Wed Jun 2 08:15:16 2021 +0200 varnishtest: Rename the no_sanitizer feature diff --git a/bin/varnishtest/tests/v00004.vtc b/bin/varnishtest/tests/v00004.vtc index 2f2b75459..8760279e5 100644 --- a/bin/varnishtest/tests/v00004.vtc +++ b/bin/varnishtest/tests/v00004.vtc @@ -1,7 +1,7 @@ varnishtest "test if our default paramers make sense ..." feature 64bit -feature no_sanitizer +feature !sanitizer # Skip this test in GCOV mode, 68xx bytes extra per level makes it fail feature cmd {test -z "$GCOVPROG"} diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc index 6c83a9f4d..68ba1b0bd 100644 --- a/bin/varnishtest/tests/v00041.vtc +++ b/bin/varnishtest/tests/v00041.vtc @@ -1,6 +1,6 @@ varnishtest "Test priv_task" -feature no_sanitizer +feature !sanitizer server s1 { rxreq diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index b962bedc9..a46254246 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -420,10 +420,10 @@ ipvx_works(const char *target) * recognized as a macro. * persistent_storage * Varnish was built with the deprecated persistent storage. - * no_sanitizer - * Varnish was not built with a sanitizer. + * sanitizer + * Varnish was built with a sanitizer. * - * A feature name can be prepended with an exclamation mark (!) to skip a + * A feature name can be prefixed with an exclamation mark (!) to skip a * test if a feature is present. * * Be careful with ignore_unknown_macro, because it may cause a test with a @@ -438,9 +438,9 @@ static const unsigned with_persistent_storage = 0; #endif #if __SANITIZER -static const unsigned no_sanitizer = 0; +static const unsigned sanitizer = 1; #else -static const unsigned no_sanitizer = 1; +static const unsigned sanitizer = 0; #endif #ifdef SO_RCVTIMEO_WORKS @@ -495,7 +495,7 @@ cmd_feature(CMD_ARGS) FEATURE("user_vcache", getpwnam("vcache") != NULL); FEATURE("group_varnish", getgrnam("varnish") != NULL); FEATURE("persistent_storage", with_persistent_storage); - FEATURE("no_sanitizer", no_sanitizer); + FEATURE("sanitizer", sanitizer); FEATURE("SO_RCVTIMEO_WORKS", so_rcvtimeo_works); if (!strcmp(feat, "disable_aslr")) { From dridi.boukelmoune at gmail.com Wed Jun 2 06:30:10 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 2 Jun 2021 06:30:10 +0000 (UTC) Subject: [master] c2799eb4c varnishtest: Give disable_aslr a normal treatment Message-ID: <20210602063010.07CA57BDB@lists.varnish-cache.org> commit c2799eb4ce38a13d0eee536afa38a2f849e7e83c Author: Dridi Boukelmoune Date: Wed Jun 2 08:27:37 2021 +0200 varnishtest: Give disable_aslr a normal treatment Only the special cases get special handling. diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index a46254246..59c226e52 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -387,6 +387,20 @@ ipvx_works(const char *target) return (0); } +/**********************************************************************/ + +static int +addr_no_randomize_works(void) +{ + int r = 0; + +#ifdef HAVE_SYS_PERSONALITY_H + r = personality(0xffffffff); + r = personality(r | ADDR_NO_RANDOMIZE); +#endif + return (r >= 0); +} + /* SECTION: feature feature * * Test that the required feature(s) for a test are available, and skip @@ -488,6 +502,7 @@ cmd_feature(CMD_ARGS) FEATURE("ipv6", ipvx_works("[::1]")); FEATURE("pcre_jit", VRE_has_jit); FEATURE("64bit", sizeof(void*) == 8); + FEATURE("disable_aslr", addr_no_randomize_works()); FEATURE("dns", dns_works()); FEATURE("topbuild", iflg); FEATURE("root", !geteuid()); @@ -498,16 +513,7 @@ cmd_feature(CMD_ARGS) FEATURE("sanitizer", sanitizer); FEATURE("SO_RCVTIMEO_WORKS", so_rcvtimeo_works); - if (!strcmp(feat, "disable_aslr")) { - good = 1; - skip = neg; -#ifdef HAVE_SYS_PERSONALITY_H - r = personality(0xffffffff); - r = personality(r | ADDR_NO_RANDOMIZE); - if (r < 0) - skip = !neg; -#endif - } else if (!strcmp(feat, "cmd")) { + if (!strcmp(feat, "cmd")) { good = 1; skip = neg; av++; From phk at FreeBSD.org Wed Jun 2 11:46:05 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 2 Jun 2021 11:46:05 +0000 (UTC) Subject: [master] b490955a0 Move the RFC8941/SF numerical boundaries to vdef so vnum can see them Message-ID: <20210602114605.D372A102E6E@lists.varnish-cache.org> commit b490955a0f3e94fe0a7ed651fddeff4b7d1333ad Author: Poul-Henning Kamp Date: Wed Jun 2 10:16:51 2021 +0000 Move the RFC8941/SF numerical boundaries to vdef so vnum can see them diff --git a/include/vdef.h b/include/vdef.h index d99c02570..fb6ce4f7a 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -112,6 +112,18 @@ # define v_dont_optimize #endif +/********************************************************************* + * Fundamental numerical limits + * These limits track RFC8941 + * We use hex notation because 999999999999.999 is not perfectly + * representable in ieee64 doubles. + */ + +#define VRT_INTEGER_MAX 999999999999999 +#define VRT_INTEGER_MIN -999999999999999 +#define VRT_DECIMAL_MAX 0x1.d1a94a1fffff8p+39 +#define VRT_DECIMAL_MIN -0x1.d1a94a1fffff8p+39 + /********************************************************************* * Pointer alignment magic */ diff --git a/include/vrt.h b/include/vrt.h index 31198fc95..9d2c87e98 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -301,16 +301,6 @@ typedef vtim_real VCL_TIME; typedef struct vcl * VCL_VCL; typedef void VCL_VOID; -/* - * These limits track RFC8941 - * We use hex notation because 999999999999.999 is not perfectly - * representable in ieee64 doubles. - */ -#define VRT_INTEGER_MAX 999999999999999 -#define VRT_INTEGER_MIN -999999999999999 -#define VRT_DECIMAL_MAX 0x1.d1a94a1fffff8p+39 -#define VRT_DECIMAL_MIN -0x1.d1a94a1fffff8p+39 - /*********************************************************************** * This is the composite "context" argument for compiled VCL, VRT and * VMOD functions. From phk at FreeBSD.org Wed Jun 2 11:46:05 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 2 Jun 2021 11:46:05 +0000 (UTC) Subject: [master] efdca94ee Give SF_Parse_{Decimal|Number} a 'strict' argument so we can relax them. Message-ID: <20210602114605.E8B44102E71@lists.varnish-cache.org> commit efdca94eec5e7cd933f9713d2468620f800f08b8 Author: Poul-Henning Kamp Date: Wed Jun 2 10:18:37 2021 +0000 Give SF_Parse_{Decimal|Number} a 'strict' argument so we can relax them. Add test-coverage of internal function sf_parse_int() diff --git a/include/vnum.h b/include/vnum.h index 859d8cbb0..6016d325e 100644 --- a/include/vnum.h +++ b/include/vnum.h @@ -40,8 +40,8 @@ int64_t VNUM_bytes_unit(double r, const char *b, const char *e, uintmax_t rel, const char *VNUM_2bytes(const char *p, uintmax_t *r, uintmax_t rel); int64_t SF_Parse_Integer(const char **ipp, const char **errtxt); -double SF_Parse_Decimal(const char **ipp, const char **errtxt); -double SF_Parse_Number(const char **ipp, const char **errtxt); +double SF_Parse_Decimal(const char **ipp, int strict, const char **errtxt); +double SF_Parse_Number(const char **ipp, int strict, const char **errtxt); #define VNUM_LEGAL_DURATION \ "Legal duration units are 'ms', 's', 'm', 'h', 'd', 'w' and 'y'" diff --git a/lib/libvarnish/vnum.c b/lib/libvarnish/vnum.c index ce66fc7e1..cf532eaf0 100644 --- a/lib/libvarnish/vnum.c +++ b/lib/libvarnish/vnum.c @@ -56,22 +56,24 @@ static const char err_fractional_bytes[] = "Fractional BYTES not allowed"; if (errtxt != NULL) \ *errtxt = (txt); \ errno = EINVAL; \ - return (0); \ + return (retval); \ } while (0) static int64_t -sf_parse_int(const char **ipp, const char **errtxt, int maxdig) +sf_parse_int(const char **ipp, const char **errtxt, int *sign, int maxdig) { int64_t retval = 0; - int negative = 0, ndig = 0; + int ndig = 0; AN(ipp); AN(*ipp); + *errtxt = NULL; + *sign = 1; errno = 0; while (vct_isows(*(*ipp))) (*ipp)++; if(*(*ipp) == '-') { - negative = 1; + *sign = -1; (*ipp)++; } while (vct_isdigit(*(*ipp))) { @@ -81,34 +83,37 @@ sf_parse_int(const char **ipp, const char **errtxt, int maxdig) retval *= 10; retval += *(*ipp)++ - 0x30; } + if (ndig == 0) + BAIL(*sign < 0 ? err_invalid_num : err_miss_num); while (vct_isows(*(*ipp))) (*ipp)++; - if (ndig == 0) - BAIL(negative ? err_invalid_num : err_miss_num); - if (negative) - retval = -retval; return (retval); } int64_t SF_Parse_Integer(const char **ipp, const char **errtxt) { + int64_t retval; + int sign; - return(sf_parse_int(ipp, errtxt, 15)); + retval = sf_parse_int(ipp, errtxt, &sign, 15); + return(retval * sign); } double -SF_Parse_Decimal(const char **ipp, const char **errtxt) +SF_Parse_Number(const char **ipp, int strict, const char **errtxt) { double retval, scale = 1; - int ndig; + int sign, ndig; - retval = (double)sf_parse_int(ipp, errtxt, 12); - if (retval < 0) - scale = -scale; + retval = (double)sf_parse_int(ipp, errtxt, &sign, 15); + if (strict && errno) + return (0); do { if (*(*ipp) != '.') break; + if (retval < VRT_DECIMAL_MIN || retval > VRT_DECIMAL_MAX) + BAIL(err_fatnum); (*ipp)++; for(ndig = 0; ndig < 3; ndig++) { scale *= .1; @@ -116,40 +121,26 @@ SF_Parse_Decimal(const char **ipp, const char **errtxt) break; retval += scale * (*(*ipp)++ - 0x30); } - if (vct_isdigit(*(*ipp))) + if (strict && vct_isdigit(*(*ipp))) BAIL(err_fatnum); + while (vct_isdigit(*(*ipp))) + (*ipp)++; } while (0); while (vct_isows(*(*ipp))) (*ipp)++; - return (retval); + return (retval * sign); } double -SF_Parse_Number(const char **ipp, const char **errtxt) +SF_Parse_Decimal(const char **ipp, int strict, const char **errtxt) { - double retval, scale = 1; - int ndig; + double retval; - retval = (double)sf_parse_int(ipp, errtxt, 15); - if (retval < 0) - scale = -scale; - do { - if (*(*ipp) != '.') - break; - if (retval < -999999999999 || retval > 999999999999) - BAIL(err_fatnum); - (*ipp)++; - for(ndig = 0; ndig < 3; ndig++) { - scale *= .1; - if (!vct_isdigit(*(*ipp))) - break; - retval += scale * (*(*ipp)++ - 0x30); - } - if (vct_isdigit(*(*ipp))) - BAIL(err_fatnum); - } while (0); - while (vct_isows(*(*ipp))) - (*ipp)++; + retval = SF_Parse_Number(ipp, strict, errtxt); + if (errno) + return(retval); + if (retval < VRT_DECIMAL_MIN || retval > VRT_DECIMAL_MAX) + BAIL(err_fatnum); return (retval); } @@ -343,7 +334,7 @@ VNUM_2bytes(const char *p, uintmax_t *r, uintmax_t rel) if (p == NULL || *p == '\0') return (err_miss_num); - fval = SF_Parse_Number(&p, &errtxt); + fval = SF_Parse_Number(&p, 1, &errtxt); if (errno) return(errtxt); if (fval < 0) @@ -362,6 +353,28 @@ VNUM_2bytes(const char *p, uintmax_t *r, uintmax_t rel) * cc -o foo -DNUM_C_TEST -I../.. -I../../include vnum.c vas.c vct.c -lm */ +static const struct test_sf_parse_int { + const char *input; + int maxdig; + int64_t retval; + int consumed; + int sign; + const char *errtxt; +} test_sf_parse_int[] = { + { "1234", 3, 123, 3, 1, err_fatnum }, + { "1234", 4, 1234, 4, 1, NULL }, + { "1234", 5, 1234, 4, 1, NULL }, + { "-", 5, 0, 1, -1, err_invalid_num }, + { "-1234", 3, 123, 4, -1, err_fatnum }, + { "-1234", 4, 1234, 5, -1, NULL }, + { "-1234", 5, 1234, 5, -1, NULL }, + { " -1234", 5, 1234, 6, -1, NULL }, + { " -1234 ", 5, 1234, 7, -1, NULL }, + { " -12 34 ", 5, 12, 5, -1, NULL }, + { " - 12 34 ", 5, 0, 2, -1, err_invalid_num }, + { NULL}, +}; + static struct test_case { const char *str; uintmax_t rel; @@ -460,11 +473,40 @@ main(int argc, char *argv[]) const char **p; const char *e; double d1, d2; + const struct test_sf_parse_int *tspi; + int64_t i64; + int sign, consumed; + const char *errtxt; + const char *input; (void)argc; setbuf(stdout, NULL); setbuf(stderr, NULL); + + for (tspi = test_sf_parse_int; tspi->input != NULL; tspi++) { + errtxt = "(unset)"; + input = tspi->input; + i64 = sf_parse_int(&input, &errtxt, &sign, tspi->maxdig); + consumed = input - tspi->input; + if (i64 != tspi->retval || + sign != tspi->sign || + consumed != tspi->consumed || + errtxt != tspi->errtxt) { + ec++; + printf("sf_parse_int(%s, maxdig=%d) failed\n", + tspi->input, tspi->maxdig); + printf(" retval\texpected %jd\tgot %jd\n", + (intmax_t)tspi->retval, (intmax_t)i64); + printf(" sign\texpected %d\tgot %d\n", + tspi->sign, sign); + printf(" consumed\texpected %d\tgot %d\n", + tspi->consumed, consumed); + printf(" errtxt\texpected %s\tgot %s\n", + tspi->errtxt, errtxt); + } + } + for (p = vec; *p != NULL; p++) { e = *p; d1 = VNUM(e + 1); diff --git a/vmod/vmod_std_conversions.c b/vmod/vmod_std_conversions.c index 9aaf9a8cf..cbf9c929a 100644 --- a/vmod/vmod_std_conversions.c +++ b/vmod/vmod_std_conversions.c @@ -168,7 +168,7 @@ vmod_integer(VRT_CTX, struct VARGS(integer) *a) if (a->valid_s && a->s != NULL) { p = a->s; - r = SF_Parse_Number(&p, &errtxt); + r = SF_Parse_Number(&p, 1, &errtxt); if (!errno && *p == '\0' && modf(r, &tmp) == 0.0) return (r); r = NAN; @@ -267,7 +267,7 @@ vmod_real(VRT_CTX, struct VARGS(real) *a) if (a->valid_s && a->s != NULL) { p = a->s; - r = SF_Parse_Decimal(&p, &errtxt); + r = SF_Parse_Decimal(&p, 1, &errtxt); if (!errno && *p == '\0') return (r); } From phk at FreeBSD.org Wed Jun 2 11:46:06 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 2 Jun 2021 11:46:06 +0000 (UTC) Subject: [master] 05e10eee3 Fully implement and test non-strict mode SF_Parse() Message-ID: <20210602114606.10876102E75@lists.varnish-cache.org> commit 05e10eee3076e4c416ebd460cfa70f4fa8fcdb67 Author: Poul-Henning Kamp Date: Wed Jun 2 11:44:48 2021 +0000 Fully implement and test non-strict mode SF_Parse() diff --git a/lib/libvarnish/vnum.c b/lib/libvarnish/vnum.c index cf532eaf0..31b2edb7f 100644 --- a/lib/libvarnish/vnum.c +++ b/lib/libvarnish/vnum.c @@ -44,11 +44,15 @@ #include "vas.h" #include "vct.h" -static const char err_miss_num[] = "Missing number"; -static const char err_fatnum[] = "Too many digits"; +/* The distinction between these two is used internally */ static const char err_invalid_num[] = "Invalid number"; +static const char err_no_digits[] = "Invalid number"; + +static const char err_fatnum[] = "Too many digits"; + static const char err_unknown_bytes[] = "Unknown BYTES unit of measurement ([KMGTP][B])"; + static const char err_fractional_bytes[] = "Fractional BYTES not allowed"; #define BAIL(txt) \ @@ -84,7 +88,7 @@ sf_parse_int(const char **ipp, const char **errtxt, int *sign, int maxdig) retval += *(*ipp)++ - 0x30; } if (ndig == 0) - BAIL(*sign < 0 ? err_invalid_num : err_miss_num); + BAIL(err_no_digits); while (vct_isows(*(*ipp))) (*ipp)++; return (retval); @@ -109,11 +113,15 @@ SF_Parse_Number(const char **ipp, int strict, const char **errtxt) retval = (double)sf_parse_int(ipp, errtxt, &sign, 15); if (strict && errno) return (0); + if (*(*ipp) != '.') + return (retval * sign); + if (retval < VRT_DECIMAL_MIN || retval > VRT_DECIMAL_MAX) + BAIL(err_fatnum); + if (*errtxt == err_no_digits && (!vct_isdigit((*ipp)[1]))) + BAIL(err_no_digits); + *errtxt = NULL; + errno = 0; do { - if (*(*ipp) != '.') - break; - if (retval < VRT_DECIMAL_MIN || retval > VRT_DECIMAL_MAX) - BAIL(err_fatnum); (*ipp)++; for(ndig = 0; ndig < 3; ndig++) { scale *= .1; @@ -121,6 +129,8 @@ SF_Parse_Number(const char **ipp, int strict, const char **errtxt) break; retval += scale * (*(*ipp)++ - 0x30); } + if (strict && ndig == 0) + BAIL(err_invalid_num); if (strict && vct_isdigit(*(*ipp))) BAIL(err_fatnum); while (vct_isdigit(*(*ipp))) @@ -332,7 +342,7 @@ VNUM_2bytes(const char *p, uintmax_t *r, uintmax_t rel) const char *errtxt; if (p == NULL || *p == '\0') - return (err_miss_num); + return (err_invalid_num); fval = SF_Parse_Number(&p, 1, &errtxt); if (errno) @@ -364,14 +374,50 @@ static const struct test_sf_parse_int { { "1234", 3, 123, 3, 1, err_fatnum }, { "1234", 4, 1234, 4, 1, NULL }, { "1234", 5, 1234, 4, 1, NULL }, - { "-", 5, 0, 1, -1, err_invalid_num }, + { "-", 5, 0, 1, -1, err_no_digits }, + { " ", 5, 0, 2, 1, err_no_digits }, { "-1234", 3, 123, 4, -1, err_fatnum }, { "-1234", 4, 1234, 5, -1, NULL }, { "-1234", 5, 1234, 5, -1, NULL }, { " -1234", 5, 1234, 6, -1, NULL }, { " -1234 ", 5, 1234, 7, -1, NULL }, { " -12 34 ", 5, 12, 5, -1, NULL }, - { " - 12 34 ", 5, 0, 2, -1, err_invalid_num }, + { " - 12 34 ", 5, 0, 2, -1, err_no_digits }, + { NULL}, +}; + +static const struct test_sf_parse_number { + const char *input; + int strict; + double retval; + int consumed; + const char *errtxt; +} test_sf_parse_number[] = { + { "1234", 1, 1234.000, 4, NULL }, + { " 1234", 1, 1234.000, 5, NULL }, + { " 1234 ", 1, 1234.000, 6, NULL }, + { " 1234. ", 1, 1234.000, 6, err_invalid_num }, + { " 123456789012.0 ", 1, 123456789012.000, 16, NULL }, + { " 1234567890123.0 ", 1, 1234567890123.000, 14, err_fatnum }, + { " 123456789012.123 ", 1, 123456789012.123, 18, NULL }, + { " 123456789012.1234 ",1, 123456789012.123, 17, err_fatnum }, + { " -0.123456 ", 1, .123, 7, err_fatnum }, + { " -.123456 ", 1, 0., 2, err_no_digits }, + { " .123456 ", 1, 0., 1, err_no_digits }, + { " 0. ", 1, 0., 3, err_invalid_num }, + { " .0 ", 1, 0., 1, err_no_digits }, + + { " 123456789012.1234 ",0, 123456789012.123, 19, NULL }, + { " -0.123456 ", 0, -.123, 11, NULL }, + { " -.123456 ", 0, -.123, 10, NULL }, + { " .123456 ", 0, .123, 9, NULL }, + { " 0. ", 0, 0., 4, NULL }, + { " .0 ", 0, 0., 4, NULL }, + { " -0. ", 0, -0., 5, NULL }, + { " -.0 ", 0, -0., 5, NULL }, + { " - ", 0, -0., 2, err_no_digits }, + { " -. ", 0, 0., 2, err_no_digits }, + { " . ", 0, 0., 1, err_no_digits }, { NULL}, }; @@ -425,12 +471,12 @@ static struct test_case { { "3%", (uintmax_t)1024, (uintmax_t)30 }, /* Check the error checks */ - { "", 0, 0, err_miss_num }, + { "", 0, 0, err_invalid_num }, { "-1", 0, 0, err_invalid_num }, { "1.3", 0, 0, err_fractional_bytes}, { "1.5011%", 0, 0, err_fatnum }, - { "-", 0, 0, err_invalid_num }, - { "m", 0, 0, err_miss_num }, + { "-", 0, 0, err_no_digits }, + { "m", 0, 0, err_no_digits }, { "4%", 0, 0, err_unknown_bytes }, { "3*", 0, 0, err_unknown_bytes }, @@ -474,10 +520,14 @@ main(int argc, char *argv[]) const char *e; double d1, d2; const struct test_sf_parse_int *tspi; + const struct test_sf_parse_number *tspn; int64_t i64; + volatile double dbl; int sign, consumed; const char *errtxt; const char *input; + char buf1[30]; + char buf2[30]; (void)argc; @@ -502,11 +552,43 @@ main(int argc, char *argv[]) tspi->sign, sign); printf(" consumed\texpected %d\tgot %d\n", tspi->consumed, consumed); + printf(" errtxt\texpected %p\tgot %p\n", + tspi->errtxt, errtxt); printf(" errtxt\texpected %s\tgot %s\n", tspi->errtxt, errtxt); } } + for (tspn = test_sf_parse_number; tspn->input != NULL; tspn++) { + errtxt = "(unset)"; + input = tspn->input; + dbl = SF_Parse_Number(&input, tspn->strict, &errtxt); + consumed = input - tspn->input; + bprintf(buf1, "%.4f", dbl); + bprintf(buf2, "%.4f", tspn->retval); + if (strcmp(buf1, buf2) || + consumed != tspn->consumed || + errtxt != tspn->errtxt) { + ec++; + printf("sf_parse_number(%s, strict=%d) failed\n", + tspn->input, tspn->strict); + printf(" retval\texpected %.4f\tgot %.4f\t(%e)\n", + tspn->retval, dbl, dbl - tspn->retval); + printf(" retval\texpected %a\tgot %a\n", + tspn->retval, dbl); + printf(" retval\texpected %s\tgot %s\n", + buf2, buf1); + printf(" retval\tdelta %e\n", + dbl - tspn->retval); + printf(" consumed\texpected %d\tgot %d\n", + tspn->consumed, consumed); + printf(" errtxt\texpected %p\tgot %p\n", + tspn->errtxt, errtxt); + printf(" errtxt\texpected %s\tgot %s\n", + tspn->errtxt, errtxt); + } + } + for (p = vec; *p != NULL; p++) { e = *p; d1 = VNUM(e + 1); diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c index d6a30d0bb..9a8fa8283 100644 --- a/lib/libvarnishapi/vsl_query.c +++ b/lib/libvarnishapi/vsl_query.c @@ -122,8 +122,7 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec) const struct vex_rhs *rhs; long long lhs_int = 0; double lhs_float = 0.; - const char *b, *e, *q, *t; - char *p; + const char *b, *e, *q; int i, dq; AN(vex); @@ -208,25 +207,13 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec) return (0); switch (rhs->type) { case VEX_INT: - lhs_int = strtoll(b, &p, 0); - if (*p == '.' || *p == 'e') { - t = ""; /* assume valid float */ - lhs_float = VNUMpfx(b, &q); - if (isnan(lhs_float)) - return (0); - if (q != NULL) - t = (q > p) ? q - 1 : q; - p = TRUST_ME(t); - lhs_int = (long long)lhs_float; - } - if (*p != '\0' && !isspace(*p)) - return (0); /* Can't parse - no match */ + lhs_int = (long long)SF_Parse_Number(&b, 0, &q); + if (errno) + return (0); break; case VEX_FLOAT: - lhs_float = VNUMpfx(b, &q); - if (isnan(lhs_float)) - return (0); - if (q != NULL && q > b && !isspace(q[-1])) + lhs_float = SF_Parse_Decimal(&b, 0, &q); + if (errno) return (0); break; default: diff --git a/vmod/vmod_std_conversions.c b/vmod/vmod_std_conversions.c index cbf9c929a..bcdfac438 100644 --- a/vmod/vmod_std_conversions.c +++ b/vmod/vmod_std_conversions.c @@ -168,7 +168,7 @@ vmod_integer(VRT_CTX, struct VARGS(integer) *a) if (a->valid_s && a->s != NULL) { p = a->s; - r = SF_Parse_Number(&p, 1, &errtxt); + r = SF_Parse_Number(&p, 0, &errtxt); if (!errno && *p == '\0' && modf(r, &tmp) == 0.0) return (r); r = NAN; @@ -267,7 +267,7 @@ vmod_real(VRT_CTX, struct VARGS(real) *a) if (a->valid_s && a->s != NULL) { p = a->s; - r = SF_Parse_Decimal(&p, 1, &errtxt); + r = SF_Parse_Decimal(&p, 0, &errtxt); if (!errno && *p == '\0') return (r); } From nils.goroll at uplex.de Wed Jun 2 13:37:05 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 2 Jun 2021 13:37:05 +0000 (UTC) Subject: [master] 48b9d1bf5 Promote the null blob to vrt Message-ID: <20210602133705.10775106124@lists.varnish-cache.org> commit 48b9d1bf538fe563f505b75043f297d91690caa3 Author: Nils Goroll Date: Wed Jun 2 15:21:12 2021 +0200 Promote the null blob to vrt diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 4939e6dd5..7b334e4e2 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -60,6 +60,11 @@ const struct strands *vrt_null_strands = &(struct strands){ .n = 0, .p = (const char *[1]){NULL} }; +const struct vrt_blob *vrt_null_blob = &(struct vrt_blob){ + .type = VRT_NULL_BLOB_TYPE, + .len = 0, + .blob = "\0" +}; /*--------------------------------------------------------------------*/ diff --git a/include/vrt.h b/include/vrt.h index 9d2c87e98..e2c3646c9 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -55,6 +55,8 @@ * * NEXT (2021-09-15) * vrt_null_strands added + * vrt_null_blob added + * VRT_NULL_BLOB_TYPE added as the .type of vrt_null_blob * 13.0 (2021-03-15) * Move VRT_synth_page() to deprecated status * Add VRT_synth_strands() and VRT_synth_blob() @@ -271,6 +273,9 @@ struct vrt_blob { const void *blob; }; +#define VRT_NULL_BLOB_TYPE 0xfade4fa0 +extern const struct vrt_blob *vrt_null_blob; + /*********************************************************************** * This is the central definition of the mapping from VCL types to * C-types. The python scripts read these from here. diff --git a/vmod/vmod_blob.c b/vmod/vmod_blob.c index 8274ea6f3..acf7469bd 100644 --- a/vmod/vmod_blob.c +++ b/vmod/vmod_blob.c @@ -114,13 +114,6 @@ static const struct vmod_blob_fptr { static char empty[1] = { '\0' }; -static const struct vrt_blob null_blob[1] = {{ -#define VMOD_BLOB_NULL_TYPE 0xfade4fa0 - .type = VMOD_BLOB_NULL_TYPE, - .len = 0, - .blob = empty, -}}; - static enum encoding parse_encoding(VCL_ENUM e) { @@ -235,7 +228,7 @@ vmod_blob__init(VRT_CTX, struct vmod_blob_blob **blobp, const char *vcl_name, } if (len == 0) { free(buf); - memcpy(&b->blob, null_blob, sizeof b->blob); + memcpy(&b->blob, vrt_null_blob, sizeof b->blob); return; } b->blob.len = len; @@ -359,7 +352,7 @@ vmod_decode(VRT_CTX, VCL_ENUM decs, VCL_INT length, VCL_STRANDS strings) } if (len == 0) { WS_Release(ctx->ws, 0); - return (null_blob); + return (vrt_null_blob); } WS_Release(ctx->ws, len); @@ -547,7 +540,7 @@ vmod_sub(VRT_CTX, VCL_BLOB b, VCL_BYTES n, VCL_BYTES off) } if (n == 0) - return (null_blob); + return (vrt_null_blob); return (VRT_blob(ctx, "blob.sub", From nils.goroll at uplex.de Wed Jun 2 13:37:05 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 2 Jun 2021 13:37:05 +0000 (UTC) Subject: [master] 0a4884c6b VRT_blob() to return vrt_null_blob for empty blobs Message-ID: <20210602133705.26F1B106127@lists.varnish-cache.org> commit 0a4884c6b1bba370aa0fa3c1cdf6570fe82d539f Author: Nils Goroll Date: Wed Jun 2 15:25:36 2021 +0200 VRT_blob() to return vrt_null_blob for empty blobs diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 7b334e4e2..d6f982b1d 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -1163,6 +1163,9 @@ VRT_blob(VRT_CTX, const char *err, const void *src, size_t len, unsigned type) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->ws, WS_MAGIC); + if (src == NULL || len == 0) + return (vrt_null_blob); + p = (void *)WS_Alloc(ctx->ws, sizeof *p); if (p == NULL) { VRT_fail(ctx, "Workspace overflow (%s)", err); diff --git a/include/vrt.h b/include/vrt.h index e2c3646c9..012d8a6ec 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -57,6 +57,8 @@ * vrt_null_strands added * vrt_null_blob added * VRT_NULL_BLOB_TYPE added as the .type of vrt_null_blob + * VRT_blob() changed to return vrt_null_blob for + * len == 0 or src == NULL arguments * 13.0 (2021-03-15) * Move VRT_synth_page() to deprecated status * Add VRT_synth_strands() and VRT_synth_blob() diff --git a/vmod/vmod_blob.c b/vmod/vmod_blob.c index acf7469bd..98a1527cd 100644 --- a/vmod/vmod_blob.c +++ b/vmod/vmod_blob.c @@ -539,10 +539,6 @@ vmod_sub(VRT_CTX, VCL_BLOB b, VCL_BYTES n, VCL_BYTES off) return (NULL); } - if (n == 0) - return (vrt_null_blob); - - return (VRT_blob(ctx, "blob.sub", (const char *)b->blob + off, n, b->type)); } From phk at FreeBSD.org Wed Jun 2 15:17:05 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 2 Jun 2021 15:17:05 +0000 (UTC) Subject: [master] 9b1f1d170 Fix PeekToken(): Should never be called with EOI token, and thus cannot return NULL. Message-ID: <20210602151705.39FD5108C61@lists.varnish-cache.org> commit 9b1f1d170781ffa8a881b46455327ca9d7d5c514 Author: Poul-Henning Kamp Date: Wed Jun 2 15:11:55 2021 +0000 Fix PeekToken(): Should never be called with EOI token, and thus cannot return NULL. diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index d7e4a63b3..724d5bfe8 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -457,7 +457,7 @@ vcc_acl_entry(struct vcc *tl) */ static void -vcc_acl_emit_tokens(struct vcc *tl, const struct acl_e *ae) +vcc_acl_emit_tokens(const struct vcc *tl, const struct acl_e *ae) { struct token *t; const char *sep = ""; @@ -490,7 +490,7 @@ vcc_acl_emit_tokens(struct vcc *tl, const struct acl_e *ae) */ static unsigned -vcc_acl_emit_tables(struct vcc *tl, unsigned n, const char *name) +vcc_acl_emit_tables(const struct vcc *tl, unsigned n, const char *name) { struct acl_e *ae; unsigned rv = sizeof(ae->data) + 3; diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 40c46ca5d..3f3ccc3c0 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -415,13 +415,13 @@ void vcc_Warn(struct vcc *); void vcc__Expect(struct vcc *tl, unsigned tok, unsigned line); int vcc_IdIs(const struct token *t, const char *p); -void vcc_PrintTokens(struct vcc *tl, const struct token *tb, +void vcc_PrintTokens(const struct vcc *tl, const struct token *tb, const struct token *te); void vcc_ExpectVid(struct vcc *tl, const char *what); void vcc_Lexer(struct vcc *tl, struct source *sp); void vcc_NextToken(struct vcc *tl); -struct token * vcc_PeekToken(struct vcc *tl); -struct token * vcc_PeekTokenFrom(struct vcc *tl, const struct token *t); +struct token * vcc_PeekToken(const struct vcc *tl); +struct token * vcc_PeekTokenFrom(const struct vcc *tl, const struct token *t); void vcc__ErrInternal(struct vcc *tl, const char *func, unsigned line); diff --git a/lib/libvcc/vcc_token.c b/lib/libvcc/vcc_token.c index b965c298d..77ce5c075 100644 --- a/lib/libvcc/vcc_token.c +++ b/lib/libvcc/vcc_token.c @@ -264,24 +264,20 @@ vcc_ErrWhere(struct vcc *tl, const struct token *t) /*--------------------------------------------------------------------*/ struct token * -vcc_PeekTokenFrom(struct vcc *tl, const struct token *t) +vcc_PeekTokenFrom(const struct vcc *tl, const struct token *t) { - struct token *tn; + struct token *t2; CHECK_OBJ_NOTNULL(tl, VCC_MAGIC); AN(t); - tn = VTAILQ_NEXT(t, list); - if (tn == NULL) { - VSB_cat(tl->sb, - "Ran out of input, something is missing or" - " maybe unbalanced (...) or {...}\n"); - tl->err = 1; - } - return (tn); + assert(t->tok != EOI); + t2 = VTAILQ_NEXT(t, list); + AN(t2); + return (t2); } struct token * -vcc_PeekToken(struct vcc *tl) +vcc_PeekToken(const struct vcc *tl) { CHECK_OBJ_NOTNULL(tl, VCC_MAGIC); @@ -330,7 +326,8 @@ vcc_IdIs(const struct token *t, const char *p) */ void -vcc_PrintTokens(struct vcc *tl, const struct token *tb, const struct token *te) +vcc_PrintTokens(const struct vcc *tl, + const struct token *tb, const struct token *te) { CHECK_OBJ_NOTNULL(tl, VCC_MAGIC); From nils.goroll at uplex.de Wed Jun 2 15:22:05 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 2 Jun 2021 15:22:05 +0000 (UTC) Subject: [master] 2bc56a4b7 vcc: Add missing ERRCHK to vcc_expr5() Message-ID: <20210602152205.20FA5108FAD@lists.varnish-cache.org> commit 2bc56a4b789fb770548aa6d528c8c8d6d296d4a1 Author: Nils Goroll Date: Wed Jun 2 17:09:27 2021 +0200 vcc: Add missing ERRCHK to vcc_expr5() VCC_SymbolGet() could fail, e,g, if the token is reserved Ref cf396384595ef47cc66f1eedcf0b6730d5b1b659 diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 55d311ea8..b41395aff 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -745,6 +745,7 @@ vcc_expr5(struct vcc *tl, struct expr **e, vcc_type_t fmt) if (sym == NULL && fmt->global_pfx != NULL && t1->tok != '.') { sym = VCC_SymbolGet(tl, SYM_MAIN, SYM_NONE, SYMTAB_CREATE, XREF_REF); + ERRCHK(tl); AN(sym); VCC_GlobalSymbol(sym, fmt); } From dridi at varni.sh Wed Jun 2 15:30:06 2021 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 2 Jun 2021 15:30:06 +0000 Subject: [master] 2bc56a4b7 vcc: Add missing ERRCHK to vcc_expr5() In-Reply-To: <20210602152205.20FA5108FAD@lists.varnish-cache.org> References: <20210602152205.20FA5108FAD@lists.varnish-cache.org> Message-ID: On Wed, Jun 2, 2021 at 3:22 PM Nils Goroll wrote: > > > commit 2bc56a4b789fb770548aa6d528c8c8d6d296d4a1 > Author: Nils Goroll > Date: Wed Jun 2 17:09:27 2021 +0200 > > vcc: Add missing ERRCHK to vcc_expr5() > > VCC_SymbolGet() could fail, e,g, if the token is reserved That's not supposed to happen, just before the symbol creation we probed the symbol table and a reserved symbol should have been returned. It would have then failed to satisfy the expression type. > Ref cf396384595ef47cc66f1eedcf0b6730d5b1b659 Actually, since 9b1f1d170781ffa8a881b46455327ca9d7d5c514 we should actively be removing lots of ERRCHK() now that vcc_PeekToken*() and vcc_NextToken() functions are fail-safe. > diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c > index 55d311ea8..b41395aff 100644 > --- a/lib/libvcc/vcc_expr.c > +++ b/lib/libvcc/vcc_expr.c > @@ -745,6 +745,7 @@ vcc_expr5(struct vcc *tl, struct expr **e, vcc_type_t fmt) > if (sym == NULL && fmt->global_pfx != NULL && t1->tok != '.') { > sym = VCC_SymbolGet(tl, SYM_MAIN, SYM_NONE, > SYMTAB_CREATE, XREF_REF); > + ERRCHK(tl); We should rather AZ(tl->err) here, or do nothing. Trying to create a symbol that already exists should fail. > AN(sym); > VCC_GlobalSymbol(sym, fmt); > } > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From phk at FreeBSD.org Wed Jun 2 15:53:05 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 2 Jun 2021 15:53:05 +0000 (UTC) Subject: [master] e109aab02 Flexelinting Message-ID: <20210602155305.293DE109F00@lists.varnish-cache.org> commit e109aab0291413d75514105d5372daa4b31bba3e Author: Poul-Henning Kamp Date: Wed Jun 2 15:51:48 2021 +0000 Flexelinting diff --git a/vmod/vmod_std_conversions.c b/vmod/vmod_std_conversions.c index bcdfac438..662aec4b5 100644 --- a/vmod/vmod_std_conversions.c +++ b/vmod/vmod_std_conversions.c @@ -170,7 +170,7 @@ vmod_integer(VRT_CTX, struct VARGS(integer) *a) p = a->s; r = SF_Parse_Number(&p, 0, &errtxt); if (!errno && *p == '\0' && modf(r, &tmp) == 0.0) - return (r); + return ((VCL_INT)r); r = NAN; } From phk at FreeBSD.org Wed Jun 2 15:56:05 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 2 Jun 2021 15:56:05 +0000 (UTC) Subject: [master] 020614503 Dont penalize our code coverage with never executed debugging output. Message-ID: <20210602155605.C8BF810A19D@lists.varnish-cache.org> commit 020614503901c0c4094b95d13c33e9bf7f6cffbb Author: Poul-Henning Kamp Date: Wed Jun 2 15:54:34 2021 +0000 Dont penalize our code coverage with never executed debugging output. diff --git a/lib/libvarnish/vnum.c b/lib/libvarnish/vnum.c index 31b2edb7f..09ae56a33 100644 --- a/lib/libvarnish/vnum.c +++ b/lib/libvarnish/vnum.c @@ -360,7 +360,8 @@ VNUM_2bytes(const char *p, uintmax_t *r, uintmax_t rel) #ifdef NUM_C_TEST /* * Compile with: - * cc -o foo -DNUM_C_TEST -I../.. -I../../include vnum.c vas.c vct.c -lm + * cc -o foo -DNUM_C_TEST -DTEST_VERBOSE \ + * -I../.. -I../../include vnum.c vas.c vct.c -lm */ static const struct test_sf_parse_int { @@ -546,6 +547,7 @@ main(int argc, char *argv[]) ec++; printf("sf_parse_int(%s, maxdig=%d) failed\n", tspi->input, tspi->maxdig); +#ifdef TEST_VERBOSE printf(" retval\texpected %jd\tgot %jd\n", (intmax_t)tspi->retval, (intmax_t)i64); printf(" sign\texpected %d\tgot %d\n", @@ -556,6 +558,7 @@ main(int argc, char *argv[]) tspi->errtxt, errtxt); printf(" errtxt\texpected %s\tgot %s\n", tspi->errtxt, errtxt); +#endif } } @@ -572,6 +575,7 @@ main(int argc, char *argv[]) ec++; printf("sf_parse_number(%s, strict=%d) failed\n", tspn->input, tspn->strict); +#ifdef TEST_VERBOSE printf(" retval\texpected %.4f\tgot %.4f\t(%e)\n", tspn->retval, dbl, dbl - tspn->retval); printf(" retval\texpected %a\tgot %a\n", @@ -586,6 +590,7 @@ main(int argc, char *argv[]) tspn->errtxt, errtxt); printf(" errtxt\texpected %s\tgot %s\n", tspn->errtxt, errtxt); +#endif } } From nils.goroll at uplex.de Wed Jun 2 16:56:04 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 2 Jun 2021 16:56:04 +0000 (UTC) Subject: [master] 2f8e02a3d Check VCL_HEADER vmod arguments Message-ID: <20210602165604.B447610DDBD@lists.varnish-cache.org> commit 2f8e02a3d52d47864ba8bd453f7b7182c0e20472 Author: Nils Goroll Date: Wed Jun 2 18:23:59 2021 +0200 Check VCL_HEADER vmod arguments When a VCL_HEADER argument is a vmod return value, it could be - NULL for an error condition - from a different context Also, we should not assert on HDR_OBJ in VRT_selecthttp() to facilitate error handling in vmods. Fixes #3623 diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index d6f982b1d..1289c1ac2 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -257,6 +257,9 @@ VRT_selecthttp(VRT_CTX, enum gethdr_e where) case HDR_RESP: hp = ctx->http_resp; break; + case HDR_OBJ: + hp = NULL; + break; default: WRONG("VRT_selecthttp 'where' invalid"); } diff --git a/bin/varnishtest/tests/m00006.vtc b/bin/varnishtest/tests/m00006.vtc index 24ad5f934..1004277c5 100644 --- a/bin/varnishtest/tests/m00006.vtc +++ b/bin/varnishtest/tests/m00006.vtc @@ -13,6 +13,15 @@ varnish v1 -vcl+backend { sub vcl_recv { std.collect(req.http.foo); } + sub vcl_hash { + hash_data("/"); + return (lookup); + } + sub vcl_hit { + if (req.url == "/obj") { + std.collect(obj.http.foo); + } + } sub vcl_backend_fetch { std.collect(bereq.http.baz); } @@ -29,4 +38,7 @@ client c1 { rxresp expect resp.http.bar == "a, b" expect resp.http.qux == "c; d" + txreq -url "/obj" + rxresp + expect resp.status == 503 } -run diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index fa75dd130..f75016da0 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -789,26 +789,34 @@ xyzzy_collect(VRT_CTX, VCL_STRANDS s) /* cf. VRT_SetHdr() */ VCL_VOID -xyzzy_sethdr(VRT_CTX, VCL_HEADER hs, VCL_STRANDS s) +xyzzy_sethdr(VRT_CTX, VCL_HEADER hdr, VCL_STRANDS s) { struct http *hp; const char *b; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - AN(hs); - AN(hs->what); - hp = VRT_selecthttp(ctx, hs->where); + if (hdr == NULL) { + VRT_fail(ctx, "debug.sethdr(): header argument is NULL"); + return; + } + hp = VRT_selecthttp(ctx, hdr->where); + if (hp == NULL) { + VRT_fail(ctx, "debug.sethdr(): header argument " + "can not be used here"); + return; + } + AN(hdr->what); CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); if (s->n == 0) { - http_Unset(hp, hs->what); + http_Unset(hp, hdr->what); } else { - b = VRT_StrandsWS(hp->ws, hs->what + 1, s); + b = VRT_StrandsWS(hp->ws, hdr->what + 1, s); if (b == NULL) { - VSLb(ctx->vsl, SLT_LostHeader, "%s", hs->what + 1); + VSLb(ctx->vsl, SLT_LostHeader, "%s", hdr->what + 1); } else { if (*b != '\0') WS_Assert_Allocated(hp->ws, b, strlen(b) + 1); - http_Unset(hp, hs->what); + http_Unset(hp, hdr->what); http_SetHeader(hp, b); } } diff --git a/vmod/vmod_std.c b/vmod/vmod_std.c index bb24e04ce..719c5051a 100644 --- a/vmod/vmod_std.c +++ b/vmod/vmod_std.c @@ -182,7 +182,16 @@ vmod_collect(VRT_CTX, VCL_HEADER hdr, VCL_STRING sep) struct http *hp; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + if (hdr == NULL) { + VRT_fail(ctx, "std.collect(): header argument is NULL"); + return; + } hp = VRT_selecthttp(ctx, hdr->where); + if (hp == NULL) { + VRT_fail(ctx, "std.collect(): header argument " + "can not be used here"); + return; + } http_CollectHdrSep(hp, hdr->what, sep); } diff --git a/vmod/vmod_std.vcc b/vmod/vmod_std.vcc index e9812ee8e..fd0bf1a4a 100644 --- a/vmod/vmod_std.vcc +++ b/vmod/vmod_std.vcc @@ -73,6 +73,8 @@ headers, with an additional whitespace for pretty printing. Care should be taken when collapsing headers. In particular collapsing ``Set-Cookie`` will lead to unexpected results on the browser side. +Using *hdr* from ``obj.http`` triggers a VCL failure. + Examples:: std.collect(req.http.accept); From dridi.boukelmoune at gmail.com Thu Jun 3 06:28:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 3 Jun 2021 06:28:06 +0000 (UTC) Subject: [master] 903ea6039 Revert "vcc: Add missing ERRCHK to vcc_expr5()" Message-ID: <20210603062806.1836F6421E@lists.varnish-cache.org> commit 903ea60399cf4480e8d1550323bab6bbada67d51 Author: Dridi Boukelmoune Date: Thu Jun 3 08:19:57 2021 +0200 Revert "vcc: Add missing ERRCHK to vcc_expr5()" This reverts commit 2bc56a4b789fb770548aa6d528c8c8d6d296d4a1. We already assert that sym is not null, and we certainly don't expect a reserved symbol to have slipped through since we already attempted the least specific symnbol lookup. This is not a case where we expect to fail because of user input. diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index b41395aff..55d311ea8 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -745,7 +745,6 @@ vcc_expr5(struct vcc *tl, struct expr **e, vcc_type_t fmt) if (sym == NULL && fmt->global_pfx != NULL && t1->tok != '.') { sym = VCC_SymbolGet(tl, SYM_MAIN, SYM_NONE, SYMTAB_CREATE, XREF_REF); - ERRCHK(tl); AN(sym); VCC_GlobalSymbol(sym, fmt); } From dridi.boukelmoune at gmail.com Thu Jun 3 06:43:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 3 Jun 2021 06:43:05 +0000 (UTC) Subject: [master] 3730f544d vcc: Promote error checks to assertions Message-ID: <20210603064305.D938E64B18@lists.varnish-cache.org> commit 3730f544db89ab487c0e0e775969edf9c12f63b6 Author: Dridi Boukelmoune Date: Thu Jun 3 08:39:32 2021 +0200 vcc: Promote error checks to assertions Introduced by eabe58bc6633012dd01ba8f2a131f4e8cd351b2e but soon after made obsolete by 9b1f1d170781ffa8a881b46455327ca9d7d5c514. diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index 724d5bfe8..557b3a21f 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -477,7 +477,6 @@ vcc_acl_emit_tokens(const struct vcc *tl, const struct acl_e *ae) if (t == ae->t_mask) break; t = vcc_PeekTokenFrom(tl, t); - ERRCHK(tl); AN(t); sep = " "; } while (ae->t_mask != NULL); diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 55d311ea8..e85111178 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -739,7 +739,7 @@ vcc_expr5(struct vcc *tl, struct expr **e, vcc_type_t fmt) case ID: t = tl->t; t1 = vcc_PeekToken(tl); - ERRCHK(tl); + AN(t1); sym = VCC_SymbolGet(tl, SYM_MAIN, SYM_NONE, SYMTAB_PARTIAL_NOERR, XREF_REF); if (sym == NULL && fmt->global_pfx != NULL && t1->tok != '.') { diff --git a/lib/libvcc/vcc_token.c b/lib/libvcc/vcc_token.c index 77ce5c075..82ab2885f 100644 --- a/lib/libvcc/vcc_token.c +++ b/lib/libvcc/vcc_token.c @@ -350,15 +350,15 @@ vcc_ExpectVid(struct vcc *tl, const char *what) ERRCHK(tl); t2 = vcc_PeekToken(tl); - ERRCHK(tl); + AN(t2); while (t2->tok == '.') { bad = "."; t2 = vcc_PeekTokenFrom(tl, t2); - ERRCHK(tl); + AN(t2); if (t2->tok != ID) break; t2 = vcc_PeekTokenFrom(tl, t2); - ERRCHK(tl); + AN(t2); } if (bad == NULL) bad = VCT_invalid_name(tl->t->b, tl->t->e); diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index d0719ec47..496164f23 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -316,7 +316,7 @@ vcc_ParseImport(struct vcc *tl) ExpectErr(tl, ID); /* "vmod_name" */ mod = tl->t; tmod = vcc_PeekTokenFrom(tl, mod); - ERRCHK(tl); + AN(tmod); if (tmod->tok == ID && vcc_IdIs(tmod, "as")) { vcc_NextToken(tl); /* "vmod_name" */ vcc_NextToken(tl); /* "as" */ diff --git a/lib/libvcc/vcc_xref.c b/lib/libvcc/vcc_xref.c index cf2be5532..3e5573eb6 100644 --- a/lib/libvcc/vcc_xref.c +++ b/lib/libvcc/vcc_xref.c @@ -124,7 +124,7 @@ vcc_AddUses(struct vcc *tl, const struct token *t1, const struct token *t2, pu->t2 = t2; if (pu->t2 == NULL) { pu->t2 = vcc_PeekTokenFrom(tl, t1); - ERRCHK(tl); + AN(pu->t2); } pu->sym = sym; pu->use = use; From nils.goroll at uplex.de Thu Jun 3 06:46:20 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 3 Jun 2021 08:46:20 +0200 Subject: [master] 903ea6039 Revert "vcc: Add missing ERRCHK to vcc_expr5()" In-Reply-To: <20210603062806.1836F6421E@lists.varnish-cache.org> References: <20210603062806.1836F6421E@lists.varnish-cache.org> Message-ID: <4da8402f-2fe4-5900-f8bc-ffe196cf9ad0@uplex.de> I should have added the test case straight away, but I wrongly assumed the case would be obvious I noticed it working on #3622 where it happens with acls being evaluated as an expression, but there is also a case without that PR: commit 581c5b48b13192e4d1ed8935e79cdd10aeab3516 Author: Nils Goroll Date: Thu Jun 3 08:44:58 2021 +0200 vtc for dridi diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc index 47b899585..a12f3295e 100644 --- a/bin/varnishtest/tests/v00020.vtc +++ b/bin/varnishtest/tests/v00020.vtc @@ -42,6 +42,14 @@ varnish v1 -errvcl {Symbol 'acl' type (reserved) can not be used in expression.} sub vcl_recv { call acl; } } +varnish v1 -errvcl {Symbols named 'vcl_*' are reserved.} { + import debug; + + sub vcl_recv { + debug.test_probe(vcl_sub1, vcl_sub2); + } +} + varnish v1 -errvcl {Operator * not possible on type STRING.} { sub vcl_recv { set req.http.foo = "bla" * "foo"; I think we should emit a proper error here rather than failing an assertion: ** top === varnish v1 -errvcl {Symbols named 'vcl_*' are reserved.} { **** v1 CLI TX|vcl.inline vcl7 << %XJEIFLH|)Xspa8P **** v1 CLI TX|vcl 4.1; **** v1 CLI TX| **** v1 CLI TX|\timport debug; **** v1 CLI TX| **** v1 CLI TX|\tsub vcl_recv { **** v1 CLI TX|\t\tdebug.test_probe(vcl_sub1, vcl_sub2); **** v1 CLI TX|\t} **** v1 CLI TX| **** v1 CLI TX|%XJEIFLH|)Xspa8P **** dT 0.491 *** v1 debug|Could not delete 'vcl_vcl7.1622702730.393641/vgc.sym': No such file or directory *** v1 CLI RX 106 **** v1 CLI RX|Message from VCC-compiler: **** v1 CLI RX|Assert error in vcc_expr5(), vcc_expr.c line 748: **** v1 CLI RX| Condition((sym) != 0) not true. **** v1 CLI RX|Running VCC-compiler failed, signal 6 **** v1 CLI RX|VCL compilation failed ** v1 VCL compilation failed (as expected) ---- v1 Did not find expected string: ("Symbols named 'vcl_*' are reserved.") On 03/06/2021 08:28, Dridi Boukelmoune wrote: > > commit 903ea60399cf4480e8d1550323bab6bbada67d51 > Author: Dridi Boukelmoune > Date: Thu Jun 3 08:19:57 2021 +0200 > > Revert "vcc: Add missing ERRCHK to vcc_expr5()" > > This reverts commit 2bc56a4b789fb770548aa6d528c8c8d6d296d4a1. > > We already assert that sym is not null, and we certainly don't expect a > reserved symbol to have slipped through since we already attempted the > least specific symnbol lookup. > > This is not a case where we expect to fail because of user input. > > diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c > index b41395aff..55d311ea8 100644 > --- a/lib/libvcc/vcc_expr.c > +++ b/lib/libvcc/vcc_expr.c > @@ -745,7 +745,6 @@ vcc_expr5(struct vcc *tl, struct expr **e, vcc_type_t fmt) > if (sym == NULL && fmt->global_pfx != NULL && t1->tok != '.') { > sym = VCC_SymbolGet(tl, SYM_MAIN, SYM_NONE, > SYMTAB_CREATE, XREF_REF); > - ERRCHK(tl); > AN(sym); > VCC_GlobalSymbol(sym, fmt); > } > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit > -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From nils.goroll at uplex.de Thu Jun 3 07:05:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 3 Jun 2021 07:05:06 +0000 (UTC) Subject: [master] 0e86a1ed8 vcc: Add missing ERRCHK to vcc_expr5() Message-ID: <20210603070506.BA54265998@lists.varnish-cache.org> commit 0e86a1ed81fc6445a037f9425c76d8a3898537fc Author: Nils Goroll Date: Wed Jun 2 17:09:27 2021 +0200 vcc: Add missing ERRCHK to vcc_expr5() VCC_SymbolGet() could fail, e,g, if the token is reserved Now with vtc to make the issue more obvious. OKed by Dridi via IRC Ref cf396384595ef47cc66f1eedcf0b6730d5b1b659 diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc index 47b899585..a12f3295e 100644 --- a/bin/varnishtest/tests/v00020.vtc +++ b/bin/varnishtest/tests/v00020.vtc @@ -42,6 +42,14 @@ varnish v1 -errvcl {Symbol 'acl' type (reserved) can not be used in expression.} sub vcl_recv { call acl; } } +varnish v1 -errvcl {Symbols named 'vcl_*' are reserved.} { + import debug; + + sub vcl_recv { + debug.test_probe(vcl_sub1, vcl_sub2); + } +} + varnish v1 -errvcl {Operator * not possible on type STRING.} { sub vcl_recv { set req.http.foo = "bla" * "foo"; diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index e85111178..fb5220e4e 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -745,6 +745,7 @@ vcc_expr5(struct vcc *tl, struct expr **e, vcc_type_t fmt) if (sym == NULL && fmt->global_pfx != NULL && t1->tok != '.') { sym = VCC_SymbolGet(tl, SYM_MAIN, SYM_NONE, SYMTAB_CREATE, XREF_REF); + ERRCHK(tl); AN(sym); VCC_GlobalSymbol(sym, fmt); } From phk at FreeBSD.org Thu Jun 3 10:26:05 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 3 Jun 2021 10:26:05 +0000 (UTC) Subject: [master] 9eff155de Replace VNUMpfx() by SF_Parse_Number() Message-ID: <20210603102605.DA02C104056@lists.varnish-cache.org> commit 9eff155de95f419733812e2e8311e3b7a3ad15e4 Author: Poul-Henning Kamp Date: Thu Jun 3 10:25:17 2021 +0000 Replace VNUMpfx() by SF_Parse_Number() diff --git a/lib/libvarnish/vnum.c b/lib/libvarnish/vnum.c index 09ae56a33..6f02fe3a5 100644 --- a/lib/libvarnish/vnum.c +++ b/lib/libvarnish/vnum.c @@ -154,47 +154,7 @@ SF_Parse_Decimal(const char **ipp, int strict, const char **errtxt) return (retval); } -/********************************************************************** - * Convert (all of!) a string to a floating point number, and if we can - * not, return NAN. - */ - -double -VNUMpfx(const char *p, const char **t) -{ - double m = 0., ee = 0.; - double ms = 1.0; - double es = 1.0, e = 1.0, ne = 0.0; - - AN(p); - AN(t); - *t = NULL; - while (vct_issp(*p)) - p++; - - if (*p == '-' || *p == '+') - ms = (*p++ == '-' ? -1.0 : 1.0); - - for (; *p != '\0'; p++) { - if (vct_isdigit(*p)) { - m *= 10.; - m += *p - '0'; - e = ne; - if (e) - ne = e - 1.0; - } else if (*p == '.' && ne == 0.0) { - ne = -1.0; - } else - break; - } - if (e > 0.0) - return (nan("")); // No digits - while (vct_issp(*p)) - p++; - if (*p != '\0') - *t = p; - return (ms * m * pow(10., e + es * ee)); -} +/**********************************************************************/ double VNUM(const char *p) @@ -202,8 +162,8 @@ VNUM(const char *p) const char *t; double r; - r = VNUMpfx(p, &t); - if (t != NULL) + r = SF_Parse_Number(&p, 0, &t); + if (errno || *p != '\0') r = nan(""); return (r); } @@ -268,12 +228,12 @@ VNUM_duration(const char *p) if (p == NULL) return (nan("")); - r = VNUMpfx(p, &t); + r = SF_Parse_Number(&p, 0, &t); - if (isnan(r) || t == NULL) + if (errno) return (nan("")); - return (VNUM_duration_unit(r, t, NULL)); + return (VNUM_duration_unit(r, p, NULL)); } /**********************************************************************/ @@ -634,8 +594,9 @@ main(int argc, char *argv[]) ++ec; } d1 = VNUM_duration(" 365.24219d "); - if (d1 < 31556925.2159 || d1 > 31556925.2161) { - printf("%s: VNUM_Duration() wrong: %g\n", *argv, d1); + if (d1 != 31556908.8) { + printf("%s: VNUM_Duration() wrong, %.3f delta = %e\n", + *argv, d1, d1 - 31556908.8); ++ec; } /* TODO: test invalid strings */ From phk at FreeBSD.org Fri Jun 4 08:11:08 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 4 Jun 2021 08:11:08 +0000 (UTC) Subject: [master] ae9dfb849 Commentary. Make errtxt optional arg. Message-ID: <20210604081108.3A89A101711@lists.varnish-cache.org> commit ae9dfb849da3a1b6bb9b679cd0bafbb2d12a2608 Author: Poul-Henning Kamp Date: Fri Jun 4 07:53:13 2021 +0000 Commentary. Make errtxt optional arg. diff --git a/lib/libvarnish/vnum.c b/lib/libvarnish/vnum.c index 6f02fe3a5..c6c59e0ba 100644 --- a/lib/libvarnish/vnum.c +++ b/lib/libvarnish/vnum.c @@ -27,7 +27,8 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * Deal with numbers with data storage suffix scaling + * Deal with numbers. + * */ #include "config.h" @@ -63,6 +64,11 @@ static const char err_fractional_bytes[] = "Fractional BYTES not allowed"; return (retval); \ } while (0) +/* + * Internal function for parsing an integer with a limited + * number of digits. + */ + static int64_t sf_parse_int(const char **ipp, const char **errtxt, int *sign, int maxdig) { @@ -71,7 +77,8 @@ sf_parse_int(const char **ipp, const char **errtxt, int *sign, int maxdig) AN(ipp); AN(*ipp); - *errtxt = NULL; + if (errtxt != NULL) + *errtxt = NULL; *sign = 1; errno = 0; while (vct_isows(*(*ipp))) @@ -94,6 +101,14 @@ sf_parse_int(const char **ipp, const char **errtxt, int *sign, int maxdig) return (retval); } +/********************************************************************** + * Parse a RFC8941 `sf-integer`. + * + * If `errno` is non-zero the conversion failed. + * If `errtxt` is provided it summarily tells why. + * The input argument points to the first character not consumed. + */ + int64_t SF_Parse_Integer(const char **ipp, const char **errtxt) { @@ -104,6 +119,14 @@ SF_Parse_Integer(const char **ipp, const char **errtxt) return(retval * sign); } +/********************************************************************** + * Parse either a RFC8941 `sf-integer` or `sf-decimal`. + * + * If `errno` is non-zero the conversion failed. + * If `errtxt` is provided it summarily tells why. + * The input argument points to the first character not consumed. + */ + double SF_Parse_Number(const char **ipp, int strict, const char **errtxt) { @@ -141,6 +164,14 @@ SF_Parse_Number(const char **ipp, int strict, const char **errtxt) return (retval * sign); } +/********************************************************************** + * Parse a RFC8941 `sf-decimal`. + * + * If `errno` is non-zero the conversion failed. + * If `errtxt` is provided it summarily tells why. + * The input argument points to the first character not consumed. + */ + double SF_Parse_Decimal(const char **ipp, int strict, const char **errtxt) { @@ -154,7 +185,12 @@ SF_Parse_Decimal(const char **ipp, int strict, const char **errtxt) return (retval); } -/**********************************************************************/ +/********************************************************************** + * Parse a "Varnish number". + * + * Varnish numbers are the union of RFC8941 sf-integer and sf-decimal. + * If `errno` is non-zero the conversion failed and NAN is returned. + */ double VNUM(const char *p) From phk at FreeBSD.org Fri Jun 4 08:11:08 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 4 Jun 2021 08:11:08 +0000 (UTC) Subject: [master] 2e846937e Update API info, remove VNUMpfx prototype. Message-ID: <20210604081108.4F421101714@lists.varnish-cache.org> commit 2e846937e11c8626f71f31063fd3019af378d3ac Author: Poul-Henning Kamp Date: Fri Jun 4 08:09:07 2021 +0000 Update API info, remove VNUMpfx prototype. diff --git a/include/vnum.h b/include/vnum.h index 6016d325e..cd08c3216 100644 --- a/include/vnum.h +++ b/include/vnum.h @@ -32,7 +32,6 @@ /* from libvarnish/vnum.c */ double VNUM(const char *p); -double VNUMpfx(const char *p, const char **e); vtim_dur VNUM_duration_unit(vtim_dur r, const char *b, const char *e); vtim_dur VNUM_duration(const char *p); int64_t VNUM_bytes_unit(double r, const char *b, const char *e, uintmax_t rel, diff --git a/include/vrt.h b/include/vrt.h index 012d8a6ec..e61156401 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -54,6 +54,7 @@ * binary/load-time compatible, increment MAJOR version * * NEXT (2021-09-15) + * VNUMpfx() removed, SF_Parse_{Integer|Decimal|Number} added * vrt_null_strands added * vrt_null_blob added * VRT_NULL_BLOB_TYPE added as the .type of vrt_null_blob From phk at FreeBSD.org Fri Jun 4 08:11:08 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 4 Jun 2021 08:11:08 +0000 (UTC) Subject: [master] f18fe9cba Use SF_Parse_Decimal() for double style parameters Message-ID: <20210604081108.6AF92101717@lists.varnish-cache.org> commit f18fe9cba3b77cd7a545bb183f0c984b0be9037b Author: Poul-Henning Kamp Date: Fri Jun 4 08:09:35 2021 +0000 Use SF_Parse_Decimal() for double style parameters Annotate parameter tweak functions with v_matchproto_() diff --git a/bin/varnishd/mgt/mgt_param_bits.c b/bin/varnishd/mgt/mgt_param_bits.c index d6a9c3fd2..89d1a1aa3 100644 --- a/bin/varnishd/mgt/mgt_param_bits.c +++ b/bin/varnishd/mgt/mgt_param_bits.c @@ -113,7 +113,7 @@ static const char * const VSL_tags[256] = { # include "tbl/vsl_tags.h" }; -static int +static int v_matchproto_(tweak_t) tweak_vsl_mask(struct vsb *vsb, const struct parspec *par, const char *arg) { unsigned j; @@ -171,7 +171,7 @@ static const char * const debug_tags[] = { NULL }; -static int +static int v_matchproto_(tweak_t) tweak_debug(struct vsb *vsb, const struct parspec *par, const char *arg) { const char *s; @@ -214,7 +214,7 @@ static const char * const feature_tags[] = { NULL }; -static int +static int v_matchproto_(tweak_t) tweak_feature(struct vsb *vsb, const struct parspec *par, const char *arg) { const char *s; diff --git a/bin/varnishd/mgt/mgt_param_tweak.c b/bin/varnishd/mgt/mgt_param_tweak.c index 721e05292..b3148b857 100644 --- a/bin/varnishd/mgt/mgt_param_tweak.c +++ b/bin/varnishd/mgt/mgt_param_tweak.c @@ -55,50 +55,54 @@ static int tweak_generic_double(struct vsb *vsb, volatile double *dest, const char *arg, const char *min, const char *max, const char *fmt) { - volatile double u, minv = 0, maxv = 0; + volatile double u, minv = VRT_DECIMAL_MIN, maxv = VRT_DECIMAL_MAX; + const char *p, *err; if (arg != NULL && arg != JSON_FMT) { if (min != NULL) { - minv = VNUM(min); - if (isnan(minv)) { - VSB_printf(vsb, "Illegal Min: %s\n", min); + p = min; + minv = SF_Parse_Decimal(&p, 0, &err); + if (errno) { + VSB_printf(vsb, "Min: %s (%s)\n", err, min); return (-1); } } if (max != NULL) { - maxv = VNUM(max); - if (isnan(maxv)) { - VSB_printf(vsb, "Illegal Max: %s\n", max); + p = max; + maxv = SF_Parse_Decimal(&p, 0, &err); + if (errno) { + VSB_printf(vsb, "Max: %s (%s)\n", err, max); return (-1); } } - u = VNUM(arg); - if (isnan(u)) { - VSB_printf(vsb, "Not a number(%s)\n", arg); + p = arg; + u = SF_Parse_Decimal(&p, 0, &err); + if (errno) { + VSB_printf(vsb, "%s (%s)\n", err, arg); return (-1); } - if (min != NULL && u < minv) { + if (u < minv) { VSB_printf(vsb, "Must be greater or equal to %s\n", min); return (-1); } - if (max != NULL && u > maxv) { + if (u > maxv) { VSB_printf(vsb, "Must be less than or equal to %s\n", max); return (-1); } *dest = u; - } else + } else { VSB_printf(vsb, fmt, *dest); + } return (0); } /*--------------------------------------------------------------------*/ -int -tweak_timeout(struct vsb *vsb, const struct parspec *par, - const char *arg) +int v_matchproto_(tweak_t) +tweak_timeout(struct vsb *vsb, const struct parspec *par, const char *arg) { volatile double *dest; @@ -109,7 +113,7 @@ tweak_timeout(struct vsb *vsb, const struct parspec *par, /*--------------------------------------------------------------------*/ -int +int v_matchproto_(tweak_t) tweak_double(struct vsb *vsb, const struct parspec *par, const char *arg) { volatile double *dest; @@ -120,7 +124,7 @@ tweak_double(struct vsb *vsb, const struct parspec *par, const char *arg) /*--------------------------------------------------------------------*/ -int +int v_matchproto_(tweak_t) tweak_boolean(struct vsb *vsb, const struct parspec *par, const char *arg) { volatile unsigned *dest; @@ -217,7 +221,7 @@ tweak_generic_uint(struct vsb *vsb, volatile unsigned *dest, const char *arg, /*--------------------------------------------------------------------*/ -int +int v_matchproto_(tweak_t) tweak_uint(struct vsb *vsb, const struct parspec *par, const char *arg) { volatile unsigned *dest; @@ -306,7 +310,7 @@ tweak_generic_bytes(struct vsb *vsb, volatile ssize_t *dest, const char *arg, /*--------------------------------------------------------------------*/ -int +int v_matchproto_(tweak_t) tweak_bytes(struct vsb *vsb, const struct parspec *par, const char *arg) { volatile ssize_t *dest; @@ -317,7 +321,7 @@ tweak_bytes(struct vsb *vsb, const struct parspec *par, const char *arg) /*--------------------------------------------------------------------*/ -int +int v_matchproto_(tweak_t) tweak_bytes_u(struct vsb *vsb, const struct parspec *par, const char *arg) { volatile unsigned *d1; @@ -335,7 +339,7 @@ tweak_bytes_u(struct vsb *vsb, const struct parspec *par, const char *arg) * vsl_buffer and vsl_reclen have dependencies. */ -int +int v_matchproto_(tweak_t) tweak_vsl_buffer(struct vsb *vsb, const struct parspec *par, const char *arg) { volatile unsigned *d1; @@ -350,7 +354,7 @@ tweak_vsl_buffer(struct vsb *vsb, const struct parspec *par, const char *arg) return (0); } -int +int v_matchproto_(tweak_t) tweak_vsl_reclen(struct vsb *vsb, const struct parspec *par, const char *arg) { volatile unsigned *d1; @@ -367,7 +371,7 @@ tweak_vsl_reclen(struct vsb *vsb, const struct parspec *par, const char *arg) /*--------------------------------------------------------------------*/ -int +int v_matchproto_(tweak_t) tweak_string(struct vsb *vsb, const struct parspec *par, const char *arg) { char **p = TRUST_ME(par->priv); @@ -388,7 +392,7 @@ tweak_string(struct vsb *vsb, const struct parspec *par, const char *arg) /*--------------------------------------------------------------------*/ -int +int v_matchproto_(tweak_t) tweak_poolparam(struct vsb *vsb, const struct parspec *par, const char *arg) { volatile struct poolparam *pp, px; @@ -458,7 +462,7 @@ tweak_poolparam(struct vsb *vsb, const struct parspec *par, const char *arg) * limit, so they don't end up crossing. */ -int +int v_matchproto_(tweak_t) tweak_thread_pool_min(struct vsb *vsb, const struct parspec *par, const char *arg) { @@ -472,7 +476,7 @@ tweak_thread_pool_min(struct vsb *vsb, const struct parspec *par, return (0); } -int +int v_matchproto_(tweak_t) tweak_thread_pool_max(struct vsb *vsb, const struct parspec *par, const char *arg) { From nils.goroll at uplex.de Fri Jun 4 13:25:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 4 Jun 2021 13:25:06 +0000 (UTC) Subject: [master] 40ca3c291 Support ACL as a vmod return type Message-ID: <20210604132506.28343109A68@lists.varnish-cache.org> commit 40ca3c2911c7c85d8025d5527c0739eea95b08b1 Author: Nils Goroll Date: Wed Jun 2 16:14:33 2021 +0200 Support ACL as a vmod return type to the extent that the match operator ~ will actually accept vmod function/method calls returning it. Because VMODs also need an invalid ACL value to signal error, we stop panicking for a NULL ACL and trigger a VCL failure instead. The use case is not (yet) to generate dynamic ACLs, but rather to store and recall global ACL symbols. This implementation was particularly simple thanks to Dridis work on #3555 and VCL_REGEX. diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 1289c1ac2..d38587d36 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -134,11 +134,12 @@ VRT_acl_match(VRT_CTX, VCL_ACL acl, VCL_IP ip) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(acl, VRT_ACL_MAGIC); - if (ip == NULL) { - VRT_fail(ctx, "Cannot match a null IP address"); + if (acl == NULL || ip == NULL) { + VRT_fail(ctx, "Cannot match a null %s", + acl == NULL ? "ACL" : "IP address"); return (0); } + CHECK_OBJ(acl, VRT_ACL_MAGIC); assert(VSA_Sane(ip)); return (acl->match(ctx, ip)); } diff --git a/bin/varnishtest/tests/m00023.vtc b/bin/varnishtest/tests/m00023.vtc index 3ecfbde70..6abe3b569 100644 --- a/bin/varnishtest/tests/m00023.vtc +++ b/bin/varnishtest/tests/m00023.vtc @@ -21,13 +21,31 @@ varnish v1 -vcl+backend { if (!debug.match_acl(loopback, "127.0.0.127")) { debug.fail(); } + new o_locals = debug.aclobj(locals); + new o_null = debug.aclobj(debug.null_acl()); } sub vcl_recv { - if (debug.match_acl(ip=client.ip, acl=locals)) { - return (hash); + if (req.url == "/null") { + if (client.ip ~ debug.null_acl()) { + return (synth(200)); + } } - return (synth(500)); + if (req.url == "/nullo") { + if (client.ip ~ o_null.get()) { + return (synth(200)); + } + } + if (! debug.match_acl(ip=client.ip, acl=locals)) { + return (synth(500, "match_acl")); + } + if (client.ip !~ debug.acl(locals)) { + return (synth(500, "~")); + } + if (client.ip !~ o_locals.get()) { + return (synth(500, "~")); + } + return (hash); } } -start @@ -35,4 +53,16 @@ client c1 { txreq rxresp expect resp.status == 200 -} -run + txreq -url /null + rxresp + expect resp.status == 503 +} -start + +client c2 { + txreq -url /nullo + rxresp + expect resp.status == 503 +} -start + +client c1 -wait +client c2 -wait diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc index d26295fdc..f4421a166 100644 --- a/bin/varnishtest/tests/v00016.vtc +++ b/bin/varnishtest/tests/v00016.vtc @@ -89,7 +89,7 @@ varnish v1 -errvcl {resolves to too many addresses} { } } -varnish v1 -errvcl {Symbol 'foo' has wrong type (instance)} { +varnish v1 -errvcl {Expression has type directors.shard, expected ACL} { import directors; backend b { .host = "${localhost}"; } @@ -103,7 +103,7 @@ varnish v1 -errvcl {Symbol 'foo' has wrong type (instance)} { } } -varnish v1 -syntax 4.0 -errvcl {Undefined acl foo} { +varnish v1 -syntax 4.0 -errvcl {Expression has type directors.shard, expected ACL} { import directors; backend b { .host = "${localhost}"; } diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc index a12f3295e..603f3a251 100644 --- a/bin/varnishtest/tests/v00020.vtc +++ b/bin/varnishtest/tests/v00020.vtc @@ -423,7 +423,7 @@ varnish v1 -errvcl {'||' must be preceeded by BOOL, found REAL.} { sub vcl_recv { if (std.random(0,1) || 0) { } } } -varnish v1 -errvcl {Symbol 'acl' has wrong type (reserved), expected acl:} { +varnish v1 -errvcl {Symbol 'acl' type (reserved) can not be used in expression.} { import std; sub vcl_recv { if (client.ip ~ acl) {} } } diff --git a/bin/varnishtest/tests/v00021.vtc b/bin/varnishtest/tests/v00021.vtc index 74bb36509..b8c2c4b40 100644 --- a/bin/varnishtest/tests/v00021.vtc +++ b/bin/varnishtest/tests/v00021.vtc @@ -151,13 +151,13 @@ varnish v1 -errvcl {Symbols named 'vcl_*' are reserved.} { } } -varnish v1 -errvcl {Symbol 'true' has wrong type (func), expected acl:} { +varnish v1 -errvcl {Expression has type BOOL, expected ACL} { sub vcl_recv { if (client.ip ~ true) { } } } -varnish v1 -errvcl {Symbol 'default' has wrong type} { +varnish v1 -errvcl {Symbol 'default' is a reserved word.} { sub vcl_recv { if (client.ip ~ default) { } } diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index fb5220e4e..4118d861b 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -1121,17 +1121,14 @@ cmp_regexp(struct vcc *tl, struct expr **e, const struct cmps *cp) static void v_matchproto_(cmp_f) cmp_acl(struct vcc *tl, struct expr **e, const struct cmps *cp) { - struct symbol *sym; + struct expr *e2; char buf[256]; vcc_NextToken(tl); - vcc_ExpectVid(tl, "ACL"); - sym = VCC_SymbolGet(tl, SYM_MAIN, SYM_ACL, SYMTAB_CREATE, XREF_REF); + vcc_expr0(tl, &e2, ACL); ERRCHK(tl); - AN(sym); - VCC_GlobalSymbol(sym, ACL); - bprintf(buf, "%sVRT_acl_match(ctx, %s, \v1)", cp->emit, sym->rname); - *e = vcc_expr_edit(tl, BOOL, buf, *e, NULL); + bprintf(buf, "%sVRT_acl_match(ctx, \v1, \v2)", cp->emit); + *e = vcc_expr_edit(tl, BOOL, buf, e2, *e); } static void v_matchproto_(cmp_f) diff --git a/vmod/vmod_debug.vcc b/vmod/vmod_debug.vcc index cdafcc4d3..b3ff9ac4d 100644 --- a/vmod/vmod_debug.vcc +++ b/vmod/vmod_debug.vcc @@ -174,6 +174,14 @@ $Function VOID vcl_discard_delay(PRIV_VCL, DURATION) Hold a reference to the VCL when it goes cold preventing discard for the given delay. +$Function ACL null_acl() + +Return no acl. + +$Function ACL acl(ACL acl) + +Return the argument. + $Function BOOL match_acl(ACL acl, IP ip) Perform an IP match against a named ACL. @@ -194,6 +202,10 @@ Time `rounds` sweeps from `ip0` to `ip1` through `acl`. Consider: `-p vsl_mask=-VCL_acl` +$Object aclobj(ACL) + +$Method ACL .get() + $Function VOID test_probe(PROBE probe, PROBE same = 0) Only here to make sure probe definitions are passed properly. diff --git a/vmod/vmod_debug_acl.c b/vmod/vmod_debug_acl.c index 7d708f00d..385565891 100644 --- a/vmod/vmod_debug_acl.c +++ b/vmod/vmod_debug_acl.c @@ -47,12 +47,27 @@ #include "vtim.h" #include "vcc_debug_if.h" +VCL_ACL v_matchproto_(td_debug_null_acl) +xyzzy_null_acl(VRT_CTX) +{ + + CHECK_OBJ_ORNULL(ctx, VRT_CTX_MAGIC); + return (NULL); +} + +VCL_ACL v_matchproto_(td_debug_acl) +xyzzy_acl(VRT_CTX, VCL_ACL acl) +{ + + CHECK_OBJ_ORNULL(ctx, VRT_CTX_MAGIC); + return (acl); +} + VCL_BOOL v_matchproto_(td_debug_match_acl) xyzzy_match_acl(VRT_CTX, VCL_ACL acl, VCL_IP ip) { CHECK_OBJ_ORNULL(ctx, VRT_CTX_MAGIC); - AN(acl); assert(VSA_Sane(ip)); return (VRT_acl_match(ctx, acl, ip)); @@ -254,3 +269,44 @@ xyzzy_time_acl(VRT_CTX, VCL_ACL acl, VCL_IP ip0, VCL_IP ip1, cleanup_sweep(asw); return (d); } + +struct xyzzy_debug_aclobj { + unsigned magic; +#define VMOD_DEBUG_ACLOBJ_MAGIC 0xac10ac10 + char * vcl_name; + VCL_ACL acl; +}; + +VCL_VOID v_matchproto_(td_xyzzy_debug_aclobj__init) +xyzzy_aclobj__init(VRT_CTX, struct VPFX(debug_aclobj) **op, + const char *vcl_name, VCL_ACL acl) +{ + struct VPFX(debug_aclobj) *o; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(op); + AZ(*op); + ALLOC_OBJ(o, VMOD_DEBUG_ACLOBJ_MAGIC); + AN(o); + REPLACE(o->vcl_name, vcl_name); + o->acl = acl; + *op = o; +} + +VCL_VOID v_matchproto_(td_xyzzy_debug_aclobj__fini) +xyzzy_aclobj__fini(struct VPFX(debug_aclobj) **op) +{ + struct VPFX(debug_aclobj) *o; + + TAKE_OBJ_NOTNULL(o, op, VMOD_DEBUG_ACLOBJ_MAGIC); + REPLACE(o->vcl_name, NULL); + FREE_OBJ(o); +} + +VCL_ACL v_matchproto_(td_xyzzy_debug_aclobj_get) +xyzzy_aclobj_get(VRT_CTX, struct VPFX(debug_aclobj) *o) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(o, VMOD_DEBUG_ACLOBJ_MAGIC); + return (o->acl); +} From phk at critter.freebsd.dk Wed Jun 9 12:44:06 2021 From: phk at critter.freebsd.dk (Poul-Henning Kamp) Date: Wed, 9 Jun 2021 12:44:06 +0000 (UTC) Subject: [master] a62210acb About the 503's heard around the world. Message-ID: <20210609124407.048F010FDD0@lists.varnish-cache.org> commit a62210acba5ba361160a2aaf23942c599631025e Author: Poul-Henning Kamp Date: Wed Jun 9 12:43:16 2021 +0000 About the 503's heard around the world. diff --git a/doc/sphinx/phk/503aroundtheworld.rst b/doc/sphinx/phk/503aroundtheworld.rst new file mode 100644 index 000000000..1c169ddc7 --- /dev/null +++ b/doc/sphinx/phk/503aroundtheworld.rst @@ -0,0 +1,106 @@ +.. + Copyright (c) 2021 Varnish Software AS + SPDX-License-Identifier: BSD-2-Clause + See LICENSE file for full text of license + +.. _phk_503aroundtheworld: + +================================ +The 503's heard around the world +================================ + +Yesterday, may 8th 2021, for about an hour, the world saw things like: + +.. image:: fastly_503.png + +(The alert reader will notice that it says "Mediation" instead of +"Meditation". I dont know why, but I know who to ask, and I will.) + +People have, somewhat justifiably jumped on this error message, +complaining about how terse it is, for instance `Max Rozen +`_ +who makes some good and sensible suggestions about good and sensible +error messages. + +But all his suggestions require some amount of ambient information, +information which we in the Varnish Cache project do not have and +never can have. + +When Varnish gets to the bottom of the `builtin.vcl` and errors +out, we dont know where that Varnish instance runs, who it runs +for, what information it servers, who it serves that information +to and most crucially, what information can safely be shared with +the clients. + +This is why the default "503 guru meditation" only mentions the +`XID`: The XID is a nonce, for all practical purposes a random +number, but one which allows the persons responsible for that varnish +instance to dig through their logs and find out what happened to +this specific request. + +The other thing I want to note, is that the one thing you absolutely +do not want to happen, is a "priority inversion", where it takes +more CPU, database accesses or bandwidth to produce the error +message, than it would have to produce the intended result. + +This error message is intended to be in all respects minimal. + +I'm pretty sure I have seen normal 503s from Fastly's CDN, they +dont look like this, so something must have gone badly wrong for +Fastly yesterday. + +Even though I'm sure the XID did not in any way help them debug +this specific problem, I am happy to see that our "error message +of last resort" did what it was supposed to do, as it was supposed +to do it. + +The first 503 stress test +------------------------- + +When we release the first version of Varnish the text "Varnish cache +server" was link to the project homepage. + +That link acted as a "viral marketing campaign" for us: When +web-people saw the 503 on another webpage, often a competitors +webpage, they clicked on the link to learn what software they +were running. + +And since everybody usually makes a couple of mistakes trying to get +Varnish running for the first time, more and more web-people +learned about Varnish Cache. + +So word about Varnish Cache spread around the globe. + +All the way to China. + +Where the Chinese State Railroads rolled out Varnish Cache. + +And made some kind of beginners mistake. + +A lot of chinese web-people learned about Varnish Cache that day. + +As did a lot of other chinese people, who instead of a train schedules +and train tickets, got a few lines of english, which the vast +majority of did not read or understand. + +But there were also a link. + +They did what any sensible web-user would do. + +They clicked on the link. + +And dDoS'ed the Varnish Cache project server. + +... and the Redpill-Linprox Data Center, where that server lived. + +... and the Norvegian ISPs that announced the prefix of that data center. + +In fact, they effectively dDoS'ed Norway. + +And that, my friends, is why the error message of last resort in +Varnish Cache does not even contain a link any more. + +*/phk* + +PS: The title of this post is of course an homage to +`John R. Garman's famous article `_. diff --git a/doc/sphinx/phk/fastly_503.png b/doc/sphinx/phk/fastly_503.png new file mode 100644 index 000000000..1459d4663 Binary files /dev/null and b/doc/sphinx/phk/fastly_503.png differ diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst index 8897a0662..c0374116c 100644 --- a/doc/sphinx/phk/index.rst +++ b/doc/sphinx/phk/index.rst @@ -13,6 +13,7 @@ You may or may not want to know what Poul-Henning thinks. .. toctree:: :maxdepth: 1 + 503aroundtheworld.rst legacy.rst ip_address.rst vdd19q3.rst From phk at critter.freebsd.dk Wed Jun 9 12:50:06 2021 From: phk at critter.freebsd.dk (Poul-Henning Kamp) Date: Wed, 9 Jun 2021 12:50:06 +0000 (UTC) Subject: [master] e77ebbd8b Typo Message-ID: <20210609125006.BEADD110090@lists.varnish-cache.org> commit e77ebbd8b9a531be72295eeface70add2d69ee6e Author: Poul-Henning Kamp Date: Wed Jun 9 12:49:12 2021 +0000 Typo diff --git a/doc/sphinx/phk/503aroundtheworld.rst b/doc/sphinx/phk/503aroundtheworld.rst index 1c169ddc7..ca819895c 100644 --- a/doc/sphinx/phk/503aroundtheworld.rst +++ b/doc/sphinx/phk/503aroundtheworld.rst @@ -28,7 +28,7 @@ never can have. When Varnish gets to the bottom of the `builtin.vcl` and errors out, we dont know where that Varnish instance runs, who it runs -for, what information it servers, who it serves that information +for, what information it serves, who it serves that information to and most crucially, what information can safely be shared with the clients. From phk at FreeBSD.org Wed Jun 9 13:11:05 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 9 Jun 2021 13:11:05 +0000 (UTC) Subject: [master] 01f81cf06 ArgH! Another typo Message-ID: <20210609131105.B9277110CF0@lists.varnish-cache.org> commit 01f81cf06b17c8c00472cd79a95af3c8c433ca27 Author: Poul-Henning Kamp Date: Wed Jun 9 13:10:06 2021 +0000 ArgH! Another typo diff --git a/doc/sphinx/phk/503aroundtheworld.rst b/doc/sphinx/phk/503aroundtheworld.rst index ca819895c..786e52711 100644 --- a/doc/sphinx/phk/503aroundtheworld.rst +++ b/doc/sphinx/phk/503aroundtheworld.rst @@ -81,7 +81,7 @@ A lot of chinese web-people learned about Varnish Cache that day. As did a lot of other chinese people, who instead of a train schedules and train tickets, got a few lines of english, which the vast -majority of did not read or understand. +majority of them did not read or understand. But there were also a link. From phk at FreeBSD.org Wed Jun 9 13:48:06 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 9 Jun 2021 13:48:06 +0000 (UTC) Subject: [master] 3b7c14b1b A typo and an update. Message-ID: <20210609134806.E8D48111E12@lists.varnish-cache.org> commit 3b7c14b1b2146bae3a0ce9ba1812ab26e38fb9b2 Author: Poul-Henning Kamp Date: Wed Jun 9 13:46:42 2021 +0000 A typo and an update. diff --git a/doc/sphinx/phk/503aroundtheworld.rst b/doc/sphinx/phk/503aroundtheworld.rst index 786e52711..1ee9a9ebb 100644 --- a/doc/sphinx/phk/503aroundtheworld.rst +++ b/doc/sphinx/phk/503aroundtheworld.rst @@ -9,7 +9,7 @@ The 503's heard around the world ================================ -Yesterday, may 8th 2021, for about an hour, the world saw things like: +Yesterday, june 8th 2021, for about an hour, the world saw things like: .. image:: fastly_503.png @@ -97,6 +97,17 @@ And dDoS'ed the Varnish Cache project server. In fact, they effectively dDoS'ed Norway. + +(Update: I'm now being told by people from Norway that this did not +dDoS that much. As I live in Denmark I obviously only have this +saga on second hand. (The only effect I experienced myself was that +the Trac server was unreachable). Somebody has exagerated the +story along the way. + +However, this did prompt me to track down the precise timing: +Varnish release 2.1.2, May 2010 has th elink, release 2.1.3 from +July same year does not.) + And that, my friends, is why the error message of last resort in Varnish Cache does not even contain a link any more. @@ -104,3 +115,4 @@ Varnish Cache does not even contain a link any more. PS: The title of this post is of course an homage to `John R. Garman's famous article `_. + From phk at FreeBSD.org Wed Jun 9 13:52:04 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 9 Jun 2021 13:52:04 +0000 (UTC) Subject: [master] c36610620 Argh, typo! Message-ID: <20210609135204.F21FD1120C8@lists.varnish-cache.org> commit c366106207252bc552a53248d5c11addc12eb78a Author: Poul-Henning Kamp Date: Wed Jun 9 13:51:18 2021 +0000 Argh, typo! diff --git a/doc/sphinx/phk/503aroundtheworld.rst b/doc/sphinx/phk/503aroundtheworld.rst index 1ee9a9ebb..c200ba7f5 100644 --- a/doc/sphinx/phk/503aroundtheworld.rst +++ b/doc/sphinx/phk/503aroundtheworld.rst @@ -105,7 +105,7 @@ the Trac server was unreachable). Somebody has exagerated the story along the way. However, this did prompt me to track down the precise timing: -Varnish release 2.1.2, May 2010 has th elink, release 2.1.3 from +Varnish release 2.1.2, May 2010 has the link, release 2.1.3 from July same year does not.) And that, my friends, is why the error message of last resort in From nils.goroll at uplex.de Thu Jun 10 08:35:09 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 10 Jun 2021 08:35:09 +0000 (UTC) Subject: [master] 12bbe31f7 build: Only link PCRE where it is used Message-ID: <20210610083509.C607F10855E@lists.varnish-cache.org> commit 12bbe31f7989538f6711448a91667347d8d529bf Author: Dridi Boukelmoune Date: Tue May 18 07:47:13 2021 +0200 build: Only link PCRE where it is used Otherwise we get it from either libvarnish or libvarnishapi. Refs #3559 diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am index fae47f4a5..2574fb586 100644 --- a/bin/varnishtest/Makefile.am +++ b/bin/varnishtest/Makefile.am @@ -56,7 +56,6 @@ varnishtest_LDADD = \ $(top_builddir)/lib/libvarnish/libvarnish.la \ $(top_builddir)/lib/libvgz/libvgz.a \ @SAN_LDFLAGS@ \ - @PCRE_LIBS@ \ ${PTHREAD_LIBS} ${NET_LIBS} ${LIBM} varnishtest_CFLAGS = \ diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am index 9c0b8a643..ea2ba0a3a 100644 --- a/lib/libvarnish/Makefile.am +++ b/lib/libvarnish/Makefile.am @@ -6,7 +6,7 @@ AM_CPPFLAGS = \ @PCRE_CFLAGS@ AM_CFLAGS = $(AM_LT_CFLAGS) @SAN_CFLAGS@ -AM_LDFLAGS = $(AM_LT_LDFLAGS) @SAN_LDFLAGS@ +AM_LDFLAGS = $(AM_LT_LDFLAGS) @SAN_LDFLAGS@ @PCRE_LIBS@ noinst_LTLIBRARIES = libvarnish.la diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am index 0cbff3c4d..26638491b 100644 --- a/lib/libvarnishapi/Makefile.am +++ b/lib/libvarnishapi/Makefile.am @@ -4,8 +4,7 @@ AM_LDFLAGS = $(AM_LT_LDFLAGS) AM_CPPFLAGS = \ -I$(top_srcdir)/include \ - -I$(top_builddir)/include \ - @PCRE_CFLAGS@ + -I$(top_builddir)/include lib_LTLIBRARIES = libvarnishapi.la @@ -41,7 +40,7 @@ libvarnishapi_la_CFLAGS = \ libvarnishapi_la_LIBADD = \ $(top_builddir)/lib/libvarnish/libvarnish.la \ - @SAN_LDFLAGS@ @PCRE_LIBS@ ${NET_LIBS} ${RT_LIBS} ${LIBM} + @SAN_LDFLAGS@ ${NET_LIBS} ${RT_LIBS} ${LIBM} if HAVE_LD_VERSION_SCRIPT libvarnishapi_la_LDFLAGS += -Wl,--version-script=$(srcdir)/libvarnishapi.map @@ -81,7 +80,8 @@ vxp_test_SOURCES = \ vxp_test_CFLAGS = \ -DVARNISH_STATE_DIR='"${VARNISH_STATE_DIR}"' \ -DVXP_DEBUG -vxp_test_LDADD = @PCRE_LIBS@ \ +vxp_test_LDADD = \ + $(top_builddir)/lib/libvarnish/libvarnish.la \ ${RT_LIBS} ${LIBM} ${PTHREAD_LIBS} TESTS = vsl_glob_test From nils.goroll at uplex.de Thu Jun 10 08:35:09 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 10 Jun 2021 08:35:09 +0000 (UTC) Subject: [master] cdca08cb1 build: Always build vxp_test Message-ID: <20210610083509.E04DE108561@lists.varnish-cache.org> commit cdca08cb19bf591529e55dcf82491d1e7feba455 Author: Dridi Boukelmoune Date: Tue May 18 07:48:33 2021 +0200 build: Always build vxp_test To make sure it is always readily available without an extra step, and that it doesn't rot away missing build flags or triggerring developer warnings like GCC's -Wtautological-constant-out-of-range-compare. diff --git a/.gitignore b/.gitignore index 7dba66a9f..f7aae9ebd 100644 --- a/.gitignore +++ b/.gitignore @@ -123,6 +123,7 @@ cscope.*out /lib/libvarnish/vsb_test /lib/libvarnishapi/vsl_glob_test /lib/libvarnishapi/vsl_glob_test_coverage +/lib/libvarnishapi/vxp_test # vtc-bisect.sh default vtc /bisect.vtc diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am index 26638491b..15f8a208c 100644 --- a/lib/libvarnishapi/Makefile.am +++ b/lib/libvarnishapi/Makefile.am @@ -72,17 +72,18 @@ vxp_tokens.h: \ $(srcdir)/generate.py @PYTHON@ $(srcdir)/generate.py $(srcdir) $(top_builddir) -EXTRA_PROGRAMS = vxp_test +noinst_PROGRAMS += vxp_test vxp_test_SOURCES = \ $(libvarnishapi_la_SOURCES) \ vxp_test.c vxp_test_CFLAGS = \ -DVARNISH_STATE_DIR='"${VARNISH_STATE_DIR}"' \ - -DVXP_DEBUG + -DVXP_DEBUG \ + ${SAN_CFLAGS} vxp_test_LDADD = \ $(top_builddir)/lib/libvarnish/libvarnish.la \ - ${RT_LIBS} ${LIBM} ${PTHREAD_LIBS} + ${RT_LIBS} ${LIBM} ${PTHREAD_LIBS} ${SAN_LDFLAGS} TESTS = vsl_glob_test diff --git a/lib/libvarnishapi/vxp_test.c b/lib/libvarnishapi/vxp_test.c index 359902025..9f921c9a9 100644 --- a/lib/libvarnishapi/vxp_test.c +++ b/lib/libvarnishapi/vxp_test.c @@ -58,7 +58,7 @@ main(int argc, char * const *argv) struct vsb *vsb; struct vex *vex; char *q_arg = NULL; - char opt; + int opt; while ((opt = getopt(argc, argv, "q:")) != -1) { switch (opt) { From nils.goroll at uplex.de Thu Jun 10 09:39:05 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 10 Jun 2021 09:39:05 +0000 (UTC) Subject: [master] cb44f3487 vtc: Spurious "\T" sequence in u3 Message-ID: <20210610093905.4708A10A30C@lists.varnish-cache.org> commit cb44f34877bf5d2a042b382828e8d2779af10271 Author: Dridi Boukelmoune Date: Wed May 19 11:46:26 2021 +0200 vtc: Spurious "\T" sequence in u3 It is interpreted as "T" by pcre but pcre2 is stricter and treats it as a syntax error. While there may be a bit flag to ignore unknown escape sequences, it is probably worth hardening instead. Refs #3559 diff --git a/bin/varnishtest/tests/u00003.vtc b/bin/varnishtest/tests/u00003.vtc index 3cfa187de..e67962a28 100644 --- a/bin/varnishtest/tests/u00003.vtc +++ b/bin/varnishtest/tests/u00003.vtc @@ -150,7 +150,7 @@ req (\d+) rxreq \5 - - - -$} \ %{VSL:Timestamp:Resp[2]}x %{VSL:Timestamp:foo}x %{VSL:ReqURL[2]}x"} # times -shell -match {^\{\d{4}-\d{2}-\d{2}\}T\{\d{2}:\d{2}:\d{2}\} \{\{\d{4}-\d{2}-\d{2}\T\d{2}:\d{2}:\d{2}\}\} \d+ \d+ \d+ \d{10,} \d{13,} \d{16,} \d{3} \d{6} usecx msecy} \ +shell -match {^\{\d{4}-\d{2}-\d{2}\}T\{\d{2}:\d{2}:\d{2}\} \{\{\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\}\} \d+ \d+ \d+ \d{10,} \d{13,} \d{16,} \d{3} \d{6} usecx msecy} \ {varnishncsa -n ${v1_name} -d -F "%{{%F}T{%T}}t %{{{%FT%T}}}t \ %{s}T %{ms}T %{us}T %{sec}t %{msec}t %{usec}t %{msec_frac}t %{usec_frac}t \ %{usecx}t %{msecy}t"} From nils.goroll at uplex.de Thu Jun 10 09:39:05 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 10 Jun 2021 09:39:05 +0000 (UTC) Subject: [master] d36fcf29b vtc: Print the VRE error message Message-ID: <20210610093905.5B0F210A30F@lists.varnish-cache.org> commit d36fcf29b5ea43da2f8c8e5299dbaf34c1c205a5 Author: Dridi Boukelmoune Date: Wed May 19 11:50:19 2021 +0200 vtc: Print the VRE error message It's more helpful than printing the invalid regular expression, that's how the spurious "\T" was spotted. diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index 59c226e52..6c3288d97 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -157,8 +157,9 @@ cmd_shell_engine(struct vtclog *vl, int ok, const char *cmd, if (re != NULL) { vre = VRE_compile(re, 0, &errptr, &err); if (vre == NULL) - vtc_fatal(vl, "shell_match invalid regexp (\"%s\")", - re); + vtc_fatal(vl, + "shell_match invalid regexp (\"%s\" at %d)", + errptr, err); } VSB_printf(vsb, "exec 2>&1 ; %s", cmd); AZ(VSB_finish(vsb)); From nils.goroll at uplex.de Thu Jun 10 09:39:05 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 10 Jun 2021 09:39:05 +0000 (UTC) Subject: [master] 91e731c93 vre: Acknowledge that limits aren't reentrant Message-ID: <20210610093905.7421210A312@lists.varnish-cache.org> commit 91e731c931eb091fe7712e382afd0a86f5a5c0a5 Author: Dridi Boukelmoune Date: Wed May 19 16:42:03 2021 +0200 vre: Acknowledge that limits aren't reentrant We might want to do something about it once pcre2 is in place, but it should be noted that concurrent matches on a vre_t could compete over the limits we can tweak. Refs #3559 diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c index 5f2a81a72..409c27d96 100644 --- a/lib/libvarnish/vre.c +++ b/lib/libvarnish/vre.c @@ -119,6 +119,7 @@ VRE_exec(const vre_t *code, const char *subject, int length, } if (lim != NULL) { + /* XXX: not reentrant */ code->re_extra->match_limit = lim->match; code->re_extra->flags |= PCRE_EXTRA_MATCH_LIMIT; code->re_extra->match_limit_recursion = lim->match_recursion; From nils.goroll at uplex.de Thu Jun 10 09:39:05 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 10 Jun 2021 09:39:05 +0000 (UTC) Subject: [master] 40a65166f Reduce use of PCRE linker flags differently Message-ID: <20210610093905.D041710A31B@lists.varnish-cache.org> commit 40a65166f92bd4c57b0dc3b30173ea1a78afa20d Author: Nils Goroll Date: Thu Jun 10 11:35:58 2021 +0200 Reduce use of PCRE linker flags differently ... to make our SunOS vtest build happy again ld: fatal: file /opt/local/lib/libpcre.so: wrong ELF class: ELFCLASS64 ld: fatal: file processing errors. No output written to vjsn_test collect2: error: ld returned 1 exit status The issue here was order of the -L -l arguments when a (32bit) version of a library needs to be found first in an overridden LDPATH. Ref 12bbe31f7989538f6711448a91667347d8d529bf Ref #3559 diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am index ea2ba0a3a..3be8609f1 100644 --- a/lib/libvarnish/Makefile.am +++ b/lib/libvarnish/Makefile.am @@ -6,7 +6,7 @@ AM_CPPFLAGS = \ @PCRE_CFLAGS@ AM_CFLAGS = $(AM_LT_CFLAGS) @SAN_CFLAGS@ -AM_LDFLAGS = $(AM_LT_LDFLAGS) @SAN_LDFLAGS@ @PCRE_LIBS@ +AM_LDFLAGS = $(AM_LT_LDFLAGS) @SAN_LDFLAGS@ noinst_LTLIBRARIES = libvarnish.la @@ -44,6 +44,8 @@ libvarnish_la_SOURCES = \ vtim.c \ vus.c +libvarnish_la_LIBADD = @PCRE_LIBS@ + TESTS = vjsn_test vnum_c_test vbh_test vsb_test noinst_PROGRAMS = ${TESTS} From nils.goroll at uplex.de Thu Jun 10 13:43:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 10 Jun 2021 13:43:06 +0000 (UTC) Subject: [master] dd8b31444 build: wflags.py compatibility with 3.4 and catch errors Message-ID: <20210610134306.738351147B8@lists.varnish-cache.org> commit dd8b3144498578165d5ea4eef7559328f0b7fd16 Author: Nils Goroll Date: Thu Jun 10 12:19:14 2021 +0200 build: wflags.py compatibility with 3.4 and catch errors The explicit catch on CalledProcessError will be used in a follow up commit to ensure that python errors do not stay unnoticed. Ref #3624 diff --git a/wflags.py b/wflags.py index d060d1794..7912f47c8 100644 --- a/wflags.py +++ b/wflags.py @@ -81,6 +81,27 @@ UNDESIRABLE_WFLAGS = [ "-Wno-nullability-completeness", # Barfs all over MacOSx ] + +def cc(compiler, opt, obj, src): + try: + j = subprocess.check_output( + [ + compiler, + "-c", + opt, + "-o", obj, + src + ], + stderr=subprocess.STDOUT + ) + except subprocess.CalledProcessError as err: + if err.output: + j = err.output + else: + j = ("Returncode %d" % err.returncode).encode('utf8') + return (j) + + def main(): compiler = os.environ.get("CC", "cc") @@ -91,29 +112,19 @@ def main(): use_flags = [] for i in DESIRABLE_OPTIONS + DESIRABLE_WFLAGS + UNDESIRABLE_WFLAGS: - j = subprocess.run( - [ - compiler, - "-c", - i, - "-o", obj_file.name, - src_file.name, - ], - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - if not j.returncode and not j.stdout and not j.stderr: + j = cc(compiler, i, obj_file.name, src_file.name) + if not j: use_flags.append(i) else: sys.stderr.write(compiler + " cannot " + i + '\n') - if b'error: unrecognized command line option' in j.stderr: + if b'error: unrecognized command line option' in j: # LLVM pass - elif b'warning: unknown warning option' in j.stderr: + elif b'warning: unknown warning option' in j: # GCC pass else: - sys.stderr.write("\n\t" + j.stderr.decode('utf8') + '\n') + sys.stderr.write("\n\t" + j.decode('utf8') + '\n') print(" ".join(use_flags)) if __name__ == "__main__": From nils.goroll at uplex.de Thu Jun 10 13:43:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 10 Jun 2021 13:43:06 +0000 (UTC) Subject: [master] 62a25a97b build: test-run our compiler in wflags.py Message-ID: <20210610134306.86F671147BD@lists.varnish-cache.org> commit 62a25a97b0171aa1359687c9cbe9801e965893ca Author: Nils Goroll Date: Thu Jun 10 12:43:39 2021 +0200 build: test-run our compiler in wflags.py ... to have a solid baseline diff --git a/wflags.py b/wflags.py index 7912f47c8..1778fd41b 100644 --- a/wflags.py +++ b/wflags.py @@ -83,17 +83,13 @@ UNDESIRABLE_WFLAGS = [ def cc(compiler, opt, obj, src): + a = [compiler, "-c"] + if opt is not None: + a.append(opt) + a += ["-o", obj, src] + try: - j = subprocess.check_output( - [ - compiler, - "-c", - opt, - "-o", obj, - src - ], - stderr=subprocess.STDOUT - ) + j = subprocess.check_output(a, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as err: if err.output: j = err.output @@ -110,6 +106,12 @@ def main(): src_file.flush() obj_file = tempfile.NamedTemporaryFile(suffix='.o') + j = cc(compiler, None, obj_file.name, src_file.name) + if j: + sys.stderr.write(compiler + " failed without flags\n\t" + + j.decode('utf8') + '\n') + sys.exit(1) + use_flags = [] for i in DESIRABLE_OPTIONS + DESIRABLE_WFLAGS + UNDESIRABLE_WFLAGS: j = cc(compiler, i, obj_file.name, src_file.name) From nils.goroll at uplex.de Thu Jun 10 13:43:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 10 Jun 2021 13:43:06 +0000 (UTC) Subject: [master] b28f6143f build: Error out of configure if wflags.py fails Message-ID: <20210610134306.A33C61147C2@lists.varnish-cache.org> commit b28f6143febd2bd143b1c9cbb930e0bba716362c Author: Nils Goroll Date: Thu Jun 10 15:41:40 2021 +0200 build: Error out of configure if wflags.py fails Fixes #3624 diff --git a/configure.ac b/configure.ac index 3e02d06d0..553e9f43d 100644 --- a/configure.ac +++ b/configure.ac @@ -719,6 +719,9 @@ AX_CHECK_COMPILE_FLAG([-Werror=unused-result], # This corresponds to FreeBSD's WARNS level 6 DEVELOPER_CFLAGS=`$PYTHON $srcdir/wflags.py` +if test $? -ne 0 ; then + AC_MSG_ERROR([wflags.py failure]) +fi # --enable-stack-protector AC_ARG_ENABLE(stack-protector, From nils.goroll at uplex.de Thu Jun 10 13:45:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 10 Jun 2021 13:45:06 +0000 (UTC) Subject: [master] 802884c97 vre: Enforce VRE options with masks Message-ID: <20210610134506.F1BD7114CB4@lists.varnish-cache.org> commit 802884c97601a5e52a9ec5fff37a40f906fc175e Author: Dridi Boukelmoune Date: Tue May 18 15:44:37 2021 +0200 vre: Enforce VRE options with masks The two options can't be mixed together and we should ensure that only the options we decided to support can be passed along, preventing bit smuggling. For a full PCRE spectrum, VMODs can directly use a PCRE library through its API. diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c index 409c27d96..60254aa4b 100644 --- a/lib/libvarnish/vre.c +++ b/lib/libvarnish/vre.c @@ -70,6 +70,14 @@ struct vre { const unsigned VRE_CASELESS = PCRE_CASELESS; const unsigned VRE_NOTEMPTY = PCRE_NOTEMPTY; +/* + * Even though we only have one for each case so far, keep track of masks + * to differentiate between compile and exec options and enfore the hard + * VRE linkage. + */ +#define VRE_MASK_COMPILE PCRE_CASELESS +#define VRE_MASK_EXEC PCRE_NOTEMPTY + vre_t * VRE_compile(const char *pattern, unsigned options, const char **errptr, int *erroffset) @@ -82,6 +90,7 @@ VRE_compile(const char *pattern, unsigned options, *errptr = "Out of memory for VRE"; return (NULL); } + AZ(options & (~VRE_MASK_COMPILE)); v->re = pcre_compile(pattern, options, errptr, erroffset, NULL); if (v->re == NULL) { VRE_free(&v); @@ -129,6 +138,7 @@ VRE_exec(const vre_t *code, const char *subject, int length, code->re_extra->flags &= ~PCRE_EXTRA_MATCH_LIMIT_RECURSION; } + AZ(options & (~VRE_MASK_EXEC)); return (pcre_exec(code->re, code->re_extra, subject, length, startoffset, options, ovector, ovecsize)); } From nils.goroll at uplex.de Thu Jun 10 13:54:05 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 10 Jun 2021 13:54:05 +0000 (UTC) Subject: [master] 8a8d6f5ee build: improve wflags.py for gcc Message-ID: <20210610135405.BDB3C1151EA@lists.varnish-cache.org> commit 8a8d6f5ee837593674d8da73e77a0bcd060417dd Author: Nils Goroll Date: Thu Jun 10 15:49:07 2021 +0200 build: improve wflags.py for gcc cc cannot -Wformat-y2k cc1: warning: -Wformat-y2k ignored without -Wformat [-Wformat-y2k] cc cannot -Wformat-zero-length cc1: warning: -Wformat-zero-length ignored without -Wformat [-Wformat-zero-length] diff --git a/wflags.py b/wflags.py index 1778fd41b..7cfffea57 100644 --- a/wflags.py +++ b/wflags.py @@ -48,8 +48,8 @@ DESIRABLE_WFLAGS = [ "-Wchar-subscripts", "-Wempty-body", "-Wextra", - "-Wformat-y2k", - "-Wformat-zero-length", + "-Wformat -Wformat-y2k", + "-Wformat -Wformat-zero-length", "-Wmissing-field-initializers", "-Wmissing-prototypes", "-Wnested-externs", @@ -85,7 +85,7 @@ UNDESIRABLE_WFLAGS = [ def cc(compiler, opt, obj, src): a = [compiler, "-c"] if opt is not None: - a.append(opt) + a += opt.split() a += ["-o", obj, src] try: From nils.goroll at uplex.de Thu Jun 10 14:22:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 10 Jun 2021 14:22:06 +0000 (UTC) Subject: [master] 41bb92b92 Update info on disabled vtc Message-ID: <20210610142206.1DEDE115F2D@lists.varnish-cache.org> commit 41bb92b924856538ae88f68565233b4db1119de4 Author: Nils Goroll Date: Thu Jun 10 16:02:53 2021 +0200 Update info on disabled vtc diff --git a/bin/varnishtest/tests.disabled/e00029.vtc b/bin/varnishtest/tests.disabled/e00029.vtc index ea3dd7006..3b17e2192 100644 --- a/bin/varnishtest/tests.disabled/e00029.vtc +++ b/bin/varnishtest/tests.disabled/e00029.vtc @@ -1,5 +1,9 @@ varnishtest "fun esi includes and ranges" +# this does not work yet because the ESI VDP pushes data directly +# to the next level up, which in turn is required to get the +# anything-in-gzip case right + server s1 { rxreq expect req.url == "/1" From nils.goroll at uplex.de Thu Jun 10 14:22:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 10 Jun 2021 14:22:06 +0000 (UTC) Subject: [master] 77d31e18e Remove OBE tests for good Message-ID: <20210610142206.39BDF115F30@lists.varnish-cache.org> commit 77d31e18e76e72566121769ef0771b80864aa845 Author: Nils Goroll Date: Thu Jun 10 16:07:25 2021 +0200 Remove OBE tests for good we test chunked + streaming and gzip + streaming in a number of vtcs since the do_stream default has been changed to true. And, yes, these vtcs do succeed. diff --git a/bin/varnishtest/tests.disabled/t00000.vtc b/bin/varnishtest/tests.disabled/t00000.vtc deleted file mode 100644 index c84100a8c..000000000 --- a/bin/varnishtest/tests.disabled/t00000.vtc +++ /dev/null @@ -1,53 +0,0 @@ -varnishtest "Ticket #873" - -barrier b1 cond 2 -barrier b2 cond 2 -barrier b3 cond 2 -barrier b4 cond 2 - -server s1 { - rxreq - txresp -nolen -hdr "Transfer-encoding: chunked" - chunked "<1>------------------------<1>\n" - barrier b1 sync - chunked "<2>------------------------<2>\n" - barrier b2 sync - chunked "<3>------------------------<3>\n" - barrier b3 sync - chunked "<4>------------------------<4>\n" - barrier b4 sync - chunkedlen 0 -} -start - -varnish v1 -vcl+backend { - sub vcl_backend_response { - set beresp.do_stream = true; - } -} -start - -varnish v1 -cliok "param.set debug +req_state" - -client c1 { - txreq -hdr "foo: /foo" - rxresp -no_obj - - rxchunk - expect resp.chunklen == 31 - barrier b1 sync - - rxchunk - expect resp.chunklen == 31 - barrier b2 sync - - rxchunk - expect resp.chunklen == 31 - barrier b3 sync - - rxchunk - expect resp.chunklen == 31 - barrier b4 sync - - rxchunk - expect resp.chunklen == 0 - expect resp.bodylen == 124 -} -run diff --git a/bin/varnishtest/tests.disabled/t00001.vtc b/bin/varnishtest/tests.disabled/t00001.vtc deleted file mode 100644 index a94e956aa..000000000 --- a/bin/varnishtest/tests.disabled/t00001.vtc +++ /dev/null @@ -1,50 +0,0 @@ -varnishtest "Test stream/gunzip" - -server s1 { - rxreq - expect req.url == "/bar" - txresp -body "foobar" - - rxreq - expect req.url == "/bla" - expect req.http.accept-encoding == "gzip" - txresp -gzipbody blablabla - - rxreq - expect req.url == "/foo" - txresp -body "snafu" - - rxreq - expect req.url == "/barf" - expect req.http.accept-encoding == "gzip" - txresp -gzipbody Iamoutofnonsensewords - -} -start - -varnish v1 -vcl+backend { - sub vcl_backend_response { - if (bereq.url == "/foo") { - set beresp.do_gzip = true; - } - if (bereq.url == "/barf") { - set beresp.do_gunzip = true; - } - set beresp.do_stream = true; - } -} -start - -client c1 { - txreq -url /bar - rxresp - expect resp.http.content-length == 6 - expect resp.bodylen == 6 - txreq -url /bla - rxresp - expect resp.bodylen == 9 - txreq -url /foo - rxresp - expect resp.bodylen == 5 - txreq -url /barf - rxresp - expect resp.bodylen == 21 -} -run From nils.goroll at uplex.de Thu Jun 10 14:22:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 10 Jun 2021 14:22:06 +0000 (UTC) Subject: [master] 0a445c835 Remove an OBE as can be test case from 2011 Message-ID: <20210610142206.61EDD115F34@lists.varnish-cache.org> commit 0a445c835d772d25d3bfd3832214e261015ebc42 Author: Nils Goroll Date: Thu Jun 10 16:12:38 2021 +0200 Remove an OBE as can be test case from 2011 This is from before the introduction of workspace overflow detection. Since then we would never silently process failing string assembly. diff --git a/bin/varnishtest/tests.disabled/r00693.vtc b/bin/varnishtest/tests.disabled/r00693.vtc deleted file mode 100644 index 4947dbcf6..000000000 --- a/bin/varnishtest/tests.disabled/r00693.vtc +++ /dev/null @@ -1,65 +0,0 @@ -varnishtest "check boundary condition on vrt_assemble_string()" - -feature 64bit - -server s1 { - rxreq - expect req.http.baz == - txresp -status 201 - - rxreq - expect req.http.baz == - txresp -status 202 - - rxreq - expect req.http.baz == "BAZ" - txresp -status 203 - -} -start - -varnish v1 -arg "-p workspace_client=3072" -vcl+backend { - - sub vcl_recv { - set req.http.foo = - req.http.bar + - "0123456789abcdef" + "0123456789abcdef" + - "0123456789abcdef" + "0123456789abcdef" + - "0123456789abcdef" + "0123456789abcdef" + - "0123456789abcdef" + "0123456789abcdef" + - "0123456789abcdef" + "0123456789abcdef" + - "0123456789abcdef" + "0123456789abcdef" + - "0123456789abcdef" + "0123456789abcdef" + - "0123456789abcdef" + "0123456789abcdef" + - "0123456789abcdef" + "0123456789abcdef" + - "0123456789abcdef" + "0123456789abcdef" + - "0123456789abcdef" + "0123456789abcdef" + - "0123456789abcdef" + "0123456789abcdef" + - "0123456789abcdef" + "0123456789abcdef" + - "0123456789abcde"; - set req.http.baz = "BAZ"; - return (pass); - } - sub vcl_hash { - hash_data(req.url); - return (lookup); - } -} -start - -client c1 { - # This should soak up all bytes but the last in the workspace - txreq -hdr "foo: x" -hdr "bar: A" - rxresp - expect resp.status == 201 - - # This should soak up all bytes in the workspace - txreq -hdr "foo: x" -hdr "bar: AB" - rxresp - expect resp.status == 202 - - # This overcommits the workspace, failing the "bar" set, - # Thus allowing the "baz" set to work. - txreq -hdr "foo: x" -hdr "bar: ABC" - rxresp - expect resp.status == 203 - -} -run From nils.goroll at uplex.de Fri Jun 11 07:43:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 11 Jun 2021 07:43:07 +0000 (UTC) Subject: [master] 1a81180e5 build: allow compiler in wflags.py to contain options Message-ID: <20210611074307.3F53710DA31@lists.varnish-cache.org> commit 1a81180e5cabedd1ed1b06efa7c5778144d5d264 Author: Nils Goroll Date: Fri Jun 11 09:42:18 2021 +0200 build: allow compiler in wflags.py to contain options diff --git a/wflags.py b/wflags.py index 7cfffea57..ce32037c4 100644 --- a/wflags.py +++ b/wflags.py @@ -83,7 +83,8 @@ UNDESIRABLE_WFLAGS = [ def cc(compiler, opt, obj, src): - a = [compiler, "-c"] + a = compiler.split() + a += ["-c"] if opt is not None: a += opt.split() a += ["-o", obj, src] From nils.goroll at uplex.de Fri Jun 11 09:42:05 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 11 Jun 2021 09:42:05 +0000 (UTC) Subject: [master] 6c6638366 Update the pcre recursion limit test Message-ID: <20210611094205.736A6110CAD@lists.varnish-cache.org> commit 6c663836667471d50ae26e2dc486bd00fb55e786 Author: Nils Goroll Date: Fri Jun 11 11:33:50 2021 +0200 Update the pcre recursion limit test This test had been disabled before to reduce the maintenance burdon, but I think it really should be possible to find a regex & pattern to reliably run into PCRE_ERROR_RECURSIONLIMIT / PCRE_ERROR_JIT_STACKLIMIT. And it seems that the work to find such a regex & pattern had already been done for us. :) Looking after the test made me realize that we really should consider latching VCL failure for "hard" pcre errors. This commit is to have a diff in the git log, the next commit will activate the test. diff --git a/bin/varnishtest/tests.disabled/r01576.vtc b/bin/varnishtest/tests.disabled/r01576.vtc index 9ed5f7a36..be7fe5d5a 100644 --- a/bin/varnishtest/tests.disabled/r01576.vtc +++ b/bin/varnishtest/tests.disabled/r01576.vtc @@ -2,17 +2,8 @@ varnishtest "Test recursive regexp's fail before consuming all the stack" feature pcre_jit -server s1 { - rxreq - expect req.http.found == "1" - txresp - rxreq - expect req.http.found == - txresp -} -start - # If you want to play around, uncomment the next lines and adjust -# the length of the aaaaaaaaaaa strings below to suit your needs. +# the length of the ABAB strings below to suit your needs. # Better yet: Rewrite your regexps to avoid this madness. # varnish v1 -arg "-p thread_pool_stack=48k" @@ -23,17 +14,26 @@ server s1 { # pcre_match_limit_recursion = thread_pool_stack * 2 - 9 varnish v1 -vcl+backend { + backend proforma none; + sub vcl_recv { - if (req.url ~ "^/a((?!/.).)*$") { - set req.http.found = "1"; + return (synth(200)); + } + sub vcl_synth { + # shamelessly copied from "bugzilla77 at gmail dot com" + # https://bugs.php.net/bug.php?id=70110 + if (req.url ~ "^/(A{1,2}B)+$") { + set resp.http.found = "1"; } } } -start # This should succeed with default params and JIT/no-JIT client c1 { - txreq -url /aaaaaaaa + txreq -url /ABAABABAABABABAB rxresp + expect resp.status == 200 + expect resp.http.found == 1 } -run # PCRE_ERROR_RECURSIONLIMIT (-21) @@ -44,8 +44,10 @@ logexpect l1 -v v1 { # This should fail with default params and JIT/no-JIT client c1 { - txreq -url /aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + txreq -url /ABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABAB rxresp + # XXX REALLY? shouldn't we fail the VCL for a hard PCRE error? + expect resp.status == 200 } -run logexpect l1 -wait From nils.goroll at uplex.de Fri Jun 11 09:42:05 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 11 Jun 2021 09:42:05 +0000 (UTC) Subject: [master] d6b4f7763 Activate the pcre recursion limit test again Message-ID: <20210611094205.87C6A110CB0@lists.varnish-cache.org> commit d6b4f7763aa1d2d88a16e3f2c3d3e08af24b2466 Author: Nils Goroll Date: Fri Jun 11 11:39:52 2021 +0200 Activate the pcre recursion limit test again diff --git a/bin/varnishtest/tests.disabled/r01576.vtc b/bin/varnishtest/r01576.vtc similarity index 100% rename from bin/varnishtest/tests.disabled/r01576.vtc rename to bin/varnishtest/r01576.vtc From nils.goroll at uplex.de Fri Jun 11 09:53:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 11 Jun 2021 09:53:06 +0000 (UTC) Subject: [master] 917385f5c Put the re-activated test in the right directory Message-ID: <20210611095306.644FF111468@lists.varnish-cache.org> commit 917385f5cb83984f19b16509a8675270e872cfad Author: Nils Goroll Date: Fri Jun 11 11:51:35 2021 +0200 Put the re-activated test in the right directory that was a stupid mistake diff --git a/bin/varnishtest/r01576.vtc b/bin/varnishtest/tests/r01576.vtc similarity index 100% rename from bin/varnishtest/r01576.vtc rename to bin/varnishtest/tests/r01576.vtc From nils.goroll at uplex.de Fri Jun 11 10:16:04 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 11 Jun 2021 10:16:04 +0000 (UTC) Subject: [master] 752949d70 pcre recursion test: workspace_client and http_req_size for 32bit Message-ID: <20210611101604.A7A92111F7E@lists.varnish-cache.org> commit 752949d70234e994e46bd8974a3a5e9e985efe2e Author: Nils Goroll Date: Fri Jun 11 12:13:49 2021 +0200 pcre recursion test: workspace_client and http_req_size for 32bit We tweak down the defaults on 32bit platforms, restore them to the 64bit defaults diff --git a/bin/varnishtest/tests/r01576.vtc b/bin/varnishtest/tests/r01576.vtc index be7fe5d5a..60277485d 100644 --- a/bin/varnishtest/tests/r01576.vtc +++ b/bin/varnishtest/tests/r01576.vtc @@ -13,7 +13,8 @@ feature pcre_jit # Approximate formula for FreeBSD/amd64: # pcre_match_limit_recursion = thread_pool_stack * 2 - 9 -varnish v1 -vcl+backend { +# -p: use 64bit defaults also on 32bit +varnish v1 -arg "-p workspace_client=64k -p http_req_size=32k" -vcl+backend { backend proforma none; sub vcl_recv { From fgsch at lodoss.net Sun Jun 13 21:33:12 2021 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 13 Jun 2021 21:33:12 +0000 (UTC) Subject: [master] 2ac270567 Fix compilation on macOS Big Sur Message-ID: <20210613213312.3417E102766@lists.varnish-cache.org> commit 2ac2705675acc99f47695cf6800254bc2cbbd454 Author: Federico G. Schwindt Date: Sun Jun 13 22:29:05 2021 +0100 Fix compilation on macOS Big Sur Fixes: In file included from vev.c:36: /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/pthread.h:328:6: error: macro expansion producing 'defined' has undefined behavior [-Werror,-Wexpansion-to-defined] #if !_PTHREAD_SWIFT_IMPORTER_NULLABILITY_COMPAT ^ /Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include/pthread.h:197:2: note: expanded from macro '_PTHREAD_SWIFT_IMPORTER_NULLABILITY_COMPAT' defined(SWIFT_CLASS_EXTRA) && (!defined(SWIFT_SDK_OVERLAY_PTHREAD_EPOCH) || (SWIFT_SDK_OVERLAY_PTHREAD_EPOCH < 1)) diff --git a/configure.ac b/configure.ac index 553e9f43d..6a2874f8d 100644 --- a/configure.ac +++ b/configure.ac @@ -710,6 +710,14 @@ else OCFLAGS="${OCFLAGS} -Werror"]) fi +case $target in + *-*-darwin*) + AX_CHECK_COMPILE_FLAG([-Wno-expansion-to-defined], + [CFLAGS="${CFLAGS} -Wno-expansion-to-defined" + OCFLAGS="${OCFLAGS} -Wno-expansion-to-defined"]) + ;; +esac + AX_CHECK_COMPILE_FLAG([-Werror=unused-result], [CFLAGS="${CFLAGS} -Wno-error=unused-result" OCFLAGS="${OCFLAGS} -Wno-error=unused-result"], From phk at FreeBSD.org Mon Jun 14 06:29:06 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 14 Jun 2021 06:29:06 +0000 (UTC) Subject: [master] 5fd49a1be Always assert VSB_finish() Message-ID: <20210614062906.DEFAE112FCF@lists.varnish-cache.org> commit 5fd49a1be43c6d18cef48b4638c5f2ca947cbe58 Author: Poul-Henning Kamp Date: Mon Jun 14 06:28:20 2021 +0000 Always assert VSB_finish() Spotted by: Coverity diff --git a/lib/libvarnishapi/vxp_test.c b/lib/libvarnishapi/vxp_test.c index 9f921c9a9..d44a9600a 100644 --- a/lib/libvarnishapi/vxp_test.c +++ b/lib/libvarnishapi/vxp_test.c @@ -77,7 +77,7 @@ main(int argc, char * const *argv) vex = vex_New(q_arg, vsb, 0); if (vex == NULL) { - VSB_finish(vsb); + AZ(VSB_finish(vsb)); fprintf(stderr, "Error:\n%s", VSB_data(vsb)); VSB_destroy(&vsb); free(q_arg); From nils.goroll at uplex.de Mon Jun 14 13:27:02 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 14 Jun 2021 13:27:02 +0000 (UTC) Subject: [master] bd36ad4d3 Fail the vcl for hard pcre errors Message-ID: <20210614132702.C0D805220@lists.varnish-cache.org> commit bd36ad4d38b4ccea5c6c1886469f6fe770365d65 Author: Nils Goroll Date: Fri Jun 11 11:54:32 2021 +0200 Fail the vcl for hard pcre errors When a regex match failed with a hard error (anything which is not just a "no match"), we would log a VCL_Error but otherwise just fail the match. Thus, such failures were likely go remain unnoticed, which could have serious implications in particular with negated matches. Ref 6c663836667471d50ae26e2dc486bd00fb55e786 diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c index 13135655c..63dfb9c07 100644 --- a/bin/varnishd/cache/cache_vrt_re.c +++ b/bin/varnishd/cache/cache_vrt_re.c @@ -88,7 +88,7 @@ VRT_re_match(VRT_CTX, const char *s, VCL_REGEX re) if (i >= 0) return (1); if (i < VRE_ERROR_NOMATCH ) - VSLb(ctx->vsl, SLT_VCL_Error, "Regexp matching returned %d", i); + VRT_fail(ctx, "Regexp matching returned %d", i); return (0); } @@ -122,7 +122,7 @@ VRT_regsub(VRT_CTX, int all, VCL_STRING str, VCL_REGEX re, if (i == VRE_ERROR_NOMATCH) return (str); if (i < VRE_ERROR_NOMATCH ) { - VSLb(ctx->vsl, SLT_VCL_Error, "Regexp matching returned %d", i); + VRT_fail(ctx, "Regexp matching returned %d", i); return (str); } @@ -159,8 +159,7 @@ VRT_regsub(VRT_CTX, int all, VCL_STRING str, VCL_REGEX re, &cache_param->vre_limits); if (i < VRE_ERROR_NOMATCH ) { WS_Release(ctx->ws, 0); - VSLb(ctx->vsl, SLT_VCL_Error, - "Regexp matching returned %d", i); + VRT_fail(ctx, "Regexp matching returned %d", i); return (str); } } while (i != VRE_ERROR_NOMATCH); diff --git a/bin/varnishtest/tests/r01576.vtc b/bin/varnishtest/tests/r01576.vtc index 60277485d..5dc0be6ec 100644 --- a/bin/varnishtest/tests/r01576.vtc +++ b/bin/varnishtest/tests/r01576.vtc @@ -46,9 +46,7 @@ logexpect l1 -v v1 { # This should fail with default params and JIT/no-JIT client c1 { txreq -url /ABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABAB - rxresp - # XXX REALLY? shouldn't we fail the VCL for a hard PCRE error? - expect resp.status == 200 + expect_close } -run logexpect l1 -wait From dridi.boukelmoune at gmail.com Mon Jun 14 13:43:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 14 Jun 2021 13:43:06 +0000 (UTC) Subject: [master] badee4616 req: New hash_ignore_vary flag Message-ID: <20210614134306.A25A55C20@lists.varnish-cache.org> commit badee4616bc83361c52e056bea7521b638f2b235 Author: Dridi Boukelmoune Date: Mon Jun 14 12:13:41 2021 +0200 req: New hash_ignore_vary flag It works like the hash_ignore_busy flag: it is disabled by default and survives a restart, but as the name implies will skip vary checks for object candidates. diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index d9a4e0c69..90c7a7ca8 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -435,6 +435,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp) continue; if (oc->boc && oc->boc->vary != NULL && + !req->hash_ignore_vary && !VRY_Match(req, oc->boc->vary)) { wrk->strangelove++; continue; @@ -453,7 +454,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp) continue; } - if (ObjHasAttr(wrk, oc, OA_VARY)) { + if (!req->hash_ignore_vary && ObjHasAttr(wrk, oc, OA_VARY)) { vary = ObjGetAttr(wrk, oc, OA_VARY, NULL); AN(vary); if (!VRY_Match(req, vary)) { diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index 101bf582c..e2981e69b 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -253,6 +253,7 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req) req->hash_always_miss = 0; req->hash_ignore_busy = 0; + req->hash_ignore_vary = 0; req->esi_level = 0; req->is_hit = 0; req->req_step = R_STP_TRANSPORT; diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 34fde97a1..683b08135 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -873,6 +873,7 @@ cnt_recv_prep(struct req *req, const char *ci) req->disable_esi = 0; req->hash_always_miss = 0; req->hash_ignore_busy = 0; + req->hash_ignore_vary = 0; req->client_identity = NULL; req->storage = NULL; } diff --git a/bin/varnishtest/tests/c00107.vtc b/bin/varnishtest/tests/c00107.vtc new file mode 100644 index 000000000..6f3dced1f --- /dev/null +++ b/bin/varnishtest/tests/c00107.vtc @@ -0,0 +1,42 @@ +varnishtest req.hash_ignore_vary + +server s1 { + rxreq + expect req.http.cookie ~ ab=a + txresp -hdr "vary: cookie" -body a + + rxreq + expect req.http.cookie ~ ab=b + txresp -hdr "vary: cookie" -body b +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + set req.hash_ignore_vary = req.http.user-agent ~ "bot"; + } + + sub vcl_req_cookie { + return; + } +} -start + +client ca { + txreq -hdr "cookie: ab=a" + rxresp + expect resp.body == a +} -run + +client cb { + txreq -hdr "cookie: ab=b" + rxresp + expect resp.body == b +} -run + +client cbot { + txreq -hdr "user-agent: googlebot" + rxresp + expect resp.body == b +} -run + +client ca -run +client cb -run diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index e1a673ec9..af4c7ecdf 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -352,6 +352,27 @@ req.hash_ignore_busy up content sideways from each other to avoid deadlocks. +req.hash_ignore_vary + + Type: BOOL + + Readable from: client + + Writable from: client + + Default: ``false``. + + Ignore objects vary headers during cache lookup. + + This returns the very first match regardless of the object + compatibility with the client request. This is useful when + variants are irrelevant to certain clients, and differences + in the way the resouce is presented don't change how the + client will interpret it. + + Use with caution. + + req.hash_always_miss Type: BOOL diff --git a/include/tbl/req_flags.h b/include/tbl/req_flags.h index 2e826601c..0c49540db 100644 --- a/include/tbl/req_flags.h +++ b/include/tbl/req_flags.h @@ -34,6 +34,7 @@ /* lower, vcl_r, vcl_w, doc */ REQ_FLAG(disable_esi, 0, 0, "") REQ_FLAG(hash_ignore_busy, 1, 1, "") +REQ_FLAG(hash_ignore_vary, 1, 1, "") REQ_FLAG(hash_always_miss, 1, 1, "") REQ_FLAG(is_hit, 0, 0, "") REQ_FLAG(is_hitmiss, 1, 0, "") From dridi.boukelmoune at gmail.com Thu Jun 17 09:19:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 17 Jun 2021 09:19:05 +0000 (UTC) Subject: [master] e5d6a0f4a vcc: Give the REGEX treatment to ACL expressions Message-ID: <20210617091905.262529506@lists.varnish-cache.org> commit e5d6a0f4a5a21f8db967f9c8ec4d2f03f0aa886b Author: Dridi Boukelmoune Date: Thu Jun 17 07:33:30 2021 +0200 vcc: Give the REGEX treatment to ACL expressions Except that we shouldn't start from expr0 since both use cases clearly need expr4 parsing. Instead of making assumptions based on the type in expr0, we split the final expression type check in its own function and use it to confirm that we are matching a regex or an acl. Fixes #3628 diff --git a/bin/varnishtest/tests/v00021.vtc b/bin/varnishtest/tests/v00021.vtc index b8c2c4b40..d0a4f391f 100644 --- a/bin/varnishtest/tests/v00021.vtc +++ b/bin/varnishtest/tests/v00021.vtc @@ -151,6 +151,15 @@ varnish v1 -errvcl {Symbols named 'vcl_*' are reserved.} { } } +# 3628 +varnish v1 -vcl { + backend be none; + acl foo { } + sub vcl_recv { + if (client.ip ~ foo || server.ip ~ foo) { } + } +} + varnish v1 -errvcl {Expression has type BOOL, expected ACL} { sub vcl_recv { if (client.ip ~ true) { } diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 4118d861b..329843bca 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -83,6 +83,8 @@ vcc_utype(vcc_type_t t) static void vcc_expr0(struct vcc *tl, struct expr **e, vcc_type_t fmt); static void vcc_expr_cor(struct vcc *tl, struct expr **e, vcc_type_t fmt); +static void vcc_expr_typecheck(struct vcc *tl, struct expr **e, vcc_type_t fmt, + struct token *t1); static struct expr * vcc_new_expr(vcc_type_t fmt) @@ -1107,12 +1109,16 @@ cmp_simple(struct vcc *tl, struct expr **e, const struct cmps *cp) static void v_matchproto_(cmp_f) cmp_regexp(struct vcc *tl, struct expr **e, const struct cmps *cp) { + struct token *t1; struct expr *e2; char buf[128]; *e = vcc_expr_edit(tl, STRING, "\vS", *e, NULL); vcc_NextToken(tl); - vcc_expr0(tl, &e2, REGEX); + t1 = tl->t; + vcc_expr4(tl, &e2, REGEX); + ERRCHK(tl); + vcc_expr_typecheck(tl, &e2, REGEX, t1); ERRCHK(tl); bprintf(buf, "%sVRT_re_match(ctx, \v1, \v2)", cp->emit); *e = vcc_expr_edit(tl, BOOL, buf, *e, e2); @@ -1121,11 +1127,15 @@ cmp_regexp(struct vcc *tl, struct expr **e, const struct cmps *cp) static void v_matchproto_(cmp_f) cmp_acl(struct vcc *tl, struct expr **e, const struct cmps *cp) { + struct token *t1; struct expr *e2; char buf[256]; vcc_NextToken(tl); - vcc_expr0(tl, &e2, ACL); + t1 = tl->t; + vcc_expr4(tl, &e2, ACL); + ERRCHK(tl); + vcc_expr_typecheck(tl, &e2, ACL, t1); ERRCHK(tl); bprintf(buf, "%sVRT_acl_match(ctx, \v1, \v2)", cp->emit); *e = vcc_expr_edit(tl, BOOL, buf, e2, *e); @@ -1366,8 +1376,6 @@ vcc_expr0(struct vcc *tl, struct expr **e, vcc_type_t fmt) t1 = tl->t; if (fmt->stringform) vcc_expr_cor(tl, e, STRINGS); - else if (fmt == REGEX) - vcc_expr4(tl, e, REGEX); else vcc_expr_cor(tl, e, fmt); ERRCHK(tl); @@ -1406,11 +1414,21 @@ vcc_expr0(struct vcc *tl, struct expr **e, vcc_type_t fmt) ERRCHK(tl); } + vcc_expr_typecheck(tl, e, fmt, t1); +} + +static void +vcc_expr_typecheck(struct vcc *tl, struct expr **e, vcc_type_t fmt, + struct token *t1) +{ + + assert(fmt != VOID); + assert(fmt != STRINGS); + if (fmt != (*e)->fmt) { VSB_printf(tl->sb, "Expression has type %s, expected %s\n", vcc_utype((*e)->fmt), vcc_utype(fmt)); vcc_ErrWhere2(tl, t1, tl->t); - return; } } From dridi.boukelmoune at gmail.com Thu Jun 17 12:35:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 17 Jun 2021 12:35:06 +0000 (UTC) Subject: [master] 114062dea vav: VAV_ParseTxt() to work without a null terminator Message-ID: <20210617123506.DA0EB101396@lists.varnish-cache.org> commit 114062dea9e895b1ef2136e6cae8c60f042c523a Author: Dridi Boukelmoune Date: Thu Jun 17 11:51:21 2021 +0200 vav: VAV_ParseTxt() to work without a null terminator I kept a variable called "s" to cut noise from the diff. diff --git a/include/vav.h b/include/vav.h index c94cee902..9e0208f57 100644 --- a/include/vav.h +++ b/include/vav.h @@ -31,6 +31,7 @@ */ void VAV_Free(char **argv); +char **VAV_ParseTxt(const char *b, const char *e, int *argc, int flag); char **VAV_Parse(const char *s, int *argc, int flag); char *VAV_BackSlashDecode(const char *s, const char *e); int VAV_BackSlash(const char *s, char *res); diff --git a/lib/libvarnish/vav.c b/lib/libvarnish/vav.c index 611ae4ba8..de944032c 100644 --- a/lib/libvarnish/vav.c +++ b/lib/libvarnish/vav.c @@ -138,14 +138,17 @@ static char err_invalid_backslash[] = "Invalid backslash sequence"; static char err_missing_quote[] = "Missing '\"'"; char ** -VAV_Parse(const char *s, int *argc, int flag) +VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) { char **argv; - const char *p; + const char *s, *p; int nargv, largv; int i, quote; - assert(s != NULL); + AN(b); + if (e == NULL) + e = strchr(b, '\0'); + s = b; nargv = 1; largv = 16; argv = calloc(largv, sizeof *argv); @@ -153,7 +156,7 @@ VAV_Parse(const char *s, int *argc, int flag) return (NULL); for (;;) { - if (*s == '\0') + if (s >= e) break; if (isspace(*s)) { s++; @@ -179,7 +182,7 @@ VAV_Parse(const char *s, int *argc, int flag) continue; } if (!quote) { - if (*s == '\0' || isspace(*s)) + if (s >= e || isspace(*s)) break; if ((flag & ARGV_COMMA) && *s == ',') break; @@ -188,7 +191,7 @@ VAV_Parse(const char *s, int *argc, int flag) } if (*s == '"' && !(flag & ARGV_NOESC)) break; - if (*s == '\0') { + if (s >= e) { argv[0] = err_missing_quote; return (argv); } @@ -207,7 +210,7 @@ VAV_Parse(const char *s, int *argc, int flag) } else { argv[nargv++] = VAV_BackSlashDecode(p, s); } - if (*s != '\0') + if (s < e) s++; } argv[nargv] = NULL; @@ -216,6 +219,13 @@ VAV_Parse(const char *s, int *argc, int flag) return (argv); } +char ** +VAV_Parse(const char *s, int *argc, int flag) +{ + + return (VAV_ParseTxt(s, NULL, argc, flag)); +} + void VAV_Free(char **argv) { From dridi.boukelmoune at gmail.com Thu Jun 17 12:35:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 17 Jun 2021 12:35:06 +0000 (UTC) Subject: [master] 3ebcd6d11 vav: Retire redundant variable Message-ID: <20210617123506.EEA4F101399@lists.varnish-cache.org> commit 3ebcd6d1134d8e235d16e1014396c68ae33f8385 Author: Dridi Boukelmoune Date: Thu Jun 17 11:55:09 2021 +0200 vav: Retire redundant variable See previous commit. Better diff with the --word-diff --word-diff-regex=. options. diff --git a/lib/libvarnish/vav.c b/lib/libvarnish/vav.c index de944032c..bc3eaf2cc 100644 --- a/lib/libvarnish/vav.c +++ b/lib/libvarnish/vav.c @@ -141,14 +141,13 @@ char ** VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) { char **argv; - const char *s, *p; + const char *p; int nargv, largv; int i, quote; AN(b); if (e == NULL) e = strchr(b, '\0'); - s = b; nargv = 1; largv = 16; argv = calloc(largv, sizeof *argv); @@ -156,62 +155,62 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) return (NULL); for (;;) { - if (s >= e) + if (b >= e) break; - if (isspace(*s)) { - s++; + if (isspace(*b)) { + b++; continue; } - if ((flag & ARGV_COMMENT) && *s == '#') + if ((flag & ARGV_COMMENT) && *b == '#') break; - if (*s == '"' && !(flag & ARGV_NOESC)) { - p = ++s; + if (*b == '"' && !(flag & ARGV_NOESC)) { + p = ++b; quote = 1; } else { - p = s; + p = b; quote = 0; } while (1) { - if (*s == '\\' && !(flag & ARGV_NOESC)) { - i = VAV_BackSlash(s, NULL); + if (*b == '\\' && !(flag & ARGV_NOESC)) { + i = VAV_BackSlash(b, NULL); if (i == 0) { argv[0] = err_invalid_backslash; return (argv); } - s += i; + b += i; continue; } if (!quote) { - if (s >= e || isspace(*s)) + if (b >= e || isspace(*b)) break; - if ((flag & ARGV_COMMA) && *s == ',') + if ((flag & ARGV_COMMA) && *b == ',') break; - s++; + b++; continue; } - if (*s == '"' && !(flag & ARGV_NOESC)) + if (*b == '"' && !(flag & ARGV_NOESC)) break; - if (s >= e) { + if (b >= e) { argv[0] = err_missing_quote; return (argv); } - s++; + b++; } if (nargv + 1 >= largv) { argv = realloc(argv, sizeof (*argv) * (largv += largv)); assert(argv != NULL); } if (flag & ARGV_NOESC) { - argv[nargv] = malloc(1L + (s - p)); + argv[nargv] = malloc(1L + (b - p)); assert(argv[nargv] != NULL); - memcpy(argv[nargv], p, s - p); - argv[nargv][s - p] = '\0'; + memcpy(argv[nargv], p, b - p); + argv[nargv][b - p] = '\0'; nargv++; } else { - argv[nargv++] = VAV_BackSlashDecode(p, s); + argv[nargv++] = VAV_BackSlashDecode(p, b); } - if (s < e) - s++; + if (b < e) + b++; } argv[nargv] = NULL; if (argc != NULL) From dridi.boukelmoune at gmail.com Fri Jun 18 04:11:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 18 Jun 2021 04:11:07 +0000 (UTC) Subject: [master] 78d056474 vtc: Stabilize i1 Message-ID: <20210618041107.30E2C11B0FC@lists.varnish-cache.org> commit 78d05647488d29afa5d2262c1f0f1d38246d5363 Author: Dridi Boukelmoune Date: Fri Jun 18 06:09:51 2021 +0200 vtc: Stabilize i1 Errors may be logged out of order. diff --git a/bin/varnishtest/tests/i00001.vtc b/bin/varnishtest/tests/i00001.vtc index b952a5016..08cb59add 100644 --- a/bin/varnishtest/tests/i00001.vtc +++ b/bin/varnishtest/tests/i00001.vtc @@ -38,9 +38,9 @@ varnish v1 -vcl+backend { } } -start -logexpect l1 -v v1 -g raw { - expect * 1001 VCL_Error "REAL overflow converting to string.*" - expect * 1004 VCL_Error "INT overflow converting to string.*" +logexpect l1 -v v1 -g raw -q VCL_Error { + expect ? 1001 VCL_Error "REAL overflow converting to string.*" + expect ? 1004 VCL_Error "INT overflow converting to string.*" } -start client c1 { From guillaume at varnish-software.com Fri Jun 18 07:58:06 2021 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Fri, 18 Jun 2021 07:58:06 +0000 (UTC) Subject: [master] 9d35b8417 [cci] apk packages need sudo Message-ID: <20210618075806.207576D58@lists.varnish-cache.org> commit 9d35b8417085ef7c4ea2857645595a21b61aaed2 Author: Guillaume Quintard Date: Fri Jun 18 00:57:19 2021 -0700 [cci] apk packages need sudo diff --git a/.circleci/make-apk-packages.sh b/.circleci/make-apk-packages.sh index 26b672c2c..a8b678f8f 100755 --- a/.circleci/make-apk-packages.sh +++ b/.circleci/make-apk-packages.sh @@ -2,7 +2,7 @@ set -eux -apk add -q --no-progress --update tar alpine-sdk +apk add -q --no-progress --update tar alpine-sdk sudo echo "PARAM_RELEASE: $PARAM_RELEASE" echo "PARAM_DIST: $PARAM_DIST" From dridi.boukelmoune at gmail.com Mon Jun 21 06:32:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 21 Jun 2021 06:32:06 +0000 (UTC) Subject: [master] ce6df7238 vsl: Remove redundant declarations Message-ID: <20210621063206.7163260C55@lists.varnish-cache.org> commit ce6df723814f91e860721c5461c24a47c09811a5 Author: Dridi Boukelmoune Date: Mon Jun 21 08:30:01 2021 +0200 vsl: Remove redundant declarations diff --git a/include/vapi/vsl_int.h b/include/vapi/vsl_int.h index d4b9fe54e..f4a0c86b8 100644 --- a/include/vapi/vsl_int.h +++ b/include/vapi/vsl_int.h @@ -42,7 +42,7 @@ #define VAPI_VSL_INT_H_INCLUDED #define VSL_CLASS "Log" -#define VSL_SEGMENTS 8U +#define VSL_SEGMENTS 8U // power of two /* * Shared memory log format diff --git a/include/vsl_priv.h b/include/vsl_priv.h index 73e98ffdd..7fd71b1ce 100644 --- a/include/vsl_priv.h +++ b/include/vsl_priv.h @@ -41,9 +41,6 @@ #include "vapi/vsl_int.h" -#define VSL_CLASS "Log" -#define VSL_SEGMENTS 8U // power of two - /* * Shared memory log format * From dridi.boukelmoune at gmail.com Mon Jun 21 19:54:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 21 Jun 2021 19:54:05 +0000 (UTC) Subject: [master] 31ed0b58d vtc: Stabilize t02015 Message-ID: <20210621195405.AB16EA9197@lists.varnish-cache.org> commit 31ed0b58dcd309843abf9cc978a603a733e8f81e Author: Dridi Boukelmoune Date: Mon Jun 21 21:52:35 2021 +0200 vtc: Stabilize t02015 Another case of non-deterministic log order. diff --git a/bin/varnishtest/tests/t02015.vtc b/bin/varnishtest/tests/t02015.vtc index 4dc6dec63..829873acd 100644 --- a/bin/varnishtest/tests/t02015.vtc +++ b/bin/varnishtest/tests/t02015.vtc @@ -8,9 +8,9 @@ server s1 { varnish v1 -cliok "param.set feature +http2" varnish v1 -vcl+backend "" -start -logexpect l1 -v v1 -g request { - expect * 1001 ReqAcct "46 0 46 97 12345 12442" - expect * 1003 ReqAcct "46 0 46 102 1000 1102" +logexpect l1 -v v1 -g raw -q ReqAcct { + expect ? 1001 ReqAcct "46 0 46 97 12345 12442" + expect ? 1003 ReqAcct "46 0 46 102 1000 1102" } -start client c1 { From dridi.boukelmoune at gmail.com Tue Jun 22 05:27:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 22 Jun 2021 05:27:06 +0000 (UTC) Subject: [master] 172e44101 vtc: Stabilize v56 Message-ID: <20210622052706.E355A61C94@lists.varnish-cache.org> commit 172e44101fc678606e1db9b4267b6704f9d89bd1 Author: Dridi Boukelmoune Date: Mon Jun 21 22:30:18 2021 +0200 vtc: Stabilize v56 The probe timeout is irrelevant, especially under load. And to be certain, we can witness probe updates to ensure we are testing VCL with a system in the expected state. diff --git a/bin/varnishtest/tests/v00056.vtc b/bin/varnishtest/tests/v00056.vtc index b3c559f7a..100c24ba9 100644 --- a/bin/varnishtest/tests/v00056.vtc +++ b/bin/varnishtest/tests/v00056.vtc @@ -26,7 +26,7 @@ varnish v1 -vcl { probe foo { .url = "/"; - .timeout = 1s; + .timeout = 1m; .interval = 1s; .window = 3; .threshold = 2; @@ -56,18 +56,30 @@ client c1 { expect resp.status == 500 } -run +logexpect l1 -v v1 -g raw -q Backend_health { + expect 0 0 Backend_health "default Still sick" +} -start + barrier b1 sync barrier b2 sync +logexpect l1 -wait + client c2 { txreq rxresp expect resp.status == 500 } -run +logexpect l2 -v v1 -g raw -q Backend_health { + expect 0 0 Backend_health "default Went healthy" +} -start + barrier b3 sync barrier b4 sync +logexpect l2 -wait + client c3 { txreq rxresp From dridi.boukelmoune at gmail.com Tue Jun 22 05:27:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 22 Jun 2021 05:27:06 +0000 (UTC) Subject: [master] 11b93b0a3 circleci: Disable dependency tracking on alpine Message-ID: <20210622052707.04E8261C98@lists.varnish-cache.org> commit 11b93b0a3cede665eac3a06c55d17af7fe6b659b Author: Dridi Boukelmoune Date: Tue Jun 22 07:25:16 2021 +0200 circleci: Disable dependency tracking on alpine It started failing out of the blue and it's not obvious why. diff --git a/.circleci/config.yml b/.circleci/config.yml index 53457fee0..89c16bd80 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -302,7 +302,7 @@ workflows: name: distcheck_alpine dist: alpine release: "latest" - extra_conf: --disable-developer-warnings + extra_conf: --disable-developer-warnings --disable-dependency-tracking - distcheck: name: distcheck_archlinux dist: archlinux From dridi.boukelmoune at gmail.com Tue Jun 22 05:45:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 22 Jun 2021 05:45:06 +0000 (UTC) Subject: [master] 3e7d376f7 vtc: Stabilize r1857 Message-ID: <20210622054506.62234628B8@lists.varnish-cache.org> commit 3e7d376f7303b0d42d1185f4633f925fec891cb3 Author: Dridi Boukelmoune Date: Tue Jun 22 07:41:54 2021 +0200 vtc: Stabilize r1857 It is possible to see the session close after the second request after timeout_linger triggered. diff --git a/bin/varnishtest/tests/r01857.vtc b/bin/varnishtest/tests/r01857.vtc index 5686be8fd..6e1256e8e 100644 --- a/bin/varnishtest/tests/r01857.vtc +++ b/bin/varnishtest/tests/r01857.vtc @@ -22,7 +22,7 @@ client c1 { # Give varnish a chance to update stats delay .1 -varnish v1 -expect sess_herd == 1 +varnish v1 -expect sess_herd >= 1 varnish v1 -expect sess_conn == 1 varnish v1 -expect cache_hit == 1 varnish v1 -expect cache_miss == 1 From dridi.boukelmoune at gmail.com Tue Jun 22 07:04:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 22 Jun 2021 07:04:05 +0000 (UTC) Subject: [master] 4b03546aa vtc: Stabilize c69 Message-ID: <20210622070405.C1E1F64DB5@lists.varnish-cache.org> commit 4b03546aac3bc365977f45acf9bf7ad47c944995 Author: Dridi Boukelmoune Date: Tue Jun 22 09:02:00 2021 +0200 vtc: Stabilize c69 Latch the barrier _after_ reading resp.is_streaming. diff --git a/bin/varnishtest/tests/c00069.vtc b/bin/varnishtest/tests/c00069.vtc index 55e9557d0..3f91696a6 100644 --- a/bin/varnishtest/tests/c00069.vtc +++ b/bin/varnishtest/tests/c00069.vtc @@ -27,10 +27,10 @@ varnish v1 -vcl+backend { set resp.http.streaming = resp.is_streaming; } sub vcl_deliver { + set resp.http.streaming = resp.is_streaming; if (obj.hits == 0) { vtc.barrier_sync("${b2_sock}"); } - set resp.http.streaming = resp.is_streaming; } } -start From guillaume at varnish-software.com Wed Jun 23 00:23:08 2021 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Wed, 23 Jun 2021 00:23:08 +0000 (UTC) Subject: [master] 7b422447c [cci] refactor Message-ID: <20210623002308.BAE152AC1@lists.varnish-cache.org> commit 7b422447c5c2d6dbc0b1b0159ae9e273f49656ee Author: Guillaume Quintard Date: Tue Jun 22 12:04:34 2021 -0700 [cci] refactor diff --git a/.circleci/config.yml b/.circleci/config.yml index 89c16bd80..61342131f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,11 +1,5 @@ version: 2.1 -aliases: - pkg_req: &pkg_req - requires: - - dist - - tar_pkg_tools - parameters: vc-commit: type: string @@ -89,47 +83,75 @@ jobs: - alpine.tar.gz package: parameters: - dist: - description: the Linux distribution (debian|ubuntu|centos) - type: string - release: - description: the release name (stretch|buster|xenial|bionic|7|8) - type: string - ext: - description: the package extension (deb|rpm|apk) + platform: + description: the Linux distribution, with release, e.g. debian:buster, centos:7 type: string - arch: - description: the architecture (x64|aarch64) - type: string - image: - description: the base Docker image for Dockerfile + rclass: + description: the resource class to use, usuall arm.medium or medium type: string machine: - image: ubuntu-2004:202010-01 + image: ubuntu-2004:202101-01 + resource_class: << parameters.rclass >> steps: - attach_workspace: at: ~/project - - run: ls -la ~/project - - run: - name: Activate QEMU - command: | - sudo docker run -it --rm --privileged multiarch/qemu-user-static --reset --credential yes --persistent yes - - run: - name: Create Dockerfile - command: | - echo "FROM << parameters.image >>" > Dockerfile - echo "ADD make-<< parameters.ext >>-packages.sh /usr/bin/" >> Dockerfile - echo 'CMD ["make-<< parameters.ext >>-packages.sh"]' >> Dockerfile + + - when: + condition: + matches: + pattern: ^alpine.* + value: << parameters.platform >> + steps: + - run: + # https://wiki.alpinelinux.org/wiki/Release_Notes_for_Alpine_3.14.0#faccessat2 + name: grab the latest docker version + command: | + # using https://docs.docker.com/engine/install/ubuntu/ + sudo apt-get update + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + case "<< parameters.rclass >>" in + arm.*) ARCH=arm64;; + *) ARCH=amd64;; + esac + echo \ + "deb [signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt-get install docker-ce docker-ce-cli containerd.io - run: - name: Build << parameters.dist >> << parameters.release >> << parameters.arch >> << parameters.ext >> + name: Build for << parameters.platform >> on << parameters.rclass >> command: | mkdir -p packages - cp .circleci/make-<< parameters.ext >>-packages.sh . - docker build -t varnish-<< parameters.ext >>-package-build:<< parameters.arch >> . - docker run --rm -it -e PARAM_DIST=<< parameters.dist >> -e PARAM_RELEASE=<< parameters.release >> -v$(pwd):/varnish-cache varnish-<< parameters.ext >>-package-build:<< parameters.arch >> + case "<< parameters.platform >>" in + debian:*|ubuntu:*) EXT=deb ;; + centos:*) EXT=rpm ;; + alpine:*) EXT=apk ;; + *) + echo "unrecognized platform: << parameters.platform >>" + exit 1 + esac + + case "<< parameters.rclass >>" in + arm.*) + IMG=arm64v8/<< parameters.platform >> + ;; + *) + IMG=<< parameters.platform >> + ;; + esac + + set -x + echo "FROM $IMG" > Dockerfile + echo "ADD make-$EXT-packages.sh /usr/bin/" >> Dockerfile + echo "CMD make-$EXT-packages.sh" >> Dockerfile + + cp .circleci/make-$EXT-packages.sh . + docker build -t docker_img . + docker run --rm -it -e PARAM_DIST=$(echo "<< parameters.platform >>" | cut -d: -f1) -e PARAM_RELEASE=$(echo "<< parameters.platform >>" | cut -d: -f2) -v$(pwd):/varnish-cache docker_img - run: name: List created packages - command: find ./packages -name "*.<< parameters.ext >>" + command: find ./packages -type f - persist_to_workspace: root: . paths: @@ -319,167 +341,24 @@ workflows: - dist - tar_pkg_tools - package: - name: aarch64-ubuntu-focal - dist: ubuntu - release: focal - arch: aarch64 - image: arm64v8/ubuntu:focal - ext: deb - <<: *pkg_req - - package: - name: x64-ubuntu-focal - dist: ubuntu - release: focal - arch: x64 - image: ubuntu:focal - ext: deb - <<: *pkg_req - - package: - name: aarch64-ubuntu-bionic - dist: ubuntu - release: bionic - arch: aarch64 - image: arm64v8/ubuntu:bionic - ext: deb - <<: *pkg_req - - package: - name: x64-ubuntu-bionic - dist: ubuntu - release: bionic - arch: x64 - image: ubuntu:bionic - ext: deb - <<: *pkg_req - - package: - name: aarch64-ubuntu-xenial - dist: ubuntu - release: xenial - arch: aarch64 - image: arm64v8/ubuntu:xenial - ext: deb - <<: *pkg_req - - package: - name: x64-ubuntu-xenial - dist: ubuntu - release: xenial - arch: x64 - image: ubuntu:xenial - ext: deb - <<: *pkg_req - - package: - name: aarch64-debian-bullseye - dist: debian - release: bullseye - arch: aarch64 - image: arm64v8/debian:bullseye-slim - ext: deb - <<: *pkg_req - - package: - name: x64-debian-bullseye - dist: debian - release: bullseye - arch: x64 - image: debian:bullseye-slim - ext: deb - <<: *pkg_req - - package: - name: aarch64-debian-buster - dist: debian - release: buster - arch: aarch64 - image: arm64v8/debian:buster-slim - ext: deb - <<: *pkg_req - - package: - name: x64-debian-buster - dist: debian - release: buster - arch: x64 - image: debian:buster-slim - ext: deb - <<: *pkg_req - - package: - name: aarch64-debian-stretch - dist: debian - release: stretch - arch: aarch64 - image: arm64v8/debian:stretch-slim - ext: deb - <<: *pkg_req - - package: - name: x64-debian-stretch - dist: debian - release: stretch - arch: x64 - image: debian:stretch-slim - ext: deb - <<: *pkg_req - - package: - name: aarch64-centos-7 - dist: centos - release: "7" - arch: aarch64 - image: arm64v8/centos:7 - ext: rpm - <<: *pkg_req - - package: - name: x64-centos-7 - dist: centos - release: "7" - arch: x64 - image: centos:7 - ext: rpm - <<: *pkg_req - - package: - name: aarch64-centos-8 - dist: centos - release: "8" - arch: aarch64 - image: arm64v8/centos:8 - ext: rpm - <<: *pkg_req - - package: - name: x64-centos-8 - dist: centos - release: "8" - arch: x64 - image: centos:8 - ext: rpm - <<: *pkg_req - - package: - name: x64-alpine-3 - dist: alpine - release: "3" - arch: x64 - image: alpine:3 - ext: apk - <<: *pkg_req - - package: - name: aarch64-alpine-3 - dist: alpine - release: "3" - arch: aarch64 - image: arm64v8/alpine:3 - ext: apk - <<: *pkg_req + name: << matrix.platform >> packages (<< matrix.rclass >>) + requires: + - dist + - tar_pkg_tools + matrix: + parameters: + platform: + - ubuntu:bionic + - ubuntu:xenial + - debian:bullseye + - debian:buster + - debian:stretch + - centos:7 + - centos:8 + - alpine:3 + rclass: + - arm.medium + - medium - collect_packages: requires: - - x64-ubuntu-xenial - - aarch64-ubuntu-xenial - - x64-ubuntu-bionic - - aarch64-ubuntu-bionic - - x64-ubuntu-focal - - aarch64-ubuntu-focal - - x64-debian-stretch - - aarch64-debian-stretch - - x64-debian-buster - - aarch64-debian-buster - - x64-debian-bullseye - - aarch64-debian-bullseye - - x64-centos-7 - - aarch64-centos-7 - - x64-centos-8 - - aarch64-centos-8 - - x64-alpine-3 - - aarch64-alpine-3 - + - package From dridi.boukelmoune at gmail.com Wed Jun 23 07:00:09 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 23 Jun 2021 07:00:09 +0000 (UTC) Subject: [master] 3c67da447 varnishtest: Make barriers reactive on error Message-ID: <20210623070009.DF35A92E3D@lists.varnish-cache.org> commit 3c67da44753cccbf5db4d2bef881abe5b7c817f4 Author: Dridi Boukelmoune Date: Wed Jun 23 08:06:22 2021 +0200 varnishtest: Make barriers reactive on error A failing test may take up to the vtc timeout to effectively fail if a thread is waiting for a barrier to sync. Barriers now check every 100ms whether the vtc is done to bail out. The socket barriers were already polling for 1s, so now they poll for 100ms and leave the loop early when needed. The condvar barriers now loop over a timedwait, but critical sections were too wide. There were vtc_fatal() calls under barrier locks that would defeat the timedwait loop. Thus critical sections were more carefully laid, allowing multiple barrier commands to run simultaneously. This also means that besides initialization, socket barriers no longer touch the lock. Consider the following diff: --- bin/varnishtest/tests/v00056.vtc +++ bin/varnishtest/tests/v00056.vtc @@ -72,7 +72,7 @@ client c2 { } -run logexpect l2 -v v1 -g raw -q Backend_health { - expect 0 0 Backend_health "default Went healthy" + expect 0 0 Backend_health "default Went wrong" } -start barrier b3 sync With this change l2 would fail early, typically in less than 2s, but the barriers stuck in a condwait would prevent further progress. diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index 3a83b771d..25e63bb86 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -41,6 +41,7 @@ #include "vtc.h" #include "vtcp.h" +#include "vtim.h" #include "vsa.h" enum barrier_e { @@ -119,8 +120,10 @@ barrier_cond(struct barrier *b, const char *av, struct vtclog *vl) { CHECK_OBJ_NOTNULL(b, BARRIER_MAGIC); + AZ(pthread_mutex_lock(&b->mtx)); barrier_expect(b, av, vl); b->type = BARRIER_COND; + AZ(pthread_mutex_unlock(&b->mtx)); } static void * @@ -170,11 +173,11 @@ barrier_sock_thread(void *priv) conns = calloc(b->expected, sizeof *conns); AN(conns); - while (b->active) { + while (b->active && !vtc_stop && !vtc_error) { pfd[0].fd = sock; pfd[0].events = POLLIN; - i = poll(pfd, 1, 1000); + i = poll(pfd, 1, 100); if (i == 0) continue; if (i < 0) { @@ -222,6 +225,19 @@ barrier_sock_thread(void *priv) b->active = 0; } + if (vtc_stop || vtc_error) { + AN(b->active); + /* wake up outstanding waiters */ + for (i = 0; i < b->waiters; i++) + closefd(&conns[i]); + b->waiters = 0; + b->active = 0; + } + + if (b->cyclic && b->waiters > 0) + vtc_fatal(vl, "Barrier(%s) has %u outstanding waiters", + b->name, b->waiters); + macro_undef(vl, b->name, "addr"); macro_undef(vl, b->name, "port"); macro_undef(vl, b->name, "sock"); @@ -237,6 +253,7 @@ barrier_sock(struct barrier *b, const char *av, struct vtclog *vl) { CHECK_OBJ_NOTNULL(b, BARRIER_MAGIC); + AZ(pthread_mutex_lock(&b->mtx)); barrier_expect(b, av, vl); b->type = BARRIER_SOCK; b->active = 1; @@ -247,23 +264,33 @@ barrier_sock(struct barrier *b, const char *av, struct vtclog *vl) */ AZ(pthread_create(&b->thread, NULL, barrier_sock_thread, b)); AZ(pthread_cond_wait(&b->cond, &b->mtx)); + AZ(pthread_mutex_unlock(&b->mtx)); } static void barrier_cyclic(struct barrier *b, struct vtclog *vl) { + enum barrier_e t; + int w; CHECK_OBJ_NOTNULL(b, BARRIER_MAGIC); - if (b->type == BARRIER_NONE) + AZ(pthread_mutex_lock(&b->mtx)); + t = b->type; + w = b->waiters; + AZ(pthread_mutex_unlock(&b->mtx)); + + if (t == BARRIER_NONE) vtc_fatal(vl, "Barrier(%s) use error: not initialized", b->name); - if (b->waiters != 0) + if (w != 0) vtc_fatal(vl, "Barrier(%s) use error: already in use", b->name); + AZ(pthread_mutex_lock(&b->mtx)); b->cyclic = 1; + AZ(pthread_mutex_unlock(&b->mtx)); } /********************************************************************** @@ -273,17 +300,29 @@ barrier_cyclic(struct barrier *b, struct vtclog *vl) static void barrier_cond_sync(struct barrier *b, struct vtclog *vl) { + struct timespec ts; + int r, w; CHECK_OBJ_NOTNULL(b, BARRIER_MAGIC); assert(b->type == BARRIER_COND); - assert(b->waiters <= b->expected); - if (b->waiters == b->expected) + AZ(pthread_mutex_lock(&b->mtx)); + w = b->waiters; + assert(w <= b->expected); + + if (w == b->expected) + w = -1; + else + b->waiters = ++w; + AZ(pthread_mutex_unlock(&b->mtx)); + + if (w < 0) vtc_fatal(vl, "Barrier(%s) use error: more waiters than the %u expected", b->name, b->expected); - if (++b->waiters == b->expected) { + AZ(pthread_mutex_lock(&b->mtx)); + if (w == b->expected) { vtc_log(vl, 4, "Barrier(%s) wake %u", b->name, b->expected); if (b->cyclic) b->waiters = 0; @@ -291,8 +330,13 @@ barrier_cond_sync(struct barrier *b, struct vtclog *vl) } else { vtc_log(vl, 4, "Barrier(%s) wait %u of %u", b->name, b->waiters, b->expected); - AZ(pthread_cond_wait(&b->cond, &b->mtx)); + do { + ts = VTIM_timespec(VTIM_real() + .1); + r = pthread_cond_timedwait(&b->cond, &b->mtx, &ts); + assert(r == 0 || r == ETIMEDOUT); + } while (!vtc_stop && !vtc_error && r == ETIMEDOUT); } + AZ(pthread_mutex_unlock(&b->mtx)); } static void @@ -317,11 +361,7 @@ barrier_sock_sync(struct barrier *b, struct vtclog *vl) VSB_destroy(&vsb); - /* emulate pthread_cond_wait's behavior */ - AZ(pthread_mutex_unlock(&b->mtx)); sz = read(sock, buf, sizeof buf); /* XXX loop with timeout? */ - AZ(pthread_mutex_lock(&b->mtx)); - i = errno; closefd(&sock); @@ -420,7 +460,6 @@ cmd_barrier(CMD_ARGS) break; case BARRIER_SOCK: if (b->need_join) { - b->active = 0; AZ(pthread_join(b->thread, NULL)); b->need_join = 0; } @@ -445,7 +484,6 @@ cmd_barrier(CMD_ARGS) b = barrier_new(av[0], vl); av++; - AZ(pthread_mutex_lock(&b->mtx)); for (; *av != NULL; av++) { if (!strcmp(*av, "cond")) { av++; @@ -469,5 +507,4 @@ cmd_barrier(CMD_ARGS) } vtc_fatal(vl, "Unknown barrier argument: %s", *av); } - AZ(pthread_mutex_unlock(&b->mtx)); } From dridi.boukelmoune at gmail.com Wed Jun 23 07:00:10 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 23 Jun 2021 07:00:10 +0000 (UTC) Subject: [master] 1ccba7210 varnishtest: Teach logexpect to stop with the vtc Message-ID: <20210623070010.0524E92E40@lists.varnish-cache.org> commit 1ccba7210d062e856672d443364d4fc4c7cfe9ae Author: Dridi Boukelmoune Date: Wed Jun 23 08:22:31 2021 +0200 varnishtest: Teach logexpect to stop with the vtc With a non-deterministic `expect * * ...` the logexpect thread would never consider itself to be done if the vtc ends if one entry was not seen in the vsl. diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index d1af7d733..ba7818555 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -545,7 +545,7 @@ logexp_thread(void *priv) if (le->query != NULL) vtc_log(le->vl, 4, "qry| %s", le->query); logexp_next(le); - while (! logexp_done(le)) { + while (!logexp_done(le) && !vtc_stop && !vtc_error) { i = VSLQ_Dispatch(le->vslq, logexp_dispatch, le); if (i == 2 && le->err_arg) { vtc_log(le->vl, 4, "end| failed as expected"); @@ -558,6 +558,8 @@ logexp_thread(void *priv) else if (i == 0 && ! logexp_done(le)) VTIM_sleep(0.01); } + if (!logexp_done(le)) + vtc_fatal(le->vl, "bad| outstanding expectations"); vtc_log(le->vl, 4, "end|"); return (NULL); From dridi.boukelmoune at gmail.com Wed Jun 23 16:22:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 23 Jun 2021 16:22:06 +0000 (UTC) Subject: [master] f1e86c860 varnishtest: Barriers may miss condvar signals Message-ID: <20210623162206.72958A9AAC@lists.varnish-cache.org> commit f1e86c860da7765e9179eba197673b8a43601d97 Author: Dridi Boukelmoune Date: Wed Jun 23 18:20:39 2021 +0200 varnishtest: Barriers may miss condvar signals diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index 25e63bb86..4c0ede383 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -61,6 +61,7 @@ struct barrier { int waiters; int expected; int cyclic; + int cycle; enum barrier_e type; /* fields below are only for BARRIER_SOCK */ @@ -301,7 +302,7 @@ static void barrier_cond_sync(struct barrier *b, struct vtclog *vl) { struct timespec ts; - int r, w; + int r, w, c; CHECK_OBJ_NOTNULL(b, BARRIER_MAGIC); assert(b->type == BARRIER_COND); @@ -314,6 +315,8 @@ barrier_cond_sync(struct barrier *b, struct vtclog *vl) w = -1; else b->waiters = ++w; + + c = b->cycle; AZ(pthread_mutex_unlock(&b->mtx)); if (w < 0) @@ -324,6 +327,7 @@ barrier_cond_sync(struct barrier *b, struct vtclog *vl) AZ(pthread_mutex_lock(&b->mtx)); if (w == b->expected) { vtc_log(vl, 4, "Barrier(%s) wake %u", b->name, b->expected); + b->cycle++; if (b->cyclic) b->waiters = 0; AZ(pthread_cond_broadcast(&b->cond)); @@ -334,7 +338,8 @@ barrier_cond_sync(struct barrier *b, struct vtclog *vl) ts = VTIM_timespec(VTIM_real() + .1); r = pthread_cond_timedwait(&b->cond, &b->mtx, &ts); assert(r == 0 || r == ETIMEDOUT); - } while (!vtc_stop && !vtc_error && r == ETIMEDOUT); + } while (!vtc_stop && !vtc_error && r == ETIMEDOUT && + c == b->cycle); } AZ(pthread_mutex_unlock(&b->mtx)); } From dridi.boukelmoune at gmail.com Thu Jun 24 06:28:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 24 Jun 2021 06:28:07 +0000 (UTC) Subject: [master] 1001a4a25 varnishtest: Remove useless barrier fields Message-ID: <20210624062807.4169C933CF@lists.varnish-cache.org> commit 1001a4a25b8d8985a6051ecdd1eb216f33f830d6 Author: Dridi Boukelmoune Date: Thu Jun 24 08:19:17 2021 +0200 varnishtest: Remove useless barrier fields Better diff with the --ignore-all-space option. diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index 4c0ede383..5c8ea2590 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -61,13 +61,10 @@ struct barrier { int waiters; int expected; int cyclic; - int cycle; + int cycle; /* BARRIER_COND only */ enum barrier_e type; - /* fields below are only for BARRIER_SOCK */ - pthread_t thread; - volatile unsigned active; - volatile unsigned need_join; + pthread_t thread; /* BARRIER_SOCK only */ }; static VTAILQ_HEAD(, barrier) barriers = VTAILQ_HEAD_INITIALIZER(barriers); @@ -174,7 +171,7 @@ barrier_sock_thread(void *priv) conns = calloc(b->expected, sizeof *conns); AN(conns); - while (b->active && !vtc_stop && !vtc_error) { + while (!vtc_stop && !vtc_error) { pfd[0].fd = sock; pfd[0].events = POLLIN; @@ -223,22 +220,18 @@ barrier_sock_thread(void *priv) if (b->cyclic) b->waiters = 0; else - b->active = 0; + break; } - if (vtc_stop || vtc_error) { - AN(b->active); + if (b->waiters % b->expected > 0) { /* wake up outstanding waiters */ for (i = 0; i < b->waiters; i++) closefd(&conns[i]); - b->waiters = 0; - b->active = 0; + if (!vtc_error) + vtc_fatal(vl, "Barrier(%s) has %u outstanding waiters", + b->name, b->waiters); } - if (b->cyclic && b->waiters > 0) - vtc_fatal(vl, "Barrier(%s) has %u outstanding waiters", - b->name, b->waiters); - macro_undef(vl, b->name, "addr"); macro_undef(vl, b->name, "port"); macro_undef(vl, b->name, "sock"); @@ -257,8 +250,6 @@ barrier_sock(struct barrier *b, const char *av, struct vtclog *vl) AZ(pthread_mutex_lock(&b->mtx)); barrier_expect(b, av, vl); b->type = BARRIER_SOCK; - b->active = 1; - b->need_join = 1; /* NB. We can use the BARRIER_COND's pthread_cond_t to wait until the * socket is ready for convenience. @@ -464,10 +455,7 @@ cmd_barrier(CMD_ARGS) case BARRIER_COND: break; case BARRIER_SOCK: - if (b->need_join) { - AZ(pthread_join(b->thread, NULL)); - b->need_join = 0; - } + AZ(pthread_join(b->thread, NULL)); break; default: WRONG("Wrong barrier type"); From dridi.boukelmoune at gmail.com Thu Jun 24 08:32:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 24 Jun 2021 08:32:05 +0000 (UTC) Subject: [master] 6ddb405ff varnishtest: Constify barrier_sock_sync() Message-ID: <20210624083205.5E94E96F45@lists.varnish-cache.org> commit 6ddb405ffbcc3418efce20425f7c9cd92dd40b87 Author: Dridi Boukelmoune Date: Thu Jun 24 10:24:27 2021 +0200 varnishtest: Constify barrier_sock_sync() Spotted by flexelint. diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index 5c8ea2590..98df1fc8a 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -336,7 +336,7 @@ barrier_cond_sync(struct barrier *b, struct vtclog *vl) } static void -barrier_sock_sync(struct barrier *b, struct vtclog *vl) +barrier_sock_sync(const struct barrier *b, struct vtclog *vl) { struct vsb *vsb; const char *err; From dridi.boukelmoune at gmail.com Sun Jun 27 11:46:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Sun, 27 Jun 2021 11:46:05 +0000 (UTC) Subject: [master] 3cb7f80cf vtc: Stabilize d3 Message-ID: <20210627114605.E332CA4DA7@lists.varnish-cache.org> commit 3cb7f80cf810f2a03609aaed7ed0ce89bdcc3da8 Author: Dridi Boukelmoune Date: Sun Jun 27 13:29:58 2021 +0200 vtc: Stabilize d3 There's always the possibility to fail reusing s2's first connection after it was closed. diff --git a/bin/varnishtest/tests/d00003.vtc b/bin/varnishtest/tests/d00003.vtc index fc799c72b..6f51fbed6 100644 --- a/bin/varnishtest/tests/d00003.vtc +++ b/bin/varnishtest/tests/d00003.vtc @@ -11,7 +11,7 @@ server s1 { txresp -hdr "Foo: 8" -body "8" } -start -server s2 { +server s2 -repeat 2 -keepalive { rxreq txresp -hdr "Foo: 1" -body "1" rxreq @@ -80,8 +80,6 @@ client c1 { expect resp.http.foo == "9" } -run -server s2 -start - client c1 { txreq -req "DELETE" -hdr "Host: 127.0.0.1" rxresp From dridi.boukelmoune at gmail.com Sun Jun 27 11:46:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Sun, 27 Jun 2021 11:46:05 +0000 (UTC) Subject: [master] 97afc52e4 vtc: Stabilize c93 Message-ID: <20210627114605.CE82EA4DA0@lists.varnish-cache.org> commit 97afc52e40157be7e0692ccf53bcc467e0d26339 Author: Dridi Boukelmoune Date: Sun Jun 27 13:28:16 2021 +0200 vtc: Stabilize c93 On a loaded system the delay may not be enough, a barrier should. diff --git a/bin/varnishtest/tests/c00093.vtc b/bin/varnishtest/tests/c00093.vtc index 625dc3d11..9b8899dd3 100644 --- a/bin/varnishtest/tests/c00093.vtc +++ b/bin/varnishtest/tests/c00093.vtc @@ -1,13 +1,16 @@ varnishtest "Test resp.is_streaming with a UDS backend" +barrier b1 sock 2 + server s1 -listen "${tmpdir}/s1.sock" { rxreq txresp -nolen -hdr "Content-Length: 10" - delay 1 + barrier b1 sync send "1234567890" } -start varnish v1 -vcl+backend { + import vtc; sub vcl_recv { if (req.url == "/synth") { return(synth(200, "OK")); @@ -18,11 +21,14 @@ varnish v1 -vcl+backend { } sub vcl_deliver { set resp.http.streaming = resp.is_streaming; + if (req.http.barrier) { + vtc.barrier_sync(req.http.barrier); + } } } -start client c1 { - txreq + txreq -hdr "barrier: ${b1_sock}" rxresp expect resp.http.streaming == "true" From dridi.boukelmoune at gmail.com Sun Jun 27 11:46:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Sun, 27 Jun 2021 11:46:06 +0000 (UTC) Subject: [master] 121356d90 vtc: Stabilize and reorganize b15 Message-ID: <20210627114606.247E0A4DB0@lists.varnish-cache.org> commit 121356d9060c9efdfe89d840fd84f710e83a164e Author: Dridi Boukelmoune Date: Sun Jun 27 13:34:18 2021 +0200 vtc: Stabilize and reorganize b15 While in itself this is not a slow test, on a loaded system the connection failures can degrade the run time dramatically, like 20 times slower. To save some time here and there, some client sessions were folded together where it made sense. Also no two clients share the same name to make it easier to spot the offending one in a test log. More importantly, the holddown test at the end which concerns the very first VCL is now put at the beginning where it belongs, and also where tweaking backend_remote_error_holddown will actually have the desired effect. diff --git a/bin/varnishtest/tests/b00015.vtc b/bin/varnishtest/tests/b00015.vtc index 396da7899..4e7ff3f3e 100644 --- a/bin/varnishtest/tests/b00015.vtc +++ b/bin/varnishtest/tests/b00015.vtc @@ -2,6 +2,8 @@ varnishtest "Check synthetic error page caching" # First test that an internally generated error is not cached +varnish v1 -cliok "param.set backend_remote_error_holddown 60" + varnish v1 -vcl { backend foo { .host = "${bad_backend}"; @@ -13,18 +15,15 @@ client c1 { rxresp expect resp.status == 503 expect resp.http.X-varnish == "1001" -} -run - -delay .1 -client c1 { txreq -url "/" rxresp expect resp.status == 503 - expect resp.http.X-varnish == "1004" + expect resp.http.X-varnish == "1003" } -run -delay .1 +varnish v1 -expect VBE.vcl1.foo.fail_econnrefused > 0 +varnish v1 -expect VBE.vcl1.foo.helddown > 0 # Then check that a cacheable error from the backend is @@ -42,23 +41,19 @@ varnish v1 -vcl+backend { } } -client c1 { +client c2 { txreq -url "/" rxresp expect resp.status == 301 - expect resp.http.X-varnish == "1007" -} -run + expect resp.http.X-varnish == "1006" -delay .1 - -client c1 { txreq -url "/" rxresp expect resp.status == 301 - expect resp.http.X-varnish == "1010 1008" + expect resp.http.X-varnish == "1008 1007" } -run -delay .1 +server s1 -wait # Then check that a non-cacheable error from the backend can be @@ -67,8 +62,6 @@ server s1 { txresp -status 502 } -start -varnish v1 -cliok "param.set backend_remote_error_holddown 10" - varnish v1 -vcl+backend { sub vcl_backend_response { if (beresp.status == 502) { @@ -77,23 +70,14 @@ varnish v1 -vcl+backend { } } -client c1 { +client c3 { txreq -url "/2" rxresp expect resp.status == 502 - expect resp.http.X-varnish == "1012" -} -run + expect resp.http.X-varnish == "1010" -delay .1 - -client c1 { txreq -url "/2" rxresp expect resp.status == 502 - expect resp.http.X-varnish == "1015 1013" + expect resp.http.X-varnish == "1012 1011" } -run - -delay .1 - -varnish v1 -expect VBE.vcl1.foo.fail_econnrefused > 0 -varnish v1 -expect VBE.vcl1.foo.helddown > 0 From dridi.boukelmoune at gmail.com Sun Jun 27 11:46:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Sun, 27 Jun 2021 11:46:06 +0000 (UTC) Subject: [master] 3dcb2a501 vtc: Stabilize s6 Message-ID: <20210627114606.09C52A4DAA@lists.varnish-cache.org> commit 3dcb2a50117588dd753d56ae2fd3e1803cc79783 Author: Dridi Boukelmoune Date: Sun Jun 27 13:31:38 2021 +0200 vtc: Stabilize s6 Despite 1s of delay we sometimes observe a zero Age header, which is consistent with being under the max-age of 2s and should not fail. diff --git a/bin/varnishtest/tests/s00006.vtc b/bin/varnishtest/tests/s00006.vtc index 9e8ba30db..53bb10e99 100644 --- a/bin/varnishtest/tests/s00006.vtc +++ b/bin/varnishtest/tests/s00006.vtc @@ -21,10 +21,10 @@ client c1 { expect resp.status == 200 expect resp.http.Age == 0 - delay 0.2 + delay 0.3 txreq -url "/" rxresp expect resp.status == 200 - expect resp.http.Age == 1 + expect resp.http.Age <= 1 } -run From dridi.boukelmoune at gmail.com Mon Jun 28 09:59:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 28 Jun 2021 09:59:06 +0000 (UTC) Subject: [master] e0c5201f9 vtc: Align c93 with c69 Message-ID: <20210628095906.3CC92A296A@lists.varnish-cache.org> commit e0c5201f947ab27619098ab2d14dc5e4e0c57020 Author: Dridi Boukelmoune Date: Mon Jun 28 11:56:33 2021 +0200 vtc: Align c93 with c69 diff --git a/bin/varnishtest/tests/c00069.vtc b/bin/varnishtest/tests/c00069.vtc index 3f91696a6..9aefaf4ea 100644 --- a/bin/varnishtest/tests/c00069.vtc +++ b/bin/varnishtest/tests/c00069.vtc @@ -13,7 +13,6 @@ server s1 { varnish v1 -vcl+backend { import vtc; - sub vcl_recv { if (req.url == "/synth") { return(synth(200, "OK")); diff --git a/bin/varnishtest/tests/c00093.vtc b/bin/varnishtest/tests/c00093.vtc index 9b8899dd3..0fa9aa2bd 100644 --- a/bin/varnishtest/tests/c00093.vtc +++ b/bin/varnishtest/tests/c00093.vtc @@ -1,34 +1,34 @@ varnishtest "Test resp.is_streaming with a UDS backend" +# Same as c00069 without the synth case + barrier b1 sock 2 +barrier b2 sock 2 server s1 -listen "${tmpdir}/s1.sock" { rxreq txresp -nolen -hdr "Content-Length: 10" barrier b1 sync + barrier b2 sync send "1234567890" } -start varnish v1 -vcl+backend { import vtc; - sub vcl_recv { - if (req.url == "/synth") { - return(synth(200, "OK")); - } - } - sub vcl_synth { - set resp.http.streaming = resp.is_streaming; + sub vcl_backend_response { + vtc.barrier_sync("${b1_sock}"); + return (deliver); } sub vcl_deliver { set resp.http.streaming = resp.is_streaming; - if (req.http.barrier) { - vtc.barrier_sync(req.http.barrier); + if (obj.hits == 0) { + vtc.barrier_sync("${b2_sock}"); } } } -start client c1 { - txreq -hdr "barrier: ${b1_sock}" + txreq rxresp expect resp.http.streaming == "true" @@ -37,9 +37,4 @@ client c1 { txreq rxresp expect resp.http.streaming == "false" - - txreq -url /synth - rxresp - expect resp.http.streaming == "false" } -run - From dridi.boukelmoune at gmail.com Mon Jun 28 10:03:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 28 Jun 2021 10:03:05 +0000 (UTC) Subject: [master] e830d6dd3 varnishtest: Barrier polish Message-ID: <20210628100305.43341A2D5B@lists.varnish-cache.org> commit e830d6dd386871f85e3126933bbac89bf2d8c44c Author: Dridi Boukelmoune Date: Mon Jun 28 12:01:37 2021 +0200 varnishtest: Barrier polish Make the type-specific fields stand out without the help of comments. diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index 98df1fc8a..05b3ef9e9 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -61,10 +61,12 @@ struct barrier { int waiters; int expected; int cyclic; - int cycle; /* BARRIER_COND only */ enum barrier_e type; - pthread_t thread; /* BARRIER_SOCK only */ + union { + int cond_cycle; + pthread_t sock_thread; + }; }; static VTAILQ_HEAD(, barrier) barriers = VTAILQ_HEAD_INITIALIZER(barriers); @@ -254,7 +256,7 @@ barrier_sock(struct barrier *b, const char *av, struct vtclog *vl) /* NB. We can use the BARRIER_COND's pthread_cond_t to wait until the * socket is ready for convenience. */ - AZ(pthread_create(&b->thread, NULL, barrier_sock_thread, b)); + AZ(pthread_create(&b->sock_thread, NULL, barrier_sock_thread, b)); AZ(pthread_cond_wait(&b->cond, &b->mtx)); AZ(pthread_mutex_unlock(&b->mtx)); } @@ -307,7 +309,7 @@ barrier_cond_sync(struct barrier *b, struct vtclog *vl) else b->waiters = ++w; - c = b->cycle; + c = b->cond_cycle; AZ(pthread_mutex_unlock(&b->mtx)); if (w < 0) @@ -318,7 +320,7 @@ barrier_cond_sync(struct barrier *b, struct vtclog *vl) AZ(pthread_mutex_lock(&b->mtx)); if (w == b->expected) { vtc_log(vl, 4, "Barrier(%s) wake %u", b->name, b->expected); - b->cycle++; + b->cond_cycle++; if (b->cyclic) b->waiters = 0; AZ(pthread_cond_broadcast(&b->cond)); @@ -330,7 +332,7 @@ barrier_cond_sync(struct barrier *b, struct vtclog *vl) r = pthread_cond_timedwait(&b->cond, &b->mtx, &ts); assert(r == 0 || r == ETIMEDOUT); } while (!vtc_stop && !vtc_error && r == ETIMEDOUT && - c == b->cycle); + c == b->cond_cycle); } AZ(pthread_mutex_unlock(&b->mtx)); } @@ -455,7 +457,7 @@ cmd_barrier(CMD_ARGS) case BARRIER_COND: break; case BARRIER_SOCK: - AZ(pthread_join(b->thread, NULL)); + AZ(pthread_join(b->sock_thread, NULL)); break; default: WRONG("Wrong barrier type"); From nils.goroll at uplex.de Tue Jun 29 15:06:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 29 Jun 2021 15:06:08 +0000 (UTC) Subject: [master] 50fb96898 VTCP_listen() may actually return file descriptor 0 Message-ID: <20210629150608.62EF7ACD25@lists.varnish-cache.org> commit 50fb968980d66e40385e3828248ea663af7c55e6 Author: Nils Goroll Date: Tue Jun 29 17:01:43 2021 +0200 VTCP_listen() may actually return file descriptor 0 There are more assertions on sockets not use file descriptor 0, but not being aware of any examples of those being actually hit, I prefer a minimal fix. Fixes #3636 diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c index d5afe460c..85521b8c9 100644 --- a/lib/libvarnish/vtcp.c +++ b/lib/libvarnish/vtcp.c @@ -518,7 +518,7 @@ vtcp_lo_cb(void *priv, const struct suckaddr *sa) struct helper *hp = priv; sock = VTCP_listen(sa, hp->depth, hp->errp); - if (sock > 0) { + if (sock >= 0) { *hp->errp = NULL; return (sock); } From dridi.boukelmoune at gmail.com Tue Jun 29 15:45:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 29 Jun 2021 15:45:06 +0000 (UTC) Subject: [master] 1c3570502 doc: The default storage is not (necessarily) malloc Message-ID: <20210629154506.EC579AE03D@lists.varnish-cache.org> commit 1c3570502f1f7325ffce5edb5fa75d068e5a58f3 Author: Dridi Boukelmoune Date: Tue Jun 29 17:41:50 2021 +0200 doc: The default storage is not (necessarily) malloc diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 5be41cf23..5fad3096f 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -330,20 +330,20 @@ hit-for-pass. If no ``-s`` options are given, the default is:: - -s malloc,100m + -s default,100m If no ``Transient`` storage is defined, the default is an unbound -``malloc`` storage as if defined as:: +``default`` storage as if defined as:: - -s Transient=malloc + -s Transient=default The following storage types and options are available: -s - The default storage type resolves to umem where available and malloc - otherwise. + The default storage type resolves to ``umem`` where available and + ``malloc`` otherwise. -s From dridi.boukelmoune at gmail.com Wed Jun 30 12:35:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 30 Jun 2021 12:35:06 +0000 (UTC) Subject: [master] d8f950447 vcc: Remove superfluous null checks Message-ID: <20210630123506.C476EAA627@lists.varnish-cache.org> commit d8f950447938b0c46083a3f69f1c1d605043e7f6 Author: Dridi Boukelmoune Date: Wed Jun 30 11:15:48 2021 +0200 vcc: Remove superfluous null checks diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index b41334b79..6c7c182d7 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -294,10 +294,10 @@ VCC_SymbolGet(struct vcc *tl, vcc_ns_t ns, vcc_kind_t kind, tn2 = tn; } tn1 = vcc_PeekTokenFrom(tl, tn); - if (tn1 == NULL || tn1->tok != '.') + if (tn1->tok != '.') break; tn1 = vcc_PeekTokenFrom(tl, tn1); - if (tn1 == NULL || tn1->tok != ID) + if (tn1->tok != ID) break; tn = tn1; } From dridi.boukelmoune at gmail.com Wed Jun 30 12:35:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 30 Jun 2021 12:35:06 +0000 (UTC) Subject: [master] a75371a7a vcc: Missing CBLOB Message-ID: <20210630123506.D944AAA62A@lists.varnish-cache.org> commit a75371a7aa3965862621c841e5a24f107a2fe276 Author: Dridi Boukelmoune Date: Wed Jun 30 14:30:05 2021 +0200 vcc: Missing CBLOB diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 329843bca..d40417baa 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -714,6 +714,7 @@ vcc_number(struct vcc *tl, struct expr **e, vcc_type_t fmt, const char *sign) * CNUM * FNUM * CSTR + * CBLOB */ static void From dridi.boukelmoune at gmail.com Wed Jun 30 12:51:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 30 Jun 2021 12:51:05 +0000 (UTC) Subject: [master] d083cb095 v1l: Don't mention timeouts on failed writes Message-ID: <20210630125105.A805FAAFBA@lists.varnish-cache.org> commit d083cb095f8a676f331e2544815fc7ac39c7b75c Author: Dridi Boukelmoune Date: Wed Jun 30 14:46:33 2021 +0200 v1l: Don't mention timeouts on failed writes This has confused at least one user troubleshooting this area. diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c index c275c92ef..52e1c88b0 100644 --- a/bin/varnishd/http1/cache_http1_line.c +++ b/bin/varnishd/http1/cache_http1_line.c @@ -222,9 +222,12 @@ V1L_Flush(const struct worker *wrk) * prevent slowloris attacks */ - VSLb(v1l->vsl, SLT_Debug, - "Hit idle send timeout, wrote = %zd/%zd; retrying", - i, v1l->liov); + if (errno == EWOULDBLOCK) { + VSLb(v1l->vsl, SLT_Debug, + "Hit idle send timeout, " + "wrote = %zd/%zd; retrying", + i, v1l->liov); + } if (i > 0) v1l_prune(v1l, i); From dridi.boukelmoune at gmail.com Wed Jun 30 14:58:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 30 Jun 2021 14:58:06 +0000 (UTC) Subject: [master] 801d91066 circleci: Stick alpine to 3.13 for now Message-ID: <20210630145806.2C845AE8EA@lists.varnish-cache.org> commit 801d91066960268f042cf74060780afb9390603b Author: Dridi Boukelmoune Date: Wed Jun 30 16:54:34 2021 +0200 circleci: Stick alpine to 3.13 for now Until we figure out how to work around the build failure since "latest" bumped to 3.14. diff --git a/.circleci/config.yml b/.circleci/config.yml index 61342131f..4b5ddc711 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -323,7 +323,8 @@ workflows: - distcheck: name: distcheck_alpine dist: alpine - release: "latest" + release: "3.13" + #release: "latest" extra_conf: --disable-developer-warnings --disable-dependency-tracking - distcheck: name: distcheck_archlinux