From guillaume at varnish-software.com Thu Apr 1 01:08:09 2021 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Thu, 1 Apr 2021 01:08:09 +0000 (UTC) Subject: [master] b46a77b76 introduce bob Message-ID: <20210401010809.20290AD29B@lists.varnish-cache.org> commit b46a77b7624e28c4837d716432c9b66190a28add Author: Guillaume Quintard Date: Wed Mar 31 17:50:32 2021 -0700 introduce bob diff --git a/.circleci/Dockerfile b/.circleci/Dockerfile new file mode 100644 index 000000000..7553611e0 --- /dev/null +++ b/.circleci/Dockerfile @@ -0,0 +1,15 @@ +FROM centos:7 + +RUN set -e;\ + yum install -y epel-release; \ + yum install -y \ + automake \ + git \ + jemalloc-devel \ + libedit-devel \ + libtool \ + libunwind-devel \ + make \ + pcre-devel \ + python3 \ + python-sphinx diff --git a/.circleci/Dockerfile.alpine b/.circleci/Dockerfile.alpine new file mode 100644 index 000000000..c09aa9d5a --- /dev/null +++ b/.circleci/Dockerfile.alpine @@ -0,0 +1,20 @@ +FROM alpine + +RUN set -e; \ + apk update; \ + apk add -q \ + autoconf \ + automake \ + build-base \ + ca-certificates \ + cpio \ + git \ + gzip \ + libedit-dev \ + libtool \ + libunwind-dev \ + linux-headers \ + pcre-dev \ + py-docutils \ + py3-sphinx \ + tar diff --git a/.circleci/Dockerfile.archlinux b/.circleci/Dockerfile.archlinux new file mode 100644 index 000000000..1e6344354 --- /dev/null +++ b/.circleci/Dockerfile.archlinux @@ -0,0 +1,16 @@ +FROM archlinux + +RUN set -e; \ + pacman -Sy --noconfirm \ + base-devel \ + ca-certificates \ + cpio \ + git \ + libedit \ + libtool \ + libunwind \ + linux-headers \ + pcre \ + python-docutils \ + python-sphinx \ + tar diff --git a/.circleci/Dockerfile.ubuntu b/.circleci/Dockerfile.ubuntu new file mode 100644 index 000000000..cc50bd3a0 --- /dev/null +++ b/.circleci/Dockerfile.ubuntu @@ -0,0 +1,22 @@ +FROM ubuntu + +RUN set -e; \ + export DEBIAN_FRONTEND=noninteractive; \ + export DEBCONF_NONINTERACTIVE_SEEN=true; \ + apt-get update; \ + apt-get install -y \ + autoconf \ + automake \ + build-essential \ + ca-certificates \ + cpio \ + git \ + graphviz \ + libedit-dev \ + libjemalloc-dev \ + libncurses-dev \ + libpcre3-dev \ + libtool \ + libunwind-dev \ + pkg-config \ + python3-sphinx From guillaume at varnish-software.com Fri Apr 2 15:13:05 2021 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Fri, 2 Apr 2021 15:13:05 +0000 (UTC) Subject: [master] 05780b634 disable -Wcast-qual because of musl Message-ID: <20210402151305.62CC7A0DA1@lists.varnish-cache.org> commit 05780b634c40233dd2494a2c76a8dd8ba72d79f8 Author: Guillaume Quintard Date: Fri Apr 2 08:11:27 2021 -0700 disable -Wcast-qual because of musl diff --git a/wflags.py b/wflags.py index f5dedf7e6..3c5e6d093 100644 --- a/wflags.py +++ b/wflags.py @@ -44,7 +44,6 @@ DESIRABLE_OPTIONS = [ DESIRABLE_WFLAGS = [ "-Wcast-align", - "-Wcast-qual", "-Wchar-subscripts", "-Wempty-body", "-Wextra", @@ -69,6 +68,7 @@ DESIRABLE_WFLAGS = [ ] UNDESIRABLE_WFLAGS = [ + "-Wno-cast-qual", # GCC complains about musl::sched.h "-Wno-thread-safety", # Does not understand our mutexs are wrapped "-Wno-old-style-definition", # Does not like vgz "-Wno-sign-compare", # Fixable From phk at FreeBSD.org Tue Apr 6 14:02:04 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 6 Apr 2021 14:02:04 +0000 (UTC) Subject: [master] 78ccf7bd9 Decrease sleep time when waiting for child to die. Message-ID: <20210406140204.F105DA5268@lists.varnish-cache.org> commit 78ccf7bd9c62ef434bc0b11011ae6f63ed2ccf62 Author: Simon Date: Tue Apr 6 13:13:28 2021 +0000 Decrease sleep time when waiting for child to die. diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index e55e6ca5d..84f3c182e 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -519,11 +519,11 @@ mgt_reap_child(void) XXXAN(vsb); /* Wait for child to die */ - for (i = 0; i < mgt_param.cli_timeout; i++) { + for (i = 0; i < mgt_param.cli_timeout * 10; i++) { r = waitpid(child_pid, &status, WNOHANG); if (r == child_pid) break; - (void)sleep(1); + (void)usleep(100000); } if (r == 0) { VSB_printf(vsb, "Child (%jd) not dying, killing", (intmax_t)r); From dridi.boukelmoune at gmail.com Tue Apr 6 17:26:04 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 6 Apr 2021 17:26:04 +0000 (UTC) Subject: [master] 606977bbf doc: Fix the cookie.keep_re() example Message-ID: <20210406172604.B3741ABF5B@lists.varnish-cache.org> commit 606977bbfb624ead38e9c8648beac0b3906a4294 Author: Dridi Boukelmoune Date: Tue Apr 6 19:23:28 2021 +0200 doc: Fix the cookie.keep_re() example Reported by Justin Lloyd. diff --git a/vmod/vmod_cookie.vcc b/vmod/vmod_cookie.vcc index ca13d0c0e..85e6b8519 100644 --- a/vmod/vmod_cookie.vcc +++ b/vmod/vmod_cookie.vcc @@ -124,7 +124,7 @@ Example:: sub vcl_recv { cookie.parse("cookie1: value1; cookie2: value2; cookie3: value3"); - cookie.keep_re("^cookie1,cookie2"); + cookie.keep_re("^cookie[12]$"); # get_string() will now yield # "cookie1: value1; cookie2: value2;"; } From nils.goroll at uplex.de Sat Apr 10 12:34:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 10 Apr 2021 12:34:06 +0000 (UTC) Subject: [master] cf9548efe vcc: Track the built-in subs which static calls originate from Message-ID: <20210410123406.88F0DAA1BB@lists.varnish-cache.org> commit cf9548efe2770e1597c49903f065572aaad96fd6 Author: Nils Goroll Date: Sat Apr 10 14:18:22 2021 +0200 vcc: Track the built-in subs which static calls originate from We use another "method" bitmask to record the built-in subs from which static calls originate. This allows us to indentify those subs which are called from housekeeping only, allowing us to selectively disable compiler optimizations. VGC diff for the example from https://github.com/varnishcache/varnish-cache/issues/3545#issue-824634168 ```diff @@ -2242,7 +2278,7 @@ #define END_ if (*ctx->handling) return -void v_matchproto_(vcl_func_f) +void v_dont_optimize v_matchproto_(vcl_func_f) VGC_function_a(VRT_CTX, enum vcl_func_call_e call, enum vcl_func_fail_e *failp) { ``` Fixes #3545 diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 906567977..88c8baf5b 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -191,7 +191,9 @@ vcc_EmitProc(struct vcc *tl, struct proc *p) maskcmp = "&"; } - if ((mask & VCL_MET_TASK_H) && (mask & ~VCL_MET_TASK_H) == 0) + if (dyn == 0 && + (p->calledfrom & VCL_MET_TASK_H) != 0 && + (p->calledfrom & ~VCL_MET_TASK_H) == 0) cc_adv = "v_dont_optimize "; else cc_adv = ""; @@ -210,6 +212,7 @@ vcc_EmitProc(struct vcc *tl, struct proc *p) Fh(tl, 1, "\t.n\t\t= %d,\n", nsub); Fh(tl, 1, "\t.nref\t\t= %d,\n", p->sym->nref); Fh(tl, 1, "\t.called\t\t= %d\n", p->called); + Fh(tl, 1, "\t// calledfrom\t 0x%x\n", p->calledfrom); Fh(tl, 1, "}};\n"); if (dyn) { diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index b52c53216..1ec423b30 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -213,6 +213,7 @@ struct proc { unsigned called; unsigned active; unsigned okmask; + unsigned calledfrom; struct token *return_tok[VCL_RET_MAX]; struct vsb *cname; struct vsb *prologue; diff --git a/lib/libvcc/vcc_xref.c b/lib/libvcc/vcc_xref.c index e3351ec48..433f5e7b8 100644 --- a/lib/libvcc/vcc_xref.c +++ b/lib/libvcc/vcc_xref.c @@ -207,6 +207,7 @@ vcc_CheckActionRecurse(struct vcc *tl, struct proc *p, unsigned bitmap) vcc_ErrWhere(tl, pc->t); return (1); } + pc->sym->proc->calledfrom |= p->calledfrom; pc->sym->proc->called++; pc->sym->nref += u; if (vcc_CheckActionRecurse(tl, pc->sym->proc, bitmap)) { @@ -233,10 +234,12 @@ vcc_checkaction(struct vcc *tl, const struct symbol *sym) AN(p); AN(p->name); - if (p->method == NULL) + if (p->method == NULL) { bitmap = ~0U; - else + } else { bitmap = p->method->ret_bitmap; + p->calledfrom = p->method->bitval; + } if (! vcc_CheckActionRecurse(tl, p, bitmap)) return; From nils.goroll at uplex.de Sat Apr 10 15:18:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 10 Apr 2021 15:18:06 +0000 (UTC) Subject: [master] 7c3db3fe0 varnishadm: use the -t argument as the timeout Message-ID: <20210410151806.ABEF4AE87C@lists.varnish-cache.org> commit 7c3db3fe0fb594dcb22db12a8e16dbbf397e52a6 Author: Nils Goroll Date: Sat Apr 10 17:09:21 2021 +0200 varnishadm: use the -t argument as the timeout We used the -t argument only for the VSM attach, not for the actual CLI operations. I do not think the complication of differentiated timeouts is justified, so just use the one timeout parameter we have for both. Also I think that the documentation is already adequate in its simplicity: -t timeout Wait no longer than this many seconds for an operation to finish. Fixes #3542 diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c index 08c4a4125..ebc1aa2cc 100644 --- a/bin/varnishadm/varnishadm.c +++ b/bin/varnishadm/varnishadm.c @@ -74,7 +74,7 @@ } while (0) -static const double timeout = 5; // XXX should be settable by arg ? +static double timeout = 5; static int p_arg = 0; static void @@ -423,6 +423,21 @@ n_arg_sock(const char *n_arg, const char *t_arg) return (sock); } +static int +t_arg_timeout(const char *t_arg) +{ + char *p = NULL; + + AN(t_arg); + timeout = strtod(t_arg, &p); + if ((p != NULL && *p != '\0') || + !isfinite(timeout) || timeout < 0) { + fprintf(stderr, "-t: Invalid argument: %s", t_arg); + return (-1); + } + return (1); +} + #define OPTARG "hn:pS:T:t:" int @@ -486,6 +501,9 @@ main(int argc, char * const *argv) if (sock < 0) exit(2); + if (t_arg != NULL && t_arg_timeout(t_arg) < 0) + exit(2); + if (argc > 0) { VSIG_Arm_int(); VSIG_Arm_term(); From phk at FreeBSD.org Sun Apr 11 09:38:09 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 11 Apr 2021 09:38:09 +0000 (UTC) Subject: [master] cb36d6aac Add -Wno-nullability-completeness which barfs all over MacOsX Message-ID: <20210411093809.5AA4AA5AE5@lists.varnish-cache.org> commit cb36d6aac1487eac4610456354b2ebf2ac0100aa Author: Poul-Henning Kamp Date: Sun Apr 11 09:36:40 2021 +0000 Add -Wno-nullability-completeness which barfs all over MacOsX diff --git a/wflags.py b/wflags.py index 3c5e6d093..7e4d30fed 100644 --- a/wflags.py +++ b/wflags.py @@ -78,6 +78,7 @@ UNDESIRABLE_WFLAGS = [ "-Wno-redundant-decls", # Complains about centos::stdio.h "-Wno-missing-variable-declarations", # Complains about optreset "-Wno-parentheses", # GCC complains about musl::endian.h + "-Wno-nullability-completeness", # Barfs all over MacOSx ] def main(): From phk at FreeBSD.org Sun Apr 11 10:16:05 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 11 Apr 2021 10:16:05 +0000 (UTC) Subject: [master] a3a5d2f62 Cast %ju argument to uintmax_t Message-ID: <20210411101605.0D35EA6D06@lists.varnish-cache.org> commit a3a5d2f62d08dad4e56497e40c2d7c9fbe087e07 Author: Poul-Henning Kamp Date: Sun Apr 11 10:11:44 2021 +0000 Cast %ju argument to uintmax_t Closes: #3570 diff --git a/vmod/vmod_debug_acl.c b/vmod/vmod_debug_acl.c index bff2a0ce2..7de562743 100644 --- a/vmod/vmod_debug_acl.c +++ b/vmod/vmod_debug_acl.c @@ -245,7 +245,8 @@ xyzzy_time_acl(VRT_CTX, VCL_ACL acl, VCL_IP ip0, VCL_IP ip1, } t1 = VTIM_mono(); VSLb(ctx->vsl, SLT_Debug, - "Timed ACL: %.9f -> %.9f = %.9f %.9f/round, %.9f/IP %jd IPs", - t0, t1, t1 - t0, (t1-t0) / turnus, (t1-t0) / asw->count, asw->count); + "Timed ACL: %.9f -> %.9f = %.9f %.9f/round, %.9f/IP %ju IPs", + t0, t1, t1 - t0, (t1-t0) / turnus, (t1-t0) / asw->count, + (uintmax_t)asw->count); return ((t1 - t0) / asw->count); } From dridi.boukelmoune at gmail.com Mon Apr 12 07:22:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 12 Apr 2021 07:22:06 +0000 (UTC) Subject: [master] c9b469c5b vcc: Simplify v_dont_optimize condition Message-ID: <20210412072206.CBD9FA2B5C@lists.varnish-cache.org> commit c9b469c5b5f9425ce2927855a0f6710167f62c3d Author: Dridi Boukelmoune Date: Mon Apr 12 09:16:46 2021 +0200 vcc: Simplify v_dont_optimize condition diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 88c8baf5b..8991e9a30 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -191,9 +191,7 @@ vcc_EmitProc(struct vcc *tl, struct proc *p) maskcmp = "&"; } - if (dyn == 0 && - (p->calledfrom & VCL_MET_TASK_H) != 0 && - (p->calledfrom & ~VCL_MET_TASK_H) == 0) + if (dyn == 0 && (p->calledfrom & VCL_MET_TASK_H) == p->calledfrom) cc_adv = "v_dont_optimize "; else cc_adv = ""; From phk at FreeBSD.org Mon Apr 12 07:48:04 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Apr 2021 07:48:04 +0000 (UTC) Subject: [master] 108c65d37 Dont attempt to re-release the VCL Message-ID: <20210412074804.9E294A3927@lists.varnish-cache.org> commit 108c65d37d1788b3e0f6ccc3e8837a7410ca39bb Author: Poul-Henning Kamp Date: Mon Apr 12 07:42:53 2021 +0000 Dont attempt to re-release the VCL Fixes #3571 diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 090ada6c3..9b5d59c82 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -433,7 +433,7 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk) else tmo = wrk->lastused+60.; i = Lck_CondWait(&wrk->cond, &pp->mtx, tmo); - if (i == ETIMEDOUT) + if (i == ETIMEDOUT && wrk->vcl != NULL) VCL_Rel(&wrk->vcl); if (wrk->task->func != NULL) { /* We have been handed a new task */ From dridi at varni.sh Mon Apr 12 08:15:27 2021 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 12 Apr 2021 08:15:27 +0000 Subject: [master] 7c3db3fe0 varnishadm: use the -t argument as the timeout In-Reply-To: <20210410151806.ABEF4AE87C@lists.varnish-cache.org> References: <20210410151806.ABEF4AE87C@lists.varnish-cache.org> Message-ID: On Sat, Apr 10, 2021 at 3:18 PM Nils Goroll wrote: > > > commit 7c3db3fe0fb594dcb22db12a8e16dbbf397e52a6 > Author: Nils Goroll > Date: Sat Apr 10 17:09:21 2021 +0200 > > varnishadm: use the -t argument as the timeout > > We used the -t argument only for the VSM attach, not for the actual CLI > operations. > > I do not think the complication of differentiated timeouts is justified, Agreed. > so just use the one timeout parameter we have for both. Also I think > that the documentation is already adequate in its simplicity: > > -t timeout > Wait no longer than this many seconds for an operation to > finish. We should however make the default apparent. > Fixes #3542 > > diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c > index 08c4a4125..ebc1aa2cc 100644 > --- a/bin/varnishadm/varnishadm.c > +++ b/bin/varnishadm/varnishadm.c > @@ -74,7 +74,7 @@ > } while (0) > > > -static const double timeout = 5; // XXX should be settable by arg ? > +static double timeout = 5; Should we increase the default timeout to match varnishd?s default cli_timeout instead of libvarnishapi?s t VSM argument? > static int p_arg = 0; > > static void > @@ -423,6 +423,21 @@ n_arg_sock(const char *n_arg, const char *t_arg) > return (sock); > } > > +static int > +t_arg_timeout(const char *t_arg) > +{ > + char *p = NULL; > + > + AN(t_arg); > + timeout = strtod(t_arg, &p); > + if ((p != NULL && *p != '\0') || > + !isfinite(timeout) || timeout < 0) { > + fprintf(stderr, "-t: Invalid argument: %s", t_arg); > + return (-1); > + } > + return (1); > +} > + > #define OPTARG "hn:pS:T:t:" > > int > @@ -486,6 +501,9 @@ main(int argc, char * const *argv) > if (sock < 0) > exit(2); > > + if (t_arg != NULL && t_arg_timeout(t_arg) < 0) > + exit(2); > + > if (argc > 0) { > VSIG_Arm_int(); > VSIG_Arm_term(); > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From phk at FreeBSD.org Mon Apr 12 09:50:05 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Apr 2021 09:50:05 +0000 (UTC) Subject: [master] ff20fdbb0 Add a "struct acl" to hold info about ACL's while we compile them. Message-ID: <20210412095005.C57ECA76D0@lists.varnish-cache.org> commit ff20fdbb0e05d8d50c6697c3b568ea2d2de1efca Author: Poul-Henning Kamp Date: Mon Apr 12 09:06:42 2021 +0000 Add a "struct acl" to hold info about ACL's while we compile them. diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index 82cd31985..a89d0f022 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -41,10 +41,20 @@ #include "vcc_compile.h" #include +#include #include #define ACL_MAXADDR (sizeof(struct in6_addr) + 1) +VRBT_HEAD(acl_tree, acl_e); + +struct acl { + unsigned magic; +#define VCC_ACL_MAGIC 0xb9fb3cd0 + + struct acl_tree acl_tree; +}; + struct acl_e { unsigned magic; #define VCC_ACL_E_MAGIC 0xcac81e23 @@ -175,7 +185,7 @@ vcc_acl_insert_entry(struct vcc *tl, struct acl_e **aenp) struct acl_e *ae2; CHECK_OBJ_NOTNULL(*aenp, VCC_ACL_E_MAGIC); - ae2 = VRBT_INSERT(acl_tree, &tl->acl_tree, *aenp); + ae2 = VRBT_INSERT(acl_tree, &tl->acl->acl_tree, *aenp); if (ae2 != NULL) { if (ae2->not != (*aenp)->not) { VSB_cat(tl->sb, "Conflicting ACL entries:\n"); @@ -461,7 +471,7 @@ vcc_acl_emit_tokens(const struct vcc *tl, const struct acl_e *ae) */ static void -vcc_acl_emit(struct vcc *tl, const char *name, const char *rname) +vcc_acl_emit(struct vcc *tl, const struct symbol *sym) { struct acl_e *ae; int depth, l, m, i; @@ -472,7 +482,7 @@ vcc_acl_emit(struct vcc *tl, const char *name, const char *rname) func = VSB_new_auto(); AN(func); VSB_printf(func, "match_acl_"); - VCC_PrintCName(func, name, NULL); + VCC_PrintCName(func, sym->name, NULL); AZ(VSB_finish(func)); Fh(tl, 0, "\nstatic int v_matchproto_(acl_match_f)\n"); @@ -483,7 +493,7 @@ vcc_acl_emit(struct vcc *tl, const char *name, const char *rname) Fh(tl, 0, "\n"); Fh(tl, 0, "\tfam = VRT_VSA_GetPtr(ctx, p, &a);\n"); Fh(tl, 0, "\tif (fam < 0) {\n"); - Fh(tl, 0, "\t\tVPI_acl_log(ctx, \"NO_FAM %s\");\n", name); + Fh(tl, 0, "\t\tVPI_acl_log(ctx, \"NO_FAM %s\");\n", sym->name); Fh(tl, 0, "\t\treturn(0);\n"); Fh(tl, 0, "\t}\n\n"); if (!tl->err_unref) { @@ -493,7 +503,7 @@ vcc_acl_emit(struct vcc *tl, const char *name, const char *rname) } depth = -1; at[0] = 256; - VRBT_FOREACH(ae, acl_tree, &tl->acl_tree) { + VRBT_FOREACH(ae, acl_tree, &tl->acl->acl_tree) { /* Find how much common prefix we have */ for (l = 0; l <= depth && l * 8 < (int)ae->mask - 7; l++) { @@ -537,7 +547,7 @@ vcc_acl_emit(struct vcc *tl, const char *name, const char *rname) i = ((int)ae->mask + 7) / 8; Fh(tl, 0, "\t%*sVPI_acl_log(ctx, \"%sMATCH %s \" ", - -i, "", ae->not ? "NEG_" : "", name); + -i, "", ae->not ? "NEG_" : "", sym->name); vcc_acl_emit_tokens(tl, ae); Fh(tl, 0, ");\n"); @@ -549,18 +559,18 @@ vcc_acl_emit(struct vcc *tl, const char *name, const char *rname) Fh(tl, 0, "\t%*.*s}\n", depth, depth, ""); /* Deny by default */ - Fh(tl, 0, "\tVPI_acl_log(ctx, \"NO_MATCH %s\");\n", name); + Fh(tl, 0, "\tVPI_acl_log(ctx, \"NO_MATCH %s\");\n", sym->name); Fh(tl, 0, "\treturn (0);\n}\n"); /* Emit the struct that will be referenced */ - Fh(tl, 0, "\nstatic const struct vrt_acl %s[] = {{\n", rname); + Fh(tl, 0, "\nstatic const struct vrt_acl %s[] = {{\n", sym->rname); Fh(tl, 0, "\t.magic = VRT_ACL_MAGIC,\n"); Fh(tl, 0, "\t.match = &%s,\n", VSB_data(func)); - Fh(tl, 0, "\t.name = \"%s\",\n", name); + Fh(tl, 0, "\t.name = \"%s\",\n", sym->name); Fh(tl, 0, "}};\n\n"); if (!tl->err_unref) { AN(ifp); - VSB_printf(ifp->ini, "\t(void)%s;\n", rname); + VSB_printf(ifp->ini, "\t(void)%s;\n", sym->rname); } VSB_destroy(&func); } @@ -569,9 +579,12 @@ void vcc_ParseAcl(struct vcc *tl) { struct symbol *sym; + struct acl acl[1]; + INIT_OBJ(acl, VCC_ACL_MAGIC); + tl->acl = acl; vcc_NextToken(tl); - VRBT_INIT(&tl->acl_tree); + VRBT_INIT(&acl->acl_tree); vcc_ExpectVid(tl, "ACL"); ERRCHK(tl); @@ -588,5 +601,5 @@ vcc_ParseAcl(struct vcc *tl) } SkipToken(tl, '}'); - vcc_acl_emit(tl, sym->name, sym->rname); + vcc_acl_emit(tl, sym); } diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 1ec423b30..231a2cd15 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -39,7 +39,6 @@ #include "vrt.h" #include "vcl.h" #include "vqueue.h" -#include "vtree.h" #include "vsb.h" #include "vcc_token_defs.h" @@ -57,6 +56,7 @@ /*---------------------------------------------------------------------*/ +struct acl; struct vsb; struct token; struct sockaddr_storage; @@ -235,8 +235,6 @@ struct inifin { VTAILQ_HEAD(inifinhead, inifin); -VRBT_HEAD(acl_tree, acl_e); - struct vcc { unsigned magic; #define VCC_MAGIC 0x24ad719d @@ -275,7 +273,7 @@ struct vcc { struct proc *curproc; VTAILQ_HEAD(, proc) procs; - struct acl_tree acl_tree; + struct acl *acl; int nprobe; From phk at FreeBSD.org Mon Apr 12 09:50:05 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Apr 2021 09:50:05 +0000 (UTC) Subject: [master] 1e45d97c3 Introduce 'acl +log {...}' flag and disable VSL logging by default. Message-ID: <20210412095005.DDE8AA76D3@lists.varnish-cache.org> commit 1e45d97c32b80232d84f6b2dec9c5e275a94c3f8 Author: Poul-Henning Kamp Date: Mon Apr 12 09:48:09 2021 +0000 Introduce 'acl +log {...}' flag and disable VSL logging by default. diff --git a/bin/varnishtest/tests/c00005.vtc b/bin/varnishtest/tests/c00005.vtc index 448d6f80c..6e8feaaa8 100644 --- a/bin/varnishtest/tests/c00005.vtc +++ b/bin/varnishtest/tests/c00005.vtc @@ -72,7 +72,7 @@ varnish v1 -vcl { backend dummy None; - acl acl1 { + acl acl1 +log { # bad notation (confusing) "1.2.3.4"/24; "1.2.3.66"/26; diff --git a/bin/varnishtest/tests/c00087.vtc b/bin/varnishtest/tests/c00087.vtc index d84862860..832f63993 100644 --- a/bin/varnishtest/tests/c00087.vtc +++ b/bin/varnishtest/tests/c00087.vtc @@ -6,7 +6,7 @@ server s1 -listen "${tmpdir}/s1.sock" { } -start varnish v1 -syntax 4.1 -arg "-a foo=${tmpdir}/v1.sock" -vcl+backend { - acl acl1 { + acl acl1 +log { "${localhost}"; } diff --git a/bin/varnishtest/tests/v00017.vtc b/bin/varnishtest/tests/v00017.vtc index 6e35d0c23..17f94582d 100644 --- a/bin/varnishtest/tests/v00017.vtc +++ b/bin/varnishtest/tests/v00017.vtc @@ -100,3 +100,19 @@ varnish v1 -errvcl {/mask only allowed once} { } sub vcl_recv { if (client.ip ~ a) { return(pass); } } } + +varnish v1 -errvcl {Expected ACL flag after:} { + backend b { .host = "${localhost}"; } + acl a + foobar { + "10.0.1.0/22" / 22; + } + sub vcl_recv { if (client.ip ~ a) { return(pass); } } +} + +varnish v1 -errvcl {Unknown ACL flag:} { + backend b { .host = "${localhost}"; } + acl a +foobar { + "10.0.1.0/22" / 22; + } + sub vcl_recv { if (client.ip ~ a) { return(pass); } } +} diff --git a/doc/changes.rst b/doc/changes.rst index b63db87c8..1fe26459f 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -31,6 +31,13 @@ http://varnish-cache.org/docs/trunk/whats-new/index.html and via individual releases. These documents are updated as part of the release process. +================================ +Varnish Cache 7.x.x (2021-09-15) +================================ + +* ACLs no longer produce VSL `VCL_acl` records by default, this must be + explicitly enabled with `vcl +log { ... }`. + ================================ Varnish Cache 6.6.0 (2021-03-15) ================================ diff --git a/doc/sphinx/users-guide/vcl-syntax.rst b/doc/sphinx/users-guide/vcl-syntax.rst index 2cc6fc6a5..7ef4dd7e7 100644 --- a/doc/sphinx/users-guide/vcl-syntax.rst +++ b/doc/sphinx/users-guide/vcl-syntax.rst @@ -59,6 +59,16 @@ To match an IP address against an ACL, simply use the match operator:: return (pipe); } +In Varnish versions before 7.0, ACLs would always emit a `VCL_acl` +record in the VSL log, from 7.0 and forward, this must be explicitly +enabled by specifying the `+log` flag:: + + acl local +log { + "localhost"; // myself + "192.0.2.0"/24; // and everyone on the local network + ! "192.0.2.23"; // except for the dialin router + } + Operators ~~~~~~~~~ diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index c98646d26..94ed41409 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -287,7 +287,7 @@ SLTM(Fetch_Body, 0, "Body fetched from backend", ) SLTM(VCL_acl, 0, "VCL ACL check results", - "Logs VCL ACL evaluation results.\n\n" + "ACLs with the `+log` flag emits this record with the result.\n\n" "The format is::\n\n" "\t%s [%s [%s [fixed: %s]]]\n" "\t| | | |\n" diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index a89d0f022..08f8a135f 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -52,6 +52,8 @@ struct acl { unsigned magic; #define VCC_ACL_MAGIC 0xb9fb3cd0 + int flag_log; + struct acl_tree acl_tree; }; @@ -493,7 +495,8 @@ vcc_acl_emit(struct vcc *tl, const struct symbol *sym) Fh(tl, 0, "\n"); Fh(tl, 0, "\tfam = VRT_VSA_GetPtr(ctx, p, &a);\n"); Fh(tl, 0, "\tif (fam < 0) {\n"); - Fh(tl, 0, "\t\tVPI_acl_log(ctx, \"NO_FAM %s\");\n", sym->name); + if (tl->acl->flag_log) + Fh(tl, 0, "\t\tVPI_acl_log(ctx, \"NO_FAM %s\");\n", sym->name); Fh(tl, 0, "\t\treturn(0);\n"); Fh(tl, 0, "\t}\n\n"); if (!tl->err_unref) { @@ -546,10 +549,12 @@ vcc_acl_emit(struct vcc *tl, const struct symbol *sym) i = ((int)ae->mask + 7) / 8; - Fh(tl, 0, "\t%*sVPI_acl_log(ctx, \"%sMATCH %s \" ", - -i, "", ae->not ? "NEG_" : "", sym->name); - vcc_acl_emit_tokens(tl, ae); - Fh(tl, 0, ");\n"); + if (tl->acl->flag_log) { + Fh(tl, 0, "\t%*sVPI_acl_log(ctx, \"%sMATCH %s \" ", + -i, "", ae->not ? "NEG_" : "", sym->name); + vcc_acl_emit_tokens(tl, ae); + Fh(tl, 0, ");\n"); + } Fh(tl, 0, "\t%*sreturn (%d);\n", -i, "", ae->not ? 0 : 1); } @@ -559,7 +564,8 @@ vcc_acl_emit(struct vcc *tl, const struct symbol *sym) Fh(tl, 0, "\t%*.*s}\n", depth, depth, ""); /* Deny by default */ - Fh(tl, 0, "\tVPI_acl_log(ctx, \"NO_MATCH %s\");\n", sym->name); + if (tl->acl->flag_log) + Fh(tl, 0, "\tVPI_acl_log(ctx, \"NO_MATCH %s\");\n", sym->name); Fh(tl, 0, "\treturn (0);\n}\n"); /* Emit the struct that will be referenced */ @@ -579,6 +585,7 @@ void vcc_ParseAcl(struct vcc *tl) { struct symbol *sym; + struct token *sign; struct acl acl[1]; INIT_OBJ(acl, VCC_ACL_MAGIC); @@ -592,6 +599,24 @@ vcc_ParseAcl(struct vcc *tl) ERRCHK(tl); AN(sym); + while (tl->t->tok == '-' || tl->t->tok == '+') { + sign = tl->t; + vcc_NextToken(tl); + if (tl->t->b != sign->e) { + VSB_cat(tl->sb, "Expected ACL flag after:\n"); + vcc_ErrWhere(tl, sign); + return; + } + if (vcc_IdIs(tl->t, "log")) { + acl->flag_log = sign->tok == '+'; + vcc_NextToken(tl); + } else { + VSB_cat(tl->sb, "Unknown ACL flag:\n"); + vcc_ErrWhere(tl, tl->t); + return; + } + } + SkipToken(tl, '{'); while (tl->t->tok != '}') { From phk at FreeBSD.org Mon Apr 12 12:09:05 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Apr 2021 12:09:05 +0000 (UTC) Subject: [master] d89674d12 Collapse vcc_acl_add_entry() and vcc_acl_insert_entry() Message-ID: <20210412120905.4BD5BAB67B@lists.varnish-cache.org> commit d89674d129ef0ed1c495e1169f83ec3203db1b42 Author: Poul-Henning Kamp Date: Mon Apr 12 10:49:30 2021 +0000 Collapse vcc_acl_add_entry() and vcc_acl_insert_entry() diff --git a/lib/libvcc/flint.lnt b/lib/libvcc/flint.lnt index 22852628c..4e90fc671 100644 --- a/lib/libvcc/flint.lnt +++ b/lib/libvcc/flint.lnt @@ -4,6 +4,7 @@ -sem(vcc_new_source, custodial(1)) +-sem(acl_tree_VRBT_INSERT, custodial(2)) -emacro(835, EXPR_VAR) // Info 835: A zero has been given as right argument to operator '<<' -esym(755, VCL_40) // Not used (right now) diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index 08f8a135f..6f2c2c923 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -181,30 +181,11 @@ vcc_acl_chk(struct vcc *tl, const struct acl_e *ae, const int l, return (strdup(t)); } -static void -vcc_acl_insert_entry(struct vcc *tl, struct acl_e **aenp) -{ - struct acl_e *ae2; - - CHECK_OBJ_NOTNULL(*aenp, VCC_ACL_E_MAGIC); - ae2 = VRBT_INSERT(acl_tree, &tl->acl->acl_tree, *aenp); - if (ae2 != NULL) { - if (ae2->not != (*aenp)->not) { - VSB_cat(tl->sb, "Conflicting ACL entries:\n"); - vcc_ErrWhere(tl, ae2->t_addr); - VSB_cat(tl->sb, "vs:\n"); - vcc_ErrWhere(tl, (*aenp)->t_addr); - } - return; - } - *aenp = NULL; -} - static void vcc_acl_add_entry(struct vcc *tl, const struct acl_e *ae, int l, unsigned char *u, int fam) { - struct acl_e *aen; + struct acl_e *aen, *ae2; if (fam == PF_INET && ae->mask > 32) { VSB_printf(tl->sb, @@ -239,9 +220,14 @@ vcc_acl_add_entry(struct vcc *tl, const struct acl_e *ae, int l, assert(l + 1UL <= sizeof aen->data); memcpy(aen->data + 1L, u, l); - vcc_acl_insert_entry(tl, &aen); - if (aen != NULL) + ae2 = VRBT_INSERT(acl_tree, &tl->acl->acl_tree, aen); + if (ae2 != NULL && ae2->not != aen->not) { + VSB_cat(tl->sb, "Conflicting ACL entries:\n"); + vcc_ErrWhere(tl, ae2->t_addr); + VSB_cat(tl->sb, "vs:\n"); + vcc_ErrWhere(tl, aen->t_addr); vcl_acl_free(&aen); + } } static void @@ -599,7 +585,7 @@ vcc_ParseAcl(struct vcc *tl) ERRCHK(tl); AN(sym); - while (tl->t->tok == '-' || tl->t->tok == '+') { + while (tl->t->tok == '-' || tl->t->tok == '+') { sign = tl->t; vcc_NextToken(tl); if (tl->t->b != sign->e) { From phk at FreeBSD.org Mon Apr 12 12:09:05 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 12 Apr 2021 12:09:05 +0000 (UTC) Subject: [master] cc885760c Add "+table" flag to ACL's Message-ID: <20210412120905.6360AAB67E@lists.varnish-cache.org> commit cc885760c1b2fbd6467084a3437a2c5327f6407d Author: Poul-Henning Kamp Date: Mon Apr 12 11:58:00 2021 +0000 Add "+table" flag to ACL's The +table flag causes all overlapped ACL entries to be compiled as usual, and non-overlapped entries to be emitted as a table of bytes. When testing the ACL the compiled code is run before VPI_acl_table() is used to do a binary search on the table. Even with a binary search, the table is approx 3 times slower than the regular compiled ACLs (ie: only "blindingly fast" as oppposed to "lighting fast"). The advantage of +table is that C-compilers literally take no time, no matter the size of the ACL, where they will take seconds or even minutes compiling large ACLs as code. diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 5cdaf2b3f..76b427418 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -138,6 +138,74 @@ VRT_acl_match(VRT_CTX, VCL_ACL acl, VCL_IP ip) return (acl->match(ctx, ip)); } +static int +acl_tbl_cmp(int fam, const uint8_t *key, const uint8_t *b) +{ + int rv; + + rv = fam - (int)b[3]; + if (rv == 0 && b[1] > 0) + rv = memcmp(key, b + 4, b[1]); + if (rv == 0 && b[2]) + rv = (int)(key[b[1]] & b[2]) - (int)b[4 + b[1]]; + return (rv); +} + +static const uint8_t * +bbsearch(int fam, const uint8_t *key, const uint8_t *base0, + size_t nmemb, size_t size) +{ + const uint8_t *base = base0; + size_t lim; + int cmp; + const uint8_t *p; + + for (lim = nmemb; lim != 0; lim >>= 1) { + p = base + (lim >> 1) * size; + cmp = acl_tbl_cmp(fam, key, p); + if (cmp == 0) + return (p); + if (cmp > 0) { + /* key > p: move right */ + base = p + size; + lim--; + } /* else move left */ + } + return (NULL); +} + +int +VPI_acl_table(VRT_CTX, VCL_IP p, unsigned n, unsigned m, const uint8_t *tbl, + const char * const *str, const char *fin) +{ + int fam; + const uint8_t *key; + const uint8_t *ptr; + size_t sz; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(p); + AN(n); + assert(m == 20); + AN(tbl); + AN(fin); + + fam = VRT_VSA_GetPtr(ctx, p, &key); + ptr = bbsearch(fam, key, tbl, n, m); + + if (ptr != NULL) { + sz = ptr - tbl; + AZ(sz % m); + sz /= m; + if (str != NULL) + VPI_acl_log(ctx, str[sz]); + return *ptr; + } + if (str != NULL) + VPI_acl_log(ctx, fin); + return (0); +} + /*--------------------------------------------------------------------*/ VCL_VOID diff --git a/bin/varnishtest/tests/r01312.vtc b/bin/varnishtest/tests/r01312.vtc index 583871d84..ded2787e2 100644 --- a/bin/varnishtest/tests/r01312.vtc +++ b/bin/varnishtest/tests/r01312.vtc @@ -1,24 +1,6 @@ varnishtest "acl functional (& historic miscompile)" -server s1 { - rxreq - txresp -} -start - -varnish v1 -vcl+backend { - import std; - import debug; - - acl foo { - "127.0.0.2"; - "127.0.1"/19; - } - acl bar { - "127.0.1.2"; - "127.0.1"/19; - } - - acl block { +shell {echo ' # Tests all boundary conditions "192.168.8.0" / 21; ! "192.168.16" / 21; @@ -46,6 +28,29 @@ varnish v1 -vcl+backend { ! "::0100" / 124; "::0130" / 124; "::0170" / 124; +' > ${tmpdir}/_acl.vcl +} + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + import std; + import debug; + + acl foo { + "127.0.0.2"; + "127.0.1"/19; + } + acl bar { + "127.0.1.2"; + "127.0.1"/19; + } + + acl block { + include "${tmpdir}/_acl.vcl"; } sub vcl_deliver { @@ -100,3 +105,67 @@ client c1 { } -run logexpect l1 -wait + +varnish v1 -vcl+backend { + import std; + import debug; + + acl block +table { + include "${tmpdir}/_acl.vcl"; + } + + sub vcl_deliver { + set resp.http.acl4 = debug.sweep_acl( + block, + std.ip("192.168.0.0"), + std.ip("192.168.32.255"), + 256 + ); + set resp.http.acl6 = debug.sweep_acl( + block, + std.ip("::"), + std.ip("::0200"), + 16 + ); + } +} + +client c1 { + txreq + rxresp + expect resp.http.acl4 == ":4thASR0O18ZxnoKtc4zd8KuO25rPvwvMQyAvRfilz6o=:" + expect resp.http.acl6 == ":NSi+7wpvQe7XJj8DPbESjpYPGnIzvjOsA5QCyCnW3kc=:" +} -run + +varnish v1 -vcl+backend { + import std; + import debug; + + acl block +log +table { + include "${tmpdir}/_acl.vcl"; + } + + sub vcl_deliver { + set resp.http.acl4 = debug.sweep_acl( + block, + std.ip("192.168.0.0"), + std.ip("192.168.32.255"), + 256 + ); + set resp.http.acl6 = debug.sweep_acl( + block, + std.ip("::"), + std.ip("::0200"), + 16 + ); + } +} + +client c1 { + txreq + rxresp + expect resp.http.acl4 == ":4thASR0O18ZxnoKtc4zd8KuO25rPvwvMQyAvRfilz6o=:" + expect resp.http.acl6 == ":NSi+7wpvQe7XJj8DPbESjpYPGnIzvjOsA5QCyCnW3kc=:" +} -run + +shell {rm -f ${tmpdir}/_acl.vcl} diff --git a/include/vcc_interface.h b/include/vcc_interface.h index 7ad670ea3..8c81bddc0 100644 --- a/include/vcc_interface.h +++ b/include/vcc_interface.h @@ -61,6 +61,8 @@ int VPI_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len, void VPI_Vmod_Unload(VRT_CTX, struct vmod **hdl); typedef int acl_match_f(VRT_CTX, const VCL_IP); +int VPI_acl_table(VRT_CTX, VCL_IP, unsigned n, unsigned m, const uint8_t *tbl, + const char * const *str, const char *fin); struct vrt_acl { unsigned magic; diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index 6f2c2c923..226215fda 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -53,6 +53,7 @@ struct acl { #define VCC_ACL_MAGIC 0xb9fb3cd0 int flag_log; + int flag_table; struct acl_tree acl_tree; }; @@ -65,6 +66,7 @@ struct acl_e { unsigned mask; unsigned not; unsigned para; + unsigned overlapped; char *addr; const char *fixed; struct token *t_addr; @@ -121,10 +123,38 @@ vcl_acl_cmp(const struct acl_e *ae1, const struct acl_e *ae2) return (0); } +static int +vcl_acl_disjoint(const struct acl_e *ae1, const struct acl_e *ae2) +{ + const unsigned char *p1, *p2; + unsigned m; + + CHECK_OBJ_NOTNULL(ae1, VCC_ACL_E_MAGIC); + CHECK_OBJ_NOTNULL(ae2, VCC_ACL_E_MAGIC); + + p1 = ae1->data; + p2 = ae2->data; + m = ae1->mask; + if (ae2->mask < m) + m = ae2->mask; + for (; m >= 8; m -= 8) { + CMP(*p1, *p2); + p1++; + p2++; + } + if (m) { + m = 0xff00 >> m; + m &= 0xff; + CMP(*p1 & m, *p2 & m); + } + return (0); +} + VRBT_GENERATE_INSERT_COLOR(acl_tree, acl_e, branch, static) VRBT_GENERATE_INSERT(acl_tree, acl_e, branch, vcl_acl_cmp, static) VRBT_GENERATE_MINMAX(acl_tree, acl_e, branch, static) VRBT_GENERATE_NEXT(acl_tree, acl_e, branch, static) +VRBT_GENERATE_PREV(acl_tree, acl_e, branch, static) static char * vcc_acl_chk(struct vcc *tl, const struct acl_e *ae, const int l, @@ -454,6 +484,52 @@ vcc_acl_emit_tokens(const struct vcc *tl, const struct acl_e *ae) Fh(tl, 0, "\" fixed: %s\"", ae->fixed); } +/********************************************************************* + * Emit ACL on table format + */ + +static unsigned +vcc_acl_emit_tables(const struct vcc *tl, unsigned n, const char *name) +{ + struct acl_e *ae; + unsigned rv = sizeof(ae->data) + 3; + unsigned nn = 0; + size_t sz; + + Fh(tl, 0, "\nstatic unsigned char acl_tbl_%s[%u*%u] = {\n", + name, n, rv); + VRBT_FOREACH(ae, acl_tree, &tl->acl->acl_tree) { + if (ae->overlapped) + continue; + Fh(tl, 0, "\t0x%02x,", ae->not ? 0 : 1); + Fh(tl, 0, "0x%02x,", (ae->mask >> 3) - 1); + Fh(tl, 0, "0x%02x,", (0xff00 >> (ae->mask & 7)) & 0xff); + for (sz = 0; sz < sizeof(ae->data); sz++) + Fh(tl, 0, "0x%02x,", ae->data[sz]); + for (; sz < rv - 3; sz++) + Fh(tl, 0, "0,"); + Fh(tl, 0, "\n"); + nn++; + } + assert(n == nn); + Fh(tl, 0, "};\n"); + if (tl->acl->flag_log) { + Fh(tl, 0, "\nstatic const char *acl_str_%s[%d] = {\n", + name, n); + VRBT_FOREACH(ae, acl_tree, &tl->acl->acl_tree) { + if (ae->overlapped) + continue; + Fh(tl, 0, "\t"); + Fh(tl, 0, "\"%sMATCH %s \" ", + ae->not ? "NEG_" : "", name); + vcc_acl_emit_tokens(tl, ae); + Fh(tl, 0, ",\n"); + } + Fh(tl, 0, "};\n"); + } + return (rv); +} + /********************************************************************* * Emit a function to match the ACL we have collected */ @@ -461,11 +537,12 @@ vcc_acl_emit_tokens(const struct vcc *tl, const struct acl_e *ae) static void vcc_acl_emit(struct vcc *tl, const struct symbol *sym) { - struct acl_e *ae; + struct acl_e *ae, *ae2; int depth, l, m, i; unsigned at[ACL_MAXADDR]; struct inifin *ifp = NULL; struct vsb *func; + unsigned n, no, nw = 0; func = VSB_new_auto(); AN(func); @@ -473,6 +550,32 @@ vcc_acl_emit(struct vcc *tl, const struct symbol *sym) VCC_PrintCName(func, sym->name, NULL); AZ(VSB_finish(func)); + depth = -1; + at[0] = 256; + ae2 = NULL; + n = no = 0; + VRBT_FOREACH_REVERSE(ae, acl_tree, &tl->acl->acl_tree) { + n++; + if (ae2 == NULL) { + ae2 = ae; + } else if (vcl_acl_disjoint(ae, ae2)) { + ae2 = ae; + } else { + no++; + ae->overlapped = 1; + } + } + + Fh(tl, 0, "/* acl_n_%s n %u no %u */\n", sym->name, n, no); + if (n - no < (1<<1)) + no = n; + else if (!tl->acl->flag_table) + no = n; + + if (no < n) + nw = vcc_acl_emit_tables(tl, n - no, sym->name); + + Fh(tl, 0, "\nstatic int v_matchproto_(acl_match_f)\n"); Fh(tl, 0, "%s(VRT_CTX, const VCL_IP p)\n", VSB_data(func)); Fh(tl, 0, "{\n"); @@ -490,10 +593,12 @@ vcc_acl_emit(struct vcc *tl, const struct symbol *sym) VSB_printf(ifp->ini, "\tif (0) %s(0, 0);\n", VSB_data(func)); } - depth = -1; - at[0] = 256; + VRBT_FOREACH(ae, acl_tree, &tl->acl->acl_tree) { + if (no < n && !ae->overlapped) + continue; + /* Find how much common prefix we have */ for (l = 0; l <= depth && l * 8 < (int)ae->mask - 7; l++) { assert(l >= 0); @@ -549,10 +654,24 @@ vcc_acl_emit(struct vcc *tl, const struct symbol *sym) for (; 0 <= depth; depth--) Fh(tl, 0, "\t%*.*s}\n", depth, depth, ""); - /* Deny by default */ - if (tl->acl->flag_log) - Fh(tl, 0, "\tVPI_acl_log(ctx, \"NO_MATCH %s\");\n", sym->name); - Fh(tl, 0, "\treturn (0);\n}\n"); + if (no < n) { + Fh(tl, 0, "\treturn(\n\t VPI_acl_table(ctx,\n"); + Fh(tl, 0, "\t\tp,\n"); + Fh(tl, 0, "\t\t%u, %u,\n", n - no, nw); + Fh(tl, 0, "\t\tacl_tbl_%s,\n", sym->name); + if (tl->acl->flag_log) + Fh(tl, 0, "\t\tacl_str_%s,\n", sym->name); + else + Fh(tl, 0, "\t\tNULL,\n"); + Fh(tl, 0, "\t\t\"NO MATCH %s\"\n\t )\n\t);\n", sym->name); + } else { + /* Deny by default */ + if (tl->acl->flag_log) + Fh(tl, 0, "\tVPI_acl_log(ctx, \"NO_MATCH %s\");\n", + sym->name); + Fh(tl, 0, "\treturn(0);\n"); + } + Fh(tl, 0, "}\n"); /* Emit the struct that will be referenced */ Fh(tl, 0, "\nstatic const struct vrt_acl %s[] = {{\n", sym->rname); @@ -596,6 +715,9 @@ vcc_ParseAcl(struct vcc *tl) if (vcc_IdIs(tl->t, "log")) { acl->flag_log = sign->tok == '+'; vcc_NextToken(tl); + } else if (vcc_IdIs(tl->t, "table")) { + acl->flag_table = sign->tok == '+'; + vcc_NextToken(tl); } else { VSB_cat(tl->sb, "Unknown ACL flag:\n"); vcc_ErrWhere(tl, tl->t); From dridi.boukelmoune at gmail.com Mon Apr 12 12:49:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 12 Apr 2021 12:49:05 +0000 (UTC) Subject: [master] 9a4523dfd vcc: Missing minor error handling Message-ID: <20210412124905.9597EACA07@lists.varnish-cache.org> commit 9a4523dfd993c4ffc0a22d92b0693c1f738fc5f4 Author: Dridi Boukelmoune Date: Mon Apr 12 14:47:30 2021 +0200 vcc: Missing minor error handling diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index 226215fda..ac6ca3d87 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -164,6 +164,7 @@ vcc_acl_chk(struct vcc *tl, const struct acl_e *ae, const int l, char h[VTCP_ADDRBUFSIZE]; char t[VTCP_ADDRBUFSIZE + 10]; char s[vsa_suckaddr_len]; + char *r = NULL; struct suckaddr *sa; unsigned m; int ll, ret = 0; @@ -208,7 +209,8 @@ vcc_acl_chk(struct vcc *tl, const struct acl_e *ae, const int l, vcc_ErrWhere(tl, ae->t_addr); if (tl->acl_pedantic == 0) vcc_Warn(tl); - return (strdup(t)); + REPLACE(r, t); + return (r); } static void From nils.goroll at uplex.de Mon Apr 12 12:56:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Apr 2021 14:56:07 +0200 Subject: [master] d89674d12 Collapse vcc_acl_add_entry() and vcc_acl_insert_entry() In-Reply-To: <20210412120905.4BD5BAB67B@lists.varnish-cache.org> References: <20210412120905.4BD5BAB67B@lists.varnish-cache.org> Message-ID: <9f3c1a61-573f-2a73-ba1f-c8411f729407@uplex.de> @phk, the split was intentional, see bbb5db47e02f431598e5a7ff7106a8ac316db8be Can I revert this to have a separate vcc_acl_insert_entry() again? -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From dridi.boukelmoune at gmail.com Mon Apr 12 13:14:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 12 Apr 2021 13:14:05 +0000 (UTC) Subject: [master] f44f16cc4 Revert "disable -Wcast-qual because of musl" Message-ID: <20210412131405.2D545AD952@lists.varnish-cache.org> commit f44f16cc4696e9e7f434dfd6d67b4e0447b4e97b Author: Dridi Boukelmoune Date: Mon Apr 12 15:09:13 2021 +0200 Revert "disable -Wcast-qual because of musl" This reverts commit 05780b634c40233dd2494a2c76a8dd8ba72d79f8. If we have to choose between this warning and alpine being part of our CI, this warning takes precedence. It's always possible to disable developer warnings on CCI if that's a problem with musl then we can configure its CCI job to disable developer warnings. Reopens #3565 Refs #3568 diff --git a/wflags.py b/wflags.py index 7e4d30fed..d060d1794 100644 --- a/wflags.py +++ b/wflags.py @@ -44,6 +44,7 @@ DESIRABLE_OPTIONS = [ DESIRABLE_WFLAGS = [ "-Wcast-align", + "-Wcast-qual", "-Wchar-subscripts", "-Wempty-body", "-Wextra", @@ -68,7 +69,6 @@ DESIRABLE_WFLAGS = [ ] UNDESIRABLE_WFLAGS = [ - "-Wno-cast-qual", # GCC complains about musl::sched.h "-Wno-thread-safety", # Does not understand our mutexs are wrapped "-Wno-old-style-definition", # Does not like vgz "-Wno-sign-compare", # Fixable From guillaume at varnish-software.com Mon Apr 12 14:20:05 2021 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Mon, 12 Apr 2021 14:20:05 +0000 (UTC) Subject: [master] 2baff3dc8 [cci] no need to pin glibc anymore Message-ID: <20210412142005.AFD6CAF956@lists.varnish-cache.org> commit 2baff3dc8b6e7f574cce53cac9345e0f30f72beb Author: Guillaume Quintard Date: Mon Apr 12 07:19:19 2021 -0700 [cci] no need to pin glibc anymore diff --git a/.circleci/config.yml b/.circleci/config.yml index 09d0f2db1..9a10465cc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -206,10 +206,6 @@ jobs: tar \ sudo elif [ << parameters.dist >> = archlinux ]; then - # XXX: TEMPORARY fix for https://bugs.archlinux.org/task/69563 - cd /tmp - patched_glibc=glibc-linux4-2.33-4-x86_64.pkg.tar.zst && curl -LO "https://repo.archlinuxcn.org/x86_64/$patched_glibc" && bsdtar -C / -xvf "$patched_glibc" - cd - pacman -Sy --noconfirm \ base-devel \ ca-certificates \ From guillaume at varnish-software.com Mon Apr 12 14:41:04 2021 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Mon, 12 Apr 2021 14:41:04 +0000 (UTC) Subject: [master] 6ab687ca1 Revert "[cci] no need to pin glibc anymore" Message-ID: <20210412144104.A7E2EB0411@lists.varnish-cache.org> commit 6ab687ca123831ee9e75d90db0a42b93dad61186 Author: Guillaume Quintard Date: Mon Apr 12 07:40:38 2021 -0700 Revert "[cci] no need to pin glibc anymore" This reverts commit 2baff3dc8b6e7f574cce53cac9345e0f30f72beb. diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a10465cc..09d0f2db1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -206,6 +206,10 @@ jobs: tar \ sudo elif [ << parameters.dist >> = archlinux ]; then + # XXX: TEMPORARY fix for https://bugs.archlinux.org/task/69563 + cd /tmp + patched_glibc=glibc-linux4-2.33-4-x86_64.pkg.tar.zst && curl -LO "https://repo.archlinuxcn.org/x86_64/$patched_glibc" && bsdtar -C / -xvf "$patched_glibc" + cd - pacman -Sy --noconfirm \ base-devel \ ca-certificates \ From nils.goroll at uplex.de Mon Apr 12 16:04:05 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 12 Apr 2021 16:04:05 +0000 (UTC) Subject: [master] c39800d8b Bring back {Req, Resp, Bereq, Beresp}Unset to varnishtop Message-ID: <20210412160405.292E7B29D7@lists.varnish-cache.org> commit c39800d8b6bc5cf81bb96bc165c99fcd5b22fd3a Author: Nils Goroll Date: Mon Apr 12 17:56:28 2021 +0200 Bring back {Req,Resp,Bereq,Beresp}Unset to varnishtop Fixes #3527 diff --git a/include/tbl/vsl_tags_http.h b/include/tbl/vsl_tags_http.h index 7a0198da6..29d731300 100644 --- a/include/tbl/vsl_tags_http.h +++ b/include/tbl/vsl_tags_http.h @@ -83,7 +83,7 @@ SLTH(Header, HTTP_HDR_FIRST, 1, 1, "header", HEADER_NOTICE ) -SLTH(Unset, HTTP_HDR_UNSET, 0, 0, "unset header", +SLTH(Unset, HTTP_HDR_UNSET, 1, 1, "unset header", "HTTP header contents.\n\n" "The format is::\n\n" "\t%s: %s\n" From guillaume at varnish-software.com Tue Apr 13 00:46:05 2021 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Tue, 13 Apr 2021 00:46:05 +0000 (UTC) Subject: [archlinux_tests] 149c0a97e [cci][wip] dumb test Message-ID: <20210413004605.2EF446E0C1@lists.varnish-cache.org> commit 149c0a97ef22300677f21a5d30aac66958380f91 Author: Guillaume Quintard Date: Mon Apr 12 17:45:36 2021 -0700 [cci][wip] dumb test diff --git a/.circleci/config.yml b/.circleci/config.yml index fe8f62e96..6f27d151e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -126,6 +126,13 @@ jobs: root: . paths: - "packages" + arch_test: + docker: + - image: archlinux:latest + steps: + - run: + command: | + pacman -Suy --noconfirm distcheck: parameters: dist: @@ -274,6 +281,7 @@ workflows: version: 2 commit: jobs: + - arch_test - dist - distcheck: name: distcheck_centos_7 From guillaume at varnish-software.com Tue Apr 13 00:52:04 2021 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Tue, 13 Apr 2021 00:52:04 +0000 (UTC) Subject: [archlinux_tests] 0ea31f68f [cci][wipi] ls debug Message-ID: <20210413005204.B13B06E522@lists.varnish-cache.org> commit 0ea31f68fdb43f0413f3ba32d568b8ab3f26cb3f Author: Guillaume Quintard Date: Mon Apr 12 17:50:07 2021 -0700 [cci][wipi] ls debug diff --git a/.circleci/config.yml b/.circleci/config.yml index 6f27d151e..0da8a788e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -132,6 +132,7 @@ jobs: steps: - run: command: | + ls /var/lib/pacman/ pacman -Suy --noconfirm distcheck: parameters: From phk at FreeBSD.org Tue Apr 13 13:20:05 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 13 Apr 2021 13:20:05 +0000 (UTC) Subject: [master] 9f88fa613 Split the source/preprocessing stuff into a separate file. Message-ID: <20210413132005.DE104A85C1@lists.varnish-cache.org> commit 9f88fa61306f218d948327894f0ee4000b6a63ae Author: Poul-Henning Kamp Date: Tue Apr 13 12:35:42 2021 +0000 Split the source/preprocessing stuff into a separate file. diff --git a/lib/libvcc/Makefile.am b/lib/libvcc/Makefile.am index 1e3d6f2d0..c97d709c5 100644 --- a/lib/libvcc/Makefile.am +++ b/lib/libvcc/Makefile.am @@ -25,6 +25,7 @@ libvcc_a_SOURCES = \ vcc_fixed_token.c \ vcc_obj.c \ vcc_parse.c \ + vcc_source.c \ vcc_storage.c \ vcc_symb.c \ vcc_token.c \ diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 8991e9a30..1bb0ba32b 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -603,158 +603,6 @@ EmitStruct(const struct vcc *tl) Fc(tl, 0, "};\n"); } -/*--------------------------------------------------------------------*/ - -static struct source * -vcc_new_source(const char *src, const char *name) -{ - struct source *sp; - - AN(src); - AN(name); - sp = calloc(1, sizeof *sp); - AN(sp); - REPLACE(sp->name, name); - sp->b = src; - sp->e = strchr(src, '\0'); - return (sp); -} - -/*--------------------------------------------------------------------*/ - -static void -vcc_destroy_source(struct source **spp) -{ - struct source *sp; - - AN(spp); - sp = *spp; - *spp = NULL; - - AN(sp); - free(sp->name); - free(sp->freeit); - free(sp); -} - -/*--------------------------------------------------------------------*/ - -static struct source * -vcc_file_source(const struct vcc *tl, const char *fn) -{ - char *f, *fnp; - struct source *sp; - - if (!tl->unsafe_path && strchr(fn, '/') != NULL) { - VSB_printf(tl->sb, "VCL filename '%s' is unsafe.\n", fn); - return (NULL); - } - f = NULL; - if (VFIL_searchpath(tl->vcl_path, NULL, &f, fn, &fnp) || f == NULL) { - VSB_printf(tl->sb, "Cannot read file '%s' (%s)\n", - fnp != NULL ? fnp : fn, strerror(errno)); - free(fnp); - return (NULL); - } - sp = vcc_new_source(f, fnp); - free(fnp); - sp->freeit = f; - return (sp); -} - -/*--------------------------------------------------------------------*/ - -static void -vcc_resolve_includes(struct vcc *tl) -{ - struct token *t, *t1, *t2; - struct source *sp; - const struct source *sp1; - struct vsb *vsb; - const char *p; - - VTAILQ_FOREACH(t, &tl->tokens, list) { - if (t->tok != ID || !vcc_IdIs(t, "include")) - continue; - - t1 = VTAILQ_NEXT(t, list); - AN(t1); /* There's always an EOI */ - if (t1->tok != CSTR) { - VSB_cat(tl->sb, - "include not followed by string constant.\n"); - vcc_ErrWhere(tl, t1); - return; - } - t2 = VTAILQ_NEXT(t1, list); - AN(t2); /* There's always an EOI */ - - if (t2->tok != ';') { - VSB_cat(tl->sb, - "include not followed by semicolon.\n"); - vcc_ErrWhere(tl, t1); - return; - } - - if (t1->dec[0] == '.' && t1->dec[1] == '/') { - /* - * Nested include filenames, starting with "./" are - * resolved relative to the VCL file which contains - * the include directive. - */ - if (t1->src->name[0] != '/') { - VSB_cat(tl->sb, - "include \"./xxxxx\"; needs absolute " - "filename of including file.\n"); - vcc_ErrWhere(tl, t1); - return; - } - vsb = VSB_new_auto(); - AN(vsb); - p = strrchr(t1->src->name, '/'); - AN(p); - VSB_bcat(vsb, t1->src->name, p - t1->src->name); - VSB_cat(vsb, t1->dec + 1); - AZ(VSB_finish(vsb)); - sp = vcc_file_source(tl, VSB_data(vsb)); - VSB_destroy(&vsb); - } else { - sp = vcc_file_source(tl, t1->dec); - } - if (sp == NULL) { - vcc_ErrWhere(tl, t1); - return; - } - - for (sp1 = t->src; sp1 != NULL; sp1 = sp1->parent) - if (!strcmp(sp1->name, sp->name)) - break; - if (sp1 != NULL) { - VSB_printf(tl->sb, - "Recursive include of \"%s\"\n\n", sp->name); - vcc_ErrWhere(tl, t1); - for (sp1 = t->src; sp1 != NULL; sp1 = sp1->parent) { - if (sp1->parent_tok) - vcc_ErrWhere(tl, sp1->parent_tok); - } - vcc_destroy_source(&sp); - return; - } - sp->parent = t->src; - sp->parent_tok = t1; - VTAILQ_INSERT_TAIL(&tl->sources, sp, list); - sp->idx = tl->nsources++; - tl->t = t2; - vcc_Lexer(tl, sp, 0); - - VTAILQ_REMOVE(&tl->tokens, t, list); - VTAILQ_REMOVE(&tl->tokens, t1, list); - VTAILQ_REMOVE(&tl->tokens, t2, list); - if (!tl->err) - vcc_resolve_includes(tl); - return; - } -} - /*-------------------------------------------------------------------- * Compile the VCL code from the given source and return the C-source */ diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 231a2cd15..aeca3d5c7 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -362,16 +362,10 @@ void vcc_Parse(struct vcc *); void vcc_Parse_Init(struct vcc *); sym_act_f vcc_Act_If; -/* vcc_utils.c */ -void vcc_regexp(struct vcc *tl, struct vsb *vgc_name); -void Resolve_Sockaddr(struct vcc *tl, const char *host, const char *defport, - const char **ipv4, const char **ipv4_ascii, const char **ipv6, - const char **ipv6_ascii, const char **p_ascii, int maxips, - const struct token *t_err, const char *errid); -double vcc_DurationUnit(struct vcc *); -void vcc_ByteVal(struct vcc *, double *); -void vcc_Duration(struct vcc *tl, double *); -unsigned vcc_UintVal(struct vcc *tl); +/* vcc_source.c */ +struct source * vcc_new_source(const char *src, const char *name); +struct source *vcc_file_source(const struct vcc *tl, const char *fn); +void vcc_resolve_includes(struct vcc *tl); /* vcc_storage.c */ void vcc_stevedore(struct vcc *vcc, const char *stv_name); @@ -426,6 +420,17 @@ vcc_type_t VCC_Type(const char *p); const char * VCC_Type_EvalMethod(struct vcc *, const struct symbol *); void vcc_Type_Init(struct vcc *tl); +/* vcc_utils.c */ +void vcc_regexp(struct vcc *tl, struct vsb *vgc_name); +void Resolve_Sockaddr(struct vcc *tl, const char *host, const char *defport, + const char **ipv4, const char **ipv4_ascii, const char **ipv6, + const char **ipv6_ascii, const char **p_ascii, int maxips, + const struct token *t_err, const char *errid); +double vcc_DurationUnit(struct vcc *); +void vcc_ByteVal(struct vcc *, double *); +void vcc_Duration(struct vcc *tl, double *); +unsigned vcc_UintVal(struct vcc *tl); + /* vcc_var.c */ sym_wildcard_t vcc_Var_Wildcard; diff --git a/lib/libvcc/vcc_source.c b/lib/libvcc/vcc_source.c new file mode 100644 index 000000000..61095e778 --- /dev/null +++ b/lib/libvcc/vcc_source.c @@ -0,0 +1,190 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2015 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "config.h" + +#include +#include +#include + +#include "vcc_compile.h" + +#include "vfil.h" + +struct source * +vcc_new_source(const char *src, const char *name) +{ + struct source *sp; + + AN(src); + AN(name); + sp = calloc(1, sizeof *sp); + AN(sp); + REPLACE(sp->name, name); + sp->b = src; + sp->e = strchr(src, '\0'); + return (sp); +} + +/*--------------------------------------------------------------------*/ + +static void +vcc_destroy_source(struct source **spp) +{ + struct source *sp; + + AN(spp); + sp = *spp; + *spp = NULL; + + AN(sp); + free(sp->name); + free(sp->freeit); + free(sp); +} + +/*--------------------------------------------------------------------*/ + +struct source * +vcc_file_source(const struct vcc *tl, const char *fn) +{ + char *f, *fnp; + struct source *sp; + + if (!tl->unsafe_path && strchr(fn, '/') != NULL) { + VSB_printf(tl->sb, "VCL filename '%s' is unsafe.\n", fn); + return (NULL); + } + f = NULL; + if (VFIL_searchpath(tl->vcl_path, NULL, &f, fn, &fnp) || f == NULL) { + VSB_printf(tl->sb, "Cannot read file '%s' (%s)\n", + fnp != NULL ? fnp : fn, strerror(errno)); + free(fnp); + return (NULL); + } + sp = vcc_new_source(f, fnp); + free(fnp); + sp->freeit = f; + return (sp); +} + +/*--------------------------------------------------------------------*/ + +void +vcc_resolve_includes(struct vcc *tl) +{ + struct token *t, *t1, *t2; + struct source *sp; + const struct source *sp1; + struct vsb *vsb; + const char *p; + + VTAILQ_FOREACH(t, &tl->tokens, list) { + if (t->tok != ID || !vcc_IdIs(t, "include")) + continue; + + t1 = VTAILQ_NEXT(t, list); + AN(t1); /* There's always an EOI */ + if (t1->tok != CSTR) { + VSB_cat(tl->sb, + "include not followed by string constant.\n"); + vcc_ErrWhere(tl, t1); + return; + } + t2 = VTAILQ_NEXT(t1, list); + AN(t2); /* There's always an EOI */ + + if (t2->tok != ';') { + VSB_cat(tl->sb, + "include not followed by semicolon.\n"); + vcc_ErrWhere(tl, t1); + return; + } + + if (t1->dec[0] == '.' && t1->dec[1] == '/') { + /* + * Nested include filenames, starting with "./" are + * resolved relative to the VCL file which contains + * the include directive. + */ + if (t1->src->name[0] != '/') { + VSB_cat(tl->sb, + "include \"./xxxxx\"; needs absolute " + "filename of including file.\n"); + vcc_ErrWhere(tl, t1); + return; + } + vsb = VSB_new_auto(); + AN(vsb); + p = strrchr(t1->src->name, '/'); + AN(p); + VSB_bcat(vsb, t1->src->name, p - t1->src->name); + VSB_cat(vsb, t1->dec + 1); + AZ(VSB_finish(vsb)); + sp = vcc_file_source(tl, VSB_data(vsb)); + VSB_destroy(&vsb); + } else { + sp = vcc_file_source(tl, t1->dec); + } + if (sp == NULL) { + vcc_ErrWhere(tl, t1); + return; + } + + for (sp1 = t->src; sp1 != NULL; sp1 = sp1->parent) + if (!strcmp(sp1->name, sp->name)) + break; + if (sp1 != NULL) { + VSB_printf(tl->sb, + "Recursive include of \"%s\"\n\n", sp->name); + vcc_ErrWhere(tl, t1); + for (sp1 = t->src; sp1 != NULL; sp1 = sp1->parent) { + if (sp1->parent_tok) + vcc_ErrWhere(tl, sp1->parent_tok); + } + vcc_destroy_source(&sp); + return; + } + sp->parent = t->src; + sp->parent_tok = t1; + VTAILQ_INSERT_TAIL(&tl->sources, sp, list); + sp->idx = tl->nsources++; + tl->t = t2; + vcc_Lexer(tl, sp, 0); + + VTAILQ_REMOVE(&tl->tokens, t, list); + VTAILQ_REMOVE(&tl->tokens, t1, list); + VTAILQ_REMOVE(&tl->tokens, t2, list); + if (!tl->err) + vcc_resolve_includes(tl); + return; + } +} From phk at FreeBSD.org Tue Apr 13 13:20:05 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 13 Apr 2021 13:20:05 +0000 (UTC) Subject: [master] c8174af68 Refactor "preprocesing" of VCL. Message-ID: <20210413132005.F3D8DA85C4@lists.varnish-cache.org> commit c8174af68956206115972e72e75ceddbd758116e Author: Poul-Henning Kamp Date: Tue Apr 13 13:19:00 2021 +0000 Refactor "preprocesing" of VCL. diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 1bb0ba32b..88d558a93 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -638,7 +638,7 @@ vcc_CompileSource(struct vcc *tl, struct source *sp, const char *jfile) AN(vcc_builtin); VTAILQ_INSERT_TAIL(&tl->sources, sp, list); sp->idx = tl->nsources++; - vcc_Lexer(tl, sp, 0); + vcc_lex_source(tl, sp, 0); if (tl->err) return (NULL); } @@ -648,12 +648,11 @@ vcc_CompileSource(struct vcc *tl, struct source *sp, const char *jfile) assert(sp != NULL); VTAILQ_INSERT_TAIL(&tl->sources, sp, list); sp->idx = tl->nsources++; - vcc_Lexer(tl, sp, 1); + vcc_lex_source(tl, sp, 1); if (tl->err) return (NULL); /* Expand and lex any includes in the token string */ - vcc_resolve_includes(tl); if (tl->err) return (NULL); diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index aeca3d5c7..76e923693 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -57,11 +57,18 @@ /*---------------------------------------------------------------------*/ struct acl; -struct vsb; -struct token; -struct sockaddr_storage; +struct acl_e; +struct expr; struct method; +struct proc; +struct sockaddr_storage; +struct symbol; struct symtab; +struct token; +struct vcc; +struct vjsn_val; +struct vmod_obj; +struct vsb; unsigned vcl_fixed_token(const char *p, const char **q); extern const char * const vcl_tnames[256]; @@ -71,14 +78,6 @@ void vcl_output_lang_h(struct vsb *sb); #define INDENT 2 -struct acl_e; -struct proc; -struct expr; -struct vcc; -struct vjsn_val; -struct symbol; -struct vmod_obj; - struct source { VTAILQ_ENTRY(source) list; char *name; @@ -88,6 +87,7 @@ struct source { char *freeit; const struct source *parent; const struct token *parent_tok; + VTAILQ_HEAD(, token) src_tokens; }; struct token { @@ -96,6 +96,7 @@ struct token { const char *e; const struct source *src; VTAILQ_ENTRY(token) list; + VTAILQ_ENTRY(token) src_list; unsigned cnt; char *dec; }; @@ -365,7 +366,7 @@ sym_act_f vcc_Act_If; /* vcc_source.c */ struct source * vcc_new_source(const char *src, const char *name); struct source *vcc_file_source(const struct vcc *tl, const char *fn); -void vcc_resolve_includes(struct vcc *tl); +void vcc_lex_source(struct vcc *tl, struct source *sp, int eoi); /* vcc_storage.c */ void vcc_stevedore(struct vcc *vcc, const char *stv_name); @@ -410,7 +411,7 @@ void vcc_Warn(struct vcc *); void vcc__Expect(struct vcc *tl, unsigned tok, unsigned line); int vcc_IdIs(const struct token *t, const char *p); void vcc_ExpectVid(struct vcc *tl, const char *what); -void vcc_Lexer(struct vcc *tl, const struct source *sp, int eoi); +void vcc_Lexer(struct vcc *tl, struct source *sp); void vcc_NextToken(struct vcc *tl); void vcc__ErrInternal(struct vcc *tl, const char *func, unsigned line); diff --git a/lib/libvcc/vcc_source.c b/lib/libvcc/vcc_source.c index 61095e778..48f50553e 100644 --- a/lib/libvcc/vcc_source.c +++ b/lib/libvcc/vcc_source.c @@ -51,6 +51,7 @@ vcc_new_source(const char *src, const char *name) REPLACE(sp->name, name); sp->b = src; sp->e = strchr(src, '\0'); + VTAILQ_INIT(&sp->src_tokens); return (sp); } @@ -98,93 +99,110 @@ vcc_file_source(const struct vcc *tl, const char *fn) /*--------------------------------------------------------------------*/ -void -vcc_resolve_includes(struct vcc *tl) +static struct source * +vcc_include_file(struct vcc *tl, const struct source *src_sp, + const char *filename, const struct token *parent_token) { - struct token *t, *t1, *t2; struct source *sp; const struct source *sp1; struct vsb *vsb; const char *p; - VTAILQ_FOREACH(t, &tl->tokens, list) { - if (t->tok != ID || !vcc_IdIs(t, "include")) + if (filename[0] == '.' && filename[1] == '/') { + /* + * Nested include filenames, starting with "./" are + * resolved relative to the VCL file which contains + * the include directive. + */ + if (src_sp->name[0] != '/') { + VSB_cat(tl->sb, + "include \"./xxxxx\"; needs absolute " + "filename of including file.\n"); + return (NULL); + } + vsb = VSB_new_auto(); + AN(vsb); + p = strrchr(src_sp->name, '/'); + AN(p); + VSB_bcat(vsb, src_sp->name, p - src_sp->name); + VSB_cat(vsb, filename + 1); + AZ(VSB_finish(vsb)); + sp = vcc_file_source(tl, VSB_data(vsb)); + VSB_destroy(&vsb); + } else { + sp = vcc_file_source(tl, filename); + } + if (sp == NULL) + return (sp); + + for (sp1 = src_sp; sp1 != NULL; sp1 = sp1->parent) + if (!strcmp(sp1->name, sp->name)) + break; + if (sp1 != NULL) { + VSB_printf(tl->sb, + "Recursive include of \"%s\"\n\n", sp->name); + vcc_destroy_source(&sp); + return (NULL); + } + sp->parent = src_sp; + VTAILQ_INSERT_TAIL(&tl->sources, sp, list); + sp->idx = tl->nsources++; + sp->parent_tok = parent_token; + vcc_lex_source(tl, sp, 0); + return (sp); +} + +/*--------------------------------------------------------------------*/ + +void +vcc_lex_source(struct vcc *tl, struct source *src_sp, int eoi) +{ + struct token *t, *t1; + struct source *sp; + const struct source *sp1; + + vcc_Lexer(tl, src_sp); + if (tl->err) + return; + VTAILQ_FOREACH(t, &src_sp->src_tokens, src_list) { + if (!eoi && t->tok == EOI) + break; + + if (t->tok != ID || !vcc_IdIs(t, "include")) { + VTAILQ_INSERT_TAIL(&tl->tokens, t, list); continue; + } - t1 = VTAILQ_NEXT(t, list); + t1 = VTAILQ_NEXT(t, src_list); AN(t1); /* There's always an EOI */ if (t1->tok != CSTR) { VSB_cat(tl->sb, "include not followed by string constant.\n"); + /* Not vcc_ErrWhere2(): tokens not on final list */ vcc_ErrWhere(tl, t1); return; } - t2 = VTAILQ_NEXT(t1, list); - AN(t2); /* There's always an EOI */ + t = VTAILQ_NEXT(t1, src_list); + AN(t); /* There's always an EOI */ - if (t2->tok != ';') { + if (t->tok != ';') { VSB_cat(tl->sb, "include not followed by semicolon.\n"); + /* Not vcc_ErrWhere2(): tokens not on final list */ vcc_ErrWhere(tl, t1); return; } - if (t1->dec[0] == '.' && t1->dec[1] == '/') { - /* - * Nested include filenames, starting with "./" are - * resolved relative to the VCL file which contains - * the include directive. - */ - if (t1->src->name[0] != '/') { - VSB_cat(tl->sb, - "include \"./xxxxx\"; needs absolute " - "filename of including file.\n"); - vcc_ErrWhere(tl, t1); - return; - } - vsb = VSB_new_auto(); - AN(vsb); - p = strrchr(t1->src->name, '/'); - AN(p); - VSB_bcat(vsb, t1->src->name, p - t1->src->name); - VSB_cat(vsb, t1->dec + 1); - AZ(VSB_finish(vsb)); - sp = vcc_file_source(tl, VSB_data(vsb)); - VSB_destroy(&vsb); - } else { - sp = vcc_file_source(tl, t1->dec); - } + sp = vcc_include_file(tl, src_sp, t1->dec, t1); if (sp == NULL) { vcc_ErrWhere(tl, t1); - return; - } - - for (sp1 = t->src; sp1 != NULL; sp1 = sp1->parent) - if (!strcmp(sp1->name, sp->name)) - break; - if (sp1 != NULL) { - VSB_printf(tl->sb, - "Recursive include of \"%s\"\n\n", sp->name); - vcc_ErrWhere(tl, t1); - for (sp1 = t->src; sp1 != NULL; sp1 = sp1->parent) { - if (sp1->parent_tok) - vcc_ErrWhere(tl, sp1->parent_tok); + for (sp1 = src_sp; sp1 != NULL; sp1 = sp1->parent) { + if (sp1->parent_tok == NULL) + break; + vcc_ErrWhere(tl, sp1->parent_tok); } - vcc_destroy_source(&sp); - return; } - sp->parent = t->src; - sp->parent_tok = t1; - VTAILQ_INSERT_TAIL(&tl->sources, sp, list); - sp->idx = tl->nsources++; - tl->t = t2; - vcc_Lexer(tl, sp, 0); - - VTAILQ_REMOVE(&tl->tokens, t, list); - VTAILQ_REMOVE(&tl->tokens, t1, list); - VTAILQ_REMOVE(&tl->tokens, t2, list); - if (!tl->err) - vcc_resolve_includes(tl); - return; + if (tl->err) + return; } } diff --git a/lib/libvcc/vcc_token.c b/lib/libvcc/vcc_token.c index 9edff6226..cc8392196 100644 --- a/lib/libvcc/vcc_token.c +++ b/lib/libvcc/vcc_token.c @@ -365,7 +365,7 @@ vcc_decstr(struct vcc *tl, unsigned sep) static void vcc_addtoken(struct vcc *tl, unsigned tok, - const struct source *sp, const char *b, const char *e) + struct source *sp, const char *b, const char *e) { struct token *t; @@ -375,10 +375,7 @@ vcc_addtoken(struct vcc *tl, unsigned tok, t->b = b; t->e = e; t->src = sp; - if (tl->t != NULL) - VTAILQ_INSERT_AFTER(&tl->tokens, tl->t, t, list); - else - VTAILQ_INSERT_TAIL(&tl->tokens, t, list); + VTAILQ_INSERT_TAIL(&sp->src_tokens, t, src_list); tl->t = t; } @@ -405,7 +402,7 @@ static const struct delim_def { }; static unsigned -vcc_delim_token(struct vcc *tl, const struct source *sp, const char *p, +vcc_delim_token(struct vcc *tl, struct source *sp, const char *p, const char **qp) { const struct delim_def *dd; @@ -445,7 +442,7 @@ vcc_delim_token(struct vcc *tl, const struct source *sp, const char *p, */ void -vcc_Lexer(struct vcc *tl, const struct source *sp, int eoi) +vcc_Lexer(struct vcc *tl, struct source *sp) { const char *p, *q, *r; unsigned u; @@ -611,6 +608,5 @@ vcc_Lexer(struct vcc *tl, const struct source *sp, int eoi) vcc_ErrWhere(tl, tl->t); return; } - if (eoi) - vcc_addtoken(tl, EOI, sp, sp->e, sp->e); + vcc_addtoken(tl, EOI, sp, sp->e, sp->e); } From reza at naghibi.com Tue Apr 13 13:45:06 2021 From: reza at naghibi.com (Reza Naghibi) Date: Tue, 13 Apr 2021 13:45:06 +0000 (UTC) Subject: [6.0] 765d564a9 [cci] PowerTools -> powertools Message-ID: <20210413134506.23D1AA92E0@lists.varnish-cache.org> commit 765d564a9d4d1bccdd760a86132b35c4c10e560f Author: Guillaume Quintard Date: Wed Dec 9 15:35:22 2020 -0800 [cci] PowerTools -> powertools diff --git a/.circleci/config.yml b/.circleci/config.yml index 0b8da355f..d830ae1df 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -169,7 +169,7 @@ jobs: if [ << parameters.dist >> = centos ]; then if [ << parameters.release >> = 8 ]; then dnf install -y 'dnf-command(config-manager)' - yum config-manager --set-enabled PowerTools + yum config-manager --set-enabled powertools yum install -y diffutils python3-sphinx else yum install -y python-sphinx From reza at naghibi.com Tue Apr 13 13:45:06 2021 From: reza at naghibi.com (Reza Naghibi) Date: Tue, 13 Apr 2021 13:45:06 +0000 (UTC) Subject: [6.0] 140ffe2a4 rename the PowerTools leftovers Message-ID: <20210413134506.39D3BA92E4@lists.varnish-cache.org> commit 140ffe2a46bc09139834fb531ff1c5b219b7878b Author: Guillaume Quintard Date: Thu Dec 10 21:52:44 2020 -0800 rename the PowerTools leftovers Conflicts: doc/sphinx/installation/install_source.rst diff --git a/.circleci/make-rpm-packages.sh b/.circleci/make-rpm-packages.sh index 5f7d0af1a..56acdad05 100755 --- a/.circleci/make-rpm-packages.sh +++ b/.circleci/make-rpm-packages.sh @@ -18,7 +18,7 @@ yum install -y epel-release if [ "$PARAM_DIST" = centos ]; then if [ "$PARAM_RELEASE" = 8 ]; then dnf install -y 'dnf-command(config-manager)' - yum config-manager --set-enabled PowerTools + yum config-manager --set-enabled powertools fi fi From phk at FreeBSD.org Tue Apr 13 17:23:05 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 13 Apr 2021 17:23:05 +0000 (UTC) Subject: [master] a1c936c83 Generalize the VCL flag stuff a little bit. Message-ID: <20210413172305.CF49873EC@lists.varnish-cache.org> commit a1c936c83b56d1d5ee211dd550a555830ae44396 Author: Poul-Henning Kamp Date: Tue Apr 13 15:09:16 2021 +0000 Generalize the VCL flag stuff a little bit. diff --git a/bin/varnishtest/tests/v00017.vtc b/bin/varnishtest/tests/v00017.vtc index 17f94582d..6ed35abc6 100644 --- a/bin/varnishtest/tests/v00017.vtc +++ b/bin/varnishtest/tests/v00017.vtc @@ -101,7 +101,7 @@ varnish v1 -errvcl {/mask only allowed once} { sub vcl_recv { if (client.ip ~ a) { return(pass); } } } -varnish v1 -errvcl {Expected ACL flag after:} { +varnish v1 -errvcl {Expected a flag at:} { backend b { .host = "${localhost}"; } acl a + foobar { "10.0.1.0/22" / 22; diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index ac6ca3d87..6f595f7ad 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -692,7 +692,7 @@ void vcc_ParseAcl(struct vcc *tl) { struct symbol *sym; - struct token *sign; + int sign; struct acl acl[1]; INIT_OBJ(acl, VCC_ACL_MAGIC); @@ -706,19 +706,20 @@ vcc_ParseAcl(struct vcc *tl) ERRCHK(tl); AN(sym); - while (tl->t->tok == '-' || tl->t->tok == '+') { - sign = tl->t; - vcc_NextToken(tl); - if (tl->t->b != sign->e) { - VSB_cat(tl->sb, "Expected ACL flag after:\n"); - vcc_ErrWhere(tl, sign); + while (1) { + sign = vcc_IsFlag(tl); + if (tl->err) { + VSB_cat(tl->sb, + "Valid ACL flags are `log` and `table`:\n"); return; } + if (sign < 0) + break; if (vcc_IdIs(tl->t, "log")) { - acl->flag_log = sign->tok == '+'; + acl->flag_log = sign; vcc_NextToken(tl); } else if (vcc_IdIs(tl->t, "table")) { - acl->flag_table = sign->tok == '+'; + acl->flag_table = sign; vcc_NextToken(tl); } else { VSB_cat(tl->sb, "Unknown ACL flag:\n"); diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 76e923693..995399ab6 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -431,6 +431,7 @@ double vcc_DurationUnit(struct vcc *); void vcc_ByteVal(struct vcc *, double *); void vcc_Duration(struct vcc *tl, double *); unsigned vcc_UintVal(struct vcc *tl); +int vcc_IsFlag(struct vcc *tl); /* vcc_var.c */ sym_wildcard_t vcc_Var_Wildcard; diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c index 9cc27c68b..6ecee46fb 100644 --- a/lib/libvcc/vcc_utils.c +++ b/lib/libvcc/vcc_utils.c @@ -357,3 +357,22 @@ vcc_ByteVal(struct vcc *tl, double *d) vcc_NextToken(tl); *d = v * sc; } + +/*--------------------------------------------------------------------*/ + +int +vcc_IsFlag(struct vcc *tl) +{ + struct token *sign; + + if (tl->t->tok != '-' && tl->t->tok != '+') + return (-1); + sign = tl->t; + vcc_NextToken(tl); + if (tl->t->b != sign->e) { + VSB_cat(tl->sb, "Expected a flag at:\n"); + vcc_ErrWhere(tl, sign); + return (-1); + } + return (sign->tok == '+' ? 1 : 0); +} From phk at FreeBSD.org Tue Apr 13 17:23:05 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 13 Apr 2021 17:23:05 +0000 (UTC) Subject: [master] 1a02b4744 Add a vcc_IsFlagRaw() for when the tokens are not on tl->tokens. Message-ID: <20210413172305.E3C8D73F0@lists.varnish-cache.org> commit 1a02b4744f3b3777c800ced3d74c5b21f316abd0 Author: Poul-Henning Kamp Date: Tue Apr 13 17:20:08 2021 +0000 Add a vcc_IsFlagRaw() for when the tokens are not on tl->tokens. diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 995399ab6..7d01ae9c8 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -432,6 +432,7 @@ void vcc_ByteVal(struct vcc *, double *); void vcc_Duration(struct vcc *tl, double *); unsigned vcc_UintVal(struct vcc *tl); int vcc_IsFlag(struct vcc *tl); +int vcc_IsFlagRaw(struct vcc *, const struct token *, const struct token *); /* vcc_var.c */ sym_wildcard_t vcc_Var_Wildcard; diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c index 6ecee46fb..d60d932cd 100644 --- a/lib/libvcc/vcc_utils.c +++ b/lib/libvcc/vcc_utils.c @@ -361,18 +361,27 @@ vcc_ByteVal(struct vcc *tl, double *d) /*--------------------------------------------------------------------*/ int -vcc_IsFlag(struct vcc *tl) +vcc_IsFlagRaw(struct vcc *tl, const struct token *t1, const struct token *t2) { - struct token *sign; - if (tl->t->tok != '-' && tl->t->tok != '+') + if (t1->tok != '-' && t1->tok != '+') return (-1); - sign = tl->t; - vcc_NextToken(tl); - if (tl->t->b != sign->e) { + if (t2->b != t1->e) { VSB_cat(tl->sb, "Expected a flag at:\n"); - vcc_ErrWhere(tl, sign); + vcc_ErrWhere(tl, t1); return (-1); } - return (sign->tok == '+' ? 1 : 0); + return (t1->tok == '+' ? 1 : 0); +} + +int +vcc_IsFlag(struct vcc *tl) +{ + int retval; + + + retval = vcc_IsFlagRaw(tl, tl->t, VTAILQ_NEXT(tl->t, list)); + if (retval >= 0) + vcc_NextToken(tl); + return (retval); } From phk at FreeBSD.org Tue Apr 13 17:23:06 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 13 Apr 2021 17:23:06 +0000 (UTC) Subject: [master] 195929e4a Implement `+glob` flag to `include` processing. Message-ID: <20210413172306.082F373F3@lists.varnish-cache.org> commit 195929e4aa29f46c49038b3982b842a4d647a1d5 Author: Poul-Henning Kamp Date: Tue Apr 13 17:21:27 2021 +0000 Implement `+glob` flag to `include` processing. Reimplementation of #3193 according to consensus. diff --git a/lib/libvcc/vcc_source.c b/lib/libvcc/vcc_source.c index 48f50553e..92d3ea7bd 100644 --- a/lib/libvcc/vcc_source.c +++ b/lib/libvcc/vcc_source.c @@ -31,6 +31,7 @@ #include "config.h" +#include #include #include #include @@ -99,41 +100,16 @@ vcc_file_source(const struct vcc *tl, const char *fn) /*--------------------------------------------------------------------*/ -static struct source * +static int vcc_include_file(struct vcc *tl, const struct source *src_sp, const char *filename, const struct token *parent_token) { struct source *sp; const struct source *sp1; - struct vsb *vsb; - const char *p; - if (filename[0] == '.' && filename[1] == '/') { - /* - * Nested include filenames, starting with "./" are - * resolved relative to the VCL file which contains - * the include directive. - */ - if (src_sp->name[0] != '/') { - VSB_cat(tl->sb, - "include \"./xxxxx\"; needs absolute " - "filename of including file.\n"); - return (NULL); - } - vsb = VSB_new_auto(); - AN(vsb); - p = strrchr(src_sp->name, '/'); - AN(p); - VSB_bcat(vsb, src_sp->name, p - src_sp->name); - VSB_cat(vsb, filename + 1); - AZ(VSB_finish(vsb)); - sp = vcc_file_source(tl, VSB_data(vsb)); - VSB_destroy(&vsb); - } else { - sp = vcc_file_source(tl, filename); - } + sp = vcc_file_source(tl, filename); if (sp == NULL) - return (sp); + return (-1); for (sp1 = src_sp; sp1 != NULL; sp1 = sp1->parent) if (!strcmp(sp1->name, sp->name)) @@ -142,24 +118,60 @@ vcc_include_file(struct vcc *tl, const struct source *src_sp, VSB_printf(tl->sb, "Recursive include of \"%s\"\n\n", sp->name); vcc_destroy_source(&sp); - return (NULL); + return (-1); } sp->parent = src_sp; VTAILQ_INSERT_TAIL(&tl->sources, sp, list); sp->idx = tl->nsources++; sp->parent_tok = parent_token; vcc_lex_source(tl, sp, 0); - return (sp); + return (0); } /*--------------------------------------------------------------------*/ +static int +vcc_include_glob_file(struct vcc *tl, const struct source *src_sp, + const char *filename, const struct token *parent_token) +{ + glob_t g[1]; + int rv; + unsigned u; + + memset(g, 0, sizeof g); + rv = glob(filename, 0, NULL, g); + switch (rv) { + case 0: + for (u = 0; rv == 0 && u < g->gl_pathc; u++) { + rv = vcc_include_file( + tl, src_sp, g->gl_pathv[u], parent_token); + } + break; + case GLOB_NOMATCH: + VSB_printf(tl->sb, "glob pattern matched no files.\n"); + break; + default: + VSB_printf(tl->sb, "glob(3) expansion failed (%d)\n", rv); + break; + } + globfree(g); + return (rv); +} + +/*-------------------------------------------------------------------- + * NB: We cannot use vcc_ErrWhere2() on tokens which are no on the + * NB: tl->tokens list. + */ + void vcc_lex_source(struct vcc *tl, struct source *src_sp, int eoi) { struct token *t, *t1; - struct source *sp; + int i, glob_flag = 0; const struct source *sp1; + struct vsb *vsb = NULL; + const char *filename; + const char *p; vcc_Lexer(tl, src_sp); if (tl->err) @@ -174,27 +186,72 @@ vcc_lex_source(struct vcc *tl, struct source *src_sp, int eoi) } t1 = VTAILQ_NEXT(t, src_list); - AN(t1); /* There's always an EOI */ + AN(t1); + + while (1) { + t = VTAILQ_NEXT(t1, src_list); + AN(t); + i = vcc_IsFlagRaw(tl, t1, t); + if (i < 0) + break; + if (vcc_IdIs(t, "glob")) { + glob_flag = i; + } else { + VSB_cat(tl->sb, "Unknown include flag:\n"); + vcc_ErrWhere(tl, t); + return; + } + t1 = VTAILQ_NEXT(t, src_list); + AN(t1); + } + if (t1->tok != CSTR) { VSB_cat(tl->sb, "include not followed by string constant.\n"); - /* Not vcc_ErrWhere2(): tokens not on final list */ vcc_ErrWhere(tl, t1); return; } t = VTAILQ_NEXT(t1, src_list); - AN(t); /* There's always an EOI */ + AN(t); if (t->tok != ';') { VSB_cat(tl->sb, "include not followed by semicolon.\n"); - /* Not vcc_ErrWhere2(): tokens not on final list */ vcc_ErrWhere(tl, t1); return; } - sp = vcc_include_file(tl, src_sp, t1->dec, t1); - if (sp == NULL) { + filename = t1->dec; + + if (filename[0] == '.' && filename[1] == '/') { + /* + * Nested include filenames, starting with "./" are + * resolved relative to the VCL file which contains + * the include directive. + */ + if (src_sp->name[0] != '/') { + VSB_cat(tl->sb, + "include \"./xxxxx\"; needs absolute " + "filename of including file.\n"); + return; + } + vsb = VSB_new_auto(); + AN(vsb); + p = strrchr(src_sp->name, '/'); + AN(p); + VSB_bcat(vsb, src_sp->name, p - src_sp->name); + VSB_cat(vsb, filename + 1); + AZ(VSB_finish(vsb)); + filename = VSB_data(vsb); + } + + if (glob_flag) + i = vcc_include_glob_file(tl, src_sp, filename, t1); + else + i = vcc_include_file(tl, src_sp, filename, t1); + if (vsb != NULL) + VSB_destroy(&vsb); + if (i) { vcc_ErrWhere(tl, t1); for (sp1 = src_sp; sp1 != NULL; sp1 = sp1->parent) { if (sp1->parent_tok == NULL)