From gquintard at users.noreply.github.com Fri Jul 2 16:05:07 2021 From: gquintard at users.noreply.github.com (guillaume quintard) Date: Fri, 2 Jul 2021 16:05:07 +0000 (UTC) Subject: [master] 7795c2b35 circleci: Re-enable Alpine latest Message-ID: <20210702160507.61D1D4252@lists.varnish-cache.org> commit 7795c2b357c09cd17bbb07ec667193de5bdd8d05 Author: Jordan Christiansen Date: Thu Jul 1 12:33:23 2021 -0500 circleci: Re-enable Alpine latest Alpine 3.14 includes a newer version of musl that uses the newer faccessat2 syscall, which is not yet allowlisted by the seccomp filter in older Docker versions. Docker 20.10.0+ allows this new syscall. In this patch, we configure Circle to set up a Docker 20.10+ environment where we can run Alpine 3.14 builds. diff --git a/.circleci/config.yml b/.circleci/config.yml index 4b5ddc711..1ba3ff940 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -169,15 +169,24 @@ jobs: default: "" type: string docker: - - image: << parameters.dist >>:<< parameters.release >> + - image: centos:7 working_directory: /workspace steps: + - setup_remote_docker: + version: 20.10.6 + - run: + name: Install docker + command: yum install -y docker + - checkout - run: - name: Possible << parameters.dist >>:<< parameters.release >> extra repos + name: Extract and distcheck command: | + docker create --name workspace -v /workspace << parameters.dist >>:<< parameters.release >> /bin/true + docker cp /workspace workspace:/ + docker run --volumes-from workspace -w /workspace << parameters.dist >>:<< parameters.release >> sh -c ' if [ << parameters.dist >> = centos ]; then if [ << parameters.release >> = 8 ]; then - dnf install -y 'dnf-command(config-manager)' + dnf install -y "dnf-command(config-manager)" yum config-manager --set-enabled powertools yum install -y diffutils python3-sphinx else @@ -254,10 +263,7 @@ jobs: python-sphinx \ tar fi - - checkout - - run: - name: Extract and distcheck - command: | + if [ << parameters.dist >> = archlinux ]; then useradd varnish elif [ << parameters.dist >> = centos ]; then @@ -285,6 +291,7 @@ jobs: make distcheck VERBOSE=1 -j 4 -k \ DISTCHECK_CONFIGURE_FLAGS="<< pipeline.parameters.configure_args >> \ << parameters.extra_conf >>" + ' collect_packages: docker: @@ -323,8 +330,7 @@ workflows: - distcheck: name: distcheck_alpine dist: alpine - release: "3.13" - #release: "latest" + release: "latest" extra_conf: --disable-developer-warnings --disable-dependency-tracking - distcheck: name: distcheck_archlinux From dridi.boukelmoune at gmail.com Fri Jul 2 16:22:04 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 2 Jul 2021 16:22:04 +0000 (UTC) Subject: [master] 70b7d5840 vav: Add a test driver Message-ID: <20210702162204.83BEC4C8E@lists.varnish-cache.org> commit 70b7d584064f23694c37c41ed1ed60468458394f Author: Dridi Boukelmoune Date: Fri Jul 2 17:50:02 2021 +0200 vav: Add a test driver It started with one corner case, then I found more, then I found new potential corner cases introduced by my own attempts at squaring them. I decided to take a step back and start some VAV test coverage. With something not too complicated to edit, dealing with the corner cases should increase confidence with the introduction of new cases easy to review. diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am index 3be8609f1..6d27522fb 100644 --- a/lib/libvarnish/Makefile.am +++ b/lib/libvarnish/Makefile.am @@ -46,10 +46,14 @@ libvarnish_la_SOURCES = \ libvarnish_la_LIBADD = @PCRE_LIBS@ -TESTS = vjsn_test vnum_c_test vbh_test vsb_test +TESTS = vav_test vjsn_test vnum_c_test vbh_test vsb_test noinst_PROGRAMS = ${TESTS} +vav_test_SOURCES = vav.c +vav_test_CFLAGS = $(AM_CFLAGS) -DTEST_DRIVER +vav_test_LDADD = $(AM_LDFLAGS) libvarnish.la + vbh_test_SOURCES = vbh.c vbh_test_CFLAGS = $(AM_CFLAGS) -DTEST_DRIVER vbh_test_LDADD = $(AM_LDFLAGS) libvarnish.la diff --git a/lib/libvarnish/vav.c b/lib/libvarnish/vav.c index bc3eaf2cc..c3e91f2dd 100644 --- a/lib/libvarnish/vav.c +++ b/lib/libvarnish/vav.c @@ -303,3 +303,115 @@ main(int argc, char **argv) return (0); } #endif + +#ifdef TEST_DRIVER +# include + +struct test_case { + int flag; + const char *str; + const char **argv; +}; + +static const struct test_case *tests[] = { +#define TEST_PASS(flag, str, ...) \ + &(const struct test_case){flag, str, \ + (const char **)&(const void *[]){NULL, __VA_ARGS__, NULL}} +#define TEST_FAIL(flag, str, err) \ + &(const struct test_case){flag, str, \ + (const char **)&(const void *[]){err_ ## err, NULL}} +#define K ARGV_COMMENT +#define C ARGV_COMMA +#define N ARGV_NOESC + TEST_PASS(K|C|N, "", NULL), + TEST_PASS(0 , "foo", "foo"), + TEST_PASS(0 , "foo bar", "foo", "bar"), + TEST_PASS( C , "foo bar", "foo", "bar"), + TEST_PASS( C , "foo,bar", "foo", "bar"), + TEST_PASS(0 , " foo bar ", "foo", "bar"), + TEST_PASS(0 , "foo \"bar baz\"", "foo", "bar baz"), + TEST_PASS(0 , "foo #bar", "foo", "#bar"), + TEST_PASS(K , "foo #bar", "foo"), + TEST_PASS( N, "\\", "\\"), + TEST_FAIL(0 , "\"foo", missing_quote), + NULL +#undef N +#undef C +#undef K +#undef TEST_FAIL +#undef TEST_PASS +}; + +static char ** +test_run(const struct test_case *tc, int *ret) +{ + const char *exp, *act; + char **argv, *tmp; + int argc, i; + + i = strlen(tc->str); + if (i == 0) { + argv = VAV_Parse(tc->str, &argc, tc->flag); + } else { + tmp = malloc(i); /* sanitizer-friendly */ + AN(tmp); + memcpy(tmp, tc->str, i); + argv = VAV_ParseTxt(tmp, tmp + i, &argc, tc->flag); + free(tmp); + } + AN(argv); + + if (tc->argv[0] != argv[0]) { + exp = tc->argv[0] != NULL ? tc->argv[0] : "success"; + act = argv[0] != NULL ? argv[0] : "success"; + printf( + "ERROR: Parsing string <%s> with flags %x, " + "expected <%s> got <%s>.\n", + tc->str, tc->flag, exp, act); + *ret = 1; + return (argv); + } + + for (i = 1; i < argc && tc->argv[i] != NULL && argv[i] != NULL; i++) { + if (!strcmp(tc->argv[i], argv[i])) + continue; + printf( + "ERROR: Parsing string <%s> with flags %x, " + "expected <%s> for argv[%d] got <%s>.\n", + tc->str, tc->flag, tc->argv[i], i, argv[i]); + *ret = 1; + return (argv); + } + + if (tc->argv[i] != NULL || argv[i] != NULL) { + act = i < argc ? "less" : "more"; + printf( + "ERROR: Parsing string <%s> with flags %x, " + "got %s arguments (%d) than expected.\n", + tc->str, tc->flag, act, argc); + *ret = 1; + return (argv); + } + + exp = tc->argv[0] == NULL ? "PASS" : "FAIL"; + printf("%s: <%s> with flags %x as expected.\n", exp, tc->str, tc->flag); + return (argv); +} + +int +main(int argc, char **argv) +{ + const struct test_case **tc; + int ret = 0; + + (void)argc; + (void)argv; + + for (tc = tests; ret == 0 && *tc != NULL; tc++) { + argv = test_run(*tc, &ret); + VAV_Free(argv); + } + + return (0); +} +#endif /* TEST_DRIVER */ From dridi.boukelmoune at gmail.com Fri Jul 2 16:22:04 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 2 Jul 2021 16:22:04 +0000 (UTC) Subject: [master] ffe895128 vav: Bound check after moving forward Message-ID: <20210702162204.996094C91@lists.varnish-cache.org> commit ffe8951286d170a6b75903700c454df8262c834f Author: Dridi Boukelmoune Date: Fri Jul 2 11:23:29 2021 +0200 vav: Bound check after moving forward There's no longer the guarantee of a null character at the end of a VAV string. diff --git a/lib/libvarnish/vav.c b/lib/libvarnish/vav.c index c3e91f2dd..53de58fa5 100644 --- a/lib/libvarnish/vav.c +++ b/lib/libvarnish/vav.c @@ -190,11 +190,11 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) } if (*b == '"' && !(flag & ARGV_NOESC)) break; + b++; if (b >= e) { argv[0] = err_missing_quote; return (argv); } - b++; } if (nargv + 1 >= largv) { argv = realloc(argv, sizeof (*argv) * (largv += largv)); From dridi.boukelmoune at gmail.com Fri Jul 2 16:22:04 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 2 Jul 2021 16:22:04 +0000 (UTC) Subject: [master] 2fd5e9228 vav: Missing bound check Message-ID: <20210702162204.B75644C96@lists.varnish-cache.org> commit 2fd5e9228c7224a5d954b1f1f56b29f55ab868c0 Author: Dridi Boukelmoune Date: Fri Jul 2 18:09:23 2021 +0200 vav: Missing bound check diff --git a/lib/libvarnish/vav.c b/lib/libvarnish/vav.c index 53de58fa5..0b76301b7 100644 --- a/lib/libvarnish/vav.c +++ b/lib/libvarnish/vav.c @@ -172,6 +172,10 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) } while (1) { if (*b == '\\' && !(flag & ARGV_NOESC)) { + if (b + 1 >= e) { + argv[0] = err_invalid_backslash; + return (argv); + } i = VAV_BackSlash(b, NULL); if (i == 0) { argv[0] = err_invalid_backslash; @@ -333,6 +337,7 @@ static const struct test_case *tests[] = { TEST_PASS(0 , "foo #bar", "foo", "#bar"), TEST_PASS(K , "foo #bar", "foo"), TEST_PASS( N, "\\", "\\"), + TEST_FAIL(0 , "\\", invalid_backslash), TEST_FAIL(0 , "\"foo", missing_quote), NULL #undef N From dridi.boukelmoune at gmail.com Fri Jul 2 16:22:04 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 2 Jul 2021 16:22:04 +0000 (UTC) Subject: [master] e2286dfbb vav: Fold main loop condition Message-ID: <20210702162204.CF2834CA4@lists.varnish-cache.org> commit e2286dfbba7214cf82faee220256e0975f127541 Author: Dridi Boukelmoune Date: Fri Jul 2 11:35:13 2021 +0200 vav: Fold main loop condition diff --git a/lib/libvarnish/vav.c b/lib/libvarnish/vav.c index 0b76301b7..be9db713b 100644 --- a/lib/libvarnish/vav.c +++ b/lib/libvarnish/vav.c @@ -154,9 +154,7 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) if (argv == NULL) return (NULL); - for (;;) { - if (b >= e) - break; + while (b < e) { if (isspace(*b)) { b++; continue; From dridi.boukelmoune at gmail.com Fri Jul 2 17:33:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 2 Jul 2021 17:33:05 +0000 (UTC) Subject: [master] c7fb55b22 vav: Defer missing quote check Message-ID: <20210702173305.7BB477574@lists.varnish-cache.org> commit c7fb55b22827e05e028bca6050a5839d875afcc3 Author: Dridi Boukelmoune Date: Fri Jul 2 19:29:54 2021 +0200 vav: Defer missing quote check Otherwise we can still overflow in the absence of a null terminator. diff --git a/lib/libvarnish/vav.c b/lib/libvarnish/vav.c index be9db713b..a25ed58fc 100644 --- a/lib/libvarnish/vav.c +++ b/lib/libvarnish/vav.c @@ -168,7 +168,7 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) p = b; quote = 0; } - while (1) { + while (b < e) { if (*b == '\\' && !(flag & ARGV_NOESC)) { if (b + 1 >= e) { argv[0] = err_invalid_backslash; @@ -193,10 +193,10 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) if (*b == '"' && !(flag & ARGV_NOESC)) break; b++; - if (b >= e) { - argv[0] = err_missing_quote; - return (argv); - } + } + if (b >= e && quote) { + argv[0] = err_missing_quote; + return (argv); } if (nargv + 1 >= largv) { argv = realloc(argv, sizeof (*argv) * (largv += largv)); From dridi.boukelmoune at gmail.com Fri Jul 2 17:33:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 2 Jul 2021 17:33:05 +0000 (UTC) Subject: [master] d3df1f640 vav: Treat a trailing comma as a separator Message-ID: <20210702173305.9739F7577@lists.varnish-cache.org> commit d3df1f64014bb7d6a8898a0b9f9d84e3f4dc11fb Author: Dridi Boukelmoune Date: Fri Jul 2 11:36:38 2021 +0200 vav: Treat a trailing comma as a separator Let's consider the following VAV strings: "foo bar baz" "foo,bar,baz" " foo bar baz " " foo,bar,baz " " foo bar baz " They are all equivalent because consecutive spaces are considered to form a single separator. However, consecutive commas aren't: "foo,bar,baz" "foo,,bar,,baz" In the example above the first string has 3 arguments while the second has 5 of them. This behavior was however inconsistent with trailing commas: "foo,bar,baz" "foo,bar,baz," "foo,bar,baz,," When it comes to trailing commas the first two strings above would contain 3 arguments, and the last string would contain 4 arguments. With this change, they respectively contain 3, 4 and 5 arguments. diff --git a/lib/libvarnish/vav.c b/lib/libvarnish/vav.c index a25ed58fc..8679b087c 100644 --- a/lib/libvarnish/vav.c +++ b/lib/libvarnish/vav.c @@ -143,11 +143,12 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) char **argv; const char *p; int nargv, largv; - int i, quote; + int i, quote, more; AN(b); if (e == NULL) e = strchr(b, '\0'); + more = 0; nargv = 1; largv = 16; argv = calloc(largv, sizeof *argv); @@ -161,6 +162,7 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) } if ((flag & ARGV_COMMENT) && *b == '#') break; + more = 0; if (*b == '"' && !(flag & ARGV_NOESC)) { p = ++b; quote = 1; @@ -185,8 +187,10 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) if (!quote) { if (b >= e || isspace(*b)) break; - if ((flag & ARGV_COMMA) && *b == ',') + if ((flag & ARGV_COMMA) && *b == ',') { + more = 1; break; + } b++; continue; } @@ -198,7 +202,8 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) argv[0] = err_missing_quote; return (argv); } - if (nargv + 1 >= largv) { + /* Ensure 1 slot for the new argument plus 1 more */ + if (nargv + 2 >= largv) { argv = realloc(argv, sizeof (*argv) * (largv += largv)); assert(argv != NULL); } @@ -214,6 +219,12 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) if (b < e) b++; } + if (more) { + AN(flag & ARGV_COMMA); + argv[nargv] = strdup(""); + assert(argv[nargv] != NULL); + nargv++; + } argv[nargv] = NULL; if (argc != NULL) *argc = nargv; From dridi.boukelmoune at gmail.com Fri Jul 2 17:36:04 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 2 Jul 2021 17:36:04 +0000 (UTC) Subject: [master] 17d773c38 Revert "vav: Treat a trailing comma as a separator" Message-ID: <20210702173604.B753C79A3@lists.varnish-cache.org> commit 17d773c384b70b76d7904bfd22dc4cd0f0756623 Author: Dridi Boukelmoune Date: Fri Jul 2 19:34:19 2021 +0200 Revert "vav: Treat a trailing comma as a separator" This reverts commit d3df1f64014bb7d6a8898a0b9f9d84e3f4dc11fb. I didn't mean to push it, it predates VAV's test driver. diff --git a/lib/libvarnish/vav.c b/lib/libvarnish/vav.c index 8679b087c..a25ed58fc 100644 --- a/lib/libvarnish/vav.c +++ b/lib/libvarnish/vav.c @@ -143,12 +143,11 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) char **argv; const char *p; int nargv, largv; - int i, quote, more; + int i, quote; AN(b); if (e == NULL) e = strchr(b, '\0'); - more = 0; nargv = 1; largv = 16; argv = calloc(largv, sizeof *argv); @@ -162,7 +161,6 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) } if ((flag & ARGV_COMMENT) && *b == '#') break; - more = 0; if (*b == '"' && !(flag & ARGV_NOESC)) { p = ++b; quote = 1; @@ -187,10 +185,8 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) if (!quote) { if (b >= e || isspace(*b)) break; - if ((flag & ARGV_COMMA) && *b == ',') { - more = 1; + if ((flag & ARGV_COMMA) && *b == ',') break; - } b++; continue; } @@ -202,8 +198,7 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) argv[0] = err_missing_quote; return (argv); } - /* Ensure 1 slot for the new argument plus 1 more */ - if (nargv + 2 >= largv) { + if (nargv + 1 >= largv) { argv = realloc(argv, sizeof (*argv) * (largv += largv)); assert(argv != NULL); } @@ -219,12 +214,6 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) if (b < e) b++; } - if (more) { - AN(flag & ARGV_COMMA); - argv[nargv] = strdup(""); - assert(argv[nargv] != NULL); - nargv++; - } argv[nargv] = NULL; if (argc != NULL) *argc = nargv; From dridi.boukelmoune at gmail.com Mon Jul 5 08:43:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Jul 2021 08:43:05 +0000 (UTC) Subject: [master] ec2c30ac0 git: Ignore vav_test Message-ID: <20210705084305.C350AABA97@lists.varnish-cache.org> commit ec2c30ac0e534f1f1c59be0b72864bffcfd98956 Author: Dridi Boukelmoune Date: Mon Jul 5 10:42:04 2021 +0200 git: Ignore vav_test diff --git a/.gitignore b/.gitignore index f7aae9ebd..183ecfddd 100644 --- a/.gitignore +++ b/.gitignore @@ -117,6 +117,7 @@ cscope.*out /include/_vrt_test /include/vrt_test /include/vbm_test +/lib/libvarnish/vav_test /lib/libvarnish/vbh_test /lib/libvarnish/vjsn_test /lib/libvarnish/vnum_c_test From dridi.boukelmoune at gmail.com Mon Jul 5 15:47:04 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Jul 2021 15:47:04 +0000 (UTC) Subject: [master] 97e6818ab vav: Prevent backslash-related overflows Message-ID: <20210705154704.BFA8975E3@lists.varnish-cache.org> commit 97e6818ab6562bb7008d370d683bf2a2a0d1a49a Author: Dridi Boukelmoune Date: Mon Jul 5 07:50:30 2021 +0200 vav: Prevent backslash-related overflows diff --git a/lib/libvarnish/vav.c b/lib/libvarnish/vav.c index a25ed58fc..7dc269aac 100644 --- a/lib/libvarnish/vav.c +++ b/lib/libvarnish/vav.c @@ -52,13 +52,21 @@ #include "vas.h" #include "vav.h" -int -VAV_BackSlash(const char *s, char *res) +static int +vav_backslash_txt(const char *s, const char *e, char *res) { - int r; + int r, l; char c; unsigned u; + AN(s); + if (e == NULL) + e = strchr(s, '\0'); + + l = pdiff(s, e); + if (l < 2) + return (0); + assert(*s == '\\'); r = c = 0; switch (s[1]) { @@ -84,7 +92,7 @@ VAV_BackSlash(const char *s, char *res) break; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': - for (r = 1; r < 4; r++) { + for (r = 1; r < 4 && r < l; r++) { if (!isdigit(s[r])) break; if (s[r] - '0' > 7) @@ -94,7 +102,7 @@ VAV_BackSlash(const char *s, char *res) } break; case 'x': - if (1 == sscanf(s + 1, "x%02x", &u)) { + if (l >= 4 && sscanf(s + 1, "x%02x", &u) == 1) { AZ(u & ~0xff); c = u; /*lint !e734 loss of precision */ r = 4; @@ -108,6 +116,13 @@ VAV_BackSlash(const char *s, char *res) return (r); } +int +VAV_BackSlash(const char *s, char *res) +{ + + return (vav_backslash_txt(s, NULL, res)); +} + char * VAV_BackSlashDecode(const char *s, const char *e) { @@ -126,7 +141,12 @@ VAV_BackSlashDecode(const char *s, const char *e) *r++ = *q++; continue; } - i = VAV_BackSlash(q, r); + i = vav_backslash_txt(q, e, r); + if (i == 0) { + free(p); + errno = EINVAL; + return (NULL); + } q += i; r++; } @@ -170,11 +190,7 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) } while (b < e) { if (*b == '\\' && !(flag & ARGV_NOESC)) { - if (b + 1 >= e) { - argv[0] = err_invalid_backslash; - return (argv); - } - i = VAV_BackSlash(b, NULL); + i = vav_backslash_txt(b, e, NULL); if (i == 0) { argv[0] = err_invalid_backslash; return (argv); @@ -183,7 +199,7 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) continue; } if (!quote) { - if (b >= e || isspace(*b)) + if (isspace(*b)) break; if ((flag & ARGV_COMMA) && *b == ',') break; @@ -207,10 +223,11 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) assert(argv[nargv] != NULL); memcpy(argv[nargv], p, b - p); argv[nargv][b - p] = '\0'; - nargv++; } else { - argv[nargv++] = VAV_BackSlashDecode(p, b); + argv[nargv] = VAV_BackSlashDecode(p, b); + assert(argv[nargv] != NULL); } + nargv++; if (b < e) b++; } @@ -336,6 +353,9 @@ static const struct test_case *tests[] = { TEST_PASS(K , "foo #bar", "foo"), TEST_PASS( N, "\\", "\\"), TEST_FAIL(0 , "\\", invalid_backslash), + TEST_FAIL(0 , "\\x", invalid_backslash), + TEST_FAIL(0 , "\\x2", invalid_backslash), + TEST_PASS(0 , "\\x20", " "), TEST_FAIL(0 , "\"foo", missing_quote), NULL #undef N From dridi.boukelmoune at gmail.com Mon Jul 5 15:47:04 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Jul 2021 15:47:04 +0000 (UTC) Subject: [master] 4f1fd4128 vav: Apparently we can't trust sscanf(3) Message-ID: <20210705154704.D50CA75E6@lists.varnish-cache.org> commit 4f1fd4128c5e5487c1ef103c25ba48b7a15a7b1f Author: Dridi Boukelmoune Date: Mon Jul 5 07:52:22 2021 +0200 vav: Apparently we can't trust sscanf(3) At least not on my system, where "x%02x" doesn't strictly require 2 hexadecimal digits. diff --git a/lib/libvarnish/vav.c b/lib/libvarnish/vav.c index 7dc269aac..6863b9dfb 100644 --- a/lib/libvarnish/vav.c +++ b/lib/libvarnish/vav.c @@ -102,7 +102,8 @@ vav_backslash_txt(const char *s, const char *e, char *res) } break; case 'x': - if (l >= 4 && sscanf(s + 1, "x%02x", &u) == 1) { + if (l >= 4 && isxdigit(s[2]) && isxdigit(s[3]) && + sscanf(s + 1, "x%02x", &u) == 1) { AZ(u & ~0xff); c = u; /*lint !e734 loss of precision */ r = 4; @@ -355,6 +356,7 @@ static const struct test_case *tests[] = { TEST_FAIL(0 , "\\", invalid_backslash), TEST_FAIL(0 , "\\x", invalid_backslash), TEST_FAIL(0 , "\\x2", invalid_backslash), + TEST_FAIL(0 , "\\x2O", invalid_backslash), TEST_PASS(0 , "\\x20", " "), TEST_FAIL(0 , "\"foo", missing_quote), NULL From dridi.boukelmoune at gmail.com Mon Jul 5 15:47:04 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Jul 2021 15:47:04 +0000 (UTC) Subject: [master] a2ab44dec vav: Tighten arguments parsing Message-ID: <20210705154704.EDA9675EB@lists.varnish-cache.org> commit a2ab44decc08bee407b232d3ae97b9b281b56c60 Author: Dridi Boukelmoune Date: Mon Jul 5 08:36:41 2021 +0200 vav: Tighten arguments parsing Commas unlike spaces are hard separators, so a trailing comma leads to a last empty parameter. A comma may appear after an argument's "trailing" spaces and should not result in an additional parameter. In we should expect two fields, not three. Comments are only treated as such at arguments boundaries: parses one field and parses one field , taking the shell word splitting as the model, cementing what was the existing VAV behavior in the first place. Unlike the shell, we don't expect quotes to start in the middle of a token so is invalid unless escaping was disabled. Fields that are quoted need a separator: <"foo""bar"> is therefore invalid unless escaping was disabled. diff --git a/lib/libvarnish/vav.c b/lib/libvarnish/vav.c index 6863b9dfb..ffebc3f49 100644 --- a/lib/libvarnish/vav.c +++ b/lib/libvarnish/vav.c @@ -156,19 +156,23 @@ VAV_BackSlashDecode(const char *s, const char *e) } static char err_invalid_backslash[] = "Invalid backslash sequence"; +static char err_invalid_quote[] = "Invalid '\"'"; static char err_missing_quote[] = "Missing '\"'"; +static char err_missing_separator[] = "Missing separator between arguments"; char ** VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) { char **argv; - const char *p; + const char *p, *sep; int nargv, largv; int i, quote; AN(b); if (e == NULL) e = strchr(b, '\0'); + sep = NULL; + quote = 0; nargv = 1; largv = 16; argv = calloc(largv, sizeof *argv); @@ -180,6 +184,17 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) b++; continue; } + if (sep != NULL && isspace(*sep) && + *b == ',' && (flag & ARGV_COMMA)) { + sep = NULL; + b++; + continue; + } + if (sep != NULL && *sep == '"' && *b == '"') { + argv[0] = err_missing_separator; + return (argv); + } + sep = NULL; if ((flag & ARGV_COMMENT) && *b == '#') break; if (*b == '"' && !(flag & ARGV_NOESC)) { @@ -200,22 +215,34 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) continue; } if (!quote) { - if (isspace(*b)) + if (isspace(*b)) { + sep = b; break; - if ((flag & ARGV_COMMA) && *b == ',') + } + if ((flag & ARGV_COMMA) && *b == ',') { + sep = b; break; + } + if (!(flag & ARGV_NOESC) && *b == '"') { + argv[0] = err_invalid_quote; + return (argv); + } b++; continue; } - if (*b == '"' && !(flag & ARGV_NOESC)) + if (*b == '"' && !(flag & ARGV_NOESC)) { + sep = b; + quote = 0; break; + } b++; } - if (b >= e && quote) { + if (sep == NULL && quote) { argv[0] = err_missing_quote; return (argv); } - if (nargv + 1 >= largv) { + /* Ensure slots for 1 new arg plus 1 trailing arg */ + if (nargv + 2 >= largv) { argv = realloc(argv, sizeof (*argv) * (largv += largv)); assert(argv != NULL); } @@ -232,6 +259,11 @@ VAV_ParseTxt(const char *b, const char *e, int *argc, int flag) if (b < e) b++; } + if (sep != NULL && *sep == ',') { + argv[nargv] = strdup(""); + assert(argv[nargv] != NULL); + nargv++; + } argv[nargv] = NULL; if (argc != NULL) *argc = nargv; @@ -349,9 +381,14 @@ static const struct test_case *tests[] = { TEST_PASS( C , "foo bar", "foo", "bar"), TEST_PASS( C , "foo,bar", "foo", "bar"), TEST_PASS(0 , " foo bar ", "foo", "bar"), + TEST_PASS( C , " foo , bar ", "foo", "bar"), + TEST_PASS( C , "foo bar ", "foo", "bar"), + TEST_PASS( C , "foo,bar,", "foo", "bar", ""), TEST_PASS(0 , "foo \"bar baz\"", "foo", "bar baz"), TEST_PASS(0 , "foo #bar", "foo", "#bar"), TEST_PASS(K , "foo #bar", "foo"), + TEST_PASS(0 , "foo#bar", "foo#bar"), + TEST_PASS(K , "foo#bar", "foo#bar"), TEST_PASS( N, "\\", "\\"), TEST_FAIL(0 , "\\", invalid_backslash), TEST_FAIL(0 , "\\x", invalid_backslash), @@ -359,6 +396,11 @@ static const struct test_case *tests[] = { TEST_FAIL(0 , "\\x2O", invalid_backslash), TEST_PASS(0 , "\\x20", " "), TEST_FAIL(0 , "\"foo", missing_quote), + TEST_PASS( N, "foo\"bar", "foo\"bar"), + TEST_FAIL(0 , "foo\"bar", invalid_quote), + TEST_FAIL(0 , "foo\"bar", invalid_quote), + TEST_PASS( N, "\"foo\"\"bar\"", "\"foo\"\"bar\""), + TEST_FAIL(0 , "\"foo\"\"bar\"", missing_separator), NULL #undef N #undef C @@ -397,6 +439,9 @@ test_run(const struct test_case *tc, int *ret) return (argv); } + if (tc->argv[0] != NULL) + return (argv); + for (i = 1; i < argc && tc->argv[i] != NULL && argv[i] != NULL; i++) { if (!strcmp(tc->argv[i], argv[i])) continue; From dridi.boukelmoune at gmail.com Mon Jul 5 15:48:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Jul 2021 15:48:05 +0000 (UTC) Subject: [master] eb71eae78 varnishtest: Replace macro_get() with macro_cat() Message-ID: <20210705154805.A840B7AC9@lists.varnish-cache.org> commit eb71eae78cf71412913b66517ab644630e4c21ec Author: Dridi Boukelmoune Date: Thu Jul 1 08:37:15 2021 +0200 varnishtest: Replace macro_get() with macro_cat() The latter operates on a VSB, which is always what call sites are doing anyway. It also takes the responsibility of ignoring unknown macros, in preparation for more responsibilities that will also require the ability to fail a test case. diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 7319d4d43..7a9eb6144 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -204,8 +204,8 @@ macro_undef(struct vtclog *vl, const char *instance, const char *name) AZ(pthread_mutex_unlock(¯o_mtx)); } -char * -macro_get(const char *b, const char *e) +void +macro_cat(struct vtclog *vl, struct vsb *vsb, const char *b, const char *e) { struct macro *m; int l; @@ -222,7 +222,9 @@ macro_get(const char *b, const char *e) retval = malloc(VTIM_FORMAT_SIZE); AN(retval); VTIM_format(t, retval); - return (retval); + VSB_cat(vsb, retval); + free(retval); + return; } AZ(pthread_mutex_lock(¯o_mtx)); @@ -232,9 +234,19 @@ macro_get(const char *b, const char *e) break; } if (m != NULL) - retval = strdup(m->val); + REPLACE(retval, m->val); AZ(pthread_mutex_unlock(¯o_mtx)); - return (retval); + + if (retval == NULL) { + if (!ign_unknown_macro) + vtc_fatal(vl, "Macro ${%.*s} not found", + (int)(e - b), b); + VSB_printf(vsb, "${%.*s}", (int)(e - b), b); + return; + } + + VSB_cat(vsb, retval); + free(retval); } struct vsb * @@ -259,7 +271,6 @@ macro_expand(struct vtclog *vl, const char *text) { struct vsb *vsb; const char *p, *q; - char *m; vsb = VSB_new_auto(); AN(vsb); @@ -279,19 +290,7 @@ macro_expand(struct vtclog *vl, const char *text) assert(p[1] == '{'); assert(q[0] == '}'); p += 2; - m = macro_get(p, q); - if (m == NULL) { - if (!ign_unknown_macro) { - VSB_destroy(&vsb); - vtc_fatal(vl, "Macro ${%.*s} not found", - (int)(q - p), p); - NEEDLESS(return (NULL)); - } - VSB_printf(vsb, "${%.*s}", (int)(q - p), p); - } else { - VSB_printf(vsb, "%s", m); - free(m); - } + macro_cat(vl, vsb, p, q); text = q + 1; } AZ(VSB_finish(vsb)); @@ -513,7 +512,6 @@ exec_file(const char *fn, const char *script, const char *tmpdir, FILE *f; struct vsb *vsb; const char *p; - char *q; (void)signal(SIGPIPE, SIG_IGN); @@ -537,12 +535,8 @@ exec_file(const char *fn, const char *script, const char *tmpdir, vsb = VSB_new_auto(); AN(vsb); - if (*fn != '/') { - q = macro_get("pwd", NULL); - AN(q); - VSB_cat(vsb, q); - free(q); - } + if (*fn != '/') + macro_cat(vltop, vsb, "pwd", NULL); p = strrchr(fn, '/'); if (p != NULL) { VSB_putc(vsb, '/'); diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index 5380f74cd..6e6978018 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -134,7 +134,7 @@ void macro_undef(struct vtclog *vl, const char *instance, const char *name); void macro_def(struct vtclog *vl, const char *instance, const char *name, const char *fmt, ...) v_printflike_(4, 5); -char *macro_get(const char *, const char *); +void macro_cat(struct vtclog *, struct vsb *, const char *, const char *); struct vsb *macro_expand(struct vtclog *vl, const char *text); struct vsb *macro_expandf(struct vtclog *vl, const char *, ...) v_printflike_(2, 3); diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index dc286066a..af3f31e43 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -774,12 +774,10 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, long bodylen = 0; char *b, *c; char *nullbody; - char *m; ssize_t len; int nolen = 0; int l; - (void)vl; nullbody = body; for (; *av != NULL; av++) { @@ -860,10 +858,9 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, break; } if (!nohost) { - m = macro_get("localhost", NULL); - AN(m); - VSB_printf(hp->vsb, "Host: %s%s", m, nl); - free(m); + VSB_cat(hp->vsb, "Host: "); + macro_cat(vl, hp->vsb, "localhost", NULL); + VSB_cat(hp->vsb, nl); } if (body != NULL && !nolen) VSB_printf(hp->vsb, "Content-Length: %ld%s", bodylen, nl); From dridi.boukelmoune at gmail.com Mon Jul 5 15:48:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Jul 2021 15:48:05 +0000 (UTC) Subject: [master] 12cd341e3 varnishtest: Allow macros to be backed by functions Message-ID: <20210705154805.C3BFE7ACC@lists.varnish-cache.org> commit 12cd341e3cd8d588af0811106b3ba8e91b5c6b17 Author: Dridi Boukelmoune Date: Thu Jun 17 15:49:45 2021 +0200 varnishtest: Allow macros to be backed by functions Instead of having a mere value, these would be able to compute a macro expansion. We parse the contents inside the ${...} delimiters as a VAV, but there can't be (yet?) nested curly {braces}, even quoted. The first argument inside the delimiters is the macro name, and other VAV arguments are treated as arguments to the macro's function. For example ${foo,bar,baz} would call the a macro "foo"'s function with arguments "bar" and "baz". Simple macros don't take arguments and work as usual. diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 7a9eb6144..2eb2e5d43 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -80,6 +80,7 @@ struct macro { VTAILQ_ENTRY(macro) list; char *name; char *val; + macro_f *func; }; static VTAILQ_HEAD(,macro) macro_list = VTAILQ_HEAD_INITIALIZER(macro_list); @@ -99,7 +100,7 @@ static const struct cmds top_cmds[] = { /**********************************************************************/ static struct macro * -macro_def_int(const char *name, const char *fmt, va_list ap) +macro_def_int(const char *name, macro_f *func, const char *fmt, va_list ap) { struct macro *m; char buf[512]; @@ -115,9 +116,15 @@ macro_def_int(const char *name, const char *fmt, va_list ap) VTAILQ_INSERT_TAIL(¯o_list, m, list); } AN(m); - vbprintf(buf, fmt, ap); - REPLACE(m->val, buf); - AN(m->val); + if (func != NULL) { + AZ(fmt); + m->func = func; + } else { + AN(fmt); + vbprintf(buf, fmt, ap); + REPLACE(m->val, buf); + AN(m->val); + } return (m); } @@ -128,12 +135,12 @@ macro_def_int(const char *name, const char *fmt, va_list ap) */ void -extmacro_def(const char *name, const char *fmt, ...) +extmacro_def(const char *name, macro_f *func, const char *fmt, ...) { va_list ap; va_start(ap, fmt); - (void)macro_def_int(name, fmt, ap); + (void)macro_def_int(name, func, fmt, ap); va_end(ap); } @@ -149,8 +156,13 @@ init_macro(void) struct macro *m; /* Dump the extmacros for completeness */ - VTAILQ_FOREACH(m, ¯o_list, list) - vtc_log(vltop, 4, "extmacro def %s=%s", m->name, m->val); + VTAILQ_FOREACH(m, ¯o_list, list) { + if (m->val != NULL) + vtc_log(vltop, 4, + "extmacro def %s=%s", m->name, m->val); + else + vtc_log(vltop, 4, "extmacro def %s(...)", m->name); + } AZ(pthread_mutex_init(¯o_mtx, NULL)); } @@ -172,7 +184,7 @@ macro_def(struct vtclog *vl, const char *instance, const char *name, AZ(pthread_mutex_lock(¯o_mtx)); va_start(ap, fmt); - m = macro_def_int(name, fmt, ap); + m = macro_def_int(name, NULL, fmt, ap); va_end(ap); vtc_log(vl, 4, "macro def %s=%s", name, m->val); AZ(pthread_mutex_unlock(¯o_mtx)); @@ -208,35 +220,61 @@ void macro_cat(struct vtclog *vl, struct vsb *vsb, const char *b, const char *e) { struct macro *m; - int l; - char *retval = NULL; + char **argv, *retval = NULL; + const char *err = NULL; + int argc; AN(b); if (e == NULL) e = strchr(b, '\0'); AN(e); - l = e - b; - if (l == 4 && !memcmp(b, "date", l)) { + argv = VAV_ParseTxt(b, e, &argc, ARGV_COMMA); + AN(argv); + + if (*argv != NULL) + vtc_fatal(vl, "Macro ${%.*s} parsing failed: %s", + (int)(e - b), b, *argv); + + assert(argc >= 2); + + if (!strcmp(argv[1], "date")) { double t = VTIM_real(); retval = malloc(VTIM_FORMAT_SIZE); AN(retval); VTIM_format(t, retval); VSB_cat(vsb, retval); free(retval); + VAV_Free(argv); return; } AZ(pthread_mutex_lock(¯o_mtx)); VTAILQ_FOREACH(m, ¯o_list, list) { CHECK_OBJ_NOTNULL(m, MACRO_MAGIC); - if (!strncmp(b, m->name, l) && m->name[l] == '\0') + if (!strcmp(argv[1], m->name)) break; } - if (m != NULL) - REPLACE(retval, m->val); + if (m != NULL) { + if (m->func != NULL) { + AZ(m->val); + retval = m->func(argc, argv, &err); + if (err == NULL) + AN(retval); + } else { + AN(m->val); + if (argc == 2) + REPLACE(retval, m->val); + else + err = "macro does not take arguments"; + } + } AZ(pthread_mutex_unlock(¯o_mtx)); + if (err != NULL) + vtc_fatal(vl, "Macro ${%.*s} failed: %s", + (int)(e - b), b, err); + if (retval == NULL) { if (!ign_unknown_macro) vtc_fatal(vl, "Macro ${%.*s} not found", @@ -247,6 +285,7 @@ macro_cat(struct vtclog *vl, struct vsb *vsb, const char *b, const char *e) VSB_cat(vsb, retval); free(retval); + VAV_Free(argv); } struct vsb * diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index 6e6978018..55d28b351 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -139,8 +139,9 @@ struct vsb *macro_expand(struct vtclog *vl, const char *text); struct vsb *macro_expandf(struct vtclog *vl, const char *, ...) v_printflike_(2, 3); -void extmacro_def(const char *name, const char *fmt, ...) - v_printflike_(2, 3); +typedef char* macro_f(int, char *const *, const char **); +void extmacro_def(const char *name, macro_f *func, const char *fmt, ...) + v_printflike_(3, 4); struct http; void cmd_stream(CMD_ARGS); diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 309bd876e..141dac1e5 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -167,7 +167,7 @@ parse_D_opt(char *arg) if (!q) return (0); *q++ = '\0'; - extmacro_def(p, "%s", q); + extmacro_def(p, NULL, "%s", q); return (1); } @@ -544,7 +544,7 @@ i_mode(void) } AN(topbuild); - extmacro_def("topbuild", "%s", topbuild); + extmacro_def("topbuild", NULL, "%s", topbuild); /* * Build $PATH which can find all programs in the build tree @@ -605,7 +605,7 @@ ip_magic(void) } assert(bad_backend_fd >= 0); VTCP_myname(bad_backend_fd, abuf, sizeof abuf, pbuf, sizeof(pbuf)); - extmacro_def("localhost", "%s", abuf); + extmacro_def("localhost", NULL, "%s", abuf); s = strdup(abuf); AN(s); @@ -620,9 +620,9 @@ ip_magic(void) /* Expose a backend that is forever down. */ if (VSA_Get_Proto(sa) == AF_INET) - extmacro_def("bad_backend", "%s:%s", abuf, pbuf); + extmacro_def("bad_backend", NULL, "%s:%s", abuf, pbuf); else - extmacro_def("bad_backend", "[%s]:%s", abuf, pbuf); + extmacro_def("bad_backend", NULL, "[%s]:%s", abuf, pbuf); /* our default bind/listen address */ if (VSA_Get_Proto(sa) == AF_INET) @@ -631,7 +631,7 @@ ip_magic(void) bprintf(abuf, "[%s]:0", s); free(s); - extmacro_def("listen_addr", "%s", abuf); + extmacro_def("listen_addr", NULL, "%s", abuf); default_listen_addr = strdup(abuf); AN(default_listen_addr); free(sa); @@ -644,7 +644,7 @@ ip_magic(void) * check your /proc/sys/net/ipv4/ip_nonlocal_bind setting. */ - extmacro_def("bad_ip", "%s", "192.0.2.255"); + extmacro_def("bad_ip", NULL, "%s", "192.0.2.255"); } /********************************************************************** @@ -725,7 +725,7 @@ main(int argc, char * const *argv) tmppath = strdup("/tmp"); cwd = getcwd(buf, sizeof buf); - extmacro_def("pwd", "%s", cwd); + extmacro_def("pwd", NULL, "%s", cwd); vmod_path = NULL; From dridi.boukelmoune at gmail.com Mon Jul 5 15:48:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Jul 2021 15:48:05 +0000 (UTC) Subject: [master] 1824ba01d varnishtest: Turn the ${date} macro into a function Message-ID: <20210705154805.E17557AD1@lists.varnish-cache.org> commit 1824ba01d8401c9548cf5e4d9b9ba5aa2ddda20c Author: Dridi Boukelmoune Date: Thu Jun 17 16:10:16 2021 +0200 varnishtest: Turn the ${date} macro into a function diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 2eb2e5d43..1a4988cbb 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -44,7 +44,6 @@ #include "vav.h" #include "vrnd.h" -#include "vtim.h" #define MAX_TOKENS 200 @@ -238,17 +237,6 @@ macro_cat(struct vtclog *vl, struct vsb *vsb, const char *b, const char *e) assert(argc >= 2); - if (!strcmp(argv[1], "date")) { - double t = VTIM_real(); - retval = malloc(VTIM_FORMAT_SIZE); - AN(retval); - VTIM_format(t, retval); - VSB_cat(vsb, retval); - free(retval); - VAV_Free(argv); - return; - } - AZ(pthread_mutex_lock(¯o_mtx)); VTAILQ_FOREACH(m, ¯o_list, list) { CHECK_OBJ_NOTNULL(m, MACRO_MAGIC); diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 141dac1e5..c646c6574 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -647,6 +647,32 @@ ip_magic(void) extmacro_def("bad_ip", NULL, "%s", "192.0.2.255"); } +/********************************************************************** + * Macros + */ + +static char * v_matchproto_(macro_f) +macro_func_date(int argc, char *const *argv, const char **err) +{ + double t; + char *s; + + assert(argc >= 2); + AN(argv); + AN(err); + + if (argc > 2) { + *err = "macro does not take arguments"; + return (NULL); + } + + t = VTIM_real(); + s = malloc(VTIM_FORMAT_SIZE); + AN(s); + VTIM_format(t, s); + return (s); +} + /********************************************************************** * Main */ @@ -727,6 +753,8 @@ main(int argc, char * const *argv) cwd = getcwd(buf, sizeof buf); extmacro_def("pwd", NULL, "%s", cwd); + extmacro_def("date", macro_func_date, NULL); + vmod_path = NULL; params_vsb = VSB_new_auto(); From dridi.boukelmoune at gmail.com Mon Jul 5 15:48:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Jul 2021 15:48:06 +0000 (UTC) Subject: [master] 5c248efcd varnishtest: New ${string, [, ...]} macro Message-ID: <20210705154806.2749B7AD6@lists.varnish-cache.org> commit 5c248efcdf3281c06efb642cf6dcc86896fbc6b6 Author: Dridi Boukelmoune Date: Thu Jun 17 17:08:11 2021 +0200 varnishtest: New ${string,[,...]} macro Its first action ${string,repeat,,} helps simplify many unwieldy test cases that will hopefully be easier to edit from now on. diff --git a/bin/varnishtest/tests/c00027.vtc b/bin/varnishtest/tests/c00027.vtc index 5c22daf1d..246aef15f 100644 --- a/bin/varnishtest/tests/c00027.vtc +++ b/bin/varnishtest/tests/c00027.vtc @@ -3,19 +3,19 @@ varnishtest "Test eviction" server s1 { rxreq expect req.url == "/1" - txresp -body "1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" + txresp -body "${string,repeat,1024,1}" rxreq expect req.url == "/2" - txresp -body "1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" + txresp -body "${string,repeat,1024,1}" rxreq expect req.url == "/3" - txresp -body "1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" + txresp -body "${string,repeat,1024,1}" rxreq expect req.url == "/4" - txresp -body "1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" + txresp -body "${string,repeat,1024,1}" rxreq expect req.url == "/5" - txresp -body "1111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" + txresp -body "${string,repeat,1024,1}" } -start varnish v1 -arg "-s default,1M" -vcl+backend { diff --git a/bin/varnishtest/tests/c00071.vtc b/bin/varnishtest/tests/c00071.vtc index 7c25657f4..b7f8d12fc 100644 --- a/bin/varnishtest/tests/c00071.vtc +++ b/bin/varnishtest/tests/c00071.vtc @@ -22,7 +22,7 @@ varnish v1 -arg "-p debug=+workspace" -vcl+backend { vtc.workspace_alloc(client, -10); } else if (req.url ~ "/baz") { - set resp.http.x-foo = regsub(req.url, "baz", "baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaz"); + set resp.http.x-foo = regsub(req.url, "baz", "b${string,repeat,91,a}z"); std.log("dummy"); std.syslog(8 + 7, "vtc: " + 1 + 2); } diff --git a/bin/varnishtest/tests/o00000.vtc b/bin/varnishtest/tests/o00000.vtc index 08459fbe4..c6691ba3c 100644 --- a/bin/varnishtest/tests/o00000.vtc +++ b/bin/varnishtest/tests/o00000.vtc @@ -199,7 +199,7 @@ varnish v1 -vsl_catchup # Try with appended request (See also: #1728) client c2 { - send "PROXY TCP6 1:f::3 5:a::8 1234 5678\r\nGET /3 HTTP/1.1\r\nHost: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n\r\n" + send "PROXY TCP6 1:f::3 5:a::8 1234 5678\r\nGET /3 HTTP/1.1\r\nHost: ${string,repeat,54,a}\r\n\r\n" rxresp expect resp.http.url == "/3" } -run @@ -217,7 +217,7 @@ varnish v1 -vsl_catchup # Malformed, too long (106) # NB: Should check VSL for proper disposal client c2 { - send "PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678 \r\n" + send "PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678 ${string,repeat,68," "}\r\n" expect_close } -run @@ -226,7 +226,7 @@ varnish v1 -vsl_catchup # Malformed, too long (107) # NB: Should check VSL for proper disposal client c2 { - send "PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678 \r\n" + send "PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678 ${string,repeat,69," "}\r\n" expect_close } -run @@ -235,7 +235,7 @@ varnish v1 -vsl_catchup # Malformed, too long (108) # NB: Should check VSL for proper disposal client c2 { - send "PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678 \r\n" + send "PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678 ${string,repeat,70," "}\r\n" expect_close } -run diff --git a/bin/varnishtest/tests/o00005.vtc b/bin/varnishtest/tests/o00005.vtc index f20595589..6a3c99446 100644 --- a/bin/varnishtest/tests/o00005.vtc +++ b/bin/varnishtest/tests/o00005.vtc @@ -293,25 +293,7 @@ client c1 { 01 bb 03 00 04 c5 1b 5b 2b 01 00 02 68 32 - 02 00 c8 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 61 61 61 61 61 61 61 61 61 - 61 61 + 02 00 c8 ${string,repeat,200,"61 "} 20 00 3d 01 00 00 00 00 21 00 07 54 4c 53 76 31 2e 33 diff --git a/bin/varnishtest/tests/r00498.vtc b/bin/varnishtest/tests/r00498.vtc index c9f2520b5..c11df3f77 100644 --- a/bin/varnishtest/tests/r00498.vtc +++ b/bin/varnishtest/tests/r00498.vtc @@ -3,7 +3,7 @@ varnishtest "very very very long return header" server s1 { rxreq expect req.url == "/" - txresp -hdr "Location: 111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111" -body {foo} + txresp -hdr "Location: ${string,repeat,8136,1}" -body {foo} } -start varnish v1 -vcl+backend { diff --git a/bin/varnishtest/tests/r01120.vtc b/bin/varnishtest/tests/r01120.vtc index c55064a00..d906096d7 100644 --- a/bin/varnishtest/tests/r01120.vtc +++ b/bin/varnishtest/tests/r01120.vtc @@ -18,7 +18,7 @@ client c1 { rxresp expect resp.bodylen == 4 #txreq -hdr "Foo: blablaA" - txreq -hdr "Foo: blablaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAaaaaaaaaaaaaaaaaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + txreq -hdr "Foo: blabla${string,repeat,1430,A}" rxresp expect resp.bodylen == 5 } -run diff --git a/bin/varnishtest/tests/r01274.vtc b/bin/varnishtest/tests/r01274.vtc index 90cd95deb..d90e98273 100644 --- a/bin/varnishtest/tests/r01274.vtc +++ b/bin/varnishtest/tests/r01274.vtc @@ -3,10 +3,10 @@ varnishtest "#1274 - panic when Vary field-name is too large to fit in a signed server s1 { rxreq # Vary header more than 127 characters long - txresp -hdr "Vary: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" -bodylen 9 + txresp -hdr "Vary: ${string,repeat,200,a}" -bodylen 9 rxreq # Vary header more than 127 characters long - txresp -hdr "Vary: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" -bodylen 8 + txresp -hdr "Vary: ${string,repeat,200,a}" -bodylen 8 } -start varnish v1 -vcl+backend { } -start diff --git a/bin/varnishtest/tests/r01284.vtc b/bin/varnishtest/tests/r01284.vtc index d899fac54..e188e69a5 100644 --- a/bin/varnishtest/tests/r01284.vtc +++ b/bin/varnishtest/tests/r01284.vtc @@ -6,7 +6,7 @@ server s1 { txresp -bodylen 1048290 rxreq expect req.url == "/obj2" - txresp -hdr "Long: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" -hdr "Long2: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + txresp -hdr "Long: ${string,repeat,127,a}" -hdr "Long2: ${string,repeat,135,a}" } -start varnish v1 \ diff --git a/bin/varnishtest/tests/r02219.vtc b/bin/varnishtest/tests/r02219.vtc index bb39cf78d..5cc3abafc 100644 --- a/bin/varnishtest/tests/r02219.vtc +++ b/bin/varnishtest/tests/r02219.vtc @@ -19,7 +19,7 @@ varnish v1 -arg "-p workspace_client=9k" -proto PROXY -vcl+backend { } -start client c1 { - send "PROXY TCP4 127.0.0.1 127.0.0.1 1111 2222\r\nGET /AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA HTTP/1.1\r\n\r\n" + send "PROXY TCP4 127.0.0.1 127.0.0.1 1111 2222\r\nGET /${string,repeat,752,A} HTTP/1.1\r\n\r\n" rxresp } -run @@ -29,42 +29,13 @@ client c2 { 0d 0a 0d 0a 00 0d 0a 51 55 49 54 0a 21 00 00 00 47 45 54 20 2f -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 42 -42 42 42 42 42 42 42 +${string,repeat,732,"42 "} 20 48 54 54 50 2f 31 2e 31 0d 0a 0d 0a } rxresp } -run client c3 { - send "PROXY TCP4 127.0.0.1 127.0.0.1 1111 2222\r\nGET /CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC HTTP/1.1\r\n\r\n" + send "PROXY TCP4 127.0.0.1 127.0.0.1 1111 2222\r\nGET /${string,repeat,756,C} HTTP/1.1\r\n\r\n" rxresp } -run diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index c646c6574..bd4d243eb 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -673,6 +673,64 @@ macro_func_date(int argc, char *const *argv, const char **err) return (s); } +static char * +macro_func_string_repeat(int argc, char *const *argv, const char **err) +{ + struct vsb vsb[1]; + const char *p; + char *res; + size_t l; + int i; + + if (argc != 4) { + *err = "repeat takes 2 arguments"; + return (NULL); + } + + p = argv[2]; + i = SF_Parse_Integer(&p, err); + + if (*err != NULL) + return (NULL); + + if (*p != '\0' || i < 0) { + *err = "invalid number of repetitions"; + return (NULL); + } + + l = (strlen(argv[3]) * i) + 1; + res = malloc(l); + AN(res); + AN(VSB_init(vsb, res, l)); + while (i > 0) { + AZ(VSB_cat(vsb, argv[3])); + i--; + } + AZ(VSB_finish(vsb)); + VSB_fini(vsb); + return (res); +} + +static char * +macro_func_string(int argc, char *const *argv, const char **err) +{ + + assert(argc >= 2); + AN(argv); + AN(err); + + if (argc == 2) { + *err = "missing action"; + return (NULL); + } + + if (!strcmp(argv[2], "repeat")) + return (macro_func_string_repeat(argc - 1, argv + 1, err)); + + *err = "unknown action"; + return (NULL); +} + /********************************************************************** * Main */ @@ -754,6 +812,7 @@ main(int argc, char * const *argv) extmacro_def("pwd", NULL, "%s", cwd); extmacro_def("date", macro_func_date, NULL); + extmacro_def("string", macro_func_string, NULL); vmod_path = NULL; From dridi.boukelmoune at gmail.com Mon Jul 5 15:48:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Jul 2021 15:48:06 +0000 (UTC) Subject: [master] 33953f071 doc: Add varnishtest macro syntax and built-in macros Message-ID: <20210705154806.44F0D7AD9@lists.varnish-cache.org> commit 33953f071b797b979370eb1756ffbbe6708fee1d Author: Dridi Boukelmoune Date: Thu Jul 1 12:59:33 2021 +0200 doc: Add varnishtest macro syntax and built-in macros diff --git a/doc/sphinx/reference/vtc.rst b/doc/sphinx/reference/vtc.rst index e88b44305..cd74e6c70 100644 --- a/doc/sphinx/reference/vtc.rst +++ b/doc/sphinx/reference/vtc.rst @@ -57,6 +57,64 @@ being the command and the following ones being its arguments. To continue over to a new line without breaking the argument string, you can escape the newline character (\\n) with a backslash (\\). +MACROS +====== + +When a string is processed, macro expansion is performed. Macros are in the +form ``${[,...]}``, they have a name followed by an optional +comma- or space-separated list of arguments. Leading and trailing spaces are +ignored. + +The macros ``${foo,bar,baz}`` and ``${ foo bar baz }`` are equivalent. If an +argument contains a space or a comma, arguments can be quoted. For example the +macro ``${foo,"bar,baz"}`` gives one argument ``bar,baz`` to the macro called +``foo``. + +Unless documented otherwise, all macros are simple macros that don't take +arguments. + +Built-in macros +--------------- + +``${bad_backend}`` + A socket address that will reliably never accept connections. + +``${bad_ip}`` + An unlikely IPv4 address. + +``${date}`` + The current date and time formatted for HTTP. + +``${listen_addr}`` + The default listen address various components use, by default a random + port on localhost. + +``${localhost}`` + The first IP address that resolves to "localhost". + +``${pwd}`` + The working directory from which ``varnishtest`` was executed. + +``${string,[,...]}`` + The ``string`` macro is the entry point for text generation, it takes + a specialized action with each its own set of arguments. + +``${string,repeat,,}`` + Repeat ``uint`` times the string ``str``. + +``${testdir}`` + The directory containing the VTC script of the ongoing test case + execution. + +``${tmpdir}`` + The dedicated working directory for the ongoing test case execution, + which happens to also be the current working directory. Useful when an + absolute path to the working directory is needed. + +``${topbuild}`` + Only present when the ``-i`` option is used, to work on Varnish itself + instead of a regular installation. + SYNTAX ====== From dridi.boukelmoune at gmail.com Mon Jul 5 15:49:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Jul 2021 15:49:05 +0000 (UTC) Subject: [master] 0e3667c5c vre: Extract a VRE_sub() function from VRT_regsub() Message-ID: <20210705154905.8036E93CD@lists.varnish-cache.org> commit 0e3667c5c45b2605cbac25c43df0e2a0d219c4be Author: Dridi Boukelmoune Date: Fri Jun 18 17:03:42 2021 +0200 vre: Extract a VRE_sub() function from VRT_regsub() This gives us a clean separation of VCL and pcre interactions. diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c index 63dfb9c07..886c35a8b 100644 --- a/bin/varnishd/cache/cache_vrt_re.c +++ b/bin/varnishd/cache/cache_vrt_re.c @@ -38,20 +38,6 @@ #include "cache_varnishd.h" #include "vcc_interface.h" -static void -Tadd(char **b, char *e, const char *p, int l) -{ - assert((*b) <= e); - - if (l <= 0) { - } if ((*b) + l < e) { - memcpy((*b), p, l); - (*b) += l; - } else { - (*b) = e; - } -} - void VPI_re_init(vre_t **rep, const char *re) { @@ -93,19 +79,12 @@ VRT_re_match(VRT_CTX, const char *s, VCL_REGEX re) } VCL_STRING -VRT_regsub(VRT_CTX, int all, VCL_STRING str, VCL_REGEX re, - VCL_STRING sub) +VRT_regsub(VRT_CTX, int all, VCL_STRING str, VCL_REGEX re, VCL_STRING sub) { - int ovector[30]; - int i, l; - char *res_b; - char *res_e; - char *b0; - const char *s; - unsigned u, x; - int options = 0; - int offset = 0; - size_t len; + struct vsb vsb[1]; + const char *res; + uintptr_t snap; + int i; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); AN(re); @@ -113,65 +92,18 @@ VRT_regsub(VRT_CTX, int all, VCL_STRING str, VCL_REGEX re, str = ""; if (sub == NULL) sub = ""; - memset(ovector, 0, sizeof(ovector)); - len = strlen(str); - i = VRE_exec(re, str, len, 0, options, ovector, 30, - &cache_param->vre_limits); - - /* If it didn't match, we can return the original string */ - if (i == VRE_ERROR_NOMATCH) - return (str); - if (i < VRE_ERROR_NOMATCH ) { - VRT_fail(ctx, "Regexp matching returned %d", i); - return (str); - } - - u = WS_ReserveAll(ctx->ws); - res_e = res_b = b0 = WS_Reservation(ctx->ws); - res_e += u; - - do { - /* Copy prefix to match */ - Tadd(&res_b, res_e, str + offset, ovector[0] - offset); - for (s = sub ; *s != '\0'; s++ ) { - if (*s != '\\' || s[1] == '\0') { - if (res_b < res_e) - *res_b++ = *s; - continue; - } - s++; - if (isdigit(*s)) { - x = *s - '0'; - l = ovector[2*x+1] - ovector[2*x]; - Tadd(&res_b, res_e, str + ovector[2*x], l); - continue; - } else { - if (res_b < res_e) - *res_b++ = *s; - } - } - offset = ovector[1]; - if (!all) - break; - memset(ovector, 0, sizeof(ovector)); - options |= VRE_NOTEMPTY; - i = VRE_exec(re, str, len, offset, options, ovector, 30, - &cache_param->vre_limits); - if (i < VRE_ERROR_NOMATCH ) { - WS_Release(ctx->ws, 0); - VRT_fail(ctx, "Regexp matching returned %d", i); - return (str); - } - } while (i != VRE_ERROR_NOMATCH); - /* Copy suffix to match */ - Tadd(&res_b, res_e, str + offset, 1 + len - offset); - if (res_b >= res_e) { - WS_MarkOverflow(ctx->ws); - WS_Release(ctx->ws, 0); - return (str); - } - assert(res_b <= res_e); - WS_ReleaseP(ctx->ws, res_b); - return (b0); + snap = WS_Snapshot(ctx->ws); + WS_VSB_new(vsb, ctx->ws); + i = VRE_sub(re, str, sub, vsb, &cache_param->vre_limits, all); + res = WS_VSB_finish(vsb, ctx->ws, NULL); + + if (i < VRE_ERROR_NOMATCH) + VRT_fail(ctx, "regsub: Regexp matching returned %d", i); + else if (res == NULL) + VRT_fail(ctx, "regsub: Out of workspace"); + else if (i > 0) + return (res); + WS_Reset(ctx->ws, snap); + return (str); } diff --git a/include/vre.h b/include/vre.h index 6dcf0e593..27158663a 100644 --- a/include/vre.h +++ b/include/vre.h @@ -59,6 +59,8 @@ vre_t *VRE_compile(const char *, unsigned, const char **, int *); int VRE_exec(const vre_t *code, const char *subject, int length, int startoffset, int options, int *ovector, int ovecsize, const volatile struct vre_limits *lim); +int VRE_sub(const vre_t *code, const char *subject, const char *replacement, + struct vsb *vsb, const volatile struct vre_limits *lim, int all); void VRE_free(vre_t **); void VRE_quote(struct vsb *, const char *); diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c index 60254aa4b..e7a15946d 100644 --- a/lib/libvarnish/vre.c +++ b/lib/libvarnish/vre.c @@ -31,6 +31,7 @@ #include "config.h" #include +#include #include #include @@ -143,6 +144,63 @@ VRE_exec(const vre_t *code, const char *subject, int length, startoffset, options, ovector, ovecsize)); } +int +VRE_sub(const vre_t *code, const char *subject, const char *replacement, + struct vsb *vsb, const volatile struct vre_limits *lim, int all) +{ + int ovector[30]; + int i, l; + const char *s; + unsigned x; + int options = 0; + int offset = 0; + size_t len; + + CHECK_OBJ_NOTNULL(code, VRE_MAGIC); + CHECK_OBJ_NOTNULL(vsb, VSB_MAGIC); + AN(subject); + AN(replacement); + + memset(ovector, 0, sizeof(ovector)); + len = strlen(subject); + i = VRE_exec(code, subject, len, 0, options, ovector, 30, lim); + + if (i <= VRE_ERROR_NOMATCH) + return (i); + + do { + /* Copy prefix to match */ + VSB_bcat(vsb, subject + offset, ovector[0] - offset); + for (s = replacement; *s != '\0'; s++ ) { + if (*s != '\\' || s[1] == '\0') { + VSB_putc(vsb, *s); + continue; + } + s++; + if (isdigit(*s)) { + x = *s - '0'; + l = ovector[2*x+1] - ovector[2*x]; + VSB_bcat(vsb, subject + ovector[2*x], l); + continue; + } + VSB_putc(vsb, *s); + } + offset = ovector[1]; + if (!all) + break; + memset(ovector, 0, sizeof(ovector)); + options |= VRE_NOTEMPTY; + i = VRE_exec(code, subject, len, offset, options, ovector, 30, + lim); + if (i < VRE_ERROR_NOMATCH ) + return (i); + } while (i != VRE_ERROR_NOMATCH); + + /* Copy suffix to match */ + VSB_bcat(vsb, subject + offset, 1 + len - offset); + return (1); +} + void VRE_free(vre_t **vv) { From dridi.boukelmoune at gmail.com Mon Jul 5 15:49:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Jul 2021 15:49:05 +0000 (UTC) Subject: [master] 520cd8af2 vre: Replace VRE_exec() with a simpler VRE_match() Message-ID: <20210705154905.9CDEB93D5@lists.varnish-cache.org> commit 520cd8af2a33a9e7cab2722e535fe1d1aa522191 Author: Dridi Boukelmoune Date: Fri Jun 18 17:33:17 2021 +0200 vre: Replace VRE_exec() with a simpler VRE_match() With the introduction of VRE_sub() many VRE_exec() use cases went away: - VRE_NOTEMPTY flag - start offset - ovector for capture groups The subject string length is now optional and zero can be passed to signal that the subject string is null-terminated. There are no options for VRE_match() but an option argument is kept in case we start exposing some in the future. diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c index 886c35a8b..bf58e1f71 100644 --- a/bin/varnishd/cache/cache_vrt_re.c +++ b/bin/varnishd/cache/cache_vrt_re.c @@ -70,7 +70,7 @@ VRT_re_match(VRT_CTX, const char *s, VCL_REGEX re) if (s == NULL) s = ""; AN(re); - i = VRE_exec(re, s, strlen(s), 0, 0, NULL, 0, &cache_param->vre_limits); + i = VRE_match(re, s, 0, 0, &cache_param->vre_limits); if (i >= 0) return (1); if (i < VRE_ERROR_NOMATCH ) diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index 89be37b43..f54b9c04b 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -323,7 +323,7 @@ cmd_haproxy_cli_expect(CMD_ARGS) vtc_fatal(hc->vl, "CLI regexp error: '%s' (@%d) (%s)", error, erroroffset, spec); - i = VRE_exec(vre, hc->rxbuf, strlen(hc->rxbuf), 0, 0, NULL, 0, 0); + i = VRE_match(vre, hc->rxbuf, 0, 0, NULL); VRE_free(&vre); diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index ba7818555..152144021 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -394,8 +394,7 @@ logexp_match(const struct logexp *le, struct logexp_test *test, if (test->vre && test->tag >= 0 && test->tag == tag && - VRE_ERROR_NOMATCH == VRE_exec(test->vre, data, - len, 0, 0, NULL, 0, NULL)) + VRE_ERROR_NOMATCH == VRE_match(test->vre, data, len, 0, NULL)) ok = 0; alt = (test->skip_max == LE_ALT); diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index 6c3288d97..a2b2038c7 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -193,10 +193,8 @@ cmd_shell_engine(struct vtclog *vl, int ok, const char *cmd, else vtc_log(vl, 4, "shell_expect found"); } else if (vre != NULL) { - if (VRE_exec(vre, VSB_data(vsb), VSB_len(vsb), 0, 0, - NULL, 0, NULL) < 1) - vtc_fatal(vl, - "shell_match failed: (\"%s\")", re); + if (VRE_match(vre, VSB_data(vsb), VSB_len(vsb), 0, NULL) < 1) + vtc_fatal(vl, "shell_match failed: (\"%s\")", re); else vtc_log(vl, 4, "shell_match succeeded"); VRE_free(&vre); diff --git a/bin/varnishtest/vtc_subr.c b/bin/varnishtest/vtc_subr.c index 3eb60c100..7d5607cd0 100644 --- a/bin/varnishtest/vtc_subr.c +++ b/bin/varnishtest/vtc_subr.c @@ -103,7 +103,7 @@ vtc_expect(struct vtclog *vl, if (vre == NULL) vtc_fatal(vl, "REGEXP error: %s (@%d) (%s)", error, erroroffset, rhs); - i = VRE_exec(vre, lhs, strlen(lhs), 0, 0, NULL, 0, 0); + i = VRE_match(vre, lhs, 0, 0, NULL); retval = (i >= 0 && *cmp == '~') || (i < 0 && *cmp == '!'); VRE_free(&vre); } else if (!strcmp(cmp, "==")) { diff --git a/bin/varnishtest/vtc_syslog.c b/bin/varnishtest/vtc_syslog.c index 403a394ab..221a5b6f5 100644 --- a/bin/varnishtest/vtc_syslog.c +++ b/bin/varnishtest/vtc_syslog.c @@ -411,7 +411,7 @@ cmd_syslog_expect(CMD_ARGS) vtc_fatal(s->vl, "REGEXP error: '%s' (@%d) (%s)", error, erroroffset, spec); - i = VRE_exec(vre, s->rxbuf, strlen(s->rxbuf), 0, 0, NULL, 0, 0); + i = VRE_match(vre, s->rxbuf, 0, 0, NULL); VRE_free(&vre); diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 3612aaa9c..97a876876 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -755,7 +755,7 @@ varnish_cli(struct varnish *v, const char *cli, unsigned exp, const char *re) if (exp != 0 && exp != (unsigned)u) vtc_fatal(v->vl, "FAIL CLI response %u expected %u", u, exp); if (vre != NULL) { - err = VRE_exec(vre, resp, strlen(resp), 0, 0, NULL, 0, NULL); + err = VRE_match(vre, resp, 0, 0, NULL); if (err < 1) vtc_fatal(v->vl, "Expect failed (%d)", err); VRE_free(&vre); diff --git a/include/vre.h b/include/vre.h index 27158663a..488791daf 100644 --- a/include/vre.h +++ b/include/vre.h @@ -53,12 +53,10 @@ typedef struct vre vre_t; /* And those to PCRE options */ extern const unsigned VRE_has_jit; extern const unsigned VRE_CASELESS; -extern const unsigned VRE_NOTEMPTY; vre_t *VRE_compile(const char *, unsigned, const char **, int *); -int VRE_exec(const vre_t *code, const char *subject, int length, - int startoffset, int options, int *ovector, int ovecsize, - const volatile struct vre_limits *lim); +int VRE_match(const vre_t *code, const char *subject, size_t length, + int options, const volatile struct vre_limits *lim); int VRE_sub(const vre_t *code, const char *subject, const char *replacement, struct vsb *vsb, const volatile struct vre_limits *lim, int all); void VRE_free(vre_t **); diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c index e7a15946d..5dae04f97 100644 --- a/lib/libvarnish/vre.c +++ b/lib/libvarnish/vre.c @@ -69,15 +69,14 @@ struct vre { * here. */ const unsigned VRE_CASELESS = PCRE_CASELESS; -const unsigned VRE_NOTEMPTY = PCRE_NOTEMPTY; /* * Even though we only have one for each case so far, keep track of masks - * to differentiate between compile and exec options and enfore the hard + * to differentiate between compile and match options and enfore the hard * VRE linkage. */ #define VRE_MASK_COMPILE PCRE_CASELESS -#define VRE_MASK_EXEC PCRE_NOTEMPTY +#define VRE_MASK_MATCH 0 vre_t * VRE_compile(const char *pattern, unsigned options, @@ -115,18 +114,14 @@ VRE_compile(const char *pattern, unsigned options, return (v); } -int -VRE_exec(const vre_t *code, const char *subject, int length, +static int +vre_exec(const vre_t *code, const char *subject, int length, int startoffset, int options, int *ovector, int ovecsize, const volatile struct vre_limits *lim) { - CHECK_OBJ_NOTNULL(code, VRE_MAGIC); - int ov[30]; - if (ovector == NULL) { - ovector = ov; - ovecsize = sizeof(ov)/sizeof(ov[0]); - } + CHECK_OBJ_NOTNULL(code, VRE_MAGIC); + AN(ovector); if (lim != NULL) { /* XXX: not reentrant */ @@ -139,11 +134,27 @@ VRE_exec(const vre_t *code, const char *subject, int length, code->re_extra->flags &= ~PCRE_EXTRA_MATCH_LIMIT_RECURSION; } - AZ(options & (~VRE_MASK_EXEC)); return (pcre_exec(code->re, code->re_extra, subject, length, startoffset, options, ovector, ovecsize)); } +int +VRE_match(const vre_t *code, const char *subject, size_t length, + int options, const volatile struct vre_limits *lim) +{ + int ovector[30]; + + CHECK_OBJ_NOTNULL(code, VRE_MAGIC); + AN(subject); + AZ(options & (~VRE_MASK_MATCH)); + + if (length == 0) + length = strlen(subject); + + return (vre_exec(code, subject, length, 0, options, + ovector, 30, lim)); +} + int VRE_sub(const vre_t *code, const char *subject, const char *replacement, struct vsb *vsb, const volatile struct vre_limits *lim, int all) @@ -152,7 +163,6 @@ VRE_sub(const vre_t *code, const char *subject, const char *replacement, int i, l; const char *s; unsigned x; - int options = 0; int offset = 0; size_t len; @@ -163,7 +173,7 @@ VRE_sub(const vre_t *code, const char *subject, const char *replacement, memset(ovector, 0, sizeof(ovector)); len = strlen(subject); - i = VRE_exec(code, subject, len, 0, options, ovector, 30, lim); + i = vre_exec(code, subject, len, 0, 0, ovector, 30, lim); if (i <= VRE_ERROR_NOMATCH) return (i); @@ -189,9 +199,8 @@ VRE_sub(const vre_t *code, const char *subject, const char *replacement, if (!all) break; memset(ovector, 0, sizeof(ovector)); - options |= VRE_NOTEMPTY; - i = VRE_exec(code, subject, len, offset, options, ovector, 30, - lim); + i = vre_exec(code, subject, len, offset, PCRE_NOTEMPTY, + ovector, 30, lim); if (i < VRE_ERROR_NOMATCH ) return (i); } while (i != VRE_ERROR_NOMATCH); diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index 681c69985..7fadba364 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -173,7 +173,7 @@ vsl_match_IX(struct VSL_data *vsl, const vslf_list *list, CHECK_OBJ_NOTNULL(vslf, VSLF_MAGIC); if (vslf->tags != NULL && !vbit_test(vslf->tags, tag)) continue; - if (VRE_exec(vslf->vre, cdata, len, 0, 0, NULL, 0, NULL) >= 0) + if (VRE_match(vslf->vre, cdata, len, 0, NULL) >= 0) return (1); } return (0); diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c index 9a8fa8283..adcb77957 100644 --- a/lib/libvarnishapi/vsl_query.c +++ b/lib/libvarnishapi/vsl_query.c @@ -264,13 +264,13 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec) return (0); case '~': /* ~ */ assert(rhs->type == VEX_REGEX && rhs->val_regex != NULL); - i = VRE_exec(rhs->val_regex, b, e - b, 0, 0, NULL, 0, NULL); + i = VRE_match(rhs->val_regex, b, e - b, 0, NULL); if (i != VRE_ERROR_NOMATCH) return (1); return (0); case T_NOMATCH: /* !~ */ assert(rhs->type == VEX_REGEX && rhs->val_regex != NULL); - i = VRE_exec(rhs->val_regex, b, e - b, 0, 0, NULL, 0, NULL); + i = VRE_match(rhs->val_regex, b, e - b, 0, NULL); if (i == VRE_ERROR_NOMATCH) return (1); return (0); From dridi.boukelmoune at gmail.com Mon Jul 5 15:49:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Jul 2021 15:49:05 +0000 (UTC) Subject: [master] 3e254078a vre: Extract VRE_error() from VRE_compile() Message-ID: <20210705154905.B9A6893DA@lists.varnish-cache.org> commit 3e254078a52ff9f271e995e64ca5d2b6443bae04 Author: Dridi Boukelmoune Date: Fri Jun 18 10:08:26 2021 +0200 vre: Extract VRE_error() from VRE_compile() This is a major step back in terms of error reporting, but I haven't found anything in libpcre to translate an error code to an error message. "pcre error %d" should still be a good enough hint. Looking forward, we are going to need this for libpcre2 but again the API leaves some to be desired and only works by writing error messages to buffers. The return value implies that we got a valid error code, which will be verified with pcre2. I couldn't find how to verify error codes with pcre. Writing to the VSB is however fail-safe, it's the caller's job to deal with a VSB error. diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c index bf58e1f71..bda5256f6 100644 --- a/bin/varnishd/cache/cache_vrt_re.c +++ b/bin/varnishd/cache/cache_vrt_re.c @@ -42,8 +42,7 @@ void VPI_re_init(vre_t **rep, const char *re) { vre_t *t; - const char *error; - int erroroffset; + int error, erroroffset; /* This was already check-compiled by the VCL compiler */ t = VRE_compile(re, 0, &error, &erroroffset); diff --git a/bin/varnishtest/tests/u00006.vtc b/bin/varnishtest/tests/u00006.vtc index c331d7ddf..b2d7dcef9 100644 --- a/bin/varnishtest/tests/u00006.vtc +++ b/bin/varnishtest/tests/u00006.vtc @@ -44,7 +44,7 @@ shell -err -expect {-I: "foo" matches zero tags} \ "varnishlog -I foo:bar" shell -err -expect {-I: "Resp" is ambiguous} \ "varnishlog -I Resp:bar" -shell -err -expect {-I: Regex error at position 4 (missing ))} \ +shell -err -expect {-I: Regex error at position 4 (pcre error 14)} \ {varnishlog -I "(foo"} shell -err -expect "-t: Invalid argument" \ "varnishlog -t -1" diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index f54b9c04b..ec6ab1b40 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -298,10 +298,10 @@ static void v_matchproto_(cmd_f) cmd_haproxy_cli_expect(CMD_ARGS) { struct haproxy_cli *hc; + struct vsb vsb[1]; vre_t *vre; - const char *error; - int erroroffset, i, ret; - char *cmp, *spec; + int error, erroroffset, i, ret; + char *cmp, *spec, errbuf[VRE_ERROR_LEN]; (void)vl; CAST_OBJ_NOTNULL(hc, priv, HAPROXY_CLI_MAGIC); @@ -319,9 +319,14 @@ cmd_haproxy_cli_expect(CMD_ARGS) haproxy_cli_recv(hc); vre = VRE_compile(spec, 0, &error, &erroroffset); - if (!vre) + if (vre == NULL) { + AN(VSB_init(vsb, errbuf, sizeof errbuf)); + AZ(VRE_error(vsb, error)); + AZ(VSB_finish(vsb)); + VSB_fini(vsb); vtc_fatal(hc->vl, "CLI regexp error: '%s' (@%d) (%s)", - error, erroroffset, spec); + errbuf, erroroffset, spec); + } i = VRE_match(vre, hc->rxbuf, 0, 0, NULL); diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 152144021..a1d8a2c55 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -624,13 +624,11 @@ static void cmd_logexp_common(struct logexp *le, struct vtclog *vl, int skip_max, char * const *av) { - int vxid; - int tag; vre_t *vre; - const char *err; - int pos; + struct vsb vsb[1]; + int err, pos, tag, vxid; struct logexp_test *test; - char *end; + char *end, errbuf[VRE_ERROR_LEN]; if (!strcmp(av[2], "*")) vxid = LE_ANY; @@ -653,9 +651,14 @@ cmd_logexp_common(struct logexp *le, struct vtclog *vl, vre = NULL; if (av[4]) { vre = VRE_compile(av[4], 0, &err, &pos); - if (vre == NULL) + if (vre == NULL) { + AN(VSB_init(vsb, errbuf, sizeof errbuf)); + AZ(VRE_error(vsb, err)); + AZ(VSB_finish(vsb)); + VSB_fini(vsb); vtc_fatal(vl, "Regex error (%s): '%s' pos %d", - err, av[4], pos); + errbuf, av[4], pos); + } } ALLOC_OBJ(test, LOGEXP_TEST_MAGIC); diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index a2b2038c7..4d267c46b 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -143,23 +143,28 @@ static void cmd_shell_engine(struct vtclog *vl, int ok, const char *cmd, const char *expect, const char *re) { - struct vsb *vsb; + struct vsb *vsb, re_vsb[1]; FILE *fp; vre_t *vre = NULL; - const char *errptr; int r, c; - int err; + int err, erroff; + char errbuf[VRE_ERROR_LEN]; AN(vl); AN(cmd); vsb = VSB_new_auto(); AN(vsb); if (re != NULL) { - vre = VRE_compile(re, 0, &errptr, &err); - if (vre == NULL) + vre = VRE_compile(re, 0, &err, &erroff); + if (vre == NULL) { + AN(VSB_init(re_vsb, errbuf, sizeof errbuf)); + AZ(VRE_error(re_vsb, err)); + AZ(VSB_finish(re_vsb)); + VSB_fini(re_vsb); vtc_fatal(vl, "shell_match invalid regexp (\"%s\" at %d)", - errptr, err); + errbuf, erroff); + } } VSB_printf(vsb, "exec 2>&1 ; %s", cmd); AZ(VSB_finish(vsb)); diff --git a/bin/varnishtest/vtc_subr.c b/bin/varnishtest/vtc_subr.c index 7d5607cd0..3d78566e7 100644 --- a/bin/varnishtest/vtc_subr.c +++ b/bin/varnishtest/vtc_subr.c @@ -87,10 +87,11 @@ vtc_expect(struct vtclog *vl, const char *orhs, const char *rhs) { vre_t *vre; - const char *error; - int erroroffset; + struct vsb vsb[1]; + int error, erroroffset; int i, j, retval = -1; double fl, fr; + char errbuf[VRE_ERROR_LEN]; j = lhs == NULL || rhs == NULL; if (lhs == NULL) @@ -100,9 +101,14 @@ vtc_expect(struct vtclog *vl, if (!strcmp(cmp, "~") || !strcmp(cmp, "!~")) { vre = VRE_compile(rhs, 0, &error, &erroroffset); - if (vre == NULL) + if (vre == NULL) { + AN(VSB_init(vsb, errbuf, sizeof errbuf)); + AZ(VRE_error(vsb, error)); + AZ(VSB_finish(vsb)); + VSB_fini(vsb); vtc_fatal(vl, "REGEXP error: %s (@%d) (%s)", - error, erroroffset, rhs); + errbuf, erroroffset, rhs); + } i = VRE_match(vre, lhs, 0, 0, NULL); retval = (i >= 0 && *cmp == '~') || (i < 0 && *cmp == '!'); VRE_free(&vre); diff --git a/bin/varnishtest/vtc_syslog.c b/bin/varnishtest/vtc_syslog.c index 221a5b6f5..78ccf77ee 100644 --- a/bin/varnishtest/vtc_syslog.c +++ b/bin/varnishtest/vtc_syslog.c @@ -388,10 +388,10 @@ static void v_matchproto_(cmd_f) cmd_syslog_expect(CMD_ARGS) { struct syslog_srv *s; + struct vsb vsb[1]; vre_t *vre; - const char *error; - int erroroffset, i, ret; - char *cmp, *spec; + int error, erroroffset, i, ret; + char *cmp, *spec, errbuf[VRE_ERROR_LEN]; (void)vl; CAST_OBJ_NOTNULL(s, priv, SYSLOG_SRV_MAGIC); @@ -407,9 +407,14 @@ cmd_syslog_expect(CMD_ARGS) assert(!strcmp(cmp, "~") || !strcmp(cmp, "!~")); vre = VRE_compile(spec, 0, &error, &erroroffset); - if (!vre) + if (vre == NULL) { + AN(VSB_init(vsb, errbuf, sizeof errbuf)); + AZ(VRE_error(vsb, error)); + AZ(VSB_finish(vsb)); + VSB_fini(vsb); vtc_fatal(s->vl, "REGEXP error: '%s' (@%d) (%s)", - error, erroroffset, spec); + errbuf, erroroffset, spec); + } i = VRE_match(vre, s->rxbuf, 0, 0, NULL); diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 97a876876..a463b5665 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -739,16 +739,22 @@ static void varnish_cli(struct varnish *v, const char *cli, unsigned exp, const char *re) { enum VCLI_status_e u; + struct vsb vsb[1]; vre_t *vre = NULL; - char *resp = NULL; - const char *errptr; - int err; + char *resp = NULL, errbuf[VRE_ERROR_LEN]; + int err, erroff; VARNISH_LAUNCH(v); if (re != NULL) { - vre = VRE_compile(re, 0, &errptr, &err); - if (vre == NULL) - vtc_fatal(v->vl, "Illegal regexp"); + vre = VRE_compile(re, 0, &err, &erroff); + if (vre == NULL) { + AN(VSB_init(vsb, errbuf, sizeof errbuf)); + AZ(VRE_error(vsb, err)); + AZ(VSB_finish(vsb)); + VSB_fini(vsb); + vtc_fatal(v->vl, "Illegal regexp: %s (@%d)", + errbuf, erroff); + } } u = varnish_ask_cli(v, cli, &resp); vtc_log(v->vl, 2, "CLI %03u <%s>", u, cli); @@ -756,8 +762,13 @@ varnish_cli(struct varnish *v, const char *cli, unsigned exp, const char *re) vtc_fatal(v->vl, "FAIL CLI response %u expected %u", u, exp); if (vre != NULL) { err = VRE_match(vre, resp, 0, 0, NULL); - if (err < 1) - vtc_fatal(v->vl, "Expect failed (%d)", err); + if (err < 1) { + AN(VSB_init(vsb, errbuf, sizeof errbuf)); + AZ(VRE_error(vsb, err)); + AZ(VSB_finish(vsb)); + VSB_fini(vsb); + vtc_fatal(v->vl, "Expect failed (%s)", errbuf); + } VRE_free(&vre); } free(resp); diff --git a/include/vre.h b/include/vre.h index 488791daf..0ecdfe806 100644 --- a/include/vre.h +++ b/include/vre.h @@ -37,6 +37,8 @@ #ifndef VRE_H_INCLUDED #define VRE_H_INCLUDED +#define VRE_ERROR_LEN 128 + struct vre; struct vsb; @@ -54,7 +56,8 @@ typedef struct vre vre_t; extern const unsigned VRE_has_jit; extern const unsigned VRE_CASELESS; -vre_t *VRE_compile(const char *, unsigned, const char **, int *); +vre_t *VRE_compile(const char *, unsigned, int *, int *); +int VRE_error(struct vsb *, int err); int VRE_match(const vre_t *code, const char *subject, size_t length, int options, const volatile struct vre_limits *lim); int VRE_sub(const vre_t *code, const char *subject, const char *replacement, diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c index 5dae04f97..006b15afb 100644 --- a/lib/libvarnish/vre.c +++ b/lib/libvarnish/vre.c @@ -32,6 +32,7 @@ #include #include +#include #include #include @@ -80,24 +81,34 @@ const unsigned VRE_CASELESS = PCRE_CASELESS; vre_t * VRE_compile(const char *pattern, unsigned options, - const char **errptr, int *erroffset) + int *errptr, int *erroffset) { + const char *errstr = NULL; vre_t *v; - *errptr = NULL; *erroffset = 0; + + AN(pattern); + AZ(options & (~VRE_MASK_COMPILE)); + AN(errptr); + AN(erroffset); + + *errptr = 0; + *erroffset = -1; ALLOC_OBJ(v, VRE_MAGIC); if (v == NULL) { - *errptr = "Out of memory for VRE"; + *errptr = PCRE_ERROR_NOMEMORY; return (NULL); } AZ(options & (~VRE_MASK_COMPILE)); - v->re = pcre_compile(pattern, options, errptr, erroffset, NULL); + v->re = pcre_compile2(pattern, options, errptr, &errstr, erroffset, + NULL); if (v->re == NULL) { VRE_free(&v); return (NULL); } - v->re_extra = pcre_study(v->re, VRE_STUDY_JIT_COMPILE, errptr); - if (*errptr != NULL) { + v->re_extra = pcre_study(v->re, VRE_STUDY_JIT_COMPILE, &errstr); + if (errstr != NULL) { + *errptr = PCRE_ERROR_INTERNAL; VRE_free(&v); return (NULL); } @@ -106,7 +117,7 @@ VRE_compile(const char *pattern, unsigned options, v->re_extra = calloc(1, sizeof(pcre_extra)); v->my_extra = 1; if (v->re_extra == NULL) { - *errptr = "Out of memory for pcre_extra"; + *errptr = PCRE_ERROR_NOMEMORY; VRE_free(&v); return (NULL); } @@ -114,6 +125,15 @@ VRE_compile(const char *pattern, unsigned options, return (v); } +int +VRE_error(struct vsb *vsb, int err) +{ + + CHECK_OBJ_NOTNULL(vsb, VSB_MAGIC); + VSB_printf(vsb, "pcre error %d", err); + return (0); +} + static int vre_exec(const vre_t *code, const char *subject, int length, int startoffset, int options, int *ovector, int ovecsize, diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c index 4a8ee9a06..d42d92b24 100644 --- a/lib/libvarnishapi/vsl_arg.c +++ b/lib/libvarnishapi/vsl_arg.c @@ -46,6 +46,7 @@ #include "vnum.h" #include "vqueue.h" #include "vre.h" +#include "vsb.h" #include "vapi/vsl.h" @@ -245,11 +246,14 @@ vsl_ix_arg(struct VSL_data *vsl, int opt, const char *arg) static int vsl_IX_arg(struct VSL_data *vsl, int opt, const char *arg) { - int i, l, off; - const char *b, *e, *err; + int i, l, off, err; + const char *b, *e; vre_t *vre; + struct vsb vsb[1]; struct vslf *vslf; struct vbitmap *tags = NULL; + char errbuf[VRE_ERROR_LEN]; + CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC); AN(arg); @@ -283,8 +287,12 @@ vsl_IX_arg(struct VSL_data *vsl, int opt, const char *arg) if (vre == NULL) { if (tags) vbit_destroy(tags); + AN(VSB_init(vsb, errbuf, sizeof errbuf)); + AZ(VRE_error(vsb, err)); + AZ(VSB_finish(vsb)); + VSB_fini(vsb); return (vsl_diag(vsl, "-%c: Regex error at position %d (%s)", - (char)opt, off, err)); + (char)opt, off, errbuf)); } ALLOC_OBJ(vslf, VSLF_MAGIC); diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c index 1dfd42065..65ebff9a8 100644 --- a/lib/libvarnishapi/vxp_parse.c +++ b/lib/libvarnishapi/vxp_parse.c @@ -260,8 +260,7 @@ vxp_expr_str(struct vxp *vxp, struct vex_rhs **prhs) static void vxp_expr_regex(struct vxp *vxp, struct vex_rhs **prhs) { - const char *errptr; - int erroff; + int err, erroff; /* XXX: Caseless option */ @@ -279,10 +278,11 @@ vxp_expr_regex(struct vxp *vxp, struct vex_rhs **prhs) (*prhs)->type = VEX_REGEX; (*prhs)->val_string = strdup(vxp->t->dec); (*prhs)->val_regex = VRE_compile(vxp->t->dec, vxp->vre_options, - &errptr, &erroff); + &err, &erroff); if ((*prhs)->val_regex == NULL) { - AN(errptr); - VSB_printf(vxp->sb, "Regular expression error: %s ", errptr); + VSB_cat(vxp->sb, "Regular expression error: "); + AZ(VRE_error(vxp->sb, err)); + VSB_putc(vxp->sb, ' '); vxp_ErrWhere(vxp, vxp->t, erroff); return; } diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c index de064bea0..bc39936ff 100644 --- a/lib/libvcc/vcc_utils.c +++ b/lib/libvcc/vcc_utils.c @@ -53,16 +53,16 @@ vcc_regexp(struct vcc *tl, struct vsb *vgc_name) { char buf[BUFSIZ]; vre_t *t; - const char *error; - int erroroffset; + int error, erroroffset; struct inifin *ifp; assert(tl->t->tok == CSTR); t = VRE_compile(tl->t->dec, 0, &error, &erroroffset); if (t == NULL) { - VSB_printf(tl->sb, - "Regexp compilation error:\n\n%s\n\n", error); + VSB_cat(tl->sb, "Regexp compilation error:\n\n"); + AZ(VRE_error(tl->sb, error)); + VSB_cat(tl->sb, "\n\n"); vcc_ErrWhere(tl, tl->t); return; } From dridi.boukelmoune at gmail.com Mon Jul 5 15:49:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Jul 2021 15:49:05 +0000 (UTC) Subject: [master] 6715fb682 param: New pcre_jit_compilation boolean Message-ID: <20210705154905.D179093DF@lists.varnish-cache.org> commit 6715fb6823285ed700ac1d90d62235e7f5709b0e Author: Dridi Boukelmoune Date: Fri Jun 18 14:05:25 2021 +0200 param: New pcre_jit_compilation boolean diff --git a/include/tbl/params.h b/include/tbl/params.h index 5d1a3952e..c6ba88eb3 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -706,6 +706,17 @@ PARAM_SIMPLE( /* flags */ EXPERIMENTAL ) +PARAM_SIMPLE( + /* name */ pcre_jit_compilation, + /* type */ boolean, + /* min */ NULL, + /* max */ NULL, + /* def */ "on", + /* units */ "bool", + /* descr */ + "Use the pcre JIT compiler if available." +) + PARAM_SIMPLE( /* name */ ping_interval, /* type */ uint, From dridi.boukelmoune at gmail.com Mon Jul 5 15:49:05 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Jul 2021 15:49:05 +0000 (UTC) Subject: [master] 8cdba637f vre: Add a jit argument to VRE_compile() Message-ID: <20210705154905.EC4DC93E5@lists.varnish-cache.org> commit 8cdba637f83184e2bec32f1bdbf7af39ee1bb38e Author: Dridi Boukelmoune Date: Fri Jun 18 14:06:41 2021 +0200 vre: Add a jit argument to VRE_compile() We always use it internally except: - where the pcre2_jit_compilation parameter applies - when libvcc verifies that a regular expression compiles For the latter, the verification would attempt a jit compilation as well until now, but we no longer need to waste cycles on that. diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c index bda5256f6..4cb5e7b17 100644 --- a/bin/varnishd/cache/cache_vrt_re.c +++ b/bin/varnishd/cache/cache_vrt_re.c @@ -45,7 +45,8 @@ VPI_re_init(vre_t **rep, const char *re) int error, erroroffset; /* This was already check-compiled by the VCL compiler */ - t = VRE_compile(re, 0, &error, &erroroffset); + t = VRE_compile(re, 0, &error, &erroroffset, + cache_param->pcre_jit_compilation); AN(t); *rep = t; } diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index ec6ab1b40..7f0e11601 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -318,7 +318,7 @@ cmd_haproxy_cli_expect(CMD_ARGS) haproxy_cli_recv(hc); - vre = VRE_compile(spec, 0, &error, &erroroffset); + vre = VRE_compile(spec, 0, &error, &erroroffset, 1); if (vre == NULL) { AN(VSB_init(vsb, errbuf, sizeof errbuf)); AZ(VRE_error(vsb, error)); diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index a1d8a2c55..34c139db8 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -650,7 +650,7 @@ cmd_logexp_common(struct logexp *le, struct vtclog *vl, } vre = NULL; if (av[4]) { - vre = VRE_compile(av[4], 0, &err, &pos); + vre = VRE_compile(av[4], 0, &err, &pos, 1); if (vre == NULL) { AN(VSB_init(vsb, errbuf, sizeof errbuf)); AZ(VRE_error(vsb, err)); diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index 4d267c46b..5133ece37 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -155,7 +155,7 @@ cmd_shell_engine(struct vtclog *vl, int ok, const char *cmd, vsb = VSB_new_auto(); AN(vsb); if (re != NULL) { - vre = VRE_compile(re, 0, &err, &erroff); + vre = VRE_compile(re, 0, &err, &erroff, 1); if (vre == NULL) { AN(VSB_init(re_vsb, errbuf, sizeof errbuf)); AZ(VRE_error(re_vsb, err)); diff --git a/bin/varnishtest/vtc_subr.c b/bin/varnishtest/vtc_subr.c index 3d78566e7..4a9c7ec0e 100644 --- a/bin/varnishtest/vtc_subr.c +++ b/bin/varnishtest/vtc_subr.c @@ -100,7 +100,7 @@ vtc_expect(struct vtclog *vl, rhs = ""; if (!strcmp(cmp, "~") || !strcmp(cmp, "!~")) { - vre = VRE_compile(rhs, 0, &error, &erroroffset); + vre = VRE_compile(rhs, 0, &error, &erroroffset, 1); if (vre == NULL) { AN(VSB_init(vsb, errbuf, sizeof errbuf)); AZ(VRE_error(vsb, error)); diff --git a/bin/varnishtest/vtc_syslog.c b/bin/varnishtest/vtc_syslog.c index 78ccf77ee..ac988e9af 100644 --- a/bin/varnishtest/vtc_syslog.c +++ b/bin/varnishtest/vtc_syslog.c @@ -406,7 +406,7 @@ cmd_syslog_expect(CMD_ARGS) assert(!strcmp(cmp, "~") || !strcmp(cmp, "!~")); - vre = VRE_compile(spec, 0, &error, &erroroffset); + vre = VRE_compile(spec, 0, &error, &erroroffset, 1); if (vre == NULL) { AN(VSB_init(vsb, errbuf, sizeof errbuf)); AZ(VRE_error(vsb, error)); diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index a463b5665..15ca52016 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -746,7 +746,7 @@ varnish_cli(struct varnish *v, const char *cli, unsigned exp, const char *re) VARNISH_LAUNCH(v); if (re != NULL) { - vre = VRE_compile(re, 0, &err, &erroff); + vre = VRE_compile(re, 0, &err, &erroff, 1); if (vre == NULL) { AN(VSB_init(vsb, errbuf, sizeof errbuf)); AZ(VRE_error(vsb, err)); diff --git a/include/vre.h b/include/vre.h index 0ecdfe806..08c768cd7 100644 --- a/include/vre.h +++ b/include/vre.h @@ -56,7 +56,7 @@ typedef struct vre vre_t; extern const unsigned VRE_has_jit; extern const unsigned VRE_CASELESS; -vre_t *VRE_compile(const char *, unsigned, int *, int *); +vre_t *VRE_compile(const char *, unsigned, int *, int *, unsigned); int VRE_error(struct vsb *, int err); int VRE_match(const vre_t *code, const char *subject, size_t length, int options, const volatile struct vre_limits *lim); diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c index 006b15afb..a4e3f4a72 100644 --- a/lib/libvarnish/vre.c +++ b/lib/libvarnish/vre.c @@ -81,7 +81,7 @@ const unsigned VRE_CASELESS = PCRE_CASELESS; vre_t * VRE_compile(const char *pattern, unsigned options, - int *errptr, int *erroffset) + int *errptr, int *erroffset, unsigned jit) { const char *errstr = NULL; vre_t *v; @@ -106,7 +106,11 @@ VRE_compile(const char *pattern, unsigned options, VRE_free(&v); return (NULL); } - v->re_extra = pcre_study(v->re, VRE_STUDY_JIT_COMPILE, &errstr); + + errstr = NULL; + if (jit) + v->re_extra = pcre_study(v->re, VRE_STUDY_JIT_COMPILE, &errstr); + if (errstr != NULL) { *errptr = PCRE_ERROR_INTERNAL; VRE_free(&v); diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c index d42d92b24..f7cbcc822 100644 --- a/lib/libvarnishapi/vsl_arg.c +++ b/lib/libvarnishapi/vsl_arg.c @@ -283,7 +283,7 @@ vsl_IX_arg(struct VSL_data *vsl, int opt, const char *arg) b = e + 1; } - vre = VRE_compile(b, vsl->C_opt ? VRE_CASELESS : 0, &err, &off); + vre = VRE_compile(b, vsl->C_opt ? VRE_CASELESS : 0, &err, &off, 1); if (vre == NULL) { if (tags) vbit_destroy(tags); diff --git a/lib/libvarnishapi/vxp_parse.c b/lib/libvarnishapi/vxp_parse.c index 65ebff9a8..6d346d55d 100644 --- a/lib/libvarnishapi/vxp_parse.c +++ b/lib/libvarnishapi/vxp_parse.c @@ -278,7 +278,7 @@ vxp_expr_regex(struct vxp *vxp, struct vex_rhs **prhs) (*prhs)->type = VEX_REGEX; (*prhs)->val_string = strdup(vxp->t->dec); (*prhs)->val_regex = VRE_compile(vxp->t->dec, vxp->vre_options, - &err, &erroff); + &err, &erroff, 1); if ((*prhs)->val_regex == NULL) { VSB_cat(vxp->sb, "Regular expression error: "); AZ(VRE_error(vxp->sb, err)); diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c index bc39936ff..04dca950d 100644 --- a/lib/libvcc/vcc_utils.c +++ b/lib/libvcc/vcc_utils.c @@ -58,7 +58,7 @@ vcc_regexp(struct vcc *tl, struct vsb *vgc_name) assert(tl->t->tok == CSTR); - t = VRE_compile(tl->t->dec, 0, &error, &erroroffset); + t = VRE_compile(tl->t->dec, 0, &error, &erroroffset, 0); if (t == NULL) { VSB_cat(tl->sb, "Regexp compilation error:\n\n"); AZ(VRE_error(tl->sb, error)); From dridi.boukelmoune at gmail.com Mon Jul 5 15:49:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Jul 2021 15:49:06 +0000 (UTC) Subject: [master] a635df720 vre: New VRE_export() function Message-ID: <20210705154906.2E19893F0@lists.varnish-cache.org> commit a635df720a042aa8cfbbd53e4138e94da15ea0db Author: Dridi Boukelmoune Date: Fri Jun 18 14:20:41 2021 +0200 vre: New VRE_export() function It packs a vre_t and a pcre in a single allocation that can be used by both VRE_match() and VRE_sub(). diff --git a/include/vre.h b/include/vre.h index 08c768cd7..31dd339ab 100644 --- a/include/vre.h +++ b/include/vre.h @@ -57,6 +57,7 @@ extern const unsigned VRE_has_jit; extern const unsigned VRE_CASELESS; vre_t *VRE_compile(const char *, unsigned, int *, int *, unsigned); +vre_t *VRE_export(const vre_t *, size_t *); int VRE_error(struct vsb *, int err); int VRE_match(const vre_t *code, const char *subject, size_t length, int options, const volatile struct vre_limits *lim); diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c index a4e3f4a72..f4e54f864 100644 --- a/lib/libvarnish/vre.c +++ b/lib/libvarnish/vre.c @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -56,6 +57,8 @@ const unsigned VRE_has_jit = VRE_STUDY_JIT_COMPILE; # define pcre_free_study pcre_free #endif +#define VRE_PACKED_RE (pcre *)(-1) + struct vre { unsigned magic; #define VRE_MAGIC 0xe83097dc @@ -138,27 +141,64 @@ VRE_error(struct vsb *vsb, int err) return (0); } +static pcre * +vre_unpack(const vre_t *code) +{ + + CHECK_OBJ_NOTNULL(code, VRE_MAGIC); + if (code->re == VRE_PACKED_RE) { + AZ(code->re_extra); + AZ(code->my_extra); + return (TRUST_ME(code + 1)); + } + return (code->re); +} + +vre_t * +VRE_export(const vre_t *code, size_t *sz) +{ + pcre *re; + vre_t *exp; + + CHECK_OBJ_NOTNULL(code, VRE_MAGIC); + re = vre_unpack(code); + AZ(pcre_fullinfo(re, NULL, PCRE_INFO_SIZE, sz)); + + exp = malloc(sizeof(*exp) + *sz); + if (exp == NULL) + return (NULL); + + INIT_OBJ(exp, VRE_MAGIC); + exp->re = VRE_PACKED_RE; + memcpy(exp + 1, re, *sz); + *sz += sizeof(*exp); + return (exp); +} + static int vre_exec(const vre_t *code, const char *subject, int length, int startoffset, int options, int *ovector, int ovecsize, const volatile struct vre_limits *lim) { + pcre *re; CHECK_OBJ_NOTNULL(code, VRE_MAGIC); AN(ovector); if (lim != NULL) { /* XXX: not reentrant */ + AN(code->re_extra); code->re_extra->match_limit = lim->match; code->re_extra->flags |= PCRE_EXTRA_MATCH_LIMIT; code->re_extra->match_limit_recursion = lim->match_recursion; code->re_extra->flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION; - } else { + } else if (code->re_extra != NULL) { code->re_extra->flags &= ~PCRE_EXTRA_MATCH_LIMIT; code->re_extra->flags &= ~PCRE_EXTRA_MATCH_LIMIT_RECURSION; } - return (pcre_exec(code->re, code->re_extra, subject, length, + re = vre_unpack(code); + return (pcre_exec(re, code->re_extra, subject, length, startoffset, options, ovector, ovecsize)); } @@ -241,6 +281,13 @@ VRE_free(vre_t **vv) *vv = NULL; CHECK_OBJ(v, VRE_MAGIC); + + if (v->re == VRE_PACKED_RE) { + v->re = NULL; + AZ(v->re_extra); + AZ(v->my_extra); + } + if (v->re_extra != NULL) { if (v->my_extra) free(v->re_extra); From dridi.boukelmoune at gmail.com Mon Jul 5 15:49:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Jul 2021 15:49:06 +0000 (UTC) Subject: [master] 9219b0dcd ban: Migrate to VRE Message-ID: <20210705154906.6801D9402@lists.varnish-cache.org> commit 9219b0dcd9d3787fa85aa1482e2a685939530034 Author: Dridi Boukelmoune Date: Fri Jun 18 14:54:05 2021 +0200 ban: Migrate to VRE Bans using regular expressions will consume slightly more space, but more importantly that breaks persistence binary compatibility. That's not a concern because we are both planning for a major release where that kind of breakage is acceptable, and in the context of a pcre2 migration we would also break ban persistence. And now, VRE is the sole pcre consumer. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index ce7145efd..90e8e2b05 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -159,7 +159,6 @@ nobase_pkginclude_HEADERS = \ vcldir=$(datarootdir)/$(PACKAGE)/vcl varnishd_CFLAGS = \ - @PCRE_CFLAGS@ \ @SAN_CFLAGS@ \ -DNOT_IN_A_VMOD \ -DVARNISH_STATE_DIR='"${VARNISH_STATE_DIR}"' \ @@ -174,7 +173,6 @@ varnishd_LDADD = \ $(top_builddir)/lib/libvgz/libvgz.a \ @SAN_LDFLAGS@ \ @JEMALLOC_LDADD@ \ - @PCRE_LIBS@ \ ${DL_LIBS} ${PTHREAD_LIBS} ${NET_LIBS} ${RT_LIBS} ${LIBM} if WITH_UNWIND diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index 03a912d9e..bd02f3001 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -32,7 +32,7 @@ #include "config.h" -#include +#include #include #include "cache_varnishd.h" @@ -495,6 +495,7 @@ ban_evaluate(struct worker *wrk, const uint8_t *bsarg, struct objcore *oc, const char *p; const char *arg1; double darg1, darg2; + int rv; /* * for ttl and age, fix the point in time such that banning refers to @@ -567,15 +568,15 @@ ban_evaluate(struct worker *wrk, const uint8_t *bsarg, struct objcore *oc, } break; case BANS_OPER_MATCH: - if (arg1 == NULL || - pcre_exec(bt.arg2_spec, NULL, arg1, strlen(arg1), - 0, 0, NULL, 0) < 0) + rv = VRE_match(bt.arg2_spec, arg1, 0, 0, NULL); + xxxassert(rv >= -1); + if (arg1 == NULL || rv < 0) return (0); break; case BANS_OPER_NMATCH: - if (arg1 != NULL && - pcre_exec(bt.arg2_spec, NULL, arg1, strlen(arg1), - 0, 0, NULL, 0) >= 0) + rv = VRE_match(bt.arg2_spec, arg1, 0, 0, NULL); + xxxassert(rv >= -1); + if (arg1 == NULL || rv >= 0) return (0); break; case BANS_OPER_GT: diff --git a/bin/varnishd/cache/cache_ban.h b/bin/varnishd/cache/cache_ban.h index 1302c17be..6512e20f6 100644 --- a/bin/varnishd/cache/cache_ban.h +++ b/bin/varnishd/cache/cache_ban.h @@ -55,11 +55,6 @@ * 4 bytes - be32: length * n bytes - content * - * In a perfect world, we should vector through VRE to get to PCRE, - * but since we rely on PCRE's ability to encode the regexp into a - * byte string, that would be a little bit artificial, so this is - * the exception that confirms the rule. - * */ /*-------------------------------------------------------------------- diff --git a/bin/varnishd/cache/cache_ban_build.c b/bin/varnishd/cache/cache_ban_build.c index 1d6e3d06a..95327406c 100644 --- a/bin/varnishd/cache/cache_ban_build.c +++ b/bin/varnishd/cache/cache_ban_build.c @@ -32,7 +32,7 @@ #include "config.h" -#include +#include #include "cache_varnishd.h" #include "cache_ban.h" @@ -40,6 +40,7 @@ #include "vend.h" #include "vtim.h" #include "vnum.h" +#include "vre.h" void BAN_Build_Init(void); void BAN_Build_Fini(void); @@ -187,18 +188,26 @@ ban_parse_http(const struct ban_proto *bp, const char *a1) static const char * ban_parse_regexp(struct ban_proto *bp, const char *a3) { - const char *error; - int erroroffset, rc; + struct vsb vsb[1]; + char errbuf[VRE_ERROR_LEN]; + int errorcode, erroroffset; size_t sz; - pcre *re; - - re = pcre_compile(a3, 0, &error, &erroroffset, NULL); - if (re == NULL) - return (ban_error(bp, "Regex compile error: %s", error)); - rc = pcre_fullinfo(re, NULL, PCRE_INFO_SIZE, &sz); - AZ(rc); - ban_add_lump(bp, re, sz); - pcre_free(re); + vre_t *re, *rex; + + re = VRE_compile(a3, 0, &errorcode, &erroroffset, 0); + if (re == NULL) { + AN(VSB_init(vsb, errbuf, sizeof errbuf)); + AZ(VRE_error(vsb, errorcode)); + AZ(VSB_finish(vsb)); + VSB_fini(vsb); + return (ban_error(bp, "Regex compile error: %s", errbuf)); + } + + rex = VRE_export(re, &sz); + AN(rex); + ban_add_lump(bp, rex, sz); + VRE_free(&rex); + VRE_free(&re); return (0); } From dridi.boukelmoune at gmail.com Tue Jul 6 14:15:08 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 6 Jul 2021 14:15:08 +0000 (UTC) Subject: [master] f633c8e54 build: Don't warn on system headers Message-ID: <20210706141508.B120F793C@lists.varnish-cache.org> commit f633c8e54d8dbbc0568ececeb7bb942c06aca0c4 Author: Dridi Boukelmoune Date: Tue Jul 6 15:16:44 2021 +0200 build: Don't warn on system headers Since we turn warnings into errors, that means failing the build because something we have no control over in /usr/include does not have a strict prototype or some other shenanigan. Just in case it might be added by Wall or Wextra in a future clang or GCC release, we may disable it explicitly too. Refs #3565 diff --git a/wflags.py b/wflags.py index ce32037c4..9e9e4e902 100644 --- a/wflags.py +++ b/wflags.py @@ -61,7 +61,6 @@ DESIRABLE_WFLAGS = [ "-Wstrict-prototypes", "-Wstring-plus-int", "-Wswitch", - "-Wsystem-headers", "-Wunused-parameter", "-Wunused-parameters", "-Wunused-result", @@ -69,15 +68,12 @@ DESIRABLE_WFLAGS = [ ] UNDESIRABLE_WFLAGS = [ + "-Wno-system-headers" # Outside of our control "-Wno-thread-safety", # Does not understand our mutexs are wrapped "-Wno-old-style-definition", # Does not like vgz "-Wno-sign-compare", # Fixable "-Wno-implicit-fallthrough", # Probably Fixable - "-Wno-builtin-requires-header", # Complains about linux::pthread.h - "-Wno-incomplete-setjmp-declaration", # Clang complains about glibc::pthread.h - "-Wno-redundant-decls", # Complains about centos::stdio.h "-Wno-missing-variable-declarations", # Complains about optreset - "-Wno-parentheses", # GCC complains about musl::endian.h "-Wno-nullability-completeness", # Barfs all over MacOSx ] From dridi.boukelmoune at gmail.com Tue Jul 6 14:15:08 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 6 Jul 2021 14:15:08 +0000 (UTC) Subject: [master] e34dbe1c7 Revert "circleci: Disable developer warnings for alpine" Message-ID: <20210706141508.C9EEE7943@lists.varnish-cache.org> commit e34dbe1c739b917dec875634ec54a49c8955a74c Author: Dridi Boukelmoune Date: Tue Jul 6 15:31:34 2021 +0200 Revert "circleci: Disable developer warnings for alpine" This reverts commit f254fff7e9682f4c0bb2e0636a07945a5dd89b83. Conflicts: .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 1ba3ff940..950f0b9cd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -331,7 +331,7 @@ workflows: name: distcheck_alpine dist: alpine release: "latest" - extra_conf: --disable-developer-warnings --disable-dependency-tracking + #extra_conf: --without-jemalloc - distcheck: name: distcheck_archlinux dist: archlinux From dridi.boukelmoune at gmail.com Tue Jul 6 17:51:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 6 Jul 2021 17:51:06 +0000 (UTC) Subject: [master] 6014912e7 vre: Migrate to pcre2 Message-ID: <20210706175106.280D56E24C@lists.varnish-cache.org> commit 6014912e74de7989e37b7be2737cef370b7147ba Author: Dridi Boukelmoune Date: Mon Jun 28 17:42:01 2021 +0200 vre: Migrate to pcre2 Now that VRE is the only regular expression API we use, we can migrate its backend to pcre2. The existing 'pcre_*' parameters are also renamed to reflect this migration, and 'pcre_match_limit_recursion' gets special treatment and is renamed to pcre2_depth_limit. This creates an additional API breakage in VRE: the `match_recursion` field in `struct vre_limits` is also renamed. One last breakage is the removal of `VRE_has_jit` used by only one undocumented varnishtest feature, and the pcre_jit feature is only used by one test case that no longer fails. The pcre jit compilation feature was broken anyway: sealing it at compile time will not reflect what VRE actually links to. Once we have a test case needing the jit feature, we can introduce a better API for that check. There is one outstanding performance problem, the ovector that was previously allocated on the stack now needs to be allocated from the heap. It might be possible to implement a pcre2 context to fix that or maybe pool them, but for now we have heap allocations on the critical path. The VRE_sub() function makes sure to make a single ovector allocation (technically a pcre2_match_data allocation) since it's the only one guaranteed to loop on a single regular expression for the `regsuball()` use case. On the documentation front, the SmartOS installation instructions are hidden for lack of a pcre2 package. Closes #3616 Closes #3559 diff --git a/.circleci/Dockerfile b/.circleci/Dockerfile index 7553611e0..71bbce5be 100644 --- a/.circleci/Dockerfile +++ b/.circleci/Dockerfile @@ -10,6 +10,6 @@ RUN set -e;\ libtool \ libunwind-devel \ make \ - pcre-devel \ + pcre2-devel \ python3 \ python-sphinx diff --git a/.circleci/Dockerfile.alpine b/.circleci/Dockerfile.alpine index c09aa9d5a..b9749f284 100644 --- a/.circleci/Dockerfile.alpine +++ b/.circleci/Dockerfile.alpine @@ -14,7 +14,7 @@ RUN set -e; \ libtool \ libunwind-dev \ linux-headers \ - pcre-dev \ + pcre2-dev \ py-docutils \ py3-sphinx \ tar diff --git a/.circleci/Dockerfile.archlinux b/.circleci/Dockerfile.archlinux index 1e6344354..c90140d43 100644 --- a/.circleci/Dockerfile.archlinux +++ b/.circleci/Dockerfile.archlinux @@ -10,7 +10,7 @@ RUN set -e; \ libtool \ libunwind \ linux-headers \ - pcre \ + pcre2 \ python-docutils \ python-sphinx \ tar diff --git a/.circleci/Dockerfile.ubuntu b/.circleci/Dockerfile.ubuntu index cc50bd3a0..7c9920a19 100644 --- a/.circleci/Dockerfile.ubuntu +++ b/.circleci/Dockerfile.ubuntu @@ -15,7 +15,7 @@ RUN set -e; \ libedit-dev \ libjemalloc-dev \ libncurses-dev \ - libpcre3-dev \ + libpcre2-dev \ libtool \ libunwind-dev \ pkg-config \ diff --git a/.circleci/config.yml b/.circleci/config.yml index 950f0b9cd..95df2a5e2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,7 +33,7 @@ jobs: libtool \ libunwind-devel \ make \ - pcre-devel \ + pcre2-devel \ python3 \ python-sphinx - checkout @@ -201,7 +201,7 @@ jobs: libtool \ libunwind-devel \ make \ - pcre-devel \ + pcre2-devel \ python3 \ sudo elif [ << parameters.dist >> = debian -o << parameters.dist >> = ubuntu ]; then @@ -219,7 +219,7 @@ jobs: libedit-dev \ libjemalloc-dev \ libncurses-dev \ - libpcre3-dev \ + libpcre2-dev \ libtool \ libunwind-dev \ pkg-config \ @@ -239,7 +239,7 @@ jobs: libtool \ libunwind-dev \ linux-headers \ - pcre-dev \ + pcre2-dev \ py-docutils \ py3-sphinx \ tar \ @@ -258,7 +258,7 @@ jobs: libtool \ libunwind \ linux-headers \ - pcre \ + pcre2 \ python-docutils \ python-sphinx \ tar diff --git a/.travis.yml b/.travis.yml index af4c0b79a..be91d56d2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,7 +21,7 @@ jobs: - python3-docutils - python3-sphinx - libunwind-dev - - libpcre3-dev + - libpcre2-dev before_script: - ./autogen.sh - ./configure --enable-maintainer-mode --with-unwind diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c index 4cb5e7b17..4fc352364 100644 --- a/bin/varnishd/cache/cache_vrt_re.c +++ b/bin/varnishd/cache/cache_vrt_re.c @@ -46,7 +46,7 @@ VPI_re_init(vre_t **rep, const char *re) /* This was already check-compiled by the VCL compiler */ t = VRE_compile(re, 0, &error, &erroroffset, - cache_param->pcre_jit_compilation); + cache_param->pcre2_jit_compilation); AN(t); *rep = t; } diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt index 8f04936be..0d8f5c5e7 100644 --- a/bin/varnishd/flint.lnt +++ b/bin/varnishd/flint.lnt @@ -80,7 +80,7 @@ -emacro(835, O_LARGEFILE) // Info 835: A zero has been given as left argument to operator '<<' -emacro(845, HTTPH) // Info 845: The left argument to operator '&&' is certain to be 0 --esym(773, PCRE_DATE) // Expression-like macro '___' not parenthesized +-esym(773, PCRE2_DATE) // Expression-like macro '___' not parenthesized ////////////// // Macros defined differently in each VMOD diff --git a/bin/varnishtest/tests/r01576.vtc b/bin/varnishtest/tests/r01576.vtc deleted file mode 100644 index 5dc0be6ec..000000000 --- a/bin/varnishtest/tests/r01576.vtc +++ /dev/null @@ -1,52 +0,0 @@ -varnishtest "Test recursive regexp's fail before consuming all the stack" - -feature pcre_jit - -# If you want to play around, uncomment the next lines and adjust -# the length of the ABAB strings below to suit your needs. -# Better yet: Rewrite your regexps to avoid this madness. - -# varnish v1 -arg "-p thread_pool_stack=48k" -# varnish v1 -arg "-p pcre_match_limit=1000" -# varnish v1 -arg "-p pcre_match_limit_recursion=89" - -# Approximate formula for FreeBSD/amd64: -# pcre_match_limit_recursion = thread_pool_stack * 2 - 9 - -# -p: use 64bit defaults also on 32bit -varnish v1 -arg "-p workspace_client=64k -p http_req_size=32k" -vcl+backend { - backend proforma none; - - sub vcl_recv { - return (synth(200)); - } - sub vcl_synth { - # shamelessly copied from "bugzilla77 at gmail dot com" - # https://bugs.php.net/bug.php?id=70110 - if (req.url ~ "^/(A{1,2}B)+$") { - set resp.http.found = "1"; - } - } -} -start - -# This should succeed with default params and JIT/no-JIT -client c1 { - txreq -url /ABAABABAABABABAB - rxresp - expect resp.status == 200 - expect resp.http.found == 1 -} -run - -# PCRE_ERROR_RECURSIONLIMIT (-21) -# PCRE_ERROR_JIT_STACKLIMIT (-27) -logexpect l1 -v v1 { - expect * * VCL_Error "Regexp matching returned -2[71]" -} -start - -# This should fail with default params and JIT/no-JIT -client c1 { - txreq -url /ABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABABAB - expect_close -} -run - -logexpect l1 -wait diff --git a/bin/varnishtest/tests/r01644.vtc b/bin/varnishtest/tests/r01644.vtc index a6e4fbfa0..e5986d626 100644 --- a/bin/varnishtest/tests/r01644.vtc +++ b/bin/varnishtest/tests/r01644.vtc @@ -16,7 +16,7 @@ varnish v1 -vcl+backend { } -start -varnish v1 -cliok "param.set pcre_match_limit 100" +varnish v1 -cliok "param.set pcre2_match_limit 100" client c1 { txreq @@ -24,7 +24,7 @@ client c1 { expect resp.http.foo == 100 } -run -varnish v1 -cliok "param.set pcre_match_limit 200" +varnish v1 -cliok "param.set pcre2_match_limit 200" client c1 { txreq diff --git a/bin/varnishtest/tests/u00006.vtc b/bin/varnishtest/tests/u00006.vtc index b2d7dcef9..e420aa18d 100644 --- a/bin/varnishtest/tests/u00006.vtc +++ b/bin/varnishtest/tests/u00006.vtc @@ -44,7 +44,7 @@ shell -err -expect {-I: "foo" matches zero tags} \ "varnishlog -I foo:bar" shell -err -expect {-I: "Resp" is ambiguous} \ "varnishlog -I Resp:bar" -shell -err -expect {-I: Regex error at position 4 (pcre error 14)} \ +shell -err -expect {-I: Regex error at position 4 (missing closing parenthesis)} \ {varnishlog -I "(foo"} shell -err -expect "-t: Invalid argument" \ "varnishlog -t -1" diff --git a/bin/varnishtest/tests/v00004.vtc b/bin/varnishtest/tests/v00004.vtc index 8760279e5..805543c55 100644 --- a/bin/varnishtest/tests/v00004.vtc +++ b/bin/varnishtest/tests/v00004.vtc @@ -10,7 +10,7 @@ feature cmd {test -z "$GCOVPROG"} # - 2019 header madness # - 5 ESI levels down # - 10 VCL subs down -# - PCRE regsub +# - PCRE2 regsub server s1 { rxreq diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index 5133ece37..eaf9a99a7 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -504,7 +504,6 @@ cmd_feature(CMD_ARGS) FEATURE("ipv4", ipvx_works("127.0.0.1")); FEATURE("ipv6", ipvx_works("[::1]")); - FEATURE("pcre_jit", VRE_has_jit); FEATURE("64bit", sizeof(void*) == 8); FEATURE("disable_aslr", addr_no_randomize_works()); FEATURE("dns", dns_works()); diff --git a/configure.ac b/configure.ac index 6a2874f8d..2b96413d6 100644 --- a/configure.ac +++ b/configure.ac @@ -111,67 +111,50 @@ AC_SUBST(LIBM) m4_ifndef([PKG_PROG_PKG_CONFIG], [m4_fatal([pkg.m4 missing, please install pkg-config])]) PKG_PROG_PKG_CONFIG if test -n $PKG_CONFIG; then - PKG_CHECK_MODULES([PCRE], [libpcre]) + PKG_CHECK_MODULES([PCRE2], [libpcre2-8]) else - AC_CHECK_PROG(PCRE_CONFIG, pcre-config, pcre-config) - AC_ARG_WITH(pcre-config, - AS_HELP_STRING([--with-pcre-config=PATH], - [Location of PCRE pcre-config (auto)]), - [pcre_config="$withval"], - [pcre_config=""]) - - if test "x$pcre_config" != "x" ; then - AC_MSG_CHECKING(for $pcre_config) - - if test -f $pcre_config ; then - PCRE_CONFIG=$pcre_config + AC_CHECK_PROG(PCRE2_CONFIG, pcre2-config, pcre2-config) + AC_ARG_WITH(pcre2-config, + AS_HELP_STRING([--with-pcre2-config=PATH], + [Location of PCRE2 pcre2-config (auto)]), + [pcre2_config="$withval"], + [pcre2_config=""]) + + if test "x$pcre2_config" != "x" ; then + AC_MSG_CHECKING(for $pcre2_config) + + if test -f $pcre2_config ; then + PCRE2_CONFIG=$pcre2_config AC_MSG_RESULT(yes) else AC_MSG_RESULT(no - searching PATH) fi fi - if test "x$PCRE_CONFIG" = "x"; then - AC_CHECK_PROGS(PCRE_CONFIG, pcre-config) + if test "x$PCRE2_CONFIG" = "x"; then + AC_CHECK_PROGS(PCRE2_CONFIG, pcre2-config) fi - PCRE_CFLAGS=`$PCRE_CONFIG --cflags` - PCRE_LIBS=`$PCRE_CONFIG --libs` + PCRE2_CFLAGS=`$PCRE2_CONFIG --cflags` + PCRE2_LIBS=`$PCRE2_CONFIG --libs8` fi -AC_SUBST(PCRE_CFLAGS) -AC_SUBST(PCRE_LIBS) +AC_SUBST(PCRE2_CFLAGS) +AC_SUBST(PCRE2_LIBS) +AC_DEFINE([PCRE2_CODE_UNIT_WIDTH], [8], [Work with 8-bit characters for PCRE2]) -# --enable-pcre-jit -AC_ARG_ENABLE(pcre-jit, - AS_HELP_STRING([--enable-pcre-jit], - [use the PCRE JIT compiler (default is YES)]), +save_LIBS="${LIBS}" +LIBS="${LIBS} ${PCRE2_LIBS}" +AC_CHECK_FUNCS([pcre2_set_depth_limit_8], [ + AC_DEFINE([HAVE_PCRE2_SET_DEPTH_LIMIT], [1], [Use pcre2_set_depth_limit()]) +]) +LIBS="${save_LIBS}" + +# --enable-pcre2-jit +AC_ARG_ENABLE(pcre2-jit, + AS_HELP_STRING([--enable-pcre2-jit], + [use the PCRE2 JIT compiler (default is YES)]), [], - [enable_pcre_jit=yes]) -if test "$enable_pcre_jit" = yes; then - AC_MSG_CHECKING(for PCRE JIT usability) - save_CFLAGS="${CFLAGS}" - CFLAGS="${CFLAGS} ${PCRE_CFLAGS}" - save_LIBS="${LIBS}" - LIBS="${LIBS} ${PCRE_LIBS}" - AC_RUN_IFELSE( - [AC_LANG_PROGRAM([[ -#include -#if PCRE_MAJOR != 8 || PCRE_MINOR < 32 -#error no jit -#endif - ]],[[ -const char *error; -pcre *re; -int erroroffset; -re = pcre_compile(".", 0, &error, &erroroffset, NULL); -if (!pcre_study(re, PCRE_STUDY_JIT_COMPILE, &error)) - return (1); - ]])], - [AC_MSG_RESULT(yes) - AC_DEFINE([USE_PCRE_JIT], [1], [Use the PCRE JIT compiler]) - ], - [AC_MSG_RESULT(no)] - ) - CFLAGS="${save_CFLAGS}" - LIBS="${save_LIBS}" + [enable_pcre2_jit=yes]) +if test "$enable_pcre2_jit" = yes; then + AC_DEFINE([USE_PCRE2_JIT], [1], [Use the PCRE2 JIT compiler]) fi diff --git a/doc/sphinx/installation/install_source.rst b/doc/sphinx/installation/install_source.rst index ef67e00b3..a8e88b8b2 100644 --- a/doc/sphinx/installation/install_source.rst +++ b/doc/sphinx/installation/install_source.rst @@ -29,7 +29,7 @@ Build dependencies on FreeBSD To get the dependencies required to build varnish from source you can either:: - pkg install automake pkgconf py36-sphinx py36-docutils pcre libtool + pkg install automake pkgconf py36-sphinx py36-docutils pcre2 libtool .. XXX does cpio need to be installed on FreeBSD? @@ -60,7 +60,7 @@ them (replace ``sudo apt-get install`` if needed):: libedit-dev \ libjemalloc-dev \ libncurses-dev \ - libpcre3-dev \ + libpcre2-dev \ libtool \ pkg-config \ python3-docutils \ @@ -114,7 +114,7 @@ packages:: libtool \ libunwind-devel \ ncurses-devel \ - pcre-devel \ + pcre2-devel \ pkgconfig \ python3-docutils \ cpio @@ -176,7 +176,7 @@ As of Alpine 3, these steps should install the required dependencies: libtool \ libunwind-dev \ linux-headers \ - pcre-dev \ + pcre2-dev \ py-docutils \ py3-sphinx \ tar \ @@ -195,24 +195,26 @@ Then continue `Compiling Varnish`_, using the ``--with-unwind`` .. _Alpine Community Repository: https://wiki.alpinelinux.org/wiki/Enable_Community_Repository -Build dependencies on a SmartOS Zone ------------------------------------- - -As of SmartOS pkgsrc 2019Q4, install the following packages:: - - pkgin in autoconf automake editline libtool ncurses \ - pcre python37 py37-sphinx py37-docutils gmake gcc8 pkg-config - -*Note:* you will probably need to add ``/opt/local/gcc8/bin`` to -``PATH`` in order to have ``gcc`` available. - -Optionally, to rebuild the svg files:: - - pkgin in graphviz - -Optionally, to pull from a repository:: - - pkgin in git +.. XXX: no pcre2 on SmartOS +.. +.. Build dependencies on a SmartOS Zone +.. ------------------------------------ +.. +.. As of SmartOS pkgsrc 2019Q4, install the following packages:: +.. +.. pkgin in autoconf automake editline libtool ncurses \ +.. pcre python37 py37-sphinx py37-docutils gmake gcc8 pkg-config +.. +.. *Note:* you will probably need to add ``/opt/local/gcc8/bin`` to +.. ``PATH`` in order to have ``gcc`` available. +.. +.. Optionally, to rebuild the svg files:: +.. +.. pkgin in graphviz +.. +.. Optionally, to pull from a repository:: +.. +.. pkgin in git Building on Solaris and other Solaris-ish OSes ---------------------------------------------- diff --git a/doc/sphinx/reference/vsl-query.rst b/doc/sphinx/reference/vsl-query.rst index e7778c186..4bfd3f348 100644 --- a/doc/sphinx/reference/vsl-query.rst +++ b/doc/sphinx/reference/vsl-query.rst @@ -267,7 +267,7 @@ The following types of operands are available: * Regular expression - A PCRE regular expression. Valid for the regular expression + A PCRE2 regular expression. Valid for the regular expression operators. Boolean functions diff --git a/doc/sphinx/users-guide/increasing-your-hitrate.rst b/doc/sphinx/users-guide/increasing-your-hitrate.rst index 10f4c47fb..8c49e5a8b 100644 --- a/doc/sphinx/users-guide/increasing-your-hitrate.rst +++ b/doc/sphinx/users-guide/increasing-your-hitrate.rst @@ -154,7 +154,7 @@ difficult. Unfortunately Varnish doesn't have good tools for manipulating the Cookies. We have to use regular expressions to do the work. If you are familiar with regular expressions you'll understand whats going on. If you aren't we recommend that you either pick up a book on -the subject, read through the *pcrepattern* man page, or read through +the subject, read through the *pcre2pattern* man page, or read through one of many online guides. Lets use the Varnish Software (VS) web as an example here. Very diff --git a/doc/sphinx/users-guide/storage-backends.rst b/doc/sphinx/users-guide/storage-backends.rst index 7464c74fc..700d47aaa 100644 --- a/doc/sphinx/users-guide/storage-backends.rst +++ b/doc/sphinx/users-guide/storage-backends.rst @@ -90,7 +90,7 @@ to indicate that `libumem`_ will not only be used for storage. Likely reasons for this to be the case are: * some library ``varnishd`` is linked against was linked against - `libumem`_ (most likely ``libpcre``, check with ``ldd``) + `libumem`_ (most likely ``libpcre2-8``, check with ``ldd``) * ``LD_PRELOAD_64=/usr/lib/amd64/libumem.so.1``, ``LD_PRELOAD_32=/usr/lib/libumem.so.1`` or diff --git a/include/tbl/params.h b/include/tbl/params.h index c6ba88eb3..66c14b1e6 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -707,14 +707,14 @@ PARAM_SIMPLE( ) PARAM_SIMPLE( - /* name */ pcre_jit_compilation, + /* name */ pcre2_jit_compilation, /* type */ boolean, /* min */ NULL, /* max */ NULL, /* def */ "on", /* units */ "bool", /* descr */ - "Use the pcre JIT compiler if available." + "Use the pcre2 JIT compiler if available." ) PARAM_SIMPLE( @@ -1615,47 +1615,46 @@ PARAM_VCC( ) /*-------------------------------------------------------------------- - * PCRE parameters + * PCRE2 parameters */ -# define PARAM_PCRE(nm, pv, min, def, descr) \ +# define PARAM_PCRE2(nm, pv, min, def, descr) \ PARAM(, , nm, tweak_uint, &mgt_param.vre_limits.pv, \ min, NULL, def, NULL, descr) -PARAM_PCRE( - /* name */ pcre_match_limit, +PARAM_PCRE2( + /* name */ pcre2_match_limit, /* priv */ match, /* min */ "1", /* def */ "10000", /* descr */ - "The limit for the number of calls to the internal match()" - " function in pcre_exec().\n\n" - "(See: PCRE_EXTRA_MATCH_LIMIT in pcre docs.)\n\n" + "The limit for the number of calls to the internal match" + " logic in pcre2_match().\n\n" + "(See: pcre2_set_match_limit() in pcre2 docs.)\n\n" "This parameter limits how much CPU time" " regular expression matching can soak up." ) -PARAM_PCRE( - /* name */ pcre_match_limit_recursion, - /* priv */ match_recursion, +PARAM_PCRE2( + /* name */ pcre2_depth_limit, + /* priv */ depth, /* min */ "1", /* def */ "20", /* descr */ - "The recursion depth-limit for the internal match() function" - " in a pcre_exec().\n\n" - "(See: PCRE_EXTRA_MATCH_LIMIT_RECURSION in pcre docs.)\n\n" + "The recursion depth-limit for the internal match logic" + " in a pcre2_match().\n\n" + "(See: pcre2_set_depth_limit() in pcre2 docs.)\n\n" "This puts an upper limit on the amount of stack used" - " by PCRE for certain classes of regular expressions.\n\n" + " by PCRE2 for certain classes of regular expressions.\n\n" "We have set the default value low in order to" " prevent crashes, at the cost of possible regexp" " matching failures.\n\n" "Matching failures will show up in the log as VCL_Error" - " messages with regexp errors -27 or -21.\n\n" - "Testcase r01576 can be useful when tuning this parameter." + " messages with regexp errors -27 or -21." ) # undef PARAM_ALL -# undef PARAM_PCRE +# undef PARAM_PCRE2 # undef PARAM_STRING # undef PARAM_VCC #endif /* defined(PARAM_ALL) */ diff --git a/include/vre.h b/include/vre.h index 31dd339ab..c206078da 100644 --- a/include/vre.h +++ b/include/vre.h @@ -29,7 +29,7 @@ * * Regular expression support * - * We wrap PCRE in VRE to make to make it feasible to use something else + * We wrap PCRE2 in VRE to make to make it feasible to use something else * without hunting down stuff through out the Varnish source code. * */ @@ -44,16 +44,15 @@ struct vsb; struct vre_limits { unsigned match; - unsigned match_recursion; + unsigned depth; }; typedef struct vre vre_t; -/* This maps to PCRE error codes */ -#define VRE_ERROR_NOMATCH (-1) +/* This maps to PCRE2 error codes */ +extern const int VRE_ERROR_NOMATCH; -/* And those to PCRE options */ -extern const unsigned VRE_has_jit; +/* And those to PCRE2 options */ extern const unsigned VRE_CASELESS; vre_t *VRE_compile(const char *, unsigned, int *, int *, unsigned); diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am index 6d27522fb..eef530795 100644 --- a/lib/libvarnish/Makefile.am +++ b/lib/libvarnish/Makefile.am @@ -3,7 +3,7 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/include \ -I$(top_builddir)/include \ - @PCRE_CFLAGS@ + @PCRE2_CFLAGS@ AM_CFLAGS = $(AM_LT_CFLAGS) @SAN_CFLAGS@ AM_LDFLAGS = $(AM_LT_LDFLAGS) @SAN_LDFLAGS@ @@ -44,7 +44,7 @@ libvarnish_la_SOURCES = \ vtim.c \ vus.c -libvarnish_la_LIBADD = @PCRE_LIBS@ +libvarnish_la_LIBADD = @PCRE2_LIBS@ TESTS = vav_test vjsn_test vnum_c_test vbh_test vsb_test diff --git a/lib/libvarnish/vre.c b/lib/libvarnish/vre.c index f4e54f864..dd8eb70c3 100644 --- a/lib/libvarnish/vre.c +++ b/lib/libvarnish/vre.c @@ -30,13 +30,12 @@ #include "config.h" -#include #include -#include -#include #include #include +#include + #include "vdef.h" #include "vas.h" // XXX Flexelint "not used" - but req'ed for assert() @@ -45,48 +44,41 @@ #include "vre.h" -#if defined(USE_PCRE_JIT) -#define VRE_STUDY_JIT_COMPILE PCRE_STUDY_JIT_COMPILE -#else -#define VRE_STUDY_JIT_COMPILE 0 -#endif - -const unsigned VRE_has_jit = VRE_STUDY_JIT_COMPILE; - -#if PCRE_MAJOR < 8 || (PCRE_MAJOR == 8 && PCRE_MINOR < 20) -# define pcre_free_study pcre_free +#if !HAVE_PCRE2_SET_DEPTH_LIMIT +# define pcre2_set_depth_limit(r, d) pcre2_set_recursion_limit(r, d) #endif -#define VRE_PACKED_RE (pcre *)(-1) +#define VRE_PACKED_RE (pcre2_code *)(-1) struct vre { unsigned magic; #define VRE_MAGIC 0xe83097dc - pcre *re; - pcre_extra *re_extra; - int my_extra; + pcre2_code *re; + pcre2_match_context *re_ctx; }; /* - * We don't want to spread or even expose the majority of PCRE options - * so we establish our own options and implement hard linkage to PCRE - * here. + * We don't want to spread or even expose the majority of PCRE2 options + * and errors so we establish our own symbols and implement hard linkage + * to PCRE2 here. */ -const unsigned VRE_CASELESS = PCRE_CASELESS; +const int VRE_ERROR_NOMATCH = PCRE2_ERROR_NOMATCH; + +const unsigned VRE_CASELESS = PCRE2_CASELESS; /* * Even though we only have one for each case so far, keep track of masks * to differentiate between compile and match options and enfore the hard * VRE linkage. */ -#define VRE_MASK_COMPILE PCRE_CASELESS +#define VRE_MASK_COMPILE PCRE2_CASELESS #define VRE_MASK_MATCH 0 vre_t * VRE_compile(const char *pattern, unsigned options, int *errptr, int *erroffset, unsigned jit) { - const char *errstr = NULL; + PCRE2_SIZE erroff; vre_t *v; AN(pattern); @@ -99,70 +91,85 @@ VRE_compile(const char *pattern, unsigned options, ALLOC_OBJ(v, VRE_MAGIC); if (v == NULL) { - *errptr = PCRE_ERROR_NOMEMORY; + *errptr = PCRE2_ERROR_NOMEMORY; return (NULL); } - AZ(options & (~VRE_MASK_COMPILE)); - v->re = pcre_compile2(pattern, options, errptr, &errstr, erroffset, - NULL); + v->re = pcre2_compile((PCRE2_SPTR8)pattern, PCRE2_ZERO_TERMINATED, + options, errptr, &erroff, NULL); + *erroffset = erroff; if (v->re == NULL) { VRE_free(&v); return (NULL); } - - errstr = NULL; - if (jit) - v->re_extra = pcre_study(v->re, VRE_STUDY_JIT_COMPILE, &errstr); - - if (errstr != NULL) { - *errptr = PCRE_ERROR_INTERNAL; + v->re_ctx = pcre2_match_context_create(NULL); + if (v->re_ctx == NULL) { + *errptr = PCRE2_ERROR_NOMEMORY; VRE_free(&v); return (NULL); } - if (v->re_extra == NULL) { - /* allocate our own */ - v->re_extra = calloc(1, sizeof(pcre_extra)); - v->my_extra = 1; - if (v->re_extra == NULL) { - *errptr = PCRE_ERROR_NOMEMORY; - VRE_free(&v); - return (NULL); - } - } +#if USE_PCRE2_JIT + if (jit) + (void)pcre2_jit_compile(v->re, 0); +#else + (void)jit; +#endif return (v); } int VRE_error(struct vsb *vsb, int err) { + char buf[VRE_ERROR_LEN]; + int i; CHECK_OBJ_NOTNULL(vsb, VSB_MAGIC); - VSB_printf(vsb, "pcre error %d", err); + i = pcre2_get_error_message(err, (PCRE2_UCHAR *)buf, VRE_ERROR_LEN); + if (i == PCRE2_ERROR_BADDATA) { + VSB_printf(vsb, "unknown pcre2 error code (%d)", err); + return (-1); + } + VSB_cat(vsb, buf); return (0); } -static pcre * +static pcre2_code * vre_unpack(const vre_t *code) { CHECK_OBJ_NOTNULL(code, VRE_MAGIC); if (code->re == VRE_PACKED_RE) { - AZ(code->re_extra); - AZ(code->my_extra); + AZ(code->re_ctx); return (TRUST_ME(code + 1)); } return (code->re); } +static void +vre_limit(const vre_t *code, const volatile struct vre_limits *lim) +{ + + CHECK_OBJ_NOTNULL(code, VRE_MAGIC); + + if (lim == NULL) + return; + + assert(code->re != VRE_PACKED_RE); + + /* XXX: not reentrant */ + AN(code->re_ctx); + pcre2_set_match_limit(code->re_ctx, lim->match); + pcre2_set_depth_limit(code->re_ctx, lim->depth); +} + vre_t * VRE_export(const vre_t *code, size_t *sz) { - pcre *re; + pcre2_code *re; vre_t *exp; CHECK_OBJ_NOTNULL(code, VRE_MAGIC); re = vre_unpack(code); - AZ(pcre_fullinfo(re, NULL, PCRE_INFO_SIZE, sz)); + AZ(pcre2_pattern_info(re, PCRE2_INFO_SIZE, sz)); exp = malloc(sizeof(*exp) + *sz); if (exp == NULL) @@ -176,73 +183,75 @@ VRE_export(const vre_t *code, size_t *sz) } static int -vre_exec(const vre_t *code, const char *subject, int length, - int startoffset, int options, int *ovector, int ovecsize, - const volatile struct vre_limits *lim) +vre_match(const vre_t *code, const char *subject, size_t length, size_t offset, + int options, pcre2_match_data **datap) { - pcre *re; + pcre2_match_data *data; + pcre2_code *re; + int matches; - CHECK_OBJ_NOTNULL(code, VRE_MAGIC); - AN(ovector); - - if (lim != NULL) { - /* XXX: not reentrant */ - AN(code->re_extra); - code->re_extra->match_limit = lim->match; - code->re_extra->flags |= PCRE_EXTRA_MATCH_LIMIT; - code->re_extra->match_limit_recursion = lim->match_recursion; - code->re_extra->flags |= PCRE_EXTRA_MATCH_LIMIT_RECURSION; - } else if (code->re_extra != NULL) { - code->re_extra->flags &= ~PCRE_EXTRA_MATCH_LIMIT; - code->re_extra->flags &= ~PCRE_EXTRA_MATCH_LIMIT_RECURSION; + re = vre_unpack(code); + + if (datap != NULL && *datap != NULL) { + data = *datap; + *datap = NULL; + } else { + data = pcre2_match_data_create_from_pattern(re, NULL); + AN(data); } - re = vre_unpack(code); - return (pcre_exec(re, code->re_extra, subject, length, - startoffset, options, ovector, ovecsize)); + matches = pcre2_match(re, (PCRE2_SPTR)subject, length, offset, + options, data, NULL); + + if (datap != NULL && matches > VRE_ERROR_NOMATCH) + *datap = data; + else + pcre2_match_data_free(data); + return (matches); } int VRE_match(const vre_t *code, const char *subject, size_t length, int options, const volatile struct vre_limits *lim) { - int ovector[30]; CHECK_OBJ_NOTNULL(code, VRE_MAGIC); AN(subject); AZ(options & (~VRE_MASK_MATCH)); if (length == 0) - length = strlen(subject); - - return (vre_exec(code, subject, length, 0, options, - ovector, 30, lim)); + length = PCRE2_ZERO_TERMINATED; + vre_limit(code, lim); + return (vre_match(code, subject, length, 0, options, NULL)); } int VRE_sub(const vre_t *code, const char *subject, const char *replacement, struct vsb *vsb, const volatile struct vre_limits *lim, int all) { - int ovector[30]; + pcre2_match_data *data = NULL; + PCRE2_SIZE *ovector; int i, l; const char *s; unsigned x; int offset = 0; - size_t len; CHECK_OBJ_NOTNULL(code, VRE_MAGIC); CHECK_OBJ_NOTNULL(vsb, VSB_MAGIC); AN(subject); AN(replacement); - memset(ovector, 0, sizeof(ovector)); - len = strlen(subject); - i = vre_exec(code, subject, len, 0, 0, ovector, 30, lim); + vre_limit(code, lim); + i = vre_match(code, subject, PCRE2_ZERO_TERMINATED, offset, 0, &data); if (i <= VRE_ERROR_NOMATCH) return (i); do { + AN(data); + ovector = pcre2_get_ovector_pointer(data); + AN(ovector); + /* Copy prefix to match */ VSB_bcat(vsb, subject + offset, ovector[0] - offset); for (s = replacement; *s != '\0'; s++ ) { @@ -262,15 +271,20 @@ VRE_sub(const vre_t *code, const char *subject, const char *replacement, offset = ovector[1]; if (!all) break; - memset(ovector, 0, sizeof(ovector)); - i = vre_exec(code, subject, len, offset, PCRE_NOTEMPTY, - ovector, 30, lim); - if (i < VRE_ERROR_NOMATCH ) + i = vre_match(code, subject, PCRE2_ZERO_TERMINATED, offset, + PCRE2_NOTEMPTY, &data); + if (i < VRE_ERROR_NOMATCH) return (i); } while (i != VRE_ERROR_NOMATCH); + if (data != NULL) { + assert(i > VRE_ERROR_NOMATCH); + AZ(all); + pcre2_match_data_free(data); + } + /* Copy suffix to match */ - VSB_bcat(vsb, subject + offset, 1 + len - offset); + VSB_cat(vsb, subject + offset); return (1); } @@ -284,18 +298,13 @@ VRE_free(vre_t **vv) if (v->re == VRE_PACKED_RE) { v->re = NULL; - AZ(v->re_extra); - AZ(v->my_extra); + AZ(v->re_ctx); } - if (v->re_extra != NULL) { - if (v->my_extra) - free(v->re_extra); - else - pcre_free_study(v->re_extra); - } + if (v->re_ctx != NULL) + pcre2_match_context_free(v->re_ctx); if (v->re != NULL) - pcre_free(v->re); + pcre2_code_free(v->re); FREE_OBJ(v); } diff --git a/vmod/vmod_std.vcc b/vmod/vmod_std.vcc index fd0bf1a4a..a220b7872 100644 --- a/vmod/vmod_std.vcc +++ b/vmod/vmod_std.vcc @@ -137,7 +137,7 @@ following: * a bracket expression such as ``[abc]`` or ``[!0-9]`` is interpreted as a character class according to the rules of basic regular - expressions (*not* `pcre(3)` regexen), except that ``!`` is used for + expressions (*not* `pcre2(3)` regexen), except that ``!`` is used for character class negation instead of ``^``. If *pathname* is ``true``, then the forward slash character ``/`` is From phk at FreeBSD.org Wed Jul 7 08:14:07 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 7 Jul 2021 08:14:07 +0000 (UTC) Subject: [master] 80071c567 Pre-process for FlexeLint Message-ID: <20210707081407.BE877AEF9C@lists.varnish-cache.org> commit 80071c567a60fe6729513631f51ae6b05eb9abed Author: Poul-Henning Kamp Date: Wed Jul 7 08:13:18 2021 +0000 Pre-process for FlexeLint diff --git a/tools/flint_skel.sh b/tools/flint_skel.sh index 9d45f87dd..d6d3031f1 100755 --- a/tools/flint_skel.sh +++ b/tools/flint_skel.sh @@ -6,12 +6,27 @@ if [ "x$1" = "x-ok" -a -f _.fl ] ; then exit 0 fi + d=$(dirname $0) -l=1 if [ $d = ../../tools ] ; then l=2 + IARG="-I. -I../../include -I../.." +else + l=1 + IARG="-I. -I../include -I.." fi +IARG="${IARG} -I/usr/local/include" + +# Flexelint do not grok the macro-stacking in pcre2.h +# Solve it by running it through CPP first. + +echo ' +#include "config.h" +#include +' | cc $IARG -E - | + sed '/^# [0-9]/d' > pcre2.h + flexelint \ -D__FLEXELINT__ \ $(if [ $l -eq 2 ] ; then echo ../../flint.lnt ; fi) \ @@ -19,18 +34,12 @@ flexelint \ flint.lnt \ -zero \ -I. \ - $(if [ $l -eq 2 ] ; then - echo -I../../include - echo -I../.. - else - echo -I../include - echo -I.. - fi - ) \ - -I/usr/local/include \ + ${IARG} \ $FLOPS \ 2>&1 | tee _.fl +rm -f pcre2.h + if [ -f _.fl.old ] ; then diff -u _.fl.old _.fl fi From phk at FreeBSD.org Wed Jul 7 10:18:04 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 7 Jul 2021 10:18:04 +0000 (UTC) Subject: [master] 7123e16d8 Found the root cause, clear flint bug. Message-ID: <20210707101804.EB26EB25CC@lists.varnish-cache.org> commit 7123e16d8013f6a447a8ef5561f2a7b9340d6f42 Author: Poul-Henning Kamp Date: Wed Jul 7 10:17:14 2021 +0000 Found the root cause, clear flint bug. diff --git a/flint.lnt b/flint.lnt index 0bc076fce..53126a529 100644 --- a/flint.lnt +++ b/flint.lnt @@ -9,6 +9,8 @@ //d__flexelint_v9__=1 +fan +-hm4 + /////////////////////////////////////////////////////////////////////// // electives //+e9* @@ -60,7 +62,12 @@ /////////////////////////////////////////////////////////////////////// --emacro((835),*) // A zero has been given as ___ argument to operator '___ +// This does not work with pcre2.h, the injected /* lint ... */ comments +// interfere with macro argument concatenation. Clear flexelint bug +// because it does not happen when run with -p +// -emacro((835),*) // A zero has been given as ___ argument to operator '___ + +-e835 // A zero has been given as ___ argument to operator '___ /////////////////////////////////////////////////////////////////////// // build/config related diff --git a/tools/flint_skel.sh b/tools/flint_skel.sh index d6d3031f1..dad674362 100755 --- a/tools/flint_skel.sh +++ b/tools/flint_skel.sh @@ -18,15 +18,6 @@ fi IARG="${IARG} -I/usr/local/include" -# Flexelint do not grok the macro-stacking in pcre2.h -# Solve it by running it through CPP first. - -echo ' -#include "config.h" -#include -' | cc $IARG -E - | - sed '/^# [0-9]/d' > pcre2.h - flexelint \ -D__FLEXELINT__ \ $(if [ $l -eq 2 ] ; then echo ../../flint.lnt ; fi) \ @@ -38,8 +29,6 @@ flexelint \ $FLOPS \ 2>&1 | tee _.fl -rm -f pcre2.h - if [ -f _.fl.old ] ; then diff -u _.fl.old _.fl fi From gquintard at users.noreply.github.com Tue Jul 13 07:16:09 2021 From: gquintard at users.noreply.github.com (guillaume quintard) Date: Tue, 13 Jul 2021 07:16:09 +0000 (UTC) Subject: [master] 99251be57 doc: Fix formatting in some examples Message-ID: <20210713071609.39A356042@lists.varnish-cache.org> commit 99251be57bbdb0eff41125a171c39db3c10c05f7 Author: Jordan Christiansen Date: Mon Jul 12 16:49:21 2021 -0500 doc: Fix formatting in some examples In RST, there needs to be an empty line after ::, otherwise the whole block is interpreted as a normal paragraph where newlines are not preserved. diff --git a/vmod/vmod_std.vcc b/vmod/vmod_std.vcc index a220b7872..733d41306 100644 --- a/vmod/vmod_std.vcc +++ b/vmod/vmod_std.vcc @@ -274,6 +274,7 @@ Only one of the *s*, *real* or *integer* arguments may be given or a VCL failure will be triggered. Examples:: + set beresp.ttl = std.duration("1w", 3600s); set beresp.ttl = std.duration(real=1.5); set beresp.ttl = std.duration(integer=10); @@ -300,6 +301,7 @@ Only one of the *s*, *real* or *integer* arguments may be given or a VCL failure will be triggered. Example:: + std.cache_req_body(std.bytes(something.somewhere, 10K)); std.cache_req_body(std.bytes(integer=10*1024)); std.cache_req_body(std.bytes(real=10.0*1024)); From martin at varnish-software.com Tue Jul 13 10:05:07 2021 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 13 Jul 2021 10:05:07 +0000 (UTC) Subject: [6.6] 9a96583bf Unbreak varnishtest expect_close Message-ID: <20210713100507.DDC0363380@lists.varnish-cache.org> commit 9a96583bf3878b5ddeb69e252c1a53e5fcc4a018 Author: Martin Blix Grydeland Date: Tue Jun 22 11:47:45 2021 +0200 Unbreak varnishtest expect_close The change to VTCP_Check() in 58a21da7736295e674ebb7e8ae9eee9529c083b0 broke expect_close in varnishtest. diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index dc286066a..78e7e880c 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -1496,7 +1496,7 @@ cmd_http_expect_close(CMD_ARGS) "Expected close: poll = %d, revents = 0x%x", i, fds[0].revents); i = read(hp->sess->fd, &c, 1); - if (VTCP_Check(i)) + if (i <= 0 && VTCP_Check(i)) break; if (i == 1 && vct_islws(c)) continue; From martin at varnish-software.com Tue Jul 13 10:05:07 2021 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 13 Jul 2021 10:05:07 +0000 (UTC) Subject: [6.6] 7d73bc843 Take content length into account on H/2 request bodies Message-ID: <20210713100508.0048563383@lists.varnish-cache.org> commit 7d73bc843ea9bff79e33773f1cfe2dffa7d03ea1 Author: Martin Blix Grydeland Date: Tue Jun 22 11:47:55 2021 +0200 Take content length into account on H/2 request bodies When receiving H/2 data frames, make sure to take the advertised content length into account, and fail appropriately if the combined sum of the data frames does not match the content length. diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index 782845209..ea5eb528d 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -134,6 +134,8 @@ struct h2_req { /* Where to wake this stream up */ struct worker *wrk; + ssize_t reqbody_bytes; + VTAILQ_ENTRY(h2_req) tx_list; h2_error error; }; diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index cc894950e..3597ec169 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -551,6 +551,7 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, struct req *req, struct h2_req *r2) { h2_error h2e; + ssize_t cl; ASSERT_RXTHR(h2); assert(r2->state == H2_S_OPEN); @@ -571,16 +572,24 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, // XXX: Have I mentioned H/2 Is hodge-podge ? http_CollectHdrSep(req->http, H_Cookie, "; "); // rfc7540,l,3114,3120 + cl = http_GetContentLength(req->http); + assert(cl >= -2); + if (cl == -2) { + VSLb(h2->vsl, SLT_Debug, "Non-parseable Content-Length"); + return (H2SE_PROTOCOL_ERROR); + } + if (req->req_body_status == NULL) { - if (!http_GetHdr(req->http, H_Content_Length, NULL)) + if (cl == -1) req->req_body_status = BS_EOF; else req->req_body_status = BS_LENGTH; + req->htc->content_length = cl; } else { /* A HEADER frame contained END_STREAM */ assert (req->req_body_status == BS_NONE); r2->state = H2_S_CLOS_REM; - if (http_GetContentLength(req->http) > 0) + if (cl > 0) return (H2CE_PROTOCOL_ERROR); //rfc7540,l,1838,1840 } @@ -736,6 +745,7 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) int w1 = 0, w2 = 0; char buf[4]; unsigned wi; + ssize_t cl; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); ASSERT_RXTHR(h2); @@ -754,6 +764,23 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) Lck_Unlock(&h2->sess->mtx); return (h2->error ? h2->error : r2->error); } + + r2->reqbody_bytes += h2->rxf_len; + if (h2->rxf_flags & H2FF_DATA_END_STREAM) + r2->state = H2_S_CLOS_REM; + cl = r2->req->htc->content_length; + if (cl >= 0 && (r2->reqbody_bytes > cl || + (r2->state >= H2_S_CLOS_REM && r2->reqbody_bytes != cl))) { + VSLb(h2->vsl, SLT_Debug, + "H2: stream %u: Received data and Content-Length" + " mismatch", h2->rxf_stream); + r2->error = H2SE_PROTOCOL_ERROR; // rfc7540,l,3150,3163 + if (r2->cond) + AZ(pthread_cond_signal(r2->cond)); + Lck_Unlock(&h2->sess->mtx); + return (H2SE_PROTOCOL_ERROR); + } + AZ(h2->mailcall); h2->mailcall = r2; h2->req0->r_window -= h2->rxf_len; @@ -772,6 +799,8 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) r2->r_window += wi; w2 = 1; } + + Lck_Unlock(&h2->sess->mtx); if (w1 || w2) { @@ -794,7 +823,7 @@ h2_vfp_body(struct vfp_ctx *vc, struct vfp_entry *vfe, void *ptr, ssize_t *lp) struct h2_req *r2; struct h2_sess *h2; unsigned l; - enum vfp_status retval = VFP_OK; + enum vfp_status retval; CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC); @@ -807,7 +836,6 @@ h2_vfp_body(struct vfp_ctx *vc, struct vfp_entry *vfe, void *ptr, ssize_t *lp) *lp = 0; Lck_Lock(&h2->sess->mtx); - assert (r2->state == H2_S_OPEN); r2->cond = &vc->wrk->cond; while (h2->mailcall != r2 && h2->error == 0 && r2->error == 0) AZ(Lck_CondWait(r2->cond, &h2->sess->mtx, 0)); @@ -830,12 +858,10 @@ h2_vfp_body(struct vfp_ctx *vc, struct vfp_entry *vfe, void *ptr, ssize_t *lp) Lck_Unlock(&h2->sess->mtx); return (VFP_OK); } - if (h2->rxf_len == 0) { - if (h2->rxf_flags & H2FF_DATA_END_STREAM) { - retval = VFP_END; - r2->state = H2_S_CLOS_REM; - } - } + if (h2->rxf_len == 0 && r2->state >= H2_S_CLOS_REM) + retval = VFP_END; + else + retval = VFP_OK; h2->mailcall = NULL; AZ(pthread_cond_signal(h2->cond)); } From martin at varnish-software.com Tue Jul 13 10:05:08 2021 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 13 Jul 2021 10:05:08 +0000 (UTC) Subject: [6.6] 087896bcc VSV00007 Test case for H2 smuggling attack Message-ID: <20210713100508.1FF5E63387@lists.varnish-cache.org> commit 087896bccd7e1f51c9571f3d71660499da644a29 Author: Martin Blix Grydeland Date: Tue Jun 22 11:47:58 2021 +0200 VSV00007 Test case for H2 smuggling attack diff --git a/bin/varnishtest/tests/f00007.vtc b/bin/varnishtest/tests/f00007.vtc new file mode 100644 index 000000000..23a2b7657 --- /dev/null +++ b/bin/varnishtest/tests/f00007.vtc @@ -0,0 +1,79 @@ +varnishtest "H/2 content length smuggling attack" + +server s1 { + rxreqhdrs + expect_close +} -start + +server s2 { + rxreqhdrs + expect_close +} -start + +server s3 { + rxreq + expect_close +} -start + +server s4 { + rxreq + expect req.body == "A" + txresp +} -start + +varnish v1 -vcl+backend { + import vtc; + sub vcl_backend_fetch { + if (bereq.url == "/1") { + set bereq.backend = s1; + } else if (bereq.url == "/2") { + set bereq.backend = s2; + } else if (bereq.url == "/3") { + set bereq.backend = s3; + } else { + set bereq.backend = s4; + } + } +} -start + +varnish v1 -cliok "param.set feature +http2" +varnish v1 -cliok "param.set debug +syncvsl" + +client c1 { + stream 1 { + txreq -req POST -url /1 -hdr "content-length" "1" -nostrend + txdata -data "AGET /FAIL HTTP/1.1\r\n\r\n" + rxrst + expect rst.err == PROTOCOL_ERROR + } -run +} -run + +client c2 { + stream 1 { + txreq -req POST -url /2 -hdr "content-length" "1" -nostrend + txdata -data "AGET /FAIL HTTP/1.1\r\n\r\n" -nostrend + txdata + rxrst + expect rst.err == PROTOCOL_ERROR + } -run +} -run + +client c3 { + stream 1 { + txreq -req POST -url /3 -hdr "content-length" "1" -nostrend + txdata -data "A" -nostrend + txdata -data "GET /FAIL HTTP/1.1\r\n\r\n" + rxrst + expect rst.err == PROTOCOL_ERROR + } -run +} -run + +client c4 { + stream 1 { + txreq -req POST -url /4 -hdr "content-length" "1" -nostrend + txdata -data "A" -nostrend + txdata + rxresp + expect resp.status == 200 + } -run +} -run From martin at varnish-software.com Tue Jul 13 10:05:08 2021 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 13 Jul 2021 10:05:08 +0000 (UTC) Subject: [6.6] 8e9934082 Add VSV00007 to the changelog Message-ID: <20210713100508.3F9686338F@lists.varnish-cache.org> commit 8e99340827efdb527dfc1e014d289f5caa430889 Author: Martin Blix Grydeland Date: Thu Jul 1 16:26:16 2021 +0200 Add VSV00007 to the changelog diff --git a/doc/changes.rst b/doc/changes.rst index b63db87c8..6640e7e82 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -31,6 +31,14 @@ http://varnish-cache.org/docs/trunk/whats-new/index.html and via individual releases. These documents are updated as part of the release process. +================================ +Varnish Cache 6.6.1 (unreleased) +================================ + +* Fix an HTTP/2.0 request smuggling vulnerability. (VSV00007_) + +.. _VSV00007: https://varnish-cache.org/security/VSV00007.html + ================================ Varnish Cache 6.6.0 (2021-03-15) ================================ From martin at varnish-software.com Tue Jul 13 10:05:08 2021 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 13 Jul 2021 10:05:08 +0000 (UTC) Subject: [6.6] e6a8c8609 Prepare for 6.6.1 Message-ID: <20210713100508.5C8EA63393@lists.varnish-cache.org> commit e6a8c860944c4f6a7e1af9f40674ea78bbdcdc66 Author: Martin Blix Grydeland Date: Thu Jul 1 16:27:33 2021 +0200 Prepare for 6.6.1 diff --git a/configure.ac b/configure.ac index 0b4c32d2d..5420bb919 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ AC_PREREQ(2.59) AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS Copyright (c) 2006-2021 Varnish Software]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [6.6.0], [varnish-dev at varnish-cache.org]) +AC_INIT([Varnish], [6.6.1], [varnish-dev at varnish-cache.org]) AC_CONFIG_SRCDIR(include/miniobj.h) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/changes.rst b/doc/changes.rst index 6640e7e82..be9d413c0 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -32,7 +32,7 @@ individual releases. These documents are updated as part of the release process. ================================ -Varnish Cache 6.6.1 (unreleased) +Varnish Cache 6.6.1 (2021-07-13) ================================ * Fix an HTTP/2.0 request smuggling vulnerability. (VSV00007_) From martin at varnish-software.com Tue Jul 13 10:05:08 2021 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 13 Jul 2021 10:05:08 +0000 (UTC) Subject: [6.6] ce2a4a2f8 Install sudo on alpine Message-ID: <20210713100508.7AA7B63399@lists.varnish-cache.org> commit ce2a4a2f828272fc48a57de21c955095465efeaf Author: Martin Blix Grydeland Date: Fri Jul 2 14:47:45 2021 +0200 Install sudo on alpine diff --git a/.circleci/make-apk-packages.sh b/.circleci/make-apk-packages.sh index 26b672c2c..a8b678f8f 100755 --- a/.circleci/make-apk-packages.sh +++ b/.circleci/make-apk-packages.sh @@ -2,7 +2,7 @@ set -eux -apk add -q --no-progress --update tar alpine-sdk +apk add -q --no-progress --update tar alpine-sdk sudo echo "PARAM_RELEASE: $PARAM_RELEASE" echo "PARAM_DIST: $PARAM_DIST" From martin at varnish-software.com Tue Jul 13 10:05:08 2021 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 13 Jul 2021 10:05:08 +0000 (UTC) Subject: [6.6] b920ea690 Disable building on alpine Message-ID: <20210713100508.9651C6339D@lists.varnish-cache.org> commit b920ea6901997e0c6c2ed2fd5cf824c56b711245 Author: Martin Blix Grydeland Date: Fri Jul 2 15:34:10 2021 +0200 Disable building on alpine diff --git a/.circleci/config.yml b/.circleci/config.yml index 4eb783a6b..1f2562b9c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -439,22 +439,6 @@ workflows: image: centos:8 ext: rpm <<: *pkg_req - - package: - name: x64-alpine-3 - dist: alpine - release: "3" - arch: x64 - image: alpine:3 - ext: apk - <<: *pkg_req - - package: - name: aarch64-alpine-3 - dist: alpine - release: "3" - arch: aarch64 - image: arm64v8/alpine:3 - ext: apk - <<: *pkg_req - collect_packages: requires: - x64-ubuntu-xenial @@ -473,6 +457,3 @@ workflows: - aarch64-centos-7 - x64-centos-8 - aarch64-centos-8 - - x64-alpine-3 - - aarch64-alpine-3 - From martin at varnish-software.com Tue Jul 13 10:06:08 2021 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 13 Jul 2021 10:06:08 +0000 (UTC) Subject: [6.0] 6e27d4dee Unbreak varnishtest expect_close Message-ID: <20210713100608.611CF63EEA@lists.varnish-cache.org> commit 6e27d4dee58e0f3e63c014464cd663b6985e8c53 Author: Martin Blix Grydeland Date: Tue Jun 22 11:47:45 2021 +0200 Unbreak varnishtest expect_close The change to VTCP_Check() in 58a21da7736295e674ebb7e8ae9eee9529c083b0 broke expect_close in varnishtest. diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index da4747872..0ea454a83 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -1553,7 +1553,7 @@ cmd_http_expect_close(CMD_ARGS) "Expected close: poll = %d, revents = 0x%x", i, fds[0].revents); i = read(hp->fd, &c, 1); - if (VTCP_Check(i)) + if (i <= 0 && VTCP_Check(i)) break; if (i == 1 && vct_islws(c)) continue; From martin at varnish-software.com Tue Jul 13 10:06:08 2021 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 13 Jul 2021 10:06:08 +0000 (UTC) Subject: [6.0] 9be22198e Take content length into account on H/2 request bodies Message-ID: <20210713100608.7F82963EED@lists.varnish-cache.org> commit 9be22198e258d0e7a5c41f4291792214a29405cf Author: Martin Blix Grydeland Date: Tue Jun 22 11:47:55 2021 +0200 Take content length into account on H/2 request bodies When receiving H/2 data frames, make sure to take the advertised content length into account, and fail appropriately if the combined sum of the data frames does not match the content length. diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index c377d03aa..205b96ccb 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -131,6 +131,8 @@ struct h2_req { /* Where to wake this stream up */ struct worker *wrk; + ssize_t reqbody_bytes; + VTAILQ_ENTRY(h2_req) tx_list; h2_error error; diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index cb35bb487..98f5dc4f3 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -546,7 +546,7 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, struct req *req, struct h2_req *r2) { h2_error h2e; - const char *b; + ssize_t cl; ASSERT_RXTHR(h2); assert(r2->state == H2_S_OPEN); @@ -572,14 +572,24 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, // XXX: Have I mentioned H/2 Is hodge-podge ? http_CollectHdrSep(req->http, H_Cookie, "; "); // rfc7540,l,3114,3120 + cl = http_GetContentLength(req->http); + assert(cl >= -2); + if (cl == -2) { + VSLb(h2->vsl, SLT_Debug, "Non-parseable Content-Length"); + return (H2SE_PROTOCOL_ERROR); + } + if (req->req_body_status == REQ_BODY_INIT) { - if (!http_GetHdr(req->http, H_Content_Length, &b)) + if (cl == -1) req->req_body_status = REQ_BODY_WITHOUT_LEN; else req->req_body_status = REQ_BODY_WITH_LEN; + req->htc->content_length = cl; } else { + /* A HEADER frame contained END_STREAM */ assert (req->req_body_status == REQ_BODY_NONE); - if (http_GetContentLength(req->http) > 0) + r2->state = H2_S_CLOS_REM; + if (cl > 0) return (H2CE_PROTOCOL_ERROR); //rfc7540,l,1838,1840 } @@ -736,6 +746,7 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) int w1 = 0, w2 = 0; char buf[4]; unsigned wi; + ssize_t cl; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); ASSERT_RXTHR(h2); @@ -754,6 +765,23 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) Lck_Unlock(&h2->sess->mtx); return (h2->error ? h2->error : r2->error); } + + r2->reqbody_bytes += h2->rxf_len; + if (h2->rxf_flags & H2FF_DATA_END_STREAM) + r2->state = H2_S_CLOS_REM; + cl = r2->req->htc->content_length; + if (cl >= 0 && (r2->reqbody_bytes > cl || + (r2->state >= H2_S_CLOS_REM && r2->reqbody_bytes != cl))) { + VSLb(h2->vsl, SLT_Debug, + "H2: stream %u: Received data and Content-Length" + " mismatch", h2->rxf_stream); + r2->error = H2SE_PROTOCOL_ERROR; // rfc7540,l,3150,3163 + if (r2->cond) + AZ(pthread_cond_signal(r2->cond)); + Lck_Unlock(&h2->sess->mtx); + return (H2SE_PROTOCOL_ERROR); + } + AZ(h2->mailcall); h2->mailcall = r2; h2->req0->r_window -= h2->rxf_len; @@ -772,6 +800,8 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) r2->r_window += wi; w2 = 1; } + + Lck_Unlock(&h2->sess->mtx); if (w1 || w2) { @@ -794,7 +824,7 @@ h2_vfp_body(struct vfp_ctx *vc, struct vfp_entry *vfe, void *ptr, ssize_t *lp) struct h2_req *r2; struct h2_sess *h2; unsigned l; - enum vfp_status retval = VFP_OK; + enum vfp_status retval; CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC); @@ -807,7 +837,6 @@ h2_vfp_body(struct vfp_ctx *vc, struct vfp_entry *vfe, void *ptr, ssize_t *lp) *lp = 0; Lck_Lock(&h2->sess->mtx); - assert (r2->state == H2_S_OPEN); r2->cond = &vc->wrk->cond; while (h2->mailcall != r2 && h2->error == 0 && r2->error == 0) AZ(Lck_CondWait(r2->cond, &h2->sess->mtx, 0)); @@ -830,12 +859,10 @@ h2_vfp_body(struct vfp_ctx *vc, struct vfp_entry *vfe, void *ptr, ssize_t *lp) Lck_Unlock(&h2->sess->mtx); return (VFP_OK); } - if (h2->rxf_len == 0) { - if (h2->rxf_flags & H2FF_DATA_END_STREAM) { - retval = VFP_END; - r2->state = H2_S_CLOS_REM; - } - } + if (h2->rxf_len == 0 && r2->state >= H2_S_CLOS_REM) + retval = VFP_END; + else + retval = VFP_OK; h2->mailcall = NULL; AZ(pthread_cond_signal(h2->cond)); } From martin at varnish-software.com Tue Jul 13 10:06:08 2021 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 13 Jul 2021 10:06:08 +0000 (UTC) Subject: [6.0] cfb79576b VSV00007 Test case for H2 smuggling attack Message-ID: <20210713100608.9C99463EF2@lists.varnish-cache.org> commit cfb79576b229c6c4d7d39bbe05fec977020fa8c9 Author: Martin Blix Grydeland Date: Tue Jun 22 11:47:58 2021 +0200 VSV00007 Test case for H2 smuggling attack diff --git a/bin/varnishtest/tests/f00007.vtc b/bin/varnishtest/tests/f00007.vtc new file mode 100644 index 000000000..1cf45aad1 --- /dev/null +++ b/bin/varnishtest/tests/f00007.vtc @@ -0,0 +1,82 @@ +varnishtest "H/2 content length smuggling attack" + +server s1 { + rxreqhdrs + expect_close +} -start + +server s2 { + rxreqhdrs + expect_close +} -start + +server s3 { + rxreq + expect_close +} -start + +server s4 { + rxreq + expect req.body == "A" + txresp +} -start + +varnish v1 -vcl+backend { + import vtc; + sub vcl_backend_fetch { + if (bereq.url == "/1") { + set bereq.backend = s1; + } else if (bereq.url == "/2") { + set bereq.backend = s2; + } else if (bereq.url == "/3") { + set bereq.backend = s3; + } else { + set bereq.backend = s4; + } + } +} -start + +varnish v1 -cliok "param.set feature +http2" +varnish v1 -cliok "param.set debug +syncvsl" + +client c1 { + stream 1 { + txreq -req POST -url /1 -hdr "content-length" "1" -nostrend + txdata -data "AGET /FAIL HTTP/1.1\r\n\r\n" + rxrst + expect rst.err == PROTOCOL_ERROR + } -run +} -run + +client c2 { + stream 1 { + txreq -req POST -url /2 -hdr "content-length" "1" -nostrend + txdata -data "AGET /FAIL HTTP/1.1\r\n\r\n" -nostrend + txdata + rxrst + expect rst.err == PROTOCOL_ERROR + } -run +} -run + +client c3 { + stream 1 { + txreq -req POST -url /3 -hdr "content-length" "1" -nostrend + txdata -data "A" -nostrend + txdata -data "GET /FAIL HTTP/1.1\r\n\r\n" + rxwinup + rxrst + expect rst.err == PROTOCOL_ERROR + } -run +} -run + +client c4 { + stream 1 { + txreq -req POST -url /4 -hdr "content-length" "1" -nostrend + txdata -data "A" -nostrend + txdata + rxwinup + rxwinup + rxresp + expect resp.status == 200 + } -run +} -run From martin at varnish-software.com Tue Jul 13 10:06:08 2021 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 13 Jul 2021 10:06:08 +0000 (UTC) Subject: [6.0] 6941a1793 Add VSV00007 to the changelog Message-ID: <20210713100608.B8D5163EF6@lists.varnish-cache.org> commit 6941a17937fd5a5bbbb21ca4f40be1cb272599c5 Author: Martin Blix Grydeland Date: Thu Jul 1 16:14:45 2021 +0200 Add VSV00007 to the changelog diff --git a/doc/changes.rst b/doc/changes.rst index caedeefe3..30dc3e4f8 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -76,6 +76,8 @@ Varnish Cache 6.0.8 (YYYY-MM-DD) * Fix an issue where the `return(error)` status and reason are kept when doing a backend retry. (3525_) +* Fix an HTTP/2.0 request smuggling vulnerability. (VSV00007_) + .. _3556: https://github.com/varnishcache/varnish-cache/issues/3556 .. _3593: https://github.com/varnishcache/varnish-cache/pull/3593 .. _3537: https://github.com/varnishcache/varnish-cache/pull/3537 @@ -94,6 +96,7 @@ Varnish Cache 6.0.8 (YYYY-MM-DD) .. _3535: https://github.com/varnishcache/varnish-cache/issues/3535 .. _3546: https://github.com/varnishcache/varnish-cache/pull/3546 .. _3525: https://github.com/varnishcache/varnish-cache/issues/3525 +.. _VSV00007: https://varnish-cache.org/security/VSV00007.html ================================ Varnish Cache 6.0.7 (2020-11-06) From martin at varnish-software.com Tue Jul 13 10:06:08 2021 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 13 Jul 2021 10:06:08 +0000 (UTC) Subject: [6.0] 97e54ada6 Prepare for 6.0.8 Message-ID: <20210713100608.D55CF63EFB@lists.varnish-cache.org> commit 97e54ada6ac578af332e52b44d2038bb4fa4cd4a Author: Martin Blix Grydeland Date: Thu Jul 1 16:15:39 2021 +0200 Prepare for 6.0.8 diff --git a/configure.ac b/configure.ac index a3d28c733..bbcbad126 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ AC_PREREQ(2.59) AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS Copyright (c) 2006-2020 Varnish Software]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [6.0.7], [varnish-dev at varnish-cache.org]) +AC_INIT([Varnish], [6.0.8], [varnish-dev at varnish-cache.org]) AC_CONFIG_SRCDIR(include/miniobj.h) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/changes.rst b/doc/changes.rst index 30dc3e4f8..de40d44ff 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -27,7 +27,7 @@ individual releases. These documents are updated as part of the release process. ================================ -Varnish Cache 6.0.8 (YYYY-MM-DD) +Varnish Cache 6.0.8 (2021-07-13) ================================ * Fix an issue where a backend fetch can stall after a client has From fgsch at lodoss.net Thu Jul 15 13:59:06 2021 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 15 Jul 2021 13:59:06 +0000 (UTC) Subject: [master] 1d04fd09b Update xcode and add missing package Message-ID: <20210715135906.4B018A2492@lists.varnish-cache.org> commit 1d04fd09bbea51258efe7ad9cabff234a8db565c Author: Federico G. Schwindt Date: Thu Jul 15 14:57:18 2021 +0100 Update xcode and add missing package diff --git a/.travis.yml b/.travis.yml index be91d56d2..cc553fcbe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,6 +58,7 @@ jobs: - nghttp2 - python3-docutils - python3-sphinx + - libpcre2-dev env: ASAN=1 UBSAN=1 before_script: - | @@ -70,7 +71,7 @@ jobs: - ./configure --enable-maintainer-mode --with-unwind --enable-debugging-symbols --disable-stack-protector --with-persistent-storage --enable-asan --enable-ubsan - stage: test os: osx - osx_image: xcode12 + osx_image: xcode12.5 compiler: clang addons: homebrew: From phk at FreeBSD.org Mon Jul 19 08:39:07 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 19 Jul 2021 08:39:07 +0000 (UTC) Subject: [master] 6cef6e770 Make sure the vav is always freed. Message-ID: <20210719083907.F37FC97A48@lists.varnish-cache.org> commit 6cef6e7708a30554252f77215d7c872ce46ce865 Author: Poul-Henning Kamp Date: Mon Jul 19 08:22:45 2021 +0000 Make sure the vav is always freed. Spotted by: Coverity diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 1a4988cbb..a73cc0b87 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -259,6 +259,8 @@ macro_cat(struct vtclog *vl, struct vsb *vsb, const char *b, const char *e) } AZ(pthread_mutex_unlock(¯o_mtx)); + VAV_Free(argv); + if (err != NULL) vtc_fatal(vl, "Macro ${%.*s} failed: %s", (int)(e - b), b, err); @@ -273,7 +275,6 @@ macro_cat(struct vtclog *vl, struct vsb *vsb, const char *b, const char *e) VSB_cat(vsb, retval); free(retval); - VAV_Free(argv); } struct vsb * From phk at FreeBSD.org Mon Jul 19 15:45:06 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 19 Jul 2021 15:45:06 +0000 (UTC) Subject: [master] 30a2df725 Silence FlexeLint Message-ID: <20210719154506.D2B8EAA1CF@lists.varnish-cache.org> commit 30a2df725bbb701751697f40cb1f14e36707daa2 Author: Poul-Henning Kamp Date: Mon Jul 19 15:37:14 2021 +0000 Silence FlexeLint diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index d270b0ef1..eda2ff1f0 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -82,13 +82,15 @@ vcc_vcl_met2c(struct vsb *vsb, unsigned method) int d = 0; //lint -e{774} Boolean within 'if' always evaluates to False -#define VCL_MET_MAC(l,U,t,b) \ - if (method & VCL_MET_##U) { \ - if (d) \ - VSB_putc(vsb, '|'); \ - VSB_cat(vsb, "VCL_MET_" #U); \ - d = 1; \ - } +#define VCL_MET_MAC(l,U,t,b) \ + do { \ + if (method & VCL_MET_##U) { \ + if (d) \ + VSB_putc(vsb, '|'); \ + VSB_cat(vsb, "VCL_MET_" #U); \ + d = 1; \ + } \ + } while (0); #include "tbl/vcl_returns.h" AN(d); } From phk at FreeBSD.org Mon Jul 19 15:45:06 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 19 Jul 2021 15:45:06 +0000 (UTC) Subject: [master] 0012ea924 Silence flexelint warnings Message-ID: <20210719154506.EE0CBAA1D2@lists.varnish-cache.org> commit 0012ea924867c48cbb1f8154d2c0f03a33d89a43 Author: Poul-Henning Kamp Date: Mon Jul 19 15:44:20 2021 +0000 Silence flexelint warnings diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c index 70d8d1280..67da6f2f4 100644 --- a/bin/varnishd/storage/storage_simple.c +++ b/bin/varnishd/storage/storage_simple.c @@ -196,11 +196,13 @@ sml_slim(struct worker *wrk, struct objcore *oc) o = sml_getobj(wrk, oc); CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC); -#define OBJ_AUXATTR(U, l) \ - if (o->aa_##l != NULL) { \ - sml_stv_free(stv, o->aa_##l); \ - o->aa_##l = NULL; \ - } +#define OBJ_AUXATTR(U, l) \ + do { \ + if (o->aa_##l != NULL) { \ + sml_stv_free(stv, o->aa_##l); \ + o->aa_##l = NULL; \ + } \ + } while (0); #include "tbl/obj_attr.h" VTAILQ_FOREACH_SAFE(st, &o->list, list, stn) { @@ -688,8 +690,10 @@ SML_panic(struct vsb *vsb, const struct objcore *oc) VSB_printf(vsb, "%s = {len=%u, ptr=%p},\n", \ #U, o->va_##l##_len, o->va_##l); -#define OBJ_AUXATTR(U, l) \ - if (o->aa_##l != NULL) sml_panic_st(vsb, #U, o->aa_##l); +#define OBJ_AUXATTR(U, l) \ + do { \ + if (o->aa_##l != NULL) sml_panic_st(vsb, #U, o->aa_##l);\ + } while(0); #include "tbl/obj_attr.h" From phk at FreeBSD.org Mon Jul 19 15:55:07 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 19 Jul 2021 15:55:07 +0000 (UTC) Subject: [master] 4a9238404 Silence FlexeLint Message-ID: <20210719155507.AF77AAA907@lists.varnish-cache.org> commit 4a9238404955a8bf6916d4b9982fa6c985a1d007 Author: Poul-Henning Kamp Date: Mon Jul 19 15:54:36 2021 +0000 Silence FlexeLint diff --git a/vmod/vmod_blob.c b/vmod/vmod_blob.c index 98a1527cd..fa05d1c48 100644 --- a/vmod/vmod_blob.c +++ b/vmod/vmod_blob.c @@ -117,7 +117,10 @@ static char empty[1] = { '\0' }; static enum encoding parse_encoding(VCL_ENUM e) { -#define VMODENUM(n) if (e == VENUM(n)) return (n); +#define VMODENUM(n) \ + do { \ + if (e == VENUM(n)) return (n); \ + } while (0); #include "vmod_blob_tbl_encodings.h" WRONG("illegal encoding enum"); } From phk at FreeBSD.org Mon Jul 19 16:04:07 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 19 Jul 2021 16:04:07 +0000 (UTC) Subject: [master] 3690420a7 One more Flexelint silencing Message-ID: <20210719160407.82DB4AAE7B@lists.varnish-cache.org> commit 3690420a7de6ac9458fb0feb30d6952d2025a0cb Author: Poul-Henning Kamp Date: Mon Jul 19 16:03:22 2021 +0000 One more Flexelint silencing diff --git a/vmod/vmod_blob.c b/vmod/vmod_blob.c index fa05d1c48..393f7fb23 100644 --- a/vmod/vmod_blob.c +++ b/vmod/vmod_blob.c @@ -128,7 +128,10 @@ parse_encoding(VCL_ENUM e) static enum case_e parse_case(VCL_ENUM e) { -#define VMODENUM(n) if (e == VENUM(n)) return (n); +#define VMODENUM(n) \ + do { \ + if (e == VENUM(n)) return (n); \ + } while (0); #include "vmod_blob_tbl_case.h" WRONG("illegal case enum"); }