From phk at FreeBSD.org Wed Jun 1 06:52:05 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 1 Jun 2022 06:52:05 +0000 (UTC) Subject: [master] 0c698c598 Solve (massive-parallel-run-)race condition with -n argument. Message-ID: <20220601065205.6337F108C74@lists.varnish-cache.org> commit 0c698c5984e60fd498f060b65991ae0625bf38f5 Author: Poul-Henning Kamp Date: Wed Jun 1 06:50:43 2022 +0000 Solve (massive-parallel-run-)race condition with -n argument. diff --git a/bin/varnishtest/tests/p00000.vtc b/bin/varnishtest/tests/p00000.vtc index 05c308215..f523ef82c 100644 --- a/bin/varnishtest/tests/p00000.vtc +++ b/bin/varnishtest/tests/p00000.vtc @@ -3,7 +3,7 @@ varnishtest "Test Basic persistence" feature persistent_storage shell -err -expect {-spersistent has been deprecated} { - varnishd -spersistent -f '' + varnishd -spersistent -f '' -n ${tmpdir} } server s1 { From phk at FreeBSD.org Wed Jun 1 07:32:04 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 1 Jun 2022 07:32:04 +0000 (UTC) Subject: [master] b98f017a3 Silence FlexeLint Message-ID: <20220601073204.DD7E610A051@lists.varnish-cache.org> commit b98f017a36c516a1248d34666e48fc39f6222cb9 Author: Poul-Henning Kamp Date: Wed Jun 1 07:31:16 2022 +0000 Silence FlexeLint diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index 6a148a5bb..30871a0c0 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -359,9 +359,10 @@ static void term_match_text(struct process *pp, const char *lin, const char *col, const char *re) { - int i, x, y, l, err, erroff; + int i, l, err, erroff; struct vsb *vsb, re_vsb[1]; size_t len; + ssize_t x, y; vre_t *vre; char errbuf[VRE_ERROR_LEN]; @@ -370,12 +371,12 @@ term_match_text(struct process *pp, y = strtoul(lin, NULL, 0); if (y < 0 || y > pp->nlin) - vtc_fatal(pp->vl, "YYY %d nlin %d", y, pp->nlin); + vtc_fatal(pp->vl, "YYY %zd nlin %d", y, pp->nlin); x = strtoul(col, NULL, 0); for(l = 0; l < 10 && x > pp->ncol; l++) // wait for screen change usleep(100000); if (x < 0 || x > pp->ncol) - vtc_fatal(pp->vl, "XXX %d ncol %d", x, pp->ncol); + vtc_fatal(pp->vl, "XXX %zd ncol %d", x, pp->ncol); if (x) x--; From phk at FreeBSD.org Wed Jun 1 07:44:04 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 1 Jun 2022 07:44:04 +0000 (UTC) Subject: [master] 55c4a928f Remove wrong '&' Message-ID: <20220601074404.F124A10A921@lists.varnish-cache.org> commit 55c4a928f25783bb6d2d5105b46d40c92c5747c0 Author: Poul-Henning Kamp Date: Wed Jun 1 07:43:29 2022 +0000 Remove wrong '&' diff --git a/include/tbl/params.h b/include/tbl/params.h index 5c69d0438..b53cbc478 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1736,7 +1736,7 @@ PARAM_ALIAS(deprecated_dummy, debug) */ #define PARAM_BITS(nm, fld, def, descr) \ - PARAM(nm, fld, nm, tweak_ ## nm, &mgt_param.fld, NULL, NULL, \ + PARAM(nm, fld, nm, tweak_ ## nm, mgt_param.fld, NULL, NULL, \ def, NULL, descr) PARAM_PRE From phk at FreeBSD.org Wed Jun 1 09:14:05 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 1 Jun 2022 09:14:05 +0000 (UTC) Subject: [master] 1a75354bc Move the initialization of stevedores to the worker process, right before privileges are dropped. Message-ID: <20220601091405.9EB5610F265@lists.varnish-cache.org> commit 1a75354bcc18b0c8c18670d54eb0720d7691ffe4 Author: Poul-Henning Kamp Date: Wed Jun 1 09:13:20 2022 +0000 Move the initialization of stevedores to the worker process, right before privileges are dropped. diff --git a/bin/varnishd/common/heritage.h b/bin/varnishd/common/heritage.h index af312b265..7ef4b04ba 100644 --- a/bin/varnishd/common/heritage.h +++ b/bin/varnishd/common/heritage.h @@ -105,7 +105,6 @@ void MCH_Fd_Inherit(int fd, const char *what); #define ARGV_ERR(...) \ do { \ - ASSERT_MGT(); \ fprintf(stderr, "Error: " __VA_ARGS__); \ fprintf(stderr, "(-? gives usage)\n"); \ exit(2); \ diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h index 2ca6a8594..4d54c86c0 100644 --- a/bin/varnishd/mgt/mgt.h +++ b/bin/varnishd/mgt/mgt.h @@ -220,6 +220,7 @@ char **MGT_NamedArg(const char *, const char **, const char *); extern const char *mgt_stv_h2_rxbuf; void STV_Config(const char *spec); void STV_Config_Transient(void); +void STV_Init(void); /* mgt_vcc.c */ void mgt_DumpBuiltin(void); diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index b4ebfba41..da4308c81 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -380,6 +380,8 @@ mgt_launch_child(struct cli *cli) vext_load(); + STV_Init(); + VJ_subproc(JAIL_SUBPROC_WORKER); heritage.proc_vsmw = VSMW_New(heritage.vsm_fd, 0640, "_.index"); diff --git a/bin/varnishd/storage/mgt_stevedore.c b/bin/varnishd/storage/mgt_stevedore.c index c593c4d9d..e646d2669 100644 --- a/bin/varnishd/storage/mgt_stevedore.c +++ b/bin/varnishd/storage/mgt_stevedore.c @@ -46,7 +46,12 @@ #include "storage/storage.h" -static VTAILQ_HEAD(, stevedore) stevedores = +VTAILQ_HEAD(stevedore_head, stevedore); + +static struct stevedore_head pre_stevedores = + VTAILQ_HEAD_INITIALIZER(pre_stevedores); + +static struct stevedore_head stevedores = VTAILQ_HEAD_INITIALIZER(stevedores); /* Name of transient storage */ @@ -66,8 +71,10 @@ STV__iter(struct stevedore ** const pp) CHECK_OBJ_ORNULL(*pp, STEVEDORE_MAGIC); if (*pp != NULL) *pp = VTAILQ_NEXT(*pp, list); - else + else if (!VTAILQ_EMPTY(&stevedores)) *pp = VTAILQ_FIRST(&stevedores); + else + *pp = VTAILQ_FIRST(&pre_stevedores); return (*pp != NULL); } @@ -166,34 +173,12 @@ static const struct choice STV_choice[] = { { NULL, NULL } }; -static void -stv_check_ident(const char *spec, const char *ident) -{ - struct stevedore *stv; - unsigned found = 0; - - if (!strcmp(ident, TRANSIENT_STORAGE)) - found = (stv_transient != NULL); - else { - STV_Foreach(stv) - if (!strcmp(stv->ident, ident)) { - found = 1; - break; - } - } - - if (found) - ARGV_ERR("(-s %s) '%s' is already defined\n", spec, ident); -} - void STV_Config(const char *spec) { char **av, buf[8]; const char *ident; struct stevedore *stv; - const struct stevedore *stv2; - int ac; static unsigned seq = 0; av = MGT_NamedArg(spec, &ident, "-s"); @@ -202,47 +187,24 @@ STV_Config(const char *spec) if (av[1] == NULL) ARGV_ERR("-s argument lacks strategy {malloc, file, ...}\n"); - for (ac = 0; av[ac + 2] != NULL; ac++) - continue; - - stv2 = MGT_Pick(STV_choice, av[1], "storage"); - AN(stv2); - /* Append strategy to ident string */ VSB_printf(vident, ",-s%s", av[1]); - av += 2; + if (ident == NULL) { + bprintf(buf, "s%u", seq++); + ident = strdup(buf); + } + + VTAILQ_FOREACH(stv, &pre_stevedores, list) + if (!strcmp(stv->ident, ident)) + ARGV_ERR("(-s %s) '%s' is already defined\n", spec, ident); - CHECK_OBJ_NOTNULL(stv2, STEVEDORE_MAGIC); ALLOC_OBJ(stv, STEVEDORE_MAGIC); AN(stv); - - *stv = *stv2; - AN(stv->name); - - if (ident) { - stv->ident = ident; - } else { - bprintf(buf, "s%u", seq++); - stv->ident = strdup(buf); - } - AN(stv->ident); - stv_check_ident(spec, stv->ident); - - if (stv->init != NULL) - stv->init(stv, ac, av); - else if (ac != 0) - ARGV_ERR("(-s %s) too many arguments\n", stv->name); - - AN(stv->allocobj); - AN(stv->methods); - - if (!strcmp(stv->ident, TRANSIENT_STORAGE)) { - AZ(stv_transient); - stv_transient = stv; - } else - VTAILQ_INSERT_TAIL(&stevedores, stv, list); - /* NB: Do not free av, stevedore gets to keep it */ + stv->av = av; + stv->ident = ident; + stv->name = av[1]; + VTAILQ_INSERT_TAIL(&pre_stevedores, stv, list); } /*--------------------------------------------------------------------*/ @@ -250,11 +212,67 @@ STV_Config(const char *spec) void STV_Config_Transient(void) { + struct stevedore *stv; ASSERT_MGT(); VCLS_AddFunc(mgt_cls, MCF_AUTH, cli_stv); - if (stv_transient == NULL) - STV_Config(TRANSIENT_STORAGE "=default"); + STV_Foreach(stv) { + if (!strcmp(stv->ident, TRANSIENT_STORAGE)) + return; + } + STV_Config(TRANSIENT_STORAGE "=default"); +} + +/*-------------------------------------------------------------------- + * Initialize configured stevedores in the worker process + */ + +void +STV_Init(void) +{ + char **av; + const char *ident; + struct stevedore *stv; + const struct stevedore *stv2; + int ac; + + while (!VTAILQ_EMPTY(&pre_stevedores)) { + stv = VTAILQ_FIRST(&pre_stevedores); + VTAILQ_REMOVE(&pre_stevedores, stv, list); + CHECK_OBJ_NOTNULL(stv, STEVEDORE_MAGIC); + AN(stv->av); + av = stv->av; + AN(stv->ident); + ident = stv->ident; + + for (ac = 0; av[ac + 2] != NULL; ac++) + continue; + + stv2 = MGT_Pick(STV_choice, av[1], "storage"); + CHECK_OBJ_NOTNULL(stv2, STEVEDORE_MAGIC); + *stv = *stv2; + AN(stv->name); + + av += 2; + + stv->ident = ident; + stv->av = av; + + if (stv->init != NULL) + stv->init(stv, ac, av); + else if (ac != 0) + ARGV_ERR("(-s %s) too many arguments\n", stv->name); + + AN(stv->allocobj); + AN(stv->methods); + + if (!strcmp(stv->ident, TRANSIENT_STORAGE)) { + AZ(stv_transient); + stv_transient = stv; + } else + VTAILQ_INSERT_TAIL(&stevedores, stv, list); + /* NB: Do not free av, stevedore gets to keep it */ + } AN(stv_transient); VTAILQ_INSERT_TAIL(&stevedores, stv_transient, list); } diff --git a/bin/varnishd/storage/mgt_storage_persistent.c b/bin/varnishd/storage/mgt_storage_persistent.c index fdd4419c8..e70c8414f 100644 --- a/bin/varnishd/storage/mgt_storage_persistent.c +++ b/bin/varnishd/storage/mgt_storage_persistent.c @@ -145,11 +145,8 @@ smp_mgt_init(struct stevedore *parent, int ac, char * const *av) void *target; int i, mmap_flags; - ASSERT_MGT(); - AZ(av[ac]); - #ifdef HAVE_SYS_PERSONALITY_H i = personality(0xffffffff); /* Fetch old personality. */ if (!(i & ADDR_NO_RANDOMIZE)) { diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index b9017f579..882ab82b8 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -114,6 +114,7 @@ struct stevedore { void *priv; VTAILQ_ENTRY(stevedore) list; + char **av; const char *ident; const char *vclname; }; diff --git a/bin/varnishd/storage/storage_malloc.c b/bin/varnishd/storage/storage_malloc.c index fb7345271..2c3ce5b5b 100644 --- a/bin/varnishd/storage/storage_malloc.c +++ b/bin/varnishd/storage/storage_malloc.c @@ -178,7 +178,6 @@ sma_init(struct stevedore *parent, int ac, char * const *av) uintmax_t u; struct sma_sc *sc; - ASSERT_MGT(); ALLOC_OBJ(sc, SMA_SC_MAGIC); AN(sc); sc->sma_max = VRT_INTEGER_MAX; diff --git a/bin/varnishd/storage/storage_persistent_subr.c b/bin/varnishd/storage/storage_persistent_subr.c index 8a1ba8808..0506c57eb 100644 --- a/bin/varnishd/storage/storage_persistent_subr.c +++ b/bin/varnishd/storage/storage_persistent_subr.c @@ -279,8 +279,6 @@ smp_newsilo(struct smp_sc *sc) { struct smp_ident *si; - ASSERT_MGT(); - /* Choose a new random number */ AZ(VRND_RandomCrypto(&sc->unique, sizeof sc->unique)); diff --git a/bin/varnishd/storage/storage_umem.c b/bin/varnishd/storage/storage_umem.c index 2df9f345f..770efb7d7 100644 --- a/bin/varnishd/storage/storage_umem.c +++ b/bin/varnishd/storage/storage_umem.c @@ -297,7 +297,6 @@ smu_init(struct stevedore *parent, int ac, char * const *av) uintmax_t u; struct smu_sc *sc; - ASSERT_MGT(); ALLOC_OBJ(sc, SMU_SC_MAGIC); AN(sc); sc->smu_max = VRT_INTEGER_MAX; diff --git a/bin/varnishtest/tests/p00000.vtc b/bin/varnishtest/tests/p00000.vtc index f523ef82c..6956a7562 100644 --- a/bin/varnishtest/tests/p00000.vtc +++ b/bin/varnishtest/tests/p00000.vtc @@ -2,9 +2,14 @@ varnishtest "Test Basic persistence" feature persistent_storage -shell -err -expect {-spersistent has been deprecated} { - varnishd -spersistent -f '' -n ${tmpdir} -} +process p1 -dump { + varnishd -spersistent -b localhost -n ${tmpdir} -a :0 -d 2>&1 +} -start -expect-exit 1 + +process p1 -expect-text 0 0 {to launch} +process p1 -write "start\n" +process p1 -expect-text 0 0 {-spersistent has been deprecated} +process p1 -wait server s1 { rxreq From phk at FreeBSD.org Fri Jun 3 08:07:09 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 3 Jun 2022 08:07:09 +0000 (UTC) Subject: [master] 9c7c796a0 Make registration of stevedore implementations dynamic. Message-ID: <20220603080709.387B0114564@lists.varnish-cache.org> commit 9c7c796a086d39ccd1c2766a285a3852c780b92c Author: Poul-Henning Kamp Date: Fri Jun 3 08:05:34 2022 +0000 Make registration of stevedore implementations dynamic. diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h index 4d54c86c0..61eafbd83 100644 --- a/bin/varnishd/mgt/mgt.h +++ b/bin/varnishd/mgt/mgt.h @@ -219,7 +219,7 @@ char **MGT_NamedArg(const char *, const char **, const char *); /* stevedore_mgt.c */ extern const char *mgt_stv_h2_rxbuf; void STV_Config(const char *spec); -void STV_Config_Transient(void); +void STV_Config_Final(void); void STV_Init(void); /* mgt_vcc.c */ diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index cc09cb06f..c0bd55914 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -851,8 +851,8 @@ main(int argc, char * const *argv) if (!arg_list_count("s")) STV_Config(s_arg); - /* Configure Transient storage, if user did not */ - STV_Config_Transient(); + /* Configure CLI and Transient storage, if user did not */ + STV_Config_Final(); mgt_vcl_init(); diff --git a/bin/varnishd/storage/mgt_stevedore.c b/bin/varnishd/storage/mgt_stevedore.c index e646d2669..b62248365 100644 --- a/bin/varnishd/storage/mgt_stevedore.c +++ b/bin/varnishd/storage/mgt_stevedore.c @@ -48,6 +48,9 @@ VTAILQ_HEAD(stevedore_head, stevedore); +static struct stevedore_head proto_stevedores = + VTAILQ_HEAD_INITIALIZER(proto_stevedores); + static struct stevedore_head pre_stevedores = VTAILQ_HEAD_INITIALIZER(pre_stevedores); @@ -146,32 +149,56 @@ smp_fake_init(struct stevedore *parent, int ac, char * const *av) static const struct stevedore smp_fake_stevedore = { .magic = STEVEDORE_MAGIC, - .name = "deprecated_persistent", + .name = "persistent", .init = smp_fake_init, }; #endif /*-------------------------------------------------------------------- - * Parse a stevedore argument on the form: - * [ name '=' ] strategy [ ',' arg ] * + * Register a stevedore implementation by name. + * VEXTs get to do this first, and since the list is searched front to + * back a VEXT stevedore which inadvisably wants to steal "default" or + * the name of another stevedore implementation can do so. */ -static const struct choice STV_choice[] = { - { "file", &smf_stevedore }, - { "malloc", &sma_stevedore }, - { "debug", &smd_stevedore }, +void +STV_Register(const struct stevedore *cstv, const char *altname) +{ + struct stevedore *stv; + + CHECK_OBJ_NOTNULL(cstv, STEVEDORE_MAGIC); + ALLOC_OBJ(stv, STEVEDORE_MAGIC); + AN(stv); + *stv = *cstv; + if (altname != NULL) + stv->ident = altname; + else + stv->ident = stv->name; + VTAILQ_INSERT_TAIL(&proto_stevedores, stv, list); +} + +static void +STV_Register_The_Usual_Suspects(void) +{ + STV_Register(&smf_stevedore, NULL); + STV_Register(&sma_stevedore, NULL); + STV_Register(&smd_stevedore, NULL); #ifdef WITH_PERSISTENT_STORAGE - { "deprecated_persistent", &smp_stevedore }, - { "persistent", &smp_fake_stevedore }, + STV_Register(&smp_stevedore, NULL); + STV_Register(&smp_fake_stevedore, NULL); #endif #if defined(HAVE_UMEM_H) - { "umem", &smu_stevedore }, - { "default", &smu_stevedore }, + STV_Register(&smu_stevedore, NULL); + STV_Register(&smu_stevedore, "default"); #else - { "default", &sma_stevedore }, + STV_Register(&sma_stevedore, "default"); #endif - { NULL, NULL } -}; +} + +/*-------------------------------------------------------------------- + * Parse a stevedore argument on the form: + * [ name '=' ] strategy [ ',' arg ] * + */ void STV_Config(const char *spec) @@ -197,7 +224,8 @@ STV_Config(const char *spec) VTAILQ_FOREACH(stv, &pre_stevedores, list) if (!strcmp(stv->ident, ident)) - ARGV_ERR("(-s %s) '%s' is already defined\n", spec, ident); + ARGV_ERR("(-s %s) '%s' is already defined\n", + spec, ident); ALLOC_OBJ(stv, STEVEDORE_MAGIC); AN(stv); @@ -210,16 +238,15 @@ STV_Config(const char *spec) /*--------------------------------------------------------------------*/ void -STV_Config_Transient(void) +STV_Config_Final(void) { struct stevedore *stv; ASSERT_MGT(); VCLS_AddFunc(mgt_cls, MCF_AUTH, cli_stv); - STV_Foreach(stv) { + STV_Foreach(stv) if (!strcmp(stv->ident, TRANSIENT_STORAGE)) return; - } STV_Config(TRANSIENT_STORAGE "=default"); } @@ -236,6 +263,7 @@ STV_Init(void) const struct stevedore *stv2; int ac; + STV_Register_The_Usual_Suspects(); while (!VTAILQ_EMPTY(&pre_stevedores)) { stv = VTAILQ_FIRST(&pre_stevedores); VTAILQ_REMOVE(&pre_stevedores, stv, list); @@ -248,7 +276,12 @@ STV_Init(void) for (ac = 0; av[ac + 2] != NULL; ac++) continue; - stv2 = MGT_Pick(STV_choice, av[1], "storage"); + VTAILQ_FOREACH(stv2, &proto_stevedores, list) + if (!strcmp(stv2->ident, av[1])) + break; + if (stv2 == NULL) + ARGV_ERR("Unknown stevedore method \"%s\"\n", av[1]); + CHECK_OBJ_NOTNULL(stv2, STEVEDORE_MAGIC); *stv = *stv2; AN(stv->name); diff --git a/bin/varnishd/storage/storage.h b/bin/varnishd/storage/storage.h index 882ab82b8..c4ff0af5d 100644 --- a/bin/varnishd/storage/storage.h +++ b/bin/varnishd/storage/storage.h @@ -124,6 +124,8 @@ extern struct stevedore *stv_h2_rxbuf; /*--------------------------------------------------------------------*/ +void STV_Register(const struct stevedore *, const char *altname); + #define STV_Foreach(arg) for (arg = NULL; STV__iter(&arg);) int STV__iter(struct stevedore ** const ); From phk at FreeBSD.org Fri Jun 3 09:29:12 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 3 Jun 2022 09:29:12 +0000 (UTC) Subject: [master] a33a135ab Hopefully fix u00006 on various platforms. Message-ID: <20220603092912.9FB53116A3C@lists.varnish-cache.org> commit a33a135ab746ff583683bc5ebfd6c95db4c4743b Author: Poul-Henning Kamp Date: Fri Nov 19 18:31:00 2021 +0000 Hopefully fix u00006 on various platforms. Dont limit ourselves to a set number of VSL records to include the "0 CLI" we want, various platforms emit a number of Debug messages relating to sockopt Use the `process p%d -expect-text` mechanism to wait only as long as necessary for the "0 CLI" diff --git a/bin/varnishtest/tests/u00006.vtc b/bin/varnishtest/tests/u00006.vtc index e420aa18d..a79d592e8 100644 --- a/bin/varnishtest/tests/u00006.vtc +++ b/bin/varnishtest/tests/u00006.vtc @@ -7,9 +7,11 @@ server s1 -repeat 2 { varnish v1 -vcl+backend {} -start +# We use this to make sure we know there is a "0 CLI" in the binary log. process p1 { - exec varnishlog -n ${v1_name} -g raw -k 3 -w ${tmpdir}/vlog -A + exec varnishlog -n ${v1_name} -g raw -u -A } -start + shell { exec varnishlog -n ${v1_name} -D -P ${tmpdir}/vlog.pid \ -w ${tmpdir}/vlog.bin -R 10/s \ @@ -119,8 +121,11 @@ shell -err -expect "-R: Syntax error: Invalid duration" \ shell -err -expect "-R: Syntax error" \ "varnishlog -R 1000000000/1000000000000000000000000000000s" -process p1 -wait -shell {grep -q "0 CLI" ${tmpdir}/vlog} +# Wait until the binary also (must) contain a "0 CLI" +process p1 -expect-text 0 0 "0 CLI" +process p1 -screen_dump +process p1 -stop + shell -match "0 CLI[ ]+- Wr 200 [0-9]+ PONG" \ {varnishlog -n ${v1_name} -d -g raw -X "Wr 200 [0-9]+ [^P]"} From noreply at github.com Fri Jun 3 09:29:12 2022 From: noreply at github.com (GitHub) Date: Fri, 3 Jun 2022 09:29:12 +0000 (UTC) Subject: [master] a810c48b8 Merge branch 'master' into master Message-ID: <20220603092912.DD71D116A40@lists.varnish-cache.org> commit a810c48b89bfbc1c5c2ac2ce7a651811bae1d81c Merge: 9c7c796a0 a33a135ab Author: Poul-Henning Kamp Date: Fri Jun 3 09:28:56 2022 +0000 Merge branch 'master' into master From dridi.boukelmoune at gmail.com Mon Jun 20 09:29:07 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 20 Jun 2022 09:29:07 +0000 (UTC) Subject: [master] 66860cd0a coverity: Only run in the main repository Message-ID: <20220620092907.CFCE7102393@lists.varnish-cache.org> commit 66860cd0ac1b09b67b2e0288d25ba810f2ccad2e Author: Dridi Boukelmoune Date: Mon Jun 20 11:26:16 2022 +0200 coverity: Only run in the main repository Otherwise forks may attempt to run coveriy scan builds only to fail and spam their owners. diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index aa7a85f12..6843d91bd 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -7,6 +7,7 @@ on: jobs: coverity: + if: github.repository_owner == 'varnishcache' runs-on: ubuntu-latest steps: - run: | From phk at FreeBSD.org Mon Jun 20 10:36:06 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 20 Jun 2022 10:36:06 +0000 (UTC) Subject: [master] 6b9635154 Minor polish. Message-ID: <20220620103606.36978104C24@lists.varnish-cache.org> commit 6b9635154d4c10162bb59202726b5db6450cdbce Author: Poul-Henning Kamp Date: Mon Jun 20 10:34:38 2022 +0000 Minor polish. In other news: I've gotten much better at writing python. diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 5be4d4665..efbcd4841 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -842,7 +842,7 @@ class AliasStanza(Stanza): def find_symbol(self, tbl, name): for sym in tbl: if sym.proto is None: - continue; + continue if sym.proto.name == name: return sym err("Symbol '%s' not found\n" % name, warn=False) @@ -1049,7 +1049,7 @@ class vcc(object): fo.write("#endif\n") fo.write("\n") - self.mkdefs(fo); + self.mkdefs(fo) for j in sorted(self.enums): fo.write("extern VCL_ENUM VENUM(%s);\n" % j) @@ -1084,22 +1084,31 @@ class vcc(object): fmt_cstruct(fo, '.enum_%s =' % j, '&VENUM(%s),' % j) fo.write("};\n") - def json(self, fo): + def vmod_proto(self, fo, fnx): + fo.write("\nstatic const char Vmod_Proto[] =\n") + for i in open(fnx): + fo.write('\t"%s\\n"\n' % i.rstrip()) + fo.write('\t"static struct %s %s;";\n' % (self.csn, self.csn)) + + def iter_json(self): + jl = [["$VMOD", "1.0"]] for j in self.contents: j.json(jl) + for i in json.dumps(jl, indent=2, separators=(",", ": ")).splitlines(): + yield i + " " + def json(self, fo): fo.write("\nstatic const char Vmod_Json[] = {\n") - t = '\t"' - for i in json.dumps(jl, indent=2, separators=(",", ": ")): - if i == '\n': - fo.write(t + ' "\n') - t = '\t"' - else: - if i in '"\\': - t += '\\' - t += i - fo.write(t + '\\n"\n};\n') + + for i in self.iter_json(fnx): + fo.write('\t"') + for j in i: + if j in '"\\': + fo.write('\\') + fo.write(j) + fo.write('"\n') + fo.write('\t\"\\n\"\n};\n') def vmod_data(self, fo): vmd = "Vmod_%s_Data" % self.modname @@ -1133,7 +1142,7 @@ class vcc(object): write_c_file_warning(fo, self.inputfile) - self.mkdefs(fx); + self.mkdefs(fx) fo.write('#include "config.h"\n') for i in ["vdef", "vrt", self.pfx, "vmod_abi"]: @@ -1167,17 +1176,14 @@ class vcc(object): fx.close() - fo.write("\nstatic const char Vmod_Proto[] =\n") - for i in open(fnx): - fo.write('\t"%s\\n"\n' % i.rstrip()) - fo.write('\t"static struct %s %s;";\n' % (self.csn, self.csn)) - - os.remove(fnx) + self.vmod_proto(fo, fnx) self.json(fo) self.vmod_data(fo) + os.remove(fnx) + fo.close() ####################################################################### From phk at FreeBSD.org Mon Jun 20 10:42:05 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 20 Jun 2022 10:42:05 +0000 (UTC) Subject: [master] 2a7ea235b Gah, I guess I must have hit 'u' before saving. Message-ID: <20220620104205.3124A10503C@lists.varnish-cache.org> commit 2a7ea235b76523ea34c863818f6401115530a3d3 Author: Poul-Henning Kamp Date: Mon Jun 20 10:41:28 2022 +0000 Gah, I guess I must have hit 'u' before saving. diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index efbcd4841..6a56afb06 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -1101,7 +1101,7 @@ class vcc(object): def json(self, fo): fo.write("\nstatic const char Vmod_Json[] = {\n") - for i in self.iter_json(fnx): + for i in self.iter_json(): fo.write('\t"') for j in i: if j in '"\\': From phk at FreeBSD.org Mon Jun 20 12:41:06 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 20 Jun 2022 12:41:06 +0000 (UTC) Subject: [master] 2feafdcfd Move the VMOD C-prototypes into the JSON data. Message-ID: <20220620124106.ECD661085E9@lists.varnish-cache.org> commit 2feafdcfde7bedbad01ebbc15261dfa07f111110 Author: Poul-Henning Kamp Date: Mon Jun 20 12:39:33 2022 +0000 Move the VMOD C-prototypes into the JSON data. diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c index 39243fdfc..8a088a19d 100644 --- a/bin/varnishd/cache/cache_vrt_vmod.c +++ b/bin/varnishd/cache/cache_vrt_vmod.c @@ -132,7 +132,7 @@ VPI_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len, strcmp(d->name, nm) || d->func == NULL || d->func_len <= 0 || - d->proto == NULL || + d->proto != NULL || d->json == NULL) { VSB_printf(ctx->msg, "Loading vmod %s from %s (%s):\n", nm, backup, path); diff --git a/include/vrt.h b/include/vrt.h index 562904942..975149322 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -58,6 +58,7 @@ * binary/load-time compatible, increment MAJOR version * * NEXT (2022-09-15) + * C-prototypes moved into JSON * VRT_AddVDP() deprecated * VRT_AddVFP() deprecated * VRT_RemoveVDP() deprecated diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index 9c5be1234..cd2c3bd09 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -164,6 +164,7 @@ vcc_json_always(struct vcc *tl, const struct vjsn *vj, const char *vmod_name) struct inifin *ifp; const struct vjsn_val *vv, *vv2; double vmod_syntax = 0.0; + int cproto_seen = 0; AN(vj); AN(vmod_name); @@ -197,6 +198,8 @@ vcc_json_always(struct vcc *tl, const struct vjsn *vj, const char *vmod_name) } else if (!strcmp(vv2->value, "$ALIAS")) { } else if (!strcmp(vv2->value, "$FUNC")) { } else if (!strcmp(vv2->value, "$OBJ")) { + } else if (!strcmp(vv2->value, "$CPROTO")) { + cproto_seen = 1; } else { VTAILQ_FOREACH(vv2, &vv->children, list) fprintf(stderr, "\tt %s n %s v %s\n", @@ -204,6 +207,8 @@ vcc_json_always(struct vcc *tl, const struct vjsn *vj, const char *vmod_name) WRONG("Vmod JSON syntax error"); } } + if (!cproto_seen) + WRONG("Vmod JSON has no CPROTO"); } static const struct vmod_data * @@ -244,7 +249,8 @@ vcc_VmodSanity(struct vcc *tl, void *hdl, const struct token *mod, char *fnp) if (vmd->name == NULL || vmd->func == NULL || vmd->func_len <= 0 || - vmd->proto == NULL || + vmd->json == NULL || + vmd->proto != NULL || vmd->abi == NULL) { VSB_printf(tl->sb, "Mangled VMOD %.*s\n", PF(mod)); VSB_printf(tl->sb, "\tFile name: %s\n", fnp); @@ -335,6 +341,32 @@ vcc_VmodSymbols(struct vcc *tl, const struct symbol *sym) } } +static void +vcc_emit_c_prototypes(const struct vcc *tl, const struct vjsn *vj) +{ + const struct vjsn_val *vv, *vv2, *vv3; + + Fh(tl, 0, "\n"); + vv = vj->value; + assert (vjsn_is_array(vv)); + vv3 = NULL; + VTAILQ_FOREACH(vv2, &vv->children, list) { + assert (vjsn_is_array(vv2)); + vv3 = VTAILQ_FIRST(&vv2->children); + assert (vjsn_is_string(vv3)); + if (!strcmp(vv3->value, "$CPROTO")) + break; + } + assert(vv3 != NULL); + while (1) { + vv3 = VTAILQ_NEXT(vv3, list); + if (vv3 == NULL) + break; + assert (vjsn_is_string(vv3)); + Fh(tl, 0, "%s\n", vv3->value); + } +} + void vcc_ParseImport(struct vcc *tl) { @@ -493,7 +525,7 @@ vcc_ParseImport(struct vcc *tl) Fh(tl, 0, "\n/* --- BEGIN VMOD %.*s --- */\n\n", PF(mod)); Fh(tl, 0, "static struct vmod *VGC_vmod_%.*s;\n", PF(mod)); Fh(tl, 0, "static struct vmod_priv vmod_priv_%.*s;\n", PF(mod)); - Fh(tl, 0, "\n%s\n", vmd->proto); + vcc_emit_c_prototypes(tl, vj); Fh(tl, 0, "\n/* --- END VMOD %.*s --- */\n\n", PF(mod)); free(fnpx); } diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 6a56afb06..823e12385 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -1084,24 +1084,22 @@ class vcc(object): fmt_cstruct(fo, '.enum_%s =' % j, '&VENUM(%s),' % j) fo.write("};\n") - def vmod_proto(self, fo, fnx): - fo.write("\nstatic const char Vmod_Proto[] =\n") - for i in open(fnx): - fo.write('\t"%s\\n"\n' % i.rstrip()) - fo.write('\t"static struct %s %s;";\n' % (self.csn, self.csn)) - - def iter_json(self): + def iter_json(self, fnx): jl = [["$VMOD", "1.0"]] + jl.append(["$CPROTO"]) + for i in open(fnx): + jl[-1].append(i.rstrip()) + jl[-1].append("static struct %s %s;" % (self.csn, self.csn)) for j in self.contents: j.json(jl) for i in json.dumps(jl, indent=2, separators=(",", ": ")).splitlines(): yield i + " " - def json(self, fo): + def json(self, fo, fnx): fo.write("\nstatic const char Vmod_Json[] = {\n") - for i in self.iter_json(): + for i in self.iter_json(fnx): fo.write('\t"') for j in i: if j in '"\\': @@ -1127,7 +1125,6 @@ class vcc(object): fo.write('\t.func =\t\t&%s,\n' % self.csn) fo.write('\t.func_len =\tsizeof(%s),\n' % self.csn) fo.write('\t.func_name =\t"%s",\n' % self.csn) - fo.write('\t.proto =\tVmod_Proto,\n') fo.write('\t.json =\t\tVmod_Json,\n') fo.write('\t.abi =\t\tVMOD_ABI_Version,\n') fo.write("\t.file_id =\t\"%s\",\n" % self.file_id) @@ -1176,9 +1173,7 @@ class vcc(object): fx.close() - self.vmod_proto(fo, fnx) - - self.json(fo) + self.json(fo, fnx) self.vmod_data(fo) diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index 24c43a263..53b0ebc14 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -1087,7 +1087,7 @@ const struct vmod_data Vmod_wrong2_Data = { .func = foo_struct, .func_len = sizeof foo_struct, .func_name = "foo_struct", - .proto = "blablabla", + .json = "blablabla", }; extern const struct vmod_data Vmod_wrong3_Data; @@ -1098,7 +1098,7 @@ const struct vmod_data Vmod_wrong3_Data = { .func = foo_struct, .func_len = sizeof foo_struct, .func_name = "foo_struct", - .proto = "blablabla", + .json = "blablabla", .abi = "abiblabla", }; From nils.goroll at uplex.de Mon Jun 20 15:44:06 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 20 Jun 2022 15:44:06 +0000 (UTC) Subject: [master] e0c8a524b Coverage for ESI subrequests vs. Request body Message-ID: <20220620154406.8280410FC32@lists.varnish-cache.org> commit e0c8a524b58ea7a1d253550ef87dd942da577418 Author: Nils Goroll Date: Mon Jun 20 17:30:34 2022 +0200 Coverage for ESI subrequests vs. Request body Make sure subrequests to not see Content-Length / Transfer-Encoding diff --git a/bin/varnishtest/tests/e00036.vtc b/bin/varnishtest/tests/e00036.vtc new file mode 100644 index 000000000..9e6e7d86f --- /dev/null +++ b/bin/varnishtest/tests/e00036.vtc @@ -0,0 +1,83 @@ +varnishtest "ESI vs. request body" + +server s1 { + rxreq + expect req.method == POST + expect req.url == "/foo/bar" + expect req.bodylen == 6 + expect req.http.Content-Length == "6" + expect req.http.Transfer-Encoding == "" + txresp -body { + + Before include + + After include + } + rxreq + expect req.method == XGET + expect req.url == "/foo/body1" + expect req.bodylen == 0 + expect req.http.Content-Length == "" + expect req.http.Transfer-Encoding == "" + txresp -body { + Included file + } + + rxreq + expect req.method == POST + expect req.url == "/foo/bar" + expect req.bodylen == 66 + # std.cache_req_body() turns chunked into C-L + expect req.http.Content-Length == "66" + expect req.http.Transfer-Encoding == "" + txresp -body { + + Before include + + After include + } + rxreq + expect req.method == XGET + expect req.url == "/foo/body2" + expect req.bodylen == 0 + expect req.http.Content-Length == "" + expect req.http.Transfer-Encoding == "" + txresp -body { + Included file + } +} -start + +varnish v1 -vcl+backend { + import std; + + sub vcl_recv { + std.cache_req_body(1KB); + if (req.url ~ "^/foo/body") { + set req.method = "XGET"; + } + return (pass); + } + + sub vcl_backend_response { + if (bereq.url !~ "^/foo/body") { + set beresp.do_esi = true; + } + } +} -start + +client c1 { + txreq -url /foo/bar -method POST -body {foobar} + rxresp + expect resp.status == 200 + expect resp.bodylen == 67 + + txreq -url /foo/bar -method POST -nolen \ + -hdr "Transfer-encoding: chunked" + chunkedlen 66 + chunkedlen 0 + rxresp + expect resp.status == 200 + expect resp.bodylen == 67 +} -run + +varnish v1 -expect esi_errors == 0 From nils.goroll at uplex.de Mon Jun 20 15:44:06 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 20 Jun 2022 15:44:06 +0000 (UTC) Subject: [master] fbf2c9863 Clear Transfer-Encoding for ESI subrequests Message-ID: <20220620154406.9B20010FC35@lists.varnish-cache.org> commit fbf2c98638057b60c09767eaa4e3d0c53f3ecf74 Author: Nils Goroll Date: Mon Jun 20 17:33:58 2022 +0200 Clear Transfer-Encoding for ESI subrequests While, before this change, subrequests would see the Transfer-Encoding on the client side, I could not find a case where this caused any harm, so this change, presumably, is only to prevent potential future issues. diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index b214e9e94..9414c324d 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -174,6 +174,7 @@ ved_include(struct req *preq, const char *src, const char *host, /* Client content already taken care of */ http_Unset(req->http, H_Content_Length); + http_Unset(req->http, H_Transfer_Encoding); req->req_body_status = BS_NONE; AZ(req->vcl); From phk at FreeBSD.org Tue Jun 21 13:33:06 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 21 Jun 2022 13:33:06 +0000 (UTC) Subject: [master] eaa1c903f Rename struct vmod_open to vmod_import and keep it around. Message-ID: <20220621133306.5496C10F704@lists.varnish-cache.org> commit eaa1c903f9337ae4d4c83bc761e772bfd1021820 Author: Poul-Henning Kamp Date: Tue Jun 21 13:32:31 2022 +0000 Rename struct vmod_open to vmod_import and keep it around. diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index cd2c3bd09..db79a5aac 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -42,11 +42,12 @@ #include "vmod_abi.h" #include "vsb.h" -struct vmod_open { +struct vmod_import { unsigned magic; -#define VMOD_OPEN_MAGIC 0x9995b1f3 +#define VMOD_IMPORT_MAGIC 0x31803a5d void *hdl; const char *err; + const struct vmod_data *vmd; }; struct vmod_obj { @@ -60,14 +61,14 @@ struct vmod_obj { static int vcc_path_dlopen(void *priv, const char *fn) { - struct vmod_open *vop; + struct vmod_import *vim; - CAST_OBJ_NOTNULL(vop, priv, VMOD_OPEN_MAGIC); + CAST_OBJ_NOTNULL(vim, priv, VMOD_IMPORT_MAGIC); AN(fn); - vop->hdl = dlopen(fn, RTLD_NOW | RTLD_LOCAL); - if (vop->hdl == NULL) { - vop->err = dlerror(); + vim->hdl = dlopen(fn, RTLD_NOW | RTLD_LOCAL); + if (vim->hdl == NULL) { + vim->err = dlerror(); return (-1); } return (0); @@ -377,9 +378,9 @@ vcc_ParseImport(struct vcc *tl) struct symbol *msym, *vsym; const struct vmod_data *vmd; struct vjsn *vj; - struct vmod_open vop[1]; + struct vmod_import *vim; + const struct vmod_import *vimold; - INIT_OBJ(vop, VMOD_OPEN_MAGIC); t1 = tl->t; SkipToken(tl, ID); /* "import" */ @@ -425,8 +426,11 @@ vcc_ParseImport(struct vcc *tl) SkipToken(tl, ';'); - if (VFIL_searchpath(tl->vmod_path, vcc_path_dlopen, vop, fn, &fnpx)) { - if (vop->err == NULL) { + ALLOC_OBJ(vim, VMOD_IMPORT_MAGIC); + AN(vim); + + if (VFIL_searchpath(tl->vmod_path, vcc_path_dlopen, vim, fn, &fnpx)) { + if (vim->err == NULL) { VSB_printf(tl->sb, "Could not find VMOD %.*s\n", PF(mod)); } else { @@ -434,22 +438,25 @@ vcc_ParseImport(struct vcc *tl) "Could not open VMOD %.*s\n", PF(mod)); VSB_printf(tl->sb, "\tFile name: %s\n", fnpx != NULL ? fnpx : fn); - VSB_printf(tl->sb, "\tdlerror: %s\n", vop->err); + VSB_printf(tl->sb, "\tdlerror: %s\n", vim->err); } vcc_ErrWhere(tl, mod); free(fnpx); + FREE_OBJ(vim); return; } - vmd = vcc_VmodSanity(tl, vop->hdl, mod, fnpx); + vmd = vcc_VmodSanity(tl, vim->hdl, mod, fnpx); if (vmd == NULL || tl->err) { - AZ(dlclose(vop->hdl)); + AZ(dlclose(vim->hdl)); free(fnpx); + FREE_OBJ(vim); return; } - if (msym->extra != NULL) { - if (!strcmp(msym->extra, vmd->file_id)) { + CAST_OBJ(vimold, (const void*)(msym->extra), VMOD_IMPORT_MAGIC); + if (vimold != NULL) { + if (!strcmp(vimold->vmd->file_id, vmd->file_id)) { /* Identical import is OK */ } else { VSB_printf(tl->sb, @@ -457,8 +464,9 @@ vcc_ParseImport(struct vcc *tl) PF(tmod)); vcc_ErrWhere2(tl, t1, tl->t); } - AZ(dlclose(vop->hdl)); + AZ(dlclose(vim->hdl)); free(fnpx); + FREE_OBJ(vim); return; } msym->def_b = t1; @@ -466,14 +474,17 @@ vcc_ParseImport(struct vcc *tl) VTAILQ_FOREACH(vsym, &tl->sym_vmods, sideways) { assert(vsym->kind == SYM_VMOD); - if (!strcmp(vsym->extra, vmd->file_id)) { + CAST_OBJ_NOTNULL(vimold, (const void*)(vsym->extra), + VMOD_IMPORT_MAGIC); + if (!strcmp(vimold->vmd->file_id, vmd->file_id)) { /* Already loaded under different name */ msym->eval_priv = vsym->eval_priv; msym->extra = vsym->extra; msym->vmod_name = vsym->vmod_name; vcc_VmodSymbols(tl, msym); - AZ(dlclose(vop->hdl)); + AZ(dlclose(vim->hdl)); free(fnpx); + FREE_OBJ(vim); return; } } @@ -516,7 +527,8 @@ vcc_ParseImport(struct vcc *tl) XXXAZ(p); AN(vj); msym->eval_priv = vj; - msym->extra = TlDup(tl, vmd->file_id); + vim->vmd = vmd; + msym->extra = (const char *)vim; msym->vmod_name = TlDup(tl, vmd->name); vcc_VmodSymbols(tl, msym); From phk at FreeBSD.org Thu Jun 23 08:32:05 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 23 Jun 2022 08:32:05 +0000 (UTC) Subject: [master] 419fbadef More rework of VCC-VMOD code. Message-ID: <20220623083205.E680E10ACA7@lists.varnish-cache.org> commit 419fbadefa200830e0f5eba5a29c342ab3d22aad Author: Poul-Henning Kamp Date: Thu Jun 23 08:30:25 2022 +0000 More rework of VCC-VMOD code. diff --git a/include/vrt.h b/include/vrt.h index 975149322..c4afdc6b6 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -58,7 +58,7 @@ * binary/load-time compatible, increment MAJOR version * * NEXT (2022-09-15) - * C-prototypes moved into JSON + * VMOD C-prototypes moved into JSON * VRT_AddVDP() deprecated * VRT_AddVFP() deprecated * VRT_RemoveVDP() deprecated diff --git a/lib/libvcc/Makefile.am b/lib/libvcc/Makefile.am index 8731890da..117f38a3c 100644 --- a/lib/libvcc/Makefile.am +++ b/lib/libvcc/Makefile.am @@ -30,6 +30,8 @@ libvcc_la_SOURCES = \ vcc_utils.c \ vcc_var.c \ vcc_vmod.c \ + vcc_vmod.h \ + vcc_vmod_sym.c \ vcc_xref.c dist_noinst_SCRIPTS = \ diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 1663ba92d..d697d3381 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -194,6 +194,9 @@ struct symbol { const char *extra; + /* vcc_vmod.c */ + const struct vmod_import *import; + /* SYM_VAR */ const char *rname; unsigned r_methods; diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index db79a5aac..a9f169d10 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -26,6 +26,9 @@ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. + * + * Parse `import`, check metadata and versioning. + * */ #include "config.h" @@ -42,22 +45,31 @@ #include "vmod_abi.h" #include "vsb.h" +#include "vcc_vmod.h" + struct vmod_import { unsigned magic; #define VMOD_IMPORT_MAGIC 0x31803a5d void *hdl; const char *err; - const struct vmod_data *vmd; -}; + char *path; -struct vmod_obj { - unsigned magic; -#define VMOD_OBJ_MAGIC 0x349885f8 + double vmod_syntax; char *name; - struct type type[1]; - VTAILQ_ENTRY(vmod_obj) list; + char *func_name; + char *file_id; + + struct symbol *sym; + const struct token *t_mod; + struct vjsn *vj; +#define STANZA(UU, ll, ss) int n_##ll; + STANZA_TBL +#undef STANZA }; +typedef void vcc_do_stanza_f(struct vcc *tl, const struct vmod_import *vim, + const struct vjsn_val *vv); + static int vcc_path_dlopen(void *priv, const char *fn) { @@ -74,178 +86,111 @@ vcc_path_dlopen(void *priv, const char *fn) return (0); } -static void vcc_VmodObject(struct vcc *tl, struct symbol *sym); -static void vcc_VmodSymbols(struct vcc *tl, const struct symbol *sym); - -static void -alias_sym(struct vcc *tl, const struct symbol *psym, const struct vjsn_val *v) +static const char * +vcc_ParseJSON(const struct vcc *tl, const char *jsn, struct vmod_import *vim) { - char *alias = NULL, *func = NULL; - struct symbol *sym; - struct vsb *buf; - - buf = VSB_new_auto(); - AN(buf); - - VCC_SymName(buf, psym); - VSB_printf(buf, ".%s", v->value); - AZ(VSB_finish(buf)); - REPLACE(alias, VSB_data(buf)); - - v = VTAILQ_NEXT(v, list); - assert(vjsn_is_string(v)); - - VSB_clear(buf); - VCC_SymName(buf, psym); - VSB_printf(buf, ".%s", v->value); - AZ(VSB_finish(buf)); - REPLACE(func, VSB_data(buf)); - - sym = VCC_MkSymAlias(tl, alias, func); - AN(sym); - assert(sym->kind == SYM_FUNC || sym->kind == SYM_METHOD); - VSB_destroy(&buf); - free(alias); - free(func); -} - -static void -func_sym(struct vcc *tl, vcc_kind_t kind, const struct symbol *psym, - const struct vjsn_val *v) -{ - struct symbol *sym; - struct vsb *buf; - - if (kind == SYM_ALIAS) { - alias_sym(tl, psym, v); - return; + const struct vjsn_val *vv, *vv2, *vv3; + const char *err; + + vim->vj = vjsn_parse(jsn, &err); + if (err != NULL) + return (err); + AN(vim->vj); + + vv = vim->vj->value; + if (!vjsn_is_array(vv)) + return ("Not array[0]"); + + vv2 = VTAILQ_FIRST(&vv->children); + if (!vjsn_is_array(vv2)) + return ("Not array[1]"); + vv3 = VTAILQ_FIRST(&vv2->children); + if (!vjsn_is_string(vv3)) + return ("Not string[2]"); + if (strcmp(vv3->value, "$VMOD")) + return ("Not $VMOD[3]"); + + vv3 = VTAILQ_NEXT(vv3, list); + assert(vjsn_is_string(vv3)); + vim->vmod_syntax = strtod(vv3->value, NULL); + assert (vim->vmod_syntax == 1.0); + + vv3 = VTAILQ_NEXT(vv3, list); + assert(vjsn_is_string(vv3)); + vim->name = vv3->value; + + vv3 = VTAILQ_NEXT(vv3, list); + assert(vjsn_is_string(vv3)); + vim->func_name = vv3->value; + + vv3 = VTAILQ_NEXT(vv3, list); + assert(vjsn_is_string(vv3)); + vim->file_id = vv3->value; + + if (!vcc_IdIs(vim->t_mod, vim->name)) { + VSB_printf(tl->sb, "Wrong file for VMOD %.*s\n", + PF(vim->t_mod)); + VSB_printf(tl->sb, "\tFile name: %s\n", vim->path); + VSB_printf(tl->sb, "\tContains vmod \"%s\"\n", vim->name); + return (""); } - buf = VSB_new_auto(); - AN(buf); - - VCC_SymName(buf, psym); - VSB_printf(buf, ".%s", v->value); - AZ(VSB_finish(buf)); - sym = VCC_MkSym(tl, VSB_data(buf), SYM_MAIN, kind, VCL_LOW, VCL_HIGH); - AN(sym); - VSB_destroy(&buf); - - if (kind == SYM_OBJECT) { - sym->eval_priv = v; - sym->vmod_name = psym->vmod_name; - sym->r_methods = VCL_MET_INIT; - vcc_VmodObject(tl, sym); - vcc_VmodSymbols(tl, sym); - return; + VTAILQ_FOREACH(vv2, &vv->children, list) { + assert (vjsn_is_array(vv2)); + vv3 = VTAILQ_FIRST(&vv2->children); + assert(vjsn_is_string(vv3)); + assert(vv3->value[0] == '$'); +#define STANZA(UU, ll, ss) \ + if (!strcmp(vv3->value, "$" #UU)) {vim->n_##ll++; continue;} + STANZA_TBL +#undef STANZA + return ("Unknown entry"); } - - if (kind == SYM_METHOD) - sym->extra = psym->rname; - - v = VTAILQ_NEXT(v, list); - - assert(vjsn_is_array(v)); - sym->action = vcc_Act_Call; - sym->vmod_name = psym->vmod_name; - sym->eval = vcc_Eval_SymFunc; - sym->eval_priv = v; - v = VTAILQ_FIRST(&v->children); - assert(vjsn_is_array(v)); - v = VTAILQ_FIRST(&v->children); - assert(vjsn_is_string(v)); - sym->type = VCC_Type(v->value); - AN(sym->type); - sym->r_methods = VCL_MET_TASK_ALL; + if (vim->n_cproto != 1) + return ("Bad cproto stanza(s)"); + if (vim->n_vmod != 1) + return ("Bad vmod stanza(s)"); + return (NULL); } -static void -vcc_json_always(struct vcc *tl, const struct vjsn *vj, const char *vmod_name) -{ - struct inifin *ifp; - const struct vjsn_val *vv, *vv2; - double vmod_syntax = 0.0; - int cproto_seen = 0; - - AN(vj); - AN(vmod_name); - ifp = NULL; - - VTAILQ_FOREACH(vv, &vj->value->children, list) { - assert(vjsn_is_array(vv)); - vv2 = VTAILQ_FIRST(&vv->children); - assert(vjsn_is_string(vv2)); - if (!strcmp(vv2->value, "$VMOD")) { - vmod_syntax = - strtod(VTAILQ_NEXT(vv2, list)->value, NULL); - continue; - } - assert (vmod_syntax == 1.0); - if (!strcmp(vv2->value, "$EVENT")) { - /* XXX: What about the rest of the events ? */ - if (ifp == NULL) - ifp = New_IniFin(tl); - vv2 = VTAILQ_NEXT(vv2, list); - VSB_printf(ifp->ini, - "\tif (%s(ctx, &vmod_priv_%s, VCL_EVENT_LOAD))\n" - "\t\treturn(1);", - vv2->value, vmod_name); - VSB_printf(ifp->fin, - "\t\t(void)%s(ctx, &vmod_priv_%s,\n" - "\t\t\t VCL_EVENT_DISCARD);", - vv2->value, vmod_name); - VSB_printf(ifp->event, "%s(ctx, &vmod_priv_%s, ev)", - vv2->value, vmod_name); - } else if (!strcmp(vv2->value, "$ALIAS")) { - } else if (!strcmp(vv2->value, "$FUNC")) { - } else if (!strcmp(vv2->value, "$OBJ")) { - } else if (!strcmp(vv2->value, "$CPROTO")) { - cproto_seen = 1; - } else { - VTAILQ_FOREACH(vv2, &vv->children, list) - fprintf(stderr, "\tt %s n %s v %s\n", - vv2->type, vv2->name, vv2->value); - WRONG("Vmod JSON syntax error"); - } - } - if (!cproto_seen) - WRONG("Vmod JSON has no CPROTO"); -} +/* + * Load and check the metadata from the objectfile containing the vmod + */ -static const struct vmod_data * -vcc_VmodSanity(struct vcc *tl, void *hdl, const struct token *mod, char *fnp) +static int +vcc_VmodLoad(const struct vcc *tl, struct vmod_import *vim, char *fnp) { char buf[256]; + static const char *err; const struct vmod_data *vmd; - bprintf(buf, "Vmod_%.*s_Data", PF(mod)); - vmd = dlsym(hdl, buf); + CHECK_OBJ_NOTNULL(vim, VMOD_IMPORT_MAGIC); + bprintf(buf, "Vmod_%.*s_Data", PF(vim->t_mod)); + vmd = dlsym(vim->hdl, buf); if (vmd == NULL) { - VSB_printf(tl->sb, "Malformed VMOD %.*s\n", PF(mod)); + VSB_printf(tl->sb, "Malformed VMOD %.*s\n", PF(vim->t_mod)); VSB_printf(tl->sb, "\tFile name: %s\n", fnp); VSB_cat(tl->sb, "\t(no Vmod_Data symbol)\n"); - vcc_ErrWhere(tl, mod); - return (NULL); + return (-1); } if (vmd->vrt_major == 0 && vmd->vrt_minor == 0 && - (vmd->abi == NULL || strcmp(vmd->abi, VMOD_ABI_Version) != 0)) { - VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(mod)); + (vmd->abi == NULL || strcmp(vmd->abi, VMOD_ABI_Version))) { + VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(vim->t_mod)); VSB_printf(tl->sb, "\tFile name: %s\n", fnp); VSB_printf(tl->sb, "\tABI mismatch, expected <%s>, got <%s>\n", VMOD_ABI_Version, vmd->abi); - vcc_ErrWhere(tl, mod); - return (NULL); + return (-1); } - if (vmd->vrt_major != 0 && (vmd->vrt_major != VRT_MAJOR_VERSION || + if (vmd->vrt_major != 0 && + (vmd->vrt_major != VRT_MAJOR_VERSION || vmd->vrt_minor > VRT_MINOR_VERSION)) { - VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(mod)); + VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(vim->t_mod)); VSB_printf(tl->sb, "\tFile name: %s\n", fnp); VSB_printf(tl->sb, "\tVMOD wants ABI version %u.%u\n", vmd->vrt_major, vmd->vrt_minor); VSB_printf(tl->sb, "\tvarnishd provides ABI version %u.%u\n", VRT_MAJOR_VERSION, VRT_MINOR_VERSION); - vcc_ErrWhere(tl, mod); - return (NULL); + return (-1); } if (vmd->name == NULL || vmd->func == NULL || @@ -253,131 +198,146 @@ vcc_VmodSanity(struct vcc *tl, void *hdl, const struct token *mod, char *fnp) vmd->json == NULL || vmd->proto != NULL || vmd->abi == NULL) { - VSB_printf(tl->sb, "Mangled VMOD %.*s\n", PF(mod)); + VSB_printf(tl->sb, "Mangled VMOD %.*s\n", PF(vim->t_mod)); VSB_printf(tl->sb, "\tFile name: %s\n", fnp); VSB_cat(tl->sb, "\tInconsistent metadata\n"); - vcc_ErrWhere(tl, mod); - return (NULL); + return (-1); } - if (!vcc_IdIs(mod, vmd->name)) { - VSB_printf(tl->sb, "Wrong file for VMOD %.*s\n", PF(mod)); - VSB_printf(tl->sb, "\tFile name: %s\n", fnp); - VSB_printf(tl->sb, "\tContains vmod \"%s\"\n", vmd->name); - vcc_ErrWhere(tl, mod); - return (NULL); + + err = vcc_ParseJSON(tl, vmd->json, vim); + AZ(dlclose(vim->hdl)); + vim->hdl = NULL; + if (err != NULL && *err != '\0') { + VSB_printf(tl->sb, + "VMOD %.*s: bad metadata\n", PF(vim->t_mod)); + VSB_printf(tl->sb, "\t(%s)\n", err); + VSB_printf(tl->sb, "\tFile name: %s\n", vim->path); } - return (vmd); -} -static vcc_kind_t -vcc_vmod_kind(const char *type) -{ + if (err != NULL) + return (-1); -#define VMOD_KIND(str, kind) \ - do { \ - if (!strcmp(str, type)) \ - return (kind); \ - } while (0) - VMOD_KIND("$OBJ", SYM_OBJECT); - VMOD_KIND("$METHOD", SYM_METHOD); - VMOD_KIND("$FUNC", SYM_FUNC); - VMOD_KIND("$ALIAS", SYM_ALIAS); -#undef VMOD_KIND - return (SYM_NONE); + return(0); } -static void -vcc_VmodObject(struct vcc *tl, struct symbol *sym) +static void v_matchproto_(vcc_do_stanza_f) +vcc_do_event(struct vcc *tl, const struct vmod_import *vim, + const struct vjsn_val *vv) { - struct vmod_obj *obj; - struct vsb *buf; - - buf = VSB_new_auto(); - AN(buf); - - VSB_printf(buf, "%s.%s", sym->vmod_name, sym->name); - AZ(VSB_finish(buf)); - - ALLOC_OBJ(obj, VMOD_OBJ_MAGIC); - AN(obj); - REPLACE(obj->name, VSB_data(buf)); + struct inifin *ifp; - INIT_OBJ(obj->type, TYPE_MAGIC); - obj->type->name = obj->name; - sym->type = obj->type; - VTAILQ_INSERT_TAIL(&tl->vmod_objects, obj, list); - VSB_destroy(&buf); + ifp = New_IniFin(tl); + VSB_printf(ifp->ini, + "\tif (%s(ctx, &vmod_priv_%s, VCL_EVENT_LOAD))\n" + "\t\treturn(1);", + vv->value, vim->sym->vmod_name); + VSB_printf(ifp->fin, + "\t\t(void)%s(ctx, &vmod_priv_%s,\n" + "\t\t\t VCL_EVENT_DISCARD);", + vv->value, vim->sym->vmod_name); + VSB_printf(ifp->event, "%s(ctx, &vmod_priv_%s, ev)", + vv->value, vim->sym->vmod_name); } -static void -vcc_VmodSymbols(struct vcc *tl, const struct symbol *sym) +static void v_matchproto_(vcc_do_stanza_f) +vcc_do_cproto(struct vcc *tl, const struct vmod_import *vim, + const struct vjsn_val *vv) { - const struct vjsn *vj; - const struct vjsn_val *vv, *vv1, *vv2; - vcc_kind_t kind; - - if (sym->kind == SYM_VMOD) { - CAST_OBJ_NOTNULL(vj, sym->eval_priv, VJSN_MAGIC); - vv = VTAILQ_FIRST(&vj->value->children); - } else if (sym->kind == SYM_OBJECT) { - CAST_OBJ_NOTNULL(vv, sym->eval_priv, VJSN_VAL_MAGIC); - } else { - WRONG("symbol kind"); - } - - for (; vv != NULL; vv = VTAILQ_NEXT(vv, list)) { - if (!vjsn_is_array(vv)) - continue; - vv1 = VTAILQ_FIRST(&vv->children); - assert(vjsn_is_string(vv1)); - vv2 = VTAILQ_NEXT(vv1, list); - if (!vjsn_is_string(vv2)) - continue; - - kind = vcc_vmod_kind(vv1->value); - if (kind == SYM_NONE) - continue; - - func_sym(tl, kind, sym, vv2); - } + (void)vim; + do { + assert (vjsn_is_string(vv)); + Fh(tl, 0, "%s\n", vv->value); + vv = VTAILQ_NEXT(vv, list); + } while(vv != NULL); } static void -vcc_emit_c_prototypes(const struct vcc *tl, const struct vjsn *vj) +vcc_vj_foreach(struct vcc *tl, const struct vmod_import *vim, + const char *stanza, vcc_do_stanza_f *func) { const struct vjsn_val *vv, *vv2, *vv3; - Fh(tl, 0, "\n"); - vv = vj->value; + vv = vim->vj->value; assert (vjsn_is_array(vv)); - vv3 = NULL; VTAILQ_FOREACH(vv2, &vv->children, list) { assert (vjsn_is_array(vv2)); vv3 = VTAILQ_FIRST(&vv2->children); assert (vjsn_is_string(vv3)); - if (!strcmp(vv3->value, "$CPROTO")) - break; - } - assert(vv3 != NULL); - while (1) { - vv3 = VTAILQ_NEXT(vv3, list); - if (vv3 == NULL) - break; - assert (vjsn_is_string(vv3)); - Fh(tl, 0, "%s\n", vv3->value); + if (!strcmp(vv3->value, stanza)) + func(tl, vim, VTAILQ_NEXT(vv3, list)); } } +static void +vcc_emit_setup(struct vcc *tl, const struct vmod_import *vim) +{ + struct inifin *ifp; + const struct token *mod = vim->t_mod; + + ifp = New_IniFin(tl); + + VSB_cat(ifp->ini, "\tif (VPI_Vmod_Init(ctx,\n"); + VSB_printf(ifp->ini, "\t &VGC_vmod_%.*s,\n", PF(mod)); + VSB_printf(ifp->ini, "\t %u,\n", tl->vmod_count++); + VSB_printf(ifp->ini, "\t &%s,\n", vim->func_name); + VSB_printf(ifp->ini, "\t sizeof(%s),\n", vim->func_name); + VSB_printf(ifp->ini, "\t \"%.*s\",\n", PF(mod)); + VSB_cat(ifp->ini, "\t "); + VSB_quote(ifp->ini, vim->path, -1, VSB_QUOTE_CSTR); + VSB_cat(ifp->ini, ",\n"); + AN(vim->file_id); + VSB_printf(ifp->ini, "\t \"%s\",\n", vim->file_id); + VSB_printf(ifp->ini, "\t \"./vmod_cache/_vmod_%.*s.%s\"\n", + PF(mod), vim->file_id); + VSB_cat(ifp->ini, "\t ))\n"); + VSB_cat(ifp->ini, "\t\treturn(1);"); + + VSB_cat(tl->symtab, ",\n {\n"); + VSB_cat(tl->symtab, "\t\"dir\": \"import\",\n"); + VSB_cat(tl->symtab, "\t\"type\": \"$VMOD\",\n"); + VSB_printf(tl->symtab, "\t\"name\": \"%.*s\",\n", PF(mod)); + VSB_printf(tl->symtab, "\t\"file\": \"%s\",\n", vim->path); + VSB_printf(tl->symtab, "\t\"dst\": \"./vmod_cache/_vmod_%.*s.%s\"\n", + PF(mod), vim->file_id); + VSB_cat(tl->symtab, " }"); + + /* XXX: zero the function pointer structure ?*/ + VSB_printf(ifp->fin, "\t\tVRT_priv_fini(ctx, &vmod_priv_%.*s);", + PF(mod)); + VSB_printf(ifp->final, "\t\tVPI_Vmod_Unload(ctx, &VGC_vmod_%.*s);", + PF(mod)); + + vcc_vj_foreach(tl, vim, "$EVENT", vcc_do_event); + + Fh(tl, 0, "\n/* --- BEGIN VMOD %.*s --- */\n\n", PF(mod)); + Fh(tl, 0, "static struct vmod *VGC_vmod_%.*s;\n", PF(mod)); + Fh(tl, 0, "static struct vmod_priv vmod_priv_%.*s;\n", PF(mod)); + + vcc_vj_foreach(tl, vim, "$CPROTO", vcc_do_cproto); + + Fh(tl, 0, "\n/* --- END VMOD %.*s --- */\n\n", PF(mod)); +} + +static void +vcc_vim_destroy(struct vmod_import **vimp) +{ + struct vmod_import *vim; + + TAKE_OBJ_NOTNULL(vim, vimp, VMOD_IMPORT_MAGIC); + if (vim->path) + free(vim->path); + if (vim->vj) + vjsn_delete(&vim->vj); + FREE_OBJ(vim); +} + void vcc_ParseImport(struct vcc *tl) { - char fn[1024], *fnpx; + char fn[1024]; const char *p; struct token *mod, *tmod, *t1; - struct inifin *ifp; struct symbol *msym, *vsym; - const struct vmod_data *vmd; - struct vjsn *vj; struct vmod_import *vim; const struct vmod_import *vimold; @@ -428,8 +388,10 @@ vcc_ParseImport(struct vcc *tl) ALLOC_OBJ(vim, VMOD_IMPORT_MAGIC); AN(vim); + vim->t_mod = mod; + vim->sym = msym; - if (VFIL_searchpath(tl->vmod_path, vcc_path_dlopen, vim, fn, &fnpx)) { + if (VFIL_searchpath(tl->vmod_path, vcc_path_dlopen, vim, fn, &vim->path)) { if (vim->err == NULL) { VSB_printf(tl->sb, "Could not find VMOD %.*s\n", PF(mod)); @@ -437,26 +399,24 @@ vcc_ParseImport(struct vcc *tl) VSB_printf(tl->sb, "Could not open VMOD %.*s\n", PF(mod)); VSB_printf(tl->sb, "\tFile name: %s\n", - fnpx != NULL ? fnpx : fn); + vim->path != NULL ? vim->path : fn); VSB_printf(tl->sb, "\tdlerror: %s\n", vim->err); } vcc_ErrWhere(tl, mod); - free(fnpx); - FREE_OBJ(vim); + vcc_vim_destroy(&vim); return; } - vmd = vcc_VmodSanity(tl, vim->hdl, mod, fnpx); - if (vmd == NULL || tl->err) { - AZ(dlclose(vim->hdl)); - free(fnpx); - FREE_OBJ(vim); + if (vcc_VmodLoad(tl, vim, vim->path) < 0 || tl->err) { + vcc_ErrWhere(tl, vim->t_mod); + vcc_vim_destroy(&vim); return; } - CAST_OBJ(vimold, (const void*)(msym->extra), VMOD_IMPORT_MAGIC); + vimold = msym->import; if (vimold != NULL) { - if (!strcmp(vimold->vmd->file_id, vmd->file_id)) { + CHECK_OBJ_NOTNULL(vimold, VMOD_IMPORT_MAGIC); + if (!strcmp(vimold->file_id, vim->file_id)) { /* Identical import is OK */ } else { VSB_printf(tl->sb, @@ -464,9 +424,7 @@ vcc_ParseImport(struct vcc *tl) PF(tmod)); vcc_ErrWhere2(tl, t1, tl->t); } - AZ(dlclose(vim->hdl)); - free(fnpx); - FREE_OBJ(vim); + vcc_vim_destroy(&vim); return; } msym->def_b = t1; @@ -474,151 +432,26 @@ vcc_ParseImport(struct vcc *tl) VTAILQ_FOREACH(vsym, &tl->sym_vmods, sideways) { assert(vsym->kind == SYM_VMOD); - CAST_OBJ_NOTNULL(vimold, (const void*)(vsym->extra), - VMOD_IMPORT_MAGIC); - if (!strcmp(vimold->vmd->file_id, vmd->file_id)) { + vimold = vsym->import; + CHECK_OBJ_NOTNULL(vimold, VMOD_IMPORT_MAGIC); + if (!strcmp(vimold->file_id, vim->file_id)) { /* Already loaded under different name */ msym->eval_priv = vsym->eval_priv; - msym->extra = vsym->extra; + msym->import = vsym->import; msym->vmod_name = vsym->vmod_name; vcc_VmodSymbols(tl, msym); - AZ(dlclose(vim->hdl)); - free(fnpx); - FREE_OBJ(vim); + // XXX: insert msym in sideways ? + vcc_vim_destroy(&vim); return; } } VTAILQ_INSERT_TAIL(&tl->sym_vmods, msym, sideways); - ifp = New_IniFin(tl); - - VSB_cat(ifp->ini, "\tif (VPI_Vmod_Init(ctx,\n"); - VSB_printf(ifp->ini, "\t &VGC_vmod_%.*s,\n", PF(mod)); - VSB_printf(ifp->ini, "\t %u,\n", tl->vmod_count++); - VSB_printf(ifp->ini, "\t &%s,\n", vmd->func_name); - VSB_printf(ifp->ini, "\t sizeof(%s),\n", vmd->func_name); - VSB_printf(ifp->ini, "\t \"%.*s\",\n", PF(mod)); - VSB_cat(ifp->ini, "\t "); - VSB_quote(ifp->ini, fnpx, -1, VSB_QUOTE_CSTR); - VSB_cat(ifp->ini, ",\n"); - AN(vmd); - AN(vmd->file_id); - VSB_printf(ifp->ini, "\t \"%s\",\n", vmd->file_id); - VSB_printf(ifp->ini, "\t \"./vmod_cache/_vmod_%.*s.%s\"\n", - PF(mod), vmd->file_id); - VSB_cat(ifp->ini, "\t ))\n"); - VSB_cat(ifp->ini, "\t\treturn(1);"); - - VSB_cat(tl->symtab, ",\n {\n"); - VSB_cat(tl->symtab, "\t\"dir\": \"import\",\n"); - VSB_cat(tl->symtab, "\t\"type\": \"$VMOD\",\n"); - VSB_printf(tl->symtab, "\t\"name\": \"%.*s\",\n", PF(mod)); - VSB_printf(tl->symtab, "\t\"file\": \"%s\",\n", fnpx); - VSB_printf(tl->symtab, "\t\"dst\": \"./vmod_cache/_vmod_%.*s.%s\"\n", - PF(mod), vmd->file_id); - VSB_cat(tl->symtab, " }"); - - /* XXX: zero the function pointer structure ?*/ - VSB_printf(ifp->fin, "\t\tVRT_priv_fini(ctx, &vmod_priv_%.*s);", PF(mod)); - VSB_printf(ifp->final, "\t\tVPI_Vmod_Unload(ctx, &VGC_vmod_%.*s);", PF(mod)); - - vj = vjsn_parse(vmd->json, &p); - XXXAZ(p); - AN(vj); - msym->eval_priv = vj; - vim->vmd = vmd; - msym->extra = (const char *)vim; - msym->vmod_name = TlDup(tl, vmd->name); + msym->eval_priv = vim->vj; + msym->import = vim; + msym->vmod_name = TlDup(tl, vim->name); vcc_VmodSymbols(tl, msym); - vcc_json_always(tl, vj, msym->vmod_name); - - Fh(tl, 0, "\n/* --- BEGIN VMOD %.*s --- */\n\n", PF(mod)); - Fh(tl, 0, "static struct vmod *VGC_vmod_%.*s;\n", PF(mod)); - Fh(tl, 0, "static struct vmod_priv vmod_priv_%.*s;\n", PF(mod)); - vcc_emit_c_prototypes(tl, vj); - Fh(tl, 0, "\n/* --- END VMOD %.*s --- */\n\n", PF(mod)); - free(fnpx); -} - -void v_matchproto_(sym_act_f) -vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym) -{ - struct symbol *isym, *osym; - struct inifin *ifp; - struct vsb *buf; - const struct vjsn_val *vv, *vf; - int null_ok = 0; - - (void)sym; - (void)t; - - ExpectErr(tl, ID); - vcc_ExpectVid(tl, "VCL object"); - ERRCHK(tl); - isym = VCC_HandleSymbol(tl, INSTANCE); - ERRCHK(tl); - AN(isym); - isym->noref = 1; - isym->action = vcc_Act_Obj; - - SkipToken(tl, '='); - ExpectErr(tl, ID); - osym = VCC_SymbolGet(tl, SYM_MAIN, SYM_OBJECT, SYMTAB_EXISTING, - XREF_NONE); - ERRCHK(tl); - AN(osym); - - /* Scratch the generic INSTANCE type */ - isym->type = osym->type; - - CAST_OBJ_NOTNULL(vv, osym->eval_priv, VJSN_VAL_MAGIC); - // vv = object name - - isym->vmod_name = osym->vmod_name; - isym->eval_priv = vv; - - vv = VTAILQ_NEXT(vv, list); - // vv = flags - assert(vjsn_is_object(vv)); - VTAILQ_FOREACH(vf, &vv->children, list) - if (!strcmp(vf->name, "NULL_OK") && vjsn_is_true(vf)) - null_ok = 1; - if (!null_ok) - VTAILQ_INSERT_TAIL(&tl->sym_objects, isym, sideways); - - vv = VTAILQ_NEXT(vv, list); - // vv = struct name - - Fh(tl, 0, "static %s *%s;\n\n", vv->value, isym->rname); - vv = VTAILQ_NEXT(vv, list); - - vf = VTAILQ_FIRST(&vv->children); - vv = VTAILQ_NEXT(vv, list); - assert(vjsn_is_string(vf)); - assert(!strcmp(vf->value, "$INIT")); - - vf = VTAILQ_NEXT(vf, list); - - buf = VSB_new_auto(); - AN(buf); - VSB_printf(buf, "&%s, \"%s\"", isym->rname, isym->name); - AZ(VSB_finish(buf)); - vcc_Eval_Func(tl, vf, VSB_data(buf), osym); - VSB_destroy(&buf); - ERRCHK(tl); - SkipToken(tl, ';'); - isym->def_e = tl->t; - - vf = VTAILQ_FIRST(&vv->children); - assert(vjsn_is_string(vf)); - assert(!strcmp(vf->value, "$FINI")); - - vf = VTAILQ_NEXT(vf, list); - vf = VTAILQ_FIRST(&vf->children); - vf = VTAILQ_NEXT(vf, list); - ifp = New_IniFin(tl); - VSB_printf(ifp->fin, "\t\tif (%s)\n", isym->rname); - VSB_printf(ifp->fin, "\t\t\t\t%s(&%s);", vf->value, isym->rname); + vcc_emit_setup(tl, vim); } diff --git a/lib/libvcc/vcc_vmod.h b/lib/libvcc/vcc_vmod.h new file mode 100644 index 000000000..2f386cf5a --- /dev/null +++ b/lib/libvcc/vcc_vmod.h @@ -0,0 +1,40 @@ +/*- + * Copyright (c) 2010-2015 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#define STANZA_TBL \ + STANZA(ALIAS, alias, SYM_ALIAS) \ + STANZA(CPROTO, cproto, SYM_NONE) \ + STANZA(EVENT, evant, SYM_NONE) \ + STANZA(FUNC, func, SYM_FUNC) \ + STANZA(METHOD, method, SYM_METHOD) \ + STANZA(OBJ, obj, SYM_OBJECT) \ + STANZA(VMOD, vmod, SYM_NONE) + +void vcc_VmodSymbols(struct vcc *tl, const struct symbol *sym); diff --git a/lib/libvcc/vcc_vmod_sym.c b/lib/libvcc/vcc_vmod_sym.c new file mode 100644 index 000000000..65f7f4d6f --- /dev/null +++ b/lib/libvcc/vcc_vmod_sym.c @@ -0,0 +1,273 @@ +/*- + * Copyright (c) 2010-2015 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * Turn vmod JSON spec into symbols + * + */ + +#include "config.h" + +#include +#include + +#include "vcc_compile.h" + +#include "libvcc.h" +#include "vjsn.h" +#include "vsb.h" + +#include "vcc_vmod.h" + +struct vmod_obj { + unsigned magic; +#define VMOD_OBJ_MAGIC 0x349885f8 + char *name; + struct type type[1]; + VTAILQ_ENTRY(vmod_obj) list; +}; + +static void +vcc_VmodObject(struct vcc *tl, struct symbol *sym) +{ + struct vmod_obj *obj; + struct vsb *buf; + + buf = VSB_new_auto(); + AN(buf); + + VSB_printf(buf, "%s.%s", sym->vmod_name, sym->name); + AZ(VSB_finish(buf)); + + ALLOC_OBJ(obj, VMOD_OBJ_MAGIC); + AN(obj); + REPLACE(obj->name, VSB_data(buf)); + + INIT_OBJ(obj->type, TYPE_MAGIC); + obj->type->name = obj->name; + sym->type = obj->type; + VTAILQ_INSERT_TAIL(&tl->vmod_objects, obj, list); + VSB_destroy(&buf); +} + +static void +alias_sym(struct vcc *tl, const struct symbol *psym, const struct vjsn_val *v) +{ + char *alias = NULL, *func = NULL; + struct symbol *sym; + struct vsb *buf; + + buf = VSB_new_auto(); + AN(buf); + + VCC_SymName(buf, psym); + VSB_printf(buf, ".%s", v->value); + AZ(VSB_finish(buf)); + REPLACE(alias, VSB_data(buf)); + + v = VTAILQ_NEXT(v, list); + assert(vjsn_is_string(v)); + + VSB_clear(buf); + VCC_SymName(buf, psym); + VSB_printf(buf, ".%s", v->value); + AZ(VSB_finish(buf)); + REPLACE(func, VSB_data(buf)); + + sym = VCC_MkSymAlias(tl, alias, func); + AN(sym); + assert(sym->kind == SYM_FUNC || sym->kind == SYM_METHOD); + VSB_destroy(&buf); + free(alias); + free(func); +} + +static void +func_sym(struct vcc *tl, vcc_kind_t kind, const struct symbol *psym, + const struct vjsn_val *v) +{ + struct symbol *sym; + struct vsb *buf; + + if (kind == SYM_ALIAS) { + alias_sym(tl, psym, v); + return; + } + + buf = VSB_new_auto(); + AN(buf); + + VCC_SymName(buf, psym); + VSB_printf(buf, ".%s", v->value); + AZ(VSB_finish(buf)); + sym = VCC_MkSym(tl, VSB_data(buf), SYM_MAIN, kind, VCL_LOW, VCL_HIGH); + AN(sym); + VSB_destroy(&buf); + + if (kind == SYM_OBJECT) { + sym->eval_priv = v; + sym->vmod_name = psym->vmod_name; + sym->r_methods = VCL_MET_INIT; + vcc_VmodObject(tl, sym); + vcc_VmodSymbols(tl, sym); + return; + } + + if (kind == SYM_METHOD) + sym->extra = psym->rname; + + v = VTAILQ_NEXT(v, list); + + assert(vjsn_is_array(v)); + sym->action = vcc_Act_Call; + sym->vmod_name = psym->vmod_name; + sym->eval = vcc_Eval_SymFunc; + sym->eval_priv = v; + v = VTAILQ_FIRST(&v->children); + assert(vjsn_is_array(v)); + v = VTAILQ_FIRST(&v->children); + assert(vjsn_is_string(v)); + sym->type = VCC_Type(v->value); + AN(sym->type); + sym->r_methods = VCL_MET_TASK_ALL; +} + +void +vcc_VmodSymbols(struct vcc *tl, const struct symbol *sym) +{ + const struct vjsn *vj; + const struct vjsn_val *vv, *vv1, *vv2; + vcc_kind_t kind; + + if (sym->kind == SYM_VMOD) { + CAST_OBJ_NOTNULL(vj, sym->eval_priv, VJSN_MAGIC); + vv = VTAILQ_FIRST(&vj->value->children); + } else if (sym->kind == SYM_OBJECT) { + CAST_OBJ_NOTNULL(vv, sym->eval_priv, VJSN_VAL_MAGIC); + } else { + WRONG("symbol kind"); + } + + for (; vv != NULL; vv = VTAILQ_NEXT(vv, list)) { + if (!vjsn_is_array(vv)) + continue; + vv1 = VTAILQ_FIRST(&vv->children); + assert(vjsn_is_string(vv1)); + vv2 = VTAILQ_NEXT(vv1, list); + if (!vjsn_is_string(vv2)) + continue; + + kind = SYM_NONE; +#define STANZA(UU, ll, ss) if (!strcmp(vv1->value, "$" #UU)) kind = ss; + STANZA_TBL +#undef STANZA + if (kind != SYM_NONE) + func_sym(tl, kind, sym, vv2); + } +} + +void v_matchproto_(sym_act_f) +vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym) +{ + struct symbol *isym, *osym; + struct inifin *ifp; + struct vsb *buf; + const struct vjsn_val *vv, *vf; + int null_ok = 0; + + (void)sym; + (void)t; + + ExpectErr(tl, ID); + vcc_ExpectVid(tl, "VCL object"); + ERRCHK(tl); + isym = VCC_HandleSymbol(tl, INSTANCE); + ERRCHK(tl); + AN(isym); + isym->noref = 1; + isym->action = vcc_Act_Obj; + + SkipToken(tl, '='); + ExpectErr(tl, ID); + osym = VCC_SymbolGet(tl, SYM_MAIN, SYM_OBJECT, SYMTAB_EXISTING, + XREF_NONE); + ERRCHK(tl); + AN(osym); + + /* Scratch the generic INSTANCE type */ + isym->type = osym->type; + + CAST_OBJ_NOTNULL(vv, osym->eval_priv, VJSN_VAL_MAGIC); + // vv = object name + + isym->vmod_name = osym->vmod_name; + isym->eval_priv = vv; + + vv = VTAILQ_NEXT(vv, list); + // vv = flags + assert(vjsn_is_object(vv)); + VTAILQ_FOREACH(vf, &vv->children, list) + if (!strcmp(vf->name, "NULL_OK") && vjsn_is_true(vf)) + null_ok = 1; + if (!null_ok) + VTAILQ_INSERT_TAIL(&tl->sym_objects, isym, sideways); + + vv = VTAILQ_NEXT(vv, list); + // vv = struct name + + Fh(tl, 0, "static %s *%s;\n\n", vv->value, isym->rname); + vv = VTAILQ_NEXT(vv, list); + + vf = VTAILQ_FIRST(&vv->children); + vv = VTAILQ_NEXT(vv, list); + assert(vjsn_is_string(vf)); + assert(!strcmp(vf->value, "$INIT")); + + vf = VTAILQ_NEXT(vf, list); + + buf = VSB_new_auto(); + AN(buf); + VSB_printf(buf, "&%s, \"%s\"", isym->rname, isym->name); + AZ(VSB_finish(buf)); + vcc_Eval_Func(tl, vf, VSB_data(buf), osym); + VSB_destroy(&buf); + ERRCHK(tl); + SkipToken(tl, ';'); + isym->def_e = tl->t; + + vf = VTAILQ_FIRST(&vv->children); + assert(vjsn_is_string(vf)); + assert(!strcmp(vf->value, "$FINI")); + + vf = VTAILQ_NEXT(vf, list); + vf = VTAILQ_FIRST(&vf->children); + vf = VTAILQ_NEXT(vf, list); + ifp = New_IniFin(tl); + VSB_printf(ifp->fin, "\t\tif (%s)\n", isym->rname); + VSB_printf(ifp->fin, "\t\t\t\t%s(&%s);", vf->value, isym->rname); +} diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 823e12385..1ea7f66be 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -1086,7 +1086,7 @@ class vcc(object): def iter_json(self, fnx): - jl = [["$VMOD", "1.0"]] + jl = [["$VMOD", "1.0", self.modname, self.csn, self.file_id]] jl.append(["$CPROTO"]) for i in open(fnx): jl[-1].append(i.rstrip()) diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index 53b0ebc14..a0a11f5b6 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -1087,7 +1087,15 @@ const struct vmod_data Vmod_wrong2_Data = { .func = foo_struct, .func_len = sizeof foo_struct, .func_name = "foo_struct", - .json = "blablabla", + .json = "[" + "[" + "\"$VMOD\"" + ", \"1.0\"" + ", \"wrongN\"" + ", \"foo_struct\"" + ", \"file_id\"" + "]" + "]", }; extern const struct vmod_data Vmod_wrong3_Data; @@ -1098,7 +1106,16 @@ const struct vmod_data Vmod_wrong3_Data = { .func = foo_struct, .func_len = sizeof foo_struct, .func_name = "foo_struct", - .json = "blablabla", + .json = "[" + "[" + "\"$VMOD\"" + ", \"1.0\"" + ", \"wrongN\"" + ", \"foo_struct\"" + ", \"file_id\"" + "]" + ", [\"$CPROTO\", \"\"]" + "]", .abi = "abiblabla", }; From phk at FreeBSD.org Thu Jun 23 10:37:05 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 23 Jun 2022 10:37:05 +0000 (UTC) Subject: [master] c796ea52d Add abi, major and minor to VMOD JSON spec Message-ID: <20220623103705.260EC110615@lists.varnish-cache.org> commit c796ea52d0f3369ec8e53aa8bebbef00fa65c391 Author: Poul-Henning Kamp Date: Thu Jun 23 10:00:32 2022 +0000 Add abi, major and minor to VMOD JSON spec diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index a9f169d10..a90712f19 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -54,10 +54,14 @@ struct vmod_import { const char *err; char *path; + // From $VMOD double vmod_syntax; char *name; char *func_name; char *file_id; + char *abi; + unsigned major; + unsigned minor; struct symbol *sym; const struct token *t_mod; @@ -102,31 +106,52 @@ vcc_ParseJSON(const struct vcc *tl, const char *jsn, struct vmod_import *vim) return ("Not array[0]"); vv2 = VTAILQ_FIRST(&vv->children); + AN(vv2); if (!vjsn_is_array(vv2)) return ("Not array[1]"); vv3 = VTAILQ_FIRST(&vv2->children); + AN(vv3); if (!vjsn_is_string(vv3)) return ("Not string[2]"); if (strcmp(vv3->value, "$VMOD")) return ("Not $VMOD[3]"); vv3 = VTAILQ_NEXT(vv3, list); + AN(vv3); assert(vjsn_is_string(vv3)); vim->vmod_syntax = strtod(vv3->value, NULL); assert (vim->vmod_syntax == 1.0); vv3 = VTAILQ_NEXT(vv3, list); + AN(vv3); assert(vjsn_is_string(vv3)); vim->name = vv3->value; vv3 = VTAILQ_NEXT(vv3, list); + AN(vv3); assert(vjsn_is_string(vv3)); vim->func_name = vv3->value; vv3 = VTAILQ_NEXT(vv3, list); + AN(vv3); assert(vjsn_is_string(vv3)); vim->file_id = vv3->value; + vv3 = VTAILQ_NEXT(vv3, list); + AN(vv3); + assert(vjsn_is_string(vv3)); + vim->abi = vv3->value; + + vv3 = VTAILQ_NEXT(vv3, list); + AN(vv3); + assert(vjsn_is_number(vv3)); + vim->major = atoi(vv3->value); + + vv3 = VTAILQ_NEXT(vv3, list); + AN(vv3); + assert(vjsn_is_number(vv3)); + vim->minor = atoi(vv3->value); + if (!vcc_IdIs(vim->t_mod, vim->name)) { VSB_printf(tl->sb, "Wrong file for VMOD %.*s\n", PF(vim->t_mod)); @@ -135,6 +160,27 @@ vcc_ParseJSON(const struct vcc *tl, const char *jsn, struct vmod_import *vim) return (""); } + if (vim->major == 0 && vim->minor == 0 && + strcmp(vim->abi, VMOD_ABI_Version)) { + VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(vim->t_mod)); + VSB_printf(tl->sb, "\tFile name: %s\n", vim->path); + VSB_printf(tl->sb, "\tABI mismatch, expected <%s>, got <%s>\n", + VMOD_ABI_Version, vim->abi); + return (""); + } + if (vim->major != 0 && + (vim->major != VRT_MAJOR_VERSION || + vim->minor > VRT_MINOR_VERSION)) { + VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(vim->t_mod)); + VSB_printf(tl->sb, "\tFile name: %s\n", vim->path); + VSB_printf(tl->sb, "\tVMOD wants ABI version %u.%u\n", + vim->major, vim->minor); + VSB_printf(tl->sb, "\tvarnishd provides ABI version %u.%u\n", + VRT_MAJOR_VERSION, VRT_MINOR_VERSION); + return (""); + } + + VTAILQ_FOREACH(vv2, &vv->children, list) { assert (vjsn_is_array(vv2)); vv3 = VTAILQ_FIRST(&vv2->children); @@ -204,7 +250,9 @@ vcc_VmodLoad(const struct vcc *tl, struct vmod_import *vim, char *fnp) return (-1); } - err = vcc_ParseJSON(tl, vmd->json, vim); + assert(!memcmp(vmd->json, "VMOD_JSON_SPEC\x03", 15)); + + err = vcc_ParseJSON(tl, vmd->json + 15, vim); AZ(dlclose(vim->hdl)); vim->hdl = NULL; if (err != NULL && *err != '\0') { diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 1ea7f66be..50e5ab99b 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -639,6 +639,12 @@ class ABIStanza(Stanza): if self.vcc.strict_abi is None: err("Valid ABI types are 'strict' or 'vrt', got '%s'\n" % self.toks[1]) + if self.vcc.strict_abi: + self.vcc.vrt_major = "0" + self.vcc.vrt_minor = "0" + else: + self.vcc.vrt_major = "VRT_MAJOR_VERSION" + self.vcc.vrt_minor = "VRT_MINOR_VERSION" self.vcc.contents.append(self) @@ -1097,16 +1103,27 @@ class vcc(object): yield i + " " def json(self, fo, fnx): + fo.write('#define STRINGIFY(arg) #arg\n') fo.write("\nstatic const char Vmod_Json[] = {\n") + fo.write('\t"VMOD_JSON_SPEC\\x03"\n') - for i in self.iter_json(fnx): + for n, i in enumerate(self.iter_json(fnx)): fo.write('\t"') for j in i: if j in '"\\': fo.write('\\') fo.write(j) - fo.write('"\n') + if n == 6: + # Hand-munge the JSON to insert stuff only known by + # the C-compiler at compile-time. + fo.write(',"\n\t" \\"" VMOD_ABI_Version "\\", "\n') + fo.write('\t STRINGIFY(%s) ", "\n' % self.vrt_major) + fo.write('\t STRINGIFY(%s)\n' % self.vrt_minor) + else: + fo.write('"\n') fo.write('\t\"\\n\"\n};\n') + fo.write('#undef STRINGIFY\n') + def vmod_data(self, fo): vmd = "Vmod_%s_Data" % self.modname @@ -1115,12 +1132,8 @@ class vcc(object): fo.write("/*lint -esym(%d, %s) */\n" % (i, vmd)) fo.write("\nextern const struct vmod_data %s;\n" % vmd) fo.write("\nconst struct vmod_data %s = {\n" % vmd) - if self.strict_abi: - fo.write("\t.vrt_major =\t0,\n") - fo.write("\t.vrt_minor =\t0,\n") - else: - fo.write("\t.vrt_major =\tVRT_MAJOR_VERSION,\n") - fo.write("\t.vrt_minor =\tVRT_MINOR_VERSION,\n") + fo.write("\t.vrt_major =\t%s,\n" % self.vrt_major) + fo.write("\t.vrt_minor =\t%s,\n" % self.vrt_minor) fo.write('\t.name =\t\t"%s",\n' % self.modname) fo.write('\t.func =\t\t&%s,\n' % self.csn) fo.write('\t.func_len =\tsizeof(%s),\n' % self.csn) diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index a0a11f5b6..846b00d92 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -1087,13 +1087,15 @@ const struct vmod_data Vmod_wrong2_Data = { .func = foo_struct, .func_len = sizeof foo_struct, .func_name = "foo_struct", - .json = "[" + .json = "VMOD_JSON_SPEC\x03[" "[" "\"$VMOD\"" ", \"1.0\"" ", \"wrongN\"" ", \"foo_struct\"" ", \"file_id\"" + ", \"abi\"" + ", 0, 0" // major, minor "]" "]", }; @@ -1106,13 +1108,15 @@ const struct vmod_data Vmod_wrong3_Data = { .func = foo_struct, .func_len = sizeof foo_struct, .func_name = "foo_struct", - .json = "[" + .json = "VMOD_JSON_SPEC\x03[" "[" "\"$VMOD\"" ", \"1.0\"" ", \"wrongN\"" ", \"foo_struct\"" ", \"file_id\"" + ", \"abi\"" + ", 0, 0" // major, minor "]" ", [\"$CPROTO\", \"\"]" "]", From phk at FreeBSD.org Thu Jun 23 10:37:05 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 23 Jun 2022 10:37:05 +0000 (UTC) Subject: [master] af87f0cd1 Dont dlopen vmods in VCC sub-process. Message-ID: <20220623103705.3E322110618@lists.varnish-cache.org> commit af87f0cd1a7b8be8a124dca355f04dd641bc5512 Author: Poul-Henning Kamp Date: Thu Jun 23 10:36:17 2022 +0000 Dont dlopen vmods in VCC sub-process. diff --git a/bin/varnishtest/tests/m00003.vtc b/bin/varnishtest/tests/m00003.vtc index c2b00e45e..a9281bb8c 100644 --- a/bin/varnishtest/tests/m00003.vtc +++ b/bin/varnishtest/tests/m00003.vtc @@ -42,24 +42,8 @@ shell { ${tmpdir}/libvmod_wrong.so } -varnish v1 -errvcl {Malformed VMOD wrong} { +varnish v1 -errvcl {Wrong file for VMOD wrong} { import wrong; } -varnish v1 -errvcl {Incompatible VMOD wrong0} { - import wrong0 from "${tmpdir}/libvmod_wrong.so"; -} - -varnish v1 -errvcl {Incompatible VMOD wrong1} { - import wrong1 from "${tmpdir}/libvmod_wrong.so"; -} - -varnish v1 -errvcl {Mangled VMOD wrong2} { - import wrong2 from "${tmpdir}/libvmod_wrong.so"; -} - -varnish v1 -errvcl {Wrong file for VMOD wrong3} { - import wrong3 from "${tmpdir}/libvmod_wrong.so"; -} - shell "rm -f ${tmpdir}/libvmod_wrong.so" diff --git a/bin/varnishtest/tests/m00008.vtc b/bin/varnishtest/tests/m00008.vtc index 234689aa4..d6e9aa37e 100644 --- a/bin/varnishtest/tests/m00008.vtc +++ b/bin/varnishtest/tests/m00008.vtc @@ -51,7 +51,7 @@ varnish v1 -vcl+backend { import std from "${topbuild}/vmod/.libs/"; } -varnish v1 -errvcl {Malformed VMOD std} { +varnish v1 -errvcl {Wrong file for VMOD std} { backend default { .host = "${s1_sock}"; } import std from "${topbuild}/vmod/.libs/libvmod_debug.so"; } diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index a90712f19..0e9463e82 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -33,7 +33,6 @@ #include "config.h" -#include #include #include @@ -50,8 +49,8 @@ struct vmod_import { unsigned magic; #define VMOD_IMPORT_MAGIC 0x31803a5d - void *hdl; const char *err; + struct vsb *json; char *path; // From $VMOD @@ -75,18 +74,55 @@ typedef void vcc_do_stanza_f(struct vcc *tl, const struct vmod_import *vim, const struct vjsn_val *vv); static int -vcc_path_dlopen(void *priv, const char *fn) +vcc_path_open(void *priv, const char *fn) { struct vmod_import *vim; + const char *magic = "VMOD_JSON_SPEC\x03", *p; + int c; + FILE *f; CAST_OBJ_NOTNULL(vim, priv, VMOD_IMPORT_MAGIC); AN(fn); - vim->hdl = dlopen(fn, RTLD_NOW | RTLD_LOCAL); - if (vim->hdl == NULL) { - vim->err = dlerror(); + vim->json = VSB_new_auto(); + AN(vim->json); + + f = fopen(fn, "rb"); + if (f == NULL) { + vim->err = strerror(errno); return (-1); } + p = magic; + vim->err = "No VMOD JSON found"; + while (1) { + c = getc(f); + if (c == EOF) { + AZ(fclose(f)); + vim->err = "No VMOD JSON found"; + return (-1); + } + if (c != *p) { + p = magic; + continue; + } + p++; + if (*p == '\0') + break; + } + + while (1) { + c = getc(f); + if (c == EOF) { + AZ(fclose(f)); + vim->err = "No VMOD JSON found"; + return (-1); + } + if (c == '\0') + break; + VSB_putc(vim->json, c); + } + AZ(fclose(f)); + AZ(VSB_finish(vim->json)); return (0); } @@ -204,57 +240,13 @@ vcc_ParseJSON(const struct vcc *tl, const char *jsn, struct vmod_import *vim) */ static int -vcc_VmodLoad(const struct vcc *tl, struct vmod_import *vim, char *fnp) +vcc_VmodLoad(const struct vcc *tl, struct vmod_import *vim) { - char buf[256]; static const char *err; - const struct vmod_data *vmd; CHECK_OBJ_NOTNULL(vim, VMOD_IMPORT_MAGIC); - bprintf(buf, "Vmod_%.*s_Data", PF(vim->t_mod)); - vmd = dlsym(vim->hdl, buf); - if (vmd == NULL) { - VSB_printf(tl->sb, "Malformed VMOD %.*s\n", PF(vim->t_mod)); - VSB_printf(tl->sb, "\tFile name: %s\n", fnp); - VSB_cat(tl->sb, "\t(no Vmod_Data symbol)\n"); - return (-1); - } - if (vmd->vrt_major == 0 && vmd->vrt_minor == 0 && - (vmd->abi == NULL || strcmp(vmd->abi, VMOD_ABI_Version))) { - VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(vim->t_mod)); - VSB_printf(tl->sb, "\tFile name: %s\n", fnp); - VSB_printf(tl->sb, "\tABI mismatch, expected <%s>, got <%s>\n", - VMOD_ABI_Version, vmd->abi); - return (-1); - } - if (vmd->vrt_major != 0 && - (vmd->vrt_major != VRT_MAJOR_VERSION || - vmd->vrt_minor > VRT_MINOR_VERSION)) { - VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(vim->t_mod)); - VSB_printf(tl->sb, "\tFile name: %s\n", fnp); - VSB_printf(tl->sb, "\tVMOD wants ABI version %u.%u\n", - vmd->vrt_major, vmd->vrt_minor); - VSB_printf(tl->sb, "\tvarnishd provides ABI version %u.%u\n", - VRT_MAJOR_VERSION, VRT_MINOR_VERSION); - return (-1); - } - if (vmd->name == NULL || - vmd->func == NULL || - vmd->func_len <= 0 || - vmd->json == NULL || - vmd->proto != NULL || - vmd->abi == NULL) { - VSB_printf(tl->sb, "Mangled VMOD %.*s\n", PF(vim->t_mod)); - VSB_printf(tl->sb, "\tFile name: %s\n", fnp); - VSB_cat(tl->sb, "\tInconsistent metadata\n"); - return (-1); - } - - assert(!memcmp(vmd->json, "VMOD_JSON_SPEC\x03", 15)); - err = vcc_ParseJSON(tl, vmd->json + 15, vim); - AZ(dlclose(vim->hdl)); - vim->hdl = NULL; + err = vcc_ParseJSON(tl, VSB_data(vim->json), vim); if (err != NULL && *err != '\0') { VSB_printf(tl->sb, "VMOD %.*s: bad metadata\n", PF(vim->t_mod)); @@ -376,6 +368,8 @@ vcc_vim_destroy(struct vmod_import **vimp) free(vim->path); if (vim->vj) vjsn_delete(&vim->vj); + if (vim->json) + VSB_destroy(&vim->json); FREE_OBJ(vim); } @@ -439,7 +433,7 @@ vcc_ParseImport(struct vcc *tl) vim->t_mod = mod; vim->sym = msym; - if (VFIL_searchpath(tl->vmod_path, vcc_path_dlopen, vim, fn, &vim->path)) { + if (VFIL_searchpath(tl->vmod_path, vcc_path_open, vim, fn, &vim->path)) { if (vim->err == NULL) { VSB_printf(tl->sb, "Could not find VMOD %.*s\n", PF(mod)); @@ -455,7 +449,7 @@ vcc_ParseImport(struct vcc *tl) return; } - if (vcc_VmodLoad(tl, vim, vim->path) < 0 || tl->err) { + if (vcc_VmodLoad(tl, vim) < 0 || tl->err) { vcc_ErrWhere(tl, vim->t_mod); vcc_vim_destroy(&vim); return; diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index 846b00d92..091ad3048 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -1055,76 +1055,6 @@ xyzzy_get_ip(VRT_CTX) return (ip); } -/********************************************************************** - * For testing import code on bad vmod files (m00003.vtc) - */ - -//lint -save -e9075 external symbol '...' defined without a prior declaration - -extern const struct vmod_data Vmod_wrong0_Data; -const struct vmod_data Vmod_wrong0_Data = { - .vrt_major = 0, - .vrt_minor = 0, -}; - -//lint -save -e835 A zero has been given as left argument to operatorp'+' -extern const struct vmod_data Vmod_wrong1_Data; -const struct vmod_data Vmod_wrong1_Data = { - .vrt_major = VRT_MAJOR_VERSION, - .vrt_minor = VRT_MINOR_VERSION + 1, -}; -//lint -restore - -static const struct foo { - int bar; -} foo_struct[1]; - -extern const struct vmod_data Vmod_wrong2_Data; -const struct vmod_data Vmod_wrong2_Data = { - .vrt_major = VRT_MAJOR_VERSION, - .vrt_minor = VRT_MINOR_VERSION, - .name = "wrongN", - .func = foo_struct, - .func_len = sizeof foo_struct, - .func_name = "foo_struct", - .json = "VMOD_JSON_SPEC\x03[" - "[" - "\"$VMOD\"" - ", \"1.0\"" - ", \"wrongN\"" - ", \"foo_struct\"" - ", \"file_id\"" - ", \"abi\"" - ", 0, 0" // major, minor - "]" - "]", -}; - -extern const struct vmod_data Vmod_wrong3_Data; -const struct vmod_data Vmod_wrong3_Data = { - .vrt_major = VRT_MAJOR_VERSION, - .vrt_minor = VRT_MINOR_VERSION, - .name = "wrongN", - .func = foo_struct, - .func_len = sizeof foo_struct, - .func_name = "foo_struct", - .json = "VMOD_JSON_SPEC\x03[" - "[" - "\"$VMOD\"" - ", \"1.0\"" - ", \"wrongN\"" - ", \"foo_struct\"" - ", \"file_id\"" - ", \"abi\"" - ", 0, 0" // major, minor - "]" - ", [\"$CPROTO\", \"\"]" - "]", - .abi = "abiblabla", -}; - -//lint -restore - /*---------------------------------------------------------------------*/ struct VPFX(debug_director) { From dridi.boukelmoune at gmail.com Fri Jun 24 11:53:07 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 24 Jun 2022 11:53:07 +0000 (UTC) Subject: [master] 3c6c0917c varnishtest: Skip dispatch servers for -vcl+backend Message-ID: <20220624115307.E0F97115CE6@lists.varnish-cache.org> commit 3c6c0917c1d781bcefe5f17da43ed0ecf8aa2310 Author: Dridi Boukelmoune Date: Wed Jun 22 16:05:46 2022 +0200 varnishtest: Skip dispatch servers for -vcl+backend We don't want to list ephemeral backends when we load a new VCL after getting some s0 traffic. diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index 23606a1ac..783be0a2e 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -428,6 +428,9 @@ cmd_server_gen_vcl(struct vsb *vsb) AZ(pthread_mutex_lock(&server_mtx)); VTAILQ_FOREACH(s, &servers, list) { + if (s->sock < 0 && s->fd >= 0) /* dispatch instance */ + continue; + if (*s->listen != '/') VSB_printf(vsb, "backend %s { .host = \"%s\"; " From nils.goroll at uplex.de Fri Jun 24 14:49:02 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 24 Jun 2022 14:49:02 +0000 (UTC) Subject: [master] c23ef4c3d Avoid panic in VRT_CacheReqBody if called from outside vcl_recv {} Message-ID: <20220624144902.AA55911ABDA@lists.varnish-cache.org> commit c23ef4c3d874871061792add005fdfda9912db43 Author: Nils Goroll Date: Fri Jun 24 16:42:03 2022 +0200 Avoid panic in VRT_CacheReqBody if called from outside vcl_recv {} diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index bd856193c..ef43a3ade 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -973,12 +973,12 @@ VRT_CacheReqBody(VRT_CTX, VCL_BYTES maxsize) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); if (ctx->method != VCL_MET_RECV) { VSLb(ctx->vsl, SLT_VCL_Error, "req.body can only be cached in vcl_recv{}"); return (-1); } + CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); return (VRB_Cache(ctx->req, maxsize)); } From nils.goroll at uplex.de Fri Jun 24 15:01:03 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 24 Jun 2022 15:01:03 +0000 (UTC) Subject: [master] 870f16ceb Avoid panic in VRT_CacheReqBody if called from vcl_init{} Message-ID: <20220624150103.65CC511B306@lists.varnish-cache.org> commit 870f16cebdf2aac9e9a5e0c8d50c4e632528ae54 Author: Nils Goroll Date: Fri Jun 24 16:56:10 2022 +0200 Avoid panic in VRT_CacheReqBody if called from vcl_init{} diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index ef43a3ade..be3d56768 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -971,11 +971,16 @@ VRT_ban_string(VRT_CTX, VCL_STRING str) VCL_BYTES VRT_CacheReqBody(VRT_CTX, VCL_BYTES maxsize) { + const char * const err = "req.body can only be cached in vcl_recv{}"; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if (ctx->method != VCL_MET_RECV) { - VSLb(ctx->vsl, SLT_VCL_Error, - "req.body can only be cached in vcl_recv{}"); + if (ctx->vsl != NULL) { + VSLb(ctx->vsl, SLT_VCL_Error, err); + } else { + AN(ctx->msg); + VSB_printf(ctx->msg, "%s\n", err); + }; return (-1); } CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC);