From nils.goroll at uplex.de Sat Jan 2 13:45:09 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 2 Jan 2021 13:45:09 +0000 (UTC) Subject: [master] 6f039dc3a Remove duplicate includes Message-ID: <20210102134509.A55AAADF74@lists.varnish-cache.org> commit 6f039dc3a270e313a6f0b3e05a4a81aa46715750 Author: Nils Goroll Date: Wed Dec 30 10:54:20 2020 +0100 Remove duplicate includes cache_varnishd.h already includes: - sys/socket.h - string.h Spotted by local Flexelint install on Linux diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index 0caabb819..b7cdbcca6 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -38,7 +38,6 @@ #include #include #include -#include #include "cache_varnishd.h" diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 5aca2b4bb..4ff2ff7ce 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -41,7 +41,6 @@ #include #include -#include #include #include "cache_varnishd.h" From nils.goroll at uplex.de Sat Jan 2 13:45:09 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 2 Jan 2021 13:45:09 +0000 (UTC) Subject: [master] 6cf146bdc Remove superfluous include Message-ID: <20210102134509.D0185ADF77@lists.varnish-cache.org> commit 6cf146bdc4beab0447f2e301b1bb84d5c1325fe0 Author: Nils Goroll Date: Sat Jan 2 11:57:28 2021 +0100 Remove superfluous include Spotted by local Flexelint install on Linux diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index 55a66290f..ffcee5827 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -33,8 +33,6 @@ #include "cache/cache_varnishd.h" -#include - #include #include From nils.goroll at uplex.de Sun Jan 3 14:03:09 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 3 Jan 2021 14:03:09 +0000 (UTC) Subject: [master] 9f5dc5cc3 Plug insignificant vcc leaks Message-ID: <20210103140309.19682ACA00@lists.varnish-cache.org> commit 9f5dc5cc3f7b888211db0c3a8661a21173dab0c0 Author: Nils Goroll Date: Sun Jan 3 14:31:25 2021 +0100 Plug insignificant vcc leaks Ref Coverity CIDs 1362617, 1362618, 1362624, 1362626 diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index 47c0b4ef2..6bfc553b2 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -200,6 +200,7 @@ vcc_acl_add_entry(struct vcc *tl, const struct acl_e *ae, int l, vcc_ErrWhere(tl, ae2->t_addr); VSB_cat(tl->sb, "vs:\n"); vcc_ErrWhere(tl, aen->t_addr); + free(aen); return; } /* @@ -387,12 +388,14 @@ vcc_acl_entry(struct vcc *tl) if (*e != '\0') { VSB_cat(tl->sb, ".../mask is not numeric.\n"); vcc_ErrWhere(tl, ae->t_addr); + free(ae); return; } ae->t_mask = ae->t_addr; if (tl->t->tok == '/') { VSB_cat(tl->sb, "/mask only allowed once.\n"); vcc_ErrWhere(tl, tl->t); + free(ae); return; } } else if (tl->t->tok == '/') { diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c index 4a88eeb8b..a31cb901e 100644 --- a/lib/libvcc/vcc_backend.c +++ b/lib/libvcc/vcc_backend.c @@ -261,6 +261,7 @@ vcc_ParseProbeSpec(struct vcc *tl, const struct symbol *sym, char **name) SkipToken(tl, ';'); } + free(fs); if (t_threshold != NULL || t_window != NULL) { if (t_threshold == NULL && t_window != NULL) { @@ -353,20 +354,6 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname) unsigned u; double t; - fs = vcc_FldSpec(tl, - "?host", - "?port", - "?path", - "?host_header", - "?connect_timeout", - "?first_byte_timeout", - "?between_bytes_timeout", - "?probe", - "?max_connections", - "?proxy_header", - NULL); - - if (tl->t->tok == ID && (vcc_IdIs(tl->t, "none") || vcc_IdIs(tl->t, "None"))) { vcc_NextToken(tl); @@ -391,6 +378,19 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname) return; } + fs = vcc_FldSpec(tl, + "?host", + "?port", + "?path", + "?host_header", + "?connect_timeout", + "?first_byte_timeout", + "?between_bytes_timeout", + "?probe", + "?max_connections", + "?proxy_header", + NULL); + tl->fb = VSB_new_auto(); AN(tl->fb); @@ -512,6 +512,7 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname) } vcc_FieldsOk(tl, fs); + free(fs); ERRCHK(tl); ExpectErr(tl, '}'); From phk at FreeBSD.org Tue Jan 5 09:43:08 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Jan 2021 09:43:08 +0000 (UTC) Subject: [master] c5a4be04d Clarify a fine point in the VAV api. Message-ID: <20210105094308.CD433A0F86@lists.varnish-cache.org> commit c5a4be04d5d9f5ab2ffacc1d49373d2d339858a0 Author: Poul-Henning Kamp Date: Tue Jan 5 09:41:32 2021 +0000 Clarify a fine point in the VAV api. diff --git a/lib/libvarnish/vav.c b/lib/libvarnish/vav.c index 12fb4d6f5..334f757ef 100644 --- a/lib/libvarnish/vav.c +++ b/lib/libvarnish/vav.c @@ -31,6 +31,8 @@ * const char **VAV_Parse(const char *s, int *argc, int flag) * Parse a command like line into an argv[] * Index zero contains NULL or an error message + * (The error message is a static const char* and does not + * need saving or copying.) * "double quotes" and backslash substitution is handled. * * void VAV_Free(const char **argv) From nils.goroll at uplex.de Tue Jan 5 10:31:05 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 5 Jan 2021 10:31:05 +0000 (UTC) Subject: [master] 68cadaee3 Rename one of the two ban_error() functions Message-ID: <20210105103105.DCB81A2675@lists.varnish-cache.org> commit 68cadaee3db823e3f5b9bd5e20fef5b1ec2bc07d Author: Nils Goroll Date: Tue Jan 5 11:21:55 2021 +0100 Rename one of the two ban_error() functions pointed out by Dridi in #3486 introduced by /me in 6cf1138fd0c397bb59ad4761a31c17087f32a292 diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 3ceb7b775..14a10c9a3 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -830,7 +830,7 @@ VRT_synth_page(VRT_CTX, VCL_STRANDS s) /*--------------------------------------------------------------------*/ static VCL_VOID -ban_error(VRT_CTX, VCL_STRING err) +vrt_ban_error(VRT_CTX, VCL_STRING err) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); AN(ctx->vsl); @@ -851,19 +851,19 @@ VRT_ban_string(VRT_CTX, VCL_STRING str) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if (str == NULL) { - ban_error(ctx, "Null argument"); + vrt_ban_error(ctx, "Null argument"); return; } bp = BAN_Build(); if (bp == NULL) { - ban_error(ctx, "Out of Memory"); + vrt_ban_error(ctx, "Out of Memory"); return; } av = VAV_Parse(str, NULL, ARGV_NOESC); AN(av); if (av[0] != NULL) { - ban_error(ctx, av[0]); + vrt_ban_error(ctx, av[0]); VAV_Free(av); BAN_Abandon(bp); return; @@ -871,22 +871,22 @@ VRT_ban_string(VRT_CTX, VCL_STRING str) for (i = 0; ;) { a1 = av[++i]; if (a1 == NULL) { - ban_error(ctx, "No ban conditions found."); + vrt_ban_error(ctx, "No ban conditions found."); break; } a2 = av[++i]; if (a2 == NULL) { - ban_error(ctx, "Expected comparison operator."); + vrt_ban_error(ctx, "Expected comparison operator."); break; } a3 = av[++i]; if (a3 == NULL) { - ban_error(ctx, "Expected second operand."); + vrt_ban_error(ctx, "Expected second operand."); break; } err = BAN_AddTest(bp, a1, a2, a3); if (err != NULL) { - ban_error(ctx, err); + vrt_ban_error(ctx, err); break; } if (av[++i] == NULL) { @@ -894,7 +894,7 @@ VRT_ban_string(VRT_CTX, VCL_STRING str) if (err == NULL) bp = NULL; else - ban_error(ctx, err); + vrt_ban_error(ctx, err); break; } if (strcmp(av[i], "&&")) { From nils.goroll at uplex.de Tue Jan 5 11:37:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 5 Jan 2021 11:37:06 +0000 (UTC) Subject: [master] 629c3f173 Refactor VRT_priv_task() Message-ID: <20210105113706.5F0FAA5A7D@lists.varnish-cache.org> commit 629c3f173f47375be287cd1fb144776ad673961b Author: Nils Goroll Date: Tue Jan 5 12:26:32 2021 +0100 Refactor VRT_priv_task() Pull out determining the privs head Re-use will come with a PR diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index fa53089f0..5cd8e06ae 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -139,8 +139,8 @@ vrt_priv_dynamic(struct ws *ws, struct vrt_privs *privs, uintptr_t vmod_id) return (vp->priv); } -struct vmod_priv * -VRT_priv_task(VRT_CTX, const void *vmod_id) +static struct vrt_privs * +vrt_priv_task_context(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -156,27 +156,26 @@ VRT_priv_task(VRT_CTX, const void *vmod_id) if (ctx->req) { CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); - return (vrt_priv_dynamic( - ctx->ws, - ctx->req->privs, - (uintptr_t)vmod_id - )); + return (ctx->req->privs); } if (ctx->bo) { CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); - return (vrt_priv_dynamic( - ctx->ws, - ctx->bo->privs, - (uintptr_t)vmod_id - )); + return (ctx->bo->privs); } ASSERT_CLI(); + return (cli_task_privs); +} + +struct vmod_priv * +VRT_priv_task(VRT_CTX, const void *vmod_id) +{ + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); return (vrt_priv_dynamic( ctx->ws, - cli_task_privs, - (uintptr_t)vmod_id - )); + vrt_priv_task_context(ctx), + (uintptr_t)vmod_id)); } struct vmod_priv * From phk at FreeBSD.org Tue Jan 5 11:39:07 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Jan 2021 11:39:07 +0000 (UTC) Subject: [master] 0664f1dac Give TCP Pools a panic function. Message-ID: <20210105113907.A9B23A5D05@lists.varnish-cache.org> commit 0664f1dacd0becce79af89fb00a1297121b70326 Author: Poul-Henning Kamp Date: Tue Jan 5 11:22:44 2021 +0000 Give TCP Pools a panic function. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 2fe784892..732613d6a 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -464,11 +464,7 @@ vbe_panic(const struct director *d, struct vsb *vsb) CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC); - if (bp->ipv4_addr != NULL) - VSB_printf(vsb, "ipv4 = %s,\n", bp->ipv4_addr); - if (bp->ipv6_addr != NULL) - VSB_printf(vsb, "ipv6 = %s,\n", bp->ipv6_addr); - VSB_printf(vsb, "port = %s,\n", bp->port); + VTP_panic(vsb, bp->tcp_pool); VSB_printf(vsb, "hosthdr = %s,\n", bp->hosthdr); VSB_printf(vsb, "n_conn = %u,\n", bp->n_conn); } diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index d4451e5b7..66bdce803 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -816,6 +816,33 @@ VTP_getip(struct pfd *pfd) /*--------------------------------------------------------------------*/ +void +VTP_panic(struct vsb *vsb, struct tcp_pool *tp) +{ + char h[VTCP_ADDRBUFSIZE]; + char p[VTCP_PORTBUFSIZE]; + + if (PAN_dump_struct(vsb, tp, TCP_POOL_MAGIC, "tcp_pool")) + return; + VSB_printf(vsb, "id = 0x%jx,\n", tp->cp->id); + if (tp->uds) + VSB_printf(vsb, "uds = %s,\n", tp->uds); + if (tp->ip4 && VSA_Sane(tp->ip4)) { + VTCP_name(tp->ip4, h, sizeof h, p, sizeof p); + VSB_printf(vsb, "ipv4 = %s, ", h); + VSB_printf(vsb, "port = %s,\n", p); + } + if (tp->ip6 && VSA_Sane(tp->ip6)) { + VTCP_name(tp->ip4, h, sizeof h, p, sizeof p); + VSB_printf(vsb, "ipv6 = %s, ", h); + VSB_printf(vsb, "port = %s,\n", p); + } + VSB_indent(vsb, -2); + VSB_cat(vsb, "},\n"); +} + +/*--------------------------------------------------------------------*/ + void VTP_Init(void) { diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h index 11c7ed211..4d62da5e5 100644 --- a/bin/varnishd/cache/cache_tcp_pool.h +++ b/bin/varnishd/cache/cache_tcp_pool.h @@ -103,5 +103,7 @@ int VTP_Wait(struct worker *, struct pfd *, vtim_real tmo); * function before attempting to receive on the connection. */ +void VTP_panic(struct vsb *, struct tcp_pool *); + const struct suckaddr *VTP_getip(struct pfd *); From phk at FreeBSD.org Tue Jan 5 11:39:07 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Jan 2021 11:39:07 +0000 (UTC) Subject: [master] f44b0c31f Retire preformatted backend addresses, TcpPool provides those in case of panic. Message-ID: <20210105113907.C29C9A5D08@lists.varnish-cache.org> commit f44b0c31feae53e8e00829a36daa37af0d640af5 Author: Poul-Henning Kamp Date: Tue Jan 5 11:37:05 2021 +0000 Retire preformatted backend addresses, TcpPool provides those in case of panic. diff --git a/include/vrt.h b/include/vrt.h index 6f8b8cb52..8f17e2014 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -395,9 +395,6 @@ struct vrt_endpoint { #define VRT_BACKEND_FIELDS(rigid) \ rigid char *vcl_name; \ - rigid char *ipv4_addr; \ - rigid char *ipv6_addr; \ - rigid char *port; \ rigid char *hosthdr; \ vtim_dur connect_timeout; \ vtim_dur first_byte_timeout; \ @@ -408,9 +405,6 @@ struct vrt_endpoint { #define VRT_BACKEND_HANDLE() \ do { \ DA(vcl_name); \ - DA(ipv4_addr); \ - DA(ipv6_addr); \ - DA(port); \ DA(hosthdr); \ DN(connect_timeout); \ DN(first_byte_timeout); \ diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c index a31cb901e..c6028deea 100644 --- a/lib/libvcc/vcc_backend.c +++ b/lib/libvcc/vcc_backend.c @@ -77,15 +77,12 @@ Emit_Sockaddr(struct vcc *tl, struct vsb *vsb1, const struct token *t_host, VSB_printf(vsb1, "\t.ipv4 = (const struct suckaddr *)%s,\n", ipv4); - Fb(tl, 0, "\t.ipv4_addr = \"%s\",\n", ipv4a); } if (ipv6 != NULL) { VSB_printf(vsb1, "\t.ipv6 = (const struct suckaddr *)%s,\n", ipv6); - Fb(tl, 0, "\t.ipv6_addr = \"%s\",\n", ipv6a); } - Fb(tl, 0, "\t.port = \"%s\",\n", pa); VSB_printf(vsb1, "\t.uds_path = (void *) 0,\n"); } diff --git a/lib/libvmod_debug/vmod_debug_dyn.c b/lib/libvmod_debug/vmod_debug_dyn.c index 68739f641..8ab467085 100644 --- a/lib/libvmod_debug/vmod_debug_dyn.c +++ b/lib/libvmod_debug/vmod_debug_dyn.c @@ -75,20 +75,17 @@ dyn_dir_init(VRT_CTX, struct xyzzy_debug_dyn *dyn, INIT_OBJ(&vep, VRT_ENDPOINT_MAGIC); INIT_OBJ(&vrt, VRT_BACKEND_MAGIC); vrt.endpoint = &vep; - vrt.port = port; vrt.vcl_name = dyn->vcl_name; vrt.hosthdr = addr; vrt.probe = probe; sa = VSS_ResolveOne(NULL, addr, port, AF_UNSPEC, SOCK_STREAM, 0); AN(sa); - if (VSA_Get_Proto(sa) == AF_INET) { - vrt.ipv4_addr = addr; + if (VSA_Get_Proto(sa) == AF_INET) vep.ipv4 = sa; - } else if (VSA_Get_Proto(sa) == AF_INET6) { - vrt.ipv6_addr = addr; + else if (VSA_Get_Proto(sa) == AF_INET6) vep.ipv6 = sa; - } else + else WRONG("Wrong proto family"); dir = VRT_new_backend(ctx, &vrt); From phk at FreeBSD.org Tue Jan 5 12:18:06 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 5 Jan 2021 12:18:06 +0000 (UTC) Subject: [master] 91cacbcc7 Let VTP hash the vrt_endpoint, and dont truncate the hash. Message-ID: <20210105121807.001C1A7047@lists.varnish-cache.org> commit 91cacbcc72a6c7eda8bacc222803761d50a0ca16 Author: Poul-Henning Kamp Date: Tue Jan 5 12:13:00 2021 +0000 Let VTP hash the vrt_endpoint, and dont truncate the hash. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 732613d6a..fcd8b0c90 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -38,9 +38,7 @@ #include "cache_varnishd.h" -#include "vend.h" #include "vsa.h" -#include "vsha256.h" #include "vtcp.h" #include "vtim.h" @@ -555,27 +553,6 @@ VRT_backend_vsm_need(VRT_CTX) return (VRT_VSC_Overhead(VSC_vbe_size)); } -static uint64_t -vrt_hash_be(const struct vrt_endpoint *vep) -{ - struct VSHA256Context cx[1]; - unsigned char ident[VSHA256_DIGEST_LENGTH]; - - CHECK_OBJ_NOTNULL(vep, VRT_ENDPOINT_MAGIC); - VSHA256_Init(cx); - VSHA256_Update(cx, vbe_proto_ident, strlen(vbe_proto_ident)); - if (vep->ipv4 != NULL) - VSHA256_Update(cx, vep->ipv4, vsa_suckaddr_len); - if (vep->ipv6 != NULL) - VSHA256_Update(cx, vep->ipv6, vsa_suckaddr_len); - if (vep->uds_path != NULL) - VSHA256_Update(cx, vep->uds_path, strlen(vep->uds_path)); - if (vep->ident != NULL) - VSHA256_Update(cx, vep->ident, vep->ident_len); - VSHA256_Final(ident, cx); - return (vbe64dec(ident)); -} - VCL_BACKEND VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, const struct vrt_backend *vrt) @@ -634,8 +611,7 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, if (! vcl->temp->is_warm) VRT_VSC_Hide(be->vsc_seg); - be->tcp_pool = VTP_Ref(vep->ipv4, vep->ipv6, - vep->uds_path, vrt_hash_be(vrt->endpoint)); + be->tcp_pool = VTP_Ref(vep, vbe_proto_ident); AN(be->tcp_pool); vbp = vrt->probe; diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index 66bdce803..c2d60f794 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -37,7 +37,9 @@ #include "cache_varnishd.h" +#include "vend.h" #include "vsa.h" +#include "vsha256.h" #include "vtcp.h" #include "vus.h" #include "vtim.h" @@ -85,7 +87,7 @@ struct conn_pool { const struct cp_methods *methods; - uintmax_t id; + char ident[VSHA256_DIGEST_LENGTH]; void *priv; VTAILQ_ENTRY(conn_pool) list; @@ -201,14 +203,14 @@ vcp_handle(struct waited *w, enum wait_event ev, vtim_real now) */ static struct conn_pool * -VCP_Ref(uintmax_t id) +VCP_Ref(const uint8_t *ident) { struct conn_pool *cp; Lck_Lock(&conn_pools_mtx); VTAILQ_FOREACH(cp, &conn_pools, list) { assert(cp->refcnt > 0); - if (cp->id != id) + if (memcmp(ident, cp->ident, sizeof cp->ident)) continue; cp->refcnt++; Lck_Unlock(&conn_pools_mtx); @@ -222,8 +224,8 @@ VCP_Ref(uintmax_t id) */ static void * -VCP_New(struct conn_pool *cp, uintmax_t id, void *priv, - const struct cp_methods *cm) +VCP_New(struct conn_pool *cp, uint8_t ident[VSHA256_DIGEST_LENGTH], + void *priv, const struct cp_methods *cm) { AN(cp); @@ -232,7 +234,7 @@ VCP_New(struct conn_pool *cp, uintmax_t id, void *priv, AN(cm->close); INIT_OBJ(cp, CONN_POOL_MAGIC); - cp->id = id; + memcpy(cp->ident, ident, sizeof cp->ident); cp->priv = priv; cp->methods = cm; cp->refcnt = 1; @@ -672,31 +674,41 @@ static const struct cp_methods vus_methods = { */ struct tcp_pool * -VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, const char *uds, - uintmax_t id) +VTP_Ref(const struct vrt_endpoint *vep, const char *ident) { struct tcp_pool *tp; struct conn_pool *cp; const struct cp_methods *methods; - - assert((uds != NULL && ip4 == NULL && ip6 == NULL) || - (uds == NULL && (ip4 != NULL || ip6 != NULL))); - - cp = VCP_Ref(id); - if (cp != NULL) { - tp = cp->priv; - CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); - - if (uds != NULL) { - AN(tp->uds); - AZ(strcmp(tp->uds, uds)); + struct VSHA256Context cx[1]; + unsigned char digest[VSHA256_DIGEST_LENGTH]; + + CHECK_OBJ_NOTNULL(vep, VRT_ENDPOINT_MAGIC); + AN(ident); + VSHA256_Init(cx); + VSHA256_Update(cx, ident, strlen(ident) + 1); // include \0 + if (vep->uds_path != NULL) { + AZ(vep->ipv4); + AZ(vep->ipv6); + VSHA256_Update(cx, "UDS", 4); // include \0 + VSHA256_Update(cx, vep->uds_path, strlen(vep->uds_path)); + } else { + assert(vep->ipv4 != NULL || vep->ipv6 != NULL); + if (vep->ipv4 != NULL) { + assert(VSA_Sane(vep->ipv4)); + VSHA256_Update(cx, "IP4", 4); // include \0 + VSHA256_Update(cx, vep->ipv4, vsa_suckaddr_len); + } + if (vep->ipv6 != NULL) { + assert(VSA_Sane(vep->ipv6)); + VSHA256_Update(cx, "IP6", 4); // include \0 + VSHA256_Update(cx, vep->ipv6, vsa_suckaddr_len); } - if (ip4 != NULL) - AZ(VSA_Compare(tp->ip4, ip4)); - if (ip6 != NULL) - AZ(VSA_Compare(tp->ip6, ip6)); - return (cp->priv); } + VSHA256_Final(digest, cx); + + cp = VCP_Ref(digest); + if (cp != NULL) + return (cp->priv); /* * this is racy - we could end up with additional pools on the same id @@ -704,18 +716,19 @@ VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, const char *uds, */ ALLOC_OBJ(tp, TCP_POOL_MAGIC); AN(tp); - if (uds != NULL) { + if (vep->uds_path != NULL) { methods = &vus_methods; - tp->uds = strdup(uds); + tp->uds = strdup(vep->uds_path); + AN(tp->uds); } else { methods = &vtp_methods; - if (ip4 != NULL) - tp->ip4 = VSA_Clone(ip4); - if (ip6 != NULL) - tp->ip6 = VSA_Clone(ip6); + if (vep->ipv4 != NULL) + tp->ip4 = VSA_Clone(vep->ipv4); + if (vep->ipv6 != NULL) + tp->ip6 = VSA_Clone(vep->ipv6); } - return (VCP_New(tp->cp, id, tp, methods)); + return (VCP_New(tp->cp, digest, tp, methods)); } /*-------------------------------------------------------------------- @@ -824,7 +837,9 @@ VTP_panic(struct vsb *vsb, struct tcp_pool *tp) if (PAN_dump_struct(vsb, tp, TCP_POOL_MAGIC, "tcp_pool")) return; - VSB_printf(vsb, "id = 0x%jx,\n", tp->cp->id); + VSB_printf(vsb, "ident = "); + VSB_quote(vsb, tp->cp->ident, VSHA256_DIGEST_LENGTH, VSB_QUOTE_HEX); + VSB_printf(vsb, ",\n"); if (tp->uds) VSB_printf(vsb, "uds = %s,\n", tp->uds); if (tp->ip4 && VSA_Sane(tp->ip4)) { diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h index 4d62da5e5..0cbaa094d 100644 --- a/bin/varnishd/cache/cache_tcp_pool.h +++ b/bin/varnishd/cache/cache_tcp_pool.h @@ -54,13 +54,12 @@ void PFD_RemoteName(const struct pfd *, char *, unsigned, char *, unsigned); struct VSC_vbe; -struct tcp_pool *VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, - const char *uds, uintmax_t id); +struct tcp_pool *VTP_Ref(const struct vrt_endpoint *, const char *ident); /* - * Get a reference to a TCP pool. Either one or both of ip4 or - * ip6 arg must be non-NULL, or uds must be non-NULL. If recycling - * is to be used, the id pointer distinguishes the pool per - * protocol. + * Get a reference to a TCP pool. Either one or both of ipv4 or + * ipv6 arg must be non-NULL, or uds must be non-NULL. If recycling + * is to be used, the ident pointer distinguishes the pool from + * other pools with same {ipv4, ipv6, uds}. */ void VTP_AddRef(struct tcp_pool *); From nils.goroll at uplex.de Wed Jan 6 09:03:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 6 Jan 2021 09:03:07 +0000 (UTC) Subject: [master] 6eee0074f add std.ban() and std.ban_error() for vcl access to ban errors Message-ID: <20210106090307.74F179B577@lists.varnish-cache.org> commit 6eee0074ffd3e3b87c994bcec8a9ec28993000c9 Author: Nils Goroll Date: Mon Dec 21 16:29:07 2020 +0100 add std.ban() and std.ban_error() for vcl access to ban errors The ban() vcl action adds bans like the ban CLI command, but has no facility for in-band error reporting. Errors are only reported to VSL. We add std.ban() as a replacement for ban() with identical semantics, but returning if the ban was successfully added. std.ban_error() is added to report the error, if any. We add v00009.vtc mirroring v00011.vtc, but with std.ban() replacing ban(). The test number was chosen to fill a gap close to v00011.vtc. Implementation: We change VRT_ban_string() to return any error or NULL for success. Where the error is not a constant string, we need to format it on the workspace. For workspace overflows, we need to fall back to canned constant error strings to ensure that the error case never appears as success. diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 14a10c9a3..072f1e745 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -829,7 +829,7 @@ VRT_synth_page(VRT_CTX, VCL_STRANDS s) /*--------------------------------------------------------------------*/ -static VCL_VOID +static VCL_STRING vrt_ban_error(VRT_CTX, VCL_STRING err) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -837,77 +837,83 @@ vrt_ban_error(VRT_CTX, VCL_STRING err) AN(err); VSLb(ctx->vsl, SLT_VCL_Error, "ban(): %s", err); + return (err); } -VCL_VOID +VCL_STRING VRT_ban_string(VRT_CTX, VCL_STRING str) { char *a1, *a2, *a3; char **av; struct ban_proto *bp; - const char *err; + const char *err = NULL, *berr = NULL; int i; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - if (str == NULL) { - vrt_ban_error(ctx, "Null argument"); - return; - } + if (str == NULL) + return (vrt_ban_error(ctx, "Null argument")); bp = BAN_Build(); - if (bp == NULL) { - vrt_ban_error(ctx, "Out of Memory"); - return; - } + if (bp == NULL) + return (vrt_ban_error(ctx, "Out of Memory")); + av = VAV_Parse(str, NULL, ARGV_NOESC); AN(av); if (av[0] != NULL) { - vrt_ban_error(ctx, av[0]); + err = av[0]; VAV_Free(av); BAN_Abandon(bp); - return; + return (vrt_ban_error(ctx, err)); } for (i = 0; ;) { a1 = av[++i]; if (a1 == NULL) { - vrt_ban_error(ctx, "No ban conditions found."); + err = "No ban conditions found."; break; } a2 = av[++i]; if (a2 == NULL) { - vrt_ban_error(ctx, "Expected comparison operator."); + err = "Expected comparison operator."; break; } a3 = av[++i]; if (a3 == NULL) { - vrt_ban_error(ctx, "Expected second operand."); + err = "Expected second operand."; break; } - err = BAN_AddTest(bp, a1, a2, a3); - if (err != NULL) { - vrt_ban_error(ctx, err); + berr = BAN_AddTest(bp, a1, a2, a3); + if (berr != NULL) break; - } + if (av[++i] == NULL) { - err = BAN_Commit(bp); - if (err == NULL) + berr = BAN_Commit(bp); + if (berr == NULL) bp = NULL; - else - vrt_ban_error(ctx, err); break; } if (strcmp(av[i], "&&")) { - // XXX refactoring pending via PR - VSLb(ctx->vsl, SLT_VCL_Error, - "ban(): Expected && between conditions," - " found \"%s\"", av[i]); + err = WS_Printf(ctx->ws, "Expected && between " + "conditions, found \"%s\"", av[i]); + if (err == NULL) + err = "Expected && between conditions " + "(workspace overflow)"; break; } } + if (berr != NULL) { + AZ(err); + err = WS_Copy(ctx->ws, berr, -1); + if (err == NULL) + err = "Unknown error (workspace overflow)"; + berr = NULL; + } if (bp != NULL) BAN_Abandon(bp); VAV_Free(av); + if (err == NULL) + return (NULL); + return (vrt_ban_error(ctx, err)); } VCL_BYTES diff --git a/bin/varnishtest/tests/v00009.vtc b/bin/varnishtest/tests/v00009.vtc new file mode 100644 index 000000000..c08ff1dec --- /dev/null +++ b/bin/varnishtest/tests/v00009.vtc @@ -0,0 +1,104 @@ +varnishtest "Test std.ban()" + +# see also v00011.vtc + +server s1 { + rxreq + txresp -body "foo" + + rxreq + txresp -body "foo" +} -start + +varnish v1 -vcl+backend { + import std; + + sub vcl_synth { + set resp.http.b1 = std.ban(req.http.doesntexist); + set resp.http.b1e = std.ban_error(); + set resp.http.b2 = std.ban(""); + set resp.http.b2e = std.ban_error(); + set resp.http.b3 = std.ban("req.url"); + set resp.http.b3e = std.ban_error(); + set resp.http.b4 = std.ban("req.url //"); + set resp.http.b4e = std.ban_error(); + set resp.http.b5 = std.ban("req.url // bar"); + set resp.http.b5e = std.ban_error(); + set resp.http.b6 = std.ban("req.url == bar //"); + set resp.http.b6e = std.ban_error(); + set resp.http.b7 = std.ban("foo == bar //"); + set resp.http.b7e = std.ban_error(); + set resp.http.b8 = std.ban("obj.age == 4"); + set resp.http.b8e = std.ban_error(); + set resp.http.b9 = std.ban("obj.age // 4d"); + set resp.http.b9e = std.ban_error(); + set resp.http.b10 = std.ban("obj.http.foo > 4d"); + set resp.http.b10e = std.ban_error(); + set resp.http.b11 = std.ban("req.url ~ ^/$"); + set resp.http.b11e = std.ban_error(); + } + sub vcl_recv { + if (req.method == "BAN") { + return (synth(209,"foo")); + } + } + +} -start + +client c1 { + txreq + rxresp + expect resp.http.X-Varnish == "1001" +} -run + +logexpect l1 -v v1 -d 1 -g vxid { + expect * 1004 VCL_Error {ban[(][)]: Null argument} + expect * 1004 VCL_Error {ban[(][)]: No ban conditions found[.]} + expect * 1004 VCL_Error {ban[(][)]: Expected comparison operator[.]} + expect * 1004 VCL_Error {ban[(][)]: Expected second operand[.]} + expect * 1004 VCL_Error {ban[(][)]: expected conditional [(]==, !=, ~ or !~[)] got "//"} + expect * 1004 VCL_Error {ban[(][)]: Expected && between conditions, found "//"} + expect * 1004 VCL_Error {ban[(][)]: Unknown or unsupported field "foo"} + expect * 1004 VCL_Error {ban[(][)]: expected duration .ms|s|m|h|d|w|y. got "4"} + expect * 1004 VCL_Error {ban[(][)]: expected conditional [(]==, !=, >, >=, < or <=[)] got "//"} + expect * 1004 VCL_Error {ban[(][)]: expected conditional [(]==, !=, ~ or !~[)] got ">"} + + +} -start + +client c1 { + txreq -req "BAN" + rxresp + expect resp.http.X-Varnish == "1004" + expect resp.status == 209 + expect resp.http.b1 == false + expect resp.http.b1e == {Null argument} + expect resp.http.b2 == false + expect resp.http.b2e == {No ban conditions found.} + expect resp.http.b3 == false + expect resp.http.b3e == {Expected comparison operator.} + expect resp.http.b4 == false + expect resp.http.b4e == {Expected second operand.} + expect resp.http.b5 == false + expect resp.http.b5e == {expected conditional (==, !=, ~ or !~) got "//"} + expect resp.http.b6 == false + expect resp.http.b6e == {Expected && between conditions, found "//"} + expect resp.http.b7 == false + expect resp.http.b7e == {Unknown or unsupported field "foo"} + expect resp.http.b8 == false + expect resp.http.b8e == {expected duration [ms|s|m|h|d|w|y] got "4"} + expect resp.http.b9 == false + expect resp.http.b9e == {expected conditional (==, !=, >, >=, < or <=) got "//"} + expect resp.http.b10 == false + expect resp.http.b10e == {expected conditional (==, !=, ~ or !~) got ">"} + expect resp.http.b11 == true + expect resp.http.b11e == {} +} -run + +logexpect l1 -wait + +client c1 { + txreq + rxresp + expect resp.http.X-Varnish == "1006" +} -run diff --git a/bin/varnishtest/tests/v00011.vtc b/bin/varnishtest/tests/v00011.vtc index 8a613be33..6e75b88da 100644 --- a/bin/varnishtest/tests/v00011.vtc +++ b/bin/varnishtest/tests/v00011.vtc @@ -1,5 +1,7 @@ varnishtest "Test vcl ban()" +# see also v00009.vtc + server s1 { rxreq txresp -body "foo" diff --git a/include/vrt.h b/include/vrt.h index 8f17e2014..96bd70df6 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -58,6 +58,7 @@ * struct vmod_priv_methods added * struct vmod_priv free member replaced with methods * VRT_CTX_Assert() added + * VRT_ban_string() signature changed * 12.0 (2020-09-15) * Added VRT_DirectorResolve() * Added VCL_STRING VRT_BLOB_string(VRT_CTX, VCL_BLOB) @@ -494,7 +495,7 @@ VCL_BYTES VRT_CacheReqBody(VRT_CTX, VCL_BYTES maxsize); /* Regexp related */ const char *VRT_regsub(VRT_CTX, int all, const char *, void *, const char *); -VCL_VOID VRT_ban_string(VRT_CTX, VCL_STRING); +VCL_STRING VRT_ban_string(VRT_CTX, VCL_STRING); VCL_INT VRT_purge(VRT_CTX, VCL_DURATION, VCL_DURATION, VCL_DURATION); VCL_VOID VRT_synth(VRT_CTX, VCL_INT, VCL_STRING); VCL_VOID VRT_hit_for_pass(VRT_CTX, VCL_DURATION); diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c index 1ed104af5..eed78708a 100644 --- a/lib/libvcc/vcc_action.c +++ b/lib/libvcc/vcc_action.c @@ -187,7 +187,7 @@ vcc_act_ban(struct vcc *tl, struct token *t, struct symbol *sym) ExpectErr(tl, '('); vcc_NextToken(tl); - Fb(tl, 1, "VRT_ban_string(ctx, \n"); + Fb(tl, 1, "(void) VRT_ban_string(ctx, \n"); tl->indent += INDENT; vcc_Expr(tl, STRING); tl->indent -= INDENT; diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index 57b12c6a6..203a621b1 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -311,3 +311,51 @@ vmod_fnmatch(VRT_CTX, VCL_STRING pattern, VCL_STRING subject, flags |= FNM_PERIOD; return (fnmatch(pattern, subject, flags) != FNM_NOMATCH); } + +static const void * const priv_task_id_ban = &priv_task_id_ban; + +VCL_BOOL v_matchproto_(td_std_ban) +vmod_ban(VRT_CTX, VCL_STRING s) +{ + struct vmod_priv *priv_task; + VCL_STRING r; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + + priv_task = VRT_priv_task(ctx, priv_task_id_ban); + if (priv_task == NULL) { + VRT_fail(ctx, "std.ban(): no priv_task (out of workspace?)"); + return (0); + } + + r = VRT_ban_string(ctx, s); + + /* + * TRUST_ME: the ban error is const. We save it in the un-const priv + * pointer, but promise to only ever return it as a (const) VCL_STRING + */ + priv_task->priv = TRUST_ME(r); + + return (r == NULL); +} + +VCL_STRING v_matchproto_(td_std_ban_error) +vmod_ban_error(VRT_CTX) +{ + struct vmod_priv *priv_task; + VCL_STRING r; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + + priv_task = VRT_priv_task(ctx, priv_task_id_ban); + if (priv_task == NULL) { + VRT_fail(ctx, "std.ban_error():" + " no priv_task (out of workspace?)"); + return ("no priv_task"); + } + + r = priv_task->priv; + if (r == NULL) + r = ""; + return (r); +} diff --git a/lib/libvmod_std/vmod_std.vcc b/lib/libvmod_std/vmod_std.vcc index d0aea5707..e0a17f019 100644 --- a/lib/libvmod_std/vmod_std.vcc +++ b/lib/libvmod_std/vmod_std.vcc @@ -561,6 +561,16 @@ Example:: std.rollback(bereq); +$Function BOOL ban(STRING) + +Same as :ref:`vcl(7)_ban`, but returns true if the ban succeeded and +false otherwise. + +$Function STRING ban_error() + +Returns a textual error description of the last `std.ban()`_ call from +the same task or the empty string if there either was no error or no +`std.ban()`_ call. DEPRECATED functions ==================== @@ -630,8 +640,6 @@ Example:: set req.http.real = std.time2real(now, 1.0); - - SEE ALSO ======== From nils.goroll at uplex.de Wed Jan 6 09:05:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 6 Jan 2021 09:05:08 +0000 (UTC) Subject: [master] e136a5149 Take note of a TODO Message-ID: <20210106090508.1B7FF9B877@lists.varnish-cache.org> commit e136a5149869cc47d8d08973183d81143d014904 Author: Nils Goroll Date: Wed Jan 6 10:03:33 2021 +0100 Take note of a TODO from https://github.com/varnishcache/varnish-cache/pull/3486#pullrequestreview-561662742 as that ticket is closed now diff --git a/bin/varnishd/cache/cache_ban_build.c b/bin/varnishd/cache/cache_ban_build.c index f3891805e..5f479504c 100644 --- a/bin/varnishd/cache/cache_ban_build.c +++ b/bin/varnishd/cache/cache_ban_build.c @@ -109,6 +109,7 @@ BAN_Build(void) return (bp); } +// TODO: change to (struct ban_proto **) void BAN_Abandon(struct ban_proto *bp) { From nils.goroll at uplex.de Wed Jan 6 09:52:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 6 Jan 2021 09:52:07 +0000 (UTC) Subject: [master] bfb95f831 Deprecate ban(), update documentation for std.ban() Message-ID: <20210106095207.5E028A0FF9@lists.varnish-cache.org> commit bfb95f831a99835d65fa239196ed3371ee22ddef Author: Nils Goroll Date: Wed Jan 6 10:44:43 2021 +0100 Deprecate ban(), update documentation for std.ban() Ref https://github.com/varnishcache/varnish-cache/pull/3486#issuecomment-754019262 roadmap: v/ add std.ban*(), add implicit import std vcc aliases ban() to std.ban() (some concerns raised) v/ deprecated ban() without std. remove ban() without std. diff --git a/bin/varnishtest/tests/c00022.vtc b/bin/varnishtest/tests/c00022.vtc index 42383a63a..2ec23c2aa 100644 --- a/bin/varnishtest/tests/c00022.vtc +++ b/bin/varnishtest/tests/c00022.vtc @@ -18,15 +18,27 @@ server s1 { txresp -hdr "foo: bar8" -body "1111111\n" } -start +# code from purging.rst varnish v1 -vcl+backend { + import std; + sub vcl_recv { if (req.method == "BAN") { - ban ("req.url == " + req.url); - return (synth(410)); + if (std.ban("req.http.host == " + req.http.host + + " && req.url == " + req.url)) { + return(synth(204, "Ban added")); + } else { + # return ban error in 400 response + return(synth(400, std.ban_error())); + } } if (req.method == "BANSTR") { - ban ("" + req.http.ban); - return (synth(410)); + if (std.ban(req.http.ban)) { + return(synth(204, "Ban added")); + } else { + # return ban error in 400 response + return(synth(400, std.ban_error())); + } } } } -start @@ -44,7 +56,7 @@ client c1 { client c1 { txreq -req BAN -url /foox rxresp - expect resp.status == 410 + expect resp.status == 204 } -run varnish v1 -cliok "ban.list" @@ -61,7 +73,7 @@ client c1 { client c1 { txreq -req BAN -url /foo rxresp - expect resp.status == 410 + expect resp.status == 204 } -run varnish v1 -cliok "ban.list" @@ -78,7 +90,7 @@ client c1 { client c1 { txreq -req BANSTR -hdr "ban: req.url != /foo" rxresp - expect resp.status == 410 + expect resp.status == 204 } -run varnish v1 -cliok "ban.list" @@ -95,7 +107,7 @@ client c1 { client c1 { txreq -req BANSTR -hdr "Ban: obj.http.foo == bar6" rxresp - expect resp.status == 410 + expect resp.status == 204 } -run varnish v1 -cliok "ban.list" @@ -112,7 +124,7 @@ client c1 { client c1 { txreq -req BANSTR -hdr "Ban: obj.http.foo == bar6" rxresp - expect resp.status == 410 + expect resp.status == 204 } -run varnish v1 -cliok "ban.list" @@ -130,7 +142,7 @@ client c1 { client c1 { txreq -req BANSTR -hdr "Ban: req.http.foo == barcheck" rxresp - expect resp.status == 410 + expect resp.status == 204 } -run varnish v1 -cliok "ban.list" @@ -146,7 +158,7 @@ client c1 { client c1 { txreq -req BANSTR -hdr "Ban: obj.http.foo == barcheck" rxresp - expect resp.status == 410 + expect resp.status == 204 } -run varnish v1 -cliok "ban.list" varnish v1 -clijson "ban.list -j" @@ -158,3 +170,11 @@ client c1 { expect resp.http.foo == bar8 expect resp.bodylen == 8 } -run + +# Error +client c1 { + txreq -req BANSTR -hdr "Ban: xobj.http.foo == barcheck" + rxresp + expect resp.status == 400 + expect resp.reason == {Unknown or unsupported field "xobj.http.foo"} +} -run diff --git a/bin/varnishtest/tests/r00502.vtc b/bin/varnishtest/tests/r00502.vtc index 4b9d63905..929d3ba43 100644 --- a/bin/varnishtest/tests/r00502.vtc +++ b/bin/varnishtest/tests/r00502.vtc @@ -8,8 +8,10 @@ server s1 { } -start varnish v1 -vcl+backend { + import std; + sub vcl_recv { - ban("req.url == / && obj.http.foo ~ bar1"); + std.ban("req.url == / && obj.http.foo ~ bar1"); } } -start diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index b442b88d3..00be68429 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -118,6 +118,8 @@ execution of a VCL state subroutine (``vcl_* {}``), including all user-defined subroutines being called, ``now`` always returns the same value. +.. _vcl(7)_durations: + Durations ~~~~~~~~~ @@ -456,76 +458,10 @@ The following built-in functions are available: ban(STRING) ~~~~~~~~~~~ - Invalidates all objects in cache that match the given expression with the - ban mechanism. - - The format of *STRING* is:: - - [&& ...] - - * **: - - * string fields: - - * ``req.url``: The request url - * ``req.http.*``: Any request header - * ``obj.status``: The cache object status - * ``obj.http.*``: Any cache object header - - ``obj.status`` is treated as a string despite the fact that it - is actually an integer. - - * duration fields: - - * ``obj.ttl``: Remaining ttl at the time the ban is issued - * ``obj.age``: Object age at the time the ban is issued - * ``obj.grace``: The grace time of the object - * ``obj.keep``: The keep time of the object - - - * **: - - * for all fields: - - * ``==``: ** and ** are equal - * ``!=``: ** and ** are unequal - - strings are compared case sensitively - - * for string fields: - - * ``~``: ** matches the regular expression ** - * ``!~``:** does not match the regular expression ** - - * for duration fields: - - * ``>``: ** is greater than ** - * ``>=``: ** is greater than or equal to ** - * ``<``: ** is less than ** - * ``<=``: ** is less than or equal to ** - - - * **: - - * for string fields: - - Either a literal string or a regular expression. Note that - ** does not use any of the string delimiters like ``"`` or - ``{"``\ *...*\ ``"}`` or ``"""``\ *...*\ ``"""`` used elsewhere - in varnish. To match against strings containing whitespace, - regular expressions containing ``\s`` can be used. - - * for duration fields: - - A VCL duration like ``10s``, ``5m`` or ``1h``, see `Durations`_ - - Expressions can be chained using the *and* operator ``&&``. For *or* - semantics, use several bans. + Deprecated. See :ref:`std.ban()`. - The unset ** is not equal to any string, such that, for a - non-existing header, the operators ``==`` and ``~`` always evaluate - as false, while the operators ``!=`` and ``!~`` always evaluate as - true, respectively, for any value of **. + The ``ban()`` function is identical to :ref:`std.ban()`, but does + not provide error reporting. hash_data(input) ~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx/users-guide/purging.rst b/doc/sphinx/users-guide/purging.rst index 7ceee5ffa..696c330cf 100644 --- a/doc/sphinx/users-guide/purging.rst +++ b/doc/sphinx/users-guide/purging.rst @@ -75,7 +75,7 @@ the following command from the shell:: varnishadm ban req.http.host == example.com '&&' req.url '~' '\\.png$' -See :ref:`vcl(7)_ban` for details on the syntax of ban expressions. In +See :ref:`std.ban()` for details on the syntax of ban expressions. In particular, note that in the example given above, the quotes are required for execution from the shell and escaping the backslash in the regular expression is required by the Varnish cli interface. @@ -99,18 +99,21 @@ impact CPU usage and thereby performance. You can also add bans to Varnish via HTTP. Doing so requires a bit of VCL:: + import std; + sub vcl_recv { if (req.method == "BAN") { - # Same ACL check as above: + # Same ACL check as above: if (!client.ip ~ purge) { return(synth(403, "Not allowed.")); } - ban("req.http.host == " + req.http.host + - " && req.url == " + req.url); - - # Throw a synthetic page so the - # request won't go to the backend. - return(synth(200, "Ban added")); + if (std.ban("req.http.host == " + req.http.host + + " && req.url == " + req.url)) { + return(synth(200, "Ban added")); + } else { + # return ban error in 400 response + return(synth(400, std.ban_error())); + } } } @@ -123,21 +126,30 @@ object is not available in the `ban lurker` thread. You can use the following template to write `ban lurker` friendly bans:: + import std; + sub vcl_backend_response { - set beresp.http.url = bereq.url; + set beresp.http.url = bereq.url; } sub vcl_deliver { - unset resp.http.url; # Optional + unset resp.http.url; # Optional } sub vcl_recv { - if (req.method == "PURGE") { - if (client.ip !~ purge) { - return(synth(403, "Not allowed")); - } - ban("obj.http.url ~ " + req.url); # Assumes req.url is a regex. This might be a bit too simple - } + if (req.method == "BAN") { + # Same ACL check as above: + if (!client.ip ~ purge) { + return(synth(403, "Not allowed.")); + } + # Assumes req.url is a regex. This might be a bit too simple + if (std.ban("obj.http.url ~ " + req.url)) { + return(synth(200, "Ban added")); + } else { + # return ban error in 400 response + return(synth(400, std.ban_error())); + } + } } To inspect the current ban list, issue the ``ban.list`` command in the CLI. This diff --git a/lib/libvmod_std/vmod_std.vcc b/lib/libvmod_std/vmod_std.vcc index e0a17f019..e9812ee8e 100644 --- a/lib/libvmod_std/vmod_std.vcc +++ b/lib/libvmod_std/vmod_std.vcc @@ -563,8 +563,77 @@ Example:: $Function BOOL ban(STRING) -Same as :ref:`vcl(7)_ban`, but returns true if the ban succeeded and -false otherwise. +Invalidates all objects in cache that match the given expression with +the ban mechanism. Returns ``true`` if the ban succeeded and ``false`` +otherwise. Error details are available via `std.ban_error()`_. + +The format of *STRING* is:: + + [&& ...] + +* **: + + * string fields: + + * ``req.url``: The request url + * ``req.http.*``: Any request header + * ``obj.status``: The cache object status + * ``obj.http.*``: Any cache object header + + ``obj.status`` is treated as a string despite the fact that it + is actually an integer. + + * duration fields: + + * ``obj.ttl``: Remaining ttl at the time the ban is issued + * ``obj.age``: Object age at the time the ban is issued + * ``obj.grace``: The grace time of the object + * ``obj.keep``: The keep time of the object + + +* **: + + * for all fields: + + * ``==``: ** and ** are equal + * ``!=``: ** and ** are unequal + + strings are compared case sensitively + + * for string fields: + + * ``~``: ** matches the regular expression ** + * ``!~``:** does not match the regular expression ** + + * for duration fields: + + * ``>``: ** is greater than ** + * ``>=``: ** is greater than or equal to ** + * ``<``: ** is less than ** + * ``<=``: ** is less than or equal to ** + + +* **: + + * for string fields: + + Either a literal string or a regular expression. Note that + ** does not use any of the string delimiters like ``"`` or + ``{"``\ *...*\ ``"}`` or ``"""``\ *...*\ ``"""`` used elsewhere + in varnish. To match against strings containing whitespace, + regular expressions containing ``\s`` can be used. + + * for duration fields: + + A VCL duration like ``10s``, ``5m`` or ``1h``, see :ref:`vcl(7)_durations` + +Expressions can be chained using the *and* operator ``&&``. For *or* +semantics, use several bans. + +The unset ** is not equal to any string, such that, for a +non-existing header, the operators ``==`` and ``~`` always evaluate as +false, while the operators ``!=`` and ``!~`` always evaluate as true, +respectively, for any value of **. $Function STRING ban_error() From nils.goroll at uplex.de Wed Jan 6 10:16:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 6 Jan 2021 10:16:07 +0000 (UTC) Subject: [master] 1f5aa959d Appease flexelint Message-ID: <20210106101607.2AAA8A1C85@lists.varnish-cache.org> commit 1f5aa959d7aa287854a953dbf09a6ec3286465f6 Author: Nils Goroll Date: Wed Jan 6 11:11:43 2021 +0100 Appease flexelint cache/cache_vrt.c 916 Warning 438: Last value assigned to variable 'berr' (defined at line 849) not used berr is intentionally NULLed to clarify that no dynamic value must stay alive after the BAN_Abandon() Ref 6eee0074ffd3e3b87c994bcec8a9ec28993000c9 diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 072f1e745..327a651f1 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -908,6 +908,7 @@ VRT_ban_string(VRT_CTX, VCL_STRING str) err = "Unknown error (workspace overflow)"; berr = NULL; } + AZ(berr); if (bp != NULL) BAN_Abandon(bp); VAV_Free(av); From dridi.boukelmoune at gmail.com Thu Jan 7 11:40:10 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 7 Jan 2021 11:40:10 +0000 (UTC) Subject: [build-vmoddir] 1a7aff5f2 build: rm vmod/hex.h Message-ID: <20210107114010.0F3C4A40CC@lists.varnish-cache.org> commit 1a7aff5f23066d8fa005ffd2250948b6bf70d5fa Author: Dridi Boukelmoune Date: Thu Jan 7 11:38:48 2021 +0100 build: rm vmod/hex.h diff --git a/vmod/Makefile.am b/vmod/Makefile.am index 8a4ab5f41..8888a24c0 100644 --- a/vmod/Makefile.am +++ b/vmod/Makefile.am @@ -22,7 +22,6 @@ libvmod_blob_la_SOURCES = \ vmod_blob.h \ id.c \ base64.c \ - hex.h \ hex.c \ url.c \ tbl_encodings.h \ diff --git a/vmod/base64.c b/vmod/base64.c index 909dc11e3..06d6b6cef 100644 --- a/vmod/base64.c +++ b/vmod/base64.c @@ -36,9 +36,6 @@ #include "vmod_blob.h" -#define ILL ((int8_t) 127) -#define PAD ((int8_t) 126) - static const struct b64_alphabet { const char b64[64]; const int8_t i64[256]; diff --git a/vmod/hex.c b/vmod/hex.c index b95e45e90..539ab87a7 100644 --- a/vmod/hex.c +++ b/vmod/hex.c @@ -38,8 +38,6 @@ #include "vmod_blob.h" -#include "hex.h" - const char hex_alphabet[][16] = { "0123456789abcdef", "0123456789ABCDEF" @@ -50,7 +48,7 @@ const char hex_alphabet[][16] = { * hex digits with their binary values. This fits all of the hex digits * into 55 bytes (cacheline friendly). */ -const uint8_t nibble[] = { +const uint8_t hex_nibble[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ILL, ILL, ILL, ILL, ILL, ILL, ILL, 10, 11, 12, 13, 14, 15, ILL, ILL, ILL, ILL, ILL, ILL, ILL, @@ -74,7 +72,7 @@ hex_decode_l(size_t l) static inline char hex2byte(const unsigned char hi, const unsigned char lo) { - return ((nibble[hi - '0'] << 4) | nibble[lo - '0']); + return ((hex_nibble[hi - '0'] << 4) | hex_nibble[lo - '0']); } ssize_t diff --git a/vmod/hex.h b/vmod/hex.h deleted file mode 100644 index 203a95aaa..000000000 --- a/vmod/hex.h +++ /dev/null @@ -1,39 +0,0 @@ -/*- - * Copyright 2016 UPLEX - Nils Goroll Systemoptimierung - * All rights reserved. - * - * Authors: Nils Goroll - * Geoffrey Simmons - * - * SPDX-License-Identifier: BSD-2-Clause - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include - -#define ILL ((int8_t) 127) - -/* These are defined in hex.c */ - -extern const char hex_alphabet[][16]; - -extern const uint8_t nibble[]; diff --git a/vmod/url.c b/vmod/url.c index 79fdc39b5..9ce272ed8 100644 --- a/vmod/url.c +++ b/vmod/url.c @@ -36,8 +36,6 @@ #include "vmod_blob.h" -#include "hex.h" - /* Decoder states */ enum state_e { NORMAL, @@ -155,7 +153,7 @@ url_decode(const enum encoding dec, blob_dest_t buf, break; case PERCENT: if (isoutofrange(*s) || - (nib = nibble[*s - '0']) == ILL) { + (nib = hex_nibble[*s - '0']) == ILL) { errno = EINVAL; return (-1); } @@ -167,7 +165,7 @@ url_decode(const enum encoding dec, blob_dest_t buf, return (-1); } if (isoutofrange(*s) || - (nib2 = nibble[*s - '0']) == ILL) { + (nib2 = hex_nibble[*s - '0']) == ILL) { errno = EINVAL; return (-1); } diff --git a/vmod/vmod_blob.h b/vmod/vmod_blob.h index fd728a93f..4574bdf9f 100644 --- a/vmod/vmod_blob.h +++ b/vmod/vmod_blob.h @@ -91,6 +91,11 @@ ssize_t encode_f(const enum encoding enc, const enum case_e kase, blob_dest_t buf, blob_len_t buflen, blob_src_t in, blob_len_t inlen); +/* Special lookup characters */ + +#define ILL ((int8_t) 127) +#define PAD ((int8_t) 126) + /* * General interface for a decoder: decode the concatenation of strings * (obtained from STRANDS) into buf, and return the length of decoded @@ -135,6 +140,9 @@ len_f hex_decode_l; encode_f hex_encode; decode_f hex_decode; +extern const char hex_alphabet[][16]; +extern const uint8_t hex_nibble[]; + /* url.c */ len_f url_encode_l; len_f url_decode_l; From dridi.boukelmoune at gmail.com Thu Jan 7 11:40:10 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 7 Jan 2021 11:40:10 +0000 (UTC) Subject: [build-vmoddir] 2d1341bbb build: Merge vdir.c and misc.c into vmod_directors.c Message-ID: <20210107114010.36FB4A40CF@lists.varnish-cache.org> commit 2d1341bbb51b54dcea389717a3b1445005db08b3 Author: Dridi Boukelmoune Date: Thu Jan 7 11:50:42 2021 +0100 build: Merge vdir.c and misc.c into vmod_directors.c diff --git a/vmod/Makefile.am b/vmod/Makefile.am index 8888a24c0..32331da09 100644 --- a/vmod/Makefile.am +++ b/vmod/Makefile.am @@ -36,11 +36,10 @@ libvmod_debug_la_SOURCES = \ vmod_debug_dyn.c libvmod_directors_la_SOURCES = \ - vdir.c \ - vdir.h \ + vmod_directors.c \ + vmod_directors.h \ fall_back.c \ hash.c \ - misc.c \ random.c \ round_robin.c \ vmod_shard.c \ diff --git a/vmod/fall_back.c b/vmod/fall_back.c index 425c1093a..ac79dae47 100644 --- a/vmod/fall_back.c +++ b/vmod/fall_back.c @@ -37,7 +37,7 @@ #include "vcc_directors_if.h" -#include "vdir.h" +#include "vmod_directors.h" #include "vsb.h" #include "vbm.h" diff --git a/vmod/hash.c b/vmod/hash.c index eeddffca2..2ab92fb7c 100644 --- a/vmod/hash.c +++ b/vmod/hash.c @@ -35,7 +35,7 @@ #include "cache/cache.h" -#include "vdir.h" +#include "vmod_directors.h" #include "vcc_directors_if.h" diff --git a/vmod/misc.c b/vmod/misc.c deleted file mode 100644 index f43875d61..000000000 --- a/vmod/misc.c +++ /dev/null @@ -1,49 +0,0 @@ -/*- - * Copyright 2019 UPLEX - Nils Goroll Systemoptimierung - * All rights reserved. - * - * Author: Nils Goroll - * - * SPDX-License-Identifier: BSD-2-Clause - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include "config.h" - -#include "vdef.h" -#include "vrt.h" -#include "vcl.h" - -#include "vcc_directors_if.h" - -VCL_BACKEND -VPFX(lookup)(VRT_CTX, VCL_STRING name) -{ - if ((ctx->method & VCL_MET_TASK_H) == 0) { - VRT_fail(ctx, - "lookup() may only be called from vcl_init / vcl_fini"); - return (NULL); - } - - return (VRT_LookupDirector(ctx, name)); -} diff --git a/vmod/random.c b/vmod/random.c index b18e1f6fd..8fe711931 100644 --- a/vmod/random.c +++ b/vmod/random.c @@ -37,7 +37,7 @@ #include "vbm.h" #include "vrnd.h" -#include "vdir.h" +#include "vmod_directors.h" #include "vcc_directors_if.h" diff --git a/vmod/round_robin.c b/vmod/round_robin.c index 230bcb02e..b365a7116 100644 --- a/vmod/round_robin.c +++ b/vmod/round_robin.c @@ -37,7 +37,7 @@ #include "vcc_directors_if.h" -#include "vdir.h" +#include "vmod_directors.h" struct vmod_directors_round_robin { unsigned magic; diff --git a/vmod/vdir.c b/vmod/vmod_directors.c similarity index 95% rename from vmod/vdir.c rename to vmod/vmod_directors.c index a4a24212c..c9ed4fc57 100644 --- a/vmod/vdir.c +++ b/vmod/vmod_directors.c @@ -1,8 +1,10 @@ /*- * Copyright (c) 2013-2015 Varnish Software AS + * Copyright 2019 UPLEX - Nils Goroll Systemoptimierung * All rights reserved. * * Author: Poul-Henning Kamp + * Author: Nils Goroll * * SPDX-License-Identifier: BSD-2-Clause * @@ -36,9 +38,24 @@ #include "cache/cache.h" #include "vbm.h" +#include "vcl.h" #include "vsb.h" -#include "vdir.h" +#include "vcc_directors_if.h" + +#include "vmod_directors.h" + +VCL_BACKEND +VPFX(lookup)(VRT_CTX, VCL_STRING name) +{ + if ((ctx->method & VCL_MET_TASK_H) == 0) { + VRT_fail(ctx, + "lookup() may only be called from vcl_init / vcl_fini"); + return (NULL); + } + + return (VRT_LookupDirector(ctx, name)); +} static void vdir_expand(struct vdir *vd, unsigned n) diff --git a/vmod/vdir.h b/vmod/vmod_directors.h similarity index 100% rename from vmod/vdir.h rename to vmod/vmod_directors.h From dridi.boukelmoune at gmail.com Thu Jan 7 11:40:10 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 7 Jan 2021 11:40:10 +0000 (UTC) Subject: [build-vmoddir] d2f1fbc67 build: Prefix all VMOD sources with vmod_ Message-ID: <20210107114010.6006DA40D3@lists.varnish-cache.org> commit d2f1fbc67212af0db3a93d1a4fc0a065b639142c Author: Dridi Boukelmoune Date: Thu Jan 7 12:14:58 2021 +0100 build: Prefix all VMOD sources with vmod_ diff --git a/vmod/Makefile.am b/vmod/Makefile.am index 32331da09..6557b366e 100644 --- a/vmod/Makefile.am +++ b/vmod/Makefile.am @@ -20,12 +20,12 @@ AM_CPPFLAGS = \ libvmod_blob_la_SOURCES = \ vmod_blob.c \ vmod_blob.h \ - id.c \ - base64.c \ - hex.c \ - url.c \ - tbl_encodings.h \ - tbl_case.h + vmod_blob_id.c \ + vmod_blob_base64.c \ + vmod_blob_hex.c \ + vmod_blob_url.c \ + vmod_blob_tbl_encodings.h \ + vmod_blob_tbl_case.h libvmod_cookie_la_SOURCES = \ vmod_cookie.c @@ -38,15 +38,15 @@ libvmod_debug_la_SOURCES = \ libvmod_directors_la_SOURCES = \ vmod_directors.c \ vmod_directors.h \ - fall_back.c \ - hash.c \ - random.c \ - round_robin.c \ - vmod_shard.c \ - shard_cfg.c \ - shard_cfg.h \ - shard_dir.c \ - shard_dir.h + vmod_directors_fall_back.c \ + vmod_directors_hash.c \ + vmod_directors_random.c \ + vmod_directors_round_robin.c \ + vmod_directors_shard.c \ + vmod_directors_shard_cfg.c \ + vmod_directors_vmod_directors_shard_cfg.h \ + vmod_directors_shard_dir.c \ + vmod_directors_shard_dir.h libvmod_proxy_la_SOURCES = \ vmod_proxy.c @@ -62,7 +62,7 @@ libvmod_std_la_SOURCES = \ libvmod_unix_la_SOURCES = \ vmod_unix.c \ - cred_compat.h + vmod_unix_cred_compat.h libvmod_vtc_la_SOURCES = \ vmod_vtc.c diff --git a/vmod/flint.lnt b/vmod/flint.lnt index ac5724d6c..34dfed208 100644 --- a/vmod/flint.lnt +++ b/vmod/flint.lnt @@ -2,7 +2,7 @@ * vmod_{blob,directors} */ --efile(451, "tbl_*.h") // No include guard +-efile(451, "vmod_*_tbl_*.h") // No include guard /* * vmod_blob diff --git a/vmod/vmod_blob.c b/vmod/vmod_blob.c index c9c3dac2e..e1f474c26 100644 --- a/vmod/vmod_blob.c +++ b/vmod/vmod_blob.c @@ -121,7 +121,7 @@ static enum encoding parse_encoding(VCL_ENUM e) { #define VMODENUM(n) if (e == VENUM(n)) return (n); -#include "tbl_encodings.h" +#include "vmod_blob_tbl_encodings.h" WRONG("illegal encoding enum"); } @@ -129,7 +129,7 @@ static enum case_e parse_case(VCL_ENUM e) { #define VMODENUM(n) if (e == VENUM(n)) return (n); -#include "tbl_case.h" +#include "vmod_blob_tbl_case.h" WRONG("illegal case enum"); } diff --git a/vmod/vmod_blob.h b/vmod/vmod_blob.h index 4574bdf9f..90872b61a 100644 --- a/vmod/vmod_blob.h +++ b/vmod/vmod_blob.h @@ -36,7 +36,7 @@ enum encoding { __INVALID_ENCODING = 0, #define VMODENUM(x) x, -#include "tbl_encodings.h" +#include "vmod_blob_tbl_encodings.h" __MAX_ENCODING }; @@ -48,7 +48,7 @@ enum encoding { */ enum case_e { #define VMODENUM(x) x, -#include "tbl_case.h" +#include "vmod_blob_tbl_case.h" }; typedef const size_t blob_len_t; diff --git a/vmod/base64.c b/vmod/vmod_blob_base64.c similarity index 100% rename from vmod/base64.c rename to vmod/vmod_blob_base64.c diff --git a/vmod/hex.c b/vmod/vmod_blob_hex.c similarity index 100% rename from vmod/hex.c rename to vmod/vmod_blob_hex.c diff --git a/vmod/id.c b/vmod/vmod_blob_id.c similarity index 100% rename from vmod/id.c rename to vmod/vmod_blob_id.c diff --git a/vmod/tbl_case.h b/vmod/vmod_blob_tbl_case.h similarity index 100% rename from vmod/tbl_case.h rename to vmod/vmod_blob_tbl_case.h diff --git a/vmod/tbl_encodings.h b/vmod/vmod_blob_tbl_encodings.h similarity index 100% rename from vmod/tbl_encodings.h rename to vmod/vmod_blob_tbl_encodings.h diff --git a/vmod/url.c b/vmod/vmod_blob_url.c similarity index 100% rename from vmod/url.c rename to vmod/vmod_blob_url.c diff --git a/vmod/fall_back.c b/vmod/vmod_directors_fall_back.c similarity index 100% rename from vmod/fall_back.c rename to vmod/vmod_directors_fall_back.c diff --git a/vmod/hash.c b/vmod/vmod_directors_hash.c similarity index 100% rename from vmod/hash.c rename to vmod/vmod_directors_hash.c diff --git a/vmod/random.c b/vmod/vmod_directors_random.c similarity index 100% rename from vmod/random.c rename to vmod/vmod_directors_random.c diff --git a/vmod/round_robin.c b/vmod/vmod_directors_round_robin.c similarity index 100% rename from vmod/round_robin.c rename to vmod/vmod_directors_round_robin.c diff --git a/vmod/vmod_shard.c b/vmod/vmod_directors_shard.c similarity index 99% rename from vmod/vmod_shard.c rename to vmod/vmod_directors_shard.c index a88095c26..d12df4476 100644 --- a/vmod/vmod_shard.c +++ b/vmod/vmod_directors_shard.c @@ -40,8 +40,8 @@ #include "vend.h" #include "vcc_directors_if.h" -#include "shard_dir.h" -#include "shard_cfg.h" +#include "vmod_directors_shard_dir.h" +#include "vmod_directors_shard_cfg.h" #include "vsb.h" /* ------------------------------------------------------------------------- diff --git a/vmod/shard_cfg.c b/vmod/vmod_directors_shard_cfg.c similarity index 99% rename from vmod/shard_cfg.c rename to vmod/vmod_directors_shard_cfg.c index 1a0cb40e4..680a8cf04 100644 --- a/vmod/shard_cfg.c +++ b/vmod/vmod_directors_shard_cfg.c @@ -38,8 +38,8 @@ #include "cache/cache.h" -#include "shard_dir.h" -#include "shard_cfg.h" +#include "vmod_directors_shard_dir.h" +#include "vmod_directors_shard_cfg.h" /*lint -esym(749, shard_change_task_e::*) */ enum shard_change_task_e { diff --git a/vmod/shard_cfg.h b/vmod/vmod_directors_shard_cfg.h similarity index 100% rename from vmod/shard_cfg.h rename to vmod/vmod_directors_shard_cfg.h diff --git a/vmod/shard_dir.c b/vmod/vmod_directors_shard_dir.c similarity index 99% rename from vmod/shard_dir.c rename to vmod/vmod_directors_shard_dir.c index 00b61aa98..9b82722d1 100644 --- a/vmod/shard_dir.c +++ b/vmod/vmod_directors_shard_dir.c @@ -45,7 +45,7 @@ #include "vrnd.h" #include "vcc_directors_if.h" -#include "shard_dir.h" +#include "vmod_directors_shard_dir.h" struct shard_be_info { unsigned hostid; diff --git a/vmod/shard_dir.h b/vmod/vmod_directors_shard_dir.h similarity index 100% rename from vmod/shard_dir.h rename to vmod/vmod_directors_shard_dir.h diff --git a/vmod/vmod_unix.c b/vmod/vmod_unix.c index 0dfbd5ae2..1edfca9d5 100644 --- a/vmod/vmod_unix.c +++ b/vmod/vmod_unix.c @@ -37,7 +37,7 @@ #include "vcl.h" #include "common/heritage.h" -#include "cred_compat.h" +#include "vmod_unix_cred_compat.h" #include "vcc_unix_if.h" #define FAIL(ctx, msg) \ diff --git a/vmod/cred_compat.h b/vmod/vmod_unix_cred_compat.h similarity index 100% rename from vmod/cred_compat.h rename to vmod/vmod_unix_cred_compat.h From dridi.boukelmoune at gmail.com Thu Jan 7 11:40:10 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 7 Jan 2021 11:40:10 +0000 (UTC) Subject: [build-vmoddir] 7b39cf1a1 build: Let vmodtool generate the sources listing Message-ID: <20210107114010.7CC0AA40DE@lists.varnish-cache.org> commit 7b39cf1a1df45f369d3ffd0be3fb1a88cb1f60bd Author: Dridi Boukelmoune Date: Thu Jan 7 12:21:32 2021 +0100 build: Let vmodtool generate the sources listing Now that all VMOD sources are prefixed consistently we no longer need to maintain the lists manually. diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 8f2ec9ffb..7247b0bde 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -53,6 +53,9 @@ AMBOILERPLATE = '''\ vmod_LTLIBRARIES += libvmod_XXX.la +libvmod_XXX_la_SOURCES = \\ +\tSRC + libvmod_XXX_la_CFLAGS = \\ \t at SAN_CFLAGS@ @@ -971,11 +974,14 @@ class vcc(object): def amboilerplate(self): ''' Produce boilplate for autocrap tools ''' vcc = os.path.basename(self.inputfile) + src = glob.glob("vmod_" + self.modname + "*.[ch]") + src.sort() fn = "automake_boilerplate_" + self.modname + ".am" fo = self.openfile(fn) fo.write(AMBOILERPLATE.replace("XXX", self.modname) .replace("VCC", vcc) - .replace("PFX", self.pfx)) + .replace("PFX", self.pfx) + .replace("SRC", " \\\n\t".join(src))) fo.close() def mkdefs(self, fo): diff --git a/vmod/Makefile.am b/vmod/Makefile.am index 6557b366e..d204d9bc5 100644 --- a/vmod/Makefile.am +++ b/vmod/Makefile.am @@ -13,60 +13,6 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/bin/varnishd \ -I$(top_builddir)/include -# -# Declare vmod sources -# - -libvmod_blob_la_SOURCES = \ - vmod_blob.c \ - vmod_blob.h \ - vmod_blob_id.c \ - vmod_blob_base64.c \ - vmod_blob_hex.c \ - vmod_blob_url.c \ - vmod_blob_tbl_encodings.h \ - vmod_blob_tbl_case.h - -libvmod_cookie_la_SOURCES = \ - vmod_cookie.c - -libvmod_debug_la_SOURCES = \ - vmod_debug.c \ - vmod_debug_obj.c \ - vmod_debug_dyn.c - -libvmod_directors_la_SOURCES = \ - vmod_directors.c \ - vmod_directors.h \ - vmod_directors_fall_back.c \ - vmod_directors_hash.c \ - vmod_directors_random.c \ - vmod_directors_round_robin.c \ - vmod_directors_shard.c \ - vmod_directors_shard_cfg.c \ - vmod_directors_vmod_directors_shard_cfg.h \ - vmod_directors_shard_dir.c \ - vmod_directors_shard_dir.h - -libvmod_proxy_la_SOURCES = \ - vmod_proxy.c - -libvmod_purge_la_SOURCES = \ - vmod_purge.c - -libvmod_std_la_SOURCES = \ - vmod_std.c \ - vmod_std_conversions.c \ - vmod_std_fileread.c \ - vmod_std_querysort.c - -libvmod_unix_la_SOURCES = \ - vmod_unix.c \ - vmod_unix_cred_compat.h - -libvmod_vtc_la_SOURCES = \ - vmod_vtc.c - # # Use vmodtool.py generated automake boilerplate # diff --git a/vmod/automake_boilerplate_blob.am b/vmod/automake_boilerplate_blob.am index 81ec37e56..d604614ac 100644 --- a/vmod/automake_boilerplate_blob.am +++ b/vmod/automake_boilerplate_blob.am @@ -2,6 +2,16 @@ vmod_LTLIBRARIES += libvmod_blob.la +libvmod_blob_la_SOURCES = \ + vmod_blob.c \ + vmod_blob.h \ + vmod_blob_base64.c \ + vmod_blob_hex.c \ + vmod_blob_id.c \ + vmod_blob_tbl_case.h \ + vmod_blob_tbl_encodings.h \ + vmod_blob_url.c + libvmod_blob_la_CFLAGS = \ @SAN_CFLAGS@ diff --git a/vmod/automake_boilerplate_cookie.am b/vmod/automake_boilerplate_cookie.am index fb9b156b5..a1509d920 100644 --- a/vmod/automake_boilerplate_cookie.am +++ b/vmod/automake_boilerplate_cookie.am @@ -2,6 +2,9 @@ vmod_LTLIBRARIES += libvmod_cookie.la +libvmod_cookie_la_SOURCES = \ + vmod_cookie.c + libvmod_cookie_la_CFLAGS = \ @SAN_CFLAGS@ diff --git a/vmod/automake_boilerplate_debug.am b/vmod/automake_boilerplate_debug.am index 950cf4ede..88f89a403 100644 --- a/vmod/automake_boilerplate_debug.am +++ b/vmod/automake_boilerplate_debug.am @@ -2,6 +2,11 @@ vmod_LTLIBRARIES += libvmod_debug.la +libvmod_debug_la_SOURCES = \ + vmod_debug.c \ + vmod_debug_dyn.c \ + vmod_debug_obj.c + libvmod_debug_la_CFLAGS = \ @SAN_CFLAGS@ diff --git a/vmod/automake_boilerplate_directors.am b/vmod/automake_boilerplate_directors.am index 5ba173349..2a3618a43 100644 --- a/vmod/automake_boilerplate_directors.am +++ b/vmod/automake_boilerplate_directors.am @@ -2,6 +2,19 @@ vmod_LTLIBRARIES += libvmod_directors.la +libvmod_directors_la_SOURCES = \ + vmod_directors.c \ + vmod_directors.h \ + vmod_directors_fall_back.c \ + vmod_directors_hash.c \ + vmod_directors_random.c \ + vmod_directors_round_robin.c \ + vmod_directors_shard.c \ + vmod_directors_shard_cfg.c \ + vmod_directors_shard_cfg.h \ + vmod_directors_shard_dir.c \ + vmod_directors_shard_dir.h + libvmod_directors_la_CFLAGS = \ @SAN_CFLAGS@ diff --git a/vmod/automake_boilerplate_proxy.am b/vmod/automake_boilerplate_proxy.am index 83f2217c0..d5873e989 100644 --- a/vmod/automake_boilerplate_proxy.am +++ b/vmod/automake_boilerplate_proxy.am @@ -2,6 +2,9 @@ vmod_LTLIBRARIES += libvmod_proxy.la +libvmod_proxy_la_SOURCES = \ + vmod_proxy.c + libvmod_proxy_la_CFLAGS = \ @SAN_CFLAGS@ diff --git a/vmod/automake_boilerplate_purge.am b/vmod/automake_boilerplate_purge.am index afe7bf60b..2146f4055 100644 --- a/vmod/automake_boilerplate_purge.am +++ b/vmod/automake_boilerplate_purge.am @@ -2,6 +2,9 @@ vmod_LTLIBRARIES += libvmod_purge.la +libvmod_purge_la_SOURCES = \ + vmod_purge.c + libvmod_purge_la_CFLAGS = \ @SAN_CFLAGS@ diff --git a/vmod/automake_boilerplate_std.am b/vmod/automake_boilerplate_std.am index a5760bf78..b035ba7b7 100644 --- a/vmod/automake_boilerplate_std.am +++ b/vmod/automake_boilerplate_std.am @@ -2,6 +2,12 @@ vmod_LTLIBRARIES += libvmod_std.la +libvmod_std_la_SOURCES = \ + vmod_std.c \ + vmod_std_conversions.c \ + vmod_std_fileread.c \ + vmod_std_querysort.c + libvmod_std_la_CFLAGS = \ @SAN_CFLAGS@ diff --git a/vmod/automake_boilerplate_unix.am b/vmod/automake_boilerplate_unix.am index b2937a270..1b0274e54 100644 --- a/vmod/automake_boilerplate_unix.am +++ b/vmod/automake_boilerplate_unix.am @@ -2,6 +2,10 @@ vmod_LTLIBRARIES += libvmod_unix.la +libvmod_unix_la_SOURCES = \ + vmod_unix.c \ + vmod_unix_cred_compat.h + libvmod_unix_la_CFLAGS = \ @SAN_CFLAGS@ diff --git a/vmod/automake_boilerplate_vtc.am b/vmod/automake_boilerplate_vtc.am index e8479ba10..efe418b00 100644 --- a/vmod/automake_boilerplate_vtc.am +++ b/vmod/automake_boilerplate_vtc.am @@ -2,6 +2,9 @@ vmod_LTLIBRARIES += libvmod_vtc.la +libvmod_vtc_la_SOURCES = \ + vmod_vtc.c + libvmod_vtc_la_CFLAGS = \ @SAN_CFLAGS@ From phk at FreeBSD.org Thu Jan 7 13:00:08 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 7 Jan 2021 13:00:08 +0000 (UTC) Subject: [master] cbfaf2810 Fix a typo and remove unused endpoint->ident members. Message-ID: <20210107130008.B563DA6A4C@lists.varnish-cache.org> commit cbfaf2810f8645eafb4807b88ec70666a3f26737 Author: Poul-Henning Kamp Date: Thu Jan 7 08:21:19 2021 +0000 Fix a typo and remove unused endpoint->ident members. diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index c2d60f794..6b1f4228a 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -848,7 +848,7 @@ VTP_panic(struct vsb *vsb, struct tcp_pool *tp) VSB_printf(vsb, "port = %s,\n", p); } if (tp->ip6 && VSA_Sane(tp->ip6)) { - VTCP_name(tp->ip4, h, sizeof h, p, sizeof p); + VTCP_name(tp->ip6, h, sizeof h, p, sizeof p); VSB_printf(vsb, "ipv6 = %s, ", h); VSB_printf(vsb, "port = %s,\n", p); } diff --git a/include/vrt.h b/include/vrt.h index 96bd70df6..fe888060e 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -390,8 +390,6 @@ struct vrt_endpoint { VCL_IP ipv4; VCL_IP ipv6; const char *uds_path; - const void *ident; - int ident_len; }; #define VRT_BACKEND_FIELDS(rigid) \ From phk at FreeBSD.org Thu Jan 7 13:00:08 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 7 Jan 2021 13:00:08 +0000 (UTC) Subject: [master] 2f6eaf09e Propagate the type of the connected to address all the way down Message-ID: <20210107130008.D34CAA6A4F@lists.varnish-cache.org> commit 2f6eaf09ee6fa55f842bd13fb4b23bdeef9f7122 Author: Poul-Henning Kamp Date: Thu Jan 7 11:49:03 2021 +0000 Propagate the type of the connected to address all the way down diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index fcd8b0c90..1a91c2753 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -341,7 +341,7 @@ vbe_dir_getip(VRT_CTX, VCL_BACKEND d) CHECK_OBJ_NOTNULL(ctx->bo->htc, HTTP_CONN_MAGIC); pfd = ctx->bo->htc->priv; - return (VTP_getip(pfd)); + return (VTP_GetIp(pfd)); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 4ff2ff7ce..8fb53d756 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -285,7 +285,7 @@ vbp_poke(struct vbp_target *vt) t_start = t_now = VTIM_real(); t_end = t_start + vt->timeout; - s = VTP_Open(vt->tcp_pool, t_end - t_now, (const void **)&sa, &err); + s = VTP_Open(vt->tcp_pool, t_end - t_now, &sa, &err); if (s < 0) { bprintf(vt->resp_buf, "Open error %d (%s)", err, vstrerror(err)); Lck_Lock(&vbp_mtx); diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index 6b1f4228a..31ddd9118 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -58,7 +58,7 @@ struct pfd { #define PFD_MAGIC 0x0c5e6593 int fd; VTAILQ_ENTRY(pfd) list; - const void *priv; + VCL_IP addr; uint8_t state; struct waited waited[1]; struct conn_pool *conn_pool; @@ -69,8 +69,7 @@ struct pfd { /*-------------------------------------------------------------------- */ -typedef int cp_open_f(const struct conn_pool *, vtim_dur tmo, - const void **privp); +typedef int cp_open_f(const struct conn_pool *, vtim_dur tmo, VCL_IP *ap); typedef void cp_close_f(struct pfd *); typedef void cp_name_f(const struct pfd *, char *, unsigned, char *, unsigned); @@ -374,7 +373,7 @@ VCP_Recycle(const struct worker *wrk, struct pfd **pfdp) */ static int -VCP_Open(struct conn_pool *cp, vtim_dur tmo, const void **privp, int *err) +VCP_Open(struct conn_pool *cp, vtim_dur tmo, VCL_IP *ap, int *err) { int r; vtim_mono h; @@ -401,7 +400,7 @@ VCP_Open(struct conn_pool *cp, vtim_dur tmo, const void **privp, int *err) return (-1); } - r = cp->methods->open(cp, tmo, privp); + r = cp->methods->open(cp, tmo, ap); *err = errno; @@ -518,7 +517,7 @@ VCP_Get(struct conn_pool *cp, vtim_dur tmo, struct worker *wrk, INIT_OBJ(pfd->waited, WAITED_MAGIC); pfd->state = PFD_STATE_USED; pfd->conn_pool = cp; - pfd->fd = VCP_Open(cp, tmo, &pfd->priv, err); + pfd->fd = VCP_Open(cp, tmo, &pfd->addr, err); if (pfd->fd < 0) { FREE_OBJ(pfd); Lck_Lock(&cp->mtx); @@ -572,7 +571,7 @@ tmo2msec(vtim_dur tmo) } static int v_matchproto_(cp_open_f) -vtp_open(const struct conn_pool *cp, vtim_dur tmo, const void **privp) +vtp_open(const struct conn_pool *cp, vtim_dur tmo, VCL_IP *ap) { int s; int msec; @@ -583,17 +582,17 @@ vtp_open(const struct conn_pool *cp, vtim_dur tmo, const void **privp) msec = tmo2msec(tmo); if (cache_param->prefer_ipv6) { - *privp = tp->ip6; + *ap = tp->ip6; s = VTCP_connect(tp->ip6, msec); if (s >= 0) return (s); } - *privp = tp->ip4; + *ap = tp->ip4; s = VTCP_connect(tp->ip4, msec); if (s >= 0) return (s); if (!cache_param->prefer_ipv6) { - *privp = tp->ip6; + *ap = tp->ip6; s = VTCP_connect(tp->ip6, msec); } return (s); @@ -634,7 +633,7 @@ static const struct cp_methods vtp_methods = { */ static int v_matchproto_(cp_open_f) -vus_open(const struct conn_pool *cp, vtim_dur tmo, const void **privp) +vus_open(const struct conn_pool *cp, vtim_dur tmo, VCL_IP *ap) { int s; int msec; @@ -645,7 +644,7 @@ vus_open(const struct conn_pool *cp, vtim_dur tmo, const void **privp) AN(tp->uds); msec = tmo2msec(tmo); - *privp = bogo_ip; + *ap = bogo_ip; s = VUS_connect(tp->uds, msec); return (s); } @@ -766,9 +765,9 @@ VTP_Rel(struct tcp_pool **tpp) */ int -VTP_Open(struct tcp_pool *tp, vtim_dur tmo, const void **privp, int *err) +VTP_Open(struct tcp_pool *tp, vtim_dur tmo, VCL_IP *ap, int *err) { - return (VCP_Open(tp->cp, tmo, privp, err)); + return (VCP_Open(tp->cp, tmo, ap, err)); } /*-------------------------------------------------------------------- @@ -817,14 +816,12 @@ VTP_Wait(struct worker *wrk, struct pfd *pfd, vtim_real tmo) /*-------------------------------------------------------------------- */ -const struct suckaddr * -VTP_getip(struct pfd *pfd) +VCL_IP +VTP_GetIp(struct pfd *pfd) { - struct tcp_pool *tp; CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC); - CAST_OBJ_NOTNULL(tp, pfd->conn_pool->priv, TCP_POOL_MAGIC); - return (pfd->priv); + return (pfd->addr); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h index 0cbaa094d..01fc952cb 100644 --- a/bin/varnishd/cache/cache_tcp_pool.h +++ b/bin/varnishd/cache/cache_tcp_pool.h @@ -73,7 +73,7 @@ void VTP_Rel(struct tcp_pool **); * the pool is destroyed and all cached connections closed. */ -int VTP_Open(struct tcp_pool *, vtim_dur tmo, const void **, int*); +int VTP_Open(struct tcp_pool *, vtim_dur tmo, VCL_IP *, int*); /* * Open a new connection and return the address used. * errno will be returned in the last argument. @@ -104,5 +104,5 @@ int VTP_Wait(struct worker *, struct pfd *, vtim_real tmo); void VTP_panic(struct vsb *, struct tcp_pool *); -const struct suckaddr *VTP_getip(struct pfd *); +VCL_IP VTP_GetIp(struct pfd *); From phk at FreeBSD.org Thu Jan 7 13:00:08 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 7 Jan 2021 13:00:08 +0000 (UTC) Subject: [master] 84d08ecae Start reversing the layering of conn/tcp pool: free becomes a method. Message-ID: <20210107130009.05647A6A53@lists.varnish-cache.org> commit 84d08ecae29d0a8702ec09bb85eb5869af66a807 Author: Poul-Henning Kamp Date: Thu Jan 7 11:58:36 2021 +0000 Start reversing the layering of conn/tcp pool: free becomes a method. diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index 31ddd9118..660c6f4d0 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -72,12 +72,14 @@ struct pfd { typedef int cp_open_f(const struct conn_pool *, vtim_dur tmo, VCL_IP *ap); typedef void cp_close_f(struct pfd *); typedef void cp_name_f(const struct pfd *, char *, unsigned, char *, unsigned); +typedef void cp_free_f(void *); struct cp_methods { cp_open_f *open; cp_close_f *close; cp_name_f *local_name; cp_name_f *remote_name; + cp_free_f *free; }; struct conn_pool { @@ -105,16 +107,6 @@ struct conn_pool { int holddown_errno; }; -struct tcp_pool { - unsigned magic; -#define TCP_POOL_MAGIC 0x28b0e42a - - struct suckaddr *ip4; - struct suckaddr *ip6; - char *uds; - struct conn_pool cp[1]; -}; - static struct lock conn_pools_mtx; static VTAILQ_HEAD(, conn_pool) conn_pools = VTAILQ_HEAD_INITIALIZER(conn_pools); @@ -268,7 +260,7 @@ VCP_AddRef(struct conn_pool *cp) * Release Conn pool, destroy if last reference. */ -static int +static void VCP_Rel(struct conn_pool *cp) { struct pfd *pfd, *pfd2; @@ -279,7 +271,7 @@ VCP_Rel(struct conn_pool *cp) assert(cp->refcnt > 0); if (--cp->refcnt > 0) { Lck_Unlock(&conn_pools_mtx); - return (1); + return; } AZ(cp->n_used); VTAILQ_REMOVE(&conn_pools, cp, list); @@ -304,7 +296,7 @@ VCP_Rel(struct conn_pool *cp) Lck_Delete(&cp->mtx); AZ(cp->n_conn); AZ(cp->n_kill); - return (0); + cp->methods->free(cp->priv); } /*-------------------------------------------------------------------- @@ -564,6 +556,28 @@ VCP_Wait(struct worker *wrk, struct pfd *pfd, vtim_real tmo) /*-------------------------------------------------------------------- */ +struct tcp_pool { + unsigned magic; +#define TCP_POOL_MAGIC 0x28b0e42a + + struct suckaddr *ip4; + struct suckaddr *ip6; + char *uds; + struct conn_pool cp[1]; +}; + +static void v_matchproto_(cp_free_f) +vtp_free(void *priv) +{ + struct tcp_pool *tp; + + TAKE_OBJ_NOTNULL(tp, &priv, TCP_POOL_MAGIC); + free(tp->ip4); + free(tp->ip6); + free(tp->uds); + FREE_OBJ(tp); +} + static inline int tmo2msec(vtim_dur tmo) { @@ -627,6 +641,7 @@ static const struct cp_methods vtp_methods = { .close = vtp_close, .local_name = vtp_local_name, .remote_name = vtp_remote_name, + .free = vtp_free, }; /*-------------------------------------------------------------------- @@ -665,6 +680,7 @@ static const struct cp_methods vus_methods = { .close = vtp_close, .local_name = vus_name, .remote_name = vus_name, + .free = vtp_free, }; /*-------------------------------------------------------------------- @@ -751,13 +767,7 @@ VTP_Rel(struct tcp_pool **tpp) struct tcp_pool *tp; TAKE_OBJ_NOTNULL(tp, tpp, TCP_POOL_MAGIC); - if (VCP_Rel(tp->cp)) - return; - - free(tp->ip4); - free(tp->ip6); - free(tp->uds); - FREE_OBJ(tp); + VCP_Rel(tp->cp); } /*-------------------------------------------------------------------- From phk at FreeBSD.org Thu Jan 7 13:00:09 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 7 Jan 2021 13:00:09 +0000 (UTC) Subject: [master] 58064f342 Split the panic function into conn/tcp pool layers. Message-ID: <20210107130009.265CCA6A58@lists.varnish-cache.org> commit 58064f342ab7d1984600c232b55d57a8a2c23d35 Author: Poul-Henning Kamp Date: Thu Jan 7 12:09:25 2021 +0000 Split the panic function into conn/tcp pool layers. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 1a91c2753..3f6d4afa6 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -462,7 +462,7 @@ vbe_panic(const struct director *d, struct vsb *vsb) CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC); - VTP_panic(vsb, bp->tcp_pool); + VTP_Panic(vsb, bp->tcp_pool); VSB_printf(vsb, "hosthdr = %s,\n", bp->hosthdr); VSB_printf(vsb, "n_conn = %u,\n", bp->n_conn); } diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index 660c6f4d0..a649e7e25 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -73,6 +73,7 @@ typedef int cp_open_f(const struct conn_pool *, vtim_dur tmo, VCL_IP *ap); typedef void cp_close_f(struct pfd *); typedef void cp_name_f(const struct pfd *, char *, unsigned, char *, unsigned); typedef void cp_free_f(void *); +typedef void cp_panic_f(struct vsb *, const void *); struct cp_methods { cp_open_f *open; @@ -80,6 +81,7 @@ struct cp_methods { cp_name_f *local_name; cp_name_f *remote_name; cp_free_f *free; + cp_panic_f *panic; }; struct conn_pool { @@ -612,6 +614,54 @@ vtp_open(const struct conn_pool *cp, vtim_dur tmo, VCL_IP *ap) return (s); } +static void +vtp_panic(struct vsb *vsb, const void *priv) +{ + const struct tcp_pool *tp; + + char h[VTCP_ADDRBUFSIZE]; + char p[VTCP_PORTBUFSIZE]; + + tp = priv; + if (PAN_dump_struct(vsb, tp, TCP_POOL_MAGIC, "tcp_pool")) + return; + if (tp->uds) + VSB_printf(vsb, "uds = %s,\n", tp->uds); + if (tp->ip4 && VSA_Sane(tp->ip4)) { + VTCP_name(tp->ip4, h, sizeof h, p, sizeof p); + VSB_printf(vsb, "ipv4 = %s, ", h); + VSB_printf(vsb, "port = %s,\n", p); + } + if (tp->ip6 && VSA_Sane(tp->ip6)) { + VTCP_name(tp->ip6, h, sizeof h, p, sizeof p); + VSB_printf(vsb, "ipv6 = %s, ", h); + VSB_printf(vsb, "port = %s,\n", p); + } + VSB_indent(vsb, -2); + VSB_cat(vsb, "},\n"); +} + +/*--------------------------------------------------------------------*/ + +void +VTP_Panic(struct vsb *vsb, struct tcp_pool *tp) +{ + struct conn_pool *cp; + + cp = tp->cp; + + if (PAN_dump_struct(vsb, cp, CONN_POOL_MAGIC, "conn_pool")) + return; + VSB_printf(vsb, "ident = "); + VSB_quote(vsb, cp->ident, VSHA256_DIGEST_LENGTH, VSB_QUOTE_HEX); + VSB_printf(vsb, ",\n"); + cp->methods->panic(vsb, cp->priv); + VSB_indent(vsb, -2); + VSB_cat(vsb, "},\n"); +} + +/*--------------------------------------------------------------------*/ + static void v_matchproto_(cp_close_f) vtp_close(struct pfd *pfd) { @@ -642,6 +692,7 @@ static const struct cp_methods vtp_methods = { .local_name = vtp_local_name, .remote_name = vtp_remote_name, .free = vtp_free, + .panic = vtp_panic, }; /*-------------------------------------------------------------------- @@ -681,6 +732,7 @@ static const struct cp_methods vus_methods = { .local_name = vus_name, .remote_name = vus_name, .free = vtp_free, + .panic = vtp_panic, }; /*-------------------------------------------------------------------- @@ -834,34 +886,6 @@ VTP_GetIp(struct pfd *pfd) return (pfd->addr); } -/*--------------------------------------------------------------------*/ - -void -VTP_panic(struct vsb *vsb, struct tcp_pool *tp) -{ - char h[VTCP_ADDRBUFSIZE]; - char p[VTCP_PORTBUFSIZE]; - - if (PAN_dump_struct(vsb, tp, TCP_POOL_MAGIC, "tcp_pool")) - return; - VSB_printf(vsb, "ident = "); - VSB_quote(vsb, tp->cp->ident, VSHA256_DIGEST_LENGTH, VSB_QUOTE_HEX); - VSB_printf(vsb, ",\n"); - if (tp->uds) - VSB_printf(vsb, "uds = %s,\n", tp->uds); - if (tp->ip4 && VSA_Sane(tp->ip4)) { - VTCP_name(tp->ip4, h, sizeof h, p, sizeof p); - VSB_printf(vsb, "ipv4 = %s, ", h); - VSB_printf(vsb, "port = %s,\n", p); - } - if (tp->ip6 && VSA_Sane(tp->ip6)) { - VTCP_name(tp->ip6, h, sizeof h, p, sizeof p); - VSB_printf(vsb, "ipv6 = %s, ", h); - VSB_printf(vsb, "port = %s,\n", p); - } - VSB_indent(vsb, -2); - VSB_cat(vsb, "},\n"); -} /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h index 01fc952cb..42dddb941 100644 --- a/bin/varnishd/cache/cache_tcp_pool.h +++ b/bin/varnishd/cache/cache_tcp_pool.h @@ -102,7 +102,7 @@ int VTP_Wait(struct worker *, struct pfd *, vtim_real tmo); * function before attempting to receive on the connection. */ -void VTP_panic(struct vsb *, struct tcp_pool *); +void VTP_Panic(struct vsb *, struct tcp_pool *); VCL_IP VTP_GetIp(struct pfd *); From phk at FreeBSD.org Thu Jan 7 13:00:09 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 7 Jan 2021 13:00:09 +0000 (UTC) Subject: [master] 32aaa009c Reverse the TCP(|UDS) and Connection pool layers. Message-ID: <20210107130009.42C0DA6A5D@lists.varnish-cache.org> commit 32aaa009c63c07ba89744c47daf5e755560cfbdc Author: Poul-Henning Kamp Date: Thu Jan 7 12:40:00 2021 +0000 Reverse the TCP(|UDS) and Connection pool layers. Now the TCP|UDS simply returns a conn_pool, which then becomes the primary handle. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 3f6d4afa6..4dc9324b8 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -155,7 +155,7 @@ vbe_dir_getfd(VRT_CTX, struct worker *wrk, struct backend *bp, bo->htc->doclose = SC_NULL; FIND_TMO(connect_timeout, tmod, bo, bp); - pfd = VTP_Get(bp->tcp_pool, tmod, wrk, force_fresh, &err); + pfd = VCP_Get(bp->conn_pool, tmod, wrk, force_fresh, &err); if (pfd == NULL) { VBE_Connect_Error(bp->vsc, err); VSLb(bo->vsl, SLT_FetchError, @@ -188,7 +188,7 @@ vbe_dir_getfd(VRT_CTX, struct worker *wrk, struct backend *bp, // account as if connect failed - good idea? VSC_C_main->backend_fail++; bo->htc = NULL; - VTP_Close(&pfd); + VCP_Close(&pfd); AZ(pfd); Lck_Lock(&bp->mtx); bp->n_conn--; @@ -237,7 +237,7 @@ vbe_dir_finish(VRT_CTX, VCL_BACKEND d) if (bo->htc->doclose != SC_NULL || bp->proxy_header != 0) { VSLb(bo->vsl, SLT_BackendClose, "%d %s close", *PFD_Fd(pfd), VRT_BACKEND_string(bp->director)); - VTP_Close(&pfd); + VCP_Close(&pfd); AZ(pfd); Lck_Lock(&bp->mtx); } else { @@ -246,7 +246,7 @@ vbe_dir_finish(VRT_CTX, VCL_BACKEND d) VRT_BACKEND_string(bp->director)); Lck_Lock(&bp->mtx); VSC_C_main->backend_recycle++; - VTP_Recycle(bo->wrk, &pfd); + VCP_Recycle(bo->wrk, &pfd); } assert(bp->n_conn > 0); bp->n_conn--; @@ -295,7 +295,7 @@ vbe_dir_gethdrs(VRT_CTX, VCL_BACKEND d) &bo->acct.bereq_bodybytes); if (PFD_State(pfd) != PFD_STATE_USED) { - if (VTP_Wait(wrk, pfd, VTIM_real() + + if (VCP_Wait(wrk, pfd, VTIM_real() + bo->htc->first_byte_timeout) != 0) { bo->htc->doclose = SC_RX_TIMEOUT; VSLb(bo->vsl, SLT_FetchError, @@ -341,7 +341,7 @@ vbe_dir_getip(VRT_CTX, VCL_BACKEND d) CHECK_OBJ_NOTNULL(ctx->bo->htc, HTTP_CONN_MAGIC); pfd = ctx->bo->htc->priv; - return (VTP_GetIp(pfd)); + return (VCP_GetIp(pfd)); } /*--------------------------------------------------------------------*/ @@ -430,7 +430,7 @@ vbe_free(struct backend *be) else VTAILQ_REMOVE(&backends, be, list); VSC_C_main->n_backend--; - VTP_Rel(&be->tcp_pool); + VCP_Rel(&be->conn_pool); Lck_Unlock(&backends_mtx); #define DA(x) do { if (be->x != NULL) free(be->x); } while (0) @@ -462,7 +462,7 @@ vbe_panic(const struct director *d, struct vsb *vsb) CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC); - VTP_Panic(vsb, bp->tcp_pool); + VCP_Panic(vsb, bp->conn_pool); VSB_printf(vsb, "hosthdr = %s,\n", bp->hosthdr); VSB_printf(vsb, "n_conn = %u,\n", bp->n_conn); } @@ -611,15 +611,15 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, if (! vcl->temp->is_warm) VRT_VSC_Hide(be->vsc_seg); - be->tcp_pool = VTP_Ref(vep, vbe_proto_ident); - AN(be->tcp_pool); + be->conn_pool = VTP_Ref(vep, vbe_proto_ident); + AN(be->conn_pool); vbp = vrt->probe; if (vbp == NULL) vbp = VCL_DefaultProbe(vcl); if (vbp != NULL) { - VBP_Insert(be, vbp, be->tcp_pool); + VBP_Insert(be, vbp, be->conn_pool); m = vbe_methods; } else { be->sick = 0; diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h index 9262c5290..393b5df09 100644 --- a/bin/varnishd/cache/cache_backend.h +++ b/bin/varnishd/cache/cache_backend.h @@ -42,7 +42,7 @@ struct vbp_target; struct vrt_ctx; struct vrt_backend_probe; -struct tcp_pool; +struct conn_pool; /*-------------------------------------------------------------------- * An instance of a backend from a VCL program. @@ -67,7 +67,7 @@ struct backend { struct vsc_seg *vsc_seg; struct VSC_vbe *vsc; - struct tcp_pool *tcp_pool; + struct conn_pool *conn_pool; VCL_BACKEND director; @@ -84,7 +84,7 @@ void VBE_SetHappy(const struct backend *, uint64_t); /* cache_backend_probe.c */ void VBP_Update_Backend(struct vbp_target *vt); void VBP_Insert(struct backend *b, struct vrt_backend_probe const *p, - struct tcp_pool *); + struct conn_pool *); void VBP_Remove(struct backend *b); void VBP_Control(const struct backend *b, int stop); void VBP_Status(struct vsb *, const struct backend *, int details, int json); diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 8fb53d756..ee9e0dbf2 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -64,7 +64,7 @@ struct vbp_target { VRT_BACKEND_PROBE_FIELDS() struct backend *backend; - struct tcp_pool *tcp_pool; + struct conn_pool *conn_pool; char *req; int req_len; @@ -103,7 +103,7 @@ vbp_delete(struct vbp_target *vt) #define DN(x) /**/ VRT_BACKEND_PROBE_HANDLE(); #undef DN - VTP_Rel(&vt->tcp_pool); + VCP_Rel(&vt->conn_pool); free(vt->req); FREE_OBJ(vt); } @@ -285,7 +285,7 @@ vbp_poke(struct vbp_target *vt) t_start = t_now = VTIM_real(); t_end = t_start + vt->timeout; - s = VTP_Open(vt->tcp_pool, t_end - t_now, &sa, &err); + s = VCP_Open(vt->conn_pool, t_end - t_now, &sa, &err); if (s < 0) { bprintf(vt->resp_buf, "Open error %d (%s)", err, vstrerror(err)); Lck_Lock(&vbp_mtx); @@ -673,7 +673,7 @@ VBP_Control(const struct backend *be, int enable) void VBP_Insert(struct backend *b, const struct vrt_backend_probe *vp, - struct tcp_pool *tp) + struct conn_pool *tp) { struct vbp_target *vt; @@ -685,8 +685,8 @@ VBP_Insert(struct backend *b, const struct vrt_backend_probe *vp, ALLOC_OBJ(vt, VBP_TARGET_MAGIC); XXXAN(vt); - vt->tcp_pool = tp; - VTP_AddRef(vt->tcp_pool); + vt->conn_pool = tp; + VCP_AddRef(vt->conn_pool); vt->backend = b; b->probe = vt; diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index a62cb83f6..e9d2da664 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -377,7 +377,7 @@ child_main(int sigmagic, size_t altstksz) HTTP_Init(); VBO_Init(); - VTP_Init(); + VCP_Init(); VBP_Init(); VDI_Init(); VBE_InitCfg(); diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index a649e7e25..31423253f 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -27,7 +27,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * TCP connection pools. + * (TCP|UDS) connection pools. * */ @@ -216,7 +216,7 @@ VCP_Ref(const uint8_t *ident) /*-------------------------------------------------------------------- */ -static void * +static struct conn_pool * VCP_New(struct conn_pool *cp, uint8_t ident[VSHA256_DIGEST_LENGTH], void *priv, const struct cp_methods *cm) { @@ -240,14 +240,13 @@ VCP_New(struct conn_pool *cp, uint8_t ident[VSHA256_DIGEST_LENGTH], VTAILQ_INSERT_HEAD(&conn_pools, cp, list); Lck_Unlock(&conn_pools_mtx); - return (priv); + return (cp); } - /*-------------------------------------------------------------------- */ -static void +void VCP_AddRef(struct conn_pool *cp) { CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC); @@ -262,12 +261,13 @@ VCP_AddRef(struct conn_pool *cp) * Release Conn pool, destroy if last reference. */ -static void -VCP_Rel(struct conn_pool *cp) +void +VCP_Rel(struct conn_pool **cpp) { + struct conn_pool *cp; struct pfd *pfd, *pfd2; - CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC); + TAKE_OBJ_NOTNULL(cp, cpp, CONN_POOL_MAGIC); Lck_Lock(&conn_pools_mtx); assert(cp->refcnt > 0); @@ -305,7 +305,7 @@ VCP_Rel(struct conn_pool *cp) * Recycle a connection. */ -static void +void VCP_Recycle(const struct worker *wrk, struct pfd **pfdp) { struct pfd *pfd; @@ -366,7 +366,7 @@ VCP_Recycle(const struct worker *wrk, struct pfd **pfdp) * Open a new connection from pool. */ -static int +int VCP_Open(struct conn_pool *cp, vtim_dur tmo, VCL_IP *ap, int *err) { int r; @@ -440,7 +440,7 @@ VCP_Open(struct conn_pool *cp, vtim_dur tmo, VCL_IP *ap, int *err) * Close a connection. */ -static void +void VCP_Close(struct pfd **pfdp) { struct pfd *pfd; @@ -474,7 +474,7 @@ VCP_Close(struct pfd **pfdp) * Get a connection, possibly recycled */ -static struct pfd * +struct pfd * VCP_Get(struct conn_pool *cp, vtim_dur tmo, struct worker *wrk, unsigned force_fresh, int *err) { @@ -526,7 +526,7 @@ VCP_Get(struct conn_pool *cp, vtim_dur tmo, struct worker *wrk, /*-------------------------------------------------------------------- */ -static int +int VCP_Wait(struct worker *wrk, struct pfd *pfd, vtim_real tmo) { struct conn_pool *cp; @@ -558,6 +558,40 @@ VCP_Wait(struct worker *wrk, struct pfd *pfd, vtim_real tmo) /*-------------------------------------------------------------------- */ +VCL_IP +VCP_GetIp(struct pfd *pfd) +{ + + CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC); + return (pfd->addr); +} + +/*--------------------------------------------------------------------*/ + +void +VCP_Panic(struct vsb *vsb, struct conn_pool *cp) +{ + + if (PAN_dump_struct(vsb, cp, CONN_POOL_MAGIC, "conn_pool")) + return; + VSB_printf(vsb, "ident = "); + VSB_quote(vsb, cp->ident, VSHA256_DIGEST_LENGTH, VSB_QUOTE_HEX); + VSB_printf(vsb, ",\n"); + cp->methods->panic(vsb, cp->priv); + VSB_indent(vsb, -2); + VSB_cat(vsb, "},\n"); +} + +/*--------------------------------------------------------------------*/ + +void +VCP_Init(void) +{ + Lck_New(&conn_pools_mtx, lck_tcp_pool); +} + +/**********************************************************************/ + struct tcp_pool { unsigned magic; #define TCP_POOL_MAGIC 0x28b0e42a @@ -643,25 +677,6 @@ vtp_panic(struct vsb *vsb, const void *priv) /*--------------------------------------------------------------------*/ -void -VTP_Panic(struct vsb *vsb, struct tcp_pool *tp) -{ - struct conn_pool *cp; - - cp = tp->cp; - - if (PAN_dump_struct(vsb, cp, CONN_POOL_MAGIC, "conn_pool")) - return; - VSB_printf(vsb, "ident = "); - VSB_quote(vsb, cp->ident, VSHA256_DIGEST_LENGTH, VSB_QUOTE_HEX); - VSB_printf(vsb, ",\n"); - cp->methods->panic(vsb, cp->priv); - VSB_indent(vsb, -2); - VSB_cat(vsb, "},\n"); -} - -/*--------------------------------------------------------------------*/ - static void v_matchproto_(cp_close_f) vtp_close(struct pfd *pfd) { @@ -740,7 +755,7 @@ static const struct cp_methods vus_methods = { * it doesn't exist already. */ -struct tcp_pool * +struct conn_pool * VTP_Ref(const struct vrt_endpoint *vep, const char *ident) { struct tcp_pool *tp; @@ -775,7 +790,7 @@ VTP_Ref(const struct vrt_endpoint *vep, const char *ident) cp = VCP_Ref(digest); if (cp != NULL) - return (cp->priv); + return (cp); /* * this is racy - we could end up with additional pools on the same id @@ -797,100 +812,3 @@ VTP_Ref(const struct vrt_endpoint *vep, const char *ident) } return (VCP_New(tp->cp, digest, tp, methods)); } - -/*-------------------------------------------------------------------- - * Add a reference to a tcp_pool - */ - -void -VTP_AddRef(struct tcp_pool *tp) -{ - CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); - VCP_AddRef(tp->cp); -} - -/*-------------------------------------------------------------------- - * Release TCP pool, destroy if last reference. - */ - -void -VTP_Rel(struct tcp_pool **tpp) -{ - struct tcp_pool *tp; - - TAKE_OBJ_NOTNULL(tp, tpp, TCP_POOL_MAGIC); - VCP_Rel(tp->cp); -} - -/*-------------------------------------------------------------------- - * Open a new connection from pool. - */ - -int -VTP_Open(struct tcp_pool *tp, vtim_dur tmo, VCL_IP *ap, int *err) -{ - return (VCP_Open(tp->cp, tmo, ap, err)); -} - -/*-------------------------------------------------------------------- - * Recycle a connection. - */ - -void -VTP_Recycle(const struct worker *wrk, struct pfd **pfdp) -{ - - VCP_Recycle(wrk, pfdp); -} - -/*-------------------------------------------------------------------- - * Close a connection. - */ - -void -VTP_Close(struct pfd **pfdp) -{ - - VCP_Close(pfdp); -} - -/*-------------------------------------------------------------------- - * Get a connection - */ - -struct pfd * -VTP_Get(struct tcp_pool *tp, vtim_dur tmo, struct worker *wrk, - unsigned force_fresh, int *err) -{ - - return (VCP_Get(tp->cp, tmo, wrk, force_fresh, err)); -} - -/*-------------------------------------------------------------------- - */ - -int -VTP_Wait(struct worker *wrk, struct pfd *pfd, vtim_real tmo) -{ - return (VCP_Wait(wrk, pfd, tmo)); -} - -/*-------------------------------------------------------------------- - */ - -VCL_IP -VTP_GetIp(struct pfd *pfd) -{ - - CHECK_OBJ_NOTNULL(pfd, PFD_MAGIC); - return (pfd->addr); -} - - -/*--------------------------------------------------------------------*/ - -void -VTP_Init(void) -{ - Lck_New(&conn_pools_mtx, lck_tcp_pool); -} diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h index 42dddb941..c2fccb909 100644 --- a/bin/varnishd/cache/cache_tcp_pool.h +++ b/bin/varnishd/cache/cache_tcp_pool.h @@ -28,11 +28,11 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * Outgoing TCP connection pools + * Outgoing TCP|UDS connection pools * */ -struct tcp_pool; +struct conn_pool; struct pfd; #define PFD_STATE_AVAIL (1<<0) #define PFD_STATE_USED (1<<1) @@ -48,13 +48,12 @@ void PFD_LocalName(const struct pfd *, char *, unsigned, char *, unsigned); void PFD_RemoteName(const struct pfd *, char *, unsigned, char *, unsigned); /*--------------------------------------------------------------------- - * Prototypes */ struct VSC_vbe; -struct tcp_pool *VTP_Ref(const struct vrt_endpoint *, const char *ident); +struct conn_pool *VTP_Ref(const struct vrt_endpoint *, const char *ident); /* * Get a reference to a TCP pool. Either one or both of ipv4 or * ipv6 arg must be non-NULL, or uds must be non-NULL. If recycling @@ -62,47 +61,47 @@ struct tcp_pool *VTP_Ref(const struct vrt_endpoint *, const char *ident); * other pools with same {ipv4, ipv6, uds}. */ -void VTP_AddRef(struct tcp_pool *); +void VCP_AddRef(struct conn_pool *); /* * Get another reference to an already referenced TCP pool. */ -void VTP_Rel(struct tcp_pool **); +void VCP_Rel(struct conn_pool **); /* * Release reference to a TCP pool. When last reference is released * the pool is destroyed and all cached connections closed. */ -int VTP_Open(struct tcp_pool *, vtim_dur tmo, VCL_IP *, int*); +int VCP_Open(struct conn_pool *, vtim_dur tmo, VCL_IP *, int*); /* * Open a new connection and return the address used. * errno will be returned in the last argument. */ -void VTP_Close(struct pfd **); +void VCP_Close(struct pfd **); /* * Close a connection. */ -void VTP_Recycle(const struct worker *, struct pfd **); +void VCP_Recycle(const struct worker *, struct pfd **); /* * Recycle an open connection. */ -struct pfd *VTP_Get(struct tcp_pool *, vtim_dur tmo, struct worker *, +struct pfd *VCP_Get(struct conn_pool *, vtim_dur tmo, struct worker *, unsigned force_fresh, int *err); /* * Get a (possibly) recycled connection. * errno will be stored in err */ -int VTP_Wait(struct worker *, struct pfd *, vtim_real tmo); +int VCP_Wait(struct worker *, struct pfd *, vtim_real tmo); /* * If the connection was recycled (state != VTP_STATE_USED) call this * function before attempting to receive on the connection. */ -void VTP_Panic(struct vsb *, struct tcp_pool *); +void VCP_Panic(struct vsb *, struct conn_pool *); -VCL_IP VTP_GetIp(struct pfd *); +VCL_IP VCP_GetIp(struct pfd *); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index d74f6ed5b..e53d28a23 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -145,8 +145,8 @@ void VCA_Shutdown(void); void VBE_InitCfg(void); void VBE_Poll(void); -/* cache_backend_tcp.c */ -void VTP_Init(void); +/* cache_tcp_pool.c */ +void VCP_Init(void); /* cache_backend_poll.c */ void VBP_Init(void); From phk at FreeBSD.org Thu Jan 7 14:10:07 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 7 Jan 2021 14:10:07 +0000 (UTC) Subject: [master] 5129b3c81 We already own the priv pointer, so no need to take it. Message-ID: <20210107141007.A43E6A8E4B@lists.varnish-cache.org> commit 5129b3c81dc88e814ed84f65c9fd312b32b7ed3f Author: Poul-Henning Kamp Date: Thu Jan 7 14:09:19 2021 +0000 We already own the priv pointer, so no need to take it. (Spotted, in a round-about way by: Gcc on SunOS) diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index 31423253f..02db6c6f3 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -607,7 +607,7 @@ vtp_free(void *priv) { struct tcp_pool *tp; - TAKE_OBJ_NOTNULL(tp, &priv, TCP_POOL_MAGIC); + CAST_OBJ_NOTNULL(tp, priv, TCP_POOL_MAGIC); free(tp->ip4); free(tp->ip6); free(tp->uds); From dridi.boukelmoune at gmail.com Thu Jan 7 14:16:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 7 Jan 2021 14:16:07 +0000 (UTC) Subject: [master] 4db4565f4 build: Add VMOD names to automake boilerplates Message-ID: <20210107141607.0CA0AA9244@lists.varnish-cache.org> commit 4db4565f421f3df8d57930bf8cd4fb0ea3e55017 Author: Dridi Boukelmoune Date: Tue Jan 5 16:08:18 2021 +0100 build: Add VMOD names to automake boilerplates This would otherwise prevent VMODs from leaving in the same directory. diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index c68b30836..956ec6165 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -81,7 +81,7 @@ PFX.h vmod_XXX.rst vmod_XXX.man.rst: PFX.c PFX.c: $(vmodtool) $(srcdir)/VCC \t at PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/VCC -EXTRA_DIST = $(srcdir)/VCC automake_boilerplate.am +EXTRA_DIST = $(srcdir)/VCC automake_boilerplate_XXX.am CLEANFILES = $(builddir)/PFX.c $(builddir)/PFX.h \\ \t$(builddir)/vmod_XXX.rst \\ @@ -987,7 +987,8 @@ class vcc(object): def amboilerplate(self): ''' Produce boilplate for autocrap tools ''' vcc = os.path.basename(self.inputfile) - fo = self.openfile("automake_boilerplate.am") + fn = "automake_boilerplate_" + self.modname + ".am" + fo = self.openfile(fn) fo.write(AMBOILERPLATE.replace("XXX", self.modname) .replace("VCC", vcc) .replace("PFX", self.pfx)) diff --git a/lib/libvmod_blob/Makefile.am b/lib/libvmod_blob/Makefile.am index 040b1aceb..2ca6384e9 100644 --- a/lib/libvmod_blob/Makefile.am +++ b/lib/libvmod_blob/Makefile.am @@ -12,4 +12,4 @@ libvmod_blob_la_SOURCES = \ tbl_case.h # Use vmodtool.py generated automake boilerplate -include $(srcdir)/automake_boilerplate.am +include $(srcdir)/automake_boilerplate_blob.am diff --git a/lib/libvmod_blob/automake_boilerplate.am b/lib/libvmod_blob/automake_boilerplate_blob.am similarity index 93% rename from lib/libvmod_blob/automake_boilerplate.am rename to lib/libvmod_blob/automake_boilerplate_blob.am index 60e496959..eddb2c8b7 100644 --- a/lib/libvmod_blob/automake_boilerplate.am +++ b/lib/libvmod_blob/automake_boilerplate_blob.am @@ -30,7 +30,7 @@ vcc_blob_if.h vmod_blob.rst vmod_blob.man.rst: vcc_blob_if.c vcc_blob_if.c: $(vmodtool) $(srcdir)/vmod_blob.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_blob.vcc -EXTRA_DIST = $(srcdir)/vmod_blob.vcc automake_boilerplate.am +EXTRA_DIST = $(srcdir)/vmod_blob.vcc automake_boilerplate_blob.am CLEANFILES = $(builddir)/vcc_blob_if.c $(builddir)/vcc_blob_if.h \ $(builddir)/vmod_blob.rst \ diff --git a/lib/libvmod_cookie/Makefile.am b/lib/libvmod_cookie/Makefile.am index e9ed0cf2e..1a0b6a6d0 100644 --- a/lib/libvmod_cookie/Makefile.am +++ b/lib/libvmod_cookie/Makefile.am @@ -2,4 +2,4 @@ libvmod_cookie_la_SOURCES = \ vmod_cookie.c # Use vmodtool.py generated automake boilerplate -include $(srcdir)/automake_boilerplate.am +include $(srcdir)/automake_boilerplate_cookie.am diff --git a/lib/libvmod_cookie/automake_boilerplate.am b/lib/libvmod_cookie/automake_boilerplate_cookie.am similarity index 95% rename from lib/libvmod_cookie/automake_boilerplate.am rename to lib/libvmod_cookie/automake_boilerplate_cookie.am index 88d6167a6..8fbed5964 100644 --- a/lib/libvmod_cookie/automake_boilerplate.am +++ b/lib/libvmod_cookie/automake_boilerplate_cookie.am @@ -30,7 +30,7 @@ vcc_cookie_if.h vmod_cookie.rst vmod_cookie.man.rst: vcc_cookie_if.c vcc_cookie_if.c: $(vmodtool) $(srcdir)/vmod_cookie.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_cookie.vcc -EXTRA_DIST = $(srcdir)/vmod_cookie.vcc automake_boilerplate.am +EXTRA_DIST = $(srcdir)/vmod_cookie.vcc automake_boilerplate_cookie.am CLEANFILES = $(builddir)/vcc_cookie_if.c $(builddir)/vcc_cookie_if.h \ $(builddir)/vmod_cookie.rst \ diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am index 3f902065c..687711aef 100644 --- a/lib/libvmod_debug/Makefile.am +++ b/lib/libvmod_debug/Makefile.am @@ -6,7 +6,7 @@ libvmod_debug_la_SOURCES = \ vmod_debug_dyn.c -include $(srcdir)/automake_boilerplate.am +include $(srcdir)/automake_boilerplate_debug.am # Allow Vmod_wrong*_Data to be exported libvmod_debug_la_LDFLAGS = \ diff --git a/lib/libvmod_debug/automake_boilerplate.am b/lib/libvmod_debug/automake_boilerplate_debug.am similarity index 93% rename from lib/libvmod_debug/automake_boilerplate.am rename to lib/libvmod_debug/automake_boilerplate_debug.am index 7850094f6..1c7f86e7b 100644 --- a/lib/libvmod_debug/automake_boilerplate.am +++ b/lib/libvmod_debug/automake_boilerplate_debug.am @@ -30,7 +30,7 @@ vcc_debug_if.h vmod_debug.rst vmod_debug.man.rst: vcc_debug_if.c vcc_debug_if.c: $(vmodtool) $(srcdir)/vmod_debug.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_debug.vcc -EXTRA_DIST = $(srcdir)/vmod_debug.vcc automake_boilerplate.am +EXTRA_DIST = $(srcdir)/vmod_debug.vcc automake_boilerplate_debug.am CLEANFILES = $(builddir)/vcc_debug_if.c $(builddir)/vcc_debug_if.h \ $(builddir)/vmod_debug.rst \ diff --git a/lib/libvmod_directors/Makefile.am b/lib/libvmod_directors/Makefile.am index 646357100..f9dc599c0 100644 --- a/lib/libvmod_directors/Makefile.am +++ b/lib/libvmod_directors/Makefile.am @@ -15,4 +15,4 @@ libvmod_directors_la_SOURCES = \ shard_dir.h # Use vmodtool.py generated automake boilerplate -include $(srcdir)/automake_boilerplate.am +include $(srcdir)/automake_boilerplate_directors.am diff --git a/lib/libvmod_directors/automake_boilerplate.am b/lib/libvmod_directors/automake_boilerplate_directors.am similarity index 92% rename from lib/libvmod_directors/automake_boilerplate.am rename to lib/libvmod_directors/automake_boilerplate_directors.am index d84a44076..182f418e0 100644 --- a/lib/libvmod_directors/automake_boilerplate.am +++ b/lib/libvmod_directors/automake_boilerplate_directors.am @@ -30,7 +30,7 @@ vcc_directors_if.h vmod_directors.rst vmod_directors.man.rst: vcc_directors_if.c vcc_directors_if.c: $(vmodtool) $(srcdir)/vmod_directors.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_directors.vcc -EXTRA_DIST = $(srcdir)/vmod_directors.vcc automake_boilerplate.am +EXTRA_DIST = $(srcdir)/vmod_directors.vcc automake_boilerplate_directors.am CLEANFILES = $(builddir)/vcc_directors_if.c $(builddir)/vcc_directors_if.h \ $(builddir)/vmod_directors.rst \ diff --git a/lib/libvmod_proxy/Makefile.am b/lib/libvmod_proxy/Makefile.am index d8d5861c8..c98a060f8 100644 --- a/lib/libvmod_proxy/Makefile.am +++ b/lib/libvmod_proxy/Makefile.am @@ -4,4 +4,4 @@ libvmod_proxy_la_SOURCES = \ vmod_proxy.c # Use vmodtool.py generated automake boilerplate -include $(srcdir)/automake_boilerplate.am +include $(srcdir)/automake_boilerplate_proxy.am diff --git a/lib/libvmod_proxy/automake_boilerplate.am b/lib/libvmod_proxy/automake_boilerplate_proxy.am similarity index 93% rename from lib/libvmod_proxy/automake_boilerplate.am rename to lib/libvmod_proxy/automake_boilerplate_proxy.am index 71cd6dab3..5d744f2e8 100644 --- a/lib/libvmod_proxy/automake_boilerplate.am +++ b/lib/libvmod_proxy/automake_boilerplate_proxy.am @@ -30,7 +30,7 @@ vcc_proxy_if.h vmod_proxy.rst vmod_proxy.man.rst: vcc_proxy_if.c vcc_proxy_if.c: $(vmodtool) $(srcdir)/vmod_proxy.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_proxy.vcc -EXTRA_DIST = $(srcdir)/vmod_proxy.vcc automake_boilerplate.am +EXTRA_DIST = $(srcdir)/vmod_proxy.vcc automake_boilerplate_proxy.am CLEANFILES = $(builddir)/vcc_proxy_if.c $(builddir)/vcc_proxy_if.h \ $(builddir)/vmod_proxy.rst \ diff --git a/lib/libvmod_purge/Makefile.am b/lib/libvmod_purge/Makefile.am index 38e1e5869..7899bb056 100644 --- a/lib/libvmod_purge/Makefile.am +++ b/lib/libvmod_purge/Makefile.am @@ -4,4 +4,4 @@ libvmod_purge_la_SOURCES = \ vmod_purge.c # Use vmodtool.py generated automake boilerplate -include $(srcdir)/automake_boilerplate.am +include $(srcdir)/automake_boilerplate_purge.am diff --git a/lib/libvmod_purge/automake_boilerplate.am b/lib/libvmod_purge/automake_boilerplate_purge.am similarity index 93% rename from lib/libvmod_purge/automake_boilerplate.am rename to lib/libvmod_purge/automake_boilerplate_purge.am index f06b85afe..259232ca9 100644 --- a/lib/libvmod_purge/automake_boilerplate.am +++ b/lib/libvmod_purge/automake_boilerplate_purge.am @@ -30,7 +30,7 @@ vcc_purge_if.h vmod_purge.rst vmod_purge.man.rst: vcc_purge_if.c vcc_purge_if.c: $(vmodtool) $(srcdir)/vmod_purge.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_purge.vcc -EXTRA_DIST = $(srcdir)/vmod_purge.vcc automake_boilerplate.am +EXTRA_DIST = $(srcdir)/vmod_purge.vcc automake_boilerplate_purge.am CLEANFILES = $(builddir)/vcc_purge_if.c $(builddir)/vcc_purge_if.h \ $(builddir)/vmod_purge.rst \ diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am index 8bde75239..292239d02 100644 --- a/lib/libvmod_std/Makefile.am +++ b/lib/libvmod_std/Makefile.am @@ -7,4 +7,4 @@ libvmod_std_la_SOURCES = \ vmod_std_querysort.c # Use vmodtool.py generated automake boilerplate -include $(srcdir)/automake_boilerplate.am +include $(srcdir)/automake_boilerplate_std.am diff --git a/lib/libvmod_std/automake_boilerplate.am b/lib/libvmod_std/automake_boilerplate_std.am similarity index 94% rename from lib/libvmod_std/automake_boilerplate.am rename to lib/libvmod_std/automake_boilerplate_std.am index 5bd015b9b..aa859751c 100644 --- a/lib/libvmod_std/automake_boilerplate.am +++ b/lib/libvmod_std/automake_boilerplate_std.am @@ -30,7 +30,7 @@ vcc_std_if.h vmod_std.rst vmod_std.man.rst: vcc_std_if.c vcc_std_if.c: $(vmodtool) $(srcdir)/vmod_std.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_std.vcc -EXTRA_DIST = $(srcdir)/vmod_std.vcc automake_boilerplate.am +EXTRA_DIST = $(srcdir)/vmod_std.vcc automake_boilerplate_std.am CLEANFILES = $(builddir)/vcc_std_if.c $(builddir)/vcc_std_if.h \ $(builddir)/vmod_std.rst \ diff --git a/lib/libvmod_unix/Makefile.am b/lib/libvmod_unix/Makefile.am index 8dafebac8..c68108aef 100644 --- a/lib/libvmod_unix/Makefile.am +++ b/lib/libvmod_unix/Makefile.am @@ -5,4 +5,4 @@ libvmod_unix_la_SOURCES = \ cred_compat.h # Use vmodtool.py generated automake boilerplate -include $(srcdir)/automake_boilerplate.am +include $(srcdir)/automake_boilerplate_unix.am diff --git a/lib/libvmod_unix/automake_boilerplate.am b/lib/libvmod_unix/automake_boilerplate_unix.am similarity index 93% rename from lib/libvmod_unix/automake_boilerplate.am rename to lib/libvmod_unix/automake_boilerplate_unix.am index fa0586db3..f1256e205 100644 --- a/lib/libvmod_unix/automake_boilerplate.am +++ b/lib/libvmod_unix/automake_boilerplate_unix.am @@ -30,7 +30,7 @@ vcc_unix_if.h vmod_unix.rst vmod_unix.man.rst: vcc_unix_if.c vcc_unix_if.c: $(vmodtool) $(srcdir)/vmod_unix.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_unix.vcc -EXTRA_DIST = $(srcdir)/vmod_unix.vcc automake_boilerplate.am +EXTRA_DIST = $(srcdir)/vmod_unix.vcc automake_boilerplate_unix.am CLEANFILES = $(builddir)/vcc_unix_if.c $(builddir)/vcc_unix_if.h \ $(builddir)/vmod_unix.rst \ diff --git a/lib/libvmod_vtc/Makefile.am b/lib/libvmod_vtc/Makefile.am index 3575d7e8c..557156f6e 100644 --- a/lib/libvmod_vtc/Makefile.am +++ b/lib/libvmod_vtc/Makefile.am @@ -4,4 +4,4 @@ libvmod_vtc_la_SOURCES = \ vmod_vtc.c # Use vmodtool.py generated automake boilerplate -include $(srcdir)/automake_boilerplate.am +include $(srcdir)/automake_boilerplate_vtc.am diff --git a/lib/libvmod_vtc/automake_boilerplate.am b/lib/libvmod_vtc/automake_boilerplate_vtc.am similarity index 93% rename from lib/libvmod_vtc/automake_boilerplate.am rename to lib/libvmod_vtc/automake_boilerplate_vtc.am index 75f8ccb1f..fd6124c01 100644 --- a/lib/libvmod_vtc/automake_boilerplate.am +++ b/lib/libvmod_vtc/automake_boilerplate_vtc.am @@ -30,7 +30,7 @@ vcc_vtc_if.h vmod_vtc.rst vmod_vtc.man.rst: vcc_vtc_if.c vcc_vtc_if.c: $(vmodtool) $(srcdir)/vmod_vtc.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_vtc.vcc -EXTRA_DIST = $(srcdir)/vmod_vtc.vcc automake_boilerplate.am +EXTRA_DIST = $(srcdir)/vmod_vtc.vcc automake_boilerplate_vtc.am CLEANFILES = $(builddir)/vcc_vtc_if.c $(builddir)/vcc_vtc_if.h \ $(builddir)/vmod_vtc.rst \ From dridi.boukelmoune at gmail.com Thu Jan 7 14:16:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 7 Jan 2021 14:16:07 +0000 (UTC) Subject: [master] 26a401822 build: Move the VMODs tests out of the boilerplate Message-ID: <20210107141607.1F676A9247@lists.varnish-cache.org> commit 26a401822e7d6fb42d3d02c56c58d845d2c33962 Author: Dridi Boukelmoune Date: Tue Jan 5 16:45:01 2021 +0100 build: Move the VMODs tests out of the boilerplate We can't have more than one test suite if we put all VMODs in the same directory. diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 956ec6165..4250b816b 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -88,19 +88,6 @@ CLEANFILES = $(builddir)/PFX.c $(builddir)/PFX.h \\ \t$(builddir)/vmod_XXX.man.rst ''' -AMBOILERPLATE_CHECK = ''' -TESTS = \\ -\tVTC - -EXTRA_DIST += $(TESTS) - -vtc-refresh-tests: -\t at PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/VCC -\t at cd $(top_builddir) && ./config.status --file=$(subdir)/Makefile - -include $(top_srcdir)/vtc.am -''' - PRIVS = { 'PRIV_CALL': "struct vmod_priv *", 'PRIV_VCL': "struct vmod_priv *", @@ -992,11 +979,6 @@ class vcc(object): fo.write(AMBOILERPLATE.replace("XXX", self.modname) .replace("VCC", vcc) .replace("PFX", self.pfx)) - tests = glob.glob("tests/*.vtc") - if len(tests) > 0: - tests.sort() - fo.write(AMBOILERPLATE_CHECK.replace("VCC", vcc). - replace("VTC", " \\\n\t".join(tests))) fo.close() def mkdefs(self, fo): diff --git a/lib/libvmod_cookie/Makefile.am b/lib/libvmod_cookie/Makefile.am index 1a0b6a6d0..b85531aed 100644 --- a/lib/libvmod_cookie/Makefile.am +++ b/lib/libvmod_cookie/Makefile.am @@ -3,3 +3,29 @@ libvmod_cookie_la_SOURCES = \ # Use vmodtool.py generated automake boilerplate include $(srcdir)/automake_boilerplate_cookie.am + +TESTS = \ + tests/cookie_b00000.vtc \ + tests/cookie_b00001.vtc \ + tests/cookie_b00002.vtc \ + tests/cookie_b00003.vtc \ + tests/cookie_b00004.vtc \ + tests/cookie_b00005.vtc \ + tests/cookie_b00006.vtc \ + tests/cookie_b00007.vtc \ + tests/cookie_b00008.vtc \ + tests/cookie_b00009.vtc \ + tests/cookie_b00010.vtc \ + tests/cookie_b00011.vtc \ + tests/cookie_b00012.vtc \ + tests/cookie_b00013.vtc \ + tests/cookie_r00028.vtc \ + tests/cookie_v00000.vtc + +EXTRA_DIST += $(TESTS) + +vtc-refresh-tests: + @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_cookie.vcc + @cd $(top_builddir) && ./config.status --file=$(subdir)/Makefile + +include $(top_srcdir)/vtc.am diff --git a/lib/libvmod_cookie/automake_boilerplate_cookie.am b/lib/libvmod_cookie/automake_boilerplate_cookie.am index 8fbed5964..f03d43f1f 100644 --- a/lib/libvmod_cookie/automake_boilerplate_cookie.am +++ b/lib/libvmod_cookie/automake_boilerplate_cookie.am @@ -35,29 +35,3 @@ EXTRA_DIST = $(srcdir)/vmod_cookie.vcc automake_boilerplate_cookie.am CLEANFILES = $(builddir)/vcc_cookie_if.c $(builddir)/vcc_cookie_if.h \ $(builddir)/vmod_cookie.rst \ $(builddir)/vmod_cookie.man.rst - -TESTS = \ - tests/cookie_b00000.vtc \ - tests/cookie_b00001.vtc \ - tests/cookie_b00002.vtc \ - tests/cookie_b00003.vtc \ - tests/cookie_b00004.vtc \ - tests/cookie_b00005.vtc \ - tests/cookie_b00006.vtc \ - tests/cookie_b00007.vtc \ - tests/cookie_b00008.vtc \ - tests/cookie_b00009.vtc \ - tests/cookie_b00010.vtc \ - tests/cookie_b00011.vtc \ - tests/cookie_b00012.vtc \ - tests/cookie_b00013.vtc \ - tests/cookie_r00028.vtc \ - tests/cookie_v00000.vtc - -EXTRA_DIST += $(TESTS) - -vtc-refresh-tests: - @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_cookie.vcc - @cd $(top_builddir) && ./config.status --file=$(subdir)/Makefile - -include $(top_srcdir)/vtc.am diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am index 292239d02..9126fd613 100644 --- a/lib/libvmod_std/Makefile.am +++ b/lib/libvmod_std/Makefile.am @@ -8,3 +8,14 @@ libvmod_std_la_SOURCES = \ # Use vmodtool.py generated automake boilerplate include $(srcdir)/automake_boilerplate_std.am + +TESTS = \ + tests/std_b00001.vtc + +EXTRA_DIST += $(TESTS) + +vtc-refresh-tests: + @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_std.vcc + @cd $(top_builddir) && ./config.status --file=$(subdir)/Makefile + +include $(top_srcdir)/vtc.am diff --git a/lib/libvmod_std/automake_boilerplate_std.am b/lib/libvmod_std/automake_boilerplate_std.am index aa859751c..f4648b226 100644 --- a/lib/libvmod_std/automake_boilerplate_std.am +++ b/lib/libvmod_std/automake_boilerplate_std.am @@ -35,14 +35,3 @@ EXTRA_DIST = $(srcdir)/vmod_std.vcc automake_boilerplate_std.am CLEANFILES = $(builddir)/vcc_std_if.c $(builddir)/vcc_std_if.h \ $(builddir)/vmod_std.rst \ $(builddir)/vmod_std.man.rst - -TESTS = \ - tests/std_b00001.vtc - -EXTRA_DIST += $(TESTS) - -vtc-refresh-tests: - @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_std.vcc - @cd $(top_builddir) && ./config.status --file=$(subdir)/Makefile - -include $(top_srcdir)/vtc.am From dridi.boukelmoune at gmail.com Thu Jan 7 14:16:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 7 Jan 2021 14:16:07 +0000 (UTC) Subject: [master] a9665427f build: Make the VMOD symbols regex extensible Message-ID: <20210107141607.3B40BA924C@lists.varnish-cache.org> commit a9665427f62bcad7d0d7b2eff00744d45db79743 Author: Dridi Boukelmoune Date: Tue Jan 5 17:15:43 2021 +0100 build: Make the VMOD symbols regex extensible That will hopefully not break macos this time. Refs 8b79193094cc diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 4250b816b..ad58a3c7d 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -63,11 +63,13 @@ vmodtoolargs ?= --strict --boilerplate -o PFX vmod_LTLIBRARIES = libvmod_XXX.la -libvmod_XXX_la_CFLAGS ?= \\ +libvmod_XXX_la_CFLAGS = \\ \t at SAN_CFLAGS@ +vmod_XXX_symbols_regex ?= Vmod_XXX_Data + libvmod_XXX_la_LDFLAGS = \\ -\t-export-symbols-regex 'Vmod_XXX_Data' \\ +\t-export-symbols-regex $(vmod_XXX_symbols_regex) \\ \t$(AM_LDFLAGS) \\ \t$(VMOD_LDFLAGS) \\ \t at SAN_LDFLAGS@ diff --git a/lib/libvmod_blob/automake_boilerplate_blob.am b/lib/libvmod_blob/automake_boilerplate_blob.am index eddb2c8b7..19db88cba 100644 --- a/lib/libvmod_blob/automake_boilerplate_blob.am +++ b/lib/libvmod_blob/automake_boilerplate_blob.am @@ -12,11 +12,13 @@ vmodtoolargs ?= --strict --boilerplate -o vcc_blob_if vmod_LTLIBRARIES = libvmod_blob.la -libvmod_blob_la_CFLAGS ?= \ +libvmod_blob_la_CFLAGS = \ @SAN_CFLAGS@ +vmod_blob_symbols_regex ?= Vmod_blob_Data + libvmod_blob_la_LDFLAGS = \ - -export-symbols-regex 'Vmod_blob_Data' \ + -export-symbols-regex $(vmod_blob_symbols_regex) \ $(AM_LDFLAGS) \ $(VMOD_LDFLAGS) \ @SAN_LDFLAGS@ diff --git a/lib/libvmod_cookie/automake_boilerplate_cookie.am b/lib/libvmod_cookie/automake_boilerplate_cookie.am index f03d43f1f..aa6867d7e 100644 --- a/lib/libvmod_cookie/automake_boilerplate_cookie.am +++ b/lib/libvmod_cookie/automake_boilerplate_cookie.am @@ -12,11 +12,13 @@ vmodtoolargs ?= --strict --boilerplate -o vcc_cookie_if vmod_LTLIBRARIES = libvmod_cookie.la -libvmod_cookie_la_CFLAGS ?= \ +libvmod_cookie_la_CFLAGS = \ @SAN_CFLAGS@ +vmod_cookie_symbols_regex ?= Vmod_cookie_Data + libvmod_cookie_la_LDFLAGS = \ - -export-symbols-regex 'Vmod_cookie_Data' \ + -export-symbols-regex $(vmod_cookie_symbols_regex) \ $(AM_LDFLAGS) \ $(VMOD_LDFLAGS) \ @SAN_LDFLAGS@ diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am index 687711aef..60b0757eb 100644 --- a/lib/libvmod_debug/Makefile.am +++ b/lib/libvmod_debug/Makefile.am @@ -5,16 +5,10 @@ libvmod_debug_la_SOURCES = \ vmod_debug_obj.c \ vmod_debug_dyn.c - include $(srcdir)/automake_boilerplate_debug.am # Allow Vmod_wrong*_Data to be exported -libvmod_debug_la_LDFLAGS = \ - -export-symbols-regex 'Vmod_.*_Data' \ - $(AM_LDFLAGS) \ - $(VMOD_LDFLAGS) \ - @SAN_LDFLAGS@ - +vmod_debug_symbols_regex = 'Vmod_.*_Data' # not --strict vmodtoolargs = --boilerplate -o vcc_debug_if diff --git a/lib/libvmod_debug/automake_boilerplate_debug.am b/lib/libvmod_debug/automake_boilerplate_debug.am index 1c7f86e7b..ec97cc5be 100644 --- a/lib/libvmod_debug/automake_boilerplate_debug.am +++ b/lib/libvmod_debug/automake_boilerplate_debug.am @@ -12,11 +12,13 @@ vmodtoolargs ?= --strict --boilerplate -o vcc_debug_if vmod_LTLIBRARIES = libvmod_debug.la -libvmod_debug_la_CFLAGS ?= \ +libvmod_debug_la_CFLAGS = \ @SAN_CFLAGS@ +vmod_debug_symbols_regex ?= Vmod_debug_Data + libvmod_debug_la_LDFLAGS = \ - -export-symbols-regex 'Vmod_debug_Data' \ + -export-symbols-regex $(vmod_debug_symbols_regex) \ $(AM_LDFLAGS) \ $(VMOD_LDFLAGS) \ @SAN_LDFLAGS@ diff --git a/lib/libvmod_directors/automake_boilerplate_directors.am b/lib/libvmod_directors/automake_boilerplate_directors.am index 182f418e0..ae29235f1 100644 --- a/lib/libvmod_directors/automake_boilerplate_directors.am +++ b/lib/libvmod_directors/automake_boilerplate_directors.am @@ -12,11 +12,13 @@ vmodtoolargs ?= --strict --boilerplate -o vcc_directors_if vmod_LTLIBRARIES = libvmod_directors.la -libvmod_directors_la_CFLAGS ?= \ +libvmod_directors_la_CFLAGS = \ @SAN_CFLAGS@ +vmod_directors_symbols_regex ?= Vmod_directors_Data + libvmod_directors_la_LDFLAGS = \ - -export-symbols-regex 'Vmod_directors_Data' \ + -export-symbols-regex $(vmod_directors_symbols_regex) \ $(AM_LDFLAGS) \ $(VMOD_LDFLAGS) \ @SAN_LDFLAGS@ diff --git a/lib/libvmod_proxy/automake_boilerplate_proxy.am b/lib/libvmod_proxy/automake_boilerplate_proxy.am index 5d744f2e8..cad272c9f 100644 --- a/lib/libvmod_proxy/automake_boilerplate_proxy.am +++ b/lib/libvmod_proxy/automake_boilerplate_proxy.am @@ -12,11 +12,13 @@ vmodtoolargs ?= --strict --boilerplate -o vcc_proxy_if vmod_LTLIBRARIES = libvmod_proxy.la -libvmod_proxy_la_CFLAGS ?= \ +libvmod_proxy_la_CFLAGS = \ @SAN_CFLAGS@ +vmod_proxy_symbols_regex ?= Vmod_proxy_Data + libvmod_proxy_la_LDFLAGS = \ - -export-symbols-regex 'Vmod_proxy_Data' \ + -export-symbols-regex $(vmod_proxy_symbols_regex) \ $(AM_LDFLAGS) \ $(VMOD_LDFLAGS) \ @SAN_LDFLAGS@ diff --git a/lib/libvmod_purge/automake_boilerplate_purge.am b/lib/libvmod_purge/automake_boilerplate_purge.am index 259232ca9..9988d5dfa 100644 --- a/lib/libvmod_purge/automake_boilerplate_purge.am +++ b/lib/libvmod_purge/automake_boilerplate_purge.am @@ -12,11 +12,13 @@ vmodtoolargs ?= --strict --boilerplate -o vcc_purge_if vmod_LTLIBRARIES = libvmod_purge.la -libvmod_purge_la_CFLAGS ?= \ +libvmod_purge_la_CFLAGS = \ @SAN_CFLAGS@ +vmod_purge_symbols_regex ?= Vmod_purge_Data + libvmod_purge_la_LDFLAGS = \ - -export-symbols-regex 'Vmod_purge_Data' \ + -export-symbols-regex $(vmod_purge_symbols_regex) \ $(AM_LDFLAGS) \ $(VMOD_LDFLAGS) \ @SAN_LDFLAGS@ diff --git a/lib/libvmod_std/automake_boilerplate_std.am b/lib/libvmod_std/automake_boilerplate_std.am index f4648b226..8201d962b 100644 --- a/lib/libvmod_std/automake_boilerplate_std.am +++ b/lib/libvmod_std/automake_boilerplate_std.am @@ -12,11 +12,13 @@ vmodtoolargs ?= --strict --boilerplate -o vcc_std_if vmod_LTLIBRARIES = libvmod_std.la -libvmod_std_la_CFLAGS ?= \ +libvmod_std_la_CFLAGS = \ @SAN_CFLAGS@ +vmod_std_symbols_regex ?= Vmod_std_Data + libvmod_std_la_LDFLAGS = \ - -export-symbols-regex 'Vmod_std_Data' \ + -export-symbols-regex $(vmod_std_symbols_regex) \ $(AM_LDFLAGS) \ $(VMOD_LDFLAGS) \ @SAN_LDFLAGS@ diff --git a/lib/libvmod_unix/automake_boilerplate_unix.am b/lib/libvmod_unix/automake_boilerplate_unix.am index f1256e205..d9a4f9c0e 100644 --- a/lib/libvmod_unix/automake_boilerplate_unix.am +++ b/lib/libvmod_unix/automake_boilerplate_unix.am @@ -12,11 +12,13 @@ vmodtoolargs ?= --strict --boilerplate -o vcc_unix_if vmod_LTLIBRARIES = libvmod_unix.la -libvmod_unix_la_CFLAGS ?= \ +libvmod_unix_la_CFLAGS = \ @SAN_CFLAGS@ +vmod_unix_symbols_regex ?= Vmod_unix_Data + libvmod_unix_la_LDFLAGS = \ - -export-symbols-regex 'Vmod_unix_Data' \ + -export-symbols-regex $(vmod_unix_symbols_regex) \ $(AM_LDFLAGS) \ $(VMOD_LDFLAGS) \ @SAN_LDFLAGS@ diff --git a/lib/libvmod_vtc/automake_boilerplate_vtc.am b/lib/libvmod_vtc/automake_boilerplate_vtc.am index fd6124c01..53579929b 100644 --- a/lib/libvmod_vtc/automake_boilerplate_vtc.am +++ b/lib/libvmod_vtc/automake_boilerplate_vtc.am @@ -12,11 +12,13 @@ vmodtoolargs ?= --strict --boilerplate -o vcc_vtc_if vmod_LTLIBRARIES = libvmod_vtc.la -libvmod_vtc_la_CFLAGS ?= \ +libvmod_vtc_la_CFLAGS = \ @SAN_CFLAGS@ +vmod_vtc_symbols_regex ?= Vmod_vtc_Data + libvmod_vtc_la_LDFLAGS = \ - -export-symbols-regex 'Vmod_vtc_Data' \ + -export-symbols-regex $(vmod_vtc_symbols_regex) \ $(AM_LDFLAGS) \ $(VMOD_LDFLAGS) \ @SAN_LDFLAGS@ From dridi.boukelmoune at gmail.com Thu Jan 7 14:16:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 7 Jan 2021 14:16:07 +0000 (UTC) Subject: [master] f5aab9bf2 build: Make vmodtool args extensible per VMOD Message-ID: <20210107141607.581E3A9251@lists.varnish-cache.org> commit f5aab9bf215de1dd40dc3baf0a85306a4cbfcafb Author: Dridi Boukelmoune Date: Tue Jan 5 17:33:03 2021 +0100 build: Make vmodtool args extensible per VMOD In practice, we only tweak vmod_debug's build for coverage purposes. Another option could be to have a VTC running vmodtool.py with various VCC inputs but we currently don't have test coverage for our python scripts. diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index ad58a3c7d..b84f3464b 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -59,7 +59,7 @@ AM_CPPFLAGS = \\ \t-I$(top_builddir)/include vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs ?= --strict --boilerplate -o PFX +vmodtoolargs_XXX ?= --strict --boilerplate -o PFX vmod_LTLIBRARIES = libvmod_XXX.la @@ -81,7 +81,7 @@ $(libvmod_XXX_la_OBJECTS): PFX.h PFX.h vmod_XXX.rst vmod_XXX.man.rst: PFX.c PFX.c: $(vmodtool) $(srcdir)/VCC -\t at PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/VCC +\t at PYTHON@ $(vmodtool) $(vmodtoolargs_XXX) $(srcdir)/VCC EXTRA_DIST = $(srcdir)/VCC automake_boilerplate_XXX.am diff --git a/lib/libvmod_blob/automake_boilerplate_blob.am b/lib/libvmod_blob/automake_boilerplate_blob.am index 19db88cba..86c9a71a9 100644 --- a/lib/libvmod_blob/automake_boilerplate_blob.am +++ b/lib/libvmod_blob/automake_boilerplate_blob.am @@ -8,7 +8,7 @@ AM_CPPFLAGS = \ -I$(top_builddir)/include vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs ?= --strict --boilerplate -o vcc_blob_if +vmodtoolargs_blob ?= --strict --boilerplate -o vcc_blob_if vmod_LTLIBRARIES = libvmod_blob.la @@ -30,7 +30,7 @@ $(libvmod_blob_la_OBJECTS): vcc_blob_if.h vcc_blob_if.h vmod_blob.rst vmod_blob.man.rst: vcc_blob_if.c vcc_blob_if.c: $(vmodtool) $(srcdir)/vmod_blob.vcc - @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_blob.vcc + @PYTHON@ $(vmodtool) $(vmodtoolargs_blob) $(srcdir)/vmod_blob.vcc EXTRA_DIST = $(srcdir)/vmod_blob.vcc automake_boilerplate_blob.am diff --git a/lib/libvmod_cookie/automake_boilerplate_cookie.am b/lib/libvmod_cookie/automake_boilerplate_cookie.am index aa6867d7e..007a19199 100644 --- a/lib/libvmod_cookie/automake_boilerplate_cookie.am +++ b/lib/libvmod_cookie/automake_boilerplate_cookie.am @@ -8,7 +8,7 @@ AM_CPPFLAGS = \ -I$(top_builddir)/include vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs ?= --strict --boilerplate -o vcc_cookie_if +vmodtoolargs_cookie ?= --strict --boilerplate -o vcc_cookie_if vmod_LTLIBRARIES = libvmod_cookie.la @@ -30,7 +30,7 @@ $(libvmod_cookie_la_OBJECTS): vcc_cookie_if.h vcc_cookie_if.h vmod_cookie.rst vmod_cookie.man.rst: vcc_cookie_if.c vcc_cookie_if.c: $(vmodtool) $(srcdir)/vmod_cookie.vcc - @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_cookie.vcc + @PYTHON@ $(vmodtool) $(vmodtoolargs_cookie) $(srcdir)/vmod_cookie.vcc EXTRA_DIST = $(srcdir)/vmod_cookie.vcc automake_boilerplate_cookie.am diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am index 60b0757eb..bcd29af6f 100644 --- a/lib/libvmod_debug/Makefile.am +++ b/lib/libvmod_debug/Makefile.am @@ -11,7 +11,7 @@ include $(srcdir)/automake_boilerplate_debug.am vmod_debug_symbols_regex = 'Vmod_.*_Data' # not --strict -vmodtoolargs = --boilerplate -o vcc_debug_if +vmodtoolargs_debug = --boilerplate -o vcc_debug_if .vsc.c: $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $< diff --git a/lib/libvmod_debug/automake_boilerplate_debug.am b/lib/libvmod_debug/automake_boilerplate_debug.am index ec97cc5be..e589a1c76 100644 --- a/lib/libvmod_debug/automake_boilerplate_debug.am +++ b/lib/libvmod_debug/automake_boilerplate_debug.am @@ -8,7 +8,7 @@ AM_CPPFLAGS = \ -I$(top_builddir)/include vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs ?= --strict --boilerplate -o vcc_debug_if +vmodtoolargs_debug ?= --strict --boilerplate -o vcc_debug_if vmod_LTLIBRARIES = libvmod_debug.la @@ -30,7 +30,7 @@ $(libvmod_debug_la_OBJECTS): vcc_debug_if.h vcc_debug_if.h vmod_debug.rst vmod_debug.man.rst: vcc_debug_if.c vcc_debug_if.c: $(vmodtool) $(srcdir)/vmod_debug.vcc - @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_debug.vcc + @PYTHON@ $(vmodtool) $(vmodtoolargs_debug) $(srcdir)/vmod_debug.vcc EXTRA_DIST = $(srcdir)/vmod_debug.vcc automake_boilerplate_debug.am diff --git a/lib/libvmod_directors/automake_boilerplate_directors.am b/lib/libvmod_directors/automake_boilerplate_directors.am index ae29235f1..2c343a523 100644 --- a/lib/libvmod_directors/automake_boilerplate_directors.am +++ b/lib/libvmod_directors/automake_boilerplate_directors.am @@ -8,7 +8,7 @@ AM_CPPFLAGS = \ -I$(top_builddir)/include vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs ?= --strict --boilerplate -o vcc_directors_if +vmodtoolargs_directors ?= --strict --boilerplate -o vcc_directors_if vmod_LTLIBRARIES = libvmod_directors.la @@ -30,7 +30,7 @@ $(libvmod_directors_la_OBJECTS): vcc_directors_if.h vcc_directors_if.h vmod_directors.rst vmod_directors.man.rst: vcc_directors_if.c vcc_directors_if.c: $(vmodtool) $(srcdir)/vmod_directors.vcc - @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_directors.vcc + @PYTHON@ $(vmodtool) $(vmodtoolargs_directors) $(srcdir)/vmod_directors.vcc EXTRA_DIST = $(srcdir)/vmod_directors.vcc automake_boilerplate_directors.am diff --git a/lib/libvmod_proxy/automake_boilerplate_proxy.am b/lib/libvmod_proxy/automake_boilerplate_proxy.am index cad272c9f..066eb68dd 100644 --- a/lib/libvmod_proxy/automake_boilerplate_proxy.am +++ b/lib/libvmod_proxy/automake_boilerplate_proxy.am @@ -8,7 +8,7 @@ AM_CPPFLAGS = \ -I$(top_builddir)/include vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs ?= --strict --boilerplate -o vcc_proxy_if +vmodtoolargs_proxy ?= --strict --boilerplate -o vcc_proxy_if vmod_LTLIBRARIES = libvmod_proxy.la @@ -30,7 +30,7 @@ $(libvmod_proxy_la_OBJECTS): vcc_proxy_if.h vcc_proxy_if.h vmod_proxy.rst vmod_proxy.man.rst: vcc_proxy_if.c vcc_proxy_if.c: $(vmodtool) $(srcdir)/vmod_proxy.vcc - @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_proxy.vcc + @PYTHON@ $(vmodtool) $(vmodtoolargs_proxy) $(srcdir)/vmod_proxy.vcc EXTRA_DIST = $(srcdir)/vmod_proxy.vcc automake_boilerplate_proxy.am diff --git a/lib/libvmod_purge/automake_boilerplate_purge.am b/lib/libvmod_purge/automake_boilerplate_purge.am index 9988d5dfa..89aeabb4a 100644 --- a/lib/libvmod_purge/automake_boilerplate_purge.am +++ b/lib/libvmod_purge/automake_boilerplate_purge.am @@ -8,7 +8,7 @@ AM_CPPFLAGS = \ -I$(top_builddir)/include vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs ?= --strict --boilerplate -o vcc_purge_if +vmodtoolargs_purge ?= --strict --boilerplate -o vcc_purge_if vmod_LTLIBRARIES = libvmod_purge.la @@ -30,7 +30,7 @@ $(libvmod_purge_la_OBJECTS): vcc_purge_if.h vcc_purge_if.h vmod_purge.rst vmod_purge.man.rst: vcc_purge_if.c vcc_purge_if.c: $(vmodtool) $(srcdir)/vmod_purge.vcc - @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_purge.vcc + @PYTHON@ $(vmodtool) $(vmodtoolargs_purge) $(srcdir)/vmod_purge.vcc EXTRA_DIST = $(srcdir)/vmod_purge.vcc automake_boilerplate_purge.am diff --git a/lib/libvmod_std/automake_boilerplate_std.am b/lib/libvmod_std/automake_boilerplate_std.am index 8201d962b..57dec41a3 100644 --- a/lib/libvmod_std/automake_boilerplate_std.am +++ b/lib/libvmod_std/automake_boilerplate_std.am @@ -8,7 +8,7 @@ AM_CPPFLAGS = \ -I$(top_builddir)/include vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs ?= --strict --boilerplate -o vcc_std_if +vmodtoolargs_std ?= --strict --boilerplate -o vcc_std_if vmod_LTLIBRARIES = libvmod_std.la @@ -30,7 +30,7 @@ $(libvmod_std_la_OBJECTS): vcc_std_if.h vcc_std_if.h vmod_std.rst vmod_std.man.rst: vcc_std_if.c vcc_std_if.c: $(vmodtool) $(srcdir)/vmod_std.vcc - @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_std.vcc + @PYTHON@ $(vmodtool) $(vmodtoolargs_std) $(srcdir)/vmod_std.vcc EXTRA_DIST = $(srcdir)/vmod_std.vcc automake_boilerplate_std.am diff --git a/lib/libvmod_unix/automake_boilerplate_unix.am b/lib/libvmod_unix/automake_boilerplate_unix.am index d9a4f9c0e..02cd56b2a 100644 --- a/lib/libvmod_unix/automake_boilerplate_unix.am +++ b/lib/libvmod_unix/automake_boilerplate_unix.am @@ -8,7 +8,7 @@ AM_CPPFLAGS = \ -I$(top_builddir)/include vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs ?= --strict --boilerplate -o vcc_unix_if +vmodtoolargs_unix ?= --strict --boilerplate -o vcc_unix_if vmod_LTLIBRARIES = libvmod_unix.la @@ -30,7 +30,7 @@ $(libvmod_unix_la_OBJECTS): vcc_unix_if.h vcc_unix_if.h vmod_unix.rst vmod_unix.man.rst: vcc_unix_if.c vcc_unix_if.c: $(vmodtool) $(srcdir)/vmod_unix.vcc - @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_unix.vcc + @PYTHON@ $(vmodtool) $(vmodtoolargs_unix) $(srcdir)/vmod_unix.vcc EXTRA_DIST = $(srcdir)/vmod_unix.vcc automake_boilerplate_unix.am diff --git a/lib/libvmod_vtc/automake_boilerplate_vtc.am b/lib/libvmod_vtc/automake_boilerplate_vtc.am index 53579929b..620fd9430 100644 --- a/lib/libvmod_vtc/automake_boilerplate_vtc.am +++ b/lib/libvmod_vtc/automake_boilerplate_vtc.am @@ -8,7 +8,7 @@ AM_CPPFLAGS = \ -I$(top_builddir)/include vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs ?= --strict --boilerplate -o vcc_vtc_if +vmodtoolargs_vtc ?= --strict --boilerplate -o vcc_vtc_if vmod_LTLIBRARIES = libvmod_vtc.la @@ -30,7 +30,7 @@ $(libvmod_vtc_la_OBJECTS): vcc_vtc_if.h vcc_vtc_if.h vmod_vtc.rst vmod_vtc.man.rst: vcc_vtc_if.c vcc_vtc_if.c: $(vmodtool) $(srcdir)/vmod_vtc.vcc - @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_vtc.vcc + @PYTHON@ $(vmodtool) $(vmodtoolargs_vtc) $(srcdir)/vmod_vtc.vcc EXTRA_DIST = $(srcdir)/vmod_vtc.vcc automake_boilerplate_vtc.am From dridi.boukelmoune at gmail.com Thu Jan 7 14:16:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 7 Jan 2021 14:16:07 +0000 (UTC) Subject: [master] 3cb906ae0 build: Define $(vmodtool) at configure time Message-ID: <20210107141607.73142A9255@lists.varnish-cache.org> commit 3cb906ae00ec14eed0a321d44fbd2a04550ac150 Author: Dridi Boukelmoune Date: Tue Jan 5 17:41:15 2021 +0100 build: Define $(vmodtool) at configure time Incidentally, defining the variable in the boilerplate makes the boilerplate unusable out of tree since it assumes vmodtool is in $(srcdir) which is a show stopper for third party VMODs. diff --git a/configure.ac b/configure.ac index 2417549ac..53ba873d0 100644 --- a/configure.ac +++ b/configure.ac @@ -679,8 +679,9 @@ AC_SUBST(VARNISH_STATE_DIR) pkgsysconfdir='${sysconfdir}/varnish' AC_SUBST(pkgsysconfdir) -# VMODs directory +# VMOD variables AC_SUBST(vmoddir, [$\(pkglibdir\)/vmods]) +AC_SUBST(vmodtool, [$\(top_srcdir\)/lib/libvcc/vmodtool.py]) # Check for linker script support gl_LD_VERSION_SCRIPT diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index b84f3464b..12f5ce25a 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -58,14 +58,12 @@ AM_CPPFLAGS = \\ \t-I$(top_srcdir)/bin/varnishd \\ \t-I$(top_builddir)/include -vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs_XXX ?= --strict --boilerplate -o PFX - vmod_LTLIBRARIES = libvmod_XXX.la libvmod_XXX_la_CFLAGS = \\ \t at SAN_CFLAGS@ +vmodtoolargs_XXX ?= --strict --boilerplate -o PFX vmod_XXX_symbols_regex ?= Vmod_XXX_Data libvmod_XXX_la_LDFLAGS = \\ diff --git a/lib/libvmod_blob/automake_boilerplate_blob.am b/lib/libvmod_blob/automake_boilerplate_blob.am index 86c9a71a9..6bacd717f 100644 --- a/lib/libvmod_blob/automake_boilerplate_blob.am +++ b/lib/libvmod_blob/automake_boilerplate_blob.am @@ -7,14 +7,12 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/bin/varnishd \ -I$(top_builddir)/include -vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs_blob ?= --strict --boilerplate -o vcc_blob_if - vmod_LTLIBRARIES = libvmod_blob.la libvmod_blob_la_CFLAGS = \ @SAN_CFLAGS@ +vmodtoolargs_blob ?= --strict --boilerplate -o vcc_blob_if vmod_blob_symbols_regex ?= Vmod_blob_Data libvmod_blob_la_LDFLAGS = \ diff --git a/lib/libvmod_cookie/automake_boilerplate_cookie.am b/lib/libvmod_cookie/automake_boilerplate_cookie.am index 007a19199..1129f2aea 100644 --- a/lib/libvmod_cookie/automake_boilerplate_cookie.am +++ b/lib/libvmod_cookie/automake_boilerplate_cookie.am @@ -7,14 +7,12 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/bin/varnishd \ -I$(top_builddir)/include -vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs_cookie ?= --strict --boilerplate -o vcc_cookie_if - vmod_LTLIBRARIES = libvmod_cookie.la libvmod_cookie_la_CFLAGS = \ @SAN_CFLAGS@ +vmodtoolargs_cookie ?= --strict --boilerplate -o vcc_cookie_if vmod_cookie_symbols_regex ?= Vmod_cookie_Data libvmod_cookie_la_LDFLAGS = \ diff --git a/lib/libvmod_debug/automake_boilerplate_debug.am b/lib/libvmod_debug/automake_boilerplate_debug.am index e589a1c76..619ebcef4 100644 --- a/lib/libvmod_debug/automake_boilerplate_debug.am +++ b/lib/libvmod_debug/automake_boilerplate_debug.am @@ -7,14 +7,12 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/bin/varnishd \ -I$(top_builddir)/include -vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs_debug ?= --strict --boilerplate -o vcc_debug_if - vmod_LTLIBRARIES = libvmod_debug.la libvmod_debug_la_CFLAGS = \ @SAN_CFLAGS@ +vmodtoolargs_debug ?= --strict --boilerplate -o vcc_debug_if vmod_debug_symbols_regex ?= Vmod_debug_Data libvmod_debug_la_LDFLAGS = \ diff --git a/lib/libvmod_directors/automake_boilerplate_directors.am b/lib/libvmod_directors/automake_boilerplate_directors.am index 2c343a523..7f52f7dca 100644 --- a/lib/libvmod_directors/automake_boilerplate_directors.am +++ b/lib/libvmod_directors/automake_boilerplate_directors.am @@ -7,14 +7,12 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/bin/varnishd \ -I$(top_builddir)/include -vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs_directors ?= --strict --boilerplate -o vcc_directors_if - vmod_LTLIBRARIES = libvmod_directors.la libvmod_directors_la_CFLAGS = \ @SAN_CFLAGS@ +vmodtoolargs_directors ?= --strict --boilerplate -o vcc_directors_if vmod_directors_symbols_regex ?= Vmod_directors_Data libvmod_directors_la_LDFLAGS = \ diff --git a/lib/libvmod_proxy/automake_boilerplate_proxy.am b/lib/libvmod_proxy/automake_boilerplate_proxy.am index 066eb68dd..bd3802fd3 100644 --- a/lib/libvmod_proxy/automake_boilerplate_proxy.am +++ b/lib/libvmod_proxy/automake_boilerplate_proxy.am @@ -7,14 +7,12 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/bin/varnishd \ -I$(top_builddir)/include -vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs_proxy ?= --strict --boilerplate -o vcc_proxy_if - vmod_LTLIBRARIES = libvmod_proxy.la libvmod_proxy_la_CFLAGS = \ @SAN_CFLAGS@ +vmodtoolargs_proxy ?= --strict --boilerplate -o vcc_proxy_if vmod_proxy_symbols_regex ?= Vmod_proxy_Data libvmod_proxy_la_LDFLAGS = \ diff --git a/lib/libvmod_purge/automake_boilerplate_purge.am b/lib/libvmod_purge/automake_boilerplate_purge.am index 89aeabb4a..e3ec8bf65 100644 --- a/lib/libvmod_purge/automake_boilerplate_purge.am +++ b/lib/libvmod_purge/automake_boilerplate_purge.am @@ -7,14 +7,12 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/bin/varnishd \ -I$(top_builddir)/include -vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs_purge ?= --strict --boilerplate -o vcc_purge_if - vmod_LTLIBRARIES = libvmod_purge.la libvmod_purge_la_CFLAGS = \ @SAN_CFLAGS@ +vmodtoolargs_purge ?= --strict --boilerplate -o vcc_purge_if vmod_purge_symbols_regex ?= Vmod_purge_Data libvmod_purge_la_LDFLAGS = \ diff --git a/lib/libvmod_std/automake_boilerplate_std.am b/lib/libvmod_std/automake_boilerplate_std.am index 57dec41a3..f52815531 100644 --- a/lib/libvmod_std/automake_boilerplate_std.am +++ b/lib/libvmod_std/automake_boilerplate_std.am @@ -7,14 +7,12 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/bin/varnishd \ -I$(top_builddir)/include -vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs_std ?= --strict --boilerplate -o vcc_std_if - vmod_LTLIBRARIES = libvmod_std.la libvmod_std_la_CFLAGS = \ @SAN_CFLAGS@ +vmodtoolargs_std ?= --strict --boilerplate -o vcc_std_if vmod_std_symbols_regex ?= Vmod_std_Data libvmod_std_la_LDFLAGS = \ diff --git a/lib/libvmod_unix/automake_boilerplate_unix.am b/lib/libvmod_unix/automake_boilerplate_unix.am index 02cd56b2a..e0b5eca25 100644 --- a/lib/libvmod_unix/automake_boilerplate_unix.am +++ b/lib/libvmod_unix/automake_boilerplate_unix.am @@ -7,14 +7,12 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/bin/varnishd \ -I$(top_builddir)/include -vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs_unix ?= --strict --boilerplate -o vcc_unix_if - vmod_LTLIBRARIES = libvmod_unix.la libvmod_unix_la_CFLAGS = \ @SAN_CFLAGS@ +vmodtoolargs_unix ?= --strict --boilerplate -o vcc_unix_if vmod_unix_symbols_regex ?= Vmod_unix_Data libvmod_unix_la_LDFLAGS = \ diff --git a/lib/libvmod_vtc/automake_boilerplate_vtc.am b/lib/libvmod_vtc/automake_boilerplate_vtc.am index 620fd9430..7b883cb02 100644 --- a/lib/libvmod_vtc/automake_boilerplate_vtc.am +++ b/lib/libvmod_vtc/automake_boilerplate_vtc.am @@ -7,14 +7,12 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/bin/varnishd \ -I$(top_builddir)/include -vmodtool = $(top_srcdir)/lib/libvcc/vmodtool.py -vmodtoolargs_vtc ?= --strict --boilerplate -o vcc_vtc_if - vmod_LTLIBRARIES = libvmod_vtc.la libvmod_vtc_la_CFLAGS = \ @SAN_CFLAGS@ +vmodtoolargs_vtc ?= --strict --boilerplate -o vcc_vtc_if vmod_vtc_symbols_regex ?= Vmod_vtc_Data libvmod_vtc_la_LDFLAGS = \ From dridi.boukelmoune at gmail.com Thu Jan 7 14:16:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 7 Jan 2021 14:16:07 +0000 (UTC) Subject: [master] 1e6e2fc97 build: Reintroduce some VMOD boilerplate everywhere Message-ID: <20210107141607.8E528A9261@lists.varnish-cache.org> commit 1e6e2fc97fff79303a6553d717cf18c6bbbad78d Author: Dridi Boukelmoune Date: Tue Jan 5 18:20:28 2021 +0100 build: Reintroduce some VMOD boilerplate everywhere Anything left that would prevent two VMODs to live in the same $(srcdir) because it would be defined more than once. diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 12f5ce25a..4735d4322 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -51,14 +51,7 @@ import time AMBOILERPLATE = '''\ # Generated by vmodtool.py --boilerplate. -AM_LDFLAGS = $(AM_LT_LDFLAGS) - -AM_CPPFLAGS = \\ -\t-I$(top_srcdir)/include \\ -\t-I$(top_srcdir)/bin/varnishd \\ -\t-I$(top_builddir)/include - -vmod_LTLIBRARIES = libvmod_XXX.la +vmod_LTLIBRARIES += libvmod_XXX.la libvmod_XXX_la_CFLAGS = \\ \t at SAN_CFLAGS@ @@ -81,9 +74,9 @@ PFX.h vmod_XXX.rst vmod_XXX.man.rst: PFX.c PFX.c: $(vmodtool) $(srcdir)/VCC \t at PYTHON@ $(vmodtool) $(vmodtoolargs_XXX) $(srcdir)/VCC -EXTRA_DIST = $(srcdir)/VCC automake_boilerplate_XXX.am +EXTRA_DIST += $(srcdir)/VCC automake_boilerplate_XXX.am -CLEANFILES = $(builddir)/PFX.c $(builddir)/PFX.h \\ +CLEANFILES += $(builddir)/PFX.c $(builddir)/PFX.h \\ \t$(builddir)/vmod_XXX.rst \\ \t$(builddir)/vmod_XXX.man.rst ''' diff --git a/lib/libvmod_blob/Makefile.am b/lib/libvmod_blob/Makefile.am index 2ca6384e9..6015d15ca 100644 --- a/lib/libvmod_blob/Makefile.am +++ b/lib/libvmod_blob/Makefile.am @@ -1,5 +1,16 @@ # +AM_LDFLAGS = $(AM_LT_LDFLAGS) + +AM_CPPFLAGS = \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/bin/varnishd \ + -I$(top_builddir)/include + +vmod_LTLIBRARIES = +EXTRA_DIST = +CLEANFILES = + libvmod_blob_la_SOURCES = \ vmod_blob.c \ vmod_blob.h \ diff --git a/lib/libvmod_blob/automake_boilerplate_blob.am b/lib/libvmod_blob/automake_boilerplate_blob.am index 6bacd717f..e0e88e4ff 100644 --- a/lib/libvmod_blob/automake_boilerplate_blob.am +++ b/lib/libvmod_blob/automake_boilerplate_blob.am @@ -1,13 +1,6 @@ # Generated by vmodtool.py --boilerplate. -AM_LDFLAGS = $(AM_LT_LDFLAGS) - -AM_CPPFLAGS = \ - -I$(top_srcdir)/include \ - -I$(top_srcdir)/bin/varnishd \ - -I$(top_builddir)/include - -vmod_LTLIBRARIES = libvmod_blob.la +vmod_LTLIBRARIES += libvmod_blob.la libvmod_blob_la_CFLAGS = \ @SAN_CFLAGS@ @@ -30,8 +23,8 @@ vcc_blob_if.h vmod_blob.rst vmod_blob.man.rst: vcc_blob_if.c vcc_blob_if.c: $(vmodtool) $(srcdir)/vmod_blob.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs_blob) $(srcdir)/vmod_blob.vcc -EXTRA_DIST = $(srcdir)/vmod_blob.vcc automake_boilerplate_blob.am +EXTRA_DIST += $(srcdir)/vmod_blob.vcc automake_boilerplate_blob.am -CLEANFILES = $(builddir)/vcc_blob_if.c $(builddir)/vcc_blob_if.h \ +CLEANFILES += $(builddir)/vcc_blob_if.c $(builddir)/vcc_blob_if.h \ $(builddir)/vmod_blob.rst \ $(builddir)/vmod_blob.man.rst diff --git a/lib/libvmod_cookie/Makefile.am b/lib/libvmod_cookie/Makefile.am index b85531aed..fe762a186 100644 --- a/lib/libvmod_cookie/Makefile.am +++ b/lib/libvmod_cookie/Makefile.am @@ -1,3 +1,14 @@ +AM_LDFLAGS = $(AM_LT_LDFLAGS) + +AM_CPPFLAGS = \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/bin/varnishd \ + -I$(top_builddir)/include + +vmod_LTLIBRARIES = +EXTRA_DIST = +CLEANFILES = + libvmod_cookie_la_SOURCES = \ vmod_cookie.c diff --git a/lib/libvmod_cookie/automake_boilerplate_cookie.am b/lib/libvmod_cookie/automake_boilerplate_cookie.am index 1129f2aea..9c2f4822d 100644 --- a/lib/libvmod_cookie/automake_boilerplate_cookie.am +++ b/lib/libvmod_cookie/automake_boilerplate_cookie.am @@ -1,13 +1,6 @@ # Generated by vmodtool.py --boilerplate. -AM_LDFLAGS = $(AM_LT_LDFLAGS) - -AM_CPPFLAGS = \ - -I$(top_srcdir)/include \ - -I$(top_srcdir)/bin/varnishd \ - -I$(top_builddir)/include - -vmod_LTLIBRARIES = libvmod_cookie.la +vmod_LTLIBRARIES += libvmod_cookie.la libvmod_cookie_la_CFLAGS = \ @SAN_CFLAGS@ @@ -30,8 +23,8 @@ vcc_cookie_if.h vmod_cookie.rst vmod_cookie.man.rst: vcc_cookie_if.c vcc_cookie_if.c: $(vmodtool) $(srcdir)/vmod_cookie.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs_cookie) $(srcdir)/vmod_cookie.vcc -EXTRA_DIST = $(srcdir)/vmod_cookie.vcc automake_boilerplate_cookie.am +EXTRA_DIST += $(srcdir)/vmod_cookie.vcc automake_boilerplate_cookie.am -CLEANFILES = $(builddir)/vcc_cookie_if.c $(builddir)/vcc_cookie_if.h \ +CLEANFILES += $(builddir)/vcc_cookie_if.c $(builddir)/vcc_cookie_if.h \ $(builddir)/vmod_cookie.rst \ $(builddir)/vmod_cookie.man.rst diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am index bcd29af6f..ccb866ba1 100644 --- a/lib/libvmod_debug/Makefile.am +++ b/lib/libvmod_debug/Makefile.am @@ -1,5 +1,16 @@ # +AM_LDFLAGS = $(AM_LT_LDFLAGS) + +AM_CPPFLAGS = \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/bin/varnishd \ + -I$(top_builddir)/include + +vmod_LTLIBRARIES = +EXTRA_DIST = +CLEANFILES = + libvmod_debug_la_SOURCES = \ vmod_debug.c \ vmod_debug_obj.c \ diff --git a/lib/libvmod_debug/automake_boilerplate_debug.am b/lib/libvmod_debug/automake_boilerplate_debug.am index 619ebcef4..c3f2a4a46 100644 --- a/lib/libvmod_debug/automake_boilerplate_debug.am +++ b/lib/libvmod_debug/automake_boilerplate_debug.am @@ -1,13 +1,6 @@ # Generated by vmodtool.py --boilerplate. -AM_LDFLAGS = $(AM_LT_LDFLAGS) - -AM_CPPFLAGS = \ - -I$(top_srcdir)/include \ - -I$(top_srcdir)/bin/varnishd \ - -I$(top_builddir)/include - -vmod_LTLIBRARIES = libvmod_debug.la +vmod_LTLIBRARIES += libvmod_debug.la libvmod_debug_la_CFLAGS = \ @SAN_CFLAGS@ @@ -30,8 +23,8 @@ vcc_debug_if.h vmod_debug.rst vmod_debug.man.rst: vcc_debug_if.c vcc_debug_if.c: $(vmodtool) $(srcdir)/vmod_debug.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs_debug) $(srcdir)/vmod_debug.vcc -EXTRA_DIST = $(srcdir)/vmod_debug.vcc automake_boilerplate_debug.am +EXTRA_DIST += $(srcdir)/vmod_debug.vcc automake_boilerplate_debug.am -CLEANFILES = $(builddir)/vcc_debug_if.c $(builddir)/vcc_debug_if.h \ +CLEANFILES += $(builddir)/vcc_debug_if.c $(builddir)/vcc_debug_if.h \ $(builddir)/vmod_debug.rst \ $(builddir)/vmod_debug.man.rst diff --git a/lib/libvmod_directors/Makefile.am b/lib/libvmod_directors/Makefile.am index f9dc599c0..c461639bc 100644 --- a/lib/libvmod_directors/Makefile.am +++ b/lib/libvmod_directors/Makefile.am @@ -1,5 +1,16 @@ # +AM_LDFLAGS = $(AM_LT_LDFLAGS) + +AM_CPPFLAGS = \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/bin/varnishd \ + -I$(top_builddir)/include + +vmod_LTLIBRARIES = +EXTRA_DIST = +CLEANFILES = + libvmod_directors_la_SOURCES = \ vdir.c \ vdir.h \ diff --git a/lib/libvmod_directors/automake_boilerplate_directors.am b/lib/libvmod_directors/automake_boilerplate_directors.am index 7f52f7dca..0f6869805 100644 --- a/lib/libvmod_directors/automake_boilerplate_directors.am +++ b/lib/libvmod_directors/automake_boilerplate_directors.am @@ -1,13 +1,6 @@ # Generated by vmodtool.py --boilerplate. -AM_LDFLAGS = $(AM_LT_LDFLAGS) - -AM_CPPFLAGS = \ - -I$(top_srcdir)/include \ - -I$(top_srcdir)/bin/varnishd \ - -I$(top_builddir)/include - -vmod_LTLIBRARIES = libvmod_directors.la +vmod_LTLIBRARIES += libvmod_directors.la libvmod_directors_la_CFLAGS = \ @SAN_CFLAGS@ @@ -30,8 +23,8 @@ vcc_directors_if.h vmod_directors.rst vmod_directors.man.rst: vcc_directors_if.c vcc_directors_if.c: $(vmodtool) $(srcdir)/vmod_directors.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs_directors) $(srcdir)/vmod_directors.vcc -EXTRA_DIST = $(srcdir)/vmod_directors.vcc automake_boilerplate_directors.am +EXTRA_DIST += $(srcdir)/vmod_directors.vcc automake_boilerplate_directors.am -CLEANFILES = $(builddir)/vcc_directors_if.c $(builddir)/vcc_directors_if.h \ +CLEANFILES += $(builddir)/vcc_directors_if.c $(builddir)/vcc_directors_if.h \ $(builddir)/vmod_directors.rst \ $(builddir)/vmod_directors.man.rst diff --git a/lib/libvmod_proxy/Makefile.am b/lib/libvmod_proxy/Makefile.am index c98a060f8..c96d8c8d1 100644 --- a/lib/libvmod_proxy/Makefile.am +++ b/lib/libvmod_proxy/Makefile.am @@ -1,5 +1,16 @@ # +AM_LDFLAGS = $(AM_LT_LDFLAGS) + +AM_CPPFLAGS = \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/bin/varnishd \ + -I$(top_builddir)/include + +vmod_LTLIBRARIES = +EXTRA_DIST = +CLEANFILES = + libvmod_proxy_la_SOURCES = \ vmod_proxy.c diff --git a/lib/libvmod_proxy/automake_boilerplate_proxy.am b/lib/libvmod_proxy/automake_boilerplate_proxy.am index bd3802fd3..9e1ae138a 100644 --- a/lib/libvmod_proxy/automake_boilerplate_proxy.am +++ b/lib/libvmod_proxy/automake_boilerplate_proxy.am @@ -1,13 +1,6 @@ # Generated by vmodtool.py --boilerplate. -AM_LDFLAGS = $(AM_LT_LDFLAGS) - -AM_CPPFLAGS = \ - -I$(top_srcdir)/include \ - -I$(top_srcdir)/bin/varnishd \ - -I$(top_builddir)/include - -vmod_LTLIBRARIES = libvmod_proxy.la +vmod_LTLIBRARIES += libvmod_proxy.la libvmod_proxy_la_CFLAGS = \ @SAN_CFLAGS@ @@ -30,8 +23,8 @@ vcc_proxy_if.h vmod_proxy.rst vmod_proxy.man.rst: vcc_proxy_if.c vcc_proxy_if.c: $(vmodtool) $(srcdir)/vmod_proxy.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs_proxy) $(srcdir)/vmod_proxy.vcc -EXTRA_DIST = $(srcdir)/vmod_proxy.vcc automake_boilerplate_proxy.am +EXTRA_DIST += $(srcdir)/vmod_proxy.vcc automake_boilerplate_proxy.am -CLEANFILES = $(builddir)/vcc_proxy_if.c $(builddir)/vcc_proxy_if.h \ +CLEANFILES += $(builddir)/vcc_proxy_if.c $(builddir)/vcc_proxy_if.h \ $(builddir)/vmod_proxy.rst \ $(builddir)/vmod_proxy.man.rst diff --git a/lib/libvmod_purge/Makefile.am b/lib/libvmod_purge/Makefile.am index 7899bb056..8af2b1475 100644 --- a/lib/libvmod_purge/Makefile.am +++ b/lib/libvmod_purge/Makefile.am @@ -1,5 +1,16 @@ # +AM_LDFLAGS = $(AM_LT_LDFLAGS) + +AM_CPPFLAGS = \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/bin/varnishd \ + -I$(top_builddir)/include + +vmod_LTLIBRARIES = +EXTRA_DIST = +CLEANFILES = + libvmod_purge_la_SOURCES = \ vmod_purge.c diff --git a/lib/libvmod_purge/automake_boilerplate_purge.am b/lib/libvmod_purge/automake_boilerplate_purge.am index e3ec8bf65..3256cd271 100644 --- a/lib/libvmod_purge/automake_boilerplate_purge.am +++ b/lib/libvmod_purge/automake_boilerplate_purge.am @@ -1,13 +1,6 @@ # Generated by vmodtool.py --boilerplate. -AM_LDFLAGS = $(AM_LT_LDFLAGS) - -AM_CPPFLAGS = \ - -I$(top_srcdir)/include \ - -I$(top_srcdir)/bin/varnishd \ - -I$(top_builddir)/include - -vmod_LTLIBRARIES = libvmod_purge.la +vmod_LTLIBRARIES += libvmod_purge.la libvmod_purge_la_CFLAGS = \ @SAN_CFLAGS@ @@ -30,8 +23,8 @@ vcc_purge_if.h vmod_purge.rst vmod_purge.man.rst: vcc_purge_if.c vcc_purge_if.c: $(vmodtool) $(srcdir)/vmod_purge.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs_purge) $(srcdir)/vmod_purge.vcc -EXTRA_DIST = $(srcdir)/vmod_purge.vcc automake_boilerplate_purge.am +EXTRA_DIST += $(srcdir)/vmod_purge.vcc automake_boilerplate_purge.am -CLEANFILES = $(builddir)/vcc_purge_if.c $(builddir)/vcc_purge_if.h \ +CLEANFILES += $(builddir)/vcc_purge_if.c $(builddir)/vcc_purge_if.h \ $(builddir)/vmod_purge.rst \ $(builddir)/vmod_purge.man.rst diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am index 9126fd613..bb922ea97 100644 --- a/lib/libvmod_std/Makefile.am +++ b/lib/libvmod_std/Makefile.am @@ -1,5 +1,16 @@ # +AM_LDFLAGS = $(AM_LT_LDFLAGS) + +AM_CPPFLAGS = \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/bin/varnishd \ + -I$(top_builddir)/include + +vmod_LTLIBRARIES = +EXTRA_DIST = +CLEANFILES = + libvmod_std_la_SOURCES = \ vmod_std.c \ vmod_std_conversions.c \ diff --git a/lib/libvmod_std/automake_boilerplate_std.am b/lib/libvmod_std/automake_boilerplate_std.am index f52815531..777280cf2 100644 --- a/lib/libvmod_std/automake_boilerplate_std.am +++ b/lib/libvmod_std/automake_boilerplate_std.am @@ -1,13 +1,6 @@ # Generated by vmodtool.py --boilerplate. -AM_LDFLAGS = $(AM_LT_LDFLAGS) - -AM_CPPFLAGS = \ - -I$(top_srcdir)/include \ - -I$(top_srcdir)/bin/varnishd \ - -I$(top_builddir)/include - -vmod_LTLIBRARIES = libvmod_std.la +vmod_LTLIBRARIES += libvmod_std.la libvmod_std_la_CFLAGS = \ @SAN_CFLAGS@ @@ -30,8 +23,8 @@ vcc_std_if.h vmod_std.rst vmod_std.man.rst: vcc_std_if.c vcc_std_if.c: $(vmodtool) $(srcdir)/vmod_std.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs_std) $(srcdir)/vmod_std.vcc -EXTRA_DIST = $(srcdir)/vmod_std.vcc automake_boilerplate_std.am +EXTRA_DIST += $(srcdir)/vmod_std.vcc automake_boilerplate_std.am -CLEANFILES = $(builddir)/vcc_std_if.c $(builddir)/vcc_std_if.h \ +CLEANFILES += $(builddir)/vcc_std_if.c $(builddir)/vcc_std_if.h \ $(builddir)/vmod_std.rst \ $(builddir)/vmod_std.man.rst diff --git a/lib/libvmod_unix/Makefile.am b/lib/libvmod_unix/Makefile.am index c68108aef..db4a1cec5 100644 --- a/lib/libvmod_unix/Makefile.am +++ b/lib/libvmod_unix/Makefile.am @@ -1,5 +1,16 @@ # +AM_LDFLAGS = $(AM_LT_LDFLAGS) + +AM_CPPFLAGS = \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/bin/varnishd \ + -I$(top_builddir)/include + +vmod_LTLIBRARIES = +EXTRA_DIST = +CLEANFILES = + libvmod_unix_la_SOURCES = \ vmod_unix.c \ cred_compat.h diff --git a/lib/libvmod_unix/automake_boilerplate_unix.am b/lib/libvmod_unix/automake_boilerplate_unix.am index e0b5eca25..6281665b2 100644 --- a/lib/libvmod_unix/automake_boilerplate_unix.am +++ b/lib/libvmod_unix/automake_boilerplate_unix.am @@ -1,13 +1,6 @@ # Generated by vmodtool.py --boilerplate. -AM_LDFLAGS = $(AM_LT_LDFLAGS) - -AM_CPPFLAGS = \ - -I$(top_srcdir)/include \ - -I$(top_srcdir)/bin/varnishd \ - -I$(top_builddir)/include - -vmod_LTLIBRARIES = libvmod_unix.la +vmod_LTLIBRARIES += libvmod_unix.la libvmod_unix_la_CFLAGS = \ @SAN_CFLAGS@ @@ -30,8 +23,8 @@ vcc_unix_if.h vmod_unix.rst vmod_unix.man.rst: vcc_unix_if.c vcc_unix_if.c: $(vmodtool) $(srcdir)/vmod_unix.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs_unix) $(srcdir)/vmod_unix.vcc -EXTRA_DIST = $(srcdir)/vmod_unix.vcc automake_boilerplate_unix.am +EXTRA_DIST += $(srcdir)/vmod_unix.vcc automake_boilerplate_unix.am -CLEANFILES = $(builddir)/vcc_unix_if.c $(builddir)/vcc_unix_if.h \ +CLEANFILES += $(builddir)/vcc_unix_if.c $(builddir)/vcc_unix_if.h \ $(builddir)/vmod_unix.rst \ $(builddir)/vmod_unix.man.rst diff --git a/lib/libvmod_vtc/Makefile.am b/lib/libvmod_vtc/Makefile.am index 557156f6e..32fb0aaa6 100644 --- a/lib/libvmod_vtc/Makefile.am +++ b/lib/libvmod_vtc/Makefile.am @@ -1,5 +1,16 @@ # +AM_LDFLAGS = $(AM_LT_LDFLAGS) + +AM_CPPFLAGS = \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/bin/varnishd \ + -I$(top_builddir)/include + +vmod_LTLIBRARIES = +EXTRA_DIST = +CLEANFILES = + libvmod_vtc_la_SOURCES = \ vmod_vtc.c diff --git a/lib/libvmod_vtc/automake_boilerplate_vtc.am b/lib/libvmod_vtc/automake_boilerplate_vtc.am index 7b883cb02..9b71914b0 100644 --- a/lib/libvmod_vtc/automake_boilerplate_vtc.am +++ b/lib/libvmod_vtc/automake_boilerplate_vtc.am @@ -1,13 +1,6 @@ # Generated by vmodtool.py --boilerplate. -AM_LDFLAGS = $(AM_LT_LDFLAGS) - -AM_CPPFLAGS = \ - -I$(top_srcdir)/include \ - -I$(top_srcdir)/bin/varnishd \ - -I$(top_builddir)/include - -vmod_LTLIBRARIES = libvmod_vtc.la +vmod_LTLIBRARIES += libvmod_vtc.la libvmod_vtc_la_CFLAGS = \ @SAN_CFLAGS@ @@ -30,8 +23,8 @@ vcc_vtc_if.h vmod_vtc.rst vmod_vtc.man.rst: vcc_vtc_if.c vcc_vtc_if.c: $(vmodtool) $(srcdir)/vmod_vtc.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs_vtc) $(srcdir)/vmod_vtc.vcc -EXTRA_DIST = $(srcdir)/vmod_vtc.vcc automake_boilerplate_vtc.am +EXTRA_DIST += $(srcdir)/vmod_vtc.vcc automake_boilerplate_vtc.am -CLEANFILES = $(builddir)/vcc_vtc_if.c $(builddir)/vcc_vtc_if.h \ +CLEANFILES += $(builddir)/vcc_vtc_if.c $(builddir)/vcc_vtc_if.h \ $(builddir)/vmod_vtc.rst \ $(builddir)/vmod_vtc.man.rst From dridi.boukelmoune at gmail.com Thu Jan 7 14:16:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 7 Jan 2021 14:16:07 +0000 (UTC) Subject: [master] 3c64cf396 build: Move all VMODs to $(top_srcdir)/vmod/ Message-ID: <20210107141607.C11B6A9279@lists.varnish-cache.org> commit 3c64cf3964ce0d62f201415b7358d211b593895d Author: Dridi Boukelmoune Date: Tue Jan 5 18:29:53 2021 +0100 build: Move all VMODs to $(top_srcdir)/vmod/ This will improve the build parallelism, and will allow us to run the VMOD test suite after the main one. diff --git a/.gitignore b/.gitignore index 0c5a0624e..7dba66a9f 100644 --- a/.gitignore +++ b/.gitignore @@ -72,11 +72,11 @@ cscope.*out /bin/varnishd/VSC_*.h # Misc. generated files for included vmods. -/lib/libvmod_*/VSC_*.c -/lib/libvmod_*/VSC_*.h -/lib/libvmod_*/vcc_*_if.c -/lib/libvmod_*/vcc_*_if.h -/lib/libvmod_*/vmod_*.rst +/vmod/VSC_*.c +/vmod/VSC_*.h +/vmod/vcc_*_if.c +/vmod/vcc_*_if.h +/vmod/vmod_*.rst # Man-files and binaries /man/*.1 diff --git a/Makefile.am b/Makefile.am index f0e132389..843b67f09 100644 --- a/Makefile.am +++ b/Makefile.am @@ -1,6 +1,6 @@ ACLOCAL_AMFLAGS = -I m4 -I . -SUBDIRS = include lib bin etc doc man +SUBDIRS = include lib bin vmod etc doc man pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = varnishapi.pc diff --git a/bin/varnishd/flint.sh b/bin/varnishd/flint.sh index af06e7568..2442255bc 100755 --- a/bin/varnishd/flint.sh +++ b/bin/varnishd/flint.sh @@ -19,20 +19,8 @@ FLOPS=' ../../lib/libvarnish/*.c ../../lib/libvcc/flint.lnt ../../lib/libvcc/*.c - ../../lib/libvmod_blob/flint.lnt - ../../lib/libvmod_blob/*.c - ../../lib/libvmod_debug/flint.lnt - ../../lib/libvmod_debug/*.c - ../../lib/libvmod_directors/flint.lnt - ../../lib/libvmod_directors/*.c - ../../lib/libvmod_proxy/flint.lnt - ../../lib/libvmod_proxy/*.c - ../../lib/libvmod_purge/flint.lnt - ../../lib/libvmod_purge/*.c - ../../lib/libvmod_std/flint.lnt - ../../lib/libvmod_std/*.c - ../../lib/libvmod_vtc/flint.lnt - ../../lib/libvmod_vtc/*.c + ../../vmod/flint.lnt + ../../vmod/*.c ' . ../../tools/flint_skel.sh diff --git a/bin/varnishd/vclflint.sh b/bin/varnishd/vclflint.sh index c390719d9..742265144 100755 --- a/bin/varnishd/vclflint.sh +++ b/bin/varnishd/vclflint.sh @@ -1,7 +1,7 @@ #!/bin/sh # # Run flexelint on the VCL output -LIBS="-p vmod_path=/home/phk/Varnish/trunk/varnish-cache/lib/libvmod_std/.libs:/home/phk/Varnish/trunk/varnish-cache/lib/libvmod_debug/.libs:/home/phk/Varnish/trunk/varnish-cache/lib/libvmod_directors/.libs:/home/phk/Varnish/trunk/varnish-cache/lib/libvmod_purge/.libs:/home/phk/Varnish/trunk/varnish-cache/lib/libvmod_vtc/.libs:/home/phk/Varnish/trunk/varnish-cache/lib/libvmod_blob/.libs:/home/phk/Varnish/trunk/varnish-cache/lib/libvmod_unix/.libs:/home/phk/Varnish/trunk/varnish-cache/lib/libvmod_proxy/.libs" +LIBS="-p vmod_path=/home/phk/Varnish/trunk/varnish-cache/vmod/.libs" if [ "x$1" = "x" ] ; then ./varnishd $LIBS -C -b localhost > /tmp/_.c diff --git a/bin/varnishtest/tests/m00003.vtc b/bin/varnishtest/tests/m00003.vtc index d6e8b571e..c2b00e45e 100644 --- a/bin/varnishtest/tests/m00003.vtc +++ b/bin/varnishtest/tests/m00003.vtc @@ -9,11 +9,11 @@ server s1 { shell { echo "vcl 4.1; import std; backend dummy None;" >${tmpdir}/test.vcl - varnishd -pvmod_path=${topbuild}/lib/libvmod_std/.libs \ + varnishd -pvmod_path=${topbuild}/vmod/.libs \ -C -f ${tmpdir}/test.vcl 2>/dev/null } -varnish v1 -arg "-pvmod_path=${topbuild}/lib/libvmod_std/.libs/" -vcl+backend { +varnish v1 -arg "-pvmod_path=${topbuild}/vmod/.libs/" -vcl+backend { import std; } -start @@ -23,7 +23,7 @@ varnish v1 -errvcl {Could not find VMOD std} { import std; } -varnish v1 -cliok "param.set vmod_path ${topbuild}/lib/libvmod_std/.libs/" +varnish v1 -cliok "param.set vmod_path ${topbuild}/vmod/.libs/" varnish v1 -vcl+backend { import std; @@ -38,7 +38,7 @@ varnish v1 -errvcl {Could not open VMOD wrong} { } shell { - cp ${topbuild}/lib/libvmod_debug/.libs/libvmod_debug.so \ + cp ${topbuild}/vmod/.libs/libvmod_debug.so \ ${tmpdir}/libvmod_wrong.so } diff --git a/bin/varnishtest/tests/m00008.vtc b/bin/varnishtest/tests/m00008.vtc index 77b327370..bb0568cd1 100644 --- a/bin/varnishtest/tests/m00008.vtc +++ b/bin/varnishtest/tests/m00008.vtc @@ -30,11 +30,11 @@ varnish v1 -cliok "param.set vcc_unsafe_path off" varnish v1 -errvcl {'import ... from path ...' is unsafe.} { backend default { .host = "${s1_sock}"; } - import std from "${topbuild}/lib/libvmod_std/.libs/"; + import std from "${topbuild}/vmod/.libs/"; } varnish v1 \ - -cliok "param.set vmod_path /nowhere:${topbuild}/lib/libvmod_std/.libs/" + -cliok "param.set vmod_path /nowhere:${topbuild}/vmod/.libs/" varnish v1 -vcl+backend { import std; @@ -45,10 +45,10 @@ varnish v1 -cliok "param.set vcc_unsafe_path on" varnish v1 -cliok "param.set vmod_path /nowhere:/else" varnish v1 -vcl+backend { - import std from "${topbuild}/lib/libvmod_std/.libs/"; + import std from "${topbuild}/vmod/.libs/"; } varnish v1 -errvcl {Malformed VMOD std} { backend default { .host = "${s1_sock}"; } - import std from "${topbuild}/lib/libvmod_debug/.libs/libvmod_debug.so"; + import std from "${topbuild}/vmod/.libs/libvmod_debug.so"; } diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 1b322f8de..6110540c2 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -559,7 +559,7 @@ i_mode(void) */ VSB_clear(vsb); - build_path(topbuild, "lib", "libvmod_", "/.libs", vsb); + build_path(topbuild, "vmod", ".libs", "", vsb); AZ(VSB_finish(vsb)); vmod_path = strdup(VSB_data(vsb)); AN(vmod_path); diff --git a/configure.ac b/configure.ac index 53ba873d0..5c7c9dd7e 100644 --- a/configure.ac +++ b/configure.ac @@ -876,17 +876,9 @@ AC_CONFIG_FILES([ lib/libvarnishapi/Makefile lib/libvcc/Makefile lib/libvgz/Makefile - lib/libvmod_cookie/Makefile - lib/libvmod_debug/Makefile - lib/libvmod_std/Makefile - lib/libvmod_directors/Makefile - lib/libvmod_purge/Makefile - lib/libvmod_vtc/Makefile - lib/libvmod_blob/Makefile - lib/libvmod_unix/Makefile - lib/libvmod_proxy/Makefile man/Makefile varnishapi.pc varnishapi-uninstalled.pc + vmod/Makefile ]) AC_OUTPUT diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index 02a691cb9..a85815071 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -174,36 +174,36 @@ BUILT_SOURCES += include/vtc-syntax.rst # XXX copy/paste rules need some TLC -include/vmod_std.generated.rst: $(top_builddir)/lib/libvmod_std/vmod_std.rst - cp $(top_builddir)/lib/libvmod_std/vmod_std.rst $@ +include/vmod_std.generated.rst: $(top_builddir)/vmod/vmod_std.rst + cp $(top_builddir)/vmod/vmod_std.rst $@ BUILT_SOURCES += include/vmod_std.generated.rst -include/vmod_directors.generated.rst: $(top_builddir)/lib/libvmod_directors/vmod_directors.rst - cp $(top_builddir)/lib/libvmod_directors/vmod_directors.rst $@ +include/vmod_directors.generated.rst: $(top_builddir)/vmod/vmod_directors.rst + cp $(top_builddir)/vmod/vmod_directors.rst $@ BUILT_SOURCES += include/vmod_directors.generated.rst -include/vmod_purge.generated.rst: $(top_builddir)/lib/libvmod_purge/vmod_purge.rst - cp $(top_builddir)/lib/libvmod_purge/vmod_purge.rst $@ +include/vmod_purge.generated.rst: $(top_builddir)/vmod/vmod_purge.rst + cp $(top_builddir)/vmod/vmod_purge.rst $@ BUILT_SOURCES += include/vmod_purge.generated.rst -include/vmod_vtc.generated.rst: $(top_builddir)/lib/libvmod_vtc/vmod_vtc.rst - cp $(top_builddir)/lib/libvmod_vtc/vmod_vtc.rst $@ +include/vmod_vtc.generated.rst: $(top_builddir)/vmod/vmod_vtc.rst + cp $(top_builddir)/vmod/vmod_vtc.rst $@ BUILT_SOURCES += include/vmod_vtc.generated.rst -include/vmod_blob.generated.rst: $(top_builddir)/lib/libvmod_blob/vmod_blob.rst - cp $(top_builddir)/lib/libvmod_blob/vmod_blob.rst $@ +include/vmod_blob.generated.rst: $(top_builddir)/vmod/vmod_blob.rst + cp $(top_builddir)/vmod/vmod_blob.rst $@ BUILT_SOURCES += include/vmod_blob.generated.rst -include/vmod_cookie.generated.rst: $(top_builddir)/lib/libvmod_cookie/vmod_cookie.rst - cp $(top_builddir)/lib/libvmod_cookie/vmod_cookie.rst $@ +include/vmod_cookie.generated.rst: $(top_builddir)/vmod/vmod_cookie.rst + cp $(top_builddir)/vmod/vmod_cookie.rst $@ BUILT_SOURCES += include/vmod_cookie.generated.rst -include/vmod_unix.generated.rst: $(top_builddir)/lib/libvmod_unix/vmod_unix.rst - cp $(top_builddir)/lib/libvmod_unix/vmod_unix.rst $@ +include/vmod_unix.generated.rst: $(top_builddir)/vmod/vmod_unix.rst + cp $(top_builddir)/vmod/vmod_unix.rst $@ BUILT_SOURCES += include/vmod_unix.generated.rst -include/vmod_proxy.generated.rst: $(top_builddir)/lib/libvmod_proxy/vmod_proxy.rst - cp $(top_builddir)/lib/libvmod_proxy/vmod_proxy.rst $@ +include/vmod_proxy.generated.rst: $(top_builddir)/vmod/vmod_proxy.rst + cp $(top_builddir)/vmod/vmod_proxy.rst $@ BUILT_SOURCES += include/vmod_proxy.generated.rst EXTRA_DIST += $(BUILT_SOURCES) diff --git a/lib/Makefile.am b/lib/Makefile.am index 1cd630954..0f6873c50 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -4,13 +4,4 @@ SUBDIRS = \ libvarnish \ libvarnishapi \ libvcc \ - libvgz \ - libvmod_cookie \ - libvmod_debug \ - libvmod_std \ - libvmod_directors \ - libvmod_purge \ - libvmod_vtc \ - libvmod_blob \ - libvmod_unix \ - libvmod_proxy + libvgz diff --git a/lib/libvmod_blob/Makefile.am b/lib/libvmod_blob/Makefile.am deleted file mode 100644 index 6015d15ca..000000000 --- a/lib/libvmod_blob/Makefile.am +++ /dev/null @@ -1,26 +0,0 @@ -# - -AM_LDFLAGS = $(AM_LT_LDFLAGS) - -AM_CPPFLAGS = \ - -I$(top_srcdir)/include \ - -I$(top_srcdir)/bin/varnishd \ - -I$(top_builddir)/include - -vmod_LTLIBRARIES = -EXTRA_DIST = -CLEANFILES = - -libvmod_blob_la_SOURCES = \ - vmod_blob.c \ - vmod_blob.h \ - id.c \ - base64.c \ - hex.h \ - hex.c \ - url.c \ - tbl_encodings.h \ - tbl_case.h - -# Use vmodtool.py generated automake boilerplate -include $(srcdir)/automake_boilerplate_blob.am diff --git a/lib/libvmod_blob/flint.lnt b/lib/libvmod_blob/flint.lnt deleted file mode 100644 index 016a8329d..000000000 --- a/lib/libvmod_blob/flint.lnt +++ /dev/null @@ -1,4 +0,0 @@ --efile(451, "tbl_*.h") // No include guard - --e784 // Nul character truncated from string - diff --git a/lib/libvmod_cookie/Makefile.am b/lib/libvmod_cookie/Makefile.am deleted file mode 100644 index fe762a186..000000000 --- a/lib/libvmod_cookie/Makefile.am +++ /dev/null @@ -1,42 +0,0 @@ -AM_LDFLAGS = $(AM_LT_LDFLAGS) - -AM_CPPFLAGS = \ - -I$(top_srcdir)/include \ - -I$(top_srcdir)/bin/varnishd \ - -I$(top_builddir)/include - -vmod_LTLIBRARIES = -EXTRA_DIST = -CLEANFILES = - -libvmod_cookie_la_SOURCES = \ - vmod_cookie.c - -# Use vmodtool.py generated automake boilerplate -include $(srcdir)/automake_boilerplate_cookie.am - -TESTS = \ - tests/cookie_b00000.vtc \ - tests/cookie_b00001.vtc \ - tests/cookie_b00002.vtc \ - tests/cookie_b00003.vtc \ - tests/cookie_b00004.vtc \ - tests/cookie_b00005.vtc \ - tests/cookie_b00006.vtc \ - tests/cookie_b00007.vtc \ - tests/cookie_b00008.vtc \ - tests/cookie_b00009.vtc \ - tests/cookie_b00010.vtc \ - tests/cookie_b00011.vtc \ - tests/cookie_b00012.vtc \ - tests/cookie_b00013.vtc \ - tests/cookie_r00028.vtc \ - tests/cookie_v00000.vtc - -EXTRA_DIST += $(TESTS) - -vtc-refresh-tests: - @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_cookie.vcc - @cd $(top_builddir) && ./config.status --file=$(subdir)/Makefile - -include $(top_srcdir)/vtc.am diff --git a/lib/libvmod_cookie/flint.lnt b/lib/libvmod_cookie/flint.lnt deleted file mode 100644 index e69de29bb..000000000 diff --git a/lib/libvmod_cookie/flint.sh b/lib/libvmod_cookie/flint.sh deleted file mode 100755 index 522e30d4d..000000000 --- a/lib/libvmod_cookie/flint.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -FLOPS=' - -I../../bin/varnishd - *.c -' - -. ../../tools/flint_skel.sh diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am deleted file mode 100644 index ccb866ba1..000000000 --- a/lib/libvmod_debug/Makefile.am +++ /dev/null @@ -1,37 +0,0 @@ -# - -AM_LDFLAGS = $(AM_LT_LDFLAGS) - -AM_CPPFLAGS = \ - -I$(top_srcdir)/include \ - -I$(top_srcdir)/bin/varnishd \ - -I$(top_builddir)/include - -vmod_LTLIBRARIES = -EXTRA_DIST = -CLEANFILES = - -libvmod_debug_la_SOURCES = \ - vmod_debug.c \ - vmod_debug_obj.c \ - vmod_debug_dyn.c - -include $(srcdir)/automake_boilerplate_debug.am - -# Allow Vmod_wrong*_Data to be exported -vmod_debug_symbols_regex = 'Vmod_.*_Data' - -# not --strict -vmodtoolargs_debug = --boilerplate -o vcc_debug_if - -.vsc.c: - $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $< - -VSC_SRC = VSC_debug.vsc -VSC_GEN_C = VSC_debug.c -VSC_GEN_H = VSC_debug.h - -CLEANFILES += $(VSC_GEN_C) $(VSC_GEN_H) - -BUILT_SOURCES = $(VSC_GEN_C) -libvmod_debug_la_SOURCES += $(VSC_SRC) diff --git a/lib/libvmod_debug/flint.sh b/lib/libvmod_debug/flint.sh deleted file mode 100755 index 522e30d4d..000000000 --- a/lib/libvmod_debug/flint.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -FLOPS=' - -I../../bin/varnishd - *.c -' - -. ../../tools/flint_skel.sh diff --git a/lib/libvmod_directors/Makefile.am b/lib/libvmod_directors/Makefile.am deleted file mode 100644 index c461639bc..000000000 --- a/lib/libvmod_directors/Makefile.am +++ /dev/null @@ -1,29 +0,0 @@ -# - -AM_LDFLAGS = $(AM_LT_LDFLAGS) - -AM_CPPFLAGS = \ - -I$(top_srcdir)/include \ - -I$(top_srcdir)/bin/varnishd \ - -I$(top_builddir)/include - -vmod_LTLIBRARIES = -EXTRA_DIST = -CLEANFILES = - -libvmod_directors_la_SOURCES = \ - vdir.c \ - vdir.h \ - fall_back.c \ - hash.c \ - misc.c \ - random.c \ - round_robin.c \ - vmod_shard.c \ - shard_cfg.c \ - shard_cfg.h \ - shard_dir.c \ - shard_dir.h - -# Use vmodtool.py generated automake boilerplate -include $(srcdir)/automake_boilerplate_directors.am diff --git a/lib/libvmod_directors/flint.lnt b/lib/libvmod_directors/flint.lnt deleted file mode 100644 index 1b6f0888f..000000000 --- a/lib/libvmod_directors/flint.lnt +++ /dev/null @@ -1 +0,0 @@ --efile(451, "tbl_*.h") // No include guard diff --git a/lib/libvmod_directors/flint.sh b/lib/libvmod_directors/flint.sh deleted file mode 100755 index 522e30d4d..000000000 --- a/lib/libvmod_directors/flint.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -FLOPS=' - -I../../bin/varnishd - *.c -' - -. ../../tools/flint_skel.sh diff --git a/lib/libvmod_proxy/Makefile.am b/lib/libvmod_proxy/Makefile.am deleted file mode 100644 index c96d8c8d1..000000000 --- a/lib/libvmod_proxy/Makefile.am +++ /dev/null @@ -1,18 +0,0 @@ -# - -AM_LDFLAGS = $(AM_LT_LDFLAGS) - -AM_CPPFLAGS = \ - -I$(top_srcdir)/include \ - -I$(top_srcdir)/bin/varnishd \ - -I$(top_builddir)/include - -vmod_LTLIBRARIES = -EXTRA_DIST = -CLEANFILES = - -libvmod_proxy_la_SOURCES = \ - vmod_proxy.c - -# Use vmodtool.py generated automake boilerplate -include $(srcdir)/automake_boilerplate_proxy.am diff --git a/lib/libvmod_proxy/flint.lnt b/lib/libvmod_proxy/flint.lnt deleted file mode 100644 index e69de29bb..000000000 diff --git a/lib/libvmod_proxy/flint.sh b/lib/libvmod_proxy/flint.sh deleted file mode 100755 index 522e30d4d..000000000 --- a/lib/libvmod_proxy/flint.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -FLOPS=' - -I../../bin/varnishd - *.c -' - -. ../../tools/flint_skel.sh diff --git a/lib/libvmod_purge/Makefile.am b/lib/libvmod_purge/Makefile.am deleted file mode 100644 index 8af2b1475..000000000 --- a/lib/libvmod_purge/Makefile.am +++ /dev/null @@ -1,18 +0,0 @@ -# - -AM_LDFLAGS = $(AM_LT_LDFLAGS) - -AM_CPPFLAGS = \ - -I$(top_srcdir)/include \ - -I$(top_srcdir)/bin/varnishd \ - -I$(top_builddir)/include - -vmod_LTLIBRARIES = -EXTRA_DIST = -CLEANFILES = - -libvmod_purge_la_SOURCES = \ - vmod_purge.c - -# Use vmodtool.py generated automake boilerplate -include $(srcdir)/automake_boilerplate_purge.am diff --git a/lib/libvmod_purge/flint.lnt b/lib/libvmod_purge/flint.lnt deleted file mode 100644 index e69de29bb..000000000 diff --git a/lib/libvmod_purge/flint.sh b/lib/libvmod_purge/flint.sh deleted file mode 100755 index 522e30d4d..000000000 --- a/lib/libvmod_purge/flint.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -FLOPS=' - -I../../bin/varnishd - *.c -' - -. ../../tools/flint_skel.sh diff --git a/lib/libvmod_std/Makefile.am b/lib/libvmod_std/Makefile.am deleted file mode 100644 index bb922ea97..000000000 --- a/lib/libvmod_std/Makefile.am +++ /dev/null @@ -1,32 +0,0 @@ -# - -AM_LDFLAGS = $(AM_LT_LDFLAGS) - -AM_CPPFLAGS = \ - -I$(top_srcdir)/include \ - -I$(top_srcdir)/bin/varnishd \ - -I$(top_builddir)/include - -vmod_LTLIBRARIES = -EXTRA_DIST = -CLEANFILES = - -libvmod_std_la_SOURCES = \ - vmod_std.c \ - vmod_std_conversions.c \ - vmod_std_fileread.c \ - vmod_std_querysort.c - -# Use vmodtool.py generated automake boilerplate -include $(srcdir)/automake_boilerplate_std.am - -TESTS = \ - tests/std_b00001.vtc - -EXTRA_DIST += $(TESTS) - -vtc-refresh-tests: - @PYTHON@ $(vmodtool) $(vmodtoolargs) $(srcdir)/vmod_std.vcc - @cd $(top_builddir) && ./config.status --file=$(subdir)/Makefile - -include $(top_srcdir)/vtc.am diff --git a/lib/libvmod_std/flint.lnt b/lib/libvmod_std/flint.lnt deleted file mode 100644 index e69de29bb..000000000 diff --git a/lib/libvmod_std/flint.sh b/lib/libvmod_std/flint.sh deleted file mode 100755 index 522e30d4d..000000000 --- a/lib/libvmod_std/flint.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -FLOPS=' - -I../../bin/varnishd - *.c -' - -. ../../tools/flint_skel.sh diff --git a/lib/libvmod_unix/Makefile.am b/lib/libvmod_unix/Makefile.am deleted file mode 100644 index db4a1cec5..000000000 --- a/lib/libvmod_unix/Makefile.am +++ /dev/null @@ -1,19 +0,0 @@ -# - -AM_LDFLAGS = $(AM_LT_LDFLAGS) - -AM_CPPFLAGS = \ - -I$(top_srcdir)/include \ - -I$(top_srcdir)/bin/varnishd \ - -I$(top_builddir)/include - -vmod_LTLIBRARIES = -EXTRA_DIST = -CLEANFILES = - -libvmod_unix_la_SOURCES = \ - vmod_unix.c \ - cred_compat.h - -# Use vmodtool.py generated automake boilerplate -include $(srcdir)/automake_boilerplate_unix.am diff --git a/lib/libvmod_unix/flint.lnt b/lib/libvmod_unix/flint.lnt deleted file mode 100644 index e69de29bb..000000000 diff --git a/lib/libvmod_unix/flint.sh b/lib/libvmod_unix/flint.sh deleted file mode 100755 index 522e30d4d..000000000 --- a/lib/libvmod_unix/flint.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -FLOPS=' - -I../../bin/varnishd - *.c -' - -. ../../tools/flint_skel.sh diff --git a/lib/libvmod_vtc/Makefile.am b/lib/libvmod_vtc/Makefile.am deleted file mode 100644 index 32fb0aaa6..000000000 --- a/lib/libvmod_vtc/Makefile.am +++ /dev/null @@ -1,18 +0,0 @@ -# - -AM_LDFLAGS = $(AM_LT_LDFLAGS) - -AM_CPPFLAGS = \ - -I$(top_srcdir)/include \ - -I$(top_srcdir)/bin/varnishd \ - -I$(top_builddir)/include - -vmod_LTLIBRARIES = -EXTRA_DIST = -CLEANFILES = - -libvmod_vtc_la_SOURCES = \ - vmod_vtc.c - -# Use vmodtool.py generated automake boilerplate -include $(srcdir)/automake_boilerplate_vtc.am diff --git a/lib/libvmod_vtc/flint.lnt b/lib/libvmod_vtc/flint.lnt deleted file mode 100644 index e69de29bb..000000000 diff --git a/lib/libvmod_vtc/flint.sh b/lib/libvmod_vtc/flint.sh deleted file mode 100755 index 522e30d4d..000000000 --- a/lib/libvmod_vtc/flint.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -FLOPS=' - -I../../bin/varnishd - *.c -' - -. ../../tools/flint_skel.sh diff --git a/man/Makefile.am b/man/Makefile.am index 5c6ebae88..15bf7c9fc 100644 --- a/man/Makefile.am +++ b/man/Makefile.am @@ -100,28 +100,28 @@ varnishhist.1: \ $(top_builddir)/doc/sphinx/include/varnishhist_synopsis.rst $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishhist.rst $@ -vmod_cookie.3: $(top_builddir)/lib/libvmod_cookie/vmod_cookie.man.rst +vmod_cookie.3: $(top_builddir)/vmod/vmod_cookie.man.rst $(BUILD_MAN) $? $@ -vmod_directors.3: $(top_builddir)/lib/libvmod_directors/vmod_directors.man.rst +vmod_directors.3: $(top_builddir)/vmod/vmod_directors.man.rst $(BUILD_MAN) $? $@ -vmod_purge.3: $(top_builddir)/lib/libvmod_purge/vmod_purge.man.rst +vmod_purge.3: $(top_builddir)/vmod/vmod_purge.man.rst $(BUILD_MAN) $? $@ -vmod_std.3: $(top_builddir)/lib/libvmod_std/vmod_std.man.rst +vmod_std.3: $(top_builddir)/vmod/vmod_std.man.rst $(BUILD_MAN) $? $@ -vmod_vtc.3: $(top_builddir)/lib/libvmod_vtc/vmod_vtc.man.rst +vmod_vtc.3: $(top_builddir)/vmod/vmod_vtc.man.rst $(BUILD_MAN) $? $@ -vmod_blob.3: $(top_builddir)/lib/libvmod_blob/vmod_blob.man.rst +vmod_blob.3: $(top_builddir)/vmod/vmod_blob.man.rst $(BUILD_MAN) $? $@ -vmod_unix.3: $(top_builddir)/lib/libvmod_unix/vmod_unix.man.rst +vmod_unix.3: $(top_builddir)/vmod/vmod_unix.man.rst $(BUILD_MAN) $? $@ -vmod_proxy.3: $(top_builddir)/lib/libvmod_proxy/vmod_proxy.man.rst +vmod_proxy.3: $(top_builddir)/vmod/vmod_proxy.man.rst $(BUILD_MAN) $? $@ .NOPATH: $(dist_man_MANS) diff --git a/vmod/Makefile.am b/vmod/Makefile.am new file mode 100644 index 000000000..8241e183d --- /dev/null +++ b/vmod/Makefile.am @@ -0,0 +1,142 @@ +# + +AM_LDFLAGS = $(AM_LT_LDFLAGS) + +AM_CPPFLAGS = \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/bin/varnishd \ + -I$(top_builddir)/include + +# +# Declare vmod sources +# + +libvmod_blob_la_SOURCES = \ + vmod_blob.c \ + vmod_blob.h \ + id.c \ + base64.c \ + hex.h \ + hex.c \ + url.c \ + tbl_encodings.h \ + tbl_case.h + +libvmod_cookie_la_SOURCES = \ + vmod_cookie.c + +.vsc.c: + $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -c $< + +.vsc.h: + $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -h $< + +libvmod_debug_extra_sources = \ + VSC_debug.vsc + +libvmod_debug_built_sources = \ + VSC_debug.c \ + VSC_debug.h + +libvmod_debug_la_SOURCES = \ + vmod_debug.c \ + vmod_debug_obj.c \ + vmod_debug_dyn.c + +libvmod_directors_la_SOURCES = \ + vdir.c \ + vdir.h \ + fall_back.c \ + hash.c \ + misc.c \ + random.c \ + round_robin.c \ + vmod_shard.c \ + shard_cfg.c \ + shard_cfg.h \ + shard_dir.c \ + shard_dir.h + +libvmod_proxy_la_SOURCES = \ + vmod_proxy.c + +libvmod_purge_la_SOURCES = \ + vmod_purge.c + +libvmod_std_la_SOURCES = \ + vmod_std.c \ + vmod_std_conversions.c \ + vmod_std_fileread.c \ + vmod_std_querysort.c + +libvmod_unix_la_SOURCES = \ + vmod_unix.c \ + cred_compat.h + +libvmod_vtc_la_SOURCES = \ + vmod_vtc.c + +# +# Prepare build variables +# + +vmod_LTLIBRARIES = +EXTRA_DIST = $(libvmod_debug_extra_sources) +CLEANFILES = $(libvmod_debug_built_sources) +BUILT_SOURCES = $(libvmod_debug_built_sources) + +# +# Use vmodtool.py generated automake boilerplate +# + +include $(srcdir)/automake_boilerplate_blob.am +include $(srcdir)/automake_boilerplate_cookie.am +include $(srcdir)/automake_boilerplate_debug.am +include $(srcdir)/automake_boilerplate_directors.am +include $(srcdir)/automake_boilerplate_proxy.am +include $(srcdir)/automake_boilerplate_purge.am +include $(srcdir)/automake_boilerplate_std.am +include $(srcdir)/automake_boilerplate_unix.am +include $(srcdir)/automake_boilerplate_vtc.am + +# +# Post-boilerplate tweaks +# + +nodist_libvmod_debug_la_SOURCES += $(libvmod_debug_built_sources) + +# Allow Vmod_wrong*_Data to be exported +vmod_debug_symbols_regex = 'Vmod_.*_Data' + +# not --strict +vmodtoolargs_debug = --boilerplate -o vcc_debug_if + +# +# Manage the vmod test suite +# + +TESTS = \ + tests/cookie_b00000.vtc \ + tests/cookie_b00001.vtc \ + tests/cookie_b00002.vtc \ + tests/cookie_b00003.vtc \ + tests/cookie_b00004.vtc \ + tests/cookie_b00005.vtc \ + tests/cookie_b00006.vtc \ + tests/cookie_b00007.vtc \ + tests/cookie_b00008.vtc \ + tests/cookie_b00009.vtc \ + tests/cookie_b00010.vtc \ + tests/cookie_b00011.vtc \ + tests/cookie_b00012.vtc \ + tests/cookie_b00013.vtc \ + tests/cookie_r00028.vtc \ + tests/cookie_v00000.vtc \ + tests/std_b00001.vtc + +EXTRA_DIST += $(TESTS) + +vtc-refresh-tests: + @cd $(top_builddir) && ./config.status --file=$(subdir)/Makefile + +include $(top_srcdir)/vtc.am diff --git a/lib/libvmod_debug/VSC_debug.vsc b/vmod/VSC_debug.vsc similarity index 100% rename from lib/libvmod_debug/VSC_debug.vsc rename to vmod/VSC_debug.vsc diff --git a/lib/libvmod_blob/automake_boilerplate_blob.am b/vmod/automake_boilerplate_blob.am similarity index 100% rename from lib/libvmod_blob/automake_boilerplate_blob.am rename to vmod/automake_boilerplate_blob.am diff --git a/lib/libvmod_cookie/automake_boilerplate_cookie.am b/vmod/automake_boilerplate_cookie.am similarity index 100% rename from lib/libvmod_cookie/automake_boilerplate_cookie.am rename to vmod/automake_boilerplate_cookie.am diff --git a/lib/libvmod_debug/automake_boilerplate_debug.am b/vmod/automake_boilerplate_debug.am similarity index 100% rename from lib/libvmod_debug/automake_boilerplate_debug.am rename to vmod/automake_boilerplate_debug.am diff --git a/lib/libvmod_directors/automake_boilerplate_directors.am b/vmod/automake_boilerplate_directors.am similarity index 100% rename from lib/libvmod_directors/automake_boilerplate_directors.am rename to vmod/automake_boilerplate_directors.am diff --git a/lib/libvmod_proxy/automake_boilerplate_proxy.am b/vmod/automake_boilerplate_proxy.am similarity index 100% rename from lib/libvmod_proxy/automake_boilerplate_proxy.am rename to vmod/automake_boilerplate_proxy.am diff --git a/lib/libvmod_purge/automake_boilerplate_purge.am b/vmod/automake_boilerplate_purge.am similarity index 100% rename from lib/libvmod_purge/automake_boilerplate_purge.am rename to vmod/automake_boilerplate_purge.am diff --git a/lib/libvmod_std/automake_boilerplate_std.am b/vmod/automake_boilerplate_std.am similarity index 100% rename from lib/libvmod_std/automake_boilerplate_std.am rename to vmod/automake_boilerplate_std.am diff --git a/lib/libvmod_unix/automake_boilerplate_unix.am b/vmod/automake_boilerplate_unix.am similarity index 100% rename from lib/libvmod_unix/automake_boilerplate_unix.am rename to vmod/automake_boilerplate_unix.am diff --git a/lib/libvmod_vtc/automake_boilerplate_vtc.am b/vmod/automake_boilerplate_vtc.am similarity index 100% rename from lib/libvmod_vtc/automake_boilerplate_vtc.am rename to vmod/automake_boilerplate_vtc.am diff --git a/lib/libvmod_blob/base64.c b/vmod/base64.c similarity index 100% rename from lib/libvmod_blob/base64.c rename to vmod/base64.c diff --git a/lib/libvmod_unix/cred_compat.h b/vmod/cred_compat.h similarity index 100% rename from lib/libvmod_unix/cred_compat.h rename to vmod/cred_compat.h diff --git a/lib/libvmod_directors/fall_back.c b/vmod/fall_back.c similarity index 100% rename from lib/libvmod_directors/fall_back.c rename to vmod/fall_back.c diff --git a/lib/libvmod_debug/flint.lnt b/vmod/flint.lnt similarity index 69% rename from lib/libvmod_debug/flint.lnt rename to vmod/flint.lnt index 2a43e7057..ac5724d6c 100644 --- a/lib/libvmod_debug/flint.lnt +++ b/vmod/flint.lnt @@ -1,3 +1,19 @@ +/* + * vmod_{blob,directors} + */ + +-efile(451, "tbl_*.h") // No include guard + +/* + * vmod_blob + */ + +-e784 // Nul character truncated from string + +/* + * vmod_debug + */ + -esym(759, xyzzy_enum_*) // header declaration for symbol '___' defined at (___) -esym(765, xyzzy_enum_*) // external '___' (___) could be made static diff --git a/lib/libvmod_blob/flint.sh b/vmod/flint.sh similarity index 100% rename from lib/libvmod_blob/flint.sh rename to vmod/flint.sh diff --git a/lib/libvmod_directors/hash.c b/vmod/hash.c similarity index 100% rename from lib/libvmod_directors/hash.c rename to vmod/hash.c diff --git a/lib/libvmod_blob/hex.c b/vmod/hex.c similarity index 100% rename from lib/libvmod_blob/hex.c rename to vmod/hex.c diff --git a/lib/libvmod_blob/hex.h b/vmod/hex.h similarity index 100% rename from lib/libvmod_blob/hex.h rename to vmod/hex.h diff --git a/lib/libvmod_blob/id.c b/vmod/id.c similarity index 100% rename from lib/libvmod_blob/id.c rename to vmod/id.c diff --git a/lib/libvmod_directors/misc.c b/vmod/misc.c similarity index 100% rename from lib/libvmod_directors/misc.c rename to vmod/misc.c diff --git a/lib/libvmod_directors/random.c b/vmod/random.c similarity index 100% rename from lib/libvmod_directors/random.c rename to vmod/random.c diff --git a/lib/libvmod_directors/round_robin.c b/vmod/round_robin.c similarity index 100% rename from lib/libvmod_directors/round_robin.c rename to vmod/round_robin.c diff --git a/lib/libvmod_directors/shard_cfg.c b/vmod/shard_cfg.c similarity index 100% rename from lib/libvmod_directors/shard_cfg.c rename to vmod/shard_cfg.c diff --git a/lib/libvmod_directors/shard_cfg.h b/vmod/shard_cfg.h similarity index 100% rename from lib/libvmod_directors/shard_cfg.h rename to vmod/shard_cfg.h diff --git a/lib/libvmod_directors/shard_dir.c b/vmod/shard_dir.c similarity index 100% rename from lib/libvmod_directors/shard_dir.c rename to vmod/shard_dir.c diff --git a/lib/libvmod_directors/shard_dir.h b/vmod/shard_dir.h similarity index 100% rename from lib/libvmod_directors/shard_dir.h rename to vmod/shard_dir.h diff --git a/lib/libvmod_blob/tbl_case.h b/vmod/tbl_case.h similarity index 100% rename from lib/libvmod_blob/tbl_case.h rename to vmod/tbl_case.h diff --git a/lib/libvmod_blob/tbl_encodings.h b/vmod/tbl_encodings.h similarity index 100% rename from lib/libvmod_blob/tbl_encodings.h rename to vmod/tbl_encodings.h diff --git a/lib/libvmod_cookie/tests/cookie_b00000.vtc b/vmod/tests/cookie_b00000.vtc similarity index 100% rename from lib/libvmod_cookie/tests/cookie_b00000.vtc rename to vmod/tests/cookie_b00000.vtc diff --git a/lib/libvmod_cookie/tests/cookie_b00001.vtc b/vmod/tests/cookie_b00001.vtc similarity index 100% rename from lib/libvmod_cookie/tests/cookie_b00001.vtc rename to vmod/tests/cookie_b00001.vtc diff --git a/lib/libvmod_cookie/tests/cookie_b00002.vtc b/vmod/tests/cookie_b00002.vtc similarity index 100% rename from lib/libvmod_cookie/tests/cookie_b00002.vtc rename to vmod/tests/cookie_b00002.vtc diff --git a/lib/libvmod_cookie/tests/cookie_b00003.vtc b/vmod/tests/cookie_b00003.vtc similarity index 100% rename from lib/libvmod_cookie/tests/cookie_b00003.vtc rename to vmod/tests/cookie_b00003.vtc diff --git a/lib/libvmod_cookie/tests/cookie_b00004.vtc b/vmod/tests/cookie_b00004.vtc similarity index 100% rename from lib/libvmod_cookie/tests/cookie_b00004.vtc rename to vmod/tests/cookie_b00004.vtc diff --git a/lib/libvmod_cookie/tests/cookie_b00005.vtc b/vmod/tests/cookie_b00005.vtc similarity index 100% rename from lib/libvmod_cookie/tests/cookie_b00005.vtc rename to vmod/tests/cookie_b00005.vtc diff --git a/lib/libvmod_cookie/tests/cookie_b00006.vtc b/vmod/tests/cookie_b00006.vtc similarity index 100% rename from lib/libvmod_cookie/tests/cookie_b00006.vtc rename to vmod/tests/cookie_b00006.vtc diff --git a/lib/libvmod_cookie/tests/cookie_b00007.vtc b/vmod/tests/cookie_b00007.vtc similarity index 100% rename from lib/libvmod_cookie/tests/cookie_b00007.vtc rename to vmod/tests/cookie_b00007.vtc diff --git a/lib/libvmod_cookie/tests/cookie_b00008.vtc b/vmod/tests/cookie_b00008.vtc similarity index 100% rename from lib/libvmod_cookie/tests/cookie_b00008.vtc rename to vmod/tests/cookie_b00008.vtc diff --git a/lib/libvmod_cookie/tests/cookie_b00009.vtc b/vmod/tests/cookie_b00009.vtc similarity index 100% rename from lib/libvmod_cookie/tests/cookie_b00009.vtc rename to vmod/tests/cookie_b00009.vtc diff --git a/lib/libvmod_cookie/tests/cookie_b00010.vtc b/vmod/tests/cookie_b00010.vtc similarity index 100% rename from lib/libvmod_cookie/tests/cookie_b00010.vtc rename to vmod/tests/cookie_b00010.vtc diff --git a/lib/libvmod_cookie/tests/cookie_b00011.vtc b/vmod/tests/cookie_b00011.vtc similarity index 100% rename from lib/libvmod_cookie/tests/cookie_b00011.vtc rename to vmod/tests/cookie_b00011.vtc diff --git a/lib/libvmod_cookie/tests/cookie_b00012.vtc b/vmod/tests/cookie_b00012.vtc similarity index 100% rename from lib/libvmod_cookie/tests/cookie_b00012.vtc rename to vmod/tests/cookie_b00012.vtc diff --git a/lib/libvmod_cookie/tests/cookie_b00013.vtc b/vmod/tests/cookie_b00013.vtc similarity index 100% rename from lib/libvmod_cookie/tests/cookie_b00013.vtc rename to vmod/tests/cookie_b00013.vtc diff --git a/lib/libvmod_cookie/tests/cookie_r00028.vtc b/vmod/tests/cookie_r00028.vtc similarity index 100% rename from lib/libvmod_cookie/tests/cookie_r00028.vtc rename to vmod/tests/cookie_r00028.vtc diff --git a/lib/libvmod_cookie/tests/cookie_v00000.vtc b/vmod/tests/cookie_v00000.vtc similarity index 100% rename from lib/libvmod_cookie/tests/cookie_v00000.vtc rename to vmod/tests/cookie_v00000.vtc diff --git a/lib/libvmod_std/tests/std_b00001.vtc b/vmod/tests/std_b00001.vtc similarity index 100% rename from lib/libvmod_std/tests/std_b00001.vtc rename to vmod/tests/std_b00001.vtc diff --git a/lib/libvmod_blob/url.c b/vmod/url.c similarity index 100% rename from lib/libvmod_blob/url.c rename to vmod/url.c diff --git a/lib/libvmod_directors/vdir.c b/vmod/vdir.c similarity index 100% rename from lib/libvmod_directors/vdir.c rename to vmod/vdir.c diff --git a/lib/libvmod_directors/vdir.h b/vmod/vdir.h similarity index 100% rename from lib/libvmod_directors/vdir.h rename to vmod/vdir.h diff --git a/lib/libvmod_blob/vmod_blob.c b/vmod/vmod_blob.c similarity index 100% rename from lib/libvmod_blob/vmod_blob.c rename to vmod/vmod_blob.c diff --git a/lib/libvmod_blob/vmod_blob.h b/vmod/vmod_blob.h similarity index 100% rename from lib/libvmod_blob/vmod_blob.h rename to vmod/vmod_blob.h diff --git a/lib/libvmod_blob/vmod_blob.vcc b/vmod/vmod_blob.vcc similarity index 100% rename from lib/libvmod_blob/vmod_blob.vcc rename to vmod/vmod_blob.vcc diff --git a/lib/libvmod_cookie/vmod_cookie.c b/vmod/vmod_cookie.c similarity index 100% rename from lib/libvmod_cookie/vmod_cookie.c rename to vmod/vmod_cookie.c diff --git a/lib/libvmod_cookie/vmod_cookie.vcc b/vmod/vmod_cookie.vcc similarity index 100% rename from lib/libvmod_cookie/vmod_cookie.vcc rename to vmod/vmod_cookie.vcc diff --git a/lib/libvmod_debug/vmod_debug.c b/vmod/vmod_debug.c similarity index 100% rename from lib/libvmod_debug/vmod_debug.c rename to vmod/vmod_debug.c diff --git a/lib/libvmod_debug/vmod_debug.vcc b/vmod/vmod_debug.vcc similarity index 100% rename from lib/libvmod_debug/vmod_debug.vcc rename to vmod/vmod_debug.vcc diff --git a/lib/libvmod_debug/vmod_debug_dyn.c b/vmod/vmod_debug_dyn.c similarity index 100% rename from lib/libvmod_debug/vmod_debug_dyn.c rename to vmod/vmod_debug_dyn.c diff --git a/lib/libvmod_debug/vmod_debug_obj.c b/vmod/vmod_debug_obj.c similarity index 100% rename from lib/libvmod_debug/vmod_debug_obj.c rename to vmod/vmod_debug_obj.c diff --git a/lib/libvmod_directors/vmod_directors.vcc b/vmod/vmod_directors.vcc similarity index 100% rename from lib/libvmod_directors/vmod_directors.vcc rename to vmod/vmod_directors.vcc diff --git a/lib/libvmod_proxy/vmod_proxy.c b/vmod/vmod_proxy.c similarity index 100% rename from lib/libvmod_proxy/vmod_proxy.c rename to vmod/vmod_proxy.c diff --git a/lib/libvmod_proxy/vmod_proxy.vcc b/vmod/vmod_proxy.vcc similarity index 100% rename from lib/libvmod_proxy/vmod_proxy.vcc rename to vmod/vmod_proxy.vcc diff --git a/lib/libvmod_purge/vmod_purge.c b/vmod/vmod_purge.c similarity index 100% rename from lib/libvmod_purge/vmod_purge.c rename to vmod/vmod_purge.c diff --git a/lib/libvmod_purge/vmod_purge.vcc b/vmod/vmod_purge.vcc similarity index 100% rename from lib/libvmod_purge/vmod_purge.vcc rename to vmod/vmod_purge.vcc diff --git a/lib/libvmod_directors/vmod_shard.c b/vmod/vmod_shard.c similarity index 100% rename from lib/libvmod_directors/vmod_shard.c rename to vmod/vmod_shard.c diff --git a/lib/libvmod_std/vmod_std.c b/vmod/vmod_std.c similarity index 100% rename from lib/libvmod_std/vmod_std.c rename to vmod/vmod_std.c diff --git a/lib/libvmod_std/vmod_std.vcc b/vmod/vmod_std.vcc similarity index 100% rename from lib/libvmod_std/vmod_std.vcc rename to vmod/vmod_std.vcc diff --git a/lib/libvmod_std/vmod_std_conversions.c b/vmod/vmod_std_conversions.c similarity index 100% rename from lib/libvmod_std/vmod_std_conversions.c rename to vmod/vmod_std_conversions.c diff --git a/lib/libvmod_std/vmod_std_fileread.c b/vmod/vmod_std_fileread.c similarity index 100% rename from lib/libvmod_std/vmod_std_fileread.c rename to vmod/vmod_std_fileread.c diff --git a/lib/libvmod_std/vmod_std_querysort.c b/vmod/vmod_std_querysort.c similarity index 100% rename from lib/libvmod_std/vmod_std_querysort.c rename to vmod/vmod_std_querysort.c diff --git a/lib/libvmod_unix/vmod_unix.c b/vmod/vmod_unix.c similarity index 100% rename from lib/libvmod_unix/vmod_unix.c rename to vmod/vmod_unix.c diff --git a/lib/libvmod_unix/vmod_unix.vcc b/vmod/vmod_unix.vcc similarity index 100% rename from lib/libvmod_unix/vmod_unix.vcc rename to vmod/vmod_unix.vcc diff --git a/lib/libvmod_vtc/vmod_vtc.c b/vmod/vmod_vtc.c similarity index 100% rename from lib/libvmod_vtc/vmod_vtc.c rename to vmod/vmod_vtc.c diff --git a/lib/libvmod_vtc/vmod_vtc.vcc b/vmod/vmod_vtc.vcc similarity index 100% rename from lib/libvmod_vtc/vmod_vtc.vcc rename to vmod/vmod_vtc.vcc From dridi.boukelmoune at gmail.com Thu Jan 7 14:16:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 7 Jan 2021 14:16:07 +0000 (UTC) Subject: [master] 0d1838161 build: Manage the VMOD test suite like the main one Message-ID: <20210107141607.D9ECDA9284@lists.varnish-cache.org> commit 0d18381615a7f78c5ef1f342bd1bd0e98eb9db77 Author: Dridi Boukelmoune Date: Tue Jan 5 18:34:10 2021 +0100 build: Manage the VMOD test suite like the main one We no longer need a special per-VMOD case since we now have one unified VMOD test suite. diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am index 2dd014f26..75f84cb56 100644 --- a/bin/varnishtest/Makefile.am +++ b/bin/varnishtest/Makefile.am @@ -2,9 +2,6 @@ TESTS = @VTC_TESTS@ -vtc-refresh-tests: - @cd $(top_builddir) && ./config.status --recheck - include $(top_srcdir)/vtc.am DISTCLEANFILES = _.ok diff --git a/configure.ac b/configure.ac index 5c7c9dd7e..e44b61fff 100644 --- a/configure.ac +++ b/configure.ac @@ -842,6 +842,10 @@ VTC_TESTS="$(cd $srcdir/bin/varnishtest && echo tests/*.vtc)" AC_SUBST(VTC_TESTS) AM_SUBST_NOTMAKE(VTC_TESTS) +VMOD_TESTS="$(cd $srcdir/vmod && echo tests/*.vtc)" +AC_SUBST(VMOD_TESTS) +AM_SUBST_NOTMAKE(VMOD_TESTS) + VSC_SRC="$(cd $srcdir/bin/varnishd && echo *.vsc)" VSC_GEN_C="$(echo $VSC_SRC | sed 's:\.vsc:.c:g')" VSC_GEN_H="$(echo $VSC_SRC | sed 's:\.vsc:.h:g')" diff --git a/vmod/Makefile.am b/vmod/Makefile.am index 8241e183d..8bf96c96e 100644 --- a/vmod/Makefile.am +++ b/vmod/Makefile.am @@ -1,5 +1,9 @@ # +TESTS = @VMOD_TESTS@ + +include $(top_srcdir)/vtc.am + AM_LDFLAGS = $(AM_LT_LDFLAGS) AM_CPPFLAGS = \ @@ -81,7 +85,7 @@ libvmod_vtc_la_SOURCES = \ # vmod_LTLIBRARIES = -EXTRA_DIST = $(libvmod_debug_extra_sources) +EXTRA_DIST = $(libvmod_debug_extra_sources) $(TESTS) CLEANFILES = $(libvmod_debug_built_sources) BUILT_SOURCES = $(libvmod_debug_built_sources) @@ -110,33 +114,3 @@ vmod_debug_symbols_regex = 'Vmod_.*_Data' # not --strict vmodtoolargs_debug = --boilerplate -o vcc_debug_if - -# -# Manage the vmod test suite -# - -TESTS = \ - tests/cookie_b00000.vtc \ - tests/cookie_b00001.vtc \ - tests/cookie_b00002.vtc \ - tests/cookie_b00003.vtc \ - tests/cookie_b00004.vtc \ - tests/cookie_b00005.vtc \ - tests/cookie_b00006.vtc \ - tests/cookie_b00007.vtc \ - tests/cookie_b00008.vtc \ - tests/cookie_b00009.vtc \ - tests/cookie_b00010.vtc \ - tests/cookie_b00011.vtc \ - tests/cookie_b00012.vtc \ - tests/cookie_b00013.vtc \ - tests/cookie_r00028.vtc \ - tests/cookie_v00000.vtc \ - tests/std_b00001.vtc - -EXTRA_DIST += $(TESTS) - -vtc-refresh-tests: - @cd $(top_builddir) && ./config.status --file=$(subdir)/Makefile - -include $(top_srcdir)/vtc.am diff --git a/vtc.am b/vtc.am index 840687a43..f67a4969c 100644 --- a/vtc.am +++ b/vtc.am @@ -17,5 +17,5 @@ vtc-check-tests: @mkdir -p tests ; \ LC_ALL=C; \ if [ "$$(cd $(srcdir) && echo tests/*.vtc)" != "$(TESTS)" ]; then \ - $(MAKE) $(AM_MAKEFLAGS) vtc-refresh-tests ; \ + cd $(top_builddir) && ./config.status --recheck ; \ fi From dridi.boukelmoune at gmail.com Thu Jan 7 14:16:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 7 Jan 2021 14:16:07 +0000 (UTC) Subject: [master] 2a453c877 build: Less variables and more make rules for vmods Message-ID: <20210107141608.00CF6A9288@lists.varnish-cache.org> commit 2a453c87728bda258d78837b8bf07eda333db4a3 Author: Dridi Boukelmoune Date: Wed Jan 6 16:08:50 2021 +0100 build: Less variables and more make rules for vmods diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 4735d4322..8f2ec9ffb 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -67,6 +67,10 @@ libvmod_XXX_la_LDFLAGS = \\ nodist_libvmod_XXX_la_SOURCES = PFX.c PFX.h +EXTRA_libvmod_XXX_la_DEPENDENCIES = $(nodist_libvmod_XXX_la_SOURCES) + +EXTRA_DIST += $(srcdir)/VCC automake_boilerplate_XXX.am + $(libvmod_XXX_la_OBJECTS): PFX.h PFX.h vmod_XXX.rst vmod_XXX.man.rst: PFX.c @@ -74,11 +78,11 @@ PFX.h vmod_XXX.rst vmod_XXX.man.rst: PFX.c PFX.c: $(vmodtool) $(srcdir)/VCC \t at PYTHON@ $(vmodtool) $(vmodtoolargs_XXX) $(srcdir)/VCC -EXTRA_DIST += $(srcdir)/VCC automake_boilerplate_XXX.am +clean-local: clean-vmod-XXX -CLEANFILES += $(builddir)/PFX.c $(builddir)/PFX.h \\ -\t$(builddir)/vmod_XXX.rst \\ -\t$(builddir)/vmod_XXX.man.rst +clean-vmod-XXX: + rm -f $(nodist_libvmod_XXX_la_SOURCES) + rm -f vmod_XXX.rst vmod_XXX.man.rst ''' PRIVS = { diff --git a/vmod/Makefile.am b/vmod/Makefile.am index 8bf96c96e..8a4ab5f41 100644 --- a/vmod/Makefile.am +++ b/vmod/Makefile.am @@ -4,6 +4,8 @@ TESTS = @VMOD_TESTS@ include $(top_srcdir)/vtc.am +EXTRA_DIST = $(TESTS) + AM_LDFLAGS = $(AM_LT_LDFLAGS) AM_CPPFLAGS = \ @@ -29,19 +31,6 @@ libvmod_blob_la_SOURCES = \ libvmod_cookie_la_SOURCES = \ vmod_cookie.c -.vsc.c: - $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -c $< - -.vsc.h: - $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -h $< - -libvmod_debug_extra_sources = \ - VSC_debug.vsc - -libvmod_debug_built_sources = \ - VSC_debug.c \ - VSC_debug.h - libvmod_debug_la_SOURCES = \ vmod_debug.c \ vmod_debug_obj.c \ @@ -81,17 +70,10 @@ libvmod_vtc_la_SOURCES = \ vmod_vtc.c # -# Prepare build variables +# Use vmodtool.py generated automake boilerplate # vmod_LTLIBRARIES = -EXTRA_DIST = $(libvmod_debug_extra_sources) $(TESTS) -CLEANFILES = $(libvmod_debug_built_sources) -BUILT_SOURCES = $(libvmod_debug_built_sources) - -# -# Use vmodtool.py generated automake boilerplate -# include $(srcdir)/automake_boilerplate_blob.am include $(srcdir)/automake_boilerplate_cookie.am @@ -107,7 +89,16 @@ include $(srcdir)/automake_boilerplate_vtc.am # Post-boilerplate tweaks # -nodist_libvmod_debug_la_SOURCES += $(libvmod_debug_built_sources) +.vsc.c: + $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -c $< + +.vsc.h: + $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -h $< + +EXTRA_DIST += VSC_debug.vsc +nodist_libvmod_debug_la_SOURCES += \ + VSC_debug.c \ + VSC_debug.h # Allow Vmod_wrong*_Data to be exported vmod_debug_symbols_regex = 'Vmod_.*_Data' diff --git a/vmod/automake_boilerplate_blob.am b/vmod/automake_boilerplate_blob.am index e0e88e4ff..81ec37e56 100644 --- a/vmod/automake_boilerplate_blob.am +++ b/vmod/automake_boilerplate_blob.am @@ -16,6 +16,10 @@ libvmod_blob_la_LDFLAGS = \ nodist_libvmod_blob_la_SOURCES = vcc_blob_if.c vcc_blob_if.h +EXTRA_libvmod_blob_la_DEPENDENCIES = $(nodist_libvmod_blob_la_SOURCES) + +EXTRA_DIST += $(srcdir)/vmod_blob.vcc automake_boilerplate_blob.am + $(libvmod_blob_la_OBJECTS): vcc_blob_if.h vcc_blob_if.h vmod_blob.rst vmod_blob.man.rst: vcc_blob_if.c @@ -23,8 +27,8 @@ vcc_blob_if.h vmod_blob.rst vmod_blob.man.rst: vcc_blob_if.c vcc_blob_if.c: $(vmodtool) $(srcdir)/vmod_blob.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs_blob) $(srcdir)/vmod_blob.vcc -EXTRA_DIST += $(srcdir)/vmod_blob.vcc automake_boilerplate_blob.am +clean-local: clean-vmod-blob -CLEANFILES += $(builddir)/vcc_blob_if.c $(builddir)/vcc_blob_if.h \ - $(builddir)/vmod_blob.rst \ - $(builddir)/vmod_blob.man.rst +clean-vmod-blob: + rm -f $(nodist_libvmod_blob_la_SOURCES) + rm -f vmod_blob.rst vmod_blob.man.rst diff --git a/vmod/automake_boilerplate_cookie.am b/vmod/automake_boilerplate_cookie.am index 9c2f4822d..fb9b156b5 100644 --- a/vmod/automake_boilerplate_cookie.am +++ b/vmod/automake_boilerplate_cookie.am @@ -16,6 +16,10 @@ libvmod_cookie_la_LDFLAGS = \ nodist_libvmod_cookie_la_SOURCES = vcc_cookie_if.c vcc_cookie_if.h +EXTRA_libvmod_cookie_la_DEPENDENCIES = $(nodist_libvmod_cookie_la_SOURCES) + +EXTRA_DIST += $(srcdir)/vmod_cookie.vcc automake_boilerplate_cookie.am + $(libvmod_cookie_la_OBJECTS): vcc_cookie_if.h vcc_cookie_if.h vmod_cookie.rst vmod_cookie.man.rst: vcc_cookie_if.c @@ -23,8 +27,8 @@ vcc_cookie_if.h vmod_cookie.rst vmod_cookie.man.rst: vcc_cookie_if.c vcc_cookie_if.c: $(vmodtool) $(srcdir)/vmod_cookie.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs_cookie) $(srcdir)/vmod_cookie.vcc -EXTRA_DIST += $(srcdir)/vmod_cookie.vcc automake_boilerplate_cookie.am +clean-local: clean-vmod-cookie -CLEANFILES += $(builddir)/vcc_cookie_if.c $(builddir)/vcc_cookie_if.h \ - $(builddir)/vmod_cookie.rst \ - $(builddir)/vmod_cookie.man.rst +clean-vmod-cookie: + rm -f $(nodist_libvmod_cookie_la_SOURCES) + rm -f vmod_cookie.rst vmod_cookie.man.rst diff --git a/vmod/automake_boilerplate_debug.am b/vmod/automake_boilerplate_debug.am index c3f2a4a46..950cf4ede 100644 --- a/vmod/automake_boilerplate_debug.am +++ b/vmod/automake_boilerplate_debug.am @@ -16,6 +16,10 @@ libvmod_debug_la_LDFLAGS = \ nodist_libvmod_debug_la_SOURCES = vcc_debug_if.c vcc_debug_if.h +EXTRA_libvmod_debug_la_DEPENDENCIES = $(nodist_libvmod_debug_la_SOURCES) + +EXTRA_DIST += $(srcdir)/vmod_debug.vcc automake_boilerplate_debug.am + $(libvmod_debug_la_OBJECTS): vcc_debug_if.h vcc_debug_if.h vmod_debug.rst vmod_debug.man.rst: vcc_debug_if.c @@ -23,8 +27,8 @@ vcc_debug_if.h vmod_debug.rst vmod_debug.man.rst: vcc_debug_if.c vcc_debug_if.c: $(vmodtool) $(srcdir)/vmod_debug.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs_debug) $(srcdir)/vmod_debug.vcc -EXTRA_DIST += $(srcdir)/vmod_debug.vcc automake_boilerplate_debug.am +clean-local: clean-vmod-debug -CLEANFILES += $(builddir)/vcc_debug_if.c $(builddir)/vcc_debug_if.h \ - $(builddir)/vmod_debug.rst \ - $(builddir)/vmod_debug.man.rst +clean-vmod-debug: + rm -f $(nodist_libvmod_debug_la_SOURCES) + rm -f vmod_debug.rst vmod_debug.man.rst diff --git a/vmod/automake_boilerplate_directors.am b/vmod/automake_boilerplate_directors.am index 0f6869805..5ba173349 100644 --- a/vmod/automake_boilerplate_directors.am +++ b/vmod/automake_boilerplate_directors.am @@ -16,6 +16,10 @@ libvmod_directors_la_LDFLAGS = \ nodist_libvmod_directors_la_SOURCES = vcc_directors_if.c vcc_directors_if.h +EXTRA_libvmod_directors_la_DEPENDENCIES = $(nodist_libvmod_directors_la_SOURCES) + +EXTRA_DIST += $(srcdir)/vmod_directors.vcc automake_boilerplate_directors.am + $(libvmod_directors_la_OBJECTS): vcc_directors_if.h vcc_directors_if.h vmod_directors.rst vmod_directors.man.rst: vcc_directors_if.c @@ -23,8 +27,8 @@ vcc_directors_if.h vmod_directors.rst vmod_directors.man.rst: vcc_directors_if.c vcc_directors_if.c: $(vmodtool) $(srcdir)/vmod_directors.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs_directors) $(srcdir)/vmod_directors.vcc -EXTRA_DIST += $(srcdir)/vmod_directors.vcc automake_boilerplate_directors.am +clean-local: clean-vmod-directors -CLEANFILES += $(builddir)/vcc_directors_if.c $(builddir)/vcc_directors_if.h \ - $(builddir)/vmod_directors.rst \ - $(builddir)/vmod_directors.man.rst +clean-vmod-directors: + rm -f $(nodist_libvmod_directors_la_SOURCES) + rm -f vmod_directors.rst vmod_directors.man.rst diff --git a/vmod/automake_boilerplate_proxy.am b/vmod/automake_boilerplate_proxy.am index 9e1ae138a..83f2217c0 100644 --- a/vmod/automake_boilerplate_proxy.am +++ b/vmod/automake_boilerplate_proxy.am @@ -16,6 +16,10 @@ libvmod_proxy_la_LDFLAGS = \ nodist_libvmod_proxy_la_SOURCES = vcc_proxy_if.c vcc_proxy_if.h +EXTRA_libvmod_proxy_la_DEPENDENCIES = $(nodist_libvmod_proxy_la_SOURCES) + +EXTRA_DIST += $(srcdir)/vmod_proxy.vcc automake_boilerplate_proxy.am + $(libvmod_proxy_la_OBJECTS): vcc_proxy_if.h vcc_proxy_if.h vmod_proxy.rst vmod_proxy.man.rst: vcc_proxy_if.c @@ -23,8 +27,8 @@ vcc_proxy_if.h vmod_proxy.rst vmod_proxy.man.rst: vcc_proxy_if.c vcc_proxy_if.c: $(vmodtool) $(srcdir)/vmod_proxy.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs_proxy) $(srcdir)/vmod_proxy.vcc -EXTRA_DIST += $(srcdir)/vmod_proxy.vcc automake_boilerplate_proxy.am +clean-local: clean-vmod-proxy -CLEANFILES += $(builddir)/vcc_proxy_if.c $(builddir)/vcc_proxy_if.h \ - $(builddir)/vmod_proxy.rst \ - $(builddir)/vmod_proxy.man.rst +clean-vmod-proxy: + rm -f $(nodist_libvmod_proxy_la_SOURCES) + rm -f vmod_proxy.rst vmod_proxy.man.rst diff --git a/vmod/automake_boilerplate_purge.am b/vmod/automake_boilerplate_purge.am index 3256cd271..afe7bf60b 100644 --- a/vmod/automake_boilerplate_purge.am +++ b/vmod/automake_boilerplate_purge.am @@ -16,6 +16,10 @@ libvmod_purge_la_LDFLAGS = \ nodist_libvmod_purge_la_SOURCES = vcc_purge_if.c vcc_purge_if.h +EXTRA_libvmod_purge_la_DEPENDENCIES = $(nodist_libvmod_purge_la_SOURCES) + +EXTRA_DIST += $(srcdir)/vmod_purge.vcc automake_boilerplate_purge.am + $(libvmod_purge_la_OBJECTS): vcc_purge_if.h vcc_purge_if.h vmod_purge.rst vmod_purge.man.rst: vcc_purge_if.c @@ -23,8 +27,8 @@ vcc_purge_if.h vmod_purge.rst vmod_purge.man.rst: vcc_purge_if.c vcc_purge_if.c: $(vmodtool) $(srcdir)/vmod_purge.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs_purge) $(srcdir)/vmod_purge.vcc -EXTRA_DIST += $(srcdir)/vmod_purge.vcc automake_boilerplate_purge.am +clean-local: clean-vmod-purge -CLEANFILES += $(builddir)/vcc_purge_if.c $(builddir)/vcc_purge_if.h \ - $(builddir)/vmod_purge.rst \ - $(builddir)/vmod_purge.man.rst +clean-vmod-purge: + rm -f $(nodist_libvmod_purge_la_SOURCES) + rm -f vmod_purge.rst vmod_purge.man.rst diff --git a/vmod/automake_boilerplate_std.am b/vmod/automake_boilerplate_std.am index 777280cf2..a5760bf78 100644 --- a/vmod/automake_boilerplate_std.am +++ b/vmod/automake_boilerplate_std.am @@ -16,6 +16,10 @@ libvmod_std_la_LDFLAGS = \ nodist_libvmod_std_la_SOURCES = vcc_std_if.c vcc_std_if.h +EXTRA_libvmod_std_la_DEPENDENCIES = $(nodist_libvmod_std_la_SOURCES) + +EXTRA_DIST += $(srcdir)/vmod_std.vcc automake_boilerplate_std.am + $(libvmod_std_la_OBJECTS): vcc_std_if.h vcc_std_if.h vmod_std.rst vmod_std.man.rst: vcc_std_if.c @@ -23,8 +27,8 @@ vcc_std_if.h vmod_std.rst vmod_std.man.rst: vcc_std_if.c vcc_std_if.c: $(vmodtool) $(srcdir)/vmod_std.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs_std) $(srcdir)/vmod_std.vcc -EXTRA_DIST += $(srcdir)/vmod_std.vcc automake_boilerplate_std.am +clean-local: clean-vmod-std -CLEANFILES += $(builddir)/vcc_std_if.c $(builddir)/vcc_std_if.h \ - $(builddir)/vmod_std.rst \ - $(builddir)/vmod_std.man.rst +clean-vmod-std: + rm -f $(nodist_libvmod_std_la_SOURCES) + rm -f vmod_std.rst vmod_std.man.rst diff --git a/vmod/automake_boilerplate_unix.am b/vmod/automake_boilerplate_unix.am index 6281665b2..b2937a270 100644 --- a/vmod/automake_boilerplate_unix.am +++ b/vmod/automake_boilerplate_unix.am @@ -16,6 +16,10 @@ libvmod_unix_la_LDFLAGS = \ nodist_libvmod_unix_la_SOURCES = vcc_unix_if.c vcc_unix_if.h +EXTRA_libvmod_unix_la_DEPENDENCIES = $(nodist_libvmod_unix_la_SOURCES) + +EXTRA_DIST += $(srcdir)/vmod_unix.vcc automake_boilerplate_unix.am + $(libvmod_unix_la_OBJECTS): vcc_unix_if.h vcc_unix_if.h vmod_unix.rst vmod_unix.man.rst: vcc_unix_if.c @@ -23,8 +27,8 @@ vcc_unix_if.h vmod_unix.rst vmod_unix.man.rst: vcc_unix_if.c vcc_unix_if.c: $(vmodtool) $(srcdir)/vmod_unix.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs_unix) $(srcdir)/vmod_unix.vcc -EXTRA_DIST += $(srcdir)/vmod_unix.vcc automake_boilerplate_unix.am +clean-local: clean-vmod-unix -CLEANFILES += $(builddir)/vcc_unix_if.c $(builddir)/vcc_unix_if.h \ - $(builddir)/vmod_unix.rst \ - $(builddir)/vmod_unix.man.rst +clean-vmod-unix: + rm -f $(nodist_libvmod_unix_la_SOURCES) + rm -f vmod_unix.rst vmod_unix.man.rst diff --git a/vmod/automake_boilerplate_vtc.am b/vmod/automake_boilerplate_vtc.am index 9b71914b0..e8479ba10 100644 --- a/vmod/automake_boilerplate_vtc.am +++ b/vmod/automake_boilerplate_vtc.am @@ -16,6 +16,10 @@ libvmod_vtc_la_LDFLAGS = \ nodist_libvmod_vtc_la_SOURCES = vcc_vtc_if.c vcc_vtc_if.h +EXTRA_libvmod_vtc_la_DEPENDENCIES = $(nodist_libvmod_vtc_la_SOURCES) + +EXTRA_DIST += $(srcdir)/vmod_vtc.vcc automake_boilerplate_vtc.am + $(libvmod_vtc_la_OBJECTS): vcc_vtc_if.h vcc_vtc_if.h vmod_vtc.rst vmod_vtc.man.rst: vcc_vtc_if.c @@ -23,8 +27,8 @@ vcc_vtc_if.h vmod_vtc.rst vmod_vtc.man.rst: vcc_vtc_if.c vcc_vtc_if.c: $(vmodtool) $(srcdir)/vmod_vtc.vcc @PYTHON@ $(vmodtool) $(vmodtoolargs_vtc) $(srcdir)/vmod_vtc.vcc -EXTRA_DIST += $(srcdir)/vmod_vtc.vcc automake_boilerplate_vtc.am +clean-local: clean-vmod-vtc -CLEANFILES += $(builddir)/vcc_vtc_if.c $(builddir)/vcc_vtc_if.h \ - $(builddir)/vmod_vtc.rst \ - $(builddir)/vmod_vtc.man.rst +clean-vmod-vtc: + rm -f $(nodist_libvmod_vtc_la_SOURCES) + rm -f vmod_vtc.rst vmod_vtc.man.rst From dridi.boukelmoune at gmail.com Thu Jan 7 14:16:08 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 7 Jan 2021 14:16:08 +0000 (UTC) Subject: [master] 9458b5287 build: rm vmod/hex.h Message-ID: <20210107141608.1CAD6A928D@lists.varnish-cache.org> commit 9458b5287ab95a837bc32cc524a9ff9784dd0315 Author: Dridi Boukelmoune Date: Thu Jan 7 11:38:48 2021 +0100 build: rm vmod/hex.h diff --git a/vmod/Makefile.am b/vmod/Makefile.am index 8a4ab5f41..8888a24c0 100644 --- a/vmod/Makefile.am +++ b/vmod/Makefile.am @@ -22,7 +22,6 @@ libvmod_blob_la_SOURCES = \ vmod_blob.h \ id.c \ base64.c \ - hex.h \ hex.c \ url.c \ tbl_encodings.h \ diff --git a/vmod/base64.c b/vmod/base64.c index 909dc11e3..06d6b6cef 100644 --- a/vmod/base64.c +++ b/vmod/base64.c @@ -36,9 +36,6 @@ #include "vmod_blob.h" -#define ILL ((int8_t) 127) -#define PAD ((int8_t) 126) - static const struct b64_alphabet { const char b64[64]; const int8_t i64[256]; diff --git a/vmod/hex.c b/vmod/hex.c index b95e45e90..539ab87a7 100644 --- a/vmod/hex.c +++ b/vmod/hex.c @@ -38,8 +38,6 @@ #include "vmod_blob.h" -#include "hex.h" - const char hex_alphabet[][16] = { "0123456789abcdef", "0123456789ABCDEF" @@ -50,7 +48,7 @@ const char hex_alphabet[][16] = { * hex digits with their binary values. This fits all of the hex digits * into 55 bytes (cacheline friendly). */ -const uint8_t nibble[] = { +const uint8_t hex_nibble[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ILL, ILL, ILL, ILL, ILL, ILL, ILL, 10, 11, 12, 13, 14, 15, ILL, ILL, ILL, ILL, ILL, ILL, ILL, @@ -74,7 +72,7 @@ hex_decode_l(size_t l) static inline char hex2byte(const unsigned char hi, const unsigned char lo) { - return ((nibble[hi - '0'] << 4) | nibble[lo - '0']); + return ((hex_nibble[hi - '0'] << 4) | hex_nibble[lo - '0']); } ssize_t diff --git a/vmod/hex.h b/vmod/hex.h deleted file mode 100644 index 203a95aaa..000000000 --- a/vmod/hex.h +++ /dev/null @@ -1,39 +0,0 @@ -/*- - * Copyright 2016 UPLEX - Nils Goroll Systemoptimierung - * All rights reserved. - * - * Authors: Nils Goroll - * Geoffrey Simmons - * - * SPDX-License-Identifier: BSD-2-Clause - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -#include - -#define ILL ((int8_t) 127) - -/* These are defined in hex.c */ - -extern const char hex_alphabet[][16]; - -extern const uint8_t nibble[]; diff --git a/vmod/url.c b/vmod/url.c index 79fdc39b5..9ce272ed8 100644 --- a/vmod/url.c +++ b/vmod/url.c @@ -36,8 +36,6 @@ #include "vmod_blob.h" -#include "hex.h" - /* Decoder states */ enum state_e { NORMAL, @@ -155,7 +153,7 @@ url_decode(const enum encoding dec, blob_dest_t buf, break; case PERCENT: if (isoutofrange(*s) || - (nib = nibble[*s - '0']) == ILL) { + (nib = hex_nibble[*s - '0']) == ILL) { errno = EINVAL; return (-1); } @@ -167,7 +165,7 @@ url_decode(const enum encoding dec, blob_dest_t buf, return (-1); } if (isoutofrange(*s) || - (nib2 = nibble[*s - '0']) == ILL) { + (nib2 = hex_nibble[*s - '0']) == ILL) { errno = EINVAL; return (-1); } diff --git a/vmod/vmod_blob.h b/vmod/vmod_blob.h index fd728a93f..4574bdf9f 100644 --- a/vmod/vmod_blob.h +++ b/vmod/vmod_blob.h @@ -91,6 +91,11 @@ ssize_t encode_f(const enum encoding enc, const enum case_e kase, blob_dest_t buf, blob_len_t buflen, blob_src_t in, blob_len_t inlen); +/* Special lookup characters */ + +#define ILL ((int8_t) 127) +#define PAD ((int8_t) 126) + /* * General interface for a decoder: decode the concatenation of strings * (obtained from STRANDS) into buf, and return the length of decoded @@ -135,6 +140,9 @@ len_f hex_decode_l; encode_f hex_encode; decode_f hex_decode; +extern const char hex_alphabet[][16]; +extern const uint8_t hex_nibble[]; + /* url.c */ len_f url_encode_l; len_f url_decode_l; From dridi.boukelmoune at gmail.com Thu Jan 7 14:16:08 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 7 Jan 2021 14:16:08 +0000 (UTC) Subject: [master] 4160eec6e build: Merge vdir.c and misc.c into vmod_directors.c Message-ID: <20210107141608.3AC62A9291@lists.varnish-cache.org> commit 4160eec6ef923e5208bfcbc8994824fc17f9631a Author: Dridi Boukelmoune Date: Thu Jan 7 11:50:42 2021 +0100 build: Merge vdir.c and misc.c into vmod_directors.c diff --git a/vmod/Makefile.am b/vmod/Makefile.am index 8888a24c0..32331da09 100644 --- a/vmod/Makefile.am +++ b/vmod/Makefile.am @@ -36,11 +36,10 @@ libvmod_debug_la_SOURCES = \ vmod_debug_dyn.c libvmod_directors_la_SOURCES = \ - vdir.c \ - vdir.h \ + vmod_directors.c \ + vmod_directors.h \ fall_back.c \ hash.c \ - misc.c \ random.c \ round_robin.c \ vmod_shard.c \ diff --git a/vmod/fall_back.c b/vmod/fall_back.c index 425c1093a..ac79dae47 100644 --- a/vmod/fall_back.c +++ b/vmod/fall_back.c @@ -37,7 +37,7 @@ #include "vcc_directors_if.h" -#include "vdir.h" +#include "vmod_directors.h" #include "vsb.h" #include "vbm.h" diff --git a/vmod/hash.c b/vmod/hash.c index eeddffca2..2ab92fb7c 100644 --- a/vmod/hash.c +++ b/vmod/hash.c @@ -35,7 +35,7 @@ #include "cache/cache.h" -#include "vdir.h" +#include "vmod_directors.h" #include "vcc_directors_if.h" diff --git a/vmod/misc.c b/vmod/misc.c deleted file mode 100644 index f43875d61..000000000 --- a/vmod/misc.c +++ /dev/null @@ -1,49 +0,0 @@ -/*- - * Copyright 2019 UPLEX - Nils Goroll Systemoptimierung - * All rights reserved. - * - * Author: Nils Goroll - * - * SPDX-License-Identifier: BSD-2-Clause - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include "config.h" - -#include "vdef.h" -#include "vrt.h" -#include "vcl.h" - -#include "vcc_directors_if.h" - -VCL_BACKEND -VPFX(lookup)(VRT_CTX, VCL_STRING name) -{ - if ((ctx->method & VCL_MET_TASK_H) == 0) { - VRT_fail(ctx, - "lookup() may only be called from vcl_init / vcl_fini"); - return (NULL); - } - - return (VRT_LookupDirector(ctx, name)); -} diff --git a/vmod/random.c b/vmod/random.c index b18e1f6fd..8fe711931 100644 --- a/vmod/random.c +++ b/vmod/random.c @@ -37,7 +37,7 @@ #include "vbm.h" #include "vrnd.h" -#include "vdir.h" +#include "vmod_directors.h" #include "vcc_directors_if.h" diff --git a/vmod/round_robin.c b/vmod/round_robin.c index 230bcb02e..b365a7116 100644 --- a/vmod/round_robin.c +++ b/vmod/round_robin.c @@ -37,7 +37,7 @@ #include "vcc_directors_if.h" -#include "vdir.h" +#include "vmod_directors.h" struct vmod_directors_round_robin { unsigned magic; diff --git a/vmod/vdir.c b/vmod/vmod_directors.c similarity index 95% rename from vmod/vdir.c rename to vmod/vmod_directors.c index a4a24212c..c9ed4fc57 100644 --- a/vmod/vdir.c +++ b/vmod/vmod_directors.c @@ -1,8 +1,10 @@ /*- * Copyright (c) 2013-2015 Varnish Software AS + * Copyright 2019 UPLEX - Nils Goroll Systemoptimierung * All rights reserved. * * Author: Poul-Henning Kamp + * Author: Nils Goroll * * SPDX-License-Identifier: BSD-2-Clause * @@ -36,9 +38,24 @@ #include "cache/cache.h" #include "vbm.h" +#include "vcl.h" #include "vsb.h" -#include "vdir.h" +#include "vcc_directors_if.h" + +#include "vmod_directors.h" + +VCL_BACKEND +VPFX(lookup)(VRT_CTX, VCL_STRING name) +{ + if ((ctx->method & VCL_MET_TASK_H) == 0) { + VRT_fail(ctx, + "lookup() may only be called from vcl_init / vcl_fini"); + return (NULL); + } + + return (VRT_LookupDirector(ctx, name)); +} static void vdir_expand(struct vdir *vd, unsigned n) diff --git a/vmod/vdir.h b/vmod/vmod_directors.h similarity index 100% rename from vmod/vdir.h rename to vmod/vmod_directors.h From dridi.boukelmoune at gmail.com Thu Jan 7 14:16:08 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 7 Jan 2021 14:16:08 +0000 (UTC) Subject: [master] 1b49d5687 build: Prefix all VMOD sources with vmod_ Message-ID: <20210107141608.5E897A929E@lists.varnish-cache.org> commit 1b49d56877e81d09c352a456fdcb11fe977cf673 Author: Dridi Boukelmoune Date: Thu Jan 7 12:14:58 2021 +0100 build: Prefix all VMOD sources with vmod_ diff --git a/vmod/Makefile.am b/vmod/Makefile.am index 32331da09..6557b366e 100644 --- a/vmod/Makefile.am +++ b/vmod/Makefile.am @@ -20,12 +20,12 @@ AM_CPPFLAGS = \ libvmod_blob_la_SOURCES = \ vmod_blob.c \ vmod_blob.h \ - id.c \ - base64.c \ - hex.c \ - url.c \ - tbl_encodings.h \ - tbl_case.h + vmod_blob_id.c \ + vmod_blob_base64.c \ + vmod_blob_hex.c \ + vmod_blob_url.c \ + vmod_blob_tbl_encodings.h \ + vmod_blob_tbl_case.h libvmod_cookie_la_SOURCES = \ vmod_cookie.c @@ -38,15 +38,15 @@ libvmod_debug_la_SOURCES = \ libvmod_directors_la_SOURCES = \ vmod_directors.c \ vmod_directors.h \ - fall_back.c \ - hash.c \ - random.c \ - round_robin.c \ - vmod_shard.c \ - shard_cfg.c \ - shard_cfg.h \ - shard_dir.c \ - shard_dir.h + vmod_directors_fall_back.c \ + vmod_directors_hash.c \ + vmod_directors_random.c \ + vmod_directors_round_robin.c \ + vmod_directors_shard.c \ + vmod_directors_shard_cfg.c \ + vmod_directors_vmod_directors_shard_cfg.h \ + vmod_directors_shard_dir.c \ + vmod_directors_shard_dir.h libvmod_proxy_la_SOURCES = \ vmod_proxy.c @@ -62,7 +62,7 @@ libvmod_std_la_SOURCES = \ libvmod_unix_la_SOURCES = \ vmod_unix.c \ - cred_compat.h + vmod_unix_cred_compat.h libvmod_vtc_la_SOURCES = \ vmod_vtc.c diff --git a/vmod/flint.lnt b/vmod/flint.lnt index ac5724d6c..34dfed208 100644 --- a/vmod/flint.lnt +++ b/vmod/flint.lnt @@ -2,7 +2,7 @@ * vmod_{blob,directors} */ --efile(451, "tbl_*.h") // No include guard +-efile(451, "vmod_*_tbl_*.h") // No include guard /* * vmod_blob diff --git a/vmod/vmod_blob.c b/vmod/vmod_blob.c index c9c3dac2e..e1f474c26 100644 --- a/vmod/vmod_blob.c +++ b/vmod/vmod_blob.c @@ -121,7 +121,7 @@ static enum encoding parse_encoding(VCL_ENUM e) { #define VMODENUM(n) if (e == VENUM(n)) return (n); -#include "tbl_encodings.h" +#include "vmod_blob_tbl_encodings.h" WRONG("illegal encoding enum"); } @@ -129,7 +129,7 @@ static enum case_e parse_case(VCL_ENUM e) { #define VMODENUM(n) if (e == VENUM(n)) return (n); -#include "tbl_case.h" +#include "vmod_blob_tbl_case.h" WRONG("illegal case enum"); } diff --git a/vmod/vmod_blob.h b/vmod/vmod_blob.h index 4574bdf9f..90872b61a 100644 --- a/vmod/vmod_blob.h +++ b/vmod/vmod_blob.h @@ -36,7 +36,7 @@ enum encoding { __INVALID_ENCODING = 0, #define VMODENUM(x) x, -#include "tbl_encodings.h" +#include "vmod_blob_tbl_encodings.h" __MAX_ENCODING }; @@ -48,7 +48,7 @@ enum encoding { */ enum case_e { #define VMODENUM(x) x, -#include "tbl_case.h" +#include "vmod_blob_tbl_case.h" }; typedef const size_t blob_len_t; diff --git a/vmod/base64.c b/vmod/vmod_blob_base64.c similarity index 100% rename from vmod/base64.c rename to vmod/vmod_blob_base64.c diff --git a/vmod/hex.c b/vmod/vmod_blob_hex.c similarity index 100% rename from vmod/hex.c rename to vmod/vmod_blob_hex.c diff --git a/vmod/id.c b/vmod/vmod_blob_id.c similarity index 100% rename from vmod/id.c rename to vmod/vmod_blob_id.c diff --git a/vmod/tbl_case.h b/vmod/vmod_blob_tbl_case.h similarity index 100% rename from vmod/tbl_case.h rename to vmod/vmod_blob_tbl_case.h diff --git a/vmod/tbl_encodings.h b/vmod/vmod_blob_tbl_encodings.h similarity index 100% rename from vmod/tbl_encodings.h rename to vmod/vmod_blob_tbl_encodings.h diff --git a/vmod/url.c b/vmod/vmod_blob_url.c similarity index 100% rename from vmod/url.c rename to vmod/vmod_blob_url.c diff --git a/vmod/fall_back.c b/vmod/vmod_directors_fall_back.c similarity index 100% rename from vmod/fall_back.c rename to vmod/vmod_directors_fall_back.c diff --git a/vmod/hash.c b/vmod/vmod_directors_hash.c similarity index 100% rename from vmod/hash.c rename to vmod/vmod_directors_hash.c diff --git a/vmod/random.c b/vmod/vmod_directors_random.c similarity index 100% rename from vmod/random.c rename to vmod/vmod_directors_random.c diff --git a/vmod/round_robin.c b/vmod/vmod_directors_round_robin.c similarity index 100% rename from vmod/round_robin.c rename to vmod/vmod_directors_round_robin.c diff --git a/vmod/vmod_shard.c b/vmod/vmod_directors_shard.c similarity index 99% rename from vmod/vmod_shard.c rename to vmod/vmod_directors_shard.c index a88095c26..d12df4476 100644 --- a/vmod/vmod_shard.c +++ b/vmod/vmod_directors_shard.c @@ -40,8 +40,8 @@ #include "vend.h" #include "vcc_directors_if.h" -#include "shard_dir.h" -#include "shard_cfg.h" +#include "vmod_directors_shard_dir.h" +#include "vmod_directors_shard_cfg.h" #include "vsb.h" /* ------------------------------------------------------------------------- diff --git a/vmod/shard_cfg.c b/vmod/vmod_directors_shard_cfg.c similarity index 99% rename from vmod/shard_cfg.c rename to vmod/vmod_directors_shard_cfg.c index 1a0cb40e4..680a8cf04 100644 --- a/vmod/shard_cfg.c +++ b/vmod/vmod_directors_shard_cfg.c @@ -38,8 +38,8 @@ #include "cache/cache.h" -#include "shard_dir.h" -#include "shard_cfg.h" +#include "vmod_directors_shard_dir.h" +#include "vmod_directors_shard_cfg.h" /*lint -esym(749, shard_change_task_e::*) */ enum shard_change_task_e { diff --git a/vmod/shard_cfg.h b/vmod/vmod_directors_shard_cfg.h similarity index 100% rename from vmod/shard_cfg.h rename to vmod/vmod_directors_shard_cfg.h diff --git a/vmod/shard_dir.c b/vmod/vmod_directors_shard_dir.c similarity index 99% rename from vmod/shard_dir.c rename to vmod/vmod_directors_shard_dir.c index 00b61aa98..9b82722d1 100644 --- a/vmod/shard_dir.c +++ b/vmod/vmod_directors_shard_dir.c @@ -45,7 +45,7 @@ #include "vrnd.h" #include "vcc_directors_if.h" -#include "shard_dir.h" +#include "vmod_directors_shard_dir.h" struct shard_be_info { unsigned hostid; diff --git a/vmod/shard_dir.h b/vmod/vmod_directors_shard_dir.h similarity index 100% rename from vmod/shard_dir.h rename to vmod/vmod_directors_shard_dir.h diff --git a/vmod/vmod_unix.c b/vmod/vmod_unix.c index 0dfbd5ae2..1edfca9d5 100644 --- a/vmod/vmod_unix.c +++ b/vmod/vmod_unix.c @@ -37,7 +37,7 @@ #include "vcl.h" #include "common/heritage.h" -#include "cred_compat.h" +#include "vmod_unix_cred_compat.h" #include "vcc_unix_if.h" #define FAIL(ctx, msg) \ diff --git a/vmod/cred_compat.h b/vmod/vmod_unix_cred_compat.h similarity index 100% rename from vmod/cred_compat.h rename to vmod/vmod_unix_cred_compat.h From dridi.boukelmoune at gmail.com Thu Jan 7 14:16:08 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 7 Jan 2021 14:16:08 +0000 (UTC) Subject: [master] a4812afc4 build: Let vmodtool generate the sources listing Message-ID: <20210107141608.787E7A92A3@lists.varnish-cache.org> commit a4812afc4cb90d8de54217461812c58dd8b7e9da Author: Dridi Boukelmoune Date: Thu Jan 7 12:21:32 2021 +0100 build: Let vmodtool generate the sources listing Now that all VMOD sources are prefixed consistently we no longer need to maintain the lists manually. diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 8f2ec9ffb..7247b0bde 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -53,6 +53,9 @@ AMBOILERPLATE = '''\ vmod_LTLIBRARIES += libvmod_XXX.la +libvmod_XXX_la_SOURCES = \\ +\tSRC + libvmod_XXX_la_CFLAGS = \\ \t at SAN_CFLAGS@ @@ -971,11 +974,14 @@ class vcc(object): def amboilerplate(self): ''' Produce boilplate for autocrap tools ''' vcc = os.path.basename(self.inputfile) + src = glob.glob("vmod_" + self.modname + "*.[ch]") + src.sort() fn = "automake_boilerplate_" + self.modname + ".am" fo = self.openfile(fn) fo.write(AMBOILERPLATE.replace("XXX", self.modname) .replace("VCC", vcc) - .replace("PFX", self.pfx)) + .replace("PFX", self.pfx) + .replace("SRC", " \\\n\t".join(src))) fo.close() def mkdefs(self, fo): diff --git a/vmod/Makefile.am b/vmod/Makefile.am index 6557b366e..d204d9bc5 100644 --- a/vmod/Makefile.am +++ b/vmod/Makefile.am @@ -13,60 +13,6 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/bin/varnishd \ -I$(top_builddir)/include -# -# Declare vmod sources -# - -libvmod_blob_la_SOURCES = \ - vmod_blob.c \ - vmod_blob.h \ - vmod_blob_id.c \ - vmod_blob_base64.c \ - vmod_blob_hex.c \ - vmod_blob_url.c \ - vmod_blob_tbl_encodings.h \ - vmod_blob_tbl_case.h - -libvmod_cookie_la_SOURCES = \ - vmod_cookie.c - -libvmod_debug_la_SOURCES = \ - vmod_debug.c \ - vmod_debug_obj.c \ - vmod_debug_dyn.c - -libvmod_directors_la_SOURCES = \ - vmod_directors.c \ - vmod_directors.h \ - vmod_directors_fall_back.c \ - vmod_directors_hash.c \ - vmod_directors_random.c \ - vmod_directors_round_robin.c \ - vmod_directors_shard.c \ - vmod_directors_shard_cfg.c \ - vmod_directors_vmod_directors_shard_cfg.h \ - vmod_directors_shard_dir.c \ - vmod_directors_shard_dir.h - -libvmod_proxy_la_SOURCES = \ - vmod_proxy.c - -libvmod_purge_la_SOURCES = \ - vmod_purge.c - -libvmod_std_la_SOURCES = \ - vmod_std.c \ - vmod_std_conversions.c \ - vmod_std_fileread.c \ - vmod_std_querysort.c - -libvmod_unix_la_SOURCES = \ - vmod_unix.c \ - vmod_unix_cred_compat.h - -libvmod_vtc_la_SOURCES = \ - vmod_vtc.c - # # Use vmodtool.py generated automake boilerplate # diff --git a/vmod/automake_boilerplate_blob.am b/vmod/automake_boilerplate_blob.am index 81ec37e56..d604614ac 100644 --- a/vmod/automake_boilerplate_blob.am +++ b/vmod/automake_boilerplate_blob.am @@ -2,6 +2,16 @@ vmod_LTLIBRARIES += libvmod_blob.la +libvmod_blob_la_SOURCES = \ + vmod_blob.c \ + vmod_blob.h \ + vmod_blob_base64.c \ + vmod_blob_hex.c \ + vmod_blob_id.c \ + vmod_blob_tbl_case.h \ + vmod_blob_tbl_encodings.h \ + vmod_blob_url.c + libvmod_blob_la_CFLAGS = \ @SAN_CFLAGS@ diff --git a/vmod/automake_boilerplate_cookie.am b/vmod/automake_boilerplate_cookie.am index fb9b156b5..a1509d920 100644 --- a/vmod/automake_boilerplate_cookie.am +++ b/vmod/automake_boilerplate_cookie.am @@ -2,6 +2,9 @@ vmod_LTLIBRARIES += libvmod_cookie.la +libvmod_cookie_la_SOURCES = \ + vmod_cookie.c + libvmod_cookie_la_CFLAGS = \ @SAN_CFLAGS@ diff --git a/vmod/automake_boilerplate_debug.am b/vmod/automake_boilerplate_debug.am index 950cf4ede..88f89a403 100644 --- a/vmod/automake_boilerplate_debug.am +++ b/vmod/automake_boilerplate_debug.am @@ -2,6 +2,11 @@ vmod_LTLIBRARIES += libvmod_debug.la +libvmod_debug_la_SOURCES = \ + vmod_debug.c \ + vmod_debug_dyn.c \ + vmod_debug_obj.c + libvmod_debug_la_CFLAGS = \ @SAN_CFLAGS@ diff --git a/vmod/automake_boilerplate_directors.am b/vmod/automake_boilerplate_directors.am index 5ba173349..2a3618a43 100644 --- a/vmod/automake_boilerplate_directors.am +++ b/vmod/automake_boilerplate_directors.am @@ -2,6 +2,19 @@ vmod_LTLIBRARIES += libvmod_directors.la +libvmod_directors_la_SOURCES = \ + vmod_directors.c \ + vmod_directors.h \ + vmod_directors_fall_back.c \ + vmod_directors_hash.c \ + vmod_directors_random.c \ + vmod_directors_round_robin.c \ + vmod_directors_shard.c \ + vmod_directors_shard_cfg.c \ + vmod_directors_shard_cfg.h \ + vmod_directors_shard_dir.c \ + vmod_directors_shard_dir.h + libvmod_directors_la_CFLAGS = \ @SAN_CFLAGS@ diff --git a/vmod/automake_boilerplate_proxy.am b/vmod/automake_boilerplate_proxy.am index 83f2217c0..d5873e989 100644 --- a/vmod/automake_boilerplate_proxy.am +++ b/vmod/automake_boilerplate_proxy.am @@ -2,6 +2,9 @@ vmod_LTLIBRARIES += libvmod_proxy.la +libvmod_proxy_la_SOURCES = \ + vmod_proxy.c + libvmod_proxy_la_CFLAGS = \ @SAN_CFLAGS@ diff --git a/vmod/automake_boilerplate_purge.am b/vmod/automake_boilerplate_purge.am index afe7bf60b..2146f4055 100644 --- a/vmod/automake_boilerplate_purge.am +++ b/vmod/automake_boilerplate_purge.am @@ -2,6 +2,9 @@ vmod_LTLIBRARIES += libvmod_purge.la +libvmod_purge_la_SOURCES = \ + vmod_purge.c + libvmod_purge_la_CFLAGS = \ @SAN_CFLAGS@ diff --git a/vmod/automake_boilerplate_std.am b/vmod/automake_boilerplate_std.am index a5760bf78..b035ba7b7 100644 --- a/vmod/automake_boilerplate_std.am +++ b/vmod/automake_boilerplate_std.am @@ -2,6 +2,12 @@ vmod_LTLIBRARIES += libvmod_std.la +libvmod_std_la_SOURCES = \ + vmod_std.c \ + vmod_std_conversions.c \ + vmod_std_fileread.c \ + vmod_std_querysort.c + libvmod_std_la_CFLAGS = \ @SAN_CFLAGS@ diff --git a/vmod/automake_boilerplate_unix.am b/vmod/automake_boilerplate_unix.am index b2937a270..1b0274e54 100644 --- a/vmod/automake_boilerplate_unix.am +++ b/vmod/automake_boilerplate_unix.am @@ -2,6 +2,10 @@ vmod_LTLIBRARIES += libvmod_unix.la +libvmod_unix_la_SOURCES = \ + vmod_unix.c \ + vmod_unix_cred_compat.h + libvmod_unix_la_CFLAGS = \ @SAN_CFLAGS@ diff --git a/vmod/automake_boilerplate_vtc.am b/vmod/automake_boilerplate_vtc.am index e8479ba10..efe418b00 100644 --- a/vmod/automake_boilerplate_vtc.am +++ b/vmod/automake_boilerplate_vtc.am @@ -2,6 +2,9 @@ vmod_LTLIBRARIES += libvmod_vtc.la +libvmod_vtc_la_SOURCES = \ + vmod_vtc.c + libvmod_vtc_la_CFLAGS = \ @SAN_CFLAGS@ From nils.goroll at uplex.de Thu Jan 7 14:18:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 7 Jan 2021 14:18:07 +0000 (UTC) Subject: [master] 888b2ea3f flint scripts to handle one- or two-level structure Message-ID: <20210107141807.279C6AA185@lists.varnish-cache.org> commit 888b2ea3f49ef1bfd64c156fd4b527e9c345ff00 Author: Nils Goroll Date: Thu Jan 7 15:07:54 2021 +0100 flint scripts to handle one- or two-level structure diff --git a/bin/varnishadm/flint.sh b/bin/varnishadm/flint.sh index 631bb1394..e214869e8 100755 --- a/bin/varnishadm/flint.sh +++ b/bin/varnishadm/flint.sh @@ -4,6 +4,4 @@ FLOPS=' *.c ../../lib/libvarnishapi/flint.lnt ../../lib/libvarnishapi/*.c -' - -. ../../tools/flint_skel.sh +' ../../tools/flint_skel.sh diff --git a/bin/varnishd/flint.sh b/bin/varnishd/flint.sh index 2442255bc..4228abdc7 100755 --- a/bin/varnishd/flint.sh +++ b/bin/varnishd/flint.sh @@ -21,6 +21,4 @@ FLOPS=' ../../lib/libvcc/*.c ../../vmod/flint.lnt ../../vmod/*.c -' - -. ../../tools/flint_skel.sh +' ../../tools/flint_skel.sh diff --git a/bin/varnishhist/flint.sh b/bin/varnishhist/flint.sh index 631bb1394..e214869e8 100755 --- a/bin/varnishhist/flint.sh +++ b/bin/varnishhist/flint.sh @@ -4,6 +4,4 @@ FLOPS=' *.c ../../lib/libvarnishapi/flint.lnt ../../lib/libvarnishapi/*.c -' - -. ../../tools/flint_skel.sh +' ../../tools/flint_skel.sh diff --git a/bin/varnishlog/flint.sh b/bin/varnishlog/flint.sh index 74b7d70e8..6a5decb69 100755 --- a/bin/varnishlog/flint.sh +++ b/bin/varnishlog/flint.sh @@ -5,7 +5,4 @@ FLOPS=' *.c ../../lib/libvarnishapi/flint.lnt ../../lib/libvarnishapi/*.c -' - -. ../../tools/flint_skel.sh - +' ../../tools/flint_skel.sh diff --git a/bin/varnishncsa/flint.sh b/bin/varnishncsa/flint.sh index 631bb1394..e214869e8 100755 --- a/bin/varnishncsa/flint.sh +++ b/bin/varnishncsa/flint.sh @@ -4,6 +4,4 @@ FLOPS=' *.c ../../lib/libvarnishapi/flint.lnt ../../lib/libvarnishapi/*.c -' - -. ../../tools/flint_skel.sh +' ../../tools/flint_skel.sh diff --git a/bin/varnishstat/flint.sh b/bin/varnishstat/flint.sh index 67dd342d4..4663fc1de 100755 --- a/bin/varnishstat/flint.sh +++ b/bin/varnishstat/flint.sh @@ -7,6 +7,4 @@ FLOPS=' ../../lib/libvarnishapi/flint.lnt ../../lib/libvarnishapi/*.c -' - -. ../../tools/flint_skel.sh +' ../../tools/flint_skel.sh diff --git a/bin/varnishtest/flint.sh b/bin/varnishtest/flint.sh index 2512c7e04..0caf8a248 100755 --- a/bin/varnishtest/flint.sh +++ b/bin/varnishtest/flint.sh @@ -6,7 +6,4 @@ FLOPS=' -DTOP_BUILDDIR="foo" -I../../lib/libvgz *.c -' - -. ../../tools/flint_skel.sh - +' ../../tools/flint_skel.sh diff --git a/bin/varnishtop/flint.sh b/bin/varnishtop/flint.sh index 631bb1394..e214869e8 100755 --- a/bin/varnishtop/flint.sh +++ b/bin/varnishtop/flint.sh @@ -4,6 +4,4 @@ FLOPS=' *.c ../../lib/libvarnishapi/flint.lnt ../../lib/libvarnishapi/*.c -' - -. ../../tools/flint_skel.sh +' ../../tools/flint_skel.sh diff --git a/lib/libvarnish/flint.sh b/lib/libvarnish/flint.sh index 0964d4c89..80600044a 100755 --- a/lib/libvarnish/flint.sh +++ b/lib/libvarnish/flint.sh @@ -2,6 +2,4 @@ FLOPS=' *.c -' - -. ../../tools/flint_skel.sh +' ../../tools/flint_skel.sh diff --git a/lib/libvarnishapi/flint.sh b/lib/libvarnishapi/flint.sh index 0964d4c89..80600044a 100755 --- a/lib/libvarnishapi/flint.sh +++ b/lib/libvarnishapi/flint.sh @@ -2,6 +2,4 @@ FLOPS=' *.c -' - -. ../../tools/flint_skel.sh +' ../../tools/flint_skel.sh diff --git a/lib/libvcc/flint.sh b/lib/libvcc/flint.sh index 0964d4c89..80600044a 100755 --- a/lib/libvcc/flint.sh +++ b/lib/libvcc/flint.sh @@ -2,6 +2,4 @@ FLOPS=' *.c -' - -. ../../tools/flint_skel.sh +' ../../tools/flint_skel.sh diff --git a/tools/flint_skel.sh b/tools/flint_skel.sh old mode 100644 new mode 100755 index 561a7d05e..decb073ce --- a/tools/flint_skel.sh +++ b/tools/flint_skel.sh @@ -6,15 +6,27 @@ if [ "x$1" = "x-ok" -a -f _.fl ] ; then exit 0 fi +d=$(dirname $0) +l=1 +if [ $d = ../../tools ] ; then + l=2 +fi + flexelint \ -D__FLEXELINT__ \ - ../../flint.lnt \ + $(if [ $l -eq 2 ] ; then echo ../../flint.lnt ; fi) \ ../flint.lnt \ flint.lnt \ -zero \ -I. \ - -I../../include \ - -I../.. \ + $(if [ $l -eq 2 ] ; then + echo -I../../include + echo -I../.. + else + echo -I../include + echo -I.. + fi + ) \ -I/usr/local/include \ $FLOPS \ 2>&1 | tee _.fl From nils.goroll at uplex.de Thu Jan 7 14:18:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 7 Jan 2021 14:18:07 +0000 (UTC) Subject: [master] 363b1b76c flint individual vmods Message-ID: <20210107141807.52302AA189@lists.varnish-cache.org> commit 363b1b76c06e5c57c9a3c430d78b996be2b4daee Author: Nils Goroll Date: Thu Jan 7 15:09:32 2021 +0100 flint individual vmods diff --git a/vmod/flint.sh b/vmod/flint.sh index 522e30d4d..ee1d06532 100755 --- a/vmod/flint.sh +++ b/vmod/flint.sh @@ -1,8 +1,11 @@ #!/bin/sh -FLOPS=' - -I../../bin/varnishd - *.c -' - -. ../../tools/flint_skel.sh +for vmod in vmod_*.vcc ; do + vmod="${vmod%.vcc}" + vmod="${vmod#vmod_}" + echo "=====================" + echo "${vmod}" + echo "=====================" + FLOPS="-I../bin/varnishd vcc_${vmod}_if.c vmod_${vmod}*.c" \ + ../tools/flint_skel.sh +done From nils.goroll at uplex.de Thu Jan 7 14:30:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 7 Jan 2021 14:30:07 +0000 (UTC) Subject: [master] c021fbca5 do not lint vmods with varnishd Message-ID: <20210107143007.6EE2FAAC6F@lists.varnish-cache.org> commit c021fbca5133f6282ff13092030b1e35fc1bc5e1 Author: Nils Goroll Date: Thu Jan 7 15:29:13 2021 +0100 do not lint vmods with varnishd diff --git a/bin/varnishd/flint.sh b/bin/varnishd/flint.sh index 4228abdc7..05ade4472 100755 --- a/bin/varnishd/flint.sh +++ b/bin/varnishd/flint.sh @@ -19,6 +19,4 @@ FLOPS=' ../../lib/libvarnish/*.c ../../lib/libvcc/flint.lnt ../../lib/libvcc/*.c - ../../vmod/flint.lnt - ../../vmod/*.c ' ../../tools/flint_skel.sh From nils.goroll at uplex.de Thu Jan 7 15:38:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 7 Jan 2021 15:38:06 +0000 (UTC) Subject: [master] c56e4a85d vmod lint: change header format Message-ID: <20210107153806.61664AC786@lists.varnish-cache.org> commit c56e4a85d1ea3a14c859e4046d8723c9df87711c Author: Nils Goroll Date: Thu Jan 7 15:32:30 2021 +0100 vmod lint: change header format diff --git a/vmod/flint.sh b/vmod/flint.sh index ee1d06532..30896053c 100755 --- a/vmod/flint.sh +++ b/vmod/flint.sh @@ -2,10 +2,10 @@ for vmod in vmod_*.vcc ; do vmod="${vmod%.vcc}" - vmod="${vmod#vmod_}" echo "=====================" echo "${vmod}" echo "=====================" + vmod="${vmod#vmod_}" FLOPS="-I../bin/varnishd vcc_${vmod}_if.c vmod_${vmod}*.c" \ ../tools/flint_skel.sh done From nils.goroll at uplex.de Thu Jan 7 16:23:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 7 Jan 2021 16:23:07 +0000 (UTC) Subject: [master] 331a02f7a Add dependency from vmod_debug to its vsc Message-ID: <20210107162307.55BA9ADAFE@lists.varnish-cache.org> commit 331a02f7a821e838b60fd8055b639c82185fa254 Author: Nils Goroll Date: Thu Jan 7 17:21:40 2021 +0100 Add dependency from vmod_debug to its vsc Without BUILT_SOURCES, we need explicit dependencies diff --git a/vmod/Makefile.am b/vmod/Makefile.am index d204d9bc5..2884cc33b 100644 --- a/vmod/Makefile.am +++ b/vmod/Makefile.am @@ -44,6 +44,8 @@ nodist_libvmod_debug_la_SOURCES += \ VSC_debug.c \ VSC_debug.h +vmod_debug.c: VSC_debug.c + # Allow Vmod_wrong*_Data to be exported vmod_debug_symbols_regex = 'Vmod_.*_Data' From nils.goroll at uplex.de Thu Jan 7 16:29:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 7 Jan 2021 16:29:07 +0000 (UTC) Subject: [master] 988bf0dbf one dependency is never enough Message-ID: <20210107162907.D2D0CADF26@lists.varnish-cache.org> commit 988bf0dbf7f8f1ef5cb24d8db97cdba7354a4556 Author: Nils Goroll Date: Thu Jan 7 17:28:10 2021 +0100 one dependency is never enough diff --git a/vmod/Makefile.am b/vmod/Makefile.am index 2884cc33b..957e069c4 100644 --- a/vmod/Makefile.am +++ b/vmod/Makefile.am @@ -44,7 +44,7 @@ nodist_libvmod_debug_la_SOURCES += \ VSC_debug.c \ VSC_debug.h -vmod_debug.c: VSC_debug.c +vmod_debug.c: VSC_debug.c VSC_debug.h # Allow Vmod_wrong*_Data to be exported vmod_debug_symbols_regex = 'Vmod_.*_Data' From nils.goroll at uplex.de Thu Jan 7 17:23:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 7 Jan 2021 17:23:06 +0000 (UTC) Subject: [master] 5a202b8f9 Add back libvmod debug when flint'ing varnishd Message-ID: <20210107172306.E03D3AF71A@lists.varnish-cache.org> commit 5a202b8f9a818df2f48170d90479eb0047e88301 Author: Nils Goroll Date: Thu Jan 7 17:45:30 2021 +0100 Add back libvmod debug when flint'ing varnishd My intention behind c021fbca5133f6282ff13092030b1e35fc1bc5e1 was that flexelint complains about symbols being defined multiple times. Yet when we omit all vmods, we end up with lots of 759 (could be moved from header to module). So add back vmod_debug and have that refer more of the API. diff --git a/bin/varnishd/flint.sh b/bin/varnishd/flint.sh index 05ade4472..fe6311907 100755 --- a/bin/varnishd/flint.sh +++ b/bin/varnishd/flint.sh @@ -19,4 +19,8 @@ FLOPS=' ../../lib/libvarnish/*.c ../../lib/libvcc/flint.lnt ../../lib/libvcc/*.c + ../../vmod/flint.lnt + ../../vmod/vcc_debug_if.c + ../../vmod/vmod_debug*.c + ../../vmod/VSC_debug*.c ' ../../tools/flint_skel.sh diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index 7c50561f5..0e01ee3d3 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -505,8 +505,15 @@ event_warm(VRT_CTX, const struct vmod_priv *priv) { struct priv_vcl *priv_vcl; char buf[32]; + const char *p[2]; + struct strands msg[1]; - VSL(SLT_Debug, 0, "%s: VCL_EVENT_WARM", VCL_Name(ctx->vcl)); + // Using VSLs for coverage + msg->n = 2; + msg->p = p; + p[0] = VCL_Name(ctx->vcl); + p[1] = ": VCL_EVENT_WARM"; + VSLs(SLT_Debug, 0, msg); AN(ctx->msg); if (cache_param->max_esi_depth == 42) { @@ -805,6 +812,20 @@ xyzzy_sethdr(VRT_CTX, VCL_HEADER hs, VCL_STRANDS s) } } +// coverage +static void +mylog(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + if (vsl != NULL) + VSLbv(vsl, tag, fmt, ap); + else + VSLv(tag, 0, fmt, ap); + va_end(ap); +} + VCL_DURATION xyzzy_priv_perf(VRT_CTX, VCL_INT size, VCL_INT rounds) { @@ -838,7 +859,7 @@ xyzzy_priv_perf(VRT_CTX, VCL_INT size, VCL_INT rounds) d = (t1 - t0) * 1e9 / ((double)size * (double)rounds); - VSLb(ctx->vsl, SLT_Debug, + mylog(ctx->vsl, SLT_Debug, "perf size %jd rounds %jd time %.1fns check %jd", (intmax_t)size, (intmax_t)rounds, d, (uintmax_t)check); @@ -850,6 +871,7 @@ xyzzy_return_strands(VRT_CTX, VCL_STRANDS strand) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + VSLbs(ctx->vsl, SLT_Debug, strand); return (strand); } From nils.goroll at uplex.de Thu Jan 7 17:23:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 7 Jan 2021 17:23:06 +0000 (UTC) Subject: [master] 496c02569 vmod interfaces do not need stdio.h Message-ID: <20210107172307.01E9BAF71D@lists.varnish-cache.org> commit 496c025697c35549f05675d8393a37f61bdc6499 Author: Nils Goroll Date: Thu Jan 7 18:11:19 2021 +0100 vmod interfaces do not need stdio.h diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 7247b0bde..25d3f977e 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -1097,7 +1097,6 @@ class vcc(object): self.mkdefs(fx); fo.write('#include "config.h"\n') - fo.write('#include \n') for i in ["vdef", "vrt", self.pfx, "vmod_abi"]: fo.write('#include "%s.h"\n' % i) fo.write("\n") From nils.goroll at uplex.de Thu Jan 7 17:23:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 7 Jan 2021 17:23:07 +0000 (UTC) Subject: [master] 44213be66 Flexelinting Message-ID: <20210107172307.2F29EAF722@lists.varnish-cache.org> commit 44213be662a6e0b4e3a8dbe760a15a2cea93abda Author: Nils Goroll Date: Thu Jan 7 18:11:45 2021 +0100 Flexelinting diff --git a/vmod/flint.lnt b/vmod/flint.lnt index 34dfed208..4bf457011 100644 --- a/vmod/flint.lnt +++ b/vmod/flint.lnt @@ -1,8 +1,11 @@ +/* we always want config.h included even where not actually required */ +-efile(766, config.h) + /* * vmod_{blob,directors} */ - -efile(451, "vmod_*_tbl_*.h") // No include guard +-efile(537, "vmod_*_tbl_*.h") // Repeated include /* * vmod_blob From nils.goroll at uplex.de Thu Jan 7 17:23:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 7 Jan 2021 17:23:07 +0000 (UTC) Subject: [master] 195860033 gc includes Message-ID: <20210107172307.4ED9EAF726@lists.varnish-cache.org> commit 195860033dc706a3bc2e199e75669ce477a36c7e Author: Nils Goroll Date: Thu Jan 7 18:15:26 2021 +0100 gc includes diff --git a/vmod/vmod_blob.h b/vmod/vmod_blob.h index 90872b61a..3ca99c024 100644 --- a/vmod/vmod_blob.h +++ b/vmod/vmod_blob.h @@ -31,7 +31,6 @@ /* vrt.h must be included before this header (for struct strands). */ #include -#include enum encoding { __INVALID_ENCODING = 0, diff --git a/vmod/vmod_debug_dyn.c b/vmod/vmod_debug_dyn.c index 8ab467085..879a382b2 100644 --- a/vmod/vmod_debug_dyn.c +++ b/vmod/vmod_debug_dyn.c @@ -33,7 +33,6 @@ #include #include #include -#include #include #include diff --git a/vmod/vmod_unix_cred_compat.h b/vmod/vmod_unix_cred_compat.h index 9ac8438f8..c75fdb5f2 100644 --- a/vmod/vmod_unix_cred_compat.h +++ b/vmod/vmod_unix_cred_compat.h @@ -27,7 +27,6 @@ * */ -#include #include #if defined(HAVE_GETPEEREID) From nils.goroll at uplex.de Thu Jan 7 17:37:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 7 Jan 2021 17:37:07 +0000 (UTC) Subject: [master] c9a19d9fa cache_varnishd.h include vre.h via common_param.h Message-ID: <20210107173707.0DAFCB033E@lists.varnish-cache.org> commit c9a19d9fa7bd79548aeaf21535bd5fe0b9c59199 Author: Nils Goroll Date: Thu Jan 7 18:34:35 2021 +0100 cache_varnishd.h include vre.h via common_param.h diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index 0e01ee3d3..2288c4ff1 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -39,7 +39,6 @@ #include "cache/cache_varnishd.h" #include "cache/cache_filter.h" -#include "vre.h" #include "vsa.h" #include "vtim.h" #include "vcc_debug_if.h" From nils.goroll at uplex.de Thu Jan 7 17:37:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 7 Jan 2021 17:37:07 +0000 (UTC) Subject: [master] 3272b6fae Surrender on two flint warnings for now Message-ID: <20210107173707.23149B0341@lists.varnish-cache.org> commit 3272b6faeedfcf0e7dfa58cef262b5cf7f3eab69 Author: Nils Goroll Date: Thu Jan 7 18:34:39 2021 +0100 Surrender on two flint warnings for now diff --git a/vmod/flint.lnt b/vmod/flint.lnt index 4bf457011..5231099c7 100644 --- a/vmod/flint.lnt +++ b/vmod/flint.lnt @@ -23,3 +23,6 @@ -esym(765, Vmod_wrong*_Data) // external '___' (___) could be made static -esym(714, Vmod_wrong*_Data) // external '___' (___) could be made static -esym(754, foo::bar) // local struct member not referenced +// TODO - don't understand why this pops up now +-esym(526, VSC_*) // Symbol ... not defined +-esym(755, VSC_*) // global macro '___' (___) not referenced From nils.goroll at uplex.de Thu Jan 7 17:37:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 7 Jan 2021 17:37:07 +0000 (UTC) Subject: [master] 94fba10c0 Revert "gc includes" Message-ID: <20210107173707.4361AB0345@lists.varnish-cache.org> commit 94fba10c0f6abb8d58a1e887a2e1c76c385b261b Author: Nils Goroll Date: Thu Jan 7 18:36:10 2021 +0100 Revert "gc includes" needed on FreeBSD This reverts commit 195860033dc706a3bc2e199e75669ce477a36c7e. diff --git a/vmod/vmod_blob.h b/vmod/vmod_blob.h index 3ca99c024..90872b61a 100644 --- a/vmod/vmod_blob.h +++ b/vmod/vmod_blob.h @@ -31,6 +31,7 @@ /* vrt.h must be included before this header (for struct strands). */ #include +#include enum encoding { __INVALID_ENCODING = 0, diff --git a/vmod/vmod_debug_dyn.c b/vmod/vmod_debug_dyn.c index 879a382b2..8ab467085 100644 --- a/vmod/vmod_debug_dyn.c +++ b/vmod/vmod_debug_dyn.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include diff --git a/vmod/vmod_unix_cred_compat.h b/vmod/vmod_unix_cred_compat.h index c75fdb5f2..9ac8438f8 100644 --- a/vmod/vmod_unix_cred_compat.h +++ b/vmod/vmod_unix_cred_compat.h @@ -27,6 +27,7 @@ * */ +#include #include #if defined(HAVE_GETPEEREID) From phk at FreeBSD.org Thu Jan 7 21:53:11 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 7 Jan 2021 21:53:11 +0000 (UTC) Subject: [master] 377c9f6b1 Fix flint scripts Message-ID: <20210107215311.AB08A90DA@lists.varnish-cache.org> commit 377c9f6b184f99906a94df142fcae8133ed21069 Author: Poul-Henning Kamp Date: Thu Jan 7 21:52:15 2021 +0000 Fix flint scripts diff --git a/bin/varnishd/flint.sh b/bin/varnishd/flint.sh index fe6311907..47ac810e3 100755 --- a/bin/varnishd/flint.sh +++ b/bin/varnishd/flint.sh @@ -23,4 +23,4 @@ FLOPS=' ../../vmod/vcc_debug_if.c ../../vmod/vmod_debug*.c ../../vmod/VSC_debug*.c -' ../../tools/flint_skel.sh +' ../../tools/flint_skel.sh $* diff --git a/tools/flint_skel.sh b/tools/flint_skel.sh index decb073ce..bf1b3dea6 100755 --- a/tools/flint_skel.sh +++ b/tools/flint_skel.sh @@ -1,5 +1,7 @@ #!/bin/sh +set -x + if [ "x$1" = "x-ok" -a -f _.fl ] ; then echo "Saved as reference" mv _.fl _.fl.old @@ -34,8 +36,3 @@ flexelint \ if [ -f _.fl.old ] ; then diff -u _.fl.old _.fl fi - -if [ "x$1" = "x-ok" ] ; then - echo "Saved as reference" - mv _.fl _.fl.old -fi From nils.goroll at uplex.de Fri Jan 8 09:47:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 8 Jan 2021 09:47:08 +0000 (UTC) Subject: [master] 2e663f90b remove accidental leftover Message-ID: <20210108094709.13D27A3A63@lists.varnish-cache.org> commit 2e663f90b9289a011fe830798815a992e40a1424 Author: Nils Goroll Date: Fri Jan 8 10:46:00 2021 +0100 remove accidental leftover diff --git a/tools/flint_skel.sh b/tools/flint_skel.sh index bf1b3dea6..9d45f87dd 100755 --- a/tools/flint_skel.sh +++ b/tools/flint_skel.sh @@ -1,7 +1,5 @@ #!/bin/sh -set -x - if [ "x$1" = "x-ok" -a -f _.fl ] ; then echo "Saved as reference" mv _.fl _.fl.old From nils.goroll at uplex.de Fri Jan 8 15:14:02 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 8 Jan 2021 15:14:02 +0000 (UTC) Subject: [master] 2b63ee2d8 Add VSA_getsockname() Message-ID: <20210108151403.048B7AC73A@lists.varnish-cache.org> commit 2b63ee2d869cf87e2fa2a6a3df78464fa2300790 Author: Nils Goroll Date: Fri Jan 8 11:48:09 2021 +0100 Add VSA_getsockname() Flexelint on Linux exhibits exactly the problem described by phk in his rant at the top of vsa.c, quote: Do I need to tell you that static code analysis tools have a really hard time coping with (struct suckaddr_storage), and that they give a lot of false negatives which confuse people ? One instance of this issue: --- Module: cache/cache_acceptor.c (C) _ #... _assert(!!((getsockname(sp->fd, (void*)&ss, &sl)) == 0)) /*lint -restore * #... getsockname(sp->fd, (void*)&ss, &sl)) == 0); } while (0) AZ(getsockname(sp->fd, (void*)&ss, &sl)); cache/cache_acceptor.c 330 Error 64: Type mismatch (arg. no. 2) (union __SOCKADDR_ARG = void *) _ #... SA_Build(sa, &ss, sl)) != 0); } while (0) AN(VSA_Build(sa, &ss, sl)); cache/cache_acceptor.c 332 Warning 603: Symbol 'ss' (line 318) not initialized cache/cache_acceptor.c 318 Info 830: Location cited in prior message _ i = accept(ls->sock, (void*)&wa.acceptaddr, cache/cache_acceptor.c 458 Error 64: Type mismatch (arg. no. 2) (union __SOCKADDR_ARG = void *) There is one more in cache/cache_backend_probe.c As the purpose of VSA is to avoid exactly this kind of trouble, we provide a VSA-wrapper for getsockname. diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index b7cdbcca6..d7dedb6c7 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -315,8 +315,6 @@ vca_mk_tcp(const struct wrk_accept *wa, struct sess *sp, char *laddr, char *lport, char *raddr, char *rport) { struct suckaddr *sa; - struct sockaddr_storage ss; - socklen_t sl; AN(SES_Reserve_remote_addr(sp, &sa)); AN(VSA_Build(sa, &wa->acceptaddr, wa->acceptaddrlen)); @@ -326,10 +324,9 @@ vca_mk_tcp(const struct wrk_accept *wa, AN(SES_Set_String_Attr(sp, SA_CLIENT_IP, raddr)); AN(SES_Set_String_Attr(sp, SA_CLIENT_PORT, rport)); - sl = sizeof ss; - AZ(getsockname(sp->fd, (void*)&ss, &sl)); + AN(SES_Reserve_local_addr(sp, &sa)); - AN(VSA_Build(sa, &ss, sl)); + AN(VSA_getsockname(sp->fd, sa, vsa_suckaddr_len)); sp->sattr[SA_SERVER_ADDR] = sp->sattr[SA_LOCAL_ADDR]; VTCP_name(sa, laddr, VTCP_ADDRBUFSIZE, lport, VTCP_PORTBUFSIZE); } diff --git a/include/vsa.h b/include/vsa.h index eb6e4a2df..88b76f866 100644 --- a/include/vsa.h +++ b/include/vsa.h @@ -42,6 +42,7 @@ unsigned VSA_Port(const struct suckaddr *); int VSA_Compare(const struct suckaddr *, const struct suckaddr *); int VSA_Compare_IP(const struct suckaddr *, const struct suckaddr *); struct suckaddr *VSA_Clone(const struct suckaddr *sua); +struct suckaddr *VSA_getsockname(int, void *, size_t); const void *VSA_Get_Sockaddr(const struct suckaddr *, socklen_t *sl); int VSA_Get_Proto(const struct suckaddr *); diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c index a604d4201..2ff6bdaea 100644 --- a/lib/libvarnish/vsa.c +++ b/lib/libvarnish/vsa.c @@ -418,3 +418,26 @@ VSA_Port(const struct suckaddr *sua) return (0); } } + +/* VSA_Build from socket name of a file descriptor */ +struct suckaddr * +VSA_getsockname(int fd, void *d, size_t l) +{ + struct suckaddr *sua; + socklen_t sl; + int r; + + AN(d); + if (l != vsa_suckaddr_len) { + errno = EINVAL; + return (NULL); + } + + sua = d; + + INIT_OBJ(sua, SUCKADDR_MAGIC); + sl = sizeof(sua->sa); + r = getsockname(fd, &sua->sa, &sl); + + return (r == 0 ? sua : NULL); +} From dridi.boukelmoune at gmail.com Fri Jan 8 16:04:09 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 8 Jan 2021 16:04:09 +0000 (UTC) Subject: [master] 32ea9c59d vtc: Flip the feature check in v4 Message-ID: <20210108160409.80F18ADDA4@lists.varnish-cache.org> commit 32ea9c59df6beeab94bb38c0b559a58686cb7f89 Author: Dridi Boukelmoune Date: Fri Jan 8 16:58:44 2021 +0100 vtc: Flip the feature check in v4 In order to run the test we want to NOT be using gcov, so we should test that the environment variable is empty. In theory that should mean that GCOV runs should currently fail since the feature check is currently backwards so I may revert this change if that suddenly happens. I find `test -z ...` more readable than `[ -z ... ]` in this context, so if it comes to a revert, I will keep the test(1) syntax. diff --git a/bin/varnishtest/tests/v00004.vtc b/bin/varnishtest/tests/v00004.vtc index f3a28062a..1cac50de0 100644 --- a/bin/varnishtest/tests/v00004.vtc +++ b/bin/varnishtest/tests/v00004.vtc @@ -3,7 +3,7 @@ varnishtest "test if our default paramers make sense ..." feature 64bit # Skip this test in GCOV mode, 68xx bytes extra per level makes it fail -feature cmd {[ -n "$GCOV_PROG" ]} +feature cmd {test -z "$GCOV_PROG"} # ... for our definition of a standard use case: # - 2019 header madness From nils.goroll at uplex.de Fri Jan 8 16:54:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 8 Jan 2021 16:54:08 +0000 (UTC) Subject: [master] c1352b0b0 Use VSA_getsockname() - avoid sockaddr_storage Message-ID: <20210108165408.69AEDAF4BD@lists.varnish-cache.org> commit c1352b0b0ece8028317c838751d9042a55347cd5 Author: Nils Goroll Date: Fri Jan 8 16:17:52 2021 +0100 Use VSA_getsockname() - avoid sockaddr_storage avoids an second getsockname() in vbp_write_proxy_v1() avoids a memcpy() in VTCP_my_suckaddr() diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index ee9e0dbf2..d638a9f8e 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -251,20 +251,22 @@ vbp_write_proxy_v1(struct vbp_target *vt, int *sock) char buf[105]; /* maximum size for a TCP6 PROXY line with null char */ char addr[VTCP_ADDRBUFSIZE]; char port[VTCP_PORTBUFSIZE]; - struct sockaddr_storage ss; + char vsabuf[vsa_suckaddr_len]; + struct suckaddr *sua; + int proto; struct vsb vsb; - socklen_t l; - VTCP_myname(*sock, addr, sizeof addr, port, sizeof port); + sua = VSA_getsockname(*sock, vsabuf, sizeof vsabuf); + AN(sua); + VTCP_name(sua, addr, sizeof addr, port, sizeof port); AN(VSB_init(&vsb, buf, sizeof buf)); - l = sizeof ss; - AZ(getsockname(*sock, (void *)&ss, &l)); - if (ss.ss_family == AF_INET || ss.ss_family == AF_INET6) { + proto = VSA_Get_Proto(sua); + if (proto == AF_INET || proto == AF_INET6) VSB_printf(&vsb, "PROXY %s %s %s %s %s\r\n", - ss.ss_family == AF_INET ? "TCP4" : "TCP6", + proto == AF_INET ? "TCP4" : "TCP6", addr, addr, port, port); - } else + else VSB_cat(&vsb, "PROXY UNKNOWN\r\n"); AZ(VSB_finish(&vsb)); diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c index 90857c329..61b580e67 100644 --- a/lib/libvarnish/vtcp.c +++ b/lib/libvarnish/vtcp.c @@ -48,6 +48,7 @@ #include #include #include +#include #include "vdef.h" #include "vas.h" @@ -105,12 +106,8 @@ VTCP_name(const struct suckaddr *addr, char *abuf, unsigned alen, struct suckaddr * VTCP_my_suckaddr(int sock) { - struct sockaddr_storage addr_s; - socklen_t l; - - l = sizeof addr_s; - AZ(getsockname(sock, (void *)&addr_s, &l)); - return (VSA_Malloc(&addr_s, l)); + return (VSA_getsockname(sock, + malloc(vsa_suckaddr_len), vsa_suckaddr_len)); } /*--------------------------------------------------------------------*/ @@ -118,12 +115,10 @@ VTCP_my_suckaddr(int sock) void VTCP_myname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen) { - struct sockaddr_storage addr_s; - socklen_t l; + char buf[vsa_suckaddr_len]; - l = sizeof addr_s; - AZ(getsockname(sock, (void *)&addr_s, &l)); - vtcp_sa_to_ascii(&addr_s, l, abuf, alen, pbuf, plen); + VTCP_name(VSA_getsockname(sock, buf, sizeof buf), + abuf, alen, pbuf, plen); } /*--------------------------------------------------------------------*/ From nils.goroll at uplex.de Fri Jan 8 16:54:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 8 Jan 2021 16:54:08 +0000 (UTC) Subject: [master] 9cfd20660 Generalize VSA_getsockname() to VSA_getpeername() Message-ID: <20210108165408.84613AF4C0@lists.varnish-cache.org> commit 9cfd2066013bd935a1f6b43a92ad34bdb0cda6d0 Author: Nils Goroll Date: Fri Jan 8 16:51:05 2021 +0100 Generalize VSA_getsockname() to VSA_getpeername() diff --git a/include/vsa.h b/include/vsa.h index 88b76f866..e9fd9108a 100644 --- a/include/vsa.h +++ b/include/vsa.h @@ -43,6 +43,7 @@ int VSA_Compare(const struct suckaddr *, const struct suckaddr *); int VSA_Compare_IP(const struct suckaddr *, const struct suckaddr *); struct suckaddr *VSA_Clone(const struct suckaddr *sua); struct suckaddr *VSA_getsockname(int, void *, size_t); +struct suckaddr *VSA_getpeername(int, void *, size_t); const void *VSA_Get_Sockaddr(const struct suckaddr *, socklen_t *sl); int VSA_Get_Proto(const struct suckaddr *); diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c index 2ff6bdaea..921ff0f04 100644 --- a/lib/libvarnish/vsa.c +++ b/lib/libvarnish/vsa.c @@ -419,25 +419,29 @@ VSA_Port(const struct suckaddr *sua) } } -/* VSA_Build from socket name of a file descriptor */ -struct suckaddr * -VSA_getsockname(int fd, void *d, size_t l) -{ - struct suckaddr *sua; - socklen_t sl; - int r; - - AN(d); - if (l != vsa_suckaddr_len) { - errno = EINVAL; - return (NULL); - } - - sua = d; - - INIT_OBJ(sua, SUCKADDR_MAGIC); - sl = sizeof(sua->sa); - r = getsockname(fd, &sua->sa, &sl); - - return (r == 0 ? sua : NULL); -} +#define VSA_getname(which) \ +struct suckaddr * \ +VSA_get ## which ## name(int fd, void *d, size_t l) \ +{ \ + struct suckaddr *sua; \ + socklen_t sl; \ + int r; \ + \ + AN(d); \ + if (l != vsa_suckaddr_len) { \ + errno = EINVAL; \ + return (NULL); \ + } \ + \ + sua = d; \ + \ + INIT_OBJ(sua, SUCKADDR_MAGIC); \ + sl = sizeof(sua->sa); \ + r = get ## which ## name(fd, &sua->sa, &sl); \ + \ + return (r == 0 ? sua : NULL); \ +} \ + +VSA_getname(sock) +VSA_getname(peer) +#undef VSA_getname From nils.goroll at uplex.de Fri Jan 8 16:54:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 8 Jan 2021 16:54:08 +0000 (UTC) Subject: [master] bee05daeb Use VSA_getpeername() - avoid sockaddr_storage Message-ID: <20210108165408.A0B13AF4C3@lists.varnish-cache.org> commit bee05daeb1050e6087b9e8380cec3c5bed80ef47 Author: Nils Goroll Date: Fri Jan 8 16:54:50 2021 +0100 Use VSA_getpeername() - avoid sockaddr_storage diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c index 61b580e67..9f2629de6 100644 --- a/lib/libvarnish/vtcp.c +++ b/lib/libvarnish/vtcp.c @@ -126,12 +126,12 @@ VTCP_myname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen) void VTCP_hisname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen) { - struct sockaddr_storage addr_s; - socklen_t l; + char buf[vsa_suckaddr_len]; + struct suckaddr *sua; - l = sizeof addr_s; - if (!getpeername(sock, (void*)&addr_s, &l)) - vtcp_sa_to_ascii(&addr_s, l, abuf, alen, pbuf, plen); + sua = VSA_getpeername(sock, buf, sizeof buf); + if (sua != NULL) + VTCP_name(sua, abuf, alen, pbuf, plen); else { (void)snprintf(abuf, alen, ""); (void)snprintf(pbuf, plen, ""); From nils.goroll at uplex.de Fri Jan 8 16:54:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 8 Jan 2021 16:54:08 +0000 (UTC) Subject: [master] 33736b200 Additional size checking for session attributes Message-ID: <20210108165408.C10CBAF4CC@lists.varnish-cache.org> commit 33736b200b7afb5b5e2a4017f60defdbbd5caf60 Author: Nils Goroll Date: Fri Jan 8 17:21:20 2021 +0100 Additional size checking for session attributes When we reserve a session attribute, we now check that the reserved space is of the expected size. diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index d7dedb6c7..beedeaf5c 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -315,8 +315,10 @@ vca_mk_tcp(const struct wrk_accept *wa, struct sess *sp, char *laddr, char *lport, char *raddr, char *rport) { struct suckaddr *sa; + ssize_t sz; - AN(SES_Reserve_remote_addr(sp, &sa)); + AN(SES_Reserve_remote_addr(sp, &sa, &sz)); + assert(sz == vsa_suckaddr_len); AN(VSA_Build(sa, &wa->acceptaddr, wa->acceptaddrlen)); sp->sattr[SA_CLIENT_ADDR] = sp->sattr[SA_REMOTE_ADDR]; @@ -325,8 +327,8 @@ vca_mk_tcp(const struct wrk_accept *wa, AN(SES_Set_String_Attr(sp, SA_CLIENT_PORT, rport)); - AN(SES_Reserve_local_addr(sp, &sa)); - AN(VSA_getsockname(sp->fd, sa, vsa_suckaddr_len)); + AN(SES_Reserve_local_addr(sp, &sa, &sz)); + AN(VSA_getsockname(sp->fd, sa, sz)); sp->sattr[SA_SERVER_ADDR] = sp->sattr[SA_LOCAL_ADDR]; VTCP_name(sa, laddr, VTCP_ADDRBUFSIZE, lport, VTCP_PORTBUFSIZE); } @@ -336,9 +338,11 @@ vca_mk_uds(struct wrk_accept *wa, struct sess *sp, char *laddr, char *lport, char *raddr, char *rport) { struct suckaddr *sa; + ssize_t sz; (void) wa; - AN(SES_Reserve_remote_addr(sp, &sa)); + AN(SES_Reserve_remote_addr(sp, &sa, &sz)); + assert(sz == vsa_suckaddr_len); AZ(SES_Set_remote_addr(sp, bogo_ip)); sp->sattr[SA_CLIENT_ADDR] = sp->sattr[SA_REMOTE_ADDR]; sp->sattr[SA_LOCAL_ADDR] = sp->sattr[SA_REMOTE_ADDR]; diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index a50e1c295..dbbce45d0 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -115,14 +115,17 @@ ses_set_attr(const struct sess *sp, enum sess_attr a, const void *src, int sz) } static int -ses_res_attr(struct sess *sp, enum sess_attr a, void **dst, int sz) +ses_res_attr(struct sess *sp, enum sess_attr a, void **dst, ssize_t *szp) { unsigned o; + ssize_t sz; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); assert(a < SA_LAST); - assert(sz >= 0); AN(dst); + sz = *szp; + *szp = 0; + assert(sz >= 0); if (WS_ReserveSize(sp->ws, sz) == 0) return (0); o = WS_ReservationOffset(sp->ws); @@ -131,6 +134,7 @@ ses_res_attr(struct sess *sp, enum sess_attr a, void **dst, int sz) return (0); } *dst = WS_Reservation(sp->ws); + *szp = sz; sp->sattr[a] = (uint16_t)o; WS_Release(sp->ws, sz); return (1); @@ -152,10 +156,12 @@ ses_res_attr(struct sess *sp, enum sess_attr a, void **dst, int sz) } \ \ int \ - SES_Reserve_##low(struct sess *sp, typ **dst) \ + SES_Reserve_##low(struct sess *sp, typ **dst, ssize_t *sz) \ { \ assert(len > 0); \ - return (ses_res_attr(sp, SA_##UP, (void**)dst, len)); \ + AN(sz); \ + *sz = len; \ + return (ses_res_attr(sp, SA_##UP, (void**)dst, sz)); \ } #include "tbl/sess_attr.h" @@ -164,6 +170,7 @@ int SES_Set_String_Attr(struct sess *sp, enum sess_attr a, const char *src) { void *q; + ssize_t l, sz; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); AN(src); @@ -172,8 +179,10 @@ SES_Set_String_Attr(struct sess *sp, enum sess_attr a, const char *src) if (strcmp(sess_attr[a].type, "char")) WRONG("wrong sess_attr: not char"); - if (! ses_res_attr(sp, a, &q, strlen(src) + 1)) + l = sz = strlen(src) + 1; + if (! ses_res_attr(sp, a, &q, &sz)) return (0); + assert(l == sz); strcpy(q, src); return (1); } diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index e53d28a23..96fb33ff8 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -408,7 +408,7 @@ enum htc_status_e HTC_RxStuff(struct http_conn *, htc_complete_f *, #define SESS_ATTR(UP, low, typ, len) \ int SES_Set_##low(const struct sess *sp, const typ *src); \ - int SES_Reserve_##low(struct sess *sp, typ **dst); + int SES_Reserve_##low(struct sess *sp, typ **dst, ssize_t *sz); #include "tbl/sess_attr.h" int SES_Set_String_Attr(struct sess *sp, enum sess_attr a, const char *src); diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c index 1d1e7c2e6..6a339f5a4 100644 --- a/bin/varnishd/http1/cache_http1_fsm.c +++ b/bin/varnishd/http1/cache_http1_fsm.c @@ -107,6 +107,7 @@ http1_new_session(struct worker *wrk, void *arg) struct sess *sp; struct req *req; uintptr_t *u; + ssize_t sz; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CAST_OBJ_NOTNULL(req, arg, REQ_MAGIC); @@ -114,7 +115,7 @@ http1_new_session(struct worker *wrk, void *arg) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); HTC_RxInit(req->htc, req->ws); - if (!SES_Reserve_proto_priv(sp, &u)) { + if (!SES_Reserve_proto_priv(sp, &u, &sz)) { /* Out of session workspace. Free the req, close the sess, * and do not set a new task func, which will exit the * worker thread. */ @@ -123,6 +124,7 @@ http1_new_session(struct worker *wrk, void *arg) SES_Delete(sp, SC_RX_JUNK, NAN); return; } + assert(sz == sizeof u); http1_setstate(sp, H1NEWREQ); wrk->task->func = http1_req; wrk->task->priv = req; diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index 7d6cffe00..29e806826 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -75,6 +75,7 @@ vpx_proto1(const struct worker *wrk, const struct req *req) int i; char *p, *q; struct suckaddr *sa; + ssize_t sz; int pfam = -1; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -121,8 +122,9 @@ vpx_proto1(const struct worker *wrk, const struct req *req) return (-1); } - if (! SES_Reserve_client_addr(req->sp, &sa)) + if (! SES_Reserve_client_addr(req->sp, &sa, &sz)) return (vpx_ws_err(req)); + assert (sz == vsa_suckaddr_len); if (VSS_ResolveOne(sa, fld[1], fld[3], pfam, SOCK_STREAM, AI_NUMERICHOST | AI_NUMERICSERV) == NULL) { @@ -135,8 +137,9 @@ vpx_proto1(const struct worker *wrk, const struct req *req) if (! SES_Set_String_Attr(req->sp, SA_CLIENT_PORT, fld[3])) return (vpx_ws_err(req)); - if (! SES_Reserve_server_addr(req->sp, &sa)) + if (! SES_Reserve_server_addr(req->sp, &sa, &sz)) return (vpx_ws_err(req)); + assert (sz == vsa_suckaddr_len); if (VSS_ResolveOne(sa, fld[2], fld[4], pfam, SOCK_STREAM, AI_NUMERICHOST | AI_NUMERICSERV) == NULL) { @@ -329,6 +332,7 @@ vpx_proto2(const struct worker *wrk, struct req *req) char *d, *tlv_start; sa_family_t pfam = 0xff; struct suckaddr *sa = NULL; + ssize_t sz; char ha[VTCP_ADDRBUFSIZE]; char pa[VTCP_PORTBUFSIZE]; char hb[VTCP_ADDRBUFSIZE]; @@ -414,8 +418,9 @@ vpx_proto2(const struct worker *wrk, struct req *req) pp = ap + 2 * alen; /* src/client */ - if (! SES_Reserve_client_addr(req->sp, &sa)) + if (! SES_Reserve_client_addr(req->sp, &sa, &sz)) return (vpx_ws_err(req)); + assert(sz == vsa_suckaddr_len); AN(VSA_BuildFAP(sa, pfam, ap, alen, pp, plen)); VTCP_name(sa, hb, sizeof hb, pb, sizeof pb); @@ -423,8 +428,9 @@ vpx_proto2(const struct worker *wrk, struct req *req) pp += plen; /* dst/server */ - if (! SES_Reserve_server_addr(req->sp, &sa)) + if (! SES_Reserve_server_addr(req->sp, &sa, &sz)) return (vpx_ws_err(req)); + assert(sz == vsa_suckaddr_len); AN(VSA_BuildFAP(sa, pfam, ap, alen, pp, plen)); VTCP_name(sa, ha, sizeof ha, pa, sizeof pa); @@ -467,8 +473,9 @@ vpx_proto2(const struct worker *wrk, struct req *req) INIT_OBJ(tlv, VPX_TLV_MAGIC); tlv->len = tlv_len; memcpy(tlv->tlv, tlv_start, tlv_len); - if (! SES_Reserve_proxy_tlv(req->sp, &up)) + if (! SES_Reserve_proxy_tlv(req->sp, &up, &sz)) return (vpx_ws_err(req)); + assert(sz == sizeof up); *up = (uintptr_t)tlv; return (0); } From nils.goroll at uplex.de Fri Jan 8 17:27:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 8 Jan 2021 17:27:06 +0000 (UTC) Subject: [master] 4d11c0726 Fix VTCP_my_suckaddr() refactor Message-ID: <20210108172706.E4481B091F@lists.varnish-cache.org> commit 4d11c0726445c5abfc032c191da0be98f6cdc5a9 Author: Nils Goroll Date: Fri Jan 8 18:20:02 2021 +0100 Fix VTCP_my_suckaddr() refactor Before c1352b0b0ece8028317c838751d9042a55347cd5, the function asserted that getsockname() succeed. We now restore this behaviour in a way which Flexelint understands (we are returning the memory which we allocated). diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c index 9f2629de6..78a093738 100644 --- a/lib/libvarnish/vtcp.c +++ b/lib/libvarnish/vtcp.c @@ -106,8 +106,11 @@ VTCP_name(const struct suckaddr *addr, char *abuf, unsigned alen, struct suckaddr * VTCP_my_suckaddr(int sock) { - return (VSA_getsockname(sock, - malloc(vsa_suckaddr_len), vsa_suckaddr_len)); + struct suckaddr *r; + + r = malloc(vsa_suckaddr_len); + AN(VSA_getsockname(sock, r, vsa_suckaddr_len)); + return (r); } /*--------------------------------------------------------------------*/ From phk at FreeBSD.org Fri Jan 8 20:19:12 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 8 Jan 2021 20:19:12 +0000 (UTC) Subject: [master] 75d56fe13 The vtester running in a IPv6 enabled jail, attempting u000000 says: Message-ID: <20210108201912.9659D59EE@lists.varnish-cache.org> commit 75d56fe136c2bf39471e40c921b090d33346bb1a Author: Poul-Henning Kamp Date: Fri Jan 8 20:16:36 2021 +0000 The vtester running in a IPv6 enabled jail, attempting u000000 says: Connection failed (fec0:: 19925): (null) The's not an IPv6 address. *This* is an IPv6 address: fec0::205 diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c index 921ff0f04..85a3e6785 100644 --- a/lib/libvarnish/vsa.c +++ b/lib/libvarnish/vsa.c @@ -172,7 +172,7 @@ struct suckaddr { struct sockaddr sa; struct sockaddr_in sa4; struct sockaddr_in6 sa6; - }; + } u; }; const size_t vsa_suckaddr_len = sizeof(struct suckaddr); @@ -204,15 +204,15 @@ VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst) return (-1); CHECK_OBJ_NOTNULL(sua, SUCKADDR_MAGIC); - switch (sua->sa.sa_family) { + switch (sua->u.sa.sa_family) { case PF_INET: - assert(sua->sa.sa_family == sua->sa4.sin_family); - *dst = (const unsigned char *)&sua->sa4.sin_addr; - return (sua->sa4.sin_family); + assert(sua->u.sa.sa_family == sua->u.sa4.sin_family); + *dst = (const unsigned char *)&sua->u.sa4.sin_addr; + return (sua->u.sa4.sin_family); case PF_INET6: - assert(sua->sa.sa_family == sua->sa6.sin6_family); - *dst = (const unsigned char *)&sua->sa6.sin6_addr; - return (sua->sa6.sin6_family); + assert(sua->u.sa.sa_family == sua->u.sa6.sin6_family); + *dst = (const unsigned char *)&sua->u.sa6.sin6_addr; + return (sua->u.sa6.sin6_family); default: *dst = NULL; return (-1); @@ -317,17 +317,17 @@ VSA_Build(void *d, const void *s, unsigned sal) INIT_OBJ(sua, SUCKADDR_MAGIC); switch (l) { - case sizeof sua->sa4: - memcpy(&sua->sa4, s, l); + case sizeof sua->u.sa4: + memcpy(&sua->u.sa4, s, l); break; - case sizeof sua->sa6: - memcpy(&sua->sa6, s, l); + case sizeof sua->u.sa6: + memcpy(&sua->u.sa6, s, l); break; default: WRONG("VSA protocol vs. size"); } #ifdef HAVE_STRUCT_SOCKADDR_SA_LEN - sua->sa.sa_len = (unsigned char)l; + sua->u.sa.sa_len = (unsigned char)l; #endif return (sua); } @@ -339,11 +339,11 @@ VSA_Get_Sockaddr(const struct suckaddr *sua, socklen_t *slp) CHECK_OBJ_NOTNULL(sua, SUCKADDR_MAGIC); AN(slp); - sl = sua_len(&sua->sa); + sl = sua_len(&sua->u.sa); if (sl == 0) return (NULL); *slp = sl; - return (&sua->sa); + return (&sua->u.sa); } int @@ -351,13 +351,13 @@ VSA_Get_Proto(const struct suckaddr *sua) { CHECK_OBJ_NOTNULL(sua, SUCKADDR_MAGIC); - return (sua->sa.sa_family); + return (sua->u.sa.sa_family); } int VSA_Sane(const struct suckaddr *sua) { - return (VALID_OBJ(sua, SUCKADDR_MAGIC) && sua_len(&sua->sa) != 0); + return (VALID_OBJ(sua, SUCKADDR_MAGIC) && sua_len(&sua->u.sa) != 0); } int @@ -376,16 +376,16 @@ VSA_Compare_IP(const struct suckaddr *sua1, const struct suckaddr *sua2) assert(VSA_Sane(sua1)); assert(VSA_Sane(sua2)); - if (sua1->sa.sa_family != sua2->sa.sa_family) + if (sua1->u.sa.sa_family != sua2->u.sa.sa_family) return (-1); - switch (sua1->sa.sa_family) { + switch (sua1->u.sa.sa_family) { case PF_INET: - return (memcmp(&sua1->sa4.sin_addr, - &sua2->sa4.sin_addr, sizeof(struct in_addr))); + return (memcmp(&sua1->u.sa4.sin_addr, + &sua2->u.sa4.sin_addr, sizeof(struct in_addr))); case PF_INET6: - return (memcmp(&sua1->sa6.sin6_addr, - &sua2->sa6.sin6_addr, sizeof(struct in6_addr))); + return (memcmp(&sua1->u.sa6.sin6_addr, + &sua2->u.sa6.sin6_addr, sizeof(struct in6_addr))); default: WRONG("Just plain insane"); } @@ -409,11 +409,11 @@ VSA_Port(const struct suckaddr *sua) { CHECK_OBJ_NOTNULL(sua, SUCKADDR_MAGIC); - switch (sua->sa.sa_family) { + switch (sua->u.sa.sa_family) { case PF_INET: - return (ntohs(sua->sa4.sin_port)); + return (ntohs(sua->u.sa4.sin_port)); case PF_INET6: - return (ntohs(sua->sa6.sin6_port)); + return (ntohs(sua->u.sa6.sin6_port)); default: return (0); } @@ -436,8 +436,8 @@ VSA_get ## which ## name(int fd, void *d, size_t l) \ sua = d; \ \ INIT_OBJ(sua, SUCKADDR_MAGIC); \ - sl = sizeof(sua->sa); \ - r = get ## which ## name(fd, &sua->sa, &sl); \ + sl = sizeof(sua->u); \ + r = get ## which ## name(fd, &sua->u.sa, &sl); \ \ return (r == 0 ? sua : NULL); \ } \ From slink at schokola.de Sat Jan 9 11:02:37 2021 From: slink at schokola.de (Nils Goroll) Date: Sat, 9 Jan 2021 12:02:37 +0100 Subject: [master] 75d56fe13 The vtester running in a IPv6 enabled jail, attempting u000000 says: In-Reply-To: <20210108201912.9659D59EE@lists.varnish-cache.org> References: <20210108201912.9659D59EE@lists.varnish-cache.org> Message-ID: <06f833b3-5583-5716-d286-9d2f3980baa8@schokola.de> Thank you! Do I understand correctly that the relevant fix was this bit? On 08/01/2021 21:19, Poul-Henning Kamp wrote: > - sl = sizeof(sua->sa); \ > - r = get ## which ## name(fd, &sua->sa, &sl); \ > + sl = sizeof(sua->u); \ > + r = get ## which ## name(fd, &sua->u.sa, &sl); \ From phk at phk.freebsd.dk Sat Jan 9 11:31:06 2021 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Sat, 09 Jan 2021 11:31:06 +0000 Subject: [master] 75d56fe13 The vtester running in a IPv6 enabled jail, attempting u000000 says: In-Reply-To: <06f833b3-5583-5716-d286-9d2f3980baa8@schokola.de> References: <20210108201912.9659D59EE@lists.varnish-cache.org> <06f833b3-5583-5716-d286-9d2f3980baa8@schokola.de> Message-ID: <78368.1610191866@critter.freebsd.dk> -------- Nils Goroll writes: > Do I understand correctly that the relevant fix was this bit? > > On 08/01/2021 21:19, Poul-Henning Kamp wrote: > > - sl = sizeof(sua->sa); \ > > - r = get ## which ## name(fd, &sua->sa, &sl); \ > > + sl = sizeof(sua->u); \ > > + r = get ## which ## name(fd, &sua->u.sa, &sl); \ Yes. You have it only the length of "struct sockaddr", that is not (always) enough to hold a IPv6 address, so it got truncated. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From nils.goroll at uplex.de Sat Jan 9 20:17:14 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 9 Jan 2021 20:17:14 +0000 (UTC) Subject: [master] 889b9e0b8 varnishd: Support "-b None" argument Message-ID: <20210109201714.EA77D457E@lists.varnish-cache.org> commit 889b9e0b87ac7808bba1ad8c9c9c11c9950b815d Author: Nils Goroll Date: Sat Jan 9 18:12:40 2021 +0100 varnishd: Support "-b None" argument for "backend default none;" Ref #3067 diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index b848df76c..64e404c88 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -102,6 +102,7 @@ usage(void) printf(FMT, "", "Proto can be \"PROXY\" or \"HTTP\" (default)"); printf(FMT, "", "user, group and mode set permissions for"); printf(FMT, "", " a Unix domain socket."); + printf(FMT, "-b none", "No backend"); printf(FMT, "-b [addr[:port]|path]", "Backend address and port"); printf(FMT, "", " or socket file path"); printf(FMT, "", " default: \":80\""); @@ -626,14 +627,15 @@ main(int argc, char * const *argv) vsb = VSB_new_auto(); AN(vsb); VSB_cat(vsb, "vcl 4.1;\n"); - VSB_cat(vsb, "backend default {\n"); - if (*optarg != '/') - VSB_printf(vsb, " .host = \"%s\";\n", - optarg); + VSB_cat(vsb, "backend default "); + if (! strcasecmp(optarg, "none")) + VSB_cat(vsb, "none;\n"); + else if (*optarg != '/') + VSB_printf(vsb, "{\n .host = \"%s\";\n}\n", + optarg); else - VSB_printf(vsb, " .path = \"%s\";\n", - optarg); - VSB_cat(vsb, "}\n"); + VSB_printf(vsb, "{\n .path = \"%s\";\n}\n", + optarg); AZ(VSB_finish(vsb)); fa->src = strdup(VSB_data(vsb)); AN(fa->src); From nils.goroll at uplex.de Sat Jan 9 20:17:14 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 9 Jan 2021 20:17:14 +0000 (UTC) Subject: [master] 0376b7f4b vtcp: make vtcp_sa_to_ascii error output more helpful Message-ID: <20210109201715.129994581@lists.varnish-cache.org> commit 0376b7f4b05954fac7902c75c9b6189094a48476 Author: Nils Goroll Date: Sat Jan 9 14:47:53 2021 +0100 vtcp: make vtcp_sa_to_ascii error output more helpful - write to stderr - output errno for EAI_SYSTEM diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c index 78a093738..84832cb65 100644 --- a/lib/libvarnish/vtcp.c +++ b/lib/libvarnish/vtcp.c @@ -73,7 +73,9 @@ vtcp_sa_to_ascii(const void *sa, socklen_t l, char *abuf, unsigned alen, * XXX this printf is shitty, but we may not have space * for the gai_strerror in the bufffer :-( */ - printf("getnameinfo = %d %s\n", i, gai_strerror(i)); + fprintf(stderr, "getnameinfo = %d %s\n", i, gai_strerror(i)); + if (i == EAI_SYSTEM) + fprintf(stderr, "errno = %d %s\n", errno, vstrerror(errno)); if (abuf != NULL) (void)snprintf(abuf, alen, "Conversion"); if (pbuf != NULL) From nils.goroll at uplex.de Sat Jan 9 20:17:14 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 9 Jan 2021 20:17:14 +0000 (UTC) Subject: [master] 2aa9a8b0b varnishd: Fix debug.listen_address for full length IPv6 Message-ID: <20210109201715.15C9A4582@lists.varnish-cache.org> commit 2aa9a8b0bc7b75f95b4eb4ffd71894b9659e555a Author: Nils Goroll Date: Sat Jan 9 14:53:11 2021 +0100 varnishd: Fix debug.listen_address for full length IPv6 32 bytes are not enough to hold the colons of a full length IPv6 address. diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index beedeaf5c..471ecc824 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -654,7 +654,7 @@ static void v_matchproto_(cli_func_t) ccf_listen_address(struct cli *cli, const char * const *av, void *priv) { struct listen_sock *ls; - char h[32], p[32]; + char h[VTCP_ADDRBUFSIZE], p[VTCP_PORTBUFSIZE]; (void)cli; (void)av; From nils.goroll at uplex.de Sat Jan 9 20:17:15 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 9 Jan 2021 20:17:15 +0000 (UTC) Subject: [master] 1c19f3fed vtest: Untie vtest from 127.0.0.1 and support IPv6 only setups Message-ID: <20210109201715.29EB04584@lists.varnish-cache.org> commit 1c19f3fed3da8fae79ef200f2ebf01625a42bb27 Author: Nils Goroll Date: Sat Jan 9 14:42:38 2021 +0100 vtest: Untie vtest from 127.0.0.1 and support IPv6 only setups to support IPv6 only hosts, we try to resolve "localhost" if resolving "127.0.0.1" fails. Whatever address we determine is stored in the ${localhost} macro. Because of the different address:port vs. [address]:port formats for IPv4 vs. IPv6, we store "${localhost}:0" / "[${localhost}]:0" in the ${listen_addr} macro. Likewise we unify the *_sock macros to ${Xaddr}:${Xport} (the colon used to be missing) for IPv4 and [${Xaddr}]:${Xport} for IPv6. We also save ${listen_addr} in default_listen_addr for use from within the vtest code whenever we want to bind to or listen on an emphemeral port. Ref #3490 diff --git a/bin/varnishtest/tests/b00053.vtc b/bin/varnishtest/tests/b00053.vtc index ea33a284a..6e7f8301b 100644 --- a/bin/varnishtest/tests/b00053.vtc +++ b/bin/varnishtest/tests/b00053.vtc @@ -76,5 +76,5 @@ client c1 -connect ${v3_sock} { expect resp.http.a0_sock == "${tmpdir}/v1.sock" expect resp.http.a1_addr != "0.0.0.0" expect resp.http.a1_port != "0" - expect resp.http.a1_sock ~ "[^ ]+ [^ ]+" + expect resp.http.a1_sock ~ "[^ ]+:[^ ]+" } -run diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index 1c716508f..046df86d0 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -81,6 +81,7 @@ extern char *vmod_path; extern struct vsb *params_vsb; extern int leave_temp; extern int ign_unknown_macro; +extern const char *default_listen_addr; void init_server(void); void init_syslog(void); diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index ee334953f..5600401fa 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -41,6 +41,7 @@ #include "vtc.h" #include "vtcp.h" +#include "vsa.h" enum barrier_e { BARRIER_NONE = 0, @@ -128,7 +129,10 @@ barrier_sock_thread(void *priv) struct barrier *b; struct vtclog *vl; const char *err; - char abuf[16], pbuf[6]; + char buf[vsa_suckaddr_len]; + struct suckaddr *sua; + + char abuf[VTCP_ADDRBUFSIZE], pbuf[VTCP_PORTBUFSIZE]; int i, sock, *conns; struct pollfd pfd[1]; @@ -140,7 +144,7 @@ barrier_sock_thread(void *priv) vl = vtc_logopen("%s", b->name); pthread_cleanup_push(vtc_logclose, vl); - sock = VTCP_listen_on("127.0.0.1:0", NULL, b->expected, &err); + sock = VTCP_listen_on(default_listen_addr, NULL, b->expected, &err); if (sock < 0) { AZ(pthread_cond_signal(&b->cond)); AZ(pthread_mutex_unlock(&b->mtx)); @@ -149,11 +153,16 @@ barrier_sock_thread(void *priv) } assert(sock > 0); VTCP_nonblocking(sock); - VTCP_myname(sock, abuf, sizeof abuf, pbuf, sizeof pbuf); + sua = VSA_getsockname(sock, buf, sizeof buf); + AN(sua); + VTCP_name(sua, abuf, sizeof abuf, pbuf, sizeof pbuf); macro_def(vl, b->name, "addr", "%s", abuf); macro_def(vl, b->name, "port", "%s", pbuf); - macro_def(vl, b->name, "sock", "%s:%s", abuf, pbuf); + if (VSA_Get_Proto(sua) == AF_INET) + macro_def(vl, b->name, "sock", "%s:%s", abuf, pbuf); + else + macro_def(vl, b->name, "sock", "[%s]:%s", abuf, pbuf); AZ(pthread_cond_signal(&b->cond)); AZ(pthread_mutex_unlock(&b->mtx)); diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c index 07a8d48c1..3ef81a0a8 100644 --- a/bin/varnishtest/vtc_client.c +++ b/bin/varnishtest/vtc_client.c @@ -106,7 +106,7 @@ client_tcp_connect(struct vtclog *vl, const char *addr, double tmo, const char **errp) { int fd; - char mabuf[32], mpbuf[32]; + char mabuf[VTCP_ADDRBUFSIZE], mpbuf[VTCP_PORTBUFSIZE]; AN(addr); AN(errp); diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index 46b4001da..ca0cb9259 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -45,6 +45,7 @@ #include "vpf.h" #include "vre.h" #include "vtcp.h" +#include "vsa.h" #include "vtim.h" #define HAPROXY_PROGRAM_ENV_VAR "HAPROXY_PROGRAM" @@ -173,7 +174,7 @@ haproxy_cli_tcp_connect(struct vtclog *vl, const char *addr, double tmo, const char **errp) { int fd; - char mabuf[32], mpbuf[32]; + char mabuf[VTCP_ADDRBUFSIZE], mpbuf[VTCP_PORTBUFSIZE]; AN(addr); AN(errp); @@ -519,16 +520,23 @@ haproxy_create_mcli(struct haproxy *h) int sock; const char *err; char buf[128], addr[128], port[128]; + char vsabuf[vsa_suckaddr_len]; + struct suckaddr *sua; - sock = VTCP_listen_on("127.0.0.1:0", NULL, 100, &err); + sock = VTCP_listen_on(default_listen_addr, NULL, 100, &err); if (err != NULL) vtc_fatal(h->vl, "Create listen socket failed: %s", err); assert(sock > 0); + sua = VSA_getsockname(sock, vsabuf, sizeof vsabuf); + AN(sua); - VTCP_myname(sock, addr, sizeof addr, port, sizeof port); + VTCP_name(sua, addr, sizeof addr, port, sizeof port); bprintf(buf, "%s_mcli", h->name); - macro_def(h->vl, buf, "sock", "%s %s", addr, port); + if (VSA_Get_Proto(sua) == AF_INET) + macro_def(h->vl, buf, "sock", "%s:%s", addr, port); + else + macro_def(h->vl, buf, "sock", "[%s]:%s", addr, port); macro_def(h->vl, buf, "addr", "%s", addr); macro_def(h->vl, buf, "port", "%s", port); @@ -558,6 +566,8 @@ haproxy_new(const char *name) int closed_sock; char addr[128], port[128]; const char *err; + char vsabuf[vsa_suckaddr_len]; + struct suckaddr *sua; ALLOC_OBJ(h, HAPROXY_MAGIC); AN(h); @@ -597,8 +607,13 @@ haproxy_new(const char *name) vtc_fatal(h->vl, "Create listen socket failed: %s", err); assert(closed_sock > 0); - VTCP_myname(closed_sock, addr, sizeof addr, port, sizeof port); - macro_def(h->vl, h->closed_sock, "sock", "%s %s", addr, port); + sua = VSA_getsockname(closed_sock, vsabuf, sizeof vsabuf); + AN(sua); + VTCP_name(sua, addr, sizeof addr, port, sizeof port); + if (VSA_Get_Proto(sua) == AF_INET) + macro_def(h->vl, h->closed_sock, "sock", "%s:%s", addr, port); + else + macro_def(h->vl, h->closed_sock, "sock", "[%s]:%s", addr, port); macro_def(h->vl, h->closed_sock, "addr", "%s", addr); macro_def(h->vl, h->closed_sock, "port", "%s", port); VTCP_close(&closed_sock); @@ -836,6 +851,8 @@ haproxy_build_backends(struct haproxy *h, const char *vsb_data) int sock; char buf[128], addr[128], port[128]; const char *err; + char vsabuf[vsa_suckaddr_len]; + struct suckaddr *sua; p = strstr(p, HAPROXY_BE_FD_STR); if (!p) @@ -853,10 +870,15 @@ haproxy_build_backends(struct haproxy *h, const char *vsb_data) vtc_fatal(h->vl, "Create listen socket failed: %s", err); assert(sock > 0); + sua = VSA_getsockname(sock, vsabuf, sizeof vsabuf); + AN(sua); - VTCP_myname(sock, addr, sizeof addr, port, sizeof port); + VTCP_name(sua, addr, sizeof addr, port, sizeof port); bprintf(buf, "%s_%s", h->name, p); - macro_def(h->vl, buf, "sock", "%s %s", addr, port); + if (VSA_Get_Proto(sua) == AF_INET) + macro_def(h->vl, buf, "sock", "%s:%s", addr, port); + else + macro_def(h->vl, buf, "sock", "[%s]:%s", addr, port); macro_def(h->vl, buf, "addr", "%s", addr); macro_def(h->vl, buf, "port", "%s", port); diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 6110540c2..9b5a60a25 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -51,6 +51,7 @@ #include "vnum.h" #include "vrnd.h" #include "vss.h" +#include "vsa.h" #include "vsub.h" #include "vtcp.h" #include "vtim.h" @@ -116,6 +117,7 @@ static int bad_backend_fd; static int cleaner_fd = -1; static pid_t cleaner_pid; +const char *default_listen_addr; static struct buf * get_buf(void) @@ -594,8 +596,13 @@ ip_magic(void) sa = VSS_ResolveOne(NULL, "127.0.0.1", "0", 0, SOCK_STREAM, 0); AN(sa); bad_backend_fd = VTCP_bind(sa, NULL); + if (bad_backend_fd < 0) { + free(sa); + sa = VSS_ResolveFirst(NULL, "localhost", "0", 0, SOCK_STREAM, 0); + AN(sa); + bad_backend_fd = VTCP_bind(sa, NULL); + } assert(bad_backend_fd >= 0); - free(sa); VTCP_myname(bad_backend_fd, abuf, sizeof abuf, pbuf, sizeof(pbuf)); extmacro_def("localhost", "%s", abuf); @@ -609,7 +616,21 @@ ip_magic(void) #endif /* Expose a backend that is forever down. */ - extmacro_def("bad_backend", "%s %s", abuf, pbuf); + if (VSA_Get_Proto(sa) == AF_INET) + extmacro_def("bad_backend", "%s:%s", abuf, pbuf); + else + extmacro_def("bad_backend", "[%s]:%s", abuf, pbuf); + + /* our default bind/listen address */ + if (VSA_Get_Proto(sa) == AF_INET) + bprintf(abuf, "%s:0", macro_get("localhost", NULL)); + else + bprintf(abuf, "[%s]:0", macro_get("localhost", NULL)); + + extmacro_def("listen_addr", "%s", abuf); + default_listen_addr = strdup(abuf); + AN(default_listen_addr); + free(sa); /* * We need an IP number which will not repond, ever, and that is a diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index f98f2b7bb..87637ed73 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -39,6 +39,7 @@ #include #include +#include "vsa.h" #include "vtc.h" #include "vtcp.h" @@ -59,8 +60,8 @@ struct server { int sock; int fd; char listen[256]; - char aaddr[32]; - char aport[32]; + char aaddr[VTCP_ADDRBUFSIZE]; + char aport[VTCP_PORTBUFSIZE]; pthread_t tp; }; @@ -88,7 +89,7 @@ server_new(const char *name, struct vtclog *vl) s->vsp = Sess_New(s->vl, name); AN(s->vsp); - bprintf(s->listen, "%s", "127.0.0.1 0"); + bprintf(s->listen, "%s", default_listen_addr); s->depth = 10; s->sock = -1; s->fd = -1; @@ -183,17 +184,27 @@ server_listen_uds(struct server *s, const char **errp) static void server_listen_tcp(struct server *s, const char **errp) { + char buf[vsa_suckaddr_len]; + struct suckaddr *sua; + s->sock = VTCP_listen_on(s->listen, "0", s->depth, errp); if (*errp != NULL) return; assert(s->sock > 0); - VTCP_myname(s->sock, s->aaddr, sizeof s->aaddr, + sua = VSA_getsockname(s->sock, buf, sizeof buf); + AN(sua); + VTCP_name(sua, s->aaddr, sizeof s->aaddr, s->aport, sizeof s->aport); + + /* Record the actual port, and reuse it on subsequent starts */ + if (VSA_Get_Proto(sua) == AF_INET) + bprintf(s->listen, "%s:%s", s->aaddr, s->aport); + else + bprintf(s->listen, "[%s]:%s", s->aaddr, s->aport); + macro_def(s->vl, s->name, "addr", "%s", s->aaddr); macro_def(s->vl, s->name, "port", "%s", s->aport); - macro_def(s->vl, s->name, "sock", "%s %s", s->aaddr, s->aport); - /* Record the actual port, and reuse it on subsequent starts */ - bprintf(s->listen, "%s %s", s->aaddr, s->aport); + macro_def(s->vl, s->name, "sock", "%s", s->listen); } static void diff --git a/bin/varnishtest/vtc_syslog.c b/bin/varnishtest/vtc_syslog.c index f57bd38d1..403a394ab 100644 --- a/bin/varnishtest/vtc_syslog.c +++ b/bin/varnishtest/vtc_syslog.c @@ -265,7 +265,7 @@ syslog_new(const char *name, struct vtclog *vl) AN(s->vl); vtc_log_set_cmd(s->vl, syslog_cmds); - bprintf(s->bind, "%s", "127.0.0.1 0"); + bprintf(s->bind, "%s", default_listen_addr); s->repeat = 1; s->sock = -1; s->lvl = -1; @@ -358,6 +358,8 @@ syslog_bind(struct syslog_srv *s) const char *err; char aaddr[VTCP_ADDRBUFSIZE]; char aport[VTCP_PORTBUFSIZE]; + char buf[vsa_suckaddr_len]; + struct suckaddr *sua; CHECK_OBJ_NOTNULL(s, SYSLOG_SRV_MAGIC); @@ -369,10 +371,15 @@ syslog_bind(struct syslog_srv *s) "Syslog server bind address (%s) cannot be resolved: %s", s->bind, err); assert(s->sock > 0); - VTCP_myname(s->sock, aaddr, sizeof aaddr, aport, sizeof aport); + sua = VSA_getsockname(s->sock, buf, sizeof buf); + AN(sua); + VTCP_name(sua, aaddr, sizeof aaddr, aport, sizeof aport); macro_def(s->vl, s->name, "addr", "%s", aaddr); macro_def(s->vl, s->name, "port", "%s", aport); - macro_def(s->vl, s->name, "sock", "%s %s", aaddr, aport); + if (VSA_Get_Proto(sua) == AF_INET) + macro_def(s->vl, s->name, "sock", "%s:%s", aaddr, aport); + else + macro_def(s->vl, s->name, "sock", "[%s]:%s", aaddr, aport); /* Record the actual port, and reuse it on subsequent starts */ bprintf(s->bind, "%s %s", aaddr, aport); } diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 1a4a3e88a..4698aa4e9 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -392,7 +392,7 @@ varnish_launch(struct varnish *v) char *r = NULL; /* Create listener socket */ - v->cli_fd = VTCP_listen_on("127.0.0.1:0", NULL, 1, &err); + v->cli_fd = VTCP_listen_on(default_listen_addr, NULL, 1, &err); if (err != NULL) vtc_fatal(v->vl, "Create CLI listen socket failed: %s", err); assert(v->cli_fd > 0); @@ -420,7 +420,7 @@ varnish_launch(struct varnish *v) VSB_cat(vsb, " -p h2_initial_window_size=1m"); VSB_cat(vsb, " -p h2_rx_window_low_water=64k"); if (!v->has_a_arg) { - VSB_printf(vsb, " -a '%s'", "127.0.0.1:0"); + VSB_printf(vsb, " -a '%s'", default_listen_addr); if (v->proto != NULL) VSB_printf(vsb, ",%s", v->proto); } @@ -572,12 +572,14 @@ varnish_listen(const struct varnish *v, char *la) AN(*a); AN(*p); - if (*p != '-') { - bprintf(s, "%s %s", a, p); - } else { + if (*p == '-') { bprintf(s, "%s", a); a = "0.0.0.0"; p = "0"; + } else if (strchr(a, ':')) { + bprintf(s, "[%s]:%s", a, p); + } else { + bprintf(s, "%s:%s", a, p); } if (first) { From nils.goroll at uplex.de Sat Jan 9 20:17:15 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 9 Jan 2021 20:17:15 +0000 (UTC) Subject: [master] 069311348 vtc: Make tests compatible with IPv6 only Message-ID: <20210109201715.B87D04599@lists.varnish-cache.org> commit 06931134800b88da228402ad64e13d668819f8b9 Author: Nils Goroll Date: Sat Jan 9 14:59:28 2021 +0100 vtc: Make tests compatible with IPv6 only For 127.0.0.1, use "None", "${localhost}" or "${bind_addr}" as appropriate. Instead of ${Xaddr}:${Xport}, use ${Xsock} For some tests, we need to add a Host header to make them stable for differing server addresses. Ref #3490 diff --git a/bin/varnishtest/tests/a02022.vtc b/bin/varnishtest/tests/a02022.vtc index d3cc0d01b..c089cd6a6 100644 --- a/bin/varnishtest/tests/a02022.vtc +++ b/bin/varnishtest/tests/a02022.vtc @@ -20,4 +20,4 @@ server s1 { } -start -shell { nghttp http://${s1_addr}:${s1_port} -nu } +shell { nghttp http://${s1_sock} -nu } diff --git a/bin/varnishtest/tests/b00032.vtc b/bin/varnishtest/tests/b00032.vtc index d395fef69..7fd326289 100644 --- a/bin/varnishtest/tests/b00032.vtc +++ b/bin/varnishtest/tests/b00032.vtc @@ -25,7 +25,7 @@ varnish v1 -clierr 300 "vcl.discard vcl1" varnish v1 -clierr 106 "vcl.discard vcl0" -varnish v1 -clierr 106 {vcl.inline vcl2 "vcl 4.0; backend foo {.host = \"127.0.0.1\";} "} +varnish v1 -clierr 106 {vcl.inline vcl2 "vcl 4.0; backend foo {.host = \"${localhost}\";} "} varnish v1 -clierr 106 {vcl.load vcl3 ./nonexistent.vcl} diff --git a/bin/varnishtest/tests/b00041.vtc b/bin/varnishtest/tests/b00041.vtc index ca66c4539..b829efc17 100644 --- a/bin/varnishtest/tests/b00041.vtc +++ b/bin/varnishtest/tests/b00041.vtc @@ -1,6 +1,6 @@ varnishtest "Test varnishadm and the Telnet CLI" -varnish v1 -vcl {backend foo { .host = "127.0.0.1"; } } -start +varnish v1 -vcl {backend foo { .host = "${localhost}"; } } -start shell -err -expect {Usage: varnishadm} \ "varnishadm -7" @@ -16,7 +16,7 @@ server s1 { } -start shell -err -expect {Rejected 400} \ - {varnishadm -T ${s1_addr}:${s1_port} -S /etc/group} + {varnishadm -T ${s1_sock} -S /etc/group} server s1 { send "107 59 \n" @@ -26,7 +26,7 @@ server s1 { } -start shell -err -expect {Authentication required} \ - {varnishadm -T ${s1_addr}:${s1_port}} + {varnishadm -T ${s1_sock}} server s1 { send "107 59 \n" @@ -36,7 +36,7 @@ server s1 { } -start shell -err -expect {Cannot open } \ - {varnishadm -T ${s1_addr}:${s1_port} -S ${v1_name}/_.nonexistent} + {varnishadm -T ${s1_sock} -S ${v1_name}/_.nonexistent} server s1 { send "107 59 \n" @@ -50,7 +50,7 @@ server s1 { } -start shell -err -expect {Rejected 599} \ - {varnishadm -T ${s1_addr}:${s1_port} -S ${v1_name}/_.secret} + {varnishadm -T ${s1_sock} -S ${v1_name}/_.secret} server s1 { send "107 59 \n" @@ -66,7 +66,7 @@ server s1 { } -start shell -err -expect {No pong received from server} \ - {varnishadm -T ${s1_addr}:${s1_port} -S ${v1_name}/_.secret} + {varnishadm -T ${s1_sock} -S ${v1_name}/_.secret} server s1 { send "107 59 \n" @@ -90,6 +90,6 @@ server s1 { } -start shell -expect {Tested} \ - {varnishadm -T ${s1_addr}:${s1_port} -S ${v1_name}/_.secret test} + {varnishadm -T ${s1_sock} -S ${v1_name}/_.secret test} shell "varnishadm -n ${v1_name} help > /dev/null" diff --git a/bin/varnishtest/tests/b00042.vtc b/bin/varnishtest/tests/b00042.vtc index 6470a7260..6c338ae51 100644 --- a/bin/varnishtest/tests/b00042.vtc +++ b/bin/varnishtest/tests/b00042.vtc @@ -1,6 +1,6 @@ varnishtest "param edge cases" -varnish v1 -vcl {backend foo {.host = "127.0.0.1";}} -start +varnish v1 -vcl {backend foo {.host = "${localhost}";}} -start varnish v1 -clierr "106" "param.set default_ttl -1" diff --git a/bin/varnishtest/tests/b00045.vtc b/bin/varnishtest/tests/b00045.vtc index 47a676f07..c687f2d49 100644 --- a/bin/varnishtest/tests/b00045.vtc +++ b/bin/varnishtest/tests/b00045.vtc @@ -15,13 +15,13 @@ client c1 { delay .2 shell -err -expect {Error: Could not open pid-file} { - varnishd -P /dev/tty -b 127.0.0.1:80 -a :0 -n ${tmpdir} + varnishd -P /dev/tty -b None -a :0 -n ${tmpdir} } shell -err -expect {Error: Varnishd is already running} { - varnishd -P ${v1_name}/varnishd.pid -b 127.0.0.1:80 -a :0 -n ${tmpdir} + varnishd -P ${v1_name}/varnishd.pid -b None -a :0 -n ${tmpdir} } shell -err -expect {Error: Varnishd is already running} { - varnishd -b 127.0.0.1:80 -a:0 -n ${tmpdir}/v1 -F + varnishd -b None -a:0 -n ${tmpdir}/v1 -F } diff --git a/bin/varnishtest/tests/b00053.vtc b/bin/varnishtest/tests/b00053.vtc index 6e7f8301b..1c9bf152b 100644 --- a/bin/varnishtest/tests/b00053.vtc +++ b/bin/varnishtest/tests/b00053.vtc @@ -5,7 +5,7 @@ server s1 -listen "${tmpdir}/s1.sock" { txresp -body "012345\n" } -start -varnish v1 -arg "-a ${tmpdir}/v1.sock -a ${localhost}:0" -vcl+backend { +varnish v1 -arg "-a ${tmpdir}/v1.sock -a ${listen_addr}" -vcl+backend { sub vcl_backend_response { set beresp.do_stream = false; } diff --git a/bin/varnishtest/tests/b00067.vtc b/bin/varnishtest/tests/b00067.vtc index 9479617a8..786ddcd93 100644 --- a/bin/varnishtest/tests/b00067.vtc +++ b/bin/varnishtest/tests/b00067.vtc @@ -1,7 +1,7 @@ varnishtest "Check timeout_idle" varnish v1 -arg "-p timeout_idle=1" \ - -arg "-a 127.0.0.1:0" \ + -arg "-a ${listen_addr}" \ -arg "-a ${tmpdir}/v1.sock" \ -vcl { backend dummy { .host = "${bad_backend}"; } diff --git a/bin/varnishtest/tests/b00068.vtc b/bin/varnishtest/tests/b00068.vtc index 2251b5290..dea15e802 100644 --- a/bin/varnishtest/tests/b00068.vtc +++ b/bin/varnishtest/tests/b00068.vtc @@ -8,7 +8,7 @@ varnishtest "Check timeout_linger" # varnish v1 -arg "-p timeout_linger=1" \ - -arg "-a 127.0.0.1:0" \ + -arg "-a ${listen_addr}" \ -arg "-a ${tmpdir}/v1.sock" \ -vcl { import std; diff --git a/bin/varnishtest/tests/c00003.vtc b/bin/varnishtest/tests/c00003.vtc index d5a8ef9d4..5c3b7fabd 100644 --- a/bin/varnishtest/tests/c00003.vtc +++ b/bin/varnishtest/tests/c00003.vtc @@ -3,19 +3,23 @@ varnishtest "Check that we fail to start with erroneous -a/-b arguments" # Duplicate -a arguments # XXX: this cannot be tested reliably, we tolerate port collision shell -err -match "have same address|already in use|Address in use" { - varnishd -d -a 127.0.0.1:38484 -a 127.0.0.1:38484 -b localhost:80 + if echo "${localhost}" | grep ":" >/dev/null ; then + varnishd -d -a "[${localhost}]":38484 -a "[${localhost}]":38484 -b None + else + varnishd -d -a ${localhost}:38484 -a ${localhost}:38484 -b None + fi } shell -err -match "have same address|already in use|Address in use" { - varnishd -d -a ${tmpdir}/vtc.sock -a ${tmpdir}/vtc.sock -b localhost:80 + varnishd -d -a ${tmpdir}/vtc.sock -a ${tmpdir}/vtc.sock -b None } # -a bad protocol specs shell -err -expect "Too many protocol sub-args" { - varnishd -a 127.0.0.1:80000,PROXY,FOO -d + varnishd -a ${localhost}:80000,PROXY,FOO -d } shell -err -expect "Too many protocol sub-args" { - varnishd -a 127.0.0.1:80000,HTTP,FOO -d + varnishd -a ${localhost}:80000,HTTP,FOO -d } # -a UDS path too long @@ -30,15 +34,15 @@ shell -err -expect "Unix domain socket addresses must be absolute paths" { # -a args for UDS permissions not permitted with IP addresses shell -err -expect "Invalid sub-arg user=u for IP addresses" { - varnishd -a 127.0.0.1:80000,user=u -d + varnishd -a ${localhost}:80000,user=u -d } shell -err -expect "Invalid sub-arg group=g for IP addresses" { - varnishd -a 127.0.0.1:80000,group=g -d + varnishd -a ${localhost}:80000,group=g -d } shell -err -expect "Invalid sub-arg mode=660 for IP addresses" { - varnishd -a 127.0.0.1:80000,mode=660 -d + varnishd -a ${localhost}:80000,mode=660 -d } # Illegal mode sub-args @@ -123,5 +127,5 @@ shell -err -expect "Error: Could not get socket" { # old style address list shell -err -expect "Unknown protocol" { - varnishd -F -a "127.0.0.1:0,${bad_ip}:0" -b '***' -n ${tmpdir} + varnishd -F -a "${listen_addr},${bad_ip}:0" -b '***' -n ${tmpdir} } diff --git a/bin/varnishtest/tests/c00005.vtc b/bin/varnishtest/tests/c00005.vtc index a39d5ee04..448d6f80c 100644 --- a/bin/varnishtest/tests/c00005.vtc +++ b/bin/varnishtest/tests/c00005.vtc @@ -19,7 +19,7 @@ varnish v1 -errvcl {Name acl1 must have type 'acl'.} { } backend acl1 { - .host = "127.0.0.1"; + .host = "${localhost}"; } } diff --git a/bin/varnishtest/tests/c00007.vtc b/bin/varnishtest/tests/c00007.vtc index 016ae78e2..ec780cec2 100644 --- a/bin/varnishtest/tests/c00007.vtc +++ b/bin/varnishtest/tests/c00007.vtc @@ -1,5 +1,5 @@ varnishtest "Test banning a hash" -varnish v1 -arg "-b 127.0.0.1:80" -start +varnish v1 -arg "-b None" -start varnish v1 -clierr 101 "ban.hash foo" diff --git a/bin/varnishtest/tests/c00039.vtc b/bin/varnishtest/tests/c00039.vtc index 4c36c2ac2..c7350219d 100644 --- a/bin/varnishtest/tests/c00039.vtc +++ b/bin/varnishtest/tests/c00039.vtc @@ -21,19 +21,19 @@ varnish v1 -cliok "param.set http_req_size 256" varnish v1 -cliok "param.set http_req_hdr_len 40" client c1 { - txreq -url "/1" -hdr "1...5: ..0....5....0....5....0....5....0" + txreq -url "/1" -hdr "host: 127.0.0.1" -hdr "1...5: ..0....5....0....5....0....5....0" rxresp expect resp.status == 200 - txreq -url "/1" -hdr "1...5....0....5....0....5....0....5....0." + txreq -url "/1" -hdr "host: 127.0.0.1" -hdr "1...5....0....5....0....5....0....5....0." rxresp expect resp.status == 400 } -run client c1 { - txreq -url "/2" -hdr "1...5: ..0....5\n ..0....5....0....5....0" + txreq -url "/2" -hdr "host: 127.0.0.1" -hdr "1...5: ..0....5\n ..0....5....0....5....0" rxresp expect resp.status == 200 - txreq -url "/2" -hdr "1...5....0....5\n ..0....5....0....5....0." + txreq -url "/2" -hdr "host: 127.0.0.1" -hdr "1...5....0....5\n ..0....5....0....5....0." rxresp expect resp.status == 400 } -run diff --git a/bin/varnishtest/tests/c00084.vtc b/bin/varnishtest/tests/c00084.vtc index 37b3be238..dff306944 100644 --- a/bin/varnishtest/tests/c00084.vtc +++ b/bin/varnishtest/tests/c00084.vtc @@ -6,7 +6,7 @@ varnish v1 \ -vcl { import directors; - acl my-acl { "127.0.0.1"; } + acl my-acl { "${localhost}"; } probe my-pb { } backend my-be { diff --git a/bin/varnishtest/tests/d00003.vtc b/bin/varnishtest/tests/d00003.vtc index 238d5e52d..fc799c72b 100644 --- a/bin/varnishtest/tests/d00003.vtc +++ b/bin/varnishtest/tests/d00003.vtc @@ -51,31 +51,31 @@ varnish v1 -vcl+backend { } -start client c1 { - txreq -url /1 + txreq -url /1 -hdr "Host: 127.0.0.1" rxresp expect resp.http.foo == "1" - txreq -url /3 + txreq -url /3 -hdr "Host: 127.0.0.1" rxresp expect resp.http.foo == "2" - txreq -url /13 + txreq -url /13 -hdr "Host: 127.0.0.1" rxresp expect resp.http.foo == "3" - txreq -url /8 + txreq -url /8 -hdr "Host: 127.0.0.1" rxresp expect resp.http.foo == "4" - txreq -url "/emptystring" + txreq -url "/emptystring" -hdr "Host: 127.0.0.1" rxresp expect resp.http.foo == "6" - txreq -url "/nohdr" + txreq -url "/nohdr" -hdr "Host: 127.0.0.1" rxresp expect resp.http.foo == "8" - txreq -url "/ip" + txreq -url "/ip" -hdr "Host: 127.0.0.1" rxresp expect resp.http.foo == "9" } -run @@ -83,19 +83,19 @@ client c1 { server s2 -start client c1 { - txreq -req "DELETE" + txreq -req "DELETE" -hdr "Host: 127.0.0.1" rxresp expect resp.status == 204 - txreq + txreq -hdr "Host: 127.0.0.1" rxresp expect resp.http.foo == "1" - txreq + txreq -hdr "Host: 127.0.0.1" rxresp expect resp.http.foo == "4" - txreq + txreq -hdr "Host: 127.0.0.1" rxresp expect resp.http.foo == "9" } -run diff --git a/bin/varnishtest/tests/d00006.vtc b/bin/varnishtest/tests/d00006.vtc index fc35395c1..f6f9a29d8 100644 --- a/bin/varnishtest/tests/d00006.vtc +++ b/bin/varnishtest/tests/d00006.vtc @@ -24,7 +24,7 @@ server s4 { varnish v1 -errvcl {Name of VCL object, 'rr1.xx', contains illegal character '.'} { import directors; - backend b1 { .host = "127.0.0.1"; .port = "8080";} + backend b1 { .host = "${localhost}"; .port = "8080";} sub vcl_init { new rr1.xx = directors.round_robin(); } diff --git a/bin/varnishtest/tests/h00001.vtc b/bin/varnishtest/tests/h00001.vtc index f8b98a2ec..7f8c11599 100644 --- a/bin/varnishtest/tests/h00001.vtc +++ b/bin/varnishtest/tests/h00001.vtc @@ -17,7 +17,7 @@ haproxy h1 -conf { timeout client 30s backend be1 - server srv1 ${s1_addr}:${s1_port} + server srv1 ${s1_sock} frontend http1 use_backend be1 diff --git a/bin/varnishtest/tests/h00002.vtc b/bin/varnishtest/tests/h00002.vtc index 935f2331a..c8c9bc1dd 100644 --- a/bin/varnishtest/tests/h00002.vtc +++ b/bin/varnishtest/tests/h00002.vtc @@ -17,7 +17,7 @@ haproxy h1 -D -conf { timeout client 30s backend be1 - server srv1 ${s1_addr}:${s1_port} + server srv1 ${s1_sock} frontend http1 use_backend be1 diff --git a/bin/varnishtest/tests/h00005.vtc b/bin/varnishtest/tests/h00005.vtc index 098bf19ab..b6281040e 100644 --- a/bin/varnishtest/tests/h00005.vtc +++ b/bin/varnishtest/tests/h00005.vtc @@ -8,7 +8,7 @@ server s1 { txresp } -start -syslog S1 -level notice -bind "127.0.0.1:0" { +syslog S1 -level notice -bind "${listen_addr}" { recv recv recv info @@ -17,7 +17,7 @@ syslog S1 -level notice -bind "127.0.0.1:0" { haproxy h1 -conf { global - log ${S1_addr}:${S1_port} local0 + log ${S1_sock} local0 defaults log global @@ -32,7 +32,7 @@ haproxy h1 -conf { default_backend be1 backend be1 - server srv1 ${s1_addr}:${s1_port} + server srv1 ${s1_sock} } -start client c1 -connect ${h1_fe_1_sock} { @@ -42,4 +42,3 @@ client c1 -connect ${h1_fe_1_sock} { } -run syslog S1 -wait - diff --git a/bin/varnishtest/tests/h00006.vtc b/bin/varnishtest/tests/h00006.vtc index 556cd6d05..18e099162 100644 --- a/bin/varnishtest/tests/h00006.vtc +++ b/bin/varnishtest/tests/h00006.vtc @@ -30,6 +30,7 @@ varnish v1 -arg "-a ${tmpdir}/v1.sock,PROXY" -vcl+backend { "localhost"; "127.0.0.1"; "::1"; + "${localhost}"; "${s1_addr}"; // Jails IPv4 address "${h1_fe1_addr}"; // Jails IPv6 address } diff --git a/bin/varnishtest/tests/h00007.vtc b/bin/varnishtest/tests/h00007.vtc index fa42fcb8f..b153590ba 100644 --- a/bin/varnishtest/tests/h00007.vtc +++ b/bin/varnishtest/tests/h00007.vtc @@ -22,7 +22,7 @@ haproxy h1 -D -conf { listen ssloff bind "fd@${fe1}" - server v1 ${v1_addr}:${v1_port} send-proxy-v2 + server v1 ${v1_sock} send-proxy-v2 } -start varnish v1 -vcl+backend { @@ -32,6 +32,7 @@ varnish v1 -vcl+backend { "localhost"; "127.0.0.1"; "::1"; + "${localhost}"; "${s1_addr}"; # Jail IPv4 address "${h1_fe1_addr}"; # Jail IPv6 address } diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc index a64fa7d06..48fc749c0 100644 --- a/bin/varnishtest/tests/m00000.vtc +++ b/bin/varnishtest/tests/m00000.vtc @@ -173,7 +173,7 @@ varnish v1 -errvcl {Symbol not found: 'obj'} { varnish v1 -errvcl {Failed initialization} { import debug; import directors; - backend default { .host = "127.0.0.1"; } + backend default { .host = "${localhost}"; } sub vcl_init { return (fail); # uninitialized objects coverage diff --git a/bin/varnishtest/tests/m00008.vtc b/bin/varnishtest/tests/m00008.vtc index bb0568cd1..95ea39af0 100644 --- a/bin/varnishtest/tests/m00008.vtc +++ b/bin/varnishtest/tests/m00008.vtc @@ -3,7 +3,7 @@ varnishtest "VCL compiler vmod coverage / vmod loading" feature topbuild varnish v1 -errvcl {Symbol 'debug' has wrong type (sub):} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub debug {} import debug; diff --git a/bin/varnishtest/tests/m00017.vtc b/bin/varnishtest/tests/m00017.vtc index 78a47e49f..78e3bdcbc 100644 --- a/bin/varnishtest/tests/m00017.vtc +++ b/bin/varnishtest/tests/m00017.vtc @@ -313,21 +313,21 @@ server s1 -repeat 3 -keepalive { } -start client c6 { - txreq -url /8 + txreq -url /8 -hdr "Host: 127.0.0.1" rxresp expect resp.status == 200 expect resp.http.fetch == "Fetch value 1" expect resp.http.response == "123" expect resp.http.response2 == "Another response" - txreq -url /9 + txreq -url /9 -hdr "Host: 127.0.0.1" rxresp expect resp.status == 200 expect resp.http.fetch == "Fetch value 1" expect resp.http.response == "123" expect resp.http.response2 == "Another response" - txreq -url /10 + txreq -url /10 -hdr "Host: 127.0.0.1" rxresp expect resp.status == 200 expect resp.http.fetch == "Fetch value 1" diff --git a/bin/varnishtest/tests/m00019.vtc b/bin/varnishtest/tests/m00019.vtc index edc0e3ab7..bebbfe4d7 100644 --- a/bin/varnishtest/tests/m00019.vtc +++ b/bin/varnishtest/tests/m00019.vtc @@ -72,7 +72,7 @@ delay .1 varnish v1 -errvcl {Argument 'one' already used} { import debug; - backend b1 {.host = "127.0.0.1";} + backend b1 {.host = "${localhost}";} sub vcl_deliver { set resp.http.foo5 = debug.argtest("1", one="1"); } @@ -80,7 +80,7 @@ varnish v1 -errvcl {Argument 'one' already used} { varnish v1 -errvcl {Argument 'one' missing} { import debug; - backend b1 {.host = "127.0.0.1";} + backend b1 {.host = "${localhost}";} sub vcl_deliver { set resp.http.foo5 = debug.argtest(two=2.0, three="3"); } @@ -88,7 +88,7 @@ varnish v1 -errvcl {Argument 'one' missing} { varnish v1 -errvcl {Unknown argument 'five'} { import debug; - backend b1 {.host = "127.0.0.1";} + backend b1 {.host = "${localhost}";} sub vcl_deliver { set resp.http.foo5 = debug.argtest("1", two=2.0, five="3"); } diff --git a/bin/varnishtest/tests/m00025.vtc b/bin/varnishtest/tests/m00025.vtc index 432043f84..1ac8ea712 100644 --- a/bin/varnishtest/tests/m00025.vtc +++ b/bin/varnishtest/tests/m00025.vtc @@ -3,7 +3,7 @@ varnishtest "Pass probe definitions to VMODs" varnish v1 -vcl { import debug; - backend be { .host = "127.0.0.1"; } + backend be { .host = "${localhost}"; } probe pb { } sub vcl_init { diff --git a/bin/varnishtest/tests/r00310.vtc b/bin/varnishtest/tests/r00310.vtc index c00e4c87e..6231149ba 100644 --- a/bin/varnishtest/tests/r00310.vtc +++ b/bin/varnishtest/tests/r00310.vtc @@ -1,7 +1,7 @@ varnishtest "Test obj.http.x-cache in vcl_hit" varnish v1 -errvcl {Variable is read only.} { - backend foo { .host = "127.0.0.1"; } + backend foo { .host = "${localhost}"; } sub vcl_hit { set obj.http.x-cache = "hit"; diff --git a/bin/varnishtest/tests/r00409.vtc b/bin/varnishtest/tests/r00409.vtc index 80acd3461..1ef578932 100644 --- a/bin/varnishtest/tests/r00409.vtc +++ b/bin/varnishtest/tests/r00409.vtc @@ -1,7 +1,7 @@ varnishtest "Regression test for ticket 409" varnish v1 -errvcl {Expected CSTR got '!'} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { if ( req.url ~ ! "\.(png|jpg|gif|js|css)$" ) { return (pass); diff --git a/bin/varnishtest/tests/r00916.vtc b/bin/varnishtest/tests/r00916.vtc index 6c951fb7a..e73ef10cd 100644 --- a/bin/varnishtest/tests/r00916.vtc +++ b/bin/varnishtest/tests/r00916.vtc @@ -6,7 +6,7 @@ server s1 { } -start varnish v1 -errvcl {Symbol not found: 's-1'} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub s1 { } sub vcl_backend_response { diff --git a/bin/varnishtest/tests/r00921.vtc b/bin/varnishtest/tests/r00921.vtc index 7eac5bbcf..9c0b556de 100644 --- a/bin/varnishtest/tests/r00921.vtc +++ b/bin/varnishtest/tests/r00921.vtc @@ -10,7 +10,7 @@ server s1 { varnish v1 -vcl+backend { sub vcl_recv { set req.http.bar = regsub(req.url, ".*", client.ip); - set req.http.foo = regsub(client.ip, ":.*", client.ip); + set req.http.foo = regsub(client.ip, "#.*", client.ip); } } -start diff --git a/bin/varnishtest/tests/r00948.vtc b/bin/varnishtest/tests/r00948.vtc index d029990cb..e0d01f2e4 100644 --- a/bin/varnishtest/tests/r00948.vtc +++ b/bin/varnishtest/tests/r00948.vtc @@ -11,10 +11,10 @@ server s1 { varnish v1 -vcl+backend { sub vcl_recv { - if (client.ip == "${s1_addr}" || client.ip == "127.0.0.1") { + if (client.ip == "${s1_addr}" || client.ip == "${localhost}") { set req.http.foo1 = "Match"; } - if (client.ip == "127.0.0.1" || client.ip == "${s1_addr}") { + if (client.ip == "${localhost}" || client.ip == "${s1_addr}") { set req.http.foo2 = "Match"; } } diff --git a/bin/varnishtest/tests/r01002.vtc b/bin/varnishtest/tests/r01002.vtc index fbede7ce8..3d6e700dd 100644 --- a/bin/varnishtest/tests/r01002.vtc +++ b/bin/varnishtest/tests/r01002.vtc @@ -3,7 +3,7 @@ varnishtest "Real relational comparisons" varnish v1 -vcl { import std; - backend foo { .host = "127.0.0.1"; } + backend foo { .host = "${localhost}"; } sub vcl_recv { if (std.random(0,5) < 1.0) { return (pipe); diff --git a/bin/varnishtest/tests/r01144.vtc b/bin/varnishtest/tests/r01144.vtc index 284c8489c..00a4312aa 100644 --- a/bin/varnishtest/tests/r01144.vtc +++ b/bin/varnishtest/tests/r01144.vtc @@ -8,7 +8,7 @@ server s1 { varnish v1 -vcl+backend { backend fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210 { - .host = "127.0.0.1"; + .host = "${localhost}"; .port = "54321"; } sub vcl_backend_fetch { @@ -27,7 +27,7 @@ client c1 { varnish v1 -errvcl {Expected ID got '0'} { backend 0fedcba9876543210fedcba9876543210fedcba9876543210fedcba9876543210 { - .host = "127.0.0.1"; + .host = "${localhost}"; .port = "54321"; } sub vcl_backend_fetch { diff --git a/bin/varnishtest/tests/r01164.vtc b/bin/varnishtest/tests/r01164.vtc index 295148232..7d5b1740a 100644 --- a/bin/varnishtest/tests/r01164.vtc +++ b/bin/varnishtest/tests/r01164.vtc @@ -2,7 +2,7 @@ varnishtest "Regression test for #1163: allow backends to be named storage*" varnish v1 -vcl { backend storage_foo { - .host = "127.0.0.1"; + .host = "${localhost}"; } sub vcl_backend_fetch { @@ -12,7 +12,7 @@ varnish v1 -vcl { varnish v1 -vcl { backend storagefoo { - .host = "127.0.0.1"; + .host = "${localhost}"; } sub vcl_backend_fetch { diff --git a/bin/varnishtest/tests/r01212.vtc b/bin/varnishtest/tests/r01212.vtc index 236b459ab..d22c16a33 100644 --- a/bin/varnishtest/tests/r01212.vtc +++ b/bin/varnishtest/tests/r01212.vtc @@ -2,7 +2,7 @@ varnishtest "#1212 - Vmod with HEADER argument given a STRING asserts the VCL co varnish v1 -errvcl {Expression has type STRING, expected HEADER} { import std; - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { std.collect("foo"); } diff --git a/bin/varnishtest/tests/r01312.vtc b/bin/varnishtest/tests/r01312.vtc index 762da43bf..14674221f 100644 --- a/bin/varnishtest/tests/r01312.vtc +++ b/bin/varnishtest/tests/r01312.vtc @@ -17,8 +17,8 @@ varnish v1 -vcl+backend { "127.0.1"/19; } sub vcl_deliver { - set resp.http.ACLfoo = std.ip("127.0.0.1", client.ip) ~ foo; - set resp.http.ACLbar = std.ip("127.0.0.1", client.ip) ~ bar; + set resp.http.ACLfoo = std.ip("${localhost}", client.ip) ~ foo; + set resp.http.ACLbar = std.ip("${localhost}", client.ip) ~ bar; } } -start diff --git a/bin/varnishtest/tests/r01498.vtc b/bin/varnishtest/tests/r01498.vtc index 0c81af543..8b5cfad61 100644 --- a/bin/varnishtest/tests/r01498.vtc +++ b/bin/varnishtest/tests/r01498.vtc @@ -4,7 +4,7 @@ varnish v1 -vcl { vcl 4.0; import directors; backend s1 { - .host = "127.0.0.1"; + .host = "${localhost}"; .port = "80"; } sub vcl_init { diff --git a/bin/varnishtest/tests/r01504.vtc b/bin/varnishtest/tests/r01504.vtc index 9767ff0b4..da4d92c61 100644 --- a/bin/varnishtest/tests/r01504.vtc +++ b/bin/varnishtest/tests/r01504.vtc @@ -6,10 +6,10 @@ varnish v1 -arg "-p vcc_err_unref=off" -vcl { .host = "${bad_backend}"; } acl foo { - "127.0.0.1"; + "${localhost}"; } acl bar { - "127.0.0.1"; + "${localhost}"; } sub vcl_recv { if (vtc.no_ip() ~ bar) { diff --git a/bin/varnishtest/tests/r01510.vtc b/bin/varnishtest/tests/r01510.vtc index 1092ff7e1..c521bd943 100644 --- a/bin/varnishtest/tests/r01510.vtc +++ b/bin/varnishtest/tests/r01510.vtc @@ -11,7 +11,7 @@ varnish v1 -errvcl {Instance 'first' redefined.} { varnish v1 -errvcl {Name 'first' already defined.} { import debug; - backend first { .host = "127.0.0.1"; } + backend first { .host = "${localhost}"; } sub vcl_init { new first = debug.obj("FOO"); diff --git a/bin/varnishtest/tests/r01566.vtc b/bin/varnishtest/tests/r01566.vtc index d78c8ac4e..a317feb15 100644 --- a/bin/varnishtest/tests/r01566.vtc +++ b/bin/varnishtest/tests/r01566.vtc @@ -1,7 +1,7 @@ varnishtest "escape issue in regexp" varnish v1 -vcl { - backend b1 { .host = "127.0.0.1"; } + backend b1 { .host = "${localhost}"; } sub vcl_recv { set req.url = regsuball(req.url, "\??(p|pi)=.*?(&|$)", ""); diff --git a/bin/varnishtest/tests/r01569.vtc b/bin/varnishtest/tests/r01569.vtc index aed91b184..94cf48f16 100644 --- a/bin/varnishtest/tests/r01569.vtc +++ b/bin/varnishtest/tests/r01569.vtc @@ -5,7 +5,7 @@ varnish v1 -errvcl {Name 'debug' already defined.} { import debug; backend debug { - .host = "127.0.0.1"; + .host = "${localhost}"; .port = "80"; } diff --git a/bin/varnishtest/tests/r01804.vtc b/bin/varnishtest/tests/r01804.vtc index 7fcbaa1e6..252dd5845 100644 --- a/bin/varnishtest/tests/r01804.vtc +++ b/bin/varnishtest/tests/r01804.vtc @@ -6,7 +6,7 @@ server s1 { } -start -varnish v1 -arg "-a foo=127.0.0.1:0,PROXY" +varnish v1 -arg "-a foo=${listen_addr},PROXY" varnish v1 -arg "-p thread_pools=1" varnish v1 -vcl+backend "" -start diff --git a/bin/varnishtest/tests/r01837.vtc b/bin/varnishtest/tests/r01837.vtc index e96a44cf7..7a4034e75 100644 --- a/bin/varnishtest/tests/r01837.vtc +++ b/bin/varnishtest/tests/r01837.vtc @@ -2,7 +2,7 @@ varnishtest "Test VCC errors out if probe is used before it is defined" varnish v1 -errvcl "Symbol not found: 'p'" { backend b { - .host = "127.0.0.1"; + .host = "${localhost}"; .probe = p; } probe p { diff --git a/bin/varnishtest/tests/r01990.vtc b/bin/varnishtest/tests/r01990.vtc index c6a83292c..56359dc09 100644 --- a/bin/varnishtest/tests/r01990.vtc +++ b/bin/varnishtest/tests/r01990.vtc @@ -9,7 +9,7 @@ varnish v1 -vcl { sub vcl_backend_fetch { # avoid LostHeader b Host: %s - set bereq.http.Host = "127.0.0.1"; + set bereq.http.Host = "${localhost}"; vtc.workspace_alloc(backend, vtc.workspace_free(backend)); } diff --git a/bin/varnishtest/tests/r02157.vtc b/bin/varnishtest/tests/r02157.vtc index 4aaba65ee..62ba0ac80 100644 --- a/bin/varnishtest/tests/r02157.vtc +++ b/bin/varnishtest/tests/r02157.vtc @@ -8,10 +8,10 @@ server s1 { varnish v1 -vcl+backend {} -start # 64 chars vlc name, 64 chars backend name -varnish v1 -cliok { vcl.inline abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789 "vcl 4.0; backend abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789 {.host = \"127.0.0.1\"; .port = \"${s1_port}\";} " } +varnish v1 -cliok { vcl.inline abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789 "vcl 4.0; backend abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789 {.host = \"${localhost}\"; .port = \"${s1_port}\";} " } # 127 chars vlc name, 1 char backend name -varnish v1 -cliok { vcl.inline abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef012345678 "vcl 4.0; backend a {.host = \"127.0.0.1\"; .port = \"${s1_port}\";} " } +varnish v1 -cliok { vcl.inline abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef012345678 "vcl 4.0; backend a {.host = \"${localhost}\"; .port = \"${s1_port}\";} " } client c1 { txreq diff --git a/bin/varnishtest/tests/r02839.vtc b/bin/varnishtest/tests/r02839.vtc index 275d934ff..2b8aa4e38 100644 --- a/bin/varnishtest/tests/r02839.vtc +++ b/bin/varnishtest/tests/r02839.vtc @@ -10,7 +10,7 @@ varnish v1 -vcl+backend "" -start varnish v1 -errvcl "Object nullobj not initialized" { import debug; - backend default { .host = "127.0.0.1"; } + backend default { .host = "${localhost}"; } sub vcl_init { if (false) { @@ -22,7 +22,7 @@ varnish v1 -errvcl "Object nullobj not initialized" { varnish v1 -vcl { import debug; - backend default { .host = "127.0.0.1"; } + backend default { .host = "${localhost}"; } sub vcl_init { if (false) { diff --git a/bin/varnishtest/tests/r03360.vtc b/bin/varnishtest/tests/r03360.vtc index 72668ff9b..9c1e81c10 100644 --- a/bin/varnishtest/tests/r03360.vtc +++ b/bin/varnishtest/tests/r03360.vtc @@ -5,11 +5,11 @@ shell {echo include '"_recurse2.vcl";' > ${tmpdir}/_recurse1.vcl} shell {echo include '"_recurse1.vcl";' > ${tmpdir}/_recurse2.vcl} varnish v1 -arg "-p vcl_path=${tmpdir}" -errvcl "Recursive include of" { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } include "_recurse.vcl" ; } varnish v2 -arg "-p vcl_path=${tmpdir}" -errvcl "Recursive include of" { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } include "_recurse1.vcl" ; } diff --git a/bin/varnishtest/tests/u00000.vtc b/bin/varnishtest/tests/u00000.vtc index 29e602253..dbed3ce58 100644 --- a/bin/varnishtest/tests/u00000.vtc +++ b/bin/varnishtest/tests/u00000.vtc @@ -1,15 +1,15 @@ varnishtest "Code coverage of mgt_main, (VCL compiler and RSTdump etc)" -shell "varnishd -b 127.0.0.1:80 -C 2> ${tmpdir}/_.c" +shell "varnishd -b None -C 2> ${tmpdir}/_.c" shell { - varnishd -n ${tmpdir}/no_keep -C -b 127.0.0.1:80 2> no_keep.c + varnishd -n ${tmpdir}/no_keep -C -b None 2> no_keep.c test -s no_keep.c && ! test -d no_keep || \ (test -f no_keep/*/vgc.gcda || test -d no_keep/*/vgc.so.dSYM) } shell { - varnishd -n ${tmpdir}/keep -p debug=+vcl_keep -C -b 127.0.0.1:80 2> keep.c + varnishd -n ${tmpdir}/keep -p debug=+vcl_keep -C -b None 2> keep.c test -s keep.c && test -d keep } @@ -54,8 +54,8 @@ shell -expect {Copyright (c) 2006} "varnishd -V" shell -err -expect {Only one of -d or -F can be specified} "varnishd -d -F " shell -err -expect {Only one of -b or -f can be specified} "varnishd -b a -f b " -shell -err -expect {-d makes no sense with -C} "varnishd -C -b 127.0.0.1:80 -d " -shell -err -expect {-F makes no sense with -C} "varnishd -C -b 127.0.0.1:80 -F " +shell -err -expect {-d makes no sense with -C} "varnishd -C -b None -d " +shell -err -expect {-F makes no sense with -C} "varnishd -C -b None -F " shell -err -expect {Neither -b nor -f given} { varnishd -n ${tmpdir}/v0 } # Test -I and -l arguments (former a00016) diff --git a/bin/varnishtest/tests/u00003.vtc b/bin/varnishtest/tests/u00003.vtc index 680b1fa7a..1191f085f 100644 --- a/bin/varnishtest/tests/u00003.vtc +++ b/bin/varnishtest/tests/u00003.vtc @@ -29,7 +29,7 @@ shell { -w ${tmpdir}/ncsa.log -R 100/s } -process p1 -winsz 25 132 {varnishncsa -n ${v1_name}} -start +process p1 -winsz 25 200 {varnishncsa -n ${v1_name}} -start delay 1 @@ -125,11 +125,11 @@ ${localhost} -$} \ # -f and standard formatters shell {echo "%b %D %H %h %I %{baz}i %l %m %{Age}o %O %q %s %t %{%Y}t \ %T %U %u" > ${tmpdir}/format} -shell -match {^100 \d+ HTTP/1.1 \d+.\d+.\d+.\d+ \d+ - - GET 0 \d+ (?# +shell -match {^100 \d+ HTTP/1.1 (?:\d+.\d+.\d+.\d+|[a-f0-9:]+) \d+ - - GET 0 \d+ (?# )\?foo=bar 200 \[../.../(20[1-9][0-9]):..:..:.. [+-]....\] \1 \d+ /1 user -\d+ \d+ HTTP/1.1 \d+.\d+.\d+.\d+ \d+ qux - GET - \d+ (?# +\d+ \d+ HTTP/1.1 (?:\d+.\d+.\d+.\d+|[a-f0-9:]+) \d+ qux - GET - \d+ (?# )\?foo=bar 404 \[../.../(20[1-9][0-9]):..:..:.. [+-]....\] \1 \d+ /1 - -- \d+ HTTP/1.1 \d+.\d+.\d+.\d+ \d+ - - GET - \d+ (?# +- \d+ HTTP/1.1 (?:\d+.\d+.\d+.\d+|[a-f0-9:]+) \d+ - - GET - \d+ (?# ) - \[../.../(20[1-9][0-9]):..:..:.. [+-]....\] \1 \d+ /2 -$} \ {varnishncsa -n ${v1_name} -d -f ${tmpdir}/format} diff --git a/bin/varnishtest/tests/v00005.vtc b/bin/varnishtest/tests/v00005.vtc index cd70d39c8..4e28d5388 100644 --- a/bin/varnishtest/tests/v00005.vtc +++ b/bin/varnishtest/tests/v00005.vtc @@ -3,7 +3,7 @@ varnishtest "VCL: test backend probe syntax" # Check status definition varnish v1 -vcl { backend b1 { - .host = "127.0.0.1"; + .host = "${localhost}"; .probe = { .expected_response = 204; } @@ -13,7 +13,7 @@ varnish v1 -vcl { # Check url definition varnish v1 -vcl { backend b1 { - .host = "127.0.0.1"; + .host = "${localhost}"; .probe = { .url = "/"; } @@ -23,7 +23,7 @@ varnish v1 -vcl { # Check request definition varnish v1 -vcl { backend b1 { - .host = "127.0.0.1"; + .host = "${localhost}"; .probe = { .request = "GET / HTTP/1.1" @@ -35,7 +35,7 @@ varnish v1 -vcl { # Check redefinition varnish v1 -errvcl {Probe request redefinition at:} { backend b1 { - .host = "127.0.0.1"; + .host = "${localhost}"; .probe = { .url = "/"; .request = @@ -48,7 +48,7 @@ varnish v1 -errvcl {Probe request redefinition at:} { # Check redefinition the other way varnish v1 -errvcl {Probe request redefinition at:} { backend b1 { - .host = "127.0.0.1"; + .host = "${localhost}"; .probe = { .request = "GET / HTTP/1.1" @@ -60,7 +60,7 @@ varnish v1 -errvcl {Probe request redefinition at:} { varnish v1 -errvcl {Expected CNUM got '"120s"'} { backend default { - .host = "127.0.0.1"; + .host = "${localhost}"; .probe = { .timeout = "120s"; } diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc index 6f09d10fb..250bdd51c 100644 --- a/bin/varnishtest/tests/v00016.vtc +++ b/bin/varnishtest/tests/v00016.vtc @@ -5,19 +5,19 @@ feature dns shell "true > ${tmpdir}/_varnishtest_empty_file" varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } include "${tmpdir}/_varnishtest_empty_file" ; } varnish v1 -errvcl {include not followed by semicolon.} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } include "${tmpdir}/_varnishtest_empty_file" | } shell "rm -f ${tmpdir}/_varnishtest_empty_file" varnish v1 -errvcl {include not followed by string constant.} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } include << } @@ -31,18 +31,18 @@ varnish v1 -errvcl {include not followed by string constant.} { } varnish v1 -errvcl {Unknown duration unit 'k'} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_backend_response { set beresp.ttl = 1. k; } } varnish v1 -errvcl {Operator > not possible on BACKEND} { - backend a { .host = "127.0.0.1"; } - backend b { .host = "127.0.0.1"; } + backend a { .host = "${localhost}"; } + backend b { .host = "${localhost}"; } sub vcl_recv { if (a > b) { } } } varnish v1 -errvcl {Unknown property 'foo' for type HTTP} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_hash { if (req.foo != "bar") { } } } @@ -53,21 +53,21 @@ varnish v1 -errvcl {Symbol not found: 'foo.bar'} { } varnish v1 -errvcl {Cannot be set in subroutine 'vcl_pipe'} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_pipe { set bereq.first_byte_timeout = 10s; } } varnish v1 -errvcl {Cannot be set in subroutine 'vcl_pipe'.} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_pipe { set bereq.between_bytes_timeout = 10s; } } varnish v1 -errvcl {Symbol not found: 'c'} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_backend_response { if (beresp.backend == c) { set beresp.ttl = 1h; @@ -76,14 +76,14 @@ varnish v1 -errvcl {Symbol not found: 'c'} { } varnish v1 -errvcl {Regexp compilation error:} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { if (req.url ~ "[a") {} } } varnish v1 -errvcl {resolves to too many addresses} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { if (remote.ip == "dns-canary-multi.varnish-cache.org") {} } @@ -91,7 +91,7 @@ varnish v1 -errvcl {resolves to too many addresses} { varnish v1 -errvcl {Symbol 'foo' has wrong type (instance):} { import directors; - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_init { new foo = directors.shard(); @@ -105,7 +105,7 @@ varnish v1 -errvcl {Symbol 'foo' has wrong type (instance):} { varnish v1 -syntax 4.0 -errvcl {Undefined acl foo} { import directors; - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_init { new foo = directors.shard(); @@ -130,7 +130,7 @@ varnish v1 -errvcl {Undefined sub foo} { varnish v1 -syntax 4.0 -errvcl {Symbol not found: 'directors.foo' At:} { import directors; - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_init { new foo = directors.foo(); @@ -139,10 +139,10 @@ At:} { # 'foo' overloaded varnish v1 -syntax 4.0 -errvcl {Symbol not found: 'foo'} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } acl foo { - "127.0.0.1"/32; + "${localhost}"/32; } sub vcl_init { new bar = foo; diff --git a/bin/varnishtest/tests/v00017.vtc b/bin/varnishtest/tests/v00017.vtc index 2809b2fe1..6e35d0c23 100644 --- a/bin/varnishtest/tests/v00017.vtc +++ b/bin/varnishtest/tests/v00017.vtc @@ -1,19 +1,19 @@ varnishtest "VCL compiler coverage test: vcc_acl.c" varnish v1 -errvcl {Too wide mask (/33) for IPv4 address} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } acl a { "10.1.2.3"/33; } sub vcl_recv { if (client.ip ~ a) { return(pass); } } } varnish v1 -errvcl {Too wide mask (/129) for IPv6 address} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } acl a { "1::2"/129; } sub vcl_recv { if (client.ip ~ a) { return(pass); } } } varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } acl a { "1.2.3.4"/31; "1.2.3.4"/31; @@ -22,7 +22,7 @@ varnish v1 -vcl { } varnish v1 -errvcl {Conflicting ACL entries:} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } acl a { "1.2.3.4"; !"1.2.3.4"; @@ -31,31 +31,31 @@ varnish v1 -errvcl {Conflicting ACL entries:} { } varnish v1 -errvcl {DNS lookup(...com): } { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } acl a { "...com"; } sub vcl_recv { if (client.ip ~ a) { return(pass); } } } varnish v1 -errvcl {DNS lookup(10.1..2): } { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } acl a { "10.1..2"; } sub vcl_recv { if (client.ip ~ a) { return(pass); } } } varnish v1 -errvcl {Expected ')' got ';'} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } acl a { ( "10.1.2"; } sub vcl_recv { if (client.ip ~ a) { return(pass); } } } varnish v1 -errvcl {Expected ';' got ')'} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } acl a { "10.1.2" ); } sub vcl_recv { if (client.ip ~ a) { return(pass); } } } varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } acl a { ! "10.1.3"; ("...com" / 22); @@ -65,12 +65,12 @@ varnish v1 -vcl { } varnish v1 -errvcl {Operator > not possible on IP} { - backend b { .host = "127.0.0.1"; } - sub vcl_recv { if (client.ip > "127.0.0.1") { return(pass); } } + backend b { .host = "${localhost}"; } + sub vcl_recv { if (client.ip > "${localhost}") { return(pass); } } } varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } acl a { "10.1.1"/25; "10.1.3"/26; @@ -86,7 +86,7 @@ varnish v1 -vcl { } varnish v1 -errvcl {.../mask is not numeric.} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } acl a { "10.0.1.0/1bc"; } @@ -94,7 +94,7 @@ varnish v1 -errvcl {.../mask is not numeric.} { } varnish v1 -errvcl {/mask only allowed once} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } acl a { "10.0.1.0/22" / 22; } diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc index d2728bcaf..56116f9ab 100644 --- a/bin/varnishtest/tests/v00018.vtc +++ b/bin/varnishtest/tests/v00018.vtc @@ -1,121 +1,121 @@ varnishtest "VCL compiler coverage test: vcc_action.c" varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_miss { return(synth(100,req.url)); } sub vcl_hit { return(synth(100,"the butter please")); } sub vcl_deliver { return(synth(resp.status, resp.reason)); } } varnish v1 -errvcl {Variable is read only.} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_miss { set now = 1000; } } varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_backend_response { set beresp.ttl /= 2; } } varnish v1 -errvcl {Expected '=' got '>>'} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_backend_response { set beresp.ttl >>= 2; } } varnish v1 -errvcl {Expected '=' got '+='} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_backend_fetch { set bereq.backend += b; } } varnish v1 -errvcl {Expected ';' got 'if'} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } /* XXX: This should not really be an synth */ sub vcl_recv { set req.url = "foo" if "bar"; } } varnish v1 -errvcl {Unknown property 'foo' for type HTTP} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_hash { hash_data(req.foo); } } varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { set req.url = 1; } } varnish v1 -errvcl {Expected '=' got '+='} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_backend_response { set beresp.do_gzip += 1; } } varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_backend_response { set beresp.do_gzip = true; } } varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_backend_response { set beresp.do_gzip = false; } } varnish v1 -errvcl {Symbol not found: 'mu'} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_backend_response { set beresp.do_gzip = mu; } } varnish v1 -errvcl {Variable cannot be unset} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_backend_response { unset beresp.do_gzip; } } varnish v1 -errvcl {Variable cannot be set.} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_backend_fetch { set bereq.body = "foo"; } } varnish v1 -errvcl {Unknown token '<<' when looking for STRING} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { ban (<<); } } varnish v1 -errvcl {Symbol not found} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { ban_hash (if); } } varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { ban ("req.url ~ foo"); } } varnish v1 -errvcl "Symbol not found" { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { kluf ; } } varnish v1 -errvcl {Unknown token '<<' when looking for STRING} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_synth { synthetic( << "foo"; } } varnish v1 -errvcl {Missing argument.} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { return(synth); } } varnish v1 -errvcl {Arguments not allowed.} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { return(pipe(XXX); } } varnish v1 -errvcl {Expected return action name.} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { return(foobar); } } # issue #936 varnish v1 -errvcl {Not a valid action in subroutine 'vcl_recv'} { - backend foo { .host = "127.0.0.1"; } + backend foo { .host = "${localhost}"; } sub vcl_recv { synthetic("XXX"); return (synth(503)); diff --git a/bin/varnishtest/tests/v00019.vtc b/bin/varnishtest/tests/v00019.vtc index 904586790..0d3bd6960 100644 --- a/bin/varnishtest/tests/v00019.vtc +++ b/bin/varnishtest/tests/v00019.vtc @@ -3,43 +3,43 @@ varnishtest "VCL compiler coverage test: vcc_token.c" varnish v1 -errvcl {Unterminated inline C source, starting at} " C{ " varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } # comment sub vcl_recv { set req.url = "x"; } } varnish v1 -errvcl {Unterminated /* ... */ comment, starting at} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } /* } varnish v1 -errvcl {Unterminated long-string, starting at} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } {" } } varnish v1 -errvcl {Unterminated long-string, starting at} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } """ "" } varnish v1 -errvcl {Unterminated string, starting at} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } " } varnish v1 -cliok "param.set vcc_allow_inline_c true" -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { C{ int i; (void)i; }C } } varnish v1 -errvcl {Syntax error at} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } ? } varnish v1 -errvcl {Comparison of different types: STRING '==' INT} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { if ("foo" + "bar" == 777) { set req.http.host = 1; @@ -48,7 +48,7 @@ varnish v1 -errvcl {Comparison of different types: STRING '==' INT} { } varnish v1 -errvcl {Comparison of different types: STRING '==' INT} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { if ("foo" + "bar" == 777) { @@ -58,7 +58,7 @@ varnish v1 -errvcl {Comparison of different types: STRING '==' INT} { } varnish v1 -errvcl {Comparison of different types: STRING '==' INT} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { if ("foo" + "bar" == @@ -70,7 +70,7 @@ varnish v1 -errvcl {Comparison of different types: STRING '==' INT} { } varnish v1 -errvcl {Symbol not found: 'req.http.req.http.foo'} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { set req.http.req.http.foo = "bar"; } diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc index 4c2a526ae..45f9474cd 100644 --- a/bin/varnishtest/tests/v00020.vtc +++ b/bin/varnishtest/tests/v00020.vtc @@ -1,7 +1,7 @@ varnishtest "VCL compiler coverage test: vcc_parse.c & vcc_expr.c" varnish v1 -cliok "param.set vcc_allow_inline_c true" -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } C{ #include }C @@ -57,7 +57,7 @@ varnish v1 -errvcl {BOOL + BOOL not possible.} { } varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { set req.http.foo = "foo" + "bar"; set req.http.foo = "foo" + 1; @@ -225,7 +225,7 @@ varnish v1 -errvcl {Function returns VOID} { varnish v1 -errvcl {Not available in subroutine 'vcl_recv'.} { import blob; - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { blob.encode(HEX, LOWER, req.hash); } @@ -233,14 +233,14 @@ varnish v1 -errvcl {Not available in subroutine 'vcl_recv'.} { varnish v1 -errvcl {Not available in subroutine 'vcl_hash'.} { import blob; - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_hash { blob.encode(HEX, LOWER, req.hash); } } varnish v1 -errvcl {Not available in subroutine 'vcl_recv'.} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { set req.http.foo = 100 + beresp.status; } @@ -264,7 +264,7 @@ Not available from subroutine 'vcl_recv'. sub vcl_recv { call foo; } ----------------------------###--- } { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub foo { set req.http.foo = 100 + beresp.status; } sub vcl_recv { call foo; } } @@ -293,7 +293,7 @@ Not available from subroutine 'vcl_recv'. sub vcl_recv { call bar; } ----------------------------###--- } { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub foo { set req.http.foo = 100 + beresp.status; } sub bar { call foo; } sub vcl_recv { call bar; } diff --git a/bin/varnishtest/tests/v00021.vtc b/bin/varnishtest/tests/v00021.vtc index ab0618f98..4c8e8ee26 100644 --- a/bin/varnishtest/tests/v00021.vtc +++ b/bin/varnishtest/tests/v00021.vtc @@ -1,37 +1,37 @@ varnishtest "VCL compiler coverage test: vcc_xref.c vcc_var.c vcc_symb.c" varnish v1 -errvcl {Variable is read only.} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_deliver { set obj.ttl = 1 w; } } varnish v1 -errvcl {Variable is read only.} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub foo { set obj.ttl = 1 w; } sub vcl_deliver { call foo; } } varnish v1 -errvcl {Not available in subroutine 'vcl_recv'.} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { set obj.ttl = 1 w; } } varnish v1 -errvcl {Not available from subroutine 'vcl_recv'.} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub foo { set obj.ttl = 1 w; } sub vcl_recv { call foo; } } varnish v1 -errvcl "Symbol not found" { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { discard; } } varnish v1 -errvcl "Symbol not found" { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub foo { discard; } sub vcl_recv { call foo; } @@ -55,7 +55,7 @@ Subroutine recurses on sub vcl_recv { call foo; } ----------------------------###--- } { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub foo { call foo; } sub vcl_recv { call foo; } @@ -85,7 +85,7 @@ Subroutine recurses on sub vcl_recv { call foo; } ----------------------------###--- } { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub bar { call foo; } sub foo { call bar; } @@ -93,14 +93,14 @@ Subroutine recurses on } varnish v1 -errvcl {Unused acl foo, defined:} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } - acl foo { "127.0.0.1"; } + acl foo { "${localhost}"; } } varnish v1 -errvcl {Unused sub foo, defined:} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub foo { } @@ -108,14 +108,14 @@ varnish v1 -errvcl {Unused sub foo, defined:} { # deliberately testing for name "none" varnish v1 -errvcl {Unused sub none, defined:} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub none { } } varnish v1 -errvcl {Invalid return "deliver"} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } sub vcl_recv { call foo; @@ -128,7 +128,7 @@ varnish v1 -errvcl {Invalid return "deliver"} { varnish v1 -errvcl {HTTP header (buckinghambuckingham..) is too long.} { - backend foo { .host = "127.0.0.1"; } + backend foo { .host = "${localhost}"; } sub vcl_deliver { set resp.http.buckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambucking = "foobar"; @@ -137,7 +137,7 @@ varnish v1 -errvcl {HTTP header (buckinghambuckingham..) is too long.} { varnish v1 -vcl { - backend foo { .host = "127.0.0.1"; } + backend foo { .host = "${localhost}"; } sub vcl_deliver { set resp.http.buckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckinghambuckin = "foobar"; @@ -145,7 +145,7 @@ varnish v1 -vcl { } varnish v1 -errvcl {Symbol not found: 'req.foobar'} { - backend foo { .host = "127.0.0.1"; } + backend foo { .host = "${localhost}"; } sub vcl_recv { set req.foobar = 3; } } diff --git a/bin/varnishtest/tests/v00031.vtc b/bin/varnishtest/tests/v00031.vtc index 70ca757af..902d4fa51 100644 --- a/bin/varnishtest/tests/v00031.vtc +++ b/bin/varnishtest/tests/v00031.vtc @@ -1,20 +1,20 @@ varnishtest "param vcc_err_unref" varnish v1 -errvcl {Unused backend c, defined:} { - backend b { .host = "127.0.0.1"; } - backend c { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } + backend c { .host = "${localhost}"; } } varnish v1 -cliok "param.set vcc_err_unref false" varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } - backend c { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } + backend c { .host = "${localhost}"; } } varnish v1 -cliok "param.set vcc_err_unref true" varnish v1 -errvcl {Unused backend c, defined:} { - backend b { .host = "127.0.0.1"; } - backend c { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } + backend c { .host = "${localhost}"; } } diff --git a/bin/varnishtest/tests/v00034.vtc b/bin/varnishtest/tests/v00034.vtc index 768ef69ce..6bfe5fb08 100644 --- a/bin/varnishtest/tests/v00034.vtc +++ b/bin/varnishtest/tests/v00034.vtc @@ -8,25 +8,25 @@ server s1 { varnish v1 -vcl+backend { } -start varnish v1 -errvcl {Subroutine 'c1' redefined} { - backend foo { .host = "127.0.0.1"; } + backend foo { .host = "${localhost}"; } sub c1 { } sub c1 { } sub vcl_recv { call c1; } } varnish v1 -errvcl {Backend 's1' redefined} { - backend s1 { .host = "127.0.0.1"; } - backend s1 { .host = "127.0.0.1"; } + backend s1 { .host = "${localhost}"; } + backend s1 { .host = "${localhost}"; } } varnish v1 -errvcl {Probe 'p1' redefined} { probe p1 { } probe p1 { } - backend s1 { .host = "127.0.0.1"; .probe = p1;} + backend s1 { .host = "${localhost}"; .probe = p1;} } varnish v1 -errvcl {Expected '(' got ';'} { - backend s1 { .host = "127.0.0.1"; } + backend s1 { .host = "${localhost}"; } sub vcl_recv { return; } } diff --git a/bin/varnishtest/tests/v00038.vtc b/bin/varnishtest/tests/v00038.vtc index 6662eefb5..eca463012 100644 --- a/bin/varnishtest/tests/v00038.vtc +++ b/bin/varnishtest/tests/v00038.vtc @@ -14,7 +14,7 @@ varnish v1 -errvcl "IPv6 address has wrong port separator" { varnish v1 -errvcl "with exactly three digits" { backend b1 { - .host = "127.0.0.1"; + .host = "${localhost}"; .probe = { .expected_response = 1000; } @@ -23,7 +23,7 @@ varnish v1 -errvcl "with exactly three digits" { varnish v1 -errvcl "Must specify .threshold with .window" { backend b1 { - .host = "127.0.0.1"; + .host = "${localhost}"; .probe = { .window = 32; } @@ -32,7 +32,7 @@ varnish v1 -errvcl "Must specify .threshold with .window" { varnish v1 -errvcl "Threshold must be 64 or less" { backend b1 { - .host = "127.0.0.1"; + .host = "${localhost}"; .probe = { .threshold = 65; } @@ -41,7 +41,7 @@ varnish v1 -errvcl "Threshold must be 64 or less" { varnish v1 -errvcl "Window must be 64 or less" { backend b1 { - .host = "127.0.0.1"; + .host = "${localhost}"; .probe = { .window = 65; .threshold = 64; @@ -51,7 +51,7 @@ varnish v1 -errvcl "Window must be 64 or less" { varnish v1 -errvcl "Threshold can not be greater than window" { backend b1 { - .host = "127.0.0.1"; + .host = "${localhost}"; .probe = { .window = 63; .threshold = 64; @@ -61,20 +61,20 @@ varnish v1 -errvcl "Threshold can not be greater than window" { varnish v1 -errvcl "NB: Backend Syntax has changed:" { backend b1 { - set .host = "127.0.0.1"; + set .host = "${localhost}"; } } varnish v1 -errvcl "Expected '{' or name of probe, got" { backend b1 { - .host = "127.0.0.1"; + .host = "${localhost}"; .probe = "NONE"; } } varnish v1 -errvcl "Field 'port' redefined at:" { backend b1 { - .host = "127.0.0.1"; + .host = "${localhost}"; .port = "NONE"; .port = "NONE"; } @@ -82,7 +82,7 @@ varnish v1 -errvcl "Field 'port' redefined at:" { varnish v1 -errvcl "Unknown field:" { backend b1 { - .host = "127.0.0.1"; + .host = "${localhost}"; .fourscoreandsevenyearsago = "NONE"; } } @@ -100,19 +100,19 @@ varnish v1 -errvcl "No default probe defined" { } varnish v1 -errvcl "Only one default director possible." { - backend b1 { .host = "127.0.0.1"; } - backend default { .host = "127.0.0.1"; } - backend default { .host = "127.0.0.1"; } + backend b1 { .host = "${localhost}"; } + backend default { .host = "${localhost}"; } + backend default { .host = "${localhost}"; } } varnish v1 -errvcl "Unused backend b1, defined:" { - backend b1 { .host = "127.0.0.1"; } - backend default { .host = "127.0.0.1"; } + backend b1 { .host = "${localhost}"; } + backend default { .host = "${localhost}"; } } varnish v1 -errvcl "Address redefinition at:" { backend b1 { - .host = "127.0.0.1"; + .host = "${localhost}"; .path = "/path/to/uds"; } } diff --git a/bin/varnishtest/tests/v00046.vtc b/bin/varnishtest/tests/v00046.vtc index 2bbe9d4f1..458266211 100644 --- a/bin/varnishtest/tests/v00046.vtc +++ b/bin/varnishtest/tests/v00046.vtc @@ -3,13 +3,13 @@ varnishtest "Test relative to vcl_path, dot-include and absolute includes" # relative plain shell "true > ${tmpdir}/_start.vcl" varnish v1 -arg "-p vcl_path=${tmpdir}" -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } include "_start.vcl" ; } # absolute include varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } include "${tmpdir}/_start.vcl" ; } @@ -18,20 +18,20 @@ shell "mkdir -p ${tmpdir}/1/2/3" shell "true > ${tmpdir}/1/2/b.vcl" shell "echo 'include \"./2/b.vcl\";' > ${tmpdir}/1/a.vcl" varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } include "${tmpdir}/1/a.vcl" ; } # same but relative to vcl_path shell "echo 'include \"1/2/b.vcl\";' > ${tmpdir}/1/ab.vcl" varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } include "1/ab.vcl" ; } # dot-relative -> relative varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } include "1/a.vcl" ; } @@ -39,28 +39,28 @@ varnish v1 -vcl { shell "echo 'include \"./3/c.vcl\";' > ${tmpdir}/1/2/b.vcl" shell "true > ${tmpdir}/1/2/3/c.vcl" varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } include "1/a.vcl" ; } # relative -> absolute shell "echo 'include \"${tmpdir}/1/2/3/c.vcl\";' > ${tmpdir}/1/aa.vcl" varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } include "1/aa.vcl" ; } # relative -> absolute -> relative shell "echo 'include \"${tmpdir}/1/2/b.vcl\";' > ${tmpdir}/1/aaa.vcl" varnish v1 -vcl { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } include "1/aaa.vcl" ; } # includes and parses out shell "echo 'zool' > ${tmpdir}/1/2/3/c.vcl" varnish v1 -errvcl {Found: 'zool' at} { - backend b { .host = "127.0.0.1"; } + backend b { .host = "${localhost}"; } include "1/a.vcl"; } diff --git a/bin/varnishtest/tests/v00049.vtc b/bin/varnishtest/tests/v00049.vtc index 53e2cb705..e90d4173c 100644 --- a/bin/varnishtest/tests/v00049.vtc +++ b/bin/varnishtest/tests/v00049.vtc @@ -1,19 +1,19 @@ varnishtest "VCL syntax numbers" -varnish v1 -vcl {backend b1 { .host = "127.0.0.1:8080"; }} -start +varnish v1 -vcl {backend b1 None;} -start varnish v1 -syntax 3.9 -errvcl "VCL version 3.9 not supported." { - backend b1 { .host = "127.0.0.1:8080"; } + backend b1 None; } varnish v1 -syntax 4.0 -errvcl "silly buggers" { vcl 4.01; - backend b1 { .host = "127.0.0.1:8080"; } + backend b1 None; } varnish v1 -syntax 4.0 -errvcl "VCL version 9.9 not supported" { vcl 9.9; - backend b1 { .host = "127.0.0.1:8080"; } + backend b1 None; } varnish v1 -cliexpect {Don't play silly buggers with VCL version numbers} \ @@ -24,4 +24,3 @@ varnish v1 -cliexpect {Don't play silly buggers with VCL version numbers} \ varnish v1 -cliexpect {Expected 'vcl N.N;' found no semi-colon} \ {vcl.inline t2 "vcl 4.0 backend b { .host = \"localhost\";} "} - From nils.goroll at uplex.de Sat Jan 9 20:17:15 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 9 Jan 2021 20:17:15 +0000 (UTC) Subject: [master] 29f79b4ae vtc: tests incompatible with IPv6 Message-ID: <20210109201715.D163E459E@lists.varnish-cache.org> commit 29f79b4aebc9097a8db3174243fe27393de1a3f8 Author: Nils Goroll Date: Sat Jan 9 20:13:38 2021 +0100 vtc: tests incompatible with IPv6 Ref #3490 diff --git a/bin/varnishtest/tests/o00005.vtc b/bin/varnishtest/tests/o00005.vtc index 7acd25bbf..d46792462 100644 --- a/bin/varnishtest/tests/o00005.vtc +++ b/bin/varnishtest/tests/o00005.vtc @@ -1,5 +1,9 @@ varnishtest "PROXY v2 TLV test" +# XXX this does does not work with IPv6, the workspace overflow test +# is too brittle +feature cmd {test "${localhost}" = "127.0.0.1"} + server s1 { rxreq txresp diff --git a/bin/varnishtest/tests/r01312.vtc b/bin/varnishtest/tests/r01312.vtc index 14674221f..cec7ad2c6 100644 --- a/bin/varnishtest/tests/r01312.vtc +++ b/bin/varnishtest/tests/r01312.vtc @@ -1,5 +1,8 @@ varnishtest "acl miscompile" +# XXX this test only works if we know a network prefix of localhost +feature cmd {test "${localhost}" = "127.0.0.1"} + server s1 { rxreq txresp From nils.goroll at uplex.de Sun Jan 10 16:56:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 10 Jan 2021 16:56:08 +0000 (UTC) Subject: [master] e95760818 vtest: plug insignificant leak Message-ID: <20210110165608.3C9B9AE938@lists.varnish-cache.org> commit e95760818552ba954d3750638c0b084ee4879214 Author: Nils Goroll Date: Sun Jan 10 17:54:07 2021 +0100 vtest: plug insignificant leak introduced with 1c19f3fed3da8fae79ef200f2ebf01625a42bb27 CID 1471724 Ref #3490 diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 9b5a60a25..fa7524c2d 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -585,6 +585,7 @@ ip_magic(void) struct suckaddr *sa; char abuf[VTCP_ADDRBUFSIZE]; char pbuf[VTCP_PORTBUFSIZE]; + char *s; /* * In FreeBSD jails localhost/127.0.0.1 becomes the jails IP# @@ -622,10 +623,13 @@ ip_magic(void) extmacro_def("bad_backend", "[%s]:%s", abuf, pbuf); /* our default bind/listen address */ + s = macro_get("localhost", NULL); + AN(s); if (VSA_Get_Proto(sa) == AF_INET) - bprintf(abuf, "%s:0", macro_get("localhost", NULL)); + bprintf(abuf, "%s:0", s); else - bprintf(abuf, "[%s]:0", macro_get("localhost", NULL)); + bprintf(abuf, "[%s]:0", s); + free(s); extmacro_def("listen_addr", "%s", abuf); default_listen_addr = strdup(abuf); From nils.goroll at uplex.de Sun Jan 10 17:26:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 10 Jan 2021 17:26:06 +0000 (UTC) Subject: [master] 698a4742e vcc: Plug more insignificant leaks Message-ID: <20210110172606.68C98AF7C1@lists.varnish-cache.org> commit 698a4742efb8e14f7706fe593a5d4b8774abb63a Author: Nils Goroll Date: Sun Jan 10 17:59:32 2021 +0100 vcc: Plug more insignificant leaks Coverity CID 1362617, CID 1362625, CID 1362626, CID 1362627 diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index 6bfc553b2..361e900c7 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -194,8 +194,10 @@ vcc_acl_add_entry(struct vcc *tl, const struct acl_e *ae, int l, * If the two rules agree, silently ignore it * XXX: is that counter intuitive ? */ - if (aen->not == ae2->not) + if (aen->not == ae2->not) { + free(aen); return; + } VSB_cat(tl->sb, "Conflicting ACL entries:\n"); vcc_ErrWhere(tl, ae2->t_addr); VSB_cat(tl->sb, "vs:\n"); diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c index c6028deea..af75bbc04 100644 --- a/lib/libvcc/vcc_backend.c +++ b/lib/libvcc/vcc_backend.c @@ -149,15 +149,15 @@ vcc_Redef(struct vcc *tl, const char *redef, struct token **t_did, */ static void -vcc_ParseProbeSpec(struct vcc *tl, const struct symbol *sym, char **name) +vcc_ParseProbeSpec(struct vcc *tl, const struct symbol *sym, char **namep) { struct fld_spec *fs; struct token *t_field; struct token *t_did = NULL, *t_window = NULL, *t_threshold = NULL; struct token *t_initial = NULL; - struct vsb *vsb; - char *retval; unsigned window, threshold, initial, status; + char buf[32]; + const char *name; double t; fs = vcc_FldSpec(tl, @@ -173,25 +173,21 @@ vcc_ParseProbeSpec(struct vcc *tl, const struct symbol *sym, char **name) SkipToken(tl, '{'); - vsb = VSB_new_auto(); - AN(vsb); - if (sym != NULL) - VSB_cat(vsb, sym->rname); - else - VSB_printf(vsb, "vgc_probe__%d", tl->nprobe++); - AZ(VSB_finish(vsb)); - retval = TlDup(tl, VSB_data(vsb)); - AN(retval); - VSB_destroy(&vsb); - if (name != NULL) - *name = retval; + if (sym != NULL) { + name = sym->rname; + } else { + bprintf(buf, "vgc_probe__%d", tl->nprobe++); + name = buf; + } + Fh(tl, 0, "static const struct vrt_backend_probe %s[] = {{\n", name); + Fh(tl, 0, "\t.magic = VRT_BACKEND_PROBE_MAGIC,\n"); + if (namep != NULL) + *namep = TlDup(tl, name); window = 0; threshold = 0; initial = 0; status = 0; - Fh(tl, 0, "static const struct vrt_backend_probe %s[] = {{\n", retval); - Fh(tl, 0, "\t.magic = VRT_BACKEND_PROBE_MAGIC,\n"); while (tl->t->tok != '}') { vcc_IsField(tl, &t_field, fs); @@ -323,7 +319,7 @@ vcc_ParseProbe(struct vcc *tl) sym = VCC_HandleSymbol(tl, PROBE, "vgc_probe"); ERRCHK(tl); AN(sym); - vcc_ParseProbeSpec(tl, sym, &p); + vcc_ParseProbeSpec(tl, sym, NULL); } } @@ -480,6 +476,7 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname) } else if (vcc_IdIs(t_field, "probe") && tl->t->tok == '{') { vcc_ParseProbeSpec(tl, NULL, &p); Fb(tl, 0, "\t.probe = %s,\n", p); + free(p); ERRCHK(tl); } else if (vcc_IdIs(t_field, "probe") && tl->t->tok == ID) { if (vcc_IdIs(tl->t, "default")) { From phk at FreeBSD.org Mon Jan 11 08:55:08 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 11 Jan 2021 08:55:08 +0000 (UTC) Subject: [master] e75a5d70c Try to break VTEST by having our most fundamental test exercise both IPv4 and IPv6 Message-ID: <20210111085508.EA27AA029C@lists.varnish-cache.org> commit e75a5d70ce7a6d13a32ac616770d593e6f237015 Author: Poul-Henning Kamp Date: Mon Jan 11 08:53:33 2021 +0000 Try to break VTEST by having our most fundamental test exercise both IPv4 and IPv6 diff --git a/bin/varnishtest/tests/b00000.vtc b/bin/varnishtest/tests/b00000.vtc index 388c9b845..86d0aa6ed 100644 --- a/bin/varnishtest/tests/b00000.vtc +++ b/bin/varnishtest/tests/b00000.vtc @@ -5,7 +5,19 @@ server s1 { txresp -body "012345\n" } -start +server s2 -listen [::1]:0 { + rxreq + txresp -body "012345\n" +} -start + varnish v1 -vcl+backend { + sub vcl_recv { + if (req.url == "/1") { + set req.backend_hint = s1; + } else { + set req.backend_hint = s2; + } + } sub vcl_backend_response { set beresp.do_stream = false; } @@ -14,7 +26,7 @@ varnish v1 -vcl+backend { varnish v1 -cliok "param.set debug +workspace" varnish v1 -cliok "param.set debug +witness" -varnish v1 -vsc * +#varnish v1 -vsc * varnish v1 -expect MAIN.n_object == 0 varnish v1 -expect MAIN.sess_conn == 0 @@ -22,7 +34,7 @@ varnish v1 -expect MAIN.client_req == 0 varnish v1 -expect MAIN.cache_miss == 0 client c1 { - txreq -url "/" + txreq -url "/1" rxresp expect resp.status == 200 } -run @@ -34,3 +46,18 @@ varnish v1 -expect cache_miss == 1 varnish v1 -expect s_sess == 1 varnish v1 -expect s_resp_bodybytes == 7 varnish v1 -expect s_resp_hdrbytes == 178 + +client c1 { + txreq -url "/2" + rxresp + expect resp.status == 200 +} -run + +# varnish v1 -vsc * +varnish v1 -expect n_object == 2 +varnish v1 -expect sess_conn == 2 +varnish v1 -expect client_req == 2 +varnish v1 -expect cache_miss == 2 +varnish v1 -expect s_sess == 2 +varnish v1 -expect s_resp_bodybytes == 14 +varnish v1 -expect s_resp_hdrbytes == 356 From phk at FreeBSD.org Mon Jan 11 09:39:07 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 11 Jan 2021 09:39:07 +0000 (UTC) Subject: [master] c862d49ee Add two new feature-checks: `feature ipv4` and `feature ipv6` Message-ID: <20210111093907.92324A1964@lists.varnish-cache.org> commit c862d49eedbc609b8bb4cb7b2fe15706ec92ffaf Author: Poul-Henning Kamp Date: Mon Jan 11 09:37:59 2021 +0000 Add two new feature-checks: `feature ipv4` and `feature ipv6` diff --git a/bin/varnishtest/tests/b00000.vtc b/bin/varnishtest/tests/b00000.vtc index 86d0aa6ed..59cb4c119 100644 --- a/bin/varnishtest/tests/b00000.vtc +++ b/bin/varnishtest/tests/b00000.vtc @@ -1,6 +1,9 @@ varnishtest "Does anything get through at all ?" -server s1 { +feature ipv4 +feature ipv6 + +server s1 -listen 127.0.0.1:0 { rxreq txresp -body "012345\n" } -start diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index ca85fabd2..180bafb8b 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -365,6 +365,28 @@ dns_works(void) return (0); } +/********************************************************************** + * Test if IPv4/IPv6 works + */ + +static int +ipvx_works(const char *target) +{ + struct suckaddr *sa; + int fd; + + sa = VSS_ResolveOne(NULL, target, "0", 0, SOCK_STREAM, 0); + if (sa == NULL) + return (0); + fd = VTCP_bind(sa, NULL); + free(sa); + if (fd >= 0) { + VTCP_close(&fd); + return (1); + } + return(0); +} + /* SECTION: feature feature * * Test that the required feature(s) for a test are available, and skip @@ -439,6 +461,8 @@ cmd_feature(CMD_ARGS) #endif } + FEATURE("ipv4", ipvx_works("127.0.0.1")); + FEATURE("ipv6", ipvx_works("[::1]")); FEATURE("pcre_jit", VRE_has_jit); FEATURE("64bit", sizeof(void*) == 8); FEATURE("dns", dns_works()); From phk at FreeBSD.org Mon Jan 11 09:56:06 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 11 Jan 2021 09:56:06 +0000 (UTC) Subject: [master] 240c5cd8d Document new ipv4/ipv6 features Message-ID: <20210111095606.2CB42A227C@lists.varnish-cache.org> commit 240c5cd8da313b5236b02a0199a61be3e80fa12d Author: Poul-Henning Kamp Date: Mon Jan 11 09:55:10 2021 +0000 Document new ipv4/ipv6 features diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index 180bafb8b..88437b757 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -397,6 +397,10 @@ ipvx_works(const char *target) * The SO_RCVTIMEO socket option is working * 64bit * The environment is 64 bits + * ipv4 + * 127.0.0.1 work + * ipv6 + * [::1] work * dns * DNS lookups are working * topbuild @@ -414,7 +418,6 @@ ipvx_works(const char *target) * ignore_unknown_macro * Do not fail the test if a string of the form ${...} is not * recognized as a macro. - * * persistent_storage * Varnish was built with the deprecated persistent storage. * From nils.goroll at uplex.de Mon Jan 11 11:14:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 11:14:07 +0000 (UTC) Subject: [master] 48758ee50 doc: Build dependencies on Alpine Linux Message-ID: <20210111111407.AC5DFA49DE@lists.varnish-cache.org> commit 48758ee50df9f10b84f12bb6333f371207998209 Author: Nils Goroll Date: Mon Jan 11 12:12:25 2021 +0100 doc: Build dependencies on Alpine Linux diff --git a/doc/sphinx/installation/install_source.rst b/doc/sphinx/installation/install_source.rst index 084ac44d3..0df5e8c99 100644 --- a/doc/sphinx/installation/install_source.rst +++ b/doc/sphinx/installation/install_source.rst @@ -151,6 +151,45 @@ dependencies: Then continue `Compiling Varnish`_ +Build dependencies on Alpine Linux +---------------------------------- + +As of Alpine 3, these steps should install the required dependencies: + +* Add the `Alpine Community Repository`_ + +* Install dependencies:: + + apk add -q \ + autoconf \ + automake \ + build-base \ + ca-certificates \ + cpio \ + gzip \ + libedit-dev \ + libtool \ + libunwind-dev \ + linux-headers \ + pcre-dev \ + py-docutils \ + py3-sphinx \ + tar \ + sudo + +Optionally, to rebuild the svg files:: + + apk add -q graphviz + +Optionally, to pull from a repository:: + + apk add -q git + +Then continue `Compiling Varnish`_, using the ``--with-unwind`` +``configure`` option. + +.. _Alpine Community Repository: https://wiki.alpinelinux.org/wiki/Enable_Community_Repository + Build dependencies on a SmartOS Zone ------------------------------------ From dridi.boukelmoune at gmail.com Mon Jan 11 11:40:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 11 Jan 2021 11:40:06 +0000 (UTC) Subject: [master] b917fbe26 vcc: Constify struct token pointers Message-ID: <20210111114006.6BD43A56FD@lists.varnish-cache.org> commit b917fbe26fb84901433fd4aaf09df2f15a97c196 Author: Dridi Boukelmoune Date: Fri Nov 15 10:25:13 2019 +0100 vcc: Constify struct token pointers Refs #3134 diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c index af75bbc04..98a92c650 100644 --- a/lib/libvcc/vcc_backend.c +++ b/lib/libvcc/vcc_backend.c @@ -131,8 +131,8 @@ Emit_UDS_Path(struct vcc *tl, struct vsb *vsb1, */ static void -vcc_Redef(struct vcc *tl, const char *redef, struct token **t_did, - struct token *t_field) +vcc_Redef(struct vcc *tl, const char *redef, const struct token **t_did, + const struct token *t_field) { if (*t_did != NULL) { VSB_printf(tl->sb, "%s redefinition at:\n", redef); @@ -152,8 +152,8 @@ static void vcc_ParseProbeSpec(struct vcc *tl, const struct symbol *sym, char **namep) { struct fld_spec *fs; - struct token *t_field; - struct token *t_did = NULL, *t_window = NULL, *t_threshold = NULL; + const struct token *t_field; + const struct token *t_did = NULL, *t_window = NULL, *t_threshold = NULL; struct token *t_initial = NULL; unsigned window, threshold, initial, status; char buf[32]; @@ -332,14 +332,14 @@ vcc_ParseProbe(struct vcc *tl) static void vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname) { - struct token *t_field; - struct token *t_val; - struct token *t_host = NULL; - struct token *t_port = NULL; - struct token *t_path = NULL; - struct token *t_hosthdr = NULL; + const struct token *t_field; + const struct token *t_val; + const struct token *t_host = NULL; + const struct token *t_port = NULL; + const struct token *t_path = NULL; + const struct token *t_hosthdr = NULL; + const struct token *t_did = NULL; struct symbol *pb; - struct token *t_did = NULL; struct fld_spec *fs; struct inifin *ifp; struct vsb *vsb1; diff --git a/lib/libvcc/vcc_backend_util.c b/lib/libvcc/vcc_backend_util.c index fa3583a72..eb6cc83ab 100644 --- a/lib/libvcc/vcc_backend_util.c +++ b/lib/libvcc/vcc_backend_util.c @@ -88,7 +88,7 @@ vcc_FldSpec(struct vcc *tl, const char *first, ...) } void -vcc_IsField(struct vcc *tl, struct token **t, struct fld_spec *fs) +vcc_IsField(struct vcc *tl, const struct token **t, struct fld_spec *fs) { struct token *t_field; diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 815b25918..dd94de055 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -307,7 +307,7 @@ void vcc_Backend_Init(struct vcc *tl); void vcc_ParseProbe(struct vcc *tl); void vcc_ParseBackend(struct vcc *tl); struct fld_spec * vcc_FldSpec(struct vcc *tl, const char *first, ...); -void vcc_IsField(struct vcc *tl, struct token **t, struct fld_spec *fs); +void vcc_IsField(struct vcc *tl, const struct token **t, struct fld_spec *fs); void vcc_FieldsOk(struct vcc *tl, const struct fld_spec *fs); /* vcc_compile.c */ From nils.goroll at uplex.de Mon Jan 11 12:27:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 12:27:06 +0000 (UTC) Subject: [master] 05356fb7c varnishd: VTCP_name() is not for UDS Message-ID: <20210111122706.F21E8A6CFC@lists.varnish-cache.org> commit 05356fb7c6c10d3ddd9766f474da679046922422 Author: Nils Goroll Date: Mon Jan 11 13:18:09 2021 +0100 varnishd: VTCP_name() is not for UDS fixes a regression introduced by c1352b0b0ece8028317c838751d9042a55347cd5: by not using a suckaddr, VTCP_myname() would also work on UDS, but none of the VTCP was ever intended to (and there are other places where that is asserted). So one could argue that 9837045a011db7496b6a0680687c48fbbac41186 was already wrong, but I do not want to deflect from my own mistake in any way. Fixes #3492 diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index d638a9f8e..cd99c9b76 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -258,16 +258,17 @@ vbp_write_proxy_v1(struct vbp_target *vt, int *sock) sua = VSA_getsockname(*sock, vsabuf, sizeof vsabuf); AN(sua); - VTCP_name(sua, addr, sizeof addr, port, sizeof port); AN(VSB_init(&vsb, buf, sizeof buf)); proto = VSA_Get_Proto(sua); - if (proto == AF_INET || proto == AF_INET6) + if (proto == AF_INET || proto == AF_INET6) { + VTCP_name(sua, addr, sizeof addr, port, sizeof port); VSB_printf(&vsb, "PROXY %s %s %s %s %s\r\n", proto == AF_INET ? "TCP4" : "TCP6", addr, addr, port, port); - else + } else { VSB_cat(&vsb, "PROXY UNKNOWN\r\n"); + } AZ(VSB_finish(&vsb)); VSB_fini(&vsb); diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c index 84832cb65..e56cd3ba9 100644 --- a/lib/libvarnish/vtcp.c +++ b/lib/libvarnish/vtcp.c @@ -100,6 +100,7 @@ VTCP_name(const struct suckaddr *addr, char *abuf, unsigned alen, socklen_t sl; sa = VSA_Get_Sockaddr(addr, &sl); + AN(sa); vtcp_sa_to_ascii(sa, sl, abuf, alen, pbuf, plen); } From nils.goroll at uplex.de Mon Jan 11 14:26:03 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 14:26:03 +0000 (UTC) Subject: [master] 7d6d1d853 Use req->ws in VDI_Http1Pipe() Message-ID: <20210111142603.44B95AA1EB@lists.varnish-cache.org> commit 7d6d1d853bf9561a0d62c5d8c4d74049020a60ec Author: Reza Naghibi Date: Fri Sep 11 10:19:01 2020 -0400 Use req->ws in VDI_Http1Pipe() diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index c7e701026..50e487f63 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -208,6 +208,7 @@ VDI_Http1Pipe(struct req *req, struct busyobj *bo) INIT_OBJ(ctx, VRT_CTX_MAGIC); VCL_Req2Ctx(ctx, req); VCL_Bo2Ctx(ctx, bo); + ctx->ws = req->ws; d = VDI_Resolve(ctx); if (d == NULL || d->vdir->methods->http1pipe == NULL) { From nils.goroll at uplex.de Mon Jan 11 14:26:03 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 14:26:03 +0000 (UTC) Subject: [master] 16e98e969 Use req->ws in the ctx when piping Message-ID: <20210111142603.59D4AAA1EE@lists.varnish-cache.org> commit 16e98e9695dce19fc7c9523ea6118bedc7c5bc27 Author: Reza Naghibi Date: Fri Sep 11 10:19:57 2020 -0400 Use req->ws in the ctx when piping Previously bo->ws was used which led to workspace mixing. req->ws safely outlives both the bo and req object when piping. diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c index 9b4d44c3c..96537a943 100644 --- a/bin/varnishd/cache/cache_vrt_vcl.c +++ b/bin/varnishd/cache/cache_vrt_vcl.c @@ -459,11 +459,13 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo, VCL_Req2Ctx(&ctx, req); } if (bo != NULL) { - if (req) - assert(method == VCL_MET_PIPE); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(bo->vcl, VCL_MAGIC); VCL_Bo2Ctx(&ctx, bo); + if (req) { + assert(method == VCL_MET_PIPE); + ctx.ws = req->ws; + } } assert(ctx.now != 0); ctx.specific = specific; diff --git a/bin/varnishtest/tests/r03329.vtc b/bin/varnishtest/tests/r03329.vtc new file mode 100644 index 000000000..f3453bb61 --- /dev/null +++ b/bin/varnishtest/tests/r03329.vtc @@ -0,0 +1,18 @@ +varnishtest "Lost reason from pipe to synth" + +server s1 {} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + return (pipe); + } + sub vcl_pipe { + return (synth(505, req.http.foo + "/bar")); + } +} -start + +client c1 { + txreq -hdr "foo: foo" + rxresp + expect resp.reason == "foo/bar" +} -run diff --git a/bin/varnishtest/tests/r03385.vtc b/bin/varnishtest/tests/r03385.vtc new file mode 100644 index 000000000..cee34301f --- /dev/null +++ b/bin/varnishtest/tests/r03385.vtc @@ -0,0 +1,23 @@ +varnishtest "Use a priv in vcl_pipe" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + import debug; + + sub vcl_recv { + return (pipe); + } + + sub vcl_pipe { + debug.test_priv_task(); + } +} -start + +client c1 { + txreq + rxresp +} -run From nils.goroll at uplex.de Mon Jan 11 14:26:03 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 14:26:03 +0000 (UTC) Subject: [master] f4e21600d slightly polish the previous two commits Message-ID: <20210111142603.79509AA1F2@lists.varnish-cache.org> commit f4e21600dfff2d5566a57c784449431df4c36248 Author: Nils Goroll Date: Fri Nov 27 17:28:45 2020 +0100 slightly polish the previous two commits instead of overriding ctx.ws spefically, just let the Req2Ctx settings win This also moves pipe logging to the client side where it belongs. diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index 50e487f63..595265f55 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -206,9 +206,8 @@ VDI_Http1Pipe(struct req *req, struct busyobj *bo) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); INIT_OBJ(ctx, VRT_CTX_MAGIC); - VCL_Req2Ctx(ctx, req); VCL_Bo2Ctx(ctx, bo); - ctx->ws = req->ws; + VCL_Req2Ctx(ctx, req); d = VDI_Resolve(ctx); if (d == NULL || d->vdir->methods->http1pipe == NULL) { diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c index 96537a943..04cc68107 100644 --- a/bin/varnishd/cache/cache_vrt_vcl.c +++ b/bin/varnishd/cache/cache_vrt_vcl.c @@ -451,22 +451,20 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo, CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); INIT_OBJ(&ctx, VRT_CTX_MAGIC); + if (bo != NULL) { + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + CHECK_OBJ_NOTNULL(bo->vcl, VCL_MAGIC); + VCL_Bo2Ctx(&ctx, bo); + } if (req != NULL) { + if (bo != NULL) + assert(method == VCL_MET_PIPE); CHECK_OBJ(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(req->vcl, VCL_MAGIC); CHECK_OBJ_NOTNULL(req->top, REQTOP_MAGIC); VCL_Req2Ctx(&ctx, req); } - if (bo != NULL) { - CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - CHECK_OBJ_NOTNULL(bo->vcl, VCL_MAGIC); - VCL_Bo2Ctx(&ctx, bo); - if (req) { - assert(method == VCL_MET_PIPE); - ctx.ws = req->ws; - } - } assert(ctx.now != 0); ctx.specific = specific; ctx.method = method; diff --git a/bin/varnishtest/tests/v00051.vtc b/bin/varnishtest/tests/v00051.vtc index 42c12ed3e..8948d76d5 100644 --- a/bin/varnishtest/tests/v00051.vtc +++ b/bin/varnishtest/tests/v00051.vtc @@ -158,9 +158,6 @@ logexpect l2 -v v1 -g vxid -q "vxid == 1012" { expect 0 = BereqHeader {^X-Forwarded-For: } expect 0 = BereqHeader {^X-Varnish: 1011} expect 0 = BereqHeader {^Connection: close} - expect 0 = VCL_call {^PIPE} - expect 0 = VCL_Error {^Forced failure} - expect 0 = VCL_return {^fail} expect 0 = BereqAcct {^0 0 0 0 0 0} expect 0 = End } -start @@ -182,6 +179,9 @@ logexpect l3 -v v1 -g vxid -q "vxid == 1011" { expect 0 = VCL_call {^HASH} expect 0 = VCL_return {^lookup} expect 0 = Link {^bereq 1012 pipe} + expect 0 = VCL_call {^PIPE} + expect 0 = VCL_Error {^Forced failure} + expect 0 = VCL_return {^fail} expect 0 = RespProtocol {^HTTP/1.1} expect 0 = RespStatus {^503} expect 0 = RespReason {^VCL failed} From nils.goroll at uplex.de Mon Jan 11 14:26:03 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 14:26:03 +0000 (UTC) Subject: [master] 33ad6de7f Move bereq.http to req->ws for vcl_pipe Message-ID: <20210111142603.990BFAA1FB@lists.varnish-cache.org> commit 33ad6de7fe590d90425c94a9c79ce912291a3f43 Author: Reza Naghibi Date: Thu Oct 15 09:40:09 2020 -0400 Move bereq.http to req->ws for vcl_pipe Also make sure we didn't overflow before entering vcl_pipe. This would mean we have lost important connection headers. diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 7c4e7c036..452dc344a 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -745,7 +745,7 @@ cnt_pipe(struct worker *wrk, struct req *req) bo->sp = req->sp; SES_Ref(bo->sp); - HTTP_Setup(bo->bereq, bo->ws, bo->vsl, SLT_BereqMethod); + HTTP_Setup(bo->bereq, req->ws, bo->vsl, SLT_BereqMethod); http_FilterReq(bo->bereq, req->http, 0); // XXX: 0 ? http_PrintfHeader(bo->bereq, "X-Varnish: %u", VXID(req->vsl->wid)); http_ForceHeader(bo->bereq, H_Connection, "close"); @@ -756,7 +756,10 @@ cnt_pipe(struct worker *wrk, struct req *req) } bo->wrk = wrk; - VCL_pipe_method(req->vcl, wrk, req, bo, NULL); + if (WS_Overflowed(req->ws)) + wrk->handling = VCL_RET_FAIL; + else + VCL_pipe_method(req->vcl, wrk, req, bo, NULL); switch (wrk->handling) { case VCL_RET_SYNTH: diff --git a/bin/varnishtest/tests/r03329.vtc b/bin/varnishtest/tests/r03329.vtc index f3453bb61..8889f4061 100644 --- a/bin/varnishtest/tests/r03329.vtc +++ b/bin/varnishtest/tests/r03329.vtc @@ -3,11 +3,20 @@ varnishtest "Lost reason from pipe to synth" server s1 {} -start varnish v1 -vcl+backend { + import vtc; sub vcl_recv { + if (req.http.workspace) { + vtc.workspace_alloc(client, -10); + } return (pipe); } sub vcl_pipe { - return (synth(505, req.http.foo + "/bar")); + if (req.http.foo) { + return (synth(505, req.http.foo + "/bar")); + } else { + set bereq.http.baz = "baz"; + return (synth(505, bereq.http.baz)); + } } } -start @@ -15,4 +24,12 @@ client c1 { txreq -hdr "foo: foo" rxresp expect resp.reason == "foo/bar" + + txreq + rxresp + expect resp.reason == "baz" + + txreq -hdr "workspace: true" + rxresp + expect resp.status == 503 } -run From nils.goroll at uplex.de Mon Jan 11 14:26:03 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 14:26:03 +0000 (UTC) Subject: [master] 7de4ae97e outlaw std.rollback(req) in vcl_pipe {} Message-ID: <20210111142603.B206FAA209@lists.varnish-cache.org> commit 7de4ae97e18ba2f31f7d6a24c1194746e6834e5e Author: Nils Goroll Date: Fri Nov 27 18:28:47 2020 +0100 outlaw std.rollback(req) in vcl_pipe {} With the previous commits, as the bereq uses the req workspace, we cannot roll back the req without also rolling back the bereq. As the use case is rather exotic, we spare the additional complications and just outlaw rollbacks from vcl_pipe. std.rollback(bereq) is already forbidden by the compiler because the bereq symbol is not available in vcl_pipe {}. We could, alternative to the runtime failure for std.rollback(req), remove the req symbol from vcl_pipe {}, but that could break use of req to vmod functions also. So we chose the more compatible route of selectively failing rollback rather than making impossible other use cases. diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 327a651f1..67abc24b2 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -795,6 +795,10 @@ VRT_Rollback(VRT_CTX, VCL_HTTP hp) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); + if (ctx->method & VCL_MET_PIPE) { + VRT_fail(ctx, "Cannot rollback in vcl_pipe {}"); + return; + } if (hp == ctx->http_req) { CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); Req_Rollback(ctx->req); diff --git a/bin/varnishtest/tests/m00017.vtc b/bin/varnishtest/tests/m00017.vtc index 78e3bdcbc..012e92989 100644 --- a/bin/varnishtest/tests/m00017.vtc +++ b/bin/varnishtest/tests/m00017.vtc @@ -3,6 +3,8 @@ varnishtest "Test std.rollback" # bug regressions: # - #3009 # - #3083 +# - #3329 +# - #3385 server s1 { rxreq @@ -335,3 +337,40 @@ client c6 { } -run server s1 -wait + +# this could work, but would need additional coding to save +# the right snapshot of the bereq on the req ws +varnish v1 -errvcl {Not available in subroutine 'vcl_pipe'} { + import std; + + backend proforma None; + + sub vcl_pipe { + std.rollback(bereq); + } +} + +# vcl_pipe { std.rollback(req); } cannot work unless it also implied +# rolling back the bereq first. +# We would want to remove req from vcl_pipe, but that could break +# vmods, so we fail specifically at runtime + +varnish v1 -vcl { + import std; + + backend proforma None; + + sub vcl_recv { + return (pipe); + } + sub vcl_pipe { + std.rollback(req); + } +} + +client c7 { + txreq -url / + rxresp + expect resp.status == 503 + expect resp.reason == "VCL failed" +} -run From nils.goroll at uplex.de Mon Jan 11 14:26:03 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 14:26:03 +0000 (UTC) Subject: [master] c12449820 update comment Message-ID: <20210111142603.C8A2BAA210@lists.varnish-cache.org> commit c12449820b41513e2eb700d26e97a73a0c71df36 Author: Nils Goroll Date: Fri Nov 27 19:00:23 2020 +0100 update comment diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index 5cd8e06ae..00ef67309 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -144,12 +144,8 @@ vrt_priv_task_context(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - /* - * XXX when coming from VRT_DirectorResolve() in pipe mode - * (ctx->method == 0), both req and bo are set. - * see #3329 #3330: we should make up our mind where - * pipe objects live - */ + + /* In pipe mode, both req and bo are set. We use req */ assert(ctx->req == NULL || ctx->bo == NULL || ctx->method == VCL_MET_PIPE || ctx->method == 0); From nils.goroll at uplex.de Mon Jan 11 14:26:03 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 14:26:03 +0000 (UTC) Subject: [master] c63ddc8f5 assert magic of vrt_privs when finalizing Message-ID: <20210111142603.E03F5AA214@lists.varnish-cache.org> commit c63ddc8f5b504764dfb6fed4f580655fa909e8a1 Author: Nils Goroll Date: Fri Nov 27 15:27:31 2020 +0100 assert magic of vrt_privs when finalizing Originally suggested by @rezan in https://github.com/varnishcache/varnish-cache/pull/3408/commits/4caad2c1858afb399f75230916183d31f57fc908 diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index 00ef67309..9987abe21 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -238,7 +238,9 @@ VCL_TaskLeave(struct vrt_privs *privs) * a costly operation. Instead we safely walk the whole tree and clear * the head at the very end. */ - VRBT_FOREACH_SAFE(vp, vrt_privs, privs, vp1) + VRBT_FOREACH_SAFE(vp, vrt_privs, privs, vp1) { + CHECK_OBJ(vp, VRT_PRIV_MAGIC); VRT_priv_fini(vp->priv); + } ZERO_OBJ(privs, sizeof *privs); } From nils.goroll at uplex.de Mon Jan 11 14:26:04 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 14:26:04 +0000 (UTC) Subject: [master] ff504802c make bereq.xid available in vcl_pipe Message-ID: <20210111142604.0CAE3AA221@lists.varnish-cache.org> commit ff504802c9033f28eb05642901947e26172ec5c2 Author: Nils Goroll Date: Mon Jan 11 15:02:37 2021 +0100 make bereq.xid available in vcl_pipe as suggested by Dridi diff --git a/bin/varnishtest/tests/b00001.vtc b/bin/varnishtest/tests/b00001.vtc index 3a07e5eaf..db16d4bdd 100644 --- a/bin/varnishtest/tests/b00001.vtc +++ b/bin/varnishtest/tests/b00001.vtc @@ -20,6 +20,7 @@ varnish v1 -vcl+backend { if (req.url == "/2") { set bereq.http.connection = req.http.connection; } + set bereq.http.xid = bereq.xid; } } -start diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index 828c1e60e..c9fa94df9 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -480,7 +480,7 @@ bereq.xid Type: STRING - Readable from: backend + Readable from: vcl_pipe, backend Unique ID of this request. From nils.goroll at uplex.de Mon Jan 11 16:04:02 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 16:04:02 +0000 (UTC) Subject: [master] 06cf60a02 Add VRT_priv_task_lookup() Message-ID: <20210111160402.BFB0BAD3B7@lists.varnish-cache.org> commit 06cf60a020a81e49d001ee42e4d85f16580f3fe0 Author: Nils Goroll Date: Tue Jan 5 12:47:20 2021 +0100 Add VRT_priv_task_lookup() VRT_priv_task() inserts a priv_task if it does not exist. In some cases, it is more efficient to only insert a priv_task under certain conditions. These require an API function to only return a priv_task if it exists. This is VRT_priv_task_lookup(). A TODO comment has been added for a sensible performance optimization, avoiding one VRBT traversal for the VRT_priv_task() case. I would address this after this suggestion got accepted, if so. diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index 9987abe21..1685ec06e 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -116,19 +116,36 @@ vrt_priv_dyncmp(const struct vrt_priv *vp1, const struct vrt_priv *vp2) VRBT_GENERATE_STATIC(vrt_privs, vrt_priv, entry, vrt_priv_dyncmp) static struct vmod_priv * -vrt_priv_dynamic(struct ws *ws, struct vrt_privs *privs, uintptr_t vmod_id) +vrt_priv_dynamic_get(struct ws *ws, struct vrt_privs *privs, uintptr_t vmod_id) { struct vrt_priv *vp; + const struct vrt_priv needle = {.vmod_id = vmod_id}; + vp = VRBT_FIND(vrt_privs, privs, &needle); + if (vp == NULL) + return (NULL); + + CHECK_OBJ(vp, VRT_PRIV_MAGIC); + assert(vp->vmod_id == vmod_id); + return (vp->priv); +} + +static struct vmod_priv * +vrt_priv_dynamic(struct ws *ws, struct vrt_privs *privs, uintptr_t vmod_id) +{ + struct vrt_priv *vp; + static struct vmod_priv *r; + AN(vmod_id); - vp = VRBT_FIND(vrt_privs, privs, &needle); - if (vp) { - CHECK_OBJ(vp, VRT_PRIV_MAGIC); - assert(vp->vmod_id == vmod_id); - return (vp->priv); - } + /* + * TODO: for insert == 1, we can avoid the VRBT_FIND: + * call only VRT_INSERT and reset the ws if the element existed + */ + r = vrt_priv_dynamic_get(ws, privs, vmod_id); + if (r) + return (r); vp = WS_Alloc(ws, sizeof *vp); if (vp == NULL) @@ -162,6 +179,17 @@ vrt_priv_task_context(VRT_CTX) return (cli_task_privs); } +struct vmod_priv * +VRT_priv_task_get(VRT_CTX, const void *vmod_id) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + + return (vrt_priv_dynamic_get( + ctx->ws, + vrt_priv_task_context(ctx), + (uintptr_t)vmod_id)); +} + struct vmod_priv * VRT_priv_task(VRT_CTX, const void *vmod_id) { diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc index 48fc749c0..49c9cc234 100644 --- a/bin/varnishtest/tests/m00000.vtc +++ b/bin/varnishtest/tests/m00000.vtc @@ -47,6 +47,7 @@ varnish v1 -vcl+backend { } sub priv_task { + debug.test_priv_task_get(); debug.test_priv_task("foo"); } diff --git a/include/vrt.h b/include/vrt.h index fe888060e..23de2361b 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -59,6 +59,7 @@ * struct vmod_priv free member replaced with methods * VRT_CTX_Assert() added * VRT_ban_string() signature changed + * VRT_priv_task_get() added * 12.0 (2020-09-15) * Added VRT_DirectorResolve() * Added VCL_STRING VRT_BLOB_string(VRT_CTX, VCL_BLOB) @@ -597,6 +598,7 @@ struct vmod_priv { void VRT_priv_fini(const struct vmod_priv *p); struct vmod_priv *VRT_priv_task(VRT_CTX, const void *vmod_id); +struct vmod_priv *VRT_priv_task_get(VRT_CTX, const void *vmod_id); struct vmod_priv *VRT_priv_top(VRT_CTX, const void *vmod_id); /* Stevedore related functions */ diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index 2288c4ff1..bb663ea94 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -243,6 +243,14 @@ xyzzy_test_priv_call(VRT_CTX, struct vmod_priv *priv) } } +VCL_VOID v_matchproto_(td_debug_test_priv_task_get) +xyzzy_test_priv_task_get(VRT_CTX) +{ + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AZ(VRT_priv_task_get(ctx, NULL)); +} + static void priv_task_free(void *ptr) { @@ -848,7 +856,7 @@ xyzzy_priv_perf(VRT_CTX, VCL_INT size, VCL_INT rounds) t0 = VTIM_mono(); for (r = 0; r < rounds; r++) { for (s = 1; s <= size; s++) { - p = VRT_priv_task(ctx, (void *)(uintptr_t)s); + p = VRT_priv_task_get(ctx, (void *)(uintptr_t)s); AN(p); check += (uintptr_t)p->priv; p->priv = (void *)(uintptr_t)(s * rounds + r); @@ -1035,7 +1043,7 @@ xyzzy_get_ip(VRT_CTX) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - priv = VRT_priv_task(ctx, &store_ip_token); + priv = VRT_priv_task_get(ctx, &store_ip_token); AN(priv); AZ(priv->methods); diff --git a/vmod/vmod_debug.vcc b/vmod/vmod_debug.vcc index 4eee74547..fa81c8cf8 100644 --- a/vmod/vmod_debug.vcc +++ b/vmod/vmod_debug.vcc @@ -56,6 +56,10 @@ $Function VOID test_priv_vcl(PRIV_VCL) Test function for VCL private pointers +$Function VOID test_priv_task_get() + +Assert that the priv_task for the NULL pointer is NULL. + $Function STRING test_priv_task(PRIV_TASK, STRING s="") Test function for TASK private pointers From nils.goroll at uplex.de Mon Jan 11 16:04:02 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 16:04:02 +0000 (UTC) Subject: [master] ce3dbbed2 Assert that VRBT_INSERT did not find the element to insert Message-ID: <20210111160402.D4BA2AD3BA@lists.varnish-cache.org> commit ce3dbbed23749d1585f3da5b32063b8b746f9f42 Author: Nils Goroll Date: Mon Jan 11 16:08:58 2021 +0100 Assert that VRBT_INSERT did not find the element to insert just to clarify for the next commit. diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index 1685ec06e..059392386 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -152,7 +152,7 @@ vrt_priv_dynamic(struct ws *ws, struct vrt_privs *privs, uintptr_t vmod_id) return (NULL); INIT_OBJ(vp, VRT_PRIV_MAGIC); vp->vmod_id = vmod_id; - VRBT_INSERT(vrt_privs, privs, vp); + AZ(VRBT_INSERT(vrt_privs, privs, vp)); return (vp->priv); } From nils.goroll at uplex.de Mon Jan 11 16:35:32 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 17:35:32 +0100 Subject: [master] 06cf60a02 Add VRT_priv_task_lookup() In-Reply-To: <20210111160402.BFB0BAD3B7@lists.varnish-cache.org> References: <20210111160402.BFB0BAD3B7@lists.varnish-cache.org> Message-ID: <975b3314-fc5a-68e4-abc1-08018eb9d2ce@uplex.de> On 11/01/2021 17:04, Nils Goroll wrote: > Add VRT_priv_task_lookup() I should have edited that commit message before merge. It should have read "Add VRT_priv_task_get()". Sorry. Nils -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From nils.goroll at uplex.de Mon Jan 11 16:45:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 16:45:07 +0000 (UTC) Subject: [master] 8edfddaaa Remove superfluous argument Message-ID: <20210111164507.B9E4AAEB32@lists.varnish-cache.org> commit 8edfddaaaaf3867f20b3e365ceb5c0c4fd3362fb Author: Nils Goroll Date: Mon Jan 11 17:38:37 2021 +0100 Remove superfluous argument polishes 06cf60a020a81e49d001ee42e4d85f16580f3fe0 I should have taken more time for that merge. diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index 059392386..3beac7f3b 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -116,7 +116,7 @@ vrt_priv_dyncmp(const struct vrt_priv *vp1, const struct vrt_priv *vp2) VRBT_GENERATE_STATIC(vrt_privs, vrt_priv, entry, vrt_priv_dyncmp) static struct vmod_priv * -vrt_priv_dynamic_get(struct ws *ws, struct vrt_privs *privs, uintptr_t vmod_id) +vrt_priv_dynamic_get(struct vrt_privs *privs, uintptr_t vmod_id) { struct vrt_priv *vp; @@ -143,7 +143,7 @@ vrt_priv_dynamic(struct ws *ws, struct vrt_privs *privs, uintptr_t vmod_id) * TODO: for insert == 1, we can avoid the VRBT_FIND: * call only VRT_INSERT and reset the ws if the element existed */ - r = vrt_priv_dynamic_get(ws, privs, vmod_id); + r = vrt_priv_dynamic_get(privs, vmod_id); if (r) return (r); @@ -185,7 +185,6 @@ VRT_priv_task_get(VRT_CTX, const void *vmod_id) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); return (vrt_priv_dynamic_get( - ctx->ws, vrt_priv_task_context(ctx), (uintptr_t)vmod_id)); } From nils.goroll at uplex.de Mon Jan 11 16:45:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 16:45:07 +0000 (UTC) Subject: [master] 6efccf76a Avoid a second rbtree lookup in VRT_priv_task() Message-ID: <20210111164507.DAFC1AEB35@lists.varnish-cache.org> commit 6efccf76a0fa4dfd43a2ce2f740a3a839545422d Author: Nils Goroll Date: Mon Jan 11 17:02:21 2021 +0100 Avoid a second rbtree lookup in VRT_priv_task() For a red-black tree insert in vrt_priv_dynamic(), we performed a lookup and then an insert, resulting in basically the same tree traversal twice. We now prepare a new struct vrt_priv on the workspace and release it if the respective priv already existed on the tree. For a full workspace, we fall back to a lookup via vrt_priv_dynamic_get(). diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index 3beac7f3b..b006f52ae 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -134,26 +134,24 @@ vrt_priv_dynamic_get(struct vrt_privs *privs, uintptr_t vmod_id) static struct vmod_priv * vrt_priv_dynamic(struct ws *ws, struct vrt_privs *privs, uintptr_t vmod_id) { - struct vrt_priv *vp; - static struct vmod_priv *r; + struct vrt_priv *vp, *ovp; AN(vmod_id); - /* - * TODO: for insert == 1, we can avoid the VRBT_FIND: - * call only VRT_INSERT and reset the ws if the element existed - */ - r = vrt_priv_dynamic_get(privs, vmod_id); - if (r) - return (r); + /* even if ws is full, return any existing priv */ + if (WS_ReserveSize(ws, sizeof *vp) == 0) + return (vrt_priv_dynamic_get(privs, vmod_id)); - vp = WS_Alloc(ws, sizeof *vp); - if (vp == NULL) - return (NULL); + vp = WS_Reservation(ws); INIT_OBJ(vp, VRT_PRIV_MAGIC); vp->vmod_id = vmod_id; - AZ(VRBT_INSERT(vrt_privs, privs, vp)); - return (vp->priv); + ovp = VRBT_INSERT(vrt_privs, privs, vp); + if (ovp == NULL) { + WS_Release(ws, sizeof *vp); + return (vp->priv); + } + WS_Release(ws, 0); + return (ovp->priv); } static struct vrt_privs * From dridi.boukelmoune at gmail.com Mon Jan 11 17:06:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 11 Jan 2021 17:06:07 +0000 (UTC) Subject: [master] 3694a418f Push pool stats to global stats on delayed VCL release Message-ID: <20210111170607.1E19BAF7B9@lists.varnish-cache.org> commit 3694a418f1234bb0b8196676ab4cf1698a152343 Author: Martin Blix Grydeland Date: Thu May 14 14:18:40 2020 +0200 Push pool stats to global stats on delayed VCL release Often in test cases the siutation comes where multiple threads go idle at the same time, and no new activity happens after that. This would potentially create the situation where the last thread to go idle sees that some thread is currently pushing stats to global, so it leaves its stats contribution on the pool stats to avoid contention. But without new activity in the future, the pool stats may stay in the pool accumulator indefinitely. This patch adds a check to see if there are unaccumulated statistics lingering in the pool accumulator when a thread wakes after idling to release its VCL reference. If there are, the thread takes on the task of pushing the pool stats to the global stats. This should help with test case stability, and potentially help with underflowed stats reported by some users. diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 39e4b4361..b7b31296c 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -398,10 +398,32 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk) i = Lck_CondWait(&wrk->cond, &pp->mtx, tmo); if (i == ETIMEDOUT) VCL_Rel(&wrk->vcl); - } while (wrk->task->func == NULL); - tpx = *wrk->task; - tp = &tpx; - wrk->stats->summs++; + if (wrk->task->func != NULL) { + /* We have been handed a new task */ + tpx = *wrk->task; + tp = &tpx; + wrk->stats->summs++; + } else if (pp->b_stat != NULL && + pp->a_stat->summs) { + /* Woken up to release the VCL, + * and noticing that there are + * pool stats not pushed to the + * global stats and no active + * thread currently doing + * it. Remove ourself from the + * idle queue and take on the + * task. */ + assert(pp->nidle > 0); + VTAILQ_REMOVE(&pp->idle_queue, + wrk->task, list); + pp->nidle--; + tps.func = pool_stat_summ; + tps.priv = pp->a_stat; + pp->a_stat = pp->b_stat; + pp->b_stat = NULL; + tp = &tps; + } + } while (tp == NULL); } Lck_Unlock(&pp->mtx); From dridi.boukelmoune at gmail.com Mon Jan 11 17:06:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 11 Jan 2021 17:06:07 +0000 (UTC) Subject: [master] 9ab876d7e Special case late pool stats push also for DBG_VCLREL Message-ID: <20210111170607.35FABAF7BE@lists.varnish-cache.org> commit 9ab876d7ead322d3acffe3af9fdeb5a80512375d Author: Martin Blix Grydeland Date: Tue May 19 12:54:13 2020 +0200 Special case late pool stats push also for DBG_VCLREL If the DBG_VCLREL is in effect and there are unpushed pool stats, make sure that some thread is waking up to perform the late push. This should help with stability for VTC test cases using the DBG_VCLREL flag. diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index b7b31296c..ee0185146 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -389,7 +389,17 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk) pp->nidle++; do { // see signaling_note at the top for explanation - if (wrk->vcl == NULL) + if (DO_DEBUG(DBG_VCLREL) && + pp->b_stat == NULL && pp->a_stat->summs) + /* We've released the VCL, but + * there are pool stats not pushed + * to the global stats and some + * thread is busy pushing + * stats. Set a 1 second timeout + * so that we'll wake up and get a + * chance to push stats. */ + tmo = wrk->lastused + 1.; + else if (wrk->vcl == NULL) tmo = 0; else if (DO_DEBUG(DBG_VTC_MODE)) tmo = wrk->lastused+1.; From dridi.boukelmoune at gmail.com Mon Jan 11 17:06:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 11 Jan 2021 17:06:07 +0000 (UTC) Subject: [master] 69dfe4980 vsc: New VSC_Value() function to clamp gauges Message-ID: <20210111170607.5B3C3AF7C3@lists.varnish-cache.org> commit 69dfe49809322242037dea26e883ed7d5a5d21a6 Author: Dridi Boukelmoune Date: Tue Oct 20 16:13:58 2020 +0200 vsc: New VSC_Value() function to clamp gauges When gauges are incremented and decremented by different threads, there is a chance that decrements are published first, provoking an underflow of the VSC. Values with the MSB set are considered underflowed, adjusted to zero instead. diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c index 75c88311f..695ca092c 100644 --- a/bin/varnishstat/varnishstat.c +++ b/bin/varnishstat/varnishstat.c @@ -67,7 +67,7 @@ do_xml_cb(void *priv, const struct VSC_point * const pt) if (pt == NULL) return (0); AZ(strcmp(pt->ctype, "uint64_t")); - val = *(const volatile uint64_t*)pt->ptr; + val = VSC_Value(pt); printf("\t\n"); printf("\t\t%s\n", pt->name); @@ -106,7 +106,7 @@ do_json_cb(void *priv, const struct VSC_point * const pt) return (0); AZ(strcmp(pt->ctype, "uint64_t")); - val = (uintmax_t)*(const volatile uint64_t*)pt->ptr; + val = (uintmax_t)VSC_Value(pt); sep = priv; @@ -167,7 +167,7 @@ do_once_cb_first(void *priv, const struct VSC_point * const pt) AZ(strcmp(pt->ctype, "uint64_t")); if (strcmp(pt->name, "MAIN.uptime")) return (0); - val = *(const volatile uint64_t*)pt->ptr; + val = VSC_Value(pt); op->up = (double)val; return (1); } @@ -183,7 +183,7 @@ do_once_cb(void *priv, const struct VSC_point * const pt) return (0); op = priv; AZ(strcmp(pt->ctype, "uint64_t")); - val = *(const volatile uint64_t*)pt->ptr; + val = VSC_Value(pt); i = 0; i += printf("%s", pt->name); if (i >= op->pad) diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c index f7c486db1..4c3ff9127 100644 --- a/bin/varnishstat/varnishstat_curses.c +++ b/bin/varnishstat/varnishstat_curses.c @@ -256,7 +256,7 @@ sample_points(void) VTAILQ_FOREACH(pt, &ptlist, list) { AN(pt->vpt); AN(pt->vpt->ptr); - v = *pt->vpt->ptr; + v = VSC_Value(pt->vpt); if (v == 0 && !pt->seen) continue; if (!pt->seen) { @@ -1104,7 +1104,7 @@ newpt(void *priv, const struct VSC_point *const vpt) rebuild |= REBUILD_NEXT; AN(pt); pt->vpt = vpt; - pt->last = *pt->vpt->ptr; + pt->last = VSC_Value(vpt); pt->ma_10.nmax = 10; pt->ma_100.nmax = 100; pt->ma_1000.nmax = 1000; diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 4698aa4e9..b051b8e08 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -864,7 +864,7 @@ do_stat_dump_cb(void *priv, const struct VSC_point * const pt) if (strcmp(pt->ctype, "uint64_t")) return (0); - u = *pt->ptr; + u = VSC_Value(pt); if (strcmp(dp->arg, "*")) { if (fnmatch(dp->arg, pt->name, 0)) @@ -926,7 +926,7 @@ do_expect_cb(void *priv, const struct VSC_point * const pt) if (!sp->lhs.good && stat_match(sp->lhs.pattern, pt->name) == 0) { AZ(strcmp(pt->ctype, "uint64_t")); AN(pt->ptr); - sp->lhs.val = *pt->ptr; + sp->lhs.val = VSC_Value(pt); sp->lhs.good = 1; } @@ -936,7 +936,7 @@ do_expect_cb(void *priv, const struct VSC_point * const pt) stat_match(sp->rhs.pattern, pt->name) == 0) { AZ(strcmp(pt->ctype, "uint64_t")); AN(pt->ptr); - sp->rhs.val = *pt->ptr; + sp->rhs.val = VSC_Value(pt); sp->rhs.good = 1; } diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h index c09d6b150..5eaa85c41 100644 --- a/include/vapi/vsc.h +++ b/include/vapi/vsc.h @@ -159,4 +159,15 @@ const struct VSC_level_desc *VSC_ChangeLevel(const struct VSC_level_desc*, int); * Change a level up or down. */ +static inline uint64_t +VSC_Value(const struct VSC_point * const pt) +{ + uint64_t val; + + val = *pt->ptr; + if (pt->semantics == 'g' && val > INT64_MAX) + val = 0; + return (val); +} + #endif /* VAPI_VSC_H_INCLUDED */ From dridi.boukelmoune at gmail.com Mon Jan 11 17:06:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 11 Jan 2021 17:06:07 +0000 (UTC) Subject: [master] 78aff3e97 vsc: Control adjusted values with a 'r'aw argument Message-ID: <20210111170607.789DBAF7CD@lists.varnish-cache.org> commit 78aff3e977db1732584a98d7276204d4b97c862b Author: Dridi Boukelmoune Date: Tue Dec 8 18:37:28 2020 +0100 vsc: Control adjusted values with a 'r'aw argument diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h index 5eaa85c41..fc6448911 100644 --- a/include/vapi/vsc.h +++ b/include/vapi/vsc.h @@ -69,6 +69,7 @@ struct VSC_point { const char *sdesc; /* short description */ const char *ldesc; /* long description */ void *priv; /* return val from VSC_new_f */ + unsigned raw; /* adjusted or raw value */ }; /*--------------------------------------------------------------------- @@ -118,6 +119,7 @@ int VSC_Arg(struct vsc *, char arg, const char *opt); * 'X' - field exclusion glob * 'R' - required field glob * 'f' - legacy field filter glob (deprecated) + * 'r' - toggle raw gauges * * Return: * -1 error, VSM_Error() returns diagnostic string @@ -165,7 +167,7 @@ VSC_Value(const struct VSC_point * const pt) uint64_t val; val = *pt->ptr; - if (pt->semantics == 'g' && val > INT64_MAX) + if (!pt->raw && pt->semantics == 'g' && val > INT64_MAX) val = 0; return (val); } diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 05e3c11e2..ae597b0ca 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -98,6 +98,7 @@ struct vsc { unsigned magic; #define VSC_MAGIC 0x3373554a + unsigned raw; struct vsc_sf_head sf_list; VTAILQ_HEAD(,vsc_seg) segs; @@ -185,6 +186,7 @@ VSC_Arg(struct vsc *vsc, char arg, const char *opt) case 'X': return (vsc_sf_arg(vsc, opt, VSC_SF_EXCLUDE)); case 'R': return (vsc_sf_arg(vsc, opt, VSC_SF_REQUIRE)); case 'f': return (vsc_f_arg(vsc, opt)); + case 'r': vsc->raw = !vsc->raw; return (1); default: return (0); } } @@ -299,6 +301,7 @@ vsc_fill_point(const struct vsc *vsc, const struct vsc_seg *seg, AN(vt); point->point.ptr = (volatile void*)(seg->body + atoi(vt->value)); + point->point.raw = vsc->raw; } static void From dridi.boukelmoune at gmail.com Mon Jan 11 17:06:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 11 Jan 2021 17:06:07 +0000 (UTC) Subject: [master] 0f5f457c8 varnishstat: New -r option Message-ID: <20210111170607.90331AF7D4@lists.varnish-cache.org> commit 0f5f457c8a01066b62773ed1687886f6371aa31d Author: Dridi Boukelmoune Date: Tue Dec 8 18:41:27 2020 +0100 varnishstat: New -r option diff --git a/bin/varnishstat/varnishstat.c b/bin/varnishstat/varnishstat.c index 695ca092c..34c073947 100644 --- a/bin/varnishstat/varnishstat.c +++ b/bin/varnishstat/varnishstat.c @@ -315,6 +315,9 @@ main(int argc, char * const *argv) AN(VSC_Arg(vsc, opt, optarg)); has_f = 1; break; + case 'r': + AN(VSC_Arg(vsc, opt, optarg)); + break; case 'V': AN(VUT_Arg(vut, opt, optarg)); break; diff --git a/bin/varnishstat/varnishstat_options.h b/bin/varnishstat/varnishstat_options.h index b98ba11c0..945bde425 100644 --- a/bin/varnishstat/varnishstat_options.h +++ b/bin/varnishstat/varnishstat_options.h @@ -47,6 +47,10 @@ "Lists the available fields to use with the -f option", \ "Lists the available fields to use with the -f option." \ ) +#define STAT_OPT_r \ + VOPT("r", "[-r]", "Toggle raw or adjusted gauges", \ + "Toggle raw or adjusted gauges, adjusted is the default." \ + ) #define STAT_OPT_x \ VOPT("x", "[-x]", "Print statistics to stdout as XML", \ "Print statistics to stdout as XML." \ @@ -59,6 +63,7 @@ VSC_OPT_I STAT_OPT_j STAT_OPT_l VUT_OPT_n +STAT_OPT_r VUT_OPT_t VUT_GLOBAL_OPT_V VSC_OPT_X From dridi.boukelmoune at gmail.com Mon Jan 11 17:06:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 11 Jan 2021 17:06:07 +0000 (UTC) Subject: [master] 9f04dd402 varnishstat: New 'r'aw key binding Message-ID: <20210111170607.AE74FAF7D9@lists.varnish-cache.org> commit 9f04dd402218ec72e43b12c9ce0eb907d8df91ad Author: Dridi Boukelmoune Date: Thu Dec 10 09:33:42 2020 +0100 varnishstat: New 'r'aw key binding diff --git a/bin/varnishstat/varnishstat_bindings.h b/bin/varnishstat/varnishstat_bindings.h index 4051a53eb..565b5cc49 100644 --- a/bin/varnishstat/varnishstat_bindings.h +++ b/bin/varnishstat/varnishstat_bindings.h @@ -72,6 +72,14 @@ BINDING(UNSEEN, "\tof varnishstat. Defaults to hide unseen counters." ) +BINDING_KEY('r', "r",) +BINDING(RAW, + "\tToggle between showing raw and adjusted gauges. When a gauge\n" + "\tis decremented faster than it is incremented, it may appear as\n" + "\ta large integer with its most significant bit set. By default\n" + "\tsuch values are adjusted to zero." +) + BINDING_KEY('e', "e",) BINDING(SCALE, "\tToggle scaling of values.") diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c index 4c3ff9127..0b5fc4c62 100644 --- a/bin/varnishstat/varnishstat_curses.c +++ b/bin/varnishstat/varnishstat_curses.c @@ -935,7 +935,7 @@ handle_common_keypress(enum kb_e kb) } static void -handle_points_keypress(enum kb_e kb) +handle_points_keypress(struct vsc *vsc, enum kb_e kb) { switch (kb) { @@ -973,6 +973,10 @@ handle_points_keypress(enum kb_e kb) hide_unseen = 1 - hide_unseen; rebuild = REBUILD_NEXT; break; + case KB_RAW: + VSC_Arg(vsc, 'r', NULL); + rebuild = REBUILD_NEXT; + break; case KB_SCALE: scale = 1 - scale; rebuild = REBUILD_NEXT; @@ -1046,6 +1050,7 @@ handle_help_keypress(enum kb_e kb) help_line = bindings_help_len; break; case KB_UNSEEN: + case KB_RAW: case KB_SCALE: case KB_ACCEL: case KB_DECEL: @@ -1072,7 +1077,7 @@ handle_help_keypress(enum kb_e kb) } static void -handle_keypress(int ch) +handle_keypress(struct vsc *vsc, int ch) { enum kb_e kb; @@ -1091,7 +1096,7 @@ handle_keypress(int ch) if (show_help) handle_help_keypress(kb); else - handle_points_keypress(kb); + handle_points_keypress(vsc, kb); } static void * v_matchproto_(VSC_new_f) @@ -1201,7 +1206,7 @@ do_curses(struct vsm *vsm, struct vsc *vsc) break; #endif default: - handle_keypress(ch); + handle_keypress(vsc, ch); break; } } From dridi.boukelmoune at gmail.com Mon Jan 11 17:06:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 11 Jan 2021 17:06:07 +0000 (UTC) Subject: [master] 95b5819d1 vsc: New VSC_IsRaw() function in the API Message-ID: <20210111170607.CA53BAF7DE@lists.varnish-cache.org> commit 95b5819d1ee7054d165cc30dbdde38a2d47f17a4 Author: Dridi Boukelmoune Date: Thu Dec 10 09:44:15 2020 +0100 vsc: New VSC_IsRaw() function in the API diff --git a/include/vapi/vsc.h b/include/vapi/vsc.h index fc6448911..a2f6f186e 100644 --- a/include/vapi/vsc.h +++ b/include/vapi/vsc.h @@ -161,6 +161,11 @@ const struct VSC_level_desc *VSC_ChangeLevel(const struct VSC_level_desc*, int); * Change a level up or down. */ +unsigned VSC_IsRaw(const struct vsc *); + /* + * Returns zero if gauges are adjusted by VSC_Value(). + */ + static inline uint64_t VSC_Value(const struct VSC_point * const pt) { diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map index 4436a9205..2d8e29a97 100644 --- a/lib/libvarnishapi/libvarnishapi.map +++ b/lib/libvarnishapi/libvarnishapi.map @@ -195,3 +195,11 @@ LIBVARNISHAPI_2.5 { /* 2020-09-15 release */ local: *; }; + +LIBVARNISHAPI_2.6 { /* 2020-03-15 release */ + global: + # vsc.c + VSC_IsRaw; + local: + *; +}; diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index ae597b0ca..1a84d6aeb 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -191,6 +191,13 @@ VSC_Arg(struct vsc *vsc, char arg, const char *opt) } } +unsigned +VSC_IsRaw(const struct vsc *vsc) +{ + + CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC); + return (vsc->raw); +} /*-------------------------------------------------------------------- */ From dridi.boukelmoune at gmail.com Mon Jan 11 17:06:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 11 Jan 2021 17:06:07 +0000 (UTC) Subject: [master] 244c06a04 varnishstat: Give a visual cue in curses mode Message-ID: <20210111170607.E3559AF7E2@lists.varnish-cache.org> commit 244c06a04d667ec7b09b20e2cf77395ee179b474 Author: Dridi Boukelmoune Date: Thu Dec 10 09:50:06 2020 +0100 varnishstat: Give a visual cue in curses mode diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c index 0b5fc4c62..d031d4978 100644 --- a/bin/varnishstat/varnishstat_curses.c +++ b/bin/varnishstat/varnishstat_curses.c @@ -123,6 +123,7 @@ static int show_help = 0; static int help_line = 0; static int keep_running = 1; static int hide_unseen = 1; +static int raw_vsc = 0; static int page_start = 0; static int current = 0; static int rebuild = 0; @@ -869,8 +870,12 @@ draw_bar_b(void) verbosity->label); X -= strlen(verbosity->label) + 2; } - if (!hide_unseen) + if (!hide_unseen) { mvwprintw(w_bar_b, 0, X - 6, "%s", "UNSEEN"); + X -= 8; + } + if (raw_vsc) + mvwprintw(w_bar_b, 0, X - 3, "%s", "RAW"); wnoutrefresh(w_bar_b); } @@ -975,6 +980,7 @@ handle_points_keypress(struct vsc *vsc, enum kb_e kb) break; case KB_RAW: VSC_Arg(vsc, 'r', NULL); + raw_vsc = VSC_IsRaw(vsc); rebuild = REBUILD_NEXT; break; case KB_SCALE: @@ -1171,6 +1177,7 @@ do_curses(struct vsm *vsm, struct vsc *vsc) VSC_State(vsc, newpt, delpt, NULL); + raw_vsc = VSC_IsRaw(vsc); rebuild |= REBUILD_FIRST; (void)VSC_Iter(vsc, vsm, NULL, NULL); build_pt_array(); From dridi.boukelmoune at gmail.com Mon Jan 11 17:06:08 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 11 Jan 2021 17:06:08 +0000 (UTC) Subject: [master] 849c08441 http1: Optionally honor thread_stats_rate Message-ID: <20210111170608.0F18FAF7EF@lists.varnish-cache.org> commit 849c084417066c1f7fdb9421943143a0ade22c32 Author: Dridi Boukelmoune Date: Tue Oct 20 17:33:07 2020 +0200 http1: Optionally honor thread_stats_rate When a session is continuously used, for example in a multi-tier setup, it can take a very long time before an HTTP/1 session runs out of tasks and unwinds back to the point where the worker might update its stats. This phenomenon is amplified by thread_stats_rate, so it might take even longer to get statistics published. This is famously observed with load testing campaigns where a certain set of VSCs may not increase at all until the very end of a load run. This change makes it possible for HTTP/1 sessions to at least publish their statistics into the pool every thread_stats_rate requests. To avoid a potential performance cost, this behavior (technically complying with the documentation) is guarded by a feature flag. The flag is generic so this behavior may be extended to other busy loops, present (if any) or future. diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 96fb33ff8..70dcdb8ba 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -478,6 +478,7 @@ void VMOD_Panic(struct vsb *); /* cache_wrk.c */ void WRK_Init(void); +void WRK_AddStat(struct worker *); /* cache_ws.c */ void WS_Id(const struct ws *ws, char *id); diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index ee0185146..0fbe71229 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -169,6 +169,25 @@ pool_addstat(struct VSC_main_wrk *dst, struct VSC_main_wrk *src) memset(src, 0, sizeof *src); } +void +WRK_AddStat(struct worker *wrk) +{ + struct pool *pp; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + pp = wrk->pool; + CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); + if (++wrk->stats->summs >= cache_param->wthread_stats_rate) { + Lck_Lock(&pp->mtx); + pool_addstat(pp->a_stat, wrk->stats); + Lck_Unlock(&pp->mtx); + } +} + +/*-------------------------------------------------------------------- + * Pool reserve calculation + */ + static unsigned pool_reserve(void) { diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c index 6a339f5a4..393088520 100644 --- a/bin/varnishd/http1/cache_http1_fsm.c +++ b/bin/varnishd/http1/cache_http1_fsm.c @@ -414,6 +414,8 @@ HTTP1_Session(struct worker *wrk, struct req *req) HTC_RxInit(req->htc, req->ws); if (req->htc->rxbuf_e != req->htc->rxbuf_b) wrk->stats->sess_readahead++; + if (FEATURE(FEATURE_BUSY_STATS_RATE)) + WRK_AddStat(wrk); http1_setstate(sp, H1NEWREQ); } else { WRONG("Wrong H1 session state"); diff --git a/include/tbl/feature_bits.h b/include/tbl/feature_bits.h index 207e18e11..d51b22cef 100644 --- a/include/tbl/feature_bits.h +++ b/include/tbl/feature_bits.h @@ -78,6 +78,10 @@ FEATURE_BIT(VALIDATE_HEADERS, validate_headers, "Validate all header set operations to conform to RFC7230." ) +FEATURE_BIT(BUSY_STATS_RATE, busy_stats_rate, + "Make busy workers comply with thread_stats_rate." +) + #undef FEATURE_BIT /*lint -restore */ From dridi.boukelmoune at gmail.com Mon Jan 11 17:06:08 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 11 Jan 2021 17:06:08 +0000 (UTC) Subject: [master] 771c02cfa wrk: Centralize pool_addstat() conditions Message-ID: <20210111170608.2EA39AF7F4@lists.varnish-cache.org> commit 771c02cfa98cfa1e4960f0db91482c1e8d433769 Author: Dridi Boukelmoune Date: Wed Oct 21 15:21:29 2020 +0200 wrk: Centralize pool_addstat() conditions It's a bit dirty because one code paths comes locked while the other one has the pool unlocked, but this removes the need to duplicate conditions. The pool_addstat() function was unrolled to its single call site. diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 0fbe71229..d32bf16d6 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -160,28 +160,39 @@ WRK_Thread(struct pool *qp, size_t stacksize, unsigned thread_workspace) * Summing of stats into pool counters */ -static void -pool_addstat(struct VSC_main_wrk *dst, struct VSC_main_wrk *src) +static unsigned +wrk_addstat(struct worker *wrk, const struct pool_task *tp, unsigned locked) { + struct pool *pp; - dst->summs++; - VSC_main_Summ_wrk_wrk(dst, src); - memset(src, 0, sizeof *src); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + pp = wrk->pool; + CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); + if (locked) + Lck_AssertHeld(&pp->mtx); + + if ((tp == NULL && wrk->stats->summs > 0) || + (wrk->stats->summs >= cache_param->wthread_stats_rate)) { + if (!locked) + Lck_Lock(&pp->mtx); + + pp->a_stat->summs++; + VSC_main_Summ_wrk_wrk(pp->a_stat, wrk->stats); + memset(wrk->stats, 0, sizeof *wrk->stats); + + if (!locked) + Lck_Unlock(&pp->mtx); + } + + return (tp == NULL ? 0 : 1); } void WRK_AddStat(struct worker *wrk) { - struct pool *pp; - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - pp = wrk->pool; - CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); - if (++wrk->stats->summs >= cache_param->wthread_stats_rate) { - Lck_Lock(&pp->mtx); - pool_addstat(pp->a_stat, wrk->stats); - Lck_Unlock(&pp->mtx); - } + (void)wrk_addstat(wrk, wrk->task, 0); + wrk->stats->summs++; } /*-------------------------------------------------------------------- @@ -385,11 +396,7 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk) } } - if ((tp == NULL && wrk->stats->summs > 0) || - (wrk->stats->summs >= cache_param->wthread_stats_rate)) - pool_addstat(pp->a_stat, wrk->stats); - - if (tp != NULL) { + if (wrk_addstat(wrk, tp, 1)) { wrk->stats->summs++; } else if (pp->b_stat != NULL && pp->a_stat->summs) { /* Nothing to do, push pool stats into global pool */ From nils.goroll at uplex.de Mon Jan 11 18:36:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 18:36:07 +0000 (UTC) Subject: [master] b417f5549 constify 06cf60a020a81e49d001ee42e4d85f16580f3fe0 Message-ID: <20210111183607.A962EB2A90@lists.varnish-cache.org> commit b417f5549ed2190bccad0f8c8caf367e038a5faf Author: Nils Goroll Date: Mon Jan 11 17:53:10 2021 +0100 constify 06cf60a020a81e49d001ee42e4d85f16580f3fe0 diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index b006f52ae..8e7ef17e1 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -116,7 +116,7 @@ vrt_priv_dyncmp(const struct vrt_priv *vp1, const struct vrt_priv *vp2) VRBT_GENERATE_STATIC(vrt_privs, vrt_priv, entry, vrt_priv_dyncmp) static struct vmod_priv * -vrt_priv_dynamic_get(struct vrt_privs *privs, uintptr_t vmod_id) +vrt_priv_dynamic_get(const struct vrt_privs *privs, uintptr_t vmod_id) { struct vrt_priv *vp; From nils.goroll at uplex.de Mon Jan 11 18:36:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 18:36:07 +0000 (UTC) Subject: [master] 9f9837fef Avoid creating a priv_task in std.ban() unless we need it Message-ID: <20210111183607.BC978B2A93@lists.varnish-cache.org> commit 9f9837fef5a6908df2104cc85de1577f6ecdfcd9 Author: Nils Goroll Date: Mon Jan 11 17:53:33 2021 +0100 Avoid creating a priv_task in std.ban() unless we need it We only need a new priv_task if we need to log an error. If one existed before, we need to clear or overwrite the error. diff --git a/vmod/vmod_std.c b/vmod/vmod_std.c index 203a621b1..1cf60162c 100644 --- a/vmod/vmod_std.c +++ b/vmod/vmod_std.c @@ -322,14 +322,20 @@ vmod_ban(VRT_CTX, VCL_STRING s) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - priv_task = VRT_priv_task(ctx, priv_task_id_ban); + r = VRT_ban_string(ctx, s); + priv_task = VRT_priv_task_get(ctx, priv_task_id_ban); + + if (r == NULL && priv_task == NULL) + return (1); + + if (priv_task == NULL) + priv_task = VRT_priv_task(ctx, priv_task_id_ban); + if (priv_task == NULL) { VRT_fail(ctx, "std.ban(): no priv_task (out of workspace?)"); return (0); } - r = VRT_ban_string(ctx, s); - /* * TRUST_ME: the ban error is const. We save it in the un-const priv * pointer, but promise to only ever return it as a (const) VCL_STRING @@ -347,12 +353,9 @@ vmod_ban_error(VRT_CTX) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - priv_task = VRT_priv_task(ctx, priv_task_id_ban); - if (priv_task == NULL) { - VRT_fail(ctx, "std.ban_error():" - " no priv_task (out of workspace?)"); - return ("no priv_task"); - } + priv_task = VRT_priv_task_get(ctx, priv_task_id_ban); + if (priv_task == NULL) + return (""); r = priv_task->priv; if (r == NULL) From nils.goroll at uplex.de Mon Jan 11 18:36:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 18:36:07 +0000 (UTC) Subject: [master] 3375af5e9 doc VRT_priv_task_get? Message-ID: <20210111183607.D42C0B2A96@lists.varnish-cache.org> commit 3375af5e9084ea81ca2968fb36e49fffe66a9a9e Author: Nils Goroll Date: Mon Jan 11 18:06:46 2021 +0100 doc VRT_priv_task_get? diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst index afc97d820..19f2ac79e 100644 --- a/doc/sphinx/reference/vmod.rst +++ b/doc/sphinx/reference/vmod.rst @@ -568,7 +568,7 @@ convenient yet unofficial interface which was not originally intended for this purpose and will likely be replaced with a more suitable interface. - +.. XXX add VRT_priv_task_get() and make it official? .. _ref-vmod-event-functions: From nils.goroll at uplex.de Mon Jan 11 18:36:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 18:36:07 +0000 (UTC) Subject: [master] 885a7bcb6 rename shard_param_task -> shard_param_task_l Message-ID: <20210111183607.EDE72B2A9B@lists.varnish-cache.org> commit 885a7bcb6f7a275b79067f375b668b6eb9fd22a0 Author: Nils Goroll Date: Mon Jan 11 18:07:35 2021 +0100 rename shard_param_task -> shard_param_task_l diff --git a/vmod/vmod_directors_shard.c b/vmod/vmod_directors_shard.c index d12df4476..25dc054ef 100644 --- a/vmod/vmod_directors_shard.c +++ b/vmod/vmod_directors_shard.c @@ -159,7 +159,7 @@ shard_param_stack(struct vmod_directors_shard_param *p, const struct vmod_directors_shard_param *pa, const char *who); static struct vmod_directors_shard_param * -shard_param_task(VRT_CTX, const void *id, +shard_param_task_l(VRT_CTX, const void *id, const struct vmod_directors_shard_param *pa); static const struct vmod_directors_shard_param * @@ -609,7 +609,7 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, resolve = VENUM(NOW); if (ctx->method & SHARD_VCL_TASK_BEREQ) { - pp = shard_param_task(ctx, vshard->shardd, + pp = shard_param_task_l(ctx, vshard->shardd, vshard->shardd->param); if (pp == NULL) return (NULL); @@ -870,7 +870,7 @@ shard_param_stack(struct vmod_directors_shard_param *p, * if id != pa and pa has VCL scope, also get a task scoped param struct for pa */ static struct vmod_directors_shard_param * -shard_param_task(VRT_CTX, const void *id, +shard_param_task_l(VRT_CTX, const void *id, const struct vmod_directors_shard_param *pa) { struct vmod_directors_shard_param *p; @@ -908,7 +908,7 @@ shard_param_task(VRT_CTX, const void *id, if (id == pa || pa->scope != SCOPE_VCL) p->defaults = pa; else - p->defaults = shard_param_task(ctx, pa, pa); + p->defaults = shard_param_task_l(ctx, pa, pa); return (p); } @@ -925,7 +925,7 @@ shard_param_prep(VRT_CTX, struct vmod_directors_shard_param *p, "in vcl_init and in backend/pipe context", who); return (NULL); } else if (ctx->method & SHARD_VCL_TASK_BEREQ) - p = shard_param_task(ctx, p, p); + p = shard_param_task_l(ctx, p, p); else assert(ctx->method & VCL_MET_TASK_H); @@ -970,7 +970,7 @@ vmod_shard_param_read(VRT_CTX, const void *id, (void) who; // XXX if (ctx->method == 0 || (ctx->method & SHARD_VCL_TASK_BEREQ)) - p = shard_param_task(ctx, id, p); + p = shard_param_task_l(ctx, id, p); if (p == NULL) return (NULL); From nils.goroll at uplex.de Mon Jan 11 18:36:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 18:36:08 +0000 (UTC) Subject: [master] 08b642d70 shard: make use of VRT_priv_task_get() Message-ID: <20210111183608.12748B2AA4@lists.varnish-cache.org> commit 08b642d70ff15346eb1818171862f70a3a9266af Author: Nils Goroll Date: Mon Jan 11 19:00:53 2021 +0100 shard: make use of VRT_priv_task_get() diff --git a/vmod/vmod_directors_shard.c b/vmod/vmod_directors_shard.c index 25dc054ef..b018ced81 100644 --- a/vmod/vmod_directors_shard.c +++ b/vmod/vmod_directors_shard.c @@ -59,7 +59,7 @@ * not defined. * * Actual resolution of the various parameter objects does not happen before - * they are used, which enabled changing them independently (ie, shard + * they are used, which enables changing them independently (ie, shard * .backend() parameters have precedence over an associated parameter object, * which by itself can be overridden). * @@ -158,6 +158,10 @@ static struct vmod_directors_shard_param * shard_param_stack(struct vmod_directors_shard_param *p, const struct vmod_directors_shard_param *pa, const char *who); +static const struct vmod_directors_shard_param * +shard_param_task_r(VRT_CTX, const void *id, + const struct vmod_directors_shard_param *pa); + static struct vmod_directors_shard_param * shard_param_task_l(VRT_CTX, const void *id, const struct vmod_directors_shard_param *pa); @@ -608,14 +612,6 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, else resolve = VENUM(NOW); - if (ctx->method & SHARD_VCL_TASK_BEREQ) { - pp = shard_param_task_l(ctx, vshard->shardd, - vshard->shardd->param); - if (pp == NULL) - return (NULL); - pp->vcl_name = vshard->shardd->name; - } - if (resolve == VENUM(LAZY)) { if ((args & ~arg_resolve) == 0) { AN(vshard->dir); @@ -643,7 +639,16 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, WRONG("resolve enum"); } + if (ctx->method & SHARD_VCL_TASK_BEREQ) { + pp = shard_param_task_l(ctx, vshard->shardd, + vshard->shardd->param); + if (pp == NULL) + return (NULL); + pp->vcl_name = vshard->shardd->name; + } + AN(pp); + if (args & arg_param) { ppt = shard_param_blob(a->param); if (ppt == NULL) { @@ -865,6 +870,34 @@ shard_param_stack(struct vmod_directors_shard_param *p, return (p); } + +const static struct vmod_directors_shard_param * +shard_param_task_r(VRT_CTX, const void *id, + const struct vmod_directors_shard_param *pa) +{ + const struct vmod_directors_shard_param *p; + const struct vmod_priv *task; + const void *task_id; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(pa, VMOD_SHARD_SHARD_PARAM_MAGIC); + assert(pa->scope > _SCOPE_INVALID); + + task_id = (const char *)id + task_off_param; + task = VRT_priv_task_get(ctx, task_id); + + if (task) { + CAST_OBJ_NOTNULL(p, task->priv, VMOD_SHARD_SHARD_PARAM_MAGIC); + assert(p->scope == SCOPE_TASK); + return (p); + } + + if (id == pa || pa->scope != SCOPE_VCL) + return (pa); + + return (shard_param_task_r(ctx, pa, pa)); +} + /* * get a task scoped param struct for id defaulting to pa * if id != pa and pa has VCL scope, also get a task scoped param struct for pa @@ -970,10 +1003,7 @@ vmod_shard_param_read(VRT_CTX, const void *id, (void) who; // XXX if (ctx->method == 0 || (ctx->method & SHARD_VCL_TASK_BEREQ)) - p = shard_param_task_l(ctx, id, p); - - if (p == NULL) - return (NULL); + p = shard_param_task_r(ctx, id, p); pp = shard_param_stack(pstk, p, p->vcl_name); AN(pp); From nils.goroll at uplex.de Mon Jan 11 18:36:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 18:36:08 +0000 (UTC) Subject: [master] 10d3b211a shard: tighten assertions Message-ID: <20210111183608.2EB70B2AB1@lists.varnish-cache.org> commit 10d3b211ae1dbb1bdcefdfca8f8171f70dc402f2 Author: Nils Goroll Date: Mon Jan 11 19:09:02 2021 +0100 shard: tighten assertions Now that we do not make allocations for reading parameters, we can assert that we always have them. diff --git a/vmod/vmod_directors_shard.c b/vmod/vmod_directors_shard.c index b018ced81..27b722eab 100644 --- a/vmod/vmod_directors_shard.c +++ b/vmod/vmod_directors_shard.c @@ -699,9 +699,8 @@ vmod_shard_resolve(VRT_CTX, VCL_BACKEND dir) CAST_OBJ_NOTNULL(shardd, dir->priv, SHARDDIR_MAGIC); pp = vmod_shard_param_read(ctx, shardd, shardd->param, - pstk, "shard_resolve"); - if (pp == NULL) - return (NULL); + pstk, "shard_resolve"); + CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return (sharddir_pick_be(ctx, shardd, shard_get_key(ctx, pp), pp->alt, pp->warmup, @@ -1005,8 +1004,8 @@ vmod_shard_param_read(VRT_CTX, const void *id, if (ctx->method == 0 || (ctx->method & SHARD_VCL_TASK_BEREQ)) p = shard_param_task_r(ctx, id, p); + CHECK_OBJ_NOTNULL(p, VMOD_SHARD_SHARD_PARAM_MAGIC); pp = shard_param_stack(pstk, p, p->vcl_name); - AN(pp); shard_param_merge(pp, p); return (pp); } @@ -1019,8 +1018,7 @@ vmod_shard_param_get_by(VRT_CTX, const struct vmod_directors_shard_param *pp; pp = vmod_shard_param_read(ctx, p, p, &pstk, "shard_param.get_by()"); - if (pp == NULL) - return (NULL); + CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return (default_by(pp->by)); } @@ -1032,8 +1030,7 @@ vmod_shard_param_get_key(VRT_CTX, const struct vmod_directors_shard_param *pp; pp = vmod_shard_param_read(ctx, p, p, &pstk, "shard_param.get_key()"); - if (pp == NULL) - return (-1); + CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return ((VCL_INT)shard_get_key(ctx, pp)); } VCL_INT v_matchproto_(td_directors_shard_param_get_alt) @@ -1043,10 +1040,8 @@ vmod_shard_param_get_alt(VRT_CTX, struct vmod_directors_shard_param pstk; const struct vmod_directors_shard_param *pp; - pp = vmod_shard_param_read(ctx, p, p, &pstk, - "shard_param.get_alt()"); - if (pp == NULL) - return (-1); + pp = vmod_shard_param_read(ctx, p, p, &pstk, "shard_param.get_alt()"); + CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return (pp->alt); } @@ -1058,9 +1053,8 @@ vmod_shard_param_get_warmup(VRT_CTX, const struct vmod_directors_shard_param *pp; pp = vmod_shard_param_read(ctx, p, p, &pstk, - "shard_param.get_warmup()"); - if (pp == NULL) - return (-2); + "shard_param.get_warmup()"); + CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return (pp->warmup); } @@ -1072,9 +1066,8 @@ vmod_shard_param_get_rampup(VRT_CTX, const struct vmod_directors_shard_param *pp; pp = vmod_shard_param_read(ctx, p, p, &pstk, - "shard_param.get_rampup()"); - if (pp == NULL) - return (0); + "shard_param.get_rampup()"); + CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return (pp->rampup); } @@ -1086,9 +1079,8 @@ vmod_shard_param_get_healthy(VRT_CTX, const struct vmod_directors_shard_param *pp; pp = vmod_shard_param_read(ctx, p, p, &pstk, - "shard_param.get_healthy()"); - if (pp == NULL) - return (NULL); + "shard_param.get_healthy()"); + CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return (default_healthy(pp->healthy)); } From nils.goroll at uplex.de Mon Jan 11 18:36:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 18:36:08 +0000 (UTC) Subject: [master] 9ea2f29f9 shard: gc unused parameter Message-ID: <20210111183608.48017B2AB7@lists.varnish-cache.org> commit 9ea2f29f9be641e0569214326f2974baa8960077 Author: Nils Goroll Date: Mon Jan 11 19:13:41 2021 +0100 shard: gc unused parameter the "who" parameter was intended for error reporting, but we never used it. And now that vmod_shard_param_read() cannot fail, we are sure we never will. diff --git a/vmod/vmod_directors_shard.c b/vmod/vmod_directors_shard.c index 27b722eab..b5f945a62 100644 --- a/vmod/vmod_directors_shard.c +++ b/vmod/vmod_directors_shard.c @@ -172,7 +172,7 @@ shard_param_blob(VCL_BLOB blob); static const struct vmod_directors_shard_param * vmod_shard_param_read(VRT_CTX, const void *id, const struct vmod_directors_shard_param *p, - struct vmod_directors_shard_param *pstk, const char *who); + struct vmod_directors_shard_param *pstk); // XXX #3329 #3330 revisit - for now, treat pipe like backend #define SHARD_VCL_TASK_REQ (VCL_MET_TASK_C & ~VCL_MET_PIPE) @@ -698,8 +698,7 @@ vmod_shard_resolve(VRT_CTX, VCL_BACKEND dir) CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(shardd, dir->priv, SHARDDIR_MAGIC); - pp = vmod_shard_param_read(ctx, shardd, shardd->param, - pstk, "shard_resolve"); + pp = vmod_shard_param_read(ctx, shardd, shardd->param, pstk); CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return (sharddir_pick_be(ctx, shardd, @@ -993,13 +992,12 @@ vmod_shard_param_clear(VRT_CTX, static const struct vmod_directors_shard_param * vmod_shard_param_read(VRT_CTX, const void *id, const struct vmod_directors_shard_param *p, - struct vmod_directors_shard_param *pstk, const char *who) + struct vmod_directors_shard_param *pstk) { struct vmod_directors_shard_param *pp; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(p, VMOD_SHARD_SHARD_PARAM_MAGIC); - (void) who; // XXX if (ctx->method == 0 || (ctx->method & SHARD_VCL_TASK_BEREQ)) p = shard_param_task_r(ctx, id, p); @@ -1017,7 +1015,7 @@ vmod_shard_param_get_by(VRT_CTX, struct vmod_directors_shard_param pstk; const struct vmod_directors_shard_param *pp; - pp = vmod_shard_param_read(ctx, p, p, &pstk, "shard_param.get_by()"); + pp = vmod_shard_param_read(ctx, p, p, &pstk); CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return (default_by(pp->by)); } @@ -1029,7 +1027,7 @@ vmod_shard_param_get_key(VRT_CTX, struct vmod_directors_shard_param pstk; const struct vmod_directors_shard_param *pp; - pp = vmod_shard_param_read(ctx, p, p, &pstk, "shard_param.get_key()"); + pp = vmod_shard_param_read(ctx, p, p, &pstk); CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return ((VCL_INT)shard_get_key(ctx, pp)); } @@ -1040,7 +1038,7 @@ vmod_shard_param_get_alt(VRT_CTX, struct vmod_directors_shard_param pstk; const struct vmod_directors_shard_param *pp; - pp = vmod_shard_param_read(ctx, p, p, &pstk, "shard_param.get_alt()"); + pp = vmod_shard_param_read(ctx, p, p, &pstk); CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return (pp->alt); } @@ -1052,8 +1050,7 @@ vmod_shard_param_get_warmup(VRT_CTX, struct vmod_directors_shard_param pstk; const struct vmod_directors_shard_param *pp; - pp = vmod_shard_param_read(ctx, p, p, &pstk, - "shard_param.get_warmup()"); + pp = vmod_shard_param_read(ctx, p, p, &pstk); CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return (pp->warmup); } @@ -1065,8 +1062,7 @@ vmod_shard_param_get_rampup(VRT_CTX, struct vmod_directors_shard_param pstk; const struct vmod_directors_shard_param *pp; - pp = vmod_shard_param_read(ctx, p, p, &pstk, - "shard_param.get_rampup()"); + pp = vmod_shard_param_read(ctx, p, p, &pstk); CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return (pp->rampup); } @@ -1078,8 +1074,7 @@ vmod_shard_param_get_healthy(VRT_CTX, struct vmod_directors_shard_param pstk; const struct vmod_directors_shard_param *pp; - pp = vmod_shard_param_read(ctx, p, p, &pstk, - "shard_param.get_healthy()"); + pp = vmod_shard_param_read(ctx, p, p, &pstk); CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return (default_healthy(pp->healthy)); } From nils.goroll at uplex.de Mon Jan 11 18:36:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 18:36:08 +0000 (UTC) Subject: [master] 95dd39a05 constify 771c02cfa98cfa1e4960f0db91482c1e8d433769 Message-ID: <20210111183608.5F63BB2ABD@lists.varnish-cache.org> commit 95dd39a05d9e1ca0ab66ce4e4696c658dc20ba05 Author: Nils Goroll Date: Mon Jan 11 19:18:25 2021 +0100 constify 771c02cfa98cfa1e4960f0db91482c1e8d433769 diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 70dcdb8ba..1ecab14d3 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -478,7 +478,7 @@ void VMOD_Panic(struct vsb *); /* cache_wrk.c */ void WRK_Init(void); -void WRK_AddStat(struct worker *); +void WRK_AddStat(const struct worker *); /* cache_ws.c */ void WS_Id(const struct ws *ws, char *id); diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index d32bf16d6..3a83f1642 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -161,7 +161,7 @@ WRK_Thread(struct pool *qp, size_t stacksize, unsigned thread_workspace) */ static unsigned -wrk_addstat(struct worker *wrk, const struct pool_task *tp, unsigned locked) +wrk_addstat(const struct worker *wrk, const struct pool_task *tp, unsigned locked) { struct pool *pp; @@ -188,7 +188,7 @@ wrk_addstat(struct worker *wrk, const struct pool_task *tp, unsigned locked) } void -WRK_AddStat(struct worker *wrk) +WRK_AddStat(const struct worker *wrk) { (void)wrk_addstat(wrk, wrk->task, 0); From nils.goroll at uplex.de Mon Jan 11 18:36:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 18:36:08 +0000 (UTC) Subject: [master] afcdbf646 flexelint 771c02cfa98cfa1e4960f0db91482c1e8d433769 Message-ID: <20210111183608.77F37B2AC8@lists.varnish-cache.org> commit afcdbf6467e55aaa5f5b9a617a1804a938a8d8ab Author: Nils Goroll Date: Mon Jan 11 19:24:10 2021 +0100 flexelint 771c02cfa98cfa1e4960f0db91482c1e8d433769 it does not understand that wrk_addstat() only returns true for tp != NULL. diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 3a83f1642..9b6933cae 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -398,6 +398,7 @@ Pool_Work_Thread(struct pool *pp, struct worker *wrk) if (wrk_addstat(wrk, tp, 1)) { wrk->stats->summs++; + AN(tp); } else if (pp->b_stat != NULL && pp->a_stat->summs) { /* Nothing to do, push pool stats into global pool */ tps.func = pool_stat_summ; From nils.goroll at uplex.de Mon Jan 11 18:44:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 18:44:06 +0000 (UTC) Subject: [master] bdab64f8f flexelint 95b5819d1ee7054d165cc30dbdde38a2d47f17a4 Message-ID: <20210111184406.AD836BE9A4@lists.varnish-cache.org> commit bdab64f8fc9a350168bdec89456e38188d5bc62a Author: Nils Goroll Date: Mon Jan 11 19:41:53 2021 +0100 flexelint 95b5819d1ee7054d165cc30dbdde38a2d47f17a4 diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c index d031d4978..6863d6278 100644 --- a/bin/varnishstat/varnishstat_curses.c +++ b/bin/varnishstat/varnishstat_curses.c @@ -979,7 +979,7 @@ handle_points_keypress(struct vsc *vsc, enum kb_e kb) rebuild = REBUILD_NEXT; break; case KB_RAW: - VSC_Arg(vsc, 'r', NULL); + AN(VSC_Arg(vsc, 'r', NULL)); raw_vsc = VSC_IsRaw(vsc); rebuild = REBUILD_NEXT; break; From nils.goroll at uplex.de Mon Jan 11 18:44:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 18:44:06 +0000 (UTC) Subject: [master] e02f9cebf flexelint 08b642d70ff15346eb1818171862f70a3a9266af Message-ID: <20210111184406.C0B5ABE9A7@lists.varnish-cache.org> commit e02f9cebfcbd581150d58a47909cbda739a26dbc Author: Nils Goroll Date: Mon Jan 11 19:43:12 2021 +0100 flexelint 08b642d70ff15346eb1818171862f70a3a9266af diff --git a/vmod/vmod_directors_shard.c b/vmod/vmod_directors_shard.c index b5f945a62..afe753854 100644 --- a/vmod/vmod_directors_shard.c +++ b/vmod/vmod_directors_shard.c @@ -869,7 +869,7 @@ shard_param_stack(struct vmod_directors_shard_param *p, return (p); } -const static struct vmod_directors_shard_param * +static const struct vmod_directors_shard_param * shard_param_task_r(VRT_CTX, const void *id, const struct vmod_directors_shard_param *pa) { From nils.goroll at uplex.de Mon Jan 11 22:31:11 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 11 Jan 2021 22:31:11 +0000 (UTC) Subject: [master] 4b895d763 use "feature ipv4" Message-ID: <20210111223111.D9EF59F81@lists.varnish-cache.org> commit 4b895d7632152e1526f5e7e95aa1bc6593a2820c Author: Nils Goroll Date: Mon Jan 11 23:30:02 2021 +0100 use "feature ipv4" from c862d49eedbc609b8bb4cb7b2fe15706ec92ffaf diff --git a/bin/varnishtest/tests/o00005.vtc b/bin/varnishtest/tests/o00005.vtc index d46792462..f20595589 100644 --- a/bin/varnishtest/tests/o00005.vtc +++ b/bin/varnishtest/tests/o00005.vtc @@ -1,8 +1,7 @@ varnishtest "PROXY v2 TLV test" -# XXX this does does not work with IPv6, the workspace overflow test -# is too brittle -feature cmd {test "${localhost}" = "127.0.0.1"} +# this does does not work with IPv6, the workspace overflow test is too brittle +feature ipv4 server s1 { rxreq diff --git a/bin/varnishtest/tests/r01312.vtc b/bin/varnishtest/tests/r01312.vtc index cec7ad2c6..1309dbbfb 100644 --- a/bin/varnishtest/tests/r01312.vtc +++ b/bin/varnishtest/tests/r01312.vtc @@ -1,7 +1,7 @@ varnishtest "acl miscompile" -# XXX this test only works if we know a network prefix of localhost -feature cmd {test "${localhost}" = "127.0.0.1"} +# this test only works if we know a network prefix of localhost +feature ipv4 server s1 { rxreq From phk at FreeBSD.org Tue Jan 12 08:44:07 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 12 Jan 2021 08:44:07 +0000 (UTC) Subject: [master] cf7304c64 Merge HAproxy support from vtest Message-ID: <20210112084407.DA4C8107EB4@lists.varnish-cache.org> commit cf7304c646d75b74fd84ccd040d2fcefe9c2ef5b Author: Poul-Henning Kamp Date: Mon Jan 11 10:01:15 2021 +0000 Merge HAproxy support from vtest diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index ca0cb9259..012a89cba 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -45,10 +45,10 @@ #include "vpf.h" #include "vre.h" #include "vtcp.h" -#include "vsa.h" #include "vtim.h" #define HAPROXY_PROGRAM_ENV_VAR "HAPROXY_PROGRAM" +#define HAPROXY_ARGS_ENV_VAR "HAPROXY_ARGS" #define HAPROXY_OPT_WORKER "-W" #define HAPROXY_OPT_MCLI "-S" #define HAPROXY_OPT_DAEMON "-D" @@ -174,7 +174,7 @@ haproxy_cli_tcp_connect(struct vtclog *vl, const char *addr, double tmo, const char **errp) { int fd; - char mabuf[VTCP_ADDRBUFSIZE], mpbuf[VTCP_PORTBUFSIZE]; + char mabuf[32], mpbuf[32]; AN(addr); AN(errp); @@ -520,23 +520,16 @@ haproxy_create_mcli(struct haproxy *h) int sock; const char *err; char buf[128], addr[128], port[128]; - char vsabuf[vsa_suckaddr_len]; - struct suckaddr *sua; - sock = VTCP_listen_on(default_listen_addr, NULL, 100, &err); + sock = VTCP_listen_on("127.0.0.1:0", NULL, 100, &err); if (err != NULL) vtc_fatal(h->vl, "Create listen socket failed: %s", err); assert(sock > 0); - sua = VSA_getsockname(sock, vsabuf, sizeof vsabuf); - AN(sua); - VTCP_name(sua, addr, sizeof addr, port, sizeof port); + VTCP_myname(sock, addr, sizeof addr, port, sizeof port); bprintf(buf, "%s_mcli", h->name); - if (VSA_Get_Proto(sua) == AF_INET) - macro_def(h->vl, buf, "sock", "%s:%s", addr, port); - else - macro_def(h->vl, buf, "sock", "[%s]:%s", addr, port); + macro_def(h->vl, buf, "sock", "%s %s", addr, port); macro_def(h->vl, buf, "addr", "%s", addr); macro_def(h->vl, buf, "port", "%s", port); @@ -566,14 +559,18 @@ haproxy_new(const char *name) int closed_sock; char addr[128], port[128]; const char *err; - char vsabuf[vsa_suckaddr_len]; - struct suckaddr *sua; + const char *env_args; ALLOC_OBJ(h, HAPROXY_MAGIC); AN(h); REPLACE(h->name, name); h->args = VSB_new_auto(); + env_args = getenv(HAPROXY_ARGS_ENV_VAR); + if (env_args) { + VSB_cat(h->args, env_args); + VSB_cat(h->args, " "); + } h->vl = vtc_logopen("%s", name); vtc_log_set_cmd(h->vl, haproxy_cli_cmds); @@ -607,13 +604,8 @@ haproxy_new(const char *name) vtc_fatal(h->vl, "Create listen socket failed: %s", err); assert(closed_sock > 0); - sua = VSA_getsockname(closed_sock, vsabuf, sizeof vsabuf); - AN(sua); - VTCP_name(sua, addr, sizeof addr, port, sizeof port); - if (VSA_Get_Proto(sua) == AF_INET) - macro_def(h->vl, h->closed_sock, "sock", "%s:%s", addr, port); - else - macro_def(h->vl, h->closed_sock, "sock", "[%s]:%s", addr, port); + VTCP_myname(closed_sock, addr, sizeof addr, port, sizeof port); + macro_def(h->vl, h->closed_sock, "sock", "%s %s", addr, port); macro_def(h->vl, h->closed_sock, "addr", "%s", addr); macro_def(h->vl, h->closed_sock, "port", "%s", port); VTCP_close(&closed_sock); @@ -851,8 +843,6 @@ haproxy_build_backends(struct haproxy *h, const char *vsb_data) int sock; char buf[128], addr[128], port[128]; const char *err; - char vsabuf[vsa_suckaddr_len]; - struct suckaddr *sua; p = strstr(p, HAPROXY_BE_FD_STR); if (!p) @@ -870,15 +860,10 @@ haproxy_build_backends(struct haproxy *h, const char *vsb_data) vtc_fatal(h->vl, "Create listen socket failed: %s", err); assert(sock > 0); - sua = VSA_getsockname(sock, vsabuf, sizeof vsabuf); - AN(sua); - VTCP_name(sua, addr, sizeof addr, port, sizeof port); + VTCP_myname(sock, addr, sizeof addr, port, sizeof port); bprintf(buf, "%s_%s", h->name, p); - if (VSA_Get_Proto(sua) == AF_INET) - macro_def(h->vl, buf, "sock", "%s:%s", addr, port); - else - macro_def(h->vl, buf, "sock", "[%s]:%s", addr, port); + macro_def(h->vl, buf, "sock", "%s %s", addr, port); macro_def(h->vl, buf, "addr", "%s", addr); macro_def(h->vl, buf, "port", "%s", port); diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index 88437b757..75a4b7c38 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -384,7 +384,7 @@ ipvx_works(const char *target) VTCP_close(&fd); return (1); } - return(0); + return(0); } /* SECTION: feature feature From phk at FreeBSD.org Tue Jan 12 08:44:07 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 12 Jan 2021 08:44:07 +0000 (UTC) Subject: [master] 53bc920fa Unbreak this test, which tests a fine point of ACL compilation which does not depend on ipv4/ipv6 Message-ID: <20210112084407.F38E0107EB7@lists.varnish-cache.org> commit 53bc920fa6f77f191fdb26603bce283a5d15ae2d Author: Poul-Henning Kamp Date: Tue Jan 12 08:43:28 2021 +0000 Unbreak this test, which tests a fine point of ACL compilation which does not depend on ipv4/ipv6 diff --git a/bin/varnishtest/tests/r01312.vtc b/bin/varnishtest/tests/r01312.vtc index 1309dbbfb..762da43bf 100644 --- a/bin/varnishtest/tests/r01312.vtc +++ b/bin/varnishtest/tests/r01312.vtc @@ -1,8 +1,5 @@ varnishtest "acl miscompile" -# this test only works if we know a network prefix of localhost -feature ipv4 - server s1 { rxreq txresp @@ -20,8 +17,8 @@ varnish v1 -vcl+backend { "127.0.1"/19; } sub vcl_deliver { - set resp.http.ACLfoo = std.ip("${localhost}", client.ip) ~ foo; - set resp.http.ACLbar = std.ip("${localhost}", client.ip) ~ bar; + set resp.http.ACLfoo = std.ip("127.0.0.1", client.ip) ~ foo; + set resp.http.ACLbar = std.ip("127.0.0.1", client.ip) ~ bar; } } -start From dridi at varni.sh Tue Jan 12 09:07:03 2021 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 12 Jan 2021 09:07:03 +0000 Subject: [master] cf7304c64 Merge HAproxy support from vtest In-Reply-To: <20210112084407.DA4C8107EB4@lists.varnish-cache.org> References: <20210112084407.DA4C8107EB4@lists.varnish-cache.org> Message-ID: On Tue, Jan 12, 2021 at 8:44 AM Poul-Henning Kamp wrote: > > > commit cf7304c646d75b74fd84ccd040d2fcefe9c2ef5b > Author: Poul-Henning Kamp > Date: Mon Jan 11 10:01:15 2021 +0000 > > Merge HAproxy support from vtest > > diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c > index ca0cb9259..012a89cba 100644 > --- a/bin/varnishtest/vtc_haproxy.c > +++ b/bin/varnishtest/vtc_haproxy.c > @@ -45,10 +45,10 @@ > #include "vpf.h" > #include "vre.h" > #include "vtcp.h" > -#include "vsa.h" > #include "vtim.h" > > #define HAPROXY_PROGRAM_ENV_VAR "HAPROXY_PROGRAM" > +#define HAPROXY_ARGS_ENV_VAR "HAPROXY_ARGS" > #define HAPROXY_OPT_WORKER "-W" > #define HAPROXY_OPT_MCLI "-S" > #define HAPROXY_OPT_DAEMON "-D" > @@ -174,7 +174,7 @@ haproxy_cli_tcp_connect(struct vtclog *vl, const char *addr, double tmo, > const char **errp) > { > int fd; > - char mabuf[VTCP_ADDRBUFSIZE], mpbuf[VTCP_PORTBUFSIZE]; > + char mabuf[32], mpbuf[32]; This is undoing some of the fixes from Nils that should have been merged the other way around too. > AN(addr); > AN(errp); > @@ -520,23 +520,16 @@ haproxy_create_mcli(struct haproxy *h) > int sock; > const char *err; > char buf[128], addr[128], port[128]; > - char vsabuf[vsa_suckaddr_len]; > - struct suckaddr *sua; > > - sock = VTCP_listen_on(default_listen_addr, NULL, 100, &err); > + sock = VTCP_listen_on("127.0.0.1:0", NULL, 100, &err); Another occurrence, there are others below. > if (err != NULL) > vtc_fatal(h->vl, > "Create listen socket failed: %s", err); > assert(sock > 0); > - sua = VSA_getsockname(sock, vsabuf, sizeof vsabuf); > - AN(sua); > > - VTCP_name(sua, addr, sizeof addr, port, sizeof port); > + VTCP_myname(sock, addr, sizeof addr, port, sizeof port); > bprintf(buf, "%s_mcli", h->name); > - if (VSA_Get_Proto(sua) == AF_INET) > - macro_def(h->vl, buf, "sock", "%s:%s", addr, port); > - else > - macro_def(h->vl, buf, "sock", "[%s]:%s", addr, port); > + macro_def(h->vl, buf, "sock", "%s %s", addr, port); > macro_def(h->vl, buf, "addr", "%s", addr); > macro_def(h->vl, buf, "port", "%s", port); > > @@ -566,14 +559,18 @@ haproxy_new(const char *name) > int closed_sock; > char addr[128], port[128]; > const char *err; > - char vsabuf[vsa_suckaddr_len]; > - struct suckaddr *sua; > + const char *env_args; > > ALLOC_OBJ(h, HAPROXY_MAGIC); > AN(h); > REPLACE(h->name, name); > > h->args = VSB_new_auto(); > + env_args = getenv(HAPROXY_ARGS_ENV_VAR); > + if (env_args) { > + VSB_cat(h->args, env_args); > + VSB_cat(h->args, " "); > + } > > h->vl = vtc_logopen("%s", name); > vtc_log_set_cmd(h->vl, haproxy_cli_cmds); > @@ -607,13 +604,8 @@ haproxy_new(const char *name) > vtc_fatal(h->vl, > "Create listen socket failed: %s", err); > assert(closed_sock > 0); > - sua = VSA_getsockname(closed_sock, vsabuf, sizeof vsabuf); > - AN(sua); > - VTCP_name(sua, addr, sizeof addr, port, sizeof port); > - if (VSA_Get_Proto(sua) == AF_INET) > - macro_def(h->vl, h->closed_sock, "sock", "%s:%s", addr, port); > - else > - macro_def(h->vl, h->closed_sock, "sock", "[%s]:%s", addr, port); > + VTCP_myname(closed_sock, addr, sizeof addr, port, sizeof port); > + macro_def(h->vl, h->closed_sock, "sock", "%s %s", addr, port); > macro_def(h->vl, h->closed_sock, "addr", "%s", addr); > macro_def(h->vl, h->closed_sock, "port", "%s", port); > VTCP_close(&closed_sock); > @@ -851,8 +843,6 @@ haproxy_build_backends(struct haproxy *h, const char *vsb_data) > int sock; > char buf[128], addr[128], port[128]; > const char *err; > - char vsabuf[vsa_suckaddr_len]; > - struct suckaddr *sua; > > p = strstr(p, HAPROXY_BE_FD_STR); > if (!p) > @@ -870,15 +860,10 @@ haproxy_build_backends(struct haproxy *h, const char *vsb_data) > vtc_fatal(h->vl, > "Create listen socket failed: %s", err); > assert(sock > 0); > - sua = VSA_getsockname(sock, vsabuf, sizeof vsabuf); > - AN(sua); > > - VTCP_name(sua, addr, sizeof addr, port, sizeof port); > + VTCP_myname(sock, addr, sizeof addr, port, sizeof port); > bprintf(buf, "%s_%s", h->name, p); > - if (VSA_Get_Proto(sua) == AF_INET) > - macro_def(h->vl, buf, "sock", "%s:%s", addr, port); > - else > - macro_def(h->vl, buf, "sock", "[%s]:%s", addr, port); > + macro_def(h->vl, buf, "sock", "%s %s", addr, port); > macro_def(h->vl, buf, "addr", "%s", addr); > macro_def(h->vl, buf, "port", "%s", port); > > diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c > index 88437b757..75a4b7c38 100644 > --- a/bin/varnishtest/vtc_misc.c > +++ b/bin/varnishtest/vtc_misc.c > @@ -384,7 +384,7 @@ ipvx_works(const char *target) > VTCP_close(&fd); > return (1); > } > - return(0); > + return(0); > } > > /* SECTION: feature feature > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From phk at phk.freebsd.dk Tue Jan 12 09:10:09 2021 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Tue, 12 Jan 2021 09:10:09 +0000 Subject: [master] cf7304c64 Merge HAproxy support from vtest In-Reply-To: References: <20210112084407.DA4C8107EB4@lists.varnish-cache.org> Message-ID: <22185.1610442609@critter.freebsd.dk> -------- Dridi Boukelmoune writes: > This is undoing some of the fixes from Nils that should have been > merged the other way around too. I know, my fault: I forgot to merge this from VTEST to varnish cache when it happened. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From phk at FreeBSD.org Tue Jan 12 10:21:07 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 12 Jan 2021 10:21:07 +0000 (UTC) Subject: [master] 45f989ee9 Add a test to exercise closing connections on tcp_pool teardown Message-ID: <20210112102107.31B1F10AA9B@lists.varnish-cache.org> commit 45f989ee923640fce50799347fc4f0a4de14f2a1 Author: Poul-Henning Kamp Date: Tue Jan 12 10:20:13 2021 +0000 Add a test to exercise closing connections on tcp_pool teardown diff --git a/bin/varnishtest/tests/c00102.vtc b/bin/varnishtest/tests/c00102.vtc new file mode 100644 index 000000000..ce49e0d57 --- /dev/null +++ b/bin/varnishtest/tests/c00102.vtc @@ -0,0 +1,60 @@ +varnishtest "TCP pool teardown with open connection" + +# when c1's request has gotten to s1 +barrier b1 cond 3 + +# when vcls have been swizzled +barrier b2 cond 3 + +server s1 { + rxreq + expect req.url == "/s1" + barrier b1 sync + barrier b2 sync + txresp -hdr "Host: s1" + expect_close +} -start + +varnish v1 -cliok "param.set debug +vclrel" + +varnish v1 -vcl+backend {} -start + +client c1 { + txreq -url /s1 + barrier b1 sync + barrier b2 sync + rxresp + expect resp.http.host == "s1" + + txreq -url /s2 + rxresp + expect resp.http.host == "s2" +} -start + +barrier b1 sync + +server s2 { + rxreq + expect req.url == "/s2" + txresp -hdr "Host: s2" +} -start + +varnish v1 -vcl { + backend b2 { + .host = "${s2_sock}"; + } +} + +varnish v1 -vsc LCK.tcp_pool.* +varnish v1 -cliok "vcl.discard vcl1" +varnish v1 -cliok "vcl.list" +varnish v1 -vsc LCK.tcp_pool.* +varnish v1 -expect LCK.tcp_pool.destroy == 0 + +barrier b2 sync + +client c1 -wait +varnish v1 -cliok "vcl.list" +server s1 -wait +varnish v1 -vsc LCK.tcp_pool.* +varnish v1 -expect LCK.tcp_pool.destroy == 1 From nils.goroll at uplex.de Tue Jan 12 11:07:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 11:07:06 +0000 (UTC) Subject: [master] 6a77c5e72 Revert "Merge HAproxy support from vtest" Message-ID: <20210112110706.DDF3A10DD81@lists.varnish-cache.org> commit 6a77c5e72f0973fb47165270b7c44344885c8da6 Author: Nils Goroll Date: Tue Jan 12 11:59:04 2021 +0100 Revert "Merge HAproxy support from vtest" Will apply the specific vtest patch instead This reverts commit cf7304c646d75b74fd84ccd040d2fcefe9c2ef5b. diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index 012a89cba..ca0cb9259 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -45,10 +45,10 @@ #include "vpf.h" #include "vre.h" #include "vtcp.h" +#include "vsa.h" #include "vtim.h" #define HAPROXY_PROGRAM_ENV_VAR "HAPROXY_PROGRAM" -#define HAPROXY_ARGS_ENV_VAR "HAPROXY_ARGS" #define HAPROXY_OPT_WORKER "-W" #define HAPROXY_OPT_MCLI "-S" #define HAPROXY_OPT_DAEMON "-D" @@ -174,7 +174,7 @@ haproxy_cli_tcp_connect(struct vtclog *vl, const char *addr, double tmo, const char **errp) { int fd; - char mabuf[32], mpbuf[32]; + char mabuf[VTCP_ADDRBUFSIZE], mpbuf[VTCP_PORTBUFSIZE]; AN(addr); AN(errp); @@ -520,16 +520,23 @@ haproxy_create_mcli(struct haproxy *h) int sock; const char *err; char buf[128], addr[128], port[128]; + char vsabuf[vsa_suckaddr_len]; + struct suckaddr *sua; - sock = VTCP_listen_on("127.0.0.1:0", NULL, 100, &err); + sock = VTCP_listen_on(default_listen_addr, NULL, 100, &err); if (err != NULL) vtc_fatal(h->vl, "Create listen socket failed: %s", err); assert(sock > 0); + sua = VSA_getsockname(sock, vsabuf, sizeof vsabuf); + AN(sua); - VTCP_myname(sock, addr, sizeof addr, port, sizeof port); + VTCP_name(sua, addr, sizeof addr, port, sizeof port); bprintf(buf, "%s_mcli", h->name); - macro_def(h->vl, buf, "sock", "%s %s", addr, port); + if (VSA_Get_Proto(sua) == AF_INET) + macro_def(h->vl, buf, "sock", "%s:%s", addr, port); + else + macro_def(h->vl, buf, "sock", "[%s]:%s", addr, port); macro_def(h->vl, buf, "addr", "%s", addr); macro_def(h->vl, buf, "port", "%s", port); @@ -559,18 +566,14 @@ haproxy_new(const char *name) int closed_sock; char addr[128], port[128]; const char *err; - const char *env_args; + char vsabuf[vsa_suckaddr_len]; + struct suckaddr *sua; ALLOC_OBJ(h, HAPROXY_MAGIC); AN(h); REPLACE(h->name, name); h->args = VSB_new_auto(); - env_args = getenv(HAPROXY_ARGS_ENV_VAR); - if (env_args) { - VSB_cat(h->args, env_args); - VSB_cat(h->args, " "); - } h->vl = vtc_logopen("%s", name); vtc_log_set_cmd(h->vl, haproxy_cli_cmds); @@ -604,8 +607,13 @@ haproxy_new(const char *name) vtc_fatal(h->vl, "Create listen socket failed: %s", err); assert(closed_sock > 0); - VTCP_myname(closed_sock, addr, sizeof addr, port, sizeof port); - macro_def(h->vl, h->closed_sock, "sock", "%s %s", addr, port); + sua = VSA_getsockname(closed_sock, vsabuf, sizeof vsabuf); + AN(sua); + VTCP_name(sua, addr, sizeof addr, port, sizeof port); + if (VSA_Get_Proto(sua) == AF_INET) + macro_def(h->vl, h->closed_sock, "sock", "%s:%s", addr, port); + else + macro_def(h->vl, h->closed_sock, "sock", "[%s]:%s", addr, port); macro_def(h->vl, h->closed_sock, "addr", "%s", addr); macro_def(h->vl, h->closed_sock, "port", "%s", port); VTCP_close(&closed_sock); @@ -843,6 +851,8 @@ haproxy_build_backends(struct haproxy *h, const char *vsb_data) int sock; char buf[128], addr[128], port[128]; const char *err; + char vsabuf[vsa_suckaddr_len]; + struct suckaddr *sua; p = strstr(p, HAPROXY_BE_FD_STR); if (!p) @@ -860,10 +870,15 @@ haproxy_build_backends(struct haproxy *h, const char *vsb_data) vtc_fatal(h->vl, "Create listen socket failed: %s", err); assert(sock > 0); + sua = VSA_getsockname(sock, vsabuf, sizeof vsabuf); + AN(sua); - VTCP_myname(sock, addr, sizeof addr, port, sizeof port); + VTCP_name(sua, addr, sizeof addr, port, sizeof port); bprintf(buf, "%s_%s", h->name, p); - macro_def(h->vl, buf, "sock", "%s %s", addr, port); + if (VSA_Get_Proto(sua) == AF_INET) + macro_def(h->vl, buf, "sock", "%s:%s", addr, port); + else + macro_def(h->vl, buf, "sock", "[%s]:%s", addr, port); macro_def(h->vl, buf, "addr", "%s", addr); macro_def(h->vl, buf, "port", "%s", port); diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index 75a4b7c38..88437b757 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -384,7 +384,7 @@ ipvx_works(const char *target) VTCP_close(&fd); return (1); } - return(0); + return(0); } /* SECTION: feature feature From nils.goroll at uplex.de Tue Jan 12 11:07:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 11:07:06 +0000 (UTC) Subject: [master] b21a771e5 vtc_haproxy: support passing extra arguments via HAPROXY_ARGS Message-ID: <20210112110706.F1E1110DD84@lists.varnish-cache.org> commit b21a771e5cfafedccca611381a81ff25ee78a556 Author: Willy Tarreau Date: Wed Dec 16 10:19:27 2020 +0100 vtc_haproxy: support passing extra arguments via HAPROXY_ARGS We're missing the ability to easily pass some extra debugging options, typically memory poisonning or memory limitations, while executing tests. Let's add the consideration for the HAPROXY_ARGS environment variable, to optionally prepend a series of arguments. diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index ca0cb9259..c345ef42d 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -49,6 +49,7 @@ #include "vtim.h" #define HAPROXY_PROGRAM_ENV_VAR "HAPROXY_PROGRAM" +#define HAPROXY_ARGS_ENV_VAR "HAPROXY_ARGS" #define HAPROXY_OPT_WORKER "-W" #define HAPROXY_OPT_MCLI "-S" #define HAPROXY_OPT_DAEMON "-D" @@ -566,6 +567,7 @@ haproxy_new(const char *name) int closed_sock; char addr[128], port[128]; const char *err; + const char *env_args; char vsabuf[vsa_suckaddr_len]; struct suckaddr *sua; @@ -574,6 +576,11 @@ haproxy_new(const char *name) REPLACE(h->name, name); h->args = VSB_new_auto(); + env_args = getenv(HAPROXY_ARGS_ENV_VAR); + if (env_args) { + VSB_cat(h->args, env_args); + VSB_cat(h->args, " "); + } h->vl = vtc_logopen("%s", name); vtc_log_set_cmd(h->vl, haproxy_cli_cmds); From nils.goroll at uplex.de Tue Jan 12 11:07:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 11:07:07 +0000 (UTC) Subject: [master] 238bc837b Typo Message-ID: <20210112110707.199C210DD87@lists.varnish-cache.org> commit 238bc837bc9d5ac3226ce4ccd2205a373f376f20 Author: Poul-Henning Kamp Date: Mon Jan 11 10:01:48 2021 +0000 Typo diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index 88437b757..75a4b7c38 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -384,7 +384,7 @@ ipvx_works(const char *target) VTCP_close(&fd); return (1); } - return(0); + return(0); } /* SECTION: feature feature From nils.goroll at uplex.de Tue Jan 12 11:29:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 11:29:06 +0000 (UTC) Subject: [master] a19e22625 name the ban lurker thread Message-ID: <20210112112906.A058A10E968@lists.varnish-cache.org> commit a19e22625caf062a501023cfc4878bbda52422a5 Author: Nils Goroll Date: Tue Jan 12 12:28:35 2021 +0100 name the ban lurker thread diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c index d0990f410..6676ce746 100644 --- a/bin/varnishd/cache/cache_ban_lurker.c +++ b/bin/varnishd/cache/cache_ban_lurker.c @@ -420,6 +420,8 @@ ban_lurker(struct worker *wrk, void *priv) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); AZ(priv); + THR_SetName("ban_lurker"); + VSL_Setup(&vsl, NULL, 0); while (!ban_shutdown) { From nils.goroll at uplex.de Tue Jan 12 12:38:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 12:38:08 +0000 (UTC) Subject: [master] 685de98b0 Revert "name the ban lurker thread" Message-ID: <20210112123808.2B2EE110483@lists.varnish-cache.org> commit 685de98b032b0dbdeebdc3f78b0adca8b1831057 Author: Nils Goroll Date: Tue Jan 12 13:34:47 2021 +0100 Revert "name the ban lurker thread" Sorry, this was a stupid oversight. The worker thread gets already named by wrk_bgthread This reverts commit a19e22625caf062a501023cfc4878bbda52422a5. diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c index 6676ce746..d0990f410 100644 --- a/bin/varnishd/cache/cache_ban_lurker.c +++ b/bin/varnishd/cache/cache_ban_lurker.c @@ -420,8 +420,6 @@ ban_lurker(struct worker *wrk, void *priv) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); AZ(priv); - THR_SetName("ban_lurker"); - VSL_Setup(&vsl, NULL, 0); while (!ban_shutdown) { From phk at FreeBSD.org Tue Jan 12 12:50:06 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 12 Jan 2021 12:50:06 +0000 (UTC) Subject: [master] ee6898163 Cover more of cache_tcp_pool.c by using ipv6 Message-ID: <20210112125006.8FF67110A59@lists.varnish-cache.org> commit ee6898163358ed71ad32e5edea2e611422011d15 Author: Poul-Henning Kamp Date: Tue Jan 12 12:49:25 2021 +0000 Cover more of cache_tcp_pool.c by using ipv6 diff --git a/bin/varnishtest/tests/c00102.vtc b/bin/varnishtest/tests/c00102.vtc index ce49e0d57..96bca461a 100644 --- a/bin/varnishtest/tests/c00102.vtc +++ b/bin/varnishtest/tests/c00102.vtc @@ -1,12 +1,14 @@ varnishtest "TCP pool teardown with open connection" +feature ipv6 + # when c1's request has gotten to s1 barrier b1 cond 3 # when vcls have been swizzled barrier b2 cond 3 -server s1 { +server s1 -listen "::1" { rxreq expect req.url == "/s1" barrier b1 sync @@ -33,7 +35,9 @@ client c1 { barrier b1 sync -server s2 { +varnish v1 -cliok "param.set debug +vclrel" + +server s2 -listen "::1" { rxreq expect req.url == "/s2" txresp -hdr "Host: s2" From phk at FreeBSD.org Tue Jan 12 14:55:08 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 12 Jan 2021 14:55:08 +0000 (UTC) Subject: [master] 97d2ffb34 Polish Message-ID: <20210112145508.983661139AB@lists.varnish-cache.org> commit 97d2ffb3407b920a1a73d3ee6795b2f328348a75 Author: Poul-Henning Kamp Date: Tue Jan 12 14:47:29 2021 +0000 Polish diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 4dc9324b8..b93c1e9e4 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -574,8 +574,9 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, VRT_fail(ctx, "%s: Illegal IP", __func__); return (NULL); } - } else + } else { assert(vep->ipv4== NULL && vep->ipv6== NULL); + } vcl = ctx->vcl; AN(vcl); @@ -583,7 +584,8 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, /* Create new backend */ ALLOC_OBJ(be, BACKEND_MAGIC); - XXXAN(be); + if (be == NULL) + return (NULL); Lck_New(&be->mtx, lck_backend); #define DA(x) do { if (vrt->x != NULL) REPLACE((be->x), (vrt->x)); } while (0) diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index 02db6c6f3..5614c9412 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -109,7 +109,8 @@ struct conn_pool { int holddown_errno; }; -static struct lock conn_pools_mtx; +static struct lock conn_pools_mtx; + static VTAILQ_HEAD(, conn_pool) conn_pools = VTAILQ_HEAD_INITIALIZER(conn_pools); @@ -488,9 +489,9 @@ VCP_Get(struct conn_pool *cp, vtim_dur tmo, struct worker *wrk, Lck_Lock(&cp->mtx); pfd = VTAILQ_FIRST(&cp->connlist); CHECK_OBJ_ORNULL(pfd, PFD_MAGIC); - if (force_fresh || pfd == NULL || pfd->state == PFD_STATE_STOLEN) + if (force_fresh || pfd == NULL || pfd->state == PFD_STATE_STOLEN) { pfd = NULL; - else { + } else { assert(pfd->conn_pool == cp); assert(pfd->state == PFD_STATE_AVAIL); VTAILQ_REMOVE(&cp->connlist, pfd, list); diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h index c2fccb909..00e8518f5 100644 --- a/bin/varnishd/cache/cache_tcp_pool.h +++ b/bin/varnishd/cache/cache_tcp_pool.h @@ -34,6 +34,7 @@ struct conn_pool; struct pfd; + #define PFD_STATE_AVAIL (1<<0) #define PFD_STATE_USED (1<<1) #define PFD_STATE_STOLEN (1<<2) @@ -51,8 +52,6 @@ void PFD_RemoteName(const struct pfd *, char *, unsigned, char *, unsigned); * Prototypes */ -struct VSC_vbe; - struct conn_pool *VTP_Ref(const struct vrt_endpoint *, const char *ident); /* * Get a reference to a TCP pool. Either one or both of ipv4 or @@ -101,7 +100,5 @@ int VCP_Wait(struct worker *, struct pfd *, vtim_real tmo); * function before attempting to receive on the connection. */ -void VCP_Panic(struct vsb *, struct conn_pool *); - VCL_IP VCP_GetIp(struct pfd *); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 1ecab14d3..489691625 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -145,10 +145,6 @@ void VCA_Shutdown(void); void VBE_InitCfg(void); void VBE_Poll(void); -/* cache_tcp_pool.c */ -void VCP_Init(void); - -/* cache_backend_poll.c */ void VBP_Init(void); /* cache_ban.c */ @@ -421,6 +417,13 @@ void VSL_ChgId(struct vsl_log *vsl, const char *typ, const char *why, void VSL_End(struct vsl_log *vsl); void VSL_Flush(struct vsl_log *, int overflow); +/* cache_tcp_pool.c */ +struct conn_pool; +void VCP_Init(void); +void VCP_Panic(struct vsb *, struct conn_pool *); + +/* cache_backend_poll.c */ + /* cache_vary.c */ int VRY_Create(struct busyobj *bo, struct vsb **psb); int VRY_Match(struct req *, const uint8_t *vary); From nils.goroll at uplex.de Tue Jan 12 15:30:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 15:30:08 +0000 (UTC) Subject: [master] d89c8bf59 Test per-instance PRIV_TASK Message-ID: <20210112153008.30AA31148E9@lists.varnish-cache.org> commit d89c8bf59b9da8263f1a767c68e4ee35d2569d11 Author: Nils Goroll Date: Tue Jan 12 16:24:27 2021 +0100 Test per-instance PRIV_TASK this is how we have been using VRT_priv_task() for long. Before making it the official way, bring vmod_debug and our own tests in sync. This tests - that PRIV_TASK is different for client and backend side - that vcl_pipe is client side wrt PRIV_TASK - that finalizers are called and probably things I forgot. diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc index ce162378e..a01903b53 100644 --- a/bin/varnishtest/tests/v00041.vtc +++ b/bin/varnishtest/tests/v00041.vtc @@ -20,35 +20,51 @@ varnish v1 -arg "-p debug=+vclrel -p workspace_client=1m" -vcl+backend { import std; sub vcl_init { - new objx = debug.obj(); + new objc = debug.obj(); + new objb = debug.obj(); + } + + sub log_obj { + std.log("objc " + objc.test_priv_task()); + std.log("objb " + objb.test_priv_task()); } sub vcl_init { debug.test_priv_task("something"); debug.test_priv_task("to remember"); std.log("func " + debug.test_priv_task()); - std.log("obj " + objx.test_priv_task()); + objc.test_priv_task("initX"); + objb.test_priv_task("initY"); + call log_obj; } sub vcl_recv { if (req.url == "/perf") { return (synth(200)); } - if (req.url == "/pipe") { - return (pipe); - } debug.test_priv_task(req.url); set req.http.x0 = debug.test_priv_task(); debug.test_priv_task("bazz"); + call log_obj; + objc.test_priv_task("c" + req.xid); + if (req.url == "/pipe") { + return (pipe); + } } sub vcl_pipe { + call log_obj; + objc.test_priv_task("p" + req.xid); + debug.test_priv_task(req.url); set req.http.x0 = debug.test_priv_task(); debug.test_priv_task("bazz"); } sub vcl_synth { + call log_obj; + objc.test_priv_task("s" + req.xid); + std.log("discard 1000 " + debug.priv_perf(1000)); std.log("perf 1 " + debug.priv_perf(1)); std.log("perf 10 " + debug.priv_perf(10)); @@ -58,66 +74,156 @@ varnish v1 -arg "-p debug=+vclrel -p workspace_client=1m" -vcl+backend { } sub vcl_deliver { + call log_obj; + objc.test_priv_task("d" + req.xid); + set resp.http.x0 = req.http.x0; set resp.http.x1 = debug.test_priv_task(); - set resp.http.o1 = objx.test_priv_task(); + set resp.http.objc = objc.test_priv_task(); } sub vcl_backend_fetch { - objx.test_priv_task("b"); - std.log("foo"); + call log_obj; + objb.test_priv_task("f" + bereq.xid); + + debug.test_priv_task("b"); set bereq.http.bx0 = debug.test_priv_task(bereq.url); - std.log("bar"); } sub vcl_backend_response { + call log_obj; + objb.test_priv_task("r" + bereq.xid); + set beresp.http.bx0 = bereq.http.bx0; set beresp.http.bx1 = debug.test_priv_task(""); - set beresp.http.bo1 = objx.test_priv_task(""); + set beresp.http.objb = objb.test_priv_task(""); } sub vcl_fini { debug.test_priv_task("cleaning"); debug.test_priv_task("up"); std.log("func " + debug.test_priv_task()); - std.log("obj " + objx.test_priv_task()); + std.log("obj " + objc.test_priv_task()); } } -start -logexpect l1 -v v1 -g raw -d 1 { +logexpect l0 -v v1 -g raw -d 1 -q "vxid == 0 and not ExpKill" { expect 0 0 CLI {^Rd vcl.load} expect 0 = Debug {^test_priv_task.*new.$} expect 0 = Debug {^test_priv_task.*update.$} expect 0 = Debug {^test_priv_task.*exists.$} expect 0 = VCL_Log {^func something to remember} - expect 0 = Debug {^test_priv_task.*exists.$} - expect 0 = VCL_Log {^obj something to remember} + expect 0 = Debug {^objc.priv_task.. = .*"initX". .new.} + expect 0 = Debug {^objb.priv_task.. = .*"initY". .new.} + expect 0 = Debug {^objc.priv_task.. = .*"initX"} + expect 0 = VCL_Log {^objc initX} + expect 0 = Debug {^objb.priv_task.. = .*"initY"} + expect 0 = VCL_Log {^objb initY} + expect 0 = Debug {^priv_task_free} + expect 0 = Debug {^obj_priv_task_fini.*"initX"} + expect 0 = Debug {^obj_priv_task_fini.*"initY"} + expect 0 = Debug {^vcl1: VCL_EVENT_WARM} - expect * 1002 Begin fetch$ - expect * = VCL_call ^BACKEND_FETCH - expect 0 = VCL_Log ^foo - expect 0 = BereqHeader {^bx0: b /foobar} - expect 0 = VCL_Log ^bar - - expect * 1004 Begin fetch$ - expect * = VCL_call ^BACKEND_FETCH - expect 0 = VCL_Log ^foo - expect 0 = BereqHeader {^bx0: b /snafu} - expect 0 = VCL_Log ^bar - - # this is not the order of events because std.log logs - # into the CLI ctx->vsl, while vmod_debug VSL() goes straight - # to the log. - expect * 0 Debug {^vcl1: VCL_EVENT_COLD} - expect * 0 CLI {^Rd vcl.discard} + # 1001/1002 + expect * = Debug {^test_priv_task.*new.$} + expect 0 = Debug {^test_priv_task.*exists.$} + expect 0 = Debug {^test_priv_task.*update.$} expect 0 = Debug {^test_priv_task.*new.$} expect 0 = Debug {^test_priv_task.*update.$} expect 0 = Debug {^test_priv_task.*exists.$} + expect 0 = Debug {^priv_task_free} + expect 0 = Debug {^obj_priv_task_fini.*"r1002"} + expect 0 = Debug {^test_priv_task.*exists} + expect 0 = Debug {^priv_task_free} + expect 0 = Debug {^obj_priv_task_fini.*"d1001"} + + # ... + # 1006 pipe + expect * = Debug {^obj_priv_task_fini.*"p1006"} + + # vcl_fini + expect * = Debug {^vcl1: VCL_EVENT_COLD} + expect 0 = CLI {^Wr 200 0 } + expect 0 = CLI {^Rd vcl.discard vcl1} + expect 0 = Debug {^test_priv_task.*new.$} + expect 0 = Debug {^test_priv_task.*update.$} expect 0 = Debug {^test_priv_task.*exists.$} expect 0 = Debug {^priv_task_free} - expect * = VCL_Log {^func cleaning up} - expect 0 = VCL_Log {^obj cleaning up} + expect 0 = VCL_Log {^func cleaning up} + expect 0 = Debug {^objc.priv_task.. = NULL} + expect 0 = VCL_Log {^obj } + expect 0 = CLI {^Wr 200 0 } +} -start + +logexpect l1001 -v v1 -g vxid -q "vxid == 1001" { + expect * 1001 VCL_call {^RECV} + expect 0 = ReqHeader {^x0: /foobar} + expect 0 = Debug {^objc.priv_task.. = NULL} + expect 0 = VCL_Log {^objc } + expect 0 = Debug {^objb.priv_task.. = NULL} + expect 0 = VCL_Log {^objb } + expect 0 = Debug {^objc.priv_task.. =.*"c1001". .new.} + + expect * = VCL_call {^DELIVER} + expect 0 = Debug {^objc.priv_task.. =.*"c1001".} + expect 0 = VCL_Log {^objc c1001} + expect 0 = Debug {^objb.priv_task.. = NULL} + expect 0 = VCL_Log {^objb } + expect 0 = Debug {^objc.priv_task.. =.*"d1001". .update.} + expect 0 = RespHeader {^x0: /foobar} + expect 0 = RespHeader {^x1: /foobar bazz} + expect 0 = Debug {^objc.priv_task.. =.*"d1001".} + expect 0 = RespHeader {^objc: d1001} + expect 0 = VCL_return {^deliver} +} -start + +logexpect l1002 -v v1 -g vxid -q "vxid == 1002" { + expect * 1002 VCL_call {^BACKEND_FETCH} + expect 0 = Debug {^objc.priv_task.. = NULL} + expect 0 = VCL_Log {^objc } + expect 0 = Debug {^objb.priv_task.. = NULL} + expect 0 = VCL_Log {^objb } + expect 0 = Debug {^objb.priv_task.. =.*"f1002". .new.} + expect 0 = BereqHeader {^bx0: b /foobar} + expect 0 = VCL_return {^fetch} + + expect * = VCL_call {^BACKEND_RESPONSE} + expect 0 = Debug {^objc.priv_task.. = NULL} + expect 0 = VCL_Log {^objc } + expect 0 = Debug {^objb.priv_task.. =.*"f1002".} + expect 0 = VCL_Log {^objb f1002} + expect 0 = Debug {^objb.priv_task.. =.*"r1002". .update.} + expect 0 = BerespHeader {^bx0: b /foobar} + expect 0 = BerespHeader {^bx1: b /foobar} + expect 0 = Debug {^objb.priv_task.. =.*"r1002".} + expect 0 = BerespHeader {^objb: r1002} + expect 0 = VCL_return {^deliver} +} -start + +logexpect l1006 -v v1 -g vxid -q "vxid == 1006" { + expect * 1006 VCL_call {^RECV} + expect 0 = ReqHeader {^x0: /pipe} + expect 0 = Debug {^objc.priv_task.. = NULL} + expect 0 = VCL_Log {^objc } + expect 0 = Debug {^objb.priv_task.. = NULL} + expect 0 = VCL_Log {^objb } + expect 0 = Debug {^objc.priv_task.. =.*"c1006". .new.} + expect 0 = VCL_return {^pipe} + + expect 0 = VCL_call {^HASH} + expect 0 = VCL_return {^lookup} + + expect 0 = Link {^bereq 1007 pipe} + expect 0 = VCL_call {^PIPE} + expect 0 = Debug {^objc.priv_task.. =.*"c1006".} + expect 0 = VCL_Log {^objc c1006} + expect 0 = Debug {^objb.priv_task.. = NULL} + expect 0 = VCL_Log {^objb } + expect 0 = Debug {^objc.priv_task.. =.*"p1006". .update.} + expect 0 = ReqUnset {^x0: /pipe} + expect 0 = ReqHeader {^x0: /pipe bazz /pipe} + expect 0 = VCL_return {^pipe} } -start client c1 { @@ -125,19 +231,19 @@ client c1 { rxresp expect resp.http.x0 == /foobar expect resp.http.x1 == "/foobar bazz" - expect resp.http.o1 == "/foobar bazz" + expect resp.http.objc == "d1001" expect resp.http.bx0 == "b /foobar" expect resp.http.bx1 == "b /foobar" - expect resp.http.bo1 == "b /foobar" + expect resp.http.objb == "r1002" txreq -url /snafu rxresp expect resp.http.x0 == /snafu expect resp.http.x1 == "/snafu bazz" - expect resp.http.o1 == "/snafu bazz" + expect resp.http.objc == "d1003" expect resp.http.bx0 == "b /snafu" expect resp.http.bx1 == "b /snafu" - expect resp.http.bo1 == "b /snafu" + expect resp.http.objb == "r1004" txreq -url /perf rxresp @@ -161,4 +267,7 @@ client c1 { varnish v1 -cliok "vcl.list" -logexpect l1 -wait +logexpect l0 -wait +logexpect l1001 -wait +logexpect l1002 -wait +logexpect l1006 -wait diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index bb663ea94..2be114a4f 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -64,6 +64,7 @@ static struct VSC_debug *vsc = NULL; static int loads; static const int store_ip_token; static const int fail_rollback_token; +extern void mylog(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...); /**********************************************************************/ @@ -819,8 +820,7 @@ xyzzy_sethdr(VRT_CTX, VCL_HEADER hs, VCL_STRANDS s) } } -// coverage -static void +void mylog(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...) { va_list ap; diff --git a/vmod/vmod_debug.vcc b/vmod/vmod_debug.vcc index fa81c8cf8..a787bb2c8 100644 --- a/vmod/vmod_debug.vcc +++ b/vmod/vmod_debug.vcc @@ -112,12 +112,9 @@ Test method for VCL private pointers Objects share the ``PRIV_*`` state with other objects and methods from the same vmod - IOW the ``PRIV_*`` state is per vmod, not per object. -$Method STRING .test_priv_task(PRIV_TASK, STRING s="") +$Method STRING .test_priv_task(STRING s="") -Test method for TASK private pointers - -Objects share the ``PRIV_*`` state with other objects and methods from -the same vmod - IOW the ``PRIV_*`` state is per vmod, not per object. +Test per-object priv_task via VRT_priv_task() / VRT_priv_get() $Method STRING .test_priv_top(PRIV_TOP, STRING) diff --git a/vmod/vmod_debug_obj.c b/vmod/vmod_debug_obj.c index 8bc328290..1ccb2f826 100644 --- a/vmod/vmod_debug_obj.c +++ b/vmod/vmod_debug_obj.c @@ -37,12 +37,16 @@ #include "vcc_debug_if.h" +// vmod_debug.c +extern void mylog(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...); + struct xyzzy_debug_obj { unsigned magic; #define VMOD_DEBUG_OBJ_MAGIC 0xccbd9b77 int foobar; const char *string; const char *number; + VCL_STRING vcl_name; }; VCL_VOID @@ -52,7 +56,6 @@ xyzzy_obj__init(VRT_CTX, struct xyzzy_debug_obj **op, struct xyzzy_debug_obj *o; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - (void)vcl_name; AN(op); AZ(*op); ALLOC_OBJ(o, VMOD_DEBUG_OBJ_MAGIC); @@ -61,6 +64,7 @@ xyzzy_obj__init(VRT_CTX, struct xyzzy_debug_obj **op, o->foobar = 42; o->string = s; o->number = e; + o->vcl_name = vcl_name; AN(*op); } @@ -147,12 +151,55 @@ xyzzy_obj_test_priv_vcl(VRT_CTX, xyzzy_test_priv_vcl(ctx, priv); } +static void +obj_priv_task_fini(void *ptr) +{ + AN(ptr); + VSL(SLT_Debug, 0, "obj_priv_task_fini(%p = \"%s\")", ptr, ptr); +} + +static const struct vmod_priv_methods xyzzy_obj_test_priv_task_methods[1] = {{ + .magic = VMOD_PRIV_METHODS_MAGIC, + .type = "debug_onk_test_priv_task", + .fini = obj_priv_task_fini +}}; + VCL_STRING v_matchproto_() -xyzzy_obj_test_priv_task(VRT_CTX, - struct xyzzy_debug_obj *o, struct vmod_priv *priv, VCL_STRING s) +xyzzy_obj_test_priv_task(VRT_CTX, struct xyzzy_debug_obj *o, VCL_STRING s) { - (void)o; - return (xyzzy_test_priv_task(ctx, priv, s)); + struct vmod_priv *p; + + if (s == NULL || *s == '\0') { + p = VRT_priv_task_get(ctx, o); + if (p == NULL) { + mylog(ctx->vsl, SLT_Debug, "%s.priv_task() = NULL", + o->vcl_name); + return (""); + } + mylog(ctx->vsl, SLT_Debug, + "%s.priv_task() = %p .priv = %p (\"%s\")", + o->vcl_name, p, p->priv, p->priv); + return (p->priv); + } + + p = VRT_priv_task(ctx, o); + + if (p == NULL) { + mylog(ctx->vsl, SLT_Debug, "%s.priv_task() = NULL [err]", + o->vcl_name); + VRT_fail(ctx, "no priv task - out of ws?"); + return (""); + } + + mylog(ctx->vsl, SLT_Debug, + "%s.priv_task() = %p .priv = %p (\"%s\") [%s]", + o->vcl_name, p, s, s, p->priv ? "update" : "new"); + + /* minimum scope of s and priv is the task - no need to copy */ + p->priv = TRUST_ME(s); + p->methods = xyzzy_obj_test_priv_task_methods; + + return (p->priv); } VCL_STRING v_matchproto_() From nils.goroll at uplex.de Tue Jan 12 15:34:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 15:34:07 +0000 (UTC) Subject: [master] c3693c9bc vcl: Allocation-free vcl.discard Message-ID: <20210112153407.33CE2114C33@lists.varnish-cache.org> commit c3693c9bcf1581aadc63cf6e7bf3c66f48f9f6be Author: Dridi Boukelmoune Date: Tue Dec 8 11:41:28 2020 +0100 vcl: Allocation-free vcl.discard This replaces the previous mapping of VCLs to discard with a mark and sweep operation where a different head is used to track matching VCLs. To discard VCLs in topological order, we can simply discard dependencies recursively before discarding a given VCL. diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 5b8a57115..84ac7fada 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -657,14 +657,17 @@ mcf_vcl_use(struct cli *cli, const char * const *av, void *priv) } static void -mgt_vcl_discard(struct cli *cli, struct vclprog *vp) +mgt_vcl_discard(struct cli *cli, struct vclproghead *vh, struct vclprog *vp) { char *p = NULL; unsigned status; AN(vp); + AN(vp->discard); assert(vp != active_vcl); - assert(VTAILQ_EMPTY(&vp->dto)); + + while (!VTAILQ_EMPTY(&vp->dto)) + mgt_vcl_discard(cli, vh, VTAILQ_FIRST(&vp->dto)->from); if (mcf_is_label(vp)) { AN(vp->warm); @@ -678,9 +681,39 @@ mgt_vcl_discard(struct cli *cli, struct vclprog *vp) assert(status == CLIS_OK || status == CLIS_COMMS); free(p); } + VTAILQ_REMOVE(vh, vp, discard_list); mgt_vcl_del(vp); } +static int +mgt_vcl_discard_mark(struct cli *cli, struct vclproghead *vh, const char *name) +{ + struct vclprog *vp; + unsigned marked = 0; + + VTAILQ_FOREACH(vp, &vclhead, list) { + if (strcmp(name, vp->name)) + continue; + if (vp == active_vcl) { + VCLI_SetResult(cli, CLIS_CANT); + VCLI_Out(cli, "Cannot discard active VCL program %s\n", + vp->name); + return (-1); + } + if (!vp->discard) + VTAILQ_INSERT_TAIL(vh, vp, discard_list); + vp->discard = 1; + marked++; + } + + if (marked == 0) { + VCLI_SetResult(cli, CLIS_PARAM); + VCLI_Out(cli, "No VCL named %s known\n", name); + } + + return (marked); +} + static void mgt_vcl_discard_depfail(struct cli *cli, struct vclprog *vp) { @@ -710,113 +743,55 @@ mgt_vcl_discard_depfail(struct cli *cli, struct vclprog *vp) } } -static struct vclprog ** -mgt_vcl_discard_depcheck(struct cli *cli, const char * const *names, - unsigned *lp) +static int +mgt_vcl_discard_depcheck(struct cli *cli, struct vclproghead *vh) { - struct vclprog **res, *vp; + struct vclprog *vp; struct vcldep *vd; - const char * const *s; - unsigned i, j, l; - - l = 0; - s = names; - while (*s != NULL) { - l++; - s++; - } - AN(l); - - res = calloc(l, sizeof *res); - AN(res); - - /* NB: Build a list of VCL programs and labels to discard. A null - * pointer in the array is a VCL program that no longer needs to be - * discarded. - */ - for (i = 0; i < l; i++) { - vp = mcf_find_vcl(cli, names[i]); - if (vp == NULL) - break; - if (vp == active_vcl) { - VCLI_SetResult(cli, CLIS_CANT); - VCLI_Out(cli, "Cannot discard active VCL program %s\n", - vp->name); - break; - } - for (j = 0; j < i; j++) - if (vp == res[j]) - break; - if (j < i) - continue; /* skip duplicates */ - //lint -e{661} - res[i] = vp; - } - if (i < l) { - free(res); - return (NULL); + VTAILQ_FOREACH(vp, vh, list) { + VTAILQ_FOREACH(vd, &vp->dto, lto) + if (!vd->from->discard) { + mgt_vcl_discard_depfail(cli, vp); + return (-1); + } } - /* NB: Check that all direct dependent VCL programs and labels belong - * to the list of VCLs to be discarded. This mechanically ensures that - * indirect dependent VCLs also belong. - */ - for (i = 0; i < l; i++) { - if (res[i] == NULL || VTAILQ_EMPTY(&res[i]->dto)) - continue; - VTAILQ_FOREACH(vd, &res[i]->dto, lto) { - for (j = 0; j < l; j++) - if (res[j] == vd->from) - break; - if (j == l) - break; - } - if (vd != NULL) - break; - } + return (0); +} - if (i < l) { - mgt_vcl_discard_depfail(cli, res[i]); - free(res); - return (NULL); - } +static void +mgt_vcl_discard_clear(struct vclproghead *vh) +{ + struct vclprog *vp, *vp2; - *lp = l; - return (res); + VTAILQ_FOREACH_SAFE(vp, vh, discard_list, vp2) { + AN(vp->discard); + vp->discard = 0; + VTAILQ_REMOVE(vh, vp, discard_list); + } } static void v_matchproto_(cli_func_t) mcf_vcl_discard(struct cli *cli, const char * const *av, void *priv) { - struct vclprog **vp; - unsigned i, l, done; + struct vclproghead vh[1]; (void)priv; - vp = mgt_vcl_discard_depcheck(cli, av + 2, &l); - if (vp == NULL) - return; - /* NB: discard VCLs in topological order. At this point it - * can only succeed, but the loop ensures that it eventually - * completes even if the dependency check is ever broken. - */ - AN(l); - do { - done = 0; - for (i = 0; i < l; i++) { - if (vp[i] == NULL || vp[i]->nto > 0) - continue; - mgt_vcl_discard(cli, vp[i]); - vp[i] = NULL; - done++; + VTAILQ_INIT(vh); + for (av += 2; *av != NULL; av++) { + if (mgt_vcl_discard_mark(cli, vh, *av) <= 0) { + mgt_vcl_discard_clear(vh); + break; } - } while (done > 0); + } + + if (mgt_vcl_discard_depcheck(cli, vh) != 0) + mgt_vcl_discard_clear(vh); - for (i = 0; i < l; i++) - if (vp[i] != NULL) - WRONG(vp[i]->name); - free(vp); + while (!VTAILQ_EMPTY(vh)) + mgt_vcl_discard(cli, vh, VTAILQ_FIRST(vh)); } static void v_matchproto_(cli_func_t) diff --git a/bin/varnishd/mgt/mgt_vcl.h b/bin/varnishd/mgt/mgt_vcl.h index 6f3b7d4bf..b82b19413 100644 --- a/bin/varnishd/mgt/mgt_vcl.h +++ b/bin/varnishd/mgt/mgt_vcl.h @@ -68,6 +68,8 @@ struct vclprog { int nto; int loaded; VTAILQ_HEAD(, vmoddep) vmods; + unsigned discard; + VTAILQ_ENTRY(vclprog) discard_list; }; struct vmodfile { From nils.goroll at uplex.de Tue Jan 12 15:34:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 15:34:07 +0000 (UTC) Subject: [master] 19b20a88f vcl: Add support for vcl.discard glob patterns Message-ID: <20210112153407.4FCBC114C36@lists.varnish-cache.org> commit 19b20a88f8d6f59c676ee922d53a10c398785d82 Author: Dridi Boukelmoune Date: Tue Dec 8 12:19:06 2020 +0100 vcl: Add support for vcl.discard glob patterns diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 84ac7fada..705de26ce 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -33,6 +33,7 @@ #include "config.h" +#include #include #include #include @@ -686,13 +687,13 @@ mgt_vcl_discard(struct cli *cli, struct vclproghead *vh, struct vclprog *vp) } static int -mgt_vcl_discard_mark(struct cli *cli, struct vclproghead *vh, const char *name) +mgt_vcl_discard_mark(struct cli *cli, struct vclproghead *vh, const char *glob) { struct vclprog *vp; unsigned marked = 0; VTAILQ_FOREACH(vp, &vclhead, list) { - if (strcmp(name, vp->name)) + if (fnmatch(glob, vp->name, 0)) continue; if (vp == active_vcl) { VCLI_SetResult(cli, CLIS_CANT); @@ -708,7 +709,7 @@ mgt_vcl_discard_mark(struct cli *cli, struct vclproghead *vh, const char *name) if (marked == 0) { VCLI_SetResult(cli, CLIS_PARAM); - VCLI_Out(cli, "No VCL named %s known\n", name); + VCLI_Out(cli, "No VCL name matches %s\n", glob); } return (marked); diff --git a/bin/varnishtest/tests/c00077.vtc b/bin/varnishtest/tests/c00077.vtc index f176bc948..e056dd986 100644 --- a/bin/varnishtest/tests/c00077.vtc +++ b/bin/varnishtest/tests/c00077.vtc @@ -64,5 +64,6 @@ varnish v1 -clierr 106 "vcl.label vclA vcl3" varnish v1 -cliok "vcl.symtab" -varnish v1 -clierr 300 "vcl.discard vcl1 vcl2 vcl3 vcl4 vcl5 vcl6 vcl7" -varnish v1 -cliok "vcl.discard vcl1 vcl2 vcl3 vcl4 vcl5 vcl6 vcl7 vclA vclB" +varnish v1 -clierr 300 "vcl.discard vcl*" +varnish v1 -clierr 300 "vcl.discard vcl[1-7]" +varnish v1 -cliok "vcl.discard vcl[1-7A-B]" diff --git a/include/tbl/cli_cmds.h b/include/tbl/cli_cmds.h index 02d853b68..7249f8dd7 100644 --- a/include/tbl/cli_cmds.h +++ b/include/tbl/cli_cmds.h @@ -99,11 +99,13 @@ CLI_CMD(VCL_STATE, CLI_CMD(VCL_DISCARD, "vcl.discard", - "vcl.discard ...", /* XXX: allow globs */ + "vcl.discard ...", "Unload the named configurations (if possible).", - " If more than one named configuration is specified the command" + "The configname or label arguments are matched as glob patterns." + " If more than one configuration matches patterns the command" " is equivalent to individual commands in the right order with" - " respect to configurations dependencies.", + " respect to configurations dependencies." + " All individual patterns must match at least one configuration.", 1, -1 ) From nils.goroll at uplex.de Tue Jan 12 15:34:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 15:34:07 +0000 (UTC) Subject: [master] acd2f5a4f vtc: Restore vcl.discard multiple arguments coverage Message-ID: <20210112153407.6C166114C3A@lists.varnish-cache.org> commit acd2f5a4f461c42d3d53e3b78c215c2d509b7d59 Author: Dridi Boukelmoune Date: Tue Dec 8 15:37:27 2020 +0100 vtc: Restore vcl.discard multiple arguments coverage Using a single glob defeats the purpose of the previous check. diff --git a/bin/varnishtest/tests/c00077.vtc b/bin/varnishtest/tests/c00077.vtc index e056dd986..ececd1519 100644 --- a/bin/varnishtest/tests/c00077.vtc +++ b/bin/varnishtest/tests/c00077.vtc @@ -66,4 +66,4 @@ varnish v1 -cliok "vcl.symtab" varnish v1 -clierr 300 "vcl.discard vcl*" varnish v1 -clierr 300 "vcl.discard vcl[1-7]" -varnish v1 -cliok "vcl.discard vcl[1-7A-B]" +varnish v1 -cliok "vcl.discard vcl[1-7] vcl[A-B]" From nils.goroll at uplex.de Tue Jan 12 15:34:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 15:34:07 +0000 (UTC) Subject: [master] 1bb782cb9 vcl: Use a static discard list Message-ID: <20210112153407.86587114C43@lists.varnish-cache.org> commit 1bb782cb99e8ca1ef91ed48c7460873bb1559266 Author: Dridi Boukelmoune Date: Tue Dec 8 18:51:18 2020 +0100 vcl: Use a static discard list And assert that it is emptied after all vcl.discard commands. Better diff with the --word-diff --word-diff-regex='\w+' options. diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 705de26ce..c934c8114 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -61,6 +61,7 @@ static const struct vclstate VCL_STATE_LABEL[1] = {{ "label" }}; static unsigned vcl_count; struct vclproghead vclhead = VTAILQ_HEAD_INITIALIZER(vclhead); +struct vclproghead discardhead = VTAILQ_HEAD_INITIALIZER(discardhead); struct vmodfilehead vmodhead = VTAILQ_HEAD_INITIALIZER(vmodhead); static struct vclprog *active_vcl; static struct vev *e_poker; @@ -658,7 +659,7 @@ mcf_vcl_use(struct cli *cli, const char * const *av, void *priv) } static void -mgt_vcl_discard(struct cli *cli, struct vclproghead *vh, struct vclprog *vp) +mgt_vcl_discard(struct cli *cli, struct vclprog *vp) { char *p = NULL; unsigned status; @@ -668,7 +669,7 @@ mgt_vcl_discard(struct cli *cli, struct vclproghead *vh, struct vclprog *vp) assert(vp != active_vcl); while (!VTAILQ_EMPTY(&vp->dto)) - mgt_vcl_discard(cli, vh, VTAILQ_FIRST(&vp->dto)->from); + mgt_vcl_discard(cli, VTAILQ_FIRST(&vp->dto)->from); if (mcf_is_label(vp)) { AN(vp->warm); @@ -682,12 +683,12 @@ mgt_vcl_discard(struct cli *cli, struct vclproghead *vh, struct vclprog *vp) assert(status == CLIS_OK || status == CLIS_COMMS); free(p); } - VTAILQ_REMOVE(vh, vp, discard_list); + VTAILQ_REMOVE(&discardhead, vp, discard_list); mgt_vcl_del(vp); } static int -mgt_vcl_discard_mark(struct cli *cli, struct vclproghead *vh, const char *glob) +mgt_vcl_discard_mark(struct cli *cli, const char *glob) { struct vclprog *vp; unsigned marked = 0; @@ -702,7 +703,7 @@ mgt_vcl_discard_mark(struct cli *cli, struct vclproghead *vh, const char *glob) return (-1); } if (!vp->discard) - VTAILQ_INSERT_TAIL(vh, vp, discard_list); + VTAILQ_INSERT_TAIL(&discardhead, vp, discard_list); vp->discard = 1; marked++; } @@ -745,12 +746,12 @@ mgt_vcl_discard_depfail(struct cli *cli, struct vclprog *vp) } static int -mgt_vcl_discard_depcheck(struct cli *cli, struct vclproghead *vh) +mgt_vcl_discard_depcheck(struct cli *cli) { struct vclprog *vp; struct vcldep *vd; - VTAILQ_FOREACH(vp, vh, list) { + VTAILQ_FOREACH(vp, &discardhead, list) { VTAILQ_FOREACH(vd, &vp->dto, lto) if (!vd->from->discard) { mgt_vcl_discard_depfail(cli, vp); @@ -762,37 +763,36 @@ mgt_vcl_discard_depcheck(struct cli *cli, struct vclproghead *vh) } static void -mgt_vcl_discard_clear(struct vclproghead *vh) +mgt_vcl_discard_clear(void) { struct vclprog *vp, *vp2; - VTAILQ_FOREACH_SAFE(vp, vh, discard_list, vp2) { + VTAILQ_FOREACH_SAFE(vp, &discardhead, discard_list, vp2) { AN(vp->discard); vp->discard = 0; - VTAILQ_REMOVE(vh, vp, discard_list); + VTAILQ_REMOVE(&discardhead, vp, discard_list); } } static void v_matchproto_(cli_func_t) mcf_vcl_discard(struct cli *cli, const char * const *av, void *priv) { - struct vclproghead vh[1]; (void)priv; - VTAILQ_INIT(vh); + assert(VTAILQ_EMPTY(&discardhead)); for (av += 2; *av != NULL; av++) { - if (mgt_vcl_discard_mark(cli, vh, *av) <= 0) { - mgt_vcl_discard_clear(vh); + if (mgt_vcl_discard_mark(cli, *av) <= 0) { + mgt_vcl_discard_clear(); break; } } - if (mgt_vcl_discard_depcheck(cli, vh) != 0) - mgt_vcl_discard_clear(vh); + if (mgt_vcl_discard_depcheck(cli) != 0) + mgt_vcl_discard_clear(); - while (!VTAILQ_EMPTY(vh)) - mgt_vcl_discard(cli, vh, VTAILQ_FIRST(vh)); + while (!VTAILQ_EMPTY(&discardhead)) + mgt_vcl_discard(cli, VTAILQ_FIRST(&discardhead)); } static void v_matchproto_(cli_func_t) From nils.goroll at uplex.de Tue Jan 12 15:34:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 15:34:07 +0000 (UTC) Subject: [master] 2658f51cf vcl: Check all discard flags before vcl.discard Message-ID: <20210112153407.9ECF9114C47@lists.varnish-cache.org> commit 2658f51cf6fa0a87a4c3b87c421374ed2ed798fa Author: Dridi Boukelmoune Date: Tue Dec 8 18:55:02 2020 +0100 vcl: Check all discard flags before vcl.discard Suggested by slink. diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index c934c8114..d852ae375 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -777,10 +777,14 @@ mgt_vcl_discard_clear(void) static void v_matchproto_(cli_func_t) mcf_vcl_discard(struct cli *cli, const char * const *av, void *priv) { + const struct vclprog *vp; (void)priv; assert(VTAILQ_EMPTY(&discardhead)); + VTAILQ_FOREACH(vp, &vclhead, list) + AZ(vp->discard); + for (av += 2; *av != NULL; av++) { if (mgt_vcl_discard_mark(cli, *av) <= 0) { mgt_vcl_discard_clear(); From nils.goroll at uplex.de Tue Jan 12 15:34:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 15:34:07 +0000 (UTC) Subject: [master] 624350a0e cli: Rewrite the vcl.discard documentation Message-ID: <20210112153407.B9BE6114C4B@lists.varnish-cache.org> commit 624350a0e83f302b5ebb2a67b43a87a06fdbe40b Author: Dridi Boukelmoune Date: Tue Jan 12 15:58:00 2021 +0100 cli: Rewrite the vcl.discard documentation Trying to be more descriptive about glob patterns, inspired by the backend.list documentation. diff --git a/include/tbl/cli_cmds.h b/include/tbl/cli_cmds.h index 7249f8dd7..9537833f6 100644 --- a/include/tbl/cli_cmds.h +++ b/include/tbl/cli_cmds.h @@ -99,13 +99,16 @@ CLI_CMD(VCL_STATE, CLI_CMD(VCL_DISCARD, "vcl.discard", - "vcl.discard ...", - "Unload the named configurations (if possible).", - "The configname or label arguments are matched as glob patterns." - " If more than one configuration matches patterns the command" - " is equivalent to individual commands in the right order with" - " respect to configurations dependencies." - " All individual patterns must match at least one configuration.", + "vcl.discard ...", + "Unload the named configurations (when possible).", + "Unload the named configurations and labels matching at least" + " one name pattern. All matching configurations and labels" + " are discarded in the correct order with respect to potential" + " dependencies. If one configuration or label could not be" + " discarded because one of its dependencies would remain," + " nothing is discarded." + " Each individual name pattern must match at least one named" + " configuration or label.", 1, -1 ) From nils.goroll at uplex.de Tue Jan 12 15:38:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 15:38:07 +0000 (UTC) Subject: [master] 59db991be polish d89c8bf59b9da8263f1a767c68e4ee35d2569d11 Message-ID: <20210112153807.1C1E7115538@lists.varnish-cache.org> commit 59db991be5281a7f4e9519781acc4eb244aa9b3e Author: Nils Goroll Date: Tue Jan 12 16:37:21 2021 +0100 polish d89c8bf59b9da8263f1a767c68e4ee35d2569d11 diff --git a/vmod/vmod_debug_obj.c b/vmod/vmod_debug_obj.c index 1ccb2f826..8915402df 100644 --- a/vmod/vmod_debug_obj.c +++ b/vmod/vmod_debug_obj.c @@ -155,7 +155,7 @@ static void obj_priv_task_fini(void *ptr) { AN(ptr); - VSL(SLT_Debug, 0, "obj_priv_task_fini(%p = \"%s\")", ptr, ptr); + VSL(SLT_Debug, 0, "obj_priv_task_fini(%p = \"%s\")", ptr, (char *)ptr); } static const struct vmod_priv_methods xyzzy_obj_test_priv_task_methods[1] = {{ From nils.goroll at uplex.de Tue Jan 12 15:44:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 15:44:07 +0000 (UTC) Subject: [master] 3aa44b7cd flexelint 1bb782cb99e8ca1ef91ed48c7460873bb1559266 Message-ID: <20210112154407.9143D1158C4@lists.varnish-cache.org> commit 3aa44b7cdc238025c8a714d131502fbc36e59c1e Author: Nils Goroll Date: Tue Jan 12 16:43:28 2021 +0100 flexelint 1bb782cb99e8ca1ef91ed48c7460873bb1559266 diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index d852ae375..f4f7f7f23 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -61,7 +61,7 @@ static const struct vclstate VCL_STATE_LABEL[1] = {{ "label" }}; static unsigned vcl_count; struct vclproghead vclhead = VTAILQ_HEAD_INITIALIZER(vclhead); -struct vclproghead discardhead = VTAILQ_HEAD_INITIALIZER(discardhead); +static struct vclproghead discardhead = VTAILQ_HEAD_INITIALIZER(discardhead); struct vmodfilehead vmodhead = VTAILQ_HEAD_INITIALIZER(vmodhead); static struct vclprog *active_vcl; static struct vev *e_poker; From nils.goroll at uplex.de Tue Jan 12 16:39:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 16:39:07 +0000 (UTC) Subject: [master] eae1571a6 PRIV_TASK vtc: make init/fini log order predictable & fix vtcs Message-ID: <20210112163907.43404116EAA@lists.varnish-cache.org> commit eae1571a693e22aa43815728dd2e1707919de2ae Author: Nils Goroll Date: Tue Jan 12 17:16:16 2021 +0100 PRIV_TASK vtc: make init/fini log order predictable & fix vtcs Ref d89c8bf59b9da8263f1a767c68e4ee35d2569d11 diff --git a/bin/varnishtest/tests/r02451.vtc b/bin/varnishtest/tests/r02451.vtc index 0dc0a5a4c..39cf67aa2 100644 --- a/bin/varnishtest/tests/r02451.vtc +++ b/bin/varnishtest/tests/r02451.vtc @@ -25,6 +25,7 @@ varnish v1 -vcl+backend { new vd = directors.shard(); debug.test_priv_task("something"); debug.test_priv_task("to remember"); + objx.test_priv_task(debug.test_priv_task()); vd.add_backend(s1); vd.add_backend(s2); @@ -44,8 +45,18 @@ varnish v1 -vcl+backend { logexpect l1 -v v1 -g raw -d 1 { expect 0 0 CLI {^Rd vcl.load} - expect 4 = VCL_Log {^func something to remember} - expect 2 = VCL_Log {^obj something to remember} + expect 0 * Debug {^test_priv_task.*new} + expect 0 * Debug {^test_priv_task.*update} + expect 0 * Debug {^test_priv_task.*exists} + expect 0 * Debug {^objx.priv_task.*"something to remember".*new} + expect 0 * Debug {^test_priv_task.*exists} + expect 0 * VCL_Log {^func something to remember} + expect 0 * Debug {^objx.priv_task.*"something to remember".$} + expect 0 * VCL_Log {^obj something to remember} + expect 0 * Debug {^priv_task_free} + # string stored in obj priv_task has already been freed + expect 0 * Debug {^obj_priv_task_fini} + expect 0 * Debug {^vcl1: VCL_EVENT_WARM} } -start client c1 { diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc index a01903b53..55415bd1d 100644 --- a/bin/varnishtest/tests/v00041.vtc +++ b/bin/varnishtest/tests/v00041.vtc @@ -149,9 +149,9 @@ logexpect l0 -v v1 -g raw -d 1 -q "vxid == 0 and not ExpKill" { expect 0 = Debug {^test_priv_task.*new.$} expect 0 = Debug {^test_priv_task.*update.$} expect 0 = Debug {^test_priv_task.*exists.$} + expect 0 = Debug {^objc.priv_task.. = NULL} expect 0 = Debug {^priv_task_free} expect 0 = VCL_Log {^func cleaning up} - expect 0 = Debug {^objc.priv_task.. = NULL} expect 0 = VCL_Log {^obj } expect 0 = CLI {^Wr 200 0 } } -start diff --git a/bin/varnishtest/tests/v00042.vtc b/bin/varnishtest/tests/v00042.vtc index 54dcccaa0..b378a2731 100644 --- a/bin/varnishtest/tests/v00042.vtc +++ b/bin/varnishtest/tests/v00042.vtc @@ -42,6 +42,7 @@ varnish v1 -vcl+backend { sub vcl_recv { set req.http.x0 = debug.test_priv_task(req.url + req.esi_level); + o.test_priv_task(req.url + req.esi_level); } sub vcl_miss { diff --git a/vmod/vmod_debug_obj.c b/vmod/vmod_debug_obj.c index 8915402df..45f0a9d04 100644 --- a/vmod/vmod_debug_obj.c +++ b/vmod/vmod_debug_obj.c @@ -34,6 +34,7 @@ #include #include "cache/cache.h" +#include "vcl.h" #include "vcc_debug_if.h" @@ -168,15 +169,23 @@ VCL_STRING v_matchproto_() xyzzy_obj_test_priv_task(VRT_CTX, struct xyzzy_debug_obj *o, VCL_STRING s) { struct vmod_priv *p; + struct vsl_log *vsl; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + + if (ctx->method & VCL_MET_TASK_H) + vsl = NULL; + else + vsl = ctx->vsl; if (s == NULL || *s == '\0') { p = VRT_priv_task_get(ctx, o); if (p == NULL) { - mylog(ctx->vsl, SLT_Debug, "%s.priv_task() = NULL", + mylog(vsl, SLT_Debug, "%s.priv_task() = NULL", o->vcl_name); return (""); } - mylog(ctx->vsl, SLT_Debug, + mylog(vsl, SLT_Debug, "%s.priv_task() = %p .priv = %p (\"%s\")", o->vcl_name, p, p->priv, p->priv); return (p->priv); @@ -185,13 +194,13 @@ xyzzy_obj_test_priv_task(VRT_CTX, struct xyzzy_debug_obj *o, VCL_STRING s) p = VRT_priv_task(ctx, o); if (p == NULL) { - mylog(ctx->vsl, SLT_Debug, "%s.priv_task() = NULL [err]", + mylog(vsl, SLT_Debug, "%s.priv_task() = NULL [err]", o->vcl_name); VRT_fail(ctx, "no priv task - out of ws?"); return (""); } - mylog(ctx->vsl, SLT_Debug, + mylog(vsl, SLT_Debug, "%s.priv_task() = %p .priv = %p (\"%s\") [%s]", o->vcl_name, p, s, s, p->priv ? "update" : "new"); From nils.goroll at uplex.de Tue Jan 12 17:09:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 17:09:07 +0000 (UTC) Subject: [master] e01254d71 insignificant reordering Message-ID: <20210112170907.60F59117BD3@lists.varnish-cache.org> commit e01254d71494ee9c17fdb1ec761ae8920d9fb4d3 Author: Nils Goroll Date: Tue Jan 12 17:52:16 2021 +0100 insignificant reordering diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index 8e7ef17e1..67d5b5755 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -208,12 +208,12 @@ VRT_priv_top(VRT_CTX, const void *vmod_id) struct vmod_priv *priv; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - if (ctx->req == NULL) { + req = ctx->req; + if (req == NULL) { WRONG("PRIV_TOP is only accessible in client VCL context"); NEEDLESS(return (NULL)); } - req = ctx->req; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ(req, REQ_MAGIC); sp = ctx->sp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); top = req->top; From nils.goroll at uplex.de Tue Jan 12 17:09:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 17:09:07 +0000 (UTC) Subject: [master] b02c78438 PRIV_TASK vtc: tolerate fini order Message-ID: <20210112170907.7500C117BD6@lists.varnish-cache.org> commit b02c78438c4e36d967a866466c9ce0cccb32dc0f Author: Nils Goroll Date: Tue Jan 12 18:07:09 2021 +0100 PRIV_TASK vtc: tolerate fini order this turned out to be harder than I thought: I had not considered the fact that fini order of a red-black tree indexed by uintptrs depends on the memory order. Ref d89c8bf59b9da8263f1a767c68e4ee35d2569d11 diff --git a/bin/varnishtest/tests/r02451.vtc b/bin/varnishtest/tests/r02451.vtc index 39cf67aa2..b25163a65 100644 --- a/bin/varnishtest/tests/r02451.vtc +++ b/bin/varnishtest/tests/r02451.vtc @@ -53,9 +53,10 @@ logexpect l1 -v v1 -g raw -d 1 { expect 0 * VCL_Log {^func something to remember} expect 0 * Debug {^objx.priv_task.*"something to remember".$} expect 0 * VCL_Log {^obj something to remember} - expect 0 * Debug {^priv_task_free} - # string stored in obj priv_task has already been freed - expect 0 * Debug {^obj_priv_task_fini} + # string stored in obj priv_task can already been freed, + # depending on the order + expect 0 * Debug {^priv_task_free|^obj_priv_task_fini} + expect 0 * Debug {^priv_task_free|^obj_priv_task_fini} expect 0 * Debug {^vcl1: VCL_EVENT_WARM} } -start diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc index 55415bd1d..f3febd423 100644 --- a/bin/varnishtest/tests/v00041.vtc +++ b/bin/varnishtest/tests/v00041.vtc @@ -120,9 +120,9 @@ logexpect l0 -v v1 -g raw -d 1 -q "vxid == 0 and not ExpKill" { expect 0 = VCL_Log {^objc initX} expect 0 = Debug {^objb.priv_task.. = .*"initY"} expect 0 = VCL_Log {^objb initY} - expect 0 = Debug {^priv_task_free} - expect 0 = Debug {^obj_priv_task_fini.*"initX"} - expect 0 = Debug {^obj_priv_task_fini.*"initY"} + expect 0 = Debug {^priv_task_free|^obj_priv_task_fini} + expect 0 = Debug {^priv_task_free|^obj_priv_task_fini} + expect 0 = Debug {^priv_task_free|^obj_priv_task_fini} expect 0 = Debug {^vcl1: VCL_EVENT_WARM} # 1001/1002 @@ -135,8 +135,8 @@ logexpect l0 -v v1 -g raw -d 1 -q "vxid == 0 and not ExpKill" { expect 0 = Debug {^priv_task_free} expect 0 = Debug {^obj_priv_task_fini.*"r1002"} expect 0 = Debug {^test_priv_task.*exists} - expect 0 = Debug {^priv_task_free} - expect 0 = Debug {^obj_priv_task_fini.*"d1001"} + expect 0 = Debug {^priv_task_free|^obj_priv_task_fini.*"d1001"} + expect 0 = Debug {^priv_task_free|^obj_priv_task_fini.*"d1001"} # ... # 1006 pipe From dridi at varni.sh Tue Jan 12 17:23:04 2021 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 12 Jan 2021 17:23:04 +0000 Subject: [master] b02c78438 PRIV_TASK vtc: tolerate fini order In-Reply-To: <20210112170907.7500C117BD6@lists.varnish-cache.org> References: <20210112170907.7500C117BD6@lists.varnish-cache.org> Message-ID: On Tue, Jan 12, 2021 at 5:09 PM Nils Goroll wrote: > > > commit b02c78438c4e36d967a866466c9ce0cccb32dc0f > Author: Nils Goroll > Date: Tue Jan 12 18:07:09 2021 +0100 > > PRIV_TASK vtc: tolerate fini order > > this turned out to be harder than I thought: I had not considered the > fact that fini order of a red-black tree indexed by uintptrs depends on > the memory order. > > Ref d89c8bf59b9da8263f1a767c68e4ee35d2569d11 Feature idea, logexpect undefined order: expect ? = Debug ^priv_task_free expect ? = Debug ^obj_priv_task_fini The "?" token would mean that consecutive expect statements using it could appear in a non-deterministic order. Thoughts? > diff --git a/bin/varnishtest/tests/r02451.vtc b/bin/varnishtest/tests/r02451.vtc > index 39cf67aa2..b25163a65 100644 > --- a/bin/varnishtest/tests/r02451.vtc > +++ b/bin/varnishtest/tests/r02451.vtc > @@ -53,9 +53,10 @@ logexpect l1 -v v1 -g raw -d 1 { > expect 0 * VCL_Log {^func something to remember} > expect 0 * Debug {^objx.priv_task.*"something to remember".$} > expect 0 * VCL_Log {^obj something to remember} > - expect 0 * Debug {^priv_task_free} > - # string stored in obj priv_task has already been freed > - expect 0 * Debug {^obj_priv_task_fini} > + # string stored in obj priv_task can already been freed, > + # depending on the order > + expect 0 * Debug {^priv_task_free|^obj_priv_task_fini} > + expect 0 * Debug {^priv_task_free|^obj_priv_task_fini} > expect 0 * Debug {^vcl1: VCL_EVENT_WARM} > } -start > > diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc > index 55415bd1d..f3febd423 100644 > --- a/bin/varnishtest/tests/v00041.vtc > +++ b/bin/varnishtest/tests/v00041.vtc > @@ -120,9 +120,9 @@ logexpect l0 -v v1 -g raw -d 1 -q "vxid == 0 and not ExpKill" { > expect 0 = VCL_Log {^objc initX} > expect 0 = Debug {^objb.priv_task.. = .*"initY"} > expect 0 = VCL_Log {^objb initY} > - expect 0 = Debug {^priv_task_free} > - expect 0 = Debug {^obj_priv_task_fini.*"initX"} > - expect 0 = Debug {^obj_priv_task_fini.*"initY"} > + expect 0 = Debug {^priv_task_free|^obj_priv_task_fini} > + expect 0 = Debug {^priv_task_free|^obj_priv_task_fini} > + expect 0 = Debug {^priv_task_free|^obj_priv_task_fini} > expect 0 = Debug {^vcl1: VCL_EVENT_WARM} > > # 1001/1002 > @@ -135,8 +135,8 @@ logexpect l0 -v v1 -g raw -d 1 -q "vxid == 0 and not ExpKill" { > expect 0 = Debug {^priv_task_free} > expect 0 = Debug {^obj_priv_task_fini.*"r1002"} > expect 0 = Debug {^test_priv_task.*exists} > - expect 0 = Debug {^priv_task_free} > - expect 0 = Debug {^obj_priv_task_fini.*"d1001"} > + expect 0 = Debug {^priv_task_free|^obj_priv_task_fini.*"d1001"} > + expect 0 = Debug {^priv_task_free|^obj_priv_task_fini.*"d1001"} > > # ... > # 1006 pipe > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From nils.goroll at uplex.de Tue Jan 12 17:35:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 17:35:06 +0000 (UTC) Subject: [master] 05e795741 PRIV_TASK vtc: tolerate fini order Message-ID: <20210112173507.00316118907@lists.varnish-cache.org> commit 05e7957414467684db375ba6319f79a861a1b742 Author: Nils Goroll Date: Tue Jan 12 18:26:41 2021 +0100 PRIV_TASK vtc: tolerate fini order see previous, fix typo along the way Ref d89c8bf59b9da8263f1a767c68e4ee35d2569d11 diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc index f3febd423..c4ec0123a 100644 --- a/bin/varnishtest/tests/v00041.vtc +++ b/bin/varnishtest/tests/v00041.vtc @@ -132,8 +132,8 @@ logexpect l0 -v v1 -g raw -d 1 -q "vxid == 0 and not ExpKill" { expect 0 = Debug {^test_priv_task.*new.$} expect 0 = Debug {^test_priv_task.*update.$} expect 0 = Debug {^test_priv_task.*exists.$} - expect 0 = Debug {^priv_task_free} - expect 0 = Debug {^obj_priv_task_fini.*"r1002"} + expect 0 = Debug {^priv_task_free|^obj_priv_task_fini.*"r1002"} + expect 0 = Debug {^priv_task_free|^obj_priv_task_fini.*"r1002"} expect 0 = Debug {^test_priv_task.*exists} expect 0 = Debug {^priv_task_free|^obj_priv_task_fini.*"d1001"} expect 0 = Debug {^priv_task_free|^obj_priv_task_fini.*"d1001"} diff --git a/vmod/vmod_debug_obj.c b/vmod/vmod_debug_obj.c index 45f0a9d04..d6db70ff4 100644 --- a/vmod/vmod_debug_obj.c +++ b/vmod/vmod_debug_obj.c @@ -161,7 +161,7 @@ obj_priv_task_fini(void *ptr) static const struct vmod_priv_methods xyzzy_obj_test_priv_task_methods[1] = {{ .magic = VMOD_PRIV_METHODS_MAGIC, - .type = "debug_onk_test_priv_task", + .type = "debug_obj_test_priv_task", .fini = obj_priv_task_fini }}; From nils.goroll at uplex.de Tue Jan 12 17:41:23 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 18:41:23 +0100 Subject: [master] b02c78438 PRIV_TASK vtc: tolerate fini order In-Reply-To: References: <20210112170907.7500C117BD6@lists.varnish-cache.org> Message-ID: <9b56cd50-16f6-86ee-32a9-a57fc2de7e33@uplex.de> On 12/01/2021 18:23, Dridi Boukelmoune wrote: > Feature idea, logexpect undefined order: > > expect ? = Debug ^priv_task_free > expect ? = Debug ^obj_priv_task_fini > > The "?" token would mean that consecutive expect statements using it > could appear in a non-deterministic order. > > Thoughts? great idea! The end result would be the same as the regex with alternation, but it would look _much_ cleaner. -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From guillaume at varnish-software.com Tue Jan 12 18:08:07 2021 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Tue, 12 Jan 2021 18:08:07 +0000 (UTC) Subject: [master] 916f77719 [cci] add archlinux Message-ID: <20210112180807.5965D2C31@lists.varnish-cache.org> commit 916f777199cffc04641e37ee78c0a4389f467611 Author: Guillaume Quintard Date: Tue Jan 12 10:07:08 2021 -0800 [cci] add archlinux diff --git a/.circleci/config.yml b/.circleci/config.yml index 71f8bb03a..631b619ba 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -281,6 +281,12 @@ workflows: #extra_conf: --without-jemalloc requires: - dist + - distcheck: + name: distcheck_arch + dist: archlinux + release: "latest" + requires: + - dist nightly: triggers: - schedule: From guillaume at varnish-software.com Tue Jan 12 18:18:06 2021 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Tue, 12 Jan 2021 18:18:06 +0000 (UTC) Subject: [master] 90550650e [cci] special case user addition for arch Message-ID: <20210112181806.B103B412D@lists.varnish-cache.org> commit 90550650e44b9bc0e469b3e0b8600107fc453cad Author: Guillaume Quintard Date: Tue Jan 12 10:17:20 2021 -0800 [cci] special case user addition for arch diff --git a/.circleci/config.yml b/.circleci/config.yml index 631b619ba..adcaae3b6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -209,7 +209,9 @@ jobs: name: Extract and distcheck command: | tar xavf varnish-*.tar.gz --strip 1 - if [ << parameters.dist >> = centos ]; then + if [ << parameters.dist >> = archlinux ]; then + useradd varnish + elif [ << parameters.dist >> = centos ]; then adduser varnish else adduser --disabled-password --gecos "" varnish @@ -282,7 +284,7 @@ workflows: requires: - dist - distcheck: - name: distcheck_arch + name: distcheck_archlinux dist: archlinux release: "latest" requires: From guillaume at varnish-software.com Tue Jan 12 18:30:07 2021 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Tue, 12 Jan 2021 18:30:07 +0000 (UTC) Subject: [master] 7e9c6a7ba [cci] use su instead of sudo Message-ID: <20210112183007.9D0364823@lists.varnish-cache.org> commit 7e9c6a7ba293ffb7b2875d4b8d168c44cc289eaf Author: Guillaume Quintard Date: Tue Jan 12 10:29:09 2021 -0800 [cci] use su instead of sudo diff --git a/.circleci/config.yml b/.circleci/config.yml index adcaae3b6..899dd36cb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -224,17 +224,17 @@ jobs: export TSAN_OPTIONS=abort_on_error=1,halt_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/tsan.suppr export UBSAN_OPTIONS=halt_on_error=1,print_stacktrace=1,use_sigaltstack=0,suppressions=$(pwd)/tools/ubsan.suppr - sudo -u varnish ./configure \ + su varnish -c './configure \ --quiet \ --with-unwind \ --enable-developer-warnings \ --enable-debugging-symbols \ --disable-stack-protector \ --with-persistent-storage \ - << parameters.extra_conf >> - sudo -u varnish \ - --preserve-env=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \ - make distcheck VERBOSE=1 -j 12 -k + << parameters.extra_conf >>' + su varnish \ + --whitelist-environment=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \ + -c 'make distcheck VERBOSE=1 -j 12 -k' collect_packages: docker: From guillaume at varnish-software.com Tue Jan 12 18:39:06 2021 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Tue, 12 Jan 2021 18:39:06 +0000 (UTC) Subject: [master] c660aa365 Revert "[cci] use su instead of sudo" Message-ID: <20210112183906.B1D694D43@lists.varnish-cache.org> commit c660aa365f00de48535764d499c190189c982707 Author: Guillaume Quintard Date: Tue Jan 12 10:36:00 2021 -0800 Revert "[cci] use su instead of sudo" This reverts commit 7e9c6a7ba293ffb7b2875d4b8d168c44cc289eaf. diff --git a/.circleci/config.yml b/.circleci/config.yml index 899dd36cb..adcaae3b6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -224,17 +224,17 @@ jobs: export TSAN_OPTIONS=abort_on_error=1,halt_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/tsan.suppr export UBSAN_OPTIONS=halt_on_error=1,print_stacktrace=1,use_sigaltstack=0,suppressions=$(pwd)/tools/ubsan.suppr - su varnish -c './configure \ + sudo -u varnish ./configure \ --quiet \ --with-unwind \ --enable-developer-warnings \ --enable-debugging-symbols \ --disable-stack-protector \ --with-persistent-storage \ - << parameters.extra_conf >>' - su varnish \ - --whitelist-environment=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \ - -c 'make distcheck VERBOSE=1 -j 12 -k' + << parameters.extra_conf >> + sudo -u varnish \ + --preserve-env=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \ + make distcheck VERBOSE=1 -j 12 -k collect_packages: docker: From guillaume at varnish-software.com Tue Jan 12 18:39:06 2021 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Tue, 12 Jan 2021 18:39:06 +0000 (UTC) Subject: [master] 3210e23ee [cci] install sudo on arch Message-ID: <20210112183906.C4C374D46@lists.varnish-cache.org> commit 3210e23ee519bf730888f8bf8d999119f4b859d4 Author: Guillaume Quintard Date: Tue Jan 12 10:38:13 2021 -0800 [cci] install sudo on arch diff --git a/.circleci/config.yml b/.circleci/config.yml index adcaae3b6..f7873b711 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -210,6 +210,7 @@ jobs: command: | tar xavf varnish-*.tar.gz --strip 1 if [ << parameters.dist >> = archlinux ]; then + pacman -Sy vim --noconfirm sudo useradd varnish elif [ << parameters.dist >> = centos ]; then adduser varnish From nils.goroll at uplex.de Tue Jan 12 18:41:09 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 19:41:09 +0100 Subject: [master] b02c78438 PRIV_TASK vtc: tolerate fini order In-Reply-To: <9b56cd50-16f6-86ee-32a9-a57fc2de7e33@uplex.de> References: <20210112170907.7500C117BD6@lists.varnish-cache.org> <9b56cd50-16f6-86ee-32a9-a57fc2de7e33@uplex.de> Message-ID: <1a4f7a33-bbcc-9cd5-6bc2-09c3bc255ffd@uplex.de> On 12/01/2021 18:41, Nils Goroll wrote: > On 12/01/2021 18:23, Dridi Boukelmoune wrote: >> Feature idea, logexpect undefined order: >> >> expect ? = Debug ^priv_task_free >> expect ? = Debug ^obj_priv_task_fini >> >> The "?" token would mean that consecutive expect statements using it >> could appear in a non-deterministic order. >> >> Thoughts? > > great idea! The end result would be the same as the regex with alternation, but > it would look _much_ cleaner. Actually, no, it would not be the same. Only the proposed solution could ensure that we see the required number of matches per line. -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From guillaume at varnish-software.com Tue Jan 12 18:51:06 2021 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Tue, 12 Jan 2021 18:51:06 +0000 (UTC) Subject: [master] 7c403fe78 [cci] arch needs a bit more than just sudo Message-ID: <20210112185106.740D35951@lists.varnish-cache.org> commit 7c403fe78fc6e296f24b79891cee394dedbfdf70 Author: Guillaume Quintard Date: Tue Jan 12 10:50:25 2021 -0800 [cci] arch needs a bit more than just sudo diff --git a/.circleci/config.yml b/.circleci/config.yml index f7873b711..f40caacc5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -202,6 +202,23 @@ jobs: py3-sphinx \ tar \ sudo + elif [ << parameters.dist >> = alpine ]; then + pacman -Sy --noconfirm \ + autoconf \ + automake \ + build-base \ + ca-certificates \ + cpio \ + gzip \ + libedit \ + libtool \ + libunwind \ + linux-headers \ + pcre \ + python-docutils \ + python-sphinx \ + tar \ + sudo fi - attach_workspace: at: /workspace @@ -210,7 +227,6 @@ jobs: command: | tar xavf varnish-*.tar.gz --strip 1 if [ << parameters.dist >> = archlinux ]; then - pacman -Sy vim --noconfirm sudo useradd varnish elif [ << parameters.dist >> = centos ]; then adduser varnish From guillaume at varnish-software.com Tue Jan 12 18:59:07 2021 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Tue, 12 Jan 2021 18:59:07 +0000 (UTC) Subject: [master] d5fb6c2a9 [cci] fix arch dist testwq Message-ID: <20210112185907.472645E2C@lists.varnish-cache.org> commit d5fb6c2a91b79bb246c15cab1cdcd09276f3d4df Author: Guillaume Quintard Date: Tue Jan 12 10:58:23 2021 -0800 [cci] fix arch dist testwq diff --git a/.circleci/config.yml b/.circleci/config.yml index f40caacc5..eed43496c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -202,7 +202,7 @@ jobs: py3-sphinx \ tar \ sudo - elif [ << parameters.dist >> = alpine ]; then + elif [ << parameters.dist >> = archlinux ]; then pacman -Sy --noconfirm \ autoconf \ automake \ From dridi at varni.sh Tue Jan 12 19:04:55 2021 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 12 Jan 2021 19:04:55 +0000 Subject: [master] e95760818 vtest: plug insignificant leak In-Reply-To: <20210110165608.3C9B9AE938@lists.varnish-cache.org> References: <20210110165608.3C9B9AE938@lists.varnish-cache.org> Message-ID: On Sun, Jan 10, 2021 at 4:56 PM Nils Goroll wrote: > > > commit e95760818552ba954d3750638c0b084ee4879214 > Author: Nils Goroll > Date: Sun Jan 10 17:54:07 2021 +0100 > > vtest: plug insignificant leak > > introduced with 1c19f3fed3da8fae79ef200f2ebf01625a42bb27 > > CID 1471724 > > Ref #3490 This patch fails 100% of our VTCs on macos. > diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c > index 9b5a60a25..fa7524c2d 100644 > --- a/bin/varnishtest/vtc_main.c > +++ b/bin/varnishtest/vtc_main.c > @@ -585,6 +585,7 @@ ip_magic(void) > struct suckaddr *sa; > char abuf[VTCP_ADDRBUFSIZE]; > char pbuf[VTCP_PORTBUFSIZE]; > + char *s; > > /* > * In FreeBSD jails localhost/127.0.0.1 becomes the jails IP# > @@ -622,10 +623,13 @@ ip_magic(void) > extmacro_def("bad_backend", "[%s]:%s", abuf, pbuf); > > /* our default bind/listen address */ > + s = macro_get("localhost", NULL); > + AN(s); > if (VSA_Get_Proto(sa) == AF_INET) > - bprintf(abuf, "%s:0", macro_get("localhost", NULL)); > + bprintf(abuf, "%s:0", s); > else > - bprintf(abuf, "[%s]:0", macro_get("localhost", NULL)); > + bprintf(abuf, "[%s]:0", s); > + free(s); > > extmacro_def("listen_addr", "%s", abuf); > default_listen_addr = strdup(abuf); > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From guillaume at varnish-software.com Tue Jan 12 19:15:06 2021 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Tue, 12 Jan 2021 19:15:06 +0000 (UTC) Subject: [master] 461c9639e [cci] no build-base in arch Message-ID: <20210112191506.B125C6827@lists.varnish-cache.org> commit 461c9639ef6bba300d2800ee2420112ebfaad611 Author: Guillaume Quintard Date: Tue Jan 12 11:14:42 2021 -0800 [cci] no build-base in arch diff --git a/.circleci/config.yml b/.circleci/config.yml index eed43496c..371654ba7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -206,7 +206,6 @@ jobs: pacman -Sy --noconfirm \ autoconf \ automake \ - build-base \ ca-certificates \ cpio \ gzip \ From guillaume at varnish-software.com Tue Jan 12 19:23:07 2021 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Tue, 12 Jan 2021 19:23:07 +0000 (UTC) Subject: [master] 9b5c541bd [cci] base-devel covers some of our needs Message-ID: <20210112192307.68D096D86@lists.varnish-cache.org> commit 9b5c541bd3bb3f03ad4f1e264b7e57df3b5b4c2d Author: Guillaume Quintard Date: Tue Jan 12 11:22:42 2021 -0800 [cci] base-devel covers some of our needs diff --git a/.circleci/config.yml b/.circleci/config.yml index 371654ba7..fd5658910 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -204,11 +204,9 @@ jobs: sudo elif [ << parameters.dist >> = archlinux ]; then pacman -Sy --noconfirm \ - autoconf \ - automake \ + base-devel \ ca-certificates \ cpio \ - gzip \ libedit \ libtool \ libunwind \ @@ -216,8 +214,7 @@ jobs: pcre \ python-docutils \ python-sphinx \ - tar \ - sudo + tar fi - attach_workspace: at: /workspace From guillaume at varnish-software.com Tue Jan 12 20:14:12 2021 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Tue, 12 Jan 2021 20:14:12 +0000 (UTC) Subject: [master] 31042bece [cci] distcheck from git Message-ID: <20210112201412.534F39540@lists.varnish-cache.org> commit 31042bece195872a7f8404635ee6c0afe0d9a4ce Author: Guillaume Quintard Date: Tue Jan 12 11:38:40 2021 -0800 [cci] distcheck from git diff --git a/.circleci/config.yml b/.circleci/config.yml index fd5658910..9a10465cc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -156,6 +156,7 @@ jobs: yum install -y epel-release yum install -y \ automake \ + git \ jemalloc-devel \ libedit-devel \ libtool \ @@ -174,6 +175,7 @@ jobs: build-essential \ ca-certificates \ cpio \ + git \ graphviz \ libedit-dev \ libjemalloc-dev \ @@ -192,6 +194,7 @@ jobs: build-base \ ca-certificates \ cpio \ + git \ gzip \ libedit-dev \ libtool \ @@ -207,6 +210,7 @@ jobs: base-devel \ ca-certificates \ cpio \ + git \ libedit \ libtool \ libunwind \ @@ -216,12 +220,10 @@ jobs: python-sphinx \ tar fi - - attach_workspace: - at: /workspace + - checkout - run: name: Extract and distcheck command: | - tar xavf varnish-*.tar.gz --strip 1 if [ << parameters.dist >> = archlinux ]; then useradd varnish elif [ << parameters.dist >> = centos ]; then @@ -237,7 +239,10 @@ jobs: export TSAN_OPTIONS=abort_on_error=1,halt_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/tsan.suppr export UBSAN_OPTIONS=halt_on_error=1,print_stacktrace=1,use_sigaltstack=0,suppressions=$(pwd)/tools/ubsan.suppr - sudo -u varnish ./configure \ + sudo -u varnish \ + autoreconf -i -v + sudo -u varnish \ + ./configure \ --quiet \ --with-unwind \ --enable-developer-warnings \ @@ -274,34 +279,24 @@ workflows: name: distcheck_centos_7 dist: centos release: "7" - requires: - - dist - distcheck: name: distcheck_centos_8 dist: centos release: "8" - requires: - - dist - distcheck: name: distcheck_debian_buster dist: debian release: buster extra_conf: --enable-asan --enable-ubsan - requires: - - dist - distcheck: name: distcheck_alpine dist: alpine release: "latest" #extra_conf: --without-jemalloc - requires: - - dist - distcheck: name: distcheck_archlinux dist: archlinux release: "latest" - requires: - - dist nightly: triggers: - schedule: From dridi.boukelmoune at gmail.com Tue Jan 12 21:20:10 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 12 Jan 2021 21:20:10 +0000 (UTC) Subject: [master] f855fa46f varnishtest: Bandaid for an uninitialized mutex Message-ID: <20210112212010.41C866138F@lists.varnish-cache.org> commit f855fa46fb1ad66f16e264d57129d605e87f33c4 Author: Dridi Boukelmoune Date: Tue Jan 12 22:15:49 2021 +0100 varnishtest: Bandaid for an uninitialized mutex diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index d1a14cf8c..d22e36037 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -139,9 +139,12 @@ extmacro_def(const char *name, const char *fmt, ...) /********************************************************************** * Below this point is run inside the testing child-process. + * + * XXX: This is no longer the case for macro_get() that is currently + * used in the main process. */ -static pthread_mutex_t macro_mtx; +static pthread_mutex_t macro_mtx = PTHREAD_MUTEX_INITIALIZER; static void init_macro(void) @@ -151,8 +154,6 @@ init_macro(void) /* Dump the extmacros for completeness */ VTAILQ_FOREACH(m, ¯o_list, list) vtc_log(vltop, 4, "extmacro def %s=%s", m->name, m->val); - - AZ(pthread_mutex_init(¯o_mtx, NULL)); } void From dridi.boukelmoune at gmail.com Tue Jan 12 21:20:10 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 12 Jan 2021 21:20:10 +0000 (UTC) Subject: [master] 8cf6443d7 varnishtest: Allocate the exact amount for ${date} Message-ID: <20210112212010.595A461391@lists.varnish-cache.org> commit 8cf6443d794a7ec71457901a49a263e6ea47c0f9 Author: Dridi Boukelmoune Date: Tue Jan 12 22:17:28 2021 +0100 varnishtest: Allocate the exact amount for ${date} It might not make a difference, except for the disappearance of a magic number. diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index d22e36037..97e895ca0 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -220,7 +220,7 @@ macro_get(const char *b, const char *e) if (l == 4 && !memcmp(b, "date", l)) { double t = VTIM_real(); - retval = malloc(64); + retval = malloc(VTIM_FORMAT_SIZE); AN(retval); VTIM_format(t, retval); return (retval); From nils.goroll at uplex.de Tue Jan 12 22:11:12 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 22:11:12 +0000 (UTC) Subject: [master] 27f28e44d vtest: refactor logexp_dispatch Message-ID: <20210112221112.7996062E78@lists.varnish-cache.org> commit 27f28e44d8924eb8a242db7871b1f0c8cfd4a303 Author: Nils Goroll Date: Tue Jan 12 20:18:39 2021 +0100 vtest: refactor logexp_dispatch pull out the match code diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 5f7f006c5..89b4e6872 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -264,16 +264,78 @@ logexp_next(struct logexp *le) vtc_log(le->vl, 3, "expecting| %s", VSB_data(le->test->str)); } +static int +logexp_match(struct logexp *le, const char *data, int vxid, int tag, + int type, int len) +{ + const char *legend; + int ok = 1, skip = 0; + + if (le->test->vxid == LE_LAST) { + if (le->vxid_last != vxid) + ok = 0; + } else if (le->test->vxid >= 0) { + if (le->test->vxid != vxid) + ok = 0; + } + if (le->test->tag == LE_LAST) { + if (le->tag_last != tag) + ok = 0; + } else if (le->test->tag >= 0) { + if (le->test->tag != tag) + ok = 0; + } + if (le->test->vre && + le->test->tag >= 0 && + le->test->tag == tag && + VRE_ERROR_NOMATCH == VRE_exec(le->test->vre, data, + len, 0, 0, NULL, 0, NULL)) + ok = 0; + + if (!ok && (le->test->skip_max == LE_ANY || + le->test->skip_max > le->skip_cnt)) + skip = 1; + + if (ok) + legend = "match"; + else if (skip && le->m_arg) + legend = "miss"; + else if (skip) + legend = NULL; + else + legend = "err"; + + if (legend != NULL) + vtc_log(le->vl, 4, "%-5s| %10u %-15s %c %.*s", + legend, vxid, VSL_tags[tag], type, len, + data); + + if (ok) { + le->vxid_last = vxid; + le->tag_last = tag; + le->skip_cnt = 0; + logexp_next(le); + if (le->test == NULL) + /* End of test script */ + return (1); + } else if (skip) + le->skip_cnt++; + else { + /* Signal fail */ + return (2); + } + return (0); +} + static int v_matchproto_(VSLQ_dispatch_f) logexp_dispatch(struct VSL_data *vsl, struct VSL_transaction * const pt[], void *priv) { struct logexp *le; struct VSL_transaction *t; - int i; - int ok, skip; + int i, r; int vxid, tag, type, len; - const char *legend, *data; + const char *data; CAST_OBJ_NOTNULL(le, priv, LOGEXP_MAGIC); @@ -284,71 +346,20 @@ logexp_dispatch(struct VSL_data *vsl, struct VSL_transaction * const pt[], CHECK_OBJ_NOTNULL(le->test, LOGEXP_TEST_MAGIC); AN(t->c->rec.ptr); - vxid = VSL_ID(t->c->rec.ptr); tag = VSL_TAG(t->c->rec.ptr); - data = VSL_CDATA(t->c->rec.ptr); - len = VSL_LEN(t->c->rec.ptr) - 1; if (tag == SLT__Batch || tag == SLT_Witness) continue; - ok = 1; - if (le->test->vxid == LE_LAST) { - if (le->vxid_last != vxid) - ok = 0; - } else if (le->test->vxid >= 0) { - if (le->test->vxid != vxid) - ok = 0; - } - if (le->test->tag == LE_LAST) { - if (le->tag_last != tag) - ok = 0; - } else if (le->test->tag >= 0) { - if (le->test->tag != tag) - ok = 0; - } - if (le->test->vre && - le->test->tag >= 0 && - le->test->tag == tag && - VRE_ERROR_NOMATCH == VRE_exec(le->test->vre, data, - len, 0, 0, NULL, 0, NULL)) - ok = 0; - - skip = 0; - if (!ok && (le->test->skip_max == LE_ANY || - le->test->skip_max > le->skip_cnt)) - skip = 1; - - if (ok) - legend = "match"; - else if (skip && le->m_arg) - legend = "miss"; - else if (skip) - legend = NULL; - else - legend = "err"; + vxid = VSL_ID(t->c->rec.ptr); + data = VSL_CDATA(t->c->rec.ptr); + len = VSL_LEN(t->c->rec.ptr) - 1; type = VSL_CLIENT(t->c->rec.ptr) ? 'c' : VSL_BACKEND(t->c->rec.ptr) ? 'b' : '-'; - if (legend != NULL) - vtc_log(le->vl, 4, "%-5s| %10u %-15s %c %.*s", - legend, vxid, VSL_tags[tag], type, len, - data); - - if (ok) { - le->vxid_last = vxid; - le->tag_last = tag; - le->skip_cnt = 0; - logexp_next(le); - if (le->test == NULL) - /* End of test script */ - return (1); - } else if (skip) - le->skip_cnt++; - else { - /* Signal fail */ - return (2); - } + r = logexp_match(le, data, vxid, tag, type, len); + if (r) + return (r); } } From nils.goroll at uplex.de Tue Jan 12 22:11:12 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 22:11:12 +0000 (UTC) Subject: [master] b5f9f0b74 vtest: replace le->test with local variable Message-ID: <20210112221112.82E4262E7A@lists.varnish-cache.org> commit b5f9f0b74d771004acf08134f057bcea9aee3805 Author: Nils Goroll Date: Tue Jan 12 20:38:15 2021 +0100 vtest: replace le->test with local variable diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 89b4e6872..a567c8a91 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -268,32 +268,36 @@ static int logexp_match(struct logexp *le, const char *data, int vxid, int tag, int type, int len) { + struct logexp_test *test; const char *legend; int ok = 1, skip = 0; - if (le->test->vxid == LE_LAST) { + test = le->test; + AN(test); + + if (test->vxid == LE_LAST) { if (le->vxid_last != vxid) ok = 0; - } else if (le->test->vxid >= 0) { - if (le->test->vxid != vxid) + } else if (test->vxid >= 0) { + if (test->vxid != vxid) ok = 0; } - if (le->test->tag == LE_LAST) { + if (test->tag == LE_LAST) { if (le->tag_last != tag) ok = 0; - } else if (le->test->tag >= 0) { - if (le->test->tag != tag) + } else if (test->tag >= 0) { + if (test->tag != tag) ok = 0; } - if (le->test->vre && - le->test->tag >= 0 && - le->test->tag == tag && - VRE_ERROR_NOMATCH == VRE_exec(le->test->vre, data, + if (test->vre && + test->tag >= 0 && + test->tag == tag && + VRE_ERROR_NOMATCH == VRE_exec(test->vre, data, len, 0, 0, NULL, 0, NULL)) ok = 0; - if (!ok && (le->test->skip_max == LE_ANY || - le->test->skip_max > le->skip_cnt)) + if (!ok && (test->skip_max == LE_ANY || + test->skip_max > le->skip_cnt)) skip = 1; if (ok) From nils.goroll at uplex.de Tue Jan 12 22:11:12 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 22:11:12 +0000 (UTC) Subject: [master] 5a08d4cf7 vtest: Implement logexpect alternatives Message-ID: <20210112221112.C739462E84@lists.varnish-cache.org> commit 5a08d4cf7881ea29e4723be7fbd48c8bd5cf970b Author: Nils Goroll Date: Tue Jan 12 20:50:36 2021 +0100 vtest: Implement logexpect alternatives This was another great idea by Dridi, thank you! We see cases in VSL where the order of some events can not be (is hard to) fix. For these cases, we add "all of these have to match, but we do not care about the order" semantics through "?" in the skip column of "expect" lines. Example: expect 0 = Debug 1 expect ? = Debug a expect ? = Debug b expect ? = Debug c expect 0 = Debug 2 After a line has matched "1", lines matching a, b or c in any order must follow and then a line matching "2". In the log, an "alt" line is emitted whenever an alternative expect is evaluated. When an alternative matches, matching will continue with the same "expect" as before. Implementation: The skip_max field of (struct logexp_test) was already used for LE_ANY if not a non-negative integer. We add LE_ALT and LE_SEEN. The former denotes an expect with alternatives (that is, the instruction to look for alternatives after this expect) and the latter denotes an LE_ALT that has been matched. Obviously, LE_SEEN is internal and can not result from the logexpect configuration. When a match on an LE_ALT fails, we do not advance le->test, but rather look forward in the expect list for any other altnertives and try them until we either have a match or run out of alternatives. diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index f0d14b086..320bfe599 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -128,6 +128,8 @@ #define LE_ANY (-1) #define LE_LAST (-2) +#define LE_ALT (-3) +#define LE_SEEN (-4) struct logexp_test { unsigned magic; @@ -141,6 +143,8 @@ struct logexp_test { int skip_max; }; +VTAILQ_HEAD(tests_head,logexp_test); + struct logexp { unsigned magic; #define LOGEXP_MAGIC 0xE81D9F1B @@ -150,7 +154,7 @@ struct logexp { char *vname; struct vtclog *vl; char run; - VTAILQ_HEAD(,logexp_test) tests; + struct tests_head tests; struct logexp_test *test; int skip_cnt; @@ -248,19 +252,57 @@ logexp_new(const char *name, const char *varg) return (le); } +static void +logexp_clean(const struct tests_head *head) +{ + struct logexp_test *test; + + VTAILQ_FOREACH(test, head, list) + if (test->skip_max == LE_SEEN) + test->skip_max = LE_ALT; +} + +static struct logexp_test * +logexp_alt(struct logexp_test *test) +{ + assert(test->skip_max == LE_ALT); + + do + test = VTAILQ_NEXT(test, list); + while (test != NULL && test->skip_max == LE_SEEN); + + if (test == NULL || test->skip_max != LE_ALT) + return (NULL); + + return (test); +} + static void logexp_next(struct logexp *le) { CHECK_OBJ_NOTNULL(le, LOGEXP_MAGIC); - if (le->test) { + if (le->test && le->test->skip_max == LE_ALT) { + /* + * if an alternative was not seen, continue at this expection + * with the next vsl + */ + (void)0; + } else if (le->test) { CHECK_OBJ_NOTNULL(le->test, LOGEXP_TEST_MAGIC); le->test = VTAILQ_NEXT(le->test, list); - } else + } else { + logexp_clean(&le->tests); le->test = VTAILQ_FIRST(&le->tests); + } + + if (le->test == NULL) + return; - CHECK_OBJ_ORNULL(le->test, LOGEXP_TEST_MAGIC); - if (le->test) + CHECK_OBJ(le->test, LOGEXP_TEST_MAGIC); + if (le->test->skip_max == LE_SEEN) + logexp_next(le); + else vtc_log(le->vl, 3, "expecting| %s", VSB_data(le->test->str)); } @@ -275,10 +317,11 @@ logexp_match(const struct logexp *le, struct logexp_test *test, const char *data, int vxid, int tag, int type, int len) { const char *legend; - int ok = 1, skip = 0; + int ok = 1, skip = 0, alt = 0; AN(le); AN(test); + assert(test->skip_max != LE_SEEN); if (test->vxid == LE_LAST) { if (le->vxid_last != vxid) @@ -301,7 +344,10 @@ logexp_match(const struct logexp *le, struct logexp_test *test, len, 0, 0, NULL, 0, NULL)) ok = 0; - if (!ok && (test->skip_max == LE_ANY || + if (test->skip_max == LE_ALT) + alt = 1; + + if (!ok && !alt && (test->skip_max == LE_ANY || test->skip_max > le->skip_cnt)) skip = 1; @@ -309,7 +355,7 @@ logexp_match(const struct logexp *le, struct logexp_test *test, legend = "match"; else if (skip && le->m_arg) legend = "miss"; - else if (skip) + else if (skip || alt) legend = NULL; else legend = "err"; @@ -319,8 +365,18 @@ logexp_match(const struct logexp *le, struct logexp_test *test, legend, vxid, VSL_tags[tag], type, len, data); - if (ok) + if (ok) { + if (alt) + test->skip_max = LE_SEEN; return (LEM_OK); + } + if (alt) { + test = logexp_alt(test); + if (test == NULL) + return (LEM_FAIL); + vtc_log(le->vl, 3, "alt | %s", VSB_data(test->str)); + return (logexp_match(le, test, data, vxid, tag, type, len)); + } if (skip) return (LEM_SKIP); return (LEM_FAIL); @@ -485,6 +541,8 @@ cmd_logexp_expect(CMD_ARGS) if (!strcmp(av[1], "*")) skip_max = LE_ANY; + else if (!strcmp(av[1], "?")) + skip_max = LE_ALT; else { skip_max = (int)strtol(av[1], &end, 10); if (*end != '\0' || skip_max < 0) From nils.goroll at uplex.de Tue Jan 12 22:11:12 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 22:11:12 +0000 (UTC) Subject: [master] 91fb76abc vtest: divorce matching from match result handling Message-ID: <20210112221112.AA43362E7F@lists.varnish-cache.org> commit 91fb76abcda1cf6c3795c8cd64e62fef8c670c53 Author: Nils Goroll Date: Tue Jan 12 21:24:37 2021 +0100 vtest: divorce matching from match result handling diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index a567c8a91..f0d14b086 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -264,15 +264,20 @@ logexp_next(struct logexp *le) vtc_log(le->vl, 3, "expecting| %s", VSB_data(le->test->str)); } -static int -logexp_match(struct logexp *le, const char *data, int vxid, int tag, - int type, int len) +enum le_match_e { + LEM_OK, + LEM_SKIP, + LEM_FAIL +}; + +static enum le_match_e +logexp_match(const struct logexp *le, struct logexp_test *test, + const char *data, int vxid, int tag, int type, int len) { - struct logexp_test *test; const char *legend; int ok = 1, skip = 0; - test = le->test; + AN(le); AN(test); if (test->vxid == LE_LAST) { @@ -314,21 +319,11 @@ logexp_match(struct logexp *le, const char *data, int vxid, int tag, legend, vxid, VSL_tags[tag], type, len, data); - if (ok) { - le->vxid_last = vxid; - le->tag_last = tag; - le->skip_cnt = 0; - logexp_next(le); - if (le->test == NULL) - /* End of test script */ - return (1); - } else if (skip) - le->skip_cnt++; - else { - /* Signal fail */ - return (2); - } - return (0); + if (ok) + return (LEM_OK); + if (skip) + return (LEM_SKIP); + return (LEM_FAIL); } static int v_matchproto_(VSLQ_dispatch_f) @@ -337,7 +332,8 @@ logexp_dispatch(struct VSL_data *vsl, struct VSL_transaction * const pt[], { struct logexp *le; struct VSL_transaction *t; - int i, r; + int i; + enum le_match_e r; int vxid, tag, type, len; const char *data; @@ -361,9 +357,22 @@ logexp_dispatch(struct VSL_data *vsl, struct VSL_transaction * const pt[], type = VSL_CLIENT(t->c->rec.ptr) ? 'c' : VSL_BACKEND(t->c->rec.ptr) ? 'b' : '-'; - r = logexp_match(le, data, vxid, tag, type, len); - if (r) + r = logexp_match(le, le->test, + data, vxid, tag, type, len); + if (r == LEM_FAIL) return (r); + if (r == LEM_SKIP) { + le->skip_cnt++; + continue; + } + assert(r == LEM_OK); + le->vxid_last = vxid; + le->tag_last = tag; + le->skip_cnt = 0; + logexp_next(le); + if (le->test == NULL) + /* End of test script */ + return (1); } } From nils.goroll at uplex.de Tue Jan 12 22:11:12 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 22:11:12 +0000 (UTC) Subject: [master] e7449d3b9 Revert "PRIV_TASK vtc: tolerate fini order" Message-ID: <20210112221112.E6C0A62E8A@lists.varnish-cache.org> commit e7449d3b9ea32445bc2d8763533683b6bca77a87 Author: Nils Goroll Date: Tue Jan 12 22:04:35 2021 +0100 Revert "PRIV_TASK vtc: tolerate fini order" These efforts lead nowhere. The problem needed to be solved at the root. This reverts commit 05e7957414467684db375ba6319f79a861a1b742. This reverts commit b02c78438c4e36d967a866466c9ce0cccb32dc0f. diff --git a/bin/varnishtest/tests/r02451.vtc b/bin/varnishtest/tests/r02451.vtc index b25163a65..39cf67aa2 100644 --- a/bin/varnishtest/tests/r02451.vtc +++ b/bin/varnishtest/tests/r02451.vtc @@ -53,10 +53,9 @@ logexpect l1 -v v1 -g raw -d 1 { expect 0 * VCL_Log {^func something to remember} expect 0 * Debug {^objx.priv_task.*"something to remember".$} expect 0 * VCL_Log {^obj something to remember} - # string stored in obj priv_task can already been freed, - # depending on the order - expect 0 * Debug {^priv_task_free|^obj_priv_task_fini} - expect 0 * Debug {^priv_task_free|^obj_priv_task_fini} + expect 0 * Debug {^priv_task_free} + # string stored in obj priv_task has already been freed + expect 0 * Debug {^obj_priv_task_fini} expect 0 * Debug {^vcl1: VCL_EVENT_WARM} } -start diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc index c4ec0123a..55415bd1d 100644 --- a/bin/varnishtest/tests/v00041.vtc +++ b/bin/varnishtest/tests/v00041.vtc @@ -120,9 +120,9 @@ logexpect l0 -v v1 -g raw -d 1 -q "vxid == 0 and not ExpKill" { expect 0 = VCL_Log {^objc initX} expect 0 = Debug {^objb.priv_task.. = .*"initY"} expect 0 = VCL_Log {^objb initY} - expect 0 = Debug {^priv_task_free|^obj_priv_task_fini} - expect 0 = Debug {^priv_task_free|^obj_priv_task_fini} - expect 0 = Debug {^priv_task_free|^obj_priv_task_fini} + expect 0 = Debug {^priv_task_free} + expect 0 = Debug {^obj_priv_task_fini.*"initX"} + expect 0 = Debug {^obj_priv_task_fini.*"initY"} expect 0 = Debug {^vcl1: VCL_EVENT_WARM} # 1001/1002 @@ -132,11 +132,11 @@ logexpect l0 -v v1 -g raw -d 1 -q "vxid == 0 and not ExpKill" { expect 0 = Debug {^test_priv_task.*new.$} expect 0 = Debug {^test_priv_task.*update.$} expect 0 = Debug {^test_priv_task.*exists.$} - expect 0 = Debug {^priv_task_free|^obj_priv_task_fini.*"r1002"} - expect 0 = Debug {^priv_task_free|^obj_priv_task_fini.*"r1002"} + expect 0 = Debug {^priv_task_free} + expect 0 = Debug {^obj_priv_task_fini.*"r1002"} expect 0 = Debug {^test_priv_task.*exists} - expect 0 = Debug {^priv_task_free|^obj_priv_task_fini.*"d1001"} - expect 0 = Debug {^priv_task_free|^obj_priv_task_fini.*"d1001"} + expect 0 = Debug {^priv_task_free} + expect 0 = Debug {^obj_priv_task_fini.*"d1001"} # ... # 1006 pipe From nils.goroll at uplex.de Tue Jan 12 22:11:13 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 22:11:13 +0000 (UTC) Subject: [master] 2e8472921 PRIV_TASK vtc: Use the new alternation (?) feature Message-ID: <20210112221113.12A7662E91@lists.varnish-cache.org> commit 2e847292100e81acb35c83ddb2cc7ba7a93ebe5c Author: Nils Goroll Date: Tue Jan 12 22:16:48 2021 +0100 PRIV_TASK vtc: Use the new alternation (?) feature ... where log order is undeterministic (I mean, not really, but from the perspective of our tests it can not be determined). diff --git a/bin/varnishtest/tests/r02451.vtc b/bin/varnishtest/tests/r02451.vtc index 39cf67aa2..80e197939 100644 --- a/bin/varnishtest/tests/r02451.vtc +++ b/bin/varnishtest/tests/r02451.vtc @@ -53,9 +53,9 @@ logexpect l1 -v v1 -g raw -d 1 { expect 0 * VCL_Log {^func something to remember} expect 0 * Debug {^objx.priv_task.*"something to remember".$} expect 0 * VCL_Log {^obj something to remember} - expect 0 * Debug {^priv_task_free} # string stored in obj priv_task has already been freed - expect 0 * Debug {^obj_priv_task_fini} + expect ? * Debug {^priv_task_free} + expect ? * Debug {^obj_priv_task_fini} expect 0 * Debug {^vcl1: VCL_EVENT_WARM} } -start diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc index 55415bd1d..6d5b6345b 100644 --- a/bin/varnishtest/tests/v00041.vtc +++ b/bin/varnishtest/tests/v00041.vtc @@ -108,7 +108,7 @@ varnish v1 -arg "-p debug=+vclrel -p workspace_client=1m" -vcl+backend { } -start -logexpect l0 -v v1 -g raw -d 1 -q "vxid == 0 and not ExpKill" { +logexpect l0 -v v1 -g raw -d 1 -m -q "vxid == 0 and not ExpKill" { expect 0 0 CLI {^Rd vcl.load} expect 0 = Debug {^test_priv_task.*new.$} expect 0 = Debug {^test_priv_task.*update.$} @@ -120,9 +120,9 @@ logexpect l0 -v v1 -g raw -d 1 -q "vxid == 0 and not ExpKill" { expect 0 = VCL_Log {^objc initX} expect 0 = Debug {^objb.priv_task.. = .*"initY"} expect 0 = VCL_Log {^objb initY} - expect 0 = Debug {^priv_task_free} - expect 0 = Debug {^obj_priv_task_fini.*"initX"} - expect 0 = Debug {^obj_priv_task_fini.*"initY"} + expect ? = Debug {^priv_task_free} + expect ? = Debug {^obj_priv_task_fini.*"initX"} + expect ? = Debug {^obj_priv_task_fini.*"initY"} expect 0 = Debug {^vcl1: VCL_EVENT_WARM} # 1001/1002 @@ -132,11 +132,13 @@ logexpect l0 -v v1 -g raw -d 1 -q "vxid == 0 and not ExpKill" { expect 0 = Debug {^test_priv_task.*new.$} expect 0 = Debug {^test_priv_task.*update.$} expect 0 = Debug {^test_priv_task.*exists.$} - expect 0 = Debug {^priv_task_free} - expect 0 = Debug {^obj_priv_task_fini.*"r1002"} - expect 0 = Debug {^test_priv_task.*exists} - expect 0 = Debug {^priv_task_free} - expect 0 = Debug {^obj_priv_task_fini.*"d1001"} + # client task can race with backend fini, so all of these are + # in any order (? = alternatives) + expect ? = Debug {^priv_task_free} + expect ? = Debug {^obj_priv_task_fini.*"r1002"} + expect ? = Debug {^test_priv_task.*exists} + expect ? = Debug {^priv_task_free} + expect ? = Debug {^obj_priv_task_fini.*"d1001"} # ... # 1006 pipe From nils.goroll at uplex.de Tue Jan 12 22:11:20 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 23:11:20 +0100 Subject: [master] e95760818 vtest: plug insignificant leak In-Reply-To: References: <20210110165608.3C9B9AE938@lists.varnish-cache.org> Message-ID: <44a149da-1256-ce7d-0d41-13be6fbc4aae@uplex.de> On 12/01/2021 20:04, Dridi Boukelmoune wrote: >> commit e95760818552ba954d3750638c0b084ee4879214 >> Author: Nils Goroll >> Date: Sun Jan 10 17:54:07 2021 +0100 >> >> vtest: plug insignificant leak >> >> introduced with 1c19f3fed3da8fae79ef200f2ebf01625a42bb27 >> >> CID 1471724 >> >> Ref #3490 > > This patch fails 100% of our VTCs on macos. is this still the case or does your mutex bandaid apply to this issue? -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From dridi at varni.sh Tue Jan 12 22:19:22 2021 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 12 Jan 2021 22:19:22 +0000 Subject: [master] e95760818 vtest: plug insignificant leak In-Reply-To: <44a149da-1256-ce7d-0d41-13be6fbc4aae@uplex.de> References: <20210110165608.3C9B9AE938@lists.varnish-cache.org> <44a149da-1256-ce7d-0d41-13be6fbc4aae@uplex.de> Message-ID: On Tue, Jan 12, 2021 at 10:12 PM Nils Goroll wrote: > > On 12/01/2021 20:04, Dridi Boukelmoune wrote: > >> commit e95760818552ba954d3750638c0b084ee4879214 > >> Author: Nils Goroll > >> Date: Sun Jan 10 17:54:07 2021 +0100 > >> > >> vtest: plug insignificant leak > >> > >> introduced with 1c19f3fed3da8fae79ef200f2ebf01625a42bb27 > >> > >> CID 1471724 > >> > >> Ref #3490 > > > > This patch fails 100% of our VTCs on macos. > > is this still the case or does your mutex bandaid apply to this issue? It solved the macos failures: https://travis-ci.org/github/varnishcache/varnish-cache/jobs/754189091 Now we need to solve the problem of calling a job process function from the main varnishtest process. We can probably avoid it in the first place. From nils.goroll at uplex.de Tue Jan 12 22:24:13 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 22:24:13 +0000 (UTC) Subject: [master] c79419bf0 vtest: Do not call macro_get() before macro_init() Message-ID: <20210112222413.9338464101@lists.varnish-cache.org> commit c79419bf0279f3e2f7be3389388f1c4fe584fe15 Author: Nils Goroll Date: Tue Jan 12 23:19:26 2021 +0100 vtest: Do not call macro_get() before macro_init() I overlooked use of an uninialized mutex. Fixes https://github.com/vtest/VTest/issues/26 diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index fa7524c2d..31f8cb712 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -606,6 +606,8 @@ ip_magic(void) assert(bad_backend_fd >= 0); VTCP_myname(bad_backend_fd, abuf, sizeof abuf, pbuf, sizeof(pbuf)); extmacro_def("localhost", "%s", abuf); + s = strdup(abuf); + AN(s); #if defined (__APPLE__) /* @@ -623,8 +625,6 @@ ip_magic(void) extmacro_def("bad_backend", "[%s]:%s", abuf, pbuf); /* our default bind/listen address */ - s = macro_get("localhost", NULL); - AN(s); if (VSA_Get_Proto(sa) == AF_INET) bprintf(abuf, "%s:0", s); else From nils.goroll at uplex.de Tue Jan 12 22:24:13 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 22:24:13 +0000 (UTC) Subject: [master] 518a64d7c Revert "varnishtest: Bandaid for an uninitialized mutex" Message-ID: <20210112222413.8B748640E0@lists.varnish-cache.org> commit 518a64d7c08939f58068a79c4a570b0a3a964ea7 Author: Nils Goroll Date: Tue Jan 12 23:17:51 2021 +0100 Revert "varnishtest: Bandaid for an uninitialized mutex" Fix in next commit This reverts commit f855fa46fb1ad66f16e264d57129d605e87f33c4. diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 97e895ca0..017c53950 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -139,12 +139,9 @@ extmacro_def(const char *name, const char *fmt, ...) /********************************************************************** * Below this point is run inside the testing child-process. - * - * XXX: This is no longer the case for macro_get() that is currently - * used in the main process. */ -static pthread_mutex_t macro_mtx = PTHREAD_MUTEX_INITIALIZER; +static pthread_mutex_t macro_mtx; static void init_macro(void) @@ -154,6 +151,8 @@ init_macro(void) /* Dump the extmacros for completeness */ VTAILQ_FOREACH(m, ¯o_list, list) vtc_log(vltop, 4, "extmacro def %s=%s", m->name, m->val); + + AZ(pthread_mutex_init(¯o_mtx, NULL)); } void From nils.goroll at uplex.de Tue Jan 12 22:45:10 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 22:45:10 +0000 (UTC) Subject: [master] 9dcbe09df PRIV_TASK vtc: tolerate even more racing until the fini executes Message-ID: <20210112224510.6691D64CEC@lists.varnish-cache.org> commit 9dcbe09dfe9ae3abcf03adff3d5a59ed048b0aa6 Author: Nils Goroll Date: Tue Jan 12 23:42:44 2021 +0100 PRIV_TASK vtc: tolerate even more racing until the fini executes diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc index 6d5b6345b..388ab5f8c 100644 --- a/bin/varnishtest/tests/v00041.vtc +++ b/bin/varnishtest/tests/v00041.vtc @@ -140,6 +140,14 @@ logexpect l0 -v v1 -g raw -d 1 -m -q "vxid == 0 and not ExpKill" { expect ? = Debug {^priv_task_free} expect ? = Debug {^obj_priv_task_fini.*"d1001"} + # 1003, but need to include in our alt to tolerate race + expect ? = Debug {^test_priv_task.*new.$} + expect ? = Debug {^test_priv_task.*exists.$} + expect ? = Debug {^test_priv_task.*update.$} + expect ? = Debug {^test_priv_task.*new.$} + expect ? = Debug {^test_priv_task.*update.$} + expect ? = Debug {^test_priv_task.*exists.$} + # ... # 1006 pipe expect * = Debug {^obj_priv_task_fini.*"p1006"} From nils.goroll at uplex.de Tue Jan 12 22:52:13 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 12 Jan 2021 23:52:13 +0100 Subject: [master] e95760818 vtest: plug insignificant leak In-Reply-To: References: <20210110165608.3C9B9AE938@lists.varnish-cache.org> <44a149da-1256-ce7d-0d41-13be6fbc4aae@uplex.de> Message-ID: On 12/01/2021 23:19, Dridi Boukelmoune wrote: > > https://travis-ci.org/github/varnishcache/varnish-cache/jobs/754189091 > > Now we need to solve the problem of calling a job process function > from the main varnishtest process. We can probably avoid it in the > first place. yeah, already done -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From dridi.boukelmoune at gmail.com Wed Jan 13 06:26:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 13 Jan 2021 06:26:06 +0000 (UTC) Subject: [master] d895f1f52 vsl: Mask ExpKill logs by default Message-ID: <20210113062607.0617A108B0E@lists.varnish-cache.org> commit d895f1f52b8797a7dfd736af944054ce9aff35da Author: Dridi Boukelmoune Date: Wed Jan 13 07:14:41 2021 +0100 vsl: Mask ExpKill logs by default Based on anecdotal evidence that it is not often used, and the concern of lock contention it might contribute to since such records are emitted unbuffered for every single expiry event, it is now disabled by default. ExpKill records are still available if needed for example for debugging purposes, and are enabled in the 3 test cases where we use them. They were also getting in the way of 1 test. Closes #3475 diff --git a/bin/varnishd/mgt/mgt_param_bits.c b/bin/varnishd/mgt/mgt_param_bits.c index 6a113e72e..756db4159 100644 --- a/bin/varnishd/mgt/mgt_param_bits.c +++ b/bin/varnishd/mgt/mgt_param_bits.c @@ -138,6 +138,7 @@ tweak_vsl_mask(struct vsb *vsb, const struct parspec *par, const char *arg) (void)bit(mgt_param.vsl_mask, SLT_ObjReason, BSET); (void)bit(mgt_param.vsl_mask, SLT_ObjStatus, BSET); (void)bit(mgt_param.vsl_mask, SLT_Debug, BSET); + (void)bit(mgt_param.vsl_mask, SLT_ExpKill, BSET); } else { return (bit_tweak(vsb, mgt_param.vsl_mask, SLT__Reserved, arg, VSL_tags, diff --git a/bin/varnishtest/tests/b00063.vtc b/bin/varnishtest/tests/b00063.vtc index 7a68e2e2f..6879968e8 100644 --- a/bin/varnishtest/tests/b00063.vtc +++ b/bin/varnishtest/tests/b00063.vtc @@ -31,6 +31,7 @@ server s1 { txresp -body "4" } -start +varnish v1 -cliok "param.set vsl_mask +ExpKill" varnish v1 -vcl+backend { sub vcl_backend_response { if (beresp.status >= 500 && bereq.is_bgfetch) { diff --git a/bin/varnishtest/tests/m00031.vtc b/bin/varnishtest/tests/m00031.vtc index 2d40cb781..267f4b882 100644 --- a/bin/varnishtest/tests/m00031.vtc +++ b/bin/varnishtest/tests/m00031.vtc @@ -14,6 +14,7 @@ server s1 { txresp -hdr "Vary: Accept-Language" -body en } -start +varnish v1 -cliok "param.set vsl_mask +ExpKill" varnish v1 -vcl+backend { import purge; diff --git a/bin/varnishtest/tests/r02339.vtc b/bin/varnishtest/tests/r02339.vtc index 178e24a39..a90479a6a 100644 --- a/bin/varnishtest/tests/r02339.vtc +++ b/bin/varnishtest/tests/r02339.vtc @@ -5,7 +5,9 @@ server s1 -repeat 12 { txresp } -start -varnish v1 -arg "-p thread_pools=1" -vcl+backend { +varnish v1 -cliok "param.set thread_pools 1" +varnish v1 -cliok "param.set vsl_mask +ExpKill" +varnish v1 -vcl+backend { import purge; import vtc; diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc index 388ab5f8c..f2b285277 100644 --- a/bin/varnishtest/tests/v00041.vtc +++ b/bin/varnishtest/tests/v00041.vtc @@ -108,7 +108,7 @@ varnish v1 -arg "-p debug=+vclrel -p workspace_client=1m" -vcl+backend { } -start -logexpect l0 -v v1 -g raw -d 1 -m -q "vxid == 0 and not ExpKill" { +logexpect l0 -v v1 -g raw -d 1 -m -q "vxid == 0" { expect 0 0 CLI {^Rd vcl.load} expect 0 = Debug {^test_priv_task.*new.$} expect 0 = Debug {^test_priv_task.*update.$} From dridi.boukelmoune at gmail.com Wed Jan 13 07:03:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 13 Jan 2021 07:03:06 +0000 (UTC) Subject: [master] c6f70f8ca Fix autoconf error in version 2.70 Message-ID: <20210113070306.C87F1109A09@lists.varnish-cache.org> commit c6f70f8ca126b34a218248109b78a540d3a4989a Author: Simon Stridsberg Date: Tue Jan 12 15:34:33 2021 +0000 Fix autoconf error in version 2.70 Fixes #3491 diff --git a/configure.ac b/configure.ac index e44b61fff..3763b3f34 100644 --- a/configure.ac +++ b/configure.ac @@ -40,9 +40,9 @@ AC_PROG_INSTALL AC_ARG_WITH([rst2man], AS_HELP_STRING([--with-rst2man=PATH], [Location of rst2man (auto)]), [RST2MAN="$withval"], - AC_CHECK_PROGS(RST2MAN, + [AC_CHECK_PROGS(RST2MAN, [rst2man-3.6 rst2man-3 rst2man rst2man.py], - [no])) + [no])]) if test "x$RST2MAN" = "xno"; then AC_MSG_ERROR( [rst2man is needed to build Varnish, please install python3-docutils.]) @@ -51,9 +51,9 @@ fi AC_ARG_WITH([sphinx-build], AS_HELP_STRING([--with-sphinx-build=PATH], [Location of sphinx-build (auto)]), [SPHINX="$withval"], - AC_CHECK_PROGS(SPHINX, + [AC_CHECK_PROGS(SPHINX, [sphinx-build-3.6 sphinx-build-3 sphinx-build], - [no])) + [no])]) if test "x$SPHINX" = "xno"; then AC_MSG_ERROR( [sphinx-build is needed to build Varnish, please install python3-sphinx.]) @@ -62,9 +62,9 @@ fi AC_ARG_WITH([rst2html], AS_HELP_STRING([--with-rst2html=PATH], [Location of rst2html (auto)]), [RST2HTML="$withval"], - AC_CHECK_PROGS(RST2HTML, + [AC_CHECK_PROGS(RST2HTML, [rst2html-3.6 rst2html-3 rst2html rst2html.py], - "no")) + "no")]) if test "x$RST2HTML" = "xno"; then AC_MSG_ERROR( From dridi.boukelmoune at gmail.com Wed Jan 13 07:35:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 13 Jan 2021 07:35:07 +0000 (UTC) Subject: [master] 37f1d9241 doc: Briefly mention logexpect's '?' skip marker Message-ID: <20210113073507.34FF710A6B9@lists.varnish-cache.org> commit 37f1d9241d91f7c3279eb5e74067732b75bd6ba1 Author: Dridi Boukelmoune Date: Wed Jan 13 08:27:18 2021 +0100 doc: Briefly mention logexpect's '?' skip marker diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 320bfe599..5d638c74f 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -95,7 +95,7 @@ * * And the arguments of the specifications lines are: * - * skip: [uint|*] + * skip: [uint|*|?] * Max number of record to skip * * vxid: [uint|*|=] @@ -108,7 +108,9 @@ * regular expression to match against (optional) * * For skip, vxid and tag, '*' matches anything, '=' expects the value of the - * previous matched record. + * previous matched record. The '?' marker is equivalent to zero, expecting a + * match on the next record. The difference is that '?' can be used when the + * order of individual consecutive logs is not deterministic. */ #include "config.h" From dridi at varni.sh Wed Jan 13 07:40:29 2021 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 13 Jan 2021 07:40:29 +0000 Subject: [master] b02c78438 PRIV_TASK vtc: tolerate fini order In-Reply-To: <1a4f7a33-bbcc-9cd5-6bc2-09c3bc255ffd@uplex.de> References: <20210112170907.7500C117BD6@lists.varnish-cache.org> <9b56cd50-16f6-86ee-32a9-a57fc2de7e33@uplex.de> <1a4f7a33-bbcc-9cd5-6bc2-09c3bc255ffd@uplex.de> Message-ID: On Tue, Jan 12, 2021 at 6:42 PM Nils Goroll wrote: > > On 12/01/2021 18:41, Nils Goroll wrote: > > On 12/01/2021 18:23, Dridi Boukelmoune wrote: > >> Feature idea, logexpect undefined order: > >> > >> expect ? = Debug ^priv_task_free > >> expect ? = Debug ^obj_priv_task_fini > >> > >> The "?" token would mean that consecutive expect statements using it > >> could appear in a non-deterministic order. > >> > >> Thoughts? > > > > great idea! The end result would be the same as the regex with alternation, but > > it would look _much_ cleaner. > > Actually, no, it would not be the same. Only the proposed solution could ensure > that we see the required number of matches per line. I don't understand that last email, but you ended up implementing the '?' marker so I suppose it's fit for purpose :) From dridi.boukelmoune at gmail.com Wed Jan 13 07:51:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 13 Jan 2021 07:51:07 +0000 (UTC) Subject: [master] 1b7d8f457 doc: Mention the VCL_STRANDS red tape Message-ID: <20210113075107.99B5E10D20D@lists.varnish-cache.org> commit 1b7d8f457b8e7546ebd825268efe39db0ef82efb Author: Dridi Boukelmoune Date: Wed Jan 13 08:50:09 2021 +0100 doc: Mention the VCL_STRANDS red tape diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst index 19f2ac79e..ee9d4f991 100644 --- a/doc/sphinx/reference/vmod.rst +++ b/doc/sphinx/reference/vmod.rst @@ -442,6 +442,9 @@ STRANDS * ``int n``: the number of strings * ``const char **p``: the array of strings with `n` elements + A VMOD should never hold onto strands beyond a function or method + execution. See ``include/vrt.h`` for the details. + TIME C-type: ``double`` From nils.goroll at uplex.de Wed Jan 13 08:48:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 13 Jan 2021 08:48:07 +0000 (UTC) Subject: [master] 3bcc36a1e varnishtest logexp: add to docs Message-ID: <20210113084807.5F88C10E8E1@lists.varnish-cache.org> commit 3bcc36a1eaa4808007cf0d869714371dfbea3895 Author: Nils Goroll Date: Wed Jan 13 09:45:46 2021 +0100 varnishtest logexp: add to docs merge a sentence which I had laying around unpushed. I think it might help explaining what is happening. diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 5d638c74f..8cfa7de29 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -110,7 +110,9 @@ * For skip, vxid and tag, '*' matches anything, '=' expects the value of the * previous matched record. The '?' marker is equivalent to zero, expecting a * match on the next record. The difference is that '?' can be used when the - * order of individual consecutive logs is not deterministic. + * order of individual consecutive logs is not deterministic. In other words, + * lines from a block of alternatives marked by '?' can be matched in any order, + * but all need to match eventually. */ #include "config.h" From nils.goroll at uplex.de Wed Jan 13 09:59:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 13 Jan 2021 09:59:06 +0000 (UTC) Subject: [master] 1952d3c9a vmod_debug: make $Method obj.test_priv_top() per-instance Message-ID: <20210113095906.C93C91104A9@lists.varnish-cache.org> commit 1952d3c9ac2ea411328a0382fe2e1a2426491357 Author: Nils Goroll Date: Wed Jan 13 10:40:02 2021 +0100 vmod_debug: make $Method obj.test_priv_top() per-instance Use a per-instance PRIV_TOP for the oo based test diff --git a/bin/varnishtest/tests/v00043.vtc b/bin/varnishtest/tests/v00043.vtc index 9aaae1aac..8933aa5ba 100644 --- a/bin/varnishtest/tests/v00043.vtc +++ b/bin/varnishtest/tests/v00043.vtc @@ -44,6 +44,7 @@ varnish v1 -cliok "param.set debug +syncvsl" -vcl+backend { sub vcl_recv { set req.http.x0 = debug.test_priv_top(req.url + req.esi_level); + o.test_priv_top(req.url + req.esi_level); } sub vcl_miss { diff --git a/vmod/vmod_debug.vcc b/vmod/vmod_debug.vcc index a787bb2c8..d085d797f 100644 --- a/vmod/vmod_debug.vcc +++ b/vmod/vmod_debug.vcc @@ -116,7 +116,9 @@ $Method STRING .test_priv_task(STRING s="") Test per-object priv_task via VRT_priv_task() / VRT_priv_get() -$Method STRING .test_priv_top(PRIV_TOP, STRING) +$Method STRING .test_priv_top(STRING s="") + +Test per-object priv_top via VRT_priv_top() $Function VOID rot52(HTTP hdr) diff --git a/vmod/vmod_debug_obj.c b/vmod/vmod_debug_obj.c index d6db70ff4..61dc8a454 100644 --- a/vmod/vmod_debug_obj.c +++ b/vmod/vmod_debug_obj.c @@ -211,12 +211,30 @@ xyzzy_obj_test_priv_task(VRT_CTX, struct xyzzy_debug_obj *o, VCL_STRING s) return (p->priv); } +static const struct vmod_priv_methods xyzzy_obj_test_priv_top_methods[1] = {{ + .magic = VMOD_PRIV_METHODS_MAGIC, + .type = "debug_obj_test_priv_top", + .fini = free +}}; + VCL_STRING v_matchproto_() -xyzzy_obj_test_priv_top(VRT_CTX, - struct xyzzy_debug_obj *o, struct vmod_priv *priv, VCL_STRING s) +xyzzy_obj_test_priv_top(VRT_CTX, struct xyzzy_debug_obj *o, VCL_STRING s) { - (void)o; - return (xyzzy_test_priv_top(ctx, priv, s)); + struct vmod_priv *p; + + p = VRT_priv_top(ctx, o); + + if (p == NULL) { + VRT_fail(ctx, "no priv task - out of ws?"); + return (""); + } + + if (p->priv == NULL) { + p->priv = strdup(s); + p->methods = xyzzy_obj_test_priv_top_methods; + } + assert (p->methods == xyzzy_obj_test_priv_top_methods); + return (p->priv); } /* ---------------------------------------------------------------------------- From nils.goroll at uplex.de Wed Jan 13 10:29:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 13 Jan 2021 10:29:06 +0000 (UTC) Subject: [master] 3228cb701 Fix VRT_priv_top() to use the topreq's workspace and change v43.vtc to test the issue Message-ID: <20210113102906.EDC051110ED@lists.varnish-cache.org> commit 3228cb70159a77a677945f4520e9c7b51edb6e95 Author: Nils Goroll Date: Wed Jan 13 11:02:17 2021 +0100 Fix VRT_priv_top() to use the topreq's workspace and change v43.vtc to test the issue This bug triggered when a PRIV_TOP was requested only from esi_level > 0 and used by a "sibling" ESI request, that is, another request on the same ESI level or its descendants. The bug was introduced by someone(tm) in 542bf9b820bbc40c2c95b5fa0282621b997ac594: I removed req = req->top->topreq but continued to use the workspace from req->ws. Fixes #3496 diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index 67d5b5755..aeb7cb87c 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -218,6 +218,8 @@ VRT_priv_top(VRT_CTX, const void *vmod_id) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); top = req->top; CHECK_OBJ_NOTNULL(top, REQTOP_MAGIC); + req = top->topreq; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); Lck_Lock(&sp->mtx); priv = vrt_priv_dynamic(req->ws, top->privs, (uintptr_t)vmod_id); diff --git a/bin/varnishtest/tests/v00043.vtc b/bin/varnishtest/tests/v00043.vtc index 8933aa5ba..949e24bfc 100644 --- a/bin/varnishtest/tests/v00043.vtc +++ b/bin/varnishtest/tests/v00043.vtc @@ -7,15 +7,18 @@ server s1 { expect req.url == "/a" expect req.http.x0 == "/a0" expect req.http.x1 == "/a0" + expect req.http.o1 == txresp -body { - + + } rxreq - expect req.url == "/foo" + expect req.url == "/foo1" expect req.http.x0 == "/a0" expect req.http.x1 == "/a0" + expect req.http.o1 == "/foo11" txresp -body { @@ -25,12 +28,24 @@ server s1 { expect req.url == "/bar" expect req.http.x0 == "/a0" expect req.http.x1 == "/a0" + expect req.http.o1 == "/foo11" txresp + rxreq + expect req.url == "/foo2" + expect req.http.x0 == "/a0" + expect req.http.x1 == "/a0" + expect req.http.o1 == "/foo11" + txresp -body { + + + } + rxreq expect req.url == "/b" expect req.http.x0 == "/b0" expect req.http.x1 == "/b0" + expect req.http.o1 == txresp } -start @@ -40,15 +55,23 @@ varnish v1 -cliok "param.set debug +syncvsl" -vcl+backend { sub vcl_init { new o = debug.obj(); + new o2 = debug.obj(); } sub vcl_recv { set req.http.x0 = debug.test_priv_top(req.url + req.esi_level); - o.test_priv_top(req.url + req.esi_level); + if (req.url == "/foo1") { + o.test_priv_top(req.url + req.esi_level); + } else { + o2.test_priv_top(req.url + req.esi_level); + } } sub vcl_miss { set req.http.x1 = debug.test_priv_top(""); + if (req.esi_level > 0) { + set req.http.o1 = o.test_priv_top(""); + } } sub vcl_backend_response { @@ -58,7 +81,6 @@ varnish v1 -cliok "param.set debug +syncvsl" -vcl+backend { sub vcl_deliver { set resp.http.x1 = debug.test_priv_top(""); - set resp.http.o1 = o.test_priv_top(""); } } -start @@ -67,12 +89,10 @@ client c1 { txreq -url /a rxresp expect resp.http.x1 == "/a0" - expect resp.http.o1 == "/a0" txreq -url /b rxresp expect resp.http.x1 == "/b0" - expect resp.http.o1 == "/b0" } -run varnish v1 -expect client_req == 2 From nils.goroll at uplex.de Wed Jan 13 15:35:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 13 Jan 2021 15:35:07 +0000 (UTC) Subject: [master] 104bd724b VRT_priv_top() code to macro for reuse in next commit Message-ID: <20210113153507.EA1CF118632@lists.varnish-cache.org> commit 104bd724b61d2e95f110a481cc4dda6e2d5caba4 Author: Nils Goroll Date: Wed Jan 13 13:08:49 2021 +0100 VRT_priv_top() code to macro for reuse in next commit diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index aeb7cb87c..c27174450 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -199,6 +199,23 @@ VRT_priv_task(VRT_CTX, const void *vmod_id) (uintptr_t)vmod_id)); } +#define VRT_PRIV_TOP_PREP(ctx, req, sp, top) do { \ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ + req = (ctx)->req; \ + if (req == NULL) { \ + WRONG("PRIV_TOP is only accessible " \ + "in client VCL context"); \ + NEEDLESS(return (NULL)); \ + } \ + CHECK_OBJ(req, REQ_MAGIC); \ + sp = (ctx)->sp; \ + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); \ + top = (req)->top; \ + CHECK_OBJ_NOTNULL(top, REQTOP_MAGIC); \ + req = (top)->topreq; \ + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); \ + } while(0) + struct vmod_priv * VRT_priv_top(VRT_CTX, const void *vmod_id) { @@ -207,19 +224,7 @@ VRT_priv_top(VRT_CTX, const void *vmod_id) struct reqtop *top; struct vmod_priv *priv; - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - req = ctx->req; - if (req == NULL) { - WRONG("PRIV_TOP is only accessible in client VCL context"); - NEEDLESS(return (NULL)); - } - CHECK_OBJ(req, REQ_MAGIC); - sp = ctx->sp; - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - top = req->top; - CHECK_OBJ_NOTNULL(top, REQTOP_MAGIC); - req = top->topreq; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + VRT_PRIV_TOP_PREP(ctx, req, sp, top); Lck_Lock(&sp->mtx); priv = vrt_priv_dynamic(req->ws, top->privs, (uintptr_t)vmod_id); From nils.goroll at uplex.de Wed Jan 13 15:35:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 13 Jan 2021 15:35:08 +0000 (UTC) Subject: [master] 18b75e959 Add VRT_priv_top_get() to parallel VRT_priv_task_get() Message-ID: <20210113153508.18EDA118635@lists.varnish-cache.org> commit 18b75e9597e61229df752d9da9c24fe55ec0d266 Author: Nils Goroll Date: Wed Jan 13 13:27:03 2021 +0100 Add VRT_priv_top_get() to parallel VRT_priv_task_get() Ref 06cf60a020a81e49d001ee42e4d85f16580f3fe0 diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index c27174450..d6884c228 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -216,6 +216,22 @@ VRT_priv_task(VRT_CTX, const void *vmod_id) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); \ } while(0) +struct vmod_priv * +VRT_priv_top_get(VRT_CTX, const void *vmod_id) +{ + struct req *req; + struct sess *sp; + struct reqtop *top; + struct vmod_priv *priv; + + VRT_PRIV_TOP_PREP(ctx, req, sp, top); + + Lck_Lock(&sp->mtx); + priv = vrt_priv_dynamic_get(top->privs, (uintptr_t)vmod_id); + Lck_Unlock(&sp->mtx); + return (priv); +} + struct vmod_priv * VRT_priv_top(VRT_CTX, const void *vmod_id) { diff --git a/include/vrt.h b/include/vrt.h index 23de2361b..12cab679d 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -60,6 +60,7 @@ * VRT_CTX_Assert() added * VRT_ban_string() signature changed * VRT_priv_task_get() added + * VRT_priv_top_get() added * 12.0 (2020-09-15) * Added VRT_DirectorResolve() * Added VCL_STRING VRT_BLOB_string(VRT_CTX, VCL_BLOB) @@ -600,6 +601,7 @@ void VRT_priv_fini(const struct vmod_priv *p); struct vmod_priv *VRT_priv_task(VRT_CTX, const void *vmod_id); struct vmod_priv *VRT_priv_task_get(VRT_CTX, const void *vmod_id); struct vmod_priv *VRT_priv_top(VRT_CTX, const void *vmod_id); +struct vmod_priv *VRT_priv_top_get(VRT_CTX, const void *vmod_id); /* Stevedore related functions */ int VRT_Stv(const char *nm); From nils.goroll at uplex.de Wed Jan 13 15:35:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 13 Jan 2021 15:35:08 +0000 (UTC) Subject: [master] 6b3e26c7d vmod_debug: obj.test_priv_top(): use VRT_priv_top_get() and workspace Message-ID: <20210113153508.4B334118639@lists.varnish-cache.org> commit 6b3e26c7d42c204fae134a1b6dc73c9ac885ed3b Author: Nils Goroll Date: Wed Jan 13 15:58:24 2021 +0100 vmod_debug: obj.test_priv_top(): use VRT_priv_top_get() and workspace Adjust our PRIV_TOP test to use VRT_priv_top_get() for read operation and copy the saved value onto the top request's workspace if need to (workspaces differ and the string to be saved is on the current request's workspace). diff --git a/bin/varnishtest/tests/v00043.vtc b/bin/varnishtest/tests/v00043.vtc index 949e24bfc..8421d890d 100644 --- a/bin/varnishtest/tests/v00043.vtc +++ b/bin/varnishtest/tests/v00043.vtc @@ -8,6 +8,7 @@ server s1 { expect req.http.x0 == "/a0" expect req.http.x1 == "/a0" expect req.http.o1 == + expect req.http.o2 == "/a0" txresp -body { @@ -19,6 +20,7 @@ server s1 { expect req.http.x0 == "/a0" expect req.http.x1 == "/a0" expect req.http.o1 == "/foo11" + expect req.http.o2 == "/a0" txresp -body { @@ -29,6 +31,7 @@ server s1 { expect req.http.x0 == "/a0" expect req.http.x1 == "/a0" expect req.http.o1 == "/foo11" + expect req.http.o2 == "/a0" txresp rxreq @@ -36,6 +39,7 @@ server s1 { expect req.http.x0 == "/a0" expect req.http.x1 == "/a0" expect req.http.o1 == "/foo11" + expect req.http.o2 == "/a0" txresp -body { @@ -46,6 +50,7 @@ server s1 { expect req.http.x0 == "/b0" expect req.http.x1 == "/b0" expect req.http.o1 == + expect req.http.o2 == "/b0" txresp } -start @@ -62,16 +67,16 @@ varnish v1 -cliok "param.set debug +syncvsl" -vcl+backend { set req.http.x0 = debug.test_priv_top(req.url + req.esi_level); if (req.url == "/foo1") { o.test_priv_top(req.url + req.esi_level); - } else { + } + if (req.esi_level == 0) { o2.test_priv_top(req.url + req.esi_level); } } sub vcl_miss { set req.http.x1 = debug.test_priv_top(""); - if (req.esi_level > 0) { - set req.http.o1 = o.test_priv_top(""); - } + set req.http.o1 = o.test_priv_top(""); + set req.http.o2 = o2.test_priv_top(""); } sub vcl_backend_response { diff --git a/vmod/vmod_debug_obj.c b/vmod/vmod_debug_obj.c index 61dc8a454..b7cb49eef 100644 --- a/vmod/vmod_debug_obj.c +++ b/vmod/vmod_debug_obj.c @@ -211,29 +211,80 @@ xyzzy_obj_test_priv_task(VRT_CTX, struct xyzzy_debug_obj *o, VCL_STRING s) return (p->priv); } +static void +obj_priv_top_fini(void *ptr) +{ + AN(ptr); + VSL(SLT_Debug, 0, "obj_priv_top_fini(%p = \"%s\")", ptr, (char *)ptr); +} + static const struct vmod_priv_methods xyzzy_obj_test_priv_top_methods[1] = {{ .magic = VMOD_PRIV_METHODS_MAGIC, .type = "debug_obj_test_priv_top", - .fini = free + .fini = obj_priv_top_fini }}; VCL_STRING v_matchproto_() xyzzy_obj_test_priv_top(VRT_CTX, struct xyzzy_debug_obj *o, VCL_STRING s) { struct vmod_priv *p; + struct req *req; + struct ws *ws; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + + req = ctx->req; + if (req == NULL) { + VRT_fail(ctx, "%s.priv_top() can only be used " + "in client VCL context", o->vcl_name); + return (""); + } + CHECK_OBJ(req, REQ_MAGIC); + + if (s == NULL || *s == '\0') { + p = VRT_priv_top_get(ctx, o); + if (p == NULL) { + VSLb(ctx->vsl, SLT_Debug, "%s.priv_top() = NULL", + o->vcl_name); + return (""); + } + assert(p->methods == xyzzy_obj_test_priv_top_methods); + VSLb(ctx->vsl, SLT_Debug, + "%s.priv_top() = %p .priv = %p (\"%s\")", + o->vcl_name, p, p->priv, p->priv); + return (p->priv); + } p = VRT_priv_top(ctx, o); + if (p == NULL) + VSLb(ctx->vsl, SLT_Debug, "%s.priv_top() = NULL [err]", + o->vcl_name); - if (p == NULL) { - VRT_fail(ctx, "no priv task - out of ws?"); + CHECK_OBJ_NOTNULL(req->top, REQTOP_MAGIC); + req = req->top->topreq; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + ws = req->ws; + + /* copy to top req's workspace if need to */ + if (ctx->ws != ws && WS_Inside(ctx->ws, s, NULL)) + s = WS_Copy(ws, s, -1); + + if (p == NULL || s == NULL) { + VRT_fail(ctx, "out of ws?"); return (""); } - if (p->priv == NULL) { - p->priv = strdup(s); + VSLb(ctx->vsl, SLT_Debug, + "%s.priv_top() = %p .priv = %p (\"%s\") [%s]", + o->vcl_name, p, s, s, p->priv ? "update" : "new"); + + if (p->priv == NULL) p->methods = xyzzy_obj_test_priv_top_methods; - } - assert (p->methods == xyzzy_obj_test_priv_top_methods); + else + assert(p->methods == xyzzy_obj_test_priv_top_methods); + + p->priv = TRUST_ME(s); + return (p->priv); } From nils.goroll at uplex.de Wed Jan 13 15:35:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 13 Jan 2021 15:35:08 +0000 (UTC) Subject: [master] 1deae5f0d vmod_debug: obj.test_priv_task() more assertions Message-ID: <20210113153508.6535F11863C@lists.varnish-cache.org> commit 1deae5f0d04b9a0ac40b475dbd3a2132f536b6d8 Author: Nils Goroll Date: Wed Jan 13 16:05:25 2021 +0100 vmod_debug: obj.test_priv_task() more assertions diff --git a/vmod/vmod_debug_obj.c b/vmod/vmod_debug_obj.c index b7cb49eef..803e9146c 100644 --- a/vmod/vmod_debug_obj.c +++ b/vmod/vmod_debug_obj.c @@ -185,6 +185,7 @@ xyzzy_obj_test_priv_task(VRT_CTX, struct xyzzy_debug_obj *o, VCL_STRING s) o->vcl_name); return (""); } + assert(p->methods == xyzzy_obj_test_priv_task_methods); mylog(vsl, SLT_Debug, "%s.priv_task() = %p .priv = %p (\"%s\")", o->vcl_name, p, p->priv, p->priv); @@ -204,9 +205,13 @@ xyzzy_obj_test_priv_task(VRT_CTX, struct xyzzy_debug_obj *o, VCL_STRING s) "%s.priv_task() = %p .priv = %p (\"%s\") [%s]", o->vcl_name, p, s, s, p->priv ? "update" : "new"); + if (p->priv == NULL) + p->methods = xyzzy_obj_test_priv_task_methods; + else + assert(p->methods == xyzzy_obj_test_priv_task_methods); + /* minimum scope of s and priv is the task - no need to copy */ p->priv = TRUST_ME(s); - p->methods = xyzzy_obj_test_priv_task_methods; return (p->priv); } From nils.goroll at uplex.de Wed Jan 13 16:19:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 13 Jan 2021 16:19:07 +0000 (UTC) Subject: [master] 06a2e92e7 fix v00043.vtc Message-ID: <20210113161907.56BE74E40@lists.varnish-cache.org> commit 06a2e92e74764e05acd7fa77556755a1a24e952c Author: Nils Goroll Date: Wed Jan 13 17:15:16 2021 +0100 fix v00043.vtc I do not understand how this has slipped through. Ref 6b3e26c7d42c204fae134a1b6dc73c9ac885ed3b diff --git a/bin/varnishtest/tests/v00043.vtc b/bin/varnishtest/tests/v00043.vtc index 8421d890d..828b06668 100644 --- a/bin/varnishtest/tests/v00043.vtc +++ b/bin/varnishtest/tests/v00043.vtc @@ -7,7 +7,7 @@ server s1 { expect req.url == "/a" expect req.http.x0 == "/a0" expect req.http.x1 == "/a0" - expect req.http.o1 == + expect req.http.o1 == "" expect req.http.o2 == "/a0" txresp -body { @@ -49,7 +49,7 @@ server s1 { expect req.url == "/b" expect req.http.x0 == "/b0" expect req.http.x1 == "/b0" - expect req.http.o1 == + expect req.http.o1 == "" expect req.http.o2 == "/b0" txresp From nils.goroll at uplex.de Wed Jan 13 16:45:09 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 13 Jan 2021 16:45:09 +0000 (UTC) Subject: [master] 4e6abff8d missing cast Message-ID: <20210113164509.247AF5CE6@lists.varnish-cache.org> commit 4e6abff8df324c0c5428b1821a1caca7674a0aff Author: Nils Goroll Date: Wed Jan 13 17:43:48 2021 +0100 missing cast Ref: 6b3e26c7d42c204fae134a1b6dc73c9ac885ed3b diff --git a/vmod/vmod_debug_obj.c b/vmod/vmod_debug_obj.c index 803e9146c..097a74f99 100644 --- a/vmod/vmod_debug_obj.c +++ b/vmod/vmod_debug_obj.c @@ -256,7 +256,7 @@ xyzzy_obj_test_priv_top(VRT_CTX, struct xyzzy_debug_obj *o, VCL_STRING s) assert(p->methods == xyzzy_obj_test_priv_top_methods); VSLb(ctx->vsl, SLT_Debug, "%s.priv_top() = %p .priv = %p (\"%s\")", - o->vcl_name, p, p->priv, p->priv); + o->vcl_name, p, p->priv, (char *)p->priv); return (p->priv); } From nils.goroll at uplex.de Wed Jan 13 18:27:09 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 13 Jan 2021 18:27:09 +0000 (UTC) Subject: [master] 4e441176c Improve vmod developer docs for private pointers Message-ID: <20210113182709.9ADE79B94@lists.varnish-cache.org> commit 4e441176c6d27974ae21d8c41633c77b88b76a86 Author: Nils Goroll Date: Wed Jan 13 19:25:11 2021 +0100 Improve vmod developer docs for private pointers See #3498 for an unresolved issue noticed on the way. diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst index ee9d4f991..5e65edf16 100644 --- a/doc/sphinx/reference/vmod.rst +++ b/doc/sphinx/reference/vmod.rst @@ -492,6 +492,8 @@ The VCL compiler supports the following private pointers: These private pointers live only for the duration of their top level request + .. XXX BROKEN https://github.com/varnishcache/varnish-cache/issues/3498 + * ``PRIV_VCL`` "per vcl" private pointers are useful for such global state that applies to all calls in this VCL, for instance flags that determine if regular expressions are case-sensitive in this vmod or @@ -499,6 +501,8 @@ The VCL compiler supports the following private pointers: to the VMOD's event function. This private pointer lives for the duration of the loaded VCL. + The ``PRIV_CALL`` vmod_privs are finalized before ``PRIV_VCL``. + The way it works in the vmod code, is that a ``struct vmod_priv *`` is passed to the functions where one of the ``PRIV_*`` argument types is specified. @@ -532,7 +536,7 @@ help debugging. vmod_priv`` when the scope ends with that ``.priv`` pointer as its only argument. -In the common case where a private data structure is allocated with +The common case where a private data structure is allocated with malloc(3) would look like this:: static const struct vmod_priv_methods mymethods[1] = {{ @@ -557,21 +561,88 @@ malloc(3) would look like this:: ... } -Note on use with objects: +Private Pointers Memory Management +---------------------------------- + +The generic malloc(3) / free(3) approach documented above works for +all private pointers. It is the simplest and less error prone (as long +as allocated memory is properly freed though the fini callback), but +comes at the cost of calling into the heap memory allocator. + +Per-vmod constant data structures can be assigned to any private +pointer type, but, obviously, free(3) must not be used on them. + +Dynamic data stored in ``PRIV_TASK`` and ``PRIV_TOP`` pointers can +also come from the workspace: + +* For ``PRIV_TASK``, any allocation from ``ctx->ws`` works, like so:: + + if (priv->priv == NULL) { + priv->priv = WS_Alloc(ctx->ws, sizeof(struct myfoo)); + if (priv->priv == NULL) { + VRT_fail(ctx, "WS_Alloc failed"); + return (...); + } + priv->methods = mymethods; + mystate = priv->priv; + mystate->foo = 21; + ... + +* For ``PRIV_TOP``, first of all keep in mind that it must only be + used from the client context, so vmod code should error out for + ``ctx->req == NULL``. + + For dynamic data, the *top request's* workspace must be used, which + complicates things a bit:: + + if (priv->priv == NULL) { + struct ws *ws; + + CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(ctx->req->top, REQTOP_MAGIC); + CHECK_OBJ_NOTNULL(ctx->req->top->topreq, REQ_MAGIC); + ws = ctx->req->top->topreq->ws; + + priv->priv = WS_Alloc(ws, sizeof(struct myfoo)); + // ... same as above for PRIV_TASK + +Notice that allocations on the workspace do not need to be freed, +their lifetime is the respective task. -The per-call vmod_privs are freed before the per-vcl vmod_priv. +Private Pointers and Objects +---------------------------- -``PRIV_TASK`` and ``PRIV_TOP`` arguments are not per object instance, -but still per vmod as for ordinary vmod functions. Thus, vmods +``PRIV_TASK`` and ``PRIV_TOP`` arguments to methods are not per object +instance, but per vmod as for ordinary vmod functions. Thus, vmods requiring per-task / per top-request state for object instances need to implement other means to associate storage with object instances. -Using ``VRT_priv_task()`` to maintain per object instance state is a -convenient yet unofficial interface which was not originally intended -for this purpose and will likely be replaced with a more suitable -interface. +This is what ``VRT_priv_task()`` / ``VRT_priv_task_get()`` and +``VRT_priv_top()`` / ``VRT_priv_top_get()`` are for: + +The non-get functions either return an existing ``PRIV_TASK`` / +``PRIV_TOP`` for a given ``void *`` argument or create one. They +return ``NULL`` in case of an allocation failure. + +The ``_get()`` functions do not create a ``PRIV_*``, but return either +an existing one or ``NULL``. + +By convention, private pointers for object instance are created on the +address of the object, as in this example for a ``PRIV_TASK``:: + + VCL_VOID + myvmod_obj_method(VRT_CTX, struct myvmod_obj *o) + { + struct vmod_priv *p; + + p = VRT_priv_task(ctx, o); + + // ... see above -.. XXX add VRT_priv_task_get() and make it official? +The ``PRIV_TOP`` case looks identical except for calling +``VRT_priv_top(ctx, o)`` in place of ``VRT_priv_task(ctx, o)``, but be +reminded that the ``VRT_priv_top*()`` functions must only be called +from client context (if ``ctx->req != NULL``). .. _ref-vmod-event-functions: From nils.goroll at uplex.de Thu Jan 14 12:55:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 14 Jan 2021 12:55:08 +0000 (UTC) Subject: [master] 2116e4dab varnishtest logexpect: add code for negative matching conditions Message-ID: <20210114125508.E53331153D3@lists.varnish-cache.org> commit 2116e4dab816e4edc92cc39983da54c902f92ac6 Author: Nils Goroll Date: Thu Jan 14 10:53:13 2021 +0100 varnishtest logexpect: add code for negative matching conditions We add a list of expects which, if matched, will trigger failure. When we encounter an expect with skip_max == LE_FAIL, we put it onto the "fail list". For skip_max == LE_CLEAR, we clear that list. For every VSL line, all expects on the fail list are tested, if any. When the list of (positive) expects has been completed, logexpect continues testing the fail list until the transaction is complete. This obviously does not work for raw grouping, so in this case the test author needs to ensure that the fail list be empty when the logexpect is expected to complete. Ref #3249 diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 8cfa7de29..50b61a828 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -130,15 +130,18 @@ #include "vtim.h" #include "vre.h" -#define LE_ANY (-1) -#define LE_LAST (-2) -#define LE_ALT (-3) -#define LE_SEEN (-4) +#define LE_ANY (-1) +#define LE_LAST (-2) +#define LE_ALT (-3) +#define LE_SEEN (-4) +#define LE_FAIL (-5) +#define LE_CLEAR (-6) // clear fail list struct logexp_test { unsigned magic; #define LOGEXP_TEST_MAGIC 0x6F62B350 VTAILQ_ENTRY(logexp_test) list; + VTAILQ_ENTRY(logexp_test) faillist; struct vsb *str; int vxid; @@ -165,6 +168,8 @@ struct logexp { int vxid_last; int tag_last; + struct tests_head fail; + int m_arg; int d_arg; enum VSL_grouping_e g_arg; @@ -297,6 +302,7 @@ logexp_next(struct logexp *le) le->test = VTAILQ_NEXT(le->test, list); } else { logexp_clean(&le->tests); + VTAILQ_INIT(&le->fail); le->test = VTAILQ_FIRST(&le->tests); } @@ -304,10 +310,24 @@ logexp_next(struct logexp *le) return; CHECK_OBJ(le->test, LOGEXP_TEST_MAGIC); - if (le->test->skip_max == LE_SEEN) + + switch (le->test->skip_max) { + case LE_SEEN: logexp_next(le); - else + return; + case LE_CLEAR: + vtc_log(le->vl, 3, "condition| fail clear"); + VTAILQ_INIT(&le->fail); + logexp_next(le); + return; + case LE_FAIL: + vtc_log(le->vl, 3, "condition| %s", VSB_data(le->test->str)); + VTAILQ_INSERT_TAIL(&le->fail, le->test, faillist); + logexp_next(le); + return; + default: vtc_log(le->vl, 3, "expecting| %s", VSB_data(le->test->str)); + } } enum le_match_e { @@ -321,11 +341,12 @@ logexp_match(const struct logexp *le, struct logexp_test *test, const char *data, int vxid, int tag, int type, int len) { const char *legend; - int ok = 1, skip = 0, alt = 0; + int ok = 1, skip = 0, alt, fail; AN(le); AN(test); assert(test->skip_max != LE_SEEN); + assert(test->skip_max != LE_CLEAR); if (test->vxid == LE_LAST) { if (le->vxid_last != vxid) @@ -348,14 +369,22 @@ logexp_match(const struct logexp *le, struct logexp_test *test, len, 0, 0, NULL, 0, NULL)) ok = 0; - if (test->skip_max == LE_ALT) - alt = 1; + alt = (test->skip_max == LE_ALT); + fail = (test->skip_max == LE_FAIL); if (!ok && !alt && (test->skip_max == LE_ANY || test->skip_max > le->skip_cnt)) skip = 1; - if (ok) + if (fail) { + if (ok) + legend = "fail"; + else if (le->m_arg) + legend = "fmiss"; + else + legend = NULL; + } + else if (ok) legend = "match"; else if (skip && le->m_arg) legend = "miss"; @@ -386,6 +415,31 @@ logexp_match(const struct logexp *le, struct logexp_test *test, return (LEM_FAIL); } +static enum le_match_e +logexp_failchk(const struct logexp *le, + const char *data, int vxid, int tag, int type, int len) +{ + struct logexp_test *test; + static enum le_match_e r; + + if (VTAILQ_FIRST(&le->fail) == NULL) + return (LEM_SKIP); + + VTAILQ_FOREACH(test, &le->fail, faillist) { + r = logexp_match(le, test, data, vxid, tag, type, len); + if (r == LEM_OK) + return (LEM_FAIL); + assert (r == LEM_FAIL); + } + return (LEM_OK); +} + +static int +logexp_done(const struct logexp *le) +{ + return ((VTAILQ_FIRST(&le->fail) == NULL) && le->test == NULL); +} + static int v_matchproto_(VSLQ_dispatch_f) logexp_dispatch(struct VSL_data *vsl, struct VSL_transaction * const pt[], void *priv) @@ -404,7 +458,6 @@ logexp_dispatch(struct VSL_data *vsl, struct VSL_transaction * const pt[], if (!VSL_Match(vsl, t->c)) continue; - CHECK_OBJ_NOTNULL(le->test, LOGEXP_TEST_MAGIC); AN(t->c->rec.ptr); tag = VSL_TAG(t->c->rec.ptr); @@ -417,6 +470,16 @@ logexp_dispatch(struct VSL_data *vsl, struct VSL_transaction * const pt[], type = VSL_CLIENT(t->c->rec.ptr) ? 'c' : VSL_BACKEND(t->c->rec.ptr) ? 'b' : '-'; + r = logexp_failchk(le, data, vxid, tag, type, len); + if (r == LEM_FAIL) + return (r); + if (le->test == NULL) { + assert (r == LEM_OK); + continue; + } + + CHECK_OBJ_NOTNULL(le->test, LOGEXP_TEST_MAGIC); + r = logexp_match(le, le->test, data, vxid, tag, type, len); if (r == LEM_FAIL) @@ -430,11 +493,13 @@ logexp_dispatch(struct VSL_data *vsl, struct VSL_transaction * const pt[], le->tag_last = tag; le->skip_cnt = 0; logexp_next(le); - if (le->test == NULL) - /* End of test script */ + if (logexp_done(le)) return (1); } } + // transaction end + if (le->g_arg != VSL_g_raw) + VTAILQ_INIT(&le->fail); return (0); } @@ -455,13 +520,13 @@ logexp_thread(void *priv) if (le->query != NULL) vtc_log(le->vl, 4, "qry| %s", le->query); logexp_next(le); - while (le->test) { + while (! logexp_done(le)) { i = VSLQ_Dispatch(le->vslq, logexp_dispatch, le); if (i == 2) vtc_fatal(le->vl, "bad| expectation failed"); else if (i < 0) vtc_fatal(le->vl, "bad| dispatch failed (%d)", i); - else if (i == 0 && le->test) + else if (i == 0 && ! logexp_done(le)) VTIM_sleep(0.01); } vtc_log(le->vl, 4, "end|"); From nils.goroll at uplex.de Thu Jan 14 12:55:09 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 14 Jan 2021 12:55:09 +0000 (UTC) Subject: [master] d8ca2e995 varnishtest logexpect refactor: pull out common code from cmd_logexp for reuse Message-ID: <20210114125509.073111153D6@lists.varnish-cache.org> commit d8ca2e99517952d34aef8142f2d0af053175329f Author: Nils Goroll Date: Thu Jan 14 11:20:45 2021 +0100 varnishtest logexpect refactor: pull out common code from cmd_logexp for reuse Ref #3249 diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 50b61a828..34997954b 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -588,11 +588,12 @@ logexp_wait(struct logexp *le) le->run = 0; } +/* shared by expect and fail: parse from av[2] (vxid) onwards */ + static void -cmd_logexp_expect(CMD_ARGS) +cmd_logexp_common(struct logexp *le, struct vtclog *vl, + int skip_max, char * const *av) { - struct logexp *le; - int skip_max; int vxid; int tag; vre_t *vre; @@ -601,22 +602,6 @@ cmd_logexp_expect(CMD_ARGS) struct logexp_test *test; char *end; - CAST_OBJ_NOTNULL(le, priv, LOGEXP_MAGIC); - if (av[1] == NULL || av[2] == NULL || av[3] == NULL) - vtc_fatal(vl, "Syntax error"); - - if (av[4] != NULL && av[5] != NULL) - vtc_fatal(vl, "Syntax error"); - - if (!strcmp(av[1], "*")) - skip_max = LE_ANY; - else if (!strcmp(av[1], "?")) - skip_max = LE_ALT; - else { - skip_max = (int)strtol(av[1], &end, 10); - if (*end != '\0' || skip_max < 0) - vtc_fatal(vl, "Not a positive integer: '%s'", av[1]); - } if (!strcmp(av[2], "*")) vxid = LE_ANY; else if (!strcmp(av[2], "=")) @@ -658,6 +643,32 @@ cmd_logexp_expect(CMD_ARGS) VTAILQ_INSERT_TAIL(&le->tests, test, list); } +static void +cmd_logexp_expect(CMD_ARGS) +{ + struct logexp *le; + int skip_max; + char *end; + + CAST_OBJ_NOTNULL(le, priv, LOGEXP_MAGIC); + if (av[1] == NULL || av[2] == NULL || av[3] == NULL) + vtc_fatal(vl, "Syntax error"); + + if (av[4] != NULL && av[5] != NULL) + vtc_fatal(vl, "Syntax error"); + + if (!strcmp(av[1], "*")) + skip_max = LE_ANY; + else if (!strcmp(av[1], "?")) + skip_max = LE_ALT; + else { + skip_max = (int)strtol(av[1], &end, 10); + if (*end != '\0' || skip_max < 0) + vtc_fatal(vl, "Not a positive integer: '%s'", av[1]); + } + cmd_logexp_common(le, vl, skip_max, av); +} + static void logexp_spec(struct logexp *le, const char *spec) { From nils.goroll at uplex.de Thu Jan 14 12:55:09 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 14 Jan 2021 12:55:09 +0000 (UTC) Subject: [master] 40055cccb varnishtest logexpect: add fail command Message-ID: <20210114125509.25F581153DA@lists.varnish-cache.org> commit 40055cccba00cb766c3ce6f2db00eb63204d66e9 Author: Nils Goroll Date: Thu Jan 14 11:22:18 2021 +0100 varnishtest logexpect: add fail command This is the VTC interface for the "fail list" facility added in a previous commit. Commands added: * fail add Add en entry to the fail list at this point in the test. From now on, logexpect failure is triggered if this entry matches * fail clear Clear the fail list Closes #3249 diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 34997954b..2c97b6b9e 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -43,6 +43,8 @@ * [vsl arguments] { * expect * expect + * fail add + * fail clear * ... * } [-start|-wait] * @@ -93,7 +95,7 @@ * \-T * Transaction end timeout * - * And the arguments of the specifications lines are: + * expect specification: * * skip: [uint|*|?] * Max number of record to skip @@ -113,6 +115,26 @@ * order of individual consecutive logs is not deterministic. In other words, * lines from a block of alternatives marked by '?' can be matched in any order, * but all need to match eventually. + * + * fail specification: + * + * add: Add to the fail list + * + * Arguments are equivalent to expect, except for skip missing + * + * clear: Clear the fail list + * + * Any number of fail specifications can be active during execution of + * a logexpect. All active fail specifications are matched against every + * log line and, if any match, the logexpect fails immediately. + * + * For transactional vsls (-g ), a fail list can be used + * without limitation: When the transaction ends, the logexpect ends + * successfully if no specification from the fail list matched. + * + * For raw mode (-g raw), however, the log never ends, so for a logexpect to + * finish successfully, a "fail clear" is required after some match which + * determines that no further negative matching is required. */ #include "config.h" @@ -185,9 +207,11 @@ static VTAILQ_HEAD(, logexp) logexps = VTAILQ_HEAD_INITIALIZER(logexps); static cmd_f cmd_logexp_expect; +static cmd_f cmd_logexp_fail; static const struct cmds logexp_cmds[] = { { "expect", cmd_logexp_expect }, + { "fail", cmd_logexp_fail }, { NULL, NULL }, }; @@ -669,6 +693,40 @@ cmd_logexp_expect(CMD_ARGS) cmd_logexp_common(le, vl, skip_max, av); } +static void +cmd_logexp_fail(CMD_ARGS) +{ + struct logexp *le; + struct logexp_test *test; + + CAST_OBJ_NOTNULL(le, priv, LOGEXP_MAGIC); + + if (av[1] == NULL) + vtc_fatal(vl, "Syntax error"); + + if (!strcmp(av[1], "clear")) { + ALLOC_OBJ(test, LOGEXP_TEST_MAGIC); + AN(test); + test->skip_max = LE_CLEAR; + test->str = VSB_new_auto(); + AN(test->str); + AZ(VSB_printf(test->str, "%s %s", + av[0], av[1])); + AZ(VSB_finish(test->str)); + + VTAILQ_INSERT_TAIL(&le->tests, test, list); + return; + } + + if (strcmp(av[1], "add")) + vtc_fatal(vl, "Unknown fail argument '%s'", av[1]); + + if (av[2] == NULL || av[3] == NULL) + vtc_fatal(vl, "Syntax error"); + + cmd_logexp_common(le, vl, LE_FAIL, av); +} + static void logexp_spec(struct logexp *le, const char *spec) { From nils.goroll at uplex.de Thu Jan 14 12:55:09 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 14 Jan 2021 12:55:09 +0000 (UTC) Subject: [master] 341c5b04d varnishtest logexpect: add -err flag to expect failure Message-ID: <20210114125509.3F0431153E0@lists.varnish-cache.org> commit 341c5b04d0cec6341b4edeb7ee6380079c259c8a Author: Nils Goroll Date: Thu Jan 14 12:28:13 2021 +0100 varnishtest logexpect: add -err flag to expect failure Ref #3249 diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 2c97b6b9e..9ffd4a92b 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -75,6 +75,10 @@ * \-m * Also emit log records for misses (only for debugging) * + * \-err + * Invert the meaning of success. Usually called once to expect the + * logexpect to fail + * * \-start * Start the logexpect thread in the background. * @@ -193,6 +197,7 @@ struct logexp { struct tests_head fail; int m_arg; + int err_arg; int d_arg; enum VSL_grouping_e g_arg; char *query; @@ -546,6 +551,10 @@ logexp_thread(void *priv) logexp_next(le); while (! logexp_done(le)) { i = VSLQ_Dispatch(le->vslq, logexp_dispatch, le); + if (i == 2 && le->err_arg) { + vtc_log(le->vl, 4, "end| failed as expected"); + return (NULL); + } if (i == 2) vtc_fatal(le->vl, "bad| expectation failed"); else if (i < 0) @@ -829,6 +838,10 @@ cmd_logexpect(CMD_ARGS) le->m_arg = !le->m_arg; continue; } + if (!strcmp(*av, "-err")) { + le->err_arg = !le->err_arg; + continue; + } if (!strcmp(*av, "-start")) { logexp_start(le); continue; From nils.goroll at uplex.de Thu Jan 14 12:55:09 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 14 Jan 2021 12:55:09 +0000 (UTC) Subject: [master] e412dbe36 varnishtest logexpect: test the tester Message-ID: <20210114125509.5DB231153EA@lists.varnish-cache.org> commit e412dbe366558885f56d79d97a14354e60ecec69 Author: Nils Goroll Date: Thu Jan 14 13:21:46 2021 +0100 varnishtest logexpect: test the tester Ref #3249 diff --git a/bin/varnishtest/tests/b00074.vtc b/bin/varnishtest/tests/b00074.vtc new file mode 100644 index 000000000..ebacf9260 --- /dev/null +++ b/bin/varnishtest/tests/b00074.vtc @@ -0,0 +1,67 @@ +varnishtest "Test logexpect fail command" + +# NOTE: this is a test of varnishtest itself, so it would fall under +# the "a" category, but has been filed under the "b" category because +# it needs varnishd + +varnish v1 -vcl { + import std; + + backend proforma None; + + sub vcl_init { + std.log("i0"); + std.log("i1"); + std.log("i2"); + std.log("i3"); + } + + sub vcl_recv { + std.log("r0"); + std.log("r1"); + std.log("r2"); + std.log("r3"); + } +} -start + +logexpect l1 -v v1 -g vxid -q "vxid == 1001" { + fail add * Error "out of workspace" + fail add * VCL_Error "Workspace overflow" +} -start + +logexpect l2 -v v1 -err -g vxid -q "vxid == 1001" { + fail add * VCL_Log ^r2 +} -start + +logexpect l3 -v v1 -g vxid -q "vxid == 1001" { + fail add * VCL_Log ^r2 + expect * 1001 VCL_Log ^r0 + expect 0 = VCL_Log ^r1 + fail clear + expect 1 = VCL_Log ^r3 +} -start + +client c1 { + txreq + rxresp +} -run + +# -g raw needs an explicit expect and "fail clear" at the end +# because the raw transaction never ends +logexpect l4 -v v1 -d 1 -g raw { + fail add * VCL_Log ^i2 + expect * 0 VCL_Log ^i0 + expect 0 = VCL_Log ^i1 + fail clear + expect 1 = VCL_Log ^i3 + fail add * Error "out of workspace" + fail add * VCL_Error "Workspace overflow" + expect * 1000 End + fail clear +} -start + + +logexpect l1 -wait +logexpect l2 -wait +logexpect l3 -wait +logexpect l4 -wait From nils.goroll at uplex.de Thu Jan 14 15:24:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 14 Jan 2021 15:24:07 +0000 (UTC) Subject: [master] b62d92ebf varnishtest logexpect: remove implicit fail list clear Message-ID: <20210114152407.6AC20119258@lists.varnish-cache.org> commit b62d92ebf282c3cd1dd1d6d929d28d90c78552b1 Author: Nils Goroll Date: Thu Jan 14 16:22:00 2021 +0100 varnishtest logexpect: remove implicit fail list clear Dridi noticed that my reasoning about this was flawed: One transaction ending does not imply no other transactions coming. diff --git a/bin/varnishtest/tests/b00074.vtc b/bin/varnishtest/tests/b00074.vtc index ebacf9260..8a0d0f0ae 100644 --- a/bin/varnishtest/tests/b00074.vtc +++ b/bin/varnishtest/tests/b00074.vtc @@ -27,6 +27,8 @@ varnish v1 -vcl { logexpect l1 -v v1 -g vxid -q "vxid == 1001" { fail add * Error "out of workspace" fail add * VCL_Error "Workspace overflow" + expect * 1001 End + fail clear } -start logexpect l2 -v v1 -err -g vxid -q "vxid == 1001" { @@ -46,8 +48,6 @@ client c1 { rxresp } -run -# -g raw needs an explicit expect and "fail clear" at the end -# because the raw transaction never ends logexpect l4 -v v1 -d 1 -g raw { fail add * VCL_Log ^i2 expect * 0 VCL_Log ^i0 diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 9ffd4a92b..d1af7d733 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -132,13 +132,13 @@ * a logexpect. All active fail specifications are matched against every * log line and, if any match, the logexpect fails immediately. * - * For transactional vsls (-g ), a fail list can be used - * without limitation: When the transaction ends, the logexpect ends - * successfully if no specification from the fail list matched. + * For a logexpect to end successfully, there must be no specs on the fail list, + * so logexpects should always end with * - * For raw mode (-g raw), however, the log never ends, so for a logexpect to - * finish successfully, a "fail clear" is required after some match which - * determines that no further negative matching is required. + * expect + * fail clear + * + * XXX can we come up with a better solution which is still safe? */ #include "config.h" @@ -526,10 +526,6 @@ logexp_dispatch(struct VSL_data *vsl, struct VSL_transaction * const pt[], return (1); } } - // transaction end - if (le->g_arg != VSL_g_raw) - VTAILQ_INIT(&le->fail); - return (0); } From dridi.boukelmoune at gmail.com Fri Jan 15 15:04:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 15 Jan 2021 15:04:07 +0000 (UTC) Subject: [master] c2d45aefb Try to send VDP_END with the last bytes Message-ID: <20210115150407.412A61179E3@lists.varnish-cache.org> commit c2d45aefb563bb105f524b67b84d20649ecb85a2 Author: Nils Goroll Date: Mon Apr 27 12:55:14 2020 +0200 Try to send VDP_END with the last bytes Follow-up to #3125 : That PR added VDP_END, but always issued another VDP_bytes() call with no data. We now make it the responsibility of the object iterator to send VDP_END and opportunistically try to issue it with the last bytes in order to avoid the additional VDP_bytes() call with no data. As noted by Martin in https://github.com/varnishcache/varnish-cache/pull/3125#issuecomment-600620905 the advantage is that "in H/2 there is the need to send a frame with a last bit set to mark the end of the stream" and in H/1 we could generate the end chunk right after the last bit of data. This change does not implement these optimizations yet, but prepares the grounds. diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 7dbe03a0a..d4a074bdc 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -217,8 +217,16 @@ VDP_Close(struct vdp_ctx *vdc) static int v_matchproto_(objiterate_f) vdp_objiterator(void *priv, unsigned flush, const void *ptr, ssize_t len) { + enum vdp_action act; - return (VDP_bytes(priv, flush ? VDP_FLUSH : VDP_NULL, ptr, len)); + if (flush == 0) + act = VDP_NULL; + else if ((flush & OBJ_ITER_END) != 0) + act = VDP_END; + else + act = VDP_FLUSH; + + return (VDP_bytes(priv, act, ptr, len)); } @@ -234,8 +242,6 @@ VDP_DeliverObj(struct vdp_ctx *vdc, struct objcore *oc) vdc->req = NULL; final = oc->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP) ? 1 : 0; r = ObjIterate(vdc->wrk, oc, vdc, vdp_objiterator, final); - if (r == 0) - r = VDP_bytes(vdc, VDP_END, NULL, 0); if (r < 0) return (r); return (0); diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index a1a5afe04..bf75ef217 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -315,6 +315,9 @@ ved_vdp_esi_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, struct ecx *ecx; int retval = 0; + if (act == VDP_END) + act = VDP_FLUSH; + AN(priv); CHECK_OBJ_NOTNULL(vdx, VDP_CTX_MAGIC); CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC); @@ -432,7 +435,6 @@ ved_vdp_esi_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, return (retval); case 3: case 4: - assert(act != VDP_END); /* * There is no guarantee that the 'l' bytes are all * in the same storage segment, so loop over storage diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c index ccba6b525..6bfb1f8f6 100644 --- a/bin/varnishd/storage/storage_simple.c +++ b/bin/varnishd/storage/storage_simple.c @@ -278,6 +278,7 @@ sml_iterator(struct worker *wrk, struct objcore *oc, p = NULL; l = 0; + u = 0; while (1) { ol = len; nl = ObjWaitExtend(wrk, oc, ol); @@ -333,11 +334,15 @@ sml_iterator(struct worker *wrk, struct objcore *oc, u = 0; if (st == NULL || final) u |= OBJ_ITER_FLUSH; + if (st == NULL && boc->state == BOS_FINISHED) + u |= OBJ_ITER_END; ret = func(priv, u, p, l); if (ret) break; } HSH_DerefBoc(wrk, oc); + if ((u & OBJ_ITER_END) == 0) + func(priv, OBJ_ITER_END, NULL, 0); return (ret); } From dridi.boukelmoune at gmail.com Fri Jan 15 15:04:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 15 Jan 2021 15:04:07 +0000 (UTC) Subject: [master] 2065315af Fix a potential race between boc state and the storage list Message-ID: <20210115150407.5652C1179E6@lists.varnish-cache.org> commit 2065315afcf7b92dd2e113636c853f52ca12b4f5 Author: Nils Goroll Date: Sun May 24 22:00:24 2020 +0200 Fix a potential race between boc state and the storage list We check for the next storage element under the boc mutex, but reference the boc state in relation to it after releasing the lock. Take a copy of the boc state to close this (potential) race. Fixes #3335, hopefully diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c index 6bfb1f8f6..bc13f443c 100644 --- a/bin/varnishd/storage/storage_simple.c +++ b/bin/varnishd/storage/storage_simple.c @@ -236,6 +236,7 @@ sml_iterator(struct worker *wrk, struct objcore *oc, void *priv, objiterate_f *func, int final) { struct boc *boc; + enum boc_state_e state; struct object *obj; struct storage *st; struct storage *checkpoint = NULL; @@ -329,12 +330,13 @@ sml_iterator(struct worker *wrk, struct objcore *oc, st = VTAILQ_NEXT(st, list); if (st != NULL && st->len == 0) st = NULL; + state = boc->state; Lck_Unlock(&boc->mtx); - assert(l > 0 || boc->state == BOS_FINISHED); + assert(l > 0 || state == BOS_FINISHED); u = 0; if (st == NULL || final) u |= OBJ_ITER_FLUSH; - if (st == NULL && boc->state == BOS_FINISHED) + if (st == NULL && state == BOS_FINISHED) u |= OBJ_ITER_END; ret = func(priv, u, p, l); if (ret) From dridi.boukelmoune at gmail.com Fri Jan 15 16:17:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 15 Jan 2021 16:17:07 +0000 (UTC) Subject: [master] 20c820e8e vcc: New REGEX type with expressions Message-ID: <20210115161707.30CAC1198A1@lists.varnish-cache.org> commit 20c820e8e59278b8b5a7f4776994a1422f3fdb90 Author: Dridi Boukelmoune Date: Fri Jan 8 12:34:41 2021 +0100 vcc: New REGEX type with expressions By introducing a proper REGEX type in VCL we can centralize the regular expressions literals parsing. The type itself is currently not very potent since VMODs aren't aware of it, but as a first step it allows error messages to be improved at compile time. diff --git a/bin/varnishtest/tests/r00409.vtc b/bin/varnishtest/tests/r00409.vtc index 1ef578932..6a9db628e 100644 --- a/bin/varnishtest/tests/r00409.vtc +++ b/bin/varnishtest/tests/r00409.vtc @@ -1,10 +1,28 @@ varnishtest "Regression test for ticket 409" -varnish v1 -errvcl {Expected CSTR got '!'} { - backend b { .host = "${localhost}"; } +varnish v1 -errvcl {Unknown token '!' when looking for REGEX} { + backend be none; sub vcl_recv { if ( req.url ~ ! "\.(png|jpg|gif|js|css)$" ) { return (pass); } } } + +varnish v1 -errvcl {Expression has type STRING, expected REGEX} { + backend be none; + sub vcl_recv { + set req.http.regex = "\.(png|jpg|gif|js|css)$"; + if (req.url ~ req.http.regex) { + return (pass); + } + } +} + +varnish v1 -errvcl {Expression has type STRING, expected REGEX} { + backend be none; + sub vcl_recv { + set req.http.regex = "\?.*"; + set req.url = regsub(req.url, req.http.regex, ""); + } +} diff --git a/include/vrt.h b/include/vrt.h index 12cab679d..c88d8cb9e 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -273,6 +273,7 @@ typedef int64_t VCL_INT; typedef const struct suckaddr * VCL_IP; typedef const struct vrt_backend_probe * VCL_PROBE; typedef double VCL_REAL; +typedef const struct vre * VCL_REGEX; typedef const struct stevedore * VCL_STEVEDORE; typedef const struct strands * VCL_STRANDS; typedef const char * VCL_STRING; diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 376e8f614..1d766ba52 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -771,6 +771,10 @@ vcc_expr5(struct vcc *tl, struct expr **e, vcc_type_t fmt) ERRCHK(tl); e1 = vcc_mk_expr(IP, "%s", ip); ERRCHK(tl); + } else if (fmt == REGEX) { + e1 = vcc_new_expr(REGEX); + vcc_regexp(tl, e1->vsb); + AZ(VSB_finish(e1->vsb)); } else { e1 = vcc_new_expr(STRINGS); EncToken(e1->vsb, tl->t); @@ -1075,20 +1079,15 @@ cmp_simple(struct vcc *tl, struct expr **e, const struct cmps *cp) static void v_matchproto_(cmp_f) cmp_regexp(struct vcc *tl, struct expr **e, const struct cmps *cp) { + struct expr *e2; char buf[128]; - struct vsb vsb; *e = vcc_expr_edit(tl, STRING, "\vS", *e, NULL); vcc_NextToken(tl); - ExpectErr(tl, CSTR); - AN(VSB_init(&vsb, buf, sizeof buf)); - VSB_printf(&vsb, "%sVRT_re_match(ctx, \v1, ", cp->emit); - vcc_regexp(tl, &vsb); + vcc_expr0(tl, &e2, REGEX); ERRCHK(tl); - VSB_cat(&vsb, ")"); - AZ(VSB_finish(&vsb)); - *e = vcc_expr_edit(tl, BOOL, VSB_data(&vsb), *e, NULL); - VSB_fini(&vsb); + bprintf(buf, "%sVRT_re_match(ctx, \v1, \v2)", cp->emit); + *e = vcc_expr_edit(tl, BOOL, buf, *e, e2); } static void v_matchproto_(cmp_f) @@ -1342,6 +1341,8 @@ vcc_expr0(struct vcc *tl, struct expr **e, vcc_type_t fmt) t1 = tl->t; if (fmt->stringform) vcc_expr_cor(tl, e, STRINGS); + else if (fmt == REGEX) + vcc_expr4(tl, e, REGEX); else vcc_expr_cor(tl, e, fmt); ERRCHK(tl); @@ -1457,10 +1458,9 @@ static void v_matchproto_(sym_expr_t) vcc_Eval_Regsub(struct vcc *tl, struct expr **e, struct token *t, struct symbol *sym, vcc_type_t fmt) { - struct expr *e2; + struct expr *e2, *e3; int all = sym->eval_priv == NULL ? 0 : 1; char buf[128]; - struct vsb vsb; (void)t; (void)fmt; @@ -1468,15 +1468,11 @@ vcc_Eval_Regsub(struct vcc *tl, struct expr **e, struct token *t, vcc_expr0(tl, &e2, STRING); ERRCHK(tl); SkipToken(tl, ','); - ExpectErr(tl, CSTR); - - AN(VSB_init(&vsb, buf, sizeof buf)); - VSB_printf(&vsb, "VRT_regsub(ctx, %d,\v+\n\v1,\n", all); - vcc_regexp(tl, &vsb); + vcc_expr0(tl, &e3, REGEX); ERRCHK(tl); - AZ(VSB_finish(&vsb)); - *e = vcc_expr_edit(tl, STRING, VSB_data(&vsb), e2, NULL); - VSB_fini(&vsb); + + bprintf(buf, "VRT_regsub(ctx, %d,\v+\n\v1,\n\v2", all); + *e = vcc_expr_edit(tl, STRING, buf, e2, e3); SkipToken(tl, ','); vcc_expr0(tl, &e2, STRING); ERRCHK(tl); diff --git a/lib/libvcc/vcc_types.c b/lib/libvcc/vcc_types.c index 576e55cee..7fcf5078c 100644 --- a/lib/libvcc/vcc_types.c +++ b/lib/libvcc/vcc_types.c @@ -151,6 +151,11 @@ const struct type REAL[1] = {{ .multype = REAL, }}; +const struct type REGEX[1] = {{ + .magic = TYPE_MAGIC, + .name = "REGEX", +}}; + static const struct vcc_method stevedore_methods[] = { #define VRTSTVVAR(nm, vtype, ctype, dval) \ { VCC_METHOD_MAGIC, vtype, #nm, "VRT_stevedore_" #nm "(\v1)", 0}, diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c index ce149d8c5..419df3fcb 100644 --- a/lib/libvcc/vcc_utils.c +++ b/lib/libvcc/vcc_utils.c @@ -57,9 +57,8 @@ vcc_regexp(struct vcc *tl, struct vsb *vgc_name) int erroroffset; struct inifin *ifp; - Expect(tl, CSTR); - if (tl->err) - return; + assert(tl->t->tok == CSTR); + t = VRE_compile(tl->t->dec, 0, &error, &erroroffset); if (t == NULL) { VSB_printf(tl->sb, @@ -78,7 +77,6 @@ vcc_regexp(struct vcc *tl, struct vsb *vgc_name) EncToken(ifp->ini, tl->t); VSB_cat(ifp->ini, ");"); VSB_printf(ifp->fin, "\t\tVRT_re_fini(%s);", buf); - vcc_NextToken(tl); } /* From dridi.boukelmoune at gmail.com Fri Jan 15 16:17:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 15 Jan 2021 16:17:07 +0000 (UTC) Subject: [master] 5391143e1 vrt: Replace void pointers with the regex types Message-ID: <20210115161707.4BF051198A4@lists.varnish-cache.org> commit 5391143e1aa5246ab58a6390a9c68b151a05abc1 Author: Dridi Boukelmoune Date: Tue Jan 12 17:06:32 2021 +0100 vrt: Replace void pointers with the regex types Now that we have a proper type for regular expressions, use it in the VRT API where it makes sense. The VRT_re_{init,fini}() functions were moved to VPI since they are here strictly to support libvcc. Summary: in VRT we use VCL_REGEX but in VPI we use struct vre. diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c index f38515151..13135655c 100644 --- a/bin/varnishd/cache/cache_vrt_re.c +++ b/bin/varnishd/cache/cache_vrt_re.c @@ -36,6 +36,7 @@ #include #include "cache_varnishd.h" +#include "vcc_interface.h" static void Tadd(char **b, char *e, const char *p, int l) @@ -52,7 +53,7 @@ Tadd(char **b, char *e, const char *p, int l) } void -VRT_re_init(void **rep, const char *re) +VPI_re_init(vre_t **rep, const char *re) { vre_t *t; const char *error; @@ -65,7 +66,7 @@ VRT_re_init(void **rep, const char *re) } void -VRT_re_fini(void *rep) +VPI_re_fini(vre_t *rep) { vre_t *vv; @@ -74,18 +75,16 @@ VRT_re_fini(void *rep) VRE_free(&vv); } -int -VRT_re_match(VRT_CTX, const char *s, void *re) +VCL_BOOL +VRT_re_match(VRT_CTX, const char *s, VCL_REGEX re) { - vre_t *t; int i; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if (s == NULL) s = ""; AN(re); - t = re; - i = VRE_exec(t, s, strlen(s), 0, 0, NULL, 0, &cache_param->vre_limits); + i = VRE_exec(re, s, strlen(s), 0, 0, NULL, 0, &cache_param->vre_limits); if (i >= 0) return (1); if (i < VRE_ERROR_NOMATCH ) @@ -93,12 +92,11 @@ VRT_re_match(VRT_CTX, const char *s, void *re) return (0); } -const char * -VRT_regsub(VRT_CTX, int all, const char *str, void *re, - const char *sub) +VCL_STRING +VRT_regsub(VRT_CTX, int all, VCL_STRING str, VCL_REGEX re, + VCL_STRING sub) { int ovector[30]; - vre_t *t; int i, l; char *res_b; char *res_e; @@ -115,10 +113,9 @@ VRT_regsub(VRT_CTX, int all, const char *str, void *re, str = ""; if (sub == NULL) sub = ""; - t = re; memset(ovector, 0, sizeof(ovector)); len = strlen(str); - i = VRE_exec(t, str, len, 0, options, ovector, 30, + i = VRE_exec(re, str, len, 0, options, ovector, 30, &cache_param->vre_limits); /* If it didn't match, we can return the original string */ @@ -158,7 +155,7 @@ VRT_regsub(VRT_CTX, int all, const char *str, void *re, break; memset(ovector, 0, sizeof(ovector)); options |= VRE_NOTEMPTY; - i = VRE_exec(t, str, len, offset, options, ovector, 30, + i = VRE_exec(re, str, len, offset, options, ovector, 30, &cache_param->vre_limits); if (i < VRE_ERROR_NOMATCH ) { WS_Release(ctx->ws, 0); diff --git a/include/vcc_interface.h b/include/vcc_interface.h index c754af479..7303a0544 100644 --- a/include/vcc_interface.h +++ b/include/vcc_interface.h @@ -71,3 +71,8 @@ struct vpi_ii { const void * p; const char * const name; }; + +/* Compile time regexp */ + +void VPI_re_init(struct vre **, const char *); +void VPI_re_fini(struct vre *); diff --git a/include/vrt.h b/include/vrt.h index c88d8cb9e..70fb7e36b 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -61,6 +61,10 @@ * VRT_ban_string() signature changed * VRT_priv_task_get() added * VRT_priv_top_get() added + * VRT_re_init removed + * VRT_re_fini removed + * VRT_re_match signature changed + * VRT_regsub signature changed * 12.0 (2020-09-15) * Added VRT_DirectorResolve() * Added VCL_STRING VRT_BLOB_string(VRT_CTX, VCL_BLOB) @@ -458,14 +462,6 @@ VCL_BACKEND VRT_DirectorResolve(VRT_CTX, VCL_BACKEND); void VRT_acl_log(VRT_CTX, const char *); int VRT_acl_match(VRT_CTX, VCL_ACL, VCL_IP); -/*********************************************************************** - * Compile time regexp - */ - -void VRT_re_init(void **, const char *); -void VRT_re_fini(void *); -int VRT_re_match(VRT_CTX, const char *, void *); - /*********************************************************************** * Getting hold of the various struct http */ @@ -495,7 +491,8 @@ VCL_BYTES VRT_CacheReqBody(VRT_CTX, VCL_BYTES maxsize); /* Regexp related */ -const char *VRT_regsub(VRT_CTX, int all, const char *, void *, const char *); +VCL_BOOL VRT_re_match(VRT_CTX, VCL_STRING, VCL_REGEX); +VCL_STRING VRT_regsub(VRT_CTX, int all, VCL_STRING, VCL_REGEX, VCL_STRING); VCL_STRING VRT_ban_string(VRT_CTX, VCL_STRING); VCL_INT VRT_purge(VRT_CTX, VCL_DURATION, VCL_DURATION, VCL_DURATION); VCL_VOID VRT_synth(VRT_CTX, VCL_INT, VCL_STRING); diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c index 419df3fcb..9cc27c68b 100644 --- a/lib/libvcc/vcc_utils.c +++ b/lib/libvcc/vcc_utils.c @@ -71,12 +71,12 @@ vcc_regexp(struct vcc *tl, struct vsb *vgc_name) if (vgc_name) VSB_cat(vgc_name, buf); - Fh(tl, 0, "static void *%s;\n", buf); + Fh(tl, 0, "static struct vre *%s;\n", buf); ifp = New_IniFin(tl); - VSB_printf(ifp->ini, "\tVRT_re_init(&%s, ",buf); + VSB_printf(ifp->ini, "\tVPI_re_init(&%s, ",buf); EncToken(ifp->ini, tl->t); VSB_cat(ifp->ini, ");"); - VSB_printf(ifp->fin, "\t\tVRT_re_fini(%s);", buf); + VSB_printf(ifp->fin, "\t\tVPI_re_fini(%s);", buf); } /* From dridi.boukelmoune at gmail.com Fri Jan 15 16:17:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 15 Jan 2021 16:17:07 +0000 (UTC) Subject: [master] 2b1320487 vmod: Teach VMODs to use REGEX expressions Message-ID: <20210115161707.70FAF1198A8@lists.varnish-cache.org> commit 2b13204874467d88e4638b6fcf2a452a34ad3a3f Author: Dridi Boukelmoune Date: Fri Jan 8 12:49:30 2021 +0100 vmod: Teach VMODs to use REGEX expressions When they take a regular expression as an argument, they can benefit from compile time checks and avoid PRIV_CALL or on-the-fly regex compilation. diff --git a/bin/varnishtest/tests/c00103.vtc b/bin/varnishtest/tests/c00103.vtc new file mode 100644 index 000000000..11da0f0b9 --- /dev/null +++ b/bin/varnishtest/tests/c00103.vtc @@ -0,0 +1,32 @@ +varnishtest "REGEX expressions in VCL" + +varnish v1 -vcl { + import debug; + backend be none; + sub vcl_recv { + # NB: the REGEX expression below is needlessly complicated + # on purpose to ensure we don't treat REGEX expressions + # differently. + if (req.url ~ (debug.just_return_regex( + debug.just_return_regex("hello")))) { + return (synth(200)); + } + return (synth(500)); + } + sub vcl_synth { + set resp.reason = regsub(resp.reason, + debug.just_return_regex("OK"), "world"); + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 500 + expect resp.reason == "Internal Server Error" + + txreq -url "/hello" + rxresp + expect resp.status == 200 + expect resp.reason == world +} -run diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst index 5e65edf16..08a791fd8 100644 --- a/doc/sphinx/reference/vmod.rst +++ b/doc/sphinx/reference/vmod.rst @@ -381,6 +381,11 @@ REAL A floating point value. +REGEX + C-type: ``const struct vre *`` + + This is an opaque type for regular expressions with a VCL scope. + STRING C-type: ``const char *`` diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 25d3f977e..8bc43a807 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -110,6 +110,7 @@ CTYPES = { 'IP': "VCL_IP", 'PROBE': "VCL_PROBE", 'REAL': "VCL_REAL", + 'REGEX': "VCL_REGEX", 'STEVEDORE': "VCL_STEVEDORE", 'STRANDS': "VCL_STRANDS", 'STRING': "VCL_STRING", diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index 2be114a4f..997dfe33a 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -1278,3 +1278,12 @@ xyzzy_validhdr(VRT_CTX, VCL_STRANDS s) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); return (VRT_ValidHdr(ctx, s)); } + +VCL_REGEX v_matchproto_(td_xyzzy_regex) +xyzzy_just_return_regex(VRT_CTX, VCL_REGEX r) +{ + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(r); + return (r); +} diff --git a/vmod/vmod_debug.vcc b/vmod/vmod_debug.vcc index d085d797f..3f2e13bcc 100644 --- a/vmod/vmod_debug.vcc +++ b/vmod/vmod_debug.vcc @@ -319,3 +319,7 @@ A function mixing a named PRIV_TASK with optional parameters. $Function BOOL validhdr(STRANDS) Test if the argument is a valid header according to RFC7230 section 3.2. + +$Function REGEX just_return_regex(REGEX) + +Take a REGEX argument and return it. From dridi.boukelmoune at gmail.com Fri Jan 15 16:17:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 15 Jan 2021 16:17:07 +0000 (UTC) Subject: [master] 259e2b8fc vmod_cookie: Use the new REGEX type Message-ID: <20210115161707.916FA1198AC@lists.varnish-cache.org> commit 259e2b8fc4e958f0f5aecb382d3291e5e0780d88 Author: Dridi Boukelmoune Date: Tue Jan 12 17:42:24 2021 +0100 vmod_cookie: Use the new REGEX type And get a glimpse at how we can simplify VMODs making use of regular expressions without breaking existing VCL. diff --git a/vmod/vmod_cookie.c b/vmod/vmod_cookie.c index fdaa2f594..129d68bdc 100644 --- a/vmod/vmod_cookie.c +++ b/vmod/vmod_cookie.c @@ -42,7 +42,6 @@ #include #include -#include #include "vcc_cookie_if.h" @@ -53,8 +52,6 @@ enum filter_action { whitelist }; -static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; - struct cookie { unsigned magic; #define VMOD_COOKIE_ENTRY_MAGIC 0x3BB41543 @@ -237,76 +234,23 @@ vmod_get(VRT_CTX, struct vmod_priv *priv, VCL_STRING name) } -static vre_t * -compile_re(VRT_CTX, VCL_STRING expression) -{ - vre_t *vre; - const char *error; - int erroroffset; - - vre = VRE_compile(expression, 0, &error, &erroroffset); - if (vre == NULL) { - VSLb(ctx->vsl, SLT_Error, - "cookie: PCRE compile error at char %i: %s", - erroroffset, error); - } - return (vre); -} - -static void -free_re(void *priv) -{ - vre_t *vre; - - AN(priv); - vre = priv; - VRE_free(&vre); - AZ(vre); -} - -static const struct vmod_priv_methods cookie_re_priv_methods[1] = {{ - .magic = VMOD_PRIV_METHODS_MAGIC, - .type = "vmod_cookie_re", - .fini = free_re -}}; - VCL_STRING -vmod_get_re(VRT_CTX, struct vmod_priv *priv, struct vmod_priv *priv_call, - VCL_STRING expression) +vmod_get_re(VRT_CTX, struct vmod_priv *priv, VCL_REGEX re) { struct vmod_cookie *vcp = cobj_get(priv); - int i, ovector[VRE_MAX_GROUPS]; struct cookie *cookie = NULL; struct cookie *current; - vre_t *vre = NULL; + unsigned match; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - if (expression == NULL || *expression == '\0') - return (NULL); - - if (priv_call->priv == NULL) { - AZ(pthread_mutex_lock(&mtx)); - vre = compile_re(ctx, expression); - if (vre == NULL) { - AZ(pthread_mutex_unlock(&mtx)); - return (NULL); - } - - priv_call->priv = vre; - priv_call->methods = cookie_re_priv_methods; - AZ(pthread_mutex_unlock(&mtx)); - } + AN(re); VTAILQ_FOREACH(current, &vcp->cookielist, list) { CHECK_OBJ_NOTNULL(current, VMOD_COOKIE_ENTRY_MAGIC); VSLb(ctx->vsl, SLT_Debug, "cookie: checking %s", current->name); - i = VRE_exec(vre, current->name, strlen(current->name), 0, 0, - ovector, VRE_MAX_GROUPS, NULL); - if (i < 0) + match = VRT_re_match(ctx, current->name, re); + if (!match) continue; - - VSLb(ctx->vsl, SLT_Debug, "cookie: %s is a match for regex '%s'", - current->name, expression); cookie = current; break; } @@ -425,37 +369,23 @@ vmod_filter(VRT_CTX, struct vmod_priv *priv, VCL_STRING blacklist_s) } static VCL_VOID -re_filter(VRT_CTX, struct vmod_priv *priv, struct vmod_priv *priv_call, - VCL_STRING expression, enum filter_action mode) +re_filter(VRT_CTX, struct vmod_priv *priv, VCL_REGEX re, enum filter_action mode) { struct vmod_cookie *vcp = cobj_get(priv); struct cookie *current, *safeptr; - int i, ovector[VRE_MAX_GROUPS]; - vre_t *vre; + unsigned match; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - if (priv_call->priv == NULL) { - AZ(pthread_mutex_lock(&mtx)); - vre = compile_re(ctx, expression); - if (vre == NULL) { - AZ(pthread_mutex_unlock(&mtx)); - return; // Not much else to do, error already logged. - } - - priv_call->priv = vre; - priv_call->methods = cookie_re_priv_methods; - AZ(pthread_mutex_unlock(&mtx)); - } + AN(re); VTAILQ_FOREACH_SAFE(current, &vcp->cookielist, list, safeptr) { CHECK_OBJ_NOTNULL(current, VMOD_COOKIE_ENTRY_MAGIC); - i = VRE_exec(priv_call->priv, current->name, - strlen(current->name), 0, 0, ovector, VRE_MAX_GROUPS, NULL); + match = VRT_re_match(ctx, current->name, re); switch (mode) { case blacklist: - if (i < 0) + if (!match) continue; VSLb(ctx->vsl, SLT_Debug, "Removing matching cookie %s (value: %s)", @@ -463,12 +393,8 @@ re_filter(VRT_CTX, struct vmod_priv *priv, struct vmod_priv *priv_call, VTAILQ_REMOVE(&vcp->cookielist, current, list); break; case whitelist: - if (i >= 0) { - VSLb(ctx->vsl, SLT_Debug, - "Cookie %s matches expression '%s'", - current->name, expression); + if (match) continue; - } VSLb(ctx->vsl, SLT_Debug, "Removing cookie %s (value: %s)", @@ -483,22 +409,20 @@ re_filter(VRT_CTX, struct vmod_priv *priv, struct vmod_priv *priv_call, VCL_VOID -vmod_keep_re(VRT_CTX, struct vmod_priv *priv, struct vmod_priv *priv_call, - VCL_STRING expression) +vmod_keep_re(VRT_CTX, struct vmod_priv *priv, VCL_REGEX re) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - re_filter(ctx, priv, priv_call, expression, whitelist); + re_filter(ctx, priv, re, whitelist); } VCL_VOID -vmod_filter_re(VRT_CTX, struct vmod_priv *priv, struct vmod_priv *priv_call, - VCL_STRING expression) +vmod_filter_re(VRT_CTX, struct vmod_priv *priv, VCL_REGEX re) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - re_filter(ctx, priv, priv_call, expression, blacklist); + re_filter(ctx, priv, re, blacklist); } diff --git a/vmod/vmod_cookie.vcc b/vmod/vmod_cookie.vcc index 321f544b9..29888f457 100644 --- a/vmod/vmod_cookie.vcc +++ b/vmod/vmod_cookie.vcc @@ -81,7 +81,7 @@ Example:: } -$Function VOID filter_re(PRIV_TASK, PRIV_CALL, STRING expression) +$Function VOID filter_re(PRIV_TASK, REGEX expression) Delete all cookies from internal vmod storage that matches the regular expression ``expression``. @@ -109,7 +109,7 @@ Example:: # "cookie1: value1; cookie2: value2;"; } -$Function VOID keep_re(PRIV_TASK, PRIV_CALL, STRING expression) +$Function VOID keep_re(PRIV_TASK, REGEX expression) Delete all cookies from internal vmod storage that does not match expression ``expression``. @@ -155,7 +155,7 @@ Example:: std.log("cookie1 value is: " + cookie.get("cookie1")); } -$Function STRING get_re(PRIV_TASK, PRIV_CALL, STRING expression) +$Function STRING get_re(PRIV_TASK, REGEX expression) Get the value of the first cookie in internal vmod storage that matches regular expression ``expression``. If nothing matches, an empty string From dridi.boukelmoune at gmail.com Fri Jan 15 16:17:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 15 Jan 2021 16:17:07 +0000 (UTC) Subject: [master] ecb767a83 doc: Draw the VCL_REGEX line Message-ID: <20210115161707.AD1951198B1@lists.varnish-cache.org> commit ecb767a83a654dbf797ecf24e094de5d26e19d4c Author: Dridi Boukelmoune Date: Fri Jan 15 16:24:51 2021 +0100 doc: Draw the VCL_REGEX line Requested by phk. diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst index 08a791fd8..48e99a28a 100644 --- a/doc/sphinx/reference/vmod.rst +++ b/doc/sphinx/reference/vmod.rst @@ -385,6 +385,9 @@ REGEX C-type: ``const struct vre *`` This is an opaque type for regular expressions with a VCL scope. + The REGEX type is only meant for regular expression literals + managed by the VCL compiler. For dynamic regular expressions or + complex usage see the API from the ``include/vre.h`` file. STRING C-type: ``const char *`` From nils.goroll at uplex.de Fri Jan 15 17:27:09 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 15 Jan 2021 17:27:09 +0000 (UTC) Subject: [master] cfed0152c fix unrelated syntax errors in vtcs. Message-ID: <20210115172709.AD2965F6B@lists.varnish-cache.org> commit cfed0152c0b846285b18c897e96f1e4e4b56a98d Author: Nils Goroll Date: Fri Jan 15 18:23:41 2021 +0100 fix unrelated syntax errors in vtcs. These tests had syntax errors besides the error handling actually being tested, which triggered with future changes to VCC. Originally from #3163 diff --git a/bin/varnishtest/tests/e00030.vtc b/bin/varnishtest/tests/e00030.vtc index f5b5f168c..0a529d1f9 100644 --- a/bin/varnishtest/tests/e00030.vtc +++ b/bin/varnishtest/tests/e00030.vtc @@ -2,6 +2,8 @@ varnishtest "Test req_top.* in an ESI context" varnish v1 -arg "-p feature=+esi_disable_xml_check" \ -errvcl {Variable is read only.} { + backend foo None; + sub vcl_recv { set req_top.url = "/foo"; } diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc index 56116f9ab..7e8b2b827 100644 --- a/bin/varnishtest/tests/v00018.vtc +++ b/bin/varnishtest/tests/v00018.vtc @@ -9,7 +9,7 @@ varnish v1 -vcl { varnish v1 -errvcl {Variable is read only.} { backend b { .host = "${localhost}"; } - sub vcl_miss { set now = 1000; } + sub vcl_miss { set now += 1s; } } varnish v1 -vcl { From nils.goroll at uplex.de Fri Jan 15 17:49:09 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 15 Jan 2021 17:49:09 +0000 (UTC) Subject: [master] 5efd7ebb1 Add error handling for last-resort VDP_END Message-ID: <20210115174909.A5C5A6BA5@lists.varnish-cache.org> commit 5efd7ebb10701c745a0e00a6cf2c85b4ca669d39 Author: Nils Goroll Date: Fri Jan 15 18:45:25 2021 +0100 Add error handling for last-resort VDP_END We lacked error handling in the call to ensure that VDP_END is being sent. Now we return the first error seen, which can be the one from the last resort VDP_END call. Flexelint c2d45aefb563bb105f524b67b84d20649ecb85a2 Ref #3298 diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c index bc13f443c..314da53fe 100644 --- a/bin/varnishd/storage/storage_simple.c +++ b/bin/varnishd/storage/storage_simple.c @@ -243,7 +243,7 @@ sml_iterator(struct worker *wrk, struct objcore *oc, const struct stevedore *stv; ssize_t checkpoint_len = 0; ssize_t len = 0; - int ret = 0; + int ret = 0, ret2; ssize_t ol; ssize_t nl; ssize_t sl; @@ -343,8 +343,11 @@ sml_iterator(struct worker *wrk, struct objcore *oc, break; } HSH_DerefBoc(wrk, oc); - if ((u & OBJ_ITER_END) == 0) - func(priv, OBJ_ITER_END, NULL, 0); + if ((u & OBJ_ITER_END) == 0) { + ret2 = func(priv, OBJ_ITER_END, NULL, 0); + if (ret == 0) + ret = ret2; + } return (ret); } From nils.goroll at uplex.de Fri Jan 15 18:24:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 15 Jan 2021 18:24:07 +0000 (UTC) Subject: [master] c2e893650 Flexelint 259e2b8fc4e958f0f5aecb382d3291e5e0780d88 Message-ID: <20210115182407.BAEA97D1D@lists.varnish-cache.org> commit c2e89365025293957f12a1a8b282a1d053e39cda Author: Nils Goroll Date: Fri Jan 15 19:22:52 2021 +0100 Flexelint 259e2b8fc4e958f0f5aecb382d3291e5e0780d88 gc unreferenced macro diff --git a/vmod/vmod_cookie.c b/vmod/vmod_cookie.c index 129d68bdc..2a7807766 100644 --- a/vmod/vmod_cookie.c +++ b/vmod/vmod_cookie.c @@ -45,8 +45,6 @@ #include "vcc_cookie_if.h" -#define VRE_MAX_GROUPS 8 - enum filter_action { blacklist, whitelist From nils.goroll at uplex.de Sat Jan 16 10:46:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 16 Jan 2021 10:46:07 +0000 (UTC) Subject: [master] dbd35b950 vmod_debug: obj fini test code to macro & unify Message-ID: <20210116104607.24A62111CD5@lists.varnish-cache.org> commit dbd35b9506919e6ba5743fcfee7fdbf034f737f4 Author: Nils Goroll Date: Sat Jan 16 11:17:39 2021 +0100 vmod_debug: obj fini test code to macro & unify - use a custom fini function for all priv* tests - unify naming to _fini - move common code to macro Mostly taken from #3472 to slim that PR diff --git a/bin/varnishtest/tests/r02451.vtc b/bin/varnishtest/tests/r02451.vtc index 80e197939..50bf1c71b 100644 --- a/bin/varnishtest/tests/r02451.vtc +++ b/bin/varnishtest/tests/r02451.vtc @@ -54,7 +54,7 @@ logexpect l1 -v v1 -g raw -d 1 { expect 0 * Debug {^objx.priv_task.*"something to remember".$} expect 0 * VCL_Log {^obj something to remember} # string stored in obj priv_task has already been freed - expect ? * Debug {^priv_task_free} + expect ? * Debug {^priv_task_fini} expect ? * Debug {^obj_priv_task_fini} expect 0 * Debug {^vcl1: VCL_EVENT_WARM} } -start diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc index f2b285277..88a869b9c 100644 --- a/bin/varnishtest/tests/v00041.vtc +++ b/bin/varnishtest/tests/v00041.vtc @@ -120,7 +120,7 @@ logexpect l0 -v v1 -g raw -d 1 -m -q "vxid == 0" { expect 0 = VCL_Log {^objc initX} expect 0 = Debug {^objb.priv_task.. = .*"initY"} expect 0 = VCL_Log {^objb initY} - expect ? = Debug {^priv_task_free} + expect ? = Debug {^priv_task_fini} expect ? = Debug {^obj_priv_task_fini.*"initX"} expect ? = Debug {^obj_priv_task_fini.*"initY"} expect 0 = Debug {^vcl1: VCL_EVENT_WARM} @@ -134,10 +134,10 @@ logexpect l0 -v v1 -g raw -d 1 -m -q "vxid == 0" { expect 0 = Debug {^test_priv_task.*exists.$} # client task can race with backend fini, so all of these are # in any order (? = alternatives) - expect ? = Debug {^priv_task_free} + expect ? = Debug {^priv_task_fini} expect ? = Debug {^obj_priv_task_fini.*"r1002"} expect ? = Debug {^test_priv_task.*exists} - expect ? = Debug {^priv_task_free} + expect ? = Debug {^priv_task_fini} expect ? = Debug {^obj_priv_task_fini.*"d1001"} # 1003, but need to include in our alt to tolerate race @@ -160,7 +160,7 @@ logexpect l0 -v v1 -g raw -d 1 -m -q "vxid == 0" { expect 0 = Debug {^test_priv_task.*update.$} expect 0 = Debug {^test_priv_task.*exists.$} expect 0 = Debug {^objc.priv_task.. = NULL} - expect 0 = Debug {^priv_task_free} + expect 0 = Debug {^priv_task_fini} expect 0 = VCL_Log {^func cleaning up} expect 0 = VCL_Log {^obj } expect 0 = CLI {^Wr 200 0 } diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index 997dfe33a..e27e6d5cc 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -225,11 +225,31 @@ xyzzy_author(VRT_CTX, VCL_ENUM person, VCL_ENUM someone) WRONG("Illegal VMOD enum"); } -static const struct vmod_priv_methods xyzzy_test_priv_call_methods[1] = {{ - .magic = VMOD_PRIV_METHODS_MAGIC, - .type = "debug_test_priv_call", - .fini = free -}}; +#define AN0(x) (void) 0 +#define AN1(x) AN(x) +#define PRIV_FINI(name, assert) \ +static void v_matchproto_(vmod_priv_fini_f) \ +priv_ ## name ## _fini(void *ptr) \ +{ \ + const char * const fmt = "priv_" #name "_fini(%p)"; \ + \ + AN ## assert (ptr); \ + VSL(SLT_Debug, 0, fmt, ptr); \ + free(ptr); \ +} \ + \ +static const struct vmod_priv_methods \ +xyzzy_test_priv_ ## name ## _methods[1] = {{ \ + .magic = VMOD_PRIV_METHODS_MAGIC, \ + .type = "debug_test_priv_" #name, \ + .fini = priv_ ## name ## _fini \ + }}; +PRIV_FINI(call, 0) +PRIV_FINI(task, 1) +PRIV_FINI(top, 1) +#undef PRIV_FINI +#undef AN0 +#undef AN1 VCL_VOID v_matchproto_(td_debug_test_priv_call) xyzzy_test_priv_call(VRT_CTX, struct vmod_priv *priv) @@ -252,20 +272,6 @@ xyzzy_test_priv_task_get(VRT_CTX) AZ(VRT_priv_task_get(ctx, NULL)); } -static void -priv_task_free(void *ptr) -{ - AN(ptr); - VSL(SLT_Debug, 0, "priv_task_free(%p)", ptr); - free(ptr); -} - -static const struct vmod_priv_methods xyzzy_test_priv_task_methods[1] = {{ - .magic = VMOD_PRIV_METHODS_MAGIC, - .type = "debug_test_priv_task", - .fini = priv_task_free -}}; - VCL_STRING v_matchproto_(td_debug_test_priv_task) xyzzy_test_priv_task(VRT_CTX, struct vmod_priv *priv, VCL_STRING s) { @@ -295,12 +301,6 @@ xyzzy_test_priv_task(VRT_CTX, struct vmod_priv *priv, VCL_STRING s) return (priv->priv); } -static const struct vmod_priv_methods xyzzy_test_priv_top_methods[1] = {{ - .magic = VMOD_PRIV_METHODS_MAGIC, - .type = "debug_test_priv_top", - .fini = free -}}; - VCL_STRING v_matchproto_(td_debug_test_priv_top) xyzzy_test_priv_top(VRT_CTX, struct vmod_priv *priv, VCL_STRING s) { @@ -403,7 +403,7 @@ xyzzy_fail2(VRT_CTX) } static void v_matchproto_(vmod_priv_fini_f) -priv_vcl_free(void *priv) +priv_vcl_fini(void *priv) { struct priv_vcl *priv_vcl; @@ -422,8 +422,8 @@ priv_vcl_free(void *priv) static const struct vmod_priv_methods priv_vcl_methods[1] = {{ .magic = VMOD_PRIV_METHODS_MAGIC, - .type = "debug_priv_vcl_free", - .fini = priv_vcl_free + .type = "debug_priv_vcl_fini", + .fini = priv_vcl_fini }}; static int diff --git a/vmod/vmod_debug_obj.c b/vmod/vmod_debug_obj.c index 097a74f99..8c0e60070 100644 --- a/vmod/vmod_debug_obj.c +++ b/vmod/vmod_debug_obj.c @@ -152,18 +152,25 @@ xyzzy_obj_test_priv_vcl(VRT_CTX, xyzzy_test_priv_vcl(ctx, priv); } -static void -obj_priv_task_fini(void *ptr) -{ - AN(ptr); - VSL(SLT_Debug, 0, "obj_priv_task_fini(%p = \"%s\")", ptr, (char *)ptr); -} - -static const struct vmod_priv_methods xyzzy_obj_test_priv_task_methods[1] = {{ - .magic = VMOD_PRIV_METHODS_MAGIC, - .type = "debug_obj_test_priv_task", - .fini = obj_priv_task_fini -}}; +#define PRIV_FINI(name) \ +static void v_matchproto_(vmod_priv_fini_f) \ +obj_priv_ ## name ## _fini(void *ptr) \ +{ \ + const char * const fmt = "obj_priv_" #name "_fini(%p = \"%s\")"; \ + \ + AN(ptr); \ + VSL(SLT_Debug, 0, fmt, ptr, ptr); \ +} \ + \ +static const struct vmod_priv_methods \ +xyzzy_obj_test_priv_ ## name ## _methods[1] = {{ \ + .magic = VMOD_PRIV_METHODS_MAGIC, \ + .type = "debug_obj_test_priv_" #name, \ + .fini = obj_priv_ ## name ## _fini \ + }}; +PRIV_FINI(task) +PRIV_FINI(top) +#undef PRIV_FINI VCL_STRING v_matchproto_() xyzzy_obj_test_priv_task(VRT_CTX, struct xyzzy_debug_obj *o, VCL_STRING s) @@ -216,19 +223,6 @@ xyzzy_obj_test_priv_task(VRT_CTX, struct xyzzy_debug_obj *o, VCL_STRING s) return (p->priv); } -static void -obj_priv_top_fini(void *ptr) -{ - AN(ptr); - VSL(SLT_Debug, 0, "obj_priv_top_fini(%p = \"%s\")", ptr, (char *)ptr); -} - -static const struct vmod_priv_methods xyzzy_obj_test_priv_top_methods[1] = {{ - .magic = VMOD_PRIV_METHODS_MAGIC, - .type = "debug_obj_test_priv_top", - .fini = obj_priv_top_fini -}}; - VCL_STRING v_matchproto_() xyzzy_obj_test_priv_top(VRT_CTX, struct xyzzy_debug_obj *o, VCL_STRING s) { From nils.goroll at uplex.de Sat Jan 16 10:46:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 16 Jan 2021 10:46:07 +0000 (UTC) Subject: [master] ff573493e vmod_debug: add assertions on methods ptr Message-ID: <20210116104607.3B38F111CD8@lists.varnish-cache.org> commit ff573493e4c238d92d9f23195d82df1083933806 Author: Nils Goroll Date: Sat Jan 16 11:43:36 2021 +0100 vmod_debug: add assertions on methods ptr diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index e27e6d5cc..e92a4671a 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -260,6 +260,7 @@ xyzzy_test_priv_call(VRT_CTX, struct vmod_priv *priv) priv->priv = strdup("BAR"); priv->methods = xyzzy_test_priv_call_methods; } else { + assert(priv->methods == xyzzy_test_priv_call_methods); assert(!strcmp(priv->priv, "BAR")); } } @@ -309,6 +310,8 @@ xyzzy_test_priv_top(VRT_CTX, struct vmod_priv *priv, VCL_STRING s) if (priv->priv == NULL) { priv->priv = strdup(s); priv->methods = xyzzy_test_priv_top_methods; + } else { + assert(priv->methods == xyzzy_test_priv_top_methods); } return (priv->priv); } From nils.goroll at uplex.de Sat Jan 16 11:14:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 16 Jan 2021 11:14:06 +0000 (UTC) Subject: [master] 6394421d8 forgotten (char *) cast for %s format Message-ID: <20210116111406.EBCAB1129D0@lists.varnish-cache.org> commit 6394421d877005f1a0d01c0ff556e2b6bb150362 Author: Nils Goroll Date: Sat Jan 16 12:11:39 2021 +0100 forgotten (char *) cast for %s format seems to be a real benefit of FreeBSD 13 vtest over all the other vtesters. Not even the compulsively obsessive Flexelint complained. diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index e92a4671a..4a29a8049 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -234,7 +234,7 @@ priv_ ## name ## _fini(void *ptr) \ const char * const fmt = "priv_" #name "_fini(%p)"; \ \ AN ## assert (ptr); \ - VSL(SLT_Debug, 0, fmt, ptr); \ + VSL(SLT_Debug, 0, fmt, (char *)ptr); \ free(ptr); \ } \ \ diff --git a/vmod/vmod_debug_obj.c b/vmod/vmod_debug_obj.c index 8c0e60070..25ec4733d 100644 --- a/vmod/vmod_debug_obj.c +++ b/vmod/vmod_debug_obj.c @@ -159,7 +159,7 @@ obj_priv_ ## name ## _fini(void *ptr) \ const char * const fmt = "obj_priv_" #name "_fini(%p = \"%s\")"; \ \ AN(ptr); \ - VSL(SLT_Debug, 0, fmt, ptr, ptr); \ + VSL(SLT_Debug, 0, fmt, ptr, (char *)ptr); \ } \ \ static const struct vmod_priv_methods \ From nils.goroll at uplex.de Sat Jan 16 11:55:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 16 Jan 2021 11:55:06 +0000 (UTC) Subject: [master] 7da412fa4 use v_printflike_ Message-ID: <20210116115506.4356F113B1F@lists.varnish-cache.org> commit 7da412fa4006eb48e4a31f00deac64cd763d0cd0 Author: Nils Goroll Date: Sat Jan 16 12:22:23 2021 +0100 use v_printflike_ see previous commit: to hopefully help avoid the same oversight with mylog() diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index 4a29a8049..4f4a3f32d 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -64,7 +64,8 @@ static struct VSC_debug *vsc = NULL; static int loads; static const int store_ip_token; static const int fail_rollback_token; -extern void mylog(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...); +extern void mylog(struct vsl_log *vsl, enum VSL_tag_e tag, + const char *fmt, ...) v_printflike_(3,4); /**********************************************************************/ diff --git a/vmod/vmod_debug_obj.c b/vmod/vmod_debug_obj.c index 25ec4733d..19386d985 100644 --- a/vmod/vmod_debug_obj.c +++ b/vmod/vmod_debug_obj.c @@ -39,7 +39,8 @@ #include "vcc_debug_if.h" // vmod_debug.c -extern void mylog(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...); +extern void mylog(struct vsl_log *vsl, enum VSL_tag_e tag, + const char *fmt, ...) v_printflike_(3,4); struct xyzzy_debug_obj { unsigned magic; From nils.goroll at uplex.de Sat Jan 16 11:55:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 16 Jan 2021 11:55:06 +0000 (UTC) Subject: [master] c9dd14b11 vmod_debug: move test_priv_task logging to the transaction Message-ID: <20210116115506.5A969113B22@lists.varnish-cache.org> commit c9dd14b11d30f1a5863d4b66b626604ce06ccb9a Author: Nils Goroll Date: Sat Jan 16 12:53:16 2021 +0100 vmod_debug: move test_priv_task logging to the transaction This removes logging from xid 0 and makes clearer what is happening diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc index 88a869b9c..03a4448a0 100644 --- a/bin/varnishtest/tests/v00041.vtc +++ b/bin/varnishtest/tests/v00041.vtc @@ -125,29 +125,18 @@ logexpect l0 -v v1 -g raw -d 1 -m -q "vxid == 0" { expect ? = Debug {^obj_priv_task_fini.*"initY"} expect 0 = Debug {^vcl1: VCL_EVENT_WARM} + # need an anchor for the ? expects to begin + expect * = CLI {^Rd debug.xid} + expect 0 = CLI {^Wr 200} + expect 0 = CLI {^Rd debug.listen_address} + expect 0 = CLI {^Wr 200} + # 1001/1002 - expect * = Debug {^test_priv_task.*new.$} - expect 0 = Debug {^test_priv_task.*exists.$} - expect 0 = Debug {^test_priv_task.*update.$} - expect 0 = Debug {^test_priv_task.*new.$} - expect 0 = Debug {^test_priv_task.*update.$} - expect 0 = Debug {^test_priv_task.*exists.$} - # client task can race with backend fini, so all of these are - # in any order (? = alternatives) expect ? = Debug {^priv_task_fini} expect ? = Debug {^obj_priv_task_fini.*"r1002"} - expect ? = Debug {^test_priv_task.*exists} expect ? = Debug {^priv_task_fini} expect ? = Debug {^obj_priv_task_fini.*"d1001"} - # 1003, but need to include in our alt to tolerate race - expect ? = Debug {^test_priv_task.*new.$} - expect ? = Debug {^test_priv_task.*exists.$} - expect ? = Debug {^test_priv_task.*update.$} - expect ? = Debug {^test_priv_task.*new.$} - expect ? = Debug {^test_priv_task.*update.$} - expect ? = Debug {^test_priv_task.*exists.$} - # ... # 1006 pipe expect * = Debug {^obj_priv_task_fini.*"p1006"} @@ -156,19 +145,22 @@ logexpect l0 -v v1 -g raw -d 1 -m -q "vxid == 0" { expect * = Debug {^vcl1: VCL_EVENT_COLD} expect 0 = CLI {^Wr 200 0 } expect 0 = CLI {^Rd vcl.discard vcl1} - expect 0 = Debug {^test_priv_task.*new.$} - expect 0 = Debug {^test_priv_task.*update.$} - expect 0 = Debug {^test_priv_task.*exists.$} - expect 0 = Debug {^objc.priv_task.. = NULL} - expect 0 = Debug {^priv_task_fini} - expect 0 = VCL_Log {^func cleaning up} - expect 0 = VCL_Log {^obj } + expect ? = Debug {^test_priv_task.*new.$} + expect ? = Debug {^test_priv_task.*update.$} + expect ? = Debug {^test_priv_task.*exists.$} + expect ? = Debug {^objc.priv_task.. = NULL} + expect ? = Debug {^priv_task_fini} + expect ? = VCL_Log {^func cleaning up} + expect ? = VCL_Log {^obj } expect 0 = CLI {^Wr 200 0 } } -start logexpect l1001 -v v1 -g vxid -q "vxid == 1001" { expect * 1001 VCL_call {^RECV} + expect 0 = Debug {^test_priv_task.*new.$} + expect 0 = Debug {^test_priv_task.*exists.$} expect 0 = ReqHeader {^x0: /foobar} + expect 0 = Debug {^test_priv_task.*update.$} expect 0 = Debug {^objc.priv_task.. = NULL} expect 0 = VCL_Log {^objc } expect 0 = Debug {^objb.priv_task.. = NULL} @@ -182,6 +174,7 @@ logexpect l1001 -v v1 -g vxid -q "vxid == 1001" { expect 0 = VCL_Log {^objb } expect 0 = Debug {^objc.priv_task.. =.*"d1001". .update.} expect 0 = RespHeader {^x0: /foobar} + expect 0 = Debug {^test_priv_task.*exists.$} expect 0 = RespHeader {^x1: /foobar bazz} expect 0 = Debug {^objc.priv_task.. =.*"d1001".} expect 0 = RespHeader {^objc: d1001} @@ -195,6 +188,8 @@ logexpect l1002 -v v1 -g vxid -q "vxid == 1002" { expect 0 = Debug {^objb.priv_task.. = NULL} expect 0 = VCL_Log {^objb } expect 0 = Debug {^objb.priv_task.. =.*"f1002". .new.} + expect 0 = Debug {^test_priv_task.*new.$} + expect 0 = Debug {^test_priv_task.*update.$} expect 0 = BereqHeader {^bx0: b /foobar} expect 0 = VCL_return {^fetch} @@ -205,6 +200,7 @@ logexpect l1002 -v v1 -g vxid -q "vxid == 1002" { expect 0 = VCL_Log {^objb f1002} expect 0 = Debug {^objb.priv_task.. =.*"r1002". .update.} expect 0 = BerespHeader {^bx0: b /foobar} + expect 0 = Debug {^test_priv_task.*exists.$} expect 0 = BerespHeader {^bx1: b /foobar} expect 0 = Debug {^objb.priv_task.. =.*"r1002".} expect 0 = BerespHeader {^objb: r1002} @@ -213,7 +209,10 @@ logexpect l1002 -v v1 -g vxid -q "vxid == 1002" { logexpect l1006 -v v1 -g vxid -q "vxid == 1006" { expect * 1006 VCL_call {^RECV} + expect 0 = Debug {^test_priv_task.*new.$} + expect 0 = Debug {^test_priv_task.*exists.$} expect 0 = ReqHeader {^x0: /pipe} + expect 0 = Debug {^test_priv_task.*update.$} expect 0 = Debug {^objc.priv_task.. = NULL} expect 0 = VCL_Log {^objc } expect 0 = Debug {^objb.priv_task.. = NULL} @@ -231,8 +230,11 @@ logexpect l1006 -v v1 -g vxid -q "vxid == 1006" { expect 0 = Debug {^objb.priv_task.. = NULL} expect 0 = VCL_Log {^objb } expect 0 = Debug {^objc.priv_task.. =.*"p1006". .update.} + expect 0 = Debug {^test_priv_task.*update.$} + expect 0 = Debug {^test_priv_task.*exists.$} expect 0 = ReqUnset {^x0: /pipe} expect 0 = ReqHeader {^x0: /pipe bazz /pipe} + expect 0 = Debug {^test_priv_task.*update.$} expect 0 = VCL_return {^pipe} } -start diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index 4f4a3f32d..1223d11b4 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -280,12 +280,12 @@ xyzzy_test_priv_task(VRT_CTX, struct vmod_priv *priv, VCL_STRING s) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if (s == NULL || *s == '\0') { - VSL(SLT_Debug, 0, "test_priv_task(%p) = %p (exists)", + mylog(ctx->vsl, SLT_Debug, "test_priv_task(%p) = %p (exists)", priv, priv->priv); } else if (priv->priv == NULL) { priv->priv = strdup(s); priv->methods = xyzzy_test_priv_task_methods; - VSL(SLT_Debug, 0, "test_priv_task(%p) = %p (new)", + mylog(ctx->vsl, SLT_Debug, "test_priv_task(%p) = %p (new)", priv, priv->priv); } else { char *n = realloc(priv->priv, @@ -295,7 +295,7 @@ xyzzy_test_priv_task(VRT_CTX, struct vmod_priv *priv, VCL_STRING s) strcat(n, " "); strcat(n, s); priv->priv = n; - VSL(SLT_Debug, 0, "test_priv_task(%p) = %p (update)", + mylog(ctx->vsl, SLT_Debug, "test_priv_task(%p) = %p (update)", priv, priv->priv); } if (priv->priv != NULL) From nils.goroll at uplex.de Sat Jan 16 12:12:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 16 Jan 2021 12:12:06 +0000 (UTC) Subject: [master] 1bb39901b fix another %s cast Message-ID: <20210116121206.8C03D11451B@lists.varnish-cache.org> commit 1bb39901bdca35cae2834be5642c0d92279645c7 Author: Nils Goroll Date: Sat Jan 16 13:09:52 2021 +0100 fix another %s cast 7da412fa4006eb48e4a31f00deac64cd763d0cd0 at work diff --git a/vmod/vmod_debug_obj.c b/vmod/vmod_debug_obj.c index 19386d985..266f4be6a 100644 --- a/vmod/vmod_debug_obj.c +++ b/vmod/vmod_debug_obj.c @@ -196,7 +196,7 @@ xyzzy_obj_test_priv_task(VRT_CTX, struct xyzzy_debug_obj *o, VCL_STRING s) assert(p->methods == xyzzy_obj_test_priv_task_methods); mylog(vsl, SLT_Debug, "%s.priv_task() = %p .priv = %p (\"%s\")", - o->vcl_name, p, p->priv, p->priv); + o->vcl_name, p, p->priv, (char *)p->priv); return (p->priv); } From nils.goroll at uplex.de Sun Jan 17 11:55:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 17 Jan 2021 11:55:08 +0000 (UTC) Subject: [master] 10c29c5a7 clarify the shard_param example a bit Message-ID: <20210117115508.E2CFC11249B@lists.varnish-cache.org> commit 10c29c5a70e6c1a501581fa9cdc485197f270bd5 Author: Nils Goroll Date: Sun Jan 17 12:53:55 2021 +0100 clarify the shard_param example a bit diff --git a/vmod/vmod_directors.vcc b/vmod/vmod_directors.vcc index ddcbf6a3a..1ea5ff79d 100644 --- a/vmod/vmod_directors.vcc +++ b/vmod/vmod_directors.vcc @@ -636,7 +636,7 @@ implement retries on alternative backends:: sub vcl_backend_fetch { # changes dir_A and dir_B behaviour - shard_param.set(alt=bereq.retries); + shard_param.set(alt=bereq.retries, by=URL); } $Method VOID .clear() From nils.goroll at uplex.de Mon Jan 18 10:13:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 18 Jan 2021 11:13:07 +0100 Subject: New Defects reported by Coverity Scan for varnish In-Reply-To: <6004288c99fd4_21fc62b1adfc6cf58243fb@prd-scan-dashboard-0.mail> References: <6004288c99fd4_21fc62b1adfc6cf58243fb@prd-scan-dashboard-0.mail> Message-ID: On 17/01/2021 13:07, scan-admin at coverity.com wrote: > ** CID 1472212: Control flow issues (DEADCODE) > /vmod/vmod_directors_shard.c: 635 in vmod_shard_backend() I will address that now -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From dridi.boukelmoune at gmail.com Mon Jan 18 14:31:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 18 Jan 2021 14:31:07 +0000 (UTC) Subject: [master] 23693c4c1 varnishncsa: log request time in milliseconds Message-ID: <20210118143107.E05F9116DF7@lists.varnish-cache.org> commit 23693c4c1e3a8322b050d45804892c21044d9950 Author: Asad Sajjad Ahmed Date: Thu Jan 14 11:01:23 2021 +0100 varnishncsa: log request time in milliseconds Allow the user to log the request time in milliseconds through the new format specifier: %{ms}T . Signed-off-by: Asad Sajjad Ahmed diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 9e2fc22d4..9078eed92 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -290,10 +290,6 @@ format_time(const struct format *format) t_end = t_start; switch (format->time_type) { - case 'D': - AZ(VSB_printf(CTX.vsb, "%d", - (int)((t_end - t_start) * 1e6))); - break; case 't': AN(format->time_fmt); t = (long)floor(t_start); @@ -302,7 +298,16 @@ format_time(const struct format *format) AZ(VSB_cat(CTX.vsb, buf)); break; case 'T': - AZ(VSB_printf(CTX.vsb, "%d", (int)(t_end - t_start))); + AN(format->time_fmt); + if (!strcmp(format->time_fmt, "s")) /* same as %T */ + t = t_end - t_start; + else if (!strcmp(format->time_fmt, "ms")) + t = (t_end - t_start) * 1e3; + else if (!strcmp(format->time_fmt, "us")) /* same as %D */ + t = (t_end - t_start) * 1e6; + else + WRONG("Unreachable branch"); + AZ(VSB_printf(CTX.vsb, "%d", (int)t)); break; default: WRONG("Time format specifier"); @@ -440,12 +445,16 @@ addf_time(char type, const char *fmt) ALLOC_OBJ(f, FORMAT_MAGIC); AN(f); + AN(fmt); f->func = format_time; f->time_type = type; - if (fmt != NULL) { - f->time_fmt = strdup(fmt); - AN(f->time_fmt); - } + f->time_fmt = strdup(fmt); + AN(f->time_fmt); + + if (f->time_type == 'T' && strcmp(f->time_fmt, "s") && + strcmp(f->time_fmt, "ms") && strcmp(f->time_fmt, "us")) + VUT_Error(vut, 1, "Unknown specifier: %%{%s}T", f->time_fmt); + VTAILQ_INSERT_TAIL(&CTX.format, f, list); } @@ -661,7 +670,7 @@ parse_format(const char *format) addf_fragment(&CTX.frag[F_b], "-"); break; case 'D': /* Float request time */ - addf_time(*p, NULL); + addf_time('T', "us"); break; case 'h': /* Client host name / IP Address */ addf_fragment(&CTX.frag[F_h], "-"); @@ -694,7 +703,7 @@ parse_format(const char *format) addf_time(*p, TIME_FMT); break; case 'T': /* Int request time */ - addf_time(*p, NULL); + addf_time(*p, "s"); break; case 'u': /* Remote user from auth */ addf_auth(); @@ -724,6 +733,9 @@ parse_format(const char *format) case 't': addf_time(*q, buf); break; + case 'T': + addf_time(*q, buf); + break; case 'x': parse_x_format(buf); break; diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst index 09eac91bd..d2b61e16c 100644 --- a/doc/sphinx/reference/varnishncsa.rst +++ b/doc/sphinx/reference/varnishncsa.rst @@ -84,7 +84,7 @@ Supported formatters are: %D In client mode, time taken to serve the request, in microseconds. In backend mode, time from the request was sent to the entire body - had been received. + had been received. This is equivalent to %{us}T. %H The request protocol. Defaults to HTTP/1.0 if not known. @@ -138,7 +138,13 @@ Supported formatters are: %T In client mode, time taken to serve the request, in seconds. In backend mode, time from the request was sent to the entire body had - been received. + been received. This is equivalent to %{s}T. + +%{X}T + In client mode, time taken to serve the request, in the format + specified by X. In backend mode, time from the request was sent to + the entire body had been received. The time specification format can + be one of the following: s (same as %T), ms or us (same as %D). %U The request URL without the query string. Defaults to '-' if not From dridi.boukelmoune at gmail.com Mon Jan 18 15:00:11 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 18 Jan 2021 15:00:11 +0000 (UTC) Subject: [master] eba8c51d2 vsl: Kill dead check Message-ID: <20210118150011.428C3117ACB@lists.varnish-cache.org> commit eba8c51d2b8738279b5dd6bddcfedfc9696a9088 Author: Dridi Boukelmoune Date: Wed Oct 28 15:31:33 2020 +0100 vsl: Kill dead check There's only one call site for vtx_dispatch() and it guarantees that len is greater than zero. diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c index 8ce001409..cf11a92a0 100644 --- a/lib/libvarnishapi/vsl_dispatch.c +++ b/lib/libvarnishapi/vsl_dispatch.c @@ -439,8 +439,7 @@ vtx_append(struct VSLQ *vslq, struct vtx *vtx, const struct VSLC_ptr *start, enum vsl_check i; AN(vtx); - if (len == 0) - return; + AN(len); AN(start); i = VSL_Check(vslq->c, start); From dridi.boukelmoune at gmail.com Mon Jan 18 15:00:11 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 18 Jan 2021 15:00:11 +0000 (UTC) Subject: [master] 729e69be0 vsl: Give up when it's too late to buffer records Message-ID: <20210118150011.5A8AF117ACE@lists.varnish-cache.org> commit 729e69be09f428906a57b22480f642c62c852ee2 Author: Dridi Boukelmoune Date: Wed Oct 28 15:38:42 2020 +0100 vsl: Give up when it's too late to buffer records When a VUT is slow enough, it might very well be overrun while it is scanning logs. For our built-in VUTs like varnishncsa or varnishlog this can happen if writing the output can block waiting for IO ops or when the output is piped to a slow consumer. diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c index cf11a92a0..a6d226c4e 100644 --- a/lib/libvarnishapi/vsl_dispatch.c +++ b/lib/libvarnishapi/vsl_dispatch.c @@ -431,7 +431,7 @@ chunk_shm_to_buf(struct VSLQ *vslq, struct chunk *chunk) } /* Append a set of records to a vtx structure */ -static void +static enum vsl_status vtx_append(struct VSLQ *vslq, struct vtx *vtx, const struct VSLC_ptr *start, size_t len) { @@ -443,6 +443,9 @@ vtx_append(struct VSLQ *vslq, struct vtx *vtx, const struct VSLC_ptr *start, AN(start); i = VSL_Check(vslq->c, start); + if (i == vsl_check_e_inval) + return (vsl_e_overrun); + if (i == vsl_check_valid && !VTAILQ_EMPTY(&vtx->shmchunks_free)) { /* Shmref it */ chunk = VTAILQ_FIRST(&vtx->shmchunks_free); @@ -457,7 +460,6 @@ vtx_append(struct VSLQ *vslq, struct vtx *vtx, const struct VSLC_ptr *start, /* Append to shmref list */ VTAILQ_INSERT_TAIL(&vslq->shmrefs, chunk, shm.shmref); } else { - assert(i != vsl_check_e_inval); /* Buffer it */ chunk = VTAILQ_LAST(&vtx->chunks, chunkhead); CHECK_OBJ_ORNULL(chunk, CHUNK_MAGIC); @@ -472,6 +474,7 @@ vtx_append(struct VSLQ *vslq, struct vtx *vtx, const struct VSLC_ptr *start, } } vtx->len += len; + return (vsl_more); } /* Allocate a new vtx structure */ @@ -1314,8 +1317,9 @@ vslq_next(struct VSLQ *vslq) AN(vtx); } if (vtx != NULL) { - vtx_append(vslq, vtx, &c->rec, len); - vtx_scan(vslq, vtx); + r = vtx_append(vslq, vtx, &c->rec, len); + if (r == vsl_more) + vtx_scan(vslq, vtx); } return (r); From nils.goroll at uplex.de Mon Jan 18 15:18:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 18 Jan 2021 15:18:07 +0000 (UTC) Subject: [master] cc42bac5c shard director refactor: .shardd to local variable Message-ID: <20210118151807.14952118587@lists.varnish-cache.org> commit cc42bac5c3a51b7ab521505940033cb842bbc9cf Author: Nils Goroll Date: Mon Jan 18 12:24:06 2021 +0100 shard director refactor: .shardd to local variable diff --git a/vmod/vmod_directors_shard.c b/vmod/vmod_directors_shard.c index afe753854..4222f92c6 100644 --- a/vmod/vmod_directors_shard.c +++ b/vmod/vmod_directors_shard.c @@ -595,6 +595,7 @@ VCL_BACKEND v_matchproto_(td_directors_shard_backend) vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, struct VARGS(shard_backend) *a) { + struct sharddir *shardd; struct vmod_directors_shard_param pstk; struct vmod_directors_shard_param *pp = NULL; const struct vmod_directors_shard_param *ppt; @@ -603,6 +604,8 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); + shardd = vshard->shardd; + CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); assert((args & ~arg_mask_) == 0); if (args & arg_resolve) @@ -619,7 +622,7 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, } if ((ctx->method & SHARD_VCL_TASK_BEREQ) == 0) { - shard_fail(ctx, vshard->shardd->name, "%s", + shard_fail(ctx, shardd->name, "%s", ".backend(resolve=LAZY) with other " "parameters can only be used in backend/pipe " "context"); @@ -627,24 +630,23 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, } } else if (resolve == VENUM(NOW)) { if (ctx->method & VCL_MET_TASK_H) { - shard_fail(ctx, vshard->shardd->name, "%s", + shard_fail(ctx, shardd->name, "%s", ".backend(resolve=NOW) can not be " "used in vcl_init{}/vcl_fini{}"); return (NULL); } pp = shard_param_stack(&pstk, - pp != NULL ? pp : vshard->shardd->param, - vshard->shardd->name); + pp != NULL ? pp : shardd->param, + shardd->name); } else { WRONG("resolve enum"); } if (ctx->method & SHARD_VCL_TASK_BEREQ) { - pp = shard_param_task_l(ctx, vshard->shardd, - vshard->shardd->param); + pp = shard_param_task_l(ctx, shardd, shardd->param); if (pp == NULL) return (NULL); - pp->vcl_name = vshard->shardd->name; + pp->vcl_name = shardd->name; } AN(pp); @@ -652,7 +654,7 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, if (args & arg_param) { ppt = shard_param_blob(a->param); if (ppt == NULL) { - shard_fail(ctx, vshard->shardd->name, "%s", + shard_fail(ctx, shardd->name, "%s", ".backend(key_blob) param invalid"); return (NULL); } @@ -671,9 +673,8 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, assert(resolve == VENUM(NOW)); shard_param_merge(pp, pp->defaults); - return (sharddir_pick_be(ctx, vshard->shardd, - shard_get_key(ctx, pp), pp->alt, pp->warmup, - pp->rampup, pp->healthy)); + return (sharddir_pick_be(ctx, shardd, shard_get_key(ctx, pp), + pp->alt, pp->warmup, pp->rampup, pp->healthy)); } static VCL_BOOL v_matchproto_(vdi_healthy) From nils.goroll at uplex.de Mon Jan 18 15:18:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 18 Jan 2021 15:18:07 +0000 (UTC) Subject: [master] 33ceb67d5 shard director: improve assertions on parameters Message-ID: <20210118151807.2BB0511858A@lists.varnish-cache.org> commit 33ceb67d532381a071feee3f43dcb5e6694f4ec1 Author: Nils Goroll Date: Mon Jan 18 12:38:22 2021 +0100 shard director: improve assertions on parameters We use shard_param_task_{r,l} on (struct sharddir *) and (struct vmod_directors_shard_param *), so the id parameter is (const void *). Yet we still want to make sure that we always retrieve the intended PRIV_TASK, and we can by simply asserting that the vcl_name attribute matches. Note that a simple pointer comparison is sufficient here because, unltimately, the PRIV_TASK vcl_name is always that of a vmod object, assigned by the constructor. This brings back the "who" parameter from 9ea2f29f9be641e0569214326f2974baa8960077, but used differently. diff --git a/vmod/vmod_directors_shard.c b/vmod/vmod_directors_shard.c index 4222f92c6..ce52bb5e8 100644 --- a/vmod/vmod_directors_shard.c +++ b/vmod/vmod_directors_shard.c @@ -159,18 +159,18 @@ shard_param_stack(struct vmod_directors_shard_param *p, const struct vmod_directors_shard_param *pa, const char *who); static const struct vmod_directors_shard_param * -shard_param_task_r(VRT_CTX, const void *id, +shard_param_task_r(VRT_CTX, const void *id, const char *who, const struct vmod_directors_shard_param *pa); static struct vmod_directors_shard_param * -shard_param_task_l(VRT_CTX, const void *id, +shard_param_task_l(VRT_CTX, const void *id, const char *who, const struct vmod_directors_shard_param *pa); static const struct vmod_directors_shard_param * shard_param_blob(VCL_BLOB blob); static const struct vmod_directors_shard_param * -vmod_shard_param_read(VRT_CTX, const void *id, +vmod_shard_param_read(VRT_CTX, const void *id, const char *who, const struct vmod_directors_shard_param *p, struct vmod_directors_shard_param *pstk); @@ -643,10 +643,10 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, } if (ctx->method & SHARD_VCL_TASK_BEREQ) { - pp = shard_param_task_l(ctx, shardd, shardd->param); + pp = shard_param_task_l(ctx, shardd, shardd->name, + shardd->param); if (pp == NULL) return (NULL); - pp->vcl_name = shardd->name; } AN(pp); @@ -699,7 +699,8 @@ vmod_shard_resolve(VRT_CTX, VCL_BACKEND dir) CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(shardd, dir->priv, SHARDDIR_MAGIC); - pp = vmod_shard_param_read(ctx, shardd, shardd->param, pstk); + pp = vmod_shard_param_read(ctx, shardd, shardd->name, + shardd->param, pstk); CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return (sharddir_pick_be(ctx, shardd, @@ -871,7 +872,7 @@ shard_param_stack(struct vmod_directors_shard_param *p, } static const struct vmod_directors_shard_param * -shard_param_task_r(VRT_CTX, const void *id, +shard_param_task_r(VRT_CTX, const void *id, const char *who, const struct vmod_directors_shard_param *pa) { const struct vmod_directors_shard_param *p; @@ -888,13 +889,14 @@ shard_param_task_r(VRT_CTX, const void *id, if (task) { CAST_OBJ_NOTNULL(p, task->priv, VMOD_SHARD_SHARD_PARAM_MAGIC); assert(p->scope == SCOPE_TASK); + assert(who == p->vcl_name); return (p); } if (id == pa || pa->scope != SCOPE_VCL) return (pa); - return (shard_param_task_r(ctx, pa, pa)); + return (shard_param_task_r(ctx, pa, pa->vcl_name, pa)); } /* @@ -902,7 +904,7 @@ shard_param_task_r(VRT_CTX, const void *id, * if id != pa and pa has VCL scope, also get a task scoped param struct for pa */ static struct vmod_directors_shard_param * -shard_param_task_l(VRT_CTX, const void *id, +shard_param_task_l(VRT_CTX, const void *id, const char *who, const struct vmod_directors_shard_param *pa) { struct vmod_directors_shard_param *p; @@ -917,30 +919,31 @@ shard_param_task_l(VRT_CTX, const void *id, task = VRT_priv_task(ctx, task_id); if (task == NULL) { - shard_fail(ctx, pa->vcl_name, "%s", "no priv_task"); + shard_fail(ctx, who, "%s", "no priv_task"); return (NULL); } if (task->priv) { CAST_OBJ_NOTNULL(p, task->priv, VMOD_SHARD_SHARD_PARAM_MAGIC); assert(p->scope == SCOPE_TASK); + assert(who == p->vcl_name); return (p); } p = WS_Alloc(ctx->ws, sizeof *p); if (p == NULL) { - shard_fail(ctx, pa->vcl_name, "%s", "WS_Alloc failed"); + shard_fail(ctx, who, "%s", "WS_Alloc failed"); return (NULL); } task->priv = p; INIT_OBJ(p, VMOD_SHARD_SHARD_PARAM_MAGIC); - p->vcl_name = pa->vcl_name; + p->vcl_name = who; p->scope = SCOPE_TASK; if (id == pa || pa->scope != SCOPE_VCL) p->defaults = pa; else - p->defaults = shard_param_task_l(ctx, pa, pa); + p->defaults = shard_param_task_l(ctx, pa, pa->vcl_name, pa); return (p); } @@ -957,7 +960,7 @@ shard_param_prep(VRT_CTX, struct vmod_directors_shard_param *p, "in vcl_init and in backend/pipe context", who); return (NULL); } else if (ctx->method & SHARD_VCL_TASK_BEREQ) - p = shard_param_task_l(ctx, p, p); + p = shard_param_task_l(ctx, p, p->vcl_name, p); else assert(ctx->method & VCL_MET_TASK_H); @@ -991,7 +994,7 @@ vmod_shard_param_clear(VRT_CTX, } static const struct vmod_directors_shard_param * -vmod_shard_param_read(VRT_CTX, const void *id, +vmod_shard_param_read(VRT_CTX, const void *id, const char *who, const struct vmod_directors_shard_param *p, struct vmod_directors_shard_param *pstk) { @@ -1001,7 +1004,7 @@ vmod_shard_param_read(VRT_CTX, const void *id, CHECK_OBJ_NOTNULL(p, VMOD_SHARD_SHARD_PARAM_MAGIC); if (ctx->method == 0 || (ctx->method & SHARD_VCL_TASK_BEREQ)) - p = shard_param_task_r(ctx, id, p); + p = shard_param_task_r(ctx, id, who, p); CHECK_OBJ_NOTNULL(p, VMOD_SHARD_SHARD_PARAM_MAGIC); pp = shard_param_stack(pstk, p, p->vcl_name); @@ -1016,7 +1019,7 @@ vmod_shard_param_get_by(VRT_CTX, struct vmod_directors_shard_param pstk; const struct vmod_directors_shard_param *pp; - pp = vmod_shard_param_read(ctx, p, p, &pstk); + pp = vmod_shard_param_read(ctx, p, p->vcl_name, p, &pstk); CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return (default_by(pp->by)); } @@ -1028,7 +1031,7 @@ vmod_shard_param_get_key(VRT_CTX, struct vmod_directors_shard_param pstk; const struct vmod_directors_shard_param *pp; - pp = vmod_shard_param_read(ctx, p, p, &pstk); + pp = vmod_shard_param_read(ctx, p, p->vcl_name, p, &pstk); CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return ((VCL_INT)shard_get_key(ctx, pp)); } @@ -1039,7 +1042,7 @@ vmod_shard_param_get_alt(VRT_CTX, struct vmod_directors_shard_param pstk; const struct vmod_directors_shard_param *pp; - pp = vmod_shard_param_read(ctx, p, p, &pstk); + pp = vmod_shard_param_read(ctx, p, p->vcl_name, p, &pstk); CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return (pp->alt); } @@ -1051,7 +1054,7 @@ vmod_shard_param_get_warmup(VRT_CTX, struct vmod_directors_shard_param pstk; const struct vmod_directors_shard_param *pp; - pp = vmod_shard_param_read(ctx, p, p, &pstk); + pp = vmod_shard_param_read(ctx, p, p->vcl_name, p, &pstk); CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return (pp->warmup); } @@ -1063,7 +1066,7 @@ vmod_shard_param_get_rampup(VRT_CTX, struct vmod_directors_shard_param pstk; const struct vmod_directors_shard_param *pp; - pp = vmod_shard_param_read(ctx, p, p, &pstk); + pp = vmod_shard_param_read(ctx, p, p->vcl_name, p, &pstk); CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return (pp->rampup); } @@ -1075,7 +1078,7 @@ vmod_shard_param_get_healthy(VRT_CTX, struct vmod_directors_shard_param pstk; const struct vmod_directors_shard_param *pp; - pp = vmod_shard_param_read(ctx, p, p, &pstk); + pp = vmod_shard_param_read(ctx, p, p->vcl_name, p, &pstk); CHECK_OBJ_NOTNULL(pp, VMOD_SHARD_SHARD_PARAM_MAGIC); return (default_healthy(pp->healthy)); } From nils.goroll at uplex.de Mon Jan 18 15:18:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 18 Jan 2021 15:18:07 +0000 (UTC) Subject: [master] c2beacdfd shard director: handle nested workspace overflow Message-ID: <20210118151807.5A5F011858E@lists.varnish-cache.org> commit c2beacdfdc4dbad0f73a5dfda9bd05ebf0133261 Author: Nils Goroll Date: Mon Jan 18 12:50:03 2021 +0100 shard director: handle nested workspace overflow Propagate upwards any workspace allocation failure of the recursive shard_param_task_l() case. diff --git a/vmod/vmod_directors_shard.c b/vmod/vmod_directors_shard.c index ce52bb5e8..2a214279d 100644 --- a/vmod/vmod_directors_shard.c +++ b/vmod/vmod_directors_shard.c @@ -945,6 +945,9 @@ shard_param_task_l(VRT_CTX, const void *id, const char *who, else p->defaults = shard_param_task_l(ctx, pa, pa->vcl_name, pa); + if (p->defaults == NULL) + return (NULL); + return (p); } From nils.goroll at uplex.de Mon Jan 18 15:18:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 18 Jan 2021 15:18:07 +0000 (UTC) Subject: [master] d1ef9b9bf shard director: Fix regression from 08b642d70ff15346eb1818171862f70a3a9266af Message-ID: <20210118151807.8F14B118594@lists.varnish-cache.org> commit d1ef9b9bf7ae4aaf32aec0f27a4bd3995bb1cd2b Author: Nils Goroll Date: Mon Jan 18 14:14:22 2021 +0100 shard director: Fix regression from 08b642d70ff15346eb1818171862f70a3a9266af and add vtc code to avoid it from happening again. The mentioned commit lead to resolve=NOW modify the shard's parameter PRIV_TASK when only parameters on the stack should have been modified. We will adjust this to use shard_param_task_r() in a follow-up Spotted thanks to Coverity, ref CID 1472212 diff --git a/bin/varnishtest/tests/d00021.vtc b/bin/varnishtest/tests/d00021.vtc index cf0a1be27..730c32705 100644 --- a/bin/varnishtest/tests/d00021.vtc +++ b/bin/varnishtest/tests/d00021.vtc @@ -70,6 +70,10 @@ varnish v1 -vcl+backend { set bereq.backend = vd.backend(resolve=LAZY, by=KEY, key=4294967295); } + set bereq.http.bereq-now-1 = vd.backend(resolve=NOW); + set bereq.http.bereq-now-mod = + vd.backend(resolve=NOW, alt=1); + set bereq.http.bereq-now-2 = vd.backend(resolve=NOW); } sub backend_fetch_layered { @@ -83,6 +87,10 @@ varnish v1 -vcl+backend { if (bereq.url == "/3") { lp.set(by=KEY, key=4294967295); } + set bereq.http.bereq-now-1 = l.backend(resolve=NOW); + set bereq.http.bereq-now-mod = + l.backend(resolve=NOW, alt=1); + set bereq.http.bereq-now-2 = l.backend(resolve=NOW); } sub vcl_backend_fetch { @@ -106,6 +114,9 @@ varnish v1 -vcl+backend { } set beresp.http.director = bereq.backend; set beresp.http.backend = beresp.backend; + set beresp.http.bereq-now-1 = bereq.http.bereq-now-1; + set beresp.http.bereq-now-mod = bereq.http.bereq-now-mod; + set beresp.http.bereq-now-2 = bereq.http.bereq-now-2; } } -start @@ -117,6 +128,9 @@ client c1 { expect resp.http.healthy == "true" expect resp.http.director == "vd" expect resp.http.backend == "s1" + expect resp.http.backend == resp.http.bereq-now-1 + expect resp.http.backend != resp.http.bereq-now-mod + expect resp.http.backend == resp.http.bereq-now-2 txreq -url /2 rxresp @@ -124,6 +138,9 @@ client c1 { expect resp.http.healthy == "true" expect resp.http.director == "vd" expect resp.http.backend == "s2" + expect resp.http.backend == resp.http.bereq-now-1 + expect resp.http.backend != resp.http.bereq-now-mod + expect resp.http.backend == resp.http.bereq-now-2 txreq -url /3 rxresp @@ -131,6 +148,9 @@ client c1 { expect resp.http.healthy == "true" expect resp.http.director == "vd" expect resp.http.backend == "s3" + expect resp.http.backend == resp.http.bereq-now-1 + expect resp.http.backend != resp.http.bereq-now-mod + expect resp.http.backend == resp.http.bereq-now-2 txreq -url /1 -hdr "layered: true" rxresp @@ -139,6 +159,9 @@ client c1 { expect resp.http.director == "ll" expect resp.http.backend == "s1" expect resp.http.backend == resp.http.backend-now + expect resp.http.backend == resp.http.bereq-now-1 + expect resp.http.backend != resp.http.bereq-now-mod + expect resp.http.backend == resp.http.bereq-now-2 txreq -url /2 -hdr "layered: true" rxresp @@ -147,6 +170,9 @@ client c1 { expect resp.http.director == "ll" expect resp.http.backend == "s2" expect resp.http.backend == resp.http.backend-now + expect resp.http.backend == resp.http.bereq-now-1 + expect resp.http.backend != resp.http.bereq-now-mod + expect resp.http.backend == resp.http.bereq-now-2 txreq -url /3 -hdr "layered: true" rxresp @@ -155,6 +181,9 @@ client c1 { expect resp.http.director == "ll" expect resp.http.backend == "s3" expect resp.http.backend == resp.http.backend-now + expect resp.http.backend == resp.http.bereq-now-1 + expect resp.http.backend != resp.http.bereq-now-mod + expect resp.http.backend == resp.http.bereq-now-2 txreq -url /1 -hdr "resolve: true" rxresp @@ -163,6 +192,9 @@ client c1 { expect resp.http.director == "s1" expect resp.http.backend == "s1" expect resp.http.backend == resp.http.backend-now + expect resp.http.backend == resp.http.bereq-now-1 + expect resp.http.backend != resp.http.bereq-now-mod + expect resp.http.backend == resp.http.bereq-now-2 txreq -url /2 -hdr "resolve: true" rxresp @@ -171,6 +203,9 @@ client c1 { expect resp.http.director == "s2" expect resp.http.backend == "s2" expect resp.http.backend == resp.http.backend-now + expect resp.http.backend == resp.http.bereq-now-1 + expect resp.http.backend != resp.http.bereq-now-mod + expect resp.http.backend == resp.http.bereq-now-2 txreq -url /3 -hdr "resolve: true" rxresp @@ -179,4 +214,7 @@ client c1 { expect resp.http.director == "s3" expect resp.http.backend == "s3" expect resp.http.backend == resp.http.backend-now + expect resp.http.backend == resp.http.bereq-now-1 + expect resp.http.backend != resp.http.bereq-now-mod + expect resp.http.backend == resp.http.bereq-now-2 } -run diff --git a/vmod/vmod_directors_shard.c b/vmod/vmod_directors_shard.c index 2a214279d..1d0ce41f7 100644 --- a/vmod/vmod_directors_shard.c +++ b/vmod/vmod_directors_shard.c @@ -615,6 +615,13 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, else resolve = VENUM(NOW); + if (ctx->method & SHARD_VCL_TASK_BEREQ) { + pp = shard_param_task_l(ctx, shardd, shardd->name, + shardd->param); + if (pp == NULL) + return (NULL); + } + if (resolve == VENUM(LAZY)) { if ((args & ~arg_resolve) == 0) { AN(vshard->dir); @@ -642,13 +649,6 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, WRONG("resolve enum"); } - if (ctx->method & SHARD_VCL_TASK_BEREQ) { - pp = shard_param_task_l(ctx, shardd, shardd->name, - shardd->param); - if (pp == NULL) - return (NULL); - } - AN(pp); if (args & arg_param) { From nils.goroll at uplex.de Mon Jan 18 15:18:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 18 Jan 2021 15:18:07 +0000 (UTC) Subject: [master] e7b720ead shard director: Use VRT_priv_task_get() properly Message-ID: <20210118151807.B9AD01185A0@lists.varnish-cache.org> commit e7b720eadba23b5b7d544de8163087d8400a714e Author: Nils Goroll Date: Mon Jan 18 14:27:40 2021 +0100 shard director: Use VRT_priv_task_get() properly This is how 08b642d70ff15346eb1818171862f70a3a9266af should have looked like. diff --git a/vmod/vmod_directors_shard.c b/vmod/vmod_directors_shard.c index 1d0ce41f7..77fab676a 100644 --- a/vmod/vmod_directors_shard.c +++ b/vmod/vmod_directors_shard.c @@ -615,13 +615,6 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, else resolve = VENUM(NOW); - if (ctx->method & SHARD_VCL_TASK_BEREQ) { - pp = shard_param_task_l(ctx, shardd, shardd->name, - shardd->param); - if (pp == NULL) - return (NULL); - } - if (resolve == VENUM(LAZY)) { if ((args & ~arg_resolve) == 0) { AN(vshard->dir); @@ -635,6 +628,11 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, "context"); return (NULL); } + + pp = shard_param_task_l(ctx, shardd, shardd->name, + shardd->param); + if (pp == NULL) + return (NULL); } else if (resolve == VENUM(NOW)) { if (ctx->method & VCL_MET_TASK_H) { shard_fail(ctx, shardd->name, "%s", @@ -642,9 +640,10 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, "used in vcl_init{}/vcl_fini{}"); return (NULL); } - pp = shard_param_stack(&pstk, - pp != NULL ? pp : shardd->param, - shardd->name); + ppt = shard_param_task_r(ctx, shardd, shardd->name, + shardd->param); + AN(ppt); + pp = shard_param_stack(&pstk, ppt, shardd->name); } else { WRONG("resolve enum"); } From nils.goroll at uplex.de Mon Jan 18 15:18:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 18 Jan 2021 15:18:07 +0000 (UTC) Subject: [master] 59d4d33f2 shard director: coverage Message-ID: <20210118151807.EADDA1185AD@lists.varnish-cache.org> commit 59d4d33f29bf9b41eec7da29ad24b2875a2eb6fd Author: Nils Goroll Date: Mon Jan 18 14:44:23 2021 +0100 shard director: coverage diff --git a/bin/varnishtest/tests/d00015.vtc b/bin/varnishtest/tests/d00015.vtc index c283732be..e0b7395f3 100644 --- a/bin/varnishtest/tests/d00015.vtc +++ b/bin/varnishtest/tests/d00015.vtc @@ -130,6 +130,9 @@ varnish v1 -vcl+backend { return(fail("reconfigure failed")); } + vd.set_warmup(1); + vd.set_warmup(-0.001); + std.log("-- END"); ## gcov - semantic testing via the client request & .list @@ -268,8 +271,10 @@ logexpect l1 -v v1 -g raw -d 1 { expect 0 0 Debug {^vmod_directors: shard:.*point = d51dafe6, host = 0} expect 0 0 Debug {^vmod_directors: shard:.*point = eb74a7d5, host = 4} expect 0 0 Debug {^vmod_directors: shard:.*point = f493ce58, host = 2} + expect 0 0 Notice {^\Qvmod_directors: shard vd: .set_warmup(1.000000) ignored\E$} + expect 0 0 Notice {^\Qvmod_directors: shard vd: .set_warmup(-0.001000) ignored\E$} -expect 0 0 VCL_Log {^-- END$} + expect 0 0 VCL_Log {^-- END$} } -start client c1 { diff --git a/bin/varnishtest/tests/d00020.vtc b/bin/varnishtest/tests/d00020.vtc index 5e99f3758..877672f06 100644 --- a/bin/varnishtest/tests/d00020.vtc +++ b/bin/varnishtest/tests/d00020.vtc @@ -85,6 +85,7 @@ varnish v1 -vcl+backend { if (bereq.url ~ "^/b/c/hash/") { set beresp.http.override = bereq.url; p_def.set(by=HASH, alt=7); + p_hash.clear(); p_hash.set(by=HASH, alt=8); p_url.set(by=HASH, alt=9); p_key.set(by=HASH, alt=10); diff --git a/bin/varnishtest/tests/d00021.vtc b/bin/varnishtest/tests/d00021.vtc index 730c32705..2ed251a46 100644 --- a/bin/varnishtest/tests/d00021.vtc +++ b/bin/varnishtest/tests/d00021.vtc @@ -36,6 +36,7 @@ varnish v1 -vcl+backend { new l = directors.shard(); new lp = directors.shard_param(); + l.associate(); l.associate(lp.use()); if (!l.add_backend(s1)) { return(fail("add s1 failed")); diff --git a/bin/varnishtest/tests/d00030.vtc b/bin/varnishtest/tests/d00030.vtc index 070dfa74b..c458a6b37 100644 --- a/bin/varnishtest/tests/d00030.vtc +++ b/bin/varnishtest/tests/d00030.vtc @@ -4,7 +4,7 @@ varnish v1 -vcl { import directors; import blob; - backend dummy { .host = "${bad_backend}"; } + backend dummy None; sub vcl_init { new shard = directors.shard(); @@ -55,7 +55,7 @@ varnish v1 -errvcl {shard .associate param invalid} { import directors; import blob; - backend dummy { .host = "${bad_backend}"; } + backend dummy None; sub vcl_init { new shard = directors.shard(); @@ -67,7 +67,7 @@ varnish v1 -errvcl {missing key argument with by=KEY} { import directors; import blob; - backend dummy { .host = "${bad_backend}"; } + backend dummy None; sub vcl_init { new p = directors.shard_param(); @@ -79,7 +79,7 @@ varnish v1 -errvcl {invalid key argument -5 with by=KEY} { import directors; import blob; - backend dummy { .host = "${bad_backend}"; } + backend dummy None; sub vcl_init { new p = directors.shard_param(); @@ -91,7 +91,7 @@ varnish v1 -errvcl {missing key_blob argument with by=BLOB} { import directors; import blob; - backend dummy { .host = "${bad_backend}"; } + backend dummy None; sub vcl_init { new p = directors.shard_param(); @@ -103,7 +103,7 @@ varnish v1 -errvcl {key and key_blob arguments are invalid with by=URL} { import directors; import blob; - backend dummy { .host = "${bad_backend}"; } + backend dummy None; sub vcl_init { new p = directors.shard_param(); @@ -115,7 +115,7 @@ varnish v1 -errvcl {key and key_blob arguments are invalid with by=HASH} { import directors; import blob; - backend dummy { .host = "${bad_backend}"; } + backend dummy None; sub vcl_init { new p = directors.shard_param(); @@ -127,7 +127,7 @@ varnish v1 -errvcl {invalid alt argument -1} { import directors; import blob; - backend dummy { .host = "${bad_backend}"; } + backend dummy None; sub vcl_init { new p = directors.shard_param(); @@ -139,7 +139,7 @@ varnish v1 -errvcl {invalid warmup argument -0.5} { import directors; import blob; - backend dummy { .host = "${bad_backend}"; } + backend dummy None; sub vcl_init { new p = directors.shard_param(); @@ -151,7 +151,7 @@ varnish v1 -errvcl {invalid warmup argument 1.1} { import directors; import blob; - backend dummy { .host = "${bad_backend}"; } + backend dummy None; sub vcl_init { new p = directors.shard_param(); @@ -163,7 +163,7 @@ varnish v1 -errvcl {vmod_directors: shard shard: .backend(resolve=LAZY) with oth import directors; import blob; - backend dummy { .host = "${bad_backend}"; } + backend dummy None; sub vcl_init { new shard = directors.shard(); @@ -176,7 +176,7 @@ varnish v1 -errvcl {vmod_directors: shard shard: .backend(resolve=NOW) can not b import directors; import blob; - backend dummy { .host = "${bad_backend}"; } + backend dummy None; sub vcl_init { new shard = directors.shard(); @@ -184,3 +184,25 @@ varnish v1 -errvcl {vmod_directors: shard shard: .backend(resolve=NOW) can not b rr.add_backend(shard.backend(resolve=NOW)); } } + +varnish v1 -errvcl {vmod_directors: shard shard: NULL backend cannot be added}{ + import directors; + + backend dummy None; + + sub vcl_init { + new shard = directors.shard(); + shard.add_backend(dummy); + } +} + +varnish v1 -errvcl {vmod_directors: shard shard: .remove_backend(): either backend or ident are required}{ + import directors; + + backend dummy None; + + sub vcl_init { + new shard = directors.shard(); + shard.remove_backend(); + } +} diff --git a/bin/varnishtest/tests/d00041.vtc b/bin/varnishtest/tests/d00041.vtc index 587f1b897..3e108b81b 100644 --- a/bin/varnishtest/tests/d00041.vtc +++ b/bin/varnishtest/tests/d00041.vtc @@ -29,6 +29,9 @@ varnish v1 -vcl+backend { if (!vd.add_backend(s2, weight=2)) { return(fail("add s2 failed")); } + # weight ignored + vd.add_backend(s3, weight=0.3); + vd.remove_backend(s3); if (!vd.add_backend(s3, weight=3)) { return(fail("add s3 failed")); } @@ -49,6 +52,7 @@ varnish v1 -vcl+backend { logexpect l1 -v v1 -g raw -d 1 { expect 0 0 CLI "^Rd vcl.load" + expect 0 = Notice {^\Qvmod_directors: shard vd: .add_backend(weight=0.300000) ignored\E$} expect 0 = Debug {^vmod_directors: shard: hashcircle.* 0. = .point = *238d0ef, host = 2.} expect 0 = Debug {^vmod_directors: shard: hashcircle.* 1. = .point = *321c598, host = 2.} expect 0 = Debug {^vmod_directors: shard: hashcircle.* 2. = .point = *3b6b56a, host = 0.} diff --git a/vmod/vmod_directors_shard.c b/vmod/vmod_directors_shard.c index 77fab676a..242723684 100644 --- a/vmod/vmod_directors_shard.c +++ b/vmod/vmod_directors_shard.c @@ -348,7 +348,7 @@ vmod_shard_remove_backend(VRT_CTX, struct vmod_directors_shard *vshard, if (be == NULL && ident == NULL) { shard_fail(ctx, vshard->shardd->name, "%s", - ".backend_remove(): either backend or ident are required"); + ".remove_backend(): either backend or ident are required"); return (0); } From nils.goroll at uplex.de Mon Jan 18 15:18:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 18 Jan 2021 15:18:08 +0000 (UTC) Subject: [master] 35cadf113 s:NULL backend:None backend: Message-ID: <20210118151808.1AC9B118608@lists.varnish-cache.org> commit 35cadf1132a6934e52895874c5134730276d1268 Author: Nils Goroll Date: Mon Jan 18 14:50:50 2021 +0100 s:NULL backend:None backend: diff --git a/bin/varnishtest/tests/d00030.vtc b/bin/varnishtest/tests/d00030.vtc index c458a6b37..9238f83c0 100644 --- a/bin/varnishtest/tests/d00030.vtc +++ b/bin/varnishtest/tests/d00030.vtc @@ -185,7 +185,7 @@ varnish v1 -errvcl {vmod_directors: shard shard: .backend(resolve=NOW) can not b } } -varnish v1 -errvcl {vmod_directors: shard shard: NULL backend cannot be added}{ +varnish v1 -errvcl {vmod_directors: shard shard: None backend cannot be added}{ import directors; backend dummy None; diff --git a/bin/varnishtest/tests/r02036.vtc b/bin/varnishtest/tests/r02036.vtc index f6bfcfcc3..caa1876e0 100644 --- a/bin/varnishtest/tests/r02036.vtc +++ b/bin/varnishtest/tests/r02036.vtc @@ -7,7 +7,7 @@ server s1 { varnish v1 -vcl+backend {} -start -varnish v1 -errvcl "NULL backend cannot be added" { +varnish v1 -errvcl "None backend cannot be added" { import directors; probe cacheprobe { diff --git a/bin/varnishtest/tests/v00060.vtc b/bin/varnishtest/tests/v00060.vtc index 28bbcb89f..c01abc809 100644 --- a/bin/varnishtest/tests/v00060.vtc +++ b/bin/varnishtest/tests/v00060.vtc @@ -1,4 +1,4 @@ -varnishtest "NULL backend allowed" +varnishtest "None backend allowed" server s1 { rxreq diff --git a/vmod/vmod_directors.c b/vmod/vmod_directors.c index c9ed4fc57..09f18719e 100644 --- a/vmod/vmod_directors.c +++ b/vmod/vmod_directors.c @@ -137,7 +137,7 @@ vdir_add_backend(VRT_CTX, struct vdir *vd, VCL_BACKEND be, double weight) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC); if (be == NULL) { - VRT_fail(ctx, "%s: NULL backend cannot be added", + VRT_fail(ctx, "%s: None backend cannot be added", VRT_BACKEND_string(vd->dir)); return; } @@ -160,7 +160,7 @@ vdir_remove_backend(VRT_CTX, struct vdir *vd, VCL_BACKEND be, unsigned *cur) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vd, VDIR_MAGIC); if (be == NULL) { - VRT_fail(ctx, "%s: NULL backend cannot be removed", + VRT_fail(ctx, "%s: None backend cannot be removed", VRT_BACKEND_string(vd->dir)); return; } diff --git a/vmod/vmod_directors_shard.c b/vmod/vmod_directors_shard.c index 242723684..19f48339b 100644 --- a/vmod/vmod_directors_shard.c +++ b/vmod/vmod_directors_shard.c @@ -319,7 +319,7 @@ vmod_shard_add_backend(VRT_CTX, struct vmod_directors_shard *vshard, if (args->backend == NULL) { shard_fail(ctx, vshard->shardd->name, "%s", - "NULL backend cannot be added"); + "None backend cannot be added"); return (0); } From nils.goroll at uplex.de Mon Jan 18 15:40:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 18 Jan 2021 15:40:07 +0000 (UTC) Subject: [master] d0db4c975 Avoid panic with PRIV_TOP arguments Message-ID: <20210118154007.EC6FC1197AB@lists.varnish-cache.org> commit d0db4c975f281c8d096b48b6fd6014cdfc606ab5 Author: Nils Goroll Date: Mon Jan 18 16:34:48 2021 +0100 Avoid panic with PRIV_TOP arguments Instead of triggering a WRONG() panic, we now fail the VCL when any vmod function with PRIV_TOP argument(s) is present in a sub called from outside client context. Note that the VCL failure happens not when the vmod function with the PRIV_TOP argument is called, but rather when the containing SUB is called. This is because we prepare PRIV_TOP arguments in the function preamble. Fixes #3498 diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index d6884c228..fcdece918 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -199,13 +199,20 @@ VRT_priv_task(VRT_CTX, const void *vmod_id) (uintptr_t)vmod_id)); } +/* + * XXX #3498 on VRT_fail(): Would be better to move the PRIV_TOP check to VCC + * + * This will fail in the preamble of any VCL SUB containing a call to a vmod + * function with a PRIV_TOP argument, which might not exactly be pola + */ + #define VRT_PRIV_TOP_PREP(ctx, req, sp, top) do { \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ req = (ctx)->req; \ if (req == NULL) { \ - WRONG("PRIV_TOP is only accessible " \ + VRT_fail(ctx, "PRIV_TOP is only accessible " \ "in client VCL context"); \ - NEEDLESS(return (NULL)); \ + return (NULL); \ } \ CHECK_OBJ(req, REQ_MAGIC); \ sp = (ctx)->sp; \ diff --git a/bin/varnishtest/tests/v00043.vtc b/bin/varnishtest/tests/v00043.vtc index 828b06668..95b85770b 100644 --- a/bin/varnishtest/tests/v00043.vtc +++ b/bin/varnishtest/tests/v00043.vtc @@ -64,6 +64,8 @@ varnish v1 -cliok "param.set debug +syncvsl" -vcl+backend { } sub vcl_recv { + set req.http.x0 = debug.test_priv_top(req.url + req.esi_level); + # coverage set req.http.x0 = debug.test_priv_top(req.url + req.esi_level); if (req.url == "/foo1") { o.test_priv_top(req.url + req.esi_level); @@ -79,9 +81,28 @@ varnish v1 -cliok "param.set debug +syncvsl" -vcl+backend { set req.http.o2 = o2.test_priv_top(""); } + # XXX because PRIV_TOP arguments get initialized in the + # function preamble, the mere presence of a vmod call with a + # PRIV_TOP argument in a SUB will trigger the failure if that + # sub is called at all. + # + # So to test #3498, we need to fence test_priv_top into its + # own sub + sub callingmewill503 { + debug.test_priv_top("only works on client side"); + } + + sub vcl_backend_fetch { + if (bereq.url == "/fail") { + call callingmewill503; + } + if (bereq.url == "/failo") { + o2.test_priv_top("only works on client side"); + } + } + sub vcl_backend_response { set beresp.do_esi = true; - } sub vcl_deliver { @@ -101,3 +122,18 @@ client c1 { } -run varnish v1 -expect client_req == 2 + +client c1 { + txreq -url /fail + rxresp + expect resp.status == 503 +} -start + +client c2 { + txreq -url /failo + rxresp + expect resp.status == 503 +} -start + +client c1 -wait +client c2 -wait diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst index 48e99a28a..9c3e08317 100644 --- a/doc/sphinx/reference/vmod.rst +++ b/doc/sphinx/reference/vmod.rst @@ -496,11 +496,11 @@ The VCL compiler supports the following private pointers: * ``PRIV_TOP`` "per top-request" private pointers live for the duration of one request and all its ESI-includes. They are only defined for the client side. When used from backend VCL subs, a NULL - pointer will be passed. + pointer will potentially be passed and a VCL failure triggered. These private pointers live only for the duration of their top level request - .. XXX BROKEN https://github.com/varnishcache/varnish-cache/issues/3498 + .. PRIV_TOP see #3498 * ``PRIV_VCL`` "per vcl" private pointers are useful for such global state that applies to all calls in this VCL, for instance flags that From nils.goroll at uplex.de Mon Jan 18 15:46:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 18 Jan 2021 15:46:07 +0000 (UTC) Subject: [master] 43d9e5fb1 add a VRT_CTX argument to the vmod_priv fini function Message-ID: <20210118154607.D085B119B93@lists.varnish-cache.org> commit 43d9e5fb1a10a88ab6a5a98ad4038438025c4999 Author: Nils Goroll Date: Sat Jan 16 11:52:01 2021 +0100 add a VRT_CTX argument to the vmod_priv fini function we take the chance of the breaking change in 681c11998ec56dc4460a30585210f95a7850ea3b to also add a VRT_CTX argument to vmod_priv_fini_f. This allows a vmod_priv fini function, for example, access to the request or simpler and better logging (because the task's vsl buffer can be used where otherwise only unbuffered vsl was possible). Implementation: The meat of this commit really only is the change to the vmod_priv_fini_f typedef and VRT_priv_fini(). All other changes are either to bundled vmods or the straight forward infrastructure to make available a ctx to VRT_priv_fini(), which, for client and backend context, is called via VCL_TaskLeave(). Consequently, it made sense to also change the signature Req_Rollback() and Bereq_Rollback(). diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 0ba276571..9ca855de8 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -134,8 +134,12 @@ vbf_cleanup(struct busyobj *bo) } void -Bereq_Rollback(struct busyobj *bo) +Bereq_Rollback(VRT_CTX) { + struct busyobj *bo; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + bo = ctx->bo; CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); if (bo->htc != NULL && @@ -144,7 +148,7 @@ Bereq_Rollback(struct busyobj *bo) bo->htc->doclose = SC_RESP_CLOSE; vbf_cleanup(bo); - VCL_TaskLeave(bo->privs); + VCL_TaskLeave(ctx, bo->privs); VCL_TaskEnter(bo->privs); HTTP_Clone(bo->bereq, bo->bereq0); bo->vfp_filter_list = NULL; @@ -1006,6 +1010,7 @@ vbf_stp_done(struct worker *wrk, struct busyobj *bo) static void v_matchproto_(task_func_t) vbf_fetch_thread(struct worker *wrk, void *priv) { + struct vrt_ctx ctx[1]; struct busyobj *bo; struct objcore *oc; const struct fetch_step *stp; @@ -1050,7 +1055,9 @@ vbf_fetch_thread(struct worker *wrk, void *priv) assert(bo->director_state == DIR_S_NULL); - VCL_TaskLeave(bo->privs); + INIT_OBJ(ctx, VRT_CTX_MAGIC); + VCL_Bo2Ctx(ctx, bo); + VCL_TaskLeave(ctx, bo->privs); http_Teardown(bo->bereq); http_Teardown(bo->beresp); // can not make assumptions about the number of references here #3434 diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index 2de283512..101bf582c 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -190,11 +190,17 @@ Req_Release(struct req *req) */ void -Req_Rollback(struct req *req) +Req_Rollback(VRT_CTX) { + struct req *req; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + req = ctx->req; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + if (IS_TOPREQ(req)) - VCL_TaskLeave(req->top->privs); - VCL_TaskLeave(req->privs); + VCL_TaskLeave(ctx, req->top->privs); + VCL_TaskLeave(ctx, req->privs); VCL_TaskEnter(req->privs); if (IS_TOPREQ(req)) VCL_TaskEnter(req->top->privs); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 452dc344a..f7d378a78 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -266,6 +266,7 @@ cnt_deliver(struct worker *wrk, struct req *req) static enum req_fsm_nxt v_matchproto_(req_state_f) cnt_vclfail(struct worker *wrk, struct req *req) { + struct vrt_ctx ctx[1]; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -273,7 +274,10 @@ cnt_vclfail(struct worker *wrk, struct req *req) AZ(req->objcore); AZ(req->stale_oc); - Req_Rollback(req); + INIT_OBJ(ctx, VRT_CTX_MAGIC); + VCL_Req2Ctx(ctx, req); + + Req_Rollback(ctx); req->err_code = 503; req->err_reason = "VCL failed"; @@ -1100,6 +1104,7 @@ CNT_Embark(struct worker *wrk, struct req *req) enum req_fsm_nxt CNT_Request(struct req *req) { + struct vrt_ctx ctx[1]; struct worker *wrk; enum req_fsm_nxt nxt; @@ -1141,12 +1146,14 @@ CNT_Request(struct req *req) } wrk->vsl = NULL; if (nxt == REQ_FSM_DONE) { + INIT_OBJ(ctx, VRT_CTX_MAGIC); + VCL_Req2Ctx(ctx, req); if (IS_TOPREQ(req)) { - VCL_TaskLeave(req->top->privs); + VCL_TaskLeave(ctx, req->top->privs); if (req->top->vcl0 != NULL) VCL_Rel(&req->top->vcl0); } - VCL_TaskLeave(req->privs); + VCL_TaskLeave(ctx, req->privs); AN(req->vsl->wid); VRB_Free(req); req->wrk = NULL; diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 489691625..c035264c6 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -255,7 +255,7 @@ enum vbf_fetch_mode_e { void VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, struct objcore *oldoc, enum vbf_fetch_mode_e); const char *VBF_Get_Filter_List(struct busyobj *); -void Bereq_Rollback(struct busyobj *); +void Bereq_Rollback(VRT_CTX); /* cache_fetch_proc.c */ void VFP_Init(void); @@ -362,7 +362,7 @@ void pan_pool(struct vsb *); /* cache_req.c */ struct req *Req_New(const struct worker *, struct sess *); void Req_Release(struct req *); -void Req_Rollback(struct req *req); +void Req_Rollback(VRT_CTX); void Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req); void Req_Fail(struct req *req, enum sess_close reason); void Req_AcctLogCharge(struct VSC_main_wrk *, struct req *); @@ -473,7 +473,7 @@ const struct vrt_backend_probe *VCL_DefaultProbe(const struct vcl *); /* cache_vrt_priv.c */ extern struct vrt_privs cli_task_privs[1]; void VCL_TaskEnter(struct vrt_privs *); -void VCL_TaskLeave(struct vrt_privs *); +void VCL_TaskLeave(VRT_CTX, struct vrt_privs *); /* cache_vrt_vmod.c */ void VMOD_Init(void); diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index f0b035b7b..596a26966 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -215,7 +215,7 @@ vcl_send_event(struct vcl *vcl, enum vcl_event_e ev, struct vsb **msg) VCL_TaskEnter(cli_task_privs); r = ctx->vcl->conf->event_vcl(ctx, ev); - VCL_TaskLeave(cli_task_privs); + VCL_TaskLeave(ctx, cli_task_privs); /* if the warm event did not get to vcl_init, vcl_fini * won't be run, so handling may be zero */ diff --git a/bin/varnishd/cache/cache_vpi.c b/bin/varnishd/cache/cache_vpi.c index e02867521..3b6783ba8 100644 --- a/bin/varnishd/cache/cache_vpi.c +++ b/bin/varnishd/cache/cache_vpi.c @@ -101,7 +101,7 @@ VPI_vcl_select(VRT_CTX, VCL_VCL vcl) if (! IS_TOPREQ(req)) assert(req->vcl == req->top->vcl0); - Req_Rollback(req); + Req_Rollback(ctx); if (IS_TOPREQ(req)) { AN(req->top); diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 67abc24b2..f8b9223a3 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -801,14 +801,13 @@ VRT_Rollback(VRT_CTX, VCL_HTTP hp) } if (hp == ctx->http_req) { CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); - Req_Rollback(ctx->req); + Req_Rollback(ctx); if (ctx->method & VCL_MET_DELIVER) XXXAZ(Resp_Setup_Deliver(ctx->req)); if (ctx->method & VCL_MET_SYNTH) Resp_Setup_Synth(ctx->req); } else if (hp == ctx->http_bereq) { - CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); - Bereq_Rollback(ctx->bo); + Bereq_Rollback(ctx); } else WRONG("VRT_Rollback 'hp' invalid"); } diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index fcdece918..b35477420 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -259,10 +259,12 @@ VRT_priv_top(VRT_CTX, const void *vmod_id) */ void -VRT_priv_fini(const struct vmod_priv *p) +VRT_priv_fini(VRT_CTX, const struct vmod_priv *p) { const struct vmod_priv_methods *m; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + m = p->methods; if (m == NULL) return; @@ -271,7 +273,7 @@ VRT_priv_fini(const struct vmod_priv *p) if (p->priv == NULL || m->fini == NULL) return; - m->fini(p->priv); + m->fini(ctx, p->priv); } /*--------------------------------------------------------------------*/ @@ -284,7 +286,7 @@ VCL_TaskEnter(struct vrt_privs *privs) } void -VCL_TaskLeave(struct vrt_privs *privs) +VCL_TaskLeave(VRT_CTX, struct vrt_privs *privs) { struct vrt_priv *vp, *vp1; @@ -295,7 +297,7 @@ VCL_TaskLeave(struct vrt_privs *privs) */ VRBT_FOREACH_SAFE(vp, vrt_privs, privs, vp1) { CHECK_OBJ(vp, VRT_PRIV_MAGIC); - VRT_priv_fini(vp->priv); + VRT_priv_fini(ctx, vp->priv); } ZERO_OBJ(privs, sizeof *privs); } diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c index 6456b1657..39243fdfc 100644 --- a/bin/varnishd/cache/cache_vrt_vmod.c +++ b/bin/varnishd/cache/cache_vrt_vmod.c @@ -165,7 +165,7 @@ VPI_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len, } void -VPI_Vmod_Unload(struct vmod **hdl) +VPI_Vmod_Unload(VRT_CTX, struct vmod **hdl) { struct vmod *v; @@ -173,7 +173,7 @@ VPI_Vmod_Unload(struct vmod **hdl) TAKE_OBJ_NOTNULL(v, hdl, VMOD_MAGIC); - VCL_TaskLeave(cli_task_privs); + VCL_TaskLeave(ctx, cli_task_privs); VCL_TaskEnter(cli_task_privs); #ifndef DONT_DLCLOSE_VMODS diff --git a/bin/varnishtest/tests/v00041.vtc b/bin/varnishtest/tests/v00041.vtc index 03a4448a0..d7cf4652f 100644 --- a/bin/varnishtest/tests/v00041.vtc +++ b/bin/varnishtest/tests/v00041.vtc @@ -131,15 +131,8 @@ logexpect l0 -v v1 -g raw -d 1 -m -q "vxid == 0" { expect 0 = CLI {^Rd debug.listen_address} expect 0 = CLI {^Wr 200} - # 1001/1002 - expect ? = Debug {^priv_task_fini} - expect ? = Debug {^obj_priv_task_fini.*"r1002"} - expect ? = Debug {^priv_task_fini} - expect ? = Debug {^obj_priv_task_fini.*"d1001"} - # ... # 1006 pipe - expect * = Debug {^obj_priv_task_fini.*"p1006"} # vcl_fini expect * = Debug {^vcl1: VCL_EVENT_COLD} @@ -179,6 +172,9 @@ logexpect l1001 -v v1 -g vxid -q "vxid == 1001" { expect 0 = Debug {^objc.priv_task.. =.*"d1001".} expect 0 = RespHeader {^objc: d1001} expect 0 = VCL_return {^deliver} + expect 9 = Timestamp {^Resp} + expect ? = Debug {^priv_task_fini} + expect ? = Debug {^obj_priv_task_fini.*"d1001"} } -start logexpect l1002 -v v1 -g vxid -q "vxid == 1002" { @@ -205,6 +201,9 @@ logexpect l1002 -v v1 -g vxid -q "vxid == 1002" { expect 0 = Debug {^objb.priv_task.. =.*"r1002".} expect 0 = BerespHeader {^objb: r1002} expect 0 = VCL_return {^deliver} + expect 9 = Timestamp {^BerespBody} + expect ? = Debug {^priv_task_fini} + expect ? = Debug {^obj_priv_task_fini.*"r1002"} } -start logexpect l1006 -v v1 -g vxid -q "vxid == 1006" { @@ -236,6 +235,9 @@ logexpect l1006 -v v1 -g vxid -q "vxid == 1006" { expect 0 = ReqHeader {^x0: /pipe bazz /pipe} expect 0 = Debug {^test_priv_task.*update.$} expect 0 = VCL_return {^pipe} + expect 4 = PipeAcct + expect ? = Debug {^priv_task_fini} + expect ? = Debug {^obj_priv_task_fini.*"p1006"} } -start client c1 { diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst index 9c3e08317..f9d49d01c 100644 --- a/doc/sphinx/reference/vmod.rst +++ b/doc/sphinx/reference/vmod.rst @@ -528,7 +528,7 @@ code wants to use them for. ``.methods`` can be an optional pointer to a struct of callbacks:: - typedef void vmod_priv_fini_f(void *); + typedef void vmod_priv_fini_f(VRT_CTX, void *); struct vmod_priv_methods { unsigned magic; @@ -542,15 +542,22 @@ help debugging. ``.fini`` will be called for a non-NULL ``.priv`` of the ``struct vmod_priv`` when the scope ends with that ``.priv`` pointer as its -only argument. +second argument besides a ``VRT_CTX``. The common case where a private data structure is allocated with malloc(3) would look like this:: + static void + myfree(VRT_CTX, void *p) + { + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + free (p); + } + static const struct vmod_priv_methods mymethods[1] = {{ .magic = VMOD_PRIV_METHODS_MAGIC, .type = "mystate", - .fini = free /* free(3) */ + .fini = myfree }}; // .... diff --git a/include/vcc_interface.h b/include/vcc_interface.h index 7303a0544..1e5f9c3cd 100644 --- a/include/vcc_interface.h +++ b/include/vcc_interface.h @@ -55,7 +55,7 @@ void VPI_count(VRT_CTX, unsigned); int VPI_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len, const char *nm, const char *path, const char *file_id, const char *backup); -void VPI_Vmod_Unload(struct vmod **hdl); +void VPI_Vmod_Unload(VRT_CTX, struct vmod **hdl); typedef int acl_match_f(VRT_CTX, const VCL_IP); diff --git a/include/vrt.h b/include/vrt.h index 70fb7e36b..20782fe45 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -580,7 +580,7 @@ void VRT_Format_Proxy(struct vsb *, VCL_INT, VCL_IP, VCL_IP, VCL_STRING); typedef int vmod_event_f(VRT_CTX, struct vmod_priv *, enum vcl_event_e); /* vmod_priv related */ -typedef void vmod_priv_fini_f(void *); +typedef void vmod_priv_fini_f(VRT_CTX, void *); struct vmod_priv_methods { unsigned magic; @@ -595,7 +595,7 @@ struct vmod_priv { const struct vmod_priv_methods *methods; }; -void VRT_priv_fini(const struct vmod_priv *p); +void VRT_priv_fini(VRT_CTX, const struct vmod_priv *p); struct vmod_priv *VRT_priv_task(VRT_CTX, const void *vmod_id); struct vmod_priv *VRT_priv_task_get(VRT_CTX, const void *vmod_id); struct vmod_priv *VRT_priv_top(VRT_CTX, const void *vmod_id); diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 1d766ba52..65d62b24f 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -368,7 +368,7 @@ vcc_priv_arg(struct vcc *tl, const char *p, const struct symbol *sym) bprintf(buf, "vmod_priv_%u", tl->unique++); ifp = New_IniFin(tl); Fh(tl, 0, "static struct vmod_priv %s;\n", buf); - VSB_printf(ifp->fin, "\tVRT_priv_fini(&%s);", buf); + VSB_printf(ifp->fin, "\tVRT_priv_fini(ctx, &%s);", buf); return (vcc_mk_expr(VOID, "&%s", buf)); } diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index 039d39787..04e4375d2 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -436,8 +436,8 @@ vcc_ParseImport(struct vcc *tl) VSB_cat(tl->symtab, " }"); /* XXX: zero the function pointer structure ?*/ - VSB_printf(ifp->fin, "\t\tVRT_priv_fini(&vmod_priv_%.*s);", PF(mod)); - VSB_printf(ifp->final, "\t\tVPI_Vmod_Unload(&VGC_vmod_%.*s);", PF(mod)); + VSB_printf(ifp->fin, "\t\tVRT_priv_fini(ctx, &vmod_priv_%.*s);", PF(mod)); + VSB_printf(ifp->final, "\t\tVPI_Vmod_Unload(ctx, &VGC_vmod_%.*s);", PF(mod)); vj = vjsn_parse(vmd->json, &p); XXXAZ(p); diff --git a/vmod/vmod_cookie.c b/vmod/vmod_cookie.c index 2a7807766..98e4bc44c 100644 --- a/vmod/vmod_cookie.c +++ b/vmod/vmod_cookie.c @@ -71,10 +71,11 @@ struct vmod_cookie { }; static void -cobj_free(void *p) +cobj_free(VRT_CTX, void *p) { struct vmod_cookie *vcp; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CAST_OBJ_NOTNULL(vcp, p, VMOD_COOKIE_MAGIC); FREE_OBJ(vcp); } diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index 1223d11b4..965393f64 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -230,12 +230,13 @@ xyzzy_author(VRT_CTX, VCL_ENUM person, VCL_ENUM someone) #define AN1(x) AN(x) #define PRIV_FINI(name, assert) \ static void v_matchproto_(vmod_priv_fini_f) \ -priv_ ## name ## _fini(void *ptr) \ +priv_ ## name ## _fini(VRT_CTX, void *ptr) \ { \ const char * const fmt = "priv_" #name "_fini(%p)"; \ \ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ AN ## assert (ptr); \ - VSL(SLT_Debug, 0, fmt, (char *)ptr); \ + mylog(ctx->vsl, SLT_Debug, fmt, (char *)ptr); \ free(ptr); \ } \ \ @@ -407,7 +408,7 @@ xyzzy_fail2(VRT_CTX) } static void v_matchproto_(vmod_priv_fini_f) -priv_vcl_fini(void *priv) +priv_vcl_fini(VRT_CTX, void *priv) { struct priv_vcl *priv_vcl; @@ -416,7 +417,7 @@ priv_vcl_fini(void *priv) free(priv_vcl->foo); if (priv_vcl->obj_cb != 0) { ObjUnsubscribeEvents(&priv_vcl->obj_cb); - VSL(SLT_Debug, 0, "Unsubscribed from Object Events"); + VSLb(ctx->vsl, SLT_Debug, "Unsubscribed from Object Events"); } AZ(priv_vcl->vclref_discard); AZ(priv_vcl->vclref_cold); @@ -1194,12 +1195,15 @@ xyzzy_client_port(VRT_CTX) return (SES_Get_String_Attr(ctx->sp, SA_CLIENT_PORT)); } +void * fail_magic = &fail_magic; + static void -fail_f(void *priv) +fail_f(VRT_CTX, void *priv) { - VRT_CTX; - CAST_OBJ_NOTNULL(ctx, priv, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + assert(priv == fail_magic); + VRT_fail(ctx, "thou shalt not rollet back"); } @@ -1223,12 +1227,12 @@ xyzzy_fail_rollback(VRT_CTX) } if (p->priv != NULL) { - assert(p->priv == ctx); + assert(p->priv == fail_magic); assert(p->methods == xyzzy_fail_rollback_methods); return; } - p->priv = TRUST_ME(ctx); + p->priv = fail_magic; p->methods = xyzzy_fail_rollback_methods; } diff --git a/vmod/vmod_debug_obj.c b/vmod/vmod_debug_obj.c index 266f4be6a..11c499997 100644 --- a/vmod/vmod_debug_obj.c +++ b/vmod/vmod_debug_obj.c @@ -155,12 +155,13 @@ xyzzy_obj_test_priv_vcl(VRT_CTX, #define PRIV_FINI(name) \ static void v_matchproto_(vmod_priv_fini_f) \ -obj_priv_ ## name ## _fini(void *ptr) \ +obj_priv_ ## name ## _fini(VRT_CTX, void *ptr) \ { \ const char * const fmt = "obj_priv_" #name "_fini(%p = \"%s\")"; \ \ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ AN(ptr); \ - VSL(SLT_Debug, 0, fmt, ptr, (char *)ptr); \ + mylog(ctx->vsl, SLT_Debug, fmt, ptr, (char *)ptr); \ } \ \ static const struct vmod_priv_methods \ diff --git a/vmod/vmod_directors_shard_cfg.c b/vmod/vmod_directors_shard_cfg.c index 680a8cf04..6ba55b692 100644 --- a/vmod/vmod_directors_shard_cfg.c +++ b/vmod/vmod_directors_shard_cfg.c @@ -76,7 +76,7 @@ struct backend_reconfig { /* forward decl */ static VCL_BOOL -change_reconfigure(struct shard_change *change, VCL_INT replicas); +change_reconfigure(VRT_CTX, struct shard_change *change, VCL_INT replicas); /* * ============================================================ @@ -87,7 +87,7 @@ change_reconfigure(struct shard_change *change, VCL_INT replicas); */ static void v_matchproto_(vmod_priv_fini_f) -shard_change_fini(void * priv) +shard_change_fini(VRT_CTX, void * priv) { struct shard_change *change; @@ -96,7 +96,7 @@ shard_change_fini(void * priv) CAST_OBJ_NOTNULL(change, priv, SHARD_CHANGE_MAGIC); - (void) change_reconfigure(change, 67); + (void) change_reconfigure(ctx, change, 67); } static const struct vmod_priv_methods shard_change_priv_methods[1] = {{ @@ -632,7 +632,7 @@ shardcfg_apply_change(struct vsl_log *vsl, struct sharddir *shardd, */ static VCL_BOOL -change_reconfigure(struct shard_change *change, VCL_INT replicas) +change_reconfigure(VRT_CTX, struct shard_change *change, VCL_INT replicas) { struct sharddir *shardd; @@ -646,7 +646,7 @@ change_reconfigure(struct shard_change *change, VCL_INT replicas) sharddir_wrlock(shardd); - shardcfg_apply_change(change->vsl, shardd, change, replicas); + shardcfg_apply_change(ctx->vsl, shardd, change, replicas); shard_change_finish(change); if (shardd->hashcircle) @@ -654,7 +654,7 @@ change_reconfigure(struct shard_change *change, VCL_INT replicas) shardd->hashcircle = NULL; if (shardd->n_backend == 0) { - shard_err0(change->vsl, shardd->name, + shard_err0(ctx->vsl, shardd->name, ".reconfigure() no backends"); sharddir_unlock(shardd); return (0); @@ -681,7 +681,7 @@ shardcfg_reconfigure(VRT_CTX, struct sharddir *shardd, VCL_INT replicas) if (change == NULL) return (0); - return (change_reconfigure(change, replicas)); + return (change_reconfigure(ctx, change, replicas)); } /* diff --git a/vmod/vmod_std_fileread.c b/vmod/vmod_std_fileread.c index 1da8b670b..bd0f0fbc8 100644 --- a/vmod/vmod_std_fileread.c +++ b/vmod/vmod_std_fileread.c @@ -64,10 +64,11 @@ static VTAILQ_HEAD(, frfile) frlist = VTAILQ_HEAD_INITIALIZER(frlist); static pthread_mutex_t frmtx = PTHREAD_MUTEX_INITIALIZER; static void -free_frfile(void *ptr) +fini_frfile(VRT_CTX, void *ptr) { struct frfile *frf; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CAST_OBJ_NOTNULL(frf, ptr, CACHED_FILE_MAGIC); AZ(pthread_mutex_lock(&frmtx)); @@ -86,7 +87,7 @@ free_frfile(void *ptr) static const struct vmod_priv_methods frfile_methods[1] = {{ .magic = VMOD_PRIV_METHODS_MAGIC, .type = "vmod_std_fileread", - .fini = free_frfile + .fini = fini_frfile }}; static struct frfile * From nils.goroll at uplex.de Mon Jan 18 15:46:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 18 Jan 2021 15:46:07 +0000 (UTC) Subject: [master] 656f77146 additional sanity checks while the priv fini change soaks in Message-ID: <20210118154607.E8F14119B96@lists.varnish-cache.org> commit 656f77146540cff79e345e5a8ebfa19b866e2705 Author: Nils Goroll Date: Mon Nov 30 18:37:14 2020 +0100 additional sanity checks while the priv fini change soaks in diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index b35477420..b265e061b 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -273,6 +273,9 @@ VRT_priv_fini(VRT_CTX, const struct vmod_priv *p) if (p->priv == NULL || m->fini == NULL) return; + // XXX remove me after soak in + VRT_CTX_Assert(ctx); + m->fini(ctx, p->priv); } From dridi.boukelmoune at gmail.com Mon Jan 18 16:06:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 18 Jan 2021 16:06:07 +0000 (UTC) Subject: [master] 1000b772c varnishncsa: -E implies -c Message-ID: <20210118160607.2B87811A685@lists.varnish-cache.org> commit 1000b772c2c18c9e5e4d787589357cec1edaaf61 Author: Dridi Boukelmoune Date: Fri Nov 13 14:49:07 2020 +0100 varnishncsa: -E implies -c Otherwise the combination of -E and -b only outputs backend requests. diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 9078eed92..1bdf8fb09 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -1157,7 +1157,7 @@ main(int argc, char * const *argv) } } /* default is client mode: */ - if (!CTX.b_opt) + if (!CTX.b_opt || CTX.E_opt) CTX.c_opt = 1; if (optind != argc) diff --git a/bin/varnishncsa/varnishncsa_options.h b/bin/varnishncsa/varnishncsa_options.h index fb9ab474d..8c86761ed 100644 --- a/bin/varnishncsa/varnishncsa_options.h +++ b/bin/varnishncsa/varnishncsa_options.h @@ -70,13 +70,13 @@ " backend requests will trigger log lines." \ ) #define NCSA_OPT_c \ - VOPT("c", "[-c]", "Client mode", \ + VOPT("c", "[-c]", "Client mode", \ "Log client requests. This is the default. If -b is" \ " specified, then -c is needed to also log client requests" \ ) #define NCSA_OPT_E \ - VOPT("E", "[-E]", "Show ESI request", \ - "Show ESI request." \ + VOPT("E", "[-E]", "Show ESI requests", \ + "Show ESI requests, implies client mode." \ ) NCSA_OPT_a diff --git a/bin/varnishtest/tests/e00003.vtc b/bin/varnishtest/tests/e00003.vtc index c93c6aec1..1e73e27df 100644 --- a/bin/varnishtest/tests/e00003.vtc +++ b/bin/varnishtest/tests/e00003.vtc @@ -1,6 +1,5 @@ varnishtest "ESI include" - server s1 { rxreq expect req.http.esi0 == "foo" @@ -92,3 +91,19 @@ logexpect l2 -wait logexpect l3 -wait logexpect l4 -wait logexpect l5 -wait + +shell { + varnishncsa -n ${v1_name} -d -b -E \ + -F '%{Varnish:vxid}x %{Varnish:side}x %{VSL:Begin[3]}x' | + sort > ncsa.txt + + cat >expected.txt <<-EOF + 1001 c rxreq + 1002 b fetch + 1003 c esi + 1004 b fetch + 1005 c rxreq + 1006 c esi + EOF + diff -u expected.txt ncsa.txt +} From dridi.boukelmoune at gmail.com Mon Jan 18 16:06:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 18 Jan 2021 16:06:07 +0000 (UTC) Subject: [master] ae72e904d varnishncsa: Pass the -b and -c options downstream Message-ID: <20210118160607.41EE111A688@lists.varnish-cache.org> commit ae72e904d208997a2e2c514466dea9dc19baf204 Author: Dridi Boukelmoune Date: Fri Nov 13 15:33:33 2020 +0100 varnishncsa: Pass the -b and -c options downstream diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 1bdf8fb09..1d8fb46dd 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -1160,6 +1160,12 @@ main(int argc, char * const *argv) if (!CTX.b_opt || CTX.E_opt) CTX.c_opt = 1; + if (CTX.b_opt) + AN(VUT_Arg(vut, 'b', NULL)); + + if (CTX.c_opt) + AN(VUT_Arg(vut, 'c', NULL)); + if (optind != argc) VUT_Usage(vut, &vopt_spec, 1); From dridi.boukelmoune at gmail.com Mon Jan 18 16:06:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 18 Jan 2021 16:06:07 +0000 (UTC) Subject: [master] 8a9610c8d vsl: Only collect candidate transactions Message-ID: <20210118160607.627EA11A68C@lists.varnish-cache.org> commit 8a9610c8dc1f8b1f2ea7c1702f00df79295cdfe5 Author: Dridi Boukelmoune Date: Fri Nov 13 16:21:52 2020 +0100 vsl: Only collect candidate transactions For high-throughput scenarios we might be collecting irrelevant transactions, especially sessions that are never buffered, and drastically increase the risk of overruns with long-running sessions (typically when varnishd is behind a load balancer or a TLS terminator that we can entrust with high client connection reuse). The -b and -c options have an effect on the output, which is way too late for such a setup. These options primarily work with the MSB backend and client markers but that is not enough to rule out sessions since they are marked as client transactions. The same goes for incomplete transactions for which we are lacking a Begin tag, that would still be ruled out at the output stage. We also need to collect irrelevant transactions for the sake of grouping, because even when we do not wish to output them we have to maintain the hierarchy. Therefore, we can identify candidate transactions early and not waste memory footprint and churn keeping track of transactions that would have no effect besides increasing the risk of overruns. diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c index a6d226c4e..acb295a83 100644 --- a/lib/libvarnishapi/vsl_dispatch.c +++ b/lib/libvarnishapi/vsl_dispatch.c @@ -1276,15 +1276,54 @@ vslq_shmref_check(struct VSLQ *vslq) return (vsl_more); } +static unsigned +vslq_candidate(struct VSLQ *vslq, const uint32_t *ptr) +{ + enum VSL_transaction_e type; + enum VSL_reason_e reason; + struct VSL_data *vsl; + enum VSL_tag_e tag; + unsigned p_vxid; + int i; + + CHECK_OBJ_NOTNULL(vslq, VSLQ_MAGIC); + AN(ptr); + + assert(vslq->grouping != VSL_g_raw); + if (vslq->grouping != VSL_g_vxid) + return (1); + + vsl = vslq->vsl; + CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC); + if (!vsl->c_opt && !vsl->b_opt) + return (1); + + tag = VSL_TAG(ptr); + assert(tag == SLT_Begin); + i = vtx_parse_link(VSL_CDATA(ptr), &type, &p_vxid, &reason); + + if (i != 3 || type == VSL_t_unknown) + return (0); + if (vsl->c_opt && !vsl->b_opt && !VSL_CLIENT(ptr)) + return (0); + if (vsl->b_opt && !vsl->c_opt && !VSL_BACKEND(ptr)) + return (0); + if (type == VSL_t_sess) + return (0); + + return (1); +} + /* Process next input record */ static enum vsl_status vslq_next(struct VSLQ *vslq) { + const uint32_t *ptr; struct VSL_cursor *c; enum vsl_status r; enum VSL_tag_e tag; ssize_t len; - unsigned vxid; + unsigned vxid, keep; struct vtx *vtx; c = vslq->c; @@ -1301,10 +1340,12 @@ vslq_next(struct VSLQ *vslq) c->rec.ptr; if (len == 0) return (r); - tag = (enum VSL_tag_e)VSL_TAG(VSL_NEXT(c->rec.ptr)); + ptr = VSL_NEXT(c->rec.ptr); + tag = (enum VSL_tag_e)VSL_TAG(ptr); } else { vxid = VSL_ID(c->rec.ptr); len = VSL_NEXT(c->rec.ptr) - c->rec.ptr; + ptr = c->rec.ptr; } assert(len > 0); if (vxid == 0) @@ -1312,11 +1353,13 @@ vslq_next(struct VSLQ *vslq) return (r); vtx = vtx_lookup(vslq, vxid); - if (vtx == NULL && tag == SLT_Begin) { + keep = tag != SLT_Begin || vslq_candidate(vslq, ptr); + if (vtx == NULL && tag == SLT_Begin && keep) { vtx = vtx_add(vslq, vxid); AN(vtx); } if (vtx != NULL) { + AN(keep); r = vtx_append(vslq, vtx, &c->rec, len); if (r == vsl_more) vtx_scan(vslq, vtx); From dridi.boukelmoune at gmail.com Mon Jan 18 16:06:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 18 Jan 2021 16:06:07 +0000 (UTC) Subject: [master] cea4eb2d0 vsl: Plug leak when initial reset fails Message-ID: <20210118160607.82D4811A690@lists.varnish-cache.org> commit cea4eb2d0452acf6f946d8a5b02d7e77ce4c6df0 Author: Dridi Boukelmoune Date: Tue Nov 17 10:32:18 2020 +0100 vsl: Plug leak when initial reset fails diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c index 687d618e0..6b1284667 100644 --- a/lib/libvarnishapi/vsl_cursor.c +++ b/lib/libvarnishapi/vsl_cursor.c @@ -295,6 +295,7 @@ VSL_CursorVSM(struct VSL_data *vsl, struct vsm *vsm, unsigned options) r = vslc_vsm_reset(&c->cursor); if (r != vsl_end) { + AZ(VSM_Unmap(vsm, &vf)); (void)vsl_diag(vsl, "Cursor initialization failure (%d)", r); FREE_OBJ(c); return (NULL); From dridi.boukelmoune at gmail.com Mon Jan 18 16:06:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 18 Jan 2021 16:06:07 +0000 (UTC) Subject: [master] 28eaeb90d vut: Print the diagnostics of VSL cursor failures Message-ID: <20210118160607.A114E11A694@lists.varnish-cache.org> commit 28eaeb90d8730c5c10f5ac1a28d6acf85a8513bd Author: Dridi Boukelmoune Date: Tue Nov 17 11:01:32 2020 +0100 vut: Print the diagnostics of VSL cursor failures But paced, because they may repeat in a semi-tight loop. diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c index 24407dc80..4233a3562 100644 --- a/lib/libvarnishapi/vut.c +++ b/lib/libvarnishapi/vut.c @@ -34,6 +34,7 @@ #include "config.h" #include +#include #include #include #include @@ -382,12 +383,34 @@ VUT_Fini(struct VUT **vutp) FREE_OBJ(vut); } +static void +vut_CursorError(struct VUT *vut, vtim_mono *last) +{ + const char *diag; + vtim_mono now; + + CHECK_OBJ_NOTNULL(vut, VUT_MAGIC); + AN(vut->vsl); + AN(last); + + diag = VSL_Error(vut->vsl); + if (diag == NULL) + diag = "Missing diagnostic"; + + now = VTIM_mono(); + if (isnan(*last) || *last + 1 < now) { + fprintf(stderr, "Failed to acquire log: %s\n", diag); + *last = now; + } +} + int VUT_Main(struct VUT *vut) { struct VSL_cursor *c; int i = -1; int hascursor = -1; + vtim_mono t_failcursor = NAN; CHECK_OBJ_NOTNULL(vut, VUT_MAGIC); AN(vut->vslq); @@ -442,6 +465,7 @@ VUT_Main(struct VUT *vut) (vut->d_opt ? VSL_COPT_TAILSTOP : VSL_COPT_TAIL) | VSL_COPT_BATCH); if (c == NULL) { + vut_CursorError(vut, &t_failcursor); VSL_ResetError(vut->vsl); continue; } From dridi.boukelmoune at gmail.com Mon Jan 18 16:06:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 18 Jan 2021 16:06:07 +0000 (UTC) Subject: [master] d054a37aa vsm: Remove stale comment Message-ID: <20210118160607.BB24711A698@lists.varnish-cache.org> commit d054a37aa18d93d5b84e79b02d1a7615da95d811 Author: Dridi Boukelmoune Date: Wed Nov 18 12:10:00 2020 +0100 vsm: Remove stale comment diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c index 8171983d1..46b558ec7 100644 --- a/bin/varnishd/common/common_vsmw.c +++ b/bin/varnishd/common/common_vsmw.c @@ -29,9 +29,6 @@ * * VSM stuff common to manager and child. * - * Please see comments in for details of protocols and - * data consistency. - * */ #include "config.h" From dridi.boukelmoune at gmail.com Mon Jan 18 16:06:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 18 Jan 2021 16:06:07 +0000 (UTC) Subject: [master] d85bd6fc0 vut: Whitespace OCD Message-ID: <20210118160607.DAB2811A69C@lists.varnish-cache.org> commit d85bd6fc0a7e4c1197ed4403ba0770f9e35d9c9d Author: Dridi Boukelmoune Date: Mon Nov 23 11:45:17 2020 +0100 vut: Whitespace OCD diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c index 4233a3562..3c458f031 100644 --- a/lib/libvarnishapi/vut.c +++ b/lib/libvarnishapi/vut.c @@ -479,8 +479,8 @@ VUT_Main(struct VUT *vut) do i = VSLQ_Dispatch(vut->vslq, vut_dispatch, vut); while (i == vsl_more && - VSIG_usr1 == vut->last_sigusr1 && - VSIG_hup == vut->last_sighup); + VSIG_usr1 == vut->last_sigusr1 && + VSIG_hup == vut->last_sighup); if (i == vsl_more) continue; From dridi.boukelmoune at gmail.com Mon Jan 18 16:06:08 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 18 Jan 2021 16:06:08 +0000 (UTC) Subject: [master] 0275c1996 vtc: Increase varnishncsa -[bcE] test coverage Message-ID: <20210118160608.070F511A6A1@lists.varnish-cache.org> commit 0275c19960427eaf757be9bb061f7ee3a40ae259 Author: Dridi Boukelmoune Date: Mon Nov 23 13:56:25 2020 +0100 vtc: Increase varnishncsa -[bcE] test coverage diff --git a/bin/varnishtest/tests/e00003.vtc b/bin/varnishtest/tests/e00003.vtc index 1e73e27df..11ba3de74 100644 --- a/bin/varnishtest/tests/e00003.vtc +++ b/bin/varnishtest/tests/e00003.vtc @@ -92,6 +92,44 @@ logexpect l3 -wait logexpect l4 -wait logexpect l5 -wait +shell { + varnishncsa -n ${v1_name} -d \ + -F '%{Varnish:vxid}x %{Varnish:side}x %{VSL:Begin[3]}x' | + sort > ncsa.txt + + cat >expected.txt <<-EOF + 1001 c rxreq + 1005 c rxreq + EOF + diff -u expected.txt ncsa.txt +} + +shell { + varnishncsa -n ${v1_name} -d -b \ + -F '%{Varnish:vxid}x %{Varnish:side}x %{VSL:Begin[3]}x' | + sort > ncsa.txt + + cat >expected.txt <<-EOF + 1002 b fetch + 1004 b fetch + EOF + diff -u expected.txt ncsa.txt +} + +shell { + varnishncsa -n ${v1_name} -d -E \ + -F '%{Varnish:vxid}x %{Varnish:side}x %{VSL:Begin[3]}x' | + sort > ncsa.txt + + cat >expected.txt <<-EOF + 1001 c rxreq + 1003 c esi + 1005 c rxreq + 1006 c esi + EOF + diff -u expected.txt ncsa.txt +} + shell { varnishncsa -n ${v1_name} -d -b -E \ -F '%{Varnish:vxid}x %{Varnish:side}x %{VSL:Begin[3]}x' | diff --git a/bin/varnishtest/tests/u00003.vtc b/bin/varnishtest/tests/u00003.vtc index 1191f085f..ae81e664c 100644 --- a/bin/varnishtest/tests/u00003.vtc +++ b/bin/varnishtest/tests/u00003.vtc @@ -159,3 +159,5 @@ process p2 {varnishncsa -t 5 -n nonexistent} -start delay 1 process p2 -expect-exit 1 -kill INT -wait shell {grep -q "VSM: Attach interrupted" ${p2_err}} + +# ESI coverage in e00003.vtc diff --git a/bin/varnishtest/tests/u00006.vtc b/bin/varnishtest/tests/u00006.vtc index aef71d393..c331d7ddf 100644 --- a/bin/varnishtest/tests/u00006.vtc +++ b/bin/varnishtest/tests/u00006.vtc @@ -164,3 +164,5 @@ shell -err -expect "Cannot open ${tmpdir}/foo: " \ shell "echo foobar > ${tmpdir}/foo" shell -err -expect "Not a VSL file: ${tmpdir}/foo" \ "varnishlog -r ${tmpdir}/foo" + +# ESI coverage in e00003.vtc From dridi.boukelmoune at gmail.com Mon Jan 18 16:06:08 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 18 Jan 2021 16:06:08 +0000 (UTC) Subject: [master] 53b83f275 vsl: Implement varnishncsa -E in libvarnishapi Message-ID: <20210118160608.29C1311A6B3@lists.varnish-cache.org> commit 53b83f275fa24585eae3905b6aff57ddc55411ff Author: Dridi Boukelmoune Date: Mon Nov 23 14:32:46 2020 +0100 vsl: Implement varnishncsa -E in libvarnishapi This doesn't change the behavior of varnishncsa that already ignores ESI sub-requests unless the -E option is specified. This is however breaking the default behavior of other VSL processing tools that gained the -E option, aligning with varnishncsa's behavior. Except varnishhist, that gained the E filter in its -P argument instead of the -E option. This allows to also not collect ESI transactions in VXID mode to further reduce churn and reduce the risk for overruns in varnishncsa. diff --git a/bin/varnishhist/varnishhist.c b/bin/varnishhist/varnishhist.c index 63bf8e73e..539245305 100644 --- a/bin/varnishhist/varnishhist.c +++ b/bin/varnishhist/varnishhist.c @@ -510,8 +510,8 @@ main(int argc, char **argv) break; } /* else check if valid definition */ - if (colon == optarg + 1 && - (*optarg == 'b' || *optarg == 'c')) { + if (colon == optarg + 1 && (*optarg == 'b' || + *optarg == 'c' || *optarg == 'E')) { cli_p.VSL_arg = *optarg; ptag = colon + 1; colon = strchr(colon + 1, ':'); @@ -605,7 +605,8 @@ main(int argc, char **argv) VUT_Error(vut, 1, "-P: No such profile '%s'", profile); assert(active_profile->VSL_arg == 'b' || - active_profile->VSL_arg == 'c'); + active_profile->VSL_arg == 'c' || + active_profile->VSL_arg == 'E'); assert(VUT_Arg(vut, active_profile->VSL_arg, NULL)); match_tag = active_profile->tag; fnum = active_profile->field; diff --git a/bin/varnishhist/varnishhist_options.h b/bin/varnishhist/varnishhist_options.h index 28d2fc4c7..c575bc85f 100644 --- a/bin/varnishhist/varnishhist_options.h +++ b/bin/varnishhist/varnishhist_options.h @@ -52,12 +52,12 @@ VOPT("P:", "[-P <[cb:]tag:[prefix]:field_num[:min:max]>]", \ "Custom profile definition", \ "Graph the given custom definition defined as: an optional" \ - " (c)lient or (b)ackend filter (defaults to client), the" \ - " tag we'll look for, a prefix to look for (can be empty," \ - " but must be terminated by a colon) and the field number" \ - " of the value we are interested in. min and max are the" \ - " boundaries of the graph in powers of ten and default to" \ - " -6 and 3." \ + " (c)lient, (b)ackend or (E)SI filter (defaults to client),"\ + " the tag we'll look for, a prefix to look for (can be" \ + " empty, but must be terminated by a colon) and the field" \ + " number of the value we are interested in. min and max are"\ + " the boundaries of the graph in powers of ten and default" \ + " to -6 and 3." \ ) #define HIS_OPT_B \ diff --git a/bin/varnishlog/varnishlog_options.h b/bin/varnishlog/varnishlog_options.h index 85394c204..046808663 100644 --- a/bin/varnishlog/varnishlog_options.h +++ b/bin/varnishlog/varnishlog_options.h @@ -62,6 +62,7 @@ VSL_OPT_c VSL_OPT_C VUT_OPT_d VUT_GLOBAL_OPT_D +VSL_OPT_E VUT_OPT_g VUT_OPT_h VSL_OPT_i diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 1d8fb46dd..7cfa347a0 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -1166,6 +1166,9 @@ main(int argc, char * const *argv) if (CTX.c_opt) AN(VUT_Arg(vut, 'c', NULL)); + if (CTX.E_opt) + AN(VUT_Arg(vut, 'E', NULL)); + if (optind != argc) VUT_Usage(vut, &vopt_spec, 1); diff --git a/bin/varnishtest/tests/e00003.vtc b/bin/varnishtest/tests/e00003.vtc index 11ba3de74..96bd91b32 100644 --- a/bin/varnishtest/tests/e00003.vtc +++ b/bin/varnishtest/tests/e00003.vtc @@ -145,3 +145,9 @@ shell { EOF diff -u expected.txt ncsa.txt } + +shell -err {varnishlog -n ${v1_name} -d -c -i Begin | grep esi} +shell -err {varnishlog -n ${v1_name} -d -c -i Begin -g request | grep esi} +shell {varnishlog -n ${v1_name} -d -c -i Begin -g raw | grep esi} +shell {varnishlog -n ${v1_name} -d -E -i Begin | grep esi} +shell {varnishlog -n ${v1_name} -d -E -i Begin | grep rxreq} diff --git a/bin/varnishtop/varnishtop_options.h b/bin/varnishtop/varnishtop_options.h index 7070a4618..c77e48fbc 100644 --- a/bin/varnishtop/varnishtop_options.h +++ b/bin/varnishtop/varnishtop_options.h @@ -67,6 +67,7 @@ VSL_OPT_b VSL_OPT_c VSL_OPT_C TOP_OPT_d +VSL_OPT_E TOP_OPT_f VUT_OPT_g VUT_OPT_h diff --git a/include/vapi/vapi_options.h b/include/vapi/vapi_options.h index ad5fe86a9..498ed08ed 100644 --- a/include/vapi/vapi_options.h +++ b/include/vapi/vapi_options.h @@ -52,6 +52,11 @@ "Do all regular expression and string matching caseless." \ ) +#define VSL_OPT_E \ + VOPT("E", "[-E]", "Display ESI transactions", \ + "Display ESI transactions and other client transactions." \ + ) + #define VSL_OPT_i \ VOPT("i:", "[-i ]", "Include tags", \ "Include log records of these tags in output. Taglist is" \ diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index 761cc4402..681c69985 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -346,6 +346,8 @@ VSL_PrintTransactions(struct VSL_data *vsl, struct VSL_transaction * const pt[], case VSL_t_req: if (!vsl->c_opt) continue; + if (t->reason == VSL_r_esi && !vsl->E_opt) + continue; break; case VSL_t_bereq: if (!vsl->b_opt) diff --git a/lib/libvarnishapi/vsl_api.h b/lib/libvarnishapi/vsl_api.h index 600ee994f..b374be38c 100644 --- a/lib/libvarnishapi/vsl_api.h +++ b/lib/libvarnishapi/vsl_api.h @@ -85,6 +85,7 @@ struct VSL_data { int b_opt; int c_opt; int C_opt; + int E_opt; int L_opt; int R_opt_l; vtim_dur R_opt_p; diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c index eb7985c9f..0ec414164 100644 --- a/lib/libvarnishapi/vsl_arg.c +++ b/lib/libvarnishapi/vsl_arg.c @@ -361,6 +361,10 @@ VSL_Arg(struct VSL_data *vsl, int opt, const char *arg) /* Caseless regular expressions */ vsl->C_opt = 1; return (1); + case 'E': + vsl->E_opt = 1; + vsl->c_opt = 1; + return (1); case 'i': case 'x': return (vsl_ix_arg(vsl, opt, arg)); case 'I': case 'X': return (vsl_IX_arg(vsl, opt, arg)); case 'L': diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c index acb295a83..e4cdbe1e5 100644 --- a/lib/libvarnishapi/vsl_dispatch.c +++ b/lib/libvarnishapi/vsl_dispatch.c @@ -1310,6 +1310,8 @@ vslq_candidate(struct VSLQ *vslq, const uint32_t *ptr) return (0); if (type == VSL_t_sess) return (0); + if (reason == VSL_r_esi && !vsl->E_opt) + return (0); return (1); } From dridi.boukelmoune at gmail.com Mon Jan 18 16:06:08 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 18 Jan 2021 16:06:08 +0000 (UTC) Subject: [master] 12fca0581 varnishncsa: Pass the -E option downstream Message-ID: <20210118160608.4474811A6DA@lists.varnish-cache.org> commit 12fca0581db1acf30de273c6b00ccfb5708bb0ba Author: Dridi Boukelmoune Date: Mon Nov 23 14:53:45 2020 +0100 varnishncsa: Pass the -E option downstream And remove [bcE]_opt handling entirely out of varnishncsa, relying on libvarnishapi to do this filtering. As such we can consolidate the tag management without backend marker fiddling. diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 7cfa347a0..c492f770d 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -140,9 +140,6 @@ VTAILQ_HEAD(vsl_watch_head, vsl_watch); static struct ctx { /* Options */ int a_opt; - int b_opt; - int c_opt; - int E_opt; char *w_arg; FILE *fo; @@ -886,30 +883,20 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], enum VSL_tag_e tag; const char *b, *e, *p; struct watch *w; - int i, skip, be_mark; + int i, skip; (void)vsl; (void)priv; -#define BACKEND_MARKER (INT_MAX / 2 + 1) - assert(BACKEND_MARKER >= VSL_t__MAX); - for (t = pt[0]; t != NULL; t = *++pt) { CTX.gen++; - /* Consider client requests only if in client mode. - Consider backend requests only if in backend mode. */ - if (t->type == VSL_t_req && CTX.c_opt) { + if (t->type == VSL_t_req) { CTX.side = "c"; - be_mark = 0; - } else if (t->type == VSL_t_bereq && CTX.b_opt) { + } else if (t->type == VSL_t_bereq) { CTX.side = "b"; - be_mark = BACKEND_MARKER; } else - continue; - if (t->reason == VSL_r_esi && !CTX.E_opt) - /* Skip ESI requests */ - continue; + WRONG("unexpected"); CTX.hitmiss = "-"; CTX.handling = "-"; CTX.vxid = t->vxid; @@ -924,8 +911,7 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], while (e > b && e[-1] == '\0') e--; - switch (tag + be_mark) { - case SLT_HttpGarbage + BACKEND_MARKER: + switch (tag) { case SLT_HttpGarbage: skip = 1; break; @@ -935,7 +921,7 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], 4, &CTX.frag[F_O], 0, NULL); break; - case (SLT_BackendOpen + BACKEND_MARKER): + case SLT_BackendOpen: frag_fields(1, b, e, 3, &CTX.frag[F_h], 0, NULL); @@ -945,11 +931,11 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], 1, &CTX.frag[F_h], 0, NULL); break; - case (SLT_BereqMethod + BACKEND_MARKER): + case SLT_BereqMethod: case SLT_ReqMethod: frag_line(0, b, e, &CTX.frag[F_m]); break; - case (SLT_BereqURL + BACKEND_MARKER): + case SLT_BereqURL: case SLT_ReqURL: p = memchr(b, '?', e - b); if (p == NULL) @@ -957,15 +943,15 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], frag_line(0, b, p, &CTX.frag[F_U]); frag_line(0, p, e, &CTX.frag[F_q]); break; - case (SLT_BereqProtocol + BACKEND_MARKER): + case SLT_BereqProtocol: case SLT_ReqProtocol: frag_line(0, b, e, &CTX.frag[F_H]); break; - case (SLT_BerespStatus + BACKEND_MARKER): + case SLT_BerespStatus: case SLT_RespStatus: frag_line(1, b, e, &CTX.frag[F_s]); break; - case (SLT_BereqAcct + BACKEND_MARKER): + case SLT_BereqAcct: case SLT_ReqAcct: frag_fields(0, b, e, 3, &CTX.frag[F_I], @@ -973,7 +959,6 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], 6, &CTX.frag[F_O], 0, NULL); break; - case (SLT_Timestamp + BACKEND_MARKER): case SLT_Timestamp: #define ISPREFIX(a, b, c, d) isprefix(a, strlen(a), b, c, d) if (ISPREFIX("Start:", b, e, &p)) { @@ -993,8 +978,9 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], &CTX.frag[F_ttfb], 0, NULL); } break; - case (SLT_BereqHeader + BACKEND_MARKER): + case SLT_BereqHeader: case SLT_ReqHeader: + process_hdr(&CTX.watch_reqhdr, b, e); if (ISPREFIX("Authorization:", b, e, &p) && ISPREFIX("basic ", p, e, &p)) frag_line(0, p, e, @@ -1004,6 +990,10 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], &CTX.frag[F_host]); #undef ISPREFIX break; + case SLT_BerespHeader: + case SLT_RespHeader: + process_hdr(&CTX.watch_resphdr, b, e); + break; case SLT_VCL_call: if (!strcasecmp(b, "recv")) { CTX.hitmiss = "-"; @@ -1032,7 +1022,6 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], } else if (!strcasecmp(b, "restart")) skip = 1; break; - case (SLT_VCL_Log + BACKEND_MARKER): case SLT_VCL_Log: VTAILQ_FOREACH(w, &CTX.watch_vcl_log, list) { CHECK_OBJ_NOTNULL(w, WATCH_MAGIC); @@ -1047,13 +1036,6 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], break; } - if ((tag == SLT_ReqHeader && CTX.c_opt) || - (tag == SLT_BereqHeader && CTX.b_opt)) - process_hdr(&CTX.watch_reqhdr, b, e); - else if ((tag == SLT_RespHeader && CTX.c_opt) || - (tag == SLT_BerespHeader && CTX.b_opt)) - process_hdr(&CTX.watch_resphdr, b, e); - process_vsl(&CTX.watch_vsl, tag, b, e); } if (skip) @@ -1098,6 +1080,7 @@ main(int argc, char * const *argv) { signed char opt; char *format = NULL; + int mode_opt = 0; vut = VUT_InitProg(argc, argv, &vopt_spec); AN(vut); @@ -1119,17 +1102,11 @@ main(int argc, char * const *argv) /* Append to file */ CTX.a_opt = 1; break; - case 'b': - /* backend mode */ - CTX.b_opt = 1; - break; - case 'c': - /* client mode */ - CTX.c_opt = 1; - break; - case 'E': - /* show ESI */ - CTX.E_opt = 1; + case 'b': /* backend mode */ + case 'c': /* client mode */ + case 'E': /* show ESI */ + AN(VUT_Arg(vut, opt, NULL)); + mode_opt = 1; break; case 'F': if (format != NULL) @@ -1156,19 +1133,11 @@ main(int argc, char * const *argv) VUT_Usage(vut, &vopt_spec, 1); } } - /* default is client mode: */ - if (!CTX.b_opt || CTX.E_opt) - CTX.c_opt = 1; - if (CTX.b_opt) - AN(VUT_Arg(vut, 'b', NULL)); - - if (CTX.c_opt) + /* default is client mode: */ + if (!mode_opt) AN(VUT_Arg(vut, 'c', NULL)); - if (CTX.E_opt) - AN(VUT_Arg(vut, 'E', NULL)); - if (optind != argc) VUT_Usage(vut, &vopt_spec, 1); From dridi.boukelmoune at gmail.com Mon Jan 18 16:06:08 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 18 Jan 2021 16:06:08 +0000 (UTC) Subject: [master] fd76408f5 varnishlog: New -u option for unbuffered output Message-ID: <20210118160608.6201011A6F0@lists.varnish-cache.org> commit fd76408f5bee70447ef9b863385832d0b85748bb Author: Dridi Boukelmoune Date: Mon Nov 23 14:57:56 2020 +0100 varnishlog: New -u option for unbuffered output The option was present in libvarnishapi but never exposed. diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c index d9c5170f3..0e049f003 100644 --- a/bin/varnishlog/varnishlog.c +++ b/bin/varnishlog/varnishlog.c @@ -60,6 +60,7 @@ static struct log { /* Options */ int a_opt; int A_opt; + int u_opt; char *w_arg; /* State */ @@ -74,7 +75,7 @@ openout(int append) if (LOG.A_opt) LOG.fo = fopen(LOG.w_arg, append ? "a" : "w"); else - LOG.fo = VSL_WriteOpen(vut->vsl, LOG.w_arg, append, 0); + LOG.fo = VSL_WriteOpen(vut->vsl, LOG.w_arg, append, LOG.u_opt); if (LOG.fo == NULL) VUT_Error(vut, 2, "Cannot open output file (%s)", LOG.A_opt ? strerror(errno) : VSL_Error(vut->vsl)); @@ -127,6 +128,10 @@ main(int argc, char * const *argv) case 'h': /* Usage help */ VUT_Usage(vut, &vopt_spec, 0); + case 'u': + /* Unbuffered output */ + LOG.u_opt = 1; + break; case 'w': /* Write to file */ REPLACE(LOG.w_arg, optarg); diff --git a/bin/varnishlog/varnishlog_options.h b/bin/varnishlog/varnishlog_options.h index 046808663..470023060 100644 --- a/bin/varnishlog/varnishlog_options.h +++ b/bin/varnishlog/varnishlog_options.h @@ -44,6 +44,12 @@ " data in ascii format." \ ) +#define LOG_OPT_u \ + VOPT("u", "[-u]", "Unbuffered output", \ + "When writing output to a file with the -w option, output" \ + " data is not buffered." \ + ) + #define LOG_OPT_w \ VOPT("w:", "[-w ]", "Output filename", \ "Redirect output to file. The file will be overwritten" \ @@ -77,6 +83,7 @@ VUT_OPT_r VSL_OPT_R VUT_OPT_t VSL_OPT_T +LOG_OPT_u VSL_OPT_v VUT_GLOBAL_OPT_V LOG_OPT_w From dridi.boukelmoune at gmail.com Mon Jan 18 16:06:08 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 18 Jan 2021 16:06:08 +0000 (UTC) Subject: [master] 3b74d51ce vsl: Also skip sessions for request grouping Message-ID: <20210118160608.82FE611A6FB@lists.varnish-cache.org> commit 3b74d51ce31755df536ca25e4cf4f7f58de0ee36 Author: Martin Blix Grydeland Date: Thu Dec 17 08:19:01 2020 +0100 vsl: Also skip sessions for request grouping diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c index e4cdbe1e5..47b100ea4 100644 --- a/lib/libvarnishapi/vsl_dispatch.c +++ b/lib/libvarnishapi/vsl_dispatch.c @@ -1290,27 +1290,31 @@ vslq_candidate(struct VSLQ *vslq, const uint32_t *ptr) AN(ptr); assert(vslq->grouping != VSL_g_raw); - if (vslq->grouping != VSL_g_vxid) - return (1); + if (vslq->grouping == VSL_g_session) + return (1); /* All are needed */ vsl = vslq->vsl; CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC); - if (!vsl->c_opt && !vsl->b_opt) - return (1); + if (vslq->grouping == VSL_g_vxid) { + if (!vsl->c_opt && !vsl->b_opt) + return (1); /* Implies also !vsl->E_opt */ + if (!vsl->b_opt && !VSL_CLIENT(ptr)) + return (0); + if (!vsl->c_opt && !VSL_BACKEND(ptr)) + return (0); + /* Need to parse the Begin tag - fallthrough to below */ + } tag = VSL_TAG(ptr); assert(tag == SLT_Begin); i = vtx_parse_link(VSL_CDATA(ptr), &type, &p_vxid, &reason); - if (i != 3 || type == VSL_t_unknown) return (0); - if (vsl->c_opt && !vsl->b_opt && !VSL_CLIENT(ptr)) - return (0); - if (vsl->b_opt && !vsl->c_opt && !VSL_BACKEND(ptr)) - return (0); + if (type == VSL_t_sess) return (0); - if (reason == VSL_r_esi && !vsl->E_opt) + + if (vslq->grouping == VSL_g_vxid && reason == VSL_r_esi && !vsl->E_opt) return (0); return (1); From dridi.boukelmoune at gmail.com Mon Jan 18 16:52:08 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 18 Jan 2021 16:52:08 +0000 (UTC) Subject: [master] 06f70fc4b varnishncsa: Polish time unit conversions Message-ID: <20210118165208.7084C6CD8@lists.varnish-cache.org> commit 06f70fc4b237bb4f0f26bce13997947547242961 Author: Dridi Boukelmoune Date: Mon Jan 18 17:49:08 2021 +0100 varnishncsa: Polish time unit conversions The time_t t variable is not guaranteed to have a specific size, so instead we use a dedicated long l variable. The explicit cast from double is also restored, spotted by flexelint. Refs #3499 diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index c492f770d..bec52e87e 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -261,6 +261,7 @@ format_time(const struct format *format) char *p; char buf[64]; time_t t; + long l; struct tm tm; CHECK_OBJ_NOTNULL(format, FORMAT_MAGIC); @@ -297,14 +298,14 @@ format_time(const struct format *format) case 'T': AN(format->time_fmt); if (!strcmp(format->time_fmt, "s")) /* same as %T */ - t = t_end - t_start; + l = (long)(t_end - t_start); else if (!strcmp(format->time_fmt, "ms")) - t = (t_end - t_start) * 1e3; + l = (long)((t_end - t_start) * 1e3); else if (!strcmp(format->time_fmt, "us")) /* same as %D */ - t = (t_end - t_start) * 1e6; + l = (long)((t_end - t_start) * 1e6); else WRONG("Unreachable branch"); - AZ(VSB_printf(CTX.vsb, "%d", (int)t)); + AZ(VSB_printf(CTX.vsb, "%ld", l)); break; default: WRONG("Time format specifier"); From nils.goroll at uplex.de Mon Jan 18 17:08:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 18 Jan 2021 17:08:07 +0000 (UTC) Subject: [master] 306b43a59 flexelint 43d9e5fb1a10a88ab6a5a98ad4038438025c4999 Message-ID: <20210118170807.0D0B77660@lists.varnish-cache.org> commit 306b43a59a0fce7299c8c614992034bb0bd11399 Author: Nils Goroll Date: Mon Jan 18 18:07:22 2021 +0100 flexelint 43d9e5fb1a10a88ab6a5a98ad4038438025c4999 forgotten static declaration diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index 965393f64..3b7c1bbb4 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -1195,7 +1195,7 @@ xyzzy_client_port(VRT_CTX) return (SES_Get_String_Attr(ctx->sp, SA_CLIENT_PORT)); } -void * fail_magic = &fail_magic; +static void * fail_magic = &fail_magic; static void fail_f(VRT_CTX, void *priv) From dridi.boukelmoune at gmail.com Mon Jan 18 17:57:06 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 18 Jan 2021 17:57:06 +0000 (UTC) Subject: [master] d5bbb1a92 vsl: Tolerate float LHS with int RHS in queries Message-ID: <20210118175706.BBD7D9CA6@lists.varnish-cache.org> commit d5bbb1a92322232591f48cc491dc7d266cdcfa11 Author: Dridi Boukelmoune Date: Wed Jan 13 12:00:48 2021 +0100 vsl: Tolerate float LHS with int RHS in queries Only if strtoll(3) stumbled upon a VNUM special character. Fixes #3463 diff --git a/bin/varnishtest/tests/r03463.vtc b/bin/varnishtest/tests/r03463.vtc new file mode 100644 index 000000000..9d0818e7f --- /dev/null +++ b/bin/varnishtest/tests/r03463.vtc @@ -0,0 +1,52 @@ +varnishtest "VSL query lenient int comparisons" + +varnish v1 -vcl { + import std; + backend be none; + sub vcl_recv { + if (req.http.skip != "log") { + std.log("float1: 123.456"); + std.log("float2: 123."); + std.log("float3: .456"); + std.log("float4: 12.3e1"); + std.log("float5: e1"); + } + return (synth(200)); + } +} -start + +logexpect l1 -v v1 -q "VCL_Log:float1 >= 123" { + expect 0 1001 Begin rxreq +} -start + +logexpect l2 -v v1 -q "VCL_Log:float2 <= 123" { + expect 0 1001 Begin rxreq +} -start + +logexpect l3 -v v1 -q "VCL_Log:float3 == 0" { + expect 0 1001 Begin rxreq +} -start + +logexpect l4 -v v1 -q "VCL_Log:float4 == 123" { + expect 0 1001 Begin rxreq +} -start + +logexpect l5 -v v1 -q "VCL_Log:float5 != 42 or ReqHeader:skip eq log" { + fail add 1001 Begin rxreq + expect * 1002 Begin rxreq + fail clear +} -start + +client c1 { + txreq + rxresp + + txreq -hdr "skip: log" + rxresp +} -run + +logexpect l1 -wait +logexpect l2 -wait +logexpect l3 -wait +logexpect l4 -wait +logexpect l5 -wait diff --git a/doc/sphinx/reference/vsl-query.rst b/doc/sphinx/reference/vsl-query.rst index 71534a4b0..6cfff6100 100644 --- a/doc/sphinx/reference/vsl-query.rst +++ b/doc/sphinx/reference/vsl-query.rst @@ -246,13 +246,15 @@ The following types of operands are available: A number without any fractional part, valid for the numerical comparison operators. The integer type is used when the operand does - not contain any period (.) characters. + not contain any period (.) nor exponent (e) characters. However if + the record evaluates as a float, only its integral part is used for + the comparison. * Float A number with a fractional part, valid for the numerical comparison operators. The float type is used when the operand does contain a - period (.) character. + period (.) or exponent (e) character. * String @@ -329,7 +331,8 @@ QUERY EXPRESSION EXAMPLES HISTORY ======= -This document was written by Martin Blix Grydeland. +This document was initially written by Martin Blix Grydeland and amended +by others. COPYRIGHT diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c index eb08bf18a..b214a38da 100644 --- a/lib/libvarnishapi/vsl_query.c +++ b/lib/libvarnishapi/vsl_query.c @@ -122,7 +122,7 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec) const struct vex_rhs *rhs; long long lhs_int = 0; double lhs_float = 0.; - const char *b, *e, *q; + const char *b, *e, *q, *t; char *p; int i, dq; @@ -209,6 +209,16 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec) switch (rhs->type) { case VEX_INT: lhs_int = strtoll(b, &p, 0); + if (*p == '.' || *p == 'e') { + t = ""; /* assume valid float */ + lhs_float = VNUMpfx(b, &q); + if (isnan(lhs_float)) + return (0); + if (q != NULL) + t = (q > p) ? q - 1 : q; + p = TRUST_ME(t); + lhs_int = trunc(lhs_float); + } if (*p != '\0' && !isspace(*p)) return (0); /* Can't parse - no match */ break; From phk at FreeBSD.org Tue Jan 19 12:10:08 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 19 Jan 2021 12:10:08 +0000 (UTC) Subject: [master] 21d4a83bf Use the correct name of the VTEST-GCOV indicating variable. Message-ID: <20210119121008.E89E611597D@lists.varnish-cache.org> commit 21d4a83bfbdd296ed71ea9f78450a3c6f02029ab Author: Poul-Henning Kamp Date: Tue Jan 19 12:09:25 2021 +0000 Use the correct name of the VTEST-GCOV indicating variable. Not sure where things went wrong on this... diff --git a/bin/varnishtest/tests/v00004.vtc b/bin/varnishtest/tests/v00004.vtc index 1cac50de0..a5345b9f7 100644 --- a/bin/varnishtest/tests/v00004.vtc +++ b/bin/varnishtest/tests/v00004.vtc @@ -3,7 +3,7 @@ varnishtest "test if our default paramers make sense ..." feature 64bit # Skip this test in GCOV mode, 68xx bytes extra per level makes it fail -feature cmd {test -z "$GCOV_PROG"} +feature cmd {test -z "$GCOVPROG"} # ... for our definition of a standard use case: # - 2019 header madness From nils.goroll at uplex.de Wed Jan 20 14:39:09 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 20 Jan 2021 14:39:09 +0000 (UTC) Subject: [master] 03757a35b rename debug.fail_rollback() & debug.ok_rollback() Message-ID: <20210120143909.7DF2D1178D7@lists.varnish-cache.org> commit 03757a35be7a9587dfa504323606f0d83ef908b3 Author: Nils Goroll Date: Wed Jan 20 10:18:20 2021 +0100 rename debug.fail_rollback() & debug.ok_rollback() Though introduced for the purpose of failing upon rollback, they actually just call VRT_fail() from the PRIV_* fini callback and thus should be named accordingly. diff --git a/bin/varnishtest/tests/m00017.vtc b/bin/varnishtest/tests/m00017.vtc index 012e92989..4e2e8cdd8 100644 --- a/bin/varnishtest/tests/m00017.vtc +++ b/bin/varnishtest/tests/m00017.vtc @@ -215,10 +215,10 @@ varnish v1 -vcl+backend { sub vcl_deliver { if (req.url ~ "/veto") { - debug.fail_rollback(); + debug.fail_task_fini(); } if (req.url ~ "/ok") { - debug.ok_rollback(); + debug.ok_task_fini(); } std.rollback(req); set resp.http.test = req.http.test; diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index 3b7c1bbb4..49dd3276c 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -63,7 +63,7 @@ static struct vsc_seg *vsc_seg = NULL; static struct VSC_debug *vsc = NULL; static int loads; static const int store_ip_token; -static const int fail_rollback_token; +static const int fail_task_fini_token; extern void mylog(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...) v_printflike_(3,4); @@ -1204,23 +1204,23 @@ fail_f(VRT_CTX, void *priv) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); assert(priv == fail_magic); - VRT_fail(ctx, "thou shalt not rollet back"); + VRT_fail(ctx, "thou shalt not fini"); } -static const struct vmod_priv_methods xyzzy_fail_rollback_methods[1] = {{ +static const struct vmod_priv_methods xyzzy_fail_task_fini_methods[1] = {{ .magic = VMOD_PRIV_METHODS_MAGIC, - .type = "debug_fail_rollback", + .type = "debug_fail_task_fini", .fini = fail_f }}; -VCL_VOID v_matchproto_(td_xyzzy_debug_fail_rollback) -xyzzy_fail_rollback(VRT_CTX) +VCL_VOID v_matchproto_(td_xyzzy_debug_fail_task_fini) +xyzzy_fail_task_fini(VRT_CTX) { struct vmod_priv *p; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - p = VRT_priv_task(ctx, &fail_rollback_token); + p = VRT_priv_task(ctx, &fail_task_fini_token); if (p == NULL) { VRT_fail(ctx, "no priv task - out of ws?"); return; @@ -1228,22 +1228,22 @@ xyzzy_fail_rollback(VRT_CTX) if (p->priv != NULL) { assert(p->priv == fail_magic); - assert(p->methods == xyzzy_fail_rollback_methods); + assert(p->methods == xyzzy_fail_task_fini_methods); return; } p->priv = fail_magic; - p->methods = xyzzy_fail_rollback_methods; + p->methods = xyzzy_fail_task_fini_methods; } -VCL_VOID v_matchproto_(td_xyzzy_debug_ok_rollback) -xyzzy_ok_rollback(VRT_CTX) +VCL_VOID v_matchproto_(td_xyzzy_debug_ok_task_fini) +xyzzy_ok_task_fini(VRT_CTX) { struct vmod_priv *p; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - p = VRT_priv_task(ctx, &fail_rollback_token); + p = VRT_priv_task(ctx, &fail_task_fini_token); if (p == NULL) { VRT_fail(ctx, "no priv task - out of ws?"); return; diff --git a/vmod/vmod_debug.vcc b/vmod/vmod_debug.vcc index 3f2e13bcc..8ee1f27cd 100644 --- a/vmod/vmod_debug.vcc +++ b/vmod/vmod_debug.vcc @@ -300,13 +300,13 @@ $Function STRING client_port() Get the stringified client port from the session attr -$Function VOID fail_rollback() +$Function VOID fail_task_fini() -fail any rollback before ok_rollback() is called +fail any task fini before ok_task_fini() is called -$Function VOID ok_rollback() +$Function VOID ok_task_fini() -Allow rollbacks. Must be called before the end of the task. +Allow task_fini. Must be called before the end of the task. $Function STRING re_quote(STRING) From nils.goroll at uplex.de Wed Jan 20 14:39:09 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 20 Jan 2021 14:39:09 +0000 (UTC) Subject: [master] 4ccd356d1 Clarify (struct vrt_ctx).handling for PRIVs Message-ID: <20210120143909.A0A441178DA@lists.varnish-cache.org> commit 4ccd356d1bdcd49af4c76399c492c3d278940324 Author: Nils Goroll Date: Wed Jan 20 14:16:11 2021 +0100 Clarify (struct vrt_ctx).handling for PRIVs Ref 43d9e5fb1a10a88ab6a5a98ad4038438025c4999 : PRIV_* fini methods need to leave (struct vrt_ctx).handling alone, except that they might call VRT_fail(), see also 746384b20cbc24ff8afd2df35e1510087404fbf4 Thus we add assertions that handling be either 0 or VCL_RET_FAIL outside the FSM. To be able to do so, we need to change VCL_RET_OK into 0 when vcl_init{} has returned successfully. The vcl_fini{} case is slightly more complicated: By design, only "ok" (VCL_RET_OK) is allowed, but VRT_fail() also added VCL_RET_FAIL, so we de-facto get a "fail" return if any vmod code called VRT_fail(). Because PRIV_* handling happens from VCC generated code via VGC_Discard(), we need to check and change (struct vrt_ctx).handling right after calling vcl_fini{} / VGC_function_vcl_fini() from VGC_Discard(). This is VPI_vcl_fini(). Implementation note: I also considered void VPI_vcl_fini(VRT_CTX, vcl_func_f fini_sub), having VPI_vcl_fini call the fini_sub, but that stirred up includes of VPI where vcl.h is not included. diff --git a/bin/varnishd/cache/cache_vpi.c b/bin/varnishd/cache/cache_vpi.c index 3b6783ba8..06eb802d5 100644 --- a/bin/varnishd/cache/cache_vpi.c +++ b/bin/varnishd/cache/cache_vpi.c @@ -62,6 +62,26 @@ VPI_count(VRT_CTX, unsigned u) ctx->vcl->conf->ref[u].line, ctx->vcl->conf->ref[u].pos); } +/* + * After vcl_fini {} == VGC_function_vcl_fini() is called from VGC_Discard(), + * handling must either be OK from VCL "return (ok)" or FAIL from VRT_fail(). + * + * replace OK with 0 for _fini callbacks because that handling has meaning only + * when returning from VCL subs + */ + +void +VPI_vcl_fini(VRT_CTX) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(ctx->handling); + + if (*ctx->handling == VCL_RET_FAIL) + return; + assert(*ctx->handling == VCL_RET_OK); + *ctx->handling = 0; +} + VCL_VCL VPI_vcl_get(VRT_CTX, const char *name) { diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index b265e061b..40434dc6e 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -277,6 +277,7 @@ VRT_priv_fini(VRT_CTX, const struct vmod_priv *p) VRT_CTX_Assert(ctx); m->fini(ctx, p->priv); + assert(*ctx->handling == 0 || *ctx->handling == VCL_RET_FAIL); } /*--------------------------------------------------------------------*/ @@ -293,6 +294,10 @@ VCL_TaskLeave(VRT_CTX, struct vrt_privs *privs) { struct vrt_priv *vp, *vp1; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(ctx->handling); + assert(*ctx->handling == 0 || *ctx->handling == VCL_RET_FAIL); + /* * NB: We don't bother removing entries as we finish them because it's * a costly operation. Instead we safely walk the whole tree and clear diff --git a/include/vcc_interface.h b/include/vcc_interface.h index 1e5f9c3cd..865c8504e 100644 --- a/include/vcc_interface.h +++ b/include/vcc_interface.h @@ -52,6 +52,7 @@ struct vpi_ref { }; void VPI_count(VRT_CTX, unsigned); +void VPI_vcl_fini(VRT_CTX); int VPI_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len, const char *nm, const char *path, const char *file_id, const char *backup); diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 6deaff9e4..a7b2a690a 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -364,10 +364,10 @@ EmitInitFini(const struct vcc *tl) Fc(tl, 0, "\n"); Fc(tl, 0, "\tif (*ctx->handling != VCL_RET_OK)\n"); Fc(tl, 0, "\t\treturn(1);\n"); + Fc(tl, 0, "\t*ctx->handling = 0;\n"); VTAILQ_FOREACH(sy, &tl->sym_objects, sideways) { Fc(tl, 0, "\tif (!%s) {\n", sy->rname); - Fc(tl, 0, "\t\t*ctx->handling = 0;\n"); Fc(tl, 0, "\t\tVRT_fail(ctx, " "\"Object %s not initialized\");\n" , sy->name); Fc(tl, 0, "\t\treturn(1);\n"); @@ -741,7 +741,7 @@ vcc_CompileSource(struct vcc *tl, struct source *sp, const char *jfile) * must always be called, also on failure. */ ifp->ignore_errors = 1; - VSB_cat(ifp->fin, "\t\tVGC_function_vcl_fini(ctx);"); + VSB_cat(ifp->fin, "\t\tVGC_function_vcl_fini(ctx);VPI_vcl_fini(ctx);"); /* Emit method functions */ Fh(tl, 1, "\n"); From nils.goroll at uplex.de Wed Jan 20 14:39:09 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 20 Jan 2021 14:39:09 +0000 (UTC) Subject: [master] e6f531bc0 Add missing bits of VRT_fail() handling during vcl_init {} / vcl_fini{} Message-ID: <20210120143909.CD3111178DE@lists.varnish-cache.org> commit e6f531bc0a82d0f143b050687fd08dbfd50b6575 Author: Nils Goroll Date: Wed Jan 20 12:15:21 2021 +0100 Add missing bits of VRT_fail() handling during vcl_init {} / vcl_fini{} This change is based on the precondition that VRT_fail() should work wherever we have a VRT_CTX and DTRT, see also previous. on vcl_init {} -------------- We already handled a "direct" VRT_fail() via a vmod function correctly. This commit adds a test. Yet we can also VRT_fail() via a PRIV_TASK fini callback and did not handle that case (triggered an assertion in VRT_fail()). Notice that this case has worked in client / backend tasks for long and a test case has been added with 746384b20cbc24ff8afd2df35e1510087404fbf4 . It has now gained relevance with 43d9e5fb1a10a88ab6a5a98ad4038438025c4999 on vcl_fini {} -------------- By design, vcl_fini {} must not fail, which is also why the only valid return() value is "ok" (VCL_RET_OK). Consequently, we had if (method && *ctx->handling && *ctx->handling != VCL_RET_OK) { assert(ev == VCL_EVENT_LOAD); in vcl_send_event(). And then came VRT_fail(). If we wanted to sustain an assertion similar to the above, we would need to require vmod authors to never call VRT_fail() from VCL_MET_FINI - which would complicate code, be likely overlooked and receive little to no test coverage. So instead we now ignore errors during vcl_fini {} and emit a VCL_error log line. I considered the alternative to check for VCL_MET_FINI in VRT_fail(), but decided to handle the exception where it applies, which is in vcl_send_event(). I am aware that this part of the change may trigger some controversy and I am open to discussing alternatives. If anything, we now avoid unmotivated assertion failures triggered for the new tests in v00051.vtc for the time being. diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 596a26966..abc41de2c 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -172,6 +172,39 @@ VCL_Rel_CliCtx(struct vrt_ctx **ctx) /*--------------------------------------------------------------------*/ +/* VRT_fail() can be called + * - from the vcl sub via a vmod + * - via a PRIV_TASK .fini callback + * + * if this happens during init, we fail it + * if during fini, we ignore, because otherwise VMOD authors would be forced to + * handle VCL_MET_FINI specifically everywhere. + */ + +static int +vcl_event_handling(VRT_CTX) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + + if (*ctx->handling == 0) + return (0); + + assert(*ctx->handling == VCL_RET_FAIL); + + if (ctx->method == VCL_MET_INIT) + return (1); + + /* + * EVENT_WARM / EVENT_COLD: method == 0 + * must not set handling + */ + assert(ctx->method == VCL_MET_FINI); + + *ctx->handling = 0; + VRT_fail(ctx, "VRT_fail() from vcl_fini{} has no effect"); + return (0); +} + static int vcl_send_event(struct vcl *vcl, enum vcl_event_e ev, struct vsb **msg) { @@ -216,13 +249,7 @@ vcl_send_event(struct vcl *vcl, enum vcl_event_e ev, struct vsb **msg) VCL_TaskEnter(cli_task_privs); r = ctx->vcl->conf->event_vcl(ctx, ev); VCL_TaskLeave(ctx, cli_task_privs); - - /* if the warm event did not get to vcl_init, vcl_fini - * won't be run, so handling may be zero */ - if (method && *ctx->handling && *ctx->handling != VCL_RET_OK) { - assert(ev == VCL_EVENT_LOAD); - r = 1; - } + r |= vcl_event_handling(ctx); *msg = VCL_Rel_CliCtx(&ctx); diff --git a/bin/varnishtest/tests/v00051.vtc b/bin/varnishtest/tests/v00051.vtc index 8948d76d5..913932146 100644 --- a/bin/varnishtest/tests/v00051.vtc +++ b/bin/varnishtest/tests/v00051.vtc @@ -488,3 +488,71 @@ varnish v1 -expect vcl_fail == 14 varnish v1 -expect sc_vcl_failure == 11 logexpect l1 -wait + +####################################################################### +# Fail in vcl_init with direct VRT_fail() + +varnish v1 -errvcl {Forced failure} { + import debug; + backend proforma none; + sub vcl_init { + debug.fail(); + } +} + +####################################################################### +# Fail in vcl_init via fini callback + +varnish v1 -errvcl {thou shalt not fini} { + import debug; + backend proforma none; + sub vcl_init { + debug.fail_task_fini(); + } +} + +####################################################################### +# Fail in vcl_fini with direct VRT_fail() + +varnish v1 -vcl { + import debug; + backend proforma none; + sub vcl_fini { + debug.fail(); + } +} + +varnish v1 -vcl+backend { +} + +logexpect l1 -v v1 -g raw { + expect * 0 CLI "^Rd vcl.state vcl16 0cold" + expect 3 0 VCL_Error "^Forced failure" + expect 0 0 VCL_Error {^\QVRT_fail() from vcl_fini{} has no effect\E$} +} -start + +varnish v1 -cliok "vcl.discard vcl16" +logexpect l1 -wait + +####################################################################### +# Fail in vcl_fini via fini callback - ignored but logged as VMOD BUG + +varnish v1 -vcl { + import debug; + backend proforma none; + sub vcl_fini { + debug.fail_task_fini(); + } +} + +varnish v1 -vcl+backend { +} + +logexpect l1 -v v1 -g raw { + expect * 0 CLI "^Rd vcl.state vcl18 0cold" + expect 3 0 VCL_Error "^thou shalt not fini" + expect 0 0 VCL_Error {^\QVRT_fail() from vcl_fini{} has no effect\E$} +} -start + +varnish v1 -cliok "vcl.discard vcl18" +logexpect l1 -wait From nils.goroll at uplex.de Wed Jan 20 15:05:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 20 Jan 2021 15:05:08 +0000 (UTC) Subject: [master] cccb8c1d4 Flexelint d5bbb1a92322232591f48cc491dc7d266cdcfa11 Message-ID: <20210120150508.B45E81188C0@lists.varnish-cache.org> commit cccb8c1d44daff940e89649be5f69d7b8797ad85 Author: Nils Goroll Date: Wed Jan 20 16:03:45 2021 +0100 Flexelint d5bbb1a92322232591f48cc491dc7d266cdcfa11 We do not need to call trunc() to truncate a double into an integert. Mark the intention for flexelint with a cast. diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c index b214a38da..d6a30d0bb 100644 --- a/lib/libvarnishapi/vsl_query.c +++ b/lib/libvarnishapi/vsl_query.c @@ -217,7 +217,7 @@ vslq_test_rec(const struct vex *vex, const struct VSLC_ptr *rec) if (q != NULL) t = (q > p) ? q - 1 : q; p = TRUST_ME(t); - lhs_int = trunc(lhs_float); + lhs_int = (long long)lhs_float; } if (*p != '\0' && !isspace(*p)) return (0); /* Can't parse - no match */ From dridi at varni.sh Wed Jan 20 15:06:27 2021 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 20 Jan 2021 15:06:27 +0000 Subject: [master] 4ccd356d1 Clarify (struct vrt_ctx).handling for PRIVs In-Reply-To: <20210120143909.A0A441178DA@lists.varnish-cache.org> References: <20210120143909.A0A441178DA@lists.varnish-cache.org> Message-ID: On Wed, Jan 20, 2021 at 2:39 PM Nils Goroll wrote: > > > commit 4ccd356d1bdcd49af4c76399c492c3d278940324 > Author: Nils Goroll > Date: Wed Jan 20 14:16:11 2021 +0100 > > Clarify (struct vrt_ctx).handling for PRIVs > > Ref 43d9e5fb1a10a88ab6a5a98ad4038438025c4999 : > > PRIV_* fini methods need to leave (struct vrt_ctx).handling alone, > except that they might call VRT_fail(), see also > 746384b20cbc24ff8afd2df35e1510087404fbf4 > > Thus we add assertions that handling be either 0 or VCL_RET_FAIL > outside the FSM. > > To be able to do so, we need to change VCL_RET_OK into 0 when > vcl_init{} has returned successfully. > > The vcl_fini{} case is slightly more complicated: > > By design, only "ok" (VCL_RET_OK) is allowed, but VRT_fail() also > added VCL_RET_FAIL, so we de-facto get a "fail" return if any vmod > code called VRT_fail(). > > Because PRIV_* handling happens from VCC generated code via > VGC_Discard(), we need to check and change (struct vrt_ctx).handling > right after calling vcl_fini{} / VGC_function_vcl_fini() from > VGC_Discard(). This is VPI_vcl_fini(). > > Implementation note: > > I also considered void VPI_vcl_fini(VRT_CTX, vcl_func_f fini_sub), > having VPI_vcl_fini call the fini_sub, but that stirred up includes of > VPI where vcl.h is not included. > > diff --git a/bin/varnishd/cache/cache_vpi.c b/bin/varnishd/cache/cache_vpi.c > index 3b6783ba8..06eb802d5 100644 > --- a/bin/varnishd/cache/cache_vpi.c > +++ b/bin/varnishd/cache/cache_vpi.c > @@ -62,6 +62,26 @@ VPI_count(VRT_CTX, unsigned u) > ctx->vcl->conf->ref[u].line, ctx->vcl->conf->ref[u].pos); > } > > +/* > + * After vcl_fini {} == VGC_function_vcl_fini() is called from VGC_Discard(), > + * handling must either be OK from VCL "return (ok)" or FAIL from VRT_fail(). > + * > + * replace OK with 0 for _fini callbacks because that handling has meaning only > + * when returning from VCL subs > + */ > + > +void > +VPI_vcl_fini(VRT_CTX) > +{ > + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); > + AN(ctx->handling); > + > + if (*ctx->handling == VCL_RET_FAIL) > + return; > + assert(*ctx->handling == VCL_RET_OK); > + *ctx->handling = 0; > +} > + > VCL_VCL > VPI_vcl_get(VRT_CTX, const char *name) > { > diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c > index b265e061b..40434dc6e 100644 > --- a/bin/varnishd/cache/cache_vrt_priv.c > +++ b/bin/varnishd/cache/cache_vrt_priv.c > @@ -277,6 +277,7 @@ VRT_priv_fini(VRT_CTX, const struct vmod_priv *p) > VRT_CTX_Assert(ctx); > > m->fini(ctx, p->priv); > + assert(*ctx->handling == 0 || *ctx->handling == VCL_RET_FAIL); > } > > /*--------------------------------------------------------------------*/ > @@ -293,6 +294,10 @@ VCL_TaskLeave(VRT_CTX, struct vrt_privs *privs) > { > struct vrt_priv *vp, *vp1; > > + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); > + AN(ctx->handling); > + assert(*ctx->handling == 0 || *ctx->handling == VCL_RET_FAIL); > + > /* > * NB: We don't bother removing entries as we finish them because it's > * a costly operation. Instead we safely walk the whole tree and clear > diff --git a/include/vcc_interface.h b/include/vcc_interface.h > index 1e5f9c3cd..865c8504e 100644 > --- a/include/vcc_interface.h > +++ b/include/vcc_interface.h > @@ -52,6 +52,7 @@ struct vpi_ref { > }; > > void VPI_count(VRT_CTX, unsigned); > +void VPI_vcl_fini(VRT_CTX); > > int VPI_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len, > const char *nm, const char *path, const char *file_id, const char *backup); > diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c > index 6deaff9e4..a7b2a690a 100644 > --- a/lib/libvcc/vcc_compile.c > +++ b/lib/libvcc/vcc_compile.c > @@ -364,10 +364,10 @@ EmitInitFini(const struct vcc *tl) > Fc(tl, 0, "\n"); > Fc(tl, 0, "\tif (*ctx->handling != VCL_RET_OK)\n"); > Fc(tl, 0, "\t\treturn(1);\n"); > + Fc(tl, 0, "\t*ctx->handling = 0;\n"); > > VTAILQ_FOREACH(sy, &tl->sym_objects, sideways) { > Fc(tl, 0, "\tif (!%s) {\n", sy->rname); > - Fc(tl, 0, "\t\t*ctx->handling = 0;\n"); > Fc(tl, 0, "\t\tVRT_fail(ctx, " > "\"Object %s not initialized\");\n" , sy->name); > Fc(tl, 0, "\t\treturn(1);\n"); > @@ -741,7 +741,7 @@ vcc_CompileSource(struct vcc *tl, struct source *sp, const char *jfile) > * must always be called, also on failure. > */ > ifp->ignore_errors = 1; > - VSB_cat(ifp->fin, "\t\tVGC_function_vcl_fini(ctx);"); > + VSB_cat(ifp->fin, "\t\tVGC_function_vcl_fini(ctx);VPI_vcl_fini(ctx);"); Can we at least format the generated code to be readable? Break line and indent? > /* Emit method functions */ > Fh(tl, 1, "\n"); > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From nils.goroll at uplex.de Wed Jan 20 15:21:53 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 20 Jan 2021 16:21:53 +0100 Subject: [master] 4ccd356d1 Clarify (struct vrt_ctx).handling for PRIVs In-Reply-To: References: <20210120143909.A0A441178DA@lists.varnish-cache.org> Message-ID: <6ba9b3e7-452e-df71-3c74-25b736cf9e98@uplex.de> On 20/01/2021 16:06, Dridi Boukelmoune wrote: >> + VSB_cat(ifp->fin, "\t\tVGC_function_vcl_fini(ctx);VPI_vcl_fini(ctx);"); > Can we at least format the generated code to be readable? > > Break line and indent? yes, but this gets post-processed as \t%s\n, so it would need to be \t\t...;\n\t\t\t... Can do... -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From nils.goroll at uplex.de Wed Jan 20 15:53:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 20 Jan 2021 15:53:07 +0000 (UTC) Subject: [master] 9ac1e8b01 VGC cstyle Message-ID: <20210120155307.44B8111A050@lists.varnish-cache.org> commit 9ac1e8b011b5d3c350ca6fc230fd854d05f76d30 Author: Nils Goroll Date: Wed Jan 20 16:45:29 2021 +0100 VGC cstyle do someone a favor and reformat the VPI_vcl_fini() call. Because ifp->fin basically gets output via "\t%s\n", this triggers my OCD in a some other way ;), but I am not going down that rabbit hole at this point... diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index a7b2a690a..b55de8642 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -741,7 +741,8 @@ vcc_CompileSource(struct vcc *tl, struct source *sp, const char *jfile) * must always be called, also on failure. */ ifp->ignore_errors = 1; - VSB_cat(ifp->fin, "\t\tVGC_function_vcl_fini(ctx);VPI_vcl_fini(ctx);"); + VSB_cat(ifp->fin, "\t\tVGC_function_vcl_fini(ctx);\n"); + VSB_cat(ifp->fin, "\t\t\tVPI_vcl_fini(ctx);"); /* Emit method functions */ Fh(tl, 1, "\n"); From dridi.boukelmoune at gmail.com Wed Jan 20 16:50:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 20 Jan 2021 16:50:07 +0000 (UTC) Subject: [master] ad37cd85a varnishncsa: Skip irrelevant transactions (again) Message-ID: <20210120165007.8E88A5A01@lists.varnish-cache.org> commit ad37cd85ada72a4704aaa7e9489cd7ce79c0e149 Author: Dridi Boukelmoune Date: Wed Jan 20 17:27:46 2021 +0100 varnishncsa: Skip irrelevant transactions (again) This was already the behavior before #3468, but then the filtering was moved to libvarnishapi to prevent varnishncsa from duplicating the logic behind transaction collection. Unfortunately, despite libvarnishapi's effort to not consider a client or backend transaction without a Begin tag as a candidate, a transaction missing one will get a synthetic Begin prior to dispatch. Regardless, that would have only applied to vxid grouping so even if varnishncsa was presented by default with only relevant transactions, it could still see incomplete transactions in request grouping panic for the same reason. Fixes #3501 diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index bec52e87e..21d95e952 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -897,7 +897,8 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], } else if (t->type == VSL_t_bereq) { CTX.side = "b"; } else - WRONG("unexpected"); + continue; + CTX.hitmiss = "-"; CTX.handling = "-"; CTX.vxid = t->vxid; From nils.goroll at uplex.de Thu Jan 21 11:31:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 21 Jan 2021 11:31:07 +0000 (UTC) Subject: [master] b682b6917 take note of an optimization we might want eventually Message-ID: <20210121113107.7074C113703@lists.varnish-cache.org> commit b682b6917506f2590a01a0760babc71ae78a7aae Author: Nils Goroll Date: Thu Jan 21 12:27:51 2021 +0100 take note of an optimization we might want eventually ... which is not important enough for me to tackle it right now, in particular because I really do not understand enough about how this code works yet, e.g. which of the if() blocks can possibly happen in which order. *cough* fsm? *cough* diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index bf75ef217..87cb9d03d 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -627,6 +627,16 @@ ved_gzgz_init(struct vdp_ctx *vdc, void **priv, struct objcore *oc) return (0); } +/* + * XXX: for act == VDP_END || act == VDP_FLUSH, we send a flush more often than + * we need. The VDP_END case would trip our "at most one VDP_END call" assertion + * in VDP_bytes(), but ved_bytes() covers it. + * + * To avoid unnecessary chunks downstream, it would be nice to re-structure the + * code to intendify the last block, send VDP_END/VDP_FLUSH for that one and + * VDP_NULL for anything before it. + */ + static int v_matchproto_(vdp_bytes_f) ved_gzgz_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, const void *ptr, ssize_t len) From nils.goroll at uplex.de Fri Jan 22 17:08:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 22 Jan 2021 17:08:06 +0000 (UTC) Subject: [master] fbba48784 cleanup: do not call a vfp_ctx pointer "bo" Message-ID: <20210122170806.CE94911B310@lists.varnish-cache.org> commit fbba48784613e1acb6a27fb0bc43d503b3369a14 Author: Nils Goroll Date: Fri Jan 22 13:59:24 2021 +0100 cleanup: do not call a vfp_ctx pointer "bo" diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index c035264c6..90fd67532 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -263,8 +263,8 @@ struct vfp_entry *VFP_Push(struct vfp_ctx *, const struct vfp *); enum vfp_status VFP_GetStorage(struct vfp_ctx *, ssize_t *sz, uint8_t **ptr); void VFP_Extend(const struct vfp_ctx *, ssize_t sz, enum vfp_status); void VFP_Setup(struct vfp_ctx *vc, struct worker *wrk); -int VFP_Open(struct vfp_ctx *bo); -uint64_t VFP_Close(struct vfp_ctx *bo); +int VFP_Open(struct vfp_ctx *); +uint64_t VFP_Close(struct vfp_ctx *); extern const struct vfp VFP_gunzip; extern const struct vfp VFP_gzip; From nils.goroll at uplex.de Fri Jan 22 17:08:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 22 Jan 2021 17:08:06 +0000 (UTC) Subject: [master] 50780f307 Fix object access in ESI VFP when there is no object Message-ID: <20210122170807.0B41E11B313@lists.varnish-cache.org> commit 50780f307600fe14c367892bbd3e9c90535b85a2 Author: Nils Goroll Date: Fri Jan 22 17:33:34 2021 +0100 Fix object access in ESI VFP when there is no object We set up fetch filters (VFPs) before initializing a storage object to fetch into. If that fails, we close the filters again. The esi_gzip VFP interacts with the storage object and was missing error handling for the case of a teardown with no storage object. Implementation: We make sure that for the VFP_ERROR case, we do not interact with the storage object (which makes no sense if that is going to be C-4'ed any moment). vfp_vep_callback() keeps its hands off if (struct vef_priv).error is set, so we use that to avoid complications in the code. diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c index 0a0d1e2aa..508d46e26 100644 --- a/bin/varnishd/cache/cache_esi_fetch.c +++ b/bin/varnishd/cache/cache_esi_fetch.c @@ -120,6 +120,11 @@ vfp_esi_end(struct vfp_ctx *vc, struct vef_priv *vef, CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); CHECK_OBJ_NOTNULL(vef, VEF_MAGIC); + if (retval == VFP_ERROR) + vef->error = errno ? errno : EINVAL; + else + assert(retval == VFP_END); + vsb = VEP_Finish(vef->vep); if (vsb != NULL) { @@ -137,7 +142,8 @@ vfp_esi_end(struct vfp_ctx *vc, struct vef_priv *vef, } if (vef->vgz != NULL) { - VGZ_UpdateObj(vc, vef->vgz, VUA_END_GZIP); + if (retval == VFP_END) + VGZ_UpdateObj(vc, vef->vgz, VUA_END_GZIP); if (VGZ_Destroy(&vef->vgz) != VGZ_END) retval = VFP_Error(vc, "ESI+Gzip Failed at the very end"); @@ -292,8 +298,13 @@ vfp_esi_fini(struct vfp_ctx *vc, struct vfp_entry *vfe) CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC); - if (vfe->priv1 != NULL) - (void)vfp_esi_end(vc, vfe->priv1, VFP_ERROR); + if (vfe->priv1 == NULL) + return; + + if (vc->oc->stobj->stevedore == NULL) + errno = ENOMEM; + + (void)vfp_esi_end(vc, vfe->priv1, VFP_ERROR); vfe->priv1 = NULL; } diff --git a/bin/varnishtest/tests/r03502.vtc b/bin/varnishtest/tests/r03502.vtc new file mode 100644 index 000000000..bde14a2ca --- /dev/null +++ b/bin/varnishtest/tests/r03502.vtc @@ -0,0 +1,56 @@ +varnishtest "#3502 Panic in VEP_Finish() for out-of-storage in vbf_beresp2obj()" + +server s1 { + # First consume (almost) all of the storage - the value + # is brittle, see l1 fail + rxreq + expect req.url == /url1 + txresp -bodylen 1048336 + + rxreq + expect req.http.accept-encoding == gzip + txresp -bodylen 1 +} -start + +varnish v1 -arg "-sTransient=default,1M -p debug=+syncvsl -p nuke_limit=0" -vcl+backend { + sub vcl_recv { + if (req.url == "/") { + return (pass); + } + } + sub vcl_backend_response { + set beresp.http.free = storage.Transient.free_space; + if (bereq.url == "/") { + set beresp.do_gzip = true; + set beresp.do_esi = true; + } + } +} -start + +logexpect l1 -v v1 -g vxid -q "vxid == 1004" { + expect 25 1004 VCL_call {^BACKEND_RESPONSE} + expect 0 = BerespHeader {^free:} + expect 0 = VCL_return {^deliver} + expect 0 = Timestamp {^Process} + expect 0 = Filters {^ esi_gzip} + expect 0 = BerespUnset {^Content-Length: 1} + expect 0 = BerespHeader {^Content-Encoding: gzip} + expect 0 = BerespHeader {^Vary: Accept-Encoding} + # Ensure the FetchError is in vbf_beresp2obj() + # not later in the VFP. Otherwise we have too much free_space + fail add = Storage + expect 0 = FetchError {^Could not get storage} + fail clear +} -start + +client c1 { + txreq -url /url1 + rxresp + expect resp.status == 200 + + txreq -hdr "Accept-Encoding: gzip" + # no storage for synth either + expect_close +} -run + +logexpect l1 -wait From nils.goroll at uplex.de Fri Jan 22 18:01:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 22 Jan 2021 18:01:08 +0000 (UTC) Subject: [master] c7523b63e Tigthen #1637 testcase Message-ID: <20210122180108.21B1C6BDD@lists.varnish-cache.org> commit c7523b63ec5c9749a7f4e7cfa625a12d9e8cbc05 Author: Nils Goroll Date: Fri Jan 22 18:57:26 2021 +0100 Tigthen #1637 testcase This is similar to #3502, but the out-of-storage condition happens after the object has been created successfully. Use a logexpect to ensure we test as intended. diff --git a/bin/varnishtest/tests/r01637.vtc b/bin/varnishtest/tests/r01637.vtc index ee0093b3a..117462eea 100644 --- a/bin/varnishtest/tests/r01637.vtc +++ b/bin/varnishtest/tests/r01637.vtc @@ -1,57 +1,53 @@ -varnishtest "do_esi + do_gzip + out of storage: #1637" +varnishtest "do_esi + do_gzip + out of storage in VFP: #1637" + +# see also r03502 for failure case in vbf_beresp2obj() server s1 { # First consume (almost) all of the storage rxreq expect req.url == /url1 - txresp -bodylen 260000 - - rxreq - expect req.url == /url2 - txresp -bodylen 260000 - - rxreq - expect req.url == /url3 - txresp -bodylen 260000 + txresp -bodylen 1040000 rxreq - expect req.url == /url4 - txresp -bodylen 260000 - - rxreq - expect req.url == /url5 + expect req.url == / txresp -bodylen 9000 } -start varnish v1 -arg "-sdefault,1M" -arg "-p nuke_limit=0 -p gzip_level=0" \ -vcl+backend { sub vcl_backend_response { - if (bereq.url == "/url5") { + set beresp.http.free = storage.s0.free_space; + if (bereq.url == "/") { set beresp.do_esi = true; set beresp.do_gzip = true; } } } -start +logexpect l1 -v v1 -g vxid -q "vxid == 1004" { + expect 25 1004 VCL_call {^BACKEND_RESPONSE} + expect 0 = BerespHeader {^free:} + expect 0 = VCL_return {^deliver} + expect 0 = Timestamp {^Process} + expect 0 = Filters {^ esi_gzip} + expect 0 = BerespUnset {^Content-Length:} + expect 0 = BerespHeader {^Content-Encoding: gzip} + expect 0 = BerespHeader {^Vary: Accept-Encoding} + expect 0 = Storage { s0$} + expect 0 = Fetch_Body + expect 0 = FetchError {^Could not get storage} + expect 0 = Gzip + expect 0 = BackendClose +} -start client c1 { txreq -url /url1 rxresp expect resp.status == 200 - txreq -url /url2 - rxresp - expect resp.status == 200 - - txreq -url /url3 - rxresp - expect resp.status == 200 - - txreq -url /url4 - rxresp - expect resp.status == 200 - - txreq -url /url5 + txreq rxresp expect resp.status == 503 } -run + +logexpect l1 -wait diff --git a/bin/varnishtest/tests/r03502.vtc b/bin/varnishtest/tests/r03502.vtc index bde14a2ca..88b56e2ec 100644 --- a/bin/varnishtest/tests/r03502.vtc +++ b/bin/varnishtest/tests/r03502.vtc @@ -1,5 +1,7 @@ varnishtest "#3502 Panic in VEP_Finish() for out-of-storage in vbf_beresp2obj()" +# see also r01637 for failure case in VFP + server s1 { # First consume (almost) all of the storage - the value # is brittle, see l1 fail From nils.goroll at uplex.de Sat Jan 23 09:28:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 23 Jan 2021 09:28:06 +0000 (UTC) Subject: [master] a2c85e397 Polish 50780f307600fe14c367892bbd3e9c90535b85a2 Message-ID: <20210123092806.B5A6E10F8F9@lists.varnish-cache.org> commit a2c85e397e0a1bfba0478cd7c2e180ee1fc7d589 Author: Nils Goroll Date: Sat Jan 23 09:38:26 2021 +0100 Polish 50780f307600fe14c367892bbd3e9c90535b85a2 Do not overwrite an already latched errno. Note that it does not make any difference at the moment because we only ever use ENOMEM Ref #3502 diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c index 508d46e26..0624fefe1 100644 --- a/bin/varnishd/cache/cache_esi_fetch.c +++ b/bin/varnishd/cache/cache_esi_fetch.c @@ -120,10 +120,12 @@ vfp_esi_end(struct vfp_ctx *vc, struct vef_priv *vef, CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); CHECK_OBJ_NOTNULL(vef, VEF_MAGIC); - if (retval == VFP_ERROR) - vef->error = errno ? errno : EINVAL; - else + if (retval == VFP_ERROR) { + if (vef->error == 0) + vef->error = errno ? errno : EINVAL; + } else { assert(retval == VFP_END); + } vsb = VEP_Finish(vef->vep); From nils.goroll at uplex.de Sat Jan 23 09:28:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 23 Jan 2021 09:28:06 +0000 (UTC) Subject: [master] 4bb1a1181 Stabilize v51.vtc Message-ID: <20210123092806.C97A410F8FC@lists.varnish-cache.org> commit 4bb1a11813a5f4cf2c899074d3f40dec860cedf7 Author: Nils Goroll Date: Sat Jan 23 09:49:56 2021 +0100 Stabilize v51.vtc This test fails constantly now on vtest arm / Freebsd 13.0 for timeout. As vcl compilations are probably the most expensive operations of vtcs, we speed up this test by reducing the number of vcls loaded from 19 to 11. We do this by collapsing all of the client side failure tests into a single vcl. Being at it, we also reduce logexpect synchronization points. The number of vcl loads could be reduced further by also collapsing the backend side, but as this would reduce clarity more than for the client side alone, I first wanted to see if this first step brings down tuntime enough. diff --git a/bin/varnishtest/tests/v00051.vtc b/bin/varnishtest/tests/v00051.vtc index 913932146..8b8453b32 100644 --- a/bin/varnishtest/tests/v00051.vtc +++ b/bin/varnishtest/tests/v00051.vtc @@ -9,12 +9,89 @@ server s1 { varnish v1 -vcl+backend { import debug; sub vcl_recv { + # vxid 1009, 1018, 1020, 1022 + if (req.http.foo ~ "^(deliver|hit|miss|hash)") { + return(hash); + } + # vxid 1016 + if (req.http.foo == "purge") { + return(purge); + } + # vxid 1014 + if (req.http.foo == "pass") { + return(pass); + } + # vxid 1012 + if (req.http.foo == "pipe") { + return(pipe); + } + # vxid 1007 + if (req.http.foo == "synth") { + return(synth(748)); + } + # vxid 1001, 1003 if (req.restarts == 0) { debug.fail(); set req.http.not = "Should not happen"; } } + sub vcl_hash { + # vxid 1009 + if (req.http.foo == "hash") { + debug.fail(); + set req.http.not = "Should not happen"; + } + # vxid 1018, 1020, 1022 default lookup + } + sub vcl_hit { + # vxid 1020 + if (req.http.foo == "hit") { + debug.fail(); + set req.http.not = "Should not happen"; + } + } + sub vcl_miss { + # vxid 1018 + if (req.http.foo == "miss") { + debug.fail(); + set req.http.not = "Should not happen"; + } + } + sub vcl_pass { + # vxid 1014 + if (req.http.foo == "pass") { + debug.fail(); + set req.http.not = "Should not happen"; + } + } + sub vcl_pipe { + # vxid 1012 + if (req.http.foo == "pipe") { + debug.fail(); + set req.http.not = "Should not happen"; + } + } + sub vcl_purge { + # vxid 1016 + if (req.http.foo == "purge") { + debug.fail(); + set req.http.not = "Should not happen"; + } + } + sub vcl_deliver { + # vxid 1022 + if (req.http.foo == "deliver") { + debug.fail(); + set req.http.not = "Should not happen"; + } + } sub vcl_synth { + # vxid 1007 + if (resp.status == 748) { + debug.fail(); + set req.http.not = "Should not happen"; + } + # vxid 1001, 1003 if (req.restarts == 0 && req.http.foo == "restart") { return (restart); } @@ -24,7 +101,7 @@ varnish v1 -vcl+backend { ####################################################################### # Fail in vcl_recv -logexpect l1 -v v1 -g raw { +logexpect l1001 -v v1 -g raw { expect * 1001 VCL_call "RECV" expect 0 1001 VCL_Error "Forced failure" expect 0 1001 VCL_return "fail" @@ -40,12 +117,10 @@ client c1 { varnish v1 -expect vcl_fail == 1 varnish v1 -expect sc_vcl_failure == 1 -logexpect l1 -wait - ####################################################################### # Fail in vcl_recv, vcl_synth restarts successfully -logexpect l1 -v v1 -g raw { +logexpect l1003 -v v1 -g raw { expect * 1003 VCL_call "RECV" expect 0 1003 VCL_Error "Forced failure" expect 0 1003 VCL_return "fail" @@ -64,27 +139,10 @@ varnish v1 -expect vcl_fail == 2 # NB: This is correct, req->doclose = SC_VCL_FAILURE latches varnish v1 -expect sc_vcl_failure == 2 -logexpect l1 -wait - ####################################################################### # Fail in vcl_synth -varnish v1 -vcl+backend { - import debug; - sub vcl_recv { - if (req.http.foo == "synth") { - return(synth(748)); - } - } - sub vcl_synth { - if (resp.status == 748) { - debug.fail(); - set req.http.not = "Should not happen"; - } - } -} - -logexpect l1 -v v1 -g raw { +logexpect l1007 -v v1 -g raw { expect * 1007 VCL_call "SYNTH" expect * 1007 VCL_Error "Forced failure" expect 0 1007 VCL_return "fail" @@ -98,22 +156,10 @@ client c1 { varnish v1 -expect vcl_fail == 3 varnish v1 -expect sc_vcl_failure == 3 -logexpect l1 -wait - ####################################################################### # Fail in vcl_hash -varnish v1 -vcl+backend { - import debug; - sub vcl_hash { - if (req.http.foo == "hash") { - debug.fail(); - set req.http.not = "Should not happen"; - } - } -} - -logexpect l1 -v v1 -g raw { +logexpect l1009 -v v1 -g raw { expect * 1009 VCL_call "HASH" expect 0 1009 VCL_Error "Forced failure" expect 0 1009 VCL_return "fail" @@ -129,26 +175,10 @@ client c1 { varnish v1 -expect vcl_fail == 4 varnish v1 -expect sc_vcl_failure == 4 -logexpect l1 -wait - ####################################################################### # Fail in vcl_pipe -varnish v1 -vcl+backend { - import debug; - sub vcl_recv { - if (req.http.foo == "pipe") { return(pipe); } - } - sub vcl_pipe { - if (req.http.foo == "pipe") { - debug.fail(); - set req.http.not = "Should not happen"; - } - } -} - - -logexpect l2 -v v1 -g vxid -q "vxid == 1012" { +logexpect l1012 -v v1 -g vxid -q "vxid == 1012" { expect 0 1012 Begin {^bereq 1011 pipe} expect 0 = BereqMethod {^GET} expect 0 = BereqURL {^/} @@ -162,11 +192,11 @@ logexpect l2 -v v1 -g vxid -q "vxid == 1012" { expect 0 = End } -start -logexpect l3 -v v1 -g vxid -q "vxid == 1011" { +logexpect l1011 -v v1 -g vxid -q "vxid == 1011" { expect 0 1011 Begin {^req 1010 rxreq} expect 0 = Timestamp {^Start: } expect 0 = Timestamp {^Req: } - expect 0 = VCL_use {^vcl4} + expect 0 = VCL_use {^vcl1} expect 0 = ReqStart expect 0 = ReqMethod {^GET} expect 0 = ReqURL {^/} @@ -197,26 +227,10 @@ client c1 { varnish v1 -expect vcl_fail == 5 varnish v1 -expect sc_vcl_failure == 5 -logexpect l2 -wait -logexpect l3 -wait - ####################################################################### # Fail in vcl_pass, no handling in vcl_synth -varnish v1 -vcl+backend { - import debug; - sub vcl_recv { - if (req.http.foo == "pass") { return(pass); } - } - sub vcl_pass { - if (req.http.foo == "pass") { - debug.fail(); - set req.http.not = "Should not happen"; - } - } -} - -logexpect l1 -v v1 -g raw { +logexpect l1014 -v v1 -g raw { expect * 1014 VCL_call "PASS" expect 0 1014 VCL_Error "Forced failure" expect 0 1014 VCL_return "fail" @@ -232,25 +246,10 @@ client c1 { varnish v1 -expect vcl_fail == 6 varnish v1 -expect sc_vcl_failure == 6 -logexpect l1 -wait - ####################################################################### # Fail in vcl_purge -varnish v1 -vcl+backend { - import debug; - sub vcl_recv { - if (req.http.foo == "purge") { return(purge); } - } - sub vcl_purge { - if (req.http.foo == "purge") { - debug.fail(); - set req.http.not = "Should not happen"; - } - } -} - -logexpect l1 -v v1 -g raw { +logexpect l1016 -v v1 -g raw { expect * 1016 VCL_call "PURGE" expect 0 1016 VCL_Error "Forced failure" expect 0 1016 VCL_return "fail" @@ -266,22 +265,10 @@ client c1 { varnish v1 -expect vcl_fail == 7 varnish v1 -expect sc_vcl_failure == 7 -logexpect l1 -wait - ####################################################################### # Fail in vcl_miss -varnish v1 -vcl+backend { - import debug; - sub vcl_miss { - if (req.http.foo == "miss") { - debug.fail(); - set req.http.not = "Should not happen"; - } - } -} - -logexpect l1 -v v1 -g raw { +logexpect l1018 -v v1 -g raw { expect * 1018 VCL_call "MISS" expect 0 1018 VCL_Error "Forced failure" expect 0 1018 VCL_return "fail" @@ -297,22 +284,10 @@ client c1 { varnish v1 -expect vcl_fail == 8 varnish v1 -expect sc_vcl_failure == 8 -logexpect l1 -wait - ####################################################################### # Fail in vcl_hit -varnish v1 -vcl+backend { - import debug; - sub vcl_hit { - if (req.http.foo == "hit") { - debug.fail(); - set req.http.not = "Should not happen"; - } - } -} - -logexpect l1 -v v1 -g raw { +logexpect l1020 -v v1 -g raw { expect * 1020 VCL_call "HIT" expect 0 1020 VCL_Error "Forced failure" expect 0 1020 VCL_return "fail" @@ -328,22 +303,10 @@ client c1 { varnish v1 -expect vcl_fail == 9 varnish v1 -expect sc_vcl_failure == 9 -logexpect l1 -wait - ####################################################################### # Fail in vcl_deliver -varnish v1 -vcl+backend { - import debug; - sub vcl_deliver { - if (req.http.foo == "deliver") { - debug.fail(); - set req.http.not = "Should not happen"; - } - } -} - -logexpect l1 -v v1 -g raw { +logexpect l1022 -v v1 -g raw { expect * 1022 VCL_call "DELIVER" expect 0 1022 VCL_Error "Forced failure" expect 0 1022 VCL_return "fail" @@ -359,7 +322,21 @@ client c1 { varnish v1 -expect vcl_fail == 10 varnish v1 -expect sc_vcl_failure == 10 -logexpect l1 -wait +####################################################################### +#wait for all client side logexpects +####################################################################### + +logexpect l1001 -wait +logexpect l1003 -wait +logexpect l1007 -wait +logexpect l1009 -wait +logexpect l1012 -wait +logexpect l1011 -wait +logexpect l1014 -wait +logexpect l1016 -wait +logexpect l1018 -wait +logexpect l1020 -wait +logexpect l1022 -wait ####################################################################### # Fail in vcl_backend_fetch @@ -526,12 +503,12 @@ varnish v1 -vcl+backend { } logexpect l1 -v v1 -g raw { - expect * 0 CLI "^Rd vcl.state vcl16 0cold" + expect * 0 CLI "^Rd vcl.state vcl8 0cold" expect 3 0 VCL_Error "^Forced failure" expect 0 0 VCL_Error {^\QVRT_fail() from vcl_fini{} has no effect\E$} } -start -varnish v1 -cliok "vcl.discard vcl16" +varnish v1 -cliok "vcl.discard vcl8" logexpect l1 -wait ####################################################################### @@ -549,10 +526,10 @@ varnish v1 -vcl+backend { } logexpect l1 -v v1 -g raw { - expect * 0 CLI "^Rd vcl.state vcl18 0cold" + expect * 0 CLI "^Rd vcl.state vcl10 0cold" expect 3 0 VCL_Error "^thou shalt not fini" expect 0 0 VCL_Error {^\QVRT_fail() from vcl_fini{} has no effect\E$} } -start -varnish v1 -cliok "vcl.discard vcl18" +varnish v1 -cliok "vcl.discard vcl10" logexpect l1 -wait From nils.goroll at uplex.de Sat Jan 23 13:44:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 23 Jan 2021 13:44:07 +0000 (UTC) Subject: [master] 757992390 Fix privs panic formatting Message-ID: <20210123134407.53A96115F9F@lists.varnish-cache.org> commit 7579923908dfadac459ef5a0dc09f510655b265e Author: Nils Goroll Date: Sat Jan 23 14:10:00 2021 +0100 Fix privs panic formatting 55dffc3585685f2a67e3228094aea9ca9d986798 introduced a PAN_dump_struct() call, which implies printing '{' and VSB_indent(vsb, 2). Fix missing '}' and wrong indentation. Also remove redundant "priv" output, which now happens in PAN_dump_struct() Diff best viewed ignoring whitespace (-b option). diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index 40434dc6e..1cd4898c3 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -77,17 +77,19 @@ pan_privs(struct vsb *vsb, const struct vrt_privs *privs) //lint -e{774} if (p == NULL) { // should never happen - VSB_printf(vsb, "priv NULL vmod %jx\n", + VSB_printf(vsb, "NULL vmod %jx\n", (uintmax_t)vp->vmod_id); - continue; + } else { + m = p->methods; + VSB_printf(vsb, + "{p %p l %ld m %p t \"%s\"} vmod %jx\n", + p->priv, p->len, m, + m != NULL ? m->type : "", + (uintmax_t)vp->vmod_id + ); } - m = p->methods; - VSB_printf(vsb, - "priv {p %p l %ld m %p t \"%s\"} vmod %jx\n", - p->priv, p->len, m, - m != NULL ? m->type : "", - (uintmax_t)vp->vmod_id - ); + VSB_indent(vsb, -2); + VSB_cat(vsb, "},\n"); } VSB_indent(vsb, -2); VSB_cat(vsb, "},\n"); From nils.goroll at uplex.de Sat Jan 23 13:44:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 23 Jan 2021 13:44:07 +0000 (UTC) Subject: [master] c13c71f80 Fix pan_top() expected magic Message-ID: <20210123134407.6A249115FA2@lists.varnish-cache.org> commit c13c71f8065659a51fb364bb95153e7cee8abd03 Author: Nils Goroll Date: Sat Jan 23 14:26:53 2021 +0100 Fix pan_top() expected magic A copy-pasta in 55dffc3585685f2a67e3228094aea9ca9d986798 prevented the top request from being dumped. diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 91268deaa..954d2949b 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -411,7 +411,7 @@ pan_busyobj(struct vsb *vsb, const struct busyobj *bo) static void pan_top(struct vsb *vsb, const struct reqtop *top) { - if (PAN_dump_struct(vsb, top, REQ_MAGIC, "top")) + if (PAN_dump_struct(vsb, top, REQTOP_MAGIC, "top")) return; pan_req(vsb, top->topreq); pan_privs(vsb, top->privs); From nils.goroll at uplex.de Sat Jan 23 18:27:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 23 Jan 2021 18:27:07 +0000 (UTC) Subject: [6.3] e1d857150 Fix vrt_priv_dyncmp() to provide total ordering Message-ID: <20210123182707.52E76615A@lists.varnish-cache.org> commit e1d85715065a2d1684d06062a6c059d857c9cb10 Author: Nils Goroll Date: Sat Jan 23 18:51:13 2021 +0100 Fix vrt_priv_dyncmp() to provide total ordering Fixes #3505, see ticket for detailed explanation. diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index 76152cbf3..ad91c01b0 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -96,9 +96,13 @@ VRTPRIV_init(struct vrt_privs *privs) static inline int vrt_priv_dyncmp(const struct vrt_priv *vp1, const struct vrt_priv *vp2) { - if (vp1->vmod_id < vp2->vmod_id || vp1->id < vp2->id) + if (vp1->vmod_id < vp2->vmod_id) return (-1); - if (vp1->vmod_id > vp2->vmod_id || vp1->id > vp2->id) + if (vp1->vmod_id > vp2->vmod_id) + return (1); + if (vp1->id < vp2->id) + return (-1); + if (vp1->id > vp2->id) return (1); return (0); } From dridi.boukelmoune at gmail.com Mon Jan 25 08:51:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 25 Jan 2021 08:51:07 +0000 (UTC) Subject: [master] b8856a121 Add PAN_dump_oneline() Message-ID: <20210125085107.EE5D510DB83@lists.varnish-cache.org> commit b8856a12125767d16362f23b38de32dd181b0c81 Author: Nils Goroll Date: Sat Jan 23 14:38:38 2021 +0100 Add PAN_dump_oneline() adds a variant of PAN_dump_struct() which does not start another indentation level on a new line. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index c7825f8fa..a6bf831be 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -848,9 +848,12 @@ Tlen(const txt t) */ #define W_TIM_real(w) ((w)->lastused = VTIM_real()) -int PAN_dump_struct2(struct vsb *vsb, const void *ptr, +int PAN_dump_struct2(struct vsb *vsb, int block, const void *ptr, const char *smagic, unsigned magic, const char *fmt, ...) - v_printflike_(5,6); + v_printflike_(6,7); -#define PAN_dump_struct(vsb, ptr, magic, ...) \ - PAN_dump_struct2(vsb, ptr, #magic, magic, __VA_ARGS__) +#define PAN_dump_struct(vsb, ptr, magic, ...) \ + PAN_dump_struct2(vsb, 1, ptr, #magic, magic, __VA_ARGS__) + +#define PAN_dump_oneline(vsb, ptr, magic, ...) \ + PAN_dump_struct2(vsb, 0, ptr, #magic, magic, __VA_ARGS__) diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 954d2949b..8331d5f23 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -110,7 +110,7 @@ static const void *already_list[N_ALREADY]; static int already_idx; int -PAN_dump_struct2(struct vsb *vsb, const void *ptr, +PAN_dump_struct2(struct vsb *vsb, int block, const void *ptr, const char *smagic, unsigned magic, const char *fmt, ...) { va_list ap; @@ -125,10 +125,14 @@ PAN_dump_struct2(struct vsb *vsb, const void *ptr, VSB_printf(vsb, " = NULL\n"); return (-1); } - VSB_printf(vsb, " = %p {\n", ptr); + VSB_printf(vsb, " = %p {", ptr); + if (block) + VSB_putc(vsb, '\n'); for (i = 0; i < already_idx; i++) { if (already_list[i] == ptr) { - VSB_cat(vsb, " [Already dumped, see above]\n"); + VSB_cat(vsb, " [Already dumped, see above]"); + if (block) + VSB_putc(vsb, '\n'); VSB_cat(vsb, "},\n"); return (-2); } @@ -138,11 +142,14 @@ PAN_dump_struct2(struct vsb *vsb, const void *ptr, uptr = ptr; if (*uptr != magic) { VSB_printf(vsb, " .magic = 0x%08x", *uptr); - VSB_printf(vsb, " EXPECTED: %s=0x%08x\n", smagic, magic); + VSB_printf(vsb, " EXPECTED: %s=0x%08x", smagic, magic); + if (block) + VSB_putc(vsb, '\n'); VSB_printf(vsb, "}\n"); return (-3); } - VSB_indent(vsb, 2); + if (block) + VSB_indent(vsb, 2); return (0); } diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index 1cd4898c3..3aa166514 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -71,25 +71,23 @@ pan_privs(struct vsb *vsb, const struct vrt_privs *privs) VSB_printf(vsb, "privs = %p {\n", privs); VSB_indent(vsb, 2); VRBT_FOREACH(vp, vrt_privs, privs) { - if (PAN_dump_struct(vsb, vp, VRT_PRIV_MAGIC, "priv")) + if (PAN_dump_oneline(vsb, vp, VRT_PRIV_MAGIC, "priv")) continue; p = vp->priv; //lint -e{774} if (p == NULL) { // should never happen - VSB_printf(vsb, "NULL vmod %jx\n", + VSB_printf(vsb, "NULL vmod %jx},\n", (uintmax_t)vp->vmod_id); - } else { - m = p->methods; - VSB_printf(vsb, - "{p %p l %ld m %p t \"%s\"} vmod %jx\n", - p->priv, p->len, m, - m != NULL ? m->type : "", - (uintmax_t)vp->vmod_id - ); + continue; } - VSB_indent(vsb, -2); - VSB_cat(vsb, "},\n"); + m = p->methods; + VSB_printf(vsb, + "{p %p l %ld m %p t \"%s\"} vmod %jx},\n", + p->priv, p->len, m, + m != NULL ? m->type : "", + (uintmax_t)vp->vmod_id + ); } VSB_indent(vsb, -2); VSB_cat(vsb, "},\n"); From dridi.boukelmoune at gmail.com Mon Jan 25 13:46:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 25 Jan 2021 13:46:07 +0000 (UTC) Subject: [master] 8587fc242 build: Make the recheck target recursive Message-ID: <20210125134607.A1F7B114F45@lists.varnish-cache.org> commit 8587fc2422cfcb68cd219653eda66cfcb0c2999e Author: Dridi Boukelmoune Date: Mon Jan 25 14:43:34 2021 +0100 build: Make the recheck target recursive This way every failure can be tested again without selecting a specific subdirectory. diff --git a/configure.ac b/configure.ac index 3763b3f34..0201b72ec 100644 --- a/configure.ac +++ b/configure.ac @@ -20,6 +20,7 @@ AC_LANG(C) AM_MAINTAINER_MODE([disable]) AM_INIT_AUTOMAKE([1.11 foreign color-tests parallel-tests subdir-objects]) +AM_EXTRA_RECURSIVE_TARGETS([recheck]) AM_SILENT_RULES([yes]) AC_DISABLE_STATIC AC_PROG_LIBTOOL From nils.goroll at uplex.de Mon Jan 25 17:34:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 25 Jan 2021 17:34:07 +0000 (UTC) Subject: [master] 903fa456c Address two Coverity issues with a signed to unsigned wrapper Message-ID: <20210125173407.2F64711AC68@lists.varnish-cache.org> commit 903fa456cd54985e340f00c74f3a1e6236d2ac97 Author: Nils Goroll Date: Sun Jan 24 16:07:16 2021 +0100 Address two Coverity issues with a signed to unsigned wrapper Simply take an ssize_t and return size_t asserting that the value is greater than or equal to zero. The name might still not be optimimal, as well as choice of *size_t. Addresses Coverity CID 1472401, CID 1472402 diff --git a/bin/varnishtest/vtc_gzip.c b/bin/varnishtest/vtc_gzip.c index 9615efdfd..c06e0a66e 100644 --- a/bin/varnishtest/vtc_gzip.c +++ b/bin/varnishtest/vtc_gzip.c @@ -54,6 +54,13 @@ vtc_report_gz_bits(struct vtclog *vl, const z_stream *vz) } #endif +static size_t +APOS(ssize_t sz) +{ + assert(sz >= 0); + return (sz); +} + static struct vsb * vtc_gzip_vsb(struct vtclog *vl, int fatal, int gzip_level, const struct vsb *vin, int *residual) { @@ -68,7 +75,7 @@ vtc_gzip_vsb(struct vtclog *vl, int fatal, int gzip_level, const struct vsb *vin AN(vout); vz.next_in = (void*)VSB_data(vin); - vz.avail_in = VSB_len(vin); + vz.avail_in = APOS(VSB_len(vin)); assert(Z_OK == deflateInit2(&vz, gzip_level, Z_DEFLATED, 31, 9, Z_DEFAULT_STRATEGY)); @@ -114,10 +121,10 @@ vtc_gzip(struct http *hp, const char *input, char **body, long *bodylen) "Wrong gzip residual got %d wanted %d", res, hp->gzipresidual); #endif - *body = malloc(VSB_len(vout) + 1); + *body = malloc(APOS(VSB_len(vout) + 1)); AN(*body); - memcpy(*body, VSB_data(vout), VSB_len(vout) + 1); - *bodylen = VSB_len(vout); + memcpy(*body, VSB_data(vout), APOS(VSB_len(vout) + 1)); + *bodylen = APOS(VSB_len(vout)); VSB_destroy(&vout); vtc_log(hp->vl, 3, "new bodylen %ld", *bodylen); vtc_dump(hp->vl, 4, "body", *body, *bodylen); @@ -137,7 +144,7 @@ vtc_gunzip_vsb(struct vtclog *vl, int fatal, const struct vsb *vin) AN(vout); vz.next_in = (void*)VSB_data(vin); - vz.avail_in = VSB_len(vin); + vz.avail_in = APOS(VSB_len(vin)); assert(Z_OK == inflateInit2(&vz, 31)); @@ -177,8 +184,8 @@ vtc_gunzip(struct http *hp, char *body, long *bodylen) vout = vtc_gunzip_vsb(hp->vl, hp->fatal, vin); VSB_destroy(&vin); - memcpy(body, VSB_data(vout), VSB_len(vout) + 1); - *bodylen = VSB_len(vout); + memcpy(body, VSB_data(vout), APOS(VSB_len(vout) + 1)); + *bodylen = APOS(VSB_len(vout)); VSB_destroy(&vout); vtc_log(hp->vl, 3, "new bodylen %ld", *bodylen); vtc_dump(hp->vl, 4, "body", body, *bodylen); From dridi.boukelmoune at gmail.com Mon Jan 25 18:18:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 25 Jan 2021 18:18:07 +0000 (UTC) Subject: [master] 5bf609bdb vcc: Cross reference VMOD symbols Message-ID: <20210125181807.5B9C811BE41@lists.varnish-cache.org> commit 5bf609bdb3e1dd8bfb51d56acd45523cab2a6d7f Author: Dridi Boukelmoune Date: Tue Jan 19 06:49:41 2021 +0100 vcc: Cross reference VMOD symbols With a default read mask based on the type of symbol. Starting with a default comprehensive mask should then help implement the $Restrict stanza from VIP4, simply by AND-ing the masks. diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 65d62b24f..e5373b5b4 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -455,14 +455,14 @@ vcc_do_arg(struct vcc *tl, struct func_arg *fa) static void vcc_func(struct vcc *tl, struct expr **e, const void *priv, - const char *extra, const struct symbol *sym) + const char *extra, struct symbol *sym) { vcc_type_t rfmt; const char *cfunc; struct expr *e1; struct func_arg *fa, *fa2; VTAILQ_HEAD(,func_arg) head; - struct token *t1; + struct token *tf, *t1; const struct vjsn_val *vv, *vvp; const char *sa, *extra_sep; char ssa[64]; @@ -492,6 +492,7 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv, AN((*e)->instance); extra = (*e)->instance->rname; } + tf = VTAILQ_PREV(tl->t, tokenhead, list); SkipToken(tl, '('); if (extra == NULL) { extra = ""; @@ -631,6 +632,7 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv, *e = vcc_expr_edit(tl, e1->fmt, "\v1\v-\n)", e1, NULL); } SkipToken(tl, ')'); + vcc_AddUses(tl, tf, NULL, sym, XREF_READ); } diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index 04e4375d2..4ecd217c6 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -97,6 +97,7 @@ func_sym(struct vcc *tl, vcc_kind_t kind, const struct symbol *psym, if (kind == SYM_OBJECT) { sym->eval_priv = v; sym->vmod_name = psym->vmod_name; + sym->r_methods = VCL_MET_INIT; vcc_VmodObject(tl, sym); vcc_VmodSymbols(tl, sym); return; @@ -118,6 +119,7 @@ func_sym(struct vcc *tl, vcc_kind_t kind, const struct symbol *psym, assert(v->type == VJSN_STRING); sym->type = VCC_Type(v->value); AN(sym->type); + sym->r_methods = VCL_MET_TASK_C|VCL_MET_TASK_B|VCL_MET_TASK_H; } static void From dridi.boukelmoune at gmail.com Mon Jan 25 18:18:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 25 Jan 2021 18:18:07 +0000 (UTC) Subject: [master] 6d49b18f1 vcc: Restrict PRIV_TOP arguments to client tasks Message-ID: <20210125181807.82DF711BE44@lists.varnish-cache.org> commit 6d49b18f1a7ad612b3c9ba6b3a8a5704fe84ea61 Author: Dridi Boukelmoune Date: Tue Jan 19 06:53:19 2021 +0100 vcc: Restrict PRIV_TOP arguments to client tasks Refs #3498 diff --git a/bin/varnishtest/tests/v00043.vtc b/bin/varnishtest/tests/v00043.vtc index 95b85770b..1a3d9e0e7 100644 --- a/bin/varnishtest/tests/v00043.vtc +++ b/bin/varnishtest/tests/v00043.vtc @@ -55,6 +55,22 @@ server s1 { txresp } -start +varnish v1 -errvcl "Not available in subroutine 'vcl_backend_fetch'" { + import debug; + backend be none; + sub vcl_backend_fetch { + debug.test_priv_top("only works on client side"); + } +} + +varnish v1 -errvcl "Not available in subroutine 'vcl_init'" { + import debug; + backend be none; + sub vcl_init { + debug.test_priv_top("only works on client side"); + } +} + varnish v1 -cliok "param.set debug +syncvsl" -vcl+backend { import debug; @@ -81,22 +97,9 @@ varnish v1 -cliok "param.set debug +syncvsl" -vcl+backend { set req.http.o2 = o2.test_priv_top(""); } - # XXX because PRIV_TOP arguments get initialized in the - # function preamble, the mere presence of a vmod call with a - # PRIV_TOP argument in a SUB will trigger the failure if that - # sub is called at all. - # - # So to test #3498, we need to fence test_priv_top into its - # own sub - sub callingmewill503 { - debug.test_priv_top("only works on client side"); - } - sub vcl_backend_fetch { if (bereq.url == "/fail") { - call callingmewill503; - } - if (bereq.url == "/failo") { + # dynamic priv not checked at compile time o2.test_priv_top("only works on client side"); } } @@ -123,17 +126,8 @@ client c1 { varnish v1 -expect client_req == 2 -client c1 { - txreq -url /fail - rxresp - expect resp.status == 503 -} -start - client c2 { - txreq -url /failo + txreq -url /fail rxresp expect resp.status == 503 -} -start - -client c1 -wait -client c2 -wait +} -run diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index dd94de055..d4b1ea8db 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -341,7 +341,7 @@ sym_expr_t vcc_Eval_Handle; sym_expr_t vcc_Eval_SymFunc; sym_expr_t vcc_Eval_TypeMethod; void vcc_Eval_Func(struct vcc *, const struct vjsn_val *, - const char *, const struct symbol *); + const char *, struct symbol *); void VCC_GlobalSymbol(struct symbol *, vcc_type_t fmt, const char *pfx); struct symbol *VCC_HandleSymbol(struct vcc *, vcc_type_t , const char *); void VCC_SymName(struct vsb *, const struct symbol *); diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index e5373b5b4..06d2c7d75 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -351,7 +351,7 @@ vcc_Eval_Var(struct vcc *tl, struct expr **e, struct token *t, */ static struct expr * -vcc_priv_arg(struct vcc *tl, const char *p, const struct symbol *sym) +vcc_priv_arg(struct vcc *tl, const char *p, struct symbol *sym) { char buf[64]; struct inifin *ifp; @@ -377,6 +377,7 @@ vcc_priv_arg(struct vcc *tl, const char *p, const struct symbol *sym) marklist = &tl->curproc->priv_tasks; } else if (!strcmp(p, "PRIV_TOP")) { f = "top"; + sym->r_methods &= VCL_MET_TASK_C; marklist = &tl->curproc->priv_tops; } else { WRONG("Wrong PRIV_ type"); @@ -641,7 +642,7 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv, void vcc_Eval_Func(struct vcc *tl, const struct vjsn_val *spec, - const char *extra, const struct symbol *sym) + const char *extra, struct symbol *sym) { struct expr *e = NULL; From dridi.boukelmoune at gmail.com Mon Jan 25 18:18:07 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 25 Jan 2021 18:18:07 +0000 (UTC) Subject: [master] f368e540a vcc: Complete mask with VCL_MET_TASK_ALL Message-ID: <20210125181807.A28E511BE48@lists.varnish-cache.org> commit f368e540aaacc5de1f874bba63ca55c475df3119 Author: Dridi Boukelmoune Date: Tue Jan 19 07:06:53 2021 +0100 vcc: Complete mask with VCL_MET_TASK_ALL The rationale is that if we add another type of task in the future, for example probe tasks, we won't need to update the default mask for VMOD functions and object methods. diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 12b8c417e..823fedc5b 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -625,6 +625,10 @@ for i in sorted(task.keys()): fo.write(tbl40("#define VCL_MET_TASK_%s" % i.upper(), "( " + (" | \\\n\t\t\t\t\t ").join(task[i]) + " )\n")) +fo.write("\n") +fo.write(tbl40("#define VCL_MET_TASK_ALL", "( VCL_MET_TASK_")) +fo.write(" | \\\n\t\t\t\t\t VCL_MET_TASK_".join(map(str.upper, task.keys()))) +fo.write(" )") fo.write("\n/* VCL Returns */\n") n = 1 diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index 4ecd217c6..361357b32 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -119,7 +119,7 @@ func_sym(struct vcc *tl, vcc_kind_t kind, const struct symbol *psym, assert(v->type == VJSN_STRING); sym->type = VCC_Type(v->value); AN(sym->type); - sym->r_methods = VCL_MET_TASK_C|VCL_MET_TASK_B|VCL_MET_TASK_H; + sym->r_methods = VCL_MET_TASK_ALL; } static void From dridi.boukelmoune at gmail.com Tue Jan 26 09:36:08 2021 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 26 Jan 2021 09:36:08 +0000 (UTC) Subject: [master] 94af15e8e doc: Mention the VCL standalone return statement Message-ID: <20210126093608.C2B02A6D83@lists.varnish-cache.org> commit 94af15e8ed993377ac7cae187f3e70ffb33c9d42 Author: Dridi Boukelmoune Date: Tue Jan 26 10:30:31 2021 +0100 doc: Mention the VCL standalone return statement Salvaged from #3503 diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index 00be68429..9dc04d6ca 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -434,6 +434,17 @@ The ongoing ``vcl_*`` subroutine execution ends when a The ** specifies how execution should proceed. The context defines which actions are available. +It is possible to exit a subroutine that is not part of the built-in ones +using a simple ``return`` statement without specifying an action. It exits +the subroutine without transitioning to a different state:: + + sub filter_cookies { + if (!req.http.cookie) { + return; + } + # complex cookie filtering + } + Multiple subroutines ~~~~~~~~~~~~~~~~~~~~ @@ -538,7 +549,7 @@ HISTORY VCL was developed by Poul-Henning Kamp in cooperation with Verdens Gang AS, Redpill Linpro and Varnish Software. This manual page is written by Per Buer, Poul-Henning Kamp, Martin Blix Grydeland, -Kristian Lyngst?l, Lasse Karstensen and possibly others. +Kristian Lyngst?l, Lasse Karstensen and others. COPYRIGHT ========= From phk at FreeBSD.org Tue Jan 26 13:48:06 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 26 Jan 2021 13:48:06 +0000 (UTC) Subject: [master] d34a356cd Add VRT_Endpoint_Clone() and give backends a copy of the endpoint information to make them easier to clone. Message-ID: <20210126134806.DE03FAD8DE@lists.varnish-cache.org> commit d34a356cdda34ed671de90f77c9b84ff18f98bd2 Author: Poul-Henning Kamp Date: Tue Jan 26 13:46:30 2021 +0000 Add VRT_Endpoint_Clone() and give backends a copy of the endpoint information to make them easier to clone. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index b93c1e9e4..da7be381f 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -438,6 +438,7 @@ vbe_free(struct backend *be) VRT_BACKEND_HANDLE(); #undef DA #undef DN + free(be->endpoint); Lck_Delete(&be->mtx); FREE_OBJ(be); @@ -588,6 +589,8 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, return (NULL); Lck_New(&be->mtx, lck_backend); + be->endpoint = VRT_Endpoint_Clone(vrt->endpoint); + vep = be->endpoint; #define DA(x) do { if (vrt->x != NULL) REPLACE((be->x), (vrt->x)); } while (0) #define DN(x) do { be->x = vrt->x; } while (0) VRT_BACKEND_HANDLE(); diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h index 393b5df09..b2c211d9e 100644 --- a/bin/varnishd/cache/cache_backend.h +++ b/bin/varnishd/cache/cache_backend.h @@ -57,6 +57,8 @@ struct backend { VTAILQ_ENTRY(backend) list; struct lock mtx; + struct vrt_endpoint *endpoint; + VRT_BACKEND_FIELDS() unsigned sick; diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index f8b9223a3..6d1cb6b77 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -33,6 +33,8 @@ #include "config.h" +#include + #include "cache_varnishd.h" #include "cache_objhead.h" @@ -1061,3 +1063,47 @@ VRT_Format_Proxy(struct vsb *vsb, VCL_INT version, VCL_IP sac, VCL_IP sas, { VPX_Format_Proxy(vsb, (int)version, sac, sas, auth); } + +/* + * Clone a struct vrt_endpoint in a single malloc() allocation + */ + +struct vrt_endpoint * +VRT_Endpoint_Clone(const struct vrt_endpoint *vep) +{ + size_t sz; + struct vrt_endpoint *nvep; + struct suckaddr *sa; + char *p; + + CHECK_OBJ_NOTNULL(vep, VRT_ENDPOINT_MAGIC); + sz = sizeof *nvep; + if (vep->ipv4) + sz += vsa_suckaddr_len; + if (vep->ipv6) + sz += vsa_suckaddr_len; + if (vep->uds_path != NULL) + sz += strlen(vep->uds_path) + 1; + p = malloc(sz); + AN(p); + nvep = (void*)p; + p += sizeof *nvep; + INIT_OBJ(nvep, VRT_ENDPOINT_MAGIC); + if (vep->ipv4) { + sa = (void*)p; //lint !e826 + memcpy(sa, vep->ipv4, vsa_suckaddr_len); //lint !e826 + nvep->ipv4 = sa; + p += vsa_suckaddr_len; + } + if (vep->ipv6) { + sa = (void*)p; //lint !e826 + memcpy(sa, vep->ipv6, vsa_suckaddr_len); //lint !e826 + nvep->ipv6 = sa; + p += vsa_suckaddr_len; + } + if (vep->uds_path != NULL) { + strcpy(p, vep->uds_path); + nvep->uds_path = p; + } + return (nvep); +} diff --git a/include/vrt.h b/include/vrt.h index 20782fe45..abd4fb2fb 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -53,8 +53,9 @@ * binary/load-time compatible, increment MAJOR version * * 13.0 (2021-03-15) + * VRT_Endpoint_Clone() added. * Calling convention for VDP implementation changed - * Added VRT_ValidHdr() + * VRT_ValidHdr() added. * struct vmod_priv_methods added * struct vmod_priv free member replaced with methods * VRT_CTX_Assert() added @@ -521,6 +522,7 @@ VCL_BACKEND VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *, const struct vrt_backend *); size_t VRT_backend_vsm_need(VRT_CTX); void VRT_delete_backend(VRT_CTX, VCL_BACKEND *); +struct vrt_endpoint *VRT_Endpoint_Clone(const struct vrt_endpoint *vep); /* VSM related */ struct vsmw_cluster *VRT_VSM_Cluster_New(VRT_CTX, size_t); From nils.goroll at uplex.de Wed Jan 27 20:11:21 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 27 Jan 2021 20:11:21 +0000 (UTC) Subject: [master] da2409c38 Improve vtest.sh variable quoting Message-ID: <20210127201121.DB7017525@lists.varnish-cache.org> commit da2409c3863d1da8f2dac0ace9ae05fa0e743d81 Author: Nils Goroll Date: Wed Jan 27 21:08:13 2021 +0100 Improve vtest.sh variable quoting Advise from shellcheck taken undogmatically. (I always forget: is $() for `..` posix-sh or not?) diff --git a/tools/vtest.sh b/tools/vtest.sh index bc601ca69..936a96020 100755 --- a/tools/vtest.sh +++ b/tools/vtest.sh @@ -66,11 +66,11 @@ export VTEST_REPORT="${REPORTDIR}/_log" ####################################################################### # Establish TMPDIR -mkdir -p ${TMPDIR} -rm -rf ${TMPDIR}/* +mkdir -p "${TMPDIR}" +rm -rf "${TMPDIR:?}"/* # Try to make varnish own TMPDIR, in case we run as root -chown varnish ${TMPDIR} > /dev/null 2>&1 || true +chown varnish "${TMPDIR}" > /dev/null 2>&1 || true ####################################################################### # Establish the SRCDIR we build/run/test @@ -91,7 +91,7 @@ if [ ! -f vt_key.pub ] ; then fi pack () ( - cd ${REPORTDIR} + cd "${REPORTDIR}" tar czf - _log \ `grep '^MANIFEST ' _log | sort -u | sed 's/^MANIFEST *//'` \ ) @@ -104,15 +104,15 @@ submit () ( -o NumberOfPasswordPrompts=0 \ -o RequestTTY=no \ -i vt_key \ - ${SSH_DST} \ + "${SSH_DST}" \ true \ - < ${1} + < "${1}" ) -rm -f ${TMPDIR}/_report.tgz -touch ${TMPDIR}/_report.tgz +rm -f "${TMPDIR}"/_report.tgz +touch "${TMPDIR}"/_report.tgz -if ! submit ${TMPDIR}/_report.tgz; then +if ! submit "${TMPDIR}"/_report.tgz; then echo "Test submit failed" echo echo "You probably need to email this VTEST specific ssh-key" @@ -193,7 +193,7 @@ failedtests () ( grep -l ':test-result: FAIL' "$LOGDIR"/*.trs | while read trs do - name=`basename $trs .trs` + name=`basename "${trs}" .trs` vtc="${name}.vtc" log="${name}.log" rev=`git log -n 1 --pretty=format:%H "${VTCDIR}/${vtc}"` @@ -226,12 +226,12 @@ do (cd "${SRCDIR}" && chmod -R +w varnish-trunk && rm -rf varnish-trunk > /dev/null 2>&1 || true) rev=`cd "${SRCDIR}" && git show -s --pretty=format:%H` if [ "x${rev}" != "x${orev}" ] ; then - waitcur=${WAITMIN} + waitcur=${WAITMIN} elif [ "${waitnext}" -gt 0 ] ; then sleep ${WAITPERIOD} waitnext=`expr ${waitnext} - 1 || true` continue - else + else waitcur=`expr ${waitcur} + 1` if [ ${waitcur} -gt ${WAITMAX} ] ; then waitcur=${WAITMAX} @@ -253,55 +253,55 @@ do # NB: Only change the report version number when the format/content # NB: of the report changes. Corresponding changes on the backend # NB: will be required. Coordinate with phk at . - echo "VTEST 1.05" > ${VTEST_REPORT} - echo "DATE `date +%s`" >> ${VTEST_REPORT} - echo "BRANCH trunk" >> ${VTEST_REPORT} - echo "HOST `hostname`" >> ${VTEST_REPORT} - echo "UNAME `uname -a`" >> ${VTEST_REPORT} - echo "UGID `id`" >> ${VTEST_REPORT} + echo "VTEST 1.05" > "${VTEST_REPORT}" + echo "DATE `date +%s`" >> "${VTEST_REPORT}" + echo "BRANCH trunk" >> "${VTEST_REPORT}" + echo "HOST `hostname`" >> "${VTEST_REPORT}" + echo "UNAME `uname -a`" >> "${VTEST_REPORT}" + echo "UGID `id`" >> "${VTEST_REPORT}" if [ -x /usr/bin/lsb_release ] ; then - echo "LSB `lsb_release -d`" >> ${VTEST_REPORT} + echo "LSB `lsb_release -d`" >> "${VTEST_REPORT}" else - echo "LSB none" >> ${VTEST_REPORT} + echo "LSB none" >> "${VTEST_REPORT}" fi - echo "MESSAGE ${MESSAGE}" >> ${VTEST_REPORT} - echo "GITREV $rev" >> ${VTEST_REPORT} + echo "MESSAGE ${MESSAGE}" >> "${VTEST_REPORT}" + echo "GITREV $rev" >> "${VTEST_REPORT}" find . -name '*.gc??' -print | xargs rm -f - if ! autogen >> ${REPORTDIR}/_autogen 2>&1 ; then - echo "AUTOGEN BAD" >> ${VTEST_REPORT} - echo "MANIFEST _autogen" >> ${VTEST_REPORT} + if ! autogen >> "${REPORTDIR}"/_autogen 2>&1 ; then + echo "AUTOGEN BAD" >> "${VTEST_REPORT}" + echo "MANIFEST _autogen" >> "${VTEST_REPORT}" else - echo "AUTOGEN GOOD" >> ${VTEST_REPORT} + echo "AUTOGEN GOOD" >> "${VTEST_REPORT}" if $enable_gcov ; then - if makegcov >> ${REPORTDIR}/_makegcov 2>&1 ; then - mv ${SRCDIR}/_gcov ${REPORTDIR}/ - echo "MAKEGCOV GOOD" >> ${VTEST_REPORT} - echo "MANIFEST _gcov" >> ${VTEST_REPORT} + if makegcov >> "${REPORTDIR}"/_makegcov 2>&1 ; then + mv ${SRCDIR}/_gcov "${REPORTDIR}"/ + echo "MAKEGCOV GOOD" >> "${VTEST_REPORT}" + echo "MANIFEST _gcov" >> "${VTEST_REPORT}" waitcur=${WAITMAX} waitnext=${WAITMAX} else - echo "MAKEGCOV BAD" >> ${VTEST_REPORT} - echo "MANIFEST _makegcov" >> ${VTEST_REPORT} - failedtests >> ${VTEST_REPORT} + echo "MAKEGCOV BAD" >> "${VTEST_REPORT}" + echo "MANIFEST _makegcov" >> "${VTEST_REPORT}" + failedtests >> "${VTEST_REPORT}" fi - elif ! makedistcheck >> ${REPORTDIR}/_makedistcheck 2>&1 ; then - echo "MAKEDISTCHECK BAD" >> ${VTEST_REPORT} - echo "MANIFEST _autogen" >> ${VTEST_REPORT} - echo "MANIFEST _makedistcheck" >> ${VTEST_REPORT} - failedtests >> ${VTEST_REPORT} + elif ! makedistcheck >> "${REPORTDIR}"/_makedistcheck 2>&1 ; then + echo "MAKEDISTCHECK BAD" >> "${VTEST_REPORT}" + echo "MANIFEST _autogen" >> "${VTEST_REPORT}" + echo "MANIFEST _makedistcheck" >> "${VTEST_REPORT}" + failedtests >> "${VTEST_REPORT}" else - echo "MAKEDISTCHECK GOOD" >> ${VTEST_REPORT} + echo "MAKEDISTCHECK GOOD" >> "${VTEST_REPORT}" waitnext=${WAITMAX} waitcur=${WAITMAX} fi fi - echo "VTEST END" >> ${VTEST_REPORT} - pack > ${TMPDIR}/_report.tgz + echo "VTEST END" >> "${VTEST_REPORT}" + pack > "${TMPDIR}"/_report.tgz - submit ${TMPDIR}/_report.tgz || \ + submit "${TMPDIR}"/_report.tgz || \ sleep 300 || \ - submit ${TMPDIR}/_report.tgz || \ + submit "${TMPDIR}"/_report.tgz || \ true done From dridi at varni.sh Thu Jan 28 07:52:37 2021 From: dridi at varni.sh (Dridi Boukelmoune) Date: Thu, 28 Jan 2021 07:52:37 +0000 Subject: [master] da2409c38 Improve vtest.sh variable quoting In-Reply-To: <20210127201121.DB7017525@lists.varnish-cache.org> References: <20210127201121.DB7017525@lists.varnish-cache.org> Message-ID: On Wed, Jan 27, 2021 at 8:11 PM Nils Goroll wrote: > > > commit da2409c3863d1da8f2dac0ace9ae05fa0e743d81 > Author: Nils Goroll > Date: Wed Jan 27 21:08:13 2021 +0100 > > Improve vtest.sh variable quoting > > Advise from shellcheck taken undogmatically. > > (I always forget: is $() for `..` posix-sh or not?) $(...) is in POSIX, not (or no longer) a bashism, at least since 2001. https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_03 > diff --git a/tools/vtest.sh b/tools/vtest.sh > index bc601ca69..936a96020 100755 > --- a/tools/vtest.sh > +++ b/tools/vtest.sh > @@ -66,11 +66,11 @@ export VTEST_REPORT="${REPORTDIR}/_log" > ####################################################################### > # Establish TMPDIR > > -mkdir -p ${TMPDIR} > -rm -rf ${TMPDIR}/* > +mkdir -p "${TMPDIR}" > +rm -rf "${TMPDIR:?}"/* > > # Try to make varnish own TMPDIR, in case we run as root > -chown varnish ${TMPDIR} > /dev/null 2>&1 || true > +chown varnish "${TMPDIR}" > /dev/null 2>&1 || true > > ####################################################################### > # Establish the SRCDIR we build/run/test > @@ -91,7 +91,7 @@ if [ ! -f vt_key.pub ] ; then > fi > > pack () ( > - cd ${REPORTDIR} > + cd "${REPORTDIR}" > tar czf - _log \ > `grep '^MANIFEST ' _log | sort -u | sed 's/^MANIFEST *//'` \ > ) > @@ -104,15 +104,15 @@ submit () ( > -o NumberOfPasswordPrompts=0 \ > -o RequestTTY=no \ > -i vt_key \ > - ${SSH_DST} \ > + "${SSH_DST}" \ > true \ > - < ${1} > + < "${1}" > ) > > -rm -f ${TMPDIR}/_report.tgz > -touch ${TMPDIR}/_report.tgz > +rm -f "${TMPDIR}"/_report.tgz > +touch "${TMPDIR}"/_report.tgz > > -if ! submit ${TMPDIR}/_report.tgz; then > +if ! submit "${TMPDIR}"/_report.tgz; then > echo "Test submit failed" > echo > echo "You probably need to email this VTEST specific ssh-key" > @@ -193,7 +193,7 @@ failedtests () ( > grep -l ':test-result: FAIL' "$LOGDIR"/*.trs | > while read trs > do > - name=`basename $trs .trs` > + name=`basename "${trs}" .trs` > vtc="${name}.vtc" > log="${name}.log" > rev=`git log -n 1 --pretty=format:%H "${VTCDIR}/${vtc}"` > @@ -226,12 +226,12 @@ do > (cd "${SRCDIR}" && chmod -R +w varnish-trunk && rm -rf varnish-trunk > /dev/null 2>&1 || true) > rev=`cd "${SRCDIR}" && git show -s --pretty=format:%H` > if [ "x${rev}" != "x${orev}" ] ; then > - waitcur=${WAITMIN} > + waitcur=${WAITMIN} > elif [ "${waitnext}" -gt 0 ] ; then > sleep ${WAITPERIOD} > waitnext=`expr ${waitnext} - 1 || true` > continue > - else > + else > waitcur=`expr ${waitcur} + 1` > if [ ${waitcur} -gt ${WAITMAX} ] ; then > waitcur=${WAITMAX} > @@ -253,55 +253,55 @@ do > # NB: Only change the report version number when the format/content > # NB: of the report changes. Corresponding changes on the backend > # NB: will be required. Coordinate with phk at . > - echo "VTEST 1.05" > ${VTEST_REPORT} > - echo "DATE `date +%s`" >> ${VTEST_REPORT} > - echo "BRANCH trunk" >> ${VTEST_REPORT} > - echo "HOST `hostname`" >> ${VTEST_REPORT} > - echo "UNAME `uname -a`" >> ${VTEST_REPORT} > - echo "UGID `id`" >> ${VTEST_REPORT} > + echo "VTEST 1.05" > "${VTEST_REPORT}" > + echo "DATE `date +%s`" >> "${VTEST_REPORT}" > + echo "BRANCH trunk" >> "${VTEST_REPORT}" > + echo "HOST `hostname`" >> "${VTEST_REPORT}" > + echo "UNAME `uname -a`" >> "${VTEST_REPORT}" > + echo "UGID `id`" >> "${VTEST_REPORT}" > if [ -x /usr/bin/lsb_release ] ; then > - echo "LSB `lsb_release -d`" >> ${VTEST_REPORT} > + echo "LSB `lsb_release -d`" >> "${VTEST_REPORT}" > else > - echo "LSB none" >> ${VTEST_REPORT} > + echo "LSB none" >> "${VTEST_REPORT}" > fi > - echo "MESSAGE ${MESSAGE}" >> ${VTEST_REPORT} > - echo "GITREV $rev" >> ${VTEST_REPORT} > + echo "MESSAGE ${MESSAGE}" >> "${VTEST_REPORT}" > + echo "GITREV $rev" >> "${VTEST_REPORT}" > > find . -name '*.gc??' -print | xargs rm -f > > - if ! autogen >> ${REPORTDIR}/_autogen 2>&1 ; then > - echo "AUTOGEN BAD" >> ${VTEST_REPORT} > - echo "MANIFEST _autogen" >> ${VTEST_REPORT} > + if ! autogen >> "${REPORTDIR}"/_autogen 2>&1 ; then > + echo "AUTOGEN BAD" >> "${VTEST_REPORT}" > + echo "MANIFEST _autogen" >> "${VTEST_REPORT}" > else > - echo "AUTOGEN GOOD" >> ${VTEST_REPORT} > + echo "AUTOGEN GOOD" >> "${VTEST_REPORT}" > if $enable_gcov ; then > - if makegcov >> ${REPORTDIR}/_makegcov 2>&1 ; then > - mv ${SRCDIR}/_gcov ${REPORTDIR}/ > - echo "MAKEGCOV GOOD" >> ${VTEST_REPORT} > - echo "MANIFEST _gcov" >> ${VTEST_REPORT} > + if makegcov >> "${REPORTDIR}"/_makegcov 2>&1 ; then > + mv ${SRCDIR}/_gcov "${REPORTDIR}"/ > + echo "MAKEGCOV GOOD" >> "${VTEST_REPORT}" > + echo "MANIFEST _gcov" >> "${VTEST_REPORT}" > waitcur=${WAITMAX} > waitnext=${WAITMAX} > else > - echo "MAKEGCOV BAD" >> ${VTEST_REPORT} > - echo "MANIFEST _makegcov" >> ${VTEST_REPORT} > - failedtests >> ${VTEST_REPORT} > + echo "MAKEGCOV BAD" >> "${VTEST_REPORT}" > + echo "MANIFEST _makegcov" >> "${VTEST_REPORT}" > + failedtests >> "${VTEST_REPORT}" > fi > - elif ! makedistcheck >> ${REPORTDIR}/_makedistcheck 2>&1 ; then > - echo "MAKEDISTCHECK BAD" >> ${VTEST_REPORT} > - echo "MANIFEST _autogen" >> ${VTEST_REPORT} > - echo "MANIFEST _makedistcheck" >> ${VTEST_REPORT} > - failedtests >> ${VTEST_REPORT} > + elif ! makedistcheck >> "${REPORTDIR}"/_makedistcheck 2>&1 ; then > + echo "MAKEDISTCHECK BAD" >> "${VTEST_REPORT}" > + echo "MANIFEST _autogen" >> "${VTEST_REPORT}" > + echo "MANIFEST _makedistcheck" >> "${VTEST_REPORT}" > + failedtests >> "${VTEST_REPORT}" > else > - echo "MAKEDISTCHECK GOOD" >> ${VTEST_REPORT} > + echo "MAKEDISTCHECK GOOD" >> "${VTEST_REPORT}" > waitnext=${WAITMAX} > waitcur=${WAITMAX} > fi > fi > - echo "VTEST END" >> ${VTEST_REPORT} > - pack > ${TMPDIR}/_report.tgz > + echo "VTEST END" >> "${VTEST_REPORT}" > + pack > "${TMPDIR}"/_report.tgz > > - submit ${TMPDIR}/_report.tgz || \ > + submit "${TMPDIR}"/_report.tgz || \ > sleep 300 || \ > - submit ${TMPDIR}/_report.tgz || \ > + submit "${TMPDIR}"/_report.tgz || \ > true > done > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From nils.goroll at uplex.de Thu Jan 28 09:06:31 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 28 Jan 2021 10:06:31 +0100 Subject: [master] da2409c38 Improve vtest.sh variable quoting In-Reply-To: References: <20210127201121.DB7017525@lists.varnish-cache.org> Message-ID: <61bde7d9-32bc-36c9-f39e-c83334cb5d38@uplex.de> On 28/01/2021 08:52, Dridi Boukelmoune wrote: > $(...) is in POSIX, not (or no longer) a bashism, at least since 2001. > > https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_03 > wonderful, abusing commits for questions worked ;) -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From nils.goroll at uplex.de Thu Jan 28 15:18:13 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 28 Jan 2021 15:18:13 +0000 (UTC) Subject: [master] 4eaa0275d Bring VRB_Iterate() back to cache.h Message-ID: <20210128151813.7DD34116102@lists.varnish-cache.org> commit 4eaa0275dbb14dd729a5c7910fc6e24a2dda993f Author: Nils Goroll Date: Thu Jan 28 16:14:56 2021 +0100 Bring VRB_Iterate() back to cache.h In fee476acc085e6563225d35fd8e19f1b10f04139 VRB_Ignore(), VRB_Cache() and VRB_Free() were rightly out of vmods' sight. But VRB_Iterate() is legitemately required for vmods, and including it in the move seems to be an accident. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index a6bf831be..952dc03ab 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -710,6 +710,11 @@ int ObjGetDouble(struct worker *, struct objcore *, enum obj_attr, double *); int ObjGetU32(struct worker *, struct objcore *, enum obj_attr, uint32_t *); int ObjGetU64(struct worker *, struct objcore *, enum obj_attr, uint64_t *); int ObjCheckFlag(struct worker *, struct objcore *, enum obj_flags of); + +/* cache_req_body.c */ +ssize_t VRB_Iterate(struct worker *, struct vsl_log *, struct req *, + objiterate_f *func, void *priv); + /* cache_session.c [SES] */ #define SESS_ATTR(UP, low, typ, len) \ diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 90fd67532..c9baf430a 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -370,8 +370,6 @@ void Req_AcctLogCharge(struct VSC_main_wrk *, struct req *); /* cache_req_body.c */ int VRB_Ignore(struct req *); ssize_t VRB_Cache(struct req *, ssize_t maxsize); -ssize_t VRB_Iterate(struct worker *, struct vsl_log *, struct req *, - objiterate_f *func, void *priv); void VRB_Free(struct req *); /* cache_req_fsm.c [CNT] */ From nils.goroll at uplex.de Thu Jan 28 15:35:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 28 Jan 2021 15:35:08 +0000 (UTC) Subject: [master] 7537590f7 Avoid our boolean parameter clashing with stdbool.h Message-ID: <20210128153508.EE914116901@lists.varnish-cache.org> commit 7537590f710b4663356fffefd4e3267a6954a3ba Author: Nils Goroll Date: Thu Jan 28 16:30:26 2021 +0100 Avoid our boolean parameter clashing with stdbool.h We have now seen a second case after 184736fad473070da79b77a16eab38faf393516e, and a simple internal rename from bool to boolean does the job. Should have no user-visible impact. diff --git a/bin/varnishd/common/common_param.h b/bin/varnishd/common/common_param.h index b7f07ace3..785e20579 100644 --- a/bin/varnishd/common/common_param.h +++ b/bin/varnishd/common/common_param.h @@ -40,10 +40,6 @@ #define VSM_CLASS_PARAM "Params" -#ifdef __MACH__ -#undef bool -#endif - enum debug_bits { #define DEBUG_BIT(U, l, d) DBG_##U, #include "tbl/debug_bits.h" @@ -82,7 +78,7 @@ typedef uint8_t feature_t[(FEATURE_Reserved+7)>>3]; struct params { -#define ptyp_bool unsigned +#define ptyp_boolean unsigned #define ptyp_bytes ssize_t #define ptyp_bytes_u unsigned #define ptyp_double double @@ -96,7 +92,7 @@ struct params { #define PARAM(typ, fld, nm, ...) \ ptyp_##typ fld; #include -#undef ptyp_bool +#undef ptyp_boolean #undef ptyp_bytes #undef ptyp_bytes_u #undef ptyp_double diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h index fd884fd02..20d82641f 100644 --- a/bin/varnishd/mgt/mgt_param.h +++ b/bin/varnishd/mgt/mgt_param.h @@ -66,7 +66,7 @@ struct parspec { char *dyn_def; }; -tweak_t tweak_bool; +tweak_t tweak_boolean; tweak_t tweak_bytes; tweak_t tweak_bytes_u; tweak_t tweak_double; diff --git a/bin/varnishd/mgt/mgt_param_tweak.c b/bin/varnishd/mgt/mgt_param_tweak.c index fe06f1ce5..8e378df04 100644 --- a/bin/varnishd/mgt/mgt_param_tweak.c +++ b/bin/varnishd/mgt/mgt_param_tweak.c @@ -122,7 +122,7 @@ tweak_double(struct vsb *vsb, const struct parspec *par, const char *arg) /*--------------------------------------------------------------------*/ int -tweak_bool(struct vsb *vsb, const struct parspec *par, const char *arg) +tweak_boolean(struct vsb *vsb, const struct parspec *par, const char *arg) { volatile unsigned *dest; diff --git a/include/tbl/params.h b/include/tbl/params.h index af751fdf2..6f4b6ad8b 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -54,7 +54,7 @@ #endif PARAM_SIMPLE( /* name */ accept_filter, - /* type */ bool, + /* type */ boolean, /* min */ NULL, /* max */ NULL, /* def */ "on", /* default adjusted in mgt_param.c */ @@ -120,7 +120,7 @@ PARAM_SIMPLE( PARAM_SIMPLE( /* name */ auto_restart, - /* type */ bool, + /* type */ boolean, /* min */ NULL, /* max */ NULL, /* def */ "on", @@ -131,7 +131,7 @@ PARAM_SIMPLE( PARAM_SIMPLE( /* name */ ban_dups, - /* type */ bool, + /* type */ boolean, /* min */ NULL, /* max */ NULL, /* def */ "on", @@ -505,7 +505,7 @@ PARAM_SIMPLE( PARAM_SIMPLE( /* name */ http_gzip_support, - /* type */ bool, + /* type */ boolean, /* min */ NULL, /* max */ NULL, /* def */ "on", @@ -544,7 +544,7 @@ PARAM_SIMPLE( PARAM_SIMPLE( /* name */ http_range_support, - /* type */ bool, + /* type */ boolean, /* min */ NULL, /* max */ NULL, /* def */ "on", @@ -745,7 +745,7 @@ PARAM_SIMPLE( PARAM_SIMPLE( /* name */ prefer_ipv6, - /* type */ bool, + /* type */ boolean, /* min */ NULL, /* max */ NULL, /* def */ "off", @@ -807,7 +807,7 @@ PARAM_SIMPLE( PARAM_SIMPLE( /* name */ sigsegv_handler, - /* type */ bool, + /* type */ boolean, /* min */ NULL, /* max */ NULL, /* def */ "on", @@ -820,7 +820,7 @@ PARAM_SIMPLE( PARAM_SIMPLE( /* name */ syslog_cli_traffic, - /* type */ bool, + /* type */ boolean, /* min */ NULL, /* max */ NULL, /* def */ "on", @@ -836,7 +836,7 @@ PARAM_SIMPLE( #endif PARAM_SIMPLE( /* name */ tcp_fastopen, - /* type */ bool, + /* type */ boolean, /* min */ NULL, /* max */ NULL, /* def */ "off", @@ -1567,7 +1567,7 @@ PARAM_STRING( */ # define PARAM_VCC(nm, def, descr) \ - PARAM(, , nm, tweak_bool, &mgt_ ## nm, NULL, NULL, def, "bool", descr) + PARAM(, , nm, tweak_boolean, &mgt_ ## nm, NULL, NULL, def, "bool", descr) PARAM_VCC( /* name */ vcc_err_unref, From nils.goroll at uplex.de Thu Jan 28 15:43:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 28 Jan 2021 15:43:06 +0000 (UTC) Subject: [master] 8aab77298 Undo over-zealous quoting Message-ID: <20210128154306.E3016116DF9@lists.varnish-cache.org> commit 8aab77298a083d241c8064a7795fa3176d76f30f Author: Nils Goroll Date: Thu Jan 28 16:41:23 2021 +0100 Undo over-zealous quoting Quote more. Do not quote too much Ref da2409c3863d1da8f2dac0ace9ae05fa0e743d81 diff --git a/tools/vtest.sh b/tools/vtest.sh index 936a96020..6b0da37d5 100755 --- a/tools/vtest.sh +++ b/tools/vtest.sh @@ -104,7 +104,7 @@ submit () ( -o NumberOfPasswordPrompts=0 \ -o RequestTTY=no \ -i vt_key \ - "${SSH_DST}" \ + ${SSH_DST} \ true \ < "${1}" ) From phk at FreeBSD.org Fri Jan 29 20:55:19 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 29 Jan 2021 20:55:19 +0000 (UTC) Subject: [master] fb09803e1 Complete the `tcp_pool` -> `conn_pool` conversion. Message-ID: <20210129205519.CFD277908@lists.varnish-cache.org> commit fb09803e1101920bc2fcc68cc63a8448e17b35e8 Author: Poul-Henning Kamp Date: Fri Jan 29 20:54:17 2021 +0000 Complete the `tcp_pool` -> `conn_pool` conversion. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index edc6a3ff3..ce7145efd 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -17,6 +17,7 @@ varnishd_SOURCES = \ cache/cache_ban_lurker.c \ cache/cache_busyobj.c \ cache/cache_cli.c \ + cache/cache_conn_pool.c \ cache/cache_deliver_proc.c \ cache/cache_director.c \ cache/cache_esi_deliver.c \ @@ -41,7 +42,6 @@ varnishd_SOURCES = \ cache/cache_rfc2616.c \ cache/cache_session.c \ cache/cache_shmlog.c \ - cache/cache_tcp_pool.c \ cache/cache_vary.c \ cache/cache_vcl.c \ cache/cache_vpi.c \ @@ -123,11 +123,11 @@ nodist_varnishd_SOURCES = \ noinst_HEADERS = \ cache/cache_ban.h \ + cache/cache_conn_pool.h \ cache/cache_esi.h \ cache/cache_obj.h \ cache/cache_objhead.h \ cache/cache_pool.h \ - cache/cache_tcp_pool.h \ cache/cache_transport.h \ cache/cache_vcl.h \ cache/cache_vgz.h \ diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index da7be381f..16e7a1258 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -43,7 +43,7 @@ #include "vtim.h" #include "cache_backend.h" -#include "cache_tcp_pool.h" +#include "cache_conn_pool.h" #include "cache_transport.h" #include "cache_vcl.h" #include "http1/cache_http1.h" @@ -616,7 +616,7 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, if (! vcl->temp->is_warm) VRT_VSC_Hide(be->vsc_seg); - be->conn_pool = VTP_Ref(vep, vbe_proto_ident); + be->conn_pool = VCP_Ref(vep, vbe_proto_ident); AN(be->conn_pool); vbp = vrt->probe; diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index cd99c9b76..e4181bcce 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -52,7 +52,7 @@ #include "vtim.h" #include "cache_backend.h" -#include "cache_tcp_pool.h" +#include "cache_conn_pool.h" /* Default averaging rate, we want something pretty responsive */ #define AVG_RATE 4 diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_conn_pool.c similarity index 83% rename from bin/varnishd/cache/cache_tcp_pool.c rename to bin/varnishd/cache/cache_conn_pool.c index 5614c9412..9b29217d8 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_conn_pool.c @@ -45,7 +45,7 @@ #include "vtim.h" #include "waiter/waiter.h" -#include "cache_tcp_pool.h" +#include "cache_conn_pool.h" #include "cache_pool.h" struct conn_pool; @@ -72,16 +72,12 @@ struct pfd { typedef int cp_open_f(const struct conn_pool *, vtim_dur tmo, VCL_IP *ap); typedef void cp_close_f(struct pfd *); typedef void cp_name_f(const struct pfd *, char *, unsigned, char *, unsigned); -typedef void cp_free_f(void *); -typedef void cp_panic_f(struct vsb *, const void *); struct cp_methods { cp_open_f *open; cp_close_f *close; cp_name_f *local_name; cp_name_f *remote_name; - cp_free_f *free; - cp_panic_f *panic; }; struct conn_pool { @@ -90,8 +86,8 @@ struct conn_pool { const struct cp_methods *methods; + struct vrt_endpoint *endpoint; char ident[VSHA256_DIGEST_LENGTH]; - void *priv; VTAILQ_ENTRY(conn_pool) list; int refcnt; @@ -114,6 +110,7 @@ static struct lock conn_pools_mtx; static VTAILQ_HEAD(, conn_pool) conn_pools = VTAILQ_HEAD_INITIALIZER(conn_pools); + /*-------------------------------------------------------------------- */ @@ -193,56 +190,6 @@ vcp_handle(struct waited *w, enum wait_event ev, vtim_real now) Lck_Unlock(&cp->mtx); } -/*-------------------------------------------------------------------- - */ - -static struct conn_pool * -VCP_Ref(const uint8_t *ident) -{ - struct conn_pool *cp; - - Lck_Lock(&conn_pools_mtx); - VTAILQ_FOREACH(cp, &conn_pools, list) { - assert(cp->refcnt > 0); - if (memcmp(ident, cp->ident, sizeof cp->ident)) - continue; - cp->refcnt++; - Lck_Unlock(&conn_pools_mtx); - return (cp); - } - Lck_Unlock(&conn_pools_mtx); - return (NULL); -} - -/*-------------------------------------------------------------------- - */ - -static struct conn_pool * -VCP_New(struct conn_pool *cp, uint8_t ident[VSHA256_DIGEST_LENGTH], - void *priv, const struct cp_methods *cm) -{ - - AN(cp); - AN(cm); - AN(cm->open); - AN(cm->close); - - INIT_OBJ(cp, CONN_POOL_MAGIC); - memcpy(cp->ident, ident, sizeof cp->ident); - cp->priv = priv; - cp->methods = cm; - cp->refcnt = 1; - cp->holddown = 0; - Lck_New(&cp->mtx, lck_tcp_pool); - VTAILQ_INIT(&cp->connlist); - VTAILQ_INIT(&cp->killlist); - - Lck_Lock(&conn_pools_mtx); - VTAILQ_INSERT_HEAD(&conn_pools, cp, list); - Lck_Unlock(&conn_pools_mtx); - - return (cp); -} /*-------------------------------------------------------------------- */ @@ -299,7 +246,8 @@ VCP_Rel(struct conn_pool **cpp) Lck_Delete(&cp->mtx); AZ(cp->n_conn); AZ(cp->n_kill); - cp->methods->free(cp->priv); + free(cp->endpoint); + free(cp); } /*-------------------------------------------------------------------- @@ -569,6 +517,30 @@ VCP_GetIp(struct pfd *pfd) /*--------------------------------------------------------------------*/ +static void +vcp_panic_endpoint(struct vsb *vsb, const struct vrt_endpoint *vep) +{ + char h[VTCP_ADDRBUFSIZE]; + char p[VTCP_PORTBUFSIZE]; + + if (PAN_dump_struct(vsb, vep, VRT_ENDPOINT_MAGIC, "vrt_endpoint")) + return; + if (vep->uds_path) + VSB_printf(vsb, "uds_path = %s,\n", vep->uds_path); + if (vep->ipv4 && VSA_Sane(vep->ipv4)) { + VTCP_name(vep->ipv4, h, sizeof h, p, sizeof p); + VSB_printf(vsb, "ipv4 = %s, ", h); + VSB_printf(vsb, "port = %s,\n", p); + } + if (vep->ipv6 && VSA_Sane(vep->ipv6)) { + VTCP_name(vep->ipv6, h, sizeof h, p, sizeof p); + VSB_printf(vsb, "ipv6 = %s, ", h); + VSB_printf(vsb, "port = %s,\n", p); + } + VSB_indent(vsb, -2); + VSB_cat(vsb, "},\n"); +} + void VCP_Panic(struct vsb *vsb, struct conn_pool *cp) { @@ -578,7 +550,7 @@ VCP_Panic(struct vsb *vsb, struct conn_pool *cp) VSB_printf(vsb, "ident = "); VSB_quote(vsb, cp->ident, VSHA256_DIGEST_LENGTH, VSB_QUOTE_HEX); VSB_printf(vsb, ",\n"); - cp->methods->panic(vsb, cp->priv); + vcp_panic_endpoint(vsb, cp->endpoint); VSB_indent(vsb, -2); VSB_cat(vsb, "},\n"); } @@ -588,33 +560,11 @@ VCP_Panic(struct vsb *vsb, struct conn_pool *cp) void VCP_Init(void) { - Lck_New(&conn_pools_mtx, lck_tcp_pool); + Lck_New(&conn_pools_mtx, lck_conn_pool); } /**********************************************************************/ -struct tcp_pool { - unsigned magic; -#define TCP_POOL_MAGIC 0x28b0e42a - - struct suckaddr *ip4; - struct suckaddr *ip6; - char *uds; - struct conn_pool cp[1]; -}; - -static void v_matchproto_(cp_free_f) -vtp_free(void *priv) -{ - struct tcp_pool *tp; - - CAST_OBJ_NOTNULL(tp, priv, TCP_POOL_MAGIC); - free(tp->ip4); - free(tp->ip6); - free(tp->uds); - FREE_OBJ(tp); -} - static inline int tmo2msec(vtim_dur tmo) { @@ -626,55 +576,27 @@ vtp_open(const struct conn_pool *cp, vtim_dur tmo, VCL_IP *ap) { int s; int msec; - struct tcp_pool *tp; CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC); - CAST_OBJ_NOTNULL(tp, cp->priv, TCP_POOL_MAGIC); msec = tmo2msec(tmo); if (cache_param->prefer_ipv6) { - *ap = tp->ip6; - s = VTCP_connect(tp->ip6, msec); + *ap = cp->endpoint->ipv6; + s = VTCP_connect(*ap, msec); if (s >= 0) return (s); } - *ap = tp->ip4; - s = VTCP_connect(tp->ip4, msec); + *ap = cp->endpoint->ipv4; + s = VTCP_connect(*ap, msec); if (s >= 0) return (s); if (!cache_param->prefer_ipv6) { - *ap = tp->ip6; - s = VTCP_connect(tp->ip6, msec); + *ap = cp->endpoint->ipv6; + s = VTCP_connect(*ap, msec); } return (s); } -static void -vtp_panic(struct vsb *vsb, const void *priv) -{ - const struct tcp_pool *tp; - - char h[VTCP_ADDRBUFSIZE]; - char p[VTCP_PORTBUFSIZE]; - - tp = priv; - if (PAN_dump_struct(vsb, tp, TCP_POOL_MAGIC, "tcp_pool")) - return; - if (tp->uds) - VSB_printf(vsb, "uds = %s,\n", tp->uds); - if (tp->ip4 && VSA_Sane(tp->ip4)) { - VTCP_name(tp->ip4, h, sizeof h, p, sizeof p); - VSB_printf(vsb, "ipv4 = %s, ", h); - VSB_printf(vsb, "port = %s,\n", p); - } - if (tp->ip6 && VSA_Sane(tp->ip6)) { - VTCP_name(tp->ip6, h, sizeof h, p, sizeof p); - VSB_printf(vsb, "ipv6 = %s, ", h); - VSB_printf(vsb, "port = %s,\n", p); - } - VSB_indent(vsb, -2); - VSB_cat(vsb, "},\n"); -} /*--------------------------------------------------------------------*/ @@ -707,8 +629,6 @@ static const struct cp_methods vtp_methods = { .close = vtp_close, .local_name = vtp_local_name, .remote_name = vtp_remote_name, - .free = vtp_free, - .panic = vtp_panic, }; /*-------------------------------------------------------------------- @@ -719,15 +639,13 @@ vus_open(const struct conn_pool *cp, vtim_dur tmo, VCL_IP *ap) { int s; int msec; - struct tcp_pool *tp; CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC); - CAST_OBJ_NOTNULL(tp, cp->priv, TCP_POOL_MAGIC); - AN(tp->uds); + AN(cp->endpoint->uds_path); msec = tmo2msec(tmo); *ap = bogo_ip; - s = VUS_connect(tp->uds, msec); + s = VUS_connect(cp->endpoint->uds_path, msec); return (s); } @@ -747,8 +665,6 @@ static const struct cp_methods vus_methods = { .close = vtp_close, .local_name = vus_name, .remote_name = vus_name, - .free = vtp_free, - .panic = vtp_panic, }; /*-------------------------------------------------------------------- @@ -757,11 +673,9 @@ static const struct cp_methods vus_methods = { */ struct conn_pool * -VTP_Ref(const struct vrt_endpoint *vep, const char *ident) +VCP_Ref(const struct vrt_endpoint *vep, const char *ident) { - struct tcp_pool *tp; - struct conn_pool *cp; - const struct cp_methods *methods; + struct conn_pool *cp, *cp2; struct VSHA256Context cx[1]; unsigned char digest[VSHA256_DIGEST_LENGTH]; @@ -789,27 +703,51 @@ VTP_Ref(const struct vrt_endpoint *vep, const char *ident) } VSHA256_Final(digest, cx); - cp = VCP_Ref(digest); - if (cp != NULL) - return (cp); - /* - * this is racy - we could end up with additional pools on the same id - * with just a single connection + * In heavy use of dynamic backends, traversing this list + * can become expensive. In order to not do so twice we + * pessimistically create the necessary pool, and discard + * it on a hit. (XXX: Consider hash or tree ?) */ - ALLOC_OBJ(tp, TCP_POOL_MAGIC); - AN(tp); - if (vep->uds_path != NULL) { - methods = &vus_methods; - tp->uds = strdup(vep->uds_path); - AN(tp->uds); + + ALLOC_OBJ(cp, CONN_POOL_MAGIC); + AN(cp); + cp->refcnt = 1; + cp->holddown = 0; + cp->endpoint = VRT_Endpoint_Clone(vep); + memcpy(cp->ident, digest, sizeof cp->ident); + if (vep->uds_path != NULL) + cp->methods = &vus_methods; + else + cp->methods = &vtp_methods; + Lck_New(&cp->mtx, lck_conn_pool); + VTAILQ_INIT(&cp->connlist); + VTAILQ_INIT(&cp->killlist); + + CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC); + Lck_Lock(&conn_pools_mtx); + VTAILQ_FOREACH(cp2, &conn_pools, list) { + CHECK_OBJ_NOTNULL(cp2, CONN_POOL_MAGIC); + assert(cp2->refcnt > 0); + if (!memcmp(digest, cp2->ident, sizeof cp2->ident)) + break; } - else { - methods = &vtp_methods; - if (vep->ipv4 != NULL) - tp->ip4 = VSA_Clone(vep->ipv4); - if (vep->ipv6 != NULL) - tp->ip6 = VSA_Clone(vep->ipv6); + if (cp2 == NULL) + VTAILQ_INSERT_HEAD(&conn_pools, cp, list); + else + cp2->refcnt++; + Lck_Unlock(&conn_pools_mtx); + + if (cp2 == NULL) { + CHECK_OBJ_NOTNULL(cp, CONN_POOL_MAGIC); + return (cp); } - return (VCP_New(tp->cp, digest, tp, methods)); + + Lck_Delete(&cp->mtx); + AZ(cp->n_conn); + AZ(cp->n_kill); + FREE_OBJ(cp->endpoint); + FREE_OBJ(cp); + CHECK_OBJ_NOTNULL(cp2, CONN_POOL_MAGIC); + return (cp2); } diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_conn_pool.h similarity index 88% rename from bin/varnishd/cache/cache_tcp_pool.h rename to bin/varnishd/cache/cache_conn_pool.h index 00e8518f5..0da4c99c6 100644 --- a/bin/varnishd/cache/cache_tcp_pool.h +++ b/bin/varnishd/cache/cache_conn_pool.h @@ -52,9 +52,9 @@ void PFD_RemoteName(const struct pfd *, char *, unsigned, char *, unsigned); * Prototypes */ -struct conn_pool *VTP_Ref(const struct vrt_endpoint *, const char *ident); +struct conn_pool *VCP_Ref(const struct vrt_endpoint *, const char *ident); /* - * Get a reference to a TCP pool. Either one or both of ipv4 or + * Get a reference to a connection pool. Either one or both of ipv4 or * ipv6 arg must be non-NULL, or uds must be non-NULL. If recycling * is to be used, the ident pointer distinguishes the pool from * other pools with same {ipv4, ipv6, uds}. @@ -62,13 +62,14 @@ struct conn_pool *VTP_Ref(const struct vrt_endpoint *, const char *ident); void VCP_AddRef(struct conn_pool *); /* - * Get another reference to an already referenced TCP pool. + * Get another reference to an already referenced connection pool. */ void VCP_Rel(struct conn_pool **); /* - * Release reference to a TCP pool. When last reference is released - * the pool is destroyed and all cached connections closed. + * Release reference to a connection pool. When last reference + * is released the pool is destroyed and all cached connections + * closed. */ int VCP_Open(struct conn_pool *, vtim_dur tmo, VCL_IP *, int*); @@ -96,7 +97,7 @@ struct pfd *VCP_Get(struct conn_pool *, vtim_dur tmo, struct worker *, int VCP_Wait(struct worker *, struct pfd *, vtim_real tmo); /* - * If the connection was recycled (state != VTP_STATE_USED) call this + * If the connection was recycled (state != VCP_STATE_USED) call this * function before attempting to receive on the connection. */ diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index c9baf430a..f2e28b2a6 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -415,7 +415,7 @@ void VSL_ChgId(struct vsl_log *vsl, const char *typ, const char *why, void VSL_End(struct vsl_log *vsl); void VSL_Flush(struct vsl_log *, int overflow); -/* cache_tcp_pool.c */ +/* cache_conn_pool.c */ struct conn_pool; void VCP_Init(void); void VCP_Panic(struct vsb *, struct conn_pool *); diff --git a/bin/varnishtest/tests/c00102.vtc b/bin/varnishtest/tests/c00102.vtc index 96bca461a..a2ad7c672 100644 --- a/bin/varnishtest/tests/c00102.vtc +++ b/bin/varnishtest/tests/c00102.vtc @@ -49,16 +49,16 @@ varnish v1 -vcl { } } -varnish v1 -vsc LCK.tcp_pool.* +varnish v1 -vsc LCK.conn_pool.* varnish v1 -cliok "vcl.discard vcl1" varnish v1 -cliok "vcl.list" -varnish v1 -vsc LCK.tcp_pool.* -varnish v1 -expect LCK.tcp_pool.destroy == 0 +varnish v1 -vsc LCK.conn_pool.* +varnish v1 -expect LCK.conn_pool.destroy == 0 barrier b2 sync client c1 -wait varnish v1 -cliok "vcl.list" server s1 -wait -varnish v1 -vsc LCK.tcp_pool.* -varnish v1 -expect LCK.tcp_pool.destroy == 1 +varnish v1 -vsc LCK.conn_pool.* +varnish v1 -expect LCK.conn_pool.destroy == 1 diff --git a/include/tbl/locks.h b/include/tbl/locks.h index ca0ed0427..8240ae2b3 100644 --- a/include/tbl/locks.h +++ b/include/tbl/locks.h @@ -44,7 +44,7 @@ LOCK(perpool) LOCK(pipestat) LOCK(probe) LOCK(sess) -LOCK(tcp_pool) +LOCK(conn_pool) LOCK(vbe) LOCK(vcapace) LOCK(vcl) From phk at FreeBSD.org Fri Jan 29 21:51:12 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 29 Jan 2021 21:51:12 +0000 (UTC) Subject: [master] d87c71cda Add a VCL_BLOB for a connection preamble to struct vrt_backend (mortally offending FlexeLint in the process.) Message-ID: <20210129215112.AEE0A603F7@lists.varnish-cache.org> commit d87c71cda531c1c7ba26a6e5af28c2758bc9060d Author: Poul-Henning Kamp Date: Fri Jan 29 21:08:19 2021 +0000 Add a VCL_BLOB for a connection preamble to struct vrt_backend (mortally offending FlexeLint in the process.) diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 6d1cb6b77..3e2b67260 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -1068,13 +1068,17 @@ VRT_Format_Proxy(struct vsb *vsb, VCL_INT version, VCL_IP sac, VCL_IP sas, * Clone a struct vrt_endpoint in a single malloc() allocation */ +//lint -e{662} Possible of out-of-bounds pointer (___ beyond end of data) +//lint -e{826} Suspicious pointer-to-pointer conversion (area to o small struct vrt_endpoint * -VRT_Endpoint_Clone(const struct vrt_endpoint *vep) +VRT_Endpoint_Clone(const struct vrt_endpoint * const vep) { size_t sz; struct vrt_endpoint *nvep; + struct vrt_blob *blob = NULL; struct suckaddr *sa; - char *p; + size_t uds_len = 0; + char *p, *e; CHECK_OBJ_NOTNULL(vep, VRT_ENDPOINT_MAGIC); sz = sizeof *nvep; @@ -1082,28 +1086,47 @@ VRT_Endpoint_Clone(const struct vrt_endpoint *vep) sz += vsa_suckaddr_len; if (vep->ipv6) sz += vsa_suckaddr_len; - if (vep->uds_path != NULL) - sz += strlen(vep->uds_path) + 1; - p = malloc(sz); + if (vep->uds_path != NULL) { + uds_len = strlen(vep->uds_path) + 1; + sz += uds_len; + } + if (vep->preamble != NULL && vep->preamble->len) { + sz += sizeof(*blob); + sz += vep->preamble->len; + } + p = calloc(1, sz); AN(p); + e = p + sz; nvep = (void*)p; p += sizeof *nvep; INIT_OBJ(nvep, VRT_ENDPOINT_MAGIC); if (vep->ipv4) { - sa = (void*)p; //lint !e826 - memcpy(sa, vep->ipv4, vsa_suckaddr_len); //lint !e826 + sa = (void*)p; + memcpy(sa, vep->ipv4, vsa_suckaddr_len); nvep->ipv4 = sa; p += vsa_suckaddr_len; } if (vep->ipv6) { - sa = (void*)p; //lint !e826 - memcpy(sa, vep->ipv6, vsa_suckaddr_len); //lint !e826 + sa = (void*)p; + memcpy(sa, vep->ipv6, vsa_suckaddr_len); nvep->ipv6 = sa; p += vsa_suckaddr_len; } - if (vep->uds_path != NULL) { - strcpy(p, vep->uds_path); + if (vep->preamble != NULL && vep->preamble->len) { + /* Before uds because we need p to be aligned still */ + blob = (void*)p; + p += sizeof(*blob); + nvep->preamble = blob; + memcpy(p, vep->preamble->blob, vep->preamble->len); + blob->len = vep->preamble->len; + blob->blob = p; + p += vep->preamble->len; + } + if (uds_len) { + memcpy(p, vep->uds_path, uds_len); nvep->uds_path = p; + p += uds_len; } + assert(p == e); return (nvep); } diff --git a/include/vrt.h b/include/vrt.h index abd4fb2fb..1fa284921 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -398,6 +398,7 @@ struct vrt_endpoint { VCL_IP ipv4; VCL_IP ipv6; const char *uds_path; + const struct vrt_blob *preamble; }; #define VRT_BACKEND_FIELDS(rigid) \ From phk at FreeBSD.org Fri Jan 29 21:51:12 2021 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 29 Jan 2021 21:51:12 +0000 (UTC) Subject: [master] 3f077667d Teach Conn_pool to send preamble, vcc to configure it and test that it works. Message-ID: <20210129215112.E6D2760441@lists.varnish-cache.org> commit 3f077667d47338dcf1d593d6765b01151968072c Author: Poul-Henning Kamp Date: Fri Jan 29 21:49:40 2021 +0000 Teach Conn_pool to send preamble, vcc to configure it and test that it works. diff --git a/bin/varnishd/cache/cache_conn_pool.c b/bin/varnishd/cache/cache_conn_pool.c index 9b29217d8..23b8d4912 100644 --- a/bin/varnishd/cache/cache_conn_pool.c +++ b/bin/varnishd/cache/cache_conn_pool.c @@ -343,9 +343,20 @@ VCP_Open(struct conn_pool *cp, vtim_dur tmo, VCL_IP *ap, int *err) return (-1); } + *err = errno = 0; r = cp->methods->open(cp, tmo, ap); - *err = errno; + if (r >= 0 && errno == 0 && cp->endpoint->preamble != NULL && + cp->endpoint->preamble->len > 0) { + if (write(r, cp->endpoint->preamble->blob, + cp->endpoint->preamble->len) != + cp->endpoint->preamble->len) { + *err = errno; + closefd(&r); + } + } else { + *err = errno; + } if (r >= 0) return (r); @@ -701,6 +712,10 @@ VCP_Ref(const struct vrt_endpoint *vep, const char *ident) VSHA256_Update(cx, vep->ipv6, vsa_suckaddr_len); } } + if (vep->preamble != NULL && vep->preamble->len > 0) { + VSHA256_Update(cx, "PRE", 4); // include \0 + VSHA256_Update(cx, vep->preamble->blob, vep->preamble->len); + } VSHA256_Final(digest, cx); /* diff --git a/bin/varnishtest/tests/b00075.vtc b/bin/varnishtest/tests/b00075.vtc new file mode 100644 index 000000000..bea1b6db2 --- /dev/null +++ b/bin/varnishtest/tests/b00075.vtc @@ -0,0 +1,27 @@ +varnishtest "Test backend preamble" + +server s1 { + rxreq + expect req.method == "POST" + expect req.url == "/preamble" + expect req.proto == "HTTP/7.3" + expect req.http.Header == "42" + rxreq + expect req.method == "GET" + expect req.url == "/" + expect req.proto == "HTTP/1.1" + txresp +} -start + +varnish v1 -vcl { + backend s1 { + .host = "${s1_sock}"; + .preamble = :UE9TVCAvcHJlYW1ibGUgSFRUUC83LjMKSGVhZGVyOiA0MgoKCg==: ; + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c index 98a92c650..d61e973d9 100644 --- a/lib/libvcc/vcc_backend.c +++ b/lib/libvcc/vcc_backend.c @@ -339,6 +339,7 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname) const struct token *t_path = NULL; const struct token *t_hosthdr = NULL; const struct token *t_did = NULL; + const struct token *t_preamble = NULL; struct symbol *pb; struct fld_spec *fs; struct inifin *ifp; @@ -382,6 +383,7 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname) "?probe", "?max_connections", "?proxy_header", + "?preamble", NULL); tl->fb = VSB_new_auto(); @@ -497,6 +499,11 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname) vcc_ErrWhere(tl, tl->t); VSB_destroy(&tl->fb); return; + } else if (vcc_IdIs(t_field, "preamble")) { + ExpectErr(tl, CBLOB); + t_preamble = tl->t; + vcc_NextToken(tl); + SkipToken(tl, ';'); } else { ErrInternal(tl); VSB_destroy(&tl->fb); @@ -534,6 +541,9 @@ vcc_ParseHostDef(struct vcc *tl, const struct token *t_be, const char *vgcname) Emit_UDS_Path(tl, vsb1, t_path, "Backend path"); ERRCHK(tl); + if (t_preamble != NULL) + VSB_printf(vsb1, "\t.preamble = %s,\n", t_preamble->dec); + VSB_printf(vsb1, "};\n"); AZ(VSB_finish(vsb1)); Fh(tl, 0, "%s", VSB_data(vsb1)); From nils.goroll at uplex.de Sat Jan 30 10:57:08 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 30 Jan 2021 10:57:08 +0000 (UTC) Subject: [master] 75acb5cc7 Add disarmed assertions on ctx->method to vgc Message-ID: <20210130105708.6AFCC10E8AC@lists.varnish-cache.org> commit 75acb5cc7aa73fcd22b150dd1a3b899c6ec9bc44 Author: Nils Goroll Date: Sat Jan 30 11:50:27 2021 +0100 Add disarmed assertions on ctx->method to vgc in the course of #3163 we want to add a preamble to VCL subs, which, ideally, would perform as many checks based on constants as possible. This is a first step towards such checks: We add a commented-out assert() on ctx->method for built-in subs only. To add the same for custom subs, we will need the okmask from #3163, for which I would first want to coordinate with phk. Also, I would want to coordinate with him on arming the assertion, as there might be a good reason why assert() is not yet defined for vgc (it is already used in unused macros, though). Having assertions would help humans, flexelint and compilers alike. Until armed, the assertion helps humans at least. The resulting vgc looks like this: void v_matchproto_(vcl_func_f) VGC_function_vcl_backend_error(VRT_CTX) { // assert(ctx->method == (VCL_MET_BACKEND_ERROR)); /* ... from ('Builtin' Line 165 Pos 23) */ diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index b55de8642..bd227fe2f 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -72,6 +72,25 @@ static const struct method method_tab[] = { { NULL, 0U, 0} }; +/*--------------------------------------------------------------------*/ + +static void +vcc_vcl_met2c(struct vsb *vsb, unsigned method) +{ + int d = 0; + +#define VCL_MET_MAC(l,U,t,b) \ + if (method & VCL_MET_##U) { \ + if (d) \ + VSB_putc(vsb, '|'); \ + VSB_cat(vsb, "VCL_MET_" #U); \ + d = 1; \ + } +#include "tbl/vcl_returns.h" + AN(d); +} + + /*--------------------------------------------------------------------*/ void * v_matchproto_(TlAlloc) @@ -147,9 +166,20 @@ vcc_NewProc(struct vcc *tl, struct symbol *sym) static void vcc_EmitProc(struct vcc *tl, struct proc *p) { + struct vsb *vsbm; + AZ(VSB_finish(p->cname)); AZ(VSB_finish(p->prologue)); AZ(VSB_finish(p->body)); + + vsbm = VSB_new_auto(); + if (p->method) { + VSB_cat(vsbm, " // assert(ctx->method == ("); + vcc_vcl_met2c(vsbm, p->method->bitval); + VSB_cat(vsbm, "));\n"); + } + AZ(VSB_finish(vsbm)); + Fh(tl, 1, "vcl_func_f %s;\n", VSB_data(p->cname)); /* * TODO: v_dont_optimize for custom subs called from vcl_init/fini only @@ -162,10 +192,12 @@ vcc_EmitProc(struct vcc *tl, struct proc *p) p->method && p->method->bitval & VCL_MET_TASK_H ? "v_dont_optimize " : ""); Fc(tl, 1, "%s(VRT_CTX)\n", VSB_data(p->cname)); - Fc(tl, 1, "{\n%s\n%s}\n", VSB_data(p->prologue), VSB_data(p->body)); + Fc(tl, 1, "{\n%s%s\n%s}\n", VSB_data(vsbm), VSB_data(p->prologue), + VSB_data(p->body)); VSB_destroy(&p->body); VSB_destroy(&p->prologue); VSB_destroy(&p->cname); + VSB_destroy(&vsbm); } /*--------------------------------------------------------------------*/ From nils.goroll at uplex.de Sat Jan 30 20:25:12 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 30 Jan 2021 20:25:12 +0000 (UTC) Subject: [master] b3f22f4cc vcc: error check in vcc_walksymbols() for the recursive case Message-ID: <20210130202513.20B7D11C9A3@lists.varnish-cache.org> commit b3f22f4ccf70801e042f6dd97649439a017bf9e3 Author: Nils Goroll Date: Sat Jan 30 21:08:14 2021 +0100 vcc: error check in vcc_walksymbols() for the recursive case Bail out upon first error, as for the non-recursive case. diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index 807015c09..96cce270f 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -445,6 +445,7 @@ vcc_walksymbols(struct vcc *tl, const struct symtab *root, assert(strcasecmp(st1->name, st2->name) >= 0); st2 = st1; vcc_walksymbols(tl, st1, func, kind); + ERRCHK(tl); } } From nils.goroll at uplex.de Sun Jan 31 11:17:06 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 31 Jan 2021 11:17:06 +0000 (UTC) Subject: [master] 6f3e06717 vcc: Change structure of method assertion generation slightly Message-ID: <20210131111706.A8EAD10DEE8@lists.varnish-cache.org> commit 6f3e06717dbcbdb6bc90330cfc46775e87f4c9dd Author: Nils Goroll Date: Sun Jan 31 12:10:48 2021 +0100 vcc: Change structure of method assertion generation slightly How I added generation of the assertion code was motivated by how I wanted to lay out more changes in that area later, but generating parts of the function body before the header turned out to not help clarity. Ref 75acb5cc7aa73fcd22b150dd1a3b899c6ec9bc44 diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index bd227fe2f..954316cba 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -172,14 +172,6 @@ vcc_EmitProc(struct vcc *tl, struct proc *p) AZ(VSB_finish(p->prologue)); AZ(VSB_finish(p->body)); - vsbm = VSB_new_auto(); - if (p->method) { - VSB_cat(vsbm, " // assert(ctx->method == ("); - vcc_vcl_met2c(vsbm, p->method->bitval); - VSB_cat(vsbm, "));\n"); - } - AZ(VSB_finish(vsbm)); - Fh(tl, 1, "vcl_func_f %s;\n", VSB_data(p->cname)); /* * TODO: v_dont_optimize for custom subs called from vcl_init/fini only @@ -191,13 +183,19 @@ vcc_EmitProc(struct vcc *tl, struct proc *p) Fc(tl, 1, "\nvoid %sv_matchproto_(vcl_func_f)\n", p->method && p->method->bitval & VCL_MET_TASK_H ? "v_dont_optimize " : ""); - Fc(tl, 1, "%s(VRT_CTX)\n", VSB_data(p->cname)); - Fc(tl, 1, "{\n%s%s\n%s}\n", VSB_data(vsbm), VSB_data(p->prologue), - VSB_data(p->body)); + Fc(tl, 1, "%s(VRT_CTX)\n{\n", VSB_data(p->cname)); + if (p->method) { + vsbm = VSB_new_auto(); + vcc_vcl_met2c(vsbm, p->method->bitval); + AZ(VSB_finish(vsbm)); + Fc(tl, 1, " // assert(ctx->method == (%s));\n", + VSB_data(vsbm)); + VSB_destroy(&vsbm); + } + Fc(tl, 1, "%s\n%s}\n", VSB_data(p->prologue), VSB_data(p->body)); VSB_destroy(&p->body); VSB_destroy(&p->prologue); VSB_destroy(&p->cname); - VSB_destroy(&vsbm); } /*--------------------------------------------------------------------*/ From nils.goroll at uplex.de Sun Jan 31 11:22:07 2021 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 31 Jan 2021 11:22:07 +0000 (UTC) Subject: [master] 2ec7eaa63 forgotten assertion Message-ID: <20210131112207.819AA10E23D@lists.varnish-cache.org> commit 2ec7eaa6321f208604f6f1f676f6311b712f8a01 Author: Nils Goroll Date: Sun Jan 31 12:20:38 2021 +0100 forgotten assertion diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 954316cba..1a2182f41 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -186,6 +186,7 @@ vcc_EmitProc(struct vcc *tl, struct proc *p) Fc(tl, 1, "%s(VRT_CTX)\n{\n", VSB_data(p->cname)); if (p->method) { vsbm = VSB_new_auto(); + AN(vsbm); vcc_vcl_met2c(vsbm, p->method->bitval); AZ(VSB_finish(vsbm)); Fc(tl, 1, " // assert(ctx->method == (%s));\n",