From phk at FreeBSD.org Fri Oct 2 11:03:08 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 2 Oct 2020 11:03:08 +0000 (UTC) Subject: [master] dc37be044 Complete the #3354-saga and also make bereq.bodybytes account only for the actual bytes in the body, not for the (chunked) overhead. Message-ID: <20201002110308.40DF894982@lists.varnish-cache.org> commit dc37be044f1954b3c7dd497e3e7a5a06c5a9bc5a Author: Poul-Henning Kamp Date: Fri Oct 2 11:01:22 2020 +0000 Complete the #3354-saga and also make bereq.bodybytes account only for the actual bytes in the body, not for the (chunked) overhead. diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c index 97c775dbf..9ab9ad711 100644 --- a/bin/varnishd/http1/cache_http1_deliver.c +++ b/bin/varnishd/http1/cache_http1_deliver.c @@ -157,17 +157,11 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody) sc = V1L_Close(req->wrk, &bytes); AZ(req->wrk->v1l); - /* Bytes accounting */ - if (bytes < hdrbytes) - req->acct.resp_hdrbytes += bytes; - else { - req->acct.resp_hdrbytes += hdrbytes; - req->acct.resp_bodybytes += bytes - hdrbytes; - } + req->acct.resp_hdrbytes += hdrbytes; + req->acct.resp_bodybytes += VDP_Close(req); if (sc == SC_NULL && err && req->sp->fd >= 0) sc = SC_REM_CLOSE; if (sc != SC_NULL) Req_Fail(req, sc); - (void)VDP_Close(req); } diff --git a/bin/varnishtest/tests/e00003.vtc b/bin/varnishtest/tests/e00003.vtc index 929e6515c..be3eb1a21 100644 --- a/bin/varnishtest/tests/e00003.vtc +++ b/bin/varnishtest/tests/e00003.vtc @@ -39,8 +39,7 @@ varnish v1 -vcl+backend { logexpect l1 -v v1 -g request { expect 0 1001 Begin "^req .* rxreq" - # ReqAcct body counts include chunked overhead - expect * = ReqAcct "^29 0 29 202 104 306$" + expect * = ReqAcct "^29 0 29 202 75 277$" expect 0 = End } -start @@ -62,9 +61,8 @@ logexpect l4 -v v1 -g request { logexpect l5 -v v1 -g request { expect * 1005 Begin "^req .* rxreq" - # ReqAcct body counts include chunked overhead # Header bytes is 5 larger than in l1 due to two item X-Varnish hdr - expect * = ReqAcct "^29 0 29 207 104 311$" + expect * = ReqAcct "^29 0 29 207 75 282$" expect 0 = End } -start diff --git a/bin/varnishtest/tests/l00003.vtc b/bin/varnishtest/tests/l00003.vtc index bdc23333c..2a60a7991 100644 --- a/bin/varnishtest/tests/l00003.vtc +++ b/bin/varnishtest/tests/l00003.vtc @@ -43,27 +43,27 @@ varnish v1 -vcl+backend { # Total: 93 bytes # Response body: -# Chunk len 5 bytes +# Chunk len - bytes # 123 3 bytes -# Chunk end 2 bytes -# Chunk len 5 bytes +# Chunk end - bytes +# Chunk len - bytes # abc 3 bytes -# Chunk end 2 bytes -# Chunk len 5 bytes +# Chunk end - bytes +# Chunk len - bytes # 123 3 bytes -# Chunk end 2 bytes -# Chunk len 5 bytes +# Chunk end - bytes +# Chunk len - bytes # def 3 bytes -# Chunk end 2 bytes -# Chunk len 5 bytes +# Chunk end - bytes +# Chunk len - bytes # ghi 3 bytes -# Chunk end 2 bytes -# Chunked end 5 bytes -# Total: 55 bytes +# Chunk end - bytes +# Chunked end - bytes +# Total: 15 bytes logexpect l1 -v v1 -g request { expect 0 1001 Begin "^req .* rxreq" - expect * = ReqAcct "^29 0 29 93 55 148$" + expect * = ReqAcct "^29 0 29 93 15 108$" expect 0 = End expect * 1003 Begin "^req .* esi" expect * = ReqAcct "^0 0 0 0 12 12$" @@ -88,4 +88,4 @@ logexpect l1 -wait varnish v1 -expect s_req_hdrbytes == 29 varnish v1 -expect s_req_bodybytes == 0 varnish v1 -expect s_resp_hdrbytes == 93 -varnish v1 -expect s_resp_bodybytes == 55 +varnish v1 -expect s_resp_bodybytes == 15 diff --git a/bin/varnishtest/tests/r3354.vtc b/bin/varnishtest/tests/r3354.vtc index b27d85af8..2886982b7 100644 --- a/bin/varnishtest/tests/r3354.vtc +++ b/bin/varnishtest/tests/r3354.vtc @@ -22,20 +22,34 @@ varnish v1 -vcl+backend { } } -start -varnish v1 -cliok "param.set vsl_mask +VfpAcct" +varnish v1 -cliok "param.set vsl_mask +VfpAcct,+VdpAcct" + +varnish v1 -cliok "param.set feature +esi_disable_xml_check" client c1 { - txreq -url /1 + txreq -method POST -url /1 -bodylen 100 rxresp expect resp.bodylen == 1000 } -run +varnish v1 -expect MAIN.s_req_bodybytes == 100 +varnish v1 -expect VBE.vcl1.s1.bereq_bodybytes == 100 varnish v1 -expect VBE.vcl1.s1.beresp_bodybytes == 1000 +varnish v1 -expect MAIN.s_resp_bodybytes == 1000 +varnish v1 -vsc *bodyb* client c1 { - txreq -url /2 + txreq -method POST -url /2 -nolen -hdr "Transfer-encoding: chunked" + chunkedlen 100 + chunkedlen 100 + chunkedlen 100 + chunkedlen 0 rxresp expect resp.bodylen == 500 } -run +varnish v1 -expect MAIN.s_req_bodybytes == 400 +varnish v1 -expect VBE.vcl1.s1.bereq_bodybytes >= 400 varnish v1 -expect VBE.vcl1.s1.beresp_bodybytes == 1500 +varnish v1 -expect MAIN.s_resp_bodybytes == 1500 +varnish v1 -vsc *bodyb* From phk at FreeBSD.org Sun Oct 4 15:39:06 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 4 Oct 2020 15:39:06 +0000 (UTC) Subject: [master] 17b9cfad4 Unset SSH_AUTH_SOCK so we always use our own key Message-ID: <20201004153906.72F7495FC8@lists.varnish-cache.org> commit 17b9cfad42fdbb6f11d90686106f99330c65ed2d Author: Poul-Henning Kamp Date: Sun Oct 4 15:38:05 2020 +0000 Unset SSH_AUTH_SOCK so we always use our own key diff --git a/tools/vtest.sh b/tools/vtest.sh index 5ebb4e165..bc601ca69 100755 --- a/tools/vtest.sh +++ b/tools/vtest.sh @@ -57,6 +57,9 @@ enable_gcov=false SSH_DST="-p 203 vtest at varnish-cache.org" +# make sure we use our own key +unset SSH_AUTH_SOCK + export REPORTDIR=`pwd`/_report export VTEST_REPORT="${REPORTDIR}/_log" From phk at FreeBSD.org Mon Oct 5 07:25:07 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 5 Oct 2020 07:25:07 +0000 (UTC) Subject: [master] f087fe6d3 Align ESI bodybytes accounting with #3354 Message-ID: <20201005072507.52186BEBE6@lists.varnish-cache.org> commit f087fe6d37679d8d709cfec85451fde4f68fbb95 Author: Poul-Henning Kamp Date: Mon Oct 5 07:23:47 2020 +0000 Align ESI bodybytes accounting with #3354 diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 9bd59b5c7..daf9ece9c 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -475,7 +475,7 @@ static inline int ved_bytes(struct req *req, struct ecx *ecx, enum vdp_action act, const void *ptr, ssize_t len) { - req->acct.resp_bodybytes += len; + (void)req; if (act == VDP_END) act = VDP_FLUSH; return (VDP_bytes(ecx->preq, act, ptr, len)); @@ -871,5 +871,5 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody) if (i && req->doclose == SC_NULL) req->doclose = SC_REM_CLOSE; - (void)VDP_Close(req); + req->acct.resp_bodybytes += VDP_Close(req); } diff --git a/bin/varnishtest/tests/e00000.vtc b/bin/varnishtest/tests/e00000.vtc index 3030a11c7..47bcdc574 100644 --- a/bin/varnishtest/tests/e00000.vtc +++ b/bin/varnishtest/tests/e00000.vtc @@ -32,3 +32,4 @@ client c1 { client c1 -run logexpect l1 -wait varnish v1 -expect esi_errors == 0 +varnish v1 -expect MAIN.s_resp_bodybytes == 33 diff --git a/bin/varnishtest/tests/e00001.vtc b/bin/varnishtest/tests/e00001.vtc index 8a4a42480..c903a0acb 100644 --- a/bin/varnishtest/tests/e00001.vtc +++ b/bin/varnishtest/tests/e00001.vtc @@ -38,3 +38,4 @@ client c1 { client c1 -run logexpect l1 -wait varnish v1 -expect esi_errors == 2 +varnish v1 -expect MAIN.s_resp_bodybytes == 40 diff --git a/bin/varnishtest/tests/e00002.vtc b/bin/varnishtest/tests/e00002.vtc index 244d4bce5..2ca2b1352 100644 --- a/bin/varnishtest/tests/e00002.vtc +++ b/bin/varnishtest/tests/e00002.vtc @@ -27,3 +27,4 @@ client c1 { client c1 -run varnish v1 -expect esi_errors == 0 +varnish v1 -expect MAIN.s_resp_bodybytes == 35 diff --git a/bin/varnishtest/tests/e00003.vtc b/bin/varnishtest/tests/e00003.vtc index be3eb1a21..23041400c 100644 --- a/bin/varnishtest/tests/e00003.vtc +++ b/bin/varnishtest/tests/e00003.vtc @@ -84,6 +84,7 @@ client c1 { client c1 -run varnish v1 -expect esi_errors == 0 +varnish v1 -expect MAIN.s_resp_bodybytes == 150 logexpect l1 -wait logexpect l2 -wait diff --git a/bin/varnishtest/tests/e00004.vtc b/bin/varnishtest/tests/e00004.vtc index 9e19b667f..44a31c760 100644 --- a/bin/varnishtest/tests/e00004.vtc +++ b/bin/varnishtest/tests/e00004.vtc @@ -33,3 +33,4 @@ client c1 { client c1 -run varnish v1 -expect esi_errors == 0 +varnish v1 -expect MAIN.s_resp_bodybytes == 67 diff --git a/bin/varnishtest/tests/e00005.vtc b/bin/varnishtest/tests/e00005.vtc index 832a6eeee..7137195b6 100644 --- a/bin/varnishtest/tests/e00005.vtc +++ b/bin/varnishtest/tests/e00005.vtc @@ -34,3 +34,4 @@ client c1 { client c1 -run varnish v1 -expect esi_errors == 0 +varnish v1 -expect MAIN.s_resp_bodybytes == 67 diff --git a/bin/varnishtest/tests/e00006.vtc b/bin/varnishtest/tests/e00006.vtc index e1b7319c0..b6b511808 100644 --- a/bin/varnishtest/tests/e00006.vtc +++ b/bin/varnishtest/tests/e00006.vtc @@ -43,6 +43,7 @@ client c1 { client c1 -run varnish v1 -expect esi_errors == 0 +varnish v1 -expect MAIN.s_resp_bodybytes == 78 # Now try with invalid URLs @@ -78,14 +79,19 @@ client c1 { rxresp expect resp.status == 200 expect resp.bodylen == 4 +} -run + +varnish v1 -expect esi_errors == 1 +varnish v1 -expect MAIN.s_resp_bodybytes == 82 + +client c1 { txreq -url /https rxresp expect resp.status == 200 expect resp.bodylen == 6 -} - -client c1 -run +} -run logexpect l1 -wait varnish v1 -expect esi_errors == 2 +varnish v1 -expect MAIN.s_resp_bodybytes == 88 diff --git a/bin/varnishtest/tests/e00007.vtc b/bin/varnishtest/tests/e00007.vtc index 015ffbb15..038f45f47 100644 --- a/bin/varnishtest/tests/e00007.vtc +++ b/bin/varnishtest/tests/e00007.vtc @@ -48,3 +48,4 @@ client c1 { client c1 -run varnish v1 -expect esi_errors == 0 +varnish v1 -expect MAIN.s_resp_bodybytes == 120 diff --git a/bin/varnishtest/tests/e00008.vtc b/bin/varnishtest/tests/e00008.vtc index 4c516e003..1264aea55 100644 --- a/bin/varnishtest/tests/e00008.vtc +++ b/bin/varnishtest/tests/e00008.vtc @@ -114,3 +114,4 @@ logexpect l1 -wait logexpect l2 -wait logexpect l3 -wait varnish v1 -expect esi_errors == 18 +varnish v1 -expect MAIN.s_resp_bodybytes == 472 diff --git a/bin/varnishtest/tests/e00009.vtc b/bin/varnishtest/tests/e00009.vtc index 70a820698..e15e56ae7 100644 --- a/bin/varnishtest/tests/e00009.vtc +++ b/bin/varnishtest/tests/e00009.vtc @@ -35,6 +35,7 @@ client c1 { } -run logexpect l1 -wait +varnish v1 -expect MAIN.s_resp_bodybytes == 57 varnish v1 -cli "param.set feature +esi_disable_xml_check" @@ -46,3 +47,4 @@ client c1 { expect resp.http.transfer-encoding == "chunked" } -run varnish v1 -expect esi_errors == 0 +varnish v1 -expect MAIN.s_resp_bodybytes == 79 diff --git a/bin/varnishtest/tests/e00012.vtc b/bin/varnishtest/tests/e00012.vtc index 0786c2181..e73f535b1 100644 --- a/bin/varnishtest/tests/e00012.vtc +++ b/bin/varnishtest/tests/e00012.vtc @@ -32,6 +32,8 @@ client c1 { expect resp.http.Transfer-Encoding == "chunked" } -run +varnish v1 -expect MAIN.s_resp_bodybytes == 67 + client c1 { txreq -url /foo/bar -proto HTTP/1.0 rxresp @@ -43,3 +45,4 @@ client c1 { } -run varnish v1 -expect esi_errors == 0 +varnish v1 -expect MAIN.s_resp_bodybytes == 134 diff --git a/bin/varnishtest/tests/e00015.vtc b/bin/varnishtest/tests/e00015.vtc index 951785115..3edc19747 100644 --- a/bin/varnishtest/tests/e00015.vtc +++ b/bin/varnishtest/tests/e00015.vtc @@ -48,6 +48,7 @@ client c1 { varnish v1 -vsl_catchup varnish v1 -expect esi_errors == 0 +varnish v1 -expect MAIN.s_resp_bodybytes == 149 server s1 { rxreq @@ -131,6 +132,8 @@ client c1 { txreq -url /top2 -hdr "fiddle: do_esi" rxresp expect resp.status == 503 + expect resp.bodylen == 251 } -run varnish v1 -expect esi_errors == 0 +varnish v1 -expect MAIN.s_resp_bodybytes == 865 diff --git a/bin/varnishtest/tests/e00016.vtc b/bin/varnishtest/tests/e00016.vtc index feb86d614..142032366 100644 --- a/bin/varnishtest/tests/e00016.vtc +++ b/bin/varnishtest/tests/e00016.vtc @@ -50,3 +50,4 @@ client c1 { client c1 -run varnish v1 -expect esi_errors == 0 +varnish v1 -expect MAIN.s_resp_bodybytes == 105 diff --git a/bin/varnishtest/tests/e00017.vtc b/bin/varnishtest/tests/e00017.vtc index dd50cedfb..09333378e 100644 --- a/bin/varnishtest/tests/e00017.vtc +++ b/bin/varnishtest/tests/e00017.vtc @@ -86,3 +86,4 @@ client c1 { client c1 -run varnish v1 -expect esi_errors == 0 +varnish v1 -expect MAIN.s_resp_bodybytes == 684 diff --git a/bin/varnishtest/tests/e00019.vtc b/bin/varnishtest/tests/e00019.vtc index 98ed56f3e..b0b08b8df 100644 --- a/bin/varnishtest/tests/e00019.vtc +++ b/bin/varnishtest/tests/e00019.vtc @@ -77,3 +77,4 @@ logexpect l1 -wait varnish v1 -expect esi_errors == 5 varnish v1 -expect esi_warnings == 1 +varnish v1 -expect MAIN.s_resp_bodybytes == 65856 From phk at FreeBSD.org Mon Oct 5 09:32:07 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 5 Oct 2020 09:32:07 +0000 (UTC) Subject: [master] 86d92bcac Eliminate unnecessary req arg from ved_bytes() Message-ID: <20201005093207.1ABC95EAC@lists.varnish-cache.org> commit 86d92bcac984bec13361fa39f7a64571ae90f613 Author: Poul-Henning Kamp Date: Mon Oct 5 07:31:51 2020 +0000 Eliminate unnecessary req arg from ved_bytes() diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index daf9ece9c..80bcca5c9 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -472,10 +472,9 @@ const struct vdp VDP_esi = { * Push bytes to preq */ static inline int -ved_bytes(struct req *req, struct ecx *ecx, enum vdp_action act, +ved_bytes(struct ecx *ecx, enum vdp_action act, const void *ptr, ssize_t len) { - (void)req; if (act == VDP_END) act = VDP_FLUSH; return (VDP_bytes(ecx->preq, act, ptr, len)); @@ -521,7 +520,7 @@ ved_pretend_gzip_bytes(struct req *req, enum vdp_action act, void **priv, (void)priv; if (l == 0) - return (ved_bytes(req, ecx, act, pv, l)); + return (ved_bytes(ecx, act, pv, l)); p = pv; @@ -537,23 +536,23 @@ ved_pretend_gzip_bytes(struct req *req, enum vdp_action act, void **priv, while (l > 0) { if (l >= 65535) { lx = 65535; - if (ved_bytes(req, ecx, VDP_NULL, buf1, sizeof buf1)) + if (ved_bytes(ecx, VDP_NULL, buf1, sizeof buf1)) return (-1); } else { lx = (uint16_t)l; buf2[0] = 0; vle16enc(buf2 + 1, lx); vle16enc(buf2 + 3, ~lx); - if (ved_bytes(req, ecx, VDP_NULL, buf2, sizeof buf2)) + if (ved_bytes(ecx, VDP_NULL, buf2, sizeof buf2)) return (-1); } - if (ved_bytes(req, ecx, VDP_NULL, p, lx)) + if (ved_bytes(ecx, VDP_NULL, p, lx)) return (-1); l -= lx; p += lx; } /* buf1 & buf2 is local, have to flush */ - return (ved_bytes(req, ecx, VDP_FLUSH, NULL, 0)); + return (ved_bytes(ecx, VDP_FLUSH, NULL, 0)); } static const struct vdp ved_pretend_gz = { @@ -623,6 +622,7 @@ ved_gzgz_bytes(struct req *req, enum vdp_action act, void **priv, ssize_t dl; ssize_t l; + (void)req; CAST_OBJ_NOTNULL(foo, *priv, VED_FOO_MAGIC); pp = ptr; if (len > 0) { @@ -643,7 +643,7 @@ ved_gzgz_bytes(struct req *req, enum vdp_action act, void **priv, if (dl > 0) { if (dl > len) dl = len; - if (ved_bytes(req, foo->ecx, act, pp, dl)) + if (ved_bytes(foo->ecx, act, pp, dl)) return(-1); foo->ll += dl; len -= dl; @@ -654,7 +654,7 @@ ved_gzgz_bytes(struct req *req, enum vdp_action act, void **priv, /* Remove the "LAST" bit */ foo->dbits[0] = *pp; foo->dbits[0] &= ~(1U << (foo->last & 7)); - if (ved_bytes(req, foo->ecx, act, foo->dbits, 1)) + if (ved_bytes(foo->ecx, act, foo->dbits, 1)) return (-1); foo->ll++; len--; @@ -666,7 +666,7 @@ ved_gzgz_bytes(struct req *req, enum vdp_action act, void **priv, if (dl > 0) { if (dl > len) dl = len; - if (ved_bytes(req, foo->ecx, act, pp, dl)) + if (ved_bytes(foo->ecx, act, pp, dl)) return (-1); foo->ll += dl; len -= dl; @@ -728,7 +728,7 @@ ved_gzgz_bytes(struct req *req, enum vdp_action act, void **priv, default: WRONG("compiler must be broken"); } - if (ved_bytes(req, foo->ecx, act, foo->dbits + 1, foo->lpad)) + if (ved_bytes(foo->ecx, act, foo->dbits + 1, foo->lpad)) return (-1); } if (len > 0) { @@ -759,6 +759,7 @@ ved_gzgz_fini(struct req *req, void **priv) uint32_t ilen; struct ved_foo *foo; + (void)req; CAST_OBJ_NOTNULL(foo, *priv, VED_FOO_MAGIC); *priv = NULL; @@ -768,7 +769,7 @@ ved_gzgz_fini(struct req *req, void **priv) * * Could rewrite use VDP_END */ - (void)ved_bytes(req, foo->ecx, VDP_FLUSH, NULL, 0); + (void)ved_bytes(foo->ecx, VDP_FLUSH, NULL, 0); icrc = vle32dec(foo->tailbuf); ilen = vle32dec(foo->tailbuf + 4); @@ -805,7 +806,7 @@ ved_vdp_bytes(struct req *req, enum vdp_action act, void **priv, CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC); - return (ved_bytes(req, ecx, act, ptr, len)); + return (ved_bytes(ecx, act, ptr, len)); } static const struct vdp ved_ved = { From phk at FreeBSD.org Mon Oct 5 09:32:07 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 5 Oct 2020 09:32:07 +0000 (UTC) Subject: [master] 30be1c800 Start changing the calling convention for VDP to make it usable on bereq. Message-ID: <20201005093207.2F5685EAF@lists.varnish-cache.org> commit 30be1c800c36c7f48f10a9eaaf750f72f31d0861 Author: Poul-Henning Kamp Date: Mon Oct 5 09:13:11 2020 +0000 Start changing the calling convention for VDP to make it usable on bereq. diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index ea186be28..0e5bdd737 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -58,17 +58,16 @@ * r > 0: Stop, breaks out early without error condition */ int -VDP_bytes(struct req *req, enum vdp_action act, const void *ptr, ssize_t len) +VDP_bytes(struct vdp_ctx *vdx, enum vdp_action act, const void *ptr, ssize_t len) { int retval; struct vdp_entry *vdpe; - struct vdp_ctx *vdc; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - vdc = req->vdc; - if (vdc->retval) - return (vdc->retval); - vdpe = vdc->nxt; + CHECK_OBJ_NOTNULL(vdx, VDP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vdx->req, REQ_MAGIC); + if (vdx->retval) + return (vdx->retval); + vdpe = vdx->nxt; CHECK_OBJ_NOTNULL(vdpe, VDP_ENTRY_MAGIC); /* at most one VDP_END call */ @@ -82,14 +81,14 @@ VDP_bytes(struct req *req, enum vdp_action act, const void *ptr, ssize_t len) assert(act == VDP_FLUSH); /* Call the present layer, while pointing to the next layer down */ - vdc->nxt = VTAILQ_NEXT(vdpe, list); + vdx->nxt = VTAILQ_NEXT(vdpe, list); vdpe->calls++; vdpe->bytes_in += len; - retval = vdpe->vdp->bytes(req, act, &vdpe->priv, ptr, len); - if (retval && (vdc->retval == 0 || retval < vdc->retval)) - vdc->retval = retval; /* Latch error value */ - vdc->nxt = vdpe; - return (vdc->retval); + retval = vdpe->vdp->bytes(vdx, act, &vdpe->priv, ptr, len); + if (retval && (vdx->retval == 0 || retval < vdx->retval)) + vdx->retval = retval; /* Latch error value */ + vdx->nxt = vdpe; + return (vdx->retval); } int @@ -177,11 +176,14 @@ VDP_DeliverObj(struct req *req) int r, final; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + req->vdc->req = req; + req->vdc->vsl = req->vsl; + req->vdc->wrk = req->wrk; final = req->objcore->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP) ? 1 : 0; - r = ObjIterate(req->wrk, req->objcore, req, vdp_objiterator, final); + r = ObjIterate(req->wrk, req->objcore, req->vdc, vdp_objiterator, final); if (r == 0) - r = VDP_bytes(req, VDP_END, NULL, 0); + r = VDP_bytes(req->vdc, VDP_END, NULL, 0); if (r < 0) return (r); return (0); diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 80bcca5c9..ff9cf8d29 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -222,7 +222,7 @@ ved_include(struct req *preq, const char *src, const char *host, #define Debug(fmt, ...) /**/ static ssize_t -ved_decode_len(struct req *req, const uint8_t **pp) +ved_decode_len(struct vsl_log *vsl, const uint8_t **pp) { const uint8_t *p; ssize_t l; @@ -242,7 +242,7 @@ ved_decode_len(struct req *req, const uint8_t **pp) p += 9; break; default: - VSLb(req->vsl, SLT_Error, + VSLb(vsl, SLT_Error, "ESI-corruption: Illegal Length %d %d\n", *p, (*p & 15)); WRONG("ESI-codes: illegal length"); } @@ -300,7 +300,7 @@ ved_vdp_esi_fini(struct req *req, void **priv) } static int v_matchproto_(vdp_bytes_f) -ved_vdp_esi_bytes(struct req *req, enum vdp_action act, void **priv, +ved_vdp_esi_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { uint8_t *q, *r; @@ -312,13 +312,14 @@ ved_vdp_esi_bytes(struct req *req, enum vdp_action act, void **priv, int retval = 0; AN(priv); + CHECK_OBJ_NOTNULL(vdx, VDP_CTX_MAGIC); CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC); pp = ptr; while (1) { switch (ecx->state) { case 0: - ecx->p = ObjGetAttr(req->wrk, req->objcore, + ecx->p = ObjGetAttr(vdx->wrk, vdx->req->objcore, OA_ESIDATA, &l); AN(ecx->p); assert(l > 0); @@ -326,7 +327,7 @@ ved_vdp_esi_bytes(struct req *req, enum vdp_action act, void **priv, if (*ecx->p == VEC_GZ) { if (ecx->pecx == NULL) - retval = VDP_bytes(req, VDP_NULL, + retval = VDP_bytes(vdx, VDP_NULL, gzip_hdr, 10); ecx->l_crc = 0; ecx->crc = crc32(0L, Z_NULL, 0); @@ -344,14 +345,14 @@ ved_vdp_esi_bytes(struct req *req, enum vdp_action act, void **priv, case VEC_V1: case VEC_V2: case VEC_V8: - ecx->l = ved_decode_len(req, &ecx->p); + ecx->l = ved_decode_len(vdx->vsl, &ecx->p); if (ecx->l < 0) return (-1); if (ecx->isgzip) { assert(*ecx->p == VEC_C1 || *ecx->p == VEC_C2 || *ecx->p == VEC_C8); - l = ved_decode_len(req, &ecx->p); + l = ved_decode_len(vdx->vsl, &ecx->p); if (l < 0) return (-1); icrc = vbe32dec(ecx->p); @@ -365,7 +366,7 @@ ved_vdp_esi_bytes(struct req *req, enum vdp_action act, void **priv, case VEC_S1: case VEC_S2: case VEC_S8: - ecx->l = ved_decode_len(req, &ecx->p); + ecx->l = ved_decode_len(vdx->vsl, &ecx->p); if (ecx->l < 0) return (-1); Debug("SKIP1(%d)\n", (int)ecx->l); @@ -378,18 +379,18 @@ ved_vdp_esi_bytes(struct req *req, enum vdp_action act, void **priv, q++; r = (void*)strchr((const char*)q, '\0'); AN(r); - if (VDP_bytes(req, VDP_FLUSH, NULL, 0)) { + if (VDP_bytes(vdx, VDP_FLUSH, NULL, 0)) { ecx->p = ecx->e; break; } Debug("INCL [%s][%s] BEGIN\n", q, ecx->p); - ved_include(req, + ved_include(vdx->req, (const char*)q, (const char*)ecx->p, ecx); Debug("INCL [%s][%s] END\n", q, ecx->p); ecx->p = r + 1; break; default: - VSLb(req->vsl, SLT_Error, + VSLb(vdx->vsl, SLT_Error, "ESI corruption line %d 0x%02x [%s]\n", __LINE__, *ecx->p, ecx->p); WRONG("ESI-codes: Illegal code"); @@ -413,14 +414,14 @@ ved_vdp_esi_bytes(struct req *req, enum vdp_action act, void **priv, /* MOD(2^32) length */ vle32enc(tailbuf + 9, ecx->l_crc); - retval = VDP_bytes(req, VDP_END, tailbuf, 13); + retval = VDP_bytes(vdx, VDP_END, tailbuf, 13); } else if (ecx->pecx != NULL) { ecx->pecx->crc = crc32_combine(ecx->pecx->crc, ecx->crc, ecx->l_crc); ecx->pecx->l_crc += ecx->l_crc; - retval = VDP_bytes(req, VDP_FLUSH, NULL, 0); + retval = VDP_bytes(vdx, VDP_FLUSH, NULL, 0); } else { - retval = VDP_bytes(req, VDP_END, NULL, 0); + retval = VDP_bytes(vdx, VDP_END, NULL, 0); } ecx->state = 99; return (retval); @@ -434,7 +435,7 @@ ved_vdp_esi_bytes(struct req *req, enum vdp_action act, void **priv, */ if (ecx->l <= len) { if (ecx->state == 3) - retval = VDP_bytes(req, act, + retval = VDP_bytes(vdx, act, pp, ecx->l); len -= ecx->l; pp += ecx->l; @@ -442,7 +443,7 @@ ved_vdp_esi_bytes(struct req *req, enum vdp_action act, void **priv, break; } if (ecx->state == 3 && len > 0) - retval = VDP_bytes(req, act, pp, len); + retval = VDP_bytes(vdx, act, pp, len); ecx->l -= len; return (retval); case 99: @@ -477,7 +478,7 @@ ved_bytes(struct ecx *ecx, enum vdp_action act, { if (act == VDP_END) act = VDP_FLUSH; - return (VDP_bytes(ecx->preq, act, ptr, len)); + return (VDP_bytes(ecx->preq->vdc, act, ptr, len)); } /*--------------------------------------------------------------------- @@ -507,7 +508,7 @@ ved_pretend_gzip_fini(struct req *req, void **priv) } static int v_matchproto_(vdp_bytes_f) -ved_pretend_gzip_bytes(struct req *req, enum vdp_action act, void **priv, +ved_pretend_gzip_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, const void *pv, ssize_t l) { uint8_t buf1[5], buf2[5]; @@ -515,7 +516,8 @@ ved_pretend_gzip_bytes(struct req *req, enum vdp_action act, void **priv, uint16_t lx; struct ecx *ecx; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(vdx, VDP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vdx->req, REQ_MAGIC); CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC); (void)priv; @@ -614,7 +616,7 @@ ved_gzgz_init(struct req *req, void **priv) } static int v_matchproto_(vdp_bytes_f) -ved_gzgz_bytes(struct req *req, enum vdp_action act, void **priv, +ved_gzgz_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { struct ved_foo *foo; @@ -622,7 +624,7 @@ ved_gzgz_bytes(struct req *req, enum vdp_action act, void **priv, ssize_t dl; ssize_t l; - (void)req; + (void)vdx; CAST_OBJ_NOTNULL(foo, *priv, VED_FOO_MAGIC); pp = ptr; if (len > 0) { @@ -799,12 +801,12 @@ ved_vdp_fini(struct req *req, void **priv) } static int v_matchproto_(vdp_bytes_f) -ved_vdp_bytes(struct req *req, enum vdp_action act, void **priv, +ved_vdp_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { struct ecx *ecx; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + (void)vdx; CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC); return (ved_bytes(ecx, act, ptr, len)); } diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index 483147196..4a190f6dc 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -110,8 +110,10 @@ typedef int vdp_init_f(struct req *, void **priv); * positive: Don't push this VDP anyway */ +struct vdp_ctx; + typedef int vdp_fini_f(struct req *, void **priv); -typedef int vdp_bytes_f(struct req *, enum vdp_action, void **priv, +typedef int vdp_bytes_f(struct vdp_ctx *, enum vdp_action, void **priv, const void *ptr, ssize_t len); struct vdp { @@ -140,9 +142,12 @@ struct vdp_ctx { int retval; struct vdp_entry_s vdp; struct vdp_entry *nxt; + struct worker *wrk; + struct vsl_log *vsl; + struct req *req; }; -int VDP_bytes(struct req *, enum vdp_action act, const void *ptr, ssize_t len); +int VDP_bytes(struct vdp_ctx *, enum vdp_action act, const void *ptr, ssize_t len); int VDP_Push(struct req *, const struct vdp *, void *priv); void VRT_AddVDP(VRT_CTX, const struct vdp *); void VRT_RemoveVDP(VRT_CTX, const struct vdp *); diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 569ca1e56..b5c04db83 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -354,7 +354,7 @@ vdp_gunzip_fini(struct req *req, void **priv) } static int v_matchproto_(vdp_bytes_f) -vdp_gunzip_bytes(struct req *req, enum vdp_action act, void **priv, +vdp_gunzip_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { enum vgzret_e vr; @@ -363,8 +363,8 @@ vdp_gunzip_bytes(struct req *req, enum vdp_action act, void **priv, struct worker *wrk; struct vgz *vg; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - wrk = req->wrk; + CHECK_OBJ_NOTNULL(vdx, VDP_CTX_MAGIC); + wrk = vdx->wrk; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); (void)act; @@ -386,8 +386,8 @@ vdp_gunzip_bytes(struct req *req, enum vdp_action act, void **priv, if (vr < VGZ_OK) return (-1); if (vg->m_len == vg->m_sz || vr != VGZ_OK) { - if (VDP_bytes(req, VDP_FLUSH, vg->m_buf, vg->m_len)) - return (req->vdc->retval); + if (VDP_bytes(vdx, VDP_FLUSH, vg->m_buf, vg->m_len)) + return (vdx->retval); vg->m_len = 0; VGZ_Obuf(vg, vg->m_buf, vg->m_sz); } diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index 970e75a75..fe9943718 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -61,7 +61,7 @@ vrg_range_fini(struct req *req, void **priv) } static int v_matchproto_(vdp_bytes_f) -vrg_range_bytes(struct req *req, enum vdp_action act, void **priv, +vrg_range_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { int retval = 0; @@ -69,7 +69,7 @@ vrg_range_bytes(struct req *req, enum vdp_action act, void **priv, const char *p = ptr; struct vrg_priv *vrg_priv; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(vdx, VDP_CTX_MAGIC); AN(priv); CAST_OBJ_NOTNULL(vrg_priv, *priv, VRG_PRIV_MAGIC); @@ -88,9 +88,9 @@ vrg_range_bytes(struct req *req, enum vdp_action act, void **priv, if (vrg_priv->range_off >= vrg_priv->range_high) act = VDP_END; if (l > 0) - retval = VDP_bytes(req, act, p, l); + retval = VDP_bytes(vdx, act, p, l); else if (l == 0 && act > VDP_NULL) - retval = VDP_bytes(req, act, p, 0); + retval = VDP_bytes(vdx, act, p, 0); return (retval || act == VDP_END ? 1 : 0); } diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c index 9ab9ad711..42262d384 100644 --- a/bin/varnishd/http1/cache_http1_deliver.c +++ b/bin/varnishd/http1/cache_http1_deliver.c @@ -38,19 +38,19 @@ /*--------------------------------------------------------------------*/ static int v_matchproto_(vdp_bytes_f) -v1d_bytes(struct req *req, enum vdp_action act, void **priv, +v1d_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { ssize_t wl = 0; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(vdx, VDP_CTX_MAGIC); (void)priv; - AZ(req->vdc->nxt); /* always at the bottom of the pile */ + AZ(vdx->nxt); /* always at the bottom of the pile */ if (len > 0) - wl = V1L_Write(req->wrk, ptr, len); - if (act > VDP_NULL && V1L_Flush(req->wrk)) + wl = V1L_Write(vdx->wrk, ptr, len); + if (act > VDP_NULL && V1L_Flush(vdx->wrk)) return (-1); if (len != wl) return (-1); diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index 190356d5d..24e462564 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -100,12 +100,13 @@ h2_fini(struct req *req, void **priv) } static int v_matchproto_(vdp_bytes_f) -h2_bytes(struct req *req, enum vdp_action act, void **priv, +h2_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { struct h2_req *r2; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(vdx, VDP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vdx->req, REQ_MAGIC); CAST_OBJ_NOTNULL(r2, *priv, H2_REQ_MAGIC); (void)act; @@ -113,9 +114,9 @@ h2_bytes(struct req *req, enum vdp_action act, void **priv, return (-1); if (len == 0) return (0); - H2_Send_Get(req->wrk, r2->h2sess, r2); - H2_Send(req->wrk, r2, H2_F_DATA, H2FF_NONE, len, ptr, - &req->acct.resp_bodybytes); + H2_Send_Get(vdx->wrk, r2->h2sess, r2); + H2_Send(vdx->wrk, r2, H2_F_DATA, H2FF_NONE, len, ptr, + &vdx->req->acct.resp_bodybytes); H2_Send_Rel(r2->h2sess, r2); return (0); } From phk at FreeBSD.org Mon Oct 5 09:32:07 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 5 Oct 2020 09:32:07 +0000 (UTC) Subject: [master] da6568c4b Make it possible for VDP filters to report partial processing. Message-ID: <20201005093207.556025EB3@lists.varnish-cache.org> commit da6568c4b62b5289aac3db07b9c87f24897220a0 Author: Poul-Henning Kamp Date: Mon Oct 5 09:27:52 2020 +0000 Make it possible for VDP filters to report partial processing. (Also changes forgotten in previous commit) diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 0e5bdd737..04b57936e 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -83,8 +83,9 @@ VDP_bytes(struct vdp_ctx *vdx, enum vdp_action act, const void *ptr, ssize_t len /* Call the present layer, while pointing to the next layer down */ vdx->nxt = VTAILQ_NEXT(vdpe, list); vdpe->calls++; - vdpe->bytes_in += len; + vdx->bytes_done = len; retval = vdpe->vdp->bytes(vdx, act, &vdpe->priv, ptr, len); + vdpe->bytes_in += vdx->bytes_done; if (retval && (vdx->retval == 0 || retval < vdx->retval)) vdx->retval = retval; /* Latch error value */ vdx->nxt = vdpe; diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index 4a190f6dc..e106b5c1f 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -140,6 +140,7 @@ struct vdp_ctx { unsigned magic; #define VDP_CTX_MAGIC 0xee501df7 int retval; + uint64_t bytes_done; struct vdp_entry_s vdp; struct vdp_entry *nxt; struct worker *wrk; diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index 24e462564..367fd9bf9 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -106,7 +106,6 @@ h2_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, struct h2_req *r2; CHECK_OBJ_NOTNULL(vdx, VDP_CTX_MAGIC); - CHECK_OBJ_NOTNULL(vdx->req, REQ_MAGIC); CAST_OBJ_NOTNULL(r2, *priv, H2_REQ_MAGIC); (void)act; @@ -115,8 +114,8 @@ h2_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, if (len == 0) return (0); H2_Send_Get(vdx->wrk, r2->h2sess, r2); - H2_Send(vdx->wrk, r2, H2_F_DATA, H2FF_NONE, len, ptr, - &vdx->req->acct.resp_bodybytes); + vdx->bytes_done = 0; + H2_Send(vdx->wrk, r2, H2_F_DATA, H2FF_NONE, len, ptr, &vdx->bytes_done); H2_Send_Rel(r2->h2sess, r2); return (0); } @@ -332,5 +331,5 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody) } AZ(req->wrk->v1l); - (void)VDP_Close(req); + req->acct.resp_bodybytes += VDP_Close(req); } diff --git a/bin/varnishtest/tests/r02219.vtc b/bin/varnishtest/tests/r02219.vtc index 0315a2b4a..bb39cf78d 100644 --- a/bin/varnishtest/tests/r02219.vtc +++ b/bin/varnishtest/tests/r02219.vtc @@ -19,7 +19,7 @@ varnish v1 -arg "-p workspace_client=9k" -proto PROXY -vcl+backend { } -start client c1 { - send "PROXY TCP4 127.0.0.1 127.0.0.1 1111 2222\r\nGET /AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA HTTP/1.1\r\n\r\n" + send "PROXY TCP4 127.0.0.1 127.0.0.1 1111 2222\r\nGET /AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA HTTP/1.1\r\n\r\n" rxresp } -run @@ -65,6 +65,6 @@ client c2 { } -run client c3 { - send "PROXY TCP4 127.0.0.1 127.0.0.1 1111 2222\r\nGET /CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC HTTP/1.1\r\n\r\n" + send "PROXY TCP4 127.0.0.1 127.0.0.1 1111 2222\r\nGET /CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC HTTP/1.1\r\n\r\n" rxresp } -run diff --git a/include/vrt.h b/include/vrt.h index 5b16c6261..5e5b032f9 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -52,6 +52,8 @@ * Whenever something is deleted or changed in a way which is not * binary/load-time compatible, increment MAJOR version * + * 13.0 (2020-03-15) + * Calling convention for VDP implementation changed * 12.0 (2020-09-15) * Added VRT_DirectorResolve() * Added VCL_STRING VRT_BLOB_string(VRT_CTX, VCL_BLOB) @@ -166,7 +168,7 @@ * vrt_acl type added */ -#define VRT_MAJOR_VERSION 12U +#define VRT_MAJOR_VERSION 13U #define VRT_MINOR_VERSION 0U diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index af7598b6b..fc8232b91 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -111,17 +111,18 @@ xyzzy_rot13_init(struct req *req, void **priv) } static int v_matchproto_(vdp_bytes_f) -xyzzy_rot13_bytes(struct req *req, enum vdp_action act, void **priv, +xyzzy_rot13_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { char *q; const char *pp; int i, j, retval = 0; + CHECK_OBJ_NOTNULL(vdx, VDP_CTX_MAGIC); AN(priv); AN(*priv); if (len <= 0) - return (VDP_bytes(req, act, ptr, len)); + return (VDP_bytes(vdx, act, ptr, len)); AN(ptr); if (act != VDP_END) act = VDP_FLUSH; @@ -136,14 +137,14 @@ xyzzy_rot13_bytes(struct req *req, enum vdp_action act, void **priv, else q[i] = pp[j]; if (i == ROT13_BUFSZ - 1) { - retval = VDP_bytes(req, act, q, ROT13_BUFSZ); + retval = VDP_bytes(vdx, act, q, ROT13_BUFSZ); if (retval != 0) return (retval); i = -1; } } if (i >= 0) - retval = VDP_bytes(req, act, q, i + 1L); + retval = VDP_bytes(vdx, act, q, i + 1L); return (retval); } From phk at FreeBSD.org Mon Oct 5 11:03:06 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 5 Oct 2020 11:03:06 +0000 (UTC) Subject: [master] 559c10b7c Skip this test in GCOV mode, 68xx bytes per esi level is too much. Message-ID: <20201005110306.AEE799839@lists.varnish-cache.org> commit 559c10b7c7b69ce4c5ea54f55e68d05483ffc076 Author: Poul-Henning Kamp Date: Mon Oct 5 11:02:27 2020 +0000 Skip this test in GCOV mode, 68xx bytes per esi level is too much. diff --git a/bin/varnishtest/tests/v00004.vtc b/bin/varnishtest/tests/v00004.vtc index 0b52fa11b..f3a28062a 100644 --- a/bin/varnishtest/tests/v00004.vtc +++ b/bin/varnishtest/tests/v00004.vtc @@ -2,6 +2,9 @@ varnishtest "test if our default paramers make sense ..." feature 64bit +# Skip this test in GCOV mode, 68xx bytes extra per level makes it fail +feature cmd {[ -n "$GCOV_PROG" ]} + # ... for our definition of a standard use case: # - 2019 header madness # - 5 ESI levels down From nils.goroll at uplex.de Mon Oct 5 11:49:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Oct 2020 11:49:06 +0000 (UTC) Subject: [master] 8816c6f9d dump move info about our client connection to panic output Message-ID: <20201005114906.9CCC460D99@lists.varnish-cache.org> commit 8816c6f9df513ba90ec60712ba629c78fa75ec9c Author: Nils Goroll Date: Mon Oct 5 13:46:55 2020 +0200 dump move info about our client connection to panic output motivated by #3415 diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index fa1ff00d4..7364bfdf7 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -52,6 +52,7 @@ #include "vcli_serve.h" #include "vtim.h" #include "vcs.h" +#include "vtcp.h" /* * The panic string is constructed in a VSB, then copied to the @@ -533,6 +534,16 @@ pan_req(struct vsb *vsb, const struct req *req) /*--------------------------------------------------------------------*/ +#define pan_addr(vsb, sp, field) do { \ + struct suckaddr *sa; \ + char h[VTCP_ADDRBUFSIZE]; \ + char p[VTCP_PORTBUFSIZE]; \ + \ + (void) SES_Get_##field##_addr((sp), &sa); \ + VTCP_name(sa, h, sizeof h, p, sizeof p); \ + VSB_printf((vsb), "%s.ip = %s:%s,\n", #field, h, p); \ + } while (0) + static void pan_sess(struct vsb *vsb, const struct sess *sp) { @@ -561,6 +572,7 @@ pan_sess(struct vsb *vsb, const struct sess *sp) VSB_cat(vsb, "}"); } VSB_cat(vsb, "\n"); + // duplicated below, remove ? ci = SES_Get_String_Attr(sp, SA_CLIENT_IP); cp = SES_Get_String_Attr(sp, SA_CLIENT_PORT); if (VALID_OBJ(sp->listen_sock, LISTEN_SOCK_MAGIC)) @@ -569,6 +581,17 @@ pan_sess(struct vsb *vsb, const struct sess *sp) else VSB_printf(vsb, "client = %s %s \n", ci, cp); + if (VALID_OBJ(sp->listen_sock, LISTEN_SOCK_MAGIC)) { + VSB_printf(vsb, "local.endpoint = %s,\n", + sp->listen_sock->endpoint); + VSB_printf(vsb, "local.socket = %s,\n", + sp->listen_sock->name); + } + pan_addr(vsb, sp, local); + pan_addr(vsb, sp, remote); + pan_addr(vsb, sp, server); + pan_addr(vsb, sp, client); + VSB_indent(vsb, -2); VSB_cat(vsb, "},\n"); } From dridi at varni.sh Mon Oct 5 12:05:14 2020 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 5 Oct 2020 12:05:14 +0000 Subject: [master] 8816c6f9d dump move info about our client connection to panic output In-Reply-To: <20201005114906.9CCC460D99@lists.varnish-cache.org> References: <20201005114906.9CCC460D99@lists.varnish-cache.org> Message-ID: On Mon, Oct 5, 2020 at 11:49 AM Nils Goroll wrote: > > > commit 8816c6f9df513ba90ec60712ba629c78fa75ec9c > Author: Nils Goroll > Date: Mon Oct 5 13:46:55 2020 +0200 > > dump move info about our client connection to panic output > > motivated by #3415 > > diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c > index fa1ff00d4..7364bfdf7 100644 > --- a/bin/varnishd/cache/cache_panic.c > +++ b/bin/varnishd/cache/cache_panic.c > @@ -52,6 +52,7 @@ > #include "vcli_serve.h" > #include "vtim.h" > #include "vcs.h" > +#include "vtcp.h" > > /* > * The panic string is constructed in a VSB, then copied to the > @@ -533,6 +534,16 @@ pan_req(struct vsb *vsb, const struct req *req) > > /*--------------------------------------------------------------------*/ > > +#define pan_addr(vsb, sp, field) do { \ > + struct suckaddr *sa; \ > + char h[VTCP_ADDRBUFSIZE]; \ > + char p[VTCP_PORTBUFSIZE]; \ > + \ > + (void) SES_Get_##field##_addr((sp), &sa); \ > + VTCP_name(sa, h, sizeof h, p, sizeof p); \ Not sure how safe this is, the two lines above can panic so I'd rather avoid dumping addresses from cache_panic.c to mitigate the risk of a truncated output. > + VSB_printf((vsb), "%s.ip = %s:%s,\n", #field, h, p); \ > + } while (0) > + > static void > pan_sess(struct vsb *vsb, const struct sess *sp) > { > @@ -561,6 +572,7 @@ pan_sess(struct vsb *vsb, const struct sess *sp) > VSB_cat(vsb, "}"); > } > VSB_cat(vsb, "\n"); > + // duplicated below, remove ? > ci = SES_Get_String_Attr(sp, SA_CLIENT_IP); > cp = SES_Get_String_Attr(sp, SA_CLIENT_PORT); > if (VALID_OBJ(sp->listen_sock, LISTEN_SOCK_MAGIC)) > @@ -569,6 +581,17 @@ pan_sess(struct vsb *vsb, const struct sess *sp) > else > VSB_printf(vsb, "client = %s %s \n", ci, cp); > > + if (VALID_OBJ(sp->listen_sock, LISTEN_SOCK_MAGIC)) { > + VSB_printf(vsb, "local.endpoint = %s,\n", > + sp->listen_sock->endpoint); > + VSB_printf(vsb, "local.socket = %s,\n", > + sp->listen_sock->name); > + } > + pan_addr(vsb, sp, local); > + pan_addr(vsb, sp, remote); > + pan_addr(vsb, sp, server); > + pan_addr(vsb, sp, client); > + > VSB_indent(vsb, -2); > VSB_cat(vsb, "},\n"); > } > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From nils.goroll at uplex.de Mon Oct 5 12:38:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Oct 2020 12:38:06 +0000 (UTC) Subject: [master] b550ef4b0 avoid asserting in VSA_Sane() Message-ID: <20201005123806.3147162513@lists.varnish-cache.org> commit b550ef4b0f6238262a35d047cb8d2f4a8f4caaa3 Author: Nils Goroll Date: Mon Oct 5 14:29:36 2020 +0200 avoid asserting in VSA_Sane() so far, it is only used in assertions - this change enables use in panic code. diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c index a46e8ac08..a604d4201 100644 --- a/lib/libvarnish/vsa.c +++ b/lib/libvarnish/vsa.c @@ -357,8 +357,7 @@ VSA_Get_Proto(const struct suckaddr *sua) int VSA_Sane(const struct suckaddr *sua) { - CHECK_OBJ_NOTNULL(sua, SUCKADDR_MAGIC); - return (sua_len(&sua->sa) != 0); + return (VALID_OBJ(sua, SUCKADDR_MAGIC) && sua_len(&sua->sa) != 0); } int From nils.goroll at uplex.de Mon Oct 5 12:38:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Oct 2020 12:38:06 +0000 (UTC) Subject: [master] 46901a4d3 avoid potential assertion failures in pan_addr() Message-ID: <20201005123806.476B862516@lists.varnish-cache.org> commit 46901a4d32901fd140441046cdf79783391e9a17 Author: Nils Goroll Date: Mon Oct 5 14:35:27 2020 +0200 avoid potential assertion failures in pan_addr() ref 8816c6f9df513ba90ec60712ba629c78fa75ec9c diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 7364bfdf7..6d7aa5afa 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -53,6 +53,7 @@ #include "vtim.h" #include "vcs.h" #include "vtcp.h" +#include "vsa.h" /* * The panic string is constructed in a VSB, then copied to the @@ -540,6 +541,8 @@ pan_req(struct vsb *vsb, const struct req *req) char p[VTCP_PORTBUFSIZE]; \ \ (void) SES_Get_##field##_addr((sp), &sa); \ + if (! VSA_Sane(sa)) \ + break; \ VTCP_name(sa, h, sizeof h, p, sizeof p); \ VSB_printf((vsb), "%s.ip = %s:%s,\n", #field, h, p); \ } while (0) From nils.goroll at uplex.de Mon Oct 5 13:06:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Oct 2020 13:06:06 +0000 (UTC) Subject: [master] d686166ad http: Add missing HTTP response code Message-ID: <20201005130606.CDCD863474@lists.varnish-cache.org> commit d686166ad469cad4043ed24328f4377e0d5f71a7 Author: BARON David Date: Thu Oct 1 12:50:18 2020 +0900 http: Add missing HTTP response code - Complete the missing HTTP response codes / messages from : https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml diff --git a/bin/varnishtest/tests/p00010.vtc b/bin/varnishtest/tests/p00010.vtc new file mode 100644 index 000000000..1a5a8a25d --- /dev/null +++ b/bin/varnishtest/tests/p00010.vtc @@ -0,0 +1,101 @@ +varnishtest "status codes" + +server s1 {} -start + +varnish v1 -vcl+backend { + import std; + sub vcl_recv { + return (synth(std.integer(req.http.x-vtc-response-code, 999))); + } +} -start + +client c1 { + txreq -hdr "x-vtc-response-code: 102" + rxresp + expect resp.reason == "Processing" + + txreq -hdr "x-vtc-response-code: 103" + rxresp + expect resp.reason == "Early Hints" + + txreq -hdr "x-vtc-response-code: 207" + rxresp + expect resp.reason == "Multi-Status" + + txreq -hdr "x-vtc-response-code: 208" + rxresp + expect resp.reason == "Already Reported" + + txreq -hdr "x-vtc-response-code: 226" + rxresp + expect resp.reason == "IM Used" + + txreq -hdr "x-vtc-response-code: 308" + rxresp + expect resp.reason == "Permanent Redirect" + + txreq -hdr "x-vtc-response-code: 421" + rxresp + expect resp.reason == "Misdirected Request" + + txreq -hdr "x-vtc-response-code: 422" + rxresp + expect resp.reason == "Unprocessable Entity" + + txreq -hdr "x-vtc-response-code: 423" + rxresp + expect resp.reason == "Locked" + + txreq -hdr "x-vtc-response-code: 424" + rxresp + expect resp.reason == "Failed Dependency" + + txreq -hdr "x-vtc-response-code: 425" + rxresp + expect resp.reason == "Too Early" + + txreq -hdr "x-vtc-response-code: 426" + rxresp + expect resp.reason == "Upgrade Required" + + txreq -hdr "x-vtc-response-code: 428" + rxresp + expect resp.reason == "Precondition Required" + + txreq -hdr "x-vtc-response-code: 429" + rxresp + expect resp.reason == "Too Many Requests" + + txreq -hdr "x-vtc-response-code: 431" + rxresp + expect resp.reason == "Request Header Fields Too Large" + txreq -hdr "x-vtc-response-code: 451" + rxresp + expect resp.reason == "Unavailable For Legal Reasons" + + txreq -hdr "x-vtc-response-code: 506" + rxresp + expect resp.reason == "Variant Also Negotiates" + + txreq -hdr "x-vtc-response-code: 507" + rxresp + expect resp.reason == "Insufficient Storage" + + txreq -hdr "x-vtc-response-code: 508" + rxresp + expect resp.reason == "Loop Detected" + + txreq -hdr "x-vtc-response-code: 510" + rxresp + expect resp.reason == "Not Extended" + + txreq -hdr "x-vtc-response-code: 511" + rxresp + expect resp.reason == "Network Authentication Required" + + txreq -hdr "x-vtc-response-code: vtc" + rxresp + expect resp.status == 999 + expect resp.reason == "Unknown HTTP Status" +} -run + diff --git a/include/tbl/http_response.h b/include/tbl/http_response.h index 8f2314639..567353968 100644 --- a/include/tbl/http_response.h +++ b/include/tbl/http_response.h @@ -34,6 +34,8 @@ HTTP_RESP(100, "Continue") HTTP_RESP(101, "Switching Protocols") +HTTP_RESP(102, "Processing") +HTTP_RESP(103, "Early Hints") HTTP_RESP(200, "OK") HTTP_RESP(201, "Created") HTTP_RESP(202, "Accepted") @@ -41,6 +43,9 @@ HTTP_RESP(203, "Non-Authoritative Information") HTTP_RESP(204, "No Content") HTTP_RESP(205, "Reset Content") HTTP_RESP(206, "Partial Content") +HTTP_RESP(207, "Multi-Status") +HTTP_RESP(208, "Already Reported") +HTTP_RESP(226, "IM Used") HTTP_RESP(300, "Multiple Choices") HTTP_RESP(301, "Moved Permanently") HTTP_RESP(302, "Found") @@ -49,6 +54,7 @@ HTTP_RESP(304, "Not Modified") HTTP_RESP(305, "Use Proxy") HTTP_RESP(306, "(Unused)") HTTP_RESP(307, "Temporary Redirect") +HTTP_RESP(308, "Permanent Redirect") HTTP_RESP(400, "Bad Request") HTTP_RESP(401, "Unauthorized") HTTP_RESP(402, "Payment Required") @@ -67,12 +73,27 @@ HTTP_RESP(414, "Request-URI Too Long") HTTP_RESP(415, "Unsupported Media Type") HTTP_RESP(416, "Requested Range Not Satisfiable") HTTP_RESP(417, "Expectation Failed") +HTTP_RESP(421, "Misdirected Request") +HTTP_RESP(422, "Unprocessable Entity") +HTTP_RESP(423, "Locked") +HTTP_RESP(424, "Failed Dependency") +HTTP_RESP(425, "Too Early") +HTTP_RESP(426, "Upgrade Required") +HTTP_RESP(428, "Precondition Required") +HTTP_RESP(429, "Too Many Requests") +HTTP_RESP(431, "Request Header Fields Too Large") +HTTP_RESP(451, "Unavailable For Legal Reasons") HTTP_RESP(500, "Internal Server Error") HTTP_RESP(501, "Not Implemented") HTTP_RESP(502, "Bad Gateway") HTTP_RESP(503, "Service Unavailable") HTTP_RESP(504, "Gateway Timeout") HTTP_RESP(505, "HTTP Version Not Supported") +HTTP_RESP(506, "Variant Also Negotiates") +HTTP_RESP(507, "Insufficient Storage") +HTTP_RESP(508, "Loop Detected") +HTTP_RESP(510, "Not Extended") +HTTP_RESP(511, "Network Authentication Required") #undef HTTP_RESP /*lint -restore */ From dridi.boukelmoune at gmail.com Mon Oct 5 13:09:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Oct 2020 13:09:07 +0000 (UTC) Subject: [master] 4e316c260 vmod: Objects can have methods named like constructors Message-ID: <20201005130907.399DB63758@lists.varnish-cache.org> commit 4e316c260a02c142c1a94a3f8efd7b5e445b7177 Author: Dridi Boukelmoune Date: Wed Sep 30 11:38:19 2020 +0200 vmod: Objects can have methods named like constructors I came across a VMOD that has a method named like the constructor to conceptually reinitialize the object's state. That's an implementation detail of the VMOD in question and arguably undefined behavior in Varnish. In order to not accidentally break this assumption, add some test coverage and document this possibility. diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc index 5a861c680..a64fa7d06 100644 --- a/bin/varnishtest/tests/m00000.vtc +++ b/bin/varnishtest/tests/m00000.vtc @@ -36,6 +36,7 @@ varnish v1 -vcl+backend { sub vcl_init { new objx = dbg.obj(); + objx.obj(); dbg.vsc_new(); call log; } diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst index f4edad883..4497fe095 100644 --- a/doc/sphinx/reference/vmod.rst +++ b/doc/sphinx/reference/vmod.rst @@ -211,6 +211,13 @@ after the instantiation):: foo.method(...); } +Nothing prevents a method to be named like the constructor and the +meaning of such a method is up to the vmod author:: + + $Object foo(...) + $Method .bar(...) + $Method .foo(...) + Object instances are represented as pointers to vmod-implemented C structs. Varnish only provides space to store the address of object instances and ensures that the right object address gets passed to C diff --git a/lib/libvmod_debug/vmod_debug.vcc b/lib/libvmod_debug/vmod_debug.vcc index 094db5311..f565b456c 100644 --- a/lib/libvmod_debug/vmod_debug.vcc +++ b/lib/libvmod_debug/vmod_debug.vcc @@ -74,6 +74,10 @@ $Method VOID .enum(ENUM { phk, des, kristian, mithrandir, martin }) Testing that enums work as part of object and that the parser isn't (too) buggy. +$Method VOID .obj() + +Covering the fact that a method can be named like the constructor. + $Method STRING .string() getter for string diff --git a/lib/libvmod_debug/vmod_debug_obj.c b/lib/libvmod_debug/vmod_debug_obj.c index 8efd6ad50..8bc328290 100644 --- a/lib/libvmod_debug/vmod_debug_obj.c +++ b/lib/libvmod_debug/vmod_debug_obj.c @@ -82,6 +82,14 @@ xyzzy_obj_enum(VRT_CTX, struct xyzzy_debug_obj *o, VCL_ENUM e) assert(!strcmp(e, "martin")); } +VCL_VOID v_matchproto_(td_xyzzy_obj_enum) +xyzzy_obj_obj(VRT_CTX, struct xyzzy_debug_obj *o) +{ + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(o, VMOD_DEBUG_OBJ_MAGIC); +} + VCL_STRING v_matchproto_() xyzzy_obj_foo(VRT_CTX, struct xyzzy_debug_obj *o, VCL_STRING s) { From dridi at varni.sh Mon Oct 5 13:36:48 2020 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 5 Oct 2020 13:36:48 +0000 Subject: [master] 46901a4d3 avoid potential assertion failures in pan_addr() In-Reply-To: <20201005123806.476B862516@lists.varnish-cache.org> References: <20201005123806.476B862516@lists.varnish-cache.org> Message-ID: On Mon, Oct 5, 2020 at 12:38 PM Nils Goroll wrote: > > > commit 46901a4d32901fd140441046cdf79783391e9a17 > Author: Nils Goroll > Date: Mon Oct 5 14:35:27 2020 +0200 > > avoid potential assertion failures in pan_addr() > > ref 8816c6f9df513ba90ec60712ba629c78fa75ec9c > > diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c > index 7364bfdf7..6d7aa5afa 100644 > --- a/bin/varnishd/cache/cache_panic.c > +++ b/bin/varnishd/cache/cache_panic.c > @@ -53,6 +53,7 @@ > #include "vtim.h" > #include "vcs.h" > #include "vtcp.h" > +#include "vsa.h" > > /* > * The panic string is constructed in a VSB, then copied to the > @@ -540,6 +541,8 @@ pan_req(struct vsb *vsb, const struct req *req) > char p[VTCP_PORTBUFSIZE]; \ > \ > (void) SES_Get_##field##_addr((sp), &sa); \ > + if (! VSA_Sane(sa)) \ > + break; \ > VTCP_name(sa, h, sizeof h, p, sizeof p); \ This should solve the risk of crash in VTCP_name() but session attributes can still trigger. > VSB_printf((vsb), "%s.ip = %s:%s,\n", #field, h, p); \ > } while (0) > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From nils.goroll at uplex.de Mon Oct 5 13:44:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Oct 2020 13:44:07 +0000 (UTC) Subject: [master] 2b91e1791 vbf: Prevent pooling of a Connection:close bereq Message-ID: <20201005134407.8E024648EC@lists.varnish-cache.org> commit 2b91e1791c595dcf28feb7a0ac006060a1595efd Author: Dridi Boukelmoune Date: Tue Sep 29 08:14:58 2020 +0200 vbf: Prevent pooling of a Connection:close bereq Once we ask the backend to close the connection after a given request there is no benefit from putting the backend connection back in the pool. It's actually a surefire way to force a subsequent backend fetch to fail its first attempt and go straight to its extra chance. Since we try to maximize connection reuse this would have to come from VCL and a user asking for the backend to close the connection should have a good reason to do so, for example when the backend is known to misbehave under certain circumstances. Closes #3400 Refs #3405 diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 065043871..3f848db63 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -451,6 +451,10 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) if (http_IsStatus(bo->beresp, 304) && vbf_304_logic(bo) < 0) return (F_STP_ERROR); + if (bo->htc->doclose == SC_NULL && + http_GetHdrField(bo->bereq, H_Connection, "close", NULL)) + bo->htc->doclose = SC_REQ_CLOSE; + VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL); if (bo->htc != NULL && bo->htc->doclose == SC_NULL && diff --git a/bin/varnishtest/tests/b00073.vtc b/bin/varnishtest/tests/b00073.vtc index 6cae03450..3372ba094 100644 --- a/bin/varnishtest/tests/b00073.vtc +++ b/bin/varnishtest/tests/b00073.vtc @@ -2,28 +2,54 @@ varnishtest "backend connection close" server s1 { rxreq + expect req.http.connection ~ close + expect req.http.beresp-connection !~ close + txresp + expect_close + + accept + rxreq + expect req.http.connection !~ close expect req.http.beresp-connection ~ close txresp expect_close accept rxreq + expect req.http.connection !~ close expect req.http.beresp-connection !~ close txresp -hdr "connection: close" expect_close + + accept + rxreq + expect req.http.connection ~ close + expect req.http.unset-connection == true + txresp + expect_close } -start varnish v1 -vcl+backend { sub vcl_recv { return (pass); } + sub vcl_backend_fetch { + set bereq.http.connection = bereq.http.bereq-connection; + } sub vcl_backend_response { + if (bereq.http.unset-connection) { + unset bereq.http.connection; + } # NB: this overrides unconditionally on purpose set beresp.http.connection = bereq.http.beresp-connection; } } -start client c1 { + txreq -hdr "bereq-connection: close, x-varnish" + rxresp + expect resp.status == 200 + txreq -hdr "beresp-connection: close, x-varnish" rxresp expect resp.status == 200 @@ -31,6 +57,10 @@ client c1 { txreq rxresp expect resp.status == 200 + + txreq -hdr "bereq-connection: close" -hdr "unset-connection: true" + rxresp + expect resp.status == 200 } -run server s1 -wait From nils.goroll at uplex.de Mon Oct 5 13:46:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Oct 2020 13:46:06 +0000 (UTC) Subject: [master] b2a37dcde v1l rename res->ws_snap Message-ID: <20201005134606.71D0064B6E@lists.varnish-cache.org> commit b2a37dcde8e78c23d9d3ba5416966e2babfd0946 Author: Nils Goroll Date: Wed Sep 16 10:50:55 2020 +0200 v1l rename res->ws_snap to clarify that it holds a workspace snapshot diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c index 7b595ca02..cd5020bd5 100644 --- a/bin/varnishd/http1/cache_http1_line.c +++ b/bin/varnishd/http1/cache_http1_line.c @@ -66,7 +66,7 @@ struct v1l { struct vsl_log *vsl; ssize_t cnt; /* Flushed byte count */ struct ws *ws; - uintptr_t res; + uintptr_t ws_snap; }; /*-------------------------------------------------------------------- @@ -80,7 +80,7 @@ V1L_Open(struct worker *wrk, struct ws *ws, int *fd, struct vsl_log *vsl, { struct v1l *v1l; unsigned u; - uintptr_t res; + uintptr_t ws_snap; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); AZ(wrk->v1l); @@ -91,7 +91,7 @@ V1L_Open(struct worker *wrk, struct ws *ws, int *fd, struct vsl_log *vsl, if (niov != 0) assert(niov >= 3); - res = WS_Snapshot(ws); + ws_snap = WS_Snapshot(ws); v1l = WS_Alloc(ws, sizeof *v1l); if (v1l == NULL) @@ -99,7 +99,7 @@ V1L_Open(struct worker *wrk, struct ws *ws, int *fd, struct vsl_log *vsl, INIT_OBJ(v1l, V1L_MAGIC); v1l->ws = ws; - v1l->res = res; + v1l->ws_snap = ws_snap; u = WS_ReserveLumps(ws, sizeof(struct iovec)); if (u < 3) { @@ -139,7 +139,7 @@ V1L_Close(struct worker *wrk, uint64_t *cnt) *cnt = v1l->cnt; if (WS_IsReserved(v1l->ws)) WS_Release(v1l->ws, 0); - WS_Rollback(v1l->ws, v1l->res); + WS_Rollback(v1l->ws, v1l->ws_snap); ZERO_OBJ(v1l, sizeof *v1l); return (sc); } From nils.goroll at uplex.de Mon Oct 5 13:46:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Oct 2020 13:46:06 +0000 (UTC) Subject: [master] 33330796d v1l: simplify workspace reservation handling Message-ID: <20201005134606.8729E64B72@lists.varnish-cache.org> commit 33330796df3526a649352539f2bc860e83ab5aca Author: Nils Goroll Date: Wed Sep 16 10:55:16 2020 +0200 v1l: simplify workspace reservation handling For niov == 0, we reserve up to IOV_MAX io vectors, otherwise niov io vectors capped to IOV_MAX. We used to handle the two cases differently and delay the WS_Release for niov == 0 until V1L_Close(). There is no need to do so, in both cases we can just reduce the reservation (turned into allocation) to the amount of space we actually use. As we are rolling back the workspace in V1L_Close(), this neither has any influence on the state after the close. diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c index cd5020bd5..2499c68cb 100644 --- a/bin/varnishd/http1/cache_http1_line.c +++ b/bin/varnishd/http1/cache_http1_line.c @@ -120,8 +120,7 @@ V1L_Open(struct worker *wrk, struct ws *ws, int *fd, struct vsl_log *vsl, v1l->vsl = vsl; wrk->v1l = v1l; - if (niov != 0) - WS_Release(ws, u * sizeof(struct iovec)); + WS_Release(ws, u * sizeof(struct iovec)); } enum sess_close @@ -137,8 +136,6 @@ V1L_Close(struct worker *wrk, uint64_t *cnt) wrk->v1l = NULL; CHECK_OBJ_NOTNULL(v1l, V1L_MAGIC); *cnt = v1l->cnt; - if (WS_IsReserved(v1l->ws)) - WS_Release(v1l->ws, 0); WS_Rollback(v1l->ws, v1l->ws_snap); ZERO_OBJ(v1l, sizeof *v1l); return (sc); From nils.goroll at uplex.de Mon Oct 5 13:46:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Oct 2020 13:46:06 +0000 (UTC) Subject: [master] 177d97c1a vsb: Harden destroy and fini operations Message-ID: <20201005134606.A113E64B77@lists.varnish-cache.org> commit 177d97c1a193494a30dbaeb5dfdf09e309655cd0 Author: Dridi Boukelmoune Date: Mon Sep 14 11:39:15 2020 +0200 vsb: Harden destroy and fini operations They either take both a dynamic struct and buffer, or both are static. diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c index 0cbc66305..0a185c9d6 100644 --- a/lib/libvarnish/vsb.c +++ b/lib/libvarnish/vsb.c @@ -529,6 +529,7 @@ VSB_fini(struct vsb *s) assert_VSB_integrity(s); assert(!VSB_ISDYNAMIC(s)); + assert(!VSB_ISDYNSTRUCT(s)); memset(s, 0, sizeof(*s)); } @@ -539,6 +540,7 @@ VSB_destroy(struct vsb **s) AN(s); assert_VSB_integrity(*s); assert(VSB_ISDYNAMIC(*s)); + assert(VSB_ISDYNSTRUCT(*s)); SBFREE((*s)->s_buf); memset(*s, 0, sizeof(**s)); SBFREE(*s); From dridi.boukelmoune at gmail.com Mon Oct 5 13:47:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Oct 2020 13:47:06 +0000 (UTC) Subject: [master] afc5b41e1 http: Mark Keep-Alive as a hop-by-hop header Message-ID: <20201005134706.D786065049@lists.varnish-cache.org> commit afc5b41e16253564bb1186165a0e3471f58b24b8 Author: Mark Felder Date: Tue Sep 22 17:09:53 2020 -0500 http: Mark Keep-Alive as a hop-by-hop header diff --git a/include/tbl/http_headers.h b/include/tbl/http_headers.h index 5ce6b673a..324adcfc1 100644 --- a/include/tbl/http_headers.h +++ b/include/tbl/http_headers.h @@ -52,7 +52,6 @@ #define S HTTPH_A_PASS #define H(s,e,f) HTTPH(s, e, f) -H("Keep-Alive", H_Keep_Alive, P|F |S) // 2068 H("Accept", H_Accept, 0 ) // 2616 14.1 H("Accept-Charset", H_Accept_Charset, 0 ) // 2616 14.2 H("Accept-Encoding", H_Accept_Encoding, 0 ) // 2616 14.3 @@ -83,6 +82,7 @@ H("If-Modified-Since", H_If_Modified_Since, F ) // 2616 14.25 H("If-None-Match", H_If_None_Match, F ) // 2616 14.26 H("If-Range", H_If_Range, F ) // 2616 14.27 H("If-Unmodified-Since",H_If_Unmodified_Since, F ) // 2616 14.28 +H("Keep-Alive", H_Keep_Alive, P|F|I|S) // 2616 13.5.1 H("Last-Modified", H_Last_Modified, 0 ) // 2616 14.29 H("Location", H_Location, 0 ) // 2616 14.30 H("Max-Forwards", H_Max_Forwards, 0 ) // 2616 14.31 From dridi.boukelmoune at gmail.com Mon Oct 5 13:47:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Oct 2020 13:47:06 +0000 (UTC) Subject: [master] a5d7ec335 vtc: Test case for backend Keep-Alive filtering Message-ID: <20201005134707.011E26504C@lists.varnish-cache.org> commit a5d7ec335d4c4ab109e2cfa15b6784c251035efb Author: Dridi Boukelmoune Date: Tue Sep 29 11:23:22 2020 +0200 vtc: Test case for backend Keep-Alive filtering Close #3417 diff --git a/bin/varnishtest/tests/r03417.vtc b/bin/varnishtest/tests/r03417.vtc new file mode 100644 index 000000000..fcb89f601 --- /dev/null +++ b/bin/varnishtest/tests/r03417.vtc @@ -0,0 +1,14 @@ +varnishtest "Filter Keep-Alive out from beresp to resp" + +server s1 { + rxreq + txresp -hdr "Keep-Alive: timeout=5, max=1000" +} -start + +varnish v1 -vcl+backend "" -start + +client c1 { + txreq + rxresp + expect resp.http.Keep-Alive == +} -run From dridi.boukelmoune at gmail.com Mon Oct 5 13:47:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 5 Oct 2020 13:47:07 +0000 (UTC) Subject: [master] 9210c4685 http2: Don't send resp connection-specific headers Message-ID: <20201005134707.2373C65051@lists.varnish-cache.org> commit 9210c4685f7b0ca9bb3ae5598f014843c8c43bd1 Author: Dridi Boukelmoune Date: Tue Sep 29 12:44:00 2020 +0200 http2: Don't send resp connection-specific headers Some browsers are strict about this and simply drop responses containing such headers. Since this is not filtering a context switch between a client and a backend transaction (or cache hit) a new filtering flag is added to the HTTP headers table for connection-specific headers. This new flag cannot be compounded as HTTPH_R_FETCH|HTTPH_A_INS because the TE header is an exception and left alone, even though trailers aren't supported. Better diff with the --ignore-all-space option. We could go further and consider any client request containing one as malformed as mandated by RFC 7540. Closes #3416 diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 40fafc766..215ed8759 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -651,11 +651,13 @@ int HTTP_IterHdrPack(struct worker *, struct objcore *, const char **); for ((ptr) = NULL; HTTP_IterHdrPack(wrk, oc, &(ptr));) const char *HTTP_GetHdrPack(struct worker *, struct objcore *, const char *hdr); enum sess_close http_DoConnection(struct http *hp); +int http_IsFiltered(const struct http *hp, unsigned u, unsigned how); -#define HTTPH_R_PASS (1 << 0) /* Request (c->b) in pass mode */ -#define HTTPH_R_FETCH (1 << 1) /* Request (c->b) for fetch */ -#define HTTPH_A_INS (1 << 2) /* Response (b->o) for insert */ -#define HTTPH_A_PASS (1 << 3) /* Response (b->o) for pass */ +#define HTTPH_R_PASS (1 << 0) /* Request (c->b) in pass mode */ +#define HTTPH_R_FETCH (1 << 1) /* Request (c->b) for fetch */ +#define HTTPH_A_INS (1 << 2) /* Response (b->o) for insert */ +#define HTTPH_A_PASS (1 << 3) /* Response (b->o) for pass */ +#define HTTPH_C_SPECIFIC (1 << 4) /* Connection-specific */ #define HTTPH(a, b, c) extern char b[]; #include "tbl/http_headers.h" diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 9fa4c1f64..6986860ce 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -904,6 +904,13 @@ http_isfiltered(const struct http *fm, unsigned u, unsigned how) return (0); } +int +http_IsFiltered(const struct http *fm, unsigned u, unsigned how) +{ + + return (http_isfiltered(fm, u, how)); +} + /*-------------------------------------------------------------------- * Estimate how much workspace we need to Filter this header according * to 'how'. diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index 367fd9bf9..c93145322 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -241,6 +241,9 @@ h2_build_headers(struct vsb *resp, struct req *req) r = strchr(hp->hd[u].b, ':'); AN(r); + if (http_IsFiltered(hp, u, HTTPH_C_SPECIFIC)) + continue; //rfc7540,l,2999,3006 + hps = hp_idx[tolower(*hp->hd[u].b)]; sz = 1 + r - hp->hd[u].b; assert(sz > 0); @@ -295,6 +298,8 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody) VSLb(req->vsl, SLT_RespProtocol, "HTTP/2.0"); + (void)http_DoConnection(req->resp); + ss = WS_Snapshot(req->ws); WS_VSB_new(resp, req->ws); diff --git a/bin/varnishtest/tests/r03416.vtc b/bin/varnishtest/tests/r03416.vtc new file mode 100644 index 000000000..3d7431353 --- /dev/null +++ b/bin/varnishtest/tests/r03416.vtc @@ -0,0 +1,25 @@ +varnishtest "Filter hop-by-hop headers out of h2 responses" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -cliok "param.set feature +http2" +varnish v1 -vcl+backend { + sub vcl_deliver { + set resp.http.Keep-Alive = "timeout=5, max=1000"; + set resp.http.Connection = "other"; + set resp.http.Other = "foo"; + } +} -start + +client c1 { + stream 1 { + txreq + rxresp + expect resp.http.keep-alive == + expect resp.http.connection == + expect resp.http.other == + } -run +} -run diff --git a/include/tbl/http_headers.h b/include/tbl/http_headers.h index 324adcfc1..416d6a8f6 100644 --- a/include/tbl/http_headers.h +++ b/include/tbl/http_headers.h @@ -42,7 +42,8 @@ /* Shorthand for this file only, to keep table narrow */ -#if defined(P) || defined(F) || defined(I) || defined(H) || defined(S) +#if defined(P) || defined(F) || defined(I) || defined(H) || defined(S) || \ + defined(K) #error "Macro overloading" // Trust but verify #endif @@ -50,65 +51,67 @@ #define F HTTPH_R_FETCH #define I HTTPH_A_INS #define S HTTPH_A_PASS +#define K HTTPH_C_SPECIFIC #define H(s,e,f) HTTPH(s, e, f) -H("Accept", H_Accept, 0 ) // 2616 14.1 -H("Accept-Charset", H_Accept_Charset, 0 ) // 2616 14.2 -H("Accept-Encoding", H_Accept_Encoding, 0 ) // 2616 14.3 -H("Accept-Language", H_Accept_Language, 0 ) // 2616 14.4 -H("Accept-Ranges", H_Accept_Ranges, P|F|I ) // 2616 14.5 -H("Age", H_Age, I|S) // 2616 14.6 -H("Allow", H_Allow, 0 ) // 2616 14.7 -H("Authorization", H_Authorization, 0 ) // 2616 14.8 -H("Cache-Control", H_Cache_Control, F ) // 2616 14.9 -H("Connection", H_Connection, P|F|I|S) // 2616 14.10 -H("Content-Encoding", H_Content_Encoding, 0 ) // 2616 14.11 -H("Content-Language", H_Content_Language, 0 ) // 2616 14.12 -H("Content-Length", H_Content_Length, 0 ) // 2616 14.13 -H("Content-Location", H_Content_Location, 0 ) // 2616 14.14 -H("Content-MD5", H_Content_MD5, 0 ) // 2616 14.15 -H("Content-Range", H_Content_Range, F|I ) // 2616 14.16 -H("Content-Type", H_Content_Type, 0 ) // 2616 14.17 -H("Cookie", H_Cookie, 0 ) // 6265 4.2 -H("Date", H_Date, 0 ) // 2616 14.18 -H("ETag", H_ETag, 0 ) // 2616 14.19 -H("Expect", H_Expect, 0 ) // 2616 14.20 -H("Expires", H_Expires, 0 ) // 2616 14.21 -H("From", H_From, 0 ) // 2616 14.22 -H("Host", H_Host, 0 ) // 2616 14.23 -H("HTTP2-Settings", H_HTTP2_Settings, P|F|I|S) // 7540 3.2.1 -H("If-Match", H_If_Match, F ) // 2616 14.24 -H("If-Modified-Since", H_If_Modified_Since, F ) // 2616 14.25 -H("If-None-Match", H_If_None_Match, F ) // 2616 14.26 -H("If-Range", H_If_Range, F ) // 2616 14.27 -H("If-Unmodified-Since",H_If_Unmodified_Since, F ) // 2616 14.28 -H("Keep-Alive", H_Keep_Alive, P|F|I|S) // 2616 13.5.1 -H("Last-Modified", H_Last_Modified, 0 ) // 2616 14.29 -H("Location", H_Location, 0 ) // 2616 14.30 -H("Max-Forwards", H_Max_Forwards, 0 ) // 2616 14.31 -H("Pragma", H_Pragma, 0 ) // 2616 14.32 -H("Proxy-Authenticate", H_Proxy_Authenticate, F|I ) // 2616 14.33 -H("Proxy-Authorization",H_Proxy_Authorization, F|I ) // 2616 14.34 -H("Range", H_Range, F|I ) // 2616 14.35 -H("Referer", H_Referer, 0 ) // 2616 14.36 -H("Retry-After", H_Retry_After, 0 ) // 2616 14.37 -H("Server", H_Server, 0 ) // 2616 14.38 -H("Set-Cookie", H_Set_Cookie, 0 ) // 6265 4.1 -H("TE", H_TE, P|F|I|S) // 2616 14.39 -H("Trailer", H_Trailer, P|F|I|S) // 2616 14.40 -H("Transfer-Encoding", H_Transfer_Encoding, P|F|I|S) // 2616 14.41 -H("Upgrade", H_Upgrade, P|F|I|S) // 2616 14.42 -H("User-Agent", H_User_Agent, 0 ) // 2616 14.43 -H("Vary", H_Vary, 0 ) // 2616 14.44 -H("Via", H_Via, 0 ) // 2616 14.45 -H("Warning", H_Warning, 0 ) // 2616 14.46 -H("WWW-Authenticate", H_WWW_Authenticate, 0 ) // 2616 14.47 -H("X-Forwarded-For", H_X_Forwarded_For, 0 ) // No RFC +H("Accept", H_Accept, 0 ) // 2616 14.1 +H("Accept-Charset", H_Accept_Charset, 0 ) // 2616 14.2 +H("Accept-Encoding", H_Accept_Encoding, 0 ) // 2616 14.3 +H("Accept-Language", H_Accept_Language, 0 ) // 2616 14.4 +H("Accept-Ranges", H_Accept_Ranges, P|F|I ) // 2616 14.5 +H("Age", H_Age, I|S ) // 2616 14.6 +H("Allow", H_Allow, 0 ) // 2616 14.7 +H("Authorization", H_Authorization, 0 ) // 2616 14.8 +H("Cache-Control", H_Cache_Control, F ) // 2616 14.9 +H("Connection", H_Connection, P|F|I|S|K) // 2616 14.10 +H("Content-Encoding", H_Content_Encoding, 0 ) // 2616 14.11 +H("Content-Language", H_Content_Language, 0 ) // 2616 14.12 +H("Content-Length", H_Content_Length, 0 ) // 2616 14.13 +H("Content-Location", H_Content_Location, 0 ) // 2616 14.14 +H("Content-MD5", H_Content_MD5, 0 ) // 2616 14.15 +H("Content-Range", H_Content_Range, F|I ) // 2616 14.16 +H("Content-Type", H_Content_Type, 0 ) // 2616 14.17 +H("Cookie", H_Cookie, 0 ) // 6265 4.2 +H("Date", H_Date, 0 ) // 2616 14.18 +H("ETag", H_ETag, 0 ) // 2616 14.19 +H("Expect", H_Expect, 0 ) // 2616 14.20 +H("Expires", H_Expires, 0 ) // 2616 14.21 +H("From", H_From, 0 ) // 2616 14.22 +H("Host", H_Host, 0 ) // 2616 14.23 +H("HTTP2-Settings", H_HTTP2_Settings, P|F|I|S|K) // 7540 3.2.1 +H("If-Match", H_If_Match, F ) // 2616 14.24 +H("If-Modified-Since", H_If_Modified_Since, F ) // 2616 14.25 +H("If-None-Match", H_If_None_Match, F ) // 2616 14.26 +H("If-Range", H_If_Range, F ) // 2616 14.27 +H("If-Unmodified-Since",H_If_Unmodified_Since, F ) // 2616 14.28 +H("Keep-Alive", H_Keep_Alive, P|F|I|S|K) // 2616 13.5.1 +H("Last-Modified", H_Last_Modified, 0 ) // 2616 14.29 +H("Location", H_Location, 0 ) // 2616 14.30 +H("Max-Forwards", H_Max_Forwards, 0 ) // 2616 14.31 +H("Pragma", H_Pragma, 0 ) // 2616 14.32 +H("Proxy-Authenticate", H_Proxy_Authenticate, F|I ) // 2616 14.33 +H("Proxy-Authorization",H_Proxy_Authorization, F|I ) // 2616 14.34 +H("Range", H_Range, F|I ) // 2616 14.35 +H("Referer", H_Referer, 0 ) // 2616 14.36 +H("Retry-After", H_Retry_After, 0 ) // 2616 14.37 +H("Server", H_Server, 0 ) // 2616 14.38 +H("Set-Cookie", H_Set_Cookie, 0 ) // 6265 4.1 +H("TE", H_TE, P|F|I|S ) // 2616 14.39 +H("Trailer", H_Trailer, P|F|I|S ) // 2616 14.40 +H("Transfer-Encoding", H_Transfer_Encoding, P|F|I|S|K) // 2616 14.41 +H("Upgrade", H_Upgrade, P|F|I|S|K) // 2616 14.42 +H("User-Agent", H_User_Agent, 0 ) // 2616 14.43 +H("Vary", H_Vary, 0 ) // 2616 14.44 +H("Via", H_Via, 0 ) // 2616 14.45 +H("Warning", H_Warning, 0 ) // 2616 14.46 +H("WWW-Authenticate", H_WWW_Authenticate, 0 ) // 2616 14.47 +H("X-Forwarded-For", H_X_Forwarded_For, 0 ) // No RFC #undef P #undef F #undef I #undef S +#undef K #undef H #undef HTTPH From nils.goroll at uplex.de Mon Oct 5 14:44:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Oct 2020 14:44:06 +0000 (UTC) Subject: [master] d0e986a6d dump argv to panic buffer Message-ID: <20201005144406.AAC476ED0F@lists.varnish-cache.org> commit d0e986a6dd0cd0801531d5c7f9c05d11c24a676b Author: Nils Goroll Date: Mon Oct 5 16:42:24 2020 +0200 dump argv to panic buffer This is solely to make bug reports more useful diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 6d7aa5afa..61bde5f43 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -713,6 +713,22 @@ pan_threadattr(struct vsb *vsb) } #endif +static void +pan_argv(struct vsb *vsb) +{ + int i; + + VSB_printf(pan_vsb, "argv = {\n"); + VSB_indent(vsb, 2); + for (i = 0; i < heritage.argc; i++) { + VSB_printf(vsb, "[%d] = ", i); + VSB_quote(vsb, heritage.argv[i], -1, VSB_QUOTE_CSTR); + VSB_cat(vsb, ",\n"); + } + VSB_cat(vsb, "}\n"); + VSB_indent(vsb, -2); + +} /*--------------------------------------------------------------------*/ static void __attribute__((__noreturn__)) @@ -781,6 +797,8 @@ pan_ic(const char *func, const char *file, int line, const char *cond, if (err) VSB_printf(pan_vsb, "errno = %d (%s)\n", err, vstrerror(err)); + pan_argv(pan_vsb); + VSB_printf(pan_vsb, "pthread.self = %p\n", TRUST_ME(pthread_self())); q = THR_GetName(); diff --git a/bin/varnishd/common/heritage.h b/bin/varnishd/common/heritage.h index d33fb4d3f..e6daa8db3 100644 --- a/bin/varnishd/common/heritage.h +++ b/bin/varnishd/common/heritage.h @@ -89,6 +89,9 @@ struct heritage { unsigned min_vcl_version; unsigned max_vcl_version; + + int argc; + char * const * argv; }; extern struct heritage heritage; diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 7eb7669cc..65ccfcb3d 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -489,6 +489,9 @@ main(int argc, char * const *argv) struct vsb *vsb; pid_t pid; + heritage.argc = argc; + heritage.argv = argv; + setbuf(stdout, NULL); setbuf(stderr, NULL); From nils.goroll at uplex.de Mon Oct 5 15:02:49 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Oct 2020 17:02:49 +0200 Subject: [master] 46901a4d3 avoid potential assertion failures in pan_addr() In-Reply-To: References: <20201005123806.476B862516@lists.varnish-cache.org> Message-ID: <46237493-4c7f-a9e6-2837-2efa2c94a55a@uplex.de> On 05/10/2020 15:36, Dridi Boukelmoune wrote: > This should solve the risk of crash in VTCP_name() but session > attributes can still trigger. yes, that existed before, and not any more. But we'll likely have the same issue elsewhere - in general, we should not deref any pointers from structs with mismatched magic. I won't clean up all the places (now) because I have not seen such an issue for ages. Nils -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From nils.goroll at uplex.de Mon Oct 5 15:04:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 5 Oct 2020 15:04:06 +0000 (UTC) Subject: [master] 8b3ac69c0 panic: do not dump session attrs for magic mismatch Message-ID: <20201005150406.7124993A2D@lists.varnish-cache.org> commit 8b3ac69c0d4e4eef76d2895b4113787ae5dfd2db Author: Nils Goroll Date: Mon Oct 5 16:56:12 2020 +0200 panic: do not dump session attrs for magic mismatch diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 61bde5f43..253fa674a 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -559,12 +559,19 @@ pan_sess(struct vsb *vsb, const struct sess *sp) return; VSB_indent(vsb, 2); PAN_CheckMagic(vsb, sp, SESS_MAGIC); - xp = XPORT_ByNumber(sp->sattr[SA_TRANSPORT]); VSB_printf(vsb, "fd = %d, vxid = %u,\n", sp->fd, VXID(sp->vxid)); VSB_printf(vsb, "t_open = %f,\n", sp->t_open); VSB_printf(vsb, "t_idle = %f,\n", sp->t_idle); + + if (! VALID_OBJ(sp, SESS_MAGIC)) { + VSB_indent(vsb, -2); + VSB_cat(vsb, "},\n"); + return; + } + WS_Panic(sp->ws, vsb); + xp = XPORT_ByNumber(sp->sattr[SA_TRANSPORT]); VSB_printf(vsb, "transport = %s", xp == NULL ? "" : xp->name); if (xp != NULL && xp->sess_panic != NULL) { @@ -575,6 +582,7 @@ pan_sess(struct vsb *vsb, const struct sess *sp) VSB_cat(vsb, "}"); } VSB_cat(vsb, "\n"); + // duplicated below, remove ? ci = SES_Get_String_Attr(sp, SA_CLIENT_IP); cp = SES_Get_String_Attr(sp, SA_CLIENT_PORT); From nils.goroll at uplex.de Tue Oct 6 10:16:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 6 Oct 2020 10:16:06 +0000 (UTC) Subject: [master] fa4df4811 No need to set a parameter to its default Message-ID: <20201006101606.BFF675FA2@lists.varnish-cache.org> commit fa4df48111710be22d112506e4f70da1e30a3eed Author: Nils Goroll Date: Tue Oct 6 12:13:31 2020 +0200 No need to set a parameter to its default sigsegv_handler is on by default since 2015, ref 1955e4601d9e53758884c598b70eeffca59de173 noticed via argv dump in panic d0e986a6dd0cd0801531d5c7f9c05d11c24a676b diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 09e49d258..78705f897 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -424,7 +424,6 @@ varnish_launch(struct varnish *v) VSB_cat(vsb, " -l 2m"); VSB_cat(vsb, " -p auto_restart=off"); VSB_cat(vsb, " -p syslog_cli_traffic=off"); - VSB_cat(vsb, " -p sigsegv_handler=on"); VSB_cat(vsb, " -p thread_pool_min=10"); VSB_cat(vsb, " -p debug=+vtc_mode"); VSB_cat(vsb, " -p vsl_mask=+Debug"); From nils.goroll at uplex.de Tue Oct 6 14:25:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 6 Oct 2020 14:25:06 +0000 (UTC) Subject: [master] f782b2f93 Back to trunk Message-ID: <20201006142506.4363263A76@lists.varnish-cache.org> commit f782b2f9394fd41dc40713cd37dff2d05508716a Author: Nils Goroll Date: Tue Oct 6 16:23:57 2020 +0200 Back to trunk diff --git a/configure.ac b/configure.ac index 419e63b0a..b63840512 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ AC_PREREQ(2.59) AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS Copyright (c) 2006-2020 Varnish Software]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [6.5.1], [varnish-dev at varnish-cache.org]) +AC_INIT([Varnish], [trunk], [varnish-dev at varnish-cache.org]) AC_CONFIG_SRCDIR(include/miniobj.h) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) From phk at FreeBSD.org Wed Oct 7 07:05:07 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 7 Oct 2020 07:05:07 +0000 (UTC) Subject: [6.0] c728b22de http: Add missing HTTP response code Message-ID: <20201007070507.98651AFABC@lists.varnish-cache.org> commit c728b22de8d03f41072e5a8f92259cd26f5173a2 Author: BARON David Date: Wed Oct 7 12:59:24 2020 +0900 http: Add missing HTTP response code - Complete the missing HTTP response codes / messages from : https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml patch from master at d686166ad469cad4043ed24328f4377e0d5f71a7 diff --git a/bin/varnishtest/tests/p00010.vtc b/bin/varnishtest/tests/p00010.vtc new file mode 100644 index 000000000..1a5a8a25d --- /dev/null +++ b/bin/varnishtest/tests/p00010.vtc @@ -0,0 +1,101 @@ +varnishtest "status codes" + +server s1 {} -start + +varnish v1 -vcl+backend { + import std; + sub vcl_recv { + return (synth(std.integer(req.http.x-vtc-response-code, 999))); + } +} -start + +client c1 { + txreq -hdr "x-vtc-response-code: 102" + rxresp + expect resp.reason == "Processing" + + txreq -hdr "x-vtc-response-code: 103" + rxresp + expect resp.reason == "Early Hints" + + txreq -hdr "x-vtc-response-code: 207" + rxresp + expect resp.reason == "Multi-Status" + + txreq -hdr "x-vtc-response-code: 208" + rxresp + expect resp.reason == "Already Reported" + + txreq -hdr "x-vtc-response-code: 226" + rxresp + expect resp.reason == "IM Used" + + txreq -hdr "x-vtc-response-code: 308" + rxresp + expect resp.reason == "Permanent Redirect" + + txreq -hdr "x-vtc-response-code: 421" + rxresp + expect resp.reason == "Misdirected Request" + + txreq -hdr "x-vtc-response-code: 422" + rxresp + expect resp.reason == "Unprocessable Entity" + + txreq -hdr "x-vtc-response-code: 423" + rxresp + expect resp.reason == "Locked" + + txreq -hdr "x-vtc-response-code: 424" + rxresp + expect resp.reason == "Failed Dependency" + + txreq -hdr "x-vtc-response-code: 425" + rxresp + expect resp.reason == "Too Early" + + txreq -hdr "x-vtc-response-code: 426" + rxresp + expect resp.reason == "Upgrade Required" + + txreq -hdr "x-vtc-response-code: 428" + rxresp + expect resp.reason == "Precondition Required" + + txreq -hdr "x-vtc-response-code: 429" + rxresp + expect resp.reason == "Too Many Requests" + + txreq -hdr "x-vtc-response-code: 431" + rxresp + expect resp.reason == "Request Header Fields Too Large" + txreq -hdr "x-vtc-response-code: 451" + rxresp + expect resp.reason == "Unavailable For Legal Reasons" + + txreq -hdr "x-vtc-response-code: 506" + rxresp + expect resp.reason == "Variant Also Negotiates" + + txreq -hdr "x-vtc-response-code: 507" + rxresp + expect resp.reason == "Insufficient Storage" + + txreq -hdr "x-vtc-response-code: 508" + rxresp + expect resp.reason == "Loop Detected" + + txreq -hdr "x-vtc-response-code: 510" + rxresp + expect resp.reason == "Not Extended" + + txreq -hdr "x-vtc-response-code: 511" + rxresp + expect resp.reason == "Network Authentication Required" + + txreq -hdr "x-vtc-response-code: vtc" + rxresp + expect resp.status == 999 + expect resp.reason == "Unknown HTTP Status" +} -run + diff --git a/include/tbl/http_response.h b/include/tbl/http_response.h index 0e21d00d6..386c285d5 100644 --- a/include/tbl/http_response.h +++ b/include/tbl/http_response.h @@ -32,6 +32,8 @@ HTTP_RESP(100, "Continue") HTTP_RESP(101, "Switching Protocols") +HTTP_RESP(102, "Processing") +HTTP_RESP(103, "Early Hints") HTTP_RESP(200, "OK") HTTP_RESP(201, "Created") HTTP_RESP(202, "Accepted") @@ -39,6 +41,9 @@ HTTP_RESP(203, "Non-Authoritative Information") HTTP_RESP(204, "No Content") HTTP_RESP(205, "Reset Content") HTTP_RESP(206, "Partial Content") +HTTP_RESP(207, "Multi-Status") +HTTP_RESP(208, "Already Reported") +HTTP_RESP(226, "IM Used") HTTP_RESP(300, "Multiple Choices") HTTP_RESP(301, "Moved Permanently") HTTP_RESP(302, "Found") @@ -47,6 +52,7 @@ HTTP_RESP(304, "Not Modified") HTTP_RESP(305, "Use Proxy") HTTP_RESP(306, "(Unused)") HTTP_RESP(307, "Temporary Redirect") +HTTP_RESP(308, "Permanent Redirect") HTTP_RESP(400, "Bad Request") HTTP_RESP(401, "Unauthorized") HTTP_RESP(402, "Payment Required") @@ -65,12 +71,27 @@ HTTP_RESP(414, "Request-URI Too Long") HTTP_RESP(415, "Unsupported Media Type") HTTP_RESP(416, "Requested Range Not Satisfiable") HTTP_RESP(417, "Expectation Failed") +HTTP_RESP(421, "Misdirected Request") +HTTP_RESP(422, "Unprocessable Entity") +HTTP_RESP(423, "Locked") +HTTP_RESP(424, "Failed Dependency") +HTTP_RESP(425, "Too Early") +HTTP_RESP(426, "Upgrade Required") +HTTP_RESP(428, "Precondition Required") +HTTP_RESP(429, "Too Many Requests") +HTTP_RESP(431, "Request Header Fields Too Large") +HTTP_RESP(451, "Unavailable For Legal Reasons") HTTP_RESP(500, "Internal Server Error") HTTP_RESP(501, "Not Implemented") HTTP_RESP(502, "Bad Gateway") HTTP_RESP(503, "Service Unavailable") HTTP_RESP(504, "Gateway Timeout") HTTP_RESP(505, "HTTP Version Not Supported") +HTTP_RESP(506, "Variant Also Negotiates") +HTTP_RESP(507, "Insufficient Storage") +HTTP_RESP(508, "Loop Detected") +HTTP_RESP(510, "Not Extended") +HTTP_RESP(511, "Network Authentication Required") #undef HTTP_RESP /*lint -restore */ From nils.goroll at uplex.de Wed Oct 7 08:34:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 7 Oct 2020 08:34:06 +0000 (UTC) Subject: [master] ed13c9f24 rename `OBJ_ITER_FINAL` to `OBJ_ITER_END` for clarity Message-ID: <20201007083406.C7C50B1EDB@lists.varnish-cache.org> commit ed13c9f24e60627b356d27f23f6548e4a44515df Author: Nils Goroll Date: Wed Oct 7 10:27:45 2020 +0200 rename `OBJ_ITER_FINAL` to `OBJ_ITER_END` for clarity We use the `final` argument in object iteration to denote `OC_F_PRIVATE | OC_F_HFM | OC_F_HFP`. As we now have `VDP_END` to mark the final bit of data, so should the iterator flag be named `_END`. This also parallels `VDP_FLUSH` vs. `OBJ_ITER_FLUSH`. Supported by martin Ref #3298 diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 215ed8759..fc4d5c71d 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -710,7 +710,7 @@ const void *ObjGetAttr(struct worker *, struct objcore *, enum obj_attr, typedef int objiterate_f(void *priv, unsigned flush, const void *ptr, ssize_t len); #define OBJ_ITER_FLUSH 0x01 -#define OBJ_ITER_FINAL 0x02 +#define OBJ_ITER_END 0x02 int ObjIterate(struct worker *, struct objcore *, void *priv, objiterate_f *func, int final); diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c index 231a50d6b..ccba6b525 100644 --- a/bin/varnishd/storage/storage_simple.c +++ b/bin/varnishd/storage/storage_simple.c @@ -261,7 +261,7 @@ sml_iterator(struct worker *wrk, struct objcore *oc, VTAILQ_FOREACH_SAFE(st, &obj->list, list, checkpoint) { u = 0; if (VTAILQ_NEXT(st, list) == NULL) - u |= OBJ_ITER_FINAL; + u |= OBJ_ITER_END; if (final) u |= OBJ_ITER_FLUSH; if (ret == 0 && st->len > 0) From reza at naghibi.com Wed Oct 7 14:23:06 2020 From: reza at naghibi.com (Reza Naghibi) Date: Wed, 7 Oct 2020 14:23:06 +0000 (UTC) Subject: [6.0] b10f51e5c Update changelog to include HTTP response reasons Message-ID: <20201007142306.B0BC260A09@lists.varnish-cache.org> commit b10f51e5ce09004cbf3b859c4dcd90fa8bf61d62 Author: Reza Naghibi Date: Wed Oct 7 10:22:36 2020 -0400 Update changelog to include HTTP response reasons diff --git a/doc/changes.rst b/doc/changes.rst index 08ccb7bf0..228c4210e 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -30,7 +30,9 @@ release process. Varnish Cache 6.0.7 (YYYY-MM-DD) ================================ -* Add weighted backend to vmod_shard. +* Add support for more HTTP response code reasons (3428_). + +* Add weighted backends to vmod_shard. * Fix an issue where an undefined value is returned when converting a VCL_TIME to a string. (3308_) @@ -75,6 +77,7 @@ Varnish Cache 6.0.7 (YYYY-MM-DD) * Expose the master and worker PIDs via the CLI: `varnishadm pid`. (3171_) +.. _3428: https://github.com/varnishcache/varnish-cache/pull/3428 .. _3308: https://github.com/varnishcache/varnish-cache/pull/3308 .. _3385: https://github.com/varnishcache/varnish-cache/issues/3385 .. _3361: https://github.com/varnishcache/varnish-cache/issues/3361 From phk at FreeBSD.org Wed Oct 7 19:23:06 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 7 Oct 2020 19:23:06 +0000 (UTC) Subject: [master] 115742b07 Merge ObjTrimStore into ObjExtend, as aid to #3298 Message-ID: <20201007192306.7566A94AC4@lists.varnish-cache.org> commit 115742b07c8bad6d465f1c981ee264f934a4492b Author: Poul-Henning Kamp Date: Wed Oct 7 19:21:19 2020 +0000 Merge ObjTrimStore into ObjExtend, as aid to #3298 diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c index 221822059..0a0d1e2aa 100644 --- a/bin/varnishd/cache/cache_esi_fetch.c +++ b/bin/varnishd/cache/cache_esi_fetch.c @@ -100,7 +100,7 @@ vfp_vep_callback(struct vfp_ctx *vc, void *priv, ssize_t l, enum vgz_flag flg) VGZ_UpdateObj(vc, vef->vgz, VUA_UPDATE); if (dl > 0) { vef->tot += dl; - VFP_Extend(vc, dl); + VFP_Extend(vc, dl, VFP_OK); } } while (i != VGZ_ERROR && (!VGZ_IbufEmpty(vef->vgz) || VGZ_ObufFull(vef->vgz))); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 3f848db63..c8958dfb4 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -517,7 +517,7 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) */ static enum fetch_step -vbf_stp_fetchbody(struct worker *wrk, struct busyobj *bo) +vbf_stp_fetchbody(const struct worker *wrk, struct busyobj *bo) { ssize_t l; uint8_t *ptr; @@ -561,8 +561,8 @@ vbf_stp_fetchbody(struct worker *wrk, struct busyobj *bo) AZ(vfc->failed); vfps = VFP_Suck(vfc, ptr, &l); - if (l > 0 && vfps != VFP_ERROR) { - VFP_Extend(vfc, l); + if (l >= 0 && vfps != VFP_ERROR) { + VFP_Extend(vfc, l, vfps); if (est >= l) est -= l; else @@ -584,7 +584,6 @@ vbf_stp_fetchbody(struct worker *wrk, struct busyobj *bo) } } - ObjTrimStore(wrk, oc); return (F_STP_FETCHEND); } @@ -729,7 +728,7 @@ vbf_objiterator(void *priv, unsigned flush, const void *ptr, ssize_t len) if (len < l) l = len; memcpy(pd, ps, l); - VFP_Extend(bo->vfc, l); + VFP_Extend(bo->vfc, l, l == len ? VFP_END : VFP_OK); ps += l; len -= l; } @@ -929,7 +928,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) if (l > ll) l = ll; memcpy(ptr, VSB_data(synth_body) + o, l); - VFP_Extend(bo->vfc, l); + VFP_Extend(bo->vfc, l, l == ll ? VFP_END : VFP_OK); ll -= l; o += l; } diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c index d9fa10716..0b7c69913 100644 --- a/bin/varnishd/cache/cache_fetch_proc.c +++ b/bin/varnishd/cache/cache_fetch_proc.c @@ -95,11 +95,11 @@ VFP_GetStorage(struct vfp_ctx *vc, ssize_t *sz, uint8_t **ptr) } void -VFP_Extend(const struct vfp_ctx *vc, ssize_t sz) +VFP_Extend(const struct vfp_ctx *vc, ssize_t sz, enum vfp_status flg) { CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); - ObjExtend(vc->wrk, vc->oc, sz); + ObjExtend(vc->wrk, vc->oc, sz, flg == VFP_END); } /********************************************************************** diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index ef10c10b7..76a90f9cf 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -47,7 +47,6 @@ * 2 ObjGetSpace() allocates space * 2 ObjExtend() commits content * 2 ObjWaitExtend() waits for content - used to implement ObjIterate()) - * 2 ObjTrimStore() signals end of content addition * * 2 ObjSetAttr() * 2 ObjCopyAttr() @@ -212,23 +211,33 @@ ObjGetSpace(struct worker *wrk, struct objcore *oc, ssize_t *sz, uint8_t **ptr) * * This function extends the used part of the object a number of bytes * into the last space returned by ObjGetSpace() + * + * The final flag must be set on the last call, and it will release any + * surplus space allocated. */ void -ObjExtend(struct worker *wrk, struct objcore *oc, ssize_t l) +ObjExtend(struct worker *wrk, struct objcore *oc, ssize_t l, int final) { const struct obj_methods *om = obj_getmethods(oc); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC); - assert(l > 0); + (void)final; + assert(l >= 0); Lck_Lock(&oc->boc->mtx); AN(om->objextend); - om->objextend(wrk, oc, l); - oc->boc->len_so_far += l; + if (l > 0) { + om->objextend(wrk, oc, l); + oc->boc->len_so_far += l; + AZ(pthread_cond_broadcast(&oc->boc->cond)); + } Lck_Unlock(&oc->boc->mtx); - AZ(pthread_cond_broadcast(&oc->boc->cond)); + + assert(oc->boc == NULL || oc->boc->state < BOS_FINISHED); + if (final && om->objtrimstore != NULL) + om->objtrimstore(wrk, oc); } /*==================================================================== @@ -320,26 +329,6 @@ ObjGetLen(struct worker *wrk, struct objcore *oc) return (len); } -/*==================================================================== - * ObjTrimStore() - * - * Release any surplus space allocated, we promise not to call ObjExtend() - * any more. - */ - -void -ObjTrimStore(struct worker *wrk, struct objcore *oc) -{ - const struct obj_methods *om = obj_getmethods(oc); - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - - assert(oc->boc == NULL || oc->boc->state < BOS_FINISHED); - - if (om->objtrimstore != NULL) - om->objtrimstore(wrk, oc); -} - /*==================================================================== * ObjSlim() * diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c index 0d417ab6c..a3a771163 100644 --- a/bin/varnishd/cache/cache_req_body.c +++ b/bin/varnishd/cache/cache_req_body.c @@ -124,7 +124,7 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv) if (r) break; } else { - ObjExtend(req->wrk, req->body_oc, l); + ObjExtend(req->wrk, req->body_oc, l, vfps == VFP_END); } } @@ -142,7 +142,6 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv) return (r); } - ObjTrimStore(req->wrk, req->body_oc); AZ(ObjSetU64(req->wrk, req->body_oc, OA_LEN, req_bodybytes)); HSH_DerefBoc(req->wrk, req->body_oc); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 919a34d15..6cb1f7a67 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -337,7 +337,7 @@ cnt_synth(struct worker *wrk, struct req *req) if (sz > 0 && ObjGetSpace(wrk, req->objcore, &sz, &ptr) && sz >= szl) { memcpy(ptr, VSB_data(synth_body), szl); - ObjExtend(wrk, req->objcore, szl); + ObjExtend(wrk, req->objcore, szl, 1); } else if (sz > 0) { szl = -1; } diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index c365ce057..f95f3c773 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -248,7 +248,7 @@ void Bereq_Rollback(struct busyobj *); /* cache_fetch_proc.c */ void VFP_Init(void); enum vfp_status VFP_GetStorage(struct vfp_ctx *, ssize_t *sz, uint8_t **ptr); -void VFP_Extend(const struct vfp_ctx *, ssize_t sz); +void VFP_Extend(const struct vfp_ctx *, ssize_t sz, enum vfp_status); void VFP_Setup(struct vfp_ctx *vc, struct worker *wrk); int VFP_Open(struct vfp_ctx *bo); uint64_t VFP_Close(struct vfp_ctx *bo); @@ -295,13 +295,12 @@ void ObjInit(void); struct objcore * ObjNew(const struct worker *); void ObjDestroy(const struct worker *, struct objcore **); int ObjGetSpace(struct worker *, struct objcore *, ssize_t *sz, uint8_t **ptr); -void ObjExtend(struct worker *, struct objcore *, ssize_t l); +void ObjExtend(struct worker *, struct objcore *, ssize_t l, int final); uint64_t ObjWaitExtend(const struct worker *, const struct objcore *, uint64_t l); void ObjSetState(struct worker *, const struct objcore *, enum boc_state_e next); void ObjWaitState(const struct objcore *, enum boc_state_e want); -void ObjTrimStore(struct worker *, struct objcore *); void ObjTouch(struct worker *, struct objcore *, vtim_real now); void ObjFreeObj(struct worker *, struct objcore *); void ObjSlim(struct worker *, struct objcore *); From phk at FreeBSD.org Wed Oct 7 20:25:06 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 7 Oct 2020 20:25:06 +0000 (UTC) Subject: [master] 7417ceea5 Update the collection copyright year range Message-ID: <20201007202506.CB4FB96653@lists.varnish-cache.org> commit 7417ceea561264440647fdd88488f416ccac6ee4 Author: Poul-Henning Kamp Date: Wed Oct 7 19:56:42 2020 +0000 Update the collection copyright year range diff --git a/LICENSE b/LICENSE index db1cc57f8..b32a79c55 100644 --- a/LICENSE +++ b/LICENSE @@ -2,7 +2,7 @@ The compilation of software known as "Varnish Cache" is distributed under the following terms: Copyright (c) 2006 Verdens Gang AS -Copyright (c) 2006-2016 Varnish Software AS +Copyright (c) 2006-2020 Varnish Software AS All rights reserved. SPDX-License-Identifier: BSD-2-Clause From phk at FreeBSD.org Wed Oct 7 20:25:06 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 7 Oct 2020 20:25:06 +0000 (UTC) Subject: [master] e6be754f0 White space OCD Message-ID: <20201007202506.E38D996656@lists.varnish-cache.org> commit e6be754f021f2448ef3fe5356050e016af4a0558 Author: Poul-Henning Kamp Date: Wed Oct 7 20:23:51 2020 +0000 White space OCD diff --git a/bin/varnishtest/tests/c00016.vtc b/bin/varnishtest/tests/c00016.vtc index 8cbbfc79d..fa86a59bf 100644 --- a/bin/varnishtest/tests/c00016.vtc +++ b/bin/varnishtest/tests/c00016.vtc @@ -57,7 +57,7 @@ varnish v1 -vcl+backend { sub vcl_deliver { set resp.http.foo = resp.http.foox; } -} +} client c1 { txreq -hdr "Baax: 1" -hdr "Baax: 2" -hdr "Connection: baax" diff --git a/bin/varnishtest/tests/d00021.vtc b/bin/varnishtest/tests/d00021.vtc index ee85ace22..e691def28 100644 --- a/bin/varnishtest/tests/d00021.vtc +++ b/bin/varnishtest/tests/d00021.vtc @@ -88,9 +88,9 @@ varnish v1 -vcl+backend { sub vcl_backend_fetch { if (bereq.http.layered) { call backend_fetch_layered; - } else if (bereq.http.resolve) { - call backend_fetch_layered; - set bereq.backend = bereq.backend.resolve(); + } else if (bereq.http.resolve) { + call backend_fetch_layered; + set bereq.backend = bereq.backend.resolve(); } else { call backend_fetch_direct; } diff --git a/bin/varnishtest/tests/h00007.vtc b/bin/varnishtest/tests/h00007.vtc index 237c45cae..fa42fcb8f 100644 --- a/bin/varnishtest/tests/h00007.vtc +++ b/bin/varnishtest/tests/h00007.vtc @@ -45,7 +45,7 @@ varnish v1 -vcl+backend { set resp.http.notstdip = std.ip("" + client.ip, resolve = false) !~ localhost; } -} +} client c1 -connect ${h1_fe1_sock} { txreq -url "/" diff --git a/bin/varnishtest/tests/p00010.vtc b/bin/varnishtest/tests/p00010.vtc index 1a5a8a25d..67dc4447e 100644 --- a/bin/varnishtest/tests/p00010.vtc +++ b/bin/varnishtest/tests/p00010.vtc @@ -72,7 +72,7 @@ client c1 { txreq -hdr "x-vtc-response-code: 451" rxresp expect resp.reason == "Unavailable For Legal Reasons" - + txreq -hdr "x-vtc-response-code: 506" rxresp expect resp.reason == "Variant Also Negotiates" diff --git a/bin/varnishtest/tests/r03169.vtc b/bin/varnishtest/tests/r03169.vtc index 650b15c83..9127475ab 100644 --- a/bin/varnishtest/tests/r03169.vtc +++ b/bin/varnishtest/tests/r03169.vtc @@ -21,9 +21,9 @@ server s1 { varnish v1 -vcl+backend { sub vcl_backend_response { - set beresp.ttl = 1s; - set beresp.grace = 0s; - set beresp.keep = 1d; + set beresp.ttl = 1s; + set beresp.grace = 0s; + set beresp.keep = 1d; } } -start @@ -58,4 +58,4 @@ client c1 { expect resp.status == 200 expect resp.http.ETag == {"bar"} expect resp.http.Content-Encoding == "gzip" -} -run \ No newline at end of file +} -run diff --git a/bin/varnishtest/tests/r3354.vtc b/bin/varnishtest/tests/r03354.vtc similarity index 100% rename from bin/varnishtest/tests/r3354.vtc rename to bin/varnishtest/tests/r03354.vtc diff --git a/bin/varnishtest/tests/r03360.vtc b/bin/varnishtest/tests/r03360.vtc index 842988119..72668ff9b 100644 --- a/bin/varnishtest/tests/r03360.vtc +++ b/bin/varnishtest/tests/r03360.vtc @@ -7,9 +7,9 @@ shell {echo include '"_recurse1.vcl";' > ${tmpdir}/_recurse2.vcl} varnish v1 -arg "-p vcl_path=${tmpdir}" -errvcl "Recursive include of" { backend b { .host = "127.0.0.1"; } include "_recurse.vcl" ; -} +} varnish v2 -arg "-p vcl_path=${tmpdir}" -errvcl "Recursive include of" { backend b { .host = "127.0.0.1"; } include "_recurse1.vcl" ; -} +} diff --git a/bin/varnishtest/tests/v00064.vtc b/bin/varnishtest/tests/v00064.vtc index 080029b4d..19544d686 100644 --- a/bin/varnishtest/tests/v00064.vtc +++ b/bin/varnishtest/tests/v00064.vtc @@ -22,11 +22,11 @@ varnish v1 \ -syntax 4.0 \ -vcl+backend { sub vcl_backend_response { - if (bereq.url == "/transient") { - set beresp.storage = storage.Transient; - # Unset Date header to not change the object sizes - unset beresp.http.Date; - } + if (bereq.url == "/transient") { + set beresp.storage = storage.Transient; + # Unset Date header to not change the object sizes + unset beresp.http.Date; + } } } -start From phk at FreeBSD.org Wed Oct 7 22:55:07 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 7 Oct 2020 22:55:07 +0000 (UTC) Subject: [master] ef377655f Merge verrno into vas, it's too trivial. Message-ID: <20201007225507.25587A070A@lists.varnish-cache.org> commit ef377655f0fa4c476fefcc40e5ec6aec0a104c03 Author: Poul-Henning Kamp Date: Wed Oct 7 22:54:15 2020 +0000 Merge verrno into vas, it's too trivial. diff --git a/include/Makefile.am b/include/Makefile.am index 78ff414f2..869ac7684 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -65,7 +65,6 @@ nobase_pkginclude_HEADERS += \ vbm.h \ vcl.h \ vcs.h \ - verrno.h \ vmod_abi.h \ vqueue.h \ vre.h \ diff --git a/include/vas.h b/include/vas.h index a73b6d927..90fac3482 100644 --- a/include/vas.h +++ b/include/vas.h @@ -40,7 +40,10 @@ #ifndef VAS_H_INCLUDED #define VAS_H_INCLUDED -#include "verrno.h" + +#include + +const char * vstrerror(int e); enum vas_e { VAS_WRONG, diff --git a/include/verrno.h b/include/verrno.h deleted file mode 100644 index f457aa2c7..000000000 --- a/include/verrno.h +++ /dev/null @@ -1,11 +0,0 @@ -/*- - * Written by Nils Goroll based upon a draft by Poul-Henning Kamp - * - * This file is in the public domain. - * - * trivial strerror() wrapper never returning NULL - */ - -#include - -const char * vstrerror(int e); diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am index 6fbdfcba5..9c0b8a643 100644 --- a/lib/libvarnish/Makefile.am +++ b/lib/libvarnish/Makefile.am @@ -22,7 +22,6 @@ libvarnish_la_SOURCES = \ vcli_serve.c \ vct.c \ venc.c \ - verrno.c \ version.c \ vev.c \ vfil.c \ diff --git a/lib/libvarnish/vas.c b/lib/libvarnish/vas.c index 13161bcea..392303fdd 100644 --- a/lib/libvarnish/vas.c +++ b/lib/libvarnish/vas.c @@ -41,6 +41,20 @@ #include "vas.h" +const char * +vstrerror(int e) +{ + const char *p; + int oerrno = errno; + + p = strerror(e); + if (p != NULL) + return (p); + + errno = oerrno; + return ("strerror(3) returned NULL"); +} + vas_f *VAS_Fail_Func v_noreturn_; void v_noreturn_ diff --git a/lib/libvarnish/verrno.c b/lib/libvarnish/verrno.c deleted file mode 100644 index 924010215..000000000 --- a/lib/libvarnish/verrno.c +++ /dev/null @@ -1,27 +0,0 @@ -/*- - * Written by Nils Goroll based upon a draft by Poul-Henning Kamp - * - * This file is in the public domain. - * - * trivial strerror() wrapper never returning NULL - */ - -#include "config.h" - -#include - -#include "verrno.h" - -const char * -vstrerror(int e) -{ - const char *p; - int oerrno = errno; - - p = strerror(e); - if (p != NULL) - return (p); - - errno = oerrno; - return ("strerror(3) returned NULL"); -} From nils.goroll at uplex.de Thu Oct 8 09:34:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 8 Oct 2020 09:34:06 +0000 (UTC) Subject: [master] bd96f26bf fix off-by-one in vdp rot13 Message-ID: <20201008093406.C8641B0EE2@lists.varnish-cache.org> commit bd96f26bfb587c997c41eeb1a784b01eb2698a84 Author: Nils Goroll Date: Wed Oct 7 13:47:49 2020 +0200 fix off-by-one in vdp rot13 It added a random byte. diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index fc8232b91..13f602b16 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -144,7 +144,7 @@ xyzzy_rot13_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, } } if (i >= 0) - retval = VDP_bytes(vdx, act, q, i + 1L); + retval = VDP_bytes(vdx, act, q, i); return (retval); } From nils.goroll at uplex.de Thu Oct 8 09:34:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 8 Oct 2020 09:34:06 +0000 (UTC) Subject: [master] 0062e8cef fix rot13 vdp VDP_END handling and make it produce more vdp calls Message-ID: <20201008093406.DB678B0EE6@lists.varnish-cache.org> commit 0062e8cef7768cd89bf5dfe4b13183ab6a9be11c Author: Nils Goroll Date: Wed Oct 7 13:58:15 2020 +0200 fix rot13 vdp VDP_END handling and make it produce more vdp calls de6288cbe4ac102ebd3ca66cddb64bc12069ee49 was incorrect in that it could generate multiple VDP_ENDs, which did not show up during testing because of the buffer being large enough to fit any testcase. Make sure such issues pop up with rot13 testing by using a deliberately minuscule buffer. (and, yeah, we could take it from workspace, but the point is to demonstrate and test VDPs...) diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 13f602b16..e301c6291 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -97,7 +97,8 @@ static const struct vfp xyzzy_rot13 = { /**********************************************************************/ -#define ROT13_BUFSZ (1 << 13) +// deliberately fragmenting the stream to make testing more interesting +#define ROT13_BUFSZ 8 static int v_matchproto_(vdp_init_f) xyzzy_rot13_init(struct req *req, void **priv) @@ -136,8 +137,8 @@ xyzzy_rot13_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, q[i] = (((pp[j] - 'a') + 13) % 26) + 'a'; else q[i] = pp[j]; - if (i == ROT13_BUFSZ - 1) { - retval = VDP_bytes(vdx, act, q, ROT13_BUFSZ); + if (i == ROT13_BUFSZ - 1 && j < len - 1) { + retval = VDP_bytes(vdx, VDP_FLUSH, q, ROT13_BUFSZ); if (retval != 0) return (retval); i = -1; From nils.goroll at uplex.de Thu Oct 8 09:34:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 8 Oct 2020 09:34:06 +0000 (UTC) Subject: [master] fb36b7b1e Test processing ESI includes with custom VDPs Message-ID: <20201008093407.05C79B0EEB@lists.varnish-cache.org> commit fb36b7b1e29edb347a4296f1f143ba8197cdfd97 Author: Nils Goroll Date: Thu Oct 8 11:24:13 2020 +0200 Test processing ESI includes with custom VDPs ... to the extent possible today. See comment in the VTC for more background. Ref Martin's comments in #3298 diff --git a/bin/varnishtest/tests/e00034.vtc b/bin/varnishtest/tests/e00034.vtc new file mode 100644 index 000000000..7050a2048 --- /dev/null +++ b/bin/varnishtest/tests/e00034.vtc @@ -0,0 +1,135 @@ +varnishtest "ESI requests, gzip and custom VDPs" + +# XXX THIS TEST CASE IS NOT YET COMPLETE. +# + +# To select the embedding vdps (gzgz / pretendgz), ESI currently only +# looks at the cache object's ESI status. So for VDP-processing an +# include of a gzip'ed ESI object, we need to restore the encoding +# after filtering. Thus, we would currently need a gzip VDP to be able +# to use `set resp.filters += " gunzip rot13 gzip"` +# +# Or we would need way to communicate down the VDP chain the current +# Content-Encoding, such that ESI could determine which embedding vdps +# to choose, which would make `set resp.filters += " gunzip rot13"` +# possible. +# +# Pursuing that idea further, filters could specify which CEs they +# have for input and output, allowing auto-config of the filter chain +# with respect to encodings. + +server s1 { + rxreq + expect req.url == /esi_plain + txresp -body { + + Before include + + Between includes + + } + + rxreq + expect req.url == "/body_plain" + txresp -body { +
Included file
+ } + + rxreq + expect req.url == "/body_gzip" + txresp -gzipbody { +
Included file
+ } + + rxreq + expect req.url == /esi_gzip + txresp -gzipbody { + + Before include + + Between includes + + } + + rxreq + expect req.url == /recurse_plain + txresp -body { + + TOP Before include + + TOP Between includes + + } + + rxreq + expect req.url == /recurse_gzip + txresp -gzipbody { + + TOP Before include + + TOP Between includes + + } +} -start + +varnish v1 -syntax 4.1 -vcl+backend { + import debug; + + sub vcl_deliver { + if (req.http.rot-level == "" + req.esi_level) { + if (resp.http.Content-Encoding == "gzip") { + # XXX no gzip vdp yet + # set resp.filters += " gunzip rot13 gzip"; + } else { + set resp.filters += " rot13"; + } + } + } + sub vcl_backend_response { + set beresp.do_esi = true; + } +} -start + +# basic case +client c1 { + txreq -url "/esi_plain" + rxresp + expect resp.bodylen == 111 + + txreq -url "/esi_gzip" + rxresp + expect resp.bodylen == 111 + + txreq -url "/recurse_plain" + rxresp + expect resp.bodylen == 283 + + txreq -url "/recurse_gzip" + rxresp + expect resp.bodylen == 283 +} -run + +# VDP for non-ESI includes +client c2 { + txreq -url "/esi_plain" -hdr "rot-level: 1" + rxresp + expect resp.bodylen == 111 + + txreq -url "/esi_gzip" -hdr "rot-level: 1" + rxresp + expect resp.bodylen == 111 +} -start + +# VDP for ESI includes +client c3 { + txreq -url "/recurse_plain" -hdr "rot-level: 1" + rxresp + expect resp.bodylen == 283 + + txreq -url "/recurse_gzip" -hdr "rot-level: 1" + rxresp + expect resp.bodylen == 283 +} -start + +client c2 -wait +client c3 -wait From nils.goroll at uplex.de Fri Oct 9 08:38:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Oct 2020 08:38:06 +0000 (UTC) Subject: [master] 804014ada Add a debug VDP to be pendantic about VDP_END Message-ID: <20201009083806.74975ABC8C@lists.varnish-cache.org> commit 804014ada87886d08a22ae3ba101887e2baaed51 Author: Nils Goroll Date: Thu Oct 8 16:18:46 2020 +0200 Add a debug VDP to be pendantic about VDP_END the sole purpose is to ensure that we send a VDP_END. Later, we might consider to arm the assertion in VDP_Close(). At this point, a couple of VTCs fail. We would need to at least improve error handling. diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 04b57936e..79234e9f3 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -157,6 +157,11 @@ VDP_Close(struct req *req) VTAILQ_REMOVE(&vdc->vdp, vdpe, list); } vdc->nxt = VTAILQ_FIRST(&vdc->vdp); +#ifdef VDP_PEDANTIC_ARMED + // enable when we are confident to get VDP_END right + if (vdc->nxt == NULL && vdc->retval >= 0) + assert(vdpe->end == VDP_END); +#endif } return (rv); } diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index e301c6291..40ecc13ca 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -166,6 +166,41 @@ static const struct vdp xyzzy_vdp_rot13 = { .fini = xyzzy_rot13_fini, }; +/********************************************************************** + * assert that we see a VDP_END + * + * note: + * we could lookup our own vdpe in _fini and check for vdpe->end == VDP_END + * yet that would cross the API + */ + +void * end_marker = &end_marker; + +static int v_matchproto_(vdp_bytes_f) +xyzzy_pedantic_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, + const void *ptr, ssize_t len) +{ + AZ(*priv); + if (act == VDP_END) + *priv = end_marker; + return (VDP_bytes(vdx, act, ptr, len)); +} + +static int v_matchproto_(vdp_fini_f) +xyzzy_pedantic_fini(struct req *req, void **priv) +{ + (void) req; + assert (*priv == end_marker); + *priv = NULL; + return (0); +} + +static const struct vdp xyzzy_vdp_pedantic = { + .name = "debug.pedantic", + .bytes = xyzzy_pedantic_bytes, + .fini = xyzzy_pedantic_fini, +}; + /**********************************************************************/ VCL_STRING v_matchproto_(td_debug_author) @@ -379,13 +414,10 @@ event_load(VRT_CTX, struct vmod_priv *priv) priv->priv = priv_vcl; priv->free = priv_vcl_free; - /* - * NB: This is a proof of concept, until we decide what the real - * API should look like, do NOT do this anywhere else. - */ VRT_AddVFP(ctx, &xyzzy_rot13); VRT_AddVDP(ctx, &xyzzy_vdp_rot13); + VRT_AddVDP(ctx, &xyzzy_vdp_pedantic); return (0); } @@ -544,6 +576,7 @@ event_discard(VRT_CTX, void *priv) VRT_RemoveVFP(ctx, &xyzzy_rot13); VRT_RemoveVDP(ctx, &xyzzy_vdp_rot13); + VRT_RemoveVDP(ctx, &xyzzy_vdp_pedantic); if (--loads) return (0); From nils.goroll at uplex.de Fri Oct 9 08:38:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Oct 2020 08:38:06 +0000 (UTC) Subject: [master] 3470a286a Properly generate VDP_END for gunzip Message-ID: <20201009083806.881ADABC8F@lists.varnish-cache.org> commit 3470a286a5af9406b538b095eb6c46872cb4c759 Author: Nils Goroll Date: Thu Oct 8 16:22:00 2020 +0200 Properly generate VDP_END for gunzip diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index b5c04db83..47dad1e52 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -386,7 +386,8 @@ vdp_gunzip_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, if (vr < VGZ_OK) return (-1); if (vg->m_len == vg->m_sz || vr != VGZ_OK) { - if (VDP_bytes(vdx, VDP_FLUSH, vg->m_buf, vg->m_len)) + if (VDP_bytes(vdx, vr == VGZ_END ? VDP_END : VDP_FLUSH, + vg->m_buf, vg->m_len)) return (vdx->retval); vg->m_len = 0; VGZ_Obuf(vg, vg->m_buf, vg->m_sz); diff --git a/bin/varnishtest/tests/g00002.vtc b/bin/varnishtest/tests/g00002.vtc index 6afddd8d9..8b83b815b 100644 --- a/bin/varnishtest/tests/g00002.vtc +++ b/bin/varnishtest/tests/g00002.vtc @@ -18,9 +18,15 @@ varnish v1 \ -cliok "param.set gzip_memlevel 1" \ -vcl+backend { + import debug; + sub vcl_backend_response { set beresp.do_esi = true; } + + sub vcl_deliver { + set resp.filters += " debug.pedantic"; + } } -start varnish v1 -cliok "param.set fetch_chunksize 4k" diff --git a/bin/varnishtest/tests/m00048.vtc b/bin/varnishtest/tests/m00048.vtc index 3b2c3d78c..f67f72a23 100644 --- a/bin/varnishtest/tests/m00048.vtc +++ b/bin/varnishtest/tests/m00048.vtc @@ -66,7 +66,7 @@ varnish v1 -vcl+backend { sub vcl_deliver { if (req.http.Rot13) { - set resp.filters = "rot13"; + set resp.filters = "rot13 debug.pedantic"; } } } From nils.goroll at uplex.de Fri Oct 9 08:38:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Oct 2020 08:38:06 +0000 (UTC) Subject: [master] 13b458a80 fix VDP_END in ved_vdp_esi_bytes case 2 Message-ID: <20201009083806.A21CAABC92@lists.varnish-cache.org> commit 13b458a808638451212ccb981715f60fc805b21d Author: Nils Goroll Date: Thu Oct 8 16:28:44 2020 +0200 fix VDP_END in ved_vdp_esi_bytes case 2 This only concerns VDP_END at esi_level > 0, which I considered not relevant until Martin pointed out that it is. diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index ff9cf8d29..405aec8e2 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -397,6 +397,8 @@ ved_vdp_esi_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, } break; case 2: + ptr = NULL; + len = 0; if (ecx->isgzip && ecx->pecx == NULL) { /* * We are bytealigned here, so simply emit @@ -414,15 +416,14 @@ ved_vdp_esi_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, /* MOD(2^32) length */ vle32enc(tailbuf + 9, ecx->l_crc); - retval = VDP_bytes(vdx, VDP_END, tailbuf, 13); + ptr = tailbuf; + len = 13; } else if (ecx->pecx != NULL) { ecx->pecx->crc = crc32_combine(ecx->pecx->crc, ecx->crc, ecx->l_crc); ecx->pecx->l_crc += ecx->l_crc; - retval = VDP_bytes(vdx, VDP_FLUSH, NULL, 0); - } else { - retval = VDP_bytes(vdx, VDP_END, NULL, 0); } + retval = VDP_bytes(vdx, VDP_END, ptr, len); ecx->state = 99; return (retval); case 3: diff --git a/bin/varnishtest/tests/e00034.vtc b/bin/varnishtest/tests/e00034.vtc index 7050a2048..2749b3df3 100644 --- a/bin/varnishtest/tests/e00034.vtc +++ b/bin/varnishtest/tests/e00034.vtc @@ -84,6 +84,7 @@ varnish v1 -syntax 4.1 -vcl+backend { set resp.filters += " rot13"; } } + set resp.filters += " debug.pedantic"; } sub vcl_backend_response { set beresp.do_esi = true; From nils.goroll at uplex.de Fri Oct 9 09:26:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Oct 2020 09:26:06 +0000 (UTC) Subject: [master] 4bf0b9df5 directors: Optional weight in hash::add_backend() Message-ID: <20201009092606.95E85AD28C@lists.varnish-cache.org> commit 4bf0b9df5a555368c6db4c7cb3d18b76a3c30629 Author: Dridi Boukelmoune Date: Thu Oct 8 18:28:27 2020 +0200 directors: Optional weight in hash::add_backend() It makes it easier to substitute the hash director with say, the round robin director, when weight is not a concern. diff --git a/bin/varnishtest/tests/d00003.vtc b/bin/varnishtest/tests/d00003.vtc index 30c73cfb5..238d5e52d 100644 --- a/bin/varnishtest/tests/d00003.vtc +++ b/bin/varnishtest/tests/d00003.vtc @@ -26,7 +26,7 @@ varnish v1 -vcl+backend { sub vcl_init { new h1 = directors.hash(); - h1.add_backend(s1, 1); + h1.add_backend(s1); h1.add_backend(s2, 1); } diff --git a/lib/libvmod_directors/vmod_directors.vcc b/lib/libvmod_directors/vmod_directors.vcc index 5238250f1..f0a32af50 100644 --- a/lib/libvmod_directors/vmod_directors.vcc +++ b/lib/libvmod_directors/vmod_directors.vcc @@ -201,23 +201,24 @@ Example:: new vdir = directors.hash(); -$Method VOID .add_backend(BACKEND, REAL) +$Method VOID .add_backend(BACKEND, REAL weight = 1.0) Add a backend to the director with a certain weight. -Weight is used as in the random director. Recommended value is 1.0 -unless you have special needs. +Weight is used as in the random director. Recommended and default value +is 1.0 unless you have special needs. Example:: - vdir.add_backend(backend1, 1.0); + vdir.add_backend(normal_backend); + vdir.add_backend(larger_backend, 1.5); $Method VOID .remove_backend(BACKEND) Remove a backend from the director. Example:: - vdir.remove_backend(backend1); + vdir.remove_backend(larger_backend); $Method BACKEND .backend(STRANDS) From nils.goroll at uplex.de Fri Oct 9 09:36:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Oct 2020 09:36:07 +0000 (UTC) Subject: [master] a29507a10 http: Record which peer wants to close the session Message-ID: <20201009093607.14DA9AD7CF@lists.varnish-cache.org> commit a29507a1045dd7fa6571eeef2bbcaed96cf62ad6 Author: Dridi Boukelmoune Date: Thu Oct 8 16:34:00 2020 +0200 http: Record which peer wants to close the session Spotted by Martin. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index fc4d5c71d..4497d20dd 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -650,7 +650,7 @@ int HTTP_IterHdrPack(struct worker *, struct objcore *, const char **); #define HTTP_FOREACH_PACK(wrk, oc, ptr) \ for ((ptr) = NULL; HTTP_IterHdrPack(wrk, oc, &(ptr));) const char *HTTP_GetHdrPack(struct worker *, struct objcore *, const char *hdr); -enum sess_close http_DoConnection(struct http *hp); +enum sess_close http_DoConnection(struct http *hp, enum sess_close sc_close); int http_IsFiltered(const struct http *hp, unsigned u, unsigned how); #define HTTPH_R_PASS (1 << 0) /* Request (c->b) in pass mode */ diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 6986860ce..09fce7def 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -730,12 +730,15 @@ http_GetContentLength(const struct http *hp) */ enum sess_close -http_DoConnection(struct http *hp) +http_DoConnection(struct http *hp, enum sess_close sc_close) { const char *h, *b, *e; enum sess_close retval; unsigned u, v; + CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); + assert(sc_close == SC_REQ_CLOSE || sc_close == SC_RESP_CLOSE); + if (hp->protover == 10) retval = SC_REQ_HTTP10; else @@ -748,7 +751,7 @@ http_DoConnection(struct http *hp) while (http_split(&h, NULL, ",", &b, &e)) { u = pdiff(b, e); if (u == 5 && !strncasecmp(b, "close", u)) - retval = SC_REQ_CLOSE; + retval = sc_close; if (u == 10 && !strncasecmp(b, "keep-alive", u)) retval = SC_NULL; diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 6cb1f7a67..47ed06b31 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -83,7 +83,7 @@ cnt_transport(struct worker *wrk, struct req *req) AZ(req->err_code); - req->doclose = http_DoConnection(req->http); + req->doclose = http_DoConnection(req->http, SC_REQ_CLOSE); if (req->doclose == SC_RX_BAD) { (void)req->transport->minimal_response(req, 400); return (REQ_FSM_DONE); diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c index 3892b1544..99a4c58d9 100644 --- a/bin/varnishd/http1/cache_http1_fetch.c +++ b/bin/varnishd/http1/cache_http1_fetch.c @@ -234,7 +234,7 @@ V1F_FetchRespHdr(struct busyobj *bo) return (-1); } - htc->doclose = http_DoConnection(hp); + htc->doclose = http_DoConnection(hp, SC_RESP_CLOSE); /* * Figure out how the fetch is supposed to happen, before the diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index c93145322..54be4148b 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -298,7 +298,7 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody) VSLb(req->vsl, SLT_RespProtocol, "HTTP/2.0"); - (void)http_DoConnection(req->resp); + (void)http_DoConnection(req->resp, SC_RESP_CLOSE); ss = WS_Snapshot(req->ws); From nils.goroll at uplex.de Fri Oct 9 11:43:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Oct 2020 11:43:06 +0000 (UTC) Subject: [master] 08d3a70d2 add a trivial VRT function to check if a string is a valid header value Message-ID: <20201009114306.C1030B188E@lists.varnish-cache.org> commit 08d3a70d258108d0dffb8ae208e6e29891565534 Author: Nils Goroll Date: Wed Sep 9 15:34:42 2020 +0200 add a trivial VRT function to check if a string is a valid header value Ref: https://httpwg.org/specs/rfc7230.html#header.fields field-value = *( field-content / obs-fold ) field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ] field-vchar = VCHAR / obs-text - we do not accept the obsolete line folding (obs-fold) - VCHAR is 0x20 to 0x7e - obs-text is 0x80 to 0xff So we end up with !VCT_CTL || HTAB (LF) This change would be the basis for new use cases, like - a type method .valid_header - a feature flag +validate_headers which would call VRT_ValidHdr for all SetHdr calls diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index aef672e27..f88daaac1 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -590,6 +590,35 @@ VRT_UpperLowerStrands(VRT_CTX, VCL_STRANDS s, int up) return (r); } +// RFC7232, 3.2 without obsolete line folding: +// ASCII VCHAR + TAB + obs-text (0x80-ff) +static inline VCL_BOOL +validhdr(const char *p) +{ + AN(p); + for(;*p != '\0'; p++) + if (vct_isctl(*p) && !vct_issp(*p)) + return (0); + return (1); +} + +/*--------------------------------------------------------------------*/ +VCL_BOOL +VRT_ValidHdr(VRT_CTX, VCL_STRANDS s) +{ + int i; + + (void) ctx; + + for (i = 0; i < s->n; i++) { + if (s->p[i] == NULL || s->p[i][0] == '\0') + continue; + if (! validhdr(s->p[i])) + return (0); + } + + return (1); +} /*--------------------------------------------------------------------*/ VCL_VOID diff --git a/include/vrt.h b/include/vrt.h index 5e5b032f9..7e31480a4 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -496,6 +496,7 @@ VCL_INT VRT_purge(VRT_CTX, VCL_DURATION, VCL_DURATION, VCL_DURATION); VCL_VOID VRT_synth(VRT_CTX, VCL_INT, VCL_STRING); VCL_VOID VRT_hit_for_pass(VRT_CTX, VCL_DURATION); +VCL_BOOL VRT_ValidHdr(VRT_CTX, VCL_STRANDS); VCL_VOID VRT_SetHdr(VRT_CTX, VCL_HEADER, const char *, ...); VCL_VOID VRT_handling(VRT_CTX, unsigned hand); VCL_VOID VRT_fail(VRT_CTX, const char *fmt, ...) v_printflike_(2,3); From nils.goroll at uplex.de Fri Oct 9 11:43:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Oct 2020 11:43:06 +0000 (UTC) Subject: [master] 018c0fda4 SQUASHME: switch to vct_ishdrval() Message-ID: <20201009114306.D6A34B1891@lists.varnish-cache.org> commit 018c0fda4c62081f20f7309d02a37907d0f42ad5 Author: Nils Goroll Date: Thu Sep 17 12:31:44 2020 +0200 SQUASHME: switch to vct_ishdrval() diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index f88daaac1..68597ad84 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -597,7 +597,7 @@ validhdr(const char *p) { AN(p); for(;*p != '\0'; p++) - if (vct_isctl(*p) && !vct_issp(*p)) + if (! vct_ishdrval(*p)) return (0); return (1); } diff --git a/include/vct.h b/include/vct.h index 7c5825849..92e4123b9 100644 --- a/include/vct.h +++ b/include/vct.h @@ -84,6 +84,7 @@ vct_is(int x, uint16_t y) #define vct_isxmlnamestart(x) vct_is(x, VCT_XMLNAMESTART) #define vct_isxmlname(x) vct_is(x, VCT_XMLNAMESTART | VCT_XMLNAME) #define vct_istchar(x) vct_is(x, VCT_ALPHA | VCT_DIGIT | VCT_TCHAR) +#define vct_ishdrval(x) (((x) >= 0x20 && (x) != 0x7f) || (x) == 0x09) static inline int vct_iscrlf(const char* p, const char* end) From nils.goroll at uplex.de Fri Oct 9 12:40:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Oct 2020 12:40:06 +0000 (UTC) Subject: [master] d23db4026 Use the right version of the vct_ishdrval() macro Message-ID: <20201009124006.D7086BE025@lists.varnish-cache.org> commit d23db4026600c964915b873ed41520e993fc8327 Author: Nils Goroll Date: Fri Oct 9 14:36:42 2020 +0200 Use the right version of the vct_ishdrval() macro I accidentally committed an earlier, wrong version. o/ Dridi Ref #3407 https://github.com/varnishcache/varnish-cache/pull/3407#issuecomment-696146625 diff --git a/include/vct.h b/include/vct.h index 92e4123b9..7ccbbf233 100644 --- a/include/vct.h +++ b/include/vct.h @@ -84,7 +84,8 @@ vct_is(int x, uint16_t y) #define vct_isxmlnamestart(x) vct_is(x, VCT_XMLNAMESTART) #define vct_isxmlname(x) vct_is(x, VCT_XMLNAMESTART | VCT_XMLNAME) #define vct_istchar(x) vct_is(x, VCT_ALPHA | VCT_DIGIT | VCT_TCHAR) -#define vct_ishdrval(x) (((x) >= 0x20 && (x) != 0x7f) || (x) == 0x09) +#define vct_ishdrval(x) \ + (((uint8_t)(x) >= 0x20 && (uint8_t)(x) != 0x7f) ||(uint8_t)(x) == 0x09) static inline int vct_iscrlf(const char* p, const char* end) From nils.goroll at uplex.de Fri Oct 9 13:30:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Oct 2020 13:30:07 +0000 (UTC) Subject: [master] e082a30c8 validate_headers feature Message-ID: <20201009133007.C4704BF3DF@lists.varnish-cache.org> commit e082a30c83657004727b94df2444b366d550a888 Author: Nils Goroll Date: Fri Oct 9 13:45:41 2020 +0200 validate_headers feature We now validate all header set operations to conform with the allowed characters by RFC7230: * HTAB 0x09 * VCHAR 0x20 to 0x7e * obs-text 0x80 to 0xff Ref https://httpwg.org/specs/rfc7230.html#header.fields See #3407 diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 68597ad84..25fb07fd8 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -633,19 +633,23 @@ VRT_SetHdr(VRT_CTX , VCL_HEADER hs, const char *p, ...) AN(hs->what); hp = VRT_selecthttp(ctx, hs->where); CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); - va_start(ap, p); if (p == vrt_magic_string_unset) { http_Unset(hp, hs->what); } else { + va_start(ap, p); b = VRT_String(hp->ws, hs->what + 1, p, ap); + va_end(ap); if (b == NULL) { VSLb(ctx->vsl, SLT_LostHeader, "%s", hs->what + 1); - } else { - http_Unset(hp, hs->what); - http_SetHeader(hp, b); + return; + } + if (FEATURE(FEATURE_VALIDATE_HEADERS) && ! validhdr(b)) { + VRT_fail(ctx, "Bad header %s", b); + return; } + http_Unset(hp, hs->what); + http_SetHeader(hp, b); } - va_end(ap); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/mgt/mgt_param_bits.c b/bin/varnishd/mgt/mgt_param_bits.c index cc0812624..6a113e72e 100644 --- a/bin/varnishd/mgt/mgt_param_bits.c +++ b/bin/varnishd/mgt/mgt_param_bits.c @@ -222,7 +222,12 @@ tweak_feature(struct vsb *vsb, const struct parspec *par, const char *arg) (void)par; if (arg != NULL && arg != JSON_FMT) { - if (!strcmp(arg, "none")) { + if (!strcmp(arg, "default")) { + memset(mgt_param.feature_bits, + 0, sizeof mgt_param.feature_bits); + (void)bit(mgt_param.feature_bits, + FEATURE_VALIDATE_HEADERS, BSET); + } else if (!strcmp(arg, "none")) { memset(mgt_param.feature_bits, 0, sizeof mgt_param.feature_bits); } else { @@ -271,9 +276,10 @@ struct parspec VSL_parspec[] = { #undef DEBUG_BIT }, { "feature", tweak_feature, NULL, - NULL, NULL, "none", + NULL, NULL, "default", NULL, "Enable/Disable various minor features.\n" + "\tdefault\tSet default value\n" "\tnone\tDisable all features.\n\n" "Use +/- prefix to enable/disable individual feature:" #define FEATURE_BIT(U, l, d) "\n\t" #l "\t" d diff --git a/bin/varnishtest/tests/b00040.vtc b/bin/varnishtest/tests/b00040.vtc index cc7479d05..5e396b386 100644 --- a/bin/varnishtest/tests/b00040.vtc +++ b/bin/varnishtest/tests/b00040.vtc @@ -1,12 +1,22 @@ -varnishtest "test certain mailformed requests" +varnishtest "test certain malformed requests and validate_headers" server s1 { rxreq expect req.url == /4 txresp + rxreq + expect req.url == /9 + txresp } -start -varnish v1 -vcl+backend { } -start +varnish v1 -vcl+backend { + sub vcl_recv { + if (req.url == "/9") { + set req.http.foo = {" + "}; + } + } +} -start logexpect l1 -v v1 -g raw { expect * 1001 BogoHeader {1st header has white space:.*} @@ -16,6 +26,7 @@ logexpect l1 -v v1 -g raw { expect * 1012 BogoHeader {Header has ctrl char 0x0d} expect * 1014 BogoHeader {Header has ctrl char 0x0d} expect * 1016 BogoHeader {Missing header name:.*} + expect * 1018 VCL_Error {Bad header foo:} } -start client c1 { @@ -80,5 +91,18 @@ client c1 { expect resp.status == 400 } -run +client c1 { + txreq -url /9 + rxresp + expect resp.status == 503 +} -run + logexpect l1 -wait +varnish v1 -cliok "param.set feature -validate_headers" + +client c1 { + txreq -url /9 + rxresp + expect resp.status == 200 +} -run diff --git a/doc/changes.rst b/doc/changes.rst index 5a81f289e..b2c192e43 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -33,6 +33,10 @@ Varnish Cache Next (2021-03-15) * counters MAIN.s_req_bodybytes and VBE.*.tools.beresp_bodybytes are now always the number of bodybytes moved on the wire. +* Unless the new ``validate_headers`` feature is disabled, all newly + set headers are now validated to contain only characters allowed by + RFC7230. A (runtime) VCL failure is triggered if not. + ================================ Varnish Cache 6.5.1 (2020-09-25) ================================ diff --git a/include/tbl/feature_bits.h b/include/tbl/feature_bits.h index 2f314cfb2..207e18e11 100644 --- a/include/tbl/feature_bits.h +++ b/include/tbl/feature_bits.h @@ -74,6 +74,10 @@ FEATURE_BIT(WAIT_SILO, wait_silo, "Wait for persistent silos to completely load before serving requests." ) +FEATURE_BIT(VALIDATE_HEADERS, validate_headers, + "Validate all header set operations to conform to RFC7230." +) + #undef FEATURE_BIT /*lint -restore */ From nils.goroll at uplex.de Fri Oct 9 13:30:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Oct 2020 13:30:07 +0000 (UTC) Subject: [master] aee1434be use the new header check macro for consistency Message-ID: <20201009133007.D7392BF3E2@lists.varnish-cache.org> commit aee1434be153f2f619804010b3199fd2d7d9fda8 Author: Nils Goroll Date: Fri Oct 9 15:28:29 2020 +0200 use the new header check macro for consistency Ref #3407 diff --git a/bin/varnishd/http1/cache_http1_proto.c b/bin/varnishd/http1/cache_http1_proto.c index 0e932d7c9..ebf9ad1f1 100644 --- a/bin/varnishd/http1/cache_http1_proto.c +++ b/bin/varnishd/http1/cache_http1_proto.c @@ -125,7 +125,7 @@ http1_dissect_hdrs(struct http *hp, char *p, struct http_conn *htc, if (vct_iscrlf(p, htc->rxbuf_e)) break; while (r < htc->rxbuf_e) { - if (!vct_isctl(*r) || vct_issp(*r)) { + if (vct_ishdrval(*r)) { r++; continue; } From nils.goroll at uplex.de Fri Oct 9 14:48:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Oct 2020 14:48:07 +0000 (UTC) Subject: [master] 7649bb1ae shard: use a configuration context per instance Message-ID: <20201009144807.3081354DF@lists.varnish-cache.org> commit 7649bb1aed54cbb6c2a7e19f7a2b0e201e33ae61 Author: Nils Goroll Date: Fri Oct 9 16:22:16 2020 +0200 shard: use a configuration context per instance The shard director had the limitation to only support adding/removing backends of one instance before `.reconfigure()` on that instance had to be called, then the next instance could be changed. This limitation came from the time when it was still unclear that calling `VRT_priv_task()` on a per-object pointer was the accepted interface for per-instance privs (varnish historians' recommended read: https://github.com/varnishcache/varnish-cache/wiki/VIP1:-PRIV_*-visibility-and-lifetime-control ) It is about time to lift that limitation now. o/ Dridi diff --git a/bin/varnishtest/tests/d00015.vtc b/bin/varnishtest/tests/d00015.vtc index 5050cff45..37899b829 100644 --- a/bin/varnishtest/tests/d00015.vtc +++ b/bin/varnishtest/tests/d00015.vtc @@ -107,6 +107,7 @@ varnish v1 -vcl+backend { vd.add_backend(s3, "2"); vd.add_backend(s3, "3"); vd2.clear(); + vd2.add_backend(s3, "4"); vd.add_backend(s3, "4"); vd.add_backend(s3, "5"); vd.add_backend(s3, "6"); @@ -241,7 +242,6 @@ logexpect l1 -v v1 -g raw -d 1 { expect 0 0 Debug {^shard:.*point = 6e040182, host = 0} expect 0 0 VCL_Log {^-- re-add some - no 2nd director$} - expect 0 0 Error {^shard vd2: cannot change more than} expect 0 0 Debug {^shard:.*point = 3d1fe97, host = 3} expect 0 0 Debug {^shard:.*point = a25a43b, host = 6} expect 0 0 Debug {^shard:.*point = 2b20d9a2, host = 1} diff --git a/bin/varnishtest/tests/d00016.vtc b/bin/varnishtest/tests/d00016.vtc index ac0bb7644..5452e0908 100644 --- a/bin/varnishtest/tests/d00016.vtc +++ b/bin/varnishtest/tests/d00016.vtc @@ -110,6 +110,7 @@ varnish v1 -vcl+backend { vd.add_backend(s3, "2"); vd.add_backend(s3, "3"); vd2.clear(); + vd2.add_backend(s3, "4"); vd.add_backend(s3, "4"); vd.add_backend(s3, "5"); vd.add_backend(s3, "6"); @@ -169,7 +170,6 @@ logexpect l1 -v v1 -g raw -d 1 { expect 0 1001 VCL_Log {^-- remove s1_2 specifically$} expect 0 1001 VCL_Log {^-- remove all instances of s1$} expect 0 1001 VCL_Log {^-- re-add some - no 2nd director$} - expect 0 1001 Error {^shard vd2: cannot change more than} expect 0 1001 VCL_Log {^-- remove second-last$} expect 0 1001 VCL_Log {^-- remove last$} expect 0 1001 VCL_Log {^-- END$} diff --git a/doc/changes.rst b/doc/changes.rst index b2c192e43..a2ab4f8b1 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -37,6 +37,10 @@ Varnish Cache Next (2021-03-15) set headers are now validated to contain only characters allowed by RFC7230. A (runtime) VCL failure is triggered if not. +* The shard director now supports reconfiguration (adding/removing + backends) of several instances without any special ordering + requirement. + ================================ Varnish Cache 6.5.1 (2020-09-25) ================================ diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c index 82c51fd16..5b71ffd82 100644 --- a/lib/libvmod_directors/shard_cfg.c +++ b/lib/libvmod_directors/shard_cfg.c @@ -80,27 +80,22 @@ struct backend_reconfig { * * for backend reconfiguration, we create a change list on the VCL workspace in * a PRIV_TASK state, which we work in reconfigure. - * - * for now, we allow to only reconfigure one shard director at a time. */ static struct shard_change * -shard_change_get(VRT_CTX, struct vmod_priv *priv, - const struct sharddir * const shardd) +shard_change_get(VRT_CTX, const struct sharddir * const shardd) { + struct vmod_priv *task; struct shard_change *change; + const void *id = (const char *)shardd + task_off_cfg; - if (priv->priv) { - CAST_OBJ_NOTNULL(change, priv->priv, SHARD_CHANGE_MAGIC); - if (change->shardd == NULL) { - change->shardd = shardd; - VSTAILQ_INIT(&change->tasks); - } else if (change->shardd != shardd) { - shard_err0(ctx, shardd, - "cannot change more than one shard director " - "at a time"); - return (NULL); - } + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + + task = VRT_priv_task(ctx, id); + + if (task->priv != NULL) { + CAST_OBJ_NOTNULL(change, task->priv, SHARD_CHANGE_MAGIC); + assert (change->shardd == shardd); return (change); } @@ -114,7 +109,7 @@ shard_change_get(VRT_CTX, struct vmod_priv *priv, change->space = NULL; change->shardd = shardd; VSTAILQ_INIT(&change->tasks); - priv->priv = change; + task->priv = change; return (change); } @@ -124,7 +119,6 @@ shard_change_finish(struct shard_change *change) { CHECK_OBJ_NOTNULL(change, SHARD_CHANGE_MAGIC); - change->shardd = NULL; VSTAILQ_INIT(&change->tasks); } @@ -150,8 +144,7 @@ shard_change_task_add(VRT_CTX, struct shard_change *change, } static inline struct shard_change_task * -shard_change_task_backend(VRT_CTX, - struct vmod_priv *priv, const struct sharddir *shardd, +shard_change_task_backend(VRT_CTX, const struct sharddir *shardd, enum shard_change_task_e task_e, VCL_BACKEND be, VCL_STRING ident, VCL_DURATION rampup) { @@ -161,7 +154,7 @@ shard_change_task_backend(VRT_CTX, CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); assert(task_e == ADD_BE || task_e == REMOVE_BE); - change = shard_change_get(ctx, priv, shardd); + change = shard_change_get(ctx, shardd); if (change == NULL) return (NULL); @@ -183,16 +176,15 @@ shard_change_task_backend(VRT_CTX, * director reconfiguration tasks */ VCL_BOOL -shardcfg_add_backend(VRT_CTX, struct vmod_priv *priv, - const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident, - VCL_DURATION rampup, VCL_REAL weight) +shardcfg_add_backend(VRT_CTX, const struct sharddir *shardd, + VCL_BACKEND be, VCL_STRING ident, VCL_DURATION rampup, VCL_REAL weight) { struct shard_change_task *task; assert (weight >= 1); AN(be); - task = shard_change_task_backend(ctx, priv, shardd, ADD_BE, + task = shard_change_task_backend(ctx, shardd, ADD_BE, be, ident, rampup); if (task == NULL) @@ -203,21 +195,21 @@ shardcfg_add_backend(VRT_CTX, struct vmod_priv *priv, } VCL_BOOL -shardcfg_remove_backend(VRT_CTX, struct vmod_priv *priv, - const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident) +shardcfg_remove_backend(VRT_CTX, const struct sharddir *shardd, + VCL_BACKEND be, VCL_STRING ident) { - return (shard_change_task_backend(ctx, priv, shardd, REMOVE_BE, + return (shard_change_task_backend(ctx, shardd, REMOVE_BE, be, ident, 0) != NULL); } VCL_BOOL -shardcfg_clear(VRT_CTX, struct vmod_priv *priv, const struct sharddir *shardd) +shardcfg_clear(VRT_CTX, const struct sharddir *shardd) { struct shard_change *change; CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); - change = shard_change_get(ctx, priv, shardd); + change = shard_change_get(ctx, shardd); if (change == NULL) return (0); @@ -609,8 +601,7 @@ shardcfg_apply_change(VRT_CTX, struct sharddir *shardd, */ VCL_BOOL -shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv, - struct sharddir *shardd, VCL_INT replicas) +shardcfg_reconfigure(VRT_CTX, struct sharddir *shardd, VCL_INT replicas) { struct shard_change *change; @@ -621,7 +612,7 @@ shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv, return (0); } - change = shard_change_get(ctx, priv, shardd); + change = shard_change_get(ctx, shardd); if (change == NULL) return (0); diff --git a/lib/libvmod_directors/shard_cfg.h b/lib/libvmod_directors/shard_cfg.h index 0b8b8612c..5c1dc5c20 100644 --- a/lib/libvmod_directors/shard_cfg.h +++ b/lib/libvmod_directors/shard_cfg.h @@ -28,15 +28,11 @@ * SUCH DAMAGE. */ -VCL_BOOL shardcfg_add_backend(VRT_CTX, struct vmod_priv *priv, - const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident, - VCL_DURATION rampup, VCL_REAL weight); -VCL_BOOL shardcfg_remove_backend(VRT_CTX, struct vmod_priv *priv, - const struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident); -VCL_BOOL shardcfg_clear(VRT_CTX, struct vmod_priv *priv, - const struct sharddir *shardd); -VCL_BOOL shardcfg_reconfigure(VRT_CTX, struct vmod_priv *priv, - struct sharddir *shardd, VCL_INT replicas); +VCL_BOOL shardcfg_add_backend(VRT_CTX, const struct sharddir *shardd, + VCL_BACKEND be, VCL_STRING ident, VCL_DURATION rampup, VCL_REAL weight); +VCL_BOOL shardcfg_remove_backend(VRT_CTX, const struct sharddir *shardd, + VCL_BACKEND be, VCL_STRING ident); +VCL_BOOL shardcfg_clear(VRT_CTX, const struct sharddir *shardd); +VCL_BOOL shardcfg_reconfigure(VRT_CTX, struct sharddir *shardd, VCL_INT); VCL_VOID shardcfg_set_warmup(struct sharddir *shardd, VCL_REAL ratio); -VCL_VOID shardcfg_set_rampup(struct sharddir *shardd, - VCL_DURATION duration); +VCL_VOID shardcfg_set_rampup(struct sharddir *shardd, VCL_DURATION duration); diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h index 67b709aac..c2c8ddeab 100644 --- a/lib/libvmod_directors/shard_dir.h +++ b/lib/libvmod_directors/shard_dir.h @@ -73,6 +73,12 @@ struct sharddir { uint32_t n_points; }; +/* VRT_priv_task() id offsets */ +enum shard_task_off_e { + task_off_param = 0, + task_off_cfg = 1 +}; + static inline VCL_BACKEND sharddir_backend(const struct sharddir *shardd, unsigned id) { diff --git a/lib/libvmod_directors/vmod_directors.vcc b/lib/libvmod_directors/vmod_directors.vcc index f0a32af50..f548b3f5d 100644 --- a/lib/libvmod_directors/vmod_directors.vcc +++ b/lib/libvmod_directors/vmod_directors.vcc @@ -376,7 +376,7 @@ association. The association can be changed per backend request using the *param* argument of `xshard.backend()`_. -$Method BOOL .add_backend(PRIV_TASK, BACKEND backend, +$Method BOOL .add_backend(BACKEND backend, [STRING ident], [DURATION rampup], [REAL weight]) Add a backend *backend* to the director. @@ -399,28 +399,25 @@ effect of *weight* is also capped such that the total number of replicas does not exceed `UINT32_MAX`. NOTE: Backend changes need to be finalized with -`xshard.reconfigure()`_ and are only supported on one -shard director at a time. +`xshard.reconfigure()`_. -$Method BOOL .remove_backend(PRIV_TASK, [BACKEND backend=0], [STRING ident=0]) +$Method BOOL .remove_backend([BACKEND backend=0], [STRING ident=0]) Remove backend(s) from the director. Either *backend* or *ident* must be specified. *ident* removes a specific instance. If *backend* is given without *ident*, all instances of this backend are removed. NOTE: Backend changes need to be finalized with -`xshard.reconfigure()`_ and are only supported on one -shard director at a time. +`xshard.reconfigure()`_. -$Method BOOL .clear(PRIV_TASK) +$Method BOOL .clear() Remove all backends from the director. NOTE: Backend changes need to be finalized with -`xshard.reconfigure()`_ and are only supported on one -shard director at a time. +`xshard.reconfigure()`_. -$Method BOOL .reconfigure(PRIV_TASK, INT replicas=67) +$Method BOOL .reconfigure(INT replicas=67) Reconfigure the consistent hashing ring to reflect backend changes. diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index fbcd1d132..18c3465ce 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -326,11 +326,10 @@ vmod_shard_add_backend(VRT_CTX, struct vmod_directors_shard *vshard, ".add_backend(weight=%f) ignored", args->weight); } - return shardcfg_add_backend(ctx, args->arg1, - vshard->shardd, args->backend, + return (shardcfg_add_backend(ctx, vshard->shardd, args->backend, args->valid_ident ? args->ident : NULL, args->valid_rampup ? args->rampup : nan(""), - weight); + weight)); } VCL_BOOL v_matchproto_(td_directors_shard_remove_backend) @@ -349,23 +348,21 @@ vmod_shard_remove_backend(VRT_CTX, struct vmod_directors_shard *vshard, return (0); } - return shardcfg_remove_backend(ctx, args->arg1, vshard->shardd, - be, ident); + return (shardcfg_remove_backend(ctx, vshard->shardd, be, ident)); } VCL_BOOL v_matchproto_(td_directors_shard_clear) -vmod_shard_clear(VRT_CTX, struct vmod_directors_shard *vshard, - struct vmod_priv *priv) +vmod_shard_clear(VRT_CTX, struct vmod_directors_shard *vshard) { CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); - return shardcfg_clear(ctx, priv, vshard->shardd); + return (shardcfg_clear(ctx, vshard->shardd)); } VCL_BOOL v_matchproto_(td_directors_shard_reconfigure) vmod_shard_reconfigure(VRT_CTX, struct vmod_directors_shard *vshard, - struct vmod_priv *priv, VCL_INT replicas) + VCL_INT replicas) { - return shardcfg_reconfigure(ctx, priv, vshard->shardd, replicas); + return (shardcfg_reconfigure(ctx, vshard->shardd, replicas)); } static inline uint32_t @@ -878,12 +875,14 @@ shard_param_task(VRT_CTX, const void *id, { struct vmod_directors_shard_param *p; struct vmod_priv *task; + const void *task_id; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(pa, VMOD_SHARD_SHARD_PARAM_MAGIC); assert(pa->scope > _SCOPE_INVALID); - task = VRT_priv_task(ctx, id); + task_id = (const char *)id + task_off_param; + task = VRT_priv_task(ctx, task_id); if (task == NULL) { VRT_fail(ctx, "no priv_task"); From nils.goroll at uplex.de Fri Oct 9 17:46:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Oct 2020 17:46:06 +0000 (UTC) Subject: [master] 61f3d9a93 vslc: New mmap cursor type for regular files Message-ID: <20201009174606.655B860FAB@lists.varnish-cache.org> commit 61f3d9a936bd75c68b54955bd8f17b5a539ffa26 Author: Dridi Boukelmoune Date: Wed Oct 7 14:35:04 2020 +0200 vslc: New mmap cursor type for regular files This cursor is the simplest of all as of now, and also the fastest way to skim through a -r input when it's a regular file. This is essentially a file cursor with much less memory churn. On a 13GB log dump captured during a performance benchmark varnishlog takes around 6m15s to print all the transactions to /dev/null but the mmap cursor consistently reduces this time to around 1m01s. This is particularly useful to audit logs at rest, be it error logs after an incident or transactions captured during a benchmark-type scenario. diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c index cf5349ade..bb79dc9b3 100644 --- a/lib/libvarnishapi/vsl_cursor.c +++ b/lib/libvarnishapi/vsl_cursor.c @@ -33,12 +33,15 @@ #include "config.h" +#include +#include +#include + #include #include #include #include #include -#include #include #include "vdef.h" @@ -399,9 +402,115 @@ static const struct vslc_tbl vslc_file_tbl = { .check = NULL, }; +struct vslc_mmap { + unsigned magic; +#define VSLC_MMAP_MAGIC 0x7de15f61 + int fd; + char *b; + char *e; + struct VSL_cursor cursor; + struct VSLC_ptr next; +}; + +static void +vslc_mmap_delete(const struct VSL_cursor *cursor) +{ + struct vslc_mmap *c; + + CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_MMAP_MAGIC); + assert(&c->cursor == cursor); + AZ(munmap(c->b, c->e - c->b)); + FREE_OBJ(c); +} + +static enum vsl_status v_matchproto_(vslc_next_f) +vslc_mmap_next(const struct VSL_cursor *cursor) +{ + struct vslc_mmap *c; + const char *t; + + CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_MMAP_MAGIC); + assert(&c->cursor == cursor); + c->cursor.rec = c->next; + t = TRUST_ME(c->cursor.rec.ptr); + if (t == c->e) + return (vsl_e_eof); + c->next.ptr = VSL_NEXT(c->next.ptr); + t = TRUST_ME(c->next.ptr); + if (t > c->e) + return (vsl_e_io); + return (vsl_more); +} + +static enum vsl_status v_matchproto_(vslc_reset_f) +vslc_mmap_reset(const struct VSL_cursor *cursor) +{ + struct vslc_mmap *c; + + CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_MMAP_MAGIC); + assert(&c->cursor == cursor); + return (vsl_e_eof); +} + +static enum vsl_check v_matchproto_(vslc_check_f) +vslc_mmap_check(const struct VSL_cursor *cursor, const struct VSLC_ptr *ptr) +{ + struct vslc_mmap *c; + + CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_MMAP_MAGIC); + assert(&c->cursor == cursor); + AN(ptr->ptr); + return (vsl_check_valid); +} + +static const struct vslc_tbl vslc_mmap_tbl = { + .magic = VSLC_TBL_MAGIC, + .delete = vslc_mmap_delete, + .next = vslc_mmap_next, + .reset = vslc_mmap_reset, + .check = vslc_mmap_check, +}; + +static struct VSL_cursor * +vsl_cursor_mmap(struct VSL_data *vsl, int fd) +{ + struct vslc_mmap *c; + struct stat st[1]; + void *p; + + AZ(fstat(fd, st)); + if ((st->st_mode & S_IFMT) != S_IFREG) + return (MAP_FAILED); + + assert(st->st_size >= sizeof VSL_FILE_ID); + p = mmap(NULL, st->st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (p == MAP_FAILED) { + vsl_diag(vsl, "Cannot mmap: %s", strerror(errno)); + return (MAP_FAILED); + } + + ALLOC_OBJ(c, VSLC_MMAP_MAGIC); + if (c == NULL) { + (void)munmap(p, st->st_size); + (void)close(fd); + vsl_diag(vsl, "Out of memory"); + return (NULL); + } + c->cursor.priv_tbl = &vslc_mmap_tbl; + c->cursor.priv_data = c; + + c->fd = fd; + c->b = p; + c->e = c->b + st->st_size; + c->next.ptr = TRUST_ME(c->b + sizeof VSL_FILE_ID); + + return (&c->cursor); +} + struct VSL_cursor * VSL_CursorFile(struct VSL_data *vsl, const char *name, unsigned options) { + struct VSL_cursor *mc; struct vslc_file *c; int fd; int close_fd = 0; @@ -440,6 +549,12 @@ VSL_CursorFile(struct VSL_data *vsl, const char *name, unsigned options) return (NULL); } + mc = vsl_cursor_mmap(vsl, fd); + if (mc == NULL) + return (NULL); + if (mc != MAP_FAILED) + return (mc); + ALLOC_OBJ(c, VSLC_FILE_MAGIC); if (c == NULL) { if (close_fd) From nils.goroll at uplex.de Fri Oct 9 17:46:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Oct 2020 17:46:06 +0000 (UTC) Subject: [master] 4f763c79f SQUASHME: assert Message-ID: <20201009174606.7963460FAE@lists.varnish-cache.org> commit 4f763c79f98f2131b4f61efe9ba5242ab363afe2 Author: Dridi Boukelmoune Date: Thu Oct 8 10:57:59 2020 +0200 SQUASHME: assert diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c index bb79dc9b3..a9d83de36 100644 --- a/lib/libvarnishapi/vsl_cursor.c +++ b/lib/libvarnishapi/vsl_cursor.c @@ -456,10 +456,14 @@ static enum vsl_check v_matchproto_(vslc_check_f) vslc_mmap_check(const struct VSL_cursor *cursor, const struct VSLC_ptr *ptr) { struct vslc_mmap *c; + const char *t; CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_MMAP_MAGIC); assert(&c->cursor == cursor); AN(ptr->ptr); + t = TRUST_ME(ptr->ptr); + assert(t > c->b); + assert(t <= c->e); return (vsl_check_valid); } From nils.goroll at uplex.de Fri Oct 9 18:06:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Oct 2020 18:06:06 +0000 (UTC) Subject: [master] 7b2c2be46 fix a comment based on insights from #3434 Message-ID: <20201009180606.91A9561B04@lists.varnish-cache.org> commit 7b2c2be46f1e6c749fb57e90e362b2fbb3b4d2e7 Author: Nils Goroll Date: Fri Oct 9 19:50:19 2020 +0200 fix a comment based on insights from #3434 diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index c8958dfb4..cd975927c 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -1028,9 +1028,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv) VCL_TaskLeave(bo->privs); http_Teardown(bo->bereq); http_Teardown(bo->beresp); - // XXX after 6.4 release: - // bereq_body should have 0 or 1 references remaining, - // see VRB_Free() for the other end + // can not make assumptions about the number of references here #3434 if (bo->bereq_body != NULL) (void) HSH_DerefObjCore(bo->wrk, &bo->bereq_body, 0); From nils.goroll at uplex.de Fri Oct 9 18:06:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Oct 2020 18:06:06 +0000 (UTC) Subject: [master] 1dc2103db typo Message-ID: <20201009180606.A29B161B0B@lists.varnish-cache.org> commit 1dc2103dbddae7bf1cd6e98671011bc2ce41e3ba Author: Nils Goroll Date: Fri Oct 9 18:56:10 2020 +0200 typo diff --git a/lib/libvmod_directors/vmod_directors.vcc b/lib/libvmod_directors/vmod_directors.vcc index f548b3f5d..266ce560c 100644 --- a/lib/libvmod_directors/vmod_directors.vcc +++ b/lib/libvmod_directors/vmod_directors.vcc @@ -222,7 +222,7 @@ Example:: $Method BACKEND .backend(STRANDS) -Pick a backend from the backend director. +Pick a backend from the hash director. Use the string or list of strings provided to pick the backend. From nils.goroll at uplex.de Fri Oct 9 18:06:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Oct 2020 18:06:06 +0000 (UTC) Subject: [master] 4b844189f gc unused struct member Message-ID: <20201009180606.BB07661B12@lists.varnish-cache.org> commit 4b844189f081ba1555367783edde1906c80043b5 Author: Nils Goroll Date: Fri Oct 9 16:50:01 2020 +0200 gc unused struct member diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c index 5b71ffd82..6976b96bd 100644 --- a/lib/libvmod_directors/shard_cfg.c +++ b/lib/libvmod_directors/shard_cfg.c @@ -63,7 +63,6 @@ struct shard_change { unsigned magic; #define SHARD_CHANGE_MAGIC 0xdff5c9a6 const struct sharddir *shardd; - void *space; VSTAILQ_HEAD(,shard_change_task) tasks; }; @@ -106,7 +105,6 @@ shard_change_get(VRT_CTX, const struct sharddir * const shardd) } INIT_OBJ(change, SHARD_CHANGE_MAGIC); - change->space = NULL; change->shardd = shardd; VSTAILQ_INIT(&change->tasks); task->priv = change; From nils.goroll at uplex.de Fri Oct 9 18:06:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Oct 2020 18:06:06 +0000 (UTC) Subject: [master] 1217f4b28 fix confusing example Message-ID: <20201009180606.D905C61B19@lists.varnish-cache.org> commit 1217f4b28c733e55e418ae5722fe6128119d790e Author: Nils Goroll Date: Fri Oct 9 19:03:46 2020 +0200 fix confusing example diff --git a/lib/libvmod_directors/vmod_directors.vcc b/lib/libvmod_directors/vmod_directors.vcc index 266ce560c..542330cc8 100644 --- a/lib/libvmod_directors/vmod_directors.vcc +++ b/lib/libvmod_directors/vmod_directors.vcc @@ -623,12 +623,12 @@ implement retries on alternative backends:: new dir_A = directors.shard(); dir_A.add_backend(...); - dir_A.reconfigure(shard_param); + dir_A.reconfigure(); dir_A.associate(shard_param.use()); # <-- ! new dir_B = directors.shard(); dir_B.add_backend(...); - dir_B.reconfigure(shard_param); + dir_B.reconfigure(); dir_B.associate(shard_param.use()); # <-- ! } From nils.goroll at uplex.de Fri Oct 9 18:06:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 9 Oct 2020 18:06:06 +0000 (UTC) Subject: [master] 6f7a1d564 shard: make `.reconfigure()` call optional Message-ID: <20201009180607.0CEBF61B20@lists.varnish-cache.org> commit 6f7a1d5640cdaef571ab5861e70d9b717a117190 Author: Nils Goroll Date: Fri Oct 9 18:55:55 2020 +0200 shard: make `.reconfigure()` call optional Making use of the PRIV_TASK `.free()` callback, we can finalize the reconfiguration when the tasks ends, in case `.reconfigure()` has not been called explicitly. o/ Dridi Also turn `SLT_Error` `"(notice)"` logs into `SLT_Notice`. diff --git a/bin/varnishtest/tests/d00015.vtc b/bin/varnishtest/tests/d00015.vtc index 37899b829..0a5b0da57 100644 --- a/bin/varnishtest/tests/d00015.vtc +++ b/bin/varnishtest/tests/d00015.vtc @@ -182,12 +182,10 @@ varnish v1 -vcl+backend { vd.add_backend(s2, "5"); vd.remove_backend(s1, "5"); vd.remove_backend(s1, "4"); - vd.add_backend(s3, "4"); - vd.add_backend(s3, "5"); - if (!vd.reconfigure(replicas=1)) { - return(fail("reconfigure failed")); - } + vd.add_backend(s3, "4a"); + vd.add_backend(s3, "5a"); vd.set_warmup(0); + # implicit .reconfigure() - checked via backend.list } sub vcl_recv { @@ -220,14 +218,14 @@ logexpect l1 -v v1 -g raw -d 1 { expect 0 0 VCL_Log {^reconfigure failed} expect 0 0 VCL_Log {^-- duplicate add$} - expect 0 0 Error {^shard vd: .notice. backend s1 already exists - skipping$} + expect 0 0 Notice {^shard vd: backend s1 already exists - skipping$} expect 0 0 Debug {^shard:.*point = 6e040182, host = 1} expect 0 0 Debug {^shard:.*point = f08ad325, host = 0} expect 0 0 VCL_Log {^-- duplicate add with idents$} - expect 0 0 Error {^shard vd: .notice. backend s1 already exists - skipping} - expect 0 0 Error {^shard vd: .notice. backend s1/s1_1 already exists - skipping} - expect 0 0 Error {^shard vd: .notice. backend s1/s1_2 already exists - skipping} + expect 0 0 Notice {^shard vd: backend s1 already exists - skipping} + expect 0 0 Notice {^shard vd: backend s1/s1_1 already exists - skipping} + expect 0 0 Notice {^shard vd: backend s1/s1_2 already exists - skipping} expect 0 0 Debug {^shard:.*point = 6e040182, host = 3} expect 0 0 Debug {^shard:.*point = 732c7bbe, host = 2} expect 0 0 Debug {^shard:.*point = bae80b0b, host = 1} @@ -282,4 +280,4 @@ client c1 { logexpect l1 -wait -varnish v1 -cliexpect {(?s)Ident.*s3 +1 +h.*s3 +2 +h.*s3 +4 +h.*s3 +5 +h.*s3 +6 +h.*s3 +7 +h} "backend.list -p" +varnish v1 -cliexpect {(?s)Ident.*s3 +1 +h.*s3 +2 +h.*s3 +4a +h.*s3 +5a +h.*s3 +6 +h.*s3 +7 +h} "backend.list -p" diff --git a/bin/varnishtest/tests/d00016.vtc b/bin/varnishtest/tests/d00016.vtc index 5452e0908..a6a8ed70a 100644 --- a/bin/varnishtest/tests/d00016.vtc +++ b/bin/varnishtest/tests/d00016.vtc @@ -1,6 +1,8 @@ varnishtest "shard director/int reconfiguration outside init" server s1 { + rxreq + txresp -body "ech3Ooj" } -start server s2 { @@ -9,6 +11,8 @@ server s2 { server s3 { rxreq txresp -body "xiuFi3Pe" + rxreq + txresp -body "xiuFi3Pe" } -start varnish v1 -vcl+backend { @@ -22,7 +26,17 @@ varnish v1 -vcl+backend { } sub vcl_recv { - + if (req.http.change == "apply") { + # test implicit reconfigure at the end of task + vd.clear(); + vd.add_backend(s1); + set req.backend_hint = vd.backend(); + return (pass); + } + if (req.http.change == "test") { + set req.backend_hint = vd.backend(); + return (pass); + } std.log("-- invalid replicas"); if (!vd.reconfigure(replicas=0)) { # continue intentionally @@ -162,11 +176,11 @@ logexpect l1 -v v1 -g raw -d 1 { expect 0 1001 VCL_Log {^reconfigure failed} expect 0 1001 VCL_Log {^-- duplicate add$} - expect 0 1001 Error {^shard vd: .notice. backend s1 already exists - skipping$} + expect 0 1001 Notice {^shard vd: backend s1 already exists - skipping$} expect 0 1001 VCL_Log {^-- duplicate add with idents$} - expect 0 1001 Error {^shard vd: .notice. backend s1 already exists - skipping} - expect 0 1001 Error {^shard vd: .notice. backend s1/s1_1 already exists - skipping} - expect 0 1001 Error {^shard vd: .notice. backend s1/s1_2 already exists - skipping} + expect 0 1001 Notice {^shard vd: backend s1 already exists - skipping} + expect 0 1001 Notice {^shard vd: backend s1/s1_1 already exists - skipping} + expect 0 1001 Notice {^shard vd: backend s1/s1_2 already exists - skipping} expect 0 1001 VCL_Log {^-- remove s1_2 specifically$} expect 0 1001 VCL_Log {^-- remove all instances of s1$} expect 0 1001 VCL_Log {^-- re-add some - no 2nd director$} @@ -220,6 +234,16 @@ client c1 { txreq rxresp expect resp.body == "xiuFi3Pe" + + # director not yet changed for this task + txreq -hdr "change: apply" + rxresp + expect resp.body == "xiuFi3Pe" + + # change active + txreq -hdr "change: test" + rxresp + expect resp.body == "ech3Ooj" } -run logexpect l1 -wait diff --git a/doc/changes.rst b/doc/changes.rst index a2ab4f8b1..7c4bbba93 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -41,6 +41,13 @@ Varnish Cache Next (2021-03-15) backends) of several instances without any special ordering requirement. +* Calling the shard director ``.reconfigure()`` method is now + optional. If not called explicitly, any shard director backend + changes are applied at the end of the current task. + +* Shard director ``Error`` log messages with ``(notice)`` have been + turned into ``Notice`` log messages. + ================================ Varnish Cache 6.5.1 (2020-09-25) ================================ diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c index 6976b96bd..b61cb9392 100644 --- a/lib/libvmod_directors/shard_cfg.c +++ b/lib/libvmod_directors/shard_cfg.c @@ -62,7 +62,8 @@ struct shard_change_task { struct shard_change { unsigned magic; #define SHARD_CHANGE_MAGIC 0xdff5c9a6 - const struct sharddir *shardd; + struct vsl_log *vsl; + struct sharddir *shardd; VSTAILQ_HEAD(,shard_change_task) tasks; }; @@ -73,6 +74,10 @@ struct backend_reconfig { unsigned hole_i; // index hint on first hole }; +/* forward decl */ +static VCL_BOOL +change_reconfigure(struct shard_change *change, VCL_INT replicas); + /* * ============================================================ * change / task list @@ -81,8 +86,21 @@ struct backend_reconfig { * a PRIV_TASK state, which we work in reconfigure. */ +static void v_matchproto_(vmod_priv_free_f) +shard_change_fini(void * priv) +{ + struct shard_change *change; + + if (priv == NULL) + return; + + CAST_OBJ_NOTNULL(change, priv, SHARD_CHANGE_MAGIC); + + (void) change_reconfigure(change, 67); +} + static struct shard_change * -shard_change_get(VRT_CTX, const struct sharddir * const shardd) +shard_change_get(VRT_CTX, struct sharddir * const shardd) { struct vmod_priv *task; struct shard_change *change; @@ -94,6 +112,7 @@ shard_change_get(VRT_CTX, const struct sharddir * const shardd) if (task->priv != NULL) { CAST_OBJ_NOTNULL(change, task->priv, SHARD_CHANGE_MAGIC); + assert (change->vsl == ctx->vsl); assert (change->shardd == shardd); return (change); } @@ -105,9 +124,11 @@ shard_change_get(VRT_CTX, const struct sharddir * const shardd) } INIT_OBJ(change, SHARD_CHANGE_MAGIC); + change->vsl = ctx->vsl; change->shardd = shardd; VSTAILQ_INIT(&change->tasks); task->priv = change; + task->free = shard_change_fini; return (change); } @@ -142,7 +163,7 @@ shard_change_task_add(VRT_CTX, struct shard_change *change, } static inline struct shard_change_task * -shard_change_task_backend(VRT_CTX, const struct sharddir *shardd, +shard_change_task_backend(VRT_CTX, struct sharddir *shardd, enum shard_change_task_e task_e, VCL_BACKEND be, VCL_STRING ident, VCL_DURATION rampup) { @@ -174,7 +195,7 @@ shard_change_task_backend(VRT_CTX, const struct sharddir *shardd, * director reconfiguration tasks */ VCL_BOOL -shardcfg_add_backend(VRT_CTX, const struct sharddir *shardd, +shardcfg_add_backend(VRT_CTX, struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident, VCL_DURATION rampup, VCL_REAL weight) { struct shard_change_task *task; @@ -193,7 +214,7 @@ shardcfg_add_backend(VRT_CTX, const struct sharddir *shardd, } VCL_BOOL -shardcfg_remove_backend(VRT_CTX, const struct sharddir *shardd, +shardcfg_remove_backend(VRT_CTX, struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident) { return (shard_change_task_backend(ctx, shardd, REMOVE_BE, @@ -201,7 +222,7 @@ shardcfg_remove_backend(VRT_CTX, const struct sharddir *shardd, } VCL_BOOL -shardcfg_clear(VRT_CTX, const struct sharddir *shardd) +shardcfg_clear(VRT_CTX, struct sharddir *shardd) { struct shard_change *change; @@ -510,7 +531,7 @@ shardcfg_backend_finalize(struct backend_reconfig *re) */ static void -shardcfg_apply_change(VRT_CTX, struct sharddir *shardd, +shardcfg_apply_change(struct vsl_log *vsl, struct sharddir *shardd, const struct shard_change *change, VCL_INT replicas) { struct shard_change_task *task, *clear; @@ -577,8 +598,8 @@ shardcfg_apply_change(VRT_CTX, struct sharddir *shardd, const char * const ident = b->ident; - shard_err(ctx, shardd, "(notice) backend %s%s%s " - "already exists - skipping", + sharddir_err(vsl, SLT_Notice, "shard %s: backend %s%s%s " + "already exists - skipping", shardd->name, VRT_BACKEND_string(b->backend), ident ? "/" : "", ident ? ident : ""); @@ -598,28 +619,22 @@ shardcfg_apply_change(VRT_CTX, struct sharddir *shardd, * top reconfiguration function */ -VCL_BOOL -shardcfg_reconfigure(VRT_CTX, struct sharddir *shardd, VCL_INT replicas) +static VCL_BOOL +change_reconfigure(struct shard_change *change, VCL_INT replicas) { - struct shard_change *change; + struct sharddir *shardd; + CHECK_OBJ_NOTNULL(change, SHARD_CHANGE_MAGIC); + assert (replicas > 0); + shardd = change->shardd; CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); - if (replicas <= 0) { - shard_err(ctx, shardd, - ".reconfigure() invalid replicas argument %ld", replicas); - return (0); - } - - change = shard_change_get(ctx, shardd); - if (change == NULL) - return (0); if (VSTAILQ_FIRST(&change->tasks) == NULL) return (1); sharddir_wrlock(shardd); - shardcfg_apply_change(ctx, shardd, change, replicas); + shardcfg_apply_change(change->vsl, shardd, change, replicas); shard_change_finish(change); if (shardd->hashcircle) @@ -627,7 +642,8 @@ shardcfg_reconfigure(VRT_CTX, struct sharddir *shardd, VCL_INT replicas) shardd->hashcircle = NULL; if (shardd->n_backend == 0) { - shard_err0(ctx, shardd, ".reconfigure() no backends"); + sharddir_err(change->vsl, SLT_Error, "shard %s: .reconfigure() " + "no backends", shardd->name); sharddir_unlock(shardd); return (0); } @@ -637,6 +653,26 @@ shardcfg_reconfigure(VRT_CTX, struct sharddir *shardd, VCL_INT replicas) return (1); } +VCL_BOOL +shardcfg_reconfigure(VRT_CTX, struct sharddir *shardd, VCL_INT replicas) +{ + struct shard_change *change; + + CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); + if (replicas <= 0) { + sharddir_err(ctx->vsl, SLT_Error, + "shard %s: .reconfigure() invalid replicas argument %ld", + shardd->name, replicas); + return (0); + } + + change = shard_change_get(ctx, shardd); + if (change == NULL) + return (0); + + return (change_reconfigure(change, replicas)); +} + /* * ============================================================ * misc config related diff --git a/lib/libvmod_directors/shard_cfg.h b/lib/libvmod_directors/shard_cfg.h index 5c1dc5c20..87a6f69e4 100644 --- a/lib/libvmod_directors/shard_cfg.h +++ b/lib/libvmod_directors/shard_cfg.h @@ -28,11 +28,11 @@ * SUCH DAMAGE. */ -VCL_BOOL shardcfg_add_backend(VRT_CTX, const struct sharddir *shardd, +VCL_BOOL shardcfg_add_backend(VRT_CTX, struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident, VCL_DURATION rampup, VCL_REAL weight); -VCL_BOOL shardcfg_remove_backend(VRT_CTX, const struct sharddir *shardd, +VCL_BOOL shardcfg_remove_backend(VRT_CTX, struct sharddir *shardd, VCL_BACKEND be, VCL_STRING ident); -VCL_BOOL shardcfg_clear(VRT_CTX, const struct sharddir *shardd); -VCL_BOOL shardcfg_reconfigure(VRT_CTX, struct sharddir *shardd, VCL_INT); +VCL_BOOL shardcfg_clear(VRT_CTX, struct sharddir *shardd); +VCL_BOOL shardcfg_reconfigure(VRT_CTX, struct sharddir *, VCL_INT); VCL_VOID shardcfg_set_warmup(struct sharddir *shardd, VCL_REAL ratio); VCL_VOID shardcfg_set_rampup(struct sharddir *shardd, VCL_DURATION duration); diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c index 809fd67a5..1d4419b66 100644 --- a/lib/libvmod_directors/shard_dir.c +++ b/lib/libvmod_directors/shard_dir.c @@ -78,13 +78,13 @@ sharddir_debug(struct sharddir *shardd, const uint32_t flags) } void -sharddir_err(VRT_CTX, enum VSL_tag_e tag, const char *fmt, ...) +sharddir_err(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...) { va_list ap; va_start(ap, fmt); - if (ctx->vsl) - VSLbv(ctx->vsl, tag, fmt, ap); + if (vsl != NULL) + VSLbv(vsl, tag, fmt, ap); else VSLv(tag, 0, fmt, ap); va_end(ap); diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h index c2c8ddeab..27c0aee60 100644 --- a/lib/libvmod_directors/shard_dir.h +++ b/lib/libvmod_directors/shard_dir.h @@ -94,18 +94,18 @@ sharddir_backend(const struct sharddir *shardd, unsigned id) #define shard_err(ctx, shardd, fmt, ...) \ do { \ - sharddir_err(ctx, SLT_Error, "shard %s: " fmt, \ + sharddir_err((ctx)->vsl, SLT_Error, "shard %s: " fmt, \ (shardd)->name, __VA_ARGS__); \ } while (0) #define shard_err0(ctx, shardd, msg) \ do { \ - sharddir_err(ctx, SLT_Error, "shard %s: %s", \ + sharddir_err((ctx)->vsl, SLT_Error, "shard %s: %s", \ (shardd)->name, (msg)); \ } while (0) void sharddir_debug(struct sharddir *shardd, const uint32_t flags); -void sharddir_err(VRT_CTX, enum VSL_tag_e tag, const char *fmt, ...); +void sharddir_err(struct vsl_log *, enum VSL_tag_e tag, const char *fmt, ...); void sharddir_new(struct sharddir **sharddp, const char *vcl_name, const struct vmod_directors_shard_param *param); void sharddir_set_param(struct sharddir *shardd, diff --git a/lib/libvmod_directors/vmod_directors.vcc b/lib/libvmod_directors/vmod_directors.vcc index 542330cc8..161841531 100644 --- a/lib/libvmod_directors/vmod_directors.vcc +++ b/lib/libvmod_directors/vmod_directors.vcc @@ -3,7 +3,7 @@ # itself. See LICENCE for details. # # Copyright (c) 2013-2015 Varnish Software AS -# Copyright 2009-2018 UPLEX - Nils Goroll Systemoptimierung +# Copyright 2009-2020 UPLEX - Nils Goroll Systemoptimierung # All rights reserved. # # Authors: Poul-Henning Kamp @@ -234,14 +234,6 @@ $Object shard() Create a shard director. -Note that the shard director needs to be configured using at least one -`xshard.add_backend()`_ call(s) **followed by a** -`xshard.reconfigure()`_ **call** before it can hand out -backends. - -_Note_ that due to various restrictions (documented below), it is -recommended to use the shard director on the backend side. - Introduction ```````````` @@ -315,12 +307,14 @@ The drawbacks are: Method `````` -When `xshard.reconfigure()`_ is called, a consistent -hashing circular data structure gets built from the last 32 bits of -SHA256 hash values of **\ ** (default *ident* being the -backend name) for each backend and for a running number *n* from 1 to -*replicas*. Hashing creates the seemingly random order for placement -of backends on the consistent hashing ring. When +When `xshard.reconfigure()`_ is called explicitly (or implicitly at +the end of any task containing reconfigurations like +`xshard.add_backend()`_), a consistent hashing circular data structure +gets built from the last 32 bits of SHA256 hash values of **\ +** (default *ident* being the backend name) for each backend and +for a running number *n* from 1 to the *replicas* argument to +`xshard.reconfigure()`_. Hashing creates the seemingly random order +for placement of backends on the consistent hashing ring. When `xshard.add_backend()`_ was called with a *weight* argument, *replicas* is scaled by that weight to add proportionally more copies of the that backend on the ring. @@ -352,6 +346,11 @@ when configuring the shard director, you are advised to check:: varnishlog -I Error:^shard +Additional information may be provided as Notices, which can be +checked using + + varnishlog -I Notice:^shard + $Method VOID .set_warmup(REAL probability=0.0) Set the default warmup probability. See the *warmup* parameter of @@ -398,31 +397,26 @@ at least 1. Values above 10 probably do not make much sense. The effect of *weight* is also capped such that the total number of replicas does not exceed `UINT32_MAX`. -NOTE: Backend changes need to be finalized with -`xshard.reconfigure()`_. - $Method BOOL .remove_backend([BACKEND backend=0], [STRING ident=0]) Remove backend(s) from the director. Either *backend* or *ident* must be specified. *ident* removes a specific instance. If *backend* is given without *ident*, all instances of this backend are removed. -NOTE: Backend changes need to be finalized with -`xshard.reconfigure()`_. - $Method BOOL .clear() Remove all backends from the director. -NOTE: Backend changes need to be finalized with -`xshard.reconfigure()`_. - $Method BOOL .reconfigure(INT replicas=67) -Reconfigure the consistent hashing ring to reflect backend changes. +.. !! mirror changes to shard_cfg.c: shard_change_fini() + +Explicitly reconfigure the consistent hashing ring to reflect backend +changes to become effective immediately. -This method must be called at least once before the director can be -used. +If this method is not called explicitly, reconfiguration happens at +the end of the current task (after ``vcl_init {}`` or when the current +client or backend task is finished). $Method INT .key(STRANDS) diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 18c3465ce..42dbcaf85 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -540,7 +540,7 @@ shard_param_args(VRT_CTX, } if (key_blob == NULL || key_blob->len == 0 || key_blob->blob == NULL) { - sharddir_err(ctx, SLT_Error, "%s %s: " + sharddir_err(ctx->vsl, SLT_Error, "%s %s: " "by=BLOB but no or empty key_blob " "- using key 0", who, p->vcl_name); From dridi at varni.sh Fri Oct 9 20:42:52 2020 From: dridi at varni.sh (Dridi Boukelmoune) Date: Fri, 9 Oct 2020 20:42:52 +0000 Subject: [master] 6f7a1d564 shard: make `.reconfigure()` call optional In-Reply-To: <20201009180607.0CEBF61B20@lists.varnish-cache.org> References: <20201009180607.0CEBF61B20@lists.varnish-cache.org> Message-ID: > +Additional information may be provided as Notices, which can be > +checked using > + > + varnishlog -I Notice:^shard > + This is not how SLT_Notice works, check the vsl(7) manual. From nils.goroll at uplex.de Sat Oct 10 08:45:37 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 10 Oct 2020 10:45:37 +0200 Subject: [master] 6f7a1d564 shard: make `.reconfigure()` call optional In-Reply-To: References: <20201009180607.0CEBF61B20@lists.varnish-cache.org> Message-ID: <7aeb4814-6764-9c46-f796-cafa4b0e8824@uplex.de> On 09/10/2020 22:42, Dridi Boukelmoune wrote: >> +Additional information may be provided as Notices, which can be >> +checked using >> + >> + varnishlog -I Notice:^shard >> + > > This is not how SLT_Notice works, check the vsl(7) manual. > Thank you, Dridi Nils -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From nils.goroll at uplex.de Sat Oct 10 10:24:07 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 10 Oct 2020 10:24:07 +0000 (UTC) Subject: [master] 8c9cde8fc polish shard director logging Message-ID: <20201010102407.2CD10ACE63@lists.varnish-cache.org> commit 8c9cde8fc3eb691f87757578158447bd4b8f7802 Author: Nils Goroll Date: Sat Oct 10 12:21:14 2020 +0200 polish shard director logging SLT_Notice requires messages to be prefixed by the vmod name. Using this format for notices only would appear inconsistent, so we change all shard log messages to that format. diff --git a/bin/varnishtest/tests/d00015.vtc b/bin/varnishtest/tests/d00015.vtc index 0a5b0da57..d87bd4498 100644 --- a/bin/varnishtest/tests/d00015.vtc +++ b/bin/varnishtest/tests/d00015.vtc @@ -199,75 +199,75 @@ logexpect l1 -v v1 -g raw -d 1 { expect 0 0 CLI {^Rd vcl.load} expect 0 0 VCL_Log {^-- invalid replicas$} - expect 0 0 Error {^shard vd: .reconfigure.. invalid replicas argument 0} + expect 0 0 Error {^vmod_directors shard vd: .reconfigure.. invalid replicas argument 0} expect 0 0 VCL_Log {^reconfigure failed} expect 0 0 VCL_Log {^-- no changes - no debug output$} expect 0 0 VCL_Log {^-- no backends$} - expect 0 0 Error {^shard vd: .reconfigure.. no backends} + expect 0 0 Error {^vmod_directors shard vd: .reconfigure.. no backends} expect 0 0 VCL_Log {^reconfigure failed} expect 0 0 VCL_Log {^-- one backend$} - expect 0 0 Debug {^shard:.*point = f08ad325, host = 0} + expect 0 0 Debug {^vmod_directors shard:.*point = f08ad325, host = 0} expect 0 0 VCL_Log {^-- no change - no output$} expect 0 0 VCL_Log {^-- change, clear, no backends$} - expect 0 0 Error {^shard vd: .reconfigure.. no backends} + expect 0 0 Error {^vmod_directors shard vd: .reconfigure.. no backends} expect 0 0 VCL_Log {^reconfigure failed} expect 0 0 VCL_Log {^-- duplicate add$} - expect 0 0 Notice {^shard vd: backend s1 already exists - skipping$} - expect 0 0 Debug {^shard:.*point = 6e040182, host = 1} - expect 0 0 Debug {^shard:.*point = f08ad325, host = 0} + expect 0 0 Notice {^vmod_directors shard vd: backend s1 already exists - skipping$} + expect 0 0 Debug {^vmod_directors shard:.*point = 6e040182, host = 1} + expect 0 0 Debug {^vmod_directors shard:.*point = f08ad325, host = 0} expect 0 0 VCL_Log {^-- duplicate add with idents$} - expect 0 0 Notice {^shard vd: backend s1 already exists - skipping} - expect 0 0 Notice {^shard vd: backend s1/s1_1 already exists - skipping} - expect 0 0 Notice {^shard vd: backend s1/s1_2 already exists - skipping} - expect 0 0 Debug {^shard:.*point = 6e040182, host = 3} - expect 0 0 Debug {^shard:.*point = 732c7bbe, host = 2} - expect 0 0 Debug {^shard:.*point = bae80b0b, host = 1} - expect 0 0 Debug {^shard:.*point = f08ad325, host = 0} + expect 0 0 Notice {^vmod_directors shard vd: backend s1 already exists - skipping} + expect 0 0 Notice {^vmod_directors shard vd: backend s1/s1_1 already exists - skipping} + expect 0 0 Notice {^vmod_directors shard vd: backend s1/s1_2 already exists - skipping} + expect 0 0 Debug {^vmod_directors shard:.*point = 6e040182, host = 3} + expect 0 0 Debug {^vmod_directors shard:.*point = 732c7bbe, host = 2} + expect 0 0 Debug {^vmod_directors shard:.*point = bae80b0b, host = 1} + expect 0 0 Debug {^vmod_directors shard:.*point = f08ad325, host = 0} expect 0 0 VCL_Log {^-- remove s1_2 specifically$} - expect 0 0 Debug {^shard:.*point = 6e040182, host = 2} - expect 0 0 Debug {^shard:.*point = bae80b0b, host = 1} - expect 0 0 Debug {^shard:.*point = f08ad325, host = 0} + expect 0 0 Debug {^vmod_directors shard:.*point = 6e040182, host = 2} + expect 0 0 Debug {^vmod_directors shard:.*point = bae80b0b, host = 1} + expect 0 0 Debug {^vmod_directors shard:.*point = f08ad325, host = 0} expect 0 0 VCL_Log {^-- remove all instances of s1$} - expect 0 0 Debug {^shard:.*point = 6e040182, host = 0} + expect 0 0 Debug {^vmod_directors shard:.*point = 6e040182, host = 0} expect 0 0 VCL_Log {^-- re-add some - no 2nd director$} - expect 0 0 Debug {^shard:.*point = 3d1fe97, host = 3} - expect 0 0 Debug {^shard:.*point = a25a43b, host = 6} - expect 0 0 Debug {^shard:.*point = 2b20d9a2, host = 1} - expect 0 0 Debug {^shard:.*point = 6337e62c, host = 8} - expect 0 0 Debug {^shard:.*point = c9803f17, host = 5} - expect 0 0 Debug {^shard:.*point = d51dafe6, host = 0} - expect 0 0 Debug {^shard:.*point = eb74a7d5, host = 4} - expect 0 0 Debug {^shard:.*point = f493ce58, host = 2} - expect 0 0 Debug {^shard:.*point = fc1a5162, host = 7} + expect 0 0 Debug {^vmod_directors shard:.*point = 3d1fe97, host = 3} + expect 0 0 Debug {^vmod_directors shard:.*point = a25a43b, host = 6} + expect 0 0 Debug {^vmod_directors shard:.*point = 2b20d9a2, host = 1} + expect 0 0 Debug {^vmod_directors shard:.*point = 6337e62c, host = 8} + expect 0 0 Debug {^vmod_directors shard:.*point = c9803f17, host = 5} + expect 0 0 Debug {^vmod_directors shard:.*point = d51dafe6, host = 0} + expect 0 0 Debug {^vmod_directors shard:.*point = eb74a7d5, host = 4} + expect 0 0 Debug {^vmod_directors shard:.*point = f493ce58, host = 2} + expect 0 0 Debug {^vmod_directors shard:.*point = fc1a5162, host = 7} expect 0 0 VCL_Log {^-- remove second-last$} - expect 0 0 Debug {^shard:.*point = 3d1fe97, host = 3} - expect 0 0 Debug {^shard:.*point = a25a43b, host = 6} - expect 0 0 Debug {^shard:.*point = 2b20d9a2, host = 1} - expect 0 0 Debug {^shard:.*point = 6337e62c, host = 7} - expect 0 0 Debug {^shard:.*point = c9803f17, host = 5} - expect 0 0 Debug {^shard:.*point = d51dafe6, host = 0} - expect 0 0 Debug {^shard:.*point = eb74a7d5, host = 4} - expect 0 0 Debug {^shard:.*point = f493ce58, host = 2} + expect 0 0 Debug {^vmod_directors shard:.*point = 3d1fe97, host = 3} + expect 0 0 Debug {^vmod_directors shard:.*point = a25a43b, host = 6} + expect 0 0 Debug {^vmod_directors shard:.*point = 2b20d9a2, host = 1} + expect 0 0 Debug {^vmod_directors shard:.*point = 6337e62c, host = 7} + expect 0 0 Debug {^vmod_directors shard:.*point = c9803f17, host = 5} + expect 0 0 Debug {^vmod_directors shard:.*point = d51dafe6, host = 0} + expect 0 0 Debug {^vmod_directors shard:.*point = eb74a7d5, host = 4} + expect 0 0 Debug {^vmod_directors shard:.*point = f493ce58, host = 2} expect 0 0 VCL_Log {^-- remove last$} - expect 0 0 Debug {^shard:.*point = 3d1fe97, host = 3} - expect 0 0 Debug {^shard:.*point = a25a43b, host = 6} - expect 0 0 Debug {^shard:.*point = 2b20d9a2, host = 1} - expect 0 0 Debug {^shard:.*point = c9803f17, host = 5} - expect 0 0 Debug {^shard:.*point = d51dafe6, host = 0} - expect 0 0 Debug {^shard:.*point = eb74a7d5, host = 4} - expect 0 0 Debug {^shard:.*point = f493ce58, host = 2} + expect 0 0 Debug {^vmod_directors shard:.*point = 3d1fe97, host = 3} + expect 0 0 Debug {^vmod_directors shard:.*point = a25a43b, host = 6} + expect 0 0 Debug {^vmod_directors shard:.*point = 2b20d9a2, host = 1} + expect 0 0 Debug {^vmod_directors shard:.*point = c9803f17, host = 5} + expect 0 0 Debug {^vmod_directors shard:.*point = d51dafe6, host = 0} + expect 0 0 Debug {^vmod_directors shard:.*point = eb74a7d5, host = 4} + expect 0 0 Debug {^vmod_directors shard:.*point = f493ce58, host = 2} expect 0 0 VCL_Log {^-- END$} } -start diff --git a/bin/varnishtest/tests/d00016.vtc b/bin/varnishtest/tests/d00016.vtc index a6a8ed70a..0c093a6ac 100644 --- a/bin/varnishtest/tests/d00016.vtc +++ b/bin/varnishtest/tests/d00016.vtc @@ -158,13 +158,13 @@ varnish v1 -vcl+backend { logexpect l1 -v v1 -g raw -d 1 { expect * 1001 VCL_Log {^-- invalid replicas$} - expect 0 1001 Error {^shard vd: .reconfigure.. invalid replicas argument 0} + expect 0 1001 Error {^vmod_directors shard vd: .reconfigure.. invalid replicas argument 0} expect 0 1001 VCL_Log {^reconfigure failed} expect 0 1001 VCL_Log {^-- no changes - no debug output$} expect 0 1001 VCL_Log {^-- no backends$} - expect 0 1001 Error {^shard vd: .reconfigure.. no backends} + expect 0 1001 Error {^vmod_directors shard vd: .reconfigure.. no backends} expect 0 1001 VCL_Log {^reconfigure failed} expect 0 1001 VCL_Log {^-- one backend$} @@ -172,15 +172,15 @@ logexpect l1 -v v1 -g raw -d 1 { expect 0 1001 VCL_Log {^-- no change - no output$} expect 0 1001 VCL_Log {^-- change, clear, no backends$} - expect 0 1001 Error {^shard vd: .reconfigure.. no backends} + expect 0 1001 Error {^vmod_directors shard vd: .reconfigure.. no backends} expect 0 1001 VCL_Log {^reconfigure failed} expect 0 1001 VCL_Log {^-- duplicate add$} - expect 0 1001 Notice {^shard vd: backend s1 already exists - skipping$} + expect 0 1001 Notice {^vmod_directors shard vd: backend s1 already exists - skipping$} expect 0 1001 VCL_Log {^-- duplicate add with idents$} - expect 0 1001 Notice {^shard vd: backend s1 already exists - skipping} - expect 0 1001 Notice {^shard vd: backend s1/s1_1 already exists - skipping} - expect 0 1001 Notice {^shard vd: backend s1/s1_2 already exists - skipping} + expect 0 1001 Notice {^vmod_directors shard vd: backend s1 already exists - skipping} + expect 0 1001 Notice {^vmod_directors shard vd: backend s1/s1_1 already exists - skipping} + expect 0 1001 Notice {^vmod_directors shard vd: backend s1/s1_2 already exists - skipping} expect 0 1001 VCL_Log {^-- remove s1_2 specifically$} expect 0 1001 VCL_Log {^-- remove all instances of s1$} expect 0 1001 VCL_Log {^-- re-add some - no 2nd director$} @@ -190,41 +190,41 @@ logexpect l1 -v v1 -g raw -d 1 { } logexpect l2 -v v1 -g raw -d 1 { - expect * 0 Debug {^shard:.*point = f08ad325, host = 0} - expect 0 0 Debug {^shard:.*point = 6e040182, host = 1} - expect 0 0 Debug {^shard:.*point = f08ad325, host = 0} - expect 0 0 Debug {^shard:.*point = 6e040182, host = 3} - expect 0 0 Debug {^shard:.*point = 732c7bbe, host = 2} - expect 0 0 Debug {^shard:.*point = bae80b0b, host = 1} - expect 0 0 Debug {^shard:.*point = f08ad325, host = 0} - expect 0 0 Debug {^shard:.*point = 6e040182, host = 2} - expect 0 0 Debug {^shard:.*point = bae80b0b, host = 1} - expect 0 0 Debug {^shard:.*point = f08ad325, host = 0} - expect 0 0 Debug {^shard:.*point = 6e040182, host = 0} - expect 0 0 Debug {^shard:.*point = 3d1fe97, host = 3} - expect 0 0 Debug {^shard:.*point = a25a43b, host = 6} - expect 0 0 Debug {^shard:.*point = 2b20d9a2, host = 1} - expect 0 0 Debug {^shard:.*point = 6337e62c, host = 8} - expect 0 0 Debug {^shard:.*point = c9803f17, host = 5} - expect 0 0 Debug {^shard:.*point = d51dafe6, host = 0} - expect 0 0 Debug {^shard:.*point = eb74a7d5, host = 4} - expect 0 0 Debug {^shard:.*point = f493ce58, host = 2} - expect 0 0 Debug {^shard:.*point = fc1a5162, host = 7} - expect 0 0 Debug {^shard:.*point = 3d1fe97, host = 3} - expect 0 0 Debug {^shard:.*point = a25a43b, host = 6} - expect 0 0 Debug {^shard:.*point = 2b20d9a2, host = 1} - expect 0 0 Debug {^shard:.*point = 6337e62c, host = 7} - expect 0 0 Debug {^shard:.*point = c9803f17, host = 5} - expect 0 0 Debug {^shard:.*point = d51dafe6, host = 0} - expect 0 0 Debug {^shard:.*point = eb74a7d5, host = 4} - expect 0 0 Debug {^shard:.*point = f493ce58, host = 2} - expect 0 0 Debug {^shard:.*point = 3d1fe97, host = 3} - expect 0 0 Debug {^shard:.*point = a25a43b, host = 6} - expect 0 0 Debug {^shard:.*point = 2b20d9a2, host = 1} - expect 0 0 Debug {^shard:.*point = c9803f17, host = 5} - expect 0 0 Debug {^shard:.*point = d51dafe6, host = 0} - expect 0 0 Debug {^shard:.*point = eb74a7d5, host = 4} - expect 0 0 Debug {^shard:.*point = f493ce58, host = 2} + expect * 0 Debug {^vmod_directors shard:.*point = f08ad325, host = 0} + expect 0 0 Debug {^vmod_directors shard:.*point = 6e040182, host = 1} + expect 0 0 Debug {^vmod_directors shard:.*point = f08ad325, host = 0} + expect 0 0 Debug {^vmod_directors shard:.*point = 6e040182, host = 3} + expect 0 0 Debug {^vmod_directors shard:.*point = 732c7bbe, host = 2} + expect 0 0 Debug {^vmod_directors shard:.*point = bae80b0b, host = 1} + expect 0 0 Debug {^vmod_directors shard:.*point = f08ad325, host = 0} + expect 0 0 Debug {^vmod_directors shard:.*point = 6e040182, host = 2} + expect 0 0 Debug {^vmod_directors shard:.*point = bae80b0b, host = 1} + expect 0 0 Debug {^vmod_directors shard:.*point = f08ad325, host = 0} + expect 0 0 Debug {^vmod_directors shard:.*point = 6e040182, host = 0} + expect 0 0 Debug {^vmod_directors shard:.*point = 3d1fe97, host = 3} + expect 0 0 Debug {^vmod_directors shard:.*point = a25a43b, host = 6} + expect 0 0 Debug {^vmod_directors shard:.*point = 2b20d9a2, host = 1} + expect 0 0 Debug {^vmod_directors shard:.*point = 6337e62c, host = 8} + expect 0 0 Debug {^vmod_directors shard:.*point = c9803f17, host = 5} + expect 0 0 Debug {^vmod_directors shard:.*point = d51dafe6, host = 0} + expect 0 0 Debug {^vmod_directors shard:.*point = eb74a7d5, host = 4} + expect 0 0 Debug {^vmod_directors shard:.*point = f493ce58, host = 2} + expect 0 0 Debug {^vmod_directors shard:.*point = fc1a5162, host = 7} + expect 0 0 Debug {^vmod_directors shard:.*point = 3d1fe97, host = 3} + expect 0 0 Debug {^vmod_directors shard:.*point = a25a43b, host = 6} + expect 0 0 Debug {^vmod_directors shard:.*point = 2b20d9a2, host = 1} + expect 0 0 Debug {^vmod_directors shard:.*point = 6337e62c, host = 7} + expect 0 0 Debug {^vmod_directors shard:.*point = c9803f17, host = 5} + expect 0 0 Debug {^vmod_directors shard:.*point = d51dafe6, host = 0} + expect 0 0 Debug {^vmod_directors shard:.*point = eb74a7d5, host = 4} + expect 0 0 Debug {^vmod_directors shard:.*point = f493ce58, host = 2} + expect 0 0 Debug {^vmod_directors shard:.*point = 3d1fe97, host = 3} + expect 0 0 Debug {^vmod_directors shard:.*point = a25a43b, host = 6} + expect 0 0 Debug {^vmod_directors shard:.*point = 2b20d9a2, host = 1} + expect 0 0 Debug {^vmod_directors shard:.*point = c9803f17, host = 5} + expect 0 0 Debug {^vmod_directors shard:.*point = d51dafe6, host = 0} + expect 0 0 Debug {^vmod_directors shard:.*point = eb74a7d5, host = 4} + expect 0 0 Debug {^vmod_directors shard:.*point = f493ce58, host = 2} } logexpect l1 -start diff --git a/bin/varnishtest/tests/d00017.vtc b/bin/varnishtest/tests/d00017.vtc index 36793a386..eeabcde5f 100644 --- a/bin/varnishtest/tests/d00017.vtc +++ b/bin/varnishtest/tests/d00017.vtc @@ -56,91 +56,91 @@ varnish v1 -vcl+backend { # check that the hashcircle points are decremented by one compared to vslp # # generated from vslp test output -# perl -ne /tmp/regression.logexp +# perl -ne /tmp/regression.logexp logexpect l1 -v v1 -g raw -d 1 { expect 0 0 CLI "^Rd vcl.load" - expect 0 = Debug {^shard: hashcircle.* 0. = .point = 3b6b56a, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 1. = .point = 66986a7, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 2. = .point = 7e41e30, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 3. = .point = b749e7b, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 4. = .point = e543430, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 5. = .point = 10136c05, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 6. = .point = 102d847f, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 7. = .point = 1112f910, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 8. = .point = 1119a7c7, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 9. = .point = 22464ee9, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 10. = .point = 22b35675, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 11. = .point = 2363bebb, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 12. = .point = 259eeccf, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 13. = .point = 26f0c3e7, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 14. = .point = 28340f35, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 15. = .point = 285e8475, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 16. = .point = 28ec7a6f, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 17. = .point = 2da0e37b, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 18. = .point = 3392487a, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 19. = .point = 37597c4c, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 20. = .point = 4b1f5b22, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 21. = .point = 523723f2, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 22. = .point = 539234db, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 23. = .point = 564ca84f, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 24. = .point = 5d2df428, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 25. = .point = 5fa294ee, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 26. = .point = 60dded53, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 27. = .point = 6257bc27, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 28. = .point = 64014b25, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 29. = .point = 6bfd5a2d, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 30. = .point = 6e040182, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 31. = .point = 6e3819f7, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 32. = .point = 7232b381, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 33. = .point = 74c384ad, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 34. = .point = 83ce71ce, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 35. = .point = 888b6447, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 36. = .point = 8997c018, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 37. = .point = 8aa6b5b4, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 38. = .point = 8b47e6ac, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 39. = .point = 8bc8bc11, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 40. = .point = 8e2d3849, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 41. = .point = 8e7e012c, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 42. = .point = 99892987, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 43. = .point = 9a6f2f00, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 44. = .point = 9ef9125d, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 45. = .point = 9f33cd30, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 46. = .point = 9fc69b51, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 47. = .point = a19f99eb, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 48. = .point = a28b9595, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 49. = .point = a8afe9c4, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 50. = .point = ad923ad3, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 51. = .point = ae8946c6, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 52. = .point = b197e339, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 53. = .point = b3c305e6, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 54. = .point = b6bf43ea, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 55. = .point = b9004d3d, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 56. = .point = bbcc0bad, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 57. = .point = c2542a5d, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 58. = .point = c6c43fa7, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 59. = .point = c945958a, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 60. = .point = cbd9198a, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 61. = .point = d4c93105, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 62. = .point = d7de63b6, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 63. = .point = d937a7df, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 64. = .point = dac52229, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 65. = .point = db7840f0, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 66. = .point = dd5c6bef, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 67. = .point = dfd5333b, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 68. = .point = e991584c, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 69. = .point = ec8891c5, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 70. = .point = ef6b4ab5, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 71. = .point = f08ad325, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 72. = .point = f3325ba2, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 73. = .point = f6530dd1, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 74. = .point = fc28e8d2, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 0. = .point = 3b6b56a, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 1. = .point = 66986a7, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 2. = .point = 7e41e30, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 3. = .point = b749e7b, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 4. = .point = e543430, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 5. = .point = 10136c05, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 6. = .point = 102d847f, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 7. = .point = 1112f910, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 8. = .point = 1119a7c7, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 9. = .point = 22464ee9, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 10. = .point = 22b35675, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 11. = .point = 2363bebb, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 12. = .point = 259eeccf, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 13. = .point = 26f0c3e7, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 14. = .point = 28340f35, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 15. = .point = 285e8475, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 16. = .point = 28ec7a6f, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 17. = .point = 2da0e37b, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 18. = .point = 3392487a, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 19. = .point = 37597c4c, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 20. = .point = 4b1f5b22, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 21. = .point = 523723f2, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 22. = .point = 539234db, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 23. = .point = 564ca84f, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 24. = .point = 5d2df428, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 25. = .point = 5fa294ee, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 26. = .point = 60dded53, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 27. = .point = 6257bc27, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 28. = .point = 64014b25, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 29. = .point = 6bfd5a2d, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 30. = .point = 6e040182, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 31. = .point = 6e3819f7, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 32. = .point = 7232b381, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 33. = .point = 74c384ad, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 34. = .point = 83ce71ce, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 35. = .point = 888b6447, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 36. = .point = 8997c018, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 37. = .point = 8aa6b5b4, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 38. = .point = 8b47e6ac, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 39. = .point = 8bc8bc11, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 40. = .point = 8e2d3849, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 41. = .point = 8e7e012c, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 42. = .point = 99892987, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 43. = .point = 9a6f2f00, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 44. = .point = 9ef9125d, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 45. = .point = 9f33cd30, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 46. = .point = 9fc69b51, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 47. = .point = a19f99eb, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 48. = .point = a28b9595, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 49. = .point = a8afe9c4, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 50. = .point = ad923ad3, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 51. = .point = ae8946c6, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 52. = .point = b197e339, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 53. = .point = b3c305e6, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 54. = .point = b6bf43ea, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 55. = .point = b9004d3d, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 56. = .point = bbcc0bad, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 57. = .point = c2542a5d, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 58. = .point = c6c43fa7, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 59. = .point = c945958a, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 60. = .point = cbd9198a, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 61. = .point = d4c93105, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 62. = .point = d7de63b6, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 63. = .point = d937a7df, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 64. = .point = dac52229, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 65. = .point = db7840f0, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 66. = .point = dd5c6bef, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 67. = .point = dfd5333b, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 68. = .point = e991584c, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 69. = .point = ec8891c5, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 70. = .point = ef6b4ab5, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 71. = .point = f08ad325, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 72. = .point = f3325ba2, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 73. = .point = f6530dd1, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 74. = .point = fc28e8d2, host = 2.} expect 0 = CLI Loaded - expect * = Debug {^shard: lookup key 68b902f7 idx 29 host 0} - expect * = Debug {^shard: lookup key 39dc4613 idx 20 host 1} - expect * = Debug {^shard: lookup key c7793505 idx 59 host 2} + expect * = Debug {^vmod_directors shard: lookup key 68b902f7 idx 29 host 0} + expect * = Debug {^vmod_directors shard: lookup key 39dc4613 idx 20 host 1} + expect * = Debug {^vmod_directors shard: lookup key c7793505 idx 59 host 2} } -start client c1 { diff --git a/bin/varnishtest/tests/d00030.vtc b/bin/varnishtest/tests/d00030.vtc index 912314b03..c83ba5db9 100644 --- a/bin/varnishtest/tests/d00030.vtc +++ b/bin/varnishtest/tests/d00030.vtc @@ -24,14 +24,14 @@ varnish v1 -vcl { logexpect l1 -v v1 -g raw -d 1 { expect 0 0 CLI {^Rd vcl.load} - expect 0 0 Error {by=BLOB but no or empty key_blob - using key 0} + expect 0 0 Error {vmod_directors shard p: shard_param.set.. by=BLOB but no or empty key_blob - using key 0} } -start -wait logexpect l2 -v v1 -g raw { - expect * 1001 VCL_Error {shard .backend param invalid} + expect * 1001 VCL_Error {vmod_directors shard shard: .backend.key_blob. param invalid} } -start logexpect l3 -v v1 -g raw { - expect * 1003 VCL_Error {shard_param.set.. may only be used in vcl_init and in backend/pipe context} + expect * 1003 VCL_Error {vmod_directors shard p: shard_param.set.. may only be used in vcl_init and in backend/pipe context} } -start client c1 { @@ -159,7 +159,7 @@ varnish v1 -errvcl {invalid warmup argument 1.1} { } } -varnish v1 -errvcl {resolve=LAZY with other parameters can only be used in backend/pipe context} { +varnish v1 -errvcl {vmod_directors shard shard: .backend(resolve=LAZY) with other parameters can only be used in backend/pipe context} { import directors; import blob; @@ -172,7 +172,7 @@ varnish v1 -errvcl {resolve=LAZY with other parameters can only be used in backe } } -varnish v1 -errvcl {resolve=NOW can not be used in vcl_init} { +varnish v1 -errvcl {vmod_directors shard shard: .backend(resolve=NOW) can not be used in vcl_init{}/vcl_fini{}} { import directors; import blob; diff --git a/bin/varnishtest/tests/d00041.vtc b/bin/varnishtest/tests/d00041.vtc index 0f7d5b058..5d80e0c19 100644 --- a/bin/varnishtest/tests/d00041.vtc +++ b/bin/varnishtest/tests/d00041.vtc @@ -49,162 +49,162 @@ varnish v1 -vcl+backend { logexpect l1 -v v1 -g raw -d 1 { expect 0 0 CLI "^Rd vcl.load" - expect 0 = Debug {^shard: hashcircle.* 0. = .point = *238d0ef, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 1. = .point = *321c598, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 2. = .point = *3b6b56a, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 3. = .point = *408ec1e, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 4. = .point = *66986a7, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 5. = .point = *7e41e30, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 6. = .point = *b749e7b, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 7. = .point = *e543430, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 8. = .point = *10136c05, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 9. = .point = *102d847f, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 10. = .point = *1112f910, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 11. = .point = *1119a7c7, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 12. = .point = *14d95c44, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 13. = .point = *150fea1f, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 14. = .point = *1643ecb6, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 15. = .point = *189ff2f2, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 16. = .point = *19cfe9f3, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 17. = .point = *1e1c78c3, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 18. = .point = *1fe0dea0, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 19. = .point = *22464ee9, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 20. = .point = *22b35675, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 21. = .point = *2363bebb, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 22. = .point = *24f827bb, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 23. = .point = *259eeccf, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 24. = .point = *26f0c3e7, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 25. = .point = *271874d4, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 26. = .point = *28340f35, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 27. = .point = *285e8475, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 28. = .point = *28ec7a6f, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 29. = .point = *299c6298, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 30. = .point = *2aedc3f7, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 31. = .point = *2b031742, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 32. = .point = *2da0e37b, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 33. = .point = *310bd2ca, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 34. = .point = *31e5f2df, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 35. = .point = *32d6b3ed, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 36. = .point = *33047373, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 37. = .point = *3392487a, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 38. = .point = *37597c4c, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 39. = .point = *3f6b2b89, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 40. = .point = *43cf6426, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 41. = .point = *46a58f28, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 42. = .point = *4b1f5b22, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 43. = .point = *523723f2, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 44. = .point = *539234db, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 45. = .point = *564ca84f, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 46. = .point = *58501380, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 47. = .point = *58704432, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 48. = .point = *5b1bcbbe, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 49. = .point = *5d2df428, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 50. = .point = *5fa294ee, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 51. = .point = *606fd878, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 52. = .point = *60dded53, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 53. = .point = *616cdb68, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 54. = .point = *6257bc27, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 55. = .point = *64014b25, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 56. = .point = *6918f467, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 57. = .point = *6a08c380, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 58. = .point = *6bfd5a2d, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 59. = .point = *6c0b607a, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 60. = .point = *6c74d296, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 61. = .point = *6e040182, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 62. = .point = *6e3819f7, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 63. = .point = *720ec1a4, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 64. = .point = *7232b381, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 65. = .point = *74c384ad, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 66. = .point = *76d47350, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 67. = .point = *791eb3a3, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 68. = .point = *7a048f20, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 69. = .point = *7f874929, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 70. = .point = *83ce71ce, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 71. = .point = *888b6447, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 72. = .point = *8997c018, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 73. = .point = *89b7d09c, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 74. = .point = *8aa6b5b4, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 75. = .point = *8ae34bde, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 76. = .point = *8b382e03, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 77. = .point = *8b47e6ac, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 78. = .point = *8bc76115, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 79. = .point = *8bc8bc11, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 80. = .point = *8e2d3849, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 81. = .point = *8e7e012c, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 82. = .point = *8f5b4c63, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 83. = .point = *94a94162, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 84. = .point = *99892987, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 85. = .point = *9a6f2f00, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 86. = .point = *9b970b49, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 87. = .point = *9e09a3a7, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 88. = .point = *9ef9125d, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 89. = .point = *9f33cd30, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 90. = .point = *9fc69b51, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 91. = .point = *a19f99eb, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 92. = .point = *a28b9595, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 93. = .point = *a3582038, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 94. = .point = *a4b6a3b9, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 95. = .point = *a66da9cb, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 96. = .point = *a8657c76, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 97. = .point = *a8afe9c4, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 98. = .point = *aa488703, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 99. = .point = *ac7b4454, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 100. = .point = *ad923ad3, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 101. = .point = *ae8946c6, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 102. = .point = *b197e339, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 103. = .point = *b3c305e6, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 104. = .point = *b4dab004, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 105. = .point = *b6bf43ea, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 106. = .point = *b9004d3d, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 107. = .point = *b96b6455, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 108. = .point = *b9a0edb9, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 109. = .point = *b9ec6465, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 110. = .point = *bb8eed4d, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 111. = .point = *bbcc0bad, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 112. = .point = *bcfea141, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 113. = .point = *be300622, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 114. = .point = *bf514d68, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 115. = .point = *c1afc7d2, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 116. = .point = *c2542a5d, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 117. = .point = *c6c43fa7, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 118. = .point = *c945958a, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 119. = .point = *c9f304a4, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 120. = .point = *cb896aa8, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 121. = .point = *cbd9198a, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 122. = .point = *ccd61dad, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 123. = .point = *d07e4431, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 124. = .point = *d21fe35f, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 125. = .point = *d4c93105, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 126. = .point = *d570b815, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 127. = .point = *d7de63b6, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 128. = .point = *d8634aef, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 129. = .point = *d92d916d, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 130. = .point = *d937a7df, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 131. = .point = *dac52229, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 132. = .point = *db7840f0, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 133. = .point = *dd5c6bef, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 134. = .point = *dded5798, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 135. = .point = *dfd5333b, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 136. = .point = *e183345a, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 137. = .point = *e2c71c27, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 138. = .point = *e49bf9d8, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 139. = .point = *e72bc224, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 140. = .point = *e8b27f41, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 141. = .point = *e991584c, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 142. = .point = *ea201c5e, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 143. = .point = *ec8891c5, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 144. = .point = *edcc8dd9, host = 1.} - expect 0 = Debug {^shard: hashcircle.* 145. = .point = *ef6b4ab5, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 146. = .point = *f08ad325, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 147. = .point = *f3325ba2, host = 2.} - expect 0 = Debug {^shard: hashcircle.* 148. = .point = *f6530dd1, host = 0.} - expect 0 = Debug {^shard: hashcircle.* 149. = .point = *fc28e8d2, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 0. = .point = *238d0ef, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 1. = .point = *321c598, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 2. = .point = *3b6b56a, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 3. = .point = *408ec1e, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 4. = .point = *66986a7, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 5. = .point = *7e41e30, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 6. = .point = *b749e7b, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 7. = .point = *e543430, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 8. = .point = *10136c05, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 9. = .point = *102d847f, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 10. = .point = *1112f910, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 11. = .point = *1119a7c7, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 12. = .point = *14d95c44, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 13. = .point = *150fea1f, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 14. = .point = *1643ecb6, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 15. = .point = *189ff2f2, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 16. = .point = *19cfe9f3, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 17. = .point = *1e1c78c3, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 18. = .point = *1fe0dea0, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 19. = .point = *22464ee9, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 20. = .point = *22b35675, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 21. = .point = *2363bebb, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 22. = .point = *24f827bb, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 23. = .point = *259eeccf, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 24. = .point = *26f0c3e7, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 25. = .point = *271874d4, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 26. = .point = *28340f35, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 27. = .point = *285e8475, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 28. = .point = *28ec7a6f, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 29. = .point = *299c6298, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 30. = .point = *2aedc3f7, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 31. = .point = *2b031742, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 32. = .point = *2da0e37b, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 33. = .point = *310bd2ca, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 34. = .point = *31e5f2df, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 35. = .point = *32d6b3ed, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 36. = .point = *33047373, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 37. = .point = *3392487a, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 38. = .point = *37597c4c, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 39. = .point = *3f6b2b89, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 40. = .point = *43cf6426, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 41. = .point = *46a58f28, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 42. = .point = *4b1f5b22, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 43. = .point = *523723f2, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 44. = .point = *539234db, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 45. = .point = *564ca84f, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 46. = .point = *58501380, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 47. = .point = *58704432, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 48. = .point = *5b1bcbbe, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 49. = .point = *5d2df428, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 50. = .point = *5fa294ee, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 51. = .point = *606fd878, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 52. = .point = *60dded53, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 53. = .point = *616cdb68, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 54. = .point = *6257bc27, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 55. = .point = *64014b25, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 56. = .point = *6918f467, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 57. = .point = *6a08c380, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 58. = .point = *6bfd5a2d, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 59. = .point = *6c0b607a, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 60. = .point = *6c74d296, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 61. = .point = *6e040182, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 62. = .point = *6e3819f7, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 63. = .point = *720ec1a4, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 64. = .point = *7232b381, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 65. = .point = *74c384ad, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 66. = .point = *76d47350, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 67. = .point = *791eb3a3, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 68. = .point = *7a048f20, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 69. = .point = *7f874929, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 70. = .point = *83ce71ce, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 71. = .point = *888b6447, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 72. = .point = *8997c018, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 73. = .point = *89b7d09c, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 74. = .point = *8aa6b5b4, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 75. = .point = *8ae34bde, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 76. = .point = *8b382e03, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 77. = .point = *8b47e6ac, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 78. = .point = *8bc76115, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 79. = .point = *8bc8bc11, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 80. = .point = *8e2d3849, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 81. = .point = *8e7e012c, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 82. = .point = *8f5b4c63, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 83. = .point = *94a94162, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 84. = .point = *99892987, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 85. = .point = *9a6f2f00, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 86. = .point = *9b970b49, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 87. = .point = *9e09a3a7, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 88. = .point = *9ef9125d, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 89. = .point = *9f33cd30, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 90. = .point = *9fc69b51, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 91. = .point = *a19f99eb, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 92. = .point = *a28b9595, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 93. = .point = *a3582038, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 94. = .point = *a4b6a3b9, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 95. = .point = *a66da9cb, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 96. = .point = *a8657c76, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 97. = .point = *a8afe9c4, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 98. = .point = *aa488703, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 99. = .point = *ac7b4454, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 100. = .point = *ad923ad3, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 101. = .point = *ae8946c6, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 102. = .point = *b197e339, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 103. = .point = *b3c305e6, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 104. = .point = *b4dab004, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 105. = .point = *b6bf43ea, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 106. = .point = *b9004d3d, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 107. = .point = *b96b6455, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 108. = .point = *b9a0edb9, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 109. = .point = *b9ec6465, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 110. = .point = *bb8eed4d, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 111. = .point = *bbcc0bad, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 112. = .point = *bcfea141, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 113. = .point = *be300622, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 114. = .point = *bf514d68, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 115. = .point = *c1afc7d2, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 116. = .point = *c2542a5d, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 117. = .point = *c6c43fa7, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 118. = .point = *c945958a, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 119. = .point = *c9f304a4, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 120. = .point = *cb896aa8, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 121. = .point = *cbd9198a, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 122. = .point = *ccd61dad, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 123. = .point = *d07e4431, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 124. = .point = *d21fe35f, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 125. = .point = *d4c93105, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 126. = .point = *d570b815, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 127. = .point = *d7de63b6, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 128. = .point = *d8634aef, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 129. = .point = *d92d916d, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 130. = .point = *d937a7df, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 131. = .point = *dac52229, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 132. = .point = *db7840f0, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 133. = .point = *dd5c6bef, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 134. = .point = *dded5798, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 135. = .point = *dfd5333b, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 136. = .point = *e183345a, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 137. = .point = *e2c71c27, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 138. = .point = *e49bf9d8, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 139. = .point = *e72bc224, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 140. = .point = *e8b27f41, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 141. = .point = *e991584c, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 142. = .point = *ea201c5e, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 143. = .point = *ec8891c5, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 144. = .point = *edcc8dd9, host = 1.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 145. = .point = *ef6b4ab5, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 146. = .point = *f08ad325, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 147. = .point = *f3325ba2, host = 2.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 148. = .point = *f6530dd1, host = 0.} + expect 0 = Debug {^vmod_directors shard: hashcircle.* 149. = .point = *fc28e8d2, host = 2.} expect 0 = CLI Loaded - expect * = Debug {^shard: lookup key 564ca84f idx 45 host 0} - expect * = Debug {^shard: lookup key 19cfe9f3 idx 16 host 1} - expect * = Debug {^shard: lookup key 46a58f28 idx 41 host 2} + expect * = Debug {^vmod_directors shard: lookup key 564ca84f idx 45 host 0} + expect * = Debug {^vmod_directors shard: lookup key 19cfe9f3 idx 16 host 1} + expect * = Debug {^vmod_directors shard: lookup key 46a58f28 idx 41 host 2} } -start client c1 { diff --git a/doc/changes.rst b/doc/changes.rst index 7c4bbba93..59a6dc52a 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -48,6 +48,9 @@ Varnish Cache Next (2021-03-15) * Shard director ``Error`` log messages with ``(notice)`` have been turned into ``Notice`` log messages. +* All shard ``Error`` and ``Notice`` messages now use the unified + prefix ``vmod_directors shard %s``. + ================================ Varnish Cache 6.5.1 (2020-09-25) ================================ diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c index b61cb9392..a6e87fd5c 100644 --- a/lib/libvmod_directors/shard_cfg.c +++ b/lib/libvmod_directors/shard_cfg.c @@ -119,7 +119,7 @@ shard_change_get(VRT_CTX, struct sharddir * const shardd) change = WS_Alloc(ctx->ws, sizeof(*change)); if (change == NULL) { - VRT_fail(ctx, "could not get workspace"); + shard_fail(ctx, shardd->name, "%s", "could not get workspace"); return (NULL); } @@ -151,7 +151,8 @@ shard_change_task_add(VRT_CTX, struct shard_change *change, task = WS_Alloc(ctx->ws, sizeof(*task)); if (task == NULL) { - VRT_fail(ctx, "could not get workspace for task"); + shard_fail(ctx, change->shardd->name, "%s", + "could not get workspace for task"); return (NULL); } INIT_OBJ(task, SHARD_CHANGE_TASK_MAGIC); @@ -179,7 +180,8 @@ shard_change_task_backend(VRT_CTX, struct sharddir *shardd, b = WS_Alloc(ctx->ws, sizeof(*b)); if (b == NULL) { - VRT_fail(ctx, "could not get workspace for change"); + shard_fail(ctx, change->shardd->name, "%s", + "could not get workspace for task"); return (NULL); } @@ -598,8 +600,8 @@ shardcfg_apply_change(struct vsl_log *vsl, struct sharddir *shardd, const char * const ident = b->ident; - sharddir_err(vsl, SLT_Notice, "shard %s: backend %s%s%s " - "already exists - skipping", shardd->name, + shard_notice(vsl, shardd->name, + "backend %s%s%s already exists - skipping", VRT_BACKEND_string(b->backend), ident ? "/" : "", ident ? ident : ""); @@ -642,8 +644,8 @@ change_reconfigure(struct shard_change *change, VCL_INT replicas) shardd->hashcircle = NULL; if (shardd->n_backend == 0) { - sharddir_err(change->vsl, SLT_Error, "shard %s: .reconfigure() " - "no backends", shardd->name); + shard_err0(change->vsl, shardd->name, + ".reconfigure() no backends"); sharddir_unlock(shardd); return (0); } @@ -660,9 +662,8 @@ shardcfg_reconfigure(VRT_CTX, struct sharddir *shardd, VCL_INT replicas) CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); if (replicas <= 0) { - sharddir_err(ctx->vsl, SLT_Error, - "shard %s: .reconfigure() invalid replicas argument %ld", - shardd->name, replicas); + shard_err(ctx->vsl, shardd->name, + ".reconfigure() invalid replicas argument %ld", replicas); return (0); } diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c index 1d4419b66..00b61aa98 100644 --- a/lib/libvmod_directors/shard_dir.c +++ b/lib/libvmod_directors/shard_dir.c @@ -78,7 +78,7 @@ sharddir_debug(struct sharddir *shardd, const uint32_t flags) } void -sharddir_err(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...) +sharddir_log(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...) { va_list ap; @@ -242,11 +242,11 @@ validate_alt(VRT_CTX, const struct sharddir *shardd, VCL_INT *alt) const VCL_INT alt_max = shardd->n_backend - 1; if (*alt < 0) { - shard_err(ctx, shardd, + shard_err(ctx->vsl, shardd->name, "invalid negative parameter alt=%ld, set to 0", *alt); *alt = 0; } else if (*alt > alt_max) { - shard_err(ctx, shardd, + shard_err(ctx->vsl, shardd->name, "parameter alt=%ld limited to %ld", *alt, alt_max); *alt = alt_max; } @@ -408,7 +408,7 @@ sharddir_pick_be(VRT_CTX, struct sharddir *shardd, uint32_t key, VCL_INT alt, sharddir_rdlock(shardd); if (shardd->n_backend == 0) { - shard_err0(ctx, shardd, "no backends"); + shard_err0(ctx->vsl, shardd->name, "no backends"); sharddir_unlock(shardd); return (NULL); } diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h index 27c0aee60..4ec22a1d2 100644 --- a/lib/libvmod_directors/shard_dir.h +++ b/lib/libvmod_directors/shard_dir.h @@ -89,23 +89,25 @@ sharddir_backend(const struct sharddir *shardd, unsigned id) #define SHDBG(flag, shardd, ...) \ do { \ if ((shardd)->debug_flags & (flag)) \ - VSL(SLT_Debug, 0, "shard: " __VA_ARGS__); \ + VSL(SLT_Debug, 0, "vmod_directors shard: " \ + __VA_ARGS__); \ } while (0) -#define shard_err(ctx, shardd, fmt, ...) \ - do { \ - sharddir_err((ctx)->vsl, SLT_Error, "shard %s: " fmt, \ - (shardd)->name, __VA_ARGS__); \ - } while (0) +#define shard_log(vsl, tag, name, fmt, ...) \ + sharddir_log(vsl, tag, "vmod_directors shard %s: " fmt, \ + name, __VA_ARGS__) -#define shard_err0(ctx, shardd, msg) \ - do { \ - sharddir_err((ctx)->vsl, SLT_Error, "shard %s: %s", \ - (shardd)->name, (msg)); \ - } while (0) +#define shard_fail(ctx, name, fmt, ...) \ + VRT_fail(ctx, "vmod_directors shard %s: " fmt, name, __VA_ARGS__) +#define shard_err(vsl, name, fmt, ...) \ + shard_log(vsl, SLT_Error, name, fmt, __VA_ARGS__) +#define shard_err0(vsl, name, msg) \ + shard_log(vsl, SLT_Error, name, "%s", msg) +#define shard_notice(vsl, name, fmt, ...) \ + shard_log(vsl, SLT_Notice, name, fmt, __VA_ARGS__) void sharddir_debug(struct sharddir *shardd, const uint32_t flags); -void sharddir_err(struct vsl_log *, enum VSL_tag_e tag, const char *fmt, ...); +void sharddir_log(struct vsl_log *, enum VSL_tag_e tag, const char *fmt, ...); void sharddir_new(struct sharddir **sharddp, const char *vcl_name, const struct vmod_directors_shard_param *param); void sharddir_set_param(struct sharddir *shardd, diff --git a/lib/libvmod_directors/vmod_directors.vcc b/lib/libvmod_directors/vmod_directors.vcc index 161841531..ddcbf6a3a 100644 --- a/lib/libvmod_directors/vmod_directors.vcc +++ b/lib/libvmod_directors/vmod_directors.vcc @@ -344,12 +344,12 @@ Error Reporting Failing methods should report errors to VSL with the Error tag, so when configuring the shard director, you are advised to check:: - varnishlog -I Error:^shard + varnishlog -I Error:^vmod_directors.shard Additional information may be provided as Notices, which can be checked using - varnishlog -I Notice:^shard + varnishlog -I Notice:^vmod_directors.shard $Method VOID .set_warmup(REAL probability=0.0) diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 42dbcaf85..5e0137bbc 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -266,7 +266,7 @@ vmod_shard_set_warmup(VRT_CTX, struct vmod_directors_shard *vshard, { CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); if (probability < 0 || probability >= 1) { - shard_err(ctx, vshard->shardd, + shard_notice(ctx->vsl, vshard->shardd->name, ".set_warmup(%f) ignored", probability); return; } @@ -297,7 +297,8 @@ vmod_shard_associate(VRT_CTX, ppt = shard_param_blob(b); if (ppt == NULL) { - VRT_fail(ctx, "shard .associate param invalid"); + shard_fail(ctx, vshard->shardd->name, "%s", + "shard .associate param invalid"); return; } @@ -313,8 +314,8 @@ vmod_shard_add_backend(VRT_CTX, struct vmod_directors_shard *vshard, CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); if (args->backend == NULL) { - VRT_fail(ctx, "%s: NULL backend cannot be added", - vshard->shardd->name); + shard_fail(ctx, vshard->shardd->name, "%s", + "NULL backend cannot be added"); return (0); } @@ -322,7 +323,7 @@ vmod_shard_add_backend(VRT_CTX, struct vmod_directors_shard *vshard, if (args->weight >= 1) weight = args->weight; else - shard_err(ctx, vshard->shardd, + shard_notice(ctx->vsl, vshard->shardd->name, ".add_backend(weight=%f) ignored", args->weight); } @@ -342,9 +343,8 @@ vmod_shard_remove_backend(VRT_CTX, struct vmod_directors_shard *vshard, CHECK_OBJ_NOTNULL(vshard, VMOD_SHARD_SHARD_MAGIC); if (be == NULL && ident == NULL) { - VRT_fail(ctx, "%s.backend_remove(): " - "either backend or ident are required", - vshard->shardd->name); + shard_fail(ctx, vshard->shardd->name, "%s", + ".backend_remove(): either backend or ident are required"); return (0); } @@ -499,7 +499,7 @@ shard_param_set_mask(const struct VARGS(shard_param_set) * const a) */ static struct vmod_directors_shard_param * shard_param_args(VRT_CTX, - struct vmod_directors_shard_param *p, const char *who, + struct vmod_directors_shard_param *p, const char *func, uint32_t args, VCL_ENUM by_s, VCL_INT key_int, VCL_BLOB key_blob, VCL_INT alt, VCL_REAL warmup, VCL_BOOL rampup, VCL_ENUM healthy_s) { @@ -516,16 +516,15 @@ shard_param_args(VRT_CTX, /* by_s / key_int / key_blob */ if (by_s == VENUM(KEY)) { if ((args & arg_key) == 0) { - VRT_fail(ctx, "%s %s: " - "missing key argument with by=%s", - who, p->vcl_name, by_s); + shard_fail(ctx, p->vcl_name, + "%s missing key argument with by=%s", + func, by_s); return (NULL); } if (key_int < 0 || key_int > UINT32_MAX) { - VRT_fail(ctx, "%s %s: " - "invalid key argument %jd with by=%s", - who, p->vcl_name, - (intmax_t)key_int, by_s); + shard_fail(ctx, p->vcl_name, + "%s invalid key argument %jd with by=%s", + func, (intmax_t)key_int, by_s); return (NULL); } assert(key_int >= 0); @@ -533,26 +532,24 @@ shard_param_args(VRT_CTX, p->key = (uint32_t)key_int; } else if (by_s == VENUM(BLOB)) { if ((args & arg_key_blob) == 0) { - VRT_fail(ctx, "%s %s: " - "missing key_blob argument with by=%s", - who, p->vcl_name, by_s); + shard_fail(ctx, p->vcl_name, + "%s missing key_blob argument with by=%s", + func, by_s); return (NULL); } if (key_blob == NULL || key_blob->len == 0 || key_blob->blob == NULL) { - sharddir_err(ctx->vsl, SLT_Error, "%s %s: " - "by=BLOB but no or empty key_blob " - "- using key 0", - who, p->vcl_name); + shard_err(ctx->vsl, p->vcl_name, + "%s by=BLOB but no or empty key_blob - using key 0", + func); p->key = 0; } else p->key = shard_blob_key(key_blob); } else if (by_s == VENUM(HASH) || by_s == VENUM(URL)) { if (args & (arg_key|arg_key_blob)) { - VRT_fail(ctx, "%s %s: " - "key and key_blob arguments are " - "invalid with by=%s", - who, p->vcl_name, by_s); + shard_fail(ctx, p->vcl_name, + "%s key and key_blob arguments are " + "invalid with by=%s", func, by_s); return (NULL); } } else { @@ -562,9 +559,9 @@ shard_param_args(VRT_CTX, if (args & arg_alt) { if (alt < 0) { - VRT_fail(ctx, "%s %s: " - "invalid alt argument %jd", - who, p->vcl_name, (intmax_t)alt); + shard_fail(ctx, p->vcl_name, + "%s invalid alt argument %jd", + func, (intmax_t)alt); return (NULL); } p->alt = alt; @@ -572,9 +569,9 @@ shard_param_args(VRT_CTX, if (args & arg_warmup) { if ((warmup < 0 && warmup != -1) || warmup > 1) { - VRT_fail(ctx, "%s %s: " - "invalid warmup argument %f", - who, p->vcl_name, warmup); + shard_fail(ctx, p->vcl_name, + "%s invalid warmup argument %f", + func, warmup); return (NULL); } p->warmup = warmup; @@ -618,9 +615,10 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, } if ((ctx->method & SHARD_VCL_TASK_BEREQ) == 0) { - VRT_fail(ctx, "shard .backend resolve=LAZY with other " - "parameters can only be used in backend/pipe " - "context"); + shard_fail(ctx, vshard->shardd->name, "%s", + ".backend(resolve=LAZY) with other " + "parameters can only be used in backend/pipe " + "context"); return (NULL); } @@ -633,9 +631,9 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, pp->vcl_name = vshard->shardd->name; } else if (resolve == VENUM(NOW)) { if (ctx->method & VCL_MET_TASK_H) { - VRT_fail(ctx, - "shard .backend resolve=NOW can not be " - "used in vcl_init{}/vcl_fini{}"); + shard_fail(ctx, vshard->shardd->name, "%s", + ".backend(resolve=NOW) can not be " + "used in vcl_init{}/vcl_fini{}"); return (NULL); } pp = shard_param_stack(&pstk, vshard->shardd->param, @@ -648,7 +646,8 @@ vmod_shard_backend(VRT_CTX, struct vmod_directors_shard *vshard, if (args & arg_param) { ppt = shard_param_blob(a->param); if (ppt == NULL) { - VRT_fail(ctx, "shard .backend param invalid"); + shard_fail(ctx, vshard->shardd->name, "%s", + ".backend(key_blob) param invalid"); return (NULL); } pp->defaults = ppt; @@ -885,7 +884,7 @@ shard_param_task(VRT_CTX, const void *id, task = VRT_priv_task(ctx, task_id); if (task == NULL) { - VRT_fail(ctx, "no priv_task"); + shard_fail(ctx, pa->vcl_name, "%s", "no priv_task"); return (NULL); } @@ -897,7 +896,7 @@ shard_param_task(VRT_CTX, const void *id, p = WS_Alloc(ctx->ws, sizeof *p); if (p == NULL) { - VRT_fail(ctx, "shard_param_task WS_Alloc failed"); + shard_fail(ctx, pa->vcl_name, "%s", "WS_Alloc failed"); return (NULL); } task->priv = p; @@ -921,8 +920,8 @@ shard_param_prep(VRT_CTX, struct vmod_directors_shard_param *p, CHECK_OBJ_NOTNULL(p, VMOD_SHARD_SHARD_PARAM_MAGIC); if (ctx->method & SHARD_VCL_TASK_REQ) { - VRT_fail(ctx, "%s may only be used " - "in vcl_init and in backend/pipe context", who); + shard_fail(ctx, p->vcl_name, "%s may only be used " + "in vcl_init and in backend/pipe context", who); return (NULL); } else if (ctx->method & SHARD_VCL_TASK_BEREQ) p = shard_param_task(ctx, p, p); From nils.goroll at uplex.de Sat Oct 10 11:02:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 10 Oct 2020 11:02:06 +0000 (UTC) Subject: [master] f7cc84a7c Fix request body reference count assertion Message-ID: <20201010110206.B5DFAADF1B@lists.varnish-cache.org> commit f7cc84a7ce699e068997d77f35118925fef8481d Author: Nils Goroll Date: Sat Oct 10 12:58:17 2020 +0200 Fix request body reference count assertion Test case by Reza, thank you Fixes #3433 Closes #3434 diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c index a3a771163..ecb78d769 100644 --- a/bin/varnishd/cache/cache_req_body.c +++ b/bin/varnishd/cache/cache_req_body.c @@ -268,8 +268,9 @@ VRB_Free(struct req *req) r = HSH_DerefObjCore(req->wrk, &req->body_oc, 0); - // a busyobj may have gained a reference - assert (r == 0 || r == 1); + + // each busyobj may have gained a reference + assert (r <= req->restarts + 1); } /*---------------------------------------------------------------------- diff --git a/bin/varnishtest/tests/r03433.vtc b/bin/varnishtest/tests/r03433.vtc new file mode 100644 index 000000000..0ebbb5d9b --- /dev/null +++ b/bin/varnishtest/tests/r03433.vtc @@ -0,0 +1,41 @@ +varnishtest "req.body and restarts" + +server s1 -repeat 4 { + rxreq + txresp -bodylen 1000 +} -start + +varnish v1 -vcl+backend { + import std; + import vtc; + + sub vcl_recv { + std.cache_req_body(1MB); + return (hash); + } + + sub vcl_backend_response { + set beresp.ttl = 0.1s; + } + + sub vcl_deliver { + if (!req.restarts) { + set req.url = "/2"; + return (restart); + } + } +} -start + +client c1 { + txreq -req PUT -url /1 -bodylen 250000 + rxresp + expect resp.status == 200 + + delay 0.2 + + txreq -req PUT -url /1 -bodylen 250000 + rxresp + expect resp.status == 200 +} -run + +delay 0.2 From nils.goroll at uplex.de Sat Oct 10 11:09:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 10 Oct 2020 11:09:06 +0000 (UTC) Subject: [master] 5367be3f6 HSH_DerefObjCore signedness fix Message-ID: <20201010110906.5602BAF335@lists.varnish-cache.org> commit 5367be3f62f45380ef7b5348eefb88a9c1507318 Author: Nils Goroll Date: Sat Oct 10 13:05:33 2020 +0200 HSH_DerefObjCore signedness fix noticed by Dridi here: https://github.com/varnishcache/varnish-cache/pull/3434/commits/f9f40b991f301fb96d2e2d0d52b86c3d4b1f9f7f#r502302785 The reference count returned already is an unsigned value diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 9e80efcb1..c9eafa92b 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -975,7 +975,7 @@ HSH_DerefBoc(struct worker *wrk, struct objcore *oc) * Returns zero if target was destroyed. */ -int +unsigned HSH_DerefObjCore(struct worker *wrk, struct objcore **ocp, int rushmax) { struct objcore *oc; diff --git a/bin/varnishd/cache/cache_objhead.h b/bin/varnishd/cache/cache_objhead.h index bc1782379..d9a015417 100644 --- a/bin/varnishd/cache/cache_objhead.h +++ b/bin/varnishd/cache/cache_objhead.h @@ -66,7 +66,7 @@ struct boc *HSH_RefBoc(const struct objcore *); void HSH_DerefBoc(struct worker *wrk, struct objcore *); void HSH_DeleteObjHead(const struct worker *, struct objhead *); -int HSH_DerefObjCore(struct worker *, struct objcore **, int rushmax); +unsigned HSH_DerefObjCore(struct worker *, struct objcore **, int rushmax); #define HSH_RUSH_POLICY -1 enum lookup_e HSH_Lookup(struct req *, struct objcore **, struct objcore **); diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c index ecb78d769..fdec1bd4c 100644 --- a/bin/varnishd/cache/cache_req_body.c +++ b/bin/varnishd/cache/cache_req_body.c @@ -259,7 +259,7 @@ VRB_Ignore(struct req *req) void VRB_Free(struct req *req) { - int r; + unsigned r; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -268,7 +268,6 @@ VRB_Free(struct req *req) r = HSH_DerefObjCore(req->wrk, &req->body_oc, 0); - // each busyobj may have gained a reference assert (r <= req->restarts + 1); } From nils.goroll at uplex.de Sat Oct 10 16:41:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 10 Oct 2020 16:41:06 +0000 (UTC) Subject: [master] 65e7c1f43 try to stabilize vtc Message-ID: <20201010164106.885355141@lists.varnish-cache.org> commit 65e7c1f4338f29b679f63014b43f7e35152db5f8 Author: Nils Goroll Date: Sat Oct 10 18:38:58 2020 +0200 try to stabilize vtc seen in vtest: **** v1 vsl| 1004 Begin b bereq 1003 fetch **** v1 vsl| 1004 VCL_use b vcl1 **** v1 vsl| 1004 Timestamp b Start: 1602334643.946723 0.000000 0.000000 **** v1 vsl| 1004 BereqMethod b PUT **** v1 vsl| 1004 BereqURL b /2 **** v1 vsl| 1004 BereqProtocol b HTTP/1.1 **** v1 vsl| 1004 BereqHeader b Host: 127.0.0.1 **** v1 vsl| 1004 BereqHeader b Content-Length: 250000 **** v1 vsl| 1004 BereqHeader b X-Forwarded-For: 127.0.0.1 **** v1 vsl| 1004 BereqMethod b GET **** v1 vsl| 1004 BereqHeader b Accept-Encoding: gzip **** v1 vsl| 1004 BereqHeader b X-Varnish: 1004 **** v1 vsl| 1004 VCL_call b BACKEND_FETCH **** v1 vsl| 1004 VCL_return b fetch **** v1 vsl| 1004 BackendOpen b 26 s1 127.0.0.1 36579 127.0.0.1 60878 connect **** v1 vsl| 1004 Timestamp b Bereq: 1602334643.956883 0.010159 0.010159 **** v1 vsl| 1004 FetchError b HTC eof (-1) Ref #3433 diff --git a/bin/varnishtest/tests/r03433.vtc b/bin/varnishtest/tests/r03433.vtc index 0ebbb5d9b..d361caada 100644 --- a/bin/varnishtest/tests/r03433.vtc +++ b/bin/varnishtest/tests/r03433.vtc @@ -1,6 +1,11 @@ varnishtest "req.body and restarts" -server s1 -repeat 4 { +server s1 -repeat 2 { + rxreq + txresp -bodylen 1000 +} -start + +server s2 -repeat 2 { rxreq txresp -bodylen 1000 } -start @@ -14,6 +19,14 @@ varnish v1 -vcl+backend { return (hash); } + sub vcl_backend_fetch { + if (bereq.url == "/1") { + set bereq.backend = s1; + } else { + set bereq.backend = s2; + } + } + sub vcl_backend_response { set beresp.ttl = 0.1s; } From dridi.boukelmoune at gmail.com Sat Oct 10 18:31:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Sat, 10 Oct 2020 18:31:06 +0000 (UTC) Subject: [master] 303ff55ab vsl: Typo Message-ID: <20201010183106.341507FB7@lists.varnish-cache.org> commit 303ff55ab37e3c6fe163e8c3413d0329f844e2e4 Author: Dridi Boukelmoune Date: Sat Oct 10 20:28:09 2020 +0200 vsl: Typo diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 49dd6ec95..9a14d0d58 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -683,7 +683,7 @@ SLTM(VCL_use, 0, "VCL in use", ) SLTM(Notice, 0, "Informational messages about request handling", - "Informational log messages on events occured during request" + "Informational log messages on events occurred during request" " handling.\n\n" "The format is::\n\n" "\t%s: %s\n" From dridi.boukelmoune at gmail.com Sun Oct 11 12:44:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Sun, 11 Oct 2020 12:44:06 +0000 (UTC) Subject: [master] d2fd7b60e directors: Missing shard error handling Message-ID: <20201011124406.D9478AD038@lists.varnish-cache.org> commit d2fd7b60e80bc5fdea7c409f97090a6d55d925c0 Author: Dridi Boukelmoune Date: Sun Oct 11 14:42:20 2020 +0200 directors: Missing shard error handling Spotted by Coverity. diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c index a6e87fd5c..312e9c722 100644 --- a/lib/libvmod_directors/shard_cfg.c +++ b/lib/libvmod_directors/shard_cfg.c @@ -109,6 +109,10 @@ shard_change_get(VRT_CTX, struct sharddir * const shardd) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); task = VRT_priv_task(ctx, id); + if (task == NULL) { + shard_fail(ctx, shardd->name, "%s", "no priv_task"); + return (NULL); + } if (task->priv != NULL) { CAST_OBJ_NOTNULL(change, task->priv, SHARD_CHANGE_MAGIC); From nils.goroll at uplex.de Sun Oct 11 13:21:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 11 Oct 2020 13:21:06 +0000 (UTC) Subject: [master] 76c9e6657 shard: one more log message polish iteration Message-ID: <20201011132106.55FC5AF032@lists.varnish-cache.org> commit 76c9e6657c92b471819daef389475ec047424c91 Author: Nils Goroll Date: Sun Oct 11 15:14:26 2020 +0200 shard: one more log message polish iteration Dridi pointed out that there was still a colon missing to conform with the SLT_Notice documentation in include/tbl/vsl_tags.h diff --git a/bin/varnishtest/tests/d00015.vtc b/bin/varnishtest/tests/d00015.vtc index d87bd4498..c283732be 100644 --- a/bin/varnishtest/tests/d00015.vtc +++ b/bin/varnishtest/tests/d00015.vtc @@ -199,75 +199,75 @@ logexpect l1 -v v1 -g raw -d 1 { expect 0 0 CLI {^Rd vcl.load} expect 0 0 VCL_Log {^-- invalid replicas$} - expect 0 0 Error {^vmod_directors shard vd: .reconfigure.. invalid replicas argument 0} + expect 0 0 Error {^vmod_directors: shard vd: .reconfigure.. invalid replicas argument 0} expect 0 0 VCL_Log {^reconfigure failed} expect 0 0 VCL_Log {^-- no changes - no debug output$} expect 0 0 VCL_Log {^-- no backends$} - expect 0 0 Error {^vmod_directors shard vd: .reconfigure.. no backends} + expect 0 0 Error {^vmod_directors: shard vd: .reconfigure.. no backends} expect 0 0 VCL_Log {^reconfigure failed} expect 0 0 VCL_Log {^-- one backend$} - expect 0 0 Debug {^vmod_directors shard:.*point = f08ad325, host = 0} + expect 0 0 Debug {^vmod_directors: shard:.*point = f08ad325, host = 0} expect 0 0 VCL_Log {^-- no change - no output$} expect 0 0 VCL_Log {^-- change, clear, no backends$} - expect 0 0 Error {^vmod_directors shard vd: .reconfigure.. no backends} + expect 0 0 Error {^vmod_directors: shard vd: .reconfigure.. no backends} expect 0 0 VCL_Log {^reconfigure failed} expect 0 0 VCL_Log {^-- duplicate add$} - expect 0 0 Notice {^vmod_directors shard vd: backend s1 already exists - skipping$} - expect 0 0 Debug {^vmod_directors shard:.*point = 6e040182, host = 1} - expect 0 0 Debug {^vmod_directors shard:.*point = f08ad325, host = 0} + expect 0 0 Notice {^vmod_directors: shard vd: backend s1 already exists - skipping$} + expect 0 0 Debug {^vmod_directors: shard:.*point = 6e040182, host = 1} + expect 0 0 Debug {^vmod_directors: shard:.*point = f08ad325, host = 0} expect 0 0 VCL_Log {^-- duplicate add with idents$} - expect 0 0 Notice {^vmod_directors shard vd: backend s1 already exists - skipping} - expect 0 0 Notice {^vmod_directors shard vd: backend s1/s1_1 already exists - skipping} - expect 0 0 Notice {^vmod_directors shard vd: backend s1/s1_2 already exists - skipping} - expect 0 0 Debug {^vmod_directors shard:.*point = 6e040182, host = 3} - expect 0 0 Debug {^vmod_directors shard:.*point = 732c7bbe, host = 2} - expect 0 0 Debug {^vmod_directors shard:.*point = bae80b0b, host = 1} - expect 0 0 Debug {^vmod_directors shard:.*point = f08ad325, host = 0} + expect 0 0 Notice {^vmod_directors: shard vd: backend s1 already exists - skipping} + expect 0 0 Notice {^vmod_directors: shard vd: backend s1/s1_1 already exists - skipping} + expect 0 0 Notice {^vmod_directors: shard vd: backend s1/s1_2 already exists - skipping} + expect 0 0 Debug {^vmod_directors: shard:.*point = 6e040182, host = 3} + expect 0 0 Debug {^vmod_directors: shard:.*point = 732c7bbe, host = 2} + expect 0 0 Debug {^vmod_directors: shard:.*point = bae80b0b, host = 1} + expect 0 0 Debug {^vmod_directors: shard:.*point = f08ad325, host = 0} expect 0 0 VCL_Log {^-- remove s1_2 specifically$} - expect 0 0 Debug {^vmod_directors shard:.*point = 6e040182, host = 2} - expect 0 0 Debug {^vmod_directors shard:.*point = bae80b0b, host = 1} - expect 0 0 Debug {^vmod_directors shard:.*point = f08ad325, host = 0} + expect 0 0 Debug {^vmod_directors: shard:.*point = 6e040182, host = 2} + expect 0 0 Debug {^vmod_directors: shard:.*point = bae80b0b, host = 1} + expect 0 0 Debug {^vmod_directors: shard:.*point = f08ad325, host = 0} expect 0 0 VCL_Log {^-- remove all instances of s1$} - expect 0 0 Debug {^vmod_directors shard:.*point = 6e040182, host = 0} + expect 0 0 Debug {^vmod_directors: shard:.*point = 6e040182, host = 0} expect 0 0 VCL_Log {^-- re-add some - no 2nd director$} - expect 0 0 Debug {^vmod_directors shard:.*point = 3d1fe97, host = 3} - expect 0 0 Debug {^vmod_directors shard:.*point = a25a43b, host = 6} - expect 0 0 Debug {^vmod_directors shard:.*point = 2b20d9a2, host = 1} - expect 0 0 Debug {^vmod_directors shard:.*point = 6337e62c, host = 8} - expect 0 0 Debug {^vmod_directors shard:.*point = c9803f17, host = 5} - expect 0 0 Debug {^vmod_directors shard:.*point = d51dafe6, host = 0} - expect 0 0 Debug {^vmod_directors shard:.*point = eb74a7d5, host = 4} - expect 0 0 Debug {^vmod_directors shard:.*point = f493ce58, host = 2} - expect 0 0 Debug {^vmod_directors shard:.*point = fc1a5162, host = 7} + expect 0 0 Debug {^vmod_directors: shard:.*point = 3d1fe97, host = 3} + expect 0 0 Debug {^vmod_directors: shard:.*point = a25a43b, host = 6} + expect 0 0 Debug {^vmod_directors: shard:.*point = 2b20d9a2, host = 1} + expect 0 0 Debug {^vmod_directors: shard:.*point = 6337e62c, host = 8} + expect 0 0 Debug {^vmod_directors: shard:.*point = c9803f17, host = 5} + expect 0 0 Debug {^vmod_directors: shard:.*point = d51dafe6, host = 0} + expect 0 0 Debug {^vmod_directors: shard:.*point = eb74a7d5, host = 4} + expect 0 0 Debug {^vmod_directors: shard:.*point = f493ce58, host = 2} + expect 0 0 Debug {^vmod_directors: shard:.*point = fc1a5162, host = 7} expect 0 0 VCL_Log {^-- remove second-last$} - expect 0 0 Debug {^vmod_directors shard:.*point = 3d1fe97, host = 3} - expect 0 0 Debug {^vmod_directors shard:.*point = a25a43b, host = 6} - expect 0 0 Debug {^vmod_directors shard:.*point = 2b20d9a2, host = 1} - expect 0 0 Debug {^vmod_directors shard:.*point = 6337e62c, host = 7} - expect 0 0 Debug {^vmod_directors shard:.*point = c9803f17, host = 5} - expect 0 0 Debug {^vmod_directors shard:.*point = d51dafe6, host = 0} - expect 0 0 Debug {^vmod_directors shard:.*point = eb74a7d5, host = 4} - expect 0 0 Debug {^vmod_directors shard:.*point = f493ce58, host = 2} + expect 0 0 Debug {^vmod_directors: shard:.*point = 3d1fe97, host = 3} + expect 0 0 Debug {^vmod_directors: shard:.*point = a25a43b, host = 6} + expect 0 0 Debug {^vmod_directors: shard:.*point = 2b20d9a2, host = 1} + expect 0 0 Debug {^vmod_directors: shard:.*point = 6337e62c, host = 7} + expect 0 0 Debug {^vmod_directors: shard:.*point = c9803f17, host = 5} + expect 0 0 Debug {^vmod_directors: shard:.*point = d51dafe6, host = 0} + expect 0 0 Debug {^vmod_directors: shard:.*point = eb74a7d5, host = 4} + expect 0 0 Debug {^vmod_directors: shard:.*point = f493ce58, host = 2} expect 0 0 VCL_Log {^-- remove last$} - expect 0 0 Debug {^vmod_directors shard:.*point = 3d1fe97, host = 3} - expect 0 0 Debug {^vmod_directors shard:.*point = a25a43b, host = 6} - expect 0 0 Debug {^vmod_directors shard:.*point = 2b20d9a2, host = 1} - expect 0 0 Debug {^vmod_directors shard:.*point = c9803f17, host = 5} - expect 0 0 Debug {^vmod_directors shard:.*point = d51dafe6, host = 0} - expect 0 0 Debug {^vmod_directors shard:.*point = eb74a7d5, host = 4} - expect 0 0 Debug {^vmod_directors shard:.*point = f493ce58, host = 2} + expect 0 0 Debug {^vmod_directors: shard:.*point = 3d1fe97, host = 3} + expect 0 0 Debug {^vmod_directors: shard:.*point = a25a43b, host = 6} + expect 0 0 Debug {^vmod_directors: shard:.*point = 2b20d9a2, host = 1} + expect 0 0 Debug {^vmod_directors: shard:.*point = c9803f17, host = 5} + expect 0 0 Debug {^vmod_directors: shard:.*point = d51dafe6, host = 0} + expect 0 0 Debug {^vmod_directors: shard:.*point = eb74a7d5, host = 4} + expect 0 0 Debug {^vmod_directors: shard:.*point = f493ce58, host = 2} expect 0 0 VCL_Log {^-- END$} } -start diff --git a/bin/varnishtest/tests/d00016.vtc b/bin/varnishtest/tests/d00016.vtc index 0c093a6ac..a0e5a2ea2 100644 --- a/bin/varnishtest/tests/d00016.vtc +++ b/bin/varnishtest/tests/d00016.vtc @@ -158,13 +158,13 @@ varnish v1 -vcl+backend { logexpect l1 -v v1 -g raw -d 1 { expect * 1001 VCL_Log {^-- invalid replicas$} - expect 0 1001 Error {^vmod_directors shard vd: .reconfigure.. invalid replicas argument 0} + expect 0 1001 Error {^vmod_directors: shard vd: .reconfigure.. invalid replicas argument 0} expect 0 1001 VCL_Log {^reconfigure failed} expect 0 1001 VCL_Log {^-- no changes - no debug output$} expect 0 1001 VCL_Log {^-- no backends$} - expect 0 1001 Error {^vmod_directors shard vd: .reconfigure.. no backends} + expect 0 1001 Error {^vmod_directors: shard vd: .reconfigure.. no backends} expect 0 1001 VCL_Log {^reconfigure failed} expect 0 1001 VCL_Log {^-- one backend$} @@ -172,15 +172,15 @@ logexpect l1 -v v1 -g raw -d 1 { expect 0 1001 VCL_Log {^-- no change - no output$} expect 0 1001 VCL_Log {^-- change, clear, no backends$} - expect 0 1001 Error {^vmod_directors shard vd: .reconfigure.. no backends} + expect 0 1001 Error {^vmod_directors: shard vd: .reconfigure.. no backends} expect 0 1001 VCL_Log {^reconfigure failed} expect 0 1001 VCL_Log {^-- duplicate add$} - expect 0 1001 Notice {^vmod_directors shard vd: backend s1 already exists - skipping$} + expect 0 1001 Notice {^vmod_directors: shard vd: backend s1 already exists - skipping$} expect 0 1001 VCL_Log {^-- duplicate add with idents$} - expect 0 1001 Notice {^vmod_directors shard vd: backend s1 already exists - skipping} - expect 0 1001 Notice {^vmod_directors shard vd: backend s1/s1_1 already exists - skipping} - expect 0 1001 Notice {^vmod_directors shard vd: backend s1/s1_2 already exists - skipping} + expect 0 1001 Notice {^vmod_directors: shard vd: backend s1 already exists - skipping} + expect 0 1001 Notice {^vmod_directors: shard vd: backend s1/s1_1 already exists - skipping} + expect 0 1001 Notice {^vmod_directors: shard vd: backend s1/s1_2 already exists - skipping} expect 0 1001 VCL_Log {^-- remove s1_2 specifically$} expect 0 1001 VCL_Log {^-- remove all instances of s1$} expect 0 1001 VCL_Log {^-- re-add some - no 2nd director$} @@ -190,41 +190,41 @@ logexpect l1 -v v1 -g raw -d 1 { } logexpect l2 -v v1 -g raw -d 1 { - expect * 0 Debug {^vmod_directors shard:.*point = f08ad325, host = 0} - expect 0 0 Debug {^vmod_directors shard:.*point = 6e040182, host = 1} - expect 0 0 Debug {^vmod_directors shard:.*point = f08ad325, host = 0} - expect 0 0 Debug {^vmod_directors shard:.*point = 6e040182, host = 3} - expect 0 0 Debug {^vmod_directors shard:.*point = 732c7bbe, host = 2} - expect 0 0 Debug {^vmod_directors shard:.*point = bae80b0b, host = 1} - expect 0 0 Debug {^vmod_directors shard:.*point = f08ad325, host = 0} - expect 0 0 Debug {^vmod_directors shard:.*point = 6e040182, host = 2} - expect 0 0 Debug {^vmod_directors shard:.*point = bae80b0b, host = 1} - expect 0 0 Debug {^vmod_directors shard:.*point = f08ad325, host = 0} - expect 0 0 Debug {^vmod_directors shard:.*point = 6e040182, host = 0} - expect 0 0 Debug {^vmod_directors shard:.*point = 3d1fe97, host = 3} - expect 0 0 Debug {^vmod_directors shard:.*point = a25a43b, host = 6} - expect 0 0 Debug {^vmod_directors shard:.*point = 2b20d9a2, host = 1} - expect 0 0 Debug {^vmod_directors shard:.*point = 6337e62c, host = 8} - expect 0 0 Debug {^vmod_directors shard:.*point = c9803f17, host = 5} - expect 0 0 Debug {^vmod_directors shard:.*point = d51dafe6, host = 0} - expect 0 0 Debug {^vmod_directors shard:.*point = eb74a7d5, host = 4} - expect 0 0 Debug {^vmod_directors shard:.*point = f493ce58, host = 2} - expect 0 0 Debug {^vmod_directors shard:.*point = fc1a5162, host = 7} - expect 0 0 Debug {^vmod_directors shard:.*point = 3d1fe97, host = 3} - expect 0 0 Debug {^vmod_directors shard:.*point = a25a43b, host = 6} - expect 0 0 Debug {^vmod_directors shard:.*point = 2b20d9a2, host = 1} - expect 0 0 Debug {^vmod_directors shard:.*point = 6337e62c, host = 7} - expect 0 0 Debug {^vmod_directors shard:.*point = c9803f17, host = 5} - expect 0 0 Debug {^vmod_directors shard:.*point = d51dafe6, host = 0} - expect 0 0 Debug {^vmod_directors shard:.*point = eb74a7d5, host = 4} - expect 0 0 Debug {^vmod_directors shard:.*point = f493ce58, host = 2} - expect 0 0 Debug {^vmod_directors shard:.*point = 3d1fe97, host = 3} - expect 0 0 Debug {^vmod_directors shard:.*point = a25a43b, host = 6} - expect 0 0 Debug {^vmod_directors shard:.*point = 2b20d9a2, host = 1} - expect 0 0 Debug {^vmod_directors shard:.*point = c9803f17, host = 5} - expect 0 0 Debug {^vmod_directors shard:.*point = d51dafe6, host = 0} - expect 0 0 Debug {^vmod_directors shard:.*point = eb74a7d5, host = 4} - expect 0 0 Debug {^vmod_directors shard:.*point = f493ce58, host = 2} + expect * 0 Debug {^vmod_directors: shard:.*point = f08ad325, host = 0} + expect 0 0 Debug {^vmod_directors: shard:.*point = 6e040182, host = 1} + expect 0 0 Debug {^vmod_directors: shard:.*point = f08ad325, host = 0} + expect 0 0 Debug {^vmod_directors: shard:.*point = 6e040182, host = 3} + expect 0 0 Debug {^vmod_directors: shard:.*point = 732c7bbe, host = 2} + expect 0 0 Debug {^vmod_directors: shard:.*point = bae80b0b, host = 1} + expect 0 0 Debug {^vmod_directors: shard:.*point = f08ad325, host = 0} + expect 0 0 Debug {^vmod_directors: shard:.*point = 6e040182, host = 2} + expect 0 0 Debug {^vmod_directors: shard:.*point = bae80b0b, host = 1} + expect 0 0 Debug {^vmod_directors: shard:.*point = f08ad325, host = 0} + expect 0 0 Debug {^vmod_directors: shard:.*point = 6e040182, host = 0} + expect 0 0 Debug {^vmod_directors: shard:.*point = 3d1fe97, host = 3} + expect 0 0 Debug {^vmod_directors: shard:.*point = a25a43b, host = 6} + expect 0 0 Debug {^vmod_directors: shard:.*point = 2b20d9a2, host = 1} + expect 0 0 Debug {^vmod_directors: shard:.*point = 6337e62c, host = 8} + expect 0 0 Debug {^vmod_directors: shard:.*point = c9803f17, host = 5} + expect 0 0 Debug {^vmod_directors: shard:.*point = d51dafe6, host = 0} + expect 0 0 Debug {^vmod_directors: shard:.*point = eb74a7d5, host = 4} + expect 0 0 Debug {^vmod_directors: shard:.*point = f493ce58, host = 2} + expect 0 0 Debug {^vmod_directors: shard:.*point = fc1a5162, host = 7} + expect 0 0 Debug {^vmod_directors: shard:.*point = 3d1fe97, host = 3} + expect 0 0 Debug {^vmod_directors: shard:.*point = a25a43b, host = 6} + expect 0 0 Debug {^vmod_directors: shard:.*point = 2b20d9a2, host = 1} + expect 0 0 Debug {^vmod_directors: shard:.*point = 6337e62c, host = 7} + expect 0 0 Debug {^vmod_directors: shard:.*point = c9803f17, host = 5} + expect 0 0 Debug {^vmod_directors: shard:.*point = d51dafe6, host = 0} + expect 0 0 Debug {^vmod_directors: shard:.*point = eb74a7d5, host = 4} + expect 0 0 Debug {^vmod_directors: shard:.*point = f493ce58, host = 2} + expect 0 0 Debug {^vmod_directors: shard:.*point = 3d1fe97, host = 3} + expect 0 0 Debug {^vmod_directors: shard:.*point = a25a43b, host = 6} + expect 0 0 Debug {^vmod_directors: shard:.*point = 2b20d9a2, host = 1} + expect 0 0 Debug {^vmod_directors: shard:.*point = c9803f17, host = 5} + expect 0 0 Debug {^vmod_directors: shard:.*point = d51dafe6, host = 0} + expect 0 0 Debug {^vmod_directors: shard:.*point = eb74a7d5, host = 4} + expect 0 0 Debug {^vmod_directors: shard:.*point = f493ce58, host = 2} } logexpect l1 -start diff --git a/bin/varnishtest/tests/d00017.vtc b/bin/varnishtest/tests/d00017.vtc index eeabcde5f..357efe57a 100644 --- a/bin/varnishtest/tests/d00017.vtc +++ b/bin/varnishtest/tests/d00017.vtc @@ -56,91 +56,91 @@ varnish v1 -vcl+backend { # check that the hashcircle points are decremented by one compared to vslp # # generated from vslp test output -# perl -ne /tmp/regression.logexp +# perl -ne /tmp/regression.logexp logexpect l1 -v v1 -g raw -d 1 { expect 0 0 CLI "^Rd vcl.load" - expect 0 = Debug {^vmod_directors shard: hashcircle.* 0. = .point = 3b6b56a, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 1. = .point = 66986a7, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 2. = .point = 7e41e30, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 3. = .point = b749e7b, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 4. = .point = e543430, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 5. = .point = 10136c05, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 6. = .point = 102d847f, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 7. = .point = 1112f910, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 8. = .point = 1119a7c7, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 9. = .point = 22464ee9, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 10. = .point = 22b35675, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 11. = .point = 2363bebb, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 12. = .point = 259eeccf, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 13. = .point = 26f0c3e7, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 14. = .point = 28340f35, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 15. = .point = 285e8475, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 16. = .point = 28ec7a6f, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 17. = .point = 2da0e37b, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 18. = .point = 3392487a, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 19. = .point = 37597c4c, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 20. = .point = 4b1f5b22, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 21. = .point = 523723f2, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 22. = .point = 539234db, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 23. = .point = 564ca84f, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 24. = .point = 5d2df428, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 25. = .point = 5fa294ee, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 26. = .point = 60dded53, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 27. = .point = 6257bc27, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 28. = .point = 64014b25, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 29. = .point = 6bfd5a2d, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 30. = .point = 6e040182, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 31. = .point = 6e3819f7, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 32. = .point = 7232b381, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 33. = .point = 74c384ad, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 34. = .point = 83ce71ce, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 35. = .point = 888b6447, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 36. = .point = 8997c018, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 37. = .point = 8aa6b5b4, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 38. = .point = 8b47e6ac, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 39. = .point = 8bc8bc11, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 40. = .point = 8e2d3849, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 41. = .point = 8e7e012c, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 42. = .point = 99892987, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 43. = .point = 9a6f2f00, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 44. = .point = 9ef9125d, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 45. = .point = 9f33cd30, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 46. = .point = 9fc69b51, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 47. = .point = a19f99eb, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 48. = .point = a28b9595, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 49. = .point = a8afe9c4, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 50. = .point = ad923ad3, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 51. = .point = ae8946c6, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 52. = .point = b197e339, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 53. = .point = b3c305e6, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 54. = .point = b6bf43ea, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 55. = .point = b9004d3d, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 56. = .point = bbcc0bad, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 57. = .point = c2542a5d, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 58. = .point = c6c43fa7, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 59. = .point = c945958a, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 60. = .point = cbd9198a, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 61. = .point = d4c93105, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 62. = .point = d7de63b6, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 63. = .point = d937a7df, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 64. = .point = dac52229, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 65. = .point = db7840f0, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 66. = .point = dd5c6bef, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 67. = .point = dfd5333b, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 68. = .point = e991584c, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 69. = .point = ec8891c5, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 70. = .point = ef6b4ab5, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 71. = .point = f08ad325, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 72. = .point = f3325ba2, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 73. = .point = f6530dd1, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 74. = .point = fc28e8d2, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 0. = .point = 3b6b56a, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 1. = .point = 66986a7, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 2. = .point = 7e41e30, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 3. = .point = b749e7b, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 4. = .point = e543430, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 5. = .point = 10136c05, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 6. = .point = 102d847f, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 7. = .point = 1112f910, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 8. = .point = 1119a7c7, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 9. = .point = 22464ee9, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 10. = .point = 22b35675, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 11. = .point = 2363bebb, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 12. = .point = 259eeccf, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 13. = .point = 26f0c3e7, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 14. = .point = 28340f35, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 15. = .point = 285e8475, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 16. = .point = 28ec7a6f, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 17. = .point = 2da0e37b, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 18. = .point = 3392487a, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 19. = .point = 37597c4c, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 20. = .point = 4b1f5b22, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 21. = .point = 523723f2, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 22. = .point = 539234db, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 23. = .point = 564ca84f, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 24. = .point = 5d2df428, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 25. = .point = 5fa294ee, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 26. = .point = 60dded53, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 27. = .point = 6257bc27, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 28. = .point = 64014b25, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 29. = .point = 6bfd5a2d, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 30. = .point = 6e040182, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 31. = .point = 6e3819f7, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 32. = .point = 7232b381, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 33. = .point = 74c384ad, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 34. = .point = 83ce71ce, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 35. = .point = 888b6447, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 36. = .point = 8997c018, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 37. = .point = 8aa6b5b4, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 38. = .point = 8b47e6ac, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 39. = .point = 8bc8bc11, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 40. = .point = 8e2d3849, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 41. = .point = 8e7e012c, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 42. = .point = 99892987, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 43. = .point = 9a6f2f00, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 44. = .point = 9ef9125d, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 45. = .point = 9f33cd30, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 46. = .point = 9fc69b51, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 47. = .point = a19f99eb, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 48. = .point = a28b9595, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 49. = .point = a8afe9c4, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 50. = .point = ad923ad3, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 51. = .point = ae8946c6, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 52. = .point = b197e339, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 53. = .point = b3c305e6, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 54. = .point = b6bf43ea, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 55. = .point = b9004d3d, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 56. = .point = bbcc0bad, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 57. = .point = c2542a5d, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 58. = .point = c6c43fa7, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 59. = .point = c945958a, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 60. = .point = cbd9198a, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 61. = .point = d4c93105, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 62. = .point = d7de63b6, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 63. = .point = d937a7df, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 64. = .point = dac52229, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 65. = .point = db7840f0, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 66. = .point = dd5c6bef, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 67. = .point = dfd5333b, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 68. = .point = e991584c, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 69. = .point = ec8891c5, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 70. = .point = ef6b4ab5, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 71. = .point = f08ad325, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 72. = .point = f3325ba2, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 73. = .point = f6530dd1, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 74. = .point = fc28e8d2, host = 2.} expect 0 = CLI Loaded - expect * = Debug {^vmod_directors shard: lookup key 68b902f7 idx 29 host 0} - expect * = Debug {^vmod_directors shard: lookup key 39dc4613 idx 20 host 1} - expect * = Debug {^vmod_directors shard: lookup key c7793505 idx 59 host 2} + expect * = Debug {^vmod_directors: shard: lookup key 68b902f7 idx 29 host 0} + expect * = Debug {^vmod_directors: shard: lookup key 39dc4613 idx 20 host 1} + expect * = Debug {^vmod_directors: shard: lookup key c7793505 idx 59 host 2} } -start client c1 { diff --git a/bin/varnishtest/tests/d00030.vtc b/bin/varnishtest/tests/d00030.vtc index c83ba5db9..070dfa74b 100644 --- a/bin/varnishtest/tests/d00030.vtc +++ b/bin/varnishtest/tests/d00030.vtc @@ -24,14 +24,14 @@ varnish v1 -vcl { logexpect l1 -v v1 -g raw -d 1 { expect 0 0 CLI {^Rd vcl.load} - expect 0 0 Error {vmod_directors shard p: shard_param.set.. by=BLOB but no or empty key_blob - using key 0} + expect 0 0 Error {vmod_directors: shard p: shard_param.set.. by=BLOB but no or empty key_blob - using key 0} } -start -wait logexpect l2 -v v1 -g raw { - expect * 1001 VCL_Error {vmod_directors shard shard: .backend.key_blob. param invalid} + expect * 1001 VCL_Error {vmod_directors: shard shard: .backend.key_blob. param invalid} } -start logexpect l3 -v v1 -g raw { - expect * 1003 VCL_Error {vmod_directors shard p: shard_param.set.. may only be used in vcl_init and in backend/pipe context} + expect * 1003 VCL_Error {vmod_directors: shard p: shard_param.set.. may only be used in vcl_init and in backend/pipe context} } -start client c1 { @@ -159,7 +159,7 @@ varnish v1 -errvcl {invalid warmup argument 1.1} { } } -varnish v1 -errvcl {vmod_directors shard shard: .backend(resolve=LAZY) with other parameters can only be used in backend/pipe context} { +varnish v1 -errvcl {vmod_directors: shard shard: .backend(resolve=LAZY) with other parameters can only be used in backend/pipe context} { import directors; import blob; @@ -172,7 +172,7 @@ varnish v1 -errvcl {vmod_directors shard shard: .backend(resolve=LAZY) with othe } } -varnish v1 -errvcl {vmod_directors shard shard: .backend(resolve=NOW) can not be used in vcl_init{}/vcl_fini{}} { +varnish v1 -errvcl {vmod_directors: shard shard: .backend(resolve=NOW) can not be used in vcl_init{}/vcl_fini{}} { import directors; import blob; diff --git a/bin/varnishtest/tests/d00041.vtc b/bin/varnishtest/tests/d00041.vtc index 5d80e0c19..587f1b897 100644 --- a/bin/varnishtest/tests/d00041.vtc +++ b/bin/varnishtest/tests/d00041.vtc @@ -49,162 +49,162 @@ varnish v1 -vcl+backend { logexpect l1 -v v1 -g raw -d 1 { expect 0 0 CLI "^Rd vcl.load" - expect 0 = Debug {^vmod_directors shard: hashcircle.* 0. = .point = *238d0ef, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 1. = .point = *321c598, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 2. = .point = *3b6b56a, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 3. = .point = *408ec1e, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 4. = .point = *66986a7, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 5. = .point = *7e41e30, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 6. = .point = *b749e7b, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 7. = .point = *e543430, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 8. = .point = *10136c05, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 9. = .point = *102d847f, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 10. = .point = *1112f910, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 11. = .point = *1119a7c7, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 12. = .point = *14d95c44, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 13. = .point = *150fea1f, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 14. = .point = *1643ecb6, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 15. = .point = *189ff2f2, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 16. = .point = *19cfe9f3, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 17. = .point = *1e1c78c3, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 18. = .point = *1fe0dea0, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 19. = .point = *22464ee9, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 20. = .point = *22b35675, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 21. = .point = *2363bebb, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 22. = .point = *24f827bb, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 23. = .point = *259eeccf, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 24. = .point = *26f0c3e7, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 25. = .point = *271874d4, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 26. = .point = *28340f35, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 27. = .point = *285e8475, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 28. = .point = *28ec7a6f, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 29. = .point = *299c6298, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 30. = .point = *2aedc3f7, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 31. = .point = *2b031742, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 32. = .point = *2da0e37b, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 33. = .point = *310bd2ca, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 34. = .point = *31e5f2df, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 35. = .point = *32d6b3ed, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 36. = .point = *33047373, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 37. = .point = *3392487a, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 38. = .point = *37597c4c, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 39. = .point = *3f6b2b89, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 40. = .point = *43cf6426, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 41. = .point = *46a58f28, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 42. = .point = *4b1f5b22, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 43. = .point = *523723f2, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 44. = .point = *539234db, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 45. = .point = *564ca84f, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 46. = .point = *58501380, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 47. = .point = *58704432, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 48. = .point = *5b1bcbbe, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 49. = .point = *5d2df428, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 50. = .point = *5fa294ee, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 51. = .point = *606fd878, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 52. = .point = *60dded53, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 53. = .point = *616cdb68, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 54. = .point = *6257bc27, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 55. = .point = *64014b25, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 56. = .point = *6918f467, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 57. = .point = *6a08c380, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 58. = .point = *6bfd5a2d, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 59. = .point = *6c0b607a, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 60. = .point = *6c74d296, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 61. = .point = *6e040182, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 62. = .point = *6e3819f7, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 63. = .point = *720ec1a4, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 64. = .point = *7232b381, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 65. = .point = *74c384ad, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 66. = .point = *76d47350, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 67. = .point = *791eb3a3, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 68. = .point = *7a048f20, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 69. = .point = *7f874929, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 70. = .point = *83ce71ce, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 71. = .point = *888b6447, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 72. = .point = *8997c018, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 73. = .point = *89b7d09c, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 74. = .point = *8aa6b5b4, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 75. = .point = *8ae34bde, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 76. = .point = *8b382e03, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 77. = .point = *8b47e6ac, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 78. = .point = *8bc76115, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 79. = .point = *8bc8bc11, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 80. = .point = *8e2d3849, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 81. = .point = *8e7e012c, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 82. = .point = *8f5b4c63, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 83. = .point = *94a94162, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 84. = .point = *99892987, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 85. = .point = *9a6f2f00, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 86. = .point = *9b970b49, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 87. = .point = *9e09a3a7, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 88. = .point = *9ef9125d, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 89. = .point = *9f33cd30, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 90. = .point = *9fc69b51, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 91. = .point = *a19f99eb, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 92. = .point = *a28b9595, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 93. = .point = *a3582038, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 94. = .point = *a4b6a3b9, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 95. = .point = *a66da9cb, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 96. = .point = *a8657c76, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 97. = .point = *a8afe9c4, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 98. = .point = *aa488703, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 99. = .point = *ac7b4454, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 100. = .point = *ad923ad3, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 101. = .point = *ae8946c6, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 102. = .point = *b197e339, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 103. = .point = *b3c305e6, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 104. = .point = *b4dab004, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 105. = .point = *b6bf43ea, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 106. = .point = *b9004d3d, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 107. = .point = *b96b6455, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 108. = .point = *b9a0edb9, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 109. = .point = *b9ec6465, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 110. = .point = *bb8eed4d, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 111. = .point = *bbcc0bad, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 112. = .point = *bcfea141, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 113. = .point = *be300622, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 114. = .point = *bf514d68, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 115. = .point = *c1afc7d2, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 116. = .point = *c2542a5d, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 117. = .point = *c6c43fa7, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 118. = .point = *c945958a, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 119. = .point = *c9f304a4, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 120. = .point = *cb896aa8, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 121. = .point = *cbd9198a, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 122. = .point = *ccd61dad, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 123. = .point = *d07e4431, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 124. = .point = *d21fe35f, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 125. = .point = *d4c93105, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 126. = .point = *d570b815, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 127. = .point = *d7de63b6, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 128. = .point = *d8634aef, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 129. = .point = *d92d916d, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 130. = .point = *d937a7df, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 131. = .point = *dac52229, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 132. = .point = *db7840f0, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 133. = .point = *dd5c6bef, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 134. = .point = *dded5798, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 135. = .point = *dfd5333b, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 136. = .point = *e183345a, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 137. = .point = *e2c71c27, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 138. = .point = *e49bf9d8, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 139. = .point = *e72bc224, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 140. = .point = *e8b27f41, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 141. = .point = *e991584c, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 142. = .point = *ea201c5e, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 143. = .point = *ec8891c5, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 144. = .point = *edcc8dd9, host = 1.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 145. = .point = *ef6b4ab5, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 146. = .point = *f08ad325, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 147. = .point = *f3325ba2, host = 2.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 148. = .point = *f6530dd1, host = 0.} - expect 0 = Debug {^vmod_directors shard: hashcircle.* 149. = .point = *fc28e8d2, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 0. = .point = *238d0ef, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 1. = .point = *321c598, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 2. = .point = *3b6b56a, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 3. = .point = *408ec1e, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 4. = .point = *66986a7, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 5. = .point = *7e41e30, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 6. = .point = *b749e7b, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 7. = .point = *e543430, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 8. = .point = *10136c05, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 9. = .point = *102d847f, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 10. = .point = *1112f910, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 11. = .point = *1119a7c7, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 12. = .point = *14d95c44, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 13. = .point = *150fea1f, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 14. = .point = *1643ecb6, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 15. = .point = *189ff2f2, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 16. = .point = *19cfe9f3, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 17. = .point = *1e1c78c3, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 18. = .point = *1fe0dea0, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 19. = .point = *22464ee9, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 20. = .point = *22b35675, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 21. = .point = *2363bebb, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 22. = .point = *24f827bb, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 23. = .point = *259eeccf, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 24. = .point = *26f0c3e7, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 25. = .point = *271874d4, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 26. = .point = *28340f35, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 27. = .point = *285e8475, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 28. = .point = *28ec7a6f, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 29. = .point = *299c6298, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 30. = .point = *2aedc3f7, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 31. = .point = *2b031742, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 32. = .point = *2da0e37b, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 33. = .point = *310bd2ca, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 34. = .point = *31e5f2df, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 35. = .point = *32d6b3ed, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 36. = .point = *33047373, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 37. = .point = *3392487a, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 38. = .point = *37597c4c, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 39. = .point = *3f6b2b89, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 40. = .point = *43cf6426, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 41. = .point = *46a58f28, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 42. = .point = *4b1f5b22, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 43. = .point = *523723f2, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 44. = .point = *539234db, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 45. = .point = *564ca84f, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 46. = .point = *58501380, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 47. = .point = *58704432, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 48. = .point = *5b1bcbbe, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 49. = .point = *5d2df428, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 50. = .point = *5fa294ee, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 51. = .point = *606fd878, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 52. = .point = *60dded53, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 53. = .point = *616cdb68, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 54. = .point = *6257bc27, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 55. = .point = *64014b25, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 56. = .point = *6918f467, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 57. = .point = *6a08c380, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 58. = .point = *6bfd5a2d, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 59. = .point = *6c0b607a, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 60. = .point = *6c74d296, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 61. = .point = *6e040182, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 62. = .point = *6e3819f7, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 63. = .point = *720ec1a4, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 64. = .point = *7232b381, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 65. = .point = *74c384ad, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 66. = .point = *76d47350, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 67. = .point = *791eb3a3, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 68. = .point = *7a048f20, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 69. = .point = *7f874929, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 70. = .point = *83ce71ce, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 71. = .point = *888b6447, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 72. = .point = *8997c018, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 73. = .point = *89b7d09c, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 74. = .point = *8aa6b5b4, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 75. = .point = *8ae34bde, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 76. = .point = *8b382e03, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 77. = .point = *8b47e6ac, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 78. = .point = *8bc76115, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 79. = .point = *8bc8bc11, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 80. = .point = *8e2d3849, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 81. = .point = *8e7e012c, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 82. = .point = *8f5b4c63, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 83. = .point = *94a94162, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 84. = .point = *99892987, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 85. = .point = *9a6f2f00, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 86. = .point = *9b970b49, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 87. = .point = *9e09a3a7, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 88. = .point = *9ef9125d, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 89. = .point = *9f33cd30, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 90. = .point = *9fc69b51, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 91. = .point = *a19f99eb, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 92. = .point = *a28b9595, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 93. = .point = *a3582038, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 94. = .point = *a4b6a3b9, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 95. = .point = *a66da9cb, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 96. = .point = *a8657c76, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 97. = .point = *a8afe9c4, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 98. = .point = *aa488703, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 99. = .point = *ac7b4454, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 100. = .point = *ad923ad3, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 101. = .point = *ae8946c6, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 102. = .point = *b197e339, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 103. = .point = *b3c305e6, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 104. = .point = *b4dab004, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 105. = .point = *b6bf43ea, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 106. = .point = *b9004d3d, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 107. = .point = *b96b6455, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 108. = .point = *b9a0edb9, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 109. = .point = *b9ec6465, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 110. = .point = *bb8eed4d, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 111. = .point = *bbcc0bad, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 112. = .point = *bcfea141, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 113. = .point = *be300622, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 114. = .point = *bf514d68, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 115. = .point = *c1afc7d2, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 116. = .point = *c2542a5d, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 117. = .point = *c6c43fa7, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 118. = .point = *c945958a, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 119. = .point = *c9f304a4, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 120. = .point = *cb896aa8, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 121. = .point = *cbd9198a, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 122. = .point = *ccd61dad, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 123. = .point = *d07e4431, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 124. = .point = *d21fe35f, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 125. = .point = *d4c93105, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 126. = .point = *d570b815, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 127. = .point = *d7de63b6, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 128. = .point = *d8634aef, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 129. = .point = *d92d916d, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 130. = .point = *d937a7df, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 131. = .point = *dac52229, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 132. = .point = *db7840f0, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 133. = .point = *dd5c6bef, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 134. = .point = *dded5798, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 135. = .point = *dfd5333b, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 136. = .point = *e183345a, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 137. = .point = *e2c71c27, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 138. = .point = *e49bf9d8, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 139. = .point = *e72bc224, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 140. = .point = *e8b27f41, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 141. = .point = *e991584c, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 142. = .point = *ea201c5e, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 143. = .point = *ec8891c5, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 144. = .point = *edcc8dd9, host = 1.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 145. = .point = *ef6b4ab5, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 146. = .point = *f08ad325, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 147. = .point = *f3325ba2, host = 2.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 148. = .point = *f6530dd1, host = 0.} + expect 0 = Debug {^vmod_directors: shard: hashcircle.* 149. = .point = *fc28e8d2, host = 2.} expect 0 = CLI Loaded - expect * = Debug {^vmod_directors shard: lookup key 564ca84f idx 45 host 0} - expect * = Debug {^vmod_directors shard: lookup key 19cfe9f3 idx 16 host 1} - expect * = Debug {^vmod_directors shard: lookup key 46a58f28 idx 41 host 2} + expect * = Debug {^vmod_directors: shard: lookup key 564ca84f idx 45 host 0} + expect * = Debug {^vmod_directors: shard: lookup key 19cfe9f3 idx 16 host 1} + expect * = Debug {^vmod_directors: shard: lookup key 46a58f28 idx 41 host 2} } -start client c1 { diff --git a/doc/changes.rst b/doc/changes.rst index 59a6dc52a..a80914c52 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -49,7 +49,7 @@ Varnish Cache Next (2021-03-15) turned into ``Notice`` log messages. * All shard ``Error`` and ``Notice`` messages now use the unified - prefix ``vmod_directors shard %s``. + prefix ``vmod_directors: shard %s``. ================================ Varnish Cache 6.5.1 (2020-09-25) diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h index 4ec22a1d2..92f505df1 100644 --- a/lib/libvmod_directors/shard_dir.h +++ b/lib/libvmod_directors/shard_dir.h @@ -89,16 +89,16 @@ sharddir_backend(const struct sharddir *shardd, unsigned id) #define SHDBG(flag, shardd, ...) \ do { \ if ((shardd)->debug_flags & (flag)) \ - VSL(SLT_Debug, 0, "vmod_directors shard: " \ + VSL(SLT_Debug, 0, "vmod_directors_ shard: " \ __VA_ARGS__); \ } while (0) #define shard_log(vsl, tag, name, fmt, ...) \ - sharddir_log(vsl, tag, "vmod_directors shard %s: " fmt, \ + sharddir_log(vsl, tag, "vmod_directors: shard %s: " fmt, \ name, __VA_ARGS__) #define shard_fail(ctx, name, fmt, ...) \ - VRT_fail(ctx, "vmod_directors shard %s: " fmt, name, __VA_ARGS__) + VRT_fail(ctx, "vmod_directors: shard %s: " fmt, name, __VA_ARGS__) #define shard_err(vsl, name, fmt, ...) \ shard_log(vsl, SLT_Error, name, fmt, __VA_ARGS__) #define shard_err0(vsl, name, msg) \ From nils.goroll at uplex.de Sun Oct 11 13:21:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 11 Oct 2020 13:21:06 +0000 (UTC) Subject: [master] 0bbc1793d polish vsl.rst in line with include/tbl/vsl_tags.h Message-ID: <20201011132106.6D561AF035@lists.varnish-cache.org> commit 0bbc1793ddf3cd600bf6bdacf5a0e8321fad9504 Author: Nils Goroll Date: Sun Oct 11 15:19:18 2020 +0200 polish vsl.rst in line with include/tbl/vsl_tags.h diff --git a/doc/sphinx/reference/vsl.rst b/doc/sphinx/reference/vsl.rst index 583bd07b9..9b245344e 100644 --- a/doc/sphinx/reference/vsl.rst +++ b/doc/sphinx/reference/vsl.rst @@ -107,9 +107,9 @@ NOTICE MESSAGES Notice messages contain informational messages about the handling of a request. These can be exceptional circumstances encountered that causes -deviation from the normal handling. The messages are prefixed with ``vsl`` +deviation from the normal handling. The messages are prefixed with ``vsl:`` for core Varnish generated messages, and VMOD authors are encouraged to -use ``vmod_`` for their own notice messages. This matches the name +use ``vmod_:`` for their own notice messages. This matches the name of the manual page where detailed descriptions of notice messages are expected. From dridi.boukelmoune at gmail.com Mon Oct 12 06:42:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 12 Oct 2020 06:42:06 +0000 (UTC) Subject: [master] 1642140aa shard: Typo Message-ID: <20201012064206.CB7E9A13C8@lists.varnish-cache.org> commit 1642140aaf959749fe32e09bec3d7e52666866e4 Author: Dridi Boukelmoune Date: Mon Oct 12 08:40:41 2020 +0200 shard: Typo diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h index 92f505df1..47a66b92a 100644 --- a/lib/libvmod_directors/shard_dir.h +++ b/lib/libvmod_directors/shard_dir.h @@ -89,7 +89,7 @@ sharddir_backend(const struct sharddir *shardd, unsigned id) #define SHDBG(flag, shardd, ...) \ do { \ if ((shardd)->debug_flags & (flag)) \ - VSL(SLT_Debug, 0, "vmod_directors_ shard: " \ + VSL(SLT_Debug, 0, "vmod_directors: shard: " \ __VA_ARGS__); \ } while (0) From dridi.boukelmoune at gmail.com Wed Oct 14 09:57:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 14 Oct 2020 09:57:06 +0000 (UTC) Subject: [master] ac4cf5099 vrt: Ensure backends always have a host_header Message-ID: <20201014095706.5C814105457@lists.varnish-cache.org> commit ac4cf50993debbce6aa82b8907dcde457cac1bdc Author: Dridi Boukelmoune Date: Wed Oct 14 11:51:59 2020 +0200 vrt: Ensure backends always have a host_header This is enforced by libvcc for static backends, but not guaranteed for dynamic backends. With no fallback we cannot meet the requirement since HTTP/1.1 that requests always have a host header if it is unset from VCL, and probes expect hosthdr to always be set. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 4eb18b8b9..763622798 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -586,6 +586,8 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, struct vcl *vcl; const struct vrt_backend_probe *vbp; const struct vdi_methods *m; + const struct suckaddr *sa; + char abuf[VTCP_ADDRBUFSIZE]; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC); @@ -613,6 +615,19 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, #undef DA #undef DN + if (be->hosthdr == NULL) { + if (vrt->path != NULL) + sa = bogo_ip; + else if (cache_param->prefer_ipv6 && vrt->ipv6_suckaddr != NULL) + sa = vrt->ipv6_suckaddr; + else if (vrt->ipv4_suckaddr != NULL) + sa = vrt->ipv4_suckaddr; + else + sa = vrt->ipv6_suckaddr; + VTCP_name(sa, abuf, sizeof abuf, NULL, 0); + REPLACE(be->hosthdr, abuf); + } + be->vsc = VSC_vbe_New(vc, &be->vsc_seg, "%s.%s", VCL_Name(ctx->vcl), vrt->vcl_name); AN(be->vsc); From nils.goroll at uplex.de Wed Oct 14 10:46:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Oct 2020 10:46:06 +0000 (UTC) Subject: [master] 6c6f8d1e3 flexelinting Message-ID: <20201014104606.BCAA51066AF@lists.varnish-cache.org> commit 6c6f8d1e34153c7423ad6ea7a73d1d26ce475355 Author: Nils Goroll Date: Wed Oct 14 12:33:36 2020 +0200 flexelinting diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c index 0b7c69913..079279c30 100644 --- a/bin/varnishd/cache/cache_fetch_proc.c +++ b/bin/varnishd/cache/cache_fetch_proc.c @@ -99,7 +99,7 @@ VFP_Extend(const struct vfp_ctx *vc, ssize_t sz, enum vfp_status flg) { CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); - ObjExtend(vc->wrk, vc->oc, sz, flg == VFP_END); + ObjExtend(vc->wrk, vc->oc, sz, flg == VFP_END ? 1 : 0); } /********************************************************************** diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c index fdec1bd4c..9a52edf5b 100644 --- a/bin/varnishd/cache/cache_req_body.c +++ b/bin/varnishd/cache/cache_req_body.c @@ -124,7 +124,8 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv) if (r) break; } else { - ObjExtend(req->wrk, req->body_oc, l, vfps == VFP_END); + ObjExtend(req->wrk, req->body_oc, l, + vfps == VFP_END ? 1 : 0); } } diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c index a9d83de36..687d618e0 100644 --- a/lib/libvarnishapi/vsl_cursor.c +++ b/lib/libvarnishapi/vsl_cursor.c @@ -486,7 +486,7 @@ vsl_cursor_mmap(struct VSL_data *vsl, int fd) if ((st->st_mode & S_IFMT) != S_IFREG) return (MAP_FAILED); - assert(st->st_size >= sizeof VSL_FILE_ID); + assert(st->st_size >= (off_t)(sizeof VSL_FILE_ID)); p = mmap(NULL, st->st_size, PROT_READ, MAP_PRIVATE, fd, 0); if (p == MAP_FAILED) { vsl_diag(vsl, "Cannot mmap: %s", strerror(errno)); diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 40ecc13ca..3f7e57b1a 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -174,7 +174,7 @@ static const struct vdp xyzzy_vdp_rot13 = { * yet that would cross the API */ -void * end_marker = &end_marker; +static void * end_marker = &end_marker; static int v_matchproto_(vdp_bytes_f) xyzzy_pedantic_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, From nils.goroll at uplex.de Wed Oct 14 13:00:08 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Oct 2020 13:00:08 +0000 (UTC) Subject: [master] 40b6f254e fix rfc reference Message-ID: <20201014130008.50FAA10974D@lists.varnish-cache.org> commit 40b6f254ef68d4c5020c82d113b68e974d6c7e66 Author: Nils Goroll Date: Wed Oct 14 12:54:24 2020 +0200 fix rfc reference diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 25fb07fd8..ceb6f3441 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -590,7 +590,7 @@ VRT_UpperLowerStrands(VRT_CTX, VCL_STRANDS s, int up) return (r); } -// RFC7232, 3.2 without obsolete line folding: +// rfc7230,l,1243,1244 // ASCII VCHAR + TAB + obs-text (0x80-ff) static inline VCL_BOOL validhdr(const char *p) From nils.goroll at uplex.de Wed Oct 14 13:00:08 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 14 Oct 2020 13:00:08 +0000 (UTC) Subject: [master] 85a42d7d7 test and document VRT_ValidHdr() Message-ID: <20201014130008.775D0109750@lists.varnish-cache.org> commit 85a42d7d740c42eed677a3cbd2796eec3dd5d135 Author: Nils Goroll Date: Wed Oct 14 13:30:40 2020 +0200 test and document VRT_ValidHdr() diff --git a/bin/varnishtest/tests/b00040.vtc b/bin/varnishtest/tests/b00040.vtc index 5e396b386..902bcc8ca 100644 --- a/bin/varnishtest/tests/b00040.vtc +++ b/bin/varnishtest/tests/b00040.vtc @@ -10,12 +10,21 @@ server s1 { } -start varnish v1 -vcl+backend { + import debug; + sub vcl_recv { if (req.url == "/9") { set req.http.foo = {" "}; } } + sub vcl_deliver { + if (req.url == "/9") { + set resp.http.valid1 = debug.validhdr({" + "}); + set resp.http.valid2 = debug.validhdr("a"); + } + } } -start logexpect l1 -v v1 -g raw { @@ -105,4 +114,6 @@ client c1 { txreq -url /9 rxresp expect resp.status == 200 + expect resp.http.valid1 == false + expect resp.http.valid2 == true } -run diff --git a/doc/changes.rst b/doc/changes.rst index a80914c52..4b7c61886 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -37,6 +37,11 @@ Varnish Cache Next (2021-03-15) set headers are now validated to contain only characters allowed by RFC7230. A (runtime) VCL failure is triggered if not. +* ``VRT_ValidHdr()`` has been added for vmods to conduct the same + check as the ``validate_headers`` feature, for example when headers + are set by vmods using the ``cache_http.c`` Functions like + ``http_ForceHeader()`` from untrusted input. + * The shard director now supports reconfiguration (adding/removing backends) of several instances without any special ordering requirement. diff --git a/include/vrt.h b/include/vrt.h index 7e31480a4..f9894ed73 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -52,8 +52,9 @@ * Whenever something is deleted or changed in a way which is not * binary/load-time compatible, increment MAJOR version * - * 13.0 (2020-03-15) + * 13.0 (2021-03-15) * Calling convention for VDP implementation changed + * Added VRT_ValidHdr() * 12.0 (2020-09-15) * Added VRT_DirectorResolve() * Added VCL_STRING VRT_BLOB_string(VRT_CTX, VCL_BLOB) diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 3f7e57b1a..65ce4bed5 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -1208,3 +1208,10 @@ xyzzy_priv_task_with_option(VRT_CTX, struct VARGS(priv_task_with_option) *args) args->priv->priv = WS_Copy(ctx->ws, args->opt, -1); return (args->priv->priv); } + +VCL_BOOL v_matchproto_(td_xyzzy_validhdr) +xyzzy_validhdr(VRT_CTX, VCL_STRANDS s) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + return (VRT_ValidHdr(ctx, s)); +} diff --git a/lib/libvmod_debug/vmod_debug.vcc b/lib/libvmod_debug/vmod_debug.vcc index f565b456c..4eee74547 100644 --- a/lib/libvmod_debug/vmod_debug.vcc +++ b/lib/libvmod_debug/vmod_debug.vcc @@ -312,3 +312,7 @@ Quote an input string to be usable for an exact match in a regular expression. $Function STRING priv_task_with_option(PRIV_TASK priv, [STRING opt]) A function mixing a named PRIV_TASK with optional parameters. + +$Function BOOL validhdr(STRANDS) + +Test if the argument is a valid header according to RFC7230 section 3.2. From phk at FreeBSD.org Thu Oct 15 08:04:06 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 15 Oct 2020 08:04:06 +0000 (UTC) Subject: [master] 0051cbe3b Import from VTest: Don't use explicit buffer size with no buffer Message-ID: <20201015080406.1A0C7100578@lists.varnish-cache.org> commit 0051cbe3b9cc24586e0ad99cff72041c6df944b8 Author: Poul-Henning Kamp Date: Thu Oct 15 07:35:04 2020 +0000 Import from VTest: Don't use explicit buffer size with no buffer diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 1deea4ab7..3c3887450 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -698,7 +698,7 @@ main(int argc, char * const *argv) else tmppath = strdup("/tmp"); - cwd = getcwd(NULL, PATH_MAX); + cwd = getcwd(NULL, 0); extmacro_def("pwd", "%s", cwd); vmod_path = NULL; From phk at FreeBSD.org Thu Oct 15 08:04:06 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 15 Oct 2020 08:04:06 +0000 (UTC) Subject: [master] 745b2f664 Move the "loop" word up to global level, so it works everywhere. Message-ID: <20201015080406.2E28310057B@lists.varnish-cache.org> commit 745b2f664bb369dd66033d5b62f4758cf12db597 Author: Poul-Henning Kamp Date: Thu Oct 15 08:02:29 2020 +0000 Move the "loop" word up to global level, so it works everywhere. diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 88c0eba11..b6346f398 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -316,6 +316,7 @@ parse_string(const char *spec, const struct cmds *cmd, void *priv, char *e, *p, *q, *f, *buf; int nest_brace; int tn; + unsigned n, m; const struct cmds *cp; AN(spec); @@ -424,6 +425,24 @@ parse_string(const char *spec, const struct cmds *cmd, void *priv, } } + +/* SECTION: loop loop + * + * loop NUMBER STRING + * Process STRING as a specification, NUMBER times. + * + * This works inside all specification strings + */ + + if (!strcmp(token_s[0], "loop")) { + n = strtoul(token_s[1], NULL, 0); + for (m = 0; m < n; m++) { + vtc_log(vl, 4, "Loop #%u", m); + parse_string(token_s[2], cmd, priv, vl); + } + continue; + } + for (cp = cmd; cp->name != NULL; cp++) if (!strcmp(token_s[0], cp->name)) break; diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 4a747f0dd..dd53bfd49 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -1573,29 +1573,6 @@ cmd_http_accept(CMD_ARGS) vtc_log(vl, 3, "Accepted socket fd is %d", hp->fd); } -/* SECTION: client-server.spec.loop - * - * loop NUMBER STRING - * Process STRING as a specification, NUMBER times. - */ - -static void -cmd_http_loop(CMD_ARGS) -{ - struct http *hp; - unsigned n, m; - - CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); - AN(av[1]); - AN(av[2]); - AZ(av[3]); - n = strtoul(av[1], NULL, 0); - for (m = 1 ; m <= n; m++) { - vtc_log(vl, 4, "Loop #%u", m); - parse_string(av[2], cmd, hp, vl); - } -} - /* SECTION: client-server.spec.fatal * * fatal|non_fatal @@ -1777,7 +1754,6 @@ const struct cmds http_cmds[] = { /* spec */ CMD_HTTP(fatal) - CMD_HTTP(loop) CMD_HTTP(non_fatal) /* body */ From nils.goroll at uplex.de Thu Oct 15 09:27:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 15 Oct 2020 09:27:06 +0000 (UTC) Subject: [master] 7a1d8def8 fix signal handling in VUT Message-ID: <20201015092706.875AA1025A0@lists.varnish-cache.org> commit 7a1d8def8a8dac1f8ffec995110ec28d1c60ecef Author: Nils Goroll Date: Wed Oct 14 16:26:34 2020 +0200 fix signal handling in VUT 55995ab0f96c458b1b585afe49c3af737e363205 turned the VSIG_* flags into counters, and changed the top of the VUT_Main() accordingly. We need to reflect the change also further down in the tight loop calling VSLQ_Dispatch(). Fixes #3436 diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c index 6e8384572..24407dc80 100644 --- a/lib/libvarnishapi/vut.c +++ b/lib/libvarnishapi/vut.c @@ -454,7 +454,9 @@ VUT_Main(struct VUT *vut) do i = VSLQ_Dispatch(vut->vslq, vut_dispatch, vut); - while (i == vsl_more && !VSIG_hup && !VSIG_usr1); + while (i == vsl_more && + VSIG_usr1 == vut->last_sigusr1 && + VSIG_hup == vut->last_sighup); if (i == vsl_more) continue; From nils.goroll at uplex.de Thu Oct 15 10:06:06 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 15 Oct 2020 10:06:06 +0000 (UTC) Subject: [master] f04d7df6e vtest: put cwd on the stack Message-ID: <20201015100606.D82E61034C8@lists.varnish-cache.org> commit f04d7df6e6d5b350740e01d24749a3930390b7a0 Author: Nils Goroll Date: Thu Oct 15 12:01:35 2020 +0200 vtest: put cwd on the stack 0051cbe3b9cc24586e0ad99cff72041c6df944b8 did not work on solaris-descendents, the man page clearly states that the size argument also determines the buffer to be malloc()ed for a NULL buffer argument. diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 3c3887450..1b322f8de 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -686,6 +686,7 @@ main(int argc, char * const *argv) int use_cleaner = 0; uintmax_t bufsiz; const char *p; + char buf[PATH_MAX]; argv0 = strrchr(argv[0], '/'); if (argv0 == NULL) @@ -698,7 +699,7 @@ main(int argc, char * const *argv) else tmppath = strdup("/tmp"); - cwd = getcwd(NULL, 0); + cwd = getcwd(buf, sizeof buf); extmacro_def("pwd", "%s", cwd); vmod_path = NULL; From dridi at varni.sh Thu Oct 15 10:29:09 2020 From: dridi at varni.sh (Dridi Boukelmoune) Date: Thu, 15 Oct 2020 10:29:09 +0000 Subject: [master] f04d7df6e vtest: put cwd on the stack In-Reply-To: <20201015100606.D82E61034C8@lists.varnish-cache.org> References: <20201015100606.D82E61034C8@lists.varnish-cache.org> Message-ID: On Thu, Oct 15, 2020 at 10:06 AM Nils Goroll wrote: > > > commit f04d7df6e6d5b350740e01d24749a3930390b7a0 > Author: Nils Goroll > Date: Thu Oct 15 12:01:35 2020 +0200 > > vtest: put cwd on the stack > > 0051cbe3b9cc24586e0ad99cff72041c6df944b8 did not work on > solaris-descendents, the man page clearly states that the size argument > also determines the buffer to be malloc()ed for a NULL buffer argument. I think this the way to go [1], POSIX says this about the size argument: > The size argument is the size in bytes of the character array pointed to > by the buf argument. If buf is a null pointer, the behavior of getcwd() is > unspecified. 0051cbe3b9cc was triggered by a GCC very eager to enforce *a* behavior on Linux: https://github.com/vtest/VTest/pull/22 Cheers [1] unless someone wants to fix GCC instead :-p From nils.goroll at uplex.de Thu Oct 15 12:57:47 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 15 Oct 2020 14:57:47 +0200 Subject: [master] f04d7df6e vtest: put cwd on the stack In-Reply-To: References: <20201015100606.D82E61034C8@lists.varnish-cache.org> Message-ID: <02c8c037-6fad-aa08-1c8d-8129cac99360@uplex.de> On 15/10/2020 12:29, Dridi Boukelmoune wrote: > [1] unless someone wants to fix GCC instead :-P at least I tried to make them aware https://gcc.gnu.org/bugzilla/show_bug.cgi?id=96832#c4 the second last comment seems to suggest to me that they are at least considering to implement this check differently -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From dridi at varni.sh Thu Oct 15 14:44:31 2020 From: dridi at varni.sh (Dridi Boukelmoune) Date: Thu, 15 Oct 2020 14:44:31 +0000 Subject: [master] f04d7df6e vtest: put cwd on the stack In-Reply-To: <02c8c037-6fad-aa08-1c8d-8129cac99360@uplex.de> References: <20201015100606.D82E61034C8@lists.varnish-cache.org> <02c8c037-6fad-aa08-1c8d-8129cac99360@uplex.de> Message-ID: On Thu, Oct 15, 2020 at 12:57 PM Nils Goroll wrote: > > On 15/10/2020 12:29, Dridi Boukelmoune wrote: > > [1] unless someone wants to fix GCC instead :-P > > at least I tried to make them aware > > https://gcc.gnu.org/bugzilla/show_bug.cgi?id=96832#c4 Thanks! From phk at FreeBSD.org Mon Oct 19 08:35:07 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 19 Oct 2020 08:35:07 +0000 (UTC) Subject: [master] 79578879b Move the file descriptor up to the vtc_sess Message-ID: <20201019083507.5BFB411974E@lists.varnish-cache.org> commit 79578879bbe817bafa9069578dc38e64850e7538 Author: Poul-Henning Kamp Date: Mon Oct 19 06:53:30 2020 +0000 Move the file descriptor up to the vtc_sess diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index 23a3a6472..af2d37bd7 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -92,7 +92,7 @@ void init_syslog(void); struct vtc_sess *Sess_New(struct vtclog *vl, const char *name); void Sess_Destroy(struct vtc_sess **spp); int Sess_GetOpt(struct vtc_sess *, char * const **); -int sess_process(struct vtclog *vl, const struct vtc_sess *, +int sess_process(struct vtclog *vl, struct vtc_sess *, const char *spec, int sock, int *sfd, const char *addr); typedef int sess_conn_f(void *priv, struct vtclog *); @@ -109,9 +109,6 @@ Sess_Start_Thread( ); -int http_process(struct vtclog *vl, const char *spec, int sock, int *sfd, - const char *addr, int rcvbuf); - char * synth_body(const char *len, int rnd); void cmd_server_gen_vcl(struct vsb *vsb); diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index dd53bfd49..08ceabd52 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -217,7 +217,7 @@ http_write(const struct http *hp, int lvl, const char *pfx) AZ(VSB_finish(hp->vsb)); vtc_dump(hp->vl, lvl, pfx, VSB_data(hp->vsb), VSB_len(hp->vsb)); - if (VSB_tofile(hp->vsb, hp->fd)) + if (VSB_tofile(hp->vsb, hp->sess->fd)) vtc_log(hp->vl, hp->fatal, "Write failed: %s", strerror(errno)); } @@ -495,7 +495,7 @@ http_rxchar(struct http *hp, int n, int eof) struct pollfd pfd[1]; while (n > 0) { - pfd[0].fd = hp->fd; + pfd[0].fd = hp->sess->fd; pfd[0].events = POLLIN; pfd[0].revents = 0; i = poll(pfd, 1, hp->timeout); @@ -504,34 +504,34 @@ http_rxchar(struct http *hp, int n, int eof) if (i == 0) { vtc_log(hp->vl, hp->fatal, "HTTP rx timeout (fd:%d %u ms)", - hp->fd, hp->timeout); + hp->sess->fd, hp->timeout); continue; } if (i < 0) { vtc_log(hp->vl, hp->fatal, "HTTP rx failed (fd:%d poll: %s)", - hp->fd, strerror(errno)); + hp->sess->fd, strerror(errno)); continue; } assert(i > 0); assert(hp->rx_p + n < hp->rx_e); - i = read(hp->fd, hp->rx_p, n); + i = read(hp->sess->fd, hp->rx_p, n); if (!(pfd[0].revents & POLLIN)) vtc_log(hp->vl, 4, "HTTP rx poll (fd:%d revents: %x n=%d, i=%d)", - hp->fd, pfd[0].revents, n, i); + hp->sess->fd, pfd[0].revents, n, i); if (i == 0 && eof) return (i); if (i == 0) { vtc_log(hp->vl, hp->fatal, "HTTP rx EOF (fd:%d read: %s) %d", - hp->fd, strerror(errno), n); + hp->sess->fd, strerror(errno), n); return (-1); } if (i < 0) { vtc_log(hp->vl, hp->fatal, "HTTP rx failed (fd:%d read: %s)", - hp->fd, strerror(errno)); + hp->sess->fd, strerror(errno)); return (-1); } hp->rx_p += i; @@ -1271,7 +1271,7 @@ cmd_http_recv(CMD_ARGS) AZ(av[2]); n = strtoul(av[1], NULL, 0); while (n > 0) { - i = read(hp->fd, u, n > 32 ? 32 : n); + i = read(hp->sess->fd, u, n > 32 ? 32 : n); if (i > 0) vtc_dump(hp->vl, 4, "recv", u, i); else @@ -1299,7 +1299,7 @@ cmd_http_send(CMD_ARGS) AN(av[1]); AZ(av[2]); vtc_dump(hp->vl, 4, "send", av[1], -1); - i = write(hp->fd, av[1], strlen(av[1])); + i = write(hp->sess->fd, av[1], strlen(av[1])); if (i != strlen(av[1])) vtc_log(hp->vl, hp->fatal, "Write error in http_send(): %s", strerror(errno)); @@ -1327,7 +1327,7 @@ cmd_http_send_n(CMD_ARGS) vtc_dump(hp->vl, 4, "send_n", av[2], -1); l = strlen(av[2]); while (n--) { - i = write(hp->fd, av[2], l); + i = write(hp->sess->fd, av[2], l); if (i != l) vtc_log(hp->vl, hp->fatal, "Write error in http_send(): %s", @@ -1353,7 +1353,7 @@ cmd_http_send_urgent(CMD_ARGS) AN(av[1]); AZ(av[2]); vtc_dump(hp->vl, 4, "send_urgent", av[1], -1); - i = send(hp->fd, av[1], strlen(av[1]), MSG_OOB); + i = send(hp->sess->fd, av[1], strlen(av[1]), MSG_OOB); if (i != strlen(av[1])) vtc_log(hp->vl, hp->fatal, "Write error in http_send_urgent(): %s", strerror(errno)); @@ -1381,7 +1381,7 @@ cmd_http_sendhex(CMD_ARGS) vsb = vtc_hex_to_bin(hp->vl, av[1]); assert(VSB_len(vsb) >= 0); vtc_hexdump(hp->vl, 4, "sendhex", VSB_data(vsb), VSB_len(vsb)); - if (VSB_tofile(vsb, hp->fd)) + if (VSB_tofile(vsb, hp->sess->fd)) vtc_log(hp->vl, hp->fatal, "Write failed: %s", strerror(errno)); VSB_destroy(&vsb); @@ -1493,11 +1493,11 @@ cmd_http_expect_close(CMD_ARGS) CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); AZ(av[1]); - vtc_log(vl, 4, "Expecting close (fd = %d)", hp->fd); + vtc_log(vl, 4, "Expecting close (fd = %d)", hp->sess->fd); if (hp->h2) stop_h2(hp); while (1) { - fds[0].fd = hp->fd; + fds[0].fd = hp->sess->fd; fds[0].events = POLLIN; fds[0].revents = 0; i = poll(fds, 1, hp->timeout); @@ -1509,7 +1509,7 @@ cmd_http_expect_close(CMD_ARGS) vtc_log(vl, hp->fatal, "Expected close: poll = %d, revents = 0x%x", i, fds[0].revents); - i = read(hp->fd, &c, 1); + i = read(hp->sess->fd, &c, 1); if (VTCP_Check(i)) break; if (i == 1 && vct_islws(c)) @@ -1517,7 +1517,7 @@ cmd_http_expect_close(CMD_ARGS) vtc_log(vl, hp->fatal, "Expecting close: read = %d, c = 0x%02x", i, c); } - vtc_log(vl, 4, "fd=%d EOF, as expected", hp->fd); + vtc_log(vl, 4, "fd=%d EOF, as expected", hp->sess->fd); } /* SECTION: client-server.spec.close @@ -1540,7 +1540,7 @@ cmd_http_close(CMD_ARGS) assert(*hp->sfd >= 0); if (hp->h2) stop_h2(hp); - VTCP_close(&hp->fd); + VTCP_close(&hp->sess->fd); vtc_log(vl, 4, "Closed"); } @@ -1564,13 +1564,13 @@ cmd_http_accept(CMD_ARGS) assert(*hp->sfd >= 0); if (hp->h2) stop_h2(hp); - if (hp->fd >= 0) - VTCP_close(&hp->fd); + if (hp->sess->fd >= 0) + VTCP_close(&hp->sess->fd); vtc_log(vl, 4, "Accepting"); - hp->fd = accept(*hp->sfd, NULL, NULL); - if (hp->fd < 0) + hp->sess->fd = accept(*hp->sfd, NULL, NULL); + if (hp->sess->fd < 0) vtc_log(vl, hp->fatal, "Accepted failed: %s", strerror(errno)); - vtc_log(vl, 3, "Accepted socket fd is %d", hp->fd); + vtc_log(vl, 3, "Accepted socket fd is %d", hp->sess->fd); } /* SECTION: client-server.spec.fatal @@ -1619,12 +1619,12 @@ cmd_http_txpri(CMD_ARGS) vtc_dump(hp->vl, 4, "txpri", PREFACE, sizeof(PREFACE)); /* Dribble out the preface */ - l = write(hp->fd, PREFACE, 18); + l = write(hp->sess->fd, PREFACE, 18); if (l != 18) vtc_log(vl, hp->fatal, "Write failed: (%zd vs %zd) %s", l, sizeof(PREFACE), strerror(errno)); usleep(10000); - l = write(hp->fd, PREFACE + 18, sizeof(PREFACE) - 18); + l = write(hp->sess->fd, PREFACE + 18, sizeof(PREFACE) - 18); if (l != sizeof(PREFACE) - 18) vtc_log(vl, hp->fatal, "Write failed: (%zd vs %zd) %s", l, sizeof(PREFACE), strerror(errno)); @@ -1808,8 +1808,8 @@ http_process_cleanup(void *arg) } int -http_process(struct vtclog *vl, const char *spec, int sock, int *sfd, - const char *addr, int rcvbuf) +http_process(struct vtclog *vl, struct vtc_sess *vsp, const char *spec, + int sock, int *sfd, const char *addr, int rcvbuf) { struct http *hp; int retval, oldbuf; @@ -1818,7 +1818,8 @@ http_process(struct vtclog *vl, const char *spec, int sock, int *sfd, (void)sfd; ALLOC_OBJ(hp, HTTP_MAGIC); AN(hp); - hp->fd = sock; + hp->sess = vsp; + hp->sess->fd = sock; hp->timeout = vtc_maxdur * 1000 / 2; if (rcvbuf) { @@ -1827,12 +1828,12 @@ http_process(struct vtclog *vl, const char *spec, int sock, int *sfd, hp->rcvbuf = rcvbuf; oldbuf = 0; - AZ(getsockopt(hp->fd, SOL_SOCKET, SO_RCVBUF, &oldbuf, &intlen)); - AZ(setsockopt(hp->fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, intlen)); - AZ(getsockopt(hp->fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, &intlen)); + AZ(getsockopt(hp->sess->fd, SOL_SOCKET, SO_RCVBUF, &oldbuf, &intlen)); + AZ(setsockopt(hp->sess->fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, intlen)); + AZ(getsockopt(hp->sess->fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, &intlen)); vtc_log(vl, 3, "-rcvbuf fd=%d old=%d new=%d actual=%d", - hp->fd, oldbuf, hp->rcvbuf, rcvbuf); + hp->sess->fd, oldbuf, hp->rcvbuf, rcvbuf); } hp->nrxbuf = 2048*1024; @@ -1868,7 +1869,7 @@ http_process(struct vtclog *vl, const char *spec, int sock, int *sfd, } pthread_cleanup_push(http_process_cleanup, hp); parse_string(spec, http_cmds, hp, vl); - retval = hp->fd; + retval = hp->sess->fd; pthread_cleanup_pop(0); http_process_cleanup(hp); return (retval); diff --git a/bin/varnishtest/vtc_http.h b/bin/varnishtest/vtc_http.h index d1690a4a3..6e5bcbdf5 100644 --- a/bin/varnishtest/vtc_http.h +++ b/bin/varnishtest/vtc_http.h @@ -30,11 +30,23 @@ #define MAX_HDR 50 +struct vtc_sess { + unsigned magic; +#define VTC_SESS_MAGIC 0x932bd565 + struct vtclog *vl; + char *name; + int repeat; + int keepalive; + int fd; + + ssize_t rcvbuf; +}; + struct http { unsigned magic; #define HTTP_MAGIC 0x2f02169c - int fd; int *sfd; + struct vtc_sess *sess; int timeout; struct vtclog *vl; @@ -76,3 +88,7 @@ struct http { uint64_t iws; int64_t ws; }; + +int http_process(struct vtclog *vl, struct vtc_sess *vsp, const char *spec, + int sock, int *sfd, const char *addr, int rcvbuf); + diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index b47295af1..ebf17d23a 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -156,7 +156,7 @@ http_write(const struct http *hp, int lvl, AN(pfx); vtc_dump(hp->vl, lvl, pfx, buf, s); - l = write(hp->fd, buf, s); + l = write(hp->sess->fd, buf, s); if (l != s) vtc_log(hp->vl, hp->fatal, "Write failed: (%zd vs %d) %s", l, s, strerror(errno)); @@ -172,7 +172,7 @@ get_bytes(const struct http *hp, char *buf, int n) AN(buf); while (n > 0) { - pfd[0].fd = hp->fd; + pfd[0].fd = hp->sess->fd; pfd[0].events = POLLIN; pfd[0].revents = 0; i = poll(pfd, 1, hp->timeout); @@ -181,26 +181,26 @@ get_bytes(const struct http *hp, char *buf, int n) if (i == 0) vtc_log(hp->vl, 3, "HTTP2 rx timeout (fd:%d %u ms)", - hp->fd, hp->timeout); + hp->sess->fd, hp->timeout); if (i < 0) vtc_log(hp->vl, 3, "HTTP2 rx failed (fd:%d poll: %s)", - hp->fd, strerror(errno)); + hp->sess->fd, strerror(errno)); if (i <= 0) return (i); - i = read(hp->fd, buf, n); + i = read(hp->sess->fd, buf, n); if (!(pfd[0].revents & POLLIN)) vtc_log(hp->vl, 4, "HTTP2 rx poll (fd:%d revents: %x n=%d, i=%d)", - hp->fd, pfd[0].revents, n, i); + hp->sess->fd, pfd[0].revents, n, i); if (i == 0) vtc_log(hp->vl, 3, "HTTP2 rx EOF (fd:%d read: %s)", - hp->fd, strerror(errno)); + hp->sess->fd, strerror(errno)); if (i < 0) vtc_log(hp->vl, 3, "HTTP2 rx failed (fd:%d read: %s)", - hp->fd, strerror(errno)); + hp->sess->fd, strerror(errno)); if (i <= 0) return (i); n -= i; @@ -334,14 +334,14 @@ write_frame(struct http *hp, const struct frame *f, const unsigned lock) if (lock) AZ(pthread_mutex_lock(&hp->mtx)); - l = write(hp->fd, hdr, sizeof(hdr)); + l = write(hp->sess->fd, hdr, sizeof(hdr)); if (l != sizeof(hdr)) vtc_log(hp->vl, hp->fatal, "Write failed: (%zd vs %zd) %s", l, sizeof(hdr), strerror(errno)); if (f->size) { AN(f->data); - l = write(hp->fd, f->data, f->size); + l = write(hp->sess->fd, f->data, f->size); if (l != f->size) vtc_log(hp->vl, hp->fatal, "Write failed: (%zd vs %d) %s", diff --git a/bin/varnishtest/vtc_sess.c b/bin/varnishtest/vtc_sess.c index 1b06b81dd..f1813e819 100644 --- a/bin/varnishtest/vtc_sess.c +++ b/bin/varnishtest/vtc_sess.c @@ -37,17 +37,7 @@ #include #include "vtc.h" - -struct vtc_sess { - unsigned magic; -#define VTC_SESS_MAGIC 0x932bd565 - struct vtclog *vl; - char *name; - int repeat; - int keepalive; - - ssize_t rcvbuf; -}; +#include "vtc_http.h" struct thread_arg { unsigned magic; @@ -113,14 +103,14 @@ Sess_GetOpt(struct vtc_sess *vsp, char * const **avp) } int -sess_process(struct vtclog *vl, const struct vtc_sess *vsp, +sess_process(struct vtclog *vl, struct vtc_sess *vsp, const char *spec, int sock, int *sfd, const char *addr) { int rv; CHECK_OBJ_NOTNULL(vsp, VTC_SESS_MAGIC); - rv = http_process(vl, spec, sock, sfd, addr, vsp->rcvbuf); + rv = http_process(vl, vsp, spec, sock, sfd, addr, vsp->rcvbuf); return (rv); } From phk at FreeBSD.org Mon Oct 19 08:35:07 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 19 Oct 2020 08:35:07 +0000 (UTC) Subject: [master] d604cdbaa Give HTTP2 streams their own log + identifier on the form "[cs]%d.%d" Message-ID: <20201019083507.74227119751@lists.varnish-cache.org> commit d604cdbaac4073be48aef78b47474cd902a84b7c Author: Poul-Henning Kamp Date: Mon Oct 19 08:33:32 2020 +0000 Give HTTP2 streams their own log + identifier on the form "[cs]%d.%d" diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index af2d37bd7..9983527db 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -115,7 +115,7 @@ void cmd_server_gen_vcl(struct vsb *vsb); void cmd_server_gen_haproxy_conf(struct vsb *vsb); void vtc_loginit(char *buf, unsigned buflen); -struct vtclog *vtc_logopen(const char *id); +struct vtclog *vtc_logopen(const char *id, ...) v_printflike_(1, 2); void vtc_logclose(void *arg); void vtc_log(struct vtclog *vl, int lvl, const char *fmt, ...) v_printflike_(3, 4); diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index 4eccf309d..6af18d252 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -137,7 +137,7 @@ barrier_sock_thread(void *priv) AZ(pthread_mutex_lock(&b->mtx)); - vl = vtc_logopen(b->name); + vl = vtc_logopen("%s", b->name); pthread_cleanup_push(vtc_logclose, vl); sock = VTCP_listen_on("127.0.0.1:0", NULL, b->expected, &err); diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c index 41cc3a581..ab22147f5 100644 --- a/bin/varnishtest/vtc_client.c +++ b/bin/varnishtest/vtc_client.c @@ -233,7 +233,7 @@ client_new(const char *name) ALLOC_OBJ(c, CLIENT_MAGIC); AN(c); REPLACE(c->name, name); - c->vl = vtc_logopen(name); + c->vl = vtc_logopen("%s", name); AN(c->vl); c->vsp = Sess_New(c->vl, name); AN(c->vsp); diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index c650f81b5..b42831798 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -565,7 +565,7 @@ haproxy_new(const char *name) h->args = VSB_new_auto(); - h->vl = vtc_logopen(name); + h->vl = vtc_logopen("%s", name); AN(h->vl); h->filename = getenv(HAPROXY_PROGRAM_ENV_VAR); diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index ebf17d23a..fda5143e0 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -93,6 +93,7 @@ struct stream { unsigned magic; #define STREAM_MAGIC 0x63f1fac2 uint32_t id; + struct vtclog *vl; char *spec; char *name; VTAILQ_ENTRY(stream) list; @@ -134,14 +135,14 @@ clean_headers(struct hpk_hdr *h) #define ONLY_H2_CLIENT(hp, av) \ do { \ if (hp->sfd != NULL) \ - vtc_fatal(hp->vl, \ + vtc_fatal(s->vl, \ "\"%s\" only possible in client", av[0]); \ } while (0) #define ONLY_H2_SERVER(hp, av) \ do { \ if (hp->sfd == NULL) \ - vtc_fatal(hp->vl, \ + vtc_fatal(s->vl, \ "\"%s\" only possible in server", av[0]); \ } while (0) @@ -316,17 +317,20 @@ clean_frame(struct frame **fp) } static void -write_frame(struct http *hp, const struct frame *f, const unsigned lock) +write_frame(const struct stream *sp, const struct frame *f, const unsigned lock) { + struct http *hp; ssize_t l; char hdr[9]; + CHECK_OBJ_NOTNULL(sp, STREAM_MAGIC); + hp = sp->hp; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); CHECK_OBJ_NOTNULL(f, FRAME_MAGIC); writeFrameHeader(hdr, f); - vtc_log(hp->vl, 3, "tx: stream: %d, type: %s (%d), " + vtc_log(sp->vl, 3, "tx: stream: %d, type: %s (%d), " "flags: 0x%02x, size: %d", f->stid, f->type < TYPE_MAX ? h2_types[f->type] : "?", @@ -336,14 +340,14 @@ write_frame(struct http *hp, const struct frame *f, const unsigned lock) AZ(pthread_mutex_lock(&hp->mtx)); l = write(hp->sess->fd, hdr, sizeof(hdr)); if (l != sizeof(hdr)) - vtc_log(hp->vl, hp->fatal, "Write failed: (%zd vs %zd) %s", + vtc_log(sp->vl, hp->fatal, "Write failed: (%zd vs %zd) %s", l, sizeof(hdr), strerror(errno)); if (f->size) { AN(f->data); l = write(hp->sess->fd, f->data, f->size); if (l != f->size) - vtc_log(hp->vl, hp->fatal, + vtc_log(sp->vl, hp->fatal, "Write failed: (%zd vs %d) %s", l, f->size, strerror(errno)); } @@ -401,7 +405,7 @@ parse_data(struct stream *s, struct frame *f) if (f->flags & PADDED) { f->md.padded = *((uint8_t *)data); if (f->md.padded >= size) { - vtc_log(hp->vl, hp->fatal, + vtc_log(s->vl, hp->fatal, "invalid padding: %d reported," "but size is only %d", f->md.padded, size); @@ -410,7 +414,7 @@ parse_data(struct stream *s, struct frame *f) } data++; size -= f->md.padded + 1; - vtc_log(hp->vl, 4, "padding: %3d", f->md.padded); + vtc_log(s->vl, 4, "padding: %3d", f->md.padded); } if (s->id) @@ -420,7 +424,7 @@ parse_data(struct stream *s, struct frame *f) if (!size) { AZ(data); - vtc_log(hp->vl, 4, "s%u - no data", s->id); + vtc_log(s->vl, 4, "s%u - no data", s->id); return; } @@ -491,7 +495,7 @@ parse_hdr(struct stream *s, struct frame *f, struct vsb *vsb) if (f->flags & PADDED && f->type != TYPE_CONTINUATION) { f->md.padded = *((uint8_t *)data); if (f->md.padded >= size) { - vtc_log(hp->vl, hp->fatal, + vtc_log(s->vl, hp->fatal, "invalid padding: %d reported," "but size is only %d", f->md.padded, size); @@ -500,7 +504,7 @@ parse_hdr(struct stream *s, struct frame *f, struct vsb *vsb) } shift += 1; size -= f->md.padded; - vtc_log(hp->vl, 4, "padding: %3d", f->md.padded); + vtc_log(s->vl, 4, "padding: %3d", f->md.padded); } if (f->type == TYPE_HEADERS && f->flags & PRIORITY){ @@ -513,8 +517,8 @@ parse_hdr(struct stream *s, struct frame *f, struct vsb *vsb) if (exclusive) exclusive_stream_dependency(s); - vtc_log(hp->vl, 4, "stream->dependency: %u", s->dependency); - vtc_log(hp->vl, 4, "stream->weight: %u", s->weight); + vtc_log(s->vl, 4, "stream->dependency: %u", s->dependency); + vtc_log(s->vl, 4, "stream->weight: %u", s->weight); } else if (f->type == TYPE_PUSH_PROMISE){ shift += 4; n = vbe32dec(f->data); @@ -536,7 +540,7 @@ parse_prio(struct stream *s, struct frame *f) CAST_OBJ_NOTNULL(hp, s->hp, HTTP_MAGIC);; if (f->size != 5) - vtc_fatal(hp->vl, "Size should be 5, but isn't (%d)", f->size); + vtc_fatal(s->vl, "Size should be 5, but isn't (%d)", f->size); buf = f->data; AN(buf); @@ -554,8 +558,8 @@ parse_prio(struct stream *s, struct frame *f) f->md.prio.weight = *buf; s->weight = f->md.prio.weight; - vtc_log(hp->vl, 3, "prio->stream: %u", f->md.prio.stream); - vtc_log(hp->vl, 3, "prio->weight: %u", f->md.prio.weight); + vtc_log(s->vl, 3, "prio->stream: %u", f->md.prio.stream); + vtc_log(s->vl, 3, "prio->weight: %u", f->md.prio.weight); } static void @@ -569,17 +573,17 @@ parse_rst(const struct stream *s, struct frame *f) CAST_OBJ_NOTNULL(hp, s->hp, HTTP_MAGIC);; if (f->size != 4) - vtc_fatal(hp->vl, "Size should be 4, but isn't (%d)", f->size); + vtc_fatal(s->vl, "Size should be 4, but isn't (%d)", f->size); err = vbe32dec(f->data); f->md.rst_err = err; - vtc_log(hp->vl, 2, "ouch"); + vtc_log(s->vl, 2, "ouch"); if (err <= ERR_MAX) buf = h2_errs[err]; else buf = "unknown"; - vtc_log(hp->vl, 4, "rst->err: %s (%d)", buf, err); + vtc_log(s->vl, 4, "rst->err: %s (%d)", buf, err); } @@ -596,7 +600,7 @@ parse_settings(const struct stream *s, struct frame *f) CAST_OBJ_NOTNULL(hp, s->hp, HTTP_MAGIC);; if (f->size % 6) - vtc_fatal(hp->vl, + vtc_fatal(s->vl, "Size should be a multiple of 6, but isn't (%d)", f->size); for (u = 0; u <= SETTINGS_MAX; u++) @@ -618,7 +622,7 @@ parse_settings(const struct stream *s, struct frame *f) assert(r == hpk_done); } - vtc_log(hp->vl, 4, "settings->%s (%u): %d", buf, t, v); + vtc_log(s->vl, 4, "settings->%s (%u): %d", buf, t, v); } } @@ -631,12 +635,12 @@ parse_ping(const struct stream *s, struct frame *f) CHECK_OBJ_NOTNULL(s, STREAM_MAGIC); CAST_OBJ_NOTNULL(hp, s->hp, HTTP_MAGIC);; if (f->size != 8) - vtc_fatal(hp->vl, "Size should be 8, but isn't (%d)", f->size); + vtc_fatal(s->vl, "Size should be 8, but isn't (%d)", f->size); f->md.ping.ack = f->flags & ACK; memcpy(f->md.ping.data, f->data, 8); f->md.ping.data[8] = '\0'; - vtc_log(hp->vl, 4, "ping->data: %s", f->md.ping.data); + vtc_log(s->vl, 4, "ping->data: %s", f->md.ping.data); } @@ -651,10 +655,10 @@ parse_goaway(const struct stream *s, struct frame *f) CAST_OBJ_NOTNULL(hp, s->hp, HTTP_MAGIC);; if (f->size < 8) - vtc_fatal(hp->vl, + vtc_fatal(s->vl, "Size should be at least 8, but isn't (%d)", f->size); if (f->data[0] & (1<<7)) - vtc_fatal(hp->vl, + vtc_fatal(s->vl, "First bit of data is reserved and should be 0"); stid = vbe32dec(f->data); @@ -675,10 +679,10 @@ parse_goaway(const struct stream *s, struct frame *f) memcpy(f->md.goaway.debug, f->data + 8, f->size - 8); } - vtc_log(hp->vl, 3, "goaway->laststream: %d", stid); - vtc_log(hp->vl, 3, "goaway->err: %s (%d)", err_buf, err); + vtc_log(s->vl, 3, "goaway->laststream: %d", stid); + vtc_log(s->vl, 3, "goaway->err: %s (%d)", err_buf, err); if (f->md.goaway.debug) - vtc_log(hp->vl, 3, "goaway->debug: %s", f->md.goaway.debug); + vtc_log(s->vl, 3, "goaway->debug: %s", f->md.goaway.debug); } static void @@ -691,15 +695,15 @@ parse_winup(const struct stream *s, struct frame *f) CAST_OBJ_NOTNULL(hp, s->hp, HTTP_MAGIC);; if (f->size != 4) - vtc_fatal(hp->vl, "Size should be 4, but isn't (%d)", f->size); + vtc_fatal(s->vl, "Size should be 4, but isn't (%d)", f->size); if (f->data[0] & (1<<7)) - vtc_log(hp->vl, s->hp->fatal, + vtc_log(s->vl, s->hp->fatal, "First bit of data is reserved and should be 0"); size = vbe32dec(f->data); f->md.winup_size = size; - vtc_log(hp->vl, 3, "winup->size: %d", size); + vtc_log(s->vl, 3, "winup->size: %d", size); } /* read a frame and queue it in the relevant stream, wait if not present yet. @@ -780,9 +784,10 @@ receive_frame(void *priv) } AZ(pthread_mutex_unlock(&hp->mtx)); + AN(s); if (expect_cont && (f->type != TYPE_CONTINUATION || expect_cont != s->id)) - vtc_fatal(hp->vl, "Expected CONTINUATION frame for " + vtc_fatal(s->vl, "Expected CONTINUATION frame for " "stream %u", expect_cont); /* parse the frame according to it type, and fill the metada */ @@ -875,7 +880,7 @@ do { \ #define CHECK_LAST_FRAME(TYPE) \ if (!f || f->type != TYPE_ ## TYPE) { \ - vtc_fatal(s->hp->vl, "Last frame was not of type " #TYPE); \ + vtc_fatal(s->vl, "Last frame was not of type " #TYPE); \ } #define RETURN_SETTINGS(idx) \ @@ -1080,7 +1085,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) if (!strncmp(spec, "frame.", 6)) { spec += 6; if (!f) - vtc_fatal(s->hp->vl, "No frame received yet."); + vtc_fatal(s->vl, "No frame received yet."); if (!strcmp(spec, "data")) { return (f->data); } else if (!strcmp(spec, "type")) { RETURN_BUFFED(f->type); } else if (!strcmp(spec, "size")) { RETURN_BUFFED(f->size); } @@ -1089,7 +1094,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) if (f->type != TYPE_DATA && f->type != TYPE_HEADERS && f->type != TYPE_PUSH_PROMISE) - vtc_fatal(s->hp->vl, + vtc_fatal(s->vl, "Last frame was not of type " "DATA, HEADERS or PUSH"); RETURN_BUFFED(f->md.padded); @@ -1645,7 +1650,7 @@ cmd_tx11obj(CMD_ARGS) f.size += 4; f.data = buf; HPK_FreeIter(iter); - write_frame(s->hp, &f, 1); + write_frame(s, &f, 1); free(buf); if (!body) @@ -1654,7 +1659,7 @@ cmd_tx11obj(CMD_ARGS) INIT_FRAME(f, DATA, bodylen, s->id, END_STREAM); f.data = body; - write_frame(s->hp, &f, 1); + write_frame(s, &f, 1); free(body); } @@ -1739,7 +1744,7 @@ cmd_txdata(CMD_ARGS) f.size = strlen(body); f.data = body; } - write_frame(s->hp, &f, 1); + write_frame(s, &f, 1); free(body); free(pad); free(data); @@ -1787,7 +1792,7 @@ cmd_txrst(CMD_ARGS) err = htonl(err); f.data = (void *)&err; - write_frame(s->hp, &f, 1); + write_frame(s, &f, 1); } /* SECTION: stream.spec.prio_txprio txprio @@ -1842,7 +1847,7 @@ cmd_txprio(CMD_ARGS) vbe32enc(buf, (stid | exclusive)); buf[4] = s->weight; - write_frame(s->hp, &f, 1); + write_frame(s, &f, 1); } #define PUT_KV(av, vl, name, val, code) \ @@ -1941,7 +1946,8 @@ cmd_txsettings(CMD_ARGS) if (*av != NULL) vtc_fatal(vl, "Unknown txsettings spec: %s\n", *av); - write_frame(hp, &f, 0); + AN(s->hp); + write_frame(s, &f, 0); AZ(pthread_mutex_unlock(&hp->mtx)); } @@ -1984,7 +1990,7 @@ cmd_txping(CMD_ARGS) vtc_fatal(vl, "Unknown txping spec: %s\n", *av); if (!f.data) f.data = buf; - write_frame(s->hp, &f, 1); + write_frame(s, &f, 1); } /* @@ -2052,7 +2058,7 @@ cmd_txgoaway(CMD_ARGS) } vbe32enc(f.data, ls); vbe32enc(f.data + 4, err); - write_frame(s->hp, &f, 1); + write_frame(s, &f, 1); free(f.data); } @@ -2101,7 +2107,7 @@ cmd_txwinup(CMD_ARGS) size = htonl(size); f.data = (void *)&size; - write_frame(hp, &f, 1); + write_frame(s, &f, 1); } static struct frame * @@ -2451,7 +2457,7 @@ cmd_rxframe(CMD_ARGS) (void)av; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); if (rxstuff(s) == NULL) - vtc_fatal(s->hp->vl, "No frame received"); + vtc_fatal(s->vl, "No frame received"); } static void @@ -2523,7 +2529,7 @@ cmd_write_body(CMD_ARGS) AZ(av[2]); AZ(strcmp(av[0], "write_body")); if (VFIL_writefile(NULL, av[1], s->body, s->bodylen) != 0) - vtc_fatal(s->hp->vl, "failed to write body: %s (%d)", + vtc_fatal(s->vl, "failed to write body: %s (%d)", strerror(errno), errno); } @@ -2573,8 +2579,8 @@ stream_thread(void *priv) struct stream *s; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); - parse_string(s->spec, stream_cmds, s, s->hp->vl); - vtc_log(s->hp->vl, 2, "Ending stream %u", s->id); + parse_string(s->spec, stream_cmds, s, s->vl); + vtc_log(s->vl, 2, "Ending stream %u", s->id); return (NULL); } /********************************************************************** @@ -2591,16 +2597,17 @@ stream_new(const char *name, struct http *h) AN(s); AZ(pthread_cond_init(&s->cond, NULL)); REPLACE(s->name, name); - AN(name); + AN(s->name); VTAILQ_INIT(&s->fq); s->ws = h->iws; + s->vl = vtc_logopen("%s.%s", h->sess->name, name); s->weight = 16; s->dependency = 0; - STRTOU32(s->id, name, p, h->vl, "stream"); + STRTOU32(s->id, name, p, s->vl, "stream"); if (s->id & (1U << 31)) - vtc_fatal(h->vl, "Stream id must be a 31-bits integer " + vtc_fatal(s->vl, "Stream id must be a 31-bits integer " "(found %s)", name); CHECK_OBJ_NOTNULL(h, HTTP_MAGIC); @@ -2628,6 +2635,7 @@ stream_delete(struct stream *s) VTAILQ_REMOVE(&s->fq, f, list); clean_frame(&f); } + vtc_logclose(s->vl); clean_headers(s->req); clean_headers(s->resp); AZ(s->frame); @@ -2645,7 +2653,7 @@ static void stream_start(struct stream *s) { CHECK_OBJ_NOTNULL(s, STREAM_MAGIC); - vtc_log(s->hp->vl, 2, "Starting stream %p", s); + vtc_log(s->hp->vl, 2, "Starting stream %s (%p)", s->name, s); AZ(pthread_create(&s->tp, NULL, stream_thread, s)); s->running = 1; } diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c index 645a5f11a..c60eab94b 100644 --- a/bin/varnishtest/vtc_log.c +++ b/bin/varnishtest/vtc_log.c @@ -47,7 +47,7 @@ static unsigned vtclog_left; struct vtclog { unsigned magic; #define VTCLOG_MAGIC 0x82731202 - const char *id; + char *id; struct vsb *vsb; pthread_mutex_t mtx; int act; @@ -77,13 +77,19 @@ static double t0; struct vtclog * -vtc_logopen(const char *id) +vtc_logopen(const char *fmt, ...) { struct vtclog *vl; + va_list ap; + char buf[BUFSIZ]; + + va_start(ap, fmt); + vbprintf(buf, fmt, ap); + va_end(ap); ALLOC_OBJ(vl, VTCLOG_MAGIC); AN(vl); - vl->id = id; + REPLACE(vl->id, buf); vl->vsb = VSB_new_auto(); AZ(pthread_mutex_init(&vl->mtx, NULL)); AZ(pthread_setspecific(log_key, vl)); @@ -100,6 +106,7 @@ vtc_logclose(void *arg) AZ(pthread_setspecific(log_key, NULL)); VSB_destroy(&vl->vsb); AZ(pthread_mutex_destroy(&vl->mtx)); + REPLACE(vl->id, NULL); FREE_OBJ(vl); } @@ -130,7 +137,7 @@ vtc_leadinv(const struct vtclog *vl, int lvl, const char *fmt, va_list ap) assert(lvl < (int)NLEAD); assert(lvl >= 0); - VSB_printf(vl->vsb, "%s %-4s ", + VSB_printf(vl->vsb, "%s %-5s ", lead[lvl < 0 ? 1: lvl], vl->id); if (fmt != NULL) (void)VSB_vprintf(vl->vsb, fmt, ap); @@ -162,7 +169,7 @@ vtc_log_emit(const struct vtclog *vl) if (t_last != t_this) { assert(vtclog_left > 25); i = snprintf(vtclog_buf, vtclog_left, - "**** dT %d.%03d\n", t_this / 1000, t_this % 1000); + "**** dT %d.%03d\n", t_this / 1000, t_this % 1000); t_last = t_this; vtclog_buf += i; vtclog_left -= i; @@ -224,7 +231,7 @@ vtc_dump(struct vtclog *vl, int lvl, const char *pfx, const char *str, int len) if (str == NULL) vtc_leadin(vl, lvl, "%s(null)\n", pfx); else { - bprintf(buf, "%s %-4s %s|", + bprintf(buf, "%s %-5s %s|", lead[lvl < 0 ? 1: lvl], vl->id, pfx); if (len < 0) len = strlen(str); diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 4719ad034..32a64d7aa 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -214,7 +214,7 @@ logexp_new(const char *name, const char *varg) ALLOC_OBJ(le, LOGEXP_MAGIC); AN(le); REPLACE(le->name, name); - le->vl = vtc_logopen(name); + le->vl = vtc_logopen("%s", name); VTAILQ_INIT(&le->tests); le->d_arg = 0; diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index f8ad2baa8..c1b4835d7 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -386,7 +386,7 @@ process_new(const char *name) REPLACE(p->name, name); AZ(pthread_mutex_init(&p->mtx, NULL)); - p->vl = vtc_logopen(name); + p->vl = vtc_logopen("%s", name); AN(p->vl); PROCESS_EXPAND(dir, "${tmpdir}/%s", name); diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index 4043c27da..0f44c75c3 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -83,7 +83,7 @@ server_new(const char *name, struct vtclog *vl) ALLOC_OBJ(s, SERVER_MAGIC); AN(s); REPLACE(s->name, name); - s->vl = vtc_logopen(s->name); + s->vl = vtc_logopen("%s", s->name); AN(s->vl); s->vsp = Sess_New(s->vl, name); AN(s->vsp); @@ -302,7 +302,7 @@ server_dispatch_wrk(void *priv) CAST_OBJ_NOTNULL(s, priv, SERVER_MAGIC); assert(s->sock < 0); - vl = vtc_logopen(s->name); + vl = vtc_logopen("%s", s->name); pthread_cleanup_push(vtc_logclose, vl); fd = s->fd; @@ -334,7 +334,7 @@ server_dispatch_thread(void *priv) CAST_OBJ_NOTNULL(s, priv, SERVER_MAGIC); assert(s->sock >= 0); - vl = vtc_logopen(s->name); + vl = vtc_logopen("%s", s->name); pthread_cleanup_push(vtc_logclose, vl); vtc_log(vl, 2, "Dispatch started on %s", s->listen); diff --git a/bin/varnishtest/vtc_sess.c b/bin/varnishtest/vtc_sess.c index f1813e819..d9766bf28 100644 --- a/bin/varnishtest/vtc_sess.c +++ b/bin/varnishtest/vtc_sess.c @@ -128,7 +128,7 @@ sess_thread(void *priv) vsp = ta.vsp; CHECK_OBJ_NOTNULL(vsp, VTC_SESS_MAGIC); - vl = vtc_logopen(vsp->name); + vl = vtc_logopen("%s", vsp->name); pthread_cleanup_push(vtc_logclose, vl); assert(vsp->repeat > 0); diff --git a/bin/varnishtest/vtc_syslog.c b/bin/varnishtest/vtc_syslog.c index c388ac8e6..aa81095f7 100644 --- a/bin/varnishtest/vtc_syslog.c +++ b/bin/varnishtest/vtc_syslog.c @@ -246,7 +246,7 @@ syslog_new(const char *name, struct vtclog *vl) ALLOC_OBJ(s, SYSLOG_SRV_MAGIC); AN(s); REPLACE(s->name, name); - s->vl = vtc_logopen(s->name); + s->vl = vtc_logopen("%s", s->name); AN(s->vl); bprintf(s->bind, "%s", "127.0.0.1 0"); @@ -255,7 +255,7 @@ syslog_new(const char *name, struct vtclog *vl) s->lvl = -1; s->timeout = vtc_maxdur * .5; // XXX - vl = vtc_logopen(s->name); + vl = vtc_logopen("%s", s->name); AN(vl); s->rxbuf_sz = s->rxbuf_left = 2048*1024; diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 78705f897..8bb701baa 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -320,7 +320,7 @@ varnish_new(const char *name) REPLACE(v->jail, ""); - v->vl = vtc_logopen(name); + v->vl = vtc_logopen("%s", name); AN(v->vl); vsb = macro_expandf(v->vl, "${tmpdir}/%s", name); From phk at FreeBSD.org Mon Oct 19 09:29:07 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 19 Oct 2020 09:29:07 +0000 (UTC) Subject: [master] 16848b72a Register the current commands on the current vtclog, and complain on mismatch. Message-ID: <20201019092907.8B8944F83@lists.varnish-cache.org> commit 16848b72a207b4699fec88b3659ac28fb4906945 Author: Poul-Henning Kamp Date: Mon Oct 19 08:53:20 2020 +0000 Register the current commands on the current vtclog, and complain on mismatch. diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index b6346f398..4968b4803 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -89,7 +89,7 @@ static const struct cmds global_cmds[] = { { NULL, NULL } }; -static const struct cmds cmds[] = { +static const struct cmds top_cmds[] = { #define CMD_TOP(n) { #n, cmd_##n }, #include "cmds.h" { NULL, NULL } @@ -319,6 +319,7 @@ parse_string(const char *spec, const struct cmds *cmd, void *priv, unsigned n, m; const struct cmds *cp; + vtc_log_chk_cmd(vl, cmd); AN(spec); buf = strdup(spec); AN(buf); @@ -492,7 +493,7 @@ fail_out(void) vtc_stop = 1; vtc_log(vltop, 1, "RESETTING after %s", tfn); reset_cmds(global_cmds); - reset_cmds(cmds); + reset_cmds(top_cmds); vtc_error |= old_err; if (vtc_error) @@ -525,6 +526,7 @@ exec_file(const char *fn, const char *script, const char *tmpdir, vtc_loginit(logbuf, loglen); vltop = vtc_logopen("top"); AN(vltop); + vtc_log_set_cmd(vltop, top_cmds); vtc_log(vltop, 1, "TEST %s starting", fn); @@ -564,6 +566,6 @@ exec_file(const char *fn, const char *script, const char *tmpdir, vtc_stop = 0; vtc_thread = pthread_self(); - parse_string(script, cmds, NULL, vltop); + parse_string(script, top_cmds, NULL, vltop); return (fail_out()); } diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index 9983527db..103c8e58a 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -114,6 +114,8 @@ char * synth_body(const char *len, int rnd); void cmd_server_gen_vcl(struct vsb *vsb); void cmd_server_gen_haproxy_conf(struct vsb *vsb); +void vtc_log_set_cmd(struct vtclog *vl, const void *cmds); +void vtc_log_chk_cmd(struct vtclog *vl, const void *cmds); void vtc_loginit(char *buf, unsigned buflen); struct vtclog *vtc_logopen(const char *id, ...) v_printflike_(1, 2); void vtc_logclose(void *arg); diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index b42831798..d49d1555c 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -478,6 +478,7 @@ haproxy_cli_new(struct haproxy *h) AN(hc); hc->vl = h->vl; + vtc_log_set_cmd(hc->vl, haproxy_cli_cmds); hc->sock = -1; bprintf(hc->connect, "${%s_cli_sock}", h->name); @@ -500,6 +501,7 @@ haproxy_mcli_new(struct haproxy *h) AN(hc); hc->vl = h->vl; + vtc_log_set_cmd(hc->vl, haproxy_cli_cmds); hc->sock = -1; bprintf(hc->connect, "${%s_mcli_sock}", h->name); @@ -566,6 +568,7 @@ haproxy_new(const char *name) h->args = VSB_new_auto(); h->vl = vtc_logopen("%s", name); + vtc_log_set_cmd(h->vl, haproxy_cli_cmds); AN(h->vl); h->filename = getenv(HAPROXY_PROGRAM_ENV_VAR); diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 08ceabd52..a62c817c8 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -1822,6 +1822,7 @@ http_process(struct vtclog *vl, struct vtc_sess *vsp, const char *spec, hp->sess->fd = sock; hp->timeout = vtc_maxdur * 1000 / 2; + if (rcvbuf) { // XXX setsockopt() too late on SunOS // https://github.com/varnishcache/varnish-cache/pull/2980#issuecomment-486214661 @@ -1855,6 +1856,7 @@ http_process(struct vtclog *vl, struct vtc_sess *vsp, const char *spec, AN(hp->rem_port); hp->vl = vl; + vtc_log_set_cmd(hp->vl, http_cmds); hp->gziplevel = 0; hp->gzipresidual = -1; diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index fda5143e0..07d7339b5 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -2601,6 +2601,7 @@ stream_new(const char *name, struct http *h) VTAILQ_INIT(&s->fq); s->ws = h->iws; s->vl = vtc_logopen("%s.%s", h->sess->name, name); + vtc_log_set_cmd(s->vl, stream_cmds); s->weight = 16; s->dependency = 0; diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c index c60eab94b..5c7a41d64 100644 --- a/bin/varnishtest/vtc_log.c +++ b/bin/varnishtest/vtc_log.c @@ -51,11 +51,25 @@ struct vtclog { struct vsb *vsb; pthread_mutex_t mtx; int act; + const void *cmds; }; static pthread_key_t log_key; static double t0; +void +vtc_log_set_cmd(struct vtclog *vl, const void *cmds) +{ + vl->cmds = cmds; +} + +void +vtc_log_chk_cmd(struct vtclog *vl, const void *cmds) +{ + if (vl->cmds != cmds) + vtc_log(vl, 4, "LOGCMDS mismatch %p vs %p", vl->cmds, cmds); +} + /**********************************************************************/ #define GET_VL(vl) \ @@ -323,4 +337,3 @@ vtc_loginit(char *buf, unsigned buflen) AZ(pthread_mutex_init(&vtclog_mtx, NULL)); AZ(pthread_key_create(&log_key, NULL)); } - diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 32a64d7aa..538387f31 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -171,6 +171,13 @@ struct logexp { static VTAILQ_HEAD(, logexp) logexps = VTAILQ_HEAD_INITIALIZER(logexps); +static cmd_f cmd_logexp_expect; + +static const struct cmds logexp_cmds[] = { + { "expect", cmd_logexp_expect }, + { NULL, NULL }, +}; + static void logexp_delete_tests(struct logexp *le) { @@ -215,6 +222,7 @@ logexp_new(const char *name, const char *varg) AN(le); REPLACE(le->name, name); le->vl = vtc_logopen("%s", name); + vtc_log_set_cmd(le->vl, logexp_cmds); VTAILQ_INIT(&le->tests); le->d_arg = 0; @@ -500,11 +508,6 @@ cmd_logexp_expect(CMD_ARGS) VTAILQ_INSERT_TAIL(&le->tests, test, list); } -static const struct cmds logexp_cmds[] = { - { "expect", cmd_logexp_expect }, - { NULL, NULL }, -}; - static void logexp_spec(struct logexp *le, const char *spec) { diff --git a/bin/varnishtest/vtc_syslog.c b/bin/varnishtest/vtc_syslog.c index aa81095f7..1bf58f78a 100644 --- a/bin/varnishtest/vtc_syslog.c +++ b/bin/varnishtest/vtc_syslog.c @@ -72,6 +72,21 @@ static pthread_mutex_t syslog_mtx; static VTAILQ_HEAD(, syslog_srv) syslogs = VTAILQ_HEAD_INITIALIZER(syslogs); +#define SYSLOGCMDS \ + CMD_SYSLOG(expect) \ + CMD_SYSLOG(recv) + +#define CMD_SYSLOG(nm) static cmd_f cmd_syslog_##nm; +SYSLOGCMDS +#undef CMD_SYSLOG + +static const struct cmds syslog_cmds[] = { +#define CMD_SYSLOG(n) { #n, cmd_syslog_##n }, +SYSLOGCMDS +#undef CMD_SYSLOG + { NULL, NULL } +}; + static const char * const syslog_levels[] = { "emerg", "alert", @@ -248,6 +263,7 @@ syslog_new(const char *name, struct vtclog *vl) REPLACE(s->name, name); s->vl = vtc_logopen("%s", s->name); AN(s->vl); + vtc_log_set_cmd(s->vl, syslog_cmds); bprintf(s->bind, "%s", "127.0.0.1 0"); s->repeat = 1; @@ -419,15 +435,6 @@ cmd_syslog_recv(CMD_ARGS) syslog_rx(s, lvl); } -static const struct cmds syslog_cmds[] = { -#define CMD_SYSLOG(n) { #n, cmd_syslog_##n }, - /* session */ - CMD_SYSLOG(expect) - CMD_SYSLOG(recv) -#undef CMD_SYSLOG - { NULL, NULL } -}; - /********************************************************************** * Syslog server thread */ From phk at FreeBSD.org Mon Oct 19 09:29:07 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 19 Oct 2020 09:29:07 +0000 (UTC) Subject: [master] ae32f5a01 Eliminate the cmd argument from parse_string argument, pick it up from vtclog. Message-ID: <20201019092907.B042D4F86@lists.varnish-cache.org> commit ae32f5a011cfad5cb57726c0434914ee11f94cbc Author: Poul-Henning Kamp Date: Mon Oct 19 09:17:19 2020 +0000 Eliminate the cmd argument from parse_string argument, pick it up from vtclog. diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index 4968b4803..e70e7a578 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -40,6 +40,7 @@ #include #include "vtc.h" +#include "vtc_log.h" #include "vav.h" #include "vrnd.h" @@ -308,8 +309,7 @@ macro_expand(struct vtclog *vl, const char *text) void -parse_string(const char *spec, const struct cmds *cmd, void *priv, - struct vtclog *vl) +parse_string(struct vtclog *vl, void *priv, const char *spec) { char *token_s[MAX_TOKENS], *token_e[MAX_TOKENS]; struct vsb *token_exp; @@ -319,7 +319,6 @@ parse_string(const char *spec, const struct cmds *cmd, void *priv, unsigned n, m; const struct cmds *cp; - vtc_log_chk_cmd(vl, cmd); AN(spec); buf = strdup(spec); AN(buf); @@ -439,12 +438,13 @@ parse_string(const char *spec, const struct cmds *cmd, void *priv, n = strtoul(token_s[1], NULL, 0); for (m = 0; m < n; m++) { vtc_log(vl, 4, "Loop #%u", m); - parse_string(token_s[2], cmd, priv, vl); + parse_string(vl, priv, token_s[2]); } continue; } - for (cp = cmd; cp->name != NULL; cp++) + AN(vl->cmds); + for (cp = vl->cmds; cp->name != NULL; cp++) if (!strcmp(token_s[0], cp->name)) break; @@ -458,7 +458,7 @@ parse_string(const char *spec, const struct cmds *cmd, void *priv, vtc_fatal(vl, "Unknown command: \"%s\"", token_s[0]); assert(cp->cmd != NULL); - cp->cmd(token_s, priv, cmd, vl); + cp->cmd(token_s, priv, vl->cmds, vl); } } @@ -566,6 +566,6 @@ exec_file(const char *fn, const char *script, const char *tmpdir, vtc_stop = 0; vtc_thread = pthread_self(); - parse_string(script, top_cmds, NULL, vltop); + parse_string(vltop, NULL, script); return (fail_out()); } diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index 103c8e58a..17fd4fe15 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -67,8 +67,7 @@ struct cmds { cmd_f *cmd; }; -void parse_string(const char *spec, const struct cmds *cmd, void *priv, - struct vtclog *vl); +void parse_string(struct vtclog *vl, void *priv, const char *spec); int fail_out(void); #define CMD_GLOBAL(n) cmd_f cmd_##n; @@ -108,14 +107,12 @@ Sess_Start_Thread( const char *spec ); - char * synth_body(const char *len, int rnd); void cmd_server_gen_vcl(struct vsb *vsb); void cmd_server_gen_haproxy_conf(struct vsb *vsb); -void vtc_log_set_cmd(struct vtclog *vl, const void *cmds); -void vtc_log_chk_cmd(struct vtclog *vl, const void *cmds); +void vtc_log_set_cmd(struct vtclog *vl, const struct cmds *cmds); void vtc_loginit(char *buf, unsigned buflen); struct vtclog *vtc_logopen(const char *id, ...) v_printflike_(1, 2); void vtc_logclose(void *arg); diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index d49d1555c..4167097e0 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -366,7 +366,7 @@ haproxy_cli_thread(void *priv) "CLI failed to open %s: %s", VSB_data(vsb), err); VTCP_blocking(fd); hc->sock = fd; - parse_string(hc->spec, haproxy_cli_cmds, hc, hc->vl); + parse_string(hc->vl, hc, hc->spec); vtc_log(hc->vl, 2, "CLI ending"); VSB_destroy(&vsb); return (NULL); diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index a62c817c8..c669ca476 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -1007,6 +1007,7 @@ cmd_http_upgrade(CMD_ARGS) char *h; struct http *hp; + (void)cmd; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); ONLY_SERVER(hp, av); AN(hp->sfd); @@ -1024,21 +1025,24 @@ cmd_http_upgrade(CMD_ARGS) if (!h) vtc_fatal(vl, "Req misses \"HTTP2-Settings\" header"); - - parse_string("txresp -status 101 " - "-hdr \"Connection: Upgrade\" " - "-hdr \"Upgrade: h2c\"\n", cmd, hp, vl); + parse_string(vl, hp, + "txresp -status 101" + " -hdr \"Connection: Upgrade\"" + " -hdr \"Upgrade: h2c\"\n" + ); b64_settings(hp, h); - parse_string("rxpri\n" - "stream 0 {\n" - "txsettings\n" - "rxsettings\n" - "txsettings -ack\n" - "rxsettings\n" - "expect settings.ack == true\n" - "} -start\n", cmd, hp, vl); + parse_string(vl, hp, + "rxpri\n" + "stream 0 {\n" + " txsettings\n" + " rxsettings\n" + " txsettings -ack\n" + " rxsettings\n" + " expect settings.ack == true\n" + "} -start\n" + ); } /********************************************************************** @@ -1235,19 +1239,23 @@ cmd_http_txreq(CMD_ARGS) http_write(hp, 4, "txreq"); if (up) { - parse_string("rxresp\n" - "expect resp.status == 101\n" - "expect resp.http.connection == Upgrade\n" - "expect resp.http.upgrade == h2c\n" - "txpri\n", http_cmds, hp, vl); + parse_string(vl, hp, + "rxresp\n" + "expect resp.status == 101\n" + "expect resp.http.connection == Upgrade\n" + "expect resp.http.upgrade == h2c\n" + "txpri\n" + ); b64_settings(hp, up); - parse_string("stream 0 {\n" - "txsettings\n" - "rxsettings\n" - "txsettings -ack\n" - "rxsettings\n" - "expect settings.ack == true" - "} -start\n", http_cmds, hp, vl); + parse_string(vl, hp, + "stream 0 {\n" + " txsettings\n" + " rxsettings\n" + " txsettings -ack\n" + " rxsettings\n" + " expect settings.ack == true" + "} -start\n" + ); } } @@ -1697,16 +1705,18 @@ cmd_http_stream(CMD_ARGS) if (!hp->h2) { vtc_log(hp->vl, 4, "Not in H/2 mode, do what's needed"); if (hp->sfd) - parse_string("rxpri", http_cmds, hp, vl); + parse_string(vl, hp, "rxpri"); else - parse_string("txpri", http_cmds, hp, vl); - parse_string("stream 0 {\n" - "txsettings\n" - "rxsettings\n" - "txsettings -ack\n" - "rxsettings\n" - "expect settings.ack == true" - "} -run\n", http_cmds, hp, vl); + parse_string(vl, hp, "txpri"); + parse_string(vl, hp, + "stream 0 {\n" + " txsettings\n" + " rxsettings\n" + " txsettings -ack\n" + " rxsettings\n" + " expect settings.ack == true" + "} -run\n" + ); } cmd_stream(av, hp, cmd, vl); } @@ -1870,7 +1880,7 @@ http_process(struct vtclog *vl, struct vtc_sess *vsp, const char *spec, hp->rem_path = strdup(addr); } pthread_cleanup_push(http_process_cleanup, hp); - parse_string(spec, http_cmds, hp, vl); + parse_string(vl, hp, spec); retval = hp->sess->fd; pthread_cleanup_pop(0); http_process_cleanup(hp); diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 07d7339b5..aa3e0d495 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -2579,7 +2579,7 @@ stream_thread(void *priv) struct stream *s; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); - parse_string(s->spec, stream_cmds, s, s->vl); + parse_string(s->vl, s, s->spec); vtc_log(s->vl, 2, "Ending stream %u", s->id); return (NULL); } diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c index 5c7a41d64..ec4dbda0c 100644 --- a/bin/varnishtest/vtc_log.c +++ b/bin/varnishtest/vtc_log.c @@ -37,6 +37,7 @@ #include #include "vtc.h" +#include "vtc_log.h" #include "vtim.h" @@ -44,32 +45,16 @@ static pthread_mutex_t vtclog_mtx; static char *vtclog_buf; static unsigned vtclog_left; -struct vtclog { - unsigned magic; -#define VTCLOG_MAGIC 0x82731202 - char *id; - struct vsb *vsb; - pthread_mutex_t mtx; - int act; - const void *cmds; -}; - static pthread_key_t log_key; static double t0; void -vtc_log_set_cmd(struct vtclog *vl, const void *cmds) +vtc_log_set_cmd(struct vtclog *vl, const struct cmds *cmds) { + AN(cmds); vl->cmds = cmds; } -void -vtc_log_chk_cmd(struct vtclog *vl, const void *cmds) -{ - if (vl->cmds != cmds) - vtc_log(vl, 4, "LOGCMDS mismatch %p vs %p", vl->cmds, cmds); -} - /**********************************************************************/ #define GET_VL(vl) \ diff --git a/bin/varnishtest/vtc_log.h b/bin/varnishtest/vtc_log.h new file mode 100644 index 000000000..0075a9001 --- /dev/null +++ b/bin/varnishtest/vtc_log.h @@ -0,0 +1,40 @@ +/*- + * Copyright (c) 2008-2011 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + + +struct vtclog { + unsigned magic; +#define VTCLOG_MAGIC 0x82731202 + char *id; + struct vsb *vsb; + pthread_mutex_t mtx; + int act; + const struct cmds *cmds; +}; diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 538387f31..486424f2d 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -515,7 +515,7 @@ logexp_spec(struct logexp *le, const char *spec) logexp_delete_tests(le); - parse_string(spec, logexp_cmds, le, le->vl); + parse_string(le->vl, le, spec); } void diff --git a/bin/varnishtest/vtc_syslog.c b/bin/varnishtest/vtc_syslog.c index 1bf58f78a..28f502638 100644 --- a/bin/varnishtest/vtc_syslog.c +++ b/bin/varnishtest/vtc_syslog.c @@ -452,7 +452,7 @@ syslog_thread(void *priv) for (i = 0; i < s->repeat; i++) { if (s->repeat > 1) vtc_log(s->vl, 3, "Iteration %d", i); - parse_string(s->spec, syslog_cmds, s, s->vl); + parse_string(s->vl, s, s->spec); vtc_log(s->vl, 3, "shutting fd %d", s->sock); } VUDP_close(&s->sock); From phk at FreeBSD.org Mon Oct 19 09:29:07 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 19 Oct 2020 09:29:07 +0000 (UTC) Subject: [master] 946ec369f Eliminate the now unused 'cmds' argument from cli command functions Message-ID: <20201019092907.DB7044F8A@lists.varnish-cache.org> commit 946ec369fe4d9535d516d9a577c18860765033e8 Author: Poul-Henning Kamp Date: Mon Oct 19 09:28:14 2020 +0000 Eliminate the now unused 'cmds' argument from cli command functions diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c index e70e7a578..d1a14cf8c 100644 --- a/bin/varnishtest/vtc.c +++ b/bin/varnishtest/vtc.c @@ -458,7 +458,7 @@ parse_string(struct vtclog *vl, void *priv, const char *spec) vtc_fatal(vl, "Unknown command: \"%s\"", token_s[0]); assert(cp->cmd != NULL); - cp->cmd(token_s, priv, vl->cmds, vl); + cp->cmd(token_s, priv, vl); } } @@ -471,7 +471,7 @@ reset_cmds(const struct cmds *cmd) { for (; cmd->name != NULL; cmd++) - cmd->cmd(NULL, NULL, NULL, NULL); + cmd->cmd(NULL, NULL, NULL); } /********************************************************************** diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index 17fd4fe15..1c716508f 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -54,11 +54,9 @@ } while (0) struct vtclog; -struct cmds; struct suckaddr; -#define CMD_ARGS \ - char * const *av, void *priv, const struct cmds *cmd, struct vtclog *vl +#define CMD_ARGS char * const *av, void *priv, struct vtclog *vl typedef void cmd_f(CMD_ARGS); diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index 6af18d252..ee334953f 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -400,7 +400,6 @@ cmd_barrier(CMD_ARGS) int r; (void)priv; - (void)cmd; if (av == NULL) { /* Reset and free */ diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c index ab22147f5..07a8d48c1 100644 --- a/bin/varnishtest/vtc_client.c +++ b/bin/varnishtest/vtc_client.c @@ -330,7 +330,6 @@ cmd_client(CMD_ARGS) struct client *c, *c2; (void)priv; - (void)cmd; if (av == NULL) { /* Reset and free */ diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index 4167097e0..b7f467cb7 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -200,7 +200,6 @@ cmd_haproxy_cli_send(CMD_ARGS) struct haproxy_cli *hc; int j; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hc, priv, HAPROXY_CLI_MAGIC); AZ(strcmp(av[0], "send")); @@ -302,7 +301,6 @@ cmd_haproxy_cli_expect(CMD_ARGS) int erroroffset, i, ret; char *cmp, *spec; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hc, priv, HAPROXY_CLI_MAGIC); AZ(strcmp(av[0], "expect")); @@ -1012,7 +1010,6 @@ cmd_haproxy(CMD_ARGS) struct haproxy *h, *h2; (void)priv; - (void)cmd; if (av == NULL) { /* Reset and free */ diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index c669ca476..dc286066a 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -357,7 +357,6 @@ cmd_http_expect(CMD_ARGS) char *cmp; const char *rhs; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); AZ(strcmp(av[0], "expect")); @@ -381,7 +380,6 @@ cmd_http_expect_pattern(CMD_ARGS) struct http *hp; char t = '0'; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); AZ(strcmp(av[0], "expect_pattern")); @@ -692,7 +690,6 @@ cmd_http_rxresp(CMD_ARGS) struct http *hp; int has_obj = 1; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); ONLY_CLIENT(hp, av); @@ -734,7 +731,6 @@ cmd_http_rxresphdrs(CMD_ARGS) { struct http *hp; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); ONLY_CLIENT(hp, av); @@ -761,7 +757,6 @@ cmd_http_gunzip(CMD_ARGS) struct http *hp; (void)av; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); @@ -965,7 +960,6 @@ cmd_http_txresp(CMD_ARGS) const char *reason = "OK"; char* body = NULL; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); ONLY_SERVER(hp, av); @@ -1007,7 +1001,6 @@ cmd_http_upgrade(CMD_ARGS) char *h; struct http *hp; - (void)cmd; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); ONLY_SERVER(hp, av); AN(hp->sfd); @@ -1059,7 +1052,6 @@ cmd_http_rxreq(CMD_ARGS) { struct http *hp; - (void)cmd; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); ONLY_SERVER(hp, av); AZ(strcmp(av[0], "rxreq")); @@ -1086,7 +1078,6 @@ cmd_http_rxreqhdrs(CMD_ARGS) { struct http *hp; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); AZ(strcmp(av[0], "rxreqhdrs")); @@ -1111,7 +1102,6 @@ cmd_http_rxreqbody(CMD_ARGS) { struct http *hp; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); ONLY_SERVER(hp, av); @@ -1138,7 +1128,6 @@ cmd_http_rxrespbody(CMD_ARGS) struct http *hp; int max = 0; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); ONLY_CLIENT(hp, av); @@ -1169,7 +1158,6 @@ cmd_http_rxchunk(CMD_ARGS) struct http *hp; int ll, i; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); ONLY_CLIENT(hp, av); @@ -1197,7 +1185,6 @@ cmd_http_txreq(CMD_ARGS) const char *up = NULL; unsigned nohost; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); ONLY_CLIENT(hp, av); @@ -1272,7 +1259,6 @@ cmd_http_recv(CMD_ARGS) int i, n; char u[32]; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); AN(av[1]); @@ -1301,7 +1287,6 @@ cmd_http_send(CMD_ARGS) struct http *hp; int i; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); AN(av[1]); @@ -1325,7 +1310,6 @@ cmd_http_send_n(CMD_ARGS) struct http *hp; int i, n, l; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); AN(av[1]); @@ -1355,7 +1339,6 @@ cmd_http_send_urgent(CMD_ARGS) struct http *hp; int i; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); AN(av[1]); @@ -1381,7 +1364,6 @@ cmd_http_sendhex(CMD_ARGS) struct vsb *vsb; struct http *hp; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); AN(av[1]); @@ -1406,7 +1388,6 @@ cmd_http_chunked(CMD_ARGS) { struct http *hp; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); AN(av[1]); @@ -1432,7 +1413,6 @@ cmd_http_chunkedlen(CMD_ARGS) char buf[16384]; struct http *hp; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); AN(av[1]); @@ -1472,7 +1452,6 @@ cmd_http_timeout(CMD_ARGS) struct http *hp; double d; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); AN(av[1]); @@ -1496,7 +1475,6 @@ cmd_http_expect_close(CMD_ARGS) char c; int i; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); AZ(av[1]); @@ -1539,7 +1517,6 @@ cmd_http_close(CMD_ARGS) { struct http *hp; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); ONLY_SERVER(hp, av); @@ -1563,7 +1540,6 @@ cmd_http_accept(CMD_ARGS) { struct http *hp; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); ONLY_SERVER(hp, av); @@ -1593,13 +1569,14 @@ cmd_http_fatal(CMD_ARGS) struct http *hp; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); + (void)vl; AZ(av[1]); - if (!strcmp(av[0], "fatal")) + if (!strcmp(av[0], "fatal")) { hp->fatal = 0; - else if (!strcmp(av[0], "non_fatal")) + } else { + assert(!strcmp(av[0], "non_fatal")); hp->fatal = -1; - else - vtc_fatal(vl, "XXX: fatal %s", cmd->name); + } } #define cmd_http_non_fatal cmd_http_fatal @@ -1621,7 +1598,7 @@ cmd_http_txpri(CMD_ARGS) { size_t l; struct http *hp; - (void)cmd; + CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); ONLY_CLIENT(hp, av); @@ -1651,7 +1628,7 @@ static void cmd_http_rxpri(CMD_ARGS) { struct http *hp; - (void)cmd; + CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); ONLY_SERVER(hp, av); @@ -1677,7 +1654,6 @@ cmd_http_settings(CMD_ARGS) char *p; struct http *hp; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); - (void)cmd; if (!hp->h2) vtc_fatal(hp->vl, "Only possible in H/2 mode"); @@ -1718,7 +1694,7 @@ cmd_http_stream(CMD_ARGS) "} -run\n" ); } - cmd_stream(av, hp, cmd, vl); + cmd_stream(av, hp, vl); } /* SECTION: client-server.spec.write_body @@ -1734,7 +1710,6 @@ cmd_http_write_body(CMD_ARGS) { struct http *hp; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(hp, priv, HTTP_MAGIC); AN(av[0]); diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index aa3e0d495..f2deff037 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -1249,7 +1249,6 @@ cmd_sendhex(CMD_ARGS) struct stream *s; struct vsb *vsb; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); CAST_OBJ_NOTNULL(hp, s->hp, HTTP_MAGIC); @@ -1417,7 +1416,6 @@ cmd_tx11obj(CMD_ARGS) struct hpk_hdr hdr; char *cmd_str = *av; char *b, *p; - (void)cmd; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); INIT_FRAME(f, CONTINUATION, 0, s->id, END_HEADERS); @@ -1692,7 +1690,6 @@ cmd_txdata(CMD_ARGS) struct frame f; char *body = NULL; char *data = NULL; - (void)cmd; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); @@ -1767,7 +1764,7 @@ cmd_txrst(CMD_ARGS) char *p; uint32_t err = 0; struct frame f; - (void)cmd; + CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); INIT_FRAME(f, RST_STREAM, 4, s->id, 0); @@ -1821,7 +1818,6 @@ cmd_txprio(CMD_ARGS) uint32_t exclusive = 0; uint8_t buf[5]; - (void)cmd; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); INIT_FRAME(f, PRIORITY, 5, s->id, 0); @@ -1898,7 +1894,6 @@ cmd_txsettings(CMD_ARGS) char buf[512]; char *cursor = buf; - (void)cmd; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); CAST_OBJ_NOTNULL(hp, s->hp, HTTP_MAGIC); @@ -1968,7 +1963,6 @@ cmd_txping(CMD_ARGS) struct frame f; char buf[8]; - (void)cmd; memset(buf, 0, 8); CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); INIT_FRAME(f, PING, 8, s->id, 0); @@ -2020,7 +2014,6 @@ cmd_txgoaway(CMD_ARGS) uint32_t ls = 0; struct frame f; - (void)cmd; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); INIT_FRAME(f, GOAWAY, 8, s->id, 0); @@ -2080,7 +2073,6 @@ cmd_txwinup(CMD_ARGS) char buf[8]; uint32_t size = 0; - (void)cmd; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); CAST_OBJ_NOTNULL(hp, s->hp, HTTP_MAGIC); memset(buf, 0, 8); @@ -2170,7 +2162,6 @@ cmd_rxhdrs(CMD_ARGS) unsigned rcv = 0; enum h2_type expect = TYPE_HEADERS; - (void)cmd; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); while (*++av) { @@ -2208,7 +2199,6 @@ cmd_rxcont(CMD_ARGS) unsigned long int times = 1; unsigned rcv = 0; - (void)cmd; (void)av; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); @@ -2259,7 +2249,6 @@ cmd_rxdata(CMD_ARGS) unsigned long int times = 1; unsigned rcv = 0; - (void)cmd; (void)av; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); @@ -2305,7 +2294,6 @@ cmd_rxmsg(CMD_ARGS) int end_stream; int rcv = 0; - (void)cmd; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); if (!strcmp(av[0], "rxreq")) @@ -2364,7 +2352,6 @@ cmd_rxpush(CMD_ARGS) unsigned rcv = 0; enum h2_type expect = TYPE_PUSH_PROMISE; - (void)cmd; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); while (*++av) { @@ -2396,7 +2383,6 @@ cmd_rxpush(CMD_ARGS) static void \ cmd_rx ## lctype(CMD_ARGS) { \ struct stream *s; \ - (void)cmd; \ (void)av; \ CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); \ s->frame = rxstuff(s); \ @@ -2452,7 +2438,7 @@ static void cmd_rxframe(CMD_ARGS) { struct stream *s; - (void)cmd; + (void)vl; (void)av; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); @@ -2470,7 +2456,6 @@ cmd_expect(CMD_ARGS) const char *rhs; char buf[20]; - (void)cmd; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); hp = s->hp; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); @@ -2501,7 +2486,6 @@ cmd_gunzip(CMD_ARGS) struct stream *s; (void)av; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); @@ -2521,7 +2505,6 @@ cmd_write_body(CMD_ARGS) { struct stream *s; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); AN(av[0]); @@ -2752,7 +2735,6 @@ cmd_stream(CMD_ARGS) struct stream *s; struct http *h; - (void)cmd; (void)vl; CAST_OBJ_NOTNULL(h, priv, HTTP_MAGIC); diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 486424f2d..5f7f006c5 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -452,7 +452,6 @@ cmd_logexp_expect(CMD_ARGS) struct logexp_test *test; char *end; - (void)cmd; CAST_OBJ_NOTNULL(le, priv, LOGEXP_MAGIC); if (av[1] == NULL || av[2] == NULL || av[3] == NULL) vtc_fatal(vl, "Syntax error"); @@ -525,7 +524,6 @@ cmd_logexpect(CMD_ARGS) int i; (void)priv; - (void)cmd; if (av == NULL) { /* Reset and free */ diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index db604ecc6..ca85fabd2 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -69,7 +69,6 @@ cmd_vtest(CMD_ARGS) { (void)priv; - (void)cmd; (void)vl; if (av == NULL) @@ -91,7 +90,6 @@ cmd_varnishtest(CMD_ARGS) { (void)priv; - (void)cmd; (void)vl; if (av == NULL) @@ -215,7 +213,6 @@ cmd_shell(CMD_ARGS) int ok = 0; (void)priv; - (void)cmd; if (av == NULL) return; @@ -262,7 +259,6 @@ void v_matchproto_(cmd_f) cmd_err_shell(CMD_ARGS) { (void)priv; - (void)cmd; if (av == NULL) return; @@ -295,7 +291,6 @@ cmd_setenv(CMD_ARGS) int force; (void)priv; - (void)cmd; if (av == NULL) return; @@ -330,7 +325,6 @@ cmd_delay(CMD_ARGS) double f; (void)priv; - (void)cmd; if (av == NULL) return; AN(av[1]); @@ -420,7 +414,6 @@ cmd_feature(CMD_ARGS) int good; (void)priv; - (void)cmd; if (av == NULL) return; diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index c1b4835d7..3ef102ffa 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -929,7 +929,6 @@ cmd_process(CMD_ARGS) int spec_set = 0; (void)priv; - (void)cmd; if (av == NULL) { /* Reset and free */ diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index 0f44c75c3..f98f2b7bb 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -474,7 +474,6 @@ cmd_server(CMD_ARGS) struct server *s; (void)priv; - (void)cmd; if (av == NULL) { /* Reset and free */ diff --git a/bin/varnishtest/vtc_syslog.c b/bin/varnishtest/vtc_syslog.c index 28f502638..f57bd38d1 100644 --- a/bin/varnishtest/vtc_syslog.c +++ b/bin/varnishtest/vtc_syslog.c @@ -387,7 +387,6 @@ cmd_syslog_expect(CMD_ARGS) char *cmp, *spec; (void)vl; - (void)cmd; CAST_OBJ_NOTNULL(s, priv, SYSLOG_SRV_MAGIC); AZ(strcmp(av[0], "expect")); av++; @@ -423,7 +422,6 @@ cmd_syslog_recv(CMD_ARGS) struct syslog_srv *s; CAST_OBJ_NOTNULL(s, priv, SYSLOG_SRV_MAGIC); - (void)cmd; (void)vl; AZ(strcmp(av[0], "recv")); av++; @@ -555,7 +553,6 @@ cmd_syslog(CMD_ARGS) struct syslog_srv *s; (void)priv; - (void)cmd; if (av == NULL) { /* Reset and free */ diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 8bb701baa..5476806bc 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -1074,7 +1074,6 @@ cmd_varnish(CMD_ARGS) struct varnish *v, *v2; (void)priv; - (void)cmd; if (av == NULL) { /* Reset and free */ From phk at FreeBSD.org Mon Oct 19 09:38:07 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 19 Oct 2020 09:38:07 +0000 (UTC) Subject: [master] 597a34c2d Fix a Flexelint warning which has bothered me for ages. Message-ID: <20201019093807.6EBD358DA@lists.varnish-cache.org> commit 597a34c2d8b6e8d9e97bb24ccf14361e178440b5 Author: Poul-Henning Kamp Date: Mon Oct 19 09:32:22 2020 +0000 Fix a Flexelint warning which has bothered me for ages. The message makes very little sense when you read it, and it was only a pure accident which finally resolved what it means: Info 768: global struct member 'backend' (line 51, file cache/cache_backend.h) not referenced cache/cache_backend.h 51 Info 830: Location cited in prior message If you have: struct backend; void somefunc(struct backend *); struct backend { ... }; All is fine. But without the `somefunc` prototype, the first vacuous mention of `struct backend` is unused, and FlexeLint uses the name *of* the struct as the name of a field *in* the struct, and compounds the confusion by emitting the message on the line where struct backend is finally flushed out. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 4497d20dd..22acb7c7f 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -90,7 +90,6 @@ enum { struct VSC_lck; struct VSC_main; struct VSC_main_wrk; -struct backend; struct ban; struct ban_proto; struct cli; From phk at FreeBSD.org Mon Oct 19 09:50:07 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 19 Oct 2020 09:50:07 +0000 (UTC) Subject: [master] b584e04ab Remember to tell autocrap about the new .h file Message-ID: <20201019095007.6A72B6066@lists.varnish-cache.org> commit b584e04abb3bb34b613ef729f33e65340b6403bd Author: Poul-Henning Kamp Date: Mon Oct 19 09:49:19 2020 +0000 Remember to tell autocrap about the new .h file diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am index 61c2be73d..2dd014f26 100644 --- a/bin/varnishtest/Makefile.am +++ b/bin/varnishtest/Makefile.am @@ -40,6 +40,7 @@ varnishtest_SOURCES = \ vtc_http.c \ vtc_http.h \ vtc_http2.c \ + vtc_log.h \ vtc_log.c \ vtc_logexp.c \ vtc_misc.c \ From dridi.boukelmoune at gmail.com Mon Oct 19 17:55:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 19 Oct 2020 17:55:06 +0000 (UTC) Subject: [master] c9fc38420 varnishtest: Tweak default h2 window parameters Message-ID: <20201019175506.C5D2F102BA1@lists.varnish-cache.org> commit c9fc384202b418faed3f1768c0caa60ec26555e0 Author: Dridi Boukelmoune Date: Mon Oct 19 19:29:23 2020 +0200 varnishtest: Tweak default h2 window parameters This change increases the initial size and reduces the low watermark. RFC7540 says this: > Flow-controlled frames from the sender and WINDOW_UPDATE frames from > the receiver are completely asynchronous with respect to each other. > This property allows a receiver to aggressively update the window > size kept by the sender to prevent streams from stalling. The default parameters are very much on the low-latency aggressive updates end of the spectrum, which increases asynchronicity at the expense of determinism in test cases. The tweaks made by varnishtest allows basic tests to send a few request bodies before being bothered by window update race conditions. Test cases that cover h2 flow control or anything else related to window updates may reset parameters or pick other specific values. This frees us from a bunch of barriers where the purpose of mitigating this race was rarely even documented. This successfully passed the following test locally: git grep -Fl +http2 -- '*.vtc' | xargs bin/varnishtest/varnishtest -i -n100 -j32 We can hope that h2 test cases will be overall more stable from now on. Refs #3442 diff --git a/bin/varnishtest/tests/r02305.vtc b/bin/varnishtest/tests/r02305.vtc index 7da2d486e..dfd60398a 100644 --- a/bin/varnishtest/tests/r02305.vtc +++ b/bin/varnishtest/tests/r02305.vtc @@ -1,7 +1,6 @@ varnishtest "#2305: h/2 reembark with a request body" barrier b1 cond 2 -barrier b2 sock 2 server s1 { rxreq @@ -16,15 +15,9 @@ varnish v1 -cliok "param.set debug +syncvsl" varnish v1 -cliok "param.set debug +waitinglist" varnish v1 -vcl+backend { - import vtc; sub vcl_recv { return (hash); } - sub vcl_deliver { - if (req.http.sync) { - vtc.barrier_sync("${b2_sock}"); - } - } } -start client c1 { @@ -35,11 +28,7 @@ client c1 { } -start stream 3 { barrier b1 sync - txreq -req POST -hdr sync 1 -body "foo" - rxwinup - # barrier b2 is here to make HEADERS vs WINDOW_UPDATE - # less racy - barrier b2 sync + txreq -req POST -body "foo" rxresp expect resp.status == 200 } -run diff --git a/bin/varnishtest/tests/r02679.vtc b/bin/varnishtest/tests/r02679.vtc index c0029782a..d1284e2f5 100644 --- a/bin/varnishtest/tests/r02679.vtc +++ b/bin/varnishtest/tests/r02679.vtc @@ -16,6 +16,7 @@ varnish v1 -vcl+backend { varnish v1 -cliok "param.set feature +http2" varnish v1 -cliok "param.set h2_rx_window_low_water 65535" +varnish v1 -cliok "param.reset h2_initial_window_size" client c1 { stream 1 { diff --git a/bin/varnishtest/tests/t02000.vtc b/bin/varnishtest/tests/t02000.vtc index b3cbc1a22..789b5d822 100644 --- a/bin/varnishtest/tests/t02000.vtc +++ b/bin/varnishtest/tests/t02000.vtc @@ -33,6 +33,7 @@ client c1 { } -run varnish v1 -cliok "param.set feature +http2" +varnish v1 -cliok "param.reset h2_initial_window_size" client c1 { stream 1 { @@ -67,8 +68,10 @@ varnish v1 -expect MEMPOOL.sess1.live == 0 process p1 -stop # shell {cat ${tmpdir}/vlog} -shell -match {1001 H2TxHdr c \[000006040000000000\]} \ - {cat ${tmpdir}/vlog} +# SETTINGS with default initial window size +shell -match {1001 H2TxHdr c \[000006040000000000\]} { + cat ${tmpdir}/vlog +} # While we're here, test sess.xid over H2 as well diff --git a/bin/varnishtest/tests/t02005.vtc b/bin/varnishtest/tests/t02005.vtc index 2bcd1da24..03c9a85ae 100644 --- a/bin/varnishtest/tests/t02005.vtc +++ b/bin/varnishtest/tests/t02005.vtc @@ -1,7 +1,5 @@ varnishtest "H2 POST" -barrier b1 cond 2 - barrier b2 sock 2 barrier b3 sock 2 @@ -9,7 +7,6 @@ server s1 { rxreq expect req.http.content-length == 7 expect req.http.transfer-encoding == - barrier b1 sync txresp -hdr "Content-Type: text/plain" -body response rxreq @@ -30,7 +27,7 @@ varnish v1 -cliok "param.set debug +syncvsl" logexpect l1 -v v1 -g raw { expect * 1001 ReqAcct "80 7 87 106 8 114" - expect * 1000 ReqAcct "45 8 53 72 22 94" + expect * 1000 ReqAcct "45 8 53 54 20 74" } -start client c1 { @@ -38,14 +35,8 @@ client c1 { txping rxping } -run - stream 0 { - rxwinup - } -start stream 1 { txreq -req POST -hdr content-type text/plain -hdr content-length 7 -body request - - rxwinup - barrier b1 sync # First, HTTP checks rxresp expect resp.http.content-Type == "text/plain" @@ -53,21 +44,18 @@ client c1 { # Then, payload checks expect resp.body == response } -run - stream 0 -wait } -run -client c1 { +client c2 { stream 0 { barrier b2 sync delay 1 barrier b3 sync - rxwinup } -start stream 1 { txreq -url "/a" -req POST -nostrend txdata -datalen 100 rxresp - rxwinup expect resp.status == 503 } -run stream 3 { diff --git a/bin/varnishtest/tests/t02006.vtc b/bin/varnishtest/tests/t02006.vtc index 30e4ac805..9056682d3 100644 --- a/bin/varnishtest/tests/t02006.vtc +++ b/bin/varnishtest/tests/t02006.vtc @@ -1,13 +1,10 @@ varnishtest "H2 POST w/ 100 Continue" -barrier b1 cond 2 - server s1 { rxreq expect req.http.content-length == expect req.http.transfer-encoding == chunked expect req.proto == HTTP/1.1 - barrier b1 sync txresp -hdr "Content-Type: text/plain" -body response } -start @@ -31,8 +28,6 @@ client c1 { txdata \ -data request - rxwinup - barrier b1 sync rxresp expect resp.status == 200 expect resp.http.content-Type == "text/plain" diff --git a/bin/varnishtest/tests/t02007.vtc b/bin/varnishtest/tests/t02007.vtc index b3fd521d5..f0b1f1abd 100644 --- a/bin/varnishtest/tests/t02007.vtc +++ b/bin/varnishtest/tests/t02007.vtc @@ -1,7 +1,5 @@ varnishtest "H2 Huge response headers" -barrier b1 sock 2 - server s1 { rxreq expect req.proto == HTTP/1.1 @@ -25,14 +23,7 @@ varnish v1 -cliok "param.set feature +http2" varnish v1 -cliok "param.set debug +syncvsl" varnish v1 -cliok "param.set debug +h2_nocheck" -varnish v1 -vcl+backend { - import vtc; - sub vcl_deliver { - if (req.url == "/1") { - vtc.barrier_sync("${b1_sock}"); - } - } -} -start +varnish v1 -vcl+backend {} -start client c1 { stream 0 { @@ -61,8 +52,6 @@ client c1 { txdata \ -data request - rxwinup - barrier b1 sync rxresp expect resp.status == 200 expect resp.http.content-Type == "text/plain" diff --git a/bin/varnishtest/tests/t02014.vtc b/bin/varnishtest/tests/t02014.vtc index 17489d362..e73477311 100644 --- a/bin/varnishtest/tests/t02014.vtc +++ b/bin/varnishtest/tests/t02014.vtc @@ -17,6 +17,7 @@ varnish v1 -vcl+backend { varnish v1 -cliok "param.set debug +syncvsl" varnish v1 -cliok "param.set feature +http2" +varnish v1 -cliok "param.reset h2_initial_window_size" client c1 { stream 0 { diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 5476806bc..1df5dd93d 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -427,6 +427,8 @@ varnish_launch(struct varnish *v) VSB_cat(vsb, " -p thread_pool_min=10"); VSB_cat(vsb, " -p debug=+vtc_mode"); VSB_cat(vsb, " -p vsl_mask=+Debug"); + VSB_cat(vsb, " -p h2_initial_window_size=1m"); + VSB_cat(vsb, " -p h2_rx_window_low_water=64k"); if (!v->has_a_arg) { VSB_printf(vsb, " -a '%s'", "127.0.0.1:0"); if (v->proto != NULL) From dridi.boukelmoune at gmail.com Tue Oct 20 05:38:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 20 Oct 2020 05:38:07 +0000 (UTC) Subject: [master] bc46b11d2 vrt: Make client.identity readable in backend tasks Message-ID: <20201020053807.26784114173@lists.varnish-cache.org> commit bc46b11d22088073e75554a07cc72912f834c69f Author: Dridi Boukelmoune Date: Fri Oct 9 15:57:43 2020 +0200 vrt: Make client.identity readable in backend tasks This puts client.identity at the same level as client.ip which happens to be its fall-back before being explicitly set. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 22acb7c7f..6a01cd5b5 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -440,6 +440,8 @@ struct busyobj { uint16_t err_code; const char *err_reason; + + const char *client_identity; }; diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index 1e77272c0..d80bd1ecc 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -132,6 +132,11 @@ VBO_GetBusyObj(struct worker *wrk, const struct req *req) bo->do_stream = 1; + if (req->client_identity != NULL) { + bo->client_identity = WS_Copy(bo->ws, req->client_identity, -1); + XXXAN(bo->client_identity); + } + bo->director_req = req->director_hint; bo->vcl = req->vcl; VCL_Ref(bo->vcl); diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 1f3138e1c..957d49cdb 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -296,11 +296,19 @@ VRT_r_beresp_uncacheable(VRT_CTX) VCL_STRING VRT_r_client_identity(VRT_CTX) { + const char *id; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); - if (ctx->req->client_identity != NULL) - return (ctx->req->client_identity); - return (SES_Get_String_Attr(ctx->req->sp, SA_CLIENT_IP)); + if (ctx->req != NULL) { + CHECK_OBJ(ctx->req, REQ_MAGIC); + id = ctx->req->client_identity; + } else { + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + id = ctx->bo->client_identity; + } + if (id != NULL) + return (id); + return (SES_Get_String_Attr(ctx->sp, SA_CLIENT_IP)); } VCL_VOID diff --git a/bin/varnishtest/tests/v00025.vtc b/bin/varnishtest/tests/v00025.vtc index d3d0cf684..790f24873 100644 --- a/bin/varnishtest/tests/v00025.vtc +++ b/bin/varnishtest/tests/v00025.vtc @@ -2,8 +2,10 @@ varnishtest "More VCL coverage" server s1 { rxreq + expect req.http.c_id == "Samuel B. Nobody" txresp rxreq + expect req.http.c_id == ${localhost} txresp } -start @@ -85,6 +87,7 @@ varnish v1 -syntax 4.0 -vcl+backend { } sub vcl_backend_fetch { + set bereq.http.c_id = client.identity; if (bereq.between_bytes_timeout < 10s) { set bereq.http.quick = "please"; } @@ -147,6 +150,10 @@ varnish v1 -syntax 4.0 -errvcl {Symbol not found: 'local.socket' (Only available } varnish v1 -syntax 4.1 -vcl+backend { + sub vcl_backend_fetch { + set bereq.http.c_id = client.identity; + } + sub vcl_backend_response { set beresp.http.B-Sess-XID = sess.xid; set beresp.http.B-Endpoint = local.endpoint; diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index a6e67d2d2..72098379a 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -117,7 +117,7 @@ client.identity Type: STRING - Readable from: client + Readable from: client, backend Writable from: client From dridi.boukelmoune at gmail.com Tue Oct 20 07:34:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 20 Oct 2020 07:34:06 +0000 (UTC) Subject: [master] 7f21d7ec4 vtc: Stabilize r02305.vtc Message-ID: <20201020073406.AF4F5116BAE@lists.varnish-cache.org> commit 7f21d7ec4d3c219a6e6313a7e1dd6349d0c8fdb1 Author: Dridi Boukelmoune Date: Tue Oct 20 09:30:40 2020 +0200 vtc: Stabilize r02305.vtc We don't need a delay, we need to sync operations. diff --git a/bin/varnishtest/tests/r02305.vtc b/bin/varnishtest/tests/r02305.vtc index dfd60398a..d5a40fe73 100644 --- a/bin/varnishtest/tests/r02305.vtc +++ b/bin/varnishtest/tests/r02305.vtc @@ -1,12 +1,14 @@ varnishtest "#2305: h/2 reembark with a request body" barrier b1 cond 2 +barrier b2 cond 2 +barrier b3 cond 2 server s1 { rxreq expect req.url == "/" barrier b1 sync - delay 2 + barrier b2 sync txresp } -start @@ -25,10 +27,13 @@ client c1 { txreq rxresp expect resp.status == 200 + barrier b3 sync } -start stream 3 { barrier b1 sync txreq -req POST -body "foo" + barrier b2 sync + barrier b3 sync rxresp expect resp.status == 200 } -run From dridi.boukelmoune at gmail.com Wed Oct 21 06:45:10 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 21 Oct 2020 06:45:10 +0000 (UTC) Subject: [master] 89a558e56 man: Reference sources from $(srcdir) Message-ID: <20201021064510.BA2C0112BB7@lists.varnish-cache.org> commit 89a558e56390d425c52732a6c94087eec9083115 Author: Dridi Boukelmoune Date: Wed Oct 21 08:03:42 2020 +0200 man: Reference sources from $(srcdir) Better diff with the --word-diff --word-diff-regex=. options. diff --git a/man/Makefile.am b/man/Makefile.am index f04a0647c..641a384f1 100644 --- a/man/Makefile.am +++ b/man/Makefile.am @@ -30,70 +30,70 @@ RST2ANY_FLAGS = --halt=2 BUILD_MAN = $(AM_V_GEN) $(RST2MAN) $(RST2ANY_FLAGS) -varnish-cli.7: $(top_builddir)/doc/sphinx/reference/varnish-cli.rst \ +varnish-cli.7: $(top_srcdir)/doc/sphinx/reference/varnish-cli.rst \ $(top_builddir)/doc/sphinx/include/cli.rst - $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnish-cli.rst $@ + $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnish-cli.rst $@ -varnish-counters.7: $(top_builddir)/doc/sphinx/reference/varnish-counters.rst \ +varnish-counters.7: $(top_srcdir)/doc/sphinx/reference/varnish-counters.rst \ $(top_builddir)/doc/sphinx/include/counters.rst - $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnish-counters.rst $@ + $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnish-counters.rst $@ -vcl.7: $(top_builddir)/doc/sphinx/reference/vcl.rst \ - $(top_builddir)/doc/sphinx/reference/vcl_var.rst \ +vcl.7: $(top_srcdir)/doc/sphinx/reference/vcl.rst \ + $(top_srcdir)/doc/sphinx/reference/vcl_var.rst \ $(top_srcdir)/bin/varnishd/builtin.vcl - $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/vcl.rst $@ + $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/vcl.rst $@ -vsl.7: $(top_builddir)/doc/sphinx/reference/vsl.rst \ +vsl.7: $(top_srcdir)/doc/sphinx/reference/vsl.rst \ $(top_builddir)/doc/sphinx/include/vsl-tags.rst - $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/vsl.rst $@ + $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/vsl.rst $@ -vsl-query.7: $(top_builddir)/doc/sphinx/reference/vsl-query.rst - $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/vsl-query.rst $@ +vsl-query.7: $(top_srcdir)/doc/sphinx/reference/vsl-query.rst + $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/vsl-query.rst $@ -varnishadm.1: $(top_builddir)/doc/sphinx/reference/varnishadm.rst - $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishadm.rst $@ +varnishadm.1: $(top_srcdir)/doc/sphinx/reference/varnishadm.rst + $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishadm.rst $@ varnishd.1: \ - $(top_builddir)/doc/sphinx/reference/varnishd.rst \ + $(top_srcdir)/doc/sphinx/reference/varnishd.rst \ $(top_builddir)/doc/sphinx/include/params.rst - $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishd.rst $@ + $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishd.rst $@ varnishncsa.1: \ - $(top_builddir)/doc/sphinx/reference/varnishncsa.rst \ + $(top_srcdir)/doc/sphinx/reference/varnishncsa.rst \ $(top_builddir)/doc/sphinx/include/varnishncsa_options.rst \ $(top_builddir)/doc/sphinx/include/varnishncsa_synopsis.rst - $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishncsa.rst $@ + $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishncsa.rst $@ varnishlog.1: \ - $(top_builddir)/doc/sphinx/reference/varnishlog.rst \ + $(top_srcdir)/doc/sphinx/reference/varnishlog.rst \ $(top_builddir)/doc/sphinx/include/varnishlog_options.rst \ $(top_builddir)/doc/sphinx/include/varnishlog_synopsis.rst - $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishlog.rst $@ + $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishlog.rst $@ -varnishstat.1: $(top_builddir)/doc/sphinx/reference/varnishstat.rst \ +varnishstat.1: $(top_srcdir)/doc/sphinx/reference/varnishstat.rst \ $(top_builddir)/doc/sphinx/include/varnishstat_options.rst \ $(top_builddir)/doc/sphinx/include/varnishstat_synopsis.rst \ $(top_builddir)/doc/sphinx/include/varnishstat_bindings.rst - $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishstat.rst $@ + $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishstat.rst $@ -varnishtest.1: $(top_builddir)/doc/sphinx/reference/varnishtest.rst - $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishtest.rst $@ +varnishtest.1: $(top_srcdir)/doc/sphinx/reference/varnishtest.rst + $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishtest.rst $@ -vtc.7: $(top_builddir)/doc/sphinx/reference/vtc.rst \ +vtc.7: $(top_srcdir)/doc/sphinx/reference/vtc.rst \ $(top_builddir)/doc/sphinx/include/vtc-syntax.rst - $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/vtc.rst $@ + $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/vtc.rst $@ varnishtop.1: \ - $(top_builddir)/doc/sphinx/reference/varnishtop.rst \ + $(top_srcdir)/doc/sphinx/reference/varnishtop.rst \ $(top_builddir)/doc/sphinx/include/varnishtop_options.rst \ $(top_builddir)/doc/sphinx/include/varnishtop_synopsis.rst - $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishtop.rst $@ + $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishtop.rst $@ varnishhist.1: \ - $(top_builddir)/doc/sphinx/reference/varnishhist.rst \ + $(top_srcdir)/doc/sphinx/reference/varnishhist.rst \ $(top_builddir)/doc/sphinx/include/varnishhist_options.rst \ $(top_builddir)/doc/sphinx/include/varnishhist_synopsis.rst - $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishhist.rst $@ + $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishhist.rst $@ vmod_cookie.3: $(top_builddir)/lib/libvmod_cookie/vmod_cookie.man.rst $(BUILD_MAN) $? $@ From dridi.boukelmoune at gmail.com Wed Oct 21 13:51:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 21 Oct 2020 13:51:07 +0000 (UTC) Subject: [master] e954c73d1 param: Promote mempools in the parameters table Message-ID: <20201021135107.6A24363A0@lists.varnish-cache.org> commit e954c73d12e1092c9263ece25e60eae236f7934b Author: Dridi Boukelmoune Date: Wed Oct 21 15:44:31 2020 +0200 param: Promote mempools in the parameters table We had everything we needed to make this happen, except for the mismatch between parameter names pool_xxx and struct member names xxx_pool. Refs #3250 diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index d80bd1ecc..5e07ccd46 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -49,7 +49,7 @@ void VBO_Init(void) { - vbopool = MPL_New("busyobj", &cache_param->vbo_pool, + vbopool = MPL_New("busyobj", &cache_param->pool_vbo, &cache_param->workspace_backend); AN(vbopool); } diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index fa6ab0b96..f927538fd 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -630,10 +630,10 @@ SES_NewPool(struct pool *pp, unsigned pool_no) CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); bprintf(nb, "req%u", pool_no); - pp->mpl_req = MPL_New(nb, &cache_param->req_pool, + pp->mpl_req = MPL_New(nb, &cache_param->pool_req, &cache_param->workspace_client); bprintf(nb, "sess%u", pool_no); - pp->mpl_sess = MPL_New(nb, &cache_param->sess_pool, + pp->mpl_sess = MPL_New(nb, &cache_param->pool_sess, &cache_param->workspace_session); pp->waiter = Waiter_New(); diff --git a/bin/varnishd/common/common_param.h b/bin/varnishd/common/common_param.h index 5896bde0a..ed8ccf8a4 100644 --- a/bin/varnishd/common/common_param.h +++ b/bin/varnishd/common/common_param.h @@ -115,10 +115,6 @@ struct params { struct vre_limits vre_limits; - struct poolparam req_pool; - struct poolparam sess_pool; - struct poolparam vbo_pool; - uint8_t vsl_mask[256>>3]; uint8_t debug_bits[(DBG_Reserved+7)>>3]; uint8_t feature_bits[(FEATURE_Reserved+7)>>3]; diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c index fec7493a9..869fdedcd 100644 --- a/bin/varnishd/mgt/mgt_param_tbl.c +++ b/bin/varnishd/mgt/mgt_param_tbl.c @@ -38,12 +38,6 @@ #include "mgt/mgt_param.h" -#define MEMPOOL_TEXT \ - "The three numbers are:\n" \ - "\tmin_pool\tminimum size of free pool.\n" \ - "\tmax_pool\tmaximum size of free pool.\n" \ - "\tmax_age\tmax age of free element." - struct parspec mgt_parspec[] = { #define PARAM(nm, ty, ...) { #nm, tweak_##ty, &mgt_param.nm, __VA_ARGS__ }, #include "tbl/params.h" @@ -122,21 +116,6 @@ struct parspec mgt_parspec[] = { "Matching failures will show up in the log as VCL_Error" " messages with regexp errors -27 or -21.\n\n" "Testcase r01576 can be useful when tuning this parameter." }, - { "pool_req", tweak_poolparam, &mgt_param.req_pool, - NULL, NULL, "10,100,10", - NULL, - "Parameters for per worker pool request memory pool.\n\n" - MEMPOOL_TEXT }, - { "pool_sess", tweak_poolparam, &mgt_param.sess_pool, - NULL, NULL, "10,100,10", - NULL, - "Parameters for per worker pool session memory pool.\n\n" - MEMPOOL_TEXT }, - { "pool_vbo", tweak_poolparam, &mgt_param.vbo_pool, - NULL, NULL, "10,100,10", - NULL, - "Parameters for backend object fetch memory pool.\n\n" - MEMPOOL_TEXT }, { NULL, NULL, NULL } }; diff --git a/include/tbl/params.h b/include/tbl/params.h index b67c3c0dc..23874857b 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1209,6 +1209,50 @@ PARAM( "HTTP2 maximum size of an uncompressed header list." ) +#define MEMPOOL_TEXT \ + "The three numbers are:\n" \ + "\tmin_pool\tminimum size of free pool.\n" \ + "\tmax_pool\tmaximum size of free pool.\n" \ + "\tmax_age\tmax age of free element." + +PARAM( + /* name */ pool_req, + /* type */ poolparam, + /* min */ NULL, + /* max */ NULL, + /* def */ "10,100,10", + /* units */ NULL, + /* descr */ + "Parameters for per worker pool request memory pool.\n" + MEMPOOL_TEXT +) + +PARAM( + /* name */ pool_sess, + /* type */ poolparam, + /* min */ NULL, + /* max */ NULL, + /* def */ "10,100,10", + /* units */ NULL, + /* descr */ + "Parameters for per worker pool session memory pool.\n" + MEMPOOL_TEXT +) + +PARAM( + /* name */ pool_vbo, + /* type */ poolparam, + /* min */ NULL, + /* max */ NULL, + /* def */ "10,100,10", + /* units */ NULL, + /* descr */ + "Parameters for backend object fetch memory pool.\n" + MEMPOOL_TEXT +) + +#undef MEMPOOL_TEXT + #if 0 /* NOT ACTUALLY DEFINED HERE */ /* actual location mgt_param_bits.c*/ /* see tbl/debug_bits.h */ @@ -1286,45 +1330,6 @@ PARAM( "recursions in a pcre_exec() execution." ) -/* actual location mgt_param_tbl.c */ -PARAM( - /* name */ pool_req, - /* type */ poolparam, - /* min */ NULL, - /* max */ NULL, - /* def */ "10,100,10", - /* units */ NULL, - /* descr */ - "Parameters for per worker pool request memory pool.\n" - MEMPOOL_TEXT -) - -/* actual location mgt_param_tbl.c */ -PARAM( - /* name */ pool_sess, - /* type */ poolparam, - /* min */ NULL, - /* max */ NULL, - /* def */ "10,100,10", - /* units */ NULL, - /* descr */ - "Parameters for per worker pool session memory pool.\n" - MEMPOOL_TEXT -) - -/* actual location mgt_param_tbl.c */ -PARAM( - /* name */ pool_vbo, - /* type */ poolparam, - /* min */ NULL, - /* max */ NULL, - /* def */ "10,100,10", - /* units */ NULL, - /* descr */ - "Parameters for backend object fetch memory pool.\n" - MEMPOOL_TEXT -) - /* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_add_delay, From dridi.boukelmoune at gmail.com Wed Oct 21 17:24:13 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 21 Oct 2020 17:24:13 +0000 (UTC) Subject: [master] 7cb016a3d param: Introduce the notion of string parameters Message-ID: <20201021172413.26460631B2@lists.varnish-cache.org> commit 7cb016a3d5cfce51c603174fee8b8f4d2f96f06f Author: Dridi Boukelmoune Date: Wed Oct 21 17:28:19 2020 +0200 param: Introduce the notion of string parameters Following the model of simple parameters, it is possible to find a home for string parameters in params.h too. A new PARAM_STRING() macro deals with the specificities of such parameters. Since string parameters are typically not inherited by the cache process they are guarded by a PARAM_ALL macro that defines whether a subset or the whole list of parameters is processed when params.h is #include'd. Refs #3250 diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c index 4122a1cfd..4ee4e23f3 100644 --- a/bin/varnishd/mgt/mgt_param_tbl.c +++ b/bin/varnishd/mgt/mgt_param_tbl.c @@ -39,33 +39,10 @@ struct parspec mgt_parspec[] = { +#define PARAM_ALL #define PARAM(ty, nm, ...) { #nm, __VA_ARGS__ }, #include "tbl/params.h" - { "cc_command", tweak_string, &mgt_cc_cmd, - NULL, NULL, VCC_CC, - NULL, - "Command used for compiling the C source code to a " - "dlopen(3) loadable object. Any occurrence of %s in " - "the string will be replaced with the source file name, " - "and %o will be replaced with the output file name.", - MUST_RELOAD, - NULL, NULL, "defined when Varnish is built" }, - { "vcl_path", tweak_string, &mgt_vcl_path, - NULL, NULL, VARNISH_VCL_DIR, - NULL, - "Directory (or colon separated list of directories) " - "from which relative VCL filenames (vcl.load and " - "include) are to be found. By default Varnish searches " - "VCL files in both the system configuration and shared " - "data directories to allow packages to drop their VCL " - "files in a standard location where relative includes " - "would work." }, - { "vmod_path", tweak_string, &mgt_vmod_path, - NULL, NULL, VARNISH_VMOD_DIR, - NULL, - "Directory (or colon separated list of directories) " - "where VMODs are to be found." }, { "vcc_err_unref", tweak_bool, &mgt_vcc_err_unref, NULL, NULL, "on", "bool", diff --git a/include/tbl/params.h b/include/tbl/params.h index 13211986d..4209669d9 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1260,6 +1260,57 @@ PARAM_SIMPLE( #undef MEMPOOL_TEXT +#if defined(PARAM_ALL) + +/*-------------------------------------------------------------------- + * String parameters + */ + +# define PARAM_STRING(nm, pv, def, ...) \ + PARAM(, nm, tweak_string, pv, NULL, NULL, def, NULL, __VA_ARGS__) + +PARAM_STRING( + /* name */ cc_command, + /* priv */ &mgt_cc_cmd, + /* def */ VCC_CC, + /* descr */ + "Command used for compiling the C source code to a " + "dlopen(3) loadable object. Any occurrence of %s in " + "the string will be replaced with the source file name, " + "and %o will be replaced with the output file name.", + /* flags */ MUST_RELOAD +) + +PARAM_STRING( + /* name */ vcl_path, + /* priv */ &mgt_vcl_path, + /* def */ VARNISH_VCL_DIR, + /* descr */ + "Directory (or colon separated list of directories) " + "from which relative VCL filenames (vcl.load and " + "include) are to be found. By default Varnish searches " + "VCL files in both the system configuration and shared " + "data directories to allow packages to drop their VCL " + "files in a standard location where relative includes " + "would work." +) + +PARAM_STRING( + /* name */ vmod_path, + /* priv */ &mgt_vmod_path, + /* def */ VARNISH_VMOD_DIR, + /* descr */ + "Directory (or colon separated list of directories) " + "where VMODs are to be found." +) + +# undef PARAM_ALL +# undef PARAM_STRING +#endif /* defined(PARAM_ALL) */ + +#undef PARAM_SIMPLE +#undef PARAM + #if 0 /* NOT ACTUALLY DEFINED HERE */ /* actual location mgt_param_bits.c*/ /* see tbl/debug_bits.h */ @@ -1637,7 +1688,4 @@ PARAM( ) #endif /* NOT ACTUALLY DEFINED HERE */ -#undef PARAM_SIMPLE -#undef PARAM - /*lint -restore */ From dridi.boukelmoune at gmail.com Wed Oct 21 17:24:12 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 21 Oct 2020 17:24:12 +0000 (UTC) Subject: [master] 087329a51 param: Introduce the notion of simple parameters Message-ID: <20201021172413.0E1F8631AE@lists.varnish-cache.org> commit 087329a5127e85c66c57cd9d7833f05f32c9f77a Author: Dridi Boukelmoune Date: Wed Oct 21 17:16:39 2020 +0200 param: Introduce the notion of simple parameters The simple parameters are those that successfully migrated to params.h because they were trivial to declare in a way that could be used in all the places where they are needed. The PARAM() macro is now changed so that the outstanding `type` argument is separated from arguments matching struct parspec fields. To minimize disruption the new PARAM_SIMPLE() macro takes parameters in the order of the previous PARAM() definition. Refs #3250 diff --git a/bin/varnishd/common/common_param.h b/bin/varnishd/common/common_param.h index ed8ccf8a4..6e612593f 100644 --- a/bin/varnishd/common/common_param.h +++ b/bin/varnishd/common/common_param.h @@ -82,7 +82,7 @@ struct params { #define ptyp_uint unsigned #define ptyp_vsl_buffer unsigned #define ptyp_vsl_reclen unsigned -#define PARAM(nm, ty, ...) \ +#define PARAM(ty, nm, ...) \ ptyp_##ty nm; #include #undef ptyp_bool diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c index 869fdedcd..4122a1cfd 100644 --- a/bin/varnishd/mgt/mgt_param_tbl.c +++ b/bin/varnishd/mgt/mgt_param_tbl.c @@ -39,7 +39,7 @@ struct parspec mgt_parspec[] = { -#define PARAM(nm, ty, ...) { #nm, tweak_##ty, &mgt_param.nm, __VA_ARGS__ }, +#define PARAM(ty, nm, ...) { #nm, __VA_ARGS__ }, #include "tbl/params.h" { "cc_command", tweak_string, &mgt_cc_cmd, diff --git a/include/tbl/params.h b/include/tbl/params.h index 23874857b..13211986d 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -27,14 +27,21 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * PARAM(nm, ty, ...) + * PARAM(type, ...) * - * Variable arguments refer to struct parspec fields from min to - * dyn_def_reason. + * Variable arguments refer to struct parspec fields, except non-const + * strings. */ /*lint -save -e525 -e539 */ +/*-------------------------------------------------------------------- + * * Simple parameters + * */ + +#define PARAM_SIMPLE(nm, ty, ...) \ + PARAM(ty, nm, tweak_##ty, &mgt_param.nm, __VA_ARGS__) + #if defined(XYZZY) #error "Temporary macro XYZZY already defined" #endif @@ -44,7 +51,7 @@ #else #define XYZZY NOT_IMPLEMENTED #endif -PARAM( +PARAM_SIMPLE( /* name */ accept_filter, /* type */ bool, /* min */ NULL, @@ -65,7 +72,7 @@ PARAM( ) #undef XYZZY -PARAM( +PARAM_SIMPLE( /* name */ acceptor_sleep_decay, /* type */ double, /* min */ "0", @@ -80,7 +87,7 @@ PARAM( /* flags */ EXPERIMENTAL ) -PARAM( +PARAM_SIMPLE( /* name */ acceptor_sleep_incr, /* type */ timeout, /* min */ "0", @@ -95,7 +102,7 @@ PARAM( /* flags */ EXPERIMENTAL ) -PARAM( +PARAM_SIMPLE( /* name */ acceptor_sleep_max, /* type */ timeout, /* min */ "0", @@ -110,7 +117,7 @@ PARAM( /* flags */ EXPERIMENTAL ) -PARAM( +PARAM_SIMPLE( /* name */ auto_restart, /* type */ bool, /* min */ NULL, @@ -121,7 +128,7 @@ PARAM( "Automatically restart the child/worker process if it dies." ) -PARAM( +PARAM_SIMPLE( /* name */ ban_dups, /* type */ bool, /* min */ NULL, @@ -135,7 +142,7 @@ PARAM( "identical." ) -PARAM( +PARAM_SIMPLE( /* name */ ban_cutoff, /* type */ uint, /* min */ "0", @@ -165,7 +172,7 @@ PARAM( /* flags */ EXPERIMENTAL ) -PARAM( +PARAM_SIMPLE( /* name */ ban_lurker_age, /* type */ timeout, /* min */ "0", @@ -181,7 +188,7 @@ PARAM( "This should be set to the approximate time which a ban-burst takes." ) -PARAM( +PARAM_SIMPLE( /* name */ ban_lurker_batch, /* type */ uint, /* min */ "1", @@ -194,7 +201,7 @@ PARAM( " Use this to pace the ban-lurker if it eats too many resources." ) -PARAM( +PARAM_SIMPLE( /* name */ ban_lurker_sleep, /* type */ timeout, /* min */ "0", @@ -208,7 +215,7 @@ PARAM( "A value of zero will disable the ban lurker entirely." ) -PARAM( +PARAM_SIMPLE( /* name */ ban_lurker_holdoff, /* type */ timeout, /* min */ "0", @@ -221,7 +228,7 @@ PARAM( /* flags */ EXPERIMENTAL ) -PARAM( +PARAM_SIMPLE( /* name */ first_byte_timeout, /* type */ timeout, /* min */ "0", @@ -236,7 +243,7 @@ PARAM( "This parameter does not apply to pipe'ed requests." ) -PARAM( +PARAM_SIMPLE( /* name */ between_bytes_timeout, /* type */ timeout, /* min */ "0", @@ -250,7 +257,7 @@ PARAM( "This parameter does not apply to pipe'ed requests." ) -PARAM( +PARAM_SIMPLE( /* name */ backend_idle_timeout, /* type */ timeout, /* min */ "1", @@ -261,7 +268,7 @@ PARAM( "Timeout before we close unused backend connections." ) -PARAM( +PARAM_SIMPLE( /* name */ backend_local_error_holddown, /* type */ timeout, /* min */ "0.000", @@ -279,7 +286,7 @@ PARAM( /* flags */ EXPERIMENTAL ) -PARAM( +PARAM_SIMPLE( /* name */ backend_remote_error_holddown, /* type */ timeout, /* min */ "0.000", @@ -296,7 +303,7 @@ PARAM( /* flags */ EXPERIMENTAL ) -PARAM( +PARAM_SIMPLE( /* name */ cli_limit, /* type */ bytes_u, /* min */ "128b", @@ -309,7 +316,7 @@ PARAM( "line will indicate the truncation." ) -PARAM( +PARAM_SIMPLE( /* name */ cli_timeout, /* type */ timeout, /* min */ "0.000", @@ -321,7 +328,7 @@ PARAM( "mgt_param." ) -PARAM( +PARAM_SIMPLE( /* name */ clock_skew, /* type */ uint, /* min */ "0", @@ -333,7 +340,7 @@ PARAM( "and our own clock." ) -PARAM( +PARAM_SIMPLE( /* name */ clock_step, /* type */ timeout, /* min */ "0.000", @@ -345,7 +352,7 @@ PARAM( "we panic." ) -PARAM( +PARAM_SIMPLE( /* name */ connect_timeout, /* type */ timeout, /* min */ "0.000", @@ -359,7 +366,7 @@ PARAM( "request." ) -PARAM( +PARAM_SIMPLE( /* name */ critbit_cooloff, /* type */ timeout, /* min */ "60.000", @@ -372,7 +379,7 @@ PARAM( /* flags */ WIZARD ) -PARAM( +PARAM_SIMPLE( /* name */ default_grace, /* type */ timeout, /* min */ "0.000", @@ -386,7 +393,7 @@ PARAM( /* flags */ OBJ_STICKY ) -PARAM( +PARAM_SIMPLE( /* name */ default_keep, /* type */ timeout, /* min */ "0.000", @@ -401,7 +408,7 @@ PARAM( /* flags */ OBJ_STICKY ) -PARAM( +PARAM_SIMPLE( /* name */ default_ttl, /* type */ timeout, /* min */ "0.000", @@ -414,7 +421,7 @@ PARAM( /* flags */ OBJ_STICKY ) -PARAM( +PARAM_SIMPLE( /* name */ http1_iovs, /* type */ uint, /* min */ "5", @@ -428,7 +435,7 @@ PARAM( /* flags */ WIZARD ) -PARAM( +PARAM_SIMPLE( /* name */ fetch_chunksize, /* type */ bytes, /* min */ "4k", @@ -443,7 +450,7 @@ PARAM( /* flags */ EXPERIMENTAL ) -PARAM( +PARAM_SIMPLE( /* name */ fetch_maxchunksize, /* type */ bytes, /* min */ "64k", @@ -456,7 +463,7 @@ PARAM( /* flags */ EXPERIMENTAL ) -PARAM( +PARAM_SIMPLE( /* name */ gzip_buffer, /* type */ bytes_u, /* min */ "2k", @@ -472,7 +479,7 @@ PARAM( /* flags */ EXPERIMENTAL ) -PARAM( +PARAM_SIMPLE( /* name */ gzip_level, /* type */ uint, /* min */ "0", @@ -483,7 +490,7 @@ PARAM( "Gzip compression level: 0=debug, 1=fast, 9=best" ) -PARAM( +PARAM_SIMPLE( /* name */ gzip_memlevel, /* type */ uint, /* min */ "1", @@ -495,7 +502,7 @@ PARAM( "Memory impact is 1=1k, 2=2k, ... 9=256k." ) -PARAM( +PARAM_SIMPLE( /* name */ http_gzip_support, /* type */ bool, /* min */ NULL, @@ -519,7 +526,7 @@ PARAM( /* XXX: what about the effect on beresp.filters? */ ) -PARAM( +PARAM_SIMPLE( /* name */ http_max_hdr, /* type */ uint, /* min */ "32", @@ -534,7 +541,7 @@ PARAM( "Note that the first line occupies five header lines." ) -PARAM( +PARAM_SIMPLE( /* name */ http_range_support, /* type */ bool, /* min */ NULL, @@ -546,7 +553,7 @@ PARAM( /* XXX: what about the effect on beresp.filters? */ ) -PARAM( +PARAM_SIMPLE( /* name */ http_req_hdr_len, /* type */ bytes_u, /* min */ "40b", @@ -558,7 +565,7 @@ PARAM( "The limit is inclusive its continuation lines." ) -PARAM( +PARAM_SIMPLE( /* name */ http_req_size, /* type */ bytes_u, /* min */ "0.25k", @@ -574,7 +581,7 @@ PARAM( "that the request is allowed to take up." ) -PARAM( +PARAM_SIMPLE( /* name */ http_resp_hdr_len, /* type */ bytes_u, /* min */ "40b", @@ -586,7 +593,7 @@ PARAM( " The limit is inclusive its continuation lines." ) -PARAM( +PARAM_SIMPLE( /* name */ http_resp_size, /* type */ bytes_u, /* min */ "0.25k", @@ -611,7 +618,7 @@ PARAM( #else #define XYZZY NOT_IMPLEMENTED #endif -PARAM( +PARAM_SIMPLE( /* name */ idle_send_timeout, /* type */ timeout, /* min */ "0.000", @@ -628,7 +635,7 @@ PARAM( ) #undef XYZZY -PARAM( +PARAM_SIMPLE( /* name */ listen_depth, /* type */ uint, /* min */ "0", @@ -640,7 +647,7 @@ PARAM( /* flags */ MUST_RESTART ) -PARAM( +PARAM_SIMPLE( /* name */ lru_interval, /* type */ timeout, /* min */ "0.000", @@ -656,7 +663,7 @@ PARAM( /* flags */ EXPERIMENTAL ) -PARAM( +PARAM_SIMPLE( /* name */ max_esi_depth, /* type */ uint, /* min */ "0", @@ -667,7 +674,7 @@ PARAM( "Maximum depth of esi:include processing." ) -PARAM( +PARAM_SIMPLE( /* name */ max_restarts, /* type */ uint, /* min */ "0", @@ -678,7 +685,7 @@ PARAM( "Upper limit on how many times a request can restart." ) -PARAM( +PARAM_SIMPLE( /* name */ max_retries, /* type */ uint, /* min */ "0", @@ -689,7 +696,7 @@ PARAM( "Upper limit on how many times a backend fetch can retry." ) -PARAM( +PARAM_SIMPLE( /* name */ nuke_limit, /* type */ uint, /* min */ "0", @@ -702,7 +709,7 @@ PARAM( /* flags */ EXPERIMENTAL ) -PARAM( +PARAM_SIMPLE( /* name */ ping_interval, /* type */ uint, /* min */ "0", @@ -716,7 +723,7 @@ PARAM( /* flags */ MUST_RESTART ) -PARAM( +PARAM_SIMPLE( /* name */ pipe_sess_max, /* type */ uint, /* min */ "0", @@ -727,7 +734,7 @@ PARAM( "Maximum number of sessions dedicated to pipe transactions." ) -PARAM( +PARAM_SIMPLE( /* name */ pipe_timeout, /* type */ timeout, /* min */ "0.000", @@ -739,7 +746,7 @@ PARAM( "either direction for this many seconds, the session is closed." ) -PARAM( +PARAM_SIMPLE( /* name */ prefer_ipv6, /* type */ bool, /* min */ NULL, @@ -751,7 +758,7 @@ PARAM( "IPv4 and IPv6 addresses." ) -PARAM( +PARAM_SIMPLE( /* name */ rush_exponent, /* type */ uint, /* min */ "2", @@ -775,7 +782,7 @@ PARAM( #else #define XYZZY NOT_IMPLEMENTED #endif -PARAM( +PARAM_SIMPLE( /* name */ send_timeout, /* type */ timeout, /* min */ "0.000", @@ -793,7 +800,7 @@ PARAM( ) #undef XYZZY -PARAM( +PARAM_SIMPLE( /* name */ shortlived, /* type */ timeout, /* min */ "0.000", @@ -805,7 +812,7 @@ PARAM( "always put in transient storage." ) -PARAM( +PARAM_SIMPLE( /* name */ sigsegv_handler, /* type */ bool, /* min */ NULL, @@ -818,7 +825,7 @@ PARAM( /* flags */ MUST_RESTART ) -PARAM( +PARAM_SIMPLE( /* name */ syslog_cli_traffic, /* type */ bool, /* min */ NULL, @@ -834,7 +841,7 @@ PARAM( #else #define XYZZY NOT_IMPLEMENTED #endif -PARAM( +PARAM_SIMPLE( /* name */ tcp_fastopen, /* type */ bool, /* min */ NULL, @@ -852,7 +859,7 @@ PARAM( #else #define XYZZY NOT_IMPLEMENTED #endif -PARAM( +PARAM_SIMPLE( /* name */ tcp_keepalive_intvl, /* type */ timeout, /* min */ "1", @@ -868,7 +875,7 @@ PARAM( /* dyn_def_reason */ "platform dependent" ) -PARAM( +PARAM_SIMPLE( /* name */ tcp_keepalive_probes, /* type */ uint, /* min */ "1", @@ -885,7 +892,7 @@ PARAM( /* dyn_def_reason */ "platform dependent" ) -PARAM( +PARAM_SIMPLE( /* name */ tcp_keepalive_time, /* type */ timeout, /* min */ "1", @@ -912,7 +919,7 @@ PARAM( #else #define XYZZY NOT_IMPLEMENTED #endif -PARAM( +PARAM_SIMPLE( /* name */ timeout_idle, /* type */ timeout, /* min */ "0.000", @@ -930,7 +937,7 @@ PARAM( ) #undef XYZZY -PARAM( +PARAM_SIMPLE( /* name */ timeout_linger, /* type */ timeout, /* min */ "0.000", @@ -948,7 +955,7 @@ PARAM( /* flags */ EXPERIMENTAL ) -PARAM( +PARAM_SIMPLE( /* name */ vcl_cooldown, /* type */ timeout, /* min */ "1.000", @@ -960,7 +967,7 @@ PARAM( "active VCL (granularity approximately 30 seconds)." ) -PARAM( +PARAM_SIMPLE( /* name */ max_vcl_handling, /* type */ uint, /* min */ "0", @@ -974,7 +981,7 @@ PARAM( "\n* 2 - Refuse loading VCLs." ) -PARAM( +PARAM_SIMPLE( /* name */ max_vcl, /* type */ uint, /* min */ "0", @@ -986,7 +993,7 @@ PARAM( " Parameter max_vcl_handling determines behaviour." ) -PARAM( +PARAM_SIMPLE( /* name */ vsm_free_cooldown, /* type */ timeout, /* min */ "10.000", @@ -998,7 +1005,7 @@ PARAM( "(granularity approximately 2 seconds)." ) -PARAM( +PARAM_SIMPLE( /* name */ vsl_buffer, /* type */ vsl_buffer, /* min */ "267", @@ -1017,7 +1024,7 @@ PARAM( /* dyn_min_reason */ "vsl_reclen + 12 bytes" ) -PARAM( +PARAM_SIMPLE( /* name */ vsl_reclen, /* type */ vsl_reclen, /* min */ "16b", @@ -1031,7 +1038,7 @@ PARAM( /* dyn_max_reason */ "vsl_buffer - 12 bytes" ) -PARAM( +PARAM_SIMPLE( /* name */ vsl_space, /* type */ bytes, /* min */ "1M", @@ -1046,7 +1053,7 @@ PARAM( /* flags */ MUST_RESTART ) -PARAM( +PARAM_SIMPLE( /* name */ vsm_space, /* type */ bytes, /* min */ "1M", @@ -1058,7 +1065,7 @@ PARAM( "There is no global limit on amount of shared memory now." ) -PARAM( +PARAM_SIMPLE( /* name */ workspace_backend, /* type */ bytes_u, /* min */ "1k", @@ -1071,7 +1078,7 @@ PARAM( /* flags */ DELAYED_EFFECT ) -PARAM( +PARAM_SIMPLE( /* name */ workspace_client, /* type */ bytes_u, /* min */ "9k", @@ -1088,7 +1095,7 @@ PARAM( /* flags */ DELAYED_EFFECT ) -PARAM( +PARAM_SIMPLE( /* name */ workspace_session, /* type */ bytes_u, /* min */ "0.25k", @@ -1102,7 +1109,7 @@ PARAM( /* flags */ DELAYED_EFFECT ) -PARAM( +PARAM_SIMPLE( /* name */ workspace_thread, /* type */ bytes_u, /* min */ "0.25k", @@ -1122,7 +1129,7 @@ PARAM( /* flags */ DELAYED_EFFECT ) -PARAM( +PARAM_SIMPLE( /* name */ h2_rx_window_low_water, /* type */ bytes_u, /* min */ "65535", @@ -1136,7 +1143,7 @@ PARAM( /* flags */ WIZARD ) -PARAM( +PARAM_SIMPLE( /* name */ h2_rx_window_increment, /* type */ bytes_u, /* min */ "1M", @@ -1150,7 +1157,7 @@ PARAM( /* flags */ WIZARD ) -PARAM( +PARAM_SIMPLE( /* name */ h2_header_table_size, /* type */ bytes_u, /* min */ "0b", @@ -1163,7 +1170,7 @@ PARAM( "decoding table." ) -PARAM( +PARAM_SIMPLE( /* name */ h2_max_concurrent_streams, /* type */ uint, /* min */ "0", @@ -1176,7 +1183,7 @@ PARAM( "at the same time for a single HTTP2 connection." ) -PARAM( +PARAM_SIMPLE( /* name */ h2_initial_window_size, /* type */ bytes_u, /* min */ "0", @@ -1187,7 +1194,7 @@ PARAM( "HTTP2 initial flow control window size." ) -PARAM( +PARAM_SIMPLE( /* name */ h2_max_frame_size, /* type */ bytes_u, /* min */ "16k", @@ -1198,7 +1205,7 @@ PARAM( "HTTP2 maximum per frame payload size we are willing to accept." ) -PARAM( +PARAM_SIMPLE( /* name */ h2_max_header_list_size, /* type */ bytes_u, /* min */ "0b", @@ -1215,7 +1222,7 @@ PARAM( "\tmax_pool\tmaximum size of free pool.\n" \ "\tmax_age\tmax age of free element." -PARAM( +PARAM_SIMPLE( /* name */ pool_req, /* type */ poolparam, /* min */ NULL, @@ -1227,7 +1234,7 @@ PARAM( MEMPOOL_TEXT ) -PARAM( +PARAM_SIMPLE( /* name */ pool_sess, /* type */ poolparam, /* min */ NULL, @@ -1239,7 +1246,7 @@ PARAM( MEMPOOL_TEXT ) -PARAM( +PARAM_SIMPLE( /* name */ pool_vbo, /* type */ poolparam, /* min */ NULL, @@ -1630,6 +1637,7 @@ PARAM( ) #endif /* NOT ACTUALLY DEFINED HERE */ +#undef PARAM_SIMPLE #undef PARAM /*lint -restore */ From dridi.boukelmoune at gmail.com Wed Oct 21 17:24:13 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 21 Oct 2020 17:24:13 +0000 (UTC) Subject: [master] 17b53dbe5 param: Introduce the notion of VCC parameters Message-ID: <20201021172413.4C4D1631B5@lists.varnish-cache.org> commit 17b53dbe5568bce71a271779b7dc739c243890ba Author: Dridi Boukelmoune Date: Wed Oct 21 17:40:37 2020 +0200 param: Introduce the notion of VCC parameters Just like string parameters, they need to be guarded by PARAM_ALL since they are not inherited by the cache process. Refs #3250 diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c index 4ee4e23f3..5b295208b 100644 --- a/bin/varnishd/mgt/mgt_param_tbl.c +++ b/bin/varnishd/mgt/mgt_param_tbl.c @@ -43,32 +43,6 @@ struct parspec mgt_parspec[] = { #define PARAM(ty, nm, ...) { #nm, __VA_ARGS__ }, #include "tbl/params.h" - { "vcc_err_unref", tweak_bool, &mgt_vcc_err_unref, - NULL, NULL, "on", - "bool", - "Unreferenced VCL objects result in error." }, - { "vcc_acl_pedantic", tweak_bool, &mgt_vcc_acl_pedantic, - NULL, NULL, "off", - "bool", - "Insist that network numbers used in ACLs have an " - "all-zero host part, e.g. make 1.2.3.4/24 an error.\n" - "With this option set to off (the default), the host " - "part of network numbers is being fixed to all-zeroes " - "(e.g. the above changed to 1.2.3.0/24), a warning is " - "output during VCL compilation and any ACL entry hits " - "are logged with the fixed address as \"fixed: ...\" " - "after the original VCL entry.\n" - "With this option set to on, any ACL entries with non-zero " - "host parts cause VCL compilation to fail." }, - { "vcc_allow_inline_c", tweak_bool, &mgt_vcc_allow_inline_c, - NULL, NULL, "off", - "bool", - "Allow inline C code in VCL." }, - { "vcc_unsafe_path", tweak_bool, &mgt_vcc_unsafe_path, - NULL, NULL, "on", - "bool", - "Allow '/' in vmod & include paths.\n" - "Allow 'import ... from ...'." }, { "pcre_match_limit", tweak_uint, &mgt_param.vre_limits.match, "1", NULL, "10000", diff --git a/include/tbl/params.h b/include/tbl/params.h index 4209669d9..139565924 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1304,8 +1304,54 @@ PARAM_STRING( "where VMODs are to be found." ) +/*-------------------------------------------------------------------- + * VCC parameters + */ + +# define PARAM_VCC(nm, def, descr) \ + PARAM(, nm, tweak_bool, &mgt_ ## nm, NULL, NULL, def, "bool", descr) + +PARAM_VCC( + /* name */ vcc_err_unref, + /* def */ "on", + /* descr */ + "Unreferenced VCL objects result in error." +) + +PARAM_VCC( + /* name */ vcc_acl_pedantic, + /* def */ "off", + /* descr */ + "Insist that network numbers used in ACLs have an " + "all-zero host part, e.g. make 1.2.3.4/24 an error.\n" + "With this option set to off (the default), the host " + "part of network numbers is being fixed to all-zeroes " + "(e.g. the above changed to 1.2.3.0/24), a warning is " + "output during VCL compilation and any ACL entry hits " + "are logged with the fixed address as \"fixed: ...\" " + "after the original VCL entry.\n" + "With this option set to on, any ACL entries with non-zero " + "host parts cause VCL compilation to fail." +) + +PARAM_VCC( + /* name */ vcc_allow_inline_c, + /* def */ "off", + /* descr */ + "Allow inline C code in VCL." +) + +PARAM_VCC( + /* name */ vcc_unsafe_path, + /* def */ "on", + /* descr */ + "Allow '/' in vmod & include paths.\n" + "Allow 'import ... from ...'." +) + # undef PARAM_ALL # undef PARAM_STRING +# undef PARAM_VCC #endif /* defined(PARAM_ALL) */ #undef PARAM_SIMPLE @@ -1614,63 +1660,6 @@ PARAM( /* flags */ EXPERIMENTAL ) -/* actual location mgt_param_tbl.c */ -PARAM( - /* name */ vcc_acl_pedantic, - /* type */ bool, - /* min */ NULL, - /* max */ NULL, - /* def */ "off", // XXX change to on in 7.x ? - /* units */ "bool", - /* descr */ - "Insist that network numbers used in ACLs have an all-zero host part, " - "e.g. make 1.2.3.4/24 an error.\n" - "With this option set to off (the default), the host part of network " - "numbers is being fixed to all-zeroes (e.g. the above changed to " - "1.2.3.0/24), a warning is output during VCL compilation and any ACL " - "entry hits are logged with the fixed address as \"fixed: ...\" " - "after the original VCL entry.\n" - "With this option set to on, any ACL entries with non-zero host parts " - "cause VCL compilation to fail." -) - -/* actual location mgt_param_tbl.c */ -PARAM( - /* name */ vcc_allow_inline_c, - /* type */ bool, - /* min */ NULL, - /* max */ NULL, - /* def */ "off", - /* units */ "bool", - /* descr */ - "Allow inline C code in VCL." -) - -/* actual location mgt_param_tbl.c */ -PARAM( - /* name */ vcc_err_unref, - /* type */ bool, - /* min */ NULL, - /* max */ NULL, - /* def */ "on", - /* units */ "bool", - /* descr */ - "Unreferenced VCL objects result in error." -) - -/* actual location mgt_param_tbl.c */ -PARAM( - /* name */ vcc_unsafe_path, - /* type */ bool, - /* min */ NULL, - /* max */ NULL, - /* def */ "on", - /* units */ "bool", - /* descr */ - "Allow '/' in vmod & include paths.\n" - "Allow 'import ... from ...'." -) - /* actual location mgt_param_bits.c*/ PARAM( /* name */ vsl_mask, From dridi.boukelmoune at gmail.com Wed Oct 21 17:24:13 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 21 Oct 2020 17:24:13 +0000 (UTC) Subject: [master] 75b25921b param: Introduce the notion of PCRE parameters Message-ID: <20201021172413.66F4B631BB@lists.varnish-cache.org> commit 75b25921b2061852478a94e88789df6a34e8a835 Author: Dridi Boukelmoune Date: Wed Oct 21 17:46:55 2020 +0200 param: Introduce the notion of PCRE parameters Unlike string or VCC parameters they are inherited, but their definition is not as straightforward as simple parameters. They also need to be guarded by PARAM_ALL because inside struct params they are groupped in a dedicated struct vre_limits, and there is no need for disruption in that area. Refs #3250 diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c index 5b295208b..f911aff20 100644 --- a/bin/varnishd/mgt/mgt_param_tbl.c +++ b/bin/varnishd/mgt/mgt_param_tbl.c @@ -34,39 +34,11 @@ #include #include "mgt/mgt.h" - #include "mgt/mgt_param.h" - struct parspec mgt_parspec[] = { #define PARAM_ALL #define PARAM(ty, nm, ...) { #nm, __VA_ARGS__ }, #include "tbl/params.h" - - { "pcre_match_limit", tweak_uint, - &mgt_param.vre_limits.match, - "1", NULL, "10000", - NULL, - "The limit for the number of calls to the internal match()" - " function in pcre_exec().\n\n" - "(See: PCRE_EXTRA_MATCH_LIMIT in pcre docs.)\n\n" - "This parameter limits how much CPU time" - " regular expression matching can soak up." }, - { "pcre_match_limit_recursion", tweak_uint, - &mgt_param.vre_limits.match_recursion, - "1", NULL, "20", - NULL, - "The recursion depth-limit for the internal match() function" - " in a pcre_exec().\n\n" - "(See: PCRE_EXTRA_MATCH_LIMIT_RECURSION in pcre docs.)\n\n" - "This puts an upper limit on the amount of stack used" - " by PCRE for certain classes of regular expressions.\n\n" - "We have set the default value low in order to" - " prevent crashes, at the cost of possible regexp" - " matching failures.\n\n" - "Matching failures will show up in the log as VCL_Error" - " messages with regexp errors -27 or -21.\n\n" - "Testcase r01576 can be useful when tuning this parameter." }, - { NULL, NULL, NULL } }; diff --git a/include/tbl/params.h b/include/tbl/params.h index 139565924..c3292b92b 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1349,7 +1349,48 @@ PARAM_VCC( "Allow 'import ... from ...'." ) +/*-------------------------------------------------------------------- + * PCRE parameters + */ + +# define PARAM_PCRE(nm, pv, min, def, descr) \ + PARAM(, nm, tweak_uint, &mgt_param.vre_limits.pv, \ + min, NULL, def, NULL, descr) + +PARAM_PCRE( + /* name */ pcre_match_limit, + /* priv */ match, + /* min */ "1", + /* def */ "10000", + /* descr */ + "The limit for the number of calls to the internal match()" + " function in pcre_exec().\n\n" + "(See: PCRE_EXTRA_MATCH_LIMIT in pcre docs.)\n\n" + "This parameter limits how much CPU time" + " regular expression matching can soak up." +) + +PARAM_PCRE( + /* name */ pcre_match_limit_recursion, + /* priv */ match_recursion, + /* min */ "1", + /* def */ "20", + /* descr */ + "The recursion depth-limit for the internal match() function" + " in a pcre_exec().\n\n" + "(See: PCRE_EXTRA_MATCH_LIMIT_RECURSION in pcre docs.)\n\n" + "This puts an upper limit on the amount of stack used" + " by PCRE for certain classes of regular expressions.\n\n" + "We have set the default value low in order to" + " prevent crashes, at the cost of possible regexp" + " matching failures.\n\n" + "Matching failures will show up in the log as VCL_Error" + " messages with regexp errors -27 or -21.\n\n" + "Testcase r01576 can be useful when tuning this parameter." +) + # undef PARAM_ALL +# undef PARAM_PCRE # undef PARAM_STRING # undef PARAM_VCC #endif /* defined(PARAM_ALL) */ @@ -1408,32 +1449,6 @@ PARAM( " esi_remove_bom Remove UTF-8 BOM" ) -/* actual location mgt_param_tbl.c */ -PARAM( - /* name */ pcre_match_limit, - /* type */ uint, - /* min */ "1", - /* max */ NULL, - /* def */ "1.000", - /* units */ NULL, - /* descr */ - "The limit for the number of internal matching function calls in " - "a pcre_exec() execution." -) - -/* actual location mgt_param_tbl.c */ -PARAM( - /* name */ pcre_match_limit_recursion, - /* type */ uint, - /* min */ "1", - /* max */ NULL, - /* def */ "1.000", - /* units */ NULL, - /* descr */ - "The limit for the number of internal matching function " - "recursions in a pcre_exec() execution." -) - /* actual location mgt_pool.c */ PARAM( /* name */ thread_pool_add_delay, From dridi.boukelmoune at gmail.com Wed Oct 21 17:24:13 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 21 Oct 2020 17:24:13 +0000 (UTC) Subject: [master] fa681397f param: Introduce the notion of thread pool parameters Message-ID: <20201021172413.8F7B4631BF@lists.varnish-cache.org> commit fa681397fa6b6849b25ea5ef738eb0044d147093 Author: Dridi Boukelmoune Date: Wed Oct 21 18:40:31 2020 +0200 param: Introduce the notion of thread pool parameters Unlike simple parameters, the field names in struct params don't match the parameters names. To deal with that a new field is added to PARAM() macros to allow simple and thread parameters to set themselves apart in that regard without getting any more intrusive. Refs #3250 diff --git a/bin/varnishd/common/common_param.h b/bin/varnishd/common/common_param.h index 6e612593f..fd4cfc0fd 100644 --- a/bin/varnishd/common/common_param.h +++ b/bin/varnishd/common/common_param.h @@ -73,23 +73,27 @@ struct poolparam { struct params { -#define ptyp_bool unsigned -#define ptyp_bytes ssize_t -#define ptyp_bytes_u unsigned -#define ptyp_double double -#define ptyp_poolparam struct poolparam -#define ptyp_timeout double -#define ptyp_uint unsigned -#define ptyp_vsl_buffer unsigned -#define ptyp_vsl_reclen unsigned -#define PARAM(ty, nm, ...) \ - ptyp_##ty nm; +#define ptyp_bool unsigned +#define ptyp_bytes ssize_t +#define ptyp_bytes_u unsigned +#define ptyp_double double +#define ptyp_poolparam struct poolparam +#define ptyp_thread_pool_max unsigned +#define ptyp_thread_pool_min unsigned +#define ptyp_timeout double +#define ptyp_uint unsigned +#define ptyp_vsl_buffer unsigned +#define ptyp_vsl_reclen unsigned +#define PARAM(typ, fld, nm, ...) \ + ptyp_##typ fld; #include #undef ptyp_bool #undef ptyp_bytes #undef ptyp_bytes_u #undef ptyp_double #undef ptyp_poolparam +#undef ptyp_thread_pool_max +#undef ptyp_thread_pool_min #undef ptyp_timeout #undef ptyp_uint #undef ptyp_vsl_buffer @@ -99,20 +103,6 @@ struct params { uid_t uid; gid_t gid; - /* Worker threads and pool */ - unsigned wthread_min; - unsigned wthread_max; - unsigned wthread_reserve; - double wthread_timeout; - unsigned wthread_pools; - double wthread_add_delay; - double wthread_fail_delay; - double wthread_destroy_delay; - double wthread_watchdog; - unsigned wthread_stats_rate; - ssize_t wthread_stacksize; - unsigned wthread_queue_limit; - struct vre_limits vre_limits; uint8_t vsl_mask[256>>3]; diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index df62c2859..6d342297c 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -658,7 +658,6 @@ MCF_InitParams(struct cli *cli) ssize_t def, low; MCF_AddParams(mgt_parspec); - MCF_AddParams(WRK_parspec); MCF_AddParams(VSL_parspec); MCF_TcpParams(); diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h index a23616686..41bc4d9f5 100644 --- a/bin/varnishd/mgt/mgt_param.h +++ b/bin/varnishd/mgt/mgt_param.h @@ -71,6 +71,8 @@ tweak_t tweak_bytes_u; tweak_t tweak_double; tweak_t tweak_poolparam; tweak_t tweak_string; +tweak_t tweak_thread_pool_min; +tweak_t tweak_thread_pool_max; tweak_t tweak_timeout; tweak_t tweak_uint; tweak_t tweak_vsl_buffer; diff --git a/bin/varnishd/mgt/mgt_param_tbl.c b/bin/varnishd/mgt/mgt_param_tbl.c index f911aff20..6b4110b08 100644 --- a/bin/varnishd/mgt/mgt_param_tbl.c +++ b/bin/varnishd/mgt/mgt_param_tbl.c @@ -38,7 +38,7 @@ struct parspec mgt_parspec[] = { #define PARAM_ALL -#define PARAM(ty, nm, ...) { #nm, __VA_ARGS__ }, +#define PARAM(typ, fld, nm, ...) { #nm, __VA_ARGS__ }, #include "tbl/params.h" { NULL, NULL, NULL } }; diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c index a2e73245f..02606bcc0 100644 --- a/bin/varnishd/mgt/mgt_pool.c +++ b/bin/varnishd/mgt/mgt_pool.c @@ -55,7 +55,7 @@ * limit, so they don't end up crossing. */ -static int +int tweak_thread_pool_min(struct vsb *vsb, const struct parspec *par, const char *arg) { @@ -69,7 +69,7 @@ tweak_thread_pool_min(struct vsb *vsb, const struct parspec *par, return (0); } -static int +int tweak_thread_pool_max(struct vsb *vsb, const struct parspec *par, const char *arg) { @@ -81,192 +81,3 @@ tweak_thread_pool_max(struct vsb *vsb, const struct parspec *par, "%u", mgt_param.wthread_max); return (0); } - -/*-------------------------------------------------------------------- - * The thread pool parameter definitions used to generate the varnishd - * manual. Check the generated RST after updating. - */ - -struct parspec WRK_parspec[] = { - { "thread_pools", tweak_uint, &mgt_param.wthread_pools, - "1", NULL, /* maximum defined in mgt_param.c */ "2", - "pools", - "Number of worker thread pools.\n" - "\n" - "Increasing the number of worker pools decreases lock " - "contention. Each worker pool also has a thread accepting " - "new connections, so for very high rates of incoming new " - "connections on systems with many cores, increasing the " - "worker pools may be required.\n" - "\n" - "Too many pools waste CPU and RAM resources, and more than one " - "pool for each CPU is most likely detrimental to performance.\n" - "\n" - "Can be increased on the fly, but decreases require a " - "restart to take effect, unless the drop_pools experimental " - "debug flag is set.", - EXPERIMENTAL | DELAYED_EFFECT, - NULL, "defined when Varnish is built" }, - { "thread_pool_max", tweak_thread_pool_max, &mgt_param.wthread_max, - NULL, NULL, "5000", - "threads", - "The maximum number of worker threads in each pool.\n" - "\n" - "Do not set this higher than you have to, since excess " - "worker threads soak up RAM and CPU and generally just get " - "in the way of getting work done.", - DELAYED_EFFECT, - "thread_pool_min" }, - { "thread_pool_min", tweak_thread_pool_min, &mgt_param.wthread_min, - "5" /* TASK_QUEUE__END */, NULL, "100", - "threads", - "The minimum number of worker threads in each pool.\n" - "\n" - "Increasing this may help ramp up faster from low load " - "situations or when threads have expired.\n" - "\n" - "Technical minimum is 5 threads, " // TASK_QUEUE__END - "but this parameter is strongly recommended to be " - "at least 10", // 2 * TASK_QUEUE__END - DELAYED_EFFECT, - NULL, "thread_pool_max" }, - { "thread_pool_reserve", tweak_uint, - &mgt_param.wthread_reserve, - NULL, NULL, "0", - "threads", - "The number of worker threads reserved for vital tasks " - "in each pool.\n" - "\n" - "Tasks may require other tasks to complete (for example, " - "client requests may require backend requests, http2 sessions " - "require streams, which require requests). This reserve is to " - "ensure that lower priority tasks do not prevent higher " - "priority tasks from running even under high load.\n" - "\n" - "The effective value is at least 5 (the number of internal " - // ^ TASK_QUEUE__END - "priority classes), irrespective of this parameter.\n" - "Default is 0 to auto-tune (5% of thread_pool_min).\n" - "Minimum is 1 otherwise, maximum is 95% of thread_pool_min.", - DELAYED_EFFECT, - NULL, "95% of thread_pool_min" }, - { "thread_pool_timeout", - tweak_timeout, &mgt_param.wthread_timeout, - "10", NULL, "300", - "seconds", - "Thread idle threshold.\n" - "\n" - "Threads in excess of thread_pool_min, which have been idle " - "for at least this long, will be destroyed.", - EXPERIMENTAL | DELAYED_EFFECT }, - { "thread_pool_watchdog", - tweak_timeout, &mgt_param.wthread_watchdog, - "0.1", NULL, "60", - "seconds", - "Thread queue stuck watchdog.\n" - "\n" - "If no queued work have been released for this long," - " the worker process panics itself.", - EXPERIMENTAL }, - { "thread_pool_destroy_delay", - tweak_timeout, &mgt_param.wthread_destroy_delay, - "0.01", NULL, "1", - "seconds", - "Wait this long after destroying a thread.\n" - "\n" - "This controls the decay of thread pools when idle(-ish).", - EXPERIMENTAL | DELAYED_EFFECT }, - { "thread_pool_add_delay", - tweak_timeout, &mgt_param.wthread_add_delay, - "0", NULL, "0", - "seconds", - "Wait at least this long after creating a thread.\n" - "\n" - "Some (buggy) systems may need a short (sub-second) " - "delay between creating threads.\n" - "Set this to a few milliseconds if you see the " - "'threads_failed' counter grow too much.\n" - "\n" - "Setting this too high results in insufficient worker threads.", - EXPERIMENTAL }, - { "thread_pool_fail_delay", - tweak_timeout, &mgt_param.wthread_fail_delay, - "10e-3", NULL, "0.2", - "seconds", - "Wait at least this long after a failed thread creation " - "before trying to create another thread.\n" - "\n" - "Failure to create a worker thread is often a sign that " - " the end is near, because the process is running out of " - "some resource. " - "This delay tries to not rush the end on needlessly.\n" - "\n" - "If thread creation failures are a problem, check that " - "thread_pool_max is not too high.\n" - "\n" - "It may also help to increase thread_pool_timeout and " - "thread_pool_min, to reduce the rate at which treads are " - "destroyed and later recreated.", - EXPERIMENTAL }, - { "thread_stats_rate", - tweak_uint, &mgt_param.wthread_stats_rate, - "0", NULL, "10", - "requests", - "Worker threads accumulate statistics, and dump these into " - "the global stats counters if the lock is free when they " - "finish a job (request/fetch etc.)\n" - "This parameters defines the maximum number of jobs " - "a worker thread may handle, before it is forced to dump " - "its accumulated stats into the global counters.", - EXPERIMENTAL }, - { "thread_queue_limit", tweak_uint, &mgt_param.wthread_queue_limit, - "0", NULL, "20", - NULL, - "Permitted request queue length per thread-pool.\n" - "\n" - "This sets the number of requests we will queue, waiting " - "for an available thread. Above this limit sessions will " - "be dropped instead of queued.", - EXPERIMENTAL }, - { "thread_pool_stack", - tweak_bytes, &mgt_param.wthread_stacksize, - NULL, NULL, NULL, // default set in mgt_param.c - "bytes", - "Worker thread stack size.\n" - "This will likely be rounded up to a multiple of 4k" - " (or whatever the page_size might be) by the kernel.\n" - "\n" - "The required stack size is primarily driven by the" - " depth of the call-tree. The most common relevant" - " determining factors in varnish core code are GZIP" - " (un)compression, ESI processing and regular" - " expression matches. VMODs may also require" - " significant amounts of additional stack. The" - " nesting depth of VCL subs is another factor," - " although typically not predominant.\n" - "\n" - "The stack size is per thread, so the maximum total" - " memory required for worker thread stacks is in the" - " order of size = thread_pools x thread_pool_max x" - " thread_pool_stack.\n" - "\n" - "Thus, in particular for setups with many threads," - " keeping the stack size at a minimum helps reduce" - " the amount of memory required by Varnish.\n" - "\n" - "On the other hand, thread_pool_stack must be large" - " enough under all circumstances, otherwise varnish" - " will crash due to a stack overflow. Usually, a" - " stack overflow manifests itself as a segmentation" - " fault (aka segfault / SIGSEGV) with the faulting" - " address being near the stack pointer (sp).\n" - "\n" - "Unless stack usage can be reduced," - " thread_pool_stack must be increased when a stack" - " overflow occurs. Setting it in 150%-200%" - " increments is recommended until stack overflows" - " cease to occur.", - DELAYED_EFFECT, - NULL, NULL, "sysconf(_SC_THREAD_STACK_MIN)" }, - { NULL, NULL, NULL } -}; diff --git a/include/tbl/params.h b/include/tbl/params.h index c3292b92b..34180d927 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -27,10 +27,11 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * PARAM(type, ...) + * PARAM(typ, fld, ...) * - * Variable arguments refer to struct parspec fields, except non-const - * strings. + * typ: parameter type + * fld: struct params field name + * ...: struct parspec fields, except the non-const string fields */ /*lint -save -e525 -e539 */ @@ -39,8 +40,8 @@ * * Simple parameters * */ -#define PARAM_SIMPLE(nm, ty, ...) \ - PARAM(ty, nm, tweak_##ty, &mgt_param.nm, __VA_ARGS__) +#define PARAM_SIMPLE(nm, typ, ...) \ + PARAM(typ, nm, nm, tweak_##typ, &mgt_param.nm, __VA_ARGS__) #if defined(XYZZY) #error "Temporary macro XYZZY already defined" @@ -1260,6 +1261,285 @@ PARAM_SIMPLE( #undef MEMPOOL_TEXT +/*-------------------------------------------------------------------- + * Thread pool parameters + */ + +#define PARAM_THREAD(nm, fld, typ, ...) \ + PARAM(typ, wthread_ ## fld, nm, tweak_ ## typ, \ + &mgt_param.wthread_ ## fld, __VA_ARGS__) + +PARAM_THREAD( + /* name */ thread_pools, + /* field */ pools, + /* type */ uint, + /* min */ "1", + /* max */ NULL, + /* def */ "2", + /* units */ "pools", + /* descr */ + "Number of worker thread pools.\n" + "\n" + "Increasing the number of worker pools decreases lock " + "contention. Each worker pool also has a thread accepting " + "new connections, so for very high rates of incoming new " + "connections on systems with many cores, increasing the " + "worker pools may be required.\n" + "\n" + "Too many pools waste CPU and RAM resources, and more than one " + "pool for each CPU is most likely detrimental to performance.\n" + "\n" + "Can be increased on the fly, but decreases require a " + "restart to take effect, unless the drop_pools experimental " + "debug flag is set.", + /* flags */ EXPERIMENTAL | DELAYED_EFFECT +) + +PARAM_THREAD( + /* name */ thread_pool_max, + /* field */ max, + /* type */ thread_pool_max, + /* min */ NULL, + /* max */ NULL, + /* def */ "5000", + /* units */ "threads", + /* descr */ + "The maximum number of worker threads in each pool.\n" + "\n" + "Do not set this higher than you have to, since excess " + "worker threads soak up RAM and CPU and generally just get " + "in the way of getting work done.", + /* flags */ DELAYED_EFFECT, + /* dyn_min_reason */ "thread_pool_min" +) + +PARAM_THREAD( + /* name */ thread_pool_min, + /* field */ min, + /* type */ thread_pool_min, + /* min */ "5" /* TASK_QUEUE__END */, + /* max */ NULL, + /* def */ "100", + /* units */ "threads", + /* descr */ + "The minimum number of worker threads in each pool.\n" + "\n" + "Increasing this may help ramp up faster from low load " + "situations or when threads have expired.\n" + "\n" + "Technical minimum is 5 threads, but this parameter is " + /* ^ TASK_QUEUE__END */ + "strongly recommended to be at least 10", + /* 2 * TASK_QUEUE__END ^^ */ + /* flags */ DELAYED_EFFECT, + /* dyn_min_reason */ NULL, + /* dyn_max_reason */ "thread_pool_max" +) + +PARAM_THREAD( + /* name */ thread_pool_reserve, + /* field */ reserve, + /* type */ uint, + /* min */ NULL, + /* max */ NULL, + /* def */ "0", + /* units */ "threads", + /* descr */ + "The number of worker threads reserved for vital tasks " + "in each pool.\n" + "\n" + "Tasks may require other tasks to complete (for example, " + "client requests may require backend requests, http2 sessions " + "require streams, which require requests). This reserve is to " + "ensure that lower priority tasks do not prevent higher " + "priority tasks from running even under high load.\n" + "\n" + "The effective value is at least 5 (the number of internal " + /* ^ TASK_QUEUE__END */ + "priority classes), irrespective of this parameter.", + /* flags */ DELAYED_EFFECT, + /* dyn_min_reason */ NULL, + /* dyn_max_reason */ "95% of thread_pool_min", + /* dyn_def_reason */ "0 (auto-tune: 5% of thread_pool_min)" +) + +PARAM_THREAD( + /* name */ thread_pool_timeout, + /* field */ timeout, + /* type */ timeout, + /* min */ "10", + /* max */ NULL, + /* def */ "300", + /* units */ "seconds", + /* descr */ + "Thread idle threshold.\n" + "\n" + "Threads in excess of thread_pool_min, which have been idle " + "for at least this long, will be destroyed.", + /* flags */ EXPERIMENTAL | DELAYED_EFFECT +) + +PARAM_THREAD( + /* name */ thread_pool_watchdog, + /* field */ watchdog, + /* type */ timeout, + /* min */ "0.1", + /* max */ NULL, + /* def */ "60", + /* units */ "seconds", + /* descr */ + "Thread queue stuck watchdog.\n" + "\n" + "If no queued work have been released for this long," + " the worker process panics itself.", + /* flags */ EXPERIMENTAL +) + +PARAM_THREAD( + /* name */ thread_pool_destroy_delay, + /* field */ destroy_delay, + /* type */ timeout, + /* min */ "0.01", + /* max */ NULL, + /* def */ "1", + /* units */ "seconds", + /* descr */ + "Wait this long after destroying a thread.\n" + "\n" + "This controls the decay of thread pools when idle(-ish).", + /* flags */ EXPERIMENTAL | DELAYED_EFFECT +) + +PARAM_THREAD( + /* name */ thread_pool_add_delay, + /* field */ add_delay, + /* type */ timeout, + /* min */ "0", + /* max */ NULL, + /* def */ "0", + /* units */ "seconds", + /* descr */ + "Wait at least this long after creating a thread.\n" + "\n" + "Some (buggy) systems may need a short (sub-second) " + "delay between creating threads.\n" + "Set this to a few milliseconds if you see the " + "'threads_failed' counter grow too much.\n" + "\n" + "Setting this too high results in insufficient worker threads.", + /* flags */ EXPERIMENTAL +) + +PARAM_THREAD( + /* name */ thread_pool_fail_delay, + /* field */ fail_delay, + /* type */ timeout, + /* min */ "10e-3", + /* max */ NULL, + /* def */ "0.2", + /* units */ "seconds", + /* descr */ + "Wait at least this long after a failed thread creation " + "before trying to create another thread.\n" + "\n" + "Failure to create a worker thread is often a sign that " + " the end is near, because the process is running out of " + "some resource. " + "This delay tries to not rush the end on needlessly.\n" + "\n" + "If thread creation failures are a problem, check that " + "thread_pool_max is not too high.\n" + "\n" + "It may also help to increase thread_pool_timeout and " + "thread_pool_min, to reduce the rate at which treads are " + "destroyed and later recreated.", + /* flags */ EXPERIMENTAL +) + +PARAM_THREAD( + /* name */ thread_stats_rate, + /* field */ stats_rate, + /* type */ uint, + /* min */ "0", + /* max */ NULL, + /* def */ "10", + /* units */ "requests", + /* descr */ + "Worker threads accumulate statistics, and dump these into " + "the global stats counters if the lock is free when they " + "finish a job (request/fetch etc.)\n" + "This parameters defines the maximum number of jobs " + "a worker thread may handle, before it is forced to dump " + "its accumulated stats into the global counters.", + /* flags */ EXPERIMENTAL +) + +PARAM_THREAD( + /* name */ thread_queue_limit, + /* field */ queue_limit, + /* type */ uint, + /* min */ "0", + /* max */ NULL, + /* def */ "20", + /* units */ "requests", + /* descr */ + "Permitted request queue length per thread-pool.\n" + "\n" + "This sets the number of requests we will queue, waiting " + "for an available thread. Above this limit sessions will " + "be dropped instead of queued.", + /* flags */ EXPERIMENTAL +) + +PARAM_THREAD( + /* name */ thread_pool_stack, + /* field */ stacksize, + /* type */ bytes, + /* min */ NULL, + /* max */ NULL, + /* def */ NULL, /* default set in mgt_param.c */ + /* units */ "bytes", + /* descr */ + "Worker thread stack size.\n" + "This will likely be rounded up to a multiple of 4k" + " (or whatever the page_size might be) by the kernel.\n" + "\n" + "The required stack size is primarily driven by the" + " depth of the call-tree. The most common relevant" + " determining factors in varnish core code are GZIP" + " (un)compression, ESI processing and regular" + " expression matches. VMODs may also require" + " significant amounts of additional stack. The" + " nesting depth of VCL subs is another factor," + " although typically not predominant.\n" + "\n" + "The stack size is per thread, so the maximum total" + " memory required for worker thread stacks is in the" + " order of size = thread_pools x thread_pool_max x" + " thread_pool_stack.\n" + "\n" + "Thus, in particular for setups with many threads," + " keeping the stack size at a minimum helps reduce" + " the amount of memory required by Varnish.\n" + "\n" + "On the other hand, thread_pool_stack must be large" + " enough under all circumstances, otherwise varnish" + " will crash due to a stack overflow. Usually, a" + " stack overflow manifests itself as a segmentation" + " fault (aka segfault / SIGSEGV) with the faulting" + " address being near the stack pointer (sp).\n" + "\n" + "Unless stack usage can be reduced," + " thread_pool_stack must be increased when a stack" + " overflow occurs. Setting it in 150%-200%" + " increments is recommended until stack overflows" + " cease to occur.", + /* flags */ DELAYED_EFFECT, + /* dyn_min_reason */ NULL, + /* dyn_max_reason */ NULL, + /* dyn_def_reason */ "sysconf(_SC_THREAD_STACK_MIN)" +) + #if defined(PARAM_ALL) /*-------------------------------------------------------------------- @@ -1267,7 +1547,7 @@ PARAM_SIMPLE( */ # define PARAM_STRING(nm, pv, def, ...) \ - PARAM(, nm, tweak_string, pv, NULL, NULL, def, NULL, __VA_ARGS__) + PARAM(, , nm, tweak_string, pv, NULL, NULL, def, NULL, __VA_ARGS__) PARAM_STRING( /* name */ cc_command, @@ -1309,7 +1589,7 @@ PARAM_STRING( */ # define PARAM_VCC(nm, def, descr) \ - PARAM(, nm, tweak_bool, &mgt_ ## nm, NULL, NULL, def, "bool", descr) + PARAM(, , nm, tweak_bool, &mgt_ ## nm, NULL, NULL, def, "bool", descr) PARAM_VCC( /* name */ vcc_err_unref, @@ -1354,7 +1634,7 @@ PARAM_VCC( */ # define PARAM_PCRE(nm, pv, min, def, descr) \ - PARAM(, nm, tweak_uint, &mgt_param.vre_limits.pv, \ + PARAM(, , nm, tweak_uint, &mgt_param.vre_limits.pv, \ min, NULL, def, NULL, descr) PARAM_PCRE( @@ -1396,6 +1676,7 @@ PARAM_PCRE( #endif /* defined(PARAM_ALL) */ #undef PARAM_SIMPLE +#undef PARAM_THREAD #undef PARAM #if 0 /* NOT ACTUALLY DEFINED HERE */ @@ -1449,232 +1730,6 @@ PARAM( " esi_remove_bom Remove UTF-8 BOM" ) -/* actual location mgt_pool.c */ -PARAM( - /* name */ thread_pool_add_delay, - /* type */ timeout, - /* min */ "0.000", - /* max */ NULL, - /* def */ "0.000", - /* units */ "seconds", - /* descr */ - "Wait at least this long after creating a thread.\n" - "\n" - "Some (buggy) systems may need a short (sub-second) delay between " - "creating threads.\n" - "Set this to a few milliseconds if you see the 'threads_failed' " - "counter grow too much.\n" - "Setting this too high results in insufficient worker threads.", - /* flags */ EXPERIMENTAL -) - -/* actual location mgt_pool.c */ -PARAM( - /* name */ thread_pool_watchdog, - /* type */ timeout, - /* min */ "0.1", - /* max */ NULL, - /* def */ "60.000", - /* units */ "seconds", - /* descr */ - "Thread queue stuck watchdog.\n" - "\n" - "If no queued work have been released for this long," - " the worker process panics itself.", - /* flags */ EXPERIMENTAL -) - -/* actual location mgt_pool.c */ -PARAM( - /* name */ thread_pool_destroy_delay, - /* type */ timeout, - /* min */ "0.010", - /* max */ NULL, - /* def */ "1.000", - /* units */ "seconds", - /* descr */ - "Wait this long after destroying a thread.\n" - "This controls the decay of thread pools when idle(-ish).", - /* flags */ DELAYED_EFFECT| EXPERIMENTAL -) - -/* actual location mgt_pool.c */ -PARAM( - /* name */ thread_pool_fail_delay, - /* type */ timeout, - /* min */ "0.010", - /* max */ NULL, - /* def */ "0.200", - /* units */ "seconds", - /* descr */ - "Wait at least this long after a failed thread creation before " - "trying to create another thread.\n" - "\n" - "Failure to create a worker thread is often a sign that the end " - "is near, because the process is running out of some resource. " - "This delay tries to not rush the end on needlessly.\n" - "\n" - "If thread creation failures are a problem, check that " - "thread_pool_max is not too high.\n" - "\n" - "It may also help to increase thread_pool_timeout and " - "thread_pool_min, to reduce the rate at which treads are destroyed " - "and later recreated.", - /* flags */ EXPERIMENTAL -) - -/* actual location mgt_pool.c */ -PARAM( - /* name */ thread_pool_max, - /* type */ thread_pool_max, - /* min */ "100", - /* max */ NULL, - /* def */ "5000", - /* units */ "threads", - /* descr */ - "The maximum number of worker threads in each pool.\n" - "\n" - "Do not set this higher than you have to, since excess worker " - "threads soak up RAM and CPU and generally just get in the way of " - "getting work done.", - /* flags */ DELAYED_EFFECT -) - -/* actual location mgt_pool.c */ -PARAM( - /* name */ thread_pool_min, - /* type */ thread_pool_min, - /* min */ NULL, - /* max */ "5000", - /* def */ "100", - /* units */ "threads", - /* descr */ - "The minimum number of worker threads in each pool.\n" - "\n" - "Increasing this may help ramp up faster from low load situations " - "or when threads have expired." - "Minimum is 10 threads.", - /* flags */ DELAYED_EFFECT -) - -/* actual location mgt_pool.c */ -PARAM( - /* name */ thread_pool_reserve, - /* type */ thread_pool_reserve, - /* min */ NULL, - /* max */ NULL, - /* def */ "0", - /* units */ "threads", - /* descr */ - "The number of worker threads reserved for vital tasks " - "in each pool.\n" - "\n" - "Tasks may require other tasks to complete (for example, " - "client requests may require backend requests). This reserve " - "is to ensure that such tasks still get to run even under high " - "load.\n" - "\n" - "Increasing the reserve may help setups with a high number of " - "backend requests at the expense of client performance. " - "Setting it too high will waste resources by keeping threads " - "unused.\n" - "\n" - "Default is 0 to auto-tune (currently 5% of thread_pool_min).\n" - "Minimum is 1 otherwise, maximum is 95% of thread_pool_min.", - /* flags */ DELAYED_EFFECT| EXPERIMENTAL -) - -/* actual location mgt_pool.c */ -PARAM( - /* name */ thread_pool_stack, - /* type */ bytes, - /* min */ "2k", - /* max */ NULL, - /* def */ "56k", - /* units */ "bytes", - /* descr */ - "Worker thread stack size.\n" - "This will likely be rounded up to a multiple of 4k (or whatever " - "the page_size might be) by the kernel.", - /* flags */ EXPERIMENTAL -) - -/* actual location mgt_pool.c */ -PARAM( - /* name */ thread_pool_timeout, - /* type */ timeout, - /* min */ "10.000", - /* max */ NULL, - /* def */ "300.000", - /* units */ "seconds", - /* descr */ - "Thread idle threshold.\n" - "\n" - "Threads in excess of thread_pool_min, which have been idle for at " - "least this long, will be destroyed.", - /* flags */ DELAYED_EFFECT| EXPERIMENTAL -) - -/* actual location mgt_pool.c */ -PARAM( - /* name */ thread_pools, - /* type */ uint, - /* min */ "1", - /* max */ NULL, - /* def */ "2", - /* units */ "pools", - /* descr */ - "Number of worker thread pools.\n" - "\n" - "Increasing the number of worker pools decreases lock " - "contention. Each worker pool also has a thread accepting " - "new connections, so for very high rates of incoming new " - "connections on systems with many cores, increasing the " - "worker pools may be required.\n" - "\n" - "Too many pools waste CPU and RAM resources, and more than one " - "pool for each CPU is most likely detrimental to performance.\n" - "\n" - "Can be increased on the fly, but decreases require a restart to " - "take effect.", - /* flags */ DELAYED_EFFECT| EXPERIMENTAL -) - -/* actual location mgt_pool.c */ -PARAM( - /* name */ thread_queue_limit, - /* type */ uint, - /* min */ "0", - /* max */ NULL, - /* def */ "20", - /* units */ NULL, - /* descr */ - "Permitted request queue length per thread-pool.\n" - "\n" - "This sets the number of requests we will queue, waiting for an " - "available thread. Above this limit sessions will be dropped " - "instead of queued.", - /* flags */ EXPERIMENTAL -) - -/* actual location mgt_pool.c */ -PARAM( - /* name */ thread_stats_rate, - /* type */ uint, - /* min */ "0", - /* max */ NULL, - /* def */ "10", - /* units */ "requests", - /* descr */ - "Worker threads accumulate statistics, and dump these into the " - "global stats counters if the lock is free when they finish a job " - "(request/fetch etc).\n" - "This parameters defines the maximum number of jobs a worker " - "thread may handle, before it is forced to dump its accumulated " - "stats into the global counters.", - /* flags */ EXPERIMENTAL -) - /* actual location mgt_param_bits.c*/ PARAM( /* name */ vsl_mask, From dridi.boukelmoune at gmail.com Wed Oct 21 17:24:13 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 21 Oct 2020 17:24:13 +0000 (UTC) Subject: [master] 74c119c88 param: Fold mgt_pool.c into mgt_param_tweak.c Message-ID: <20201021172413.AE9E4631C6@lists.varnish-cache.org> commit 74c119c88d77636843d0416c23060fd80d0cbf24 Author: Dridi Boukelmoune Date: Wed Oct 21 18:50:52 2020 +0200 param: Fold mgt_pool.c into mgt_param_tweak.c Refs #3250 diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 3b87ddfe6..edc6a3ff3 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -88,7 +88,6 @@ varnishd_SOURCES = \ mgt/mgt_param_tbl.c \ mgt/mgt_param_tcp.c \ mgt/mgt_param_tweak.c \ - mgt/mgt_pool.c \ mgt/mgt_shmem.c \ mgt/mgt_symtab.c \ mgt/mgt_util.c \ diff --git a/bin/varnishd/mgt/mgt_param_tweak.c b/bin/varnishd/mgt/mgt_param_tweak.c index a947fd52b..fe06f1ce5 100644 --- a/bin/varnishd/mgt/mgt_param_tweak.c +++ b/bin/varnishd/mgt/mgt_param_tweak.c @@ -449,3 +449,37 @@ tweak_poolparam(struct vsb *vsb, const struct parspec *par, const char *arg) } return (retval); } + +/*-------------------------------------------------------------------- + * Thread pool tweaks. + * + * The min/max values automatically update the opposites appropriate + * limit, so they don't end up crossing. + */ + +int +tweak_thread_pool_min(struct vsb *vsb, const struct parspec *par, + const char *arg) +{ + if (tweak_uint(vsb, par, arg)) + return (-1); + + MCF_ParamConf(MCF_MINIMUM, "thread_pool_max", + "%u", mgt_param.wthread_min); + MCF_ParamConf(MCF_MAXIMUM, "thread_pool_reserve", + "%u", mgt_param.wthread_min * 950 / 1000); + return (0); +} + +int +tweak_thread_pool_max(struct vsb *vsb, const struct parspec *par, + const char *arg) +{ + + if (tweak_uint(vsb, par, arg)) + return (-1); + + MCF_ParamConf(MCF_MAXIMUM, "thread_pool_min", + "%u", mgt_param.wthread_max); + return (0); +} diff --git a/bin/varnishd/mgt/mgt_pool.c b/bin/varnishd/mgt/mgt_pool.c deleted file mode 100644 index 02606bcc0..000000000 --- a/bin/varnishd/mgt/mgt_pool.c +++ /dev/null @@ -1,83 +0,0 @@ -/*- - * Copyright (c) 2006 Verdens Gang AS - * Copyright (c) 2006-2011 Varnish Software AS - * All rights reserved. - * - * Author: Poul-Henning Kamp - * - * SPDX-License-Identifier: BSD-2-Clause - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * We maintain a number of worker thread pools, to spread lock contention. - * - * Pools can be added on the fly, as a means to mitigate lock contention, - * but can only be removed again by a restart. (XXX: we could fix that) - * - * Two threads herd the pools, one eliminates idle threads and aggregates - * statistics for all the pools, the other thread creates new threads - * on demand, subject to various numerical constraints. - * - * The algorithm for when to create threads needs to be reactive enough - * to handle startup spikes, but sufficiently attenuated to not cause - * thread pileups. This remains subject for improvement. - */ - -#include "config.h" - -#include - -#include "mgt/mgt.h" - -#include "mgt/mgt_param.h" - -/*-------------------------------------------------------------------- - * The min/max values automatically update the opposites appropriate - * limit, so they don't end up crossing. - */ - -int -tweak_thread_pool_min(struct vsb *vsb, const struct parspec *par, - const char *arg) -{ - if (tweak_uint(vsb, par, arg)) - return (-1); - - MCF_ParamConf(MCF_MINIMUM, "thread_pool_max", - "%u", mgt_param.wthread_min); - MCF_ParamConf(MCF_MAXIMUM, "thread_pool_reserve", - "%u", mgt_param.wthread_min * 950 / 1000); - return (0); -} - -int -tweak_thread_pool_max(struct vsb *vsb, const struct parspec *par, - const char *arg) -{ - - if (tweak_uint(vsb, par, arg)) - return (-1); - - MCF_ParamConf(MCF_MAXIMUM, "thread_pool_min", - "%u", mgt_param.wthread_max); - return (0); -} From dridi.boukelmoune at gmail.com Wed Oct 21 17:24:13 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 21 Oct 2020 17:24:13 +0000 (UTC) Subject: [master] 29568e06b param: Introduce the notion of memory pool parameters Message-ID: <20201021172413.CBAD0631CE@lists.varnish-cache.org> commit 29568e06b74c862d678be35b6c3709c798ce1f64 Author: Dridi Boukelmoune Date: Wed Oct 21 19:11:58 2020 +0200 param: Introduce the notion of memory pool parameters Refs #3250 diff --git a/include/tbl/params.h b/include/tbl/params.h index 34180d927..5b64b462e 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1217,49 +1217,39 @@ PARAM_SIMPLE( "HTTP2 maximum size of an uncompressed header list." ) -#define MEMPOOL_TEXT \ - "The three numbers are:\n" \ - "\tmin_pool\tminimum size of free pool.\n" \ - "\tmax_pool\tmaximum size of free pool.\n" \ - "\tmax_age\tmax age of free element." +/*-------------------------------------------------------------------- + * Memory pool parameters + */ -PARAM_SIMPLE( - /* name */ pool_req, - /* type */ poolparam, - /* min */ NULL, - /* max */ NULL, - /* def */ "10,100,10", - /* units */ NULL, - /* descr */ - "Parameters for per worker pool request memory pool.\n" - MEMPOOL_TEXT -) +#define PARAM_MEMPOOL(nm, def, descr) \ + PARAM(poolparam, nm, nm, tweak_poolparam, &mgt_param.nm, \ + NULL, NULL, def, NULL, \ + descr \ + "The three numbers are:\n" \ + "\tmin_pool\tminimum size of free pool.\n" \ + "\tmax_pool\tmaximum size of free pool.\n" \ + "\tmax_age\tmax age of free element.") -PARAM_SIMPLE( - /* name */ pool_sess, - /* type */ poolparam, - /* min */ NULL, - /* max */ NULL, - /* def */ "10,100,10", - /* units */ NULL, - /* descr */ - "Parameters for per worker pool session memory pool.\n" - MEMPOOL_TEXT +PARAM_MEMPOOL( + /* name */ pool_req, + /* def */ "10,100,10", + /* descr */ + "Parameters for per worker pool request memory pool.\n\n" ) -PARAM_SIMPLE( - /* name */ pool_vbo, - /* type */ poolparam, - /* min */ NULL, - /* max */ NULL, - /* def */ "10,100,10", - /* units */ NULL, - /* descr */ - "Parameters for backend object fetch memory pool.\n" - MEMPOOL_TEXT +PARAM_MEMPOOL( + /* name */ pool_sess, + /* def */ "10,100,10", + /* descr */ + "Parameters for per worker pool session memory pool.\n\n" ) -#undef MEMPOOL_TEXT +PARAM_MEMPOOL( + /* name */ pool_vbo, + /* def */ "10,100,10", + /* descr */ + "Parameters for backend object fetch memory pool.\n\n" +) /*-------------------------------------------------------------------- * Thread pool parameters @@ -1675,6 +1665,7 @@ PARAM_PCRE( # undef PARAM_VCC #endif /* defined(PARAM_ALL) */ +#undef PARAM_MEMPOOL #undef PARAM_SIMPLE #undef PARAM_THREAD #undef PARAM From dridi.boukelmoune at gmail.com Wed Oct 21 17:24:13 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 21 Oct 2020 17:24:13 +0000 (UTC) Subject: [master] 0f193e508 param: Introduce a PLATFORM_DEPENDENT parameter flag Message-ID: <20201021172413.E6176631D4@lists.varnish-cache.org> commit 0f193e508d18db15f5108a0cbae897ab640ef33d Author: Dridi Boukelmoune Date: Thu Mar 12 15:27:29 2020 +0100 param: Introduce a PLATFORM_DEPENDENT parameter flag The problem with NOT_IMPLEMENTED is that it's contextual, so we can't reliably generate documentation that marks all conditional parameters as such. While at it, unsign the flags in parameter specs. Slightly better diff with the --word-diff option. Refs #3250 diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 6d342297c..cdd9a01af 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -108,7 +108,7 @@ static const char NOT_IMPLEMENTED_TEXT[] = "This parameter depends on a feature which is not available" " on this platform."; -static const char NOT_IMPLEMENTED_DOC[] = +static const char PLATFORM_DEPENDENT_TEXT[] = "NB: This parameter depends on a feature which is not available" " on all platforms."; @@ -763,8 +763,8 @@ MCF_DumpRstParam(void) printf("~"); printf("\n"); - if (pp->flags && pp->flags & NOT_IMPLEMENTED) - printf("\n%s\n\n", NOT_IMPLEMENTED_DOC); + if (pp->flags && pp->flags & PLATFORM_DEPENDENT) + printf("\n%s\n\n", PLATFORM_DEPENDENT_TEXT); if (pp->units != NULL && *pp->units != '\0') printf("\t* Units: %s\n", pp->units); @@ -783,7 +783,7 @@ MCF_DumpRstParam(void) * XXX: that say if ->min/->max are valid, so we * XXX: can emit those also in help texts. */ - if (pp->flags & ~NOT_IMPLEMENTED) { + if (pp->flags & ~(NOT_IMPLEMENTED|PLATFORM_DEPENDENT)) { printf("\t* Flags: "); q = ""; diff --git a/bin/varnishd/mgt/mgt_param.h b/bin/varnishd/mgt/mgt_param.h index 41bc4d9f5..fd884fd02 100644 --- a/bin/varnishd/mgt/mgt_param.h +++ b/bin/varnishd/mgt/mgt_param.h @@ -46,16 +46,17 @@ struct parspec { const char *def; const char *units; const char *descr; - int flags; -#define DELAYED_EFFECT (1<<0) -#define EXPERIMENTAL (1<<1) -#define MUST_RESTART (1<<2) -#define MUST_RELOAD (1<<3) -#define WIZARD (1<<4) -#define PROTECTED (1<<5) -#define OBJ_STICKY (1<<6) -#define ONLY_ROOT (1<<7) -#define NOT_IMPLEMENTED (1<<8) + unsigned flags; +#define DELAYED_EFFECT (1<<0) +#define EXPERIMENTAL (1<<1) +#define MUST_RESTART (1<<2) +#define MUST_RELOAD (1<<3) +#define WIZARD (1<<4) +#define PROTECTED (1<<5) +#define OBJ_STICKY (1<<6) +#define ONLY_ROOT (1<<7) +#define NOT_IMPLEMENTED (1<<8) +#define PLATFORM_DEPENDENT (1<<9) const char *dyn_min_reason; const char *dyn_max_reason; diff --git a/include/tbl/params.h b/include/tbl/params.h index 5b64b462e..14986dfec 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -43,14 +43,14 @@ #define PARAM_SIMPLE(nm, typ, ...) \ PARAM(typ, nm, nm, tweak_##typ, &mgt_param.nm, __VA_ARGS__) -#if defined(XYZZY) - #error "Temporary macro XYZZY already defined" +#if defined(PLATFORM_FLAGS) +# error "Temporary macro PLATFORM_FLAGS already defined" #endif #if defined(HAVE_ACCEPT_FILTERS) - #define XYZZY MUST_RESTART +# define PLATFORM_FLAGS MUST_RESTART #else - #define XYZZY NOT_IMPLEMENTED +# define PLATFORM_FLAGS NOT_IMPLEMENTED #endif PARAM_SIMPLE( /* name */ accept_filter, @@ -66,12 +66,12 @@ PARAM_SIMPLE( "in the first place. Malformed requests may go unnoticed and not " "increase the client_req_400 counter. GET or HEAD requests with a " "body may be blocked altogether.", - /* flags */ XYZZY, + /* flags */ PLATFORM_DEPENDENT | PLATFORM_FLAGS, /* dyn_min_reason */ NULL, /* dyn_max_reason */ NULL, /* dyn_def_reason */ "on (if your platform supports accept filters)" ) -#undef XYZZY +#undef PLATFORM_FLAGS PARAM_SIMPLE( /* name */ acceptor_sleep_decay, @@ -610,14 +610,10 @@ PARAM_SIMPLE( "of that the response is allowed to take up." ) -#if defined(XYZZY) - #error "Temporary macro XYZZY already defined" -#endif - #if defined(SO_SNDTIMEO_WORKS) - #define XYZZY DELAYED_EFFECT +# define PLATFORM_FLAGS DELAYED_EFFECT #else - #define XYZZY NOT_IMPLEMENTED +# define PLATFORM_FLAGS NOT_IMPLEMENTED #endif PARAM_SIMPLE( /* name */ idle_send_timeout, @@ -632,9 +628,9 @@ PARAM_SIMPLE( "When this timeout is hit, the session is closed.\n\n" "See the man page for `setsockopt(2)` or `socket(7)` under" " ``SO_SNDTIMEO`` for more information.", - /* flags */ XYZZY + /* flags */ PLATFORM_DEPENDENT | PLATFORM_FLAGS ) -#undef XYZZY +#undef PLATFORM_FLAGS PARAM_SIMPLE( /* name */ listen_depth, @@ -774,14 +770,10 @@ PARAM_SIMPLE( /* flags */ EXPERIMENTAL ) -#if defined(XYZZY) - #error "Temporary macro XYZZY already defined" -#endif - #if defined(SO_SNDTIMEO_WORKS) - #define XYZZY DELAYED_EFFECT +# define PLATFORM_FLAGS DELAYED_EFFECT #else - #define XYZZY NOT_IMPLEMENTED +# define PLATFORM_FLAGS NOT_IMPLEMENTED #endif PARAM_SIMPLE( /* name */ send_timeout, @@ -797,9 +789,9 @@ PARAM_SIMPLE( " timeout is extended unless the total time already taken for sending" " the response in its entirety exceeds this many seconds.\n\n" "When this timeout is hit, the session is closed", - /* flags */ XYZZY + /* flags */ PLATFORM_DEPENDENT | PLATFORM_FLAGS ) -#undef XYZZY +#undef PLATFORM_FLAGS PARAM_SIMPLE( /* name */ shortlived, @@ -838,9 +830,9 @@ PARAM_SIMPLE( ) #if defined(HAVE_TCP_FASTOPEN) - #define XYZZY MUST_RESTART +# define PLATFORM_FLAGS MUST_RESTART #else - #define XYZZY NOT_IMPLEMENTED +# define PLATFORM_FLAGS NOT_IMPLEMENTED #endif PARAM_SIMPLE( /* name */ tcp_fastopen, @@ -851,14 +843,14 @@ PARAM_SIMPLE( /* units */ "bool", /* descr */ "Enable TCP Fast Open extension.", - /* flags */ XYZZY + /* flags */ PLATFORM_FLAGS ) -#undef XYZZY +#undef PLATFORM_FLAGS #if defined(HAVE_TCP_KEEP) - #define XYZZY EXPERIMENTAL +# define PLATFORM_FLAGS EXPERIMENTAL #else - #define XYZZY NOT_IMPLEMENTED +# define PLATFORM_FLAGS NOT_IMPLEMENTED #endif PARAM_SIMPLE( /* name */ tcp_keepalive_intvl, @@ -870,7 +862,7 @@ PARAM_SIMPLE( /* descr */ "The number of seconds between TCP keep-alive probes. " "Ignored for Unix domain sockets.", - /* flags */ XYZZY, + /* flags */ PLATFORM_DEPENDENT | PLATFORM_FLAGS, /* dyn_min_reason */ NULL, /* dyn_max_reason */ NULL, /* dyn_def_reason */ "platform dependent" @@ -887,7 +879,7 @@ PARAM_SIMPLE( "The maximum number of TCP keep-alive probes to send before giving " "up and killing the connection if no response is obtained from the " "other end. Ignored for Unix domain sockets.", - /* flags */ XYZZY, + /* flags */ PLATFORM_DEPENDENT | PLATFORM_FLAGS, /* dyn_min_reason */ NULL, /* dyn_max_reason */ NULL, /* dyn_def_reason */ "platform dependent" @@ -904,21 +896,17 @@ PARAM_SIMPLE( "The number of seconds a connection needs to be idle before TCP " "begins sending out keep-alive probes. " "Ignored for Unix domain sockets.", - /* flags */ XYZZY, + /* flags */ PLATFORM_DEPENDENT | PLATFORM_FLAGS, /* dyn_min_reason */ NULL, /* dyn_max_reason */ NULL, /* dyn_def_reason */ "platform dependent" ) -#undef XYZZY - -#if defined(XYZZY) - #error "Temporary macro XYZZY already defined" -#endif +#undef PLATFORM_FLAGS #if defined(SO_RCVTIMEO_WORKS) - #define XYZZY 0 +# define PLATFORM_FLAGS 0 #else - #define XYZZY NOT_IMPLEMENTED +# define PLATFORM_FLAGS NOT_IMPLEMENTED #endif PARAM_SIMPLE( /* name */ timeout_idle, @@ -934,9 +922,9 @@ PARAM_SIMPLE( "This parameter is particularly relevant for HTTP1 keepalive " " connections which are closed unless the next request is received" " before this timeout is reached.", - /* flags */ XYZZY + /* flags */ PLATFORM_FLAGS ) -#undef XYZZY +#undef PLATFORM_FLAGS PARAM_SIMPLE( /* name */ timeout_linger, From dridi.boukelmoune at gmail.com Wed Oct 21 20:13:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 21 Oct 2020 20:13:06 +0000 (UTC) Subject: [master] f7c41e2f4 param: Missing PLATFORM_DEPENDENT flags Message-ID: <20201021201307.006A3101258@lists.varnish-cache.org> commit f7c41e2f480d48155e3ff52803036a618435642b Author: Dridi Boukelmoune Date: Wed Oct 21 22:11:14 2020 +0200 param: Missing PLATFORM_DEPENDENT flags Refs #3250 diff --git a/include/tbl/params.h b/include/tbl/params.h index 14986dfec..51ad42456 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -843,7 +843,7 @@ PARAM_SIMPLE( /* units */ "bool", /* descr */ "Enable TCP Fast Open extension.", - /* flags */ PLATFORM_FLAGS + /* flags */ PLATFORM_DEPENDENT | PLATFORM_FLAGS ) #undef PLATFORM_FLAGS @@ -922,7 +922,7 @@ PARAM_SIMPLE( "This parameter is particularly relevant for HTTP1 keepalive " " connections which are closed unless the next request is received" " before this timeout is reached.", - /* flags */ PLATFORM_FLAGS + /* flags */ PLATFORM_DEPENDENT | PLATFORM_FLAGS ) #undef PLATFORM_FLAGS From dridi.boukelmoune at gmail.com Thu Oct 22 08:22:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 22 Oct 2020 08:22:06 +0000 (UTC) Subject: [master] c8d8d1879 Missing newline character between the secret and the challenge Message-ID: <20201022082206.E92801132D9@lists.varnish-cache.org> commit c8d8d1879bf4474633cbd02f12c2af21c6b78691 Author: ThijsFeryn Date: Thu Oct 22 10:12:54 2020 +0200 Missing newline character between the secret and the challenge diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst index b8575c740..0c30b63e6 100644 --- a/doc/sphinx/reference/varnish-cli.rst +++ b/doc/sphinx/reference/varnish-cli.rst @@ -382,10 +382,11 @@ The authenticator is calculated by applying the SHA256 function to the following byte sequence: * Challenge string -* Newline (0x0a) character. +* Newline (0x0a) character * Contents of the secret file +* Newline (0x0a) character * Challenge string -* Newline (0x0a) character. +* Newline (0x0a) character and dumping the resulting digest in lower-case hex. From nils.goroll at uplex.de Thu Oct 22 15:40:25 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 22 Oct 2020 17:40:25 +0200 Subject: [master] 89a558e56 man: Reference sources from $(srcdir) In-Reply-To: <20201021064510.BA2C0112BB7@lists.varnish-cache.org> References: <20201021064510.BA2C0112BB7@lists.varnish-cache.org> Message-ID: <425b0851-fcdf-3aa9-80f6-7cfc16ef2149@uplex.de> FTR, this was done on purpose because the previous (incomplete) solution to the VPATH problem relied on all the rst files residing in the single builddir tree. I hope PR "Accomodate VPATH builds (#3443)" really solves the issue, otherwise this change would go in the wrong direction, IMHO. Unfortunately, I have other priorities overriding a timely review. -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From dridi at varni.sh Thu Oct 22 15:47:07 2020 From: dridi at varni.sh (Dridi Boukelmoune) Date: Thu, 22 Oct 2020 15:47:07 +0000 Subject: [master] 89a558e56 man: Reference sources from $(srcdir) In-Reply-To: <425b0851-fcdf-3aa9-80f6-7cfc16ef2149@uplex.de> References: <20201021064510.BA2C0112BB7@lists.varnish-cache.org> <425b0851-fcdf-3aa9-80f6-7cfc16ef2149@uplex.de> Message-ID: On Thu, Oct 22, 2020 at 3:40 PM Nils Goroll wrote: > > FTR, this was done on purpose because the previous (incomplete) solution to the > VPATH problem relied on all the rst files residing in the single builddir tree. I'm starting to have recollections of a triple commit revert. > I hope PR "Accomodate VPATH builds (#3443)" really solves the issue, otherwise > this change would go in the wrong direction, IMHO. I hope so too. > Unfortunately, I have other priorities overriding a timely review. Got it. From dridi.boukelmoune at gmail.com Thu Oct 22 16:13:06 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 22 Oct 2020 16:13:06 +0000 (UTC) Subject: [master] 293624a5a Revert "Missing newline character between the secret and the challenge" Message-ID: <20201022161306.E10DE95FA@lists.varnish-cache.org> commit 293624a5a7a0b15397ff89765fc0554265e9966c Author: Dridi Boukelmoune Date: Thu Oct 22 16:10:21 2020 +0000 Revert "Missing newline character between the secret and the challenge" This reverts commit c8d8d1879bf4474633cbd02f12c2af21c6b78691. diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst index 0c30b63e6..b8575c740 100644 --- a/doc/sphinx/reference/varnish-cli.rst +++ b/doc/sphinx/reference/varnish-cli.rst @@ -382,11 +382,10 @@ The authenticator is calculated by applying the SHA256 function to the following byte sequence: * Challenge string -* Newline (0x0a) character +* Newline (0x0a) character. * Contents of the secret file -* Newline (0x0a) character * Challenge string -* Newline (0x0a) character +* Newline (0x0a) character. and dumping the resulting digest in lower-case hex. From phk at FreeBSD.org Fri Oct 23 09:03:09 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 23 Oct 2020 09:03:09 +0000 (UTC) Subject: [master] cb6d61637 Clarify the description of the authentication Message-ID: <20201023090309.4DEA61121EB@lists.varnish-cache.org> commit cb6d6163758344e74c8515d6e0a9218dabaa4fe1 Author: Poul-Henning Kamp Date: Fri Oct 23 09:01:41 2020 +0000 Clarify the description of the authentication diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst index b8575c740..b9cde35e0 100644 --- a/doc/sphinx/reference/varnish-cli.rst +++ b/doc/sphinx/reference/varnish-cli.rst @@ -346,7 +346,9 @@ file on the fly. Use the unix file permissions to control access to the file. -An authenticated session looks like this:: +An authenticated session looks like this: + +.. code-block:: text critter phk> telnet localhost 1234 Trying ::1... @@ -363,8 +365,8 @@ An authenticated session looks like this:: ----------------------------- Varnish Cache CLI 1.0 ----------------------------- - Linux,4.4.0-1-amd64,x86_64,-jnone,-smalloc,-smalloc,-hcritbit - varnish-trunk revision dc360a4 + FreeBSD,13.0-CURRENT,amd64,-jnone,-sdefault,-sdefault,-hcritbit + varnish-trunk revision 89a558e56390d425c52732a6c94087eec9083115 Type 'help' for command list. Type 'quit' to close CLI session. @@ -389,23 +391,28 @@ following byte sequence: and dumping the resulting digest in lower-case hex. -In the above example, the secret file contained foo\n and thus:: +In the above example, the secret file contains `foo\\n` and thus: + +.. code-block:: text - critter phk> cat > _ + critter phk> hexdump secret + 00000000 66 6f 6f 0a |foo.| + 00000004 + critter phk> cat > tmpfile ixslvvxrgkjptxmcgnnsdxsvdmvfympg foo ixslvvxrgkjptxmcgnnsdxsvdmvfympg ^D - critter phk> hexdump -C _ + critter phk> hexdump -C tmpfile 00000000 69 78 73 6c 76 76 78 72 67 6b 6a 70 74 78 6d 63 |ixslvvxrgkjptxmc| 00000010 67 6e 6e 73 64 78 73 76 64 6d 76 66 79 6d 70 67 |gnnsdxsvdmvfympg| 00000020 0a 66 6f 6f 0a 69 78 73 6c 76 76 78 72 67 6b 6a |.foo.ixslvvxrgkj| 00000030 70 74 78 6d 63 67 6e 6e 73 64 78 73 76 64 6d 76 |ptxmcgnnsdxsvdmv| 00000040 66 79 6d 70 67 0a |fympg.| 00000046 - critter phk> sha256 _ + critter phk> sha256 tmpfile SHA256 (_) = 455ce847f0073c7ab3b1465f74507b75d3dc064c1e7de3b71e00de9092fdc89a - critter phk> openssl dgst -sha256 < _ + critter phk> openssl dgst -sha256 < tmpfile 455ce847f0073c7ab3b1465f74507b75d3dc064c1e7de3b71e00de9092fdc89a The sourcefile lib/libvarnish/cli_auth.c contains a useful function From dridi.boukelmoune at gmail.com Fri Oct 23 09:11:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 23 Oct 2020 09:11:07 +0000 (UTC) Subject: [master] 3e34b796c doc: Typo Message-ID: <20201023091107.334E7112610@lists.varnish-cache.org> commit 3e34b796ce5ceb33c6e61168b5af970f59c50efa Author: Dridi Boukelmoune Date: Fri Oct 23 11:10:07 2020 +0200 doc: Typo diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst index b9cde35e0..f78e19472 100644 --- a/doc/sphinx/reference/varnish-cli.rst +++ b/doc/sphinx/reference/varnish-cli.rst @@ -391,7 +391,7 @@ following byte sequence: and dumping the resulting digest in lower-case hex. -In the above example, the secret file contains `foo\\n` and thus: +In the above example, the secret file contains ``foo\\n`` and thus: .. code-block:: text From phk at FreeBSD.org Fri Oct 23 10:33:06 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 23 Oct 2020 10:33:06 +0000 (UTC) Subject: [master] acdc13eb5 Remove a surplus backslash Message-ID: <20201023103306.E6AAF1145B9@lists.varnish-cache.org> commit acdc13eb54f7882204d4a6c90f76d42ee134da51 Author: Poul-Henning Kamp Date: Fri Oct 23 10:31:22 2020 +0000 Remove a surplus backslash diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst index f78e19472..2b425c220 100644 --- a/doc/sphinx/reference/varnish-cli.rst +++ b/doc/sphinx/reference/varnish-cli.rst @@ -391,7 +391,7 @@ following byte sequence: and dumping the resulting digest in lower-case hex. -In the above example, the secret file contains ``foo\\n`` and thus: +In the above example, the secret file contains ``foo\n`` and thus: .. code-block:: text From guillaume at varnish-software.com Fri Oct 23 21:57:07 2020 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Fri, 23 Oct 2020 21:57:07 +0000 (UTC) Subject: [6.0] d5c2696cc backport cci conf Message-ID: <20201023215707.443141029D4@lists.varnish-cache.org> commit d5c2696cc836be5634d8387163d15619fb376c5e Author: Guillaume Quintard Date: Thu Oct 15 16:27:51 2020 -0700 backport cci conf diff --git a/.circleci/README.rst b/.circleci/README.rst new file mode 100644 index 000000000..0c52c4235 --- /dev/null +++ b/.circleci/README.rst @@ -0,0 +1,93 @@ +Multiarch building, testing & packaging +======================================= + +Varnish Cache uses CircleCI_ for building, testing and creating packages for +several Linux distributions for both x86_64 and aarch64 architectures. + +Since CircleCI provides only x86_64 VMs the setup uses Docker and QEMU to be +able to build, test and create packages for aarch64. This is accomplished by +registering ``qemu-user-static`` for the CircleCI ``machine`` executor:: + + sudo docker run --rm --privileged multiarch/qemu-user-static --reset --credential yes --persistent yes + +Note 1: **--credential yes** is needed so that *setuid* flag is working. +Without it ``sudo`` does not work in the Docker containers with architecture +different than x86_64. + +Note 2: **--persistent yes** is needed so that there is no need to use +``:register`` tag. This way one can run locally pure foreign arch Docker +images, like the official ``arm64v8/***`` ones. + +With QEMU registered each build step can start a Docker image for any of the +supported architectures to execute the ``configure``, ``make``, package steps. + +Workflows +--------- + +There are two CircleCI workflows: + +commit +~~~~~~ + +It is executed after each push to any branch, including Pull Requests + +The ``commit`` workflow runs two jobs: + +- ``dist`` - this job creates the source code distribution of Varnish Cache as + compressed archive (``varnish-${VERSION}.tar.gz``). + +- ``distcheck`` - untars the source code distribution from ``dist`` job and + builds (*configure*, *make*) on different Linux distributions + +nightly +~~~~~~~ + +It is executed once per day at 04:00 AM UTC time. + +This workflow also builds binary packages for different Linux distributions +and CPU architectures (x86_64 & aarch64) and for this reason its run takes +longer. + +It runs the following jobs: + +- The first two jobs that run in parallel are: + + - ``tar_pkg_tools`` - this step checks out pkg-varnish-cache_ with the + packaging descriptions for Debian, RedHat and Alpine, and stores them in + the build workspace for the next steps in the pipeline. + + - ``dist`` - this step creates the source code distribution of Varnish Cache + as compressed archive (``varnish-${VERSION}.tar.gz``). This archive is + also stored in the build workspace and used later by the packaging steps. + + +- The next job in the workflow is ``package`` - a job that creates the + packages (e.g. .rpm, .deb) for each supported CPU architecture, Linux + distribution and its major version (e.g. *x64_centos_7*, + *aarch64_ubuntu_bionic*, *x64_alpine_3*, etc.). This step creates a + Dockerfile on the fly by using a base Docker image. This custom Docker image + executes a Shell script that has the recipe for creating the package for the + specific Linux flavor, e.g. *make-rpm-packages.sh*. The step stores the + packages in the build workspace. + +- Finally, if the previous jobs are successful, a final step is executed - + ``collect_packages``. This step creates an archive with all packages and + stores it as an artifact that can be uploaded to PackageCloud_. + + +More +---- + +This setup can be easily extended for any CPU architectures supported by QEMU +and for any Linux distributions which have Docker image. To do this one needs +to add a new ``package`` job with the proper parameters for it. + +At the moment the setup uses *raw* Docker images and installs the required +Linux distribution dependencies before running the tests/build/packaging code. +This could be optimized to save some execution time by creating custom Docker +images that extend the current ones and pre-installs the required +dependencies. + +.. _CircleCI: https://app.circleci.com/pipelines/github/varnishcache/varnish-cache +.. _pkg-varnish-cache: https://github.com/varnishcache/pkg-varnish-cache +.. _PackageCloud: https://packagecloud.io/varnishcache/ diff --git a/.circleci/config.yml b/.circleci/config.yml index 7e9905b09..2efe13d56 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,13 +1,363 @@ version: 2.1 +aliases: + pkg_req: &pkg_req + requires: + - dist + - tar_pkg_tools + +parameters: + vc-commit: + type: string + default: "HEAD" + pkg-commit: + type: string + default: "" jobs: - dummy: + dist: + description: Builds varnish-x.y.z.tar.gz that is used later for the packaging jobs + docker: + - image: centos:7 + steps: + - run: + name: Install deps + command: | + yum install -y epel-release + yum install -y \ + automake \ + jemalloc-devel \ + git \ + libedit-devel \ + libtool \ + libunwind-devel \ + make \ + pcre-devel \ + python \ + python-sphinx + - checkout + - run: + name: Create the dist tarball + command: | + git checkout << pipeline.parameters.vc-commit >> + # if version is "trunk", it's a weekly tarball, override the version + if grep 'AC_INIT.*trunk.*' ./configure.ac; then + sed -i -e "s/^AC_INIT.*trunk.*/AC_INIT([Varnish], [$(date +%Y%m%d)], [varnish-dev at varnish-cache.org])/" ./configure.ac + touch .is_weekly + fi + ./autogen.des --quiet + make dist -j 16 + - persist_to_workspace: + root: . + paths: + - .is_weekly + - varnish*.tar.gz + - tools/*.suppr + - .circleci + tar_pkg_tools: + description: Builds archives with the packaging tools from https://github.com/varnishcache/pkg-varnish-cache docker: - - image: alpine:3 + - image: centos:7 + steps: + - add_ssh_keys: + fingerprints: + - "11:ed:57:75:32:81:9d:d0:a4:5e:af:15:4b:d8:74:27" + - run: + name: Grab the pkg repo + command: | + yum install -y git + mkdir -p ~/.ssh + ssh-keyscan -H github.com >> ~/.ssh/known_hosts + echo ${CIRCLE_REPOSITORY_URL} + git clone https://github.com/varnishcache/pkg-varnish-cache.git . + git checkout << pipeline.parameters.pkg-commit >> + tar cvzf debian.tar.gz debian --dereference + tar cvzf redhat.tar.gz redhat --dereference + tar cvzf alpine.tar.gz alpine --dereference + - persist_to_workspace: + root: . + paths: + - debian.tar.gz + - redhat.tar.gz + - alpine.tar.gz + package: + parameters: + dist: + description: the Linux distribution (debian|ubuntu|centos) + type: string + release: + description: the release name (stretch|buster|xenial|bionic|7|8) + type: string + ext: + description: the package extension (deb|rpm|apk) + type: string + arch: + description: the architecture (x64|aarch64) + type: string + image: + description: the base Docker image for Dockerfile + type: string + machine: + image: ubuntu-1604:201903-01 steps: - - run: echo ok + - attach_workspace: + at: ~/project + - run: ls -la ~/project + - run: + name: Activate QEMU + command: | + sudo docker run -it --rm --privileged multiarch/qemu-user-static --reset --credential yes --persistent yes + - run: + name: Create Dockerfile + command: | + echo "FROM << parameters.image >>" > Dockerfile + echo "ADD make-<< parameters.ext >>-packages.sh /usr/bin/" >> Dockerfile + echo 'CMD ["make-<< parameters.ext >>-packages.sh"]' >> Dockerfile + - run: + name: Build << parameters.dist >> << parameters.release >> << parameters.arch >> << parameters.ext >> + command: | + mkdir -p packages + cp .circleci/make-<< parameters.ext >>-packages.sh . + docker build -t varnish-<< parameters.ext >>-package-build:<< parameters.arch >> . + docker run --rm -it -e PARAM_DIST=<< parameters.dist >> -e PARAM_RELEASE=<< parameters.release >> -v$(pwd):/varnish-cache varnish-<< parameters.ext >>-package-build:<< parameters.arch >> + - run: + name: List created packages + command: find ./packages -name "*.<< parameters.ext >>" + - persist_to_workspace: + root: . + paths: + - "packages" + distcheck: + parameters: + dist: + description: the Linux distribution (debian|ubuntu) + type: string + release: + description: the release name (stretch|buster|xenial|bionic) + type: string + extra_conf: + description: platform-specific configure arguments + default: "" + type: string + docker: + - image: << parameters.dist >>:<< parameters.release >> + working_directory: /workspace + steps: + - run: + name: Possible << parameters.dist >>:<< parameters.release >> extra repos + command: | + if [ << parameters.dist >> = centos ]; then + if [ << parameters.release >> = 8 ]; then + dnf install -y 'dnf-command(config-manager)' + yum config-manager --set-enabled PowerTools + yum install -y diffutils python3-sphinx + else + yum install -y python-sphinx + fi + yum install -y epel-release + yum install -y \ + automake \ + jemalloc-devel \ + libedit-devel \ + libtool \ + libunwind-devel \ + make \ + pcre-devel \ + python3 \ + sudo + elif [ << parameters.dist >> = debian -o << parameters.dist >> = ubuntu ]; then + export DEBIAN_FRONTEND=noninteractive + export DEBCONF_NONINTERACTIVE_SEEN=true + apt-get update + apt-get install -y \ + autoconf \ + automake \ + build-essential \ + ca-certificates \ + cpio \ + graphviz \ + libedit-dev \ + libjemalloc-dev \ + libncurses-dev \ + libpcre3-dev \ + libtool \ + libunwind-dev \ + pkg-config \ + python3-sphinx \ + sudo + elif [ << parameters.dist >> = alpine ]; then + apk update + apk add -q \ + autoconf \ + automake \ + build-base \ + ca-certificates \ + cpio \ + gzip \ + libedit-dev \ + libtool \ + libunwind-dev \ + linux-headers \ + pcre-dev \ + py-docutils \ + py3-sphinx \ + tar \ + sudo + fi + - attach_workspace: + at: /workspace + - run: + name: Extract and distcheck + command: | + tar xavf varnish-*.tar.gz --strip 1 + if [ << parameters.dist >> = centos ]; then + adduser varnish + else + adduser --disabled-password --gecos "" varnish + fi + + chown -R varnish:varnish . + + export ASAN_OPTIONS=abort_on_error=1,detect_odr_violation=1,detect_leaks=1,detect_stack_use_after_return=1,detect_invalid_pointer_pairs=1,handle_segv=0,handle_sigbus=0,use_sigaltstack=0,disable_coredump=0 + export LSAN_OPTIONS=abort_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/lsan.suppr + export TSAN_OPTIONS=abort_on_error=1,halt_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/tsan.suppr + export UBSAN_OPTIONS=halt_on_error=1,print_stacktrace=1,use_sigaltstack=0,suppressions=$(pwd)/tools/ubsan.suppr + + sudo -u varnish ./configure \ + --quiet \ + --with-unwind \ + --enable-developer-warnings \ + --enable-debugging-symbols \ + --disable-stack-protector \ + --with-persistent-storage \ + << parameters.extra_conf >> + sudo -u varnish \ + --preserve-env=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \ + make distcheck VERBOSE=1 -j 12 -k + + collect_packages: + docker: + - image: centos:7 + steps: + - attach_workspace: + at: ~/project + - run: ls -la ~/project/ + - run: + name: Tar the packages + command: | + tar cvzf packages.tar.gz packages + - store_artifacts: + destination: packages.tar.gz + path: packages.tar.gz + workflows: - build: + version: 2 + commit: jobs: - - dummy + - distcheck: + name: distcheck_centos_7 + dist: centos + release: "7" + requires: + - dist + - distcheck: + name: distcheck_debian_buster + dist: debian + release: buster + extra_conf: --enable-asan --enable-ubsan + requires: + - dist + - dist + - tar_pkg_tools + - package: + name: aarch64-ubuntu-bionic + dist: ubuntu + release: bionic + arch: aarch64 + image: arm64v8/ubuntu:bionic + ext: deb + <<: *pkg_req + - package: + name: x64-ubuntu-bionic + dist: ubuntu + release: bionic + arch: x64 + image: ubuntu:bionic + ext: deb + <<: *pkg_req + - package: + name: aarch64-ubuntu-xenial + dist: ubuntu + release: xenial + arch: aarch64 + image: arm64v8/ubuntu:xenial + ext: deb + <<: *pkg_req + - package: + name: x64-ubuntu-xenial + dist: ubuntu + release: xenial + arch: x64 + image: ubuntu:xenial + ext: deb + <<: *pkg_req + - package: + name: aarch64-debian-buster + dist: debian + release: buster + arch: aarch64 + image: arm64v8/debian:buster-slim + ext: deb + <<: *pkg_req + - package: + name: x64-debian-buster + dist: debian + release: buster + arch: x64 + image: debian:buster-slim + ext: deb + <<: *pkg_req + - package: + name: aarch64-debian-stretch + dist: debian + release: stretch + arch: aarch64 + image: arm64v8/debian:stretch-slim + ext: deb + <<: *pkg_req + - package: + name: x64-debian-stretch + dist: debian + release: stretch + arch: x64 + image: debian:stretch-slim + ext: deb + <<: *pkg_req + - package: + name: aarch64-centos-7 + dist: centos + release: "7" + arch: aarch64 + image: arm64v8/centos:7 + ext: rpm + <<: *pkg_req + - package: + name: x64-centos-7 + dist: centos + release: "7" + arch: x64 + image: centos:7 + ext: rpm + <<: *pkg_req + - collect_packages: + requires: + - x64-ubuntu-xenial + - aarch64-ubuntu-xenial + - x64-ubuntu-bionic + - aarch64-ubuntu-bionic + - x64-debian-stretch + - aarch64-debian-stretch + - x64-debian-buster + - aarch64-debian-buster + - x64-centos-7 + - aarch64-centos-7 diff --git a/.circleci/make-apk-packages.sh b/.circleci/make-apk-packages.sh new file mode 100755 index 000000000..26b672c2c --- /dev/null +++ b/.circleci/make-apk-packages.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env sh + +set -eux + +apk add -q --no-progress --update tar alpine-sdk + +echo "PARAM_RELEASE: $PARAM_RELEASE" +echo "PARAM_DIST: $PARAM_DIST" + +if [ -z "$PARAM_RELEASE" ]; then + echo "Env variable PARAM_RELEASE is not set! For example PARAM_RELEASE=8, for CentOS 8" + exit 1 +elif [ -z "$PARAM_DIST" ]; then + echo "Env variable PARAM_DIST is not set! For example PARAM_DIST=centos" + exit 1 +fi + +cd /varnish-cache +tar xazf alpine.tar.gz --strip 1 + +adduser -D builder +echo "builder ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers +addgroup builder abuild +mkdir -p /var/cache/distfiles +chmod -R a+w /var/cache/distfiles + +echo "Generate key" +su builder -c "abuild-keygen -nai" + +echo "Fix APKBUILD's variables" +tar xavf varnish-*.tar.gz +VERSION=$(varnish-*/configure --version | awk 'NR == 1 {print $NF}') +echo "Version: $VERSION" +sed -i "s/@VERSION@/$VERSION/" APKBUILD +rm -rf varnish-*/ + +echo "Change the ownership so that abuild is able to write its logs" +chown builder -R . +echo "Fix checksums, build" +su builder -c "abuild checksum" +su builder -c "abuild -r" + +echo "Fix the APKBUILD's version" +su builder -c "mkdir apks" +ARCH=`uname -m` +su builder -c "cp /home/builder/packages/$ARCH/*.apk apks" + +echo "Import the packages into the workspace" +mkdir -p packages/$PARAM_DIST/$PARAM_RELEASE/$ARCH/ +mv /home/builder/packages/$ARCH/*.apk packages/$PARAM_DIST/$PARAM_RELEASE/$ARCH/ + +echo "Allow to read the packages by 'circleci' user outside of Docker after 'chown builder -R .' above" +chmod -R a+rwx . diff --git a/.circleci/make-deb-packages.sh b/.circleci/make-deb-packages.sh new file mode 100755 index 000000000..3b3ba0847 --- /dev/null +++ b/.circleci/make-deb-packages.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +set -eux + +export DEBIAN_FRONTEND=noninteractive +export DEBCONF_NONINTERACTIVE_SEEN=true +apt-get update +apt-get install -y dpkg-dev debhelper devscripts equivs pkg-config apt-utils fakeroot + +echo "PARAM_RELEASE: $PARAM_RELEASE" +echo "PARAM_DIST: $PARAM_DIST" + + +if [ -z "$PARAM_RELEASE" ]; then + echo "Env variable PARAM_RELEASE is not set! For example PARAM_RELEASE=8, for CentOS 8" + exit 1 +elif [ -z "$PARAM_DIST" ]; then + echo "Env variable PARAM_DIST is not set! For example PARAM_DIST=centos" + exit 1 +fi + +# Ubuntu 20.04 aarch64 fails when using fakeroot-sysv with: +# semop(1): encountered an error: Function not implemented +update-alternatives --set fakeroot /usr/bin/fakeroot-tcp + +cd /varnish-cache +ls -la + +echo "Untar debian..." +tar xavf debian.tar.gz + +echo "Untar orig..." +tar xavf varnish-*.tar.gz --strip 1 + +echo "Update changelog version..." +if [ -e .is_weekly ]; then + WEEKLY='-weekly' +else + WEEKLY= +fi +VERSION=$(./configure --version | awk 'NR == 1 {print $NF}')$WEEKLY~$PARAM_RELEASE +sed -i -e "s|@VERSION@|$VERSION-1|" "debian/changelog" + +echo "Install Build-Depends packages..." +yes | mk-build-deps --install debian/control || true + +echo "Build the packages..." +dpkg-buildpackage -us -uc -j16 + +echo "Prepare the packages for storage..." +mkdir -p packages/$PARAM_DIST/$PARAM_RELEASE/ +mv ../*.deb packages/$PARAM_DIST/$PARAM_RELEASE/ + +if [ "`uname -m`" = "x86_64" ]; then + ARCH="amd64" +else + ARCH="arm64" +fi + +DSC_FILE=$(ls ../*.dsc) +DSC_FILE_WO_EXT=$(basename ${DSC_FILE%.*}) +mv $DSC_FILE packages/$PARAM_DIST/$PARAM_RELEASE/${DSC_FILE_WO_EXT}_${ARCH}.dsc diff --git a/.circleci/make-rpm-packages.sh b/.circleci/make-rpm-packages.sh new file mode 100755 index 000000000..b443c9139 --- /dev/null +++ b/.circleci/make-rpm-packages.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +set -eux + +echo "PARAM_RELEASE: $PARAM_RELEASE" +echo "PARAM_DIST: $PARAM_DIST" + +if [ -z "$PARAM_RELEASE" ]; then + echo "Env variable PARAM_RELEASE is not set! For example PARAM_RELEASE=8, for CentOS 8" + exit 1 +elif [ -z "$PARAM_DIST" ]; then + echo "Env variable PARAM_DIST is not set! For example PARAM_DIST=centos" + exit 1 +fi + +yum install -y epel-release + +if [ "$PARAM_DIST" = centos ]; then + if [ "$PARAM_RELEASE" = 8 ]; then + dnf install -y 'dnf-command(config-manager)' + yum config-manager --set-enabled PowerTools + fi +fi + +yum install -y rpm-build yum-utils + +export DIST_DIR=build + +cd /varnish-cache +rm -rf $DIST_DIR +mkdir $DIST_DIR + + +echo "Untar redhat..." +tar xavf redhat.tar.gz -C $DIST_DIR + +echo "Untar orig..." +tar xavf varnish-*.tar.gz -C $DIST_DIR --strip 1 + +echo "Build Packages..." +if [ -e .is_weekly ]; then + WEEKLY='.weekly' +else + WEEKLY= +fi +VERSION=$("$DIST_DIR"/configure --version | awk 'NR == 1 {print $NF}')$WEEKLY + +cp -r -L "$DIST_DIR"/redhat/* "$DIST_DIR"/ +tar zcf "$DIST_DIR.tgz" --exclude "$DIST_DIR/redhat" "$DIST_DIR"/ + +RPMVERSION="$VERSION" + +RESULT_DIR="rpms" +CUR_DIR="$(pwd)" + +rpmbuild() { + command rpmbuild \ + --define "_smp_mflags -j10" \ + --define "_sourcedir $CUR_DIR" \ + --define "_srcrpmdir $CUR_DIR/${RESULT_DIR}" \ + --define "_rpmdir $CUR_DIR/${RESULT_DIR}" \ + --define "versiontag ${RPMVERSION}" \ + --define "releasetag 0.0" \ + --define "srcname $DIST_DIR" \ + --define "nocheck 1" \ + "$@" +} + +yum-builddep -y "$DIST_DIR"/redhat/varnish.spec +rpmbuild -bs "$DIST_DIR"/redhat/varnish.spec +rpmbuild --rebuild "$RESULT_DIR"/varnish-*.src.rpm + +echo "Prepare the packages for storage..." +mkdir -p packages/$PARAM_DIST/$PARAM_RELEASE/ +mv rpms/*/*.rpm packages/$PARAM_DIST/$PARAM_RELEASE/ From guillaume at varnish-software.com Sat Oct 24 00:34:07 2020 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Sat, 24 Oct 2020 00:34:07 +0000 (UTC) Subject: [6.0] 3aa711b6e [cci] fix package revision Message-ID: <20201024003407.652011062FD@lists.varnish-cache.org> commit 3aa711b6e37da39159f2aa4fc6b8fd5c95d19d22 Author: Guillaume Quintard Date: Fri Oct 23 15:02:29 2020 -0700 [cci] fix package revision diff --git a/.circleci/make-deb-packages.sh b/.circleci/make-deb-packages.sh index 3b3ba0847..8437de094 100755 --- a/.circleci/make-deb-packages.sh +++ b/.circleci/make-deb-packages.sh @@ -38,8 +38,8 @@ if [ -e .is_weekly ]; then else WEEKLY= fi -VERSION=$(./configure --version | awk 'NR == 1 {print $NF}')$WEEKLY~$PARAM_RELEASE -sed -i -e "s|@VERSION@|$VERSION-1|" "debian/changelog" +VERSION=$(./configure --version | awk 'NR == 1 {print $NF}')$WEEKLY-1~$PARAM_RELEASE +sed -i -e "s|@VERSION@|$VERSION|" "debian/changelog" echo "Install Build-Depends packages..." yes | mk-build-deps --install debian/control || true diff --git a/.circleci/make-rpm-packages.sh b/.circleci/make-rpm-packages.sh index b443c9139..5f7d0af1a 100755 --- a/.circleci/make-rpm-packages.sh +++ b/.circleci/make-rpm-packages.sh @@ -60,7 +60,7 @@ rpmbuild() { --define "_srcrpmdir $CUR_DIR/${RESULT_DIR}" \ --define "_rpmdir $CUR_DIR/${RESULT_DIR}" \ --define "versiontag ${RPMVERSION}" \ - --define "releasetag 0.0" \ + --define "releasetag 1" \ --define "srcname $DIST_DIR" \ --define "nocheck 1" \ "$@" From phk at FreeBSD.org Mon Oct 26 08:32:07 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 26 Oct 2020 08:32:07 +0000 (UTC) Subject: [master] 8907c7d8b Make the internal states of the backend-fetch code private matter. Message-ID: <20201026083207.7DFFC10DCB2@lists.varnish-cache.org> commit 8907c7d8bc286b78da16211e09d425f97cb120fb Author: Poul-Henning Kamp Date: Mon Oct 26 08:30:30 2020 +0000 Make the internal states of the backend-fetch code private matter. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 6a01cd5b5..f763a9fa6 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -46,7 +46,7 @@ #include "vdef.h" #include "vrt.h" -#define CACHE_H_INCLUDED +#define CACHE_H_INCLUDED // After vrt.h include. #include "miniobj.h" #include "vas.h" @@ -121,12 +121,6 @@ enum req_step { #include "tbl/steps.h" }; -enum fetch_step { - F_STP_NONE = 0, -#define FETCH_STEP(l, U, arg) F_STP_##U, -#include "tbl/steps.h" -}; - /*--------------------------------------------------------------------*/ struct lock { void *priv; }; // Opaque diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index cd975927c..218df239a 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -39,6 +39,31 @@ #include "vcl.h" #include "vtim.h" +#define FETCH_STEPS \ + FETCH_STEP(mkbereq, MKBEREQ) \ + FETCH_STEP(retry, RETRY) \ + FETCH_STEP(startfetch, STARTFETCH) \ + FETCH_STEP(condfetch, CONDFETCH) \ + FETCH_STEP(fetch, FETCH) \ + FETCH_STEP(fetchbody, FETCHBODY) \ + FETCH_STEP(fetchend, FETCHEND) \ + FETCH_STEP(error, ERROR) \ + FETCH_STEP(fail, FAIL) \ + FETCH_STEP(done, DONE) + +typedef const struct fetch_step *vbf_state_f(struct worker *, struct busyobj *); + +struct fetch_step { + const char *name; + vbf_state_f *func; +}; + +#define FETCH_STEP(l, U) \ + static vbf_state_f vbf_stp_##l; \ + static const struct fetch_step F_STP_##U[1] = {{ .name = "Fetch Step" #l, .func = vbf_stp_##l, }}; +FETCH_STEPS +#undef FETCH_STEP + /*-------------------------------------------------------------------- * Allocate an object, with fall-back to Transient. * XXX: This somewhat overlaps the stuff in stevedore.c @@ -213,7 +238,7 @@ vbf_beresp2obj(struct busyobj *bo) * Copy req->bereq and release req if no body */ -static enum fetch_step +static const struct fetch_step * v_matchproto_(vbf_state_f) vbf_stp_mkbereq(struct worker *wrk, struct busyobj *bo) { const char *q; @@ -278,7 +303,7 @@ vbf_stp_mkbereq(struct worker *wrk, struct busyobj *bo) * Prepare the busyobj and fetch processors */ -static enum fetch_step +static const struct fetch_step * v_matchproto_(vbf_state_f) vbf_stp_retry(struct worker *wrk, struct busyobj *bo) { CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -352,7 +377,7 @@ vbf_304_logic(struct busyobj *bo) * Setup bereq from bereq0, run vcl_backend_fetch */ -static enum fetch_step +static const struct fetch_step * v_matchproto_(vbf_state_f) vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) { int i; @@ -516,8 +541,8 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) /*-------------------------------------------------------------------- */ -static enum fetch_step -vbf_stp_fetchbody(const struct worker *wrk, struct busyobj *bo) +static const struct fetch_step * v_matchproto_(vbf_state_f) +vbf_stp_fetchbody(struct worker *wrk, struct busyobj *bo) { ssize_t l; uint8_t *ptr; @@ -587,7 +612,7 @@ vbf_stp_fetchbody(const struct worker *wrk, struct busyobj *bo) return (F_STP_FETCHEND); } -static enum fetch_step +static const struct fetch_step * v_matchproto_(vbf_state_f) vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) { struct objcore *oc; @@ -673,7 +698,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) return (F_STP_FETCHEND); } -static enum fetch_step +static const struct fetch_step * v_matchproto_(vbf_state_f) vbf_stp_fetchend(struct worker *wrk, struct busyobj *bo) { @@ -735,7 +760,7 @@ vbf_objiterator(void *priv, unsigned flush, const void *ptr, ssize_t len) return (0); } -static enum fetch_step +static const struct fetch_step * v_matchproto_(vbf_state_f) vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) { struct boc *stale_boc; @@ -824,7 +849,7 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) * default true) */ -static enum fetch_step +static const struct fetch_step * v_matchproto_(vbf_state_f) vbf_stp_error(struct worker *wrk, struct busyobj *bo) { ssize_t l, ll, o; @@ -945,8 +970,8 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) /*-------------------------------------------------------------------- */ -static enum fetch_step -vbf_stp_fail(struct worker *wrk, const struct busyobj *bo) +static const struct fetch_step * v_matchproto_(vbf_state_f) +vbf_stp_fail(struct worker *wrk, struct busyobj *bo) { struct objcore *oc; @@ -966,9 +991,12 @@ vbf_stp_fail(struct worker *wrk, const struct busyobj *bo) /*-------------------------------------------------------------------- */ -static enum fetch_step -vbf_stp_done(void) +static const struct fetch_step * v_matchproto_(vbf_state_f) +vbf_stp_done(struct worker *wrk, struct busyobj *bo) { + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); WRONG("Just plain wrong"); NEEDLESS(return (F_STP_DONE)); } @@ -978,7 +1006,7 @@ vbf_fetch_thread(struct worker *wrk, void *priv) { struct busyobj *bo; struct objcore *oc; - enum fetch_step stp; + const struct fetch_step *stp; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC); @@ -1012,15 +1040,10 @@ vbf_fetch_thread(struct worker *wrk, void *priv) AN(bo->req); else AZ(bo->req); - switch (stp) { -#define FETCH_STEP(l, U, arg) \ - case F_STP_##U: \ - stp = vbf_stp_##l arg; \ - break; -#include "tbl/steps.h" - default: - WRONG("Illegal fetch_step"); - } + AN(stp); + AN(stp->name); + AN(stp->func); + stp = stp->func(wrk, bo); } assert(bo->director_state == DIR_S_NULL); diff --git a/include/tbl/steps.h b/include/tbl/steps.h index 7fa119557..b16a65bfc 100644 --- a/include/tbl/steps.h +++ b/include/tbl/steps.h @@ -49,18 +49,4 @@ #undef REQ_STEP #endif -#ifdef FETCH_STEP - FETCH_STEP(mkbereq, MKBEREQ, (wrk, bo)) - FETCH_STEP(retry, RETRY, (wrk, bo)) - FETCH_STEP(startfetch, STARTFETCH, (wrk, bo)) - FETCH_STEP(condfetch, CONDFETCH, (wrk, bo)) - FETCH_STEP(fetch, FETCH, (wrk, bo)) - FETCH_STEP(fetchbody, FETCHBODY, (wrk, bo)) - FETCH_STEP(fetchend, FETCHEND, (wrk, bo)) - FETCH_STEP(error, ERROR, (wrk, bo)) - FETCH_STEP(fail, FAIL, (wrk, bo)) - FETCH_STEP(done, DONE, ()) - #undef FETCH_STEP -#endif - /*lint -restore */ From phk at FreeBSD.org Mon Oct 26 09:14:06 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 26 Oct 2020 09:14:06 +0000 (UTC) Subject: [master] efca75a54 Initialize req->req_step in Req_New() Message-ID: <20201026091406.9A9CF10EE3D@lists.varnish-cache.org> commit efca75a54b22099653a9bf6498589eb047a1836d Author: Poul-Henning Kamp Date: Mon Oct 26 08:49:03 2020 +0000 Initialize req->req_step in Req_New() diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 405aec8e2..66dbb8c22 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -181,7 +181,7 @@ ved_include(struct req *preq, const char *src, const char *host, req->vcl = preq->vcl; VCL_Ref(req->vcl); - req->req_step = R_STP_TRANSPORT; + assert(req->req_step == R_STP_TRANSPORT); req->t_req = preq->t_req; req->transport = &VED_transport; diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index 43cb3837b..0592318b4 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -152,6 +152,8 @@ Req_New(const struct worker *wrk, struct sess *sp) req->t_prev = NAN; req->t_req = NAN; + req->req_step = R_STP_TRANSPORT; + return (req); } diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index c079656b1..80993232c 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -597,7 +597,7 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, } AN(req->http->hd[HTTP_HDR_PROTO].b); - req->req_step = R_STP_TRANSPORT; + assert(req->req_step == R_STP_TRANSPORT); req->task->func = h2_do_req; req->task->priv = req; r2->scheduled = 1; diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c index 4b216f85c..0637984b8 100644 --- a/bin/varnishd/http2/cache_http2_session.c +++ b/bin/varnishd/http2/cache_http2_session.c @@ -277,7 +277,7 @@ h2_ou_session(struct worker *wrk, struct h2_sess *h2, /* Start req thread */ r2 = h2_new_req(wrk, h2, 1, req); req->transport = &H2_transport; - req->req_step = R_STP_TRANSPORT; + assert(req->req_step == R_STP_TRANSPORT); req->task->func = h2_do_req; req->task->priv = req; r2->scheduled = 1; From phk at FreeBSD.org Mon Oct 26 09:14:06 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 26 Oct 2020 09:14:06 +0000 (UTC) Subject: [master] ae6dfe2e2 Further reduce transport code frobbing of req->req_step Message-ID: <20201026091406.D23A510EE40@lists.varnish-cache.org> commit ae6dfe2e2fc2973163fc7927f98b042918bf04d1 Author: Poul-Henning Kamp Date: Mon Oct 26 09:12:28 2020 +0000 Further reduce transport code frobbing of req->req_step diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 66dbb8c22..5e702a3be 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -187,10 +187,10 @@ ved_include(struct req *preq, const char *src, const char *host, req->transport = &VED_transport; req->transport_priv = ecx; - CNT_Embark(wrk, req); VCL_TaskEnter(req->privs); while (1) { + CNT_Embark(wrk, req); ecx->woken = 0; s = CNT_Request(req); if (s == REQ_FSM_DONE) @@ -204,7 +204,6 @@ ved_include(struct req *preq, const char *src, const char *host, &ecx->preq->wrk->cond, &sp->mtx, 0); Lck_Unlock(&sp->mtx); AZ(req->wrk); - CNT_Embark(wrk, req); } VCL_Rel(&req->vcl); diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index 0592318b4..0a3bc65a5 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -250,6 +250,7 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req) req->hash_ignore_busy = 0; req->esi_level = 0; req->is_hit = 0; + req->req_step = R_STP_TRANSPORT; if (WS_Overflowed(req->ws)) wrk->stats->ws_client_overflow++; diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c index 9625bed21..de3662f1e 100644 --- a/bin/varnishd/http1/cache_http1_fsm.c +++ b/bin/varnishd/http1/cache_http1_fsm.c @@ -389,17 +389,15 @@ HTTP1_Session(struct worker *wrk, struct req *req) return; } } - req->req_step = R_STP_TRANSPORT; + assert(req->req_step == R_STP_TRANSPORT); + VCL_TaskEnter(req->privs); + VCL_TaskEnter(req->top->privs); http1_setstate(sp, H1PROC); } else if (st == H1PROC) { req->task->func = http1_req; req->task->priv = req; wrk->stats->client_req++; CNT_Embark(wrk, req); - if (req->req_step == R_STP_TRANSPORT) { - VCL_TaskEnter(req->privs); - VCL_TaskEnter(req->top->privs); - } if (CNT_Request(req) == REQ_FSM_DISEMBARK) return; AZ(req->top->vcl0); diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 80993232c..614d13781 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -525,10 +525,6 @@ h2_do_req(struct worker *wrk, void *priv) CAST_OBJ_NOTNULL(r2, req->transport_priv, H2_REQ_MAGIC); THR_SetRequest(req); CNT_Embark(wrk, req); - if (req->req_step == R_STP_TRANSPORT) { - VCL_TaskEnter(req->privs); - VCL_TaskEnter(req->top->privs); - } wrk->stats->client_req++; if (CNT_Request(req) != REQ_FSM_DISEMBARK) { @@ -598,6 +594,8 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, AN(req->http->hd[HTTP_HDR_PROTO].b); assert(req->req_step == R_STP_TRANSPORT); + VCL_TaskEnter(req->privs); + VCL_TaskEnter(req->top->privs); req->task->func = h2_do_req; req->task->priv = req; r2->scheduled = 1; From phk at FreeBSD.org Mon Oct 26 09:55:07 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 26 Oct 2020 09:55:07 +0000 (UTC) Subject: [master] 0ac6be1bf Make the internal steps of the request-handling a (much more) private matter Message-ID: <20201026095507.538F6110029@lists.varnish-cache.org> commit 0ac6be1bf2b4d5ace463ca4287158fb220c5cc5a Author: Poul-Henning Kamp Date: Mon Oct 26 09:53:43 2020 +0000 Make the internal steps of the request-handling a (much more) private matter diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index f763a9fa6..dbeabe7b1 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -103,6 +103,7 @@ struct transport; struct worker; struct listen_sock; struct vcf; +struct req_step; #define DIGEST_LEN 32 @@ -115,14 +116,6 @@ typedef struct { /*--------------------------------------------------------------------*/ -enum req_step { - R_STP_NONE = 0, -#define REQ_STEP(l, u, arg) R_STP_##u, -#include "tbl/steps.h" -}; - -/*--------------------------------------------------------------------*/ - struct lock { void *priv; }; // Opaque /*-------------------------------------------------------------------- @@ -451,13 +444,19 @@ struct reqtop { struct req { unsigned magic; -#define REQ_MAGIC 0x2751aaa1 +#define REQ_MAGIC 0xfb4abf6d - enum req_step req_step; body_status_t req_body_status; enum sess_close doclose; unsigned restarts; unsigned esi_level; + + /* Delivery mode */ + unsigned res_mode; +#define RES_ESI (1<<4) +#define RES_PIPE (1<<7) + + const struct req_step *req_step; struct reqtop *top; /* esi_level == 0 request */ #define REQ_FLAG(l, r, w, d) unsigned l:1; @@ -522,11 +521,6 @@ struct req { struct vdp_ctx *vdc; const char *filter_list; - /* Delivery mode */ - unsigned res_mode; -#define RES_ESI (1<<4) -#define RES_PIPE (1<<7) - /* Transaction VSL buffer */ struct vsl_log vsl[1]; diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 253fa674a..a0292e023 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -454,7 +454,6 @@ pan_top(struct vsb *vsb, const struct reqtop *top) static void pan_req(struct vsb *vsb, const struct req *req) { - const char *stp; const struct transport *xp; VSB_printf(vsb, "req = %p {\n", req); @@ -474,15 +473,10 @@ pan_req(struct vsb *vsb, const struct req *req) VSB_cat(vsb, "}"); } VSB_cat(vsb, "\n"); - switch (req->req_step) { -#define REQ_STEP(l, u, arg) case R_STP_##u: stp = "R_STP_" #u; break; -#include "tbl/steps.h" - default: stp = NULL; - } - if (stp != NULL) - VSB_printf(vsb, "step = %s,\n", stp); + if (req->req_step == NULL) + VSB_cat(vsb, "step = R_STP_TRANSPORT\n"); else - VSB_printf(vsb, "step = 0x%x,\n", req->req_step); + VSB_printf(vsb, "step = %s\n", req->req_step->name); VSB_printf(vsb, "req_body = %s,\n", req->req_body_status ? req->req_body_status->name : "NULL"); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 47ed06b31..b84591bb8 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -53,11 +53,35 @@ #include "vsha256.h" #include "vtim.h" +#define REQ_STEPS \ + REQ_STEP(transport, TRANSPORT, ) \ + REQ_STEP(restart, RESTART, static) \ + REQ_STEP(recv, RECV, ) \ + REQ_STEP(pipe, PIPE, static) \ + REQ_STEP(pass, PASS, static) \ + REQ_STEP(lookup, LOOKUP, static) \ + REQ_STEP(purge, PURGE, static) \ + REQ_STEP(miss, MISS, static) \ + REQ_STEP(fetch, FETCH, static) \ + REQ_STEP(deliver, DELIVER, static) \ + REQ_STEP(vclfail, VCLFAIL, static) \ + REQ_STEP(synth, SYNTH, static) \ + REQ_STEP(transmit, TRANSMIT, static) + +#define REQ_STEP(l, U, priv) \ + static req_state_f cnt_##l; \ + priv const struct req_step R_STP_##U[1] = {{ \ + .name = "Fetch Step" #l, \ + .func = cnt_##l, \ + }}; +REQ_STEPS +#undef REQ_STEP + /*-------------------------------------------------------------------- * Handle "Expect:" and "Connection:" on incoming request */ -static enum req_fsm_nxt +static enum req_fsm_nxt v_matchproto_(req_state_f) cnt_transport(struct worker *wrk, struct req *req) { const char *p; @@ -177,7 +201,7 @@ Resp_Setup_Synth(struct req *req) http_SetHeader(h, "Connection: close"); } -static enum req_fsm_nxt +static enum req_fsm_nxt v_matchproto_(req_state_f) cnt_deliver(struct worker *wrk, struct req *req) { @@ -239,8 +263,8 @@ cnt_deliver(struct worker *wrk, struct req *req) * VCL failed, die horribly */ -static enum req_fsm_nxt -cnt_vclfail(const struct worker *wrk, struct req *req) +static enum req_fsm_nxt v_matchproto_(req_state_f) +cnt_vclfail(struct worker *wrk, struct req *req) { CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -263,7 +287,7 @@ cnt_vclfail(const struct worker *wrk, struct req *req) * Emit a synthetic response */ -static enum req_fsm_nxt +static enum req_fsm_nxt v_matchproto_(req_state_f) cnt_synth(struct worker *wrk, struct req *req) { struct vsb *synth_body; @@ -365,7 +389,7 @@ cnt_synth(struct worker *wrk, struct req *req) * The mechanics of sending a response (from deliver or synth) */ -static enum req_fsm_nxt +static enum req_fsm_nxt v_matchproto_(req_state_f) cnt_transmit(struct worker *wrk, struct req *req) { struct boc *boc; @@ -458,7 +482,7 @@ cnt_transmit(struct worker *wrk, struct req *req) * Initiated a fetch (pass/miss) which we intend to deliver */ -static enum req_fsm_nxt +static enum req_fsm_nxt v_matchproto_(req_state_f) cnt_fetch(struct worker *wrk, struct req *req) { @@ -487,7 +511,7 @@ cnt_fetch(struct worker *wrk, struct req *req) * this state if we get suspended on a busy objhdr. */ -static enum req_fsm_nxt +static enum req_fsm_nxt v_matchproto_(req_state_f) cnt_lookup(struct worker *wrk, struct req *req) { struct objcore *oc, *busy; @@ -614,7 +638,7 @@ cnt_lookup(struct worker *wrk, struct req *req) * Cache miss. */ -static enum req_fsm_nxt +static enum req_fsm_nxt v_matchproto_(req_state_f) cnt_miss(struct worker *wrk, struct req *req) { @@ -659,7 +683,7 @@ cnt_miss(struct worker *wrk, struct req *req) * Pass processing */ -static enum req_fsm_nxt +static enum req_fsm_nxt v_matchproto_(req_state_f) cnt_pass(struct worker *wrk, struct req *req) { @@ -697,7 +721,7 @@ cnt_pass(struct worker *wrk, struct req *req) * Pipe mode */ -static enum req_fsm_nxt +static enum req_fsm_nxt v_matchproto_(req_state_f) cnt_pipe(struct worker *wrk, struct req *req) { struct busyobj *bo; @@ -769,7 +793,7 @@ cnt_pipe(struct worker *wrk, struct req *req) * Handle restart events */ -static enum req_fsm_nxt +static enum req_fsm_nxt v_matchproto_(req_state_f) cnt_restart(struct worker *wrk, struct req *req) { @@ -803,7 +827,7 @@ cnt_restart(struct worker *wrk, struct req *req) * - remove duplicatation with Req_Cleanup() */ -static void +static void v_matchproto_(req_state_f) cnt_recv_prep(struct req *req, const char *ci) { const char *xff; @@ -849,7 +873,7 @@ cnt_recv_prep(struct req *req, const char *ci) * a interior request during ESI delivery. */ -static enum req_fsm_nxt +static enum req_fsm_nxt v_matchproto_(req_state_f) cnt_recv(struct worker *wrk, struct req *req) { unsigned recv_handling; @@ -986,7 +1010,7 @@ cnt_recv(struct worker *wrk, struct req *req) * In VCL, a restart is necessary to get a new object */ -static enum req_fsm_nxt +static enum req_fsm_nxt v_matchproto_(req_state_f) cnt_purge(struct worker *wrk, struct req *req) { struct objcore *oc, *boc; @@ -1037,7 +1061,7 @@ cnt_purge(struct worker *wrk, struct req *req) * */ -static void +static void v_matchproto_(req_state_f) cnt_diag(struct req *req, const char *state) { @@ -1102,17 +1126,12 @@ CNT_Request(struct req *req) CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - switch (req->req_step) { -#define REQ_STEP(l,u,arg) \ - case R_STP_##u: \ - if (DO_DEBUG(DBG_REQ_STATE)) \ - cnt_diag(req, #u); \ - nxt = cnt_##l arg; \ - break; -#include "tbl/steps.h" - default: - WRONG("State engine misfire"); - } + AN(req->req_step); + AN(req->req_step->name); + AN(req->req_step->func); + if (DO_DEBUG(DBG_REQ_STATE)) + cnt_diag(req, req->req_step->name); + nxt = req->req_step->func(wrk, req); CHECK_OBJ_ORNULL(wrk->nobjhead, OBJHEAD_MAGIC); } wrk->vsl = NULL; diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index f95f3c773..bc9750910 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -47,12 +47,23 @@ # include "VSC_main.h" #endif -/* -------------------------------------------------------------------*/ +/*--------------------------------------------------------------------*/ struct vfp; struct cli_proto; struct poolparam; +/*--------------------------------------------------------------------*/ + +typedef enum req_fsm_nxt req_state_f(struct worker *, struct req *); +struct req_step { + const char *name; + req_state_f *func; +}; + +extern const struct req_step R_STP_TRANSPORT[1]; +extern const struct req_step R_STP_RECV[1]; + /*-------------------------------------------------------------------- * HTTP Protocol connection structure * diff --git a/include/Makefile.am b/include/Makefile.am index 869ac7684..818011d3d 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -30,7 +30,6 @@ nobase_pkginclude_HEADERS = \ tbl/req_flags.h \ tbl/sess_attr.h \ tbl/sess_close.h \ - tbl/steps.h \ tbl/symbol_kind.h \ tbl/vcl_returns.h \ tbl/vcl_states.h \ diff --git a/include/tbl/steps.h b/include/tbl/steps.h deleted file mode 100644 index b16a65bfc..000000000 --- a/include/tbl/steps.h +++ /dev/null @@ -1,52 +0,0 @@ -/*- - * Copyright (c) 2006 Verdens Gang AS - * Copyright (c) 2006-2015 Varnish Software AS - * All rights reserved. - * - * Author: Poul-Henning Kamp - * - * SPDX-License-Identifier: BSD-2-Clause - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - */ - -/*lint -save -e525 -e539 */ - -#ifdef REQ_STEP - REQ_STEP(transport, TRANSPORT, (wrk, req)) - REQ_STEP(restart, RESTART, (wrk, req)) - REQ_STEP(recv, RECV, (wrk, req)) - REQ_STEP(pipe, PIPE, (wrk, req)) - REQ_STEP(pass, PASS, (wrk, req)) - REQ_STEP(lookup, LOOKUP, (wrk, req)) - REQ_STEP(purge, PURGE, (wrk, req)) - REQ_STEP(miss, MISS, (wrk, req)) - REQ_STEP(fetch, FETCH, (wrk, req)) - REQ_STEP(deliver, DELIVER, (wrk, req)) - REQ_STEP(vclfail, VCLFAIL, (wrk, req)) - REQ_STEP(synth, SYNTH, (wrk, req)) - REQ_STEP(transmit, TRANSMIT, (wrk, req)) - #undef REQ_STEP -#endif - -/*lint -restore */ From guillaume at varnish-software.com Mon Oct 26 17:43:09 2020 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Mon, 26 Oct 2020 17:43:09 +0000 (UTC) Subject: [6.0] e0691b85e [cci] add centos:8 and ubuntu:focal Message-ID: <20201026174309.6255D64B4@lists.varnish-cache.org> commit e0691b85eb5afced50d965164f03dd32759b61c8 Author: Guillaume Quintard Date: Mon Oct 26 08:45:13 2020 -0700 [cci] add centos:8 and ubuntu:focal diff --git a/.circleci/config.yml b/.circleci/config.yml index 2efe13d56..726d789db 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -269,6 +269,22 @@ workflows: - dist - dist - tar_pkg_tools + - package: + name: aarch64-ubuntu-focal + dist: ubuntu + release: focal + arch: aarch64 + image: arm64v8/ubuntu:focal + ext: deb + <<: *pkg_req + - package: + name: x64-ubuntu-focal + dist: ubuntu + release: focal + arch: x64 + image: ubuntu:focal + ext: deb + <<: *pkg_req - package: name: aarch64-ubuntu-bionic dist: ubuntu @@ -333,6 +349,22 @@ workflows: image: debian:stretch-slim ext: deb <<: *pkg_req + - package: + name: aarch64-centos-8 + dist: centos + release: "8" + arch: aarch64 + image: arm64v8/centos:8 + ext: rpm + <<: *pkg_req + - package: + name: x64-centos-8 + dist: centos + release: "8" + arch: x64 + image: centos:8 + ext: rpm + <<: *pkg_req - package: name: aarch64-centos-7 dist: centos @@ -355,9 +387,13 @@ workflows: - aarch64-ubuntu-xenial - x64-ubuntu-bionic - aarch64-ubuntu-bionic + - x64-ubuntu-focal + - aarch64-ubuntu-focal - x64-debian-stretch - aarch64-debian-stretch - x64-debian-buster - aarch64-debian-buster + - x64-centos-8 + - aarch64-centos-8 - x64-centos-7 - aarch64-centos-7 From phk at FreeBSD.org Mon Oct 26 20:07:09 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 26 Oct 2020 20:07:09 +0000 (UTC) Subject: [master] 276ca5e86 Give VDP a proper VDP_Init() function Message-ID: <20201026200709.54AD362353@lists.varnish-cache.org> commit 276ca5e860d436bca8b483b1653b41e0fb989f69 Author: Poul-Henning Kamp Date: Mon Oct 26 18:14:31 2020 +0000 Give VDP a proper VDP_Init() function diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 79234e9f3..7ba2450a5 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -34,6 +34,13 @@ #include "cache_varnishd.h" #include "cache_filter.h" +void +VDP_Init(struct vdp_ctx *vdx) +{ + INIT_OBJ(vdx, VDP_CTX_MAGIC); + VTAILQ_INIT(&vdx->vdp); +} + /* VDP_bytes * * Pushes len bytes at ptr down the delivery processor list. @@ -57,6 +64,7 @@ * r == 0: Continue * r > 0: Stop, breaks out early without error condition */ + int VDP_bytes(struct vdp_ctx *vdx, enum vdp_action act, const void *ptr, ssize_t len) { diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index e106b5c1f..f185171a8 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -148,6 +148,7 @@ struct vdp_ctx { struct req *req; }; +void VDP_Init(struct vdp_ctx *vdx); int VDP_bytes(struct vdp_ctx *, enum vdp_action act, const void *ptr, ssize_t len); int VDP_Push(struct req *, const struct vdp *, void *priv); void VRT_AddVDP(VRT_CTX, const struct vdp *); diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index 0a3bc65a5..7f94d8348 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -131,8 +131,7 @@ Req_New(const struct worker *wrk, struct sess *sp) p = (void*)PRNDUP(p + sizeof(*req->htc)); req->vdc = (void*)p; - INIT_OBJ(req->vdc, VDP_CTX_MAGIC); - VTAILQ_INIT(&req->vdc->vdp); + VDP_Init(req->vdc); p = (void*)PRNDUP(p + sizeof(*req->vdc)); req->htc = (void*)p; From phk at FreeBSD.org Mon Oct 26 20:07:09 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 26 Oct 2020 20:07:09 +0000 (UTC) Subject: [master] dfe6df9a6 Rename bo->filter_list -> bo->vfp_filter_list Message-ID: <20201026200709.6C58762357@lists.varnish-cache.org> commit dfe6df9a6c4b693a3e18d7350734667ef624df0e Author: Poul-Henning Kamp Date: Mon Oct 26 18:21:36 2020 +0000 Rename bo->filter_list -> bo->vfp_filter_list diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index dbeabe7b1..b8dbd1303 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -382,7 +382,7 @@ struct busyobj { struct worker *wrk; struct vfp_ctx *vfc; - const char *filter_list; + const char *vfp_filter_list; struct ws ws[1]; uintptr_t ws_bo; diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 218df239a..13d240a83 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -127,7 +127,7 @@ vbf_cleanup(struct busyobj *bo) CHECK_OBJ_NOTNULL(vfc, VFP_CTX_MAGIC); bo->acct.beresp_bodybytes += VFP_Close(vfc); - bo->filter_list = NULL; + bo->vfp_filter_list = NULL; if (bo->director_state != DIR_S_NULL) VDI_Finish(bo); @@ -147,7 +147,7 @@ Bereq_Rollback(struct busyobj *bo) VCL_TaskLeave(bo->privs); VCL_TaskEnter(bo->privs); HTTP_Clone(bo->bereq, bo->bereq0); - bo->filter_list = NULL; + bo->vfp_filter_list = NULL; bo->err_reason = NULL; WS_Rollback(bo->bereq->ws, bo->ws_bo); WS_Rollback(bo->ws, bo->ws_bo); @@ -635,13 +635,13 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) http_Unset(bo->beresp, H_Content_Encoding); bo->do_gzip = bo->do_gunzip = 0; bo->do_stream = 0; - bo->filter_list = ""; - } else if (bo->filter_list == NULL) { - bo->filter_list = VBF_Get_Filter_List(bo); + bo->vfp_filter_list = ""; + } else if (bo->vfp_filter_list == NULL) { + bo->vfp_filter_list = VBF_Get_Filter_List(bo); } - if (bo->filter_list == NULL || - VCL_StackVFP(bo->vfc, bo->vcl, bo->filter_list)) { + if (bo->vfp_filter_list == NULL || + VCL_StackVFP(bo->vfc, bo->vcl, bo->vfp_filter_list)) { (bo)->htc->doclose = SC_OVERLOAD; vbf_cleanup(bo); return (F_STP_ERROR); diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index a0292e023..abd637e2f 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -391,8 +391,8 @@ pan_busyobj(struct vsb *vsb, const struct busyobj *bo) if (bo->vfc != NULL) pan_vfp(vsb, bo->vfc); - if (bo->filter_list != NULL) - VSB_printf(vsb, "filter_list = \"%s\",\n", bo->filter_list); + if (bo->vfp_filter_list != NULL) + VSB_printf(vsb, "filter_list = \"%s\",\n", bo->vfp_filter_list); WS_Panic(bo->ws, vsb); VSB_printf(vsb, "ws_bo = %p,\n", (void *)bo->ws_bo); diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c index 5f972520e..54002bec4 100644 --- a/bin/varnishd/cache/cache_vrt_filter.c +++ b/bin/varnishd/cache/cache_vrt_filter.c @@ -386,14 +386,14 @@ resp_Get_Filter_List(struct req *req) /*--------------------------------------------------------------------*/ -#define FILTER_VAR(vcl, in, func) \ +#define FILTER_VAR(vcl, in, func, fld) \ VCL_STRING \ VRT_r_##vcl##_filters(VRT_CTX) \ { \ \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ - if (ctx->in->filter_list != NULL) \ - return(ctx->in->filter_list); \ + if (ctx->in->fld != NULL) \ + return(ctx->in->fld); \ return (func(ctx->in)); \ } \ \ @@ -410,8 +410,8 @@ resp_Get_Filter_List(struct req *req) if (b == NULL) \ WS_MarkOverflow(ctx->in->ws); \ else \ - ctx->in->filter_list = b; \ + ctx->in->fld = b; \ } -FILTER_VAR(beresp, bo, VBF_Get_Filter_List) -FILTER_VAR(resp, req, resp_Get_Filter_List) +FILTER_VAR(beresp, bo, VBF_Get_Filter_List, vfp_filter_list) +FILTER_VAR(resp, req, resp_Get_Filter_List, filter_list) diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 957d49cdb..b9165d96b 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -189,9 +189,10 @@ VRT_r_obj_reason(VRT_CTX) static inline int beresp_filter_fixed(VRT_CTX, const char *s) { - if (ctx->bo->filter_list == NULL) + if (ctx->bo->vfp_filter_list == NULL) return (0); - VRT_fail(ctx, "beresp.filters are already fixed, beresp.%s is undefined", s); + VRT_fail(ctx, + "beresp.filters are already fixed, beresp.%s is undefined", s); return (1); } From phk at FreeBSD.org Mon Oct 26 20:07:09 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 26 Oct 2020 20:07:09 +0000 (UTC) Subject: [master] faa60dc52 Make arguments to VDP_DeliverObj() explicit Message-ID: <20201026200709.852CA6235F@lists.varnish-cache.org> commit faa60dc526859eb0ba6b8a0baec38689803ff27c Author: Poul-Henning Kamp Date: Mon Oct 26 18:55:31 2020 +0000 Make arguments to VDP_DeliverObj() explicit diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 7ba2450a5..5cf465866 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -184,20 +184,23 @@ vdp_objiterator(void *priv, unsigned flush, const void *ptr, ssize_t len) } -int -VDP_DeliverObj(struct req *req) +int VDP_DeliverObj(struct vdp_ctx *vdc, struct objcore *oc, struct worker *wrk, + struct vsl_log *vsl, struct req *req) { int r, final; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - req->vdc->req = req; - req->vdc->vsl = req->vsl; - req->vdc->wrk = req->wrk; - final = req->objcore->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP) - ? 1 : 0; - r = ObjIterate(req->wrk, req->objcore, req->vdc, vdp_objiterator, final); + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + AN(vsl); + CHECK_OBJ_ORNULL(req, REQ_MAGIC); + vdc->vsl = vsl; + vdc->wrk = wrk; + vdc->req = req; + final = oc->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP) ? 1 : 0; + r = ObjIterate(wrk, oc, vdc, vdp_objiterator, final); if (r == 0) - r = VDP_bytes(req->vdc, VDP_END, NULL, 0); + r = VDP_bytes(vdc, VDP_END, NULL, 0); if (r < 0) return (r); return (0); diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 5e702a3be..4548b5a51 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -865,7 +865,8 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody) } if (i == 0) { - i = VDP_DeliverObj(req); + i = VDP_DeliverObj( + req->vdc, req->objcore, req->wrk, req->vsl, req); } else { VSLb(req->vsl, SLT_Error, "Failure to push ESI processors"); req->doclose = SC_OVERLOAD; diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index f185171a8..fbf9b76ad 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -148,6 +148,14 @@ struct vdp_ctx { struct req *req; }; +extern const struct vdp VDP_gunzip; +extern const struct vdp VDP_esi; +extern const struct vdp VDP_range; + +uint64_t VDP_Close(struct req *req); +int VDP_DeliverObj(struct vdp_ctx *vdc, struct objcore *oc, struct worker *wrk, + struct vsl_log *vsl, struct req *req); + void VDP_Init(struct vdp_ctx *vdx); int VDP_bytes(struct vdp_ctx *, enum vdp_action act, const void *ptr, ssize_t len); int VDP_Push(struct req *, const struct vdp *, void *priv); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index bc9750910..385c38e0d 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -233,14 +233,6 @@ void CLI_Init(void); void CLI_Run(void); void CLI_AddFuncs(struct cli_proto *p); -/* cache_deliver_proc.c */ -uint64_t VDP_Close(struct req *req); -int VDP_DeliverObj(struct req *req); - -extern const struct vdp VDP_gunzip; -extern const struct vdp VDP_esi; -extern const struct vdp VDP_range; - /* cache_expire.c */ void EXP_Init(void); void EXP_Shutdown(void); diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c index 42262d384..ec8826168 100644 --- a/bin/varnishd/http1/cache_http1_deliver.c +++ b/bin/varnishd/http1/cache_http1_deliver.c @@ -149,7 +149,8 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody) (void)V1L_Flush(req->wrk); if (chunked) V1L_Chunked(req->wrk); - err = VDP_DeliverObj(req); + err = VDP_DeliverObj( + req->vdc, req->objcore, req->wrk, req->vsl, req); if (!err && chunked) V1L_EndChunk(req->wrk); } diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index 54be4148b..1ebfd3253 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -332,7 +332,8 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody) /* XXX someone into H2 please add appropriate error handling */ if (sendbody) { if (!VDP_Push(req, &h2_vdp, NULL)) - (void)VDP_DeliverObj(req); + (void)VDP_DeliverObj( + req->vdc, req->objcore, req->wrk, req->vsl, req); } AZ(req->wrk->v1l); From phk at FreeBSD.org Mon Oct 26 20:07:09 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 26 Oct 2020 20:07:09 +0000 (UTC) Subject: [master] 9d32fa37c Replace VDP->fini()'s req arg with vdp_ctx. Message-ID: <20201026200709.9F84C62363@lists.varnish-cache.org> commit 9d32fa37c1bee40bdb300ca1fcb8441d0040299c Author: Poul-Henning Kamp Date: Mon Oct 26 19:03:46 2020 +0000 Replace VDP->fini()'s req arg with vdp_ctx. diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 5cf465866..30e573d93 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -160,7 +160,7 @@ VDP_Close(struct req *req) if (vdpe != NULL) { CHECK_OBJ(vdpe, VDP_ENTRY_MAGIC); if (vdpe->vdp->fini != NULL) - AZ(vdpe->vdp->fini(req, &vdpe->priv)); + AZ(vdpe->vdp->fini(vdc, &vdpe->priv)); AZ(vdpe->priv); VTAILQ_REMOVE(&vdc->vdp, vdpe, list); } diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 4548b5a51..decd0162d 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -286,11 +286,11 @@ ved_vdp_esi_init(struct req *req, void **priv) } static int v_matchproto_(vdp_fini_f) -ved_vdp_esi_fini(struct req *req, void **priv) +ved_vdp_esi_fini(struct vdp_ctx *vdc, void **priv) { struct ecx *ecx; - (void)req; + (void)vdc; AN(priv); CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC); FREE_OBJ(ecx); @@ -500,9 +500,9 @@ ved_bytes(struct ecx *ecx, enum vdp_action act, */ static int v_matchproto_(vdp_fini_f) -ved_pretend_gzip_fini(struct req *req, void **priv) +ved_pretend_gzip_fini(struct vdp_ctx *vdc, void **priv) { - (void)req; + (void)vdc; *priv = NULL; return (0); } @@ -755,13 +755,13 @@ ved_gzgz_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, } static int v_matchproto_(vdp_fini_f) -ved_gzgz_fini(struct req *req, void **priv) +ved_gzgz_fini(struct vdp_ctx *vdc, void **priv) { uint32_t icrc; uint32_t ilen; struct ved_foo *foo; - (void)req; + (void)vdc; CAST_OBJ_NOTNULL(foo, *priv, VED_FOO_MAGIC); *priv = NULL; @@ -793,9 +793,9 @@ static const struct vdp ved_gzgz = { */ static int v_matchproto_(vdp_fini_f) -ved_vdp_fini(struct req *req, void **priv) +ved_vdp_fini(struct vdp_ctx *vdc, void **priv) { - (void)req; + (void)vdc; *priv = NULL; return (0); } diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index fbf9b76ad..cf837df5a 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -112,7 +112,7 @@ typedef int vdp_init_f(struct req *, void **priv); struct vdp_ctx; -typedef int vdp_fini_f(struct req *, void **priv); +typedef int vdp_fini_f(struct vdp_ctx *, void **priv); typedef int vdp_bytes_f(struct vdp_ctx *, enum vdp_action, void **priv, const void *ptr, ssize_t len); diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 47dad1e52..67bbc69f2 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -341,11 +341,11 @@ vdp_gunzip_init(struct req *req, void **priv) } static int v_matchproto_(vdp_fini_f) -vdp_gunzip_fini(struct req *req, void **priv) +vdp_gunzip_fini(struct vdp_ctx *vdc, void **priv) { struct vgz *vg; - (void)req; + (void)vdc; CAST_OBJ_NOTNULL(vg, *priv, VGZ_MAGIC); AN(vg->m_buf); (void)VGZ_Destroy(&vg); diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index fe9943718..a5516ae7d 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -48,14 +48,15 @@ struct vrg_priv { }; static int v_matchproto_(vdp_fini_f) -vrg_range_fini(struct req *req, void **priv) +vrg_range_fini(struct vdp_ctx *vdc, void **priv) { struct vrg_priv *vrg_priv; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vdc->req, REQ_MAGIC); CAST_OBJ_NOTNULL(vrg_priv, *priv, VRG_PRIV_MAGIC); if (vrg_priv->range_off < vrg_priv->range_high) - Req_Fail(req, SC_RANGE_SHORT); + Req_Fail(vdc->req, SC_RANGE_SHORT); *priv = NULL; /* struct on ws, no need to free */ return (0); } diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index 1ebfd3253..8bdbdc50d 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -83,18 +83,19 @@ h2_init(struct req *req, void **priv) } static int v_matchproto_(vdp_fini_f) -h2_fini(struct req *req, void **priv) +h2_fini(struct vdp_ctx *vdc, void **priv) { struct h2_req *r2; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vdc->wrk, WORKER_MAGIC); TAKE_OBJ_NOTNULL(r2, priv, H2_REQ_MAGIC); if (r2->error) return (0); - H2_Send_Get(req->wrk, r2->h2sess, r2); - H2_Send(req->wrk, r2, H2_F_DATA, H2FF_DATA_END_STREAM, 0, "", NULL); + H2_Send_Get(vdc->wrk, r2->h2sess, r2); + H2_Send(vdc->wrk, r2, H2_F_DATA, H2FF_DATA_END_STREAM, 0, "", NULL); H2_Send_Rel(r2->h2sess, r2); return (0); } diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 65ce4bed5..aacebc4d9 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -150,9 +150,9 @@ xyzzy_rot13_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, } static int v_matchproto_(vdp_fini_f) -xyzzy_rot13_fini(struct req *req, void **priv) +xyzzy_rot13_fini(struct vdp_ctx *vdc, void **priv) { - (void)req; + (void)vdc; AN(priv); free(*priv); *priv = NULL; @@ -187,9 +187,9 @@ xyzzy_pedantic_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, } static int v_matchproto_(vdp_fini_f) -xyzzy_pedantic_fini(struct req *req, void **priv) +xyzzy_pedantic_fini(struct vdp_ctx *vdc, void **priv) { - (void) req; + (void) vdc; assert (*priv == end_marker); *priv = NULL; return (0); From phk at FreeBSD.org Mon Oct 26 20:07:09 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 26 Oct 2020 20:07:09 +0000 (UTC) Subject: [master] 5b212a2c5 Postpone VDP_Init() until we need it. Zero in the meantime. Message-ID: <20201026200709.B67EA62369@lists.varnish-cache.org> commit 5b212a2c5966698e243402d230d18c7bd0496c01 Author: Poul-Henning Kamp Date: Mon Oct 26 19:29:21 2020 +0000 Postpone VDP_Init() until we need it. Zero in the meantime. diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index 7f94d8348..2de283512 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -131,7 +131,7 @@ Req_New(const struct worker *wrk, struct sess *sp) p = (void*)PRNDUP(p + sizeof(*req->htc)); req->vdc = (void*)p; - VDP_Init(req->vdc); + memset(req->vdc, 0, sizeof *req->vdc); p = (void*)PRNDUP(p + sizeof(*req->vdc)); req->htc = (void*)p; diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index b84591bb8..9ad804dea 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -425,6 +425,7 @@ cnt_transmit(struct worker *wrk, struct req *req) sendbody = 1; } + VDP_Init(req->vdc); if (req->filter_list == NULL) req->filter_list = resp_Get_Filter_List(req); if (req->filter_list == NULL || From phk at FreeBSD.org Mon Oct 26 20:07:09 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 26 Oct 2020 20:07:09 +0000 (UTC) Subject: [master] dfaa97033 Move the initialization arguments from VDP_DeliverObj() to VDP_init() Message-ID: <20201026200709.CE23B6236D@lists.varnish-cache.org> commit dfaa9703373cc2df659c8b387a1b273499ad0427 Author: Poul-Henning Kamp Date: Mon Oct 26 19:46:44 2020 +0000 Move the initialization arguments from VDP_DeliverObj() to VDP_init() diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 30e573d93..65bcc6ff6 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -35,10 +35,19 @@ #include "cache_filter.h" void -VDP_Init(struct vdp_ctx *vdx) +VDP_Init(struct vdp_ctx *vdc, struct worker *wrk, struct vsl_log *vsl, + struct req *req) { - INIT_OBJ(vdx, VDP_CTX_MAGIC); - VTAILQ_INIT(&vdx->vdp); + AN(vdc); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + AN(vsl); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + + INIT_OBJ(vdc, VDP_CTX_MAGIC); + VTAILQ_INIT(&vdc->vdp); + vdc->wrk = wrk; + vdc->vsl = vsl; + vdc->req = req; } /* VDP_bytes @@ -184,21 +193,16 @@ vdp_objiterator(void *priv, unsigned flush, const void *ptr, ssize_t len) } -int VDP_DeliverObj(struct vdp_ctx *vdc, struct objcore *oc, struct worker *wrk, - struct vsl_log *vsl, struct req *req) +int VDP_DeliverObj(struct vdp_ctx *vdc, struct objcore *oc) { int r, final; CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - AN(vsl); - CHECK_OBJ_ORNULL(req, REQ_MAGIC); - vdc->vsl = vsl; - vdc->wrk = wrk; - vdc->req = req; + CHECK_OBJ_NOTNULL(vdc->wrk, WORKER_MAGIC); + AN(vdc->vsl); final = oc->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP) ? 1 : 0; - r = ObjIterate(wrk, oc, vdc, vdp_objiterator, final); + r = ObjIterate(vdc->wrk, oc, vdc, vdp_objiterator, final); if (r == 0) r = VDP_bytes(vdc, VDP_END, NULL, 0); if (r < 0) diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index decd0162d..43ebe9bf0 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -865,8 +865,7 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody) } if (i == 0) { - i = VDP_DeliverObj( - req->vdc, req->objcore, req->wrk, req->vsl, req); + i = VDP_DeliverObj(req->vdc, req->objcore); } else { VSLb(req->vsl, SLT_Error, "Failure to push ESI processors"); req->doclose = SC_OVERLOAD; diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index cf837df5a..4ab1ca74e 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -152,11 +152,12 @@ extern const struct vdp VDP_gunzip; extern const struct vdp VDP_esi; extern const struct vdp VDP_range; +void VDP_Init(struct vdp_ctx *vdx, struct worker *wrk, struct vsl_log *vsl, + struct req *req); + uint64_t VDP_Close(struct req *req); -int VDP_DeliverObj(struct vdp_ctx *vdc, struct objcore *oc, struct worker *wrk, - struct vsl_log *vsl, struct req *req); +int VDP_DeliverObj(struct vdp_ctx *vdc, struct objcore *oc); -void VDP_Init(struct vdp_ctx *vdx); int VDP_bytes(struct vdp_ctx *, enum vdp_action act, const void *ptr, ssize_t len); int VDP_Push(struct req *, const struct vdp *, void *priv); void VRT_AddVDP(VRT_CTX, const struct vdp *); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 9ad804dea..2bf6c485f 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -425,7 +425,7 @@ cnt_transmit(struct worker *wrk, struct req *req) sendbody = 1; } - VDP_Init(req->vdc); + VDP_Init(req->vdc, req->wrk, req->vsl, req); if (req->filter_list == NULL) req->filter_list = resp_Get_Filter_List(req); if (req->filter_list == NULL || diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c index ec8826168..03e52514f 100644 --- a/bin/varnishd/http1/cache_http1_deliver.c +++ b/bin/varnishd/http1/cache_http1_deliver.c @@ -149,8 +149,7 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody) (void)V1L_Flush(req->wrk); if (chunked) V1L_Chunked(req->wrk); - err = VDP_DeliverObj( - req->vdc, req->objcore, req->wrk, req->vsl, req); + err = VDP_DeliverObj(req->vdc, req->objcore); if (!err && chunked) V1L_EndChunk(req->wrk); } diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index 8bdbdc50d..4ead4dcd3 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -333,8 +333,7 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody) /* XXX someone into H2 please add appropriate error handling */ if (sendbody) { if (!VDP_Push(req, &h2_vdp, NULL)) - (void)VDP_DeliverObj( - req->vdc, req->objcore, req->wrk, req->vsl, req); + (void)VDP_DeliverObj(req->vdc, req->objcore); } AZ(req->wrk->v1l); From phk at FreeBSD.org Mon Oct 26 20:07:09 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 26 Oct 2020 20:07:09 +0000 (UTC) Subject: [master] 57359c500 Give VDP->init() vdp_ctx arg instead of req Message-ID: <20201026200709.E75EA62373@lists.varnish-cache.org> commit 57359c500d293189dfba11a19b64d02989dca8ae Author: Poul-Henning Kamp Date: Mon Oct 26 19:50:50 2020 +0000 Give VDP->init() vdp_ctx arg instead of req diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 65bcc6ff6..a8951305c 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -141,7 +141,7 @@ VDP_Push(struct req *req, const struct vdp *vdp, void *priv) AZ(vdc->retval); if (vdpe->vdp->init != NULL) - vdc->retval = vdpe->vdp->init(req, &vdpe->priv); + vdc->retval = vdpe->vdp->init(vdc, &vdpe->priv); if (vdc->retval > 0) { VTAILQ_REMOVE(&vdc->vdp, vdpe, list); vdc->nxt = VTAILQ_FIRST(&vdc->vdp); diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 43ebe9bf0..242cd19dc 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -254,15 +254,18 @@ ved_decode_len(struct vsl_log *vsl, const uint8_t **pp) */ static int v_matchproto_(vdp_init_f) -ved_vdp_esi_init(struct req *req, void **priv) +ved_vdp_esi_init(struct vdp_ctx *vdc, void **priv) { struct ecx *ecx; + struct req *req; + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); + req = vdc->req; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); AN(priv); AZ(*priv); - if (!ObjHasAttr(req->wrk, req->objcore, OA_ESIDATA)) + if (!ObjHasAttr(vdc->wrk, req->objcore, OA_ESIDATA)) return (1); ALLOC_OBJ(ecx, ECX_MAGIC); @@ -584,26 +587,29 @@ struct ved_foo { }; static int v_matchproto_(vdp_fini_f) -ved_gzgz_init(struct req *req, void **priv) +ved_gzgz_init(struct vdp_ctx *vdc, void **priv) { ssize_t l; const char *p; struct ved_foo *foo; + struct req *req; + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); + req = vdc->req; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CAST_OBJ_NOTNULL(foo, *priv, VED_FOO_MAGIC); memset(foo->tailbuf, 0xdd, sizeof foo->tailbuf); - AN(ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED)); + AN(ObjCheckFlag(vdc->wrk, req->objcore, OF_GZIPED)); - p = ObjGetAttr(req->wrk, req->objcore, OA_GZIPBITS, &l); + p = ObjGetAttr(vdc->wrk, req->objcore, OA_GZIPBITS, &l); AN(p); assert(l == 32); foo->start = vbe64dec(p); foo->last = vbe64dec(p + 8); foo->stop = vbe64dec(p + 16); - foo->olen = ObjGetLen(req->wrk, req->objcore); + foo->olen = ObjGetLen(vdc->wrk, req->objcore); assert(foo->start > 0 && foo->start < foo->olen * 8); assert(foo->last > 0 && foo->last < foo->olen * 8); assert(foo->stop > 0 && foo->stop < foo->olen * 8); diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index 4ab1ca74e..e83706951 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -102,7 +102,7 @@ enum vdp_action { VDP_END, /* Last buffer or after, implies VDP_FLUSH */ }; -typedef int vdp_init_f(struct req *, void **priv); +typedef int vdp_init_f(struct vdp_ctx *, void **priv); /* * Return value: * negative: Error - abandon delivery diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 67bbc69f2..147fe67f0 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -288,15 +288,18 @@ VGZ_Gzip(struct vgz *vg, const void **pptr, ssize_t *plen, enum vgz_flag flags) */ static int v_matchproto_(vdp_init_f) -vdp_gunzip_init(struct req *req, void **priv) +vdp_gunzip_init(struct vdp_ctx *vdp, void **priv) { struct vgz *vg; struct boc *boc; + struct req *req; enum boc_state_e bos; const char *p; ssize_t dl; uint64_t u; + CHECK_OBJ_NOTNULL(vdp, VDP_CTX_MAGIC); + req = vdp->req; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index a5516ae7d..016a95f90 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -243,29 +243,31 @@ vrg_ifrange(struct req *req) } static int v_matchproto_(vdp_init_f) -vrg_range_init(struct req *req, void **priv) +vrg_range_init(struct vdp_ctx *vdc, void **priv) { const char *r; const char *err; - assert(http_GetHdr(req->http, H_Range, &r)); - if (!vrg_ifrange(req)) // rfc7233,l,455,456 + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vdc->req, REQ_MAGIC); + assert(http_GetHdr(vdc->req->http, H_Range, &r)); + if (!vrg_ifrange(vdc->req)) // rfc7233,l,455,456 return (1); - err = vrg_dorange(req, r, priv); + err = vrg_dorange(vdc->req, r, priv); if (err == NULL) return (*priv == NULL ? 1 : 0); - VSLb(req->vsl, SLT_Debug, "RANGE_FAIL %s", err); - if (req->resp_len >= 0) - http_PrintfHeader(req->resp, + VSLb(vdc->vsl, SLT_Debug, "RANGE_FAIL %s", err); + if (vdc->req->resp_len >= 0) + http_PrintfHeader(vdc->req->resp, "Content-Range: bytes */%jd", - (intmax_t)req->resp_len); - http_PutResponse(req->resp, "HTTP/1.1", 416, NULL); + (intmax_t)vdc->req->resp_len); + http_PutResponse(vdc->req->resp, "HTTP/1.1", 416, NULL); /* * XXX: We ought to produce a body explaining things. * XXX: That really calls for us to hit vcl_synth{} */ - req->resp_len = 0; + vdc->req->resp_len = 0; return (1); } diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index 4ead4dcd3..729fd6bec 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -75,10 +75,12 @@ V2D_Init(void) /**********************************************************************/ static int v_matchproto_(vdp_init_f) -h2_init(struct req *req, void **priv) +h2_init(struct vdp_ctx *vdc, void **priv) { - *priv = req->transport_priv; + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vdc->req, REQ_MAGIC); + *priv = vdc->req->transport_priv; return (0); } diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index aacebc4d9..44afe6098 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -101,9 +101,9 @@ static const struct vfp xyzzy_rot13 = { #define ROT13_BUFSZ 8 static int v_matchproto_(vdp_init_f) -xyzzy_rot13_init(struct req *req, void **priv) +xyzzy_rot13_init(struct vdp_ctx *vdc, void **priv) { - (void)req; + (void)vdc; AN(priv); *priv = malloc(ROT13_BUFSZ); if (*priv == NULL) From phk at FreeBSD.org Mon Oct 26 20:07:10 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 26 Oct 2020 20:07:10 +0000 (UTC) Subject: [master] ff82e01a9 Give VDP_Push() a vdp_ctx argument instead of req Message-ID: <20201026200710.0BA176237A@lists.varnish-cache.org> commit ff82e01a93a2f424e6356523846e25a66a528232 Author: Poul-Henning Kamp Date: Mon Oct 26 19:58:44 2020 +0000 Give VDP_Push() a vdp_ctx argument instead of req diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index a8951305c..c1fe1ad9a 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -81,7 +81,6 @@ VDP_bytes(struct vdp_ctx *vdx, enum vdp_action act, const void *ptr, ssize_t len struct vdp_entry *vdpe; CHECK_OBJ_NOTNULL(vdx, VDP_CTX_MAGIC); - CHECK_OBJ_NOTNULL(vdx->req, REQ_MAGIC); if (vdx->retval) return (vdx->retval); vdpe = vdx->nxt; @@ -110,13 +109,12 @@ VDP_bytes(struct vdp_ctx *vdx, enum vdp_action act, const void *ptr, ssize_t len } int -VDP_Push(struct req *req, const struct vdp *vdp, void *priv) +VDP_Push(struct vdp_ctx *vdc, struct ws *ws, const struct vdp *vdp, void *priv) { struct vdp_entry *vdpe; - struct vdp_ctx *vdc; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - vdc = req->vdc; + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); + AN(ws); AN(vdp); AN(vdp->name); AN(vdp->bytes); @@ -125,9 +123,9 @@ VDP_Push(struct req *req, const struct vdp *vdp, void *priv) return (vdc->retval); if (DO_DEBUG(DBG_PROCESSORS)) - VSLb(req->vsl, SLT_Debug, "VDP_push(%s)", vdp->name); + VSLb(vdc->vsl, SLT_Debug, "VDP_push(%s)", vdp->name); - vdpe = WS_Alloc(req->ws, sizeof *vdpe); + vdpe = WS_Alloc(ws, sizeof *vdpe); if (vdpe == NULL) { AZ(vdc->retval); vdc->retval = -1; diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 242cd19dc..d7030cc35 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -860,14 +860,14 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody) INIT_OBJ(foo, VED_FOO_MAGIC); foo->ecx = ecx; - i = VDP_Push(req, &ved_gzgz, foo); + i = VDP_Push(req->vdc, req->ws, &ved_gzgz, foo); } else if (ecx->isgzip && !i) { /* Non-Gzip'ed include in gzip'ed parent */ - i = VDP_Push(req, &ved_pretend_gz, ecx); + i = VDP_Push(req->vdc, req->ws, &ved_pretend_gz, ecx); } else { /* Anything else goes straight through */ - i = VDP_Push(req, &ved_ved, ecx); + i = VDP_Push(req->vdc, req->ws, &ved_ved, ecx); } if (i == 0) { diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index e83706951..60a2a53c1 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -159,6 +159,6 @@ uint64_t VDP_Close(struct req *req); int VDP_DeliverObj(struct vdp_ctx *vdc, struct objcore *oc); int VDP_bytes(struct vdp_ctx *, enum vdp_action act, const void *ptr, ssize_t len); -int VDP_Push(struct req *, const struct vdp *, void *priv); +int VDP_Push(struct vdp_ctx *, struct ws *, const struct vdp *, void *priv); void VRT_AddVDP(VRT_CTX, const struct vdp *); void VRT_RemoveVDP(VRT_CTX, const struct vdp *); diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c index 54002bec4..485915c11 100644 --- a/bin/varnishd/cache/cache_vrt_filter.c +++ b/bin/varnishd/cache/cache_vrt_filter.c @@ -238,7 +238,7 @@ VCL_StackVDP(struct req *req, const struct vcl *vcl, const char *fl) "Filter '...%s' not found", fl); return (-1); } - if (VDP_Push(req, vp->vdp, NULL)) + if (VDP_Push(req->vdc, req->ws, vp->vdp, NULL)) return (-1); } } diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c index 03e52514f..aad32808f 100644 --- a/bin/varnishd/http1/cache_http1_deliver.c +++ b/bin/varnishd/http1/cache_http1_deliver.c @@ -114,7 +114,7 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody) req->doclose = SC_TX_EOF; } } - if (VDP_Push(req, &v1d_vdp, NULL)) { + if (VDP_Push(req->vdc, req->ws, &v1d_vdp, NULL)) { v1d_error(req, "workspace_thread overflow"); AZ(req->wrk->v1l); return; diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index 729fd6bec..3e257f151 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -334,7 +334,7 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody) /* XXX someone into H2 please add appropriate error handling */ if (sendbody) { - if (!VDP_Push(req, &h2_vdp, NULL)) + if (!VDP_Push(req->vdc, req->ws, &h2_vdp, NULL)) (void)VDP_DeliverObj(req->vdc, req->objcore); } From phk at FreeBSD.org Mon Oct 26 20:07:10 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 26 Oct 2020 20:07:10 +0000 (UTC) Subject: [master] dc6f44ce9 Finally, make VDP_Close() take vdp_ctx instead of req arg. Message-ID: <20201026200710.2212F6238B@lists.varnish-cache.org> commit dc6f44ce9450cd184fd34144fd9d4f77a0599415 Author: Poul-Henning Kamp Date: Mon Oct 26 20:05:50 2020 +0000 Finally, make VDP_Close() take vdp_ctx instead of req arg. diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index c1fe1ad9a..076532bcf 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -149,18 +149,16 @@ VDP_Push(struct vdp_ctx *vdc, struct ws *ws, const struct vdp *vdp, void *priv) } uint64_t -VDP_Close(struct req *req) +VDP_Close(struct vdp_ctx *vdc) { struct vdp_entry *vdpe; - struct vdp_ctx *vdc; uint64_t rv = 0; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - vdc = req->vdc; + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); while (!VTAILQ_EMPTY(&vdc->vdp)) { vdpe = VTAILQ_FIRST(&vdc->vdp); rv = vdpe->bytes_in; - VSLb(req->vsl, SLT_VdpAcct, "%s %ju %ju", vdpe->vdp->name, + VSLb(vdc->vsl, SLT_VdpAcct, "%s %ju %ju", vdpe->vdp->name, (uintmax_t)vdpe->calls, (uintmax_t)rv); if (vdc->retval >= 0) AN(vdpe); diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index d7030cc35..6179b7bcc 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -880,5 +880,5 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody) if (i && req->doclose == SC_NULL) req->doclose = SC_REM_CLOSE; - req->acct.resp_bodybytes += VDP_Close(req); + req->acct.resp_bodybytes += VDP_Close(req->vdc); } diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index 60a2a53c1..e3b3cc613 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -155,7 +155,7 @@ extern const struct vdp VDP_range; void VDP_Init(struct vdp_ctx *vdx, struct worker *wrk, struct vsl_log *vsl, struct req *req); -uint64_t VDP_Close(struct req *req); +uint64_t VDP_Close(struct vdp_ctx *); int VDP_DeliverObj(struct vdp_ctx *vdc, struct objcore *oc); int VDP_bytes(struct vdp_ctx *, enum vdp_action act, const void *ptr, ssize_t len); diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c index aad32808f..77ef66f3f 100644 --- a/bin/varnishd/http1/cache_http1_deliver.c +++ b/bin/varnishd/http1/cache_http1_deliver.c @@ -158,7 +158,7 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody) AZ(req->wrk->v1l); req->acct.resp_hdrbytes += hdrbytes; - req->acct.resp_bodybytes += VDP_Close(req); + req->acct.resp_bodybytes += VDP_Close(req->vdc); if (sc == SC_NULL && err && req->sp->fd >= 0) sc = SC_REM_CLOSE; diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index 3e257f151..4ee707816 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -339,5 +339,5 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody) } AZ(req->wrk->v1l); - req->acct.resp_bodybytes += VDP_Close(req); + req->acct.resp_bodybytes += VDP_Close(req->vdc); } From nils.goroll at uplex.de Mon Oct 26 21:11:58 2020 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 26 Oct 2020 22:11:58 +0100 Subject: [master] dc6f44ce9 Finally, make VDP_Close() take vdp_ctx instead of req arg. In-Reply-To: <20201026200710.2212F6238B@lists.varnish-cache.org> References: <20201026200710.2212F6238B@lists.varnish-cache.org> Message-ID: So now that this has broken PRs again, are there any reasons not to merge #3298 and #3287 after rebasing one last time? -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From dridi.boukelmoune at gmail.com Tue Oct 27 09:36:15 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 27 Oct 2020 09:36:15 +0000 (UTC) Subject: [master] 0e12fb1d8 vsl: Fold Backend_health window bits docs Message-ID: <20201027093615.74C171113E1@lists.varnish-cache.org> commit 0e12fb1d8935a3a63e99e68cce03cb6853cd48f4 Author: Dridi Boukelmoune Date: Tue Oct 27 10:31:01 2020 +0100 vsl: Fold Backend_health window bits docs The net result is the same, but should be more readable. Better diff with the --word-diff --word-diff-regex=. options. diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 33ee14be4..016cc5784 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -455,15 +455,15 @@ SLTM(Backend_health, 0, "Backend health check", "\n" "Probe window bits are::\n\n" - "\t" "'-'" ": " "Could not connect" "\n" - "\t" "'4'" ": " "Good IPv4" "\n" - "\t" "'6'" ": " "Good IPv6" "\n" - "\t" "'U'" ": " "Good UNIX" "\n" - "\t" "'x'" ": " "Error Xmit" "\n" - "\t" "'X'" ": " "Good Xmit" "\n" - "\t" "'r'" ": " "Error Recv" "\n" - "\t" "'R'" ": " "Good Recv" "\n" - "\t" "'H'" ": " "Happy" "\n" + "\t'-': Could not connect\n" + "\t'4': Good IPv4\n" + "\t'6': Good IPv6\n" + "\t'U': Good UNIX\n" + "\t'x': Error Xmit\n" + "\t'X': Good Xmit\n" + "\t'r': Error Recv\n" + "\t'R': Good Recv\n" + "\t'H': Happy\n" "\n" "When the backend is just created, the window bits for health check\n" "slots that haven't run yet appear as '-' like failures to connect.\n" From dridi.boukelmoune at gmail.com Tue Oct 27 09:36:15 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 27 Oct 2020 09:36:15 +0000 (UTC) Subject: [master] 79b38482e vsl: Document Backend_health '-' window bit Message-ID: <20201027093615.5C5861113DE@lists.varnish-cache.org> commit 79b38482e78e95907eba7043d2162d9201f3ea1a Author: Dridi Boukelmoune Date: Tue Oct 27 10:25:59 2020 +0100 vsl: Document Backend_health '-' window bit diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 9a14d0d58..33ee14be4 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -455,6 +455,7 @@ SLTM(Backend_health, 0, "Backend health check", "\n" "Probe window bits are::\n\n" + "\t" "'-'" ": " "Could not connect" "\n" "\t" "'4'" ": " "Good IPv4" "\n" "\t" "'6'" ": " "Good IPv6" "\n" "\t" "'U'" ": " "Good UNIX" "\n" @@ -464,6 +465,9 @@ SLTM(Backend_health, 0, "Backend health check", "\t" "'R'" ": " "Good Recv" "\n" "\t" "'H'" ": " "Happy" "\n" "\n" + "When the backend is just created, the window bits for health check\n" + "slots that haven't run yet appear as '-' like failures to connect.\n" + "\n" ) SLTM(VCL_Log, 0, "Log statement from VCL", From dridi.boukelmoune at gmail.com Tue Oct 27 18:11:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 27 Oct 2020 18:11:07 +0000 (UTC) Subject: [master] b33f0173d Revert "man: Reference sources from $(srcdir)" Message-ID: <20201027181107.58C7D7C1D@lists.varnish-cache.org> commit b33f0173d5b5b2a220796df8f864299a627089c6 Author: Dridi Boukelmoune Date: Thu Oct 22 17:52:07 2020 +0200 Revert "man: Reference sources from $(srcdir)" This reverts commit 89a558e56390d425c52732a6c94087eec9083115. I also added a note to avoid falling in the same trap again. diff --git a/man/Makefile.am b/man/Makefile.am index 641a384f1..5c6ebae88 100644 --- a/man/Makefile.am +++ b/man/Makefile.am @@ -30,70 +30,75 @@ RST2ANY_FLAGS = --halt=2 BUILD_MAN = $(AM_V_GEN) $(RST2MAN) $(RST2ANY_FLAGS) -varnish-cli.7: $(top_srcdir)/doc/sphinx/reference/varnish-cli.rst \ +# NB: everything happens in $(builddir) on purpose, even though some +# RST files are always present in $(srcdir). Links from $(builddir) +# to $(srcdir) are created for sphinx builds that don't support +# VPATH. + +varnish-cli.7: $(top_builddir)/doc/sphinx/reference/varnish-cli.rst \ $(top_builddir)/doc/sphinx/include/cli.rst - $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnish-cli.rst $@ + $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnish-cli.rst $@ -varnish-counters.7: $(top_srcdir)/doc/sphinx/reference/varnish-counters.rst \ +varnish-counters.7: $(top_builddir)/doc/sphinx/reference/varnish-counters.rst \ $(top_builddir)/doc/sphinx/include/counters.rst - $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnish-counters.rst $@ + $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnish-counters.rst $@ -vcl.7: $(top_srcdir)/doc/sphinx/reference/vcl.rst \ - $(top_srcdir)/doc/sphinx/reference/vcl_var.rst \ +vcl.7: $(top_builddir)/doc/sphinx/reference/vcl.rst \ + $(top_builddir)/doc/sphinx/reference/vcl_var.rst \ $(top_srcdir)/bin/varnishd/builtin.vcl - $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/vcl.rst $@ + $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/vcl.rst $@ -vsl.7: $(top_srcdir)/doc/sphinx/reference/vsl.rst \ +vsl.7: $(top_builddir)/doc/sphinx/reference/vsl.rst \ $(top_builddir)/doc/sphinx/include/vsl-tags.rst - $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/vsl.rst $@ + $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/vsl.rst $@ -vsl-query.7: $(top_srcdir)/doc/sphinx/reference/vsl-query.rst - $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/vsl-query.rst $@ +vsl-query.7: $(top_builddir)/doc/sphinx/reference/vsl-query.rst + $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/vsl-query.rst $@ -varnishadm.1: $(top_srcdir)/doc/sphinx/reference/varnishadm.rst - $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishadm.rst $@ +varnishadm.1: $(top_builddir)/doc/sphinx/reference/varnishadm.rst + $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishadm.rst $@ varnishd.1: \ - $(top_srcdir)/doc/sphinx/reference/varnishd.rst \ + $(top_builddir)/doc/sphinx/reference/varnishd.rst \ $(top_builddir)/doc/sphinx/include/params.rst - $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishd.rst $@ + $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishd.rst $@ varnishncsa.1: \ - $(top_srcdir)/doc/sphinx/reference/varnishncsa.rst \ + $(top_builddir)/doc/sphinx/reference/varnishncsa.rst \ $(top_builddir)/doc/sphinx/include/varnishncsa_options.rst \ $(top_builddir)/doc/sphinx/include/varnishncsa_synopsis.rst - $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishncsa.rst $@ + $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishncsa.rst $@ varnishlog.1: \ - $(top_srcdir)/doc/sphinx/reference/varnishlog.rst \ + $(top_builddir)/doc/sphinx/reference/varnishlog.rst \ $(top_builddir)/doc/sphinx/include/varnishlog_options.rst \ $(top_builddir)/doc/sphinx/include/varnishlog_synopsis.rst - $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishlog.rst $@ + $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishlog.rst $@ -varnishstat.1: $(top_srcdir)/doc/sphinx/reference/varnishstat.rst \ +varnishstat.1: $(top_builddir)/doc/sphinx/reference/varnishstat.rst \ $(top_builddir)/doc/sphinx/include/varnishstat_options.rst \ $(top_builddir)/doc/sphinx/include/varnishstat_synopsis.rst \ $(top_builddir)/doc/sphinx/include/varnishstat_bindings.rst - $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishstat.rst $@ + $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishstat.rst $@ -varnishtest.1: $(top_srcdir)/doc/sphinx/reference/varnishtest.rst - $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishtest.rst $@ +varnishtest.1: $(top_builddir)/doc/sphinx/reference/varnishtest.rst + $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishtest.rst $@ -vtc.7: $(top_srcdir)/doc/sphinx/reference/vtc.rst \ +vtc.7: $(top_builddir)/doc/sphinx/reference/vtc.rst \ $(top_builddir)/doc/sphinx/include/vtc-syntax.rst - $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/vtc.rst $@ + $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/vtc.rst $@ varnishtop.1: \ - $(top_srcdir)/doc/sphinx/reference/varnishtop.rst \ + $(top_builddir)/doc/sphinx/reference/varnishtop.rst \ $(top_builddir)/doc/sphinx/include/varnishtop_options.rst \ $(top_builddir)/doc/sphinx/include/varnishtop_synopsis.rst - $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishtop.rst $@ + $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishtop.rst $@ varnishhist.1: \ - $(top_srcdir)/doc/sphinx/reference/varnishhist.rst \ + $(top_builddir)/doc/sphinx/reference/varnishhist.rst \ $(top_builddir)/doc/sphinx/include/varnishhist_options.rst \ $(top_builddir)/doc/sphinx/include/varnishhist_synopsis.rst - $(BUILD_MAN) $(top_srcdir)/doc/sphinx/reference/varnishhist.rst $@ + $(BUILD_MAN) $(top_builddir)/doc/sphinx/reference/varnishhist.rst $@ vmod_cookie.3: $(top_builddir)/lib/libvmod_cookie/vmod_cookie.man.rst $(BUILD_MAN) $? $@ From dridi.boukelmoune at gmail.com Tue Oct 27 18:11:07 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 27 Oct 2020 18:11:07 +0000 (UTC) Subject: [master] 4232439a7 sphinx: Close link race Message-ID: <20201027181107.77CDB7C20@lists.varnish-cache.org> commit 4232439a76ff1a16d58ed73926145f907db14af5 Author: Dridi Boukelmoune Date: Thu Oct 22 18:44:19 2020 +0200 sphinx: Close link race We can't start linking files from $(srcdir) to $(builddir) until we are done populating files in $(builddir). Make sure we defer linking until after all sphinx sources are ready. diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index b5cf56dc5..02a691cb9 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -20,7 +20,7 @@ clean: # use index.rst as an indicator if we have copied already .PHONY: link_srcdir -link_srcdir: +link_srcdir: graphviz conf.py $(BUILT_SOURCES) if test "x$(srcdir)" != "x$(builddir)" && test ! -f index.rst; then \ s=`realpath $(srcdir)`; \ for f in `cd $$s && find . -type f`; do \ @@ -35,7 +35,7 @@ link_srcdir: graphviz: cd ../graphviz && $(MAKE) html -sphinx_prereq: link_srcdir graphviz conf.py +sphinx_prereq: link_srcdir all: link_srcdir From phk at FreeBSD.org Tue Oct 27 22:18:08 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 27 Oct 2020 22:18:08 +0000 (UTC) Subject: [master] 99016d84b Move pan_vdp() to VDP_Panic() Message-ID: <20201027221808.12A0D1007BF@lists.varnish-cache.org> commit 99016d84b8080d55e3c5ce02122aff56b52649ef Author: Poul-Henning Kamp Date: Tue Oct 27 17:31:55 2020 +0000 Move pan_vdp() to VDP_Panic() diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 076532bcf..1b871bd49 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -34,6 +34,33 @@ #include "cache_varnishd.h" #include "cache_filter.h" +void +VDP_Panic(struct vsb *vsb, const struct vdp_ctx *vdc) +{ + struct vdp_entry *vde; + + VSB_printf(vsb, "vdc = %p {\n", vdc); + VSB_indent(vsb, 2); + PAN_CheckMagic(vsb, vdc, VDP_CTX_MAGIC); + VSB_printf(vsb, "nxt = %p,\n", vdc->nxt); + VSB_printf(vsb, "retval = %d,\n", vdc->retval); + + if (!VTAILQ_EMPTY(&vdc->vdp)) { + VSB_cat(vsb, "filters = {\n"); + VSB_indent(vsb, 2); + VTAILQ_FOREACH(vde, &vdc->vdp, list) + VSB_printf(vsb, "%s = %p { priv = %p }\n", + vde->vdp->name, vde, vde->priv); + VSB_indent(vsb, -2); + VSB_cat(vsb, "},\n"); + } + + VSB_indent(vsb, -2); + VSB_cat(vsb, "},\n"); +} + + + void VDP_Init(struct vdp_ctx *vdc, struct worker *wrk, struct vsl_log *vsl, struct req *req) diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index abd637e2f..c0443ebed 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -344,31 +344,6 @@ pan_vfp(struct vsb *vsb, const struct vfp_ctx *vfc) VSB_cat(vsb, "},\n"); } -static void -pan_vdp(struct vsb *vsb, const struct vdp_ctx *vdc) -{ - struct vdp_entry *vde; - - VSB_printf(vsb, "vdc = %p {\n", vdc); - VSB_indent(vsb, 2); - PAN_CheckMagic(vsb, vdc, VDP_CTX_MAGIC); - VSB_printf(vsb, "nxt = %p,\n", vdc->nxt); - VSB_printf(vsb, "retval = %d,\n", vdc->retval); - - if (!VTAILQ_EMPTY(&vdc->vdp)) { - VSB_cat(vsb, "filters = {\n"); - VSB_indent(vsb, 2); - VTAILQ_FOREACH(vde, &vdc->vdp, list) - VSB_printf(vsb, "%s = %p { priv = %p }\n", - vde->vdp->name, vde, vde->priv); - VSB_indent(vsb, -2); - VSB_cat(vsb, "},\n"); - } - - VSB_indent(vsb, -2); - VSB_cat(vsb, "},\n"); -} - static void pan_busyobj(struct vsb *vsb, const struct busyobj *bo) { @@ -502,7 +477,7 @@ pan_req(struct vsb *vsb, const struct req *req) if (req->resp != NULL && req->resp->ws != NULL) pan_http(vsb, "resp", req->resp); if (req->vdc != NULL) - pan_vdp(vsb, req->vdc); + VDP_Panic(vsb, req->vdc); VCL_Panic(vsb, "vcl", req->vcl); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 385c38e0d..6446699e2 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -173,6 +173,9 @@ void VDI_Panic(const struct director *, struct vsb *, const char *nm); void VDI_Event(const struct director *d, enum vcl_event_e ev); void VDI_Init(void); +/* cache_deliver_proc.c */ +void VDP_Panic(struct vsb *vsb, const struct vdp_ctx *vdc); + /* cache_exp.c */ vtim_real EXP_Ttl(const struct req *, const struct objcore *); vtim_real EXP_Ttl_grace(const struct req *, const struct objcore *oc); From phk at FreeBSD.org Tue Oct 27 22:18:08 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 27 Oct 2020 22:18:08 +0000 (UTC) Subject: [master] 49641a8d1 Stash more internal stuff away (re: #2800) Message-ID: <20201027221808.2C3701007C2@lists.varnish-cache.org> commit 49641a8d1c26f971b0e09f2ae26b41270213c251 Author: Poul-Henning Kamp Date: Tue Oct 27 22:16:52 2020 +0000 Stash more internal stuff away (re: #2800) diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index e3b3cc613..ff07ac563 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -32,6 +32,7 @@ struct req; struct vfp_entry; struct vfp_ctx; +struct vdp_ctx; /* Fetch processors --------------------------------------------------*/ @@ -87,7 +88,6 @@ struct vfp_ctx { unsigned obj_flags; }; -struct vfp_entry *VFP_Push(struct vfp_ctx *, const struct vfp *); enum vfp_status VFP_Suck(struct vfp_ctx *, void *p, ssize_t *lp); enum vfp_status VFP_Error(struct vfp_ctx *, const char *fmt, ...) v_printflike_(2, 3); @@ -110,8 +110,6 @@ typedef int vdp_init_f(struct vdp_ctx *, void **priv); * positive: Don't push this VDP anyway */ -struct vdp_ctx; - typedef int vdp_fini_f(struct vdp_ctx *, void **priv); typedef int vdp_bytes_f(struct vdp_ctx *, enum vdp_action, void **priv, const void *ptr, ssize_t len); @@ -148,17 +146,6 @@ struct vdp_ctx { struct req *req; }; -extern const struct vdp VDP_gunzip; -extern const struct vdp VDP_esi; -extern const struct vdp VDP_range; - -void VDP_Init(struct vdp_ctx *vdx, struct worker *wrk, struct vsl_log *vsl, - struct req *req); - -uint64_t VDP_Close(struct vdp_ctx *); -int VDP_DeliverObj(struct vdp_ctx *vdc, struct objcore *oc); - -int VDP_bytes(struct vdp_ctx *, enum vdp_action act, const void *ptr, ssize_t len); -int VDP_Push(struct vdp_ctx *, struct ws *, const struct vdp *, void *priv); +int VDP_bytes(struct vdp_ctx *, enum vdp_action act, const void *, ssize_t); void VRT_AddVDP(VRT_CTX, const struct vdp *); void VRT_RemoveVDP(VRT_CTX, const struct vdp *); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 6446699e2..835e74d7e 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -50,6 +50,7 @@ /*--------------------------------------------------------------------*/ struct vfp; +struct vdp; struct cli_proto; struct poolparam; @@ -174,7 +175,16 @@ void VDI_Event(const struct director *d, enum vcl_event_e ev); void VDI_Init(void); /* cache_deliver_proc.c */ +void VDP_Init(struct vdp_ctx *vdx, struct worker *wrk, struct vsl_log *vsl, + struct req *req); +uint64_t VDP_Close(struct vdp_ctx *); void VDP_Panic(struct vsb *vsb, const struct vdp_ctx *vdc); +int VDP_Push(struct vdp_ctx *, struct ws *, const struct vdp *, void *priv); +int VDP_DeliverObj(struct vdp_ctx *vdc, struct objcore *oc); +extern const struct vdp VDP_gunzip; +extern const struct vdp VDP_esi; +extern const struct vdp VDP_range; + /* cache_exp.c */ vtim_real EXP_Ttl(const struct req *, const struct objcore *); @@ -253,6 +263,7 @@ void Bereq_Rollback(struct busyobj *); /* cache_fetch_proc.c */ void VFP_Init(void); +struct vfp_entry *VFP_Push(struct vfp_ctx *, const struct vfp *); enum vfp_status VFP_GetStorage(struct vfp_ctx *, ssize_t *sz, uint8_t **ptr); void VFP_Extend(const struct vfp_ctx *, ssize_t sz, enum vfp_status); void VFP_Setup(struct vfp_ctx *vc, struct worker *wrk); From dridi.boukelmoune at gmail.com Wed Oct 28 08:03:10 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 28 Oct 2020 08:03:10 +0000 (UTC) Subject: [master] c7ac52663 vtc: Add varnish macros for all listen addresses Message-ID: <20201028080310.A195110FCB5@lists.varnish-cache.org> commit c7ac5266377ed6171548fcd61cbf5f095f98fa41 Author: Dridi Boukelmoune Date: Wed Oct 28 08:31:30 2020 +0100 vtc: Add varnish macros for all listen addresses When a test case successfully runs this: varnish v1 -start The following macros are created: - ${v1_addr} - ${v1_port} - ${v1_sock} When v1 listens to a UNIX domain socket, those macros become useless and we generally have to repeat the socket path wherever we need it. From now on, varnish listen address macros for unix domain sockets match their server counterparts: - ${v1_addr} expands to "0.0.0.0" - ${v1_port} expands to "0" - ${v1_sock} expands to the socket path It is now possible to use the following construct regardless of the socket type: client c1 -connect "${v1_sock}" In addition, the debug.listen_address command now outputs the listen address name in addition to its endpoint. For example, in a test case with the following command: varnish v1 -arg "-a :0 -a :0 -b '${bad_backend}'" -start The output for debug.listen_address could look like this: a0 127.0.0.1 34567 a0 ::1 34567 a1 127.0.0.1 45678 a1 ::1 45678 The resulting macros would be: - ${v1_addr} - ${v1_port} - ${v1_sock} - ${v1_a0_addr} - ${v1_a0_port} - ${v1_a0_sock} - ${v1_a1_addr} - ${v1_a1_port} - ${v1_a1_sock} It becomes possible to reliably listen to multiple endpoints and use either in a test case. On a dual-stack system where we might end up with multiple endpoints sharing the same name, only the first one is used for a given name. In the example above all macros would expand to IPv4 addresses. diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index 77dbfaa43..0caabb819 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -673,10 +673,10 @@ ccf_listen_address(struct cli *cli, const char * const *av, void *priv) VTAILQ_FOREACH(ls, &heritage.socks, list) { if (!ls->uds) { VTCP_myname(ls->sock, h, sizeof h, p, sizeof p); - VCLI_Out(cli, "%s %s\n", h, p); + VCLI_Out(cli, "%s %s %s\n", ls->name, h, p); } else - VCLI_Out(cli, "%s -\n", ls->endpoint); + VCLI_Out(cli, "%s %s -\n", ls->name, ls->endpoint); } AZ(pthread_mutex_unlock(&shut_mtx)); } diff --git a/bin/varnishtest/tests/b00053.vtc b/bin/varnishtest/tests/b00053.vtc index e8cc9ab99..ea33a284a 100644 --- a/bin/varnishtest/tests/b00053.vtc +++ b/bin/varnishtest/tests/b00053.vtc @@ -5,7 +5,7 @@ server s1 -listen "${tmpdir}/s1.sock" { txresp -body "012345\n" } -start -varnish v1 -arg "-a ${tmpdir}/v1.sock" -vcl+backend { +varnish v1 -arg "-a ${tmpdir}/v1.sock -a ${localhost}:0" -vcl+backend { sub vcl_backend_response { set beresp.do_stream = false; } @@ -19,7 +19,7 @@ varnish v1 -expect sess_conn == 0 varnish v1 -expect client_req == 0 varnish v1 -expect cache_miss == 0 -client c1 -connect "${tmpdir}/v1.sock" { +client c1 -connect "${v1_sock}" { txreq -url "/" rxresp expect resp.status == 200 @@ -56,13 +56,25 @@ varnish v3 -vcl { set resp.http.addr = "${v1_addr}"; set resp.http.port = "${v1_port}"; set resp.http.sock = "${v1_sock}"; + set resp.http.a0_addr = "${v1_a0_addr}"; + set resp.http.a0_port = "${v1_a0_port}"; + set resp.http.a0_sock = "${v1_a0_sock}"; + set resp.http.a1_addr = "${v1_a1_addr}"; + set resp.http.a1_port = "${v1_a1_port}"; + set resp.http.a1_sock = "${v1_a1_sock}"; } } -start client c1 -connect ${v3_sock} { txreq rxresp - expect resp.http.addr == "${tmpdir}/v1.sock" - expect resp.http.port == "-" - expect resp.http.sock == "${tmpdir}/v1.sock -" + expect resp.http.addr == "0.0.0.0" + expect resp.http.port == "0" + expect resp.http.sock == "${tmpdir}/v1.sock" + expect resp.http.a0_addr == "0.0.0.0" + expect resp.http.a0_port == "0" + expect resp.http.a0_sock == "${tmpdir}/v1.sock" + expect resp.http.a1_addr != "0.0.0.0" + expect resp.http.a1_port != "0" + expect resp.http.a1_sock ~ "[^ ]+ [^ ]+" } -run diff --git a/bin/varnishtest/tests/m00046.vtc b/bin/varnishtest/tests/m00046.vtc index 8d2e2fd61..8c276ce35 100644 --- a/bin/varnishtest/tests/m00046.vtc +++ b/bin/varnishtest/tests/m00046.vtc @@ -11,21 +11,21 @@ varnish v2 -arg "-a ${tmpdir}/v2.sock" -vcl { } sub vcl_synth { - set resp.http.v1_addr = std.ip("${v1_addr}", client.ip); + set resp.http.v1_addr = std.ip("${v1_sock}", client.ip); } } -start -client c1 -connect "${tmpdir}/v2.sock" { +client c1 -connect "${v2_sock}" { txreq rxresp expect resp.http.v1_addr == "0.0.0.0" } -run -varnish v2 -errvcl {Cannot convert to an IP address: '"${v1_addr}"'} { +varnish v2 -errvcl {Cannot convert to an IP address: '"${v1_sock}"'} { import std; sub vcl_recv { - set resp.http.v1_port = std.port("${v1_addr}"); + set resp.http.v1_port = std.port("${v1_sock}"); } } @@ -40,7 +40,7 @@ varnish v2 -vcl { } } -client c1 -connect "${tmpdir}/v2.sock" { +client c1 -connect "${v2_sock}" { txreq rxresp expect resp.status == 200 diff --git a/bin/varnishtest/tests/m00047.vtc b/bin/varnishtest/tests/m00047.vtc index e9c05b032..9298f9d7a 100644 --- a/bin/varnishtest/tests/m00047.vtc +++ b/bin/varnishtest/tests/m00047.vtc @@ -123,7 +123,7 @@ varnish v2 -arg "-a ${tmpdir}/v2.sock" -vcl+backend { # Check the log output for the response header values to see how this # worked on your platform. -client c2 -connect "${v2_addr}" { +client c2 -connect "${v2_sock}" { txreq rxresp expect resp.status ~ "^(200|503)$" diff --git a/bin/varnishtest/tests/r02702.vtc b/bin/varnishtest/tests/r02702.vtc index a68c9acb3..438efeaef 100644 --- a/bin/varnishtest/tests/r02702.vtc +++ b/bin/varnishtest/tests/r02702.vtc @@ -47,7 +47,7 @@ varnish v2 -arg "-a ${tmpdir}/v2.sock,PROXY,mode=0777" -vcl { varnish v3 -vcl { backend bp { - .path = "${v2_addr}"; + .path = "${v2_sock}"; .host_header = "v2"; .proxy_header = 2; diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 1df5dd93d..52447b11b 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -544,11 +544,69 @@ varnish_launch(struct varnish *v) * Start a Varnish */ +static void +varnish_listen(struct varnish *v, char *la) +{ + const char *a, *p, *n, *n2; + char m[64], s[256]; + unsigned first; + + n2 = ""; + first = 1; + + while (*la != '\0') { + n = la; + la = strchr(n, ' '); + AN(la); + *la = '\0'; + a = la + 1; + la = strchr(a, ' '); + AN(la); + *la = '\0'; + p = la + 1; + la = strchr(p, '\n'); + AN(la); + *la = '\0'; + la++; + + AN(*n); + AN(*a); + AN(*p); + + if (*p != '-') { + bprintf(s, "%s %s", a, p); + } else { + bprintf(s, "%s", a); + a = "0.0.0.0"; + p = "0"; + } + + if (first) { + vtc_log(v->vl, 2, "Listen on %s %s", a, p); + macro_def(v->vl, v->name, "addr", "%s", a); + macro_def(v->vl, v->name, "port", "%s", p); + macro_def(v->vl, v->name, "sock", "%s", s); + first = 0; + } + + if (!strcmp(n, n2)) + continue; + + bprintf(m, "%s_addr", n); + macro_def(v->vl, v->name, m, "%s", a); + bprintf(m, "%s_port", n); + macro_def(v->vl, v->name, m, "%s", p); + bprintf(m, "%s_sock", n); + macro_def(v->vl, v->name, m, "%s", s); + n2 = n; + } +} + static void varnish_start(struct varnish *v) { enum VCLI_status_e u; - char *resp = NULL, *h, *p; + char *resp = NULL; if (v->cli_fd < 0) varnish_launch(v); @@ -577,17 +635,7 @@ varnish_start(struct varnish *v) if (u != CLIS_OK) vtc_fatal(v->vl, "CLI debug.listen_address command failed: %u %s", u, resp); - h = resp; - p = strchr(h, '\n'); - if (p != NULL) - *p = '\0'; - p = strchr(h, ' '); - AN(p); - *p++ = '\0'; - vtc_log(v->vl, 2, "Listen on %s %s", h, p); - macro_def(v->vl, v->name, "addr", "%s", h); - macro_def(v->vl, v->name, "port", "%s", p); - macro_def(v->vl, v->name, "sock", "%s %s", h, p); + varnish_listen(v, resp); free(resp); /* Wait for vsl logging to get underway */ while (v->vsl_rec == 0) @@ -1018,6 +1066,12 @@ varnish_expect(const struct varnish *v, char * const *av) * \-start * Start the child process. * + * Once successfully started, the following macros are available for + * the default listen address: ``${vNAME_addr}``, ``${vNAME_port}`` + * and ``${vNAME_sock}``. Additional macros are available, including + * the listen address name for each address vNAME listens to, like for + * example: ``${vNAME_a0_addr}``. + * * \-stop * Stop the child process. * From dridi.boukelmoune at gmail.com Wed Oct 28 09:48:08 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 28 Oct 2020 09:48:08 +0000 (UTC) Subject: [master] 1b0a51cc1 test: Over-breeding is still a thing Message-ID: <20201028094808.236D9112473@lists.varnish-cache.org> commit 1b0a51cc1ff3890ea1ff1949fec7fd9297f3735e Author: Dridi Boukelmoune Date: Wed Oct 28 10:44:37 2020 +0100 test: Over-breeding is still a thing Spotted on CircleCI: ---- v1 Not true: MAIN.threads (26) == 10 (10) Refs #2942 diff --git a/bin/varnishtest/tests/v00006.vtc b/bin/varnishtest/tests/v00006.vtc index 66d3d989c..89bf9a619 100644 --- a/bin/varnishtest/tests/v00006.vtc +++ b/bin/varnishtest/tests/v00006.vtc @@ -23,10 +23,10 @@ varnish v1 \ # >thread_pool_timeout to allow for any over-bred workers to be kissed # to death. # NB: This is important for to avoid mis-ordering of the workers. - delay 11 -varnish v1 -expect MAIN.threads == 10 +# But all in all, the delay does not fully prevent over-breeding. +varnish v1 -expect MAIN.threads >= 10 client c1 { txreq -url "/bar" From dridi.boukelmoune at gmail.com Wed Oct 28 13:53:11 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 28 Oct 2020 13:53:11 +0000 (UTC) Subject: [master] fda02024b cocci: Remove redundant AN() checks after REPLACE() Message-ID: <20201028135311.1E055117FF1@lists.varnish-cache.org> commit fda02024b2040c0d3a1807a5ab201057c63afc6a Author: Dridi Boukelmoune Date: Wed Oct 28 14:51:13 2020 +0100 cocci: Remove redundant AN() checks after REPLACE() Except those present in varnishtest/vtest. diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c index a60efd918..8171983d1 100644 --- a/bin/varnishd/common/common_vsmw.c +++ b/bin/varnishd/common/common_vsmw.c @@ -246,7 +246,6 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg) } else { vsmw_mkent(vsmw, vsmw->idx); REPLACE(t, VSB_data(vsmw->vsb)); - AN(t); fd = openat(vsmw->vdirfd, t, O_WRONLY|O_CREAT|O_EXCL, vsmw->mode); assert(fd >= 0); diff --git a/bin/varnishd/mgt/mgt_symtab.c b/bin/varnishd/mgt/mgt_symtab.c index d4666328a..530c89d17 100644 --- a/bin/varnishd/mgt/mgt_symtab.c +++ b/bin/varnishd/mgt/mgt_symtab.c @@ -137,7 +137,6 @@ mgt_vcl_import_vmod(struct vclprog *vp, const struct vjsn_val *vv) ALLOC_OBJ(vf, VMODFILE_MAGIC); AN(vf); REPLACE(vf->fname, v_dst); - AN(vf->fname); VTAILQ_INIT(&vf->vcls); AZ(mgt_vcl_cache_vmod(v_name, v_file, v_dst)); VTAILQ_INSERT_TAIL(&vmodhead, vf, list); diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 084cf4e70..520ea6dfb 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -1122,7 +1122,6 @@ main(int argc, char * const *argv) if (format != NULL) VUT_Error(vut, 1, "Format already set"); REPLACE(format, optarg); - AN(format); break; case 'f': if (format != NULL) diff --git a/tools/coccinelle/replace.cocci b/tools/coccinelle/replace.cocci index f70b7aeb8..706b28482 100644 --- a/tools/coccinelle/replace.cocci +++ b/tools/coccinelle/replace.cocci @@ -10,3 +10,10 @@ expression ptr, val; - ptr = strdup(val); - AN(ptr); + REPLACE(ptr, val); + +@@ +expression ptr, val; +@@ + +REPLACE(ptr, val); +- AN(ptr); From dridi.boukelmoune at gmail.com Wed Oct 28 15:27:09 2020 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 28 Oct 2020 15:27:09 +0000 (UTC) Subject: [master] 1b606ff94 ci: Increase parallelism in Travis CI builds Message-ID: <20201028152709.B7DD611A473@lists.varnish-cache.org> commit 1b606ff940309d3ba62a392ac3050902aa740b8c Author: Dridi Boukelmoune Date: Wed Oct 28 16:21:38 2020 +0100 ci: Increase parallelism in Travis CI builds They are taking too long, needlessly holding onto job slots forever. Align with CircleCI for now, even though we could probably do more. diff --git a/.travis.yml b/.travis.yml index 7175105fd..90ce54d2f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,9 +28,9 @@ jobs: script: &script-common - | if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then - make -j8 distcheck VERBOSE=1 + make -j12 distcheck VERBOSE=1 else - make -j8 check VERBOSE=1 + make -j16 check VERBOSE=1 fi - <<: *test-linux arch: arm64 @@ -47,7 +47,7 @@ jobs: - ./configure --enable-maintainer-mode - <<: *test-linux env: WITNESS=1 - script: make -j8 witness VERBOSE=1 + script: make -j16 witness VERBOSE=1 - <<: *test-linux stage: sanitizers addons: From dridi at varni.sh Wed Oct 28 19:40:18 2020 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 28 Oct 2020 19:40:18 +0000 Subject: [master] 1b606ff94 ci: Increase parallelism in Travis CI builds In-Reply-To: <20201028152709.B7DD611A473@lists.varnish-cache.org> References: <20201028152709.B7DD611A473@lists.varnish-cache.org> Message-ID: On Wed, Oct 28, 2020 at 3:27 PM Dridi Boukelmoune wrote: > > > commit 1b606ff940309d3ba62a392ac3050902aa740b8c > Author: Dridi Boukelmoune > Date: Wed Oct 28 16:21:38 2020 +0100 > > ci: Increase parallelism in Travis CI builds > > They are taking too long, needlessly holding onto job slots forever. We saved 10 minutes per build (a couple minutes per job). But I think most of the time is wasted waiting for and booting VMs instead of container builds... From dridi at varni.sh Wed Oct 28 22:10:40 2020 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 28 Oct 2020 22:10:40 +0000 Subject: [master] 1b606ff94 ci: Increase parallelism in Travis CI builds In-Reply-To: References: <20201028152709.B7DD611A473@lists.varnish-cache.org> Message-ID: > We saved 10 minutes per build (a couple minutes per job). But I think > most of the time is wasted waiting for and booting VMs instead of > container builds... I was a bit behind on travis build environments and got mentally stuck to the old "historical" container setup: https://docs.travis-ci.com/user/reference/overview/#deprecated-virtualization-environments Too my surprise nowadays x86_64 systems are the only ones that don't containerize on Travis CI and I'm wondering whether we shouldn't instead use all the other architectures, especially those that we don't get with Circle CI that I occasionally test using Fedora's infrastructure: https://docs.travis-ci.com/user/reference/overview/#which-one-do-i-use Except for the macos and sanitizer jobs, we could run "normal" jobs in aarch64, ppc64le and s390x containers with a faster spin up, but possibly slower execution speed (at least that's how aarch64 looks today). Thoughts? From phk at phk.freebsd.dk Wed Oct 28 22:17:52 2020 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Wed, 28 Oct 2020 22:17:52 +0000 Subject: [master] 1b606ff94 ci: Increase parallelism in Travis CI builds In-Reply-To: References: <20201028152709.B7DD611A473@lists.varnish-cache.org> Message-ID: <92388.1603923472@critter.freebsd.dk> -------- Dridi Boukelmoune writes: > https://docs.travis-ci.com/user/reference/overview/#which-one-do-i-use > > Except for the macos and sanitizer jobs, we could run "normal" jobs in > aarch64, ppc64le and s390x containers with a faster spin up, but > possibly slower execution speed (at least that's how aarch64 looks > today). > > Thoughts? The more the merrier. ARM64 is going to be much more important now that Amazon rolled it out and Apple ships laptops with it. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From phk at FreeBSD.org Wed Oct 28 22:30:13 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 28 Oct 2020 22:30:13 +0000 (UTC) Subject: [master] 541ca1338 Test correct sess_close reason on range error Message-ID: <20201028223013.C2AFE6E568@lists.varnish-cache.org> commit 541ca1338268a0337685410f7e73d6658f5159e0 Author: Poul-Henning Kamp Date: Wed Oct 28 19:28:50 2020 +0000 Test correct sess_close reason on range error diff --git a/bin/varnishtest/tests/r02258.vtc b/bin/varnishtest/tests/r02258.vtc index 707d4b637..097682c88 100644 --- a/bin/varnishtest/tests/r02258.vtc +++ b/bin/varnishtest/tests/r02258.vtc @@ -48,4 +48,4 @@ client c1 { } -run } -run - +varnish v1 -expect MAIN.sc_range_short == 1 From phk at FreeBSD.org Wed Oct 28 22:30:13 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 28 Oct 2020 22:30:13 +0000 (UTC) Subject: [master] 178bb20e9 Use correct req-pointer, and dont insist on it, if we dont need it. Message-ID: <20201028223013.CA9006E569@lists.varnish-cache.org> commit 178bb20e98048b3f33ebdbd600fb2ac0fee2999d Author: Poul-Henning Kamp Date: Wed Oct 28 19:47:32 2020 +0000 Use correct req-pointer, and dont insist on it, if we dont need it. diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 6179b7bcc..c34f77302 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -321,7 +321,7 @@ ved_vdp_esi_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, while (1) { switch (ecx->state) { case 0: - ecx->p = ObjGetAttr(vdx->wrk, vdx->req->objcore, + ecx->p = ObjGetAttr(vdx->wrk, ecx->preq->objcore, OA_ESIDATA, &l); AN(ecx->p); assert(l > 0); @@ -386,7 +386,7 @@ ved_vdp_esi_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, break; } Debug("INCL [%s][%s] BEGIN\n", q, ecx->p); - ved_include(vdx->req, + ved_include(ecx->preq, (const char*)q, (const char*)ecx->p, ecx); Debug("INCL [%s][%s] END\n", q, ecx->p); ecx->p = r + 1; @@ -520,7 +520,6 @@ ved_pretend_gzip_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, struct ecx *ecx; CHECK_OBJ_NOTNULL(vdx, VDP_CTX_MAGIC); - CHECK_OBJ_NOTNULL(vdx->req, REQ_MAGIC); CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC); (void)priv; From phk at FreeBSD.org Wed Oct 28 22:30:13 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 28 Oct 2020 22:30:13 +0000 (UTC) Subject: [master] c7bc8b659 Make vdp_range cache req from ->init and NULL vdp_ctx->req during data-phase Message-ID: <20201028223013.E94946E56D@lists.varnish-cache.org> commit c7bc8b659c335df65afeaf323a19fdf39f2338d6 Author: Poul-Henning Kamp Date: Wed Oct 28 22:29:07 2020 +0000 Make vdp_range cache req from ->init and NULL vdp_ctx->req during data-phase diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 1b871bd49..6370ffe28 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -224,6 +224,7 @@ int VDP_DeliverObj(struct vdp_ctx *vdc, struct objcore *oc) CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); CHECK_OBJ_NOTNULL(vdc->wrk, WORKER_MAGIC); AN(vdc->vsl); + vdc->req = NULL; final = oc->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP) ? 1 : 0; r = ObjIterate(vdc->wrk, oc, vdc, vdp_objiterator, final); if (r == 0) diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index 016a95f90..b705cd526 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -42,6 +42,7 @@ struct vrg_priv { unsigned magic; #define VRG_PRIV_MAGIC 0xb886e711 + struct req *req; ssize_t range_low; ssize_t range_high; ssize_t range_off; @@ -53,10 +54,9 @@ vrg_range_fini(struct vdp_ctx *vdc, void **priv) struct vrg_priv *vrg_priv; CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); - CHECK_OBJ_NOTNULL(vdc->req, REQ_MAGIC); CAST_OBJ_NOTNULL(vrg_priv, *priv, VRG_PRIV_MAGIC); if (vrg_priv->range_off < vrg_priv->range_high) - Req_Fail(vdc->req, SC_RANGE_SHORT); + Req_Fail(vrg_priv->req, SC_RANGE_SHORT); *priv = NULL; /* struct on ws, no need to free */ return (0); } @@ -177,6 +177,7 @@ vrg_dorange(struct req *req, const char *r, void **priv) return ("WS too small"); INIT_OBJ(vrg_priv, VRG_PRIV_MAGIC); + vrg_priv->req = req; vrg_priv->range_off = 0; vrg_priv->range_low = low; vrg_priv->range_high = high + 1; @@ -247,27 +248,29 @@ vrg_range_init(struct vdp_ctx *vdc, void **priv) { const char *r; const char *err; + struct req *req; CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); - CHECK_OBJ_NOTNULL(vdc->req, REQ_MAGIC); - assert(http_GetHdr(vdc->req->http, H_Range, &r)); - if (!vrg_ifrange(vdc->req)) // rfc7233,l,455,456 + req = vdc->req; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + assert(http_GetHdr(req->http, H_Range, &r)); + if (!vrg_ifrange(req)) // rfc7233,l,455,456 return (1); - err = vrg_dorange(vdc->req, r, priv); + err = vrg_dorange(req, r, priv); if (err == NULL) return (*priv == NULL ? 1 : 0); VSLb(vdc->vsl, SLT_Debug, "RANGE_FAIL %s", err); - if (vdc->req->resp_len >= 0) - http_PrintfHeader(vdc->req->resp, + if (req->resp_len >= 0) + http_PrintfHeader(req->resp, "Content-Range: bytes */%jd", - (intmax_t)vdc->req->resp_len); - http_PutResponse(vdc->req->resp, "HTTP/1.1", 416, NULL); + (intmax_t)req->resp_len); + http_PutResponse(req->resp, "HTTP/1.1", 416, NULL); /* * XXX: We ought to produce a body explaining things. * XXX: That really calls for us to hit vcl_synth{} */ - vdc->req->resp_len = 0; + req->resp_len = 0; return (1); } From phk at FreeBSD.org Sat Oct 31 09:44:13 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 31 Oct 2020 09:44:13 +0000 (UTC) Subject: [6.1] f326a87ac Make this compile with modern CLANG Message-ID: <20201031094413.6101510EC2D@lists.varnish-cache.org> commit f326a87ac037d1d3090263a314453ef4cb2f5268 Author: Poul-Henning Kamp Date: Sat Oct 31 09:43:18 2020 +0000 Make this compile with modern CLANG diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index ba10fa513..648e3e71e 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -70,7 +70,7 @@ vmod_integer(VRT_CTX, VCL_STRING p, VCL_INT i) return (i); r = trunc(r); - if (r > LONG_MAX || r < LONG_MIN) + if (r > (double)LONG_MAX || r < (double)LONG_MIN) return (i); return ((VCL_INT)r); @@ -147,7 +147,7 @@ vmod_real2integer(VRT_CTX, VCL_REAL r, VCL_INT i) if (!isfinite(r)) return (i); r = round(r); - if (r > LONG_MAX || r < LONG_MIN) + if (r > (double)LONG_MAX || r < (double)LONG_MIN) return(i); return ((VCL_INT)r); } @@ -171,7 +171,7 @@ vmod_time2integer(VRT_CTX, VCL_TIME t, VCL_INT i) if (!isfinite(t)) return (i); t = round(t); - if (t > LONG_MAX || t < LONG_MIN) + if (t > (double)LONG_MAX || t < (double)LONG_MIN) return(i); return ((VCL_INT)t); } From phk at FreeBSD.org Sat Oct 31 09:45:08 2020 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 31 Oct 2020 09:45:08 +0000 (UTC) Subject: [6.0] 29ce1958f Make this compile with modern CLANG Message-ID: <20201031094508.4EB8810ED89@lists.varnish-cache.org> commit 29ce1958f7137db1463378c8079253a86c4090b4 Author: Poul-Henning Kamp Date: Sat Oct 31 09:44:12 2020 +0000 Make this compile with modern CLANG diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index a30086106..0bfa0ad7f 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -71,7 +71,7 @@ vmod_integer(VRT_CTX, VCL_STRING p, VCL_INT i) return (i); r = trunc(r); - if (r > LONG_MAX || r < LONG_MIN) + if (r > (double)LONG_MAX || r < (double)LONG_MIN) return (i); return ((VCL_INT)r); @@ -130,7 +130,7 @@ vmod_real2integer(VRT_CTX, VCL_REAL r, VCL_INT i) if (!isfinite(r)) return (i); r = round(r); - if (r > LONG_MAX || r < LONG_MIN) + if (r > (double)LONG_MAX || r < (double)LONG_MIN) return(i); return ((VCL_INT)r); } @@ -154,7 +154,7 @@ vmod_time2integer(VRT_CTX, VCL_TIME t, VCL_INT i) if (!isfinite(t)) return (i); t = round(t); - if (t > LONG_MAX || t < LONG_MIN) + if (t > (double)LONG_MAX || t < (double)LONG_MIN) return(i); return ((VCL_INT)t); }