From phk at FreeBSD.org Wed Jan 2 12:38:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 2 Jan 2019 12:38:07 +0000 (UTC) Subject: [master] ae99e82dd Initialize ESIs from the correct header set Message-ID: <20190102123807.52F889C896@lists.varnish-cache.org> commit ae99e82dd2992f4ce2bedf812acb26a98c24b55b Author: Dag Haavi Finstad Date: Mon Dec 17 13:08:53 2018 +0100 Initialize ESIs from the correct header set Ensure ESI children are set up from preq->http0 diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 5642f31e1..cbb54298d 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -141,7 +141,7 @@ ved_include(struct req *preq, const char *src, const char *host, req->top = preq->top; HTTP_Setup(req->http, req->ws, req->vsl, SLT_ReqMethod); - HTTP_Copy(req->http, preq->http); + HTTP_Copy(req->http, preq->http0); http_SetH(req->http, HTTP_HDR_URL, src); if (host != NULL && *host != '\0') { diff --git a/bin/varnishtest/tests/e00003.vtc b/bin/varnishtest/tests/e00003.vtc index a541596bf..0a1d657ad 100644 --- a/bin/varnishtest/tests/e00003.vtc +++ b/bin/varnishtest/tests/e00003.vtc @@ -3,6 +3,7 @@ varnishtest "ESI include" server s1 { rxreq + expect req.http.esi0 == "foo" txresp -body { Before include @@ -12,6 +13,7 @@ server s1 { } rxreq expect req.url == "/body1" + expect req.http.esi0 != "foo" txresp -body { Included file } @@ -21,6 +23,8 @@ varnish v1 -vcl+backend { sub vcl_recv { if (req.esi_level > 0) { set req.url = req.url + req.esi_level; + } else { + set req.http.esi0 = "foo"; } } sub vcl_backend_response { From hermunn at varnish-software.com Wed Jan 2 12:54:07 2019 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Wed, 2 Jan 2019 12:54:07 +0000 (UTC) Subject: [4.1] 49ad1b211 Fix a panic in VRB_Cache Message-ID: <20190102125407.905739CEED@lists.varnish-cache.org> commit 49ad1b2119eb984280b26551d702cf122fd9eb24 Author: P?l Hermunn Johansen Date: Wed Jan 2 13:44:23 2019 +0100 Fix a panic in VRB_Cache This adds error handling for STV_NewObject(.., TRANSIENT) in VRB_Cache, which would fail when transient is full. This is a back port of 6045eaaa6a2. Fixes: #2831 Conflicts: bin/varnishd/cache/cache_req_body.c bin/varnishtest/tests/r02831.vtc diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c index ccd73081d..90b27ea4f 100644 --- a/bin/varnishd/cache/cache_req_body.c +++ b/bin/varnishd/cache/cache_req_body.c @@ -239,7 +239,12 @@ VRB_Cache(struct req *req, ssize_t maxsize) req->body_oc = HSH_Private(req->wrk); AN(req->body_oc); - XXXAN(STV_NewObject(req->body_oc, req->wrk, TRANSIENT_STORAGE, 8)); + if (STV_NewObject(req->body_oc, req->wrk, TRANSIENT_STORAGE, 8) == 0) { + (void)VFP_Error(vfc, "Object allocation failed -" + " ran out of space in Transient"); + req->req_body_status = REQ_BODY_FAIL; + return (-1); + } vfc->http = req->http; vfc->oc = req->body_oc; diff --git a/bin/varnishtest/tests/r02831.vtc b/bin/varnishtest/tests/r02831.vtc new file mode 100644 index 000000000..29282b5d5 --- /dev/null +++ b/bin/varnishtest/tests/r02831.vtc @@ -0,0 +1,56 @@ +varnishtest "#2831: Out of storage in cache_req_body" + +server s1 { + rxreq + expect req.url == "/obj1" + txresp -bodylen 1048400 +} -start + +varnish v1 \ + -arg "-p nuke_limit=0" \ + -arg "-sTransient=malloc,1m" \ + -vcl+backend { + import ${vmod_std}; + sub vcl_recv { + if (req.method == "POST") { + std.cache_req_body(1KB); + } + } + sub vcl_backend_response { + set beresp.do_stream = false; + set beresp.storage_hint = "Transient"; + # Unset Date header to not change the object sizes + unset beresp.http.Date; + } +} -start + +varnish v1 -cliok "param.set debug +syncvsl" + +delay .1 + +client c1 { + # Fill transient + txreq -url "/obj1" + rxresp + expect resp.status == 200 +} -run + +delay .1 + +varnish v1 -expect SMA.Transient.g_bytes > 1048400 +varnish v1 -expect SMA.Transient.g_space < 100 + +client c1 { + # No space for caching this req.body + txreq -req "POST" -body "foobar" + delay 1 +} -run + +varnish v1 -expect SMA.Transient.c_fail == 1 + +client c1 { + # Check that Varnish is still alive + txreq -url "/obj1" + rxresp + expect resp.status == 200 +} -run diff --git a/doc/changes.rst b/doc/changes.rst index 5a4e7e2c1..b712f9bfc 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -1,3 +1,13 @@ +================================= +Varnish Cache 4.1.11 (unreleased) +================================= + +Changes since 4.1.10: + +* New counter added: cache_hit_grace (2831_) + +.. _2831: https://github.com/varnishcache/varnish-cache/issues/2831 + ================================= Varnish Cache 4.1.10 (2018-04-25) ================================= From phk at FreeBSD.org Thu Jan 3 00:46:06 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 3 Jan 2019 00:46:06 +0000 (UTC) Subject: [master] 0d5da11fb Do you feel lucky Punk ? Message-ID: <20190103004606.9CD54AD81A@lists.varnish-cache.org> commit 0d5da11fbea10ab04fa241502056f71b01330d3e Author: Poul-Henning Kamp Date: Thu Jan 3 00:44:48 2019 +0000 Do you feel lucky Punk ? diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst index 7962d9ea5..d258654d8 100644 --- a/doc/sphinx/phk/index.rst +++ b/doc/sphinx/phk/index.rst @@ -8,6 +8,7 @@ You may or may not want to know what Poul-Henning thinks. .. toctree:: :maxdepth: 1 + lucky.rst apispaces.rst VSV00001.rst somethinghappened.rst diff --git a/doc/sphinx/phk/lucky.rst b/doc/sphinx/phk/lucky.rst new file mode 100644 index 000000000..328f66e6a --- /dev/null +++ b/doc/sphinx/phk/lucky.rst @@ -0,0 +1,179 @@ +.. _phk_lucky: + +=================== +Do you feel lucky ? +=================== + +Whenever a corporate cotton-mouth says anything, their laywers +always make them add a footnote to the effect that *"Past performance +is not predictive of future results."* so they do not get sued for +being insufficiently clairvoyant. + +The lawyers are wrong. + +Past performance *is* predictive of future results: That is how +we determine the odds. + +Just never forget that the roll of the dice is still pure luck. + +Or as author Neil Gaiman said it in +`a commencement speech in 2012 `_: + + *?Often you will discover that the harder you work, + and the more wisely you work, the luckier you get. + But there is luck, and it helps.?* + +Approximately four million real websites use Varnish now and the +number seems to grow by half a million sites per year, as it has +been doing for the last 8 years. + +That took a lot of luck, and wisdom was probably also involved, but +mostly it was lot of hard work by a lot of people. + +Wisdom is a tricky thing, the hypothesized "older?wiser" correlation +is still in clinical testing and in the meantime Neil Gaiman suggests: + + *?So be wise, because the world needs more wisdom, and if you + cannot be wise, pretend to be someone who is wise, and then just + behave like they would.?* + +Works for me. + +---- +2018 +---- + +Despite sucking in pretty much any other aspect, 2018 was a good +year for Open Source Software in general and Varnish Cache in +particular. + +People have finally started to understand that Free does not mean +Gratis, and that quality software takes time and effort. + +From to the dot-com generation reinventing reproducible builds (like +we had then in the 1980'ies) to the EU setting up a pot +of `850M? bug-bounties +`_ [#f1]_, +things are moving in the right direction with respect to software +quality. + +In 2018 the Varnish Cache project had settled into our "March and +September 15th" release strategy, and released 6.0 in March and 6.1 +in September, as promised, and the next release will be out in ten +weeks. + +We also had no security issues, and we have managed to keep the +number of open issues and bug reports down. + +Writing it like that makes it sound boring, but with four million +web sites depending on Varnish, boring is good thing. + +No news is indeed good news. + +--------------- +2019 and HTTP/3 +--------------- + +The Next Big Thing in our world seems like it will be HTTP/3 ("The +protocol formerly known as QUIC"), and I suspect it will drive +much of our work in 2019. + +It is far too early to say anything about if, when or how, but I +do spend a lot of time with pencil and paper, pretending to be +somebody who is good at designing secure and efficient software. + +Around the time of the 2019-03-15 release we will gather for a VDD +(Varnish Developer Day), and the big topic there will be HTTP/3, +and then we will know and be ready to say something more detailed. + +I don't think it is realistic to roll out any kind of H3 support +in the september release, that release will probably only contain +a some of the necessary prepatory reorganization, so expect to +run production on 6.1 for a while. + +--------------------- +Varnish Moral License +--------------------- + +I want to thank the companies who have paid for a `Varnish +Moral License `_: + +* Fastly + +* Uplex + +* Varnish Software + +* Section.io + +* Globo + +The VML funding is why Varnish Cache is not on EU's hit-list and +why another half million websites who started using Varnish in +2018 will not regret it. + +Much appreciated! + +------- +ENOLUCK +------- + +For me 2018 ended on a sour note, when my dear friend `Jacob Sparre +Andersen `_ died from cancer a week +before christmas. + +Society as such knows how to deal with deaths, and all sorts of +procedures and rules kick in, to tie the loose ends up, respectfully +and properly. + +The Internet is not there yet, people on the Internet have only +just started dying, and there are not yet any automatic routines +or generally perceived procedures for informing the people and +communities who should know, or for tying up the loose ends, accounts, +repositories and memberships on the Internet. + +But deaths happen, and I can tell you from personal experience that +few things feel more awful, than having sent an email to somebody, +to receive the reply from their heartbroken spouse, that you are +many months too late [#f2]_. + +Jacob was not a major persona on the Internet, but between doing a +lot of interesting stuff as a multi-disicpline phd. in physics, +being a really good Ada programmer, a huge Lego enthusiast, an +incredibly helpful person *and* really *good* at helping, he had a +lot of friends in many corners of the Internet. + +Jacob knew what was coming, and being his usual helpful self, he +used the last few weeks to make a list of who to tell online, where +things were stored, what the passwords were, and he even appointed +a close friend to be his "digital executor", who will help his widow +sort all these things out in the coming months. + +When people die in our age-bracket, they usually do not get a few +weeks notice. If Jacob had been hit by a bus, his widow would have +have been stuck in an almost impossible digital situation, starting +with the need to guess, well, pretty much everything, including +the passwords. + +In honour of my helpful friend Jacob, and for the sake of your loved +ones, please sit down tonight, and write your own list of digital +who, what and where, including how to gain access to the necessary +passwords, and file it away in a way where it will be found, if +you run out of luck. + +Good luck! + +*phk* + +.. rubric:: Footnotes + +.. [#f1] I am not a big fan of bug-bounties, but I will grudingly admit + that wiser men than me, notably `Dan Geer + `_, have proposed that + tax-money be used to snatch the vulnerabilities up, before bad guys + get hold of them, and they seem to have a point. + +.. [#f2] And it does not feel any less awful if the loved ones + left behind tries to fill the blanks by asking you how you knew + each other and if you have any memories you could share with them. + From hermunn at varnish-software.com Thu Jan 3 09:19:06 2019 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Thu, 3 Jan 2019 09:19:06 +0000 (UTC) Subject: [4.1] 29c950457 Update changelog.rst for 4.1.11 Message-ID: <20190103091906.BE2B561E7C@lists.varnish-cache.org> commit 29c9504571ce90ff452aa54b57cf6991b7ea01a9 Author: P?l Hermunn Johansen Date: Wed Jan 2 17:20:00 2019 +0100 Update changelog.rst for 4.1.11 diff --git a/doc/changes.rst b/doc/changes.rst index b712f9bfc..629f27725 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -4,8 +4,28 @@ Varnish Cache 4.1.11 (unreleased) Changes since 4.1.10: +* Reintroduce the req.grace variable, change keep behavior. The use of + req.grace is documented on the same page as grace_ is documented. + +.. _grace: http://varnish-cache.org/docs/4.1/users-guide/vcl-grace.html + +Bugs fixed +---------- + +* On startup, tell what Varnish version this is (2661_) +* Ban lurker should back off on seeing a busy object (2681_) +* Fix http_resp_size documentation (2684_) +* Panic on return (retry) of a conditional fetch (2700_) +* Set the task arguments under the lock (2719_) +* Stabilize the test case b000064.vtc for real (2751_) * New counter added: cache_hit_grace (2831_) +.. _2661: https://github.com/varnishcache/varnish-cache/issues/2661 +.. _2681: https://github.com/varnishcache/varnish-cache/issues/2681 +.. _2684: https://github.com/varnishcache/varnish-cache/issues/2684 +.. _2700: https://github.com/varnishcache/varnish-cache/issues/2700 +.. _2719: https://github.com/varnishcache/varnish-cache/issues/2719 +.. _2751: https://github.com/varnishcache/varnish-cache/issues/2751 .. _2831: https://github.com/varnishcache/varnish-cache/issues/2831 ================================= From phk at FreeBSD.org Thu Jan 3 12:43:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 3 Jan 2019 12:43:07 +0000 (UTC) Subject: [master] 37fc9699c Improve function call argument error messages. Message-ID: <20190103124307.1557265B20@lists.varnish-cache.org> commit 37fc9699c885bcec3c4dbd824decf7911050d6bb Author: Poul-Henning Kamp Date: Thu Jan 3 12:40:05 2019 +0000 Improve function call argument error messages. Fixes: #2874 diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 780befa0c..b92781047 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -576,6 +576,10 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv, break; } vcc_do_arg(tl, fa); + if (tl->err) + VSB_printf(tl->sb, "Expected argument: %s %s\n\n", + fa->type->name, + fa->name ? fa->name : "(unnamed argument)"); ERRCHK(tl); if (tl->t->tok == ')') break; @@ -666,6 +670,8 @@ vcc_Eval_Func(struct vcc *tl, const struct vjsn_val *spec, struct expr *e = NULL; vcc_func(tl, &e, spec, extra, sym); + if (tl->err) + VSB_printf(tl->sb, "While compiling function call:\n"); ERRCHK(tl); vcc_expr_fmt(tl->fb, tl->indent, e); VSB_cat(tl->fb, ";\n"); @@ -752,6 +758,11 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt) AN(sym->eval); AZ(*e); sym->eval(tl, e, t, sym, fmt); + if (tl->err) { + VSB_printf(tl->sb, + "While compiling function call:\n\n"); + vcc_ErrWhere2(tl, t, tl->t); + } ERRCHK(tl); /* Unless asked for a HEADER, fold to string here */ if (*e && fmt != HEADER && (*e)->fmt == HEADER) { @@ -1372,6 +1383,7 @@ vcc_Act_Call(struct vcc *tl, struct token *t, struct symbol *sym) SkipToken(tl, ';'); VSB_cat(tl->fb, ";\n"); } else if (t != tl->t) { + VSB_printf(tl->sb, "While compiling function call:\n\n"); vcc_ErrWhere2(tl, t, tl->t); } vcc_delete_expr(e); From fgsch at lodoss.net Thu Jan 3 20:02:09 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 3 Jan 2019 20:02:09 +0000 (UTC) Subject: [master] c143dbee3 Enable some features based on the input Message-ID: <20190103200209.9A1B19834B@lists.varnish-cache.org> commit c143dbee3de9fee57789f7ae902e9282d6ddbc26 Author: Federico G. Schwindt Date: Thu Jan 3 20:01:37 2019 +0000 Enable some features based on the input diff --git a/bin/varnishd/fuzzers/esi_parse_fuzzer.c b/bin/varnishd/fuzzers/esi_parse_fuzzer.c index 972ffa0b1..34548b90f 100644 --- a/bin/varnishd/fuzzers/esi_parse_fuzzer.c +++ b/bin/varnishd/fuzzers/esi_parse_fuzzer.c @@ -94,8 +94,17 @@ LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) VSC_C_main = &__VSC_C_main; cache_param = &__cache_param; - /* Zero out the esi feature bits for now */ memset(&__cache_param, 0, sizeof(__cache_param)); +#define BSET(b, no) (b)[(no) >> 3] |= (0x80 >> ((no) & 7)) + if (data[0] & 0x8f) + BSET(__cache_param.feature_bits, FEATURE_ESI_IGNORE_HTTPS); + if (size > 1 && data[1] & 0x8f) + BSET(__cache_param.feature_bits, FEATURE_ESI_DISABLE_XML_CHECK); + if (size > 2 && data[2] & 0x8f) + BSET(__cache_param.feature_bits, FEATURE_ESI_IGNORE_OTHER_ELEMENTS); + if (size > 3 && data[3] & 0x8f) + BSET(__cache_param.feature_bits, FEATURE_ESI_REMOVE_BOM); +#undef BSET /* Setup req */ req.hd = hd; From fgsch at lodoss.net Thu Jan 3 20:16:06 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 3 Jan 2019 20:16:06 +0000 (UTC) Subject: [master] 42f3ee8c8 Field values are case insensitive Message-ID: <20190103201606.677469893B@lists.varnish-cache.org> commit 42f3ee8c8d1e7b646f60051f5b0282a6c59820b8 Author: Federico G. Schwindt Date: Thu Dec 27 17:14:37 2018 +0000 Field values are case insensitive Should address "HTTP cache must not use a cached response with Cache-Control: No-CaChE, even with max-age and Expires" from https://cache-tests.fyi. diff --git a/bin/varnishd/builtin.vcl b/bin/varnishd/builtin.vcl index a578a9c33..4835a231c 100644 --- a/bin/varnishd/builtin.vcl +++ b/bin/varnishd/builtin.vcl @@ -156,9 +156,9 @@ sub vcl_backend_response { return (deliver); } else if (beresp.ttl <= 0s || beresp.http.Set-Cookie || - beresp.http.Surrogate-control ~ "no-store" || + beresp.http.Surrogate-control ~ "(?i)no-store" || (!beresp.http.Surrogate-Control && - beresp.http.Cache-Control ~ "no-cache|no-store|private") || + beresp.http.Cache-Control ~ "(?i:no-cache|no-store|private)") || beresp.http.Vary == "*") { # Mark as "Hit-For-Miss" for the next 2 minutes set beresp.ttl = 120s; From fgsch at lodoss.net Thu Jan 3 20:16:06 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 3 Jan 2019 20:16:06 +0000 (UTC) Subject: [master] 2433f397b Ignore fields with invalid C-C values Message-ID: <20190103201606.7B3469893E@lists.varnish-cache.org> commit 2433f397be91ac02a4b16ef2dc651a002c1f6e17 Author: Federico G. Schwindt Date: Thu Dec 27 17:14:55 2018 +0000 Ignore fields with invalid C-C values Should address "HTTP cache must not reuse a response with an invalid Cache-Control: max-age (trailing alpha)" from https://cache-tests.fyi. diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index bd5ed88c6..e2bffa216 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -34,6 +34,7 @@ #include "cache_varnishd.h" #include "vtim.h" +#include "vct.h" /*-------------------------------------------------------------------- * TTL and Age calculation in Varnish @@ -61,6 +62,21 @@ * */ +static inline int +rfc2616_time(const char *p) +{ + char *ep; + int val; + if (*p == '-') + return (0); + val = strtoul(p, &ep, 10); + for (; *ep != '\0' && vct_issp(*ep); ep++) + continue; + if (*ep == '\0' || *ep == ',') + return (val); + return (0); +} + void RFC2616_Ttl(struct busyobj *bo, vtim_real now, vtim_real *t_origin, float *ttl, float *grace, float *keep) @@ -140,12 +156,7 @@ RFC2616_Ttl(struct busyobj *bo, vtim_real now, vtim_real *t_origin, if ((http_GetHdrField(hp, H_Cache_Control, "s-maxage", &p) || http_GetHdrField(hp, H_Cache_Control, "max-age", &p)) && p != NULL) { - - if (*p == '-') - max_age = 0; - else - max_age = strtoul(p, NULL, 0); - + max_age = rfc2616_time(p); *ttl = max_age; break; } @@ -190,11 +201,7 @@ RFC2616_Ttl(struct busyobj *bo, vtim_real now, vtim_real *t_origin, */ if (*ttl >= 0 && http_GetHdrField(hp, H_Cache_Control, "stale-while-revalidate", &p) && p != NULL) { - - if (*p == '-') - *grace = 0; - else - *grace = strtoul(p, NULL, 0); + *grace = rfc2616_time(p); } VSLb(bo->vsl, SLT_TTL, From fgsch at lodoss.net Thu Jan 3 20:16:06 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 3 Jan 2019 20:16:06 +0000 (UTC) Subject: [master] fb9619753 Test max-age, s-maxage and Expires Message-ID: <20190103201606.A2E3D98944@lists.varnish-cache.org> commit fb961975395a50abae3f37d2235ab37453e16ee8 Author: Federico G. Schwindt Date: Thu Jan 3 20:15:18 2019 +0000 Test max-age, s-maxage and Expires diff --git a/bin/varnishtest/tests/b00066.vtc b/bin/varnishtest/tests/b00066.vtc new file mode 100644 index 000000000..7eb286f4b --- /dev/null +++ b/bin/varnishtest/tests/b00066.vtc @@ -0,0 +1,102 @@ +varnishtest "Test CC:max-age, CC:s-maxage and Expires handling" + +server s1 { + rxreq + txresp -hdr "Cache-Control: max-age=-2" + rxreq + txresp -hdr "Cache-Control: max-age=a2" + rxreq + txresp -hdr "Cache-Control: max-age=2a" + rxreq + txresp -hdr "Cache-Control: s-maxage=-2" + rxreq + txresp -hdr "Cache-Control: s-maxage=a2" + rxreq + txresp -hdr "Cache-Control: s-maxage=2a" + rxreq + txresp -hdr "Expires: THU, 18 Aug 2050 02:01:18 GMT" + rxreq + txresp -hdr "Expires: Thu, 18 AUG 2050 02:01:18 GMT" + rxreq + txresp -hdr "Expires: Thu, 18 Aug 2050 02:01:18 gMT" + rxreq + txresp -hdr "Cache-Control: max-age=5, s-maxage=1" + rxreq + txresp -hdr "Cache-Control: s-maxage=2, max-age=5" + rxreq + txresp \ + -hdr "Cache-Control: max-age=5" \ + -hdr "Cache-Control: s-maxage=3" +} -start + +varnish v1 -arg "-pdefault_ttl=0" -vcl+backend { + sub vcl_backend_response { + set beresp.http.ttl = beresp.ttl; + set beresp.uncacheable = true; + } +} -start + +client c1 { + # negative max-age + txreq + rxresp + expect resp.http.ttl == 0.000 + + # invalid max-age - leading alpha + txreq + rxresp + expect resp.http.ttl == 0.000 + + # invalid max-age - trailing alpha + txreq + rxresp + expect resp.http.ttl == 0.000 + + # negative s-maxage + txreq + rxresp + expect resp.http.ttl == 0.000 + + # invalid s-maxage - leading alpha + txreq + rxresp + expect resp.http.ttl == 0.000 + + # invalid s-maxage - trailing alpha + txreq + rxresp + expect resp.http.ttl == 0.000 + + # Expires using wrong case (weekday) + txreq + rxresp + expect resp.http.ttl == 0.000 + + # Expires using wrong case (month) + txreq + rxresp + expect resp.http.ttl == 0.000 + + # Expires using wrong case (tz) + txreq + rxresp + expect resp.http.ttl == 0.000 + + # s-maxage wins over longer max-age + txreq + rxresp + expect resp.http.ttl == 1.000 + + # s-maxage wins over longer max-age - reversed + txreq + rxresp + expect resp.http.ttl == 2.000 + + # s-maxage wins over longer max-age - multiple headers + txreq + rxresp + expect resp.http.ttl == 3.000 +} -run + + +varnish v1 -expect *.s1.req == 12 diff --git a/bin/varnishtest/tests/r00887.vtc b/bin/varnishtest/tests/r00887.vtc deleted file mode 100644 index 627e2c5fa..000000000 --- a/bin/varnishtest/tests/r00887.vtc +++ /dev/null @@ -1,20 +0,0 @@ -varnishtest "Ticket #887" - -server s1 { - rxreq - txresp -hdr "Cache-control: max-age=-1000" -body "FOO" - rxreq - txresp -body "FOOBAR" -} -start - -varnish v1 -vcl+backend { -} -start - -client c1 { - txreq - rxresp - expect resp.bodylen == 3 - txreq - rxresp - expect resp.bodylen == 6 -} -run From fgsch at lodoss.net Fri Jan 4 00:16:07 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 4 Jan 2019 00:16:07 +0000 (UTC) Subject: [master] 032db57bf Polish Message-ID: <20190104001607.1FF4DA0D9D@lists.varnish-cache.org> commit 032db57bfb1bb76e1d38316bcffd581be1b102f4 Author: Federico G. Schwindt Date: Fri Jan 4 00:13:33 2019 +0000 Polish diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c index 82540cc0d..5e83a89a7 100644 --- a/bin/varnishd/cache/cache_vrt_filter.c +++ b/bin/varnishd/cache/cache_vrt_filter.c @@ -354,7 +354,6 @@ static void v_matchproto_(filter_list_t) resp_default_filter_list(void *arg, struct vsb *vsb) { struct req *req; - const char *r; CAST_OBJ_NOTNULL(req, arg, REQ_MAGIC); @@ -369,7 +368,7 @@ resp_default_filter_list(void *arg, struct vsb *vsb) if (cache_param->http_range_support && http_GetStatus(req->resp) == 200 && - http_GetHdr(req->http, H_Range, &r)) + http_GetHdr(req->http, H_Range, NULL)) VSB_cat(vsb, " range"); } diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index d71b6949a..ee24e65c5 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -561,7 +561,6 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, struct req *req, struct h2_req *r2) { h2_error h2e; - const char *b; ASSERT_RXTHR(h2); assert(r2->state == H2_S_OPEN); @@ -588,7 +587,7 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, http_CollectHdrSep(req->http, H_Cookie, "; "); // rfc7540,l,3114,3120 if (req->req_body_status == REQ_BODY_INIT) { - if (!http_GetHdr(req->http, H_Content_Length, &b)) + if (!http_GetHdr(req->http, H_Content_Length, NULL)) req->req_body_status = REQ_BODY_WITHOUT_LEN; else req->req_body_status = REQ_BODY_WITH_LEN; From phk at FreeBSD.org Fri Jan 4 10:18:05 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 4 Jan 2019 10:18:05 +0000 (UTC) Subject: [master] e46ae71aa Make VCC not fail if you try to pass STRANDS to STRING_LIST. Message-ID: <20190104101805.DE00EADF6B@lists.varnish-cache.org> commit e46ae71aaf57047b1e50f07fd55f9758c870fa65 Author: Poul-Henning Kamp Date: Fri Jan 4 10:15:59 2019 +0000 Make VCC not fail if you try to pass STRANDS to STRING_LIST. Strenghen the case against STRING_LIST in the docs. See also: #2873 diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc index 355bb3c56..e7f1d4ad1 100644 --- a/bin/varnishtest/tests/m00000.vtc +++ b/bin/varnishtest/tests/m00000.vtc @@ -120,3 +120,10 @@ varnish v1 -errvcl {Expression has type STRING, expected REAL} { set resp.http.who = std.random("foo", "bar"); } } + +varnish v1 -errvcl {Cannot convert type STRING(STRANDS) to STRING(STRING_LIST)} { + import debug; + sub vcl_deliver { + set resp.http.who = debug.return_strands(req.url + "bar"); + } +} diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst index d8cf6e4b0..36ef2b51c 100644 --- a/doc/sphinx/reference/vmod.rst +++ b/doc/sphinx/reference/vmod.rst @@ -394,9 +394,8 @@ STEVEDORE STRING_LIST C-type: ``const char *, ...`` - `Notice: New vmod developments for 6.1 and higher should - consider STRANDS as a better alternative to STRING_LIST, which - will eventually be replaced entirely.` + `Notice: New vmod developments for 6.1 and later must + use STRANDS instead of STRING_LIST, which is going away.` A multi-component text-string. We try very hard to avoid doing text-processing in Varnish, and this is one way we diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index b92781047..295b81e14 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -1311,7 +1311,14 @@ vcc_expr0(struct vcc *tl, struct expr **e, vcc_type_t fmt) else vcc_expr_cor(tl, e, fmt); ERRCHK(tl); - assert(!(*e)->fmt->stringform); + + if ((*e)->fmt->stringform) { + VSB_printf(tl->sb, "Cannot convert type %s(%s) to %s(%s)\n", + vcc_utype((*e)->fmt), (*e)->fmt->name, + vcc_utype(fmt), fmt->name); + vcc_ErrWhere2(tl, t1, tl->t); + return; + } if ((*e)->fmt != STRINGS && fmt->stringform) vcc_expr_tostring(tl, e, STRINGS); diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index e3fb3cf12..5faa718bf 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -227,3 +227,5 @@ $Method STRING .meth_opt(PRIV_CALL, PRIV_VCL, PRIV_TASK, [STRING s], [BOOL b]) Test object method with all the fancy stuff. + +$Function STRANDS return_strands(STRANDS strand) diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 474bc064f..8738ee243 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -599,3 +599,11 @@ xyzzy_priv_perf(VRT_CTX, VCL_INT size, VCL_INT rounds) return (d); } + +VCL_STRANDS +xyzzy_return_strands(VRT_CTX, VCL_STRANDS strand) +{ + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + return (strand); +} From lasse.karstensen at gmail.com Wed Jan 9 07:59:06 2019 From: lasse.karstensen at gmail.com (Lasse Karstensen) Date: Wed, 9 Jan 2019 07:59:06 +0000 (UTC) Subject: [master] 3e15c14ff Handle Home and End keys in varnishstat. Message-ID: <20190109075906.C6B49937C4@lists.varnish-cache.org> commit 3e15c14ff6fcd009b90f5136c8c4b3265ac20f60 Author: Lasse Karstensen Date: Wed Jan 9 08:41:26 2019 +0100 Handle Home and End keys in varnishstat. This has bugged me (slightly) for ages. Too trivial to warrant its own line(s) in test-case/u00008.vtc. diff --git a/bin/varnishstat/varnishstat_curses.c b/bin/varnishstat/varnishstat_curses.c index 4b1703e6c..1038f0200 100644 --- a/bin/varnishstat/varnishstat_curses.c +++ b/bin/varnishstat/varnishstat_curses.c @@ -890,6 +890,12 @@ handle_keypress(int ch) if (page_start + l_points < n_ptarray - 1) page_start += l_points; break; + case KEY_HOME: + current = 0; + break; + case KEY_END: + current = n_ptarray - 1; + break; case 'd': hide_unseen = 1 - hide_unseen; rebuild = 1; diff --git a/doc/sphinx/reference/varnishstat.rst b/doc/sphinx/reference/varnishstat.rst index 023a6d784..d8b7f2399 100644 --- a/doc/sphinx/reference/varnishstat.rst +++ b/doc/sphinx/reference/varnishstat.rst @@ -85,6 +85,12 @@ The following keys control the interactive display: or Navigate the counter list one page down. + + Navigate the counter list to the top. + + + Navigate the counter list to the bottom. + Toggle between showing and hiding unseen counters. Unseen counters are those that has been zero for the entire runtime From phk at FreeBSD.org Wed Jan 9 10:47:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 9 Jan 2019 10:47:07 +0000 (UTC) Subject: [master] 64e2ba279 Wrap the "if (ctx->handling) return; " check in a macro for readability Message-ID: <20190109104707.0D018977A2@lists.varnish-cache.org> commit 64e2ba27932aefcd5ab4754bf11c28563fe5e7f4 Author: Poul-Henning Kamp Date: Wed Jan 9 09:32:31 2019 +0000 Wrap the "if (ctx->handling) return;" check in a macro for readability diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index adfee9579..6182eadea 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -579,6 +579,8 @@ vcc_CompileSource(struct vcc *tl, struct source *sp) Fh(tl, 0, "/* ---===### VCC generated .h code ###===---*/\n"); Fc(tl, 0, "\n/* ---===### VCC generated .c code ###===---*/\n"); + Fc(tl, 0, "\n#define END_ if (*ctx->handling) return\n"); + vcc_Parse_Init(tl); vcc_Expr_Init(tl); diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c index 2b5075358..af8e6a45e 100644 --- a/lib/libvcc/vcc_parse.c +++ b/lib/libvcc/vcc_parse.c @@ -149,7 +149,7 @@ vcc_Compound(struct vcc *tl) Fb(tl, 1, "{\n"); tl->indent += INDENT; C(tl, ";"); - Fb(tl, 1, "if (*ctx->handling) return;\n"); + Fb(tl, 1, "END_;\n"); while (1) { ERRCHK(tl); t = tl->t; @@ -206,7 +206,7 @@ vcc_Compound(struct vcc *tl) vcc_ErrWhere(tl, tl->t); return; } - Fb(tl, 1, "if (*ctx->handling) return;\n"); + Fb(tl, 1, "END_;\n"); } } From phk at FreeBSD.org Wed Jan 9 10:47:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 9 Jan 2019 10:47:07 +0000 (UTC) Subject: [master] 1425d2ae3 VSB_quote(QUOTE_CSTR) should not indent with "\t" unless asked. Message-ID: <20190109104707.20A9E977A5@lists.varnish-cache.org> commit 1425d2ae3cfff0a7ca8dd4b3696118ebf474b0d3 Author: Poul-Henning Kamp Date: Wed Jan 9 09:37:39 2019 +0000 VSB_quote(QUOTE_CSTR) should not indent with "\t" unless asked. diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c index 0e92b1771..5101a9007 100644 --- a/lib/libvarnish/vsb.c +++ b/lib/libvarnish/vsb.c @@ -573,7 +573,7 @@ VSB_quote_pfx(struct vsb *s, const char *pfx, const void *v, int len, int how) break; case '\n': if (how & VSB_QUOTE_CSTR) { - (void)VSB_printf(s, "\\n\"\n%s\t\"", pfx); + (void)VSB_printf(s, "\\n\"\n%s\"", pfx); } else if (how & (VSB_QUOTE_NONL|VSB_QUOTE_UNSAFE)) { (void)VSB_printf(s, "\n"); nl = 1; diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 6182eadea..d9bc4f84b 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -233,9 +233,8 @@ EmitCoordinates(const struct vcc *tl, struct vsb *vsb) VTAILQ_FOREACH(sp, &tl->sources, list) { VSB_printf(vsb, " /* "); VSB_quote(vsb, sp->name, -1, VSB_QUOTE_CSTR); - VSB_printf(vsb, "*/\n"); - VSB_printf(vsb, "\t"); - VSB_quote(vsb, sp->b, sp->e - sp->b, VSB_QUOTE_CSTR); + VSB_printf(vsb, " */\n"); + VSB_quote_pfx(vsb, "\t", sp->b, sp->e - sp->b, VSB_QUOTE_CSTR); VSB_printf(vsb, ",\n"); } VSB_printf(vsb, "};\n\n"); From phk at FreeBSD.org Wed Jan 9 10:47:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 9 Jan 2019 10:47:07 +0000 (UTC) Subject: [master] fb65f18f0 Improve indentation of VGC Message-ID: <20190109104707.3A20D977AA@lists.varnish-cache.org> commit fb65f18f090fbe4901f17e954ace5d01ef19d2d2 Author: Poul-Henning Kamp Date: Wed Jan 9 09:48:02 2019 +0000 Improve indentation of VGC diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 295b81e14..61dca8e19 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -130,8 +130,9 @@ vcc_delete_expr(struct expr *e) * \v1 insert subexpression 1 * \v2 insert subexpression 2 * \vS insert subexpression 1(STRINGS) as STRING + * \vs insert subexpression 2(STRINGS) as STRING * \vT insert subexpression 1(STRINGS) as STRANDS - * \vt insert subexpression 1(STRINGS) as STRANDS + * \vt insert subexpression 2(STRINGS) as STRANDS * \v+ increase indentation * \v- decrease indentation * anything else is literal @@ -161,8 +162,8 @@ vcc_expr_edit(struct vcc *tl, vcc_type_t fmt, const char *p, struct expr *e1, } assert(*p == '\v'); switch (*++p) { - case '+': VSB_cat(e->vsb, "\v+"); break; - case '-': VSB_cat(e->vsb, "\v-"); break; + case '+': VSB_cat(e->vsb, "\v+"); nl = 0; break; + case '-': VSB_cat(e->vsb, "\v-"); nl = 0; break; case 'S': case 's': e3 = (*p == 'S' ? e1 : e2); @@ -654,7 +655,7 @@ vcc_func(struct vcc *tl, struct expr **e, const void *priv, if (sa != NULL) { *e = vcc_expr_edit(tl, e1->fmt, "\v1\n})\v-", e1, NULL); } else { - *e = vcc_expr_edit(tl, e1->fmt, "\v1\n)\v-", e1, NULL); + *e = vcc_expr_edit(tl, e1->fmt, "\v1\v-\n)", e1, NULL); } SkipToken(tl, ')'); } @@ -1336,7 +1337,7 @@ vcc_expr0(struct vcc *tl, struct expr **e, vcc_type_t fmt) if ((*e)->fmt == STRING_LIST) *e = vcc_expr_edit(tl, STRING_LIST, - "\v+\n\v1,\nvrt_magic_string_end\v-", *e, NULL); + "\n\v1,\nvrt_magic_string_end", *e, NULL); if (fmt == BOOL) { vcc_expr_tobool(tl, e); From phk at FreeBSD.org Wed Jan 9 10:47:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 9 Jan 2019 10:47:07 +0000 (UTC) Subject: [master] 02916817d Implement stringifcation of STRANDS via a workspace allocation. Message-ID: <20190109104707.54FE0977B0@lists.varnish-cache.org> commit 02916817dff50fb50b02e042a0e975434499ff1b Author: Poul-Henning Kamp Date: Wed Jan 9 10:44:55 2019 +0000 Implement stringifcation of STRANDS via a workspace allocation. When a VMOD returns a STRANDS, it is a requirement that the storage be stable, but that is not really different from any other type of return value. Completes #2873 diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc index e7f1d4ad1..355bb3c56 100644 --- a/bin/varnishtest/tests/m00000.vtc +++ b/bin/varnishtest/tests/m00000.vtc @@ -120,10 +120,3 @@ varnish v1 -errvcl {Expression has type STRING, expected REAL} { set resp.http.who = std.random("foo", "bar"); } } - -varnish v1 -errvcl {Cannot convert type STRING(STRANDS) to STRING(STRING_LIST)} { - import debug; - sub vcl_deliver { - set resp.http.who = debug.return_strands(req.url + "bar"); - } -} diff --git a/bin/varnishtest/tests/v00058.vtc b/bin/varnishtest/tests/v00058.vtc index e9869c29e..cef91e8b7 100644 --- a/bin/varnishtest/tests/v00058.vtc +++ b/bin/varnishtest/tests/v00058.vtc @@ -78,6 +78,13 @@ varnish v1 -arg "-i foobar" -vcl { req.http.Foo + req.http.Unset + req.http.Bar); debug.sethdr(resp.http.Hdr-6, req.http.Foo); debug.sethdr(resp.http.Hdr-7, req.http.Unset); + + set resp.http.Hdr-8 = + debug.return_strands( + debug.return_strands( + req.url + "<-->" + req.url + ) + ); } } -start @@ -108,6 +115,7 @@ client c1 { expect resp.http.Hdr-5 == "foobar" expect resp.http.Hdr-6 == "foo" expect resp.http.Hdr-7 == "" + expect resp.http.Hdr-8 == "/<-->/" } -run # out of workspace @@ -140,8 +148,7 @@ varnish v1 -vcl+backend { = debug.concatenate(req.http.Foo + req.http.Bar + req.http.Baz + req.http.Quux); - } - elsif (req.url == "/2") { + } elsif (req.url == "/2") { # VRT_CollectStrands() invokes VCL failure. set req.http.Result = debug.collect(req.http.Foo + req.http.Bar diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 61dca8e19..00b6ca700 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -186,8 +186,8 @@ vcc_expr_edit(struct vcc *tl, vcc_type_t fmt, const char *p, struct expr *e1, " const char * strs_%u_s[%d];\n", tl->unique, tl->unique, e3->nstr); VSB_printf(e->vsb, - "\v+\nVRT_BundleStrands(%d, &strs_%u_a, strs_%u_s," - "\v+\n%s,\nvrt_magic_string_end)\v-\v-", + "VRT_BundleStrands(%d, &strs_%u_a, strs_%u_s," + "\v+\n%s,\nvrt_magic_string_end\v-\n)", e3->nstr, tl->unique, tl->unique, VSB_data(e3->vsb)); tl->unique++; @@ -1313,6 +1313,12 @@ vcc_expr0(struct vcc *tl, struct expr **e, vcc_type_t fmt) vcc_expr_cor(tl, e, fmt); ERRCHK(tl); + if ((*e)->fmt == fmt) + return; + + if ((*e)->fmt != STRINGS && fmt->stringform) + vcc_expr_tostring(tl, e, STRINGS); + if ((*e)->fmt->stringform) { VSB_printf(tl->sb, "Cannot convert type %s(%s) to %s(%s)\n", vcc_utype((*e)->fmt), (*e)->fmt->name, @@ -1321,9 +1327,6 @@ vcc_expr0(struct vcc *tl, struct expr **e, vcc_type_t fmt) return; } - if ((*e)->fmt != STRINGS && fmt->stringform) - vcc_expr_tostring(tl, e, STRINGS); - if ((*e)->fmt == STRINGS && fmt->stringform) { if (fmt == STRING_LIST) (*e)->fmt = STRING_LIST; diff --git a/lib/libvcc/vcc_types.c b/lib/libvcc/vcc_types.c index 2e5660994..2d9208c64 100644 --- a/lib/libvcc/vcc_types.c +++ b/lib/libvcc/vcc_types.c @@ -138,6 +138,7 @@ const struct type STRANDS[1] = {{ .magic = TYPE_MAGIC, .name = "STRANDS", .stringform = 1, + .tostring = "VRT_CollectStrands(ctx,\v+\n\v1\v-\n)", }}; const struct type STRINGS[1] = {{ From phk at FreeBSD.org Wed Jan 9 21:52:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 9 Jan 2019 21:52:08 +0000 (UTC) Subject: [master] 708cec815 This file is derived from FreeBSD's flopen.c and does not need "verror.h" but only "errno.h" Message-ID: <20190109215208.20FCFA95A2@lists.varnish-cache.org> commit 708cec815a88609ac6b466078bf747657588aaab Author: Poul-Henning Kamp Date: Wed Jan 9 21:51:29 2019 +0000 This file is derived from FreeBSD's flopen.c and does not need "verror.h" but only "errno.h" diff --git a/lib/libvarnish/vfl.c b/lib/libvarnish/vfl.c index fa7804569..9af496a91 100644 --- a/lib/libvarnish/vfl.c +++ b/lib/libvarnish/vfl.c @@ -32,13 +32,13 @@ #include #include +#include #include #include #include #include #include "vfl.h" -#include "verrno.h" /* * Reliably open and lock a file. From phk at FreeBSD.org Fri Jan 11 10:29:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 11 Jan 2019 10:29:08 +0000 (UTC) Subject: [master] 7119d790b Avoid printing %s, NULL in case of errors we do not expect. Message-ID: <20190111102908.DE07C2EFC@lists.varnish-cache.org> commit 7119d790b590e7fb560ad602cedfda5185c7e841 Author: Poul-Henning Kamp Date: Fri Jan 11 10:26:44 2019 +0000 Avoid printing %s,NULL in case of errors we do not expect. Fixes #2879 diff --git a/lib/libvarnish/vnum.c b/lib/libvarnish/vnum.c index b619199c6..59e804ec8 100644 --- a/lib/libvarnish/vnum.c +++ b/lib/libvarnish/vnum.c @@ -349,15 +349,17 @@ main(int argc, char *argv[]) for (tc = test_cases; tc->str; ++tc) { e = VNUM_2bytes(tc->str, &val, tc->rel); - if (e != tc->err) { - printf("%s: VNUM_2bytes(\"%s\", %ju) (%s) != (%s)\n", - *argv, tc->str, tc->rel, tc->err, e); - ++ec; - } else if (e == NULL && val != tc->val) { - printf("%s: VNUM_2bytes(\"%s\", %ju) %ju != %ju (%s)\n", - *argv, tc->str, tc->rel, val, tc->val, e); - ++ec; - } + if (e != NULL) + val = 0; + if (e == tc->err && val == tc->val) + continue; + ++ec; + printf("%s: VNUM_2bytes(\"%s\", %ju)\n", + *argv, tc->str, tc->rel); + printf("\tExpected:\tstatus %s - value %ju\n", + tc->err ? tc->err : "Success", tc->val); + printf("\tGot:\t\tstatus %s - value %ju\n", + e ? e : "Success", val); } if (!isnan(VNUM_duration(NULL))) { printf("%s: VNUM_Duration(NULL) fail\n", *argv); From geoff at uplex.de Fri Jan 11 11:16:07 2019 From: geoff at uplex.de (Geoff Simmons) Date: Fri, 11 Jan 2019 11:16:07 +0000 (UTC) Subject: [master] ddd9f277e Use VSBs to generate code for VMOD objects. Message-ID: <20190111111607.BEBEF4E1A@lists.varnish-cache.org> commit ddd9f277e7c2388c4b991f9ddc8f8619d8c2bbad Author: Geoff Simmons Date: Fri Jan 11 11:42:16 2019 +0100 Use VSBs to generate code for VMOD objects. Fixes #2880 diff --git a/bin/varnishtest/tests/r02880.vtc b/bin/varnishtest/tests/r02880.vtc new file mode 100644 index 000000000..1d554230b --- /dev/null +++ b/bin/varnishtest/tests/r02880.vtc @@ -0,0 +1,13 @@ +varnishtest "Long VMOD object names" + +varnish v1 -vcl { + import debug; + backend b { .host = "${bad_ip}"; } + + sub vcl_init { + new l234567890123456789012345678901234567890123456789012345678 + = debug.obj(); + new l2345678901234567890123456789012345678901234567890123456789 + = debug.obj(); + } +} diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index babd70eef..9fe6ddddf 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -38,6 +38,7 @@ #include "vfil.h" #include "vjsn.h" #include "vmod_abi.h" +#include "vsb.h" static int vcc_path_dlopen(void *priv, const char *fn) @@ -349,8 +350,7 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym) { struct symbol *sy1, *sy2, *sy3; struct inifin *ifp; - char buf1[128]; - char buf2[128]; + struct vsb *buf; const struct vjsn_val *vv, *vf; const char *p; @@ -393,8 +393,10 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym) vf = VTAILQ_NEXT(vf, list); - bprintf(buf1, ", &%s, \"%s\"", sy1->rname, sy1->name); - vcc_Eval_Func(tl, vf, buf1, sy2); + buf = VSB_new_auto(); + VSB_printf(buf, ", &%s, \"%s\"", sy1->rname, sy1->name); + VSB_finish(buf); + vcc_Eval_Func(tl, vf, VSB_data(buf), sy2); ERRCHK(tl); SkipToken(tl, ';'); sy1->def_e = tl->t; @@ -411,8 +413,10 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym) VSB_printf(ifp->fin, "\t\t%s(&%s);", vf->value, sy1->rname); /* Instantiate symbols for the methods */ - bprintf(buf1, ", %s", sy1->rname); - p = TlDup(tl, buf1); + VSB_clear(buf); + VSB_printf(buf, ", %s", sy1->rname); + VSB_finish(buf); + p = TlDup(tl, VSB_data(buf)); while (vv != NULL) { vf = VTAILQ_FIRST(&vv->children); assert(vf->type == VJSN_STRING); @@ -420,11 +424,14 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym) vf = VTAILQ_NEXT(vf, list); assert(vf->type == VJSN_STRING); - bprintf(buf2, "%s.%s", sy1->name, vf->value); - sy3 = VCC_MkSym(tl, buf2, SYM_FUNC, VCL_LOW, VCL_HIGH); + VSB_clear(buf); + VSB_printf(buf, "%s.%s", sy1->name, vf->value); + VSB_finish(buf); + sy3 = VCC_MkSym(tl, VSB_data(buf), SYM_FUNC, VCL_LOW, VCL_HIGH); AN(sy3); func_sym(sy3, sy2->vmod, VTAILQ_NEXT(vf, list)); sy3->extra = p; vv = VTAILQ_NEXT(vv, list); } + VSB_destroy(&buf); } From dridi.boukelmoune at gmail.com Fri Jan 11 13:31:08 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 11 Jan 2019 13:31:08 +0000 (UTC) Subject: [master] 068ba48f5 Missing closing bracket Message-ID: <20190111133108.0D3E075D8@lists.varnish-cache.org> commit 068ba48f50d9453775bd7741232afe16d0aa64fa Author: Dridi Boukelmoune Date: Fri Jan 11 14:30:05 2019 +0100 Missing closing bracket diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index cbb54298d..b3dd3c0c4 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -120,7 +120,7 @@ ved_include(struct req *preq, const char *src, const char *host, if (preq->esi_level >= cache_param->max_esi_depth) { VSLb(preq->vsl, SLT_VCL_Error, - "ESI depth limit reach (param max_esi_depth = %u", + "ESI depth limit reach (param max_esi_depth = %u)", cache_param->max_esi_depth); return; } From phk at FreeBSD.org Fri Jan 11 17:02:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 11 Jan 2019 17:02:08 +0000 (UTC) Subject: [master] a08b9d5c4 assert VSB_finish() works Message-ID: <20190111170208.0E78C6253C@lists.varnish-cache.org> commit a08b9d5c4f7496aca3b82ed4b02f6afd26441037 Author: Poul-Henning Kamp Date: Fri Jan 11 17:01:17 2019 +0000 assert VSB_finish() works diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index 9fe6ddddf..8b308c03f 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -395,7 +395,7 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym) buf = VSB_new_auto(); VSB_printf(buf, ", &%s, \"%s\"", sy1->rname, sy1->name); - VSB_finish(buf); + AZ(VSB_finish(buf)); vcc_Eval_Func(tl, vf, VSB_data(buf), sy2); ERRCHK(tl); SkipToken(tl, ';'); @@ -415,7 +415,7 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym) /* Instantiate symbols for the methods */ VSB_clear(buf); VSB_printf(buf, ", %s", sy1->rname); - VSB_finish(buf); + AZ(VSB_finish(buf)); p = TlDup(tl, VSB_data(buf)); while (vv != NULL) { vf = VTAILQ_FIRST(&vv->children); @@ -426,7 +426,7 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym) VSB_clear(buf); VSB_printf(buf, "%s.%s", sy1->name, vf->value); - VSB_finish(buf); + AZ(VSB_finish(buf)); sy3 = VCC_MkSym(tl, VSB_data(buf), SYM_FUNC, VCL_LOW, VCL_HIGH); AN(sy3); func_sym(sy3, sy2->vmod, VTAILQ_NEXT(vf, list)); From phk at FreeBSD.org Mon Jan 14 09:33:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 14 Jan 2019 09:33:07 +0000 (UTC) Subject: [master] 7e9a31831 Explain string->boolean better. Message-ID: <20190114093307.9F69560428@lists.varnish-cache.org> commit 7e9a31831133d8fe868a0c8ecb846a5e8ac00fa5 Author: Poul-Henning Kamp Date: Mon Jan 14 09:32:08 2019 +0000 Explain string->boolean better. Fixes #2846 diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index fbd9cde3f..2f31f4d3f 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -93,7 +93,12 @@ Booleans can be either ``true`` or ``false``. In addition, in a boolean context some data types will evaluate to ``true`` or ``false`` depending on their value. -String types will evaluate to ``false`` if they are not set; backend types +String types will evaluate to ``false`` if they are unset. This allows +checks of the type ``if (req.http.opthdr) {}`` to test if a header +exists, even if it is empty, whereas ``if (req.http.opthdr == "") {}`` +does not distinguish if the header does not exist or if it is empty. + +Backend types will evaluate to ``false`` if they don't have a backend assigned; integer types will evaluate to ``false`` if their value is zero; duration types will evaluate to ``false`` if their value is equal or less than zero. From phk at FreeBSD.org Mon Jan 14 11:22:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 14 Jan 2019 11:22:08 +0000 (UTC) Subject: [master] 0f1d40cfa Introduce VCL_use SLT which tells us which VCL we're running in Message-ID: <20190114112208.A490462751@lists.varnish-cache.org> commit 0f1d40cfa05563ed36167d84804b936fb4a9cfba Author: Poul-Henning Kamp Date: Mon Jan 14 11:19:47 2019 +0000 Introduce VCL_use SLT which tells us which VCL we're running in diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 02ad2cc56..1e42d0201 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -896,6 +896,7 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, bo = VBO_GetBusyObj(wrk, req); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); + AN(bo->vcl); boc = HSH_RefBoc(oc); CHECK_OBJ_NOTNULL(boc, BOC_MAGIC); @@ -917,6 +918,7 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, } VSLb(bo->vsl, SLT_Begin, "bereq %u %s", VXID(req->vsl->wid), how); + VSLb(bo->vsl, SLT_VCL_use, "%s", VCL_Name(bo->vcl)); VSLb(req->vsl, SLT_Link, "bereq %u %s", VXID(bo->vsl->wid), how); THR_SetBusyobj(bo); @@ -924,8 +926,6 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, bo->sp = req->sp; SES_Ref(bo->sp); - AN(bo->vcl); - oc->boc->vary = req->vary_b; req->vary_b = NULL; diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index f5e6b16f7..f11fd96bf 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -1029,6 +1029,7 @@ CNT_Embark(struct worker *wrk, struct req *req) VCL_Refresh(&wrk->vcl); req->vcl = wrk->vcl; wrk->vcl = NULL; + VSLb(req->vsl, SLT_VCL_use, "%s", VCL_Name(req->vcl)); } AN(req->vcl); diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c index 051d16c9f..bfd4effe6 100644 --- a/bin/varnishd/cache/cache_vrt_vcl.c +++ b/bin/varnishd/cache/cache_vrt_vcl.c @@ -344,8 +344,8 @@ VRT_vcl_select(VRT_CTX, VCL_VCL vcl) VCL_TaskLeave(req->vcl, req->privs); VCL_Rel(&req->vcl); vcl_get(&req->vcl, vcl); - /* XXX: better logging */ - VSLb(ctx->req->vsl, SLT_Debug, "Now using %s VCL", vcl->loaded_name); + VSLb(ctx->req->vsl, SLT_VCL_use, "%s via %s", + req->vcl->loaded_name, vcl->loaded_name); VCL_TaskEnter(req->vcl, req->privs); } diff --git a/bin/varnishtest/tests/u00010.vtc b/bin/varnishtest/tests/u00010.vtc index dd37a15d0..329f92567 100644 --- a/bin/varnishtest/tests/u00010.vtc +++ b/bin/varnishtest/tests/u00010.vtc @@ -20,7 +20,7 @@ client c1 { varnish v1 -vsl_catchup -process p1 -expect-text 1 1 {list length 64} +process p1 -expect-text 1 1 {list length 65} process p1 -writehex 0c diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index d9be207ac..0ff57bd96 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -192,7 +192,7 @@ include/varnishstat_synopsis.rst: $(top_builddir)/bin/varnishstat/varnishstat BUILT_SOURCES += include/varnishstat_options.rst \ include/varnishstat_synopsis.rst -include/vsl-tags.rst: $(top_builddir)/lib/libvarnishapi/vsl2rst +include/vsl-tags.rst: $(top_builddir)/lib/libvarnishapi/vsl2rst $(top_builddir)/lib/libvarnishapi/vsl2rst > ${@}_ mv ${@}_ ${@} BUILT_SOURCES += include/vsl-tags.rst diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 49b27cf9b..cc02f18d4 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -642,6 +642,15 @@ SLTM(SessError, 0, "Client connection accept failed", "\n" ) +SLTM(VCL_use, 0, "VCL in use", + "Records the name of the VCL being used.\n\n" + "The format is::\n\n" + "\t%s [via %s]\n" + "\t| |\n" + "\t| +- Name of label used to find it (optional)\n" + "\t+--------- Name of VCL put in use\n" +) + #undef NODEF_NOTICE #undef SLTM From fgsch at lodoss.net Mon Jan 14 15:57:09 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 14 Jan 2019 15:57:09 +0000 (UTC) Subject: [master] da3c0bd9d Clarify this is a 3-fields tag Message-ID: <20190114155709.DB51991F91@lists.varnish-cache.org> commit da3c0bd9d767da265b077e0b65b2b82ac9aa1dfa Author: Federico G. Schwindt Date: Mon Jan 14 15:55:31 2019 +0000 Clarify this is a 3-fields tag diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index cc02f18d4..8353bd966 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -645,10 +645,12 @@ SLTM(SessError, 0, "Client connection accept failed", SLTM(VCL_use, 0, "VCL in use", "Records the name of the VCL being used.\n\n" "The format is::\n\n" - "\t%s [via %s]\n" - "\t| |\n" - "\t| +- Name of label used to find it (optional)\n" + "\t%s [ %s %s ]\n" + "\t| | |\n" + "\t| | +- Name of label used to find it\n" + "\t| +---- \"via\"\n" "\t+--------- Name of VCL put in use\n" + "\n" ) #undef NODEF_NOTICE From phk at FreeBSD.org Mon Jan 14 23:36:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 14 Jan 2019 23:36:08 +0000 (UTC) Subject: [master] b9708eac8 Fix a recently introduced issue with logging of ESI request headers. Message-ID: <20190114233609.0CCCA9E208@lists.varnish-cache.org> commit b9708eac809feac0358ba2156aca0a5000582eb2 Author: Poul-Henning Kamp Date: Mon Jan 14 21:46:40 2019 +0000 Fix a recently introduced issue with logging of ESI request headers. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 86db641dc..ff5fa29da 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -595,7 +595,8 @@ extern pthread_t cli_thread; /* cache_http.c */ unsigned HTTP_estimate(unsigned nhttp); -void HTTP_Copy(struct http *to, const struct http * const fm); +void HTTP_Clone(struct http *to, const struct http * const fm); +void HTTP_Dup(struct http *to, const struct http * const fm); struct http *HTTP_create(void *p, uint16_t nhttp, unsigned); const char *http_Status2Reason(unsigned, const char **); unsigned http_EstimateWS(const struct http *fm, unsigned how); diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index b3dd3c0c4..f164b4bc5 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -141,7 +141,7 @@ ved_include(struct req *preq, const char *src, const char *host, req->top = preq->top; HTTP_Setup(req->http, req->ws, req->vsl, SLT_ReqMethod); - HTTP_Copy(req->http, preq->http0); + HTTP_Dup(req->http, preq->http0); http_SetH(req->http, HTTP_HDR_URL, src); if (host != NULL && *host != '\0') { diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 1e42d0201..57aa5bd0f 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -208,7 +208,7 @@ vbf_stp_mkbereq(struct worker *wrk, struct busyobj *bo) HTTP_Setup(bo->bereq, bo->ws, bo->vsl, SLT_BereqMethod); bo->ws_bo = WS_Snapshot(bo->ws); - HTTP_Copy(bo->bereq, bo->bereq0); + HTTP_Clone(bo->bereq, bo->bereq0); if (bo->req->req_body_status == REQ_BODY_NONE) { bo->req = NULL; diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 34cd11a02..534cd5d2e 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -190,19 +190,39 @@ http_Teardown(struct http *hp) memset(hp->hdf, 0, sizeof *hp->hdf * hp->shd); } -/*--------------------------------------------------------------------*/ +/*-------------------------------------------------------------------- + * Duplicate the http content into another http + * We cannot just memcpy the struct because the hd & hdf are private + * storage to the struct http. + */ void -HTTP_Copy(struct http *to, const struct http * const fm) +HTTP_Dup(struct http *to, const struct http * const fm) { assert(fm->nhd <= to->shd); - memcpy(&to->nhd, &fm->nhd, sizeof *to - offsetof(struct http, nhd)); memcpy(to->hd, fm->hd, fm->nhd * sizeof *to->hd); memcpy(to->hdf, fm->hdf, fm->nhd * sizeof *to->hdf); + to->nhd = fm->nhd; + to->logtag = fm->logtag; + to->status = fm->status; to->protover = fm->protover; } + +/*-------------------------------------------------------------------- + * Clone the entire http structure, including vsl & ws + */ + +void +HTTP_Clone(struct http *to, const struct http * const fm) +{ + + HTTP_Dup(to, fm); + to->vsl = fm->vsl; + to->ws = fm->ws; +} + /*--------------------------------------------------------------------*/ void diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index d2f3c2d31..11ada9c1e 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -189,7 +189,7 @@ Req_Rollback(struct req *req) { VCL_TaskLeave(req->vcl, req->privs); VCL_TaskEnter(req->vcl, req->privs); - HTTP_Copy(req->http, req->http0); + HTTP_Clone(req->http, req->http0); if (WS_Overflowed(req->ws)) req->wrk->stats->ws_client_overflow++; WS_Reset(req->ws, req->ws_req); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index f11fd96bf..9650ec59d 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -94,7 +94,7 @@ cnt_transport(struct worker *wrk, struct req *req) } req->ws_req = WS_Snapshot(req->ws); - HTTP_Copy(req->http0, req->http); // For ESI & restart + HTTP_Clone(req->http0, req->http); // For ESI & restart req->req_step = R_STP_RECV; return (REQ_FSM_MORE); } diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 9ab637fed..e1ac810c4 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -639,7 +639,7 @@ VRT_Rollback(VRT_CTX, VCL_HTTP hp) // -> VBO_Rollback ? VCL_TaskLeave(ctx->bo->vcl, ctx->bo->privs); VCL_TaskEnter(ctx->bo->vcl, ctx->bo->privs); - HTTP_Copy(ctx->bo->bereq, ctx->bo->bereq0); + HTTP_Clone(ctx->bo->bereq, ctx->bo->bereq0); WS_Reset(ctx->bo->bereq->ws, ctx->bo->ws_bo); WS_Reset(ctx->bo->ws, ctx->bo->ws_bo); } else diff --git a/include/vrt.h b/include/vrt.h index 19bdb7b08..35d9dcaa1 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -52,6 +52,10 @@ * binary/load-time compatible, increment MAJOR version * * + * 9.0 (scheduled for 2019-03-15) + * HTTP_Copy() removed + * HTTP_Dup() added + * HTTP_Clone() added * 8.0 (2018-09-15) * VRT_Strands() added * VRT_StrandsWS() added From phk at FreeBSD.org Mon Jan 14 23:36:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 14 Jan 2019 23:36:09 +0000 (UTC) Subject: [master] d67993451 Now that we have decided bump VRT_MAJOR, drop the reqtop struct again and all the overhead it brought. Message-ID: <20190114233609.2D9BD9E20F@lists.varnish-cache.org> commit d67993451bda7eeb7c0ea3c5528a8353f283a190 Author: Poul-Henning Kamp Date: Mon Jan 14 23:22:16 2019 +0000 Now that we have decided bump VRT_MAJOR, drop the reqtop struct again and all the overhead it brought. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index ff5fa29da..8f450ee63 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -443,11 +443,6 @@ struct busyobj { /*--------------------------------------------------------------------*/ -struct reqtop { - unsigned magic; -#define REQTOP_MAGIC 0x57fbda52 - struct req *topreq; -}; struct req { unsigned magic; @@ -458,7 +453,7 @@ struct req { enum sess_close doclose; int restarts; int esi_level; - struct reqtop *top; /* esi_level == 0 request */ + struct req *topreq; /* esi_level == 0 request */ #define REQ_FLAG(l, r, w, d) unsigned l:1; #include "tbl/req_flags.h" @@ -537,7 +532,7 @@ struct req { struct vrt_privs privs[1]; }; -#define IS_TOPREQ(req) ((req)->top == NULL || (req)->top->topreq == (req)) +#define IS_TOPREQ(req) ((req)->topreq == (req)) /*-------------------------------------------------------------------- * Struct sess is a high memory-load structure because sessions typically diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index f164b4bc5..067d77973 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -112,7 +112,7 @@ ved_include(struct req *preq, const char *src, const char *host, enum req_fsm_nxt s; CHECK_OBJ_NOTNULL(preq, REQ_MAGIC); - CHECK_OBJ_NOTNULL(preq->top, REQTOP_MAGIC); + CHECK_OBJ_NOTNULL(preq->topreq, REQ_MAGIC); sp = preq->sp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(ecx, ECX_MAGIC); @@ -138,7 +138,7 @@ ved_include(struct req *preq, const char *src, const char *host, req->esi_level = preq->esi_level + 1; - req->top = preq->top; + req->topreq = preq->topreq; HTTP_Setup(req->http, req->ws, req->vsl, SLT_ReqMethod); HTTP_Dup(req->http, preq->http0); @@ -258,16 +258,6 @@ ved_vdp_esi_init(struct req *req, void **priv) *priv = ecx; RFC2616_Weaken_Etag(req->resp); - if (IS_TOPREQ(req)) { - Req_MakeTop(req); - if (req->top == NULL) { - VSLb(req->vsl, SLT_Error, - "(top)request workspace overflow"); - Req_Fail(req, SC_OVERLOAD); - return (-1); - } - } - req->res_mode |= RES_ESI; if (req->resp_len != 0) req->resp_len = -1; diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index c0ef87130..db1c498bc 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -455,20 +455,6 @@ pan_busyobj(struct vsb *vsb, const struct busyobj *bo) /*--------------------------------------------------------------------*/ -static void -pan_top(struct vsb *vsb, const struct reqtop *top) -{ - VSB_printf(vsb, "top = %p {\n", top); - if (PAN_already(vsb, top)) - return; - VSB_indent(vsb, 2); - pan_req(vsb, top->topreq); - VSB_indent(vsb, -2); - VSB_printf(vsb, "},\n"); -} - -/*--------------------------------------------------------------------*/ - static void pan_req(struct vsb *vsb, const struct req *req) { @@ -542,8 +528,11 @@ pan_req(struct vsb *vsb, const struct req *req) pan_privs(vsb, req->privs); - if (req->top != NULL) - pan_top(vsb, req->top); + VSB_printf(vsb, "topreq = {\n"); + VSB_indent(vsb, 2); + pan_req(vsb, req->topreq); + VSB_indent(vsb, -2); + VSB_printf(vsb, "},\n"); VSB_indent(vsb, -2); VSB_printf(vsb, "},\n"); diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index 11ada9c1e..34c99907e 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -63,6 +63,7 @@ Req_AcctLogCharge(struct VSC_main_wrk *ds, struct req *req) (uintmax_t)(a->resp_hdrbytes + a->resp_bodybytes)); } + AN(req->topreq); if (IS_TOPREQ(req)) { #define ACCT(foo) ds->s_##foo += a->foo; #include "tbl/acct_fields_req.h" @@ -147,6 +148,8 @@ Req_New(const struct worker *wrk, struct sess *sp) req->t_prev = NAN; req->t_req = NAN; + req->topreq = req; + return (req); } @@ -174,7 +177,7 @@ Req_Release(struct req *req) MPL_AssertSane(req); VSL_Flush(req->vsl, 0); req->sp = NULL; - req->top = NULL; + req->topreq = NULL; MPL_Free(pp->mpl_req, req); } @@ -193,7 +196,6 @@ Req_Rollback(struct req *req) if (WS_Overflowed(req->ws)) req->wrk->stats->ws_client_overflow++; WS_Reset(req->ws, req->ws_req); - req->top = NULL; } /*---------------------------------------------------------------------- @@ -207,6 +209,7 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->topreq, REQ_MAGIC); assert(sp == req->sp); req->director_hint = NULL; @@ -238,7 +241,6 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req) req->hash_ignore_busy = 0; req->esi_level = 0; req->is_hit = 0; - req->top = 0; if (WS_Overflowed(req->ws)) wrk->stats->ws_client_overflow++; @@ -257,20 +259,3 @@ Req_Fail(struct req *req, enum sess_close reason) AN(req->transport->req_fail); req->transport->req_fail(req, reason); } - -/*---------------------------------------------------------------------- - */ - -void -Req_MakeTop(struct req *req) -{ - - CHECK_OBJ_ORNULL(req->top, REQTOP_MAGIC); - if (req->top != NULL) - return; - req->top = WS_Alloc(req->ws, sizeof *req->top); - if (req->top != NULL) { - INIT_OBJ(req->top, REQTOP_MAGIC); - req->top->topreq = req; - } -} diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 9650ec59d..dc6da227f 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -1021,6 +1021,7 @@ CNT_Embark(struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->topreq, REQ_MAGIC); /* wrk can have changed for restarts */ req->vfc->wrk = req->wrk = wrk; diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index b8c53a6df..563fafc18 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -314,7 +314,6 @@ void Req_Rollback(struct req *req); void Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req); void Req_Fail(struct req *req, enum sess_close reason); void Req_AcctLogCharge(struct VSC_main_wrk *, struct req *); -void Req_MakeTop(struct req *req); /* cache_req_body.c */ int VRB_Ignore(struct req *); diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 52c0f0a80..17c56463f 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -91,15 +91,12 @@ VCL_Req2Ctx(struct vrt_ctx *ctx, struct req *req) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_ORNULL(req->topreq, REQ_MAGIC); ctx->vcl = req->vcl; ctx->vsl = req->vsl; ctx->http_req = req->http; - CHECK_OBJ_ORNULL(req->top, REQTOP_MAGIC); - if (req->top != NULL) - ctx->http_req_top = req->top->topreq->http; - else - ctx->http_req_top = req->http; + ctx->http_req_top = req->topreq->http; ctx->http_resp = req->resp; ctx->req = req; ctx->sp = req->sp; diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index 211990cc3..13ab454e2 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -166,13 +166,7 @@ VRT_priv_top(VRT_CTX, const void *vmod_id) NEEDLESS(return NULL); } CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); - if (ctx->req->top != NULL) { - CHECK_OBJ_NOTNULL(ctx->req->top, REQTOP_MAGIC); - CHECK_OBJ_NOTNULL(ctx->req->top->topreq, REQ_MAGIC); - req = ctx->req->top->topreq; - } else { - req = ctx->req; - } + req = ctx->req->topreq; CAST_OBJ_NOTNULL(vps, req->privs, VRT_PRIVS_MAGIC); return (vrt_priv_dynamic(req->ws, vps, (uintptr_t)vmod_id)); } diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c index bfd4effe6..ae7d44a3d 100644 --- a/bin/varnishd/cache/cache_vrt_vcl.c +++ b/bin/varnishd/cache/cache_vrt_vcl.c @@ -430,7 +430,6 @@ vcl_call_method(struct worker *wrk, struct req *req, struct busyobj *bo, CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->sp, SESS_MAGIC); CHECK_OBJ_NOTNULL(req->vcl, VCL_MAGIC); - CHECK_OBJ_ORNULL(req->top, REQTOP_MAGIC); VCL_Req2Ctx(&ctx, req); } if (bo != NULL) { From phk at FreeBSD.org Tue Jan 15 12:58:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 15 Jan 2019 12:58:08 +0000 (UTC) Subject: [master] f8ef73c32 Add a -m ("Miss") argument which also logs the records which do not match Message-ID: <20190115125808.85CFEB0778@lists.varnish-cache.org> commit f8ef73c329a084879465f36a108901dac7fe64a1 Author: Poul-Henning Kamp Date: Tue Jan 15 12:50:09 2019 +0000 Add a -m ("Miss") argument which also logs the records which do not match diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 43e300acf..521c9ae3b 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -65,6 +65,8 @@ * \-q query * Filter records using a query expression, see ``man vsl-query`` for * more information. + * \-m + * Also emit log records for misses (only for debugging) * * \-start * Start the logexpect thread in the background. @@ -153,6 +155,7 @@ struct logexp { int vxid_last; int tag_last; + int m_arg; int d_arg; enum VSL_grouping_e g_arg; char *query; @@ -308,6 +311,8 @@ logexp_dispatch(struct VSL_data *vsl, struct VSL_transaction * const pt[], if (ok) legend = "match"; + else if (skip && le->m_arg) + legend = "miss"; else if (skip) legend = NULL; else @@ -597,6 +602,10 @@ cmd_logexpect(CMD_ARGS) av++; continue; } + if (!strcmp(*av, "-m")) { + le->m_arg = !le->m_arg; + continue; + } if (!strcmp(*av, "-start")) { logexp_start(le); continue; From phk at FreeBSD.org Tue Jan 15 12:58:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 15 Jan 2019 12:58:08 +0000 (UTC) Subject: [master] c50a0d9f7 Make ESI:included requests start out in the same VCL as the top-req did. Message-ID: <20190115125808.9DB8BB077B@lists.varnish-cache.org> commit c50a0d9f77f2affa544727d0a16fec2105dcf8c0 Author: Poul-Henning Kamp Date: Tue Jan 15 12:55:37 2019 +0000 Make ESI:included requests start out in the same VCL as the top-req did. This is more complex than it sounds because the active VCL at the time the topreq got scheduled, may no longer be by the time the esi:include req gets processed. Fixes: #2849 diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 8f450ee63..ce68faf59 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -454,6 +454,7 @@ struct req { int restarts; int esi_level; struct req *topreq; /* esi_level == 0 request */ + struct vcl *vcl0; #define REQ_FLAG(l, r, w, d) unsigned l:1; #include "tbl/req_flags.h" diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 067d77973..ae8669f21 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -169,7 +169,10 @@ ved_include(struct req *preq, const char *src, const char *host, req->req_body_status = REQ_BODY_NONE; AZ(req->vcl); - req->vcl = preq->vcl; + if (req->topreq->vcl0) + req->vcl = req->topreq->vcl0; + else + req->vcl = preq->vcl; VCL_Ref(req->vcl); req->req_step = R_STP_TRANSPORT; diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index db1c498bc..bf4f9dfaf 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -448,7 +448,7 @@ pan_busyobj(struct vsb *vsb, const struct busyobj *bo) VSB_printf(vsb, "director_resp = director_req,\n"); else VDI_Panic(bo->director_resp, vsb, "director_resp"); - VCL_Panic(vsb, bo->vcl); + VCL_Panic(vsb, "vcl", bo->vcl); VSB_indent(vsb, -2); VSB_printf(vsb, "},\n"); } @@ -512,7 +512,8 @@ pan_req(struct vsb *vsb, const struct req *req) if (req->resp->ws != NULL) pan_http(vsb, "resp", req->resp); - VCL_Panic(vsb, req->vcl); + VCL_Panic(vsb, "vcl", req->vcl); + VCL_Panic(vsb, "vcl0", req->vcl0); if (req->body_oc != NULL) pan_objcore(vsb, "BODY", req->body_oc); diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index 34c99907e..0cd80c20a 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -211,6 +211,7 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->topreq, REQ_MAGIC); assert(sp == req->sp); + AZ(req->vcl0); req->director_hint = NULL; req->restarts = 0; diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index dc6da227f..1768efee1 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -1033,6 +1033,7 @@ CNT_Embark(struct worker *wrk, struct req *req) VSLb(req->vsl, SLT_VCL_use, "%s", VCL_Name(req->vcl)); } + AZ(req->vcl0); AN(req->vcl); } @@ -1043,6 +1044,8 @@ CNT_Request(struct req *req) enum req_fsm_nxt nxt; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + assert(IS_TOPREQ(req) || req->vcl0 == NULL); + wrk = req->wrk; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -1085,6 +1088,8 @@ CNT_Request(struct req *req) } wrk->vsl = NULL; if (nxt == REQ_FSM_DONE) { + if (IS_TOPREQ(req) && req->vcl0 != NULL) + VCL_Rel(&req->vcl0); VCL_TaskLeave(req->vcl, req->privs); AN(req->vsl->wid); VRB_Free(req); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 563fafc18..d85892de3 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -378,7 +378,7 @@ void VRY_Finish(struct req *req, enum vry_finish_flag); VCL_BACKEND VCL_DefaultDirector(const struct vcl *); const struct vrt_backend_probe *VCL_DefaultProbe(const struct vcl *); void VCL_Init(void); -void VCL_Panic(struct vsb *, const struct vcl *); +void VCL_Panic(struct vsb *, const char *nm, const struct vcl *); void VCL_Poll(void); void VCL_Ref(struct vcl *); void VCL_Refresh(struct vcl **); diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 17c56463f..270452132 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -193,14 +193,14 @@ vcl_find(const char *name) /*--------------------------------------------------------------------*/ void -VCL_Panic(struct vsb *vsb, const struct vcl *vcl) +VCL_Panic(struct vsb *vsb, const char *nm, const struct vcl *vcl) { int i; AN(vsb); if (vcl == NULL) return; - VSB_printf(vsb, "vcl = {\n"); + VSB_printf(vsb, "%s = {\n", nm); VSB_indent(vsb, 2); PAN_CheckMagic(vsb, vcl, VCL_MAGIC); VSB_printf(vsb, "name = \"%s\",\n", vcl->loaded_name); diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c index ae7d44a3d..c6eb3a927 100644 --- a/bin/varnishd/cache/cache_vrt_vcl.c +++ b/bin/varnishd/cache/cache_vrt_vcl.c @@ -341,8 +341,17 @@ VRT_vcl_select(VRT_CTX, VCL_VCL vcl) struct req *req = ctx->req; CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); + + if (IS_TOPREQ(req) && req->vcl0 != NULL) + return; // Illega, req-FSM will fail this later. + VCL_TaskLeave(req->vcl, req->privs); - VCL_Rel(&req->vcl); + if (IS_TOPREQ(req)) { + req->vcl0 = req->vcl; + req->vcl = NULL; + } else { + VCL_Rel(&req->vcl); + } vcl_get(&req->vcl, vcl); VSLb(ctx->req->vsl, SLT_VCL_use, "%s via %s", req->vcl->loaded_name, vcl->loaded_name); diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c index deee26649..f754413a2 100644 --- a/bin/varnishd/http1/cache_http1_fsm.c +++ b/bin/varnishd/http1/cache_http1_fsm.c @@ -447,6 +447,7 @@ HTTP1_Session(struct worker *wrk, struct req *req) VCL_TaskEnter(req->vcl, req->privs); if (CNT_Request(req) == REQ_FSM_DISEMBARK) return; + AZ(req->vcl0); req->task.func = NULL; req->task.priv = NULL; AZ(req->ws->r); diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index ee24e65c5..8269d0fb1 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -541,6 +541,7 @@ h2_do_req(struct worker *wrk, void *priv) wrk->stats->client_req++; if (CNT_Request(req) != REQ_FSM_DISEMBARK) { AZ(req->ws->r); + AZ(req->vcl0); h2 = r2->h2sess; CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); Lck_Lock(&h2->sess->mtx); diff --git a/bin/varnishtest/tests/r02849.vtc b/bin/varnishtest/tests/r02849.vtc new file mode 100644 index 000000000..9b4677524 --- /dev/null +++ b/bin/varnishtest/tests/r02849.vtc @@ -0,0 +1,123 @@ +varnishtest "ESI included req's start in the same VCL the top started." + +server s1 { + rxreq + expect req.url == /l3a + txresp -body "_Correct_" + + rxreq + expect req.url == /l3b + txresp -body "_Wrong1_" + + rxreq + expect req.url == /l3c + txresp -body "_Wrong2_" + + rxreq + expect req.url == /l1 + txresp -body {} + + rxreq + expect req.url == /l2 + txresp -body {} +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + if (req.url == "/l3") { + set req.url = "/l3b"; + } + } + sub vcl_backend_response { + set beresp.do_esi = True; + } + sub vcl_deliver { + set resp.http.vcl = "lab1"; + } +} -start + +varnish v1 -cliok "vcl.label lab1 vcl1" + +varnish v1 -vcl+backend { + sub vcl_recv { + if (req.url == "/l3") { + set req.url = "/l3a"; + } + } + sub vcl_backend_response { + set beresp.do_esi = True; + } + sub vcl_deliver { + set resp.http.vcl = "lab2"; + } +} + +varnish v1 -cliok "vcl.label lab2 vcl2" + +varnish v1 -vcl+backend { + sub vcl_recv { + if (req.url == "/l1") { + return (vcl(lab1)); + } + if (req.url == "/l3") { + return (vcl(lab2)); + } + if (req.url == "/l3") { + set req.url = "/l3c"; + } + } + sub vcl_backend_response { + set beresp.do_esi = True; + } + sub vcl_deliver { + set resp.http.vcl = "vcl3"; + } +} + +# First cache the two possible inclusions + +client c1 { + txreq -url /l3a + rxresp + expect resp.http.vcl == vcl3 + txreq -url /l3b + rxresp + expect resp.http.vcl == vcl3 + txreq -url /l3c + rxresp + expect resp.http.vcl == vcl3 +} -run + +varnish v1 -vsl_catchup + +logexpect l1 -v v1 -g raw { + + expect * 1009 VCL_use "vcl1" + expect * 1009 BeReqURL "/l1" + + expect * 1011 VCL_use "vcl3" + expect * 1011 BeReqURL "/l2" + + expect * 1012 ReqURL "/l3" + expect * 1012 ReqURL "/l3" + expect * 1012 VCL_use "vcl2 via lab2" + expect * 1012 ReqURL "/l3a" + + expect * 1010 ReqURL "/l2" + expect * 1010 ReqURL "/l2" + + expect * 1008 VCL_use "vcl3" + expect * 1008 ReqURL "/l1" + expect * 1008 VCL_use "vcl1 via lab1" +} -start + +# The test which one is picked + +client c1 { + txreq -url /l1 + rxresp + expect resp.http.vcl == lab1 + expect resp.body == {_Correct_} +} -run + +logexpect l1 -wait diff --git a/doc/sphinx/users-guide/esi.rst b/doc/sphinx/users-guide/esi.rst index afa9236df..a88e9cc91 100644 --- a/doc/sphinx/users-guide/esi.rst +++ b/doc/sphinx/users-guide/esi.rst @@ -89,8 +89,8 @@ Doing ESI on JSON and other non-XML'ish content ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Varnish will peek at the first byte of an object and if it is not -a "<" Varnish assumes you didn't really mean to ESI process. -You can alter this behaviour by:: +a "<" Varnish assumes you didn't really mean to ESI process it. +You can disable this check by:: param.set feature +esi_disable_xml_check @@ -134,3 +134,11 @@ If you really know what you are doing, you can use this workaround:: set beresp.status = 1206; } } + +ESI and return(vcl(...)) +~~~~~~~~~~~~~~~~~~~~~~~~ + +If the original client request switched to a different VCL using +``return(vcl(...))`` in ``vcl_recv``, any esi:include-requests +will still start out in the same VCL as the original did, *not* +in the one it switched to. From phk at FreeBSD.org Tue Jan 15 13:47:06 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 15 Jan 2019 13:47:06 +0000 (UTC) Subject: [master] e25351cf3 Sync with VTEST Message-ID: <20190115134706.A6D82493A@lists.varnish-cache.org> commit e25351cf3f10dca6def9202074b71a1b054a4a03 Author: Poul-Henning Kamp Date: Tue Jan 15 13:32:16 2019 +0000 Sync with VTEST diff --git a/bin/varnishtest/Makefile.am b/bin/varnishtest/Makefile.am index 357ef9269..b9c1def06 100644 --- a/bin/varnishtest/Makefile.am +++ b/bin/varnishtest/Makefile.am @@ -69,6 +69,8 @@ varnishtest_LDADD = \ varnishtest_CFLAGS = \ @SAN_CFLAGS@ \ + -DVTEST_WITH_VTC_LOGEXPECT \ + -DVTEST_WITH_VTC_VARNISH \ -DTOP_BUILDDIR='"${top_builddir}"' EXTRA_DIST = $(top_srcdir)/bin/varnishtest/tests/*.vtc \ diff --git a/bin/varnishtest/cmds.h b/bin/varnishtest/cmds.h index fd140efe2..8eb2fa4f1 100644 --- a/bin/varnishtest/cmds.h +++ b/bin/varnishtest/cmds.h @@ -44,12 +44,16 @@ CMD_TOP(client) CMD_TOP(err_shell) CMD_TOP(feature) CMD_TOP(haproxy) +#ifdef VTEST_WITH_VTC_LOGEXPECT CMD_TOP(logexpect) +#endif CMD_TOP(process) CMD_TOP(server) CMD_TOP(setenv) CMD_TOP(syslog) +#ifdef VTEST_WITH_VTC_VARNISH CMD_TOP(varnish) +#endif CMD_TOP(varnishtest) CMD_TOP(vtest) #undef CMD_TOP diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 521c9ae3b..11f046f95 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -26,6 +26,8 @@ * SUCH DAMAGE. */ +#ifdef VTEST_WITH_VTC_LOGEXPECT + /* SECTION: logexpect logexpect * * Reads the VSL and looks for records matching a given specification. It will @@ -628,3 +630,5 @@ cmd_logexpect(CMD_ARGS) logexp_spec(le, *av); } } + +#endif /* VTEST_WITH_VTC_LOGEXPECT */ diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 4aa4680a5..02f6da13f 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -26,6 +26,8 @@ * SUCH DAMAGE. */ +#ifdef VTEST_WITH_VTC_VARNISH + #include "config.h" #include @@ -1237,3 +1239,5 @@ cmd_varnish(CMD_ARGS) vtc_fatal(v->vl, "Unknown varnish argument: %s", *av); } } + +#endif /* VTEST_WITH_VTC_VARNISH */ From phk at FreeBSD.org Tue Jan 15 13:47:06 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 15 Jan 2019 13:47:06 +0000 (UTC) Subject: [master] b336becf1 Sync with VTEST Message-ID: <20190115134706.C1FEB493D@lists.varnish-cache.org> commit b336becf101895db9faca5615e005244e92a1fda Author: Poul-Henning Kamp Date: Tue Jan 15 13:45:27 2019 +0000 Sync with VTEST diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index ff9376018..2a52de427 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -478,7 +478,7 @@ haproxy_new(const char *name) h->cli = haproxy_cli_new(h); AN(h->cli); - bprintf(buf, "rm -rf %s ; mkdir -p %s", h->workdir, h->workdir); + bprintf(buf, "rm -rf \"%s\" ; mkdir -p \"%s\"", h->workdir, h->workdir); AZ(system(buf)); VTAILQ_INSERT_TAIL(&haproxies, h, list); @@ -499,7 +499,7 @@ haproxy_delete(struct haproxy *h) vtc_logclose(h->vl); if (!leave_temp) { - bprintf(buf, "rm -rf %s", h->workdir); + bprintf(buf, "rm -rf \"%s\"", h->workdir); AZ(system(buf)); } @@ -549,7 +549,7 @@ haproxy_start(struct haproxy *h) vsb = VSB_new_auto(); AN(vsb); - VSB_printf(vsb, "exec %s", h->filename); + VSB_printf(vsb, "exec \"%s\"", h->filename); if (h->opt_check_mode) VSB_printf(vsb, " -c"); else if (h->opt_daemon) @@ -562,13 +562,13 @@ haproxy_start(struct haproxy *h) VSB_printf(vsb, " %s", VSB_data(h->args)); - VSB_printf(vsb, " -f %s ", h->cfg_fn); + VSB_printf(vsb, " -f \"%s\" ", h->cfg_fn); if (h->opt_worker || h->opt_daemon) { bprintf(buf, "%s/pid", h->workdir); h->pid_fn = strdup(buf); AN(h->pid_fn); - VSB_printf(vsb, " -p %s", h->pid_fn); + VSB_printf(vsb, " -p \"%s\"", h->pid_fn); } AZ(VSB_finish(vsb)); @@ -755,7 +755,7 @@ haproxy_write_conf(const struct haproxy *h, const char *cfg, int auto_be) vsb2 = VSB_new_auto(); AN(vsb2); - VSB_printf(vsb, " global\n\tstats socket %s " + VSB_printf(vsb, " global\n\tstats socket \"%s\" " "level admin mode 600\n", h->cli_fn); VSB_printf(vsb, " stats socket \"fd@${cli}\" level admin\n"); AZ(VSB_cat(vsb, cfg)); diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index e01ebc181..a8461530d 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -796,8 +796,11 @@ cmd_http_gunzip(CMD_ARGS) static void gzip_body(const struct http *hp, const char *txt, char **body, int *bodylen) { - int l, i; + int l; z_stream vz; +#ifdef VGZ_EXTENSIONS + int i; +#endif memset(&vz, 0, sizeof vz); From phk at FreeBSD.org Tue Jan 15 13:47:06 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 15 Jan 2019 13:47:06 +0000 (UTC) Subject: [master] e4f4f69d7 Synchronize with VTEST Message-ID: <20190115134706.E1FCD4941@lists.varnish-cache.org> commit e4f4f69d7cf3c856a765f3cddfd2350b4af81db4 Author: Poul-Henning Kamp Date: Tue Jan 15 13:45:58 2019 +0000 Synchronize with VTEST diff --git a/lib/libvarnish/vfil.c b/lib/libvarnish/vfil.c index 4dfc30faf..8e34c9cb8 100644 --- a/lib/libvarnish/vfil.c +++ b/lib/libvarnish/vfil.c @@ -49,7 +49,7 @@ #ifdef HAVE_SYS_VFS_H # include #endif -#if defined(__linux__) && defined(HAVE_FALLOCATE) +#ifdef HAVE_FALLOCATE # include #endif From phk at FreeBSD.org Tue Jan 15 21:54:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 15 Jan 2019 21:54:08 +0000 (UTC) Subject: [master] 45d4464d0 Remove a wrong vintage 2010 optimization (5f7d80c5034c3ce) Message-ID: <20190115215408.D3635915E1@lists.varnish-cache.org> commit 45d4464d0f12123cef34719979730ad21debbe01 Author: Poul-Henning Kamp Date: Tue Jan 15 21:47:17 2019 +0000 Remove a wrong vintage 2010 optimization (5f7d80c5034c3ce) Solaris, (remember Solaris?) will return connection related errnos on fd metadata operations such as ioctl(2), [gs]etsockopt(2) etc. We propagated these errors up from VTC_(non)blocking() which could, theoretically, bail out earlier than it would otherwise have done. This complicated code far more than it speeded anything up. Instead just ensure that any errors returned are indeed connection related, and then ignore them, and let Solaris deal with the broken connection same way as other operating systems. diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c index 972595ea0..6ee32f809 100644 --- a/lib/libvarnish/vtcp.c +++ b/lib/libvarnish/vtcp.c @@ -213,9 +213,11 @@ VTCP_fastopen(int sock, int depth) * us to do two syscalls, one to get and one to set, the latter of * which mucks about a bit before it ends up calling ioctl(FIONBIO), * at least on FreeBSD. + * On Solaris ioctl(FIONBIO) can fail with connection related errnos, + * but as long as that is how they fail, we're fine. */ -int +void VTCP_blocking(int sock) { int i, j; @@ -223,10 +225,9 @@ VTCP_blocking(int sock) i = 0; j = ioctl(sock, FIONBIO, &i); VTCP_Assert(j); - return (j); } -int +void VTCP_nonblocking(int sock) { int i, j; @@ -234,7 +235,6 @@ VTCP_nonblocking(int sock) i = 1; j = ioctl(sock, FIONBIO, &i); VTCP_Assert(j); - return (j); } /*-------------------------------------------------------------------- From phk at FreeBSD.org Tue Jan 15 22:16:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 15 Jan 2019 22:16:07 +0000 (UTC) Subject: [master] 8d6c4a1b5 The rest of previous commit. Message-ID: <20190115221607.1672491F2E@lists.varnish-cache.org> commit 8d6c4a1b5bdc70aa663efdaf45a4912ee092c405 Author: Poul-Henning Kamp Date: Tue Jan 15 22:14:43 2019 +0000 The rest of previous commit. diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index ba4e84be9..f47bd57e0 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -368,12 +368,7 @@ vca_make_session(struct worker *wrk, void *arg) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CAST_OBJ_NOTNULL(wa, arg, WRK_ACCEPT_MAGIC); - if (VTCP_blocking(wa->acceptsock)) { - closefd(&wa->acceptsock); - wrk->stats->sess_drop++; // XXX Better counter ? - WS_Release(wrk->aws, 0); - return; - } + VTCP_blocking(wa->acceptsock); /* Turn accepted socket into a session */ AN(wrk->aws->r); diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 470b769b3..bf3fd14ff 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -440,10 +440,7 @@ SES_Wait(struct sess *sp, const struct transport *xp) * XXX: waiter_epoll prevents us from zeroing the struct because * XXX: it keeps state across calls. */ - if (VTCP_nonblocking(sp->fd)) { - SES_Delete(sp, SC_REM_CLOSE, NAN); - return; - } + VTCP_nonblocking(sp->fd); /* * put struct waited on the workspace diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c index f754413a2..05c1c4f06 100644 --- a/bin/varnishd/http1/cache_http1_fsm.c +++ b/bin/varnishd/http1/cache_http1_fsm.c @@ -176,30 +176,6 @@ http1_req_fail(struct req *req, enum sess_close reason) SES_Close(req->sp, reason); } -/*---------------------------------------------------------------------- - */ - -static int -http1_req_cleanup(struct sess *sp, struct worker *wrk, struct req *req) -{ - AZ(wrk->aws->r); - AZ(req->ws->r); - Req_Cleanup(sp, wrk, req); - - if (sp->fd >= 0 && req->doclose != SC_NULL) - SES_Close(sp, req->doclose); - - if (sp->fd < 0) { - wrk->stats->sess_closed++; - AZ(req->vcl); - Req_Release(req); - SES_Delete(sp, SC_NULL, NAN); - return (1); - } - - return (0); -} - static int v_matchproto_(vtr_minimal_response_f) http1_minimal_response(struct req *req, uint16_t status) { @@ -336,18 +312,9 @@ HTTP1_Session(struct worker *wrk, struct req *req) * Whenever we come in from the acceptor or waiter, we need to set * blocking mode. It would be simpler to do this in the acceptor * or waiter, but we'd rather do the syscall in the worker thread. - * On systems which return errors for ioctl, we close early */ - if (http1_getstate(sp) == H1NEWREQ && VTCP_blocking(sp->fd)) { - AN(req->htc->ws->r); - if (errno == ECONNRESET) - SES_Close(sp, SC_REM_CLOSE); - else - SES_Close(sp, SC_TX_ERROR); - WS_Release(req->htc->ws, 0); - AN(http1_req_cleanup(sp, wrk, req)); - return; - } + if (http1_getstate(sp) == H1NEWREQ) + VTCP_blocking(sp->fd); req->transport = &HTTP1_transport; @@ -454,8 +421,22 @@ HTTP1_Session(struct worker *wrk, struct req *req) AZ(wrk->aws->r); http1_setstate(sp, H1CLEANUP); } else if (st == H1CLEANUP) { - if (http1_req_cleanup(sp, wrk, req)) + + AZ(wrk->aws->r); + AZ(req->ws->r); + + if (sp->fd >= 0 && req->doclose != SC_NULL) + SES_Close(sp, req->doclose); + + if (sp->fd < 0) { + wrk->stats->sess_closed++; + Req_Cleanup(sp, wrk, req); + Req_Release(req); + SES_Delete(sp, SC_NULL, NAN); return; + } + + Req_Cleanup(sp, wrk, req); HTC_RxInit(req->htc, req->ws); if (req->htc->rxbuf_e != req->htc->rxbuf_b) wrk->stats->sess_readahead++; diff --git a/include/vtcp.h b/include/vtcp.h index 29f83cb75..a5431aba6 100644 --- a/include/vtcp.h +++ b/include/vtcp.h @@ -45,8 +45,8 @@ void VTCP_hisname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen); int VTCP_filter_http(int sock); int VTCP_fastopen(int sock, int depth); -int VTCP_blocking(int sock); -int VTCP_nonblocking(int sock); +void VTCP_blocking(int sock); +void VTCP_nonblocking(int sock); int VTCP_linger(int sock, int linger); int VTCP_check_hup(int sock); From phk at FreeBSD.org Wed Jan 16 09:58:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 16 Jan 2019 09:58:08 +0000 (UTC) Subject: [master] 78836dc70 Flexelinting Message-ID: <20190116095808.70294A4109@lists.varnish-cache.org> commit 78836dc701f3104fb74c88baf2c1d03e8b930493 Author: Poul-Henning Kamp Date: Wed Jan 16 09:56:57 2019 +0000 Flexelinting diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index 8b308c03f..00f0cb0d0 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -394,9 +394,11 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym) vf = VTAILQ_NEXT(vf, list); buf = VSB_new_auto(); + AN(buf); VSB_printf(buf, ", &%s, \"%s\"", sy1->rname, sy1->name); AZ(VSB_finish(buf)); vcc_Eval_Func(tl, vf, VSB_data(buf), sy2); + VSB_destroy(&buf); ERRCHK(tl); SkipToken(tl, ';'); sy1->def_e = tl->t; @@ -413,7 +415,8 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym) VSB_printf(ifp->fin, "\t\t%s(&%s);", vf->value, sy1->rname); /* Instantiate symbols for the methods */ - VSB_clear(buf); + buf = VSB_new_auto(); + AN(buf); VSB_printf(buf, ", %s", sy1->rname); AZ(VSB_finish(buf)); p = TlDup(tl, VSB_data(buf)); From phk at FreeBSD.org Wed Jan 16 22:56:06 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 16 Jan 2019 22:56:06 +0000 (UTC) Subject: [master] 7609d3f03 Use the vdp priv as intended Message-ID: <20190116225606.804016417@lists.varnish-cache.org> commit 7609d3f03a915714e40fd449ccd96b1e6d3148c6 Author: Poul-Henning Kamp Date: Wed Jan 16 20:58:15 2019 +0000 Use the vdp priv as intended diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index 20a300811..f19565011 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -72,17 +72,25 @@ V2D_Init(void) /**********************************************************************/ +static int v_matchproto_(vdp_init_f) +h2_init(struct req *req, void **priv) +{ + + *priv = req->transport_priv; + return (0); +} + static int v_matchproto_(vdp_fini_f) h2_fini(struct req *req, void **priv) { struct h2_req *r2; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - (void)priv; - CAST_OBJ_NOTNULL(r2, req->transport_priv, H2_REQ_MAGIC); + CAST_OBJ_NOTNULL(r2, *priv, H2_REQ_MAGIC); H2_Send_Get(req->wrk, r2->h2sess, r2); H2_Send(req->wrk, r2, H2_F_DATA, H2FF_DATA_END_STREAM, 0, ""); H2_Send_Rel(r2->h2sess, r2); + *priv = NULL; return (0); } @@ -93,9 +101,8 @@ h2_bytes(struct req *req, enum vdp_action act, void **priv, struct h2_req *r2; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - CAST_OBJ_NOTNULL(r2, req->transport_priv, H2_REQ_MAGIC); + CAST_OBJ_NOTNULL(r2, *priv, H2_REQ_MAGIC); (void)act; - (void)priv; if ((r2->h2sess->error || r2->error)) return (-1); @@ -108,6 +115,7 @@ h2_bytes(struct req *req, enum vdp_action act, void **priv, static const struct vdp h2_vdp = { .name = "H2B", + .init = h2_init, .bytes = h2_bytes, .fini = h2_fini, }; From phk at FreeBSD.org Wed Jan 16 22:56:06 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 16 Jan 2019 22:56:06 +0000 (UTC) Subject: [master] e7d497e27 Polish Message-ID: <20190116225606.A073E641A@lists.varnish-cache.org> commit e7d497e270d4f388ff146a5caa6d6a0e02ed2f52 Author: Poul-Henning Kamp Date: Wed Jan 16 21:50:15 2019 +0000 Polish diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index ae8669f21..6d00c6efa 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -64,6 +64,7 @@ struct ecx { int woken; struct req *preq; + struct ecx *pecx; ssize_t l_crc; uint32_t crc; }; @@ -264,6 +265,13 @@ ved_vdp_esi_init(struct req *req, void **priv) req->res_mode |= RES_ESI; if (req->resp_len != 0) req->resp_len = -1; + if (req->esi_level > 0) { + assert(req->transport == &VED_transport); + CAST_OBJ_NOTNULL(ecx->pecx, req->transport_priv, ECX_MAGIC); + if (!ecx->pecx->isgzip) + ecx->pecx = NULL; + } + return (0); } @@ -292,15 +300,9 @@ ved_vdp_esi_bytes(struct req *req, enum vdp_action act, void **priv, int retval = 0; CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC); + pecx = ecx->pecx; pp = ptr; - if (req->esi_level > 0) { - assert(req->transport == &VED_transport); - CAST_OBJ_NOTNULL(pecx, req->transport_priv, ECX_MAGIC); - if (!pecx->isgzip) - pecx = NULL; - } - while (1) { switch (ecx->state) { case 0: @@ -455,11 +457,11 @@ const struct vdp VDP_esi = { * Push bytes to preq */ static inline int -ved_bytes(struct req *req, struct req *preq, enum vdp_action act, +ved_bytes(struct req *req, struct ecx *ecx, enum vdp_action act, const void *ptr, ssize_t len) { req->acct.resp_bodybytes += len; - return (VDP_bytes(preq, act, ptr, len)); + return (VDP_bytes(ecx->preq, act, ptr, len)); } /*--------------------------------------------------------------------- @@ -496,15 +498,13 @@ ved_pretend_gzip_bytes(struct req *req, enum vdp_action act, void **priv, const uint8_t *p; uint16_t lx; struct ecx *ecx; - struct req *preq; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC); - preq = ecx->preq; (void)priv; if (l == 0) - return (ved_bytes(req, ecx->preq, act, pv, l)); + return (ved_bytes(req, ecx, act, pv, l)); p = pv; @@ -520,23 +520,23 @@ ved_pretend_gzip_bytes(struct req *req, enum vdp_action act, void **priv, while (l > 0) { if (l >= 65535) { lx = 65535; - if (ved_bytes(req, preq, VDP_NULL, buf1, sizeof buf1)) + if (ved_bytes(req, ecx, VDP_NULL, buf1, sizeof buf1)) return (-1); } else { lx = (uint16_t)l; buf2[0] = 0; vle16enc(buf2 + 1, lx); vle16enc(buf2 + 3, ~lx); - if (ved_bytes(req, preq, VDP_NULL, buf2, sizeof buf2)) + if (ved_bytes(req, ecx, VDP_NULL, buf2, sizeof buf2)) return (-1); } - if (ved_bytes(req, preq, VDP_NULL, p, lx)) + if (ved_bytes(req, ecx, VDP_NULL, p, lx)) return (-1); l -= lx; p += lx; } /* buf2 is local, have to flush */ - return (ved_bytes(req, preq, VDP_FLUSH, NULL, 0)); + return (ved_bytes(req, ecx, VDP_FLUSH, NULL, 0)); } static const struct vdp ved_vdp_pgz = { @@ -567,8 +567,8 @@ static const struct vdp ved_vdp_pgz = { struct ved_foo { unsigned magic; #define VED_FOO_MAGIC 0x6a5a262d + struct ecx *ecx; struct req *req; - struct req *preq; ssize_t start, last, stop, lpad; ssize_t ll; uint64_t olen; @@ -605,7 +605,7 @@ ved_objiterate(void *priv, unsigned flush, const void *ptr, ssize_t len) if (dl > 0) { if (dl > len) dl = len; - if (ved_bytes(foo->req, foo->preq, VDP_NULL, pp, dl)) + if (ved_bytes(foo->req, foo->ecx, VDP_NULL, pp, dl)) return(-1); foo->ll += dl; len -= dl; @@ -616,7 +616,7 @@ ved_objiterate(void *priv, unsigned flush, const void *ptr, ssize_t len) /* Remove the "LAST" bit */ foo->dbits[0] = *pp; foo->dbits[0] &= ~(1U << (foo->last & 7)); - if (ved_bytes(foo->req, foo->preq, VDP_NULL, foo->dbits, 1)) + if (ved_bytes(foo->req, foo->ecx, VDP_NULL, foo->dbits, 1)) return (-1); foo->ll++; len--; @@ -628,7 +628,7 @@ ved_objiterate(void *priv, unsigned flush, const void *ptr, ssize_t len) if (dl > 0) { if (dl > len) dl = len; - if (ved_bytes(foo->req, foo->preq, VDP_NULL, pp, dl)) + if (ved_bytes(foo->req, foo->ecx, VDP_NULL, pp, dl)) return (-1); foo->ll += dl; len -= dl; @@ -690,7 +690,7 @@ ved_objiterate(void *priv, unsigned flush, const void *ptr, ssize_t len) default: WRONG("compiler must be broken"); } - if (ved_bytes(foo->req, foo->preq, + if (ved_bytes(foo->req, foo->ecx, VDP_NULL, foo->dbits + 1, foo->lpad)) return (-1); } @@ -731,8 +731,8 @@ ved_stripgzip(struct req *req, const struct boc *boc) CAST_OBJ_NOTNULL(ecx, req->transport_priv, ECX_MAGIC); INIT_OBJ(&foo, VED_FOO_MAGIC); + foo.ecx = ecx; foo.req = req; - foo.preq = ecx->preq; memset(foo.tailbuf, 0xdd, sizeof foo.tailbuf); /* OA_GZIPBITS is not valid until BOS_FINISHED */ @@ -773,7 +773,7 @@ ved_stripgzip(struct req *req, const struct boc *boc) foo.dbits = dbits; (void)ObjIterate(req->wrk, req->objcore, &foo, ved_objiterate, 0); /* XXX: error check ?? */ - (void)ved_bytes(req, foo.preq, VDP_FLUSH, NULL, 0); + (void)ved_bytes(req, ecx, VDP_FLUSH, NULL, 0); icrc = vle32dec(foo.tailbuf); ilen = vle32dec(foo.tailbuf + 4); @@ -796,11 +796,11 @@ static int v_matchproto_(vdp_bytes_f) ved_vdp_bytes(struct req *req, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { - struct req *preq; + struct ecx *ecx; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - CAST_OBJ_NOTNULL(preq, *priv, REQ_MAGIC); - return (ved_bytes(req, preq, act, ptr, len)); + CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC); + return (ved_bytes(req, ecx, act, ptr, len)); } static const struct vdp ved_ved = { @@ -836,7 +836,7 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody) if (ecx->isgzip && !i) (void)VDP_Push(req, &ved_vdp_pgz, ecx); else - (void)VDP_Push(req, &ved_ved, ecx->preq); + (void)VDP_Push(req, &ved_ved, ecx); (void)VDP_DeliverObj(req); (void)VDP_bytes(req, VDP_FLUSH, NULL, 0); } From phk at FreeBSD.org Wed Jan 16 22:56:06 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 16 Jan 2019 22:56:06 +0000 (UTC) Subject: [master] bc292854a Polish Message-ID: <20190116225606.BE1E2641E@lists.varnish-cache.org> commit bc292854a2b70d5ada3fea5ad7af413efbdf0df0 Author: Poul-Henning Kamp Date: Wed Jan 16 21:57:26 2019 +0000 Polish diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 6d00c6efa..dd1894dc7 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -296,11 +296,10 @@ ved_vdp_esi_bytes(struct req *req, enum vdp_action act, void **priv, uint32_t icrc = 0; uint8_t tailbuf[8 + 5]; const uint8_t *pp; - struct ecx *ecx, *pecx = NULL; + struct ecx *ecx; int retval = 0; CAST_OBJ_NOTNULL(ecx, *priv, ECX_MAGIC); - pecx = ecx->pecx; pp = ptr; while (1) { @@ -313,7 +312,7 @@ ved_vdp_esi_bytes(struct req *req, enum vdp_action act, void **priv, ecx->e = ecx->p + l; if (*ecx->p == VEC_GZ) { - if (pecx == NULL) + if (ecx->pecx == NULL) retval = VDP_bytes(req, VDP_NULL, gzip_hdr, 10); ecx->l_crc = 0; @@ -384,7 +383,7 @@ ved_vdp_esi_bytes(struct req *req, enum vdp_action act, void **priv, } break; case 2: - if (ecx->isgzip && pecx == NULL) { + if (ecx->isgzip && ecx->pecx == NULL) { /* * We are bytealigned here, so simply emit * a gzip literal block with finish bit set. @@ -402,10 +401,10 @@ ved_vdp_esi_bytes(struct req *req, enum vdp_action act, void **priv, vle32enc(tailbuf + 9, ecx->l_crc); (void)VDP_bytes(req, VDP_NULL, tailbuf, 13); - } else if (pecx != NULL) { - pecx->crc = crc32_combine(pecx->crc, + } else if (ecx->pecx != NULL) { + ecx->pecx->crc = crc32_combine(ecx->pecx->crc, ecx->crc, ecx->l_crc); - pecx->l_crc += ecx->l_crc; + ecx->pecx->l_crc += ecx->l_crc; } retval = VDP_bytes(req, VDP_FLUSH, NULL, 0); ecx->state = 99; From phk at FreeBSD.org Wed Jan 16 22:56:06 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 16 Jan 2019 22:56:06 +0000 (UTC) Subject: [master] 72c46cdcf No return value from VTCP_[non]blocking() to throw away Message-ID: <20190116225606.DC44A6422@lists.varnish-cache.org> commit 72c46cdcf72da369c742f012f12f9a10f37058dc Author: Poul-Henning Kamp Date: Wed Jan 16 21:57:34 2019 +0000 No return value from VTCP_[non]blocking() to throw away diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index f47bd57e0..f80d4b2f4 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -380,7 +380,7 @@ vca_make_session(struct worker *wrk, void *arg) * to send an intelligent message back. */ vca_pace_bad(); - (void)VTCP_nonblocking(wa->acceptsock); + VTCP_nonblocking(wa->acceptsock); closefd(&wa->acceptsock); wrk->stats->sess_drop++; WS_Release(wrk->aws, 0); diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c index d8318b444..c5ccee3b0 100644 --- a/bin/varnishd/http1/cache_http1_fetch.c +++ b/bin/varnishd/http1/cache_http1_fetch.c @@ -97,7 +97,7 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes, VSLb(bo->vsl, SLT_BackendStart, "%s %s", abuf, pbuf); - (void)VTCP_blocking(*htc->rfd); /* XXX: we should timeout instead */ + VTCP_blocking(*htc->rfd); /* XXX: we should timeout instead */ V1L_Open(wrk, wrk->aws, htc->rfd, bo->vsl, bo->t_prev, 0); hdrbytes = HTTP1_Write(wrk, hp, HTTP1_Req); diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 8269d0fb1..5b6ce30c0 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -1076,7 +1076,7 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2) char b[8]; ASSERT_RXTHR(h2); - (void)VTCP_blocking(*h2->htc->rfd); + VTCP_blocking(*h2->htc->rfd); h2->sess->t_idle = VTIM_real(); hs = HTC_RxStuff(h2->htc, h2_frame_complete, NULL, NULL, NAN, diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index 7a7ac31f5..e7fc1e5ae 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -146,7 +146,7 @@ barrier_sock_thread(void *priv) b->name, err, strerror(errno), errno); } assert(sock > 0); - (void)VTCP_nonblocking(sock); + VTCP_nonblocking(sock); VTCP_myname(sock, abuf, sizeof abuf, pbuf, sizeof pbuf); macro_def(vl, b->name, "addr", "%s", abuf); diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c index 6966e462b..72b7cfb71 100644 --- a/bin/varnishtest/vtc_client.c +++ b/bin/varnishtest/vtc_client.c @@ -148,7 +148,7 @@ uds_open(void *priv, const struct sockaddr_un *uds) if (s < 0) return (s); - (void) VTCP_nonblocking(s); + VTCP_nonblocking(s); i = connect(s, (const void*)uds, sl); if (i == 0) return(s); @@ -229,7 +229,7 @@ client_thread(void *priv) vtc_fatal(c->vl, "Failed to open %s: %s", VSB_data(vsb), err); /* VTCP_blocking does its own checks, trust it */ - (void)VTCP_blocking(fd); + VTCP_blocking(fd); if (c->proxy_spec != NULL) client_proxy(vl, fd, c->proxy_version, c->proxy_spec); if (! c->keepalive) diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index 2a52de427..2fee33b30 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -300,7 +300,7 @@ haproxy_cli_thread(void *priv) if (fd < 0) vtc_fatal(hc->vl, "CLI failed to open %s: %s", VSB_data(vsb), err); - (void)VTCP_blocking(fd); + VTCP_blocking(fd); hc->sock = fd; parse_string(hc->spec, haproxy_cli_cmds, hc, hc->vl); vtc_log(hc->vl, 2, "CLI ending"); diff --git a/bin/varnishtest/vtc_subr.c b/bin/varnishtest/vtc_subr.c index 0e5922978..ffcc51ef7 100644 --- a/bin/varnishtest/vtc_subr.c +++ b/bin/varnishtest/vtc_subr.c @@ -192,7 +192,7 @@ vtc_record(struct vtclog *vl, int fd, struct vsb *vsb) struct pollfd fds[1]; int i; - (void)VTCP_nonblocking(fd); + VTCP_nonblocking(fd); while (1) { memset(fds, 0, sizeof fds); fds->fd = fd; diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c index 6ee32f809..baf61f27e 100644 --- a/lib/libvarnish/vtcp.c +++ b/lib/libvarnish/vtcp.c @@ -264,7 +264,7 @@ VTCP_connected(int s) return (-1); } - (void)VTCP_blocking(s); + VTCP_blocking(s); return (s); } @@ -291,7 +291,7 @@ VTCP_connect(const struct suckaddr *name, int msec) /* Set the socket non-blocking */ if (msec != 0) - (void)VTCP_nonblocking(s); + VTCP_nonblocking(s); val = 1; AZ(setsockopt(s, IPPROTO_TCP, TCP_NODELAY, &val, sizeof val)); diff --git a/lib/libvarnish/vus.c b/lib/libvarnish/vus.c index ba8a3d507..12ee9af31 100644 --- a/lib/libvarnish/vus.c +++ b/lib/libvarnish/vus.c @@ -118,7 +118,7 @@ VUS_connect(const char *path, int msec) /* Set the socket non-blocking */ if (msec != 0) - (void)VTCP_nonblocking(s); + VTCP_nonblocking(s); i = connect(s, (const void*)&uds, sl); if (i == 0) From phk at FreeBSD.org Wed Jan 16 22:56:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 16 Jan 2019 22:56:07 +0000 (UTC) Subject: [master] 3fbdda3d7 Polish Message-ID: <20190116225607.0386A6426@lists.varnish-cache.org> commit 3fbdda3d7335e806dd22ebde49a6fb7ceda6e14a Author: Poul-Henning Kamp Date: Wed Jan 16 22:32:56 2019 +0000 Polish diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index dd1894dc7..66751b280 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -571,7 +571,7 @@ struct ved_foo { ssize_t start, last, stop, lpad; ssize_t ll; uint64_t olen; - uint8_t *dbits; + uint8_t dbits[8]; uint8_t tailbuf[8]; }; @@ -721,18 +721,17 @@ ved_stripgzip(struct req *req, const struct boc *boc) const char *p; uint32_t icrc; uint32_t ilen; - uint8_t *dbits; struct ecx *ecx; - struct ved_foo foo; + struct ved_foo foo[1]; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); CAST_OBJ_NOTNULL(ecx, req->transport_priv, ECX_MAGIC); - INIT_OBJ(&foo, VED_FOO_MAGIC); - foo.ecx = ecx; - foo.req = req; - memset(foo.tailbuf, 0xdd, sizeof foo.tailbuf); + INIT_OBJ(foo, VED_FOO_MAGIC); + foo->ecx = ecx; + foo->req = req; + memset(foo->tailbuf, 0xdd, sizeof foo->tailbuf); /* OA_GZIPBITS is not valid until BOS_FINISHED */ if (boc != NULL) @@ -754,28 +753,25 @@ ved_stripgzip(struct req *req, const struct boc *boc) p = ObjGetAttr(req->wrk, req->objcore, OA_GZIPBITS, &l); AN(p); assert(l == 32); - foo.start = vbe64dec(p); - foo.last = vbe64dec(p + 8); - foo.stop = vbe64dec(p + 16); - foo.olen = ObjGetLen(req->wrk, req->objcore); - assert(foo.start > 0 && foo.start < foo.olen * 8); - assert(foo.last > 0 && foo.last < foo.olen * 8); - assert(foo.stop > 0 && foo.stop < foo.olen * 8); - assert(foo.last >= foo.start); - assert(foo.last < foo.stop); + foo->start = vbe64dec(p); + foo->last = vbe64dec(p + 8); + foo->stop = vbe64dec(p + 16); + foo->olen = ObjGetLen(req->wrk, req->objcore); + assert(foo->start > 0 && foo->start < foo->olen * 8); + assert(foo->last > 0 && foo->last < foo->olen * 8); + assert(foo->stop > 0 && foo->stop < foo->olen * 8); + assert(foo->last >= foo->start); + assert(foo->last < foo->stop); /* The start bit must be byte aligned. */ - AZ(foo.start & 7); + AZ(foo->start & 7); - dbits = WS_Alloc(req->ws, 8); - AN(dbits); - foo.dbits = dbits; (void)ObjIterate(req->wrk, req->objcore, &foo, ved_objiterate, 0); /* XXX: error check ?? */ (void)ved_bytes(req, ecx, VDP_FLUSH, NULL, 0); - icrc = vle32dec(foo.tailbuf); - ilen = vle32dec(foo.tailbuf + 4); + icrc = vle32dec(foo->tailbuf); + ilen = vle32dec(foo->tailbuf + 4); ecx->crc = crc32_combine(ecx->crc, icrc, ilen); ecx->l_crc += ilen; From phk at FreeBSD.org Wed Jan 16 22:56:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 16 Jan 2019 22:56:07 +0000 (UTC) Subject: [master] d42817991 Reorganize the "include gzip in gzip" path as VDP Message-ID: <20190116225607.2FF31642A@lists.varnish-cache.org> commit d428179919e311aef312e99cb013068f38272351 Author: Poul-Henning Kamp Date: Wed Jan 16 22:54:36 2019 +0000 Reorganize the "include gzip in gzip" path as VDP diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 66751b280..360bd3c2b 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -547,44 +547,35 @@ static const struct vdp ved_vdp_pgz = { /*--------------------------------------------------------------------- * Include an object in a gzip'ed ESI object delivery * - * This is not written as a VDP (yet) because it relies on the - * OA_GZIPBITS which only becomes available when the input side - * has fully digested the object and located the magic bit positions. + * This is the interesting case: Deliver all the deflate blocks, stripping + * the "LAST" bit of the last one and padding it, as necessary, to a byte + * boundary. * - * We can improve this two ways. - * - * One is to run a gunzip instance here, to find the stopbit ourselves, - * but that would be double work, in particular when passing a gziped - * object, where we would have two null-gunzips. - * - * The other is to have the input side guarantee that OA_GZIPBITS::stopbit - * always is committed before the chunk of data containing it. We would - * be required to poll OA_GZIPBITS on every chunk presented, but that is - * much cheaper than running a gunzip instance. */ struct ved_foo { unsigned magic; #define VED_FOO_MAGIC 0x6a5a262d struct ecx *ecx; - struct req *req; - ssize_t start, last, stop, lpad; - ssize_t ll; - uint64_t olen; - uint8_t dbits[8]; - uint8_t tailbuf[8]; + ssize_t start, last, stop, lpad; + ssize_t ll; + uint64_t olen; + uint8_t dbits[8]; + uint8_t tailbuf[8]; }; -static int v_matchproto_(objiterate_f) -ved_objiterate(void *priv, unsigned flush, const void *ptr, ssize_t len) +static int v_matchproto_(vdp_bytes_f) +ved_zap_bytes(struct req *req, enum vdp_action act, void **priv, + const void *ptr, ssize_t len) { struct ved_foo *foo; const uint8_t *pp; ssize_t dl; ssize_t l; - CAST_OBJ_NOTNULL(foo, priv, VED_FOO_MAGIC); - (void)flush; + CAST_OBJ_NOTNULL(foo, *priv, VED_FOO_MAGIC); + (void)req; + (void)act; pp = ptr; if (len > 0) { /* Skip over the GZIP header */ @@ -604,7 +595,7 @@ ved_objiterate(void *priv, unsigned flush, const void *ptr, ssize_t len) if (dl > 0) { if (dl > len) dl = len; - if (ved_bytes(foo->req, foo->ecx, VDP_NULL, pp, dl)) + if (ved_bytes(req, foo->ecx, VDP_NULL, pp, dl)) return(-1); foo->ll += dl; len -= dl; @@ -615,7 +606,7 @@ ved_objiterate(void *priv, unsigned flush, const void *ptr, ssize_t len) /* Remove the "LAST" bit */ foo->dbits[0] = *pp; foo->dbits[0] &= ~(1U << (foo->last & 7)); - if (ved_bytes(foo->req, foo->ecx, VDP_NULL, foo->dbits, 1)) + if (ved_bytes(req, foo->ecx, VDP_NULL, foo->dbits, 1)) return (-1); foo->ll++; len--; @@ -627,7 +618,7 @@ ved_objiterate(void *priv, unsigned flush, const void *ptr, ssize_t len) if (dl > 0) { if (dl > len) dl = len; - if (ved_bytes(foo->req, foo->ecx, VDP_NULL, pp, dl)) + if (ved_bytes(req, foo->ecx, VDP_NULL, pp, dl)) return (-1); foo->ll += dl; len -= dl; @@ -689,7 +680,7 @@ ved_objiterate(void *priv, unsigned flush, const void *ptr, ssize_t len) default: WRONG("compiler must be broken"); } - if (ved_bytes(foo->req, foo->ecx, + if (ved_bytes(req, foo->ecx, VDP_NULL, foo->dbits + 1, foo->lpad)) return (-1); } @@ -714,42 +705,20 @@ ved_objiterate(void *priv, unsigned flush, const void *ptr, ssize_t len) return (0); } -static void -ved_stripgzip(struct req *req, const struct boc *boc) +static int v_matchproto_(vdp_fini_f) +ved_zap_init(struct req *req, void **priv) { ssize_t l; const char *p; - uint32_t icrc; - uint32_t ilen; - struct ecx *ecx; - struct ved_foo foo[1]; + struct ved_foo *foo; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); - CAST_OBJ_NOTNULL(ecx, req->transport_priv, ECX_MAGIC); + CAST_OBJ_NOTNULL(foo, *priv, VED_FOO_MAGIC); - INIT_OBJ(foo, VED_FOO_MAGIC); - foo->ecx = ecx; - foo->req = req; memset(foo->tailbuf, 0xdd, sizeof foo->tailbuf); - /* OA_GZIPBITS is not valid until BOS_FINISHED */ - if (boc != NULL) - ObjWaitState(req->objcore, BOS_FINISHED); - if (req->objcore->flags & OC_F_FAILED) { - /* No way of signalling errors in the middle of - the ESI body. Omit this ESI fragment. */ - return; - } - AN(ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED)); - /* - * This is the interesting case: Deliver all the deflate - * blocks, stripping the "LAST" bit of the last one and - * padding it, as necessary, to a byte boundary. - */ - p = ObjGetAttr(req->wrk, req->objcore, OA_GZIPBITS, &l); AN(p); assert(l == 32); @@ -765,18 +734,36 @@ ved_stripgzip(struct req *req, const struct boc *boc) /* The start bit must be byte aligned. */ AZ(foo->start & 7); + return (0); +} + +static int v_matchproto_(vdp_fini_f) +ved_zap_fini(struct req *req, void **priv) +{ + uint32_t icrc; + uint32_t ilen; + struct ved_foo *foo; + + CAST_OBJ_NOTNULL(foo, *priv, VED_FOO_MAGIC); + *priv = NULL; - (void)ObjIterate(req->wrk, req->objcore, &foo, ved_objiterate, 0); - /* XXX: error check ?? */ - (void)ved_bytes(req, ecx, VDP_FLUSH, NULL, 0); + (void)ved_bytes(req, foo->ecx, VDP_FLUSH, NULL, 0); icrc = vle32dec(foo->tailbuf); ilen = vle32dec(foo->tailbuf + 4); + foo->ecx->crc = crc32_combine(foo->ecx->crc, icrc, ilen); + foo->ecx->l_crc += ilen; - ecx->crc = crc32_combine(ecx->crc, icrc, ilen); - ecx->l_crc += ilen; + return (0); } +static const struct vdp ved_zap = { + .name = "VZP", + .init = ved_zap_init, + .bytes = ved_zap_bytes, + .fini = ved_zap_fini, +}; + /*--------------------------------------------------------------------*/ static int v_matchproto_(vdp_fini_f) @@ -811,6 +798,7 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody) { int i; struct ecx *ecx; + struct ved_foo foo[1]; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_ORNULL(boc, BOC_MAGIC); @@ -826,14 +814,24 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody) i = ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED); if (ecx->isgzip && i && !(req->res_mode & RES_ESI)) { - ved_stripgzip(req, boc); + /* OA_GZIPBITS is not valid until BOS_FINISHED */ + if (boc != NULL) + ObjWaitState(req->objcore, BOS_FINISHED); + if (req->objcore->flags & OC_F_FAILED) { + /* No way of signalling errors in the middle of + the ESI body. Omit this ESI fragment. */ + return; + } + INIT_OBJ(foo, VED_FOO_MAGIC); + foo->ecx = ecx; + (void)VDP_Push(req, &ved_zap, foo); } else { if (ecx->isgzip && !i) (void)VDP_Push(req, &ved_vdp_pgz, ecx); else (void)VDP_Push(req, &ved_ved, ecx); - (void)VDP_DeliverObj(req); - (void)VDP_bytes(req, VDP_FLUSH, NULL, 0); } + (void)VDP_DeliverObj(req); + (void)VDP_bytes(req, VDP_FLUSH, NULL, 0); VDP_close(req); } From phk at FreeBSD.org Wed Jan 16 23:12:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 16 Jan 2019 23:12:07 +0000 (UTC) Subject: [master] 943a0cf89 Polish Message-ID: <20190116231207.62B3E704F@lists.varnish-cache.org> commit 943a0cf8943e811f19cd1efb3e03d1688cf447b5 Author: Poul-Henning Kamp Date: Wed Jan 16 23:11:25 2019 +0000 Polish diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 360bd3c2b..d51df4ac8 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -538,14 +538,14 @@ ved_pretend_gzip_bytes(struct req *req, enum vdp_action act, void **priv, return (ved_bytes(req, ecx, VDP_FLUSH, NULL, 0)); } -static const struct vdp ved_vdp_pgz = { +static const struct vdp ved_pretend_gz = { .name = "PGZ", .bytes = ved_pretend_gzip_bytes, .fini = ved_pretend_gzip_fini, }; /*--------------------------------------------------------------------- - * Include an object in a gzip'ed ESI object delivery + * Include a gzip'ed object in a gzip'ed ESI object delivery * * This is the interesting case: Deliver all the deflate blocks, stripping * the "LAST" bit of the last one and padding it, as necessary, to a byte @@ -564,8 +564,40 @@ struct ved_foo { uint8_t tailbuf[8]; }; +static int v_matchproto_(vdp_fini_f) +ved_gzgz_init(struct req *req, void **priv) +{ + ssize_t l; + const char *p; + struct ved_foo *foo; + + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CAST_OBJ_NOTNULL(foo, *priv, VED_FOO_MAGIC); + + memset(foo->tailbuf, 0xdd, sizeof foo->tailbuf); + + AN(ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED)); + + p = ObjGetAttr(req->wrk, req->objcore, OA_GZIPBITS, &l); + AN(p); + assert(l == 32); + foo->start = vbe64dec(p); + foo->last = vbe64dec(p + 8); + foo->stop = vbe64dec(p + 16); + foo->olen = ObjGetLen(req->wrk, req->objcore); + assert(foo->start > 0 && foo->start < foo->olen * 8); + assert(foo->last > 0 && foo->last < foo->olen * 8); + assert(foo->stop > 0 && foo->stop < foo->olen * 8); + assert(foo->last >= foo->start); + assert(foo->last < foo->stop); + + /* The start bit must be byte aligned. */ + AZ(foo->start & 7); + return (0); +} + static int v_matchproto_(vdp_bytes_f) -ved_zap_bytes(struct req *req, enum vdp_action act, void **priv, +ved_gzgz_bytes(struct req *req, enum vdp_action act, void **priv, const void *ptr, ssize_t len) { struct ved_foo *foo; @@ -706,39 +738,7 @@ ved_zap_bytes(struct req *req, enum vdp_action act, void **priv, } static int v_matchproto_(vdp_fini_f) -ved_zap_init(struct req *req, void **priv) -{ - ssize_t l; - const char *p; - struct ved_foo *foo; - - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - CAST_OBJ_NOTNULL(foo, *priv, VED_FOO_MAGIC); - - memset(foo->tailbuf, 0xdd, sizeof foo->tailbuf); - - AN(ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED)); - - p = ObjGetAttr(req->wrk, req->objcore, OA_GZIPBITS, &l); - AN(p); - assert(l == 32); - foo->start = vbe64dec(p); - foo->last = vbe64dec(p + 8); - foo->stop = vbe64dec(p + 16); - foo->olen = ObjGetLen(req->wrk, req->objcore); - assert(foo->start > 0 && foo->start < foo->olen * 8); - assert(foo->last > 0 && foo->last < foo->olen * 8); - assert(foo->stop > 0 && foo->stop < foo->olen * 8); - assert(foo->last >= foo->start); - assert(foo->last < foo->stop); - - /* The start bit must be byte aligned. */ - AZ(foo->start & 7); - return (0); -} - -static int v_matchproto_(vdp_fini_f) -ved_zap_fini(struct req *req, void **priv) +ved_gzgz_fini(struct req *req, void **priv) { uint32_t icrc; uint32_t ilen; @@ -757,11 +757,11 @@ ved_zap_fini(struct req *req, void **priv) return (0); } -static const struct vdp ved_zap = { - .name = "VZP", - .init = ved_zap_init, - .bytes = ved_zap_bytes, - .fini = ved_zap_fini, +static const struct vdp ved_gzgz = { + .name = "VZZ", + .init = ved_gzgz_init, + .bytes = ved_gzgz_bytes, + .fini = ved_gzgz_fini, }; /*--------------------------------------------------------------------*/ @@ -814,22 +814,28 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody) i = ObjCheckFlag(req->wrk, req->objcore, OF_GZIPED); if (ecx->isgzip && i && !(req->res_mode & RES_ESI)) { - /* OA_GZIPBITS is not valid until BOS_FINISHED */ + /* A gzip'ed include which is not ESI processed */ + + /* OA_GZIPBITS are not valid until BOS_FINISHED */ if (boc != NULL) ObjWaitState(req->objcore, BOS_FINISHED); + if (req->objcore->flags & OC_F_FAILED) { /* No way of signalling errors in the middle of the ESI body. Omit this ESI fragment. */ return; } + INIT_OBJ(foo, VED_FOO_MAGIC); foo->ecx = ecx; - (void)VDP_Push(req, &ved_zap, foo); + (void)VDP_Push(req, &ved_gzgz, foo); + + } else if (ecx->isgzip && !i) { + /* Non-Gzip'ed include in gzip'ed parent */ + (void)VDP_Push(req, &ved_pretend_gz, ecx); } else { - if (ecx->isgzip && !i) - (void)VDP_Push(req, &ved_vdp_pgz, ecx); - else - (void)VDP_Push(req, &ved_ved, ecx); + /* Anything else goes straight through */ + (void)VDP_Push(req, &ved_ved, ecx); } (void)VDP_DeliverObj(req); (void)VDP_bytes(req, VDP_FLUSH, NULL, 0); From fgsch at lodoss.net Thu Jan 17 12:09:07 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 17 Jan 2019 12:09:07 +0000 (UTC) Subject: [master] 1dc74a392 Make clear that the second field is actually 2 Message-ID: <20190117120907.B28DE97145@lists.varnish-cache.org> commit 1dc74a392f8aa01e111fbc40c003d0479be5b777 Author: Federico G. Schwindt Date: Thu Jan 17 12:02:49 2019 +0000 Make clear that the second field is actually 2 Addresses the doc part of #2872. diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 8353bd966..3730e7a85 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -413,17 +413,18 @@ SLTM(Hash, SLT_F_UNSAFE, "Value added to hash", SLTM(Backend_health, 0, "Backend health check", "The result of a backend health probe.\n\n" "The format is::\n\n" - "\t%s %s %s %u %u %u %f %f %s\n" - "\t| | | | | | | | |\n" - "\t| | | | | | | | +- Probe HTTP response / error information\n" - "\t| | | | | | | +---- Average response time\n" - "\t| | | | | | +------- Response time\n" - "\t| | | | | +---------- Probe window size\n" - "\t| | | | +------------- Probe threshold level\n" - "\t| | | +---------------- Number of good probes in window\n" - "\t| | +------------------- Probe window bits\n" - "\t| +---------------------- Status message\n" - "\t+------------------------- Backend name\n" + "\t%s %s %s %s %u %u %u %f %f %s\n" + "\t| | | | | | | | | |\n" + "\t| | | | | | | | | +- Probe HTTP response / error information\n" + "\t| | | | | | | | +---- Average response time\n" + "\t| | | | | | | +------- Response time\n" + "\t| | | | | | +---------- Probe window size\n" + "\t| | | | | +------------- Probe threshold level\n" + "\t| | | | +---------------- Number of good probes in window\n" + "\t| | | +------------------- Probe window bits\n" + "\t| | +---------------------- \"healthy\" or \"sick\"\n" + "\t| +------------------------- \"Back\", \"Still\" or \"Went\"\n" + "\t+---------------------------- Backend name\n" "\n" "Probe window bits are::\n\n" From fgsch at lodoss.net Thu Jan 17 12:09:07 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 17 Jan 2019 12:09:07 +0000 (UTC) Subject: [master] 1a0e5fb64 Whitespace Message-ID: <20190117120907.CC19D97149@lists.varnish-cache.org> commit 1a0e5fb6415b1dd1d76bc23fe2186005f9a275d5 Author: Federico G. Schwindt Date: Thu Jan 17 12:04:51 2019 +0000 Whitespace diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index 0ff57bd96..d9be207ac 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -192,7 +192,7 @@ include/varnishstat_synopsis.rst: $(top_builddir)/bin/varnishstat/varnishstat BUILT_SOURCES += include/varnishstat_options.rst \ include/varnishstat_synopsis.rst -include/vsl-tags.rst: $(top_builddir)/lib/libvarnishapi/vsl2rst +include/vsl-tags.rst: $(top_builddir)/lib/libvarnishapi/vsl2rst $(top_builddir)/lib/libvarnishapi/vsl2rst > ${@}_ mv ${@}_ ${@} BUILT_SOURCES += include/vsl-tags.rst diff --git a/doc/sphinx/phk/lucky.rst b/doc/sphinx/phk/lucky.rst index 328f66e6a..6a4393c97 100644 --- a/doc/sphinx/phk/lucky.rst +++ b/doc/sphinx/phk/lucky.rst @@ -96,7 +96,7 @@ Varnish Moral License --------------------- I want to thank the companies who have paid for a `Varnish -Moral License `_: +Moral License `_: * Fastly @@ -176,4 +176,4 @@ Good luck! .. [#f2] And it does not feel any less awful if the loved ones left behind tries to fill the blanks by asking you how you knew each other and if you have any memories you could share with them. - + From phk at FreeBSD.org Mon Jan 21 09:18:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 21 Jan 2019 09:18:07 +0000 (UTC) Subject: [master] 5f46a0ebf Monday morning flexelinting Message-ID: <20190121091807.2D8E36E999@lists.varnish-cache.org> commit 5f46a0ebf70a1c93e31694b1605b2a0187106661 Author: Poul-Henning Kamp Date: Mon Jan 21 09:16:46 2019 +0000 Monday morning flexelinting diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c index 16393ecde..98fb33104 100644 --- a/bin/varnishadm/varnishadm.c +++ b/bin/varnishadm/varnishadm.c @@ -73,7 +73,7 @@ } while (0) -static double timeout = 5; +static double timeout = 5; // XXX should be settable by arg ? static void cli_write(int sock, const char *s) @@ -192,7 +192,7 @@ send_line(char *l) cli_write(_line_sock, l); cli_write(_line_sock, "\n"); if (*l) - add_history(l); + AZ(add_history(l)); rl_callback_handler_install("varnish> ", send_line); } else { RL_EXIT(0); @@ -234,6 +234,29 @@ varnishadm_completion (const char *text, int start, int end) return (matches); } +static void +pass_answer(int fd) +{ + unsigned u, status; + char *answer = NULL; + + u = VCLI_ReadResult(fd, &status, &answer, timeout); + if (u) { + if (status == CLIS_COMMS) + RL_EXIT(0); + if (answer) + fprintf(stderr, "%s\n", answer); + RL_EXIT(1); + } + + printf("%u\n", status); + if (answer) { + printf("%s\n", answer); + free(answer); + } + (void)fflush(stdout); +} + /* * No arguments given, simply pass bytes on stdin/stdout and CLI socket * Send a "banner" to varnish, to provoke a welcome message. @@ -242,7 +265,6 @@ static void interactive(int sock) { struct pollfd fds[2]; - char buf[1024]; int i; char *answer = NULL; unsigned u, status; @@ -293,25 +315,8 @@ interactive(int sock) assert(i > 0); if (fds[0].revents & POLLIN) { /* Get rid of the prompt, kinda hackish */ - u = write(1, "\r \r", 13); - u = VCLI_ReadResult(fds[0].fd, &status, &answer, - timeout); - if (u) { - if (status == CLIS_COMMS) - RL_EXIT(0); - if (answer) - fprintf(stderr, "%s\n", answer); - RL_EXIT(1); - } - - bprintf(buf, "%u\n", status); - u = write(1, buf, strlen(buf)); - if (answer) { - u = write(1, answer, strlen(answer)); - u = write(1, "\n", 1); - free(answer); - answer = NULL; - } + printf("\r \r"); + pass_answer(fds[0].fd); rl_forced_update_display(); } if (fds[1].revents & POLLIN) { @@ -329,8 +334,6 @@ pass(int sock) struct pollfd fds[2]; char buf[1024]; int i; - char *answer = NULL; - unsigned u, status; ssize_t n; fds[0].fd = sock; @@ -343,26 +346,8 @@ pass(int sock) continue; } assert(i > 0); - if (fds[0].revents & POLLIN) { - u = VCLI_ReadResult(fds[0].fd, &status, &answer, - timeout); - if (u) { - if (status == CLIS_COMMS) - RL_EXIT(0); - if (answer) - fprintf(stderr, "%s\n", answer); - RL_EXIT(1); - } - - bprintf(buf, "%u\n", status); - u = write(1, buf, strlen(buf)); - if (answer) { - u = write(1, answer, strlen(answer)); - u = write(1, "\n", 1); - free(answer); - answer = NULL; - } - } + if (fds[0].revents & POLLIN) + pass_answer(fds[0].fd); if (fds[1].revents & POLLIN || fds[1].revents & POLLHUP) { n = read(fds[1].fd, buf, sizeof buf - 1); if (n == 0) { @@ -392,8 +377,8 @@ usage(int status) static int n_arg_sock(const char *n_arg, const char *t_arg) { - char *T_arg = NULL, *T_start = NULL; - char *S_arg = NULL; + char *T_arg, *T_start; + char *S_arg; struct vsm *vsm; char *p; int sock; diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index d51df4ac8..1462832a1 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -764,7 +764,9 @@ static const struct vdp ved_gzgz = { .fini = ved_gzgz_fini, }; -/*--------------------------------------------------------------------*/ +/*-------------------------------------------------------------------- + * Straight through without processing. + */ static int v_matchproto_(vdp_fini_f) ved_vdp_fini(struct req *req, void **priv) diff --git a/lib/libvmod_blob/base64.c b/lib/libvmod_blob/base64.c index 62ad71e39..594b84881 100644 --- a/lib/libvmod_blob/base64.c +++ b/lib/libvmod_blob/base64.c @@ -139,6 +139,7 @@ base64_decode(const enum encoding dec, char *restrict const buf, unsigned u = 0, term = 0; int n = 0; size_t len = SIZE_MAX; + const char *s; AN(buf); AN(alpha); @@ -148,7 +149,7 @@ base64_decode(const enum encoding dec, char *restrict const buf, len = inlen; for (int i = 0; len > 0 && i < strings->n; i++) { - const char *s = strings->p[i]; + s = strings->p[i]; if (s == NULL) continue; @@ -158,7 +159,7 @@ base64_decode(const enum encoding dec, char *restrict const buf, } while (*s && len) { while (n < 4) { - char b = alpha->i64[(unsigned) *s++]; + char b = alpha->i64[(uint8_t) *s++]; u <<= 6; if (b == ILL) { errno = EINVAL; @@ -169,7 +170,7 @@ base64_decode(const enum encoding dec, char *restrict const buf, term++; continue; } - u |= (unsigned) b; + u |= (uint8_t) b; if (--len == 0) break; if (!*s) diff --git a/lib/libvmod_debug/vmod_debug_obj.c b/lib/libvmod_debug/vmod_debug_obj.c index 9038bcb81..4e716dae4 100644 --- a/lib/libvmod_debug/vmod_debug_obj.c +++ b/lib/libvmod_debug/vmod_debug_obj.c @@ -182,6 +182,8 @@ xyzzy_obj_opt__init(VRT_CTX, if (args->valid_s) AN(args->s); + (void)args->valid_b; + (void)args->b; AN(op); AZ(*op); diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h index 62bc2df53..a1e51e244 100644 --- a/lib/libvmod_directors/shard_dir.h +++ b/lib/libvmod_directors/shard_dir.h @@ -42,10 +42,8 @@ enum healthy_e { }; enum resolve_e { - _RESOLVE_E_INVALID = 0, #define VMODENUM(x) x, #include "tbl_resolve.h" - _RESOLVE_E_MAX }; struct vbitmap; From geoff at uplex.de Mon Jan 21 09:31:08 2019 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 21 Jan 2019 09:31:08 +0000 (UTC) Subject: [master] 42ab3c100 Document disuse of SLT_Backend. Message-ID: <20190121093108.3DFB46EF1C@lists.varnish-cache.org> commit 42ab3c1003e7bf68ddf1e58daf3df55b095a2e8b Author: Geoff Simmons Date: Mon Jan 21 10:29:59 2019 +0100 Document disuse of SLT_Backend. Closes #2870 diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 3730e7a85..03488cdc8 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -160,6 +160,10 @@ SLTM(Backend, 0, "Backend selected", "\t| | +- Backend display name\n" "\t| +---- VCL name\n" "\t+------- Connection file descriptor\n" + "\t\n" + "\tNOTE: This tag is currently not in use in the Varnish log.\n" + "\tIt is mentioned here to document legacy versions of the log,\n" + "\tand reserved for possible use in future versions.\n" "\n" ) From phk at FreeBSD.org Mon Jan 21 09:32:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 21 Jan 2019 09:32:07 +0000 (UTC) Subject: [master] c7cba86e4 Define VTEST_WITH_ macros Message-ID: <20190121093207.E3883910C0@lists.varnish-cache.org> commit c7cba86e4edfe43165ed5b4670420dc8717a80e4 Author: Poul-Henning Kamp Date: Mon Jan 21 09:31:00 2019 +0000 Define VTEST_WITH_ macros diff --git a/bin/varnishtest/flint.sh b/bin/varnishtest/flint.sh index 49f5e2db8..2512c7e04 100755 --- a/bin/varnishtest/flint.sh +++ b/bin/varnishtest/flint.sh @@ -1,6 +1,8 @@ #!/bin/sh FLOPS=' + -DVTEST_WITH_VTC_LOGEXPECT + -DVTEST_WITH_VTC_VARNISH -DTOP_BUILDDIR="foo" -I../../lib/libvgz *.c From phk at FreeBSD.org Mon Jan 21 09:35:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 21 Jan 2019 09:35:09 +0000 (UTC) Subject: [master] 03aa46ee9 add_history() has variant definitions Message-ID: <20190121093509.54DAD91382@lists.varnish-cache.org> commit 03aa46ee9297e966b896242c19e25c4c24ac0d65 Author: Poul-Henning Kamp Date: Mon Jan 21 09:34:18 2019 +0000 add_history() has variant definitions diff --git a/bin/varnishadm/varnishadm.c b/bin/varnishadm/varnishadm.c index 98fb33104..db16284f6 100644 --- a/bin/varnishadm/varnishadm.c +++ b/bin/varnishadm/varnishadm.c @@ -192,7 +192,7 @@ send_line(char *l) cli_write(_line_sock, l); cli_write(_line_sock, "\n"); if (*l) - AZ(add_history(l)); + add_history(l); rl_callback_handler_install("varnish> ", send_line); } else { RL_EXIT(0); diff --git a/flint.lnt b/flint.lnt index a20d04af2..60c016cb0 100644 --- a/flint.lnt +++ b/flint.lnt @@ -196,6 +196,11 @@ -esym(785,VSL_tags) // Sparse array +/////////////////////////////////////////////////////////////////////// +// readline etc. + +-esym(534, add_history) + /////////////////////////////////////////////////////////////////////// // -lcurses -esym(534, beep) From phk at FreeBSD.org Mon Jan 21 09:51:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 21 Jan 2019 09:51:08 +0000 (UTC) Subject: [master] dfee06d5c Ignore a curses function return value Message-ID: <20190121095108.C8E9D91A8F@lists.varnish-cache.org> commit dfee06d5cbb4fef168314a32d0c0a270890e69c3 Author: Poul-Henning Kamp Date: Mon Jan 21 09:50:02 2019 +0000 Ignore a curses function return value diff --git a/flint.lnt b/flint.lnt index 60c016cb0..1a0b88caf 100644 --- a/flint.lnt +++ b/flint.lnt @@ -212,6 +212,7 @@ -esym(534, intrflush) -esym(534, keypad) -esym(534, mvprintw) +-esym(534, waddnstr) -esym(534, mvwprintw) -esym(534, nodelay) -esym(534, noecho) From phk at FreeBSD.org Mon Jan 21 09:51:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 21 Jan 2019 09:51:08 +0000 (UTC) Subject: [master] 4703c26fd Test more of the fallback director Message-ID: <20190121095108.E503591A92@lists.varnish-cache.org> commit 4703c26fd93a78f27db0f2bc339861712f949597 Author: Poul-Henning Kamp Date: Mon Jan 21 09:50:17 2019 +0000 Test more of the fallback director diff --git a/bin/varnishtest/tests/d00001.vtc b/bin/varnishtest/tests/d00001.vtc index 405147615..1ebae04d2 100644 --- a/bin/varnishtest/tests/d00001.vtc +++ b/bin/varnishtest/tests/d00001.vtc @@ -16,6 +16,7 @@ server s3 { } -start varnish v1 -vcl+backend { + import std; import directors; sub vcl_init { @@ -33,11 +34,17 @@ varnish v1 -vcl+backend { return (pass); } + sub vcl_deliver { + set resp.http.health = std.healthy(fb1.backend()); + } + sub vcl_backend_fetch { set bereq.backend = fb1.backend(); } } -start +varnish v1 -cliok "param.set debug +vclrel" + varnish v1 -cliok "backend.set_health s1 sick" varnish v1 -cliok "backend.set_health s2 sick" @@ -48,6 +55,8 @@ client c1 { expect resp.http.foo == "3" } -run +varnish v1 -vsl_catchup + varnish v1 -cliok "backend.set_health s2 healthy" client c1 { @@ -56,6 +65,8 @@ client c1 { expect resp.http.foo == "2" } -run +varnish v1 -vsl_catchup + varnish v1 -cliok "backend.set_health s1 healthy" client c1 { @@ -64,6 +75,8 @@ client c1 { expect resp.http.foo == "1" } -run +varnish v1 -vsl_catchup + varnish v1 -cliok "backend.set_health s1 sick" server s3 -start @@ -75,3 +88,15 @@ client c1 { rxresp expect resp.http.foo == "3" } -run + +varnish v1 -vsl_catchup + +varnish v1 -vcl+backend { + sub vcl_recv { + set req.backend_hint = s1; + set req.backend_hint = s2; + set req.backend_hint = s3; + } +} +varnish v1 -cliok "vcl.discard vcl1" +varnish v1 -cliok "vcl.list" From nils.goroll at uplex.de Tue Jan 22 06:06:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 22 Jan 2019 06:06:07 +0000 (UTC) Subject: [master] 883578ce7 improve vmod_blob description Message-ID: <20190122060607.3ECEEB0F5C@lists.varnish-cache.org> commit 883578ce7eaf108e93dcd460a6ae4beea2ff031a Author: Nils Goroll Date: Tue Jan 22 07:00:47 2019 +0100 improve vmod_blob description Encoding and decoding are as much a function as handling blobs, in particular with transcode(). diff --git a/lib/libvmod_blob/vmod.vcc b/lib/libvmod_blob/vmod.vcc index 1ea3716d4..a5b6ae4fd 100644 --- a/lib/libvmod_blob/vmod.vcc +++ b/lib/libvmod_blob/vmod.vcc @@ -6,7 +6,7 @@ # Geoffrey Simmons # -$Module blob 3 "Utilities for the VCL blob type" +$Module blob 3 "Utilities for the VCL blob type, encoding and decoding" $ABI strict From dridi.boukelmoune at gmail.com Tue Jan 22 07:38:07 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 22 Jan 2019 07:38:07 +0000 (UTC) Subject: [master] 7c568d948 Document the link between varnishd -l and -p Message-ID: <20190122073807.36AFFB2A78@lists.varnish-cache.org> commit 7c568d948527923eabfe285d0271e685c4e4b101 Author: Dridi Boukelmoune Date: Tue Jan 22 08:33:48 2019 +0100 Document the link between varnishd -l and -p diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 57c657711..9f6f8fb77 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -207,9 +207,9 @@ Tuning options -l - Specifies size of the space for the VSL records. - Scaling suffixes like 'K' and 'M' can be used up to (G)igabytes. - Default is 80 Megabytes. + Specifies size of the space for the VSL records, shorthand for + ``-p vsl_space=``. Scaling suffixes like 'K' and 'M' can be + used up to (G)igabytes. See `vsl_space`_ for more information. Security options ---------------- From phk at FreeBSD.org Tue Jan 22 08:24:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 22 Jan 2019 08:24:07 +0000 (UTC) Subject: [master] e5afe5346 Better coverage of random director Message-ID: <20190122082407.D1A5049C7@lists.varnish-cache.org> commit e5afe534609c47026b0e0b3cecc6903c3e170837 Author: Poul-Henning Kamp Date: Tue Jan 22 07:30:41 2019 +0000 Better coverage of random director diff --git a/bin/varnishtest/tests/d00002.vtc b/bin/varnishtest/tests/d00002.vtc index b13518ca1..a5ed1aeb2 100644 --- a/bin/varnishtest/tests/d00002.vtc +++ b/bin/varnishtest/tests/d00002.vtc @@ -9,6 +9,7 @@ server s1 { varnish v1 -vcl+backend { import directors; + import std; sub vcl_init { new foo = directors.random(); @@ -22,6 +23,7 @@ varnish v1 -vcl+backend { sub vcl_backend_response { set beresp.http.where = bereq.backend + "-->" + beresp.backend; + set beresp.http.health = std.healthy(foo.backend()); } } -start From phk at FreeBSD.org Tue Jan 22 08:24:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 22 Jan 2019 08:24:07 +0000 (UTC) Subject: [master] 6233b0884 More work on refactoring req-cleanup Message-ID: <20190122082407.E927949CA@lists.varnish-cache.org> commit 6233b08842d8ea48e78b9fc5381a26a2c753862e Author: Poul-Henning Kamp Date: Tue Jan 22 08:23:02 2019 +0000 More work on refactoring req-cleanup diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index 0cd80c20a..630e47f07 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -43,8 +43,8 @@ #include "vtim.h" -void -Req_AcctLogCharge(struct VSC_main_wrk *ds, struct req *req) +static void +req_AcctLogCharge(struct VSC_main_wrk *ds, struct req *req) { struct acct_req *a; @@ -167,8 +167,7 @@ Req_Release(struct req *req) #include "tbl/acct_fields_req.h" AZ(req->vcl); - if (req->vsl->wid) - VSL_End(req->vsl); + AZ(req->vsl->wid); sp = req->sp; CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); pp = sp->pool; @@ -222,10 +221,9 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req) VCL_Recache(wrk, &req->vcl); /* Charge and log byte counters */ - if (req->vsl->wid) { - Req_AcctLogCharge(wrk->stats, req); + req_AcctLogCharge(wrk->stats, req); + if (req->vsl->wid) VSL_End(req->vsl); - } req->req_bodybytes = 0; if (!isnan(req->t_prev) && req->t_prev > 0. && req->t_prev > sp->t_idle) diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index d85892de3..38ca918a7 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -313,7 +313,6 @@ void Req_Release(struct req *); void Req_Rollback(struct req *req); void Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req); void Req_Fail(struct req *req, enum sess_close reason); -void Req_AcctLogCharge(struct VSC_main_wrk *, struct req *); /* cache_req_body.c */ int VRB_Ignore(struct req *); diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c index 05c1c4f06..14b36d4d2 100644 --- a/bin/varnishd/http1/cache_http1_fsm.c +++ b/bin/varnishd/http1/cache_http1_fsm.c @@ -337,7 +337,7 @@ HTTP1_Session(struct worker *wrk, struct req *req) if (hs < HTC_S_EMPTY) { req->acct.req_hdrbytes += req->htc->rxbuf_e - req->htc->rxbuf_b; - Req_AcctLogCharge(wrk->stats, req); + Req_Cleanup(sp, wrk, req); Req_Release(req); switch (hs) { case HTC_S_CLOSE: @@ -358,6 +358,7 @@ HTTP1_Session(struct worker *wrk, struct req *req) } if (hs == HTC_S_IDLE) { wrk->stats->sess_herd++; + Req_Cleanup(sp, wrk, req); Req_Release(req); SES_Wait(sp, &HTTP1_transport); return; diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c index 115ec6b8d..eee053931 100644 --- a/bin/varnishd/http2/cache_http2_session.c +++ b/bin/varnishd/http2/cache_http2_session.c @@ -240,7 +240,7 @@ h2_ou_rel(struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); AZ(req->vcl); - Req_AcctLogCharge(wrk->stats, req); + Req_Cleanup(req->sp, wrk, req); Req_Release(req); return (0); } From phk at FreeBSD.org Tue Jan 22 09:04:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 22 Jan 2019 09:04:07 +0000 (UTC) Subject: [master] 9d3f1941a Localize req->req_bodybytes Message-ID: <20190122090407.4CCEA57FC@lists.varnish-cache.org> commit 9d3f1941a5ebcec36b022c46d4a0b356913c305c Author: Poul-Henning Kamp Date: Tue Jan 22 09:03:47 2019 +0000 Localize req->req_bodybytes diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index ce68faf59..2018c9bce 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -486,7 +486,6 @@ struct req { vtim_dur d_ttl; vtim_dur d_grace; - ssize_t req_bodybytes; /* Parsed req bodybytes */ const struct stevedore *storage; const struct director *director_hint; diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index 630e47f07..c6abfe1ae 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -142,8 +142,6 @@ Req_New(const struct worker *wrk, struct sess *sp) WS_Init(req->ws, "req", p, e - p); - req->req_bodybytes = 0; - req->t_first = NAN; req->t_prev = NAN; req->t_req = NAN; @@ -224,7 +222,6 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req) req_AcctLogCharge(wrk->stats, req); if (req->vsl->wid) VSL_End(req->vsl); - req->req_bodybytes = 0; if (!isnan(req->t_prev) && req->t_prev > 0. && req->t_prev > sp->t_idle) sp->t_idle = req->t_prev; diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c index 488815d8a..903c973b5 100644 --- a/bin/varnishd/cache/cache_req_body.c +++ b/bin/varnishd/cache/cache_req_body.c @@ -56,6 +56,7 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv) uint8_t *ptr; enum vfp_status vfps = VFP_ERROR; const struct stevedore *stv; + ssize_t req_bodybytes = 0; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -91,7 +92,6 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv) return (-1); } - AZ(req->req_bodybytes); AN(req->htc); yet = req->htc->content_length; if (yet != 0 && req->want100cont) { @@ -102,7 +102,7 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv) yet = 0; do { AZ(vfc->failed); - if (maxsize >= 0 && req->req_bodybytes > maxsize) { + if (maxsize >= 0 && req_bodybytes > maxsize) { (void)VFP_Error(vfc, "Request body too big to cache"); break; } @@ -114,7 +114,7 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv) AN(l); vfps = VFP_Suck(vfc, ptr, &l); if (l > 0 && vfps != VFP_ERROR) { - req->req_bodybytes += l; + req_bodybytes += l; req->acct.req_bodybytes += l; if (yet >= l) yet -= l; @@ -142,7 +142,7 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv) } ObjTrimStore(req->wrk, req->body_oc); - AZ(ObjSetU64(req->wrk, req->body_oc, OA_LEN, req->req_bodybytes)); + AZ(ObjSetU64(req->wrk, req->body_oc, OA_LEN, req_bodybytes)); HSH_DerefBoc(req->wrk, req->body_oc); if (vfps != VFP_END) { @@ -151,22 +151,22 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv) return (-1); } - assert(req->req_bodybytes >= 0); - if (req->req_bodybytes != req->htc->content_length) { + assert(req_bodybytes >= 0); + if (req_bodybytes != req->htc->content_length) { /* We must update also the "pristine" req.* copy */ http_Unset(req->http0, H_Content_Length); http_Unset(req->http0, H_Transfer_Encoding); http_PrintfHeader(req->http0, "Content-Length: %ju", - (uintmax_t)req->req_bodybytes); + (uintmax_t)req_bodybytes); http_Unset(req->http, H_Content_Length); http_Unset(req->http, H_Transfer_Encoding); http_PrintfHeader(req->http, "Content-Length: %ju", - (uintmax_t)req->req_bodybytes); + (uintmax_t)req_bodybytes); } req->req_body_status = REQ_BODY_CACHED; - return (req->req_bodybytes); + return (req_bodybytes); } /*---------------------------------------------------------------------- @@ -188,8 +188,8 @@ VRB_Iterate(struct req *req, objiterate_f *func, void *priv) switch (req->req_body_status) { case REQ_BODY_CACHED: - if (req->req_bodybytes > 0 && - ObjIterate(req->wrk, req->body_oc, priv, func, 0)) + AN(req->body_oc); + if (ObjIterate(req->wrk, req->body_oc, priv, func, 0)) return (-1); return (0); case REQ_BODY_NONE: @@ -280,6 +280,7 @@ VRB_Free(struct req *req) ssize_t VRB_Cache(struct req *req, ssize_t maxsize) { + uint64_t u; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); assert(maxsize >= 0); @@ -295,7 +296,8 @@ VRB_Cache(struct req *req, ssize_t maxsize) assert (req->req_step == R_STP_RECV); switch (req->req_body_status) { case REQ_BODY_CACHED: - return (req->req_bodybytes); + AZ(ObjGetU64(req->wrk, req->body_oc, OA_LEN, &u)); + return (u); case REQ_BODY_FAIL: return (-1); case REQ_BODY_NONE: From phk at FreeBSD.org Tue Jan 22 13:12:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 22 Jan 2019 13:12:08 +0000 (UTC) Subject: [master] b2b9393ec Put each vmod on its own HTML page Message-ID: <20190122131208.7A36B6140D@lists.varnish-cache.org> commit b2b9393ec60d67a5ff3c25c36dfc657a849b9773 Author: Poul-Henning Kamp Date: Tue Jan 22 12:03:36 2019 +0000 Put each vmod on its own HTML page diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst index 75ec38e99..89375dcb7 100644 --- a/doc/sphinx/reference/index.rst +++ b/doc/sphinx/reference/index.rst @@ -20,7 +20,13 @@ The Varnish Reference Manual varnishtop.rst vsm.rst vmod.rst - vmod_generated.rst + vmod_std.rst + vmod_directors.rst + vmod_purge.rst + vmod_blob.rst + vmod_unix.rst + vmod_proxy.rst + vmod_vtc.rst directors.rst varnish-counters.rst vsl.rst diff --git a/doc/sphinx/reference/vmod_blob.rst b/doc/sphinx/reference/vmod_blob.rst new file mode 100644 index 000000000..e6b473c4f --- /dev/null +++ b/doc/sphinx/reference/vmod_blob.rst @@ -0,0 +1,2 @@ + +.. include:: ../include/vmod_blob.generated.rst diff --git a/doc/sphinx/reference/vmod_directors.rst b/doc/sphinx/reference/vmod_directors.rst new file mode 100644 index 000000000..fdb536602 --- /dev/null +++ b/doc/sphinx/reference/vmod_directors.rst @@ -0,0 +1,2 @@ + +.. include:: ../include/vmod_directors.generated.rst diff --git a/doc/sphinx/reference/vmod_generated.rst b/doc/sphinx/reference/vmod_generated.rst deleted file mode 100644 index ec89bb9a9..000000000 --- a/doc/sphinx/reference/vmod_generated.rst +++ /dev/null @@ -1,15 +0,0 @@ - -.. include:: ../include/vmod_std.generated.rst - -.. include:: ../include/vmod_directors.generated.rst - -.. include:: ../include/vmod_vtc.generated.rst - -.. include:: ../include/vmod_purge.generated.rst - -.. include:: ../include/vmod_blob.generated.rst - -.. include:: ../include/vmod_unix.generated.rst - -.. include:: ../include/vmod_proxy.generated.rst - diff --git a/doc/sphinx/reference/vmod_proxy.rst b/doc/sphinx/reference/vmod_proxy.rst new file mode 100644 index 000000000..224b323a0 --- /dev/null +++ b/doc/sphinx/reference/vmod_proxy.rst @@ -0,0 +1,3 @@ + +.. include:: ../include/vmod_proxy.generated.rst + diff --git a/doc/sphinx/reference/vmod_purge.rst b/doc/sphinx/reference/vmod_purge.rst new file mode 100644 index 000000000..24f9ffbd2 --- /dev/null +++ b/doc/sphinx/reference/vmod_purge.rst @@ -0,0 +1,3 @@ + +.. include:: ../include/vmod_purge.generated.rst + diff --git a/doc/sphinx/reference/vmod_std.rst b/doc/sphinx/reference/vmod_std.rst new file mode 100644 index 000000000..5124783bd --- /dev/null +++ b/doc/sphinx/reference/vmod_std.rst @@ -0,0 +1,2 @@ + +.. include:: ../include/vmod_std.generated.rst diff --git a/doc/sphinx/reference/vmod_unix.rst b/doc/sphinx/reference/vmod_unix.rst new file mode 100644 index 000000000..fe26290f0 --- /dev/null +++ b/doc/sphinx/reference/vmod_unix.rst @@ -0,0 +1,2 @@ + +.. include:: ../include/vmod_unix.generated.rst diff --git a/doc/sphinx/reference/vmod_vtc.rst b/doc/sphinx/reference/vmod_vtc.rst new file mode 100644 index 000000000..f71a68588 --- /dev/null +++ b/doc/sphinx/reference/vmod_vtc.rst @@ -0,0 +1,2 @@ + +.. include:: ../include/vmod_vtc.generated.rst From phk at FreeBSD.org Tue Jan 22 13:12:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 22 Jan 2019 13:12:08 +0000 (UTC) Subject: [master] 502b7f143 Refactor vmodtoll and the .rst doc it produces. Message-ID: <20190122131208.93CD761410@lists.varnish-cache.org> commit 502b7f1434a19bc6c2b0123445c3a642bd645322 Author: Poul-Henning Kamp Date: Tue Jan 22 13:11:18 2019 +0000 Refactor vmodtoll and the .rst doc it produces. Not what I came here for, but... diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 32d4e6f2e..17956311b 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -147,10 +147,11 @@ def write_rst_file_warning(fo): def write_rst_hdr(fo, s, below="-", above=None): - if above is not None: + fo.write('\n') + if above: fo.write(above * len(s) + "\n") fo.write(s + "\n") - if below is not None: + if below: fo.write(below * len(s) + "\n") ####################################################################### @@ -245,19 +246,16 @@ class CType(object): break assert w == "," - def vcl(self): + def vcl(self, terse=False): if self.vt in ("STRING_LIST", "STRANDS"): return "STRING" + if terse: + return self.vt if self.spec is None: return self.vt return self.vt + " {" + ", ".join(self.spec) + "}" - def synopsis(self): - if self.vt in ("STRING_LIST", "STRANDS"): - return "STRING" - return self.vt - - def json(self, jl): + def jsonproto(self, jl): jl.append([self.vt]) while jl[-1][-1] is None: jl[-1].pop(-1) @@ -294,7 +292,7 @@ class arg(CType): x = unquote(x) self.defval = x - def json(self, jl): + def jsonproto(self, jl): jl.append([self.vt, self.nm, self.defval, self.spec]) if self.opt: jl[-1].append(True) @@ -363,7 +361,7 @@ class ProtoType(object): t.nm2 = t.nm self.args.append(t) - def vcl_proto(self, short, pfx=""): + def vcl_proto(self, terse, pfx=""): if isinstance(self.st, MethodStanza): pfx += pfx s = pfx @@ -380,22 +378,19 @@ class ProtoType(object): s += self.name + "(" ll = [] for i in self.args: - if short: - t = i.synopsis() - else: - t = i.vcl() + t = i.vcl(terse) if t in PRIVS: continue if i.nm is not None: t += " " + i.nm - if not short: + if not terse: if i.defval is not None: t += "=" + i.defval if i.opt: t = "[" + t + "]" ll.append(t) t = ",@".join(ll) - if len(s + t) > 68 and not short: + if len(s + t) > 68 and not terse: s += "\n" + pfx + pfx s += t.replace("@", "\n" + pfx + pfx) s += "\n" + pfx + ")" @@ -409,15 +404,9 @@ class ProtoType(object): write_rst_hdr(fo, s, '-') else: s = self.vcl_proto(True) - if len(s) > 60: - s = self.name + "(...)" write_rst_hdr(fo, s, '-') fo.write("\n::\n\n" + self.vcl_proto(False, pfx=" ") + "\n") - def synopsis(self, fo, unused_man): - fo.write(self.vcl_proto(True, pfx=" ") + "\n") - fo.write(" \n") - def cname(self, pfx=False): r = self.name.replace(".", "_") if pfx: @@ -458,44 +447,49 @@ class ProtoType(object): s += "};\n" return s - def cstuff(self, args, where): + def cproto(self, eargs, where): + ''' Produce C language prototype ''' s = "" if where == 'h': if self.argstruct: s += self.argstructure() - s += lwrap(self.proto(args, self.cname(True))) + s += lwrap(self.proto(eargs, self.cname(True))) elif where == 'c': - s += lwrap(self.typedef(args)) + s += lwrap(self.typedef(eargs)) elif where == 'o': if self.argstruct: s += self.argstructure() - s += lwrap(self.typedef(args)) + s += lwrap(self.typedef(eargs)) else: assert False return s - def json(self, jl, cfunc): + def jsonproto(self, jl, cfunc): + ''' Produce VCL prototype as JSON ''' ll = [] - self.retval.json(ll) + self.retval.jsonproto(ll) ll.append('Vmod_%s_Func.%s' % (self.st.vcc.modname, cfunc)) if self.argstruct: ll.append(self.argstructname()) else: ll.append("") for i in self.args: - i.json(ll) + i.jsonproto(ll) jl.append(ll) ####################################################################### class Stanza(object): - def __init__(self, toks, l0, doc, vcc): + + ''' Base class for all $-Stanzas ''' + + def __init__(self, vcc, toks, doc): self.toks = toks - self.line = l0 - while doc and doc[0] == '': + doc = doc.split('\n') + while doc and not doc[0].strip(): doc.pop(0) - while doc and doc[-1] == '': + while doc and not doc[-1].strip(): doc.pop(-1) self.doc = doc self.vcc = vcc @@ -507,9 +501,6 @@ class Stanza(object): def parse(self): assert "subclass should have defined" == "parse method" - def dump(self): - print(type(self), self.line) - def syntax(self): err("Syntax error.\n" + "\tShould be: " + self.__doc__.strip() + "\n" + @@ -517,29 +508,26 @@ class Stanza(object): warn=False) def rstfile(self, fo, man): - if self.rstlbl is not None: - fo.write(".. _" + self.rstlbl + ":\n\n") - + if self.rstlbl: + fo.write("\n.. _" + self.rstlbl + ":\n") self.rsthead(fo, man) - fo.write("\n") - self.rstmid(fo, man) - fo.write("\n") - self.rsttail(fo, man) - fo.write("\n") + self.rstdoc(fo, man) def rsthead(self, fo, unused_man): - if self.proto is not None: + ''' Emit the systematic part of the documentation ''' + if self.proto: self.proto.rsthead(fo) + fo.write("\n") - def rstmid(self, fo, unused_man): + def rstdoc(self, fo, unused_man): + ''' Emit the explanatory part of the documentation ''' fo.write("\n".join(self.doc) + "\n") - def rsttail(self, unused_fo, unused_man): - return - def synopsis(self, fo, man): - if self.proto is not None: - self.proto.synopsis(fo, man) + if man and self.proto: + fo.write(self.proto.vcl_proto(True, pfx=" ") + '\n \n') + elif self.proto and self.rstlbl: + fo.write(' :ref:`%s`\n \n' % self.rstlbl) def cstuff(self, unused_fo, unused_where): return @@ -548,6 +536,7 @@ class Stanza(object): return def json(self, unused_jl): + ''' Add to the json we hand VCC ''' return ####################################################################### @@ -575,49 +564,25 @@ class ModuleStanza(Stanza): def rsthead(self, fo, man): - write_rst_hdr(fo, self.vcc.sympfx + self.vcc.modname, "=", "=") - fo.write("\n") - - write_rst_hdr(fo, self.vcc.moddesc, "-", "-") - - fo.write("\n") - fo.write(":Manual section: " + self.vcc.mansection + "\n") + if man: + write_rst_hdr(fo, self.vcc.sympfx + self.vcc.modname, "=", "=") + write_rst_hdr(fo, self.vcc.moddesc, "-", "-") + fo.write("\n") + fo.write(":Manual section: " + self.vcc.mansection + "\n") + else: + write_rst_hdr(fo, + self.vcc.sympfx + self.vcc.modname + + ' - ' + self.vcc.moddesc, + "=", "=") if self.vcc.auto_synopsis: - fo.write("\n") write_rst_hdr(fo, "SYNOPSIS", "=") fo.write("\n") - fo.write("\n::\n\n") - fo.write(' import %s [from "path"] ;\n' % self.vcc.modname) - fo.write(" \n") + fo.write(".. parsed-literal::\n\n") + fo.write(' import %s [from "path"]\n' % self.vcc.modname) + fo.write(" \n") for c in self.vcc.contents: c.synopsis(fo, man) - fo.write("\n") - - def rsttail(self, fo, man): - - if man: - return - - write_rst_hdr(fo, "CONTENTS", "=") - fo.write("\n") - - ll = [] - for i in self.vcc.contents[1:]: - j = i.rstlbl - if j is not None: - ll.append([j.split("_", 1)[1], j]) - if i.methods is None: - continue - for x in i.methods: - j = x.rstlbl - ll.append([j.split("_", 1)[1], j]) - - ll.sort() - for i in ll: - fo.write("* :ref:`%s`\n" % i[1]) - fo.write("\n") - class ABIStanza(Stanza): @@ -699,13 +664,16 @@ class EventStanza(Stanza): class FunctionStanza(Stanza): + + ''' $Function TYPE name ( ARGUMENSTS ) ''' + def parse(self): self.proto = ProtoType(self) self.rstlbl = "func_" + self.proto.name self.vcc.contents.append(self) def cstuff(self, fo, where): - fo.write(self.proto.cstuff(['VRT_CTX'], where)) + fo.write(self.proto.cproto(['VRT_CTX'], where)) def cstruct(self, fo, define): if define: @@ -715,10 +683,13 @@ class FunctionStanza(Stanza): def json(self, jl): jl.append(["$FUNC", "%s" % self.proto.name]) - self.proto.json(jl[-1], self.proto.cname()) + self.proto.jsonproto(jl[-1], self.proto.cname()) class ObjectStanza(Stanza): + + ''' $Object TYPE class ( ARGUMENSTS ) ''' + def parse(self): self.proto = ProtoType(self, retval=False) self.proto.obj = "x" + self.proto.name @@ -737,29 +708,34 @@ class ObjectStanza(Stanza): def rsthead(self, fo, man): self.proto.rsthead(fo) - - fo.write("\n" + "\n".join(self.doc) + "\n\n") - + fo.write("\n" + "\n".join(self.doc) + "\n") for i in self.methods: i.rstfile(fo, man) - def rstmid(self, unused_fo, unused_man): + def rstdoc(self, unused_fo, unused_man): return def synopsis(self, fo, man): - self.proto.synopsis(fo, man) - for i in self.methods: - i.proto.synopsis(fo, man) + if man and self.proto: + fo.write(self.proto.vcl_proto(True, pfx=" ") + '\n \n') + for i in self.methods: + if i.proto: + fo.write(i.proto.vcl_proto(True, pfx=" ") + '\n \n') + elif self.proto and self.rstlbl: + fo.write(' :ref:`%s`\n \n' % self.rstlbl) + for i in self.methods: + if i.proto and i.rstlbl: + fo.write(' :ref:`%s`\n \n' % i.rstlbl) def cstuff(self, fo, w): sn = self.vcc.sympfx + self.vcc.modname + "_" + self.proto.name fo.write("struct %s;\n" % sn) - fo.write(self.init.cstuff( + fo.write(self.init.cproto( ['VRT_CTX', 'struct %s **' % sn, 'const char *'], w)) - fo.write(self.fini.cstuff(['struct %s **' % sn], w)) + fo.write(self.fini.cproto(['struct %s **' % sn], w)) for i in self.methods: - fo.write(i.proto.cstuff(['VRT_CTX', 'struct %s *' % sn], w)) + fo.write(i.proto.cproto(['VRT_CTX', 'struct %s *' % sn], w)) fo.write("\n") def cstruct(self, fo, define): @@ -784,26 +760,24 @@ class ObjectStanza(Stanza): l2 = ["$INIT"] ll.append(l2) - self.init.json(l2, self.init.name) + self.init.jsonproto(l2, self.init.name) l2 = ["$FINI"] ll.append(l2) - self.fini.json(l2, self.fini.name) + self.fini.jsonproto(l2, self.fini.name) for i in self.methods: i.json(ll) jl.append(ll) - def dump(self): - super(ObjectStanza, self).dump() - for i in self.methods: - i.dump() - ####################################################################### class MethodStanza(Stanza): + + ''' $Method TYPE . method ( ARGUMENSTS ) ''' + def parse(self): p = self.vcc.contents[-1] assert isinstance(p, ObjectStanza) @@ -824,7 +798,7 @@ class MethodStanza(Stanza): def json(self, jl): jl.append(["$METHOD", self.proto.name[len(self.pfx)+1:]]) - self.proto.json(jl[-1], self.proto.cname()) + self.proto.jsonproto(jl[-1], self.proto.cname()) ####################################################################### @@ -877,12 +851,11 @@ class vcc(object): ss = re.split('\n([^\t ])', s.pop(0), maxsplit=1) toks = self.tokenize(ss[0]) inputline = '$' + ' '.join(toks) - c = ss[0].split() - d = "".join(ss[1:]) - m = DISPATCH.get(toks[0]) - if m is None: + docstr = "".join(ss[1:]) + stanzaclass = DISPATCH.get(toks[0]) + if stanzaclass is None: err("Unknown stanza $%s" % toks[0], warn=False) - m(toks, [c[0], " ".join(c[1:])], d.split('\n'), self) + stanzaclass(self, toks, docstr) inputline = None def tokenize(self, txt, seps=None, quotes=None): @@ -922,41 +895,43 @@ class vcc(object): # print("\t", [i]) return out - - def rst_copyright(self, fo): - write_rst_hdr(fo, "COPYRIGHT", "=") - fo.write("\n::\n\n") - a = self.copyright - a = a.replace("\n#", "\n ") - if a[:2] == "#\n": - a = a[2:] - if a[:3] == "#-\n": - a = a[3:] - fo.write(a + "\n") - def rstfile(self, man=False): + ''' Produce rst documentation ''' fn = os.path.join(self.rstdir, "vmod_" + self.modname) if man: fn += ".man" fn += ".rst" fo = self.openfile(fn) write_rst_file_warning(fo) - fo.write(".. role:: ref(emphasis)\n\n") + if man: + fo.write(".. role:: ref(emphasis)\n") + else: + fo.write('\n:tocdepth: 1\n') for i in self.contents: i.rstfile(fo, man) if self.copyright: - self.rst_copyright(fo) + write_rst_hdr(fo, "COPYRIGHT", "=") + fo.write("\n::\n\n") + a = self.copyright + a = a.replace("\n#", "\n ") + if a[:2] == "#\n": + a = a[2:] + if a[:3] == "#-\n": + a = a[3:] + fo.write(a + "\n") fo.close() def amboilerplate(self): + ''' Produce boilplate for autocrap tools ''' fo = self.openfile("automake_boilerplate.am") fo.write(AMBOILERPLATE.replace("XXX", self.modname)) fo.close() - def hfile(self): + def mkhfile(self): + ''' Produce vcc_if.h file ''' fn = self.pfx + ".h" fo = self.openfile(fn) write_c_file_warning(fo) @@ -998,30 +973,23 @@ class vcc(object): for j in self.contents: j.json(jl) - bz = bytearray(json.dumps(jl, separators=(",", ":")), - encoding="ascii") + b"\0" - fo.write("\nstatic const char Vmod_Json[%d] = {\n" % len(bz)) - t = "\t" - for i in bz: - t += "%d," % i - if len(t) >= 69: - fo.write(t + "\n") - t = "\t" - if len(t) > 1: - fo.write(t[:-1]) - fo.write("\n};\n\n") - for i in json.dumps(jl, indent=2, separators=(',', ': ')).split("\n"): - j = "// " + i - if len(j) > 72: - fo.write(j[:72] + "[...]\n") + fo.write("\nstatic const char Vmod_Json[] = {\n") + t = '\t"' + for i in json.dumps(jl, indent=2, separators=(",", ": ")): + if i == '\n': + fo.write(t + '\\n"\n') + t = '\t"' else: - fo.write(j + "\n") - fo.write("\n") + if i in '"\\': + t += '\\' + t += i + fo.write(t + '\\n"\n};\n') def vmod_data(self, fo): vmd = "Vmod_%s_Data" % self.modname + fo.write('\n') for i in (714, 759, 765): - fo.write("\n/*lint -esym(%d, %s) */\n" % (i, vmd)) + fo.write("/*lint -esym(%d, %s) */\n" % (i, vmd)) fo.write("\nextern const struct vmod_data %s;\n" % vmd) fo.write("\nconst struct vmod_data %s = {\n" % vmd) if self.strict_abi: @@ -1039,7 +1007,8 @@ class vcc(object): fo.write("\t.file_id =\t\"%s\",\n" % self.file_id) fo.write("};\n") - def cfile(self): + def mkcfile(self): + ''' Produce vcc_if.c file ''' fno = self.pfx + ".c" fo = self.openfile(fno) fnx = fno + ".tmp2" @@ -1102,8 +1071,8 @@ def runmain(inputvcc, rstdir, outputprefix): v.rstfile(man=False) v.rstfile(man=True) - v.hfile() - v.cfile() + v.mkhfile() + v.mkcfile() if opts.boilerplate: v.amboilerplate() diff --git a/lib/libvmod_blob/vmod.vcc b/lib/libvmod_blob/vmod.vcc index a5b6ae4fd..ab545480e 100644 --- a/lib/libvmod_blob/vmod.vcc +++ b/lib/libvmod_blob/vmod.vcc @@ -6,9 +6,8 @@ # Geoffrey Simmons # -$Module blob 3 "Utilities for the VCL blob type, encoding and decoding" - $ABI strict +$Module blob 3 "Utilities for the VCL blob type, encoding and decoding" DESCRIPTION =========== diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index 5faa718bf..65464fd32 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -26,9 +26,9 @@ # SUCH DAMAGE. # -$Module debug 3 "Development, test and debug" $ABI strict $Prefix xyzzy +$Module debug 3 "Development, test and debug" $Synopsis auto DESCRIPTION =========== diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc index 54b111fba..a98352983 100644 --- a/lib/libvmod_directors/vmod.vcc +++ b/lib/libvmod_directors/vmod.vcc @@ -32,8 +32,8 @@ # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. -$Module directors 3 "Varnish Directors Module" $ABI strict +$Module directors 3 "Varnish Directors Module" DESCRIPTION =========== diff --git a/lib/libvmod_proxy/vmod.vcc b/lib/libvmod_proxy/vmod.vcc index 79aa4b678..2d0663582 100644 --- a/lib/libvmod_proxy/vmod.vcc +++ b/lib/libvmod_proxy/vmod.vcc @@ -25,8 +25,8 @@ # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. -$Module proxy 3 "Varnish Module to extract TLV attributes from PROXYv2" $ABI strict +$Module proxy 3 "Varnish Module to extract TLV attributes from PROXYv2" DESCRIPTION =========== diff --git a/lib/libvmod_purge/vmod.vcc b/lib/libvmod_purge/vmod.vcc index e158aac69..c8d8185cc 100644 --- a/lib/libvmod_purge/vmod.vcc +++ b/lib/libvmod_purge/vmod.vcc @@ -25,8 +25,8 @@ # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. -$Module purge 3 "Varnish Purge Module" $ABI strict +$Module purge 3 "Varnish Purge Module" DESCRIPTION =========== diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index fc5ef3736..0ae941c12 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -25,8 +25,8 @@ # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. -$Module std 3 "Varnish Standard Module" $ABI strict +$Module std 3 "Varnish Standard Module" DESCRIPTION =========== diff --git a/lib/libvmod_unix/vmod.vcc b/lib/libvmod_unix/vmod.vcc index e227e9f99..38e48a7d3 100644 --- a/lib/libvmod_unix/vmod.vcc +++ b/lib/libvmod_unix/vmod.vcc @@ -5,9 +5,8 @@ # Authors: Geoffrey Simmons # -$Module unix 3 "Utilities for Unix domain sockets" - $ABI strict +$Module unix 3 "Utilities for Unix domain sockets" DESCRIPTION =========== diff --git a/lib/libvmod_vtc/vmod.vcc b/lib/libvmod_vtc/vmod.vcc index 248f4b2d6..e1644fffc 100644 --- a/lib/libvmod_vtc/vmod.vcc +++ b/lib/libvmod_vtc/vmod.vcc @@ -25,8 +25,8 @@ # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. -$Module vtc 3 "Utility module for varnishtest" $ABI strict +$Module vtc 3 "Utility module for varnishtest" DESCRIPTION =========== From nils.goroll at uplex.de Wed Jan 23 18:14:02 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 23 Jan 2019 18:14:02 +0000 (UTC) Subject: [master] 545055090 Change type of VCL_BLOB to newly introduced struct vrt_blob Message-ID: <20190123181403.071517B09@lists.varnish-cache.org> commit 5450550906d0809dfc21ae9d3a5b14446c1809d4 Author: Nils Goroll Date: Wed Jan 23 18:58:30 2019 +0100 Change type of VCL_BLOB to newly introduced struct vrt_blob Re-using struct vmod_priv helped simplicity, but had the following shortcomings: - in ddfda3d7c996e67a1894fc3955eb546ebd0b721b we documented what was long implied: return values of vmod functions are assumed immutable and so are blobs. So the blob pointer should be const, yet the struct vmod_priv pointer can't be. - the struct vmod_priv free pointer implies that there would be automatic memory management provided by varnish core, yet this does not exist for a good reason: We would otherwise need to track all blobs ever returned by vmod functions/methods. So we turn the data pointer into a const and remove the free function callback. We also add a type field, which is to be viewed similar to the miniobj magic, except that it should not be asserted upon. The type field is intended for additional (yet unreliable) checks of vmods using BLOBs to carry around vmod-specific private data. diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index e1ac810c4..45b4e1399 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -839,21 +839,24 @@ VRT_ipcmp(VCL_IP sua1, VCL_IP sua2) return (VSA_Compare_IP(sua1, sua2)); } +/* + * the pointer passed as src must have at least VCL_TASK lifetime + */ VCL_BLOB -VRT_blob(VRT_CTX, const char *err, const void *src, size_t len) +VRT_blob(VRT_CTX, const char *err, const void *src, size_t len, unsigned type) { - struct vmod_priv *p; - void *d; + struct vrt_blob *p; p = (void *)WS_Alloc(ctx->ws, sizeof *p); - d = WS_Copy(ctx->ws, src, len); - if (p == NULL || d == NULL) { + if (p == NULL) { VRT_fail(ctx, "Workspace overflow (%s)", err); return (NULL); } - memset(p, 0, sizeof *p); + + p->type = type; p->len = len; - p->priv = d; + p->blob = src; + return (p); } diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index a141df54d..9b4f5fedb 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -891,12 +891,16 @@ VRT_BODY_L(resp) /*--------------------------------------------------------------------*/ + /* digest */ +#define BLOB_HASH_TYPE 0x00d16357 + VCL_BLOB VRT_r_req_hash(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); - return (VRT_blob(ctx, "req.hash", ctx->req->digest, DIGEST_LEN)); + return (VRT_blob(ctx, "req.hash", ctx->req->digest, DIGEST_LEN, + BLOB_HASH_TYPE)); } VCL_BLOB @@ -904,7 +908,8 @@ VRT_r_bereq_hash(VRT_CTX) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); - return (VRT_blob(ctx, "bereq.hash", ctx->bo->digest, DIGEST_LEN)); + return (VRT_blob(ctx, "bereq.hash", ctx->bo->digest, DIGEST_LEN, + BLOB_HASH_TYPE)); } /*--------------------------------------------------------------------*/ diff --git a/include/vrt.h b/include/vrt.h index 35d9dcaa1..5ad307294 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -56,6 +56,8 @@ * HTTP_Copy() removed * HTTP_Dup() added * HTTP_Clone() added + * changed type of VCL_BLOB to newly introduced struct vrt_blob * + * changed VRT_blob() * 8.0 (2018-09-15) * VRT_Strands() added * VRT_StrandsWS() added @@ -118,7 +120,7 @@ * vrt_acl type added */ -#define VRT_MAJOR_VERSION 8U +#define VRT_MAJOR_VERSION 9U #define VRT_MINOR_VERSION 0U @@ -150,6 +152,32 @@ struct strands { const char **p; }; +/* + * VCL_BLOB: + * + * opaque, immutable data (pointer + length), minimum lifetime is the VCL task. + * + * type (optional) is owned by the creator of the blob. blob consumers may use + * it for checks, but should not assert on it. + * + * The data behind the blob pointer is assumed to be immutable for the blob's + * lifetime. + * + * Memory management is either implicit or up to the vmod: + * + * - memory for shortlived blobs should come from the respective workspace + * + * - management of memory for longer lived blobs is up to the vmod + * (in which case the blob will probably be embedded in an object or + * referenced by other state with vcl lifetime) + */ + +struct vrt_blob { + unsigned type; + size_t len; + const void *blob; +}; + /*********************************************************************** * This is the central definition of the mapping from VCL types to * C-types. The python scripts read these from here. @@ -158,7 +186,7 @@ struct strands { typedef const struct vrt_acl * VCL_ACL; typedef const struct director * VCL_BACKEND; -typedef const struct vmod_priv * VCL_BLOB; +typedef const struct vrt_blob * VCL_BLOB; typedef const char * VCL_BODY; typedef unsigned VCL_BOOL; typedef int64_t VCL_BYTES; @@ -417,7 +445,7 @@ VCL_VOID VRT_hashdata(VRT_CTX, const char *str, ...); int VRT_strcmp(const char *s1, const char *s2); void VRT_memmove(void *dst, const void *src, unsigned len); VCL_BOOL VRT_ipcmp(VCL_IP, VCL_IP); -VCL_BLOB VRT_blob(VRT_CTX, const char *, const void *, size_t); +VCL_BLOB VRT_blob(VRT_CTX, const char *, const void *, size_t, unsigned); VCL_VOID VRT_Rollback(VRT_CTX, VCL_HTTP); diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c index 28e486856..66b9f4e24 100644 --- a/lib/libvmod_blob/vmod_blob.c +++ b/lib/libvmod_blob/vmod_blob.c @@ -34,10 +34,13 @@ #include "vcc_if.h" #include "vmod_blob.h" +#define VMOD_BLOB_TYPE 0xfade4faa + struct vmod_blob_blob { unsigned magic; #define VMOD_BLOB_MAGIC 0xfade4fa9 - struct vmod_priv blob; + struct vrt_blob blob; + void *freeptr; char *encoding[__MAX_ENCODING][2]; pthread_mutex_t lock; }; @@ -105,14 +108,12 @@ static const struct vmod_blob_fptr { static char empty[1] = { '\0' }; -static const struct vmod_priv null_blob[1] = -{ - { - .priv = empty, - .len = 0, - .free = NULL - } -}; +static const struct vrt_blob null_blob[1] = {{ +#define VMOD_BLOB_NULL_TYPE 0xfade4fa0 + .type = VMOD_BLOB_NULL_TYPE, + .len = 0, + .blob = empty, +}}; static enum encoding parse_encoding(VCL_ENUM e) @@ -187,6 +188,7 @@ vmod_blob__init(VRT_CTX, struct vmod_blob_blob **blobp, const char *vcl_name, { struct vmod_blob_blob *b; enum encoding dec = parse_encoding(decs); + void *buf; ssize_t len; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -201,36 +203,37 @@ vmod_blob__init(VRT_CTX, struct vmod_blob_blob **blobp, const char *vcl_name, *blobp = b; AZ(pthread_mutex_init(&b->lock, NULL)); + b->blob.type = VMOD_BLOB_TYPE; + len = decode_l(dec, strings); if (len == 0) return; assert(len > 0); - b->blob.priv = malloc(len); - if (b->blob.priv == NULL) { + buf = malloc(len); + if (buf == NULL) { VERRNOMEM(ctx, "cannot create blob %s", vcl_name); return; } errno = 0; - len = func[dec].decode(dec, b->blob.priv, len, -1, strings); + len = func[dec].decode(dec, buf, len, -1, strings); if (len == -1) { assert(errno == EINVAL); - free(b->blob.priv); - b->blob.priv = NULL; + free(buf); VERR(ctx, "cannot create blob %s, illegal encoding beginning " "with \"%s\"", vcl_name, strings->p[0]); return; } if (len == 0) { - b->blob.len = 0; - free(b->blob.priv); - b->blob.priv = NULL; + free(buf); + memcpy(&b->blob, null_blob, sizeof b->blob); return; } b->blob.len = len; + b->blob.blob = b->freeptr = buf; } VCL_BLOB v_matchproto_(td_blob_blob_get) @@ -238,7 +241,7 @@ vmod_blob_get(VRT_CTX, struct vmod_blob_blob *b) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(b, VMOD_BLOB_MAGIC); - return &b->blob; + return (&b->blob); } VCL_STRING v_matchproto_(td_blob_blob_encode) @@ -276,7 +279,7 @@ vmod_blob_encode(VRT_CTX, struct vmod_blob_blob *b, VCL_ENUM encs, len = func[enc].encode( enc, kase, s, len, - b->blob.priv, + b->blob.blob, b->blob.len); assert(len >= 0); if (len == 0) { @@ -304,9 +307,9 @@ vmod_blob__fini(struct vmod_blob_blob **blobp) b = *blobp; *blobp = NULL; CHECK_OBJ(b, VMOD_BLOB_MAGIC); - if (b->blob.priv != NULL) { - free(b->blob.priv); - b->blob.priv = NULL; + if (b->freeptr != NULL) { + free(b->freeptr); + b->blob.blob = NULL; } for (int i = 0; i < __MAX_ENCODING; i++) for (int j = 0; j < 2; j++) { @@ -326,9 +329,7 @@ VCL_BLOB v_matchproto_(td_blob_decode) vmod_decode(VRT_CTX, VCL_ENUM decs, VCL_INT length, VCL_STRANDS strings) { enum encoding dec = parse_encoding(decs); - struct vmod_priv *b; char *buf; - uintptr_t snap; ssize_t len; unsigned space; @@ -337,12 +338,6 @@ vmod_decode(VRT_CTX, VCL_ENUM decs, VCL_INT length, VCL_STRANDS strings) AN(strings); CHECK_OBJ_NOTNULL(ctx->ws, WS_MAGIC); - snap = WS_Snapshot(ctx->ws); - if ((b = WS_Alloc(ctx->ws, sizeof(struct vmod_priv))) == NULL) { - ERRNOMEM(ctx, "cannot decode"); - return NULL; - } - buf = WS_Front(ctx->ws); space = WS_Reserve(ctx->ws, 0); @@ -354,19 +349,17 @@ vmod_decode(VRT_CTX, VCL_ENUM decs, VCL_INT length, VCL_STRANDS strings) if (len == -1) { err_decode(ctx, strings->p[0]); WS_Release(ctx->ws, 0); - WS_Reset(ctx->ws, snap); return NULL; } if (len == 0) { WS_Release(ctx->ws, 0); - WS_Reset(ctx->ws, snap); return null_blob; } WS_Release(ctx->ws, len); - b->priv = buf; - b->len = len; - b->free = NULL; - return (b); + + assert(len > 0); + + return (VRT_blob(ctx, "blob.decode", buf, len, VMOD_BLOB_TYPE)); } static VCL_STRING @@ -387,7 +380,7 @@ encode(VRT_CTX, enum encoding enc, enum case_e kase, VCL_BLOB b) buf = WS_Front(ctx->ws); space = WS_Reserve(ctx->ws, 0); - len = func[enc].encode(enc, kase, buf, space, b->priv, b->len); + len = func[enc].encode(enc, kase, buf, space, b->blob, b->len); if (len == -1) { ERRNOMEM(ctx, "cannot encode"); @@ -424,7 +417,7 @@ vmod_transcode(VRT_CTX, VCL_ENUM decs, VCL_ENUM encs, VCL_ENUM case_s, enum encoding dec = parse_encoding(decs); enum encoding enc = parse_encoding(encs); enum case_e kase = parse_case(case_s); - struct vmod_priv b; + struct vrt_blob b; VCL_STRING r; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -444,15 +437,15 @@ vmod_transcode(VRT_CTX, VCL_ENUM decs, VCL_ENUM encs, VCL_ENUM case_s, size_t l = decode_l(dec, strings); if (l == 0) return ""; + /* XXX: handle stack overflow? */ char buf[l]; - b.free = NULL; - b.priv = buf; if (length <= 0) length = -1; errno = 0; b.len = func[dec].decode(dec, buf, l, length, strings); + b.blob = buf; if (b.len == -1) { err_decode(ctx, strings->p[0]); @@ -487,7 +480,7 @@ vmod_same(VRT_CTX, VCL_BLOB b1, VCL_BLOB b2) return 1; if (b1 == NULL || b2 == NULL) return 0; - return (b1->len == b2->len && b1->priv == b2->priv); + return (b1->len == b2->len && b1->blob == b2->blob); } VCL_BOOL v_matchproto_(td_blob_equal) @@ -501,11 +494,11 @@ vmod_equal(VRT_CTX, VCL_BLOB b1, VCL_BLOB b2) return 0; if (b1->len != b2->len) return 0; - if (b1->priv == b2->priv) + if (b1->blob == b2->blob) return 1; - if (b1->priv == NULL || b2->priv == NULL) + if (b1->blob == NULL || b2->blob == NULL) return 0; - return (memcmp(b1->priv, b2->priv, b1->len) == 0); + return (memcmp(b1->blob, b2->blob, b1->len) == 0); } VCL_INT v_matchproto_(td_blob_length) @@ -521,21 +514,20 @@ vmod_length(VRT_CTX, VCL_BLOB b) VCL_BLOB v_matchproto_(td_blob_sub) vmod_sub(VRT_CTX, VCL_BLOB b, VCL_BYTES n, VCL_BYTES off) { - uintptr_t snap; - struct vmod_priv *sub; - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); assert(n >= 0); assert(off >= 0); - if (b == NULL || b->len == 0 || b->priv == NULL) { + if (b == NULL || b->len == 0 || b->blob == NULL) { ERR(ctx, "blob is empty in blob.sub()"); return NULL; } - assert(b->len >= 0); + + assert(b->len > 0); + if (off + n > b->len) { VERR(ctx, "size %jd from offset %jd requires more bytes than " - "blob length %d in blob.sub()", + "blob length %zd in blob.sub()", (intmax_t)n, (intmax_t)off, b->len); return NULL; } @@ -543,17 +535,7 @@ vmod_sub(VRT_CTX, VCL_BLOB b, VCL_BYTES n, VCL_BYTES off) if (n == 0) return null_blob; - snap = WS_Snapshot(ctx->ws); - if ((sub = WS_Alloc(ctx->ws, sizeof(*sub))) == NULL) { - ERRNOMEM(ctx, "Allocating BLOB result in blob.sub()"); - return NULL; - } - if ((sub->priv = WS_Alloc(ctx->ws, n)) == NULL) { - VERRNOMEM(ctx, "Allocating %jd bytes in blob.sub()", (intmax_t)n); - WS_Reset(ctx->ws, snap); - return NULL; - } - memcpy(sub->priv, (char *)b->priv + off, n); - sub->len = n; - return sub; + + return (VRT_blob(ctx, "blob.sub()", + (const char *)b->blob + off, n, b->type)); } diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index 3f4924bc3..9e8abcbc0 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -113,6 +113,8 @@ enum vmod_directors_shard_param_scope { struct vmod_directors_shard_param; +#define VMOD_SHARD_SHARD_PARAM_BLOB 0xdf5ca116 + struct vmod_directors_shard_param { unsigned magic; #define VMOD_SHARD_SHARD_PARAM_MAGIC 0xdf5ca117 @@ -157,7 +159,7 @@ shard_param_task(VRT_CTX, const void *id, const struct vmod_directors_shard_param *pa); static const struct vmod_directors_shard_param * -shard_param_blob(const VCL_BLOB blob); +shard_param_blob(VCL_BLOB blob); static const struct vmod_directors_shard_param * vmod_shard_param_read(VRT_CTX, const void *id, @@ -469,19 +471,19 @@ static uint32_t shard_blob_key(VCL_BLOB key_blob) { uint8_t k[4] = { 0 }; - uint8_t *b; + const uint8_t *b; int i, ki; - assert(key_blob); + AN(key_blob); + AN(key_blob->blob); assert(key_blob->len > 0); - assert(key_blob->priv != NULL); if (key_blob->len >= 4) ki = 0; else ki = 4 - key_blob->len; - b = key_blob->priv; + b = key_blob->blob; for (i = 0; ki < 4; i++, ki++) k[ki] = b[i]; assert(i <= key_blob->len); @@ -570,7 +572,7 @@ shard_param_args(VRT_CTX, return (NULL); } if (key_blob == NULL || key_blob->len <= 0 || - key_blob->priv == NULL) { + key_blob->blob == NULL) { sharddir_err(ctx, SLT_Error, "%s %s: " "by=BLOB but no or empty key_blob " "- using key 0", @@ -1014,12 +1016,17 @@ vmod_shard_param_get_healthy(VRT_CTX, } static const struct vmod_directors_shard_param * -shard_param_blob(const VCL_BLOB blob) +shard_param_blob(VCL_BLOB blob) { - if (blob && blob->priv && - blob->len == sizeof(struct vmod_directors_shard_param) && - *(unsigned *)blob->priv == VMOD_SHARD_SHARD_PARAM_MAGIC) - return (blob->priv); + const struct vmod_directors_shard_param *p; + + if (blob && blob->type == VMOD_SHARD_SHARD_PARAM_BLOB && + blob->blob != NULL && + blob->len == sizeof(struct vmod_directors_shard_param)) { + CAST_OBJ_NOTNULL(p, blob->blob, VMOD_SHARD_SHARD_PARAM_MAGIC); + return (p); + } + return (NULL); } @@ -1027,20 +1034,9 @@ VCL_BLOB v_matchproto_(td_directors_shard_param_use) vmod_shard_param_use(VRT_CTX, struct vmod_directors_shard_param *p) { - struct vmod_priv *blob; - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(p, VMOD_SHARD_SHARD_PARAM_MAGIC); - blob = (void *)WS_Alloc(ctx->ws, sizeof *blob); - if (blob == NULL) { - VRT_fail(ctx, "Workspace overflow (param.use())"); - return (NULL); - } - - memset(blob, 0, sizeof *blob); - blob->len = sizeof *p; - blob->priv = p; - - return (blob); + return (VRT_blob(ctx, "xshard_param.use()", p, sizeof *p, + VMOD_SHARD_SHARD_PARAM_BLOB)); } From nils.goroll at uplex.de Thu Jan 24 16:20:11 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 24 Jan 2019 16:20:11 +0000 (UTC) Subject: [master] 1f32c1597 varnishlog: When writing binary logs, always include Begin/End/Link tags Message-ID: <20190124162011.75582A874E@lists.varnish-cache.org> commit 1f32c1597a0927e3cd0f728069613e3377572a83 Author: Nils Goroll Date: Thu Jan 24 17:18:14 2019 +0100 varnishlog: When writing binary logs, always include Begin/End/Link tags Closes #2852 diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c index 09f8983d5..b6a76d7df 100644 --- a/bin/varnishlog/varnishlog.c +++ b/bin/varnishlog/varnishlog.c @@ -142,10 +142,15 @@ main(int argc, char * const *argv) VUT_Error(vut, 1, "Missing -w option"); /* Setup output */ - if (LOG.A_opt || !LOG.w_arg) + if (LOG.A_opt || !LOG.w_arg) { vut->dispatch_f = VSL_PrintTransactions; - else + } else { vut->dispatch_f = VSL_WriteTransactions; + // inefficient but not crossing API layers + AN(VUT_Arg(vut, 'i', "Link")); + AN(VUT_Arg(vut, 'i', "Begin")); + AN(VUT_Arg(vut, 'i', "End")); + } if (LOG.w_arg) { openout(LOG.a_opt); AN(LOG.fo); From geoff at uplex.de Fri Jan 25 11:42:07 2019 From: geoff at uplex.de (Geoff Simmons) Date: Fri, 25 Jan 2019 11:42:07 +0000 (UTC) Subject: [master] cdebe44a3 Revert "varnishlog: When writing binary logs, always include Begin/End/Link tags" Message-ID: <20190125114207.E9E9891BE2@lists.varnish-cache.org> commit cdebe44a3cedb925a871ebda020c26438a6b7d60 Author: Geoff Simmons Date: Fri Jan 25 12:31:37 2019 +0100 Revert "varnishlog: When writing binary logs, always include Begin/End/Link tags" This reverts commit 1f32c1597a0927e3cd0f728069613e3377572a83. VUT_Arg('i') excludes all records with tags that are not explicitly marked as included, so this change had the effect that only records with Begin, End and Link, and any other tags that might be mentioned for inclusion, get written to binary logs. For u00006.vtc in particular, only those three record types were written to the log. What we need here is that the three record types are never excluded, in order to get transaction grouping right for reads from binary logs. diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c index b6a76d7df..09f8983d5 100644 --- a/bin/varnishlog/varnishlog.c +++ b/bin/varnishlog/varnishlog.c @@ -142,15 +142,10 @@ main(int argc, char * const *argv) VUT_Error(vut, 1, "Missing -w option"); /* Setup output */ - if (LOG.A_opt || !LOG.w_arg) { + if (LOG.A_opt || !LOG.w_arg) vut->dispatch_f = VSL_PrintTransactions; - } else { + else vut->dispatch_f = VSL_WriteTransactions; - // inefficient but not crossing API layers - AN(VUT_Arg(vut, 'i', "Link")); - AN(VUT_Arg(vut, 'i', "Begin")); - AN(VUT_Arg(vut, 'i', "End")); - } if (LOG.w_arg) { openout(LOG.a_opt); AN(LOG.fo); From nils.goroll at uplex.de Fri Jan 25 12:34:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 25 Jan 2019 12:34:07 +0000 (UTC) Subject: [master] 26776d12f Take two: When writing binary logs, always include Begin/End/Link tags Message-ID: <20190125123407.86CC092EA8@lists.varnish-cache.org> commit 26776d12ff3eafd233b832e2bf228d23bd44457d Author: Nils Goroll Date: Fri Jan 25 13:16:58 2019 +0100 Take two: When writing binary logs, always include Begin/End/Link tags Apologies for the bad version in 1f32c1597a0927e3cd0f728069613e3377572a83 Closes #2852 diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c index 09f8983d5..003528b46 100644 --- a/bin/varnishlog/varnishlog.c +++ b/bin/varnishlog/varnishlog.c @@ -142,10 +142,19 @@ main(int argc, char * const *argv) VUT_Error(vut, 1, "Missing -w option"); /* Setup output */ - if (LOG.A_opt || !LOG.w_arg) + if (LOG.A_opt || !LOG.w_arg) { vut->dispatch_f = VSL_PrintTransactions; - else + } else { vut->dispatch_f = VSL_WriteTransactions; + /* + * inefficient but not crossing API layers + * first x argument avoids initial suppression of all tags + */ + AN(VUT_Arg(vut, 'x', "Link")); + AN(VUT_Arg(vut, 'i', "Link")); + AN(VUT_Arg(vut, 'i', "Begin")); + AN(VUT_Arg(vut, 'i', "End")); + } if (LOG.w_arg) { openout(LOG.a_opt); AN(LOG.fo); From fgsch at lodoss.net Fri Jan 25 14:50:09 2019 From: fgsch at lodoss.net (Federico Schwindt) Date: Fri, 25 Jan 2019 14:50:09 +0000 Subject: [master] cdebe44a3 Revert "varnishlog: When writing binary logs, always include Begin/End/Link tags" In-Reply-To: <20190125114207.E9E9891BE2@lists.varnish-cache.org> References: <20190125114207.E9E9891BE2@lists.varnish-cache.org> Message-ID: This smells like we are missing a test. On Fri, Jan 25, 2019 at 11:42 AM Geoff Simmons wrote: > > commit cdebe44a3cedb925a871ebda020c26438a6b7d60 > Author: Geoff Simmons > Date: Fri Jan 25 12:31:37 2019 +0100 > > Revert "varnishlog: When writing binary logs, always include > Begin/End/Link tags" > > This reverts commit 1f32c1597a0927e3cd0f728069613e3377572a83. > > VUT_Arg('i') excludes all records with tags that are not explicitly > marked as included, so this change had the effect that only records > with Begin, End and Link, and any other tags that might be mentioned > for inclusion, get written to binary logs. For u00006.vtc in > particular, only those three record types were written to the log. > > What we need here is that the three record types are never excluded, > in order to get transaction grouping right for reads from binary logs. > > diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c > index b6a76d7df..09f8983d5 100644 > --- a/bin/varnishlog/varnishlog.c > +++ b/bin/varnishlog/varnishlog.c > @@ -142,15 +142,10 @@ main(int argc, char * const *argv) > VUT_Error(vut, 1, "Missing -w option"); > > /* Setup output */ > - if (LOG.A_opt || !LOG.w_arg) { > + if (LOG.A_opt || !LOG.w_arg) > vut->dispatch_f = VSL_PrintTransactions; > - } else { > + else > vut->dispatch_f = VSL_WriteTransactions; > - // inefficient but not crossing API layers > - AN(VUT_Arg(vut, 'i', "Link")); > - AN(VUT_Arg(vut, 'i', "Begin")); > - AN(VUT_Arg(vut, 'i', "End")); > - } > if (LOG.w_arg) { > openout(LOG.a_opt); > AN(LOG.fo); > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit > -------------- next part -------------- An HTML attachment was scrubbed... URL: From nils.goroll at uplex.de Fri Jan 25 14:57:37 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 25 Jan 2019 15:57:37 +0100 Subject: [master] cdebe44a3 Revert "varnishlog: When writing binary logs, always include Begin/End/Link tags" In-Reply-To: References: <20190125114207.E9E9891BE2@lists.varnish-cache.org> Message-ID: On 25/01/2019 15:50, Federico Schwindt wrote: > This smells like we are missing a test. No. The only smell this has is that someone(tm) failed run make check :((( Apologies again -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From geoff at uplex.de Fri Jan 25 14:57:41 2019 From: geoff at uplex.de (Geoff Simmons) Date: Fri, 25 Jan 2019 15:57:41 +0100 Subject: [master] cdebe44a3 Revert "varnishlog: When writing binary logs, always include Begin/End/Link tags" In-Reply-To: References: <20190125114207.E9E9891BE2@lists.varnish-cache.org> Message-ID: <367a17ce-6eb5-570e-ac2b-e1e1b3423a08@uplex.de> On 1/25/19 15:50, Federico Schwindt wrote: > This smells like we are missing a test. Not sure what you mean -- u00006.vtc failed as a result of the commit that was reverted, with a few hundred VTEST failures overnight, and no OKs: https://varnish-cache.org/vtest/bugs_u00006.html#1f32c15 So u00006.vtc was the test that exposed the problem. -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg Tel +49 40 2880 5731 Mob +49 176 636 90917 Fax +49 40 42949753 http://uplex.de -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 833 bytes Desc: OpenPGP digital signature URL: From fgsch at lodoss.net Fri Jan 25 16:14:11 2019 From: fgsch at lodoss.net (Federico Schwindt) Date: Fri, 25 Jan 2019 16:14:11 +0000 Subject: [master] cdebe44a3 Revert "varnishlog: When writing binary logs, always include Begin/End/Link tags" In-Reply-To: <367a17ce-6eb5-570e-ac2b-e1e1b3423a08@uplex.de> References: <20190125114207.E9E9891BE2@lists.varnish-cache.org> <367a17ce-6eb5-570e-ac2b-e1e1b3423a08@uplex.de> Message-ID: Ah, ok. I assumed the tests were passing. On Fri, Jan 25, 2019 at 2:57 PM Geoff Simmons wrote: > On 1/25/19 15:50, Federico Schwindt wrote: > > This smells like we are missing a test. > > Not sure what you mean -- u00006.vtc failed as a result of the commit > that was reverted, with a few hundred VTEST failures overnight, and no OKs: > > https://varnish-cache.org/vtest/bugs_u00006.html#1f32c15 > > So u00006.vtc was the test that exposed the problem. > > -- > ** * * UPLEX - Nils Goroll Systemoptimierung > > Scheffelstra?e 32 > 22301 Hamburg > > Tel +49 40 2880 5731 > Mob +49 176 636 90917 > Fax +49 40 42949753 > > http://uplex.de > > -------------- next part -------------- An HTML attachment was scrubbed... URL: From phk at phk.freebsd.dk Sat Jan 26 09:13:37 2019 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Sat, 26 Jan 2019 09:13:37 +0000 Subject: [master] 26776d12f Take two: When writing binary logs, always include Begin/End/Link tags In-Reply-To: <20190125123407.86CC092EA8@lists.varnish-cache.org> References: <20190125123407.86CC092EA8@lists.varnish-cache.org> Message-ID: <12354.1548494017@critter.freebsd.dk> -------- In message <20190125123407.86CC092EA8 at lists.varnish-cache.org>, Nils Goroll writes: > Take two: When writing binary logs, always include Begin/End/Link tags > >diff --git a/bin/varnishlog/varnishlog.c b/bin/varnishlog/varnishlog.c >index 09f8983d5..003528b46 100644 >--- a/bin/varnishlog/varnishlog.c >+++ b/bin/varnishlog/varnishlog.c >@@ -142,10 +142,19 @@ main(int argc, char * const *argv) > VUT_Error(vut, 1, "Missing -w option"); > > /* Setup output */ >- if (LOG.A_opt || !LOG.w_arg) >+ if (LOG.A_opt || !LOG.w_arg) { > vut->dispatch_f = VSL_PrintTransactions; >- else >+ } else { > vut->dispatch_f = VSL_WriteTransactions; >+ /* >+ * inefficient but not crossing API layers >+ * first x argument avoids initial suppression of all tags >+ */ >+ AN(VUT_Arg(vut, 'x', "Link")); >+ AN(VUT_Arg(vut, 'i', "Link")); >+ AN(VUT_Arg(vut, 'i', "Begin")); >+ AN(VUT_Arg(vut, 'i', "End")); >+ } > if (LOG.w_arg) { > openout(LOG.a_opt); > AN(LOG.fo); Doesn't this change the interpretation of user-provided arguments ? -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From nils.goroll at uplex.de Sat Jan 26 13:46:36 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 26 Jan 2019 14:46:36 +0100 Subject: [master] 26776d12f Take two: When writing binary logs, always include Begin/End/Link tags In-Reply-To: <12354.1548494017@critter.freebsd.dk> References: <20190125123407.86CC092EA8@lists.varnish-cache.org> <12354.1548494017@critter.freebsd.dk> Message-ID: On 26/01/2019 10:13, Poul-Henning Kamp wrote: > Doesn't this change the interpretation of user-provided arguments ? My mistake with the initial patch was that an initial -i/-I argument clears the tag mask as "no tags except for the given ones". With the initial -x argument added, - if no user arguments were given, the above does not happen due to the initial -x argument and and all tags remain included - if user arguments were given, the -x Link is a noop. And this change only concerns the -w and not -A case. For the rest of the discussion, the scenario at hand is varnishlog -w file ... varnishlog -r file ... and we should be concerned with the output of the second varnishlog. The varnishlog -w will now always write Link/Begin/End, which is the right thing to do, IMHO: The varnishlog -r needs these tags for -g xvid | request |session, so this change is a fix. IIUC, the only side effect is that varnishlog -r file will output the Link/Begin/End tags if no -iIxX filter is given. Does anyone see an issue with this? Or am I missing aynthing else? Nils -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From phk at phk.freebsd.dk Sat Jan 26 14:10:10 2019 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Sat, 26 Jan 2019 14:10:10 +0000 Subject: [master] 26776d12f Take two: When writing binary logs, always include Begin/End/Link tags In-Reply-To: References: <20190125123407.86CC092EA8@lists.varnish-cache.org> <12354.1548494017@critter.freebsd.dk> Message-ID: <13509.1548511810@critter.freebsd.dk> -------- In message , Nils Goroll writes: >> Doesn't this change the interpretation of user-provided arguments ? >IIUC, the only side effect is that varnishlog -r file will output the >Link/Begin/End tags if no -iIxX filter is given. That's what I meant, is that OK ? -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From fgsch at lodoss.net Sun Jan 27 14:55:09 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 27 Jan 2019 14:55:09 +0000 (UTC) Subject: [master] 5cd4e2be9 Spelling Message-ID: <20190127145509.96B0CAF852@lists.varnish-cache.org> commit 5cd4e2be9ea69ad2bca0c5eb51e0efed067cc231 Author: Federico G. Schwindt Date: Sat Jan 26 12:04:41 2019 +0000 Spelling diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 17956311b..b1e4cd590 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -665,7 +665,7 @@ class EventStanza(Stanza): class FunctionStanza(Stanza): - ''' $Function TYPE name ( ARGUMENSTS ) ''' + ''' $Function TYPE name ( ARGUMENTS ) ''' def parse(self): self.proto = ProtoType(self) @@ -688,7 +688,7 @@ class FunctionStanza(Stanza): class ObjectStanza(Stanza): - ''' $Object TYPE class ( ARGUMENSTS ) ''' + ''' $Object TYPE class ( ARGUMENTS ) ''' def parse(self): self.proto = ProtoType(self, retval=False) @@ -776,7 +776,7 @@ class ObjectStanza(Stanza): class MethodStanza(Stanza): - ''' $Method TYPE . method ( ARGUMENSTS ) ''' + ''' $Method TYPE . method ( ARGUMENTS ) ''' def parse(self): p = self.vcc.contents[-1] From phk at FreeBSD.org Mon Jan 28 08:19:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 28 Jan 2019 08:19:07 +0000 (UTC) Subject: [master] 7e10a4658 Stabilize this test with syncvsl Message-ID: <20190128081907.365EF95D31@lists.varnish-cache.org> commit 7e10a4658a64b5d745a81bb5c89d415d01f016b1 Author: Poul-Henning Kamp Date: Mon Jan 28 08:18:10 2019 +0000 Stabilize this test with syncvsl diff --git a/bin/varnishtest/tests/r02849.vtc b/bin/varnishtest/tests/r02849.vtc index 9b4677524..3022bcf78 100644 --- a/bin/varnishtest/tests/r02849.vtc +++ b/bin/varnishtest/tests/r02849.vtc @@ -36,6 +36,8 @@ varnish v1 -vcl+backend { } } -start +varnish v1 -cliok "param.set debug +syncvsl" + varnish v1 -cliok "vcl.label lab1 vcl1" varnish v1 -vcl+backend { @@ -92,23 +94,25 @@ varnish v1 -vsl_catchup logexpect l1 -v v1 -g raw { + expect * 1008 VCL_use "vcl3" + expect * 1008 ReqURL "/l1" + expect * 1008 VCL_use "vcl1 via lab1" + expect * 1009 VCL_use "vcl1" expect * 1009 BeReqURL "/l1" + # yes, twice! + expect * 1010 ReqURL "/l2" + expect * 1010 ReqURL "/l2" + expect * 1011 VCL_use "vcl3" expect * 1011 BeReqURL "/l2" + # yes, twice! expect * 1012 ReqURL "/l3" expect * 1012 ReqURL "/l3" expect * 1012 VCL_use "vcl2 via lab2" expect * 1012 ReqURL "/l3a" - - expect * 1010 ReqURL "/l2" - expect * 1010 ReqURL "/l2" - - expect * 1008 VCL_use "vcl3" - expect * 1008 ReqURL "/l1" - expect * 1008 VCL_use "vcl1 via lab1" } -start # The test which one is picked From phk at FreeBSD.org Mon Jan 28 08:19:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 28 Jan 2019 08:19:07 +0000 (UTC) Subject: [master] 8c2ed731a OCD fix Message-ID: <20190128081907.4B7D495D34@lists.varnish-cache.org> commit 8c2ed731af694eb78611d648b11d25a2894635f8 Author: Poul-Henning Kamp Date: Mon Jan 28 08:18:40 2019 +0000 OCD fix diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 11f046f95..957436d5d 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -323,7 +323,7 @@ logexp_dispatch(struct VSL_data *vsl, struct VSL_transaction * const pt[], VSL_BACKEND(t->c->rec.ptr) ? 'b' : '-'; if (legend != NULL) - vtc_log(le->vl, 4, "%3s| %10u %-15s %c %.*s", + vtc_log(le->vl, 4, "%-5s| %10u %-15s %c %.*s", legend, vxid, VSL_tags[tag], type, len, data); From phk at FreeBSD.org Mon Jan 28 08:45:11 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 28 Jan 2019 08:45:11 +0000 (UTC) Subject: [master] 8840d7618 Note VRT breakage Message-ID: <20190128084512.0CFC7968BE@lists.varnish-cache.org> commit 8840d7618545d0eb79981952c9d8425a9e4f29ec Author: Poul-Henning Kamp Date: Mon Jan 28 08:44:10 2019 +0000 Note VRT breakage Fixes #2885 diff --git a/include/vrt.h b/include/vrt.h index 5ad307294..b2fa4ec4f 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -58,6 +58,8 @@ * HTTP_Clone() added * changed type of VCL_BLOB to newly introduced struct vrt_blob * * changed VRT_blob() + * req->req_bodybytes removed + * use: AZ(ObjGetU64(req->wrk, req->body_oc, OA_LEN, &u)); * 8.0 (2018-09-15) * VRT_Strands() added * VRT_StrandsWS() added From phk at FreeBSD.org Mon Jan 28 09:37:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 28 Jan 2019 09:37:08 +0000 (UTC) Subject: [master] b09f97eeb Rescule the VTLA list from the old wiki snapshot Message-ID: <20190128093708.1A45B97B65@lists.varnish-cache.org> commit b09f97eebd1f92ac2dd4927ee5a8df0062641d69 Author: Poul-Henning Kamp Date: Mon Jan 28 09:35:44 2019 +0000 Rescule the VTLA list from the old wiki snapshot diff --git a/doc/sphinx/reference/index.rst b/doc/sphinx/reference/index.rst index 89375dcb7..c8afd4994 100644 --- a/doc/sphinx/reference/index.rst +++ b/doc/sphinx/reference/index.rst @@ -32,6 +32,7 @@ The Varnish Reference Manual vsl.rst vsl-query.rst vtc.rst + vtla.rst .. todo:: The programs: diff --git a/doc/sphinx/reference/vtla.rst b/doc/sphinx/reference/vtla.rst new file mode 100644 index 000000000..905d2d6c1 --- /dev/null +++ b/doc/sphinx/reference/vtla.rst @@ -0,0 +1,124 @@ +.. role:: ref(emphasis) + +.. _vtla: + +==================================== +VTLA - Varnish Three Letter Acronyms +==================================== + +Very early in the project, we made a fortunate bargain on eBay, +buying up a batch of 676 three letter acronyms, all starting with +'V'. + +This page tells you what we use them for, if & when we remember to +add them... + +VAV + Varnish Arg Vector -- Argv parsing. + +VBE + Varnish Back End -- Code for contacting backends + (bin/varnishd/cache_backend.c) + +VBP + Varnish Backend Polling -- Health checks of backends + (bin/varnishd/cache_backend_poll.c) + +VCA + Varnish Connection Acceptor -- The code that receives/accepts the + TCP connections (bin/varnishd/cache_acceptor.c) + +VCC + VCL to C Compiler -- The code that compiles VCL to C code. (lib/libvcl) + +VCL + Varnish Configuration Language -- The domain-specific programming + language used for configuring a varnishd. + +VCT + Varnish CType(3) -- Character classification for RFC2616 and XML parsing. + +VDD + Varnish (Core) Developer Day -- Quarterly invite-only meeting strictly + for Varnish core (C) developers, packagers and VMOD hackers. + +VEV + Varnish EVent -- library functions to implement a simple event-dispatcher. + +VGB + Varnish Governing Board -- May or may not exist. + If you need to ask, you are not on it. + +VGC + Varnish Generated Code -- Code generated by VCC from VCL. + +VIN + Varnish Instance Naming -- Resolution of -n arguments. + +VLU + Varnish Line Up -- library functions to collect stream of bytes + into lines for processing. (lib/libvarnish/vlu.c) + +VRE + Varnish Regular Expression -- library functions for regular expression + based matching and substring replacement. (lib/libvarnish/vre.c) + +VRT + Varnish Run Time -- functions called from compiled code. + (bin/varnishd/cache_vrt.c) + +VRY + VaRY -- Related to processing of Vary: HTTP headers. + (bin/varnishd/cache_vary.c) + +VSL + Varnish Shared memory Log -- The log written into the shared + memory segment for varnish{log,ncsa,top,hist} to see. + +VSB + Varnish string Buffer -- a copy of the FreeBSD "sbuf" library, + for safe string handling. + +VSC + Varnish Statistics Counter -- counters for various stats, + exposed via varnishapi. + +VSS + Varnish Session Stuff -- library functions to wrap DNS/TCP. + (lib/libvarnish/vss.c) + +VTC + Varnish Test Code -- a test-specification for the varnishtest program. + +VTLA + Varnish Three Letter Acronym -- No rule without an exception. + +VUG + Varnish User Group meeting -- Half-yearly event where the users and + developers of Varnish Cache gather to share experiences and plan + future development. + +VWx + Varnish Waiter 'x' -- A code module to monitor idle sessions. + +VWE + Varnish Waiter Epoll -- epoll(2) (linux) based waiter module. + +VWK + Varnish Waiter Kqueue -- kqueue(2) (freebsd) based waiter module. + +VWP + Varnish Waiter Poll -- poll(2) based waiter module. + +VWS + Varnish Waiter Solaris -- Solaris ports(2) based waiter module. + + + +COPYRIGHT +========= + +This document is licensed under the same licence as Varnish +itself. See LICENCE for details. + +* Copyright (c) 2019 Varnish Software AS From dridi at varni.sh Mon Jan 28 09:48:32 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 28 Jan 2019 10:48:32 +0100 Subject: [master] 26776d12f Take two: When writing binary logs, always include Begin/End/Link tags In-Reply-To: <13509.1548511810@critter.freebsd.dk> References: <20190125123407.86CC092EA8@lists.varnish-cache.org> <12354.1548494017@critter.freebsd.dk> <13509.1548511810@critter.freebsd.dk> Message-ID: > That's what I meant, is that OK ? I think it is. Nils, can you please add a test case where varnishlog is run with both -w and -i, and then -r? You can make that r02852.vtc Dridi From dridi.boukelmoune at gmail.com Mon Jan 28 10:10:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 28 Jan 2019 10:10:09 +0000 (UTC) Subject: [master] 1ac8efd82 Recommend 6.0 to users for the next cycles Message-ID: <20190128101009.CF15D9B8A0@lists.varnish-cache.org> commit 1ac8efd8252e9af12d6ba21bfb81d03220a61c71 Author: Dridi Boukelmoune Date: Mon Jan 28 11:08:31 2019 +0100 Recommend 6.0 to users for the next cycles diff --git a/doc/sphinx/phk/lucky.rst b/doc/sphinx/phk/lucky.rst index 6a4393c97..b7737f442 100644 --- a/doc/sphinx/phk/lucky.rst +++ b/doc/sphinx/phk/lucky.rst @@ -89,7 +89,7 @@ and then we will know and be ready to say something more detailed. I don't think it is realistic to roll out any kind of H3 support in the september release, that release will probably only contain a some of the necessary prepatory reorganization, so expect to -run production on 6.1 for a while. +run production on 6.0 LTS for a while. --------------------- Varnish Moral License From geoff at uplex.de Mon Jan 28 12:27:08 2019 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 28 Jan 2019 12:27:08 +0000 (UTC) Subject: [master] 18e255742 VCC warns (only) if stat(UDS backend) fails with ENOENT or EACCES. Message-ID: <20190128122708.299A2A1786@lists.varnish-cache.org> commit 18e255742334720be7e7a02cc4c088d78be1fb67 Author: Geoff Simmons Date: Fri Jan 25 16:44:20 2019 +0100 VCC warns (only) if stat(UDS backend) fails with ENOENT or EACCES. This makes it easier to start the UDS-listening peer, and/or set permissions on the socket file, after starting Varnish or loading VCL with a UDS backend. Closes #2884 diff --git a/bin/varnishtest/tests/c00086.vtc b/bin/varnishtest/tests/c00086.vtc index e14943729..2ab6350ee 100644 --- a/bin/varnishtest/tests/c00086.vtc +++ b/bin/varnishtest/tests/c00086.vtc @@ -1,4 +1,4 @@ -varnishtest "-a sub-args user, group and mode" +varnishtest "-a sub-args user, group and mode; and warn if stat(EACCES) fails for UDS" feature user_vcache feature group_varnish @@ -51,3 +51,14 @@ shell -match "vcache" { ls -l ${tmpdir}/v6.sock } varnish v7 -arg "-a ${tmpdir}/v7.sock,group=varnish" -vcl+backend {} shell -match "root.+varnish" { ls -l ${tmpdir}/v7.sock } + +# VCC warns, but does not fail, if stat(UDS) fails with EACCES. +shell { mkdir ${tmpdir}/dir } +server s2 -listen ${tmpdir}/dir/s1.sock {} -start + +shell { chmod go-rx ${tmpdir}/dir } + +varnish v8 -arg "-j unix,user=vcache" -vcl { backend b {.host="${bad_ip}";} } + +varnish v8 -cliexpect "(?s)Cannot stat:.+That was just a warning" \ + {vcl.inline test "vcl 4.1; backend b {.path=\"${tmpdir}/dir/s1.sock\";}"} diff --git a/bin/varnishtest/tests/v00038.vtc b/bin/varnishtest/tests/v00038.vtc index d93d4d5f6..d408896b8 100644 --- a/bin/varnishtest/tests/v00038.vtc +++ b/bin/varnishtest/tests/v00038.vtc @@ -123,16 +123,40 @@ varnish v1 -errvcl "Must be an absolute path:" { } } -shell { rm -f ${tmpdir}/foo } - -varnish v1 -errvcl "Cannot stat:" { - backend b1 { - .path = "${tmpdir}/foo"; - } -} - varnish v1 -errvcl "Not a socket:" { backend b1 { .path = "${tmpdir}"; } } + +# VCC warns, but does not fail, if stat(UDS) fails with ENOENT. +shell { rm -f ${tmpdir}/foo } + +varnish v1 -vcl { backend b {.host="${bad_ip}";} } + +varnish v1 -cliexpect "(?s)Cannot stat:.+That was just a warning" \ + {vcl.inline test "vcl 4.1; backend b {.path=\"${tmpdir}/foo\";}"} + +# VCC also warns but doesn't fail for EACCES. Tested in c00086.vtc. + +# The following test verifies that Varnish continues connecting to a +# socket file, even if the listener at that location changes. + +server s1 -listen "${tmpdir}/server.sock" { + rxreq + txresp -hdr "Connection: close" -hdr "Cache-Control: max-age=0" +} -start + +varnish v1 -vcl { + backend b {.path = "${s1_sock}"; } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 +} -run + +server s1 -start + +client c1 -run diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c index e92be266f..f03762b34 100644 --- a/lib/libvcc/vcc_utils.c +++ b/lib/libvcc/vcc_utils.c @@ -276,12 +276,16 @@ Emit_UDS_Path(struct vcc *tl, const struct token *t_path, const char *errid) } errno = 0; if (stat(t_path->dec, &st) != 0) { + int err = errno; VSB_printf(tl->sb, "%s: Cannot stat: %s\n", errid, strerror(errno)); vcc_ErrWhere(tl, t_path); - return; - } - if (!S_ISSOCK(st.st_mode)) { + if (err == ENOENT || err == EACCES) { + VSB_printf(tl->sb, "(That was just a warning)\n"); + tl->err = 0; + } else + return; + } else if (!S_ISSOCK(st.st_mode)) { VSB_printf(tl->sb, "%s: Not a socket:\n", errid); vcc_ErrWhere(tl, t_path); return; From geoff at uplex.de Mon Jan 28 12:27:08 2019 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 28 Jan 2019 12:27:08 +0000 (UTC) Subject: [master] f878f3044 Document the relaxed restrictions on backend UDS declarations. Message-ID: <20190128122708.3ED6FA1789@lists.varnish-cache.org> commit f878f30444ca6cc19175e54c87c84fca69f9f697 Author: Geoff Simmons Date: Fri Jan 25 16:53:59 2019 +0100 Document the relaxed restrictions on backend UDS declarations. diff --git a/doc/sphinx/reference/vcl.rst b/doc/sphinx/reference/vcl.rst index 2f31f4d3f..c1de845e7 100644 --- a/doc/sphinx/reference/vcl.rst +++ b/doc/sphinx/reference/vcl.rst @@ -233,9 +233,17 @@ parameters. The following attributes are available: is declared. ``.path`` (``VCL >= 4.1``) + The absolute path of a Unix domain socket at which a backend is - listening. The file at that path must exist and must be accessible - to Varnish at VCL load time, and it must be a socket. One of + listening. If the file at that path does not exist or is not + accessible to Varnish at VCL load time, then the VCL compiler + issues a warning, but does not fail. This makes it possible to + start the UDS-listening peer, or set the socket file's + permissions, after starting Varnish or loading VCL with a UDS + backend. But the socket file must exist and have necessary + permissions before the first connection is attempted, otherwise + fetches will fail. If the file does exist and is accessible, then + it must be a socket; otherwise the VCL load fails. One of ``.path`` or ``.host`` must be declared (but not both). ``.path`` may only be used in VCL since version 4.1. From dridi.boukelmoune at gmail.com Mon Jan 28 12:50:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 28 Jan 2019 12:50:10 +0000 (UTC) Subject: [master] f81b172db Wearing my tinfoil hat... Message-ID: <20190128125010.2004FA2252@lists.varnish-cache.org> commit f81b172db5d8d98133756051e51b9c0a0a11f7d7 Author: Dridi Boukelmoune Date: Mon Jan 28 13:48:53 2019 +0100 Wearing my tinfoil hat... Refs #2884 diff --git a/bin/varnishtest/tests/v00038.vtc b/bin/varnishtest/tests/v00038.vtc index d408896b8..00cd8fff2 100644 --- a/bin/varnishtest/tests/v00038.vtc +++ b/bin/varnishtest/tests/v00038.vtc @@ -157,6 +157,12 @@ client c1 { expect resp.status == 200 } -run +shell { + # for good measure + test -S "${s1_sock}" && rm -vf "${s1_sock}" + test ! -S "${s1_sock}" +} + server s1 -start client c1 -run From fgsch at lodoss.net Mon Jan 28 12:57:34 2019 From: fgsch at lodoss.net (Federico Schwindt) Date: Mon, 28 Jan 2019 12:57:34 +0000 Subject: [master] 26776d12f Take two: When writing binary logs, always include Begin/End/Link tags In-Reply-To: References: <20190125123407.86CC092EA8@lists.varnish-cache.org> <12354.1548494017@critter.freebsd.dk> <13509.1548511810@critter.freebsd.dk> Message-ID: I'd rather use the existing u00006 test if possible. On Mon, Jan 28, 2019 at 9:49 AM Dridi Boukelmoune wrote: > > That's what I meant, is that OK ? > > I think it is. > > Nils, can you please add a test case where varnishlog is run with both > -w and -i, and then -r? > > You can make that r02852.vtc > > Dridi > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit > -------------- next part -------------- An HTML attachment was scrubbed... URL: From dridi.boukelmoune at gmail.com Mon Jan 28 14:08:08 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 28 Jan 2019 14:08:08 +0000 (UTC) Subject: [master] 73dc986d9 Revert "Wearing my tinfoil hat..." Message-ID: <20190128140808.F16C5A5558@lists.varnish-cache.org> commit 73dc986d995250c99c7fbf1d68641234a043d12c Author: Dridi Boukelmoune Date: Mon Jan 28 15:07:16 2019 +0100 Revert "Wearing my tinfoil hat..." Aka remove linux blinders. This reverts commit f81b172db5d8d98133756051e51b9c0a0a11f7d7. diff --git a/bin/varnishtest/tests/v00038.vtc b/bin/varnishtest/tests/v00038.vtc index 00cd8fff2..d408896b8 100644 --- a/bin/varnishtest/tests/v00038.vtc +++ b/bin/varnishtest/tests/v00038.vtc @@ -157,12 +157,6 @@ client c1 { expect resp.status == 200 } -run -shell { - # for good measure - test -S "${s1_sock}" && rm -vf "${s1_sock}" - test ! -S "${s1_sock}" -} - server s1 -start client c1 -run From geoff at uplex.de Mon Jan 28 14:49:08 2019 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 28 Jan 2019 14:49:08 +0000 (UTC) Subject: [master] 7f17cbec4 Configure jail in the test for UDS access so that it runs with VTEST. Message-ID: <20190128144908.86368A6446@lists.varnish-cache.org> commit 7f17cbec42c5ae8faafe8f589db75a9af4336079 Author: Geoff Simmons Date: Mon Jan 28 15:48:03 2019 +0100 Configure jail in the test for UDS access so that it runs with VTEST. diff --git a/bin/varnishtest/tests/c00086.vtc b/bin/varnishtest/tests/c00086.vtc index 2ab6350ee..62b6ba72d 100644 --- a/bin/varnishtest/tests/c00086.vtc +++ b/bin/varnishtest/tests/c00086.vtc @@ -58,7 +58,10 @@ server s2 -listen ${tmpdir}/dir/s1.sock {} -start shell { chmod go-rx ${tmpdir}/dir } -varnish v8 -arg "-j unix,user=vcache" -vcl { backend b {.host="${bad_ip}";} } +varnish v8 \ + -jail "-junix,user=vcache,ccgroup=varnish,workuser=vcache" \ + -vcl { backend b {.host="${bad_ip}";} +} varnish v8 -cliexpect "(?s)Cannot stat:.+That was just a warning" \ {vcl.inline test "vcl 4.1; backend b {.path=\"${tmpdir}/dir/s1.sock\";}"} From phk at FreeBSD.org Mon Jan 28 15:05:11 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 28 Jan 2019 15:05:11 +0000 (UTC) Subject: [master] b3532c1a1 Cover the rest of the vsl_glob_test program too Message-ID: <20190128150511.356D0A6AA5@lists.varnish-cache.org> commit b3532c1a14abc5b1bdd7eef830c99139e342079a Author: Poul-Henning Kamp Date: Mon Jan 28 15:04:14 2019 +0000 Cover the rest of the vsl_glob_test program too diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am index 77e96d666..fa0f3078b 100644 --- a/lib/libvarnishapi/Makefile.am +++ b/lib/libvarnishapi/Makefile.am @@ -98,10 +98,16 @@ vxp_test_CFLAGS = \ vxp_test_LDADD = @PCRE_LIBS@ \ ${RT_LIBS} ${LIBM} ${PTHREAD_LIBS} -TESTS = vsl_glob_test +TESTS = vsl_glob_test vsl_glob_test_coverage noinst_PROGRAMS += ${TESTS} vsl_glob_test_SOURCES = vsl_glob_test.c vsl_glob_test_CFLAGS = @SAN_CFLAGS@ vsl_glob_test_LDADD = libvarnishapi.la @SAN_LDFLAGS@ + +vsl_glob_test_coverage: + echo './vsl_glob_test 1 2 3 2> /dev/null || true' > ${builddir}/_ + echo './vsl_glob_test "Req*" > /dev/null' >> ${builddir}/_ + mv ${builddir}/_ ${builddir}/vsl_glob_test_coverage + chmod +x ${builddir}/vsl_glob_test_coverage From phk at FreeBSD.org Mon Jan 28 16:18:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 28 Jan 2019 16:18:09 +0000 (UTC) Subject: [master] 8a2b01b70 Fix build. Message-ID: <20190128161809.AB34CA8155@lists.varnish-cache.org> commit 8a2b01b70f4b56c91b821604afa7dfbf8389d2f2 Author: Poul-Henning Kamp Date: Mon Jan 28 16:17:38 2019 +0000 Fix build. In other news: I hate autocrap diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am index fa0f3078b..a2dd76bbc 100644 --- a/lib/libvarnishapi/Makefile.am +++ b/lib/libvarnishapi/Makefile.am @@ -98,16 +98,20 @@ vxp_test_CFLAGS = \ vxp_test_LDADD = @PCRE_LIBS@ \ ${RT_LIBS} ${LIBM} ${PTHREAD_LIBS} -TESTS = vsl_glob_test vsl_glob_test_coverage +TESTS = vsl_glob_test -noinst_PROGRAMS += ${TESTS} +noinst_PROGRAMS += vsl_glob_test vsl_glob_test_SOURCES = vsl_glob_test.c vsl_glob_test_CFLAGS = @SAN_CFLAGS@ vsl_glob_test_LDADD = libvarnishapi.la @SAN_LDFLAGS@ +TESTS += vsl_glob_test_coverage + vsl_glob_test_coverage: echo './vsl_glob_test 1 2 3 2> /dev/null || true' > ${builddir}/_ echo './vsl_glob_test "Req*" > /dev/null' >> ${builddir}/_ mv ${builddir}/_ ${builddir}/vsl_glob_test_coverage chmod +x ${builddir}/vsl_glob_test_coverage + +CLEANFILES += ${builddir}/vsl_glob_test_coverage From phk at FreeBSD.org Mon Jan 28 16:47:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 28 Jan 2019 16:47:09 +0000 (UTC) Subject: [master] 399619e86 uid=varnish, not vcache Message-ID: <20190128164709.37821A8B1D@lists.varnish-cache.org> commit 399619e865872d6ef0feb76e0801e4b519ccbf22 Author: Poul-Henning Kamp Date: Mon Jan 28 16:46:21 2019 +0000 uid=varnish, not vcache diff --git a/bin/varnishtest/tests/c00086.vtc b/bin/varnishtest/tests/c00086.vtc index 62b6ba72d..0e6979ef3 100644 --- a/bin/varnishtest/tests/c00086.vtc +++ b/bin/varnishtest/tests/c00086.vtc @@ -59,7 +59,7 @@ server s2 -listen ${tmpdir}/dir/s1.sock {} -start shell { chmod go-rx ${tmpdir}/dir } varnish v8 \ - -jail "-junix,user=vcache,ccgroup=varnish,workuser=vcache" \ + -jail "-junix,user=varnish,ccgroup=varnish,workuser=vcache" \ -vcl { backend b {.host="${bad_ip}";} } From phk at FreeBSD.org Tue Jan 29 09:43:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Jan 2019 09:43:08 +0000 (UTC) Subject: [master] 2d39b0790 Collect VCL_DURATION type conversion in a single function and restrict acceptable whitespace to SP and TAB. Message-ID: <20190129094308.1430065C3C@lists.varnish-cache.org> commit 2d39b0790f523ef7e894bea38e26f0406b269a1b Author: Poul-Henning Kamp Date: Tue Jan 29 08:56:34 2019 +0000 Collect VCL_DURATION type conversion in a single function and restrict acceptable whitespace to SP and TAB. Part of fix for #2882 diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc index 1d1637f8c..a53bcc579 100644 --- a/bin/varnishtest/tests/v00016.vtc +++ b/bin/varnishtest/tests/v00016.vtc @@ -30,7 +30,7 @@ varnish v1 -errvcl {include not followed by string constant.} { { } ( ) * + - / % > < = ; ! & . | ~ , } -varnish v1 -errvcl {Unknown time unit 'k'. Legal are 'ms', 's', 'm', 'h', 'd', 'w' and 'y'} { +varnish v1 -errvcl {Unknown duration unit 'k'} { backend b { .host = "127.0.0.1"; } sub vcl_backend_response { set beresp.ttl = 1. k; } } diff --git a/include/vnum.h b/include/vnum.h index 266f156f5..ae8a8e1f2 100644 --- a/include/vnum.h +++ b/include/vnum.h @@ -31,5 +31,9 @@ /* from libvarnish/vnum.c */ double VNUM(const char *p); double VNUMpfx(const char *p, const char **e); +vtim_dur VNUM_duration_unit(vtim_dur r, const char *b, const char *e); vtim_dur VNUM_duration(const char *p); const char *VNUM_2bytes(const char *p, uintmax_t *r, uintmax_t rel); + +#define VNUM_LEGAL_DURATION \ + "Legal duration units are 'ms', 's', 'm', 'h', 'd', 'w' and 'y'" diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am index 271b13c43..026955a5e 100644 --- a/lib/libvarnish/Makefile.am +++ b/lib/libvarnish/Makefile.am @@ -51,7 +51,7 @@ binheap_CFLAGS = -DTEST_DRIVER vnum_c_test_SOURCES = vnum.c vas.c vnum_c_test_CFLAGS = -DNUM_C_TEST -include config.h -vnum_c_test_LDADD = ${LIBM} +vnum_c_test_LDADD = ${LIBM} libvarnish.a vjsn_test_SOURCES = vjsn.c vjsn_test_CFLAGS = -DVJSN_TEST @SAN_CFLAGS@ diff --git a/lib/libvarnish/vnum.c b/lib/libvarnish/vnum.c index 59e804ec8..99a81a8f5 100644 --- a/lib/libvarnish/vnum.c +++ b/lib/libvarnish/vnum.c @@ -30,14 +30,15 @@ #include "config.h" -#include #include #include #include #include +#include #include "vdef.h" +#include "vct.h" #include "vnum.h" #include "vas.h" @@ -61,14 +62,14 @@ VNUMpfx(const char *p, const char **t) AN(p); AN(t); *t = NULL; - while (isspace(*p)) + while (vct_issp(*p)) p++; if (*p == '-' || *p == '+') ms = (*p++ == '-' ? -1.0 : 1.0); for (; *p != '\0'; p++) { - if (isdigit(*p)) { + if (vct_isdigit(*p)) { m = m * 10. + *p - '0'; e = ne; if (e) @@ -84,12 +85,12 @@ VNUMpfx(const char *p, const char **t) p++; if (*p == '-' || *p == '+') es = (*p++ == '-' ? -1.0 : 1.0); - if (!isdigit(*p)) + if (!vct_isdigit(*p)) return (nan("")); - for (; isdigit(*p); p++) + for (; vct_isdigit(*p); p++) ee = ee * 10. + *p - '0'; } - while (isspace(*p)) + while (vct_issp(*p)) p++; if (*p != '\0') *t = p; @@ -111,31 +112,25 @@ VNUM(const char *p) /**********************************************************************/ vtim_dur -VNUM_duration(const char *p) +VNUM_duration_unit(vtim_dur r, const char *b, const char *e) { - const char *t; - vtim_dur r; double sc = 1.0; - if (p == NULL) - return (nan("")); - - r = VNUMpfx(p, &t); + if (e == NULL) + e = strchr(b, '\0'); - if (isnan(r) || t == NULL) + while (b < e && vct_issp(*b)) + b++; + if (b == e) return (nan("")); - while (isspace(*t)) - t++; - - // keep in sync with vcc_expr.c vcc_TimeUnit() - switch (*t++) { + switch (*b++) { case 's': break; case 'm': - if (*t == 's') { + if (b < e && *b == 's') { sc = 1e-3; - t++; + b++; } else sc = 60.0; break; @@ -155,15 +150,32 @@ VNUM_duration(const char *p) return (nan("")); } - while (isspace(*t)) - t++; + while (b < e && vct_issp(*b)) + b++; - if (*t != '\0') + if (b < e) return (nan("")); return (r * sc); } +vtim_dur +VNUM_duration(const char *p) +{ + const char *t; + vtim_dur r; + + if (p == NULL) + return (nan("")); + + r = VNUMpfx(p, &t); + + if (isnan(r) || t == NULL) + return (nan("")); + + return (VNUM_duration_unit(r, t, NULL)); +} + /**********************************************************************/ const char * diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am index a2dd76bbc..a94efee04 100644 --- a/lib/libvarnishapi/Makefile.am +++ b/lib/libvarnishapi/Makefile.am @@ -20,6 +20,7 @@ libvarnishapi_la_SOURCES = \ ../../include/vcs_version.h \ ../libvarnish/version.c \ ../libvarnish/vcli_proto.c \ + ../libvarnish/vct.c \ ../libvarnish/vfil.c \ ../libvarnish/vfl.c \ ../libvarnish/vin.c \ diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 3a98c76e0..4e7e94c05 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -334,7 +334,7 @@ void Resolve_Sockaddr(struct vcc *tl, const char *host, const char *defport, const struct token *t_err, const char *errid); void Emit_UDS_Path(struct vcc *tl, const struct token *t_path, const char *errid); -double vcc_TimeUnit(struct vcc *); +double vcc_DurationUnit(struct vcc *); void vcc_ByteVal(struct vcc *, double *); void vcc_Duration(struct vcc *tl, double *); unsigned vcc_UintVal(struct vcc *tl); diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 00b6ca700..1c1f659ad 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -838,7 +838,7 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt) vcc_NextToken(tl); if (tl->t->tok == ID) { e1 = vcc_mk_expr(DURATION, "(%s%.*s) * %g", - sign, PF(t), vcc_TimeUnit(tl)); + sign, PF(t), vcc_DurationUnit(tl)); ERRCHK(tl); } else if (fmt == REAL || t->tok == FNUM) { e1 = vcc_mk_expr(REAL, "%s%.*s", sign, PF(t)); diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c index f03762b34..abe4f69b5 100644 --- a/lib/libvcc/vcc_utils.c +++ b/lib/libvcc/vcc_utils.c @@ -40,6 +40,7 @@ #include "vcc_compile.h" #include "vre.h" +#include "vnum.h" #include "vsa.h" #include "vss.h" #include "vtcp.h" @@ -296,39 +297,25 @@ Emit_UDS_Path(struct vcc *tl, const struct token *t_path, const char *errid) } /*-------------------------------------------------------------------- - * Recognize and convert units of time, return seconds. + * Recognize and convert units of duration, return seconds. */ double -vcc_TimeUnit(struct vcc *tl) +vcc_DurationUnit(struct vcc *tl) { - double sc = 1.0; + double sc; assert(tl->t->tok == ID); - if (vcc_IdIs(tl->t, "ms")) - sc = 1e-3; - else if (vcc_IdIs(tl->t, "s")) - sc = 1.0; - else if (vcc_IdIs(tl->t, "m")) - sc = 60.0; - else if (vcc_IdIs(tl->t, "h")) - sc = 60.0 * 60.0; - else if (vcc_IdIs(tl->t, "d")) - sc = 60.0 * 60.0 * 24.0; - else if (vcc_IdIs(tl->t, "w")) - sc = 60.0 * 60.0 * 24.0 * 7.0; - else if (vcc_IdIs(tl->t, "y")) - sc = 60.0 * 60.0 * 24.0 * 365.0; - else { - VSB_printf(tl->sb, "Unknown time unit "); - vcc_ErrToken(tl, tl->t); - VSB_printf(tl->sb, - ". Legal are 'ms', 's', 'm', 'h', 'd', 'w' and 'y'\n"); - vcc_ErrWhere(tl, tl->t); - return (1.0); + sc = VNUM_duration_unit(1.0, tl->t->b, tl->t->e); + if (!isnan(sc)) { + vcc_NextToken(tl); + return (sc); } - vcc_NextToken(tl); - return (sc); + VSB_printf(tl->sb, "Unknown duration unit "); + vcc_ErrToken(tl, tl->t); + VSB_printf(tl->sb, "\n%s\n", VNUM_LEGAL_DURATION); + vcc_ErrWhere(tl, tl->t); + return (1.0); } /*-------------------------------------------------------------------- @@ -374,7 +361,7 @@ vcc_Duration(struct vcc *tl, double *d) v = vcc_DoubleVal(tl); ERRCHK(tl); ExpectErr(tl, ID); - sc = vcc_TimeUnit(tl); + sc = vcc_DurationUnit(tl); *d = v * sc; } diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index ba10fa513..05c17495b 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -47,10 +47,10 @@ VCL_DURATION v_matchproto_(td_std_duration) vmod_duration(VRT_CTX, VCL_STRING p, VCL_DURATION d) { - double r = VNUM_duration(p); - - (void) ctx; + double r; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + r = VNUM_duration(p); return (isnan(r) ? d : r); } From phk at FreeBSD.org Tue Jan 29 09:43:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Jan 2019 09:43:08 +0000 (UTC) Subject: [master] f543c06b5 Overhaul and unify handling of BYTES suffixes. Message-ID: <20190129094308.2FFD065C3F@lists.varnish-cache.org> commit f543c06b5e65b24e9d701adb0107877c92fb8728 Author: Poul-Henning Kamp Date: Tue Jan 29 09:38:35 2019 +0000 Overhaul and unify handling of BYTES suffixes. Closes #2882 diff --git a/bin/varnishtest/tests/v00033.vtc b/bin/varnishtest/tests/v00033.vtc index bbe21d1ee..fcd7e422b 100644 --- a/bin/varnishtest/tests/v00033.vtc +++ b/bin/varnishtest/tests/v00033.vtc @@ -43,7 +43,7 @@ varnish v1 -syntax 4.0 -errvcl {Expected BYTES unit (B, KB, MB...) got '"X"'} { } } } -varnish v1 -syntax 4.0 -errvcl {Unknown BYTES unit 'X'. Legal are 'B', 'KB', 'MB', 'GB' and 'TB'} { +varnish v1 -syntax 4.0 -errvcl {Unknown BYTES unit 'X'} { sub vcl_recv { if (storage.Transient.free_space > 4 X) { } diff --git a/include/vnum.h b/include/vnum.h index ae8a8e1f2..53da2a8e3 100644 --- a/include/vnum.h +++ b/include/vnum.h @@ -33,7 +33,11 @@ double VNUM(const char *p); double VNUMpfx(const char *p, const char **e); vtim_dur VNUM_duration_unit(vtim_dur r, const char *b, const char *e); vtim_dur VNUM_duration(const char *p); +double VNUM_bytes_unit(double r, const char *b, const char *e, uintmax_t rel); const char *VNUM_2bytes(const char *p, uintmax_t *r, uintmax_t rel); #define VNUM_LEGAL_DURATION \ "Legal duration units are 'ms', 's', 'm', 'h', 'd', 'w' and 'y'" + +#define VNUM_LEGAL_BYTES \ + "Legal byte units are 'B', 'KB', 'MB', 'GB', 'TB' and 'PB'" diff --git a/lib/libvarnish/vnum.c b/lib/libvarnish/vnum.c index 99a81a8f5..43baa7da5 100644 --- a/lib/libvarnish/vnum.c +++ b/lib/libvarnish/vnum.c @@ -44,7 +44,6 @@ static const char err_miss_num[] = "Missing number"; static const char err_invalid_num[] = "Invalid number"; -static const char err_abs_req[] = "Absolute number required"; static const char err_invalid_suff[] = "Invalid suffix"; /********************************************************************** @@ -114,7 +113,7 @@ VNUM(const char *p) vtim_dur VNUM_duration_unit(vtim_dur r, const char *b, const char *e) { - double sc = 1.0; + double sc; if (e == NULL) e = strchr(b, '\0'); @@ -126,6 +125,7 @@ VNUM_duration_unit(vtim_dur r, const char *b, const char *e) switch (*b++) { case 's': + sc = 1.0; break; case 'm': if (b < e && *b == 's') { @@ -178,6 +178,44 @@ VNUM_duration(const char *p) /**********************************************************************/ +double +VNUM_bytes_unit(double r, const char *b, const char *e, uintmax_t rel) +{ + double sc = 1.0; + + if (e == NULL) + e = strchr(b, '\0'); + + while (b < e && vct_issp(*b)) + b++; + if (b == e) + return (nan("")); + + if (rel != 0 && *b == '%') { + r *= rel * 0.01; + b++; + } else { + switch (*b) { + case 'k': case 'K': sc = exp2(10); b++; break; + case 'm': case 'M': sc = exp2(20); b++; break; + case 'g': case 'G': sc = exp2(30); b++; break; + case 't': case 'T': sc = exp2(40); b++; break; + case 'p': case 'P': sc = exp2(50); b++; break; + case 'b': case 'B': + break; + default: + return (nan("")); + } + if (b < e && (*b == 'b' || *b == 'B')) + b++; + } + while (b < e && vct_issp(*b)) + b++; + if (b < e) + return (nan("")); + return (sc * r); +} + const char * VNUM_2bytes(const char *p, uintmax_t *r, uintmax_t rel) { @@ -196,48 +234,9 @@ VNUM_2bytes(const char *p, uintmax_t *r, uintmax_t rel) return (NULL); } - if (end[0] == '%' && end[1] == '\0') { - if (rel == 0) - return (err_abs_req); - fval *= rel / 100.0; - } else { - /* accept a space before the multiplier */ - if (end[0] == ' ' && end[1] != '\0') - ++end; - - switch (end[0]) { - case 'k': case 'K': - fval *= (uintmax_t)1 << 10; - ++end; - break; - case 'm': case 'M': - fval *= (uintmax_t)1 << 20; - ++end; - break; - case 'g': case 'G': - fval *= (uintmax_t)1 << 30; - ++end; - break; - case 't': case 'T': - fval *= (uintmax_t)1 << 40; - ++end; - break; - case 'p': case 'P': - fval *= (uintmax_t)1 << 50; - ++end; - break; - default: - break; - } - - /* [bB] is a generic suffix of no effect */ - if (end[0] == 'b' || end[0] == 'B') - end++; - - if (end[0] != '\0') - return (err_invalid_suff); - } - + fval = VNUM_bytes_unit(fval, end, NULL, rel); + if (isnan(fval)) + return (err_invalid_suff); *r = (uintmax_t)round(fval); return (NULL); } @@ -279,11 +278,11 @@ static struct test_case { { "1T", (uintmax_t)0, (uintmax_t)1<<40 }, { "1TB", (uintmax_t)0, (uintmax_t)1<<40 }, { "1.3TB", (uintmax_t)0, (uintmax_t)1429365116109ULL }, - { "1.7TB", (uintmax_t)0, (uintmax_t)1869169767219ULL }, + { "1.7\tTB", (uintmax_t)0, (uintmax_t)1869169767219ULL }, { "1125899906842624", (uintmax_t)0, (uintmax_t)1125899906842624ULL}, - { "1P", (uintmax_t)0, (uintmax_t)1125899906842624ULL}, - { "1PB", (uintmax_t)0, (uintmax_t)1125899906842624ULL}, + { "1P\t", (uintmax_t)0, (uintmax_t)1125899906842624ULL}, + { "1PB ", (uintmax_t)0, (uintmax_t)1125899906842624ULL}, { "1.3 PB", (uintmax_t)0, (uintmax_t)1463669878895411ULL}, { "1%", (uintmax_t)1024, (uintmax_t)10 }, @@ -293,7 +292,7 @@ static struct test_case { /* Check the error checks */ { "", 0, 0, err_miss_num }, { "m", 0, 0, err_invalid_num }, - { "4%", 0, 0, err_abs_req }, + { "4%", 0, 0, err_invalid_suff }, { "3*", 0, 0, err_invalid_suff }, /* TODO: add more */ diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c index abe4f69b5..832e5357f 100644 --- a/lib/libvcc/vcc_utils.c +++ b/lib/libvcc/vcc_utils.c @@ -381,25 +381,14 @@ vcc_ByteVal(struct vcc *tl, double *d) vcc_ErrWhere(tl, tl->t); return; } - if (vcc_IdIs(tl->t, "B")) - sc = 1.; - else if (vcc_IdIs(tl->t, "KB")) - sc = 1024.; - else if (vcc_IdIs(tl->t, "MB")) - sc = 1024. * 1024.; - else if (vcc_IdIs(tl->t, "GB")) - sc = 1024. * 1024. * 1024.; - else if (vcc_IdIs(tl->t, "TB")) - sc = 1024. * 1024. * 1024. * 1024.; - else { + sc = VNUM_bytes_unit(1.0, tl->t->b, tl->t->e, 0); + if (isnan(sc)) { VSB_printf(tl->sb, "Unknown BYTES unit "); vcc_ErrToken(tl, tl->t); - VSB_printf(tl->sb, - ". Legal are 'B', 'KB', 'MB', 'GB' and 'TB'\n"); + VSB_printf(tl->sb, "\n%s\n", VNUM_LEGAL_BYTES); vcc_ErrWhere(tl, tl->t); return; } vcc_NextToken(tl); *d = v * sc; } - From phk at FreeBSD.org Tue Jan 29 10:25:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Jan 2019 10:25:10 +0000 (UTC) Subject: [master] e8ffdc452 Add the missing std.bytes() conversion function Message-ID: <20190129102510.4FF606ED9E@lists.varnish-cache.org> commit e8ffdc452a80835acdae47bc7bdff65baf9c5a26 Author: Poul-Henning Kamp Date: Tue Jan 29 10:22:50 2019 +0000 Add the missing std.bytes() conversion function Part of #2863 diff --git a/bin/varnishtest/tests/m00005.vtc b/bin/varnishtest/tests/m00005.vtc index 13e8c2dc9..9ce01bf42 100644 --- a/bin/varnishtest/tests/m00005.vtc +++ b/bin/varnishtest/tests/m00005.vtc @@ -1,4 +1,4 @@ -varnishtest "Test std.duration()" +varnishtest "Test std.duration() and std.bytes()" server s1 { rxreq @@ -9,68 +9,77 @@ varnish v1 -vcl+backend { import std; sub vcl_deliver { - set resp.http.ttl = std.duration(req.http.ttl, 1s) + 1000000s; + set resp.http.duration = std.duration(req.http.duration, 1s) + 1000000s; + set resp.http.bytes = std.bytes(req.http.bytes, 10K) + 512B; } } -start client c1 { - txreq -hdr "ttl: 0.010s" + txreq -hdr "duration: 0.010s" rxresp - expect resp.http.ttl == 1000000.010 + expect resp.http.duration == 1000000.010 + expect resp.http.bytes == 10752.000 - txreq -hdr "ttl: 10ms" + txreq -hdr "duration: 10ms" -hdr "bytes: 2k" rxresp - expect resp.http.ttl == 1000000.010 + expect resp.http.duration == 1000000.010 + expect resp.http.bytes == 2560.000 - txreq -hdr "ttl: 10.1s" + txreq -hdr "duration: 10.1s" -hdr "bytes: 3 m" rxresp - expect resp.http.ttl == 1000010.100 + expect resp.http.duration == 1000010.100 + expect resp.http.bytes == 3146240.000 - txreq -hdr "ttl: 10m" + txreq -hdr "duration: 10m" -hdr "bytes:4.5 g" rxresp - expect resp.http.ttl == 1000600.000 + expect resp.http.duration == 1000600.000 + expect resp.http.bytes == 4831838720.000 - txreq -hdr "ttl: 10h" + txreq -hdr "duration: 10h" -hdr "bytes: 0.12 TB" rxresp - expect resp.http.ttl == 1036000.000 + expect resp.http.duration == 1036000.000 + expect resp.http.bytes == 131941395845.000 - txreq -hdr "ttl: 10d" + txreq -hdr "duration: 10d" -hdr "bytes: 0.25 PB" rxresp - expect resp.http.ttl == 1864000.000 + expect resp.http.duration == 1864000.000 + expect resp.http.bytes == 281474976711168.000 - txreq -hdr "ttl: 10w" + txreq -hdr "duration: 10w" -hdr "bytes: 34%" rxresp - expect resp.http.ttl == 7048000.000 + expect resp.http.duration == 7048000.000 + expect resp.http.bytes == 10752.000 - txreq -hdr "ttl: 1y" + txreq -hdr "duration: 1y" -hdr "bytes: 34x" rxresp - expect resp.http.ttl == 32536000.000 + expect resp.http.duration == 32536000.000 + expect resp.http.bytes == 10752.000 - txreq -hdr "ttl: -100s" + txreq -hdr "duration: -100s" rxresp - expect resp.http.ttl == 999900.000 + expect resp.http.duration == 999900.000 - txreq -hdr "ttl: s" + txreq -hdr "duration: s" rxresp - expect resp.http.ttl == 1000001.000 + expect resp.http.duration == 1000001.000 - txreq -hdr "ttl: 3wx" + txreq -hdr "duration: 3wx" rxresp - expect resp.http.ttl == 1000001.000 + expect resp.http.duration == 1000001.000 - txreq -hdr "ttl: -inf" + txreq -hdr "duration: -inf" rxresp - expect resp.http.ttl == 1000001.000 + expect resp.http.duration == 1000001.000 - txreq -hdr "ttl: 2x" + txreq -hdr "duration: 2x" rxresp - expect resp.http.ttl == 1000001.000 + expect resp.http.duration == 1000001.000 - txreq -hdr "ttl: 2h x" + txreq -hdr "duration: 2h x" rxresp - expect resp.http.ttl == 1000001.000 + expect resp.http.duration == 1000001.000 - txreq -hdr "ttl: 100" + txreq -hdr "duration: 100" rxresp - expect resp.http.ttl == 1000001.000 + expect resp.http.duration == 1000001.000 } -run diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index 0ae941c12..afab2bed3 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -154,6 +154,15 @@ Description Example set beresp.ttl = std.duration("1w", 3600s); +$Function BYTES bytes(STRING s, BYTES fallback) + +Description + Converts the string *s* to bytes. *s* can be quantified + with a multiplier (k, m, g, t, p). If conversion fails, + *fallback* will be returned. +Example + std.cache_req_body(std.bytes(something.somewhere, 10K)); + $Function INT integer(STRING s, INT fallback) Description diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index 05c17495b..659c148c3 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -54,6 +54,17 @@ vmod_duration(VRT_CTX, VCL_STRING p, VCL_DURATION d) return (isnan(r) ? d : r); } +VCL_BYTES v_matchproto_(td_std_bytes) +vmod_bytes(VRT_CTX, VCL_STRING p, VCL_BYTES d) +{ + uintmax_t r; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + if (VNUM_2bytes(p, &r, 0) != NULL) + return (d); + return (r); +} + VCL_INT v_matchproto_(td_std_integer) vmod_integer(VRT_CTX, VCL_STRING p, VCL_INT i) { From phk at FreeBSD.org Tue Jan 29 14:40:12 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Jan 2019 14:40:12 +0000 (UTC) Subject: [master] 75e4de5c1 Add coverage Message-ID: <20190129144012.6F6519625D@lists.varnish-cache.org> commit 75e4de5c1160c32bceb5166a042aa45051796b0e Author: Poul-Henning Kamp Date: Tue Jan 29 14:39:38 2019 +0000 Add coverage diff --git a/bin/varnishtest/tests/u00006.vtc b/bin/varnishtest/tests/u00006.vtc index 9ffb331e7..55b2fe909 100644 --- a/bin/varnishtest/tests/u00006.vtc +++ b/bin/varnishtest/tests/u00006.vtc @@ -60,6 +60,10 @@ shell -err -expect "Cannot open output file (No such file or directory)" \ shell -err -expect "Only one of -n and -r options may be used" \ {varnishlog -n ${v1_name} -r ${v1_name}/_.vsm} +shell -err -expect "Unterminated string" \ + {varnishlog -q '"b\\la'} +shell -err -expect "Syntax error" \ + {varnishlog -q ' / '} shell -err -expect "Expected integer got '}" \ "varnishlog -q {}" shell -err -expect "Expected positive integer" \ From phk at FreeBSD.org Tue Jan 29 19:30:18 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Jan 2019 19:30:18 +0000 (UTC) Subject: [master] 83ca874cf Test all the atypical GZIP headers Message-ID: <20190129193018.77D05A2316@lists.varnish-cache.org> commit 83ca874cf376da8ae39c7974b184280e50fc0264 Author: Poul-Henning Kamp Date: Tue Jan 29 19:28:38 2019 +0000 Test all the atypical GZIP headers diff --git a/bin/varnishtest/tests/g00008.vtc b/bin/varnishtest/tests/g00008.vtc new file mode 100644 index 000000000..3929b6999 --- /dev/null +++ b/bin/varnishtest/tests/g00008.vtc @@ -0,0 +1,30 @@ +varnishtest "Test uncommon GZIP header fields" + +# With a a handcrafted GZIP file with all optional header fields + +server s1 { + rxreq + txresp -hdr "content-encoding: gzip" -nolen + sendhex "1f 8b" + sendhex "08" + sendhex "1f" + sendhex "12 34 56 78" + sendhex "00" + sendhex "03" + sendhex "08 00 50 48 04 00 4b 61 6d 70" + sendhex "46 4e 41 4d 45 00" + sendhex "46 43 4f 4d 4d 45 4e 54 00" + sendhex "96 cc" + sendhex "f3 48 cd c9 c9 57 70 8f 52 08 cf 2f ca 49 e1 02 00" + sendhex "3a 0b 41 35" + sendhex "0f 00 00 00" +} -start + +varnish v1 -vcl+backend { } -start + +client c1 { + txreq + rxresp + expect resp.status == 200 + expect resp.body == "Hello GZ World\n" +} -run From fgsch at lodoss.net Wed Jan 30 11:21:07 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 30 Jan 2019 11:21:07 +0000 (UTC) Subject: [master] 763426043 Fix build with sanitizers enabled Message-ID: <20190130112107.CDDB96BD1@lists.varnish-cache.org> commit 7634260435bf4a4043347ab91f64dda49e992ba9 Author: Federico G. Schwindt Date: Wed Jan 30 11:18:33 2019 +0000 Fix build with sanitizers enabled diff --git a/lib/libvarnish/Makefile.am b/lib/libvarnish/Makefile.am index 026955a5e..a7a57dd49 100644 --- a/lib/libvarnish/Makefile.am +++ b/lib/libvarnish/Makefile.am @@ -50,8 +50,8 @@ binheap_SOURCES = binary_heap.c vas.c vrnd.c binheap_CFLAGS = -DTEST_DRIVER vnum_c_test_SOURCES = vnum.c vas.c -vnum_c_test_CFLAGS = -DNUM_C_TEST -include config.h -vnum_c_test_LDADD = ${LIBM} libvarnish.a +vnum_c_test_CFLAGS = -DNUM_C_TEST -include config.h @SAN_CFLAGS@ +vnum_c_test_LDADD = ${LIBM} libvarnish.a @SAN_LDFLAGS@ vjsn_test_SOURCES = vjsn.c vjsn_test_CFLAGS = -DVJSN_TEST @SAN_CFLAGS@ From fgsch at lodoss.net Wed Jan 30 11:21:07 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 30 Jan 2019 11:21:07 +0000 (UTC) Subject: [master] 7294ccedd Sync ignored files Message-ID: <20190130112108.00DF16BD4@lists.varnish-cache.org> commit 7294ccedde39c4a3224eeb5aeb2e5feb79089da8 Author: Federico G. Schwindt Date: Wed Jan 30 11:19:44 2019 +0000 Sync ignored files diff --git a/.gitignore b/.gitignore index f6870aa1b..37b62a9fa 100644 --- a/.gitignore +++ b/.gitignore @@ -124,6 +124,7 @@ cscope.*out /lib/libvarnish/vnum_c_test /lib/libvarnishapi/*.log /lib/libvarnishapi/vsl_glob_test +/lib/libvarnishapi/vsl_glob_test_coverage # vtest.sh droppings /tools/tmp/ From phk at FreeBSD.org Thu Jan 31 08:51:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 31 Jan 2019 08:51:08 +0000 (UTC) Subject: [master] c09032032 Make the rst anchors we generated for vmods be the vcl-syntax name: Ie: Message-ID: <20190131085108.7389CA5A7C@lists.varnish-cache.org> commit c09032032700f91fcf06e3018d206cbb730d1772 Author: Poul-Henning Kamp Date: Thu Jan 31 08:49:07 2019 +0000 Make the rst anchors we generated for vmods be the vcl-syntax name: Ie: vmod.cache_req_body instead of func_cache_req_body diff --git a/doc/sphinx/reference/directors.rst b/doc/sphinx/reference/directors.rst index b359133ae..8758b4535 100644 --- a/doc/sphinx/reference/directors.rst +++ b/doc/sphinx/reference/directors.rst @@ -155,7 +155,7 @@ Health Probes ============= It is possible in a VCL program to query the health of a director (see -:ref:`func_healthy`). A director can report its health if it implements the +:ref:`vmod_std.healthy`). A director can report its health if it implements the ``healthy`` function, it is otherwise always considered healthy. Unless you are making a dynamic backend, you need to take care of the diff --git a/doc/sphinx/whats-new/upgrading-5.1.rst b/doc/sphinx/whats-new/upgrading-5.1.rst index f350c58ff..62a1726c4 100644 --- a/doc/sphinx/whats-new/upgrading-5.1.rst +++ b/doc/sphinx/whats-new/upgrading-5.1.rst @@ -197,7 +197,7 @@ vcl_recv * Added ``req.storage``, which tells Varnish which storage backend to use if you choose to save the request body (see - :ref:`func_cache_req_body`). + :ref:`vmod_std.cache_req_body`). * ``return(vcl(LABEL))`` may not be called after a restart. It can only be called from the active VCL instance. @@ -232,9 +232,9 @@ nuke limit is used in all cases. vmod_std ~~~~~~~~ -* Added ``std.getenv()``, see :ref:`func_getenv`. +* Added ``std.getenv()``, see :ref:`vmod_std.getenv`. -* Added ``std.late_100_continue()``, see :ref:`func_late_100_continue`. +* Added ``std.late_100_continue()``, see :ref:`vmod_std.late_100_continue`. Other changes ============= diff --git a/doc/sphinx/whats-new/upgrading-5.2.rst b/doc/sphinx/whats-new/upgrading-5.2.rst index fa137c35f..8edd71b6b 100644 --- a/doc/sphinx/whats-new/upgrading-5.2.rst +++ b/doc/sphinx/whats-new/upgrading-5.2.rst @@ -121,7 +121,7 @@ situation. vmod_std ~~~~~~~~ -Added :ref:`func_file_exists`. +Added :ref:`vmod_std.file_exists`. New VMODs in the standard distribution ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index 510993143..7a0227995 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -461,9 +461,9 @@ backend, or set a value for the Host header in VCL. VMOD std -------- -:ref:`std.port(IP) ` always returns 0 when applied to a +:ref:`std.port(IP) ` always returns 0 when applied to a ``*.ip`` variable whose value is set to ``0.0.0.0`` because the -listener is UDS. :ref:`std.set_ip_tos(INT) ` is +listener is UDS. :ref:`std.set_ip_tos(INT) ` is silently ignored when the listener is UDS. The ``shard`` director @@ -519,7 +519,7 @@ except for ``req.restarts`` and ``req.xid``, which change by design. If you need to reset the client request headers to their original state (before changes in VCL), call -:ref:`std.rollback(req) `. +:ref:`std.rollback(req) `. ``return(restart)`` can now be called from ``vcl_recv{}``. diff --git a/doc/sphinx/whats-new/upgrading-6.1.rst b/doc/sphinx/whats-new/upgrading-6.1.rst index a082b3cc5..7a5499af4 100644 --- a/doc/sphinx/whats-new/upgrading-6.1.rst +++ b/doc/sphinx/whats-new/upgrading-6.1.rst @@ -138,7 +138,7 @@ Other changes to VCL VMODs ===== -Added the :ref:`func_fnmatch` function to :ref:`vmod_std(3)`, which +Added the :ref:`vmod_std.fnmatch` function to :ref:`vmod_std(3)`, which you can use for shell-style wildcard matching. Wildcard patterns may be a good fit for matching URLs, to match against a pattern like ``/foo/*/bar/*``. The patterns can be built at runtime, if you need to diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index b1e4cd590..6ad42f421 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -398,13 +398,13 @@ class ProtoType(object): s += t.replace("@", " ") + ")" return s - def rsthead(self, fo): + def rst_proto(self, fo, sep='-'): s = self.vcl_proto(False) if len(s) < 60: - write_rst_hdr(fo, s, '-') + write_rst_hdr(fo, s, sep) else: s = self.vcl_proto(True) - write_rst_hdr(fo, s, '-') + write_rst_hdr(fo, s, sep) fo.write("\n::\n\n" + self.vcl_proto(False, pfx=" ") + "\n") def cname(self, pfx=False): @@ -508,15 +508,15 @@ class Stanza(object): warn=False) def rstfile(self, fo, man): - if self.rstlbl: - fo.write("\n.. _" + self.rstlbl + ":\n") self.rsthead(fo, man) self.rstdoc(fo, man) def rsthead(self, fo, unused_man): ''' Emit the systematic part of the documentation ''' + if self.rstlbl: + fo.write('\n.. _' + self.rstlbl + ':\n') if self.proto: - self.proto.rsthead(fo) + self.proto.rst_proto(fo) fo.write("\n") def rstdoc(self, fo, unused_man): @@ -527,7 +527,7 @@ class Stanza(object): if man and self.proto: fo.write(self.proto.vcl_proto(True, pfx=" ") + '\n \n') elif self.proto and self.rstlbl: - fo.write(' :ref:`%s`\n \n' % self.rstlbl) + fo.write(' :ref:`%s`\n \n' % self.rstlbl) def cstuff(self, unused_fo, unused_where): return @@ -556,10 +556,7 @@ class ModuleStanza(Stanza): else: print("\nNOTICE: Please put $Module description in quotes.\n") self.vcc.moddesc = " ".join(self.toks[3:]) - self.rstlbl = "vmod_%s(%s)" % ( - self.vcc.modname, - self.vcc.mansection - ) + self.rstlbl = "vmod_%s(%d)" % (self.vcc.modname, 3) self.vcc.contents.append(self) def rsthead(self, fo, man): @@ -570,6 +567,8 @@ class ModuleStanza(Stanza): fo.write("\n") fo.write(":Manual section: " + self.vcc.mansection + "\n") else: + if self.rstlbl: + fo.write('\n.. _' + self.rstlbl + ':\n') write_rst_hdr(fo, self.vcc.sympfx + self.vcc.modname + ' - ' + self.vcc.moddesc, @@ -669,7 +668,7 @@ class FunctionStanza(Stanza): def parse(self): self.proto = ProtoType(self) - self.rstlbl = "func_" + self.proto.name + self.rstlbl = 'vmod_%s.%s' % (self.vcc.modname, self.proto.name) self.vcc.contents.append(self) def cstuff(self, fo, where): @@ -702,12 +701,14 @@ class ObjectStanza(Stanza): self.fini.argstruct = False self.fini.args = [] - self.rstlbl = "obj_" + self.proto.name + self.rstlbl = 'vmod_%s.%s' % (self.vcc.modname, self.proto.name) self.vcc.contents.append(self) self.methods = [] def rsthead(self, fo, man): - self.proto.rsthead(fo) + if self.rstlbl: + fo.write('\n.. _' + self.rstlbl + ':\n') + self.proto.rst_proto(fo) fo.write("\n" + "\n".join(self.doc) + "\n") for i in self.methods: i.rstfile(fo, man) @@ -725,7 +726,7 @@ class ObjectStanza(Stanza): fo.write(' :ref:`%s`\n \n' % self.rstlbl) for i in self.methods: if i.proto and i.rstlbl: - fo.write(' :ref:`%s`\n \n' % i.rstlbl) + fo.write(' :ref:`%s`\n \n' % i.rstlbl) def cstuff(self, fo, w): sn = self.vcc.sympfx + self.vcc.modname + "_" + self.proto.name @@ -787,7 +788,7 @@ class MethodStanza(Stanza): err("$Method %s: Method names need to start with . (dot)" % self.proto.bname, warn=False) self.proto.obj = "x" + self.pfx - self.rstlbl = "func_" + self.proto.name + self.rstlbl = 'vmod_%s.%s' % ( self.vcc.modname, self.proto.name) p.methods.append(self) def cstruct(self, fo, define): diff --git a/lib/libvmod_blob/vmod.vcc b/lib/libvmod_blob/vmod.vcc index ab545480e..302293f76 100644 --- a/lib/libvmod_blob/vmod.vcc +++ b/lib/libvmod_blob/vmod.vcc @@ -55,7 +55,7 @@ Examples:: } ENCODING SCHEMES -================ +---------------- Binary-to-text encoding schemes are specified by ENUMs in the VMOD's constructor, methods and functions. Decodings convert a (possibly @@ -91,7 +91,7 @@ case of a string, use the ``toupper`` or ``tolower`` functions from :ref:`vmod_std(3)`. IDENTITY --------- +~~~~~~~~ The simplest encoding converts between the BLOB and STRING data types, leaving the contents byte-identical. @@ -120,7 +120,7 @@ be written as:: The ``case`` ENUM MUST be set to ``DEFAULT`` for ``IDENTITY`` encodings. BASE64* -------- +~~~~~~~ The base64 encoding schemes use 4 characters to encode 3 bytes. There are no newlines or maximal line lengths -- whitespace is not @@ -143,7 +143,7 @@ The ``case`` ENUM MUST be set to ``DEFAULT`` for for all of the ``BASE64*`` encodings. HEX ---- +~~~ The ``HEX`` encoding scheme converts hex strings into blobs and vice versa. For encodings, you may use the ``case`` ENUM to specify upper- @@ -164,7 +164,7 @@ byte. For example:: encoded=resp.http.First + resp.http.Second)); URL ---- +~~~ The ``URL`` decoding replaces any ``%<2-hex-digits>`` substrings with the binary value of the hexadecimal number after the % sign. @@ -197,6 +197,7 @@ Example:: # convert string to blob blob.decode(encoded="foo"); + $Function STRING encode(ENUM {IDENTITY, BASE64, BASE64URL, BASE64URLNOPAD, HEX, URL} encoding="IDENTITY", ENUM {LOWER, UPPER, DEFAULT} case="DEFAULT", BLOB blob) diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc index a98352983..71ec5be1b 100644 --- a/lib/libvmod_directors/vmod.vcc +++ b/lib/libvmod_directors/vmod.vcc @@ -349,13 +349,13 @@ Set the default rampup duration. See `rampup` parameter of $Method VOID .associate(BLOB param=0) -Associate a default `obj_shard_param`_ object or clear an association. +Associate a default `vmod_directors.shard_param`_ object or clear an association. The value of the `param` argument must be a call to the -`func_shard_param.use`_ method. No argument clears the association. +`vmod_directors.shard_param.use`_ method. No argument clears the association. The association can be changed per backend request using the `param` -argument of `func_shard.backend`_. +argument of `vmod_directors.shard.backend`_. $Method BOOL .add_backend(PRIV_TASK, BACKEND backend, [STRING ident], [DURATION rampup]) @@ -371,7 +371,7 @@ backend name. `rampup`: Optionally specify a specific rampup time for this backend. Otherwise, the per-director rampup time is used (see -:ref:`func_shard.set_rampup`). +:ref:`vmod_directors.shard.set_rampup`). NOTE: Backend changes need to be finalized with `shard.reconfigure()` and are only supported on one shard director at a time. @@ -536,14 +536,14 @@ is _not_ the order given when backends are added. * `param` Use or associate a parameter set. The value of the `param` argument - must be a call to the `func_shard_param.use`_ method. + must be a call to the `vmod_directors.shard_param.use`_ method. - default: as set by `func_shard.associate`_ or unset. + default: as set by `vmod_directors.shard.associate`_ or unset. * for ``resolve=NOW`` take parameter defaults from the - `obj_shard_param`_ parameter set + `vmod_directors.shard_param`_ parameter set - * for ``resolve=LAZY`` associate the `obj_shard_param`_ parameter + * for ``resolve=LAZY`` associate the `vmod_directors.shard_param`_ parameter set for this backend request Implementation notes for use of parameter sets with @@ -557,7 +557,7 @@ is _not_ the order given when backends are added. and are kept even if the parameter set given by the `param` argument is subsequently changed within the same backend request. - * Each call to `func_shard.backend`_ overrides any previous call. + * Each call to `vmod_directors.shard.backend`_ overrides any previous call. $Method VOID .debug(INT) @@ -567,7 +567,7 @@ $Object shard_param() Create a shard parameter set. -A parameter set allows for re-use of `func_shard.backend`_ arguments +A parameter set allows for re-use of `vmod_directors.shard.backend`_ arguments across many shard director instances and simplifies advanced use cases (e.g. shard director with custom parameters layered below other directors). @@ -586,7 +586,7 @@ Parameter sets can not be used in client context. $Method VOID .clear() Reset the parameter set to default values as documented for -`func_shard.backend`_. +`vmod_directors.shard.backend`_. * in ``vcl_init{}``, resets the parameter set default for this VCL * in backend context, resets the parameter set for this backend @@ -604,7 +604,7 @@ $Method VOID .set( [ ENUM {CHOSEN, IGNORE, ALL} healthy ]) Change the given parameters of a parameter set as documented for -`func_shard.backend`_. +`vmod_directors.shard.backend`_. * in ``vcl_init{}``, changes the parameter set default for this VCL @@ -618,39 +618,39 @@ $Method STRING .get_by() Get a string representation of the `by` enum argument which denotes how a shard director using this parameter object would derive the -shard key. See `func_shard.backend`_. +shard key. See `vmod_directors.shard.backend`_. $Method INT .get_key() Get the key which a shard director using this parameter object would -use. See `func_shard.backend`_. +use. See `vmod_directors.shard.backend`_. $Method INT .get_alt() Get the `alt` parameter which a shard director using this parameter -object would use. See `func_shard.backend`_. +object would use. See `vmod_directors.shard.backend`_. $Method REAL .get_warmup() Get the `warmup` parameter which a shard director using this parameter -object would use. See `func_shard.backend`_. +object would use. See `vmod_directors.shard.backend`_. $Method BOOL .get_rampup() Get the `rampup` parameter which a shard director using this parameter -object would use. See `func_shard.backend`_. +object would use. See `vmod_directors.shard.backend`_. $Method STRING .get_healthy() Get a string representation of the `healthy` enum argument which a shard director using this parameter object would use. See -`func_shard.backend`_. +`vmod_directors.shard.backend`_. $Method BLOB .use() This method may only be used in backend context. -For use with the `param` argument of `func_shard.backend`_ to associate +For use with the `param` argument of `vmod_directors.shard.backend`_ to associate this shard parameter set with a shard director. ACKNOWLEDGEMENTS diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index afab2bed3..cf919d250 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -214,7 +214,7 @@ $Function TIME real2time(REAL r, TIME fallback) Description Rounds the real *r* to the nearest integer (see - `func_real2integer`_) and returns the corresponding time when + `vmod_std.real2integer`_) and returns the corresponding time when interpreted as a unix epoch. If conversion fails, *fallback* will be returned. Example From phk at FreeBSD.org Thu Jan 31 10:27:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 31 Jan 2019 10:27:09 +0000 (UTC) Subject: [master] a70d412cf Use unique names for the function-container in vmods. Message-ID: <20190131102709.B3319A79E0@lists.varnish-cache.org> commit a70d412cf452e84f79818a11ed7bbcf01f1ba855 Author: Poul-Henning Kamp Date: Thu Jan 31 10:25:27 2019 +0000 Use unique names for the function-container in vmods. diff --git a/bin/varnishd/vclflint.sh b/bin/varnishd/vclflint.sh index 24d47bf0a..c390719d9 100755 --- a/bin/varnishd/vclflint.sh +++ b/bin/varnishd/vclflint.sh @@ -1,11 +1,12 @@ #!/bin/sh # # Run flexelint on the VCL output +LIBS="-p vmod_path=/home/phk/Varnish/trunk/varnish-cache/lib/libvmod_std/.libs:/home/phk/Varnish/trunk/varnish-cache/lib/libvmod_debug/.libs:/home/phk/Varnish/trunk/varnish-cache/lib/libvmod_directors/.libs:/home/phk/Varnish/trunk/varnish-cache/lib/libvmod_purge/.libs:/home/phk/Varnish/trunk/varnish-cache/lib/libvmod_vtc/.libs:/home/phk/Varnish/trunk/varnish-cache/lib/libvmod_blob/.libs:/home/phk/Varnish/trunk/varnish-cache/lib/libvmod_unix/.libs:/home/phk/Varnish/trunk/varnish-cache/lib/libvmod_proxy/.libs" if [ "x$1" = "x" ] ; then - ./varnishd -C -b localhost > /tmp/_.c + ./varnishd $LIBS -C -b localhost > /tmp/_.c elif [ -f $1 ] ; then - ./varnishd -C -f $1 > /tmp/_.c + ./varnishd $LIBS -C -f $1 > /tmp/_.c else echo "usage!" 1>&2 fi diff --git a/include/vrt.h b/include/vrt.h index b2fa4ec4f..9cbf6a6d7 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -270,6 +270,7 @@ struct vmod_data { const char *file_id; const char *name; + const char *func_name; const void *func; int func_len; const char *proto; diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index 00f0cb0d0..678821429 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -307,8 +307,8 @@ vcc_ParseImport(struct vcc *tl) VSB_printf(ifp->ini, "\tif (VRT_Vmod_Init(ctx,\n"); VSB_printf(ifp->ini, "\t &VGC_vmod_%.*s,\n", PF(mod)); VSB_printf(ifp->ini, "\t %u,\n", tl->vmod_count++); - VSB_printf(ifp->ini, "\t &Vmod_%.*s_Func,\n", PF(mod)); - VSB_printf(ifp->ini, "\t sizeof(Vmod_%.*s_Func),\n", PF(mod)); + VSB_printf(ifp->ini, "\t &%s,\n", vmd->func_name); + VSB_printf(ifp->ini, "\t sizeof(%s),\n", vmd->func_name); VSB_printf(ifp->ini, "\t \"%.*s\",\n", PF(mod)); VSB_printf(ifp->ini, "\t "); VSB_quote(ifp->ini, fnp, -1, VSB_QUOTE_CSTR); diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 6ad42f421..37338888b 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -468,7 +468,7 @@ class ProtoType(object): ''' Produce VCL prototype as JSON ''' ll = [] self.retval.jsonproto(ll) - ll.append('Vmod_%s_Func.%s' % (self.st.vcc.modname, cfunc)) + ll.append('%s.%s' % (self.st.vcc.csn, cfunc)) if self.argstruct: ll.append(self.argstructname()) else: @@ -656,10 +656,7 @@ class EventStanza(Stanza): fo.write("\t%s,\n" % self.event_func) def json(self, jl): - jl.append([ - "$EVENT", - "Vmod_%s_Func._event" % self.vcc.modname - ]) + jl.append(["$EVENT", "%s._event" % self.vcc.csn]) class FunctionStanza(Stanza): @@ -832,6 +829,7 @@ class vcc(object): self.strict_abi = True self.auto_synopsis = True self.modname = None + self.csn = None def openfile(self, fn): self.commit_files.append(fn) @@ -858,6 +856,7 @@ class vcc(object): err("Unknown stanza $%s" % toks[0], warn=False) stanzaclass(self, toks, docstr) inputline = None + self.csn = "Vmod_%s%s_Func" % (self.sympfx, self.modname) def tokenize(self, txt, seps=None, quotes=None): if seps is None: @@ -952,16 +951,16 @@ class vcc(object): j.cstuff(fo, 'h') fo.close() - def cstruct(self, fo, csn): - fo.write("\n%s {\n" % csn) + def cstruct(self, fo): + fo.write("\nstruct %s {\n" % self.csn) for j in self.contents: j.cstruct(fo, True) for j in sorted(self.enums): fo.write("\tVCL_ENUM\t\t\t*enum_%s;\n" % j) fo.write("};\n") - def cstruct_init(self, fo, csn): - fo.write("\nstatic const %s Vmod_Func = {\n" % csn) + def cstruct_init(self, fo): + fo.write("\nstatic const struct %s %s = {\n" % (self.csn, self.csn)) for j in self.contents: j.cstruct(fo, False) fo.write("\n") @@ -1000,8 +999,9 @@ class vcc(object): fo.write("\t.vrt_major =\tVRT_MAJOR_VERSION,\n") fo.write("\t.vrt_minor =\tVRT_MINOR_VERSION,\n") fo.write('\t.name =\t\t"%s",\n' % self.modname) - fo.write('\t.func =\t\t&Vmod_Func,\n') - fo.write('\t.func_len =\tsizeof(Vmod_Func),\n') + fo.write('\t.func =\t\t&%s,\n' % self.csn) + fo.write('\t.func_len =\tsizeof(%s),\n' % self.csn) + fo.write('\t.func_name =\t"%s",\n' % self.csn) fo.write('\t.proto =\tVmod_Proto,\n') fo.write('\t.json =\t\tVmod_Json,\n') fo.write('\t.abi =\t\tVMOD_ABI_Version,\n') @@ -1038,21 +1038,18 @@ class vcc(object): i.cstuff(fo, 'c') i.cstuff(fx, 'o') - csn = "Vmod_%s_Func" % self.modname - scsn = "struct " + csn + self.cstruct(fo) + self.cstruct(fx) - self.cstruct(fo, scsn) - self.cstruct(fx, scsn) - - fo.write("\n/*lint -esym(754, " + csn + "::*) */\n") - self.cstruct_init(fo, scsn) + fo.write("\n/*lint -esym(754, " + self.csn + "::*) */\n") + self.cstruct_init(fo) fx.close() fo.write("\nstatic const char Vmod_Proto[] =\n") for i in open(fnx): fo.write('\t"%s\\n"\n' % i.rstrip()) - fo.write('\t"static struct %s %s;";\n' % (csn, csn)) + fo.write('\t"static struct %s %s;";\n' % (self.csn, self.csn)) os.remove(fnx) From phk at FreeBSD.org Thu Jan 31 11:45:13 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 31 Jan 2019 11:45:13 +0000 (UTC) Subject: [master] d8ae26b5a Give the typedefs unique names and $prefix event functions Message-ID: <20190131114514.0A9D0A928C@lists.varnish-cache.org> commit d8ae26b5a0b4a1102360a61a145e4730e52d68bc Author: Poul-Henning Kamp Date: Thu Jan 31 11:43:57 2019 +0000 Give the typedefs unique names and $prefix event functions diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 37338888b..1e92145e4 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -127,6 +127,13 @@ def unquote(txt): assert is_quoted(txt) return txt[1:-1] +def fmt_cstruct(fo, a, b): + ''' Output line in vmod struct ''' + t = "\t%s\t" % a + while len(t.expandtabs()) < 40: + t += "\t" + fo.write("%s%s\n" % (t, b)) + ####################################################################### @@ -177,15 +184,6 @@ def lwrap(s, width=64): return "\n".join(ll) + "\n" -def fmt_cstruct(fo, mn, x): - """ - Align fields in C struct - """ - a = "\ttd_" + mn + "_" + x - while len(a.expandtabs()) < 40: - a += "\t" - fo.write("%s*%s;\n" % (a, x)) - ####################################################################### @@ -424,9 +422,12 @@ class ProtoType(object): s += ", ".join(ll) return s + ');' + def typedef_name(self): + return 'td_' + self.st.vcc.sympfx + \ + self.st.vcc.modname + '_' + self.cname() + def typedef(self, args): - tn = 'td_' + self.st.vcc.modname + '_' + self.cname() - return "typedef " + self.proto(args, name=tn) + return "typedef " + self.proto(args, name=self.typedef_name()) def argstructname(self): return "struct %s_arg" % self.cname(True) @@ -532,6 +533,20 @@ class Stanza(object): def cstuff(self, unused_fo, unused_where): return + def fmt_cstruct_proto(self, fo, proto, define): + if define: + fmt_cstruct( + fo, + proto.typedef_name(), + '*' + proto.cname() + ';' + ) + else: + fmt_cstruct( + fo, + '.' + proto.cname() + ' =', + '*' + self.vcc.sympfx + proto.cname() + ',' + ) + def cstruct(self, unused_fo, unused_define): return @@ -647,13 +662,16 @@ class EventStanza(Stanza): def cstuff(self, fo, where): if where == 'h': - fo.write("vmod_event_f %s;\n" % self.event_func) + fo.write("vmod_event_f %s%s;\n" % + (self.vcc.sympfx, self.event_func)) def cstruct(self, fo, define): if define: - fo.write("\tvmod_event_f\t\t\t*_event;\n") + fmt_cstruct(fo, "vmod_event_f", "*_event;") else: - fo.write("\t%s,\n" % self.event_func) + fmt_cstruct(fo, + "._event =", + '*' + self.vcc.sympfx + self.event_func + ',') def json(self, jl): jl.append(["$EVENT", "%s._event" % self.vcc.csn]) @@ -672,10 +690,7 @@ class FunctionStanza(Stanza): fo.write(self.proto.cproto(['VRT_CTX'], where)) def cstruct(self, fo, define): - if define: - fmt_cstruct(fo, self.vcc.modname, self.proto.cname()) - else: - fo.write("\t" + self.proto.cname(pfx=True) + ",\n") + self.fmt_cstruct_proto(fo, self.proto, define) def json(self, jl): jl.append(["$FUNC", "%s" % self.proto.name]) @@ -737,13 +752,8 @@ class ObjectStanza(Stanza): fo.write("\n") def cstruct(self, fo, define): - if define: - fmt_cstruct(fo, self.vcc.modname, self.init.name) - fmt_cstruct(fo, self.vcc.modname, self.fini.name) - else: - p = "\t" + self.vcc.sympfx - fo.write(p + self.init.name + ",\n") - fo.write(p + self.fini.name + ",\n") + self.fmt_cstruct_proto(fo, self.init, define) + self.fmt_cstruct_proto(fo, self.fini, define) for i in self.methods: i.cstruct(fo, define) fo.write("\n") @@ -789,10 +799,7 @@ class MethodStanza(Stanza): p.methods.append(self) def cstruct(self, fo, define): - if define: - fmt_cstruct(fo, self.vcc.modname, self.proto.cname()) - else: - fo.write('\t' + self.proto.cname(pfx=True) + ",\n") + self.fmt_cstruct_proto(fo, self.proto, define) def json(self, jl): jl.append(["$METHOD", self.proto.name[len(self.pfx)+1:]]) @@ -956,7 +963,7 @@ class vcc(object): for j in self.contents: j.cstruct(fo, True) for j in sorted(self.enums): - fo.write("\tVCL_ENUM\t\t\t*enum_%s;\n" % j) + fmt_cstruct(fo, 'VCL_ENUM', '*enum_%s;' % j) fo.write("};\n") def cstruct_init(self, fo): diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 8738ee243..43e203262 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -371,7 +371,7 @@ event_cold(VRT_CTX, const struct vmod_priv *priv) } int v_matchproto_(vmod_event_f) -event_function(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e) +xyzzy_event_function(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e) { switch (e) { From gquintard at users.noreply.github.com Thu Jan 31 22:43:08 2019 From: gquintard at users.noreply.github.com (guillaume quintard) Date: Thu, 31 Jan 2019 22:43:08 +0000 (UTC) Subject: [master] b23dddd3e Add param.reset command to varnishadm Message-ID: <20190131224308.927B963843@lists.varnish-cache.org> commit b23dddd3ee02cd5cf093ae7fd534ace34ef90656 Author: Guillaume Quintard Date: Mon Jan 14 21:35:27 2019 -0800 Add param.reset command to varnishadm diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 9c6a74685..2bf69a0e5 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -505,6 +505,8 @@ MCF_ParamSet(struct cli *cli, const char *param, const char *val) VCLI_Out(cli, "parameter \"%s\" is protected.", param); return; } + if (!val) + val = pp->def; if (pp->func(cli->sb, pp, val)) VCLI_SetResult(cli, CLIS_PARAM); @@ -534,6 +536,16 @@ mcf_param_set(struct cli *cli, const char * const *av, void *priv) MCF_ParamSet(cli, av[2], av[3]); } +/*--------------------------------------------------------------------*/ + +static void v_matchproto_(cli_func_t) +mcf_param_reset(struct cli *cli, const char * const *av, void *priv) +{ + + (void)priv; + MCF_ParamSet(cli, av[2], NULL); +} + /*-------------------------------------------------------------------- * Add a group of parameters to the global set and sort by name. */ @@ -595,6 +607,7 @@ static struct cli_proto cli_params[] = { { CLICMD_PARAM_SHOW, "", mcf_param_show, mcf_param_show_json }, { CLICMD_PARAM_SET, "", mcf_param_set }, + { CLICMD_PARAM_RESET, "", mcf_param_reset }, { NULL } }; diff --git a/bin/varnishtest/tests/b00008.vtc b/bin/varnishtest/tests/b00008.vtc index 6134ff0e6..4fdf4205b 100644 --- a/bin/varnishtest/tests/b00008.vtc +++ b/bin/varnishtest/tests/b00008.vtc @@ -38,6 +38,12 @@ varnish v1 -clijson "ping -j" varnish v1 -clierr 106 "param.set waiter HASH(0x8839c4c)" +varnish v1 -cliexpect 60 "param.show first_byte_timeout" +varnish v1 -cliok "param.set first_byte_timeout 120" +varnish v1 -cliexpect 120 "param.show first_byte_timeout" +varnish v1 -cliok "param.reset first_byte_timeout" +varnish v1 -cliexpect 60 "param.show first_byte_timeout" + varnish v1 -cliok "param.set cli_limit 128" varnish v1 -clierr 201 "param.show" diff --git a/doc/changes.rst b/doc/changes.rst index 79e353b15..37b7dfbba 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -110,6 +110,8 @@ Varnish Cache trunk (ongoing) * added ``req.is_hitmiss`` and ``req.is_hitpass`` (2743_) +* Add ``param.reset`` command to ``varnishadm`` + .. _2809: https://github.com/varnishcache/varnish-cache/issues/2809 .. _2820: https://github.com/varnishcache/varnish-cache/issues/2820 .. _2815: https://github.com/varnishcache/varnish-cache/issues/2815 diff --git a/include/tbl/cli_cmds.h b/include/tbl/cli_cmds.h index 25f13d4ac..57efeaab6 100644 --- a/include/tbl/cli_cmds.h +++ b/include/tbl/cli_cmds.h @@ -133,6 +133,14 @@ CLI_CMD(VCL_LABEL, 2, 2 ) +CLI_CMD(PARAM_RESET, + "param.reset", + "param.reset ", + "Reset parameter to default value.", + "", + 1,1 +) + CLI_CMD(PARAM_SHOW, "param.show", "param.show [-l|-j] [|changed]",