From geoff at uplex.de Mon Jul 2 10:49:10 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 2 Jul 2018 10:49:10 +0000 (UTC) Subject: [master] 36a1f756a For backends with PROXYv1, send UNKNOWN in a probe if the backend's address family is neither of AF_INET or AF_INET6. Message-ID: <20180702104910.09EADA755A@lists.varnish-cache.org> commit 36a1f756ae2825e1f0091a023efc85becb0c48b5 Author: Geoff Simmons Date: Mon Jul 2 00:13:12 2018 +0200 For backends with PROXYv1, send UNKNOWN in a probe if the backend's address family is neither of AF_INET or AF_INET6. This means that UNKNOWN is sent in probes to UDS backends when .proxy_header=1 is set. Also verify that a UDS backend receives PROXY LOCAL in a probe when .proxy_header=2 is set. Fixes #2702 Closes #2726 diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index e63cd02c4..026a8928c 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -269,8 +269,9 @@ vbp_write_proxy_v1(struct vbp_target *vt, int *sock) else if (ss.ss_family == AF_INET) VSB_printf(&vsb, " TCP4 "); else - WRONG("Unknown family"); - VSB_printf(&vsb, "%s %s %s %s\r\n", addr, addr, port, port); + VSB_printf(&vsb, " UNKNOWN\r\n"); + if (ss.ss_family == AF_INET6 || ss.ss_family == AF_INET) + VSB_printf(&vsb, "%s %s %s %s\r\n", addr, addr, port, port); AZ(VSB_finish(&vsb)); return (vbp_write(vt, sock, VSB_data(&vsb), VSB_len(&vsb))); diff --git a/bin/varnishtest/tests/r02702.vtc b/bin/varnishtest/tests/r02702.vtc new file mode 100644 index 000000000..6208f0c07 --- /dev/null +++ b/bin/varnishtest/tests/r02702.vtc @@ -0,0 +1,74 @@ +varnishtest "probes to UDS backends with .proxy_header in [12]" + +# Since we can only reliably forward IP addresses via PROXY with +# Varnish, we can't use the trick from o00002.vtc (use a Varnish +# instance listening for PROXY as a backend) to verify PROXYv1 (but +# the PROXY header can be viewed in the test log). + +# We just verify that the probe is good (and that Varnish doesn't +# crash with WRONG). + +server s1 -listen "${tmpdir}/s1.sock" { + rxreq + txresp +} -start + +# For PROXYv1 with a UDS backend, we send PROXY UNKNOWN in the probe. +varnish v1 -vcl { + backend s1 { + .path = "${s1_sock}"; + .proxy_header = 1; + + .probe = { + .window = 1; + .threshold = 1; + .interval = 0.5s; + } + } +} -start + +delay 1 + +varnish v1 -cliexpect "vcl1.s1[ ]+probe[ ]+1/1[ ]+good" backend.list + +# For PROXYv2, we apply a trick similar to o0000[24].vtc, since +# Varnish accepts (and ignores) PROXY LOCAL. + +server s2 { + rxreq + expect req.http.Host == "v2" + expect req.http.X-Forwarded-For == "0.0.0.0" + txresp +} -start + +varnish v2 -arg "-a ${tmpdir}/v2.sock,PROXY" -vcl { + backend s2 { .host = "${s2_addr}"; .port = "${s2_port}"; } +} -start + +varnish v3 -vcl { + backend bp { + .path = "${v2_addr}"; + .host_header = "v2"; + .proxy_header = 2; + + .probe = { + .window = 1; + .threshold = 1; + .interval = 0.5s; + } + } +} -start + +server s1 -wait + +delay 1 + +varnish v3 -cliexpect "vcl1.bp[ ]+probe[ ]+1/1[ ]+good" backend.list + +# Verify in the v2 log that PROXY LOCAL was sent. + +logexpect l1 -v v2 -d 1 -g session -q "Proxy" { + expect 0 * Begin sess + expect * = Proxy {^\d+ local local local local$} + expect * = End +} -run From hermunn at varnish-software.com Mon Jul 2 12:39:07 2018 From: hermunn at varnish-software.com (hermunn) Date: Mon, 2 Jul 2018 12:39:07 +0000 (UTC) Subject: [master] ff38535ac Reintroduce the req.grace variable, change keep behavior Message-ID: <20180702123907.F33DF52B2@lists.varnish-cache.org> commit ff38535acd6d9c3b87d7fbbcc9c6917f7106d216 Author: P?l Hermunn Johansen Date: Tue Jun 12 16:33:07 2018 +0200 Reintroduce the req.grace variable, change keep behavior The req.grace variable can be set in vcl_recv to cap the grace of objects in the cache, in the same way as in 3.0.x The "keep" behavior changes with this patch. We now always go to vcl_miss when the expired object is out of grace, or we go to the waiting list. The result is that it is no longer possible to deliver a "keep" object in vcl_hit. Note that when we get to vcl_miss, we will still have the 304 candidate, but without the detour by vcl_hit. This commit changes VCL, but only slightly, so we aim to back port this to earlier versions of Varnish Cache. Refs: #1799 and #2519 diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 551a3e1d0..c5c5cb9b0 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -471,6 +471,7 @@ struct req { uint8_t digest[DIGEST_LEN]; double d_ttl; + double d_grace; ssize_t req_bodybytes; /* Parsed req bodybytes */ const struct stevedore *storage; diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c index efe3814c6..6aa1ae07c 100644 --- a/bin/varnishd/cache/cache_expire.c +++ b/bin/varnishd/cache/cache_expire.c @@ -74,6 +74,24 @@ EXP_Ttl(const struct req *req, const struct objcore *oc) return (oc->t_origin + r); } +/*-------------------------------------------------------------------- + * Calculate an object's effective ttl+grace time, taking req.grace into + * account if it is available. + */ + +double +EXP_Ttl_grace(const struct req *req, const struct objcore *oc) +{ + double g; + + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + + g = oc->grace; + if (req != NULL && req->d_grace >= 0. && req->d_grace < g) + g = req->d_grace; + return (EXP_Ttl(req, oc) + g); +} + /*-------------------------------------------------------------------- * Post an objcore to the exp_thread's inbox. */ diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 55a451cf7..38531b149 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -478,36 +478,50 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, busy_found = 0; } + if (exp_oc != NULL && EXP_Ttl_grace(req, exp_oc) < req->t_req) { + /* the newest object is out of grace */ + if (!busy_found) { + /* + * here we get to insert the busy object. This + * translates to a MISS with a 304 candidate. + */ + assert(oh->refcnt > 1); + assert(exp_oc->objhead == oh); + exp_oc->refcnt++; + /* Insert objcore in objecthead and release mutex */ + *bocp = hsh_insert_busyobj(wrk, oh); + /* NB: no deref of objhead, new object inherits reference */ + Lck_Unlock(&oh->mtx); + *ocp = exp_oc; + return (HSH_MISS); + } else { + /* we have no use for this very expired object */ + exp_oc = NULL; + } + } if (exp_oc != NULL) { + /* + * here the object is within grace, so we expect it to + * be delivered + */ assert(oh->refcnt > 1); assert(exp_oc->objhead == oh); + exp_oc->refcnt++; if (!busy_found) { *bocp = hsh_insert_busyobj(wrk, oh); retval = HSH_EXPBUSY; } else { AZ(req->hash_ignore_busy); - /* - * here we have a busy object, but if the stale object - * is not under grace we go to the waiting list - * instead of returning the stale object - */ - if (EXP_Ttl(req, exp_oc) + exp_oc->grace < req->t_req) - retval = HSH_BUSY; - else - retval = HSH_EXP; - } - if (retval != HSH_BUSY) { - exp_oc->refcnt++; - - if (exp_oc->hits < LONG_MAX) - exp_oc->hits++; - Lck_Unlock(&oh->mtx); - if (retval == HSH_EXP) - assert(HSH_DerefObjHead(wrk, &oh)); - *ocp = exp_oc; - return (retval); + retval = HSH_EXP; } + if (exp_oc->hits < LONG_MAX) + exp_oc->hits++; + Lck_Unlock(&oh->mtx); + if (retval == HSH_EXP) + assert(HSH_DerefObjHead(wrk, &oh)); + *ocp = exp_oc; + return (retval); } if (!busy_found) { diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 5a5e5ebb5..c65434308 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -507,14 +507,23 @@ cnt_lookup(struct worker *wrk, struct req *req) AZ(req->objcore); if (lr == HSH_MISS) { - /* Found nothing */ - AZ(oc); - if (busy != NULL) { + /* Found nothing or an object that was out of grace */ + if (oc == NULL) { + /* Nothing */ + if (busy != NULL) { + AN(busy->flags & OC_F_BUSY); + req->objcore = busy; + req->req_step = R_STP_MISS; + } else { + req->req_step = R_STP_PASS; + } + } else { + /* Object out of grace */ + AN(busy); /* Else we should be on waiting list */ AN(busy->flags & OC_F_BUSY); req->objcore = busy; + req->stale_oc = oc; req->req_step = R_STP_MISS; - } else { - req->req_step = R_STP_PASS; } return (REQ_FSM_MORE); } @@ -804,6 +813,7 @@ cnt_recv_prep(struct req *req, const char *ci) AN(req->director_hint); req->d_ttl = -1; + req->d_grace = -1; req->disable_esi = 0; req->hash_always_miss = 0; req->hash_ignore_busy = 0; diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 292e33631..88c9f2cf7 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -126,6 +126,7 @@ void VDI_Init(void); /* cache_exp.c */ double EXP_Ttl(const struct req *, const struct objcore *); +double EXP_Ttl_grace(const struct req *, const struct objcore *oc); void EXP_Insert(struct worker *wrk, struct objcore *oc); void EXP_Remove(struct objcore *); diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index ea8ea5e6e..175bee8a3 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -434,7 +434,7 @@ VRT_r_obj_storage(VRT_CTX) /*--------------------------------------------------------------------*/ -#define REQ_VAR_L(nm, elem, type,extra) \ +#define REQ_VAR_L(nm, elem, type, extra) \ \ VCL_VOID \ VRT_l_req_##nm(VRT_CTX, type arg) \ @@ -459,6 +459,8 @@ REQ_VAR_L(backend_hint, director_hint, VCL_BACKEND,) REQ_VAR_R(backend_hint, director_hint, VCL_BACKEND) REQ_VAR_L(ttl, d_ttl, VCL_DURATION, if (!(arg>0.0)) arg = 0;) REQ_VAR_R(ttl, d_ttl, VCL_DURATION) +REQ_VAR_L(grace, d_grace, VCL_DURATION, if (!(arg>0.0)) arg = 0;) +REQ_VAR_R(grace, d_grace, VCL_DURATION) /*--------------------------------------------------------------------*/ diff --git a/bin/varnishtest/tests/b00098.vtc b/bin/varnishtest/tests/b00062.vtc similarity index 100% rename from bin/varnishtest/tests/b00098.vtc rename to bin/varnishtest/tests/b00062.vtc diff --git a/bin/varnishtest/tests/b00064.vtc b/bin/varnishtest/tests/b00064.vtc new file mode 100644 index 000000000..6054ec1e6 --- /dev/null +++ b/bin/varnishtest/tests/b00064.vtc @@ -0,0 +1,126 @@ +varnishtest "Test that req.grace will hold a client when a miss is anticipated" + +barrier b1 cond 2 + +server s1 { + rxreq + expect req.url == "/" + txresp -body "0" + + rxreq + expect req.url == "/" + txresp -body "1" + + # second time we get a request, we use some time to serve it + rxreq + expect req.url == "/" + barrier b1 sync + delay .1 + txresp -body "2" + + # Last request, to a different URL to catch it if varnish asks for "/" too many times + rxreq + expect req.url == "/2" + txresp -body "x" +} -start + +varnish v1 -vcl+backend { + import std; + + sub vcl_recv { + # When we know in vcl_recv that we will have no grace, it is now + # possible to signal this to the lookup function: + if (req.http.X-no-grace) { + set req.grace = 0s; + } + } + sub vcl_hit { + set req.http.X-grace = obj.grace; + } + sub vcl_backend_response { + set beresp.ttl = 0.1s; + set beresp.grace = 1m; + } + sub vcl_deliver { + if (req.http.X-grace) { + set resp.http.X-grace = req.http.X-grace; + set resp.http.X-req-grace = req.grace; + } + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 + expect resp.body == "0" + expect resp.http.X-grace == + # let the object's ttl expire + delay .2 + # get a genuinely fresh object by disabling grace + # we will not get to vcl_hit to see the grace + txreq -hdr "X-no-grace: true" + rxresp + expect resp.status == 200 + expect resp.body == "1" + expect resp.http.X-grace == + expect resp.http.X-req-grace == +} -run + +# let the latest object's ttl expire. +delay .2 + +varnish v1 -expect n_object == 1 + +# c2 asks for the object under grace +client c2 { + txreq + rxresp + # we did not disable grace in the request, so we should get the graced object here + expect resp.status == 200 + expect resp.body == "1" + expect resp.http.X-grace == "60.000" + expect resp.http.X-req-grace < 0. +} -start + +delay .1 + +# c3 asks for graced object, but now we disable grace. +client c3 { + txreq -hdr "X-no-grace: true" + rxresp + expect resp.status == 200 + # Here we have disable grace and should get the object from the background fetch, + # which will take us into vcl_hit + expect resp.body == "2" + expect resp.http.X-grace == "60.000" + expect resp.http.X-req-grace == "0.000" +} -start + +delay .1 + +# c4 does not disable grace, and should get the grace object even +# though c3 is waiting on the background thread to deliver a new +# version. + +client c4 { + txreq + rxresp + barrier b1 sync + expect resp.status == 200 + # We should get what c1 got in the very beginning + expect resp.body == "1" + expect resp.http.X-grace == "60.000" + expect resp.http.X-req-grace < 0. +} -start + +client c2 -wait +client c3 -wait +client c4 -wait + +client c5 { + txreq -url "/2" + rxresp + expect resp.status == 200 + expect resp.body == "x" +} -run diff --git a/bin/varnishtest/tests/r02705.vtc b/bin/varnishtest/tests/r02705.vtc new file mode 100644 index 000000000..54bec95c2 --- /dev/null +++ b/bin/varnishtest/tests/r02705.vtc @@ -0,0 +1,51 @@ +varnishtest "No vcl_hit when grace has run out, with working IMS" + +server s1 { + rxreq + txresp -hdr {Etag: "Foo"} -body "1" + + rxreq + txresp -status 304 + + rxreq + expect req.url == "/2" + txresp -body "3" +} -start + +varnish v1 -vcl+backend { + sub vcl_backend_response { + set beresp.ttl = 1ms; + set beresp.grace = 0s; + set beresp.keep = 1m; + if (bereq.http.Hit) { + set beresp.http.Hit = bereq.http.Hit; + } + } + sub vcl_hit { + set req.http.Hit = "HIT"; + } +} -start + +client c1 { + txreq + rxresp + expect resp.http.Hit == + expect resp.http.Etag == {"Foo"} + expect resp.status == 200 + expect resp.body == "1" + delay .2 + txreq + rxresp + expect resp.http.Hit == + expect resp.http.Etag == {"Foo"} + expect resp.status == 200 + expect resp.body == "1" +} -run + +client c2 { + txreq -url "/2" + rxresp + expect resp.http.Hit == + expect resp.body == "3" +} -run + diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index 332f1c344..12423f2ec 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -304,6 +304,21 @@ req.ttl Deprecated and scheduled for removal with varnish release 7. +req.grace + + Type: DURATION + + Readable from: client + + Writable from: client + + + Upper limit on the object grace. + + During lookup the minimum of req.grace and the object's stored + grace value will be used as the object's grace. + + req.xid Type: STRING From hermunn at varnish-software.com Mon Jul 2 12:39:08 2018 From: hermunn at varnish-software.com (hermunn) Date: Mon, 2 Jul 2018 12:39:08 +0000 (UTC) Subject: [master] 12578f449 Undeprecate req.ttl Message-ID: <20180702123908.220FA52B5@lists.varnish-cache.org> commit 12578f4493090a4c919cf22ea7185d18eae8919b Author: P?l Hermunn Johansen Date: Thu Jun 7 14:21:04 2018 +0200 Undeprecate req.ttl Right now there are use cases that cannot be solved without req.ttl, so we undeprecate it. diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index 12423f2ec..692a5888c 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -296,13 +296,6 @@ req.ttl Upper limit on the object age for cache lookups to return hit. - Usage of req.ttl should be replaced with a check on - obj.ttl in vcl_hit, returning miss when needed, but - this currently hits bug #1799, so an additional - workaround is required. - - Deprecated and scheduled for removal with varnish release 7. - req.grace From hermunn at varnish-software.com Mon Jul 2 12:39:08 2018 From: hermunn at varnish-software.com (hermunn) Date: Mon, 2 Jul 2018 12:39:08 +0000 (UTC) Subject: [master] 7494d6adc Update the documentation on grace and keep Message-ID: <20180702123908.46C8252B9@lists.varnish-cache.org> commit 7494d6adc9bad382cf267b352c21f5c1069bb14c Author: P?l Hermunn Johansen Date: Thu Jun 7 14:56:24 2018 +0200 Update the documentation on grace and keep The main point is to clearly recomend using req.grace for the most common use case - using different grace time when the backend is healthy. diff --git a/doc/sphinx/users-guide/vcl-grace.rst b/doc/sphinx/users-guide/vcl-grace.rst index 8db2a4649..dddfac5f7 100644 --- a/doc/sphinx/users-guide/vcl-grace.rst +++ b/doc/sphinx/users-guide/vcl-grace.rst @@ -34,15 +34,11 @@ Keep ~~~~ Setting an object's `keep` tells Varnish that it should keep an object -in the cache for some additional time. There are two reasons to do this: - -* To use the object to construct a conditional GET backend request (with - If-Modified-Since: and/or ?f-None-Match: headers), allowing the backend - to reply with a 304 Not Modified response, which may be more efficient - on the backend and saves re-transmitting the unchanged body. -* To be able to serve the object when grace has expired but we have a - problem with getting a fresh object from the backend. This will require - a change in ``sub vcl_hit``, as described below. +in the cache for some additional time. The reasons to set `keep` is to +use the object to construct a conditional GET backend request (with +If-Modified-Since: and/or ?f-None-Match: headers), allowing the +backend to reply with a 304 Not Modified response, which may be more +efficient on the backend and saves re-transmitting the unchanged body. The values are additive, so if grace is 10 seconds and keep is 1 minute, then objects will survive in cache for 70 seconds after the TTL has @@ -64,9 +60,9 @@ The effect of grace and keep For most users setting the default grace and/or a suitable grace for each object is enough. The default VCL will do the right thing and -behave as described above. However, if you want to customize how varnish -behaves by changing ``sub vcl_hit``, then you should know some of the -details on how this works. +behave as described above. However, if you want to customize how +varnish behaves, then you should know some of the details on how this +works. When ``sub vcl_recv`` ends with ``return (lookup)`` (which is the default behavior), Varnish will look for a matching object in its @@ -78,13 +74,15 @@ will consider the following: Then, Varnish reacts using the following rules: -* If there is no backend request for the object, one is scheduled and - ``sub vcl_hit`` is called immediately. -* If there is a backend request going on, but the object is under grace, - ``sub vcl_hit`` is called immediately. -* If there is a backend request going on, but the grace has expired, - processing is halted until the backend request has finished and a - fresh object is available. +* If the `grace period` has run out and there is no ongoing backend + request, then ``sub vcl_miss`` is called immediately, and the object + will be used as a 304 candidate. +* If the `grace period` has run out and there is an ongoing backend + request, then the request will wait until the backend request + finishes. +* If there is no backend request for the object, one is scheduled. +* Assuming the object will be delivered, ``sub vcl_hit`` is called + immediately. Note that the backend fetch happens asynchronously, and the moment the new object is in it will replace the one we've already got. @@ -106,9 +104,17 @@ used. It looks like this:: return (miss); } -If you follow the code, you see that Varnish delivers graced objects -while fetching fresh copies, but if grace has expired the clients have to -wait until a new copy is available. +The effect of the built-in VCL is in fact equivalent to the following:: + + sub vcl_hit { + return (deliver); + } + +This is because ``obj.ttl + obj.grace > 0s`` always will evaluate to +true. However, the the VCL is as it is to show users how to +differentiate between a pure hit and a `grace` hit. With the next +major version of Varnish, the default VCL is planned to change to the +latter, shorter version. Misbehaving servers ~~~~~~~~~~~~~~~~~~~ @@ -118,104 +124,42 @@ web- and application servers. If you have enabled :ref:`users-guide-advanced_backend_servers-health` you can check if the backend is sick and modify the behavior when it -comes to grace. There are essentially two ways of doing this. You can -explicitly deliver kept object (that is not within grace) when you see -that the backend is sick, or you can explicitly `not` serve an expired -object when you know that the backend is healthy. The two methods have -slightly different characteristics, as we shall see. - -In both cases we assume that you avoid inserting objects into the cache -when you get certain errors from the backend, for example by using the -following:: - - sub vcl_backend_response { - if (beresp.status == 503 && bereq.is_bgfetch) { - return (abandon); - } - } - -Method 1: When the backend is healthy, use a lower grace value -============================================================== - -Imagine that you have set an object's grace to a high value that you -wish to use when the backend is sick, for example:: +comes to grace. This can done in the following way:: sub vcl_backend_response { set beresp.grace = 24h; - // no keep + // no keep - the grace should be enough for 304 candidates } -Then you can use the following code as your ``sub vcl_hit``:: - - if (std.healthy(req.backend_hint)) { - // change the behavior for health backends: Cap grace to 10s - if (obj.ttl + obj.grace > 0s && obj.ttl + 10s > 0s) { - return (deliver); - } else { - return (miss); - } - } - -The effect of this is that, when the backend is healthy, objects with -grace above 10 seconds will have an `effective` grace of 10 seconds. -When the backend is sick, the default VCL kicks in, and the long grace -is used. - -This method has one potentially serious problem when more than one -client asks for an object that has expired its TTL. If the second of -these requests arrives after the effective grace, but before the first -request has completed, then the second request will be turned into a -`pass`. - -In practice this method works well in most cases, but if you -experience excessive `pass` behavior, this translates to a reduced -hit rate and higher load on the backend. When this happens you will -see the error message `vcl_hit{} returns miss without busy object` in -the log. - -Method 2: When the backend is sick, deliver kept objects -======================================================== - -With this method, we assume that we have used `sub backend_response` -to set `beresp.grace` to a value that is suitable for healthy backends, -and with a `beresp.keep` that corresponds to the time we want to serve -the object when the backend is sick. For example:: - - sub vcl_backend_response { - set beresp.grace = 10s; - set beresp.keep = 24h; + sub vcl_recv { + if (std.healthy(req.backend_hint)) { + // change the behavior for healthy backends: Cap grace to 10s + set req.grace = 10s; + } } -The appropriate code for ``vcl_hit`` then becomes:: - - if (!std.healthy(req.backend_hint) && (obj.ttl + obj.grace + obj.keep > 0s)) { - return (deliver); - } +In the example above, the special variable ``req.grace`` is set. The +effect is that, when the backend is healthy, objects with grace above +10 seconds will have an `effective` grace of 10 seconds. When the +backend is sick, the default VCL kicks in, and the long grace is used. -Typically you can omit the second part of the if test due to the -expiry thread deleting objects where `grace + keep` has expired. It is -possible that the `expiry thread` can be lagging slightly behind, but -for almost all practical purposes you are probably fine with the -following:: +Additionally, you might want to stop cache insertion when a backend fetch +returns an ``5xx`` error:: - if (!std.healthy(req.backend_hint)) { - return (deliver); - } - -The problem with this solution concerns requests that are waiting for -a backend fetch to finish. If the backend fetch gets to ``return -(abandon)``, then all the requests that are waiting will get to ``sub -vcl_hit`` with an `error object` created by the error handling -code/VCL. In other words, you risk that some clients will get errors -instead of the more desirable stale objects. + sub vcl_backend_response { + if (beresp.status >= 500 && bereq.is_bgfetch) { + return (abandon); + } + } Summary ~~~~~~~ -Grace mode allows Varnish to deliver slightly stale content to clients while -getting a fresh version from the backend. The result is faster load times -with a low cost. +Grace mode allows Varnish to deliver slightly stale content to clients +while getting a fresh version from the backend. The result is faster +load times at lower cost. -It is possible to change the behavior when it comes to grace and keep, for -example by changing the `effective` grace depending on the health of the -backend, but you have to be careful. +It is possible to limit the grace during lookup by setting +``req.grace`` and then change the behavior when it comes to +grace. Often this is done to change the `effective` grace depending on +the health of the backend. From hermunn at varnish-software.com Mon Jul 2 12:39:08 2018 From: hermunn at varnish-software.com (hermunn) Date: Mon, 2 Jul 2018 12:39:08 +0000 (UTC) Subject: [master] 95f33d35c Test case that shows return(abandon) to avoid cache insertion Message-ID: <20180702123908.672F052BE@lists.varnish-cache.org> commit 95f33d35c134ee09f24ebcda892073d45cbda2ba Author: P?l Hermunn Johansen Date: Fri Jun 15 10:54:48 2018 +0200 Test case that shows return(abandon) to avoid cache insertion diff --git a/bin/varnishtest/tests/b00063.vtc b/bin/varnishtest/tests/b00063.vtc new file mode 100644 index 000000000..1adfe2680 --- /dev/null +++ b/bin/varnishtest/tests/b00063.vtc @@ -0,0 +1,90 @@ +varnishtest "Abandon background fetch when backend serves 5xx" + +barrier b1 cond 2 +barrier b2 cond 3 + +server s1 { + # This is what we want to get to all client requests below + rxreq + expect req.url == "/1" + txresp -body "1" + + # 503s will be abandoned when we have a bgfetch + rxreq + expect req.url == "/1" + txresp -status 503 -body "2" + + # varnish will disconnect on a 503 + accept + rxreq + expect req.url == "/1" + # wait until varnish has delivered 200 before replying + # with the 404 + barrier b2 sync + delay .1 + # this response will not be abandoned + txresp -status 404 -reason "Not Found" -body "3" + + # some other resource at the end + rxreq + expect req.url == "/2" + txresp -body "4" +} -start + +varnish v1 -vcl+backend { + sub vcl_backend_response { + if (beresp.status >= 500 && bereq.is_bgfetch) { + return (abandon); + } + if (beresp.status >= 400) { + set beresp.ttl = 1m; + } else { + set beresp.ttl = 1ms; + } + set beresp.grace = 1m; + } +} -start + +client c1 { + txreq -url "/1" + rxresp + expect resp.status == 200 + expect resp.body == "1" + delay .2 + txreq -url "/1" + rxresp + expect resp.status == 200 + expect resp.body == "1" + delay .2 + barrier b1 sync + txreq -url "/1" + rxresp + expect resp.status == 200 + expect resp.body == "1" + barrier b2 sync +} -start + +client c2 { + barrier b1 sync + txreq -url "/1" + rxresp + expect resp.status == 200 + expect resp.body == "1" + barrier b2 sync +} -start + +client c1 -wait +client c2 -wait + +client c3 { + delay .1 + # We should now get a HIT on the 404: + txreq -url "/1" + rxresp + expect resp.status == 404 + expect resp.body == "3" + # do a different resource to make sure we got the right number of reqs to /1 + txreq -url "/2" + rxresp + expect resp.body == "4" +} -run From hermunn at varnish-software.com Mon Jul 2 12:39:08 2018 From: hermunn at varnish-software.com (hermunn) Date: Mon, 2 Jul 2018 12:39:08 +0000 (UTC) Subject: [master] 4a370dc4f simplify cnt_lookup Message-ID: <20180702123908.8F4AD52C2@lists.varnish-cache.org> commit 4a370dc4f91706b19c2e3d3103c4f13957349bbc Author: Nils Goroll Date: Tue Jun 19 15:43:25 2018 +0200 simplify cnt_lookup diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index c65434308..686f25452 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -507,23 +507,16 @@ cnt_lookup(struct worker *wrk, struct req *req) AZ(req->objcore); if (lr == HSH_MISS) { - /* Found nothing or an object that was out of grace */ - if (oc == NULL) { - /* Nothing */ - if (busy != NULL) { - AN(busy->flags & OC_F_BUSY); - req->objcore = busy; - req->req_step = R_STP_MISS; - } else { - req->req_step = R_STP_PASS; - } - } else { - /* Object out of grace */ - AN(busy); /* Else we should be on waiting list */ + if (busy != NULL) { + /* hitmiss, out-of-grace or ordinary miss */ AN(busy->flags & OC_F_BUSY); req->objcore = busy; req->stale_oc = oc; req->req_step = R_STP_MISS; + } else { + /* hitpass */ + AZ(oc); + req->req_step = R_STP_PASS; } return (REQ_FSM_MORE); } From hermunn at varnish-software.com Mon Jul 2 12:39:08 2018 From: hermunn at varnish-software.com (hermunn) Date: Mon, 2 Jul 2018 12:39:08 +0000 (UTC) Subject: [master] 89631a9ae reshuffle HSH_Lookup() code a bit to condense and clarify Message-ID: <20180702123908.B45D452C7@lists.varnish-cache.org> commit 89631a9aed793f14d64cd16315b0e11ea0642e0a Author: Nils Goroll Date: Tue Jun 19 17:02:36 2018 +0200 reshuffle HSH_Lookup() code a bit to condense and clarify With the req.grace reintroduction, we got two identical cases where we insert a busy object: When there is no exp_oc (obviously) or when the exp_oc does not fulfil the req.grace limit. This lead to some code duplication. IMHO, handling first the case where we need to insert a busy object (with or without exp_oc) and then handling the exception that we do not wait on a busy object if the exp_oc is in grace is cleaner. Please note that comparing this change using diff -u does not make much sense, I believe the simplicity of the reshuffling change becomes clear when looking at the old and new code side-by-side diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 38531b149..3fbbc702d 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -478,58 +478,45 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp, busy_found = 0; } - if (exp_oc != NULL && EXP_Ttl_grace(req, exp_oc) < req->t_req) { - /* the newest object is out of grace */ - if (!busy_found) { - /* - * here we get to insert the busy object. This - * translates to a MISS with a 304 candidate. - */ + if (!busy_found) { + /* Insert objcore in objecthead */ + *bocp = hsh_insert_busyobj(wrk, oh); + + if (exp_oc != NULL) { assert(oh->refcnt > 1); assert(exp_oc->objhead == oh); exp_oc->refcnt++; - /* Insert objcore in objecthead and release mutex */ - *bocp = hsh_insert_busyobj(wrk, oh); - /* NB: no deref of objhead, new object inherits reference */ Lck_Unlock(&oh->mtx); *ocp = exp_oc; - return (HSH_MISS); + + if (EXP_Ttl_grace(req, exp_oc) < req->t_req) { + retval = HSH_MISS; + } else { + if (exp_oc->hits < LONG_MAX) + exp_oc->hits++; + retval = HSH_EXPBUSY; + } } else { - /* we have no use for this very expired object */ - exp_oc = NULL; + Lck_Unlock(&oh->mtx); + retval = HSH_MISS; } + + return (retval); } - if (exp_oc != NULL) { - /* - * here the object is within grace, so we expect it to - * be delivered - */ + + AN(busy_found); + if (exp_oc != NULL && EXP_Ttl_grace(req, exp_oc) >= req->t_req) { + /* we do not wait on the busy object if in grace */ assert(oh->refcnt > 1); assert(exp_oc->objhead == oh); exp_oc->refcnt++; - - if (!busy_found) { - *bocp = hsh_insert_busyobj(wrk, oh); - retval = HSH_EXPBUSY; - } else { - AZ(req->hash_ignore_busy); - retval = HSH_EXP; - } - if (exp_oc->hits < LONG_MAX) - exp_oc->hits++; Lck_Unlock(&oh->mtx); - if (retval == HSH_EXP) - assert(HSH_DerefObjHead(wrk, &oh)); *ocp = exp_oc; - return (retval); - } - if (!busy_found) { - /* Insert objcore in objecthead and release mutex */ - *bocp = hsh_insert_busyobj(wrk, oh); - /* NB: no deref of objhead, new object inherits reference */ - Lck_Unlock(&oh->mtx); - return (HSH_MISS); + assert(HSH_DerefObjHead(wrk, &oh)); + if (exp_oc->hits < LONG_MAX) + exp_oc->hits++; + return (HSH_EXP); } /* There are one or more busy objects, wait for them */ From phk at FreeBSD.org Mon Jul 2 14:46:09 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 2 Jul 2018 14:46:09 +0000 (UTC) Subject: [master] 87766181d Set the mode on the UDS socket, in case we are running as root with varnish user etc. Message-ID: <20180702144610.04B6C9087@lists.varnish-cache.org> commit 87766181dfda2a674fa5cb0453ae07d68f303894 Author: Poul-Henning Kamp Date: Mon Jul 2 14:45:22 2018 +0000 Set the mode on the UDS socket, in case we are running as root with varnish user etc. diff --git a/bin/varnishtest/tests/r02702.vtc b/bin/varnishtest/tests/r02702.vtc index 6208f0c07..c9a938a90 100644 --- a/bin/varnishtest/tests/r02702.vtc +++ b/bin/varnishtest/tests/r02702.vtc @@ -41,7 +41,7 @@ server s2 { txresp } -start -varnish v2 -arg "-a ${tmpdir}/v2.sock,PROXY" -vcl { +varnish v2 -arg "-a ${tmpdir}/v2.sock,PROXY,mode=0777" -vcl { backend s2 { .host = "${s2_addr}"; .port = "${s2_port}"; } } -start From phk at FreeBSD.org Mon Jul 2 15:35:11 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 2 Jul 2018 15:35:11 +0000 (UTC) Subject: [master] fefa8364d Make sure the expiry is in the VSL before proceeding Message-ID: <20180702153512.101AA9F31@lists.varnish-cache.org> commit fefa8364df09ba0f0f2d734f100d6bf8500f2392 Author: Poul-Henning Kamp Date: Mon Jul 2 15:34:05 2018 +0000 Make sure the expiry is in the VSL before proceeding diff --git a/bin/varnishtest/tests/b00063.vtc b/bin/varnishtest/tests/b00063.vtc index 1adfe2680..7a68e2e2f 100644 --- a/bin/varnishtest/tests/b00063.vtc +++ b/bin/varnishtest/tests/b00063.vtc @@ -45,6 +45,12 @@ varnish v1 -vcl+backend { } } -start +logexpect l1 -v v1 -g raw { + expect * * ExpKill EXP_expire + expect * * ExpKill EXP_When + expect * * ExpKill EXP_expire +} -start + client c1 { txreq -url "/1" rxresp @@ -76,6 +82,9 @@ client c2 { client c1 -wait client c2 -wait +# Make sure the expiry has happened +logexpect l1 -wait + client c3 { delay .1 # We should now get a HIT on the 404: From daghf at varnish-software.com Tue Jul 3 14:11:08 2018 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Tue, 3 Jul 2018 14:11:08 +0000 (UTC) Subject: [master] a3cadf035 Remember to zero h2->mailcall on h2 rxbody vfp failure Message-ID: <20180703141108.A24ECA5493@lists.varnish-cache.org> commit a3cadf0357dc2c39d78c4e72b934f517b867d974 Author: Dag Haavi Finstad Date: Tue Jul 3 16:05:00 2018 +0200 Remember to zero h2->mailcall on h2 rxbody vfp failure Fixes: #2572 diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index f4cc294e5..3851089cd 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -835,8 +835,10 @@ h2_vfp_body_fini(struct vfp_ctx *vc, struct vfp_entry *vfe) H2SE_REFUSED_STREAM); Lck_Lock(&h2->sess->mtx); r2->error = H2SE_REFUSED_STREAM; - if (h2->mailcall == r2) + if (h2->mailcall == r2) { + h2->mailcall = NULL; AZ(pthread_cond_signal(h2->cond)); + } Lck_Unlock(&h2->sess->mtx); } } From fgsch at lodoss.net Tue Jul 3 19:26:12 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 3 Jul 2018 19:26:12 +0000 (UTC) Subject: [master] 146fbcd82 Polish Message-ID: <20180703192612.32B2B5C5A@lists.varnish-cache.org> commit 146fbcd82eb3ba9c09169c1574ef0e929e0f41b0 Author: Federico G. Schwindt Date: Tue Jul 3 19:29:06 2018 +0100 Polish diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 026a8928c..1b1190ebd 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -260,18 +260,15 @@ vbp_write_proxy_v1(struct vbp_target *vt, int *sock) VTCP_myname(*sock, addr, sizeof addr, port, sizeof port); AN(VSB_new(&vsb, buf, sizeof buf, VSB_FIXEDLEN)); - AZ(VSB_cat(&vsb, "PROXY")); l = sizeof ss; AZ(getsockname(*sock, (void *)&ss, &l)); - if (ss.ss_family == AF_INET6) - VSB_printf(&vsb, " TCP6 "); - else if (ss.ss_family == AF_INET) - VSB_printf(&vsb, " TCP4 "); - else - VSB_printf(&vsb, " UNKNOWN\r\n"); - if (ss.ss_family == AF_INET6 || ss.ss_family == AF_INET) - VSB_printf(&vsb, "%s %s %s %s\r\n", addr, addr, port, port); + if (ss.ss_family == AF_INET || ss.ss_family == AF_INET6) { + VSB_printf(&vsb, "PROXY %s %s %s %s %s\r\n", + ss.ss_family == AF_INET ? "TCP4" : "TCP6", + addr, addr, port, port); + } else + VSB_cat(&vsb, "PROXY UNKNOWN\r\n"); AZ(VSB_finish(&vsb)); return (vbp_write(vt, sock, VSB_data(&vsb), VSB_len(&vsb))); From fgsch at lodoss.net Tue Jul 3 19:26:12 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 3 Jul 2018 19:26:12 +0000 (UTC) Subject: [master] 57c1d53e5 python 3 takes priority over python 2 Message-ID: <20180703192612.514FA5C5D@lists.varnish-cache.org> commit 57c1d53e5adcd71add274809c7fd9c0132bb127f Author: Federico G. Schwindt Date: Tue Jul 3 19:52:31 2018 +0100 python 3 takes priority over python 2 This was the original intent but got lost in autoconf translation. diff --git a/varnish.m4 b/varnish.m4 index 8a92d5f10..f052aa145 100644 --- a/varnish.m4 +++ b/varnish.m4 @@ -130,10 +130,12 @@ AC_DEFUN([_VARNISH_CHECK_DEVEL], [ # --------------------- AC_DEFUN([_VARNISH_CHECK_PYTHON], [ - AM_PATH_PYTHON([2.7], [], [ + AC_PATH_PROGS(PYTHON, [python3.7 python3.6 python3.5 python3.4 python2.7 python3 python2 python], [ + AC_MSG_ERROR([no suitable Python interpreter found]) + ]) + AM_PYTHON_CHECK_VERSION([$PYTHON], [2.7], [], [ AC_MSG_ERROR([Python >= 2.7 is required.]) ]) - ]) # _VARNISH_VMOD_LDFLAGS From fgsch at lodoss.net Tue Jul 3 19:58:08 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 3 Jul 2018 19:58:08 +0000 (UTC) Subject: [master] 9a23bd5e6 Revert 57c1d53e for now Message-ID: <20180703195808.18DF869D0@lists.varnish-cache.org> commit 9a23bd5e661b55ea504cc5c2fcdf4d17e0f4b1f0 Author: Federico G. Schwindt Date: Tue Jul 3 20:56:37 2018 +0100 Revert 57c1d53e for now For some strange reason this is failing with 3.6 in travis. diff --git a/varnish.m4 b/varnish.m4 index f052aa145..8a92d5f10 100644 --- a/varnish.m4 +++ b/varnish.m4 @@ -130,12 +130,10 @@ AC_DEFUN([_VARNISH_CHECK_DEVEL], [ # --------------------- AC_DEFUN([_VARNISH_CHECK_PYTHON], [ - AC_PATH_PROGS(PYTHON, [python3.7 python3.6 python3.5 python3.4 python2.7 python3 python2 python], [ - AC_MSG_ERROR([no suitable Python interpreter found]) - ]) - AM_PYTHON_CHECK_VERSION([$PYTHON], [2.7], [], [ + AM_PATH_PYTHON([2.7], [], [ AC_MSG_ERROR([Python >= 2.7 is required.]) ]) + ]) # _VARNISH_VMOD_LDFLAGS From nils.goroll at uplex.de Wed Jul 4 15:01:12 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 4 Jul 2018 15:01:12 +0000 (UTC) Subject: [master] 2efb4633c document format change from cc8f349f809d4a91761bec73d2e4a92792690349 Message-ID: <20180704150112.291B2A1235@lists.varnish-cache.org> commit 2efb4633c93d20675ad6902f8a7679e8b53bb5a0 Author: Nils Goroll Date: Wed Jul 4 17:00:02 2018 +0200 document format change from cc8f349f809d4a91761bec73d2e4a92792690349 diff --git a/doc/changes.rst b/doc/changes.rst index 97ce46696..9c398cc53 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -67,6 +67,12 @@ bundled tools for custom profile definitions to also contain a prefix to match the tag against. +logging +------- + +* The backend name logged under the ``Backend_health`` tag has been + changed back from `vcl name`.`backend name` to just `backend name` + C APIs (for vmod and utility authors) ------------------------------------- From dridi.boukelmoune at gmail.com Wed Jul 4 15:15:09 2018 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 4 Jul 2018 15:15:09 +0000 (UTC) Subject: [master] a09c3b46e Revert "Terminate varnishtop -d automatically" Message-ID: <20180704151509.CEBD0A17D8@lists.varnish-cache.org> commit a09c3b46efb0b22b55c6a6f3c1d27f9edb31269e Author: Dridi Boukelmoune Date: Wed Jul 4 16:14:39 2018 +0200 Revert "Terminate varnishtop -d automatically" This reverts commit d1b78e8d13a993931d19f2c11acd19959e258ce9. The problem is that upon exiting the screen is likely to be cleared on most terminals (not the one we embed in varnishtest for example) so it doesn't give any chance to the user to read the output. The consensus is to restore the previous behavior and fix the manual instead. Conflicts: bin/varnishtest/tests/u00004.vtc bin/varnishtop/varnishtop.c Breaks: bin/varnishtest/tests/r02686.vtc Refs #2721 diff --git a/bin/varnishtest/tests/r02686.vtc b/bin/varnishtest/tests/r02686.vtc index eb85b9914..a58a4f545 100644 --- a/bin/varnishtest/tests/r02686.vtc +++ b/bin/varnishtest/tests/r02686.vtc @@ -18,7 +18,7 @@ delay 2 process p1 -expect-text 1 1 "list length 1" process p1 -expect-text 1 75 "(EOF)" process p1 -expect-text 3 6 "1.00 ReqMethod GET" -process p1 -screen_dump -wait +process p1 -screen_dump -write q -wait process p2 -dump {varnishtop -n ${v1_name} -1 -i ReqMethod} -start process p2 -winsz 30 80 diff --git a/bin/varnishtest/tests/u00004.vtc b/bin/varnishtest/tests/u00004.vtc index 5435f9865..5dfaa3310 100644 --- a/bin/varnishtest/tests/u00004.vtc +++ b/bin/varnishtest/tests/u00004.vtc @@ -14,8 +14,10 @@ client c1 { shell -expect "fetch" "varnishtop -n ${v1_name} -1" -process p1 "varnishtop -n ${v1_name} -d" -run -screen_dump -process p1 -expect-text 0 0 fetch +process p1 "varnishtop -n ${v1_name} -d" -start +delay 1 +process p1 -screen_dump -expect-text 0 0 fetch +process p1 -write q -wait # without -f shell -match "1\\.00 RespHeader Date: [^\\n]+\\n" { diff --git a/bin/varnishtop/varnishtop.c b/bin/varnishtop/varnishtop.c index 9372c5e36..942e25b97 100644 --- a/bin/varnishtop/varnishtop.c +++ b/bin/varnishtop/varnishtop.c @@ -76,13 +76,13 @@ struct top { }; static int period = 60; /* seconds */ +static int end_of_file = 0; static unsigned ntop; static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER; static int f_flag = 0; static unsigned maxfieldlen = 0; static const char *ident; -static volatile sig_atomic_t end_of_file = 0; static volatile sig_atomic_t quit = 0; static VRB_HEAD(t_order, top) h_order = VRB_INITIALIZER(&h_order); @@ -203,7 +203,7 @@ static void update(int p) { struct top *tp, *tp2; - int l, len, eof; + int l, len; double t = 0; static time_t last = 0; static unsigned n = 0; @@ -221,8 +221,7 @@ update(int p) AC(erase()); q = ident; len = COLS - strlen(q); - eof = end_of_file; - if (eof) + if (end_of_file) AC(mvprintw(0, len - (1 + 6), "%s (EOF)", q)); else AC(mvprintw(0, len - 1, "%s", q)); @@ -240,7 +239,7 @@ update(int p) len, len, tp->rec_data)); t = tp->count; } - if (eof) + if (end_of_file) continue; tp->count += (1.0/3.0 - tp->count) / (double)n; if (tp->count * 10 < t || l > LINES * 10) { @@ -252,8 +251,6 @@ update(int p) } } AC(refresh()); - if (eof) - quit = 1; } static void * From dridi.boukelmoune at gmail.com Wed Jul 4 15:15:09 2018 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 4 Jul 2018 15:15:09 +0000 (UTC) Subject: [master] 76bed41db varnishtop -d does not exit by itself Message-ID: <20180704151509.EBA6DA17DB@lists.varnish-cache.org> commit 76bed41dbbe1c976bcab622a502402ac64e9333a Author: Dridi Boukelmoune Date: Wed Jul 4 16:19:15 2018 +0200 varnishtop -d does not exit by itself Unless -1 is used too, but -1 already implies -d so it doesn't really count. Fixes #2721 diff --git a/bin/varnishtop/varnishtop_options.h b/bin/varnishtop/varnishtop_options.h index ebf8912c1..6605d463a 100644 --- a/bin/varnishtop/varnishtop_options.h +++ b/bin/varnishtop/varnishtop_options.h @@ -37,6 +37,11 @@ " statistics once and exit. Implies ``-d``." \ ) +#define TOP_OPT_d \ + VOPT("d", "[-d]", "Process old log entries", \ + "Process log records at the head of the log." \ + ) + #define TOP_OPT_f \ VOPT("f", "[-f]", "First field only", \ "Sort and group only on the first field of each log entry." \ @@ -59,7 +64,7 @@ TOP_OPT_1 VSL_OPT_b VSL_OPT_c VSL_OPT_C -VUT_OPT_d +TOP_OPT_d TOP_OPT_f VUT_OPT_g VUT_OPT_h From phk at phk.freebsd.dk Wed Jul 4 15:18:36 2018 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Wed, 04 Jul 2018 15:18:36 +0000 Subject: [master] a09c3b46e Revert "Terminate varnishtop -d automatically" In-Reply-To: <20180704151509.CEBD0A17D8@lists.varnish-cache.org> References: <20180704151509.CEBD0A17D8@lists.varnish-cache.org> Message-ID: <97424.1530717516@critter.freebsd.dk> -------- In message <20180704151509.CEBD0A17D8 at lists.varnish-cache.org>, Dridi Boukelmoune writes: >commit a09c3b46efb0b22b55c6a6f3c1d27f9edb31269e >Author: Dridi Boukelmoune >Date: Wed Jul 4 16:14:39 2018 +0200 Does this mean that endwin() is no longer called ? That will leave the terminal in raw mode, unless your shell implements termio-sanity code. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From dridi at varni.sh Wed Jul 4 15:53:24 2018 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 4 Jul 2018 17:53:24 +0200 Subject: [master] a09c3b46e Revert "Terminate varnishtop -d automatically" In-Reply-To: <97424.1530717516@critter.freebsd.dk> References: <20180704151509.CEBD0A17D8@lists.varnish-cache.org> <97424.1530717516@critter.freebsd.dk> Message-ID: On Wed, Jul 4, 2018 at 5:18 PM, Poul-Henning Kamp wrote: > -------- > In message <20180704151509.CEBD0A17D8 at lists.varnish-cache.org>, Dridi Boukelmoune writes: > >>commit a09c3b46efb0b22b55c6a6f3c1d27f9edb31269e >>Author: Dridi Boukelmoune >>Date: Wed Jul 4 16:14:39 2018 +0200 > > Does this mean that endwin() is no longer called ? > > That will leave the terminal in raw mode, unless your shell implements > termio-sanity code. I believe we call it once we leave the main loop in do_curses(). This time I manually checked to avoid the same mistake than last time, relying too much on passing test cases. Dridi From phk at FreeBSD.org Wed Jul 11 21:29:10 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 11 Jul 2018 21:29:10 +0000 (UTC) Subject: [master] 47dc5fd73 Postpone conversion to BOOL until we absolutely have to. Message-ID: <20180711212910.C4CC7615F5@lists.varnish-cache.org> commit 47dc5fd732eddff135d7c75672f2d3e28f324705 Author: Poul-Henning Kamp Date: Wed Jul 11 21:28:18 2018 +0000 Postpone conversion to BOOL until we absolutely have to. Fixes 2727 diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc index ee2f2a22d..60e1fb346 100644 --- a/bin/varnishtest/tests/v00020.vtc +++ b/bin/varnishtest/tests/v00020.vtc @@ -277,11 +277,45 @@ varnish v1 -errvcl {Comparison of different types: BACKEND '==' STRING} { } } +varnish v1 -errvcl {'!' must be followed by BOOL, found REAL.} { + import std; + sub vcl_recv { if (!std.random(0,1)) { } } +} + +varnish v1 -errvcl {'&&' must be followed by BOOL, found REAL.} { + import std; + sub vcl_recv { if (0 && std.random(0,1)) { } } +} + +varnish v1 -errvcl {'||' must be followed by BOOL, found REAL.} { + import std; + sub vcl_recv { if (0 || std.random(0,1)) { } } +} + +varnish v1 -errvcl {'&&' must be preceeded by BOOL, found REAL.} { + import std; + sub vcl_recv { if (std.random(0,1) && 0) { } } +} + +varnish v1 -errvcl {'||' must be preceeded by BOOL, found REAL.} { + import std; + sub vcl_recv { if (std.random(0,1) || 0) { } } +} + server s1 { rxreq txresp -hdr "bar: X" } -start +varnish v1 -vcl+backend { + // Ticket 2727 + sub vcl_recv { + if ((2 - 1) > 0) { + # nothing + } + } +} + varnish v1 -vcl+backend { sub vcl_deliver { set resp.http.foo = diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index e5ef67416..b9ad25a8e 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -246,6 +246,28 @@ vcc_expr_fmt(struct vsb *d, int ind, const struct expr *e1) } } +/*-------------------------------------------------------------------- + */ + +static void +vcc_expr_tobool(struct vcc *tl, struct expr **e) +{ + + if ((*e)->fmt == BOOL) + return; + if ((*e)->fmt == BACKEND || (*e)->fmt == INT) + *e = vcc_expr_edit(tl, BOOL, "(\v1 != 0)", *e, NULL); + else if ((*e)->fmt == DURATION) + *e = vcc_expr_edit(tl, BOOL, "(\v1 > 0)", *e, NULL); + else if ((*e)->fmt == STRINGS) + *e = vcc_expr_edit(tl, BOOL, "(\vS != 0)", *e, NULL); + /* + * We do not provide automatic folding from REAL to BOOL + * because comparing to zero is seldom an exact science + * and we want to force people to explicitly get it right. + */ +} + /*-------------------------------------------------------------------- */ @@ -1126,16 +1148,6 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, vcc_type_t fmt) default: break; } - if (fmt != BOOL) - return; - if ((*e)->fmt == BACKEND || (*e)->fmt == INT) - *e = vcc_expr_edit(tl, BOOL, "(\v1 != 0)", *e, NULL); - else if ((*e)->fmt == DURATION) - *e = vcc_expr_edit(tl, BOOL, "(\v1 > 0)", *e, NULL); - else if ((*e)->fmt == STRINGS) - *e = vcc_expr_edit(tl, BOOL, "(\vS != 0)", *e, NULL); - else - INCOMPL(); } /*-------------------------------------------------------------------- @@ -1147,63 +1159,92 @@ vcc_expr_cmp(struct vcc *tl, struct expr **e, vcc_type_t fmt) static void vcc_expr_not(struct vcc *tl, struct expr **e, vcc_type_t fmt) { - struct expr *e2; struct token *tk; *e = NULL; - if (fmt != BOOL || tl->t->tok != '!') { - vcc_expr_cmp(tl, e, fmt); - return; - } - - vcc_NextToken(tl); tk = tl->t; - vcc_expr_cmp(tl, &e2, fmt); + if (tl->t->tok == '!') + vcc_NextToken(tl); + vcc_expr_cmp(tl, e, fmt); ERRCHK(tl); - if (e2->fmt != BOOL) { + if (tk->tok != '!') + return; + vcc_expr_tobool(tl, e); + ERRCHK(tl); + if ((*e)->fmt != BOOL) { VSB_printf(tl->sb, "'!' must be followed by BOOL, found "); - VSB_printf(tl->sb, "%s.\n", vcc_utype(e2->fmt)); + VSB_printf(tl->sb, "%s.\n", vcc_utype((*e)->fmt)); vcc_ErrWhere2(tl, tk, tl->t); } else { - *e = vcc_expr_edit(tl, BOOL, "!(\v1)", e2, NULL); + *e = vcc_expr_edit(tl, BOOL, "!(\v1)", *e, NULL); } } /*-------------------------------------------------------------------- - * SYNTAX: - * ExprCand: - * ExprNot { '&&' ExprNot } * + * CAND and COR are identical save for a few details, but they are + * stacked so handling them in the same function is not simpler. + * Instead have them both call this helper function to do everything. */ +typedef void upfunc(struct vcc *tl, struct expr **e, vcc_type_t fmt); + static void -vcc_expr_cand(struct vcc *tl, struct expr **e, vcc_type_t fmt) +vcc_expr_bin_bool(struct vcc *tl, struct expr **e, vcc_type_t fmt, + unsigned ourtok, upfunc *up, const char *tokstr) { struct expr *e2; struct token *tk; + char buf[32]; *e = NULL; - vcc_expr_not(tl, e, fmt); + tk = tl->t; + up(tl, e, fmt); ERRCHK(tl); - if ((*e)->fmt != BOOL || tl->t->tok != T_CAND) + if (tl->t->tok != ourtok) return; + vcc_expr_tobool(tl, e); + ERRCHK(tl); + if ((*e)->fmt != BOOL) { + VSB_printf(tl->sb, + "'%s' must be preceeded by BOOL," + " found %s.\n", tokstr, vcc_utype((*e)->fmt)); + vcc_ErrWhere2(tl, tk, tl->t); + return; + } *e = vcc_expr_edit(tl, BOOL, "(\v+\n\v1", *e, NULL); - while (tl->t->tok == T_CAND) { + while (tl->t->tok == ourtok) { vcc_NextToken(tl); tk = tl->t; vcc_expr_not(tl, &e2, fmt); ERRCHK(tl); + vcc_expr_tobool(tl, &e2); + ERRCHK(tl); if (e2->fmt != BOOL) { VSB_printf(tl->sb, - "'&&' must be followed by BOOL," - " found %s.\n", vcc_utype(e2->fmt)); + "'%s' must be followed by BOOL," + " found %s.\n", tokstr, vcc_utype(e2->fmt)); vcc_ErrWhere2(tl, tk, tl->t); return; } - *e = vcc_expr_edit(tl, BOOL, "\v1\v-\n&&\v+\n\v2", *e, e2); + bprintf(buf, "\v1\v-\n%s\v+\n\v2", tokstr); + *e = vcc_expr_edit(tl, BOOL, buf, *e, e2); } *e = vcc_expr_edit(tl, BOOL, "\v1\v-\n)", *e, NULL); } +/*-------------------------------------------------------------------- + * SYNTAX: + * ExprCand: + * ExprNot { '&&' ExprNot } * + */ + +static void +vcc_expr_cand(struct vcc *tl, struct expr **e, vcc_type_t fmt) +{ + + vcc_expr_bin_bool(tl, e, fmt, T_CAND, vcc_expr_not, "&&"); +} + /*-------------------------------------------------------------------- * SYNTAX: * ExprCOR: @@ -1213,30 +1254,8 @@ vcc_expr_cand(struct vcc *tl, struct expr **e, vcc_type_t fmt) static void vcc_expr_cor(struct vcc *tl, struct expr **e, vcc_type_t fmt) { - struct expr *e2; - struct token *tk; - *e = NULL; - vcc_expr_cand(tl, e, fmt); - ERRCHK(tl); - if ((*e)->fmt != BOOL || tl->t->tok != T_COR) - return; - *e = vcc_expr_edit(tl, BOOL, "(\v+\n\v1", *e, NULL); - while (tl->t->tok == T_COR) { - vcc_NextToken(tl); - tk = tl->t; - vcc_expr_cand(tl, &e2, fmt); - ERRCHK(tl); - if (e2->fmt != BOOL) { - VSB_printf(tl->sb, - "'||' must be followed by BOOL," - " found %s.\n", vcc_utype(e2->fmt)); - vcc_ErrWhere2(tl, tk, tl->t); - return; - } - *e = vcc_expr_edit(tl, BOOL, "\v1\v-\n||\v+\n\v2", *e, e2); - } - *e = vcc_expr_edit(tl, BOOL, "\v1\v-\n)", *e, NULL); + vcc_expr_bin_bool(tl, e, fmt, T_COR, vcc_expr_cand, "||"); } /*-------------------------------------------------------------------- @@ -1274,6 +1293,11 @@ vcc_expr0(struct vcc *tl, struct expr **e, vcc_type_t fmt) *e = vcc_expr_edit(tl, STRING_LIST, "\v+\n\v1,\nvrt_magic_string_end\v-", *e, NULL); + if (fmt == BOOL) { + vcc_expr_tobool(tl, e); + ERRCHK(tl); + } + if (fmt != (*e)->fmt) { VSB_printf(tl->sb, "Expression has type %s, expected %s\n", vcc_utype((*e)->fmt), vcc_utype(fmt)); From fgsch at lodoss.net Fri Jul 13 11:06:11 2018 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Fri, 13 Jul 2018 11:06:11 +0000 (UTC) Subject: [master] d8afb649e Set the task arguments under the lock Message-ID: <20180713110611.3439D986C6@lists.varnish-cache.org> commit d8afb649e465f3dce249dfd9985e34c87a056555 Author: Federico G. Schwindt Date: Fri Jul 13 12:02:23 2018 +0100 Set the task arguments under the lock I've been torturing varnish with this change for some time and was not able to reproduce the problem. Should fix #2719. diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 2c0ad150f..7cd02533d 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -214,13 +214,12 @@ Pool_Task_Arg(struct worker *wrk, enum task_prio prio, task_func_t *func, wrk2 = wrk; retval = 0; } - Lck_Unlock(&pp->mtx); AZ(wrk2->task.func); - assert(arg_len <= WS_Reserve(wrk2->aws, arg_len)); memcpy(wrk2->aws->f, arg, arg_len); wrk2->task.func = func; wrk2->task.priv = wrk2->aws->f; + Lck_Unlock(&pp->mtx); if (retval) AZ(pthread_cond_signal(&wrk2->cond)); return (retval); From geoff at uplex.de Mon Jul 16 08:31:14 2018 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 16 Jul 2018 08:31:14 +0000 (UTC) Subject: [master] 30005bb36 Set socket options correctly for the accepted socket. Message-ID: <20180716083114.26909A613B@lists.varnish-cache.org> commit 30005bb3665cf4bbbd986ce3e310855d539301bd Author: Geoff Simmons Date: Mon Jul 16 10:29:30 2018 +0200 Set socket options correctly for the accepted socket. Closes #2722 diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index 455697fe1..908613f47 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -221,17 +221,16 @@ vca_tcp_opt_init(void) } static void -vca_tcp_opt_test(const struct listen_sock *ls) +vca_tcp_opt_test(const int sock, const unsigned uds) { - int i, n, sock; + int i, n; struct tcp_opt *to; socklen_t l; void *ptr; - sock = ls->sock; for (n = 0; n < n_tcp_opts; n++) { to = &tcp_opts[n]; - if (to->iponly && ls->uds) + if (to->iponly && uds) continue; to->need = 1; ptr = calloc(1, to->sz); @@ -247,15 +246,14 @@ vca_tcp_opt_test(const struct listen_sock *ls) } static void -vca_tcp_opt_set(const struct listen_sock *ls, int force) +vca_tcp_opt_set(const int sock, const unsigned uds, const int force) { - int n, sock; + int n; struct tcp_opt *to; - sock = ls->sock; for (n = 0; n < n_tcp_opts; n++) { to = &tcp_opts[n]; - if (to->iponly && ls->uds) + if (to->iponly && uds) continue; if (to->need || force) { VTCP_Assert(setsockopt(sock, @@ -426,10 +424,10 @@ vca_make_session(struct worker *wrk, void *arg) wrk->stats->sess_conn++; if (need_test) { - vca_tcp_opt_test(wa->acceptlsock); + vca_tcp_opt_test(sp->fd, wa->acceptlsock->uds); need_test = 0; } - vca_tcp_opt_set(wa->acceptlsock, 0); + vca_tcp_opt_set(sp->fd, wa->acceptlsock->uds, 0); req = Req_New(wrk, sp); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -607,7 +605,7 @@ vca_acct(void *arg) if (ls->sock == -2) continue; // VCA_Shutdown assert (ls->sock > 0); - vca_tcp_opt_set(ls, 1); + vca_tcp_opt_set(ls->sock, ls->uds, 1); } AZ(pthread_mutex_unlock(&shut_mtx)); } @@ -646,7 +644,7 @@ ccf_start(struct cli *cli, const char * const *av, void *priv) ls->endpoint, strerror(errno)); return; } - vca_tcp_opt_set(ls, 1); + vca_tcp_opt_set(ls->sock, ls->uds, 1); if (cache_param->accept_filter) { int i; i = VTCP_filter_http(ls->sock); From phk at FreeBSD.org Tue Jul 17 09:32:12 2018 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 17 Jul 2018 09:32:12 +0000 (UTC) Subject: [master] 8d8e109d5 Increase listen-depth to 100 Message-ID: <20180717093212.349599B7CE@lists.varnish-cache.org> commit 8d8e109d5bf65a4537cd06da28e696d91a0baec1 Author: Poul-Henning Kamp Date: Tue Jul 17 09:30:34 2018 +0000 Increase listen-depth to 100 Submitted by: Frederic Lecaille diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index 3cc12fe28..2de165031 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -389,7 +389,7 @@ haproxy_build_backends(const struct haproxy *h, const char *vsb_data) break; *q++ = '\0'; - sock = VTCP_listen_on("localhost:0", NULL, 1, &err); + sock = VTCP_listen_on("localhost:0", NULL, 100, &err); if (err != NULL) vtc_fatal(h->vl, "Create listen socket failed: %s", err); From geoff at uplex.de Tue Jul 17 13:41:08 2018 From: geoff at uplex.de (Geoff Simmons) Date: Tue, 17 Jul 2018 13:41:08 +0000 (UTC) Subject: [master] 0139f83cf Remove an unnecessary #include from VMOD blob. Message-ID: <20180717134108.EF3A7A167C@lists.varnish-cache.org> commit 0139f83cf92330111fd3d79e76498e4597a2d9b5 Author: Geoff Simmons Date: Tue Jul 17 15:39:12 2018 +0200 Remove an unnecessary #include from VMOD blob. Had been used in its previous life as a 3rd-party VMOD. diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c index a53d8c415..920be37e0 100644 --- a/lib/libvmod_blob/vmod_blob.c +++ b/lib/libvmod_blob/vmod_blob.c @@ -26,7 +26,6 @@ * */ -#include "config.h" #include #include From dridi at varni.sh Tue Jul 17 14:08:56 2018 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 17 Jul 2018 16:08:56 +0200 Subject: [master] 0139f83cf Remove an unnecessary #include from VMOD blob. In-Reply-To: <20180717134108.EF3A7A167C@lists.varnish-cache.org> References: <20180717134108.EF3A7A167C@lists.varnish-cache.org> Message-ID: On Tue, Jul 17, 2018 at 3:41 PM, Geoff Simmons wrote: > > commit 0139f83cf92330111fd3d79e76498e4597a2d9b5 > Author: Geoff Simmons > Date: Tue Jul 17 15:39:12 2018 +0200 > > Remove an unnecessary #include from VMOD blob. > > Had been used in its previous life as a 3rd-party VMOD. Technically we should have it as the first include of any C file to honor our configuration and possibly effect system-wide includes (for example if macros like _POSIX_C_SOURCE was added at configure time). Dridi > diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c > index a53d8c415..920be37e0 100644 > --- a/lib/libvmod_blob/vmod_blob.c > +++ b/lib/libvmod_blob/vmod_blob.c > @@ -26,7 +26,6 @@ > * > */ > > -#include "config.h" > #include > #include > > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From phk at phk.freebsd.dk Tue Jul 17 14:15:01 2018 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Tue, 17 Jul 2018 14:15:01 +0000 Subject: [master] 0139f83cf Remove an unnecessary #include from VMOD blob. In-Reply-To: References: <20180717134108.EF3A7A167C@lists.varnish-cache.org> Message-ID: <16327.1531836901@critter.freebsd.dk> -------- In message , Dridi Boukelmoune writes: >On Tue, Jul 17, 2018 at 3:41 PM, Geoff Simmons wrote: >> >> commit 0139f83cf92330111fd3d79e76498e4597a2d9b5 >> Author: Geoff Simmons >> Date: Tue Jul 17 15:39:12 2018 +0200 >> >> Remove an unnecessary #include from VMOD blob. >> >> Had been used in its previous life as a 3rd-party VMOD. > >Technically we should have it as the first include of any C file to >honor our configuration and possibly effect system-wide includes (for >example if macros like _POSIX_C_SOURCE was added at configure time). +1 -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From geoff at uplex.de Tue Jul 17 14:19:32 2018 From: geoff at uplex.de (Geoff Simmons) Date: Tue, 17 Jul 2018 16:19:32 +0200 Subject: [master] 0139f83cf Remove an unnecessary #include from VMOD blob. In-Reply-To: <16327.1531836901@critter.freebsd.dk> References: <20180717134108.EF3A7A167C@lists.varnish-cache.org> <16327.1531836901@critter.freebsd.dk> Message-ID: <0c02a075-37fa-d669-4d5f-544ace1baf94@uplex.de> On 07/17/2018 04:15 PM, Poul-Henning Kamp wrote: > -------- > In message > , Dridi Boukelmoune writes: >>> >>> Remove an unnecessary #include from VMOD blob. >> >> Technically we should have it as the first include of any C file to >> honor our configuration and possibly effect system-wide includes (for >> example if macros like _POSIX_C_SOURCE was added at configure time). > > +1 All right, then I'll revert it. (Previously it had been used to return VERSION as a version string for the VMOD, so I thought we didn't need it any more.) -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg Tel +49 40 2880 5731 Mob +49 176 636 90917 Fax +49 40 42949753 http://uplex.de -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 833 bytes Desc: OpenPGP digital signature URL: From geoff at uplex.de Tue Jul 17 14:21:07 2018 From: geoff at uplex.de (Geoff Simmons) Date: Tue, 17 Jul 2018 14:21:07 +0000 (UTC) Subject: [master] 51ae885b6 Revert "Remove an unnecessary #include from VMOD blob." Message-ID: <20180717142107.C8226A2759@lists.varnish-cache.org> commit 51ae885b6bd189241f369f7df4acd1093fd12a98 Author: Geoff Simmons Date: Tue Jul 17 16:20:03 2018 +0200 Revert "Remove an unnecessary #include from VMOD blob." This reverts commit 0139f83cf92330111fd3d79e76498e4597a2d9b5. Despite its obsolete purpose in the previous life of the VMOD, we include config.h in all C sources for consistent configuration. diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c index 920be37e0..a53d8c415 100644 --- a/lib/libvmod_blob/vmod_blob.c +++ b/lib/libvmod_blob/vmod_blob.c @@ -26,6 +26,7 @@ * */ +#include "config.h" #include #include From geoff at uplex.de Tue Jul 17 16:12:11 2018 From: geoff at uplex.de (Geoff Simmons) Date: Tue, 17 Jul 2018 16:12:11 +0000 (UTC) Subject: [master] 2e5bc18be Test some out of workspace conditions for VMOD blob. Message-ID: <20180717161211.18C66A4AF6@lists.varnish-cache.org> commit 2e5bc18bebbb015e48717eab01f4f35f8961d9a7 Author: Geoff Simmons Date: Tue Jul 17 18:10:16 2018 +0200 Test some out of workspace conditions for VMOD blob. To get some better GCOV coverage. So far just IDENTITY encoding. diff --git a/bin/varnishtest/tests/m00049.vtc b/bin/varnishtest/tests/m00049.vtc new file mode 100644 index 000000000..bca702cad --- /dev/null +++ b/bin/varnishtest/tests/m00049.vtc @@ -0,0 +1,97 @@ +varnishtest "VMOD blob workspace overflow conditions" + +varnish v1 -vcl { + import blob; + import vtc; + backend b { .host = "${bad_ip}"; } + + sub vcl_recv { + if (req.url == "/1") { + # Too small for the vmod_priv object. + vtc.workspace_alloc(client, -8); + } + elsif (req.url == "/2") { + # Likely large enough for the vmod_priv object, + # but not enough left over for the decode. + vtc.workspace_alloc(client, -33); + } + elsif (req.url == "/3") { + # Enough for the decode, but not enough left + # over for the encode. + vtc.workspace_alloc(client, -50); + } + set req.http.Decode + = blob.encode(blob=blob.decode(encoded="1234567890")); + } + + sub vcl_miss { + if (req.url == "/4") { + # Enough for the req.hash BLOB (vmod_priv + 32 + # bytes), but not enough left over for the + # encoding. + vtc.workspace_alloc(client, -65); + set req.http.Encode = blob.encode(blob=req.hash); + } + return( synth(200) ); + } +} -start + +logexpect l1 -v v1 -d 0 -g vxid -q "VCL_Error" { + expect 0 * Begin req + expect * = VCL_Error {^vmod blob error: cannot decode, out of space$} + expect * = End +} -start + +client c1 { + txreq -url "/1" + rxresp + expect resp.status == 503 + expect resp.reason == "VCL failed" +} -run + +logexpect l1 -wait + +logexpect l1 -v v1 -d 0 -g vxid -q "VCL_Error" { + expect 0 * Begin req + expect * = VCL_Error {^vmod blob error: cannot decode, out of space$} + expect * = End +} -start + +client c1 { + txreq -url "/2" + rxresp + expect resp.status == 503 + expect resp.reason == "VCL failed" +} -run + +logexpect l1 -wait + +logexpect l1 -v v1 -d 0 -g vxid -q "VCL_Error" { + expect 0 * Begin req + expect * = VCL_Error {^vmod blob error: cannot encode, out of space$} + expect * = End +} -start + +client c1 { + txreq -url "/3" + rxresp + expect resp.status == 503 + expect resp.reason == "VCL failed" +} -run + +logexpect l1 -wait + +logexpect l1 -v v1 -d 0 -g vxid -q "VCL_Error" { + expect 0 * Begin req + expect * = VCL_Error {^vmod blob error: cannot encode, out of space$} + expect * = End +} -start + +client c1 { + txreq -url "/4" + rxresp + expect resp.status == 503 + expect resp.reason == "VCL failed" +} -run + +logexpect l1 -wait From geoff at uplex.de Wed Jul 18 08:22:09 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 18 Jul 2018 08:22:09 +0000 (UTC) Subject: [master] 068391678 Remove logexpect from the tests for VMOD blob out-of-workspace. Message-ID: <20180718082209.81D9F90721@lists.varnish-cache.org> commit 068391678aacdb8e54342f943d234146d394f503 Author: Geoff Simmons Date: Wed Jul 18 10:19:05 2018 +0200 Remove logexpect from the tests for VMOD blob out-of-workspace. On the various platforms, esp 32-bit, workspace may be exhausted before it gets to the VMOD at all, so we can't reliably expect specific error messages. This has nevertheless improved GCOV coverage, which was the goal. diff --git a/bin/varnishtest/tests/m00049.vtc b/bin/varnishtest/tests/m00049.vtc index bca702cad..f94f5afc6 100644 --- a/bin/varnishtest/tests/m00049.vtc +++ b/bin/varnishtest/tests/m00049.vtc @@ -36,12 +36,6 @@ varnish v1 -vcl { } } -start -logexpect l1 -v v1 -d 0 -g vxid -q "VCL_Error" { - expect 0 * Begin req - expect * = VCL_Error {^vmod blob error: cannot decode, out of space$} - expect * = End -} -start - client c1 { txreq -url "/1" rxresp @@ -49,14 +43,6 @@ client c1 { expect resp.reason == "VCL failed" } -run -logexpect l1 -wait - -logexpect l1 -v v1 -d 0 -g vxid -q "VCL_Error" { - expect 0 * Begin req - expect * = VCL_Error {^vmod blob error: cannot decode, out of space$} - expect * = End -} -start - client c1 { txreq -url "/2" rxresp @@ -64,14 +50,6 @@ client c1 { expect resp.reason == "VCL failed" } -run -logexpect l1 -wait - -logexpect l1 -v v1 -d 0 -g vxid -q "VCL_Error" { - expect 0 * Begin req - expect * = VCL_Error {^vmod blob error: cannot encode, out of space$} - expect * = End -} -start - client c1 { txreq -url "/3" rxresp @@ -79,19 +57,9 @@ client c1 { expect resp.reason == "VCL failed" } -run -logexpect l1 -wait - -logexpect l1 -v v1 -d 0 -g vxid -q "VCL_Error" { - expect 0 * Begin req - expect * = VCL_Error {^vmod blob error: cannot encode, out of space$} - expect * = End -} -start - client c1 { txreq -url "/4" rxresp expect resp.status == 503 expect resp.reason == "VCL failed" } -run - -logexpect l1 -wait From geoff at uplex.de Wed Jul 18 09:42:06 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 18 Jul 2018 09:42:06 +0000 (UTC) Subject: [master] fc95898d9 Fix the out-of-workspace tests for VMOD blob. Message-ID: <20180718094207.1C634921FF@lists.varnish-cache.org> commit fc95898d97fc4c205f739f2397040f12ac65b2c6 Author: Geoff Simmons Date: Wed Jul 18 11:40:25 2018 +0200 Fix the out-of-workspace tests for VMOD blob. The error may be status 500 on 32-bit systems. diff --git a/bin/varnishtest/tests/m00049.vtc b/bin/varnishtest/tests/m00049.vtc index f94f5afc6..81e0ff530 100644 --- a/bin/varnishtest/tests/m00049.vtc +++ b/bin/varnishtest/tests/m00049.vtc @@ -39,27 +39,31 @@ varnish v1 -vcl { client c1 { txreq -url "/1" rxresp - expect resp.status == 503 - expect resp.reason == "VCL failed" + expect resp.status >= 500 + expect resp.status <= 503 + expect resp.reason ~ {(?i)^VCL Failed$|^Internal Server Error$} } -run client c1 { txreq -url "/2" rxresp - expect resp.status == 503 - expect resp.reason == "VCL failed" + expect resp.status >= 500 + expect resp.status <= 503 + expect resp.reason ~ {(?i)^VCL Failed$|^Internal Server Error$} } -run client c1 { txreq -url "/3" rxresp - expect resp.status == 503 - expect resp.reason == "VCL failed" + expect resp.status >= 500 + expect resp.status <= 503 + expect resp.reason ~ {(?i)^VCL Failed$|^Internal Server Error$} } -run client c1 { txreq -url "/4" rxresp - expect resp.status == 503 - expect resp.reason == "VCL failed" + expect resp.status >= 500 + expect resp.status <= 503 + expect resp.reason ~ {(?i)^VCL Failed$|^Internal Server Error$} } -run From geoff at uplex.de Wed Jul 18 11:21:05 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 18 Jul 2018 11:21:05 +0000 (UTC) Subject: [master] 5ba6b682d VMOD blob out-of-workspace tests with all encoding schemes. Message-ID: <20180718112105.BDDEA94145@lists.varnish-cache.org> commit 5ba6b682d42ad02cf1acf7b863975db98821cfb9 Author: Geoff Simmons Date: Wed Jul 18 13:20:31 2018 +0200 VMOD blob out-of-workspace tests with all encoding schemes. diff --git a/bin/varnishtest/tests/m00049.vtc b/bin/varnishtest/tests/m00049.vtc index 81e0ff530..6850ffc26 100644 --- a/bin/varnishtest/tests/m00049.vtc +++ b/bin/varnishtest/tests/m00049.vtc @@ -44,7 +44,7 @@ client c1 { expect resp.reason ~ {(?i)^VCL Failed$|^Internal Server Error$} } -run -client c1 { +client c2 { txreq -url "/2" rxresp expect resp.status >= 500 @@ -52,7 +52,7 @@ client c1 { expect resp.reason ~ {(?i)^VCL Failed$|^Internal Server Error$} } -run -client c1 { +client c3 { txreq -url "/3" rxresp expect resp.status >= 500 @@ -60,10 +60,104 @@ client c1 { expect resp.reason ~ {(?i)^VCL Failed$|^Internal Server Error$} } -run -client c1 { +client c4 { txreq -url "/4" rxresp expect resp.status >= 500 expect resp.status <= 503 expect resp.reason ~ {(?i)^VCL Failed$|^Internal Server Error$} } -run + +# Similar tests with BASE64 encoding +varnish v1 -vcl { + import blob; + import vtc; + backend b { .host = "${bad_ip}"; } + + sub vcl_recv { + if (req.url == "/1") { + vtc.workspace_alloc(client, -33); + } + elsif (req.url == "/2") { + vtc.workspace_alloc(client, -50); + } + set req.http.Decode + = blob.encode(blob=blob.decode(BASE64, + encoded="MTIzNDU2Nzg5MA==")); + } + + sub vcl_miss { + if (req.url == "/3") { + vtc.workspace_alloc(client, -65); + set req.http.Encode + = blob.encode(BASE64, blob=req.hash); + } + return( synth(200) ); + } +} + +client c1 -run +client c2 -run +client c3 -run + +# URL encoding +varnish v1 -vcl { + import blob; + import vtc; + backend b { .host = "${bad_ip}"; } + + sub vcl_recv { + if (req.url == "/1") { + vtc.workspace_alloc(client, -33); + } + elsif (req.url == "/2") { + vtc.workspace_alloc(client, -50); + } + set req.http.Decode + = blob.encode(blob=blob.decode(URL, + encoded="1234567890")); + } + + sub vcl_miss { + if (req.url == "/3") { + vtc.workspace_alloc(client, -65); + set req.http.Encode = blob.encode(URL, blob=req.hash); + } + return( synth(200) ); + } +} + +client c1 -run +client c2 -run +client c3 -run + +# HEX encoding +varnish v1 -vcl { + import blob; + import vtc; + backend b { .host = "${bad_ip}"; } + + sub vcl_recv { + if (req.url == "/1") { + vtc.workspace_alloc(client, -33); + } + elsif (req.url == "/2") { + vtc.workspace_alloc(client, -50); + } + set req.http.Decode + = blob.encode(blob=blob.decode(HEX, + encoded="31323334353637383930")); + } + + sub vcl_miss { + if (req.url == "/3") { + vtc.workspace_alloc(client, -65); + set req.http.Encode = blob.encode(HEX, blob=req.hash); + } + return( synth(200) ); + } +} + +client c1 -run +client c2 -run +client c3 -run From geoff at uplex.de Wed Jul 18 12:47:07 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 18 Jul 2018 12:47:07 +0000 (UTC) Subject: [master] a468ee22d Add an out-of-workspace test for VMOD blob's sub() function. Message-ID: <20180718124707.563F095C6B@lists.varnish-cache.org> commit a468ee22deedbaa24c88e46864af638331c734e8 Author: Geoff Simmons Date: Wed Jul 18 14:45:50 2018 +0200 Add an out-of-workspace test for VMOD blob's sub() function. diff --git a/bin/varnishtest/tests/m00049.vtc b/bin/varnishtest/tests/m00049.vtc index 6850ffc26..17ea49e84 100644 --- a/bin/varnishtest/tests/m00049.vtc +++ b/bin/varnishtest/tests/m00049.vtc @@ -161,3 +161,26 @@ varnish v1 -vcl { client c1 -run client c2 -run client c3 -run + +# sub() function +varnish v1 -vcl { + import blob; + import vtc; + backend b { .host = "${bad_ip}"; } + + sub vcl_miss { + if (req.url == "/1") { + # Not enough for req.hash + vmod_priv + vtc.workspace_alloc(client, -65); + } + elsif (req.url == "/2") { + # Not enough for req.hash + vmod_priv + 30 bytes + vtc.workspace_alloc(client, -90); + } + set req.http.Encode = blob.encode(blob=blob.sub(req.hash, 30B)); + return( synth(200) ); + } +} + +client c1 -run +client c2 -run From geoff at uplex.de Wed Jul 18 14:19:07 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 18 Jul 2018 14:19:07 +0000 (UTC) Subject: [master] 651a249f4 Retire the WB mini-interface used by VMOD blob. Message-ID: <20180718141907.995A29784C@lists.varnish-cache.org> commit 651a249f4a78fe433bda6b0dc3883b370dc73242 Author: Geoff Simmons Date: Wed Jul 18 16:16:21 2018 +0200 Retire the WB mini-interface used by VMOD blob. This was a proof of concept for an alternative WS interface -- write incrementally to reserved workspace as an "open buffer". That may be generally useful for Varnish, but for now we return to the usual WS_* idiom. diff --git a/lib/libvmod_blob/Makefile.am b/lib/libvmod_blob/Makefile.am index 2695913a2..fafce9acd 100644 --- a/lib/libvmod_blob/Makefile.am +++ b/lib/libvmod_blob/Makefile.am @@ -9,8 +9,6 @@ libvmod_blob_la_SOURCES = \ hex.h \ hex.c \ url.c \ - wb.h \ - wb.c \ tbl_encodings.h \ tbl_case.h diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c index a53d8c415..87591fb52 100644 --- a/lib/libvmod_blob/vmod_blob.c +++ b/lib/libvmod_blob/vmod_blob.c @@ -34,7 +34,6 @@ #include "vcc_if.h" #include "vmod_blob.h" -#include "wb.h" struct vmod_blob_blob { unsigned magic; @@ -357,11 +356,11 @@ vmod_decode(VRT_CTX, VCL_ENUM decs, VCL_INT length, const char *p, ...) { enum encoding dec = parse_encoding(decs); va_list ap; - struct wb_s wb; struct vmod_priv *b; char *buf; uintptr_t snap; ssize_t len; + unsigned space; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); AENC(dec); @@ -373,33 +372,28 @@ vmod_decode(VRT_CTX, VCL_ENUM decs, VCL_INT length, const char *p, ...) return NULL; } - if (wb_create(ctx->ws, &wb) == NULL) { - WS_Reset(ctx->ws, snap); - ERRNOMEM(ctx, "cannot decode"); - return NULL; - } - buf = wb_buf(&wb); + buf = WS_Front(ctx->ws); + space = WS_Reserve(ctx->ws, 0); if (length <= 0) length = -1; va_start(ap, p); errno = 0; - len = func[dec].decode(dec, buf, wb_space(&wb), length, p, ap); + len = func[dec].decode(dec, buf, space, length, p, ap); va_end(ap); if (len == -1) { err_decode(ctx, p); - wb_reset(&wb); + WS_Release(ctx->ws, 0); WS_Reset(ctx->ws, snap); return NULL; } if (len == 0) { - wb_reset(&wb); + WS_Release(ctx->ws, 0); WS_Reset(ctx->ws, snap); return null_blob; } - wb_advance(&wb, len); - WS_ReleaseP(ctx->ws, wb_buf(&wb)); + WS_Release(ctx->ws, len); b->priv = buf; b->len = len; b->free = NULL; @@ -409,8 +403,10 @@ vmod_decode(VRT_CTX, VCL_ENUM decs, VCL_INT length, const char *p, ...) static VCL_STRING encode(VRT_CTX, enum encoding enc, enum case_e kase, VCL_BLOB b) { - struct wb_s wb; ssize_t len; + char *buf; + uintptr_t snap; + unsigned space; AENC(enc); @@ -418,25 +414,25 @@ encode(VRT_CTX, enum encoding enc, enum case_e kase, VCL_BLOB b) return NULL; CHECK_OBJ_NOTNULL(ctx->ws, WS_MAGIC); - if (wb_create(ctx->ws, &wb) == NULL) { - ERRNOMEM(ctx, "cannot encode"); - return NULL; - } + snap = WS_Snapshot(ctx->ws); + buf = WS_Front(ctx->ws); + space = WS_Reserve(ctx->ws, 0); - len = func[enc].encode(enc, kase, - wb_buf(&wb), wb_space(&wb), b->priv, b->len); + len = func[enc].encode(enc, kase, buf, space, b->priv, b->len); if (len == -1) { ERRNOMEM(ctx, "cannot encode"); - wb_reset(&wb); + WS_Release(ctx->ws, 0); + WS_Reset(ctx->ws, snap); return NULL; } if (len == 0) { - wb_reset(&wb); + WS_Release(ctx->ws, 0); + WS_Reset(ctx->ws, snap); return ""; } - wb_advance(&wb, len); - return wb_finish(&wb, NULL); + WS_Release(ctx->ws, len + 1); + return buf; } VCL_STRING v_matchproto_(td_blob_encode) diff --git a/lib/libvmod_blob/wb.c b/lib/libvmod_blob/wb.c deleted file mode 100644 index 1b54b3e72..000000000 --- a/lib/libvmod_blob/wb.c +++ /dev/null @@ -1,78 +0,0 @@ -/*- - * Copyright 2015 UPLEX - Nils Goroll Systemoptimierung - * All rights reserved. - * - * Authors: Nils Goroll - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * write buffer: utility functions to append-write on a varnish workspace - */ - -#include "config.h" - -#include - -#include "cache/cache.h" -#include "wb.h" - -char * -wb_create(struct ws *ws, struct wb_s *wb) -{ - if (WS_Reserve(ws, 0) == 0) { - WS_Release(ws, 0); - wb->w = NULL; - wb->ws = NULL; - return NULL; - } - wb->w = ws->f; - wb->ws = ws; - - return wb->w; -} - -void -wb_reset(struct wb_s *wb) -{ - WS_Release(wb->ws, 0); - memset(wb, 0, sizeof(*wb)); -} - -/* - * release varnish workspace - * - * return start of buffer - */ -char * -wb_finish(struct wb_s *wb, ssize_t *l) -{ - char *r = wb->ws->f; - assert(wb->ws->r - wb->w > 0); - if (l) - *l = wb_len(wb); - - *wb->w = '\0'; - wb->w++; - - /* amount of space used */ - WS_ReleaseP(wb->ws, wb->w); - - return r; -} diff --git a/lib/libvmod_blob/wb.h b/lib/libvmod_blob/wb.h deleted file mode 100644 index e2c820d50..000000000 --- a/lib/libvmod_blob/wb.h +++ /dev/null @@ -1,68 +0,0 @@ -/*- - * Copyright 2015-2016 UPLEX - Nils Goroll Systemoptimierung - * All rights reserved. - * - * Author: Nils Goroll - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -struct wb_s { - struct ws *ws; // varnish workspace - char *w; // current write position -}; - -/* return one byte less for the final zero byte */ -static inline const char* -wb_end(const struct wb_s *wb) { - return wb->ws->r - 1; -} - -/* return the write position */ -static inline char* -wb_buf(const struct wb_s *wb) { - return wb->w; -} - -/* return one byte less for the final zero byte */ -static inline ssize_t -wb_space(const struct wb_s *wb) { - ssize_t f = wb->ws->r - wb->w; - assert(f > 0); - return f - 1; -} - -static inline ssize_t -wb_len(const struct wb_s *wb) { - ssize_t l = wb->w - wb->ws->f; - assert(l >= 0); - return l; -} - -static inline void -wb_advance(struct wb_s *wb, ssize_t l) { - wb->w += l; // final byte - assert(wb->w < wb_end(wb)); -} - -char *wb_create(struct ws *ws, struct wb_s *wb); -void wb_reset(struct wb_s *wb); -char *wb_finish(struct wb_s *wb, ssize_t *l); From geoff at uplex.de Wed Jul 18 14:35:07 2018 From: geoff at uplex.de (Geoff Simmons) Date: Wed, 18 Jul 2018 14:35:07 +0000 (UTC) Subject: [master] cdf9a180d VMOD blob encoder: don't forget the terminating null byte Message-ID: <20180718143507.B33AA97E81@lists.varnish-cache.org> commit cdf9a180d4acb2dd577d12781a120076100af025 Author: Geoff Simmons Date: Wed Jul 18 16:33:39 2018 +0200 VMOD blob encoder: don't forget the terminating null byte diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c index 87591fb52..632580e91 100644 --- a/lib/libvmod_blob/vmod_blob.c +++ b/lib/libvmod_blob/vmod_blob.c @@ -431,6 +431,7 @@ encode(VRT_CTX, enum encoding enc, enum case_e kase, VCL_BLOB b) WS_Reset(ctx->ws, snap); return ""; } + buf[len] = '\0'; WS_Release(ctx->ws, len + 1); return buf; } From fgsch at users.noreply.github.com Fri Jul 20 16:24:06 2018 From: fgsch at users.noreply.github.com (Federico G. Schwindt) Date: Fri, 20 Jul 2018 16:24:06 +0000 (UTC) Subject: [master] 3d5e0651f minor typo in upgrading-6.0.rst Message-ID: <20180720162406.721FF40D1@lists.varnish-cache.org> commit 3d5e0651f312ff680c752f05a639144a18d5468a Author: Valentin Matei Date: Fri Jul 20 18:30:26 2018 +0300 minor typo in upgrading-6.0.rst diff --git a/doc/sphinx/whats-new/upgrading-6.0.rst b/doc/sphinx/whats-new/upgrading-6.0.rst index eaea169a3..2006a97b1 100644 --- a/doc/sphinx/whats-new/upgrading-6.0.rst +++ b/doc/sphinx/whats-new/upgrading-6.0.rst @@ -584,7 +584,7 @@ community support for those platforms. Services ~~~~~~~~ -As a result we ended up with ended up with systemd-only platforms for +As a result we ended up with systemd-only platforms for the official packages. The old services are still available as we archived them in the ``pkg-varnish-cache`` source tree. This was the occasion to remove differences between Red Hat and Debian derivatives From gquintard at users.noreply.github.com Mon Jul 23 20:01:10 2018 From: gquintard at users.noreply.github.com (guillaume quintard) Date: Mon, 23 Jul 2018 20:01:10 +0000 (UTC) Subject: [master] ef8ffbdd9 Accommodate musl libc that has a smaller stack Message-ID: <20180723200110.4B11A64223@lists.varnish-cache.org> commit ef8ffbdd9a2c8091b80e4950b6e74e23fcd30bc6 Author: Guillaume Quintard Date: Tue Jul 17 13:09:29 2018 -0700 Accommodate musl libc that has a smaller stack diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c index 9e2e2a31e..c48b72396 100644 --- a/bin/varnishd/waiter/cache_waiter_epoll.c +++ b/bin/varnishd/waiter/cache_waiter_epoll.c @@ -33,6 +33,8 @@ //lint -e{766} #include "config.h" +#include + #if defined(HAVE_EPOLL_CTL) #include @@ -69,7 +71,7 @@ struct vwe { static void * vwe_thread(void *priv) { - struct epoll_event ev[NEEV], *ep; + struct epoll_event *ev, *ep; struct waited *wp; struct waiter *w; double now, then; @@ -82,6 +84,8 @@ vwe_thread(void *priv) CHECK_OBJ_NOTNULL(w, WAITER_MAGIC); THR_SetName("cache-epoll"); THR_Init(); + ev = malloc(sizeof(struct epoll_event) * NEEV); + AN(ev); now = VTIM_real(); while (1) { @@ -146,6 +150,7 @@ vwe_thread(void *priv) if (vwe->nwaited == 0 && vwe->die) break; } + free(ev); closefd(&vwe->pipe[0]); closefd(&vwe->pipe[1]); closefd(&vwe->epfd); From nils.goroll at uplex.de Sat Jul 28 12:24:08 2018 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 28 Jul 2018 12:24:08 +0000 (UTC) Subject: [master] ead72749b Test adding (dynamic) backends with probe to already warm vcl Message-ID: <20180728122409.0D27578EE@lists.varnish-cache.org> commit ead72749b78d99f941649e879deadbb683891d10 Author: Nils Goroll Date: Thu Jul 12 12:53:58 2018 +0200 Test adding (dynamic) backends with probe to already warm vcl this tests d912ffe4f7a5e643275699c94b2722d486fd9e7f diff --git a/bin/varnishtest/tests/d00007.vtc b/bin/varnishtest/tests/d00007.vtc index eac4a4d17..93c0a0c7f 100644 --- a/bin/varnishtest/tests/d00007.vtc +++ b/bin/varnishtest/tests/d00007.vtc @@ -3,6 +3,10 @@ varnishtest "Test dynamic backends" server s1 { rxreq txresp + close + accept + rxreq + txresp } -start varnish v1 -vcl { @@ -10,11 +14,14 @@ varnish v1 -vcl { backend dummy { .host = "${bad_backend}"; } + probe pr {} + sub vcl_init { - new s1 = debug.dyn("${s1_addr}", "${s1_port}"); + new s1 = debug.dyn("0.0.0.0", "0"); } sub vcl_recv { + s1.refresh("${s1_addr}", "${s1_port}", pr); set req.backend_hint = s1.backend(); } } -start diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index d6094bcae..3e6b06108 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -129,7 +129,7 @@ $Function VOID fail() Function to fail vcl code. (See also: RFC748) -$Object dyn(STRING addr, STRING port) +$Object dyn(STRING addr, STRING port, PROBE probe=0) Dynamically create a single-backend director, addr and port must not be empty. @@ -137,7 +137,7 @@ $Method BACKEND .backend() Return the dynamic backend. -$Method VOID .refresh(STRING addr, STRING port) +$Method VOID .refresh(STRING addr, STRING port, PROBE probe=0) Dynamically refresh & (always!) replace the backend by a new one. diff --git a/lib/libvmod_debug/vmod_debug_dyn.c b/lib/libvmod_debug/vmod_debug_dyn.c index ba7c974ca..cd0b7b6a3 100644 --- a/lib/libvmod_debug/vmod_debug_dyn.c +++ b/lib/libvmod_debug/vmod_debug_dyn.c @@ -60,7 +60,7 @@ struct xyzzy_debug_dyn_uds { static void dyn_dir_init(VRT_CTX, struct xyzzy_debug_dyn *dyn, - VCL_STRING addr, VCL_STRING port) + VCL_STRING addr, VCL_STRING port, VCL_PROBE probe) { struct addrinfo hints, *res = NULL; struct suckaddr *sa; @@ -75,6 +75,7 @@ dyn_dir_init(VRT_CTX, struct xyzzy_debug_dyn *dyn, vrt.port = port; vrt.vcl_name = dyn->vcl_name; vrt.hosthdr = addr; + vrt.probe = probe; memset(&hints, 0, sizeof(hints)); hints.ai_family = AF_UNSPEC; @@ -116,7 +117,7 @@ dyn_dir_init(VRT_CTX, struct xyzzy_debug_dyn *dyn, VCL_VOID xyzzy_dyn__init(VRT_CTX, struct xyzzy_debug_dyn **dynp, - const char *vcl_name, VCL_STRING addr, VCL_STRING port) + const char *vcl_name, VCL_STRING addr, VCL_STRING port, VCL_PROBE probe) { struct xyzzy_debug_dyn *dyn; @@ -139,7 +140,7 @@ xyzzy_dyn__init(VRT_CTX, struct xyzzy_debug_dyn **dynp, AZ(pthread_mutex_init(&dyn->mtx, NULL)); - dyn_dir_init(ctx, dyn, addr, port); + dyn_dir_init(ctx, dyn, addr, port, probe); XXXAN(dyn->dir); *dynp = dyn; } @@ -172,11 +173,11 @@ xyzzy_dyn_backend(VRT_CTX, struct xyzzy_debug_dyn *dyn) VCL_VOID xyzzy_dyn_refresh(VRT_CTX, struct xyzzy_debug_dyn *dyn, - VCL_STRING addr, VCL_STRING port) + VCL_STRING addr, VCL_STRING port, VCL_PROBE probe) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(dyn, VMOD_DEBUG_DYN_MAGIC); - dyn_dir_init(ctx, dyn, addr, port); + dyn_dir_init(ctx, dyn, addr, port, probe); } static int From dridi at varni.sh Sat Jul 28 18:00:55 2018 From: dridi at varni.sh (Dridi Boukelmoune) Date: Sat, 28 Jul 2018 20:00:55 +0200 Subject: [master] ead72749b Test adding (dynamic) backends with probe to already warm vcl In-Reply-To: <20180728122409.0D27578EE@lists.varnish-cache.org> References: <20180728122409.0D27578EE@lists.varnish-cache.org> Message-ID: On Sat, Jul 28, 2018 at 2:24 PM, Nils Goroll wrote: > > commit ead72749b78d99f941649e879deadbb683891d10 > Author: Nils Goroll > Date: Thu Jul 12 12:53:58 2018 +0200 > > Test adding (dynamic) backends with probe to already warm vcl > > this tests d912ffe4f7a5e643275699c94b2722d486fd9e7f Thanks a lot! From noreply at github.com Tue Jul 31 10:06:03 2018 From: noreply at github.com (GitHub) Date: Tue, 31 Jul 2018 10:06:03 +0000 (UTC) Subject: [master] 02b73e11c Update doc Message-ID: <20180731100604.356876310C@lists.varnish-cache.org> commit 02b73e11c2740f96e778bebe8de73a67b2939bc2 Author: Valentin Matei Date: Tue Jul 31 12:47:45 2018 +0300 Update doc client content -> client context diff --git a/lib/libvmod_directors/vmod.vcc b/lib/libvmod_directors/vmod.vcc index cc5f9aed2..8fecc08e7 100644 --- a/lib/libvmod_directors/vmod.vcc +++ b/lib/libvmod_directors/vmod.vcc @@ -438,7 +438,7 @@ is _not_ the order given when backends are added. * when called in backend context: Use the varnish hash value as set by `vcl_hash` - * when called in client content: hash `req.url` + * when called in client context: hash `req.url` * `URL`: hash req.url / bereq.url