From martin at varnish-software.com Thu May 2 12:45:10 2019 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 2 May 2019 12:45:10 +0000 (UTC) Subject: [master] cc06c2278 Revert "Fix the order of HSH_Fail() and ObjSetState()" Message-ID: <20190502124510.8419C9C73D@lists.varnish-cache.org> commit cc06c2278f186bd4bc4c1aea80a8e31a7e2cc436 Author: Martin Blix Grydeland Date: Fri Apr 26 15:32:10 2019 +0200 Revert "Fix the order of HSH_Fail() and ObjSetState()" This reverts commit 501246e939430dd60d44e8e3af40d6978f11186d. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 5535e00b7..63738e1fe 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -809,6 +809,7 @@ vbf_stp_fail(struct worker *wrk, const struct busyobj *bo) assert(bo->fetch_objcore->boc->state < BOS_FINISHED); ObjSetState(wrk, bo->fetch_objcore, BOS_FAILED); + HSH_Fail(bo->fetch_objcore); if (!(bo->fetch_objcore->flags & OC_F_BUSY)) HSH_Kill(bo->fetch_objcore); return (F_STP_DONE); diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index d723975da..89bb0497d 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -84,7 +84,6 @@ #include #include "cache_varnishd.h" -#include "cache_objhead.h" #include "cache_obj.h" #include "vend.h" #include "storage/storage.h" @@ -258,7 +257,8 @@ ObjWaitExtend(const struct worker *wrk, const struct objcore *oc, uint64_t l) */ void -ObjSetState(struct worker *wrk, struct objcore *oc, enum boc_state_e next) +ObjSetState(struct worker *wrk, const struct objcore *oc, + enum boc_state_e next) { const struct obj_methods *om; @@ -275,13 +275,6 @@ ObjSetState(struct worker *wrk, struct objcore *oc, enum boc_state_e next) om->objsetstate(wrk, oc, next); } - if (next == BOS_FAILED) { - /* Signal to cache_hash.c that this object is failed. This - * needs to happen before we signal the boc so that code - * will see the OC_F_FAILED flag after ObjWaitState() */ - HSH_Fail(oc); - } - Lck_Lock(&oc->boc->mtx); oc->boc->state = next; AZ(pthread_cond_broadcast(&oc->boc->cond)); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index fab80d6ba..b3f766080 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -293,7 +293,8 @@ int ObjGetSpace(struct worker *, struct objcore *, ssize_t *sz, uint8_t **ptr); void ObjExtend(struct worker *, struct objcore *, ssize_t l); uint64_t ObjWaitExtend(const struct worker *, const struct objcore *, uint64_t l); -void ObjSetState(struct worker *, struct objcore *, enum boc_state_e next); +void ObjSetState(struct worker *, const struct objcore *, + enum boc_state_e next); void ObjWaitState(const struct objcore *, enum boc_state_e want); void ObjTrimStore(struct worker *, struct objcore *); void ObjTouch(struct worker *, struct objcore *, vtim_real now); From martin at varnish-software.com Thu May 2 12:45:10 2019 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 2 May 2019 12:45:10 +0000 (UTC) Subject: [master] 7046979e6 Revert "Resolve race on waitinglist rushing between OC_F_BUSY and boc->state" Message-ID: <20190502124510.B09149C73F@lists.varnish-cache.org> commit 7046979e6e29461b075a7d3b35c51d1b06f98dc4 Author: Martin Blix Grydeland Date: Fri Apr 26 15:32:16 2019 +0200 Revert "Resolve race on waitinglist rushing between OC_F_BUSY and boc->state" This reverts commit 4130055c457903e7f904fa56c0db58d7e7467dc1. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 63738e1fe..b3304bedb 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -562,8 +562,9 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) assert(bo->fetch_objcore->boc->state == BOS_REQ_DONE); if (bo->do_stream) { - ObjSetState(wrk, bo->fetch_objcore, BOS_STREAM); + ObjSetState(wrk, bo->fetch_objcore, BOS_PREP_STREAM); HSH_Unbusy(wrk, bo->fetch_objcore); + ObjSetState(wrk, bo->fetch_objcore, BOS_STREAM); } VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s", @@ -588,20 +589,18 @@ vbf_stp_fetchend(struct worker *wrk, struct busyobj *bo) AZ(ObjSetU64(wrk, bo->fetch_objcore, OA_LEN, bo->fetch_objcore->boc->len_so_far)); - /* Recycle the backend connection before setting BOS_FINISHED to - give predictable backend reuse behavior for varnishtest */ - VDI_Finish(bo); - if (bo->do_stream) assert(bo->fetch_objcore->boc->state == BOS_STREAM); - else + else { assert(bo->fetch_objcore->boc->state == BOS_REQ_DONE); - ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED); - - if (!bo->do_stream) HSH_Unbusy(wrk, bo->fetch_objcore); - AZ(bo->fetch_objcore->flags & OC_F_BUSY); + } + /* Recycle the backend connection before setting BOS_FINISHED to + give predictable backend reuse behavior for varnishtest */ + VDI_Finish(bo); + + ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED); VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk)); if (bo->stale_oc != NULL) HSH_Kill(bo->stale_oc); @@ -655,8 +654,9 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_GZIPBITS)); if (bo->do_stream) { - ObjSetState(wrk, bo->fetch_objcore, BOS_STREAM); + ObjSetState(wrk, bo->fetch_objcore, BOS_PREP_STREAM); HSH_Unbusy(wrk, bo->fetch_objcore); + ObjSetState(wrk, bo->fetch_objcore, BOS_STREAM); } if (ObjIterate(wrk, bo->stale_oc, bo, vbf_objiterator, 0)) @@ -790,10 +790,10 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) } AZ(ObjSetU64(wrk, bo->fetch_objcore, OA_LEN, o)); VSB_destroy(&synth_body); - ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED); HSH_Unbusy(wrk, bo->fetch_objcore); if (stale != NULL && bo->fetch_objcore->ttl > 0) HSH_Kill(stale); + ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED); return (F_STP_DONE); } @@ -808,10 +808,10 @@ vbf_stp_fail(struct worker *wrk, const struct busyobj *bo) CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC); assert(bo->fetch_objcore->boc->state < BOS_FINISHED); - ObjSetState(wrk, bo->fetch_objcore, BOS_FAILED); HSH_Fail(bo->fetch_objcore); if (!(bo->fetch_objcore->flags & OC_F_BUSY)) HSH_Kill(bo->fetch_objcore); + ObjSetState(wrk, bo->fetch_objcore, BOS_FAILED); return (F_STP_DONE); } @@ -982,6 +982,8 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, ObjWaitState(oc, BOS_STREAM); if (oc->boc->state == BOS_FAILED) { AN((oc->flags & OC_F_FAILED)); + } else { + AZ(oc->flags & OC_F_BUSY); } } } diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 66167df33..ead55932c 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -420,11 +420,11 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp) continue; CHECK_OBJ_ORNULL(oc->boc, BOC_MAGIC); - if (oc->flags & OC_F_BUSY) { + if (oc->boc != NULL && oc->boc->state < BOS_STREAM) { if (req->hash_ignore_busy) continue; - if (oc->boc != NULL && oc->boc->vary != NULL && + if (oc->boc->vary != NULL && !VRY_Match(req, oc->boc->vary)) continue; diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index 89bb0497d..b533eefcf 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -267,6 +267,7 @@ ObjSetState(struct worker *wrk, const struct objcore *oc, assert(next > oc->boc->state); CHECK_OBJ_ORNULL(oc->stobj->stevedore, STEVEDORE_MAGIC); + assert(next != BOS_STREAM || oc->boc->state == BOS_PREP_STREAM); assert(next != BOS_FINISHED || (oc->oa_present & (1 << OA_LEN))); if (oc->stobj->stevedore != NULL) { diff --git a/include/tbl/boc_state.h b/include/tbl/boc_state.h index 751456ca6..a5c4a61ff 100644 --- a/include/tbl/boc_state.h +++ b/include/tbl/boc_state.h @@ -30,6 +30,7 @@ BOC_STATE(INVALID, invalid) /* don't touch (yet) */ BOC_STATE(REQ_DONE, req_done) /* bereq.* can be examined */ +BOC_STATE(PREP_STREAM, prep_stream) /* Prepare for streaming */ BOC_STATE(STREAM, stream) /* beresp.* can be examined */ BOC_STATE(FINISHED, finished) /* object is complete */ BOC_STATE(FAILED, failed) /* something went wrong */ From martin at varnish-software.com Thu May 2 12:45:10 2019 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Thu, 2 May 2019 12:45:10 +0000 (UTC) Subject: [master] a280f82b1 Resolve race on waitinglist rushing between HSH_Unbusy and boc->state Message-ID: <20190502124510.D55159C743@lists.varnish-cache.org> commit a280f82b18d8b736339370204f39141f31813d13 Author: Martin Blix Grydeland Date: Thu May 2 13:21:54 2019 +0200 Resolve race on waitinglist rushing between HSH_Unbusy and boc->state This is the second attempt at a fix for this, the previous one was reverted. When an object is ready for delivery, HSH_Unbusy was called before calling ObjSetState([BOS_STREAM|BOS_FINISHED]). The HSH_Unbusy() call does the waitinglist rushing, but HSH_Lookup() wanted to look at the boc->state and if BOS_STREAM had been reached. This could cause requests woken to find that the stream state still hadn't been reached (ObjSetState still hadn't executed), and go back on the waitinglist. To fix this, this patch reverts commit 0375791cac1f2333ab54932ba1d2025261082fab, and goes back to considering OC_F_BUSY as the gate keeper for HSH_Lookup. This eliminates the race, because HSH_Unbusy and HSH_Lookup then uses the same mutex. That change opens up the possiblity that req code after HSH_Lookup() sees an object that has not yet reached BOS_STREAM. To counter this, once a boc reference is found during delivery, an ObjWaitState(BOS_STREAM) is done before continuing. Lastly, an ObjSetState(BOS_PREP_STREAM) is added just before HSH_Unbusy() in vbf_stp_error() and vbf_stp_fetchend() for the case where do_stream==false. This makes the order of events consistent throughout. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index b3304bedb..eb25de6e9 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -593,6 +593,7 @@ vbf_stp_fetchend(struct worker *wrk, struct busyobj *bo) assert(bo->fetch_objcore->boc->state == BOS_STREAM); else { assert(bo->fetch_objcore->boc->state == BOS_REQ_DONE); + ObjSetState(wrk, bo->fetch_objcore, BOS_PREP_STREAM); HSH_Unbusy(wrk, bo->fetch_objcore); } @@ -790,6 +791,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) } AZ(ObjSetU64(wrk, bo->fetch_objcore, OA_LEN, o)); VSB_destroy(&synth_body); + ObjSetState(wrk, bo->fetch_objcore, BOS_PREP_STREAM); HSH_Unbusy(wrk, bo->fetch_objcore); if (stale != NULL && bo->fetch_objcore->ttl > 0) HSH_Kill(stale); diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index ead55932c..8796e2adf 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -420,11 +420,11 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp) continue; CHECK_OBJ_ORNULL(oc->boc, BOC_MAGIC); - if (oc->boc != NULL && oc->boc->state < BOS_STREAM) { + if (oc->flags & OC_F_BUSY) { if (req->hash_ignore_busy) continue; - if (oc->boc->vary != NULL && + if (oc->boc && oc->boc->vary != NULL && !VRY_Match(req, oc->boc->vary)) continue; diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 5a7ef97e3..d4b343b46 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -353,6 +353,8 @@ cnt_transmit(struct worker *wrk, struct req *req) /* Grab a ref to the bo if there is one (=streaming) */ boc = HSH_RefBoc(req->objcore); + if (boc && boc->state < BOS_STREAM) + ObjWaitState(req->objcore, BOS_STREAM); clval = http_GetContentLength(req->resp); /* RFC 7230, 3.3.3 */ status = http_GetStatus(req->resp); From phk at FreeBSD.org Fri May 3 06:48:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 3 May 2019 06:48:07 +0000 (UTC) Subject: [master] e407f7c10 Revert "Don't panic if you try to relabel a VCL with the same label." Message-ID: <20190503064807.E887D60F80@lists.varnish-cache.org> commit e407f7c10cbc2be786e0e2d890e2548d3faf1886 Author: Nils Goroll Date: Mon Feb 18 14:23:19 2019 +0100 Revert "Don't panic if you try to relabel a VCL with the same label." This reverts commit 4709fae3a703b5904a799b8aa1df2872e803219f. diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 161063221..6e1bc5639 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -923,13 +923,6 @@ mcf_vcl_label(struct cli *cli, const char * const *av, void *priv) VCLI_Out(cli, "%s is not a label", vpl->name); return; } - if (!VTAILQ_EMPTY(&vpl->dfrom) && - VTAILQ_FIRST(&vpl->dfrom)->to == vpt) { - VCLI_SetResult(cli, CLIS_PARAM); - VCLI_Out(cli, "VCL '%s' already has label '%s'", - vpt->name, vpl->name); - return; - } if (!VTAILQ_EMPTY(&vpt->dfrom) && !VTAILQ_EMPTY(&vpl->dto)) { VCLI_SetResult(cli, CLIS_PARAM); diff --git a/bin/varnishtest/tests/c00077.vtc b/bin/varnishtest/tests/c00077.vtc index c32983893..49ed76797 100644 --- a/bin/varnishtest/tests/c00077.vtc +++ b/bin/varnishtest/tests/c00077.vtc @@ -13,10 +13,6 @@ varnish v1 -vcl+backend { varnish v1 -clierr 106 "vcl.label vcl.A vcl1" varnish v1 -cliok "vcl.label vclA vcl1" -varnish v1 -clierr 106 "vcl.label vclA vcl1" -varnish v1 -cliexpect {VCL 'vcl1' already has label 'vclA'} { - vcl.label vclA vcl1 -} varnish v1 -vcl+backend { sub vcl_recv { From phk at FreeBSD.org Fri May 3 06:48:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 3 May 2019 06:48:08 +0000 (UTC) Subject: [master] 424c7b85d change semantics of the vcl 'auto' state and centralize vcl mgt Message-ID: <20190503064808.0FDA460F84@lists.varnish-cache.org> commit 424c7b85dd7c95fa5dba804472d8a64f2adb9961 Author: Nils Goroll Date: Sat Nov 17 19:19:48 2018 +0100 change semantics of the vcl 'auto' state and centralize vcl mgt Conceptually, the auto state was a variant of the warm state which would automatically cool the vcl. Yet, cooling did not only transition the temperature, but also the state, so 'auto' only worked one way - except that vcl.use or moving a label (by labeling another vcl) would also set 'auto', so a manual warm/cold setting would get lost. Now the auto-state will remain no matter the actual temperature or labeling, so when a vcl needs to implicitly change temperature (due to being used or being labeled), an auto vcl will remain auto, and a cold/warm vcl will change state, but never become 'auto' implicitly. The vcl state/temperature test v00003.vtc, besides testing the new auto semantics, now also checks for the right vcl.list output and has been reduced by a duplicate check (warm event check has been integrated into an existing warm event). On other code changes: * mgt_vcl_setstate is now only concerned with the state, the temperature will be changed implicitly if so required. The state will either end up changed or restored, depending on success. owner of changes to the (struct vclprog).state member * mgt_vcl_settemp responsible for the right action to change the temperature. For auto, it will only change the temperature, for non-auto, also the state. owner of changes to the (struct vclprog).warm member * mgt_vcl_tellchild Inform the child about a change and/or temperature change * mgt_vcl_set_cooldown Update the cooldown (go_cold) appropriately, should be called after a change/temperature change. Fixes #2834 Closes #2801 diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 6e1bc5639..f54a9beff 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -105,6 +105,9 @@ static struct vclprog *active_vcl; static struct vev *e_poker; static int mgt_vcl_setstate(struct cli *, struct vclprog *, const char *); +static int mgt_vcl_settemp(struct cli *, struct vclprog *, unsigned); +static int mgt_vcl_tellchild(struct cli *, struct vclprog *, unsigned); +static void mgt_vcl_set_cooldown(struct vclprog *, vtim_mono); /*--------------------------------------------------------------------*/ @@ -198,13 +201,19 @@ mgt_vcl_dep_add(struct vclprog *vp_from, struct vclprog *vp_to) CHECK_OBJ_NOTNULL(vp_from, VCLPROG_MAGIC); CHECK_OBJ_NOTNULL(vp_to, VCLPROG_MAGIC); ALLOC_OBJ(vd, VCLDEP_MAGIC); + + mgt_vcl_set_cooldown(vp_from, -1); + mgt_vcl_set_cooldown(vp_to, -1); + XXXAN(vd); vd->from = vp_from; VTAILQ_INSERT_TAIL(&vp_from->dfrom, vd, lfrom); vd->to = vp_to; VTAILQ_INSERT_TAIL(&vp_to->dto, vd, lto); vp_to->nto++; + assert(vp_to->state == VCL_STATE_WARM || /* vcl.label ... */ + vp_to->state == VCL_STATE_AUTO || /* vcl.label ... */ vp_to->state == VCL_STATE_LABEL); /* return(vcl(...)) */ } @@ -216,12 +225,8 @@ mgt_vcl_dep_del(struct vcldep *vd) VTAILQ_REMOVE(&vd->from->dfrom, vd, lfrom); VTAILQ_REMOVE(&vd->to->dto, vd, lto); vd->to->nto--; - if (vd->to->nto == 0 && vd->to->state == VCL_STATE_WARM) { - vd->to->state = VCL_STATE_AUTO; - AZ(vd->to->go_cold); - (void)mgt_vcl_setstate(NULL, vd->to, VCL_STATE_AUTO); - AN(vd->to->go_cold); - } + if (vd->to->nto == 0) + mgt_vcl_set_cooldown(vd->to, VTIM_mono()); FREE_OBJ(vd); } @@ -398,50 +403,79 @@ mgt_has_vcl(void) return (!VTAILQ_EMPTY(&vclhead)); } +/* + * go_cold + * + * -1: leave alone + * 0: timer not started - not currently used + * >0: when timer started + */ +static void +mgt_vcl_set_cooldown(struct vclprog *vp, vtim_mono now) +{ + CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC); + + if (vp == active_vcl || + vp->state != VCL_STATE_AUTO || + vp->warm == 0 || + !VTAILQ_EMPTY(&vp->dto) || + !VTAILQ_EMPTY(&vp->dfrom)) + vp->go_cold = -1; + else + vp->go_cold = now; +} + static unsigned -mgt_vcl_cooldown(struct vclprog *vp) +mgt_vcl_cooldown(struct vclprog *vp, vtim_mono now) { - double now; + CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC); - if (vp->state != VCL_STATE_AUTO) + if (vp->go_cold < 0) return (0); - now = VTIM_mono(); - if (vp->go_cold > 0 && vp->go_cold + mgt_param.vcl_cooldown < now) - return (1); + if (vp->go_cold == 0) { + mgt_vcl_set_cooldown(vp, now); + return (0); + } - if (vp->go_cold == 0 && vp != active_vcl) - vp->go_cold = now; + assert(vp->go_cold > 0); + + if (vp->go_cold + mgt_param.vcl_cooldown < now) + return (1); return (0); } static int -mgt_vcl_setstate(struct cli *cli, struct vclprog *vp, const char *vs) +mgt_vcl_settemp(struct cli *cli, struct vclprog *vp, unsigned warm) { - unsigned status, warm; - char *p; int i; - assert(vs != VCL_STATE_LABEL); + CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC); - if (vp == active_vcl || mcf_is_label(vp)) { - AN(vp->warm); - /* Only the poker sends COLD indiscriminately, ignore it */ - if (vs == VCL_STATE_COLD) - AZ(cli); + if (warm == vp->warm) return (0); + + if (vp->state == VCL_STATE_AUTO || vp->state == VCL_STATE_LABEL) { + mgt_vcl_set_cooldown(vp, -1); + i = mgt_vcl_tellchild(cli, vp, warm); + mgt_vcl_set_cooldown(vp, VTIM_mono()); + } else { + i = mgt_vcl_setstate(cli, vp, + warm ? VCL_STATE_WARM : VCL_STATE_COLD); } - if (vs == VCL_STATE_AUTO) - vs = (mgt_vcl_cooldown(vp) ? VCL_STATE_COLD : VCL_STATE_WARM); - else - vp->go_cold = 0; + return (i); +} - warm = (vs == VCL_STATE_WARM ? 1 : 0); +static int +mgt_vcl_tellchild(struct cli *cli, struct vclprog *vp, unsigned warm) +{ + unsigned status; + char *p; + int i; - if (vp->warm == warm) - return (0); + CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC); if (!MCH_Running()) { vp->warm = warm; @@ -449,7 +483,7 @@ mgt_vcl_setstate(struct cli *cli, struct vclprog *vp, const char *vs) } i = mgt_cli_askchild(&status, &p, "vcl.state %s %d%s\n", - vp->name, warm, vs); + vp->name, warm, vp->state); if (i && cli != NULL) { VCLI_SetResult(cli, status); VCLI_Out(cli, "%s", p); @@ -468,6 +502,47 @@ mgt_vcl_setstate(struct cli *cli, struct vclprog *vp, const char *vs) return (i); } +static int +mgt_vcl_setstate(struct cli *cli, struct vclprog *vp, const char *vs) +{ + unsigned warm; + int i; + const char *os; + + CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC); + + assert(vs != VCL_STATE_LABEL); + + if (mcf_is_label(vp)) { + AN(vp->warm); + /* do not touch labels */ + return (0); + } + + if (vp->state == vs) + return (0); + + os = vp->state; + vp->state = vs; + + if (vp == active_vcl) { + assert (vs == VCL_STATE_WARM || vs == VCL_STATE_AUTO); + AN(vp->warm); + warm = 1; + } else if (vs == VCL_STATE_AUTO) { + warm = vp->warm; + } else { + warm = (vs == VCL_STATE_WARM ? 1 : 0); + } + + i = mgt_vcl_tellchild(cli, vp, warm); + if (i == 0) + mgt_vcl_set_cooldown(vp, VTIM_mono()); + else + vp->state = os; + return (i); +} + /*--------------------------------------------------------------------*/ static void @@ -537,8 +612,7 @@ mgt_new_vcl(struct cli *cli, const char *vclname, const char *vclsrc, } free(p); - if (vp->warm && vp->state == VCL_STATE_AUTO) - vp->go_cold = VTIM_mono(); + mgt_vcl_set_cooldown(vp, VTIM_mono()); } /*--------------------------------------------------------------------*/ @@ -556,8 +630,8 @@ mgt_vcl_startup(struct cli *cli, const char *vclsrc, const char *vclname, bprintf(buf, "boot%d", n++); vclname = buf; } + active_vcl = NULL; mgt_new_vcl(cli, vclname, vclsrc, origin, NULL, C_flag); - active_vcl = mcf_vcl_byname(vclname); } /*--------------------------------------------------------------------*/ @@ -584,7 +658,7 @@ mgt_push_vcls(struct cli *cli, unsigned *status, char **p) AN(active_vcl); /* The VCL has not been loaded yet, it cannot fail */ - AZ(mgt_vcl_setstate(cli, active_vcl, VCL_STATE_WARM)); + (void)cli; VTAILQ_FOREACH(vp, &vclhead, list) vp->loaded = 0; @@ -677,36 +751,21 @@ mcf_vcl_state(struct cli *cli, const char * const *av, void *priv) return; } - if (!VTAILQ_EMPTY(&vp->dto)) { - assert(vp->state != VCL_STATE_COLD); - if (state == VCL_STATE_COLD) { + if (state == VCL_STATE_COLD) { + if (!VTAILQ_EMPTY(&vp->dto)) { + assert(vp->state != VCL_STATE_COLD); VCLI_Out(cli, "A labeled VCL cannot be set cold"); VCLI_SetResult(cli, CLIS_CANT); return; } - } - - if (vp->state == state) - return; - - if (state == VCL_STATE_AUTO) { - vp->state = VCL_STATE_AUTO; - (void)mgt_vcl_setstate(cli, vp, VCL_STATE_AUTO); - } else if (state == VCL_STATE_COLD) { if (vp == active_vcl) { VCLI_Out(cli, "Cannot set the active VCL cold."); VCLI_SetResult(cli, CLIS_CANT); return; } - vp->state = VCL_STATE_AUTO; - (void)mgt_vcl_setstate(cli, vp, VCL_STATE_COLD); - } else if (state == VCL_STATE_WARM) { - if (mgt_vcl_setstate(cli, vp, VCL_STATE_WARM) == 0) - vp->state = VCL_STATE_WARM; - } else { - VCLI_Out(cli, "State must be one of auto, cold or warm."); - VCLI_SetResult(cli, CLIS_PARAM); } + + (void)mgt_vcl_setstate(cli, vp, state); } static void v_matchproto_(cli_func_t) @@ -715,6 +774,7 @@ mcf_vcl_use(struct cli *cli, const char * const *av, void *priv) unsigned status; char *p = NULL; struct vclprog *vp, *vp2; + vtim_mono now; (void)priv; vp = mcf_find_vcl(cli, av[2]); @@ -722,22 +782,22 @@ mcf_vcl_use(struct cli *cli, const char * const *av, void *priv) return; if (vp == active_vcl) return; - if (mgt_vcl_setstate(cli, vp, VCL_STATE_WARM)) + + if (mgt_vcl_settemp(cli, vp, 1)) return; + if (MCH_Running() && mgt_cli_askchild(&status, &p, "vcl.use %s\n", av[2])) { VCLI_SetResult(cli, status); VCLI_Out(cli, "%s", p); - AZ(vp->go_cold); - (void)mgt_vcl_setstate(cli, vp, VCL_STATE_AUTO); } else { VCLI_Out(cli, "VCL '%s' now active", av[2]); vp2 = active_vcl; active_vcl = vp; - if (vp2 != NULL) { - AZ(vp2->go_cold); - (void)mgt_vcl_setstate(cli, vp2, VCL_STATE_AUTO); - } + now = VTIM_mono(); + mgt_vcl_set_cooldown(vp, now); + if (vp2 != NULL) + mgt_vcl_set_cooldown(vp2, now); } free(p); } @@ -779,12 +839,14 @@ mcf_vcl_discard(struct cli *cli, const char * const *av, void *priv) } return; } - if (mcf_is_label(vp)) + if (mcf_is_label(vp)) { AN(vp->warm); - else + vp->warm = 0; + } else { (void)mgt_vcl_setstate(cli, vp, VCL_STATE_COLD); + } if (MCH_Running()) { - assert(vp->state != VCL_STATE_WARM); + AZ(vp->warm); if (mgt_cli_askchild(&status, &p, "vcl.discard %s\n", av[2])) assert(status == CLIS_OK || status == CLIS_COMMS); free(p); @@ -937,9 +999,8 @@ mcf_vcl_label(struct cli *cli, const char * const *av, void *priv) } } - if (mgt_vcl_setstate(cli, vpt, VCL_STATE_WARM)) + if (mgt_vcl_settemp(cli, vpt, 1)) return; - vpt->state = VCL_STATE_WARM; /* XXX: race with the poker? */ if (vpl != NULL) { mgt_vcl_dep_del(VTAILQ_FIRST(&vpl->dfrom)); @@ -949,7 +1010,8 @@ mcf_vcl_label(struct cli *cli, const char * const *av, void *priv) } AN(vpl); - vpl->warm = 1; + if (mgt_vcl_settemp(cli, vpl, 1)) + return; mgt_vcl_dep_add(vpl, vpt); if (!MCH_Running()) @@ -969,13 +1031,16 @@ static int v_matchproto_(vev_cb_f) mgt_vcl_poker(const struct vev *e, int what) { struct vclprog *vp; + vtim_mono now; (void)e; (void)what; e_poker->timeout = mgt_param.vcl_cooldown * .45; - VTAILQ_FOREACH(vp, &vclhead, list) - if (mgt_vcl_cooldown(vp)) - (void)mgt_vcl_setstate(NULL, vp, VCL_STATE_COLD); + now = VTIM_mono(); + VTAILQ_FOREACH(vp, &vclhead, list) { + if (mgt_vcl_cooldown(vp, now)) + (void)mgt_vcl_settemp(NULL, vp, 0); + } return (0); } diff --git a/bin/varnishtest/tests/c00077.vtc b/bin/varnishtest/tests/c00077.vtc index 49ed76797..8b2c35df1 100644 --- a/bin/varnishtest/tests/c00077.vtc +++ b/bin/varnishtest/tests/c00077.vtc @@ -13,6 +13,8 @@ varnish v1 -vcl+backend { varnish v1 -clierr 106 "vcl.label vcl.A vcl1" varnish v1 -cliok "vcl.label vclA vcl1" +# labeling twice #2834 +varnish v1 -cliok "vcl.label vclA vcl1" varnish v1 -vcl+backend { sub vcl_recv { diff --git a/bin/varnishtest/tests/r02432.vtc b/bin/varnishtest/tests/r02432.vtc index 97b5d30d9..2365f1725 100644 --- a/bin/varnishtest/tests/r02432.vtc +++ b/bin/varnishtest/tests/r02432.vtc @@ -11,6 +11,7 @@ varnish v1 -vcl+backend { } -start varnish v1 -cliok "vcl.label label1 vcl1" +varnish v1 -cliok "vcl.list" varnish v1 -vcl+backend { sub vcl_recv { diff --git a/bin/varnishtest/tests/v00003.vtc b/bin/varnishtest/tests/v00003.vtc index 5ef44b95b..449356791 100644 --- a/bin/varnishtest/tests/v00003.vtc +++ b/bin/varnishtest/tests/v00003.vtc @@ -4,7 +4,6 @@ server s1 -repeat 20 { rxreq txresp delay .2 - close } -start # The debug vmod logs temperature vcl events @@ -12,7 +11,8 @@ varnish v1 -arg "-p vcl_cooldown=1" -vcl { import debug; backend default { .host = "${s1_addr}"; - .probe = { .interval = 1s; .initial = 1;} + .port = "${s1_port}"; + .probe = { .interval = 1s; } } } -start @@ -24,7 +24,8 @@ varnish v1 -cliok "backend.list -p *.*" varnish v1 -vcl { backend default { .host = "${s1_addr}"; - .probe = { .interval = 1s; .initial = 1;} + .port = "${s1_port}"; + .probe = { .interval = 1s; } } } @@ -33,64 +34,64 @@ delay .4 varnish v1 -expect VBE.vcl1.default.happy >= 0 varnish v1 -expect VBE.vcl2.default.happy >= 0 -# Freeze the first VCL -varnish v1 -cliok "vcl.state vcl1 cold" -delay .4 -varnish v1 -expect !VBE.vcl1.default.happy +# We are about to freeze vcl, then implicitly thaw it via use. +# check that we see the events -# Set it auto should be a no-op -varnish v1 -cliok "vcl.state vcl1 auto" +logexpect l1 -v v1 -g raw { + expect * 0 Debug "vcl1: VCL_EVENT_COLD" + expect * 0 Debug "vcl1: VCL_EVENT_WARM" +} -start + +# Freeze vcl1 +varnish v1 -cliok "vcl.state vcl1 cold" +varnish v1 -cliexpect "available *cold *cold *[0-9]+ *vcl1\\s+active *auto *warm *[0-9]+ *vcl2" "vcl.list" delay .4 varnish v1 -expect !VBE.vcl1.default.happy -# Use it, and it should come alive +# Use it, and it should come warm (but not auto) varnish v1 -cliok "vcl.use vcl1" +varnish v1 -cliexpect "active *warm *warm *[0-9]+ *vcl1\\s+available *auto *warm *[0-9]+ *vcl2" "vcl.list" delay .4 varnish v1 -expect VBE.vcl1.default.happy >= 0 varnish v1 -expect VBE.vcl2.default.happy >= 0 +logexpect l1 -wait + # and the unused one should go cold delay 4 +varnish v1 -cliexpect "active *warm *warm *[0-9]+ *vcl1\\s+available *auto *cold *[0-9]+ *vcl2" "vcl.list" varnish v1 -expect !VBE.vcl2.default.happy -# Mark the used warm and use the other -varnish v1 -cliok "vcl.state vcl1 warm" +# use the other varnish v1 -cliok "vcl.use vcl2" +varnish v1 -cliexpect "available *warm *warm *[0-9]+ *vcl1\\s+active *auto *warm *[0-9]+ *vcl2" "vcl.list" -# It will stay warm even after the cooldown period +# the non-auto vcl will stay warm even after the cooldown period delay 4 +varnish v1 -cliexpect "available *warm *warm *[0-9]+ *vcl1\\s+active *auto *warm *[0-9]+ *vcl2" "vcl.list" varnish v1 -expect VBE.vcl1.default.happy >= 0 varnish v1 -expect VBE.vcl2.default.happy >= 0 # You can't freeze the active VCL varnish v1 -clierr 300 "vcl.state vcl2 cold" -# However a warm event is guaranteed... -logexpect l1 -v v1 -g raw { - expect * 0 Debug "vcl1: VCL_EVENT_COLD" - expect * 0 Debug "vcl1: VCL_EVENT_WARM" -} -start - -# ...when you use a cold VCL -varnish v1 -cliok "vcl.state vcl1 cold" -varnish v1 -cliok "vcl.use vcl1" - -logexpect l1 -wait - -# It will apply the cooldown period once inactive -varnish v1 -cliok "vcl.use vcl2" +# the non-auto vcl will apply the cooldown again once changed back to auto +varnish v1 -cliok "vcl.state vcl1 auto" +varnish v1 -cliexpect "available *auto *warm *[0-9]+ *vcl1\\s+active *auto *warm *[0-9]+ *vcl2" "vcl.list" delay .4 varnish v1 -expect VBE.vcl1.default.happy >= 0 delay 4 +varnish v1 -cliexpect "available *auto *cold *[0-9]+ *vcl1\\s+active *auto *warm *[0-9]+ *vcl2" "vcl.list" varnish v1 -expect !VBE.vcl1.default.happy # A VMOD's warm-up can fail varnish v1 -cliok "param.set max_esi_depth 42" varnish v1 -clierr 300 "vcl.state vcl1 warm" -varnish v1 -cliexpect "available *cold *cold *[0-9]+ *vcl1\\s+active *warm *warm *[0-9]+ *vcl2" "vcl.list" +varnish v1 -cliexpect "available *auto *cold *[0-9]+ *vcl1\\s+active *auto *warm *[0-9]+ *vcl2" "vcl.list" # A warm-up failure can also fail a child start varnish v1 -cliok stop varnish v1 -cliok "vcl.state vcl1 warm" +varnish v1 -cliok "vcl.list" varnish v1 -clierr 300 start From phk at FreeBSD.org Fri May 3 06:48:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 3 May 2019 06:48:08 +0000 (UTC) Subject: [master] adf18f4df no implicit warmup for manual temperature control Message-ID: <20190503064808.2B80560F87@lists.varnish-cache.org> commit adf18f4df2aaae0701ab2ddc04ca518e71c68a55 Author: Nils Goroll Date: Mon Nov 19 14:45:26 2018 +0100 no implicit warmup for manual temperature control As discussed during bugwash, we should be consistent about the manual temperature controls and not transition cold->warm, but rather fail. diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index f54a9beff..ca45018a0 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -468,6 +468,18 @@ mgt_vcl_settemp(struct cli *cli, struct vclprog *vp, unsigned warm) return (i); } +static int +mgt_vcl_requirewarm(struct cli *cli, struct vclprog *vp) +{ + if (vp->state == VCL_STATE_COLD) { + VCLI_SetResult(cli, CLIS_CANT); + VCLI_Out(cli, "VCL '%s' is cold - set to auto or warm first", + vp->name); + return (1); + } + return (mgt_vcl_settemp(cli, vp, 1)); +} + static int mgt_vcl_tellchild(struct cli *cli, struct vclprog *vp, unsigned warm) { @@ -783,7 +795,7 @@ mcf_vcl_use(struct cli *cli, const char * const *av, void *priv) if (vp == active_vcl) return; - if (mgt_vcl_settemp(cli, vp, 1)) + if (mgt_vcl_requirewarm(cli, vp)) return; if (MCH_Running() && @@ -999,10 +1011,13 @@ mcf_vcl_label(struct cli *cli, const char * const *av, void *priv) } } - if (mgt_vcl_settemp(cli, vpt, 1)) + if (mgt_vcl_requirewarm(cli, vpt)) return; if (vpl != NULL) { + /* potentially fail before we delete the dependency */ + if (mgt_vcl_requirewarm(cli, vpl)) + return; mgt_vcl_dep_del(VTAILQ_FIRST(&vpl->dfrom)); AN(VTAILQ_EMPTY(&vpl->dfrom)); } else { @@ -1010,7 +1025,7 @@ mcf_vcl_label(struct cli *cli, const char * const *av, void *priv) } AN(vpl); - if (mgt_vcl_settemp(cli, vpl, 1)) + if (mgt_vcl_requirewarm(cli, vpl)) return; mgt_vcl_dep_add(vpl, vpt); diff --git a/bin/varnishtest/tests/v00003.vtc b/bin/varnishtest/tests/v00003.vtc index 449356791..4a8d15898 100644 --- a/bin/varnishtest/tests/v00003.vtc +++ b/bin/varnishtest/tests/v00003.vtc @@ -48,7 +48,10 @@ varnish v1 -cliexpect "available *cold *cold *[0-9]+ *vcl1\\s+active *auto *warm delay .4 varnish v1 -expect !VBE.vcl1.default.happy -# Use it, and it should come warm (but not auto) +# Manual temperature control needs to be explicit before use +varnish v1 -clierr 300 "vcl.use vcl1" + +varnish v1 -cliok "vcl.state vcl1 warm" varnish v1 -cliok "vcl.use vcl1" varnish v1 -cliexpect "active *warm *warm *[0-9]+ *vcl1\\s+available *auto *warm *[0-9]+ *vcl2" "vcl.list" delay .4 From phk at FreeBSD.org Fri May 3 06:48:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 3 May 2019 06:48:08 +0000 (UTC) Subject: [master] df909f16c update for #2471 Message-ID: <20190503064808.4A4BA60F8C@lists.varnish-cache.org> commit df909f16c50bafdbe0c8edace2891600df29334e Author: Nils Goroll Date: Thu Apr 25 09:59:09 2019 +0200 update for #2471 diff --git a/bin/varnishtest/tests/r02471.vtc b/bin/varnishtest/tests/r02471.vtc index d9056ebc6..cbc2165e0 100644 --- a/bin/varnishtest/tests/r02471.vtc +++ b/bin/varnishtest/tests/r02471.vtc @@ -27,6 +27,7 @@ varnish v1 -cliexpect "cold cold 0 vcl1" "vcl.list" # Grab hold of vcl1 +varnish v1 -cliok "vcl.state vcl1 auto" varnish v1 -cliok "vcl.use vcl1" client c1 { txreq -url "/hold" @@ -45,6 +46,7 @@ varnish v1 -cliok "vcl.state vcl1 cold" varnish v1 -cliexpect "cold busy [12] vcl1" "vcl.list" # Release hold on vcl1 +varnish v1 -cliok "vcl.state vcl1 auto" varnish v1 -cliok "vcl.use vcl1" client c1 { txreq -url "/release" From phk at FreeBSD.org Fri May 3 06:49:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 3 May 2019 06:49:07 +0000 (UTC) Subject: [master] 56c37d90e Make this work with llvm's code coverage tools Message-ID: <20190503064907.33D3261597@lists.varnish-cache.org> commit 56c37d90e568e5ff3ffa01d77c1c2d11de6a4adf Author: Poul-Henning Kamp Date: Tue Apr 30 06:37:02 2019 +0000 Make this work with llvm's code coverage tools diff --git a/tools/gcov_digest.py b/tools/gcov_digest.py index 2e2f6ebba..90828a43f 100644 --- a/tools/gcov_digest.py +++ b/tools/gcov_digest.py @@ -106,14 +106,14 @@ def run_gcov(prog, subdir): if root[-6:] == "/.libs": x = subprocess.check_output( ["cd " + root + "/.. && " + - "exec " + prog + " -r .libs/" + fn], + "exec " + prog + " .libs/" + fn], stderr=subprocess.STDOUT, shell=True, universal_newlines=True) pf = ".." else: x = subprocess.check_output( ["cd " + root + " && " + - "exec " + prog + " -r " + fn], + "exec " + prog + " " + fn], stderr=subprocess.STDOUT, shell=True, universal_newlines=True) pf = "" @@ -122,7 +122,7 @@ def run_gcov(prog, subdir): ln = ln.split() if not ln: continue - if ln[0] == "Creating": + if ln[0].find("reating") != -1: gn = ln[1].strip("'") assert gn[-5:] == ".gcov" sn = gn[:-5] @@ -178,7 +178,7 @@ if __name__ == "__main__": optlist, args = getopt.getopt(sys.argv[1:], "g:o:x:") fo = sys.stdout - gcovprog = "gcov6" + gcovprog = "gcov6 -r" for f, v in optlist: if f == '-o' and v == '-': From dridi.boukelmoune at gmail.com Fri May 3 07:22:07 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 3 May 2019 07:22:07 +0000 (UTC) Subject: [master] f41055d32 Verbosity no longer cycles through in varnishstat Message-ID: <20190503072207.E8FA1620C9@lists.varnish-cache.org> commit f41055d326fdfabd75a1afd0653ea9357a13d25e Author: Dridi Boukelmoune Date: Fri May 3 09:19:54 2019 +0200 Verbosity no longer cycles through in varnishstat Refs #2990 diff --git a/doc/sphinx/reference/varnishstat.rst b/doc/sphinx/reference/varnishstat.rst index d8b7f2399..a6f81c8b8 100644 --- a/doc/sphinx/reference/varnishstat.rst +++ b/doc/sphinx/reference/varnishstat.rst @@ -106,8 +106,12 @@ The following keys control the interactive display: Go to the bottom of the counter list. - Cycle through the verbosity levels. Defaults to only showing - informational counters. + Increase verbosity. Defaults to only showing informational + counters. + + + Decrease verbosity. Defaults to only showing informational + counters. Quit. From phk at FreeBSD.org Fri May 3 09:19:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 3 May 2019 09:19:07 +0000 (UTC) Subject: [master] 2c8d4a786 Simplifications Message-ID: <20190503091907.B49BF65222@lists.varnish-cache.org> commit 2c8d4a786e484e6f881dab44ad14b7464ca84d6d Author: Poul-Henning Kamp Date: Fri May 3 08:33:09 2019 +0000 Simplifications diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index ca45018a0..af520299b 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -106,20 +106,22 @@ static struct vev *e_poker; static int mgt_vcl_setstate(struct cli *, struct vclprog *, const char *); static int mgt_vcl_settemp(struct cli *, struct vclprog *, unsigned); -static int mgt_vcl_tellchild(struct cli *, struct vclprog *, unsigned); +static int mgt_vcl_askchild(struct cli *, struct vclprog *, unsigned); static void mgt_vcl_set_cooldown(struct vclprog *, vtim_mono); /*--------------------------------------------------------------------*/ static const char * -mcf_vcl_parse_state(const char *s) +mcf_vcl_parse_state(struct cli *cli, const char *s) { - if (s == NULL || *s == '\0') - return (NULL); -#define VCL_STATE(sym, str) \ - if (!strcmp(s, VCL_STATE_ ## sym)) \ - return (VCL_STATE_ ## sym); + if (s != NULL) { +#define VCL_STATE(sym, str) \ + if (!strcmp(s, VCL_STATE_ ## sym)) \ + return (VCL_STATE_ ## sym); #include "tbl/vcl_states.h" + } + VCLI_Out(cli, "State must be one of auto, cold or warm."); + VCLI_SetResult(cli, CLIS_PARAM); return (NULL); } @@ -200,21 +202,19 @@ mgt_vcl_dep_add(struct vclprog *vp_from, struct vclprog *vp_to) CHECK_OBJ_NOTNULL(vp_from, VCLPROG_MAGIC); CHECK_OBJ_NOTNULL(vp_to, VCLPROG_MAGIC); + assert(vp_to->state != VCL_STATE_COLD); + ALLOC_OBJ(vd, VCLDEP_MAGIC); + AN(vd); mgt_vcl_set_cooldown(vp_from, -1); mgt_vcl_set_cooldown(vp_to, -1); - XXXAN(vd); vd->from = vp_from; VTAILQ_INSERT_TAIL(&vp_from->dfrom, vd, lfrom); vd->to = vp_to; VTAILQ_INSERT_TAIL(&vp_to->dto, vd, lto); vp_to->nto++; - - assert(vp_to->state == VCL_STATE_WARM || /* vcl.label ... */ - vp_to->state == VCL_STATE_AUTO || /* vcl.label ... */ - vp_to->state == VCL_STATE_LABEL); /* return(vcl(...)) */ } static void @@ -425,27 +425,6 @@ mgt_vcl_set_cooldown(struct vclprog *vp, vtim_mono now) vp->go_cold = now; } -static unsigned -mgt_vcl_cooldown(struct vclprog *vp, vtim_mono now) -{ - CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC); - - if (vp->go_cold < 0) - return (0); - - if (vp->go_cold == 0) { - mgt_vcl_set_cooldown(vp, now); - return (0); - } - - assert(vp->go_cold > 0); - - if (vp->go_cold + mgt_param.vcl_cooldown < now) - return (1); - - return (0); -} - static int mgt_vcl_settemp(struct cli *cli, struct vclprog *vp, unsigned warm) { @@ -458,7 +437,7 @@ mgt_vcl_settemp(struct cli *cli, struct vclprog *vp, unsigned warm) if (vp->state == VCL_STATE_AUTO || vp->state == VCL_STATE_LABEL) { mgt_vcl_set_cooldown(vp, -1); - i = mgt_vcl_tellchild(cli, vp, warm); + i = mgt_vcl_askchild(cli, vp, warm); mgt_vcl_set_cooldown(vp, VTIM_mono()); } else { i = mgt_vcl_setstate(cli, vp, @@ -481,7 +460,7 @@ mgt_vcl_requirewarm(struct cli *cli, struct vclprog *vp) } static int -mgt_vcl_tellchild(struct cli *cli, struct vclprog *vp, unsigned warm) +mgt_vcl_askchild(struct cli *cli, struct vclprog *vp, unsigned warm) { unsigned status; char *p; @@ -547,7 +526,7 @@ mgt_vcl_setstate(struct cli *cli, struct vclprog *vp, const char *vs) warm = (vs == VCL_STATE_WARM ? 1 : 0); } - i = mgt_vcl_tellchild(cli, vp, warm); + i = mgt_vcl_askchild(cli, vp, warm); if (i == 0) mgt_vcl_set_cooldown(vp, VTIM_mono()); else @@ -582,13 +561,10 @@ mgt_new_vcl(struct cli *cli, const char *vclname, const char *vclsrc, if (state == NULL) state = VCL_STATE_AUTO; else - state = mcf_vcl_parse_state(state); + state = mcf_vcl_parse_state(cli, state); - if (state == NULL) { - VCLI_Out(cli, "State must be one of auto, cold or warm."); - VCLI_SetResult(cli, CLIS_PARAM); + if (state == NULL) return; - } vp = mgt_vcl_add(vclname, state); lib = mgt_VccCompile(cli, vp, vclname, vclsrc, vclsrcfile, C_flag); @@ -756,12 +732,9 @@ mcf_vcl_state(struct cli *cli, const char * const *av, void *priv) return; } - state = mcf_vcl_parse_state(av[3]); - if (state == NULL) { - VCLI_Out(cli, "State must be one of auto, cold or warm."); - VCLI_SetResult(cli, CLIS_PARAM); + state = mcf_vcl_parse_state(cli, av[3]); + if (state == NULL) return; - } if (state == VCL_STATE_COLD) { if (!VTAILQ_EMPTY(&vp->dto)) { @@ -1053,7 +1026,10 @@ mgt_vcl_poker(const struct vev *e, int what) e_poker->timeout = mgt_param.vcl_cooldown * .45; now = VTIM_mono(); VTAILQ_FOREACH(vp, &vclhead, list) { - if (mgt_vcl_cooldown(vp, now)) + if (vp->go_cold == 0) + mgt_vcl_set_cooldown(vp, now); + else if (vp->go_cold > 0 && + vp->go_cold + mgt_param.vcl_cooldown < now) (void)mgt_vcl_settemp(NULL, vp, 0); } return (0); From dridi.boukelmoune at gmail.com Fri May 3 15:55:08 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 3 May 2019 15:55:08 +0000 (UTC) Subject: [master] 535192c3a Timid start of Coccinelle type isomorphisms Message-ID: <20190503155508.EED7EA0ECA@lists.varnish-cache.org> commit 535192c3ab8545c9b9bc849327879d773e05804a Author: Dridi Boukelmoune Date: Fri May 3 17:35:38 2019 +0200 Timid start of Coccinelle type isomorphisms This way we don't need to constantly deal with typedef ORing when we encounter a varnish-defined type. We still need to declare typedefs that we use, but it's already one order of magnitude more convenient. diff --git a/tools/coccinelle/varnish.iso b/tools/coccinelle/varnish.iso new file mode 100644 index 000000000..1beb51e90 --- /dev/null +++ b/tools/coccinelle/varnish.iso @@ -0,0 +1,35 @@ +/* + * This file contains isomorphisms specific to the Varnish code base and + * out of tree Varnish projects such as VUTs for VMODs. + * + * It can be used directly by semantic patches from the same directory + * with the following syntax: + * + * using "varnish.iso" + * + * XXX: way incomplete. + * XXX: consider autogeneration. + */ + +/* This section contains VCL types used by VMODs */ + +Type +@ vrt_ctx @ +type VRT_CTX; +@@ + +VRT_CTX <=> const struct vrt_ctx * + +Type +@ vcl_void @ +type VCL_VOID; +@@ + +VCL_VOID <=> void + +Type +@ vcl_bool @ +type VCL_BOOL; +@@ + +VCL_BOOL <=> unsigned From dridi.boukelmoune at gmail.com Fri May 3 16:15:08 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 3 May 2019 16:15:08 +0000 (UTC) Subject: [master] c722eb774 Swap isomorphism subjects Message-ID: <20190503161508.5C7CDA16FE@lists.varnish-cache.org> commit c722eb774e4fd5412de94a9cc01a49d9f0f3f76d Author: Dridi Boukelmoune Date: Fri May 3 18:06:29 2019 +0200 Swap isomorphism subjects In this order it becomes possible to use VCL_VOID transparently as void without having to declare it again as a typedef. It doesn't matter for VRT_CTX because it tends to be easier to use it instead of its actual C type (with the typedef-in-patch penalty). Wave in my general direction if you're not slink and yet can make sense of the previous paragraph. diff --git a/tools/coccinelle/varnish.iso b/tools/coccinelle/varnish.iso index 1beb51e90..8bbc11c53 100644 --- a/tools/coccinelle/varnish.iso +++ b/tools/coccinelle/varnish.iso @@ -25,11 +25,11 @@ Type type VCL_VOID; @@ -VCL_VOID <=> void +void <=> VCL_VOID Type @ vcl_bool @ type VCL_BOOL; @@ -VCL_BOOL <=> unsigned +unsigned <=> VCL_BOOL From dridi.boukelmoune at gmail.com Fri May 3 17:25:08 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 3 May 2019 17:25:08 +0000 (UTC) Subject: [master] 06e7329d6 Help Coccinelle a tiny bit more Message-ID: <20190503172508.E6E33A2F92@lists.varnish-cache.org> commit 06e7329d6525c3f3721e201a47ed3919d8d725ce Author: Dridi Boukelmoune Date: Fri May 3 19:16:59 2019 +0200 Help Coccinelle a tiny bit more It currently freaks out when it's not aware of a type, or when it encounters v_*_ macros in a function signature. This doesn't improve the results of the existing patches but we should take a habit of specifying where additional C includes are. diff --git a/tools/coccinelle/README.rst b/tools/coccinelle/README.rst index ff01e80bf..bf01fc8e3 100644 --- a/tools/coccinelle/README.rst +++ b/tools/coccinelle/README.rst @@ -6,6 +6,6 @@ for both the in-tree code style or out-of-tree VMOD and VUT development. Unless noted otherwise, all patches should work when invoked as:: - spatch --dir . --in-place --sp-file $COCCI + spatch -I include/ -I bin/varnishd/ --dir . --in-place --sp-file $COCCI .. _coccinelle: http://coccinelle.lip6.fr/ From fgsch at lodoss.net Sun May 5 08:15:13 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 5 May 2019 08:15:13 +0000 (UTC) Subject: [master] e6717f5ac Split this test into tcp and uds Message-ID: <20190505081513.222F7AEFF8@lists.varnish-cache.org> commit e6717f5acb7c851f4469eebe10cc02f0702e8ce6 Author: Federico G. Schwindt Date: Sun May 5 01:18:37 2019 +0100 Split this test into tcp and uds The former fails in some platforms but we can test the latter. diff --git a/bin/varnishtest/tests/s00010.vtc b/bin/varnishtest/tests/s00010.vtc index 81dbfee00..169ffa8d1 100644 --- a/bin/varnishtest/tests/s00010.vtc +++ b/bin/varnishtest/tests/s00010.vtc @@ -1,7 +1,9 @@ -varnishtest "client h1 send timeouts" +varnishtest "client h1 send timeouts - tcp" # XXX See https://github.com/varnishcache/varnish-cache/pull/2980#issuecomment-486214661 -feature cmd {test $(uname) != "SunOS"} +feature cmd {test $(uname) != "SunOS" && test $(uname) != "Darwin"} + +feature SO_RCVTIMEO_WORKS server s1 { rxreq @@ -13,7 +15,6 @@ varnish v1 \ -arg "-p idle_send_timeout=.1" \ -arg "-p send_timeout=.1" \ -arg "-a 127.0.0.1:0" \ - -arg "-a ${tmpdir}/v1.sock" \ -vcl+backend { import debug; @@ -24,7 +25,6 @@ varnish v1 \ logexpect l1 -v v1 { expect * * Debug "Hit total send timeout" - expect * * Debug "Hit total send timeout" } -start client c1 -rcvbuf 128 { @@ -35,25 +35,14 @@ client c1 -rcvbuf 128 { delay 2 } -start -client c1u -connect "${tmpdir}/v1.sock" -rcvbuf 128 { - txreq - non_fatal - rxresphdrs - # keep the session open for 2 seconds - delay 2 -} -start - client c1 -wait -client c1u -wait logexpect l1 -wait -feature SO_RCVTIMEO_WORKS varnish v1 -cliok "param.set idle_send_timeout 1" varnish v1 -cliok "param.reset send_timeout" logexpect l2 -v v1 { expect * * Debug "Hit idle send timeout" - expect * * Debug "Hit idle send timeout" } -start client c2 -rcvbuf 128 { @@ -63,13 +52,5 @@ client c2 -rcvbuf 128 { delay 2 } -start -client c2u -connect "${tmpdir}/v1.sock" -rcvbuf 128 { - txreq - rxresphdrs - # keep the session open for 2 seconds - delay 2 -} -start - client c2 -wait -client c2u -wait logexpect l2 -wait diff --git a/bin/varnishtest/tests/s00012.vtc b/bin/varnishtest/tests/s00012.vtc new file mode 100644 index 000000000..392cb2830 --- /dev/null +++ b/bin/varnishtest/tests/s00012.vtc @@ -0,0 +1,53 @@ +varnishtest "client h1 send timeouts - uds" + +feature SO_RCVTIMEO_WORKS + +server s1 { + rxreq + txresp -bodylen 100000 +} -start + +varnish v1 \ + -arg "-p timeout_idle=1" \ + -arg "-p idle_send_timeout=.1" \ + -arg "-p send_timeout=.1" \ + -arg "-a ${tmpdir}/v1.sock" \ + -vcl+backend { + import debug; + + sub vcl_deliver { + debug.sndbuf(128b); + } +} -start + +logexpect l1 -v v1 { + expect * * Debug "Hit total send timeout" +} -start + +client c1 -connect "${tmpdir}/v1.sock" -rcvbuf 128 { + txreq + non_fatal + rxresphdrs + # keep the session open for 2 seconds + delay 2 +} -start + +client c1 -wait +logexpect l1 -wait + +varnish v1 -cliok "param.set idle_send_timeout 1" +varnish v1 -cliok "param.reset send_timeout" + +logexpect l2 -v v1 { + expect * * Debug "Hit idle send timeout" +} -start + +client c2 -connect "${tmpdir}/v1.sock" -rcvbuf 128 { + txreq + rxresphdrs + # keep the session open for 2 seconds + delay 2 +} -start + +client c2 -wait +logexpect l2 -wait From fgsch at lodoss.net Sun May 5 08:15:13 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 5 May 2019 08:15:13 +0000 (UTC) Subject: [master] ac85bd89d Whitespace Message-ID: <20190505081513.3DE17AEFFB@lists.varnish-cache.org> commit ac85bd89dd04026093adb63da32d9a787e759d83 Author: Federico G. Schwindt Date: Sun May 5 01:23:18 2019 +0100 Whitespace diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index af520299b..2710e82fd 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -202,7 +202,7 @@ mgt_vcl_dep_add(struct vclprog *vp_from, struct vclprog *vp_to) CHECK_OBJ_NOTNULL(vp_from, VCLPROG_MAGIC); CHECK_OBJ_NOTNULL(vp_to, VCLPROG_MAGIC); - assert(vp_to->state != VCL_STATE_COLD); + assert(vp_to->state != VCL_STATE_COLD); ALLOC_OBJ(vd, VCLDEP_MAGIC); AN(vd); diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 110256cb0..e0194e8c9 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -449,7 +449,7 @@ VSC_Iter(struct vsc *vsc, struct vsm *vsm, VSC_iter_f *fiter, void *priv) i = vsc_iter_seg(vsc, sp, fiter, priv); sp = VTAILQ_NEXT(sp, list); } - + if (i) break; } From fgsch at lodoss.net Sun May 5 08:45:09 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 5 May 2019 08:45:09 +0000 (UTC) Subject: [master] 7557c64b3 This seems to also fail under sunos Message-ID: <20190505084509.5085EAFBE6@lists.varnish-cache.org> commit 7557c64b334d5f2b8bb54046976d9f742fc0b3a8 Author: Federico G. Schwindt Date: Sun May 5 09:35:40 2019 +0100 This seems to also fail under sunos diff --git a/bin/varnishtest/tests/s00012.vtc b/bin/varnishtest/tests/s00012.vtc index 392cb2830..277625f46 100644 --- a/bin/varnishtest/tests/s00012.vtc +++ b/bin/varnishtest/tests/s00012.vtc @@ -1,5 +1,7 @@ varnishtest "client h1 send timeouts - uds" +feature cmd {test $(uname) != "SunOS"} + feature SO_RCVTIMEO_WORKS server s1 { From fgsch at lodoss.net Sun May 5 16:35:07 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 5 May 2019 16:35:07 +0000 (UTC) Subject: [master] a1dc310a1 We have cmd for a while so drop !osx Message-ID: <20190505163507.E75C370DB@lists.varnish-cache.org> commit a1dc310a10303bf5a7aceba9557453724ae5e4a6 Author: Federico G. Schwindt Date: Sun May 5 17:11:30 2019 +0100 We have cmd for a while so drop !osx diff --git a/bin/varnishtest/tests/r00962.vtc b/bin/varnishtest/tests/r00962.vtc index 84f5eea23..91c093075 100644 --- a/bin/varnishtest/tests/r00962.vtc +++ b/bin/varnishtest/tests/r00962.vtc @@ -5,7 +5,7 @@ feature persistent_storage feature disable_aslr # VM-remapping is too random on OSX -feature !OSX +feature cmd {test $(uname) != "Darwin"} # Same on some hardened Linux feature cmd "test ! -c /dev/grsec" diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index e496de120..69c1c2c75 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -382,8 +382,6 @@ dns_works(void) * The SO_RCVTIMEO socket option is working * 64bit * The environment is 64 bits - * !OSX - * The environment is not OSX * dns * DNS lookups are working * topbuild @@ -449,13 +447,6 @@ cmd_feature(CMD_ARGS) #endif } - if (!strcmp(*av, "!OSX")) { -#if !defined(__APPLE__) || !defined(__MACH__) - good = 1; -#else - vtc_stop = 2; -#endif - } FEATURE("pcre_jit", VRE_has_jit); FEATURE("64bit", sizeof(void*) == 8); FEATURE("dns", dns_works()); From phk at FreeBSD.org Mon May 6 09:14:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 6 May 2019 09:14:08 +0000 (UTC) Subject: [master] a2bb3507e Whitespace OCD Message-ID: <20190506091408.7BFEEA4B3C@lists.varnish-cache.org> commit a2bb3507e27060e043d5faaae0dec9b34dba80e3 Author: Poul-Henning Kamp Date: Mon May 6 08:59:06 2019 +0000 Whitespace OCD diff --git a/bin/varnishtest/tests/s00010.vtc b/bin/varnishtest/tests/s00010.vtc index 169ffa8d1..eb71ddc57 100644 --- a/bin/varnishtest/tests/s00010.vtc +++ b/bin/varnishtest/tests/s00010.vtc @@ -10,7 +10,7 @@ server s1 { txresp -bodylen 100000 } -start -varnish v1 \ +varnish v1 \ -arg "-p timeout_idle=1" \ -arg "-p idle_send_timeout=.1" \ -arg "-p send_timeout=.1" \ diff --git a/bin/varnishtest/tests/s00012.vtc b/bin/varnishtest/tests/s00012.vtc index 277625f46..2ea5bdf16 100644 --- a/bin/varnishtest/tests/s00012.vtc +++ b/bin/varnishtest/tests/s00012.vtc @@ -9,7 +9,7 @@ server s1 { txresp -bodylen 100000 } -start -varnish v1 \ +varnish v1 \ -arg "-p timeout_idle=1" \ -arg "-p idle_send_timeout=.1" \ -arg "-p send_timeout=.1" \ From phk at FreeBSD.org Mon May 6 09:14:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 6 May 2019 09:14:08 +0000 (UTC) Subject: [master] aa7295e51 Move VRT_Vmod_{Init|Unload} To VPI Message-ID: <20190506091408.95E63A4B3F@lists.varnish-cache.org> commit aa7295e51762f7f9ad42f71ab2ca536289a78ba0 Author: Poul-Henning Kamp Date: Mon May 6 09:02:59 2019 +0000 Move VRT_Vmod_{Init|Unload} To VPI Relevant to #2800 diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c index fd8031149..bfc9496c5 100644 --- a/bin/varnishd/cache/cache_vrt_vmod.c +++ b/bin/varnishd/cache/cache_vrt_vmod.c @@ -38,6 +38,7 @@ #include #include "vcli_serve.h" +#include "vcc_interface.h" #include "vmod_abi.h" /*-------------------------------------------------------------------- @@ -78,7 +79,7 @@ vmod_abi_mismatch(const struct vmod_data *d) } int -VRT_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len, +VPI_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len, const char *nm, const char *path, const char *file_id, const char *backup) { struct vmod *v; @@ -162,7 +163,7 @@ VRT_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len, } void -VRT_Vmod_Unload(VRT_CTX, struct vmod **hdl) +VPI_Vmod_Unload(VRT_CTX, struct vmod **hdl) { struct vmod *v; diff --git a/include/vcc_interface.h b/include/vcc_interface.h index cf1845a8d..347c20077 100644 --- a/include/vcc_interface.h +++ b/include/vcc_interface.h @@ -50,3 +50,8 @@ struct vpi_ref { }; void VPI_count(VRT_CTX, unsigned); + +int VPI_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len, + const char *nm, const char *path, const char *file_id, const char *backup); +void VPI_Vmod_Unload(VRT_CTX, struct vmod **hdl); + diff --git a/include/vrt.h b/include/vrt.h index f83218efa..f5789e9e0 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -52,6 +52,8 @@ * binary/load-time compatible, increment MAJOR version * * unreleased (planned for 2019-09-15) + * VRT_Vmod_{Init|Unload} moved to vcc_interface.h + * VRT_count moved to vcc_interface.h * VRT_VCL_Busy() and VRT_VCL_Unbusy() added. * VRT_vcl_get moved to vcc_interface.h * VRT_vcl_rel emoved to vcc_interface.h @@ -511,11 +513,6 @@ void VRT_DelDirector(VCL_BACKEND *); /* Suckaddr related */ int VRT_VSA_GetPtr(VCL_IP sua, const unsigned char ** dst); -/* VMOD/Modules related */ -int VRT_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len, - const char *nm, const char *path, const char *file_id, const char *backup); -void VRT_Vmod_Unload(VRT_CTX, struct vmod **hdl); - typedef int vmod_event_f(VRT_CTX, struct vmod_priv *, enum vcl_event_e); typedef void vmod_priv_free_f(void *); diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index a455bbb8e..61380edf2 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -303,7 +303,7 @@ vcc_ParseImport(struct vcc *tl) ifp = New_IniFin(tl); - VSB_printf(ifp->ini, "\tif (VRT_Vmod_Init(ctx,\n"); + VSB_printf(ifp->ini, "\tif (VPI_Vmod_Init(ctx,\n"); VSB_printf(ifp->ini, "\t &VGC_vmod_%.*s,\n", PF(mod)); VSB_printf(ifp->ini, "\t %u,\n", tl->vmod_count++); VSB_printf(ifp->ini, "\t &%s,\n", vmd->func_name); @@ -326,7 +326,7 @@ vcc_ParseImport(struct vcc *tl) /* XXX: zero the function pointer structure ?*/ VSB_printf(ifp->fin, "\t\tVRT_priv_fini(&vmod_priv_%.*s);", PF(mod)); VSB_printf(ifp->final, - "\t\tVRT_Vmod_Unload(ctx, &VGC_vmod_%.*s);", PF(mod)); + "\t\tVPI_Vmod_Unload(ctx, &VGC_vmod_%.*s);", PF(mod)); vj = vjsn_parse(vmd->json, &p); XXXAZ(p); From phk at FreeBSD.org Mon May 6 09:14:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 6 May 2019 09:14:08 +0000 (UTC) Subject: [master] f9c9db4a0 Move struct vmod_data to VPI namespace Message-ID: <20190506091408.AE5B1A4B43@lists.varnish-cache.org> commit f9c9db4a0f18d740aa5f008074717affe4c6e0ef Author: Poul-Henning Kamp Date: Mon May 6 09:13:29 2019 +0000 Move struct vmod_data to VPI namespace diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c index bfc9496c5..7f5df6066 100644 --- a/bin/varnishd/cache/cache_vrt_vmod.c +++ b/bin/varnishd/cache/cache_vrt_vmod.c @@ -68,7 +68,7 @@ struct vmod { static VTAILQ_HEAD(,vmod) vmods = VTAILQ_HEAD_INITIALIZER(vmods); static unsigned -vmod_abi_mismatch(const struct vmod_data *d) +vmod_abi_mismatch(const struct vpi_vmod_data *d) { if (d->vrt_major == 0 && d->vrt_minor == 0) @@ -83,7 +83,7 @@ VPI_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len, const char *nm, const char *path, const char *file_id, const char *backup) { struct vmod *v; - const struct vmod_data *d; + const struct vpi_vmod_data *d; char buf[256]; void *dlhdl; diff --git a/include/vcc_interface.h b/include/vcc_interface.h index 347c20077..a9224f3bc 100644 --- a/include/vcc_interface.h +++ b/include/vcc_interface.h @@ -37,6 +37,25 @@ VCL_VCL VPI_vcl_get(VRT_CTX, const char *); void VPI_vcl_rel(VRT_CTX, VCL_VCL); void VPI_vcl_select(VRT_CTX, VCL_VCL); +/*********************************************************************** + * This is the interface structure to a compiled VMOD + */ + +struct vpi_vmod_data { + /* The version/id fields must be first, they protect the rest */ + unsigned vrt_major; + unsigned vrt_minor; + const char *file_id; + + const char *name; + const char *func_name; + const void *func; + int func_len; + const char *proto; + const char *json; + const char *abi; +}; + /*********************************************************************** * VPI_count() refers to this structure for coordinates into the VCL source. */ diff --git a/include/vrt.h b/include/vrt.h index f5789e9e0..934443037 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -52,6 +52,7 @@ * binary/load-time compatible, increment MAJOR version * * unreleased (planned for 2019-09-15) + * struct vmod_data moved to vcc_interface.h * VRT_Vmod_{Init|Unload} moved to vcc_interface.h * VRT_count moved to vcc_interface.h * VRT_VCL_Busy() and VRT_VCL_Unbusy() added. @@ -270,25 +271,6 @@ struct vrt_ctx { #define VRT_CTX const struct vrt_ctx *ctx -/*********************************************************************** - * This is the interface structure to a compiled VMOD - */ - -struct vmod_data { - /* The version/id fields must be first, they protect the rest */ - unsigned vrt_major; - unsigned vrt_minor; - const char *file_id; - - const char *name; - const char *func_name; - const void *func; - int func_len; - const char *proto; - const char *json; - const char *abi; -}; - /*********************************************************************** * Enum for events sent to compiled VCL and from there to Vmods */ diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index 61380edf2..d11615ac2 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -38,6 +38,7 @@ #include "vfil.h" #include "vjsn.h" #include "vmod_abi.h" +#include "vcc_interface.h" #include "vsb.h" static int @@ -162,7 +163,7 @@ vcc_ParseImport(struct vcc *tl) struct token *mod, *t1; struct inifin *ifp; struct symbol *msym; - const struct vmod_data *vmd; + const struct vpi_vmod_data *vmd; struct vjsn *vj; int again = 0; diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 1b40b10b8..2d5c2d9cb 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -1036,8 +1036,8 @@ class vcc(object): fo.write('\n') for i in (714, 759, 765): fo.write("/*lint -esym(%d, %s) */\n" % (i, vmd)) - fo.write("\nextern const struct vmod_data %s;\n" % vmd) - fo.write("\nconst struct vmod_data %s = {\n" % vmd) + fo.write("\nextern const struct vpi_vmod_data %s;\n" % vmd) + fo.write("\nconst struct vpi_vmod_data %s = {\n" % vmd) if self.strict_abi: fo.write("\t.vrt_major =\t0,\n") fo.write("\t.vrt_minor =\t0,\n") @@ -1067,7 +1067,7 @@ class vcc(object): fo.write('#include "config.h"\n') fo.write('#include \n') - for i in ["vdef", "vrt", self.pfx, "vmod_abi"]: + for i in ["vdef", "vrt", self.pfx, "vmod_abi", "vcc_interface"]: fo.write('#include "%s.h"\n' % i) fo.write("\n") From phk at FreeBSD.org Mon May 6 10:45:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 6 May 2019 10:45:10 +0000 (UTC) Subject: [master] e5072d107 Revert "Move struct vmod_data to VPI namespace" Message-ID: <20190506104510.A7237A6D2F@lists.varnish-cache.org> commit e5072d1077e23fb8217d1c6265926b9db7683272 Author: Poul-Henning Kamp Date: Mon May 6 10:44:25 2019 +0000 Revert "Move struct vmod_data to VPI namespace" This reverts commit f9c9db4a0f18d740aa5f008074717affe4c6e0ef. diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c index 7f5df6066..bfc9496c5 100644 --- a/bin/varnishd/cache/cache_vrt_vmod.c +++ b/bin/varnishd/cache/cache_vrt_vmod.c @@ -68,7 +68,7 @@ struct vmod { static VTAILQ_HEAD(,vmod) vmods = VTAILQ_HEAD_INITIALIZER(vmods); static unsigned -vmod_abi_mismatch(const struct vpi_vmod_data *d) +vmod_abi_mismatch(const struct vmod_data *d) { if (d->vrt_major == 0 && d->vrt_minor == 0) @@ -83,7 +83,7 @@ VPI_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len, const char *nm, const char *path, const char *file_id, const char *backup) { struct vmod *v; - const struct vpi_vmod_data *d; + const struct vmod_data *d; char buf[256]; void *dlhdl; diff --git a/include/vcc_interface.h b/include/vcc_interface.h index a9224f3bc..347c20077 100644 --- a/include/vcc_interface.h +++ b/include/vcc_interface.h @@ -37,25 +37,6 @@ VCL_VCL VPI_vcl_get(VRT_CTX, const char *); void VPI_vcl_rel(VRT_CTX, VCL_VCL); void VPI_vcl_select(VRT_CTX, VCL_VCL); -/*********************************************************************** - * This is the interface structure to a compiled VMOD - */ - -struct vpi_vmod_data { - /* The version/id fields must be first, they protect the rest */ - unsigned vrt_major; - unsigned vrt_minor; - const char *file_id; - - const char *name; - const char *func_name; - const void *func; - int func_len; - const char *proto; - const char *json; - const char *abi; -}; - /*********************************************************************** * VPI_count() refers to this structure for coordinates into the VCL source. */ diff --git a/include/vrt.h b/include/vrt.h index 934443037..f5789e9e0 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -52,7 +52,6 @@ * binary/load-time compatible, increment MAJOR version * * unreleased (planned for 2019-09-15) - * struct vmod_data moved to vcc_interface.h * VRT_Vmod_{Init|Unload} moved to vcc_interface.h * VRT_count moved to vcc_interface.h * VRT_VCL_Busy() and VRT_VCL_Unbusy() added. @@ -271,6 +270,25 @@ struct vrt_ctx { #define VRT_CTX const struct vrt_ctx *ctx +/*********************************************************************** + * This is the interface structure to a compiled VMOD + */ + +struct vmod_data { + /* The version/id fields must be first, they protect the rest */ + unsigned vrt_major; + unsigned vrt_minor; + const char *file_id; + + const char *name; + const char *func_name; + const void *func; + int func_len; + const char *proto; + const char *json; + const char *abi; +}; + /*********************************************************************** * Enum for events sent to compiled VCL and from there to Vmods */ diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index d11615ac2..61380edf2 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -38,7 +38,6 @@ #include "vfil.h" #include "vjsn.h" #include "vmod_abi.h" -#include "vcc_interface.h" #include "vsb.h" static int @@ -163,7 +162,7 @@ vcc_ParseImport(struct vcc *tl) struct token *mod, *t1; struct inifin *ifp; struct symbol *msym; - const struct vpi_vmod_data *vmd; + const struct vmod_data *vmd; struct vjsn *vj; int again = 0; diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 2d5c2d9cb..1b40b10b8 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -1036,8 +1036,8 @@ class vcc(object): fo.write('\n') for i in (714, 759, 765): fo.write("/*lint -esym(%d, %s) */\n" % (i, vmd)) - fo.write("\nextern const struct vpi_vmod_data %s;\n" % vmd) - fo.write("\nconst struct vpi_vmod_data %s = {\n" % vmd) + fo.write("\nextern const struct vmod_data %s;\n" % vmd) + fo.write("\nconst struct vmod_data %s = {\n" % vmd) if self.strict_abi: fo.write("\t.vrt_major =\t0,\n") fo.write("\t.vrt_minor =\t0,\n") @@ -1067,7 +1067,7 @@ class vcc(object): fo.write('#include "config.h"\n') fo.write('#include \n') - for i in ["vdef", "vrt", self.pfx, "vmod_abi", "vcc_interface"]: + for i in ["vdef", "vrt", self.pfx, "vmod_abi"]: fo.write('#include "%s.h"\n' % i) fo.write("\n") From hermunn at varnish-software.com Mon May 6 11:12:08 2019 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Mon, 6 May 2019 11:12:08 +0000 (UTC) Subject: [master] b3a14937a Typo Message-ID: <20190506111208.C1EEBA76D2@lists.varnish-cache.org> commit b3a14937ad7ce51e4e8f9abf71aaf94a1fb1d321 Author: P?l Hermunn Johansen Date: Sun Mar 3 22:46:53 2019 +0100 Typo diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c index 569f7e06e..053324d10 100644 --- a/bin/varnishd/cache/cache_ban_lurker.c +++ b/bin/varnishd/cache/cache_ban_lurker.c @@ -365,7 +365,7 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl) /* * conceptually, all obans are now completed. Remove the tail. If it - * containted the first oban, all obans were on the tail and we're + * contained the first oban, all obans were on the tail and we're * done. */ if (ban_cleantail(VTAILQ_FIRST(&obans))) From hermunn at varnish-software.com Mon May 6 11:12:08 2019 From: hermunn at varnish-software.com (Pål Hermunn Johansen) Date: Mon, 6 May 2019 11:12:08 +0000 (UTC) Subject: [master] bc421898d Optimize ban lurker and dying objects Message-ID: <20190506111208.E2A50A76D5@lists.varnish-cache.org> commit bc421898d3bfc37b4ef162638171ac544a1db171 Author: P?l Hermunn Johansen Date: Mon May 6 11:22:57 2019 +0200 Optimize ban lurker and dying objects When an object is dying, we would rather not have the ban lurker evalueate it against any existing bans. Now it will, when a dying object is found, simply remove the oc off the ban list completely. This is very cheap since we are already holding the oh and ban mutexes. diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index 0bfca2d04..16e9351d7 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -266,6 +266,8 @@ BAN_DestroyObj(struct objcore *oc) { CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + if (oc->ban == NULL) + return; Lck_Lock(&ban_mtx); CHECK_OBJ_ORNULL(oc->ban, BAN_MAGIC); if (oc->ban != NULL) { diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c index 053324d10..55a6b6402 100644 --- a/bin/varnishd/cache/cache_ban_lurker.c +++ b/bin/varnishd/cache/cache_ban_lurker.c @@ -167,8 +167,23 @@ ban_lurker_getfirst(struct vsl_log *vsl, struct ban *bt) oh = oc->objhead; CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); if (!Lck_Trylock(&oh->mtx)) { - if (oc->refcnt == 0 || oc->flags & OC_F_BUSY) { + if (oc->flags & OC_F_BUSY) { Lck_Unlock(&oh->mtx); + } else if (oc->refcnt == 0 || + oc->flags & (OC_F_DYING | OC_F_FAILED)) { + /* + * We seize the opportunity to remove + * the object completely off the ban + * list, now that we have both the oh + * and ban mutexes. + */ + noc = VTAILQ_NEXT(oc, ban_list); + VTAILQ_REMOVE(&bt->objcore, oc, ban_list); + oc->ban = NULL; + bt->refcount--; + Lck_Unlock(&oh->mtx); + oc = noc; + continue; } else { /* * We got the lock, and the oc is not being From phk at FreeBSD.org Mon May 6 19:04:12 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 6 May 2019 19:04:12 +0000 (UTC) Subject: [master] 069341b84 Merge from VTEST Message-ID: <20190506190412.0FF6ABEAA8@lists.varnish-cache.org> commit 069341b84f2d3dcc1c710183ca0850bbe4a71662 Author: Poul-Henning Kamp Date: Mon May 6 19:03:18 2019 +0000 Merge from VTEST diff --git a/bin/varnishtest/vtc_syslog.c b/bin/varnishtest/vtc_syslog.c index b9862eb50..d4f7dbc15 100644 --- a/bin/varnishtest/vtc_syslog.c +++ b/bin/varnishtest/vtc_syslog.c @@ -151,7 +151,10 @@ VUDP_close(int *s) static int VUDP_bind(const struct suckaddr *sa, const char **errp) { - int sd, val, e; +#ifdef IPV6_V6ONLY + int val; +#endif + int sd, e; socklen_t sl; const struct sockaddr *so; int proto; @@ -166,15 +169,6 @@ VUDP_bind(const struct suckaddr *sa, const char **errp) *errp = "socket(2)"; return (-1); } - val = 1; - if (setsockopt(sd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof val) != 0) { - if (errp != NULL) - *errp = "setsockopt(SO_REUSEADDR, 1)"; - e = errno; - closefd(&sd); - errno = e; - return (-1); - } #ifdef IPV6_V6ONLY /* forcibly use separate sockets for IPv4 and IPv6 */ From phk at FreeBSD.org Mon May 6 19:57:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 6 May 2019 19:57:08 +0000 (UTC) Subject: [master] d7a81fe82 Introduce a VSS_ResolveOne() function and use it the places where we want a single unique suckaddr. Message-ID: <20190506195708.3D9F2BFA94@lists.varnish-cache.org> commit d7a81fe82ca09c9f3b3a9fd67265de00da7a4315 Author: Poul-Henning Kamp Date: Mon May 6 19:55:58 2019 +0000 Introduce a VSS_ResolveOne() function and use it the places where we want a single unique suckaddr. diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index 86fc2f97d..867c48ccb 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -40,6 +40,7 @@ #include "vend.h" #include "vsa.h" +#include "vss.h" #include "vtcp.h" struct vpx_tlv { @@ -61,7 +62,6 @@ vpx_proto1(const struct worker *wrk, const struct req *req) const char *fld[5]; int i; char *p, *q; - struct addrinfo hints, *res; struct suckaddr *sa; int pfam = -1; @@ -100,58 +100,34 @@ vpx_proto1(const struct worker *wrk, const struct req *req) } if (!strcmp(fld[0], "TCP4")) - pfam = AF_INET; + pfam = PF_INET; else if (!strcmp(fld[0], "TCP6")) - pfam = AF_INET6; + pfam = PF_INET6; else { VSL(SLT_ProxyGarbage, req->sp->vxid, "PROXY1: Wrong TCP[46] field"); return (-1); } - memset(&hints, 0, sizeof hints); - hints.ai_socktype = SOCK_STREAM; - hints.ai_flags = AI_NUMERICHOST | AI_NUMERICSERV; - - i = getaddrinfo(fld[1], fld[3], &hints, &res); - if (i != 0) { - VSL(SLT_ProxyGarbage, req->sp->vxid, - "PROXY1: Cannot resolve source address (%s)", - gai_strerror(i)); - return (-1); - } - AZ(res->ai_next); - if (res->ai_family != pfam) { + SES_Reserve_client_addr(req->sp, &sa); + if (VSS_ResolveOne(sa, fld[1], fld[3], + pfam, SOCK_STREAM, AI_NUMERICHOST | AI_NUMERICSERV) == NULL) { VSL(SLT_ProxyGarbage, req->sp->vxid, - "PROXY1: %s got wrong protocol (%d)", - fld[0], res->ai_family); - freeaddrinfo(res); + "PROXY1: Cannot resolve source address"); return (-1); } - SES_Reserve_client_addr(req->sp, &sa); - AN(VSA_Build(sa, res->ai_addr, res->ai_addrlen)); SES_Set_String_Attr(req->sp, SA_CLIENT_IP, fld[1]); SES_Set_String_Attr(req->sp, SA_CLIENT_PORT, fld[3]); - freeaddrinfo(res); - i = getaddrinfo(fld[2], fld[4], &hints, &res); - if (i != 0) { - VSL(SLT_ProxyGarbage, req->sp->vxid, - "PROXY1: Cannot resolve destination address (%s)", - gai_strerror(i)); - return (-1); - } - AZ(res->ai_next); - if (res->ai_family != pfam) { + SES_Reserve_server_addr(req->sp, &sa); + if (VSS_ResolveOne(sa, fld[2], fld[4], + pfam, SOCK_STREAM, AI_NUMERICHOST | AI_NUMERICSERV) == NULL) { VSL(SLT_ProxyGarbage, req->sp->vxid, - "PROXY1: %s got wrong protocol (%d)", - fld[0], res->ai_family); - freeaddrinfo(res); + "PROXY1: Cannot resolve destination address"); return (-1); } - SES_Reserve_server_addr(req->sp, &sa); - AN(VSA_Build(sa, res->ai_addr, res->ai_addrlen)); - freeaddrinfo(res); + SES_Set_String_Attr(req->sp, SA_CLIENT_IP, fld[1]); + SES_Set_String_Attr(req->sp, SA_CLIENT_PORT, fld[3]); VSL(SLT_Proxy, req->sp->vxid, "1 %s %s %s %s", fld[1], fld[3], fld[2], fld[4]); diff --git a/bin/varnishtest/tests/o00000.vtc b/bin/varnishtest/tests/o00000.vtc index 1e5dcabf8..08459fbe4 100644 --- a/bin/varnishtest/tests/o00000.vtc +++ b/bin/varnishtest/tests/o00000.vtc @@ -34,10 +34,10 @@ logexpect l1 -v v1 { expect * 1002 ProxyGarbage "PROXY1: Too few fields" expect * 1003 ProxyGarbage "PROXY1: Too many fields" expect * 1004 ProxyGarbage "PROXY1: Wrong TCP\\[46\\] field" - expect * 1005 ProxyGarbage "PROXY1: Cannot resolve source address \$.*\$" - expect * 1007 ProxyGarbage "PROXY1: Cannot resolve destination address \$.*\$" - expect * 1013 ProxyGarbage "PROXY1: TCP4 got wrong protocol \$[0-9]*\$" - expect * 1014 ProxyGarbage "PROXY1: TCP6 got wrong protocol \$[0-9]*\$" + expect * 1005 ProxyGarbage "PROXY1: Cannot resolve source address" + expect * 1007 ProxyGarbage "PROXY1: Cannot resolve destination address" + expect * 1009 ProxyGarbage "PROXY1: Cannot resolve source address" + expect * 1011 ProxyGarbage "PROXY1: Cannot resolve source address" expect * 1015 Proxy "1 1.2.3.4 1234 5.6.7.8 5678" expect * 1018 Proxy "1 1:f::2 1234 5:a::8 5678" expect * 1021 Proxy "1 1:f::3 1234 5:a::8 5678" @@ -50,7 +50,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY " @@ -58,7 +58,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY A B C D\r\n" @@ -66,7 +66,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY A B C D E F\r\n" @@ -74,7 +74,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY A B C D E\r\n" @@ -82,7 +82,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY TCP4 B C D E\r\n" @@ -90,7 +90,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY TCP4 1.2.3.4 C D E\r\n" @@ -98,7 +98,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY TCP4 1.2.3.4 D 1234 E\r\n" @@ -106,7 +106,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY TCP4 1.2.3.4 5.6.7.8 1234 E\r\n" @@ -114,7 +114,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY TCP6 B C D E\r\n" @@ -122,7 +122,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY TCP6 1:f::2 C D E\r\n" @@ -130,7 +130,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY TCP6 1:f::2 1234 D E\r\n" @@ -138,7 +138,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY TCP6 1:f::2 5:a::8 1234 E\r\n" @@ -146,7 +146,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY TCP4 1:f::2 5:a::8 1234 5678\r\n" @@ -154,7 +154,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY TCP6 1.2.3.4 5.6.7.8 1234 5678\r\n" @@ -162,7 +162,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup # Finally try something which works... client c1 -proxy1 "1.2.3.4:1234 5.6.7.8:5678" { @@ -179,7 +179,7 @@ client c1 -proxy1 "1.2.3.4:1234 5.6.7.8:5678" { expect resp.http.rp != "1234" } -run -delay .1 +varnish v1 -vsl_catchup client c1 -proxy1 "[1:f::2]:1234 [5:a::8]:5678" { txreq -url /2 @@ -195,7 +195,7 @@ client c1 -proxy1 "[1:f::2]:1234 [5:a::8]:5678" { expect resp.http.rp != "1234" } -run -delay .1 +varnish v1 -vsl_catchup # Try with appended request (See also: #1728) client c2 { @@ -204,13 +204,15 @@ client c2 { expect resp.http.url == "/3" } -run +varnish v1 -vsl_catchup + # Malformed (missing \r) client c2 { send "PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678\n" expect_close } -run -delay .1 +varnish v1 -vsl_catchup # Malformed, too long (106) # NB: Should check VSL for proper disposal @@ -219,7 +221,7 @@ client c2 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup # Malformed, too long (107) # NB: Should check VSL for proper disposal @@ -228,7 +230,7 @@ client c2 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup # Malformed, too long (108) # NB: Should check VSL for proper disposal @@ -237,6 +239,6 @@ client c2 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup logexpect l1 -wait diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c index 783246e3e..da9033f48 100644 --- a/bin/varnishtest/vtc_client.c +++ b/bin/varnishtest/vtc_client.c @@ -31,6 +31,7 @@ #include #include +#include #include #include #include @@ -73,21 +74,11 @@ static VTAILQ_HEAD(, client) clients = * Send the proxy header */ -static int v_matchproto_(vss_resolved_f) -proxy_cb(void *priv, const struct suckaddr *sa) -{ - struct suckaddr **addr = priv; - *addr = VSA_Clone(sa); - return (1); -} - static void client_proxy(struct vtclog *vl, int fd, int version, const char *spec) { struct suckaddr *sac, *sas; - const char *err; char *p, *p2; - int error; p = strdup(spec); AN(p); @@ -95,14 +86,12 @@ client_proxy(struct vtclog *vl, int fd, int version, const char *spec) AN(p2); *p2++ = '\0'; - error = VSS_resolver(p, NULL, proxy_cb, &sac, &err); - if (err != NULL) - vtc_fatal(vl, "Could not resolve client address: %s", err); - assert(error == 1); - error = VSS_resolver(p2, NULL, proxy_cb, &sas, &err); - if (err != NULL) - vtc_fatal(vl, "Could not resolve server address: %s", err); - assert(error == 1); + sac = VSS_ResolveOne(NULL, p, NULL, 0, SOCK_STREAM, AI_PASSIVE); + if (sac == NULL) + vtc_fatal(vl, "Could not resolve client address"); + sas = VSS_ResolveOne(NULL, p2, NULL, 0, SOCK_STREAM, AI_PASSIVE); + if (sas == NULL) + vtc_fatal(vl, "Could not resolve server address"); if (vtc_send_proxy(fd, version, sac, sas)) vtc_fatal(vl, "Write failed: %s", strerror(errno)); free(p); diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index f90a92973..20b32476b 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -38,6 +38,7 @@ #include #include #include +#include #include "vtc.h" @@ -550,18 +551,11 @@ i_mode(void) * Figure out what IP related magic */ -static int v_matchproto_(vss_resolved_f) -bind_cb(void *priv, const struct suckaddr *sa) -{ - (void)priv; - return (VTCP_bind(sa, NULL)); -} - static void ip_magic(void) { - const char *p; int fd; + struct suckaddr *sa; char abuf[VTCP_ADDRBUFSIZE]; char pbuf[VTCP_PORTBUFSIZE]; @@ -572,7 +566,9 @@ ip_magic(void) * XXX: "localhost", but that doesn't work out of the box. * XXX: Things like "prefer_ipv6" parameter complicates things. */ - fd = VSS_resolver("127.0.0.1", NULL, bind_cb, NULL, &p); + sa = VSS_ResolveOne(NULL, "127.0.0.1", "0", 0, SOCK_STREAM, 0); + AN(sa); + fd = VTCP_bind(sa, NULL); assert(fd >= 0); VTCP_myname(fd, abuf, sizeof abuf, pbuf, sizeof(pbuf)); extmacro_def("localhost", "%s", abuf); diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index 69c1c2c75..8009820e4 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -344,30 +344,19 @@ cmd_delay(CMD_ARGS) * DNS services. This is a basic sanity check for those. */ -static int v_matchproto_(vss_resolved_f) -dns_cb(void *priv, const struct suckaddr *sa) +static int +dns_works(void) { + struct suckaddr *sa; char abuf[VTCP_ADDRBUFSIZE]; char pbuf[VTCP_PORTBUFSIZE]; - int *ret = priv; + sa = VSS_ResolveOne(NULL, "dns-canary.freebsd.dk", NULL, 0, 0, 0); + if (sa == NULL) + return (0); VTCP_name(sa, abuf, sizeof abuf, pbuf, sizeof pbuf); - if (strcmp(abuf, "192.0.2.255")) { - fprintf(stderr, "DNS-test: Wrong response: %s\n", abuf); - *ret = -1; - } else if (*ret == 0) - *ret = 1; - return (0); -} - -static int -dns_works(void) -{ - int ret = 0, error; - const char *msg; - - error = VSS_resolver("dns-canary.freebsd.dk", NULL, dns_cb, &ret, &msg); - if (error || msg != NULL || ret != 1) + free(sa); + if (strcmp(abuf, "192.0.2.255")) return (0); return (1); } diff --git a/include/vss.h b/include/vss.h index aa645bdc7..ab4cdc33d 100644 --- a/include/vss.h +++ b/include/vss.h @@ -34,3 +34,6 @@ int VSS_resolver(const char *addr, const char *def_port, vss_resolved_f *func, void *priv, const char **err); int VSS_resolver_socktype(const char *addr, const char *def_port, vss_resolved_f *func, void *priv, const char **err, int socktype); +struct suckaddr *VSS_ResolveOne(void *dst, + const char *addr, const char *port, + int family, int socktype, int flags); diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c index 7282019de..38c766203 100644 --- a/lib/libvarnish/vss.c +++ b/lib/libvarnish/vss.c @@ -154,3 +154,42 @@ VSS_resolver(const char *addr, const char *def_port, vss_resolved_f *func, return (VSS_resolver_socktype( addr, def_port, func, priv, err, SOCK_STREAM)); } + +#include + +struct suckaddr * +VSS_ResolveOne(void *dst, const char *addr, const char *port, + int family, int socktype, int flags) +{ + struct addrinfo hints, *res = NULL; + struct suckaddr *retval = NULL; + char *p = NULL, *hp, *pp; + int error; + + memset(&hints, 0, sizeof hints); + hints.ai_family = family; + hints.ai_socktype = socktype; + hints.ai_flags = flags; + + AN(addr); + if (port != NULL) { + error = getaddrinfo(addr, port, &hints, &res); + } else { + p = strdup(addr); + AN(p); + if (vss_parse(p, &hp, &pp) != NULL || pp == NULL) { + free(p); + return (NULL); + } + error = getaddrinfo(hp, pp, &hints, &res); + free(p); + } + if (!error && res != NULL && res->ai_next == NULL) { + if (dst == NULL) + retval = VSA_Malloc(res->ai_addr, res->ai_addrlen); + else + retval = VSA_Build(dst, res->ai_addr, res->ai_addrlen); + freeaddrinfo(res); + } + return (retval); +} diff --git a/lib/libvmod_debug/vmod_debug_dyn.c b/lib/libvmod_debug/vmod_debug_dyn.c index fca6bdb86..6dc0af797 100644 --- a/lib/libvmod_debug/vmod_debug_dyn.c +++ b/lib/libvmod_debug/vmod_debug_dyn.c @@ -38,6 +38,7 @@ #include "cache/cache.h" #include "vsa.h" +#include "vss.h" #include "vcc_if.h" struct xyzzy_debug_dyn { @@ -60,7 +61,6 @@ static void dyn_dir_init(VRT_CTX, struct xyzzy_debug_dyn *dyn, VCL_STRING addr, VCL_STRING port, VCL_PROBE probe) { - struct addrinfo hints, *res = NULL; struct suckaddr *sa; VCL_BACKEND dir, dir2; struct vrt_backend vrt; @@ -75,13 +75,7 @@ dyn_dir_init(VRT_CTX, struct xyzzy_debug_dyn *dyn, vrt.hosthdr = addr; vrt.probe = probe; - memset(&hints, 0, sizeof(hints)); - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - AZ(getaddrinfo(addr, port, &hints, &res)); - XXXAZ(res->ai_next); - - sa = VSA_Malloc(res->ai_addr, res->ai_addrlen); + sa = VSS_ResolveOne(NULL, addr, port, AF_UNSPEC, SOCK_STREAM, 0); AN(sa); if (VSA_Get_Proto(sa) == AF_INET) { vrt.ipv4_addr = addr; @@ -92,8 +86,6 @@ dyn_dir_init(VRT_CTX, struct xyzzy_debug_dyn *dyn, } else WRONG("Wrong proto family"); - freeaddrinfo(res); - dir = VRT_new_backend(ctx, &vrt); AN(dir); diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index 7bb4a67ff..64aa12691 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -40,6 +40,7 @@ #include "vnum.h" #include "vsa.h" +#include "vss.h" #include "vtim.h" #include "vcc_if.h" @@ -189,34 +190,6 @@ vmod_integer(VRT_CTX, struct VARGS(integer) *a) return (0); } -static VCL_IP -lookup(VCL_IP *p, const char *s, int resolve) -{ - struct addrinfo hints, *res0 = NULL; - const struct addrinfo *res; - int error; - VCL_IP retval = NULL; - - if (s == NULL) - return (retval); - - memset(&hints, 0, sizeof(hints)); - hints.ai_family = PF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - if (!resolve) - hints.ai_flags |= AI_NUMERICHOST; - error = getaddrinfo(s, "80", &hints, &res0); - if (!error) { - for (res = res0; res != NULL; res = res->ai_next) { - retval = VSA_Build(p, res->ai_addr, res->ai_addrlen); - if (retval != NULL) - break; - } - freeaddrinfo(res0); - } - return (retval); -} - VCL_IP vmod_ip(VRT_CTX, struct VARGS(ip) *a) { @@ -234,7 +207,8 @@ vmod_ip(VRT_CTX, struct VARGS(ip) *a) return (NULL); } - retval = lookup(p, a->s, a->resolve); + retval = VSS_ResolveOne(p, a->s, "80", PF_UNSPEC, SOCK_STREAM, + a->resolve ? 0 : AI_NUMERICHOST); if (retval != NULL) return (retval); From phk at FreeBSD.org Tue May 7 17:07:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 7 May 2019 17:07:10 +0000 (UTC) Subject: [master] 5df6f8d46 Change VRT_re[fl]_vcl() to VRT_VCL_{Prevent|Allow}_Discard() Message-ID: <20190507170710.4959610C20F@lists.varnish-cache.org> commit 5df6f8d46cb08ae435f866b8e6c1bb333cebdee3 Author: Poul-Henning Kamp Date: Tue May 7 14:47:38 2019 +0000 Change VRT_re[fl]_vcl() to VRT_VCL_{Prevent|Allow}_Discard() diff --git a/bin/varnishd/cache/cache_vcl.h b/bin/varnishd/cache/cache_vcl.h index aa530cb5b..d0154266f 100644 --- a/bin/varnishd/cache/cache_vcl.h +++ b/bin/varnishd/cache/cache_vcl.h @@ -60,7 +60,7 @@ struct vcl { struct vclref { unsigned magic; #define VCLREF_MAGIC 0x47fb6848 - const struct vcl *vcl; + struct vcl *vcl; VTAILQ_ENTRY(vclref) list; char desc[32]; }; diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c index 30fd2a26a..db1e15243 100644 --- a/bin/varnishd/cache/cache_vrt_vcl.c +++ b/bin/varnishd/cache/cache_vrt_vcl.c @@ -323,7 +323,7 @@ VCL_DefaultProbe(const struct vcl *vcl) } struct vclref * -VRT_ref_vcl(VRT_CTX, const char *desc) +VRT_VCL_Prevent_Discard(VRT_CTX, const char *desc) { struct vcl *vcl; struct vclref* ref; @@ -351,7 +351,7 @@ VRT_ref_vcl(VRT_CTX, const char *desc) } void -VRT_rel_vcl(VRT_CTX, struct vclref **refp) +VRT_VCL_Allow_Discard(struct vclref **refp) { struct vcl *vcl; struct vclref *ref; @@ -360,12 +360,9 @@ VRT_rel_vcl(VRT_CTX, struct vclref **refp) ref = *refp; *refp = NULL; - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ref, VCLREF_MAGIC); - - vcl = ctx->vcl; + vcl = ref->vcl; CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - assert(vcl == ref->vcl); /* NB: A VCL may be released by a VMOD at any time, but it must happen * after a warmup and before the end of a cooldown. The release may or diff --git a/include/vrt.h b/include/vrt.h index f5789e9e0..8db9bf88c 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -52,6 +52,7 @@ * binary/load-time compatible, increment MAJOR version * * unreleased (planned for 2019-09-15) + * VRT_re[fl]_vcl changed to VRT_VCL_{Prevent|Allow}_Discard * VRT_Vmod_{Init|Unload} moved to vcc_interface.h * VRT_count moved to vcc_interface.h * VRT_VCL_Busy() and VRT_VCL_Unbusy() added. @@ -523,8 +524,8 @@ struct vmod_priv { }; struct vclref; -struct vclref * VRT_ref_vcl(VRT_CTX, const char *); -void VRT_rel_vcl(VRT_CTX, struct vclref **); +struct vclref * VRT_VCL_Prevent_Discard(VRT_CTX, const char *); +void VRT_VCL_Allow_Discard(struct vclref **); void VRT_priv_fini(const struct vmod_priv *p); struct vmod_priv *VRT_priv_task(VRT_CTX, const void *vmod_id); diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index e2fef3125..1e6d06078 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -47,7 +47,6 @@ struct priv_vcl { #define PRIV_VCL_MAGIC 0x8E62FA9D char *foo; uintptr_t obj_cb; - struct vcl *vcl; struct vclref *vclref; }; @@ -341,7 +340,6 @@ priv_vcl_free(void *priv) ObjUnsubscribeEvents(&priv_vcl->obj_cb); VSL(SLT_Debug, 0, "Unsubscribed from Object Events"); } - AZ(priv_vcl->vcl); AZ(priv_vcl->vclref); FREE_OBJ(priv_vcl); AZ(priv_vcl); @@ -393,31 +391,23 @@ event_warm(VRT_CTX, const struct vmod_priv *priv) } CAST_OBJ_NOTNULL(priv_vcl, priv->priv, PRIV_VCL_MAGIC); - AZ(priv_vcl->vcl); AZ(priv_vcl->vclref); bprintf(buf, "vmod-debug ref on %s", VCL_Name(ctx->vcl)); - priv_vcl->vcl = ctx->vcl; - priv_vcl->vclref = VRT_ref_vcl(ctx, buf); + priv_vcl->vclref = VRT_VCL_Prevent_Discard(ctx, buf); return (0); } static void* cooldown_thread(void *priv) { - struct vrt_ctx ctx; struct priv_vcl *priv_vcl; CAST_OBJ_NOTNULL(priv_vcl, priv, PRIV_VCL_MAGIC); - AN(priv_vcl->vcl); AN(priv_vcl->vclref); - INIT_OBJ(&ctx, VRT_CTX_MAGIC); - ctx.vcl = priv_vcl->vcl; - VTIM_sleep(vcl_release_delay); - VRT_rel_vcl(&ctx, &priv_vcl->vclref); - priv_vcl->vcl = NULL; + VRT_VCL_Allow_Discard(&priv_vcl->vclref); return (NULL); } @@ -428,14 +418,12 @@ event_cold(VRT_CTX, const struct vmod_priv *priv) struct priv_vcl *priv_vcl; CAST_OBJ_NOTNULL(priv_vcl, priv->priv, PRIV_VCL_MAGIC); - AN(priv_vcl->vcl); AN(priv_vcl->vclref); VSL(SLT_Debug, 0, "%s: VCL_EVENT_COLD", VCL_Name(ctx->vcl)); if (vcl_release_delay == 0.0) { - VRT_rel_vcl(ctx, &priv_vcl->vclref); - priv_vcl->vcl = NULL; + VRT_VCL_Allow_Discard(&priv_vcl->vclref); return (0); } From phk at FreeBSD.org Wed May 8 08:43:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 8 May 2019 08:43:09 +0000 (UTC) Subject: [master] 1d0298e1b Bring more clarity to what and how VMODs can veto cold&discard for their VCL. Message-ID: <20190508084309.F148C7801@lists.varnish-cache.org> commit 1d0298e1b4a1f106bbcebe8069ed7aa8d17582f8 Author: Poul-Henning Kamp Date: Wed May 8 08:41:34 2019 +0000 Bring more clarity to what and how VMODs can veto cold&discard for their VCL. Closes: #2471 diff --git a/bin/varnishd/cache/cache_vcl.h b/bin/varnishd/cache/cache_vcl.h index d0154266f..ac01706e2 100644 --- a/bin/varnishd/cache/cache_vcl.h +++ b/bin/varnishd/cache/cache_vcl.h @@ -62,7 +62,7 @@ struct vclref { #define VCLREF_MAGIC 0x47fb6848 struct vcl *vcl; VTAILQ_ENTRY(vclref) list; - char desc[32]; + char *desc; }; extern struct lock vcl_mtx; diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c index db1e15243..451d7aea3 100644 --- a/bin/varnishd/cache/cache_vrt_vcl.c +++ b/bin/varnishd/cache/cache_vrt_vcl.c @@ -142,28 +142,6 @@ VCL_Rel(struct vcl **vcc) /*--------------------------------------------------------------------*/ -void -VRT_VCL_Busy(VRT_CTX) -{ - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - - CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); - VCL_Ref(ctx->vcl); -} - -void -VRT_VCL_Unbusy(VRT_CTX) -{ - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - struct vcl *vcl; - - CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); - vcl = ctx->vcl; - VCL_Rel(&vcl); -} - -/*--------------------------------------------------------------------*/ - VCL_BACKEND VRT_AddDirector(VRT_CTX, const struct vdi_methods *m, void *priv, const char *fmt, ...) @@ -322,6 +300,55 @@ VCL_DefaultProbe(const struct vcl *vcl) return (vcl->conf->default_probe); } +/*--------------------------------------------------------------------*/ + +struct vclref * +VRT_VCL_Prevent_Cold(VRT_CTX, const char *desc) +{ + struct vclref* ref; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); + + ALLOC_OBJ(ref, VCLREF_MAGIC); + AN(ref); + ref->vcl = ctx->vcl; + REPLACE(ref->desc, desc); + + VCL_Ref(ctx->vcl); + + Lck_Lock(&vcl_mtx); + VTAILQ_INSERT_TAIL(&ctx->vcl->ref_list, ref, list); + Lck_Unlock(&vcl_mtx); + + return(ref); +} + +void +VRT_VCL_Allow_Cold(struct vclref **refp) +{ + struct vcl *vcl; + struct vclref *ref; + + AN(refp); + ref = *refp; + *refp = NULL; + + CHECK_OBJ_NOTNULL(ref, VCLREF_MAGIC); + vcl = ref->vcl; + CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); + + Lck_Lock(&vcl_mtx); + assert(!VTAILQ_EMPTY(&vcl->ref_list)); + VTAILQ_REMOVE(&vcl->ref_list, ref, list); + Lck_Unlock(&vcl_mtx); + + VCL_Rel(&vcl); + + REPLACE(ref->desc, NULL); + FREE_OBJ(ref); +} + struct vclref * VRT_VCL_Prevent_Discard(VRT_CTX, const char *desc) { @@ -340,7 +367,7 @@ VRT_VCL_Prevent_Discard(VRT_CTX, const char *desc) ALLOC_OBJ(ref, VCLREF_MAGIC); AN(ref); ref->vcl = vcl; - bprintf(ref->desc, "%s", desc); + REPLACE(ref->desc, desc); Lck_Lock(&vcl_mtx); VTAILQ_INSERT_TAIL(&vcl->ref_list, ref, list); @@ -377,6 +404,7 @@ VRT_VCL_Allow_Discard(struct vclref **refp) /* No garbage collection here, for the same reasons as in VCL_Rel. */ Lck_Unlock(&vcl_mtx); + REPLACE(ref->desc, NULL); FREE_OBJ(ref); } From phk at FreeBSD.org Wed May 8 08:52:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 8 May 2019 08:52:07 +0000 (UTC) Subject: [master] 54908ac19 Duh! Last committed from wrong directory. Message-ID: <20190508085207.E48DB7D24@lists.varnish-cache.org> commit 54908ac19ffce067fe231cb5e8be1ee7bf77683d Author: Poul-Henning Kamp Date: Wed May 8 08:50:42 2019 +0000 Duh! Last committed from wrong directory. diff --git a/bin/varnishtest/tests/r02471.vtc b/bin/varnishtest/tests/r02471.vtc index cbc2165e0..a573d4a3c 100644 --- a/bin/varnishtest/tests/r02471.vtc +++ b/bin/varnishtest/tests/r02471.vtc @@ -10,9 +10,9 @@ varnish v1 -vcl+backend { sub vcl_recv { if (req.url == "/hold") { - debug.hold_vcl_busy(); + debug.vcl_prevent_cold(); } else if (req.url == "/release") { - debug.release_vcl_busy(); + debug.vcl_allow_cold(); } return (synth(200)); } diff --git a/bin/varnishtest/tests/v00045.vtc b/bin/varnishtest/tests/v00045.vtc index 2591f9d36..60504ce73 100644 --- a/bin/varnishtest/tests/v00045.vtc +++ b/bin/varnishtest/tests/v00045.vtc @@ -6,7 +6,7 @@ server s1 -start varnish v1 -vcl+backend { import debug; sub vcl_init { - debug.vcl_release_delay(3s); + debug.vcl_discard_delay(3s); } } -start diff --git a/include/vrt.h b/include/vrt.h index 8db9bf88c..87319107f 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -52,10 +52,11 @@ * binary/load-time compatible, increment MAJOR version * * unreleased (planned for 2019-09-15) + * VRT_VCL_{Busy|Unbusy} changed to VRT_VCL_{Prevent|Allow}_Cold * VRT_re[fl]_vcl changed to VRT_VCL_{Prevent|Allow}_Discard * VRT_Vmod_{Init|Unload} moved to vcc_interface.h * VRT_count moved to vcc_interface.h - * VRT_VCL_Busy() and VRT_VCL_Unbusy() added. + * VRT_VCL_Prevent_Cold() and VRT_VCL_Allow_Cold() added. * VRT_vcl_get moved to vcc_interface.h * VRT_vcl_rel emoved to vcc_interface.h * VRT_vcl_select emoved to vcc_interface.h @@ -523,10 +524,6 @@ struct vmod_priv { vmod_priv_free_f *free; }; -struct vclref; -struct vclref * VRT_VCL_Prevent_Discard(VRT_CTX, const char *); -void VRT_VCL_Allow_Discard(struct vclref **); - void VRT_priv_fini(const struct vmod_priv *p); struct vmod_priv *VRT_priv_task(VRT_CTX, const void *vmod_id); struct vmod_priv *VRT_priv_top(VRT_CTX, const void *vmod_id); @@ -563,8 +560,12 @@ void VRT_VSC_Reveal(const struct vsc_seg *); size_t VRT_VSC_Overhead(size_t); /* - * API to prevent VCL from going cold + * API to restrict the VCL in various ways */ -void VRT_VCL_Busy(VRT_CTX); -void VRT_VCL_Unbusy(VRT_CTX); +struct vclref; +struct vclref * VRT_VCL_Prevent_Cold(VRT_CTX, const char *); +void VRT_VCL_Allow_Cold(struct vclref **); + +struct vclref * VRT_VCL_Prevent_Discard(VRT_CTX, const char *); +void VRT_VCL_Allow_Discard(struct vclref **); diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index 20ee901da..aa2a3634e 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -160,9 +160,10 @@ $Method VOID .refresh(STRING path) Dynamically refresh & (always!) replace the backend by a new UDS backend. -$Function VOID vcl_release_delay(DURATION) +$Function VOID vcl_discard_delay(PRIV_VCL, DURATION) -Hold a reference to the VCL when it goes cold for the given delay. +Hold a reference to the VCL when it goes cold preventing +discard for the given delay. $Function BOOL match_acl(ACL acl, IP ip) @@ -244,11 +245,11 @@ should now only be used for diagnostic purposes. 0B is returned if no sensible value can be determined. -$Function VOID hold_vcl_busy() +$Function VOID vcl_prevent_cold(PRIV_VCL) Prevent VCL from going cold -$Function VOID release_vcl_busy() +$Function VOID vcl_allow_cold(PRIV_VCL) Allow VCL to go cold diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 1e6d06078..4dac3f3dc 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -47,10 +47,11 @@ struct priv_vcl { #define PRIV_VCL_MAGIC 0x8E62FA9D char *foo; uintptr_t obj_cb; - struct vclref *vclref; + struct vclref *vclref_discard; + struct vclref *vclref_cold; + VCL_DURATION vcl_discard_delay; }; -static VCL_DURATION vcl_release_delay = 0.0; static pthread_mutex_t vsc_mtx = PTHREAD_MUTEX_INITIALIZER; static struct vsc_seg *vsc_seg = NULL; @@ -340,7 +341,8 @@ priv_vcl_free(void *priv) ObjUnsubscribeEvents(&priv_vcl->obj_cb); VSL(SLT_Debug, 0, "Unsubscribed from Object Events"); } - AZ(priv_vcl->vclref); + AZ(priv_vcl->vclref_discard); + AZ(priv_vcl->vclref_cold); FREE_OBJ(priv_vcl); AZ(priv_vcl); } @@ -376,6 +378,32 @@ event_load(VRT_CTX, struct vmod_priv *priv) return (0); } +VCL_VOID +xyzzy_vcl_prevent_cold(VRT_CTX, struct vmod_priv *priv) +{ + struct priv_vcl *priv_vcl; + char buf[32]; + + CAST_OBJ_NOTNULL(priv_vcl, priv->priv, PRIV_VCL_MAGIC); + AZ(priv_vcl->vclref_cold); + + bprintf(buf, "vmod-debug ref on %s", VCL_Name(ctx->vcl)); + priv_vcl->vclref_cold = VRT_VCL_Prevent_Cold(ctx, buf); +} + +VCL_VOID +xyzzy_vcl_allow_cold(VRT_CTX, struct vmod_priv *priv) +{ + struct priv_vcl *priv_vcl; + + (void)ctx; + + CAST_OBJ_NOTNULL(priv_vcl, priv->priv, PRIV_VCL_MAGIC); + AN(priv_vcl->vclref_cold); + VRT_VCL_Allow_Cold(&priv_vcl->vclref_cold); +} + + static int event_warm(VRT_CTX, const struct vmod_priv *priv) { @@ -391,10 +419,10 @@ event_warm(VRT_CTX, const struct vmod_priv *priv) } CAST_OBJ_NOTNULL(priv_vcl, priv->priv, PRIV_VCL_MAGIC); - AZ(priv_vcl->vclref); + AZ(priv_vcl->vclref_discard); bprintf(buf, "vmod-debug ref on %s", VCL_Name(ctx->vcl)); - priv_vcl->vclref = VRT_VCL_Prevent_Discard(ctx, buf); + priv_vcl->vclref_discard = VRT_VCL_Prevent_Discard(ctx, buf); return (0); } @@ -404,10 +432,10 @@ cooldown_thread(void *priv) struct priv_vcl *priv_vcl; CAST_OBJ_NOTNULL(priv_vcl, priv, PRIV_VCL_MAGIC); - AN(priv_vcl->vclref); + AN(priv_vcl->vclref_discard); - VTIM_sleep(vcl_release_delay); - VRT_VCL_Allow_Discard(&priv_vcl->vclref); + VTIM_sleep(priv_vcl->vcl_discard_delay); + VRT_VCL_Allow_Discard(&priv_vcl->vclref_discard); return (NULL); } @@ -418,12 +446,12 @@ event_cold(VRT_CTX, const struct vmod_priv *priv) struct priv_vcl *priv_vcl; CAST_OBJ_NOTNULL(priv_vcl, priv->priv, PRIV_VCL_MAGIC); - AN(priv_vcl->vclref); + AN(priv_vcl->vclref_discard); VSL(SLT_Debug, 0, "%s: VCL_EVENT_COLD", VCL_Name(ctx->vcl)); - if (vcl_release_delay == 0.0) { - VRT_VCL_Allow_Discard(&priv_vcl->vclref); + if (priv_vcl->vcl_discard_delay == 0.0) { + VRT_VCL_Allow_Discard(&priv_vcl->vclref_discard); return (0); } @@ -469,13 +497,15 @@ xyzzy_event_function(VRT_CTX, struct vmod_priv *priv, enum vcl_event_e e) } } -VCL_VOID v_matchproto_(td_debug_vcl_release_delay) -xyzzy_vcl_release_delay(VRT_CTX, VCL_DURATION delay) +VCL_VOID v_matchproto_(td_debug_vcl_discard_delay) +xyzzy_vcl_discard_delay(VRT_CTX, struct vmod_priv *priv, VCL_DURATION delay) { + struct priv_vcl *priv_vcl; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CAST_OBJ_NOTNULL(priv_vcl, priv->priv, PRIV_VCL_MAGIC); assert(delay > 0.0); - vcl_release_delay = delay; + priv_vcl->vcl_discard_delay = delay; } VCL_BOOL v_matchproto_(td_debug_match_acl) @@ -782,20 +812,6 @@ xyzzy_stk(VRT_CTX) return (0); } -VCL_VOID -xyzzy_hold_vcl_busy(VRT_CTX) -{ - - VRT_VCL_Busy(ctx); -} - -VCL_VOID -xyzzy_release_vcl_busy(VRT_CTX) -{ - - VRT_VCL_Unbusy(ctx); -} - VCL_VOID xyzzy_sndbuf(VRT_CTX, VCL_BYTES arg) { From phk at FreeBSD.org Thu May 9 06:04:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 9 May 2019 06:04:10 +0000 (UTC) Subject: [master] 1f5526a72 Coverity doesnt know that getaddrinfo only allocates res on success. Message-ID: <20190509060410.5A327110EF8@lists.varnish-cache.org> commit 1f5526a72e63d1d1db4f9a944f8b63ea386d0e6e Author: Poul-Henning Kamp Date: Thu May 9 06:03:02 2019 +0000 Coverity doesnt know that getaddrinfo only allocates res on success. diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c index 38c766203..0ad31a5b9 100644 --- a/lib/libvarnish/vss.c +++ b/lib/libvarnish/vss.c @@ -189,7 +189,7 @@ VSS_ResolveOne(void *dst, const char *addr, const char *port, retval = VSA_Malloc(res->ai_addr, res->ai_addrlen); else retval = VSA_Build(dst, res->ai_addr, res->ai_addrlen); - freeaddrinfo(res); } + freeaddrinfo(res); return (retval); } From dridi at varni.sh Thu May 9 06:59:23 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Thu, 9 May 2019 08:59:23 +0200 Subject: [master] 1f5526a72 Coverity doesnt know that getaddrinfo only allocates res on success. In-Reply-To: <20190509060410.5A327110EF8@lists.varnish-cache.org> References: <20190509060410.5A327110EF8@lists.varnish-cache.org> Message-ID: On Thu, May 9, 2019 at 8:04 AM Poul-Henning Kamp wrote: > > > commit 1f5526a72e63d1d1db4f9a944f8b63ea386d0e6e > Author: Poul-Henning Kamp > Date: Thu May 9 06:03:02 2019 +0000 > > Coverity doesnt know that getaddrinfo only allocates res on success. > > diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c > index 38c766203..0ad31a5b9 100644 > --- a/lib/libvarnish/vss.c > +++ b/lib/libvarnish/vss.c > @@ -189,7 +189,7 @@ VSS_ResolveOne(void *dst, const char *addr, const char *port, > retval = VSA_Malloc(res->ai_addr, res->ai_addrlen); > else > retval = VSA_Build(dst, res->ai_addr, res->ai_addrlen); > - freeaddrinfo(res); > } > + freeaddrinfo(res); It fails badly on SunOS, I think we should revert this and mark the Coverity defect as a false positive instead. > return (retval); > } > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From phk at FreeBSD.org Thu May 9 07:18:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 9 May 2019 07:18:09 +0000 (UTC) Subject: [master] c8559db55 Only free addrinfo if it is non-NULL Message-ID: <20190509071809.66AB01124F7@lists.varnish-cache.org> commit c8559db555f63d4cddfef845ff4d0d8f48c9b5a3 Author: Poul-Henning Kamp Date: Thu May 9 07:17:10 2019 +0000 Only free addrinfo if it is non-NULL diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c index 0ad31a5b9..4431dea0d 100644 --- a/lib/libvarnish/vss.c +++ b/lib/libvarnish/vss.c @@ -108,7 +108,7 @@ int VSS_resolver_socktype(const char *addr, const char *def_port, vss_resolved_f *func, void *priv, const char **err, int socktype) { - struct addrinfo hints, *res0, *res; + struct addrinfo hints, *res0 = NULL, *res; struct suckaddr *vsa; char *h; char *adp, *hop; @@ -143,7 +143,8 @@ VSS_resolver_socktype(const char *addr, const char *def_port, break; } } - freeaddrinfo(res0); + if (res0 != NULL) + freeaddrinfo(res0); return (ret); } From phk at FreeBSD.org Thu May 9 07:44:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 9 May 2019 07:44:08 +0000 (UTC) Subject: [master] cf7c39281 Duh: Never rush a fix. Message-ID: <20190509074408.A0309112D23@lists.varnish-cache.org> commit cf7c39281b392cabe81ad49d5221e53e65cc5337 Author: Poul-Henning Kamp Date: Thu May 9 07:43:21 2019 +0000 Duh: Never rush a fix. diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c index 4431dea0d..410fab9f3 100644 --- a/lib/libvarnish/vss.c +++ b/lib/libvarnish/vss.c @@ -108,7 +108,7 @@ int VSS_resolver_socktype(const char *addr, const char *def_port, vss_resolved_f *func, void *priv, const char **err, int socktype) { - struct addrinfo hints, *res0 = NULL, *res; + struct addrinfo hints, *res0, *res; struct suckaddr *vsa; char *h; char *adp, *hop; @@ -143,8 +143,7 @@ VSS_resolver_socktype(const char *addr, const char *def_port, break; } } - if (res0 != NULL) - freeaddrinfo(res0); + freeaddrinfo(res0); return (ret); } @@ -191,6 +190,7 @@ VSS_ResolveOne(void *dst, const char *addr, const char *port, else retval = VSA_Build(dst, res->ai_addr, res->ai_addrlen); } - freeaddrinfo(res); + if (res != NULL) + freeaddrinfo(res); return (retval); } From dridi at varni.sh Thu May 9 07:43:42 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Thu, 9 May 2019 09:43:42 +0200 Subject: [master] c8559db55 Only free addrinfo if it is non-NULL In-Reply-To: <20190509071809.66AB01124F7@lists.varnish-cache.org> References: <20190509071809.66AB01124F7@lists.varnish-cache.org> Message-ID: On Thu, May 9, 2019 at 9:18 AM Poul-Henning Kamp wrote: > > > commit c8559db555f63d4cddfef845ff4d0d8f48c9b5a3 > Author: Poul-Henning Kamp > Date: Thu May 9 07:17:10 2019 +0000 > > Only free addrinfo if it is non-NULL > > diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c > index 0ad31a5b9..4431dea0d 100644 > --- a/lib/libvarnish/vss.c > +++ b/lib/libvarnish/vss.c > @@ -108,7 +108,7 @@ int > VSS_resolver_socktype(const char *addr, const char *def_port, > vss_resolved_f *func, void *priv, const char **err, int socktype) > { > - struct addrinfo hints, *res0, *res; > + struct addrinfo hints, *res0 = NULL, *res; > struct suckaddr *vsa; > char *h; > char *adp, *hop; > @@ -143,7 +143,8 @@ VSS_resolver_socktype(const char *addr, const char *def_port, > break; > } > } > - freeaddrinfo(res0); > + if (res0 != NULL) > + freeaddrinfo(res0); For some reason, still failing on SunOS. I think we should really mark the Coverity defect as a false positive instead so they can fix it. > return (ret); > } > > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi at varni.sh Thu May 9 07:56:22 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Thu, 9 May 2019 09:56:22 +0200 Subject: [master] cf7c39281 Duh: Never rush a fix. In-Reply-To: <20190509074408.A0309112D23@lists.varnish-cache.org> References: <20190509074408.A0309112D23@lists.varnish-cache.org> Message-ID: On Thu, May 9, 2019 at 9:44 AM Poul-Henning Kamp wrote: > > > commit cf7c39281b392cabe81ad49d5221e53e65cc5337 > Author: Poul-Henning Kamp > Date: Thu May 9 07:43:21 2019 +0000 > > Duh: Never rush a fix. Looks better now, I think we can leave Coverity alone this time. > diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c > index 4431dea0d..410fab9f3 100644 > --- a/lib/libvarnish/vss.c > +++ b/lib/libvarnish/vss.c > @@ -108,7 +108,7 @@ int > VSS_resolver_socktype(const char *addr, const char *def_port, > vss_resolved_f *func, void *priv, const char **err, int socktype) > { > - struct addrinfo hints, *res0 = NULL, *res; > + struct addrinfo hints, *res0, *res; > struct suckaddr *vsa; > char *h; > char *adp, *hop; > @@ -143,8 +143,7 @@ VSS_resolver_socktype(const char *addr, const char *def_port, > break; > } > } > - if (res0 != NULL) > - freeaddrinfo(res0); > + freeaddrinfo(res0); > return (ret); > } > > @@ -191,6 +190,7 @@ VSS_ResolveOne(void *dst, const char *addr, const char *port, > else > retval = VSA_Build(dst, res->ai_addr, res->ai_addrlen); > } > - freeaddrinfo(res); > + if (res != NULL) > + freeaddrinfo(res); > return (retval); > } > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi.boukelmoune at gmail.com Thu May 9 08:21:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 9 May 2019 08:21:09 +0000 (UTC) Subject: [master] 5c1e69602 GC #include Message-ID: <20190509082109.164BB113B10@lists.varnish-cache.org> commit 5c1e69602c017a0f48d8a84473ac57c50b4b3cbe Author: Dridi Boukelmoune Date: Thu May 9 10:18:12 2019 +0200 GC #include diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c index 410fab9f3..6ba726d85 100644 --- a/lib/libvarnish/vss.c +++ b/lib/libvarnish/vss.c @@ -155,8 +155,6 @@ VSS_resolver(const char *addr, const char *def_port, vss_resolved_f *func, addr, def_port, func, priv, err, SOCK_STREAM)); } -#include - struct suckaddr * VSS_ResolveOne(void *dst, const char *addr, const char *port, int family, int socktype, int flags) From dridi.boukelmoune at gmail.com Thu May 9 08:21:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 9 May 2019 08:21:09 +0000 (UTC) Subject: [master] ed66b6edd Improve vss_parse documentation Message-ID: <20190509082109.2BC53113B13@lists.varnish-cache.org> commit ed66b6edd27572b1838e1c703cb27e42a29b922a Author: Dridi Boukelmoune Date: Thu May 9 10:19:33 2019 +0200 Improve vss_parse documentation diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c index 6ba726d85..ebbdf4f2b 100644 --- a/lib/libvarnish/vss.c +++ b/lib/libvarnish/vss.c @@ -46,14 +46,15 @@ /* * Take a string provided by the user and break it up into address and - * port parts. Examples of acceptable input include: + * port parts. The address and port separator may be either a colon or + * a whitespace. Examples of acceptable input include: * - * "localhost" - "localhost:80" - * "127.0.0.1" - "127.0.0.1:80" - * "0.0.0.0" - "0.0.0.0:80" - * "[::1]" - "[::1]:80" - * "[::]" - "[::]:80" - * "::1" - "[::1]:80" + * "localhost" - "localhost:80" - "localhost 80" + * "127.0.0.1" - "127.0.0.1:80" - "127.0.0.1 80" + * "0.0.0.0" - "0.0.0.0:80" - "0.0.0.0 80" + * "[::1]" - "[::1]:80" - "[::1] 80" + * "[::]" - "[::]:80" - "[::] 80" + * "::1" - "[::1]:80" - "::1 80" * * See also RFC5952 */ From phk at FreeBSD.org Sun May 12 14:04:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 12 May 2019 14:04:08 +0000 (UTC) Subject: [master] 9cacb0c85 Make our bad-backend look less suspicious to Coverity Message-ID: <20190512140408.9829210024E@lists.varnish-cache.org> commit 9cacb0c859a051b5f370ea62b25c498ce1218834 Author: Poul-Henning Kamp Date: Sun May 12 13:57:39 2019 +0000 Make our bad-backend look less suspicious to Coverity diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 20b32476b..beb4bfe1d 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -108,6 +108,7 @@ struct vsb *params_vsb = NULL; int leave_temp; int vtc_witness = 0; static struct vsb *cbvsb; +static int bad_backend_fd; static int cleaner_fd = -1; static pid_t cleaner_pid; @@ -554,7 +555,6 @@ i_mode(void) static void ip_magic(void) { - int fd; struct suckaddr *sa; char abuf[VTCP_ADDRBUFSIZE]; char pbuf[VTCP_PORTBUFSIZE]; @@ -568,9 +568,9 @@ ip_magic(void) */ sa = VSS_ResolveOne(NULL, "127.0.0.1", "0", 0, SOCK_STREAM, 0); AN(sa); - fd = VTCP_bind(sa, NULL); - assert(fd >= 0); - VTCP_myname(fd, abuf, sizeof abuf, pbuf, sizeof(pbuf)); + bad_backend_fd = VTCP_bind(sa, NULL); + assert(bad_backend_fd >= 0); + VTCP_myname(bad_backend_fd, abuf, sizeof abuf, pbuf, sizeof(pbuf)); extmacro_def("localhost", "%s", abuf); #if defined (__APPLE__) @@ -579,7 +579,7 @@ ip_magic(void) * instead of refusing the connection so close it and hope * for the best. */ - VTCP_close(&fd); + VTCP_close(&bad_backend_fd); #endif /* Expose a backend that is forever down. */ @@ -793,6 +793,7 @@ main(int argc, char * const *argv) i = VEV_Once(vb); } cleaner_finish(); + (void)close(bad_backend_fd); if (vtc_continue) fprintf(stderr, "%d tests failed, %d tests skipped, %d tests passed\n", From phk at FreeBSD.org Mon May 13 05:46:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 13 May 2019 05:46:07 +0000 (UTC) Subject: [master] d404df0ad Make this test more patient for people silly enough to run it on an RPI3b. Message-ID: <20190513054607.73AE111137D@lists.varnish-cache.org> commit d404df0adbd1e10adcf1e4afb367c1d21af5746b Author: Poul-Henning Kamp Date: Mon May 13 05:44:48 2019 +0000 Make this test more patient for people silly enough to run it on an RPI3b. diff --git a/bin/varnishtest/tests/s00010.vtc b/bin/varnishtest/tests/s00010.vtc index eb71ddc57..456d48741 100644 --- a/bin/varnishtest/tests/s00010.vtc +++ b/bin/varnishtest/tests/s00010.vtc @@ -31,8 +31,8 @@ client c1 -rcvbuf 128 { txreq non_fatal rxresphdrs - # keep the session open for 2 seconds - delay 2 + # keep the session open for 4 seconds + delay 4 } -start client c1 -wait @@ -48,8 +48,8 @@ logexpect l2 -v v1 { client c2 -rcvbuf 128 { txreq rxresphdrs - # keep the session open for 2 seconds - delay 2 + # keep the session open for 4 seconds + delay 4 } -start client c2 -wait From phk at FreeBSD.org Mon May 13 07:27:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 13 May 2019 07:27:08 +0000 (UTC) Subject: [master] 4270a297c Try to make this test more stable on heavily loaded systems. Message-ID: <20190513072708.55C13112FD2@lists.varnish-cache.org> commit 4270a297c25c3252a3342751cf2082373e122402 Author: Poul-Henning Kamp Date: Mon May 13 07:26:27 2019 +0000 Try to make this test more stable on heavily loaded systems. diff --git a/bin/varnishtest/tests/c00094.vtc b/bin/varnishtest/tests/c00094.vtc index d8c7eef72..d0b35efd9 100644 --- a/bin/varnishtest/tests/c00094.vtc +++ b/bin/varnishtest/tests/c00094.vtc @@ -3,7 +3,7 @@ varnishtest "Test Backend Polling with a backend listening at a UDS" barrier b1 cond 2 server s1 -listen "${tmpdir}/s1.sock" { - timeout 3 + timeout 8 fatal # Probes @@ -44,7 +44,7 @@ varnish v1 -vcl { backend foo { .path = "${s1_sock}"; .probe = { - .timeout = 1 s; + .timeout = 7 s; .interval = 0.5 s; } } From phk at FreeBSD.org Mon May 13 17:45:16 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 13 May 2019 17:45:16 +0000 (UTC) Subject: [master] 62a934b7e Pick up default gcov processor from environment. Message-ID: <20190513174516.63D1298E7@lists.varnish-cache.org> commit 62a934b7e4fe17347cd1f4599052c76d71c89630 Author: Poul-Henning Kamp Date: Mon May 13 17:07:08 2019 +0000 Pick up default gcov processor from environment. diff --git a/tools/gcov_digest.py b/tools/gcov_digest.py index 90828a43f..797d9dea0 100644 --- a/tools/gcov_digest.py +++ b/tools/gcov_digest.py @@ -178,7 +178,9 @@ if __name__ == "__main__": optlist, args = getopt.getopt(sys.argv[1:], "g:o:x:") fo = sys.stdout - gcovprog = "gcov6 -r" + gcovprog = os.environ.get('GCOVPROG') + if gcovprog is None: + gcovprog = "gcov6 -r" for f, v in optlist: if f == '-o' and v == '-': From phk at FreeBSD.org Mon May 13 17:45:16 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 13 May 2019 17:45:16 +0000 (UTC) Subject: [master] 1aef67a54 Switch gcov to LLVM 8.0 Message-ID: <20190513174516.7D17F98EA@lists.varnish-cache.org> commit 1aef67a543ee0516b42a3bf700b0e1267a39229d Author: Poul-Henning Kamp Date: Mon May 13 17:43:55 2019 +0000 Switch gcov to LLVM 8.0 diff --git a/tools/vtest.sh b/tools/vtest.sh index 4464826e5..c4bc26e02 100755 --- a/tools/vtest.sh +++ b/tools/vtest.sh @@ -199,7 +199,9 @@ failedtests () ( ) if $enable_gcov ; then - export CC=gcc6 + #export CC=gcc6 + export CC=clang80 + export GCOVPROG='llvm-cov80 gcov' export CFLAGS="-fprofile-arcs -ftest-coverage -fstack-protector -DDONT_DLCLOSE_VMODS" export MAKEFLAGS=-j1 fi From fgsch at lodoss.net Mon May 13 22:09:12 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 13 May 2019 22:09:12 +0000 (UTC) Subject: [master] 3e02788d8 Polish and more coverage Message-ID: <20190513220912.B94C16CF70@lists.varnish-cache.org> commit 3e02788d8c7be9324afc55d841b22defd5047eb1 Author: Federico G. Schwindt Date: Mon May 13 11:49:44 2019 +0100 Polish and more coverage diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc index 5ddd706ce..b0e894419 100644 --- a/bin/varnishtest/tests/m00000.vtc +++ b/bin/varnishtest/tests/m00000.vtc @@ -44,6 +44,7 @@ varnish v1 -vcl+backend { set resp.http.bar = std.tolower(resp.http.bar); set resp.http.who = debug.author(mithrandir); set resp.http.really = debug.author(); + set resp.http.when = objx.date(); set resp.http.what = vtc.typesize("dfijlopsz"); set resp.http.not = vtc.typesize("*"); debug.test_priv_call(); @@ -67,6 +68,7 @@ client c1 { expect resp.http.bar == "foo" expect resp.http.who == "Tollef" expect resp.http.really == "Poul-Henning" + expect resp.http.when == "Thu, 01 Jan 1970 00:00:21 GMT" expect resp.http.encrypted == "ROT52" expect resp.http.what >= 16 expect resp.http.not == -1 @@ -91,6 +93,7 @@ logexpect l1 -v v1 -g raw -d 1 { expect 0 = RespHeader {^bar: foo} expect 0 = RespHeader {^who: Tollef} expect 0 = RespHeader {^really: Poul-Henning} + expect 0 = RespHeader {^when: Thu, 01 Jan 1970 00:00:21 GMT} expect 0 = RespHeader {^what: [1-9][0-9]} expect 0 = RespHeader {^not: -1} expect 0 = VCL_Log {^VCL initiated log} diff --git a/bin/varnishtest/tests/o00004.vtc b/bin/varnishtest/tests/o00004.vtc index 5effa2890..e582df24d 100644 --- a/bin/varnishtest/tests/o00004.vtc +++ b/bin/varnishtest/tests/o00004.vtc @@ -12,8 +12,6 @@ server s1 { } -start varnish v1 -proto PROXY -vcl+backend { - import debug; - sub vcl_recv { if (client.ip != remote.ip || server.ip != local.ip) { return (synth(400)); From dridi.boukelmoune at gmail.com Tue May 14 09:13:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 14 May 2019 09:13:09 +0000 (UTC) Subject: [master] 3a1fd9bb6 Kill strcat and strcpy usage in VIN_n_Arg Message-ID: <20190514091309.3E1AC10D032@lists.varnish-cache.org> commit 3a1fd9bb611927b47ee45979dcd6781e67bdfc0f Author: Dridi Boukelmoune Date: Tue May 14 00:24:53 2019 +0200 Kill strcat and strcpy usage in VIN_n_Arg If an absolute path is provided as n_arg with a length of exactly PATH_MAX-1 then the combination of strcpy and strcat for the trailing slash '/' overflows dn by one byte, writing its new null-terminating character '\0' right after dn's upper bound. By using a fixed-length VSB we can simply ensure that we stay within bounds at a reasonable cost. Guarding VSB operations should silence Flexelint as a nice side effect. VIN_n_Arg is not exposed outside of the source tree, and both callers today provide a valid dir argument, so we can now make it part of the contract with an assertion, simplifying the strdup error handling. diff --git a/lib/libvarnish/vin.c b/lib/libvarnish/vin.c index 6c098afb7..053e9ed36 100644 --- a/lib/libvarnish/vin.c +++ b/lib/libvarnish/vin.c @@ -39,15 +39,19 @@ #include "vdef.h" -#include "vas.h" // XXX Flexelint "not used" - but req'ed for assert() +#include "vas.h" #include "vin.h" +#include "vsb.h" int VIN_n_Arg(const char *n_arg, char **dir) { char nm[PATH_MAX]; char dn[PATH_MAX]; + struct vsb vsb[1]; + int i; + AN(dir); /* First: determine the name */ @@ -61,25 +65,26 @@ VIN_n_Arg(const char *n_arg, char **dir) } else bprintf(nm, "%s", n_arg); - /* Second: find the directory name */ + AN(VSB_new(vsb, dn, sizeof dn, VSB_FIXEDLEN)); + if (*nm == '/') - strcpy(dn, nm); - else if (strlen(VARNISH_STATE_DIR) + 1 + strlen(nm) >= sizeof dn){ - /* preliminary length check to avoid overflowing dm */ + i = VSB_printf(vsb, "%s/", nm); + else + i = VSB_printf(vsb, "%s/%s/", VARNISH_STATE_DIR, nm); + + if (i != 0) { errno = ENAMETOOLONG; return (-1); - } else { - bprintf(dn, "%s/%s", VARNISH_STATE_DIR, nm); } - strcat(dn, "/"); + AZ(VSB_finish(vsb)); + VSB_clear(vsb); + + *dir = strdup(dn); + if (*dir == NULL) + return (-1); - if (dir != NULL) { - *dir = strdup(dn); - if (*dir == NULL) - return (-1); - } return (0); } From phk at FreeBSD.org Wed May 15 11:00:15 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 15 May 2019 11:00:15 +0000 (UTC) Subject: [master] d33b89e8d Unused include Message-ID: <20190515110015.439639C076@lists.varnish-cache.org> commit d33b89e8d599d4cf6415ffa4d50f456197aea26a Author: Poul-Henning Kamp Date: Wed May 15 07:57:26 2019 +0000 Unused include diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c index da9033f48..9e4816da8 100644 --- a/bin/varnishtest/vtc_client.c +++ b/bin/varnishtest/vtc_client.c @@ -40,7 +40,6 @@ #include "vtc.h" -#include "vsa.h" #include "vss.h" #include "vtcp.h" #include "vus.h" From phk at FreeBSD.org Wed May 15 11:00:15 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 15 May 2019 11:00:15 +0000 (UTC) Subject: [master] f0ad3c0df Un-special-case the 'default' more. Message-ID: <20190515110015.66A5E9C079@lists.varnish-cache.org> commit f0ad3c0dfd11cb19dc6f7e8b11addad804eba560 Author: Poul-Henning Kamp Date: Wed May 15 10:58:50 2019 +0000 Un-special-case the 'default' more. diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 1c1f659ad..e8fd18a28 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -302,55 +302,6 @@ vcc_expr_tostring(struct vcc *tl, struct expr **e, vcc_type_t fmt) } } -/*-------------------------------------------------------------------- - */ - -static void v_matchproto_(sym_expr_t) -vcc_Eval_Regsub(struct vcc *tl, struct expr **e, struct token *t, - struct symbol *sym, vcc_type_t fmt) -{ - struct expr *e2; - int all = sym->eval_priv == NULL ? 0 : 1; - char buf[128]; - struct vsb vsb; - - (void)t; - (void)fmt; - SkipToken(tl, '('); - vcc_expr0(tl, &e2, STRING); - ERRCHK(tl); - SkipToken(tl, ','); - ExpectErr(tl, CSTR); - - AN(VSB_new(&vsb, buf, sizeof buf, VSB_FIXEDLEN)); - VSB_printf(&vsb, "VRT_regsub(ctx, %d,\v+\n\v1,\n", all); - vcc_regexp(tl, &vsb); - ERRCHK(tl); - AZ(VSB_finish(&vsb)); - *e = vcc_expr_edit(tl, STRING, VSB_data(&vsb), e2, NULL); - SkipToken(tl, ','); - vcc_expr0(tl, &e2, STRING); - ERRCHK(tl); - *e = vcc_expr_edit(tl, STRINGS, "\v1,\n\v2)\v-", *e, e2); - (*e)->nstr = 1; - SkipToken(tl, ')'); -} - -/*-------------------------------------------------------------------- - */ - -static void v_matchproto_(sym_expr_t) -vcc_Eval_BoolConst(struct vcc *tl, struct expr **e, struct token *t, - struct symbol *sym, vcc_type_t fmt) -{ - - (void)t; - (void)tl; - (void)fmt; - *e = vcc_mk_expr(BOOL, "(0==%d)", sym->eval_priv == NULL ? 1 : 0); - (*e)->constant = EXPR_CONST; -} - /*-------------------------------------------------------------------- */ @@ -734,17 +685,6 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt) } switch (tl->t->tok) { case ID: - if (vcc_IdIs(tl->t, "default") && fmt == PROBE) { - vcc_NextToken(tl); - *e = vcc_mk_expr(PROBE, "%s", vcc_default_probe(tl)); - return; - } - if (vcc_IdIs(tl->t, "default") && fmt == BACKEND) { - vcc_NextToken(tl); - *e = vcc_mk_expr(BACKEND, - "*(VCL_conf.default_director)"); - return; - } t = tl->t; sym = VCC_SymbolGet(tl, SYM_NONE, "Symbol not found", XREF_REF); @@ -1399,6 +1339,77 @@ vcc_Act_Call(struct vcc *tl, struct token *t, struct symbol *sym) } vcc_delete_expr(e); } +/*-------------------------------------------------------------------- + */ + +static void v_matchproto_(sym_expr_t) +vcc_Eval_Regsub(struct vcc *tl, struct expr **e, struct token *t, + struct symbol *sym, vcc_type_t fmt) +{ + struct expr *e2; + int all = sym->eval_priv == NULL ? 0 : 1; + char buf[128]; + struct vsb vsb; + + (void)t; + (void)fmt; + SkipToken(tl, '('); + vcc_expr0(tl, &e2, STRING); + ERRCHK(tl); + SkipToken(tl, ','); + ExpectErr(tl, CSTR); + + AN(VSB_new(&vsb, buf, sizeof buf, VSB_FIXEDLEN)); + VSB_printf(&vsb, "VRT_regsub(ctx, %d,\v+\n\v1,\n", all); + vcc_regexp(tl, &vsb); + ERRCHK(tl); + AZ(VSB_finish(&vsb)); + *e = vcc_expr_edit(tl, STRING, VSB_data(&vsb), e2, NULL); + SkipToken(tl, ','); + vcc_expr0(tl, &e2, STRING); + ERRCHK(tl); + *e = vcc_expr_edit(tl, STRINGS, "\v1,\n\v2)\v-", *e, e2); + (*e)->nstr = 1; + SkipToken(tl, ')'); +} + +/*-------------------------------------------------------------------- + */ + +static void v_matchproto_(sym_expr_t) +vcc_Eval_BoolConst(struct vcc *tl, struct expr **e, struct token *t, + struct symbol *sym, vcc_type_t fmt) +{ + + (void)t; + (void)tl; + (void)fmt; + *e = vcc_mk_expr(BOOL, "(0==%d)", sym->eval_priv == NULL ? 1 : 0); + (*e)->constant = EXPR_CONST; +} + +/*-------------------------------------------------------------------- + */ + +static void v_matchproto_(sym_expr_t) +vcc_Eval_Default(struct vcc *tl, struct expr **e, struct token *t, + struct symbol *sym, vcc_type_t fmt) +{ + (void)e; + (void)fmt; + (void)sym; + (void)t; + + if (fmt == PROBE) + *e = vcc_mk_expr(PROBE, "%s", vcc_default_probe(tl)); + else if (fmt == BACKEND) + *e = vcc_mk_expr(BACKEND, "*(VCL_conf.default_director)"); + else { + VSB_printf(tl->sb, + "Symbol 'default' is a reserved word.\n"); + vcc_ErrWhere(tl, t); + } +} /*-------------------------------------------------------------------- */ @@ -1431,4 +1442,9 @@ vcc_Expr_Init(struct vcc *tl) sym->type = BOOL; sym->eval = vcc_Eval_BoolConst; sym->eval_priv = NULL; + + sym = VCC_MkSym(tl, "default", SYM_FUNC, VCL_LOW, VCL_HIGH); + AN(sym); + sym->type = BACKEND; // ... can also (sometimes) deliver PROBE + sym->eval = vcc_Eval_Default; } diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c index 1e0bbd355..ce0deb08c 100644 --- a/lib/libvcc/vcc_parse.c +++ b/lib/libvcc/vcc_parse.c @@ -357,7 +357,6 @@ static struct toplev { { "probe", vcc_ParseProbe, VCL_41, VCL_HIGH }, { "import", vcc_ParseImport, VCL_41, VCL_HIGH }, { "vcl", vcc_ParseVcl, VCL_41, VCL_HIGH }, - { "default", NULL, VCL_40, VCL_HIGH }, { NULL, NULL } }; diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index 1ff8a4943..e898c884b 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -240,11 +240,7 @@ VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x) if (kind != SYM_NONE && kind != sym->kind) { VSB_printf(tl->sb, "Symbol "); vcc_ErrToken(tl, tl->t); - if (sym->kind == SYM_NONE) - VSB_printf(tl->sb, " is a reserved word."); - else - VSB_printf(tl->sb, " has wrong type (%s): ", - sym->kind->name); + VSB_printf(tl->sb, " has wrong type (%s): ", sym->kind->name); VSB_cat(tl->sb, "\nAt: "); vcc_ErrWhere(tl, tl->t); if (sym->def_b != NULL) { From phk at FreeBSD.org Wed May 15 11:11:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 15 May 2019 11:11:10 +0000 (UTC) Subject: [master] b81fd3bbf Sigh, last commit was from wrong subdirectory... Message-ID: <20190515111110.1B91C9C7AE@lists.varnish-cache.org> commit b81fd3bbf458c7fd744ab5676f05338b71b875e5 Author: Poul-Henning Kamp Date: Wed May 15 11:09:45 2019 +0000 Sigh, last commit was from wrong subdirectory... diff --git a/bin/varnishtest/tests/v00021.vtc b/bin/varnishtest/tests/v00021.vtc index 689b3faea..44c4c0f3d 100644 --- a/bin/varnishtest/tests/v00021.vtc +++ b/bin/varnishtest/tests/v00021.vtc @@ -115,7 +115,7 @@ varnish v1 -errvcl {Symbol 'true' has wrong type (func):} { } } -varnish v1 -errvcl {Symbol 'default' is a reserved word.} { +varnish v1 -errvcl {Symbol 'default' has wrong type} { sub vcl_recv { if (client.ip ~ default) { } } From dridi.boukelmoune at gmail.com Wed May 15 15:43:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 15 May 2019 15:43:09 +0000 (UTC) Subject: [master] e55d9eae5 Make r02433.vtc relevant again Message-ID: <20190515154309.63902A895A@lists.varnish-cache.org> commit e55d9eae55690ca8d60392090c247563a034ec52 Author: Dridi Boukelmoune Date: Wed May 15 17:38:49 2019 +0200 Make r02433.vtc relevant again With recent changes in the VCL state and temperature handling this test case lost its relevance and started testing a no-op operation. And since it wasn't apparently obvious what it was trying to check between two possible interpretations it now tests both and better verifies it. Reported by fgs. When I initially wrote this test case it used to trigger a panic in the then young VCL labeling. diff --git a/bin/varnishtest/tests/r02433.vtc b/bin/varnishtest/tests/r02433.vtc index b78777e64..81f76cc4c 100644 --- a/bin/varnishtest/tests/r02433.vtc +++ b/bin/varnishtest/tests/r02433.vtc @@ -2,22 +2,34 @@ varnishtest "label a cold vcl" server s1 { } -start +varnish v1 -cliok "param.set vcl_cooldown 0.1" varnish v1 -vcl+backend { import debug; # can fail a VCL warmup - - sub vcl_recv { - return (synth(200)); - } } -start -# a dummy vcl to freely use vcl1 +# a dummy vcl2 to freely use vcl1 varnish v1 -vcl+backend { } -# the magic parameter that fails a VCL warmup +# cool vcl1 down +delay 4 +varnish v1 -cliok ping + +shell -match "auto +cold" { varnishadm -n ${v1_name} vcl.list | grep vcl1 } + +# first we try to label a cold VCL in auto state + +# the magic parameter that fails a VCL warmup in vmod-debug varnish v1 -cliok "param.set max_esi_depth 42" +varnish v1 -cliexpect "max_esi_depth is not the answer" "vcl.label label1 vcl1" + +shell -err { varnishadm -n ${v1_name} vcl.list | grep label1 } + +# second we try to label VCL in cold state varnish v1 -cliok "vcl.state vcl1 cold" -varnish v1 -clierr 300 "vcl.label label1 vcl1" -# check that creating the label actually failed -varnish v1 -clierr 106 "vcl.discard label1" +shell -match "cold +cold" { varnishadm -n ${v1_name} vcl.list | grep vcl1 } + +varnish v1 -cliexpect "set to auto or warm first" "vcl.label label1 vcl1" + +shell -err { varnishadm -n ${v1_name} vcl.list | grep label1 } From phk at FreeBSD.org Wed May 15 21:14:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 15 May 2019 21:14:10 +0000 (UTC) Subject: [master] f8f8f1f72 Cover double import Message-ID: <20190515211410.36A5AB118F@lists.varnish-cache.org> commit f8f8f1f72d3fd9e87d0aa63f3b20a630bc947bfb Author: Poul-Henning Kamp Date: Wed May 15 21:13:07 2019 +0000 Cover double import diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc index b0e894419..1931db422 100644 --- a/bin/varnishtest/tests/m00000.vtc +++ b/bin/varnishtest/tests/m00000.vtc @@ -10,6 +10,7 @@ varnish v1 -vcl+backend { import std; import debug; import vtc; + import debug; // again sub vcl_init { new objx = debug.obj(); From phk at FreeBSD.org Thu May 16 08:12:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 16 May 2019 08:12:10 +0000 (UTC) Subject: [master] b072bb7ec Copyediting Message-ID: <20190516081210.89E20626A6@lists.varnish-cache.org> commit b072bb7ecd11f4fb106d347c79cfc154a52c3f58 Author: Poul-Henning Kamp Date: Wed May 15 21:48:37 2019 +0000 Copyediting diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst index 574da32dc..f069d10b1 100644 --- a/doc/sphinx/reference/vmod.rst +++ b/doc/sphinx/reference/vmod.rst @@ -466,8 +466,9 @@ The VCL compiler supports the following private pointers: * ``PRIV_CALL`` "per call" private pointers are useful to cache/store state relative to the specific call or its arguments, for instance a - compiled regular expression specific to a regsub() statement or a - simply caching the last output of some expensive lookup. + compiled regular expression specific to a regsub() statement or + simply caching the most recent output of some expensive operation. + These private pointers live for the duration of the loaded VCL. * ``PRIV_TASK`` "per task" private pointers are useful for state that applies to calls for either a specific request or a backend @@ -476,17 +477,21 @@ The VCL compiler supports the following private pointers: for the client side and the backend side, so use in ``vcl_backend_*`` will yield a different private pointer from the one used on the client side. + These private pointers live only for the duration of their task. * ``PRIV_TOP`` "per top-request" private pointers live for the duration of one request and all its ESI-includes. They are only defined for the client side. When used from backend VCL subs, a NULL pointer will be passed. + These private pointers live only for the duration of their top + level request * ``PRIV_VCL`` "per vcl" private pointers are useful for such global state that applies to all calls in this VCL, for instance flags that determine if regular expressions are case-sensitive in this vmod or similar. The ``PRIV_VCL`` object is the same object that is passed to the VMOD's event function. + This private pointer lives for the duration of the loaded VCL. The way it works in the vmod code, is that a ``struct vmod_priv *`` is passed to the functions where one of the ``PRIV_*`` argument types is @@ -501,22 +506,16 @@ This structure contains three members:: vmod_priv_free_f *free; }; -The "priv" element can be used for whatever the vmod code wants to -use it for, it defaults to a NULL pointer. +The "priv" and "len" elements can be used for whatever the vmod +code wants to use them for, and the "free" element provides a +callback to clean them up. -The "len" element is used primarily for BLOBs to indicate its size. - -The "free" element defaults to NULL, and it is the modules responsibility -to set it to a suitable function, which can clean up whatever the "priv" -pointer points to. - -When a VCL program is discarded, all private pointers are checked -to see if both the "priv" and "free" elements are non-NULL, and if -they are, the "free" function will be called with the "priv" pointer -as the only argument. +If both the "priv" and "free" pointers are non-NULL when the scope +ends, the "free" function will be called with the "priv" pointer +as its only argument. In the common case where a private data structure is allocated with -malloc would look like this:: +malloc(3) would look like this:: if (priv->priv == NULL) { priv->priv = calloc(1, sizeof(struct myfoo)); @@ -532,10 +531,10 @@ malloc would look like this:: ... } -The per-call vmod_privs are freed before the per-vcl vmod_priv. - Note on use with objects: +The per-call vmod_privs are freed before the per-vcl vmod_priv. + ``PRIV_TASK`` and ``PRIV_TOP`` arguments are not per object instance, but still per vmod as for ordinary vmod functions. Thus, vmods requiring per-task / per top-request state for object instances need From phk at FreeBSD.org Thu May 16 08:12:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 16 May 2019 08:12:10 +0000 (UTC) Subject: [master] b2a86df62 Introduce a "reserved" symboltype and wrangle an error message or two. Message-ID: <20190516081210.9F53D626A9@lists.varnish-cache.org> commit b2a86df62afc206ec03dc034aada220bc659db1e Author: Poul-Henning Kamp Date: Thu May 16 08:11:13 2019 +0000 Introduce a "reserved" symboltype and wrangle an error message or two. diff --git a/bin/varnishtest/tests/m00001.vtc b/bin/varnishtest/tests/m00001.vtc index 6e0781262..1a676c673 100644 --- a/bin/varnishtest/tests/m00001.vtc +++ b/bin/varnishtest/tests/m00001.vtc @@ -66,7 +66,7 @@ varnish v1 -cliok "vcl.discard vcl2" varnish v1 -cliok "vcl.list" varnish v1 -cliok "debug.vmod" -varnish v1 -errvcl {Symbol type (vmod) can not be used in expression.} { +varnish v1 -errvcl {Symbol 'std' type (vmod) can not be used in expression.} { import std; sub vcl_recv { diff --git a/bin/varnishtest/tests/v00020.vtc b/bin/varnishtest/tests/v00020.vtc index 1c61f02e5..919624a41 100644 --- a/bin/varnishtest/tests/v00020.vtc +++ b/bin/varnishtest/tests/v00020.vtc @@ -20,7 +20,7 @@ varnish v1 -errvcl {Found: '0' at} { 0; } # VCLs tokenstream. # XXX: A better error message would be desirable -varnish v1 -errvcl {Symbol not found} " sub vcl_recv { { } { " +varnish v1 -errvcl {Symbol cannot be used here} " sub vcl_recv { { } { " varnish v1 -errvcl {Comparison of different types: INT '!=' STRING} { sub vcl_recv { @@ -31,7 +31,7 @@ varnish v1 -errvcl {Comparison of different types: INT '!=' STRING} { } } -varnish v1 -errvcl {Symbol type (sub) can not be used in expression.} { +varnish v1 -errvcl {Symbol 'vcl_recv' type (sub) can not be used in expression.} { sub vcl_recv { set req.http.foo = vcl_recv; } @@ -307,6 +307,11 @@ varnish v1 -errvcl {'||' must be preceeded by BOOL, found REAL.} { sub vcl_recv { if (std.random(0,1) || 0) { } } } +varnish v1 -errvcl {Symbol 'acl' has wrong type (reserved):} { + import std; + sub vcl_recv { call acl; } +} + server s1 { rxreq txresp -hdr "bar: X" diff --git a/include/tbl/symbol_kind.h b/include/tbl/symbol_kind.h index bfae2c1e2..8c0294373 100644 --- a/include/tbl/symbol_kind.h +++ b/include/tbl/symbol_kind.h @@ -30,6 +30,7 @@ /*lint -save -e525 -e539 */ VCC_KIND(NONE, none) +VCC_KIND(RESERVED, reserved) VCC_KIND(ACL, acl) VCC_KIND(ACTION, action) VCC_KIND(BACKEND, backend) diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c index af1d7fee6..8126f9924 100644 --- a/lib/libvcc/vcc_action.c +++ b/lib/libvcc/vcc_action.c @@ -46,11 +46,12 @@ vcc_act_call(struct vcc *tl, struct token *t, struct symbol *sym) (void)t; ExpectErr(tl, ID); sym = VCC_SymbolGet(tl, SYM_SUB, SYMTAB_CREATE, XREF_REF); - AN(sym); - vcc_AddCall(tl, sym); - VCC_GlobalSymbol(sym, SUB, "VGC_function"); - Fb(tl, 1, "%s(ctx);\n", sym->rname); - SkipToken(tl, ';'); + if (sym != NULL) { + vcc_AddCall(tl, sym); + VCC_GlobalSymbol(sym, SUB, "VGC_function"); + Fb(tl, 1, "%s(ctx);\n", sym->rname); + SkipToken(tl, ';'); + } } /*--------------------------------------------------------------------*/ diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index e8fd18a28..d04444dae 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -713,9 +713,9 @@ vcc_expr4(struct vcc *tl, struct expr **e, vcc_type_t fmt) return; } VSB_printf(tl->sb, - "Symbol type (%s) can not be used in expression.\n", - sym->kind->name); - vcc_ErrWhere(tl, tl->t); + "Symbol '%.*s' type (%s) can not be used in expression.\n", + PF(t), sym->kind->name); + vcc_ErrWhere(tl, t); if (sym->def_b != NULL) { VSB_printf(tl->sb, "That symbol was defined here:\n"); vcc_ErrWhere(tl, sym->def_b); diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c index ce0deb08c..bb695ba92 100644 --- a/lib/libvcc/vcc_parse.c +++ b/lib/libvcc/vcc_parse.c @@ -434,5 +434,5 @@ vcc_Parse_Init(struct vcc *tl) struct toplev *tp; for (tp = toplev; tp->name != NULL; tp++) - AN(VCC_MkSym(tl, tp->name, SYM_NONE, tp->vcllo, tp->vclhi)); + AN(VCC_MkSym(tl, tp->name, SYM_RESERVED, tp->vcllo, tp->vclhi)); } From phk at FreeBSD.org Fri May 17 08:14:11 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 17 May 2019 08:14:11 +0000 (UTC) Subject: [master] f0cba1de0 Create VCC_SymbolGetTok() variant Message-ID: <20190517081411.7C8B1117CF5@lists.varnish-cache.org> commit f0cba1de0c497111090f80b073db9677fbff93fc Author: Poul-Henning Kamp Date: Fri May 17 06:00:21 2019 +0000 Create VCC_SymbolGetTok() variant diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 1a3c778d9..00b45d15a 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -356,6 +356,8 @@ extern const char SYMTAB_NOERR[]; extern const char SYMTAB_CREATE[]; struct symbol *VCC_SymbolGet(struct vcc *, vcc_kind_t, const char *, const char *); +struct symbol *VCC_SymbolGetTok(struct vcc *, vcc_kind_t, const char *, const char *, struct token *); + typedef void symwalk_f(struct vcc *tl, const struct symbol *s); void VCC_WalkSymbols(struct vcc *tl, symwalk_f *func, vcc_kind_t); diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index e898c884b..a7c8880e0 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -198,30 +198,31 @@ const char SYMTAB_NOERR[] = "sym_noerror"; const char SYMTAB_CREATE[] = "sym_create"; struct symbol * -VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x) +VCC_SymbolGetTok(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x, + struct token *t) { struct symbol *sym; AN(e); if (tl->syntax >= VCL_41 && e == SYMTAB_CREATE && kind != SYM_SUB && - (tl->t->b[0] == 'v'|| tl->t->b[0] == 'V') && - (tl->t->b[1] == 'c'|| tl->t->b[1] == 'C') && - (tl->t->b[2] == 'l'|| tl->t->b[2] == 'L') && - (tl->t->b[3] == '_')) { + (t->b[0] == 'v'|| t->b[0] == 'V') && + (t->b[1] == 'c'|| t->b[1] == 'C') && + (t->b[2] == 'l'|| t->b[2] == 'L') && + (t->b[3] == '_')) { VSB_printf(tl->sb, "Symbols named 'vcl_*' are reserved.\nAt:"); - vcc_ErrWhere(tl, tl->t); + vcc_ErrWhere(tl, t); return (NULL); } - sym = VCC_Symbol(tl, NULL, tl->t->b, tl->t->e, kind, + sym = VCC_Symbol(tl, NULL, t->b, t->e, kind, e == SYMTAB_CREATE ? 1 : 0, tl->syntax, tl->syntax); if (sym == NULL && e == SYMTAB_NOERR) return (sym); if (sym == NULL) { VSB_printf(tl->sb, "%s: ", e); - vcc_ErrToken(tl, tl->t); - sym = VCC_Symbol(tl, NULL, tl->t->b, tl->t->e, kind, 0, + vcc_ErrToken(tl, t); + sym = VCC_Symbol(tl, NULL, t->b, t->e, kind, 0, VCL_LOW, VCL_HIGH); if (sym != NULL) { VSB_printf(tl->sb, " (Only available when"); @@ -233,16 +234,16 @@ VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x) VSB_printf(tl->sb, ")"); } VSB_cat(tl->sb, "\nAt: "); - vcc_ErrWhere(tl, tl->t); + vcc_ErrWhere(tl, t); return (NULL); } assert (sym->lorev <= tl->syntax && sym->hirev >= tl->syntax); if (kind != SYM_NONE && kind != sym->kind) { VSB_printf(tl->sb, "Symbol "); - vcc_ErrToken(tl, tl->t); + vcc_ErrToken(tl, t); VSB_printf(tl->sb, " has wrong type (%s): ", sym->kind->name); VSB_cat(tl->sb, "\nAt: "); - vcc_ErrWhere(tl, tl->t); + vcc_ErrWhere(tl, t); if (sym->def_b != NULL) { VSB_printf(tl->sb, "Symbol was defined here: "); vcc_ErrWhere(tl, sym->def_b); @@ -256,22 +257,30 @@ VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x) } if (x == XREF_DEF) { if (sym->def_b == NULL) - sym->def_b = tl->t; + sym->def_b = t; sym->ndef++; } else if (x == XREF_REF) { if (sym->ref_b == NULL) - sym->ref_b = tl->t; + sym->ref_b = t; sym->nref++; } else { assert (x == XREF_NONE); } - vcc_NextToken(tl); return (sym); } struct symbol * -VCC_MkSym(struct vcc *tl, const char *b, vcc_kind_t kind, - int vlo, int vhi) +VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x) +{ + struct symbol *sym; + sym = VCC_SymbolGetTok(tl, kind, e, x, tl->t); + if (sym != NULL) + vcc_NextToken(tl); + return (sym); +} + +struct symbol * +VCC_MkSym(struct vcc *tl, const char *b, vcc_kind_t kind, int vlo, int vhi) { struct symbol *sym; From phk at FreeBSD.org Fri May 17 08:14:11 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 17 May 2019 08:14:11 +0000 (UTC) Subject: [master] 5d3d3eeaf Make vcc typenames available in lower-case as well Message-ID: <20190517081411.69468117CF2@lists.varnish-cache.org> commit 5d3d3eeaf9650c2e69fa7dfb18071e6c6b280067 Author: Poul-Henning Kamp Date: Thu May 16 08:25:52 2019 +0000 Make vcc typenames available in lower-case as well diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 8da81c378..7e7723ed4 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -793,7 +793,7 @@ file_header(ft) lint_start(ft) for vcltype in sorted(vcltypes.keys()): - ft.write("VCC_TYPE(" + vcltype + ")\n") + ft.write("VCC_TYPE(" + vcltype + ", " + vcltype.lower() +")\n") ft.write("#undef VCC_TYPE\n") lint_end(ft) ft.close() diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index ff9fee700..1a3c778d9 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -106,7 +106,7 @@ struct type { int stringform; }; -#define VCC_TYPE(foo) extern const struct type foo[1]; +#define VCC_TYPE(UC, lc) extern const struct type UC[1]; #include "tbl/vcc_types.h" /*---------------------------------------------------------------------*/ diff --git a/lib/libvcc/vcc_types.c b/lib/libvcc/vcc_types.c index 2d9208c64..3611dcef5 100644 --- a/lib/libvcc/vcc_types.c +++ b/lib/libvcc/vcc_types.c @@ -178,7 +178,7 @@ vcc_type_t VCC_Type(const char *p) { -#define VCC_TYPE(foo) if (!strcmp(p, #foo)) return (foo); +#define VCC_TYPE(UC, lc) if (!strcmp(p, #UC)) return (UC); #include "tbl/vcc_types.h" return (NULL); } From phk at FreeBSD.org Fri May 17 08:14:11 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 17 May 2019 08:14:11 +0000 (UTC) Subject: [master] f4f41c51c Use vmod symbol instead of vmod name Message-ID: <20190517081411.A5E67117CF8@lists.varnish-cache.org> commit f4f41c51cb5c3f9dd2e21b9e785ae94bffcb85e7 Author: Poul-Henning Kamp Date: Fri May 17 06:19:09 2019 +0000 Use vmod symbol instead of vmod name diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 00b45d15a..f164a9e92 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -144,7 +144,7 @@ struct symbol { int hirev; struct symbol *parent; - const char *vmod; + const struct symbol *vmod; sym_wildcard_t *wildcard; vcc_kind_t kind; @@ -356,7 +356,8 @@ extern const char SYMTAB_NOERR[]; extern const char SYMTAB_CREATE[]; struct symbol *VCC_SymbolGet(struct vcc *, vcc_kind_t, const char *, const char *); -struct symbol *VCC_SymbolGetTok(struct vcc *, vcc_kind_t, const char *, const char *, struct token *); +struct symbol *VCC_SymbolGetTok(struct vcc *, vcc_kind_t, const char *, + const char *, const struct token *); typedef void symwalk_f(struct vcc *tl, const struct symbol *s); void VCC_WalkSymbols(struct vcc *tl, symwalk_f *func, vcc_kind_t); diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index d04444dae..9638b519f 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -355,22 +355,24 @@ vcc_priv_arg(struct vcc *tl, const char *p, const struct symbol *sym) { char buf[64]; struct inifin *ifp; - const char *vmod, *f = NULL; + const char *f = NULL; struct procprivhead *marklist = NULL; AN(sym); AN(sym->vmod); - vmod = sym->vmod; - if (!strcmp(p, "PRIV_VCL")) { - return (vcc_mk_expr(VOID, "&vmod_priv_%s", vmod)); - } else if (!strcmp(p, "PRIV_CALL")) { + if (!strcmp(p, "PRIV_VCL")) + return (vcc_mk_expr(VOID, "&vmod_priv_%s", sym->vmod->name)); + + if (!strcmp(p, "PRIV_CALL")) { bprintf(buf, "vmod_priv_%u", tl->unique++); ifp = New_IniFin(tl); Fh(tl, 0, "static struct vmod_priv %s;\n", buf); VSB_printf(ifp->fin, "\tVRT_priv_fini(&%s);", buf); return (vcc_mk_expr(VOID, "&%s", buf)); - } else if (!strcmp(p, "PRIV_TASK")) { + } + + if (!strcmp(p, "PRIV_TASK")) { f = "task"; marklist = &tl->curproc->priv_tasks; } else if (!strcmp(p, "PRIV_TOP")) { @@ -381,9 +383,9 @@ vcc_priv_arg(struct vcc *tl, const char *p, const struct symbol *sym) } AN(f); AN(marklist); - bprintf(buf, "ARG_priv_%s_%s", f, vmod); + bprintf(buf, "ARG_priv_%s_%s", f, sym->vmod->name); - if (vcc_MarkPriv(tl, marklist, vmod) == NULL) + if (vcc_MarkPriv(tl, marklist, sym->vmod->name) == NULL) VSB_printf(tl->curproc->prologue, " struct vmod_priv *%s = " "VRT_priv_%s(ctx, &VGC_vmod_%s);\n" @@ -392,7 +394,7 @@ vcc_priv_arg(struct vcc *tl, const char *p, const struct symbol *sym) "for vmod %s\");\n" " return;\n" " }\n", - buf, f, vmod, buf, f, vmod); + buf, f, sym->vmod->name, buf, f, sym->vmod->name); return (vcc_mk_expr(VOID, "%s", buf)); } diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index a7c8880e0..25a2573cb 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -199,7 +199,7 @@ const char SYMTAB_CREATE[] = "sym_create"; struct symbol * VCC_SymbolGetTok(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x, - struct token *t) + const struct token *t) { struct symbol *sym; diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index 61380edf2..6eadd44dc 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -57,7 +57,8 @@ vcc_path_dlopen(void *priv, const char *fn) } static void -func_sym(struct symbol *sym, const char *vmod, const struct vjsn_val *v) +func_sym(struct symbol *sym, const struct symbol *vmod, + const struct vjsn_val *v) { assert(v->type == VJSN_ARRAY); @@ -139,13 +140,13 @@ vcc_json_wildcard(struct vcc *tl, struct symbol *msym, struct symbol *tsym) !strcmp(vv2->value, tsym->name)) { tsym->kind = SYM_FUNC; tsym->noref = 1; - func_sym(tsym, msym->name, VTAILQ_NEXT(vv2, list)); + func_sym(tsym, msym, VTAILQ_NEXT(vv2, list)); return; } else if (!strcmp(vv1->value, "$OBJ") && !strcmp(vv2->value, tsym->name)) { tsym->kind = SYM_OBJECT; tsym->eval_priv = vv2; - tsym->vmod = msym->name; + tsym->vmod = msym; return; } } From phk at FreeBSD.org Fri May 17 08:14:11 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 17 May 2019 08:14:11 +0000 (UTC) Subject: [master] 0060ee941 Found a sneaky way to tests VCC's vmod-tasting code. Message-ID: <20190517081411.D5D67117CFD@lists.varnish-cache.org> commit 0060ee9410e0fa1d8ca9785761b790abdab7d656 Author: Poul-Henning Kamp Date: Fri May 17 08:12:44 2019 +0000 Found a sneaky way to tests VCC's vmod-tasting code. diff --git a/bin/varnishtest/tests/m00003.vtc b/bin/varnishtest/tests/m00003.vtc index e2f6dccab..4e5a60ff4 100644 --- a/bin/varnishtest/tests/m00003.vtc +++ b/bin/varnishtest/tests/m00003.vtc @@ -14,7 +14,7 @@ varnish v1 -arg "-pvmod_path=${topbuild}/lib/libvmod_std/.libs/" \ varnish v1 -cliok "param.set vmod_path /nonexistent" -varnish v1 -errvcl {Could not load VMOD std} { +varnish v1 -errvcl {Could not find VMOD std} { import std; } @@ -23,3 +23,35 @@ varnish v1 -cliok "param.set vmod_path ${topbuild}/lib/libvmod_std/.libs/" varnish v1 -vcl+backend { import std; } + +varnish v1 -cliok "param.set vmod_path ${tmpdir}" + +shell "true > ${tmpdir}/libvmod_wrong.so" + +varnish v1 -errvcl {Could not open VMOD wrong} { + import wrong; +} + +shell "cp ${topbuild}/lib/libvmod_debug/.libs/libvmod_debug.so ${tmpdir}/libvmod_wrong.so" + +varnish v1 -errvcl {Malformed VMOD wrong} { + import wrong; +} + +varnish v1 -errvcl {Incompatible VMOD wrong0} { + import wrong0 from "${tmpdir}/libvmod_wrong.so"; +} + +varnish v1 -errvcl {Incompatible VMOD wrong1} { + import wrong1 from "${tmpdir}/libvmod_wrong.so"; +} + +varnish v1 -errvcl {Mangled VMOD wrong2} { + import wrong2 from "${tmpdir}/libvmod_wrong.so"; +} + +varnish v1 -errvcl {Wrong file for VMOD wrong3} { + import wrong3 from "${tmpdir}/libvmod_wrong.so"; +} + +shell "rm -f ${tmpdir}/libvmod_wrong.so" diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 6b0ac88a6..c0ffc19ba 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -769,6 +769,7 @@ VCC_New(void) VTAILQ_INIT(&tl->sources); VTAILQ_INIT(&tl->procs); VTAILQ_INIT(&tl->sym_objects); + VTAILQ_INIT(&tl->sym_vmods); tl->nsources = 0; diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index f164a9e92..58eb570c6 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -258,6 +258,7 @@ struct vcc { const char *default_probe; VTAILQ_HEAD(, symbol) sym_objects; + VTAILQ_HEAD(, symbol) sym_vmods; unsigned unique; unsigned vmod_count; diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index 6eadd44dc..ff4831198 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -40,19 +40,26 @@ #include "vmod_abi.h" #include "vsb.h" +struct vmod_open { + unsigned magic; +#define VMOD_OPEN_MAGIC 0x9995b1f3 + void *hdl; + const char *err; +}; + static int vcc_path_dlopen(void *priv, const char *fn) { - void *hdl, **pp; + struct vmod_open *vop; - AN(priv); + CAST_OBJ_NOTNULL(vop, priv, VMOD_OPEN_MAGIC); AN(fn); - hdl = dlopen(fn, RTLD_NOW | RTLD_LOCAL); - if (hdl == NULL) + vop->hdl = dlopen(fn, RTLD_NOW | RTLD_LOCAL); + if (vop->hdl == NULL) { + vop->err = dlerror(); return (-1); - pp = priv; - *pp = hdl; + } return (0); } @@ -156,7 +163,6 @@ vcc_json_wildcard(struct vcc *tl, struct symbol *msym, struct symbol *tsym) void vcc_ParseImport(struct vcc *tl) { - void *hdl; char fn[1024], *fnp, *fnpx; char buf[256]; const char *p; @@ -166,14 +172,17 @@ vcc_ParseImport(struct vcc *tl) const struct vmod_data *vmd; struct vjsn *vj; int again = 0; + struct vmod_open vop[1]; + INIT_OBJ(vop, VMOD_OPEN_MAGIC); t1 = tl->t; SkipToken(tl, ID); /* "import" */ ExpectErr(tl, ID); mod = tl->t; - msym = VCC_SymbolGet(tl, SYM_NONE, SYMTAB_NOERR, XREF_NONE); + vcc_NextToken(tl); + msym = VCC_SymbolGetTok(tl, SYM_NONE, SYMTAB_NOERR, XREF_NONE, mod); if (msym != NULL && msym->kind != SYM_VMOD) { /* @@ -188,13 +197,16 @@ vcc_ParseImport(struct vcc *tl) again = 1; } else { - msym = VCC_SymbolGet(tl, SYM_VMOD, SYMTAB_CREATE, XREF_NONE); + msym = VCC_SymbolGetTok(tl, SYM_VMOD, + SYMTAB_CREATE, XREF_NONE, mod); ERRCHK(tl); AN(msym); msym->def_b = t1; msym->def_e = tl->t; } + VTAILQ_INSERT_TAIL(&tl->sym_vmods, msym, sideways); + if (tl->t->tok == ID) { if (!vcc_IdIs(tl->t, "from")) { VSB_printf(tl->sb, "Expected 'from path ...'\n"); @@ -225,12 +237,17 @@ vcc_ParseImport(struct vcc *tl) if (!again) msym->def_e = tl->t; - - if (VFIL_searchpath(tl->vmod_path, vcc_path_dlopen, &hdl, fn, &fnpx)) { - VSB_printf(tl->sb, "Could not load VMOD %.*s\n", PF(mod)); - VSB_printf(tl->sb, "\tFile name: %s\n", - fnpx != NULL ? fnpx : fn); - VSB_printf(tl->sb, "\tdlerror: %s\n", dlerror()); + if (VFIL_searchpath(tl->vmod_path, vcc_path_dlopen, vop, fn, &fnpx)) { + if (vop->err == NULL) { + VSB_printf(tl->sb, + "Could not find VMOD %.*s\n", PF(mod)); + } else { + VSB_printf(tl->sb, + "Could not open VMOD %.*s\n", PF(mod)); + VSB_printf(tl->sb, "\tFile name: %s\n", + fnpx != NULL ? fnpx : fn); + VSB_printf(tl->sb, "\tdlerror: %s\n", vop->err); + } vcc_ErrWhere(tl, mod); free(fnpx); return; @@ -241,7 +258,7 @@ vcc_ParseImport(struct vcc *tl) free(fnpx); bprintf(buf, "Vmod_%.*s_Data", PF(mod)); - vmd = dlsym(hdl, buf); + vmd = dlsym(vop->hdl, buf); if (vmd == NULL) { VSB_printf(tl->sb, "Malformed VMOD %.*s\n", PF(mod)); VSB_printf(tl->sb, "\tFile name: %s\n", fnp); @@ -250,7 +267,7 @@ vcc_ParseImport(struct vcc *tl) return; } if (vmd->vrt_major == 0 && vmd->vrt_minor == 0 && - strcmp(vmd->abi, VMOD_ABI_Version) != 0) { + (vmd->abi == NULL || strcmp(vmd->abi, VMOD_ABI_Version) != 0)) { VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(mod)); VSB_printf(tl->sb, "\tFile name: %s\n", fnp); VSB_printf(tl->sb, "\tABI mismatch, expected <%s>, got <%s>\n", @@ -282,7 +299,7 @@ vcc_ParseImport(struct vcc *tl) } if (!vcc_IdIs(mod, vmd->name)) { - VSB_printf(tl->sb, "Wrong VMOD file %.*s\n", PF(mod)); + VSB_printf(tl->sb, "Wrong file for VMOD %.*s\n", PF(mod)); VSB_printf(tl->sb, "\tFile name: %s\n", fnp); VSB_printf(tl->sb, "\tContains vmod \"%s\"\n", vmd->name); vcc_ErrWhere(tl, mod); @@ -298,7 +315,7 @@ vcc_ParseImport(struct vcc *tl) vcc_ErrWhere2(tl, msym->def_b, msym->def_e); } if (again) { - AZ(dlclose(hdl)); + AZ(dlclose(vop->hdl)); return; } diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am index 4823e1b4d..8c6df5232 100644 --- a/lib/libvmod_debug/Makefile.am +++ b/lib/libvmod_debug/Makefile.am @@ -5,8 +5,17 @@ libvmod_debug_la_SOURCES = \ vmod_debug_obj.c \ vmod_debug_dyn.c + include $(srcdir)/automake_boilerplate.am +# Allow Vmod_wrong*_Data to be exported +libvmod_debug_la_LDFLAGS = \ + -export-symbols-regex 'Vmod_.*_Data' \ + $(AM_LDFLAGS) \ + $(VMOD_LDFLAGS) \ + @SAN_LDFLAGS@ + + # not --strict vmodtoolargs = --boilerplate diff --git a/lib/libvmod_debug/flint.lnt b/lib/libvmod_debug/flint.lnt index 36d225583..2a43e7057 100644 --- a/lib/libvmod_debug/flint.lnt +++ b/lib/libvmod_debug/flint.lnt @@ -1,3 +1,6 @@ -esym(759, xyzzy_enum_*) // header declaration for symbol '___' defined at (___) -esym(765, xyzzy_enum_*) // external '___' (___) could be made static +-esym(765, Vmod_wrong*_Data) // external '___' (___) could be made static +-esym(714, Vmod_wrong*_Data) // external '___' (___) could be made static +-esym(754, foo::bar) // local struct member not referenced diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 4dac3f3dc..0d9f9d356 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -134,7 +134,7 @@ xyzzy_rot13_bytes(struct req *req, enum vdp_action act, void **priv, } } if (i >= 0) - retval = VDP_bytes(req, VDP_FLUSH, q, i + 1); + retval = VDP_bytes(req, VDP_FLUSH, q, i + 1L); return (retval); } @@ -854,3 +854,44 @@ xyzzy_sndbuf(VRT_CTX, VCL_BYTES arg) VSLb(ctx->vsl, SLT_Debug, "SO_SNDBUF fd=%d old=%d new=%d actual=%d", fd, oldbuf, buflen, newbuf); } + +/********************************************************************** + * For testing import code on bad vmod files (m00003.vtc) + */ + +const struct vmod_data Vmod_wrong0_Data = { + .vrt_major = 0, + .vrt_minor = 0, +}; + +//lint -save -e835 A zero has been given as left argument to operatorp'+' +const struct vmod_data Vmod_wrong1_Data = { + .vrt_major = VRT_MAJOR_VERSION, + .vrt_minor = VRT_MINOR_VERSION + 1, +}; +//lint -restore + +static const struct foo { + int bar; +} foo_struct[1]; + +const struct vmod_data Vmod_wrong2_Data = { + .vrt_major = VRT_MAJOR_VERSION, + .vrt_minor = VRT_MINOR_VERSION, + .name = "wrongN", + .func = foo_struct, + .func_len = sizeof foo_struct, + .func_name = "foo_struct", + .proto = "blablabla", +}; + +const struct vmod_data Vmod_wrong3_Data = { + .vrt_major = VRT_MAJOR_VERSION, + .vrt_minor = VRT_MINOR_VERSION, + .name = "wrongN", + .func = foo_struct, + .func_len = sizeof foo_struct, + .func_name = "foo_struct", + .proto = "blablabla", + .abi = "abiblabla", +}; From phk at FreeBSD.org Fri May 17 19:09:12 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 17 May 2019 19:09:12 +0000 (UTC) Subject: [master] 659f2af42 I had a good idea. Message-ID: <20190517190912.5EB43109021@lists.varnish-cache.org> commit 659f2af428935b83155c5e638d03e7c8501f2eaf Author: Poul-Henning Kamp Date: Fri May 17 19:08:12 2019 +0000 I had a good idea. diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst index d258654d8..70a7edc16 100644 --- a/doc/sphinx/phk/index.rst +++ b/doc/sphinx/phk/index.rst @@ -8,6 +8,7 @@ You may or may not want to know what Poul-Henning thinks. .. toctree:: :maxdepth: 1 + patent.rst lucky.rst apispaces.rst VSV00001.rst diff --git a/doc/sphinx/phk/patent.rst b/doc/sphinx/phk/patent.rst new file mode 100644 index 000000000..426e9ba13 --- /dev/null +++ b/doc/sphinx/phk/patent.rst @@ -0,0 +1,233 @@ +.. _phk_patent: + +A patently good idea +==================== + +When I was in USA, diplomas on the wall was very much a thing. + +I don't think I fully reverse-engineered the protocol for which +diplomas would get hung and which would be filed away, apart from +the unbreakable rule that, like it or not, anything your company +handed out was mandatory on the office-wall, no matter how embarrasing. + +Our paediatrician had diplomas for five or six steps of her education. + +My favourite pizzeria had a diploma for "Authentic Italian Food" +from a US organization suffering from fuzzy territorial perception. + +Co-workers had diplomas from their universities, OSHA, USAF, DoE, +CalTrans and who knows what. + +But the gold-standard of diplomas, at least amongst the engineers, +was having a US Patent on the wall, even if it only ever made them +a single dollar in assignment fee. + +I asked one of them about his patent and he answered wryly: *"It +proves to my boss and my mom that I had at least one unique idea +in my career."* + +Personally I do not think the patent system does what people think +it does, ie: protect the small inventor from big companies, so I +have no patents to my name, and in fact no diplomas on my wall at +all. + +But I still mentally carve a notch when I see one of my ideas +being validated in some form. + +Containers and Zones are not jails, but they know, and I know, where +they got the basic idea from, and that is plenty of validation +for my ego. + +Today is Store Bededag in Denmark, loosely translated "All Prayers +Day", by definition a friday and we, like many other danes, have +eloped to the beach-house for a long weekend. + +But being self-employed I puttered around with VCC, the VCL compiler, +this morning, and as a result, you will soon be able to say:: + + import vmod_with_impractically_long_name as v; + +My idea that Varnish would be configured in a Domain Specific +Language compiled to native code is obviously one of my better, +and about 10 years ago, that was becoming very obvious. + +In Norway `Varnish Software `_ were +being spun out of the Redpill-Linpro company. + +Artur Bergman, one of the first Varnish Cache power users, who ran +Wikias content delivery and hit our project like a blast-oven with +ideas, patches, measurements, general good cheer and incredibly low +tolerance for bull-shit, started the `Fastly CDN `_. + +Prior to that, I had done a bit of soul-searching myself, wondering +if I should try to take Varnish and run with it? + +In conventional economic theory, I would have patented the +VCL idea, and become as rich as the idea was good. + +But in all probable worlds, that would only have meant that the +idea would be dead as a doornail, I would not have made any money +from it, it would never have helped improve the web, and I would +have wasted much more of my life in meetings than would be good for +anybodys health. + +As if that wasn't enough, the very thought of having to hire somebody +scared me, but not nearly as much as the realization that if I built +a company with any number of employees, sooner or later I would +have to fire someone again. + +Writing code? Yes. + +Running a growing company? No. + +The result of my soul-searching was this email to announce@ where +I took myself out of the game: + +.. code-block:: text + + Subject: For the record: Varnish and Money + From: Poul-Henning Kamp + To: varnish-announce at varnish-cache.org + Date: Fri Nov 19 14:03:22 CET 2010 + + Just so everybody know where I stand on this... + + Poul-Henning + + -----BEGIN PGP SIGNED MESSAGE----- + Hash: SHA1 + + + Introduction + - ------------ + + As the main developer of the Varnish Software and the de-facto leader + of the Varnish Open Source Project, it is my desire to see Varnish + used and adopted as widely as possible. + + To the same ends, the founders of the Varnish Project chose the BSD + license to facilitate commercial exploitation of Varnish in all + forms, while protecting the independence of the Open Source Project. + + The BSD license is non-discriminatory, and makes no attempt to + separate the good guys from the bad guys, and neither should it. + + The Varnish Project, as a community, is a different story. + + While the BSD license can guarantee that Varnish, as software, will + always be available, a thriving Open Source Community takes a fair + bit more effort to hold together. + + Nothing can rip apart an Open Source project faster than competing + commercial interests playing dirty, and since Varnish has started + to cause serious amounts of money to shift around, it is time to + take this issue a bit more seriously. + + + Non-competition pledge: + - ----------------------- + + My interest in Varnish is developing capable quality software, and + making a living at the same time. + + In addition to Varnish, I have some long time good customers for + whom I do various weird things with computers and software, and + since they have stuck with me and paid my bills, I will stick with + them and send them more bills. + + The Varnish Moral License (VML) was drawn up to provide a money-stream + that can fund my Varnish-habit, and it was designed as an "arms-length" + construction to prevent it from taking control of the projects + direction. + + Therefore acquiring a VML does not mean that you get to tell me + what to do, or in which order I should do it. There is no "tit for + tat" involved. The only thing you get out of the VML, is that the + next version of Varnish will be better than the one we have now. + + Therefore: + + As long as I can keep my family fed, happy and warm this + way, I will not enter any other commercial activity related + to Varnish, and am more than happy to leave that field open + to everybody and anybody, who wants to try their hand. + + + Fairness pledge: + - ---------------- + + As the de-facto leader of the Varnish community, I believe that + the success or failure of open source rises and falls with the + community which backs it up. + + In general, there is a tacit assumption, that you take something + from the pot and you try put something back in the pot, each to his + own means and abilities. + + And the pot has plenty that needs filling: From answers to newbies + questions, bug-reports, patches, documentation, advocacy, VML funding, + hosting VUG meetings, writing articles for magazines, HOW-TO's for + blogs and so on, so this is no onerous demand for anybody. + + But the BSD license allows you to not participate in or contribute + to the community, and there are special times and circumstances + where that is the right thing, or even the only thing you can do, + and I recognize that. + + Therefore: + + I will treat everybody, who do not contribute negatively to + the Varnish community, equally and fairly, and try to foster + cooperation and justly resolve conflicts to the best of my + abilities. + + + Policy on Gifts: + - ---------------- + + People sometimes prefer to show their appreciation of Varnish by + sending me gifts. + + I really love that + + But please understand, that any and gifts or other appreciations I + may receive, from cartoons on my Amazon Wishlist, up to and including + pre-owned tropical tax-shelter islands, with conveniently unlocked + bank vaults filled with gold bars (one can always dream...), will + all be received and interpreted the same way: As tokens of + appreciation for deeds already done, and encouragement to me to + keep doing what is right and best for Varnish in the future. + + + Poul-Henning Kamp + + Signed with my PGP-key, November 19th, 2010, Slagelse, Denmark. + -----BEGIN PGP SIGNATURE----- + Version: GnuPG v1.4.10 (FreeBSD) + + iEYEARECAAYFAkzmdRkACgkQlftZhnGqOJOJwwCffytQ5kGP+Grh2unpNIIw8G2R + QcQAn18fGLT4Lx2ACBivtk5wEFy6fUcu + =3V52 + -----END PGP SIGNATURE----- + -- + Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 + phk at FreeBSD.ORG | TCP/IP since RFC 956 + FreeBSD committer | BSD since 4.3-tahoe + Never attribute to malice what can adequately be explained by incompetence. + +Today (20190517) Arturs `Fastly `_, company +went public on the New York Stock Exchange, and went up from $16 +to $24 in a matter of hours. So-called "financial analysts" write +that as a consequence Fastly is now worth 2+ Billion Dollars. + +I can say with 100% certainty and honesty that there is no way +I could *ever* have done that, that is entirely Arturs doing and +I know and admire how hard he worked to make it happen. + +Congratulations to Artur and the Fastly Crew! + +But I will steal some of Arturs thunder, and point to Fastlys IPO +as proof that at least once in my career, I had a unique idea worth +a billion dollars :-) + +*phk* From dridi at varni.sh Sat May 18 09:22:20 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Sat, 18 May 2019 11:22:20 +0200 Subject: [master] 0060ee941 Found a sneaky way to tests VCC's vmod-tasting code. In-Reply-To: <20190517081411.D5D67117CFD@lists.varnish-cache.org> References: <20190517081411.D5D67117CFD@lists.varnish-cache.org> Message-ID: On Fri, May 17, 2019 at 10:14 AM Poul-Henning Kamp wrote: > > > commit 0060ee9410e0fa1d8ca9785761b790abdab7d656 > Author: Poul-Henning Kamp > Date: Fri May 17 08:12:44 2019 +0000 > > Found a sneaky way to tests VCC's vmod-tasting code. > > diff --git a/bin/varnishtest/tests/m00003.vtc b/bin/varnishtest/tests/m00003.vtc > index e2f6dccab..4e5a60ff4 100644 > --- a/bin/varnishtest/tests/m00003.vtc > +++ b/bin/varnishtest/tests/m00003.vtc > @@ -14,7 +14,7 @@ varnish v1 -arg "-pvmod_path=${topbuild}/lib/libvmod_std/.libs/" \ > > varnish v1 -cliok "param.set vmod_path /nonexistent" > > -varnish v1 -errvcl {Could not load VMOD std} { > +varnish v1 -errvcl {Could not find VMOD std} { > import std; > } > > @@ -23,3 +23,35 @@ varnish v1 -cliok "param.set vmod_path ${topbuild}/lib/libvmod_std/.libs/" > varnish v1 -vcl+backend { > import std; > } > + > +varnish v1 -cliok "param.set vmod_path ${tmpdir}" > + > +shell "true > ${tmpdir}/libvmod_wrong.so" Did you lose your touch? By the way ${tmpdir} is the current directory so we can omit it when we don't need an absolute path. > +varnish v1 -errvcl {Could not open VMOD wrong} { > + import wrong; > +} > + > +shell "cp ${topbuild}/lib/libvmod_debug/.libs/libvmod_debug.so ${tmpdir}/libvmod_wrong.so" > + > +varnish v1 -errvcl {Malformed VMOD wrong} { > + import wrong; > +} > + > +varnish v1 -errvcl {Incompatible VMOD wrong0} { > + import wrong0 from "${tmpdir}/libvmod_wrong.so"; > +} > + > +varnish v1 -errvcl {Incompatible VMOD wrong1} { > + import wrong1 from "${tmpdir}/libvmod_wrong.so"; > +} > + > +varnish v1 -errvcl {Mangled VMOD wrong2} { > + import wrong2 from "${tmpdir}/libvmod_wrong.so"; > +} > + > +varnish v1 -errvcl {Wrong file for VMOD wrong3} { > + import wrong3 from "${tmpdir}/libvmod_wrong.so"; > +} > + > +shell "rm -f ${tmpdir}/libvmod_wrong.so" > diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c > index 6b0ac88a6..c0ffc19ba 100644 > --- a/lib/libvcc/vcc_compile.c > +++ b/lib/libvcc/vcc_compile.c > @@ -769,6 +769,7 @@ VCC_New(void) > VTAILQ_INIT(&tl->sources); > VTAILQ_INIT(&tl->procs); > VTAILQ_INIT(&tl->sym_objects); > + VTAILQ_INIT(&tl->sym_vmods); > > tl->nsources = 0; > > diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h > index f164a9e92..58eb570c6 100644 > --- a/lib/libvcc/vcc_compile.h > +++ b/lib/libvcc/vcc_compile.h > @@ -258,6 +258,7 @@ struct vcc { > const char *default_probe; > > VTAILQ_HEAD(, symbol) sym_objects; > + VTAILQ_HEAD(, symbol) sym_vmods; > > unsigned unique; > unsigned vmod_count; > diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c > index 6eadd44dc..ff4831198 100644 > --- a/lib/libvcc/vcc_vmod.c > +++ b/lib/libvcc/vcc_vmod.c > @@ -40,19 +40,26 @@ > #include "vmod_abi.h" > #include "vsb.h" > > +struct vmod_open { > + unsigned magic; > +#define VMOD_OPEN_MAGIC 0x9995b1f3 > + void *hdl; > + const char *err; > +}; > + > static int > vcc_path_dlopen(void *priv, const char *fn) > { > - void *hdl, **pp; > + struct vmod_open *vop; > > - AN(priv); > + CAST_OBJ_NOTNULL(vop, priv, VMOD_OPEN_MAGIC); > AN(fn); > > - hdl = dlopen(fn, RTLD_NOW | RTLD_LOCAL); > - if (hdl == NULL) > + vop->hdl = dlopen(fn, RTLD_NOW | RTLD_LOCAL); > + if (vop->hdl == NULL) { > + vop->err = dlerror(); > return (-1); > - pp = priv; > - *pp = hdl; > + } > return (0); > } > > @@ -156,7 +163,6 @@ vcc_json_wildcard(struct vcc *tl, struct symbol *msym, struct symbol *tsym) > void > vcc_ParseImport(struct vcc *tl) > { > - void *hdl; > char fn[1024], *fnp, *fnpx; > char buf[256]; > const char *p; > @@ -166,14 +172,17 @@ vcc_ParseImport(struct vcc *tl) > const struct vmod_data *vmd; > struct vjsn *vj; > int again = 0; > + struct vmod_open vop[1]; > > + INIT_OBJ(vop, VMOD_OPEN_MAGIC); > t1 = tl->t; > SkipToken(tl, ID); /* "import" */ > > > ExpectErr(tl, ID); > mod = tl->t; > - msym = VCC_SymbolGet(tl, SYM_NONE, SYMTAB_NOERR, XREF_NONE); > + vcc_NextToken(tl); > + msym = VCC_SymbolGetTok(tl, SYM_NONE, SYMTAB_NOERR, XREF_NONE, mod); > > if (msym != NULL && msym->kind != SYM_VMOD) { > /* > @@ -188,13 +197,16 @@ vcc_ParseImport(struct vcc *tl) > again = 1; > } else { > > - msym = VCC_SymbolGet(tl, SYM_VMOD, SYMTAB_CREATE, XREF_NONE); > + msym = VCC_SymbolGetTok(tl, SYM_VMOD, > + SYMTAB_CREATE, XREF_NONE, mod); > ERRCHK(tl); > AN(msym); > msym->def_b = t1; > msym->def_e = tl->t; > } > > + VTAILQ_INSERT_TAIL(&tl->sym_vmods, msym, sideways); > + > if (tl->t->tok == ID) { > if (!vcc_IdIs(tl->t, "from")) { > VSB_printf(tl->sb, "Expected 'from path ...'\n"); > @@ -225,12 +237,17 @@ vcc_ParseImport(struct vcc *tl) > if (!again) > msym->def_e = tl->t; > > - > - if (VFIL_searchpath(tl->vmod_path, vcc_path_dlopen, &hdl, fn, &fnpx)) { > - VSB_printf(tl->sb, "Could not load VMOD %.*s\n", PF(mod)); > - VSB_printf(tl->sb, "\tFile name: %s\n", > - fnpx != NULL ? fnpx : fn); > - VSB_printf(tl->sb, "\tdlerror: %s\n", dlerror()); > + if (VFIL_searchpath(tl->vmod_path, vcc_path_dlopen, vop, fn, &fnpx)) { > + if (vop->err == NULL) { > + VSB_printf(tl->sb, > + "Could not find VMOD %.*s\n", PF(mod)); > + } else { > + VSB_printf(tl->sb, > + "Could not open VMOD %.*s\n", PF(mod)); > + VSB_printf(tl->sb, "\tFile name: %s\n", > + fnpx != NULL ? fnpx : fn); > + VSB_printf(tl->sb, "\tdlerror: %s\n", vop->err); > + } > vcc_ErrWhere(tl, mod); > free(fnpx); > return; > @@ -241,7 +258,7 @@ vcc_ParseImport(struct vcc *tl) > free(fnpx); > > bprintf(buf, "Vmod_%.*s_Data", PF(mod)); > - vmd = dlsym(hdl, buf); > + vmd = dlsym(vop->hdl, buf); > if (vmd == NULL) { > VSB_printf(tl->sb, "Malformed VMOD %.*s\n", PF(mod)); > VSB_printf(tl->sb, "\tFile name: %s\n", fnp); > @@ -250,7 +267,7 @@ vcc_ParseImport(struct vcc *tl) > return; > } > if (vmd->vrt_major == 0 && vmd->vrt_minor == 0 && > - strcmp(vmd->abi, VMOD_ABI_Version) != 0) { > + (vmd->abi == NULL || strcmp(vmd->abi, VMOD_ABI_Version) != 0)) { > VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(mod)); > VSB_printf(tl->sb, "\tFile name: %s\n", fnp); > VSB_printf(tl->sb, "\tABI mismatch, expected <%s>, got <%s>\n", > @@ -282,7 +299,7 @@ vcc_ParseImport(struct vcc *tl) > } > > if (!vcc_IdIs(mod, vmd->name)) { > - VSB_printf(tl->sb, "Wrong VMOD file %.*s\n", PF(mod)); > + VSB_printf(tl->sb, "Wrong file for VMOD %.*s\n", PF(mod)); > VSB_printf(tl->sb, "\tFile name: %s\n", fnp); > VSB_printf(tl->sb, "\tContains vmod \"%s\"\n", vmd->name); > vcc_ErrWhere(tl, mod); > @@ -298,7 +315,7 @@ vcc_ParseImport(struct vcc *tl) > vcc_ErrWhere2(tl, msym->def_b, msym->def_e); > } > if (again) { > - AZ(dlclose(hdl)); > + AZ(dlclose(vop->hdl)); > return; > } > > diff --git a/lib/libvmod_debug/Makefile.am b/lib/libvmod_debug/Makefile.am > index 4823e1b4d..8c6df5232 100644 > --- a/lib/libvmod_debug/Makefile.am > +++ b/lib/libvmod_debug/Makefile.am > @@ -5,8 +5,17 @@ libvmod_debug_la_SOURCES = \ > vmod_debug_obj.c \ > vmod_debug_dyn.c > > + > include $(srcdir)/automake_boilerplate.am > > +# Allow Vmod_wrong*_Data to be exported > +libvmod_debug_la_LDFLAGS = \ > + -export-symbols-regex 'Vmod_.*_Data' \ > + $(AM_LDFLAGS) \ > + $(VMOD_LDFLAGS) \ > + @SAN_LDFLAGS@ > + > + > # not --strict > vmodtoolargs = --boilerplate > > diff --git a/lib/libvmod_debug/flint.lnt b/lib/libvmod_debug/flint.lnt > index 36d225583..2a43e7057 100644 > --- a/lib/libvmod_debug/flint.lnt > +++ b/lib/libvmod_debug/flint.lnt > @@ -1,3 +1,6 @@ > -esym(759, xyzzy_enum_*) // header declaration for symbol '___' defined at (___) > -esym(765, xyzzy_enum_*) // external '___' (___) could be made static > > +-esym(765, Vmod_wrong*_Data) // external '___' (___) could be made static > +-esym(714, Vmod_wrong*_Data) // external '___' (___) could be made static > +-esym(754, foo::bar) // local struct member not referenced > diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c > index 4dac3f3dc..0d9f9d356 100644 > --- a/lib/libvmod_debug/vmod_debug.c > +++ b/lib/libvmod_debug/vmod_debug.c > @@ -134,7 +134,7 @@ xyzzy_rot13_bytes(struct req *req, enum vdp_action act, void **priv, > } > } > if (i >= 0) > - retval = VDP_bytes(req, VDP_FLUSH, q, i + 1); > + retval = VDP_bytes(req, VDP_FLUSH, q, i + 1L); > return (retval); > } > > @@ -854,3 +854,44 @@ xyzzy_sndbuf(VRT_CTX, VCL_BYTES arg) > VSLb(ctx->vsl, SLT_Debug, "SO_SNDBUF fd=%d old=%d new=%d actual=%d", > fd, oldbuf, buflen, newbuf); > } > + > +/********************************************************************** > + * For testing import code on bad vmod files (m00003.vtc) > + */ > + > +const struct vmod_data Vmod_wrong0_Data = { > + .vrt_major = 0, > + .vrt_minor = 0, > +}; > + > +//lint -save -e835 A zero has been given as left argument to operatorp'+' > +const struct vmod_data Vmod_wrong1_Data = { > + .vrt_major = VRT_MAJOR_VERSION, > + .vrt_minor = VRT_MINOR_VERSION + 1, > +}; > +//lint -restore > + > +static const struct foo { > + int bar; > +} foo_struct[1]; > + > +const struct vmod_data Vmod_wrong2_Data = { > + .vrt_major = VRT_MAJOR_VERSION, > + .vrt_minor = VRT_MINOR_VERSION, > + .name = "wrongN", > + .func = foo_struct, > + .func_len = sizeof foo_struct, > + .func_name = "foo_struct", > + .proto = "blablabla", > +}; > + > +const struct vmod_data Vmod_wrong3_Data = { > + .vrt_major = VRT_MAJOR_VERSION, > + .vrt_minor = VRT_MINOR_VERSION, > + .name = "wrongN", > + .func = foo_struct, > + .func_len = sizeof foo_struct, > + .func_name = "foo_struct", > + .proto = "blablabla", > + .abi = "abiblabla", > +}; > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From phk at phk.freebsd.dk Sat May 18 09:38:26 2019 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Sat, 18 May 2019 09:38:26 +0000 Subject: [master] 0060ee941 Found a sneaky way to tests VCC's vmod-tasting code. In-Reply-To: References: <20190517081411.D5D67117CFD@lists.varnish-cache.org> Message-ID: <6877.1558172306@critter.freebsd.dk> -------- In message , Dridi Boukelmoune writes: >On Fri, May 17, 2019 at 10:14 AM Poul-Henning Kamp wrote: >> +varnish v1 -cliok "param.set vmod_path ${tmpdir}" >> + >> +shell "true > ${tmpdir}/libvmod_wrong.so" > >Did you lose your touch? I'm not sure I follow you here ? >By the way ${tmpdir} is the current directory so we can omit it when >we don't need an absolute path. I know, but I dont mind the verbosity if it makes it easier to see what the test-case actually does. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From dridi at varni.sh Sun May 19 16:03:27 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Sun, 19 May 2019 18:03:27 +0200 Subject: [master] 0060ee941 Found a sneaky way to tests VCC's vmod-tasting code. In-Reply-To: <6877.1558172306@critter.freebsd.dk> References: <20190517081411.D5D67117CFD@lists.varnish-cache.org> <6877.1558172306@critter.freebsd.dk> Message-ID: On Sun, May 19, 2019 at 8:54 AM Poul-Henning Kamp wrote: > > -------- > In message > , Dridi Boukelmoune writes: > >On Fri, May 17, 2019 at 10:14 AM Poul-Henning Kamp wrote: > > >> +varnish v1 -cliok "param.set vmod_path ${tmpdir}" > >> + > >> +shell "true > ${tmpdir}/libvmod_wrong.so" > > > >Did you lose your touch? > > I'm not sure I follow you here ? Isn't touch(1) good enough at this point in the test case to create an empty file? From phk at phk.freebsd.dk Mon May 20 05:54:40 2019 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Mon, 20 May 2019 05:54:40 +0000 Subject: [master] 0060ee941 Found a sneaky way to tests VCC's vmod-tasting code. In-Reply-To: References: <20190517081411.D5D67117CFD@lists.varnish-cache.org> <6877.1558172306@critter.freebsd.dk> Message-ID: <4513.1558331680@critter.freebsd.dk> -------- In message , Dridi Boukelmoune writes: >> >> +shell "true > ${tmpdir}/libvmod_wrong.so" >> > >> >Did you lose your touch? >> >> I'm not sure I follow you here ? > >Isn't touch(1) good enough at this point in the test case to create an >empty file? 1. Touch(1) would not truncate an existing file (not that there should be one.) 2. True(1) is usually built into the shell, so we save an exec(2). -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From phk at FreeBSD.org Mon May 20 10:22:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 20 May 2019 10:22:10 +0000 (UTC) Subject: [master] 8ae3d9782 Extend the syntax to 'import vmod [as name] [from "path"]; ' Message-ID: <20190520102210.25E9C1090B1@lists.varnish-cache.org> commit 8ae3d978224587ae6021a78d5821b9978ae65f24 Author: Poul-Henning Kamp Date: Mon May 20 10:20:43 2019 +0000 Extend the syntax to 'import vmod [as name] [from "path"];' There are a surprising number of subtle corner cases here, and I think I have handled them all correctly, but caveat emptor... diff --git a/bin/varnishtest/tests/m00000.vtc b/bin/varnishtest/tests/m00000.vtc index 1931db422..a62ff527b 100644 --- a/bin/varnishtest/tests/m00000.vtc +++ b/bin/varnishtest/tests/m00000.vtc @@ -11,10 +11,12 @@ varnish v1 -vcl+backend { import debug; import vtc; import debug; // again + import debug as dbg; + import debug as dbg; // again sub vcl_init { - new objx = debug.obj(); - debug.vsc_new(); + new objx = dbg.obj(); + dbg.vsc_new(); } sub vcl_synth { diff --git a/bin/varnishtest/tests/m00008.vtc b/bin/varnishtest/tests/m00008.vtc index a0bc5b2a3..7b5647487 100644 --- a/bin/varnishtest/tests/m00008.vtc +++ b/bin/varnishtest/tests/m00008.vtc @@ -12,6 +12,11 @@ varnish v1 -errvcl {Module debug conflicts with other symbol.} { } } +varnish v1 -errvcl {Another module already imported as foo} { + import debug as foo; + import directors as foo; +} + server s1 { rxreq txresp diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 58eb570c6..20213c0b9 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -144,7 +144,7 @@ struct symbol { int hirev; struct symbol *parent; - const struct symbol *vmod; + const char *vmod_name; sym_wildcard_t *wildcard; vcc_kind_t kind; diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 9638b519f..c74cc4c4b 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -359,10 +359,10 @@ vcc_priv_arg(struct vcc *tl, const char *p, const struct symbol *sym) struct procprivhead *marklist = NULL; AN(sym); - AN(sym->vmod); + AN(sym->vmod_name); if (!strcmp(p, "PRIV_VCL")) - return (vcc_mk_expr(VOID, "&vmod_priv_%s", sym->vmod->name)); + return (vcc_mk_expr(VOID, "&vmod_priv_%s", sym->vmod_name)); if (!strcmp(p, "PRIV_CALL")) { bprintf(buf, "vmod_priv_%u", tl->unique++); @@ -383,9 +383,9 @@ vcc_priv_arg(struct vcc *tl, const char *p, const struct symbol *sym) } AN(f); AN(marklist); - bprintf(buf, "ARG_priv_%s_%s", f, sym->vmod->name); + bprintf(buf, "ARG_priv_%s_%s", f, sym->vmod_name); - if (vcc_MarkPriv(tl, marklist, sym->vmod->name) == NULL) + if (vcc_MarkPriv(tl, marklist, sym->vmod_name) == NULL) VSB_printf(tl->curproc->prologue, " struct vmod_priv *%s = " "VRT_priv_%s(ctx, &VGC_vmod_%s);\n" @@ -394,7 +394,7 @@ vcc_priv_arg(struct vcc *tl, const char *p, const struct symbol *sym) "for vmod %s\");\n" " return;\n" " }\n", - buf, f, sym->vmod->name, buf, f, sym->vmod->name); + buf, f, sym->vmod_name, buf, f, sym->vmod_name); return (vcc_mk_expr(VOID, "%s", buf)); } diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index ff4831198..38863ceb5 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -64,13 +64,12 @@ vcc_path_dlopen(void *priv, const char *fn) } static void -func_sym(struct symbol *sym, const struct symbol *vmod, - const struct vjsn_val *v) +func_sym(struct symbol *sym, const char *vmod_name, const struct vjsn_val *v) { assert(v->type == VJSN_ARRAY); sym->action = vcc_Act_Call; - sym->vmod = vmod; + sym->vmod_name = vmod_name; sym->eval = vcc_Eval_SymFunc; sym->eval_priv = v; v = VTAILQ_FIRST(&v->children); @@ -82,17 +81,16 @@ func_sym(struct symbol *sym, const struct symbol *vmod, } static void -vcc_json_always(struct vcc *tl, const struct symbol *msym) +vcc_json_always(struct vcc *tl, const struct vjsn *vj, const char *vmod_name) { struct inifin *ifp; - const struct vjsn *vj; const struct vjsn_val *vv, *vv2; double vmod_syntax = 0.0; + AN(vj); + AN(vmod_name); ifp = NULL; - CAST_OBJ_NOTNULL(vj, msym->eval_priv, VJSN_MAGIC); - VTAILQ_FOREACH(vv, &vj->value->children, list) { assert(vv->type == VJSN_ARRAY); vv2 = VTAILQ_FIRST(&vv->children); @@ -111,13 +109,13 @@ vcc_json_always(struct vcc *tl, const struct symbol *msym) VSB_printf(ifp->ini, "\tif (%s(ctx, &vmod_priv_%s, VCL_EVENT_LOAD))\n" "\t\treturn(1);", - vv2->value, msym->name); + vv2->value, vmod_name); VSB_printf(ifp->fin, "\t\t(void)%s(ctx, &vmod_priv_%s,\n" "\t\t\t VCL_EVENT_DISCARD);", - vv2->value, msym->name); + vv2->value, vmod_name); VSB_printf(ifp->event, "%s(ctx, &vmod_priv_%s, ev)", - vv2->value, msym->name); + vv2->value, vmod_name); } else if (!strcmp(vv2->value, "$FUNC")) { } else if (!strcmp(vv2->value, "$OBJ")) { } else { @@ -147,66 +145,104 @@ vcc_json_wildcard(struct vcc *tl, struct symbol *msym, struct symbol *tsym) !strcmp(vv2->value, tsym->name)) { tsym->kind = SYM_FUNC; tsym->noref = 1; - func_sym(tsym, msym, VTAILQ_NEXT(vv2, list)); + func_sym(tsym, msym->vmod_name, VTAILQ_NEXT(vv2, list)); return; } else if (!strcmp(vv1->value, "$OBJ") && !strcmp(vv2->value, tsym->name)) { tsym->kind = SYM_OBJECT; tsym->eval_priv = vv2; - tsym->vmod = msym; + tsym->vmod_name = msym->vmod_name; return; } } tl->err = 1; } +static const struct vmod_data * +vcc_VmodSanity(struct vcc *tl, void *hdl, struct token *mod, char *fnp) +{ + char buf[256]; + const struct vmod_data *vmd; + + bprintf(buf, "Vmod_%.*s_Data", PF(mod)); + vmd = dlsym(hdl, buf); + if (vmd == NULL) { + VSB_printf(tl->sb, "Malformed VMOD %.*s\n", PF(mod)); + VSB_printf(tl->sb, "\tFile name: %s\n", fnp); + VSB_printf(tl->sb, "\t(no Vmod_Data symbol)\n"); + vcc_ErrWhere(tl, mod); + return (NULL); + } + if (vmd->vrt_major == 0 && vmd->vrt_minor == 0 && + (vmd->abi == NULL || strcmp(vmd->abi, VMOD_ABI_Version) != 0)) { + VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(mod)); + VSB_printf(tl->sb, "\tFile name: %s\n", fnp); + VSB_printf(tl->sb, "\tABI mismatch, expected <%s>, got <%s>\n", + VMOD_ABI_Version, vmd->abi); + vcc_ErrWhere(tl, mod); + return (NULL); + } + if (vmd->vrt_major != 0 && (vmd->vrt_major != VRT_MAJOR_VERSION || + vmd->vrt_minor > VRT_MINOR_VERSION)) { + VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(mod)); + VSB_printf(tl->sb, "\tFile name: %s\n", fnp); + VSB_printf(tl->sb, "\tVMOD wants ABI version %u.%u\n", + vmd->vrt_major, vmd->vrt_minor); + VSB_printf(tl->sb, "\tvarnishd provides ABI version %u.%u\n", + VRT_MAJOR_VERSION, VRT_MINOR_VERSION); + vcc_ErrWhere(tl, mod); + return (NULL); + } + if (vmd->name == NULL || + vmd->func == NULL || + vmd->func_len <= 0 || + vmd->proto == NULL || + vmd->abi == NULL) { + VSB_printf(tl->sb, "Mangled VMOD %.*s\n", PF(mod)); + VSB_printf(tl->sb, "\tFile name: %s\n", fnp); + VSB_printf(tl->sb, "\tInconsistent metadata\n"); + vcc_ErrWhere(tl, mod); + return (NULL); + } + if (!vcc_IdIs(mod, vmd->name)) { + VSB_printf(tl->sb, "Wrong file for VMOD %.*s\n", PF(mod)); + VSB_printf(tl->sb, "\tFile name: %s\n", fnp); + VSB_printf(tl->sb, "\tContains vmod \"%s\"\n", vmd->name); + vcc_ErrWhere(tl, mod); + return (NULL); + } + return (vmd); +} + void vcc_ParseImport(struct vcc *tl) { - char fn[1024], *fnp, *fnpx; - char buf[256]; + char fn[1024], *fnpx; const char *p; - struct token *mod, *t1; + struct token *mod, *tmod, *t1; struct inifin *ifp; - struct symbol *msym; + struct symbol *msym, *vsym; const struct vmod_data *vmd; struct vjsn *vj; - int again = 0; struct vmod_open vop[1]; INIT_OBJ(vop, VMOD_OPEN_MAGIC); t1 = tl->t; SkipToken(tl, ID); /* "import" */ - - ExpectErr(tl, ID); + ExpectErr(tl, ID); /* "vmod_name" */ mod = tl->t; vcc_NextToken(tl); - msym = VCC_SymbolGetTok(tl, SYM_NONE, SYMTAB_NOERR, XREF_NONE, mod); - - if (msym != NULL && msym->kind != SYM_VMOD) { - /* - * We need to make sure the entire std.* namespace is empty - */ - VSB_printf(tl->sb, "Module %.*s conflicts with other symbol.\n", - PF(mod)); - vcc_ErrWhere2(tl, t1, tl->t); - return; - } - if (msym != NULL) { - again = 1; - } else { - msym = VCC_SymbolGetTok(tl, SYM_VMOD, - SYMTAB_CREATE, XREF_NONE, mod); - ERRCHK(tl); - AN(msym); - msym->def_b = t1; - msym->def_e = tl->t; + if (tl->t->tok == ID && vcc_IdIs(tl->t, "as")) { + SkipToken(tl, ID); /* "as" */ + ExpectErr(tl, ID); /* "vcl_name" */ + tmod = tl->t; + vcc_NextToken(tl); + } else { + tmod = mod; } - VTAILQ_INSERT_TAIL(&tl->sym_vmods, msym, sideways); - if (tl->t->tok == ID) { if (!vcc_IdIs(tl->t, "from")) { VSB_printf(tl->sb, "Expected 'from path ...'\n"); @@ -234,9 +270,6 @@ vcc_ParseImport(struct vcc *tl) SkipToken(tl, ';'); - if (!again) - msym->def_e = tl->t; - if (VFIL_searchpath(tl->vmod_path, vcc_path_dlopen, vop, fn, &fnpx)) { if (vop->err == NULL) { VSB_printf(tl->sb, @@ -253,72 +286,58 @@ vcc_ParseImport(struct vcc *tl) return; } - AN(fnpx); - fnp = TlDup(tl, fnpx); - free(fnpx); - - bprintf(buf, "Vmod_%.*s_Data", PF(mod)); - vmd = dlsym(vop->hdl, buf); - if (vmd == NULL) { - VSB_printf(tl->sb, "Malformed VMOD %.*s\n", PF(mod)); - VSB_printf(tl->sb, "\tFile name: %s\n", fnp); - VSB_printf(tl->sb, "\t(no Vmod_Data symbol)\n"); - vcc_ErrWhere(tl, mod); - return; - } - if (vmd->vrt_major == 0 && vmd->vrt_minor == 0 && - (vmd->abi == NULL || strcmp(vmd->abi, VMOD_ABI_Version) != 0)) { - VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(mod)); - VSB_printf(tl->sb, "\tFile name: %s\n", fnp); - VSB_printf(tl->sb, "\tABI mismatch, expected <%s>, got <%s>\n", - VMOD_ABI_Version, vmd->abi); - vcc_ErrWhere(tl, mod); - return; - } - if (vmd->vrt_major != 0 && (vmd->vrt_major != VRT_MAJOR_VERSION || - vmd->vrt_minor > VRT_MINOR_VERSION)) { - VSB_printf(tl->sb, "Incompatible VMOD %.*s\n", PF(mod)); - VSB_printf(tl->sb, "\tFile name: %s\n", fnp); - VSB_printf(tl->sb, "\tVMOD wants ABI version %u.%u\n", - vmd->vrt_major, vmd->vrt_minor); - VSB_printf(tl->sb, "\tvarnishd provides ABI version %u.%u\n", - VRT_MAJOR_VERSION, VRT_MINOR_VERSION); - vcc_ErrWhere(tl, mod); - return; - } - if (vmd->name == NULL || - vmd->func == NULL || - vmd->func_len <= 0 || - vmd->proto == NULL || - vmd->abi == NULL) { - VSB_printf(tl->sb, "Mangled VMOD %.*s\n", PF(mod)); - VSB_printf(tl->sb, "\tFile name: %s\n", fnp); - VSB_printf(tl->sb, "\tInconsistent metadata\n"); - vcc_ErrWhere(tl, mod); + vmd = vcc_VmodSanity(tl, vop->hdl, mod, fnpx); + if (vmd == NULL || tl->err) { + AZ(dlclose(vop->hdl)); + free(fnpx); return; } - if (!vcc_IdIs(mod, vmd->name)) { - VSB_printf(tl->sb, "Wrong file for VMOD %.*s\n", PF(mod)); - VSB_printf(tl->sb, "\tFile name: %s\n", fnp); - VSB_printf(tl->sb, "\tContains vmod \"%s\"\n", vmd->name); - vcc_ErrWhere(tl, mod); - return; - } + msym = VCC_SymbolGetTok(tl, SYM_NONE, SYMTAB_NOERR, XREF_NONE, tmod); - if (again && strcmp(vmd->file_id, msym->extra)) { + if (msym != NULL && msym->kind == SYM_VMOD && + !strcmp(msym->extra, vmd->file_id)) { + /* Identical import is OK */ + AZ(dlclose(vop->hdl)); + free(fnpx); + return; + } else if (msym != NULL && msym->kind == SYM_VMOD) { VSB_printf(tl->sb, - "Different version of module %.*s already imported.\n", - PF(mod)); + "Another module already imported as %.*s.\n", PF(tmod)); + vcc_ErrWhere2(tl, t1, tl->t); + AZ(dlclose(vop->hdl)); + free(fnpx); + return; + } else if (msym != NULL) { + VSB_printf(tl->sb, + "Module %.*s conflicts with other symbol.\n", PF(tmod)); vcc_ErrWhere2(tl, t1, tl->t); - VSB_printf(tl->sb, "Previous import was here:\n"); - vcc_ErrWhere2(tl, msym->def_b, msym->def_e); - } - if (again) { AZ(dlclose(vop->hdl)); + free(fnpx); return; } + msym = VCC_SymbolGetTok(tl, SYM_VMOD, SYMTAB_CREATE, XREF_NONE, tmod); + ERRCHK(tl); + AN(msym); + msym->def_b = t1; + msym->def_e = tl->t; + + VTAILQ_FOREACH(vsym, &tl->sym_vmods, sideways) { + assert(vsym->kind == SYM_VMOD); + if (!strcmp(vsym->extra, vmd->file_id)) { + /* Already loaded under different name */ + msym->eval_priv = vsym->eval_priv; + msym->wildcard = vsym->wildcard; + msym->extra = vsym->extra; + msym->vmod_name = vsym->vmod_name; + AZ(dlclose(vop->hdl)); + return; + } + } + + VTAILQ_INSERT_TAIL(&tl->sym_vmods, msym, sideways); + ifp = New_IniFin(tl); VSB_printf(ifp->ini, "\tif (VPI_Vmod_Init(ctx,\n"); @@ -328,7 +347,7 @@ vcc_ParseImport(struct vcc *tl) VSB_printf(ifp->ini, "\t sizeof(%s),\n", vmd->func_name); VSB_printf(ifp->ini, "\t \"%.*s\",\n", PF(mod)); VSB_printf(ifp->ini, "\t "); - VSB_quote(ifp->ini, fnp, -1, VSB_QUOTE_CSTR); + VSB_quote(ifp->ini, fnpx, -1, VSB_QUOTE_CSTR); VSB_printf(ifp->ini, ",\n"); AN(vmd); AN(vmd->file_id); @@ -339,7 +358,7 @@ vcc_ParseImport(struct vcc *tl) VSB_printf(ifp->ini, "\t\treturn(1);"); VSB_printf(tl->fi, "%s VMOD %s ./vmod_cache/_vmod_%.*s.%s */\n", - VCC_INFO_PREFIX, fnp, PF(mod), vmd->file_id); + VCC_INFO_PREFIX, fnpx, PF(mod), vmd->file_id); /* XXX: zero the function pointer structure ?*/ VSB_printf(ifp->fin, "\t\tVRT_priv_fini(&vmod_priv_%.*s);", PF(mod)); @@ -352,14 +371,16 @@ vcc_ParseImport(struct vcc *tl) msym->eval_priv = vj; msym->wildcard = vcc_json_wildcard; msym->extra = TlDup(tl, vmd->file_id); + msym->vmod_name = TlDup(tl, vmd->name); - vcc_json_always(tl, msym); + vcc_json_always(tl, vj, msym->vmod_name); Fh(tl, 0, "\n/* --- BEGIN VMOD %.*s --- */\n\n", PF(mod)); Fh(tl, 0, "static struct vmod *VGC_vmod_%.*s;\n", PF(mod)); Fh(tl, 0, "static struct vmod_priv vmod_priv_%.*s;\n", PF(mod)); Fh(tl, 0, "\n%s\n", vmd->proto); Fh(tl, 0, "\n/* --- END VMOD %.*s --- */\n\n", PF(mod)); + free(fnpx); } void v_matchproto_(sym_act_f) @@ -462,7 +483,7 @@ vcc_Act_New(struct vcc *tl, struct token *t, struct symbol *sym) AZ(VSB_finish(buf)); sy3 = VCC_MkSym(tl, VSB_data(buf), SYM_FUNC, VCL_LOW, VCL_HIGH); AN(sy3); - func_sym(sy3, sy2->vmod, VTAILQ_NEXT(vf, list)); + func_sym(sy3, sy2->vmod_name, VTAILQ_NEXT(vf, list)); sy3->extra = p; vv = VTAILQ_NEXT(vv, list); } diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 1b40b10b8..4d8734d4c 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -603,7 +603,7 @@ class ModuleStanza(Stanza): write_rst_hdr(fo, "SYNOPSIS", "=") fo.write("\n") fo.write(".. parsed-literal::\n\n") - fo.write(' import %s [from "path"]\n' % self.vcc.modname) + fo.write(' import %s [as name] [from "path"]\n' % self.vcc.modname) fo.write(" \n") for c in self.vcc.contents: c.synopsis(fo, man) From dridi.boukelmoune at gmail.com Mon May 20 11:14:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 20 May 2019 11:14:09 +0000 (UTC) Subject: [master] cfe0beeed Add Debug to the default VSL mask Message-ID: <20190520111409.E3C4110A069@lists.varnish-cache.org> commit cfe0beeedefd435be722f5bd3acdfdae7dcd3663 Author: Dridi Boukelmoune Date: Thu May 16 10:27:49 2019 +0200 Add Debug to the default VSL mask The name implies that this is not for production usage. diff --git a/bin/varnishd/mgt/mgt_param_bits.c b/bin/varnishd/mgt/mgt_param_bits.c index 263d8a34d..7ea66d7e1 100644 --- a/bin/varnishd/mgt/mgt_param_bits.c +++ b/bin/varnishd/mgt/mgt_param_bits.c @@ -135,6 +135,7 @@ tweak_vsl_mask(struct vsb *vsb, const struct parspec *par, const char *arg) (void)bit(mgt_param.vsl_mask, SLT_ObjProtocol, BSET); (void)bit(mgt_param.vsl_mask, SLT_ObjReason, BSET); (void)bit(mgt_param.vsl_mask, SLT_ObjStatus, BSET); + (void)bit(mgt_param.vsl_mask, SLT_Debug, BSET); } else { return (bit_tweak(vsb, mgt_param.vsl_mask, SLT__Reserved, arg, VSL_tags, diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 90e4d403d..010c2d915 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -424,6 +424,7 @@ varnish_launch(struct varnish *v) VSB_printf(vsb, " -p sigsegv_handler=on"); VSB_printf(vsb, " -p thread_pool_min=10"); VSB_printf(vsb, " -p debug=+vtc_mode"); + VSB_printf(vsb, " -p vsl_mask=+Debug"); if (!v->has_a_arg) { VSB_printf(vsb, " -a '%s'", "127.0.0.1:0"); if (v->proto != NULL) From phk at FreeBSD.org Tue May 21 07:25:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 21 May 2019 07:25:09 +0000 (UTC) Subject: [master] ea3c109c5 Skip magic check when on boc when gunzip streaming Message-ID: <20190521072509.A6C69630D4@lists.varnish-cache.org> commit ea3c109c58314a78bbfec3a5997ab3c62a9c2016 Author: Reza Naghibi Date: Tue May 14 13:12:22 2019 -0400 Skip magic check when on boc when gunzip streaming We do not hold a reference, the magic can be unstable. diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 9be9b1d7c..46146beb3 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -294,7 +294,6 @@ vdp_gunzip_init(struct req *req, void **priv) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); - CHECK_OBJ_ORNULL(req->objcore->boc, BOC_MAGIC); vg = VGZ_NewGunzip(req->vsl, "U D -"); AN(vg); From phk at FreeBSD.org Tue May 21 07:25:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 21 May 2019 07:25:09 +0000 (UTC) Subject: [master] c611b9a61 Guard against OA_GZIPBITS streaming race Message-ID: <20190521072509.7C855630D1@lists.varnish-cache.org> commit c611b9a615b2d035c8631febc9f60efb339af67e Author: Reza Naghibi Date: Wed Feb 13 10:45:40 2019 -0500 Guard against OA_GZIPBITS streaming race diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index a48715055..9be9b1d7c 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -292,6 +292,10 @@ vdp_gunzip_init(struct req *req, void **priv) ssize_t dl; uint64_t u; + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_NOTNULL(req->objcore, OBJCORE_MAGIC); + CHECK_OBJ_ORNULL(req->objcore->boc, BOC_MAGIC); + vg = VGZ_NewGunzip(req->vsl, "U D -"); AN(vg); if (vgz_getmbuf(vg)) { @@ -306,6 +310,10 @@ vdp_gunzip_init(struct req *req, void **priv) req->resp_len = -1; + /* OA_GZIPBITS is not stable yet */ + if (req->objcore->boc) + return (0); + p = ObjGetAttr(req->wrk, req->objcore, OA_GZIPBITS, &dl); if (p != NULL && dl == 32) { u = vbe64dec(p + 24); From dridi.boukelmoune at gmail.com Tue May 21 07:46:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 21 May 2019 07:46:09 +0000 (UTC) Subject: [master] ce47e75d8 Don't defer an std.ip() failure to WS_Alloc'ate Message-ID: <20190521074609.449E963C49@lists.varnish-cache.org> commit ce47e75d80ce17b6fc21b994db057474093c75a9 Author: Dridi Boukelmoune Date: Thu May 9 15:21:11 2019 +0200 Don't defer an std.ip() failure to WS_Alloc'ate At this point the workspace already overflowed so there's no point making further progress to fail on the next workspace operation. Incidentally, we are failing for an IP conversion, not an integer. diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index 64aa12691..0856f7a23 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -202,8 +202,7 @@ vmod_ip(VRT_CTX, struct VARGS(ip) *a) p = WS_Alloc(ctx->ws, vsa_suckaddr_len); if (p == NULL) { - VSLb(ctx->vsl, SLT_VCL_Error, - "vmod std.ip(): insufficient workspace"); + VRT_fail(ctx, "std.ip: insufficient workspace"); return (NULL); } @@ -217,7 +216,7 @@ vmod_ip(VRT_CTX, struct VARGS(ip) *a) if (a->valid_fallback) return (a->fallback); - VRT_fail(ctx, "std.integer: conversion failed"); + VRT_fail(ctx, "std.ip: conversion failed"); return (NULL); } From dridi.boukelmoune at gmail.com Tue May 21 07:46:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 21 May 2019 07:46:09 +0000 (UTC) Subject: [master] 937ef0ec8 Extract the resolution logic to vss_resolve Message-ID: <20190521074609.9699F63C4E@lists.varnish-cache.org> commit 937ef0ec84c440b0a0cc27b843811ad51df255bd Author: Dridi Boukelmoune Date: Thu May 9 10:35:17 2019 +0200 Extract the resolution logic to vss_resolve It can now be shared by established callback-based resolvers and the new VSS_ResolveOne. This also changes the semantics of VSS_ResolveOne in the sense that the port is now a default port, overriden by the address if it contains one. Also make it clear that VTCP was already relying on a VSS function that didn't and still doesn't allow a null errp argument, while conversely all VTCP_open call sites provide a valid errp argument. diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c index 9e4816da8..29915a0b3 100644 --- a/bin/varnishtest/vtc_client.c +++ b/bin/varnishtest/vtc_client.c @@ -109,6 +109,8 @@ client_tcp_connect(struct vtclog *vl, const char *addr, double tmo, int fd; char mabuf[32], mpbuf[32]; + AN(addr); + AN(errp); fd = VTCP_open(addr, NULL, tmo, errp); if (fd < 0) return (fd); diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index a3a5ed21e..a93a93c3b 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -167,6 +167,8 @@ haproxy_cli_tcp_connect(struct vtclog *vl, const char *addr, double tmo, int fd; char mabuf[32], mpbuf[32]; + AN(addr); + AN(errp); fd = VTCP_open(addr, NULL, tmo, errp); if (fd < 0) return (fd); diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c index ebbdf4f2b..206b5acb3 100644 --- a/lib/libvarnish/vss.c +++ b/lib/libvarnish/vss.c @@ -98,6 +98,43 @@ vss_parse(char *str, char **addr, char **port) return (NULL); } +static int +vss_resolve(const char *addr, const char *def_port, int family, int socktype, + int flags, struct addrinfo **res, const char **errp) +{ + struct addrinfo hints; + char *p = NULL, *hp, *pp; + int ret; + + AN(addr); + AN(res); + AZ(*res); + AN(errp); + *errp = NULL; + + memset(&hints, 0, sizeof hints); + hints.ai_family = family; + hints.ai_socktype = socktype; + hints.ai_flags = flags; + + p = strdup(addr); + AN(p); + *errp = vss_parse(p, &hp, &pp); + if (*errp != NULL) { + free(p); + return (-1); + } + if (pp != NULL) + def_port = pp; + ret = getaddrinfo(hp, def_port, &hints, res); + free(p); + + if (ret != 0) + *errp = gai_strerror(ret); + + return (ret); +} + /* * Look up an address, using a default port if provided, and call * the callback function with the suckaddrs we find. @@ -107,34 +144,21 @@ vss_parse(char *str, char **addr, char **port) int VSS_resolver_socktype(const char *addr, const char *def_port, - vss_resolved_f *func, void *priv, const char **err, int socktype) + vss_resolved_f *func, void *priv, const char **errp, int socktype) { - struct addrinfo hints, *res0, *res; + struct addrinfo *res0 = NULL, *res; struct suckaddr *vsa; - char *h; - char *adp, *hop; int ret; - *err = NULL; - h = strdup(addr); - AN(h); - *err = vss_parse(h, &hop, &adp); - if (*err != NULL) { - free(h); - return (-1); - } - if (adp != NULL) - def_port = adp; + AN(addr); + AN(func); + AN(errp); - memset(&hints, 0, sizeof hints); - hints.ai_socktype = socktype; - hints.ai_flags = AI_PASSIVE; - ret = getaddrinfo(hop, def_port, &hints, &res0); - free(h); - if (ret != 0) { - *err = gai_strerror(ret); + ret = vss_resolve(addr, def_port, AF_UNSPEC, socktype, AI_PASSIVE, + &res0, errp); + if (ret != 0) return (-1); - } + for (res = res0; res != NULL; res = res->ai_next) { vsa = VSA_Malloc(res->ai_addr, res->ai_addrlen); if (vsa != NULL) { @@ -150,40 +174,25 @@ VSS_resolver_socktype(const char *addr, const char *def_port, int VSS_resolver(const char *addr, const char *def_port, vss_resolved_f *func, - void *priv, const char **err) + void *priv, const char **errp) { return (VSS_resolver_socktype( - addr, def_port, func, priv, err, SOCK_STREAM)); + addr, def_port, func, priv, errp, SOCK_STREAM)); } struct suckaddr * -VSS_ResolveOne(void *dst, const char *addr, const char *port, +VSS_ResolveOne(void *dst, const char *addr, const char *def_port, int family, int socktype, int flags) { - struct addrinfo hints, *res = NULL; + struct addrinfo *res = NULL; struct suckaddr *retval = NULL; - char *p = NULL, *hp, *pp; - int error; - - memset(&hints, 0, sizeof hints); - hints.ai_family = family; - hints.ai_socktype = socktype; - hints.ai_flags = flags; + const char *err; + int ret; AN(addr); - if (port != NULL) { - error = getaddrinfo(addr, port, &hints, &res); - } else { - p = strdup(addr); - AN(p); - if (vss_parse(p, &hp, &pp) != NULL || pp == NULL) { - free(p); - return (NULL); - } - error = getaddrinfo(hp, pp, &hints, &res); - free(p); - } - if (!error && res != NULL && res->ai_next == NULL) { + ret = vss_resolve(addr, def_port, family, socktype, flags, &res, &err); + if (ret == 0 && res != NULL && res->ai_next == NULL) { + AZ(err); if (dst == NULL) retval = VSA_Malloc(res->ai_addr, res->ai_addrlen); else diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c index 521ab5343..bc4d88ee5 100644 --- a/lib/libvarnish/vtcp.c +++ b/lib/libvarnish/vtcp.c @@ -381,20 +381,12 @@ int VTCP_open(const char *addr, const char *def_port, vtim_dur timeout, const char **errp) { - int error; - const char *err; - if (errp != NULL) - *errp = NULL; + AN(errp); assert(timeout >= 0); - error = VSS_resolver(addr, def_port, vtcp_open_callback, - &timeout, &err); - if (err != NULL) { - if (errp != NULL) - *errp = err; - return (-1); - } - return (error); + + return (VSS_resolver(addr, def_port, vtcp_open_callback, + &timeout, errp)); } /*-------------------------------------------------------------------- @@ -513,6 +505,7 @@ VTCP_listen_on(const char *addr, const char *def_port, int depth, struct helper h; int sock; + AN(errp); h.depth = depth; h.errp = errp; From dridi.boukelmoune at gmail.com Tue May 21 07:46:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 21 May 2019 07:46:09 +0000 (UTC) Subject: [master] fc2aed271 Add a VSS_ResolveFirst primitive as well Message-ID: <20190521074609.B53D763C51@lists.varnish-cache.org> commit fc2aed271876789a1f62e32c255135fd7974f677 Author: Dridi Boukelmoune Date: Thu May 9 20:06:42 2019 +0200 Add a VSS_ResolveFirst primitive as well This is how std.ip is documented, so VSS_ResolveOne doesn't work there. It might not be the only migration to VSS_ResolveOne that requires attention. Speaking of attention, VSA_Malloc may require some. diff --git a/include/vss.h b/include/vss.h index ab4cdc33d..bf1fb2b6d 100644 --- a/include/vss.h +++ b/include/vss.h @@ -37,3 +37,6 @@ int VSS_resolver_socktype(const char *addr, const char *def_port, struct suckaddr *VSS_ResolveOne(void *dst, const char *addr, const char *port, int family, int socktype, int flags); +struct suckaddr *VSS_ResolveFirst(void *dst, + const char *addr, const char *port, + int family, int socktype, int flags); diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c index 40d84fa83..296f01374 100644 --- a/lib/libvarnish/vsa.c +++ b/lib/libvarnish/vsa.c @@ -252,6 +252,10 @@ VSA_Malloc(const void *s, unsigned sal) } if (l != 0) { ALLOC_OBJ(sua, SUCKADDR_MAGIC); + /* XXX: shouldn't we AN(sua) instead of mixing up failed + * allocations with unsupported address family or bogus + * sockaddr? + */ if (sua != NULL) memcpy(&sua->sa, s, l); } diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c index 206b5acb3..a37879a97 100644 --- a/lib/libvarnish/vss.c +++ b/lib/libvarnish/vss.c @@ -135,6 +135,17 @@ vss_resolve(const char *addr, const char *def_port, int family, int socktype, return (ret); } +static struct suckaddr * +vss_alloc_suckaddr(void *dst, const struct addrinfo *ai) +{ + + AN(ai); + if (dst == NULL) + return (VSA_Malloc(ai->ai_addr, ai->ai_addrlen)); + + return (VSA_Build(dst, ai->ai_addr, ai->ai_addrlen)); +} + /* * Look up an address, using a default port if provided, and call * the callback function with the suckaddrs we find. @@ -193,12 +204,33 @@ VSS_ResolveOne(void *dst, const char *addr, const char *def_port, ret = vss_resolve(addr, def_port, family, socktype, flags, &res, &err); if (ret == 0 && res != NULL && res->ai_next == NULL) { AZ(err); - if (dst == NULL) - retval = VSA_Malloc(res->ai_addr, res->ai_addrlen); - else - retval = VSA_Build(dst, res->ai_addr, res->ai_addrlen); + retval = vss_alloc_suckaddr(dst, res); } if (res != NULL) freeaddrinfo(res); return (retval); } + +struct suckaddr * +VSS_ResolveFirst(void *dst, const char *addr, const char *def_port, + int family, int socktype, int flags) +{ + struct addrinfo *res0 = NULL, *res; + struct suckaddr *retval = NULL; + const char *err; + int ret; + + AN(addr); + ret = vss_resolve(addr, def_port, family, socktype, flags, &res0, &err); + if (ret == 0) + AZ(err); + + for (res = res0; res != NULL; res = res->ai_next) { + retval = vss_alloc_suckaddr(dst, res); + if (retval != NULL) + break; + } + if (res0 != NULL) + freeaddrinfo(res0); + return (retval); +} diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index 0856f7a23..49e3594eb 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -206,7 +206,7 @@ vmod_ip(VRT_CTX, struct VARGS(ip) *a) return (NULL); } - retval = VSS_ResolveOne(p, a->s, "80", PF_UNSPEC, SOCK_STREAM, + retval = VSS_ResolveFirst(p, a->s, "80", PF_UNSPEC, SOCK_STREAM, a->resolve ? 0 : AI_NUMERICHOST); if (retval != NULL) return (retval); From dridi.boukelmoune at gmail.com Tue May 21 07:46:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 21 May 2019 07:46:09 +0000 (UTC) Subject: [master] 9dba9bc7e Use AF_UNSPEC for the address family Message-ID: <20190521074609.D11E963C55@lists.varnish-cache.org> commit 9dba9bc7ea29e3b324b8e884bceca07266a69d21 Author: Dridi Boukelmoune Date: Thu May 9 20:11:44 2019 +0200 Use AF_UNSPEC for the address family diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index 49e3594eb..d38740ec0 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -206,7 +206,7 @@ vmod_ip(VRT_CTX, struct VARGS(ip) *a) return (NULL); } - retval = VSS_ResolveFirst(p, a->s, "80", PF_UNSPEC, SOCK_STREAM, + retval = VSS_ResolveFirst(p, a->s, "80", AF_UNSPEC, SOCK_STREAM, a->resolve ? 0 : AI_NUMERICHOST); if (retval != NULL) return (retval); From dridi.boukelmoune at gmail.com Tue May 21 07:46:09 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 21 May 2019 07:46:09 +0000 (UTC) Subject: [master] e475ad019 Add an optional default port argument to std.ip() Message-ID: <20190521074609.EF51263C59@lists.varnish-cache.org> commit e475ad0196812992d378ff24eef3ee54ec1a23be Author: Dridi Boukelmoune Date: Wed May 15 11:13:20 2019 +0200 Add an optional default port argument to std.ip() And sync the documentation with the current behavior, part of which used to be implicit. diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index 789c8c125..05ec3aec7 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -222,8 +222,7 @@ Returns ``true`` if the backend *be* is healthy. $Function INT port(IP ip) Returns the port number of the IP address *ip*. Always returns ``0`` -for a ``*.ip`` variable whose value is ``0.0.0.0`` because the listen -address is a Unix domain socket. +for a ``*.ip`` variable when the address is a Unix domain socket. Type Conversion functions ========================= @@ -328,15 +327,26 @@ Examples:: set resp.http.answer = std.integer(real=126.42/3); -$Function IP ip(STRING s, [IP fallback], BOOL resolve = 1) +$Function IP ip(STRING s, [IP fallback], BOOL resolve = 1, [STRING p]) -Converts the string *s* to the first IP number returned by -the system library function `getaddrinfo(3)`. If conversion -fails, *fallback* will be returned or VCL failure will happen. +Converts the string *s* to the first IP number returned by the system +library function `getaddrinfo(3)`. If conversion fails, *fallback* will +be returned or VCL failure will happen. + +The IP address includes a port number that can be found with ``std.port()`` +that defaults to 80. The default port can be set to a different value with +the *p* argument. It will be overriden if *s* contains both an IP address +and a port number or service name. + +When *s* contains both, the syntax is either ``address:port`` or +``address port``. If the address is a numerical IPv6 address it must be +enclosed between brackets, for example ``[::1] 80`` or ``[::1]:http``. +The *fallback* may also contain both an address and a port. If *resolve* is false, `getaddrinfo(3)` is called using ``AI_NUMERICHOST`` -to avoid network lookups. This makes "numerical" IP strings cheaper -to convert. +and ``AI_NUMERICSERV`` to avoid network lookups depending on the system's +`getaddrinfo(3)` or nsswitch configuration. This makes "numerical" IP +strings and services cheaper to convert. Example:: diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index d38740ec0..b0c07298f 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -206,8 +206,11 @@ vmod_ip(VRT_CTX, struct VARGS(ip) *a) return (NULL); } - retval = VSS_ResolveFirst(p, a->s, "80", AF_UNSPEC, SOCK_STREAM, - a->resolve ? 0 : AI_NUMERICHOST); + retval = VSS_ResolveFirst( + p, a->s, a->valid_p ? a->p : "80", + AF_UNSPEC, SOCK_STREAM, + a->resolve ? 0 : AI_NUMERICHOST|AI_NUMERICSERV); + if (retval != NULL) return (retval); From dridi.boukelmoune at gmail.com Tue May 21 07:46:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 21 May 2019 07:46:10 +0000 (UTC) Subject: [master] 0b442e085 Missing debug.sndbuf() documentation Message-ID: <20190521074610.1DC5363C64@lists.varnish-cache.org> commit 0b442e08547182580ad0d4169154a22b47efa82e Author: Dridi Boukelmoune Date: Wed May 15 11:18:48 2019 +0200 Missing debug.sndbuf() documentation diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index aa2a3634e..530d94571 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -254,3 +254,7 @@ $Function VOID vcl_allow_cold(PRIV_VCL) Allow VCL to go cold $Function VOID sndbuf(BYTES sndbuf) + +Set the client socket' send buffer size to *sndbuf*. The previous, desired +and actual values appear in the logs. Not currently implemented for backend +transactions. From dridi.boukelmoune at gmail.com Tue May 21 07:46:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 21 May 2019 07:46:10 +0000 (UTC) Subject: [master] 64fc34269 Increase std.ip() and std.port() coverage Message-ID: <20190521074610.3E45B63C77@lists.varnish-cache.org> commit 64fc34269040ca5f986156c0fe32b2527839cf47 Author: Dridi Boukelmoune Date: Wed May 15 11:59:16 2019 +0200 Increase std.ip() and std.port() coverage diff --git a/bin/varnishtest/tests/m00011.vtc b/bin/varnishtest/tests/m00011.vtc index 95517c7c8..6c761fff4 100644 --- a/bin/varnishtest/tests/m00011.vtc +++ b/bin/varnishtest/tests/m00011.vtc @@ -7,25 +7,76 @@ server s1 { varnish v1 -vcl+backend { import std; + import debug; sub vcl_deliver { std.timestamp("t0"); - set resp.http.foo0 = std.ip("8.8.8.*", client.ip); + + debug.store_ip(std.ip("8.8.8.*", client.ip)); + set resp.http.ip0 = debug.get_ip(); + set resp.http.port0 = std.port(debug.get_ip()); std.timestamp("8.8.8.*, client.ip"); - set resp.http.foo1 = std.ip("9.9.9.*", server.ip); + + debug.store_ip(std.ip("9.9.9.*", server.ip)); + set resp.http.ip1 = debug.get_ip(); + set resp.http.port1 = std.port(debug.get_ip()); std.timestamp("9.9.9.*, server.ip"); - set resp.http.foo2 = std.ip("1.2.3.*", "127.0.0.2"); + + debug.store_ip(std.ip("1.2.3.*", "127.0.0.2")); + set resp.http.ip2 = debug.get_ip(); + set resp.http.port2 = std.port(debug.get_ip()); std.timestamp("1.2.3.*"); - set resp.http.foo3 = std.ip("1.2.3.5", "127.0.0.3"); + + debug.store_ip(std.ip("1.2.3.5", "127.0.0.3")); + set resp.http.ip3 = debug.get_ip(); + set resp.http.port3 = std.port(debug.get_ip()); std.timestamp("1.2.3.5"); - set resp.http.foo4 = std.ip("2001:db8::", "[::1]"); + + debug.store_ip(std.ip("2001:db8::", "[::1]")); + set resp.http.ip4 = debug.get_ip(); + set resp.http.port4 = std.port(debug.get_ip()); std.timestamp("2001:db8::"); - set resp.http.foo5 = std.ip("2001::db8::", "[::1]"); + + debug.store_ip(std.ip("2001::db8::", "[::1]")); + set resp.http.ip5 = debug.get_ip(); + set resp.http.port5 = std.port(debug.get_ip()); std.timestamp("2001::db8::"); - set resp.http.foo6 = std.ip("localhost", "0.0.0.0", resolve = false); + + debug.store_ip(std.ip("localhost", "0.0.0.0", resolve = false)); + set resp.http.ip6 = debug.get_ip(); + set resp.http.port6 = std.port(debug.get_ip()); std.timestamp("localhost, resolve = false"); - set resp.http.foo7 = std.ip("1.2.3.4", "0.0.0.0", resolve = false); + + debug.store_ip(std.ip("1.2.3.4", "0.0.0.0", resolve = false)); + set resp.http.ip7 = debug.get_ip(); + set resp.http.port7 = std.port(debug.get_ip()); std.timestamp("1.2.3.4, resolve = false"); + + debug.store_ip(std.ip("1.2.3.4 8080", "0.0.0.0")); + set resp.http.ip8 = debug.get_ip(); + set resp.http.port8 = std.port(debug.get_ip()); + std.timestamp("1.2.3.4 8080"); + + debug.store_ip(std.ip("1.2.3.4:443", "0.0.0.0")); + set resp.http.ip9 = debug.get_ip(); + set resp.http.port9 = std.port(debug.get_ip()); + std.timestamp("1.2.3.4:443"); + + debug.store_ip(std.ip("1.2.3.4", "0.0.0.0", resolve = false)); + set resp.http.ip10 = debug.get_ip(); + set resp.http.port10 = std.port(debug.get_ip()); + std.timestamp("1.2.3.4, resolve = false"); + + debug.store_ip(std.ip("9.9.9.*", "${s1_sock}")); + set resp.http.ip10 = debug.get_ip(); + set resp.http.port10 = std.port(debug.get_ip()); + std.timestamp("9.9.9.*, ${s1_sock}"); + + debug.store_ip(std.ip("1.2.3.4 80", "0.0.0.0", p = "443")); + set resp.http.ip11 = debug.get_ip(); + set resp.http.port11 = std.port(debug.get_ip()); + std.timestamp("1.2.3.4 80, p = 443"); + } } -start @@ -34,12 +85,29 @@ client c1 { timeout 60 txreq rxresp - expect resp.http.foo0 == "${localhost}" - expect resp.http.foo1 == "${localhost}" - expect resp.http.foo2 == "127.0.0.2" - expect resp.http.foo3 == "1.2.3.5" - expect resp.http.foo4 == "2001:db8::" - expect resp.http.foo5 == "::1" - expect resp.http.foo6 == "0.0.0.0" - expect resp.http.foo7 == "1.2.3.4" + expect resp.http.ip0 == ${localhost} + expect resp.http.port0 != 0 + expect resp.http.port0 != 80 + expect resp.http.ip1 == ${v1_addr} + expect resp.http.port1 == ${v1_port} + expect resp.http.ip2 == 127.0.0.2 + expect resp.http.port2 == 80 + expect resp.http.ip3 == 1.2.3.5 + expect resp.http.port3 == 80 + expect resp.http.ip4 == 2001:db8:: + expect resp.http.port4 == 80 + expect resp.http.ip5 == ::1 + expect resp.http.port5 == 80 + expect resp.http.ip6 == 0.0.0.0 + expect resp.http.port6 == 80 + expect resp.http.ip7 == 1.2.3.4 + expect resp.http.port7 == 80 + expect resp.http.ip8 == 1.2.3.4 + expect resp.http.port8 == 8080 + expect resp.http.ip9 == 1.2.3.4 + expect resp.http.port9 == 443 + expect resp.http.ip10 == ${s1_addr} + expect resp.http.port10 == ${s1_port} + expect resp.http.ip11 == 1.2.3.4 + expect resp.http.port11 == 80 } -run diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index 530d94571..4a40ecc9b 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -258,3 +258,13 @@ $Function VOID sndbuf(BYTES sndbuf) Set the client socket' send buffer size to *sndbuf*. The previous, desired and actual values appear in the logs. Not currently implemented for backend transactions. + +$Function VOID store_ip(PRIV_TASK, IP) + +Store an IP address to be later found by ``debug.get_ip()`` in the same +transaction. + +$Function IP get_ip(PRIV_TASK) + +Get the IP address previously stored by ``debug.store_ip()`` in the same +transaction. diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 0d9f9d356..9e1463101 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -855,6 +855,33 @@ xyzzy_sndbuf(VRT_CTX, VCL_BYTES arg) fd, oldbuf, buflen, newbuf); } +VCL_VOID +xyzzy_store_ip(VRT_CTX, struct vmod_priv *priv, VCL_IP ip) +{ + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(priv); + AZ(priv->free); + assert(VSA_Sane(ip)); + + priv->priv = TRUST_ME(ip); +} + +VCL_IP +xyzzy_get_ip(VRT_CTX, struct vmod_priv *priv) +{ + VCL_IP ip; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(priv); + AZ(priv->free); + + ip = priv->priv; + assert(VSA_Sane(ip)); + + return (ip); +} + /********************************************************************** * For testing import code on bad vmod files (m00003.vtc) */ From dridi.boukelmoune at gmail.com Tue May 21 07:46:10 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 21 May 2019 07:46:10 +0000 (UTC) Subject: [master] 5aea717e2 The fallback default port for std.ip() is always 80 Message-ID: <20190521074610.57F4463C7B@lists.varnish-cache.org> commit 5aea717e2be067fa1b6914e992baa8abb38918cd Author: Dridi Boukelmoune Date: Wed May 15 12:02:05 2019 +0200 The fallback default port for std.ip() is always 80 diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index 05ec3aec7..fafc8ae29 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -341,7 +341,8 @@ and a port number or service name. When *s* contains both, the syntax is either ``address:port`` or ``address port``. If the address is a numerical IPv6 address it must be enclosed between brackets, for example ``[::1] 80`` or ``[::1]:http``. -The *fallback* may also contain both an address and a port. +The *fallback* may also contain both an address and a port, but its default +port is always 80. If *resolve* is false, `getaddrinfo(3)` is called using ``AI_NUMERICHOST`` and ``AI_NUMERICSERV`` to avoid network lookups depending on the system's From phk at FreeBSD.org Tue May 21 08:28:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 21 May 2019 08:28:09 +0000 (UTC) Subject: [master] 3cd5a9934 Add VCS_String() public function, which depending on the argument returns version related strings. Message-ID: <20190521082809.ABEDF65498@lists.varnish-cache.org> commit 3cd5a9934d3d93c0b3f5f24c9f985d8eb59071dd Author: Poul-Henning Kamp Date: Tue May 21 08:26:40 2019 +0000 Add VCS_String() public function, which depending on the argument returns version related strings. Get rid of global, but pretty useles variable VCS_version. Closes: #2936 diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index a0641897b..60a521ece 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -734,7 +734,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond, break; } VSB_printf(pan_vsb, "version = %s, vrt api = %u.%u\n", - VCS_version, VRT_MAJOR_VERSION, VRT_MINOR_VERSION); + VCS_String("V"), VRT_MAJOR_VERSION, VRT_MINOR_VERSION); VSB_printf(pan_vsb, "ident = %s,%s\n", heritage.ident, Waiter_GetName()); VSB_printf(pan_vsb, "now = %f (mono), %f (real)\n", diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c index ed38683de..63a696eaa 100644 --- a/bin/varnishd/mgt/mgt_cli.c +++ b/bin/varnishd/mgt/mgt_cli.c @@ -81,7 +81,7 @@ mcf_banner(struct cli *cli, const char *const *av, void *priv) VCLI_Out(cli, "Varnish Cache CLI 1.0\n"); VCLI_Out(cli, "-----------------------------\n"); VCLI_Out(cli, "%s\n", VSB_data(vident) + 1); - VCLI_Out(cli, "%s\n", VCS_version); + VCLI_Out(cli, "%s\n", VCS_String("V")); VCLI_Out(cli, "\n"); VCLI_Out(cli, "Type 'help' for command list.\n"); VCLI_Out(cli, "Type 'quit' to close CLI session.\n"); diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 7f6622a01..02b9c7ed0 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -843,7 +843,7 @@ main(int argc, char * const *argv) if (pfh2 != NULL) VPF_Write(pfh2); - MGT_Complain(C_DEBUG, "Version: %s", VCS_version); + MGT_Complain(C_DEBUG, "Version: %s", VCS_String("V")); MGT_Complain(C_DEBUG, "Platform: %s", VSB_data(vident) + 1); if (d_flag) diff --git a/include/vcs.h b/include/vcs.h index d3a1585fa..8e605eb97 100644 --- a/include/vcs.h +++ b/include/vcs.h @@ -29,5 +29,5 @@ */ /* from libvarnish/version.c */ -extern const char *VCS_version; void VCS_Message(const char *); +const char *VCS_String(const char *which); diff --git a/lib/libvarnish/version.c b/lib/libvarnish/version.c index b2258ccde..db3398f22 100644 --- a/lib/libvarnish/version.c +++ b/lib/libvarnish/version.c @@ -33,16 +33,49 @@ #include +#include "vdef.h" +#include "vas.h" #include "vcs.h" #include "vcs_version.h" -const char *VCS_version = - PACKAGE_TARNAME "-" PACKAGE_VERSION " revision " VCS_Version; - void VCS_Message(const char *progname) { - fprintf(stderr, "%s (%s)\n", progname, VCS_version); - fprintf(stderr, "Copyright (c) 2006 Verdens Gang AS\n"); - fprintf(stderr, "Copyright (c) 2006-2019 Varnish Software AS\n"); + fprintf(stderr, "%s %s", progname, VCS_String("M")); +} + +const char * +VCS_String(const char *which) +{ + AN(which); + assert(which[1] == '\0'); + + switch(*which) { + case 'T': + return (PACKAGE_TARNAME); + case 'P': + return (PACKAGE_VERSION); + case 'R': + return (VCS_Version); + case 'V': + return ( + PACKAGE_TARNAME + "-" PACKAGE_VERSION + " revision " VCS_Version + ); + case 'M': + return ( + "(" + PACKAGE_TARNAME + "-" PACKAGE_VERSION + " revision " VCS_Version + ")" + "\n" + "Copyright (c) 2006 Verdens Gang AS\n" + "Copyright (c) 2006-2019 Varnish Software AS\n" + ); + default: + WRONG("Wrong argument to VCS_String"); + } + NEEDLESS(return (0)); } diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map index addfb3077..17f982ddd 100644 --- a/lib/libvarnishapi/libvarnishapi.map +++ b/lib/libvarnishapi/libvarnishapi.map @@ -167,3 +167,10 @@ LIBVARNISHAPI_2.2 { local: *; }; + +LIBVARNISHAPI_2.3 { /* 2019-09-15 release */ + global: + VCS_String; + local: + *; +}; diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c index 42596af10..a60bc6017 100644 --- a/lib/libvarnishapi/vsl_query.c +++ b/lib/libvarnishapi/vsl_query.c @@ -348,7 +348,7 @@ vslq_exec(const struct vex *vex, struct VSL_transaction * const ptrans[]) default: return (vslq_test(vex, ptrans)); } - NEEDLESS(return 0); + NEEDLESS(return (0)); } struct vslq_query * diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 4d8734d4c..d431df1cf 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -1023,7 +1023,7 @@ class vcc(object): t = '\t"' for i in json.dumps(jl, indent=2, separators=(",", ": ")): if i == '\n': - fo.write(t + '\\n"\n') + fo.write(t + ' "\n') t = '\t"' else: if i in '"\\': From dridi at varni.sh Tue May 21 08:31:02 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 21 May 2019 10:31:02 +0200 Subject: [master] 3cd5a9934 Add VCS_String() public function, which depending on the argument returns version related strings. In-Reply-To: <20190521082809.ABEDF65498@lists.varnish-cache.org> References: <20190521082809.ABEDF65498@lists.varnish-cache.org> Message-ID: On Tue, May 21, 2019 at 10:28 AM Poul-Henning Kamp wrote: > > > commit 3cd5a9934d3d93c0b3f5f24c9f985d8eb59071dd > Author: Poul-Henning Kamp > Date: Tue May 21 08:26:40 2019 +0000 > > Add VCS_String() public function, which depending on the argument > returns version related strings. > > Get rid of global, but pretty useles variable VCS_version. > > Closes: #2936 > > diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c > index a0641897b..60a521ece 100644 > --- a/bin/varnishd/cache/cache_panic.c > +++ b/bin/varnishd/cache/cache_panic.c > @@ -734,7 +734,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond, > break; > } > VSB_printf(pan_vsb, "version = %s, vrt api = %u.%u\n", > - VCS_version, VRT_MAJOR_VERSION, VRT_MINOR_VERSION); > + VCS_String("V"), VRT_MAJOR_VERSION, VRT_MINOR_VERSION); > VSB_printf(pan_vsb, "ident = %s,%s\n", > heritage.ident, Waiter_GetName()); > VSB_printf(pan_vsb, "now = %f (mono), %f (real)\n", > diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c > index ed38683de..63a696eaa 100644 > --- a/bin/varnishd/mgt/mgt_cli.c > +++ b/bin/varnishd/mgt/mgt_cli.c > @@ -81,7 +81,7 @@ mcf_banner(struct cli *cli, const char *const *av, void *priv) > VCLI_Out(cli, "Varnish Cache CLI 1.0\n"); > VCLI_Out(cli, "-----------------------------\n"); > VCLI_Out(cli, "%s\n", VSB_data(vident) + 1); > - VCLI_Out(cli, "%s\n", VCS_version); > + VCLI_Out(cli, "%s\n", VCS_String("V")); > VCLI_Out(cli, "\n"); > VCLI_Out(cli, "Type 'help' for command list.\n"); > VCLI_Out(cli, "Type 'quit' to close CLI session.\n"); > diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c > index 7f6622a01..02b9c7ed0 100644 > --- a/bin/varnishd/mgt/mgt_main.c > +++ b/bin/varnishd/mgt/mgt_main.c > @@ -843,7 +843,7 @@ main(int argc, char * const *argv) > if (pfh2 != NULL) > VPF_Write(pfh2); > > - MGT_Complain(C_DEBUG, "Version: %s", VCS_version); > + MGT_Complain(C_DEBUG, "Version: %s", VCS_String("V")); > MGT_Complain(C_DEBUG, "Platform: %s", VSB_data(vident) + 1); > > if (d_flag) > diff --git a/include/vcs.h b/include/vcs.h > index d3a1585fa..8e605eb97 100644 > --- a/include/vcs.h > +++ b/include/vcs.h > @@ -29,5 +29,5 @@ > */ > > /* from libvarnish/version.c */ > -extern const char *VCS_version; > void VCS_Message(const char *); > +const char *VCS_String(const char *which); > diff --git a/lib/libvarnish/version.c b/lib/libvarnish/version.c > index b2258ccde..db3398f22 100644 > --- a/lib/libvarnish/version.c > +++ b/lib/libvarnish/version.c > @@ -33,16 +33,49 @@ > > #include > > +#include "vdef.h" > +#include "vas.h" > #include "vcs.h" > #include "vcs_version.h" > > -const char *VCS_version = > - PACKAGE_TARNAME "-" PACKAGE_VERSION " revision " VCS_Version; > - > void > VCS_Message(const char *progname) > { > - fprintf(stderr, "%s (%s)\n", progname, VCS_version); > - fprintf(stderr, "Copyright (c) 2006 Verdens Gang AS\n"); > - fprintf(stderr, "Copyright (c) 2006-2019 Varnish Software AS\n"); > + fprintf(stderr, "%s %s", progname, VCS_String("M")); > +} > + > +const char * > +VCS_String(const char *which) > +{ > + AN(which); > + assert(which[1] == '\0'); Why not simply take a char here? > + switch(*which) { > + case 'T': > + return (PACKAGE_TARNAME); > + case 'P': > + return (PACKAGE_VERSION); > + case 'R': > + return (VCS_Version); > + case 'V': > + return ( > + PACKAGE_TARNAME > + "-" PACKAGE_VERSION > + " revision " VCS_Version > + ); > + case 'M': > + return ( > + "(" > + PACKAGE_TARNAME > + "-" PACKAGE_VERSION > + " revision " VCS_Version > + ")" > + "\n" > + "Copyright (c) 2006 Verdens Gang AS\n" > + "Copyright (c) 2006-2019 Varnish Software AS\n" > + ); > + default: > + WRONG("Wrong argument to VCS_String"); > + } > + NEEDLESS(return (0)); > } > diff --git a/lib/libvarnishapi/libvarnishapi.map b/lib/libvarnishapi/libvarnishapi.map > index addfb3077..17f982ddd 100644 > --- a/lib/libvarnishapi/libvarnishapi.map > +++ b/lib/libvarnishapi/libvarnishapi.map > @@ -167,3 +167,10 @@ LIBVARNISHAPI_2.2 { > local: > *; > }; > + > +LIBVARNISHAPI_2.3 { /* 2019-09-15 release */ > + global: > + VCS_String; > + local: > + *; > +}; > diff --git a/lib/libvarnishapi/vsl_query.c b/lib/libvarnishapi/vsl_query.c > index 42596af10..a60bc6017 100644 > --- a/lib/libvarnishapi/vsl_query.c > +++ b/lib/libvarnishapi/vsl_query.c > @@ -348,7 +348,7 @@ vslq_exec(const struct vex *vex, struct VSL_transaction * const ptrans[]) > default: > return (vslq_test(vex, ptrans)); > } > - NEEDLESS(return 0); > + NEEDLESS(return (0)); > } > > struct vslq_query * > diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py > index 4d8734d4c..d431df1cf 100755 > --- a/lib/libvcc/vmodtool.py > +++ b/lib/libvcc/vmodtool.py > @@ -1023,7 +1023,7 @@ class vcc(object): > t = '\t"' > for i in json.dumps(jl, indent=2, separators=(",", ": ")): > if i == '\n': > - fo.write(t + '\\n"\n') > + fo.write(t + ' "\n') > t = '\t"' > else: > if i in '"\\': > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From phk at phk.freebsd.dk Tue May 21 08:34:51 2019 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Tue, 21 May 2019 08:34:51 +0000 Subject: [master] 3cd5a9934 Add VCS_String() public function, which depending on the argument returns version related strings. In-Reply-To: References: <20190521082809.ABEDF65498@lists.varnish-cache.org> Message-ID: <46098.1558427691@critter.freebsd.dk> -------- In message , Dridi Boukelmoune writes: >> +const char * >> +VCS_String(const char *which) >> +{ >> + AN(which); >> + assert(which[1] == '\0'); > >Why not simply take a char here? Because I want to keep the door open for a printf-like specification down the road. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From dridi at varni.sh Tue May 21 08:36:54 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 21 May 2019 10:36:54 +0200 Subject: [master] 3cd5a9934 Add VCS_String() public function, which depending on the argument returns version related strings. In-Reply-To: <46098.1558427691@critter.freebsd.dk> References: <20190521082809.ABEDF65498@lists.varnish-cache.org> <46098.1558427691@critter.freebsd.dk> Message-ID: On Tue, May 21, 2019 at 10:34 AM Poul-Henning Kamp wrote: > > -------- > In message , Dridi Boukelmoune writes: > > >> +const char * > >> +VCS_String(const char *which) > >> +{ > >> + AN(which); > >> + assert(which[1] == '\0'); > > > >Why not simply take a char here? > > Because I want to keep the door open for a printf-like > specification down the road. Good call! Dridi From dridi at varni.sh Tue May 21 08:43:36 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 21 May 2019 10:43:36 +0200 Subject: [master] 3cd5a9934 Add VCS_String() public function, which depending on the argument returns version related strings. In-Reply-To: References: <20190521082809.ABEDF65498@lists.varnish-cache.org> <46098.1558427691@critter.freebsd.dk> Message-ID: On Tue, May 21, 2019 at 10:36 AM Dridi Boukelmoune wrote: > > On Tue, May 21, 2019 at 10:34 AM Poul-Henning Kamp wrote: > > > > -------- > > In message , Dridi Boukelmoune writes: > > > > >> +const char * > > >> +VCS_String(const char *which) > > >> +{ > > >> + AN(which); > > >> + assert(which[1] == '\0'); > > > > > >Why not simply take a char here? > > > > Because I want to keep the door open for a printf-like > > specification down the road. > > Good call! On second thought, having a dynamic version string format would likely force the return value to drop the const qualifier. You might as well introduce a VCS_Printf function if we ever want to generalize to that extent. Dridi From phk at phk.freebsd.dk Tue May 21 08:46:23 2019 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Tue, 21 May 2019 08:46:23 +0000 Subject: [master] 3cd5a9934 Add VCS_String() public function, which depending on the argument returns version related strings. In-Reply-To: References: <20190521082809.ABEDF65498@lists.varnish-cache.org> <46098.1558427691@critter.freebsd.dk> Message-ID: <46179.1558428383@critter.freebsd.dk> -------- In message , Dridi Boukelmoune writes: >On Tue, May 21, 2019 at 10:36 AM Dridi Boukelmoune wrote: >> >> On Tue, May 21, 2019 at 10:34 AM Poul-Henning Kamp wrote: >> > >> > -------- >> > In message , Dridi Boukelmoune writes: >> > >> > >> +const char * >> > >> +VCS_String(const char *which) >> > >> +{ >> > >> + AN(which); >> > >> + assert(which[1] == '\0'); >> > > >> > >Why not simply take a char here? >> > >> > Because I want to keep the door open for a printf-like >> > specification down the road. >> >> Good call! > >On second thought, having a dynamic version string format would likely >force the return value to drop the const qualifier. I would probably use a single static buffer. It would not be a facility very applicable to multithreaded usage. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From dridi.boukelmoune at gmail.com Tue May 21 09:45:13 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 21 May 2019 09:45:13 +0000 (UTC) Subject: [master] 1a6af67c7 Fail on NULL IPs in VRT Message-ID: <20190521094513.6DD621012EA@lists.varnish-cache.org> commit 1a6af67c731cd295899a5598057d63d3b9f93836 Author: Dridi Boukelmoune Date: Mon Jan 28 13:36:25 2019 +0100 Fail on NULL IPs in VRT This breaks the API since some functions didn't take a VRT_CTX until now. Closes #2860 diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index da1b7f8ed..f1fd423e6 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -534,10 +534,12 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vrt, VRT_BACKEND_MAGIC); - if (vrt->path == NULL) - assert(vrt->ipv4_suckaddr != NULL || - vrt->ipv6_suckaddr != NULL); - else + if (vrt->path == NULL) { + if (vrt->ipv4_suckaddr == NULL && vrt->ipv6_suckaddr == NULL) { + VRT_fail(ctx, "%s: Illegal IP", __func__); + return (NULL); + } + } else assert(vrt->ipv4_suckaddr == NULL && vrt->ipv6_suckaddr == NULL); diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index bf1676653..10da99437 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -571,8 +571,10 @@ VRT_IP_string(VRT_CTX, VCL_IP ip) unsigned len; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - if (ip == NULL) + if (ip == NULL) { + VRT_fail(ctx, "%s: Illegal IP", __func__); return (NULL); + } len = WS_ReserveAll(ctx->ws); if (len == 0) { WS_Release(ctx->ws, 0); @@ -836,10 +838,15 @@ VRT_memmove(void *dst, const void *src, unsigned len) } VCL_BOOL -VRT_ipcmp(VCL_IP sua1, VCL_IP sua2) +VRT_ipcmp(VRT_CTX, VCL_IP sua1, VCL_IP sua2) { - if (sua1 == NULL || sua2 == NULL) - return (1); + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + + if (sua1 == NULL || sua2 == NULL) { + VRT_fail(ctx, "%s: Illegal IP", __func__); + return(1); + } return (VSA_Compare_IP(sua1, sua2)); } @@ -851,6 +858,9 @@ VRT_blob(VRT_CTX, const char *err, const void *src, size_t len, unsigned type) { struct vrt_blob *p; + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx->ws, WS_MAGIC); + p = (void *)WS_Alloc(ctx->ws, sizeof *p); if (p == NULL) { VRT_fail(ctx, "Workspace overflow (%s)", err); @@ -865,7 +875,16 @@ VRT_blob(VRT_CTX, const char *err, const void *src, size_t len, unsigned type) } int -VRT_VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst) +VRT_VSA_GetPtr(VRT_CTX, const struct suckaddr *sua, const unsigned char ** dst) { + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(dst); + + if (sua == NULL) { + VRT_fail(ctx, "%s: Illegal IP", __func__); + *dst = NULL; + return (-1); + } return (VSA_GetPtr(sua, dst)); } diff --git a/include/vrt.h b/include/vrt.h index 87319107f..3b010ea28 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -60,6 +60,8 @@ * VRT_vcl_get moved to vcc_interface.h * VRT_vcl_rel emoved to vcc_interface.h * VRT_vcl_select emoved to vcc_interface.h + * VRT_VSA_GetPtr() changed + * VRT_ipcmp() changed * [cache.h] WS_ReserveAll() added * [cache.h] WS_Reserve(ws, 0) deprecated * 9.0 (2019-03-15) @@ -67,8 +69,8 @@ * HTTP_Copy() removed * HTTP_Dup() added * HTTP_Clone() added - * changed type of VCL_BLOB to newly introduced struct vrt_blob * - * changed VRT_blob() + * VCL_BLOB changed to newly introduced struct vrt_blob * + * VRT_blob() changed * req->req_bodybytes removed * use: AZ(ObjGetU64(req->wrk, req->body_oc, OA_LEN, &u)); * struct vdi_methods .list callback signature changed @@ -447,7 +449,7 @@ VCL_VOID VRT_hashdata(VRT_CTX, const char *str, ...); /* Simple stuff */ int VRT_strcmp(const char *s1, const char *s2); void VRT_memmove(void *dst, const void *src, unsigned len); -VCL_BOOL VRT_ipcmp(VCL_IP, VCL_IP); +VCL_BOOL VRT_ipcmp(VRT_CTX, VCL_IP, VCL_IP); VCL_BLOB VRT_blob(VRT_CTX, const char *, const void *, size_t, unsigned); VCL_VOID VRT_Rollback(VRT_CTX, VCL_HTTP); @@ -513,7 +515,7 @@ VCL_BACKEND VRT_LookupDirector(VRT_CTX, VCL_STRING); void VRT_DelDirector(VCL_BACKEND *); /* Suckaddr related */ -int VRT_VSA_GetPtr(VCL_IP sua, const unsigned char ** dst); +int VRT_VSA_GetPtr(VRT_CTX, VCL_IP sua, const unsigned char ** dst); typedef int vmod_event_f(VRT_CTX, struct vmod_priv *, enum vcl_event_e); diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index d038e1a3e..43a5b43a1 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -364,7 +364,7 @@ vcc_acl_emit(struct vcc *tl, const char *name, const char *rname, int anon) Fh(tl, 0, "\tconst unsigned char *a;\n"); Fh(tl, 0, "\tint fam;\n"); Fh(tl, 0, "\n"); - Fh(tl, 0, "\tfam = VRT_VSA_GetPtr(p, &a);\n"); + Fh(tl, 0, "\tfam = VRT_VSA_GetPtr(ctx, p, &a);\n"); Fh(tl, 0, "\tif (fam < 0) {\n"); Fh(tl, 0, "\t\tVRT_acl_log(ctx, \"NO_FAM %s\");\n", name); Fh(tl, 0, "\t\treturn(0);\n"); diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index c74cc4c4b..dae6fe2f2 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -1067,8 +1067,8 @@ static const struct cmps vcc_cmps[] = { {BOOL, T_EQ, cmp_simple, "((!(\v1)) == (!(\v2)))" }, {BOOL, T_NEQ, cmp_simple, "((!(\v1)) != (!(\v2)))" }, - {IP, T_EQ, cmp_simple, "!VRT_ipcmp(\v1, \v2)" }, - {IP, T_NEQ, cmp_simple, "VRT_ipcmp(\v1, \v2)" }, + {IP, T_EQ, cmp_simple, "!VRT_ipcmp(ctx, \v1, \v2)" }, + {IP, T_NEQ, cmp_simple, "VRT_ipcmp(ctx, \v1, \v2)" }, {IP, '~', cmp_acl, "" }, {IP, T_NOMATCH, cmp_acl, "!" }, From dridi.boukelmoune at gmail.com Tue May 21 09:45:13 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 21 May 2019 09:45:13 +0000 (UTC) Subject: [master] d73b1e46e Typo Message-ID: <20190521094513.8D6961012ED@lists.varnish-cache.org> commit d73b1e46e53a4c4a51f05b3f108ccff0ab124f1a Author: Dridi Boukelmoune Date: Tue May 21 11:41:35 2019 +0200 Typo diff --git a/include/vrt.h b/include/vrt.h index 3b010ea28..7a227ebc9 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -58,8 +58,8 @@ * VRT_count moved to vcc_interface.h * VRT_VCL_Prevent_Cold() and VRT_VCL_Allow_Cold() added. * VRT_vcl_get moved to vcc_interface.h - * VRT_vcl_rel emoved to vcc_interface.h - * VRT_vcl_select emoved to vcc_interface.h + * VRT_vcl_rel moved to vcc_interface.h + * VRT_vcl_select moved to vcc_interface.h * VRT_VSA_GetPtr() changed * VRT_ipcmp() changed * [cache.h] WS_ReserveAll() added From dridi.boukelmoune at gmail.com Tue May 21 16:29:08 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 21 May 2019 16:29:08 +0000 (UTC) Subject: [master] 3474e45d8 Code style OCD Message-ID: <20190521162908.8754710862F@lists.varnish-cache.org> commit 3474e45d85d19bf5f738ac6c1ad2b6cb8e02f744 Author: Dridi Boukelmoune Date: Tue May 21 18:25:14 2019 +0200 Code style OCD Done with git tricks involving: sed ' s:NEEDLESS(return NULL):NEEDLESS(return (NULL)): s:NEEDLESS(return(:NEEDLESS(return (: ' Unfortunately it won't work with a Coccinelle patch, so I couldn't update return.cocci to handle the NEEDLESS case. diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index 8300d1b5d..3336fd89f 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -616,7 +616,7 @@ vca_acct(void *arg) } vca_periodic(t0); } - NEEDLESS(return NULL); + NEEDLESS(return (NULL)); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index c87c73465..571154859 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -491,7 +491,7 @@ vbp_thread(struct worker *wrk, void *priv) } } NEEDLESS(Lck_Unlock(&vbp_mtx)); - NEEDLESS(return NULL); + NEEDLESS(return (NULL)); } diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c index 55a6b6402..ba083cddb 100644 --- a/bin/varnishd/cache/cache_ban_lurker.c +++ b/bin/varnishd/cache/cache_ban_lurker.c @@ -447,5 +447,5 @@ ban_lurker(struct worker *wrk, void *priv) Lck_Unlock(&ban_mtx); } pthread_exit(0); - NEEDLESS(return NULL); + NEEDLESS(return (NULL)); } diff --git a/bin/varnishd/cache/cache_expire.c b/bin/varnishd/cache/cache_expire.c index 85a435741..288c6a4d8 100644 --- a/bin/varnishd/cache/cache_expire.c +++ b/bin/varnishd/cache/cache_expire.c @@ -361,7 +361,7 @@ exp_thread(struct worker *wrk, void *priv) else tnext = exp_expire(ep, t); } - NEEDLESS(return NULL); + NEEDLESS(return (NULL)); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index eb25de6e9..88ee91cd8 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -824,7 +824,7 @@ static enum fetch_step vbf_stp_done(void) { WRONG("Just plain wrong"); - NEEDLESS(return(F_STP_DONE)); + NEEDLESS(return (F_STP_DONE)); } static void v_matchproto_(task_func_t) diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c index 35be8ad00..75ed4bd94 100644 --- a/bin/varnishd/cache/cache_pool.c +++ b/bin/varnishd/cache/cache_pool.c @@ -233,7 +233,7 @@ pool_poolherder(void *priv) Lck_Unlock(&pool_mtx); VSC_C_main->thread_queue_len = u; } - NEEDLESS(return NULL); + NEEDLESS(return (NULL)); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index fe9da5d77..daa667aab 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -163,7 +163,7 @@ VRT_priv_top(VRT_CTX, const void *vmod_id) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if (ctx->req == NULL) { WRONG("PRIV_TOP is only accessible in client VCL context"); - NEEDLESS(return NULL); + NEEDLESS(return (NULL)); } CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); req = ctx->req->topreq; diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 2481f9cdf..a0662c221 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -97,7 +97,7 @@ wrk_bgthread(void *arg) WRONG("BgThread terminated"); - NEEDLESS(return NULL); + NEEDLESS(return (NULL)); } void diff --git a/bin/varnishd/hash/hash_critbit.c b/bin/varnishd/hash/hash_critbit.c index 515de002c..1f1a36c1a 100644 --- a/bin/varnishd/hash/hash_critbit.c +++ b/bin/varnishd/hash/hash_critbit.c @@ -328,7 +328,7 @@ hcb_cleaner(struct worker *wrk, void *priv) Pool_Sumstat(wrk); VTIM_sleep(cache_param->critbit_cooloff); } - NEEDLESS(return NULL); + NEEDLESS(return (NULL)); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/storage/storage_persistent.c b/bin/varnishd/storage/storage_persistent.c index 95f7b4f88..3125c8d35 100644 --- a/bin/varnishd/storage/storage_persistent.c +++ b/bin/varnishd/storage/storage_persistent.c @@ -312,7 +312,7 @@ smp_thread(struct worker *wrk, void *priv) Lck_Unlock(&sc->mtx); pthread_exit(0); - NEEDLESS(return NULL); + NEEDLESS(return (NULL)); } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/waiter/cache_waiter_poll.c b/bin/varnishd/waiter/cache_waiter_poll.c index 74c65aca7..1736ba300 100644 --- a/bin/varnishd/waiter/cache_waiter_poll.c +++ b/bin/varnishd/waiter/cache_waiter_poll.c @@ -206,7 +206,7 @@ vwp_main(void *priv) if (vwp->pollfd[0].revents) vwp_dopipe(vwp); } - NEEDLESS(return NULL); + NEEDLESS(return (NULL)); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index 1a4054a4b..5b05f19cc 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -350,7 +350,7 @@ server_dispatch_thread(void *priv) } pthread_cleanup_pop(0); vtc_logclose(vl); - NEEDLESS(return(NULL)); + NEEDLESS(return (NULL)); } static void diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c index 296f01374..836747eac 100644 --- a/lib/libvarnish/vsa.c +++ b/lib/libvarnish/vsa.c @@ -363,7 +363,7 @@ VSA_Compare_IP(const struct suckaddr *sua1, const struct suckaddr *sua2) default: WRONG("Just plain insane"); } - NEEDLESS(return(-1)); + NEEDLESS(return (-1)); } struct suckaddr * From dridi.boukelmoune at gmail.com Wed May 22 07:37:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 May 2019 07:37:11 +0000 (UTC) Subject: [master] 2a5dc8770 Minor doc fix Message-ID: <20190522073711.76A87118E0F@lists.varnish-cache.org> commit 2a5dc877041bc6bba090b2cd2ca5de1c9f6ddfaa Author: Dridi Boukelmoune Date: Wed May 22 09:36:06 2019 +0200 Minor doc fix diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c index a37879a97..4313b3b05 100644 --- a/lib/libvarnish/vss.c +++ b/lib/libvarnish/vss.c @@ -54,7 +54,7 @@ * "0.0.0.0" - "0.0.0.0:80" - "0.0.0.0 80" * "[::1]" - "[::1]:80" - "[::1] 80" * "[::]" - "[::]:80" - "[::] 80" - * "::1" - "[::1]:80" - "::1 80" + * "::1" - "[::1]:80" - "[::1] 80" * * See also RFC5952 */ From phk at FreeBSD.org Wed May 22 08:50:14 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 22 May 2019 08:50:14 +0000 (UTC) Subject: [master] 7b5d416f4 Add from-to variant named vjsn_parse_end() Message-ID: <20190522085014.C6F6F4591@lists.varnish-cache.org> commit 7b5d416f4aec9f640428ba2e07775dab21b027dc Author: Poul-Henning Kamp Date: Wed May 22 05:54:00 2019 +0000 Add from-to variant named vjsn_parse_end() diff --git a/include/vjsn.h b/include/vjsn.h index 3b2f16b30..ebd4cafc5 100644 --- a/include/vjsn.h +++ b/include/vjsn.h @@ -54,6 +54,7 @@ struct vjsn { const char *err; }; +struct vjsn *vjsn_parse_end(const char *, const char *, const char **); struct vjsn *vjsn_parse(const char *, const char **); void vjsn_delete(struct vjsn **); void vjsn_dump(const struct vjsn *js, FILE *fo); diff --git a/lib/libvarnish/vjsn.c b/lib/libvarnish/vjsn.c index 6a265cbe0..653fb340d 100644 --- a/lib/libvarnish/vjsn.c +++ b/lib/libvarnish/vjsn.c @@ -395,20 +395,28 @@ vjsn_value(struct vjsn *js) } struct vjsn * -vjsn_parse(const char *src, const char **err) +vjsn_parse_end(const char *from, const char *to, const char **err) { struct vjsn *js; char *p, *e; + size_t sz; - AN(src); + AN(from); AN(err); *err = NULL; - p = strdup(src); + if (to == NULL) + to = strchr(from, '\0'); + AN(to); + + sz = to - from; + + p = malloc(sz + 1L); AN(p); - e = strchr(p, '\0'); - AN(e); + memcpy(p, from, sz); + p[sz] = '\0'; + e = p + sz; ALLOC_OBJ(js, VJSN_MAGIC); AN(js); @@ -431,6 +439,13 @@ vjsn_parse(const char *src, const char **err) return (js); } +struct vjsn * +vjsn_parse(const char *src, const char **err) +{ + + return (vjsn_parse_end(src, NULL, err)); +} + struct vjsn_val * vjsn_child(const struct vjsn_val *vv, const char *key) { From phk at FreeBSD.org Wed May 22 08:50:14 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 22 May 2019 08:50:14 +0000 (UTC) Subject: [master] cf14a0fd7 Add convenience function VSB_tofile() to write a VSB to a fd. Message-ID: <20190522085014.DD2064598@lists.varnish-cache.org> commit cf14a0fd774fcbcd4bfd2d50bcbe029f63ff7e0e Author: Poul-Henning Kamp Date: Wed May 22 06:55:20 2019 +0000 Add convenience function VSB_tofile() to write a VSB to a fd. diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c index 7a2d9c040..498045d4e 100644 --- a/bin/varnishd/common/common_vsmw.c +++ b/bin/varnishd/common/common_vsmw.c @@ -169,7 +169,6 @@ static void vsmw_addseg(struct vsmw *vsmw, struct vsmwseg *seg) { int fd; - ssize_t s; VTAILQ_INSERT_TAIL(&vsmw->segs, seg, list); fd = openat(vsmw->vdirfd, vsmw->idx, O_APPEND | O_WRONLY); @@ -177,9 +176,7 @@ vsmw_addseg(struct vsmw *vsmw, struct vsmwseg *seg) VSB_clear(vsmw->vsb); vsmw_fmt_index(vsmw, seg); AZ(VSB_finish(vsmw->vsb)); - s = write(fd, VSB_data(vsmw->vsb), VSB_len(vsmw->vsb)); - // XXX handle ENOSPC? #2764 - assert(s == VSB_len(vsmw->vsb)); + XXXAZ(VSB_tofile(fd, vsmw->vsb)); // XXX handle ENOSPC? #2764 closefd(&fd); } @@ -189,7 +186,6 @@ static void vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx) { char *t = NULL; - ssize_t s; int fd; CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC); @@ -212,9 +208,7 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx) VTAILQ_FOREACH(seg, &vsmw->segs, list) vsmw_fmt_index(vsmw, seg); AZ(VSB_finish(vsmw->vsb)); - s = write(fd, VSB_data(vsmw->vsb), VSB_len(vsmw->vsb)); - // XXX handle ENOSPC? #2764 - assert(s == VSB_len(vsmw->vsb)); + XXXAZ(VSB_tofile(fd, vsmw->vsb)); // XXX handle ENOSPC? #2764 closefd(&fd); AZ(renameat(vsmw->vdirfd, t, vsmw->vdirfd, vsmw->idx)); REPLACE(t, NULL); diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c index 63a696eaa..2798eceb6 100644 --- a/bin/varnishd/mgt/mgt_cli.c +++ b/bin/varnishd/mgt/mgt_cli.c @@ -144,8 +144,7 @@ mcf_askchild(struct cli *cli, const char * const *av, void *priv) } VSB_putc(vsb, '\n'); AZ(VSB_finish(vsb)); - i = write(cli_o, VSB_data(vsb), VSB_len(vsb)); - if (i != VSB_len(vsb)) { + if (VSB_tofile(cli_o, vsb)) { VSB_destroy(&vsb); VCLI_SetResult(cli, CLIS_COMMS); VCLI_Out(cli, "CLI communication error"); @@ -176,7 +175,7 @@ static struct cli_proto cli_askchild[] = { int mgt_cli_askchild(unsigned *status, char **resp, const char *fmt, ...) { - int i, j; + int i; va_list ap; unsigned u; @@ -202,8 +201,7 @@ mgt_cli_askchild(unsigned *status, char **resp, const char *fmt, ...) AZ(VSB_finish(cli_buf)); i = VSB_len(cli_buf); assert(i > 0 && VSB_data(cli_buf)[i - 1] == '\n'); - j = write(cli_o, VSB_data(cli_buf), i); - if (j != i) { + if (VSB_tofile(cli_o, cli_buf)) { if (status != NULL) *status = CLIS_COMMS; if (resp != NULL) diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index 867c48ccb..ee9c19601 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -645,7 +645,7 @@ VPX_Send_Proxy(int fd, int version, const struct sess *sp) WRONG("Wrong proxy version"); AZ(VSB_finish(vsb)); - (void)write(fd, VSB_data(vsb), VSB_len(vsb)); + (void)VSB_tofile(fd, vsb); // XXX: Error handling ? if (!DO_DEBUG(DBG_PROTOCOL)) { VSB_delete(vsb); return; diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index a93a93c3b..ccd0fba80 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -190,7 +190,6 @@ cmd_haproxy_cli_send(CMD_ARGS) { struct vsb *vsb; struct haproxy_cli *hc; - ssize_t wr; (void)cmd; (void)vl; @@ -221,8 +220,7 @@ cmd_haproxy_cli_send(CMD_ARGS) } vtc_dump(hc->vl, 4, "CLI send", VSB_data(vsb), -1); - wr = write(hc->sock, VSB_data(vsb), VSB_len(vsb)); - if (wr != VSB_len(vsb)) + if (VSB_tofile(hc->sock, vsb)) vtc_fatal(hc->vl, "CLI fd %d send error %s", hc->sock, strerror(errno)); diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 6e249ae24..2df6cab45 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -212,14 +212,12 @@ synth_body(const char *len, int rnd) static void http_write(const struct http *hp, int lvl, const char *pfx) { - ssize_t l; AZ(VSB_finish(hp->vsb)); vtc_dump(hp->vl, lvl, pfx, VSB_data(hp->vsb), VSB_len(hp->vsb)); - l = write(hp->fd, VSB_data(hp->vsb), VSB_len(hp->vsb)); - if (l != VSB_len(hp->vsb)) - vtc_log(hp->vl, hp->fatal, "Write failed: (%zd vs %zd) %s", - l, VSB_len(hp->vsb), strerror(errno)); + if (VSB_tofile(hp->fd, hp->vsb)) + vtc_log(hp->vl, hp->fatal, "Write failed: %s", + strerror(errno)); } /********************************************************************** @@ -1467,7 +1465,6 @@ cmd_http_sendhex(CMD_ARGS) { struct vsb *vsb; struct http *hp; - int j; (void)cmd; (void)vl; @@ -1477,8 +1474,9 @@ cmd_http_sendhex(CMD_ARGS) vsb = vtc_hex_to_bin(hp->vl, av[1]); assert(VSB_len(vsb) >= 0); vtc_hexdump(hp->vl, 4, "sendhex", VSB_data(vsb), VSB_len(vsb)); - j = write(hp->fd, VSB_data(vsb), VSB_len(vsb)); - assert(j == VSB_len(vsb)); + if (VSB_tofile(hp->fd, vsb)) + vtc_log(hp->vl, hp->fatal, "Write failed: %s", + strerror(errno)); VSB_destroy(&vsb); } diff --git a/bin/varnishtest/vtc_process.c b/bin/varnishtest/vtc_process.c index 2b1534be1..92a591786 100644 --- a/bin/varnishtest/vtc_process.c +++ b/bin/varnishtest/vtc_process.c @@ -785,7 +785,6 @@ static void process_write_hex(const struct process *p, const char *text) { struct vsb *vsb; - int j; if (!p->hasthread) vtc_fatal(p->vl, "Cannot write to a non-running process"); @@ -793,8 +792,7 @@ process_write_hex(const struct process *p, const char *text) vsb = vtc_hex_to_bin(p->vl, text); assert(VSB_len(vsb) >= 0); vtc_hexdump(p->vl, 4, "sendhex", VSB_data(vsb), VSB_len(vsb)); - j = write(p->fd_term, VSB_data(vsb), VSB_len(vsb)); - assert(j == VSB_len(vsb)); + AZ(VSB_tofile(p->fd_term, vsb)); VSB_destroy(&vsb); } diff --git a/bin/varnishtest/vtc_proxy.c b/bin/varnishtest/vtc_proxy.c index 03e186b57..4477ab056 100644 --- a/bin/varnishtest/vtc_proxy.c +++ b/bin/varnishtest/vtc_proxy.c @@ -84,7 +84,7 @@ vtc_send_proxy(int fd, int version, const struct suckaddr *sac, char pc[VTCP_PORTBUFSIZE]; char hs[VTCP_ADDRBUFSIZE]; char ps[VTCP_PORTBUFSIZE]; - int i, len; + int i; int proto; AN(sac); @@ -126,8 +126,7 @@ vtc_send_proxy(int fd, int version, const struct suckaddr *sac, WRONG("Wrong proxy version"); AZ(VSB_finish(vsb)); - len = VSB_len(vsb); - i = write(fd, VSB_data(vsb), len); + i = VSB_tofile(fd, vsb); VSB_delete(vsb); - return (i != len); + return (i); } diff --git a/include/vsb.h b/include/vsb.h index ea1ab4de8..47d66ac2e 100644 --- a/include/vsb.h +++ b/include/vsb.h @@ -86,6 +86,7 @@ void VSB_quote_pfx(struct vsb *, const char*, const void *, int len, int how); void VSB_quote(struct vsb *, const void *, int len, int how); void VSB_indent(struct vsb *, int); +int VSB_tofile(int fd, const struct vsb *); #ifdef __cplusplus }; #endif diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c index 5101a9007..f95debe0d 100644 --- a/lib/libvarnish/vsb.c +++ b/lib/libvarnish/vsb.c @@ -34,6 +34,7 @@ __FBSDID("$FreeBSD: head/sys/kern/subr_vsb.c 222004 2011-05-17 06:36:32Z phk $") #include #include #include +#include #include "vdef.h" #include "vas.h" // XXX Flexelint "not used" - but req'ed for assert() @@ -618,7 +619,7 @@ VSB_quote(struct vsb *s, const void *v, int len, int how) */ void -VSB_indent(struct vsb * s, int i) +VSB_indent(struct vsb *s, int i) { assert_VSB_integrity(s); @@ -627,3 +628,14 @@ VSB_indent(struct vsb * s, int i) else s->s_indent += i; } + +int +VSB_tofile(int fd, const struct vsb *s) +{ + int sz; + + assert_VSB_integrity(s); + assert_VSB_state(s, VSB_FINISHED); + sz = write(fd, VSB_data(s), VSB_len(s)); + return (sz == VSB_len(s) ? 0 : -1); +} From phk at FreeBSD.org Wed May 22 08:50:14 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 22 May 2019 08:50:14 +0000 (UTC) Subject: [master] 91fe120b3 Move the writing of the compiled C source to VCC Message-ID: <20190522085015.03960459C@lists.varnish-cache.org> commit 91fe120b364498ee32279ed07a7b48effe99576a Author: Poul-Henning Kamp Date: Wed May 22 07:03:25 2019 +0000 Move the writing of the compiled C source to VCC diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index cf7852e7c..8a4074efa 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -85,12 +85,11 @@ mgt_DumpBuiltin(void) static void v_matchproto_(vsub_func_f) run_vcc(void *priv) { - struct vsb *csrc; struct vsb *sb = NULL; struct vcc_priv *vp; - int fd, i, l; struct vcc *vcc; struct stevedore *stv; + int i; VJ_subproc(JAIL_SUBPROC_VCC); CAST_OBJ_NOTNULL(vp, priv, VCC_PRIV_MAGIC); @@ -108,28 +107,11 @@ run_vcc(void *priv) STV_Foreach(stv) VCC_Predef(vcc, "VCL_STEVEDORE", stv->ident); mgt_vcl_export_labels(vcc); - csrc = VCC_Compile(vcc, &sb, vp->vclsrc, vp->vclsrcfile); - AZ(VSB_finish(sb)); + i = VCC_Compile(vcc, &sb, vp->vclsrc, vp->vclsrcfile, VGC_SRC); if (VSB_len(sb)) printf("%s", VSB_data(sb)); VSB_destroy(&sb); - if (csrc == NULL) - exit(2); - - fd = open(VGC_SRC, O_WRONLY|O_TRUNC|O_CREAT, 0600); - if (fd < 0) { - fprintf(stderr, "VCC cannot open %s", vp->csrcfile); - exit(2); - } - l = VSB_len(csrc); - i = write(fd, VSB_data(csrc), l); - if (i != l) { - fprintf(stderr, "VCC cannot write %s", vp->csrcfile); - exit(2); - } - closefd(&fd); - VSB_destroy(&csrc); - exit(0); + exit(i == 0 ? 0 : 2); } /*-------------------------------------------------------------------- diff --git a/include/libvcc.h b/include/libvcc.h index 31b2a7ff0..45f012a1a 100644 --- a/include/libvcc.h +++ b/include/libvcc.h @@ -42,5 +42,5 @@ void VCC_VMOD_path(struct vcc *, const char *); void VCC_Predef(struct vcc *, const char *type, const char *name); void VCC_VCL_Range(unsigned *, unsigned *); -struct vsb *VCC_Compile(struct vcc *, struct vsb **, - const char *vclsrc, const char *vclsrcfile); +int VCC_Compile(struct vcc *, struct vsb **, + const char *, const char *, const char *); diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index c0ffc19ba..8b8c2e9d3 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -52,9 +52,11 @@ #include "config.h" +#include #include #include #include +#include #include "vcc_compile.h" @@ -729,25 +731,48 @@ VCC_VCL_Range(unsigned *lo, unsigned *hi) * formatted into the vsb. */ -struct vsb * +int VCC_Compile(struct vcc *tl, struct vsb **sb, - const char *vclsrc, const char *vclsrcfile) + const char *vclsrc, const char *vclsrcfile, + const char *ofile) { struct source *sp; struct vsb *r = NULL; + int fo, retval = 0; CHECK_OBJ_NOTNULL(tl, VCC_MAGIC); AN(sb); AN(vclsrcfile); - + AN(ofile); if (vclsrc != NULL) sp = vcc_new_source(vclsrc, NULL, vclsrcfile); else sp = vcc_file_source(tl, vclsrcfile); + if (sp != NULL) r = vcc_CompileSource(tl, sp); + + if (r != NULL) { + fo = open(ofile, O_WRONLY|O_TRUNC|O_CREAT, 0600); + if (fo < 0) { + VSB_printf(tl->sb, + "Could not open C-source file %s: %s\n", + ofile, strerror(errno)); + } else { + if (VSB_tofile(fo, r)) { + VSB_printf(tl->sb, + "Could not write C-source to %s: %s\n", + ofile, strerror(errno)); + } + closefd(&fo); + } + VSB_destroy(&r); + } else { + retval = -1; + } + AZ(VSB_finish(tl->sb)); *sb = tl->sb; - return (r); + return (retval); } /*-------------------------------------------------------------------- From phk at FreeBSD.org Wed May 22 08:50:15 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 22 May 2019 08:50:15 +0000 (UTC) Subject: [master] 873c615be Add a function to dump just a subtree Message-ID: <20190522085015.1BE7E45A0@lists.varnish-cache.org> commit 873c615be1b07d15e27cba60f9200b90598af644 Author: Poul-Henning Kamp Date: Wed May 22 08:39:17 2019 +0000 Add a function to dump just a subtree diff --git a/include/vjsn.h b/include/vjsn.h index ebd4cafc5..5051a6cfa 100644 --- a/include/vjsn.h +++ b/include/vjsn.h @@ -58,4 +58,5 @@ struct vjsn *vjsn_parse_end(const char *, const char *, const char **); struct vjsn *vjsn_parse(const char *, const char **); void vjsn_delete(struct vjsn **); void vjsn_dump(const struct vjsn *js, FILE *fo); +void vjsn_dump_val(const struct vjsn_val *jsv, FILE *fo); struct vjsn_val *vjsn_child(const struct vjsn_val *, const char *); diff --git a/lib/libvarnish/vjsn.c b/lib/libvarnish/vjsn.c index 653fb340d..d9c1a1e53 100644 --- a/lib/libvarnish/vjsn.c +++ b/lib/libvarnish/vjsn.c @@ -465,6 +465,7 @@ vjsn_dump_i(const struct vjsn_val *jsv, FILE *fo, int indent) { struct vjsn_val *jsve; + CHECK_OBJ_NOTNULL(jsv, VJSN_VAL_MAGIC); printf("%*s", indent, ""); if (jsv->name != NULL) printf("[\"%s\"]: ", jsv->name); @@ -482,6 +483,13 @@ vjsn_dump_i(const struct vjsn_val *jsv, FILE *fo, int indent) vjsn_dump_i(jsve, fo, indent + 2); } +void +vjsn_dump_val(const struct vjsn_val *jsv, FILE *fo) +{ + CHECK_OBJ_NOTNULL(jsv, VJSN_VAL_MAGIC); + vjsn_dump_i(jsv, fo, 0); +} + void vjsn_dump(const struct vjsn *js, FILE *fo) { From phk at FreeBSD.org Wed May 22 08:50:15 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 22 May 2019 08:50:15 +0000 (UTC) Subject: [master] d032e522e Make VCC produce a "proper" symbol table for the compiled VCL. Message-ID: <20190522085015.39FD345A4@lists.varnish-cache.org> commit d032e522e9e9d294569b3aa15ab6fc2e657fe7af Author: Poul-Henning Kamp Date: Wed May 22 08:48:00 2019 +0000 Make VCC produce a "proper" symbol table for the compiled VCL. diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 88844c295..c50e44dd3 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -422,7 +422,7 @@ VCL_Close(struct vcl **vclp) } /*-------------------------------------------------------------------- - * NB: This function is called from the test-load subprocess. + * NB: This function is called in/from the test-load subprocess. */ int diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h index 7575de2d8..78f79ce4d 100644 --- a/bin/varnishd/mgt/mgt.h +++ b/bin/varnishd/mgt/mgt.h @@ -48,6 +48,7 @@ #include "common/common_param.h" +struct vjsn; struct vsc_seg; struct vsmw_cluster; #include "VSC_mgt.h" @@ -216,8 +217,7 @@ void mgt_vcl_startup(struct cli *, const char *vclsrc, const char *origin, int mgt_push_vcls(struct cli *, unsigned *status, char **p); void mgt_vcl_export_labels(struct vcc *); int mgt_has_vcl(void); -void mgt_vcl_depends(struct vclprog *vp1, const char *name); -void mgt_vcl_vmod(struct vclprog *, const char *src, const char *dst); +void mgt_vcl_symtab(struct vclprog *, struct vjsn *); extern char *mgt_cc_cmd; extern const char *mgt_vcl_path; extern const char *mgt_vmod_path; diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index 8a4074efa..6187c8d97 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -45,8 +45,8 @@ #include "libvcc.h" #include "vcli_serve.h" #include "vfil.h" +#include "vjsn.h" #include "vsub.h" -#include "vav.h" #include "vtim.h" struct vcc_priv { @@ -57,6 +57,7 @@ struct vcc_priv { const char *vclsrcfile; char *csrcfile; char *libfile; + char *symfile; }; char *mgt_cc_cmd; @@ -69,6 +70,7 @@ unsigned mgt_vcc_unsafe_path; #define VGC_SRC "vgc.c" #define VGC_LIB "vgc.so" +#define VGC_SYM "vgc.sym" /*--------------------------------------------------------------------*/ @@ -107,7 +109,8 @@ run_vcc(void *priv) STV_Foreach(stv) VCC_Predef(vcc, "VCL_STEVEDORE", stv->ident); mgt_vcl_export_labels(vcc); - i = VCC_Compile(vcc, &sb, vp->vclsrc, vp->vclsrcfile, VGC_SRC); + i = VCC_Compile(vcc, &sb, vp->vclsrc, vp->vclsrcfile, + VGC_SRC, VGC_SYM); if (VSB_len(sb)) printf("%s", VSB_data(sb)); VSB_destroy(&sb); @@ -212,7 +215,9 @@ static unsigned mgt_vcc_compile(struct vcc_priv *vp, struct vsb *sb, int C_flag) { char *csrc; + const char *err; unsigned subs; + struct vjsn *vj; if (mgt_vcc_touchfile(vp->csrcfile, sb)) return (2); @@ -228,6 +233,18 @@ mgt_vcc_compile(struct vcc_priv *vp, struct vsb *sb, int C_flag) AN(csrc); VSB_cat(sb, csrc); free(csrc); + + VSB_printf(sb, "/* EXTERNAL SYMBOL TABLE\n"); + csrc = VFIL_readfile(NULL, vp->symfile, NULL); + AN(csrc); + VSB_cat(sb, csrc); + vj = vjsn_parse(csrc, &err); + if (err != NULL) + VSB_printf(sb, "# Parse error: %s\n", err); + if (vj != NULL) + vjsn_delete(&vj); + VSB_printf(sb, "*/\n"); + free(csrc); } subs = VSUB_run(sb, run_cc, vp, "C-compiler", 10); @@ -246,11 +263,10 @@ mgt_VccCompile(struct cli *cli, struct vclprog *vcl, const char *vclname, { struct vcc_priv vp; struct vsb *sb; + struct vjsn *vj; unsigned status; - char buf[1024]; - FILE *fcs; - char **av; - int ac; + const char *err; + char *p; AN(cli); @@ -319,6 +335,12 @@ mgt_VccCompile(struct cli *cli, struct vclprog *vcl, const char *vclname, AN(vp.csrcfile); VSB_clear(sb); + VSB_printf(sb, "%s/%s", vp.dir, VGC_SYM); + AZ(VSB_finish(sb)); + vp.symfile = strdup(VSB_data(sb)); + AN(vp.symfile); + VSB_clear(sb); + status = mgt_vcc_compile(&vp, sb, C_flag); AZ(VSB_finish(sb)); @@ -330,6 +352,7 @@ mgt_VccCompile(struct cli *cli, struct vclprog *vcl, const char *vclname, if (!MGT_DO_DEBUG(DBG_VCL_KEEP)) { (void)unlink(vp.csrcfile); (void)unlink(vp.libfile); + (void)unlink(vp.symfile); (void)rmdir(vp.dir); } free(vp.csrcfile); @@ -342,26 +365,17 @@ mgt_VccCompile(struct cli *cli, struct vclprog *vcl, const char *vclname, return (NULL); } - fcs = fopen(vp.csrcfile, "r"); - AN(fcs); - while (1) { - AN(fgets(buf, sizeof buf, fcs)); - if (memcmp(buf, VCC_INFO_PREFIX, strlen(VCC_INFO_PREFIX))) - break; - av = VAV_Parse(buf, &ac, 0); - AN(av); - AZ(av[0]); - AZ(strcmp(av[1], "/*")); - AZ(strcmp(av[ac-1], "*/")); - if (!strcmp(av[3], "VCL")) - mgt_vcl_depends(vcl, av[4]); - else if (!strcmp(av[3], "VMOD")) - mgt_vcl_vmod(vcl, av[4], av[5]); - else - WRONG("Wrong VCCINFO"); - VAV_Free(av); - } - AZ(fclose(fcs)); + p = VFIL_readfile(NULL, vp.symfile, NULL); + AN(p); + vj = vjsn_parse(p, &err); + if (err != NULL) + fprintf(stderr, "FATAL: Symtab parse error: %s\n%s\n", + err, p); + AZ(err); + AN(vj); + free(p); + mgt_vcl_symtab(vcl, vj); + (void)unlink(vp.symfile); if (!MGT_DO_DEBUG(DBG_VCL_KEEP)) (void)unlink(vp.csrcfile); diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 2710e82fd..1a388248c 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -45,6 +45,7 @@ #include "vcli_serve.h" #include "vct.h" #include "vev.h" +#include "vjsn.h" #include "vtim.h" #define VCL_STATE(sym, str) \ @@ -84,6 +85,7 @@ struct vclprog { unsigned warm; const char * state; double go_cold; + struct vjsn *symtab; VTAILQ_HEAD(, vcldep) dfrom; VTAILQ_HEAD(, vcldep) dto; int nto; @@ -309,17 +311,32 @@ mgt_vcl_del(struct vclprog *vp) } } free(vp->name); + if (vp->symtab) + vjsn_delete(&vp->symtab); FREE_OBJ(vp); } -void -mgt_vcl_depends(struct vclprog *vp1, const char *name) +static const char * +mgt_vcl_symtab_val(const struct vjsn_val *vv, const char *val) +{ + const struct vjsn_val *jv; + + jv = vjsn_child(vv, val); + AN(jv); + assert(jv->type == VJSN_STRING); + AN(jv->value); + return (jv->value); +} + +static void +mgt_vcl_import_vcl(struct vclprog *vp1, const struct vjsn_val *vv) { struct vclprog *vp2; CHECK_OBJ_NOTNULL(vp1, VCLPROG_MAGIC); + AN(vv); - vp2 = mcf_vcl_byname(name); + vp2 = mcf_vcl_byname(mgt_vcl_symtab_val(vv, "name")); CHECK_OBJ_NOTNULL(vp2, VCLPROG_MAGIC); mgt_vcl_dep_add(vp1, vp2); } @@ -336,8 +353,8 @@ mgt_vcl_cache_vmod(const char *nm, const char *fm, const char *to) if (fo < 0 && errno == EEXIST) return (0); if (fo < 0) { - fprintf(stderr, "Creating copy of vmod %s: %s\n", - nm, vstrerror(errno)); + fprintf(stderr, "While creating copy of vmod %s:\n\t%s: %s\n", + nm, to, vstrerror(errno)); return (1); } fi = open(fm, O_RDONLY); @@ -366,27 +383,32 @@ mgt_vcl_cache_vmod(const char *nm, const char *fm, const char *to) return (ret); } -void -mgt_vcl_vmod(struct vclprog *vp, const char *src, const char *dst) +static void +mgt_vcl_import_vmod(struct vclprog *vp, const struct vjsn_val *vv) { struct vmodfile *vf; struct vmoddep *vd; + const char *v_name; + const char *v_file; + const char *v_dst; CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC); - AN(src); - AN(dst); - assert(!strncmp(dst, "./vmod_cache/", 13)); + AN(vv); + + v_name = mgt_vcl_symtab_val(vv, "name"); + v_file = mgt_vcl_symtab_val(vv, "file"); + v_dst = mgt_vcl_symtab_val(vv, "dst"); VTAILQ_FOREACH(vf, &vmodhead, list) - if (!strcmp(vf->fname, dst)) + if (!strcmp(vf->fname, v_dst)) break; if (vf == NULL) { ALLOC_OBJ(vf, VMODFILE_MAGIC); AN(vf); - REPLACE(vf->fname, dst); + REPLACE(vf->fname, v_dst); AN(vf->fname); VTAILQ_INIT(&vf->vcls); - AZ(mgt_vcl_cache_vmod(vp->name, src, dst)); + AZ(mgt_vcl_cache_vmod(v_name, v_file, v_dst)); VTAILQ_INSERT_TAIL(&vmodhead, vf, list); } ALLOC_OBJ(vd, VMODDEP_MAGIC); @@ -396,6 +418,33 @@ mgt_vcl_vmod(struct vclprog *vp, const char *src, const char *dst) VTAILQ_INSERT_TAIL(&vf->vcls, vd, lto); } +void +mgt_vcl_symtab(struct vclprog *vp, struct vjsn *vj) +{ + struct vjsn_val *v1, *v2; + const char *typ; + + CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC); + vp->symtab = vj; + assert(vj->value->type == VJSN_ARRAY); + VTAILQ_FOREACH(v1, &vj->value->children, list) { + assert(v1->type == VJSN_OBJECT); + v2 = vjsn_child(v1, "dir"); + if (v2 == NULL) + continue; + assert(v2->type == VJSN_STRING); + if (strcmp(v2->value, "import")) + continue; + typ = mgt_vcl_symtab_val(v1, "type"); + if (!strcmp(typ, "$VMOD")) + mgt_vcl_import_vmod(vp, v1); + else if (!strcmp(typ, "$VCL")) + mgt_vcl_import_vcl(vp, v1); + else + WRONG("Bad symtab import entry"); + } +} + int mgt_has_vcl(void) { diff --git a/include/libvcc.h b/include/libvcc.h index 45f012a1a..10ed6f04d 100644 --- a/include/libvcc.h +++ b/include/libvcc.h @@ -30,8 +30,6 @@ struct vcc; -#define VCC_INFO_PREFIX "/* VCC_INFO" - struct vcc *VCC_New(void); void VCC_Allow_InlineC(struct vcc *, unsigned); void VCC_Builtin_VCL(struct vcc *, const char *); @@ -43,4 +41,4 @@ void VCC_Predef(struct vcc *, const char *type, const char *name); void VCC_VCL_Range(unsigned *, unsigned *); int VCC_Compile(struct vcc *, struct vsb **, - const char *, const char *, const char *); + const char *, const char *, const char *, const char *); diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c index 8126f9924..4e547a2ab 100644 --- a/lib/libvcc/vcc_action.c +++ b/lib/libvcc/vcc_action.c @@ -273,8 +273,11 @@ vcc_act_return_vcl(struct vcc *tl) ERRCHK(tl); AN(sym); if (sym->eval_priv == NULL) { - VSB_printf(tl->fi, "%s VCL %s */\n", VCC_INFO_PREFIX, - sym->name); + VSB_printf(tl->symtab, ",\n {\n"); + VSB_printf(tl->symtab, "\t\"dir\": \"import\",\n"); + VSB_printf(tl->symtab, "\t\"type\": \"$VCL\",\n"); + VSB_printf(tl->symtab, "\t\"name\": \"%s\"\n", sym->name); + VSB_printf(tl->symtab, " }"); bprintf(buf, "vgc_vcl_%u", tl->unique++); sym->eval_priv = strdup(buf); diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 8b8c2e9d3..cfbc62c2e 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -94,6 +94,30 @@ TlDup(struct vcc *tl, const char *s) return (p); } +static int +TLWriteVSB(struct vcc *tl, const char *fn, const struct vsb *vsb, + const char *what) +{ + int fo; + int i; + + fo = open(fn, O_WRONLY|O_TRUNC|O_CREAT, 0600); + if (fo < 0) { + VSB_printf(tl->sb, + "Could not open %s file %s: %s\n", + what, fn, strerror(errno)); + return (-1); + } + i = VSB_tofile(fo, vsb); + if (i) { + VSB_printf(tl->sb, + "Could not write %s to %s: %s\n", + what, fn, strerror(errno)); + } + closefd(&fo); + return (i); +} + /*--------------------------------------------------------------------*/ struct proc * @@ -590,7 +614,7 @@ vcc_resolve_includes(struct vcc *tl) */ static struct vsb * -vcc_CompileSource(struct vcc *tl, struct source *sp) +vcc_CompileSource(struct vcc *tl, struct source *sp, const char *jfile) { struct proc *p; struct vsb *vsb; @@ -691,14 +715,16 @@ vcc_CompileSource(struct vcc *tl, struct source *sp) VCC_XrefTable(tl); + VSB_printf(tl->symtab, "\n]\n"); + AZ(VSB_finish(tl->symtab)); + if (TLWriteVSB(tl, jfile, tl->symtab, "Symbol table")) + return (NULL); + /* Combine it all */ vsb = VSB_new_auto(); AN(vsb); - AZ(VSB_finish(tl->fi)); - VSB_cat(vsb, VSB_data(tl->fi)); - vcl_output_lang_h(vsb); EmitCoordinates(tl, vsb); @@ -734,38 +760,27 @@ VCC_VCL_Range(unsigned *lo, unsigned *hi) int VCC_Compile(struct vcc *tl, struct vsb **sb, const char *vclsrc, const char *vclsrcfile, - const char *ofile) + const char *ofile, const char *jfile) { struct source *sp; struct vsb *r = NULL; - int fo, retval = 0; + int retval = 0; CHECK_OBJ_NOTNULL(tl, VCC_MAGIC); AN(sb); AN(vclsrcfile); AN(ofile); + AN(jfile); if (vclsrc != NULL) sp = vcc_new_source(vclsrc, NULL, vclsrcfile); else sp = vcc_file_source(tl, vclsrcfile); if (sp != NULL) - r = vcc_CompileSource(tl, sp); + r = vcc_CompileSource(tl, sp, jfile); if (r != NULL) { - fo = open(ofile, O_WRONLY|O_TRUNC|O_CREAT, 0600); - if (fo < 0) { - VSB_printf(tl->sb, - "Could not open C-source file %s: %s\n", - ofile, strerror(errno)); - } else { - if (VSB_tofile(fo, r)) { - VSB_printf(tl->sb, - "Could not write C-source to %s: %s\n", - ofile, strerror(errno)); - } - closefd(&fo); - } + retval = TLWriteVSB(tl, ofile, r, "C-source"); VSB_destroy(&r); } else { retval = -1; @@ -798,8 +813,9 @@ VCC_New(void) tl->nsources = 0; - tl->fi = VSB_new_auto(); - assert(tl->fi != NULL); + tl->symtab = VSB_new_auto(); + assert(tl->symtab != NULL); + VSB_printf(tl->symtab, "[\n {\"version\": 0}"); tl->fc = VSB_new_auto(); assert(tl->fc != NULL); diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 20213c0b9..aa4496316 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -237,7 +237,7 @@ struct vcc { int hindent; unsigned cnt; - struct vsb *fi; /* VCC info to MGT */ + struct vsb *symtab; /* VCC info to MGT */ struct vsb *fc; /* C-code */ struct vsb *fh; /* H-code (before C-code) */ struct vsb *fb; /* Body of current sub diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index 38863ceb5..a9c2ffe33 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -357,8 +357,15 @@ vcc_ParseImport(struct vcc *tl) VSB_printf(ifp->ini, "\t ))\n"); VSB_printf(ifp->ini, "\t\treturn(1);"); - VSB_printf(tl->fi, "%s VMOD %s ./vmod_cache/_vmod_%.*s.%s */\n", - VCC_INFO_PREFIX, fnpx, PF(mod), vmd->file_id); + VSB_printf(tl->symtab, ",\n {\n"); + VSB_printf(tl->symtab, "\t\"dir\": \"import\",\n"); + VSB_printf(tl->symtab, "\t\"type\": \"$VMOD\",\n"); + VSB_printf(tl->symtab, "\t\"name\": \"%.*s\",\n", PF(mod)); + VSB_printf(tl->symtab, "\t\"file\": \"%s\",\n", fnpx); + VSB_printf(tl->symtab, + "\t\"dst\": \"./vmod_cache/_vmod_%.*s.%s\"\n", + PF(mod), vmd->file_id); + VSB_printf(tl->symtab, " }"); /* XXX: zero the function pointer structure ?*/ VSB_printf(ifp->fin, "\t\tVRT_priv_fini(&vmod_priv_%.*s);", PF(mod)); From dridi.boukelmoune at gmail.com Wed May 22 09:03:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 May 2019 09:03:11 +0000 (UTC) Subject: [6.0] 5852ed5cf Introduce a VSS_ResolveOne() function and use it the places where we want a single unique suckaddr. Message-ID: <20190522090311.178F95165@lists.varnish-cache.org> commit 5852ed5cf63d9e91d5242d5605a546fb19d00a3a Author: Poul-Henning Kamp Date: Mon May 6 19:55:58 2019 +0000 Introduce a VSS_ResolveOne() function and use it the places where we want a single unique suckaddr. Conflicts: bin/varnishtest/vtc_client.c lib/libvmod_std/vmod_std_conversions.c diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index fc4c451b3..1def50ebe 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -40,6 +40,7 @@ #include "vend.h" #include "vsa.h" +#include "vss.h" #include "vtcp.h" struct vpx_tlv { @@ -61,7 +62,6 @@ vpx_proto1(const struct worker *wrk, const struct req *req) const char *fld[5]; int i; char *p, *q; - struct addrinfo hints, *res; struct suckaddr *sa; int pfam = -1; @@ -100,58 +100,34 @@ vpx_proto1(const struct worker *wrk, const struct req *req) } if (!strcmp(fld[0], "TCP4")) - pfam = AF_INET; + pfam = PF_INET; else if (!strcmp(fld[0], "TCP6")) - pfam = AF_INET6; + pfam = PF_INET6; else { VSL(SLT_ProxyGarbage, req->sp->vxid, "PROXY1: Wrong TCP[46] field"); return (-1); } - memset(&hints, 0, sizeof hints); - hints.ai_socktype = SOCK_STREAM; - hints.ai_flags = AI_NUMERICHOST | AI_NUMERICSERV; - - i = getaddrinfo(fld[1], fld[3], &hints, &res); - if (i != 0) { - VSL(SLT_ProxyGarbage, req->sp->vxid, - "PROXY1: Cannot resolve source address (%s)", - gai_strerror(i)); - return (-1); - } - AZ(res->ai_next); - if (res->ai_family != pfam) { + SES_Reserve_client_addr(req->sp, &sa); + if (VSS_ResolveOne(sa, fld[1], fld[3], + pfam, SOCK_STREAM, AI_NUMERICHOST | AI_NUMERICSERV) == NULL) { VSL(SLT_ProxyGarbage, req->sp->vxid, - "PROXY1: %s got wrong protocol (%d)", - fld[0], res->ai_family); - freeaddrinfo(res); + "PROXY1: Cannot resolve source address"); return (-1); } - SES_Reserve_client_addr(req->sp, &sa); - AN(VSA_Build(sa, res->ai_addr, res->ai_addrlen)); SES_Set_String_Attr(req->sp, SA_CLIENT_IP, fld[1]); SES_Set_String_Attr(req->sp, SA_CLIENT_PORT, fld[3]); - freeaddrinfo(res); - i = getaddrinfo(fld[2], fld[4], &hints, &res); - if (i != 0) { - VSL(SLT_ProxyGarbage, req->sp->vxid, - "PROXY1: Cannot resolve destination address (%s)", - gai_strerror(i)); - return (-1); - } - AZ(res->ai_next); - if (res->ai_family != pfam) { + SES_Reserve_server_addr(req->sp, &sa); + if (VSS_ResolveOne(sa, fld[2], fld[4], + pfam, SOCK_STREAM, AI_NUMERICHOST | AI_NUMERICSERV) == NULL) { VSL(SLT_ProxyGarbage, req->sp->vxid, - "PROXY1: %s got wrong protocol (%d)", - fld[0], res->ai_family); - freeaddrinfo(res); + "PROXY1: Cannot resolve destination address"); return (-1); } - SES_Reserve_server_addr(req->sp, &sa); - AN(VSA_Build(sa, res->ai_addr, res->ai_addrlen)); - freeaddrinfo(res); + SES_Set_String_Attr(req->sp, SA_CLIENT_IP, fld[1]); + SES_Set_String_Attr(req->sp, SA_CLIENT_PORT, fld[3]); VSL(SLT_Proxy, req->sp->vxid, "1 %s %s %s %s", fld[1], fld[3], fld[2], fld[4]); diff --git a/bin/varnishtest/tests/o00000.vtc b/bin/varnishtest/tests/o00000.vtc index 1e5dcabf8..08459fbe4 100644 --- a/bin/varnishtest/tests/o00000.vtc +++ b/bin/varnishtest/tests/o00000.vtc @@ -34,10 +34,10 @@ logexpect l1 -v v1 { expect * 1002 ProxyGarbage "PROXY1: Too few fields" expect * 1003 ProxyGarbage "PROXY1: Too many fields" expect * 1004 ProxyGarbage "PROXY1: Wrong TCP\\[46\\] field" - expect * 1005 ProxyGarbage "PROXY1: Cannot resolve source address \$.*\$" - expect * 1007 ProxyGarbage "PROXY1: Cannot resolve destination address \$.*\$" - expect * 1013 ProxyGarbage "PROXY1: TCP4 got wrong protocol \$[0-9]*\$" - expect * 1014 ProxyGarbage "PROXY1: TCP6 got wrong protocol \$[0-9]*\$" + expect * 1005 ProxyGarbage "PROXY1: Cannot resolve source address" + expect * 1007 ProxyGarbage "PROXY1: Cannot resolve destination address" + expect * 1009 ProxyGarbage "PROXY1: Cannot resolve source address" + expect * 1011 ProxyGarbage "PROXY1: Cannot resolve source address" expect * 1015 Proxy "1 1.2.3.4 1234 5.6.7.8 5678" expect * 1018 Proxy "1 1:f::2 1234 5:a::8 5678" expect * 1021 Proxy "1 1:f::3 1234 5:a::8 5678" @@ -50,7 +50,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY " @@ -58,7 +58,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY A B C D\r\n" @@ -66,7 +66,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY A B C D E F\r\n" @@ -74,7 +74,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY A B C D E\r\n" @@ -82,7 +82,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY TCP4 B C D E\r\n" @@ -90,7 +90,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY TCP4 1.2.3.4 C D E\r\n" @@ -98,7 +98,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY TCP4 1.2.3.4 D 1234 E\r\n" @@ -106,7 +106,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY TCP4 1.2.3.4 5.6.7.8 1234 E\r\n" @@ -114,7 +114,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY TCP6 B C D E\r\n" @@ -122,7 +122,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY TCP6 1:f::2 C D E\r\n" @@ -130,7 +130,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY TCP6 1:f::2 1234 D E\r\n" @@ -138,7 +138,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY TCP6 1:f::2 5:a::8 1234 E\r\n" @@ -146,7 +146,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY TCP4 1:f::2 5:a::8 1234 5678\r\n" @@ -154,7 +154,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup client c1 { send "PROXY TCP6 1.2.3.4 5.6.7.8 1234 5678\r\n" @@ -162,7 +162,7 @@ client c1 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup # Finally try something which works... client c1 -proxy1 "1.2.3.4:1234 5.6.7.8:5678" { @@ -179,7 +179,7 @@ client c1 -proxy1 "1.2.3.4:1234 5.6.7.8:5678" { expect resp.http.rp != "1234" } -run -delay .1 +varnish v1 -vsl_catchup client c1 -proxy1 "[1:f::2]:1234 [5:a::8]:5678" { txreq -url /2 @@ -195,7 +195,7 @@ client c1 -proxy1 "[1:f::2]:1234 [5:a::8]:5678" { expect resp.http.rp != "1234" } -run -delay .1 +varnish v1 -vsl_catchup # Try with appended request (See also: #1728) client c2 { @@ -204,13 +204,15 @@ client c2 { expect resp.http.url == "/3" } -run +varnish v1 -vsl_catchup + # Malformed (missing \r) client c2 { send "PROXY TCP4 1.2.3.4 5.6.7.8 1234 5678\n" expect_close } -run -delay .1 +varnish v1 -vsl_catchup # Malformed, too long (106) # NB: Should check VSL for proper disposal @@ -219,7 +221,7 @@ client c2 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup # Malformed, too long (107) # NB: Should check VSL for proper disposal @@ -228,7 +230,7 @@ client c2 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup # Malformed, too long (108) # NB: Should check VSL for proper disposal @@ -237,6 +239,6 @@ client c2 { expect_close } -run -delay .1 +varnish v1 -vsl_catchup logexpect l1 -wait diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c index 5c714de0b..dac5a5840 100644 --- a/bin/varnishtest/vtc_client.c +++ b/bin/varnishtest/vtc_client.c @@ -32,6 +32,7 @@ #include #include +#include #include #include #include @@ -73,21 +74,11 @@ static VTAILQ_HEAD(, client) clients = * Send the proxy header */ -static int v_matchproto_(vss_resolved_f) -proxy_cb(void *priv, const struct suckaddr *sa) -{ - struct suckaddr **addr = priv; - *addr = VSA_Clone(sa); - return (1); -} - static void client_proxy(struct vtclog *vl, int fd, int version, const char *spec) { struct suckaddr *sac, *sas; - const char *err; char *p, *p2; - int error; p = strdup(spec); AN(p); @@ -95,14 +86,12 @@ client_proxy(struct vtclog *vl, int fd, int version, const char *spec) AN(p2); *p2++ = '\0'; - error = VSS_resolver(p, NULL, proxy_cb, &sac, &err); - if (err != NULL) - vtc_fatal(vl, "Could not resolve client address: %s", err); - assert(error == 1); - error = VSS_resolver(p2, NULL, proxy_cb, &sas, &err); - if (err != NULL) - vtc_fatal(vl, "Could not resolve server address: %s", err); - assert(error == 1); + sac = VSS_ResolveOne(NULL, p, NULL, 0, SOCK_STREAM, AI_PASSIVE); + if (sac == NULL) + vtc_fatal(vl, "Could not resolve client address"); + sas = VSS_ResolveOne(NULL, p2, NULL, 0, SOCK_STREAM, AI_PASSIVE); + if (sas == NULL) + vtc_fatal(vl, "Could not resolve server address"); if (vtc_send_proxy(fd, version, sac, sas)) vtc_fatal(vl, "Write failed: %s", strerror(errno)); free(p); diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 01ae08ba7..ee9b74c10 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -39,6 +39,7 @@ #include #include #include +#include #include "vtc.h" @@ -433,18 +434,11 @@ i_mode(void) * Figure out what IP related magic */ -static int v_matchproto_(vss_resolved_f) -bind_cb(void *priv, const struct suckaddr *sa) -{ - (void)priv; - return (VTCP_bind(sa, NULL)); -} - static void ip_magic(void) { - const char *p; int fd; + struct suckaddr *sa; char abuf[VTCP_ADDRBUFSIZE]; char pbuf[VTCP_PORTBUFSIZE]; @@ -455,7 +449,9 @@ ip_magic(void) * XXX: "localhost", but that doesn't work out of the box. * XXX: Things like "prefer_ipv6" parameter complicates things. */ - fd = VSS_resolver("127.0.0.1", NULL, bind_cb, NULL, &p); + sa = VSS_ResolveOne(NULL, "127.0.0.1", "0", 0, SOCK_STREAM, 0); + AN(sa); + fd = VTCP_bind(sa, NULL); assert(fd >= 0); VTCP_myname(fd, abuf, sizeof abuf, pbuf, sizeof(pbuf)); extmacro_def("localhost", "%s", abuf); diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index b55d4be13..8a1f80285 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -345,30 +345,19 @@ cmd_delay(CMD_ARGS) * DNS services. This is a basic sanity check for those. */ -static int v_matchproto_(vss_resolved_f) -dns_cb(void *priv, const struct suckaddr *sa) +static int +dns_works(void) { + struct suckaddr *sa; char abuf[VTCP_ADDRBUFSIZE]; char pbuf[VTCP_PORTBUFSIZE]; - int *ret = priv; + sa = VSS_ResolveOne(NULL, "dns-canary.freebsd.dk", NULL, 0, 0, 0); + if (sa == NULL) + return (0); VTCP_name(sa, abuf, sizeof abuf, pbuf, sizeof pbuf); - if (strcmp(abuf, "192.0.2.255")) { - fprintf(stderr, "DNS-test: Wrong response: %s\n", abuf); - *ret = -1; - } else if (*ret == 0) - *ret = 1; - return (0); -} - -static int -dns_works(void) -{ - int ret = 0, error; - const char *msg; - - error = VSS_resolver("dns-canary.freebsd.dk", NULL, dns_cb, &ret, &msg); - if (error || msg != NULL || ret != 1) + free(sa); + if (strcmp(abuf, "192.0.2.255")) return (0); return (1); } diff --git a/include/vss.h b/include/vss.h index aa645bdc7..ab4cdc33d 100644 --- a/include/vss.h +++ b/include/vss.h @@ -34,3 +34,6 @@ int VSS_resolver(const char *addr, const char *def_port, vss_resolved_f *func, void *priv, const char **err); int VSS_resolver_socktype(const char *addr, const char *def_port, vss_resolved_f *func, void *priv, const char **err, int socktype); +struct suckaddr *VSS_ResolveOne(void *dst, + const char *addr, const char *port, + int family, int socktype, int flags); diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c index 7282019de..38c766203 100644 --- a/lib/libvarnish/vss.c +++ b/lib/libvarnish/vss.c @@ -154,3 +154,42 @@ VSS_resolver(const char *addr, const char *def_port, vss_resolved_f *func, return (VSS_resolver_socktype( addr, def_port, func, priv, err, SOCK_STREAM)); } + +#include + +struct suckaddr * +VSS_ResolveOne(void *dst, const char *addr, const char *port, + int family, int socktype, int flags) +{ + struct addrinfo hints, *res = NULL; + struct suckaddr *retval = NULL; + char *p = NULL, *hp, *pp; + int error; + + memset(&hints, 0, sizeof hints); + hints.ai_family = family; + hints.ai_socktype = socktype; + hints.ai_flags = flags; + + AN(addr); + if (port != NULL) { + error = getaddrinfo(addr, port, &hints, &res); + } else { + p = strdup(addr); + AN(p); + if (vss_parse(p, &hp, &pp) != NULL || pp == NULL) { + free(p); + return (NULL); + } + error = getaddrinfo(hp, pp, &hints, &res); + free(p); + } + if (!error && res != NULL && res->ai_next == NULL) { + if (dst == NULL) + retval = VSA_Malloc(res->ai_addr, res->ai_addrlen); + else + retval = VSA_Build(dst, res->ai_addr, res->ai_addrlen); + freeaddrinfo(res); + } + return (retval); +} diff --git a/lib/libvmod_debug/vmod_debug_dyn.c b/lib/libvmod_debug/vmod_debug_dyn.c index 338c2518c..35fdd9baa 100644 --- a/lib/libvmod_debug/vmod_debug_dyn.c +++ b/lib/libvmod_debug/vmod_debug_dyn.c @@ -40,6 +40,7 @@ #include "cache/cache.h" #include "vsa.h" +#include "vss.h" #include "vcc_if.h" struct xyzzy_debug_dyn { @@ -62,7 +63,6 @@ static void dyn_dir_init(VRT_CTX, struct xyzzy_debug_dyn *dyn, VCL_STRING addr, VCL_STRING port, VCL_PROBE probe) { - struct addrinfo hints, *res = NULL; struct suckaddr *sa; struct director *dir, *dir2; struct vrt_backend vrt; @@ -77,13 +77,7 @@ dyn_dir_init(VRT_CTX, struct xyzzy_debug_dyn *dyn, vrt.hosthdr = addr; vrt.probe = probe; - memset(&hints, 0, sizeof(hints)); - hints.ai_family = AF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - AZ(getaddrinfo(addr, port, &hints, &res)); - XXXAZ(res->ai_next); - - sa = VSA_Malloc(res->ai_addr, res->ai_addrlen); + sa = VSS_ResolveOne(NULL, addr, port, AF_UNSPEC, SOCK_STREAM, 0); AN(sa); if (VSA_Get_Proto(sa) == AF_INET) { vrt.ipv4_addr = addr; @@ -94,8 +88,6 @@ dyn_dir_init(VRT_CTX, struct xyzzy_debug_dyn *dyn, } else WRONG("Wrong proto family"); - freeaddrinfo(res); - dir = VRT_new_backend(ctx, &vrt); AN(dir); diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index ba10fa513..e98b98cef 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -41,6 +41,7 @@ #include "vnum.h" #include "vsa.h" +#include "vss.h" #include "vtim.h" #include "vcc_if.h" @@ -79,13 +80,11 @@ vmod_integer(VRT_CTX, VCL_STRING p, VCL_INT i) VCL_IP vmod_ip(VRT_CTX, VCL_STRING s, VCL_IP d, VCL_BOOL n) { - struct addrinfo hints, *res0 = NULL; - const struct addrinfo *res; - int error; void *p; - const struct suckaddr *r; + VCL_IP retval; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(s); AN(d); assert(VSA_Sane(d)); @@ -95,30 +94,14 @@ vmod_ip(VRT_CTX, VCL_STRING s, VCL_IP d, VCL_BOOL n) "vmod std.ip(): insufficient workspace"); return (d); } - r = NULL; - - if (s != NULL) { - memset(&hints, 0, sizeof(hints)); - hints.ai_family = PF_UNSPEC; - hints.ai_socktype = SOCK_STREAM; - if (!n) - hints.ai_flags |= AI_NUMERICHOST; - error = getaddrinfo(s, "80", &hints, &res0); - if (!error) { - for (res = res0; res != NULL; res = res->ai_next) { - r = VSA_Build(p, res->ai_addr, res->ai_addrlen); - if (r != NULL) - break; - } - } - } - if (r == NULL) { - WS_Reset(ctx->ws, (uintptr_t)p); - r = d; - } - if (res0 != NULL) - freeaddrinfo(res0); - return (r); + + retval = VSS_ResolveOne(p, s, "80", PF_UNSPEC, SOCK_STREAM, + n ? 0 : AI_NUMERICHOST); + if (retval != NULL) + return (retval); + + WS_Reset(ctx->ws, (uintptr_t)p); + return (d); } VCL_REAL v_matchproto_(td_std_real) From dridi.boukelmoune at gmail.com Wed May 22 09:03:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 May 2019 09:03:11 +0000 (UTC) Subject: [6.0] 1898c0012 Coverity doesnt know that getaddrinfo only allocates res on success. Message-ID: <20190522090311.31CA95168@lists.varnish-cache.org> commit 1898c001275aaf08d0b83e75996f5ab3badf744f Author: Poul-Henning Kamp Date: Thu May 9 06:03:02 2019 +0000 Coverity doesnt know that getaddrinfo only allocates res on success. diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c index 38c766203..0ad31a5b9 100644 --- a/lib/libvarnish/vss.c +++ b/lib/libvarnish/vss.c @@ -189,7 +189,7 @@ VSS_ResolveOne(void *dst, const char *addr, const char *port, retval = VSA_Malloc(res->ai_addr, res->ai_addrlen); else retval = VSA_Build(dst, res->ai_addr, res->ai_addrlen); - freeaddrinfo(res); } + freeaddrinfo(res); return (retval); } From dridi.boukelmoune at gmail.com Wed May 22 09:03:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 May 2019 09:03:11 +0000 (UTC) Subject: [6.0] 7e57728f9 Only free addrinfo if it is non-NULL Message-ID: <20190522090311.4BC49516C@lists.varnish-cache.org> commit 7e57728f91488d645aadcf85535856b5a25a5d4d Author: Poul-Henning Kamp Date: Thu May 9 07:17:10 2019 +0000 Only free addrinfo if it is non-NULL diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c index 0ad31a5b9..4431dea0d 100644 --- a/lib/libvarnish/vss.c +++ b/lib/libvarnish/vss.c @@ -108,7 +108,7 @@ int VSS_resolver_socktype(const char *addr, const char *def_port, vss_resolved_f *func, void *priv, const char **err, int socktype) { - struct addrinfo hints, *res0, *res; + struct addrinfo hints, *res0 = NULL, *res; struct suckaddr *vsa; char *h; char *adp, *hop; @@ -143,7 +143,8 @@ VSS_resolver_socktype(const char *addr, const char *def_port, break; } } - freeaddrinfo(res0); + if (res0 != NULL) + freeaddrinfo(res0); return (ret); } From dridi.boukelmoune at gmail.com Wed May 22 09:03:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 May 2019 09:03:11 +0000 (UTC) Subject: [6.0] 39dfc1c37 Duh: Never rush a fix. Message-ID: <20190522090311.67EA15170@lists.varnish-cache.org> commit 39dfc1c37dd5a555487378eb24b9e8c47c99e3fd Author: Poul-Henning Kamp Date: Thu May 9 07:43:21 2019 +0000 Duh: Never rush a fix. diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c index 4431dea0d..410fab9f3 100644 --- a/lib/libvarnish/vss.c +++ b/lib/libvarnish/vss.c @@ -108,7 +108,7 @@ int VSS_resolver_socktype(const char *addr, const char *def_port, vss_resolved_f *func, void *priv, const char **err, int socktype) { - struct addrinfo hints, *res0 = NULL, *res; + struct addrinfo hints, *res0, *res; struct suckaddr *vsa; char *h; char *adp, *hop; @@ -143,8 +143,7 @@ VSS_resolver_socktype(const char *addr, const char *def_port, break; } } - if (res0 != NULL) - freeaddrinfo(res0); + freeaddrinfo(res0); return (ret); } @@ -191,6 +190,7 @@ VSS_ResolveOne(void *dst, const char *addr, const char *port, else retval = VSA_Build(dst, res->ai_addr, res->ai_addrlen); } - freeaddrinfo(res); + if (res != NULL) + freeaddrinfo(res); return (retval); } From dridi.boukelmoune at gmail.com Wed May 22 09:03:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 May 2019 09:03:11 +0000 (UTC) Subject: [6.0] caf7d220f GC #include Message-ID: <20190522090311.87924517D@lists.varnish-cache.org> commit caf7d220fa8c86a9dea0cbb97211717e0193439a Author: Dridi Boukelmoune Date: Thu May 9 10:18:12 2019 +0200 GC #include diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c index 410fab9f3..6ba726d85 100644 --- a/lib/libvarnish/vss.c +++ b/lib/libvarnish/vss.c @@ -155,8 +155,6 @@ VSS_resolver(const char *addr, const char *def_port, vss_resolved_f *func, addr, def_port, func, priv, err, SOCK_STREAM)); } -#include - struct suckaddr * VSS_ResolveOne(void *dst, const char *addr, const char *port, int family, int socktype, int flags) From dridi.boukelmoune at gmail.com Wed May 22 09:03:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 May 2019 09:03:11 +0000 (UTC) Subject: [6.0] 49e207fa3 Improve vss_parse documentation Message-ID: <20190522090311.9FD5E5188@lists.varnish-cache.org> commit 49e207fa30c8a77f37ba9901c831b7bbce7f9c98 Author: Dridi Boukelmoune Date: Thu May 9 10:19:33 2019 +0200 Improve vss_parse documentation diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c index 6ba726d85..ebbdf4f2b 100644 --- a/lib/libvarnish/vss.c +++ b/lib/libvarnish/vss.c @@ -46,14 +46,15 @@ /* * Take a string provided by the user and break it up into address and - * port parts. Examples of acceptable input include: + * port parts. The address and port separator may be either a colon or + * a whitespace. Examples of acceptable input include: * - * "localhost" - "localhost:80" - * "127.0.0.1" - "127.0.0.1:80" - * "0.0.0.0" - "0.0.0.0:80" - * "[::1]" - "[::1]:80" - * "[::]" - "[::]:80" - * "::1" - "[::1]:80" + * "localhost" - "localhost:80" - "localhost 80" + * "127.0.0.1" - "127.0.0.1:80" - "127.0.0.1 80" + * "0.0.0.0" - "0.0.0.0:80" - "0.0.0.0 80" + * "[::1]" - "[::1]:80" - "[::1] 80" + * "[::]" - "[::]:80" - "[::] 80" + * "::1" - "[::1]:80" - "::1 80" * * See also RFC5952 */ From dridi.boukelmoune at gmail.com Wed May 22 09:03:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 May 2019 09:03:11 +0000 (UTC) Subject: [6.0] bea21ef05 Don't defer an std.ip() failure to WS_Alloc'ate Message-ID: <20190522090311.B7F6E518D@lists.varnish-cache.org> commit bea21ef05a2631a071c8e20edfb7323aa1d2e62c Author: Dridi Boukelmoune Date: Thu May 9 15:21:11 2019 +0200 Don't defer an std.ip() failure to WS_Alloc'ate At this point the workspace already overflowed so there's no point making further progress to fail on the next workspace operation. Incidentally, we are failing for an IP conversion, not an integer. Conflicts: lib/libvmod_std/vmod_std_conversions.c diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index e98b98cef..49fe95e61 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -90,9 +90,8 @@ vmod_ip(VRT_CTX, VCL_STRING s, VCL_IP d, VCL_BOOL n) p = WS_Alloc(ctx->ws, vsa_suckaddr_len); if (p == NULL) { - VSLb(ctx->vsl, SLT_VCL_Error, - "vmod std.ip(): insufficient workspace"); - return (d); + VRT_fail(ctx, "std.ip: insufficient workspace"); + return (NULL); } retval = VSS_ResolveOne(p, s, "80", PF_UNSPEC, SOCK_STREAM, From dridi.boukelmoune at gmail.com Wed May 22 09:03:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 May 2019 09:03:11 +0000 (UTC) Subject: [6.0] a7f46d149 Extract the resolution logic to vss_resolve Message-ID: <20190522090311.D39F45197@lists.varnish-cache.org> commit a7f46d149a8ffd20adfb6d2004904b6e6168fed4 Author: Dridi Boukelmoune Date: Thu May 9 10:35:17 2019 +0200 Extract the resolution logic to vss_resolve It can now be shared by established callback-based resolvers and the new VSS_ResolveOne. This also changes the semantics of VSS_ResolveOne in the sense that the port is now a default port, overriden by the address if it contains one. Also make it clear that VTCP was already relying on a VSS function that didn't and still doesn't allow a null errp argument, while conversely all VTCP_open call sites provide a valid errp argument. diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c index dac5a5840..d0962e5bc 100644 --- a/bin/varnishtest/vtc_client.c +++ b/bin/varnishtest/vtc_client.c @@ -110,6 +110,8 @@ client_tcp_connect(struct vtclog *vl, const char *addr, double tmo, int fd; char mabuf[32], mpbuf[32]; + AN(addr); + AN(errp); fd = VTCP_open(addr, NULL, tmo, errp); if (fd < 0) return fd; diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index 22f5eb2fe..bd37aadb7 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -120,6 +120,8 @@ haproxy_cli_tcp_connect(struct vtclog *vl, const char *addr, vtim_dur tmo, int fd; char mabuf[32], mpbuf[32]; + AN(addr); + AN(errp); fd = VTCP_open(addr, NULL, tmo, errp); if (fd < 0) return fd; diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c index ebbdf4f2b..206b5acb3 100644 --- a/lib/libvarnish/vss.c +++ b/lib/libvarnish/vss.c @@ -98,6 +98,43 @@ vss_parse(char *str, char **addr, char **port) return (NULL); } +static int +vss_resolve(const char *addr, const char *def_port, int family, int socktype, + int flags, struct addrinfo **res, const char **errp) +{ + struct addrinfo hints; + char *p = NULL, *hp, *pp; + int ret; + + AN(addr); + AN(res); + AZ(*res); + AN(errp); + *errp = NULL; + + memset(&hints, 0, sizeof hints); + hints.ai_family = family; + hints.ai_socktype = socktype; + hints.ai_flags = flags; + + p = strdup(addr); + AN(p); + *errp = vss_parse(p, &hp, &pp); + if (*errp != NULL) { + free(p); + return (-1); + } + if (pp != NULL) + def_port = pp; + ret = getaddrinfo(hp, def_port, &hints, res); + free(p); + + if (ret != 0) + *errp = gai_strerror(ret); + + return (ret); +} + /* * Look up an address, using a default port if provided, and call * the callback function with the suckaddrs we find. @@ -107,34 +144,21 @@ vss_parse(char *str, char **addr, char **port) int VSS_resolver_socktype(const char *addr, const char *def_port, - vss_resolved_f *func, void *priv, const char **err, int socktype) + vss_resolved_f *func, void *priv, const char **errp, int socktype) { - struct addrinfo hints, *res0, *res; + struct addrinfo *res0 = NULL, *res; struct suckaddr *vsa; - char *h; - char *adp, *hop; int ret; - *err = NULL; - h = strdup(addr); - AN(h); - *err = vss_parse(h, &hop, &adp); - if (*err != NULL) { - free(h); - return (-1); - } - if (adp != NULL) - def_port = adp; + AN(addr); + AN(func); + AN(errp); - memset(&hints, 0, sizeof hints); - hints.ai_socktype = socktype; - hints.ai_flags = AI_PASSIVE; - ret = getaddrinfo(hop, def_port, &hints, &res0); - free(h); - if (ret != 0) { - *err = gai_strerror(ret); + ret = vss_resolve(addr, def_port, AF_UNSPEC, socktype, AI_PASSIVE, + &res0, errp); + if (ret != 0) return (-1); - } + for (res = res0; res != NULL; res = res->ai_next) { vsa = VSA_Malloc(res->ai_addr, res->ai_addrlen); if (vsa != NULL) { @@ -150,40 +174,25 @@ VSS_resolver_socktype(const char *addr, const char *def_port, int VSS_resolver(const char *addr, const char *def_port, vss_resolved_f *func, - void *priv, const char **err) + void *priv, const char **errp) { return (VSS_resolver_socktype( - addr, def_port, func, priv, err, SOCK_STREAM)); + addr, def_port, func, priv, errp, SOCK_STREAM)); } struct suckaddr * -VSS_ResolveOne(void *dst, const char *addr, const char *port, +VSS_ResolveOne(void *dst, const char *addr, const char *def_port, int family, int socktype, int flags) { - struct addrinfo hints, *res = NULL; + struct addrinfo *res = NULL; struct suckaddr *retval = NULL; - char *p = NULL, *hp, *pp; - int error; - - memset(&hints, 0, sizeof hints); - hints.ai_family = family; - hints.ai_socktype = socktype; - hints.ai_flags = flags; + const char *err; + int ret; AN(addr); - if (port != NULL) { - error = getaddrinfo(addr, port, &hints, &res); - } else { - p = strdup(addr); - AN(p); - if (vss_parse(p, &hp, &pp) != NULL || pp == NULL) { - free(p); - return (NULL); - } - error = getaddrinfo(hp, pp, &hints, &res); - free(p); - } - if (!error && res != NULL && res->ai_next == NULL) { + ret = vss_resolve(addr, def_port, family, socktype, flags, &res, &err); + if (ret == 0 && res != NULL && res->ai_next == NULL) { + AZ(err); if (dst == NULL) retval = VSA_Malloc(res->ai_addr, res->ai_addrlen); else diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c index 00f7f63af..696753b00 100644 --- a/lib/libvarnish/vtcp.c +++ b/lib/libvarnish/vtcp.c @@ -381,20 +381,12 @@ int VTCP_open(const char *addr, const char *def_port, vtim_dur timeout, const char **errp) { - int error; - const char *err; - if (errp != NULL) - *errp = NULL; + AN(errp); assert(timeout >= 0); - error = VSS_resolver(addr, def_port, vtcp_open_callback, - &timeout, &err); - if (err != NULL) { - if (errp != NULL) - *errp = err; - return (-1); - } - return (error); + + return (VSS_resolver(addr, def_port, vtcp_open_callback, + &timeout, errp)); } /*-------------------------------------------------------------------- @@ -513,6 +505,7 @@ VTCP_listen_on(const char *addr, const char *def_port, int depth, struct helper h; int sock; + AN(errp); h.depth = depth; h.errp = errp; From dridi.boukelmoune at gmail.com Wed May 22 09:03:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 May 2019 09:03:11 +0000 (UTC) Subject: [6.0] d02d21971 Add a VSS_ResolveFirst primitive as well Message-ID: <20190522090311.F23F1519C@lists.varnish-cache.org> commit d02d21971a99d696e2976b82e254d4da8ff299e8 Author: Dridi Boukelmoune Date: Thu May 9 20:06:42 2019 +0200 Add a VSS_ResolveFirst primitive as well This is how std.ip is documented, so VSS_ResolveOne doesn't work there. It might not be the only migration to VSS_ResolveOne that requires attention. Speaking of attention, VSA_Malloc may require some. Conflicts: lib/libvmod_std/vmod_std_conversions.c diff --git a/include/vss.h b/include/vss.h index ab4cdc33d..bf1fb2b6d 100644 --- a/include/vss.h +++ b/include/vss.h @@ -37,3 +37,6 @@ int VSS_resolver_socktype(const char *addr, const char *def_port, struct suckaddr *VSS_ResolveOne(void *dst, const char *addr, const char *port, int family, int socktype, int flags); +struct suckaddr *VSS_ResolveFirst(void *dst, + const char *addr, const char *port, + int family, int socktype, int flags); diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c index 40d84fa83..296f01374 100644 --- a/lib/libvarnish/vsa.c +++ b/lib/libvarnish/vsa.c @@ -252,6 +252,10 @@ VSA_Malloc(const void *s, unsigned sal) } if (l != 0) { ALLOC_OBJ(sua, SUCKADDR_MAGIC); + /* XXX: shouldn't we AN(sua) instead of mixing up failed + * allocations with unsupported address family or bogus + * sockaddr? + */ if (sua != NULL) memcpy(&sua->sa, s, l); } diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c index 206b5acb3..a37879a97 100644 --- a/lib/libvarnish/vss.c +++ b/lib/libvarnish/vss.c @@ -135,6 +135,17 @@ vss_resolve(const char *addr, const char *def_port, int family, int socktype, return (ret); } +static struct suckaddr * +vss_alloc_suckaddr(void *dst, const struct addrinfo *ai) +{ + + AN(ai); + if (dst == NULL) + return (VSA_Malloc(ai->ai_addr, ai->ai_addrlen)); + + return (VSA_Build(dst, ai->ai_addr, ai->ai_addrlen)); +} + /* * Look up an address, using a default port if provided, and call * the callback function with the suckaddrs we find. @@ -193,12 +204,33 @@ VSS_ResolveOne(void *dst, const char *addr, const char *def_port, ret = vss_resolve(addr, def_port, family, socktype, flags, &res, &err); if (ret == 0 && res != NULL && res->ai_next == NULL) { AZ(err); - if (dst == NULL) - retval = VSA_Malloc(res->ai_addr, res->ai_addrlen); - else - retval = VSA_Build(dst, res->ai_addr, res->ai_addrlen); + retval = vss_alloc_suckaddr(dst, res); } if (res != NULL) freeaddrinfo(res); return (retval); } + +struct suckaddr * +VSS_ResolveFirst(void *dst, const char *addr, const char *def_port, + int family, int socktype, int flags) +{ + struct addrinfo *res0 = NULL, *res; + struct suckaddr *retval = NULL; + const char *err; + int ret; + + AN(addr); + ret = vss_resolve(addr, def_port, family, socktype, flags, &res0, &err); + if (ret == 0) + AZ(err); + + for (res = res0; res != NULL; res = res->ai_next) { + retval = vss_alloc_suckaddr(dst, res); + if (retval != NULL) + break; + } + if (res0 != NULL) + freeaddrinfo(res0); + return (retval); +} diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index 49fe95e61..f3bfec539 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -94,7 +94,7 @@ vmod_ip(VRT_CTX, VCL_STRING s, VCL_IP d, VCL_BOOL n) return (NULL); } - retval = VSS_ResolveOne(p, s, "80", PF_UNSPEC, SOCK_STREAM, + retval = VSS_ResolveFirst(p, s, "80", PF_UNSPEC, SOCK_STREAM, n ? 0 : AI_NUMERICHOST); if (retval != NULL) return (retval); From dridi.boukelmoune at gmail.com Wed May 22 09:03:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 May 2019 09:03:12 +0000 (UTC) Subject: [6.0] 5618bade8 Use AF_UNSPEC for the address family Message-ID: <20190522090312.1783551A8@lists.varnish-cache.org> commit 5618bade89292faf9d41f8ca2d240e019202ec90 Author: Dridi Boukelmoune Date: Thu May 9 20:11:44 2019 +0200 Use AF_UNSPEC for the address family Conflicts: lib/libvmod_std/vmod_std_conversions.c diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index f3bfec539..da8a84cd6 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -94,7 +94,7 @@ vmod_ip(VRT_CTX, VCL_STRING s, VCL_IP d, VCL_BOOL n) return (NULL); } - retval = VSS_ResolveFirst(p, s, "80", PF_UNSPEC, SOCK_STREAM, + retval = VSS_ResolveFirst(p, s, "80", AF_UNSPEC, SOCK_STREAM, n ? 0 : AI_NUMERICHOST); if (retval != NULL) return (retval); From dridi.boukelmoune at gmail.com Wed May 22 09:03:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 May 2019 09:03:12 +0000 (UTC) Subject: [6.0] 24c586cec Add an optional default port argument to std.ip() Message-ID: <20190522090312.336F051AE@lists.varnish-cache.org> commit 24c586cecac87173679a65b31af2148c114a4597 Author: Dridi Boukelmoune Date: Wed May 15 11:13:20 2019 +0200 Add an optional default port argument to std.ip() And sync the documentation with the current behavior, part of which used to be implicit. Conflicts: lib/libvmod_std/vmod.vcc lib/libvmod_std/vmod_std_conversions.c diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index fc5ef3736..fa326bf52 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -164,16 +164,29 @@ Example | ... | } -$Function IP ip(STRING s, IP fallback, BOOL resolve = 1) +$Function IP ip(STRING s, IP fallback, BOOL resolve = 1, STRING p = "80") Description - Converts the string *s* to the first IP number returned by - the system library function getaddrinfo(3). If conversion - fails, *fallback* will be returned. - - If *resolve* is false, getaddrinfo() is called using *AI_NUMERICHOST* - to avoid network lookups. This makes "pure" IP strings cheaper to - convert. + Converts the string *s* to the first IP number returned by the + system library function `getaddrinfo(3)`. If conversion fails, + *fallback* will be returned or VCL failure will happen. + + The IP address includes a port number that can be found with + ``std.port()`` that defaults to 80. The default port can be set + to a different value with the *p* argument. It will be overriden + if *s* contains both an IP address and a port number or service + name. + + When *s* contains both, the syntax is either ``address:port`` or + ``address port``. If the address is a numerical IPv6 address it + must be enclosed between brackets, for example ``[::1] 80`` or + ``[::1]:http``. The *fallback* may also contain both an address + and a port. + + If *resolve* is false, `getaddrinfo(3)` is called using + ``AI_NUMERICHOST`` and ``AI_NUMERICSERV`` to avoid network lookups + depending on the system's `getaddrinfo(3)` or nsswitch configuration. + This makes "numerical" IP strings and services cheaper to convert. Example | if (std.ip(req.http.X-forwarded-for, "0.0.0.0") ~ my_acl) { diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index da8a84cd6..c89b79d03 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -78,7 +78,7 @@ vmod_integer(VRT_CTX, VCL_STRING p, VCL_INT i) } VCL_IP -vmod_ip(VRT_CTX, VCL_STRING s, VCL_IP d, VCL_BOOL n) +vmod_ip(VRT_CTX, VCL_STRING s, VCL_IP d, VCL_BOOL n, VCL_STRING default_port) { void *p; VCL_IP retval; @@ -94,8 +94,9 @@ vmod_ip(VRT_CTX, VCL_STRING s, VCL_IP d, VCL_BOOL n) return (NULL); } - retval = VSS_ResolveFirst(p, s, "80", AF_UNSPEC, SOCK_STREAM, - n ? 0 : AI_NUMERICHOST); + retval = VSS_ResolveFirst(p, s, default_port, AF_UNSPEC, SOCK_STREAM, + n ? 0 : AI_NUMERICHOST|AI_NUMERICSERV); + if (retval != NULL) return (retval); From dridi.boukelmoune at gmail.com Wed May 22 09:03:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 May 2019 09:03:12 +0000 (UTC) Subject: [6.0] 68df82ed6 Increase std.ip() and std.port() coverage Message-ID: <20190522090312.529D051BC@lists.varnish-cache.org> commit 68df82ed6f53bccdda08d3a80473b0d163396f52 Author: Dridi Boukelmoune Date: Wed May 15 11:59:16 2019 +0200 Increase std.ip() and std.port() coverage Conflicts: bin/varnishtest/tests/m00011.vtc lib/libvmod_debug/vmod.vcc lib/libvmod_debug/vmod_debug.c diff --git a/bin/varnishtest/tests/m00011.vtc b/bin/varnishtest/tests/m00011.vtc index cd4980665..6c761fff4 100644 --- a/bin/varnishtest/tests/m00011.vtc +++ b/bin/varnishtest/tests/m00011.vtc @@ -7,16 +7,76 @@ server s1 { varnish v1 -vcl+backend { import std; + import debug; sub vcl_deliver { - set resp.http.foo0 = std.ip("8.8.8.*", client.ip); - set resp.http.foo1 = std.ip("9.9.9.*", server.ip); - set resp.http.foo2 = std.ip("1.2.3.*", "127.0.0.2"); - set resp.http.foo3 = std.ip("1.2.3.5", "127.0.0.3"); - set resp.http.foo4 = std.ip("2001:db8::", "[::1]"); - set resp.http.foo5 = std.ip("2001::db8::", "[::1]"); - set resp.http.foo6 = std.ip("localhost", "0.0.0.0", resolve = false); - set resp.http.foo7 = std.ip("1.2.3.4", "0.0.0.0", resolve = false); + std.timestamp("t0"); + + debug.store_ip(std.ip("8.8.8.*", client.ip)); + set resp.http.ip0 = debug.get_ip(); + set resp.http.port0 = std.port(debug.get_ip()); + std.timestamp("8.8.8.*, client.ip"); + + debug.store_ip(std.ip("9.9.9.*", server.ip)); + set resp.http.ip1 = debug.get_ip(); + set resp.http.port1 = std.port(debug.get_ip()); + std.timestamp("9.9.9.*, server.ip"); + + debug.store_ip(std.ip("1.2.3.*", "127.0.0.2")); + set resp.http.ip2 = debug.get_ip(); + set resp.http.port2 = std.port(debug.get_ip()); + std.timestamp("1.2.3.*"); + + debug.store_ip(std.ip("1.2.3.5", "127.0.0.3")); + set resp.http.ip3 = debug.get_ip(); + set resp.http.port3 = std.port(debug.get_ip()); + std.timestamp("1.2.3.5"); + + debug.store_ip(std.ip("2001:db8::", "[::1]")); + set resp.http.ip4 = debug.get_ip(); + set resp.http.port4 = std.port(debug.get_ip()); + std.timestamp("2001:db8::"); + + debug.store_ip(std.ip("2001::db8::", "[::1]")); + set resp.http.ip5 = debug.get_ip(); + set resp.http.port5 = std.port(debug.get_ip()); + std.timestamp("2001::db8::"); + + debug.store_ip(std.ip("localhost", "0.0.0.0", resolve = false)); + set resp.http.ip6 = debug.get_ip(); + set resp.http.port6 = std.port(debug.get_ip()); + std.timestamp("localhost, resolve = false"); + + debug.store_ip(std.ip("1.2.3.4", "0.0.0.0", resolve = false)); + set resp.http.ip7 = debug.get_ip(); + set resp.http.port7 = std.port(debug.get_ip()); + std.timestamp("1.2.3.4, resolve = false"); + + debug.store_ip(std.ip("1.2.3.4 8080", "0.0.0.0")); + set resp.http.ip8 = debug.get_ip(); + set resp.http.port8 = std.port(debug.get_ip()); + std.timestamp("1.2.3.4 8080"); + + debug.store_ip(std.ip("1.2.3.4:443", "0.0.0.0")); + set resp.http.ip9 = debug.get_ip(); + set resp.http.port9 = std.port(debug.get_ip()); + std.timestamp("1.2.3.4:443"); + + debug.store_ip(std.ip("1.2.3.4", "0.0.0.0", resolve = false)); + set resp.http.ip10 = debug.get_ip(); + set resp.http.port10 = std.port(debug.get_ip()); + std.timestamp("1.2.3.4, resolve = false"); + + debug.store_ip(std.ip("9.9.9.*", "${s1_sock}")); + set resp.http.ip10 = debug.get_ip(); + set resp.http.port10 = std.port(debug.get_ip()); + std.timestamp("9.9.9.*, ${s1_sock}"); + + debug.store_ip(std.ip("1.2.3.4 80", "0.0.0.0", p = "443")); + set resp.http.ip11 = debug.get_ip(); + set resp.http.port11 = std.port(debug.get_ip()); + std.timestamp("1.2.3.4 80, p = 443"); + } } -start @@ -25,12 +85,29 @@ client c1 { timeout 60 txreq rxresp - expect resp.http.foo0 == "${localhost}" - expect resp.http.foo1 == "${localhost}" - expect resp.http.foo2 == "127.0.0.2" - expect resp.http.foo3 == "1.2.3.5" - expect resp.http.foo4 == "2001:db8::" - expect resp.http.foo5 == "::1" - expect resp.http.foo6 == "0.0.0.0" - expect resp.http.foo7 == "1.2.3.4" + expect resp.http.ip0 == ${localhost} + expect resp.http.port0 != 0 + expect resp.http.port0 != 80 + expect resp.http.ip1 == ${v1_addr} + expect resp.http.port1 == ${v1_port} + expect resp.http.ip2 == 127.0.0.2 + expect resp.http.port2 == 80 + expect resp.http.ip3 == 1.2.3.5 + expect resp.http.port3 == 80 + expect resp.http.ip4 == 2001:db8:: + expect resp.http.port4 == 80 + expect resp.http.ip5 == ::1 + expect resp.http.port5 == 80 + expect resp.http.ip6 == 0.0.0.0 + expect resp.http.port6 == 80 + expect resp.http.ip7 == 1.2.3.4 + expect resp.http.port7 == 80 + expect resp.http.ip8 == 1.2.3.4 + expect resp.http.port8 == 8080 + expect resp.http.ip9 == 1.2.3.4 + expect resp.http.port9 == 443 + expect resp.http.ip10 == ${s1_addr} + expect resp.http.port10 == ${s1_port} + expect resp.http.ip11 == 1.2.3.4 + expect resp.http.port11 == 80 } -run diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index 0409c1f0a..969d38265 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -193,3 +193,13 @@ $Method STRING .meth_opt(PRIV_CALL, PRIV_VCL, PRIV_TASK, [STRING s], [BOOL b]) Test object method with all the fancy stuff. + +$Function VOID store_ip(PRIV_TASK, IP) + +Store an IP address to be later found by ``debug.get_ip()`` in the same +transaction. + +$Function IP get_ip(PRIV_TASK) + +Get the IP address previously stored by ``debug.store_ip()`` in the same +transaction. diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 10c9bde96..75e310d54 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -414,3 +414,30 @@ xyzzy_vsc_destroy(VRT_CTX) vsc = NULL; AZ(pthread_mutex_unlock(&vsc_mtx)); } + +VCL_VOID +xyzzy_store_ip(VRT_CTX, struct vmod_priv *priv, VCL_IP ip) +{ + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(priv); + AZ(priv->free); + assert(VSA_Sane(ip)); + + priv->priv = TRUST_ME(ip); +} + +VCL_IP +xyzzy_get_ip(VRT_CTX, struct vmod_priv *priv) +{ + VCL_IP ip; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(priv); + AZ(priv->free); + + ip = priv->priv; + assert(VSA_Sane(ip)); + + return (ip); +} From dridi.boukelmoune at gmail.com Wed May 22 09:03:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 May 2019 09:03:12 +0000 (UTC) Subject: [6.0] 41d303866 The fallback default port for std.ip() is always 80 Message-ID: <20190522090312.7392B51C1@lists.varnish-cache.org> commit 41d30386691d7eda00916a2943e58bf10627a8ea Author: Dridi Boukelmoune Date: Wed May 15 12:02:05 2019 +0200 The fallback default port for std.ip() is always 80 Conflicts: lib/libvmod_std/vmod.vcc diff --git a/lib/libvmod_std/vmod.vcc b/lib/libvmod_std/vmod.vcc index fa326bf52..80b2d5c07 100644 --- a/lib/libvmod_std/vmod.vcc +++ b/lib/libvmod_std/vmod.vcc @@ -181,7 +181,7 @@ Description ``address port``. If the address is a numerical IPv6 address it must be enclosed between brackets, for example ``[::1] 80`` or ``[::1]:http``. The *fallback* may also contain both an address - and a port. + and a port, but its default port is always 80. If *resolve* is false, `getaddrinfo(3)` is called using ``AI_NUMERICHOST`` and ``AI_NUMERICSERV`` to avoid network lookups From dridi.boukelmoune at gmail.com Wed May 22 16:54:08 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 May 2019 16:54:08 +0000 (UTC) Subject: [master] c85aa3110 Tolerate a null address string in std.ip() Message-ID: <20190522165408.62E05102CE4@lists.varnish-cache.org> commit c85aa3110c6ab841ef33ba95dfbd962c8ed6c351 Author: Dridi Boukelmoune Date: Wed May 22 18:07:45 2019 +0200 Tolerate a null address string in std.ip() A regression from d7a81fe82ca09c9f3b3a9fd67265de00da7a4315 and the followup changes from #2993. The new VSS_Resolve{One,First} functions expect a non-null address string. diff --git a/bin/varnishtest/tests/m00011.vtc b/bin/varnishtest/tests/m00011.vtc index 6c761fff4..243e5daf6 100644 --- a/bin/varnishtest/tests/m00011.vtc +++ b/bin/varnishtest/tests/m00011.vtc @@ -77,6 +77,11 @@ varnish v1 -vcl+backend { set resp.http.port11 = std.port(debug.get_ip()); std.timestamp("1.2.3.4 80, p = 443"); + debug.store_ip(std.ip(req.http.non-existent, server.ip)); + set resp.http.ip12 = debug.get_ip(); + set resp.http.port12 = std.port(debug.get_ip()); + std.timestamp("NULL, server.ip"); + } } -start @@ -110,4 +115,6 @@ client c1 { expect resp.http.port10 == ${s1_port} expect resp.http.ip11 == 1.2.3.4 expect resp.http.port11 == 80 + expect resp.http.ip12 == ${v1_addr} + expect resp.http.port12 == ${v1_port} } -run diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index b0c07298f..52a006205 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -194,7 +194,7 @@ VCL_IP vmod_ip(VRT_CTX, struct VARGS(ip) *a) { void *p; - VCL_IP retval; + VCL_IP retval = NULL; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if (a->valid_fallback) @@ -206,10 +206,11 @@ vmod_ip(VRT_CTX, struct VARGS(ip) *a) return (NULL); } - retval = VSS_ResolveFirst( - p, a->s, a->valid_p ? a->p : "80", - AF_UNSPEC, SOCK_STREAM, - a->resolve ? 0 : AI_NUMERICHOST|AI_NUMERICSERV); + if (a->s != NULL) + retval = VSS_ResolveFirst( + p, a->s, a->valid_p ? a->p : "80", + AF_UNSPEC, SOCK_STREAM, + a->resolve ? 0 : AI_NUMERICHOST|AI_NUMERICSERV); if (retval != NULL) return (retval); From dridi.boukelmoune at gmail.com Wed May 22 17:34:08 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 22 May 2019 17:34:08 +0000 (UTC) Subject: [6.0] 2d37cff7d Tolerate a null address string in std.ip() Message-ID: <20190522173408.11F92103972@lists.varnish-cache.org> commit 2d37cff7df80c887e42263ecc44bb391655be05a Author: Dridi Boukelmoune Date: Wed May 22 18:07:45 2019 +0200 Tolerate a null address string in std.ip() A regression from d7a81fe82ca09c9f3b3a9fd67265de00da7a4315 and the followup changes from #2993. The new VSS_Resolve{One,First} functions expect a non-null address string. Conflicts: lib/libvmod_std/vmod_std_conversions.c diff --git a/bin/varnishtest/tests/m00011.vtc b/bin/varnishtest/tests/m00011.vtc index 6c761fff4..243e5daf6 100644 --- a/bin/varnishtest/tests/m00011.vtc +++ b/bin/varnishtest/tests/m00011.vtc @@ -77,6 +77,11 @@ varnish v1 -vcl+backend { set resp.http.port11 = std.port(debug.get_ip()); std.timestamp("1.2.3.4 80, p = 443"); + debug.store_ip(std.ip(req.http.non-existent, server.ip)); + set resp.http.ip12 = debug.get_ip(); + set resp.http.port12 = std.port(debug.get_ip()); + std.timestamp("NULL, server.ip"); + } } -start @@ -110,4 +115,6 @@ client c1 { expect resp.http.port10 == ${s1_port} expect resp.http.ip11 == 1.2.3.4 expect resp.http.port11 == 80 + expect resp.http.ip12 == ${v1_addr} + expect resp.http.port12 == ${v1_port} } -run diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index c89b79d03..a30086106 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -81,10 +81,9 @@ VCL_IP vmod_ip(VRT_CTX, VCL_STRING s, VCL_IP d, VCL_BOOL n, VCL_STRING default_port) { void *p; - VCL_IP retval; + VCL_IP retval = NULL; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - AN(s); AN(d); assert(VSA_Sane(d)); @@ -94,8 +93,9 @@ vmod_ip(VRT_CTX, VCL_STRING s, VCL_IP d, VCL_BOOL n, VCL_STRING default_port) return (NULL); } - retval = VSS_ResolveFirst(p, s, default_port, AF_UNSPEC, SOCK_STREAM, - n ? 0 : AI_NUMERICHOST|AI_NUMERICSERV); + if (s != NULL) + retval = VSS_ResolveFirst(p, s, default_port, AF_UNSPEC, + SOCK_STREAM, n ? 0 : AI_NUMERICHOST|AI_NUMERICSERV); if (retval != NULL) return (retval); From phk at FreeBSD.org Thu May 23 08:15:16 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 23 May 2019 08:15:16 +0000 (UTC) Subject: [master] 93512fea7 Attribute import ... as ... to Dridi Message-ID: <20190523081516.BBD31113CAE@lists.varnish-cache.org> commit 93512fea75511d00b3a91f82142148acc6a1d899 Author: Poul-Henning Kamp Date: Thu May 23 08:13:48 2019 +0000 Attribute import ... as ... to Dridi diff --git a/doc/sphinx/phk/patent.rst b/doc/sphinx/phk/patent.rst index 426e9ba13..7fa4b8f53 100644 --- a/doc/sphinx/phk/patent.rst +++ b/doc/sphinx/phk/patent.rst @@ -47,6 +47,8 @@ this morning, and as a result, you will soon be able to say:: import vmod_with_impractically_long_name as v; +(You can thank Dridi for suggesting that) + My idea that Varnish would be configured in a Domain Specific Language compiled to native code is obviously one of my better, and about 10 years ago, that was becoming very obvious. From fgsch at lodoss.net Thu May 23 09:19:07 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 23 May 2019 09:19:07 +0000 (UTC) Subject: [master] f045b9ba0 Plug minor leak Message-ID: <20190523091908.06AFF114F8D@lists.varnish-cache.org> commit f045b9ba0e1ece5cb2765192c6808d088d59f647 Author: Federico G. Schwindt Date: Thu May 23 10:17:55 2019 +0100 Plug minor leak diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index 6187c8d97..adb98efe6 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -357,6 +357,7 @@ mgt_VccCompile(struct cli *cli, struct vclprog *vcl, const char *vclname, } free(vp.csrcfile); free(vp.libfile); + free(vp.symfile); free(vp.dir); if (status) { VCLI_Out(cli, "VCL compilation failed"); @@ -376,6 +377,7 @@ mgt_VccCompile(struct cli *cli, struct vclprog *vcl, const char *vclname, free(p); mgt_vcl_symtab(vcl, vj); (void)unlink(vp.symfile); + free(vp.symfile); if (!MGT_DO_DEBUG(DBG_VCL_KEEP)) (void)unlink(vp.csrcfile); From fgsch at lodoss.net Thu May 23 09:19:08 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 23 May 2019 09:19:08 +0000 (UTC) Subject: [master] a147a9900 Polish Message-ID: <20190523091908.198CD114F90@lists.varnish-cache.org> commit a147a9900c1824e0aa437ba80b878a957e7b2036 Author: Federico G. Schwindt Date: Thu May 23 10:18:18 2019 +0100 Polish diff --git a/bin/varnishtest/tests/r02433.vtc b/bin/varnishtest/tests/r02433.vtc index 81f76cc4c..26cceeb08 100644 --- a/bin/varnishtest/tests/r02433.vtc +++ b/bin/varnishtest/tests/r02433.vtc @@ -14,7 +14,7 @@ varnish v1 -vcl+backend { } delay 4 varnish v1 -cliok ping -shell -match "auto +cold" { varnishadm -n ${v1_name} vcl.list | grep vcl1 } +shell -match "auto +cold +0 +vcl1" { varnishadm -n ${v1_name} vcl.list } # first we try to label a cold VCL in auto state @@ -28,7 +28,7 @@ shell -err { varnishadm -n ${v1_name} vcl.list | grep label1 } # second we try to label VCL in cold state varnish v1 -cliok "vcl.state vcl1 cold" -shell -match "cold +cold" { varnishadm -n ${v1_name} vcl.list | grep vcl1 } +shell -match "cold +cold +0 +vcl1" { varnishadm -n ${v1_name} vcl.list } varnish v1 -cliexpect "set to auto or warm first" "vcl.label label1 vcl1" From dridi.boukelmoune at gmail.com Fri May 24 18:22:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 24 May 2019 18:22:11 +0000 (UTC) Subject: [master] d6d341609 Revert "actually, the scope must not change either" Message-ID: <20190524182211.9782F115DD1@lists.varnish-cache.org> commit d6d3416090b8b7db62d2cd0958e756e23ea92352 Author: Dridi Boukelmoune Date: Fri May 24 19:29:55 2019 +0200 Revert "actually, the scope must not change either" This reverts commit 86af5ce095a6e39e313cf7c6c23a49edba4aace6. Unfortunately we have two scopes top and task that may conflict when esi_level is zero. This restores the id field, making struct vrt_priv slightly larger than when it used to be managed as a tailq. But in the context of a red-black tree for dynamic priv lookups the benefits should still outweigh the slight increase in memory footprint. Conflicts: bin/varnishd/cache/cache_vrt_priv.c Fixes #3003 diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index daa667aab..76152cbf3 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -42,6 +42,7 @@ struct vrt_priv { #define VRT_PRIV_MAGIC 0x24157a52 VRBT_ENTRY(vrt_priv) entry; struct vmod_priv priv[1]; + uintptr_t id; uintptr_t vmod_id; }; @@ -95,9 +96,9 @@ VRTPRIV_init(struct vrt_privs *privs) static inline int vrt_priv_dyncmp(const struct vrt_priv *vp1, const struct vrt_priv *vp2) { - if (vp1->vmod_id < vp2->vmod_id) + if (vp1->vmod_id < vp2->vmod_id || vp1->id < vp2->id) return (-1); - if (vp1->vmod_id > vp2->vmod_id) + if (vp1->vmod_id > vp2->vmod_id || vp1->id > vp2->id) return (1); return (0); } @@ -105,17 +106,23 @@ vrt_priv_dyncmp(const struct vrt_priv *vp1, const struct vrt_priv *vp2) VRBT_GENERATE_STATIC(vrt_priv_tree, vrt_priv, entry, vrt_priv_dyncmp) static struct vmod_priv * -vrt_priv_dynamic(struct ws *ws, struct vrt_privs *vps, uintptr_t vmod_id) +vrt_priv_dynamic(struct ws *ws, struct vrt_privs *vps, uintptr_t id, + uintptr_t vmod_id) { struct vrt_priv *vp; - const struct vrt_priv needle = {.vmod_id = vmod_id}; + const struct vrt_priv needle = { + .id = id, + .vmod_id = vmod_id, + }; CHECK_OBJ_NOTNULL(vps, VRT_PRIVS_MAGIC); + AN(id); AN(vmod_id); vp = VRBT_FIND(vrt_priv_tree, &vps->privs, &needle); if (vp) { CHECK_OBJ(vp, VRT_PRIV_MAGIC); + assert(vp->id == id); assert(vp->vmod_id == vmod_id); return (vp->priv); } @@ -124,6 +131,7 @@ vrt_priv_dynamic(struct ws *ws, struct vrt_privs *vps, uintptr_t vmod_id) if (vp == NULL) return (NULL); INIT_OBJ(vp, VRT_PRIV_MAGIC); + vp->id = id; vp->vmod_id = vmod_id; VRBT_INSERT(vrt_priv_tree, &vps->privs, vp); return (vp->priv); @@ -132,6 +140,7 @@ vrt_priv_dynamic(struct ws *ws, struct vrt_privs *vps, uintptr_t vmod_id) struct vmod_priv * VRT_priv_task(VRT_CTX, const void *vmod_id) { + uintptr_t id; struct vrt_privs *vps; struct vmod_priv *vp; @@ -141,34 +150,40 @@ VRT_priv_task(VRT_CTX, const void *vmod_id) if (ctx->req) { CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); + id = (uintptr_t)ctx->req; CAST_OBJ_NOTNULL(vps, ctx->req->privs, VRT_PRIVS_MAGIC); } else if (ctx->bo) { CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + id = (uintptr_t)ctx->bo; CAST_OBJ_NOTNULL(vps, ctx->bo->privs, VRT_PRIVS_MAGIC); } else { ASSERT_CLI(); + id = (uintptr_t)cli_task_privs; CAST_OBJ_NOTNULL(vps, cli_task_privs, VRT_PRIVS_MAGIC); } - vp = vrt_priv_dynamic(ctx->ws, vps, (uintptr_t)vmod_id); + vp = vrt_priv_dynamic(ctx->ws, vps, id, (uintptr_t)vmod_id); return (vp); } struct vmod_priv * VRT_priv_top(VRT_CTX, const void *vmod_id) { + uintptr_t id; struct vrt_privs *vps; struct req *req; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if (ctx->req == NULL) { + /* XXX: should we VRT_fail here instead? */ WRONG("PRIV_TOP is only accessible in client VCL context"); NEEDLESS(return (NULL)); } CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); req = ctx->req->topreq; + id = (uintptr_t)&req->topreq; CAST_OBJ_NOTNULL(vps, req->privs, VRT_PRIVS_MAGIC); - return (vrt_priv_dynamic(req->ws, vps, (uintptr_t)vmod_id)); + return (vrt_priv_dynamic(req->ws, vps, id, (uintptr_t)vmod_id)); } /*-------------------------------------------------------------------- diff --git a/bin/varnishtest/tests/r03003.vtc b/bin/varnishtest/tests/r03003.vtc new file mode 100644 index 000000000..a95836178 --- /dev/null +++ b/bin/varnishtest/tests/r03003.vtc @@ -0,0 +1,23 @@ +varnishtest "Test PRIV_TOP vs PRIV_TASK" + +varnish v1 -vcl { + import debug; + + backend bad { .host = "${bad_backend}"; } + + sub vcl_recv { + return (synth(200)); + } + + sub vcl_synth { + set resp.http.priv_task = debug.test_priv_task("task"); + set resp.http.priv_top = debug.test_priv_top("top"); + } +} -start + +client c1 { + txreq + rxresp + expect resp.http.priv_task == task + expect resp.http.priv_top == top +} -run From nils.goroll at uplex.de Sat May 25 13:55:11 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 25 May 2019 15:55:11 +0200 Subject: [master] d6d341609 Revert "actually, the scope must not change either" In-Reply-To: <20190524182211.9782F115DD1@lists.varnish-cache.org> References: <20190524182211.9782F115DD1@lists.varnish-cache.org> Message-ID: <1a6a2219-bd04-fafe-00b7-d7700cca48b8@uplex.de> Hi Dridi, thank you for looking after this. I wonder if it wouldn't make more sense to give PRIV_TOP its own tree head rather than an additional id to put into a shared tree? Nils -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From fgsch at lodoss.net Sun May 26 10:30:14 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 26 May 2019 10:30:14 +0000 (UTC) Subject: [master] 8e5236859 Prefer Python 3 tools Message-ID: <20190526103014.5705E5DF5@lists.varnish-cache.org> commit 8e5236859fd08b366dd881b187f0b177dcb82342 Author: Klemens Nanni Date: Tue May 14 17:43:57 2019 +0200 Prefer Python 3 tools The last three commits already made configure recommend installing Python 3 packages and look for versioned executables, however with a low priority. This is a problem on systems such as OpenBSD 6.5 with a default Python version at 2.7, where 3.7 flavored Python packages get installed with a "-3" binary suffix. That is, when both rst2man and rst2man-3 are installed at configure time, the lower version will be picked unless explicitly passed through `--with-feature' arguments. Regardless of this specific case, trying more specificly versioned tool names first seems correctly in line with recent development and less error prone, so change it accordingly. diff --git a/configure.ac b/configure.ac index fe4225069..fbfea3e45 100644 --- a/configure.ac +++ b/configure.ac @@ -41,7 +41,7 @@ AC_ARG_WITH([rst2man], AS_HELP_STRING([--with-rst2man=PATH], [Location of rst2man (auto)]), [RST2MAN="$withval"], AC_CHECK_PROGS(RST2MAN, - [rst2man rst2man.py rst2man-3.6 rst2man-3], + [rst2man-3.6 rst2man-3 rst2man rst2man.py], [no])) if test "x$RST2MAN" = "xno"; then AC_MSG_ERROR( @@ -52,7 +52,7 @@ AC_ARG_WITH([sphinx-build], AS_HELP_STRING([--with-sphinx-build=PATH], [Location of sphinx-build (auto)]), [SPHINX="$withval"], AC_CHECK_PROGS(SPHINX, - [sphinx-build sphinx-build-3.6 sphinx-build-3], + [sphinx-build-3.6 sphinx-build-3 sphinx-build], [no])) if test "x$SPHINX" = "xno"; then AC_MSG_ERROR( @@ -63,7 +63,7 @@ AC_ARG_WITH([rst2html], AS_HELP_STRING([--with-rst2html=PATH], [Location of rst2html (auto)]), [RST2HTML="$withval"], AC_CHECK_PROGS(RST2HTML, - [rst2html rst2html.py rst2html-3.6 rst2html-3], + [rst2html-3.6 rst2html-3 rst2html rst2html.py], "no")) if test "x$RST2HTML" = "xno"; then From fgsch at lodoss.net Sun May 26 18:08:10 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 26 May 2019 18:08:10 +0000 (UTC) Subject: [master] 3ef8142a5 Plug minor leak reported by coverity Message-ID: <20190526180810.2B450103396@lists.varnish-cache.org> commit 3ef8142a51d75e91bc7795b38cd1c9bb28239932 Author: Federico G. Schwindt Date: Sun May 26 19:04:09 2019 +0100 Plug minor leak reported by coverity diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index a9c2ffe33..fad1006a1 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -332,6 +332,7 @@ vcc_ParseImport(struct vcc *tl) msym->extra = vsym->extra; msym->vmod_name = vsym->vmod_name; AZ(dlclose(vop->hdl)); + free(fnpx); return; } } From dridi at varni.sh Mon May 27 07:18:18 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 27 May 2019 09:18:18 +0200 Subject: [master] d6d341609 Revert "actually, the scope must not change either" In-Reply-To: <1a6a2219-bd04-fafe-00b7-d7700cca48b8@uplex.de> References: <20190524182211.9782F115DD1@lists.varnish-cache.org> <1a6a2219-bd04-fafe-00b7-d7700cca48b8@uplex.de> Message-ID: Hi Nils, On Sat, May 25, 2019 at 4:00 PM Nils Goroll wrote: > > Hi Dridi, > > thank you for looking after this. You should also thank Martin without whom I would have remained on the wrong track probably until today's bugwash. > I wonder if it wouldn't make more sense to give PRIV_TOP its own tree head > rather than an additional id to put into a shared tree? I have no opinion (but I understand the rationale) so I will leave this question to the rest of the team. Dridi From phk at FreeBSD.org Mon May 27 08:01:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 27 May 2019 08:01:10 +0000 (UTC) Subject: [master] 02acb35b5 More work to turn our VCL dependency tracking into a proper symboltable with references. Message-ID: <20190527080110.A07C811290B@lists.varnish-cache.org> commit 02acb35b54eb22b39eada5e2ef12120d835718b9 Author: Poul-Henning Kamp Date: Mon May 27 07:32:02 2019 +0000 More work to turn our VCL dependency tracking into a proper symboltable with references. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 86ec2d460..41fffd334 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -89,6 +89,7 @@ varnishd_SOURCES = \ mgt/mgt_param_tweak.c \ mgt/mgt_pool.c \ mgt/mgt_shmem.c \ + mgt/mgt_symtab.c \ mgt/mgt_util.c \ mgt/mgt_vcc.c \ mgt/mgt_vcl.c \ @@ -137,6 +138,7 @@ noinst_HEADERS = \ http1/cache_http1.h \ http2/cache_http2.h \ mgt/mgt.h \ + mgt/mgt_vcl.h \ mgt/mgt_param.h \ storage/storage.h \ storage/storage_persistent.h \ diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h index 78f79ce4d..82c34b52a 100644 --- a/bin/varnishd/mgt/mgt.h +++ b/bin/varnishd/mgt/mgt.h @@ -215,9 +215,7 @@ void mgt_vcl_init(void); void mgt_vcl_startup(struct cli *, const char *vclsrc, const char *origin, const char *vclname, int Cflag); int mgt_push_vcls(struct cli *, unsigned *status, char **p); -void mgt_vcl_export_labels(struct vcc *); int mgt_has_vcl(void); -void mgt_vcl_symtab(struct vclprog *, struct vjsn *); extern char *mgt_cc_cmd; extern const char *mgt_vcl_path; extern const char *mgt_vmod_path; diff --git a/bin/varnishd/mgt/mgt_symtab.c b/bin/varnishd/mgt/mgt_symtab.c new file mode 100644 index 000000000..608c3c24e --- /dev/null +++ b/bin/varnishd/mgt/mgt_symtab.c @@ -0,0 +1,200 @@ +/*- + * Copyright (c) 2019 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * VCL/VMOD symbol table + */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include + +#include "mgt/mgt.h" +#include "mgt/mgt_vcl.h" + +#include "libvcc.h" +#include "vjsn.h" + +/*--------------------------------------------------------------------*/ + +static const char * +mgt_vcl_symtab_val(const struct vjsn_val *vv, const char *val) +{ + const struct vjsn_val *jv; + + jv = vjsn_child(vv, val); + AN(jv); + assert(jv->type == VJSN_STRING); + AN(jv->value); + return (jv->value); +} + +static void +mgt_vcl_import_vcl(struct vclprog *vp1, const struct vjsn_val *vv) +{ + struct vclprog *vp2; + + CHECK_OBJ_NOTNULL(vp1, VCLPROG_MAGIC); + AN(vv); + + vp2 = mcf_vcl_byname(mgt_vcl_symtab_val(vv, "name")); + CHECK_OBJ_NOTNULL(vp2, VCLPROG_MAGIC); + mgt_vcl_dep_add(vp1, vp2); +} + +static int +mgt_vcl_cache_vmod(const char *nm, const char *fm, const char *to) +{ + int fi, fo; + int ret = 0; + ssize_t sz; + char buf[BUFSIZ]; + + fo = open(to, O_WRONLY | O_CREAT | O_EXCL, 0744); + if (fo < 0 && errno == EEXIST) + return (0); + if (fo < 0) { + fprintf(stderr, "While creating copy of vmod %s:\n\t%s: %s\n", + nm, to, vstrerror(errno)); + return (1); + } + fi = open(fm, O_RDONLY); + if (fi < 0) { + fprintf(stderr, "Opening vmod %s from %s: %s\n", + nm, fm, vstrerror(errno)); + AZ(unlink(to)); + closefd(&fo); + return (1); + } + while (1) { + sz = read(fi, buf, sizeof buf); + if (sz == 0) + break; + if (sz < 0 || sz != write(fo, buf, sz)) { + fprintf(stderr, "Copying vmod %s: %s\n", + nm, vstrerror(errno)); + AZ(unlink(to)); + ret = 1; + break; + } + } + closefd(&fi); + AZ(fchmod(fo, 0444)); + closefd(&fo); + return (ret); +} + +static void +mgt_vcl_import_vmod(struct vclprog *vp, const struct vjsn_val *vv) +{ + struct vmodfile *vf; + struct vmoddep *vd; + const char *v_name; + const char *v_file; + const char *v_dst; + + CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC); + AN(vv); + + v_name = mgt_vcl_symtab_val(vv, "name"); + v_file = mgt_vcl_symtab_val(vv, "file"); + v_dst = mgt_vcl_symtab_val(vv, "dst"); + + VTAILQ_FOREACH(vf, &vmodhead, list) + if (!strcmp(vf->fname, v_dst)) + break; + if (vf == NULL) { + ALLOC_OBJ(vf, VMODFILE_MAGIC); + AN(vf); + REPLACE(vf->fname, v_dst); + AN(vf->fname); + VTAILQ_INIT(&vf->vcls); + AZ(mgt_vcl_cache_vmod(v_name, v_file, v_dst)); + VTAILQ_INSERT_TAIL(&vmodhead, vf, list); + } + ALLOC_OBJ(vd, VMODDEP_MAGIC); + AN(vd); + vd->to = vf; + VTAILQ_INSERT_TAIL(&vp->vmods, vd, lfrom); + VTAILQ_INSERT_TAIL(&vf->vcls, vd, lto); +} + +void +mgt_vcl_symtab(struct vclprog *vp, const char *input) +{ + struct vjsn *vj; + struct vjsn_val *v1, *v2; + const char *typ, *err; + + CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC); + AN(input); + vj = vjsn_parse(input, &err); + if (err != NULL) { + fprintf(stderr, "FATAL: Symtab parse error: %s\n%s\n", + err, input); + } + AZ(err); + AN(vj); + vp->symtab = vj; + assert(vj->value->type == VJSN_ARRAY); + VTAILQ_FOREACH(v1, &vj->value->children, list) { + assert(v1->type == VJSN_OBJECT); + v2 = vjsn_child(v1, "dir"); + if (v2 == NULL) + continue; + assert(v2->type == VJSN_STRING); + if (strcmp(v2->value, "import")) + continue; + typ = mgt_vcl_symtab_val(v1, "type"); + if (!strcmp(typ, "$VMOD")) + mgt_vcl_import_vmod(vp, v1); + else if (!strcmp(typ, "$VCL")) + mgt_vcl_import_vcl(vp, v1); + else + WRONG("Bad symtab import entry"); + } +} + + +/*--------------------------------------------------------------------*/ + +void +mgt_vcl_export_labels(struct vcc *vcc) +{ + struct vclprog *vp; + VTAILQ_FOREACH(vp, &vclhead, list) { + if (mcf_is_label(vp)) + VCC_Predef(vcc, "VCL_VCL", vp->name); + } +} + +/*--------------------------------------------------------------------*/ + diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index adb98efe6..5316ab299 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -39,25 +39,25 @@ #include #include "mgt/mgt.h" +#include "mgt/mgt_vcl.h" #include "common/heritage.h" #include "storage/storage.h" #include "libvcc.h" #include "vcli_serve.h" #include "vfil.h" -#include "vjsn.h" #include "vsub.h" #include "vtim.h" struct vcc_priv { unsigned magic; #define VCC_PRIV_MAGIC 0x70080cb8 - char *dir; const char *vclsrc; const char *vclsrcfile; - char *csrcfile; - char *libfile; - char *symfile; + struct vsb *dir; + struct vsb *csrcfile; + struct vsb *libfile; + struct vsb *symfile; }; char *mgt_cc_cmd; @@ -96,7 +96,7 @@ run_vcc(void *priv) VJ_subproc(JAIL_SUBPROC_VCC); CAST_OBJ_NOTNULL(vp, priv, VCC_PRIV_MAGIC); - AZ(chdir(vp->dir)); + AZ(chdir(VSB_data(vp->dir))); vcc = VCC_New(); AN(vcc); @@ -132,7 +132,7 @@ run_cc(void *priv) VJ_subproc(JAIL_SUBPROC_CC); CAST_OBJ_NOTNULL(vp, priv, VCC_PRIV_MAGIC); - AZ(chdir(vp->dir)); + AZ(chdir(VSB_data(vp->dir))); sb = VSB_new_auto(); AN(sb); @@ -180,7 +180,7 @@ run_dlopen(void *priv) VJ_subproc(JAIL_SUBPROC_VCLLOAD); CAST_OBJ_NOTNULL(vp, priv, VCC_PRIV_MAGIC); - if (VCL_TestLoad(vp->libfile)) + if (VCL_TestLoad(VSB_data(vp->libfile))) exit(1); exit(0); } @@ -215,13 +215,13 @@ static unsigned mgt_vcc_compile(struct vcc_priv *vp, struct vsb *sb, int C_flag) { char *csrc; - const char *err; unsigned subs; - struct vjsn *vj; - if (mgt_vcc_touchfile(vp->csrcfile, sb)) + AN(sb); + VSB_clear(sb); + if (mgt_vcc_touchfile(VSB_data(vp->csrcfile), sb)) return (2); - if (mgt_vcc_touchfile(vp->libfile, sb)) + if (mgt_vcc_touchfile(VSB_data(vp->libfile), sb)) return (2); subs = VSUB_run(sb, run_vcc, vp, "VCC-compiler", -1); @@ -229,20 +229,15 @@ mgt_vcc_compile(struct vcc_priv *vp, struct vsb *sb, int C_flag) return (subs); if (C_flag) { - csrc = VFIL_readfile(NULL, vp->csrcfile, NULL); + csrc = VFIL_readfile(NULL, VSB_data(vp->csrcfile), NULL); AN(csrc); VSB_cat(sb, csrc); free(csrc); VSB_printf(sb, "/* EXTERNAL SYMBOL TABLE\n"); - csrc = VFIL_readfile(NULL, vp->symfile, NULL); + csrc = VFIL_readfile(NULL, VSB_data(vp->symfile), NULL); AN(csrc); VSB_cat(sb, csrc); - vj = vjsn_parse(csrc, &err); - if (err != NULL) - VSB_printf(sb, "# Parse error: %s\n", err); - if (vj != NULL) - vjsn_delete(&vj); VSB_printf(sb, "*/\n"); free(csrc); } @@ -257,25 +252,53 @@ mgt_vcc_compile(struct vcc_priv *vp, struct vsb *sb, int C_flag) /*--------------------------------------------------------------------*/ +static void +mgt_vcc_init_vp(struct vcc_priv *vp) +{ + INIT_OBJ(vp, VCC_PRIV_MAGIC); + vp->csrcfile = VSB_new_auto(); + AN(vp->csrcfile); + vp->libfile = VSB_new_auto(); + AN(vp->libfile); + vp->symfile = VSB_new_auto(); + AN(vp->symfile); + vp->dir = VSB_new_auto(); + AN(vp->dir); +} + +static void +mgt_vcc_fini_vp(struct vcc_priv *vp, int leave_lib) +{ + if (!MGT_DO_DEBUG(DBG_VCL_KEEP)) { + (void)unlink(VSB_data(vp->csrcfile)); + (void)unlink(VSB_data(vp->symfile)); + if (!leave_lib) + (void)unlink(VSB_data(vp->libfile)); + } + (void)rmdir(VSB_data(vp->dir)); + VSB_destroy(&vp->csrcfile); + VSB_destroy(&vp->libfile); + VSB_destroy(&vp->symfile); + VSB_destroy(&vp->dir); +} + char * mgt_VccCompile(struct cli *cli, struct vclprog *vcl, const char *vclname, const char *vclsrc, const char *vclsrcfile, int C_flag) { - struct vcc_priv vp; + struct vcc_priv vp[1]; struct vsb *sb; - struct vjsn *vj; unsigned status; - const char *err; char *p; AN(cli); sb = VSB_new_auto(); - XXXAN(sb); + AN(sb); - INIT_OBJ(&vp, VCC_PRIV_MAGIC); - vp.vclsrc = vclsrc; - vp.vclsrcfile = vclsrcfile; + mgt_vcc_init_vp(vp); + vp->vclsrc = vclsrc; + vp->vclsrcfile = vclsrcfile; /* * The subdirectory must have a unique name to 100% certain evade @@ -309,56 +332,35 @@ mgt_VccCompile(struct cli *cli, struct vclprog *vcl, const char *vclname, * * The Best way to reproduce this is to have regexps in the VCL. */ - VSB_printf(sb, "vcl_%s.%.6f", vclname, VTIM_real()); - AZ(VSB_finish(sb)); - vp.dir = strdup(VSB_data(sb)); - AN(vp.dir); - if (VJ_make_subdir(vp.dir, "VCL", cli->sb)) { - free(vp.dir); + VSB_printf(vp->dir, "vcl_%s.%.6f", vclname, VTIM_real()); + AZ(VSB_finish(vp->dir)); + + VSB_printf(vp->csrcfile, "%s/%s", VSB_data(vp->dir), VGC_SRC); + AZ(VSB_finish(vp->csrcfile)); + + VSB_printf(vp->libfile, "%s/%s", VSB_data(vp->dir), VGC_LIB); + AZ(VSB_finish(vp->libfile)); + + VSB_printf(vp->symfile, "%s/%s", VSB_data(vp->dir), VGC_SYM); + AZ(VSB_finish(vp->symfile)); + + if (VJ_make_subdir(VSB_data(vp->dir), "VCL", cli->sb)) { + mgt_vcc_fini_vp(vp, 0); VSB_destroy(&sb); VCLI_Out(cli, "VCL compilation failed"); VCLI_SetResult(cli, CLIS_PARAM); return (NULL); } - VSB_clear(sb); - VSB_printf(sb, "%s/%s", vp.dir, VGC_SRC); - AZ(VSB_finish(sb)); - vp.csrcfile = strdup(VSB_data(sb)); - AN(vp.csrcfile); - VSB_clear(sb); - - VSB_printf(sb, "%s/%s", vp.dir, VGC_LIB); - AZ(VSB_finish(sb)); - vp.libfile = strdup(VSB_data(sb)); - AN(vp.csrcfile); - VSB_clear(sb); - - VSB_printf(sb, "%s/%s", vp.dir, VGC_SYM); - AZ(VSB_finish(sb)); - vp.symfile = strdup(VSB_data(sb)); - AN(vp.symfile); - VSB_clear(sb); - - status = mgt_vcc_compile(&vp, sb, C_flag); - + status = mgt_vcc_compile(vp, sb, C_flag); AZ(VSB_finish(sb)); if (VSB_len(sb) > 0) VCLI_Out(cli, "%s", VSB_data(sb)); VSB_destroy(&sb); if (status || C_flag) { - if (!MGT_DO_DEBUG(DBG_VCL_KEEP)) { - (void)unlink(vp.csrcfile); - (void)unlink(vp.libfile); - (void)unlink(vp.symfile); - (void)rmdir(vp.dir); - } - free(vp.csrcfile); - free(vp.libfile); - free(vp.symfile); - free(vp.dir); + mgt_vcc_fini_vp(vp, 0); if (status) { VCLI_Out(cli, "VCL compilation failed"); VCLI_SetResult(cli, CLIS_PARAM); @@ -366,26 +368,13 @@ mgt_VccCompile(struct cli *cli, struct vclprog *vcl, const char *vclname, return (NULL); } - p = VFIL_readfile(NULL, vp.symfile, NULL); + p = VFIL_readfile(NULL, VSB_data(vp->symfile), NULL); AN(p); - vj = vjsn_parse(p, &err); - if (err != NULL) - fprintf(stderr, "FATAL: Symtab parse error: %s\n%s\n", - err, p); - AZ(err); - AN(vj); - free(p); - mgt_vcl_symtab(vcl, vj); - (void)unlink(vp.symfile); - free(vp.symfile); - - if (!MGT_DO_DEBUG(DBG_VCL_KEEP)) - (void)unlink(vp.csrcfile); - free(vp.csrcfile); - - free(vp.dir); + mgt_vcl_symtab(vcl, p); VCLI_Out(cli, "VCL compiled.\n"); - return (vp.libfile); + REPLACE(p, VSB_data(vp->libfile)); + mgt_vcc_fini_vp(vp, 1); + return (p); } diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 1a388248c..88e8bcce0 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -31,17 +31,15 @@ #include "config.h" -#include #include #include #include #include -#include #include "mgt/mgt.h" +#include "mgt/mgt_vcl.h" #include "common/heritage.h" -#include "libvcc.h" #include "vcli_serve.h" #include "vct.h" #include "vev.h" @@ -56,53 +54,8 @@ static const char * const VCL_STATE_LABEL = "label"; static int vcl_count; -struct vclprog; -struct vmodfile; - -struct vmoddep { - unsigned magic; -#define VMODDEP_MAGIC 0xc1490542 - VTAILQ_ENTRY(vmoddep) lfrom; - struct vmodfile *to; - VTAILQ_ENTRY(vmoddep) lto; -}; - -struct vcldep { - unsigned magic; -#define VCLDEP_MAGIC 0xa9a17dc2 - struct vclprog *from; - VTAILQ_ENTRY(vcldep) lfrom; - struct vclprog *to; - VTAILQ_ENTRY(vcldep) lto; -}; - -struct vclprog { - unsigned magic; -#define VCLPROG_MAGIC 0x9ac09fea - VTAILQ_ENTRY(vclprog) list; - char *name; - char *fname; - unsigned warm; - const char * state; - double go_cold; - struct vjsn *symtab; - VTAILQ_HEAD(, vcldep) dfrom; - VTAILQ_HEAD(, vcldep) dto; - int nto; - int loaded; - VTAILQ_HEAD(, vmoddep) vmods; -}; - -struct vmodfile { - unsigned magic; -#define VMODFILE_MAGIC 0xffa1a0d5 - char *fname; - VTAILQ_ENTRY(vmodfile) list; - VTAILQ_HEAD(, vmoddep) vcls; -}; - -static VTAILQ_HEAD(, vclprog) vclhead = VTAILQ_HEAD_INITIALIZER(vclhead); -static VTAILQ_HEAD(, vmodfile) vmodhead = VTAILQ_HEAD_INITIALIZER(vmodhead); +struct vclproghead vclhead = VTAILQ_HEAD_INITIALIZER(vclhead); +struct vmodfilehead vmodhead = VTAILQ_HEAD_INITIALIZER(vmodhead); static struct vclprog *active_vcl; static struct vev *e_poker; @@ -127,7 +80,7 @@ mcf_vcl_parse_state(struct cli *cli, const char *s) return (NULL); } -static struct vclprog * +struct vclprog * mcf_vcl_byname(const char *name) { struct vclprog *vp; @@ -189,7 +142,7 @@ mcf_find_no_vcl(struct cli *cli, const char *name) return (1); } -static int +int mcf_is_label(const struct vclprog *vp) { return (vp->state == VCL_STATE_LABEL); @@ -197,7 +150,7 @@ mcf_is_label(const struct vclprog *vp) /*--------------------------------------------------------------------*/ -static void +void mgt_vcl_dep_add(struct vclprog *vp_from, struct vclprog *vp_to) { struct vcldep *vd; @@ -316,135 +269,6 @@ mgt_vcl_del(struct vclprog *vp) FREE_OBJ(vp); } -static const char * -mgt_vcl_symtab_val(const struct vjsn_val *vv, const char *val) -{ - const struct vjsn_val *jv; - - jv = vjsn_child(vv, val); - AN(jv); - assert(jv->type == VJSN_STRING); - AN(jv->value); - return (jv->value); -} - -static void -mgt_vcl_import_vcl(struct vclprog *vp1, const struct vjsn_val *vv) -{ - struct vclprog *vp2; - - CHECK_OBJ_NOTNULL(vp1, VCLPROG_MAGIC); - AN(vv); - - vp2 = mcf_vcl_byname(mgt_vcl_symtab_val(vv, "name")); - CHECK_OBJ_NOTNULL(vp2, VCLPROG_MAGIC); - mgt_vcl_dep_add(vp1, vp2); -} - -static int -mgt_vcl_cache_vmod(const char *nm, const char *fm, const char *to) -{ - int fi, fo; - int ret = 0; - ssize_t sz; - char buf[BUFSIZ]; - - fo = open(to, O_WRONLY | O_CREAT | O_EXCL, 0744); - if (fo < 0 && errno == EEXIST) - return (0); - if (fo < 0) { - fprintf(stderr, "While creating copy of vmod %s:\n\t%s: %s\n", - nm, to, vstrerror(errno)); - return (1); - } - fi = open(fm, O_RDONLY); - if (fi < 0) { - fprintf(stderr, "Opening vmod %s from %s: %s\n", - nm, fm, vstrerror(errno)); - AZ(unlink(to)); - closefd(&fo); - return (1); - } - while (1) { - sz = read(fi, buf, sizeof buf); - if (sz == 0) - break; - if (sz < 0 || sz != write(fo, buf, sz)) { - fprintf(stderr, "Copying vmod %s: %s\n", - nm, vstrerror(errno)); - AZ(unlink(to)); - ret = 1; - break; - } - } - closefd(&fi); - AZ(fchmod(fo, 0444)); - closefd(&fo); - return (ret); -} - -static void -mgt_vcl_import_vmod(struct vclprog *vp, const struct vjsn_val *vv) -{ - struct vmodfile *vf; - struct vmoddep *vd; - const char *v_name; - const char *v_file; - const char *v_dst; - - CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC); - AN(vv); - - v_name = mgt_vcl_symtab_val(vv, "name"); - v_file = mgt_vcl_symtab_val(vv, "file"); - v_dst = mgt_vcl_symtab_val(vv, "dst"); - - VTAILQ_FOREACH(vf, &vmodhead, list) - if (!strcmp(vf->fname, v_dst)) - break; - if (vf == NULL) { - ALLOC_OBJ(vf, VMODFILE_MAGIC); - AN(vf); - REPLACE(vf->fname, v_dst); - AN(vf->fname); - VTAILQ_INIT(&vf->vcls); - AZ(mgt_vcl_cache_vmod(v_name, v_file, v_dst)); - VTAILQ_INSERT_TAIL(&vmodhead, vf, list); - } - ALLOC_OBJ(vd, VMODDEP_MAGIC); - AN(vd); - vd->to = vf; - VTAILQ_INSERT_TAIL(&vp->vmods, vd, lfrom); - VTAILQ_INSERT_TAIL(&vf->vcls, vd, lto); -} - -void -mgt_vcl_symtab(struct vclprog *vp, struct vjsn *vj) -{ - struct vjsn_val *v1, *v2; - const char *typ; - - CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC); - vp->symtab = vj; - assert(vj->value->type == VJSN_ARRAY); - VTAILQ_FOREACH(v1, &vj->value->children, list) { - assert(v1->type == VJSN_OBJECT); - v2 = vjsn_child(v1, "dir"); - if (v2 == NULL) - continue; - assert(v2->type == VJSN_STRING); - if (strcmp(v2->value, "import")) - continue; - typ = mgt_vcl_symtab_val(v1, "type"); - if (!strcmp(typ, "$VMOD")) - mgt_vcl_import_vmod(vp, v1); - else if (!strcmp(typ, "$VCL")) - mgt_vcl_import_vcl(vp, v1); - else - WRONG("Bad symtab import entry"); - } -} - int mgt_has_vcl(void) { @@ -673,18 +497,6 @@ mgt_vcl_startup(struct cli *cli, const char *vclsrc, const char *vclname, /*--------------------------------------------------------------------*/ -void -mgt_vcl_export_labels(struct vcc *vcc) -{ - struct vclprog *vp; - VTAILQ_FOREACH(vp, &vclhead, list) { - if (mcf_is_label(vp)) - VCC_Predef(vcc, "VCL_VCL", vp->name); - } -} - -/*--------------------------------------------------------------------*/ - int mgt_push_vcls(struct cli *cli, unsigned *status, char **p) { diff --git a/bin/varnishd/mgt/mgt_vcl.h b/bin/varnishd/mgt/mgt_vcl.h new file mode 100644 index 000000000..21163d7ae --- /dev/null +++ b/bin/varnishd/mgt/mgt_vcl.h @@ -0,0 +1,85 @@ +/*- + * Copyright (c) 2019 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * VCL/VMOD symbol table + */ + +struct vclprog; +struct vmodfile; + +struct vmoddep { + unsigned magic; +#define VMODDEP_MAGIC 0xc1490542 + VTAILQ_ENTRY(vmoddep) lfrom; + struct vmodfile *to; + VTAILQ_ENTRY(vmoddep) lto; +}; + +struct vcldep { + unsigned magic; +#define VCLDEP_MAGIC 0xa9a17dc2 + struct vclprog *from; + VTAILQ_ENTRY(vcldep) lfrom; + struct vclprog *to; + VTAILQ_ENTRY(vcldep) lto; +}; + +struct vclprog { + unsigned magic; +#define VCLPROG_MAGIC 0x9ac09fea + VTAILQ_ENTRY(vclprog) list; + char *name; + char *fname; + unsigned warm; + const char * state; + double go_cold; + struct vjsn *symtab; + VTAILQ_HEAD(, vcldep) dfrom; + VTAILQ_HEAD(, vcldep) dto; + int nto; + int loaded; + VTAILQ_HEAD(, vmoddep) vmods; +}; + +struct vmodfile { + unsigned magic; +#define VMODFILE_MAGIC 0xffa1a0d5 + char *fname; + VTAILQ_ENTRY(vmodfile) list; + VTAILQ_HEAD(, vmoddep) vcls; +}; + +extern VTAILQ_HEAD(vclproghead, vclprog) vclhead; +extern VTAILQ_HEAD(vmodfilehead, vmodfile) vmodhead; + +struct vclprog *mcf_vcl_byname(const char *name); +void mgt_vcl_dep_add(struct vclprog *vp_from, struct vclprog *vp_to); +int mcf_is_label(const struct vclprog *vp); + +void mgt_vcl_export_labels(struct vcc *vcc); +void mgt_vcl_symtab(struct vclprog *vp, const char *input); + From phk at FreeBSD.org Mon May 27 08:01:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 27 May 2019 08:01:10 +0000 (UTC) Subject: [master] a808daa3d Add a "vcl.symtab" debug command. Message-ID: <20190527080110.B7E0011290E@lists.varnish-cache.org> commit a808daa3d0f5e2e3d0147a120c2ad15800161214 Author: Poul-Henning Kamp Date: Mon May 27 07:55:53 2019 +0000 Add a "vcl.symtab" debug command. diff --git a/bin/varnishd/mgt/mgt_symtab.c b/bin/varnishd/mgt/mgt_symtab.c index 608c3c24e..fff79a785 100644 --- a/bin/varnishd/mgt/mgt_symtab.c +++ b/bin/varnishd/mgt/mgt_symtab.c @@ -40,6 +40,7 @@ #include "mgt/mgt.h" #include "mgt/mgt_vcl.h" +#include "vcli_serve.h" #include "libvcc.h" #include "vjsn.h" @@ -198,3 +199,33 @@ mgt_vcl_export_labels(struct vcc *vcc) /*--------------------------------------------------------------------*/ +static void +mcf_vcl_vjsn_dump(struct cli *cli, const struct vjsn_val *vj, int indent) +{ + struct vjsn_val *vj1; + AN(cli); + AN(vj); + + VCLI_Out(cli, "%*s", indent, ""); + if (vj->name != NULL) + VCLI_Out(cli, "[\"%s\"]: ", vj->name); + VCLI_Out(cli, "{%s}", vj->type); + if (vj->value != NULL) + VCLI_Out(cli, " <%s>", vj->value); + VCLI_Out(cli, "\n"); + VTAILQ_FOREACH(vj1, &vj->children, list) + mcf_vcl_vjsn_dump(cli, vj1, indent + 2); +} + +void v_matchproto_(cli_func_t) +mcf_vcl_symtab(struct cli *cli, const char * const *av, void *priv) +{ + struct vclprog *vp; + (void)av; + (void)priv; + VTAILQ_FOREACH(vp, &vclhead, list) { + VCLI_Out(cli, "VCL: %s\n", vp->name); + if (vp->symtab != NULL) + mcf_vcl_vjsn_dump(cli, vp->symtab->value, 4); + } +} diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 88e8bcce0..147eaed90 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -906,6 +906,7 @@ static struct cli_proto cli_vcl[] = { { CLICMD_VCL_DISCARD, "", mcf_vcl_discard }, { CLICMD_VCL_LIST, "", mcf_vcl_list, mcf_vcl_list_json }, { CLICMD_VCL_LABEL, "", mcf_vcl_label }, + { CLICMD_DEBUG_VCL_SYMTAB, "d", mcf_vcl_symtab }, { NULL } }; diff --git a/bin/varnishd/mgt/mgt_vcl.h b/bin/varnishd/mgt/mgt_vcl.h index 21163d7ae..fe740dde4 100644 --- a/bin/varnishd/mgt/mgt_vcl.h +++ b/bin/varnishd/mgt/mgt_vcl.h @@ -82,4 +82,5 @@ int mcf_is_label(const struct vclprog *vp); void mgt_vcl_export_labels(struct vcc *vcc); void mgt_vcl_symtab(struct vclprog *vp, const char *input); +void mcf_vcl_symtab(struct cli *cli, const char * const *av, void *priv); diff --git a/bin/varnishtest/tests/c00077.vtc b/bin/varnishtest/tests/c00077.vtc index 8b2c35df1..78b9002f9 100644 --- a/bin/varnishtest/tests/c00077.vtc +++ b/bin/varnishtest/tests/c00077.vtc @@ -59,6 +59,9 @@ varnish v1 -vcl+backend { sub vcl_recv { return (vcl(vclA)); } } varnish v1 -clierr 300 "vcl.discard vclA" varnish v1 -vcl+backend { } + +varnish v1 -cliok "vcl.symtab" + varnish v1 -cliok "vcl.discard vcl3" varnish v1 -cliok "vcl.discard vcl4" varnish v1 -cliok "vcl.discard vcl5" diff --git a/include/tbl/cli_cmds.h b/include/tbl/cli_cmds.h index 99193c6c7..d1b9f2afa 100644 --- a/include/tbl/cli_cmds.h +++ b/include/tbl/cli_cmds.h @@ -362,6 +362,14 @@ CLI_CMD(DEBUG_PANIC_MASTER, 0, 0 ) +CLI_CMD(DEBUG_VCL_SYMTAB, + "vcl.symtab", + "vcl.symtab", + "Dump the VCL symbol-tables.", + "", + 0, 0 +) + CLI_CMD(DEBUG_VMOD, "debug.vmod", "debug.vmod", From nils.goroll at uplex.de Mon May 27 08:22:30 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 27 May 2019 10:22:30 +0200 Subject: [master] d6d341609 Revert "actually, the scope must not change either" In-Reply-To: References: <20190524182211.9782F115DD1@lists.varnish-cache.org> <1a6a2219-bd04-fafe-00b7-d7700cca48b8@uplex.de> Message-ID: > You should also thank Martin without whom I would have remained on the > wrong track probably until today's bugwash. Thank you, Martin! And I apologize for my bug. >> I wonder if it wouldn't make more sense to give PRIV_TOP its own tree head >> rather than an additional id to put into a shared tree? > > I have no opinion (but I understand the rationale) so I will leave > this question to the rest of the team. Using a different head would use less memory, be more CPU efficient and also, I think, result in simpler code. So a clear cut case in my opinion. Nils -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From phk at FreeBSD.org Mon May 27 08:41:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 27 May 2019 08:41:08 +0000 (UTC) Subject: [master] c1196bb7a Introduce the new unified import structure. Message-ID: <20190527084108.7F57D11389C@lists.varnish-cache.org> commit c1196bb7a7dde648e9ea3bb40ca5ef7939944af0 Author: Poul-Henning Kamp Date: Mon May 27 08:40:17 2019 +0000 Introduce the new unified import structure. diff --git a/bin/varnishd/mgt/mgt_symtab.c b/bin/varnishd/mgt/mgt_symtab.c index fff79a785..4a5dc96e3 100644 --- a/bin/varnishd/mgt/mgt_symtab.c +++ b/bin/varnishd/mgt/mgt_symtab.c @@ -59,15 +59,18 @@ mgt_vcl_symtab_val(const struct vjsn_val *vv, const char *val) } static void -mgt_vcl_import_vcl(struct vclprog *vp1, const struct vjsn_val *vv) +mgt_vcl_import_vcl(struct vclprog *vp1, struct import *ip, const struct vjsn_val *vv) { struct vclprog *vp2; CHECK_OBJ_NOTNULL(vp1, VCLPROG_MAGIC); + AN(ip); AN(vv); vp2 = mcf_vcl_byname(mgt_vcl_symtab_val(vv, "name")); CHECK_OBJ_NOTNULL(vp2, VCLPROG_MAGIC); + ip->vcl = vp2; + VTAILQ_INSERT_TAIL(&vp2->exports, ip, to); mgt_vcl_dep_add(vp1, vp2); } @@ -114,7 +117,7 @@ mgt_vcl_cache_vmod(const char *nm, const char *fm, const char *to) } static void -mgt_vcl_import_vmod(struct vclprog *vp, const struct vjsn_val *vv) +mgt_vcl_import_vmod(struct vclprog *vp, struct import *ip, const struct vjsn_val *vv) { struct vmodfile *vf; struct vmoddep *vd; @@ -123,6 +126,7 @@ mgt_vcl_import_vmod(struct vclprog *vp, const struct vjsn_val *vv) const char *v_dst; CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC); + AN(ip); AN(vv); v_name = mgt_vcl_symtab_val(vv, "name"); @@ -138,6 +142,7 @@ mgt_vcl_import_vmod(struct vclprog *vp, const struct vjsn_val *vv) REPLACE(vf->fname, v_dst); AN(vf->fname); VTAILQ_INIT(&vf->vcls); + VTAILQ_INIT(&vf->exports); AZ(mgt_vcl_cache_vmod(v_name, v_file, v_dst)); VTAILQ_INSERT_TAIL(&vmodhead, vf, list); } @@ -146,6 +151,8 @@ mgt_vcl_import_vmod(struct vclprog *vp, const struct vjsn_val *vv) vd->to = vf; VTAILQ_INSERT_TAIL(&vp->vmods, vd, lfrom); VTAILQ_INSERT_TAIL(&vf->vcls, vd, lto); + ip->vmod = vf; + VTAILQ_INSERT_TAIL(&vf->exports, ip, to); } void @@ -154,6 +161,7 @@ mgt_vcl_symtab(struct vclprog *vp, const char *input) struct vjsn *vj; struct vjsn_val *v1, *v2; const char *typ, *err; + struct import *ip; CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC); AN(input); @@ -174,16 +182,38 @@ mgt_vcl_symtab(struct vclprog *vp, const char *input) assert(v2->type == VJSN_STRING); if (strcmp(v2->value, "import")) continue; + ALLOC_OBJ(ip, IMPORT_MAGIC); + AN(ip); + ip->vj = v1; + ip->target = vp; + VTAILQ_INSERT_TAIL(&vp->imports, ip, from); typ = mgt_vcl_symtab_val(v1, "type"); if (!strcmp(typ, "$VMOD")) - mgt_vcl_import_vmod(vp, v1); + mgt_vcl_import_vmod(vp, ip, v1); else if (!strcmp(typ, "$VCL")) - mgt_vcl_import_vcl(vp, v1); + mgt_vcl_import_vcl(vp, ip, v1); else WRONG("Bad symtab import entry"); } } +void +mgt_vcl_symtab_clean(struct vclprog *vp) +{ + struct import *ip; + + if (vp->symtab) + vjsn_delete(&vp->symtab); + while (!VTAILQ_EMPTY(&vp->imports)) { + ip = VTAILQ_FIRST(&vp->imports); + VTAILQ_REMOVE(&vp->imports, ip, from); + if (ip->vmod) + VTAILQ_REMOVE(&ip->vmod->exports, ip, to); + if (ip->vcl) + VTAILQ_REMOVE(&ip->vcl->exports, ip, to); + FREE_OBJ(ip); + } +} /*--------------------------------------------------------------------*/ @@ -221,11 +251,22 @@ void v_matchproto_(cli_func_t) mcf_vcl_symtab(struct cli *cli, const char * const *av, void *priv) { struct vclprog *vp; + struct import *ip; (void)av; (void)priv; VTAILQ_FOREACH(vp, &vclhead, list) { VCLI_Out(cli, "VCL: %s\n", vp->name); - if (vp->symtab != NULL) - mcf_vcl_vjsn_dump(cli, vp->symtab->value, 4); + VCLI_Out(cli, " imports:\n"); + VTAILQ_FOREACH(ip, &vp->imports, from) { + VCLI_Out(cli, " %p -> (%p, %p)\n", + ip, ip->vcl, ip->vmod); + mcf_vcl_vjsn_dump(cli, ip->vj, 6); + } + VCLI_Out(cli, " exports:\n"); + VTAILQ_FOREACH(ip, &vp->exports, to) { + VCLI_Out(cli, " %p -> (%p, %p) %s\n", + ip, ip->vcl, ip->vmod, ip->target->name); + mcf_vcl_vjsn_dump(cli, ip->vj, 6); + } } } diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 147eaed90..892605630 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -43,7 +43,6 @@ #include "vcli_serve.h" #include "vct.h" #include "vev.h" -#include "vjsn.h" #include "vtim.h" #define VCL_STATE(sym, str) \ @@ -199,9 +198,11 @@ mgt_vcl_add(const char *name, const char *state) ALLOC_OBJ(vp, VCLPROG_MAGIC); XXXAN(vp); REPLACE(vp->name, name); + VTAILQ_INIT(&vp->exports); VTAILQ_INIT(&vp->dfrom); VTAILQ_INIT(&vp->dto); VTAILQ_INIT(&vp->vmods); + VTAILQ_INIT(&vp->imports); vp->state = state; if (vp->state != VCL_STATE_COLD) @@ -264,8 +265,7 @@ mgt_vcl_del(struct vclprog *vp) } } free(vp->name); - if (vp->symtab) - vjsn_delete(&vp->symtab); + mgt_vcl_symtab_clean(vp); FREE_OBJ(vp); } diff --git a/bin/varnishd/mgt/mgt_vcl.h b/bin/varnishd/mgt/mgt_vcl.h index fe740dde4..49adcdf51 100644 --- a/bin/varnishd/mgt/mgt_vcl.h +++ b/bin/varnishd/mgt/mgt_vcl.h @@ -30,6 +30,18 @@ struct vclprog; struct vmodfile; +struct vjsn_val; + +struct import { + unsigned magic; +#define IMPORT_MAGIC 0xce767c9b + struct vclprog *target; + VTAILQ_ENTRY(import) from; + VTAILQ_ENTRY(import) to; + struct vjsn_val *vj; + struct vmodfile *vmod; + struct vclprog *vcl; +}; struct vmoddep { unsigned magic; @@ -58,6 +70,8 @@ struct vclprog { const char * state; double go_cold; struct vjsn *symtab; + VTAILQ_HEAD(, import) imports; + VTAILQ_HEAD(, import) exports; VTAILQ_HEAD(, vcldep) dfrom; VTAILQ_HEAD(, vcldep) dto; int nto; @@ -69,6 +83,7 @@ struct vmodfile { unsigned magic; #define VMODFILE_MAGIC 0xffa1a0d5 char *fname; + VTAILQ_HEAD(, import) exports; VTAILQ_ENTRY(vmodfile) list; VTAILQ_HEAD(, vmoddep) vcls; }; @@ -80,6 +95,7 @@ struct vclprog *mcf_vcl_byname(const char *name); void mgt_vcl_dep_add(struct vclprog *vp_from, struct vclprog *vp_to); int mcf_is_label(const struct vclprog *vp); +void mgt_vcl_symtab_clean(struct vclprog *vp); void mgt_vcl_export_labels(struct vcc *vcc); void mgt_vcl_symtab(struct vclprog *vp, const char *input); void mcf_vcl_symtab(struct cli *cli, const char * const *av, void *priv); diff --git a/bin/varnishtest/tests/c00077.vtc b/bin/varnishtest/tests/c00077.vtc index 78b9002f9..060d577f9 100644 --- a/bin/varnishtest/tests/c00077.vtc +++ b/bin/varnishtest/tests/c00077.vtc @@ -51,8 +51,8 @@ client c1 { delay .2 -varnish v1 -vcl+backend { sub vcl_recv { return (vcl(vclA)); } } -varnish v1 -vcl+backend { sub vcl_recv { return (vcl(vclA)); } } +varnish v1 -vcl+backend { import std; sub vcl_recv { return (vcl(vclA)); } } +varnish v1 -vcl+backend { import debug; sub vcl_recv { return (vcl(vclA)); } } varnish v1 -vcl+backend { sub vcl_recv { return (vcl(vclA)); } } varnish v1 -vcl+backend { sub vcl_recv { return (vcl(vclA)); } } From dridi at varni.sh Mon May 27 08:41:46 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 27 May 2019 10:41:46 +0200 Subject: [master] d6d341609 Revert "actually, the scope must not change either" In-Reply-To: References: <20190524182211.9782F115DD1@lists.varnish-cache.org> <1a6a2219-bd04-fafe-00b7-d7700cca48b8@uplex.de>

Message-ID: On Mon, May 27, 2019 at 10:22 AM Nils Goroll wrote: > > > You should also thank Martin without whom I would have remained on the > > wrong track probably until today's bugwash. > > Thank you, Martin! And I apologize for my bug. > > >> I wonder if it wouldn't make more sense to give PRIV_TOP its own tree head > >> rather than an additional id to put into a shared tree? > > > > I have no opinion (but I understand the rationale) so I will leave > > this question to the rest of the team. > > Using a different head would use less memory, be more CPU efficient and also, I > think, result in simpler code. So a clear cut case in my opinion. Also less branches in general, trivial to implement etc... But I'll stay on my proverbial fence :) Dridi From dridi.boukelmoune at gmail.com Mon May 27 09:49:08 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 27 May 2019 09:49:08 +0000 (UTC) Subject: [master] 657e31f25 Easier regression tracking with vtc-bisect.sh Message-ID: <20190527094908.CD7FE114DF7@lists.varnish-cache.org> commit 657e31f25a66adc65565478fc95625119da6d867 Author: Dridi Boukelmoune Date: Mon May 27 10:34:22 2019 +0200 Easier regression tracking with vtc-bisect.sh With this script we can essentially use a VTC to track down a regression and see what broke after getting a cup of coffee, lunch, or whatever you wish to do while the script is running. It will search out of the box for a regression between the last tag and the current git head, and assumes the last tag passed the test case. It is otherwise the maintainer's responsibility to provide the good and/or bad commits beforehand. For more information: tools/vtc-bisect.sh -h I had this idea after setting up an automated bisect one time too many for #3003 and decided to generalize this in a handy script. For example: tools/vtc-bisect.sh -b d6d341609~ bin/varnishtest/tests/r03003.vtc The script works on a best-effort basis and tries to minimize rebuild time, or broken builds. It has minimal error handling and may leave you in a dirty git tree stuck in bisect mode as a result. I also noticed that 6.2.0 was not tagged in the master branch, so the example above starts with the varnish-6.1.0 tag. Tagging major releases (aka x.y.0 or dot-zero releases) from master is part of the release procedure and this facility should give one more reason to enforce the rule. Last time it happened before 6.2.0 was 5.2.0 so there's clearly something wrong with x.2.0 releases... This will of course not solve all bisect problems if for example a VTC relies on a feature that wasn't available in varnishtest when the regression was introduced and we still may need to perform manual git bisect operations but hopefully this should save us time from now on. diff --git a/.gitignore b/.gitignore index d24c7b41c..07e77ffa9 100644 --- a/.gitignore +++ b/.gitignore @@ -127,6 +127,9 @@ cscope.*out /lib/libvarnishapi/vsl_glob_test /lib/libvarnishapi/vsl_glob_test_coverage +# vtc-bisect.sh default vtc +/bisect.vtc + # vtest.sh droppings /tools/tmp/ /tools/_vtest_tmp/ diff --git a/tools/vtc-bisect.sh b/tools/vtc-bisect.sh new file mode 100755 index 000000000..050c3319c --- /dev/null +++ b/tools/vtc-bisect.sh @@ -0,0 +1,130 @@ +#!/bin/sh +# +# Copyright (c) 2019 Varnish Software AS +# All rights reserved. +# +# Author: Dridi Boukelmoune +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +set -e +set -u + +usage() { + test $# -eq 1 && + printf 'Error: %s.\n\n' "$1" + + cat <<-EOF + Usage: $SCRIPT [-b ] [-g ] [-j ] [] + $SCRIPT [-j ] -r + $SCRIPT -h + + Automatically look for the change that introduced a regression with + the help of a test case. + + Available options: + -h : show this help and exit + -b : bad revision (defaults to HEAD) + -g : good revision (defaults to latest tag before bad) + -j : number of jobs for make invocations (defaults to 8) + -r : git-bisect(1) run mode + + When is empty or missing, bisect.vtc is expected to be found + at the root of the git repository. The current source tree is used + and VPATH setups are not supported. + + This script is expected to run from the root of the git repository + as well. + EOF + exit $# +} + +build() { + make -s -j"$MAKE_JOBS" all || { + ./autogen.des + make -s -j"$MAKE_JOBS" all + } +} + +run() { + if build + then + # ignore unfortunate build-time modifications of srcdir + git checkout -- . + else + git checkout -- . + exit 125 + fi + + bin/varnishtest/varnishtest -i "$VTC_FILE" + exit $? +} + +BISECT_GOOD= +BISECT_BAD= +MAKE_JOBS= +RUN_MODE=false +VTC_FILE= + +while getopts b:g:hj:r OPT +do + case $OPT in + b) BISECT_BAD=$OPTARG ;; + g) BISECT_GOOD=$OPTARG ;; + h) usage ;; + j) MAKE_JOBS=$OPTARG ;; + r) RUN_MODE=true ;; + *) usage "wrong usage" >&2 ;; + esac +done + +shift $((OPTIND - 1)) + +test $# -gt 1 && usage "too many arguments" >&2 +test $# -eq 1 && VTC_FILE=$1 + +BISECT_BAD=${BISECT_BAD:-HEAD} +MAKE_JOBS=${MAKE_JOBS:-8} +VTC_FILE=${VTC_FILE:-bisect.vtc} + +[ -n "$BISECT_GOOD" ] || BISECT_GOOD=$(git describe --abbrev=0 "$BISECT_BAD") + +# run mode short circuit +"$RUN_MODE" && run + +# bisect mode +ROOT_DIR=$(git rev-parse --show-toplevel) + +readonly TMP_DIR=$(mktemp -d) + +trap 'rm -rf $TMP_DIR' EXIT + +cp "$ROOT_DIR"/tools/vtc-bisect.sh "$TMP_DIR" +cp "$VTC_FILE" "$TMP_DIR"/bisect.vtc + +cd "$ROOT_DIR" + +git bisect start +git bisect good "$BISECT_GOOD" +git bisect bad "$BISECT_BAD" +git bisect run "$TMP_DIR"/vtc-bisect.sh -r -j "$MAKE_JOBS" "$TMP_DIR"/bisect.vtc +git bisect reset From phk at FreeBSD.org Mon May 27 11:37:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 27 May 2019 11:37:08 +0000 (UTC) Subject: [master] a6ce0697c Avoid use after free. Message-ID: <20190527113708.D81F8116DFB@lists.varnish-cache.org> commit a6ce0697cd7be9a7fabb605af0a6a5dda7f8ce62 Author: Poul-Henning Kamp Date: Mon May 27 11:35:32 2019 +0000 Avoid use after free. Spotted by: fgs with asan (in the castle) diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 892605630..4347a3abd 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -247,6 +247,7 @@ mgt_vcl_del(struct vclprog *vp) VJ_master(JAIL_MASTER_LOW); free(vp->fname); } + mgt_vcl_symtab_clean(vp); while (!VTAILQ_EMPTY(&vp->vmods)) { vd = VTAILQ_FIRST(&vp->vmods); CHECK_OBJ(vd, VMODDEP_MAGIC); @@ -265,7 +266,6 @@ mgt_vcl_del(struct vclprog *vp) } } free(vp->name); - mgt_vcl_symtab_clean(vp); FREE_OBJ(vp); } From phk at FreeBSD.org Mon May 27 15:50:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 27 May 2019 15:50:09 +0000 (UTC) Subject: [master] 8fd310af0 Explain things for Coverity Message-ID: <20190527155009.5FB0A489A@lists.varnish-cache.org> commit 8fd310af0238a5ae3dbd1df417a2c6ac3931102e Author: Poul-Henning Kamp Date: Mon May 27 15:49:17 2019 +0000 Explain things for Coverity diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c index f95debe0d..f0e26e2f4 100644 --- a/lib/libvarnish/vsb.c +++ b/lib/libvarnish/vsb.c @@ -636,6 +636,7 @@ VSB_tofile(int fd, const struct vsb *s) assert_VSB_integrity(s); assert_VSB_state(s, VSB_FINISHED); - sz = write(fd, VSB_data(s), VSB_len(s)); - return (sz == VSB_len(s) ? 0 : -1); + assert(s->s_len >= 0); + sz = write(fd, s->s_buf, s->s_len); + return (sz == s->s_len ? 0 : -1); } diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c index 4313b3b05..10281eee6 100644 --- a/lib/libvarnish/vss.c +++ b/lib/libvarnish/vss.c @@ -103,7 +103,7 @@ vss_resolve(const char *addr, const char *def_port, int family, int socktype, int flags, struct addrinfo **res, const char **errp) { struct addrinfo hints; - char *p = NULL, *hp, *pp; + char *p, *hp, *pp; int ret; AN(addr); From phk at FreeBSD.org Mon May 27 20:47:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 27 May 2019 20:47:07 +0000 (UTC) Subject: [master] 6ef48bab3 Purge VCL list from behind to tear down dependencies in order. Message-ID: <20190527204707.F2F936307D@lists.varnish-cache.org> commit 6ef48bab3dd9be7e9836641e754bba36b1759508 Author: Poul-Henning Kamp Date: Mon May 27 20:46:20 2019 +0000 Purge VCL list from behind to tear down dependencies in order. diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 4347a3abd..1e9c6939d 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -222,8 +222,10 @@ mgt_vcl_del(struct vclprog *vp) struct vmodfile *vf; CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC); - while (!VTAILQ_EMPTY(&vp->dto)) - mgt_vcl_dep_del(VTAILQ_FIRST(&vp->dto)); + assert(VTAILQ_EMPTY(&vp->dto)); + + mgt_vcl_symtab_clean(vp); + while (!VTAILQ_EMPTY(&vp->dfrom)) mgt_vcl_dep_del(VTAILQ_FIRST(&vp->dfrom)); @@ -247,7 +249,6 @@ mgt_vcl_del(struct vclprog *vp) VJ_master(JAIL_MASTER_LOW); free(vp->fname); } - mgt_vcl_symtab_clean(vp); while (!VTAILQ_EMPTY(&vp->vmods)) { vd = VTAILQ_FIRST(&vp->vmods); CHECK_OBJ(vd, VMODDEP_MAGIC); @@ -915,16 +916,15 @@ static struct cli_proto cli_vcl[] = { static void mgt_vcl_atexit(void) { - struct vclprog *vp; + struct vclprog *vp, *vp2; if (getpid() != heritage.mgt_pid) return; active_vcl = NULL; - do { - vp = VTAILQ_FIRST(&vclhead); - if (vp != NULL) - mgt_vcl_del(vp); - } while (vp != NULL); + VTAILQ_FOREACH_REVERSE_SAFE(vp, &vclhead, vclproghead, list, vp2) { + assert(VTAILQ_EMPTY(&vp->dto)); + mgt_vcl_del(vp); + } } void From phk at FreeBSD.org Tue May 28 09:08:12 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 28 May 2019 09:08:12 +0000 (UTC) Subject: [master] f5982ebd6 Dont trust the vcl list to be tsorted. Message-ID: <20190528090812.0BA3F10C234@lists.varnish-cache.org> commit f5982ebd68aef144d52c0e7ad2b7513d91a2f95f Author: Poul-Henning Kamp Date: Tue May 28 06:43:35 2019 +0000 Dont trust the vcl list to be tsorted. diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 1e9c6939d..f78d1887f 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -921,10 +921,10 @@ mgt_vcl_atexit(void) if (getpid() != heritage.mgt_pid) return; active_vcl = NULL; - VTAILQ_FOREACH_REVERSE_SAFE(vp, &vclhead, vclproghead, list, vp2) { - assert(VTAILQ_EMPTY(&vp->dto)); - mgt_vcl_del(vp); - } + while (!VTAILQ_EMPTY(&vclhead)) + VTAILQ_FOREACH_SAFE(vp, &vclhead, list, vp2) + if (VTAILQ_EMPTY(&vp->dto)) + mgt_vcl_del(vp); } void From phk at FreeBSD.org Tue May 28 09:08:12 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 28 May 2019 09:08:12 +0000 (UTC) Subject: [master] a182f02d2 Drop the unified import structure, it doesnt make anything easier after all Message-ID: <20190528090812.1F88810C237@lists.varnish-cache.org> commit a182f02d23fa51d99fa14f531993ffa82a8d6f41 Author: Poul-Henning Kamp Date: Tue May 28 07:39:07 2019 +0000 Drop the unified import structure, it doesnt make anything easier after all diff --git a/bin/varnishd/mgt/mgt_symtab.c b/bin/varnishd/mgt/mgt_symtab.c index 4a5dc96e3..64c1bef4d 100644 --- a/bin/varnishd/mgt/mgt_symtab.c +++ b/bin/varnishd/mgt/mgt_symtab.c @@ -59,19 +59,16 @@ mgt_vcl_symtab_val(const struct vjsn_val *vv, const char *val) } static void -mgt_vcl_import_vcl(struct vclprog *vp1, struct import *ip, const struct vjsn_val *vv) +mgt_vcl_import_vcl(struct vclprog *vp1, const struct vjsn_val *vv) { struct vclprog *vp2; CHECK_OBJ_NOTNULL(vp1, VCLPROG_MAGIC); - AN(ip); AN(vv); vp2 = mcf_vcl_byname(mgt_vcl_symtab_val(vv, "name")); CHECK_OBJ_NOTNULL(vp2, VCLPROG_MAGIC); - ip->vcl = vp2; - VTAILQ_INSERT_TAIL(&vp2->exports, ip, to); - mgt_vcl_dep_add(vp1, vp2); + mgt_vcl_dep_add(vp1, vp2)->vj = vv; } static int @@ -117,7 +114,7 @@ mgt_vcl_cache_vmod(const char *nm, const char *fm, const char *to) } static void -mgt_vcl_import_vmod(struct vclprog *vp, struct import *ip, const struct vjsn_val *vv) +mgt_vcl_import_vmod(struct vclprog *vp, const struct vjsn_val *vv) { struct vmodfile *vf; struct vmoddep *vd; @@ -126,7 +123,6 @@ mgt_vcl_import_vmod(struct vclprog *vp, struct import *ip, const struct vjsn_val const char *v_dst; CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC); - AN(ip); AN(vv); v_name = mgt_vcl_symtab_val(vv, "name"); @@ -142,7 +138,6 @@ mgt_vcl_import_vmod(struct vclprog *vp, struct import *ip, const struct vjsn_val REPLACE(vf->fname, v_dst); AN(vf->fname); VTAILQ_INIT(&vf->vcls); - VTAILQ_INIT(&vf->exports); AZ(mgt_vcl_cache_vmod(v_name, v_file, v_dst)); VTAILQ_INSERT_TAIL(&vmodhead, vf, list); } @@ -151,8 +146,6 @@ mgt_vcl_import_vmod(struct vclprog *vp, struct import *ip, const struct vjsn_val vd->to = vf; VTAILQ_INSERT_TAIL(&vp->vmods, vd, lfrom); VTAILQ_INSERT_TAIL(&vf->vcls, vd, lto); - ip->vmod = vf; - VTAILQ_INSERT_TAIL(&vf->exports, ip, to); } void @@ -161,7 +154,6 @@ mgt_vcl_symtab(struct vclprog *vp, const char *input) struct vjsn *vj; struct vjsn_val *v1, *v2; const char *typ, *err; - struct import *ip; CHECK_OBJ_NOTNULL(vp, VCLPROG_MAGIC); AN(input); @@ -182,16 +174,11 @@ mgt_vcl_symtab(struct vclprog *vp, const char *input) assert(v2->type == VJSN_STRING); if (strcmp(v2->value, "import")) continue; - ALLOC_OBJ(ip, IMPORT_MAGIC); - AN(ip); - ip->vj = v1; - ip->target = vp; - VTAILQ_INSERT_TAIL(&vp->imports, ip, from); typ = mgt_vcl_symtab_val(v1, "type"); if (!strcmp(typ, "$VMOD")) - mgt_vcl_import_vmod(vp, ip, v1); + mgt_vcl_import_vmod(vp, v1); else if (!strcmp(typ, "$VCL")) - mgt_vcl_import_vcl(vp, ip, v1); + mgt_vcl_import_vcl(vp, v1); else WRONG("Bad symtab import entry"); } @@ -200,19 +187,9 @@ mgt_vcl_symtab(struct vclprog *vp, const char *input) void mgt_vcl_symtab_clean(struct vclprog *vp) { - struct import *ip; if (vp->symtab) vjsn_delete(&vp->symtab); - while (!VTAILQ_EMPTY(&vp->imports)) { - ip = VTAILQ_FIRST(&vp->imports); - VTAILQ_REMOVE(&vp->imports, ip, from); - if (ip->vmod) - VTAILQ_REMOVE(&ip->vmod->exports, ip, to); - if (ip->vcl) - VTAILQ_REMOVE(&ip->vcl->exports, ip, to); - FREE_OBJ(ip); - } } /*--------------------------------------------------------------------*/ @@ -251,22 +228,23 @@ void v_matchproto_(cli_func_t) mcf_vcl_symtab(struct cli *cli, const char * const *av, void *priv) { struct vclprog *vp; - struct import *ip; + struct vcldep *vd; + (void)av; (void)priv; VTAILQ_FOREACH(vp, &vclhead, list) { VCLI_Out(cli, "VCL: %s\n", vp->name); - VCLI_Out(cli, " imports:\n"); - VTAILQ_FOREACH(ip, &vp->imports, from) { - VCLI_Out(cli, " %p -> (%p, %p)\n", - ip, ip->vcl, ip->vmod); - mcf_vcl_vjsn_dump(cli, ip->vj, 6); + VCLI_Out(cli, " imports from:\n"); + VTAILQ_FOREACH(vd, &vp->dto, lto) { + VCLI_Out(cli, " %s\n", vd->from->name); + if (vd->vj) + mcf_vcl_vjsn_dump(cli, vd->vj, 6); } - VCLI_Out(cli, " exports:\n"); - VTAILQ_FOREACH(ip, &vp->exports, to) { - VCLI_Out(cli, " %p -> (%p, %p) %s\n", - ip, ip->vcl, ip->vmod, ip->target->name); - mcf_vcl_vjsn_dump(cli, ip->vj, 6); + VCLI_Out(cli, " exports to:\n"); + VTAILQ_FOREACH(vd, &vp->dfrom, lfrom) { + VCLI_Out(cli, " %s\n", vd->to->name); + if (vd->vj) + mcf_vcl_vjsn_dump(cli, vd->vj, 6); } } } diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index f78d1887f..47b223496 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -149,7 +149,7 @@ mcf_is_label(const struct vclprog *vp) /*--------------------------------------------------------------------*/ -void +struct vcldep * mgt_vcl_dep_add(struct vclprog *vp_from, struct vclprog *vp_to) { struct vcldep *vd; @@ -169,6 +169,7 @@ mgt_vcl_dep_add(struct vclprog *vp_from, struct vclprog *vp_to) vd->to = vp_to; VTAILQ_INSERT_TAIL(&vp_to->dto, vd, lto); vp_to->nto++; + return (vd); } static void @@ -198,11 +199,9 @@ mgt_vcl_add(const char *name, const char *state) ALLOC_OBJ(vp, VCLPROG_MAGIC); XXXAN(vp); REPLACE(vp->name, name); - VTAILQ_INIT(&vp->exports); VTAILQ_INIT(&vp->dfrom); VTAILQ_INIT(&vp->dto); VTAILQ_INIT(&vp->vmods); - VTAILQ_INIT(&vp->imports); vp->state = state; if (vp->state != VCL_STATE_COLD) @@ -862,7 +861,7 @@ mcf_vcl_label(struct cli *cli, const char * const *av, void *priv) AN(vpl); if (mgt_vcl_requirewarm(cli, vpl)) return; - mgt_vcl_dep_add(vpl, vpt); + (void)mgt_vcl_dep_add(vpl, vpt); if (!MCH_Running()) return; diff --git a/bin/varnishd/mgt/mgt_vcl.h b/bin/varnishd/mgt/mgt_vcl.h index 49adcdf51..1c25c656e 100644 --- a/bin/varnishd/mgt/mgt_vcl.h +++ b/bin/varnishd/mgt/mgt_vcl.h @@ -32,17 +32,6 @@ struct vclprog; struct vmodfile; struct vjsn_val; -struct import { - unsigned magic; -#define IMPORT_MAGIC 0xce767c9b - struct vclprog *target; - VTAILQ_ENTRY(import) from; - VTAILQ_ENTRY(import) to; - struct vjsn_val *vj; - struct vmodfile *vmod; - struct vclprog *vcl; -}; - struct vmoddep { unsigned magic; #define VMODDEP_MAGIC 0xc1490542 @@ -58,6 +47,7 @@ struct vcldep { VTAILQ_ENTRY(vcldep) lfrom; struct vclprog *to; VTAILQ_ENTRY(vcldep) lto; + const struct vjsn_val *vj; }; struct vclprog { @@ -70,8 +60,6 @@ struct vclprog { const char * state; double go_cold; struct vjsn *symtab; - VTAILQ_HEAD(, import) imports; - VTAILQ_HEAD(, import) exports; VTAILQ_HEAD(, vcldep) dfrom; VTAILQ_HEAD(, vcldep) dto; int nto; @@ -83,7 +71,6 @@ struct vmodfile { unsigned magic; #define VMODFILE_MAGIC 0xffa1a0d5 char *fname; - VTAILQ_HEAD(, import) exports; VTAILQ_ENTRY(vmodfile) list; VTAILQ_HEAD(, vmoddep) vcls; }; @@ -92,7 +79,7 @@ extern VTAILQ_HEAD(vclproghead, vclprog) vclhead; extern VTAILQ_HEAD(vmodfilehead, vmodfile) vmodhead; struct vclprog *mcf_vcl_byname(const char *name); -void mgt_vcl_dep_add(struct vclprog *vp_from, struct vclprog *vp_to); +struct vcldep *mgt_vcl_dep_add(struct vclprog *vp_from, struct vclprog *vp_to); int mcf_is_label(const struct vclprog *vp); void mgt_vcl_symtab_clean(struct vclprog *vp); From phk at FreeBSD.org Tue May 28 09:08:12 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 28 May 2019 09:08:12 +0000 (UTC) Subject: [master] 1d52b52a5 Implement a real DAG-loop detector for allowing VCL labels Message-ID: <20190528090812.4566C10C23A@lists.varnish-cache.org> commit 1d52b52a5cc85134ad90ec60daa52cd779b2f026 Author: Poul-Henning Kamp Date: Tue May 28 09:02:47 2019 +0000 Implement a real DAG-loop detector for allowing VCL labels diff --git a/bin/varnishd/mgt/mgt_symtab.c b/bin/varnishd/mgt/mgt_symtab.c index 64c1bef4d..e54274c8c 100644 --- a/bin/varnishd/mgt/mgt_symtab.c +++ b/bin/varnishd/mgt/mgt_symtab.c @@ -235,14 +235,14 @@ mcf_vcl_symtab(struct cli *cli, const char * const *av, void *priv) VTAILQ_FOREACH(vp, &vclhead, list) { VCLI_Out(cli, "VCL: %s\n", vp->name); VCLI_Out(cli, " imports from:\n"); - VTAILQ_FOREACH(vd, &vp->dto, lto) { - VCLI_Out(cli, " %s\n", vd->from->name); + VTAILQ_FOREACH(vd, &vp->dfrom, lfrom) { + VCLI_Out(cli, " %s\n", vd->to->name); if (vd->vj) mcf_vcl_vjsn_dump(cli, vd->vj, 6); } VCLI_Out(cli, " exports to:\n"); - VTAILQ_FOREACH(vd, &vp->dfrom, lfrom) { - VCLI_Out(cli, " %s\n", vd->to->name); + VTAILQ_FOREACH(vd, &vp->dto, lto) { + VCLI_Out(cli, " %s\n", vd->from->name); if (vd->vj) mcf_vcl_vjsn_dump(cli, vd->vj, 6); } diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 47b223496..80404fee6 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -802,6 +802,27 @@ mcf_vcl_list_json(struct cli *cli, const char * const *av, void *priv) } } +static int +mcf_vcl_check_dag(struct cli *cli, struct vclprog *tree, struct vclprog *target) +{ + struct vcldep *vd; + + if (target == tree) + return (1); + VTAILQ_FOREACH(vd, &tree->dfrom, lfrom) { + if (!mcf_vcl_check_dag(cli, vd->to, target)) + continue; + if (mcf_is_label(tree)) + VCLI_Out(cli, "Label %s points to vcl %s\n", + tree->name, vd->to->name); + else + VCLI_Out(cli, "Vcl %s uses Label %s\n", + tree->name, vd->to->name); + return (1); + } + return(0); +} + static void v_matchproto_(cli_func_t) mcf_vcl_label(struct cli *cli, const char * const *av, void *priv) { @@ -814,8 +835,12 @@ mcf_vcl_label(struct cli *cli, const char * const *av, void *priv) (void)priv; if (mcf_invalid_vclname(cli, av[2])) return; - if (mcf_invalid_vclname(cli, av[3])) + vpl = mcf_vcl_byname(av[2]); + if (vpl != NULL && !mcf_is_label(vpl)) { + VCLI_SetResult(cli, CLIS_PARAM); + VCLI_Out(cli, "%s is not a label", vpl->name); return; + } vpt = mcf_find_vcl(cli, av[3]); if (vpt == NULL) return; @@ -824,23 +849,13 @@ mcf_vcl_label(struct cli *cli, const char * const *av, void *priv) VCLI_Out(cli, "VCL labels cannot point to labels"); return; } - vpl = mcf_vcl_byname(av[2]); if (vpl != NULL) { - if (!mcf_is_label(vpl)) { - VCLI_SetResult(cli, CLIS_PARAM); - VCLI_Out(cli, "%s is not a label", vpl->name); - return; - } - if (!VTAILQ_EMPTY(&vpt->dfrom) && - !VTAILQ_EMPTY(&vpl->dto)) { - VCLI_SetResult(cli, CLIS_PARAM); - VCLI_Out(cli, "return(vcl) can only be used from" - " the active VCL.\n\n"); + if (VTAILQ_FIRST(&vpl->dfrom)->to != vpt && + mcf_vcl_check_dag(cli, vpt, vpl)) { VCLI_Out(cli, - "Label %s is used in return(vcl) from VCL %s\n", - vpl->name, VTAILQ_FIRST(&vpl->dto)->from->name); - VCLI_Out(cli, "and VCL %s also has return(vcl)", - vpt->name); + "Pointing label %s at %s would create a loop", + vpl->name, vpt->name); + VCLI_SetResult(cli, CLIS_PARAM); return; } } diff --git a/bin/varnishtest/tests/c00077.vtc b/bin/varnishtest/tests/c00077.vtc index 060d577f9..c25d9e4c5 100644 --- a/bin/varnishtest/tests/c00077.vtc +++ b/bin/varnishtest/tests/c00077.vtc @@ -60,6 +60,8 @@ varnish v1 -clierr 300 "vcl.discard vclA" varnish v1 -vcl+backend { } +varnish v1 -clierr 106 "vcl.label vclA vcl3" + varnish v1 -cliok "vcl.symtab" varnish v1 -cliok "vcl.discard vcl3" diff --git a/bin/varnishtest/tests/s00005.vtc b/bin/varnishtest/tests/s00005.vtc index 01c2def7e..0fa00990f 100644 --- a/bin/varnishtest/tests/s00005.vtc +++ b/bin/varnishtest/tests/s00005.vtc @@ -127,6 +127,6 @@ varnish v1 -vcl+backend { sub vcl_recv { return (vcl(lblB)); } } varnish v1 -clierr 106 "vcl.label lblA vcl5" varnish v1 -cliexpect \ - {can only be used from the active VCL} \ + {would create a loop} \ {vcl.label lblA vcl5} From phk at FreeBSD.org Wed May 29 09:54:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 29 May 2019 09:54:10 +0000 (UTC) Subject: [master] a29d1e8c0 Shuffle things around Message-ID: <20190529095410.E46731041CD@lists.varnish-cache.org> commit a29d1e8c0ced3b9c714e648d82cd1e404f039268 Author: Poul-Henning Kamp Date: Tue May 28 11:40:18 2019 +0000 Shuffle things around diff --git a/bin/varnishd/cache/cache_vpi.c b/bin/varnishd/cache/cache_vpi.c index ec35ad2c7..17ac4d1dc 100644 --- a/bin/varnishd/cache/cache_vpi.c +++ b/bin/varnishd/cache/cache_vpi.c @@ -92,7 +92,7 @@ VPI_vcl_select(VRT_CTX, VCL_VCL vcl) CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); if (IS_TOPREQ(req) && req->vcl0 != NULL) - return; // Illega, req-FSM will fail this later. + return; // Illegal, req-FSM will fail this later. VCL_TaskLeave(req->vcl, req->privs); if (IS_TOPREQ(req)) { diff --git a/bin/varnishd/mgt/mgt_symtab.c b/bin/varnishd/mgt/mgt_symtab.c index e54274c8c..be80ec593 100644 --- a/bin/varnishd/mgt/mgt_symtab.c +++ b/bin/varnishd/mgt/mgt_symtab.c @@ -194,18 +194,6 @@ mgt_vcl_symtab_clean(struct vclprog *vp) /*--------------------------------------------------------------------*/ -void -mgt_vcl_export_labels(struct vcc *vcc) -{ - struct vclprog *vp; - VTAILQ_FOREACH(vp, &vclhead, list) { - if (mcf_is_label(vp)) - VCC_Predef(vcc, "VCL_VCL", vp->name); - } -} - -/*--------------------------------------------------------------------*/ - static void mcf_vcl_vjsn_dump(struct cli *cli, const struct vjsn_val *vj, int indent) { @@ -233,18 +221,29 @@ mcf_vcl_symtab(struct cli *cli, const char * const *av, void *priv) (void)av; (void)priv; VTAILQ_FOREACH(vp, &vclhead, list) { - VCLI_Out(cli, "VCL: %s\n", vp->name); - VCLI_Out(cli, " imports from:\n"); - VTAILQ_FOREACH(vd, &vp->dfrom, lfrom) { - VCLI_Out(cli, " %s\n", vd->to->name); - if (vd->vj) - mcf_vcl_vjsn_dump(cli, vd->vj, 6); + if (mcf_is_label(vp)) + VCLI_Out(cli, "Label: %s\n", vp->name); + else + VCLI_Out(cli, "Vcl: %s\n", vp->name); + if (!VTAILQ_EMPTY(&vp->dfrom)) { + VCLI_Out(cli, " imports from:\n"); + VTAILQ_FOREACH(vd, &vp->dfrom, lfrom) { + VCLI_Out(cli, " %s\n", vd->to->name); + if (vd->vj) + mcf_vcl_vjsn_dump(cli, vd->vj, 6); + } + } + if (!VTAILQ_EMPTY(&vp->dto)) { + VCLI_Out(cli, " exports to:\n"); + VTAILQ_FOREACH(vd, &vp->dto, lto) { + VCLI_Out(cli, " %s\n", vd->from->name); + if (vd->vj) + mcf_vcl_vjsn_dump(cli, vd->vj, 6); + } } - VCLI_Out(cli, " exports to:\n"); - VTAILQ_FOREACH(vd, &vp->dto, lto) { - VCLI_Out(cli, " %s\n", vd->from->name); - if (vd->vj) - mcf_vcl_vjsn_dump(cli, vd->vj, 6); + if (vp->symtab != NULL) { + VCLI_Out(cli, " symtab:\n"); + mcf_vcl_vjsn_dump(cli, vp->symtab->value, 4); } } } diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index 5316ab299..58b4ec152 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -88,6 +88,7 @@ static void v_matchproto_(vsub_func_f) run_vcc(void *priv) { struct vsb *sb = NULL; + struct vclprog *vpg; struct vcc_priv *vp; struct vcc *vcc; struct stevedore *stv; @@ -108,7 +109,9 @@ run_vcc(void *priv) VCC_Unsafe_Path(vcc, mgt_vcc_unsafe_path); STV_Foreach(stv) VCC_Predef(vcc, "VCL_STEVEDORE", stv->ident); - mgt_vcl_export_labels(vcc); + VTAILQ_FOREACH(vpg, &vclhead, list) + if (mcf_is_label(vpg)) + VCC_Predef(vcc, "VCL_VCL", vpg->name); i = VCC_Compile(vcc, &sb, vp->vclsrc, vp->vclsrcfile, VGC_SRC, VGC_SYM); if (VSB_len(sb)) diff --git a/bin/varnishd/mgt/mgt_vcl.h b/bin/varnishd/mgt/mgt_vcl.h index 1c25c656e..e6a32fc8d 100644 --- a/bin/varnishd/mgt/mgt_vcl.h +++ b/bin/varnishd/mgt/mgt_vcl.h @@ -83,7 +83,6 @@ struct vcldep *mgt_vcl_dep_add(struct vclprog *vp_from, struct vclprog *vp_to); int mcf_is_label(const struct vclprog *vp); void mgt_vcl_symtab_clean(struct vclprog *vp); -void mgt_vcl_export_labels(struct vcc *vcc); void mgt_vcl_symtab(struct vclprog *vp, const char *input); void mcf_vcl_symtab(struct cli *cli, const char * const *av, void *priv); From phk at FreeBSD.org Wed May 29 09:54:11 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 29 May 2019 09:54:11 +0000 (UTC) Subject: [master] fd4d2e0b2 Provide a function to get a symbols full name Message-ID: <20190529095411.0A02A1041D0@lists.varnish-cache.org> commit fd4d2e0b25a19e3333cec23391c3aa045076432b Author: Poul-Henning Kamp Date: Wed May 29 07:18:28 2019 +0000 Provide a function to get a symbols full name diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index aa4496316..156ea816e 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -322,6 +322,7 @@ void vcc_Eval_Func(struct vcc *, const struct vjsn_val *, const char *, const struct symbol *); void VCC_GlobalSymbol(struct symbol *, vcc_type_t fmt, const char *pfx); struct symbol *VCC_HandleSymbol(struct vcc *, vcc_type_t , const char *); +void VCC_SymName(struct vsb *, const struct symbol *); /* vcc_obj.c */ void vcc_Var_Init(struct vcc *); diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index 25a2573cb..032f9e97a 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -72,6 +72,16 @@ VCC_PrintCName(struct vsb *vsb, const char *b, const char *e) VSB_printf(vsb, "_%02x_", *b); } +void +VCC_SymName(struct vsb *vsb, const struct symbol *sym) +{ + if (sym->parent != NULL && sym->parent->parent != NULL) { + VCC_SymName(vsb, sym->parent); + VSB_putc(vsb, '.'); + } + VSB_cat(vsb, sym->name); +} + static struct symbol * vcc_new_symbol(struct vcc *tl, const char *b, const char *e) { diff --git a/lib/libvcc/vcc_xref.c b/lib/libvcc/vcc_xref.c index bfa98a6f6..dd0893c23 100644 --- a/lib/libvcc/vcc_xref.c +++ b/lib/libvcc/vcc_xref.c @@ -326,17 +326,6 @@ vcc_CheckUses(struct vcc *tl) /*---------------------------------------------------------------------*/ -static void -vcc_pnam(struct vcc *tl, const struct symbol *sym) -{ - - if (sym->parent != tl->symbols) { - vcc_pnam(tl, sym->parent); - Fc(tl, 0, "."); - } - Fc(tl, 0, "%s", sym->name); -} - static void v_matchproto_(symwalk_f) vcc_xreftable(struct vcc *tl, const struct symbol *sym) { @@ -347,7 +336,7 @@ vcc_xreftable(struct vcc *tl, const struct symbol *sym) Fc(tl, 0, " * %-8s ", sym->kind->name); Fc(tl, 0, " %-9s ", sym->type->name); Fc(tl, 0, " %2u %2u ", sym->lorev, sym->hirev); - vcc_pnam(tl, sym); + VCC_SymName(tl->fc, sym); if (sym->wildcard != NULL) Fc(tl, 0, "*"); Fc(tl, 0, "\n"); From phk at FreeBSD.org Wed May 29 09:54:11 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 29 May 2019 09:54:11 +0000 (UTC) Subject: [master] 4a3e03d98 Get rid of a bothersome trailing '.' Message-ID: <20190529095411.22E631041D4@lists.varnish-cache.org> commit 4a3e03d98d16944733d14cdf668cd4ae09bf7fd8 Author: Poul-Henning Kamp Date: Wed May 29 08:01:45 2019 +0000 Get rid of a bothersome trailing '.' diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 7e7723ed4..892dfff5b 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -249,8 +249,8 @@ def parse_var(ln): l1 = ln.pop(0).split("``") assert len(l1) in (1, 3) vn = l1[0].strip() - if vn[-1] == '*': - vn = vn[:-1] + if vn[-2:] == '.*': + vn = vn[:-2] if len(l1) == 3: vlo, vhi = parse_vcl(l1[1]) else: From phk at FreeBSD.org Wed May 29 09:54:11 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 29 May 2019 09:54:11 +0000 (UTC) Subject: [master] bc3b93bcc Split the symbol-table into a naming and a content layer. Message-ID: <20190529095411.40BAB1041DD@lists.varnish-cache.org> commit bc3b93bcca6f5a573eb1344d8c374124b0e8aecb Author: Poul-Henning Kamp Date: Wed May 29 09:53:03 2019 +0000 Split the symbol-table into a naming and a content layer. diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 156ea816e..c9ea92d5d 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -58,6 +58,7 @@ struct vsb; struct token; struct sockaddr_storage; struct method; +struct symtab; unsigned vcl_fixed_token(const char *p, const char **q); extern const char * const vcl_tnames[256]; @@ -135,15 +136,13 @@ struct symbol { #define SYMBOL_MAGIC 0x3368c9fb VTAILQ_ENTRY(symbol) list; VTAILQ_ENTRY(symbol) sideways; - VTAILQ_HEAD(,symbol) children; - char *name; - unsigned nlen; + const char *name; int lorev; int hirev; - struct symbol *parent; + const struct symtab *symtab; const char *vmod_name; sym_wildcard_t *wildcard; @@ -222,7 +221,7 @@ struct vcc { unsigned allow_inline_c; unsigned unsafe_path; - struct symbol *symbols; + struct symtab *syms; struct inifinhead inifin; unsigned ninifin; diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index 032f9e97a..134599e6b 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -42,6 +42,17 @@ #include "tbl/symbol_kind.h" /*--------------------------------------------------------------------*/ +struct symtab { + unsigned magic; +#define SYMTAB_MAGIC 0x084d9c8a + unsigned nlen; + char *name; + struct symtab *parent; + VTAILQ_ENTRY(symtab) list; + VTAILQ_HEAD(,symtab) children; + VTAILQ_HEAD(,symbol) symbols; +}; + static vcc_kind_t VCC_HandleKind(vcc_type_t fmt) { @@ -72,88 +83,140 @@ VCC_PrintCName(struct vsb *vsb, const char *b, const char *e) VSB_printf(vsb, "_%02x_", *b); } -void -VCC_SymName(struct vsb *vsb, const struct symbol *sym) +static void +vcc_symtabname(struct vsb *vsb, const struct symtab *st) { - if (sym->parent != NULL && sym->parent->parent != NULL) { - VCC_SymName(vsb, sym->parent); + if (st->parent != NULL && st->parent->parent != NULL) { + vcc_symtabname(vsb, st->parent); VSB_putc(vsb, '.'); } - VSB_cat(vsb, sym->name); + VSB_cat(vsb, st->name); } -static struct symbol * -vcc_new_symbol(struct vcc *tl, const char *b, const char *e) +void +VCC_SymName(struct vsb *vsb, const struct symbol *sym) { - struct symbol *sym; + AN(vsb); + CHECK_OBJ_NOTNULL(sym, SYMBOL_MAGIC); + CHECK_OBJ_NOTNULL(sym->symtab, SYMTAB_MAGIC); + vcc_symtabname(vsb, sym->symtab); +} + +static char * +vcc_dup_be(const char *b, const char *e) +{ + char *p; AN(b); if (e == NULL) e = strchr(b, '\0'); AN(e); - assert(e > b); + assert(e >= b); + + p = malloc((e - b) + 1); + AN(p); + memcpy(p, b, e - b); + p[e - b] = '\0'; + return (p); +} + +static struct symtab * +vcc_symtab_new(const char *b, const char *e) +{ + struct symtab *st; + + ALLOC_OBJ(st, SYMTAB_MAGIC); + AN(st); + st->name = vcc_dup_be(b, e); + st->nlen = strlen(st->name); + VTAILQ_INIT(&st->children); + VTAILQ_INIT(&st->symbols); + return (st); +} + +static const char * const rootname = ""; + +static struct symtab * +vcc_symtab_str(struct vcc *tl, const char *b, const char *e) +{ + struct symtab *st1, *st2, *st3; + size_t l; + int i; + char *p, *q; + + AN(tl); + p = vcc_dup_be(b, e); + + if (tl->syms == NULL) + tl->syms = vcc_symtab_new(rootname, rootname); + st1 = tl->syms; + + while (1) { + q = strchr(p, '.'); + if (q == NULL) + q = strchr(p, '\0'); + else + assert(q[1] != '.' && q[1] != '\0'); + AN(q); + l = q - p; + VTAILQ_FOREACH(st2, &st1->children, list) { + i = strncasecmp(st2->name, p, l); + if (i < 0) + continue; + if (i == 0 && l == st2->nlen) + break; + st3 = vcc_symtab_new(p, q); + st3->parent = st1; + VTAILQ_INSERT_BEFORE(st2, st3, list); + st2 = st3; + break; + } + if (st2 == NULL) { + st2 = vcc_symtab_new(p, q); + st2->parent = st1; + VTAILQ_INSERT_TAIL(&st1->children, st2, list); + } + if (*q == '\0') + return (st2); + st1 = st2; + p = q + 1; + } +} + +static struct symbol * +vcc_new_symbol(struct vcc *tl, struct symtab *st) +{ + struct symbol *sym; + sym = TlAlloc(tl, sizeof *sym); INIT_OBJ(sym, SYMBOL_MAGIC); AN(sym); - sym->name = TlAlloc(tl, (e - b) + 1L); - AN(sym->name); - memcpy(sym->name, b, (e - b)); - sym->name[e - b] = '\0'; - sym->nlen = e - b; - VTAILQ_INIT(&sym->children); + sym->name = st->name; + sym->symtab = st; sym->kind = SYM_NONE; sym->type = VOID; sym->lorev = VCL_LOW; sym->hirev = VCL_HIGH; + VTAILQ_INSERT_TAIL(&st->symbols, sym, list); return (sym); } static struct symbol * -VCC_Symbol(struct vcc *tl, struct symbol *parent, +VCC_Symbol(struct vcc *tl, const char *b, const char *e, vcc_kind_t kind, int create, int vlo, int vhi) { - const char *q; - struct symbol *sym, *sym2 = NULL; - size_t l; - int i; + struct symtab *st, *pst; + struct symbol *sym, *psym; assert(vlo <= vhi); - if (tl->symbols == NULL) - tl->symbols = vcc_new_symbol(tl, "", NULL); - if (parent == NULL) - parent = tl->symbols; - AN(b); - assert(e == NULL || b < e); - if (e == NULL) - e = strchr(b, '\0'); - assert(e > b); - if (e[-1] == '.') - e--; - assert(e > b); - - q = strchr(b, '.'); - if (q == NULL || q > e) - q = e; - l = q - b; - assert(l > 0); - - VTAILQ_FOREACH(sym, &parent->children, list) { - i = strncasecmp(sym->name, b, l); - if (i < 0) - continue; - if (i > 0 || l < sym->nlen) { - sym2 = sym; - sym = NULL; - break; - } - if (l > sym->nlen) - continue; + st = vcc_symtab_str(tl, b, e); + AN(st); + + VTAILQ_FOREACH(sym, &st->symbols, list) { if (sym->lorev > vhi || sym->hirev < vlo) continue; - if (q < e) - break; if ((kind == SYM_NONE && kind == sym->kind)) continue; if (tl->syntax < VCL_41 && strcmp(sym->name, "default") && @@ -161,44 +224,30 @@ VCC_Symbol(struct vcc *tl, struct symbol *parent, continue; break; } - if (sym == NULL && create == 0 && parent->wildcard != NULL) { - AN(parent->wildcard); - sym2 = vcc_new_symbol(tl, b, e); - sym2->parent = parent; - parent->wildcard(tl, parent, sym2); - if (tl->err) - return (NULL); - VTAILQ_FOREACH(sym, &parent->children, list) { - i = strncasecmp(sym->name, b, l); - if (i > 0 || (i == 0 && l < sym->nlen)) - break; - } - sym2->lorev = vlo; - sym2->hirev = vhi; - if (sym == NULL) - VTAILQ_INSERT_TAIL(&parent->children, sym2, list); - else - VTAILQ_INSERT_BEFORE(sym, sym2, list); - return (VCC_Symbol(tl, parent, b, e, kind, -1, vlo, vhi)); - } - if (sym == NULL && create < 1) + if (sym != NULL) return (sym); - if (sym == NULL) { - sym = vcc_new_symbol(tl, b, q); - sym->parent = parent; + if (create) { + sym = vcc_new_symbol(tl, st); sym->lorev = vlo; sym->hirev = vhi; - if (sym2 != NULL) - VTAILQ_INSERT_BEFORE(sym2, sym, list); - else - VTAILQ_INSERT_TAIL(&parent->children, sym, list); - if (q == e) - sym->kind = kind; + sym->kind = kind; } - if (q == e) - return (sym); - assert(*q == '.'); - return (VCC_Symbol(tl, sym, ++q, e, kind, create, vlo, vhi)); + pst = st->parent; + if (pst == NULL) + return(sym); + psym = VTAILQ_FIRST(&pst->symbols); + if (psym == NULL) + return(sym); + if (psym->wildcard != NULL) { + sym = vcc_new_symbol(tl, st); + sym->lorev = vlo; + sym->hirev = vhi; + sym->kind = kind; + psym->wildcard(tl, psym, sym); + if (tl->err) + return(NULL); + } + return(sym); } const char XREF_NONE[] = "xref_none"; @@ -225,14 +274,14 @@ VCC_SymbolGetTok(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x, return (NULL); } - sym = VCC_Symbol(tl, NULL, t->b, t->e, kind, + sym = VCC_Symbol(tl, t->b, t->e, kind, e == SYMTAB_CREATE ? 1 : 0, tl->syntax, tl->syntax); if (sym == NULL && e == SYMTAB_NOERR) return (sym); if (sym == NULL) { VSB_printf(tl->sb, "%s: ", e); vcc_ErrToken(tl, t); - sym = VCC_Symbol(tl, NULL, t->b, t->e, kind, 0, + sym = VCC_Symbol(tl, t->b, t->e, kind, 0, VCL_LOW, VCL_HIGH); if (sym != NULL) { VSB_printf(tl->sb, " (Only available when"); @@ -283,6 +332,7 @@ struct symbol * VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x) { struct symbol *sym; + sym = VCC_SymbolGetTok(tl, kind, e, x, tl->t); if (sym != NULL) vcc_NextToken(tl); @@ -294,26 +344,34 @@ VCC_MkSym(struct vcc *tl, const char *b, vcc_kind_t kind, int vlo, int vhi) { struct symbol *sym; - sym = VCC_Symbol(tl, NULL, b, NULL, kind, 1, vlo, vhi); + AN(tl); + AN(b); + CHECK_OBJ_NOTNULL(kind, KIND_MAGIC); + + sym = VCC_Symbol(tl, b, NULL, kind, 1, vlo, vhi); + AN(sym); sym->noref = 1; return (sym); } static void -vcc_walksymbols(struct vcc *tl, const struct symbol *root, +vcc_walksymbols(struct vcc *tl, const struct symtab *root, symwalk_f *func, vcc_kind_t kind) { - struct symbol *sym, *sym2 = NULL; + struct symbol *sym; + struct symtab *st1, *st2 = NULL; - VTAILQ_FOREACH(sym, &root->children, list) { - if (sym2 != NULL) - assert(strcasecmp(sym->name, sym2->name) >= 0); - sym2 = sym; + VTAILQ_FOREACH(sym, &root->symbols, list) { if (kind == SYM_NONE || kind == sym->kind) func(tl, sym); ERRCHK(tl); - vcc_walksymbols(tl, sym, func, kind); + } + VTAILQ_FOREACH(st1, &root->children, list) { + if (st2 != NULL) + assert(strcasecmp(st1->name, st2->name) >= 0); + st2 = st1; + vcc_walksymbols(tl, st1, func, kind); } } @@ -321,7 +379,7 @@ void VCC_WalkSymbols(struct vcc *tl, symwalk_f *func, vcc_kind_t kind) { - vcc_walksymbols(tl, tl->symbols, func, kind); + vcc_walksymbols(tl, tl->syms, func, kind); } void diff --git a/lib/libvcc/vcc_var.c b/lib/libvcc/vcc_var.c index 273563c23..72931c6a0 100644 --- a/lib/libvcc/vcc_var.c +++ b/lib/libvcc/vcc_var.c @@ -43,7 +43,7 @@ vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent, struct symbol *sym) assert(parent->type == HEADER); - if (sym->nlen >= 127) { + if (strlen(sym->name) >= 127) { VSB_printf(tl->sb, "HTTP header (%.20s..) is too long.\n", sym->name); VSB_cat(tl->sb, "\nAt: "); @@ -69,7 +69,7 @@ vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent, struct symbol *sym) /* Create the static identifier */ Fh(tl, 0, "static const struct gethdr_s %s =\n", VSB_data(vsb) + 1); Fh(tl, 0, " { %s, \"\\%03o%s:\"};\n", - parent->rname, sym->nlen + 1, sym->name); + parent->rname, (unsigned int)strlen(sym->name) + 1, sym->name); /* Create the symbol r/l values */ sym->rname = TlDup(tl, VSB_data(vsb)); From phk at FreeBSD.org Wed May 29 14:38:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 29 May 2019 14:38:09 +0000 (UTC) Subject: [master] b08fa3bb0 Abstract wildcard'ing one level up Message-ID: <20190529143809.5BC7C10A9FF@lists.varnish-cache.org> commit b08fa3bb010c8dc9384f9c342b83806c64c82c06 Author: Poul-Henning Kamp Date: Wed May 29 10:55:49 2019 +0000 Abstract wildcard'ing one level up diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index 134599e6b..fb64b26cb 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -37,6 +37,8 @@ #include "vct.h" +static const char * const rootname = ""; + /*--------------------------------------------------------------------*/ #define VCC_KIND(U,l) const struct kind SYM_##U[1] = {{ KIND_MAGIC, #l}}; #include "tbl/symbol_kind.h" @@ -134,23 +136,16 @@ vcc_symtab_new(const char *b, const char *e) return (st); } -static const char * const rootname = ""; - static struct symtab * -vcc_symtab_str(struct vcc *tl, const char *b, const char *e) +vcc_symtab_str(struct symtab *st, const char *b, const char *e) { - struct symtab *st1, *st2, *st3; + struct symtab *st2, *st3; size_t l; int i; char *p, *q; - AN(tl); p = vcc_dup_be(b, e); - if (tl->syms == NULL) - tl->syms = vcc_symtab_new(rootname, rootname); - st1 = tl->syms; - while (1) { q = strchr(p, '.'); if (q == NULL) @@ -159,26 +154,26 @@ vcc_symtab_str(struct vcc *tl, const char *b, const char *e) assert(q[1] != '.' && q[1] != '\0'); AN(q); l = q - p; - VTAILQ_FOREACH(st2, &st1->children, list) { + VTAILQ_FOREACH(st2, &st->children, list) { i = strncasecmp(st2->name, p, l); if (i < 0) continue; if (i == 0 && l == st2->nlen) break; st3 = vcc_symtab_new(p, q); - st3->parent = st1; + st3->parent = st; VTAILQ_INSERT_BEFORE(st2, st3, list); st2 = st3; break; } if (st2 == NULL) { st2 = vcc_symtab_new(p, q); - st2->parent = st1; - VTAILQ_INSERT_TAIL(&st1->children, st2, list); + st2->parent = st; + VTAILQ_INSERT_TAIL(&st->children, st2, list); } if (*q == '\0') return (st2); - st1 = st2; + st = st2; p = q + 1; } } @@ -202,18 +197,12 @@ vcc_new_symbol(struct vcc *tl, struct symtab *st) } static struct symbol * -VCC_Symbol(struct vcc *tl, - const char *b, const char *e, vcc_kind_t kind, - int create, int vlo, int vhi) +vcc_sym_in_tab(struct vcc *tl, struct symtab *st, + vcc_kind_t kind, int vlo, int vhi) { - struct symtab *st, *pst; + struct symtab *pst; struct symbol *sym, *psym; - assert(vlo <= vhi); - - st = vcc_symtab_str(tl, b, e); - AN(st); - VTAILQ_FOREACH(sym, &st->symbols, list) { if (sym->lorev > vhi || sym->hirev < vlo) continue; @@ -222,15 +211,7 @@ VCC_Symbol(struct vcc *tl, if (tl->syntax < VCL_41 && strcmp(sym->name, "default") && (kind != SYM_NONE && kind != sym->kind)) continue; - break; - } - if (sym != NULL) return (sym); - if (create) { - sym = vcc_new_symbol(tl, st); - sym->lorev = vlo; - sym->hirev = vhi; - sym->kind = kind; } pst = st->parent; if (pst == NULL) @@ -238,14 +219,39 @@ VCC_Symbol(struct vcc *tl, psym = VTAILQ_FIRST(&pst->symbols); if (psym == NULL) return(sym); - if (psym->wildcard != NULL) { + if (psym->wildcard == NULL) + return(sym); + + sym = vcc_new_symbol(tl, st); + sym->lorev = vlo; + sym->hirev = vhi; + sym->kind = kind; + psym->wildcard(tl, psym, sym); + if (tl->err) + return(NULL); + return (sym); +} + +static struct symbol * +VCC_Symbol(struct vcc *tl, + const char *b, const char *e, vcc_kind_t kind, + int create, int vlo, int vhi) +{ + struct symtab *st; + struct symbol *sym; + + assert(vlo <= vhi); + if (tl->syms == NULL) + tl->syms = vcc_symtab_new(rootname, rootname); + + st = vcc_symtab_str(tl->syms, b, e); + AN(st); + sym = vcc_sym_in_tab(tl, st, kind, vlo, vhi); + if (sym == NULL && create) { sym = vcc_new_symbol(tl, st); sym->lorev = vlo; sym->hirev = vhi; sym->kind = kind; - psym->wildcard(tl, psym, sym); - if (tl->err) - return(NULL); } return(sym); } From phk at FreeBSD.org Wed May 29 14:38:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 29 May 2019 14:38:09 +0000 (UTC) Subject: [master] 126203852 Parse identifiers element by element. Message-ID: <20190529143809.73E6410AA04@lists.varnish-cache.org> commit 1262038523db35653e97977fc874fbf8ae92b41b Author: Poul-Henning Kamp Date: Wed May 29 14:36:25 2019 +0000 Parse identifiers element by element. This is not very elegant, but I wanted to maintain the same error messages for now. More to come. diff --git a/include/vct.h b/include/vct.h index 24143a332..44021f72d 100644 --- a/include/vct.h +++ b/include/vct.h @@ -43,7 +43,6 @@ #define VCT_TCHAR (1<<9) #define VCT_ID (1<<10) #define VCT_IDENT (VCT_ALPHA | VCT_DIGIT | VCT_ID) -#define VCT_VAR (1<<11) #define VCT_VT (1<<12) #define VCT_SPACE (VCT_LWS | VCT_VT) @@ -71,7 +70,6 @@ vct_is(int x, uint16_t y) #define vct_issepctl(x) vct_is(x, VCT_SEPARATOR | VCT_CTL) #define vct_isident1(x) vct_isalpha(x) #define vct_isident(x) vct_is(x, VCT_IDENT) -#define vct_isvar(x) vct_is(x, VCT_IDENT | VCT_VAR) #define vct_isxmlnamestart(x) vct_is(x, VCT_XMLNAMESTART) #define vct_isxmlname(x) vct_is(x, VCT_XMLNAMESTART | VCT_XMLNAME) #define vct_istchar(x) vct_is(x, VCT_ALPHA | VCT_DIGIT | VCT_TCHAR) diff --git a/lib/libvarnish/vct.c b/lib/libvarnish/vct.c index 73b784e32..a86e2e427 100644 --- a/lib/libvarnish/vct.c +++ b/lib/libvarnish/vct.c @@ -91,7 +91,7 @@ const uint16_t vct_typtab[256] = { [0x2b] = VCT_TCHAR, [0x2c] = VCT_SEPARATOR, [0x2d] = VCT_XMLNAME | VCT_TCHAR | VCT_ID, - [0x2e] = VCT_XMLNAME | VCT_TCHAR | VCT_VAR, + [0x2e] = VCT_XMLNAME | VCT_TCHAR, [0x2f] = VCT_SEPARATOR, [0x30] = VCT_DIGIT | VCT_HEX | VCT_XMLNAME, [0x31] = VCT_DIGIT | VCT_HEX | VCT_XMLNAME, diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index fb64b26cb..048efc896 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -262,6 +262,22 @@ const char XREF_REF[] = "xref_ref"; const char SYMTAB_NOERR[] = "sym_noerror"; const char SYMTAB_CREATE[] = "sym_create"; +static void +vcc_symxref(struct symbol *sym, const char *x, const struct token *t) +{ + if (x == XREF_DEF) { + if (sym->def_b == NULL) + sym->def_b = t; + sym->ndef++; + } else if (x == XREF_REF) { + if (sym->ref_b == NULL) + sym->ref_b = t; + sym->nref++; + } else { + assert (x == XREF_NONE); + } +} + struct symbol * VCC_SymbolGetTok(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x, const struct token *t) @@ -320,28 +336,90 @@ VCC_SymbolGetTok(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x, } return (NULL); } - if (x == XREF_DEF) { - if (sym->def_b == NULL) - sym->def_b = t; - sym->ndef++; - } else if (x == XREF_REF) { - if (sym->ref_b == NULL) - sym->ref_b = t; - sym->nref++; - } else { - assert (x == XREF_NONE); - } + vcc_symxref(sym, x, t); return (sym); } struct symbol * VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x) { + struct symtab *st; struct symbol *sym; + struct token *tn, *tn1; + + if (tl->syntax >= VCL_41 && e == SYMTAB_CREATE && kind != SYM_SUB && + (tl->t->b[0] == 'v'|| tl->t->b[0] == 'V') && + (tl->t->b[1] == 'c'|| tl->t->b[1] == 'C') && + (tl->t->b[2] == 'l'|| tl->t->b[2] == 'L') && + (tl->t->b[3] == '_')) { + VSB_printf(tl->sb, + "Symbols named 'vcl_*' are reserved.\nAt:"); + vcc_ErrWhere(tl, tl->t); + return (NULL); + } - sym = VCC_SymbolGetTok(tl, kind, e, x, tl->t); - if (sym != NULL) - vcc_NextToken(tl); + st = tl->syms; + tn = tl->t; + while (1) { + st = vcc_symtab_str(st, tn->b, tn->e); + tn1 = VTAILQ_NEXT(tn, list); + if (tn1->tok != '.') + break; + tn1 = VTAILQ_NEXT(tn1, list); + if (tn1->tok != ID) + break; + tn = tn1; + } + sym = vcc_sym_in_tab(tl, st, kind, tl->syntax, tl->syntax); + if (sym == NULL && e == SYMTAB_CREATE) { + sym = vcc_new_symbol(tl, st); + sym->lorev = tl->syntax; + sym->hirev = tl->syntax; + sym->kind = kind; + } + if (sym == NULL && e == SYMTAB_NOERR) + return (sym); + if (sym == NULL) { + VSB_printf(tl->sb, "%s: '", e); + tn = VTAILQ_NEXT(tn, list); + for (tn1 = tl->t; tn1 != tn; tn1 = VTAILQ_NEXT(tn1, list)) + VSB_printf(tl->sb, "%.*s", PF(tn1)); + VSB_printf(tl->sb, "'"); + sym = vcc_sym_in_tab(tl, st, kind, VCL_LOW, VCL_HIGH); + if (sym != NULL) { + VSB_printf(tl->sb, " (Only available when"); + if (sym->lorev >= VCL_LOW) + VSB_printf(tl->sb, " %.1f <=", .1 * sym->lorev); + VSB_printf(tl->sb, " VCL syntax"); + if (sym->hirev <= VCL_HIGH) + VSB_printf(tl->sb, " <= %.1f", .1 * sym->hirev); + VSB_printf(tl->sb, ")"); + } + VSB_cat(tl->sb, "\nAt: "); + vcc_ErrWhere(tl, tl->t); + return (NULL); + } + if (kind != SYM_NONE && kind != sym->kind) { + VSB_printf(tl->sb, "Symbol '"); + tn = VTAILQ_NEXT(tn, list); + for (tn1 = tl->t; tn1 != tn; tn1 = VTAILQ_NEXT(tn1, list)) + VSB_printf(tl->sb, "%.*s", PF(tn1)); + VSB_printf(tl->sb, "' has wrong type (%s): ", sym->kind->name); + VSB_cat(tl->sb, "\nAt: "); + vcc_ErrWhere2(tl, tl->t, tn); + if (sym->def_b != NULL) { + VSB_printf(tl->sb, "Symbol was defined here: "); + vcc_ErrWhere(tl, sym->def_b); + } else if (sym->ref_b != NULL) { + VSB_printf(tl->sb, "Symbol was declared here: "); + vcc_ErrWhere(tl, sym->ref_b); + } else { + VSB_printf(tl->sb, "Symbol was builtin\n"); + } + return (NULL); + } + vcc_symxref(sym, x, tl->t); + tl->t = VTAILQ_NEXT(tn, list); return (sym); } diff --git a/lib/libvcc/vcc_token.c b/lib/libvcc/vcc_token.c index fcd37796c..090c84439 100644 --- a/lib/libvcc/vcc_token.c +++ b/lib/libvcc/vcc_token.c @@ -299,18 +299,29 @@ vcc_IdIs(const struct token *t, const char *p) void vcc_ExpectVid(struct vcc *tl, const char *what) { - const char *bad; + const char *bad = NULL; + struct token *t2, *t3; ExpectErr(tl, ID); ERRCHK(tl); - bad = VCT_invalid_name(tl->t->b, tl->t->e); + t2 = VTAILQ_NEXT(tl->t, list); + while (t2->tok == '.') { + bad = "."; + t2 = VTAILQ_NEXT(t2, list); + if (t2->tok != ID) + break; + t2 = VTAILQ_NEXT(t2, list); + } + if (bad == NULL) + bad = VCT_invalid_name(tl->t->b, tl->t->e); if (bad != NULL) { - VSB_printf(tl->sb, "Name of %s, ", what); - vcc_ErrToken(tl, tl->t); + VSB_printf(tl->sb, "Name of %s, '", what); + for (t3 = tl->t; t3 != t2; t3 = VTAILQ_NEXT(t3, list)) + VSB_printf(tl->sb, "%.*s", PF(t3)); VSB_printf(tl->sb, - ", contains illegal character '%c'\n", *bad); - vcc_ErrWhere(tl, tl->t); + "', contains illegal character '%c'\n", *bad); + vcc_ErrWhere2(tl, tl->t, t2); return; } } @@ -493,7 +504,7 @@ vcc_Lexer(struct vcc *tl, struct source *sp) /* Match Identifiers */ if (vct_isident1(*p)) { for (q = p; q < sp->e; q++) - if (!vct_isvar(*q)) + if (!vct_isident(*q)) break; vcc_AddToken(tl, ID, p, q); p = q; From nils.goroll at uplex.de Wed May 29 17:04:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 29 May 2019 17:04:08 +0000 (UTC) Subject: [master] 44ea36eb0 fix regression: ban-lurker skips objects Message-ID: <20190529170408.6CEB210E617@lists.varnish-cache.org> commit 44ea36eb050fc8bdabc4a834bedd4e77c154a2ff Author: Nils Goroll Date: Wed May 29 18:54:06 2019 +0200 fix regression: ban-lurker skips objects 93d805014df919f9be761c6d8ad4ed86eb90c2cb made execution of ban_lurker_test_ban() conditional on bd != b, which effectively caused objects hanging off bans below request bans to not get tested against relevant bans. Because object bans (from the obans list) are being marked completed, the objects which were skipped would also be missed to get evaluated against the relevant bans at lookup time unless they were evaluated in request context. So, in effect, we would simply miss to test bans. Fixes #3007 Maybe related to #3006 diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c index ba083cddb..7be660cb8 100644 --- a/bin/varnishd/cache/cache_ban_lurker.c +++ b/bin/varnishd/cache/cache_ban_lurker.c @@ -304,7 +304,7 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt, VSC_C_main->bans_lurker_obj_killed += lok; VSC_C_main->bans_lurker_obj_killed_cutoff += lokc; tested = tested_tests = lok = lokc = 0; - if (oc->ban == bt) { + if (oc->ban == bt && bt != bd) { bt->refcount--; VTAILQ_REMOVE(&bt->objcore, oc, ban_list); oc->ban = bd; @@ -356,7 +356,7 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl) bd = NULL; VTAILQ_INIT(&obans); for (; b != NULL; b = VTAILQ_NEXT(b, list)) { - if (bd != NULL && bd != b) + if (bd != NULL) ban_lurker_test_ban(wrk, vsl, b, &obans, bd, count > cutoff); if (b->flags & BANS_FLAG_COMPLETED) diff --git a/bin/varnishtest/tests/c00049.vtc b/bin/varnishtest/tests/c00049.vtc index a442833fa..3e6ac286b 100644 --- a/bin/varnishtest/tests/c00049.vtc +++ b/bin/varnishtest/tests/c00049.vtc @@ -33,6 +33,15 @@ server s1 { expect req.url == /4 txresp -hdr "Foo: bar4.1" + rxreq + expect req.url == /r1 + txresp + rxreq + expect req.url == /r2 + txresp + rxreq + expect req.url == /r3 + txresp } -start varnish v1 -vcl+backend {} -start @@ -132,8 +141,8 @@ varnish v1 -expect bans_deleted == 2 varnish v1 -expect bans_tested == 0 varnish v1 -expect bans_tests_tested == 0 varnish v1 -expect bans_obj_killed == 0 -varnish v1 -expect bans_lurker_tested == 8 -varnish v1 -expect bans_lurker_tests_tested == 9 +varnish v1 -expect bans_lurker_tested == 10 +varnish v1 -expect bans_lurker_tests_tested == 11 varnish v1 -expect bans_lurker_obj_killed == 4 varnish v1 -expect bans_dups == 0 @@ -157,8 +166,8 @@ varnish v1 -expect bans_deleted == 2 varnish v1 -expect bans_tested == 1 varnish v1 -expect bans_tests_tested == 1 varnish v1 -expect bans_obj_killed == 0 -varnish v1 -expect bans_lurker_tested == 8 -varnish v1 -expect bans_lurker_tests_tested == 9 +varnish v1 -expect bans_lurker_tested == 10 +varnish v1 -expect bans_lurker_tests_tested == 11 varnish v1 -expect bans_lurker_obj_killed == 4 varnish v1 -expect bans_dups == 0 @@ -182,8 +191,8 @@ varnish v1 -expect bans_deleted == 5 varnish v1 -expect bans_tested == 2 varnish v1 -expect bans_tests_tested == 2 varnish v1 -expect bans_obj_killed == 1 -varnish v1 -expect bans_lurker_tested == 8 -varnish v1 -expect bans_lurker_tests_tested == 9 +varnish v1 -expect bans_lurker_tested == 10 +varnish v1 -expect bans_lurker_tests_tested == 11 varnish v1 -expect bans_lurker_obj_killed == 4 varnish v1 -expect bans_dups == 0 @@ -217,10 +226,41 @@ varnish v1 -expect bans_deleted == 10 varnish v1 -expect bans_tested == 2 varnish v1 -expect bans_tests_tested == 2 varnish v1 -expect bans_obj_killed == 1 -varnish v1 -expect bans_lurker_tested == 8 -varnish v1 -expect bans_lurker_tests_tested == 9 +varnish v1 -expect bans_lurker_tested == 10 +varnish v1 -expect bans_lurker_tests_tested == 11 varnish v1 -expect bans_lurker_obj_killed == 4 varnish v1 -expect bans_lurker_obj_killed_cutoff == 3 varnish v1 -expect bans_dups == 0 varnish v1 -expect n_object == 0 + +client c1 { + txreq -url /r1 + rxresp +} -run + +varnish v1 -cliok "ban req.http.nevermatch == 1" + +client c1 { + txreq -url /r2 + rxresp +} -run + +varnish v1 -cliok "ban req.http.nevermatch == 2" + +client c1 { + txreq -url /r3 + rxresp +} -run + +varnish v1 -cliok "ban req.http.nevermatch == 3" + +varnish v1 -cliok "ban.list" + +varnish v1 -cliok "ban obj.http.status != 0" + +delay 1 + +varnish v1 -cliok "ban.list" + +varnish v1 -expect n_object == 0 From phk at FreeBSD.org Thu May 30 14:53:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 30 May 2019 14:53:10 +0000 (UTC) Subject: [master] f1eef6edc Polish error messages Message-ID: <20190530145310.1F5A1105F98@lists.varnish-cache.org> commit f1eef6edcc6fb478778c7a8fe47b298458419159 Author: Poul-Henning Kamp Date: Thu May 30 13:07:51 2019 +0000 Polish error messages diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index 048efc896..e409f3920 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -396,7 +396,7 @@ VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x) VSB_printf(tl->sb, ")"); } VSB_cat(tl->sb, "\nAt: "); - vcc_ErrWhere(tl, tl->t); + vcc_ErrWhere2(tl, tl->t, tn); return (NULL); } if (kind != SYM_NONE && kind != sym->kind) { diff --git a/lib/libvcc/vcc_var.c b/lib/libvcc/vcc_var.c index 72931c6a0..c662947c8 100644 --- a/lib/libvcc/vcc_var.c +++ b/lib/libvcc/vcc_var.c @@ -46,8 +46,8 @@ vcc_Var_Wildcard(struct vcc *tl, struct symbol *parent, struct symbol *sym) if (strlen(sym->name) >= 127) { VSB_printf(tl->sb, "HTTP header (%.20s..) is too long.\n", sym->name); - VSB_cat(tl->sb, "\nAt: "); - vcc_ErrWhere(tl, tl->t); + tl->err = 1; + return; } AN(sym); From phk at FreeBSD.org Thu May 30 14:53:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 30 May 2019 14:53:10 +0000 (UTC) Subject: [master] 21956d6c2 Pass correct args to VSS_ResolveOne Message-ID: <20190530145310.33D92105F9B@lists.varnish-cache.org> commit 21956d6c2ec6067939cbeffe5f7789a8d09c9af8 Author: Poul-Henning Kamp Date: Thu May 30 13:33:04 2019 +0000 Pass correct args to VSS_ResolveOne diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index 8009820e4..d2c897f46 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -29,6 +29,7 @@ #include "config.h" #include +#include #include #include @@ -351,7 +352,7 @@ dns_works(void) char abuf[VTCP_ADDRBUFSIZE]; char pbuf[VTCP_PORTBUFSIZE]; - sa = VSS_ResolveOne(NULL, "dns-canary.freebsd.dk", NULL, 0, 0, 0); + sa = VSS_ResolveOne(NULL, "dns-canary.freebsd.dk", NULL, AF_UNSPEC, SOCK_STREAM, 0); if (sa == NULL) return (0); VTCP_name(sa, abuf, sizeof abuf, pbuf, sizeof pbuf); From phk at FreeBSD.org Thu May 30 14:53:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 30 May 2019 14:53:10 +0000 (UTC) Subject: [master] 192bfb64d Get rid of a single use, special case, symbol table lookup function. Message-ID: <20190530145310.4F7A8105F9E@lists.varnish-cache.org> commit 192bfb64dbd6a7399c4dce7c40aac42cf43e490d Author: Poul-Henning Kamp Date: Thu May 30 14:46:39 2019 +0000 Get rid of a single use, special case, symbol table lookup function. diff --git a/bin/varnishtest/tests/m00008.vtc b/bin/varnishtest/tests/m00008.vtc index 7b5647487..77b327370 100644 --- a/bin/varnishtest/tests/m00008.vtc +++ b/bin/varnishtest/tests/m00008.vtc @@ -2,7 +2,7 @@ varnishtest "VCL compiler vmod coverage / vmod loading" feature topbuild -varnish v1 -errvcl {Module debug conflicts with other symbol.} { +varnish v1 -errvcl {Symbol 'debug' has wrong type (sub):} { backend b { .host = "127.0.0.1"; } sub debug {} import debug; diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index c9ea92d5d..3a2116a9f 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -357,8 +357,6 @@ extern const char SYMTAB_NOERR[]; extern const char SYMTAB_CREATE[]; struct symbol *VCC_SymbolGet(struct vcc *, vcc_kind_t, const char *, const char *); -struct symbol *VCC_SymbolGetTok(struct vcc *, vcc_kind_t, const char *, - const char *, const struct token *); typedef void symwalk_f(struct vcc *tl, const struct symbol *s); void VCC_WalkSymbols(struct vcc *tl, symwalk_f *func, vcc_kind_t); diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index e409f3920..e142a2b4d 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -278,68 +278,6 @@ vcc_symxref(struct symbol *sym, const char *x, const struct token *t) } } -struct symbol * -VCC_SymbolGetTok(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x, - const struct token *t) -{ - struct symbol *sym; - - AN(e); - if (tl->syntax >= VCL_41 && e == SYMTAB_CREATE && kind != SYM_SUB && - (t->b[0] == 'v'|| t->b[0] == 'V') && - (t->b[1] == 'c'|| t->b[1] == 'C') && - (t->b[2] == 'l'|| t->b[2] == 'L') && - (t->b[3] == '_')) { - VSB_printf(tl->sb, - "Symbols named 'vcl_*' are reserved.\nAt:"); - vcc_ErrWhere(tl, t); - return (NULL); - } - - sym = VCC_Symbol(tl, t->b, t->e, kind, - e == SYMTAB_CREATE ? 1 : 0, tl->syntax, tl->syntax); - if (sym == NULL && e == SYMTAB_NOERR) - return (sym); - if (sym == NULL) { - VSB_printf(tl->sb, "%s: ", e); - vcc_ErrToken(tl, t); - sym = VCC_Symbol(tl, t->b, t->e, kind, 0, - VCL_LOW, VCL_HIGH); - if (sym != NULL) { - VSB_printf(tl->sb, " (Only available when"); - if (sym->lorev >= VCL_LOW) - VSB_printf(tl->sb, " %.1f <=", .1 * sym->lorev); - VSB_printf(tl->sb, " VCL syntax"); - if (sym->hirev <= VCL_HIGH) - VSB_printf(tl->sb, " <= %.1f", .1 * sym->hirev); - VSB_printf(tl->sb, ")"); - } - VSB_cat(tl->sb, "\nAt: "); - vcc_ErrWhere(tl, t); - return (NULL); - } - assert (sym->lorev <= tl->syntax && sym->hirev >= tl->syntax); - if (kind != SYM_NONE && kind != sym->kind) { - VSB_printf(tl->sb, "Symbol "); - vcc_ErrToken(tl, t); - VSB_printf(tl->sb, " has wrong type (%s): ", sym->kind->name); - VSB_cat(tl->sb, "\nAt: "); - vcc_ErrWhere(tl, t); - if (sym->def_b != NULL) { - VSB_printf(tl->sb, "Symbol was defined here: "); - vcc_ErrWhere(tl, sym->def_b); - } else if (sym->ref_b != NULL) { - VSB_printf(tl->sb, "Symbol was declared here: "); - vcc_ErrWhere(tl, sym->ref_b); - } else { - VSB_printf(tl->sb, "Symbol was builtin\n"); - } - return (NULL); - } - vcc_symxref(sym, x, t); - return (sym); -} - struct symbol * VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x) { diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index fad1006a1..c98e261f0 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -232,16 +232,17 @@ vcc_ParseImport(struct vcc *tl) ExpectErr(tl, ID); /* "vmod_name" */ mod = tl->t; - vcc_NextToken(tl); - - if (tl->t->tok == ID && vcc_IdIs(tl->t, "as")) { - SkipToken(tl, ID); /* "as" */ + tmod = VTAILQ_NEXT(mod, list); + if (tmod->tok == ID && vcc_IdIs(tmod, "as")) { + vcc_NextToken(tl); /* "vmod_name" */ + vcc_NextToken(tl); /* "as" */ ExpectErr(tl, ID); /* "vcl_name" */ - tmod = tl->t; - vcc_NextToken(tl); - } else { - tmod = mod; } + tmod = tl->t; + + msym = VCC_SymbolGet(tl, SYM_VMOD, SYMTAB_CREATE, XREF_NONE); + ERRCHK(tl); + AN(msym); if (tl->t->tok == ID) { if (!vcc_IdIs(tl->t, "from")) { @@ -293,33 +294,19 @@ vcc_ParseImport(struct vcc *tl) return; } - msym = VCC_SymbolGetTok(tl, SYM_NONE, SYMTAB_NOERR, XREF_NONE, tmod); - - if (msym != NULL && msym->kind == SYM_VMOD && - !strcmp(msym->extra, vmd->file_id)) { - /* Identical import is OK */ - AZ(dlclose(vop->hdl)); - free(fnpx); - return; - } else if (msym != NULL && msym->kind == SYM_VMOD) { - VSB_printf(tl->sb, - "Another module already imported as %.*s.\n", PF(tmod)); - vcc_ErrWhere2(tl, t1, tl->t); - AZ(dlclose(vop->hdl)); - free(fnpx); - return; - } else if (msym != NULL) { - VSB_printf(tl->sb, - "Module %.*s conflicts with other symbol.\n", PF(tmod)); - vcc_ErrWhere2(tl, t1, tl->t); + if (msym->extra != NULL) { + if (!strcmp(msym->extra, vmd->file_id)) { + /* Identical import is OK */ + } else { + VSB_printf(tl->sb, + "Another module already imported as %.*s.\n", + PF(tmod)); + vcc_ErrWhere2(tl, t1, tl->t); + } AZ(dlclose(vop->hdl)); free(fnpx); return; } - - msym = VCC_SymbolGetTok(tl, SYM_VMOD, SYMTAB_CREATE, XREF_NONE, tmod); - ERRCHK(tl); - AN(msym); msym->def_b = t1; msym->def_e = tl->t; From phk at FreeBSD.org Thu May 30 14:53:10 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 30 May 2019 14:53:10 +0000 (UTC) Subject: [master] 174f62a42 Code condensation. Message-ID: <20190530145310.6A159105FA3@lists.varnish-cache.org> commit 174f62a4294a6e7e09492823d8d60a49801fdfa3 Author: Poul-Henning Kamp Date: Thu May 30 14:52:22 2019 +0000 Code condensation. diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index e142a2b4d..bcc067bc6 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -262,28 +262,12 @@ const char XREF_REF[] = "xref_ref"; const char SYMTAB_NOERR[] = "sym_noerror"; const char SYMTAB_CREATE[] = "sym_create"; -static void -vcc_symxref(struct symbol *sym, const char *x, const struct token *t) -{ - if (x == XREF_DEF) { - if (sym->def_b == NULL) - sym->def_b = t; - sym->ndef++; - } else if (x == XREF_REF) { - if (sym->ref_b == NULL) - sym->ref_b = t; - sym->nref++; - } else { - assert (x == XREF_NONE); - } -} - struct symbol * VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x) { struct symtab *st; struct symbol *sym; - struct token *tn, *tn1; + struct token *t0, *tn, *tn1; if (tl->syntax >= VCL_41 && e == SYMTAB_CREATE && kind != SYM_SUB && (tl->t->b[0] == 'v'|| tl->t->b[0] == 'V') && @@ -297,6 +281,7 @@ VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x) } st = tl->syms; + t0 = tl->t; tn = tl->t; while (1) { st = vcc_symtab_str(st, tn->b, tn->e); @@ -317,10 +302,10 @@ VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x) } if (sym == NULL && e == SYMTAB_NOERR) return (sym); + tl->t = VTAILQ_NEXT(tn, list); if (sym == NULL) { VSB_printf(tl->sb, "%s: '", e); - tn = VTAILQ_NEXT(tn, list); - for (tn1 = tl->t; tn1 != tn; tn1 = VTAILQ_NEXT(tn1, list)) + for (tn1 = t0; tn1 != tl->t; tn1 = VTAILQ_NEXT(tn1, list)) VSB_printf(tl->sb, "%.*s", PF(tn1)); VSB_printf(tl->sb, "'"); sym = vcc_sym_in_tab(tl, st, kind, VCL_LOW, VCL_HIGH); @@ -334,17 +319,16 @@ VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x) VSB_printf(tl->sb, ")"); } VSB_cat(tl->sb, "\nAt: "); - vcc_ErrWhere2(tl, tl->t, tn); + vcc_ErrWhere2(tl, t0, tl->t); return (NULL); } if (kind != SYM_NONE && kind != sym->kind) { VSB_printf(tl->sb, "Symbol '"); - tn = VTAILQ_NEXT(tn, list); - for (tn1 = tl->t; tn1 != tn; tn1 = VTAILQ_NEXT(tn1, list)) + for (tn1 = t0; tn1 != tl->t; tn1 = VTAILQ_NEXT(tn1, list)) VSB_printf(tl->sb, "%.*s", PF(tn1)); VSB_printf(tl->sb, "' has wrong type (%s): ", sym->kind->name); VSB_cat(tl->sb, "\nAt: "); - vcc_ErrWhere2(tl, tl->t, tn); + vcc_ErrWhere2(tl, t0, tl->t); if (sym->def_b != NULL) { VSB_printf(tl->sb, "Symbol was defined here: "); vcc_ErrWhere(tl, sym->def_b); @@ -356,8 +340,17 @@ VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x) } return (NULL); } - vcc_symxref(sym, x, tl->t); - tl->t = VTAILQ_NEXT(tn, list); + if (x == XREF_DEF) { + if (sym->def_b == NULL) + sym->def_b = t0; + sym->ndef++; + } else if (x == XREF_REF) { + if (sym->ref_b == NULL) + sym->ref_b = t0; + sym->nref++; + } else { + assert (x == XREF_NONE); + } return (sym); } From phk at FreeBSD.org Thu May 30 18:34:12 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 30 May 2019 18:34:12 +0000 (UTC) Subject: [master] 3264dffd1 More code polishing and condensation Message-ID: <20190530183412.5DF5A10EEDC@lists.varnish-cache.org> commit 3264dffd1ba9f6ef385d24c44691040f5cf36ce3 Author: Poul-Henning Kamp Date: Thu May 30 18:33:28 2019 +0000 More code polishing and condensation diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index bcc067bc6..9883dede4 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -37,22 +37,20 @@ #include "vct.h" -static const char * const rootname = ""; - /*--------------------------------------------------------------------*/ #define VCC_KIND(U,l) const struct kind SYM_##U[1] = {{ KIND_MAGIC, #l}}; #include "tbl/symbol_kind.h" /*--------------------------------------------------------------------*/ struct symtab { - unsigned magic; -#define SYMTAB_MAGIC 0x084d9c8a - unsigned nlen; - char *name; + unsigned magic; +#define SYMTAB_MAGIC 0x084d9c8a + unsigned nlen; + const char *name; struct symtab *parent; - VTAILQ_ENTRY(symtab) list; - VTAILQ_HEAD(,symtab) children; - VTAILQ_HEAD(,symbol) symbols; + VTAILQ_ENTRY(symtab) list; + VTAILQ_HEAD(,symtab) children; + VTAILQ_HEAD(,symbol) symbols; }; static vcc_kind_t @@ -123,13 +121,13 @@ vcc_dup_be(const char *b, const char *e) } static struct symtab * -vcc_symtab_new(const char *b, const char *e) +vcc_symtab_new(const char *name) { struct symtab *st; ALLOC_OBJ(st, SYMTAB_MAGIC); AN(st); - st->name = vcc_dup_be(b, e); + st->name = name; st->nlen = strlen(st->name); VTAILQ_INIT(&st->children); VTAILQ_INIT(&st->symbols); @@ -142,44 +140,42 @@ vcc_symtab_str(struct symtab *st, const char *b, const char *e) struct symtab *st2, *st3; size_t l; int i; - char *p, *q; + const char *q; - p = vcc_dup_be(b, e); + if (e == NULL) + e = strchr(b, '\0'); - while (1) { - q = strchr(p, '.'); - if (q == NULL) - q = strchr(p, '\0'); - else - assert(q[1] != '.' && q[1] != '\0'); + while (b < e) { + for (q = b; q < e && *q != '.'; q++) + continue; AN(q); - l = q - p; + l = q - b; VTAILQ_FOREACH(st2, &st->children, list) { - i = strncasecmp(st2->name, p, l); + i = strncasecmp(st2->name, b, l); if (i < 0) continue; if (i == 0 && l == st2->nlen) break; - st3 = vcc_symtab_new(p, q); + st3 = vcc_symtab_new(vcc_dup_be(b, q)); st3->parent = st; VTAILQ_INSERT_BEFORE(st2, st3, list); st2 = st3; break; } if (st2 == NULL) { - st2 = vcc_symtab_new(p, q); + st2 = vcc_symtab_new(vcc_dup_be(b, q)); st2->parent = st; VTAILQ_INSERT_TAIL(&st->children, st2, list); } - if (*q == '\0') - return (st2); st = st2; - p = q + 1; + b = q + 1; } + return (st); } static struct symbol * -vcc_new_symbol(struct vcc *tl, struct symtab *st) +vcc_new_symbol(struct vcc *tl, struct symtab *st, + vcc_kind_t kind, int vlo, int vhi) { struct symbol *sym; @@ -188,10 +184,10 @@ vcc_new_symbol(struct vcc *tl, struct symtab *st) AN(sym); sym->name = st->name; sym->symtab = st; - sym->kind = SYM_NONE; + sym->kind = kind; sym->type = VOID; - sym->lorev = VCL_LOW; - sym->hirev = VCL_HIGH; + sym->lorev = vlo; + sym->hirev = vhi; VTAILQ_INSERT_TAIL(&st->symbols, sym, list); return (sym); } @@ -222,39 +218,13 @@ vcc_sym_in_tab(struct vcc *tl, struct symtab *st, if (psym->wildcard == NULL) return(sym); - sym = vcc_new_symbol(tl, st); - sym->lorev = vlo; - sym->hirev = vhi; - sym->kind = kind; + sym = vcc_new_symbol(tl, st, kind, vlo, vhi); psym->wildcard(tl, psym, sym); if (tl->err) return(NULL); return (sym); } -static struct symbol * -VCC_Symbol(struct vcc *tl, - const char *b, const char *e, vcc_kind_t kind, - int create, int vlo, int vhi) -{ - struct symtab *st; - struct symbol *sym; - - assert(vlo <= vhi); - if (tl->syms == NULL) - tl->syms = vcc_symtab_new(rootname, rootname); - - st = vcc_symtab_str(tl->syms, b, e); - AN(st); - sym = vcc_sym_in_tab(tl, st, kind, vlo, vhi); - if (sym == NULL && create) { - sym = vcc_new_symbol(tl, st); - sym->lorev = vlo; - sym->hirev = vhi; - sym->kind = kind; - } - return(sym); -} const char XREF_NONE[] = "xref_none"; const char XREF_DEF[] = "xref_def"; @@ -294,12 +264,8 @@ VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x) tn = tn1; } sym = vcc_sym_in_tab(tl, st, kind, tl->syntax, tl->syntax); - if (sym == NULL && e == SYMTAB_CREATE) { - sym = vcc_new_symbol(tl, st); - sym->lorev = tl->syntax; - sym->hirev = tl->syntax; - sym->kind = kind; - } + if (sym == NULL && e == SYMTAB_CREATE) + sym = vcc_new_symbol(tl, st, kind, tl->syntax, tl->syntax); if (sym == NULL && e == SYMTAB_NOERR) return (sym); tl->t = VTAILQ_NEXT(tn, list); @@ -357,19 +323,25 @@ VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x) struct symbol * VCC_MkSym(struct vcc *tl, const char *b, vcc_kind_t kind, int vlo, int vhi) { + struct symtab *st; struct symbol *sym; AN(tl); AN(b); CHECK_OBJ_NOTNULL(kind, KIND_MAGIC); + assert(vlo <= vhi); - sym = VCC_Symbol(tl, b, NULL, kind, 1, vlo, vhi); - AN(sym); + if (tl->syms == NULL) + tl->syms = vcc_symtab_new(""); + st = vcc_symtab_str(tl->syms, b, NULL); + AN(st); + sym = vcc_sym_in_tab(tl, st, kind, vlo, vhi); + AZ(sym); + sym = vcc_new_symbol(tl, st, kind, vlo, vhi); sym->noref = 1; return (sym); } - static void vcc_walksymbols(struct vcc *tl, const struct symtab *root, symwalk_f *func, vcc_kind_t kind) From nils.goroll at uplex.de Fri May 31 16:27:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 31 May 2019 16:27:08 +0000 (UTC) Subject: [master] 991d8d1a1 properly maintain the obans list when pruning the ban list tail Message-ID: <20190531162708.4F373104ACE@lists.varnish-cache.org> commit 991d8d1a1685536fe1010f124c6a6c0e67d14622 Author: Nils Goroll Date: Fri May 31 15:51:08 2019 +0200 properly maintain the obans list when pruning the ban list tail background: When the ban lurker has finished working the bottom of the ban list, conceptually we mark all bans it has evaluated as completed and then remove the tail of the ban list which has no references any more. Yet, for efficiency, we first remove the tail and then mark only those bans completed, which we did not remove. Doing so depends on knowing where in the (obans) list of bans to be completed is the new tail of the bans list after pruning. 5dd54f8390739c62c201e62c36eb515b1e03c2ee was intended to solve this, but the fix was incomplete (and also unnecessarily complicated): For example when a duplicate ban was issued, ban_lurker_test_ban() could remove a ban from the obans list which later happens to become the new ban tail. We now - hopefully - solve the problem for real by properly cleaning the obans list when we prune the ban list. Fixes #3006 Fixes #2779 Fixes #2556 for real (5dd54f8390739c62c201e62c36eb515b1e03c2ee was incomplete) diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c index 7be660cb8..2d7f7f03f 100644 --- a/bin/varnishd/cache/cache_ban_lurker.c +++ b/bin/varnishd/cache/cache_ban_lurker.c @@ -58,19 +58,18 @@ ban_kick_lurker(void) * still referenced. For already completed bans, we update statistics * accordingly, but otherwise just skip the completion step and remove directly * - * return 1 if we removed the victim, 0 otherwise + * if an obans list is passed, we clean its tail as well */ -static int -ban_cleantail(const struct ban *victim) +static void +ban_cleantail(struct banhead_s *obans) { struct ban *b, *bt; struct banhead_s freelist = VTAILQ_HEAD_INITIALIZER(freelist); - int r = 0; /* handle the zero-length tail unprotected */ if (VTAILQ_LAST(&ban_head, banhead_s) == VTAILQ_FIRST(&ban_head)) - return (r); + return; Lck_Lock(&ban_mtx); do { @@ -99,13 +98,27 @@ ban_cleantail(const struct ban *victim) Lck_Unlock(&ban_mtx); - VTAILQ_FOREACH_SAFE(b, &freelist, list, bt) { - if (b == victim) - r = 1; - BAN_Free(b); + /* oban order is head to tail, freelist tail to head */ + if (obans != NULL) + bt = VTAILQ_LAST(obans, banhead_s); + else + bt = NULL; + + if (bt != NULL) { + VTAILQ_FOREACH(b, &freelist, list) { + if (b != bt) + continue; + VTAILQ_REMOVE(obans, b, l_list); + bt = VTAILQ_LAST(obans, banhead_s); + if (bt == NULL) + break; + } } - return (r); + VTAILQ_FOREACH_SAFE(b, &freelist, list, bt) + BAN_Free(b); + + return; } /*-------------------------------------------------------------------- @@ -343,7 +356,7 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl) dt = 49.62; // Random, non-magic if (cache_param->ban_lurker_sleep == 0) { - (void)ban_cleantail(NULL); + ban_cleantail(NULL); return (dt); } if (cache_param->ban_cutoff > 0) @@ -379,42 +392,18 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl) } /* - * conceptually, all obans are now completed. Remove the tail. If it - * contained the first oban, all obans were on the tail and we're - * done. + * conceptually, all obans are now completed. Remove the tail. + * If any bans to be completed remain after the tail is cut, + * mark them completed */ - if (ban_cleantail(VTAILQ_FIRST(&obans))) - return (dt); + ban_cleantail(&obans); if (VTAILQ_FIRST(&obans) == NULL) return (dt); - /* - * Mark remaining bans completed: the tail of the obans list is now - * removed, but iterating over it is safe until we hit the new ban list - * tail - * - * bans at the tail of the list may have been completed by other means - * and, consequently, may have been removed from obans, so we skip all - * already completed bans at the tail. - * - * While traversing the ban list backwards, we check if we pass by the - * first oban, in which case we're done. - */ - bd = VTAILQ_LAST(&ban_head, banhead_s); - while (bd->flags & BANS_FLAG_COMPLETED) { - if (bd == VTAILQ_FIRST(&ban_head) || - bd == VTAILQ_FIRST(&obans)) - return (dt); - bd = VTAILQ_PREV(bd, banhead_s, list); - } - Lck_Lock(&ban_mtx); - VTAILQ_FOREACH(b, &obans, l_list) { + VTAILQ_FOREACH(b, &obans, l_list) ban_mark_completed(b); - if (b == bd) - break; - } Lck_Unlock(&ban_mtx); return (dt); } diff --git a/bin/varnishtest/tests/r03006.vtc b/bin/varnishtest/tests/r03006.vtc new file mode 100644 index 000000000..9a48e7536 --- /dev/null +++ b/bin/varnishtest/tests/r03006.vtc @@ -0,0 +1,54 @@ +varnishtest "test ban lurker destination being completed by dup ban" + +server s1 -repeat 4 -keepalive { + rxreq + txresp +} -start + +varnish v1 -vcl+backend {} -start + +varnish v1 -cliok "param.set ban_lurker_age 99" + +# this ban becomes the pruned tail +varnish v1 -cliok "ban obj.http.t == 1" +client c1 { + txreq -url /1 + rxresp +} -run + +# this ban becomes the new tail +varnish v1 -cliok "ban obj.http.t == 2" +client c1 { + txreq -url /2 + rxresp +} -run + +# req ban to define where the tail goes (at t == 2) +varnish v1 -cliok "ban req.http.barrier == here" +client c1 { + txreq -url /3 + rxresp +} -run + +# dup ban to trigger #3006 +varnish v1 -cliok "ban obj.http.t == 2" +client c1 { + txreq -url /4 + rxresp +} -run + +varnish v1 -cliok "ban.list" + +varnish v1 -cliok "param.set ban_lurker_age 0.1" + +varnish v1 -cliok "ban.list" + +delay 2 + +varnish v1 -expect bans == 3 +varnish v1 -expect bans_completed == 2 + +varnish v1 -cliok "ban.list" + +varnish v1 -expect bans == 3 +varnish v1 -expect bans_completed == 2