From dridi.boukelmoune at gmail.com Wed Apr 3 14:33:08 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 3 Apr 2019 14:33:08 +0000 (UTC) Subject: [master] f87c8e33c Polish Message-ID: <20190403143308.5B545967D2@lists.varnish-cache.org> commit f87c8e33ceeb35819e89e18dc21520f74db6e6d0 Author: Dridi Boukelmoune Date: Wed Apr 3 13:53:28 2019 +0200 Polish diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index d46f4e4da..4c0ec76bb 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -116,13 +116,10 @@ obj_deleteboc(struct boc **p) { struct boc *boc; - AN(p); - boc = *p; - *p = NULL; + TAKE_OBJ_NOTNULL(boc, p, BOC_MAGIC); Lck_Delete(&boc->mtx); AZ(pthread_cond_destroy(&boc->cond)); - if (boc->vary != NULL) - free(boc->vary); + free(boc->vary); FREE_OBJ(boc); } From dridi.boukelmoune at gmail.com Wed Apr 3 14:33:08 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 3 Apr 2019 14:33:08 +0000 (UTC) Subject: [master] 7ee86fbbb Polish Message-ID: <20190403143308.70080967D5@lists.varnish-cache.org> commit 7ee86fbbb9edd5a6fbe4b989488e1ce3d0c16108 Author: Dridi Boukelmoune Date: Wed Apr 3 14:44:09 2019 +0200 Polish diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c index 0606f997e..83e8d713c 100644 --- a/bin/varnishd/storage/storage_simple.c +++ b/bin/varnishd/storage/storage_simple.c @@ -497,8 +497,7 @@ sml_bocdone(struct worker *wrk, struct objcore *oc, struct boc *boc) if (boc->stevedore_priv != NULL) { /* Free any leftovers from Trim */ CAST_OBJ_NOTNULL(st, boc->stevedore_priv, STORAGE_MAGIC); - boc->stevedore_priv = 0; - CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC); + boc->stevedore_priv = NULL; sml_stv_free(stv, st); } From dridi.boukelmoune at gmail.com Wed Apr 3 14:33:08 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 3 Apr 2019 14:33:08 +0000 (UTC) Subject: [master] 521324839 Assert Message-ID: <20190403143308.8ACA6967DB@lists.varnish-cache.org> commit 52132483928766aa0dfc485182be9b552392bbc5 Author: Dridi Boukelmoune Date: Wed Apr 3 14:46:22 2019 +0200 Assert diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index 4c0ec76bb..5a807e869 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -367,6 +367,7 @@ ObjBocDone(struct worker *wrk, struct objcore *oc, struct boc **boc) const struct obj_methods *m; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); AN(boc); CHECK_OBJ_NOTNULL(*boc, BOC_MAGIC); if (oc->stobj->stevedore != NULL) { From dridi.boukelmoune at gmail.com Wed Apr 3 14:57:06 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 3 Apr 2019 14:57:06 +0000 (UTC) Subject: [master] 613c2270a Assert Message-ID: <20190403145706.6DBF897F2A@lists.varnish-cache.org> commit 613c2270a4ba8cc9d8317b9c8971e99e151df52a Author: Dridi Boukelmoune Date: Wed Apr 3 16:55:31 2019 +0200 Assert diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index 5a807e869..b533eefcf 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -370,6 +370,7 @@ ObjBocDone(struct worker *wrk, struct objcore *oc, struct boc **boc) CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); AN(boc); CHECK_OBJ_NOTNULL(*boc, BOC_MAGIC); + CHECK_OBJ_ORNULL(oc->stobj->stevedore, STEVEDORE_MAGIC); if (oc->stobj->stevedore != NULL) { m = obj_getmethods(oc); if (m->objbocdone != NULL) From nils.goroll at uplex.de Sat Apr 6 11:16:06 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 6 Apr 2019 11:16:06 +0000 (UTC) Subject: [master] f106ed834 add the vmod pointer to the panic output Message-ID: <20190406111606.BF988A328B@lists.varnish-cache.org> commit f106ed834f31f2ab5c798c56a6ca48225615dc45 Author: Nils Goroll Date: Sat Apr 6 12:51:17 2019 +0200 add the vmod pointer to the panic output to identify the vmod by its address in other places like vcc-assigned VRT_priv_* ids diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c index af49ef40b..fd8031149 100644 --- a/bin/varnishd/cache/cache_vrt_vmod.c +++ b/bin/varnishd/cache/cache_vrt_vmod.c @@ -199,8 +199,8 @@ VMOD_Panic(struct vsb *vsb) VSB_printf(vsb, "vmods = {\n"); VSB_indent(vsb, 2); VTAILQ_FOREACH(v, &vmods, list) - VSB_printf(vsb, "%s = {%s, %u.%u},\n", - v->nm, v->abi, v->vrt_major, v->vrt_minor); + VSB_printf(vsb, "%s = {%p, %s, %u.%u},\n", + v->nm, v, v->abi, v->vrt_major, v->vrt_minor); VSB_indent(vsb, -2); VSB_printf(vsb, "},\n"); } From nils.goroll at uplex.de Sun Apr 7 15:59:06 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 7 Apr 2019 15:59:06 +0000 (UTC) Subject: [master] 458306fa8 the XXX marks the spot Message-ID: <20190407155906.92E1A9C79B@lists.varnish-cache.org> commit 458306fa8ea4391b4567e52a5b869267b0f76ffc Author: Nils Goroll Date: Sun Apr 7 15:30:17 2019 +0200 the XXX marks the spot ... where we would need error handling diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 0524f3e32..5a12f8890 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -176,8 +176,11 @@ h2h_decode_init(const struct h2_sess *h2) INIT_OBJ(d, H2H_DECODE_MAGIC); VHD_Init(d->vhd); d->out_l = WS_Reserve(h2->new_req->http->ws, 0); - assert(d->out_l > 0); /* Can't do any work without any buffer - space. Require non-zero size. */ + /* + * Can't do any work without any buffer + * space. Require non-zero size. + */ + XXXAN(d->out_l); d->out = h2->new_req->http->ws->f; d->reset = d->out; } From nils.goroll at uplex.de Sun Apr 7 15:59:06 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 7 Apr 2019 15:59:06 +0000 (UTC) Subject: [master] 0e32f1667 Fix WS_Reserve() error handling Message-ID: <20190407155906.A54DB9C7A0@lists.varnish-cache.org> commit 0e32f1667ea55156fc2a7fa84e3f1080a05bcb2e Author: Nils Goroll Date: Sun Apr 7 17:32:48 2019 +0200 Fix WS_Reserve() error handling See #2967 for discussion diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index 7ee71ac99..5a3d01cab 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -226,8 +226,10 @@ h2_build_headers(struct vsb *resp, struct req *req) ssize_t sz, sz1; l = WS_Reserve(req->ws, 0); - if (l < 10) + if (l < 10) { + WS_Release(req->ws, 0); return (-1); + } AN(VSB_new(resp, req->ws->f, l, VSB_FIXEDLEN)); From phk at FreeBSD.org Mon Apr 8 07:37:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 8 Apr 2019 07:37:07 +0000 (UTC) Subject: [master] e9948061d From VTEST: Message-ID: <20190408073707.1A9026B02@lists.varnish-cache.org> commit e9948061d30255fb2eafc1fa5f5e905aba774789 Author: Poul-Henning Kamp Date: Mon Apr 8 07:35:42 2019 +0000 From VTEST: Reinitialize cyclic barrier only once per cycle It was done for all threads syncing on a barrier. The one reaching the limit but also all waiting threads. So it could be reset by a thread while others have already started another round, leading to an deadlock (because some increments will be lost then). Now, cyclic barrier is reset for a new round by the thread reaching the limit and only this one. Author: Christopher Faulet diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index e7fc1e5ae..cd0063cf5 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -274,15 +274,14 @@ barrier_cond_sync(struct barrier *b, struct vtclog *vl) if (++b->waiters == b->expected) { vtc_log(vl, 4, "Barrier(%s) wake %u", b->name, b->expected); + if (b->cyclic) + b->waiters = 0; AZ(pthread_cond_broadcast(&b->cond)); } else { vtc_log(vl, 4, "Barrier(%s) wait %u of %u", b->name, b->waiters, b->expected); AZ(pthread_cond_wait(&b->cond, &b->mtx)); } - - if (b->cyclic) - b->waiters = 0; } static void From phk at FreeBSD.org Mon Apr 8 07:51:05 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 8 Apr 2019 07:51:05 +0000 (UTC) Subject: [master] 9284af89e From VTEST: Message-ID: <20190408075105.35AA3709E@lists.varnish-cache.org> commit 9284af89e4137da6e8f39a351c550c310e391f94 Author: Poul-Henning Kamp Date: Mon Apr 8 07:46:57 2019 +0000 From VTEST: Missing HEAD method usage flag reset between two requests. If an HTTP client request with a method different from HEAD followed another one with HEAD method, the former was still flagged as being using the HEAD method because of a missing HEAD method usage flag reset before building an HTTP client request. Author: Fr?d?ric L?caille diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 0c772f50d..a32b0f37b 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -1275,6 +1275,7 @@ cmd_http_txreq(CMD_ARGS) VSB_clear(hp->vsb); + hp->head_method = 0; for (; *av != NULL; av++) { if (!strcmp(*av, "-url")) { url = av[1]; From nils.goroll at uplex.de Mon Apr 8 13:45:10 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 8 Apr 2019 13:45:10 +0000 (UTC) Subject: [master] 3ce0d377e ws functions are not part of vrt, so mark them Message-ID: <20190408134510.7FCC1653B0@lists.varnish-cache.org> commit 3ce0d377ed55cdb964cebf5d124c18d3c336576a Author: Nils Goroll Date: Mon Apr 8 15:18:44 2019 +0200 ws functions are not part of vrt, so mark them Yet having the changelog in one place is useful diff --git a/include/vrt.h b/include/vrt.h index a0c4d4753..05ab88d7b 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -111,11 +111,11 @@ * VRT_vcl_get added * VRT_vcl_rel added * VRT_fail added - * WS_Reset and WS_Snapshot signatures changed - * WS_Front added - * WS_ReserveLumps added - * WS_Inside added - * WS_Assert_Allocated added + * [cache.h] WS_Reset and WS_Snapshot signatures changed + * [cache.h] WS_Front added + * [cache.h] WS_ReserveLumps added + * [cache.h] WS_Inside added + * [cache.h] WS_Assert_Allocated added * 5.0: * Varnish 5.0 release "better safe than sorry" bump * 4.0: From nils.goroll at uplex.de Mon Apr 8 13:45:10 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 8 Apr 2019 13:45:10 +0000 (UTC) Subject: [master] d001cdd20 Add WS_ReserveAll() to replace WS_Reserve(ws, 0) Message-ID: <20190408134510.B18C3653B4@lists.varnish-cache.org> commit d001cdd2045aa484a968d09859cddf59dee372cb Author: Nils Goroll Date: Mon Apr 8 15:19:30 2019 +0200 Add WS_ReserveAll() to replace WS_Reserve(ws, 0) ... to un-confuse the interface Notes on changes from WS_Reserve(): * Removed the first WS_Assert because all we change is ws->r and we got a specific assert on it. * it follows from PAOK(ws->e) && PAOK(ws->f) in WS_Assert() that PAOK(ws->r) && PAOK(b), so we remove the PRNDDN() Closes #2967 diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 163c52948..cc2fc993d 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -765,6 +765,7 @@ void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, void WS_Init(struct ws *ws, const char *id, void *space, unsigned len); unsigned WS_Reserve(struct ws *ws, unsigned bytes); +unsigned WS_ReserveAll(struct ws *); unsigned WS_ReserveLumps(struct ws *ws, size_t sz); void WS_MarkOverflow(struct ws *ws); void WS_Release(struct ws *ws, unsigned bytes); diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index f43e214ff..1395f7101 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -650,7 +650,7 @@ HSH_Purge(struct worker *wrk, struct objhead *oh, vtim_real ttl_now, CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC); - ospc = WS_Reserve(wrk->aws, 0); + ospc = WS_ReserveAll(wrk->aws); assert(ospc >= sizeof *ocp); /* * Because of "soft" purges, there might be oc's in the list that has diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 5e3c92cd0..717fa2feb 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -393,7 +393,7 @@ http_CollectHdrSep(struct http *hp, const char *hdr, const char *sep) } if (b == NULL) { /* Found second header, start our collection */ - ml = WS_Reserve(hp->ws, 0); + ml = WS_ReserveAll(hp->ws); b = hp->ws->f; e = b + ml; x = Tlen(hp->hd[f]); @@ -1227,7 +1227,7 @@ http_PrintfHeader(struct http *to, const char *fmt, ...) unsigned l, n; CHECK_OBJ_NOTNULL(to, HTTP_MAGIC); - l = WS_Reserve(to->ws, 0); + l = WS_ReserveAll(to->ws); va_start(ap, fmt); n = vsnprintf(to->ws->f, l, fmt, ap); va_end(ap); diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 259110d7b..eac4773fc 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -203,7 +203,7 @@ HTC_RxInit(struct http_conn *htc, struct ws *ws) CHECK_OBJ_NOTNULL(htc, HTTP_CONN_MAGIC); htc->ws = ws; - (void)WS_Reserve(htc->ws, 0); + (void)WS_ReserveAll(htc->ws); htc->rxbuf_b = ws->f; htc->rxbuf_e = ws->f; if (htc->pipeline_b != NULL) { diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c index 46d6930fb..4af92f69c 100644 --- a/bin/varnishd/cache/cache_vary.c +++ b/bin/varnishd/cache/cache_vary.c @@ -230,7 +230,7 @@ VRY_Prep(struct req *req) AZ(req->vary_b); AZ(req->vary_l); AZ(req->vary_e); - (void)WS_Reserve(req->ws, 0); + (void)WS_ReserveAll(req->ws); } else { AN(req->ws->r); } diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 2edfb85b6..bf1676653 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -261,7 +261,7 @@ VRT_String(struct ws *ws, const char *h, const char *p, va_list ap) unsigned u, x; va_list aq; - u = WS_Reserve(ws, 0); + u = WS_ReserveAll(ws); e = b = ws->f; e += u; @@ -385,7 +385,7 @@ VRT_StrandsWS(struct ws *ws, const char *h, VCL_STRANDS s) int i; AN(s); - u = WS_Reserve(ws, 0); + u = WS_ReserveAll(ws); for (i = 0; i < s->n; i++) if (s->p[i] != NULL && *s->p[i] != '\0') { @@ -573,7 +573,7 @@ VRT_IP_string(VRT_CTX, VCL_IP ip) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if (ip == NULL) return (NULL); - len = WS_Reserve(ctx->ws, 0); + len = WS_ReserveAll(ctx->ws); if (len == 0) { WS_Release(ctx->ws, 0); return (NULL); diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c index 5e83a89a7..2f57572aa 100644 --- a/bin/varnishd/cache/cache_vrt_filter.c +++ b/bin/varnishd/cache/cache_vrt_filter.c @@ -251,7 +251,7 @@ filter_on_ws(struct ws *ws, filter_list_t *func, void *arg) AN(func); AN(arg); - u = WS_Reserve(ws, 0); + u = WS_ReserveAll(ws); if (u == 0) { WS_Release(ws, 0); WS_MarkOverflow(ws); diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c index 9cd0de3e7..ad1f44de8 100644 --- a/bin/varnishd/cache/cache_vrt_re.c +++ b/bin/varnishd/cache/cache_vrt_re.c @@ -127,7 +127,7 @@ VRT_regsub(VRT_CTX, int all, const char *str, void *re, return (str); } - u = WS_Reserve(ctx->ws, 0); + u = WS_ReserveAll(ctx->ws); res_e = res_b = b0 = ctx->ws->f; res_e += u; diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index 3ec5ec017..e0a9b1d48 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -202,7 +202,7 @@ WS_Printf(struct ws *ws, const char *fmt, ...) va_list ap; char *p; - u = WS_Reserve(ws, 0); + u = WS_ReserveAll(ws); p = ws->f; va_start(ap, fmt); v = vsnprintf(p, u, fmt, ap); @@ -227,6 +227,30 @@ WS_Snapshot(struct ws *ws) return (ws->f == ws->s ? 0 : (uintptr_t)ws->f); } +/* + * WS_Release() must be called in all cases + */ +unsigned +WS_ReserveAll(struct ws *ws) +{ + unsigned b; + + assert(ws->r == NULL); + + ws->r = ws->e; + b = pdiff(ws->f, ws->r); + + WS_Assert(ws); + DSL(DBG_WORKSPACE, 0, "WS_ReserveAll(%p) = %u", ws, b); + + return (b); +} + +/* + * bytes == 0 argument is deprecated - use WS_ReserveAll + * + * XXX rename to WS_ReserveSize and macro-wrap WS_Reserve to emit #warn ? + */ unsigned WS_Reserve(struct ws *ws, unsigned bytes) { @@ -253,11 +277,7 @@ WS_Reserve(struct ws *ws, unsigned bytes) unsigned WS_ReserveLumps(struct ws *ws, size_t sz) { - unsigned u; - - u = WS_Reserve(ws, 0); - u /= sz; - return (u); + return (WS_ReserveAll(ws) / sz); } void diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index 5a3d01cab..26e4cc5da 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -225,7 +225,7 @@ h2_build_headers(struct vsb *resp, struct req *req) uint8_t buf[6]; ssize_t sz, sz1; - l = WS_Reserve(req->ws, 0); + l = WS_ReserveAll(req->ws); if (l < 10) { WS_Release(req->ws, 0); return (-1); diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 5a12f8890..85d374269 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -175,7 +175,7 @@ h2h_decode_init(const struct h2_sess *h2) d = h2->decode; INIT_OBJ(d, H2H_DECODE_MAGIC); VHD_Init(d->vhd); - d->out_l = WS_Reserve(h2->new_req->http->ws, 0); + d->out_l = WS_ReserveAll(h2->new_req->http->ws); /* * Can't do any work without any buffer * space. Require non-zero size. diff --git a/include/vrt.h b/include/vrt.h index 05ab88d7b..9f3e52e89 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -51,7 +51,9 @@ * Whenever something is deleted or changed in a way which is not * binary/load-time compatible, increment MAJOR version * - * + * unreleased (planned for 2019-09-15) + * [cache.h] WS_ReserveAll() added + * [cache.h] WS_Reserve(ws, 0) deprecated * 9.0 (2019-03-15) * Make 'len' in vmod_priv 'long' * HTTP_Copy() removed diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c index 2f8686edf..2a14e8b20 100644 --- a/lib/libvmod_blob/vmod_blob.c +++ b/lib/libvmod_blob/vmod_blob.c @@ -337,7 +337,7 @@ vmod_decode(VRT_CTX, VCL_ENUM decs, VCL_INT length, VCL_STRANDS strings) CHECK_OBJ_NOTNULL(ctx->ws, WS_MAGIC); buf = WS_Front(ctx->ws); - space = WS_Reserve(ctx->ws, 0); + space = WS_ReserveAll(ctx->ws); if (length <= 0) length = -1; @@ -376,7 +376,7 @@ encode(VRT_CTX, enum encoding enc, enum case_e kase, VCL_BLOB b) CHECK_OBJ_NOTNULL(ctx->ws, WS_MAGIC); snap = WS_Snapshot(ctx->ws); buf = WS_Front(ctx->ws); - space = WS_Reserve(ctx->ws, 0); + space = WS_ReserveAll(ctx->ws); len = func[enc].encode(enc, kase, buf, space, b->blob, b->len); diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index d35d89f6c..11e5f8667 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -72,7 +72,7 @@ vmod_updown(VRT_CTX, int up, const char *s, va_list ap) const char *p; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - u = WS_Reserve(ctx->ws, 0); + u = WS_ReserveAll(ctx->ws); e = b = ctx->ws->f; e += u; p = s; diff --git a/lib/libvmod_vtc/vmod_vtc.c b/lib/libvmod_vtc/vmod_vtc.c index 72320ba94..ee9d23fa0 100644 --- a/lib/libvmod_vtc/vmod_vtc.c +++ b/lib/libvmod_vtc/vmod_vtc.c @@ -155,7 +155,7 @@ vmod_workspace_alloc(VRT_CTX, VCL_ENUM which, VCL_INT size) WS_Assert(ws); if (size < 0) { - size += WS_Reserve(ws, 0); + size += WS_ReserveAll(ws); WS_Release(ws, 0); } if (size <= 0) { @@ -182,7 +182,7 @@ vmod_workspace_free(VRT_CTX, VCL_ENUM which) return(-1); WS_Assert(ws); - u = WS_Reserve(ws, 0); + u = WS_ReserveAll(ws); WS_Release(ws, 0); return (u); } From dridi at varni.sh Mon Apr 8 13:51:11 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 8 Apr 2019 15:51:11 +0200 Subject: [master] 3ce0d377e ws functions are not part of vrt, so mark them In-Reply-To: <20190408134510.7FCC1653B0@lists.varnish-cache.org> References: <20190408134510.7FCC1653B0@lists.varnish-cache.org> Message-ID: On Mon, Apr 8, 2019 at 3:45 PM Nils Goroll wrote: > > > commit 3ce0d377ed55cdb964cebf5d124c18d3c336576a > Author: Nils Goroll > Date: Mon Apr 8 15:18:44 2019 +0200 > > ws functions are not part of vrt, so mark them > > Yet having the changelog in one place is useful I thought cache.h was part of VRT and cache_varnishd.h was strict ABI. From phk at FreeBSD.org Mon Apr 8 14:20:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 8 Apr 2019 14:20:08 +0000 (UTC) Subject: [master] 0bd1f42ac Use a different host while server is being upgraded Message-ID: <20190408142008.5CA8D6E60E@lists.varnish-cache.org> commit 0bd1f42ac184a5095b52bee9c80ac5e199c9f11c Author: Poul-Henning Kamp Date: Mon Apr 8 14:18:41 2019 +0000 Use a different host while server is being upgraded diff --git a/bin/varnishtest/tests/v00016.vtc b/bin/varnishtest/tests/v00016.vtc index d41d750c3..0b8b7b474 100644 --- a/bin/varnishtest/tests/v00016.vtc +++ b/bin/varnishtest/tests/v00016.vtc @@ -85,7 +85,7 @@ varnish v1 -errvcl {Regexp compilation error:} { varnish v1 -errvcl {resolves to too many addresses} { backend b { .host = "127.0.0.1"; } sub vcl_recv { - if (remote.ip == "varnish-cache.org") {} + if (remote.ip == "r.freebsd.dk") {} } } From nils.goroll at uplex.de Mon Apr 8 14:59:21 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 8 Apr 2019 16:59:21 +0200 Subject: [master] 3ce0d377e ws functions are not part of vrt, so mark them In-Reply-To: References: <20190408134510.7FCC1653B0@lists.varnish-cache.org> Message-ID: <79a92ef1-f0a2-b3d3-9a20-14f7c0f1c00c@uplex.de> On 08/04/2019 15:51, Dridi Boukelmoune wrote: > I thought cache.h was part of VRT and cache_varnishd.h was strict ABI. um yes. And no. We might have a bit of confusion wrt VRT (pun intended) in the vmod $ABI sense (for which cache.h is forbidden) and the fact that is, after all, still part of our ABI. -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From dridi at varni.sh Tue Apr 9 08:34:38 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 9 Apr 2019 10:34:38 +0200 Subject: [master] 3ce0d377e ws functions are not part of vrt, so mark them In-Reply-To: <79a92ef1-f0a2-b3d3-9a20-14f7c0f1c00c@uplex.de> References: <20190408134510.7FCC1653B0@lists.varnish-cache.org> <79a92ef1-f0a2-b3d3-9a20-14f7c0f1c00c@uplex.de> Message-ID: On Mon, Apr 8, 2019 at 4:59 PM Nils Goroll wrote: > > On 08/04/2019 15:51, Dridi Boukelmoune wrote: > > I thought cache.h was part of VRT and cache_varnishd.h was strict ABI. > > um yes. And no. > > We might have a bit of confusion wrt VRT (pun intended) in the vmod $ABI sense > (for which cache.h is forbidden) and the fact that is, after all, still part of > our ABI. I thought the whole cache_varnishd.h split was done to allow cache.h in VRT. If we want to make cache.h $ABI strict, we should move the relevant WS_* prototypes to vrt.h at least to not make 90% of $ABI vrt modules turn strict. We can keep struct ws opaque, but those are critical for VMOD authors (it also means no more CHECK_OBJ_NOTNULL but AN instead on ctx->ws). Having to always rebuild a VMOD when Varnish updates because you make workspace allocations bring a poor out-of-tree experience. That forces everyone to use a priv_task instead for simple allocations, and that priv_task may need to keep track of multiple allocations too for each VMOD. Too much complexity for $ABI strict makes it unusable. Dridi From phk at FreeBSD.org Tue Apr 9 09:12:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 9 Apr 2019 09:12:08 +0000 (UTC) Subject: [master] e2b456919 Look for rst2html same way we look for rst2man and make it non-optional Message-ID: <20190409091208.48688AE3E5@lists.varnish-cache.org> commit e2b456919d42b015c44358358e738ead18fd96c3 Author: Poul-Henning Kamp Date: Tue Apr 9 09:11:33 2019 +0000 Look for rst2html same way we look for rst2man and make it non-optional diff --git a/configure.ac b/configure.ac index 78e47a8e4..fe4225069 100644 --- a/configure.ac +++ b/configure.ac @@ -60,14 +60,16 @@ if test "x$SPHINX" = "xno"; then fi AC_ARG_WITH([rst2html], - AS_HELP_STRING([--with-rst2html=PATH], - [Location of rst2html (auto)]), - [RST2HTML="$withval"], - [AC_CHECK_PROGS(RST2HTML, [rst2html rst2html.py], "no") - if test "x$RST2HTML" = "xno"; then - AC_MSG_WARN([rst2html not found - not building changelog]) - fi]) -AM_CONDITIONAL(HAVE_RST2HTML,[test "x$RST2HTML" != "xno"]) + AS_HELP_STRING([--with-rst2html=PATH], [Location of rst2html (auto)]), + [RST2HTML="$withval"], + AC_CHECK_PROGS(RST2HTML, + [rst2html rst2html.py rst2html-3.6 rst2html-3], + "no")) + +if test "x$RST2HTML" = "xno"; then + AC_MSG_ERROR( + [rst2html not found - (Weird, we found rst2man?!)]) +fi AC_ARG_WITH([dot], AS_HELP_STRING([--with-dot=PATH], diff --git a/doc/Makefile.am b/doc/Makefile.am index e420a99f1..f9f19ea27 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -5,14 +5,7 @@ RST2ANY_FLAGS = --halt=2 EXTRA_DIST = changes.rst changes.html changes.html: changes.rst -if HAVE_RST2HTML ${RST2HTML} ${RST2ANY_FLAGS} $? $@ -else - @echo "========================================" - @echo "You need rst2html installed to make dist" - @echo "========================================" - @false -endif # build graphviz before sphinx, so sphinx docs can use svg output SUBDIRS = graphviz sphinx From dridi at varni.sh Tue Apr 9 09:52:32 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 9 Apr 2019 11:52:32 +0200 Subject: [master] e2b456919 Look for rst2html same way we look for rst2man and make it non-optional In-Reply-To: <20190409091208.48688AE3E5@lists.varnish-cache.org> References: <20190409091208.48688AE3E5@lists.varnish-cache.org> Message-ID: On Tue, Apr 9, 2019 at 11:12 AM Poul-Henning Kamp wrote: > > > commit e2b456919d42b015c44358358e738ead18fd96c3 > Author: Poul-Henning Kamp > Date: Tue Apr 9 09:11:33 2019 +0000 > > Look for rst2html same way we look for rst2man and make it non-optional The whole point of `make dist` is that downstream users of a dist archive don't need autoconf, automake, libtool or any of the things that we build at dist-time. Making rst2* and sphinx-build truly optional would solve the polishing needed on the packaging side: we wouldn't even need them as build dependencies anymore. Can I revert this and send a patch to fix our documentation build instead? Dridi > diff --git a/configure.ac b/configure.ac > index 78e47a8e4..fe4225069 100644 > --- a/configure.ac > +++ b/configure.ac > @@ -60,14 +60,16 @@ if test "x$SPHINX" = "xno"; then > fi > > AC_ARG_WITH([rst2html], > - AS_HELP_STRING([--with-rst2html=PATH], > - [Location of rst2html (auto)]), > - [RST2HTML="$withval"], > - [AC_CHECK_PROGS(RST2HTML, [rst2html rst2html.py], "no") > - if test "x$RST2HTML" = "xno"; then > - AC_MSG_WARN([rst2html not found - not building changelog]) > - fi]) > -AM_CONDITIONAL(HAVE_RST2HTML,[test "x$RST2HTML" != "xno"]) > + AS_HELP_STRING([--with-rst2html=PATH], [Location of rst2html (auto)]), > + [RST2HTML="$withval"], > + AC_CHECK_PROGS(RST2HTML, > + [rst2html rst2html.py rst2html-3.6 rst2html-3], > + "no")) > + > +if test "x$RST2HTML" = "xno"; then > + AC_MSG_ERROR( > + [rst2html not found - (Weird, we found rst2man?!)]) > +fi > > AC_ARG_WITH([dot], > AS_HELP_STRING([--with-dot=PATH], > diff --git a/doc/Makefile.am b/doc/Makefile.am > index e420a99f1..f9f19ea27 100644 > --- a/doc/Makefile.am > +++ b/doc/Makefile.am > @@ -5,14 +5,7 @@ RST2ANY_FLAGS = --halt=2 > EXTRA_DIST = changes.rst changes.html > > changes.html: changes.rst > -if HAVE_RST2HTML > ${RST2HTML} ${RST2ANY_FLAGS} $? $@ > -else > - @echo "========================================" > - @echo "You need rst2html installed to make dist" > - @echo "========================================" > - @false > -endif > > # build graphviz before sphinx, so sphinx docs can use svg output > SUBDIRS = graphviz sphinx > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From phk at phk.freebsd.dk Tue Apr 9 19:33:55 2019 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Tue, 09 Apr 2019 19:33:55 +0000 Subject: [master] e2b456919 Look for rst2html same way we look for rst2man and make it non-optional In-Reply-To: References: <20190409091208.48688AE3E5@lists.varnish-cache.org> Message-ID: <2311.1554838435@critter.freebsd.dk> -------- In message , Dridi Boukelmoune writes: >> Look for rst2html same way we look for rst2man and make it non-optional > >The whole point of `make dist` is that downstream users of a dist >archive don't need autoconf, automake, libtool or any of the things >that we build at dist-time. > >Making rst2* and sphinx-build truly optional would solve the polishing >needed on the packaging side: we wouldn't even need them as build >dependencies anymore. > >Can I revert this and send a patch to fix our documentation build instead? As long as your fix also finds Py3 rst2html: yes. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From phk at FreeBSD.org Tue Apr 9 20:11:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 9 Apr 2019 20:11:07 +0000 (UTC) Subject: [master] c9b9c6bf8 Ditch references to the old trac/wiki Message-ID: <20190409201108.1873B6584B@lists.varnish-cache.org> commit c9b9c6bf8752dc3685a4b7fcfbfd1260699878e1 Author: Poul-Henning Kamp Date: Tue Apr 9 20:09:54 2019 +0000 Ditch references to the old trac/wiki diff --git a/doc/sphinx/installation/help.rst b/doc/sphinx/installation/help.rst index ba6dbd865..47f2dfe52 100644 --- a/doc/sphinx/installation/help.rst +++ b/doc/sphinx/installation/help.rst @@ -6,9 +6,11 @@ Getting hold of the gang behind Varnish is pretty straight forward, we try to help out as much as time permits and have tried to streamline this process as much as possible. -But before you grab hold of us, spend a moment composing your thoughts and -formulate your question. From our perspective there is nothing as pointless as simply telling -us "Varnish does not work for me" with no further information. This does not give us any relevant information to use when trying to figure out whats wrong. +But before you grab hold of us, spend a moment composing your +thoughts and formulate your question. From our perspective there +is nothing as pointless as simply telling us "Varnish does not work +for me" with no further information. This does not give us any +relevant information to use when trying to figure out whats wrong. And before you even do that, do a couple of searches to see if your question is already answered, if it has been, you will get your answer @@ -33,7 +35,7 @@ families and jobs to deal with also. You are more than welcome to just hang out, and while we don't mind the occasional intrusion from the real world into our flow, we try and keep it mostly on topic, and please don't paste random links unless they are -*really* spectacular and intelligent. +*really* funny, spectacular and intelligent. Mailing Lists ============= @@ -46,7 +48,6 @@ people spend some time getting Varnish running, and then more or less forget about it. Therefore the announce list is a good way to be reminded about new releases, bugs or potential (security) vulnerabilities. - The `varnish-misc` mailing list is for general banter, questions, suggestions, ideas and so on. If you are new to Varnish it may pay off to subscribe to it, simply to have an ear to the telegraph-pole @@ -59,46 +60,32 @@ thread changes, please change the subject to match, some of us deal with hundreds of emails per day, after spam-filters, and we need all the help we can get to pick the interesting ones. - The `varnish-dev` mailing list is used by the developers and is -usually quite focused on source-code and such. Everybody on -the `-dev` list is also on `-misc`, so cross-posting only serves to annoy -those people. - -We also maintain a community wiki_ for Varnish, there you will find information on planned events, meetings, current backlog, trouble tickets , and links to resources and documentation. - -.. XXX: we should introduce the wiki (if we care about it) before -.. we start referring to it (below). Make a wiki chapter? +usually quite focused on source-code and such. Everybody on the +`-dev` list is also on `-misc`, so cross-posting only serves to +annoy those people. Trouble Tickets =============== -Please do not open a trouble ticket, unless you have spotted an actual -bug in Varnish. Ask on IRC first if you are in doubt. +Our bugtracker lives on Github, but please do not open a trouble +ticket, unless you have spotted an actual bug in Varnish. Ask on +IRC first if you are in doubt. The reason for this policy, is to avoid bugs being drowned in a -pile of other `issues`, feature suggestions for future releases, and double postings of calls for help -from people who forgot to check back on already opened Tickets. +pile of other `issues`, feature suggestions for future releases, +and double postings of calls for help from people who forgot to +check back on already opened Tickets. -.. XXX: Not sure what you want with the last sentence above. benc +New ideas may get parked in our Github wiki, until we have time +for them, or until we have thought out a good design. -We instead track suggestions and feature ideas in our `"Future_Feature" wiki page`_, -and through user support via email and IRC. Commercial Support ================== -The following companies offer commercial Varnish support, and are listed -here for your convenience. If you want your company listed here, drop -an email to phk at FreeBSD.org. - - Varnish Software - sales at varnish-software.com - - UPLEX - info at uplex.de +If you need commercial support, there are companies which offer that +and you can find a `list on our homepage. `_. .. _mailman: https://www.varnish-cache.org/lists/mailman/listinfo .. _pastebin: https://gist.github.com/ -.. _"Future_Feature" wiki page: https://www.varnish-cache.org/trac/wiki/Future_Feature -.. _wiki: https://www.varnish-cache.org/trac diff --git a/doc/sphinx/tutorial/introduction.rst b/doc/sphinx/tutorial/introduction.rst index 99862cc0e..237ff1c12 100644 --- a/doc/sphinx/tutorial/introduction.rst +++ b/doc/sphinx/tutorial/introduction.rst @@ -53,28 +53,3 @@ Varnish is written to run on modern versions of Linux and FreeBSD and the best experience is had on those platforms. Thanks to our contributors it also runs on NetBSD, OpenBSD, OS X and various Solaris-descendants like Oracle Solaris, OmniOS and SmartOS. - -About the Varnish development process -------------------------------------- - -Varnish is a community driven project. The development is overseen by -the Varnish Governing Board which currently consists of Poul-Henning Kamp -(Architect), Rogier Mulhuijzen (Fastly) and Lasse Karstensen (Independent). - -Please see https://www.varnish-cache.org/trac/wiki/Contributing as a -starting point if you would like to contribute to Varnish. - -Getting in touch ----------------- - -You can get in touch with us through many channels. For real time chat -you can reach us on IRC through the server irc.linpro.net on the #varnish -and #varnish-hacking channels. -There are two mailing lists available: one for user questions and one -for development discussions. See https://www.varnish-cache.org/lists -for information and signup. There is also a web forum on the same site. - -Now that you have a vague idea on what Varnish Cache is, let's see if we -can get it up and running. - -.. XXX:The above three paragraphs are repetitive this is already handled in previous chapters. The only new information is Governing Board which could be moved to the introduction and the paragraphs scrapped. benc diff --git a/doc/sphinx/users-guide/increasing-your-hitrate.rst b/doc/sphinx/users-guide/increasing-your-hitrate.rst index 80eec9fdb..94e1bdbad 100644 --- a/doc/sphinx/users-guide/increasing-your-hitrate.rst +++ b/doc/sphinx/users-guide/increasing-your-hitrate.rst @@ -41,9 +41,6 @@ requests coming from the client matching `/foo/bar`. For more information on how :ref:`varnishlog(1)` works please see :ref:`users-guide-logging` or then man page. -For extended diagnostics headers, see -https://www.varnish-cache.org/trac/wiki/VCLExampleHitMissHeader - Tool: lwp-request ~~~~~~~~~~~~~~~~~ From nils.goroll at uplex.de Wed Apr 10 05:46:20 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 10 Apr 2019 07:46:20 +0200 Subject: [master] c9b9c6bf8 Ditch references to the old trac/wiki In-Reply-To: <20190409201108.1873B6584B@lists.varnish-cache.org> References: <20190409201108.1873B6584B@lists.varnish-cache.org> Message-ID: <8a4eb97b-a2d4-fb77-f87f-dfb1f9b9625f@uplex.de> Dridi, did you really mean to scrap the info about the VGB? Why? -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From phk at phk.freebsd.dk Wed Apr 10 05:53:09 2019 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Wed, 10 Apr 2019 05:53:09 +0000 Subject: [master] c9b9c6bf8 Ditch references to the old trac/wiki In-Reply-To: <8a4eb97b-a2d4-fb77-f87f-dfb1f9b9625f@uplex.de> References: <20190409201108.1873B6584B@lists.varnish-cache.org> <8a4eb97b-a2d4-fb77-f87f-dfb1f9b9625f@uplex.de> Message-ID: <12174.1554875589@critter.freebsd.dk> -------- In message <8a4eb97b-a2d4-fb77-f87f-dfb1f9b9625f at uplex.de>, Nils Goroll writes: >did you really mean to scrap the info about the VGB? Why? I did a survey of the entire homepage/docs setup yesterday and the VGB goes elsewhere. I sent Geoff a snapshot of my thinking in private email last night (A: My impression is he's the only one who cares, B: Not quite material for a public mailing list.) -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From dridi at varni.sh Wed Apr 10 05:52:50 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 10 Apr 2019 07:52:50 +0200 Subject: [master] c9b9c6bf8 Ditch references to the old trac/wiki In-Reply-To: <8a4eb97b-a2d4-fb77-f87f-dfb1f9b9625f@uplex.de> References: <20190409201108.1873B6584B@lists.varnish-cache.org> <8a4eb97b-a2d4-fb77-f87f-dfb1f9b9625f@uplex.de> Message-ID: On Wed, Apr 10, 2019 at 7:46 AM Nils Goroll wrote: > > Dridi, > > did you really mean to scrap the info about the VGB? Why? How did you know about my secret plan? I didn't leave any hints. Dridi From nils.goroll at uplex.de Wed Apr 10 06:59:53 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 10 Apr 2019 08:59:53 +0200 Subject: [master] 3ce0d377e ws functions are not part of vrt, so mark them In-Reply-To: References: <20190408134510.7FCC1653B0@lists.varnish-cache.org> <79a92ef1-f0a2-b3d3-9a20-14f7c0f1c00c@uplex.de> Message-ID: <356c6c8a-8350-956d-e5f0-8473d59b6843@uplex.de> I totally agree. Anything which basically all vmods need should be in vrt.h @phk ok to move WS_* to vrt.h ? Nils On 09/04/2019 10:34, Dridi Boukelmoune wrote: > On Mon, Apr 8, 2019 at 4:59 PM Nils Goroll wrote: >> >> On 08/04/2019 15:51, Dridi Boukelmoune wrote: >>> I thought cache.h was part of VRT and cache_varnishd.h was strict ABI. >> >> um yes. And no. >> >> We might have a bit of confusion wrt VRT (pun intended) in the vmod $ABI sense >> (for which cache.h is forbidden) and the fact that is, after all, still part of >> our ABI. > > I thought the whole cache_varnishd.h split was done to allow cache.h in VRT. > > If we want to make cache.h $ABI strict, we should move the relevant > WS_* prototypes to vrt.h at least to not make 90% of $ABI vrt modules > turn strict. We can keep struct ws opaque, but those are critical for > VMOD authors (it also means no more CHECK_OBJ_NOTNULL but AN > instead on ctx->ws). > > Having to always rebuild a VMOD when Varnish updates because you make > workspace allocations bring a poor out-of-tree experience. That forces > everyone to use a priv_task instead for simple allocations, and that priv_task > may need to keep track of multiple allocations too for each VMOD. > > Too much complexity for $ABI strict makes it unusable. > > Dridi > -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From dridi at varni.sh Wed Apr 10 07:36:02 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 10 Apr 2019 09:36:02 +0200 Subject: [master] 3ce0d377e ws functions are not part of vrt, so mark them In-Reply-To: <356c6c8a-8350-956d-e5f0-8473d59b6843@uplex.de> References: <20190408134510.7FCC1653B0@lists.varnish-cache.org> <79a92ef1-f0a2-b3d3-9a20-14f7c0f1c00c@uplex.de> <356c6c8a-8350-956d-e5f0-8473d59b6843@uplex.de> Message-ID: On Wed, Apr 10, 2019 at 8:59 AM Nils Goroll wrote: > > I totally agree. Anything which basically all vmods need should be in vrt.h > > @phk ok to move WS_* to vrt.h ? And in my last sentence I meant the other way around: > Too much complexity for $ABI vrt (not strict) makes it unusable. Dridi From phk at phk.freebsd.dk Wed Apr 10 08:05:11 2019 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Wed, 10 Apr 2019 08:05:11 +0000 Subject: [master] 3ce0d377e ws functions are not part of vrt, so mark them In-Reply-To: <356c6c8a-8350-956d-e5f0-8473d59b6843@uplex.de> References: <20190408134510.7FCC1653B0@lists.varnish-cache.org> <79a92ef1-f0a2-b3d3-9a20-14f7c0f1c00c@uplex.de> <356c6c8a-8350-956d-e5f0-8473d59b6843@uplex.de> Message-ID: <12842.1554883511@critter.freebsd.dk> -------- In message <356c6c8a-8350-956d-e5f0-8473d59b6843 at uplex.de>, Nils Goroll writes: >@phk ok to move WS_* to vrt.h ? You are not going to get a fast answer to this one, and I want to use this as an example of why sometimes things take time around here. My initial leaning is: Hell NO! We're not doing nearly as good a job as I wish of keeping VRT "isolated" from varnishd internals, but that doesn't mean I'm going to throw it all overboard. If we put WS_* into vrt.h we are essentially stuck with it forever. So far I have very deliberately kept WS out of vrt.h because I want us to retain the ability to redo the basic WS concept if the static allocation becomes unworkable down the road, and it is starting to look a bit geriatric these days. The alternative to just making WS be VRT, is to add some VRT memory allocation functions which wrap WS, and I would be much more receptive to that. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From nils.goroll at uplex.de Wed Apr 10 08:13:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 10 Apr 2019 10:13:08 +0200 Subject: [master] 3ce0d377e ws functions are not part of vrt, so mark them In-Reply-To: <12842.1554883511@critter.freebsd.dk> References: <20190408134510.7FCC1653B0@lists.varnish-cache.org> <79a92ef1-f0a2-b3d3-9a20-14f7c0f1c00c@uplex.de> <356c6c8a-8350-956d-e5f0-8473d59b6843@uplex.de> <12842.1554883511@critter.freebsd.dk> Message-ID: <357f8ed2-ae05-059c-bea8-6398c923dd14@uplex.de> On 10/04/2019 10:05, Poul-Henning Kamp wrote: > The alternative to just making WS be VRT, is to add some VRT memory > allocation functions which wrap WS, and I would be much more receptive to that. All right, this sounds like a very sensible way forward. I will try to come up with something. -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From nils.goroll at uplex.de Wed Apr 10 09:29:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 10 Apr 2019 09:29:07 +0000 (UTC) Subject: [master] 4442c6d7c move busyobj cancel handling to a seprate function Message-ID: <20190410092907.F37E4ACBD3@lists.varnish-cache.org> commit 4442c6d7ccc81ebade83846d832c93607f007457 Author: Nils Goroll Date: Sun Apr 7 18:17:52 2019 +0200 move busyobj cancel handling to a seprate function diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 1395f7101..0d54fd48c 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -765,6 +765,35 @@ HSH_Abandon(struct objcore *oc) Lck_Unlock(&oh->mtx); } +/*--------------------------------------------------------------------- + * Cancel a fetch when the client does not need it any more + */ + +void +HSH_Cancel(struct worker *wrk, struct objcore *oc, struct boc *boc) +{ + struct boc *bocref = NULL; + + if ((oc->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP)) == 0) + return; + + if (boc == NULL) + bocref = boc = HSH_RefBoc(oc); + + if (oc->flags & OC_F_HFP) + AN(oc->flags & OC_F_HFM); + + if (boc != NULL) { + HSH_Abandon(oc); + ObjWaitState(oc, BOS_FINISHED); + } + + if (bocref != NULL) + HSH_DerefBoc(wrk, oc); + + ObjSlim(wrk, oc); +} + /*--------------------------------------------------------------------- * Unbusy an objcore when the object is completely fetched. */ diff --git a/bin/varnishd/cache/cache_objhead.h b/bin/varnishd/cache/cache_objhead.h index 41b7781a5..c2755dce1 100644 --- a/bin/varnishd/cache/cache_objhead.h +++ b/bin/varnishd/cache/cache_objhead.h @@ -74,3 +74,4 @@ unsigned HSH_Purge(struct worker *, struct objhead *, vtim_real ttl_now, vtim_dur ttl, vtim_dur grace, vtim_dur keep); struct objcore *HSH_Private(const struct worker *wrk); void HSH_Abandon(struct objcore *oc); +void HSH_Cancel(struct worker *, struct objcore *, struct boc *); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index a310cd32b..c6764e0c2 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -404,15 +404,7 @@ cnt_transmit(struct worker *wrk, struct req *req) VSLb_ts_req(req, "Resp", W_TIM_real(wrk)); - if (req->objcore->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP)) { - if (req->objcore->flags & OC_F_HFP) - AN(req->objcore->flags & OC_F_HFM); - if (boc != NULL) { - HSH_Abandon(req->objcore); - ObjWaitState(req->objcore, BOS_FINISHED); - } - ObjSlim(wrk, req->objcore); - } + HSH_Cancel(wrk, req->objcore, boc); if (boc != NULL) HSH_DerefBoc(wrk, req->objcore); From nils.goroll at uplex.de Wed Apr 10 09:29:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 10 Apr 2019 09:29:08 +0000 (UTC) Subject: [master] 32ef5dce5 Cancel a request for a private (streaming) object from vcl_deliver Message-ID: <20190410092908.1A2DEACBD6@lists.varnish-cache.org> commit 32ef5dce5dd3d0e435186fc638e3749ea3e7b3c0 Author: Nils Goroll Date: Sun Apr 7 18:18:03 2019 +0200 Cancel a request for a private (streaming) object from vcl_deliver for anything but return (deliver) Fixes #2963 diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index c6764e0c2..6a367bed7 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -160,6 +160,7 @@ cnt_deliver(struct worker *wrk, struct req *req) assert(req->restarts <= cache_param->max_restarts); if (wrk->handling != VCL_RET_DELIVER) { + HSH_Cancel(wrk, req->objcore, NULL); (void)HSH_DerefObjCore(wrk, &req->objcore, HSH_RUSH_POLICY); http_Teardown(req->resp); diff --git a/bin/varnishtest/tests/r02964.vtc b/bin/varnishtest/tests/r02964.vtc new file mode 100644 index 000000000..89f3d4184 --- /dev/null +++ b/bin/varnishtest/tests/r02964.vtc @@ -0,0 +1,49 @@ +varnishtest "Cancel private busy obj from vcl_deliver" + +server s1 { + rxreq + expect req.url == "/hfm" + txresp -hdr "HFM: True" -bodylen 65530 + rxreq + expect req.url == "/hfp" + txresp -hdr "HFP: True" -bodylen 65550 +} -start + +varnish v1 -arg "-s Transient=default" -vcl+backend { + sub vcl_recv { + if (req.restarts > 0) { + return (synth(200)); + } + } + sub vcl_backend_response { + if (bereq.url == "/hfm") { + set beresp.uncacheable = true; + } else if (bereq.url == "/hfp") { + return (pass(1m)); + } + } + sub vcl_deliver { + if (req.restarts == 0) { + return (restart); + } + } +} -start + +logexpect l1 -v v1 -g raw { + expect * * Storage "Transient" + expect * * Storage "Transient" +} -start + +client c1 { + txreq -url "/hfm" + rxresp + expect resp.status == 200 + txreq -url "/hfp" + rxresp + expect resp.status == 200 +} -run + +logexpect l1 -wait + +varnish v1 -expect SM?.Transient.c_bytes > 131072 +varnish v1 -expect SM?.Transient.g_bytes < 2000 From dridi at varni.sh Wed Apr 10 09:38:30 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 10 Apr 2019 11:38:30 +0200 Subject: [master] 4442c6d7c move busyobj cancel handling to a seprate function In-Reply-To: <20190410092907.F37E4ACBD3@lists.varnish-cache.org> References: <20190410092907.F37E4ACBD3@lists.varnish-cache.org> Message-ID: On Wed, Apr 10, 2019 at 11:29 AM Nils Goroll wrote: > > > commit 4442c6d7ccc81ebade83846d832c93607f007457 > Author: Nils Goroll > Date: Sun Apr 7 18:17:52 2019 +0200 > > move busyobj cancel handling to a seprate function > > diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c > index 1395f7101..0d54fd48c 100644 > --- a/bin/varnishd/cache/cache_hash.c > +++ b/bin/varnishd/cache/cache_hash.c > @@ -765,6 +765,35 @@ HSH_Abandon(struct objcore *oc) > Lck_Unlock(&oh->mtx); > } > > +/*--------------------------------------------------------------------- > + * Cancel a fetch when the client does not need it any more > + */ > + > +void > +HSH_Cancel(struct worker *wrk, struct objcore *oc, struct boc *boc) > +{ > + struct boc *bocref = NULL; > + > + if ((oc->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP)) == 0) > + return; > + > + if (boc == NULL) > + bocref = boc = HSH_RefBoc(oc); Maybe drop a comment regarding the line above. I didn't understand the intent on the first read. > + > + if (oc->flags & OC_F_HFP) > + AN(oc->flags & OC_F_HFM); > + > + if (boc != NULL) { > + HSH_Abandon(oc); > + ObjWaitState(oc, BOS_FINISHED); > + } > + > + if (bocref != NULL) > + HSH_DerefBoc(wrk, oc); > + > + ObjSlim(wrk, oc); > +} > + > /*--------------------------------------------------------------------- > * Unbusy an objcore when the object is completely fetched. > */ > diff --git a/bin/varnishd/cache/cache_objhead.h b/bin/varnishd/cache/cache_objhead.h > index 41b7781a5..c2755dce1 100644 > --- a/bin/varnishd/cache/cache_objhead.h > +++ b/bin/varnishd/cache/cache_objhead.h > @@ -74,3 +74,4 @@ unsigned HSH_Purge(struct worker *, struct objhead *, vtim_real ttl_now, > vtim_dur ttl, vtim_dur grace, vtim_dur keep); > struct objcore *HSH_Private(const struct worker *wrk); > void HSH_Abandon(struct objcore *oc); > +void HSH_Cancel(struct worker *, struct objcore *, struct boc *); > diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c > index a310cd32b..c6764e0c2 100644 > --- a/bin/varnishd/cache/cache_req_fsm.c > +++ b/bin/varnishd/cache/cache_req_fsm.c > @@ -404,15 +404,7 @@ cnt_transmit(struct worker *wrk, struct req *req) > > VSLb_ts_req(req, "Resp", W_TIM_real(wrk)); > > - if (req->objcore->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP)) { > - if (req->objcore->flags & OC_F_HFP) > - AN(req->objcore->flags & OC_F_HFM); > - if (boc != NULL) { > - HSH_Abandon(req->objcore); > - ObjWaitState(req->objcore, BOS_FINISHED); > - } > - ObjSlim(wrk, req->objcore); > - } > + HSH_Cancel(wrk, req->objcore, boc); > > if (boc != NULL) > HSH_DerefBoc(wrk, req->objcore); > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From nils.goroll at uplex.de Wed Apr 10 09:57:06 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 10 Apr 2019 09:57:06 +0000 (UTC) Subject: [master] 819241d2a Add a comment Message-ID: <20190410095706.C7E80AEA21@lists.varnish-cache.org> commit 819241d2a78425a752efc591aa76d12c25171307 Author: Nils Goroll Date: Wed Apr 10 11:56:08 2019 +0200 Add a comment as requested by @Dridi diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 0d54fd48c..b108a3062 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -777,6 +777,10 @@ HSH_Cancel(struct worker *wrk, struct objcore *oc, struct boc *boc) if ((oc->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP)) == 0) return; + /* + * the caller-provided boc is optional, if they do not have one, we grab + * a ref + */ if (boc == NULL) bocref = boc = HSH_RefBoc(oc); From nils.goroll at uplex.de Wed Apr 10 11:14:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 10 Apr 2019 11:14:07 +0000 (UTC) Subject: [master] 429835da3 try to stabilize this test Message-ID: <20190410111407.48E87B01EF@lists.varnish-cache.org> commit 429835da32f9b282e07fa895b6d506b063884bc6 Author: Nils Goroll Date: Wed Apr 10 13:13:27 2019 +0200 try to stabilize this test Ref #2963 diff --git a/bin/varnishtest/tests/r02964.vtc b/bin/varnishtest/tests/r02964.vtc index 89f3d4184..06f1c89c6 100644 --- a/bin/varnishtest/tests/r02964.vtc +++ b/bin/varnishtest/tests/r02964.vtc @@ -4,6 +4,7 @@ server s1 { rxreq expect req.url == "/hfm" txresp -hdr "HFM: True" -bodylen 65530 + accept rxreq expect req.url == "/hfp" txresp -hdr "HFP: True" -bodylen 65550 @@ -15,6 +16,9 @@ varnish v1 -arg "-s Transient=default" -vcl+backend { return (synth(200)); } } + sub vcl_backend_fetch { + set bereq.http.Connection = "close"; + } sub vcl_backend_response { if (bereq.url == "/hfm") { set beresp.uncacheable = true; From dridi.boukelmoune at gmail.com Wed Apr 10 11:24:07 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 10 Apr 2019 11:24:07 +0000 (UTC) Subject: [master] d86f1850b Clarify the comment Message-ID: <20190410112407.C9C63B079F@lists.varnish-cache.org> commit d86f1850bc6ebabd61ef7274e2ff06784be0669f Author: Dridi Boukelmoune Date: Wed Apr 10 13:21:49 2019 +0200 Clarify the comment This is the intent that was not obvious to me when I first reviewed this change. diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index b108a3062..5576ca6fd 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -778,8 +778,8 @@ HSH_Cancel(struct worker *wrk, struct objcore *oc, struct boc *boc) return; /* - * the caller-provided boc is optional, if they do not have one, we grab - * a ref + * NB: we use two distinct variables to only release the reference if + * we had to acquire one. The caller-provided boc is optional. */ if (boc == NULL) bocref = boc = HSH_RefBoc(oc); From nils.goroll at uplex.de Wed Apr 10 11:52:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 10 Apr 2019 11:52:07 +0000 (UTC) Subject: [master] 1842a555f more stabilization Message-ID: <20190410115207.535A924DD@lists.varnish-cache.org> commit 1842a555f245fafbb5dce77afba6d12a52eb62d4 Author: Nils Goroll Date: Wed Apr 10 13:50:01 2019 +0200 more stabilization I had overlooked the fact that for a canceled request we might only read parts of the body. Interestingly, this is only exposed on the vtest ARMs. Ref #2964 diff --git a/bin/varnishtest/tests/r02964.vtc b/bin/varnishtest/tests/r02964.vtc index 06f1c89c6..12cc4d603 100644 --- a/bin/varnishtest/tests/r02964.vtc +++ b/bin/varnishtest/tests/r02964.vtc @@ -49,5 +49,5 @@ client c1 { logexpect l1 -wait -varnish v1 -expect SM?.Transient.c_bytes > 131072 +varnish v1 -expect SM?.Transient.c_bytes > 0 varnish v1 -expect SM?.Transient.g_bytes < 2000 From nils.goroll at uplex.de Wed Apr 10 12:55:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 10 Apr 2019 12:55:08 +0000 (UTC) Subject: [master] 7cee37c00 more test stabilization Message-ID: <20190410125508.51DBA4901@lists.varnish-cache.org> commit 7cee37c00a5721b85d5f10b030acfd52ca7a07fa Author: Nils Goroll Date: Wed Apr 10 14:52:57 2019 +0200 more test stabilization With 32ef5dce5dd3d0e435186fc638e3749ea3e7b3c0, this test also cancels backend requests and thus may prevent the server from finishing writing the reponse body Ref #2964 diff --git a/bin/varnishtest/tests/d00022.vtc b/bin/varnishtest/tests/d00022.vtc index 08130a15b..9344234b7 100644 --- a/bin/varnishtest/tests/d00022.vtc +++ b/bin/varnishtest/tests/d00022.vtc @@ -3,8 +3,10 @@ varnishtest "shard director Restarts" server s1 { rxreq txresp -body "ech3Ooj" + accept rxreq txresp -body "ech3Ooj" + accept rxreq txresp -body "ech3Ooj" } -start @@ -12,8 +14,10 @@ server s1 { server s2 { rxreq txresp -body "ieQu2qua" + accept rxreq txresp -body "ieQu2qua" + accept rxreq txresp -body "ieQu2qua" } -start @@ -21,8 +25,10 @@ server s2 { server s3 { rxreq txresp -body "xiuFi3Pe" + accept rxreq txresp -body "xiuFi3Pe" + accept rxreq txresp -body "xiuFi3Pe" } -start @@ -38,6 +44,10 @@ varnish v1 -vcl+backend { vd.reconfigure(25); } + sub vcl_backend_fetch { + set bereq.http.Connection = "close"; + } + sub vcl_recv { set req.backend_hint = vd.backend(by=KEY, key=1756955383, From dridi.boukelmoune at gmail.com Wed Apr 10 16:28:06 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 10 Apr 2019 16:28:06 +0000 (UTC) Subject: [master] cc3cd17b5 Assert Message-ID: <20190410162806.65B1463906@lists.varnish-cache.org> commit cc3cd17b51e289822fcdc46b76d03fd5d48e632d Author: Dridi Boukelmoune Date: Wed Apr 10 18:26:42 2019 +0200 Assert diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 5576ca6fd..0d22eeae8 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -774,6 +774,8 @@ HSH_Cancel(struct worker *wrk, struct objcore *oc, struct boc *boc) { struct boc *bocref = NULL; + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + if ((oc->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP)) == 0) return; @@ -784,6 +786,8 @@ HSH_Cancel(struct worker *wrk, struct objcore *oc, struct boc *boc) if (boc == NULL) bocref = boc = HSH_RefBoc(oc); + CHECK_OBJ_ORNULL(boc, BOC_MAGIC); + if (oc->flags & OC_F_HFP) AN(oc->flags & OC_F_HFM); From fgsch at lodoss.net Wed Apr 10 21:10:09 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 10 Apr 2019 21:10:09 +0000 (UTC) Subject: [master] fe83bc4a8 Make lgtm extraction happy Message-ID: <20190410211009.CC3C893554@lists.varnish-cache.org> commit fe83bc4a8b418995b565d99d8c987c32c386b3bb Author: Federico G. Schwindt Date: Wed Apr 10 20:00:06 2019 +0100 Make lgtm extraction happy diff --git a/.lgtm.yml b/.lgtm.yml new file mode 100644 index 000000000..578bfad28 --- /dev/null +++ b/.lgtm.yml @@ -0,0 +1,4 @@ +extraction: + cpp: + prepare: + packages: "python3.7" From fgsch at lodoss.net Wed Apr 10 21:10:09 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 10 Apr 2019 21:10:09 +0000 (UTC) Subject: [master] 490a0c7c2 Drop unneded include Message-ID: <20190410211009.D7EAE93557@lists.varnish-cache.org> commit 490a0c7c23b91e2ae1e20cd88ed51397ba7702f7 Author: Federico G. Schwindt Date: Wed Apr 10 20:01:56 2019 +0100 Drop unneded include diff --git a/lib/libvmod_debug/vmod_debug_dyn.c b/lib/libvmod_debug/vmod_debug_dyn.c index e8085394e..fca6bdb86 100644 --- a/lib/libvmod_debug/vmod_debug_dyn.c +++ b/lib/libvmod_debug/vmod_debug_dyn.c @@ -32,7 +32,6 @@ #include #include #include -#include #include #include From fgsch at lodoss.net Wed Apr 10 21:10:09 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 10 Apr 2019 21:10:09 +0000 (UTC) Subject: [master] 7b00d5dfc Minor polishing Message-ID: <20190410211009.ED81B9355A@lists.varnish-cache.org> commit 7b00d5dfc31b923b4bdbdd9dbf14e975db832e9b Author: Federico G. Schwindt Date: Wed Apr 10 20:02:07 2019 +0100 Minor polishing diff --git a/tools/include_wash.py b/tools/include_wash.py index 61a0cae91..fdf7f366d 100644 --- a/tools/include_wash.py +++ b/tools/include_wash.py @@ -29,11 +29,11 @@ def check(fn): for i in ("cache/cache.h", "cache.h"): if i in l: - for i in ("stddef.h", "stdint.h", "vrt.h", + for j in ("stddef.h", "stdint.h", "vrt.h", "math.h", "pthread.h", "stdarg.h", "sys/types.h", "vdef.h", "miniobj.h", "vas.h", "vqueue.h", "vtree.h"): - if i in l: - print(fn, i + " included with cache.h") + if j in l: + print(fn, j + " included with cache.h") for (dir, dns, fns) in os.walk("."): for f in fns: From fgsch at lodoss.net Wed Apr 10 21:10:10 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 10 Apr 2019 21:10:10 +0000 (UTC) Subject: [master] 44677514a Use the stage should we have more in the future Message-ID: <20190410211010.199B39355E@lists.varnish-cache.org> commit 44677514a451cb5bfbf0c9868b72e9c901bf4d43 Author: Federico G. Schwindt Date: Wed Apr 10 20:07:19 2019 +0100 Use the stage should we have more in the future diff --git a/.travis.yml b/.travis.yml index 5f08d0109..2a19d2751 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ language: c jobs: allow_failures: - os: osx - - env: ASAN=1 UBSAN=1 + - stage: sanitizers fast_finish: true include: - &test-linux From fgsch at lodoss.net Wed Apr 10 22:42:07 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 10 Apr 2019 22:42:07 +0000 (UTC) Subject: [master] 9e1e0972c Polish Message-ID: <20190410224207.A885C95901@lists.varnish-cache.org> commit 9e1e0972c47c5ea03350c39c76370ccafd678802 Author: Federico G. Schwindt Date: Wed Apr 10 23:35:45 2019 +0100 Polish diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 85d374269..0b4b3f78d 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -141,7 +141,7 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len) if (n < HTTP_HDR_FIRST) { /* Check for duplicate pseudo-header */ - if (n < HTTP_HDR_FIRST && hp->hd[n].b != NULL) { + if (hp->hd[n].b != NULL) { VSLb(hp->vsl, SLT_BogoHeader, "Duplicate pseudo-header: %.*s", (int)(len > 20 ? 20 : len), b); From phk at FreeBSD.org Mon Apr 15 07:21:06 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Apr 2019 07:21:06 +0000 (UTC) Subject: [master] f16e95f8e Make "default" a fully reserved word in syntax 4.0 also Message-ID: <20190415072106.73CD260BA6@lists.varnish-cache.org> commit f16e95f8ea9efb6459694608eaba6cd5f7f4b0ac Author: Poul-Henning Kamp Date: Mon Apr 15 07:20:25 2019 +0000 Make "default" a fully reserved word in syntax 4.0 also Fixes #2959 diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c index faa929d5f..f47991748 100644 --- a/lib/libvcc/vcc_parse.c +++ b/lib/libvcc/vcc_parse.c @@ -357,7 +357,7 @@ static struct toplev { { "probe", vcc_ParseProbe, VCL_41, VCL_HIGH }, { "import", vcc_ParseImport, VCL_41, VCL_HIGH }, { "vcl", vcc_ParseVcl, VCL_41, VCL_HIGH }, - { "default", NULL, VCL_41, VCL_HIGH }, + { "default", NULL, VCL_40, VCL_HIGH }, { NULL, NULL } }; diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index 67d4d8707..6ab7defe4 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -146,7 +146,7 @@ VCC_Symbol(struct vcc *tl, struct symbol *parent, break; if ((kind == SYM_NONE && kind == sym->kind)) continue; - if (tl->syntax < VCL_41 && + if (tl->syntax < VCL_41 && strcmp(sym->name, "default") && (kind != SYM_NONE && kind != sym->kind)) continue; break; From dridi.boukelmoune at gmail.com Mon Apr 15 07:39:06 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 15 Apr 2019 07:39:06 +0000 (UTC) Subject: [master] 8234b485d Bump etc/example.vcl syntax to 4.1 Message-ID: <20190415073906.A49C86126E@lists.varnish-cache.org> commit 8234b485ddffdd2ab25bf4e73f09143176c33647 Author: Dridi Boukelmoune Date: Mon Apr 15 09:37:51 2019 +0200 Bump etc/example.vcl syntax to 4.1 Closes #2948 diff --git a/etc/example.vcl b/etc/example.vcl index 8302c260f..050396324 100644 --- a/etc/example.vcl +++ b/etc/example.vcl @@ -9,8 +9,8 @@ # and https://www.varnish-cache.org/trac/wiki/VCLExamples for more examples. # Marker to tell the VCL compiler that this VCL has been adapted to the -# new 4.0 format. -vcl 4.0; +# 4.0 or 4.1 syntax. +vcl 4.1; # Default backend definition. Set this to point to your content server. backend default { From dridi.boukelmoune at gmail.com Mon Apr 15 07:43:06 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 15 Apr 2019 07:43:06 +0000 (UTC) Subject: [master] 8e138d602 Don't reference the old TRAC wiki in example.vcl Message-ID: <20190415074306.32D2D6151C@lists.varnish-cache.org> commit 8e138d6026d2544fb8cdad8f8cecd81a54dbdbd7 Author: Dridi Boukelmoune Date: Mon Apr 15 09:41:38 2019 +0200 Don't reference the old TRAC wiki in example.vcl Refs #2948 diff --git a/etc/example.vcl b/etc/example.vcl index 050396324..a48247a93 100644 --- a/etc/example.vcl +++ b/etc/example.vcl @@ -5,10 +5,10 @@ # builtin VCL. The builtin VCL is called when there is no explicit # return statement. # -# See the VCL chapters in the Users Guide at https://www.varnish-cache.org/docs/ -# and https://www.varnish-cache.org/trac/wiki/VCLExamples for more examples. +# See the VCL chapters in the Users Guide for a comprehensive documentation +# at https://www.varnish-cache.org/docs/. -# Marker to tell the VCL compiler that this VCL has been adapted to the +# Marker to tell the VCL compiler that this VCL has been written with the # 4.0 or 4.1 syntax. vcl 4.1; From dridi.boukelmoune at gmail.com Mon Apr 15 07:57:07 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 15 Apr 2019 07:57:07 +0000 (UTC) Subject: [master] f75b694dc New MAIN.n_pipe gauge Message-ID: <20190415075707.1FB1661A81@lists.varnish-cache.org> commit f75b694dc8453c94a514d6dd2bf2a42a65e2ef61 Author: Dridi Boukelmoune Date: Thu Apr 11 19:21:33 2019 +0200 New MAIN.n_pipe gauge This allows users to keep track of the number of ongoing pipe transactions at any time. diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc index 0c22fead5..91d17fc8b 100644 --- a/bin/varnishd/VSC_main.vsc +++ b/bin/varnishd/VSC_main.vsc @@ -394,6 +394,11 @@ :oneliner: Total sessions seen +.. varnish_vsc:: n_pipe + :type: gauge + :oneliner: Number of ongoing pipe sessions + + .. varnish_vsc:: s_pipe :group: wrk :oneliner: Total pipe sessions seen diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 6a367bed7..fd4940167 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -44,6 +44,7 @@ #include "cache_transport.h" #include "hash/hash_slinger.h" +#include "http1/cache_http1.h" #include "storage/storage.h" #include "common/heritage.h" #include "vcl.h" @@ -704,11 +705,13 @@ cnt_pipe(struct worker *wrk, struct req *req) nxt = REQ_FSM_MORE; break; case VCL_RET_PIPE: + XXXAZ(V1P_Enter()); AZ(bo->req); bo->req = req; bo->wrk = wrk; SES_Close(req->sp, VDI_Http1Pipe(req, bo)); nxt = REQ_FSM_DONE; + V1P_Leave(); break; default: WRONG("Illegal return from vcl_pipe{}"); diff --git a/bin/varnishd/http1/cache_http1.h b/bin/varnishd/http1/cache_http1.h index a6558a22f..f642046cf 100644 --- a/bin/varnishd/http1/cache_http1.h +++ b/bin/varnishd/http1/cache_http1.h @@ -50,6 +50,8 @@ struct v1p_acct { uint64_t out; }; +int V1P_Enter(void); +void V1P_Leave(void); void V1P_Process(const struct req *, int fd, struct v1p_acct *); void V1P_Charge(struct req *, const struct v1p_acct *, struct VSC_vbe *); diff --git a/bin/varnishd/http1/cache_http1_pipe.c b/bin/varnishd/http1/cache_http1_pipe.c index 224298b01..b3c6e6e25 100644 --- a/bin/varnishd/http1/cache_http1_pipe.c +++ b/bin/varnishd/http1/cache_http1_pipe.c @@ -62,6 +62,26 @@ rdf(int fd0, int fd1, uint64_t *pcnt) return (0); } +int +V1P_Enter(void) +{ + + Lck_Lock(&pipestat_mtx); + VSC_C_main->n_pipe++; + Lck_Unlock(&pipestat_mtx); + return (0); +} + +void +V1P_Leave(void) +{ + + Lck_Lock(&pipestat_mtx); + assert(VSC_C_main->n_pipe > 0); + VSC_C_main->n_pipe--; + Lck_Unlock(&pipestat_mtx); +} + void V1P_Charge(struct req *req, const struct v1p_acct *a, struct VSC_vbe *b) { diff --git a/bin/varnishtest/tests/v00059.vtc b/bin/varnishtest/tests/v00059.vtc new file mode 100644 index 000000000..0cb4dc868 --- /dev/null +++ b/bin/varnishtest/tests/v00059.vtc @@ -0,0 +1,55 @@ +varnishtest "n_pipe gauge" + +barrier b1 cond 2 +barrier b2 cond 2 +barrier b3 cond 2 +barrier b4 cond 2 + +server s1 { + rxreq + barrier b1 sync + barrier b2 sync + txresp +} -start + +server s2 { + rxreq + barrier b3 sync + barrier b4 sync + txresp +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + if (req.url == "/c1") { + set req.backend_hint = s1; + } + elsif (req.url == "/c2") { + set req.backend_hint = s2; + } + return (pipe); + } +} -start + +varnish v1 -expect MAIN.n_pipe == 0 + +client c1 { + txreq -url "/c1" + rxresp +} -start + +barrier b1 sync + +varnish v1 -expect MAIN.n_pipe == 1 + +client c2 { + txreq -url "/c2" + rxresp +} -start + +barrier b3 sync +varnish v1 -expect MAIN.n_pipe == 2 +barrier b2 sync +varnish v1 -expect MAIN.n_pipe == 1 +barrier b4 sync +varnish v1 -expect MAIN.n_pipe == 0 From dridi.boukelmoune at gmail.com Mon Apr 15 07:57:07 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 15 Apr 2019 07:57:07 +0000 (UTC) Subject: [master] b58eb24f3 New pipe_sess_max runtime parameter Message-ID: <20190415075707.3B15061A84@lists.varnish-cache.org> commit b58eb24f334d965867c5f01de619016b40c51ea3 Author: Dridi Boukelmoune Date: Thu Apr 11 19:45:30 2019 +0200 New pipe_sess_max runtime parameter Optionally, it can limit the number of concurrent pipe transactions as they each monopolize one worker thread. diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index fd4940167..5fb7f615e 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -696,22 +696,24 @@ cnt_pipe(struct worker *wrk, struct req *req) VCL_pipe_method(req->vcl, wrk, req, bo, NULL); switch (wrk->handling) { - case VCL_RET_FAIL: - req->req_step = R_STP_VCLFAIL; - nxt = REQ_FSM_MORE; - break; case VCL_RET_SYNTH: req->req_step = R_STP_SYNTH; nxt = REQ_FSM_MORE; break; case VCL_RET_PIPE: - XXXAZ(V1P_Enter()); - AZ(bo->req); - bo->req = req; - bo->wrk = wrk; - SES_Close(req->sp, VDI_Http1Pipe(req, bo)); - nxt = REQ_FSM_DONE; - V1P_Leave(); + if (V1P_Enter() == 0) { + AZ(bo->req); + bo->req = req; + bo->wrk = wrk; + SES_Close(req->sp, VDI_Http1Pipe(req, bo)); + nxt = REQ_FSM_DONE; + V1P_Leave(); + break; + } + /* fall through */ + case VCL_RET_FAIL: + req->req_step = R_STP_VCLFAIL; + nxt = REQ_FSM_MORE; break; default: WRONG("Illegal return from vcl_pipe{}"); diff --git a/bin/varnishd/http1/cache_http1_pipe.c b/bin/varnishd/http1/cache_http1_pipe.c index b3c6e6e25..420005641 100644 --- a/bin/varnishd/http1/cache_http1_pipe.c +++ b/bin/varnishd/http1/cache_http1_pipe.c @@ -65,11 +65,16 @@ rdf(int fd0, int fd1, uint64_t *pcnt) int V1P_Enter(void) { + int retval = 0; Lck_Lock(&pipestat_mtx); - VSC_C_main->n_pipe++; + if (cache_param->pipe_sess_max == 0 || + VSC_C_main->n_pipe < cache_param->pipe_sess_max) + VSC_C_main->n_pipe++; + else + retval = -1; Lck_Unlock(&pipestat_mtx); - return (0); + return (retval); } void diff --git a/bin/varnishtest/tests/v00059.vtc b/bin/varnishtest/tests/v00059.vtc index 0cb4dc868..2b8e24148 100644 --- a/bin/varnishtest/tests/v00059.vtc +++ b/bin/varnishtest/tests/v00059.vtc @@ -1,4 +1,4 @@ -varnishtest "n_pipe gauge" +varnishtest "concurrent pipe limit" barrier b1 cond 2 barrier b2 cond 2 @@ -36,6 +36,7 @@ varnish v1 -expect MAIN.n_pipe == 0 client c1 { txreq -url "/c1" rxresp + expect resp.status == 200 } -start barrier b1 sync @@ -45,10 +46,20 @@ varnish v1 -expect MAIN.n_pipe == 1 client c2 { txreq -url "/c2" rxresp + expect resp.status == 200 } -start barrier b3 sync varnish v1 -expect MAIN.n_pipe == 2 + +varnish v1 -cliok "param.set pipe_sess_max 2" + +client c3 { + txreq + rxresp + expect resp.status == 503 +} -run + barrier b2 sync varnish v1 -expect MAIN.n_pipe == 1 barrier b4 sync diff --git a/include/tbl/params.h b/include/tbl/params.h index de34fb931..9632f47bf 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -914,6 +914,20 @@ PARAM( /* func */ NULL ) +PARAM( + /* name */ pipe_sess_max, + /* typ */ uint, + /* min */ "0", + /* max */ NULL, + /* default */ "0", + /* units */ "connections", + /* flags */ 0, + /* s-text */ + "Maximum number of sessions dedicated to pipe transactions.", + /* l-text */ "", + /* func */ NULL +) + PARAM( /* name */ pipe_timeout, /* typ */ timeout, From dridi.boukelmoune at gmail.com Mon Apr 15 07:57:07 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 15 Apr 2019 07:57:07 +0000 (UTC) Subject: [master] a4ea4a076 New counter MAIN.pipe_limited Message-ID: <20190415075707.5404361A88@lists.varnish-cache.org> commit a4ea4a07618009f7a52c3eb8bbdf24b8a8588796 Author: Dridi Boukelmoune Date: Thu Apr 11 19:52:07 2019 +0200 New counter MAIN.pipe_limited To keep track of the number of times pipe_sess_max kicked in. This one can be lazily updated. diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc index 91d17fc8b..0377f9659 100644 --- a/bin/varnishd/VSC_main.vsc +++ b/bin/varnishd/VSC_main.vsc @@ -399,6 +399,14 @@ :oneliner: Number of ongoing pipe sessions +.. varnish_vsc:: pipe_limited + :group: wrk + :oneliner: Pipes hit pipe_sess_max + + Number of times more pipes were needed, but the limit was reached. See + also parameter pipe_sess_max. + + .. varnish_vsc:: s_pipe :group: wrk :oneliner: Total pipe sessions seen diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 5fb7f615e..eb797917d 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -710,6 +710,7 @@ cnt_pipe(struct worker *wrk, struct req *req) V1P_Leave(); break; } + wrk->stats->pipe_limited++; /* fall through */ case VCL_RET_FAIL: req->req_step = R_STP_VCLFAIL; diff --git a/bin/varnishtest/tests/v00059.vtc b/bin/varnishtest/tests/v00059.vtc index 2b8e24148..26282201d 100644 --- a/bin/varnishtest/tests/v00059.vtc +++ b/bin/varnishtest/tests/v00059.vtc @@ -64,3 +64,4 @@ barrier b2 sync varnish v1 -expect MAIN.n_pipe == 1 barrier b4 sync varnish v1 -expect MAIN.n_pipe == 0 +varnish v1 -expect MAIN.pipe_limited == 1 From nils.goroll at uplex.de Mon Apr 15 09:08:06 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Apr 2019 09:08:06 +0000 (UTC) Subject: [master] e68288fed make test less time sensitive Message-ID: <20190415090806.DD53E63622@lists.varnish-cache.org> commit e68288fed524be74094aabea9bfe2936f479b9e3 Author: Nils Goroll Date: Mon Apr 15 11:06:20 2019 +0200 make test less time sensitive Seen fail on travis: ** c1 2.7 === expect resp.status == 200 **** c1 2.7 EXPECT resp.status (200) == "200" match ** c1 2.7 === expect resp.http.Age == 1 ---- c1 2.7 EXPECT resp.http.Age (2) == "1" failed diff --git a/bin/varnishtest/tests/s00006.vtc b/bin/varnishtest/tests/s00006.vtc index c347e67a6..9e8ba30db 100644 --- a/bin/varnishtest/tests/s00006.vtc +++ b/bin/varnishtest/tests/s00006.vtc @@ -21,7 +21,7 @@ client c1 { expect resp.status == 200 expect resp.http.Age == 0 - delay 1.0 + delay 0.2 txreq -url "/" rxresp From phk at FreeBSD.org Mon Apr 15 09:34:06 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 15 Apr 2019 09:34:06 +0000 (UTC) Subject: [master] 0e125f511 Stabilize this test. Message-ID: <20190415093406.BED0963FE6@lists.varnish-cache.org> commit 0e125f511f755945d7b1bff19660873ff5ab796c Author: Poul-Henning Kamp Date: Mon Apr 15 09:33:11 2019 +0000 Stabilize this test. diff --git a/bin/varnishtest/tests/a02022.vtc b/bin/varnishtest/tests/a02022.vtc index 5e0e9b4a8..d3cc0d01b 100644 --- a/bin/varnishtest/tests/a02022.vtc +++ b/bin/varnishtest/tests/a02022.vtc @@ -6,18 +6,18 @@ server s1 { rxreq upgrade - stream 3 { rxprio } -run - stream 5 { rxprio } -run - stream 7 { rxprio } -run - stream 9 { rxprio } -run - stream 11 { rxprio } -run + stream 1 { rxprio } -start + stream 3 { rxprio } -start + stream 5 { rxprio } -start + stream 7 { rxprio } -start + stream 9 { rxprio } -start + stream 11 { rxprio } -start - stream 1 { - rxprio - txresp - } -start + stream 0 -wait + stream 1 -wait { txresp } -run -} -start + stream 0 { rxgoaway } -run +} -start shell { nghttp http://${s1_addr}:${s1_port} -nu } From nils.goroll at uplex.de Mon Apr 15 11:55:10 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Apr 2019 11:55:10 +0000 (UTC) Subject: [master] 1594037cb Add a v_* attribute for deprecated functions Message-ID: <20190415115510.D7D5C91052@lists.varnish-cache.org> commit 1594037cb896970c2cbfdab101d0059ac00201f9 Author: Nils Goroll Date: Mon Apr 8 16:11:30 2019 +0200 Add a v_* attribute for deprecated functions This works with gcc 6.3.0 and clang 3.8.1-24 The test for __GNUC__ is deliberately simple and might not catch all compilers which would potentially support deprecation marks. While more specifics could be added, the aim is to raise awareness with developers and we consider it quite unlikely that anyone does not compile with one of the main stream compilers at all. diff --git a/include/vdef.h b/include/vdef.h index acf94813f..2c78fa041 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -81,6 +81,12 @@ #define v_noreturn_ __attribute__((__noreturn__)) +#ifdef __GNUC__ +# define v_deprecated_ __attribute__((deprecated)) +#else +# define v_deprecated_ +#endif + /********************************************************************* * Pointer alignment magic */ From nils.goroll at uplex.de Mon Apr 15 11:55:10 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Apr 2019 11:55:10 +0000 (UTC) Subject: [master] 4e3335977 Deprecate WS_Reserve() and replace it with WS_ReserveSize() Message-ID: <20190415115511.0342291057@lists.varnish-cache.org> commit 4e33359772a3d751b2ef4b5c4b40259f1bcd6903 Author: Nils Goroll Date: Mon Apr 8 16:26:07 2019 +0200 Deprecate WS_Reserve() and replace it with WS_ReserveSize() WS_ReserveSize() does not leave the workspace reserved when the reservation fails, so WS_Release() must be called for retval > 0 only. Besides the debug string, it is identical to WS_Reserve() except for assert(bytes > 0); Follow-up #2967 diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index cc2fc993d..d6d3e9ac2 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -764,7 +764,10 @@ void WRK_BgThread(pthread_t *thr, const char *name, bgthread_t *func, /* cache_ws.c */ void WS_Init(struct ws *ws, const char *id, void *space, unsigned len); -unsigned WS_Reserve(struct ws *ws, unsigned bytes); + +/* WS_Reserve(): Use WS_ReserveSize() or WS_ReserveAll() */ +unsigned WS_Reserve(struct ws *ws, unsigned bytes) v_deprecated_; +unsigned WS_ReserveSize(struct ws *, unsigned); unsigned WS_ReserveAll(struct ws *); unsigned WS_ReserveLumps(struct ws *ws, size_t sz); void WS_MarkOverflow(struct ws *ws); diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index eac4773fc..8c9e2d7c9 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -112,7 +112,7 @@ ses_reserve_attr(struct sess *sp, enum sess_attr a, void **dst, int sz) assert(a < SA_LAST); assert(sz >= 0); AN(dst); - o = WS_Reserve(sp->ws, sz); + o = WS_ReserveSize(sp->ws, sz); assert(o >= sz); *dst = sp->ws->f; o = sp->ws->f - sp->ws->s; @@ -415,7 +415,7 @@ ses_handle(struct waited *wp, enum wait_event ev, vtim_real now) case WAITER_ACTION: pp = sp->pool; CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); - assert(sizeof *tp <= WS_Reserve(sp->ws, sizeof *tp)); + assert(sizeof *tp <= WS_ReserveSize(sp->ws, sizeof *tp)); tp = (void*)sp->ws->f; tp->func = xp->unwait; tp->priv = sp; @@ -453,7 +453,7 @@ SES_Wait(struct sess *sp, const struct transport *xp) /* * put struct waited on the workspace */ - if (WS_Reserve(sp->ws, sizeof(struct waited)) + if (WS_ReserveSize(sp->ws, sizeof(struct waited)) < sizeof(struct waited)) { SES_Delete(sp, SC_OVERLOAD, NAN); return; diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index a0fc4eacb..c6a37b862 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -238,7 +238,7 @@ Pool_Task_Arg(struct worker *wrk, enum task_prio prio, task_func_t *func, retval = 0; } AZ(wrk2->task.func); - assert(arg_len <= WS_Reserve(wrk2->aws, arg_len)); + assert(arg_len <= WS_ReserveSize(wrk2->aws, arg_len)); memcpy(wrk2->aws->f, arg, arg_len); wrk2->task.func = func; wrk2->task.priv = wrk2->aws->f; diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index e0a9b1d48..7b8df6d23 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -247,9 +247,34 @@ WS_ReserveAll(struct ws *ws) } /* - * bytes == 0 argument is deprecated - use WS_ReserveAll - * - * XXX rename to WS_ReserveSize and macro-wrap WS_Reserve to emit #warn ? + * WS_Release() must be called for retval > 0 only + */ +unsigned +WS_ReserveSize(struct ws *ws, unsigned bytes) +{ + unsigned b2; + + WS_Assert(ws); + assert(ws->r == NULL); + assert(bytes > 0); + + b2 = PRNDDN(ws->e - ws->f); + if (bytes < b2) + b2 = PRNDUP(bytes); + + if (ws->f + b2 > ws->e) { + WS_MarkOverflow(ws); + return (0); + } + ws->r = ws->f + b2; + DSL(DBG_WORKSPACE, 0, "WS_ReserveSize(%p, %u/%u) = %u", + ws, b2, bytes, pdiff(ws->f, ws->r)); + WS_Assert(ws); + return (pdiff(ws->f, ws->r)); +} + +/* + * XXX remove for 2020-03-15 release */ unsigned WS_Reserve(struct ws *ws, unsigned bytes) diff --git a/lib/libvmod_proxy/vmod_proxy.c b/lib/libvmod_proxy/vmod_proxy.c index fa7b9617e..58f2db295 100644 --- a/lib/libvmod_proxy/vmod_proxy.c +++ b/lib/libvmod_proxy/vmod_proxy.c @@ -105,7 +105,7 @@ tlv_string(VRT_CTX, int tlv) if (VPX_tlv(ctx->req, tlv, (void **)&dst, &len)) return (NULL); - if (!WS_Reserve(ctx->ws, len+1)) + if (!WS_ReserveSize(ctx->ws, len+1)) return (NULL); d = ctx->ws->f; memcpy(d, dst, len); From nils.goroll at uplex.de Mon Apr 15 11:59:06 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Apr 2019 11:59:06 +0000 (UTC) Subject: [master] 5260e3033 make VRT_fail() work with any initializier Message-ID: <20190415115906.AFC6A914F8@lists.varnish-cache.org> commit 5260e3033018e8fb5f50aa2a6541b9943afa98f6 Author: Nils Goroll Date: Fri Mar 22 17:29:08 2019 +0100 make VRT_fail() work with any initializier Historically, all initializers called from VGC_Load() would handle failures individually (return (1) from VGC_Load) or not fail at all. Now that we got VRT_fail(), it makes sense to generalize. The upcoming use case is to properly handle failures from VRT_new_backend_clustered(). diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 203f34465..6f2e38adc 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -311,6 +311,10 @@ EmitInitFini(const struct vcc *tl) assert(p->n > 0); if (VSB_len(p->ini)) Fc(tl, 0, "\t/* %u */\n%s\n", p->n, VSB_data(p->ini)); + if (p->ignore_errors == 0) { + Fc(tl, 0, "\tif (*ctx->handling == VCL_RET_FAIL)\n"); + Fc(tl, 0, "\t\treturn(1);\n"); + } Fc(tl, 0, "\tvgc_inistep = %u;\n\n", p->n); VSB_destroy(&p->ini); @@ -664,10 +668,10 @@ vcc_CompileSource(struct vcc *tl, struct source *sp) ifp = New_IniFin(tl); VSB_printf(ifp->ini, "\tVGC_function_vcl_init(ctx);\n"); /* - * We do not return(1) if this fails, because the failure - * could be half way into vcl_init{} so vcl_fini{} must - * always be called, also on failure. + * Because the failure could be half way into vcl_init{} so vcl_fini{} + * must always be called, also on failure. */ + ifp->ignore_errors = 1; VSB_printf(ifp->fin, "\t\tVGC_function_vcl_fini(ctx);"); /* Emit method functions */ diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index ece13c662..5f9774ed5 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -200,6 +200,7 @@ struct inifin { unsigned magic; #define INIFIN_MAGIC 0x583c274c unsigned n; + unsigned ignore_errors; struct vsb *ini; struct vsb *fin; struct vsb *final; From nils.goroll at uplex.de Mon Apr 15 13:39:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Apr 2019 13:39:07 +0000 (UTC) Subject: [master] 9c10c5aca kill stale oc for backend synth and clarify when we cache errors Message-ID: <20190415133907.3808B9394E@lists.varnish-cache.org> commit 9c10c5aca8b2595997f5ce490991d734d6af6928 Author: Nils Goroll Date: Sun Mar 17 11:45:18 2019 +0100 kill stale oc for backend synth and clarify when we cache errors Fixes #2946 in the sense that we want to treat a cacheable backend synth like any other object and kill the stale object it replaces. We do not replace the stale oc when - abandoning the bereq - leaving vcl_backend_error with return (deliver) and beresp.ttl == 0s - there is a waitinglist on this object because in this case the default ttl would be 1s, so we might be looking at the same case as the previous Thank you @mbgrydeland for the review and spotting a mistake. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 33a751ea2..a7ecafa22 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -683,6 +683,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) vtim_real now; uint8_t *ptr; struct vsb *synth_body; + struct objcore *stale = NULL; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); @@ -708,6 +709,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) http_TimeHeader(bo->beresp, "Date: ", now); http_SetHeader(bo->beresp, "Server: Varnish"); + stale = bo->stale_oc; bo->fetch_objcore->t_origin = now; if (!VTAILQ_EMPTY(&bo->fetch_objcore->objhead->waitinglist)) { /* @@ -720,6 +722,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) bo->fetch_objcore->ttl = 1; bo->fetch_objcore->grace = 5; bo->fetch_objcore->keep = 5; + stale = NULL; } else { bo->fetch_objcore->ttl = 0; bo->fetch_objcore->grace = 0; @@ -775,6 +778,8 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) AZ(ObjSetU64(wrk, bo->fetch_objcore, OA_LEN, o)); VSB_destroy(&synth_body); HSH_Unbusy(wrk, bo->fetch_objcore); + if (stale != NULL && bo->fetch_objcore->ttl > 0) + HSH_Kill(stale); ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED); return (F_STP_DONE); } diff --git a/bin/varnishtest/tests/r01395.vtc b/bin/varnishtest/tests/r01395.vtc index b7e388101..ce56cf708 100644 --- a/bin/varnishtest/tests/r01395.vtc +++ b/bin/varnishtest/tests/r01395.vtc @@ -1,4 +1,4 @@ -varnishtest "Test vcl_synth is called even if vcl_backend_fetch failed" +varnishtest "Test vcl_backend_error is called if vcl_backend_fetch fails" varnish v1 -vcl { backend default { .host = "${bad_backend}"; } diff --git a/bin/varnishtest/tests/r02946.vtc b/bin/varnishtest/tests/r02946.vtc new file mode 100644 index 000000000..b0eb3e6bf --- /dev/null +++ b/bin/varnishtest/tests/r02946.vtc @@ -0,0 +1,45 @@ +varnishtest "#2946 - objcore leak for backend_synth" + +varnish v1 -vcl { + backend bad { .host = "${bad_backend}"; } + + sub vcl_backend_error { + if (bereq.http.abandon) { + return (abandon); + } + if (bereq.http.ttl0) { + set beresp.ttl = 0s; + return (deliver); + } + set beresp.status = 200; + set beresp.ttl = 0.0001s; + set beresp.grace = 1h; + return (deliver); + } +} -start + +client c1 -repeat 20 -keepalive { + txreq + rxresp + expect resp.status == 200 + + delay 0.001 +} -run + +delay 2 + +client c1 { + txreq -hdr "abandon: true" + rxresp + expect resp.status == 200 + expect resp.http.age > 1 +} -run + +client c1 { + txreq -hdr "ttl0: true" + rxresp + expect resp.status == 200 + expect resp.http.age > 1 +} -run + +varnish v1 -expect n_objectcore == 1 diff --git a/doc/sphinx/users-guide/vcl-built-in-subs.rst b/doc/sphinx/users-guide/vcl-built-in-subs.rst index a911ed005..0315b4838 100644 --- a/doc/sphinx/users-guide/vcl-built-in-subs.rst +++ b/doc/sphinx/users-guide/vcl-built-in-subs.rst @@ -495,8 +495,12 @@ vcl_backend_error This subroutine is called if we fail the backend fetch or if *max_retries* has been exceeded. -A synthetic object is generated in VCL, whose body may be constructed -using the ``synthetic()`` function. +Returning with `abandon`_ does not leave a cache object. + +If returning with ``deliver`` and a ``beresp.ttl > 0s``, a synthetic +cache object is generated in VCL, whose body may be constructed using +the ``synthetic()`` function. This may be useful to increase the +efficiency of failing backend requests. The `vcl_backend_error` subroutine may terminate with calling ``return()`` with one of the following keywords: From nils.goroll at uplex.de Mon Apr 15 14:11:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Apr 2019 14:11:07 +0000 (UTC) Subject: [master] a4d79267e Remove XXXes which have been addressed Message-ID: <20190415141107.3BA7494590@lists.varnish-cache.org> commit a4d79267eca936654d515eeecf97f5bd98b26a58 Author: Nils Goroll Date: Mon Apr 15 15:44:47 2019 +0200 Remove XXXes which have been addressed diff --git a/bin/varnishd/cache/cache_ban_build.c b/bin/varnishd/cache/cache_ban_build.c index 660bdabc0..228d18c4b 100644 --- a/bin/varnishd/cache/cache_ban_build.c +++ b/bin/varnishd/cache/cache_ban_build.c @@ -353,10 +353,6 @@ BAN_Commit(struct ban_proto *bp) VSC_C_main->bans++; VSC_C_main->bans_added++; bans_persisted_bytes += ln; - /* - * XXX absolute update of gauges - may be inaccurate for Pool_Sumstat - * race - */ VSC_C_main->bans_persisted_bytes = bans_persisted_bytes; if (b->flags & BANS_FLAG_OBJ) diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index 423eb0550..0c1098da9 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -364,7 +364,6 @@ do_list_json(struct cli *cli, struct director *d, void *priv) VCLI_Out(cli, "%s", la->jsep); la->jsep = ",\n"; - // XXX admin health "probe" for the no-probe case is confusing VCLI_JSON_str(cli, d->vdir->cli_name); VCLI_Out(cli, ": {\n"); VSB_indent(cli->sb, 2); From nils.goroll at uplex.de Mon Apr 15 14:11:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Apr 2019 14:11:07 +0000 (UTC) Subject: [master] 09253629e Add a REL_* marker for things we need to consider for releases Message-ID: <20190415141107.4E79494593@lists.varnish-cache.org> commit 09253629e6170712d4fc2bc60c2d4bd7caa6b66a Author: Nils Goroll Date: Mon Apr 15 15:45:09 2019 +0200 Add a REL_* marker for things we need to consider for releases Ref: https://github.com/varnishcache/varnish-cache/pull/2969#pullrequestreview-224802203 diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index 7b8df6d23..a8e035ccf 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -273,9 +273,7 @@ WS_ReserveSize(struct ws *ws, unsigned bytes) return (pdiff(ws->f, ws->r)); } -/* - * XXX remove for 2020-03-15 release - */ +/* REL_20200315 remove */ unsigned WS_Reserve(struct ws *ws, unsigned bytes) { From nils.goroll at uplex.de Mon Apr 15 14:11:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Apr 2019 14:11:07 +0000 (UTC) Subject: [master] 9bb68d852 remove superfluous assert Message-ID: <20190415141107.66E5D94598@lists.varnish-cache.org> commit 9bb68d852f202e28bea37ff9f10d61a082677600 Author: Nils Goroll Date: Mon Apr 15 15:48:14 2019 +0200 remove superfluous assert We handle the error case just above diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index 156e50e6f..ab96954fd 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -171,7 +171,6 @@ vrg_dorange(struct req *req, const char *r, void **priv) if (vrg_priv == NULL) return ("WS too small"); - XXXAN(vrg_priv); INIT_OBJ(vrg_priv, VRG_PRIV_MAGIC); vrg_priv->range_off = 0; vrg_priv->range_low = low; From nils.goroll at uplex.de Mon Apr 15 14:11:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Apr 2019 14:11:07 +0000 (UTC) Subject: [master] a3fcdabb9 Comment on #2951 Message-ID: <20190415141107.810DF9459D@lists.varnish-cache.org> commit a3fcdabb9d0d303998642fcde315b2a189e84d1b Author: Nils Goroll Date: Mon Apr 15 16:01:18 2019 +0200 Comment on #2951 diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index a7ecafa22..82af410e2 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -674,6 +674,19 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) /*-------------------------------------------------------------------- * Create synth object + * + * replaces a stale object unless + * - abandoning the bereq or + * - leaving vcl_backend_error with return (deliver) and beresp.ttl == 0s or + * - there is a waitinglist on this object because in this case the default ttl + * would be 1s, so we might be looking at the same case as the previous + * + * We do want the stale replacement to avoid an object pileup with short ttl and + * long grace/keep, yet there could exist cases where a cache object is + * deliberately created to momentarily override a stale object. + * + * If this case exists, we should add a vcl veto (e.g. beresp.replace_stale with + * default true) */ static enum fetch_step From nils.goroll at uplex.de Mon Apr 15 14:11:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Apr 2019 14:11:07 +0000 (UTC) Subject: [master] fa920eb05 document #2951 Message-ID: <20190415141107.9C1E6945A1@lists.varnish-cache.org> commit fa920eb053a8c36d2bef3f463ed0238659606f8a Author: Nils Goroll Date: Mon Apr 15 16:04:44 2019 +0200 document #2951 diff --git a/doc/changes.rst b/doc/changes.rst index 4e7494225..046c5e1a8 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -26,6 +26,23 @@ http://varnish-cache.org/docs/trunk/whats-new/index.html and via individual releases. These documents are updated as part of the release process. +================================ +NEXT (2019-09-15) +================================ + +* Synthetic objects created from ``vcl_backend_error {}`` now replace + existing stale objects as ordinary backend fetches would, unless: + + - abandoning the bereq or + + - leaving ``vcl_backend_error {}`` with ``return (deliver) and + ``beresp.ttl == 0s`` or + + - there is a waitinglist on the object, in which case, by default, + the synthetic object is created with ``ttl = 1s`` / ``grace = 5s`` + / ``keep = 5s`` avoid hammering on failing backends + (note this is existing behavior). + ================================ Varnish Cache 6.2.0 (2019-03-15) ================================ From nils.goroll at uplex.de Mon Apr 15 14:11:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Apr 2019 14:11:07 +0000 (UTC) Subject: [master] 8c9b944cc document #2969 Message-ID: <20190415141107.B657A945A5@lists.varnish-cache.org> commit 8c9b944cc8a717ef14794b2001ba294951792f8a Author: Nils Goroll Date: Mon Apr 15 16:09:46 2019 +0200 document #2969 diff --git a/doc/changes.rst b/doc/changes.rst index 046c5e1a8..e7666d0c7 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -43,6 +43,25 @@ NEXT (2019-09-15) / ``keep = 5s`` avoid hammering on failing backends (note this is existing behavior). +C APIs (for vmod and utility authors) +------------------------------------- + +* ``WS_Reserve()`` is now deprecated and any use should trigger a + compiler warning. It is to be replaced by + + - ``WS_ReserveAll()`` to reserve all of the remaining workspace + + It will always leave the workspace reserved even if 0 bytes are + available, so it must always be followed by a call to + ``WS_Release()`` + + - ``WS_ReserveSize()`` to reserve a fixed amount. + + It will only leave the workspace reserved if the reservation + request could be fulfilled. + + + ================================ Varnish Cache 6.2.0 (2019-03-15) ================================ From nils.goroll at uplex.de Mon Apr 15 15:17:06 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Apr 2019 15:17:06 +0000 (UTC) Subject: [master] 5dd049418 First step to vmod code maintenance automation Message-ID: <20190415151706.99B55965DB@lists.varnish-cache.org> commit 5dd049418a4edf34d96f18fd1cb66c0893593fc6 Author: Nils Goroll Date: Mon Apr 15 17:15:14 2019 +0200 First step to vmod code maintenance automation diff --git a/coccinelle/README.rst b/coccinelle/README.rst new file mode 100644 index 000000000..f279cb563 --- /dev/null +++ b/coccinelle/README.rst @@ -0,0 +1,10 @@ +This directory contains `coccinelle`_ patches to facilitate code +maintenance. + +Each patch should, in a comment section, explain its purpose. + +Unless noted otherwise, all patches should work when invoked as:: + + spatch --dir . --in-place --sp-file $COCCI + +.. _coccinelle: http://coccinelle.lip6.fr/ diff --git a/coccinelle/vmod/WS_Reserve.cocci b/coccinelle/vmod/WS_Reserve.cocci new file mode 100644 index 000000000..f40a49dbb --- /dev/null +++ b/coccinelle/vmod/WS_Reserve.cocci @@ -0,0 +1,23 @@ +/* + * patch to change code with respect to #2969: Replacement of + * WS_Reserve(ws, sz) by WS_ReserveAll(ws) / WS_ReserveSize(ws, sz) + * + * NOTE this patch does not check/fix error handling: + * - WS_ReserveAll(ws) : Always needs WS_Release(ws, sz) + * - WS_ReserveSize(ws, sz): needs WS_Release(ws, sz) if retval != 0 + */ +@@ +expression ws; +identifier ptr; +@@ + +-ptr = WS_Reserve(ws, 0); ++ptr = WS_ReserveAll(ws); + +@@ +expression ws, sz; +identifier ptr; +@@ + +-ptr = WS_Reserve(ws, sz); ++ptr = WS_ReserveSize(ws, sz); diff --git a/doc/changes.rst b/doc/changes.rst index e7666d0c7..294fb8e80 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -60,7 +60,8 @@ C APIs (for vmod and utility authors) It will only leave the workspace reserved if the reservation request could be fulfilled. - + We provide a script to automate this change in the ``coccinelle`` + subdirectory of the source tree. ================================ Varnish Cache 6.2.0 (2019-03-15) From nils.goroll at uplex.de Mon Apr 15 15:57:06 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Apr 2019 15:57:06 +0000 (UTC) Subject: [master] 626ddb029 Retire SLT_BackendStart Message-ID: <20190415155706.D938D974A5@lists.varnish-cache.org> commit 626ddb029256513b6de36d5e642299d6708d638d Author: Nils Goroll Date: Mon Apr 15 17:54:26 2019 +0200 Retire SLT_BackendStart Closes #2971 diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index d5f5bb2ee..da1b7f8ed 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -238,7 +238,6 @@ vbe_dir_gethdrs(VRT_CTX, VCL_BACKEND d) struct pfd *pfd; struct busyobj *bo; struct worker *wrk; - char abuf[VTCP_ADDRBUFSIZE], pbuf[VTCP_PORTBUFSIZE]; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); @@ -264,9 +263,8 @@ vbe_dir_gethdrs(VRT_CTX, VCL_BACKEND d) if (PFD_State(pfd) != PFD_STATE_STOLEN) extrachance = 0; - PFD_RemoteName(pfd, abuf, sizeof abuf, pbuf, sizeof pbuf); i = V1F_SendReq(wrk, bo, &bo->acct.bereq_hdrbytes, - &bo->acct.bereq_bodybytes, 0, abuf, pbuf); + &bo->acct.bereq_bodybytes, 0); if (PFD_State(pfd) != PFD_STATE_USED) { if (VTP_Wait(wrk, pfd, VTIM_real() + @@ -330,7 +328,6 @@ vbe_dir_http1pipe(VRT_CTX, VCL_BACKEND d) struct backend *bp; struct v1p_acct v1a; struct pfd *pfd; - char abuf[VTCP_ADDRBUFSIZE], pbuf[VTCP_PORTBUFSIZE]; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); @@ -352,9 +349,8 @@ vbe_dir_http1pipe(VRT_CTX, VCL_BACKEND d) retval = SC_TX_ERROR; } else { CHECK_OBJ_NOTNULL(ctx->bo->htc, HTTP_CONN_MAGIC); - PFD_RemoteName(pfd, abuf, sizeof abuf, pbuf, sizeof pbuf); i = V1F_SendReq(ctx->req->wrk, ctx->bo, - &v1a.bereq, &v1a.out, 1, abuf, pbuf); + &v1a.bereq, &v1a.out, 1); VSLb_ts_req(ctx->req, "Pipe", W_TIM_real(ctx->req->wrk)); if (i == 0) V1P_Process(ctx->req, *PFD_Fd(pfd), &v1a); diff --git a/bin/varnishd/http1/cache_http1.h b/bin/varnishd/http1/cache_http1.h index f642046cf..0bc52606c 100644 --- a/bin/varnishd/http1/cache_http1.h +++ b/bin/varnishd/http1/cache_http1.h @@ -31,7 +31,7 @@ struct VSC_vbe; /* cache_http1_fetch.c [V1F] */ int V1F_SendReq(struct worker *, struct busyobj *, uint64_t *ctr_hdrbytes, - uint64_t *ctr_bodybytes, int onlycached, char *addr, char *port); + uint64_t *ctr_bodybytes, int onlycached); int V1F_FetchRespHdr(struct busyobj *); int V1F_Setup_Fetch(struct vfp_ctx *vfc, struct http_conn *htc); diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c index f4ee4ebf0..d8ce8a1a6 100644 --- a/bin/varnishd/http1/cache_http1_fetch.c +++ b/bin/varnishd/http1/cache_http1_fetch.c @@ -69,7 +69,7 @@ vbf_iter_req_body(void *priv, unsigned flush, const void *ptr, ssize_t l) int V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes, - uint64_t *ctr_bodybytes, int onlycached, char *abuf, char *pbuf) + uint64_t *ctr_bodybytes, int onlycached) { struct http *hp; int j; @@ -95,8 +95,6 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes, do_chunked = 1; } - VSLb(bo->vsl, SLT_BackendStart, "%s %s", abuf, pbuf); - VTCP_blocking(*htc->rfd); /* XXX: we should timeout instead */ V1L_Open(wrk, wrk->aws, htc->rfd, bo->vsl, bo->t_prev, 0); hdrbytes = HTTP1_Write(wrk, hp, HTTP1_Req); diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index ca470e2a4..c17548809 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -919,9 +919,9 @@ dispatch_f(struct VSL_data *vsl, struct VSL_transaction * const pt[], 4, &CTX.frag[F_O], 0, NULL); break; - case (SLT_BackendStart + BACKEND_MARKER): + case (SLT_BackendOpen + BACKEND_MARKER): frag_fields(1, b, e, - 1, &CTX.frag[F_h], + 3, &CTX.frag[F_h], 0, NULL); break; case SLT_ReqStart: diff --git a/bin/varnishtest/tests/b00060.vtc b/bin/varnishtest/tests/b00060.vtc index be6a10634..959e765d6 100644 --- a/bin/varnishtest/tests/b00060.vtc +++ b/bin/varnishtest/tests/b00060.vtc @@ -25,5 +25,4 @@ logexpect l2 -v v1 -d 1 -g vxid { logexpect l3 -v v1 -d 1 -g vxid { expect * * Begin ^bereq expect * = BackendOpen "0.0.0.0 0 0.0.0.0 0$" - expect * = BackendStart "^0.0.0.0 0$" } -run diff --git a/bin/varnishtest/tests/u00010.vtc b/bin/varnishtest/tests/u00010.vtc index 329f92567..dd37a15d0 100644 --- a/bin/varnishtest/tests/u00010.vtc +++ b/bin/varnishtest/tests/u00010.vtc @@ -20,7 +20,7 @@ client c1 { varnish v1 -vsl_catchup -process p1 -expect-text 1 1 {list length 65} +process p1 -expect-text 1 1 {list length 64} process p1 -writehex 0c diff --git a/bin/varnishtest/tests/u00013.vtc b/bin/varnishtest/tests/u00013.vtc index 133d9992a..10fdd5523 100644 --- a/bin/varnishtest/tests/u00013.vtc +++ b/bin/varnishtest/tests/u00013.vtc @@ -1,6 +1,6 @@ varnishtest "varnishncsa outputs when UDS addresses are in use" -# The %h formatter gets its value from ReqStart or BackendStart, +# The %h formatter gets its value from ReqStart or BackendOpen, # which now may be a UDS address. # For UDS backends without a .host_header spec, the Host header is diff --git a/doc/changes.rst b/doc/changes.rst index 294fb8e80..8f8e52139 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -43,6 +43,9 @@ NEXT (2019-09-15) / ``keep = 5s`` avoid hammering on failing backends (note this is existing behavior). +* Retired the ``BackendStart`` log tag - ``BackendOpen`` contains all + the information from it + C APIs (for vmod and utility authors) ------------------------------------- diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 03488cdc8..434419409 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -592,16 +592,11 @@ SLTM(Witness, 0, "Lock order witness records", "Diagnostic recording of locking order.\n" ) -SLTM(BackendStart, 0, "Backend request start", - "Start of backend processing. Logs the backend IP address and port" - " number.\n\n" - "The format is::\n\n" - "\t%s %s\n" - "\t| |\n" - "\t| +- Backend Port number\n" - "\t+---- Backend IP4/6 address\n" - "\n" -) +/* + * REL_20190915 remove after VSLng + * kept for now for VSL binary compatibility + */ +SLTM(BackendStart, 0, "(retired)", "") SLTM(H2RxHdr, SLT_F_BINARY, "Received HTTP2 frame header", "Binary data" From dridi at varni.sh Mon Apr 15 16:04:34 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 15 Apr 2019 18:04:34 +0200 Subject: [master] 5dd049418 First step to vmod code maintenance automation In-Reply-To: <20190415151706.99B55965DB@lists.varnish-cache.org> References: <20190415151706.99B55965DB@lists.varnish-cache.org> Message-ID: On Mon, Apr 15, 2019 at 5:17 PM Nils Goroll wrote: > > > commit 5dd049418a4edf34d96f18fd1cb66c0893593fc6 > Author: Nils Goroll > Date: Mon Apr 15 17:15:14 2019 +0200 > > First step to vmod code maintenance automation > > diff --git a/coccinelle/README.rst b/coccinelle/README.rst > new file mode 100644 > index 000000000..f279cb563 > --- /dev/null > +++ b/coccinelle/README.rst Shouldn't we move this to tools/coccinelle? I will check in my spatches too. > @@ -0,0 +1,10 @@ > +This directory contains `coccinelle`_ patches to facilitate code > +maintenance. > + > +Each patch should, in a comment section, explain its purpose. > + > +Unless noted otherwise, all patches should work when invoked as:: > + > + spatch --dir . --in-place --sp-file $COCCI > + > +.. _coccinelle: http://coccinelle.lip6.fr/ > diff --git a/coccinelle/vmod/WS_Reserve.cocci b/coccinelle/vmod/WS_Reserve.cocci > new file mode 100644 > index 000000000..f40a49dbb > --- /dev/null > +++ b/coccinelle/vmod/WS_Reserve.cocci > @@ -0,0 +1,23 @@ > +/* > + * patch to change code with respect to #2969: Replacement of > + * WS_Reserve(ws, sz) by WS_ReserveAll(ws) / WS_ReserveSize(ws, sz) > + * > + * NOTE this patch does not check/fix error handling: > + * - WS_ReserveAll(ws) : Always needs WS_Release(ws, sz) > + * - WS_ReserveSize(ws, sz): needs WS_Release(ws, sz) if retval != 0 > + */ > +@@ > +expression ws; > +identifier ptr; > +@@ > + > +-ptr = WS_Reserve(ws, 0); > ++ptr = WS_ReserveAll(ws); > + > +@@ > +expression ws, sz; > +identifier ptr; > +@@ > + > +-ptr = WS_Reserve(ws, sz); > ++ptr = WS_ReserveSize(ws, sz); > diff --git a/doc/changes.rst b/doc/changes.rst > index e7666d0c7..294fb8e80 100644 > --- a/doc/changes.rst > +++ b/doc/changes.rst > @@ -60,7 +60,8 @@ C APIs (for vmod and utility authors) > It will only leave the workspace reserved if the reservation > request could be fulfilled. > > - > + We provide a script to automate this change in the ``coccinelle`` > + subdirectory of the source tree. > > ================================ > Varnish Cache 6.2.0 (2019-03-15) > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From nils.goroll at uplex.de Mon Apr 15 16:16:39 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 15 Apr 2019 18:16:39 +0200 Subject: [master] 5dd049418 First step to vmod code maintenance automation In-Reply-To: References: <20190415151706.99B55965DB@lists.varnish-cache.org> Message-ID: On 15/04/2019 18:04, Dridi Boukelmoune wrote: > Shouldn't we move this to tools/coccinelle? wfm, I really don't have much of an opinion about this -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From phk at FreeBSD.org Tue Apr 16 06:48:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 16 Apr 2019 06:48:07 +0000 (UTC) Subject: [master] 1593fdb1e Move the homepage stuff into the dev-guide. Message-ID: <20190416064807.9E800AF879@lists.varnish-cache.org> commit 1593fdb1ec90613211d348f42d91616f64373564 Author: Poul-Henning Kamp Date: Tue Apr 16 06:46:24 2019 +0000 Move the homepage stuff into the dev-guide. Add Project Governance info. diff --git a/doc/sphinx/dev-guide/homepage_contrib.rst b/doc/sphinx/dev-guide/homepage_contrib.rst new file mode 100644 index 000000000..a6c9c8720 --- /dev/null +++ b/doc/sphinx/dev-guide/homepage_contrib.rst @@ -0,0 +1,128 @@ +.. _homepage_contrib: + +How to contribute content to varnish-cache.org +============================================== + +This is where we walk you through the mechanics of adding content to +varnish-cache.org (see phk's note :ref:`homepage_dogfood` for an +insight into the innards of site). + +Git Repository +-------------- + +The web site contents live in github at: + +https://github.com/varnishcache/homepage + +To offer your own contribution, fork the project and send us a pull +request. + +Sphinx and RST +-------------- + +The web site sources are written in `RST +`_ -- reStructuredText, the +documentation format originally conceived for Python (and also used in +the Varnish distribution, as well as for formatting VMOD +docs). `Sphinx `_ is used to render web +pages from the RST sources. + +So you'll need to `learn markup with RST and Sphinx +`_; and you +will need to `install Sphinx `_ to test the rendering on your local system. + +Makefile +-------- + +Generation of web contents from the sources is driven by the ``Makefile`` +in the ``R1`` directory of the repo:: + + $ cd R1 + $ make help + Please use `make ' where is one of + html to make standalone HTML files + dirhtml to make HTML files named index.html in directories + singlehtml to make a single large HTML file + pickle to make pickle files + json to make JSON files + htmlhelp to make HTML files and a HTML help project + qthelp to make HTML files and a qthelp project + applehelp to make an Apple Help Book + devhelp to make HTML files and a Devhelp project + epub to make an epub + latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter + latexpdf to make LaTeX files and run them through pdflatex + latexpdfja to make LaTeX files and run them through platex/dvipdfmx + text to make text files + man to make manual pages + texinfo to make Texinfo files + info to make Texinfo files and run them through makeinfo + gettext to make PO message catalogs + changes to make an overview of all changed/added/deprecated items + xml to make Docutils-native XML files + pseudoxml to make pseudoxml-XML files for display purposes + linkcheck to check all external links for integrity + doctest to run all doctests embedded in the documentation (if enabled) + coverage to run coverage check of the documentation (if enabled) + +Most of the time, you'll just need ``make html`` to test the rendering +of your contribution. + +alabaster theme +--------------- + +We use the `alabaster theme `_, +which you may need to add to your local Python installation:: + + $ sudo pip install alabaster + +We have found that you may need to link the alabaster package install +directory to the directory where Sphinx expects to find themes. For +example (on my machine), alabaster was installed into:: + + /usr/local/lib/python2.7/dist-packages/alabaster + +And Sphinx expects to find themes in:: + + /usr/share/sphinx/themes + +So to get the make targets to run successfully:: + + $ cd /usr/share/sphinx/themes + $ ln -s /usr/local/lib/python2.7/dist-packages/alabaster + +Test the rendering +------------------ + +Now you can edit contents in the website repo, and test the rendering +by calling make targets in the ``R1`` directory:: + + $ cd $REPO/R1 + $ make html + sphinx-build -b html -d build/doctrees source build/html + Running Sphinx v1.2.3 + loading pickled environment... done + building [html]: targets for 1 source files that are out of date + updating environment: 0 added, 1 changed, 0 removed + reading sources... [100%] tips/contribdoc/index + looking for now-outdated files... none found + pickling environment... done + checking consistency... done + preparing documents... done + writing output... [100%] tips/index + writing additional files... genindex search + copying static files... done + copying extra files... done + dumping search index... done + dumping object inventory... done + build succeeded. + +After a successful build, the newly rendered contents are saved in the +``R1/source/build`` directory, so you can have a look with your +browser. + +Send us a pull request +---------------------- + +When you have your contribution building successfully, send us a PR, +we'll be happy to hear from you! diff --git a/doc/sphinx/dev-guide/homepage_dogfood.rst b/doc/sphinx/dev-guide/homepage_dogfood.rst new file mode 100644 index 000000000..521513741 --- /dev/null +++ b/doc/sphinx/dev-guide/homepage_dogfood.rst @@ -0,0 +1,158 @@ +.. _homepage_dogfood: + +How our website works +===================== + +The principle of eating your own dogfood is important for software +quality, that is how you experience what your users are dealing with, +and I am not the least ashamed to admit that several obvious improvements +have happened to Varnish as a result of running the project webserver. + +But it is also important to externalize what you learn doing so, and +therefore I thought I would document here how the projects new "internal +IT" works. + +Hardware +-------- + +Who cares? + +Yes, we use some kind of hardware, but to be honest I don't know what +it is. + +Our primary site runs on a `RootBSD 'Omega' `_ +virtual server somewhere near CDG/Paris. + +And as backup/integration/testing server we can use any server, +virtual or physical, as long as it has a internet connection and +contemporary performance, because the entire install is scripted +and under version control (more below). + +Operating System +---------------- + +So, dogfood: Obviously FreeBSD. + +Apart from the obvious reason that I wrote a lot of FreeBSD and +can get world-class support by bugging my buddies about it, there +are two equally serious reasons for the Varnish Project to run on +FreeBSD: Dogfood and jails. + +Varnish Cache is not "software for Linux", it is software for any +competent UNIX-like operating system, and FreeBSD is our primary +"keep us honest about this" platform. + +Jails +----- + +You have probably heard about Docker and Containers, but FreeBSD +have had jails +`since I wrote them in 1998 `_ +and they're a wonderful way to keep your server installation +sane. + +We currently have three jails: + +* Hitch - runs the `Hitch SSL proxy `_ + +* Varnish - You guessed it + +* Tools - backend webserver, currently `ACME Labs' thttpd `_ + +Script & Version Control All The Things +--------------------------------------- + +We have a git repos with shell scripts which create these jails +from scratch and also a script to configure the host machine +properly. + +That means that the procedure to install a clone of the server +is, unabridged:: + + # Install FreeBSD (if not already done by hosting) + # Configure networking (if not already done by hosting) + # Set the clock + service ntpdate forcestart + # Get git + env ASSUME_ALWAYS_YES=yes pkg install git + # Clone the private git repo + git clone ssh://example.com/root/Admin + # Edit the machines IP numbers in /etc/pf.conf + # Configure the host + sh build_host.sh |& tee _.bh + # Build the jails + foreach i (Tools Hitch Varnish) + (cd $i ; sh build* |& tee _.bj) + end + +From bare hardware to ready system in 15-30 minutes. + +It goes without saying that this git repos contains stuff +like ssh host keys, so it should *not* go on github. + +Backups +------- + +Right now there is nothing we absolutely have to backup, provided +we have an up to date copy of the Admin git repos. + +In practice we want to retain history for our development tools +(VTEST, GCOV etc.) and I rsync those file of the server on a +regular basis. + + +The Homepage +------------ + +The homepage is built with `Sphinx `_ +and lives in its own +`github project `_ (Pull requests +are very welcome!) + +We have taken snapshots of some of the old webproperties, Trac, the +Forum etc as static HTML copies. + +Why on Earth... +--------------- + +It is a little bit tedious to get a setup like this going, whenever +you tweak some config file, you need to remember to pull the change +back out and put it in your Admin repos. + +But that extra effort pays of so many times later. + +You never have to wonder "who made that change and why" or even try +to remember what changes were needed in the first place. + +For us as a project, it means, that all our sysadmin people +can build a clone of our infrastructure, if they have a copy of +our "Admin" git repos and access to github. + +And when `FreeBSD 11 `_ +comes out, or a new version of sphinx or something else, mucking +about with things until they work can be done at leisure without +guess work. (We're actually at 12 now, but the joke is too good +to delete.) + +For instance I just added the forum snapshot, by working out all +the kinks on one of my test-machines. + +Once it was as I wanted it, I pushed the changes the live machine and then:: + + varnishadm vcl.use backup + # The 'backup' VCL does a "pass" of all traffic to my server + cd Admin + git pull + cd Tools + sh build_j_tools.sh |& tee _.bj + varnishadm vcl.load foobar varnish-live.vcl + varnishadm vcl.use foobar + +For a few minutes our website was a bit slower (because of the +extra Paris-Denmark hop), but there was never any interruption. + +And by doing it this way, I *know* it will work next time also. + +2016-04-25 /phk + +PS: All that buzz about "reproducible builds" ? Yeah, not a new idea. diff --git a/doc/sphinx/dev-guide/index.rst b/doc/sphinx/dev-guide/index.rst index 633a0102c..d1e71284d 100644 --- a/doc/sphinx/dev-guide/index.rst +++ b/doc/sphinx/dev-guide/index.rst @@ -81,35 +81,20 @@ These rules are imported from the X11 project: * Provide mechanism, rather than policy. -Bundling VMODs with the Varnish distribution --------------------------------------------- - -Decisions about whether to add a new Varnish module (VMOD) to those -bundled with Varnish are guided by these criteria. - -* The VMOD is known to be in widespread use and in high demand for - common use cases. - -* Or, if the VMOD is relatively new, it provides compelling features - that the developer group agrees will be a valuable enhancement for - the project. - -* The VMOD does not create dependencies on additional external - libraries. VMODs that are "glue" for a library come from third - parties. - - * We don't want to add new burdens of dependency and compatibility - to the project. +Various policies +---------------- - * We don't want to force Varnish deployments to install more than - admins explicitly choose to install. +.. toctree:: + :maxdepth: 1 -* The VMOD code follows project conventions (passes make distcheck, - follows source code style, and so forth). + policy_governance + policy_vmods + +The varnish-cache.org homepage +------------------------------ - * A pull request can demonstrate that this is the case (after any - necessary fixups). +.. toctree:: + :maxdepth: 1 -* The developer group commits to maintaining the code for the long run - (so there will have to be a consensus that we're comfortable with - it). + homepage_dogfood + homepage_contrib diff --git a/doc/sphinx/dev-guide/policy_governance.rst b/doc/sphinx/dev-guide/policy_governance.rst new file mode 100644 index 000000000..70250ad97 --- /dev/null +++ b/doc/sphinx/dev-guide/policy_governance.rst @@ -0,0 +1,28 @@ +.. _policy-governance: + +Project Truck Factor and Governance +----------------------------------- + +We recognize that the Truck Factor of the Varnish Cache project is low. + +As mitigation we have created a Varnish Governance Board (VGB) +and implemented procedures so no project asset have a Truck Factor +less than 3. + +The Varnish Governance Board is the ultimate authority in the +project, but does little to nothing on a daily basis. + +The important thing is that there is agreement beforehand about who +has the authority to step in, if need be. + +Appointment to the VGB is by acclamation, and we prefer the members +to have enough distance and neutrality to be able to resolve +conflicts, should it ever become necessary. + +Current members of the VGB: + +* Rogier Mulhuijzen +* Lasse Karstensen +* Poul-Henning Kamp + + diff --git a/doc/sphinx/dev-guide/policy_vmods.rst b/doc/sphinx/dev-guide/policy_vmods.rst new file mode 100644 index 000000000..365e86e73 --- /dev/null +++ b/doc/sphinx/dev-guide/policy_vmods.rst @@ -0,0 +1,34 @@ +.. _policy-vmods: + +Bundling VMODs with the Varnish distribution +-------------------------------------------- + +Decisions about whether to add a new Varnish module (VMOD) to those +bundled with Varnish are guided by these criteria. + +* The VMOD is known to be in widespread use and in high demand for + common use cases. + +* Or, if the VMOD is relatively new, it provides compelling features + that the developer group agrees will be a valuable enhancement for + the project. + +* The VMOD does not create dependencies on additional external + libraries. VMODs that are "glue" for a library come from third + parties. + + * We don't want to add new burdens of dependency and compatibility + to the project. + + * We don't want to force Varnish deployments to install more than + admins explicitly choose to install. + +* The VMOD code follows project conventions (passes make distcheck, + follows source code style, and so forth). + + * A pull request can demonstrate that this is the case (after any + necessary fixups). + +* The developer group commits to maintaining the code for the long run + (so there will have to be a consensus that we're comfortable with + it). From phk at FreeBSD.org Tue Apr 16 07:13:06 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 16 Apr 2019 07:13:06 +0000 (UTC) Subject: [master] b3efff3e6 Move Twurp's Peerage to the dev-guide Message-ID: <20190416071306.15D3FB00F6@lists.varnish-cache.org> commit b3efff3e692149a33de02d7862056610ac7a43ef Author: Poul-Henning Kamp Date: Tue Apr 16 07:08:34 2019 +0000 Move Twurp's Peerage to the dev-guide diff --git a/doc/sphinx/dev-guide/index.rst b/doc/sphinx/dev-guide/index.rst index d1e71284d..d8e5381d7 100644 --- a/doc/sphinx/dev-guide/index.rst +++ b/doc/sphinx/dev-guide/index.rst @@ -98,3 +98,11 @@ The varnish-cache.org homepage homepage_dogfood homepage_contrib + +Project metadata +---------------- + +.. toctree:: + :maxdepth: 1 + + who diff --git a/doc/sphinx/dev-guide/who.rst b/doc/sphinx/dev-guide/who.rst new file mode 100644 index 000000000..d7b8c1d95 --- /dev/null +++ b/doc/sphinx/dev-guide/who.rst @@ -0,0 +1,148 @@ +.. _who_is: + +Who is ... ? +============ + +Not quite `Twurp's Peerage `_ but a Who's Who of the Varnish Cache project. + +Anders Berg +~~~~~~~~~~~ + +Blame Anders! He is the one who got the crazy idea that the world +needed another HTTP proxy server software, and convinced his employer, +the norwegian newspaper `Verdens Gang `_ to pay for the +first version to be developed. + +Here is an interview with Anders about `how it all began +`_ + +Dag-Erling Sm?rgrav +~~~~~~~~~~~~~~~~~~~ + +DES was working at Redpill-Linpro, a norwegian UNIX/Open Source company +when Anders floated his idea for a "forward HTTP cache", he lured PHK +into joining, was one of the original developers (doing Linux), project +manager and release engineer for the first three years of the project, +and forced us to adopt a non-US-ASCII charset from the start. + +Poul-Henning Kamp +~~~~~~~~~~~~~~~~~ + +PHK, as he's usually known, has written most of the code and come up with +most of the crazy ideas in Varnish, and yet he still has trouble +remembering what 'REST', 'CORS' and 'ALPN' means, and he flunked +'CSS for dummies' because he was never a webmaster or webdeveloper. +He does have 30+ years of experience in systems programming, and +that seems useful too. + +PHK's `random outbursts `_ has their own +section in the Varnish documentation. + +Per Buer +~~~~~~~~ + +Per also worked at Redpill-Linpro, and at some point when the +impedance mismatch between Linpros "normal way of doing things" and +the potential of Varnish became to steep, he convinced the company +to spin off `Varnish Software `_ +with himself at the helm. + +Do a git blame on the Varnish documentation and you will be surprised +to see how much he cares about it. Very few people notice this. + +Ingvar Hagelund +~~~~~~~~~~~~~~~ + +Ingvar works as Team Leader (read very skilled sysadmin) at Redpill-Linpro, +but his passion is reading books and blogging about it, as well as RPM +packaging. So every Fedora and EPEL (read RedHat and CentOS) Varnish user +out there owe him a thanks or two. Once in a while, he also trawls the +internet checking for the rate of Varnish adoption among top web sites. + +Stig Sandbeck Mathisen +~~~~~~~~~~~~~~~~~~~~~~ + +Stig works at Redpill-Linpro and is the guy in charge of packaging Varnish +for Debian, which means Ubuntu users owe him a thanks also. Besides this, +he maintains VCL-mode for emacs and is generally a nice and helpful guy. + + +Tollef Fog Heen +~~~~~~~~~~~~~~~ + +Tollef was product owner and responsible for Varnish while working +for Redpill-Linpro. later tech lead at Varnish Software and held +the Varnish release manager helmet for a few years. His experience with +open source (Debian, Ubuntu and many others) brought sanity to the +project in ways that are hard to measure or describe. + +Kristian Lyngst?l +~~~~~~~~~~~~~~~~~ + +Kristian was the first Varnish SuperUser, and he quite literally +wrote the book, while giving Varnish courses for Redpill-Linpro, +and he pushed boundaries where no boundaries had been pushed before +which caused a lot of improvements and "aha!" moments in Varnish. + +Artur Bergman +~~~~~~~~~~~~~ + +Artur ran Wikias webservers and CDN when he discovered Varnish and +eagerly adopted it, causing many bugreports, suggestions, patches +and improvements. At some point, he pivoted Wikias CDN into the +Varnish based startup-CDN named `Fastly `_ + +Kacper Wysocki +~~~~~~~~~~~~~~ + +Kacper was probably the first VCL long program writer. Combine this with +an interest in security and a job at Redpill-Linpro and he turned +quickly into the author of security.vcl and, later, the Varnish Security +Firewall. He does not have any commits in Varnish and still has managed +to drive quite a few changes into the project. Similarly, he has no idea +or has even thought about asking for it, and still is being added here +He maintains the VCL grammar in BNF notation, which is an unexploited +gold mine. + +Nils Goroll +~~~~~~~~~~~ + +aka 'slink' is the founder of `UPLEX `_, a five-head +tech / consultancy company with negative to zero marketing (applied +for entry into the "Earth's worst company homepage" competition). He +fell in love with Varnish when he migrated Germany's Verdens Gang +counterpart over a weekend in March 2009 and, since then, has +experienced countless moments of pure joy and happiness when, after +struggling for hours, he finally understood another piece of +beautiful, ingenious Varnish code. + +Nils' primary focus are his clients and their projects. He tries to +make those improvements to Varnish which matter to them. + +Martin Blix Grydeland +~~~~~~~~~~~~~~~~~~~~~ + +Martin was the first full-time member of the C-team at Varnish Software. +He is the main responsible for the amazing revamp of the logging +facilities and utilities in the 4.0 cycle and later the storage +rework. Besides that he fixes lots of bugs, knows varnishtest better +than most, writes vmods and is the Varnish Cache Plus architect. + +Lasse Karstensen +~~~~~~~~~~~~~~~~ + +Lasse is the current release manager and stable version maintainer of +Varnish Cache. When not doing that, he maintains build infrastructure +and runs the Varnish Software C developer team in Oslo. + +Geoff Simmons +~~~~~~~~~~~~~ + +Geoff started working at UPLEX in 2010 and soon learned to love +Varnish as much as slink does. Since then he's been contributing code +to the project, writing up various VMODs (mostly about regular +expressions, blobs, backends and directors), developing standalone +applications for logging that use Martin's VSL API, and adding custom +patches to Varnish for various customer needs. He spends most of his +days in customer projects as "the Varnish guy" on the operations +teams. From dridi.boukelmoune at gmail.com Tue Apr 16 08:08:07 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 16 Apr 2019 08:08:07 +0000 (UTC) Subject: [master] a90a099dd mv coccinelle/ tools/ Message-ID: <20190416080807.E2A15B13A7@lists.varnish-cache.org> commit a90a099dd9d476135ab0da97f498c607b18ef3c5 Author: Dridi Boukelmoune Date: Tue Apr 16 09:19:21 2019 +0200 mv coccinelle/ tools/ I also decided to lowercase semantic patch file names, and to not mislead people about the WS_Reserve change being a vmod-only thing. diff --git a/coccinelle/README.rst b/coccinelle/README.rst deleted file mode 100644 index f279cb563..000000000 --- a/coccinelle/README.rst +++ /dev/null @@ -1,10 +0,0 @@ -This directory contains `coccinelle`_ patches to facilitate code -maintenance. - -Each patch should, in a comment section, explain its purpose. - -Unless noted otherwise, all patches should work when invoked as:: - - spatch --dir . --in-place --sp-file $COCCI - -.. _coccinelle: http://coccinelle.lip6.fr/ diff --git a/doc/changes.rst b/doc/changes.rst index 8f8e52139..b3923dce8 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -63,8 +63,8 @@ C APIs (for vmod and utility authors) It will only leave the workspace reserved if the reservation request could be fulfilled. - We provide a script to automate this change in the ``coccinelle`` - subdirectory of the source tree. + We provide a script to help automate this change in the + ``tools/coccinelle`` subdirectory of the source tree. ================================ Varnish Cache 6.2.0 (2019-03-15) diff --git a/tools/coccinelle/README.rst b/tools/coccinelle/README.rst new file mode 100644 index 000000000..ff01e80bf --- /dev/null +++ b/tools/coccinelle/README.rst @@ -0,0 +1,11 @@ +This directory contains `coccinelle`_ semantic patches to facilitate code +maintenance. + +Each patch should, in a comment section, explain its purpose. They may be fit +for both the in-tree code style or out-of-tree VMOD and VUT development. + +Unless noted otherwise, all patches should work when invoked as:: + + spatch --dir . --in-place --sp-file $COCCI + +.. _coccinelle: http://coccinelle.lip6.fr/ diff --git a/coccinelle/vmod/WS_Reserve.cocci b/tools/coccinelle/ws_reserve.cocci similarity index 87% rename from coccinelle/vmod/WS_Reserve.cocci rename to tools/coccinelle/ws_reserve.cocci index f40a49dbb..14e6cf846 100644 --- a/coccinelle/vmod/WS_Reserve.cocci +++ b/tools/coccinelle/ws_reserve.cocci @@ -1,5 +1,5 @@ /* - * patch to change code with respect to #2969: Replacement of + * Patch to change code with respect to #2969: Replacement of * WS_Reserve(ws, sz) by WS_ReserveAll(ws) / WS_ReserveSize(ws, sz) * * NOTE this patch does not check/fix error handling: From dridi.boukelmoune at gmail.com Tue Apr 16 08:08:08 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 16 Apr 2019 08:08:08 +0000 (UTC) Subject: [master] 10b79a439 Add past Coccinelle patches to the collection Message-ID: <20190416080808.0B2DFB13AA@lists.varnish-cache.org> commit 10b79a439ae211bdd386e03afab6c1a795d818a4 Author: Dridi Boukelmoune Date: Tue Apr 16 10:02:46 2019 +0200 Add past Coccinelle patches to the collection Slightly enhanced compared to when they were first used. I ran them and found more opportunities to polish the code base. diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c index 4e54a4aea..ba83b862b 100644 --- a/bin/varnishd/common/common_vsmw.c +++ b/bin/varnishd/common/common_vsmw.c @@ -158,7 +158,7 @@ vsmw_mkent(const struct vsmw *vsmw, const char *pfx) if (fd < 0 && errno == ENOENT) return; if (fd >= 0) - AZ(close(fd)); + closefd(&fd); } } @@ -178,7 +178,7 @@ vsmw_addseg(struct vsmw *vsmw, struct vsmwseg *seg) AZ(VSB_finish(vsmw->vsb)); s = write(fd, VSB_data(vsmw->vsb), VSB_len(vsmw->vsb)); assert(s == VSB_len(vsmw->vsb)); - AZ(close(fd)); + closefd(&fd); } /*--------------------------------------------------------------------*/ @@ -212,7 +212,7 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx) AZ(VSB_finish(vsmw->vsb)); s = write(fd, VSB_data(vsmw->vsb), VSB_len(vsmw->vsb)); assert(s == VSB_len(vsmw->vsb)); - AZ(close(fd)); + closefd(&fd); AZ(renameat(vsmw->vdirfd, t, vsmw->vdirfd, vsmw->idx)); REPLACE(t, NULL); } @@ -250,7 +250,7 @@ vsmw_newcluster(struct vsmw *vsmw, size_t len, const char *pfx) MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED, fd, 0); - AZ(close(fd)); + closefd(&fd); assert(vc->ptr != MAP_FAILED); (void)mlock(vc->ptr, len); @@ -286,10 +286,7 @@ VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vsmcp) vsmw_lock(); CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC); - AN(vsmcp); - vc = *vsmcp; - *vsmcp = NULL; - CHECK_OBJ_NOTNULL(vc, VSMW_CLUSTER_MAGIC); + TAKE_OBJ_NOTNULL(vc, vsmcp, VSMW_CLUSTER_MAGIC); if (vc->cseg != NULL) { /* @@ -430,7 +427,7 @@ VSMW_New(int vdirfd, int mode, const char *idxname) vsmw->idx, O_APPEND | O_WRONLY | O_CREAT, vsmw->mode); assert(fd >= 0); vsmw_idx_head(vsmw, fd); - AZ(close(fd)); + closefd(&fd); vsmw_unlock(); return (vsmw); @@ -450,7 +447,7 @@ VSMW_Destroy(struct vsmw **pp) assert (errno == ENOENT); REPLACE(vsmw->idx, NULL); VSB_destroy(&vsmw->vsb); - AZ(close(vsmw->vdirfd)); + closefd(&vsmw->vdirfd); FREE_OBJ(vsmw); vsmw_unlock(); } diff --git a/lib/libvarnish/vjsn.c b/lib/libvarnish/vjsn.c index 2ef5629ca..6a265cbe0 100644 --- a/lib/libvarnish/vjsn.c +++ b/lib/libvarnish/vjsn.c @@ -95,10 +95,7 @@ vjsn_delete(struct vjsn **jp) { struct vjsn *js; - AN(jp); - js = *jp; - *jp = NULL; - CHECK_OBJ_NOTNULL(js, VJSN_MAGIC); + TAKE_OBJ_NOTNULL(js, jp, VJSN_MAGIC); if (js->value != NULL) vjsn_val_delete(js->value); free(js->raw); diff --git a/lib/libvarnish/vlu.c b/lib/libvarnish/vlu.c index bc32df9e0..d7e163616 100644 --- a/lib/libvarnish/vlu.c +++ b/lib/libvarnish/vlu.c @@ -77,10 +77,7 @@ VLU_Destroy(struct vlu **lp) { struct vlu *l; - AN(lp); - l = *lp; - *lp = NULL; - CHECK_OBJ_NOTNULL(l, LINEUP_MAGIC); + TAKE_OBJ_NOTNULL(l, lp, LINEUP_MAGIC); free(l->buf); FREE_OBJ(l); } diff --git a/tools/coccinelle/check_obj.cocci b/tools/coccinelle/check_obj.cocci new file mode 100644 index 000000000..37eff6b68 --- /dev/null +++ b/tools/coccinelle/check_obj.cocci @@ -0,0 +1,13 @@ +/* + * This patch removes a redundant null check. + */ + +@@ +expression obj, magic; +@@ + +if (obj != NULL) { +- CHECK_OBJ_NOTNULL(obj, magic); ++ CHECK_OBJ(obj, magic); +... +} diff --git a/tools/coccinelle/closefd.cocci b/tools/coccinelle/closefd.cocci new file mode 100644 index 000000000..f1f75a33a --- /dev/null +++ b/tools/coccinelle/closefd.cocci @@ -0,0 +1,24 @@ +/* + * The closefd() macro takes a pointer to the file descriptor to wipe it in + * addition to closing the file and checking there was no error. This can + * prevent use-after-close bugs. It will not work if the variable or field + * holding the file descriptor has a const qualifier. It also make sures that + * fd looks like a valid file descriptor before even calling close(). + * + * The second part of the patch undoes the change when fd is a constant such + * as STDIN_FILENO where it becomes nonsensical. + */ + +@@ +expression fd; +@@ + +- AZ(close(fd)); ++ closefd(&fd); + +@@ +constant fd; +@@ + +- closefd(&fd); ++ AZ(close(fd)); diff --git a/tools/coccinelle/free_obj.cocci b/tools/coccinelle/free_obj.cocci new file mode 100644 index 000000000..13feaf74d --- /dev/null +++ b/tools/coccinelle/free_obj.cocci @@ -0,0 +1,27 @@ +/* + * Make sure to use FREE_OBJ() instead of a plain free() to get additional + * safeguards offered by the macro. + */ + +@@ +expression obj, objp, magic; +@@ + +( +ALLOC_OBJ(obj, magic); +| +CAST_OBJ(obj, objp, magic); +| +CAST_OBJ_NOTNULL(obj, objp, magic); +| +CHECK_OBJ(obj, magic); +| +CHECK_OBJ_NOTNULL(obj, magic); +| +CHECK_OBJ_ORNULL(obj, magic); +| +TAKE_OBJ_NOTNULL(obj, objp, magic); +) +... +- free(obj); ++ FREE_OBJ(obj); diff --git a/tools/coccinelle/take_obj_notnull.cocci b/tools/coccinelle/take_obj_notnull.cocci new file mode 100644 index 000000000..98ac7ca93 --- /dev/null +++ b/tools/coccinelle/take_obj_notnull.cocci @@ -0,0 +1,39 @@ +/* + * The TAKE_OBJ_NOTNULL() macro emulates move semantics and better conveys the + * intent behind a common pattern in the code base, usually before freeing an + * object. + * + * This may fail to capture other incarnations of this pattern where the order + * of the operations is not exactly followed so we try several combinations. + */ + +@@ +expression obj, objp, magic; +@@ + +- AN(objp); +- obj = *objp; +- *objp = NULL; +- CHECK_OBJ_NOTNULL(obj, magic); ++ TAKE_OBJ_NOTNULL(obj, objp, magic); + +@@ +expression obj, objp, magic; +@@ + +- AN(objp); +- obj = *objp; +- CHECK_OBJ_NOTNULL(obj, magic); +- *objp = NULL; ++ TAKE_OBJ_NOTNULL(obj, objp, magic); + +@@ +expression obj, objp, magic; +@@ + +- AN(objp); +- obj = *objp; +- CHECK_OBJ_NOTNULL(obj, magic); ++ TAKE_OBJ_NOTNULL(obj, objp, magic); +... +- *objp = NULL; From dridi at varni.sh Tue Apr 16 08:08:19 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 16 Apr 2019 10:08:19 +0200 Subject: [master] 5dd049418 First step to vmod code maintenance automation In-Reply-To: References: <20190415151706.99B55965DB@lists.varnish-cache.org> Message-ID: > On 15/04/2019 18:04, Dridi Boukelmoune wrote: > > Shouldn't we move this to tools/coccinelle? > > wfm, I really don't have much of an opinion about this Done! From dridi.boukelmoune at gmail.com Tue Apr 16 08:56:06 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 16 Apr 2019 08:56:06 +0000 (UTC) Subject: [master] 462e9568d Restore suggestion burried in the commit log Message-ID: <20190416085606.756FE2806@lists.varnish-cache.org> commit 462e9568d79e1f3a7d8837d39e0bef4c7c69744c Author: Dridi Boukelmoune Date: Tue Apr 16 10:54:34 2019 +0200 Restore suggestion burried in the commit log diff --git a/tools/coccinelle/closefd.cocci b/tools/coccinelle/closefd.cocci index f1f75a33a..4afbcef9e 100644 --- a/tools/coccinelle/closefd.cocci +++ b/tools/coccinelle/closefd.cocci @@ -5,6 +5,11 @@ * holding the file descriptor has a const qualifier. It also make sures that * fd looks like a valid file descriptor before even calling close(). * + * It might be worth revisiting regularly cases that don't apply to this + * patch: + * + * git grep '(void)close' + * * The second part of the patch undoes the change when fd is a constant such * as STDIN_FILENO where it becomes nonsensical. */ From dridi.boukelmoune at gmail.com Tue Apr 16 09:13:06 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 16 Apr 2019 09:13:06 +0000 (UTC) Subject: [master] 20f5c0432 Also check in return.cocci Message-ID: <20190416091306.C93CC2DEE@lists.varnish-cache.org> commit 20f5c0432b274aaf9816a57e32bafbb0634c1aea Author: Dridi Boukelmoune Date: Tue Apr 16 11:11:45 2019 +0200 Also check in return.cocci This one is much more tricky to use though, certainly not for automation purposes. diff --git a/tools/coccinelle/return.cocci b/tools/coccinelle/return.cocci new file mode 100644 index 000000000..50f877841 --- /dev/null +++ b/tools/coccinelle/return.cocci @@ -0,0 +1,35 @@ +/* + * Our code style inherited from FreeBSD explicitly requires brackets around a + * function's return (value). The trick is to always add the parenthesis and + * then undo the cases where we ended with two sets of them. + * + * It has the annoying effect of joining lines that were broken because of + * the same code style and this needs to be fixed by hand. + * + * This does not apply to lib/libvgz/ as this is mostly bundled code and one + * quick way to undo changes there after running this patch is: + * + * git checkout -- lib/libvgz/ + * + * Once libvgz is out of the way, one quick way to unbreak lines that had the + * parenthesis in place is: + * + * git checkout --patch + * + * An interactive prompt will ask what to do for each hunk of updated code and + * replying 'y' to a false positive undoes the change on disk. + */ + +@@ +expression rv; +@@ + +- return rv; ++ return (rv); + +@@ +expression rv; +@@ + +- return ((rv)); ++ return (rv); From phk at FreeBSD.org Tue Apr 16 09:34:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 16 Apr 2019 09:34:07 +0000 (UTC) Subject: [master] 5548d5300 Move the installation instructions from the homepage repo to the per-release documentation and rearrange things a bit. Message-ID: <20190416093407.CCA60460A@lists.varnish-cache.org> commit 5548d530026dea6f4c23704e6d19a0badc41417b Author: Poul-Henning Kamp Date: Tue Apr 16 09:32:14 2019 +0000 Move the installation instructions from the homepage repo to the per-release documentation and rearrange things a bit. Competent review is requested. diff --git a/doc/sphinx/installation/cloud_debian.rst b/doc/sphinx/installation/cloud_debian.rst new file mode 100644 index 000000000..317141947 --- /dev/null +++ b/doc/sphinx/installation/cloud_debian.rst @@ -0,0 +1,52 @@ +.. _cloud-debian: + +Debian cloud images +------------------- + +Varnish Cache is also made available by Varnish Software in the following +clouds providers: + +Amazon Web Services (AWS EC2) +............................. + +Here is a list of the currently available images for Ubuntu LTS on +Amazon Web Services (AWS) Elastic Compute Cloud (EC2): + +* `Varnish Cache 4 on Ubuntu LTS 14.04 on AWS`_ +* `Varnish Cache 5 on Ubuntu LTS 14.04 on AWS`_ + +.. _`Varnish Cache 4 on Ubuntu LTS 14.04 on AWS`: https://aws.amazon.com/marketplace/pp/B01H2063F6 +.. _`Varnish Cache 5 on Ubuntu LTS 14.04 on AWS`: https://aws.amazon.com/marketplace/pp/B01MU4VLOA + +Microsoft Azure +............... + +Here is a list of the currently available images for Ubuntu LTS on +Microsoft's Azure cloud: + +* `Varnish Cache 4 and 5 on Ubuntu LTS 14.04 on Azure`_ + +.. _`Varnish Cache 4 and 5 on Ubuntu LTS 14.04 on Azure`: https://azuremarketplace.microsoft.com/en-us/marketplace/apps/varnish.varnish-cache_ + +Google Cloud Platform (GCP) +........................... + +Here is a list of the currently available images for Ubuntu LTS on +Google Cloud Platform (GCP): + +* `Varnish Cache 4 on Ubuntu LTS 14.04 on GCP`_ +* `Varnish Cache 5 on Ubuntu LTS 14.04 on GCP`_ + +.. _`Varnish Cache 4 on Ubuntu LTS 14.04 on GCP`: https://console.cloud.google.com/launcher/details/varnish-public/varnish-cache-4-payg-ubuntu +.. _`Varnish Cache 5 on Ubuntu LTS 14.04 on GCP`: https://console.cloud.google.com/launcher/details/varnish-public/varnish-cache-5-payg-ubuntu + +UPLEX Packages with vmods +......................... + +`UPLEX`_ provides packages of varnish-cache and various vmods for +Debian, Ubuntu and RHEL/CentOS + +* `UPLEX Packages`_ + +.. _`UPLEX Packages`: https://pkg.uplex.de/ +.. _`UPLEX`: https://uplex.de/#anchorvarnish diff --git a/doc/sphinx/installation/cloud_redhat.rst b/doc/sphinx/installation/cloud_redhat.rst new file mode 100644 index 000000000..e6e0a4b3d --- /dev/null +++ b/doc/sphinx/installation/cloud_redhat.rst @@ -0,0 +1,45 @@ +.. _cloud-redhat: + +RedHat Cloud images +=================== + +Varnish Cache is also made available by Varnish Software in the following +clouds providers: + + +Amazon Web Services (AWS EC2) +............................. + +Here is a list of the currently available images for RHEL7 on +Amazon Web Services (AWS) Elastic Compute Cloud (EC2): + +* `Varnish Cache 4 on Red Hat Enterprise Linux 7 on AWS`_ +* `Varnish Cache 5 on Red Hat Enterprise Linux 7 on AWS`_ + +.. _`Varnish Cache 4 on Red Hat Enterprise Linux 7 on AWS`: https://aws.amazon.com/marketplace/pp/B01H2061O4 +.. _`Varnish Cache 5 on Red Hat Enterprise Linux 7 on AWS`: https://aws.amazon.com/marketplace/pp/B01MR09UKM + + +Microsoft Azure +............... + +Here is a list of the currently available images for RHEL7 on +Microsoft's Azure cloud: + +* `Varnish Cache 4 and 5 on Red Hat Enterprise Linux 7 on Azure`_ + +.. _`Varnish Cache 4 and 5 on Red Hat Enterprise Linux 7 on Azure`: https://azuremarketplace.microsoft.com/en-us/marketplace/apps/varnish.varnish-cache_ + + +Google Cloud Platform (GCP) +........................... + +Here is a list of the currently available images for RHEL7 on +Google Cloud Platform (GCP): + +* `Varnish Cache 4 on Red Hat Enterprise Linux 7 on GCP`_ +* `Varnish Cache 5 on Red Hat Enterprise Linux 7 on GCP`_ + +.. _`Varnish Cache 4 on Red Hat Enterprise Linux 7 on GCP`: https://console.cloud.google.com/launcher/details/varnish-public/varnish-cache-4-payg-red-hat +.. _`Varnish Cache 5 on Red Hat Enterprise Linux 7 on GCP`: https://console.cloud.google.com/launcher/details/varnish-public/varnish-cache-5-payg-red-hat + diff --git a/doc/sphinx/installation/cloud_ubuntu.rst b/doc/sphinx/installation/cloud_ubuntu.rst new file mode 100644 index 000000000..b2231a950 --- /dev/null +++ b/doc/sphinx/installation/cloud_ubuntu.rst @@ -0,0 +1,43 @@ +.. _cloud-ubuntu: + +Ubuntu Cloud images +------------------- + +Varnish Cache is also made available by Varnish Software in the following +clouds providers: + +Amazon Web Services (AWS EC2) +............................. + +Here is a list of the currently available images for Ubuntu LTS on +Amazon Web Services (AWS) Elastic Compute Cloud (EC2): + +* `Varnish Cache 4 on Ubuntu LTS 14.04 on AWS`_ +* `Varnish Cache 5 on Ubuntu LTS 14.04 on AWS`_ + +.. _`Varnish Cache 4 on Ubuntu LTS 14.04 on AWS`: https://aws.amazon.com/marketplace/pp/B01H2063F6 +.. _`Varnish Cache 5 on Ubuntu LTS 14.04 on AWS`: https://aws.amazon.com/marketplace/pp/B01MU4VLOA + +Microsoft Azure +............... + +Here is a list of the currently available images for Ubuntu LTS on +Microsoft's Azure cloud: + +* `Varnish Cache 4 and 5 on Ubuntu LTS 14.04 on Azure`_ + +.. _`Varnish Cache 4 and 5 on Ubuntu LTS 14.04 on Azure`: https://azuremarketplace.microsoft.com/en-us/marketplace/apps/varnish.varnish-cache_ + +Google Cloud Platform (GCP) +........................... + +Here is a list of the currently available images for Ubuntu LTS on +Google Cloud Platform (GCP): + +* `Varnish Cache 4 on Ubuntu LTS 14.04 on GCP`_ +* `Varnish Cache 5 on Ubuntu LTS 14.04 on GCP`_ + +.. _`Varnish Cache 4 on Ubuntu LTS 14.04 on GCP`: https://console.cloud.google.com/launcher/details/varnish-public/varnish-cache-4-payg-ubuntu +.. _`Varnish Cache 5 on Ubuntu LTS 14.04 on GCP`: https://console.cloud.google.com/launcher/details/varnish-public/varnish-cache-5-payg-ubuntu + +(this is left here to avoid linkrot) diff --git a/doc/sphinx/installation/index.rst b/doc/sphinx/installation/index.rst index f758d8486..525fdcd44 100644 --- a/doc/sphinx/installation/index.rst +++ b/doc/sphinx/installation/index.rst @@ -3,12 +3,12 @@ Varnish Installation ==================== -This section covers installation prerequisites, a step-by-step installation procedure, how and where -to get help, and how to report bugs. It also contains a set of platform specific notes to aid you when installing Varnish on certain platforms. - -.. XXX: rewrite the last paragraph. +This section covers installation prerequisites, a step-by-step +installation procedure, how and where to get help, and how to report +bugs. .. toctree:: + :maxdepth: 2 prerequisites.rst install.rst diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst index 9010e8153..4710f7a62 100644 --- a/doc/sphinx/installation/install.rst +++ b/doc/sphinx/installation/install.rst @@ -11,207 +11,37 @@ matter of personal taste. If you don't know which method to choose, we recommend that you read this whole section and then choose the method you feel most comfortable with. +Unfortunately, something as basic as installing a piece of software +is highly operating system specific: -Source or packages? -------------------- +.. toctree:: + :maxdepth: 2 -Installing Varnish on most relevant operating systems can usually -be done with with the specific systems package manager, typical examples -being: + install_debian + install_freebsd + install_redhat -FreeBSD -------- +Cloud images of Varnish +======================= -Binary package: - ``pkg_add -r varnish4`` -From source: - ``cd /usr/ports/www/varnish4 && make install clean`` +.. toctree:: + :maxdepth: 2 -Red Hat / CentOS ----------------- - -We try to keep the latest version available as prebuilt RPMs (el5 and el6) -on `packagecloud.io/varnishcache `_. See the online -`Red Hat installation instructions -`_ for more information. - -Varnish is included in the `EPEL -`_ repository, however due to -incompatible syntax changes in newer versions of Varnish, only older -versions are available. - -We therefore recommend that you install the latest version directly from our repository, as described above. - -Debian/Ubuntu -------------- - -Varnish is distributed with both Debian and Ubuntu. In order to get -Varnish up and running type ``sudo apt-get install varnish``. Please -note that this might not be the latest version of Varnish. If you -need a later version of Varnish, please follow the online installation -instructions for `Debian -`_ or `Ubuntu -`_. + cloud_debian + cloud_redhat + cloud_ubuntu Compiling Varnish from source ============================= If there are no binary packages available for your system, or if you -want to compile Varnish from source for other reasons, follow these -steps: - -Download the appropriate release tarball, which you can find on -https://varnish-cache.org/releases/ . - -Alternatively, if you want to hack on Varnish, you should clone our -git repository by doing. - - ``git clone https://github.com/varnishcache/varnish-cache`` - - -Build dependencies on Debian / Ubuntu --------------------------------------- - -.. grep-dctrl -n -sBuild-Depends -r ^ ../../../../varnish-cache-debian/control | tr -d '\n' | awk -F,\ '{ for (i = 0; ++i <= NF;) { sub (/ .*/, "", $i); print "* `" $i "`"; }}' | egrep -v '(debhelper)' - -In order to build Varnish from source you need a number of packages -installed. On a Debian or Ubuntu system, use this command to install -them (replace ``sudo apt-get install`` if needed):: - - sudo apt-get install \ - make \ - automake \ - autotools-dev \ - libedit-dev \ - libjemalloc-dev \ - libncurses-dev \ - libpcre3-dev \ - libtool \ - pkg-config \ - python3-docutils \ - python3-sphinx - -Optionally, to rebuild the svg files:: - - sudo apt-get install graphviz - -Recommended, in particular if you plan on building custom vmods:: - - sudo apt-get install autoconf-archive - -Optionally, to pull from a repository:: - - sudo apt-get install git - -Build dependencies on Red Hat / CentOS --------------------------------------- - -.. gawk '/^BuildRequires/ {print "* `" $2 "`"}' ../../../redhat/varnish.spec | sort | uniq | egrep -v '(systemd)' - -To build Varnish on a Red Hat or CentOS system, this command should -install required packages (replace ``sudo yum install`` if needed):: - - sudo yum install \ - make \ - autoconf \ - automake \ - jemalloc-devel \ - libedit-devel \ - libtool \ - ncurses-devel \ - pcre-devel \ - pkgconfig \ - python3-docutils \ - python3-sphinx - -Optionally, to rebuild the svg files:: - - yum install graphviz - -Optionally, to pull from a repository:: - - yum install git - -.. XXX autoconf-archive ? is this any helpful on the notoriously - outdated Redhats? - -Build dependencies on a SmartOS Zone ------------------------------------- - -As of SmartOS pkgsrc 2017Q1, install the following packages:: - - pkgin in autoconf automake libedit libtool ncurses \ - pcre py27-sphinx python27 gmake gcc49 pkg-config - -Optionally, to rebuild the svg files:: - - pkgin in graphviz - -Optionally, to pull from a repository:: - - pkgin in git - -Building on Solaris and other Solaris-ish OSes ----------------------------------------------- - -Building with gcc should be straight forward, as long as the above -requirements are installed. - -By convention, consider installing Varnish under `/opt/local` using:: - - ./configure \ - --prefix=/opt/local \ - --mandir=/opt/local/man - -Alternatively, building with Solaris Studio 12.4 should work -considering the following recommendations: - -* have GNU `nm` in `$PATH` before Solaris `nm` -* Provide compiler flags for `configure` to include paths under which - dependencies are installed. Example for `/opt/local`:: - - ./configure \ - --prefix=/opt/local \ - --mandir=/opt/local/man \ - CPPFLAGS="-I/opt/local/include" \ - CFLAGS="-m64" \ - LDFLAGS="-L/opt/local/lib -R/opt/local/lib" - -Compiling Varnish ------------------ - -The configuration will need the dependencies above satisfied. Once that is -taken care of:: - - cd varnish-cache - sh autogen.sh - sh configure - make - -The `configure` script takes some arguments, but more likely than not you can -forget about that for now, almost everything in Varnish can be tweaked with run -time parameters. - -Before you install, you may want to run the test suite, make a cup of -tea while it runs, it usually takes a couple of minutes:: - - make check - -Don't worry if one or two tests fail. Some of the tests are a -bit too timing sensitive (Please tell us which so we can fix them). -However, if a lot of them fail, and in particular if the `b00000.vtc` test -fails, something is horribly wrong. You will get nowhere without -figuring this one out. - -Installing ----------- +want to compile Varnish from source for other reasons: -And finally, the true test of a brave heart: ``sudo make install`` +.. toctree:: + :maxdepth: 2 -Varnish will now be installed in ``/usr/local``. The ``varnishd`` binary is in -`/usr/local/sbin/varnishd`. To make sure that the necessary links and caches -of the most recent shared libraries are found, run ``sudo ldconfig``. + install_source Next steps ========== diff --git a/doc/sphinx/installation/install_debian.rst b/doc/sphinx/installation/install_debian.rst new file mode 100644 index 000000000..c4647ceed --- /dev/null +++ b/doc/sphinx/installation/install_debian.rst @@ -0,0 +1,40 @@ +.. _install-debian: + +Installing on Debian/Ubuntu +=========================== + +From package +------------ + +Type:: + + sudo apt-get install varnish + + +Official packages of 6 +---------------------- + +Starting from Varnish Cache 5.0, we've simplified our packaging down to two: +the main package and a development package. + +The official Varnish Cache repository is now hosted at Packagecloud.io. +Note that while Packagecloud.io provides Bash Script installs, we recommend +using the manual installation procedures. + +Instructions for installing the official repository which contains the newest +Varnish Cache 6 release are available at: + +* https://packagecloud.io/varnishcache/varnish60lts/install#manual-deb + +With the release of 6.0.2, users have to switch to switch repositories to get +the latest version. +Read more about this on `Release 6.0.2 `_. + + +Official packages of 4.1 +------------------------ + +To use Varnish Cache 4.1 packages from the official varnish-cache.org repos, +follow the instructions available at: + +* https://packagecloud.io/varnishcache/varnish41/install#manual-deb diff --git a/doc/sphinx/installation/install_freebsd.rst b/doc/sphinx/installation/install_freebsd.rst new file mode 100644 index 000000000..76cef4389 --- /dev/null +++ b/doc/sphinx/installation/install_freebsd.rst @@ -0,0 +1,27 @@ +.. _install-freebsd: + +Installing on FreeBSD +===================== + +From package +------------ + +FreeBSD offers two versions of Varnish pre-packaged:: + + pkg install varnish6 + +or, if for some reason you want the older version:: + + pkg install varnish4 + +From ports +---------- + +The FreeBSD packages are built out of the "ports" tree, and you can +install varnish directly from ports if you prefer, for instance to +get a newer version of Varnish than the current set of prebuilt +packages provide:: + + cd /usr/ports/www/varnish6 + make all install clean + diff --git a/doc/sphinx/installation/install_redhat.rst b/doc/sphinx/installation/install_redhat.rst new file mode 100644 index 000000000..235d4d734 --- /dev/null +++ b/doc/sphinx/installation/install_redhat.rst @@ -0,0 +1,56 @@ +.. _install-redhat: + +Installing on RedHat or CentOS +============================== + +Varnish is included in the `EPEL +`_ repository, however due to +incompatible syntax changes in newer versions of Varnish, only older +versions are available. + +We therefore recommend that you install the latest version directly from our repository, as described above. + +Varnish Cache is packaged in RPMs for easy installation and upgrade on Red Hat +systems. The Varnish Cache project maintains official packages for the current +Enterprise Linux versions. Varnish Cache 4.1 and 5.x are supported on EL6 and EL7. + +We try to keep the latest version available as prebuilt RPMs (el5 and el6) +on `packagecloud.io/varnishcache `_. + +Official packages of 6 +---------------------- + +Starting from Varnish Cache 5.0, we've simplified our packaging down to two: +the main package and a development package. + +The official Varnish Cache repository is now hosted at Packagecloud.io. +Note that while Packagecloud.io provides Bash Script installs, we recommend +using the manual installation procedures. + +Instructions for installing the official repository which contains the newest +Varnish Cache 6 release are available at: + +* https://packagecloud.io/varnishcache/varnish60lts/install#manual-rpm + +With the release of 6.0.2, users have to switch to switch repositories to get +the latest version. +Read more about this on `Release 6.0.2 `_. + + +Official packages of 4.1 +------------------------ + +To use Varnish Cache 4.1 packages from the official varnish-cache.org repos, +follow the instructions available at: + +* https://packagecloud.io/varnishcache/varnish41/install#manual-rpm + +External packaging +------------------ +Varnish Cache is also distributed in third party package repositories. + +.. _`Fedora EPEL`: https://fedoraproject.org/wiki/EPEL + +* `Fedora EPEL`_ does community packaging of Varnish Cache. + +* RedHat has packaged versions of Varnish Cache available since Software Collections 2.1. Announcement on . diff --git a/doc/sphinx/installation/install_source.rst b/doc/sphinx/installation/install_source.rst new file mode 100644 index 000000000..b9a95cc14 --- /dev/null +++ b/doc/sphinx/installation/install_source.rst @@ -0,0 +1,179 @@ +.. _install-src: + +Compiling Varnish from source +============================= + +If there are no binary packages available for your system, or if you +want to compile Varnish from source for other reasons, follow these +steps: + +Getting hold of the source +-------------------------- + +Download the appropriate release tarball, which you can find on +https://varnish-cache.org/releases/ . + +Alternatively, if you want to hack on Varnish, you should clone our +git repository by doing. + + ``git clone https://github.com/varnishcache/varnish-cache`` + +Build dependencies on FreeBSD +----------------------------- + +To get the dependencies required to build varnish from source +you can either:: + + pkg install automake pkgconf py36-sphinx py36-docutils pcre libtool + +And optionally, to be able to run all the testcases:: + + pkg install haproxy nghttp2 vttest + +Or if you want the built from sources:: + + cd /usr/ports/www/varnish6 + make depends clean + +Build dependencies on Debian / Ubuntu +-------------------------------------- + +.. grep-dctrl -n -sBuild-Depends -r ^ ../../../../varnish-cache-debian/control | tr -d '\n' | awk -F,\ '{ for (i = 0; ++i <= NF;) { sub (/ .*/, "", $i); print "* `" $i "`"; }}' | egrep -v '(debhelper)' + +In order to build Varnish from source you need a number of packages +installed. On a Debian or Ubuntu system, use this command to install +them (replace ``sudo apt-get install`` if needed):: + + sudo apt-get install \ + make \ + automake \ + autotools-dev \ + libedit-dev \ + libjemalloc-dev \ + libncurses-dev \ + libpcre3-dev \ + libtool \ + pkg-config \ + python3-docutils \ + python3-sphinx + +Optionally, to rebuild the svg files:: + + sudo apt-get install graphviz + +Recommended, in particular if you plan on building custom vmods:: + + sudo apt-get install autoconf-archive + +Optionally, to pull from a repository:: + + sudo apt-get install git + +Build dependencies on Red Hat / CentOS +-------------------------------------- + +.. gawk '/^BuildRequires/ {print "* `" $2 "`"}' ../../../redhat/varnish.spec | sort | uniq | egrep -v '(systemd)' + +To build Varnish on a Red Hat or CentOS system, this command should +install required packages (replace ``sudo yum install`` if needed):: + + sudo yum install \ + make \ + autoconf \ + automake \ + jemalloc-devel \ + libedit-devel \ + libtool \ + ncurses-devel \ + pcre-devel \ + pkgconfig \ + python3-docutils \ + python3-sphinx + +Optionally, to rebuild the svg files:: + + yum install graphviz + +Optionally, to pull from a repository:: + + yum install git + +.. XXX autoconf-archive ? is this any helpful on the notoriously + outdated Redhats? + +Build dependencies on a SmartOS Zone +------------------------------------ + +As of SmartOS pkgsrc 2017Q1, install the following packages:: + + pkgin in autoconf automake libedit libtool ncurses \ + pcre py27-sphinx python27 gmake gcc49 pkg-config + +Optionally, to rebuild the svg files:: + + pkgin in graphviz + +Optionally, to pull from a repository:: + + pkgin in git + +Building on Solaris and other Solaris-ish OSes +---------------------------------------------- + +Building with gcc should be straight forward, as long as the above +requirements are installed. + +By convention, consider installing Varnish under `/opt/local` using:: + + ./configure \ + --prefix=/opt/local \ + --mandir=/opt/local/man + +Alternatively, building with Solaris Studio 12.4 should work +considering the following recommendations: + +* have GNU `nm` in `$PATH` before Solaris `nm` +* Provide compiler flags for `configure` to include paths under which + dependencies are installed. Example for `/opt/local`:: + + ./configure \ + --prefix=/opt/local \ + --mandir=/opt/local/man \ + CPPFLAGS="-I/opt/local/include" \ + CFLAGS="-m64" \ + LDFLAGS="-L/opt/local/lib -R/opt/local/lib" + +Compiling Varnish +----------------- + +The configuration will need the dependencies above satisfied. Once that is +taken care of:: + + cd varnish-cache + sh autogen.sh + sh configure + make + +The `configure` script takes some arguments, but more likely than not you can +forget about that for now, almost everything in Varnish can be tweaked with run +time parameters. + +Before you install, you may want to run the test suite, make a cup of +tea while it runs, it usually takes a couple of minutes:: + + make check + +Don't worry if one or two tests fail. Some of the tests are a +bit too timing sensitive (Please tell us which so we can fix them). +However, if a lot of them fail, and in particular if the `b00000.vtc` test +fails, something is horribly wrong. You will get nowhere without +figuring this one out. + +Installing +---------- + +And finally, the true test of a brave heart: ``sudo make install`` + +Varnish will now be installed in ``/usr/local``. The ``varnishd`` binary is in +`/usr/local/sbin/varnishd`. To make sure that the necessary links and caches +of the most recent shared libraries are found, run ``sudo ldconfig``. From phk at FreeBSD.org Tue Apr 16 09:41:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 16 Apr 2019 09:41:07 +0000 (UTC) Subject: [master] ef5d2cd65 Also OpenBSD Message-ID: <20190416094107.386F749B8@lists.varnish-cache.org> commit ef5d2cd65c2fb13f1b59b18a1ebc04709d519ddb Author: Poul-Henning Kamp Date: Tue Apr 16 09:39:57 2019 +0000 Also OpenBSD diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst index 4710f7a62..a684669cc 100644 --- a/doc/sphinx/installation/install.rst +++ b/doc/sphinx/installation/install.rst @@ -19,6 +19,7 @@ is highly operating system specific: install_debian install_freebsd + install_openbsd install_redhat Cloud images of Varnish diff --git a/doc/sphinx/installation/install_openbsd.rst b/doc/sphinx/installation/install_openbsd.rst new file mode 100644 index 000000000..d8abb6909 --- /dev/null +++ b/doc/sphinx/installation/install_openbsd.rst @@ -0,0 +1,20 @@ +.. _install-openbsd: + +Installing on OpenBSD +===================== + +From package +------------ + +Varnish is distributed in the OpenBSD ports collection as 'www/varnish':: + + pkg_dd varnish + +From ports +---------- + +:: + + cd /usr/ports/www/varnish + make install + From phk at FreeBSD.org Tue Apr 16 09:51:06 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 16 Apr 2019 09:51:06 +0000 (UTC) Subject: [master] 88c8c8563 Add auxillary links to packages from homepage Message-ID: <20190416095106.C201D4DBC@lists.varnish-cache.org> commit 88c8c856351809bcde487db77eb457862e28a52f Author: Poul-Henning Kamp Date: Tue Apr 16 09:50:14 2019 +0000 Add auxillary links to packages from homepage diff --git a/doc/sphinx/installation/install.rst b/doc/sphinx/installation/install.rst index a684669cc..609d08dee 100644 --- a/doc/sphinx/installation/install.rst +++ b/doc/sphinx/installation/install.rst @@ -44,13 +44,17 @@ want to compile Varnish from source for other reasons: install_source -Next steps -========== +Other pre-built Varnish packages +================================ -After successful installation you are ready to proceed to the :ref:`tutorial-index`. +Here is a list of the ones we know about: -This tutorial is written for installations from binary packages. -In practice, it means that some configurations are not in place for installations from source code. -For example, instead of calling ``service varnish start``, you start the varnish daemon manually by typing:: +* `ArchLinux package`_ and `ArchLinux wiki`_ +* `Alpine Linux`_ +* `UPLEX Packages`_ with various vmods for Debian, Ubuntu and RHEL/CentOS + +.. _`ArchLinux package`: https://www.archlinux.org/packages/extra/x86_64/varnish/ +.. _`ArchLinux wiki`: https://wiki.archlinux.org/index.php/Varnish +.. _`Alpine Linux`: https://pkgs.alpinelinux.org/package/edge/main/x86_64/varnish +.. _`UPLEX Packages`: https://pkg.uplex.de/ - varnishd -a :6081 -T localhost:6082 -b localhost:8080 From geoff at uplex.de Tue Apr 16 11:58:07 2019 From: geoff at uplex.de (Geoff Simmons) Date: Tue, 16 Apr 2019 11:58:07 +0000 (UTC) Subject: [master] 0726d78d5 Fix the link about jails in dev-guide/dogfood. Message-ID: <20190416115807.4BB687682@lists.varnish-cache.org> commit 0726d78d5de1af76eba1408e5b8e2392f097f3da Author: Geoff Simmons Date: Tue Apr 16 13:57:09 2019 +0200 Fix the link about jails in dev-guide/dogfood. diff --git a/doc/sphinx/dev-guide/homepage_dogfood.rst b/doc/sphinx/dev-guide/homepage_dogfood.rst index 521513741..31da37fa5 100644 --- a/doc/sphinx/dev-guide/homepage_dogfood.rst +++ b/doc/sphinx/dev-guide/homepage_dogfood.rst @@ -47,7 +47,7 @@ Jails You have probably heard about Docker and Containers, but FreeBSD have had jails -`since I wrote them in 1998 `_ +`since I wrote them in 1998 `_ and they're a wonderful way to keep your server installation sane. From dridi.boukelmoune at gmail.com Tue Apr 16 17:03:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 16 Apr 2019 17:03:11 +0000 (UTC) Subject: [master] dd2e9fe4d Turn include/vtc-syntax.rst into a silent rule Message-ID: <20190416170311.9207064C1E@lists.varnish-cache.org> commit dd2e9fe4d1e0010cb8861f620497307d7acbca86 Author: Dridi Boukelmoune Date: Tue Apr 16 19:00:24 2019 +0200 Turn include/vtc-syntax.rst into a silent rule diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index d9be207ac..a9ce1bb75 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -208,8 +208,8 @@ VTCSYN_SRC = $(top_srcdir)/bin/varnishtest/vtc.c \ $(top_srcdir)/bin/varnishtest/vtc_syslog.c \ $(top_srcdir)/bin/varnishtest/vtc_varnish.c include/vtc-syntax.rst: vtc-syntax.py $(VTCSYN_SRC) - $(PYTHON) $(top_srcdir)/doc/sphinx/vtc-syntax.py $(VTCSYN_SRC) > ${@}_ - mv ${@}_ ${@} + $(AM_V_GEN) $(PYTHON) $(top_srcdir)/doc/sphinx/vtc-syntax.py $(VTCSYN_SRC) > ${@}_ + @mv ${@}_ ${@} BUILT_SOURCES += include/vtc-syntax.rst include/vmod_std.generated.rst: $(top_builddir)/lib/libvmod_std/vmod_std.rst From dridi.boukelmoune at gmail.com Tue Apr 16 17:03:08 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 16 Apr 2019 17:03:08 +0000 (UTC) Subject: [master] 5ecd3824a Only a server can rxreqhdrs Message-ID: <20190416170309.E421364C1A@lists.varnish-cache.org> commit 5ecd3824a6c72efdfef0cbbb24ab8565c9ac4652 Author: Dridi Boukelmoune Date: Tue Apr 16 18:58:38 2019 +0200 Only a server can rxreqhdrs diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index a32b0f37b..7df5a472b 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -1146,7 +1146,7 @@ cmd_http_rxreq(CMD_ARGS) /* SECTION: client-server.spec.rxreqhdrs * - * rxreqhdrs + * rxreqhdrs (server only) * Receive and parse a request's headers (but not the body). */ From dridi.boukelmoune at gmail.com Tue Apr 16 17:52:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 16 Apr 2019 17:52:12 +0000 (UTC) Subject: [master] 335c7516e Polish Message-ID: <20190416175212.36DE665DEB@lists.varnish-cache.org> commit 335c7516e5daa75c3320de1947ef6f12d81d453f Author: Dridi Boukelmoune Date: Tue Apr 16 19:50:30 2019 +0200 Polish diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c index 92ac448d3..8f2cdce77 100644 --- a/bin/varnishd/cache/cache_vrt_vcl.c +++ b/bin/varnishd/cache/cache_vrt_vcl.c @@ -181,7 +181,7 @@ VRT_AddDirector(VRT_CTX, const struct vdi_methods *m, void *priv, VSB_vprintf(vsb, fmt, ap); va_end(ap); AZ(VSB_finish(vsb)); - REPLACE((vdir->cli_name), VSB_data(vsb)); + REPLACE(vdir->cli_name, VSB_data(vsb)); VSB_destroy(&vsb); d->vcl_name = vdir->cli_name + i; From dridi.boukelmoune at gmail.com Tue Apr 16 17:52:11 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 16 Apr 2019 17:52:11 +0000 (UTC) Subject: [master] 32d2c5caa Reduce noise in VBP_Update_Backend Message-ID: <20190416175212.1DDA965DE7@lists.varnish-cache.org> commit 32d2c5caad19d2385d97fbfd68bcb2c9b7425552 Author: Dridi Boukelmoune Date: Tue Apr 16 19:50:15 2019 +0200 Reduce noise in VBP_Update_Backend diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 94757a6b3..136c7413a 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -152,6 +152,7 @@ vbp_has_poked(struct vbp_target *vt) void VBP_Update_Backend(struct vbp_target *vt) { + const struct director *dir; unsigned i = 0, chg; char bits[10]; @@ -169,7 +170,8 @@ VBP_Update_Backend(struct vbp_target *vt) bits[i] = '\0'; assert(i < sizeof bits); - if (vt->backend->director == NULL) { + dir = vt->backend->director; + if (dir == NULL) { Lck_Unlock(&vbp_mtx); return; } @@ -178,8 +180,9 @@ VBP_Update_Backend(struct vbp_target *vt) chg = (i != vt->backend->sick); vt->backend->sick = i; + AN(dir->vcl_name); VSL(SLT_Backend_health, 0, "%s %s %s %s %u %u %u %.6f %.6f %s", - vt->backend->director->vcl_name, chg ? "Went" : "Still", + dir->vcl_name, chg ? "Went" : "Still", i ? "sick" : "healthy", bits, vt->good, vt->threshold, vt->window, vt->last, vt->avg, vt->resp_buf); @@ -187,7 +190,7 @@ VBP_Update_Backend(struct vbp_target *vt) if (chg) { vt->backend->changed = VTIM_real(); - VRT_SetChanged(vt->backend->director, vt->backend->changed); + VRT_SetChanged(dir, vt->backend->changed); } Lck_Unlock(&vbp_mtx); } From dridi.boukelmoune at gmail.com Tue Apr 16 17:59:07 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 16 Apr 2019 17:59:07 +0000 (UTC) Subject: [master] e7d5fb68f Fix the VCL name of directors Message-ID: <20190416175908.771036E1DD@lists.varnish-cache.org> commit e7d5fb68f7bac45e7b96eaab48718516be60f73e Author: Dridi Boukelmoune Date: Tue Apr 16 19:55:38 2019 +0200 Fix the VCL name of directors Instead of looking like the backend's VCL name we only get the symbol name in logs, unlike the CLI. This brings back the full ${vcl}.${symbol} name everywhere. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index da1b7f8ed..2e590b85c 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -575,7 +575,7 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, be->director = VRT_AddDirector(ctx, vbp != NULL ? vbe_methods : vbe_methods_noprobe, be, - "%s", vrt->vcl_name); + "%s.%s", VCL_Name(ctx->vcl), vrt->vcl_name); if (be->director != NULL) { /* for cold VCL, update initial director state */ From dridi.boukelmoune at gmail.com Tue Apr 16 19:14:08 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 16 Apr 2019 19:14:08 +0000 (UTC) Subject: [master] 8158736c1 Revert "Fix the VCL name of directors" Message-ID: <20190416191408.7352E91D01@lists.varnish-cache.org> commit 8158736c1632dcdde2523c3e84be97e39b537284 Author: Dridi Boukelmoune Date: Tue Apr 16 21:11:15 2019 +0200 Revert "Fix the VCL name of directors" This reverts commit e7d5fb68f7bac45e7b96eaab48718516be60f73e. Apparently this change looks intentional, but it should be revisited as it makes troubleshooting significantly harder when logs (for example Backend_health records) aren't explicit about the backend they are referring to. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 2e590b85c..da1b7f8ed 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -575,7 +575,7 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, be->director = VRT_AddDirector(ctx, vbp != NULL ? vbe_methods : vbe_methods_noprobe, be, - "%s.%s", VCL_Name(ctx->vcl), vrt->vcl_name); + "%s", vrt->vcl_name); if (be->director != NULL) { /* for cold VCL, update initial director state */ From phk at phk.freebsd.dk Tue Apr 16 19:33:48 2019 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Tue, 16 Apr 2019 19:33:48 +0000 Subject: [master] 8158736c1 Revert "Fix the VCL name of directors" In-Reply-To: <20190416191408.7352E91D01@lists.varnish-cache.org> References: <20190416191408.7352E91D01@lists.varnish-cache.org> Message-ID: <58617.1555443228@critter.freebsd.dk> -------- In message <20190416191408.7352E91D01 at lists.varnish-cache.org>, Dridi Boukelmou ne writes: > Revert "Fix the VCL name of directors" The commit you revert here should at the very least have been discussed at a bugwash, as it changed one of the presumably stable and documented interfaces to Varnish. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From fgsch at lodoss.net Tue Apr 16 21:27:06 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 16 Apr 2019 21:27:06 +0000 (UTC) Subject: [master] a18ce1acb Omit some warnings in lgtm Message-ID: <20190416212706.3D7FE95B38@lists.varnish-cache.org> commit a18ce1acb9578854dbfa3264877e27995296f036 Author: Federico G. Schwindt Date: Tue Apr 16 21:44:16 2019 +0100 Omit some warnings in lgtm diff --git a/.lgtm.yml b/.lgtm.yml index 578bfad28..269898eef 100644 --- a/.lgtm.yml +++ b/.lgtm.yml @@ -1,3 +1,6 @@ +queries: + - exclude: cpp/missing-header-guard + - exclude: cpp/short-global-name extraction: cpp: prepare: From fgsch at lodoss.net Tue Apr 16 21:27:06 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 16 Apr 2019 21:27:06 +0000 (UTC) Subject: [master] d3f463888 Fix comparison is always true Message-ID: <20190416212706.5481C95B3B@lists.varnish-cache.org> commit d3f463888bfcf8bb9dc802e07e376c1ba3eadf05 Author: Federico G. Schwindt Date: Tue Apr 16 21:47:13 2019 +0100 Fix comparison is always true Via lgtm. diff --git a/lib/libvmod_blob/url.c b/lib/libvmod_blob/url.c index f035bb6ae..fd7245bb7 100644 --- a/lib/libvmod_blob/url.c +++ b/lib/libvmod_blob/url.c @@ -128,7 +128,7 @@ url_decode(const enum encoding dec, char *restrict const buf, AN(strings); assert(dec == URL); - if (n >= 0 && (size_t)n < len) + if (n >= 0) len = n; for (int i = 0; len > 0 && i < strings->n; i++) { From fgsch at lodoss.net Tue Apr 16 21:27:06 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 16 Apr 2019 21:27:06 +0000 (UTC) Subject: [master] e8cfffd2f Fix command name Message-ID: <20190416212706.6EDC095B3E@lists.varnish-cache.org> commit e8cfffd2f166dc3942811d89840247329e08f142 Author: Federico G. Schwindt Date: Tue Apr 16 22:23:29 2019 +0100 Fix command name diff --git a/doc/sphinx/installation/install_openbsd.rst b/doc/sphinx/installation/install_openbsd.rst index d8abb6909..e2dedbeb6 100644 --- a/doc/sphinx/installation/install_openbsd.rst +++ b/doc/sphinx/installation/install_openbsd.rst @@ -8,7 +8,7 @@ From package Varnish is distributed in the OpenBSD ports collection as 'www/varnish':: - pkg_dd varnish + pkg_add varnish From ports ---------- From fgsch at lodoss.net Tue Apr 16 21:27:06 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Tue, 16 Apr 2019 21:27:06 +0000 (UTC) Subject: [master] 355e7adfc Whitespace OCD Message-ID: <20190416212706.8B2FB95B44@lists.varnish-cache.org> commit 355e7adfca0b84a82a3e6ea13da4469580803d5f Author: Federico G. Schwindt Date: Tue Apr 16 22:24:01 2019 +0100 Whitespace OCD diff --git a/doc/sphinx/dev-guide/homepage_dogfood.rst b/doc/sphinx/dev-guide/homepage_dogfood.rst index 31da37fa5..3073fc4d1 100644 --- a/doc/sphinx/dev-guide/homepage_dogfood.rst +++ b/doc/sphinx/dev-guide/homepage_dogfood.rst @@ -46,7 +46,7 @@ Jails ----- You have probably heard about Docker and Containers, but FreeBSD -have had jails +have had jails `since I wrote them in 1998 `_ and they're a wonderful way to keep your server installation sane. diff --git a/doc/sphinx/dev-guide/index.rst b/doc/sphinx/dev-guide/index.rst index d8e5381d7..7d361024d 100644 --- a/doc/sphinx/dev-guide/index.rst +++ b/doc/sphinx/dev-guide/index.rst @@ -89,7 +89,7 @@ Various policies policy_governance policy_vmods - + The varnish-cache.org homepage ------------------------------ diff --git a/doc/sphinx/installation/cloud_debian.rst b/doc/sphinx/installation/cloud_debian.rst index 317141947..80464bf10 100644 --- a/doc/sphinx/installation/cloud_debian.rst +++ b/doc/sphinx/installation/cloud_debian.rst @@ -3,13 +3,13 @@ Debian cloud images ------------------- -Varnish Cache is also made available by Varnish Software in the following +Varnish Cache is also made available by Varnish Software in the following clouds providers: Amazon Web Services (AWS EC2) ............................. -Here is a list of the currently available images for Ubuntu LTS on +Here is a list of the currently available images for Ubuntu LTS on Amazon Web Services (AWS) Elastic Compute Cloud (EC2): * `Varnish Cache 4 on Ubuntu LTS 14.04 on AWS`_ @@ -21,7 +21,7 @@ Amazon Web Services (AWS) Elastic Compute Cloud (EC2): Microsoft Azure ............... -Here is a list of the currently available images for Ubuntu LTS on +Here is a list of the currently available images for Ubuntu LTS on Microsoft's Azure cloud: * `Varnish Cache 4 and 5 on Ubuntu LTS 14.04 on Azure`_ @@ -31,7 +31,7 @@ Microsoft's Azure cloud: Google Cloud Platform (GCP) ........................... -Here is a list of the currently available images for Ubuntu LTS on +Here is a list of the currently available images for Ubuntu LTS on Google Cloud Platform (GCP): * `Varnish Cache 4 on Ubuntu LTS 14.04 on GCP`_ diff --git a/doc/sphinx/installation/cloud_redhat.rst b/doc/sphinx/installation/cloud_redhat.rst index e6e0a4b3d..07bb5a67c 100644 --- a/doc/sphinx/installation/cloud_redhat.rst +++ b/doc/sphinx/installation/cloud_redhat.rst @@ -3,14 +3,14 @@ RedHat Cloud images =================== -Varnish Cache is also made available by Varnish Software in the following +Varnish Cache is also made available by Varnish Software in the following clouds providers: Amazon Web Services (AWS EC2) ............................. -Here is a list of the currently available images for RHEL7 on +Here is a list of the currently available images for RHEL7 on Amazon Web Services (AWS) Elastic Compute Cloud (EC2): * `Varnish Cache 4 on Red Hat Enterprise Linux 7 on AWS`_ @@ -23,7 +23,7 @@ Amazon Web Services (AWS) Elastic Compute Cloud (EC2): Microsoft Azure ............... -Here is a list of the currently available images for RHEL7 on +Here is a list of the currently available images for RHEL7 on Microsoft's Azure cloud: * `Varnish Cache 4 and 5 on Red Hat Enterprise Linux 7 on Azure`_ @@ -34,7 +34,7 @@ Microsoft's Azure cloud: Google Cloud Platform (GCP) ........................... -Here is a list of the currently available images for RHEL7 on +Here is a list of the currently available images for RHEL7 on Google Cloud Platform (GCP): * `Varnish Cache 4 on Red Hat Enterprise Linux 7 on GCP`_ diff --git a/doc/sphinx/installation/cloud_ubuntu.rst b/doc/sphinx/installation/cloud_ubuntu.rst index b2231a950..9693b07c6 100644 --- a/doc/sphinx/installation/cloud_ubuntu.rst +++ b/doc/sphinx/installation/cloud_ubuntu.rst @@ -3,13 +3,13 @@ Ubuntu Cloud images ------------------- -Varnish Cache is also made available by Varnish Software in the following +Varnish Cache is also made available by Varnish Software in the following clouds providers: Amazon Web Services (AWS EC2) ............................. -Here is a list of the currently available images for Ubuntu LTS on +Here is a list of the currently available images for Ubuntu LTS on Amazon Web Services (AWS) Elastic Compute Cloud (EC2): * `Varnish Cache 4 on Ubuntu LTS 14.04 on AWS`_ @@ -21,7 +21,7 @@ Amazon Web Services (AWS) Elastic Compute Cloud (EC2): Microsoft Azure ............... -Here is a list of the currently available images for Ubuntu LTS on +Here is a list of the currently available images for Ubuntu LTS on Microsoft's Azure cloud: * `Varnish Cache 4 and 5 on Ubuntu LTS 14.04 on Azure`_ @@ -31,7 +31,7 @@ Microsoft's Azure cloud: Google Cloud Platform (GCP) ........................... -Here is a list of the currently available images for Ubuntu LTS on +Here is a list of the currently available images for Ubuntu LTS on Google Cloud Platform (GCP): * `Varnish Cache 4 on Ubuntu LTS 14.04 on GCP`_ diff --git a/doc/sphinx/installation/install_debian.rst b/doc/sphinx/installation/install_debian.rst index c4647ceed..405824252 100644 --- a/doc/sphinx/installation/install_debian.rst +++ b/doc/sphinx/installation/install_debian.rst @@ -21,7 +21,7 @@ The official Varnish Cache repository is now hosted at Packagecloud.io. Note that while Packagecloud.io provides Bash Script installs, we recommend using the manual installation procedures. -Instructions for installing the official repository which contains the newest +Instructions for installing the official repository which contains the newest Varnish Cache 6 release are available at: * https://packagecloud.io/varnishcache/varnish60lts/install#manual-deb diff --git a/doc/sphinx/installation/install_redhat.rst b/doc/sphinx/installation/install_redhat.rst index 235d4d734..6d29ae203 100644 --- a/doc/sphinx/installation/install_redhat.rst +++ b/doc/sphinx/installation/install_redhat.rst @@ -27,7 +27,7 @@ The official Varnish Cache repository is now hosted at Packagecloud.io. Note that while Packagecloud.io provides Bash Script installs, we recommend using the manual installation procedures. -Instructions for installing the official repository which contains the newest +Instructions for installing the official repository which contains the newest Varnish Cache 6 release are available at: * https://packagecloud.io/varnishcache/varnish60lts/install#manual-rpm From dridi at varni.sh Wed Apr 17 05:02:49 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 17 Apr 2019 07:02:49 +0200 Subject: [master] 8158736c1 Revert "Fix the VCL name of directors" In-Reply-To: <58617.1555443228@critter.freebsd.dk> References: <20190416191408.7352E91D01@lists.varnish-cache.org> <58617.1555443228@critter.freebsd.dk> Message-ID: On Tue, Apr 16, 2019 at 9:33 PM Poul-Henning Kamp wrote: > > -------- > In message <20190416191408.7352E91D01 at lists.varnish-cache.org>, Dridi Boukelmou > ne writes: > > > Revert "Fix the VCL name of directors" > > The commit you revert here should at the very least have been > discussed at a bugwash, as it changed one of the presumably stable > and documented interfaces to Varnish. Yes, this is why I reverted it after realizing that. However I won't attend the very next bugwash so I will open an issue instead. Dridi From nils.goroll at uplex.de Wed Apr 17 06:01:50 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 17 Apr 2019 08:01:50 +0200 Subject: [master] 8158736c1 Revert "Fix the VCL name of directors" In-Reply-To: <58617.1555443228@critter.freebsd.dk> References: <20190416191408.7352E91D01@lists.varnish-cache.org> <58617.1555443228@critter.freebsd.dk> Message-ID: > The commit you revert here should at the very least have been > discussed at a bugwash, as it changed one of the presumably stable > and documented interfaces to Varnish. This really has gone back and forth, in particular wrt backend.list and the health probes over the last couple of years. Whatever we do, we should really not touch the backend names any more. From the management-side, one can adjust to anything, but we should finally stick to something. Nils -- ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ -------------- next part -------------- A non-text attachment was scrubbed... Name: signature.asc Type: application/pgp-signature Size: 488 bytes Desc: OpenPGP digital signature URL: From nils.goroll at uplex.de Wed Apr 17 08:42:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 17 Apr 2019 08:42:07 +0000 (UTC) Subject: [master] 96c9ea216 wake up vbp_thread after a probe is done Message-ID: <20190417084208.00A4EA88C3@lists.varnish-cache.org> commit 96c9ea2163ccf420070af34ae21295f6fd1c7d80 Author: Nils Goroll Date: Wed Apr 17 10:36:45 2019 +0200 wake up vbp_thread after a probe is done The due time of the probe which has just finished may be earlier than the previously earliest due time, so vbp_thread needs to be woken up to update the nxt value Fixes #2976 diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 136c7413a..6dda4273a 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -439,6 +439,7 @@ vbp_task(struct worker *wrk, void *priv) vt->due = VTIM_real() + vt->interval; binheap_delete(vbp_heap, vt->heap_idx); binheap_insert(vbp_heap, vt); + AZ(pthread_cond_signal(&vbp_cond)); } } Lck_Unlock(&vbp_mtx); From nils.goroll at uplex.de Wed Apr 17 09:42:06 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 17 Apr 2019 09:42:06 +0000 (UTC) Subject: [master] 34684ee3e reduce code within the vbp_mtx critical section insignificantly Message-ID: <20190417094206.CEFA8A9D4A@lists.varnish-cache.org> commit 34684ee3e29dfa09fdd9b736e2e4d002a6fe6c93 Author: Nils Goroll Date: Wed Apr 17 11:37:34 2019 +0200 reduce code within the vbp_mtx critical section insignificantly We could also move the VSL() outside if we copied dir->vcl_name diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 6dda4273a..c99b0abea 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -158,18 +158,18 @@ VBP_Update_Backend(struct vbp_target *vt) CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC); +#define BITMAP(n, c, t, b) \ + bits[i++] = (vt->n & 1) ? c : '-'; +#include "tbl/backend_poll.h" + bits[i] = '\0'; + assert(i < sizeof bits); + Lck_Lock(&vbp_mtx); if (vt->backend == NULL) { Lck_Unlock(&vbp_mtx); return; } -#define BITMAP(n, c, t, b) \ - bits[i++] = (vt->n & 1) ? c : '-'; -#include "tbl/backend_poll.h" - bits[i] = '\0'; - assert(i < sizeof bits); - dir = vt->backend->director; if (dir == NULL) { Lck_Unlock(&vbp_mtx); From nils.goroll at uplex.de Wed Apr 17 11:41:06 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 17 Apr 2019 11:41:06 +0000 (UTC) Subject: [master] fff848208 optimize vbp_thread wakeup Message-ID: <20190417114106.B822FAE297@lists.varnish-cache.org> commit fff848208dc2efd495f4e6aead6a898c2fca3f18 Author: Nils Goroll Date: Wed Apr 17 13:31:25 2019 +0200 optimize vbp_thread wakeup We only need to wake up vbp_thread if the probe we insert on the binheap has become the next due. As a binheap_root() call is cheap and saves posting many signals, this optimization should be worth some extra lines. Also we add a function for the two places where we insert into the heap. Ref #2976 diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index c99b0abea..c87c73465 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -409,6 +409,17 @@ vbp_poke(struct vbp_target *vt) vt->happy |= 1; } +/*-------------------------------------------------------------------- + */ +static void +vbp_heap_insert(struct vbp_target *vt) +{ + // Lck_AssertHeld(&vbp_mtx); + binheap_insert(vbp_heap, vt); + if (binheap_root(vbp_heap) == vt) + AZ(pthread_cond_signal(&vbp_cond)); +} + /*-------------------------------------------------------------------- */ @@ -438,8 +449,7 @@ vbp_task(struct worker *wrk, void *priv) if (vt->heap_idx != BINHEAP_NOIDX) { vt->due = VTIM_real() + vt->interval; binheap_delete(vbp_heap, vt->heap_idx); - binheap_insert(vbp_heap, vt); - AZ(pthread_cond_signal(&vbp_cond)); + vbp_heap_insert(vt); } } Lck_Unlock(&vbp_mtx); @@ -638,8 +648,7 @@ VBP_Control(const struct backend *be, int enable) if (enable) { assert(vt->heap_idx == BINHEAP_NOIDX); vt->due = VTIM_real(); - binheap_insert(vbp_heap, vt); - AZ(pthread_cond_signal(&vbp_cond)); + vbp_heap_insert(vt); } else { assert(vt->heap_idx != BINHEAP_NOIDX); binheap_delete(vbp_heap, vt->heap_idx); From dridi.boukelmoune at gmail.com Wed Apr 17 11:58:07 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 17 Apr 2019 11:58:07 +0000 (UTC) Subject: [master] 132e985cd Close quoted string Message-ID: <20190417115807.2BCA1AE90A@lists.varnish-cache.org> commit 132e985cd0fd4840f51ce5e0160a02f24e53e907 Author: Dridi Boukelmoune Date: Wed Apr 17 13:56:36 2019 +0200 Close quoted string diff --git a/bin/varnishtest/tests/v00004.vtc b/bin/varnishtest/tests/v00004.vtc index 5da5930f9..486f88a50 100644 --- a/bin/varnishtest/tests/v00004.vtc +++ b/bin/varnishtest/tests/v00004.vtc @@ -191,7 +191,7 @@ client c1 { -hdr "DNT: 1" \ -hdr "Connection: keep-alive" \ -hdr "Upgrade-Insecure-Requests: 1" \ - -hdr {If-None-Match: W/"9060a5e7924af13779c0437265ad2f1c} + -hdr {If-None-Match: W/"9060a5e7924af13779c0437265ad2f1c"} rxresp expect resp.status == 200 From nils.goroll at uplex.de Wed Apr 17 12:55:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 17 Apr 2019 12:55:07 +0000 (UTC) Subject: [master] c1cde2d32 docs: clarify VFP order Message-ID: <20190417125507.72809AFB1D@lists.varnish-cache.org> commit c1cde2d3294fb9da3a307517c405fd92535ba9b5 Author: Nils Goroll Date: Wed Apr 17 14:53:20 2019 +0200 docs: clarify VFP order diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index ab8459496..b1e492e06 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -939,7 +939,10 @@ beresp.filters Writable from: vcl_backend_response List of Varnish Fetch Processor (VFP) filters the beresp.body - will be pulled through. + will be pulled through. The order left to right signifies + processing from backend to cache, iow the leftmost filter is + run first on the body as received from the backend after + decoding of any transfer encodings. VFP Filters change the body before going into the cache and/or being handed to the client side, where it may get processed From nils.goroll at uplex.de Thu Apr 18 08:23:06 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 18 Apr 2019 08:23:06 +0000 (UTC) Subject: [master] b27fc03d5 mark where we might want to handle ENOSPC Message-ID: <20190418082306.94BA2A6C9A@lists.varnish-cache.org> commit b27fc03d5b8f9e2f2fffb19904d39821d622e7de Author: Nils Goroll Date: Thu Apr 18 10:15:26 2019 +0200 mark where we might want to handle ENOSPC ... which would require error handling on all of the call stack upwards, which is not that trivial an endavour. Also, for the calls via VSMW_DestroyCluster(), we are actually about to free space, but temporarily need more. Other ideas: - panic, but leave a better error message - monitor free space and refuse loading vcls if insufficient - maybe as part of some more general vcl management (e.g. "maintain max xx vcls loaded - aka auto-discard) Ref #2764 diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c index ba83b862b..7a2d9c040 100644 --- a/bin/varnishd/common/common_vsmw.c +++ b/bin/varnishd/common/common_vsmw.c @@ -122,6 +122,7 @@ vsmw_idx_head(const struct vsmw *vsmw, int fd) char buf[64]; bprintf(buf, "# %jd %jd\n", (intmax_t)vsmw->pid, (intmax_t)vsmw->birth); + // XXX handle ENOSPC? #2764 assert(write(fd, buf, strlen(buf)) == strlen(buf)); } @@ -177,6 +178,7 @@ vsmw_addseg(struct vsmw *vsmw, struct vsmwseg *seg) vsmw_fmt_index(vsmw, seg); AZ(VSB_finish(vsmw->vsb)); s = write(fd, VSB_data(vsmw->vsb), VSB_len(vsmw->vsb)); + // XXX handle ENOSPC? #2764 assert(s == VSB_len(vsmw->vsb)); closefd(&fd); } @@ -211,6 +213,7 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg, int fixidx) vsmw_fmt_index(vsmw, seg); AZ(VSB_finish(vsmw->vsb)); s = write(fd, VSB_data(vsmw->vsb), VSB_len(vsmw->vsb)); + // XXX handle ENOSPC? #2764 assert(s == VSB_len(vsmw->vsb)); closefd(&fd); AZ(renameat(vsmw->vdirfd, t, vsmw->vdirfd, vsmw->idx)); From phk at FreeBSD.org Thu Apr 18 08:55:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Apr 2019 08:55:07 +0000 (UTC) Subject: [master] 36920626b Flexelinting Message-ID: <20190418085507.C9AB5A780D@lists.varnish-cache.org> commit 36920626b0251c5ae283b258e8352b9eb4c800f4 Author: Poul-Henning Kamp Date: Thu Apr 18 07:04:25 2019 +0000 Flexelinting diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 82af410e2..b3304bedb 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -696,7 +696,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) vtim_real now; uint8_t *ptr; struct vsb *synth_body; - struct objcore *stale = NULL; + struct objcore *stale; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 0d22eeae8..2e7123205 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -751,8 +751,8 @@ HSH_Fail(struct objcore *oc) * Abandon a fetch we will not need */ -void -HSH_Abandon(struct objcore *oc) +static void +hsh_abandon(struct objcore *oc) { struct objhead *oh; @@ -792,7 +792,7 @@ HSH_Cancel(struct worker *wrk, struct objcore *oc, struct boc *boc) AN(oc->flags & OC_F_HFM); if (boc != NULL) { - HSH_Abandon(oc); + hsh_abandon(oc); ObjWaitState(oc, BOS_FINISHED); } diff --git a/bin/varnishd/cache/cache_objhead.h b/bin/varnishd/cache/cache_objhead.h index c2755dce1..286859223 100644 --- a/bin/varnishd/cache/cache_objhead.h +++ b/bin/varnishd/cache/cache_objhead.h @@ -73,5 +73,4 @@ void HSH_AddString(struct req *, void *ctx, const char *str); unsigned HSH_Purge(struct worker *, struct objhead *, vtim_real ttl_now, vtim_dur ttl, vtim_dur grace, vtim_dur keep); struct objcore *HSH_Private(const struct worker *wrk); -void HSH_Abandon(struct objcore *oc); void HSH_Cancel(struct worker *, struct objcore *, struct boc *); diff --git a/bin/varnishd/flint.lnt b/bin/varnishd/flint.lnt index 5a1942528..20166d91b 100644 --- a/bin/varnishd/flint.lnt +++ b/bin/varnishd/flint.lnt @@ -1,5 +1,14 @@ // -w4 +/////////////////////////////////////////////////////////////////////// +// deprecated + +-esym(765, WS_Reserve) +-esym(714, WS_Reserve) +-esym(759, WS_Reserve) + +/////////////////////////////////////////////////////////////////////// + -printf(3, VSL) -printf(2, http_PrintfHeader) -printf(2, VSB_printf) From phk at FreeBSD.org Thu Apr 18 08:55:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Apr 2019 08:55:07 +0000 (UTC) Subject: [master] fc1c056ac signed/unsigned flexelinting Message-ID: <20190418085507.F4160A7810@lists.varnish-cache.org> commit fc1c056ac5b1783005790e198215d1cffb142544 Author: Poul-Henning Kamp Date: Thu Apr 18 07:22:51 2019 +0000 signed/unsigned flexelinting diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index eb6304b14..06d16ee6f 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -44,7 +44,7 @@ struct acl_e { VTAILQ_ENTRY(acl_e) list; unsigned char data[ACL_MAXADDR]; - unsigned mask; + int mask; unsigned not; unsigned para; char *addr; @@ -66,7 +66,7 @@ static int vcl_acl_cmp(struct acl_e *ae1, struct acl_e *ae2) { unsigned char *p1, *p2; - unsigned m; + int m; p1 = ae1->data; p2 = ae2->data; @@ -123,7 +123,7 @@ vcc_acl_add_entry(struct vcc *tl, const struct acl_e *ae, int l, aen->data[0] = fam & 0xff; aen->mask += 8; - assert(l + 1L <= sizeof aen->data); + assert(l + 1UL <= sizeof aen->data); memcpy(aen->data + 1L, u, l); VTAILQ_FOREACH(ae2, &tl->acl, list) { diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 5f9774ed5..ff9fee700 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -140,8 +140,8 @@ struct symbol { char *name; unsigned nlen; - unsigned lorev; - unsigned hirev; + int lorev; + int hirev; struct symbol *parent; const char *vmod; @@ -348,8 +348,7 @@ void vcc_stevedore(struct vcc *vcc, const char *stv_name); /* vcc_symb.c */ void VCC_PrintCName(struct vsb *vsb, const char *b, const char *e); -struct symbol *VCC_MkSym(struct vcc *tl, const char *b, vcc_kind_t, - unsigned, unsigned); +struct symbol *VCC_MkSym(struct vcc *tl, const char *b, vcc_kind_t, int, int); extern const char XREF_NONE[]; extern const char XREF_DEF[]; extern const char XREF_REF[]; diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index 6ab7defe4..1ff8a4943 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -101,7 +101,7 @@ vcc_new_symbol(struct vcc *tl, const char *b, const char *e) static struct symbol * VCC_Symbol(struct vcc *tl, struct symbol *parent, const char *b, const char *e, vcc_kind_t kind, - int create, unsigned vlo, unsigned vhi) + int create, int vlo, int vhi) { const char *q; struct symbol *sym, *sym2 = NULL; @@ -275,7 +275,7 @@ VCC_SymbolGet(struct vcc *tl, vcc_kind_t kind, const char *e, const char *x) struct symbol * VCC_MkSym(struct vcc *tl, const char *b, vcc_kind_t kind, - unsigned vlo, unsigned vhi) + int vlo, int vhi) { struct symbol *sym; diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c index 832e5357f..0e6642552 100644 --- a/lib/libvcc/vcc_utils.c +++ b/lib/libvcc/vcc_utils.c @@ -105,7 +105,7 @@ vcc_suckaddr(struct vcc *tl, const char *host, const struct suckaddr *vsa, const int sz = sizeof(unsigned long long); const unsigned n = (vsa_suckaddr_len + sz - 1) / sz; unsigned long long b[n]; - int len; + unsigned len; char *q; VTCP_name(vsa, a, sizeof a, p, sizeof p); From phk at FreeBSD.org Thu Apr 18 08:55:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Apr 2019 08:55:08 +0000 (UTC) Subject: [master] 6334f5533 Don't rely on unsigned -1 to be huge Message-ID: <20190418085508.1B971A7814@lists.varnish-cache.org> commit 6334f55336f96bfeeb8a2ece09ea27812d340934 Author: Poul-Henning Kamp Date: Thu Apr 18 07:54:29 2019 +0000 Don't rely on unsigned -1 to be huge diff --git a/lib/libvarnish/vcli_serve.c b/lib/libvarnish/vcli_serve.c index 45363b302..5c95f4f74 100644 --- a/lib/libvarnish/vcli_serve.c +++ b/lib/libvarnish/vcli_serve.c @@ -206,11 +206,12 @@ VCLS_func_help_json(struct cli *cli, const char * const *av, void *priv) static void cls_dispatch(struct cli *cli, const struct cli_proto *cp, - char * const * av, unsigned ac) + char * const * av, int ac) { int json = 0; AN(av); + assert(ac >= 0); VSB_clear(cli->sb); @@ -234,7 +235,7 @@ cls_dispatch(struct cli *cli, const struct cli_proto *cp, return; } - if (ac - 1 > cp->desc->maxarg + json) { + if (cp->desc->maxarg >= 0 && ac - 1 > cp->desc->maxarg + json) { VCLI_Out(cli, "Too many parameters\n"); VCLI_SetResult(cli, CLIS_TOOMANY); return; @@ -257,7 +258,7 @@ cls_exec(struct VCLS_fd *cfd, char * const *av) struct VCLS *cs; struct cli_proto *clp; struct cli *cli; - unsigned na; + int na; ssize_t len; char *s; unsigned lim; From phk at FreeBSD.org Thu Apr 18 08:55:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Apr 2019 08:55:08 +0000 (UTC) Subject: [master] 76007e33e Signed/Unsigned cleanup Message-ID: <20190418085508.53F7EA7819@lists.varnish-cache.org> commit 76007e33e228406a2db84ca63fbab120e56a67ff Author: Poul-Henning Kamp Date: Thu Apr 18 07:54:57 2019 +0000 Signed/Unsigned cleanup diff --git a/include/vfil.h b/include/vfil.h index 25f3b3ac6..49369f57e 100644 --- a/include/vfil.h +++ b/include/vfil.h @@ -38,7 +38,7 @@ char *VFIL_readfile(const char *pfx, const char *fn, ssize_t *sz); int VFIL_writefile(const char *pfx, const char *fn, const char *buf, size_t sz); int VFIL_nonblocking(int fd); int VFIL_fsinfo(int fd, unsigned *pbs, uintmax_t *size, uintmax_t *space); -int VFIL_allocate(int fd, off_t size, int insist); +int VFIL_allocate(int fd, uintmax_t size, int insist); void VFIL_setpath(struct vfil_path**, const char *path); typedef int vfil_path_func_f(void *priv, const char *fn); int VFIL_searchpath(const struct vfil_path *, vfil_path_func_f *func, diff --git a/lib/libvarnish/vev.c b/lib/libvarnish/vev.c index dcb712a2e..dcace04f4 100644 --- a/lib/libvarnish/vev.c +++ b/lib/libvarnish/vev.c @@ -400,7 +400,8 @@ VEV_Once(struct vev_root *evb) { double t; struct vev *e; - int i, j, k, tmo, retval = 1; + int i, k, tmo, retval = 1; + unsigned u; CHECK_OBJ_NOTNULL(evb, VEV_BASE_MAGIC); assert(evb->thread == pthread_self()); @@ -440,21 +441,21 @@ VEV_Once(struct vev_root *evb) } k = 0; - for (j = 1; j < evb->lpfd; j++) { - evb->pev[j]->fd_events = evb->pfd[j].revents; - if (evb->pev[j]->fd_events) + for (u = 1; u < evb->lpfd; u++) { + evb->pev[u]->fd_events = evb->pfd[u].revents; + if (evb->pev[u]->fd_events) k++; } assert(k == i); DBG(evb, "EVENTS %d\n", i); while (i > 0) { - for (j = BINHEAP_NOIDX + 1; j < evb->lpfd; j++) { - e = evb->pev[j]; + for (u = BINHEAP_NOIDX + 1; u < evb->lpfd; u++) { + e = evb->pev[u]; if (e->fd_events == 0) continue; - DBG(evb, "EVENT %p j=%d fd=%d ev=0x%x %d\n", - e, j, e->fd, e->fd_events, i); + DBG(evb, "EVENT %p u=%u fd=%d ev=0x%x %d\n", + e, u, e->fd, e->fd_events, i); k = e->callback(e, e->fd_events); e->fd_events = 0; i--; diff --git a/lib/libvarnish/vfil.c b/lib/libvarnish/vfil.c index 8e34c9cb8..ad41c5644 100644 --- a/lib/libvarnish/vfil.c +++ b/lib/libvarnish/vfil.c @@ -224,7 +224,7 @@ VFIL_fsinfo(int fd, unsigned *pbs, uintmax_t *psize, uintmax_t *pspace) */ int -VFIL_allocate(int fd, off_t size, int insist) +VFIL_allocate(int fd, uintmax_t size, int insist) { struct stat st; uintmax_t fsspace; diff --git a/lib/libvarnish/vlu.c b/lib/libvarnish/vlu.c index d7e163616..718ac3b86 100644 --- a/lib/libvarnish/vlu.c +++ b/lib/libvarnish/vlu.c @@ -144,19 +144,20 @@ VLU_File(int fd, vlu_f *func, void *priv, unsigned bufsize) int VLU_Feed(struct vlu *l, const char *ptr, int len) { - int i = 0, ll; + int i = 0; + unsigned u; CHECK_OBJ_NOTNULL(l, LINEUP_MAGIC); AN(ptr); assert(len > 0); while (len > 0) { - ll = len; - if (ll > l->bufl - l->bufp) - ll = l->bufl - l->bufp; - memcpy(l->buf + l->bufp, ptr, ll); - len -= ll; - ptr += ll; - l->bufp += ll; + u = len; + if (u > l->bufl - l->bufp) + u = l->bufl - l->bufp; + memcpy(l->buf + l->bufp, ptr, u); + len -= u; + ptr += u; + l->bufp += u; i = LineUpProcess(l); if (i) return (i); From phk at FreeBSD.org Thu Apr 18 08:55:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Apr 2019 08:55:08 +0000 (UTC) Subject: [master] aebfbe944 It is OK to call realloc(3) with a NULL pointer Message-ID: <20190418085508.84821A781D@lists.varnish-cache.org> commit aebfbe9448c2a57edb21ec469abce2140beec150 Author: Poul-Henning Kamp Date: Thu Apr 18 08:39:40 2019 +0000 It is OK to call realloc(3) with a NULL pointer diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c index 70d709c43..62d2e74f1 100644 --- a/lib/libvmod_directors/shard_cfg.c +++ b/lib/libvmod_directors/shard_cfg.c @@ -382,12 +382,8 @@ shardcfg_backend_expand(const struct backend_reconfig *re) else re->shardd->l_backend <<= 1; - if (re->shardd->backend) - re->shardd->backend = realloc(re->shardd->backend, - re->shardd->l_backend * sizeof *re->shardd->backend); - else - re->shardd->backend = malloc( - re->shardd->l_backend * sizeof *re->shardd->backend); + re->shardd->backend = realloc(re->shardd->backend, + re->shardd->l_backend * sizeof *re->shardd->backend); AN(re->shardd->backend); } From phk at FreeBSD.org Thu Apr 18 09:32:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Apr 2019 09:32:08 +0000 (UTC) Subject: [master] 94414c2a4 More signed/unsigned FlexeLinting Message-ID: <20190418093208.5120DA89D8@lists.varnish-cache.org> commit 94414c2a49a9b547026037e48cdc20b39b7c554d Author: Poul-Henning Kamp Date: Thu Apr 18 09:31:38 2019 +0000 More signed/unsigned FlexeLinting diff --git a/include/vapi/voptget.h b/include/vapi/voptget.h index f01ac94f9..51de7c683 100644 --- a/include/vapi/voptget.h +++ b/include/vapi/voptget.h @@ -44,7 +44,7 @@ struct vopt_list { struct vopt_spec { const struct vopt_list *vopt_list; - unsigned vopt_list_n; + int vopt_list_n; const char *vopt_optstring; const char *vopt_synopsis; const char **vopt_usage; diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h index f91c6615a..8a8872acb 100644 --- a/include/vapi/vsl.h +++ b/include/vapi/vsl.h @@ -109,7 +109,7 @@ enum VSL_reason_e { }; struct VSL_transaction { - unsigned level; + int level; uint32_t vxid; uint32_t vxid_parent; enum VSL_transaction_e type; diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index 1818767a2..2c3c160fd 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -363,7 +363,7 @@ VSL_PrintTransactions(struct VSL_data *vsl, struct VSL_transaction * const pt[], if (t->level) { /* Print header */ if (t->level > 3) - VSL_PRINT(fo, "*%1.1u* ", t->level); + VSL_PRINT(fo, "*%1.1d* ", t->level); else VSL_PRINT(fo, "%-3.*s ", (int)(t->level), "***"); @@ -385,7 +385,7 @@ VSL_PrintTransactions(struct VSL_data *vsl, struct VSL_transaction * const pt[], if (!VSL_Match(vsl, t->c)) continue; if (t->level > 3) - VSL_PRINT(fo, "-%1.1u- ", t->level); + VSL_PRINT(fo, "-%1.1d- ", t->level); else if (t->level) VSL_PRINT(fo, "%-3.*s ", (int)(t->level), "---"); diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c index 652751c52..c9505d484 100644 --- a/lib/libvarnishapi/vsl_cursor.c +++ b/lib/libvarnishapi/vsl_cursor.c @@ -323,7 +323,7 @@ vslc_file_delete(const struct VSL_cursor *cursor) /* Read n bytes from fd into buf */ static ssize_t -vslc_file_readn(int fd, void *buf, size_t n) +vslc_file_readn(int fd, void *buf, ssize_t n) { ssize_t t = 0; ssize_t l; diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c index 57c45168f..fb284a58a 100644 --- a/lib/libvarnishapi/vsl_dispatch.c +++ b/lib/libvarnishapi/vsl_dispatch.c @@ -189,7 +189,7 @@ struct VSLQ { struct vtx_tree tree; VTAILQ_HEAD(,vtx) ready; VTAILQ_HEAD(,vtx) incomplete; - unsigned n_outstanding; + int n_outstanding; struct chunkhead shmrefs; VTAILQ_HEAD(,vtx) cache; unsigned n_cache; @@ -205,7 +205,7 @@ struct VSLQ { struct VSL_transaction *ptrans[2]; struct VSLC_ptr start; ssize_t len; - size_t offset; + ssize_t offset; } raw; }; From phk at FreeBSD.org Thu Apr 18 13:13:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 18 Apr 2019 13:13:08 +0000 (UTC) Subject: [master] f74235ab2 Signed/unsigned polish Message-ID: <20190418131308.2469CB0153@lists.varnish-cache.org> commit f74235ab21fab06035ff36be72a063bbd1b38bdd Author: Poul-Henning Kamp Date: Thu Apr 18 12:46:59 2019 +0000 Signed/unsigned polish diff --git a/bin/varnishtest/tests/v00017.vtc b/bin/varnishtest/tests/v00017.vtc index d3a0ccf7a..2809b2fe1 100644 --- a/bin/varnishtest/tests/v00017.vtc +++ b/bin/varnishtest/tests/v00017.vtc @@ -1,12 +1,12 @@ varnishtest "VCL compiler coverage test: vcc_acl.c" -varnish v1 -errvcl {Too wide mask (33) for IPv4 address} { +varnish v1 -errvcl {Too wide mask (/33) for IPv4 address} { backend b { .host = "127.0.0.1"; } acl a { "10.1.2.3"/33; } sub vcl_recv { if (client.ip ~ a) { return(pass); } } } -varnish v1 -errvcl {Too wide mask (129) for IPv6 address} { +varnish v1 -errvcl {Too wide mask (/129) for IPv6 address} { backend b { .host = "127.0.0.1"; } acl a { "1::2"/129; } sub vcl_recv { if (client.ip ~ a) { return(pass); } } diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index 06d16ee6f..d038e1a3e 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -99,7 +99,7 @@ vcc_acl_add_entry(struct vcc *tl, const struct acl_e *ae, int l, if (fam == PF_INET && ae->mask > 32) { VSB_printf(tl->sb, - "Too wide mask (%u) for IPv4 address\n", ae->mask); + "Too wide mask (/%d) for IPv4 address\n", ae->mask); if (ae->t_mask != NULL) vcc_ErrWhere(tl, ae->t_mask); else @@ -108,7 +108,7 @@ vcc_acl_add_entry(struct vcc *tl, const struct acl_e *ae, int l, } if (fam == PF_INET6 && ae->mask > 128) { VSB_printf(tl->sb, - "Too wide mask (%u) for IPv6 address\n", ae->mask); + "Too wide mask (/%d) for IPv6 address\n", ae->mask); vcc_ErrWhere(tl, ae->t_mask); return; } @@ -231,7 +231,7 @@ vcc_acl_try_getaddrinfo(struct vcc *tl, struct acl_e *ae) if (ae->t_mask != NULL && i4 > 0 && i6 > 0) { VSB_printf(tl->sb, - "Mask (%u) specified, but string resolves to" + "Mask (/%d) specified, but string resolves to" " both IPv4 and IPv6 addresses.\n", ae->mask); vcc_ErrWhere(tl, ae->t_mask); return; From nils.goroll at uplex.de Thu Apr 18 15:50:09 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 18 Apr 2019 15:50:09 +0000 (UTC) Subject: [master] 101511ccb sh syntax Message-ID: <20190418155009.AF8AF4228@lists.varnish-cache.org> commit 101511ccb81904c8535f6d839b4baa2d8e243dc2 Author: Nils Goroll Date: Thu Apr 18 17:49:19 2019 +0200 sh syntax diff --git a/autogen.des b/autogen.des index 0cc52dc42..ecb24d9fa 100755 --- a/autogen.des +++ b/autogen.des @@ -26,7 +26,8 @@ rm -f configure (cd $SRCDIR && . ./autogen.sh 2>&1 | egrep -v "(subdir-objects|is in a subdirectory)") # autoconf prior to 2.62 has issues with zsh 4.2 and newer -export CONFIG_SHELL=/bin/sh +CONFIG_SHELL=/bin/sh +export CONFIG_SHELL # NB: Workaround for make distcheck not working with # NB: FreeBSD's make on -current From geoff at uplex.de Thu Apr 18 16:26:08 2019 From: geoff at uplex.de (Geoff Simmons) Date: Thu, 18 Apr 2019 16:26:08 +0000 (UTC) Subject: [master] bdf2f6394 Fix a copy/paste error in VCL_StackVDP. Message-ID: <20190418162608.0D8934ED0@lists.varnish-cache.org> commit bdf2f63946e5abd4379f6a86ec0a273b387a4e59 Author: Geoff Simmons Date: Thu Apr 18 18:23:34 2019 +0200 Fix a copy/paste error in VCL_StackVDP. It had been iterating through the vfps list, as VCL_StackVFP does, rather than the vdps. Add a test for a VMOD-defined VDP. diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c index 2f57572aa..ddc0f2510 100644 --- a/bin/varnishd/cache/cache_vrt_filter.c +++ b/bin/varnishd/cache/cache_vrt_filter.c @@ -212,7 +212,7 @@ VCL_StackVDP(struct req *req, const struct vcl *vcl, const char *fl) AN(fl); VSLb(req->vsl, SLT_Filters, "%s", fl); while (1) { - vp = vcl_filter_list_iter(&vdp_filters, &vcl->vfps, &fl); + vp = vcl_filter_list_iter(&vdp_filters, &vcl->vdps, &fl); if (vp == NULL) return (0); if (vp == vfilter_error) { diff --git a/bin/varnishtest/tests/m00048.vtc b/bin/varnishtest/tests/m00048.vtc index bb78fdfae..3b2c3d78c 100644 --- a/bin/varnishtest/tests/m00048.vtc +++ b/bin/varnishtest/tests/m00048.vtc @@ -1,4 +1,4 @@ -varnishtest "VMOD vfp" +varnishtest "VMOD vfp & vdp" server s1 { rxreq @@ -55,3 +55,27 @@ client c1 { expect resp.status == 503 } -run +server s1 -wait +server s1 { + rxreq + txresp -body "Ponto Facto, Caesar Transit!" +} -start + +varnish v1 -vcl+backend { + import debug; + + sub vcl_deliver { + if (req.http.Rot13) { + set resp.filters = "rot13"; + } + } +} + +client c1 -repeat 2 { + txreq + rxresp + expect resp.body == "Ponto Facto, Caesar Transit!" + txreq -hdr "Rot13: please" + rxresp + expect resp.body == "Cbagb Snpgb, Pnrfne Genafvg!" +} -run diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index aa94579fe..7832f4ffb 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -89,6 +89,74 @@ static const struct vfp xyzzy_rot13 = { /**********************************************************************/ +#define ROT13_BUFSZ (1 << 13) + +static int v_matchproto_(vdp_init_f) +xyzzy_rot13_init(struct req *req, void **priv) +{ + (void)req; + AN(priv); + *priv = malloc(ROT13_BUFSZ); + if (*priv == NULL) + return (-1); + return (0); +} + +static int v_matchproto_(vdp_bytes_f) +xyzzy_rot13_bytes(struct req *req, enum vdp_action act, void **priv, + const void *ptr, ssize_t len) +{ + char *q; + const char *pp; + int i, j, retval = 0; + + (void)act; + AN(priv); + AN(*priv); + AN(ptr); + if (len <= 0) + return (0); + q = *priv; + pp = ptr; + + for (i = 0, j = 0; j < len; i++, j++) { + if (pp[j] >= 'A' && pp[j] <= 'Z') + q[i] = (((pp[j] - 'A') + 13) % 26) + 'A'; + else if (pp[j] >= 'a' && pp[j] <= 'z') + q[i] = (((pp[j] - 'a') + 13) % 26) + 'a'; + else + q[i] = pp[j]; + if (i == ROT13_BUFSZ - 1) { + retval = VDP_bytes(req, VDP_FLUSH, q, ROT13_BUFSZ); + if (retval != 0) + return (retval); + i = -1; + } + } + if (j > 0 && i >= 0) + retval = VDP_bytes(req, VDP_FLUSH, q, i + 1); + return (retval); +} + +static int v_matchproto_(vdp_fini_f) +xyzzy_rot13_fini(struct req *req, void **priv) +{ + (void)req; + AN(priv); + free(*priv); + *priv = NULL; + return (0); +} + +static const struct vdp xyzzy_vdp_rot13 = { + .name = "rot13", + .init = xyzzy_rot13_init, + .bytes = xyzzy_rot13_bytes, + .fini = xyzzy_rot13_fini, +}; + +/**********************************************************************/ + VCL_STRING v_matchproto_(td_debug_author) xyzzy_author(VRT_CTX, VCL_ENUM person, VCL_ENUM someone) { @@ -305,6 +373,8 @@ event_load(VRT_CTX, struct vmod_priv *priv) * API should look like, do NOT do this anywhere else. */ VRT_AddVFP(ctx, &xyzzy_rot13); + + VRT_AddVDP(ctx, &xyzzy_vdp_rot13); return (0); } @@ -381,6 +451,7 @@ event_discard(VRT_CTX, void *priv) (void)priv; VRT_RemoveVFP(ctx, &xyzzy_rot13); + VRT_RemoveVDP(ctx, &xyzzy_vdp_rot13); if (--loads) return (0); From nils.goroll at uplex.de Thu Apr 18 16:26:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 18 Apr 2019 16:26:08 +0000 (UTC) Subject: [master] 118ac4ed1 define enum before using it Message-ID: <20190418162608.228CA4ED3@lists.varnish-cache.org> commit 118ac4ed1326e9d1c7e54df8c406f2b5c77e79fb Author: Nils Goroll Date: Thu Apr 18 18:25:28 2019 +0200 define enum before using it exotic compilers complain about incomplete type diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index f5c4bf656..b3f766080 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -79,6 +79,11 @@ struct http_conn { vtim_dur between_bytes_timeout; }; +enum htc_status_e { +#define HTC_STATUS(e, n, s, l) HTC_S_ ## e = n, +#include "tbl/htc.h" +}; + typedef enum htc_status_e htc_complete_f(struct http_conn *); /* -------------------------------------------------------------------*/ @@ -382,12 +387,6 @@ enum htc_status_e HTC_RxStuff(struct http_conn *, htc_complete_f *, #include "tbl/sess_attr.h" void SES_Set_String_Attr(struct sess *sp, enum sess_attr a, const char *src); - -enum htc_status_e { -#define HTC_STATUS(e, n, s, l) HTC_S_ ## e = n, -#include "tbl/htc.h" -}; - /* cache_shmlog.c */ extern struct VSC_main *VSC_C_main; void VSM_Init(void); From fgsch at lodoss.net Sat Apr 20 11:24:09 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 20 Apr 2019 11:24:09 +0000 (UTC) Subject: [master] 1bef464e0 Fix comparison is always true Message-ID: <20190420112409.35FE793AA2@lists.varnish-cache.org> commit 1bef464e04f94f58c7ec2abfd24d1e4dbb438c49 Author: Federico G. Schwindt Date: Sat Apr 20 10:42:54 2019 +0100 Fix comparison is always true Via lgtm. diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 7832f4ffb..51ba7dd03 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -133,7 +133,7 @@ xyzzy_rot13_bytes(struct req *req, enum vdp_action act, void **priv, i = -1; } } - if (j > 0 && i >= 0) + if (i >= 0) retval = VDP_bytes(req, VDP_FLUSH, q, i + 1); return (retval); } From nils.goroll at uplex.de Sat Apr 20 14:45:09 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 20 Apr 2019 14:45:09 +0000 (UTC) Subject: [master] cbe9e62a0 gc unused function Message-ID: <20190420144509.568AD98DF6@lists.varnish-cache.org> commit cbe9e62a05c28b7d397636f79cbc16cf29a354e9 Author: Nils Goroll Date: Sat Apr 20 16:32:09 2019 +0200 gc unused function diff --git a/bin/varnishd/waiter/waiter_priv.h b/bin/varnishd/waiter/waiter_priv.h index 502d34961..bdf0d23af 100644 --- a/bin/varnishd/waiter/waiter_priv.h +++ b/bin/varnishd/waiter/waiter_priv.h @@ -58,14 +58,6 @@ struct waiter_impl { size_t size; }; -static inline double -Wait_Tmo(const struct waited *wp) -{ - CHECK_OBJ_ORNULL(wp, WAITED_MAGIC); - AN(wp->tmo); - return (*wp->tmo); -} - static inline double Wait_When(const struct waited *wp) { From nils.goroll at uplex.de Sat Apr 20 14:45:09 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 20 Apr 2019 14:45:09 +0000 (UTC) Subject: [master] a82d8552f fix type of (struct waited).tmo and make it immediate Message-ID: <20190420144509.69EE598DF9@lists.varnish-cache.org> commit a82d8552fc9f612b2295c357a256f896c6168368 Author: Nils Goroll Date: Sat Apr 20 16:39:09 2019 +0200 fix type of (struct waited).tmo and make it immediate the pointer indirection would make sense if we had any long lived struct waited instances so the timeout parameter could change. But we don't. Also the vtim_ type was wrong, it is a duration. diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 8c9e2d7c9..95696d4eb 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -465,7 +465,7 @@ SES_Wait(struct sess *sp, const struct transport *xp) wp->priv2 = (uintptr_t)xp; wp->idle = sp->t_idle; wp->func = ses_handle; - wp->tmo = &cache_param->timeout_idle; + wp->tmo = cache_param->timeout_idle; if (Wait_Enter(pp->waiter, wp)) SES_Delete(sp, SC_PIPE_OVERFLOW, NAN); } diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index 0e2fbd4f7..63691b91b 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -338,7 +338,7 @@ VCP_Recycle(const struct worker *wrk, struct pfd **pfdp) pfd->waited->idle = VTIM_real(); pfd->state = PFD_STATE_AVAIL; pfd->waited->func = vcp_handle; - pfd->waited->tmo = &cache_param->backend_idle_timeout; + pfd->waited->tmo = cache_param->backend_idle_timeout; if (Wait_Enter(wrk->pool->waiter, pfd->waited)) { cp->methods->close(pfd); memset(pfd, 0x33, sizeof *pfd); diff --git a/bin/varnishd/waiter/cache_waiter.c b/bin/varnishd/waiter/cache_waiter.c index d481358bb..80a0deb9f 100644 --- a/bin/varnishd/waiter/cache_waiter.c +++ b/bin/varnishd/waiter/cache_waiter.c @@ -133,7 +133,6 @@ Wait_Enter(const struct waiter *w, struct waited *wp) CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC); assert(wp->fd > 0); // stdin never comes here AN(wp->func); - AN(wp->tmo); wp->idx = BINHEAP_NOIDX; return (w->impl->enter(w->priv, wp)); } diff --git a/bin/varnishd/waiter/waiter.h b/bin/varnishd/waiter/waiter.h index 4a0d8a7d5..cbdf6a6d6 100644 --- a/bin/varnishd/waiter/waiter.h +++ b/bin/varnishd/waiter/waiter.h @@ -62,7 +62,7 @@ struct waited { void *priv1; uintptr_t priv2; waiter_handle_f *func; - volatile vtim_real *tmo; + vtim_dur tmo; vtim_real idle; }; diff --git a/bin/varnishd/waiter/waiter_priv.h b/bin/varnishd/waiter/waiter_priv.h index bdf0d23af..8ef498613 100644 --- a/bin/varnishd/waiter/waiter_priv.h +++ b/bin/varnishd/waiter/waiter_priv.h @@ -62,8 +62,7 @@ static inline double Wait_When(const struct waited *wp) { CHECK_OBJ_ORNULL(wp, WAITED_MAGIC); - AN(wp->tmo); - return (wp->idle + *wp->tmo); + return (wp->idle + wp->tmo); } void Wait_Call(const struct waiter *, struct waited *, From fgsch at lodoss.net Sat Apr 20 16:43:07 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 20 Apr 2019 16:43:07 +0000 (UTC) Subject: [master] ff01007e4 GC unused function Message-ID: <20190420164307.63EF1A0826@lists.varnish-cache.org> commit ff01007e4427ffc7e70cbd73fe370bc3d2ad842f Author: Federico G. Schwindt Date: Sat Apr 20 16:42:23 2019 +0100 GC unused function diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h index 44a109b70..29373d0fe 100644 --- a/lib/libvmod_directors/shard_dir.h +++ b/lib/libvmod_directors/shard_dir.h @@ -97,14 +97,6 @@ sharddir_backend(const struct sharddir *shardd, int id) return (shardd->backend[id].backend); } -static inline const char * -sharddir_backend_ident(const struct sharddir *shardd, int host) -{ - assert(host >= 0); - assert(host < shardd->n_backend); - return (shardd->backend[host].ident); -} - #define SHDBG(flag, shardd, ...) \ do { \ if ((shardd)->debug_flags & (flag)) \ From fgsch at lodoss.net Sun Apr 21 09:50:07 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 21 Apr 2019 09:50:07 +0000 (UTC) Subject: [master] 7554a4485 Spelling Message-ID: <20190421095007.B14A26DD2@lists.varnish-cache.org> commit 7554a4485a83106e28000d05b92655f60704d96e Author: Federico G. Schwindt Date: Sat Apr 20 17:49:39 2019 +0100 Spelling diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 717fa2feb..f1b40c861 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -519,7 +519,7 @@ http_split(const char **src, const char *stop, const char *sep, * Comparison rule for tokens: * if target string starts with '"', we use memcmp() and expect closing * double quote as well - * otherwise we use strncascmp() + * otherwise we use strncasecmp() * * On match we increment *bp past the token name. */ From fgsch at lodoss.net Sun Apr 21 09:50:07 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 21 Apr 2019 09:50:07 +0000 (UTC) Subject: [master] ef18c847f Install ca-certificates Message-ID: <20190421095007.C773D6DD5@lists.varnish-cache.org> commit ef18c847faa566b2fe0dc36021b1a07e46a84bed Author: Federico G. Schwindt Date: Sun Apr 21 10:45:31 2019 +0100 Install ca-certificates Hopefully this will fix the error with coverity. diff --git a/.travis.yml b/.travis.yml index 2a19d2751..607525635 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,6 +69,12 @@ jobs: script: *script-common - <<: *test-linux stage: coverity + addons: + apt: + packages: + - ca-certificates + - python3-docutils + - python3-sphinx env: - secure: "TndnHrwJk9FRSuVQWUk+ZrRc0jcNc0PW3TnvbRicIIwvYSLkMV5Y1tCQ5Jq/P98DA48/N/gf9DCAiFkxrNSKVeOY70FKgHYWlS130GhTv7r0c8zd+CdEXNORclcbBNV5F3Pli/LxZ+RUImjOfwcIcWV4eYv54Xv7aNFDAaDt4G9QlkSwXykLlZkoWLJQXFbhDBFioT1F1mucD9q9izEEeE+kqO1QH/IfobAq9v7/WrcS38sYI+0WvB1S0ajWuZJgRYqy1bocDNcQd05Vbr9NfAdJ9y+4VTuluZtTUyLxu3/0Tw8mAjHkcpOeNU26r3LnpdRk+5JuOFej/MrCmYRRawVfyvNGtu9RwcMkv8jl48TTs5kTf6UwFqJhe85QSlSi7IszfrE8HfB7B6u8eRr67rqjTr9k/BwEQyoBdK4JElQDj4A1GYHClomxgzmMZnVLvStnAm+IjdNlee4SfY0jj2KfPBd/v6Ms+LGVqNV9NDDKRQdOQD+H52MkIWs5Xu9fU5VaWP+xjFomA9aXex3r5FCssgyQ2P+HtWPdjNEtrkNezzfZ5b+VBVP87RdxfSqkZaRxi6gof0AgeTHWoi7GN1scseiKLxxCI7C0dfQgKrXTN7mZdcED1MMYdiaSI9mlSYQDDUHMQGeY1n3a9D6bUcC/TcmYo524PoTFBZgbbYM=" before_script: From fgsch at lodoss.net Sun Apr 21 10:42:06 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 21 Apr 2019 10:42:06 +0000 (UTC) Subject: [master] 75a091d32 Redo previous Message-ID: <20190421104206.CADF590CE@lists.varnish-cache.org> commit 75a091d3267ccb5907570ef693a5595fe384a64f Author: Federico G. Schwindt Date: Sun Apr 21 11:14:25 2019 +0100 Redo previous ca-certificates is already installed so for now just ignore any ssl errors. diff --git a/.travis.yml b/.travis.yml index 607525635..35053aac7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,16 +69,13 @@ jobs: script: *script-common - <<: *test-linux stage: coverity - addons: - apt: - packages: - - ca-certificates - - python3-docutils - - python3-sphinx env: - secure: "TndnHrwJk9FRSuVQWUk+ZrRc0jcNc0PW3TnvbRicIIwvYSLkMV5Y1tCQ5Jq/P98DA48/N/gf9DCAiFkxrNSKVeOY70FKgHYWlS130GhTv7r0c8zd+CdEXNORclcbBNV5F3Pli/LxZ+RUImjOfwcIcWV4eYv54Xv7aNFDAaDt4G9QlkSwXykLlZkoWLJQXFbhDBFioT1F1mucD9q9izEEeE+kqO1QH/IfobAq9v7/WrcS38sYI+0WvB1S0ajWuZJgRYqy1bocDNcQd05Vbr9NfAdJ9y+4VTuluZtTUyLxu3/0Tw8mAjHkcpOeNU26r3LnpdRk+5JuOFej/MrCmYRRawVfyvNGtu9RwcMkv8jl48TTs5kTf6UwFqJhe85QSlSi7IszfrE8HfB7B6u8eRr67rqjTr9k/BwEQyoBdK4JElQDj4A1GYHClomxgzmMZnVLvStnAm+IjdNlee4SfY0jj2KfPBd/v6Ms+LGVqNV9NDDKRQdOQD+H52MkIWs5Xu9fU5VaWP+xjFomA9aXex3r5FCssgyQ2P+HtWPdjNEtrkNezzfZ5b+VBVP87RdxfSqkZaRxi6gof0AgeTHWoi7GN1scseiKLxxCI7C0dfQgKrXTN7mZdcED1MMYdiaSI9mlSYQDDUHMQGeY1n3a9D6bUcC/TcmYo524PoTFBZgbbYM=" before_script: - - curl --data "token=$COVTOKEN&project=varnish" -o coverity_tool.tgz https://scan.coverity.com/download/linux64 + - curl + --data "token=$COVTOKEN&project=varnish" + --insecure + -o coverity_tool.tgz https://scan.coverity.com/download/linux64 - tar xfz coverity_tool.tgz - export PATH=$PATH:$(echo $(pwd)/cov-analysis-*/bin) script: @@ -91,6 +88,7 @@ jobs: --form file=@varnish.tgz --form version="$TRAVIS_COMMIT" --form description="$TRAVIS_BRANCH" + --insecure 'https://scan.coverity.com/builds?project=varnish' stages: From nils.goroll at uplex.de Mon Apr 22 13:49:09 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 22 Apr 2019 13:49:09 +0000 (UTC) Subject: [master] 864dd9cc6 test timeout_idle Message-ID: <20190422134910.0EEECB2317@lists.varnish-cache.org> commit 864dd9cc6b86ba45e0922ae3e36032c6911f9b24 Author: Nils Goroll Date: Mon Apr 22 15:48:04 2019 +0200 test timeout_idle diff --git a/bin/varnishtest/tests/b00067.vtc b/bin/varnishtest/tests/b00067.vtc new file mode 100644 index 000000000..3f4d6a175 --- /dev/null +++ b/bin/varnishtest/tests/b00067.vtc @@ -0,0 +1,20 @@ +varnishtest "Check timeout_idle" + +varnish v1 -arg "-p timeout_idle=1" -vcl { + backend dummy { .host = "${bad_ip}"; } + + sub vcl_backend_error { + set beresp.status = 200; + set beresp.ttl = 1h; + } +} -start + +client c1 { + txreq + rxresp + delay .2 + txreq + rxresp + delay 2 + expect_close +} -run From phk at FreeBSD.org Tue Apr 23 07:21:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Apr 2019 07:21:09 +0000 (UTC) Subject: [master] d15a62d4f Whitespace OCD Message-ID: <20190423072109.959F897C6E@lists.varnish-cache.org> commit d15a62d4f569d1627999d193f3009fb67a4bd123 Author: Poul-Henning Kamp Date: Tue Apr 23 07:10:47 2019 +0000 Whitespace OCD diff --git a/include/tbl/params.h b/include/tbl/params.h index 9632f47bf..9e52e598b 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -1793,77 +1793,77 @@ PARAM( ) PARAM( - /* name */ h2_header_table_size, - /* typ */ bytes_u, - /* min */ "0b", - /* max */ NULL, - /* default */ "4k", - /* units */ "bytes", - /* flags */ 0, + /* name */ h2_header_table_size, + /* typ */ bytes_u, + /* min */ "0b", + /* max */ NULL, + /* default */ "4k", + /* units */ "bytes", + /* flags */ 0, /* s-text */ "HTTP2 header table size.\n" "This is the size that will be used for the HPACK dynamic\n" "decoding table.", - /* l-text */ "", - /* func */ NULL + /* l-text */ "", + /* func */ NULL ) PARAM( - /* name */ h2_max_concurrent_streams, - /* typ */ uint, - /* min */ "0", - /* max */ NULL, - /* default */ "100", - /* units */ "streams", - /* flags */ 0, - /* s-text */ - "HTTP2 Maximum number of concurrent streams.\n" - "This is the number of requests that can be active\n" - "at the same time for a single HTTP2 connection.", - /* l-text */ "", - /* func */ NULL + /* name */ h2_max_concurrent_streams, + /* typ */ uint, + /* min */ "0", + /* max */ NULL, + /* default */ "100", + /* units */ "streams", + /* flags */ 0, + /* s-text */ + "HTTP2 Maximum number of concurrent streams.\n" + "This is the number of requests that can be active\n" + "at the same time for a single HTTP2 connection.", + /* l-text */ "", + /* func */ NULL ) PARAM( - /* name */ h2_initial_window_size, - /* typ */ bytes_u, - /* min */ "0", - /* max */ "2147483647b", - /* default */ "65535b", - /* units */ "bytes", - /* flags */ 0, + /* name */ h2_initial_window_size, + /* typ */ bytes_u, + /* min */ "0", + /* max */ "2147483647b", + /* default */ "65535b", + /* units */ "bytes", + /* flags */ 0, /* s-text */ "HTTP2 initial flow control window size.", - /* l-text */ "", - /* func */ NULL + /* l-text */ "", + /* func */ NULL ) PARAM( - /* name */ h2_max_frame_size, - /* typ */ bytes_u, - /* min */ "16k", - /* max */ "16777215b", - /* default */ "16k", - /* units */ "bytes", - /* flags */ 0, + /* name */ h2_max_frame_size, + /* typ */ bytes_u, + /* min */ "16k", + /* max */ "16777215b", + /* default */ "16k", + /* units */ "bytes", + /* flags */ 0, /* s-text */ "HTTP2 maximum per frame payload size we are willing to accept.", - /* l-text */ "", - /* func */ NULL + /* l-text */ "", + /* func */ NULL ) PARAM( - /* name */ h2_max_header_list_size, - /* typ */ bytes_u, - /* min */ "0b", - /* max */ NULL, - /* default */ "2147483647b", - /* units */ "bytes", - /* flags */ 0, + /* name */ h2_max_header_list_size, + /* typ */ bytes_u, + /* min */ "0b", + /* max */ NULL, + /* default */ "2147483647b", + /* units */ "bytes", + /* flags */ 0, /* s-text */ "HTTP2 maximum size of an uncompressed header list.", - /* l-text */ "", - /* func */ NULL + /* l-text */ "", + /* func */ NULL ) #undef PARAM From phk at FreeBSD.org Tue Apr 23 07:21:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Apr 2019 07:21:09 +0000 (UTC) Subject: [master] ee042fc92 A bit of signed/unsigned FlexeLinting Message-ID: <20190423072109.AB23697C73@lists.varnish-cache.org> commit ee042fc92f11770fa165ff47faa64c1b16640f13 Author: Poul-Henning Kamp Date: Tue Apr 23 07:20:01 2019 +0000 A bit of signed/unsigned FlexeLinting diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index d6d3e9ac2..67aa4a9a5 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -453,7 +453,7 @@ struct req { volatile enum req_body_state_e req_body_status; enum sess_close doclose; int restarts; - int esi_level; + unsigned esi_level; struct req *topreq; /* esi_level == 0 request */ struct vcl *vcl0; diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 8bea0e81d..498aca6f2 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -560,7 +560,7 @@ struct ved_foo { unsigned magic; #define VED_FOO_MAGIC 0x6a5a262d struct ecx *ecx; - ssize_t start, last, stop, lpad; + uint64_t start, last, stop, lpad; ssize_t ll; uint64_t olen; uint8_t dbits[8]; From phk at FreeBSD.org Tue Apr 23 09:08:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Apr 2019 09:08:07 +0000 (UTC) Subject: [master] a80c4f14d Start segregating the very private API between VCC and Varnishd which nobody else is allowed to even think about calling. Message-ID: <20190423090807.E72FAA031A@lists.varnish-cache.org> commit a80c4f14d2ec046a307a4a041d7647377a7a9a4c Author: Poul-Henning Kamp Date: Tue Apr 23 08:40:16 2019 +0000 Start segregating the very private API between VCC and Varnishd which nobody else is allowed to even think about calling. diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c index 8f2cdce77..c06066da5 100644 --- a/bin/varnishd/cache/cache_vrt_vcl.c +++ b/bin/varnishd/cache/cache_vrt_vcl.c @@ -38,6 +38,8 @@ #include "vcl.h" #include "vtim.h" +#include "vcc_interface.h" + #include "cache_director.h" #include "cache_vcl.h" diff --git a/flint.lnt b/flint.lnt index 6c98c5c94..c043f969d 100644 --- a/flint.lnt +++ b/flint.lnt @@ -35,7 +35,7 @@ /////////////////////////////////////////////////////////////////////// // General stylistic issues -e663 // Suspicious array to pointer conversion --e574 // Signed-unsigned mix with relational +//-e574 // Signed-unsigned mix with relational -e641 // Converting enum '...' to int -e716 // while(1) ... -e726 // Extraneous comma ignored diff --git a/include/Makefile.am b/include/Makefile.am index ddd5fefcf..7899eda6e 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -86,6 +86,7 @@ nobase_noinst_HEADERS = \ compat/daemon.h \ vfl.h \ libvcc.h \ + vcc_interface.h \ vcli_serve.h \ vcs_version.h \ vct.h \ diff --git a/include/vcc_interface.h b/include/vcc_interface.h new file mode 100644 index 000000000..08b4756de --- /dev/null +++ b/include/vcc_interface.h @@ -0,0 +1,38 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2019 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * Runtime support for compiled VCL programs and VMODs. + * + * This file contains prototypes for functions nobody but VCC may call. + * + * NB: When this file is changed, lib/libvcc/generate.py *MUST* be rerun. + */ + +VCL_VCL VRT_vcl_get(VRT_CTX, const char *); +void VRT_vcl_rel(VRT_CTX, VCL_VCL); +void VRT_vcl_select(VRT_CTX, VCL_VCL); diff --git a/include/vrt.h b/include/vrt.h index 9f3e52e89..e4e3b8841 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -52,6 +52,9 @@ * binary/load-time compatible, increment MAJOR version * * unreleased (planned for 2019-09-15) + * VRT_vcl_get moved to vcc_interface.h + * VRT_vcl_rel emoved to vcc_interface.h + * VRT_vcl_select emoved to vcc_interface.h * [cache.h] WS_ReserveAll() added * [cache.h] WS_Reserve(ws, 0) deprecated * 9.0 (2019-03-15) @@ -110,8 +113,6 @@ * VRT_ipcmp added * VRT_Vmod_Init signature changed * VRT_vcl_lookup removed - * VRT_vcl_get added - * VRT_vcl_rel added * VRT_fail added * [cache.h] WS_Reset and WS_Snapshot signatures changed * [cache.h] WS_Front added @@ -527,11 +528,6 @@ int VRT_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len, const char *nm, const char *path, const char *file_id, const char *backup); void VRT_Vmod_Unload(VRT_CTX, struct vmod **hdl); -/* VCL program related */ -VCL_VCL VRT_vcl_get(VRT_CTX, const char *); -void VRT_vcl_rel(VRT_CTX, VCL_VCL); -void VRT_vcl_select(VRT_CTX, VCL_VCL); - typedef int vmod_event_f(VRT_CTX, struct vmod_priv *, enum vcl_event_e); typedef void vmod_priv_free_f(void *); diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index d36012eef..b5237f8f8 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -779,6 +779,7 @@ vcl_output_lang_h(struct vsb *sb) emit_file(fo, srcroot, "include/vdef.h") emit_file(fo, srcroot, "include/vrt.h") +emit_file(fo, srcroot, "include/vcc_interface.h") emit_file(fo, buildroot, "include/vcl.h") emit_file(fo, buildroot, "include/vrt_obj.h") From phk at FreeBSD.org Tue Apr 23 09:08:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Apr 2019 09:08:08 +0000 (UTC) Subject: [master] 96da1204f Emit vcc_interface.h as the last one. Message-ID: <20190423090808.0F5B5A031D@lists.varnish-cache.org> commit 96da1204ffa46fd7d2d5ab7a4ea0f938bb3b70d3 Author: Poul-Henning Kamp Date: Tue Apr 23 08:46:18 2019 +0000 Emit vcc_interface.h as the last one. diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index b5237f8f8..8f2db0651 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -779,9 +779,9 @@ vcl_output_lang_h(struct vsb *sb) emit_file(fo, srcroot, "include/vdef.h") emit_file(fo, srcroot, "include/vrt.h") -emit_file(fo, srcroot, "include/vcc_interface.h") emit_file(fo, buildroot, "include/vcl.h") emit_file(fo, buildroot, "include/vrt_obj.h") +emit_file(fo, srcroot, "include/vcc_interface.h") fo.write("\n}\n") fo.close() From phk at FreeBSD.org Tue Apr 23 09:08:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Apr 2019 09:08:08 +0000 (UTC) Subject: [master] f58469a70 Reserve "VPI_" for functions which only VCC is allowed to call. Message-ID: <20190423090808.32CC0A0322@lists.varnish-cache.org> commit f58469a700a86932bf3502aeb58e33e7e667e551 Author: Poul-Henning Kamp Date: Tue Apr 23 08:53:32 2019 +0000 Reserve "VPI_" for functions which only VCC is allowed to call. diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c index c06066da5..0137655f3 100644 --- a/bin/varnishd/cache/cache_vrt_vcl.c +++ b/bin/varnishd/cache/cache_vrt_vcl.c @@ -325,7 +325,7 @@ VRT_count(VRT_CTX, unsigned u) } VCL_VCL -VRT_vcl_get(VRT_CTX, const char *name) +VPI_vcl_get(VRT_CTX, const char *name) { VCL_VCL vcl; @@ -339,7 +339,7 @@ VRT_vcl_get(VRT_CTX, const char *name) } void -VRT_vcl_rel(VRT_CTX, VCL_VCL vcl) +VPI_vcl_rel(VRT_CTX, VCL_VCL vcl) { CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); AN(vcl); @@ -349,7 +349,7 @@ VRT_vcl_rel(VRT_CTX, VCL_VCL vcl) } void -VRT_vcl_select(VRT_CTX, VCL_VCL vcl) +VPI_vcl_select(VRT_CTX, VCL_VCL vcl) { struct req *req = ctx->req; diff --git a/doc/sphinx/reference/vtla.rst b/doc/sphinx/reference/vtla.rst index 76dccd068..a2f2e3030 100644 --- a/doc/sphinx/reference/vtla.rst +++ b/doc/sphinx/reference/vtla.rst @@ -62,6 +62,10 @@ VLU Varnish Line Up -- library functions to collect stream of bytes into lines for processing. (lib/libvarnish/vlu.c) +VPI + VCC Private Interface -- functions in varnishd which only VCC is + allowed to call. + VRE Varnish Regular Expression -- library functions for regular expression based matching and substring replacement. (lib/libvarnish/vre.c) diff --git a/include/vcc_interface.h b/include/vcc_interface.h index 08b4756de..f86157555 100644 --- a/include/vcc_interface.h +++ b/include/vcc_interface.h @@ -33,6 +33,6 @@ * NB: When this file is changed, lib/libvcc/generate.py *MUST* be rerun. */ -VCL_VCL VRT_vcl_get(VRT_CTX, const char *); -void VRT_vcl_rel(VRT_CTX, VCL_VCL); -void VRT_vcl_select(VRT_CTX, VCL_VCL); +VCL_VCL VPI_vcl_get(VRT_CTX, const char *); +void VPI_vcl_rel(VRT_CTX, VCL_VCL); +void VPI_vcl_select(VRT_CTX, VCL_VCL); diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c index 02d323a14..af1d7fee6 100644 --- a/lib/libvcc/vcc_action.c +++ b/lib/libvcc/vcc_action.c @@ -284,12 +284,12 @@ vcc_act_return_vcl(struct vcc *tl) p = New_IniFin(tl); AN(p); - VSB_printf(p->ini, "\t%s = VRT_vcl_get(ctx, \"%s\");", + VSB_printf(p->ini, "\t%s = VPI_vcl_get(ctx, \"%s\");", buf, sym->name); - VSB_printf(p->fin, "\tVRT_vcl_rel(ctx, %s);", + VSB_printf(p->fin, "\tVPI_vcl_rel(ctx, %s);", buf); } - Fb(tl, 1, "VRT_vcl_select(ctx, %s);\t/* %s */\n", + Fb(tl, 1, "VPI_vcl_select(ctx, %s);\t/* %s */\n", (const char*)sym->eval_priv, sym->name); SkipToken(tl, ')'); } From phk at FreeBSD.org Tue Apr 23 10:28:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Apr 2019 10:28:07 +0000 (UTC) Subject: [master] 43da3e4a0 Add VRT_VCL_Busy() and VRT_VCL_Unbusy() so VMODs can prevent their VCL from going cold. Message-ID: <20190423102807.6F143A2357@lists.varnish-cache.org> commit 43da3e4a015410558da851f7ae47a6c17d9d38b3 Author: Poul-Henning Kamp Date: Tue Apr 23 10:26:24 2019 +0000 Add VRT_VCL_Busy() and VRT_VCL_Unbusy() so VMODs can prevent their VCL from going cold. One half of #2471 diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c index 0137655f3..12f7843cf 100644 --- a/bin/varnishd/cache/cache_vrt_vcl.c +++ b/bin/varnishd/cache/cache_vrt_vcl.c @@ -144,6 +144,28 @@ VCL_Rel(struct vcl **vcc) /*--------------------------------------------------------------------*/ +void +VRT_VCL_Busy(VRT_CTX) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + + CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); + VCL_Ref(ctx->vcl); +} + +void +VRT_VCL_Unbusy(VRT_CTX) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + struct vcl *vcl; + + CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); + vcl = ctx->vcl; + VCL_Rel(&vcl); +} + +/*--------------------------------------------------------------------*/ + VCL_BACKEND VRT_AddDirector(VRT_CTX, const struct vdi_methods *m, void *priv, const char *fmt, ...) diff --git a/bin/varnishtest/tests/r02471.vtc b/bin/varnishtest/tests/r02471.vtc new file mode 100644 index 000000000..24c84ee1c --- /dev/null +++ b/bin/varnishtest/tests/r02471.vtc @@ -0,0 +1,63 @@ +varnishtest "test VRT_VCL_(Un)Busy()" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend { + import debug; + + sub vcl_recv { + if (req.url == "/hold") { + debug.hold_vcl_busy(); + } else if (req.url == "/release") { + debug.release_vcl_busy(); + } + return (synth(200)); + } +} -start + +varnish v1 -vcl+backend {} + +varnish v1 -cliok "vcl.state vcl1 cold" + +# Nothing holds vcl1, so it should go gold. +varnish v1 -cliexpect "cold cold 0 vcl1" "vcl.list" + + +# Grab hold of vcl1 +varnish v1 -cliok "vcl.use vcl1" +client c1 { + txreq -url "/hold" + rxresp +} -run + +# Flush worker threads hold +varnish v1 -cliok "vcl.use vcl2" +client c1 { + txreq + rxresp +} -run + +# There should still be a single busy hold on vcl1 +varnish v1 -cliok "vcl.state vcl1 cold" +varnish v1 -cliexpect "cold busy 1 vcl1" "vcl.list" + +# Release hold on vcl1 +varnish v1 -cliok "vcl.use vcl1" +client c1 { + txreq -url "/release" + rxresp +} -run + +# Flush worker threads hold +varnish v1 -cliok "vcl.use vcl2" +client c1 { + txreq + rxresp +} -run + +# Nothing holds vcl1, so it should go gold. +varnish v1 -cliok "vcl.state vcl1 cold" +varnish v1 -cliexpect "cold cold 0 vcl1" "vcl.list" diff --git a/include/vrt.h b/include/vrt.h index e4e3b8841..cdf7bd12f 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -52,6 +52,7 @@ * binary/load-time compatible, increment MAJOR version * * unreleased (planned for 2019-09-15) + * VRT_VCL_Busy() and VRT_VCL_Unbusy() added. * VRT_vcl_get moved to vcc_interface.h * VRT_vcl_rel emoved to vcc_interface.h * VRT_vcl_select emoved to vcc_interface.h @@ -575,3 +576,10 @@ void VRT_VSC_Destroy(const char *, struct vsc_seg *); void VRT_VSC_Hide(const struct vsc_seg *); void VRT_VSC_Reveal(const struct vsc_seg *); size_t VRT_VSC_Overhead(size_t); + +/* + * API to prevent VCL from going cold + */ + +void VRT_VCL_Busy(VRT_CTX); +void VRT_VCL_Unbusy(VRT_CTX); diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index 8c6ffad05..0cd1963c7 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -243,3 +243,11 @@ This function is by no means guaranteed to work cross platform and should now only be used for diagnostic purposes. 0B is returned if no sensible value can be determined. + +$Function VOID hold_vcl_busy() + +Prevent VCL from going cold + +$Function VOID release_vcl_busy() + +Allow VCL to go cold diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 51ba7dd03..fc9720a74 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -793,3 +793,17 @@ xyzzy_stk(VRT_CTX) return (0); } + +VCL_VOID +xyzzy_hold_vcl_busy(VRT_CTX) +{ + + VRT_VCL_Busy(ctx); +} + +VCL_VOID +xyzzy_release_vcl_busy(VRT_CTX) +{ + + VRT_VCL_Unbusy(ctx); +} From phk at FreeBSD.org Tue Apr 23 10:47:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Apr 2019 10:47:08 +0000 (UTC) Subject: [master] b3094c7b2 Weaken the checks to allow for thread scheduling differences Message-ID: <20190423104708.142C3A2AFF@lists.varnish-cache.org> commit b3094c7b278210323ce4bce10d6269fd6bc1a8f4 Author: Poul-Henning Kamp Date: Tue Apr 23 10:46:14 2019 +0000 Weaken the checks to allow for thread scheduling differences diff --git a/bin/varnishtest/tests/r02471.vtc b/bin/varnishtest/tests/r02471.vtc index 24c84ee1c..d9056ebc6 100644 --- a/bin/varnishtest/tests/r02471.vtc +++ b/bin/varnishtest/tests/r02471.vtc @@ -42,7 +42,7 @@ client c1 { # There should still be a single busy hold on vcl1 varnish v1 -cliok "vcl.state vcl1 cold" -varnish v1 -cliexpect "cold busy 1 vcl1" "vcl.list" +varnish v1 -cliexpect "cold busy [12] vcl1" "vcl.list" # Release hold on vcl1 varnish v1 -cliok "vcl.use vcl1" @@ -60,4 +60,4 @@ client c1 { # Nothing holds vcl1, so it should go gold. varnish v1 -cliok "vcl.state vcl1 cold" -varnish v1 -cliexpect "cold cold 0 vcl1" "vcl.list" +varnish v1 -cliexpect "cold [a-z]+ [01] vcl1" "vcl.list" From martin at varnish-software.com Tue Apr 23 12:42:08 2019 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 23 Apr 2019 12:42:08 +0000 (UTC) Subject: [master] 373684960 Make waitinglist rushes propagate on streaming delivery Message-ID: <20190423124208.59A5BA52B1@lists.varnish-cache.org> commit 3736849608cb8907b0c3ef93b01691ea3281f48a Author: Martin Blix Grydeland Date: Wed Apr 17 16:20:26 2019 +0200 Make waitinglist rushes propagate on streaming delivery This makes waitinglist rushes happen also in HSH_Lookup when encountering cache hits. This helps to get the requests on the waitinglist restarted when doing streaming delivery. Fixes: #2977 diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 2e7123205..113615b73 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -77,7 +77,8 @@ static void hsh_rush1(const struct worker *, struct objhead *, struct rush *, int); static void hsh_rush2(struct worker *, struct rush *); static int hsh_deref_objhead(struct worker *wrk, struct objhead **poh); -static int hsh_deref_objhead_unlock(struct worker *wrk, struct objhead **poh); +static int hsh_deref_objhead_unlock(struct worker *wrk, struct objhead **poh, + int); /*---------------------------------------------------------------------*/ @@ -482,7 +483,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp) if (oc != NULL && oc->flags & OC_F_HFP) { xid = ObjGetXID(wrk, oc); dttl = EXP_Dttl(req, oc); - AN(hsh_deref_objhead_unlock(wrk, &oh)); + AN(hsh_deref_objhead_unlock(wrk, &oh, 0)); wrk->stats->cache_hitpass++; VSLb(req->vsl, SLT_HitPass, "%u %.6f", xid, dttl); return (HSH_HITPASS); @@ -501,7 +502,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp) return (HSH_HITMISS); } oc->hits++; - AN(hsh_deref_objhead_unlock(wrk, &oh)); + AN(hsh_deref_objhead_unlock(wrk, &oh, HSH_RUSH_POLICY)); return (HSH_HIT); } @@ -543,7 +544,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp) exp_oc->refcnt++; *ocp = exp_oc; exp_oc->hits++; - AN(hsh_deref_objhead_unlock(wrk, &oh)); + AN(hsh_deref_objhead_unlock(wrk, &oh, 0)); return (HSH_GRACE); } @@ -1021,9 +1022,11 @@ HSH_DerefObjCore(struct worker *wrk, struct objcore **ocp, int rushmax) } static int -hsh_deref_objhead_unlock(struct worker *wrk, struct objhead **poh) +hsh_deref_objhead_unlock(struct worker *wrk, struct objhead **poh, int max) { struct objhead *oh; + struct rush rush; + int r; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); TAKE_OBJ_NOTNULL(oh, poh, OBJHEAD_MAGIC); @@ -1038,11 +1041,19 @@ hsh_deref_objhead_unlock(struct worker *wrk, struct objhead **poh) return (1); } + INIT_OBJ(&rush, RUSH_MAGIC); + if (!VTAILQ_EMPTY(&oh->waitinglist)) { + assert(oh->refcnt > 1); + hsh_rush1(wrk, oh, &rush, max); + } + if (oh->refcnt == 1) assert(VTAILQ_EMPTY(&oh->waitinglist)); assert(oh->refcnt > 0); - return (hash->deref(wrk, oh)); + r = hash->deref(wrk, oh); /* Unlocks oh->mtx */ + hsh_rush2(wrk, &rush); + return (r); } static int @@ -1054,7 +1065,7 @@ hsh_deref_objhead(struct worker *wrk, struct objhead **poh) TAKE_OBJ_NOTNULL(oh, poh, OBJHEAD_MAGIC); Lck_Lock(&oh->mtx); - return (hsh_deref_objhead_unlock(wrk, &oh)); + return (hsh_deref_objhead_unlock(wrk, &oh, 0)); } void diff --git a/bin/varnishtest/tests/c00097.vtc b/bin/varnishtest/tests/c00097.vtc new file mode 100644 index 000000000..31be8ca5e --- /dev/null +++ b/bin/varnishtest/tests/c00097.vtc @@ -0,0 +1,44 @@ +varnishtest "Streaming delivery and waitinglist rushing" + +barrier b1 sock 4 + +server s1 { + rxreq + txresp -nolen -hdr "Transfer-Encoding: chunked" + chunkedlen 10 + barrier b1 sync + chunkedlen 10 + chunkedlen 0 +} -start + +varnish v1 -arg "-p rush_exponent=2" -arg "-p debug=+syncvsl" -vcl+backend { + import vtc; + sub vcl_hit { + vtc.barrier_sync("${b1_sock}"); + } +} -start + +client c1 { + txreq + rxresp +} -start + +client c2 { + txreq + rxresp +} -start + +client c3 { + txreq + rxresp +} -start + +client c4 { + txreq + rxresp +} -start + +client c1 -wait +client c2 -wait +client c3 -wait +client c4 -wait From martin at varnish-software.com Tue Apr 23 12:42:08 2019 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 23 Apr 2019 12:42:08 +0000 (UTC) Subject: [master] 31ec2bd22 Rush waitinglist when doing hit-for-pass Message-ID: <20190423124208.6B7F6A52B5@lists.varnish-cache.org> commit 31ec2bd22c3a9beb61a4af93c103550e5affaf05 Author: Martin Blix Grydeland Date: Wed Apr 17 16:35:55 2019 +0200 Rush waitinglist when doing hit-for-pass This rushes the waitinglist when encountering a hit-for-pass during HSH_Lookup. Without this, the rushing may be delayed until the passed object completes delivery. Test cases c00098.vtc and c00099.vtc tests this early rushing behaviour for HFP and HFM respectively. Fixes: #2978 diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 113615b73..ead55932c 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -483,7 +483,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp) if (oc != NULL && oc->flags & OC_F_HFP) { xid = ObjGetXID(wrk, oc); dttl = EXP_Dttl(req, oc); - AN(hsh_deref_objhead_unlock(wrk, &oh, 0)); + AN(hsh_deref_objhead_unlock(wrk, &oh, HSH_RUSH_POLICY)); wrk->stats->cache_hitpass++; VSLb(req->vsl, SLT_HitPass, "%u %.6f", xid, dttl); return (HSH_HITPASS); diff --git a/bin/varnishtest/tests/c00098.vtc b/bin/varnishtest/tests/c00098.vtc new file mode 100644 index 000000000..423c4f995 --- /dev/null +++ b/bin/varnishtest/tests/c00098.vtc @@ -0,0 +1,123 @@ +varnishtest "Hit-for-pass and waitinglist rushing" + +barrier b2 cond 6 + +server s1 { + rxreq + txresp -nolen -hdr "Transfer-Encoding: chunked" + chunkedlen 10 + barrier b2 sync + chunkedlen 10 + chunkedlen 0 +} -start + +server s2 { + rxreq + txresp -nolen -hdr "Transfer-Encoding: chunked" + chunkedlen 10 + barrier b2 sync + chunkedlen 10 + chunkedlen 0 +} -start + +server s3 { + rxreq + txresp -nolen -hdr "Transfer-Encoding: chunked" + chunkedlen 10 + barrier b2 sync + chunkedlen 10 + chunkedlen 0 +} -start + +server s4 { + rxreq + txresp -nolen -hdr "Transfer-Encoding: chunked" + chunkedlen 10 + barrier b2 sync + chunkedlen 10 + chunkedlen 0 +} -start + +server s5 { + rxreq + txresp -nolen -hdr "Transfer-Encoding: chunked" + chunkedlen 10 + barrier b2 sync + chunkedlen 10 + chunkedlen 0 +} -start + +server s6 { + rxreq + txresp -nolen -hdr "Transfer-Encoding: chunked" + chunkedlen 10 + barrier b2 sync + chunkedlen 10 + chunkedlen 0 +} -start + +varnish v1 -arg "-p rush_exponent=2" -arg "-p debug=+syncvsl" -vcl+backend { + import vtc; + sub vcl_backend_fetch { + if (bereq.http.client == "1") { + set bereq.backend = s1; + } else if (bereq.http.client == "2") { + set bereq.backend = s2; + } else if (bereq.http.client == "3") { + set bereq.backend = s3; + } else if (bereq.http.client == "4") { + set bereq.backend = s4; + } else if (bereq.http.client == "5") { + set bereq.backend = s5; + } else if (bereq.http.client == "6") { + set bereq.backend = s6; + } + } + sub vcl_backend_response { + return (pass(1m)); + } +} -start + +client c1 { + txreq -url /hfp -hdr "Client: 1" + rxresp +} -start + +client c2 { + txreq -url /hfp -hdr "Client: 2" + rxresp +} -start + +client c3 { + txreq -url /hfp -hdr "Client: 3" + rxresp +} -start + +client c4 { + txreq -url /hfp -hdr "Client: 4" + rxresp +} -start + +client c5 { + txreq -url /hfp -hdr "Client: 5" + rxresp +} -start + +client c6 { + txreq -url /hfp -hdr "Client: 6" + rxresp +} -start + +client c1 -wait +client c2 -wait +client c3 -wait +client c4 -wait +client c5 -wait +client c6 -wait + +server s1 -wait +server s2 -wait +server s3 -wait +server s4 -wait +server s5 -wait +server s6 -wait diff --git a/bin/varnishtest/tests/c00099.vtc b/bin/varnishtest/tests/c00099.vtc new file mode 100644 index 000000000..ec2840796 --- /dev/null +++ b/bin/varnishtest/tests/c00099.vtc @@ -0,0 +1,123 @@ +varnishtest "Hit-for-miss and waitinglist rushing" + +barrier b3 cond 6 + +server s1 { + rxreq + txresp -nolen -hdr "Transfer-Encoding: chunked" + chunkedlen 10 + barrier b3 sync + chunkedlen 10 + chunkedlen 0 +} -start + +server s2 { + rxreq + txresp -nolen -hdr "Transfer-Encoding: chunked" + chunkedlen 10 + barrier b3 sync + chunkedlen 10 + chunkedlen 0 +} -start + +server s3 { + rxreq + txresp -nolen -hdr "Transfer-Encoding: chunked" + chunkedlen 10 + barrier b3 sync + chunkedlen 10 + chunkedlen 0 +} -start + +server s4 { + rxreq + txresp -nolen -hdr "Transfer-Encoding: chunked" + chunkedlen 10 + barrier b3 sync + chunkedlen 10 + chunkedlen 0 +} -start + +server s5 { + rxreq + txresp -nolen -hdr "Transfer-Encoding: chunked" + chunkedlen 10 + barrier b3 sync + chunkedlen 10 + chunkedlen 0 +} -start + +server s6 { + rxreq + txresp -nolen -hdr "Transfer-Encoding: chunked" + chunkedlen 10 + barrier b3 sync + chunkedlen 10 + chunkedlen 0 +} -start + +varnish v1 -arg "-p rush_exponent=2" -arg "-p debug=+syncvsl" -vcl+backend { + import vtc; + sub vcl_backend_fetch { + if (bereq.http.client == "1") { + set bereq.backend = s1; + } else if (bereq.http.client == "2") { + set bereq.backend = s2; + } else if (bereq.http.client == "3") { + set bereq.backend = s3; + } else if (bereq.http.client == "4") { + set bereq.backend = s4; + } else if (bereq.http.client == "5") { + set bereq.backend = s5; + } else if (bereq.http.client == "6") { + set bereq.backend = s6; + } + } + sub vcl_backend_response { + set beresp.uncacheable = true; + } +} -start + +client c1 { + txreq -url /hfm -hdr "Client: 1" + rxresp +} -start + +client c2 { + txreq -url /hfm -hdr "Client: 2" + rxresp +} -start + +client c3 { + txreq -url /hfm -hdr "Client: 3" + rxresp +} -start + +client c4 { + txreq -url /hfm -hdr "Client: 4" + rxresp +} -start + +client c5 { + txreq -url /hfm -hdr "Client: 5" + rxresp +} -start + +client c6 { + txreq -url /hfm -hdr "Client: 6" + rxresp +} -start + +client c1 -wait +client c2 -wait +client c3 -wait +client c4 -wait +client c5 -wait +client c6 -wait + +server s1 -wait +server s2 -wait +server s3 -wait +server s4 -wait +server s5 -wait +server s6 -wait From nils.goroll at uplex.de Tue Apr 23 12:57:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 23 Apr 2019 12:57:07 +0000 (UTC) Subject: [master] 6944fe82c respect max_restarts and fix c00028.vtc Message-ID: <20190423125707.E3D44A5A95@lists.varnish-cache.org> commit 6944fe82c8df4a26bf9f2006dda80a9e9566fb32 Author: Nils Goroll Date: Sun Mar 17 12:25:47 2019 +0100 respect max_restarts and fix c00028.vtc Fixes #2949 diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index eb797917d..5a7ef97e3 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -272,6 +272,10 @@ cnt_synth(struct worker *wrk, struct req *req) return (REQ_FSM_DONE); } + if (wrk->handling == VCL_RET_RESTART && + req->restarts > cache_param->max_restarts) + wrk->handling = VCL_RET_DELIVER; + if (wrk->handling == VCL_RET_RESTART) { /* * XXX: Should we reset req->doclose = SC_VCL_FAILURE diff --git a/bin/varnishtest/tests/c00028.vtc b/bin/varnishtest/tests/c00028.vtc index 0399e9fdc..7862e89e6 100644 --- a/bin/varnishtest/tests/c00028.vtc +++ b/bin/varnishtest/tests/c00028.vtc @@ -8,7 +8,11 @@ varnish v1 -vcl { sub vcl_backend_fetch { set bereq.backend = bad; } + sub vcl_backend_error { + return (abandon); + } sub vcl_synth { + set resp.http.restarts = req.restarts; return (restart); } } -start @@ -17,4 +21,5 @@ client c1 { txreq -url "/" rxresp expect resp.status == 503 + expect resp.http.restarts == 5 } -run From nils.goroll at uplex.de Tue Apr 23 12:58:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 23 Apr 2019 12:58:08 +0000 (UTC) Subject: [master] 4bbf6df68 Add timeout_idle customization per session Message-ID: <20190423125808.74D80A5C54@lists.varnish-cache.org> commit 4bbf6df68722dbe3ea77508d160c1dd258ab1d5e Author: Nils Goroll Date: Sat Apr 20 12:03:25 2019 +0200 Add timeout_idle customization per session includes vtc Purposes: * Allow to optimize keepalive for trusted client connections (e.g. CDN, authenticated clients) * Prepare for an optimization to time out connections on inactive VCLs earlier (see #2764) Other client timeouts will be added when accepted. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 67aa4a9a5..a6ade1d39 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -572,9 +572,12 @@ struct sess { vtim_real t_open; /* fd accepted */ vtim_real t_idle; /* fd accepted or resp sent */ - + vtim_dur timeout_idle; }; +#define SESS_TMO(sp, tmo) \ + (isnan((sp)->tmo) ? cache_param->tmo : (sp)->tmo) + /* Prototypes etc ----------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index 95696d4eb..875129f68 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -378,6 +378,7 @@ SES_New(struct pool *pp) sp->t_open = NAN; sp->t_idle = NAN; + sp->timeout_idle = NAN; Lck_New(&sp->mtx, lck_sess); CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); return (sp); @@ -465,7 +466,7 @@ SES_Wait(struct sess *sp, const struct transport *xp) wp->priv2 = (uintptr_t)xp; wp->idle = sp->t_idle; wp->func = ses_handle; - wp->tmo = cache_param->timeout_idle; + wp->tmo = SESS_TMO(sp, timeout_idle); if (Wait_Enter(pp->waiter, wp)) SES_Delete(sp, SC_PIPE_OVERFLOW, NAN); } diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index c734e0a5a..8e564fa19 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -939,3 +939,27 @@ HTTP_VAR(req) HTTP_VAR(resp) HTTP_VAR(bereq) HTTP_VAR(beresp) + +/*--------------------------------------------------------------------*/ + +VCL_VOID +VRT_l_sess_timeout_idle(VRT_CTX, VCL_DURATION d) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx->sp, SESS_MAGIC); + ctx->sp->timeout_idle = d > 0.0 ? d : 0.0; +} + +VCL_DURATION +VRT_r_sess_timeout_idle(VRT_CTX) +{ + VCL_DURATION d; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx->sp, SESS_MAGIC); + + d = ctx->sp->timeout_idle; + if (isnan(d)) + return (cache_param->timeout_idle); + return (d); +} diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c index 4f6e8efd1..c289bbbf1 100644 --- a/bin/varnishd/http1/cache_http1_fsm.c +++ b/bin/varnishd/http1/cache_http1_fsm.c @@ -331,7 +331,7 @@ HTTP1_Session(struct worker *wrk, struct req *req) hs = HTC_RxStuff(req->htc, HTTP1_Complete, &req->t_first, &req->t_req, sp->t_idle + cache_param->timeout_linger, - sp->t_idle + cache_param->timeout_idle, + sp->t_idle + SESS_TMO(sp, timeout_idle), NAN, cache_param->http_req_size); AZ(req->htc->ws->r); diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 48ad050c7..7681bfff5 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -1097,8 +1097,8 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2) h2->sess->t_idle = VTIM_real(); hs = HTC_RxStuff(h2->htc, h2_frame_complete, NULL, NULL, NAN, - h2->sess->t_idle + cache_param->timeout_idle, NAN, - h2->local_settings.max_frame_size + 9); + h2->sess->t_idle + SESS_TMO(h2->sess, timeout_idle), + NAN, h2->local_settings.max_frame_size + 9); switch (hs) { case HTC_S_COMPLETE: break; diff --git a/bin/varnishtest/tests/b00067.vtc b/bin/varnishtest/tests/b00067.vtc index 3f4d6a175..5bb1fee4e 100644 --- a/bin/varnishtest/tests/b00067.vtc +++ b/bin/varnishtest/tests/b00067.vtc @@ -3,6 +3,11 @@ varnishtest "Check timeout_idle" varnish v1 -arg "-p timeout_idle=1" -vcl { backend dummy { .host = "${bad_ip}"; } + sub vcl_deliver { + if (req.url == "/sess") { + set sess.timeout_idle = 2s; + } + } sub vcl_backend_error { set beresp.status = 200; set beresp.ttl = 1h; @@ -12,9 +17,19 @@ varnish v1 -arg "-p timeout_idle=1" -vcl { client c1 { txreq rxresp - delay .2 + delay 0.2 + txreq + rxresp + delay 1.2 + expect_close +} -run + +client c1 { + txreq -url "/sess" + rxresp + delay 1.2 txreq rxresp - delay 2 + delay 2.2 expect_close } -run diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index b1e492e06..7474e02fc 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -1250,6 +1250,17 @@ sess.xid ``VCL >= 4.1`` Unique ID of this session. +sess.timeout_idle + + Type: DURATION + + Readable from: client + + Writable from: client + + Idle timeout for this session, defaults to the + ``timeout_idle`` parameter, see :ref:`varnishd(1)` + storage ~~~~~~~ From nils.goroll at uplex.de Tue Apr 23 13:25:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 23 Apr 2019 13:25:08 +0000 (UTC) Subject: [master] 3e6e584b5 Try to stabilize this test Message-ID: <20190423132509.097B8A660D@lists.varnish-cache.org> commit 3e6e584b5ff587bf176b375171565bb86819c6f8 Author: Nils Goroll Date: Tue Apr 23 15:22:38 2019 +0200 Try to stabilize this test I ignored this for ages, but now it really bothers me: This test had quite a high failure rate on systems I control. And actually I do not quite understand why the fix works, but it does survive -j100 -n1000 Additional input welcome diff --git a/bin/varnishtest/tests/r01490.vtc b/bin/varnishtest/tests/r01490.vtc index 88c2550aa..37310cc55 100644 --- a/bin/varnishtest/tests/r01490.vtc +++ b/bin/varnishtest/tests/r01490.vtc @@ -6,8 +6,8 @@ server s1 { varnish v1 \ -arg "-p debug=+syncvsl" \ -arg "-p vsl_mask=+WorkThread" \ - -arg "-p thread_pool_min=2" \ - -arg "-p thread_pool_max=3" \ + -arg "-p thread_pool_min=3" \ + -arg "-p thread_pool_max=4" \ -arg "-p thread_pools=1" \ -arg "-p thread_pool_timeout=10" \ -vcl+backend {} @@ -15,19 +15,19 @@ varnish v1 -start delay 2 -varnish v1 -expect threads == 2 +varnish v1 -expect threads == 3 logexpect l1 -v v1 -g raw { expect * 0 WorkThread {^\S+ start$} expect * 0 WorkThread {^\S+ end$} } -start -varnish v1 -cliok "param.set thread_pool_min 3" +varnish v1 -cliok "param.set thread_pool_min 4" # Have to wait longer than thread_pool_timeout delay 11 -varnish v1 -expect threads == 3 +varnish v1 -expect threads == 4 varnish v1 -cliok "param.set thread_pool_min 2" varnish v1 -cliok "param.set thread_pool_max 2" From martin at varnish-software.com Tue Apr 23 13:33:07 2019 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 23 Apr 2019 13:33:07 +0000 (UTC) Subject: [master] 441713c77 Add test case by @rezan for #2976. Message-ID: <20190423133308.089EEA6A5A@lists.varnish-cache.org> commit 441713c77340010437a1d00d4c5e06ef73f09c60 Author: Martin Blix Grydeland Date: Tue Apr 23 15:32:00 2019 +0200 Add test case by @rezan for #2976. diff --git a/bin/varnishtest/tests/r02976.vtc b/bin/varnishtest/tests/r02976.vtc new file mode 100644 index 000000000..421cb3fe9 --- /dev/null +++ b/bin/varnishtest/tests/r02976.vtc @@ -0,0 +1,53 @@ +varnishtest "Detect probe interval inversion" + +barrier bprobe cond 2 + +server s1 { + # 5 probe requests + rxreq + txresp + close + accept + rxreq + txresp + close + accept + rxreq + txresp + close + accept + rxreq + txresp + close + accept + rxreq + txresp + close + accept + barrier bprobe sync +} -start + +varnish v1 -vcl { + backend b1 { + .host = "${s1_addr}"; + .port = "${s1_port}"; + .probe = { + .interval = 1s; + } + } + + backend b2 { + .host = "${bad_ip}"; + .probe = { + .interval = 24h; + } + } + + sub vcl_recv { + if (!req.url) { + set req.backend_hint = b2; + } + } +} -start + +barrier bprobe sync \ No newline at end of file From martin at varnish-software.com Tue Apr 23 14:13:08 2019 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 23 Apr 2019 14:13:08 +0000 (UTC) Subject: [master] 730927c26 Increase the minimum thread pool for c00098.vtc and c00099.vtc Message-ID: <20190423141308.84248A783E@lists.varnish-cache.org> commit 730927c262e316db0e4266a03bda620b88d429cf Author: Martin Blix Grydeland Date: Tue Apr 23 16:10:14 2019 +0200 Increase the minimum thread pool for c00098.vtc and c00099.vtc It seems that these test cases were suffering under the problem that #2942 addresses. Set a minimum thread pool so that there will be adequate threads available before the test begins. diff --git a/bin/varnishtest/tests/c00098.vtc b/bin/varnishtest/tests/c00098.vtc index 423c4f995..af5fff417 100644 --- a/bin/varnishtest/tests/c00098.vtc +++ b/bin/varnishtest/tests/c00098.vtc @@ -56,7 +56,7 @@ server s6 { chunkedlen 0 } -start -varnish v1 -arg "-p rush_exponent=2" -arg "-p debug=+syncvsl" -vcl+backend { +varnish v1 -arg "-p thread_pool_min=20" -arg "-p rush_exponent=2" -arg "-p debug=+syncvsl" -vcl+backend { import vtc; sub vcl_backend_fetch { if (bereq.http.client == "1") { diff --git a/bin/varnishtest/tests/c00099.vtc b/bin/varnishtest/tests/c00099.vtc index ec2840796..f05681b91 100644 --- a/bin/varnishtest/tests/c00099.vtc +++ b/bin/varnishtest/tests/c00099.vtc @@ -56,7 +56,7 @@ server s6 { chunkedlen 0 } -start -varnish v1 -arg "-p rush_exponent=2" -arg "-p debug=+syncvsl" -vcl+backend { +varnish v1 -arg "-p thread_pool_min=20" -arg "-p rush_exponent=2" -arg "-p debug=+syncvsl" -vcl+backend { import vtc; sub vcl_backend_fetch { if (bereq.http.client == "1") { From nils.goroll at uplex.de Tue Apr 23 14:54:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 23 Apr 2019 14:54:08 +0000 (UTC) Subject: [master] 310e42540 Revert "Try to stabilize this test" Message-ID: <20190423145408.0E97BA86A5@lists.varnish-cache.org> commit 310e4254067e07504af1f93fa9e2a724ef1419cb Author: Nils Goroll Date: Tue Apr 23 15:44:27 2019 +0200 Revert "Try to stabilize this test" This reverts commit 3e6e584b5ff587bf176b375171565bb86819c6f8. No use, still happens diff --git a/bin/varnishtest/tests/r01490.vtc b/bin/varnishtest/tests/r01490.vtc index 37310cc55..88c2550aa 100644 --- a/bin/varnishtest/tests/r01490.vtc +++ b/bin/varnishtest/tests/r01490.vtc @@ -6,8 +6,8 @@ server s1 { varnish v1 \ -arg "-p debug=+syncvsl" \ -arg "-p vsl_mask=+WorkThread" \ - -arg "-p thread_pool_min=3" \ - -arg "-p thread_pool_max=4" \ + -arg "-p thread_pool_min=2" \ + -arg "-p thread_pool_max=3" \ -arg "-p thread_pools=1" \ -arg "-p thread_pool_timeout=10" \ -vcl+backend {} @@ -15,19 +15,19 @@ varnish v1 -start delay 2 -varnish v1 -expect threads == 3 +varnish v1 -expect threads == 2 logexpect l1 -v v1 -g raw { expect * 0 WorkThread {^\S+ start$} expect * 0 WorkThread {^\S+ end$} } -start -varnish v1 -cliok "param.set thread_pool_min 4" +varnish v1 -cliok "param.set thread_pool_min 3" # Have to wait longer than thread_pool_timeout delay 11 -varnish v1 -expect threads == 4 +varnish v1 -expect threads == 3 varnish v1 -cliok "param.set thread_pool_min 2" varnish v1 -cliok "param.set thread_pool_max 2" From nils.goroll at uplex.de Tue Apr 23 14:54:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 23 Apr 2019 14:54:08 +0000 (UTC) Subject: [master] fa3e14197 setting thread_pool_timeout is useless because vtc_mode overrides Message-ID: <20190423145408.22F79A86A8@lists.varnish-cache.org> commit fa3e141975ee5c26735c150ce07c5575ae746578 Author: Nils Goroll Date: Tue Apr 23 16:01:04 2019 +0200 setting thread_pool_timeout is useless because vtc_mode overrides the herder delay to .5 seconds anyway diff --git a/bin/varnishtest/tests/r01490.vtc b/bin/varnishtest/tests/r01490.vtc index 88c2550aa..7ca0b23e2 100644 --- a/bin/varnishtest/tests/r01490.vtc +++ b/bin/varnishtest/tests/r01490.vtc @@ -9,7 +9,6 @@ varnish v1 \ -arg "-p thread_pool_min=2" \ -arg "-p thread_pool_max=3" \ -arg "-p thread_pools=1" \ - -arg "-p thread_pool_timeout=10" \ -vcl+backend {} varnish v1 -start @@ -24,16 +23,16 @@ logexpect l1 -v v1 -g raw { varnish v1 -cliok "param.set thread_pool_min 3" -# Have to wait longer than thread_pool_timeout -delay 11 +# thread_pool_timeout overridden by vtc_mode with delay=0.5 +delay 1 varnish v1 -expect threads == 3 varnish v1 -cliok "param.set thread_pool_min 2" varnish v1 -cliok "param.set thread_pool_max 2" -# Have to wait longer than thread_pool_timeout -delay 11 +# thread_pool_timeout overridden by vtc_mode with delay=0.5 +delay 1 varnish v1 -expect threads == 2 From nils.goroll at uplex.de Tue Apr 23 14:54:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 23 Apr 2019 14:54:08 +0000 (UTC) Subject: [master] b2def9fa5 polish: move duplicated code into pool_getidleworker() Message-ID: <20190423145408.3CBC2A86AB@lists.varnish-cache.org> commit b2def9fa591ddcdeca6e79c29265a4f989eefb7b Author: Nils Goroll Date: Tue Apr 23 16:50:18 2019 +0200 polish: move duplicated code into pool_getidleworker() it should remove the worker it returns from the idle queue for clarity diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index c6a37b862..2481f9cdf 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -200,8 +200,14 @@ pool_getidleworker(struct pool *pp, enum task_prio prio) } return (NULL); } + AZ(pt->func); CAST_OBJ_NOTNULL(wrk, pt->priv, WORKER_MAGIC); + + AN(pp->nidle); + VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list); + pp->nidle--; + return (wrk); } @@ -228,12 +234,9 @@ Pool_Task_Arg(struct worker *wrk, enum task_prio prio, task_func_t *func, Lck_Lock(&pp->mtx); wrk2 = pool_getidleworker(pp, prio); - if (wrk2 != NULL) { - AN(pp->nidle); - VTAILQ_REMOVE(&pp->idle_queue, &wrk2->task, list); - pp->nidle--; + if (wrk2 != NULL) retval = 1; - } else { + else { wrk2 = wrk; retval = 0; } @@ -280,9 +283,6 @@ Pool_Task(struct pool *pp, struct pool_task *task, enum task_prio prio) wrk = pool_getidleworker(pp, prio); if (wrk != NULL) { - AN(pp->nidle); - VTAILQ_REMOVE(&pp->idle_queue, &wrk->task, list); - pp->nidle--; AZ(wrk->task.func); wrk->task.func = task->func; wrk->task.priv = task->priv; From nils.goroll at uplex.de Tue Apr 23 15:34:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 23 Apr 2019 15:34:08 +0000 (UTC) Subject: [master] 37b6c8326 Revert "setting thread_pool_timeout is useless because vtc_mode overrides" Message-ID: <20190423153408.3D4F2A970D@lists.varnish-cache.org> commit 37b6c83265ee4f7178784513e7f1177d97802590 Author: Nils Goroll Date: Tue Apr 23 17:20:28 2019 +0200 Revert "setting thread_pool_timeout is useless because vtc_mode overrides" This reverts commit fa3e141975ee5c26735c150ce07c5575ae746578. We might have over-bread such that thread_pool_timeout becomes relevant at the other place in pool_herder() diff --git a/bin/varnishtest/tests/r01490.vtc b/bin/varnishtest/tests/r01490.vtc index 7ca0b23e2..88c2550aa 100644 --- a/bin/varnishtest/tests/r01490.vtc +++ b/bin/varnishtest/tests/r01490.vtc @@ -9,6 +9,7 @@ varnish v1 \ -arg "-p thread_pool_min=2" \ -arg "-p thread_pool_max=3" \ -arg "-p thread_pools=1" \ + -arg "-p thread_pool_timeout=10" \ -vcl+backend {} varnish v1 -start @@ -23,16 +24,16 @@ logexpect l1 -v v1 -g raw { varnish v1 -cliok "param.set thread_pool_min 3" -# thread_pool_timeout overridden by vtc_mode with delay=0.5 -delay 1 +# Have to wait longer than thread_pool_timeout +delay 11 varnish v1 -expect threads == 3 varnish v1 -cliok "param.set thread_pool_min 2" varnish v1 -cliok "param.set thread_pool_max 2" -# thread_pool_timeout overridden by vtc_mode with delay=0.5 -delay 1 +# Have to wait longer than thread_pool_timeout +delay 11 varnish v1 -expect threads == 2 From nils.goroll at uplex.de Tue Apr 23 15:34:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 23 Apr 2019 15:34:08 +0000 (UTC) Subject: [master] 691d5ac9b finally get r01490.vtc right Message-ID: <20190423153408.51716A9710@lists.varnish-cache.org> commit 691d5ac9b3aaa4d22b98cc1ce44b8daa685f64e9 Author: Nils Goroll Date: Tue Apr 23 17:26:54 2019 +0200 finally get r01490.vtc right (i hope) Trouble here is that in pool_herder(), we access pp->dry unprotected, so we might see an old value, thus we might breed more than wthread_min even if the dry condition does not exist any more. So for the vtc, we need to wait until wthread_timeout has passed and the surplus thread has been kissed to death. Notice that this does not change with #2942 because there the same unprotected access happens to lqueue. diff --git a/bin/varnishtest/tests/r01490.vtc b/bin/varnishtest/tests/r01490.vtc index 88c2550aa..fa7c8af67 100644 --- a/bin/varnishtest/tests/r01490.vtc +++ b/bin/varnishtest/tests/r01490.vtc @@ -13,7 +13,8 @@ varnish v1 \ -vcl+backend {} varnish v1 -start -delay 2 +# we might have over-bred +delay 11 varnish v1 -expect threads == 2 From phk at FreeBSD.org Tue Apr 23 20:58:09 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Apr 2019 20:58:09 +0000 (UTC) Subject: [master] 75fdb7927 I belive this fixes c00097 Message-ID: <20190423205809.E0F38B1A69@lists.varnish-cache.org> commit 75fdb7927f22cc56d9b3c1e3936b809f09d0a69e Author: Poul-Henning Kamp Date: Tue Apr 23 20:57:03 2019 +0000 I belive this fixes c00097 diff --git a/bin/varnishtest/tests/c00097.vtc b/bin/varnishtest/tests/c00097.vtc index 31be8ca5e..52469a690 100644 --- a/bin/varnishtest/tests/c00097.vtc +++ b/bin/varnishtest/tests/c00097.vtc @@ -1,6 +1,7 @@ varnishtest "Streaming delivery and waitinglist rushing" barrier b1 sock 4 +barrier b2 sock 4 server s1 { rxreq @@ -20,20 +21,25 @@ varnish v1 -arg "-p rush_exponent=2" -arg "-p debug=+syncvsl" -vcl+backend { client c1 { txreq - rxresp + rxresp -no_obj + barrier b2 sync + rxrespbody } -start client c2 { + barrier b2 sync txreq rxresp } -start client c3 { + barrier b2 sync txreq rxresp } -start client c4 { + barrier b2 sync txreq rxresp } -start From phk at FreeBSD.org Tue Apr 23 21:11:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 23 Apr 2019 21:11:07 +0000 (UTC) Subject: [master] 4b4bb1aa8 Try to fix c98 and c99 the same way. Message-ID: <20190423211107.D5DC9B1F95@lists.varnish-cache.org> commit 4b4bb1aa884be8320a1af5cd7a215f5d2b308e08 Author: Poul-Henning Kamp Date: Tue Apr 23 21:09:32 2019 +0000 Try to fix c98 and c99 the same way. The trick here is that these tests depend on c1 getting in front of c2... and that cannot be assumed given random thread scheduling algorithms, it must be enforced with barriers diff --git a/bin/varnishtest/tests/c00098.vtc b/bin/varnishtest/tests/c00098.vtc index af5fff417..32a809766 100644 --- a/bin/varnishtest/tests/c00098.vtc +++ b/bin/varnishtest/tests/c00098.vtc @@ -1,6 +1,7 @@ varnishtest "Hit-for-pass and waitinglist rushing" barrier b2 cond 6 +barrier b3 cond 6 server s1 { rxreq @@ -80,30 +81,36 @@ varnish v1 -arg "-p thread_pool_min=20" -arg "-p rush_exponent=2" -arg "-p debug client c1 { txreq -url /hfp -hdr "Client: 1" - rxresp + rxresp -no_obj + barrier b3 sync } -start client c2 { + barrier b3 sync txreq -url /hfp -hdr "Client: 2" rxresp } -start client c3 { + barrier b3 sync txreq -url /hfp -hdr "Client: 3" rxresp } -start client c4 { + barrier b3 sync txreq -url /hfp -hdr "Client: 4" rxresp } -start client c5 { + barrier b3 sync txreq -url /hfp -hdr "Client: 5" rxresp } -start client c6 { + barrier b3 sync txreq -url /hfp -hdr "Client: 6" rxresp } -start diff --git a/bin/varnishtest/tests/c00099.vtc b/bin/varnishtest/tests/c00099.vtc index f05681b91..58363ffc3 100644 --- a/bin/varnishtest/tests/c00099.vtc +++ b/bin/varnishtest/tests/c00099.vtc @@ -1,6 +1,7 @@ varnishtest "Hit-for-miss and waitinglist rushing" barrier b3 cond 6 +barrier b4 cond 6 server s1 { rxreq @@ -80,30 +81,36 @@ varnish v1 -arg "-p thread_pool_min=20" -arg "-p rush_exponent=2" -arg "-p debug client c1 { txreq -url /hfm -hdr "Client: 1" - rxresp + rxresp -no_obj + barrier b4 sync } -start client c2 { + barrier b4 sync txreq -url /hfm -hdr "Client: 2" rxresp } -start client c3 { + barrier b4 sync txreq -url /hfm -hdr "Client: 3" rxresp } -start client c4 { + barrier b4 sync txreq -url /hfm -hdr "Client: 4" rxresp } -start client c5 { + barrier b4 sync txreq -url /hfm -hdr "Client: 5" rxresp } -start client c6 { + barrier b4 sync txreq -url /hfm -hdr "Client: 6" rxresp } -start From nils.goroll at uplex.de Wed Apr 24 09:32:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 09:32:08 +0000 (UTC) Subject: [master] 98af434d4 simplify Message-ID: <20190424093208.E3B3591760@lists.varnish-cache.org> commit 98af434d445304397bcff05774e7ca956ca7ef07 Author: Nils Goroll Date: Wed Apr 24 06:54:25 2019 +0200 simplify diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 8e564fa19..2b471930e 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -953,13 +953,7 @@ VRT_l_sess_timeout_idle(VRT_CTX, VCL_DURATION d) VCL_DURATION VRT_r_sess_timeout_idle(VRT_CTX) { - VCL_DURATION d; - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(ctx->sp, SESS_MAGIC); - - d = ctx->sp->timeout_idle; - if (isnan(d)) - return (cache_param->timeout_idle); - return (d); + return (SESS_TMO(ctx->sp, timeout_idle)); } From nils.goroll at uplex.de Wed Apr 24 09:32:09 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 09:32:09 +0000 (UTC) Subject: [master] 8e540d38f sync VTIM_timeval/VTIM_timespec argument type with reality Message-ID: <20190424093209.062D891763@lists.varnish-cache.org> commit 8e540d38fedf620991db8c9a936d5d2415174278 Author: Nils Goroll Date: Wed Apr 24 07:33:56 2019 +0200 sync VTIM_timeval/VTIM_timespec argument type with reality timevals and timespecs could represent vtim_real / vmtim_mono also, but for now, all use cases within varnish-cache are for vtim_dur As we typedef vtim_{real,mono,dur} to double at the moment, this should not break any vmods. diff --git a/include/vtim.h b/include/vtim.h index b1898a39d..3eab38d69 100644 --- a/include/vtim.h +++ b/include/vtim.h @@ -36,5 +36,5 @@ vtim_real VTIM_parse(const char *p); vtim_mono VTIM_mono(void); vtim_real VTIM_real(void); void VTIM_sleep(vtim_dur t); -struct timespec VTIM_timespec(vtim_real t); -struct timeval VTIM_timeval(vtim_real t); +struct timespec VTIM_timespec(vtim_dur t); +struct timeval VTIM_timeval(vtim_dur t); diff --git a/lib/libvarnish/vtim.c b/lib/libvarnish/vtim.c index 14b9752d6..ddc0201a6 100644 --- a/lib/libvarnish/vtim.c +++ b/lib/libvarnish/vtim.c @@ -414,8 +414,13 @@ VTIM_sleep(vtim_dur t) #endif } +/* + * VTIM_timeval and VTIM_timespec may need variants with different signatures + * when vtim_real / vtim_mono typedefs are changed + */ + struct timeval -VTIM_timeval(vtim_real t) +VTIM_timeval(vtim_dur t) { struct timeval tv; @@ -426,7 +431,7 @@ VTIM_timeval(vtim_real t) } struct timespec -VTIM_timespec(vtim_real t) +VTIM_timespec(vtim_dur t) { struct timespec tv; From nils.goroll at uplex.de Wed Apr 24 09:32:09 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 09:32:09 +0000 (UTC) Subject: [master] 43bbe97b3 Pull the deadline calculation out of V1L Message-ID: <20190424093209.2637591768@lists.varnish-cache.org> commit 43bbe97b38a86d706c0d9f4838bbfbd305342775 Author: Nils Goroll Date: Wed Apr 24 09:31:01 2019 +0200 Pull the deadline calculation out of V1L and change the t0 argument into a vtim_real deadline, allowing for adjustments per call. This causes changes of send_timeout to not change open V1L transactions. This also exposed the fact that we are using send_timeout also for sending the bereq.body to the backend. Is this what we want? diff --git a/bin/varnishd/http1/cache_http1.h b/bin/varnishd/http1/cache_http1.h index 0bc52606c..47c9fce5c 100644 --- a/bin/varnishd/http1/cache_http1.h +++ b/bin/varnishd/http1/cache_http1.h @@ -59,7 +59,7 @@ void V1P_Charge(struct req *, const struct v1p_acct *, struct VSC_vbe *); void V1L_Chunked(const struct worker *w); void V1L_EndChunk(const struct worker *w); void V1L_Open(struct worker *, struct ws *, int *fd, struct vsl_log *, - double t0, unsigned niov); + vtim_real deadline, unsigned niov); unsigned V1L_Flush(const struct worker *w); unsigned V1L_Close(struct worker *w, uint64_t *cnt); size_t V1L_Write(const struct worker *w, const void *ptr, ssize_t len); diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c index b2e061391..ff21ede7f 100644 --- a/bin/varnishd/http1/cache_http1_deliver.c +++ b/bin/varnishd/http1/cache_http1_deliver.c @@ -131,7 +131,8 @@ V1D_Deliver(struct req *req, struct boc *boc, int sendbody) AZ(req->wrk->v1l); V1L_Open(req->wrk, req->wrk->aws, - &req->sp->fd, req->vsl, req->t_prev, cache_param->http1_iovs); + &req->sp->fd, req->vsl, req->t_prev + cache_param->send_timeout, + cache_param->http1_iovs); if (WS_Overflowed(req->wrk->aws)) { v1d_error(req, "workspace_thread overflow"); diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c index d8ce8a1a6..dc1515ef1 100644 --- a/bin/varnishd/http1/cache_http1_fetch.c +++ b/bin/varnishd/http1/cache_http1_fetch.c @@ -96,7 +96,9 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes, } VTCP_blocking(*htc->rfd); /* XXX: we should timeout instead */ - V1L_Open(wrk, wrk->aws, htc->rfd, bo->vsl, bo->t_prev, 0); + /* XXX: what is the right timeout ? Isn't send_timeout client side? */ + V1L_Open(wrk, wrk->aws, htc->rfd, bo->vsl, + bo->t_prev + cache_param->send_timeout, 0); hdrbytes = HTTP1_Write(wrk, hp, HTTP1_Req); /* Deal with any message-body the request might (still) have */ diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c index c8352af54..8d03d9b4f 100644 --- a/bin/varnishd/http1/cache_http1_line.c +++ b/bin/varnishd/http1/cache_http1_line.c @@ -60,7 +60,7 @@ struct v1l { ssize_t liov; ssize_t cliov; unsigned ciov; /* Chunked header marker */ - double t0; + vtim_real deadline; struct vsl_log *vsl; ssize_t cnt; /* Flushed byte count */ struct ws *ws; @@ -74,7 +74,7 @@ struct v1l { void V1L_Open(struct worker *wrk, struct ws *ws, int *fd, struct vsl_log *vsl, - double t0, unsigned niov) + vtim_real deadline, unsigned niov) { struct v1l *v1l; unsigned u; @@ -114,7 +114,7 @@ V1L_Open(struct worker *wrk, struct ws *ws, int *fd, struct vsl_log *vsl, v1l->siov = u; v1l->ciov = u; v1l->wfd = fd; - v1l->t0 = t0; + v1l->deadline = deadline; v1l->vsl = vsl; wrk->v1l = v1l; @@ -209,7 +209,7 @@ V1L_Flush(const struct worker *wrk) * counter to prevent slowloris attacks */ - if (VTIM_real() - v1l->t0 > cache_param->send_timeout) { + if (VTIM_real() > v1l->deadline) { VSLb(v1l->vsl, SLT_Debug, "Hit total send timeout, " "wrote = %zd/%zd; not retrying", From nils.goroll at uplex.de Wed Apr 24 10:32:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 10:32:07 +0000 (UTC) Subject: [master] c8ecdc15f extend timeout_idle test to uds Message-ID: <20190424103207.6AD5793172@lists.varnish-cache.org> commit c8ecdc15f91e2c26a74bf0561f81690bd3f370e8 Author: Nils Goroll Date: Wed Apr 24 12:27:29 2019 +0200 extend timeout_idle test to uds diff --git a/bin/varnishtest/tests/b00067.vtc b/bin/varnishtest/tests/b00067.vtc index 5bb1fee4e..e7b43a36a 100644 --- a/bin/varnishtest/tests/b00067.vtc +++ b/bin/varnishtest/tests/b00067.vtc @@ -1,6 +1,9 @@ varnishtest "Check timeout_idle" -varnish v1 -arg "-p timeout_idle=1" -vcl { +varnish v1 -arg "-p timeout_idle=1" \ + -arg "-a 127.0.0.1:0" \ + -arg "-a ${tmpdir}/v1.sock" \ + -vcl { backend dummy { .host = "${bad_ip}"; } sub vcl_deliver { @@ -22,9 +25,29 @@ client c1 { rxresp delay 1.2 expect_close -} -run +} -start -client c1 { +client c2 { + txreq -url "/sess" + rxresp + delay 1.2 + txreq + rxresp + delay 2.2 + expect_close +} -start + +client c1u -connect "${tmpdir}/v1.sock" { + txreq + rxresp + delay 0.2 + txreq + rxresp + delay 1.2 + expect_close +} -start + +client c2u -connect "${tmpdir}/v1.sock" { txreq -url "/sess" rxresp delay 1.2 @@ -32,4 +55,9 @@ client c1 { rxresp delay 2.2 expect_close -} -run +} -start + +client c1 -wait +client c2 -wait +client c1u -wait +client c2u -wait From nils.goroll at uplex.de Wed Apr 24 11:02:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 11:02:07 +0000 (UTC) Subject: [master] 0cc146608 stabilize another test by example of r01490.vtc Message-ID: <20190424110207.EBA2593CEE@lists.varnish-cache.org> commit 0cc146608d51b62e2a694467110171a2c18a4e02 Author: Nils Goroll Date: Wed Apr 24 12:49:28 2019 +0200 stabilize another test by example of r01490.vtc hope we'll get to a better option soon. Ref: 691d5ac9b3aaa4d22b98cc1ce44b8daa685f64e9 diff --git a/bin/varnishtest/tests/v00006.vtc b/bin/varnishtest/tests/v00006.vtc index 4a187c8a7..66d3d989c 100644 --- a/bin/varnishtest/tests/v00006.vtc +++ b/bin/varnishtest/tests/v00006.vtc @@ -14,11 +14,17 @@ server s1 { } -start # Only one pool, to avoid getting more than one work thread -varnish v1 -arg "-p thread_pools=1" -vcl+backend { } -start - -# Give the varnishd a chance to start and create workers etc. +varnish v1 \ + -arg "-p thread_pools=1" \ + -arg "-p thread_pool_timeout=10" \ + -vcl+backend { } -start + +# Give the varnishd a chance to start and create workers, delaying for +# >thread_pool_timeout to allow for any over-bred workers to be kissed +# to death. # NB: This is important for to avoid mis-ordering of the workers. -# delay 1 + +delay 11 varnish v1 -expect MAIN.threads == 10 From nils.goroll at uplex.de Wed Apr 24 11:41:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 11:41:08 +0000 (UTC) Subject: [master] 654f0b5f4 Clear the v1l after resetting the workspace Message-ID: <20190424114108.1043095C6B@lists.varnish-cache.org> commit 654f0b5f493f50b016c316a8572a29a457aa3eb5 Author: Dridi Boukelmoune Date: Tue Apr 16 15:16:35 2019 +0200 Clear the v1l after resetting the workspace diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c index 8d03d9b4f..6a471b043 100644 --- a/bin/varnishd/http1/cache_http1_line.c +++ b/bin/varnishd/http1/cache_http1_line.c @@ -138,6 +138,7 @@ V1L_Close(struct worker *wrk, uint64_t *cnt) if (v1l->ws->r) WS_Release(v1l->ws, 0); WS_Reset(v1l->ws, v1l->res); + ZERO_OBJ(v1l, sizeof *v1l); return (u); } @@ -278,7 +279,7 @@ V1L_Chunked(const struct worker *wrk) assert(v1l->ciov == v1l->siov); assert(v1l->siov >= 3); /* - * If there are not space for chunked header, a chunk of data and + * If there is no space for chunked header, a chunk of data and * a chunk tail, we might as well flush right away. */ if (v1l->niov + 3 >= v1l->siov) From nils.goroll at uplex.de Wed Apr 24 11:41:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 11:41:08 +0000 (UTC) Subject: [master] d49025519 Teach varnishtest about SO_RCVBUF Message-ID: <20190424114108.373F195C6E@lists.varnish-cache.org> commit d4902551998a6c57aa3d56332ce888c25d134ddf Author: Dridi Boukelmoune Date: Wed Apr 17 13:48:19 2019 +0200 Teach varnishtest about SO_RCVBUF For now the varnishd handling of SO_SNDBUF lives in vmod-debug but could be promoted to vmod-vtc to be usable out of tree too. diff --git a/bin/varnishtest/tests/s00010.vtc b/bin/varnishtest/tests/s00010.vtc new file mode 100644 index 000000000..1c21dd58b --- /dev/null +++ b/bin/varnishtest/tests/s00010.vtc @@ -0,0 +1,46 @@ +varnishtest "client h1 send timeouts" + +server s1 { + rxreq + txresp -bodylen 100000 +} -start + +varnish v1 -vcl+backend { + import debug; + + sub vcl_deliver { + debug.sndbuf(128b); + } +} -start + +varnish v1 -cliok "param.set send_timeout 1" + +logexpect l1 -v v1 { + expect * * Debug "Hit total send timeout" +} -start + +client c1 -rcvbuf 128 { + txreq + rxresphdrs + # keep the session open for 2 seconds + delay 2 +} -run + +logexpect l1 -wait + +feature SO_RCVTIMEO_WORKS +varnish v1 -cliok "param.set idle_send_timeout 1" +varnish v1 -cliok "param.reset send_timeout" + +logexpect l2 -v v1 { + expect * * Debug "Hit idle send timeout" +} -start + +client c2 -rcvbuf 128 { + txreq + rxresphdrs + # keep the session open for 2 seconds + delay 2 +} -run + +logexpect l2 -wait diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index c091afe40..9d19f09aa 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -88,7 +88,7 @@ void init_server(void); void init_syslog(void); int http_process(struct vtclog *vl, const char *spec, int sock, int *sfd, - const char *addr); + const char *addr, int rcvbuf); char * synth_body(const char *len, int rnd); diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c index 7d199d2ef..865b7872d 100644 --- a/bin/varnishtest/vtc_client.c +++ b/bin/varnishtest/vtc_client.c @@ -54,6 +54,7 @@ struct client { char *spec; char connect[256]; + int rcvbuf; char *proxy_spec; int proxy_version; @@ -233,10 +234,12 @@ client_thread(void *priv) if (c->proxy_spec != NULL) client_proxy(vl, fd, c->proxy_version, c->proxy_spec); if (! c->keepalive) - fd = http_process(vl, c->spec, fd, NULL, addr); + fd = http_process(vl, c->spec, fd, NULL, addr, + c->rcvbuf); else while (fd >= 0 && u++ < c->repeat) - fd = http_process(vl, c->spec, fd, NULL, addr); + fd = http_process(vl, c->spec, fd, NULL, addr, + c->rcvbuf); vtc_log(vl, 3, "closing fd %d", fd); VTCP_close(&fd); } @@ -402,6 +405,11 @@ cmd_client(CMD_ARGS) c->keepalive = 1; continue; } + if (!strcmp(*av, "-rcvbuf")) { + c->rcvbuf = atoi(av[1]); + av++; + continue; + } if (!strcmp(*av, "-start")) { client_start(c); continue; diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 7df5a472b..99b46eaf2 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2008-2015 Varnish Software AS + * Copyright (c) 2008-2019 Varnish Software AS * All rights reserved. * * Author: Poul-Henning Kamp @@ -1905,10 +1905,11 @@ http_process_cleanup(void *arg) int http_process(struct vtclog *vl, const char *spec, int sock, int *sfd, - const char *addr) + const char *addr, int rcvbuf) { struct http *hp; - int retval; + int retval, oldbuf; + socklen_t intlen = sizeof(int); (void)sfd; ALLOC_OBJ(hp, HTTP_MAGIC); @@ -1916,6 +1917,18 @@ http_process(struct vtclog *vl, const char *spec, int sock, int *sfd, hp->fd = sock; hp->timeout = vtc_maxdur * 1000 / 2; + if (rcvbuf) { + hp->rcvbuf = rcvbuf; + + oldbuf = 0; + AZ(getsockopt(hp->fd, SOL_SOCKET, SO_RCVBUF, &oldbuf, &intlen)); + AZ(setsockopt(hp->fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, intlen)); + AZ(getsockopt(hp->fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, &intlen)); + + vtc_log(vl, 3, "-rcvbuf fd=%d old=%d new=%d actual=%d", + hp->fd, oldbuf, hp->rcvbuf, rcvbuf); + } + hp->nrxbuf = 2048*1024; hp->rx_b = malloc(hp->nrxbuf); AN(hp->rx_b); diff --git a/bin/varnishtest/vtc_http.h b/bin/varnishtest/vtc_http.h index 4db97a82f..d9241ff2b 100644 --- a/bin/varnishtest/vtc_http.h +++ b/bin/varnishtest/vtc_http.h @@ -38,6 +38,7 @@ struct http { struct vsb *vsb; + int rcvbuf; int nrxbuf; char *rx_b; char *rx_p; diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index 1a7c06f87..9003516c4 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -57,6 +57,7 @@ struct server { int depth; int sock; int fd; + int rcvbuf; char listen[256]; char aaddr[32]; char aport[32]; @@ -248,11 +249,12 @@ server_thread(void *priv) } else vtc_log(vl, 3, "accepted fd %d 0.0.0.0 0", fd); if (! s->keepalive) - fd = http_process(vl, s->spec, fd, &s->sock, s->listen); + fd = http_process(vl, s->spec, fd, &s->sock, s->listen, + s->rcvbuf); else while (fd >= 0 && i++ < s->repeat) fd = http_process(vl, s->spec, fd, - &s->sock, s->listen); + &s->sock, s->listen, s->rcvbuf); vtc_log(vl, 3, "shutting fd %d", fd); j = shutdown(fd, SHUT_WR); if (!VTCP_Check(j)) @@ -300,7 +302,7 @@ server_dispatch_wrk(void *priv) fd = s->fd; vtc_log(vl, 3, "start with fd %d", fd); - fd = http_process(vl, s->spec, fd, &s->sock, s->listen); + fd = http_process(vl, s->spec, fd, &s->sock, s->listen, s->rcvbuf); vtc_log(vl, 3, "shutting fd %d", fd); j = shutdown(fd, SHUT_WR); if (!VTCP_Check(j)) @@ -541,6 +543,11 @@ cmd_server(CMD_ARGS) av++; continue; } + if (!strcmp(*av, "-rcvbuf")) { + s->rcvbuf = atoi(av[1]); + av++; + continue; + } if (!strcmp(*av, "-start")) { server_start(s); continue; diff --git a/lib/libvmod_debug/vmod.vcc b/lib/libvmod_debug/vmod.vcc index 0cd1963c7..20ee901da 100644 --- a/lib/libvmod_debug/vmod.vcc +++ b/lib/libvmod_debug/vmod.vcc @@ -1,5 +1,5 @@ # -# Copyright (c) 2010-2015 Varnish Software AS +# Copyright (c) 2010-2019 Varnish Software AS # All rights reserved. # # Author: Poul-Henning Kamp @@ -251,3 +251,5 @@ Prevent VCL from going cold $Function VOID release_vcl_busy() Allow VCL to go cold + +$Function VOID sndbuf(BYTES sndbuf) diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index fc9720a74..870250ce3 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2012-2015 Varnish Software AS + * Copyright (c) 2012-2019 Varnish Software AS * All rights reserved. * * Author: Poul-Henning Kamp @@ -807,3 +807,39 @@ xyzzy_release_vcl_busy(VRT_CTX) VRT_VCL_Unbusy(ctx); } + +VCL_VOID +xyzzy_sndbuf(VRT_CTX, VCL_BYTES buflen) +{ + int fd = -1, oldbuf, newbuf; + socklen_t intlen = sizeof(int); + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + + if (ctx->bo) { + CHECK_OBJ(ctx->bo, BUSYOBJ_MAGIC); + INCOMPL(); + } + else if (ctx->req) { + CHECK_OBJ(ctx->req, REQ_MAGIC); + CHECK_OBJ(ctx->req->sp, SESS_MAGIC); + fd = ctx->req->sp->fd; + } + else { + VRT_fail(ctx, "debug.sndbuf() called outside a transaction."); + return; + } + + xxxassert(fd >= 0); + + oldbuf = 0; + AZ(getsockopt(fd, SOL_SOCKET, SO_SNDBUF, &oldbuf, &intlen)); + + newbuf = buflen; + AZ(setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &newbuf, intlen)); + AZ(getsockopt(fd, SOL_SOCKET, SO_SNDBUF, &newbuf, &intlen)); + + AN(ctx->vsl); + VSLb(ctx->vsl, SLT_Debug, "SO_SNDBUF fd=%d old=%d new=%ld actual=%d", + fd, oldbuf, buflen, newbuf); +} From nils.goroll at uplex.de Wed Apr 24 11:41:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 11:41:08 +0000 (UTC) Subject: [master] d8421dcd5 Assert priv is not null before dereferencing it Message-ID: <20190424114108.5234A95C73@lists.varnish-cache.org> commit d8421dcd59a8cf271e4587a0228a3a24954636f2 Author: Dridi Boukelmoune Date: Wed Apr 17 18:56:30 2019 +0200 Assert priv is not null before dereferencing it And make it clear that we steal the reference at this point. diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index 26e4cc5da..d5febbf38 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -86,11 +86,10 @@ h2_fini(struct req *req, void **priv) struct h2_req *r2; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - CAST_OBJ_NOTNULL(r2, *priv, H2_REQ_MAGIC); + TAKE_OBJ_NOTNULL(r2, priv, H2_REQ_MAGIC); H2_Send_Get(req->wrk, r2->h2sess, r2); H2_Send(req->wrk, r2, H2_F_DATA, H2FF_DATA_END_STREAM, 0, ""); H2_Send_Rel(r2->h2sess, r2); - *priv = NULL; return (0); } From nils.goroll at uplex.de Wed Apr 24 11:41:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 11:41:08 +0000 (UTC) Subject: [master] 3c8ecb4cc Incremental h2 request accounting Message-ID: <20190424114108.7413B95C78@lists.varnish-cache.org> commit 3c8ecb4cc0447e0149715e2fbb773a8fa9f433ce Author: Dridi Boukelmoune Date: Wed Apr 17 19:00:39 2019 +0200 Incremental h2 request accounting The dummy_acct is only here to avoid repeating null checks in the send loop below. This doesn't change the end result if the transaction completes with no problem. As such, only header and body bytes are accounted for, as before, ignoring h2 framing overhead and in general other kinds of frames that belong to the stream. In other words, the only improvement is that ReqAcct doesn't show a full delivery when the client hangs up before the end of the transaction. The split of H2_Send will allow handling of error conditions from the multiple return statements although at this point there is no change in this area. diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index 5399f2f98..83bf5460c 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -223,8 +223,8 @@ void H2_Send_Frame(struct worker *, struct h2_sess *, h2_frame type, uint8_t flags, uint32_t len, uint32_t stream, const void *); -void H2_Send(struct worker *, struct h2_req *, - h2_frame type, uint8_t flags, uint32_t len, const void *); +void H2_Send(struct worker *, struct h2_req *, h2_frame type, uint8_t flags, + uint32_t len, const void *, uint64_t *acct); /* cache_http2_proto.c */ struct h2_req * h2_new_req(const struct worker *, struct h2_sess *, diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index d5febbf38..025d431d2 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -88,7 +88,7 @@ h2_fini(struct req *req, void **priv) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); TAKE_OBJ_NOTNULL(r2, priv, H2_REQ_MAGIC); H2_Send_Get(req->wrk, r2->h2sess, r2); - H2_Send(req->wrk, r2, H2_F_DATA, H2FF_DATA_END_STREAM, 0, ""); + H2_Send(req->wrk, r2, H2_F_DATA, H2FF_DATA_END_STREAM, 0, "", NULL); H2_Send_Rel(r2->h2sess, r2); return (0); } @@ -106,9 +106,9 @@ h2_bytes(struct req *req, enum vdp_action act, void **priv, if ((r2->h2sess->error || r2->error)) return (-1); H2_Send_Get(req->wrk, r2->h2sess, r2); - H2_Send(req->wrk, r2, H2_F_DATA, H2FF_NONE, len, ptr); + H2_Send(req->wrk, r2, H2_F_DATA, H2FF_NONE, len, ptr, + &req->acct.resp_bodybytes); H2_Send_Rel(r2->h2sess, r2); - req->acct.resp_bodybytes += len; return (0); } @@ -173,7 +173,7 @@ h2_minimal_response(struct req *req, uint16_t status) H2_F_HEADERS, H2FF_HEADERS_END_HEADERS | (status < 200 ? 0 : H2FF_HEADERS_END_STREAM), - l, buf); + l, buf, NULL); H2_Send_Rel(r2->h2sess, r2); return (0); } @@ -317,9 +317,8 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody) H2_Send_Get(req->wrk, r2->h2sess, r2); H2_Send(req->wrk, r2, H2_F_HEADERS, (sendbody ? 0 : H2FF_HEADERS_END_STREAM) | H2FF_HEADERS_END_HEADERS, - sz, r); + sz, r, &req->acct.resp_hdrbytes); H2_Send_Rel(r2->h2sess, r2); - req->acct.resp_hdrbytes += sz; WS_Release(req->ws, 0); diff --git a/bin/varnishd/http2/cache_http2_send.c b/bin/varnishd/http2/cache_http2_send.c index dd20d1051..a0d8600cd 100644 --- a/bin/varnishd/http2/cache_http2_send.c +++ b/bin/varnishd/http2/cache_http2_send.c @@ -256,9 +256,9 @@ h2_do_window(struct worker *wrk, struct h2_req *r2, * XXX: priority */ -void -H2_Send(struct worker *wrk, struct h2_req *r2, - h2_frame ftyp, uint8_t flags, uint32_t len, const void *ptr) +static void +h2_send(struct worker *wrk, struct h2_req *r2, h2_frame ftyp, uint8_t flags, + uint32_t len, const void *ptr, uint64_t *acct) { struct h2_sess *h2; uint32_t mfs, tf; @@ -270,6 +270,7 @@ H2_Send(struct worker *wrk, struct h2_req *r2, h2 = r2->h2sess; CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); assert(len == 0 || ptr != NULL); + AN(acct); assert(VTAILQ_FIRST(&h2->txqueue) == r2); @@ -297,8 +298,7 @@ H2_Send(struct worker *wrk, struct h2_req *r2, Lck_Unlock(&h2->sess->mtx); if (ftyp->respect_window) { - tf = h2_do_window(wrk, r2, h2, - (len > mfs) ? mfs : len); + tf = h2_do_window(wrk, r2, h2, (len > mfs) ? mfs : len); if (h2_errcheck(r2, h2)) return; assert(VTAILQ_FIRST(&h2->txqueue) == r2); @@ -307,6 +307,7 @@ H2_Send(struct worker *wrk, struct h2_req *r2, if (len <= tf) { H2_Send_Frame(wrk, h2, ftyp, flags, len, r2->stream, ptr); + *acct += len; } else { AN(ptr); p = ptr; @@ -318,7 +319,7 @@ H2_Send(struct worker *wrk, struct h2_req *r2, tf = mfs; if (ftyp->respect_window && p != ptr) { tf = h2_do_window(wrk, r2, h2, - (len > mfs) ? mfs : len); + (len > mfs) ? mfs : len); if (h2_errcheck(r2, h2)) return; assert(VTAILQ_FIRST(&h2->txqueue) == r2); @@ -330,15 +331,28 @@ H2_Send(struct worker *wrk, struct h2_req *r2, if (ftyp->respect_window) assert(tf == len); tf = len; - H2_Send_Frame(wrk, h2, ftyp, - final_flags, tf, r2->stream, p); + H2_Send_Frame(wrk, h2, ftyp, final_flags, tf, + r2->stream, p); flags = 0; } p += tf; len -= tf; + *acct += tf; ftyp = ftyp->continuation; flags &= ftyp->flags; final_flags &= ftyp->flags; } while (!h2->error && len > 0); } } + +void +H2_Send(struct worker *wrk, struct h2_req *r2, h2_frame ftyp, uint8_t flags, + uint32_t len, const void *ptr, uint64_t *acct) +{ + uint64_t dummy_acct; + + if (acct == NULL) + acct = &dummy_acct; + + h2_send(wrk, r2, ftyp, flags, len, ptr, acct); +} diff --git a/bin/varnishtest/tests/t02015.vtc b/bin/varnishtest/tests/t02015.vtc new file mode 100644 index 000000000..3da5c24d4 --- /dev/null +++ b/bin/varnishtest/tests/t02015.vtc @@ -0,0 +1,49 @@ +varnishtest "h2 ReqAcct" + +server s1 { + rxreq + txresp -bodylen 12345 +} -start + +varnish v1 -cliok "param.set feature +http2" +varnish v1 -vcl+backend "" -start + +client c1 { + txpri + + stream 0 { + rxsettings + expect settings.ack == false + txsettings -ack + txsettings -winsize 1000 + rxsettings + expect settings.ack == true + } -run + + stream 1 { + txreq -hdr stream 1 + rxhdrs + rxdata + txwinup -size 11345 + rxdata + } -run + + stream 3 { + txreq -hdr stream 3 + rxhdrs + rxdata + txrst + } -run +} -run + +shell { + varnishncsa -n ${v1_name} -F '%{VSL:ReqAcct[5]}x' -d \ + -q 'ReqHeader:stream == 1' | + grep 12345 +} + +shell { + varnishncsa -n ${v1_name} -F '%{VSL:ReqAcct[5]}x' -d \ + -q 'ReqHeader:stream == 3' | + grep 1000 +} From nils.goroll at uplex.de Wed Apr 24 11:41:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 11:41:08 +0000 (UTC) Subject: [master] a0b8a7344 Suffix with _locked for clarity Message-ID: <20190424114108.8C9B795C7D@lists.varnish-cache.org> commit a0b8a73441511e3ee90479f11f927d0600e9ae06 Author: Dridi Boukelmoune Date: Wed Apr 17 20:02:34 2019 +0200 Suffix with _locked for clarity diff --git a/bin/varnishd/http2/cache_http2_send.c b/bin/varnishd/http2/cache_http2_send.c index a0d8600cd..7424353b1 100644 --- a/bin/varnishd/http2/cache_http2_send.c +++ b/bin/varnishd/http2/cache_http2_send.c @@ -40,7 +40,7 @@ #include "vtim.h" static void -h2_send_get(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) +h2_send_get_locked(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) { CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); @@ -64,12 +64,12 @@ H2_Send_Get(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); Lck_Lock(&h2->sess->mtx); - h2_send_get(wrk, h2, r2); + h2_send_get_locked(wrk, h2, r2); Lck_Unlock(&h2->sess->mtx); } static void -h2_send_rel(struct h2_sess *h2, const struct h2_req *r2) +h2_send_rel_locked(struct h2_sess *h2, const struct h2_req *r2) { CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); @@ -91,7 +91,7 @@ H2_Send_Rel(struct h2_sess *h2, const struct h2_req *r2) CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); Lck_Lock(&h2->sess->mtx); - h2_send_rel(h2, r2); + h2_send_rel_locked(h2, r2); Lck_Unlock(&h2->sess->mtx); } @@ -217,7 +217,7 @@ h2_do_window(struct worker *wrk, struct h2_req *r2, Lck_Lock(&h2->sess->mtx); if (r2->t_window <= 0 || h2->req0->t_window <= 0) { r2->t_winupd = VTIM_real(); - h2_send_rel(h2, r2); + h2_send_rel_locked(h2, r2); while (r2->t_window <= 0 && h2_errcheck(r2, h2) == 0) { r2->cond = &wrk->cond; AZ(Lck_CondWait(r2->cond, &h2->sess->mtx, 0)); @@ -234,7 +234,7 @@ h2_do_window(struct worker *wrk, struct h2_req *r2, h2_win_charge(r2, h2, w); assert (w > 0); } - h2_send_get(wrk, h2, r2); + h2_send_get_locked(wrk, h2, r2); } if (w == 0 && h2_errcheck(r2, h2) == 0) { From nils.goroll at uplex.de Wed Apr 24 11:41:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 11:41:08 +0000 (UTC) Subject: [master] 8c5118d17 Move redundant checks to an explicit H2_SEND_HELD() Message-ID: <20190424114108.C3B0C95C83@lists.varnish-cache.org> commit 8c5118d17cba2f4d78a672f42f16f1cbff2ca766 Author: Dridi Boukelmoune Date: Thu Apr 18 10:50:43 2019 +0200 Move redundant checks to an explicit H2_SEND_HELD() It is unclear to me whether those calls belong under the h2 session lock, but this patch doesn't change any behavior. diff --git a/bin/varnishd/http2/cache_http2_send.c b/bin/varnishd/http2/cache_http2_send.c index 7424353b1..06e029e9e 100644 --- a/bin/varnishd/http2/cache_http2_send.c +++ b/bin/varnishd/http2/cache_http2_send.c @@ -39,9 +39,12 @@ #include "vend.h" #include "vtim.h" +#define H2_SEND_HELD(h2, r2) (VTAILQ_FIRST(&(h2)->txqueue) == (r2)) + static void h2_send_get_locked(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) { + CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -51,7 +54,7 @@ h2_send_get_locked(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) ASSERT_RXTHR(h2); r2->wrk = wrk; VTAILQ_INSERT_TAIL(&h2->txqueue, r2, tx_list); - while (VTAILQ_FIRST(&h2->txqueue) != r2) + while (!H2_SEND_HELD(h2, r2)) AZ(Lck_CondWait(&wrk->cond, &h2->sess->mtx, 0)); r2->wrk = NULL; } @@ -59,6 +62,7 @@ h2_send_get_locked(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) void H2_Send_Get(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) { + CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -75,7 +79,7 @@ h2_send_rel_locked(struct h2_sess *h2, const struct h2_req *r2) CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); Lck_AssertHeld(&h2->sess->mtx); - assert(VTAILQ_FIRST(&h2->txqueue) == r2); + AN(H2_SEND_HELD(h2, r2)); VTAILQ_REMOVE(&h2->txqueue, r2, tx_list); r2 = VTAILQ_FIRST(&h2->txqueue); if (r2 != NULL) { @@ -272,7 +276,7 @@ h2_send(struct worker *wrk, struct h2_req *r2, h2_frame ftyp, uint8_t flags, assert(len == 0 || ptr != NULL); AN(acct); - assert(VTAILQ_FIRST(&h2->txqueue) == r2); + AN(H2_SEND_HELD(h2, r2)); if (h2_errcheck(r2, h2)) return; @@ -301,7 +305,7 @@ h2_send(struct worker *wrk, struct h2_req *r2, h2_frame ftyp, uint8_t flags, tf = h2_do_window(wrk, r2, h2, (len > mfs) ? mfs : len); if (h2_errcheck(r2, h2)) return; - assert(VTAILQ_FIRST(&h2->txqueue) == r2); + AN(H2_SEND_HELD(h2, r2)); } else tf = mfs; @@ -322,7 +326,7 @@ h2_send(struct worker *wrk, struct h2_req *r2, h2_frame ftyp, uint8_t flags, (len > mfs) ? mfs : len); if (h2_errcheck(r2, h2)) return; - assert(VTAILQ_FIRST(&h2->txqueue) == r2); + AN(H2_SEND_HELD(h2, r2)); } if (tf < len) { H2_Send_Frame(wrk, h2, ftyp, From nils.goroll at uplex.de Wed Apr 24 11:41:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 11:41:08 +0000 (UTC) Subject: [master] 51127b460 Avoid double log ending of h2 sessions Message-ID: <20190424114108.F20D495C87@lists.varnish-cache.org> commit 51127b4600f095fb086c5115905b90ea91731d4f Author: Dridi Boukelmoune Date: Fri Apr 19 10:29:04 2019 +0200 Avoid double log ending of h2 sessions diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index c6abfe1ae..def146b04 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -220,8 +220,12 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req) /* Charge and log byte counters */ req_AcctLogCharge(wrk->stats, req); - if (req->vsl->wid) - VSL_End(req->vsl); + if (req->vsl->wid) { + if (req->vsl->wid != sp->vxid) + VSL_End(req->vsl); + else + req->vsl->wid = 0; /* ending an h2 stream 0 */ + } if (!isnan(req->t_prev) && req->t_prev > 0. && req->t_prev > sp->t_idle) sp->t_idle = req->t_prev; From nils.goroll at uplex.de Wed Apr 24 12:04:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 12:04:07 +0000 (UTC) Subject: [master] de1aa79aa extend send_timeout test to uds Message-ID: <20190424120407.D6D5796E43@lists.varnish-cache.org> commit de1aa79aa5d720c775f379da80bced47fcbf52e7 Author: Nils Goroll Date: Wed Apr 24 14:02:25 2019 +0200 extend send_timeout test to uds diff --git a/bin/varnishtest/tests/s00010.vtc b/bin/varnishtest/tests/s00010.vtc index 1c21dd58b..c641db519 100644 --- a/bin/varnishtest/tests/s00010.vtc +++ b/bin/varnishtest/tests/s00010.vtc @@ -5,7 +5,10 @@ server s1 { txresp -bodylen 100000 } -start -varnish v1 -vcl+backend { +varnish v1 -arg "-p timeout_idle=1" \ + -arg "-a 127.0.0.1:0" \ + -arg "-a ${tmpdir}/v1.sock" \ + -vcl+backend { import debug; sub vcl_deliver { @@ -17,6 +20,7 @@ varnish v1 -cliok "param.set send_timeout 1" logexpect l1 -v v1 { expect * * Debug "Hit total send timeout" + expect * * Debug "Hit total send timeout" } -start client c1 -rcvbuf 128 { @@ -24,8 +28,17 @@ client c1 -rcvbuf 128 { rxresphdrs # keep the session open for 2 seconds delay 2 -} -run +} -start + +client c1u -connect "${tmpdir}/v1.sock" -rcvbuf 128 { + txreq + rxresphdrs + # keep the session open for 2 seconds + delay 2 +} -start +client c1 -wait +client c1u -wait logexpect l1 -wait feature SO_RCVTIMEO_WORKS @@ -34,6 +47,7 @@ varnish v1 -cliok "param.reset send_timeout" logexpect l2 -v v1 { expect * * Debug "Hit idle send timeout" + expect * * Debug "Hit idle send timeout" } -start client c2 -rcvbuf 128 { @@ -41,6 +55,15 @@ client c2 -rcvbuf 128 { rxresphdrs # keep the session open for 2 seconds delay 2 -} -run +} -start + +client c2u -connect "${tmpdir}/v1.sock" -rcvbuf 128 { + txreq + rxresphdrs + # keep the session open for 2 seconds + delay 2 +} -start +client c2 -wait +client c2u -wait logexpect l2 -wait From nils.goroll at uplex.de Wed Apr 24 12:28:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 12:28:07 +0000 (UTC) Subject: [master] 21da8c35a GC DBG_WAITER Message-ID: <20190424122807.D276F9779A@lists.varnish-cache.org> commit 21da8c35a90bf4b964b8871a37ad31dc1b52d1ee Author: Nils Goroll Date: Wed Apr 24 14:19:46 2019 +0200 GC DBG_WAITER the last uses were removed in fcbc79510967d7a985bc00bfea20edcb8993f95f diff --git a/include/tbl/debug_bits.h b/include/tbl/debug_bits.h index 83206e80d..f76eb76ec 100644 --- a/include/tbl/debug_bits.h +++ b/include/tbl/debug_bits.h @@ -33,7 +33,6 @@ DEBUG_BIT(REQ_STATE, req_state, "VSL Request state engine") DEBUG_BIT(WORKSPACE, workspace, "VSL Workspace operations") -DEBUG_BIT(WAITER, waiter, "VSL Waiter internals") DEBUG_BIT(WAITINGLIST, waitinglist, "VSL Waitinglist events") DEBUG_BIT(SYNCVSL, syncvsl, "Make VSL synchronous") DEBUG_BIT(HASHEDGE, hashedge, "Edge cases in Hash") From nils.goroll at uplex.de Wed Apr 24 12:28:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 12:28:07 +0000 (UTC) Subject: [master] bfbd2963f change debug.sndbuf to only handle int buffer lengths Message-ID: <20190424122807.E78AD9779D@lists.varnish-cache.org> commit bfbd2963fc719ac78eba5f68259a86e3401cac65 Author: Nils Goroll Date: Wed Apr 24 14:26:40 2019 +0200 change debug.sndbuf to only handle int buffer lengths should also fix printf format errors on 32bit diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index 870250ce3..e2fef3125 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -809,9 +809,9 @@ xyzzy_release_vcl_busy(VRT_CTX) } VCL_VOID -xyzzy_sndbuf(VRT_CTX, VCL_BYTES buflen) +xyzzy_sndbuf(VRT_CTX, VCL_BYTES arg) { - int fd = -1, oldbuf, newbuf; + int fd = -1, oldbuf, newbuf, buflen; socklen_t intlen = sizeof(int); CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -832,6 +832,13 @@ xyzzy_sndbuf(VRT_CTX, VCL_BYTES buflen) xxxassert(fd >= 0); + if (arg > INT_MAX) + buflen = INT_MAX; + else if (arg <= 0) + buflen = 0; + else + buflen = (int)arg; + oldbuf = 0; AZ(getsockopt(fd, SOL_SOCKET, SO_SNDBUF, &oldbuf, &intlen)); @@ -840,6 +847,6 @@ xyzzy_sndbuf(VRT_CTX, VCL_BYTES buflen) AZ(getsockopt(fd, SOL_SOCKET, SO_SNDBUF, &newbuf, &intlen)); AN(ctx->vsl); - VSLb(ctx->vsl, SLT_Debug, "SO_SNDBUF fd=%d old=%d new=%ld actual=%d", + VSLb(ctx->vsl, SLT_Debug, "SO_SNDBUF fd=%d old=%d new=%d actual=%d", fd, oldbuf, buflen, newbuf); } From nils.goroll at uplex.de Wed Apr 24 12:59:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 12:59:07 +0000 (UTC) Subject: [master] 09785c4ee the current vtc -rcvbuf implementation does not work on SunOS Message-ID: <20190424125907.7C8929B4BA@lists.varnish-cache.org> commit 09785c4ee0331a0f2111b1b501e5c2657cf39fff Author: Nils Goroll Date: Wed Apr 24 14:56:09 2019 +0200 the current vtc -rcvbuf implementation does not work on SunOS Ref #2980 diff --git a/bin/varnishtest/tests/s00010.vtc b/bin/varnishtest/tests/s00010.vtc index c641db519..d3ce9473e 100644 --- a/bin/varnishtest/tests/s00010.vtc +++ b/bin/varnishtest/tests/s00010.vtc @@ -1,5 +1,8 @@ varnishtest "client h1 send timeouts" +# XXX See https://github.com/varnishcache/varnish-cache/pull/2980#issuecomment-486214661 +feature cmd {test $(uname) != "SunOS"} + server s1 { rxreq txresp -bodylen 100000 diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 99b46eaf2..041690caa 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -1918,6 +1918,8 @@ http_process(struct vtclog *vl, const char *spec, int sock, int *sfd, hp->timeout = vtc_maxdur * 1000 / 2; if (rcvbuf) { + // XXX setsockopt() too late on SunOS + // https://github.com/varnishcache/varnish-cache/pull/2980#issuecomment-486214661 hp->rcvbuf = rcvbuf; oldbuf = 0; From nils.goroll at uplex.de Wed Apr 24 13:08:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 13:08:08 +0000 (UTC) Subject: [master] cc90aab3a enable -rcvbuf vtc only on Linux for now Message-ID: <20190424130808.B2C3E9B954@lists.varnish-cache.org> commit cc90aab3ac884ddc5c56df0662c84ef0ed9cf072 Author: Nils Goroll Date: Wed Apr 24 15:03:29 2019 +0200 enable -rcvbuf vtc only on Linux for now diff --git a/bin/varnishtest/tests/s00010.vtc b/bin/varnishtest/tests/s00010.vtc index d3ce9473e..ad1e76132 100644 --- a/bin/varnishtest/tests/s00010.vtc +++ b/bin/varnishtest/tests/s00010.vtc @@ -1,7 +1,7 @@ varnishtest "client h1 send timeouts" # XXX See https://github.com/varnishcache/varnish-cache/pull/2980#issuecomment-486214661 -feature cmd {test $(uname) != "SunOS"} +feature cmd {test $(uname) == "Linux"} server s1 { rxreq From nils.goroll at uplex.de Wed Apr 24 14:11:09 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 14:11:09 +0000 (UTC) Subject: [master] 60d3263b8 document vmod_vtc snapshot limitation Message-ID: <20190424141109.2E1099CFC3@lists.varnish-cache.org> commit 60d3263b87d1587a8285e72a0b3a45127d3baaa0 Author: Nils Goroll Date: Wed Apr 24 16:08:55 2019 +0200 document vmod_vtc snapshot limitation diff --git a/lib/libvmod_vtc/vmod.vcc b/lib/libvmod_vtc/vmod.vcc index e1644fffc..b6a8e4354 100644 --- a/lib/libvmod_vtc/vmod.vcc +++ b/lib/libvmod_vtc/vmod.vcc @@ -102,7 +102,9 @@ Find how much unallocated space there is left in a workspace. $Function VOID workspace_snapshot(ENUM { client, backend, session, thread}) -Snapshot a workspace. Only one snapshot may be active at a time. +Snapshot a workspace. Only one snapshot may be active at a time and +each VCL can save only one snapshot, so concurrent tasks requiring +snapshots are not supported. $Function VOID workspace_reset(ENUM { client, backend, session, thread }) diff --git a/lib/libvmod_vtc/vmod_vtc.c b/lib/libvmod_vtc/vmod_vtc.c index ee9d23fa0..b4022e869 100644 --- a/lib/libvmod_vtc/vmod_vtc.c +++ b/lib/libvmod_vtc/vmod_vtc.c @@ -124,6 +124,7 @@ vmod_sleep(VRT_CTX, VCL_DURATION t) /*--------------------------------------------------------------------*/ +// XXX this really should be PRIV_TASK state static uintptr_t vtc_ws_snapshot; static struct ws * From nils.goroll at uplex.de Wed Apr 24 15:03:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 15:03:07 +0000 (UTC) Subject: [master] ed418ea0a modernize enum handling Message-ID: <20190424150307.ADFDEA13C6@lists.varnish-cache.org> commit ed418ea0a9916c1bac058eb23a48ea6f81b3a18d Author: Nils Goroll Date: Wed Apr 24 16:41:59 2019 +0200 modernize enum handling diff --git a/lib/libvmod_vtc/vmod_vtc.c b/lib/libvmod_vtc/vmod_vtc.c index b4022e869..c0620bd72 100644 --- a/lib/libvmod_vtc/vmod_vtc.c +++ b/lib/libvmod_vtc/vmod_vtc.c @@ -131,13 +131,13 @@ static struct ws * vtc_ws_find(VRT_CTX, VCL_ENUM which) { - if (!strcmp(which, "client")) + if (which == VENUM(client)) return (ctx->ws); - if (!strcmp(which, "backend")) + if (which == VENUM(backend)) return (ctx->bo->ws); - if (!strcmp(which, "session")) + if (which == VENUM(session)) return (ctx->req->sp->ws); - if (!strcmp(which, "thread")) + if (which == VENUM(thread)) return (ctx->req->wrk->aws); WRONG("vtc_ws_find Illegal enum"); } From nils.goroll at uplex.de Wed Apr 24 15:03:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 15:03:07 +0000 (UTC) Subject: [master] 0da959260 add a workspace dump utility function Message-ID: <20190424150307.C1BA6A13C9@lists.varnish-cache.org> commit 0da959260561d14293f31ae4c83d46955cccb324 Author: Nils Goroll Date: Wed Apr 24 16:42:45 2019 +0200 add a workspace dump utility function diff --git a/lib/libvmod_vtc/vmod.vcc b/lib/libvmod_vtc/vmod.vcc index b6a8e4354..ccff7b654 100644 --- a/lib/libvmod_vtc/vmod.vcc +++ b/lib/libvmod_vtc/vmod.vcc @@ -119,6 +119,18 @@ $Function VOID workspace_overflow(ENUM { client, backend, session, thread }) Mark a workspace as overflowed. +$Function BLOB workspace_dump( + ENUM { client, backend, session, thread }, + ENUM { s, f, r }, + BYTES off=0, + BYTES len=64) + +Return data from a workspace's ``s``, ``f``, or ``r`` pointer as a +blob. Data is copied onto the primary workspace to avoid it being +subsequently overwritten. + +The maximum *len* is 1KB. + $Function INT typesize(STRING) Returns the size in bytes of a collection of C-datatypes: diff --git a/lib/libvmod_vtc/vmod_vtc.c b/lib/libvmod_vtc/vmod_vtc.c index c0620bd72..ceaf76ac7 100644 --- a/lib/libvmod_vtc/vmod_vtc.c +++ b/lib/libvmod_vtc/vmod_vtc.c @@ -209,6 +209,62 @@ VTC_WS_OP(VOID, , overflow, WS_MarkOverflow(ws)) VTC_WS_OP(BOOL, (0), overflowed, return (WS_Overflowed(ws))) #undef VTC_WS_OP +VCL_BLOB v_matchproto_(td_vtc_workspace_dump) +vmod_workspace_dump(VRT_CTX, VCL_ENUM which, VCL_ENUM where, + VCL_BYTES off, VCL_BYTES len) +{ + struct ws *ws; + const size_t maxlen = 1024; + unsigned char buf[maxlen]; + const char *p; + unsigned l; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + + ws = vtc_ws_find(ctx, which); + if (ws == NULL) + return (NULL); + WS_Assert(ws); + + if (len > maxlen) { + VRT_fail(ctx, "workspace_dump: max length is %zd", maxlen); + return (NULL); + } + + if (where == VENUM(s)) + p = ws->s; + else if (where == VENUM(f)) + p = ws->f; + else if (where == VENUM(r)) + p = ws->r; + else + INCOMPL(); + + if (p == NULL) { + VSLb(ctx->vsl, SLT_Error, "workspace_dump: NULL"); + return (NULL); + } + + p += off; + if (p >= ws->e) { + VSLb(ctx->vsl, SLT_Error, "workspace_dump: off limit"); + return (NULL); + } + + l = pdiff(p, ws->e); + if (len < l) + l = len; + + assert(l < maxlen); + memcpy(buf, p, l); + p = WS_Copy(ctx->ws, buf, l); + if (p == NULL) { + VRT_fail(ctx, "workspace_dump: copy failed"); + return (NULL); + } + return (VRT_blob(ctx, "workspace_dump", p, l, 0xd000d000)); +} + /*--------------------------------------------------------------------*/ VCL_INT v_matchproto_(td_vtc_typesize) From nils.goroll at uplex.de Wed Apr 24 15:03:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 15:03:07 +0000 (UTC) Subject: [master] 0c4a95487 test timeout_linger Message-ID: <20190424150307.DB2E6A13CD@lists.varnish-cache.org> commit 0c4a95487f67a1f8270eef3436dbe60f97f0c240 Author: Nils Goroll Date: Wed Apr 24 16:57:53 2019 +0200 test timeout_linger see comment in the vtc: For now this test is exploiting some implementation detail and we might want to consider adding a VSL for waiter involvement. diff --git a/bin/varnishtest/tests/b00068.vtc b/bin/varnishtest/tests/b00068.vtc new file mode 100644 index 000000000..afe44c5c4 --- /dev/null +++ b/bin/varnishtest/tests/b00068.vtc @@ -0,0 +1,92 @@ +varnishtest "Check timeout_linger" + +# XXX this test exploits the fact that the struct waited is +# left near the free pointer of the session ws when a session +# made a tour over the waiter +# +# Would we want VSL Info about waiter involvement? +# + +varnish v1 -arg "-p timeout_linger=1" \ + -arg "-a 127.0.0.1:0" \ + -arg "-a ${tmpdir}/v1.sock" \ + -vcl { + import std; + import vtc; + import blob; + + backend dummy { .host = "${bad_ip}"; } + + sub vcl_recv { + std.log(blob.encode(encoding=HEX, + blob=vtc.workspace_dump(session, f))); + } + sub vcl_backend_error { + set beresp.status = 200; + set beresp.ttl = 1h; + } +} -start + +logexpect l1 -v v1 -q "vxid == 1001" { + expect * 1001 VCL_call {^RECV} + expect 0 1001 VCL_Log "^0{128}$" +} -start + +logexpect l2 -v v1 -q "vxid == 1004" { + expect * 1004 VCL_call {^RECV} + expect 0 1004 VCL_Log "^0{128}$" +} -start + +logexpect l3 -v v1 -q "vxid == 1005" { + expect * 1005 VCL_call {^RECV} + expect 0 1005 VCL_Log "^0{128}$" +} -start + +logexpect l4 -v v1 -q "vxid == 1006" { + expect * 1006 VCL_call {^RECV} + expect 0 1006 VCL_Log "^0{128}$" +} -start + +# requests which made a de-tour over the waiter + +logexpect l5 -v v1 -q "vxid == 1007" { + expect * 1007 VCL_call {^RECV} + expect 0 1007 VCL_Log "[1-9a-f]" +} -start + +logexpect l6 -v v1 -q "vxid == 1008" { + expect * 1008 VCL_call {^RECV} + expect 0 1008 VCL_Log "[1-9a-f]" +} -start + +client c1 { + txreq + rxresp + delay 0.2 + txreq + rxresp + delay 1.2 + txreq + rxresp +} -start + +client c1u -connect "${tmpdir}/v1.sock" { + txreq + rxresp + delay 0.2 + txreq + rxresp + delay 1.2 + txreq + rxresp +} -start + +client c1 -wait +client c1u -wait + +logexpect l1 -wait +logexpect l2 -wait +logexpect l3 -wait +logexpect l4 -wait +logexpect l5 -wait +logexpect l6 -wait From nils.goroll at uplex.de Wed Apr 24 15:24:12 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 15:24:12 +0000 (UTC) Subject: [master] 1ad4bb344 rework the logexpects of new b68.vtc Message-ID: <20190424152412.78034A1F7E@lists.varnish-cache.org> commit 1ad4bb344cc16f17e05a2d350834f5f202922b97 Author: Nils Goroll Date: Wed Apr 24 17:22:50 2019 +0200 rework the logexpects of new b68.vtc Relying on fixed vxids was found to fail, sorry diff --git a/bin/varnishtest/tests/b00068.vtc b/bin/varnishtest/tests/b00068.vtc index afe44c5c4..e42b1a4e7 100644 --- a/bin/varnishtest/tests/b00068.vtc +++ b/bin/varnishtest/tests/b00068.vtc @@ -27,36 +27,22 @@ varnish v1 -arg "-p timeout_linger=1" \ } } -start -logexpect l1 -v v1 -q "vxid == 1001" { - expect * 1001 VCL_call {^RECV} - expect 0 1001 VCL_Log "^0{128}$" +logexpect l1 -v v1 -g session -q "SessOpen ~ a0" { + expect * * VCL_call {^RECV} + expect 0 = VCL_Log "^0{128}$" + expect * * VCL_call {^RECV} + expect 0 = VCL_Log "^0{128}$" + expect * * VCL_call {^RECV} + expect 0 = VCL_Log "[1-9a-f]" } -start -logexpect l2 -v v1 -q "vxid == 1004" { - expect * 1004 VCL_call {^RECV} - expect 0 1004 VCL_Log "^0{128}$" -} -start - -logexpect l3 -v v1 -q "vxid == 1005" { - expect * 1005 VCL_call {^RECV} - expect 0 1005 VCL_Log "^0{128}$" -} -start - -logexpect l4 -v v1 -q "vxid == 1006" { - expect * 1006 VCL_call {^RECV} - expect 0 1006 VCL_Log "^0{128}$" -} -start - -# requests which made a de-tour over the waiter - -logexpect l5 -v v1 -q "vxid == 1007" { - expect * 1007 VCL_call {^RECV} - expect 0 1007 VCL_Log "[1-9a-f]" -} -start - -logexpect l6 -v v1 -q "vxid == 1008" { - expect * 1008 VCL_call {^RECV} - expect 0 1008 VCL_Log "[1-9a-f]" +logexpect l2 -v v1 -g session -q "SessOpen ~ a1" { + expect * * VCL_call {^RECV} + expect 0 = VCL_Log "^0{128}$" + expect * * VCL_call {^RECV} + expect 0 = VCL_Log "^0{128}$" + expect * * VCL_call {^RECV} + expect 0 = VCL_Log "[1-9a-f]" } -start client c1 { @@ -86,7 +72,3 @@ client c1u -wait logexpect l1 -wait logexpect l2 -wait -logexpect l3 -wait -logexpect l4 -wait -logexpect l5 -wait -logexpect l6 -wait From nils.goroll at uplex.de Wed Apr 24 15:54:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Wed, 24 Apr 2019 15:54:07 +0000 (UTC) Subject: [master] 211f72da3 fix sh syntax Message-ID: <20190424155407.D64DEA2B9D@lists.varnish-cache.org> commit 211f72da35816b5b2b8767bc0fe8c111f250cafe Author: Nils Goroll Date: Wed Apr 24 17:53:44 2019 +0200 fix sh syntax diff --git a/bin/varnishtest/tests/s00010.vtc b/bin/varnishtest/tests/s00010.vtc index ad1e76132..9097e4fd9 100644 --- a/bin/varnishtest/tests/s00010.vtc +++ b/bin/varnishtest/tests/s00010.vtc @@ -1,7 +1,7 @@ varnishtest "client h1 send timeouts" # XXX See https://github.com/varnishcache/varnish-cache/pull/2980#issuecomment-486214661 -feature cmd {test $(uname) == "Linux"} +feature cmd {test $(uname) = "Linux"} server s1 { rxreq From nils.goroll at uplex.de Thu Apr 25 08:45:10 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 25 Apr 2019 08:45:10 +0000 (UTC) Subject: [master] 175c8ddea fix ban.list obj.http.* rendering Message-ID: <20190425084510.2832691A9@lists.varnish-cache.org> commit 175c8ddea715a2270d6205e74874aaa8aa12140b Author: Nils Goroll Date: Thu Apr 25 10:43:57 2019 +0200 fix ban.list obj.http.* rendering Fixes #2985 diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c index 283cad024..0bfca2d04 100644 --- a/bin/varnishd/cache/cache_ban.c +++ b/bin/varnishd/cache/cache_ban.c @@ -798,7 +798,7 @@ ban_render(struct cli *cli, const uint8_t *bs, int quote) ASSERT_BAN_OPER(bt.oper); if (BANS_HAS_ARG1_SPEC(bt.arg1)) - VCLI_Out(cli, "%s.%.*s", + VCLI_Out(cli, "%s%.*s", arg_name[BAN_ARGIDX(bt.arg1)], bt.arg1_spec[0] - 1, bt.arg1_spec + 1); else diff --git a/bin/varnishtest/tests/c00019.vtc b/bin/varnishtest/tests/c00019.vtc index 3a4a3b2f8..9c8bd4e55 100644 --- a/bin/varnishtest/tests/c00019.vtc +++ b/bin/varnishtest/tests/c00019.vtc @@ -92,5 +92,5 @@ varnish v1 -clierr 106 "ban req.url ~ [[[" # Ban expression with quoting varnish v1 -cliok {ban req.url ~ "BAR"} shell {varnishadm -n ${tmpdir}/v1 ban 'obj.http.Host ~ \"Foo\"'} -varnish v1 -cliok "ban.list" +varnish v1 -cliexpect {(?s)\d+\.\d+\s+\d+\s+-\s+\Qobj.http.Host ~ "Foo"\E} "ban.list" varnish v1 -clijson "ban.list -j" From nils.goroll at uplex.de Thu Apr 25 09:26:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 25 Apr 2019 09:26:08 +0000 (UTC) Subject: [master] 084d3c336 another REL_ marker Message-ID: <20190425092608.7440D9FA4@lists.varnish-cache.org> commit 084d3c336e0c7059437fe3763f4615e8a2599701 Author: Nils Goroll Date: Thu Apr 25 11:25:29 2019 +0200 another REL_ marker diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 434419409..54e4c3f58 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -151,6 +151,10 @@ SLTM(ProxyGarbage, 0, "Unparseable PROXY request", "A PROXY protocol header was unparseable.\n\n" ) +/* + * REL_20190915 remove after VSLng + * kept for now for VSL binary compatibility + */ SLTM(Backend, 0, "Backend selected", "Logged when a connection is selected for handling a backend" " request.\n\n" From martin at varnish-software.com Fri Apr 26 08:51:08 2019 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Fri, 26 Apr 2019 08:51:08 +0000 (UTC) Subject: [master] 4130055c4 Resolve race on waitinglist rushing between OC_F_BUSY and boc->state Message-ID: <20190426085108.73395AEEB0@lists.varnish-cache.org> commit 4130055c457903e7f904fa56c0db58d7e7467dc1 Author: Martin Blix Grydeland Date: Thu Apr 25 13:39:59 2019 +0200 Resolve race on waitinglist rushing between OC_F_BUSY and boc->state When an object is ready for delivery, HSH_Unbusy was called before calling ObjSetState([BOS_STREAM|BOS_FINISHED]). The HSH_Unbusy() call does the waitinglist rushing, but HSH_Lookup() wanted to look at the boc->state and if BOS_STREAM had been reached. This could cause requests woken to find that the stream state still hadn't been reached (ObjSetState still hadn't executed), and go back on the waitinglist. To fix this, this patch reverts commit 0375791cac1f2333ab54932ba1d2025261082fab, and goes back to considering OC_F_BUSY as the gate keeper for HSH_Lookup. This eliminates the race, because HSH_Unbusy and HSH_Lookup then uses the same mutex. That change opens up the possiblity that req code after HSH_Lookup() sees an object that has not yet reached BOS_STREAM. In order to not have to add new ObjWaitState() calls (with the additional locking cost that would bring) to wait for BOS_STREAM, the order of events is changed throughout, and calls ObjSetState([BOS_STREAM|BOS_FINISHED]) before HSH_Unbusy(). That way, an object returned from HSH_Lookup() is guaranteed to be at least BOS_STREAM. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index b3304bedb..63738e1fe 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -562,9 +562,8 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) assert(bo->fetch_objcore->boc->state == BOS_REQ_DONE); if (bo->do_stream) { - ObjSetState(wrk, bo->fetch_objcore, BOS_PREP_STREAM); - HSH_Unbusy(wrk, bo->fetch_objcore); ObjSetState(wrk, bo->fetch_objcore, BOS_STREAM); + HSH_Unbusy(wrk, bo->fetch_objcore); } VSLb(bo->vsl, SLT_Fetch_Body, "%u %s %s", @@ -589,18 +588,20 @@ vbf_stp_fetchend(struct worker *wrk, struct busyobj *bo) AZ(ObjSetU64(wrk, bo->fetch_objcore, OA_LEN, bo->fetch_objcore->boc->len_so_far)); - if (bo->do_stream) - assert(bo->fetch_objcore->boc->state == BOS_STREAM); - else { - assert(bo->fetch_objcore->boc->state == BOS_REQ_DONE); - HSH_Unbusy(wrk, bo->fetch_objcore); - } - /* Recycle the backend connection before setting BOS_FINISHED to give predictable backend reuse behavior for varnishtest */ VDI_Finish(bo); + if (bo->do_stream) + assert(bo->fetch_objcore->boc->state == BOS_STREAM); + else + assert(bo->fetch_objcore->boc->state == BOS_REQ_DONE); ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED); + + if (!bo->do_stream) + HSH_Unbusy(wrk, bo->fetch_objcore); + AZ(bo->fetch_objcore->flags & OC_F_BUSY); + VSLb_ts_busyobj(bo, "BerespBody", W_TIM_real(wrk)); if (bo->stale_oc != NULL) HSH_Kill(bo->stale_oc); @@ -654,9 +655,8 @@ vbf_stp_condfetch(struct worker *wrk, struct busyobj *bo) AZ(ObjCopyAttr(bo->wrk, bo->fetch_objcore, bo->stale_oc, OA_GZIPBITS)); if (bo->do_stream) { - ObjSetState(wrk, bo->fetch_objcore, BOS_PREP_STREAM); - HSH_Unbusy(wrk, bo->fetch_objcore); ObjSetState(wrk, bo->fetch_objcore, BOS_STREAM); + HSH_Unbusy(wrk, bo->fetch_objcore); } if (ObjIterate(wrk, bo->stale_oc, bo, vbf_objiterator, 0)) @@ -790,10 +790,10 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) } AZ(ObjSetU64(wrk, bo->fetch_objcore, OA_LEN, o)); VSB_destroy(&synth_body); + ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED); HSH_Unbusy(wrk, bo->fetch_objcore); if (stale != NULL && bo->fetch_objcore->ttl > 0) HSH_Kill(stale); - ObjSetState(wrk, bo->fetch_objcore, BOS_FINISHED); return (F_STP_DONE); } @@ -808,10 +808,10 @@ vbf_stp_fail(struct worker *wrk, const struct busyobj *bo) CHECK_OBJ_NOTNULL(bo->fetch_objcore, OBJCORE_MAGIC); assert(bo->fetch_objcore->boc->state < BOS_FINISHED); + ObjSetState(wrk, bo->fetch_objcore, BOS_FAILED); HSH_Fail(bo->fetch_objcore); if (!(bo->fetch_objcore->flags & OC_F_BUSY)) HSH_Kill(bo->fetch_objcore); - ObjSetState(wrk, bo->fetch_objcore, BOS_FAILED); return (F_STP_DONE); } @@ -982,8 +982,6 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, ObjWaitState(oc, BOS_STREAM); if (oc->boc->state == BOS_FAILED) { AN((oc->flags & OC_F_FAILED)); - } else { - AZ(oc->flags & OC_F_BUSY); } } } diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index ead55932c..66167df33 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -420,11 +420,11 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp) continue; CHECK_OBJ_ORNULL(oc->boc, BOC_MAGIC); - if (oc->boc != NULL && oc->boc->state < BOS_STREAM) { + if (oc->flags & OC_F_BUSY) { if (req->hash_ignore_busy) continue; - if (oc->boc->vary != NULL && + if (oc->boc != NULL && oc->boc->vary != NULL && !VRY_Match(req, oc->boc->vary)) continue; diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index b533eefcf..89bb0497d 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -267,7 +267,6 @@ ObjSetState(struct worker *wrk, const struct objcore *oc, assert(next > oc->boc->state); CHECK_OBJ_ORNULL(oc->stobj->stevedore, STEVEDORE_MAGIC); - assert(next != BOS_STREAM || oc->boc->state == BOS_PREP_STREAM); assert(next != BOS_FINISHED || (oc->oa_present & (1 << OA_LEN))); if (oc->stobj->stevedore != NULL) { diff --git a/include/tbl/boc_state.h b/include/tbl/boc_state.h index a5c4a61ff..751456ca6 100644 --- a/include/tbl/boc_state.h +++ b/include/tbl/boc_state.h @@ -30,7 +30,6 @@ BOC_STATE(INVALID, invalid) /* don't touch (yet) */ BOC_STATE(REQ_DONE, req_done) /* bereq.* can be examined */ -BOC_STATE(PREP_STREAM, prep_stream) /* Prepare for streaming */ BOC_STATE(STREAM, stream) /* beresp.* can be examined */ BOC_STATE(FINISHED, finished) /* object is complete */ BOC_STATE(FAILED, failed) /* something went wrong */ From martin at varnish-software.com Fri Apr 26 08:51:08 2019 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Fri, 26 Apr 2019 08:51:08 +0000 (UTC) Subject: [master] 5db61e045 Tune test cases c97-c99 Message-ID: <20190426085108.90B92AEEB3@lists.varnish-cache.org> commit 5db61e0457334392fdc4da8bb5a065a3a191692a Author: Martin Blix Grydeland Date: Thu Apr 25 14:47:39 2019 +0200 Tune test cases c97-c99 This reverts some of the previous attempts to get these test cases stable, as those attempts actually prevented the testing of the desired code paths. Also make the test cases wait until the required requests are on the waitinglist before continuing. diff --git a/bin/varnishtest/tests/c00097.vtc b/bin/varnishtest/tests/c00097.vtc index 52469a690..afc77fbd9 100644 --- a/bin/varnishtest/tests/c00097.vtc +++ b/bin/varnishtest/tests/c00097.vtc @@ -1,49 +1,64 @@ varnishtest "Streaming delivery and waitinglist rushing" -barrier b1 sock 4 -barrier b2 sock 4 +# Barrier to make sure that c1 connects to s1 +barrier b1 cond 2 + +# Barrier to make sure that all requests are on waitinglist before +# HSH_Unbusy is called +barrier b2 cond 2 + +# Barrier to control that all requests start streaming before the object +# finishes. This tests that waitinglists are rushed before +# HSH_DerefObjCore(). +barrier b3 sock 4 server s1 { rxreq + barrier b1 sync + barrier b2 sync txresp -nolen -hdr "Transfer-Encoding: chunked" chunkedlen 10 - barrier b1 sync + barrier b3 sync chunkedlen 10 chunkedlen 0 } -start -varnish v1 -arg "-p rush_exponent=2" -arg "-p debug=+syncvsl" -vcl+backend { +varnish v1 -arg "-p thread_pools=1" -arg "-p thread_pool_min=20" -arg "-p rush_exponent=2" -arg "-p debug=+syncvsl" -arg "-p debug=+waitinglist" -vcl+backend { import vtc; sub vcl_hit { - vtc.barrier_sync("${b1_sock}"); + vtc.barrier_sync("${b3_sock}"); } } -start client c1 { txreq - rxresp -no_obj - barrier b2 sync - rxrespbody + rxresp } -start +barrier b1 sync + client c2 { - barrier b2 sync txreq rxresp } -start client c3 { - barrier b2 sync txreq rxresp } -start client c4 { - barrier b2 sync txreq rxresp } -start +# Wait until c2-c4 are on the waitinglist +delay 1 +varnish v1 -expect busy_sleep == 3 + +# Open up the response headers from s1, and as a result HSH_Unbusy +barrier b2 sync + client c1 -wait client c2 -wait client c3 -wait diff --git a/bin/varnishtest/tests/c00098.vtc b/bin/varnishtest/tests/c00098.vtc index 32a809766..5b34d3a75 100644 --- a/bin/varnishtest/tests/c00098.vtc +++ b/bin/varnishtest/tests/c00098.vtc @@ -1,13 +1,24 @@ varnishtest "Hit-for-pass and waitinglist rushing" -barrier b2 cond 6 +# Barrier to make sure that s1 is run first +barrier b1 cond 2 + +# Barrier to make sure that all requests are on waitinglist before +# HSH_Unbusy is called +barrier b2 cond 2 + +# Barrier to control that all backends are reached before any request +# finishes. This tests that waitinglists are rushed before +# HSH_DerefObjCore(). barrier b3 cond 6 server s1 { rxreq + barrier b1 sync + barrier b2 sync txresp -nolen -hdr "Transfer-Encoding: chunked" chunkedlen 10 - barrier b2 sync + barrier b3 sync chunkedlen 10 chunkedlen 0 } -start @@ -16,7 +27,7 @@ server s2 { rxreq txresp -nolen -hdr "Transfer-Encoding: chunked" chunkedlen 10 - barrier b2 sync + barrier b3 sync chunkedlen 10 chunkedlen 0 } -start @@ -25,7 +36,7 @@ server s3 { rxreq txresp -nolen -hdr "Transfer-Encoding: chunked" chunkedlen 10 - barrier b2 sync + barrier b3 sync chunkedlen 10 chunkedlen 0 } -start @@ -34,7 +45,7 @@ server s4 { rxreq txresp -nolen -hdr "Transfer-Encoding: chunked" chunkedlen 10 - barrier b2 sync + barrier b3 sync chunkedlen 10 chunkedlen 0 } -start @@ -43,7 +54,7 @@ server s5 { rxreq txresp -nolen -hdr "Transfer-Encoding: chunked" chunkedlen 10 - barrier b2 sync + barrier b3 sync chunkedlen 10 chunkedlen 0 } -start @@ -52,13 +63,12 @@ server s6 { rxreq txresp -nolen -hdr "Transfer-Encoding: chunked" chunkedlen 10 - barrier b2 sync + barrier b3 sync chunkedlen 10 chunkedlen 0 } -start -varnish v1 -arg "-p thread_pool_min=20" -arg "-p rush_exponent=2" -arg "-p debug=+syncvsl" -vcl+backend { - import vtc; +varnish v1 -arg "-p thread_pools=1" -arg "-p thread_pool_min=30" -arg "-p rush_exponent=2" -arg "-p debug=+syncvsl" -arg "-p debug=+waitinglist" -vcl+backend { sub vcl_backend_fetch { if (bereq.http.client == "1") { set bereq.backend = s1; @@ -81,50 +91,47 @@ varnish v1 -arg "-p thread_pool_min=20" -arg "-p rush_exponent=2" -arg "-p debug client c1 { txreq -url /hfp -hdr "Client: 1" - rxresp -no_obj - barrier b3 sync + rxresp } -start +# This makes sure that c1->s1 is done first +barrier b1 sync + client c2 { - barrier b3 sync txreq -url /hfp -hdr "Client: 2" rxresp } -start client c3 { - barrier b3 sync txreq -url /hfp -hdr "Client: 3" rxresp } -start client c4 { - barrier b3 sync txreq -url /hfp -hdr "Client: 4" rxresp } -start client c5 { - barrier b3 sync txreq -url /hfp -hdr "Client: 5" rxresp } -start client c6 { - barrier b3 sync txreq -url /hfp -hdr "Client: 6" rxresp } -start +# Wait until c2-c6 are on the waitinglist +delay 1 +varnish v1 -expect busy_sleep == 5 + +# Open up the response headers from s1, and as a result HSH_Unbusy +barrier b2 sync + client c1 -wait client c2 -wait client c3 -wait client c4 -wait client c5 -wait client c6 -wait - -server s1 -wait -server s2 -wait -server s3 -wait -server s4 -wait -server s5 -wait -server s6 -wait diff --git a/bin/varnishtest/tests/c00099.vtc b/bin/varnishtest/tests/c00099.vtc index 58363ffc3..4bbd904a0 100644 --- a/bin/varnishtest/tests/c00099.vtc +++ b/bin/varnishtest/tests/c00099.vtc @@ -1,10 +1,21 @@ varnishtest "Hit-for-miss and waitinglist rushing" +# Barrier to make sure that s1 is run first +barrier b1 cond 2 + +# Barrier to make sure that all requests are on waitinglist before +# HSH_Unbusy is called +barrier b2 cond 2 + +# Barrier to control that all backends are reached before any request +# finishes. This tests that waitinglists are rushed before +# HSH_DerefObjCore(). barrier b3 cond 6 -barrier b4 cond 6 server s1 { rxreq + barrier b1 sync + barrier b2 sync txresp -nolen -hdr "Transfer-Encoding: chunked" chunkedlen 10 barrier b3 sync @@ -57,8 +68,7 @@ server s6 { chunkedlen 0 } -start -varnish v1 -arg "-p thread_pool_min=20" -arg "-p rush_exponent=2" -arg "-p debug=+syncvsl" -vcl+backend { - import vtc; +varnish v1 -arg "-p thread_pools=1" -arg "-p thread_pool_min=30" -arg "-p rush_exponent=2" -arg "-p debug=+syncvsl" -arg "-p debug=+waitinglist" -vcl+backend { sub vcl_backend_fetch { if (bereq.http.client == "1") { set bereq.backend = s1; @@ -81,50 +91,47 @@ varnish v1 -arg "-p thread_pool_min=20" -arg "-p rush_exponent=2" -arg "-p debug client c1 { txreq -url /hfm -hdr "Client: 1" - rxresp -no_obj - barrier b4 sync + rxresp } -start +# This makes sure that c1->s1 is done first +barrier b1 sync + client c2 { - barrier b4 sync txreq -url /hfm -hdr "Client: 2" rxresp } -start client c3 { - barrier b4 sync txreq -url /hfm -hdr "Client: 3" rxresp } -start client c4 { - barrier b4 sync txreq -url /hfm -hdr "Client: 4" rxresp } -start client c5 { - barrier b4 sync txreq -url /hfm -hdr "Client: 5" rxresp } -start client c6 { - barrier b4 sync txreq -url /hfm -hdr "Client: 6" rxresp } -start +# Wait until c2-c6 are on the waitinglist +delay 1 +varnish v1 -expect busy_sleep == 5 + +# Open up the response headers from s1, and as a result HSH_Unbusy +barrier b2 sync + client c1 -wait client c2 -wait client c3 -wait client c4 -wait client c5 -wait client c6 -wait - -server s1 -wait -server s2 -wait -server s3 -wait -server s4 -wait -server s5 -wait -server s6 -wait From martin at varnish-software.com Fri Apr 26 11:16:08 2019 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Fri, 26 Apr 2019 11:16:08 +0000 (UTC) Subject: [master] 462eab253 Remove a racy assert Message-ID: <20190426111608.6F21BB1DA9@lists.varnish-cache.org> commit 462eab25381ccfd914be4b406e46d0441257f06f Author: Martin Blix Grydeland Date: Fri Apr 26 13:13:48 2019 +0200 Remove a racy assert With the recent changes where HSH_Unbusy/HSH_Fail is called after ObjSetState, this assert becomes racy. Remove the assert. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 63738e1fe..0dc8c3697 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -980,9 +980,6 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, (void)VRB_Ignore(req); } else { ObjWaitState(oc, BOS_STREAM); - if (oc->boc->state == BOS_FAILED) { - AN((oc->flags & OC_F_FAILED)); - } } } AZ(bo); From martin at varnish-software.com Fri Apr 26 12:15:09 2019 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Fri, 26 Apr 2019 12:15:09 +0000 (UTC) Subject: [master] 865df54a9 Revert "Remove a racy assert" Message-ID: <20190426121509.70167BE00D@lists.varnish-cache.org> commit 865df54a92f120e234147ceabc7bd76cb7918868 Author: Martin Blix Grydeland Date: Fri Apr 26 13:45:21 2019 +0200 Revert "Remove a racy assert" This reverts commit 462eab25381ccfd914be4b406e46d0441257f06f. That assert was there for a good reason. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 0dc8c3697..63738e1fe 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -980,6 +980,9 @@ VBF_Fetch(struct worker *wrk, struct req *req, struct objcore *oc, (void)VRB_Ignore(req); } else { ObjWaitState(oc, BOS_STREAM); + if (oc->boc->state == BOS_FAILED) { + AN((oc->flags & OC_F_FAILED)); + } } } AZ(bo); From martin at varnish-software.com Fri Apr 26 12:15:09 2019 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Fri, 26 Apr 2019 12:15:09 +0000 (UTC) Subject: [master] 501246e93 Fix the order of HSH_Fail() and ObjSetState() Message-ID: <20190426121509.82DE3BE010@lists.varnish-cache.org> commit 501246e939430dd60d44e8e3af40d6978f11186d Author: Martin Blix Grydeland Date: Fri Apr 26 14:11:18 2019 +0200 Fix the order of HSH_Fail() and ObjSetState() The previous patch 4130055c457903e7f904fa56c0db58d7e7467dc1 went a bit too far in it's mission to reorder events, and included putting HSH_Fail() after ObjSetState(). That caused problems for code looking at the OC_F_FAILED to learn about failed fetches. Change the order of this back to normal, and move the call to HSH_Fail() into ObjSetState(), so that the order can be shown and the caveats properly commented. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 63738e1fe..5535e00b7 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -809,7 +809,6 @@ vbf_stp_fail(struct worker *wrk, const struct busyobj *bo) assert(bo->fetch_objcore->boc->state < BOS_FINISHED); ObjSetState(wrk, bo->fetch_objcore, BOS_FAILED); - HSH_Fail(bo->fetch_objcore); if (!(bo->fetch_objcore->flags & OC_F_BUSY)) HSH_Kill(bo->fetch_objcore); return (F_STP_DONE); diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index 89bb0497d..d723975da 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -84,6 +84,7 @@ #include #include "cache_varnishd.h" +#include "cache_objhead.h" #include "cache_obj.h" #include "vend.h" #include "storage/storage.h" @@ -257,8 +258,7 @@ ObjWaitExtend(const struct worker *wrk, const struct objcore *oc, uint64_t l) */ void -ObjSetState(struct worker *wrk, const struct objcore *oc, - enum boc_state_e next) +ObjSetState(struct worker *wrk, struct objcore *oc, enum boc_state_e next) { const struct obj_methods *om; @@ -275,6 +275,13 @@ ObjSetState(struct worker *wrk, const struct objcore *oc, om->objsetstate(wrk, oc, next); } + if (next == BOS_FAILED) { + /* Signal to cache_hash.c that this object is failed. This + * needs to happen before we signal the boc so that code + * will see the OC_F_FAILED flag after ObjWaitState() */ + HSH_Fail(oc); + } + Lck_Lock(&oc->boc->mtx); oc->boc->state = next; AZ(pthread_cond_broadcast(&oc->boc->cond)); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index b3f766080..fab80d6ba 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -293,8 +293,7 @@ int ObjGetSpace(struct worker *, struct objcore *, ssize_t *sz, uint8_t **ptr); void ObjExtend(struct worker *, struct objcore *, ssize_t l); uint64_t ObjWaitExtend(const struct worker *, const struct objcore *, uint64_t l); -void ObjSetState(struct worker *, const struct objcore *, - enum boc_state_e next); +void ObjSetState(struct worker *, struct objcore *, enum boc_state_e next); void ObjWaitState(const struct objcore *, enum boc_state_e want); void ObjTrimStore(struct worker *, struct objcore *); void ObjTouch(struct worker *, struct objcore *, vtim_real now); From fgsch at lodoss.net Sat Apr 27 21:57:12 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 27 Apr 2019 21:57:12 +0000 (UTC) Subject: [master] c957ad878 Add -method for consistency Message-ID: <20190427215712.EE3B5BF031@lists.varnish-cache.org> commit c957ad878bdd2c2ed065a8c7faa87e3fb3289712 Author: Federico G. Schwindt Date: Sat Apr 27 20:41:29 2019 +0100 Add -method for consistency Make -req an alias. diff --git a/bin/varnishtest/tests/a00002.vtc b/bin/varnishtest/tests/a00002.vtc index 2d8c755fe..56bb068f5 100644 --- a/bin/varnishtest/tests/a00002.vtc +++ b/bin/varnishtest/tests/a00002.vtc @@ -11,7 +11,7 @@ server s1 { server s1 -start client c1 -connect ${s1_sock} { - txreq -req PUT -proto HTTP/1.0 -url /foo + txreq -method PUT -proto HTTP/1.0 -url /foo rxresp expect resp.proto == HTTP/1.2 expect resp.status == 201 diff --git a/bin/varnishtest/tests/a02014.vtc b/bin/varnishtest/tests/a02014.vtc index d11af49b9..0dd7d71e1 100644 --- a/bin/varnishtest/tests/a02014.vtc +++ b/bin/varnishtest/tests/a02014.vtc @@ -22,7 +22,7 @@ server s1 { } -start client c1 -connect ${s1_sock} { stream 1 { - txreq -req GET -url /1 \ + txreq -method GET -url /1 \ -hdr :scheme http -hdr :authority localhost rxresp expect stream.weight == 16 diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 5d75e6ef4..d4e03999f 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -979,9 +979,12 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, * from who can send them is that the first line (request line vs * status line), so all the options are prety much the same. * - * \-req STRING (txreq only) + * \-method STRING (txreq only) * What method to use (default: "GET"). * + * \-req STRING (txreq only) + * Alias for -method. + * * \-url STRING (txreq only) * What location to use (default "/"). * @@ -1295,7 +1298,8 @@ cmd_http_txreq(CMD_ARGS) } else if (!strcmp(*av, "-proto")) { proto = av[1]; av++; - } else if (!strcmp(*av, "-req")) { + } else if (!strcmp(*av, "-method") || + !strcmp(*av, "-req")) { req = av[1]; hp->head_method = !strcasecmp(av[1], "HEAD") ; av++; diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 00f254b1e..f49f955dc 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -1286,9 +1286,12 @@ cmd_sendhex(CMD_ARGS) * \-url STRING (txreq, txpush) * Set the :path pseudo-header. * - * \-req STRING (txreq, txpush) + * \-method STRING (txreq, txpush) * Set the :method pseudo-header. * + * \-req STRING (txreq, txpush) + * Alias for -method. + * * \-scheme STRING (txreq, txpush) * Set the :scheme pseudo-header. * @@ -1437,7 +1440,7 @@ cmd_tx11obj(CMD_ARGS) av++; path_done = 1; } - else if (AV_IS("-req") && + else if ((AV_IS("-method") || AV_IS("-req")) && (CMD_IS("txreq") || CMD_IS("txpush"))) { ENC(hdr, ":method", av[1]); av++; From fgsch at lodoss.net Sat Apr 27 21:57:12 2019 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sat, 27 Apr 2019 21:57:12 +0000 (UTC) Subject: [master] 3c2e42b71 Add -bodyfrom to read the body from a file Message-ID: <20190427215712.DAE02BF02F@lists.varnish-cache.org> commit 3c2e42b71c8f488adbc1b0045cce06de558f8863 Author: Federico G. Schwindt Date: Sat Apr 27 18:52:18 2019 +0100 Add -bodyfrom to read the body from a file Name subject to bikeshedding. diff --git a/bin/varnishtest/tests/a00020.vtc b/bin/varnishtest/tests/a00020.vtc new file mode 100644 index 000000000..6b50c11df --- /dev/null +++ b/bin/varnishtest/tests/a00020.vtc @@ -0,0 +1,14 @@ +varnishtest "Test -bodyfrom" + +server s1 { + rxreq + expect req.bodylen == 241 + txresp -bodyfrom ${testdir}/a00020.vtc +} -start + +client c1 -connect ${s1_sock} { + txreq -bodyfrom ${testdir}/a00020.vtc + rxresp + expect resp.bodylen == 241 +} -run + diff --git a/bin/varnishtest/tests/a02025.vtc b/bin/varnishtest/tests/a02025.vtc new file mode 100644 index 000000000..b3088d4d9 --- /dev/null +++ b/bin/varnishtest/tests/a02025.vtc @@ -0,0 +1,17 @@ +varnishtest "Test -bodyfrom" + +server s1 { + stream 1 { + rxreq + expect req.bodylen == 286 + txresp -bodyfrom ${testdir}/a02025.vtc + } -run +} -start + +client c1 -connect ${s1_sock} { + stream 1 { + txreq -bodyfrom ${testdir}/a02025.vtc + rxresp + expect resp.bodylen == 286 + } -run +} -run diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 041690caa..5d75e6ef4 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -868,6 +868,7 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, char *b, *c; char *nullbody; char *m; + ssize_t len; int nolen = 0; int l; @@ -912,6 +913,14 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, bodylen--; } } + } else if (!strcmp(*av, "-bodyfrom")) { + assert(body == nullbody); + free(body); + body = VFIL_readfile(NULL, av[1], &len); + AN(body); + assert(len < INT_MAX); + bodylen = len; + av++; } else if (!strcmp(*av, "-bodylen")) { assert(body == nullbody); free(body); @@ -1007,6 +1016,9 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, * \-body STRING * Input STRING as body. * + * \-bodyfrom FILE + * Same as -body but content is read from FILE. + * * \-bodylen NUMBER * Generate and input a body that is NUMBER bytes-long. * diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index cb15ebe82..00f254b1e 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -1322,6 +1322,9 @@ cmd_sendhex(CMD_ARGS) * Specify a body, effectively putting STRING into a DATA frame after * the HEADER frame is sent. * + * \-bodyfrom FILE (txreq, txresp) + * Same as ``-body`` but content is read from FILE. + * * \-bodylen INT (txreq, txresp) * Do the same thing as ``-body`` but generate an string of INT length * for you. @@ -1366,6 +1369,8 @@ cmd_tx11obj(CMD_ARGS) int method_done = 1; int path_done = 1; int scheme_done = 1; + int bodylen = 0; + ssize_t len; uint32_t stid = 0, pstid; uint32_t weight = 16; uint32_t exclusive = 0; @@ -1510,12 +1515,23 @@ cmd_tx11obj(CMD_ARGS) if (AV_IS("-body")) { AZ(body); REPLACE(body, av[1]); + bodylen = strlen(body); + f.flags &= ~END_STREAM; + av++; + } + else if (AV_IS("-bodyfrom")) { + AZ(body); + body = VFIL_readfile(NULL, av[1], &len); + AN(body); + assert(len < INT_MAX); + bodylen = len; f.flags &= ~END_STREAM; av++; } else if (AV_IS("-bodylen")) { AZ(body); body = synth_body(av[1], 0); + bodylen = strlen(body); f.flags &= ~END_STREAM; av++; }else if (AV_IS("-dep")) { @@ -1585,7 +1601,7 @@ cmd_tx11obj(CMD_ARGS) if (!body) return; - INIT_FRAME(f, DATA, strlen(body), s->id, END_STREAM); + INIT_FRAME(f, DATA, bodylen, s->id, END_STREAM); f.data = body; write_frame(s->hp, &f, 1); From phk at FreeBSD.org Mon Apr 29 09:28:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 29 Apr 2019 09:28:07 +0000 (UTC) Subject: [master] 3d170c0d8 Flexelinting: 'acct' is the name of a syscall Message-ID: <20190429092807.DACA79538@lists.varnish-cache.org> commit 3d170c0d8fd029eb8604854b9a758404f19008c2 Author: Poul-Henning Kamp Date: Mon Apr 29 07:06:59 2019 +0000 Flexelinting: 'acct' is the name of a syscall diff --git a/bin/varnishd/http2/cache_http2_send.c b/bin/varnishd/http2/cache_http2_send.c index 06e029e9e..d1c3573ef 100644 --- a/bin/varnishd/http2/cache_http2_send.c +++ b/bin/varnishd/http2/cache_http2_send.c @@ -262,7 +262,7 @@ h2_do_window(struct worker *wrk, struct h2_req *r2, static void h2_send(struct worker *wrk, struct h2_req *r2, h2_frame ftyp, uint8_t flags, - uint32_t len, const void *ptr, uint64_t *acct) + uint32_t len, const void *ptr, uint64_t *counter) { struct h2_sess *h2; uint32_t mfs, tf; @@ -274,7 +274,7 @@ h2_send(struct worker *wrk, struct h2_req *r2, h2_frame ftyp, uint8_t flags, h2 = r2->h2sess; CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); assert(len == 0 || ptr != NULL); - AN(acct); + AN(counter); AN(H2_SEND_HELD(h2, r2)); @@ -311,7 +311,7 @@ h2_send(struct worker *wrk, struct h2_req *r2, h2_frame ftyp, uint8_t flags, if (len <= tf) { H2_Send_Frame(wrk, h2, ftyp, flags, len, r2->stream, ptr); - *acct += len; + *counter += len; } else { AN(ptr); p = ptr; @@ -341,7 +341,7 @@ h2_send(struct worker *wrk, struct h2_req *r2, h2_frame ftyp, uint8_t flags, } p += tf; len -= tf; - *acct += tf; + *counter += tf; ftyp = ftyp->continuation; flags &= ftyp->flags; final_flags &= ftyp->flags; @@ -351,12 +351,12 @@ h2_send(struct worker *wrk, struct h2_req *r2, h2_frame ftyp, uint8_t flags, void H2_Send(struct worker *wrk, struct h2_req *r2, h2_frame ftyp, uint8_t flags, - uint32_t len, const void *ptr, uint64_t *acct) + uint32_t len, const void *ptr, uint64_t *counter) { - uint64_t dummy_acct; + uint64_t dummy_counter; - if (acct == NULL) - acct = &dummy_acct; + if (counter == NULL) + counter = &dummy_counter; - h2_send(wrk, r2, ftyp, flags, len, ptr, acct); + h2_send(wrk, r2, ftyp, flags, len, ptr, counter); } From phk at FreeBSD.org Mon Apr 29 09:28:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 29 Apr 2019 09:28:07 +0000 (UTC) Subject: [master] 4ac43023e FlexeLinting: Be more careful about signed/unsigned Message-ID: <20190429092807.EEE02953B@lists.varnish-cache.org> commit 4ac43023e66bf0392fcb0ba40693fec3f28cdbee Author: Poul-Henning Kamp Date: Mon Apr 29 07:20:10 2019 +0000 FlexeLinting: Be more careful about signed/unsigned diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c index 62d2e74f1..b72bad27f 100644 --- a/lib/libvmod_directors/shard_cfg.c +++ b/lib/libvmod_directors/shard_cfg.c @@ -370,7 +370,7 @@ shardcfg_backend_lookup(const struct backend_reconfig *re, static void shardcfg_backend_expand(const struct backend_reconfig *re) { - unsigned min = re->hint; + int min = re->hint; CHECK_OBJ_NOTNULL(re->shardd, SHARDDIR_MAGIC); @@ -380,7 +380,7 @@ shardcfg_backend_expand(const struct backend_reconfig *re) if (re->shardd->l_backend < min) re->shardd->l_backend = min; else - re->shardd->l_backend <<= 1; + re->shardd->l_backend *= 2; re->shardd->backend = realloc(re->shardd->backend, re->shardd->l_backend * sizeof *re->shardd->backend); diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c index ac870cd6b..e295be1e9 100644 --- a/lib/libvmod_directors/shard_dir.c +++ b/lib/libvmod_directors/shard_dir.c @@ -319,15 +319,15 @@ sharddir_any_healthy(VRT_CTX, struct sharddir *shardd, VCL_TIME *changed) { unsigned retval = 0; VCL_BACKEND be; - unsigned u; + int i; vtim_real c; CHECK_OBJ_NOTNULL(shardd, SHARDDIR_MAGIC); sharddir_rdlock(shardd); if (changed != NULL) *changed = 0; - for (u = 0; u < shardd->n_backend; u++) { - be = shardd->backend[u].backend; + for (i = 0; i < shardd->n_backend; i++) { + be = shardd->backend[i].backend; CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC); retval = VRT_Healthy(ctx, be, &c); if (changed != NULL && c > *changed) diff --git a/lib/libvmod_directors/shard_dir.h b/lib/libvmod_directors/shard_dir.h index 29373d0fe..bf5bb1ed4 100644 --- a/lib/libvmod_directors/shard_dir.h +++ b/lib/libvmod_directors/shard_dir.h @@ -76,8 +76,8 @@ struct sharddir { pthread_rwlock_t mtx; - unsigned n_backend; - unsigned l_backend; + int n_backend; + int l_backend; struct shard_backend *backend; const char *name; diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index c25ebb9ec..73611b95c 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -471,7 +471,7 @@ shard_blob_key(VCL_BLOB key_blob) { uint8_t k[4] = { 0 }; const uint8_t *b; - int i, ki; + size_t i, ki; AN(key_blob); AN(key_blob->blob); @@ -762,8 +762,9 @@ vmod_shard_list(VRT_CTX, VCL_BACKEND dir, struct vsb *vsb, int pflag, int jflag) VCL_DURATION rampup_d, d; VCL_BACKEND be; VCL_BOOL h; - unsigned u, nh = 0; + unsigned nh = 0; double rampup_p; + int i; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(dir, DIRECTOR_MAGIC); @@ -785,8 +786,8 @@ vmod_shard_list(VRT_CTX, VCL_BACKEND dir, struct vsb *vsb, int pflag, int jflag) } sharddir_rdlock(shardd); - for (u = 0; u < shardd->n_backend; u++) { - sbe = &shardd->backend[u]; + for (i = 0; i < shardd->n_backend; i++) { + sbe = &shardd->backend[i]; AN(sbe); be = sbe->backend; CHECK_OBJ_NOTNULL(be, DIRECTOR_MAGIC); @@ -801,7 +802,7 @@ vmod_shard_list(VRT_CTX, VCL_BACKEND dir, struct vsb *vsb, int pflag, int jflag) continue; d = ctx->now - c; - rampup_d = shardcfg_get_rampup(shardd, u); + rampup_d = shardcfg_get_rampup(shardd, i); if (! h) { rampup_p = 0.0; rampup_d = 0.0; @@ -814,7 +815,7 @@ vmod_shard_list(VRT_CTX, VCL_BACKEND dir, struct vsb *vsb, int pflag, int jflag) } if (jflag) { - if (u) + if (i) VSB_cat(vsb, ",\n"); VSB_printf(vsb, "\"%s\": {\n", be->vcl_name); @@ -852,10 +853,10 @@ vmod_shard_list(VRT_CTX, VCL_BACKEND dir, struct vsb *vsb, int pflag, int jflag) return; if (jflag) - VSB_printf(vsb, "[%u, %u, \"%s\"]", nh, u, + VSB_printf(vsb, "[%u, %d, \"%s\"]", nh, i, nh ? "healthy" : "sick"); else - VSB_printf(vsb, "%u/%u\t%s", nh, u, nh ? "healthy" : "sick"); + VSB_printf(vsb, "%u/%d\t%s", nh, i, nh ? "healthy" : "sick"); } VCL_VOID v_matchproto_(td_directors_shard_backend) From phk at FreeBSD.org Mon Apr 29 09:28:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 29 Apr 2019 09:28:08 +0000 (UTC) Subject: [master] d3c780c5f Flexelinting: int/unsigned consistency Message-ID: <20190429092808.1627C953E@lists.varnish-cache.org> commit d3c780c5ff5cb4366e414179827db32155eb0ca2 Author: Poul-Henning Kamp Date: Mon Apr 29 07:23:20 2019 +0000 Flexelinting: int/unsigned consistency diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 9289a60ba..a0641897b 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -520,7 +520,7 @@ pan_req(struct vsb *vsb, const struct req *req) "err_code = %d, err_reason = %s,\n", req->err_code, req->err_reason ? req->err_reason : "(null)"); - VSB_printf(vsb, "restarts = %d, esi_level = %d,\n", + VSB_printf(vsb, "restarts = %d, esi_level = %u,\n", req->restarts, req->esi_level); if (req->sp != NULL) diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 8f2db0651..d6b02a0e9 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -651,14 +651,14 @@ struct VCL_conf { unsigned syntax; VCL_BACKEND *default_director; VCL_PROBE default_probe; - unsigned nref; + int nref; const struct vrt_ref *ref; - unsigned nsrc; + int nsrc; const char **srcname; const char **srcbody; - unsigned nvmod; + int nvmod; vcl_event_f *event_vcl; """) From phk at FreeBSD.org Mon Apr 29 09:28:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 29 Apr 2019 09:28:08 +0000 (UTC) Subject: [master] a1a752407 Handle EWOULDBLOCK as a short write Message-ID: <20190429092808.312A29543@lists.varnish-cache.org> commit a1a7524076e8be742fc5e8272561f39eda9d492a Author: Poul-Henning Kamp Date: Mon Apr 29 09:26:27 2019 +0000 Handle EWOULDBLOCK as a short write diff --git a/bin/varnishd/http1/cache_http1_line.c b/bin/varnishd/http1/cache_http1_line.c index 6a471b043..ae924d66e 100644 --- a/bin/varnishd/http1/cache_http1_line.c +++ b/bin/varnishd/http1/cache_http1_line.c @@ -201,7 +201,7 @@ V1L_Flush(const struct worker *wrk) i = writev(*v1l->wfd, v1l->iov, v1l->niov); if (i > 0) v1l->cnt += i; - while (i != v1l->liov && i > 0) { + while (i != v1l->liov && (i > 0 || errno == EWOULDBLOCK)) { /* Remove sent data from start of I/O vector, * then retry; we hit a timeout, but some data * was sent. @@ -223,7 +223,8 @@ V1L_Flush(const struct worker *wrk) "Hit idle send timeout, wrote = %zd/%zd; retrying", i, v1l->liov); - v1l_prune(v1l, i); + if (i > 0) + v1l_prune(v1l, i); i = writev(*v1l->wfd, v1l->iov, v1l->niov); if (i > 0) v1l->cnt += i; From phk at FreeBSD.org Mon Apr 29 09:28:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 29 Apr 2019 09:28:08 +0000 (UTC) Subject: [master] 21cc7fc5d Make this test work on FreeBSD. Message-ID: <20190429092808.4B5759548@lists.varnish-cache.org> commit 21cc7fc5d259266ce3c42583d32a74846606608f Author: Poul-Henning Kamp Date: Mon Apr 29 09:26:49 2019 +0000 Make this test work on FreeBSD. diff --git a/bin/varnishtest/tests/s00010.vtc b/bin/varnishtest/tests/s00010.vtc index 9097e4fd9..6c2a5be37 100644 --- a/bin/varnishtest/tests/s00010.vtc +++ b/bin/varnishtest/tests/s00010.vtc @@ -1,17 +1,20 @@ varnishtest "client h1 send timeouts" # XXX See https://github.com/varnishcache/varnish-cache/pull/2980#issuecomment-486214661 -feature cmd {test $(uname) = "Linux"} +feature cmd {test $(uname) != "SunOS"} server s1 { rxreq txresp -bodylen 100000 } -start -varnish v1 -arg "-p timeout_idle=1" \ - -arg "-a 127.0.0.1:0" \ - -arg "-a ${tmpdir}/v1.sock" \ - -vcl+backend { +varnish v1 \ + -arg "-p timeout_idle=1" \ + -arg "-p idle_send_timeout=.1" \ + -arg "-p send_timeout=.1" \ + -arg "-a 127.0.0.1:0" \ + -arg "-a ${tmpdir}/v1.sock" \ + -vcl+backend { import debug; sub vcl_deliver { @@ -19,8 +22,6 @@ varnish v1 -arg "-p timeout_idle=1" \ } } -start -varnish v1 -cliok "param.set send_timeout 1" - logexpect l1 -v v1 { expect * * Debug "Hit total send timeout" expect * * Debug "Hit total send timeout" From phk at FreeBSD.org Mon Apr 29 10:25:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 29 Apr 2019 10:25:08 +0000 (UTC) Subject: [master] 89d330190 Make truncated HTTP headers failure by default subject to "non_fatal" Message-ID: <20190429102508.ABDE360F29@lists.varnish-cache.org> commit 89d33019022a8ed81a9cbe5aef86b5ea8ac17c0f Author: Poul-Henning Kamp Date: Mon Apr 29 10:20:22 2019 +0000 Make truncated HTTP headers failure by default subject to "non_fatal" diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index d4e03999f..6e249ae24 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -467,11 +467,15 @@ http_splitheader(struct http *hp, int req) hh[n++] = p++; while (*p != '\0' && !vct_iscrlf(p)) p++; + if (*p == '\0') { + break; + } q = p; p += vct_skipcrlf(p); *q = '\0'; } - p += vct_skipcrlf(p); + if (*p != '\0') + p += vct_skipcrlf(p); assert(*p == '\0'); for (n = 0; n < 3 || hh[n] != NULL; n++) { @@ -670,6 +674,9 @@ http_rxhdr(struct http *hp) l = hp->rx_p - hp->rx_b; vtc_dump(hp->vl, 4, "rxhdr", hp->rx_b, l); vtc_log(hp->vl, 4, "rxhdrlen = %zd", l); + if (i < 1) + vtc_log(hp->vl, hp->fatal, "HTTP header is incomplete"); + *hp->rx_p = '\0'; hp->body = hp->rx_p; } From phk at FreeBSD.org Mon Apr 29 10:25:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 29 Apr 2019 10:25:08 +0000 (UTC) Subject: [master] 58167d61d Clients may get short HTTP headers Message-ID: <20190429102508.C00D460F2C@lists.varnish-cache.org> commit 58167d61dd3fd660717ec566406a95223fbdc35b Author: Poul-Henning Kamp Date: Mon Apr 29 10:20:45 2019 +0000 Clients may get short HTTP headers diff --git a/bin/varnishtest/tests/s00010.vtc b/bin/varnishtest/tests/s00010.vtc index 6c2a5be37..81dbfee00 100644 --- a/bin/varnishtest/tests/s00010.vtc +++ b/bin/varnishtest/tests/s00010.vtc @@ -29,6 +29,7 @@ logexpect l1 -v v1 { client c1 -rcvbuf 128 { txreq + non_fatal rxresphdrs # keep the session open for 2 seconds delay 2 @@ -36,6 +37,7 @@ client c1 -rcvbuf 128 { client c1u -connect "${tmpdir}/v1.sock" -rcvbuf 128 { txreq + non_fatal rxresphdrs # keep the session open for 2 seconds delay 2 From nils.goroll at uplex.de Mon Apr 29 12:47:07 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 29 Apr 2019 12:47:07 +0000 (UTC) Subject: [master] d01633f61 stabilize as suggested by @Dridi Message-ID: <20190429124707.5415C640BD@lists.varnish-cache.org> commit d01633f6188a7bd3d03108ac79903829899e9675 Author: Nils Goroll Date: Mon Apr 29 14:46:07 2019 +0200 stabilize as suggested by @Dridi diff --git a/bin/varnishtest/tests/t02004.vtc b/bin/varnishtest/tests/t02004.vtc index cca1fafa9..709fff70c 100644 --- a/bin/varnishtest/tests/t02004.vtc +++ b/bin/varnishtest/tests/t02004.vtc @@ -24,6 +24,8 @@ client c1 { expect_close } -run +delay 2 + varnish v1 -cliok "panic.clear" varnish v1 -expectexit 0x20 From dridi.boukelmoune at gmail.com Mon Apr 29 15:30:12 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 29 Apr 2019 15:30:12 +0000 (UTC) Subject: [master] 51a54fbd3 Backend send_timeout coverage Message-ID: <20190429153012.1A4FC91A4D@lists.varnish-cache.org> commit 51a54fbd344bb2fb2ca08c945b0331873b8cd799 Author: Dridi Boukelmoune Date: Mon Apr 29 17:22:21 2019 +0200 Backend send_timeout coverage Despite v1l code being used for both client responses and backend requests, the send timeouts only kicks in on the client side. The backend code does not enter the send loop subject to the timeout for lack of niov at V1L_Open() time. Refs 43bbe97b38a86d706c0d9f4838bbfbd305342775 diff --git a/bin/varnishtest/tests/s00011.vtc b/bin/varnishtest/tests/s00011.vtc new file mode 100644 index 000000000..50e6e4612 --- /dev/null +++ b/bin/varnishtest/tests/s00011.vtc @@ -0,0 +1,25 @@ +varnishtest "backend send timeouts" + +server s1 { + rxreq + expect req.method == POST + expect req.body == helloworld + txresp +} -start + +varnish v1 -vcl+backend "" -start + +# Despite shared code, these parameters are only for client responses. +# See 43bbe97b38a86d706c0d9f4838bbfbd305342775. + +varnish v1 -cliok "param.set idle_send_timeout 1" +varnish v1 -cliok "param.set send_timeout 1" + +client c1 { + txreq -method POST -hdr "Content-Length: 10" + send hello + delay 2 + send world + rxresp + expect resp.status == 200 +} -run From phk at FreeBSD.org Mon Apr 29 21:31:12 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 29 Apr 2019 21:31:12 +0000 (UTC) Subject: [master] d5d39f95b Add an assert Message-ID: <20190429213112.22EAAA02BF@lists.varnish-cache.org> commit d5d39f95b4c7a5e21927bb75f6ee442b22bd09cb Author: Poul-Henning Kamp Date: Mon Apr 29 10:31:57 2019 +0000 Add an assert diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index f49f955dc..9b1b27632 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -1518,6 +1518,7 @@ cmd_tx11obj(CMD_ARGS) if (AV_IS("-body")) { AZ(body); REPLACE(body, av[1]); + AN(body); bodylen = strlen(body); f.flags &= ~END_STREAM; av++; From phk at FreeBSD.org Mon Apr 29 21:31:12 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 29 Apr 2019 21:31:12 +0000 (UTC) Subject: [master] df178b49b Flexelinting Message-ID: <20190429213112.32328A02C2@lists.varnish-cache.org> commit df178b49b373fdf89dc766fddf44e5759de13eaa Author: Poul-Henning Kamp Date: Mon Apr 29 21:24:58 2019 +0000 Flexelinting diff --git a/lib/libvmod_vtc/vmod_vtc.c b/lib/libvmod_vtc/vmod_vtc.c index ceaf76ac7..c67b115eb 100644 --- a/lib/libvmod_vtc/vmod_vtc.c +++ b/lib/libvmod_vtc/vmod_vtc.c @@ -214,10 +214,9 @@ vmod_workspace_dump(VRT_CTX, VCL_ENUM which, VCL_ENUM where, VCL_BYTES off, VCL_BYTES len) { struct ws *ws; - const size_t maxlen = 1024; + VCL_BYTES l, maxlen = 1024; unsigned char buf[maxlen]; const char *p; - unsigned l; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); @@ -227,7 +226,8 @@ vmod_workspace_dump(VRT_CTX, VCL_ENUM which, VCL_ENUM where, WS_Assert(ws); if (len > maxlen) { - VRT_fail(ctx, "workspace_dump: max length is %zd", maxlen); + VRT_fail(ctx, "workspace_dump: max length is %jd", + (intmax_t)maxlen); return (NULL); } From phk at FreeBSD.org Mon Apr 29 21:31:12 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 29 Apr 2019 21:31:12 +0000 (UTC) Subject: [master] cc4455d01 Move VPI stuff out of VRT Message-ID: <20190429213112.5875CA02C7@lists.varnish-cache.org> commit cc4455d01960671f5aa5dec3a8d72370e5084cf9 Author: Poul-Henning Kamp Date: Mon Apr 29 21:30:13 2019 +0000 Move VPI stuff out of VRT diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 6b2e4753b..86ec2d460 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -44,6 +44,7 @@ varnishd_SOURCES = \ cache/cache_tcp_pool.c \ cache/cache_vary.c \ cache/cache_vcl.c \ + cache/cache_vpi.c \ cache/cache_vrt.c \ cache/cache_vrt_filter.c \ cache/cache_vrt_priv.c \ diff --git a/bin/varnishd/cache/cache_vpi.c b/bin/varnishd/cache/cache_vpi.c new file mode 100644 index 000000000..ec35ad2c7 --- /dev/null +++ b/bin/varnishd/cache/cache_vpi.c @@ -0,0 +1,108 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2019 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include "config.h" + +#include "cache_varnishd.h" + +#include "vcl.h" + +#include "vcc_interface.h" + +#include "cache_vcl.h" + +/*-------------------------------------------------------------------- + * Private & exclusive interfaces between VCC and varnishd + */ + +void +VPI_count(VRT_CTX, unsigned u) +{ + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); + CHECK_OBJ_NOTNULL(ctx->vcl->conf, VCL_CONF_MAGIC); + assert(u < ctx->vcl->conf->nref); + if (ctx->vsl != NULL) + VSLb(ctx->vsl, SLT_VCL_trace, "%s %u %u.%u.%u", + ctx->vcl->loaded_name, u, ctx->vcl->conf->ref[u].source, + ctx->vcl->conf->ref[u].line, ctx->vcl->conf->ref[u].pos); + else + VSL(SLT_VCL_trace, 0, "%s %u %u.%u.%u", + ctx->vcl->loaded_name, u, ctx->vcl->conf->ref[u].source, + ctx->vcl->conf->ref[u].line, ctx->vcl->conf->ref[u].pos); +} + +VCL_VCL +VPI_vcl_get(VRT_CTX, const char *name) +{ + VCL_VCL vcl; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + vcl = vcl_find(name); + AN(vcl); + Lck_Lock(&vcl_mtx); + vcl->nrefs++; + Lck_Unlock(&vcl_mtx); + return (vcl); +} + +void +VPI_vcl_rel(VRT_CTX, VCL_VCL vcl) +{ + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + AN(vcl); + Lck_Lock(&vcl_mtx); + vcl->nrefs--; + Lck_Unlock(&vcl_mtx); +} + +void +VPI_vcl_select(VRT_CTX, VCL_VCL vcl) +{ + struct req *req = ctx->req; + + CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); + + if (IS_TOPREQ(req) && req->vcl0 != NULL) + return; // Illega, req-FSM will fail this later. + + VCL_TaskLeave(req->vcl, req->privs); + if (IS_TOPREQ(req)) { + req->vcl0 = req->vcl; + req->vcl = NULL; + } else { + VCL_Rel(&req->vcl); + } + vcl_get(&req->vcl, vcl); + VSLb(ctx->req->vsl, SLT_VCL_use, "%s via %s", + req->vcl->loaded_name, vcl->loaded_name); + VCL_TaskEnter(req->vcl, req->privs); +} diff --git a/bin/varnishd/cache/cache_vrt_vcl.c b/bin/varnishd/cache/cache_vrt_vcl.c index 12f7843cf..30fd2a26a 100644 --- a/bin/varnishd/cache/cache_vrt_vcl.c +++ b/bin/varnishd/cache/cache_vrt_vcl.c @@ -38,8 +38,6 @@ #include "vcl.h" #include "vtim.h" -#include "vcc_interface.h" - #include "cache_director.h" #include "cache_vcl.h" @@ -324,75 +322,6 @@ VCL_DefaultProbe(const struct vcl *vcl) return (vcl->conf->default_probe); } -/*-------------------------------------------------------------------- - * VRT apis relating to VCL's as VCLS. - */ - -void -VRT_count(VRT_CTX, unsigned u) -{ - - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - CHECK_OBJ_NOTNULL(ctx->vcl, VCL_MAGIC); - CHECK_OBJ_NOTNULL(ctx->vcl->conf, VCL_CONF_MAGIC); - assert(u < ctx->vcl->conf->nref); - if (ctx->vsl != NULL) - VSLb(ctx->vsl, SLT_VCL_trace, "%s %u %u.%u.%u", - ctx->vcl->loaded_name, u, ctx->vcl->conf->ref[u].source, - ctx->vcl->conf->ref[u].line, ctx->vcl->conf->ref[u].pos); - else - VSL(SLT_VCL_trace, 0, "%s %u %u.%u.%u", - ctx->vcl->loaded_name, u, ctx->vcl->conf->ref[u].source, - ctx->vcl->conf->ref[u].line, ctx->vcl->conf->ref[u].pos); -} - -VCL_VCL -VPI_vcl_get(VRT_CTX, const char *name) -{ - VCL_VCL vcl; - - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - vcl = vcl_find(name); - AN(vcl); - Lck_Lock(&vcl_mtx); - vcl->nrefs++; - Lck_Unlock(&vcl_mtx); - return (vcl); -} - -void -VPI_vcl_rel(VRT_CTX, VCL_VCL vcl) -{ - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); - AN(vcl); - Lck_Lock(&vcl_mtx); - vcl->nrefs--; - Lck_Unlock(&vcl_mtx); -} - -void -VPI_vcl_select(VRT_CTX, VCL_VCL vcl) -{ - struct req *req = ctx->req; - - CHECK_OBJ_NOTNULL(vcl, VCL_MAGIC); - - if (IS_TOPREQ(req) && req->vcl0 != NULL) - return; // Illega, req-FSM will fail this later. - - VCL_TaskLeave(req->vcl, req->privs); - if (IS_TOPREQ(req)) { - req->vcl0 = req->vcl; - req->vcl = NULL; - } else { - VCL_Rel(&req->vcl); - } - vcl_get(&req->vcl, vcl); - VSLb(ctx->req->vsl, SLT_VCL_use, "%s via %s", - req->vcl->loaded_name, vcl->loaded_name); - VCL_TaskEnter(req->vcl, req->privs); -} - struct vclref * VRT_ref_vcl(VRT_CTX, const char *desc) { diff --git a/include/vcc_interface.h b/include/vcc_interface.h index f86157555..cf1845a8d 100644 --- a/include/vcc_interface.h +++ b/include/vcc_interface.h @@ -36,3 +36,17 @@ VCL_VCL VPI_vcl_get(VRT_CTX, const char *); void VPI_vcl_rel(VRT_CTX, VCL_VCL); void VPI_vcl_select(VRT_CTX, VCL_VCL); + +/*********************************************************************** + * VPI_count() refers to this structure for coordinates into the VCL source. + */ + +struct vpi_ref { + unsigned source; + unsigned offset; + unsigned line; + unsigned pos; + const char *token; +}; + +void VPI_count(VRT_CTX, unsigned); diff --git a/include/vrt.h b/include/vrt.h index cdf7bd12f..f83218efa 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -376,19 +376,6 @@ struct vrt_backend_probe { VRT_BACKEND_PROBE_FIELDS(const) }; -/*********************************************************************** - * VRT_count() refers to this structure for coordinates into the VCL source. - */ - -struct vrt_ref { - unsigned source; - unsigned offset; - unsigned line; - unsigned pos; - const char *token; -}; - -void VRT_count(VRT_CTX, unsigned); /*********************************************************************** * Implementation details of ACLs diff --git a/lib/libvcc/Makefile.am b/lib/libvcc/Makefile.am index 2b141b542..be4d4c66d 100644 --- a/lib/libvcc/Makefile.am +++ b/lib/libvcc/Makefile.am @@ -42,8 +42,10 @@ dist_pkgdata_SCRIPTS = \ ## keep in sync with include/Makefile.am vcc_obj.c: \ $(top_srcdir)/lib/libvcc/generate.py \ + $(top_srcdir)/include/vcc_interface.h \ $(top_srcdir)/include/vdef.h \ - $(top_srcdir)/include/vrt.h + $(top_srcdir)/include/vrt.h \ + $(top_srcdir)/include/vrt_obj.h mkdir -p $(top_builddir)/include/tbl @PYTHON@ $(top_srcdir)/lib/libvcc/generate.py \ $(top_srcdir) $(top_builddir) diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index d6b02a0e9..8da81c378 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -651,8 +651,8 @@ struct VCL_conf { unsigned syntax; VCL_BACKEND *default_director; VCL_PROBE default_probe; - int nref; - const struct vrt_ref *ref; + unsigned nref; + const struct vpi_ref *ref; int nsrc; const char **srcname; diff --git a/lib/libvcc/vcc_compile.c b/lib/libvcc/vcc_compile.c index 6f2e38adc..6b0ac88a6 100644 --- a/lib/libvcc/vcc_compile.c +++ b/lib/libvcc/vcc_compile.c @@ -243,7 +243,7 @@ EmitCoordinates(const struct vcc *tl, struct vsb *vsb) VSB_printf(vsb, "\n#define VGC_NREFS %u\n\n", tl->cnt + 1); - VSB_printf(vsb, "static const struct vrt_ref VGC_ref[VGC_NREFS] = {\n"); + VSB_printf(vsb, "static const struct vpi_ref VGC_ref[VGC_NREFS] = {\n"); lin = 1; pos = 0; sp = 0; diff --git a/lib/libvcc/vcc_parse.c b/lib/libvcc/vcc_parse.c index f47991748..1e0bbd355 100644 --- a/lib/libvcc/vcc_parse.c +++ b/lib/libvcc/vcc_parse.c @@ -46,7 +46,7 @@ static void vcc_Compound(struct vcc *tl); } while (0) #define C(tl, sep) do { \ - Fb(tl, 1, "VRT_count(ctx, %u)%s\n", ++tl->cnt, sep); \ + Fb(tl, 1, "VPI_count(ctx, %u)%s\n", ++tl->cnt, sep); \ tl->t->cnt = tl->cnt; \ } while (0) From phk at FreeBSD.org Tue Apr 30 05:18:07 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 30 Apr 2019 05:18:07 +0000 (UTC) Subject: [master] 1dee0bd19 Fix a mistake in keeping the list of VSCs up to date. Message-ID: <20190430051807.B503AAC5E8@lists.varnish-cache.org> commit 1dee0bd19e5a949740e8b18dc1b0723080dd0847 Author: Poul-Henning Kamp Date: Tue Apr 30 05:16:13 2019 +0000 Fix a mistake in keeping the list of VSCs up to date. Fixes: 2576 diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 0c243fecd..20fb86a3b 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -282,14 +282,15 @@ vsc_fill_point(const struct vsc *vsc, const struct vsc_seg *seg, } static void -vsc_del_seg(const struct vsc *vsc, struct vsm *vsm, struct vsc_seg *sp) +vsc_del_seg(const struct vsc *vsc, struct vsm *vsm, struct vsc_seg **spp) { unsigned u; struct vsc_pt *pp; + struct vsc_seg *sp; CHECK_OBJ_NOTNULL(vsc, VSC_MAGIC); AN(vsm); - CHECK_OBJ_NOTNULL(sp, VSC_SEG_MAGIC); + TAKE_OBJ_NOTNULL(sp, spp, VSC_SEG_MAGIC); AZ(VSM_Unmap(vsm, sp->fantom)); if (sp->vj != NULL) { vjsn_delete(&sp->vj); @@ -425,7 +426,6 @@ VSC_Iter(struct vsc *vsc, struct vsm *vsm, VSC_iter_f *fiter, void *priv) if (strcmp(ifantom.class, VSC_CLASS) && strcmp(ifantom.class, VSC_DOC_CLASS)) continue; - sp2 = sp; while (sp != NULL && (strcmp(ifantom.ident, sp->fantom->ident) || VSM_StillValid(vsm, sp->fantom) != VSM_valid)) { @@ -433,22 +433,21 @@ VSC_Iter(struct vsc *vsc, struct vsm *vsm, VSC_iter_f *fiter, void *priv) sp = VTAILQ_NEXT(sp, list); VTAILQ_REMOVE(&vsc->segs, sp2, list); vsc_expose(vsc, sp2, 1); - vsc_del_seg(vsc, vsm, sp2); + vsc_del_seg(vsc, vsm, &sp2); } if (sp == NULL) { sp = vsc_add_seg(vsc, vsm, &ifantom); if (sp != NULL) { VTAILQ_INSERT_TAIL(&vsc->segs, sp, list); - sp2 = NULL; vsc_expose(vsc, sp, 0); } } else { vsc_expose(vsc, sp, 0); - sp2 = VTAILQ_NEXT(sp, list); } - if (sp != NULL && fiter != NULL && sp->head->ready < 2) + if (sp != NULL && fiter != NULL && sp->head->ready < 2) { i = vsc_iter_seg(vsc, sp, fiter, priv); - sp = sp2; + sp = VTAILQ_NEXT(sp, list); + } if (i) break; } @@ -457,7 +456,7 @@ VSC_Iter(struct vsc *vsc, struct vsm *vsm, VSC_iter_f *fiter, void *priv) sp = VTAILQ_NEXT(sp, list); VTAILQ_REMOVE(&vsc->segs, sp2, list); vsc_expose(vsc, sp2, 1); - vsc_del_seg(vsc, vsm, sp2); + vsc_del_seg(vsc, vsm, &sp2); } return (i); } @@ -533,7 +532,7 @@ VSC_Destroy(struct vsc **vscp, struct vsm *vsm) VTAILQ_FOREACH_SAFE(sp, &vsc->segs, list, sp2) { VTAILQ_REMOVE(&vsc->segs, sp, list); vsc_expose(vsc, sp, 1); - vsc_del_seg(vsc, vsm, sp); + vsc_del_seg(vsc, vsm, &sp); } FREE_OBJ(vsc); } From phk at FreeBSD.org Tue Apr 30 06:26:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 30 Apr 2019 06:26:08 +0000 (UTC) Subject: [master] 679c64d06 Signed/unsigned Flexelinting Message-ID: <20190430062608.5EC44ADB29@lists.varnish-cache.org> commit 679c64d067b10fc2c6490e6bcde256881bb08bae Author: Poul-Henning Kamp Date: Tue Apr 30 05:46:17 2019 +0000 Signed/unsigned Flexelinting diff --git a/bin/varnishtest/hpack.h b/bin/varnishtest/hpack.h index a443ee4af..b1ad007bb 100644 --- a/bin/varnishtest/hpack.h +++ b/bin/varnishtest/hpack.h @@ -52,7 +52,7 @@ struct hpk_hdr { struct txt key; struct txt value; enum hpk_indexed t; - int i; + unsigned i; }; struct hpk_ctx; diff --git a/bin/varnishtest/huffman_gen.py b/bin/varnishtest/huffman_gen.py index dd4f8e1e8..1320424d2 100755 --- a/bin/varnishtest/huffman_gen.py +++ b/bin/varnishtest/huffman_gen.py @@ -70,7 +70,7 @@ struct ssym { }; struct stbl { - int msk; + unsigned msk; struct ssym *syms; }; ''') diff --git a/bin/varnishtest/tests/c00094.vtc b/bin/varnishtest/tests/c00094.vtc index ad7a06a4e..d8c7eef72 100644 --- a/bin/varnishtest/tests/c00094.vtc +++ b/bin/varnishtest/tests/c00094.vtc @@ -37,6 +37,8 @@ server s1 -listen "${tmpdir}/s1.sock" { barrier b1 sync } -start +varnish v1 -cliok "param.set debug +syncvsl" + varnish v1 -vcl { backend foo { diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index 9d19f09aa..50f5e8f1a 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -104,7 +104,7 @@ void vtc_fatal(struct vtclog *vl, const char *, ...) v_noreturn_ v_printflike_(2,3); void vtc_dump(struct vtclog *vl, int lvl, const char *pfx, const char *str, int len); -void vtc_hexdump(struct vtclog *, int , const char *, const void *, int ); +void vtc_hexdump(struct vtclog *, int , const char *, const void *, unsigned); int vtc_send_proxy(int fd, int version, const struct suckaddr *sac, const struct suckaddr *sas); diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index cd0063cf5..99ed617d5 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -54,9 +54,9 @@ struct barrier { pthread_mutex_t mtx; pthread_cond_t cond; - unsigned waiters; - unsigned expected; - unsigned cyclic; + int waiters; + int expected; + int cyclic; enum barrier_e type; /* fields below are only for BARRIER_SOCK */ diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c index 865b7872d..783246e3e 100644 --- a/bin/varnishtest/vtc_client.c +++ b/bin/varnishtest/vtc_client.c @@ -59,7 +59,7 @@ struct client { char *proxy_spec; int proxy_version; - unsigned repeat; + int repeat; unsigned keepalive; unsigned running; @@ -200,7 +200,7 @@ client_thread(void *priv) struct client *c; struct vtclog *vl; int fd; - unsigned u; + int i; struct vsb *vsb; const char *err; @@ -218,7 +218,7 @@ client_thread(void *priv) if (c->repeat != 1) vtc_log(vl, 2, "Started (%u iterations%s)", c->repeat, c->keepalive ? " using keepalive" : ""); - for (u = 0; u < c->repeat; u++) { + for (i = 0; i < c->repeat; i++) { char *addr = VSB_data(vsb); vtc_log(vl, 3, "Connect to %s", addr); @@ -237,7 +237,7 @@ client_thread(void *priv) fd = http_process(vl, c->spec, fd, NULL, addr, c->rcvbuf); else - while (fd >= 0 && u++ < c->repeat) + while (fd >= 0 && i++ < c->repeat) fd = http_process(vl, c->spec, fd, NULL, addr, c->rcvbuf); vtc_log(vl, 3, "closing fd %d", fd); diff --git a/bin/varnishtest/vtc_h2_hpack.c b/bin/varnishtest/vtc_h2_hpack.c index f7343cab1..a00208ad9 100644 --- a/bin/varnishtest/vtc_h2_hpack.c +++ b/bin/varnishtest/vtc_h2_hpack.c @@ -254,11 +254,14 @@ str_decode(struct hpk_iter *iter, struct txt *t) { uint32_t num; int huff; + assert(iter->buf < iter->end); + huff = (*iter->buf & 0x80); if (hpk_more != num_decode(&num, iter, 7)) return (hpk_err); - if (num > iter->end - iter->buf) + assert(iter->buf < iter->end); + if (num > (unsigned)(iter->end - iter->buf)) return (hpk_err); if (huff) { /*Huffman encoding */ t->ptr = malloc((num * 8L) / 5L + 1L); diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 9b1b27632..41baa8803 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -569,7 +569,8 @@ static void parse_settings(const struct stream *s, struct frame *f) { struct http *hp; - int i, t, v; + int t, v; + unsigned u; const char *buf; enum hpk_result r; CHECK_OBJ_NOTNULL(f, FRAME_MAGIC); @@ -580,19 +581,19 @@ parse_settings(const struct stream *s, struct frame *f) vtc_fatal(hp->vl, "Size should be a multiple of 6, but isn't (%d)", f->size); - for (i = 0; i <= SETTINGS_MAX; i++) - f->md.settings[i] = NAN; + for (u = 0; u <= SETTINGS_MAX; u++) + f->md.settings[u] = NAN; - for (i = 0; i < f->size;) { - t = vbe16dec(f->data + i); - i += 2; - v = vbe32dec(f->data + i); + for (u = 0; u < f->size;) { + t = vbe16dec(f->data + u); + u += 2; + v = vbe32dec(f->data + u); if (t <= SETTINGS_MAX) { buf = h2_settings[t]; f->md.settings[t] = v; } else buf = "unknown"; - i += 4; + u += 4; if (t == 1) { r = HPK_ResizeTbl(s->hp->encctx, v); @@ -2116,7 +2117,7 @@ cmd_rxhdrs(CMD_ARGS) char *p; int loop = 0; unsigned long int times = 1; - int rcv = 0; + unsigned rcv = 0; enum h2_type expect = TYPE_HEADERS; (void)cmd; @@ -2155,7 +2156,7 @@ cmd_rxcont(CMD_ARGS) char *p; int loop = 0; unsigned long int times = 1; - int rcv = 0; + unsigned rcv = 0; (void)cmd; (void)av; @@ -2206,7 +2207,7 @@ cmd_rxdata(CMD_ARGS) char *p; int loop = 0; unsigned long int times = 1; - int rcv = 0; + unsigned rcv = 0; (void)cmd; (void)av; @@ -2307,7 +2308,7 @@ cmd_rxpush(CMD_ARGS) char *p; int loop = 0; unsigned long int times = 1; - int rcv = 0; + unsigned rcv = 0; enum h2_type expect = TYPE_PUSH_PROMISE; (void)cmd; diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c index 60f0576b8..e9b540ee6 100644 --- a/bin/varnishtest/vtc_log.c +++ b/bin/varnishtest/vtc_log.c @@ -148,7 +148,7 @@ vtc_leadin(const struct vtclog *vl, int lvl, const char *fmt, ...) static void vtc_log_emit(const struct vtclog *vl) { - int l; + unsigned l; l = VSB_len(vl->vsb); if (l == 0) @@ -232,7 +232,7 @@ vtc_dump(struct vtclog *vl, int lvl, const char *pfx, const char *str, int len) void vtc_hexdump(struct vtclog *vl, int lvl, const char *pfx, - const void *ptr, int len) + const void *ptr, unsigned len) { int nl = 1; unsigned l; diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index 9003516c4..1a4054a4b 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -50,7 +50,7 @@ struct server { VTAILQ_ENTRY(server) list; char run; - unsigned repeat; + int repeat; unsigned keepalive; char *spec; diff --git a/bin/varnishtest/vtc_syslog.c b/bin/varnishtest/vtc_syslog.c index 419553930..b9862eb50 100644 --- a/bin/varnishtest/vtc_syslog.c +++ b/bin/varnishtest/vtc_syslog.c @@ -51,7 +51,7 @@ struct syslog_srv { VTAILQ_ENTRY(syslog_srv) list; char run; - unsigned repeat; + int repeat; char *spec; int sock; From phk at FreeBSD.org Tue Apr 30 06:26:08 2019 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 30 Apr 2019 06:26:08 +0000 (UTC) Subject: [master] dddadb1ab VSC iterator, take N Message-ID: <20190430062608.8CE40ADB2C@lists.varnish-cache.org> commit dddadb1ab5ad1ead28b75e33953fefbfc4324c15 Author: Poul-Henning Kamp Date: Tue Apr 30 06:24:50 2019 +0000 VSC iterator, take N diff --git a/lib/libvarnishapi/vsc.c b/lib/libvarnishapi/vsc.c index 20fb86a3b..110256cb0 100644 --- a/lib/libvarnishapi/vsc.c +++ b/lib/libvarnishapi/vsc.c @@ -444,10 +444,12 @@ VSC_Iter(struct vsc *vsc, struct vsm *vsm, VSC_iter_f *fiter, void *priv) } else { vsc_expose(vsc, sp, 0); } - if (sp != NULL && fiter != NULL && sp->head->ready < 2) { - i = vsc_iter_seg(vsc, sp, fiter, priv); + if (sp != NULL) { + if (fiter != NULL && sp->head->ready < 2) + i = vsc_iter_seg(vsc, sp, fiter, priv); sp = VTAILQ_NEXT(sp, list); } + if (i) break; } From dridi.boukelmoune at gmail.com Tue Apr 30 07:56:07 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 30 Apr 2019 07:56:07 +0000 (UTC) Subject: [master] 3648ab627 Missing null check Message-ID: <20190430075607.E3704AF893@lists.varnish-cache.org> commit 3648ab6273687f0b9df0ea0e263c6a182370d313 Author: Dridi Boukelmoune Date: Tue Apr 30 09:46:42 2019 +0200 Missing null check Spotted by Coverity Scan. diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 41baa8803..42006ef3d 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -1518,6 +1518,7 @@ cmd_tx11obj(CMD_ARGS) else if (CMD_IS("txreq") || CMD_IS("txresp")) { if (AV_IS("-body")) { AZ(body); + AN(av[1]); REPLACE(body, av[1]); AN(body); bodylen = strlen(body); From phk at phk.freebsd.dk Tue Apr 30 08:02:55 2019 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Tue, 30 Apr 2019 08:02:55 +0000 Subject: [master] 3648ab627 Missing null check In-Reply-To: <20190430075607.E3704AF893@lists.varnish-cache.org> References: <20190430075607.E3704AF893@lists.varnish-cache.org> Message-ID: <95751.1556611375@critter.freebsd.dk> -------- In message <20190430075607.E3704AF893 at lists.varnish-cache.org>, Dridi Boukelmou ne writes: That is actually not what Coverity complained about, but I had already added the missing AN(body) before we got the report. > Missing null check > > Spotted by Coverity Scan. > >diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c >index 41baa8803..42006ef3d 100644 >--- a/bin/varnishtest/vtc_http2.c >+++ b/bin/varnishtest/vtc_http2.c >@@ -1518,6 +1518,7 @@ cmd_tx11obj(CMD_ARGS) > else if (CMD_IS("txreq") || CMD_IS("txresp")) { > if (AV_IS("-body")) { > AZ(body); >+ AN(av[1]); > REPLACE(body, av[1]); > AN(body); > bodylen = strlen(body); -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From dridi at varni.sh Tue Apr 30 08:16:51 2019 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 30 Apr 2019 10:16:51 +0200 Subject: [master] 3648ab627 Missing null check In-Reply-To: <95751.1556611375@critter.freebsd.dk> References: <20190430075607.E3704AF893@lists.varnish-cache.org> <95751.1556611375@critter.freebsd.dk> Message-ID: On Tue, Apr 30, 2019 at 10:03 AM Poul-Henning Kamp wrote: > > -------- > In message <20190430075607.E3704AF893 at lists.varnish-cache.org>, Dridi Boukelmou > ne writes: > > That is actually not what Coverity complained about, but I had already > added the missing AN(body) before we got the report. Technically it was, REPLACE() already does a null check after strdup() so the only case was a null argument. I didn't notice the AN(body) but I can revert this commit. > > Missing null check > > > > Spotted by Coverity Scan. > > > >diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c > >index 41baa8803..42006ef3d 100644 > >--- a/bin/varnishtest/vtc_http2.c > >+++ b/bin/varnishtest/vtc_http2.c > >@@ -1518,6 +1518,7 @@ cmd_tx11obj(CMD_ARGS) > > else if (CMD_IS("txreq") || CMD_IS("txresp")) { > > if (AV_IS("-body")) { > > AZ(body); > >+ AN(av[1]); > > REPLACE(body, av[1]); > > AN(body); > > bodylen = strlen(body); > > > -- > Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 > phk at FreeBSD.ORG | TCP/IP since RFC 956 > FreeBSD committer | BSD since 4.3-tahoe > Never attribute to malice what can adequately be explained by incompetence. > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi.boukelmoune at gmail.com Tue Apr 30 08:19:07 2019 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 30 Apr 2019 08:19:07 +0000 (UTC) Subject: [master] 2a24ffb4c Revert "Missing null check" Message-ID: <20190430081907.C5C40B0407@lists.varnish-cache.org> commit 2a24ffb4c0a8842f116dd8445a032e3fe67908fb Author: Dridi Boukelmoune Date: Tue Apr 30 10:17:33 2019 +0200 Revert "Missing null check" This reverts commit 3648ab6273687f0b9df0ea0e263c6a182370d313. I didn't notice the null check added after REPLACE in the mean time. diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 42006ef3d..41baa8803 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -1518,7 +1518,6 @@ cmd_tx11obj(CMD_ARGS) else if (CMD_IS("txreq") || CMD_IS("txresp")) { if (AV_IS("-body")) { AZ(body); - AN(av[1]); REPLACE(body, av[1]); AN(body); bodylen = strlen(body); From nils.goroll at uplex.de Tue Apr 30 13:23:08 2019 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 30 Apr 2019 13:23:08 +0000 (UTC) Subject: [master] f8ea52835 fix -a usage info Message-ID: <20190430132308.9E1B45BE7@lists.varnish-cache.org> commit f8ea528353a09d6f9c1753c32a0ed446d8033118 Author: Nils Goroll Date: Tue Apr 30 15:22:12 2019 +0200 fix -a usage info diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 8b1cadcb6..7f6622a01 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -91,7 +91,7 @@ usage(void) printf("\nBasic options:\n"); - printf(FMT, "-a address[:port][,proto]", + printf(FMT, "-a [=]address[:port][,proto]", "HTTP listen address and port"); printf(FMT, " [,user=][,group=]", "Can be specified multiple times.");