From phk at FreeBSD.org Mon Nov 7 09:59:14 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 7 Nov 2022 09:59:14 +0000 (UTC) Subject: [master] b110c0bf7 Track zlib in FreeBSD Message-ID: <20221107095914.AF1E8A8191@lists.varnish-cache.org> commit b110c0bf7c9e716c11310a74911ddabb12bd793b Author: Poul-Henning Kamp Date: Mon Nov 7 08:15:57 2022 +0000 Track zlib in FreeBSD diff --git a/lib/libvgz/crc32.c b/lib/libvgz/crc32.c index 3ce5afd96..0f59da66a 100644 --- a/lib/libvgz/crc32.c +++ b/lib/libvgz/crc32.c @@ -108,17 +108,14 @@ local z_crc_t x2nmodp OF((z_off64_t n, unsigned k)); #endif #if defined(W) && (!defined(ARMCRC32) || defined(DYNAMIC_CRC_TABLE)) - -local z_word_t byte_swap OF((z_word_t word)); -local z_crc_t crc_word OF((z_word_t data)); -local z_word_t crc_word_big OF((z_word_t data)); - /* Swap the bytes in a z_word_t to convert between little and big endian. Any self-respecting compiler will optimize this to a single machine byte-swap instruction, if one is available. This assumes that word_t is either 32 bits or 64 bits. */ +local z_word_t byte_swap OF((z_word_t word)); + local z_word_t byte_swap(word) z_word_t word; { @@ -719,6 +716,8 @@ unsigned long ZEXPORT crc32_z(crc, buf, len) least-significant byte of the word as the first byte of data, without any pre or post conditioning. This is used to combine the CRCs of each braid. */ +local z_crc_t crc_word OF((z_word_t data)); + local z_crc_t crc_word(data) z_word_t data; { @@ -728,6 +727,8 @@ local z_crc_t crc_word(data) return (z_crc_t)data; } +local z_word_t crc_word_big OF((z_word_t data)); + local z_word_t crc_word_big(data) z_word_t data; { From nils.goroll at uplex.de Mon Nov 7 14:57:05 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 7 Nov 2022 14:57:05 +0000 (UTC) Subject: [master] 499939c57 boc: Rename len_so_far to fetched_so_far Message-ID: <20221107145705.EC40AB12F8@lists.varnish-cache.org> commit 499939c57355942db6facd775123728aa5b98ebe Author: Alf-Andr? Walla Date: Tue Dec 7 11:24:20 2021 +0100 boc: Rename len_so_far to fetched_so_far This clarifies which length this field is about. Better diff with the --word-diff --word-diff-regex='\w+' options. Signed-off-by: Dridi Boukelmoune diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index de022c726..c46ef68b9 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -295,7 +295,7 @@ struct boc { void *stevedore_priv; enum boc_state_e state; uint8_t *vary; - uint64_t len_so_far; + uint64_t fetched_so_far; }; /* Object core structure --------------------------------------------- diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 95bbc27f0..bc46e280c 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -670,7 +670,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) if (oc->flags & OC_F_PRIVATE) AN(bo->uncacheable); - oc->boc->len_so_far = 0; + oc->boc->fetched_so_far = 0; INIT_OBJ(ctx, VRT_CTX_MAGIC); VCL_Bo2Ctx(ctx, bo); @@ -738,7 +738,7 @@ vbf_stp_fetchend(struct worker *wrk, struct busyobj *bo) give predictable backend reuse behavior for varnishtest */ vbf_cleanup(bo); - AZ(ObjSetU64(wrk, oc, OA_LEN, oc->boc->len_so_far)); + AZ(ObjSetU64(wrk, oc, OA_LEN, oc->boc->fetched_so_far)); if (bo->do_stream) assert(oc->boc->state == BOS_STREAM); @@ -898,7 +898,7 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) VSLb_ts_busyobj(bo, "Error", now); if (oc->stobj->stevedore != NULL) { - oc->boc->len_so_far = 0; + oc->boc->fetched_so_far = 0; ObjFreeObj(bo->wrk, oc); } diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 337727c99..4a77e7ffd 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -520,7 +520,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp) return (HSH_HITMISS); } oc->hits++; - boc_progress = oc->boc == NULL ? -1 : oc->boc->len_so_far; + boc_progress = oc->boc == NULL ? -1 : oc->boc->fetched_so_far; AN(hsh_deref_objhead_unlock(wrk, &oh, HSH_RUSH_POLICY)); Req_LogHit(wrk, req, oc, boc_progress); return (HSH_HIT); @@ -543,7 +543,7 @@ HSH_Lookup(struct req *req, struct objcore **ocp, struct objcore **bocp) } if (exp_oc != NULL && exp_oc->boc != NULL) - boc_progress = exp_oc->boc->len_so_far; + boc_progress = exp_oc->boc->fetched_so_far; else boc_progress = -1; diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index 24785b567..781c5c2a0 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -230,7 +230,7 @@ ObjExtend(struct worker *wrk, struct objcore *oc, ssize_t l, int final) AN(om->objextend); if (l > 0) { om->objextend(wrk, oc, l); - oc->boc->len_so_far += l; + oc->boc->fetched_so_far += l; AZ(pthread_cond_broadcast(&oc->boc->cond)); } Lck_Unlock(&oc->boc->mtx); @@ -253,13 +253,13 @@ ObjWaitExtend(const struct worker *wrk, const struct objcore *oc, uint64_t l) CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC); Lck_Lock(&oc->boc->mtx); while (1) { - rv = oc->boc->len_so_far; + rv = oc->boc->fetched_so_far; assert(l <= rv || oc->boc->state == BOS_FAILED); if (rv > l || oc->boc->state >= BOS_FINISHED) break; (void)Lck_CondWait(&oc->boc->cond, &oc->boc->mtx); } - rv = oc->boc->len_so_far; + rv = oc->boc->fetched_so_far; Lck_Unlock(&oc->boc->mtx); return (rv); } diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c index 429ef4f96..5735eed7e 100644 --- a/bin/varnishd/storage/storage_simple.c +++ b/bin/varnishd/storage/storage_simple.c @@ -324,7 +324,7 @@ sml_iterator(struct worker *wrk, struct objcore *oc, l = 0; u = 0; - if (boc->len_so_far == 0) { + if (boc->fetched_so_far == 0) { ret = func(priv, OBJ_ITER_FLUSH, NULL, 0); if (ret) return (ret); From nils.goroll at uplex.de Mon Nov 7 14:57:06 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 7 Nov 2022 14:57:06 +0000 (UTC) Subject: [master] cb4214f82 obj: Group assertions Message-ID: <20221107145706.0FDDBB12FB@lists.varnish-cache.org> commit cb4214f823ecde9354beae2dc749b4cf35937bed Author: Alf-Andr? Walla Date: Tue Dec 7 11:34:29 2021 +0100 obj: Group assertions Signed-off-by: Dridi Boukelmoune diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index 781c5c2a0..0c9e68fb3 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -223,11 +223,10 @@ ObjExtend(struct worker *wrk, struct objcore *oc, ssize_t l, int final) CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC); - (void)final; + AN(om->objextend); assert(l >= 0); Lck_Lock(&oc->boc->mtx); - AN(om->objextend); if (l > 0) { om->objextend(wrk, oc, l); oc->boc->fetched_so_far += l; From nils.goroll at uplex.de Mon Nov 7 14:57:06 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 7 Nov 2022 14:57:06 +0000 (UTC) Subject: [master] b6d1f73b2 param: New transit_buffer parameter Message-ID: <20221107145706.470EDB1300@lists.varnish-cache.org> commit b6d1f73b232133c6d4731644e6d2d606ea3a17f2 Author: Alf-Andr? Walla Date: Tue Dec 7 12:30:44 2021 +0100 param: New transit_buffer parameter It pauses the fetch progress when clients are lagging behind for uncacheable streaming deliveries. Signed-off-by: Dridi Boukelmoune diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index c46ef68b9..49ef26195 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -296,6 +296,8 @@ struct boc { enum boc_state_e state; uint8_t *vary; uint64_t fetched_so_far; + uint64_t delivered_so_far; + uint64_t transit_buffer; }; /* Object core structure --------------------------------------------- diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index bc46e280c..cf1005aa4 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -549,6 +549,8 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) bo->uncacheable = 1; wrk->vpi->handling = VCL_RET_DELIVER; } + if (!bo->uncacheable || !bo->do_stream) + oc->boc->transit_buffer = 0; if (bo->uncacheable) oc->flags |= OC_F_HFM; @@ -597,6 +599,9 @@ vbf_stp_fetchbody(struct worker *wrk, struct busyobj *bo) } AZ(vfc->failed); l = est; + oc = bo->fetch_objcore; + if (oc->boc->transit_buffer > 0) + l = vmin_t(ssize_t, l, oc->boc->transit_buffer); assert(l >= 0); if (VFP_GetStorage(vfc, &l, &ptr) != VFP_OK) { bo->htc->doclose = SC_RX_BODY; @@ -970,6 +975,8 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) return (F_STP_FAIL); } + oc->boc->transit_buffer = 0; + ll = VSB_len(synth_body); o = 0; while (ll > 0) { diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index 0c9e68fb3..b839b9e68 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -109,6 +109,7 @@ obj_newboc(void) Lck_New(&boc->mtx, lck_busyobj); AZ(pthread_cond_init(&boc->cond, NULL)); boc->refcount = 1; + boc->transit_buffer = cache_param->transit_buffer; return (boc); } @@ -216,6 +217,29 @@ ObjGetSpace(struct worker *wrk, struct objcore *oc, ssize_t *sz, uint8_t **ptr) * surplus space allocated. */ +static void +obj_extend_condwait(const struct objcore *oc) +{ + + if (oc->boc->transit_buffer == 0) + return; + + assert(oc->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP)); + /* NB: strictly signaling progress both ways would be prone to + * deadlocks, so instead we wait for signals from the client side + * when delivered_so_far so far is updated, but in case the fetch + * thread was not waiting at the time of the signal, we will see + * updates to delivered_so_far after timing out. + */ + while (!(oc->flags & OC_F_CANCEL) && oc->boc->fetched_so_far > + oc->boc->delivered_so_far + oc->boc->transit_buffer) { + (void)Lck_CondWaitTimeout(&oc->boc->cond, &oc->boc->mtx, 0.1); + /* Fallback: Check if we are alone waiting on this object */ + if (oc->refcnt == 1) + break; + } +} + void ObjExtend(struct worker *wrk, struct objcore *oc, ssize_t l, int final) { @@ -228,6 +252,7 @@ ObjExtend(struct worker *wrk, struct objcore *oc, ssize_t l, int final) Lck_Lock(&oc->boc->mtx); if (l > 0) { + obj_extend_condwait(oc); om->objextend(wrk, oc, l); oc->boc->fetched_so_far += l; AZ(pthread_cond_broadcast(&oc->boc->cond)); @@ -254,6 +279,12 @@ ObjWaitExtend(const struct worker *wrk, const struct objcore *oc, uint64_t l) while (1) { rv = oc->boc->fetched_so_far; assert(l <= rv || oc->boc->state == BOS_FAILED); + if (oc->boc->transit_buffer > 0) { + assert(oc->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP)); + /* Signal the new client position */ + oc->boc->delivered_so_far = l; + AZ(pthread_cond_signal(&oc->boc->cond)); + } if (rv > l || oc->boc->state >= BOS_FINISHED) break; (void)Lck_CondWait(&oc->boc->cond, &oc->boc->mtx); diff --git a/bin/varnishtest/tests/c00110.vtc b/bin/varnishtest/tests/c00110.vtc new file mode 100644 index 000000000..ea841c505 --- /dev/null +++ b/bin/varnishtest/tests/c00110.vtc @@ -0,0 +1,22 @@ +varnishtest "Transit buffering with early close" + +feature cmd {test $(uname) != SunOS} + +server s1 { + non_fatal + rxreq + txresp -bodylen 2000000 +} -start + +varnish v1 -cliok "param.set transit_buffer 1k" +varnish v1 -vcl+backend { } -start + +client c1 -rcvbuf 128 { + txreq -method POST + rxresphdrs + expect resp.status == 200 + recv 100 +} -run + +varnish v1 -expect VBE.vcl1.s1.conn == 0 +varnish v1 -expect VBE.vcl1.s1.busy == 0 diff --git a/bin/varnishtest/tests/c00111.vtc b/bin/varnishtest/tests/c00111.vtc new file mode 100644 index 000000000..706ee7041 --- /dev/null +++ b/bin/varnishtest/tests/c00111.vtc @@ -0,0 +1,29 @@ +varnishtest "LRU error without transit buffer" + +server s1 -repeat 2 { + non_fatal + rxreq + txresp -bodylen 1850000 +} -start + +varnish v1 -arg "-s Transient=malloc,1m" -vcl+backend { } -start + +client c1 { + non_fatal + txreq -method POST + rxresp +} -run + +varnish v1 -vsl_catchup +varnish v1 -expect fetch_failed == 1 + +varnish v1 -cliok "param.set transit_buffer 4k" + +client c2 { + txreq -method POST + rxresp +} -run + +varnish v1 -vsl_catchup +varnish v1 -expect s_fetch == 2 +varnish v1 -expect fetch_failed == 1 diff --git a/bin/varnishtest/tests/c00112.vtc b/bin/varnishtest/tests/c00112.vtc new file mode 100644 index 000000000..491daedee --- /dev/null +++ b/bin/varnishtest/tests/c00112.vtc @@ -0,0 +1,39 @@ +varnishtest "Transit buffering deadlock test" + +server s1 { + rxreq + txresp -status 404 +} -start + +server s2 { + non_fatal + rxreq + txresp -bodylen 2000000 +} -start + +varnish v1 -cliok "param.set transit_buffer 1k" +varnish v1 -vcl+backend { + sub vcl_recv { + set req.backend_hint = s2; + if (req.restarts == 1) { + set req.backend_hint = s1; + } + } + sub vcl_deliver { + if (req.restarts < 1) { + return (restart); + } + } +} -start + +client c1 { + txreq -method POST + rxresp + expect resp.bodylen == 0 + expect resp.status == 404 +} -run + +varnish v1 -expect VBE.vcl1.s1.conn == 0 +varnish v1 -expect VBE.vcl1.s1.busy == 0 +varnish v1 -expect VBE.vcl1.s2.conn == 0 +varnish v1 -expect VBE.vcl1.s2.busy == 0 diff --git a/include/tbl/params.h b/include/tbl/params.h index 1d3416daa..5ed5d7f86 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -952,6 +952,23 @@ PARAM_SIMPLE( /* flags */ EXPERIMENTAL ) +PARAM_SIMPLE( + /* name */ transit_buffer, + /* type */ bytes, + /* min */ "0k", + /* max */ NULL, + /* def */ "0k", + /* units */ "bytes", + /* descr */ + "The amount of buffer allowed through Varnish for uncacheable " + "backend streaming fetches. Zero means that no buffering is " + "done and the object is fetched as fast as possible.\n\n" + "When a client is slow, this can prevent large uncacheable objects " + "from being stored indefinitely when the intent is to simply stream " + "them to the client. As a result, a slow client transaction holds " + "onto a backend connection until the end of the delivery." +) + PARAM_SIMPLE( /* name */ vary_notice, /* type */ uint, From nils.goroll at uplex.de Mon Nov 7 14:57:06 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 7 Nov 2022 14:57:06 +0000 (UTC) Subject: [master] 530dc80fb vcl: New beresp.transit_buffer variable Message-ID: <20221107145706.67B59B1303@lists.varnish-cache.org> commit 530dc80fb22b9eeebacab45230494c8caccf9919 Author: Alf-Andr? Walla Date: Tue Dec 7 13:22:35 2021 +0100 vcl: New beresp.transit_buffer variable To override the global parameter on a per-fetch basis. Signed-off-by: Dridi Boukelmoune diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 97e74b9bf..fe4f8a81e 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -303,6 +303,36 @@ VRT_r_beresp_uncacheable(VRT_CTX) /*--------------------------------------------------------------------*/ +VCL_BYTES +VRT_r_beresp_transit_buffer(VRT_CTX) +{ + struct objcore *oc; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + + oc = ctx->bo->fetch_objcore; + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + + return oc->boc->transit_buffer; +} + +VCL_VOID +VRT_l_beresp_transit_buffer(VRT_CTX, VCL_BYTES value) +{ + struct objcore *oc; + + CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); + + oc = ctx->bo->fetch_objcore; + CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); + + oc->boc->transit_buffer = value; +} + +/*--------------------------------------------------------------------*/ + VCL_STRING VRT_r_client_identity(VRT_CTX) { diff --git a/bin/varnishtest/tests/c00112.vtc b/bin/varnishtest/tests/c00112.vtc index 491daedee..5ad76ac84 100644 --- a/bin/varnishtest/tests/c00112.vtc +++ b/bin/varnishtest/tests/c00112.vtc @@ -11,7 +11,6 @@ server s2 { txresp -bodylen 2000000 } -start -varnish v1 -cliok "param.set transit_buffer 1k" varnish v1 -vcl+backend { sub vcl_recv { set req.backend_hint = s2; @@ -19,6 +18,9 @@ varnish v1 -vcl+backend { set req.backend_hint = s1; } } + sub vcl_backend_response { + set beresp.transit_buffer = 1k; + } sub vcl_deliver { if (req.restarts < 1) { return (restart); diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index bf307edd2..361801fe0 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -1059,6 +1059,17 @@ beresp.time ``vcl_backend_response {}`` was entered, or when ``vcl_backend_error {}`` was entered. +beresp.transit_buffer + + Type: BYTES + + Readable from: vcl_backend_response + + Writable from: vcl_backend_response + + The maximum number of bytes the client can be ahead of the + backend during a streaming pass if ``beresp`` is uncacheable. + beresp.ttl From nils.goroll at uplex.de Mon Nov 7 14:58:05 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 7 Nov 2022 14:58:05 +0000 (UTC) Subject: [master] caf49d8b4 transit buffer without the timeout on the condition variable Message-ID: <20221107145805.9BE6AB191E@lists.varnish-cache.org> commit caf49d8b42b97c3599e01f2e503d24991a1a0bcf Author: Nils Goroll Date: Wed Jun 1 12:55:09 2022 +0200 transit buffer without the timeout on the condition variable We do not need to periodically check the conditions in obj_extend_condwait() if we ensure that HSH_Cancel() triggers a wakeup. Readers on private objects need to ensure they call HSH_Cancel() if they abort the read, so this is consistent with our requirements. While we could add a seperate function to the object API for the sole purpose of signaling a cancel, HSH_Cancel() already calls ObjWaitState() to syncronize with the backend thread, so adding the signal there was deemed the simpler solution. As agreed during bugwash, Martin wants to add back the timeout as an optinal parameter (default: no timeout). diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index b839b9e68..b862d2c44 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -232,12 +232,8 @@ obj_extend_condwait(const struct objcore *oc) * updates to delivered_so_far after timing out. */ while (!(oc->flags & OC_F_CANCEL) && oc->boc->fetched_so_far > - oc->boc->delivered_so_far + oc->boc->transit_buffer) { - (void)Lck_CondWaitTimeout(&oc->boc->cond, &oc->boc->mtx, 0.1); - /* Fallback: Check if we are alone waiting on this object */ - if (oc->refcnt == 1) - break; - } + oc->boc->delivered_so_far + oc->boc->transit_buffer) + (void)Lck_CondWait(&oc->boc->cond, &oc->boc->mtx); } void @@ -334,6 +330,9 @@ ObjWaitState(const struct objcore *oc, enum boc_state_e want) CHECK_OBJ_NOTNULL(oc->boc, BOC_MAGIC); Lck_Lock(&oc->boc->mtx); + /* wake up obj_extend_condwait() */ + if (oc->flags & OC_F_CANCEL) + AZ(pthread_cond_signal(&oc->boc->cond)); while (1) { if (oc->boc->state >= want) break; From martin at varnish-software.com Tue Nov 8 10:02:09 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:02:09 +0000 (UTC) Subject: [master] e40007dfc Add all well-known headers to the perfect hash lookup table Message-ID: <20221108100209.24359AE0A5@lists.varnish-cache.org> commit e40007dfc2243fb5b3be9923f1ed22dfebb90002 Author: Martin Blix Grydeland Date: Thu Sep 29 14:38:05 2022 +0200 Add all well-known headers to the perfect hash lookup table This expands the perfect hash lookup table to be able to match any entry in the list of well-known headers from tbl/http_headers.h. Previously only the headers that had a non-zero filter flag section was kept in the fast match table. Fixes: VSV00010 diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 3e78feda6..fe6fcb03d 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -70,73 +70,113 @@ static char * via_hdr; * A suitable algorithm can be found with `gperf`: * * tr '" ,' ' ' < include/tbl/http_headers.h | - * awk '$1 == "H(" && $4 != "0" {print$2}' | + * awk '$1 == "H(" {print $2}' | * gperf --ignore-case * */ +#define GPERF_MIN_WORD_LENGTH 2 +#define GPERF_MAX_WORD_LENGTH 19 +#define GPERF_MAX_HASH_VALUE 79 + static const unsigned char http_asso_values[256] = { - 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, - 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, - 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, - 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, - 39, 39, 39, 39, 39, 25, 39, 0, 20, 5, 39, 39, 39, 15, 0, 39, - 10, 39, 0, 39, 15, 10, 39, 39, 0, 39, 39, 39, 39, 39, 39, 39, - 39, 39, 39, 39, 39, 25, 39, 0, 20, 5, 39, 39, 39, 15, 0, 39, - 10, 39, 0, 39, 15, 10, 39, 39, 0, 39, 39, 39, 39, 39, 39, 39, - 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, - 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, - 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, - 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, - 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, - 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, - 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, - 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39, 39 + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 0, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 5, 80, 20, 0, 0, + 5, 10, 5, 5, 80, 0, 15, 0, 20, 80, + 40, 80, 0, 35, 10, 20, 55, 45, 0, 0, + 80, 80, 80, 80, 80, 80, 80, 5, 80, 20, + 0, 0, 5, 10, 5, 5, 80, 0, 15, 0, + 20, 80, 40, 80, 0, 35, 10, 20, 55, 45, + 0, 0, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80, 80, 80, 80, 80, + 80, 80, 80, 80, 80, 80 }; static struct http_hdrflg { char *hdr; unsigned flag; -} http_hdrflg[38 + 1] = { // MAX_HASH_VALUE - { NULL }, - { NULL }, - { H_TE }, - { H_Age }, - { NULL }, +} http_hdrflg[GPERF_MAX_HASH_VALUE + 1] = { + { NULL }, { NULL }, { NULL }, { NULL }, + { H_Date }, { H_Range }, { NULL }, - { H_Upgrade }, + { H_Referer }, + { H_Age }, + { H_From }, + { H_Keep_Alive }, + { H_Retry_After }, + { H_TE }, { H_If_Range }, - { NULL }, - { H_Connection }, - { NULL }, + { H_ETag }, + { H_X_Forwarded_For }, + { H_Expect }, { H_Trailer }, - { H_If_None_Match }, - { NULL }, - { NULL }, - { NULL }, - { H_Transfer_Encoding }, - { H_Proxy_Authenticate }, - { H_Proxy_Authorization }, - { H_Keep_Alive }, - { NULL }, - { NULL }, { H_If_Match }, - { H_HTTP2_Settings }, - { NULL }, - { NULL }, - { NULL }, - { H_Content_Range }, + { H_Host }, + { H_Accept_Language }, + { H_Accept }, + { H_If_Modified_Since }, + { H_If_None_Match }, { H_If_Unmodified_Since }, { NULL }, + { H_Cookie }, + { H_Upgrade }, + { H_Last_Modified }, + { H_Accept_Charset }, + { H_Accept_Encoding }, + { H_Content_MD5 }, + { H_Content_Type }, + { H_Content_Range }, + { NULL }, { NULL }, + { H_Content_Language }, + { H_Transfer_Encoding }, + { H_Authorization }, + { H_Content_Length }, + { H_User_Agent }, + { H_Server }, + { H_Expires }, + { H_Location }, { NULL }, - { H_If_Modified_Since }, + { H_Set_Cookie }, + { H_Content_Encoding }, + { H_Max_Forwards }, { H_Cache_Control }, { NULL }, + { H_Connection }, + { H_Pragma }, { NULL }, + { H_Accept_Ranges }, + { H_HTTP2_Settings }, + { H_Allow }, + { H_Content_Location }, { NULL }, + { H_Proxy_Authenticate }, + { H_Vary }, { NULL }, - { H_Accept_Ranges } + { H_WWW_Authenticate }, + { H_Warning }, + { H_Via }, + { NULL }, { NULL }, { NULL }, { NULL }, + { NULL }, { NULL }, { NULL }, { NULL }, + { NULL }, { NULL }, { NULL }, { NULL }, + { NULL }, { NULL }, { NULL }, + { H_Proxy_Authorization } }; static struct http_hdrflg * @@ -150,11 +190,11 @@ http_hdr_flags(const char *b, const char *e) assert(b <= e); u = (unsigned)(e - b); assert(b + u == e); - if (u < 2 || u > 19) // MIN_WORD_LENGTH & MAX_WORD_LENGTH + if (u < GPERF_MIN_WORD_LENGTH || u > GPERF_MAX_WORD_LENGTH) return (NULL); - if (u > 3) - u += http_asso_values[((const uint8_t*)b)[3]]; - if (u > 38) // MAX_HASH_VALUE + u += http_asso_values[((const uint8_t *)b)[u - 1]] + + http_asso_values[((const uint8_t *)b)[0]]; + if (u > GPERF_MAX_HASH_VALUE) return (NULL); retval = &http_hdrflg[u]; if (retval->hdr == NULL) @@ -173,11 +213,9 @@ http_init_hdr(char *hdr, int flg) hdr[0] = strlen(hdr + 1); f = http_hdr_flags(hdr + 1, hdr + hdr[0]); - if (flg) { - AN(f); - assert(f->hdr == hdr); - f->flag = flg; - } + AN(f); + assert(f->hdr == hdr); + f->flag = flg; } void diff --git a/bin/varnishtest/tests/f00010.vtc b/bin/varnishtest/tests/f00010.vtc new file mode 100644 index 000000000..b381b5cf3 --- /dev/null +++ b/bin/varnishtest/tests/f00010.vtc @@ -0,0 +1,19 @@ +varnishtest "Do not allow critical headers to be marked hop-by-hop" + +varnish v1 -vcl { + backend default none; +} -start + +client c1 { + txreq -hdr "Connection: Content-Length" -body "asdf" + rxresp + expect resp.status == 400 + expect_close +} -run + +client c2 { + txreq -hdr "Connection: Host" + rxresp + expect resp.status == 400 + expect_close +} -run From martin at varnish-software.com Tue Nov 8 10:02:09 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:02:09 +0000 (UTC) Subject: [master] 687ffb645 hpack: fix pseudo-headers handling Message-ID: <20221108100209.3D5FFAE0A8@lists.varnish-cache.org> commit 687ffb6452ba570778a83b6eb1df8ac1b31d9221 Author: Asad Sajjad Ahmed Date: Fri Sep 30 14:42:53 2022 +0200 hpack: fix pseudo-headers handling We should apply the same restrictions on the list of allowed characters inside H/2 pseudo-headers as we do for H/1. This error is translated into the headers we send to a backend over H/1. Failure to do so could permit various exploits against a backend not handling malformed H/1 requests. Signed-off-by: Asad Sajjad Ahmed diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 6e67b55c5..f58788b12 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -96,13 +96,18 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len) { /* XXX: This might belong in cache/cache_http.c */ const char *b0; + int disallow_empty; unsigned n; + char *p; + int i; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); AN(b); assert(namelen >= 2); /* 2 chars from the ': ' that we added */ assert(namelen <= len); + disallow_empty = 0; + if (len > UINT_MAX) { /* XXX: cache_param max header size */ VSLb(hp->vsl, SLT_BogoHeader, "Header too large: %.20s", b); return (H2SE_ENHANCE_YOUR_CALM); @@ -117,10 +122,24 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len) b += namelen; len -= namelen; n = HTTP_HDR_METHOD; + disallow_empty = 1; + + /* First field cannot contain SP or CTL */ + for (p = b, i = 0; i < len; p++, i++) { + if (vct_issp(*p) || vct_isctl(*p)) + return (H2SE_PROTOCOL_ERROR); + } } else if (!strncmp(b, ":path: ", namelen)) { b += namelen; len -= namelen; n = HTTP_HDR_URL; + disallow_empty = 1; + + /* Second field cannot contain LWS or CTL */ + for (p = b, i = 0; i < len; p++, i++) { + if (vct_islws(*p) || vct_isctl(*p)) + return (H2SE_PROTOCOL_ERROR); + } } else if (!strncmp(b, ":scheme: ", namelen)) { /* XXX: What to do about this one? (typically "http" or "https"). For now set it as a normal @@ -128,6 +147,15 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len) b++; len-=1; n = hp->nhd; + + for (p = b + namelen, i = 0; i < len-namelen; + p++, i++) { + if (vct_issp(*p) || vct_isctl(*p)) + return (H2SE_PROTOCOL_ERROR); + } + + if (!i) + return (H2SE_PROTOCOL_ERROR); } else if (!strncmp(b, ":authority: ", namelen)) { b+=6; len-=6; @@ -164,6 +192,13 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len) hp->hd[n].b = b; hp->hd[n].e = b + len; + if (disallow_empty && !Tlen(hp->hd[n])) { + VSLb(hp->vsl, SLT_BogoHeader, + "Empty pseudo-header %.*s", + (int)namelen, b0); + return (H2SE_PROTOCOL_ERROR); + } + return (0); } diff --git a/bin/varnishtest/tests/t02023.vtc b/bin/varnishtest/tests/t02023.vtc new file mode 100644 index 000000000..cfd843da3 --- /dev/null +++ b/bin/varnishtest/tests/t02023.vtc @@ -0,0 +1,48 @@ +varnishtest "Empty pseudo-headers" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -arg "-p feature=+http2" -vcl+backend { +} -start + +client c1 { + txreq -url "" + rxresp + expect resp.status == 400 +} -run + +client c1 { + txreq -req "" + rxresp + expect resp.status == 400 +} -run + +client c1 { + txreq -proto "" + rxresp + expect resp.status == 400 +} -run + +client c1 { + stream 1 { + txreq -url "" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -scheme "" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -req "" + rxrst + } -run +} -run diff --git a/bin/varnishtest/tests/t02024.vtc b/bin/varnishtest/tests/t02024.vtc new file mode 100644 index 000000000..0d0a1abc5 --- /dev/null +++ b/bin/varnishtest/tests/t02024.vtc @@ -0,0 +1,48 @@ +varnishtest "Garbage pseudo-headers" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -arg "-p feature=+http2" -vcl+backend { +} -start + +client c1 { + txreq -url " " + rxresp + expect resp.status == 400 +} -run + +client c1 { + txreq -req " " + rxresp + expect resp.status == 400 +} -run + +client c1 { + txreq -proto " " + rxresp + expect resp.status == 400 +} -run + +client c1 { + stream 1 { + txreq -url " " + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -scheme " " + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -req " " + rxrst + } -run +} -run From martin at varnish-software.com Tue Nov 8 10:02:09 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:02:09 +0000 (UTC) Subject: [master] c9e50df69 h2: add vtc for VSV00011 Message-ID: <20221108100209.5864FAE0AE@lists.varnish-cache.org> commit c9e50df6938e0b61d935c8b2db316a3073e958ce Author: Asad Sajjad Ahmed Date: Mon Oct 3 16:08:30 2022 +0200 h2: add vtc for VSV00011 Check for correct handling of missing pseudo-headers, and invalid characters. Signed-off-by: Asad Sajjad Ahmed diff --git a/bin/varnishtest/tests/f00011.vtc b/bin/varnishtest/tests/f00011.vtc new file mode 100644 index 000000000..810918028 --- /dev/null +++ b/bin/varnishtest/tests/f00011.vtc @@ -0,0 +1,51 @@ +varnishtest "H2: Malformed pseudo-headers" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -arg "-p feature=+http2" -vcl+backend { +} -start + +client c1 { + stream 1 { + txreq -url "" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -url " \t" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -scheme "" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -scheme " \t" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -req "" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -req " \t" + rxrst + } -run +} -run From martin at varnish-software.com Tue Nov 8 10:03:07 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:03:07 +0000 (UTC) Subject: [6.0] fb6918339 Bring the CCI setup up to speed Message-ID: <20221108100307.BD202AE5DB@lists.varnish-cache.org> commit fb6918339310e75715820ae0fb3c1591070d0079 Author: Martin Blix Grydeland Date: Wed Oct 12 11:50:06 2022 +0200 Bring the CCI setup up to speed diff --git a/.circleci/config.yml b/.circleci/config.yml index e6c8027be..c272743b7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,6 +16,9 @@ parameters: dist-url: type: string default: "" + dist-url-sha256: + type: string + default: "" jobs: dist: @@ -46,7 +49,17 @@ jobs: - run: name: Download the dist tarball command: | - curl -s << pipeline.parameters.dist-url >> -o varnish-dist.tar.gz + curl -Ls << pipeline.parameters.dist-url >> -o varnish-dist.tar.gz + - when: + condition: << pipeline.parameters.dist-url-sha256 >> + steps: + - run: + name: Verify downloaded tarball + command: | + echo "<< pipeline.parameters.dist-url-sha256 >> varnish-dist.tar.gz" | sha256sum -c + - run: + name: Rename the dist tarball by parsed version + command: | mkdir parse-version-tmp cd parse-version-tmp tar xzf ../varnish-dist.tar.gz From martin at varnish-software.com Tue Nov 8 10:03:07 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:03:07 +0000 (UTC) Subject: [6.0] 88b09ede5 remove code duplication for http header filtering Message-ID: <20221108100307.DDC40AE5DE@lists.varnish-cache.org> commit 88b09ede520f3d147315c8fcc70e2d95b66c614e Author: Nils Goroll Date: Sun Mar 22 16:32:33 2020 +0100 remove code duplication for http header filtering no semantic changes, just consolidating three places in one (thus pushed during freeze) diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 78d96293b..92043c98d 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -850,6 +850,23 @@ http_PutResponse(struct http *to, const char *proto, uint16_t status, http_SetH(to, HTTP_HDR_REASON, reason); } +/*-------------------------------------------------------------------- + * check if header is filterd by the dynamic marker or the static + * definitions in http_headers.h + */ + +static inline int +http_isfiltered(const struct http *fm, unsigned u, unsigned how) +{ + if (fm->hdf[u] & HDF_FILTER) + return (1); +#define HTTPH(a, b, c) \ + if (((c) & how) && http_IsHdr(&fm->hd[u], (b))) \ + return (1); +#include "tbl/http_headers.h" + return (0); +} + /*-------------------------------------------------------------------- * Estimate how much workspace we need to Filter this header according * to 'how'. @@ -866,12 +883,8 @@ http_EstimateWS(const struct http *fm, unsigned how) if (u == HTTP_HDR_METHOD || u == HTTP_HDR_URL) continue; Tcheck(fm->hd[u]); - if (fm->hdf[u] & HDF_FILTER) + if (http_isfiltered(fm, u, how)) continue; -#define HTTPH(a, b, c) \ - if (((c) & how) && http_IsHdr(&fm->hd[u], (b))) \ - continue; -#include "tbl/http_headers.h" l += Tlen(fm->hd[u]) + 1L; } return (PRNDUP(l + 1L)); @@ -907,12 +920,8 @@ HTTP_Encode(const struct http *fm, uint8_t *p0, unsigned l, unsigned how) if (u == HTTP_HDR_METHOD || u == HTTP_HDR_URL) continue; Tcheck(fm->hd[u]); - if (fm->hdf[u] & HDF_FILTER) - continue; -#define HTTPH(a, b, c) \ - if (((c) & how) && http_IsHdr(&fm->hd[u], (b))) \ + if (http_isfiltered(fm, u, how)) continue; -#include "tbl/http_headers.h" http_VSLH(fm, u); w = Tlen(fm->hd[u]) + 1L; assert(p + w + 1 <= e); @@ -1099,13 +1108,8 @@ http_filterfields(struct http *to, const struct http *fm, unsigned how) to->status = fm->status; for (u = HTTP_HDR_FIRST; u < fm->nhd; u++) { Tcheck(fm->hd[u]); - if (fm->hdf[u] & HDF_FILTER) + if (http_isfiltered(fm, u, how)) continue; - Tcheck(fm->hd[u]); -#define HTTPH(a, b, c) \ - if (((c) & how) && http_IsHdr(&fm->hd[u], (b))) \ - continue; -#include "tbl/http_headers.h" assert (to->nhd < to->shd); to->hd[to->nhd] = fm->hd[u]; to->hdf[to->nhd] = 0; From martin at varnish-software.com Tue Nov 8 10:03:08 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:03:08 +0000 (UTC) Subject: [6.0] 1e9fae538 http: Mark Keep-Alive as a hop-by-hop header Message-ID: <20221108100308.0CAA8AE5E2@lists.varnish-cache.org> commit 1e9fae538c8a7bba2f768076479cb6b80bd997ca Author: Mark Felder Date: Tue Sep 22 17:09:53 2020 -0500 http: Mark Keep-Alive as a hop-by-hop header diff --git a/include/tbl/http_headers.h b/include/tbl/http_headers.h index 44e7d0656..4be5b6a9d 100644 --- a/include/tbl/http_headers.h +++ b/include/tbl/http_headers.h @@ -50,7 +50,6 @@ #define S HTTPH_A_PASS #define H(s,e,f) HTTPH(s, e, f) -H("Keep-Alive", H_Keep_Alive, P|F |S) // 2068 H("Accept", H_Accept, 0 ) // 2616 14.1 H("Accept-Charset", H_Accept_Charset, 0 ) // 2616 14.2 H("Accept-Encoding", H_Accept_Encoding, 0 ) // 2616 14.3 @@ -81,6 +80,7 @@ H("If-Modified-Since", H_If_Modified_Since, F ) // 2616 14.25 H("If-None-Match", H_If_None_Match, F ) // 2616 14.26 H("If-Range", H_If_Range, F ) // 2616 14.27 H("If-Unmodified-Since",H_If_Unmodified_Since, F ) // 2616 14.28 +H("Keep-Alive", H_Keep_Alive, P|F|I|S) // 2616 13.5.1 H("Last-Modified", H_Last_Modified, 0 ) // 2616 14.29 H("Location", H_Location, 0 ) // 2616 14.30 H("Max-Forwards", H_Max_Forwards, 0 ) // 2616 14.31 From martin at varnish-software.com Tue Nov 8 10:03:08 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:03:08 +0000 (UTC) Subject: [6.0] 4e5728c5c Add coverage for Keep-Alive header removal Message-ID: <20221108100308.29789AE5E7@lists.varnish-cache.org> commit 4e5728c5cc3d4242856e76f4eec1da5a6617ee3f Author: AlveElde Date: Wed Sep 28 14:53:20 2022 +0200 Add coverage for Keep-Alive header removal diff --git a/bin/varnishtest/tests/b00080.vtc b/bin/varnishtest/tests/b00080.vtc new file mode 100644 index 000000000..d1227f1be --- /dev/null +++ b/bin/varnishtest/tests/b00080.vtc @@ -0,0 +1,77 @@ +varnishtest "Keep-Alive is a hop-by-hop header" + +server s1 { + rxreq + expect req.http.req-keep-alive == "true" + expect req.http.bereq-keep-alive == "false" + expect req.http.Keep-Alive == + txresp -hdr "Keep-Alive: 1" + + rxreq + expect req.http.req-keep-alive == "true" + expect req.http.bereq-keep-alive == "false" + expect req.http.Keep-Alive == + txresp -hdr "Connection: Keep-Alive" -hdr "Keep-Alive: 2" + + rxreq + expect req.http.req-keep-alive == "true" + expect req.http.bereq-keep-alive == "false" + expect req.http.Keep-Alive == + expect req.http.Cookie == "3" + txresp -hdr "Keep-Alive: 3" + + rxreq + expect req.http.req-keep-alive == "true" + expect req.http.bereq-keep-alive == "false" + expect req.http.Keep-Alive == + expect req.http.Cookie == "4" + txresp -hdr "Connection: Keep-Alive" -hdr "Keep-Alive: 4" +} -start + +varnish v1 -vcl+backend { + sub vcl_recv { + set req.http.req-keep-alive = !(!req.http.Keep-Alive); + } + + sub vcl_backend_fetch { + set bereq.http.bereq-keep-alive = !(!bereq.http.Keep-Alive); + } + + sub vcl_backend_response { + set beresp.http.beresp-keep-alive = !(!beresp.http.Keep-Alive); + } + + sub vcl_deliver { + set resp.http.resp-keep-alive = !(!resp.http.Keep-Alive); + } +} -start + +client c1 { + txreq -url "/1" -hdr "Keep-Alive: 1" + rxresp + expect resp.status == 200 + expect resp.http.beresp-keep-alive == "true" + expect resp.http.resp-keep-alive == "false" + expect resp.http.Keep-Alive == + + txreq -url "/2" -hdr "Connection: Keep-Alive" -hdr "Keep-Alive: 2" + rxresp + expect resp.status == 200 + expect resp.http.beresp-keep-alive == "true" + expect resp.http.resp-keep-alive == "false" + expect req.http.Keep-Alive == + + txreq -url "/3" -hdr "Keep-Alive: 3" -hdr "Cookie: 3" + rxresp + expect resp.status == 200 + expect resp.http.beresp-keep-alive == "true" + expect resp.http.resp-keep-alive == "false" + expect resp.http.Keep-Alive == + + txreq -url "/4" -hdr "Connection: Keep-Alive" -hdr "Keep-Alive: 4" -hdr "Cookie: 4" + rxresp + expect resp.status == 200 + expect resp.http.beresp-keep-alive == "true" + expect resp.http.resp-keep-alive == "false" + expect req.http.Keep-Alive == +} -run From martin at varnish-software.com Tue Nov 8 10:03:08 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:03:08 +0000 (UTC) Subject: [6.0] 396b915e1 http2: Don't send resp connection-specific headers Message-ID: <20221108100308.4D741AE5EC@lists.varnish-cache.org> commit 396b915e13ab968cb303d030b19cbe00441b062d Author: Dridi Boukelmoune Date: Tue Sep 29 12:44:00 2020 +0200 http2: Don't send resp connection-specific headers Some browsers are strict about this and simply drop responses containing such headers. Since this is not filtering a context switch between a client and a backend transaction (or cache hit) a new filtering flag is added to the HTTP headers table for connection-specific headers. This new flag cannot be compounded as HTTPH_R_FETCH|HTTPH_A_INS because the TE header is an exception and left alone, even though trailers aren't supported. Better diff with the --ignore-all-space option. We could go further and consider any client request containing one as malformed as mandated by RFC 7540. Closes #3416 diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index db5467a8d..b26a2e73a 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -640,11 +640,13 @@ int HTTP_IterHdrPack(struct worker *, struct objcore *, const char **); for ((ptr) = NULL; HTTP_IterHdrPack(wrk, oc, &(ptr));) const char *HTTP_GetHdrPack(struct worker *, struct objcore *, const char *hdr); enum sess_close http_DoConnection(struct http *hp); +int http_IsFiltered(const struct http *hp, unsigned u, unsigned how); -#define HTTPH_R_PASS (1 << 0) /* Request (c->b) in pass mode */ -#define HTTPH_R_FETCH (1 << 1) /* Request (c->b) for fetch */ -#define HTTPH_A_INS (1 << 2) /* Response (b->o) for insert */ -#define HTTPH_A_PASS (1 << 3) /* Response (b->o) for pass */ +#define HTTPH_R_PASS (1 << 0) /* Request (c->b) in pass mode */ +#define HTTPH_R_FETCH (1 << 1) /* Request (c->b) for fetch */ +#define HTTPH_A_INS (1 << 2) /* Response (b->o) for insert */ +#define HTTPH_A_PASS (1 << 3) /* Response (b->o) for pass */ +#define HTTPH_C_SPECIFIC (1 << 4) /* Connection-specific */ #define HTTPH(a, b, c) extern char b[]; #include "tbl/http_headers.h" diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 92043c98d..4c1f525aa 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -867,6 +867,13 @@ http_isfiltered(const struct http *fm, unsigned u, unsigned how) return (0); } +int +http_IsFiltered(const struct http *fm, unsigned u, unsigned how) +{ + + return (http_isfiltered(fm, u, how)); +} + /*-------------------------------------------------------------------- * Estimate how much workspace we need to Filter this header according * to 'how'. diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index ceb2b46c3..b28033df4 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -244,6 +244,9 @@ h2_build_headers(struct vsb *resp, struct req *req) r = strchr(hp->hd[u].b, ':'); AN(r); + if (http_IsFiltered(hp, u, HTTPH_C_SPECIFIC)) + continue; //rfc7540,l,2999,3006 + hps = hp_idx[tolower(*hp->hd[u].b)]; sz = 1 + r - hp->hd[u].b; assert(sz > 0); @@ -298,6 +301,8 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody) VSLb(req->vsl, SLT_RespProtocol, "HTTP/2.0"); + (void)http_DoConnection(req->resp); + if (h2_build_headers(&resp, req)) { // We ran out of workspace, return minimal 500 WS_MarkOverflow(req->ws); diff --git a/bin/varnishtest/tests/r03416.vtc b/bin/varnishtest/tests/r03416.vtc new file mode 100644 index 000000000..3d7431353 --- /dev/null +++ b/bin/varnishtest/tests/r03416.vtc @@ -0,0 +1,25 @@ +varnishtest "Filter hop-by-hop headers out of h2 responses" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -cliok "param.set feature +http2" +varnish v1 -vcl+backend { + sub vcl_deliver { + set resp.http.Keep-Alive = "timeout=5, max=1000"; + set resp.http.Connection = "other"; + set resp.http.Other = "foo"; + } +} -start + +client c1 { + stream 1 { + txreq + rxresp + expect resp.http.keep-alive == + expect resp.http.connection == + expect resp.http.other == + } -run +} -run diff --git a/include/tbl/http_headers.h b/include/tbl/http_headers.h index 4be5b6a9d..274e28672 100644 --- a/include/tbl/http_headers.h +++ b/include/tbl/http_headers.h @@ -40,7 +40,8 @@ /* Shorthand for this file only, to keep table narrow */ -#if defined(P) || defined(F) || defined(I) || defined(H) || defined(S) +#if defined(P) || defined(F) || defined(I) || defined(H) || defined(S) || \ + defined(K) #error "Macro overloading" // Trust but verify #endif @@ -48,65 +49,67 @@ #define F HTTPH_R_FETCH #define I HTTPH_A_INS #define S HTTPH_A_PASS +#define K HTTPH_C_SPECIFIC #define H(s,e,f) HTTPH(s, e, f) -H("Accept", H_Accept, 0 ) // 2616 14.1 -H("Accept-Charset", H_Accept_Charset, 0 ) // 2616 14.2 -H("Accept-Encoding", H_Accept_Encoding, 0 ) // 2616 14.3 -H("Accept-Language", H_Accept_Language, 0 ) // 2616 14.4 -H("Accept-Ranges", H_Accept_Ranges, P|F|I ) // 2616 14.5 -H("Age", H_Age, I|S) // 2616 14.6 -H("Allow", H_Allow, 0 ) // 2616 14.7 -H("Authorization", H_Authorization, 0 ) // 2616 14.8 -H("Cache-Control", H_Cache_Control, F ) // 2616 14.9 -H("Connection", H_Connection, P|F|I|S) // 2616 14.10 -H("Content-Encoding", H_Content_Encoding, 0 ) // 2616 14.11 -H("Content-Language", H_Content_Language, 0 ) // 2616 14.12 -H("Content-Length", H_Content_Length, 0 ) // 2616 14.13 -H("Content-Location", H_Content_Location, 0 ) // 2616 14.14 -H("Content-MD5", H_Content_MD5, 0 ) // 2616 14.15 -H("Content-Range", H_Content_Range, F|I ) // 2616 14.16 -H("Content-Type", H_Content_Type, 0 ) // 2616 14.17 -H("Cookie", H_Cookie, 0 ) // 6265 4.2 -H("Date", H_Date, 0 ) // 2616 14.18 -H("ETag", H_ETag, 0 ) // 2616 14.19 -H("Expect", H_Expect, 0 ) // 2616 14.20 -H("Expires", H_Expires, 0 ) // 2616 14.21 -H("From", H_From, 0 ) // 2616 14.22 -H("Host", H_Host, 0 ) // 2616 14.23 -H("HTTP2-Settings", H_HTTP2_Settings, P|F|I|S) // 7540 3.2.1 -H("If-Match", H_If_Match, F ) // 2616 14.24 -H("If-Modified-Since", H_If_Modified_Since, F ) // 2616 14.25 -H("If-None-Match", H_If_None_Match, F ) // 2616 14.26 -H("If-Range", H_If_Range, F ) // 2616 14.27 -H("If-Unmodified-Since",H_If_Unmodified_Since, F ) // 2616 14.28 -H("Keep-Alive", H_Keep_Alive, P|F|I|S) // 2616 13.5.1 -H("Last-Modified", H_Last_Modified, 0 ) // 2616 14.29 -H("Location", H_Location, 0 ) // 2616 14.30 -H("Max-Forwards", H_Max_Forwards, 0 ) // 2616 14.31 -H("Pragma", H_Pragma, 0 ) // 2616 14.32 -H("Proxy-Authenticate", H_Proxy_Authenticate, F|I ) // 2616 14.33 -H("Proxy-Authorization",H_Proxy_Authorization, F|I ) // 2616 14.34 -H("Range", H_Range, F|I ) // 2616 14.35 -H("Referer", H_Referer, 0 ) // 2616 14.36 -H("Retry-After", H_Retry_After, 0 ) // 2616 14.37 -H("Server", H_Server, 0 ) // 2616 14.38 -H("Set-Cookie", H_Set_Cookie, 0 ) // 6265 4.1 -H("TE", H_TE, P|F|I|S) // 2616 14.39 -H("Trailer", H_Trailer, P|F|I|S) // 2616 14.40 -H("Transfer-Encoding", H_Transfer_Encoding, P|F|I|S) // 2616 14.41 -H("Upgrade", H_Upgrade, P|F|I|S) // 2616 14.42 -H("User-Agent", H_User_Agent, 0 ) // 2616 14.43 -H("Vary", H_Vary, 0 ) // 2616 14.44 -H("Via", H_Via, 0 ) // 2616 14.45 -H("Warning", H_Warning, 0 ) // 2616 14.46 -H("WWW-Authenticate", H_WWW_Authenticate, 0 ) // 2616 14.47 -H("X-Forwarded-For", H_X_Forwarded_For, 0 ) // No RFC +H("Accept", H_Accept, 0 ) // 2616 14.1 +H("Accept-Charset", H_Accept_Charset, 0 ) // 2616 14.2 +H("Accept-Encoding", H_Accept_Encoding, 0 ) // 2616 14.3 +H("Accept-Language", H_Accept_Language, 0 ) // 2616 14.4 +H("Accept-Ranges", H_Accept_Ranges, P|F|I ) // 2616 14.5 +H("Age", H_Age, I|S ) // 2616 14.6 +H("Allow", H_Allow, 0 ) // 2616 14.7 +H("Authorization", H_Authorization, 0 ) // 2616 14.8 +H("Cache-Control", H_Cache_Control, F ) // 2616 14.9 +H("Connection", H_Connection, P|F|I|S|K) // 2616 14.10 +H("Content-Encoding", H_Content_Encoding, 0 ) // 2616 14.11 +H("Content-Language", H_Content_Language, 0 ) // 2616 14.12 +H("Content-Length", H_Content_Length, 0 ) // 2616 14.13 +H("Content-Location", H_Content_Location, 0 ) // 2616 14.14 +H("Content-MD5", H_Content_MD5, 0 ) // 2616 14.15 +H("Content-Range", H_Content_Range, F|I ) // 2616 14.16 +H("Content-Type", H_Content_Type, 0 ) // 2616 14.17 +H("Cookie", H_Cookie, 0 ) // 6265 4.2 +H("Date", H_Date, 0 ) // 2616 14.18 +H("ETag", H_ETag, 0 ) // 2616 14.19 +H("Expect", H_Expect, 0 ) // 2616 14.20 +H("Expires", H_Expires, 0 ) // 2616 14.21 +H("From", H_From, 0 ) // 2616 14.22 +H("Host", H_Host, 0 ) // 2616 14.23 +H("HTTP2-Settings", H_HTTP2_Settings, P|F|I|S|K) // 7540 3.2.1 +H("If-Match", H_If_Match, F ) // 2616 14.24 +H("If-Modified-Since", H_If_Modified_Since, F ) // 2616 14.25 +H("If-None-Match", H_If_None_Match, F ) // 2616 14.26 +H("If-Range", H_If_Range, F ) // 2616 14.27 +H("If-Unmodified-Since",H_If_Unmodified_Since, F ) // 2616 14.28 +H("Keep-Alive", H_Keep_Alive, P|F|I|S|K) // 2616 13.5.1 +H("Last-Modified", H_Last_Modified, 0 ) // 2616 14.29 +H("Location", H_Location, 0 ) // 2616 14.30 +H("Max-Forwards", H_Max_Forwards, 0 ) // 2616 14.31 +H("Pragma", H_Pragma, 0 ) // 2616 14.32 +H("Proxy-Authenticate", H_Proxy_Authenticate, F|I ) // 2616 14.33 +H("Proxy-Authorization",H_Proxy_Authorization, F|I ) // 2616 14.34 +H("Range", H_Range, F|I ) // 2616 14.35 +H("Referer", H_Referer, 0 ) // 2616 14.36 +H("Retry-After", H_Retry_After, 0 ) // 2616 14.37 +H("Server", H_Server, 0 ) // 2616 14.38 +H("Set-Cookie", H_Set_Cookie, 0 ) // 6265 4.1 +H("TE", H_TE, P|F|I|S ) // 2616 14.39 +H("Trailer", H_Trailer, P|F|I|S ) // 2616 14.40 +H("Transfer-Encoding", H_Transfer_Encoding, P|F|I|S|K) // 2616 14.41 +H("Upgrade", H_Upgrade, P|F|I|S|K) // 2616 14.42 +H("User-Agent", H_User_Agent, 0 ) // 2616 14.43 +H("Vary", H_Vary, 0 ) // 2616 14.44 +H("Via", H_Via, 0 ) // 2616 14.45 +H("Warning", H_Warning, 0 ) // 2616 14.46 +H("WWW-Authenticate", H_WWW_Authenticate, 0 ) // 2616 14.47 +H("X-Forwarded-For", H_X_Forwarded_For, 0 ) // No RFC #undef P #undef F #undef I #undef S +#undef K #undef H #undef HTTPH From martin at varnish-software.com Tue Nov 8 10:03:08 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:03:08 +0000 (UTC) Subject: [6.0] 6329cd86b http2: Filter out all connection-specific headers Message-ID: <20221108100308.68A79AE5F0@lists.varnish-cache.org> commit 6329cd86ba23da1afc37eaca59467d522813145b Author: AlveElde Date: Thu Sep 29 16:20:49 2022 +0200 http2: Filter out all connection-specific headers Now that http_DoConnection() is used without respecting the SC_RX_BAD return value it should not return early when encountering a well-known header. diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 4c1f525aa..ba7e16cfb 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -718,17 +718,21 @@ http_DoConnection(struct http *hp) AN(h); while (http_split(&h, NULL, ",", &b, &e)) { u = pdiff(b, e); - if (u == 5 && !strncasecmp(b, "close", u)) + if (u == 5 && retval != SC_RX_BAD && + !strncasecmp(b, "close", u)) retval = SC_REQ_CLOSE; - if (u == 10 && !strncasecmp(b, "keep-alive", u)) + if (u == 10 && retval != SC_RX_BAD && + !strncasecmp(b, "keep-alive", u)) retval = SC_NULL; /* Refuse removal of well-known-headers if they would pass. */ /*lint -save -e506 [constant value boolean] */ #define HTTPH(a, x, c) \ if (!((c) & HTTPH_R_PASS) && \ - strlen(a) == u && !strncasecmp(a, b, u)) \ - return (SC_RX_BAD); + strlen(a) == u && !strncasecmp(a, b, u)) { \ + retval = SC_RX_BAD; \ + continue; \ + } #include "tbl/http_headers.h" /*lint -restore */ diff --git a/bin/varnishtest/tests/r03416.vtc b/bin/varnishtest/tests/r03416.vtc index 3d7431353..9346d6841 100644 --- a/bin/varnishtest/tests/r03416.vtc +++ b/bin/varnishtest/tests/r03416.vtc @@ -2,15 +2,16 @@ varnishtest "Filter hop-by-hop headers out of h2 responses" server s1 { rxreq - txresp + txresp -body "water" } -start varnish v1 -cliok "param.set feature +http2" varnish v1 -vcl+backend { sub vcl_deliver { set resp.http.Keep-Alive = "timeout=5, max=1000"; - set resp.http.Connection = "other"; + set resp.http.Connection = "other, Content-Length, another"; set resp.http.Other = "foo"; + set resp.http.Another = "bar"; } } -start @@ -21,5 +22,7 @@ client c1 { expect resp.http.keep-alive == expect resp.http.connection == expect resp.http.other == + expect resp.http.another == + expect resp.http.Content-Length == 5 } -run } -run From martin at varnish-software.com Tue Nov 8 10:03:08 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:03:08 +0000 (UTC) Subject: [6.0] 7c5b14c34 h2: Polish duplicate pseudo-header error Message-ID: <20221108100308.8396CAE5F5@lists.varnish-cache.org> commit 7c5b14c346eac7b5dace4d3a8eceb13ed03273ee Author: Dridi Boukelmoune Date: Thu Nov 5 08:11:24 2020 +0100 h2: Polish duplicate pseudo-header error I find ":path: /foo" more descriptive than "/foo", even though I could infer which one it was... diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index d43262916..5c791a103 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -93,6 +93,7 @@ static h2_error h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len) { /* XXX: This might belong in cache/cache_http.c */ + const char *b0; unsigned n; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); @@ -105,6 +106,7 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len) return (H2SE_ENHANCE_YOUR_CALM); } + b0 = b; if (b[0] == ':') { /* Match H/2 pseudo headers */ /* XXX: Should probably have some include tbl for @@ -143,8 +145,8 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len) /* Check for duplicate pseudo-header */ if (hp->hd[n].b != NULL) { VSLb(hp->vsl, SLT_BogoHeader, - "Duplicate pseudo-header: %.*s", - (int)(len > 20 ? 20 : len), b); + "Duplicate pseudo-header %.*s%.*s", + (int)namelen, b0, (int)(len > 20 ? 20 : len), b); return (H2SE_PROTOCOL_ERROR); // rfc7540,l,3158,3162 } } else { From martin at varnish-software.com Tue Nov 8 10:03:08 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:03:08 +0000 (UTC) Subject: [6.0] 67c08c7c5 hpack: fix pseudo-headers handling Message-ID: <20221108100308.A1273AE5F9@lists.varnish-cache.org> commit 67c08c7c55e8a31e8a023960b450aa7b980223f3 Author: Asad Sajjad Ahmed Date: Fri Sep 30 14:42:53 2022 +0200 hpack: fix pseudo-headers handling We should apply the same restrictions on the list of allowed characters inside H/2 pseudo-headers as we do for H/1. This error is translated into the headers we send to a backend over H/1. Failure to do so could permit various exploits against a backend not handling malformed H/1 requests. Signed-off-by: Asad Sajjad Ahmed diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 5c791a103..e247e8f8c 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -94,13 +94,18 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len) { /* XXX: This might belong in cache/cache_http.c */ const char *b0; + int disallow_empty; unsigned n; + char *p; + int i; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); AN(b); assert(namelen >= 2); /* 2 chars from the ': ' that we added */ assert(namelen <= len); + disallow_empty = 0; + if (len > UINT_MAX) { /* XXX: cache_param max header size */ VSLb(hp->vsl, SLT_BogoHeader, "Header too large: %.20s", b); return (H2SE_ENHANCE_YOUR_CALM); @@ -115,10 +120,24 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len) b += namelen; len -= namelen; n = HTTP_HDR_METHOD; + disallow_empty = 1; + + /* First field cannot contain SP or CTL */ + for (p = b, i = 0; i < len; p++, i++) { + if (vct_issp(*p) || vct_isctl(*p)) + return (H2SE_PROTOCOL_ERROR); + } } else if (!strncmp(b, ":path: ", namelen)) { b += namelen; len -= namelen; n = HTTP_HDR_URL; + disallow_empty = 1; + + /* Second field cannot contain LWS or CTL */ + for (p = b, i = 0; i < len; p++, i++) { + if (vct_islws(*p) || vct_isctl(*p)) + return (H2SE_PROTOCOL_ERROR); + } } else if (!strncmp(b, ":scheme: ", namelen)) { /* XXX: What to do about this one? (typically "http" or "https"). For now set it as a normal @@ -126,6 +145,15 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len) b++; len-=1; n = hp->nhd; + + for (p = b + namelen, i = 0; i < len-namelen; + p++, i++) { + if (vct_issp(*p) || vct_isctl(*p)) + return (H2SE_PROTOCOL_ERROR); + } + + if (!i) + return (H2SE_PROTOCOL_ERROR); } else if (!strncmp(b, ":authority: ", namelen)) { b+=6; len-=6; @@ -162,6 +190,13 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len) hp->hd[n].b = b; hp->hd[n].e = b + len; + if (disallow_empty && !Tlen(hp->hd[n])) { + VSLb(hp->vsl, SLT_BogoHeader, + "Empty pseudo-header %.*s", + (int)namelen, b0); + return (H2SE_PROTOCOL_ERROR); + } + return (0); } diff --git a/bin/varnishtest/tests/t02023.vtc b/bin/varnishtest/tests/t02023.vtc new file mode 100644 index 000000000..cfd843da3 --- /dev/null +++ b/bin/varnishtest/tests/t02023.vtc @@ -0,0 +1,48 @@ +varnishtest "Empty pseudo-headers" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -arg "-p feature=+http2" -vcl+backend { +} -start + +client c1 { + txreq -url "" + rxresp + expect resp.status == 400 +} -run + +client c1 { + txreq -req "" + rxresp + expect resp.status == 400 +} -run + +client c1 { + txreq -proto "" + rxresp + expect resp.status == 400 +} -run + +client c1 { + stream 1 { + txreq -url "" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -scheme "" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -req "" + rxrst + } -run +} -run diff --git a/bin/varnishtest/tests/t02024.vtc b/bin/varnishtest/tests/t02024.vtc new file mode 100644 index 000000000..0d0a1abc5 --- /dev/null +++ b/bin/varnishtest/tests/t02024.vtc @@ -0,0 +1,48 @@ +varnishtest "Garbage pseudo-headers" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -arg "-p feature=+http2" -vcl+backend { +} -start + +client c1 { + txreq -url " " + rxresp + expect resp.status == 400 +} -run + +client c1 { + txreq -req " " + rxresp + expect resp.status == 400 +} -run + +client c1 { + txreq -proto " " + rxresp + expect resp.status == 400 +} -run + +client c1 { + stream 1 { + txreq -url " " + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -scheme " " + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -req " " + rxrst + } -run +} -run From martin at varnish-software.com Tue Nov 8 10:03:08 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:03:08 +0000 (UTC) Subject: [6.0] 6fbb93775 h2: the :scheme pseudo header is not optional Message-ID: <20221108100308.BFC1BAE607@lists.varnish-cache.org> commit 6fbb937758dfa7ad131d5a40d2c9b7b5929dc7dd Author: Asad Sajjad Ahmed Date: Wed Sep 28 14:58:38 2022 +0200 h2: the :scheme pseudo header is not optional The :scheme pseudo header is not optional in H/2 except when doing CONNECT. There is also a strict requirement for it appear only once. Signed-off-by: Asad Sajjad Ahmed diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index 205b96ccb..bba2b7e7d 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -199,11 +199,14 @@ vtr_deliver_f h2_deliver; vtr_minimal_response_f h2_minimal_response; #endif /* TRANSPORT_MAGIC */ +#define H2H_DECODE_FLAG_SCHEME_SEEN 0x1 + /* http2/cache_http2_hpack.c */ struct h2h_decode { unsigned magic; #define H2H_DECODE_MAGIC 0xd092bde4 + int flags; h2_error error; enum vhd_ret_e vhd_ret; char *out; diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index e247e8f8c..45cddb34b 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -90,7 +90,7 @@ h2h_checkhdr(const struct http *hp, const char *b, size_t namelen, size_t len) } static h2_error -h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len) +h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len, int *flags) { /* XXX: This might belong in cache/cache_http.c */ const char *b0; @@ -142,9 +142,18 @@ h2h_addhdr(struct http *hp, char *b, size_t namelen, size_t len) /* XXX: What to do about this one? (typically "http" or "https"). For now set it as a normal header, stripping the first ':'. */ + if (*flags & H2H_DECODE_FLAG_SCHEME_SEEN) { + VSLb(hp->vsl, SLT_BogoHeader, + "Duplicate pseudo-header %.*s%.*s", + (int)namelen, b0, + (int)(len > 20 ? 20 : len), b); + return (H2SE_PROTOCOL_ERROR); + } + b++; len-=1; n = hp->nhd; + *flags |= H2H_DECODE_FLAG_SCHEME_SEEN; for (p = b + namelen, i = 0; i < len-namelen; p++, i++) { @@ -330,7 +339,8 @@ h2h_decode_bytes(struct h2_sess *h2, const uint8_t *in, size_t in_l) d->out_u); if (d->error) break; - d->error = h2h_addhdr(hp, d->out, d->namelen, d->out_u); + d->error = h2h_addhdr(hp, d->out, d->namelen, d->out_u, + &d->flags); if (d->error) break; d->out[d->out_u++] = '\0'; /* Zero guard */ diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 98f5dc4f3..03a8cc5f0 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -545,11 +545,13 @@ static h2_error h2_end_headers(struct worker *wrk, struct h2_sess *h2, struct req *req, struct h2_req *r2) { + int scheme_seen; h2_error h2e; ssize_t cl; ASSERT_RXTHR(h2); assert(r2->state == H2_S_OPEN); + scheme_seen = h2->decode->flags & H2H_DECODE_FLAG_SCHEME_SEEN; h2e = h2h_decode_fini(h2); h2->new_req = NULL; if (r2->req->req_body_status == REQ_BODY_NONE) { @@ -597,10 +599,17 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, VSLb(h2->vsl, SLT_Debug, "Missing :method"); return (H2SE_PROTOCOL_ERROR); //rfc7540,l,3087,3090 } + if (req->http->hd[HTTP_HDR_URL].b == NULL) { VSLb(h2->vsl, SLT_Debug, "Missing :path"); return (H2SE_PROTOCOL_ERROR); //rfc7540,l,3087,3090 } + + if (!(scheme_seen)) { + VSLb(h2->vsl, SLT_Debug, "Missing :scheme"); + return (H2SE_PROTOCOL_ERROR); //rfc7540,l,3087,3090 + } + AN(req->http->hd[HTTP_HDR_PROTO].b); req->req_step = R_STP_TRANSPORT; diff --git a/bin/varnishtest/tests/t02025.vtc b/bin/varnishtest/tests/t02025.vtc new file mode 100644 index 000000000..b464f8cdc --- /dev/null +++ b/bin/varnishtest/tests/t02025.vtc @@ -0,0 +1,48 @@ +varnishtest "Dublicate pseudo-headers" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -arg "-p feature=+http2" -vcl+backend { +} -start + +#client c1 { +# txreq -url "/some/path" -url "/some/other/path" +# rxresp +# expect resp.status == 400 +#} -run + +#client c1 { +# txreq -req "GET" -req "POST" +# rxresp +# expect resp.status == 400 +#} -run + +#client c1 { +# txreq -proto "HTTP/1.1" -proto "HTTP/2.0" +# rxresp +# expect resp.status == 400 +#} -run + +client c1 { + stream 1 { + txreq -url "/some/path" -url "/some/other/path" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -scheme "http" -scheme "https" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -req "GET" -req "POST" + rxrst + } -run +} -run From martin at varnish-software.com Tue Nov 8 10:03:08 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:03:08 +0000 (UTC) Subject: [6.0] 9c4d8a47b h2: add vtc for VSV00011 Message-ID: <20221108100308.D95A0AE618@lists.varnish-cache.org> commit 9c4d8a47b980d6535ac02bf550175b2c724248d8 Author: Asad Sajjad Ahmed Date: Mon Oct 3 16:08:30 2022 +0200 h2: add vtc for VSV00011 Check for correct handling of missing pseudo-headers, and invalid characters. Signed-off-by: Asad Sajjad Ahmed diff --git a/bin/varnishtest/tests/f00011.vtc b/bin/varnishtest/tests/f00011.vtc new file mode 100644 index 000000000..3fbcde4e5 --- /dev/null +++ b/bin/varnishtest/tests/f00011.vtc @@ -0,0 +1,51 @@ +varnishtest "H2: Empty pseudo-headers" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -arg "-p feature=+http2" -vcl+backend { +} -start + +client c1 { + stream 1 { + txreq -url "" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -url " \t" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -scheme "" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -scheme " \t" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -req "" + rxrst + } -run +} -run + +client c1 { + stream 1 { + txreq -req " \t" + rxrst + } -run +} -run From martin at varnish-software.com Tue Nov 8 10:03:08 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:03:08 +0000 (UTC) Subject: [6.0] 7ec2391c9 Fix autoconf error in version 2.70 Message-ID: <20221108100308.F2FD1AE625@lists.varnish-cache.org> commit 7ec2391c9f665c911d3b4e7e85be52c47f469bea Author: Simon Stridsberg Date: Tue Jan 12 15:34:33 2021 +0000 Fix autoconf error in version 2.70 Fixes #3491 diff --git a/configure.ac b/configure.ac index 2d72758e2..28476d480 100644 --- a/configure.ac +++ b/configure.ac @@ -40,9 +40,9 @@ AC_PROG_INSTALL AC_ARG_WITH([rst2man], AS_HELP_STRING([--with-rst2man=PATH], [Location of rst2man (auto)]), [RST2MAN="$withval"], - AC_CHECK_PROGS(RST2MAN, + [AC_CHECK_PROGS(RST2MAN, [rst2man-3.6 rst2man-3 rst2man rst2man.py], - [no])) + [no])]) if test "x$RST2MAN" = "xno"; then AC_MSG_ERROR( [rst2man is needed to build Varnish, please install python3-docutils.]) @@ -51,9 +51,9 @@ fi AC_ARG_WITH([sphinx-build], AS_HELP_STRING([--with-sphinx-build=PATH], [Location of sphinx-build (auto)]), [SPHINX="$withval"], - AC_CHECK_PROGS(SPHINX, + [AC_CHECK_PROGS(SPHINX, [sphinx-build-3.6 sphinx-build-3 sphinx-build], - [no])) + [no])]) if test "x$SPHINX" = "xno"; then AC_MSG_ERROR( [sphinx-build is needed to build Varnish, please install python3-sphinx.]) @@ -62,10 +62,11 @@ fi AC_ARG_WITH([rst2html], AS_HELP_STRING([--with-rst2html=PATH], [Location of rst2html (auto)]), [RST2HTML="$withval"], - AC_CHECK_PROGS(RST2HTML, + [AC_CHECK_PROGS(RST2HTML, [rst2html-3.6 rst2html-3 rst2html rst2html.py], - "no")) + "no")]) AM_CONDITIONAL(HAVE_RST2HTML,[test "x$RST2HTML" != "xno"]) + if test "x$RST2HTML" = "xno"; then AC_MSG_ERROR( [rst2html not found - (Weird, we found rst2man?!)]) From martin at varnish-software.com Tue Nov 8 10:03:09 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:03:09 +0000 (UTC) Subject: [6.0] 827283d5e Create a changelog for 6.0.11 Message-ID: <20221108100309.191C2AE62F@lists.varnish-cache.org> commit 827283d5eeee0f7db7341ea00582233fd10d0a93 Author: Martin Blix Grydeland Date: Wed Oct 12 11:33:45 2022 +0200 Create a changelog for 6.0.11 diff --git a/doc/changes.rst b/doc/changes.rst index 697c4a20f..36580a851 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -26,6 +26,26 @@ http://varnish-cache.org/docs/trunk/whats-new/index.html and via individual releases. These documents are updated as part of the release process. +================================= +Varnish Cache 6.0.11 (unreleased) +================================= + +* Handling of hop-by-hop headers has been fixed for HTTP/2 (3416_). + +* Handling of the ``Keep-Alive`` HTTP header as hop-by-hop has been fixed + (3417_). + +* Apply the same character set rules to HTTP/2 pseudo-headers as is done + on the corresponding HTTP/1 request-line field parsing. (VSV00011_) + +* The ``%{X}T`` format has been added to ``varnishncsa``, which + generalizes ``%D`` and ``%T``, but also support milliseconds + (``ms``) output. + +.. _3416: https://github.com/varnishcache/varnish-cache/pull/3416 +.. _3417: https://github.com/varnishcache/varnish-cache/pull/3417 +.. _VSV00011: https://varnish-cache.org/security/VSV00011.html + ================================= Varnish Cache 6.0.10 (2022-01-25) ================================= From martin at varnish-software.com Tue Nov 8 10:03:09 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:03:09 +0000 (UTC) Subject: [6.0] a3bc025c2 Prepare for 6.0.11 Message-ID: <20221108100309.3CB93AE63F@lists.varnish-cache.org> commit a3bc025c2df28e4a76e10c2c41217c9864e9963b Author: Martin Blix Grydeland Date: Wed Oct 12 11:41:21 2022 +0200 Prepare for 6.0.11 diff --git a/configure.ac b/configure.ac index 28476d480..ca100e06c 100644 --- a/configure.ac +++ b/configure.ac @@ -1,8 +1,8 @@ AC_PREREQ(2.59) AC_COPYRIGHT([Copyright (c) 2006 Verdens Gang AS -Copyright (c) 2006-2020 Varnish Software]) +Copyright (c) 2006-2022 Varnish Software]) AC_REVISION([$Id$]) -AC_INIT([Varnish], [6.0.10], [varnish-dev at varnish-cache.org]) +AC_INIT([Varnish], [6.0.11], [varnish-dev at varnish-cache.org]) AC_CONFIG_SRCDIR(include/miniobj.h) AC_CONFIG_HEADERS([config.h]) AC_CONFIG_MACRO_DIR([m4]) diff --git a/doc/changes.rst b/doc/changes.rst index 36580a851..7fdd4530e 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -27,7 +27,7 @@ individual releases. These documents are updated as part of the release process. ================================= -Varnish Cache 6.0.11 (unreleased) +Varnish Cache 6.0.11 (2022-11-08) ================================= * Handling of hop-by-hop headers has been fixed for HTTP/2 (3416_). diff --git a/doc/sphinx/index.rst b/doc/sphinx/index.rst index e5c33b1e4..7e0d27619 100644 --- a/doc/sphinx/index.rst +++ b/doc/sphinx/index.rst @@ -35,9 +35,9 @@ Conventions used in this manual include: Longer listings like example command output and VCL look like this:: $ /opt/varnish/sbin/varnishd -V - varnishd (varnish-trunk revision 199de9b) + varnishd (varnish-6.0.x revision 1234567) Copyright (c) 2006 Verdens Gang AS - Copyright (c) 2006-2020 Varnish Software AS + Copyright (c) 2006-2022 Varnish Software AS .. For maintainers: diff --git a/lib/libvarnish/version.c b/lib/libvarnish/version.c index 194eb328e..aab1951db 100644 --- a/lib/libvarnish/version.c +++ b/lib/libvarnish/version.c @@ -44,5 +44,5 @@ VCS_Message(const char *progname) { fprintf(stderr, "%s (%s)\n", progname, VCS_version); fprintf(stderr, "Copyright (c) 2006 Verdens Gang AS\n"); - fprintf(stderr, "Copyright (c) 2006-2021 Varnish Software\n"); + fprintf(stderr, "Copyright (c) 2006-2022 Varnish Software\n"); } From martin at varnish-software.com Tue Nov 8 10:03:09 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:03:09 +0000 (UTC) Subject: [6.0] 9c30d039d Update image used for packaging Message-ID: <20221108100309.56090AE64C@lists.varnish-cache.org> commit 9c30d039d22640a354157d37ded13d13f0299b4e Author: Martin Blix Grydeland Date: Wed Oct 12 13:57:16 2022 +0200 Update image used for packaging diff --git a/.circleci/config.yml b/.circleci/config.yml index c272743b7..2fdc17d32 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -131,7 +131,7 @@ jobs: description: the base Docker image for Dockerfile type: string machine: - image: ubuntu-1604:201903-01 + image: ubuntu-2004:202111-02 steps: - attach_workspace: at: ~/project From martin at varnish-software.com Tue Nov 8 10:03:09 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:03:09 +0000 (UTC) Subject: [6.0] eeb29798a Update make-rpm-packages.sh from 7.2 Message-ID: <20221108100309.73F75AE668@lists.varnish-cache.org> commit eeb29798ac75d65732d0b6fa4636aab80de7b29d Author: Martin Blix Grydeland Date: Wed Oct 12 15:03:32 2022 +0200 Update make-rpm-packages.sh from 7.2 diff --git a/.circleci/make-rpm-packages.sh b/.circleci/make-rpm-packages.sh index 56acdad05..848084bc4 100755 --- a/.circleci/make-rpm-packages.sh +++ b/.circleci/make-rpm-packages.sh @@ -6,21 +6,28 @@ echo "PARAM_RELEASE: $PARAM_RELEASE" echo "PARAM_DIST: $PARAM_DIST" if [ -z "$PARAM_RELEASE" ]; then - echo "Env variable PARAM_RELEASE is not set! For example PARAM_RELEASE=8, for CentOS 8" + echo "Env variable PARAM_RELEASE is not set! For example PARAM_RELEASE=stream, for CentOS stream" exit 1 elif [ -z "$PARAM_DIST" ]; then echo "Env variable PARAM_DIST is not set! For example PARAM_DIST=centos" exit 1 fi -yum install -y epel-release - -if [ "$PARAM_DIST" = centos ]; then - if [ "$PARAM_RELEASE" = 8 ]; then - dnf install -y 'dnf-command(config-manager)' - yum config-manager --set-enabled powertools - fi -fi +case "$PARAM_DIST:$PARAM_RELEASE" in + almalinux:9) + dnf install -y 'dnf-command(config-manager)' + yum config-manager --set-enabled crb + yum install -y epel-release + ;; + centos:stream|almalinux:8) + dnf install -y 'dnf-command(config-manager)' + yum config-manager --set-enabled powertools + yum install -y epel-release + ;; + centos:7) + yum install -y epel-release + ;; +esac yum install -y rpm-build yum-utils From martin at varnish-software.com Tue Nov 8 10:03:09 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:03:09 +0000 (UTC) Subject: [6.0] b3ed6495c Update make-deb-packages.sh from 7.2 Message-ID: <20221108100309.9340CAE681@lists.varnish-cache.org> commit b3ed6495c4f6b063f0a29001aba589753a96b9a6 Author: Martin Blix Grydeland Date: Wed Oct 12 15:06:11 2022 +0200 Update make-deb-packages.sh from 7.2 diff --git a/.circleci/make-deb-packages.sh b/.circleci/make-deb-packages.sh index 8437de094..2f05e99d2 100755 --- a/.circleci/make-deb-packages.sh +++ b/.circleci/make-deb-packages.sh @@ -12,10 +12,10 @@ echo "PARAM_DIST: $PARAM_DIST" if [ -z "$PARAM_RELEASE" ]; then - echo "Env variable PARAM_RELEASE is not set! For example PARAM_RELEASE=8, for CentOS 8" + echo "Env variable PARAM_RELEASE is not set! For example PARAM_RELEASE=focal for Ubuntu 20.04" exit 1 elif [ -z "$PARAM_DIST" ]; then - echo "Env variable PARAM_DIST is not set! For example PARAM_DIST=centos" + echo "Env variable PARAM_DIST is not set! For example PARAM_DIST=debian" exit 1 fi From martin at varnish-software.com Tue Nov 8 10:03:09 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:03:09 +0000 (UTC) Subject: [6.0] a6f489c85 Update make-apk-packages.sh from 7.2 Message-ID: <20221108100309.B381BAE693@lists.varnish-cache.org> commit a6f489c85ed72d8ea64c1149755b2aacb3054a18 Author: Martin Blix Grydeland Date: Wed Oct 12 15:41:08 2022 +0200 Update make-apk-packages.sh from 7.2 diff --git a/.circleci/make-apk-packages.sh b/.circleci/make-apk-packages.sh index 26b672c2c..a8b678f8f 100755 --- a/.circleci/make-apk-packages.sh +++ b/.circleci/make-apk-packages.sh @@ -2,7 +2,7 @@ set -eux -apk add -q --no-progress --update tar alpine-sdk +apk add -q --no-progress --update tar alpine-sdk sudo echo "PARAM_RELEASE: $PARAM_RELEASE" echo "PARAM_DIST: $PARAM_DIST" From martin at varnish-software.com Tue Nov 8 10:03:09 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:03:09 +0000 (UTC) Subject: [6.0] f9bbd9d47 Update config.yml from 7.2 Message-ID: <20221108100309.D548CAE6B2@lists.varnish-cache.org> commit f9bbd9d4701a4d37f3ce5a36efea6110f1765b18 Author: Martin Blix Grydeland Date: Wed Oct 12 15:06:40 2022 +0200 Update config.yml from 7.2 diff --git a/.circleci/config.yml b/.circleci/config.yml index 2fdc17d32..619a189ec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,11 +1,5 @@ version: 2.1 -aliases: - pkg_req: &pkg_req - requires: - - dist - - tar_pkg_tools - parameters: vc-commit: type: string @@ -19,10 +13,21 @@ parameters: dist-url-sha256: type: string default: "" + configure_args: + type: string + default: | + --with-unwind \ + --enable-developer-warnings \ + --enable-debugging-symbols \ + --disable-stack-protector \ + --with-persistent-storage \ + build-pkgs: + type: string + default: "" jobs: dist: - description: Builds varnish-x.y.z.tar.gz that is used later for the packaging jobs + description: Build or download varnish-x.y.z.tar.gz that is used later for the packaging jobs docker: - image: centos:7 steps: @@ -49,7 +54,7 @@ jobs: - run: name: Download the dist tarball command: | - curl -Ls << pipeline.parameters.dist-url >> -o varnish-dist.tar.gz + curl -Ls '<< pipeline.parameters.dist-url >>' -o varnish-dist.tar.gz - when: condition: << pipeline.parameters.dist-url-sha256 >> steps: @@ -65,7 +70,7 @@ jobs: tar xzf ../varnish-dist.tar.gz VERSION=$(varnish-*/configure --version | awk 'NR == 1 {print $NF}') cd .. - mv varnish-dist.tar.gz varnish-${VERSION}.tar.gz + mv -v varnish-dist.tar.gz varnish-${VERSION}.tar.gz - unless: condition: << pipeline.parameters.dist-url >> steps: @@ -73,12 +78,15 @@ jobs: name: Create the dist tarball command: | git checkout << pipeline.parameters.vc-commit >> - # if version is "trunk", it's a weekly tarball, override the version + # Locally built tarballs are always built with weekly in package name + touch .is_weekly + # If version is "trunk", override version to add date if grep 'AC_INIT.*trunk.*' ./configure.ac; then - sed -i -e "s/^AC_INIT.*trunk.*/AC_INIT([Varnish], [$(date +%Y%m%d)], [varnish-dev at varnish-cache.org])/" ./configure.ac - touch .is_weekly + sed -i -e "s/AC_INIT(\[$.*$\], \[$.*$\], \[$.*$\])/AC_INIT([\1], [$(date +%Y%m%d)], [\3])/" ./configure.ac + else + sed -i -e "s/AC_INIT(\[$.*$\], \[$.*$\], \[$.*$\])/AC_INIT([\1], [\2-$(date +%Y%m%d)], [\3])/" ./configure.ac fi - ./autogen.des --quiet + ./autogen.des make dist -j 16 - persist_to_workspace: root: . @@ -115,81 +123,142 @@ jobs: - alpine.tar.gz package: parameters: - dist: - description: the Linux distribution (debian|ubuntu|centos) - type: string - release: - description: the release name (stretch|buster|xenial|bionic|7|8) - type: string - ext: - description: the package extension (deb|rpm|apk) - type: string - arch: - description: the architecture (x64|aarch64) + platform: + description: the Linux distribution, with release, e.g. debian:buster, centos:7 type: string - image: - description: the base Docker image for Dockerfile + rclass: + description: the resource class to use, usuall arm.medium or medium type: string machine: image: ubuntu-2004:202111-02 + resource_class: << parameters.rclass >> steps: - attach_workspace: at: ~/project - - run: ls -la ~/project - - run: - name: Activate QEMU - command: | - sudo docker run -it --rm --privileged multiarch/qemu-user-static --reset --credential yes --persistent yes - - run: - name: Create Dockerfile - command: | - echo "FROM << parameters.image >>" > Dockerfile - echo "ADD make-<< parameters.ext >>-packages.sh /usr/bin/" >> Dockerfile - echo 'CMD ["make-<< parameters.ext >>-packages.sh"]' >> Dockerfile + + - when: + condition: + matches: + pattern: ^alpine.* + value: << parameters.platform >> + steps: + - run: + # https://wiki.alpinelinux.org/wiki/Release_Notes_for_Alpine_3.14.0#faccessat2 + name: grab the latest docker version + command: | + # using https://docs.docker.com/engine/install/ubuntu/ + sudo apt-get update + sudo apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + case "<< parameters.rclass >>" in + arm.*) ARCH=arm64;; + *) ARCH=amd64;; + esac + echo \ + "deb [signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt-get install docker-ce docker-ce-cli containerd.io - run: - name: Build << parameters.dist >> << parameters.release >> << parameters.arch >> << parameters.ext >> + name: Build for << parameters.platform >> on << parameters.rclass >> command: | mkdir -p packages - cp .circleci/make-<< parameters.ext >>-packages.sh . - docker build -t varnish-<< parameters.ext >>-package-build:<< parameters.arch >> . - docker run --rm -it -e PARAM_DIST=<< parameters.dist >> -e PARAM_RELEASE=<< parameters.release >> -v$(pwd):/varnish-cache varnish-<< parameters.ext >>-package-build:<< parameters.arch >> + case "<< parameters.platform >>" in + debian:*|ubuntu:*) EXT=deb ;; + centos:*|fedora:*) EXT=rpm ;; + almalinux:*) EXT=rpm ;; + alpine:*) EXT=apk ;; + *) + echo "unrecognized platform: << parameters.platform >>" + exit 1 + esac + + case "<< parameters.platform >>" in + centos:stream) REPO=quay.io/centos/ ;; + *) REPO= ;; + esac + + case "<< parameters.rclass >>" in + arm.*) ARCH=arm64 ;; + *) ARCH=amd64 ;; + esac + + docker run \ + --rm \ + -it \ + -e PARAM_DIST=$(echo "<< parameters.platform >>" | cut -d: -f1) \ + -e PARAM_RELEASE=$(echo "<< parameters.platform >>" | cut -d: -f2) \ + -v$(pwd):/varnish-cache \ + --platform linux/$ARCH \ + ${REPO}<< parameters.platform >> \ + /varnish-cache/.circleci/make-$EXT-packages.sh - run: name: List created packages - command: find ./packages -name "*.<< parameters.ext >>" + command: find ./packages -type f - persist_to_workspace: root: . paths: - "packages" - distcheck: + build: parameters: + prefix: + description: the container image prefix (repository or architecture) + type: string + default: "" dist: description: the Linux distribution (debian|ubuntu) type: string release: - description: the release name (stretch|buster|xenial|bionic) + description: the release name (buster|bullseye|bionic|focal|jammy) + type: string + make_target: + description: the make target to execute during the build + default: distcheck type: string extra_conf: description: platform-specific configure arguments default: "" type: string docker: - - image: << parameters.dist >>:<< parameters.release >> + - image: centos:7 working_directory: /workspace steps: + - setup_remote_docker: + version: 20.10.11 - run: - name: Possible << parameters.dist >>:<< parameters.release >> extra repos + name: Install docker + command: yum install -y docker + - checkout + - run: + name: Extract and build command: | - if [ << parameters.dist >> = centos ]; then - if [ << parameters.release >> = 8 ]; then - dnf install -y 'dnf-command(config-manager)' - yum config-manager --set-enabled powertools - yum install -y diffutils python3-sphinx - else - yum install -y python-sphinx - fi - yum install -y epel-release + docker create --name workspace -v /workspace << parameters.prefix >><< parameters.dist >>:<< parameters.release >> /bin/true + docker cp /workspace workspace:/ + docker run --volumes-from workspace -w /workspace << parameters.prefix >><< parameters.dist >>:<< parameters.release >> sh -c ' + case "<< parameters.dist >>" in + centos|almalinux|fedora) + yum groupinstall -y "Development Tools" + case "<< parameters.dist >>:<< parameters.release >>" in + almalinux:9) + dnf install -y "dnf-command(config-manager)" + yum config-manager --set-enabled crb + yum install -y diffutils + yum install -y epel-release + ;; + centos:stream|almalinux:8) + dnf install -y "dnf-command(config-manager)" + yum config-manager --set-enabled powertools + yum install -y diffutils + yum install -y epel-release + ;; + centos:7) + yum install -y epel-release + ;; + esac yum install -y \ + cpio \ automake \ + git \ jemalloc-devel \ libedit-devel \ libtool \ @@ -197,8 +266,10 @@ jobs: make \ pcre-devel \ python3 \ + /usr/bin/sphinx-build \ sudo - elif [ << parameters.dist >> = debian -o << parameters.dist >> = ubuntu ]; then + ;; + debian|ubuntu) export DEBIAN_FRONTEND=noninteractive export DEBCONF_NONINTERACTIVE_SEEN=true apt-get update @@ -208,6 +279,7 @@ jobs: build-essential \ ca-certificates \ cpio \ + git \ graphviz \ libedit-dev \ libjemalloc-dev \ @@ -218,7 +290,8 @@ jobs: pkg-config \ python3-sphinx \ sudo - elif [ << parameters.dist >> = alpine ]; then + ;; + alpine) apk update apk add -q \ autoconf \ @@ -226,6 +299,7 @@ jobs: build-base \ ca-certificates \ cpio \ + git \ gzip \ libedit-dev \ libtool \ @@ -236,18 +310,34 @@ jobs: py3-sphinx \ tar \ sudo - fi - - attach_workspace: - at: /workspace - - run: - name: Extract and distcheck - command: | - tar xavf varnish-*.tar.gz --strip 1 - if [ << parameters.dist >> = centos ]; then - adduser varnish - else - adduser --disabled-password --gecos "" varnish - fi + ;; + archlinux) + pacman -Syu --noconfirm \ + ca-certificates \ + cpio \ + git \ + libedit \ + libtool \ + libunwind \ + linux-headers \ + pcre \ + python-docutils \ + python-sphinx \ + tar + ;; + esac + + case "<< parameters.dist >>" in + archlinux) + useradd varnish + ;; + centos|almalinux|fedora) + adduser varnish + ;; + *) + adduser --disabled-password --gecos "" varnish + ;; + esac chown -R varnish:varnish . @@ -256,17 +346,18 @@ jobs: export TSAN_OPTIONS=abort_on_error=1,halt_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/tsan.suppr export UBSAN_OPTIONS=halt_on_error=1,print_stacktrace=1,use_sigaltstack=0,suppressions=$(pwd)/tools/ubsan.suppr - sudo -u varnish ./configure \ - --quiet \ - --with-unwind \ - --enable-developer-warnings \ - --enable-debugging-symbols \ - --disable-stack-protector \ - --with-persistent-storage \ - << parameters.extra_conf >> sudo -u varnish \ - --preserve-env=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \ - make distcheck VERBOSE=1 -j 12 -k + autoreconf -i -v + sudo -u varnish \ + ./configure \ + << pipeline.parameters.configure_args >> \ + << parameters.extra_conf >> + sudo -u varnish \ + --preserve-env=ASAN_OPTIONS,LSAN_OPTIONS,TSAN_OPTIONS,UBSAN_OPTIONS \ + make << parameters.make_target >> VERBOSE=1 -j 4 -k \ + DISTCHECK_CONFIGURE_FLAGS="<< pipeline.parameters.configure_args >> \ + << parameters.extra_conf >>" + ' collect_packages: docker: @@ -287,147 +378,78 @@ jobs: workflows: version: 2 commit: + unless: &packaging_cond + or: + - << pipeline.parameters.build-pkgs >> + - << pipeline.parameters.dist-url >> jobs: - - distcheck: - name: distcheck_centos_7 + - build: + name: build_centos_7 dist: centos release: "7" - requires: - - dist - - distcheck: - name: distcheck_debian_buster + - build: + name: build_almalinux_8 + dist: almalinux + release: "8" + - build: + name: build_almalinux_9 + dist: almalinux + release: "9" + - build: + name: build_debian_buster dist: debian release: buster - extra_conf: --enable-asan --enable-ubsan - requires: - - dist - - dist - - tar_pkg_tools - - package: - name: aarch64-ubuntu-focal - dist: ubuntu - release: focal - arch: aarch64 - image: arm64v8/ubuntu:focal - ext: deb - <<: *pkg_req - - package: - name: x64-ubuntu-focal - dist: ubuntu - release: focal - arch: x64 - image: ubuntu:focal - ext: deb - <<: *pkg_req - - package: - name: aarch64-ubuntu-bionic - dist: ubuntu - release: bionic - arch: aarch64 - image: arm64v8/ubuntu:bionic - ext: deb - <<: *pkg_req - - package: - name: x64-ubuntu-bionic + - build: + name: build_debian_bullseye + dist: debian + release: bullseye + - build: + name: build_ubuntu_bionic dist: ubuntu release: bionic - arch: x64 - image: ubuntu:bionic - ext: deb - <<: *pkg_req - - package: - name: aarch64-ubuntu-xenial + - build: + name: build_ubuntu_focal dist: ubuntu - release: xenial - arch: aarch64 - image: arm64v8/ubuntu:xenial - ext: deb - <<: *pkg_req - - package: - name: x64-ubuntu-xenial + release: focal + - build: + name: build_ubuntu_jammy dist: ubuntu - release: xenial - arch: x64 - image: ubuntu:xenial - ext: deb - <<: *pkg_req - - package: - name: aarch64-debian-buster - dist: debian - release: buster - arch: aarch64 - image: arm64v8/debian:buster-slim - ext: deb - <<: *pkg_req - - package: - name: x64-debian-buster - dist: debian - release: buster - arch: x64 - image: debian:buster-slim - ext: deb - <<: *pkg_req - - package: - name: aarch64-debian-stretch - dist: debian - release: stretch - arch: aarch64 - image: arm64v8/debian:stretch-slim - ext: deb - <<: *pkg_req - - package: - name: x64-debian-stretch - dist: debian - release: stretch - arch: x64 - image: debian:stretch-slim - ext: deb - <<: *pkg_req - - package: - name: aarch64-centos-8 - dist: centos - release: "8" - arch: aarch64 - image: arm64v8/centos:8 - ext: rpm - <<: *pkg_req - - package: - name: x64-centos-8 - dist: centos - release: "8" - arch: x64 - image: centos:8 - ext: rpm - <<: *pkg_req - - package: - name: aarch64-centos-7 - dist: centos - release: "7" - arch: aarch64 - image: arm64v8/centos:7 - ext: rpm - <<: *pkg_req + release: jammy + packaging: + when: *packaging_cond + jobs: &packaging_jobs + - dist + - tar_pkg_tools - package: - name: x64-centos-7 - dist: centos - release: "7" - arch: x64 - image: centos:7 - ext: rpm - <<: *pkg_req + name: << matrix.platform >> packages (<< matrix.rclass >>) + requires: + - dist + - tar_pkg_tools + matrix: + parameters: + platform: + - ubuntu:bionic + - ubuntu:focal + - ubuntu:jammy + - debian:buster + - debian:bullseye + - centos:7 + - almalinux:8 + - almalinux:9 + - fedora:latest + - alpine:3 + rclass: + - arm.medium + - medium - collect_packages: requires: - - x64-ubuntu-xenial - - aarch64-ubuntu-xenial - - x64-ubuntu-bionic - - aarch64-ubuntu-bionic - - x64-ubuntu-focal - - aarch64-ubuntu-focal - - x64-debian-stretch - - aarch64-debian-stretch - - x64-debian-buster - - aarch64-debian-buster - - x64-centos-8 - - aarch64-centos-8 - - x64-centos-7 - - aarch64-centos-7 + - package + nightly: + triggers: + - schedule: + cron: "0 4 * * *" + filters: + branches: + only: + - master + jobs: *packaging_jobs From martin at varnish-software.com Tue Nov 8 10:03:09 2022 From: martin at varnish-software.com (Martin Blix Grydeland) Date: Tue, 8 Nov 2022 10:03:09 +0000 (UTC) Subject: [6.0] 401fdb426 Add back xenial and stretch Message-ID: <20221108100309.F1CFAAE6C4@lists.varnish-cache.org> commit 401fdb42609424a95b3f9e9befd0240d9407f870 Author: Martin Blix Grydeland Date: Wed Oct 12 16:18:12 2022 +0200 Add back xenial and stretch diff --git a/.circleci/config.yml b/.circleci/config.yml index 619a189ec..d4a1970f4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -428,9 +428,11 @@ workflows: matrix: parameters: platform: + - ubuntu:xenial - ubuntu:bionic - ubuntu:focal - ubuntu:jammy + - debian:stretch - debian:buster - debian:bullseye - centos:7 From nils.goroll at uplex.de Tue Nov 8 17:51:08 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 8 Nov 2022 17:51:08 +0000 (UTC) Subject: [master] bb8bcca31 Fix typo Message-ID: <20221108175108.8C0BBA214C@lists.varnish-cache.org> commit bb8bcca318064614a263a0c491a8f7cc08bbabb9 Author: Nils Goroll Date: Tue Nov 8 18:42:20 2022 +0100 Fix typo As far as I can see, this is only used in the vim->n_ stats, which are, apparently, not used anywhere. diff --git a/lib/libvcc/vcc_vmod.h b/lib/libvcc/vcc_vmod.h index 2f386cf5a..1fe6e5360 100644 --- a/lib/libvcc/vcc_vmod.h +++ b/lib/libvcc/vcc_vmod.h @@ -31,7 +31,7 @@ #define STANZA_TBL \ STANZA(ALIAS, alias, SYM_ALIAS) \ STANZA(CPROTO, cproto, SYM_NONE) \ - STANZA(EVENT, evant, SYM_NONE) \ + STANZA(EVENT, event, SYM_NONE) \ STANZA(FUNC, func, SYM_FUNC) \ STANZA(METHOD, method, SYM_METHOD) \ STANZA(OBJ, obj, SYM_OBJECT) \ From nils.goroll at uplex.de Tue Nov 8 18:11:05 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 8 Nov 2022 18:11:05 +0000 (UTC) Subject: [master] 041c5a951 Start a NEXT release in vrt history Message-ID: <20221108181105.50E21A2E34@lists.varnish-cache.org> commit 041c5a951b7a2d9cc29501cbaf285bdc47a7ed28 Author: Nils Goroll Date: Tue Nov 8 19:09:43 2022 +0100 Start a NEXT release in vrt history to anchor patches diff --git a/include/vrt.h b/include/vrt.h index fca3c0295..d2af82cc7 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -57,6 +57,7 @@ * Whenever something is deleted or changed in a way which is not * binary/load-time compatible, increment MAJOR version * + * NEXT (2023-03-15) * 16.0 (2022-09-15) * VMOD C-prototypes moved into JSON * VRT_AddVDP() deprecated From nils.goroll at uplex.de Mon Nov 14 15:53:07 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 14 Nov 2022 15:53:07 +0000 (UTC) Subject: [master] 1e1493bdc Refactor: Centralize (struct sockaddr_un) initialization Message-ID: <20221114155307.199A664C8F@lists.varnish-cache.org> commit 1e1493bdc7ba4f8ef6dba35281fb2b18c688423c Author: Nils Goroll Date: Mon Nov 14 16:45:22 2022 +0100 Refactor: Centralize (struct sockaddr_un) initialization We had three call sites, one with and two without size checking. Other improvements: - zero (struct sockaddr_un).sun_path - set errno if the path argument does not fit Motivated by: #3863 diff --git a/bin/varnishd/mgt/mgt_acceptor.c b/bin/varnishd/mgt/mgt_acceptor.c index 612198672..9cbeb81aa 100644 --- a/bin/varnishd/mgt/mgt_acceptor.c +++ b/bin/varnishd/mgt/mgt_acceptor.c @@ -76,11 +76,17 @@ struct uds_perms { static VTAILQ_HEAD(,listen_arg) listen_args = VTAILQ_HEAD_INITIALIZER(listen_args); +static int +mac_vus_bind(void *priv, const struct sockaddr_un *uds) +{ + return (VUS_bind(uds, priv)); +} + static int mac_opensocket(struct listen_sock *ls) { int fail; - struct sockaddr_un uds; + const char *err; CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); if (ls->sock > 0) { @@ -89,11 +95,8 @@ mac_opensocket(struct listen_sock *ls) } if (!ls->uds) ls->sock = VTCP_bind(ls->addr, NULL); - else { - uds.sun_family = PF_UNIX; - bprintf(uds.sun_path, "%s", ls->endpoint); - ls->sock = VUS_bind(&uds, NULL); - } + else + ls->sock = VUS_resolver(ls->endpoint, mac_vus_bind, NULL, &err); fail = errno; if (ls->sock < 0) { AN(fail); diff --git a/lib/libvarnish/vus.c b/lib/libvarnish/vus.c index ba803b08d..0f39fb601 100644 --- a/lib/libvarnish/vus.c +++ b/lib/libvarnish/vus.c @@ -41,24 +41,41 @@ #include "vus.h" #include "vtcp.h" +static int +sun_init(struct sockaddr_un *uds, const char *path, const char **err) +{ + AN(uds); + AN(path); + assert(*path == '/'); + + if (err) + *err = NULL; + + if (strlen(path) + 1 > sizeof(uds->sun_path)) { + errno = ENAMETOOLONG; + if (err) + *err = "Path too long for a Unix domain socket"; + return (-1); + } + memset(uds->sun_path, 0, sizeof(uds->sun_path)); + bprintf(uds->sun_path, "%s", path); + uds->sun_family = PF_UNIX; + return (0); +} + int VUS_resolver(const char *path, vus_resolved_f *func, void *priv, const char **err) { struct sockaddr_un uds; - int ret = 0; - - AN(path); - assert(*path == '/'); + int ret; AN(err); - *err = NULL; - if (strlen(path) + 1 > sizeof(uds.sun_path)) { - *err = "Path too long for a Unix domain socket"; - return (-1); - } - bprintf(uds.sun_path, "%s", path); - uds.sun_family = PF_UNIX; + + ret = sun_init(&uds, path, err); + if (ret) + return (ret); + if (func != NULL) ret = func(priv, &uds); return (ret); @@ -110,8 +127,9 @@ VUS_connect(const char *path, int msec) if (path == NULL) return (-1); - uds.sun_family = PF_UNIX; - bprintf(uds.sun_path, "%s", path); + i = sun_init(&uds, path, NULL); + if (i) + return (i); AN(sl); s = socket(PF_UNIX, SOCK_STREAM, 0); From phk at FreeBSD.org Mon Nov 14 17:26:05 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 14 Nov 2022 17:26:05 +0000 (UTC) Subject: [master] 1e09e63ff Silence FlexeLint Message-ID: <20221114172605.41A8D91F40@lists.varnish-cache.org> commit 1e09e63ff0fe97ed9faff596c87d0809d6f00a34 Author: Poul-Henning Kamp Date: Mon Nov 14 17:25:15 2022 +0000 Silence FlexeLint diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index fe6fcb03d..75e66ce6d 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -192,8 +192,8 @@ http_hdr_flags(const char *b, const char *e) assert(b + u == e); if (u < GPERF_MIN_WORD_LENGTH || u > GPERF_MAX_WORD_LENGTH) return (NULL); - u += http_asso_values[((const uint8_t *)b)[u - 1]] + - http_asso_values[((const uint8_t *)b)[0]]; + u += http_asso_values[(uint8_t)(e[-1])] + + http_asso_values[(uint8_t)(b[0])]; if (u > GPERF_MAX_HASH_VALUE) return (NULL); retval = &http_hdrflg[u]; From phk at FreeBSD.org Tue Nov 15 08:31:06 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 15 Nov 2022 08:31:06 +0000 (UTC) Subject: [master] e9deae59d Use VSL_OVERHEAD instead of 2 Message-ID: <20221115083106.78ECB4B50@lists.varnish-cache.org> commit e9deae59d0b093be63b7d66bcb42b54060055cc1 Author: Poul-Henning Kamp Date: Tue Nov 15 08:29:59 2022 +0000 Use VSL_OVERHEAD instead of 2 diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c index 8c24a7521..cf0683606 100644 --- a/lib/libvarnishapi/vsl_dispatch.c +++ b/lib/libvarnishapi/vsl_dispatch.c @@ -93,7 +93,7 @@ struct synth { VTAILQ_ENTRY(synth) list; size_t offset; - uint32_t data[2 + 64 / sizeof (uint32_t)]; + uint32_t data[VSL_OVERHEAD + 64 / sizeof (uint32_t)]; }; VTAILQ_HEAD(synthhead, synth); @@ -1036,8 +1036,8 @@ vtx_synth_rec(struct vtx *vtx, unsigned tag, const char *fmt, ...) ALLOC_OBJ(synth, SYNTH_MAGIC); AN(synth); - buf = (char *)&synth->data[2]; - buflen = sizeof (synth->data) - 2 * sizeof (uint32_t); + buf = (char *)&synth->data[VSL_OVERHEAD]; + buflen = sizeof (synth->data) - VSL_OVERHEAD * sizeof (uint32_t); va_start(ap, fmt); l = vsnprintf(buf, buflen, fmt, ap); assert(l >= 0); From phk at FreeBSD.org Wed Nov 16 14:08:08 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 16 Nov 2022 14:08:08 +0000 (UTC) Subject: [master] 7da69d318 Increase the size of the VSL record header by one word, to make space for larger XIDs. Add a version field. Message-ID: <20221116140809.02B9F9525D@lists.varnish-cache.org> commit 7da69d318fdcf89cb5d6cc890c553244c7747bff Author: Poul-Henning Kamp Date: Wed Nov 16 14:06:51 2022 +0000 Increase the size of the VSL record header by one word, to make space for larger XIDs. Add a version field. diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index c166b5648..afb74535e 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -143,8 +143,11 @@ vsl_hdr(enum VSL_tag_e tag, uint32_t *p, unsigned len, vxid_t vxid) assert(tag < SLT__Reserved); AZ(len & ~VSL_LENMASK); + p[2] = 0; p[1] = vxid.vxid; - p[0] = ((((unsigned)tag & 0xff) << 24) | len); + p[0] = (((unsigned)tag & VSL_IDMASK) << VSL_IDSHIFT) | + (VSL_VERSION_3 << VSL_VERSHIFT) | + len; return (VSL_END(p, len)); } diff --git a/include/vapi/vsl_int.h b/include/vapi/vsl_int.h index dbd043908..faeb9d111 100644 --- a/include/vapi/vsl_int.h +++ b/include/vapi/vsl_int.h @@ -67,13 +67,20 @@ #define VSL_IDENTMASK (~(3U<<30)) #define VSL_LENMASK 0xffff -#define VSL_OVERHEAD 2 +#define VSL_VERMASK 0x3 +#define VSL_VERSHIFT 16 +#define VSL_IDMASK 0xff +#define VSL_IDSHIFT 24 +#define VSL_OVERHEAD 3 +#define VSL_VERSION_2 0x0 +#define VSL_VERSION_3 0x1 #define VSL_WORDS(len) (((len) + 3) / 4) #define VSL_BYTES(words) ((words) * 4) #define VSL_END(ptr, len) ((ptr) + VSL_OVERHEAD + VSL_WORDS(len)) #define VSL_NEXT(ptr) VSL_END(ptr, VSL_LEN(ptr)) #define VSL_LEN(ptr) ((ptr)[0] & VSL_LENMASK) -#define VSL_TAG(ptr) ((enum VSL_tag_e)((ptr)[0] >> 24)) +#define VSL_VER(ptr) (((ptr)[0] & VSL_VERMASK) >> VSL_VERSHIFT) +#define VSL_TAG(ptr) ((enum VSL_tag_e)((ptr)[0] >> VSL_IDSHIFT)) #define VSL_ID(ptr) (((ptr)[1]) & VSL_IDENTMASK) #define VSL_CLIENT(ptr) (((ptr)[1]) & VSL_CLIENTMARKER) #define VSL_BACKEND(ptr) (((ptr)[1]) & VSL_BACKENDMARKER) From phk at FreeBSD.org Wed Nov 16 16:55:06 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 16 Nov 2022 16:55:06 +0000 (UTC) Subject: [master] 4ed6181f8 Fix a wrap-around issue in buffered VSLs Message-ID: <20221116165506.D70CDA09A1@lists.varnish-cache.org> commit 4ed6181f8d505283c879d34ac9bcbdc07074cdd9 Author: Poul-Henning Kamp Date: Wed Nov 16 16:53:57 2022 +0000 Fix a wrap-around issue in buffered VSLs diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index afb74535e..7a217d6f8 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -151,6 +151,26 @@ vsl_hdr(enum VSL_tag_e tag, uint32_t *p, unsigned len, vxid_t vxid) return (VSL_END(p, len)); } +/*-------------------------------------------------------------------- + * Space available in a VSL buffer when accounting for overhead + */ + +static unsigned +vsl_space(const struct vsl_log *vsl) +{ + ptrdiff_t mlen; + + mlen = vsl->wle - vsl->wlp; + assert(mlen >= 0); + if (mlen < VSL_OVERHEAD + 1) + return (0); + mlen -= VSL_OVERHEAD; + mlen *= sizeof *vsl->wlp; + if (mlen > cache_param->vsl_reclen) + mlen = cache_param->vsl_reclen; + return(mlen); +} + /*-------------------------------------------------------------------- * Wrap the VSL buffer */ @@ -353,7 +373,7 @@ vslb_get(struct vsl_log *vsl, enum VSL_tag_e tag, unsigned *length) /* If it still doesn't fit, truncate */ if (VSL_END(vsl->wlp, mlen) > vsl->wle) - mlen = ((char *)vsl->wle) - VSL_DATA(vsl->wlp); + mlen = vsl_space(vsl); vsl->wlp = vsl_hdr(tag, vsl->wlp, mlen, vsl->wid); vsl->wlr++; @@ -448,7 +468,7 @@ VSLbv(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, va_list ap) vsl_sanity(vsl); - mlen = (char *)vsl->wle - VSL_DATA(vsl->wlp); + mlen = vsl_space(vsl); // First attempt, only if any space at all if (mlen > 0) { @@ -458,11 +478,10 @@ VSLbv(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, va_list ap) va_end(ap2); } - // Second attempt after a flush - if (mlen == 0 || n + 1 > mlen) { - // Second attempt after a flush + // Second attempt, if a flush might help + if (mlen == 0 || (n + 1 > mlen && n + 1 <= cache_param->vsl_reclen)) { VSL_Flush(vsl, 1); - mlen = (char *)vsl->wle - VSL_DATA(vsl->wlp); + mlen = vsl_space(vsl); p = VSL_DATA(vsl->wlp); n = vsnprintf(p, mlen, fmt, ap); } From dridi at varni.sh Wed Nov 16 17:12:53 2022 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 16 Nov 2022 17:12:53 +0000 Subject: [master] 4ed6181f8 Fix a wrap-around issue in buffered VSLs In-Reply-To: <20221116165506.D70CDA09A1@lists.varnish-cache.org> References: <20221116165506.D70CDA09A1@lists.varnish-cache.org> Message-ID: On Wed, Nov 16, 2022 at 4:55 PM Poul-Henning Kamp wrote: > > > commit 4ed6181f8d505283c879d34ac9bcbdc07074cdd9 > Author: Poul-Henning Kamp > Date: Wed Nov 16 16:53:57 2022 +0000 > > Fix a wrap-around issue in buffered VSLs > > diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c > index afb74535e..7a217d6f8 100644 > --- a/bin/varnishd/cache/cache_shmlog.c > +++ b/bin/varnishd/cache/cache_shmlog.c > @@ -151,6 +151,26 @@ vsl_hdr(enum VSL_tag_e tag, uint32_t *p, unsigned len, vxid_t vxid) > return (VSL_END(p, len)); > } > > +/*-------------------------------------------------------------------- > + * Space available in a VSL buffer when accounting for overhead > + */ > + > +static unsigned > +vsl_space(const struct vsl_log *vsl) Maybe rename to vslb_space for consistency? > +{ > + ptrdiff_t mlen; > + > + mlen = vsl->wle - vsl->wlp; > + assert(mlen >= 0); > + if (mlen < VSL_OVERHEAD + 1) > + return (0); > + mlen -= VSL_OVERHEAD; > + mlen *= sizeof *vsl->wlp; > + if (mlen > cache_param->vsl_reclen) > + mlen = cache_param->vsl_reclen; > + return(mlen); > +} > + > /*-------------------------------------------------------------------- > * Wrap the VSL buffer > */ > @@ -353,7 +373,7 @@ vslb_get(struct vsl_log *vsl, enum VSL_tag_e tag, unsigned *length) > > /* If it still doesn't fit, truncate */ > if (VSL_END(vsl->wlp, mlen) > vsl->wle) > - mlen = ((char *)vsl->wle) - VSL_DATA(vsl->wlp); > + mlen = vsl_space(vsl); > > vsl->wlp = vsl_hdr(tag, vsl->wlp, mlen, vsl->wid); > vsl->wlr++; > @@ -448,7 +468,7 @@ VSLbv(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, va_list ap) > > vsl_sanity(vsl); > > - mlen = (char *)vsl->wle - VSL_DATA(vsl->wlp); > + mlen = vsl_space(vsl); > > // First attempt, only if any space at all > if (mlen > 0) { > @@ -458,11 +478,10 @@ VSLbv(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, va_list ap) > va_end(ap2); > } > > - // Second attempt after a flush > - if (mlen == 0 || n + 1 > mlen) { > - // Second attempt after a flush > + // Second attempt, if a flush might help > + if (mlen == 0 || (n + 1 > mlen && n + 1 <= cache_param->vsl_reclen)) { > VSL_Flush(vsl, 1); > - mlen = (char *)vsl->wle - VSL_DATA(vsl->wlp); > + mlen = vsl_space(vsl); > p = VSL_DATA(vsl->wlp); > n = vsnprintf(p, mlen, fmt, ap); > } > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From phk at phk.freebsd.dk Wed Nov 16 17:29:36 2022 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Wed, 16 Nov 2022 17:29:36 +0000 Subject: [master] 4ed6181f8 Fix a wrap-around issue in buffered VSLs In-Reply-To: References: <20221116165506.D70CDA09A1@lists.varnish-cache.org> Message-ID: <202211161729.2AGHTahr095926@critter.freebsd.dk> -------- Dridi Boukelmoune writes: > On Wed, Nov 16, 2022 at 4:55 PM Poul-Henning Kamp wrote: > > +static unsigned > > +vsl_space(const struct vsl_log *vsl) > > Maybe rename to vslb_space for consistency? It is not specialized to vslb, it works on any vsl_log. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From dridi at varni.sh Wed Nov 16 17:42:29 2022 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 16 Nov 2022 17:42:29 +0000 Subject: [master] 4ed6181f8 Fix a wrap-around issue in buffered VSLs In-Reply-To: <202211161729.2AGHTahr095926@critter.freebsd.dk> References: <20221116165506.D70CDA09A1@lists.varnish-cache.org> <202211161729.2AGHTahr095926@critter.freebsd.dk> Message-ID: On Wed, Nov 16, 2022 at 5:29 PM Poul-Henning Kamp wrote: > > -------- > Dridi Boukelmoune writes: > > On Wed, Nov 16, 2022 at 4:55 PM Poul-Henning Kamp wrote: > > > > +static unsigned > > > +vsl_space(const struct vsl_log *vsl) > > > > Maybe rename to vslb_space for consistency? > > It is not specialized to vslb, it works on any vsl_log. I don't follow, either we have a vsl_log and it's buffered or we use the VSL() function and its variants. From phk at phk.freebsd.dk Wed Nov 16 17:49:57 2022 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Wed, 16 Nov 2022 17:49:57 +0000 Subject: [master] 4ed6181f8 Fix a wrap-around issue in buffered VSLs In-Reply-To: References: <20221116165506.D70CDA09A1@lists.varnish-cache.org> <202211161729.2AGHTahr095926@critter.freebsd.dk> Message-ID: <202211161749.2AGHnvDj096145@critter.freebsd.dk> -------- Dridi Boukelmoune writes: > On Wed, Nov 16, 2022 at 5:29 PM Poul-Henning Kamp wrote: > > > > -------- > > Dridi Boukelmoune writes: > > > On Wed, Nov 16, 2022 at 4:55 PM Poul-Henning Kamp wrote: > > > > > > +static unsigned > > > > +vsl_space(const struct vsl_log *vsl) > > > > > > Maybe rename to vslb_space for consistency? > > > > It is not specialized to vslb, it works on any vsl_log. > > I don't follow, either we have a vsl_log and it's buffered or we use > the VSL() function and its variants. Correct, but there may be future opportunities to use struct vsl_log also for the global log. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From dridi at varni.sh Wed Nov 16 17:53:05 2022 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 16 Nov 2022 17:53:05 +0000 Subject: [master] 4ed6181f8 Fix a wrap-around issue in buffered VSLs In-Reply-To: <202211161749.2AGHnvDj096145@critter.freebsd.dk> References: <20221116165506.D70CDA09A1@lists.varnish-cache.org> <202211161729.2AGHTahr095926@critter.freebsd.dk> <202211161749.2AGHnvDj096145@critter.freebsd.dk> Message-ID: On Wed, Nov 16, 2022 at 5:49 PM Poul-Henning Kamp wrote: > > -------- > Dridi Boukelmoune writes: > > On Wed, Nov 16, 2022 at 5:29 PM Poul-Henning Kamp wrote: > > > > > > -------- > > > Dridi Boukelmoune writes: > > > > On Wed, Nov 16, 2022 at 4:55 PM Poul-Henning Kamp wrote: > > > > > > > > +static unsigned > > > > > +vsl_space(const struct vsl_log *vsl) > > > > > > > > Maybe rename to vslb_space for consistency? > > > > > > It is not specialized to vslb, it works on any vsl_log. > > > > I don't follow, either we have a vsl_log and it's buffered or we use > > the VSL() function and its variants. > > Correct, but there may be future opportunities to use struct vsl_log > also for the global log. ACK From nils.goroll at uplex.de Sun Nov 20 19:18:07 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 20 Nov 2022 19:18:07 +0000 (UTC) Subject: [master] dfac1eeff Capture all of the child output when it fails to start Message-ID: <20221120191807.4A3816BCB@lists.varnish-cache.org> commit dfac1eeff899b2ab0ac63228412b270e319de8a8 Author: Nils Goroll Date: Sun Nov 20 20:15:10 2022 +0100 Capture all of the child output when it fails to start we only forwarded the result from a single read(), which might have missed information on the actual error. diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index da4308c81..2776f9de6 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -426,7 +426,9 @@ mgt_launch_child(struct cli *cli) assert(u == CLIS_COMMS); pidr = waitpid(pid, &i, 0); assert(pidr == pid); - (void)VLU_Fd(child_std_vlu, child_output); + do { + i = VLU_Fd(child_std_vlu, child_output); + } while (i == 0); MGT_Complain(C_ERR, "Child failed on launch"); exit(1); // XXX Harsh ? } else { From phk at FreeBSD.org Mon Nov 21 11:28:08 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 21 Nov 2022 11:28:08 +0000 (UTC) Subject: [master] 2190b5a27 Slightly longer delay to improve stability. Message-ID: <20221121112808.4B883AE8EA@lists.varnish-cache.org> commit 2190b5a2710b16bf92407f2030b388ec3ce4cbe8 Author: Poul-Henning Kamp Date: Mon Nov 21 11:27:34 2022 +0000 Slightly longer delay to improve stability. diff --git a/bin/varnishtest/tests/c00058.vtc b/bin/varnishtest/tests/c00058.vtc index 6a2d733d2..daf8548a0 100644 --- a/bin/varnishtest/tests/c00058.vtc +++ b/bin/varnishtest/tests/c00058.vtc @@ -36,7 +36,8 @@ client c1 { # But bg fetch was kicked off barrier b1 sync - delay .2 + + delay .5 # And now we get the new object txreq From phk at FreeBSD.org Mon Nov 21 12:49:05 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 21 Nov 2022 12:49:05 +0000 (UTC) Subject: [master] 3fa5baf1d Stabilize this test with a short delay Message-ID: <20221121124905.A015CB2325@lists.varnish-cache.org> commit 3fa5baf1d0014c712374b32383b884f66525cdaf Author: Poul-Henning Kamp Date: Mon Nov 21 11:35:16 2022 +0000 Stabilize this test with a short delay diff --git a/bin/varnishtest/tests/c00093.vtc b/bin/varnishtest/tests/c00093.vtc index 23a36ad36..002b68930 100644 --- a/bin/varnishtest/tests/c00093.vtc +++ b/bin/varnishtest/tests/c00093.vtc @@ -39,6 +39,8 @@ client c1 { logexpect l1 -wait +delay .5 + client c2 { txreq rxresp From phk at FreeBSD.org Mon Nov 21 12:49:05 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 21 Nov 2022 12:49:05 +0000 (UTC) Subject: [master] bfdd9ac76 Use barrier instead of delay to stabilize r01086 Message-ID: <20221121124905.CF60AB2328@lists.varnish-cache.org> commit bfdd9ac76ce1143322ee8bd0189c6545258ef30b Author: Poul-Henning Kamp Date: Mon Nov 21 12:47:45 2022 +0000 Use barrier instead of delay to stabilize r01086 diff --git a/bin/varnishtest/tests/r01086.vtc b/bin/varnishtest/tests/r01086.vtc index d15ef7afb..d5fca1602 100644 --- a/bin/varnishtest/tests/r01086.vtc +++ b/bin/varnishtest/tests/r01086.vtc @@ -1,5 +1,7 @@ varnishtest "#1086 junk after gzip from backend and streaming enabled" +barrier b1 cond 2 + server s1 { # This one will be streamed rxreq @@ -20,7 +22,7 @@ server s1 { sendhex "00000000" send "\r\n" - delay .2 + barrier b1 sync chunked "FOOBAR" non_fatal @@ -40,6 +42,7 @@ client c1 { txreq -hdr "Cookie: FOO" rxresphdrs expect resp.status == 200 + barrier b1 sync expect_close } -run From nils.goroll at uplex.de Mon Nov 21 16:06:07 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 21 Nov 2022 16:06:07 +0000 (UTC) Subject: [master] 871d00207 Wrap checks for UDS path Message-ID: <20221121160607.853AB74CD@lists.varnish-cache.org> commit 871d0020732e17642b445da8691f7dfe4efec5ea Author: Nils Goroll Date: Mon Nov 21 16:50:35 2022 +0100 Wrap checks for UDS path This commit was guided by the cocci patch below, but lines to change were manually selected. Motivated by #3864 --- 8< --- @@ expression path; @@ - *path == '/' + VUS_is(path) @@ expression path; @@ - *path != '/' + ! VUS_is(path) diff --git a/bin/varnishd/mgt/mgt_acceptor.c b/bin/varnishd/mgt/mgt_acceptor.c index 9cbeb81aa..c02941048 100644 --- a/bin/varnishd/mgt/mgt_acceptor.c +++ b/bin/varnishd/mgt/mgt_acceptor.c @@ -158,7 +158,7 @@ mk_listen_sock(const struct listen_arg *la, const struct suckaddr *sa) ls->name = la->name; ls->transport = la->transport; ls->perms = la->perms; - if (*la->endpoint == '/') + if (VUS_is(la->endpoint)) ls->uds = 1; VJ_master(JAIL_MASTER_PRIVPORT); fail = mac_opensocket(ls); @@ -273,7 +273,7 @@ MAC_Arg(const char *spec) ARGV_ERR("Unix domain socket addresses must be" " absolute paths in -a (%s)\n", la->endpoint); - if (*la->endpoint == '/' && heritage.min_vcl_version < 41) + if (VUS_is(la->endpoint) && heritage.min_vcl_version < 41) heritage.min_vcl_version = 41; for (int i = 2; av[i] != NULL; i++) { @@ -368,7 +368,7 @@ MAC_Arg(const char *spec) else AZ(la->perms); - if (*la->endpoint != '/') + if (! VUS_is(la->endpoint)) error = VSS_resolver(av[1], "80", mac_tcp, la, &err); else error = VUS_resolver(av[1], mac_uds, la, &err); diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 544359db9..50247a963 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -62,6 +62,7 @@ #include "vtim.h" #include "waiter/mgt_waiter.h" #include "vsa.h" +#include "vus.h" struct heritage heritage; unsigned d_flag = 0; @@ -483,7 +484,7 @@ mgt_b_conv(const char *b_arg) VSB_cat(vsb, "backend default "); if (!strcasecmp(b_arg, "none")) VSB_cat(vsb, "none;\n"); - else if (*b_arg != '/') + else if (! VUS_is(b_arg)) VSB_printf(vsb, "{\n .host = \"%s\";\n}\n", b_arg); else VSB_printf(vsb, "{\n .path = \"%s\";\n}\n", b_arg); diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c index 178aa1335..883aad017 100644 --- a/bin/varnishtest/vtc_client.c +++ b/bin/varnishtest/vtc_client.c @@ -186,7 +186,7 @@ client_connect(struct vtclog *vl, struct client *c) int fd; vtc_log(vl, 3, "Connect to %s", c->addr); - if (*c->addr == '/') + if (VUS_is(c->addr)) fd = client_uds_connect(vl, c->addr, 10., &err); else fd = client_tcp_connect(vl, c->addr, 10., &err); diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index 783be0a2e..437f685c6 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -216,7 +216,7 @@ server_listen(struct server *s) if (s->sock >= 0) VTCP_close(&s->sock); - if (*s->listen != '/') + if (! VUS_is(s->listen)) server_listen_tcp(s, &err); else server_listen_uds(s, &err); @@ -248,7 +248,7 @@ server_conn(void *priv, struct vtclog *vl) fd = accept(s->sock, addr, &l); if (fd < 0) vtc_fatal(vl, "Accept failed: %s", strerror(errno)); - if (*s->listen != '/') { + if (! VUS_is(s->listen)) { VTCP_hisname(fd, abuf, sizeof abuf, pbuf, sizeof pbuf); vtc_log(vl, 3, "accepted fd %d %s %s", fd, abuf, pbuf); } else @@ -431,7 +431,7 @@ cmd_server_gen_vcl(struct vsb *vsb) if (s->sock < 0 && s->fd >= 0) /* dispatch instance */ continue; - if (*s->listen != '/') + if (! VUS_is(s->listen)) VSB_printf(vsb, "backend %s { .host = \"%s\"; " ".port = \"%s\"; }\n", @@ -456,7 +456,7 @@ cmd_server_gen_haproxy_conf(struct vsb *vsb) AZ(pthread_mutex_lock(&server_mtx)); VTAILQ_FOREACH(s, &servers, list) { - if (*s->listen != '/') + if (! VUS_is(s->listen)) VSB_printf(vsb, "\n backend be%s\n" "\tserver srv%s %s:%s\n", @@ -465,7 +465,7 @@ cmd_server_gen_haproxy_conf(struct vsb *vsb) INCOMPL(); } VTAILQ_FOREACH(s, &servers, list) { - if (*s->listen != '/') + if (! VUS_is(s->listen)) VSB_printf(vsb, "\n frontend http%s\n" "\tuse_backend be%s\n" diff --git a/include/vus.h b/include/vus.h index 157b65102..c2daf8f85 100644 --- a/include/vus.h +++ b/include/vus.h @@ -36,3 +36,9 @@ int VUS_resolver(const char *path, vus_resolved_f *func, void *priv, const char **err); int VUS_bind(const struct sockaddr_un *uds, const char **errp); int VUS_connect(const char *path, int msec); + +static inline int +VUS_is(const char *path) +{ + return (*path == '/'); +} diff --git a/lib/libvarnish/vus.c b/lib/libvarnish/vus.c index 0f39fb601..07e3bf193 100644 --- a/lib/libvarnish/vus.c +++ b/lib/libvarnish/vus.c @@ -46,7 +46,7 @@ sun_init(struct sockaddr_un *uds, const char *path, const char **err) { AN(uds); AN(path); - assert(*path == '/'); + assert(VUS_is(path)); if (err) *err = NULL; diff --git a/vmod/vmod_debug_dyn.c b/vmod/vmod_debug_dyn.c index 6b562cfc7..8e9676c2c 100644 --- a/vmod/vmod_debug_dyn.c +++ b/vmod/vmod_debug_dyn.c @@ -41,6 +41,7 @@ #include "vsa.h" #include "vss.h" +#include "vus.h" #include "vcc_debug_if.h" struct xyzzy_debug_dyn { @@ -178,7 +179,7 @@ dyn_uds_init(VRT_CTX, struct xyzzy_debug_dyn_uds *uds, VCL_STRING path) VRT_fail(ctx, "path is NULL"); return (-1); } - if (*path != '/') { + if (! VUS_is(path)) { VRT_fail(ctx, "path must be an absolute path: %s", path); return (-1); } From nils.goroll at uplex.de Mon Nov 21 16:06:07 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 21 Nov 2022 16:06:07 +0000 (UTC) Subject: [master] 136bc58bc Polish VUS_is() call sites Message-ID: <20221121160607.980FF74D5@lists.varnish-cache.org> commit 136bc58bc8ff3f478bc47489bcc64c15c6b39695 Author: Nils Goroll Date: Mon Nov 21 17:02:46 2022 +0100 Polish VUS_is() call sites Motivated by #3864 diff --git a/bin/varnishd/mgt/mgt_acceptor.c b/bin/varnishd/mgt/mgt_acceptor.c index c02941048..754911a23 100644 --- a/bin/varnishd/mgt/mgt_acceptor.c +++ b/bin/varnishd/mgt/mgt_acceptor.c @@ -158,8 +158,7 @@ mk_listen_sock(const struct listen_arg *la, const struct suckaddr *sa) ls->name = la->name; ls->transport = la->transport; ls->perms = la->perms; - if (VUS_is(la->endpoint)) - ls->uds = 1; + ls->uds = VUS_is(la->endpoint); VJ_master(JAIL_MASTER_PRIVPORT); fail = mac_opensocket(ls); VJ_master(JAIL_MASTER_LOW); @@ -368,10 +367,10 @@ MAC_Arg(const char *spec) else AZ(la->perms); - if (! VUS_is(la->endpoint)) - error = VSS_resolver(av[1], "80", mac_tcp, la, &err); - else + if (VUS_is(la->endpoint)) error = VUS_resolver(av[1], mac_uds, la, &err); + else + error = VSS_resolver(av[1], "80", mac_tcp, la, &err); if (VTAILQ_EMPTY(&la->socks) || error) ARGV_ERR("Got no socket(s) for %s\n", av[1]); diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 50247a963..019da8d02 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -484,10 +484,10 @@ mgt_b_conv(const char *b_arg) VSB_cat(vsb, "backend default "); if (!strcasecmp(b_arg, "none")) VSB_cat(vsb, "none;\n"); - else if (! VUS_is(b_arg)) - VSB_printf(vsb, "{\n .host = \"%s\";\n}\n", b_arg); - else + else if (VUS_is(b_arg)) VSB_printf(vsb, "{\n .path = \"%s\";\n}\n", b_arg); + else + VSB_printf(vsb, "{\n .host = \"%s\";\n}\n", b_arg); AZ(VSB_finish(vsb)); fa->src = strdup(VSB_data(vsb)); AN(fa->src); diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index 437f685c6..08103a600 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -216,10 +216,10 @@ server_listen(struct server *s) if (s->sock >= 0) VTCP_close(&s->sock); - if (! VUS_is(s->listen)) - server_listen_tcp(s, &err); - else + if (VUS_is(s->listen)) server_listen_uds(s, &err); + else + server_listen_tcp(s, &err); if (err != NULL) vtc_fatal(s->vl, "Server listen address (%s) cannot be resolved: %s", @@ -248,11 +248,12 @@ server_conn(void *priv, struct vtclog *vl) fd = accept(s->sock, addr, &l); if (fd < 0) vtc_fatal(vl, "Accept failed: %s", strerror(errno)); - if (! VUS_is(s->listen)) { + if (VUS_is(s->listen)) + vtc_log(vl, 3, "accepted fd %d 0.0.0.0 0", fd); + else { VTCP_hisname(fd, abuf, sizeof abuf, pbuf, sizeof pbuf); vtc_log(vl, 3, "accepted fd %d %s %s", fd, abuf, pbuf); - } else - vtc_log(vl, 3, "accepted fd %d 0.0.0.0 0", fd); + } return (fd); } @@ -431,15 +432,15 @@ cmd_server_gen_vcl(struct vsb *vsb) if (s->sock < 0 && s->fd >= 0) /* dispatch instance */ continue; - if (! VUS_is(s->listen)) + if (VUS_is(s->listen)) + VSB_printf(vsb, + "backend %s { .path = \"%s\"; }\n", + s->name, s->listen); + else VSB_printf(vsb, "backend %s { .host = \"%s\"; " ".port = \"%s\"; }\n", s->name, s->aaddr, s->aport); - else - VSB_printf(vsb, - "backend %s { .path = \"%s\"; }\n", - s->name, s->listen); } AZ(pthread_mutex_unlock(&server_mtx)); } From nils.goroll at uplex.de Mon Nov 21 17:11:05 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 21 Nov 2022 17:11:05 +0000 (UTC) Subject: [master] 616bf3bf2 Add basic support of abstract sockets for accept and .path to backends Message-ID: <20221121171105.B5943619BC@lists.varnish-cache.org> commit 616bf3bf2e29f4011d4a8e5e66f59cba553199a2 Author: Nils Goroll Date: Mon Nov 14 18:11:18 2022 +0100 Add basic support of abstract sockets for accept and .path to backends We use the commonplace @ syntax to specify abstract socket names. Implements #3863 diff --git a/bin/varnishd/mgt/mgt_acceptor.c b/bin/varnishd/mgt/mgt_acceptor.c index 754911a23..9b649a7b9 100644 --- a/bin/varnishd/mgt/mgt_acceptor.c +++ b/bin/varnishd/mgt/mgt_acceptor.c @@ -221,9 +221,10 @@ mac_uds(void *priv, const struct sockaddr_un *uds) struct listen_sock *ls; CAST_OBJ_NOTNULL(la, priv, LISTEN_ARG_MAGIC); + (void) uds; VTAILQ_FOREACH(ls, &heritage.socks, list) { - if (ls->uds && strcmp(uds->sun_path, ls->endpoint) == 0) + if (ls->uds && strcmp(ls->endpoint, la->endpoint) == 0) ARGV_ERR("-a arguments %s and %s have same address\n", ls->endpoint, la->endpoint); } @@ -289,7 +290,7 @@ MAC_Arg(const char *spec) continue; } if (la->endpoint[0] != '/') - ARGV_ERR("Invalid sub-arg %s for IP addresses" + ARGV_ERR("Invalid sub-arg %s" " in -a\n", av[i]); val = eq + 1; diff --git a/bin/varnishtest/tests/c00003.vtc b/bin/varnishtest/tests/c00003.vtc index 5c3b7fabd..1ad256922 100644 --- a/bin/varnishtest/tests/c00003.vtc +++ b/bin/varnishtest/tests/c00003.vtc @@ -33,18 +33,22 @@ shell -err -expect "Unix domain socket addresses must be absolute paths" { } # -a args for UDS permissions not permitted with IP addresses -shell -err -expect "Invalid sub-arg user=u for IP addresses" { +shell -err -expect "Invalid sub-arg user=u" { varnishd -a ${localhost}:80000,user=u -d } -shell -err -expect "Invalid sub-arg group=g for IP addresses" { +shell -err -expect "Invalid sub-arg group=g" { varnishd -a ${localhost}:80000,group=g -d } -shell -err -expect "Invalid sub-arg mode=660 for IP addresses" { +shell -err -expect "Invalid sub-arg mode=660" { varnishd -a ${localhost}:80000,mode=660 -d } +shell -err -expect "Invalid sub-arg mode=660" { + varnishd -a @abstract,mode=660 -d +} + # Illegal mode sub-args shell -err -expect "Too many mode sub-args" { varnishd -a ${tmpdir}/vtc.sock,mode=660,mode=600 -d diff --git a/bin/varnishtest/tests/c00121.vtc b/bin/varnishtest/tests/c00121.vtc new file mode 100644 index 000000000..95b6aa14c --- /dev/null +++ b/bin/varnishtest/tests/c00121.vtc @@ -0,0 +1,80 @@ +varnishtest "Abstract UDS backend: change path, drop poll" + +# XXX better test to check for abstract sockets +feature cmd {socat abstract-listen:test file:/dev/null & date | socat fd:0 abstract-client:test} + +server s1 -listen "@vtc.s1.sock" { + non_fatal + timeout 3 + loop 40 { + rxreq + txresp + accept + } +} -start + +server s2 -listen "@vtc.s2.sock" { + non_fatal + timeout 3 + loop 40 { + rxreq + txresp + accept + } +} -start + +varnish v1 -arg "-a @vtc.v1.sock" -vcl { + probe default { + .window = 8; + .initial = 7; + .threshold = 8; + .interval = 0.1s; + } + backend s1 { + .path = "@vtc.s2.sock"; + } +} -start + +delay 1 + +varnish v1 -vcl { + probe default { + .window = 8; + .initial = 7; + .threshold = 8; + .interval = 0.1s; + } + backend s1 { + .path = "@vtc.s1.sock"; + } +} -cliok "vcl.use vcl2" -cliok "vcl.discard vcl1" + +delay 1 + +varnish v1 -vcl { + backend s1 { + .path = "@vtc.s1.sock"; + } +} -cliok "vcl.use vcl3" -cliok "vcl.discard vcl2" + +delay 1 + +varnish v1 -cliok "vcl.list" +varnish v1 -cliok "backend.list -p" + +server s1 -break { + rxreq + expect req.url == /foo + txresp -bodylen 4 +} -start + +delay 1 + +client c1 -connect "@vtc.v1.sock" { + txreq -url /foo + rxresp + txreq -url /foo + rxresp + expect resp.status == 200 + expect resp.bodylen == 4 +} -run diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 144910516..12bd0425f 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -93,11 +93,13 @@ Basic options (VCL4.1 and higher) Accept connections on a Unix domain socket. Path must be absolute - ("/path/to/listen.sock"). + ("/path/to/listen.sock") or "@" followed by the name of an abstract + socket ("@myvarnishd"). The user, group and mode sub-arguments may be used to specify the permissions of the socket file -- use names for user and group, and - a 3-digit octal value for mode. + a 3-digit octal value for mode. These sub-arguments do not apply to + abstract sockets. -b <[host[:port]|path]> diff --git a/doc/sphinx/reference/vcl-backend.rst b/doc/sphinx/reference/vcl-backend.rst index a1f66e9f7..048f25cdd 100644 --- a/doc/sphinx/reference/vcl-backend.rst +++ b/doc/sphinx/reference/vcl-backend.rst @@ -76,6 +76,11 @@ The absolute path to a Unix(4) domain socket of a local backend:: .path = "/var/run/http.sock"; +or, where available, ``@`` followed by the name of an abstract socket +of a local backend:: + + .path = "@mybackend"; + A warning will be issued if the uds-socket does not exist when the VCL is loaded. This makes it possible to start the UDS-listening peer, or set the socket file's permissions afterwards. diff --git a/include/vus.h b/include/vus.h index c2daf8f85..20d56fd15 100644 --- a/include/vus.h +++ b/include/vus.h @@ -40,5 +40,5 @@ int VUS_connect(const char *path, int msec); static inline int VUS_is(const char *path) { - return (*path == '/'); + return (*path == '/' || *path == '@'); } diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c index 613555e3f..d1f1c91a0 100644 --- a/lib/libvarnish/vsa.c +++ b/lib/libvarnish/vsa.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include @@ -229,11 +230,14 @@ VSA_GetPtr(const struct suckaddr *sua, const unsigned char ** dst) static inline socklen_t sua_len(const struct sockaddr *sa) { + switch (sa->sa_family) { case PF_INET: return (sizeof(struct sockaddr_in)); case PF_INET6: return (sizeof(struct sockaddr_in6)); + case AF_UNIX: + return (sizeof(struct sockaddr_un)); default: return (0); } diff --git a/lib/libvarnish/vus.c b/lib/libvarnish/vus.c index 07e3bf193..f1b943649 100644 --- a/lib/libvarnish/vus.c +++ b/lib/libvarnish/vus.c @@ -58,7 +58,10 @@ sun_init(struct sockaddr_un *uds, const char *path, const char **err) return (-1); } memset(uds->sun_path, 0, sizeof(uds->sun_path)); - bprintf(uds->sun_path, "%s", path); + if (*path == '@') + bprintf(uds->sun_path, "%c%s", 0, path + 1); + else + bprintf(uds->sun_path, "%s", path); uds->sun_family = PF_UNIX; return (0); } diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c index 8b357a58e..41b6d3739 100644 --- a/lib/libvcc/vcc_backend.c +++ b/lib/libvcc/vcc_backend.c @@ -91,6 +91,14 @@ Emit_Sockaddr(struct vcc *tl, struct vsb *vsb1, const struct token *t_host, * be accessed, and is a socket. If so, just emit the path field and set * the IP suckaddrs to NULL. */ +static void +emit_path(struct vsb *vsb1, char *path) +{ + VSB_printf(vsb1, "\t.uds_path = \"%s\",\n", path); + VSB_cat(vsb1, "\t.ipv4 = (void *) 0,\n"); + VSB_cat(vsb1, "\t.ipv6 = (void *) 0,\n"); +} + static void Emit_UDS_Path(struct vcc *tl, struct vsb *vsb1, const struct token *t_path, const char *errid) @@ -100,6 +108,10 @@ Emit_UDS_Path(struct vcc *tl, struct vsb *vsb1, AN(t_path); AN(t_path->dec); + if (*t_path->dec == '@') { + emit_path(vsb1, t_path->dec); + return; + } if (*t_path->dec != '/') { VSB_printf(tl->sb, "%s: Must be an absolute path:\n", errid); @@ -121,9 +133,7 @@ Emit_UDS_Path(struct vcc *tl, struct vsb *vsb1, vcc_ErrWhere(tl, t_path); return; } - VSB_printf(vsb1, "\t.uds_path = \"%s\",\n", t_path->dec); - VSB_cat(vsb1, "\t.ipv4 = (void *) 0,\n"); - VSB_cat(vsb1, "\t.ipv6 = (void *) 0,\n"); + emit_path(vsb1, t_path->dec); } /*-------------------------------------------------------------------- From nils.goroll at uplex.de Mon Nov 21 17:11:05 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 21 Nov 2022 17:11:05 +0000 (UTC) Subject: [master] 027296d87 Add vtc feature abstract_uds Message-ID: <20221121171105.D8BE1619C0@lists.varnish-cache.org> commit 027296d877664bb7b79469b255091d2f09aa4282 Author: Nils Goroll Date: Mon Nov 21 17:27:50 2022 +0100 Add vtc feature abstract_uds diff --git a/bin/varnishtest/tests/c00121.vtc b/bin/varnishtest/tests/c00121.vtc index 95b6aa14c..3155e957e 100644 --- a/bin/varnishtest/tests/c00121.vtc +++ b/bin/varnishtest/tests/c00121.vtc @@ -1,7 +1,6 @@ varnishtest "Abstract UDS backend: change path, drop poll" -# XXX better test to check for abstract sockets -feature cmd {socat abstract-listen:test file:/dev/null & date | socat fd:0 abstract-client:test} +feature abstract_uds server s1 -listen "@vtc.s1.sock" { non_fatal diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index 8c90cf438..10875ecd9 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -39,6 +39,7 @@ #include #include #include +#include #include #ifdef HAVE_SYS_PERSONALITY_H @@ -52,6 +53,7 @@ #include "vtcp.h" #include "vss.h" #include "vtim.h" +#include "vus.h" /* SECTION: vtest vtest * @@ -411,6 +413,28 @@ addr_no_randomize_works(void) return (r >= 0); } +/**********************************************************************/ + +static int +uds_socket(void *priv, const struct sockaddr_un *uds) +{ + + (void) priv; + return (socket(uds->sun_family, SOCK_STREAM, 0)); +} +static int +abstract_uds_works(void) +{ + const char *err; + int fd; + + fd = VUS_resolver("@vtc.feature.abstract_uds", uds_socket, NULL, &err); + if (fd < 0) + return (0); + close(fd); + return (1); +} + /* SECTION: feature feature * * Test that the required feature(s) for a test are available, and skip @@ -456,6 +480,8 @@ addr_no_randomize_works(void) * Varnish was built with a sanitizer. * workspace_emulator * Varnish was built with its workspace emulator. + * abstract_uds + * Creation of an abstract unix domain socket succeeded * * A feature name can be prefixed with an exclamation mark (!) to skip a * test if the feature is present. @@ -566,6 +592,7 @@ cmd_feature(CMD_ARGS) FEATURE("ubsan", ubsan); FEATURE("sanitizer", sanitizer); FEATURE("workspace_emulator", workspace_emulator); + FEATURE("abstract_uds", abstract_uds_works()); if (!strcmp(feat, "cmd")) { good = 1; From nils.goroll at uplex.de Mon Nov 21 17:11:05 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 21 Nov 2022 17:11:05 +0000 (UTC) Subject: [master] 105e2b306 Check for the empty abstract socket name Message-ID: <20221121171105.EFE6A619C7@lists.varnish-cache.org> commit 105e2b3064ac9a397a1c75e9e48a4623094b4c38 Author: Nils Goroll Date: Mon Nov 21 17:15:29 2022 +0100 Check for the empty abstract socket name As pointed out by Dridi, we should rather not make an attempt to support un-printable socket names, of which the empty name is the most prominent case. For all other non-printable cases (e.g. d\0r\0i\0d\0i), we have no support to pass them in the first place, because we treat uds paths as NUL-terminated strings. diff --git a/lib/libvarnish/vus.c b/lib/libvarnish/vus.c index f1b943649..c9920ef28 100644 --- a/lib/libvarnish/vus.c +++ b/lib/libvarnish/vus.c @@ -57,6 +57,13 @@ sun_init(struct sockaddr_un *uds, const char *path, const char **err) *err = "Path too long for a Unix domain socket"; return (-1); } + if (! strcmp(path, "@")) { + errno = EINVAL; + if (err) + *err = "The empty abstract socket name is not" + " supported"; + return (-1); + } memset(uds->sun_path, 0, sizeof(uds->sun_path)); if (*path == '@') bprintf(uds->sun_path, "%c%s", 0, path + 1); From nils.goroll at uplex.de Mon Nov 21 17:11:06 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 21 Nov 2022 17:11:06 +0000 (UTC) Subject: [master] f16ebb924 Use VUS_resolver() checks also for VCC Message-ID: <20221121171106.1C948619CD@lists.varnish-cache.org> commit f16ebb9240143da89e9aa012a0bffc7bb0f64e9c Author: Nils Goroll Date: Mon Nov 21 17:59:11 2022 +0100 Use VUS_resolver() checks also for VCC diff --git a/bin/varnishtest/tests/v00038.vtc b/bin/varnishtest/tests/v00038.vtc index eca463012..c7c4d61ed 100644 --- a/bin/varnishtest/tests/v00038.vtc +++ b/bin/varnishtest/tests/v00038.vtc @@ -117,12 +117,18 @@ varnish v1 -errvcl "Address redefinition at:" { } } -varnish v1 -errvcl "Must be an absolute path:" { +varnish v1 -errvcl "Must be a valid path or abstract socket:" { backend b1 { .path = "server.sock"; } } +varnish v1 -errvcl "Path too long for a Unix domain socket" { + backend b1 { + .path = "/this/super/long/path/this/super/long/path/this/super/long/path/this/super/long/path/this/super/long/path/this/super/long/path/this/super/long/path/this/super/long/path/this/super/long/path/this/super/long/path/this/super/long/path/this/super/long/path/this/super/long/path/this/super/long/path/this/super/long/path/this/super/long/path/this/super/long/path/this/super/long/path/this/super/long/path/this/super/long/path/this/super/long/path/this/super/long/path/this/super/long/path"; + } +} + varnish v1 -errvcl "Not a socket:" { backend b1 { .path = "${tmpdir}"; diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c index 41b6d3739..e4ccde185 100644 --- a/lib/libvcc/vcc_backend.c +++ b/lib/libvcc/vcc_backend.c @@ -37,6 +37,7 @@ #include #include "vcc_compile.h" +#include "vus.h" const char * vcc_default_probe(struct vcc *tl) @@ -87,10 +88,19 @@ Emit_Sockaddr(struct vcc *tl, struct vsb *vsb1, const struct token *t_host, } /* - * For UDS, we do not create a VSA. Check if it's an absolute path, can - * be accessed, and is a socket. If so, just emit the path field and set - * the IP suckaddrs to NULL. + * For UDS, we do not create a VSA. We run the VUS_resolver() checks and, if + * it's a path, can be accessed, and is a socket. If so, just emit the path + * field and set the IP suckaddrs to NULL. */ + +static int +uds_resolved(void *priv, const struct sockaddr_un *uds) +{ + (void) priv; + (void) uds; + return (42); +} + static void emit_path(struct vsb *vsb1, char *path) { @@ -104,20 +114,28 @@ Emit_UDS_Path(struct vcc *tl, struct vsb *vsb1, const struct token *t_path, const char *errid) { struct stat st; + const char *vus_err; AN(t_path); AN(t_path->dec); - if (*t_path->dec == '@') { - emit_path(vsb1, t_path->dec); + if (! VUS_is(t_path->dec)) { + VSB_printf(tl->sb, + "%s: Must be a valid path or abstract socket:\n", + errid); + vcc_ErrWhere(tl, t_path); return; } - if (*t_path->dec != '/') { - VSB_printf(tl->sb, - "%s: Must be an absolute path:\n", errid); + if (VUS_resolver(t_path->dec, uds_resolved, NULL, &vus_err) != 42) { + VSB_printf(tl->sb, "%s: %s\n", errid, vus_err); vcc_ErrWhere(tl, t_path); return; } + if (*t_path->dec == '@') { + emit_path(vsb1, t_path->dec); + return; + } + assert(*t_path->dec == '/'); errno = 0; if (stat(t_path->dec, &st) != 0) { int err = errno; From nils.goroll at uplex.de Mon Nov 21 17:11:06 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 21 Nov 2022 17:11:06 +0000 (UTC) Subject: [master] 81c913c83 Abstract socket coverage tests Message-ID: <20221121171106.34594619D3@lists.varnish-cache.org> commit 81c913c833eb24396a8af761195f7225465312af Author: Nils Goroll Date: Mon Nov 21 17:37:15 2022 +0100 Abstract socket coverage tests diff --git a/bin/varnishtest/tests/c00121.vtc b/bin/varnishtest/tests/c00121.vtc index 3155e957e..002a9aca5 100644 --- a/bin/varnishtest/tests/c00121.vtc +++ b/bin/varnishtest/tests/c00121.vtc @@ -77,3 +77,15 @@ client c1 -connect "@vtc.v1.sock" { expect resp.status == 200 expect resp.bodylen == 4 } -run + +######################################## +# coverage + +varnish v1 -errvcl {Backend path: The empty abstract socket name is not supported} { + backend s1 { + .path = "@"; + } +} + +shell -err -expect "Error: Got no socket(s) for @" \ + "varnishd -n ${tmpdir}/v0 -a @ -b None" \ No newline at end of file From nils.goroll at uplex.de Mon Nov 21 17:41:05 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 21 Nov 2022 17:41:05 +0000 (UTC) Subject: [master] 416234dc7 Fix vtc feature abstract_uds Message-ID: <20221121174105.45614631E8@lists.varnish-cache.org> commit 416234dc744d38bf4650b48b6ab669bc18dcd6fc Author: Nils Goroll Date: Mon Nov 21 18:40:02 2022 +0100 Fix vtc feature abstract_uds diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index 10875ecd9..78ec28464 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -419,8 +419,7 @@ static int uds_socket(void *priv, const struct sockaddr_un *uds) { - (void) priv; - return (socket(uds->sun_family, SOCK_STREAM, 0)); + return (VUS_bind(uds, priv)); } static int abstract_uds_works(void) @@ -431,7 +430,7 @@ abstract_uds_works(void) fd = VUS_resolver("@vtc.feature.abstract_uds", uds_socket, NULL, &err); if (fd < 0) return (0); - close(fd); + AZ(close(fd)); return (1); } From dridi.boukelmoune at gmail.com Tue Nov 22 10:17:05 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 22 Nov 2022 10:17:05 +0000 (UTC) Subject: [master] e3e2ef33a vslc: Plug mmap file descriptor leak Message-ID: <20221122101705.51177522C@lists.varnish-cache.org> commit e3e2ef33a0455a3b8b7ede38d1bcef73f18ffd61 Author: Dridi Boukelmoune Date: Thu Nov 10 14:58:27 2022 +0100 vslc: Plug mmap file descriptor leak To be consistent with how the file cursor behaves, the close_fd field is duplicated in the mmap cursor. If a VUT replaces stdin's file descriptor with a regular file's fd using dup2(2), we don't want to close it just because we managed to mmap(2) it. For some reason we don't use the closefd() macro in the VSL cursor code, potentially to avoid its underlying assertion in libvarnishapi. On the other hand we do use it in other places: $ git grep -l closefd -- lib/libvarnishapi/ lib/libvarnishapi/daemon.c lib/libvarnishapi/vsm.c So maybe in a subsequent change `(void)close(fd)` statements could turn into `closefd(&fd)` in vsl_cursor.c to harden those code paths as well. diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c index 3d356ad1c..8cf259394 100644 --- a/lib/libvarnishapi/vsl_cursor.c +++ b/lib/libvarnishapi/vsl_cursor.c @@ -409,6 +409,7 @@ struct vslc_mmap { unsigned magic; #define VSLC_MMAP_MAGIC 0x7de15f61 int fd; + int close_fd; char *b; char *e; struct VSL_cursor cursor; @@ -424,6 +425,8 @@ vslc_mmap_delete(const struct VSL_cursor *cursor) CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_MMAP_MAGIC); assert(&c->cursor == cursor); AZ(munmap(c->b, c->e - c->b)); + if (c->close_fd) + (void)close(c->fd); FREE_OBJ(c); } @@ -480,7 +483,7 @@ static const struct vslc_tbl vslc_mmap_tbl = { }; static struct VSL_cursor * -vsl_cursor_mmap(struct VSL_data *vsl, int fd) +vsl_cursor_mmap(struct VSL_data *vsl, int fd, int close_fd) { struct vslc_mmap *c; struct stat st[1]; @@ -500,7 +503,8 @@ vsl_cursor_mmap(struct VSL_data *vsl, int fd) ALLOC_OBJ(c, VSLC_MMAP_MAGIC); if (c == NULL) { (void)munmap(p, st->st_size); - (void)close(fd); + if (close_fd) + (void)close(fd); vsl_diag(vsl, "Out of memory"); return (NULL); } @@ -508,6 +512,7 @@ vsl_cursor_mmap(struct VSL_data *vsl, int fd) c->cursor.priv_data = c; c->fd = fd; + c->close_fd = close_fd; c->b = p; c->e = c->b + st->st_size; c->next.ptr = TRUST_ME(c->b + sizeof VSL_FILE_ID); @@ -557,7 +562,7 @@ VSL_CursorFile(struct VSL_data *vsl, const char *name, unsigned options) return (NULL); } - mc = vsl_cursor_mmap(vsl, fd); + mc = vsl_cursor_mmap(vsl, fd, close_fd); if (mc == NULL) return (NULL); if (mc != MAP_FAILED) From dridi.boukelmoune at gmail.com Tue Nov 22 10:20:06 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 22 Nov 2022 10:20:06 +0000 (UTC) Subject: [master] 647e1f2a9 vtc_gzip: Add the payload length in error messages Message-ID: <20221122102006.66937553B@lists.varnish-cache.org> commit 647e1f2a918d1ffeec90a6f7fb57fecd50bfc2b1 Author: Dridi Boukelmoune Date: Tue Nov 22 11:17:48 2022 +0100 vtc_gzip: Add the payload length in error messages As a convenience, the error message includes the actual total length of data being g[un]zipped, in addition to the total_in amount of data processed so far. diff --git a/bin/varnishtest/vtc_gzip.c b/bin/varnishtest/vtc_gzip.c index c06e0a66e..02700bf69 100644 --- a/bin/varnishtest/vtc_gzip.c +++ b/bin/varnishtest/vtc_gzip.c @@ -89,8 +89,9 @@ vtc_gzip_vsb(struct vtclog *vl, int fatal, int gzip_level, const struct vsb *vin } while (i == Z_OK || i == Z_BUF_ERROR); if (i != Z_STREAM_END) vtc_log(vl, fatal, - "Gzip error = %d (%s) in:%jd out:%jd", - i, vz.msg, (intmax_t)vz.total_in, (intmax_t)vz.total_out); + "Gzip error = %d (%s) in:%jd out:%jd len:%zd", + i, vz.msg, (intmax_t)vz.total_in, (intmax_t)vz.total_out, + VSB_len(vin)); AZ(VSB_finish(vout)); #ifdef VGZ_EXTENSIONS *residual = vz.stop_bit & 7; @@ -157,8 +158,9 @@ vtc_gunzip_vsb(struct vtclog *vl, int fatal, const struct vsb *vin) } while (i == Z_OK || i == Z_BUF_ERROR); if (i != Z_STREAM_END) vtc_log(vl, fatal, - "Gunzip error = %d (%s) in:%jd out:%jd", - i, vz.msg, (intmax_t)vz.total_in, (intmax_t)vz.total_out); + "Gunzip error = %d (%s) in:%jd out:%jd len:%zd", + i, vz.msg, (intmax_t)vz.total_in, (intmax_t)vz.total_out, + VSB_len(vin)); AZ(VSB_finish(vout)); #ifdef VGZ_EXTENSIONS vtc_report_gz_bits(vl, &vz); From phk at FreeBSD.org Fri Nov 25 21:07:08 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 25 Nov 2022 21:07:08 +0000 (UTC) Subject: [master] 896cd93da First installment of "When Varnish met CHERI" Message-ID: <20221125210708.E64F6BEAA8@lists.varnish-cache.org> commit 896cd93da96e1ce0298d5b854688bf94c961ffbd Author: Poul-Henning Kamp Date: Fri Nov 25 21:06:39 2022 +0000 First installment of "When Varnish met CHERI" diff --git a/doc/sphinx/phk/cheri1.rst b/doc/sphinx/phk/cheri1.rst new file mode 100644 index 000000000..fddb7939c --- /dev/null +++ b/doc/sphinx/phk/cheri1.rst @@ -0,0 +1,182 @@ +.. _phk_cheri_1: + +How Varnish met CHERI 1/N +========================= + +I should warn you up front that Robert Watson has been my friend +for a couple of decades, so I am by no means a neutral observer. + +But Robert is one of the smartest people I know, and when he first +explained his ideas for hardware capabilities, I sold all my shares +in sliced bread the very next morning. + +Roberts ideas grew to become CHERI, and if you have not heard about +it yet, you should read up on it, now: + +https://www.cl.cam.ac.uk/research/security/ctsrd/cheri/ + +The core idea in CHERI is that pointers are not integers, whicb +means that you cannot randomly make up or modify pointers to point +at random things any more, whatever your intentions might be. + +From a security point of view, this circumscribes several large +classes of commonly used attack-vectors, and Microsoft Research +found that CHERI stopped a full 43% of all the vulnerabilities they +saw in 2019: + +https://msrc-blog.microsoft.com/2022/01/20/an_armful_of_cheris/ + +Yes, we can pause while you sell all your shares in sliced bread. + +I have been eagerly waiting to see how my own Varnish HTTP Cache +Software would fare under CHERI, because one of my goals with the +Varnish software, was to turn the quality dial up to 11 to see if +it made any real-life difference. + +Robert has graciously lent me a shell-account on one of his shiny +new MORELLO machines, which rock an ARM64 prototype CPU with CHERI +capabilites. + +In this sequence of rants I will sing the saga of How Varnish meets +CHERI, as it happens. + +My hope is that it will inspire and help other software projects +to take the CHERI-plunge, and to help convince ARM that "MORELLO" +should become a permanent part of the ARM architecture. + +A very thin layer of Varnish +---------------------------- + +For those of you not familiar with Varnish, you will need to know: + +* Varnish is an afterburner cache to HTTP webservers, written in + C and it clocks in around 100KLOC. + +* Around 20% of all webtraffic in the world pass through a Varnish instance. + +* You configure Varnish with a domain-specific programming + language called VCL, which is translated to C source code, + compiled to a shared library, dlopen(3)'ed and executed. + +* Varnish runs as two processes, a "manager" and a "child". + The child process does not ``exec(2)`` after the ``fork(2)``. + +* The source code is written in a very paranoid style, around 10% + of all lines are asserts, and there are other layers of paranoia on + top of that, for instance we always check that ``close(2)`` returns zero. + +* We have 900+ test cases, which exercise 90%+ of our source lines. + +* In 16 years, we have had just a single "Oh Shit!" security issue. + +I still hate Autocrap tools +--------------------------- + +Autocrap is a hack on a hack on hack which really ruins software +portability, but it is the "industry standard" so we use it also +for Varnish, no matter how much I hate it. + +https://dl.acm.org/doi/abs/10.1145/2346916.2349257 + +Because a lot of software does not work in CHERI mode, there are two +kinds of packages for CheriBSD: Regular and Cheri. + +https://ctsrd-cheri.github.io/cheribsd-getting-started/packages/index.html + +Autocrap does not grok that some packages end up in /usr/local and +some in /usr/local64, so the first thing I had to do, was to explain +this:: + + export LIBTOOLIZE=/usr/local64/bin/libtoolize + export PCRE2_LIBS=`/usr/local/bin/pcre2-config --libs8` + export PCRE2_CFLAGS=`/usr/local/bin/pcre2-config --cflags` + ${SRCDIR}/configure \ + [the usual options] + +Things you just can't do with CHERI +----------------------------------- + +A long long time ago, I wrote a "persistent storage" module for +Varnish, and in order to not rewrite too much of the rest of +the source code, the architecture ended up files ``mmap(2)``'ed to +a consistent address, containing data structures with pointers. + +The advent of memory space layout randomization as a bandaid for +buffer-overflows (dont get me started!), made that architecture +unworkable, and for close to a decade this "stevedore" has been +named ``deprecated_persistent``. + +We still keep it around, because it helps us test the internal APIs, +but it is obviously not going to work with CHERI so:: + + ${SRCDIR}/configure \ + --without-persistent-storage \ + [the usual options] + +Dont panic (quite as detailed) +------------------------------ + +Varnish has a built in panic-handler which dumps a lot of +valuable information, so people dont need to send us 1TB +coredumps. + +Part of the information dumped is a backtrace produced with +``libunwind``, which is not available in a CHERI version (yet), +so:: + + ${SRCDIR}/configure \ + --without-unwind \ + [the usual options] + +[u]intptr_t is or isn't ? +------------------------- + +The two typedefs ``uintptr_t`` and ``intptr_t`` are big enough to +hold a pointer so that you can write "portable" code which +does the kind the kind of integer-pointer-mis-math which +CHERI prevents you from doing. + +In theory we should not have any ``[u]intptr_t`` in Varnish, +since one of our quality policies is to never convert an integer +to a pointer. + +But there are a couple of places where we have used them +because for "private" struct members, instead of unions. + +Those become the first stumbling block:: + + vsm.c:601:15: error: shift count >= width of type + vd->serial = VSM_PRIV_LOW(vd->serial + 1); + ^~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The confusing message is because In CHERI, ``[u]intptr_t``, like pointers, +are 16 bytes wide but in this case the integer-view is used as a bit-map. + +For now, I change them to ``uint64_t``, and put them on the TODO list. + +One of them is printed as part of the panic output:: + + VSB_printf(vsb, "priv2 = %zd,\n", vfe->priv2); + +But that doesn't work with the wider type, so:: + + VSB_printf(vsb, "priv2 = %jd,\n", (intmax_t)vfe->priv2); + +And with that Varnish compiles under CHERI, which we can check with:: + + % file bin/varnishd/varnishd + bin/varnishd/varnishd: [?] CheriABI [?] + +First test-run +-------------- + +Just to see how bad it is, we run the main test-scripts:: + + % cd bin/varnishtest + % ./varnistest -i -k -q tests/*.vtc + [?] + 38 tests failed, 33 tests skipped, 754 tests passed + +That's not half bad? + +/phk diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst index f7ccde00d..755e75d5c 100644 --- a/doc/sphinx/phk/index.rst +++ b/doc/sphinx/phk/index.rst @@ -13,6 +13,7 @@ You may or may not want to know what Poul-Henning thinks. .. toctree:: :maxdepth: 1 + cheri1.rst routine.rst 503aroundtheworld.rst legacy.rst From phk at FreeBSD.org Fri Nov 25 21:25:06 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Fri, 25 Nov 2022 21:25:06 +0000 (UTC) Subject: [master] 44d47f64a Duh! Save the actual corrections before committing. Message-ID: <20221125212506.C134ABF3DB@lists.varnish-cache.org> commit 44d47f64a268f43eeca5614081a01f0d28ccc6a5 Author: Poul-Henning Kamp Date: Fri Nov 25 21:24:26 2022 +0000 Duh! Save the actual corrections before committing. diff --git a/doc/sphinx/phk/cheri1.rst b/doc/sphinx/phk/cheri1.rst index fddb7939c..ce27d6495 100644 --- a/doc/sphinx/phk/cheri1.rst +++ b/doc/sphinx/phk/cheri1.rst @@ -15,7 +15,7 @@ it yet, you should read up on it, now: https://www.cl.cam.ac.uk/research/security/ctsrd/cheri/ -The core idea in CHERI is that pointers are not integers, whicb +The core idea in CHERI is that pointers are not integers, which means that you cannot randomly make up or modify pointers to point at random things any more, whatever your intentions might be. @@ -26,7 +26,7 @@ saw in 2019: https://msrc-blog.microsoft.com/2022/01/20/an_armful_of_cheris/ -Yes, we can pause while you sell all your shares in sliced bread. +(Yes, we can pause while you sell all your shares in sliced bread.) I have been eagerly waiting to see how my own Varnish HTTP Cache Software would fare under CHERI, because one of my goals with the @@ -37,8 +37,8 @@ Robert has graciously lent me a shell-account on one of his shiny new MORELLO machines, which rock an ARM64 prototype CPU with CHERI capabilites. -In this sequence of rants I will sing the saga of How Varnish meets -CHERI, as it happens. +In this sequence of rants I will sing the saga of "How Varnish meets +CHERI" - as it happens. My hope is that it will inspire and help other software projects to take the CHERI-plunge, and to help convince ARM that "MORELLO" @@ -76,15 +76,15 @@ Autocrap is a hack on a hack on hack which really ruins software portability, but it is the "industry standard" so we use it also for Varnish, no matter how much I hate it. -https://dl.acm.org/doi/abs/10.1145/2346916.2349257 +See: https://dl.acm.org/doi/abs/10.1145/2346916.2349257 Because a lot of software does not work in CHERI mode, there are two kinds of packages for CheriBSD: Regular and Cheri. -https://ctsrd-cheri.github.io/cheribsd-getting-started/packages/index.html +See: https://ctsrd-cheri.github.io/cheribsd-getting-started/packages/index.html -Autocrap does not grok that some packages end up in /usr/local and -some in /usr/local64, so the first thing I had to do, was to explain +Autocrap does not grok that some packages end up in ``/usr/local`` and +some in ``/usr/local64``, so the first thing I had to do, was to explain this:: export LIBTOOLIZE=/usr/local64/bin/libtoolize @@ -141,7 +141,7 @@ since one of our quality policies is to never convert an integer to a pointer. But there are a couple of places where we have used them -because for "private" struct members, instead of unions. +for "private" struct members, instead of unions. Those become the first stumbling block:: From phk at FreeBSD.org Sat Nov 26 10:49:05 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 26 Nov 2022 10:49:05 +0000 (UTC) Subject: [master] d766510cd Second part of the CHERI saga Message-ID: <20221126104905.D219AA761A@lists.varnish-cache.org> commit d766510cde4fdb1e17e032b2a1b19c757dbf8ed8 Author: Poul-Henning Kamp Date: Sat Nov 26 10:48:34 2022 +0000 Second part of the CHERI saga diff --git a/doc/sphinx/phk/cheri1.rst b/doc/sphinx/phk/cheri1.rst index ce27d6495..03545e568 100644 --- a/doc/sphinx/phk/cheri1.rst +++ b/doc/sphinx/phk/cheri1.rst @@ -173,10 +173,10 @@ First test-run Just to see how bad it is, we run the main test-scripts:: % cd bin/varnishtest - % ./varnistest -i -k -q tests/*.vtc + % ./varnishtest -i -k -q tests/*.vtc [?] 38 tests failed, 33 tests skipped, 754 tests passed That's not half bad? -/phk +*/phk* diff --git a/doc/sphinx/phk/cheri2.rst b/doc/sphinx/phk/cheri2.rst new file mode 100644 index 000000000..9c675af29 --- /dev/null +++ b/doc/sphinx/phk/cheri2.rst @@ -0,0 +1,121 @@ +.. _phk_cheri_2: + +How Varnish met CHERI 2/N +========================= + +CHERI capabilities are twice the size of pointers, and Varnish not +only uses a lot of pointers per request, it is also stingy with +RAM, because it is not uncommon to use 100K worker threads. + +A number of test-cases fail because they are too stingy with memory +allocations, I will deal with them as I get to them, and merely +note them here as part of the accounting:: + + Increase workspace + ================== + TEST tests/c00108.vtc + TEST tests/r01038.vtc + TEST tests/r01120.vtc + TEST tests/r02219.vtc + TEST tests/o00005.vtc + +Things you cannot do under CHERI: Pointers in Pipes +--------------------------------------------------- + +Varnish has a central "waiter" service, whose job it is to monitor +file descriptors to idle network connections, and do the right thing +if data arrives on them, or if they are, or should be closed after +a timeout. + +For reasons of performance, we have multiple implementations: +``kqueue(2)`` (BSD), ``epoll(2)`` (Linux), ``ports(2)`` (Solaris) +and ``poll(2)`` which should work everywhere POSIX has been read. + +We only have the ``poll(2)`` based waiter for portability, one +less issue to deal with during bring-up on new platforms, its +performance degrades to uselessness with contemporary loads +of open network connections. + +The way they all work is that have a single thread sitting +in the relevant system-call, monitoring tens of thousands +of file descriptors. + +Some of those system calls allows other threads to add fds to the +list, but ``poll(2)`` does not, so when we start the poll-waiter +we create a ``pipe(2)``, and have the waiter-thread listen to that +too. + +When another thread wants to add a file descriptor to the inventory, +it uses ``write(2)`` to send a pointer into that pipe. The kernel +provide all the locking and buffering for us, wakes up the waiter-thread +which reads the pointer, adds the new fd to its inventory and dives +back into ``poll(2)``. + +This is 100% safe, because nobody else can get to a pipe created +with ``pipe(2)``, but there is no way CHERI could spot that to +make an execption, so reading pointers out of a filedescriptor, +cause fully justified core-dumps. + +If the poll-waiter was actaully relevant, the proper fix would be +to let the sending thread stick things on a locked list and just +write a nonce-byte into the pipe to the waiter-thread, but that +goes at the bottom of the TODO list, and for now I just remove the +-Wpoll argument from five tests, which then pass:: + + TEST tests/b00009.vtc + TEST tests/b00048.vtc + TEST tests/b00054.vtc + TEST tests/b00059.vtc + TEST tests/c00080.vtc + +But why five tests ? + +It looks like one to test the poll-waiter and four cases of copy&paste. + +Never write your own Red-Black Trees +------------------------------------ + +In general there are few pieces of code I dare not wade into, +but there are a LOT of code I dont want to touch, if there +is any way to avoid it. + +Red-Black trees are one of them. + +In Varnish we stol^H^H^H^H imported both ```` and ```` +from FreeBSD, but as a safety measure we stuck a ``V`` prefix on +everything in them. + +Every so often I will run a small shell-script which does the +v-thing and compare the result to ``vtree.h`` and ``vqueue.h``, +to keep up with FreeBSD. + +Today that paid off handsomely: Some poor person on the CHERI +team had to wade into ``tree.h`` and stick ``__no_subobject_bounds`` +directives to pointers to make that monster work under CHERI. + +I just ran my script and 20 more tests pass:: + + TEST tests/b00068.vtc + TEST tests/c00005.vtc + TEST tests/e00003.vtc + TEST tests/e00008.vtc + TEST tests/e00019.vtc + TEST tests/l00002.vtc + TEST tests/l00003.vtc + TEST tests/l00005.vtc + TEST tests/m00053.vtc + TEST tests/r01312.vtc + TEST tests/r01441.vtc + TEST tests/r02451.vtc + TEST tests/s00012.vtc + TEST tests/u00004.vtc + TEST tests/u00010.vtc + TEST tests/v00009.vtc + TEST tests/v00011.vtc + TEST tests/v00017.vtc + TEST tests/v00041.vtc + TEST tests/v00043.vtc + +Only nine failing tests left now. + +*/phk* diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst index 755e75d5c..f5741e337 100644 --- a/doc/sphinx/phk/index.rst +++ b/doc/sphinx/phk/index.rst @@ -13,6 +13,7 @@ You may or may not want to know what Poul-Henning thinks. .. toctree:: :maxdepth: 1 + cheri2.rst cheri1.rst routine.rst 503aroundtheworld.rst From phk at FreeBSD.org Sat Nov 26 11:56:05 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 26 Nov 2022 11:56:05 +0000 (UTC) Subject: [master] c5f0211af Slight revision Message-ID: <20221126115605.34D55A9876@lists.varnish-cache.org> commit c5f0211af6142d6199d21ccc75bd151fd2d1fb00 Author: Poul-Henning Kamp Date: Sat Nov 26 11:54:58 2022 +0000 Slight revision diff --git a/doc/sphinx/phk/cheri2.rst b/doc/sphinx/phk/cheri2.rst index 9c675af29..71584b40b 100644 --- a/doc/sphinx/phk/cheri2.rst +++ b/doc/sphinx/phk/cheri2.rst @@ -7,18 +7,37 @@ CHERI capabilities are twice the size of pointers, and Varnish not only uses a lot of pointers per request, it is also stingy with RAM, because it is not uncommon to use 100K worker threads. -A number of test-cases fail because they are too stingy with memory -allocations, I will deal with them as I get to them, and merely -note them here as part of the accounting:: +A number of test-cases fail because they are stingy with memory +allocations, and other test-cases run out of cache space. - Increase workspace - ================== +The job of these test-cases is to push varnish into obscure code-paths, +using finely tuned sizes of objects, lengths of headers and parameter +settings to varnish, and the bigger pointers in Varnish trows that +tuning off. + +These test-failures have nothing to do with CHERI. + +There was enough margin that we could find magic numbers which work +on both 32 bit and 64 bit CPUs, that is with both 4 and 8 byte +pointers, but it is doubtful there is enough margin to make them +also work with 16 byte pointers, so I will merely list these tests +here as part of the accounting:: + + Workspace sizes + ===================== TEST tests/c00108.vtc TEST tests/r01038.vtc TEST tests/r01120.vtc TEST tests/r02219.vtc TEST tests/o00005.vtc + Cache sizes + ===================== + TEST tests/r03502.vtc + TEST tests/r01140.vtc + TEST tests/r02831.vtc + TEST tests/v00064.vtc + Things you cannot do under CHERI: Pointers in Pipes --------------------------------------------------- @@ -62,6 +81,8 @@ write a nonce-byte into the pipe to the waiter-thread, but that goes at the bottom of the TODO list, and for now I just remove the -Wpoll argument from five tests, which then pass:: + -Wpoll + ===================== TEST tests/b00009.vtc TEST tests/b00048.vtc TEST tests/b00054.vtc @@ -95,6 +116,8 @@ directives to pointers to make that monster work under CHERI. I just ran my script and 20 more tests pass:: + Red-Black Trees + ===================== TEST tests/b00068.vtc TEST tests/c00005.vtc TEST tests/e00003.vtc @@ -116,6 +139,6 @@ I just ran my script and 20 more tests pass:: TEST tests/v00041.vtc TEST tests/v00043.vtc -Only nine failing tests left now. +Four failures left? */phk* From dridi.boukelmoune at gmail.com Mon Nov 28 13:51:05 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 28 Nov 2022 13:51:05 +0000 (UTC) Subject: [master] a7e8a2600 vtc_server: Fix -Wclobbered variable during dispatch Message-ID: <20221128135105.D4F205C05@lists.varnish-cache.org> commit a7e8a2600d2b89e929522b3f7ecaf69bb1a3fae7 Author: Dridi Boukelmoune Date: Mon Nov 28 14:46:57 2022 +0100 vtc_server: Fix -Wclobbered variable during dispatch Spotted by GCC's fortification level 3. diff --git a/bin/varnishtest/tests/a00023.vtc b/bin/varnishtest/tests/a00023.vtc new file mode 100644 index 000000000..b35a9ec7e --- /dev/null +++ b/bin/varnishtest/tests/a00023.vtc @@ -0,0 +1,29 @@ +varnishtest "Run server -dispatch more than once" + +feature ignore_unknown_macro + +shell { + cat <<-'EOF' >_.vtc + varnishtest "Run server -dispatch more than once (nested)" + + server s0 { + rxreq + txresp + } -dispatch + + client c1 -connect ${s0_sock} { + txreq + rxresp + } -run + + server s0 -break + server s0 -dispatch + + client c1 -run + EOF + + varnishtest -v _.vtc >_.log +} + +shell -match "s1 +rxhdr" { cat _.log } +shell -match "s2 +rxhdr" { cat _.log } diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index 08103a600..f09131a24 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -336,7 +336,8 @@ static void * server_dispatch_thread(void *priv) { struct server *s, *s2; - int sn = 1, fd; + static int sn = 1; + int fd; char snbuf[8]; struct vtclog *vl; struct sockaddr_storage addr_s; From nils.goroll at uplex.de Mon Nov 28 15:03:05 2022 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 28 Nov 2022 15:03:05 +0000 (UTC) Subject: [master] 6840fbd47 Hint to the parameter relevant for watchdog panic Message-ID: <20221128150305.A7D4B93AE@lists.varnish-cache.org> commit 6840fbd47313eae8e3b2acc0ab863f00ee1b5e72 Author: Nils Goroll Date: Mon Nov 28 15:57:38 2022 +0100 Hint to the parameter relevant for watchdog panic Motivated by #3868 diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 7c2bd8036..f218d6758 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -629,7 +629,8 @@ pool_herder(void *priv) VSL(SLT_Error, NO_VXID, "Pool Herder: Queue does not move ql=%u dt=%f", pp->lqueue, VTIM_mono() - dqt); - WRONG("Worker Pool Queue does not move"); + WRONG("Worker Pool Queue does not move" + " - see thread_pool_watchdog parameter"); } wthread_min = cache_param->wthread_min; if (pp->die) From dridi.boukelmoune at gmail.com Mon Nov 28 15:33:06 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 28 Nov 2022 15:33:06 +0000 (UTC) Subject: [master] 4c8e0955b varnishtest: set -nolen on tx[req|resp] C-L or T-E Message-ID: <20221128153306.9267C61564@lists.varnish-cache.org> commit 4c8e0955badbb67ee52fd356e1f4d237b641f342 Author: AlveElde Date: Fri Sep 16 16:17:59 2022 +0200 varnishtest: set -nolen on tx[req|resp] C-L or T-E diff --git a/bin/varnishtest/tests/b00037.vtc b/bin/varnishtest/tests/b00037.vtc index 63d8014dc..ce0e84112 100644 --- a/bin/varnishtest/tests/b00037.vtc +++ b/bin/varnishtest/tests/b00037.vtc @@ -13,7 +13,7 @@ varnish v1 -vsl_catchup varnish v1 -expect client_req_400 == 1 client c1 { - txreq -method POST -hdr "Content-Length: 12" -bodylen 12 + txreq -method POST -hdr "Content-Length: 12" -hdr "Content-Length: 12" -bodylen 12 rxresp expect resp.status == 400 } -run diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 827c20da4..aada58c03 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -786,6 +786,9 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, } else if (!strcmp(*av, "-nohost")) { nohost = 1; } else if (!strcmp(*av, "-hdr")) { + if (!strncasecmp(av[1], "Content-Length:", 15) || + !strncasecmp(av[1], "Transfer-Encoding:", 18)) + nolen = 1; if (!strncasecmp(av[1], "Host:", 5)) nohost = 1; VSB_printf(hp->vsb, "%s%s", av[1], nl); From dridi.boukelmoune at gmail.com Mon Nov 28 15:33:06 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 28 Nov 2022 15:33:06 +0000 (UTC) Subject: [master] ec0c5f4a5 varnishtest: txresp Date header by default Message-ID: <20221128153306.BA12F61567@lists.varnish-cache.org> commit ec0c5f4a539c51de35e3c1b2897328244e9d0a63 Author: AlveElde Date: Thu Sep 15 12:57:19 2022 +0200 varnishtest: txresp Date header by default This commit prepares varnishtest for a change to Varnish revalidations, where the Last-Modified header must be at least one second older than the Date header for Varnish to send an If-Modified-Since header. Any explicitly defined Date header will override the default, and the Date header can be omitted with -nodate. diff --git a/bin/varnishtest/tests/b00002.vtc b/bin/varnishtest/tests/b00002.vtc index c3feb0722..63fde07a2 100644 --- a/bin/varnishtest/tests/b00002.vtc +++ b/bin/varnishtest/tests/b00002.vtc @@ -3,7 +3,7 @@ varnishtest "Check that a pass transaction works" server s1 { rxreq expect req.proto == HTTP/1.1 - txresp -hdr "Cache-Control: max-age=120" -hdr "Connection: close" -body "012345\n" + txresp -hdr "Cache-Control: max-age=120" -hdr "Connection: close" -nodate -body "012345\n" } -start varnish v1 -arg "-sTransient=default,1m" -vcl+backend { diff --git a/bin/varnishtest/tests/c00036.vtc b/bin/varnishtest/tests/c00036.vtc index 97b51d5cc..adec31ab0 100644 --- a/bin/varnishtest/tests/c00036.vtc +++ b/bin/varnishtest/tests/c00036.vtc @@ -2,13 +2,13 @@ varnishtest "Backend close retry" server s1 -repeat 1 { rxreq - txresp -bodylen 5 + txresp -nodate -bodylen 5 rxreq accept rxreq - txresp -bodylen 6 + txresp -nodate -bodylen 6 } -start diff --git a/bin/varnishtest/tests/l00004.vtc b/bin/varnishtest/tests/l00004.vtc index 5978af8b2..0bff118bb 100644 --- a/bin/varnishtest/tests/l00004.vtc +++ b/bin/varnishtest/tests/l00004.vtc @@ -4,7 +4,7 @@ server s1 { rxreq expect req.url == "/" expect req.http.test == "yes" - txresp -body "fdsa" + txresp -nodate -body "fdsa" } -start varnish v1 -vcl+backend { diff --git a/bin/varnishtest/tests/l00005.vtc b/bin/varnishtest/tests/l00005.vtc index 51667672a..12da731d0 100644 --- a/bin/varnishtest/tests/l00005.vtc +++ b/bin/varnishtest/tests/l00005.vtc @@ -3,7 +3,7 @@ varnishtest "Test backend fetch byte counters" server s1 { rxreq expect req.url == "/1" - txresp -bodylen 1000 + txresp -nodate -bodylen 1000 rxreq expect req.url == "/2" diff --git a/bin/varnishtest/tests/r00498.vtc b/bin/varnishtest/tests/r00498.vtc index c11df3f77..2b08dc202 100644 --- a/bin/varnishtest/tests/r00498.vtc +++ b/bin/varnishtest/tests/r00498.vtc @@ -3,7 +3,7 @@ varnishtest "very very very long return header" server s1 { rxreq expect req.url == "/" - txresp -hdr "Location: ${string,repeat,8136,1}" -body {foo} + txresp -hdr "Location: ${string,repeat,8136,1}" -nodate -body {foo} } -start varnish v1 -vcl+backend { diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index aada58c03..5f94452a8 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -769,7 +769,7 @@ cmd_http_gunzip(CMD_ARGS) static char* const * http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, - char *body, unsigned nohost) + char *body, unsigned nohost, unsigned nodate) { long bodylen = 0; char *b, *c; @@ -785,12 +785,16 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, nolen = 1; } else if (!strcmp(*av, "-nohost")) { nohost = 1; + } else if (!strcmp(*av, "-nodate")) { + nodate = 1; } else if (!strcmp(*av, "-hdr")) { if (!strncasecmp(av[1], "Content-Length:", 15) || !strncasecmp(av[1], "Transfer-Encoding:", 18)) nolen = 1; if (!strncasecmp(av[1], "Host:", 5)) nohost = 1; + if (!strncasecmp(av[1], "Date:", 5)) + nodate = 1; VSB_printf(hp->vsb, "%s%s", av[1], nl); av++; } else if (!strcmp(*av, "-hdrlen")) { @@ -865,6 +869,11 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, macro_cat(vl, hp->vsb, "localhost", NULL); VSB_cat(hp->vsb, nl); } + if (!nodate) { + VSB_cat(hp->vsb, "Date: "); + macro_cat(vl, hp->vsb, "date", NULL); + VSB_cat(hp->vsb, nl); + } if (body != NULL && !nolen) VSB_printf(hp->vsb, "Content-Length: %ld%s", bodylen, nl); VSB_cat(hp->vsb, nl); @@ -914,6 +923,9 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, * \-nolen * Don't include a Content-Length header. * + * \-nodate + * Don't include a Date header in the response. + * * \-hdr STRING * Add STRING as a header, it must follow this format: * "name: value". It can be called multiple times. @@ -988,7 +1000,7 @@ cmd_http_txresp(CMD_ARGS) /* send a "Content-Length: 0" header unless something else happens */ REPLACE(body, ""); - av = http_tx_parse_args(av, vl, hp, body, 1); + av = http_tx_parse_args(av, vl, hp, body, 1, 0); if (*av != NULL) vtc_fatal(hp->vl, "Unknown http txresp spec: %s\n", *av); @@ -1220,7 +1232,7 @@ cmd_http_txreq(CMD_ARGS) "HTTP2-Settings: %s%s", nl, nl, up, nl); nohost = strcmp(proto, "HTTP/1.1") != 0; - av = http_tx_parse_args(av, vl, hp, NULL, nohost); + av = http_tx_parse_args(av, vl, hp, NULL, nohost, 1); if (*av != NULL) vtc_fatal(hp->vl, "Unknown http txreq spec: %s\n", *av); http_write(hp, 4, "txreq"); From dridi.boukelmoune at gmail.com Mon Nov 28 15:33:06 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 28 Nov 2022 15:33:06 +0000 (UTC) Subject: [master] ba79652ee RFC2616: Function to derive LM header strength Message-ID: <20221128153306.DFF816156F@lists.varnish-cache.org> commit ba79652eeec65fdf46c25008fe4abc00df51ee3e Author: AlveElde Date: Fri Sep 16 16:38:49 2022 +0200 RFC2616: Function to derive LM header strength Per RFC9110, the Last-Modified header is not a strong validator unless it is at least one second older than the Date header. This is to prevent revalidating content that has been changed within a second of the last response. In the case of an intermediate cache like Varnish, a weak Last-Modified validator is "weaker" than a weak ETag, and should not be used for revalidating content. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 49ef26195..c59564737 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -880,6 +880,8 @@ unsigned RFC2616_Req_Gzip(const struct http *); int RFC2616_Do_Cond(const struct req *sp); void RFC2616_Weaken_Etag(struct http *hp); void RFC2616_Vary_AE(struct http *hp); +const char * RFC2616_Strong_LM(struct http *hp, struct worker *wrk, + struct objcore *oc); /* * We want to cache the most recent timestamp in wrk->lastused to avoid diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index 93bc30843..e45ad12fd 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -356,3 +356,36 @@ RFC2616_Vary_AE(struct http *hp) http_SetHeader(hp, "Vary: Accept-Encoding"); } } + +/*--------------------------------------------------------------------*/ + +const char * +RFC2616_Strong_LM(struct http *hp, struct worker *wrk, struct objcore *oc) +{ + const char *p = NULL, *e = NULL; + vtim_real lm, d; + + CHECK_OBJ_ORNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_ORNULL(oc, OBJCORE_MAGIC); + CHECK_OBJ_ORNULL(hp, HTTP_MAGIC); + + if (hp != NULL) { + http_GetHdr(hp, H_Last_Modified, &p); + http_GetHdr(hp, H_Date, &e); + } else if (wrk != NULL && oc != NULL) { + p = HTTP_GetHdrPack(wrk, oc, H_Last_Modified); + e = HTTP_GetHdrPack(wrk, oc, H_Date); + } + + if (p == NULL || e == NULL) + return (NULL); + + lm = VTIM_parse(p); + d = VTIM_parse(e); + + /* The cache entry includes a Date value which is at least one second + * after the Last-Modified value. + * [RFC9110 8.8.2.2-6.2] + */ + return ((lm && d && lm + 1 <= d) ? p : NULL); +} From dridi.boukelmoune at gmail.com Mon Nov 28 15:33:07 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 28 Nov 2022 15:33:07 +0000 (UTC) Subject: [master] ab2595062 fetch: A weak LM header does not make an IMS candidate Message-ID: <20221128153307.13CC76157D@lists.varnish-cache.org> commit ab25950629232b56e29251f7a9872ccb4a3f2266 Author: AlveElde Date: Fri Sep 16 16:48:50 2022 +0200 fetch: A weak LM header does not make an IMS candidate Avoid marking an object as an IMS candidate when a weak Last-Modified header is the only validator. In the case where there is an ETag AND a weak Last-Modified header, add a If-None-Match header, but do not add an If-Modified-Since header. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index cf1005aa4..2875d2f91 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -274,7 +274,7 @@ vbf_stp_mkbereq(struct worker *wrk, struct busyobj *bo) ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND) && (bo->stale_oc->boc != NULL || ObjGetLen(wrk, bo->stale_oc) != 0)) { AZ(bo->stale_oc->flags & (OC_F_HFM|OC_F_PRIVATE)); - q = HTTP_GetHdrPack(bo->wrk, bo->stale_oc, H_Last_Modified); + q = RFC2616_Strong_LM(NULL, wrk, bo->stale_oc); if (q != NULL) http_PrintfHeader(bo->bereq0, "If-Modified-Since: %s", q); @@ -700,7 +700,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) if (!(oc->flags & OC_F_HFM) && http_IsStatus(bo->beresp, 200) && ( - http_GetHdr(bo->beresp, H_Last_Modified, NULL) || + RFC2616_Strong_LM(bo->beresp, NULL, NULL) != NULL || http_GetHdr(bo->beresp, H_ETag, NULL))) ObjSetFlag(bo->wrk, oc, OF_IMSCAND, 1); diff --git a/bin/varnishtest/tests/b00079.vtc b/bin/varnishtest/tests/b00079.vtc new file mode 100644 index 000000000..efb451261 --- /dev/null +++ b/bin/varnishtest/tests/b00079.vtc @@ -0,0 +1,83 @@ +varnishtest "Test backend IMS with weak and strong LM" + +server s1 { + rxreq + txresp -hdr "Last-Modified: Wed, 11 Sep 2013 13:36:55 GMT" -nodate -body "1" + + # When origin does not send a Date, varnish inserts one, prompting IMS + rxreq + expect req.http.if-modified-since == "Wed, 11 Sep 2013 13:36:55 GMT" + txresp -status 304 -hdr "Last-Modified: Wed, 11 Sep 2013 13:36:55 GMT" \ + -hdr "Date: Wed, 11 Sep 2013 13:36:55 GMT" \ + + # LM was the same as Date + rxreq + expect req.http.if-modified-since == + txresp -hdr "Last-Modified: Wed, 11 Sep 2013 13:36:55 GMT" \ + -hdr "Date: Wed, 11 Sep 2013 13:36:56 GMT" \ + -body "2" + + # LM was one second older than Date + rxreq + expect req.http.if-modified-since == "Wed, 11 Sep 2013 13:36:55 GMT" + txresp -status 304 -hdr "Last-Modified: Wed, 11 Sep 2013 13:36:55 GMT" \ + -hdr "Date: Wed, 11 Sep 2013 13:36:55 GMT" \ + -hdr {ETag: "foo"} + + # LM was the same as Date, but we had an ETag, prompting INM + rxreq + expect req.http.if-modified-since == + expect req.http.if-none-match == {"foo"} + txresp -status 304 -hdr "Last-Modified: Wed, 11 Sep 2013 13:36:55 GMT" \ + -hdr "Date: Wed, 11 Sep 2013 13:36:55 GMT" \ + -hdr {ETag: "foo"} +} -start + +varnish v1 -vcl+backend { + sub vcl_backend_response { + set beresp.ttl = 1ms; + set beresp.grace = 0s; + set beresp.keep = 1m; + set beresp.http.was-304 = beresp.was_304; + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 + expect resp.body == "1" + expect resp.http.was-304 == "false" + + delay 0.1 + + txreq + rxresp + expect resp.status == 200 + expect resp.body == "1" + expect resp.http.was-304 == "true" + + delay 0.1 + + txreq + rxresp + expect resp.status == 200 + expect resp.body == "2" + expect resp.http.was-304 == "false" + + delay 0.1 + + txreq + rxresp + expect resp.status == 200 + expect resp.body == "2" + expect resp.http.was-304 == "true" + + delay 0.1 + + txreq + rxresp + expect resp.status == 200 + expect resp.body == "2" + expect resp.http.was-304 == "true" +} -run From phk at FreeBSD.org Mon Nov 28 16:19:05 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 28 Nov 2022 16:19:05 +0000 (UTC) Subject: [master] a50a05c93 Third installment... Message-ID: <20221128161905.A78DF63557@lists.varnish-cache.org> commit a50a05c93c3a752a7c0ec39d7e9f61ac543ff8da Author: Poul-Henning Kamp Date: Mon Nov 28 16:18:23 2022 +0000 Third installment... diff --git a/doc/sphinx/phk/cheri3.rst b/doc/sphinx/phk/cheri3.rst new file mode 100644 index 000000000..a2995bfae --- /dev/null +++ b/doc/sphinx/phk/cheri3.rst @@ -0,0 +1,66 @@ +.. _phk_cheri_3: + +How Varnish met CHERI 3/N +========================= + +Of the four tests which still fail, three uses the "file stevedore". + +Things you cannot do under CHERI: Pointers in shared files +---------------------------------------------------------- + +In varnish a "stevedore" is responsible for orchestrating storage +for the cached objects, and it is an API extension-point. + +The "malloc stevedore" is the default, it does precisely what you +think it does. + +The "file" stevedore, ``mmap(2)``'s a file in the filesystem +and partitions that out as needed, pretty much like an implementation +of ``malloc(3)`` would do. + +The "file" stevedore exists mainly because back in 2006 mapping a +file with ``MAP_NOCORE``, was the only way to avoid the entire +object cache being included in core-dumps. + +But CHERI will not allow you to put a pointer into a regular file +``mmmap(2)``'ed ``MAP_SHARED``, because that would allow another +process, maybe even on a different computer, to ``mmap(2)`` the +file ``MAP_SHARED`` later and, by implication, resurrect the pointers. + +The "persistent" stevedore mentioned in part 1, does the same thing, +and does not work under CHERI for the same reason. + +If instead we map the file ``MAP_PRIVATE``, nobody else will +ever be able to see the pointers, and those three test cases pass:: + + MAP_SHARED pointers + ===================== + TEST tests/b00005.vtc + TEST tests/r02429.vtc + TEST tests/s00003.vtc + +(We cannot do the same for the "persistent" stevedore, because the +only reason it exists is precisely to to resurrect those pointers later.) + +The final and really nasty test-case +==================================== + +As previously mentioned, when you have 100K threads, you have to be +stingy with memory, in particular with the thread stacks. + +But if you tune things too hard, the threads will step out of their +stack and coredups result. We try to be a little bit helpful +about the diagnostics in such cases, and we have a test-case +which tries to exercise that. + +That test-case is a pretty nasty piece of work, and for all +intents and purposes, it is just a cheap erzats of what CHERI is +supposed to do for us, so I am going to punt on it, and get +to the more interesting parts of this project:: + + SigSegv handler test + ===================== + TEST tests/c00057.vtc + + +*/phk* diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst index f5741e337..6ab783cab 100644 --- a/doc/sphinx/phk/index.rst +++ b/doc/sphinx/phk/index.rst @@ -13,6 +13,7 @@ You may or may not want to know what Poul-Henning thinks. .. toctree:: :maxdepth: 1 + cheri3.rst cheri2.rst cheri1.rst routine.rst From dridi.boukelmoune at gmail.com Mon Nov 28 16:22:05 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 28 Nov 2022 16:22:05 +0000 (UTC) Subject: [master] 10068db60 man: Implicit -nohost -nodate and -nolen in vtc(7) Message-ID: <20221128162205.D1169638A0@lists.varnish-cache.org> commit 10068db6019442e91ec263847d828e22e5ddef41 Author: Dridi Boukelmoune Date: Mon Nov 28 17:00:42 2022 +0100 man: Implicit -nohost -nodate and -nolen in vtc(7) diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 5f94452a8..44c0138cd 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -918,13 +918,17 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, * following ones. * * \-nohost - * Don't include a Host header in the request. + * Don't include a Host header in the request. Also Implied + * by the addition of a Host header with ``-hdr``. * * \-nolen - * Don't include a Content-Length header. + * Don't include a Content-Length header. Also implied by the + * addition of a Content-Length or Transfer-Encoding header + * with ``-hdr``. * * \-nodate - * Don't include a Date header in the response. + * Don't include a Date header in the response. Also implied + * by the addition of a Date header with ``-hdr``. * * \-hdr STRING * Add STRING as a header, it must follow this format: From dridi.boukelmoune at gmail.com Mon Nov 28 16:22:05 2022 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 28 Nov 2022 16:22:05 +0000 (UTC) Subject: [master] 39992c7e9 man: Fix rendering of certain sections in vtc(7) Message-ID: <20221128162205.E75B1638A3@lists.varnish-cache.org> commit 39992c7e95404ad2a53da3818fa76d34501b5f79 Author: Dridi Boukelmoune Date: Mon Nov 28 17:17:15 2022 +0100 man: Fix rendering of certain sections in vtc(7) The matching regular expression did not align with the substitution regular expression. Refs d2e526ce95d6d8426ec170b4cfc78d41abc07179 diff --git a/doc/sphinx/vtc-syntax.py b/doc/sphinx/vtc-syntax.py index f5533e84a..cfc5a05ab 100644 --- a/doc/sphinx/vtc-syntax.py +++ b/doc/sphinx/vtc-syntax.py @@ -38,7 +38,7 @@ import re def parse_file(fn, cl, tl, sl): p = False section = "" - resec = re.compile("\s*/\* SECTION: ") + resec = re.compile("\s*/?\* SECTION: ") try: # Python3 From phk at FreeBSD.org Mon Nov 28 17:39:05 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 28 Nov 2022 17:39:05 +0000 (UTC) Subject: [master] 7bb21524d Typos Message-ID: <20221128173905.A190E6E261@lists.varnish-cache.org> commit 7bb21524d7dad464bc36e7cfaa0892f449b75d7c Author: Poul-Henning Kamp Date: Mon Nov 28 17:38:29 2022 +0000 Typos diff --git a/doc/sphinx/phk/cheri3.rst b/doc/sphinx/phk/cheri3.rst index a2995bfae..0d45056c0 100644 --- a/doc/sphinx/phk/cheri3.rst +++ b/doc/sphinx/phk/cheri3.rst @@ -3,7 +3,7 @@ How Varnish met CHERI 3/N ========================= -Of the four tests which still fail, three uses the "file stevedore". +Of the four tests which still fail, three use the "file stevedore". Things you cannot do under CHERI: Pointers in shared files ---------------------------------------------------------- @@ -49,7 +49,7 @@ As previously mentioned, when you have 100K threads, you have to be stingy with memory, in particular with the thread stacks. But if you tune things too hard, the threads will step out of their -stack and coredups result. We try to be a little bit helpful +stack and coredumps result. We try to be a little bit helpful about the diagnostics in such cases, and we have a test-case which tries to exercise that. From phk at FreeBSD.org Tue Nov 29 09:41:06 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Nov 2022 09:41:06 +0000 (UTC) Subject: [master] abe708b1e Chapter 4 Message-ID: <20221129094106.63057BFC33@lists.varnish-cache.org> commit abe708b1e49a4766e722e87f5607cf5c432f384d Author: Poul-Henning Kamp Date: Tue Nov 29 09:40:43 2022 +0000 Chapter 4 diff --git a/doc/sphinx/phk/cheri3.rst b/doc/sphinx/phk/cheri3.rst index 0d45056c0..372d84050 100644 --- a/doc/sphinx/phk/cheri3.rst +++ b/doc/sphinx/phk/cheri3.rst @@ -43,7 +43,7 @@ ever be able to see the pointers, and those three test cases pass:: only reason it exists is precisely to to resurrect those pointers later.) The final and really nasty test-case -==================================== +------------------------------------ As previously mentioned, when you have 100K threads, you have to be stingy with memory, in particular with the thread stacks. diff --git a/doc/sphinx/phk/cheri4.rst b/doc/sphinx/phk/cheri4.rst new file mode 100644 index 000000000..26baeb015 --- /dev/null +++ b/doc/sphinx/phk/cheri4.rst @@ -0,0 +1,179 @@ +.. _phk_cheri_4: + +How Varnish met CHERI 4/N +========================= + +So what can CHERI do ? +---------------------- + +CHERI can restrict pointers, but it does not restrict the memory +they point to:: + + #include + #include + #include + + int + main() + { + char buf[20]; + char *ptr1 = cheri_perms_and(buf, CHERI_PERM_LOAD); + char *ptr2 = buf; + + strcpy(buf, "Hello World\n"); + ptr1[5] = '_'; // Will core dump + ptr2[5] = '_'; // Works fine. + puts(buf); + return (0); + } + +I suspect most programmers will find this counter-intuitive, because +normally it is the memory itself which write-protected in which +case no pointers can ever write to it. + +This is where the word "capability" comes from: The pointer is what +gives you the "capability" to access the memory, and therefore they +can be restricted separately from the memory they provide access to. + +If you could just create your own "capabilities" out of integers, +that would be no big improvement, but you can not: Under CHERI you +can only make a new capability from another capability, and the new +one can never be more potent than the one it is derived from. + +In addition to 'read' and 'write' permissions, capabilities contain the +start and the length of the piece of memory they allow access to. + +Under CHERI the printf(3) pattern "%#p" tells the full story:: + + #include + #include + #include + + int + main() + { + char buf[20]; + char *ptr1 = cheri_perms_and(buf, CHERI_PERM_LOAD); + char *ptr2 = buf; + char *ptr3; + char *ptr4; + + strcpy(buf, "Hello World\n"); + //ptr1[5] = '_'; // Will core dump + ptr2[5] = '_'; // Works fine. + puts(buf); + printf("buf:\t%#p\n", buf); + printf("ptr1:\t%#p\n", ptr1); + printf("ptr2:\t%#p\n", ptr2); + ptr3 = ptr2 + 1; + printf("ptr3:\t%#p\n", ptr3); + ptr4 = cheri_bounds_set(ptr3, 4); + printf("ptr4:\t%#p\n", ptr4); + return (0); + } + +And we get:: + + buf: 0xfffffff7ff68 [rwRW,0xfffffff7ff68-0xfffffff7ff7c] + ptr1: 0xfffffff7ff68 [r,0xfffffff7ff68-0xfffffff7ff7c] + ptr2: 0xfffffff7ff68 [rwRW,0xfffffff7ff68-0xfffffff7ff7c] + ptr3: 0xfffffff7ff69 [rwRW,0xfffffff7ff68-0xfffffff7ff7c] + ptr4: 0xfffffff7ff69 [rwRW,0xfffffff7ff69-0xfffffff7ff6d] + +(Ignore the upper-case 'RW' for now, I'll get back to those later.) + +Because of the odd things people do with pointers in C, ``ptr3`` +keeps the same range as ``ptr2`` and that's a good excuse to +address the question I imagine my readers have at this point: + +What about ``const`` ? +---------------------- + +The C language is a mess. + +Bjarne Stroustrup introduced ``const`` in "C-with-classes", +which later became C++, and in C++ it does the right thing. + +The morons on the ISO-C committee did a half-assed import +in C89, and therefore it does the wrong thing:: + + char *strchr(const char *s, int c); + +You pass a read-only pointer in, and you get a read-write +pointer into that string back ?! + +There were at least three different ways they could have done right: + +* Add a ``cstrchr`` function, having ``const`` on both sides. + +* Enable prototypes to explain this, with some gruesome hack:: + + (const) char *strchr((const) char *s, int c); + +* Allow multiple prototypes for the same function, varying only in const-ness:: + + char *strchr(char *s, int c); + const char *strchr(const char *s, int c); + +But instead they just ignored that issue, and several others like it. + +The result is that we have developed the concept of "const-poisoning", +to describe the fact that if you use "const" to any extent in your +C sources, you almost always end up needing a macro like:: + + #define TRUST_ME(ptr) ((void*)(uintptr_t)(ptr)) + +To remove const-ness where it cannot go. + +(If you think that is ISO-C's opus magnum, ask yourself why we still +cannot specify struct packing and endianess explicitly ? It's hardly +like anybody ever have to apart data-structures explicitly specified in +hardware or protocol documents, is it ?) + +Read/Write markup with CHERI +---------------------------- + +Because ``const`` is such a mess in C, the CHERI compiler does not +automatically remove the write-capability from ``const`` arguments +to functions, something I suspect (but have not checked) that they +can do in C++. + +Instead we will have to do it ourselves, so I have added two macros to +our ```` file:: + + #define RO(x) cheri_perms_and((x), CHERI_PERM_LOAD) + #define ROP(x) cheri_perms_and((x), CHERI_PERM_LOAD|CHERI_PERM_LOAD_CAP) + +Passing a pointer through the ``RO()`` macro makes it read-only, so we +can do stuff like:: + + @@ -285,7 +286,7 @@ VRT_GetHdr(VRT_CTX, VCL_HEADER hs) + [?] + - return (p); + + return (RO(p)); + } + +To explicitly give ``const`` some bite. + +The difference between ``RO`` and ``ROP`` is where the upper- and +lower-case "rw" comes into the picture: Capabilities have two +levels of read/write protection: + +* Can you read or write normal data with this pointer (``CHERI_PERM_LOAD``) + +* Can you read or write pointers with this pointer (``CHERI_PERM_LOAD_CAP``) + +Rule of thumb: Pure data: Use only the first, structs with pointers in them, +use both. + +One can also make write-only pointers with CHERI, but there are +only few places where they can be gainfully employed, outside strict +security in handling of (cryptographic) secrets. + +Right now I'm plunking ``RO()`` and ``ROP()`` into the varnish code, +and one by one relearning what atrocity the 37 uses of ``TRUST_ME()`` +hide. + +Still no bugs found. + +*/phk* diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst index 6ab783cab..5d33fef4e 100644 --- a/doc/sphinx/phk/index.rst +++ b/doc/sphinx/phk/index.rst @@ -13,6 +13,7 @@ You may or may not want to know what Poul-Henning thinks. .. toctree:: :maxdepth: 1 + cheri4.rst cheri3.rst cheri2.rst cheri1.rst From phk at FreeBSD.org Tue Nov 29 12:16:06 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Nov 2022 12:16:06 +0000 (UTC) Subject: [master] ee46f6e8f Eliminate need for TRUST_ME() Message-ID: <20221129121606.20BCF9E86@lists.varnish-cache.org> commit ee46f6e8fe23e8cf7164184bb49f10e9a54ccf55 Author: Poul-Henning Kamp Date: Tue Nov 29 12:14:31 2022 +0000 Eliminate need for TRUST_ME() diff --git a/vmod/vmod_debug_acl.c b/vmod/vmod_debug_acl.c index b7b772593..8f277e7a0 100644 --- a/vmod/vmod_debug_acl.c +++ b/vmod/vmod_debug_acl.c @@ -131,9 +131,12 @@ setup_sweep(VRT_CTX, struct acl_sweep *asw, VCL_IP ip0, VCL_IP ip1, } asw->this = asw->reset; - asw->probe = VSA_Clone(ip0); + /* Dont try this at home */ + asw->probe = malloc(vsa_suckaddr_len); + AN(asw->probe); + memcpy(asw->probe, ip0, vsa_suckaddr_len); (void)VSA_GetPtr(asw->probe, &ptr); - asw->probe_p = TRUST_ME(ptr); + asw->probe_p = ((uint8_t*)(asw->probe)) + (ptr - (uint8_t*)asw->probe); asw->step = step; From phk at FreeBSD.org Tue Nov 29 12:48:09 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Nov 2022 12:48:09 +0000 (UTC) Subject: [CHERI] 54273f22f Second part of the CHERI saga Message-ID: <20221129124809.1C6C762219@lists.varnish-cache.org> commit 54273f22fd18404839426b496b2bacc09c37ca9f Author: Poul-Henning Kamp Date: Sat Nov 26 10:48:34 2022 +0000 Second part of the CHERI saga diff --git a/doc/sphinx/phk/cheri1.rst b/doc/sphinx/phk/cheri1.rst index ce27d6495..03545e568 100644 --- a/doc/sphinx/phk/cheri1.rst +++ b/doc/sphinx/phk/cheri1.rst @@ -173,10 +173,10 @@ First test-run Just to see how bad it is, we run the main test-scripts:: % cd bin/varnishtest - % ./varnistest -i -k -q tests/*.vtc + % ./varnishtest -i -k -q tests/*.vtc [?] 38 tests failed, 33 tests skipped, 754 tests passed That's not half bad? -/phk +*/phk* diff --git a/doc/sphinx/phk/cheri2.rst b/doc/sphinx/phk/cheri2.rst new file mode 100644 index 000000000..9c675af29 --- /dev/null +++ b/doc/sphinx/phk/cheri2.rst @@ -0,0 +1,121 @@ +.. _phk_cheri_2: + +How Varnish met CHERI 2/N +========================= + +CHERI capabilities are twice the size of pointers, and Varnish not +only uses a lot of pointers per request, it is also stingy with +RAM, because it is not uncommon to use 100K worker threads. + +A number of test-cases fail because they are too stingy with memory +allocations, I will deal with them as I get to them, and merely +note them here as part of the accounting:: + + Increase workspace + ================== + TEST tests/c00108.vtc + TEST tests/r01038.vtc + TEST tests/r01120.vtc + TEST tests/r02219.vtc + TEST tests/o00005.vtc + +Things you cannot do under CHERI: Pointers in Pipes +--------------------------------------------------- + +Varnish has a central "waiter" service, whose job it is to monitor +file descriptors to idle network connections, and do the right thing +if data arrives on them, or if they are, or should be closed after +a timeout. + +For reasons of performance, we have multiple implementations: +``kqueue(2)`` (BSD), ``epoll(2)`` (Linux), ``ports(2)`` (Solaris) +and ``poll(2)`` which should work everywhere POSIX has been read. + +We only have the ``poll(2)`` based waiter for portability, one +less issue to deal with during bring-up on new platforms, its +performance degrades to uselessness with contemporary loads +of open network connections. + +The way they all work is that have a single thread sitting +in the relevant system-call, monitoring tens of thousands +of file descriptors. + +Some of those system calls allows other threads to add fds to the +list, but ``poll(2)`` does not, so when we start the poll-waiter +we create a ``pipe(2)``, and have the waiter-thread listen to that +too. + +When another thread wants to add a file descriptor to the inventory, +it uses ``write(2)`` to send a pointer into that pipe. The kernel +provide all the locking and buffering for us, wakes up the waiter-thread +which reads the pointer, adds the new fd to its inventory and dives +back into ``poll(2)``. + +This is 100% safe, because nobody else can get to a pipe created +with ``pipe(2)``, but there is no way CHERI could spot that to +make an execption, so reading pointers out of a filedescriptor, +cause fully justified core-dumps. + +If the poll-waiter was actaully relevant, the proper fix would be +to let the sending thread stick things on a locked list and just +write a nonce-byte into the pipe to the waiter-thread, but that +goes at the bottom of the TODO list, and for now I just remove the +-Wpoll argument from five tests, which then pass:: + + TEST tests/b00009.vtc + TEST tests/b00048.vtc + TEST tests/b00054.vtc + TEST tests/b00059.vtc + TEST tests/c00080.vtc + +But why five tests ? + +It looks like one to test the poll-waiter and four cases of copy&paste. + +Never write your own Red-Black Trees +------------------------------------ + +In general there are few pieces of code I dare not wade into, +but there are a LOT of code I dont want to touch, if there +is any way to avoid it. + +Red-Black trees are one of them. + +In Varnish we stol^H^H^H^H imported both ```` and ```` +from FreeBSD, but as a safety measure we stuck a ``V`` prefix on +everything in them. + +Every so often I will run a small shell-script which does the +v-thing and compare the result to ``vtree.h`` and ``vqueue.h``, +to keep up with FreeBSD. + +Today that paid off handsomely: Some poor person on the CHERI +team had to wade into ``tree.h`` and stick ``__no_subobject_bounds`` +directives to pointers to make that monster work under CHERI. + +I just ran my script and 20 more tests pass:: + + TEST tests/b00068.vtc + TEST tests/c00005.vtc + TEST tests/e00003.vtc + TEST tests/e00008.vtc + TEST tests/e00019.vtc + TEST tests/l00002.vtc + TEST tests/l00003.vtc + TEST tests/l00005.vtc + TEST tests/m00053.vtc + TEST tests/r01312.vtc + TEST tests/r01441.vtc + TEST tests/r02451.vtc + TEST tests/s00012.vtc + TEST tests/u00004.vtc + TEST tests/u00010.vtc + TEST tests/v00009.vtc + TEST tests/v00011.vtc + TEST tests/v00017.vtc + TEST tests/v00041.vtc + TEST tests/v00043.vtc + +Only nine failing tests left now. + +*/phk* diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst index 755e75d5c..f5741e337 100644 --- a/doc/sphinx/phk/index.rst +++ b/doc/sphinx/phk/index.rst @@ -13,6 +13,7 @@ You may or may not want to know what Poul-Henning thinks. .. toctree:: :maxdepth: 1 + cheri2.rst cheri1.rst routine.rst 503aroundtheworld.rst From phk at FreeBSD.org Tue Nov 29 12:48:09 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Nov 2022 12:48:09 +0000 (UTC) Subject: [CHERI] 983f9ecaa Slight revision Message-ID: <20221129124809.2D0406221B@lists.varnish-cache.org> commit 983f9ecaa225f65a1d3856e5627957c1df0ff07b Author: Poul-Henning Kamp Date: Sat Nov 26 11:54:58 2022 +0000 Slight revision diff --git a/doc/sphinx/phk/cheri2.rst b/doc/sphinx/phk/cheri2.rst index 9c675af29..71584b40b 100644 --- a/doc/sphinx/phk/cheri2.rst +++ b/doc/sphinx/phk/cheri2.rst @@ -7,18 +7,37 @@ CHERI capabilities are twice the size of pointers, and Varnish not only uses a lot of pointers per request, it is also stingy with RAM, because it is not uncommon to use 100K worker threads. -A number of test-cases fail because they are too stingy with memory -allocations, I will deal with them as I get to them, and merely -note them here as part of the accounting:: +A number of test-cases fail because they are stingy with memory +allocations, and other test-cases run out of cache space. - Increase workspace - ================== +The job of these test-cases is to push varnish into obscure code-paths, +using finely tuned sizes of objects, lengths of headers and parameter +settings to varnish, and the bigger pointers in Varnish trows that +tuning off. + +These test-failures have nothing to do with CHERI. + +There was enough margin that we could find magic numbers which work +on both 32 bit and 64 bit CPUs, that is with both 4 and 8 byte +pointers, but it is doubtful there is enough margin to make them +also work with 16 byte pointers, so I will merely list these tests +here as part of the accounting:: + + Workspace sizes + ===================== TEST tests/c00108.vtc TEST tests/r01038.vtc TEST tests/r01120.vtc TEST tests/r02219.vtc TEST tests/o00005.vtc + Cache sizes + ===================== + TEST tests/r03502.vtc + TEST tests/r01140.vtc + TEST tests/r02831.vtc + TEST tests/v00064.vtc + Things you cannot do under CHERI: Pointers in Pipes --------------------------------------------------- @@ -62,6 +81,8 @@ write a nonce-byte into the pipe to the waiter-thread, but that goes at the bottom of the TODO list, and for now I just remove the -Wpoll argument from five tests, which then pass:: + -Wpoll + ===================== TEST tests/b00009.vtc TEST tests/b00048.vtc TEST tests/b00054.vtc @@ -95,6 +116,8 @@ directives to pointers to make that monster work under CHERI. I just ran my script and 20 more tests pass:: + Red-Black Trees + ===================== TEST tests/b00068.vtc TEST tests/c00005.vtc TEST tests/e00003.vtc @@ -116,6 +139,6 @@ I just ran my script and 20 more tests pass:: TEST tests/v00041.vtc TEST tests/v00043.vtc -Only nine failing tests left now. +Four failures left? */phk* From phk at FreeBSD.org Tue Nov 29 12:48:09 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Nov 2022 12:48:09 +0000 (UTC) Subject: [CHERI] 399c0a481 vtc_server: Fix -Wclobbered variable during dispatch Message-ID: <20221129124809.52D3A6221F@lists.varnish-cache.org> commit 399c0a4811c9084b9c15ad77eaa639e66eb94f72 Author: Dridi Boukelmoune Date: Mon Nov 28 14:46:57 2022 +0100 vtc_server: Fix -Wclobbered variable during dispatch Spotted by GCC's fortification level 3. diff --git a/bin/varnishtest/tests/a00023.vtc b/bin/varnishtest/tests/a00023.vtc new file mode 100644 index 000000000..b35a9ec7e --- /dev/null +++ b/bin/varnishtest/tests/a00023.vtc @@ -0,0 +1,29 @@ +varnishtest "Run server -dispatch more than once" + +feature ignore_unknown_macro + +shell { + cat <<-'EOF' >_.vtc + varnishtest "Run server -dispatch more than once (nested)" + + server s0 { + rxreq + txresp + } -dispatch + + client c1 -connect ${s0_sock} { + txreq + rxresp + } -run + + server s0 -break + server s0 -dispatch + + client c1 -run + EOF + + varnishtest -v _.vtc >_.log +} + +shell -match "s1 +rxhdr" { cat _.log } +shell -match "s2 +rxhdr" { cat _.log } diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index 08103a600..f09131a24 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -336,7 +336,8 @@ static void * server_dispatch_thread(void *priv) { struct server *s, *s2; - int sn = 1, fd; + static int sn = 1; + int fd; char snbuf[8]; struct vtclog *vl; struct sockaddr_storage addr_s; From phk at FreeBSD.org Tue Nov 29 12:48:09 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Nov 2022 12:48:09 +0000 (UTC) Subject: [CHERI] aac3538d4 Hint to the parameter relevant for watchdog panic Message-ID: <20221129124809.6FB706222A@lists.varnish-cache.org> commit aac3538d49af8b74986d128e29d250349cb1c856 Author: Nils Goroll Date: Mon Nov 28 15:57:38 2022 +0100 Hint to the parameter relevant for watchdog panic Motivated by #3868 diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 7c2bd8036..f218d6758 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -629,7 +629,8 @@ pool_herder(void *priv) VSL(SLT_Error, NO_VXID, "Pool Herder: Queue does not move ql=%u dt=%f", pp->lqueue, VTIM_mono() - dqt); - WRONG("Worker Pool Queue does not move"); + WRONG("Worker Pool Queue does not move" + " - see thread_pool_watchdog parameter"); } wthread_min = cache_param->wthread_min; if (pp->die) From phk at FreeBSD.org Tue Nov 29 12:48:09 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Nov 2022 12:48:09 +0000 (UTC) Subject: [CHERI] ae29a70dc varnishtest: set -nolen on tx[req|resp] C-L or T-E Message-ID: <20221129124809.CBBAF62234@lists.varnish-cache.org> commit ae29a70dc66d9837a74a06192993e8a02b69e30a Author: AlveElde Date: Fri Sep 16 16:17:59 2022 +0200 varnishtest: set -nolen on tx[req|resp] C-L or T-E diff --git a/bin/varnishtest/tests/b00037.vtc b/bin/varnishtest/tests/b00037.vtc index 63d8014dc..ce0e84112 100644 --- a/bin/varnishtest/tests/b00037.vtc +++ b/bin/varnishtest/tests/b00037.vtc @@ -13,7 +13,7 @@ varnish v1 -vsl_catchup varnish v1 -expect client_req_400 == 1 client c1 { - txreq -method POST -hdr "Content-Length: 12" -bodylen 12 + txreq -method POST -hdr "Content-Length: 12" -hdr "Content-Length: 12" -bodylen 12 rxresp expect resp.status == 400 } -run diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 827c20da4..aada58c03 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -786,6 +786,9 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, } else if (!strcmp(*av, "-nohost")) { nohost = 1; } else if (!strcmp(*av, "-hdr")) { + if (!strncasecmp(av[1], "Content-Length:", 15) || + !strncasecmp(av[1], "Transfer-Encoding:", 18)) + nolen = 1; if (!strncasecmp(av[1], "Host:", 5)) nohost = 1; VSB_printf(hp->vsb, "%s%s", av[1], nl); From phk at FreeBSD.org Tue Nov 29 12:48:10 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Nov 2022 12:48:10 +0000 (UTC) Subject: [CHERI] 8fb35335e varnishtest: txresp Date header by default Message-ID: <20221129124810.36AF162251@lists.varnish-cache.org> commit 8fb35335e4773411600106b6ffa522b700e4c5be Author: AlveElde Date: Thu Sep 15 12:57:19 2022 +0200 varnishtest: txresp Date header by default This commit prepares varnishtest for a change to Varnish revalidations, where the Last-Modified header must be at least one second older than the Date header for Varnish to send an If-Modified-Since header. Any explicitly defined Date header will override the default, and the Date header can be omitted with -nodate. diff --git a/bin/varnishtest/tests/b00002.vtc b/bin/varnishtest/tests/b00002.vtc index c3feb0722..63fde07a2 100644 --- a/bin/varnishtest/tests/b00002.vtc +++ b/bin/varnishtest/tests/b00002.vtc @@ -3,7 +3,7 @@ varnishtest "Check that a pass transaction works" server s1 { rxreq expect req.proto == HTTP/1.1 - txresp -hdr "Cache-Control: max-age=120" -hdr "Connection: close" -body "012345\n" + txresp -hdr "Cache-Control: max-age=120" -hdr "Connection: close" -nodate -body "012345\n" } -start varnish v1 -arg "-sTransient=default,1m" -vcl+backend { diff --git a/bin/varnishtest/tests/c00036.vtc b/bin/varnishtest/tests/c00036.vtc index 97b51d5cc..adec31ab0 100644 --- a/bin/varnishtest/tests/c00036.vtc +++ b/bin/varnishtest/tests/c00036.vtc @@ -2,13 +2,13 @@ varnishtest "Backend close retry" server s1 -repeat 1 { rxreq - txresp -bodylen 5 + txresp -nodate -bodylen 5 rxreq accept rxreq - txresp -bodylen 6 + txresp -nodate -bodylen 6 } -start diff --git a/bin/varnishtest/tests/l00004.vtc b/bin/varnishtest/tests/l00004.vtc index 5978af8b2..0bff118bb 100644 --- a/bin/varnishtest/tests/l00004.vtc +++ b/bin/varnishtest/tests/l00004.vtc @@ -4,7 +4,7 @@ server s1 { rxreq expect req.url == "/" expect req.http.test == "yes" - txresp -body "fdsa" + txresp -nodate -body "fdsa" } -start varnish v1 -vcl+backend { diff --git a/bin/varnishtest/tests/l00005.vtc b/bin/varnishtest/tests/l00005.vtc index 51667672a..12da731d0 100644 --- a/bin/varnishtest/tests/l00005.vtc +++ b/bin/varnishtest/tests/l00005.vtc @@ -3,7 +3,7 @@ varnishtest "Test backend fetch byte counters" server s1 { rxreq expect req.url == "/1" - txresp -bodylen 1000 + txresp -nodate -bodylen 1000 rxreq expect req.url == "/2" diff --git a/bin/varnishtest/tests/r00498.vtc b/bin/varnishtest/tests/r00498.vtc index c11df3f77..2b08dc202 100644 --- a/bin/varnishtest/tests/r00498.vtc +++ b/bin/varnishtest/tests/r00498.vtc @@ -3,7 +3,7 @@ varnishtest "very very very long return header" server s1 { rxreq expect req.url == "/" - txresp -hdr "Location: ${string,repeat,8136,1}" -body {foo} + txresp -hdr "Location: ${string,repeat,8136,1}" -nodate -body {foo} } -start varnish v1 -vcl+backend { diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index aada58c03..5f94452a8 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -769,7 +769,7 @@ cmd_http_gunzip(CMD_ARGS) static char* const * http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, - char *body, unsigned nohost) + char *body, unsigned nohost, unsigned nodate) { long bodylen = 0; char *b, *c; @@ -785,12 +785,16 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, nolen = 1; } else if (!strcmp(*av, "-nohost")) { nohost = 1; + } else if (!strcmp(*av, "-nodate")) { + nodate = 1; } else if (!strcmp(*av, "-hdr")) { if (!strncasecmp(av[1], "Content-Length:", 15) || !strncasecmp(av[1], "Transfer-Encoding:", 18)) nolen = 1; if (!strncasecmp(av[1], "Host:", 5)) nohost = 1; + if (!strncasecmp(av[1], "Date:", 5)) + nodate = 1; VSB_printf(hp->vsb, "%s%s", av[1], nl); av++; } else if (!strcmp(*av, "-hdrlen")) { @@ -865,6 +869,11 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, macro_cat(vl, hp->vsb, "localhost", NULL); VSB_cat(hp->vsb, nl); } + if (!nodate) { + VSB_cat(hp->vsb, "Date: "); + macro_cat(vl, hp->vsb, "date", NULL); + VSB_cat(hp->vsb, nl); + } if (body != NULL && !nolen) VSB_printf(hp->vsb, "Content-Length: %ld%s", bodylen, nl); VSB_cat(hp->vsb, nl); @@ -914,6 +923,9 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, * \-nolen * Don't include a Content-Length header. * + * \-nodate + * Don't include a Date header in the response. + * * \-hdr STRING * Add STRING as a header, it must follow this format: * "name: value". It can be called multiple times. @@ -988,7 +1000,7 @@ cmd_http_txresp(CMD_ARGS) /* send a "Content-Length: 0" header unless something else happens */ REPLACE(body, ""); - av = http_tx_parse_args(av, vl, hp, body, 1); + av = http_tx_parse_args(av, vl, hp, body, 1, 0); if (*av != NULL) vtc_fatal(hp->vl, "Unknown http txresp spec: %s\n", *av); @@ -1220,7 +1232,7 @@ cmd_http_txreq(CMD_ARGS) "HTTP2-Settings: %s%s", nl, nl, up, nl); nohost = strcmp(proto, "HTTP/1.1") != 0; - av = http_tx_parse_args(av, vl, hp, NULL, nohost); + av = http_tx_parse_args(av, vl, hp, NULL, nohost, 1); if (*av != NULL) vtc_fatal(hp->vl, "Unknown http txreq spec: %s\n", *av); http_write(hp, 4, "txreq"); From phk at FreeBSD.org Tue Nov 29 12:48:10 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Nov 2022 12:48:10 +0000 (UTC) Subject: [CHERI] 634060677 RFC2616: Function to derive LM header strength Message-ID: <20221129124810.5D01062257@lists.varnish-cache.org> commit 63406067777f7c737b5c2f51b3f8d224dd1d6e2f Author: AlveElde Date: Fri Sep 16 16:38:49 2022 +0200 RFC2616: Function to derive LM header strength Per RFC9110, the Last-Modified header is not a strong validator unless it is at least one second older than the Date header. This is to prevent revalidating content that has been changed within a second of the last response. In the case of an intermediate cache like Varnish, a weak Last-Modified validator is "weaker" than a weak ETag, and should not be used for revalidating content. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 49ef26195..c59564737 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -880,6 +880,8 @@ unsigned RFC2616_Req_Gzip(const struct http *); int RFC2616_Do_Cond(const struct req *sp); void RFC2616_Weaken_Etag(struct http *hp); void RFC2616_Vary_AE(struct http *hp); +const char * RFC2616_Strong_LM(struct http *hp, struct worker *wrk, + struct objcore *oc); /* * We want to cache the most recent timestamp in wrk->lastused to avoid diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index 93bc30843..e45ad12fd 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -356,3 +356,36 @@ RFC2616_Vary_AE(struct http *hp) http_SetHeader(hp, "Vary: Accept-Encoding"); } } + +/*--------------------------------------------------------------------*/ + +const char * +RFC2616_Strong_LM(struct http *hp, struct worker *wrk, struct objcore *oc) +{ + const char *p = NULL, *e = NULL; + vtim_real lm, d; + + CHECK_OBJ_ORNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_ORNULL(oc, OBJCORE_MAGIC); + CHECK_OBJ_ORNULL(hp, HTTP_MAGIC); + + if (hp != NULL) { + http_GetHdr(hp, H_Last_Modified, &p); + http_GetHdr(hp, H_Date, &e); + } else if (wrk != NULL && oc != NULL) { + p = HTTP_GetHdrPack(wrk, oc, H_Last_Modified); + e = HTTP_GetHdrPack(wrk, oc, H_Date); + } + + if (p == NULL || e == NULL) + return (NULL); + + lm = VTIM_parse(p); + d = VTIM_parse(e); + + /* The cache entry includes a Date value which is at least one second + * after the Last-Modified value. + * [RFC9110 8.8.2.2-6.2] + */ + return ((lm && d && lm + 1 <= d) ? p : NULL); +} From phk at FreeBSD.org Tue Nov 29 12:48:10 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Nov 2022 12:48:10 +0000 (UTC) Subject: [CHERI] 67dd44a28 fetch: A weak LM header does not make an IMS candidate Message-ID: <20221129124810.8B82E62264@lists.varnish-cache.org> commit 67dd44a289cc2e3e3161308c975178f47aa5f35a Author: AlveElde Date: Fri Sep 16 16:48:50 2022 +0200 fetch: A weak LM header does not make an IMS candidate Avoid marking an object as an IMS candidate when a weak Last-Modified header is the only validator. In the case where there is an ETag AND a weak Last-Modified header, add a If-None-Match header, but do not add an If-Modified-Since header. diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index cf1005aa4..2875d2f91 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -274,7 +274,7 @@ vbf_stp_mkbereq(struct worker *wrk, struct busyobj *bo) ObjCheckFlag(bo->wrk, bo->stale_oc, OF_IMSCAND) && (bo->stale_oc->boc != NULL || ObjGetLen(wrk, bo->stale_oc) != 0)) { AZ(bo->stale_oc->flags & (OC_F_HFM|OC_F_PRIVATE)); - q = HTTP_GetHdrPack(bo->wrk, bo->stale_oc, H_Last_Modified); + q = RFC2616_Strong_LM(NULL, wrk, bo->stale_oc); if (q != NULL) http_PrintfHeader(bo->bereq0, "If-Modified-Since: %s", q); @@ -700,7 +700,7 @@ vbf_stp_fetch(struct worker *wrk, struct busyobj *bo) if (!(oc->flags & OC_F_HFM) && http_IsStatus(bo->beresp, 200) && ( - http_GetHdr(bo->beresp, H_Last_Modified, NULL) || + RFC2616_Strong_LM(bo->beresp, NULL, NULL) != NULL || http_GetHdr(bo->beresp, H_ETag, NULL))) ObjSetFlag(bo->wrk, oc, OF_IMSCAND, 1); diff --git a/bin/varnishtest/tests/b00079.vtc b/bin/varnishtest/tests/b00079.vtc new file mode 100644 index 000000000..efb451261 --- /dev/null +++ b/bin/varnishtest/tests/b00079.vtc @@ -0,0 +1,83 @@ +varnishtest "Test backend IMS with weak and strong LM" + +server s1 { + rxreq + txresp -hdr "Last-Modified: Wed, 11 Sep 2013 13:36:55 GMT" -nodate -body "1" + + # When origin does not send a Date, varnish inserts one, prompting IMS + rxreq + expect req.http.if-modified-since == "Wed, 11 Sep 2013 13:36:55 GMT" + txresp -status 304 -hdr "Last-Modified: Wed, 11 Sep 2013 13:36:55 GMT" \ + -hdr "Date: Wed, 11 Sep 2013 13:36:55 GMT" \ + + # LM was the same as Date + rxreq + expect req.http.if-modified-since == + txresp -hdr "Last-Modified: Wed, 11 Sep 2013 13:36:55 GMT" \ + -hdr "Date: Wed, 11 Sep 2013 13:36:56 GMT" \ + -body "2" + + # LM was one second older than Date + rxreq + expect req.http.if-modified-since == "Wed, 11 Sep 2013 13:36:55 GMT" + txresp -status 304 -hdr "Last-Modified: Wed, 11 Sep 2013 13:36:55 GMT" \ + -hdr "Date: Wed, 11 Sep 2013 13:36:55 GMT" \ + -hdr {ETag: "foo"} + + # LM was the same as Date, but we had an ETag, prompting INM + rxreq + expect req.http.if-modified-since == + expect req.http.if-none-match == {"foo"} + txresp -status 304 -hdr "Last-Modified: Wed, 11 Sep 2013 13:36:55 GMT" \ + -hdr "Date: Wed, 11 Sep 2013 13:36:55 GMT" \ + -hdr {ETag: "foo"} +} -start + +varnish v1 -vcl+backend { + sub vcl_backend_response { + set beresp.ttl = 1ms; + set beresp.grace = 0s; + set beresp.keep = 1m; + set beresp.http.was-304 = beresp.was_304; + } +} -start + +client c1 { + txreq + rxresp + expect resp.status == 200 + expect resp.body == "1" + expect resp.http.was-304 == "false" + + delay 0.1 + + txreq + rxresp + expect resp.status == 200 + expect resp.body == "1" + expect resp.http.was-304 == "true" + + delay 0.1 + + txreq + rxresp + expect resp.status == 200 + expect resp.body == "2" + expect resp.http.was-304 == "false" + + delay 0.1 + + txreq + rxresp + expect resp.status == 200 + expect resp.body == "2" + expect resp.http.was-304 == "true" + + delay 0.1 + + txreq + rxresp + expect resp.status == 200 + expect resp.body == "2" + expect resp.http.was-304 == "true" +} -run From phk at FreeBSD.org Tue Nov 29 12:48:10 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Nov 2022 12:48:10 +0000 (UTC) Subject: [CHERI] e822489d7 Third installment... Message-ID: <20221129124810.B6F466226E@lists.varnish-cache.org> commit e822489d7db63feae84a85d904d4d9e7beaeb5c9 Author: Poul-Henning Kamp Date: Mon Nov 28 16:18:23 2022 +0000 Third installment... diff --git a/doc/sphinx/phk/cheri3.rst b/doc/sphinx/phk/cheri3.rst new file mode 100644 index 000000000..a2995bfae --- /dev/null +++ b/doc/sphinx/phk/cheri3.rst @@ -0,0 +1,66 @@ +.. _phk_cheri_3: + +How Varnish met CHERI 3/N +========================= + +Of the four tests which still fail, three uses the "file stevedore". + +Things you cannot do under CHERI: Pointers in shared files +---------------------------------------------------------- + +In varnish a "stevedore" is responsible for orchestrating storage +for the cached objects, and it is an API extension-point. + +The "malloc stevedore" is the default, it does precisely what you +think it does. + +The "file" stevedore, ``mmap(2)``'s a file in the filesystem +and partitions that out as needed, pretty much like an implementation +of ``malloc(3)`` would do. + +The "file" stevedore exists mainly because back in 2006 mapping a +file with ``MAP_NOCORE``, was the only way to avoid the entire +object cache being included in core-dumps. + +But CHERI will not allow you to put a pointer into a regular file +``mmmap(2)``'ed ``MAP_SHARED``, because that would allow another +process, maybe even on a different computer, to ``mmap(2)`` the +file ``MAP_SHARED`` later and, by implication, resurrect the pointers. + +The "persistent" stevedore mentioned in part 1, does the same thing, +and does not work under CHERI for the same reason. + +If instead we map the file ``MAP_PRIVATE``, nobody else will +ever be able to see the pointers, and those three test cases pass:: + + MAP_SHARED pointers + ===================== + TEST tests/b00005.vtc + TEST tests/r02429.vtc + TEST tests/s00003.vtc + +(We cannot do the same for the "persistent" stevedore, because the +only reason it exists is precisely to to resurrect those pointers later.) + +The final and really nasty test-case +==================================== + +As previously mentioned, when you have 100K threads, you have to be +stingy with memory, in particular with the thread stacks. + +But if you tune things too hard, the threads will step out of their +stack and coredups result. We try to be a little bit helpful +about the diagnostics in such cases, and we have a test-case +which tries to exercise that. + +That test-case is a pretty nasty piece of work, and for all +intents and purposes, it is just a cheap erzats of what CHERI is +supposed to do for us, so I am going to punt on it, and get +to the more interesting parts of this project:: + + SigSegv handler test + ===================== + TEST tests/c00057.vtc + + +*/phk* diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst index f5741e337..6ab783cab 100644 --- a/doc/sphinx/phk/index.rst +++ b/doc/sphinx/phk/index.rst @@ -13,6 +13,7 @@ You may or may not want to know what Poul-Henning thinks. .. toctree:: :maxdepth: 1 + cheri3.rst cheri2.rst cheri1.rst routine.rst From phk at FreeBSD.org Tue Nov 29 12:48:10 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Nov 2022 12:48:10 +0000 (UTC) Subject: [CHERI] af8dc1ee6 man: Implicit -nohost -nodate and -nolen in vtc(7) Message-ID: <20221129124810.E4C626228A@lists.varnish-cache.org> commit af8dc1ee68eafa6865363611678e758de8973829 Author: Dridi Boukelmoune Date: Mon Nov 28 17:00:42 2022 +0100 man: Implicit -nohost -nodate and -nolen in vtc(7) diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 5f94452a8..44c0138cd 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -918,13 +918,17 @@ http_tx_parse_args(char * const *av, struct vtclog *vl, struct http *hp, * following ones. * * \-nohost - * Don't include a Host header in the request. + * Don't include a Host header in the request. Also Implied + * by the addition of a Host header with ``-hdr``. * * \-nolen - * Don't include a Content-Length header. + * Don't include a Content-Length header. Also implied by the + * addition of a Content-Length or Transfer-Encoding header + * with ``-hdr``. * * \-nodate - * Don't include a Date header in the response. + * Don't include a Date header in the response. Also implied + * by the addition of a Date header with ``-hdr``. * * \-hdr STRING * Add STRING as a header, it must follow this format: From phk at FreeBSD.org Tue Nov 29 12:48:11 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Nov 2022 12:48:11 +0000 (UTC) Subject: [CHERI] 0d11960a4 man: Fix rendering of certain sections in vtc(7) Message-ID: <20221129124811.1F11662298@lists.varnish-cache.org> commit 0d11960a4466be2486d81b966d7d052d7543d21e Author: Dridi Boukelmoune Date: Mon Nov 28 17:17:15 2022 +0100 man: Fix rendering of certain sections in vtc(7) The matching regular expression did not align with the substitution regular expression. Refs d2e526ce95d6d8426ec170b4cfc78d41abc07179 diff --git a/doc/sphinx/vtc-syntax.py b/doc/sphinx/vtc-syntax.py index f5533e84a..cfc5a05ab 100644 --- a/doc/sphinx/vtc-syntax.py +++ b/doc/sphinx/vtc-syntax.py @@ -38,7 +38,7 @@ import re def parse_file(fn, cl, tl, sl): p = False section = "" - resec = re.compile("\s*/\* SECTION: ") + resec = re.compile("\s*/?\* SECTION: ") try: # Python3 From phk at FreeBSD.org Tue Nov 29 12:48:11 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Nov 2022 12:48:11 +0000 (UTC) Subject: [CHERI] 953960e62 Typos Message-ID: <20221129124811.419BE622AB@lists.varnish-cache.org> commit 953960e6279bdc42f4422391bce9c893fe0716cb Author: Poul-Henning Kamp Date: Mon Nov 28 17:38:29 2022 +0000 Typos diff --git a/doc/sphinx/phk/cheri3.rst b/doc/sphinx/phk/cheri3.rst index a2995bfae..0d45056c0 100644 --- a/doc/sphinx/phk/cheri3.rst +++ b/doc/sphinx/phk/cheri3.rst @@ -3,7 +3,7 @@ How Varnish met CHERI 3/N ========================= -Of the four tests which still fail, three uses the "file stevedore". +Of the four tests which still fail, three use the "file stevedore". Things you cannot do under CHERI: Pointers in shared files ---------------------------------------------------------- @@ -49,7 +49,7 @@ As previously mentioned, when you have 100K threads, you have to be stingy with memory, in particular with the thread stacks. But if you tune things too hard, the threads will step out of their -stack and coredups result. We try to be a little bit helpful +stack and coredumps result. We try to be a little bit helpful about the diagnostics in such cases, and we have a test-case which tries to exercise that. From phk at FreeBSD.org Tue Nov 29 12:48:11 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Nov 2022 12:48:11 +0000 (UTC) Subject: [CHERI] 4f6566384 Chapter 4 Message-ID: <20221129124811.5FB15622B4@lists.varnish-cache.org> commit 4f6566384745465cf3d8983703d543bfd9ed8101 Author: Poul-Henning Kamp Date: Tue Nov 29 09:40:43 2022 +0000 Chapter 4 diff --git a/doc/sphinx/phk/cheri3.rst b/doc/sphinx/phk/cheri3.rst index 0d45056c0..372d84050 100644 --- a/doc/sphinx/phk/cheri3.rst +++ b/doc/sphinx/phk/cheri3.rst @@ -43,7 +43,7 @@ ever be able to see the pointers, and those three test cases pass:: only reason it exists is precisely to to resurrect those pointers later.) The final and really nasty test-case -==================================== +------------------------------------ As previously mentioned, when you have 100K threads, you have to be stingy with memory, in particular with the thread stacks. diff --git a/doc/sphinx/phk/cheri4.rst b/doc/sphinx/phk/cheri4.rst new file mode 100644 index 000000000..26baeb015 --- /dev/null +++ b/doc/sphinx/phk/cheri4.rst @@ -0,0 +1,179 @@ +.. _phk_cheri_4: + +How Varnish met CHERI 4/N +========================= + +So what can CHERI do ? +---------------------- + +CHERI can restrict pointers, but it does not restrict the memory +they point to:: + + #include + #include + #include + + int + main() + { + char buf[20]; + char *ptr1 = cheri_perms_and(buf, CHERI_PERM_LOAD); + char *ptr2 = buf; + + strcpy(buf, "Hello World\n"); + ptr1[5] = '_'; // Will core dump + ptr2[5] = '_'; // Works fine. + puts(buf); + return (0); + } + +I suspect most programmers will find this counter-intuitive, because +normally it is the memory itself which write-protected in which +case no pointers can ever write to it. + +This is where the word "capability" comes from: The pointer is what +gives you the "capability" to access the memory, and therefore they +can be restricted separately from the memory they provide access to. + +If you could just create your own "capabilities" out of integers, +that would be no big improvement, but you can not: Under CHERI you +can only make a new capability from another capability, and the new +one can never be more potent than the one it is derived from. + +In addition to 'read' and 'write' permissions, capabilities contain the +start and the length of the piece of memory they allow access to. + +Under CHERI the printf(3) pattern "%#p" tells the full story:: + + #include + #include + #include + + int + main() + { + char buf[20]; + char *ptr1 = cheri_perms_and(buf, CHERI_PERM_LOAD); + char *ptr2 = buf; + char *ptr3; + char *ptr4; + + strcpy(buf, "Hello World\n"); + //ptr1[5] = '_'; // Will core dump + ptr2[5] = '_'; // Works fine. + puts(buf); + printf("buf:\t%#p\n", buf); + printf("ptr1:\t%#p\n", ptr1); + printf("ptr2:\t%#p\n", ptr2); + ptr3 = ptr2 + 1; + printf("ptr3:\t%#p\n", ptr3); + ptr4 = cheri_bounds_set(ptr3, 4); + printf("ptr4:\t%#p\n", ptr4); + return (0); + } + +And we get:: + + buf: 0xfffffff7ff68 [rwRW,0xfffffff7ff68-0xfffffff7ff7c] + ptr1: 0xfffffff7ff68 [r,0xfffffff7ff68-0xfffffff7ff7c] + ptr2: 0xfffffff7ff68 [rwRW,0xfffffff7ff68-0xfffffff7ff7c] + ptr3: 0xfffffff7ff69 [rwRW,0xfffffff7ff68-0xfffffff7ff7c] + ptr4: 0xfffffff7ff69 [rwRW,0xfffffff7ff69-0xfffffff7ff6d] + +(Ignore the upper-case 'RW' for now, I'll get back to those later.) + +Because of the odd things people do with pointers in C, ``ptr3`` +keeps the same range as ``ptr2`` and that's a good excuse to +address the question I imagine my readers have at this point: + +What about ``const`` ? +---------------------- + +The C language is a mess. + +Bjarne Stroustrup introduced ``const`` in "C-with-classes", +which later became C++, and in C++ it does the right thing. + +The morons on the ISO-C committee did a half-assed import +in C89, and therefore it does the wrong thing:: + + char *strchr(const char *s, int c); + +You pass a read-only pointer in, and you get a read-write +pointer into that string back ?! + +There were at least three different ways they could have done right: + +* Add a ``cstrchr`` function, having ``const`` on both sides. + +* Enable prototypes to explain this, with some gruesome hack:: + + (const) char *strchr((const) char *s, int c); + +* Allow multiple prototypes for the same function, varying only in const-ness:: + + char *strchr(char *s, int c); + const char *strchr(const char *s, int c); + +But instead they just ignored that issue, and several others like it. + +The result is that we have developed the concept of "const-poisoning", +to describe the fact that if you use "const" to any extent in your +C sources, you almost always end up needing a macro like:: + + #define TRUST_ME(ptr) ((void*)(uintptr_t)(ptr)) + +To remove const-ness where it cannot go. + +(If you think that is ISO-C's opus magnum, ask yourself why we still +cannot specify struct packing and endianess explicitly ? It's hardly +like anybody ever have to apart data-structures explicitly specified in +hardware or protocol documents, is it ?) + +Read/Write markup with CHERI +---------------------------- + +Because ``const`` is such a mess in C, the CHERI compiler does not +automatically remove the write-capability from ``const`` arguments +to functions, something I suspect (but have not checked) that they +can do in C++. + +Instead we will have to do it ourselves, so I have added two macros to +our ```` file:: + + #define RO(x) cheri_perms_and((x), CHERI_PERM_LOAD) + #define ROP(x) cheri_perms_and((x), CHERI_PERM_LOAD|CHERI_PERM_LOAD_CAP) + +Passing a pointer through the ``RO()`` macro makes it read-only, so we +can do stuff like:: + + @@ -285,7 +286,7 @@ VRT_GetHdr(VRT_CTX, VCL_HEADER hs) + [?] + - return (p); + + return (RO(p)); + } + +To explicitly give ``const`` some bite. + +The difference between ``RO`` and ``ROP`` is where the upper- and +lower-case "rw" comes into the picture: Capabilities have two +levels of read/write protection: + +* Can you read or write normal data with this pointer (``CHERI_PERM_LOAD``) + +* Can you read or write pointers with this pointer (``CHERI_PERM_LOAD_CAP``) + +Rule of thumb: Pure data: Use only the first, structs with pointers in them, +use both. + +One can also make write-only pointers with CHERI, but there are +only few places where they can be gainfully employed, outside strict +security in handling of (cryptographic) secrets. + +Right now I'm plunking ``RO()`` and ``ROP()`` into the varnish code, +and one by one relearning what atrocity the 37 uses of ``TRUST_ME()`` +hide. + +Still no bugs found. + +*/phk* diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst index 6ab783cab..5d33fef4e 100644 --- a/doc/sphinx/phk/index.rst +++ b/doc/sphinx/phk/index.rst @@ -13,6 +13,7 @@ You may or may not want to know what Poul-Henning thinks. .. toctree:: :maxdepth: 1 + cheri4.rst cheri3.rst cheri2.rst cheri1.rst From phk at FreeBSD.org Tue Nov 29 12:48:11 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 29 Nov 2022 12:48:11 +0000 (UTC) Subject: [CHERI] 65a219f0b Eliminate need for TRUST_ME() Message-ID: <20221129124811.7FF49622BF@lists.varnish-cache.org> commit 65a219f0bbaa89497294e35407185b37f9e51cce Author: Poul-Henning Kamp Date: Tue Nov 29 12:14:31 2022 +0000 Eliminate need for TRUST_ME() diff --git a/vmod/vmod_debug_acl.c b/vmod/vmod_debug_acl.c index b7b772593..8f277e7a0 100644 --- a/vmod/vmod_debug_acl.c +++ b/vmod/vmod_debug_acl.c @@ -131,9 +131,12 @@ setup_sweep(VRT_CTX, struct acl_sweep *asw, VCL_IP ip0, VCL_IP ip1, } asw->this = asw->reset; - asw->probe = VSA_Clone(ip0); + /* Dont try this at home */ + asw->probe = malloc(vsa_suckaddr_len); + AN(asw->probe); + memcpy(asw->probe, ip0, vsa_suckaddr_len); (void)VSA_GetPtr(asw->probe, &ptr); - asw->probe_p = TRUST_ME(ptr); + asw->probe_p = ((uint8_t*)(asw->probe)) + (ptr - (uint8_t*)asw->probe); asw->step = step; From phk at FreeBSD.org Wed Nov 30 08:48:06 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 30 Nov 2022 08:48:06 +0000 (UTC) Subject: [master] df9bc1860 Make WS_Allocated() assert that pointers are not in the free part of WS Message-ID: <20221130084806.5B7579A48@lists.varnish-cache.org> commit df9bc1860ff4d4c659aca71ec9ebcd4bdd0d31f1 Author: Poul-Henning Kamp Date: Wed Nov 30 08:47:25 2022 +0000 Make WS_Allocated() assert that pointers are not in the free part of WS diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c index d327ad757..d6ff67370 100644 --- a/bin/varnishd/cache/cache_ws.c +++ b/bin/varnishd/cache/cache_ws.c @@ -71,6 +71,7 @@ WS_Allocated(const struct ws *ws, const void *ptr, ssize_t len) WS_Assert(ws); if (len < 0) len = strlen(p) + 1; + assert(!(p > ws->f && p <= ws->e)); return (p >= ws->s && (p + len) <= ws->f); } From dridi at varni.sh Wed Nov 30 08:53:18 2022 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 30 Nov 2022 08:53:18 +0000 Subject: [master] df9bc1860 Make WS_Allocated() assert that pointers are not in the free part of WS In-Reply-To: <20221130084806.5B7579A48@lists.varnish-cache.org> References: <20221130084806.5B7579A48@lists.varnish-cache.org> Message-ID: On Wed, Nov 30, 2022 at 8:48 AM Poul-Henning Kamp wrote: > > > commit df9bc1860ff4d4c659aca71ec9ebcd4bdd0d31f1 > Author: Poul-Henning Kamp > Date: Wed Nov 30 08:47:25 2022 +0000 > > Make WS_Allocated() assert that pointers are not in the free part of WS > > diff --git a/bin/varnishd/cache/cache_ws.c b/bin/varnishd/cache/cache_ws.c > index d327ad757..d6ff67370 100644 > --- a/bin/varnishd/cache/cache_ws.c > +++ b/bin/varnishd/cache/cache_ws.c > @@ -71,6 +71,7 @@ WS_Allocated(const struct ws *ws, const void *ptr, ssize_t len) > WS_Assert(ws); > if (len < 0) > len = strlen(p) + 1; > + assert(!(p > ws->f && p <= ws->e)); > return (p >= ws->s && (p + len) <= ws->f); > } Just in case you were wondering, the workspace emulator already guarantees this. Cheers, Dridi From phk at phk.freebsd.dk Wed Nov 30 09:41:46 2022 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Wed, 30 Nov 2022 09:41:46 +0000 Subject: [master] df9bc1860 Make WS_Allocated() assert that pointers are not in the free part of WS In-Reply-To: References: <20221130084806.5B7579A48@lists.varnish-cache.org> Message-ID: <202211300941.2AU9fkOX094298@critter.freebsd.dk> -------- Dridi Boukelmoune writes: > > + assert(!(p > ws->f && p <= ws->e)); > > return (p >= ws->s && (p + len) <= ws->f); > > } > > Just in case you were wondering, the workspace emulator already guarantees this. yes, but I see no reason to not enforce it in production too. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From dridi at varni.sh Wed Nov 30 09:52:04 2022 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 30 Nov 2022 09:52:04 +0000 Subject: [master] df9bc1860 Make WS_Allocated() assert that pointers are not in the free part of WS In-Reply-To: <202211300941.2AU9fkOX094298@critter.freebsd.dk> References: <20221130084806.5B7579A48@lists.varnish-cache.org> <202211300941.2AU9fkOX094298@critter.freebsd.dk> Message-ID: On Wed, Nov 30, 2022 at 9:41 AM Poul-Henning Kamp wrote: > > -------- > Dridi Boukelmoune writes: > > > > + assert(!(p > ws->f && p <= ws->e)); > > > return (p >= ws->s && (p + len) <= ws->f); > > > } > > > > Just in case you were wondering, the workspace emulator already guarantees this. > > yes, but I see no reason to not enforce it in production too. I was not disagreeing with the change. Cheers, Dridi From phk at FreeBSD.org Wed Nov 30 14:39:05 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 30 Nov 2022 14:39:05 +0000 (UTC) Subject: [master] 6e616f98d Chapter 5 & 6 in the CHERI saga Message-ID: <20221130143905.F184E97CE2@lists.varnish-cache.org> commit 6e616f98d83cced716fd300388fca86a00f24076 Author: Poul-Henning Kamp Date: Wed Nov 30 14:38:34 2022 +0000 Chapter 5 & 6 in the CHERI saga diff --git a/doc/sphinx/phk/cheri5.rst b/doc/sphinx/phk/cheri5.rst new file mode 100644 index 000000000..762d72c0a --- /dev/null +++ b/doc/sphinx/phk/cheri5.rst @@ -0,0 +1,197 @@ +.. _phk_cheri_5: + +How Varnish met CHERI 5/N +========================= + +Varnish Workspaces +------------------ + +To process a HTTP request or response, varnish must allocate bits +of memory which will only be used for the duration of that processing, +and all of it can be released back at the same time. + +To avoid calling ``malloc(3)`` a lot, which comes with a locking +overhead in a heavily multithreaded process, but even more to +avoid having to keep track of all these allocations in order to be able +to ``free(3)`` them all, varnish has "workspaces": + +.. code-block:: none + + struct ws { + [?] + char *s; /* (S)tart of buffer */ + char *f; /* (F)ree/front pointer */ + char *r; /* (R)eserved length */ + char *e; /* (E)nd of buffer */ + }; + +The ``s`` pointer points at the start of a slab of memory, owned +exclusively by the current thread and ``e`` points to the end. + +Initially ``f`` is the same as ``s``, but as allocations are made +from the workspace, it moves towards ``e``. The ``r`` pointer is +used to make "reservations", we will ignore that for now. + +Workspaces look easy to create: + +.. code-block:: none + + ws->s = space; + ws->e = ws->s + len; + ws->f = ws->s; + ws->r = NULL; + +? only, given the foot-shooting-abetting nature of the C language, +we have bolted on a lot of seat-belts: + +.. code-block:: none + + #define WS_ID_SIZE 4 + + struct ws { + unsigned magic; + #define WS_MAGIC 0x35fac554 + char id[WS_ID_SIZE]; /* identity */ + char *s; /* (S)tart of buffer */ + char *f; /* (F)ree/front pointer */ + char *r; /* (R)eserved length */ + char *e; /* (E)nd of buffer */ + }; + + void + WS_Init(struct ws *ws, const char *id, void *space, unsigned len) + { + unsigned l; + + DSLb(DBG_WORKSPACE, + "WS_Init(%s, %p, %p, %u)", id, ws, space, len); + assert(space != NULL); + assert(PAOK(space)); + INIT_OBJ(ws, WS_MAGIC); + ws->s = space; + l = PRNDDN(len - 1); + ws->e = ws->s + l; + memset(ws->e, WS_REDZONE_END, len - l); + ws->f = ws->s; + assert(id[0] & 0x20); // cheesy islower() + bstrcpy(ws->id, id); + WS_Assert(ws); + } + +Let me walk you through that: + +The ``DSLb()`` call can be used to trace all operations on the +workspace, so we can see what actually goes on. + +(Hint: Your ``malloc(3)`` may have something similar, +look for ``utrace`` in the manual page.) + +Next we check the provided space pointer is not NULL, and +that it is properly aligned, these are both following +a varnish style-pattern, to sprinkle asserts liberally, +both as code documentation, but also because it allows +the compiler to optimize things better. + +The ``INIT_OBJ() and ``magic`` field is a style-pattern +we use throughout varnish: Each structure is tagged with +a unique magic, which can be used to ensure that pointers +are what we are told, when they get passed through a ``void*``. + +We set the ``s`` pointer. + +We calculate a length at least one byte shorter than what +we were provided, align it, and point ``e`` at that. + +We fill that extraspace at and past ``e``, with a "canary" to +stochastically detect overruns. It catches most but not +all overruns. + +We set the name of the workspace, ensuring it is not already +marked as overflowed. + +And finally check that the resulting workspace complies with +the defined invariants, as captured in the ``WS_Assert()`` +function. + +With CHERI, it looks like this: + +.. code-block:: none + + void + WS_Init(struct ws *ws, const char *id, void *space, unsigned len) + { + unsigned l; + + DSLb(DBG_WORKSPACE, + "WS_Init(%s, %p, %p, %u)", id, ws, space, len); + assert(space != NULL); + INIT_OBJ(ws, WS_MAGIC); + assert(PAOK(space)); + ws->s = cheri_bounds_set(space, len); + ws->e = ws->s + len + ws->f = ws->s; + assert(id[0] & 0x20); // cheesy islower() + bstrcpy(ws->id, id); + WS_Assert(ws); + } + +All the gunk to implement a canary to detect overruns went +away, because with CHERI we can restrict the ``s`` pointer so writing +outside the workspace is *by definition* impossible, as long as your +pointer is derived from ``s``. + +Less memory wasted, much stronger check and more readable source-code, +what's not to like ? + +When an allocation is made from the workspace, CHERI makes it possible +to restrict the returned pointer to just the allocated space: + +.. code-block:: none + + void * + WS_Alloc(struct ws *ws, unsigned bytes) + { + char *r; + + [?] + r = ws->f; + ws->f += bytes; + return(cheri_bounds_set(r, bytes)); + } + +Varnish String Buffers +---------------------- + +Back in the mists of time, Dag-Erling Sm?rgrav and I designed a +safe string API called ``sbuf`` for the FreeBSD kernel. + +The basic idea is you set up your buffer, you call functions to stuff +text into it, and those functions do all the hard work to ensure +you do not overrun the buffer. When the string is complete, you +call a function to "finish" the buffer, and if returns a flag which +tells you if overrun (or other problems) happened, and then you can +get a pointer to the resulting string from another function. + +Varnish has adopted sbuf's under the name ``vsb``. This should +really not surprise anybody: Dag-Erling was also involved +in the birth of varnish. + +It should be obvious that internally ``vsb`` almost always operate +on a bigger buffer than the result, so this is another obvious +place to have CHERI cut a pointer down to size: + +.. code-block:: none + + char * + VSB_data(const struct vsb *s) + { + + assert_VSB_integrity(s); + assert_VSB_state(s, VSB_FINISHED); + + return (cheri_bounds_set(s->s_buf, s->s_len + 1)); + } + +Still no bugs though. + +*/phk* diff --git a/doc/sphinx/phk/cheri6.rst b/doc/sphinx/phk/cheri6.rst new file mode 100644 index 000000000..adf5d83cb --- /dev/null +++ b/doc/sphinx/phk/cheri6.rst @@ -0,0 +1,179 @@ +.. _phk_cheri_6: + +How Varnish met CHERI 6/N +========================= + +Varnish Socket Addresses +------------------------ + +Socket addresses are a bit of a mess, in particular because nobody +dared shake up all the IPv4 legacy code when IPv6 was introduced. + +In varnish we encapsulate all that uglyness in a ``struct suckaddr``, +so named because it sucks that we have to spend time and code on this. + +In a case like this, it makes sense to make the internals strictly +read-only, to ensure nobody gets sneaky ideas: + +.. code-block:: none + + struct suckaddr * + VSA_Build(void *d, const void *s, unsigned sal) + { + struct suckaddr *sua; + + [? lots of ugly stuff ?] + + return (RO(sua)); + } + +It would then seem logical to use C's ``const`` to signal this fact, +but since the current VSA api is currently defined such that users +call ``free(3)`` on the suckaddrs when they are done with them, that does +not work, because the prototype for ``free(3)`` is: + +.. code-block:: none + + void free(void?*ptr); + +So you cannot call it with a ``const`` pointer. + +All hail the ISO-C standards committee! + +This brings me to a soft point with CHERI: Allocators. + +How to free things with CHERI +----------------------------- + +A very common design-pattern in encapsulating classes look something +like this: + +.. code-block:: none + + struct real_foo { + struct foo foo; + [some metadata about foo] + }; + + const struct foo * + Make_Foo([arguments]) + { + struct real_foo *rf; + + rf = calloc(1, sizeof *rf); + if (rf == NULL) + return (rf); + [fill in rf] + return (&rf->foo); + } + + void + Free_Foo(const struct foo **ptr) + { + const struct foo *fp; + struct real_foo *rfp; + + assert(ptr != NULL); + fp = *ptr; + assert(fp != NULL); + *ptr = NULL; + + rfp = (struct real_foo*)((uintptr_t)fp); + [clean stuff up] + } + +We pass a ``**ptr`` to ``Free_Foo()``, another varnish style-pattern, +so we can NULL out the holding variable in the calling function, +to avoid a dangling pointer to the now destroyed object from +causing any kind of trouble later. + +In the calling function this looks like: + +.. code-block:: none + + const struct foo *foo_ptr; + [?] + Free_Foo(&foo_ptr); + +If we use CHERI to make the foo truly ``const`` for the users of +the API, we cannot, as above, wash the ``const`` away with a trip through +``uintptr_t`` and then write to the metadata. + +The CHERI C/C++ manual, a terse academic tome, laconically mention that: + +*?Therefore, some additional work may be required to derive +a pointer to the allocation?s metadata via another global capability, +rather than the one that has been passed to free().?* + +Despite the understatement, I am very much in favour of this, because +this is *precisely* why my own +`phkmalloc `_ +became a big hit twenty years ago: By firmly separating the metadata +from the allocated space, several new classes of mistakes using the +``malloc(3)`` API could, and was, detected. + +But this *is* going to be an embuggerance for CHERI users, because +with CHERI getting from one pointer to different one is actual work. + +The only "proper" solution is to build some kind of datastructure: +List, tree, hash, DB2 database, pick any poison you prefer, and +search out the metadata pointer using the impotent pointer as key. +Given that CHERI pointers are huge, it may be a better idea to embed +a numeric index in the object and use that the key, + +An important benefit of this ?additional work? is that if your +free-function get passed a pointer to something else, you will +find out, because it is not in your data-structure. + +It would be a good idea if CHERI came with a quality implementation +of "Find my other pointer", so that nobody is forced to crack The +Art of Computer Programming open for this. + +When the API is "fire-and-forget" like VSA, in the sense that there +is no metadata to clean up, we can leave the hard work to the +``malloc(3)`` implementation. + +Ever since ``phkmalloc`` no relevant implementation of ``malloc(3)`` +has dereferenced the freed pointer, in order to find the metadata +for the allocation. Despite its non-const C prototype ``free(3)``, +will therefore happily handle a ``const`` or even CHERIed read-only +pointer. + +But you *will* have to scrub the ``const`` off with a ``uintptr_t`` +to satisfy the C-compiler: + +.. code-block:: none + + void + VSA_free(const struct suckaddr **vsap) + { + const struct suckaddr *vsa; + + AN(vsap); + vsa = *vsap; + *vsap = NULL; + free((char *)(uintptr_t)vsa); + } + +Or in varnish style: + +.. code-block:: none + + void + VSA_free(const struct suckaddr **vsap) + { + const struct suckaddr *vsa; + + TAKE_OBJ_NOTNULL(vsa, vsap, SUCKADDR_MAGIC); + free(TRUST_ME(vsa)); + } + + +Having been all over this code now, I have decided to constify ``struct +suckaddr`` in varnish, even without CHERI, it is sounder that way. + +It is not bug, but CHERI made it a lot simpler and faster for me +to explore the consequences of this change, so I will forfeit +a score of "half a bug" to CHERI at this point. + +*/phk* diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst index 5d33fef4e..4b792547e 100644 --- a/doc/sphinx/phk/index.rst +++ b/doc/sphinx/phk/index.rst @@ -13,6 +13,8 @@ You may or may not want to know what Poul-Henning thinks. .. toctree:: :maxdepth: 1 + cheri6.rst + cheri5.rst cheri4.rst cheri3.rst cheri2.rst From phk at FreeBSD.org Wed Nov 30 15:30:07 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 30 Nov 2022 15:30:07 +0000 (UTC) Subject: [master] 3d1489800 Typo (or is "Grammaro" a thing ?) Message-ID: <20221130153008.0F4D39C988@lists.varnish-cache.org> commit 3d1489800da4d6357b3c47087b16e4fe9c9e61ab Author: Poul-Henning Kamp Date: Wed Nov 30 15:28:58 2022 +0000 Typo (or is "Grammaro" a thing ?) Spotted as always: by Dridi diff --git a/doc/sphinx/phk/cheri6.rst b/doc/sphinx/phk/cheri6.rst index adf5d83cb..02e2f9401 100644 --- a/doc/sphinx/phk/cheri6.rst +++ b/doc/sphinx/phk/cheri6.rst @@ -110,7 +110,7 @@ this is *precisely* why my own `phkmalloc `_ became a big hit twenty years ago: By firmly separating the metadata from the allocated space, several new classes of mistakes using the -``malloc(3)`` API could, and was, detected. +``malloc(3)`` API could, and were, detected. But this *is* going to be an embuggerance for CHERI users, because with CHERI getting from one pointer to different one is actual work. From phk at FreeBSD.org Wed Nov 30 21:00:16 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 30 Nov 2022 21:00:16 +0000 (UTC) Subject: [master] 62916e0ec Flexelinting Message-ID: <20221130210016.2CF3CAAF27@lists.varnish-cache.org> commit 62916e0eca246f3850c44e1bd50920c1f03d0f0e Author: Poul-Henning Kamp Date: Wed Nov 30 20:59:03 2022 +0000 Flexelinting diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index c59564737..0a3f9e092 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -880,7 +880,7 @@ unsigned RFC2616_Req_Gzip(const struct http *); int RFC2616_Do_Cond(const struct req *sp); void RFC2616_Weaken_Etag(struct http *hp); void RFC2616_Vary_AE(struct http *hp); -const char * RFC2616_Strong_LM(struct http *hp, struct worker *wrk, +const char * RFC2616_Strong_LM(const struct http *hp, struct worker *wrk, struct objcore *oc); /* diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index e45ad12fd..90d588e7b 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -360,7 +360,8 @@ RFC2616_Vary_AE(struct http *hp) /*--------------------------------------------------------------------*/ const char * -RFC2616_Strong_LM(struct http *hp, struct worker *wrk, struct objcore *oc) +RFC2616_Strong_LM(const struct http *hp, struct worker *wrk, + struct objcore *oc) { const char *p = NULL, *e = NULL; vtim_real lm, d; @@ -370,8 +371,8 @@ RFC2616_Strong_LM(struct http *hp, struct worker *wrk, struct objcore *oc) CHECK_OBJ_ORNULL(hp, HTTP_MAGIC); if (hp != NULL) { - http_GetHdr(hp, H_Last_Modified, &p); - http_GetHdr(hp, H_Date, &e); + (void)http_GetHdr(hp, H_Last_Modified, &p); + (void)http_GetHdr(hp, H_Date, &e); } else if (wrk != NULL && oc != NULL) { p = HTTP_GetHdrPack(wrk, oc, H_Last_Modified); e = HTTP_GetHdrPack(wrk, oc, H_Date); From phk at FreeBSD.org Wed Nov 30 21:00:16 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 30 Nov 2022 21:00:16 +0000 (UTC) Subject: [master] 2948843f1 Introduce VSA_free() and constify suckaddrs Message-ID: <20221130210016.999D8AAF2A@lists.varnish-cache.org> commit 2948843f182ffe1feb27ae8e82bf997775d16020 Author: Poul-Henning Kamp Date: Wed Nov 30 20:59:23 2022 +0000 Introduce VSA_free() and constify suckaddrs Inspired by: CHERI diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 594dc7627..d38a51b72 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -247,7 +247,7 @@ vbp_write_proxy_v1(struct vbp_target *vt, int *sock) char addr[VTCP_ADDRBUFSIZE]; char port[VTCP_PORTBUFSIZE]; char vsabuf[vsa_suckaddr_len]; - struct suckaddr *sua; + const struct suckaddr *sua; int proto; struct vsb vsb; diff --git a/bin/varnishd/common/heritage.h b/bin/varnishd/common/heritage.h index 06dbe177d..5b68d96a3 100644 --- a/bin/varnishd/common/heritage.h +++ b/bin/varnishd/common/heritage.h @@ -48,7 +48,7 @@ struct listen_sock { int uds; char *endpoint; const char *name; - struct suckaddr *addr; + const struct suckaddr *addr; const struct transport *transport; const struct uds_perms *perms; unsigned test_heritage; diff --git a/bin/varnishd/mgt/mgt_acceptor.c b/bin/varnishd/mgt/mgt_acceptor.c index 9b649a7b9..06946389d 100644 --- a/bin/varnishd/mgt/mgt_acceptor.c +++ b/bin/varnishd/mgt/mgt_acceptor.c @@ -163,7 +163,7 @@ mk_listen_sock(const struct listen_arg *la, const struct suckaddr *sa) fail = mac_opensocket(ls); VJ_master(JAIL_MASTER_LOW); if (fail) { - free(ls->addr); + VSA_free(&ls->addr); free(ls->endpoint); FREE_OBJ(ls); if (fail != EAFNOSUPPORT) @@ -199,7 +199,7 @@ mac_tcp(void *priv, const struct suckaddr *sa) * port number this VTCP_bind() found us, as if * it was specified by the argv. */ - free(ls->addr); + VSA_free(&ls->addr); ls->addr = VTCP_my_suckaddr(ls->sock); VTCP_myname(ls->sock, abuf, sizeof abuf, pbuf, sizeof pbuf); diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c index 5692335a5..4a21962e9 100644 --- a/bin/varnishd/mgt/mgt_cli.c +++ b/bin/varnishd/mgt/mgt_cli.c @@ -552,7 +552,7 @@ mgt_cli_telnet(const char *T_arg) struct m_addr { unsigned magic; #define M_ADDR_MAGIC 0xbc6217ed - struct suckaddr *sa; + const struct suckaddr *sa; VTAILQ_ENTRY(m_addr) list; }; diff --git a/bin/varnishtest/vtc_barrier.c b/bin/varnishtest/vtc_barrier.c index 9b1f6a026..04f5db027 100644 --- a/bin/varnishtest/vtc_barrier.c +++ b/bin/varnishtest/vtc_barrier.c @@ -133,7 +133,7 @@ barrier_sock_thread(void *priv) struct vtclog *vl; const char *err; char buf[vsa_suckaddr_len]; - struct suckaddr *sua; + const struct suckaddr *sua; char abuf[VTCP_ADDRBUFSIZE], pbuf[VTCP_PORTBUFSIZE]; int i, sock, *conns; diff --git a/bin/varnishtest/vtc_client.c b/bin/varnishtest/vtc_client.c index 883aad017..fb7277d5c 100644 --- a/bin/varnishtest/vtc_client.c +++ b/bin/varnishtest/vtc_client.c @@ -42,6 +42,7 @@ #include "vtc.h" +#include "vsa.h" #include "vss.h" #include "vtcp.h" #include "vus.h" @@ -75,7 +76,7 @@ static VTAILQ_HEAD(, client) clients = VTAILQ_HEAD_INITIALIZER(clients); static void client_proxy(struct vtclog *vl, int fd, int version, const char *spec) { - struct suckaddr *sac, *sas; + const struct suckaddr *sac, *sas; char *p, *p2; p = strdup(spec); @@ -93,8 +94,8 @@ client_proxy(struct vtclog *vl, int fd, int version, const char *spec) if (vtc_send_proxy(fd, version, sac, sas)) vtc_fatal(vl, "Write failed: %s", strerror(errno)); free(p); - free(sac); - free(sas); + VSA_free(&sac); + VSA_free(&sas); } /********************************************************************** diff --git a/bin/varnishtest/vtc_haproxy.c b/bin/varnishtest/vtc_haproxy.c index 7f0e11601..cf71a812e 100644 --- a/bin/varnishtest/vtc_haproxy.c +++ b/bin/varnishtest/vtc_haproxy.c @@ -527,7 +527,7 @@ haproxy_create_mcli(struct haproxy *h) const char *err; char buf[128], addr[128], port[128]; char vsabuf[vsa_suckaddr_len]; - struct suckaddr *sua; + const struct suckaddr *sua; sock = VTCP_listen_on(default_listen_addr, NULL, 100, &err); if (err != NULL) @@ -574,7 +574,7 @@ haproxy_new(const char *name) const char *err; const char *env_args; char vsabuf[vsa_suckaddr_len]; - struct suckaddr *sua; + const struct suckaddr *sua; ALLOC_OBJ(h, HAPROXY_MAGIC); AN(h); @@ -864,7 +864,7 @@ haproxy_build_backends(struct haproxy *h, const char *vsb_data) char buf[128], addr[128], port[128]; const char *err; char vsabuf[vsa_suckaddr_len]; - struct suckaddr *sua; + const struct suckaddr *sua; p = strstr(p, HAPROXY_BE_FD_STR); if (!p) diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index d020b37ec..c61fecc8d 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -49,8 +49,8 @@ #include "vfil.h" #include "vnum.h" #include "vrnd.h" -#include "vss.h" #include "vsa.h" +#include "vss.h" #include "vsub.h" #include "vtcp.h" #include "vtim.h" @@ -605,7 +605,7 @@ i_mode(void) static void ip_magic(void) { - struct suckaddr *sa; + const struct suckaddr *sa; char abuf[VTCP_ADDRBUFSIZE]; char pbuf[VTCP_PORTBUFSIZE]; char *s; @@ -621,7 +621,7 @@ ip_magic(void) AN(sa); bad_backend_fd = VTCP_bind(sa, NULL); if (bad_backend_fd < 0) { - free(sa); + VSA_free(&sa); sa = VSS_ResolveFirst(NULL, "localhost", "0", 0, SOCK_STREAM, 0); AN(sa); bad_backend_fd = VTCP_bind(sa, NULL); @@ -657,7 +657,7 @@ ip_magic(void) extmacro_def("listen_addr", NULL, "%s", abuf); default_listen_addr = strdup(abuf); AN(default_listen_addr); - free(sa); + VSA_free(&sa); /* * We need an IP number which will not repond, ever, and that is a diff --git a/bin/varnishtest/vtc_misc.c b/bin/varnishtest/vtc_misc.c index 78ec28464..30784d61a 100644 --- a/bin/varnishtest/vtc_misc.c +++ b/bin/varnishtest/vtc_misc.c @@ -51,6 +51,7 @@ #include "vnum.h" #include "vre.h" #include "vtcp.h" +#include "vsa.h" #include "vss.h" #include "vtim.h" #include "vus.h" @@ -356,7 +357,7 @@ cmd_delay(CMD_ARGS) static int dns_works(void) { - struct suckaddr *sa; + const struct suckaddr *sa; char abuf[VTCP_ADDRBUFSIZE]; char pbuf[VTCP_PORTBUFSIZE]; @@ -365,7 +366,7 @@ dns_works(void) if (sa == NULL) return (0); VTCP_name(sa, abuf, sizeof abuf, pbuf, sizeof pbuf); - free(sa); + VSA_free(&sa); if (strcmp(abuf, "192.0.2.255")) return (0); @@ -373,7 +374,7 @@ dns_works(void) AF_INET6, SOCK_STREAM, 0); if (sa == NULL) return (1); /* the canary is ipv4 only */ - free(sa); + VSA_free(&sa); return (0); } @@ -384,14 +385,14 @@ dns_works(void) static int ipvx_works(const char *target) { - struct suckaddr *sa; + const struct suckaddr *sa; int fd; sa = VSS_ResolveOne(NULL, target, "0", 0, SOCK_STREAM, 0); if (sa == NULL) return (0); fd = VTCP_bind(sa, NULL); - free(sa); + VSA_free(&sa); if (fd >= 0) { VTCP_close(&fd); return (1); diff --git a/bin/varnishtest/vtc_server.c b/bin/varnishtest/vtc_server.c index f09131a24..3942ae92d 100644 --- a/bin/varnishtest/vtc_server.c +++ b/bin/varnishtest/vtc_server.c @@ -185,7 +185,7 @@ static void server_listen_tcp(struct server *s, const char **errp) { char buf[vsa_suckaddr_len]; - struct suckaddr *sua; + const struct suckaddr *sua; s->sock = VTCP_listen_on(s->listen, "0", s->depth, errp); if (*errp != NULL) diff --git a/bin/varnishtest/vtc_syslog.c b/bin/varnishtest/vtc_syslog.c index ac988e9af..7dc97db7d 100644 --- a/bin/varnishtest/vtc_syslog.c +++ b/bin/varnishtest/vtc_syslog.c @@ -359,7 +359,7 @@ syslog_bind(struct syslog_srv *s) char aaddr[VTCP_ADDRBUFSIZE]; char aport[VTCP_PORTBUFSIZE]; char buf[vsa_suckaddr_len]; - struct suckaddr *sua; + const struct suckaddr *sua; CHECK_OBJ_NOTNULL(s, SYSLOG_SRV_MAGIC); diff --git a/bin/varnishtest/vtc_tunnel.c b/bin/varnishtest/vtc_tunnel.c index 342617c2b..3899cc536 100644 --- a/bin/varnishtest/vtc_tunnel.c +++ b/bin/varnishtest/vtc_tunnel.c @@ -583,7 +583,7 @@ static void tunnel_listen(struct tunnel *t) { char buf[vsa_suckaddr_len]; - struct suckaddr *sua; + const struct suckaddr *sua; const char *err; if (t->lsock >= 0) diff --git a/flint.lnt b/flint.lnt index 3724d99f1..af5a93da6 100644 --- a/flint.lnt +++ b/flint.lnt @@ -9,6 +9,9 @@ //d__flexelint_v9__=1 +fan +// No automatic custody +-ffc + -hm4 /////////////////////////////////////////////////////////////////////// diff --git a/include/vsa.h b/include/vsa.h index 7b7ffb55e..b542b2329 100644 --- a/include/vsa.h +++ b/include/vsa.h @@ -42,9 +42,10 @@ int VSA_Sane(const struct suckaddr *); unsigned VSA_Port(const struct suckaddr *); int VSA_Compare(const struct suckaddr *, const struct suckaddr *); int VSA_Compare_IP(const struct suckaddr *, const struct suckaddr *); -struct suckaddr *VSA_Clone(const struct suckaddr *sua); -struct suckaddr *VSA_getsockname(int, void *, size_t); -struct suckaddr *VSA_getpeername(int, void *, size_t); +const struct suckaddr *VSA_Clone(const struct suckaddr *sua); +const struct suckaddr *VSA_getsockname(int, void *, size_t); +const struct suckaddr *VSA_getpeername(int, void *, size_t); +void VSA_free(const struct suckaddr **); const void *VSA_Get_Sockaddr(const struct suckaddr *, socklen_t *sl); int VSA_Get_Proto(const struct suckaddr *); @@ -52,13 +53,14 @@ int VSA_Get_Proto(const struct suckaddr *); /* * 's' is a sockaddr of some kind, 'sal' is its length */ -struct suckaddr *VSA_Malloc(const void *s, unsigned sal); +const struct suckaddr *VSA_Malloc(const void *s, unsigned sal); /* - * 'd' SHALL point to vsa_suckaddr_len aligned bytes of storage, + * 'd' can point to vsa_suckaddr_len aligned bytes of storage, but + * if it is NULL, malloc will be called. * 's' is a sockaddr of some kind, 'sal' is its length. */ -struct suckaddr *VSA_Build(void *d, const void *s, unsigned sal); +const struct suckaddr *VSA_Build(void *d, const void *s, unsigned sal); /* 'd' SHALL point to vsa_suckaddr_len aligned bytes of storage * @@ -69,7 +71,7 @@ struct suckaddr *VSA_Build(void *d, const void *s, unsigned sal); * NULL or 0 length argument are ignored. * argument of the wrong length are an error (NULL return value, EINVAL) */ -struct suckaddr * VSA_BuildFAP(void *d, sa_family_t fam, +const struct suckaddr * VSA_BuildFAP(void *d, sa_family_t fam, const void *a, unsigned al, const void *p, unsigned pl); /* diff --git a/include/vss.h b/include/vss.h index 4695743b2..5db0bb7e6 100644 --- a/include/vss.h +++ b/include/vss.h @@ -36,9 +36,9 @@ int VSS_resolver(const char *addr, const char *def_port, vss_resolved_f *func, void *priv, const char **err); int VSS_resolver_socktype(const char *addr, const char *def_port, vss_resolved_f *func, void *priv, const char **err, int socktype); -struct suckaddr *VSS_ResolveOne(void *dst, +const struct suckaddr *VSS_ResolveOne(void *dst, const char *addr, const char *port, int family, int socktype, int flags); -struct suckaddr *VSS_ResolveFirst(void *dst, +const struct suckaddr *VSS_ResolveFirst(void *dst, const char *addr, const char *port, int family, int socktype, int flags); diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c index d1f1c91a0..c03fbf91f 100644 --- a/lib/libvarnish/vsa.c +++ b/lib/libvarnish/vsa.c @@ -247,10 +247,11 @@ socklen_t sua_len(const struct sockaddr *sa) * Malloc a suckaddr from a sockaddr of some kind. */ -struct suckaddr * +const struct suckaddr * VSA_Malloc(const void *s, unsigned sal) { - return (VSA_Build(malloc(vsa_suckaddr_len), s, sal)); + + return (VSA_Build(NULL, s, sal)); } /* 'd' SHALL point to vsa_suckaddr_len aligned bytes of storage @@ -262,7 +263,7 @@ VSA_Malloc(const void *s, unsigned sal) * NULL or 0 length argument are ignored. * argument of the wrong length are an error (NULL return value, EINVAL) */ -struct suckaddr * +const struct suckaddr * VSA_BuildFAP(void *d, sa_family_t fam, const void *a, unsigned al, const void *p, unsigned pl) { @@ -306,20 +307,23 @@ VSA_BuildFAP(void *d, sa_family_t fam, const void *a, unsigned al, return (NULL); } -/* 'd' SHALL point to vsa_suckaddr_len aligned bytes of storage */ -struct suckaddr * +const struct suckaddr * VSA_Build(void *d, const void *s, unsigned sal) { struct suckaddr *sua; const struct sockaddr *sa = s; unsigned l; // for flexelint - AN(d); AN(s); l = sua_len(sa); if (l == 0 || l != sal) return (NULL); + if (d == NULL) { + d = malloc(vsa_suckaddr_len); + AN(d); + } + sua = d; INIT_OBJ(sua, SUCKADDR_MAGIC); @@ -399,7 +403,7 @@ VSA_Compare_IP(const struct suckaddr *sua1, const struct suckaddr *sua2) NEEDLESS(return (-1)); } -struct suckaddr * +const struct suckaddr * VSA_Clone(const struct suckaddr *sua) { struct suckaddr *sua2; @@ -427,7 +431,7 @@ VSA_Port(const struct suckaddr *sua) } #define VSA_getname(which) \ -struct suckaddr * \ +const struct suckaddr * \ VSA_get ## which ## name(int fd, void *d, size_t l) \ { \ struct suckaddr *sua; \ @@ -452,3 +456,13 @@ VSA_get ## which ## name(int fd, void *d, size_t l) \ VSA_getname(sock) VSA_getname(peer) #undef VSA_getname + +void +VSA_free(const struct suckaddr **vsap) +{ + const struct suckaddr *vsa; + + TAKE_OBJ_NOTNULL(vsa, vsap, SUCKADDR_MAGIC); + free(TRUST_ME(vsa)); +} + diff --git a/lib/libvarnish/vss.c b/lib/libvarnish/vss.c index 45a3d2f2a..4601f27d5 100644 --- a/lib/libvarnish/vss.c +++ b/lib/libvarnish/vss.c @@ -139,7 +139,7 @@ vss_resolve(const char *addr, const char *def_port, int family, int socktype, return (ret); } -static struct suckaddr * +static const struct suckaddr * vss_alloc_suckaddr(void *dst, const struct addrinfo *ai) { @@ -162,7 +162,7 @@ VSS_resolver_socktype(const char *addr, const char *def_port, vss_resolved_f *func, void *priv, const char **errp, int socktype) { struct addrinfo *res0 = NULL, *res; - struct suckaddr *vsa; + const struct suckaddr *vsa; int ret; AN(addr); @@ -178,7 +178,7 @@ VSS_resolver_socktype(const char *addr, const char *def_port, vsa = VSA_Malloc(res->ai_addr, res->ai_addrlen); if (vsa != NULL) { ret = func(priv, vsa); - free(vsa); + VSA_free(&vsa); if (ret) break; } @@ -195,12 +195,12 @@ VSS_resolver(const char *addr, const char *def_port, vss_resolved_f *func, addr, def_port, func, priv, errp, SOCK_STREAM)); } -struct suckaddr * +const struct suckaddr * VSS_ResolveOne(void *dst, const char *addr, const char *def_port, int family, int socktype, int flags) { struct addrinfo *res = NULL; - struct suckaddr *retval = NULL; + const struct suckaddr *retval = NULL; const char *err; int ret; @@ -215,12 +215,12 @@ VSS_ResolveOne(void *dst, const char *addr, const char *def_port, return (retval); } -struct suckaddr * +const struct suckaddr * VSS_ResolveFirst(void *dst, const char *addr, const char *def_port, int family, int socktype, int flags) { struct addrinfo *res0 = NULL, *res; - struct suckaddr *retval = NULL; + const struct suckaddr *retval = NULL; const char *err; int ret; diff --git a/lib/libvarnish/vtcp.c b/lib/libvarnish/vtcp.c index 6c1d154a2..d46142b46 100644 --- a/lib/libvarnish/vtcp.c +++ b/lib/libvarnish/vtcp.c @@ -134,7 +134,7 @@ void VTCP_hisname(int sock, char *abuf, unsigned alen, char *pbuf, unsigned plen) { char buf[vsa_suckaddr_len]; - struct suckaddr *sua; + const struct suckaddr *sua; sua = VSA_getpeername(sock, buf, sizeof buf); if (sua != NULL) diff --git a/lib/libvcc/vcc_acl.c b/lib/libvcc/vcc_acl.c index 4ccefea4f..811f0bf3b 100644 --- a/lib/libvcc/vcc_acl.c +++ b/lib/libvcc/vcc_acl.c @@ -164,7 +164,7 @@ vcc_acl_chk(struct vcc *tl, const struct acl_e *ae, const int l, char t[VTCP_ADDRBUFSIZE + 10]; char s[vsa_suckaddr_len]; char *r = NULL; - struct suckaddr *sa; + const struct suckaddr *sa; unsigned m; int ll, ret = 0; diff --git a/lib/libvcc/vcc_utils.c b/lib/libvcc/vcc_utils.c index 9264200c1..50abffe75 100644 --- a/lib/libvcc/vcc_utils.c +++ b/lib/libvcc/vcc_utils.c @@ -173,8 +173,8 @@ struct rss { unsigned magic; #define RSS_MAGIC 0x11e966ab - struct suckaddr *vsa4; - struct suckaddr *vsa6; + const struct suckaddr *vsa4; + const struct suckaddr *vsa6; struct vsb *vsb; int retval; int wrong; @@ -246,8 +246,10 @@ Resolve_Sockaddr(struct vcc *tl, "(Sorry if that error message is gibberish.)\n", errid, PF(t_err), err); vcc_ErrWhere(tl, t_err); - free(rss->vsa4); - free(rss->vsa6); + if (rss->vsa4 != NULL) + VSA_free(&rss->vsa4); + if (rss->vsa6 != NULL) + VSA_free(&rss->vsa6); VSB_destroy(&rss->vsb); ZERO_OBJ(rss, sizeof rss); return; @@ -255,11 +257,11 @@ Resolve_Sockaddr(struct vcc *tl, AZ(error); if (rss->vsa4 != NULL) { vcc_suckaddr(tl, host, rss->vsa4, ipv4, ipv4_ascii, p_ascii); - free(rss->vsa4); + VSA_free(&rss->vsa4); } if (rss->vsa6 != NULL) { vcc_suckaddr(tl, host, rss->vsa6, ipv6, ipv6_ascii, p_ascii); - free(rss->vsa6); + VSA_free(&rss->vsa6); } if (rss->retval == 0) { VSB_printf(tl->sb, diff --git a/vmod/vmod_debug_dyn.c b/vmod/vmod_debug_dyn.c index 8e9676c2c..2f1a1d180 100644 --- a/vmod/vmod_debug_dyn.c +++ b/vmod/vmod_debug_dyn.c @@ -64,7 +64,7 @@ static void dyn_dir_init(VRT_CTX, struct xyzzy_debug_dyn *dyn, VCL_STRING addr, VCL_STRING port, VCL_PROBE probe) { - struct suckaddr *sa; + const struct suckaddr *sa; VCL_BACKEND dir, dir2; struct vrt_endpoint vep; struct vrt_backend vrt; @@ -105,7 +105,7 @@ dyn_dir_init(VRT_CTX, struct xyzzy_debug_dyn *dyn, if (dir2 != NULL) VRT_delete_backend(ctx, &dir2); - free(sa); + VSA_free(&sa); } VCL_VOID From phk at FreeBSD.org Wed Nov 30 21:47:08 2022 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 30 Nov 2022 21:47:08 +0000 (UTC) Subject: [master] fa66a3289 Include Message-ID: <20221130214708.1AD55AC7DD@lists.varnish-cache.org> commit fa66a32891319f4874af319816df8c823c00054b Author: Poul-Henning Kamp Date: Wed Nov 30 21:46:40 2022 +0000 Include diff --git a/lib/libvarnish/vsa.c b/lib/libvarnish/vsa.c index c03fbf91f..90b2713e7 100644 --- a/lib/libvarnish/vsa.c +++ b/lib/libvarnish/vsa.c @@ -36,6 +36,7 @@ #include "config.h" #include +#include #include #include #include