From phk at FreeBSD.org Thu Jan 12 14:17:12 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 12 Jan 2023 14:17:12 +0000 (UTC) Subject: [master] ebc18974f Various auxillary 64-bit VXID adjustments Message-ID: <20230112141712.C54371003B8@lists.varnish-cache.org> commit ebc18974f980ed2335e551f6b0d6f670941686cd Author: Poul-Henning Kamp Date: Thu Jan 12 14:16:23 2023 +0000 Various auxillary 64-bit VXID adjustments diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 87d85d935..117e710c6 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -139,10 +139,10 @@ ved_include(struct req *preq, const char *src, const char *host, wrk->stats->esi_req++; req->esi_level = preq->esi_level + 1; - VSLb(req->vsl, SLT_Begin, "req %ju esi %u", VXID(preq->vsl->wid), - req->esi_level); - VSLb(preq->vsl, SLT_Link, "req %ju esi %u", VXID(req->vsl->wid), - req->esi_level); + VSLb(req->vsl, SLT_Begin, "req %ju esi %u", + (uintmax_t)VXID(preq->vsl->wid), req->esi_level); + VSLb(preq->vsl, SLT_Link, "req %ju esi %u", + (uintmax_t)VXID(req->vsl->wid), req->esi_level); VSLb_ts_req(req, "Start", W_TIM_real(wrk)); diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index ff6ef5c39..107506938 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -179,7 +179,7 @@ THR_Init(void) * zero vxid, in order to reserve that for "unassociated" VSL records. */ -static uint32_t vxid_base; +static uint64_t vxid_base; static uint32_t vxid_chunk = 32768; static struct lock vxid_lock; @@ -221,14 +221,14 @@ cli_debug_xid(struct cli *cli, const char * const *av, void *priv) { (void)priv; if (av[2] != NULL) { - vxid_base = strtoul(av[2], NULL, 0); + vxid_base = strtoull(av[2], NULL, 0); vxid_chunk = 0; if (av[3] != NULL) vxid_chunk = strtoul(av[3], NULL, 0); if (vxid_chunk == 0) vxid_chunk = 1; } - VCLI_Out(cli, "XID is %u chunk %u", vxid_base, vxid_chunk); + VCLI_Out(cli, "XID is %ju chunk %u", (uintmax_t)vxid_base, vxid_chunk); } /* diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 2e335b97c..20077e182 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -66,7 +66,7 @@ extern const struct req_step R_STP_TRANSPORT[1]; extern const struct req_step R_STP_RECV[1]; struct vxid_pool { - uint32_t next; + uint64_t next; uint32_t count; }; diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index 3b9350f6d..d7b8216c5 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -748,7 +748,7 @@ VRT_INT_string(VRT_CTX, VCL_INT num) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if (!VRT_INT_is_valid(num)) - VRT_fail(ctx, "INT overflow converting to string (%jX)", + VRT_fail(ctx, "INT overflow converting to string (0x%jx)", (intmax_t)num); return (WS_Printf(ctx->ws, "%jd", (intmax_t)num)); } diff --git a/bin/varnishncsa/varnishncsa.c b/bin/varnishncsa/varnishncsa.c index 7b1e4234e..37b27834b 100644 --- a/bin/varnishncsa/varnishncsa.c +++ b/bin/varnishncsa/varnishncsa.c @@ -110,7 +110,7 @@ struct format { char *string; const char *const *strptr; char *time_fmt; - int32_t *int32; + int64_t *int64; }; struct watch { @@ -159,7 +159,7 @@ static struct ctx { const char *hitmiss; const char *handling; const char *side; - int32_t vxid; + int64_t vxid; } CTX; static void @@ -235,11 +235,11 @@ format_strptr(const struct format *format) } static int v_matchproto_(format_f) -format_int32(const struct format *format) +format_int64(const struct format *format) { CHECK_OBJ_NOTNULL(format, FORMAT_MAGIC); - VSB_printf(CTX.vsb, "%" PRIi32, *format->int32); + VSB_printf(CTX.vsb, "%jd", (intmax_t)*format->int64); return (1); } @@ -448,15 +448,15 @@ addf_fragment(struct fragment *frag, const char *str) } static void -addf_int32(int32_t *i) +addf_int64(int64_t *i) { struct format *f; AN(i); ALLOC_OBJ(f, FORMAT_MAGIC); AN(f); - f->func = format_int32; - f->int32 = i; + f->func = format_int64; + f->int64 = i; VTAILQ_INSERT_TAIL(&CTX.format, f, list); } @@ -626,7 +626,7 @@ parse_x_format(char *buf) return; } if (!strcmp(buf, "Varnish:vxid")) { - addf_int32(&CTX.vxid); + addf_int64(&CTX.vxid); return; } if (!strncmp(buf, "VCL_Log:", 8)) { diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 07024e23d..88cb5049f 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -180,7 +180,7 @@ struct logexp_test { VTAILQ_ENTRY(logexp_test) faillist; struct vsb *str; - int vxid; + int64_t vxid; int tag; vre_t *vre; int skip_max; @@ -201,7 +201,7 @@ struct logexp { struct logexp_test *test; int skip_cnt; - int vxid_last; + int64_t vxid_last; int tag_last; struct tests_head fail; @@ -382,7 +382,7 @@ enum le_match_e { static enum le_match_e logexp_match(const struct logexp *le, struct logexp_test *test, - const char *data, int vxid, int tag, int type, int len) + const char *data, int64_t vxid, int tag, int type, int len) { const char *legend; int ok = 1, skip = 0, alt, fail, vxid_ok = 0; @@ -446,8 +446,8 @@ logexp_match(const struct logexp *le, struct logexp_test *test, legend = "err"; if (legend != NULL) - vtc_log(le->vl, 4, "%-5s| %10u %-15s %c %.*s", - legend, vxid, VSL_tags[tag], type, len, + vtc_log(le->vl, 4, "%-5s| %10ju %-15s %c %.*s", + legend, (intmax_t)vxid, VSL_tags[tag], type, len, data); if (ok) { @@ -469,7 +469,7 @@ logexp_match(const struct logexp *le, struct logexp_test *test, static enum le_match_e logexp_failchk(const struct logexp *le, - const char *data, int vxid, int tag, int type, int len) + const char *data, int64_t vxid, int tag, int type, int len) { struct logexp_test *test; static enum le_match_e r; @@ -500,7 +500,8 @@ logexp_dispatch(struct VSL_data *vsl, struct VSL_transaction * const pt[], struct VSL_transaction *t; int i; enum le_match_e r; - int vxid, tag, type, len; + int64_t vxid; + int tag, type, len; const char *data; CAST_OBJ_NOTNULL(le, priv, LOGEXP_MAGIC); @@ -650,7 +651,8 @@ cmd_logexp_common(struct logexp *le, struct vtclog *vl, { vre_t *vre; struct vsb vsb[1]; - int err, pos, tag, vxid; + int64_t vxid; + int err, pos, tag; struct logexp_test *test; char *end, errbuf[VRE_ERROR_LEN]; @@ -659,7 +661,7 @@ cmd_logexp_common(struct logexp *le, struct vtclog *vl, else if (!strcmp(av[2], "=")) vxid = LE_LAST; else { - vxid = (int)strtol(av[2], &end, 10); + vxid = (int)strtoll(av[2], &end, 10); if (*end != '\0' || vxid < 0) vtc_fatal(vl, "Not a positive integer: '%s'", av[2]); } diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index 62e3deec0..f68ed5d2f 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -203,7 +203,7 @@ varnishlog_thread(void *priv) struct vsm *vsm; struct VSL_cursor *c; enum VSL_tag_e tag; - uint32_t vxid; + uint64_t vxid; unsigned len; const char *tagname, *data; int type, i, opt; @@ -259,11 +259,13 @@ varnishlog_thread(void *priv) VSB_quote(vsb, data, len, VSB_QUOTE_HEX); AZ(VSB_finish(vsb)); /* +2 to skip "0x" */ - vtc_log(v->vl, 4, "vsl| %10u %-15s %c [%s]", - vxid, tagname, type, VSB_data(vsb) + 2); + vtc_log(v->vl, 4, "vsl| %10ju %-15s %c [%s]", + (uintmax_t)vxid, tagname, type, + VSB_data(vsb) + 2); } else { - vtc_log(v->vl, 4, "vsl| %10u %-15s %c %.*s", - vxid, tagname, type, (int)len, data); + vtc_log(v->vl, 4, "vsl| %10ju %-15s %c %.*s", + (uintmax_t)vxid, tagname, type, (int)len, + data); } } if (i == 0) { diff --git a/include/vapi/vsl.h b/include/vapi/vsl.h index 4df8e14dc..a1f1528e6 100644 --- a/include/vapi/vsl.h +++ b/include/vapi/vsl.h @@ -112,8 +112,8 @@ enum VSL_reason_e { struct VSL_transaction { int level; - uint32_t vxid; - uint32_t vxid_parent; + int64_t vxid; + int64_t vxid_parent; enum VSL_transaction_e type; enum VSL_reason_e reason; struct VSL_cursor *c; diff --git a/lib/libvarnishapi/generate.py b/lib/libvarnishapi/generate.py index 10bef4257..6ec9a5028 100755 --- a/lib/libvarnishapi/generate.py +++ b/lib/libvarnishapi/generate.py @@ -190,6 +190,7 @@ fo.write(""" #include "config.h" #include +#include #include #include "vdef.h" diff --git a/lib/libvarnishapi/vsl.c b/lib/libvarnishapi/vsl.c index 0a7b2cbf1..8b0f095fa 100644 --- a/lib/libvarnishapi/vsl.c +++ b/lib/libvarnishapi/vsl.c @@ -261,7 +261,7 @@ vsl_print(const struct VSL_data *vsl, const struct VSL_cursor *c, void *fo, int terse) { enum VSL_tag_e tag; - uint32_t vxid; + uint64_t vxid; unsigned len; const char *data; int type; @@ -279,7 +279,7 @@ vsl_print(const struct VSL_data *vsl, const struct VSL_cursor *c, void *fo, data = VSL_CDATA(c->rec.ptr); if (!terse) - VSL_PRINT(fo, "%10u ", vxid); + VSL_PRINT(fo, "%10ju ", (uintmax_t)vxid); VSL_PRINT(fo, "%-14s ", VSL_tags[tag]); if (!terse) VSL_PRINT(fo, "%c ", type); @@ -373,11 +373,11 @@ VSL_PrintTransactions(struct VSL_data *vsl, struct VSL_transaction * const pt[], else VSL_PRINT(fo, "%-3.*s ", (int)(t->level), "***"); - VSL_PRINT(fo, "%*.s%-14s %*.s%-10u\n", + VSL_PRINT(fo, "%*.s%-14s %*.s%-10ju\n", verbose ? 10 + 1 : 0, " ", VSL_transactions[t->type], verbose ? 1 + 1 : 0, " ", - t->vxid); + (uintmax_t)t->vxid); delim = 1; } diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c index 361407f1b..34b8af199 100644 --- a/lib/libvarnishapi/vsl_dispatch.c +++ b/lib/libvarnishapi/vsl_dispatch.c @@ -137,7 +137,7 @@ struct vslc_vtx { }; struct vtx_key { - unsigned vxid; + uint64_t vxid; VRBT_ENTRY(vtx_key) entry; }; VRBT_HEAD(vtx_tree, vtx_key); @@ -603,7 +603,7 @@ vtx_retire(struct VSLQ *vslq, struct vtx **pvtx) /* Lookup a vtx by vxid from the managed list */ static struct vtx * -vtx_lookup(const struct VSLQ *vslq, unsigned vxid) +vtx_lookup(const struct VSLQ *vslq, uint64_t vxid) { struct vtx_key lkey, *key; struct vtx *vtx; @@ -619,7 +619,7 @@ vtx_lookup(const struct VSLQ *vslq, unsigned vxid) /* Insert a new vtx into the managed list */ static struct vtx * -vtx_add(struct VSLQ *vslq, unsigned vxid) +vtx_add(struct VSLQ *vslq, uint64_t vxid) { struct vtx *vtx; @@ -690,10 +690,10 @@ vtx_set_parent(struct vtx *parent, struct vtx *child) successfully parsed. */ static int vtx_parse_link(const char *str, enum VSL_transaction_e *ptype, - unsigned *pvxid, enum VSL_reason_e *preason, unsigned *psub) + uint64_t *pvxid, enum VSL_reason_e *preason, uint64_t *psub) { char type[16], reason[16]; - unsigned vxid, sub; + uintmax_t vxid, sub; int i; enum VSL_transaction_e et; enum VSL_reason_e er; @@ -703,7 +703,7 @@ vtx_parse_link(const char *str, enum VSL_transaction_e *ptype, AN(pvxid); AN(preason); - i = sscanf(str, "%15s %u %15s %u", type, &vxid, reason, &sub); + i = sscanf(str, "%15s %ju %15s %ju", type, &vxid, reason, &sub); if (i < 1) return (0); @@ -746,7 +746,7 @@ vtx_scan_begin(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr) int i; enum VSL_transaction_e type; enum VSL_reason_e reason; - unsigned p_vxid; + uint64_t p_vxid; struct vtx *p_vtx; assert(VSL_TAG(ptr) == SLT_Begin); @@ -813,7 +813,7 @@ vtx_scan_link(struct VSLQ *vslq, struct vtx *vtx, const uint32_t *ptr) int i; enum VSL_transaction_e c_type; enum VSL_reason_e c_reason; - unsigned c_vxid; + uint64_t c_vxid; struct vtx *c_vtx; assert(VSL_TAG(ptr) == SLT_Link); @@ -1295,7 +1295,7 @@ vslq_candidate(struct VSLQ *vslq, const uint32_t *ptr) enum VSL_reason_e reason; struct VSL_data *vsl; enum VSL_tag_e tag; - unsigned p_vxid, sub; + uint64_t p_vxid, sub; int i; CHECK_OBJ_NOTNULL(vslq, VSLQ_MAGIC); @@ -1341,7 +1341,8 @@ vslq_next(struct VSLQ *vslq) enum vsl_status r; enum VSL_tag_e tag; ssize_t len; - unsigned vxid, keep; + uint64_t vxid; + unsigned keep; struct vtx *vtx; c = vslq->c; diff --git a/lib/libvarnishapi/vxp.h b/lib/libvarnishapi/vxp.h index 337c4b519..ce5cd224a 100644 --- a/lib/libvarnishapi/vxp.h +++ b/lib/libvarnishapi/vxp.h @@ -85,7 +85,7 @@ struct vex_lhs { int level; int level_pm; unsigned taglist; - unsigned vxid; + int64_t vxid; }; enum vex_rhs_e { @@ -102,7 +102,7 @@ struct vex_rhs { unsigned magic; #define VEX_RHS_MAGIC 0x3F109965 enum vex_rhs_e type; - long long val_int; + int64_t val_int; double val_float; char *val_string; size_t val_stringlen; diff --git a/lib/libvarnishapi/vxp_test.c b/lib/libvarnishapi/vxp_test.c index d44a9600a..c1bc08833 100644 --- a/lib/libvarnishapi/vxp_test.c +++ b/lib/libvarnishapi/vxp_test.c @@ -31,6 +31,7 @@ #ifndef __FLEXELINT__ #include +#include #include #include #include From phk at FreeBSD.org Mon Jan 16 14:39:11 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 16 Jan 2023 14:39:11 +0000 (UTC) Subject: [master] 7da66c9b0 Improve streaming range responses Message-ID: <20230116143911.9637C114297@lists.varnish-cache.org> commit 7da66c9b03f82b97e2476d5d0af62e58e0419216 Author: Gil Pedersen Date: Tue Nov 29 19:10:31 2022 +0100 Improve streaming range responses This is an enhanced fix for #1777 diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index b9aab9aed..695ca0211 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -124,7 +124,7 @@ vrg_dorange(struct req *req, void **priv) high = req->resp_len - 1; } else if (req->resp_len >= 0 && (high >= req->resp_len || high < 0)) high = req->resp_len - 1; - else if (high < 0 || req->resp_len < 0) + else if (high < 0) return (NULL); // Allow 200 response /* * else (bo != NULL) { @@ -136,13 +136,13 @@ vrg_dorange(struct req *req, void **priv) if (req->resp_len >= 0 && low >= req->resp_len) return ("low range beyond object"); - if (req->resp_len >= 0) + if (req->resp_len >= 0) { http_PrintfHeader(req->resp, "Content-Range: bytes %jd-%jd/%jd", (intmax_t)low, (intmax_t)high, (intmax_t)req->resp_len); - else + req->resp_len = (intmax_t)(1 + high - low); + } else http_PrintfHeader(req->resp, "Content-Range: bytes %jd-%jd/*", (intmax_t)low, (intmax_t)high); - req->resp_len = (intmax_t)(1 + high - low); vrg_priv = WS_Alloc(req->ws, sizeof *vrg_priv); if (vrg_priv == NULL) diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc index bc2af742e..4858d3967 100644 --- a/bin/varnishtest/tests/c00034.vtc +++ b/bin/varnishtest/tests/c00034.vtc @@ -178,12 +178,10 @@ client c5 { -hdr "Range: bytes=2-5" \ -hdr "Accept-encoding: gzip" rxresp - expect resp.status == 200 - expect resp.http.Content-Range == + expect resp.status == 206 + expect resp.http.Content-Range == "bytes 2-5/*" expect resp.http.Content-Length == - expect resp.http.Content-Encoding == gzip - gunzip - expect resp.bodylen == 100 + expect resp.bodylen == 4 } -run # Test partial range with http2 diff --git a/bin/varnishtest/tests/e00015.vtc b/bin/varnishtest/tests/e00015.vtc index 3edc19747..9f90bd3c2 100644 --- a/bin/varnishtest/tests/e00015.vtc +++ b/bin/varnishtest/tests/e00015.vtc @@ -87,8 +87,6 @@ varnish v1 -syntax 4.1 -vcl+backend { } } -# Note on Range requests: The range VDP is active, but as it cannot -# reliably determine the size of the response, it falls back to a 200 client c1 { txreq -url /top2 rxresp @@ -106,11 +104,10 @@ client c1 { expect resp.http.filter0 == "esi" expect resp.http.filters == "esi" - # see Note on Range above txreq -url "/esi" -hdr "Range: bytes=1-2" rxresp - expect resp.bodylen == 76 - expect resp.status == 200 + expect resp.bodylen == 2 + expect resp.status == 206 expect resp.http.was == true expect resp.http.filters == "esi range" @@ -121,11 +118,10 @@ client c1 { expect resp.http.was == true expect resp.http.filters == "esi gunzip" - # see Note on Range above txreq -url "/recurse" -hdr "Range: bytes=1-2" rxresp - expect resp.bodylen == 120 - expect resp.status == 200 + expect resp.bodylen == 2 + expect resp.status == 206 expect resp.http.was == true expect resp.http.filters == "esi gunzip range" @@ -136,4 +132,4 @@ client c1 { } -run varnish v1 -expect esi_errors == 0 -varnish v1 -expect MAIN.s_resp_bodybytes == 865 +varnish v1 -expect MAIN.s_resp_bodybytes == 673 diff --git a/bin/varnishtest/tests/g00005.vtc b/bin/varnishtest/tests/g00005.vtc index 74b21de5b..1240c7a09 100644 --- a/bin/varnishtest/tests/g00005.vtc +++ b/bin/varnishtest/tests/g00005.vtc @@ -25,12 +25,13 @@ varnish v1 -cliok "param.set http_gzip_support true" -vcl+backend { } -start client c1 { - # no range support with streaming cache miss and gunzip + # cache miss txreq -hdr "Range: bytes=3-5" rxresp - expect resp.status == 200 - expect resp.bodylen == "10" + expect resp.status == 206 + expect resp.bodylen == "3" expect resp.http.content-encoding == + expect resp.body == "BAR" } -run varnish v1 -vsl_catchup diff --git a/bin/varnishtest/tests.disabled/r01506.vtc b/bin/varnishtest/tests/r01506.vtc similarity index 92% rename from bin/varnishtest/tests.disabled/r01506.vtc rename to bin/varnishtest/tests/r01506.vtc index 1d0f0bb3c..92192653a 100644 --- a/bin/varnishtest/tests.disabled/r01506.vtc +++ b/bin/varnishtest/tests/r01506.vtc @@ -33,7 +33,6 @@ client c1 { txreq -url /1 -hdr "Range: bytes=17-101" rxresphdrs expect resp.status == 206 - expect resp.http.content-length == 85 barrier b1 sync rxrespbody expect resp.bodylen == 85 @@ -59,15 +58,13 @@ client c1 { expect resp.bodylen == 136 delay .1 - # Invalid range + # Handles out of bounds range txreq -url /4 -hdr "Range: bytes=102-200" rxresphdrs expect resp.status == 206 - expect resp.http.content-length == 99 barrier b1 sync - recv 34 - delay .3 - expect_close + rxrespbody + expect resp.bodylen == 34 } -run varnish v1 -expect sc_range_short == 1 diff --git a/bin/varnishtest/tests.disabled/r01732.vtc b/bin/varnishtest/tests/r01732.vtc similarity index 100% rename from bin/varnishtest/tests.disabled/r01732.vtc rename to bin/varnishtest/tests/r01732.vtc diff --git a/bin/varnishtest/tests/r01777.vtc b/bin/varnishtest/tests/r01777.vtc index 6808023c7..ef8871153 100644 --- a/bin/varnishtest/tests/r01777.vtc +++ b/bin/varnishtest/tests/r01777.vtc @@ -14,5 +14,8 @@ varnish v1 -vcl+backend { } -start client c1 { txreq -hdr "Range: bytes=0-129" rxresp - expect resp.status == 200 + expect resp.status == 206 + expect resp.http.Content-Range == "bytes 0-129/*" + expect resp.http.Content-Length == + expect resp.bodylen == 128 } -run From phk at FreeBSD.org Mon Jan 16 14:39:11 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 16 Jan 2023 14:39:11 +0000 (UTC) Subject: [master] 9ec14234c Don't error on short range response Message-ID: <20230116143911.AECA111429A@lists.varnish-cache.org> commit 9ec14234c889324ffdc8e7362fdd1073626dbc8b Author: Gil Pedersen Date: Wed Dec 14 13:42:55 2022 +0100 Don't error on short range response diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index 695ca0211..ac1a45bc6 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -55,7 +55,8 @@ vrg_range_fini(struct vdp_ctx *vdc, void **priv) CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); CAST_OBJ_NOTNULL(vrg_priv, *priv, VRG_PRIV_MAGIC); - if (vrg_priv->range_off < vrg_priv->range_high) { + if (vrg_priv->req->resp_len >= 0 && + vrg_priv->range_off < vrg_priv->range_high) { Req_Fail(vrg_priv->req, SC_RANGE_SHORT); vrg_priv->req->vdc->retval = -1; } diff --git a/bin/varnishtest/tests/r01506.vtc b/bin/varnishtest/tests/r01506.vtc index 92192653a..c9f07d0e9 100644 --- a/bin/varnishtest/tests/r01506.vtc +++ b/bin/varnishtest/tests/r01506.vtc @@ -2,7 +2,7 @@ varnishtest "range requests on streamed response" barrier b1 cond 2 -cyclic -server s1 -repeat 4 { +server s1 -repeat 5 { rxreq txresp -nolen \ -hdr "Transfer-Encoding: chunked" \ @@ -24,11 +24,6 @@ varnish v1 -vcl+backend {} -start varnish v1 -cliok "param.set debug +syncvsl" -logexpect l1 -v v1 -g session { - expect 0 1000 Begin sess - expect * = SessClose RANGE_SHORT -} -start - client c1 { txreq -url /1 -hdr "Range: bytes=17-101" rxresphdrs @@ -65,7 +60,15 @@ client c1 { barrier b1 sync rxrespbody expect resp.bodylen == 34 + delay .1 + + # Keeps working after short response + txreq -url /5 -hdr "Range: bytes=17-101" + rxresphdrs + expect resp.status == 206 + barrier b1 sync + rxrespbody + expect resp.bodylen == 85 } -run -varnish v1 -expect sc_range_short == 1 -logexpect l1 -wait +varnish v1 -expect sc_range_short == 0 diff --git a/bin/varnishtest/tests/r02258.vtc b/bin/varnishtest/tests/r02258.vtc index 552b1b47e..84eabdf8e 100644 --- a/bin/varnishtest/tests/r02258.vtc +++ b/bin/varnishtest/tests/r02258.vtc @@ -14,6 +14,11 @@ server s1 { varnish v1 -vcl+backend { } -start +logexpect l1 -v v1 -g session { + expect 0 1000 Begin sess + expect * = SessClose RANGE_SHORT +} -start + client c1 { txreq -hdr "range: bytes=0-16" rxresp -no_obj @@ -24,6 +29,7 @@ client c1 { } -run varnish v1 -expect MAIN.sc_range_short == 1 +logexpect l1 -wait delay .3 @@ -43,3 +49,5 @@ client c2 { expect rst.err == INTERNAL_ERROR } -run } -run + +varnish v1 -expect MAIN.sc_range_short == 1 From dridi.boukelmoune at gmail.com Mon Jan 16 16:40:06 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 16 Jan 2023 16:40:06 +0000 (UTC) Subject: [master] a50b1d725 vtc: Polish r1506 Message-ID: <20230116164006.D82A8117B89@lists.varnish-cache.org> commit a50b1d725a7cd3486498bcc99938c27b88b2d9c1 Author: Dridi Boukelmoune Date: Mon Jan 16 17:37:25 2023 +0100 vtc: Polish r1506 Opportunity noticed during the review of #3872. diff --git a/bin/varnishtest/tests/r01506.vtc b/bin/varnishtest/tests/r01506.vtc index c9f07d0e9..96b7b54c9 100644 --- a/bin/varnishtest/tests/r01506.vtc +++ b/bin/varnishtest/tests/r01506.vtc @@ -2,7 +2,7 @@ varnishtest "range requests on streamed response" barrier b1 cond 2 -cyclic -server s1 -repeat 5 { +server s0 { rxreq txresp -nolen \ -hdr "Transfer-Encoding: chunked" \ @@ -18,7 +18,7 @@ server s1 -repeat 5 { send "11\r\n7_23456789abcdef\n" chunkedlen 0 -} -start +} -dispatch varnish v1 -vcl+backend {} -start @@ -31,7 +31,6 @@ client c1 { barrier b1 sync rxrespbody expect resp.bodylen == 85 - delay .1 # We cannot do tail-ranges when streaming txreq -url /2 -hdr "Range: bytes=-10" @@ -41,7 +40,6 @@ client c1 { barrier b1 sync rxrespbody expect resp.bodylen == 136 - delay .1 # We cannot do open-ranges when streaming txreq -url /3 -hdr "Range: bytes=17-" @@ -51,7 +49,6 @@ client c1 { barrier b1 sync rxrespbody expect resp.bodylen == 136 - delay .1 # Handles out of bounds range txreq -url /4 -hdr "Range: bytes=102-200" @@ -60,7 +57,6 @@ client c1 { barrier b1 sync rxrespbody expect resp.bodylen == 34 - delay .1 # Keeps working after short response txreq -url /5 -hdr "Range: bytes=17-101" From phk at FreeBSD.org Mon Jan 16 17:12:05 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 16 Jan 2023 17:12:05 +0000 (UTC) Subject: [master] d287d2da7 Add include for alpine Message-ID: <20230116171205.9531C118BA0@lists.varnish-cache.org> commit d287d2da7dacc8972363db142b72d1012c7fd85a Author: Poul-Henning Kamp Date: Mon Jan 16 17:10:55 2023 +0000 Add include for alpine diff --git a/lib/libvarnishapi/vxp_lexer.c b/lib/libvarnishapi/vxp_lexer.c index 18ae314da..827b4027a 100644 --- a/lib/libvarnishapi/vxp_lexer.c +++ b/lib/libvarnishapi/vxp_lexer.c @@ -35,6 +35,7 @@ #include #include #include +#include #include /* for MUSL */ #include "vdef.h" From phk at FreeBSD.org Tue Jan 17 10:52:08 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 17 Jan 2023 10:52:08 +0000 (UTC) Subject: [master] 925d3f05b Change CLI cmd debug.xid to set the next XID to be used, rather than "one less than the next XID to be used" Message-ID: <20230117105208.0FD8B11384D@lists.varnish-cache.org> commit 925d3f05b9d88cfb759bb6d2b0ec04457a68ee59 Author: Poul-Henning Kamp Date: Tue Jan 17 10:51:10 2023 +0000 Change CLI cmd debug.xid to set the next XID to be used, rather than "one less than the next XID to be used" diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index 107506938..ae414bc01 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -179,7 +179,7 @@ THR_Init(void) * zero vxid, in order to reserve that for "unassociated" VSL records. */ -static uint64_t vxid_base; +static uint64_t vxid_base = 1; static uint32_t vxid_chunk = 32768; static struct lock vxid_lock; @@ -193,18 +193,20 @@ VXID_Get(const struct worker *wrk, uint32_t mask) CHECK_OBJ_NOTNULL(wrk->wpriv, WORKER_PRIV_MAGIC); v = wrk->wpriv->vxid_pool; AZ(mask & VSL_IDENTMASK); - do { - if (v->count == 0) { - Lck_Lock(&vxid_lock); - v->next = vxid_base; - v->count = vxid_chunk; - vxid_base = (vxid_base + v->count) & VSL_IDENTMASK; - Lck_Unlock(&vxid_lock); - } - v->count--; - v->next++; - } while (v->next == 0); + while (v->count == 0 || v->next >= VSL_CLIENTMARKER) { + Lck_Lock(&vxid_lock); + v->next = vxid_base; + v->count = vxid_chunk; + vxid_base += v->count; + if (vxid_base >= VSL_CLIENTMARKER) + vxid_base = 1; + Lck_Unlock(&vxid_lock); + } + v->count--; + assert(v->next > 0); + assert(v->next < VSL_CLIENTMARKER); retval.vxid = v->next | mask; + v->next++; return (retval); } diff --git a/bin/varnishtest/tests/p00000.vtc b/bin/varnishtest/tests/p00000.vtc index 6956a7562..ea803b524 100644 --- a/bin/varnishtest/tests/p00000.vtc +++ b/bin/varnishtest/tests/p00000.vtc @@ -43,7 +43,7 @@ varnish v1 -cliok "debug.persistent s0 sync" varnish v1 -stop varnish v1 -start -varnish v1 -cliok "debug.xid 1999" +varnish v1 -cliok "debug.xid 2000" client c1 { txreq -url "/" diff --git a/bin/varnishtest/tests/p00004.vtc b/bin/varnishtest/tests/p00004.vtc index 986f7ec0c..57c30cdd3 100644 --- a/bin/varnishtest/tests/p00004.vtc +++ b/bin/varnishtest/tests/p00004.vtc @@ -39,7 +39,7 @@ varnish v1 -expect n_object == 2 varnish v1 -stop varnish v1 -start -varnish v1 -cliok "debug.xid 1999" +varnish v1 -cliok "debug.xid 2000" varnish v1 -expect n_vampireobject == 2 varnish v1 -expect n_object == 0 diff --git a/bin/varnishtest/tests/p00006.vtc b/bin/varnishtest/tests/p00006.vtc index dfec3571f..b2ddde8bd 100644 --- a/bin/varnishtest/tests/p00006.vtc +++ b/bin/varnishtest/tests/p00006.vtc @@ -38,7 +38,7 @@ server s1 -wait varnish v1 -stop varnish v1 -start -varnish v1 -cliok "debug.xid 1999" +varnish v1 -cliok "debug.xid 2000" varnish v1 -expect n_vampireobject == 2 diff --git a/bin/varnishtest/tests/r01762.vtc b/bin/varnishtest/tests/r01762.vtc index 8a3fbd426..5c5b39f2b 100644 --- a/bin/varnishtest/tests/r01762.vtc +++ b/bin/varnishtest/tests/r01762.vtc @@ -36,7 +36,7 @@ varnish v1 -cliok "param.set debug +syncvsl" varnish v1 -cliok "debug.xid 1073741823" logexpect l1 -v v1 -g request -T 2 { - expect 0 1 Begin "req 0" + expect 0 1 Begin "req 107374182" expect * = ReqStart expect 0 = ReqMethod GET expect 0 = ReqURL / @@ -80,7 +80,7 @@ varnish v1 -cliok "param.set debug +syncvsl" varnish v1 -cliok "debug.xid 1073741823" logexpect l1 -v v1 -g request { - expect 0 1 Begin "req 0" + expect 0 1 Begin "req 1073741823" expect * = ReqStart expect 0 = ReqMethod GET expect 0 = ReqURL / diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c index f68ed5d2f..12554b853 100644 --- a/bin/varnishtest/vtc_varnish.c +++ b/bin/varnishtest/vtc_varnish.c @@ -620,7 +620,7 @@ varnish_start(struct varnish *v) wait_running(v); free(resp); resp = NULL; - u = varnish_ask_cli(v, "debug.xid 999", &resp); + u = varnish_ask_cli(v, "debug.xid 1000", &resp); if (vtc_error) return; if (u != CLIS_OK) diff --git a/doc/changes.rst b/doc/changes.rst index 043ad7f5f..3e47502bf 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -35,6 +35,9 @@ release process. Varnish Cache NEXT (2023-03-15) =============================== +* The ``debug.xid`` CLI command now sets the next XID to be used, + rather than "one less than the next XID to be used" + ================================ Varnish Cache 7.2.0 (2022-09-15) ================================ From dridi.boukelmoune at gmail.com Tue Jan 17 17:04:08 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 17 Jan 2023 17:04:08 +0000 (UTC) Subject: [master] 79e470485 vtc: Actually do ESI in e25 Message-ID: <20230117170409.04AA111DD81@lists.varnish-cache.org> commit 79e4704858ebb8225dcf0d097ed32c8a1a9057f6 Author: Dridi Boukelmoune Date: Mon Dec 26 10:30:04 2022 +0100 vtc: Actually do ESI in e25 diff --git a/bin/varnishtest/tests/e00025.vtc b/bin/varnishtest/tests/e00025.vtc index e208dc813..fa84031da 100644 --- a/bin/varnishtest/tests/e00025.vtc +++ b/bin/varnishtest/tests/e00025.vtc @@ -1,12 +1,20 @@ varnishtest "Test that esi+gzip correctly bypasses Vary: accept-encoding" server s1 { + rxreq + expect req.http.accept-encoding == gzip + txresp -hdr "Vary: Accept-encoding" -gzipbody {} + rxreq expect req.http.accept-encoding == gzip txresp -hdr "Vary: Accept-encoding" -gzipbody {FOO} } -start -varnish v1 -vcl+backend { } -start +varnish v1 -vcl+backend { + sub vcl_backend_response { + set beresp.do_esi = true; + } +} -start client c1 { txreq -hdr "Accept-encoding: gzip" @@ -15,10 +23,12 @@ client c1 { expect resp.status == 200 gunzip expect resp.bodylen == 3 + expect resp.body == FOO txreq rxresp expect resp.http.content-encoding == expect resp.status == 200 expect resp.bodylen == 3 + expect resp.body == FOO } -run From dridi.boukelmoune at gmail.com Tue Jan 17 17:11:06 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 17 Jan 2023 17:11:06 +0000 (UTC) Subject: [master] 822541f31 req: Typo Message-ID: <20230117171106.7230C4247@lists.varnish-cache.org> commit 822541f31c4615828ba08335a88709db404e8537 Author: Dridi Boukelmoune Date: Tue Dec 27 11:13:50 2022 +0100 req: Typo diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index ac1a66a5d..3deb70e5a 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -860,7 +860,7 @@ cnt_restart(struct worker *wrk, struct req *req) * * TODO * - make restarts == 0 bit re-usable for rollback - * - remove duplicatation with Req_Cleanup() + * - remove duplication with Req_Cleanup() */ static void v_matchproto_(req_state_f) From dridi.boukelmoune at gmail.com Wed Jan 18 06:44:05 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 18 Jan 2023 06:44:05 +0000 (UTC) Subject: [master] 3e0717e51 http: Don't alias range lo and hi fallbacks Message-ID: <20230118064405.53D65111359@lists.varnish-cache.org> commit 3e0717e51d46b2338d5791ffcf4d605996f3c2a0 Author: Dridi Boukelmoune Date: Wed Oct 26 14:34:20 2022 +0200 http: Don't alias range lo and hi fallbacks diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 0a8186861..77120bf3f 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -960,15 +960,15 @@ http_GetContentRange(const struct http *hp, ssize_t *lo, ssize_t *hi) const char * http_GetRange(const struct http *hp, ssize_t *lo, ssize_t *hi) { - ssize_t tmp; + ssize_t tmp_lo, tmp_hi; const char *b, *t; CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); if (lo == NULL) - lo = &tmp; + lo = &tmp_lo; if (hi == NULL) - hi = &tmp; + hi = &tmp_hi; *lo = *hi = -1; From dridi.boukelmoune at gmail.com Wed Jan 18 06:44:05 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 18 Jan 2023 06:44:05 +0000 (UTC) Subject: [master] e8f56d916 range: Move more parsing logic to http_GetRange() Message-ID: <20230118064405.6C64D11135B@lists.varnish-cache.org> commit e8f56d916ce9eb568fd1ff983eb84b55276a3dea Author: Dridi Boukelmoune Date: Wed Oct 26 14:38:52 2022 +0200 range: Move more parsing logic to http_GetRange() diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index f1d107c15..a0e141a48 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -640,7 +640,8 @@ int http_GetHdrField(const struct http *hp, hdr_t, double http_GetHdrQ(const struct http *hp, hdr_t, const char *field); ssize_t http_GetContentLength(const struct http *hp); ssize_t http_GetContentRange(const struct http *hp, ssize_t *lo, ssize_t *hi); -const char * http_GetRange(const struct http *hp, ssize_t *lo, ssize_t *hi); +const char * http_GetRange(const struct http *hp, ssize_t *lo, ssize_t *hi, + ssize_t len); uint16_t http_GetStatus(const struct http *hp); int http_IsStatus(const struct http *hp, int); void http_SetStatus(struct http *to, uint16_t status, const char *reason); diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 77120bf3f..63818eaad 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -958,7 +958,7 @@ http_GetContentRange(const struct http *hp, ssize_t *lo, ssize_t *hi) } const char * -http_GetRange(const struct http *hp, ssize_t *lo, ssize_t *hi) +http_GetRange(const struct http *hp, ssize_t *lo, ssize_t *hi, ssize_t len) { ssize_t tmp_lo, tmp_hi; const char *b, *t; @@ -1003,6 +1003,26 @@ http_GetRange(const struct http *hp, ssize_t *lo, ssize_t *hi) b++; if (*b != '\0') return ("Trailing stuff"); + + assert(*lo >= -1); + assert(*hi >= -1); + + if (len <= 0) + return (NULL); // Allow 200 response + + if (*lo < 0) { + assert(*hi > 0); + *lo = len - *hi; + if (*lo < 0) + *lo = 0; + *hi = len - 1; + } else if (len >= 0 && (*hi >= len || *hi < 0)) { + *hi = len - 1; + } + + if (*lo >= len) + return ("low range beyond object"); + return (NULL); } diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index ac1a45bc6..dc1aa73f3 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -108,25 +108,13 @@ vrg_dorange(struct req *req, void **priv) struct vrg_priv *vrg_priv; const char *err; - err = http_GetRange(req->http, &low, &high); + err = http_GetRange(req->http, &low, &high, req->resp_len); if (err != NULL) return (err); - assert(low >= -1); - assert(high >= -1); - - if (low < 0) { - if (req->resp_len < 0 || high < 0) - return (NULL); // Allow 200 response - assert(high > 0); - low = req->resp_len - high; - if (low < 0) - low = 0; - high = req->resp_len - 1; - } else if (req->resp_len >= 0 && (high >= req->resp_len || high < 0)) - high = req->resp_len - 1; - else if (high < 0) - return (NULL); // Allow 200 response + if (low < 0 || high < 0) + return (NULL); // Allow 200 response + /* * else (bo != NULL) { * We assume that the client knows what it's doing and trust @@ -134,9 +122,6 @@ vrg_dorange(struct req *req, void **priv) * } */ - if (req->resp_len >= 0 && low >= req->resp_len) - return ("low range beyond object"); - if (req->resp_len >= 0) { http_PrintfHeader(req->resp, "Content-Range: bytes %jd-%jd/%jd", (intmax_t)low, (intmax_t)high, (intmax_t)req->resp_len); @@ -268,7 +253,7 @@ VRG_CheckBo(struct busyobj *bo) if (!cache_param->http_range_support) return (0); - err = http_GetRange(bo->bereq0, &rlo, &rhi); + err = http_GetRange(bo->bereq0, &rlo, &rhi, -1); clen = http_GetContentLength(bo->beresp); crlen = http_GetContentRange(bo->beresp, &crlo, &crhi); diff --git a/include/vrt.h b/include/vrt.h index d2af82cc7..e65e59737 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -58,6 +58,7 @@ * binary/load-time compatible, increment MAJOR version * * NEXT (2023-03-15) + * [cache.h] http_GetRange() changed * 16.0 (2022-09-15) * VMOD C-prototypes moved into JSON * VRT_AddVDP() deprecated From dridi.boukelmoune at gmail.com Wed Jan 18 06:44:05 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 18 Jan 2023 06:44:05 +0000 (UTC) Subject: [master] f01063ac5 range: Remove comment gone stale with VRG_CheckBo() Message-ID: <20230118064405.8AB6311135E@lists.varnish-cache.org> commit f01063ac518137fd2b621152aae373b3d1ca3fd5 Author: Dridi Boukelmoune Date: Wed Oct 26 14:52:19 2022 +0200 range: Remove comment gone stale with VRG_CheckBo() diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index dc1aa73f3..8161fd8ce 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -115,13 +115,6 @@ vrg_dorange(struct req *req, void **priv) if (low < 0 || high < 0) return (NULL); // Allow 200 response - /* - * else (bo != NULL) { - * We assume that the client knows what it's doing and trust - * that both low and high make sense. - * } - */ - if (req->resp_len >= 0) { http_PrintfHeader(req->resp, "Content-Range: bytes %jd-%jd/%jd", (intmax_t)low, (intmax_t)high, (intmax_t)req->resp_len); From dridi.boukelmoune at gmail.com Wed Jan 18 14:53:08 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 18 Jan 2023 14:53:08 +0000 (UTC) Subject: [master] 0d2eee601 vrt: Skip the workspace reservation in VRT_IP_string() Message-ID: <20230118145308.431C2513C@lists.varnish-cache.org> commit 0d2eee601192273ffa0b5209da36736002b69c1c Author: Dridi Boukelmoune Date: Mon Jan 16 12:33:20 2023 +0100 vrt: Skip the workspace reservation in VRT_IP_string() diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index d7b8216c5..cc4f1bffb 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -722,7 +722,6 @@ VCL_STRING v_matchproto_() VRT_IP_string(VRT_CTX, VCL_IP ip) { char buf[VTCP_ADDRBUFSIZE]; - struct vsb vsb[1]; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if (ip == NULL) { @@ -730,9 +729,7 @@ VRT_IP_string(VRT_CTX, VCL_IP ip) return (NULL); } VTCP_name(ip, buf, sizeof buf, NULL, 0); - WS_VSB_new(vsb, ctx->ws); - VSB_cat(vsb, buf); - return (WS_VSB_finish(vsb, ctx->ws, NULL)); + return (WS_Copy(ctx->ws, buf, -1)); } int From phk at FreeBSD.org Mon Jan 23 11:45:13 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 23 Jan 2023 11:45:13 +0000 (UTC) Subject: [master] b5443cc8a Make VXID's 64 bit and VRT_INTEGER limited. Message-ID: <20230123114513.720AF10E023@lists.varnish-cache.org> commit b5443cc8a2aa1550ca3e36766ce2d2ceba8a42a7 Author: Poul-Henning Kamp Date: Mon Jan 23 11:44:28 2023 +0000 Make VXID's 64 bit and VRT_INTEGER limited. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index a0e141a48..55a86960a 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -65,8 +65,8 @@ typedef struct vxids vxid_t; #define NO_VXID ((struct vxids){0}) #define IS_NO_VXID(x) ((x).vxid == 0) -#define VXID_TAG(x) ((x).vxid & (VSL_CLIENTMARKER|VSL_BACKENDMARKER)) -#define VXID(u) ((uintmax_t)(u.vxid) & VSL_IDENTMASK) +#define VXID_TAG(x) ((uintmax_t)((x).vxid & (VSL_CLIENTMARKER|VSL_BACKENDMARKER))) +#define VXID(u) ((uintmax_t)((u.vxid) & VSL_IDENTMASK)) #define IS_SAME_VXID(x, y) ((x).vxid == (y).vxid) /*--------------------------------------------------------------------*/ @@ -705,10 +705,6 @@ extern const char H__Reason[]; // rfc7233,l,1207,1208 #define http_range_at(str, tok, l) http_tok_at(str, #tok, l) -/* cache_main.c */ -vxid_t VXID_Get(const struct worker *, uint32_t marker); -extern pthread_key_t witness_key; - /* cache_lck.c */ /* Internal functions, call only through macros below */ diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index ae414bc01..370b8c741 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -184,7 +184,7 @@ static uint32_t vxid_chunk = 32768; static struct lock vxid_lock; vxid_t -VXID_Get(const struct worker *wrk, uint32_t mask) +VXID_Get(const struct worker *wrk, uint64_t mask) { struct vxid_pool *v; vxid_t retval; @@ -193,12 +193,12 @@ VXID_Get(const struct worker *wrk, uint32_t mask) CHECK_OBJ_NOTNULL(wrk->wpriv, WORKER_PRIV_MAGIC); v = wrk->wpriv->vxid_pool; AZ(mask & VSL_IDENTMASK); - while (v->count == 0 || v->next >= VSL_CLIENTMARKER) { + while (v->count == 0 || v->next >= VRT_INTEGER_MAX) { Lck_Lock(&vxid_lock); v->next = vxid_base; v->count = vxid_chunk; vxid_base += v->count; - if (vxid_base >= VSL_CLIENTMARKER) + if (vxid_base >= VRT_INTEGER_MAX) vxid_base = 1; Lck_Unlock(&vxid_lock); } diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c index 7a217d6f8..b3b2e213d 100644 --- a/bin/varnishd/cache/cache_shmlog.c +++ b/bin/varnishd/cache/cache_shmlog.c @@ -143,7 +143,7 @@ vsl_hdr(enum VSL_tag_e tag, uint32_t *p, unsigned len, vxid_t vxid) assert(tag < SLT__Reserved); AZ(len & ~VSL_LENMASK); - p[2] = 0; + p[2] = vxid.vxid >> 32; p[1] = vxid.vxid; p[0] = (((unsigned)tag & VSL_IDMASK) << VSL_IDSHIFT) | (VSL_VERSION_3 << VSL_VERSHIFT) | diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 20077e182..e6dc7e697 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -302,6 +302,9 @@ uint16_t HTTP1_DissectResponse(struct http_conn *, struct http *resp, unsigned HTTP1_Write(const struct worker *w, const struct http *hp, const int*); /* cache_main.c */ +vxid_t VXID_Get(const struct worker *, uint64_t marker); +extern pthread_key_t witness_key; + void THR_SetName(const char *name); const char* THR_GetName(void); void THR_SetBusyobj(const struct busyobj *); diff --git a/bin/varnishtest/tests/c00122.vtc b/bin/varnishtest/tests/c00122.vtc new file mode 100644 index 000000000..d624df224 --- /dev/null +++ b/bin/varnishtest/tests/c00122.vtc @@ -0,0 +1,107 @@ +varnishtest "test 64 bit vxid rollover" + +server s1 { + rxreq + txresp + + accept + rxreq + delay 5 + txresp +} -start + +varnish v1 -vcl+backend { + sub vcl_backend_response { + if (bereq.url == "/retry" && + bereq.retries == 0) { + return (retry); + } + } + sub vcl_deliver { + if (req.url == "/restart" && + req.restarts == 0) { + return (restart); + } + } + +} -start + +process p1 -winsz 100 200 {exec varnishlog -n ${v1_name} -g raw} -start +process p1 -expect-text 0 0 CLI + +varnish v1 -cliok "param.set debug +syncvsl" +varnish v1 -cliok "debug.xid 999999999999998" + +logexpect l1 -v v1 -g request -T 2 { + expect 0 1 Begin "req 999999999999998" + expect * = ReqStart + expect 0 = ReqMethod GET + expect 0 = ReqURL / + expect 0 = ReqProtocol HTTP/1.1 + expect * = ReqHeader "Foo: bar" + expect * = Link "bereq 2 fetch" + expect * = VSL "timeout" + expect * = End "synth" + + expect 0 2 Begin "bereq 1" + expect * 2 Link "bereq 3 retry" + expect * = End + + expect 0 3 Begin "bereq 2 retry" + expect * = End +} -start + +client c1 { + txreq -url "/retry" -hdr "Foo: bar" + rxresp + expect resp.status == 200 +} -run + +varnish v1 -vsl_catchup + +logexpect l1 -wait +process p1 -expect-text 0 1 "999999999999998 SessClose" +process p1 -screen_dump + + +################################################################################ + +server s1 { + rxreq + txresp +} -start + +varnish v1 -cliok "param.set debug +syncvsl" +varnish v1 -cliok "debug.xid 999999999999997" + +logexpect l1 -v v1 -g request { + expect 0 999999999999998 Begin "req 999999999999997" + expect * = ReqStart + expect 0 = ReqMethod GET + expect 0 = ReqURL / + expect 0 = ReqProtocol HTTP/1.1 + expect * = ReqHeader "Foo: bar" + expect * = Link "bereq 1 fetch" + expect * = Link "req 2 restart" + expect * = End + + expect 0 1 Begin "bereq 999999999999998" + expect * = End + + expect 0 2 Begin "req 999999999999998 restart" + expect * = End +} -start + +client c1 { + txreq -url "/restart" -hdr "Foo: bar" + rxresp + expect resp.status == 200 +} -run + +varnish v1 -vsl_catchup +logexpect l1 -wait + +process p1 -expect-text 0 1 "999999999999998 Link c req 2 restart" +process p1 -expect-text 0 1 "999999999999997 SessClose" +process p1 -screen_dump +process p1 -stop diff --git a/bin/varnishtest/tests/r01762.vtc b/bin/varnishtest/tests/r01762.vtc index 5c5b39f2b..d0f96d9bb 100644 --- a/bin/varnishtest/tests/r01762.vtc +++ b/bin/varnishtest/tests/r01762.vtc @@ -1,5 +1,7 @@ varnishtest "test vsl api handling of incomplete vtxes combined with bad vxids" +feature cmd false + server s1 { rxreq txresp diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c index 88cb5049f..7eede8a7e 100644 --- a/bin/varnishtest/vtc_logexp.c +++ b/bin/varnishtest/vtc_logexp.c @@ -661,7 +661,7 @@ cmd_logexp_common(struct logexp *le, struct vtclog *vl, else if (!strcmp(av[2], "=")) vxid = LE_LAST; else { - vxid = (int)strtoll(av[2], &end, 10); + vxid = strtoll(av[2], &end, 10); if (*end != '\0' || vxid < 0) vtc_fatal(vl, "Not a positive integer: '%s'", av[2]); } diff --git a/doc/changes.rst b/doc/changes.rst index 3e47502bf..8d16a3988 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -35,6 +35,24 @@ release process. Varnish Cache NEXT (2023-03-15) =============================== +* VXIDs are 64 bit now and the binary format of SHM and raw saved + VSL files has changed as a consequence. + + The actual valid range for VXIDs is [1?999999999999999], so it + fits in a VRT_INTEGER. + + At one million cache-missing single request sessions per second + VXIDs will roll over in a little over ten years:: + + (1e15-1) / (3 * 1e6 * 86400 * 365) = 10.57 + + That should be enough for everybody?. + + You can test if your downstream log-chewing pipeline handle the + larger VXIDs correctly using the CLI command:: + + ``debug.xid 20000000000`` + * The ``debug.xid`` CLI command now sets the next XID to be used, rather than "one less than the next XID to be used" diff --git a/include/vapi/vsl_int.h b/include/vapi/vsl_int.h index faeb9d111..a40158e04 100644 --- a/include/vapi/vsl_int.h +++ b/include/vapi/vsl_int.h @@ -62,9 +62,9 @@ * changing corresponding magic numbers in varnishd/cache/cache_shmlog.c */ -#define VSL_CLIENTMARKER (1U<<30) -#define VSL_BACKENDMARKER (1U<<31) -#define VSL_IDENTMASK (~(3U<<30)) +#define VSL_CLIENTMARKER (1ULL<<62) +#define VSL_BACKENDMARKER (1ULL<<63) +#define VSL_IDENTMASK ((1ULL<<51)-1) #define VSL_LENMASK 0xffff #define VSL_VERMASK 0x3 @@ -81,9 +81,10 @@ #define VSL_LEN(ptr) ((ptr)[0] & VSL_LENMASK) #define VSL_VER(ptr) (((ptr)[0] & VSL_VERMASK) >> VSL_VERSHIFT) #define VSL_TAG(ptr) ((enum VSL_tag_e)((ptr)[0] >> VSL_IDSHIFT)) -#define VSL_ID(ptr) (((ptr)[1]) & VSL_IDENTMASK) -#define VSL_CLIENT(ptr) (((ptr)[1]) & VSL_CLIENTMARKER) -#define VSL_BACKEND(ptr) (((ptr)[1]) & VSL_BACKENDMARKER) +#define VSL_ID64(ptr) (((uint64_t)((ptr)[2])<<32) | ((ptr)[1])) +#define VSL_ID(ptr) (VSL_ID64(ptr) & VSL_IDENTMASK) +#define VSL_CLIENT(ptr) (((ptr)[2]) & (VSL_CLIENTMARKER >> 32)) +#define VSL_BACKEND(ptr) (((ptr)[2]) & (VSL_BACKENDMARKER >> 32)) #define VSL_DATA(ptr) ((char*)((ptr)+VSL_OVERHEAD)) #define VSL_CDATA(ptr) ((const char*)((ptr)+VSL_OVERHEAD)) #define VSL_BATCHLEN(ptr) ((ptr)[1]) diff --git a/include/vrt.h b/include/vrt.h index e65e59737..e9ca4f087 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -58,6 +58,7 @@ * binary/load-time compatible, increment MAJOR version * * NEXT (2023-03-15) + * VXID is 64 bit * [cache.h] http_GetRange() changed * 16.0 (2022-09-15) * VMOD C-prototypes moved into JSON diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c index 34b8af199..3e990fb36 100644 --- a/lib/libvarnishapi/vsl_dispatch.c +++ b/lib/libvarnishapi/vsl_dispatch.c @@ -1033,6 +1033,7 @@ vtx_synth_rec(struct vtx *vtx, unsigned tag, const char *fmt, ...) va_list ap; char *buf; int l, buflen; + uint64_t vxid; ALLOC_OBJ(synth, SYNTH_MAGIC); AN(synth); @@ -1046,18 +1047,21 @@ vtx_synth_rec(struct vtx *vtx, unsigned tag, const char *fmt, ...) if (l > buflen - 1) l = buflen - 1; buf[l++] = '\0'; /* NUL-terminated */ - synth->data[1] = vtx->key.vxid; + vxid = vtx->key.vxid; switch (vtx->type) { case VSL_t_req: - synth->data[1] |= VSL_CLIENTMARKER; + vxid |= VSL_CLIENTMARKER; break; case VSL_t_bereq: - synth->data[1] |= VSL_BACKENDMARKER; + vxid |= VSL_BACKENDMARKER; break; default: break; } - synth->data[0] = (((tag & 0xff) << 24) | l); + synth->data[2] = vxid >> 32; + synth->data[1] = vxid; + synth->data[0] = (((tag & VSL_IDMASK) << VSL_IDSHIFT) | + (VSL_VERSION_3 << VSL_VERSHIFT) | l); synth->offset = vtx->c.offset; VTAILQ_FOREACH_REVERSE(it, &vtx->synth, synthhead, list) { diff --git a/vmod/vmod_vtc.c b/vmod/vmod_vtc.c index aa1c2b9ad..6c38db3ae 100644 --- a/vmod/vmod_vtc.c +++ b/vmod/vmod_vtc.c @@ -422,7 +422,7 @@ vmod_vsl(VRT_CTX, VCL_INT id, VCL_STRING tag_s, VCL_ENUM side, VCL_STRANDS s) return; } - if (id < 0 || id > VSL_IDENTMASK) { + if (id < 0 || id > VRT_INTEGER_MAX) { VRT_fail(ctx, "id out of bounds"); return; } From dridi at varni.sh Mon Jan 23 13:32:24 2023 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 23 Jan 2023 13:32:24 +0000 Subject: [master] b5443cc8a Make VXID's 64 bit and VRT_INTEGER limited. In-Reply-To: <20230123114513.720AF10E023@lists.varnish-cache.org> References: <20230123114513.720AF10E023@lists.varnish-cache.org> Message-ID: On Mon, Jan 23, 2023 at 11:45 AM Poul-Henning Kamp wrote: > > > commit b5443cc8a2aa1550ca3e36766ce2d2ceba8a42a7 > Author: Poul-Henning Kamp > Date: Mon Jan 23 11:44:28 2023 +0000 > > Make VXID's 64 bit and VRT_INTEGER limited. > > diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h > index a0e141a48..55a86960a 100644 > --- a/bin/varnishd/cache/cache.h > +++ b/bin/varnishd/cache/cache.h > @@ -65,8 +65,8 @@ typedef struct vxids vxid_t; > > #define NO_VXID ((struct vxids){0}) > #define IS_NO_VXID(x) ((x).vxid == 0) > -#define VXID_TAG(x) ((x).vxid & (VSL_CLIENTMARKER|VSL_BACKENDMARKER)) > -#define VXID(u) ((uintmax_t)(u.vxid) & VSL_IDENTMASK) > +#define VXID_TAG(x) ((uintmax_t)((x).vxid & (VSL_CLIENTMARKER|VSL_BACKENDMARKER))) > +#define VXID(u) ((uintmax_t)((u.vxid) & VSL_IDENTMASK)) > #define IS_SAME_VXID(x, y) ((x).vxid == (y).vxid) > > /*--------------------------------------------------------------------*/ > @@ -705,10 +705,6 @@ extern const char H__Reason[]; > // rfc7233,l,1207,1208 > #define http_range_at(str, tok, l) http_tok_at(str, #tok, l) > > -/* cache_main.c */ > -vxid_t VXID_Get(const struct worker *, uint32_t marker); > -extern pthread_key_t witness_key; > - > /* cache_lck.c */ > > /* Internal functions, call only through macros below */ > diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c > index ae414bc01..370b8c741 100644 > --- a/bin/varnishd/cache/cache_main.c > +++ b/bin/varnishd/cache/cache_main.c > @@ -184,7 +184,7 @@ static uint32_t vxid_chunk = 32768; > static struct lock vxid_lock; > > vxid_t > -VXID_Get(const struct worker *wrk, uint32_t mask) > +VXID_Get(const struct worker *wrk, uint64_t mask) > { > struct vxid_pool *v; > vxid_t retval; > @@ -193,12 +193,12 @@ VXID_Get(const struct worker *wrk, uint32_t mask) > CHECK_OBJ_NOTNULL(wrk->wpriv, WORKER_PRIV_MAGIC); > v = wrk->wpriv->vxid_pool; > AZ(mask & VSL_IDENTMASK); > - while (v->count == 0 || v->next >= VSL_CLIENTMARKER) { > + while (v->count == 0 || v->next >= VRT_INTEGER_MAX) { > Lck_Lock(&vxid_lock); > v->next = vxid_base; > v->count = vxid_chunk; > vxid_base += v->count; > - if (vxid_base >= VSL_CLIENTMARKER) > + if (vxid_base >= VRT_INTEGER_MAX) > vxid_base = 1; > Lck_Unlock(&vxid_lock); > } > diff --git a/bin/varnishd/cache/cache_shmlog.c b/bin/varnishd/cache/cache_shmlog.c > index 7a217d6f8..b3b2e213d 100644 > --- a/bin/varnishd/cache/cache_shmlog.c > +++ b/bin/varnishd/cache/cache_shmlog.c > @@ -143,7 +143,7 @@ vsl_hdr(enum VSL_tag_e tag, uint32_t *p, unsigned len, vxid_t vxid) > assert(tag < SLT__Reserved); > AZ(len & ~VSL_LENMASK); > > - p[2] = 0; > + p[2] = vxid.vxid >> 32; > p[1] = vxid.vxid; > p[0] = (((unsigned)tag & VSL_IDMASK) << VSL_IDSHIFT) | > (VSL_VERSION_3 << VSL_VERSHIFT) | > diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h > index 20077e182..e6dc7e697 100644 > --- a/bin/varnishd/cache/cache_varnishd.h > +++ b/bin/varnishd/cache/cache_varnishd.h > @@ -302,6 +302,9 @@ uint16_t HTTP1_DissectResponse(struct http_conn *, struct http *resp, > unsigned HTTP1_Write(const struct worker *w, const struct http *hp, const int*); > > /* cache_main.c */ > +vxid_t VXID_Get(const struct worker *, uint64_t marker); > +extern pthread_key_t witness_key; > + > void THR_SetName(const char *name); > const char* THR_GetName(void); > void THR_SetBusyobj(const struct busyobj *); > diff --git a/bin/varnishtest/tests/c00122.vtc b/bin/varnishtest/tests/c00122.vtc > new file mode 100644 > index 000000000..d624df224 > --- /dev/null > +++ b/bin/varnishtest/tests/c00122.vtc > @@ -0,0 +1,107 @@ > +varnishtest "test 64 bit vxid rollover" > + > +server s1 { > + rxreq > + txresp > + > + accept > + rxreq > + delay 5 > + txresp > +} -start > + > +varnish v1 -vcl+backend { > + sub vcl_backend_response { > + if (bereq.url == "/retry" && > + bereq.retries == 0) { > + return (retry); > + } > + } > + sub vcl_deliver { > + if (req.url == "/restart" && > + req.restarts == 0) { > + return (restart); > + } > + } > + > +} -start > + > +process p1 -winsz 100 200 {exec varnishlog -n ${v1_name} -g raw} -start > +process p1 -expect-text 0 0 CLI > + > +varnish v1 -cliok "param.set debug +syncvsl" > +varnish v1 -cliok "debug.xid 999999999999998" > + > +logexpect l1 -v v1 -g request -T 2 { > + expect 0 1 Begin "req 999999999999998" > + expect * = ReqStart > + expect 0 = ReqMethod GET > + expect 0 = ReqURL / > + expect 0 = ReqProtocol HTTP/1.1 > + expect * = ReqHeader "Foo: bar" > + expect * = Link "bereq 2 fetch" > + expect * = VSL "timeout" > + expect * = End "synth" > + > + expect 0 2 Begin "bereq 1" > + expect * 2 Link "bereq 3 retry" > + expect * = End > + > + expect 0 3 Begin "bereq 2 retry" > + expect * = End > +} -start > + > +client c1 { > + txreq -url "/retry" -hdr "Foo: bar" > + rxresp > + expect resp.status == 200 > +} -run > + > +varnish v1 -vsl_catchup > + > +logexpect l1 -wait > +process p1 -expect-text 0 1 "999999999999998 SessClose" > +process p1 -screen_dump > + > + > +################################################################################ > + > +server s1 { > + rxreq > + txresp > +} -start > + > +varnish v1 -cliok "param.set debug +syncvsl" > +varnish v1 -cliok "debug.xid 999999999999997" > + > +logexpect l1 -v v1 -g request { > + expect 0 999999999999998 Begin "req 999999999999997" > + expect * = ReqStart > + expect 0 = ReqMethod GET > + expect 0 = ReqURL / > + expect 0 = ReqProtocol HTTP/1.1 > + expect * = ReqHeader "Foo: bar" > + expect * = Link "bereq 1 fetch" > + expect * = Link "req 2 restart" > + expect * = End > + > + expect 0 1 Begin "bereq 999999999999998" > + expect * = End > + > + expect 0 2 Begin "req 999999999999998 restart" > + expect * = End > +} -start > + > +client c1 { > + txreq -url "/restart" -hdr "Foo: bar" > + rxresp > + expect resp.status == 200 > +} -run > + > +varnish v1 -vsl_catchup > +logexpect l1 -wait > + > +process p1 -expect-text 0 1 "999999999999998 Link c req 2 restart" > +process p1 -expect-text 0 1 "999999999999997 SessClose" > +process p1 -screen_dump > +process p1 -stop > diff --git a/bin/varnishtest/tests/r01762.vtc b/bin/varnishtest/tests/r01762.vtc > index 5c5b39f2b..d0f96d9bb 100644 > --- a/bin/varnishtest/tests/r01762.vtc > +++ b/bin/varnishtest/tests/r01762.vtc > @@ -1,5 +1,7 @@ > varnishtest "test vsl api handling of incomplete vtxes combined with bad vxids" > > +feature cmd false > + > server s1 { > rxreq > txresp > diff --git a/bin/varnishtest/vtc_logexp.c b/bin/varnishtest/vtc_logexp.c > index 88cb5049f..7eede8a7e 100644 > --- a/bin/varnishtest/vtc_logexp.c > +++ b/bin/varnishtest/vtc_logexp.c > @@ -661,7 +661,7 @@ cmd_logexp_common(struct logexp *le, struct vtclog *vl, > else if (!strcmp(av[2], "=")) > vxid = LE_LAST; > else { > - vxid = (int)strtoll(av[2], &end, 10); > + vxid = strtoll(av[2], &end, 10); > if (*end != '\0' || vxid < 0) > vtc_fatal(vl, "Not a positive integer: '%s'", av[2]); > } > diff --git a/doc/changes.rst b/doc/changes.rst > index 3e47502bf..8d16a3988 100644 > --- a/doc/changes.rst > +++ b/doc/changes.rst > @@ -35,6 +35,24 @@ release process. > Varnish Cache NEXT (2023-03-15) > =============================== > > +* VXIDs are 64 bit now and the binary format of SHM and raw saved > + VSL files has changed as a consequence. > + > + The actual valid range for VXIDs is [1?999999999999999], so it > + fits in a VRT_INTEGER. > + > + At one million cache-missing single request sessions per second > + VXIDs will roll over in a little over ten years:: > + > + (1e15-1) / (3 * 1e6 * 86400 * 365) = 10.57 > + > + That should be enough for everybody?. > + > + You can test if your downstream log-chewing pipeline handle the > + larger VXIDs correctly using the CLI command:: > + > + ``debug.xid 20000000000`` > + > * The ``debug.xid`` CLI command now sets the next XID to be used, > rather than "one less than the next XID to be used" > > diff --git a/include/vapi/vsl_int.h b/include/vapi/vsl_int.h > index faeb9d111..a40158e04 100644 > --- a/include/vapi/vsl_int.h > +++ b/include/vapi/vsl_int.h > @@ -62,9 +62,9 @@ > * changing corresponding magic numbers in varnishd/cache/cache_shmlog.c > */ > > -#define VSL_CLIENTMARKER (1U<<30) > -#define VSL_BACKENDMARKER (1U<<31) > -#define VSL_IDENTMASK (~(3U<<30)) > +#define VSL_CLIENTMARKER (1ULL<<62) > +#define VSL_BACKENDMARKER (1ULL<<63) > +#define VSL_IDENTMASK ((1ULL<<51)-1) Should we reserve 4 bits to track the ESI level of client requests? That would set an upper limit of 15 for the max_esi_depth, which is probably reasonable. If not, we still have a few more bits that could then be (ab)used for faster VSL query processing. Please note that I'm very well aware that the current "level" syntax for vslq has nothing to do with ESI, so I was also implicitly suggesting we could extend VSL filtering and/or to include esi_level. I understand it is hard to decide on an arbitrary maximum for parameters like max_esi_depth, but I'm willing to make that kind of trade off. Dridi > > #define VSL_LENMASK 0xffff > #define VSL_VERMASK 0x3 > @@ -81,9 +81,10 @@ > #define VSL_LEN(ptr) ((ptr)[0] & VSL_LENMASK) > #define VSL_VER(ptr) (((ptr)[0] & VSL_VERMASK) >> VSL_VERSHIFT) > #define VSL_TAG(ptr) ((enum VSL_tag_e)((ptr)[0] >> VSL_IDSHIFT)) > -#define VSL_ID(ptr) (((ptr)[1]) & VSL_IDENTMASK) > -#define VSL_CLIENT(ptr) (((ptr)[1]) & VSL_CLIENTMARKER) > -#define VSL_BACKEND(ptr) (((ptr)[1]) & VSL_BACKENDMARKER) > +#define VSL_ID64(ptr) (((uint64_t)((ptr)[2])<<32) | ((ptr)[1])) > +#define VSL_ID(ptr) (VSL_ID64(ptr) & VSL_IDENTMASK) > +#define VSL_CLIENT(ptr) (((ptr)[2]) & (VSL_CLIENTMARKER >> 32)) > +#define VSL_BACKEND(ptr) (((ptr)[2]) & (VSL_BACKENDMARKER >> 32)) > #define VSL_DATA(ptr) ((char*)((ptr)+VSL_OVERHEAD)) > #define VSL_CDATA(ptr) ((const char*)((ptr)+VSL_OVERHEAD)) > #define VSL_BATCHLEN(ptr) ((ptr)[1]) > diff --git a/include/vrt.h b/include/vrt.h > index e65e59737..e9ca4f087 100644 > --- a/include/vrt.h > +++ b/include/vrt.h > @@ -58,6 +58,7 @@ > * binary/load-time compatible, increment MAJOR version > * > * NEXT (2023-03-15) > + * VXID is 64 bit > * [cache.h] http_GetRange() changed > * 16.0 (2022-09-15) > * VMOD C-prototypes moved into JSON > diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c > index 34b8af199..3e990fb36 100644 > --- a/lib/libvarnishapi/vsl_dispatch.c > +++ b/lib/libvarnishapi/vsl_dispatch.c > @@ -1033,6 +1033,7 @@ vtx_synth_rec(struct vtx *vtx, unsigned tag, const char *fmt, ...) > va_list ap; > char *buf; > int l, buflen; > + uint64_t vxid; > > ALLOC_OBJ(synth, SYNTH_MAGIC); > AN(synth); > @@ -1046,18 +1047,21 @@ vtx_synth_rec(struct vtx *vtx, unsigned tag, const char *fmt, ...) > if (l > buflen - 1) > l = buflen - 1; > buf[l++] = '\0'; /* NUL-terminated */ > - synth->data[1] = vtx->key.vxid; > + vxid = vtx->key.vxid; > switch (vtx->type) { > case VSL_t_req: > - synth->data[1] |= VSL_CLIENTMARKER; > + vxid |= VSL_CLIENTMARKER; > break; > case VSL_t_bereq: > - synth->data[1] |= VSL_BACKENDMARKER; > + vxid |= VSL_BACKENDMARKER; > break; > default: > break; > } > - synth->data[0] = (((tag & 0xff) << 24) | l); > + synth->data[2] = vxid >> 32; > + synth->data[1] = vxid; > + synth->data[0] = (((tag & VSL_IDMASK) << VSL_IDSHIFT) | > + (VSL_VERSION_3 << VSL_VERSHIFT) | l); > synth->offset = vtx->c.offset; > > VTAILQ_FOREACH_REVERSE(it, &vtx->synth, synthhead, list) { > diff --git a/vmod/vmod_vtc.c b/vmod/vmod_vtc.c > index aa1c2b9ad..6c38db3ae 100644 > --- a/vmod/vmod_vtc.c > +++ b/vmod/vmod_vtc.c > @@ -422,7 +422,7 @@ vmod_vsl(VRT_CTX, VCL_INT id, VCL_STRING tag_s, VCL_ENUM side, VCL_STRANDS s) > return; > } > > - if (id < 0 || id > VSL_IDENTMASK) { > + if (id < 0 || id > VRT_INTEGER_MAX) { > VRT_fail(ctx, "id out of bounds"); > return; > } > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From phk at phk.freebsd.dk Mon Jan 23 13:41:48 2023 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Mon, 23 Jan 2023 13:41:48 +0000 Subject: [master] b5443cc8a Make VXID's 64 bit and VRT_INTEGER limited. In-Reply-To: References: <20230123114513.720AF10E023@lists.varnish-cache.org> Message-ID: <202301231341.30NDfmo2001847@critter.freebsd.dk> -------- Dridi Boukelmoune writes: > On Mon, Jan 23, 2023 at 11:45 AM Poul-Henning Kamp wrote: > > diff --git a/include/vapi/vsl_int.h b/include/vapi/vsl_int.h > > index faeb9d111..a40158e04 100644 > > --- a/include/vapi/vsl_int.h > > +++ b/include/vapi/vsl_int.h > > @@ -62,9 +62,9 @@ > > * changing corresponding magic numbers in varnishd/cache/cache_shmlog.c > > */ > > > > -#define VSL_CLIENTMARKER (1U<<30) > > -#define VSL_BACKENDMARKER (1U<<31) > > -#define VSL_IDENTMASK (~(3U<<30)) > > +#define VSL_CLIENTMARKER (1ULL<<62) > > +#define VSL_BACKENDMARKER (1ULL<<63) > > +#define VSL_IDENTMASK ((1ULL<<51)-1) > > Should we reserve 4 bits to track the ESI level of client requests? > > That would set an upper limit of 15 for the max_esi_depth, which is > probably reasonable. If not, we still have a few more bits that could > then be (ab)used for faster VSL query processing. Please note that I'm > very well aware that the current "level" syntax for vslq has nothing > to do with ESI, so I was also implicitly suggesting we could extend > VSL filtering and/or to include esi_level. > > I understand it is hard to decide on an arbitrary maximum for > parameters like max_esi_depth, but I'm willing to make that kind of > trade off. This reminds me I forgot to update the comment at the top of that file... We have a fair number of spare bits right now, and I have been thinking about things to do with them. I think it would be useful to define a field which VCL can set and VSLQ filter on. If people want to stuff the ESI-level in there, they can. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From phk at FreeBSD.org Mon Jan 23 13:50:07 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 23 Jan 2023 13:50:07 +0000 (UTC) Subject: [master] 361dba0c6 Update comment about VSL layout Message-ID: <20230123135007.1AAB9111C63@lists.varnish-cache.org> commit 361dba0c6e7206b9e8b2498f4e1a3cd0ab8beb4b Author: Poul-Henning Kamp Date: Mon Jan 23 13:49:25 2023 +0000 Update comment about VSL layout diff --git a/include/vapi/vsl_int.h b/include/vapi/vsl_int.h index a40158e04..b3378d103 100644 --- a/include/vapi/vsl_int.h +++ b/include/vapi/vsl_int.h @@ -50,10 +50,20 @@ * The log member points to an array of 32bit unsigned integers containing * log records. * - * Each logrecord consist of: - * [n] = ((type & 0xff) << 24) | (length & 0xffff) - * [n + 1] = ((marker & 0x03) << 30) | (identifier & 0x3fffffff) - * [n + 2] ... [m] = content (NUL-terminated) + * Each logrecord consist of four or more 32 bit words, stored in + * native endiansess: + * + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | TAG | unused |ver| length | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | 32 lower bits of VXID | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |B|C| unused (zero) | 19 upper bits of VXID | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | content ... (NUL-terminated) | + * +-+-+ +-+-+ + * | ... | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ * * Logrecords are NUL-terminated so that string functions can be run * directly on the shmlog data. From dridi.boukelmoune at gmail.com Tue Jan 24 12:46:06 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 24 Jan 2023 12:46:06 +0000 (UTC) Subject: [master] 973dce377 vxp: Missing include for in64_t Message-ID: <20230124124606.D01E9115EAA@lists.varnish-cache.org> commit 973dce377cd469ed5d156d2fe3fb155614ec7a5d Author: Dridi Boukelmoune Date: Tue Jan 24 13:42:16 2023 +0100 vxp: Missing include for in64_t Spotted by our Alpine Linux job on CircleCI. diff --git a/lib/libvarnishapi/vxp.c b/lib/libvarnishapi/vxp.c index f23ffad3f..46e086e50 100644 --- a/lib/libvarnishapi/vxp.c +++ b/lib/libvarnishapi/vxp.c @@ -33,6 +33,7 @@ #include "config.h" #include +#include #include #include #include /* for MUSL */ From phk at FreeBSD.org Mon Jan 30 08:32:09 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 30 Jan 2023 08:32:09 +0000 (UTC) Subject: [master] e54d67342 Fix printf format for uintmax_t. Message-ID: <20230130083209.B5898107AD1@lists.varnish-cache.org> commit e54d67342b44e4f0c7935f438c26c6dc5045d1ce Author: Poul-Henning Kamp Date: Mon Jan 30 08:30:42 2023 +0000 Fix printf format for uintmax_t. Spotted by: Coverity diff --git a/lib/libvarnishapi/vsl_dispatch.c b/lib/libvarnishapi/vsl_dispatch.c index 3e990fb36..a28bc70b1 100644 --- a/lib/libvarnishapi/vsl_dispatch.c +++ b/lib/libvarnishapi/vsl_dispatch.c @@ -1096,7 +1096,7 @@ static int vtx_diag_tag(struct vtx *vtx, const uint32_t *ptr, const char *reason) { - vtx_synth_rec(vtx, SLT_VSL, "%s (%u:%s \"%.*s\")", reason, VSL_ID(ptr), + vtx_synth_rec(vtx, SLT_VSL, "%s (%ju:%s \"%.*s\")", reason, VSL_ID(ptr), VSL_tags[VSL_TAG(ptr)], (int)VSL_LEN(ptr), VSL_CDATA(ptr)); return (-1); } From dridi.boukelmoune at gmail.com Mon Jan 30 10:59:05 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 30 Jan 2023 10:59:05 +0000 (UTC) Subject: [master] 118fd10c9 build: Move misplaced suncc configure check Message-ID: <20230130105905.C785C10E160@lists.varnish-cache.org> commit 118fd10c9e81920e0c31906c2bf5c62a3f25cd5b Author: Dridi Boukelmoune Date: Mon Jan 30 11:23:58 2023 +0100 build: Move misplaced suncc configure check diff --git a/configure.ac b/configure.ac index 4de4012ba..dcaffeb5e 100644 --- a/configure.ac +++ b/configure.ac @@ -249,14 +249,14 @@ LIBS="${save_LIBS}" AC_CHECK_DECL([__SUNPRO_C], [SUNCC="yes"], [SUNCC="no"]) -# Support for visibility attribute -save_CFLAGS="${CFLAGS}" if test "$SUNCC" = "yes" ; then CFLAGS="${CFLAGS} -errwarn=%all,no%E_EMPTY_TRANSLATION_UNIT" else CFLAGS="${CFLAGS} -Werror" fi +# Support for visibility attribute +save_CFLAGS="${CFLAGS}" AC_CACHE_CHECK([whether we have support for visibility attributes], [ac_cv_have_viz], [AC_RUN_IFELSE( From dridi.boukelmoune at gmail.com Mon Jan 30 10:59:05 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 30 Jan 2023 10:59:05 +0000 (UTC) Subject: [master] e41b6ca0d build: Better handle zlib-specific C flags Message-ID: <20230130105905.DCA0A10E163@lists.varnish-cache.org> commit e41b6ca0da5fff01a854aaddebf217158bad12ec Author: Dridi Boukelmoune Date: Mon Jan 30 11:27:57 2023 +0100 build: Better handle zlib-specific C flags There are two warnings that we enforce for our own code that zlib does not. There's also the visibility attribute that we check at configure time. And regarding the visibility attribute, zlib no longer relies on a NO_VIZ macro and aligned with the autoconf naming convention and wants HAVE_HIDDEN instead. diff --git a/configure.ac b/configure.ac index dcaffeb5e..93c92e29b 100644 --- a/configure.ac +++ b/configure.ac @@ -255,27 +255,25 @@ else CFLAGS="${CFLAGS} -Werror" fi -# Support for visibility attribute -save_CFLAGS="${CFLAGS}" +# zlib-specific flags +libvgz_extra_cflags="-Wno-error=strict-prototypes" + +dnl https://github.com/madler/zlib/issues/633 +libvgz_extra_cflags="$libvgz_extra_cflags -Wno-error=deprecated-non-prototype" + +AC_SUBST(libvgz_extra_cflags) + +# Support for visibility attribute (zlib) AC_CACHE_CHECK([whether we have support for visibility attributes], [ac_cv_have_viz], - [AC_RUN_IFELSE( + [AC_COMPILE_IFELSE( [AC_LANG_PROGRAM([[ - #if ((__GNUC__-0) * 10 + __GNUC_MINOR__-0 >= 33 || defined(__SUNPRO_C)) - # define ZLIB_INTERNAL __attribute__((visibility ("hidden"))) - #else - # define ZLIB_INTERNAL - #endif - int ZLIB_INTERNAL foo; + int __attribute__((visibility ("hidden"))) foo; ]],[])], - [ac_cv_have_viz=yes], - [ac_cv_have_viz=no]) + [AC_DEFINE([HAVE_HIDDEN], [1], + [Define to 1 if visibility attribute hidden is available.]) + ]) ]) -if test "$ac_cv_have_viz" = no; then - libvgz_extra_cflags="-DNO_VIZ" - AC_SUBST(libvgz_extra_cflags) -fi -CFLAGS="${save_CFLAGS}" AC_ARG_ENABLE(ubsan, AS_HELP_STRING([--enable-ubsan], diff --git a/lib/libvgz/Makefile.am b/lib/libvgz/Makefile.am index 52fd2f0fb..929646ad5 100644 --- a/lib/libvgz/Makefile.am +++ b/lib/libvgz/Makefile.am @@ -1,11 +1,13 @@ # -AM_LDFLAGS = $(AM_LT_LDFLAGS) +AM_CFLAGS = $(AM_LT_CFLAGS) \ + -D_LARGEFILE64_SOURCE=1 \ + -DZLIB_CONST \ + $(libvgz_extra_cflags) -noinst_LTLIBRARIES = libvgz.la +AM_LDFLAGS = $(AM_LT_LDFLAGS) -libvgz_la_CFLAGS = -D_LARGEFILE64_SOURCE=1 -DZLIB_CONST \ - $(libvgz_extra_cflags) +noinst_LTLIBRARIES = libvgz.la libvgz_la_SOURCES = \ adler32.c \ From phk at FreeBSD.org Mon Jan 30 12:08:06 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 30 Jan 2023 12:08:06 +0000 (UTC) Subject: [master] d2d312634 Add -Wno-unknown-warning-option to libvgz cocktail, to deal with older clangs. Message-ID: <20230130120806.1674111030F@lists.varnish-cache.org> commit d2d312634d1318877d8c2c5f84b289ce44aef295 Author: Poul-Henning Kamp Date: Mon Jan 30 12:07:29 2023 +0000 Add -Wno-unknown-warning-option to libvgz cocktail, to deal with older clangs. diff --git a/configure.ac b/configure.ac index 93c92e29b..c21d6bf63 100644 --- a/configure.ac +++ b/configure.ac @@ -256,7 +256,7 @@ else fi # zlib-specific flags -libvgz_extra_cflags="-Wno-error=strict-prototypes" +libvgz_extra_cflags="-Wno-error=strict-prototypes -Wno-unknown-warning-option" dnl https://github.com/madler/zlib/issues/633 libvgz_extra_cflags="$libvgz_extra_cflags -Wno-error=deprecated-non-prototype" From dridi.boukelmoune at gmail.com Mon Jan 30 13:24:05 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 30 Jan 2023 13:24:05 +0000 (UTC) Subject: [master] 79c7d1754 build: Revisit zlib C flags one more time Message-ID: <20230130132405.81EFD112711@lists.varnish-cache.org> commit 79c7d17543d86249c9e88ca1cf2a14f423275fff Author: Dridi Boukelmoune Date: Mon Jan 30 13:54:13 2023 +0100 build: Revisit zlib C flags one more time Trying to fix the build for clang 15 actually broke the build for GCC. The -Werror that was initially set after saving CFLAGS was meant to be part of NO_VIZ test. We turn warnings into errors later in the configure script so at this point we shouldn't care about it. If we really do, we can move this check below the line where -Werror is set. GCC chokes on -Wno-error=deprecated-non-prototype so instead we add it conditionally. To match the naming convention everywhere else, libvgz_extra_cflags was renamed to VGZ_CFLAGS. Refs 118fd10c9e81920e0c31906c2bf5c62a3f25cd5b diff --git a/configure.ac b/configure.ac index c21d6bf63..9596b8285 100644 --- a/configure.ac +++ b/configure.ac @@ -249,19 +249,14 @@ LIBS="${save_LIBS}" AC_CHECK_DECL([__SUNPRO_C], [SUNCC="yes"], [SUNCC="no"]) -if test "$SUNCC" = "yes" ; then - CFLAGS="${CFLAGS} -errwarn=%all,no%E_EMPTY_TRANSLATION_UNIT" -else - CFLAGS="${CFLAGS} -Werror" -fi - # zlib-specific flags -libvgz_extra_cflags="-Wno-error=strict-prototypes -Wno-unknown-warning-option" +VGZ_CFLAGS="-Wno-error=strict-prototypes -Wno-unknown-warning-option" dnl https://github.com/madler/zlib/issues/633 -libvgz_extra_cflags="$libvgz_extra_cflags -Wno-error=deprecated-non-prototype" +AX_CHECK_COMPILE_FLAG([-Wno-error=deprecated-non-prototype], + [VGZ_CFLAGS="$VGZ_CFLAGS -Wno-error=deprecated-non-prototype"]) -AC_SUBST(libvgz_extra_cflags) +AC_SUBST(VGZ_CFLAGS) # Support for visibility attribute (zlib) AC_CACHE_CHECK([whether we have support for visibility attributes], diff --git a/lib/libvgz/Makefile.am b/lib/libvgz/Makefile.am index 929646ad5..bbca6f5ac 100644 --- a/lib/libvgz/Makefile.am +++ b/lib/libvgz/Makefile.am @@ -3,7 +3,7 @@ AM_CFLAGS = $(AM_LT_CFLAGS) \ -D_LARGEFILE64_SOURCE=1 \ -DZLIB_CONST \ - $(libvgz_extra_cflags) + $(VGZ_CFLAGS) AM_LDFLAGS = $(AM_LT_LDFLAGS) From dridi.boukelmoune at gmail.com Mon Jan 30 14:14:05 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 30 Jan 2023 14:14:05 +0000 (UTC) Subject: [master] aad8de860 build: Move zlib checks down Message-ID: <20230130141405.5C142113F5D@lists.varnish-cache.org> commit aad8de860d8f5237bc3ec6197af04e93e6cbabfb Author: Dridi Boukelmoune Date: Mon Jan 30 14:57:50 2023 +0100 build: Move zlib checks down As I suspected, the -Werror setup for libvgz C flags were needed to properly discard them for suncc. Both warnings are conditionally not turned into errors, so -Wno-unknown-warning-option should no longer be needed. Refs madler/zlib#633 diff --git a/configure.ac b/configure.ac index 9596b8285..dc0153c50 100644 --- a/configure.ac +++ b/configure.ac @@ -249,27 +249,6 @@ LIBS="${save_LIBS}" AC_CHECK_DECL([__SUNPRO_C], [SUNCC="yes"], [SUNCC="no"]) -# zlib-specific flags -VGZ_CFLAGS="-Wno-error=strict-prototypes -Wno-unknown-warning-option" - -dnl https://github.com/madler/zlib/issues/633 -AX_CHECK_COMPILE_FLAG([-Wno-error=deprecated-non-prototype], - [VGZ_CFLAGS="$VGZ_CFLAGS -Wno-error=deprecated-non-prototype"]) - -AC_SUBST(VGZ_CFLAGS) - -# Support for visibility attribute (zlib) -AC_CACHE_CHECK([whether we have support for visibility attributes], - [ac_cv_have_viz], - [AC_COMPILE_IFELSE( - [AC_LANG_PROGRAM([[ - int __attribute__((visibility ("hidden"))) foo; - ]],[])], - [AC_DEFINE([HAVE_HIDDEN], [1], - [Define to 1 if visibility attribute hidden is available.]) - ]) -]) - AC_ARG_ENABLE(ubsan, AS_HELP_STRING([--enable-ubsan], [enable undefined behavior sanitizer (default is NO)]), @@ -721,6 +700,28 @@ if test $? -ne 0 ; then AC_MSG_ERROR([wflags.py failure]) fi +# zlib-specific flags +AX_CHECK_COMPILE_FLAG([-Wno-error=strict-prototypes], + [VGZ_CFLAGS="$VGZ_CFLAGS -Wno-error=strict-prototypes"]) + +dnl https://github.com/madler/zlib/issues/633 +AX_CHECK_COMPILE_FLAG([-Wno-error=deprecated-non-prototype], + [VGZ_CFLAGS="$VGZ_CFLAGS -Wno-error=deprecated-non-prototype"]) + +AC_SUBST(VGZ_CFLAGS) + +# Support for visibility attribute (zlib) +AC_CACHE_CHECK([whether we have support for visibility attributes], + [ac_cv_have_viz], + [AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM([[ + int __attribute__((visibility ("hidden"))) foo; + ]],[])], + [AC_DEFINE([HAVE_HIDDEN], [1], + [Define to 1 if visibility attribute hidden is available.]) + ]) +]) + # --enable-stack-protector AC_ARG_ENABLE(stack-protector, AS_HELP_STRING([--enable-stack-protector],[enable stack protector (default is YES)]), From dridi.boukelmoune at gmail.com Mon Jan 30 16:55:06 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 30 Jan 2023 16:55:06 +0000 (UTC) Subject: [master] 02636039d heritage: Remove unset params::[gu]id fields Message-ID: <20230130165506.5079A118B46@lists.varnish-cache.org> commit 02636039d4da9fbbe25362a7a17a9375873d11f4 Author: Dridi Boukelmoune Date: Tue Jan 24 11:32:09 2023 +0100 heritage: Remove unset params::[gu]id fields They are shared with the cache process but are never used. Only the VCC process uses them, but they are never set. This specific fchown(2) call in the VCC process was probably a no-op in the first place: since the fields are never set this is transferring ownership to root:root and if that succeeded the process was already root in the first place. If it failed, we never see the error message since we lacked root privileges. Both the unix and solaris jails are designed to run VCC (and CC) with limited privileges, and in the absence of a jail, the outcome should be the same: VCC creates a file with credentials suitable for the next CC invocation. diff --git a/bin/varnishd/common/common_param.h b/bin/varnishd/common/common_param.h index 4c64c4696..30e033b7c 100644 --- a/bin/varnishd/common/common_param.h +++ b/bin/varnishd/common/common_param.h @@ -140,9 +140,5 @@ struct params { #undef ptyp_vsl_mask #undef ptyp_vsl_reclen - /* Unprivileged user / group */ - uid_t uid; - gid_t gid; - struct vre_limits vre_limits; }; diff --git a/bin/varnishd/mgt/mgt_vcc.c b/bin/varnishd/mgt/mgt_vcc.c index 0fcef8a72..6561e7e77 100644 --- a/bin/varnishd/mgt/mgt_vcc.c +++ b/bin/varnishd/mgt/mgt_vcc.c @@ -259,10 +259,6 @@ mgt_vcc_touchfile(const char *fn, struct vsb *sb) VSB_printf(sb, "Failed to create %s: %s", fn, VAS_errtxt(errno)); return (2); } - if (fchown(i, mgt_param.uid, mgt_param.gid) != 0) - if (geteuid() == 0) - VSB_printf(sb, "Failed to change owner on %s: %s\n", - fn, VAS_errtxt(errno)); closefd(&i); return (0); } From dridi.boukelmoune at gmail.com Mon Jan 30 16:55:06 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 30 Jan 2023 16:55:06 +0000 (UTC) Subject: [master] db49f013b varnishtest: Turn vtc_maxdur into a vtim_dur Message-ID: <20230130165506.68BC9118B48@lists.varnish-cache.org> commit db49f013b8f9eedd336e29c29a3166189a400869 Author: Dridi Boukelmoune Date: Fri Jul 29 17:58:24 2022 +0200 varnishtest: Turn vtc_maxdur into a vtim_dur We already pass it to VEV and VCLI subsystems in places where a double is expected. Trivia: we currently parse it in two distinct ways. So for now I'm not eager to support duration units. diff --git a/bin/varnishtest/vtc.h b/bin/varnishtest/vtc.h index 55d28b351..0c33f1adf 100644 --- a/bin/varnishtest/vtc.h +++ b/bin/varnishtest/vtc.h @@ -76,7 +76,7 @@ extern volatile sig_atomic_t vtc_error; /* Error, bail out */ extern int vtc_stop; /* Abandon current test, no error */ extern pthread_t vtc_thread; extern int iflg; -extern unsigned vtc_maxdur; +extern vtim_dur vtc_maxdur; extern char *vmod_path; extern struct vsb *params_vsb; extern int leave_temp; diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index c61fecc8d..92f0add1c 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -94,7 +94,7 @@ struct vtc_job { int iflg = 0; -unsigned vtc_maxdur = 60; +vtim_dur vtc_maxdur = 60; static unsigned vtc_bufsiz = 1024 * 1024; static VTAILQ_HEAD(, vtc_tst) tst_head = VTAILQ_HEAD_INITIALIZER(tst_head); From dridi.boukelmoune at gmail.com Mon Jan 30 16:55:06 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 30 Jan 2023 16:55:06 +0000 (UTC) Subject: [master] 27017722d vtc_http: Turn http::timeout into a duration Message-ID: <20230130165506.8DC48118B4D@lists.varnish-cache.org> commit 27017722d56c6c3152d92648f6d5470bccc3a695 Author: Dridi Boukelmoune Date: Fri Jul 29 17:47:13 2022 +0200 vtc_http: Turn http::timeout into a duration This aligns with varnishd where durations are always computed in seconds instead of introducing corner cases where sometimes it's milliseconds. It also aligns with vtc_syslog that was introduced after the change to "seconds everywhere" in varnishd. diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 44c0138cd..4b79ebc1b 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -496,12 +496,12 @@ http_rxchar(struct http *hp, int n, int eof) pfd[0].fd = hp->sess->fd; pfd[0].events = POLLIN; pfd[0].revents = 0; - i = poll(pfd, 1, hp->timeout); + i = poll(pfd, 1, hp->timeout * 1000); if (i < 0 && errno == EINTR) continue; if (i == 0) { vtc_log(hp->vl, hp->fatal, - "HTTP rx timeout (fd:%d %u ms)", + "HTTP rx timeout (fd:%d %.3fs)", hp->sess->fd, hp->timeout); continue; } @@ -1473,7 +1473,7 @@ cmd_http_timeout(CMD_ARGS) d = VNUM(av[1]); if (isnan(d)) vtc_fatal(vl, "timeout is not a number (%s)", av[1]); - hp->timeout = (int)(d * 1000.0); + hp->timeout = d; } /* SECTION: client-server.spec.expect_close @@ -1500,7 +1500,7 @@ cmd_http_expect_close(CMD_ARGS) fds[0].fd = hp->sess->fd; fds[0].events = POLLIN; fds[0].revents = 0; - i = poll(fds, 1, hp->timeout); + i = poll(fds, 1, hp->timeout * 1000); if (i < 0 && errno == EINTR) continue; if (i == 0) @@ -1821,7 +1821,7 @@ http_process(struct vtclog *vl, struct vtc_sess *vsp, const char *spec, AN(hp); hp->sess = vsp; hp->sess->fd = sock; - hp->timeout = vtc_maxdur * 1000 / 2; + hp->timeout = vtc_maxdur * .5; if (rcvbuf) { // XXX setsockopt() too late on SunOS diff --git a/bin/varnishtest/vtc_http.h b/bin/varnishtest/vtc_http.h index e45b40469..c029cb80d 100644 --- a/bin/varnishtest/vtc_http.h +++ b/bin/varnishtest/vtc_http.h @@ -47,7 +47,7 @@ struct http { #define HTTP_MAGIC 0x2f02169c int *sfd; struct vtc_sess *sess; - int timeout; + vtim_dur timeout; struct vtclog *vl; struct vsb *vsb; diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 7a437eb84..7a848bb0d 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -176,12 +176,12 @@ get_bytes(const struct http *hp, char *buf, int n) pfd[0].fd = hp->sess->fd; pfd[0].events = POLLIN; pfd[0].revents = 0; - i = poll(pfd, 1, hp->timeout); + i = poll(pfd, 1, hp->timeout * 1000); if (i < 0 && errno == EINTR) continue; if (i == 0) vtc_log(hp->vl, 3, - "HTTP2 rx timeout (fd:%d %u ms)", + "HTTP2 rx timeout (fd:%d %.3fs)", hp->sess->fd, hp->timeout); if (i < 0) vtc_log(hp->vl, 3, From dridi.boukelmoune at gmail.com Mon Jan 30 17:56:05 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 30 Jan 2023 17:56:05 +0000 (UTC) Subject: [master] 9b42ca376 jail_solaris: Retire mgt_param.[gu]id usage Message-ID: <20230130175605.EA00C11AB03@lists.varnish-cache.org> commit 9b42ca376923b0cbf534c24f280bca477de27767 Author: Dridi Boukelmoune Date: Mon Jan 30 18:36:25 2023 +0100 jail_solaris: Retire mgt_param.[gu]id usage This fields were never set in the first place so they went away in #3888. We don't have SunOS coverage on Github so I noticed it after the facts. I did look at the Solaris jail but somehow missed that those fields were used there as well. Chances are that the deleted statements never ran in the first place, otherwise the assertions would have triggered. If the solaris jail should set[gu]id(2) as part of its privileges drop, it should probably grow new sub-options similar to the ones in the unix jail. Refs #3888 diff --git a/bin/varnishd/mgt/mgt_jail_solaris.c b/bin/varnishd/mgt/mgt_jail_solaris.c index 516f12fc7..749306835 100644 --- a/bin/varnishd/mgt/mgt_jail_solaris.c +++ b/bin/varnishd/mgt/mgt_jail_solaris.c @@ -447,10 +447,6 @@ static void vjs_setuid(void) { if (priv_ineffect(PRIV_PROC_SETID)) { - if (getgid() != mgt_param.gid) - XXXAZ(setgid(mgt_param.gid)); - if (getuid() != mgt_param.uid) - XXXAZ(setuid(mgt_param.uid)); AZ(setppriv(PRIV_OFF, PRIV_EFFECTIVE, vjs_proc_setid)); AZ(setppriv(PRIV_OFF, PRIV_PERMITTED, vjs_proc_setid)); } else { From dridi.boukelmoune at gmail.com Tue Jan 31 11:47:06 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 31 Jan 2023 11:47:06 +0000 (UTC) Subject: [master] ec59ba4c4 vtc_http: New struct h2_window Message-ID: <20230131114706.A5CA6115B0E@lists.varnish-cache.org> commit ec59ba4c4f5ce4d9323aa1cbaaf97d5be00e00f6 Author: Walid Boudebouda Date: Mon Dec 5 17:59:40 2022 +0100 vtc_http: New struct h2_window It groups the ws and iws fields together and hopefully conveys that h2_window::init means "initial window size" slightly better than iws. Signed-off-by: Dridi Boukelmoune diff --git a/bin/varnishtest/vtc_http.h b/bin/varnishtest/vtc_http.h index c029cb80d..51028c1ef 100644 --- a/bin/varnishtest/vtc_http.h +++ b/bin/varnishtest/vtc_http.h @@ -42,6 +42,11 @@ struct vtc_sess { ssize_t rcvbuf; }; +struct h2_window { + uint64_t init; + int64_t size; +}; + struct http { unsigned magic; #define HTTP_MAGIC 0x2f02169c @@ -85,8 +90,7 @@ struct http { pthread_cond_t cond; struct hpk_ctx *encctx; struct hpk_ctx *decctx; - uint64_t iws; - int64_t ws; + struct h2_window h2_win_self[1]; }; int http_process(struct vtclog *vl, struct vtc_sess *vsp, const char *spec, diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 7a848bb0d..990ffd291 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -420,7 +420,7 @@ parse_data(struct stream *s, struct frame *f) if (s->id) s->ws -= size; - s->hp->ws -= size; + s->hp->h2_win_self->size -= size; if (!size) { AZ(data); @@ -1121,7 +1121,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) */ if (!strcmp(spec, "stream.window")) { snprintf(buf, 20, "%jd", - (intmax_t)(s->id ? s->ws : s->hp->ws)); + (intmax_t)(s->id ? s->ws : s->hp->h2_win_self->size)); return (buf); } if (!strcmp(spec, "stream.weight")) { @@ -1931,8 +1931,8 @@ cmd_txsettings(CMD_ARGS) else if (!strcmp(*av, "-winsize")) { PUT_KV(av, vl, winsize, val, 0x4); VTAILQ_FOREACH(_s, &hp->streams, list) - _s->ws += (val - hp->iws); - hp->iws = val; + _s->ws += (val - hp->h2_win_self->init); + hp->h2_win_self->init = val; } else if (!strcmp(*av, "-framesize")) PUT_KV(av, vl, framesize, val, 0x5); @@ -2098,7 +2098,7 @@ cmd_txwinup(CMD_ARGS) AZ(pthread_mutex_lock(&hp->mtx)); if (s->id == 0) - hp->ws += size; + hp->h2_win_self->size += size; s->ws += size; AZ(pthread_mutex_unlock(&hp->mtx)); @@ -2593,7 +2593,7 @@ stream_new(const char *name, struct http *h) REPLACE(s->name, name); AN(s->name); VTAILQ_INIT(&s->fq); - s->ws = h->iws; + s->ws = h->h2_win_self->init; s->vl = vtc_logopen("%s.%s", h->sess->name, name); vtc_log_set_cmd(s->vl, stream_cmds); @@ -2840,8 +2840,8 @@ start_h2(struct http *hp) AZ(pthread_mutex_init(&hp->mtx, NULL)); AZ(pthread_cond_init(&hp->cond, NULL)); VTAILQ_INIT(&hp->streams); - hp->iws = 0xffff; - hp->ws = 0xffff; + hp->h2_win_self->init = 0xffff; + hp->h2_win_self->size = 0xffff; hp->h2 = 1; From dridi.boukelmoune at gmail.com Tue Jan 31 11:47:06 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 31 Jan 2023 11:47:06 +0000 (UTC) Subject: [master] 339650f9d vtc_http2: Avoid magic settings indices Message-ID: <20230131114706.BD6E9115B11@lists.varnish-cache.org> commit 339650f9ddcdd01be6378b78e6b4cfad201f105d Author: Walid Boudebouda Date: Mon Dec 5 18:04:09 2022 +0100 vtc_http2: Avoid magic settings indices This will be useful as varnishtest will honor more settings. Signed-off-by: Dridi Boukelmoune diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 990ffd291..04389d25f 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -70,7 +70,11 @@ static const char * const h2_settings[] = { NULL }; -#define SETTINGS_MAX (sizeof(h2_settings)/sizeof(h2_settings[0]) - 1U) +enum h2_settings_e { +#define H2_SETTING(U,l,v,...) SETTINGS_##U = v, +#include + SETTINGS_MAX +}; enum h2_type { @@ -1026,9 +1030,9 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) return (buf); } if (!strcmp(spec, "push")) { - if (isnan(f->md.settings[2])) + if (isnan(f->md.settings[SETTINGS_ENABLE_PUSH])) return (NULL); - else if (f->md.settings[2] == 1) + else if (f->md.settings[SETTINGS_ENABLE_PUSH] == 1) snprintf(buf, 20, "true"); else snprintf(buf, 20, "false"); From dridi.boukelmoune at gmail.com Tue Jan 31 11:47:06 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 31 Jan 2023 11:47:06 +0000 (UTC) Subject: [master] 281cb6e62 vtc_http2: Rename enum h2_type to match convention Message-ID: <20230131114706.D8875115B14@lists.varnish-cache.org> commit 281cb6e6208559ad18d75a894f1b318f99b1acdb Author: Dridi Boukelmoune Date: Tue Jan 31 10:42:54 2023 +0100 vtc_http2: Rename enum h2_type to match convention diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 04389d25f..6a9a4de03 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -77,7 +77,7 @@ enum h2_settings_e { }; -enum h2_type { +enum h2_type_e { #define H2_FRAME(l,u,t,f,...) TYPE_##u = t, #include TYPE_MAX @@ -2169,7 +2169,7 @@ cmd_rxhdrs(CMD_ARGS) int loop = 0; unsigned long int times = 1; unsigned rcv = 0; - enum h2_type expect = TYPE_HEADERS; + enum h2_type_e expect = TYPE_HEADERS; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); @@ -2365,7 +2365,7 @@ cmd_rxpush(CMD_ARGS) int loop = 0; unsigned long int times = 1; unsigned rcv = 0; - enum h2_type expect = TYPE_PUSH_PROMISE; + enum h2_type_e expect = TYPE_PUSH_PROMISE; CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); From dridi.boukelmoune at gmail.com Tue Jan 31 11:47:06 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 31 Jan 2023 11:47:06 +0000 (UTC) Subject: [master] c3310c64f vtc_http2: Restrict SETTINGS frames to stream 0 Message-ID: <20230131114706.F3FFE115B19@lists.varnish-cache.org> commit c3310c64f44146d66b64493108a951bfa5ae7196 Author: Walid Boudebouda Date: Mon Dec 5 18:16:21 2022 +0100 vtc_http2: Restrict SETTINGS frames to stream 0 In the unlikely event that we ever need actual coverage for this case, we can handle it better than just failing the test case. In that regard, as long as varnishtest itself can send SETTINGS frame on a wrong stream to test varnishd's behavior, we should be fine. Signed-off-by: Dridi Boukelmoune diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 6a9a4de03..d2ec55a65 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -607,6 +607,10 @@ parse_settings(const struct stream *s, struct frame *f) vtc_fatal(s->vl, "Size should be a multiple of 6, but isn't (%d)", f->size); + if (s->id != 0) + vtc_fatal(s->vl, + "Setting frames should only be on stream 0, but received on stream: %d", s->id); + for (u = 0; u <= SETTINGS_MAX; u++) f->md.settings[u] = NAN; From dridi.boukelmoune at gmail.com Tue Jan 31 11:47:07 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 31 Jan 2023 11:47:07 +0000 (UTC) Subject: [master] f43c51b6c vtc_http2: Rename stream::ws to win_self Message-ID: <20230131114707.1D344115B1C@lists.varnish-cache.org> commit f43c51b6c34eaa44c6e7831edb3a9848aa772b02 Author: Walid Boudebouda Date: Tue Jan 31 11:12:09 2023 +0100 vtc_http2: Rename stream::ws to win_self Signed-off-by: Dridi Boukelmoune diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index d2ec55a65..08209234d 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -106,7 +106,7 @@ struct stream { struct frame *frame; pthread_t tp; struct http *hp; - int64_t ws; + int64_t win_self; int wf; VTAILQ_HEAD(, frame) fq; @@ -422,7 +422,7 @@ parse_data(struct stream *s, struct frame *f) } if (s->id) - s->ws -= size; + s->win_self -= size; s->hp->h2_win_self->size -= size; @@ -1129,7 +1129,7 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) */ if (!strcmp(spec, "stream.window")) { snprintf(buf, 20, "%jd", - (intmax_t)(s->id ? s->ws : s->hp->h2_win_self->size)); + (intmax_t)(s->id ? s->win_self : s->hp->h2_win_self->size)); return (buf); } if (!strcmp(spec, "stream.weight")) { @@ -1939,7 +1939,7 @@ cmd_txsettings(CMD_ARGS) else if (!strcmp(*av, "-winsize")) { PUT_KV(av, vl, winsize, val, 0x4); VTAILQ_FOREACH(_s, &hp->streams, list) - _s->ws += (val - hp->h2_win_self->init); + _s->win_self += (val - hp->h2_win_self->init); hp->h2_win_self->init = val; } else if (!strcmp(*av, "-framesize")) @@ -2107,7 +2107,7 @@ cmd_txwinup(CMD_ARGS) AZ(pthread_mutex_lock(&hp->mtx)); if (s->id == 0) hp->h2_win_self->size += size; - s->ws += size; + s->win_self += size; AZ(pthread_mutex_unlock(&hp->mtx)); size = htonl(size); @@ -2601,7 +2601,7 @@ stream_new(const char *name, struct http *h) REPLACE(s->name, name); AN(s->name); VTAILQ_INIT(&s->fq); - s->ws = h->h2_win_self->init; + s->win_self = h->h2_win_self->init; s->vl = vtc_logopen("%s.%s", h->sess->name, name); vtc_log_set_cmd(s->vl, stream_cmds); From dridi.boukelmoune at gmail.com Tue Jan 31 11:47:07 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 31 Jan 2023 11:47:07 +0000 (UTC) Subject: [master] 75c344d20 vtc_http2: Track the peer window of h2 streams Message-ID: <20230131114707.3A537115B22@lists.varnish-cache.org> commit 75c344d20437e103520810e738b350eeb92be685 Author: Walid Boudebouda Date: Tue Jan 31 11:17:32 2023 +0100 vtc_http2: Track the peer window of h2 streams Signed-off-by: Dridi Boukelmoune diff --git a/bin/varnishtest/vtc_http.h b/bin/varnishtest/vtc_http.h index 51028c1ef..a5d43001e 100644 --- a/bin/varnishtest/vtc_http.h +++ b/bin/varnishtest/vtc_http.h @@ -91,6 +91,7 @@ struct http { struct hpk_ctx *encctx; struct hpk_ctx *decctx; struct h2_window h2_win_self[1]; + struct h2_window h2_win_peer[1]; }; int http_process(struct vtclog *vl, struct vtc_sess *vsp, const char *spec, diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index 08209234d..a3470282e 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -107,6 +107,7 @@ struct stream { pthread_t tp; struct http *hp; int64_t win_self; + int64_t win_peer; int wf; VTAILQ_HEAD(, frame) fq; @@ -321,7 +322,7 @@ clean_frame(struct frame **fp) } static void -write_frame(const struct stream *sp, const struct frame *f, const unsigned lock) +write_frame(struct stream *sp, const struct frame *f, const unsigned lock) { struct http *hp; ssize_t l; @@ -340,6 +341,11 @@ write_frame(const struct stream *sp, const struct frame *f, const unsigned lock) f->type < TYPE_MAX ? h2_types[f->type] : "?", f->type, f->flags, f->size); + if (f->type == TYPE_DATA) { + sp->win_peer -= f->size; + hp->h2_win_peer->size -= f->size; + } + if (lock) AZ(pthread_mutex_lock(&hp->mtx)); l = write(hp->sess->fd, hdr, sizeof(hdr)); @@ -2398,6 +2404,50 @@ cmd_rxpush(CMD_ARGS) s->frame = f; } +/* SECTION: stream.spec.winup_rxwinup rxwinup + * + * Receive a WINDOW_UPDATE frame. + */ +static void +cmd_rxwinup(CMD_ARGS) +{ + struct stream *s; + struct frame *f; + + CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); + s->frame = rxstuff(s); + CAST_OBJ_NOTNULL(f, s->frame, FRAME_MAGIC); + CHKFRAME(f->type, TYPE_WINDOW_UPDATE, 0, *av); + if (s->id == 0) + s->hp->h2_win_peer->size += s->frame->md.winup_size; + s->win_peer += s->frame->md.winup_size; +} + +/* SECTION: stream.spec.settings_rxsettings rxsettings + * + * Receive a SETTINGS frame. + */ +static void +cmd_rxsettings(CMD_ARGS) +{ + struct stream *s, *_s; + uint32_t val = 0; + struct http *hp; + struct frame *f; + + CAST_OBJ_NOTNULL(s, priv, STREAM_MAGIC); + CAST_OBJ_NOTNULL(hp, s->hp, HTTP_MAGIC); + s->frame = rxstuff(s); + CAST_OBJ_NOTNULL(f, s->frame, FRAME_MAGIC); + CHKFRAME(f->type, TYPE_SETTINGS, 0, *av); + if (! isnan(f->md.settings[SETTINGS_INITIAL_WINDOW_SIZE])) { + val = f->md.settings[SETTINGS_INITIAL_WINDOW_SIZE]; + VTAILQ_FOREACH(_s, &hp->streams, list) + _s->win_peer += (val - hp->h2_win_peer->init); + hp->h2_win_peer->init = val; + } +} + #define RXFUNC(lctype, upctype) \ static void \ cmd_rx ## lctype(CMD_ARGS) { \ @@ -2425,12 +2475,6 @@ RXFUNC(prio, PRIORITY) */ RXFUNC(rst, RST_STREAM) -/* SECTION: stream.spec.settings_rxsettings rxsettings - * - * Receive a SETTINGS frame. - */ -RXFUNC(settings,SETTINGS) - /* SECTION: stream.spec.ping_rxping rxping * * Receive a PING frame. @@ -2443,12 +2487,6 @@ RXFUNC(ping, PING) */ RXFUNC(goaway, GOAWAY) -/* SECTION: stream.spec.winup_rxwinup rxwinup - * - * Receive a WINDOW_UPDATE frame. - */ -RXFUNC(winup, WINDOW_UPDATE) - /* SECTION: stream.spec.frame_rxframe * * Receive a frame, any frame. @@ -2602,6 +2640,7 @@ stream_new(const char *name, struct http *h) AN(s->name); VTAILQ_INIT(&s->fq); s->win_self = h->h2_win_self->init; + s->win_peer = h->h2_win_peer->init; s->vl = vtc_logopen("%s.%s", h->sess->name, name); vtc_log_set_cmd(s->vl, stream_cmds); @@ -2850,7 +2889,8 @@ start_h2(struct http *hp) VTAILQ_INIT(&hp->streams); hp->h2_win_self->init = 0xffff; hp->h2_win_self->size = 0xffff; - + hp->h2_win_peer->init = 0xffff; + hp->h2_win_peer->size = 0xffff; hp->h2 = 1; hp->decctx = HPK_NewCtx(4096); From dridi.boukelmoune at gmail.com Tue Jan 31 11:47:07 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 31 Jan 2023 11:47:07 +0000 (UTC) Subject: [master] f5c7d24f4 vtc: New stream.peer_window variable for h2 checks Message-ID: <20230131114707.60068115B36@lists.varnish-cache.org> commit f5c7d24f4a2c2524abaf2c8ad84f0eae7d7ca73c Author: Walid Boudebouda Date: Tue Jan 31 11:21:31 2023 +0100 vtc: New stream.peer_window variable for h2 checks Closes #3681 Signed-off-by: Dridi Boukelmoune diff --git a/bin/varnishtest/tests/a02006.vtc b/bin/varnishtest/tests/a02006.vtc index e6ee7d451..ecb49745f 100644 --- a/bin/varnishtest/tests/a02006.vtc +++ b/bin/varnishtest/tests/a02006.vtc @@ -8,8 +8,20 @@ server s1 { txcont -hdr "baz" "qux" txdata -data "foo" txdata -data "bar" + expect stream.peer_window == 65529 + rxreq + txresp -bodylen 529 + expect stream.peer_window == 65000 + rxwinup + expect stream.peer_window == 65200 + } -run + stream 0 { + expect stream.peer_window == 65000 + rxwinup + expect stream.peer_window == 66000 } -run + } -start client c1 -connect ${s1_sock} { @@ -35,6 +47,15 @@ client c1 -connect ${s1_sock} { stream 0 { expect stream.window == 65529 } -run + stream 2 { + txreq + rxresp + txwinup -size 200 + } -run + stream 0 { + txwinup -size 1000 + } -run + } -run server s1 -wait diff --git a/bin/varnishtest/tests/a02010.vtc b/bin/varnishtest/tests/a02010.vtc index adf12d33e..2e77c4d2d 100644 --- a/bin/varnishtest/tests/a02010.vtc +++ b/bin/varnishtest/tests/a02010.vtc @@ -2,17 +2,23 @@ varnishtest "Verify the initial window size" server s1 { stream 0 { + expect stream.peer_window == 65535 rxsettings txsettings -ack } -run stream 1 { + expect stream.peer_window == 128 rxreq txresp -bodylen 100 + expect stream.peer_window == 28 } -run stream 0 { rxsettings txsettings -ack } -run + stream 1 { + expect stream.peer_window == -36 + } -run } -start client c1 -connect ${s1_sock} { diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index a3470282e..cc93e550e 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -1124,7 +1124,11 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) /* SECTION: stream.spec.zexpect.zstream Stream * * stream.window - * The current window size of the stream, or, if on stream 0, + * The current local window size of the stream, or, if on stream 0, + * of the connection. + * + * stream.peer_window + * The current peer window size of the stream, or, if on stream 0, * of the connection. * * stream.weight @@ -1138,6 +1142,11 @@ cmd_var_resolve(const struct stream *s, const char *spec, char *buf) (intmax_t)(s->id ? s->win_self : s->hp->h2_win_self->size)); return (buf); } + if (!strcmp(spec, "stream.peer_window")) { + snprintf(buf, 20, "%jd", + (intmax_t)(s->id ? s->win_peer : s->hp->h2_win_peer->size)); + return (buf); + } if (!strcmp(spec, "stream.weight")) { if (s->id) { snprintf(buf, 20, "%d", s->weight); From dridi.boukelmoune at gmail.com Tue Jan 31 11:47:07 2023 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 31 Jan 2023 11:47:07 +0000 (UTC) Subject: [master] 0bbadc56b vtc_http2: Avoid weak symbol names for temp variables Message-ID: <20230131114707.7D08A115B44@lists.varnish-cache.org> commit 0bbadc56b5fc2a1bb5a8100772c0b49efd036754 Author: Dridi Boukelmoune Date: Tue Jan 31 11:32:35 2023 +0100 vtc_http2: Avoid weak symbol names for temp variables diff --git a/bin/varnishtest/vtc_http2.c b/bin/varnishtest/vtc_http2.c index cc93e550e..b70944bda 100644 --- a/bin/varnishtest/vtc_http2.c +++ b/bin/varnishtest/vtc_http2.c @@ -1913,7 +1913,7 @@ cmd_txprio(CMD_ARGS) static void cmd_txsettings(CMD_ARGS) { - struct stream *s, *_s; + struct stream *s, *s2; struct http *hp; char *p; uint32_t val = 0; @@ -1953,8 +1953,8 @@ cmd_txsettings(CMD_ARGS) PUT_KV(av, vl, maxstreams, val, 0x3); else if (!strcmp(*av, "-winsize")) { PUT_KV(av, vl, winsize, val, 0x4); - VTAILQ_FOREACH(_s, &hp->streams, list) - _s->win_self += (val - hp->h2_win_self->init); + VTAILQ_FOREACH(s2, &hp->streams, list) + s2->win_self += (val - hp->h2_win_self->init); hp->h2_win_self->init = val; } else if (!strcmp(*av, "-framesize")) @@ -2439,7 +2439,7 @@ cmd_rxwinup(CMD_ARGS) static void cmd_rxsettings(CMD_ARGS) { - struct stream *s, *_s; + struct stream *s, *s2; uint32_t val = 0; struct http *hp; struct frame *f; @@ -2451,8 +2451,8 @@ cmd_rxsettings(CMD_ARGS) CHKFRAME(f->type, TYPE_SETTINGS, 0, *av); if (! isnan(f->md.settings[SETTINGS_INITIAL_WINDOW_SIZE])) { val = f->md.settings[SETTINGS_INITIAL_WINDOW_SIZE]; - VTAILQ_FOREACH(_s, &hp->streams, list) - _s->win_peer += (val - hp->h2_win_peer->init); + VTAILQ_FOREACH(s2, &hp->streams, list) + s2->win_peer += (val - hp->h2_win_peer->init); hp->h2_win_peer->init = val; } } From phk at FreeBSD.org Tue Jan 31 14:45:07 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 31 Jan 2023 14:45:07 +0000 (UTC) Subject: [master] 2c33188b3 Set ESI:include'ded objs's status to 200 Message-ID: <20230131144508.044A811B7AF@lists.varnish-cache.org> commit 2c33188b34d10b78449cd0e44211b80047ed3884 Author: Poul-Henning Kamp Date: Tue Jan 31 14:22:59 2023 +0000 Set ESI:include'ded objs's status to 200 diff --git a/bin/varnishtest/tests/r01688.vtc b/bin/varnishtest/tests/r01688.vtc index ff10e0005..9819d3675 100644 --- a/bin/varnishtest/tests/r01688.vtc +++ b/bin/varnishtest/tests/r01688.vtc @@ -18,6 +18,7 @@ varnish v1 -vcl+backend { sub vcl_synth { if (resp.status == 998) { + set resp.status = 200; set resp.body = "this is the body of an " + "included synthetic response"; return(deliver); diff --git a/bin/varnishtest/tests/r01838.vtc b/bin/varnishtest/tests/r01838.vtc index 077851f70..4e6d1c74e 100644 --- a/bin/varnishtest/tests/r01838.vtc +++ b/bin/varnishtest/tests/r01838.vtc @@ -14,6 +14,7 @@ varnish v1 -vcl+backend { sub vcl_synth { if (resp.status == 998) { + set resp.status = 200; set resp.body = "synthetic body"; return (deliver); } From phk at FreeBSD.org Tue Jan 31 14:45:08 2023 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 31 Jan 2023 14:45:08 +0000 (UTC) Subject: [master] 582ded6a2 By default do not deliver failed 200 ESI:includes. Message-ID: <20230131144508.2A23811B7B3@lists.varnish-cache.org> commit 582ded6a2d6ae1a4467b1eb500f2725b42888016 Author: Poul-Henning Kamp Date: Tue Jan 31 14:43:48 2023 +0000 By default do not deliver failed 200 ESI:includes. diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 117e710c6..56f4b82d2 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -65,7 +65,7 @@ struct ecx { ssize_t l; int isgzip; int woken; - int abrt; + int incl_cont; struct req *preq; struct ecx *pecx; @@ -382,11 +382,11 @@ ved_vdp_esi_bytes(struct vdp_ctx *vdx, enum vdp_action act, void **priv, Debug("SKIP1(%d)\n", (int)ecx->l); ecx->state = 4; break; - case VEC_IA: - ecx->abrt = + case VEC_IC: + ecx->incl_cont = FEATURE(FEATURE_ESI_INCLUDE_ONERROR); /* FALLTHROUGH */ - case VEC_IC: + case VEC_IA: ecx->p++; q = (void*)strchr((const char*)ecx->p, '\0'); AN(q); @@ -869,6 +869,14 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody) if (wantbody == 0) return; + if (!ecx->incl_cont && + req->resp->status != 200 && + req->resp->status != 204) { + req->top->topreq->vdc->retval = -1; + req->top->topreq->doclose = req->doclose; + return; + } + if (boc == NULL && ObjGetLen(req->wrk, req->objcore) == 0) return; @@ -918,7 +926,7 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody) req->acct.resp_bodybytes += VDP_Close(req->vdc); - if (i && ecx->abrt) { + if (i && !ecx->incl_cont) { req->top->topreq->vdc->retval = -1; req->top->topreq->doclose = req->doclose; } diff --git a/bin/varnishtest/tests/r03241.vtc b/bin/varnishtest/tests/r03241.vtc index ac7fab446..54a485743 100644 --- a/bin/varnishtest/tests/r03241.vtc +++ b/bin/varnishtest/tests/r03241.vtc @@ -4,12 +4,8 @@ varnishtest "ESI include out of workspace" server s1 { rxreq expect req.http.esi0 == "foo" - txresp -body { - - Before include - - After include - + txresp -body {Before includeAfter include } rxreq expect req.url == "/body1" @@ -47,10 +43,11 @@ logexpect l1 -v v1 -g raw { client c1 { txreq -hdr "Host: foo" - rxresp - # XXX this is actually wrong (missed include) - expect resp.bodylen == 57 + rxresphdrs expect resp.status == 200 + rxchunk + expect_close + expect resp.body == {Before include} } -run logexpect l1 -wait diff --git a/bin/varnishtest/tests/r03865.vtc b/bin/varnishtest/tests/r03865.vtc new file mode 100644 index 000000000..98f706b5b --- /dev/null +++ b/bin/varnishtest/tests/r03865.vtc @@ -0,0 +1,57 @@ +varnishtest "ESI onerror" + +server s1 { + rxreq + expect req.url == "/abort" + txresp -hdr {surrogate-control: content="ESI/1.0"} \ + -body {before after} + +} -start + +varnish v1 -cliok "param.set feature +esi_disable_xml_check" +varnish v1 -cliok "param.set feature +esi_include_onerror" +varnish v1 -vcl+backend { + sub vcl_backend_fetch { + if (bereq.url == "/fail") { + return (error(604)); + } + } + sub vcl_backend_response { + set beresp.do_esi = beresp.http.surrogate-control ~ "ESI/1.0"; + unset beresp.http.surrogate-control; + } + sub vcl_backend_error { + if (beresp.status == 604) { + set beresp.body = "FOOBAR"; + return(deliver); + } + } +} -start + +client c1 { + txreq -url "/abort" + non_fatal + rxresphdrs + expect resp.status == 200 + rxchunk + expect_close + expect resp.body == "before " +} -run + +varnish v1 -vsl_catchup + +server s1 -wait + +server s1 { + rxreq + expect req.url == "/continue" + txresp -hdr {surrogate-control: content="ESI/1.0"} \ + -body {before after} +} -start + +client c1 { + fatal + txreq -url "/continue" + rxresp + expect resp.body == "before FOOBAR after" +} -run diff --git a/doc/changes.rst b/doc/changes.rst index 8d16a3988..979857f08 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -35,6 +35,25 @@ release process. Varnish Cache NEXT (2023-03-15) =============================== +* Do not ESI:include failed objects unless instructed to. + + Previously, any ESI:include object would be included, no matter + what the status of it were, 200, 503, didn't matter. + + From now on, by default, only objects with 200 and 204 status + will be included and any other status code will fail the parent + ESI request. + + If objects with other status should be delivered, they should + have their status changed to 200 in VCL, for instance in + ``sub vcl_backend_error{}``, ``vcl_synth{}`` or ``vcl_deliver{}``. + + If ``param.set feature +esi_include_onerror`` is used, and the + ```` tag has a ``onerror="continue"`` attribute, + any and all ESI:include objects will be delivered, no matter what + their status might be, and not even a partial delivery of them + will fail the parent ESI request. To be used with great caution. + * VXIDs are 64 bit now and the binary format of SHM and raw saved VSL files has changed as a consequence.