From nils.goroll at uplex.de Mon Sep 2 14:02:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 2 Sep 2024 14:02:05 +0000 (UTC) Subject: [master] 197b72ab2 Generalize the VDP API Message-ID: <20240902140205.E3C08AFAA1@lists.varnish-cache.org> commit 197b72ab24bcd604602be573a130d0dcddb5fa92 Author: Nils Goroll Date: Wed Dec 27 12:39:47 2023 +0100 Generalize the VDP API This commit is to prepare use of the VDP API also for the backend side to filter bereq.body through bereq.filters by putting all pointers _intended_ to be used by a VDP init function into vdp_ctx. For background, see #4035 diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 04ff3bbec..db658d17b 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -71,18 +71,35 @@ VDP_Fini(const struct vdp_ctx *vdc) void VDP_Init(struct vdp_ctx *vdc, struct worker *wrk, struct vsl_log *vsl, - struct req *req) + struct req *req, struct busyobj *bo, intmax_t *clen) { AN(vdc); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); AN(vsl); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + + CHECK_OBJ_ORNULL(req, REQ_MAGIC); + CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC); + AN(clen); + + assert((req ? 1 : 0) ^ (bo ? 1 : 0)); + + AN(clen); + assert(*clen >= -1); INIT_OBJ(vdc, VDP_CTX_MAGIC); VTAILQ_INIT(&vdc->vdp); vdc->wrk = wrk; vdc->vsl = vsl; - vdc->req = req; + vdc->clen = clen; + + if (req != NULL) { + vdc->oc = req->objcore; + vdc->hp = req->resp; + } + else { + vdc->oc = bo->bereq_body; + vdc->hp = bo->bereq; + } } /* VDP_bytes @@ -152,6 +169,10 @@ VDP_Push(VRT_CTX, struct vdp_ctx *vdc, struct ws *ws, const struct vdp *vdp, CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); + CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC); + AN(vdc->clen); + assert(*vdc->clen >= -1); AN(ws); AN(vdp); AN(vdp->name); @@ -175,10 +196,9 @@ VDP_Push(VRT_CTX, struct vdp_ctx *vdc, struct ws *ws, const struct vdp *vdp, vdc->nxt = VTAILQ_FIRST(&vdc->vdp); AZ(vdc->retval); - if (vdpe->vdp->init != NULL) { - vdc->retval = vdpe->vdp->init(ctx, vdc, &vdpe->priv, - vdpe == vdc->nxt ? vdc->req->objcore : NULL); - } + if (vdpe->vdp->init != NULL) + vdc->retval = vdpe->vdp->init(ctx, vdc, &vdpe->priv); + vdc->oc = NULL; if (vdc->retval > 0) { VTAILQ_REMOVE(&vdc->vdp, vdpe, list); @@ -254,7 +274,9 @@ VDP_DeliverObj(struct vdp_ctx *vdc, struct objcore *oc) CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); CHECK_OBJ_NOTNULL(vdc->wrk, WORKER_MAGIC); AN(vdc->vsl); - vdc->req = NULL; + AZ(vdc->oc); + vdc->hp = NULL; + vdc->clen = NULL; final = oc->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP) ? 1 : 0; r = ObjIterate(vdc->wrk, oc, vdc, vdp_objiterate, final); if (r < 0) diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 23cdfe242..ba6b0412d 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -261,35 +261,41 @@ ved_decode_len(struct vsl_log *vsl, const uint8_t **pp) */ static int v_matchproto_(vdp_init_f) -ved_vdp_esi_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) +ved_vdp_esi_init(VRT_CTX, struct vdp_ctx *vdc, void **priv) { struct ecx *ecx; - struct req *req; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_ORNULL(ctx->req, REQ_MAGIC); CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); - CHECK_OBJ_ORNULL(oc, OBJCORE_MAGIC); - if (oc == NULL || !ObjHasAttr(vdc->wrk, oc, OA_ESIDATA)) - return (1); - - req = vdc->req; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC); + AN(vdc->clen); AN(priv); + AZ(*priv); + if (vdc->oc == NULL || !ObjHasAttr(vdc->wrk, vdc->oc, OA_ESIDATA)) + return (1); + + if (ctx->req == NULL) { + VSLb(vdc->vsl, SLT_Error, + "esi can only be used on the client side"); + return (1); + } ALLOC_OBJ(ecx, ECX_MAGIC); AN(ecx); assert(sizeof gzip_hdr == 10); - ecx->preq = req; + ecx->preq = ctx->req; *priv = ecx; - RFC2616_Weaken_Etag(req->resp); - - req->res_mode |= RES_ESI; - if (req->resp_len != 0) - req->resp_len = -1; - if (req->esi_level > 0) { - assert(req->transport == &VED_transport); - CAST_OBJ_NOTNULL(ecx->pecx, req->transport_priv, ECX_MAGIC); + RFC2616_Weaken_Etag(vdc->hp); + + ctx->req->res_mode |= RES_ESI; + if (*vdc->clen != 0) + *vdc->clen = -1; + if (ctx->req->esi_level > 0) { + assert(ctx->req->transport == &VED_transport); + CAST_OBJ_NOTNULL(ecx->pecx, ctx->req->transport_priv, ECX_MAGIC); if (!ecx->pecx->isgzip) ecx->pecx = NULL; } @@ -604,18 +610,16 @@ struct ved_foo { }; static int v_matchproto_(vdp_init_f) -ved_gzgz_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) +ved_gzgz_init(VRT_CTX, struct vdp_ctx *vdc, void **priv) { ssize_t l; const char *p; struct ved_foo *foo; - struct req *req; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); - (void)oc; - req = vdc->req; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + AN(priv); + CAST_OBJ_NOTNULL(foo, *priv, VED_FOO_MAGIC); CHECK_OBJ_NOTNULL(foo->objcore, OBJCORE_MAGIC); @@ -925,7 +929,6 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody) foo->ecx = ecx; foo->objcore = req->objcore; i = VDP_Push(ctx, req->vdc, req->ws, &ved_gzgz, foo); - } else if (ecx->isgzip && !i) { /* Non-Gzip'ed include in gzip'ed parent */ i = VDP_Push(ctx, req->vdc, req->ws, &ved_pretend_gz, ecx); diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index d06540092..329ab9982 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -90,7 +90,7 @@ struct vfp_ctx { struct http *req; struct http *resp; struct worker *wrk; - struct objcore *oc; + struct objcore *oc; // Only first filter, if at all struct vfp_entry_s vfp; struct vfp_entry *vfp_nxt; @@ -112,8 +112,8 @@ enum vdp_action { VDP_END, /* Last buffer or after, implies VDP_FLUSH */ }; -typedef int vdp_init_f(VRT_CTX, struct vdp_ctx *, void **priv, - struct objcore *); + +typedef int vdp_init_f(VRT_CTX, struct vdp_ctx *, void **priv); /* * Return value: * negative: Error - abandon delivery @@ -155,7 +155,11 @@ struct vdp_ctx { struct vdp_entry *nxt; struct worker *wrk; struct vsl_log *vsl; - struct req *req; + // NULL'ed after the first filter has been pushed + struct objcore *oc; + // NULL'ed for delivery + struct http *hp; + intmax_t *clen; }; int VDP_bytes(struct vdp_ctx *, enum vdp_action act, const void *, ssize_t); diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 472ac2e5b..a7402ca56 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -288,11 +288,10 @@ VGZ_Gzip(struct vgz *vg, const void **pptr, ssize_t *plen, enum vgz_flag flags) */ static int v_matchproto_(vdp_init_f) -vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) +vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv) { struct vgz *vg; struct boc *boc; - struct req *req; enum boc_state_e bos; const char *p; ssize_t dl; @@ -300,9 +299,10 @@ vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); - CHECK_OBJ_ORNULL(oc, OBJCORE_MAGIC); - req = vdc->req; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC); + AN(vdc->clen); + AN(priv); vg = VGZ_NewGunzip(vdc->vsl, "U D -"); AN(vg); @@ -314,27 +314,27 @@ vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) VGZ_Obuf(vg, vg->m_buf, vg->m_sz); *priv = vg; - http_Unset(req->resp, H_Content_Encoding); + http_Unset(vdc->hp, H_Content_Encoding); - req->resp_len = -1; + *vdc->clen = -1; - if (oc == NULL) + if (vdc->oc == NULL) return (0); - boc = HSH_RefBoc(oc); + boc = HSH_RefBoc(vdc->oc); if (boc != NULL) { CHECK_OBJ(boc, BOC_MAGIC); bos = boc->state; - HSH_DerefBoc(vdc->wrk, oc); + HSH_DerefBoc(vdc->wrk, vdc->oc); if (bos < BOS_FINISHED) return (0); /* OA_GZIPBITS is not stable yet */ } - p = ObjGetAttr(vdc->wrk, oc, OA_GZIPBITS, &dl); + p = ObjGetAttr(vdc->wrk, vdc->oc, OA_GZIPBITS, &dl); if (p != NULL && dl == 32) { u = vbe64dec(p + 24); if (u != 0) - req->resp_len = u; + *vdc->clen = u; } return (0); } diff --git a/bin/varnishd/cache/cache_range.c b/bin/varnishd/cache/cache_range.c index d7398f33f..a9a86abc2 100644 --- a/bin/varnishd/cache/cache_range.c +++ b/bin/varnishd/cache/cache_range.c @@ -196,33 +196,40 @@ vrg_ifrange(struct req *req) } static int v_matchproto_(vdp_init_f) -vrg_range_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) +vrg_range_init(VRT_CTX, struct vdp_ctx *vdc, void **priv) { const char *err; - struct req *req; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_ORNULL(ctx->req, REQ_MAGIC); CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); - (void)oc; - req = vdc->req; - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - if (!vrg_ifrange(req)) // rfc7233,l,455,456 + AN(priv); + + if (ctx->req == NULL) { + VSLb(vdc->vsl, SLT_Error, + "range can only be used on the client side"); + return (1); + } + + // not using vdc->{hd,cl}, because range needs req anyway for Req_Fail() + + if (!vrg_ifrange(ctx->req)) // rfc7233,l,455,456 return (1); - err = vrg_dorange(req, priv); + err = vrg_dorange(ctx->req, priv); if (err == NULL) return (*priv == NULL ? 1 : 0); VSLb(vdc->vsl, SLT_Debug, "RANGE_FAIL %s", err); - if (req->resp_len >= 0) - http_PrintfHeader(req->resp, + if (ctx->req->resp_len >= 0) + http_PrintfHeader(ctx->req->resp, "Content-Range: bytes */%jd", - (intmax_t)req->resp_len); - http_PutResponse(req->resp, "HTTP/1.1", 416, NULL); + (intmax_t)ctx->req->resp_len); + http_PutResponse(ctx->req->resp, "HTTP/1.1", 416, NULL); /* * XXX: We ought to produce a body explaining things. * XXX: That really calls for us to hit vcl_synth{} */ - req->resp_len = 0; + ctx->req->resp_len = 0; return (1); } diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 45f66defc..20e4eb13b 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -462,11 +462,11 @@ cnt_transmit(struct worker *wrk, struct req *req) sendbody = 1; } - VDP_Init(req->vdc, req->wrk, req->vsl, req); + VDP_Init(req->vdc, req->wrk, req->vsl, req, NULL, &req->resp_len); if (req->vdp_filter_list == NULL) req->vdp_filter_list = resp_Get_Filter_List(req); if (req->vdp_filter_list == NULL || - VCL_StackVDP(req, req->vcl, req->vdp_filter_list)) { + VCL_StackVDP(req->vdc, req->vcl, req->vdp_filter_list, req, NULL)) { VSLb(req->vsl, SLT_Error, "Failure to push processors"); req->doclose = SC_OVERLOAD; req->acct.resp_bodybytes += diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 016516bcd..804961493 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -193,7 +193,7 @@ void VDI_Init(void); /* cache_deliver_proc.c */ void VDP_Fini(const struct vdp_ctx *vdc); void VDP_Init(struct vdp_ctx *vdc, struct worker *wrk, struct vsl_log *vsl, - struct req *req); + struct req *req, struct busyobj *bo, intmax_t *cl); uint64_t VDP_Close(struct vdp_ctx *, struct objcore *, struct boc *); void VDP_Panic(struct vsb *vsb, const struct vdp_ctx *vdc); int VDP_Push(VRT_CTX, struct vdp_ctx *, struct ws *, const struct vdp *, @@ -511,7 +511,8 @@ void pan_privs(struct vsb *, const struct vrt_privs *); /* cache_vrt_filter.c */ int VCL_StackVFP(struct vfp_ctx *, const struct vcl *, const char *); -int VCL_StackVDP(struct req *, const struct vcl *, const char *); +int VCL_StackVDP(struct vdp_ctx *vdc, const struct vcl *vcl, const char *fl, + struct req *req, struct busyobj *bo); const char *resp_Get_Filter_List(struct req *req); void VCL_VRT_Init(void); diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c index e0ca18321..702026a47 100644 --- a/bin/varnishd/cache/cache_vrt_filter.c +++ b/bin/varnishd/cache/cache_vrt_filter.c @@ -254,15 +254,28 @@ VCL_StackVFP(struct vfp_ctx *vc, const struct vcl *vcl, const char *fl) } int -VCL_StackVDP(struct req *req, const struct vcl *vcl, const char *fl) +VCL_StackVDP(struct vdp_ctx *vdc, const struct vcl *vcl, const char *fl, + struct req *req, struct busyobj *bo) { const struct vfilter *vp; struct vrt_ctx ctx[1]; + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); + AN(vcl); AN(fl); - VSLbs(req->vsl, SLT_Filters, TOSTRAND(fl)); + + CHECK_OBJ_ORNULL(req, REQ_MAGIC); + CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC); + + assert((req ? 1 : 0) ^ (bo ? 1 : 0)); + + VSLbs(vdc->vsl, SLT_Filters, TOSTRAND(fl)); INIT_OBJ(ctx, VRT_CTX_MAGIC); - VCL_Req2Ctx(ctx, req); + + if (req) + VCL_Req2Ctx(ctx, req); + else + VCL_Bo2Ctx(ctx, bo); while (1) { vp = vcl_filter_list_iter(0, &vrt_filters, &vcl->filters, &fl); @@ -273,7 +286,7 @@ VCL_StackVDP(struct req *req, const struct vcl *vcl, const char *fl) "Filter '...%s' not found", fl); return (-1); } - if (VDP_Push(ctx, req->vdc, req->ws, vp->vdp, NULL)) + if (VDP_Push(ctx, vdc, ctx->ws, vp->vdp, NULL)) return (-1); } } diff --git a/bin/varnishd/http2/cache_http2_deliver.c b/bin/varnishd/http2/cache_http2_deliver.c index cdd356d7d..5de2a7fc4 100644 --- a/bin/varnishd/http2/cache_http2_deliver.c +++ b/bin/varnishd/http2/cache_http2_deliver.c @@ -73,7 +73,7 @@ V2D_Init(void) /**********************************************************************/ static int v_matchproto_(vdp_init_f) -h2_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) +h2_init(VRT_CTX, struct vdp_ctx *vdc, void **priv) { struct h2_req *r2; @@ -82,7 +82,6 @@ h2_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) AN(priv); CAST_OBJ_NOTNULL(r2, *priv, H2_REQ_MAGIC); (void)r2; - (void)oc; return (0); } @@ -348,7 +347,7 @@ h2_deliver(struct req *req, struct boc *boc, int sendbody) if (sendbody) { INIT_OBJ(ctx, VRT_CTX_MAGIC); VCL_Req2Ctx(ctx, req); - if (!VDP_Push(ctx, req->vdc, req->ws, &h2_vdp, r2)) + if (! VDP_Push(ctx, req->vdc, req->ws, &h2_vdp, r2)) (void)VDP_DeliverObj(req->vdc, req->objcore); } diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index e78208941..3cf1eb7b7 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -105,16 +105,21 @@ static const struct vfp xyzzy_vfp_rot13 = { #define ROT13_BUFSZ 8 static int v_matchproto_(vdp_init_f) -xyzzy_vdp_rot13_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) +xyzzy_vdp_rot13_init(VRT_CTX, struct vdp_ctx *vdc, void **priv) { - (void)vdc; - (void)oc; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); + CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC); + AN(vdc->clen); + AN(priv); + *priv = malloc(ROT13_BUFSZ); if (*priv == NULL) return (-1); + return (0); } @@ -182,21 +187,18 @@ static const struct vdp xyzzy_vdp_rot13 = { */ static int v_matchproto_(vdp_init_f) -xyzzy_vdp_chunked_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc) +xyzzy_vdp_chunked_init(VRT_CTX, struct vdp_ctx *vdc, void **priv) { - struct http *hp; - - (void)vdc; - (void)oc; - (void)priv; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); - CHECK_OBJ_NOTNULL(vdc->req, REQ_MAGIC); - hp = vdc->req->resp; - CHECK_OBJ_NOTNULL(hp, HTTP_MAGIC); - http_Unset(hp, H_Content_Length); - vdc->req->resp_len = -1; + CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC); + AN(vdc->clen); + AN(priv); + + http_Unset(vdc->hp, H_Content_Length); + *vdc->clen = -1; return (1); } @@ -248,15 +250,18 @@ static const struct vmod_priv_methods priv_pedantic_methods[1] = {{ }}; static int v_matchproto_(vdp_init_f) -xyzzy_pedantic_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, - struct objcore *oc) +xyzzy_pedantic_init(VRT_CTX, struct vdp_ctx *vdc, void **priv) { struct vdp_state_s *vdps; struct vmod_priv *p; - (void)oc; - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC); + CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC); + CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC); + AN(vdc->clen); + AN(priv); + WS_TASK_ALLOC_OBJ(ctx, vdps, VDP_STATE_MAGIC); if (vdps == NULL) return (-1); @@ -268,7 +273,6 @@ xyzzy_pedantic_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, p->priv = vdps; p->methods = priv_pedantic_methods; - AN(priv); *priv = vdps; vdps->state = VDPS_INIT; From nils.goroll at uplex.de Mon Sep 2 15:30:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 2 Sep 2024 15:30:06 +0000 (UTC) Subject: [master] d879bd3a8 Flexelint-polish of Generalize the VDP API Message-ID: <20240902153006.34309B2965@lists.varnish-cache.org> commit d879bd3a8262f02aae79992de375beafd2614b6b Author: Nils Goroll Date: Mon Sep 2 17:22:35 2024 +0200 Flexelint-polish of Generalize the VDP API diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index db658d17b..5321fa06c 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -71,14 +71,12 @@ VDP_Fini(const struct vdp_ctx *vdc) void VDP_Init(struct vdp_ctx *vdc, struct worker *wrk, struct vsl_log *vsl, - struct req *req, struct busyobj *bo, intmax_t *clen) + const struct req *req, const struct busyobj *bo, intmax_t *clen) { AN(vdc); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); AN(vsl); - CHECK_OBJ_ORNULL(req, REQ_MAGIC); - CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC); AN(clen); assert((req ? 1 : 0) ^ (bo ? 1 : 0)); @@ -93,10 +91,12 @@ VDP_Init(struct vdp_ctx *vdc, struct worker *wrk, struct vsl_log *vsl, vdc->clen = clen; if (req != NULL) { + CHECK_OBJ(req, REQ_MAGIC); vdc->oc = req->objcore; vdc->hp = req->resp; } else { + CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); vdc->oc = bo->bereq_body; vdc->hp = bo->bereq; } diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 804961493..e0b88b4aa 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -193,7 +193,7 @@ void VDI_Init(void); /* cache_deliver_proc.c */ void VDP_Fini(const struct vdp_ctx *vdc); void VDP_Init(struct vdp_ctx *vdc, struct worker *wrk, struct vsl_log *vsl, - struct req *req, struct busyobj *bo, intmax_t *cl); + const struct req *req, const struct busyobj *bo, intmax_t *cl); uint64_t VDP_Close(struct vdp_ctx *, struct objcore *, struct boc *); void VDP_Panic(struct vsb *vsb, const struct vdp_ctx *vdc); int VDP_Push(VRT_CTX, struct vdp_ctx *, struct ws *, const struct vdp *, diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c index 702026a47..40236fd9b 100644 --- a/bin/varnishd/cache/cache_vrt_filter.c +++ b/bin/varnishd/cache/cache_vrt_filter.c @@ -282,7 +282,7 @@ VCL_StackVDP(struct vdp_ctx *vdc, const struct vcl *vcl, const char *fl, if (vp == NULL) return (0); if (vp == vfilter_error) { - VSLb(req->vsl, SLT_Error, + VSLb(ctx->vsl, SLT_Error, "Filter '...%s' not found", fl); return (-1); } From nils.goroll at uplex.de Mon Sep 2 15:30:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 2 Sep 2024 15:30:06 +0000 (UTC) Subject: [master] f4bbb91ae Flexelint-polish Message-ID: <20240902153006.4E76BB2968@lists.varnish-cache.org> commit f4bbb91ae9b3af18590ba42e261da2f7153c6d6e Author: Nils Goroll Date: Mon Sep 2 17:28:16 2024 +0200 Flexelint-polish Error 82: return ; illegal with void function Ref 830525b63e5f4d42c52f1ea11c7c095a1b0edb20 diff --git a/bin/varnishd/mgt/mgt_jail_linux.c b/bin/varnishd/mgt/mgt_jail_linux.c index 4c3d99fe2..10243049c 100644 --- a/bin/varnishd/mgt/mgt_jail_linux.c +++ b/bin/varnishd/mgt/mgt_jail_linux.c @@ -96,7 +96,7 @@ static int vjl_make_workdir(const char *dname, const char *what, struct vsb *vsb } static void vjl_fixfd(int fd, enum jail_fixfd_e what) { - return jail_tech_unix.fixfd(fd, what); + jail_tech_unix.fixfd(fd, what); } const struct jail_tech jail_tech_linux = { From nils.goroll at uplex.de Thu Sep 5 11:56:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 5 Sep 2024 11:56:05 +0000 (UTC) Subject: [master] 31aae7b16 varnishd: Polish documentation of the generic -a option syntax Message-ID: <20240905115605.B39ACA24B9@lists.varnish-cache.org> commit 31aae7b16bed4f3f8e900eea539752dac99c3454 Author: Nils Goroll Date: Thu Sep 5 13:50:07 2024 +0200 varnishd: Polish documentation of the generic -a option syntax Motivated by #3976 diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 2835860a2..c9930e77c 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -63,19 +63,26 @@ OPTIONS Basic options ------------- --a <[name=][listen_address[,PROTO]]> +-a <[name=][listen_address[,PROTO|,option=value,...]]> - Accept for client requests on the specified listen_address (see below). + Generic syntax to accept client requests on a listen_address. See below for + details. - Name is referenced in logs. If name is not specified, "a0", "a1", - etc. is used. + Name is referenced in logs and available to vcl as ``local.socket``. If name + is not specified, ``a`` with a numerical sequence ("a0", "a1", etc.) is used. + + Any arguments after the listen_address separated by comma are taken as either + an acceptor ``option=value`` pair if containing a ``=``, or as a PROTO(col) + selection otherwise. + + Valid options depend on the acceptor type, see below. PROTO can be "HTTP" (the default) or "PROXY". Both version 1 and 2 of the proxy protocol can be used. Multiple -a arguments are allowed. - If no -a argument is given, the default `-a :80` will listen to + If no -a argument is given, the default `-a :80` will listen on all IPv4 and IPv6 interfaces. -a <[name=][ip_address][:port][,PROTO]> From nils.goroll at uplex.de Thu Sep 5 13:03:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 5 Sep 2024 13:03:05 +0000 (UTC) Subject: [master] 49e43dfeb varnishd usage: make more room on the left hand side Message-ID: <20240905130305.F2293A4BF2@lists.varnish-cache.org> commit 49e43dfeb048debe10e316773a1337460a770573 Author: Nils Goroll Date: Thu Sep 5 14:51:14 2024 +0200 varnishd usage: make more room on the left hand side before this patch, the longest line was 73 characters. Because we already exceeded the width of the left column, give it the additional space for a line length of up to 80 characters. diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 8b3ccfad6..56cdf6f04 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -97,7 +97,7 @@ usage(void) { char *p; -#define FMT_NONL " %-28s # %s" +#define FMT_NONL " %-35s # %s" #define FMT FMT_NONL "\n" printf( "Usage: varnishd [options]\n"); From nils.goroll at uplex.de Thu Sep 5 13:03:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 5 Sep 2024 13:03:06 +0000 (UTC) Subject: [master] 2911bd168 varnishd: harmonize -a usage help Message-ID: <20240905130306.12ED8A4BF5@lists.varnish-cache.org> commit 2911bd168bd1602e8f9a594c395f2ff0cd3f6212 Author: Nils Goroll Date: Thu Sep 5 15:02:12 2024 +0200 varnishd: harmonize -a usage help diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 56cdf6f04..1b8e49f27 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -104,14 +104,18 @@ usage(void) printf("\nBasic options:\n"); - printf(FMT, "-a [=]address[:port][,proto]", - "HTTP listen address and port"); - printf(FMT, " [,user=][,group=]", + printf(FMT, "[-a [name=][listen_address", + "HTTP listen address, protocol, options."); + printf(FMT, " [,PROTO|,option=value,...]]", "Can be specified multiple times."); - printf(FMT, " [,mode=]", " default: \":80,HTTP\""); - printf(FMT, "", "Proto can be \"PROXY\" or \"HTTP\" (default)"); - printf(FMT, "", "user, group and mode set permissions for"); - printf(FMT, "", " a Unix domain socket."); + printf(FMT, "", + " default: \":80,HTTP\""); + printf(FMT, " options:", + "Proto can be \"PROXY\" or \"HTTP\" (default)"); + printf(FMT, " [,user=][,group=]", + "user, group and mode set permissions for"); + printf(FMT, " [,mode=]", + " a Unix domain socket."); printf(FMT, "-b none", "No backend"); printf(FMT, "-b [addr[:port]|path]", "Backend address and port"); printf(FMT, "", " or socket file path"); diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index c9930e77c..1ca94c7a5 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -21,7 +21,7 @@ SYNOPSIS ======== varnishd - [-a [name=][listen_address[,PROTO]] + [-a [name=][listen_address[,PROTO|,option=value,...]] [-b [host[:port]|path]] [-C] [-d] From nils.goroll at uplex.de Fri Sep 6 11:53:10 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 6 Sep 2024 11:53:10 +0000 (UTC) Subject: [master] 0d77366e4 varnishd: Plug irrelevant leak in VPI_Vmod_Init() Message-ID: <20240906115310.791A1ABFE1@lists.varnish-cache.org> commit 0d77366e42a57d7dcb1f79b7886c552f99e9b604 Author: Nils Goroll Date: Fri Sep 6 13:51:23 2024 +0200 varnishd: Plug irrelevant leak in VPI_Vmod_Init() for error cases, we would leak the memory allocated for v->backup diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c index 8a088a19d..c1fa22070 100644 --- a/bin/varnishd/cache/cache_vrt_vmod.c +++ b/bin/varnishd/cache/cache_vrt_vmod.c @@ -109,7 +109,6 @@ VPI_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len, if (v == NULL) { ALLOC_OBJ(v, VMOD_MAGIC); AN(v); - REPLACE(v->backup, backup); v->hdl = dlhdl; @@ -151,6 +150,7 @@ VPI_Vmod_Init(VRT_CTX, struct vmod **hdl, unsigned nbr, void *ptr, int len, REPLACE(v->nm, nm); REPLACE(v->path, path); + REPLACE(v->backup, backup); VSC_C_main->vmods++; VTAILQ_INSERT_TAIL(&vmods, v, list); From nils.goroll at uplex.de Fri Sep 6 17:38:07 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 6 Sep 2024 17:38:07 +0000 (UTC) Subject: [master] 444d0bea4 doc: Changelog TLC Message-ID: <20240906173807.3A56AC1F75@lists.varnish-cache.org> commit 444d0bea46254526149f4f5daa4c179e642746a9 Author: Nils Goroll Date: Fri Sep 6 19:37:26 2024 +0200 doc: Changelog TLC diff --git a/doc/changes.rst b/doc/changes.rst index 788160c50..42c850753 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -1,6 +1,6 @@ .. - Copyright (c) 2011-2023 Varnish Software AS - Copyright 2016-2023 UPLEX - Nils Goroll Systemoptimierung + Copyright (c) 2011-2024 Varnish Software AS + Copyright 2016-2024 UPLEX - Nils Goroll Systemoptimierung SPDX-License-Identifier: BSD-2-Clause See LICENSE file for full text of license @@ -41,22 +41,138 @@ Varnish Cache NEXT (2024-09-15) .. PLEASE keep this roughly in commit order as shown by git-log / tig (new to old) -* Backend tasks can now queue if the backend has reached its max_connections. - This allows the task to wait for a connection to become available rather - than immediately failing. This feature must be enabled with the new - parameters added. +* The Varnish Delivery Processor (VDP) filter API has been generalized to also + accommodate future use for backend request bodies: - New parameters: - ``backend_wait_timeout`` sets the amount of time a task will wait. - ``backend_wait_limit`` sets the maximum number of tasks that can wait. + - ``VDP_Init()`` gained a ``struct busyobj *`` argument for use of VDPs on the + backend side, which is mutually exclusive with the existing ``struct req *`` + argument (one of the two needs to be ``NULL``). ``VDP_Init()`` also gained + an ``intmax_t *`` pointer, which needs to point to the known content length + of the body data or ``-1`` for "unknown length". Filters can change this + value. - These parameters can also be set in the backend with ``wait_timeout`` - and ``wait_limit``. + - ``struct vdp_ctx`` lost the ``req`` member, but gained ``struct objcore + *oc``, ``struct http *hp`` and ``intmax_t *clen`` members. The rationale + here is that a VDP should be concerned mainly with transforming body data + (for which ``clen`` is relevant) and optionally changing (from the + ``vdp_init_f``) the headers sent before the body data, for which ``hp`` is + intended. Some VDPs also work directly on a ``struct objcore *``, so ``oc`` + is provided to the first VDP in the chain only. - New counters: - ``backend_wait`` count of tasks that waited in queue for a connection. - ``backend_wait_fail`` count of tasks that waited in queue but did not get - a connection within the ``wait_timeout``. + Generic VDPs should specifically not access the request or be concerned with + the object. + + Yet special purpose VDPs still can take from ``VRT_CTX`` whatever references + they need in the ``vdp_init_f`` and store them in their private data. + + - Consequent to what as been explained above, ``vdp_init_f`` lost its ``struct + objcore *`` argument. + +* VDPs with no ``vdp_bytes_f`` function are now supported if the ``vdp_init_f`` + returns a value greater than zero to signify that the filter is not to be + added to the chain. This is useful to support VDPs which only need to work on + headers. + +* The ``epoll`` and ``kqueue`` waiters have been improved to correctly report + ``WAITER_REMCLOSE``, which increases the ``WAITER.*.remclose`` counter. + +* ``varnishtest`` now supports the ``shutdown`` command corresponding to the + ``shutdown(2)`` standard C library call. + +* VSC counters for waiters have been added: + + * ``conns`` to count waits on idle connections + * ``remclose`` to count idle connections closed by the peer + * ``timeout`` to count idle connections which timed out in the waiter + * ``action`` to count idle connections which resulted in a read + + These can be found under ``WAITER..``. + +* The port of a *listen_endpoint* given with the ``-a`` argument to ``varnishd`` + can now also be a numerical port range like ``80-89``, besides the existing + options of port number (e.g. ``80``) and service name (e.g. ``http``). With a + port range, Varnish will accept connections on all ports within the range. + +* To implement the aforementioned feature, ``VSS_resolver_range()`` as been + added to ``libvarnish``. + +* The ``Warning: mlock() of VSM failed`` message is now emitted when locking of + shared memory segments (via ``mlock(2)``) fails. As Varnish performance may + severely be impacted if shared memory segments are not resident in RAM, users + seeing this message are urged to review the ``RLIMIT_MEMLOCK`` resource + control as set via ``ulimit -l`` or ``LimitMEMLOCK`` with ``systemd(1)``. + +* A bug has been fixed where string comparisons in VCL could fail with the + nonsensical error message ``Comparison of different types: STRING '==' + STRING``. + +.. _RFC9110: https://www.rfc-editor.org/rfc/rfc9110.html#section-14.4 + +* An issue has been addressed in the ``builtin.vcl`` where backend responses + would fail if they contained a ``Content-Range`` header when no range was + requested. According to `RFC9110`_, this header should just be ignored, yet + some Varnish-Users might prefer stricter checks. Thus, we decided to change + the ``builtin.vcl`` only and users hitting this issue are advised to call + ``vcl_beresp_range`` in custom VCL. + +* Additional ``SessError`` VSL events are now generated for various HTTP/2 + protocol errors. Some HTTP/2 log events have been changed from ``Debug`` and + ``Error`` to ``SessError``. + +* A new ``linux`` jail has been added which is now the default on Linux. For + now, it is almost identical to the ``unix`` jail with one addition: + +* When the new ``linux`` jail is used, the ``Working directory not mounted on + tmpfs partition`` warning is now emitted if the working directory is found to + reside on a file system other than ``tmpfs``. While other file systems are + supported (and might be the right choice where administrators understand how + to avoid blocking disk IO while ``varnishd`` is writing to shared memory), + ``tmpfs`` is the failsafe option to avoid performance issues. + +* A race condition with VCL temperature transitions has been addressed, which + likely caused issues with dynamic directors. + +* The implementation of the ``transit_buffer`` has now been made the + responsibility of storage engines. + +.. _4108: https://github.com/varnishcache/varnish-cache/issues/4108 + +* Internal management of probes has been reworked to address race conditions + which could cause panics with VCL temperature changes and discards (`4108`_). + +* Backend tasks can now be instructed to queue if the backend has reached its + ``max_connections``. This allows tasks to wait for a connection to become + available rather than immediately failing. This feature must be enabled + through new global parameters or individual backend properties: + + * ``backend_wait_timeout`` sets the amount of time a task will wait. + * ``backend_wait_limit`` sets the maximum number of tasks that can wait. + + These parameters can also be set for individual backends using the + ``wait_timeout`` and ``wait_limit`` properties. + + Tasks waiting on a backend going sick (either explicitly via the + ``backend.set_health`` command or implicitly through the probe) fail + immediately. + + Global VSC counters have been added under ``MAIN``: + + * ``backend_wait`` counts tasks which waited in queue for a connection. + * ``backend_wait_fail`` counts tasks which waited in queue but failed because + ``wait_timeout`` was reached or the backend went sick. + +* The size of the buffer to hold panic messages is now tunable through the new + ``panic_buffer`` parameter. + +* The Varnish Shared Memory (VSM) and Varnish Shared Counters (VSC) consumer + implementation in ``libvarnishapi`` have been improved for stability and + performance. + +.. _4088: https://github.com/varnishcache/varnish-cache/issues/4088 + +* An issue has been fixed where Varnish Shared Log (VSL) queries (for example + using ``varnishlog -q``) with numerical values would fail in unexpected ways + due to truncation. (`4088`_) * The ObjWaitExtend() Object API function gained a ``statep`` argument to optionally return the busy object state consistent with the @@ -79,6 +195,11 @@ Varnish Cache NEXT (2024-09-15) .. _VMOD developer documentation: doc/sphinx/reference/vmod.rst +* An glitch with ttl comparisons has been fixed which could, for example, lead + to unexpected behavior with ``purge.soft()``. + +.. TODO 0e75d46357fc26ab59b9f660460d7c748f2c8be4 hpack ? + ================================ Varnish Cache 7.5.0 (2024-03-18) ================================ From dridi at varni.sh Tue Sep 10 14:15:36 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 10 Sep 2024 14:15:36 +0000 Subject: [master] 2c3e4a148 spelling: gzipped In-Reply-To: <20240819124307.A2C1811DFB6@lists.varnish-cache.org> References: <20240819124307.A2C1811DFB6@lists.varnish-cache.org> Message-ID: On Mon, Aug 19, 2024 at 1:05?PM Poul-Henning Kamp wrote: > > > commit 2c3e4a1485e5bf6a460c3d8ab272ee9888bbfaf1 > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > Date: Wed Aug 7 08:30:57 2024 -0400 > > spelling: gzipped > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > diff --git a/include/tbl/obj_attr.h b/include/tbl/obj_attr.h > index f9441b17a..dbbb2295d 100644 > --- a/include/tbl/obj_attr.h > +++ b/include/tbl/obj_attr.h > @@ -56,7 +56,7 @@ > > #ifdef OBJ_FLAG > /* upper, lower, val */ > - OBJ_FLAG(GZIPED, gziped, (1<<1)) > + OBJ_FLAG(GZIPED, gzipped, (1<<1)) We don't appear to use the lowercase token anywhere, but is it a good idea to desync? > OBJ_FLAG(CHGCE, chgce, (1<<2)) > OBJ_FLAG(IMSCAND, imscand, (1<<3)) > OBJ_FLAG(ESIPROC, esiproc, (1<<4)) > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi at varni.sh Tue Sep 10 14:17:54 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 10 Sep 2024 14:17:54 +0000 Subject: [master] 8875cbf26 spelling: functionally In-Reply-To: <20240819124712.05AF3120ADB@lists.varnish-cache.org> References: <20240819124712.05AF3120ADB@lists.varnish-cache.org> Message-ID: On Mon, Aug 19, 2024 at 1:06?PM Poul-Henning Kamp wrote: > > > commit 8875cbf2628a9dab37b9dbe088598b243c6210fc > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > Date: Wed Aug 7 08:30:57 2024 -0400 > > spelling: functionally > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > diff --git a/doc/sphinx/whats-new/changes-7.2.rst b/doc/sphinx/whats-new/changes-7.2.rst > index c7fbbbf09..a48cfa640 100644 > --- a/doc/sphinx/whats-new/changes-7.2.rst > +++ b/doc/sphinx/whats-new/changes-7.2.rst > @@ -25,7 +25,7 @@ keep a place ready for it in the overall architecture. > > Now a credible use-case finally appeared, and we have implemented > "Varnish Extensions" (VTLA: "VEXT"), which can both be used to load > -ambient VMODs and to implement entirely new functionaly, for instance > +ambient VMODs and to implement entirely new functionally, for instance Isn't the correct word functionality here? > stevedores. > > See :ref:`ref-vext` in the reference manual for more information. > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi at varni.sh Tue Sep 10 14:20:44 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 10 Sep 2024 14:20:44 +0000 Subject: [master] c1391705e spelling: important In-Reply-To: <20240819124712.C48A6120B79@lists.varnish-cache.org> References: <20240819124712.C48A6120B79@lists.varnish-cache.org> Message-ID: On Mon, Aug 19, 2024 at 1:07?PM Poul-Henning Kamp wrote: > > > commit c1391705e3b1d0dd600b5b7dff3402826fb5f86b > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > Date: Wed Aug 7 08:30:57 2024 -0400 > > spelling: important > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > diff --git a/doc/README.WRITING_RST.rst b/doc/README.WRITING_RST.rst > index bb1a5f7c1..0d203ece0 100644 > --- a/doc/README.WRITING_RST.rst > +++ b/doc/README.WRITING_RST.rst > @@ -46,7 +46,7 @@ of documentation: > > .. _varnishd(1): > > -* set link targets for imporant paramgraphs following the scheme > +* set link targets for important paramgraphs following the scheme Is paramgraphs coming up next? > ref-`doc`-`section`, for instance:: > > .. _ref-varnishd-opt_T: > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi at varni.sh Tue Sep 10 14:23:03 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 10 Sep 2024 14:23:03 +0000 Subject: [master] f102d3353 spelling: in between In-Reply-To: <20240819124712.DEFB2120B8D@lists.varnish-cache.org> References: <20240819124712.DEFB2120B8D@lists.varnish-cache.org> Message-ID: On Mon, Aug 19, 2024 at 1:07?PM Poul-Henning Kamp wrote: > > > commit f102d3353569dcd130870c083ecd6ecc5fd77b73 > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > Date: Wed Aug 7 08:30:57 2024 -0400 > > spelling: in between > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst > index 553c406f7..bac18cd03 100644 > --- a/doc/sphinx/reference/vmod.rst > +++ b/doc/sphinx/reference/vmod.rst > @@ -488,7 +488,7 @@ VCL_SUB > side). > > For more than one invocation of ``VRT_call()``, VMODs *must* > - check if ``VRT_handled()`` returns non-zero inbetween calls: > + check if ``VRT_handled()`` returns non-zero in between calls: Isn't the correct spelling in-between in this context? > The called SUB may have returned with an action (any > ``return(x)`` other than plain ``return``) or may have failed > the VCL, and in both cases the calling VMOD *must* return > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi at varni.sh Tue Sep 10 14:41:18 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 10 Sep 2024 14:41:18 +0000 Subject: [master] 81322fe59 spelling: object In-Reply-To: <20240819124715.49169120CBC@lists.varnish-cache.org> References: <20240819124715.49169120CBC@lists.varnish-cache.org> Message-ID: On Mon, Aug 19, 2024 at 1:10?PM Poul-Henning Kamp wrote: > > > commit 81322fe592fdc5ca4720b17547526592bba9385b > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > Date: Wed Aug 7 08:30:59 2024 -0400 > > spelling: object > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > diff --git a/doc/sphinx/whats-new/changes-7.4.rst b/doc/sphinx/whats-new/changes-7.4.rst > index 53c56f90f..a39e2c16b 100644 > --- a/doc/sphinx/whats-new/changes-7.4.rst > +++ b/doc/sphinx/whats-new/changes-7.4.rst > @@ -56,7 +56,7 @@ varnishlog > ========== > > Object creation failures by the selected storage engine are now logged > -under the ``Error`` tag as ``Failed to create object object from %s > +under the ``Error`` tag as ``Failed to create object from %s > %s``. Note to self, there's one more in doc/changes.rst > varnishadm > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi at varni.sh Tue Sep 10 14:43:44 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 10 Sep 2024 14:43:44 +0000 Subject: [master] 91dae394f spelling: paragraphs In-Reply-To: <20240819124715.7ABAA120CD5@lists.varnish-cache.org> References: <20240819124715.7ABAA120CD5@lists.varnish-cache.org> Message-ID: On Mon, Aug 19, 2024 at 1:11?PM Poul-Henning Kamp wrote: > > > commit 91dae394f72b70f50eb858706046703abd3ba357 > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > Date: Wed Aug 7 08:30:59 2024 -0400 > > spelling: paragraphs > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > diff --git a/doc/README.WRITING_RST.rst b/doc/README.WRITING_RST.rst > index 0d203ece0..5bb846a69 100644 > --- a/doc/README.WRITING_RST.rst > +++ b/doc/README.WRITING_RST.rst > @@ -46,7 +46,7 @@ of documentation: > > .. _varnishd(1): > > -* set link targets for important paramgraphs following the scheme > +* set link targets for important paragraphs following the scheme And there it is :) > ref-`doc`-`section`, for instance:: > > .. _ref-varnishd-opt_T: > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi at varni.sh Tue Sep 10 14:47:01 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 10 Sep 2024 14:47:01 +0000 Subject: [master] 65d828d6b spelling: silliness In-Reply-To: <20240819124717.7B13D120DE9@lists.varnish-cache.org> References: <20240819124717.7B13D120DE9@lists.varnish-cache.org> Message-ID: On Mon, Aug 19, 2024 at 1:13?PM Poul-Henning Kamp wrote: > > > commit 65d828d6b5599746a40b719fcbae07da5a71eddb > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > Date: Wed Aug 7 09:11:13 2024 -0400 > > spelling: silliness > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > diff --git a/doc/sphinx/phk/thetoolsweworkwith.rst b/doc/sphinx/phk/thetoolsweworkwith.rst > index 5ed32a7f2..b763c8b3c 100644 > --- a/doc/sphinx/phk/thetoolsweworkwith.rst > +++ b/doc/sphinx/phk/thetoolsweworkwith.rst > @@ -46,7 +46,7 @@ ISO-C standardisation working-group 14. > I won't claim that it is enough to make grown men cry, but it > certainly was enough to make me angry. > > -Let me give you an example of their utter sillyness: > +Let me give you an example of their utter silliness: > > The book which defined the C langauge had a list af reserved Is langauge coming next? > identifiers, all of them lower-case words. The UNIX libraries > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi at varni.sh Tue Sep 10 15:08:08 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 10 Sep 2024 15:08:08 +0000 Subject: [master] aba90542c spelling: unsettable In-Reply-To: <20240819124719.B644D120EE6@lists.varnish-cache.org> References: <20240819124719.B644D120EE6@lists.varnish-cache.org> Message-ID: On Mon, Aug 19, 2024 at 1:25?PM Poul-Henning Kamp wrote: > > > commit aba90542c488a341e194c67f8a2148e9f0b59b6c > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > Date: Wed Aug 7 09:14:01 2024 -0400 > > spelling: unsettable > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst > index b277150e9..e895753b2 100644 > --- a/doc/sphinx/reference/vcl_var.rst > +++ b/doc/sphinx/reference/vcl_var.rst > @@ -337,7 +337,7 @@ req.http.* > > Writable from: client > > - Unsetable from: client > + Unsettable from: client Note to self, did it break generate.py? > > The headers of request, things like ``req.http.date``. > @@ -652,7 +652,7 @@ bereq.between_bytes_timeout > > Writable from: backend > > - Unsetable from: vcl_pipe, backend > + Unsettable from: vcl_pipe, backend > > Default: ``.between_bytes_timeout`` attribute from the > :ref:`backend_definition`, which defaults to the > @@ -668,7 +668,7 @@ bereq.body > > Type: BODY > > - Unsetable from: vcl_backend_fetch > + Unsettable from: vcl_backend_fetch > > The request body. > > @@ -684,7 +684,7 @@ bereq.connect_timeout > > Writable from: vcl_pipe, backend > > - Unsetable from: vcl_pipe, backend > + Unsettable from: vcl_pipe, backend > > Default: ``.connect_timeout`` attribute from the > :ref:`backend_definition`, which defaults to the > @@ -704,7 +704,7 @@ bereq.first_byte_timeout > > Writable from: backend > > - Unsetable from: vcl_pipe, backend > + Unsettable from: vcl_pipe, backend > > Default: ``.first_byte_timeout`` attribute from the > :ref:`backend_definition`, which defaults to the > @@ -735,7 +735,7 @@ bereq.http.* > > Writable from: vcl_pipe, backend > > - Unsetable from: vcl_pipe, backend > + Unsettable from: vcl_pipe, backend > > The headers to be sent to the backend. > > @@ -856,7 +856,7 @@ bereq.task_deadline > > Writable from: vcl_pipe > > - Unsetable from: vcl_pipe > + Unsettable from: vcl_pipe > > Deadline for pipe sessions, defaults ``0s``, which falls back to the > ``pipe_task_deadline`` parameter, see :ref:`varnishd(1)` > @@ -1175,7 +1175,7 @@ beresp.http.* > > Writable from: vcl_backend_response, vcl_backend_error > > - Unsetable from: vcl_backend_response, vcl_backend_error > + Unsettable from: vcl_backend_response, vcl_backend_error > > The HTTP headers returned from the server. > > @@ -1625,7 +1625,7 @@ resp.http.* > > Writable from: vcl_deliver, vcl_synth > > - Unsetable from: vcl_deliver, vcl_synth > + Unsettable from: vcl_deliver, vcl_synth > > The HTTP headers that will be returned. > > @@ -1787,7 +1787,7 @@ sess.idle_send_timeout > > Writable from: client > > - Unsetable from: client > + Unsettable from: client > > Send timeout for individual pieces of data on client > connections, defaults to the ``idle_send_timeout`` parameter, > @@ -1804,7 +1804,7 @@ sess.send_timeout > > Writable from: client > > - Unsetable from: client > + Unsettable from: client > > Total timeout for ordinary HTTP1 responses, defaults to the > ``send_timeout`` parameter, see :ref:`varnishd(1)` > @@ -1820,7 +1820,7 @@ sess.timeout_idle > > Writable from: client > > - Unsetable from: client > + Unsettable from: client > > Idle timeout for this session, defaults to the > ``timeout_idle`` parameter, see :ref:`varnishd(1)` > @@ -1836,7 +1836,7 @@ sess.timeout_linger > > Writable from: client > > - Unsetable from: client > + Unsettable from: client > > Linger timeout for this session, defaults to the > ``timeout_linger`` parameter, see :ref:`varnishd(1)` > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi at varni.sh Tue Sep 10 15:38:49 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 10 Sep 2024 15:38:49 +0000 Subject: [master] 65d828d6b spelling: silliness In-Reply-To: References: <20240819124717.7B13D120DE9@lists.varnish-cache.org> Message-ID: On Tue, Sep 10, 2024 at 2:47?PM Dridi Boukelmoune wrote: > > On Mon, Aug 19, 2024 at 1:13?PM Poul-Henning Kamp wrote: > > > > > > commit 65d828d6b5599746a40b719fcbae07da5a71eddb > > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > > Date: Wed Aug 7 09:11:13 2024 -0400 > > > > spelling: silliness > > > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > > > diff --git a/doc/sphinx/phk/thetoolsweworkwith.rst b/doc/sphinx/phk/thetoolsweworkwith.rst > > index 5ed32a7f2..b763c8b3c 100644 > > --- a/doc/sphinx/phk/thetoolsweworkwith.rst > > +++ b/doc/sphinx/phk/thetoolsweworkwith.rst > > @@ -46,7 +46,7 @@ ISO-C standardisation working-group 14. > > I won't claim that it is enough to make grown men cry, but it > > certainly was enough to make me angry. > > > > -Let me give you an example of their utter sillyness: > > +Let me give you an example of their utter silliness: > > > > The book which defined the C langauge had a list af reserved > > Is langauge coming next? It did not, neither did af, I'm on it. From dridi at varni.sh Tue Sep 10 15:40:29 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 10 Sep 2024 15:40:29 +0000 Subject: [master] aba90542c spelling: unsettable In-Reply-To: References: <20240819124719.B644D120EE6@lists.varnish-cache.org> Message-ID: On Tue, Sep 10, 2024 at 3:08?PM Dridi Boukelmoune wrote: > > On Mon, Aug 19, 2024 at 1:25?PM Poul-Henning Kamp wrote: > > > > > > commit aba90542c488a341e194c67f8a2148e9f0b59b6c > > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > > Date: Wed Aug 7 09:14:01 2024 -0400 > > > > spelling: unsettable > > > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > > > diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst > > index b277150e9..e895753b2 100644 > > --- a/doc/sphinx/reference/vcl_var.rst > > +++ b/doc/sphinx/reference/vcl_var.rst > > @@ -337,7 +337,7 @@ req.http.* > > > > Writable from: client > > > > - Unsetable from: client > > + Unsettable from: client > > Note to self, did it break generate.py? Unfortunately yes, it got fixed in a separate commit, so this one breaks the build. From dridi.boukelmoune at gmail.com Tue Sep 10 15:46:06 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 10 Sep 2024 15:46:06 +0000 (UTC) Subject: [master] 4623d2592 doc: Spelling galore aftermath Message-ID: <20240910154606.7837A11DA0A@lists.varnish-cache.org> commit 4623d25925fb453462bfc27d1412ca944ce2b422 Author: Dridi Boukelmoune Date: Tue Sep 10 17:43:56 2024 +0200 doc: Spelling galore aftermath Better diff with the --word-diff option. diff --git a/doc/changes.rst b/doc/changes.rst index 42c850753..60c4559f1 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -591,8 +591,7 @@ Varnish Cache 7.4.0 (2023-09-15) amount of free workspace for backend connections. * Object creation failures by the selected storage engine are now - logged under the ``Error`` tag as ``Failed to create object object - from %s %s``. + logged under the ``Error`` tag as ``Failed to create object from %s %s``. * The limit on the size of ``varnishtest`` macros has been raised to 2KB. diff --git a/doc/sphinx/phk/thetoolsweworkwith.rst b/doc/sphinx/phk/thetoolsweworkwith.rst index b763c8b3c..c8b9cc782 100644 --- a/doc/sphinx/phk/thetoolsweworkwith.rst +++ b/doc/sphinx/phk/thetoolsweworkwith.rst @@ -48,7 +48,7 @@ certainly was enough to make me angry. Let me give you an example of their utter silliness: -The book which defined the C langauge had a list af reserved +The book which defined the C language had a list of reserved identifiers, all of them lower-case words. The UNIX libraries defined a lot of functions, all of them lower-case words. @@ -113,7 +113,7 @@ that, or without wasting a lot of time debugging silly mistakes. If you look in the Varnish source code, which uses pthreads, you will see that I have wrapped pthread mutexes in my own little -datastructure, to be able to do those asserts, and to get some +data structure, to be able to do those asserts, and to get some usable statistics on lock-contention. Another example where C1X did not improve on pthreads at all, was @@ -152,7 +152,7 @@ Because it's not like the call is actually guaranteed to return at 16:00Z if you ask it to, you are only promised it will not return later than that, so you have to wrap the call in a loop. -Whoever defined the select(2) and poll(2) systemcalls knew better +Whoever defined the select(2) and poll(2) system calls knew better than the POSIX and ISO-C group-think: They specified a maximum duration for the call, because then it doesn't matter what time it is, only how long time has transpired. diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst index bac18cd03..367d7e8ac 100644 --- a/doc/sphinx/reference/vmod.rst +++ b/doc/sphinx/reference/vmod.rst @@ -488,7 +488,7 @@ VCL_SUB side). For more than one invocation of ``VRT_call()``, VMODs *must* - check if ``VRT_handled()`` returns non-zero in between calls: + check if ``VRT_handled()`` returns non-zero in-between calls: The called SUB may have returned with an action (any ``return(x)`` other than plain ``return``) or may have failed the VCL, and in both cases the calling VMOD *must* return diff --git a/doc/sphinx/whats-new/changes-7.2.rst b/doc/sphinx/whats-new/changes-7.2.rst index 0d97bd998..d46e26ad1 100644 --- a/doc/sphinx/whats-new/changes-7.2.rst +++ b/doc/sphinx/whats-new/changes-7.2.rst @@ -25,7 +25,7 @@ keep a place ready for it in the overall architecture. Now a credible use-case finally appeared, and we have implemented "Varnish Extensions" (VTLA: "VEXT"), which can both be used to load -ambient VMODs and to implement entirely new functionally, for instance +ambient VMODs and to implement entirely new functionality, for instance stevedores. See :ref:`ref-vext` in the reference manual for more information. From dridi at varni.sh Wed Sep 11 06:43:49 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 11 Sep 2024 06:43:49 +0000 Subject: [master] 4623d2592 doc: Spelling galore aftermath In-Reply-To: <20240910154606.7837A11DA0A@lists.varnish-cache.org> References: <20240910154606.7837A11DA0A@lists.varnish-cache.org> Message-ID: On Tue, Sep 10, 2024 at 3:46?PM Dridi Boukelmoune wrote: > > > commit 4623d25925fb453462bfc27d1412ca944ce2b422 > Author: Dridi Boukelmoune > Date: Tue Sep 10 17:43:56 2024 +0200 > > doc: Spelling galore aftermath Apparently I managed to miss 128 commits in my spelling review, there might be another followup commit. > Better diff with the --word-diff option. > > diff --git a/doc/changes.rst b/doc/changes.rst > index 42c850753..60c4559f1 100644 > --- a/doc/changes.rst > +++ b/doc/changes.rst > @@ -591,8 +591,7 @@ Varnish Cache 7.4.0 (2023-09-15) > amount of free workspace for backend connections. > > * Object creation failures by the selected storage engine are now > - logged under the ``Error`` tag as ``Failed to create object object > - from %s %s``. > + logged under the ``Error`` tag as ``Failed to create object from %s %s``. > > * The limit on the size of ``varnishtest`` macros has been raised to > 2KB. > diff --git a/doc/sphinx/phk/thetoolsweworkwith.rst b/doc/sphinx/phk/thetoolsweworkwith.rst > index b763c8b3c..c8b9cc782 100644 > --- a/doc/sphinx/phk/thetoolsweworkwith.rst > +++ b/doc/sphinx/phk/thetoolsweworkwith.rst > @@ -48,7 +48,7 @@ certainly was enough to make me angry. > > Let me give you an example of their utter silliness: > > -The book which defined the C langauge had a list af reserved > +The book which defined the C language had a list of reserved > identifiers, all of them lower-case words. The UNIX libraries > defined a lot of functions, all of them lower-case words. > > @@ -113,7 +113,7 @@ that, or without wasting a lot of time debugging silly mistakes. > > If you look in the Varnish source code, which uses pthreads, you > will see that I have wrapped pthread mutexes in my own little > -datastructure, to be able to do those asserts, and to get some > +data structure, to be able to do those asserts, and to get some > usable statistics on lock-contention. > > Another example where C1X did not improve on pthreads at all, was > @@ -152,7 +152,7 @@ Because it's not like the call is actually guaranteed to return at > 16:00Z if you ask it to, you are only promised it will not return > later than that, so you have to wrap the call in a loop. > > -Whoever defined the select(2) and poll(2) systemcalls knew better > +Whoever defined the select(2) and poll(2) system calls knew better > than the POSIX and ISO-C group-think: They specified a maximum > duration for the call, because then it doesn't matter what time > it is, only how long time has transpired. > diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst > index bac18cd03..367d7e8ac 100644 > --- a/doc/sphinx/reference/vmod.rst > +++ b/doc/sphinx/reference/vmod.rst > @@ -488,7 +488,7 @@ VCL_SUB > side). > > For more than one invocation of ``VRT_call()``, VMODs *must* > - check if ``VRT_handled()`` returns non-zero in between calls: > + check if ``VRT_handled()`` returns non-zero in-between calls: > The called SUB may have returned with an action (any > ``return(x)`` other than plain ``return``) or may have failed > the VCL, and in both cases the calling VMOD *must* return > diff --git a/doc/sphinx/whats-new/changes-7.2.rst b/doc/sphinx/whats-new/changes-7.2.rst > index 0d97bd998..d46e26ad1 100644 > --- a/doc/sphinx/whats-new/changes-7.2.rst > +++ b/doc/sphinx/whats-new/changes-7.2.rst > @@ -25,7 +25,7 @@ keep a place ready for it in the overall architecture. > > Now a credible use-case finally appeared, and we have implemented > "Varnish Extensions" (VTLA: "VEXT"), which can both be used to load > -ambient VMODs and to implement entirely new functionally, for instance > +ambient VMODs and to implement entirely new functionality, for instance > stevedores. > > See :ref:`ref-vext` in the reference manual for more information. > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi at varni.sh Wed Sep 11 07:02:03 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 11 Sep 2024 07:02:03 +0000 Subject: [master] 2f43737b2 spelling: irix In-Reply-To: <20240819124206.DD6F411D569@lists.varnish-cache.org> References: <20240819124206.DD6F411D569@lists.varnish-cache.org> Message-ID: On Mon, Aug 19, 2024 at 12:43?PM Poul-Henning Kamp wrote: > > > commit 2f43737b28e050cc394affeeebe583d7eca477f6 > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > Date: Wed Aug 7 08:30:58 2024 -0400 > > spelling: irix > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > diff --git a/lib/libvgz/README b/lib/libvgz/README > index 77dad0005..c07b6b6b3 100644 > --- a/lib/libvgz/README > +++ b/lib/libvgz/README > @@ -59,7 +59,7 @@ Notes for some targets: > > - For Windows DLL versions, please see win32/DLL_FAQ.txt > > -- For 64-bit Irix, deflate.c must be compiled without any optimization. With > +- For 64-bit IRIX, deflate.c must be compiled without any optimization. With This one should be reverted, it's the zlib README > -O, one libpng test fails. The test works in 32 bit mode (with the -n32 > compiler flag). The compiler bug has been reported to SGI. > > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi at varni.sh Wed Sep 11 07:07:14 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 11 Sep 2024 07:07:14 +0000 Subject: [master] 8b4493c5c spelling: with In-Reply-To: <20240819124207.C790911D5CD@lists.varnish-cache.org> References: <20240819124207.C790911D5CD@lists.varnish-cache.org> Message-ID: On Mon, Aug 19, 2024 at 12:44?PM Poul-Henning Kamp wrote: > > > commit 8b4493c5c39d152c926dc8c731b6cadbd4544837 > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > Date: Wed Aug 7 08:31:03 2024 -0400 > > spelling: with > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c > index 67a4efec8..61f119458 100644 > --- a/lib/libvarnish/vsb.c > +++ b/lib/libvarnish/vsb.c > @@ -87,7 +87,7 @@ _assert_VSB_integrity(const char *fun, const struct vsb *s) > KASSERT(s != NULL, > ("%s called with a NULL vsb pointer", fun)); > KASSERT(s->magic == VSB_MAGIC, > - ("%s called wih an bogus vsb pointer", fun)); > + ("%s called with an bogus vsb pointer", fun)); Is "an bogus" coming up later? > KASSERT(s->s_buf != NULL, > ("%s called with uninitialized or corrupt vsb", fun)); > KASSERT(s->s_len < s->s_size, > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi at varni.sh Wed Sep 11 08:20:29 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 11 Sep 2024 08:20:29 +0000 Subject: [master] 606aef9ca spelling: parties In-Reply-To: <20240819124411.6EEA611EB01@lists.varnish-cache.org> References: <20240819124411.6EEA611EB01@lists.varnish-cache.org> Message-ID: On Mon, Aug 19, 2024 at 12:53?PM Poul-Henning Kamp wrote: > > > commit 606aef9ca4a62ba0af6b4f77351858b1cbde60a5 > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > Date: Wed Aug 7 08:30:59 2024 -0400 > > spelling: parties > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c > index 97ce9348f..cf9d7b465 100644 > --- a/bin/varnishd/http2/cache_http2_proto.c > +++ b/bin/varnishd/http2/cache_http2_proto.c > @@ -603,7 +603,7 @@ h2_rx_settings(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) > } > > /********************************************************************** > - * Incoming HEADERS, this is where the partys at... > + * Incoming HEADERS, this is where the parties at... EINVAL > */ > > void v_matchproto_(task_func_t) > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi at varni.sh Wed Sep 11 08:24:34 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 11 Sep 2024 08:24:34 +0000 Subject: [master] d528cd57b spelling: gunzipped In-Reply-To: <20240819124307.8059011DFA2@lists.varnish-cache.org> References: <20240819124307.8059011DFA2@lists.varnish-cache.org> Message-ID: On Mon, Aug 19, 2024 at 12:55?PM Poul-Henning Kamp wrote: > > > commit d528cd57ba1aa65518b1470c64b015c0c115bbd3 > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > Date: Wed Aug 7 08:30:57 2024 -0400 > > spelling: gunzipped > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h > index 2c35404db..5d50d8afc 100644 > --- a/include/tbl/vsl_tags.h > +++ b/include/tbl/vsl_tags.h > @@ -481,7 +481,7 @@ SLTM(VCL_Error, 0, "VCL execution error message", > SLTM(Gzip, 0, "G(un)zip performed on object", > "A Gzip record is emitted for each instance of gzip or gunzip" > " work performed. Worst case, an ESI transaction stored in" > - " gzip'ed objects but delivered gunziped, will run into many of" > + " gzip'ed objects but delivered gunzipped, will run into many of" Is s/gzip'ed/gzipped/ coming later? > " these.\n\n" > "The format is::\n\n" > "\t%c %c %c %d %d %d %d %d\n" > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi at varni.sh Wed Sep 11 08:26:45 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 11 Sep 2024 08:26:45 +0000 Subject: [master] 7b62ae756 spelling: brightness In-Reply-To: <20240819124406.EDBF211E94E@lists.varnish-cache.org> References: <20240819124406.EDBF211E94E@lists.varnish-cache.org> Message-ID: On Mon, Aug 19, 2024 at 12:55?PM Poul-Henning Kamp wrote: > > > commit 7b62ae75690c08d6a61466ee27b68c09e52a4e71 > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > Date: Wed Aug 7 08:30:53 2024 -0400 > > spelling: brightness > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > diff --git a/bin/varnishtest/teken.3 b/bin/varnishtest/teken.3 > index 840a17cec..b5263e4ad 100644 > --- a/bin/varnishtest/teken.3 > +++ b/bin/varnishtest/teken.3 > @@ -181,7 +181,7 @@ The > function is similar to > .Fn teken_256to16 > except it converts to an ANSI 8-color code. > -This is more accurate than discarding the brigtness bit in the result of > +This is more accurate than discarding the brightness bit in the result of PHK, can you commit this one in the upstream tekken code? > .Fn teken_256to16 . > .Pp > The > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From phk at phk.freebsd.dk Wed Sep 11 08:36:34 2024 From: phk at phk.freebsd.dk (Poul-Henning Kamp) Date: Wed, 11 Sep 2024 08:36:34 +0000 Subject: [master] 7b62ae756 spelling: brightness In-Reply-To: References: <20240819124406.EDBF211E94E@lists.varnish-cache.org> Message-ID: <202409110836.48B8aYa3000435@critter.freebsd.dk> -------- Dridi Boukelmoune writes: > PHK, can you commit this one in the upstream tekken code? done. -- Poul-Henning Kamp | UNIX since Zilog Zeus 3.20 phk at FreeBSD.ORG | TCP/IP since RFC 956 FreeBSD committer | BSD since 4.3-tahoe Never attribute to malice what can adequately be explained by incompetence. From dridi at varni.sh Wed Sep 11 09:33:53 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 11 Sep 2024 09:33:53 +0000 Subject: [master] 0133a7242 spelling: actually In-Reply-To: <20240819124705.B30F91207E0@lists.varnish-cache.org> References: <20240819124705.B30F91207E0@lists.varnish-cache.org> Message-ID: On Mon, Aug 19, 2024 at 12:57?PM Poul-Henning Kamp wrote: > > > commit 0133a724275a9c8045f4833c67453da77693fe20 > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > Date: Wed Aug 7 08:30:52 2024 -0400 > > spelling: actually > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > diff --git a/doc/sphinx/phk/cheri2.rst b/doc/sphinx/phk/cheri2.rst > index 71584b40b..26d312d06 100644 > --- a/doc/sphinx/phk/cheri2.rst > +++ b/doc/sphinx/phk/cheri2.rst > @@ -75,7 +75,7 @@ with ``pipe(2)``, but there is no way CHERI could spot that to > make an execption, so reading pointers out of a filedescriptor, > cause fully justified core-dumps. > > -If the poll-waiter was actaully relevant, the proper fix would be > +If the poll-waiter was actually relevant, the proper fix would be Well actually, there is another execption typo above to be checked later. > to let the sending thread stick things on a locked list and just > write a nonce-byte into the pipe to the waiter-thread, but that > goes at the bottom of the TODO list, and for now I just remove the > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi at varni.sh Wed Sep 11 09:44:38 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 11 Sep 2024 09:44:38 +0000 Subject: [master] 83e8292a2 spelling: crypto In-Reply-To: <20240819124708.249931208F7@lists.varnish-cache.org> References: <20240819124708.249931208F7@lists.varnish-cache.org> Message-ID: On Mon, Aug 19, 2024 at 1:01?PM Poul-Henning Kamp wrote: > > > commit 83e8292a28b15945bf17d0d1f26013ac7fa50a51 > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > Date: Wed Aug 7 08:30:54 2024 -0400 > > spelling: crypto > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > diff --git a/doc/sphinx/phk/ssl.rst b/doc/sphinx/phk/ssl.rst > index b38d9847e..581b8653a 100644 > --- a/doc/sphinx/phk/ssl.rst > +++ b/doc/sphinx/phk/ssl.rst > @@ -58,7 +58,7 @@ a SSL proxy in separate process ? > No, it will not, because the way varnish would have to do it would > be to ... start a separate process to do the SSL handling. > > -There is no other way we can guarantee that secret krypto-bits do > +There is no other way we can guarantee that secret crypto-bits do PHP, this one looks intentional, should I revert along with the small followup I'm pushing soon? > not leak anywhere they should not, than by fencing in the code that > deals with them in a child process, so the bulk of varnish never > gets anywhere near the certificates, not even during a core-dump. > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi at varni.sh Wed Sep 11 12:20:32 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 11 Sep 2024 12:20:32 +0000 Subject: [master] 83e8292a2 spelling: crypto In-Reply-To: References: <20240819124708.249931208F7@lists.varnish-cache.org> Message-ID: On Wed, Sep 11, 2024 at 9:44?AM Dridi Boukelmoune wrote: > > On Mon, Aug 19, 2024 at 1:01?PM Poul-Henning Kamp wrote: > > > > > > commit 83e8292a28b15945bf17d0d1f26013ac7fa50a51 > > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > > Date: Wed Aug 7 08:30:54 2024 -0400 > > > > spelling: crypto > > > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > > > diff --git a/doc/sphinx/phk/ssl.rst b/doc/sphinx/phk/ssl.rst > > index b38d9847e..581b8653a 100644 > > --- a/doc/sphinx/phk/ssl.rst > > +++ b/doc/sphinx/phk/ssl.rst > > @@ -58,7 +58,7 @@ a SSL proxy in separate process ? > > No, it will not, because the way varnish would have to do it would > > be to ... start a separate process to do the SSL handling. > > > > -There is no other way we can guarantee that secret krypto-bits do > > +There is no other way we can guarantee that secret crypto-bits do > > PHP, this one looks intentional, should I revert along with the small > followup I'm pushing soon? Of course I meant to ask PHK, not PHP. From dridi.boukelmoune at gmail.com Wed Sep 11 12:34:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 11 Sep 2024 12:34:07 +0000 (UTC) Subject: [master] 06bd1e0c0 doc: Spelling galore aftermath, part 2 Message-ID: <20240911123407.C3DC9926E@lists.varnish-cache.org> commit 06bd1e0c0ec563f5d8e58f409e3923a66fa814a5 Author: Dridi Boukelmoune Date: Wed Sep 11 14:24:00 2024 +0200 doc: Spelling galore aftermath, part 2 Courtesy of inconsistent gmail search results. This batch includes a generalized s/gzip'ed/gzipped/g edit on all files matching. Since we accepted gzipped and gunzipped as the proper spelling we might as well use it everywhere for consistency (mostly documentation and code comments). diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index ba6b0412d..c4063f355 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -504,11 +504,11 @@ ved_bytes(struct ecx *ecx, enum vdp_action act, } /*--------------------------------------------------------------------- - * If a gzip'ed ESI object includes a ungzip'ed object, we need to make - * it looked like a gzip'ed data stream. The official way to do so would + * If a gzipped ESI object includes a ungzipped object, we need to make + * it looked like a gzipped data stream. The official way to do so would * be to fire up libvgz and gzip it, but we don't, we fake it. * - * First, we cannot know if it is ungzip'ed on purpose, the admin may + * First, we cannot know if it is ungzipped on purpose, the admin may * know something we don't. * * What do you mean "BS ?" @@ -589,7 +589,7 @@ static const struct vdp ved_pretend_gz = { }; /*--------------------------------------------------------------------- - * Include a gzip'ed object in a gzip'ed ESI object delivery + * Include a gzipped object in a gzipped ESI object delivery * * This is the interesting case: Deliver all the deflate blocks, stripping * the "LAST" bit of the last one and padding it, as necessary, to a byte @@ -910,7 +910,7 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody) VCL_Req2Ctx(ctx, req); if (ecx->isgzip && i && !(req->res_mode & RES_ESI)) { - /* A gzip'ed include which is not ESI processed */ + /* A gzipped include which is not ESI processed */ /* OA_GZIPBITS are not valid until BOS_FINISHED */ if (boc != NULL) @@ -930,7 +930,7 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody) foo->objcore = req->objcore; i = VDP_Push(ctx, req->vdc, req->ws, &ved_gzgz, foo); } else if (ecx->isgzip && !i) { - /* Non-Gzip'ed include in gzip'ed parent */ + /* Non-Gzip'ed include in gzipped parent */ i = VDP_Push(ctx, req->vdc, req->ws, &ved_pretend_gz, ecx); } else { /* Anything else goes straight through */ diff --git a/bin/varnishd/cache/cache_esi_parse.c b/bin/varnishd/cache/cache_esi_parse.c index b4148e72e..5145b016c 100644 --- a/bin/varnishd/cache/cache_esi_parse.c +++ b/bin/varnishd/cache/cache_esi_parse.c @@ -1132,7 +1132,7 @@ VEP_Finish(struct vep_state *vep) lcb = vep->cb(vep->vc, vep->cb_priv, 0, VGZ_ALIGN); vep_emit_common(vep, lcb - vep->o_last, vep->last_mark); } - // NB: We don't account for PAD+SUM+LEN in gzip'ed objects + // NB: We don't account for PAD+SUM+LEN in gzipped objects (void)vep->cb(vep->vc, vep->cb_priv, 0, VGZ_FINISH); AZ(VSB_finish(vep->vsb)); diff --git a/bin/varnishd/cache/cache_rfc2616.c b/bin/varnishd/cache/cache_rfc2616.c index 38e302237..97f16ed9f 100644 --- a/bin/varnishd/cache/cache_rfc2616.c +++ b/bin/varnishd/cache/cache_rfc2616.c @@ -228,7 +228,7 @@ RFC2616_Ttl(struct busyobj *bo, vtim_real now, vtim_real *t_origin, } /*-------------------------------------------------------------------- - * Find out if the request can receive a gzip'ed response + * Find out if the request can receive a gzipped response */ unsigned diff --git a/bin/varnishd/cache/cache_vary.c b/bin/varnishd/cache/cache_vary.c index 101eca231..fa9053966 100644 --- a/bin/varnishd/cache/cache_vary.c +++ b/bin/varnishd/cache/cache_vary.c @@ -203,7 +203,7 @@ vry_cmp(const uint8_t *v1, const uint8_t *v2) http_hdr_eq(H_Accept_Encoding, (const char*) v1 + 2)) { /* * If we do gzip processing, we do not vary on Accept-Encoding, - * because we want everybody to get the gzip'ed object, and + * because we want everybody to get the gzipped object, and * varnish will gunzip as necessary. We implement the skip at * check time, rather than create time, so that object in * persistent storage can be used with either setting of diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c index 40236fd9b..4d777d464 100644 --- a/bin/varnishd/cache/cache_vrt_filter.c +++ b/bin/varnishd/cache/cache_vrt_filter.c @@ -347,8 +347,8 @@ vbf_default_filter_list(void *arg, struct vsb *vsb) * The backend Content-Encoding header tells us what we are going * to receive, which we classify in the following three classes: * - * "Content-Encoding: gzip" --> object is gzip'ed. - * no Content-Encoding --> object is not gzip'ed. + * "Content-Encoding: gzip" --> object is gzipped. + * no Content-Encoding --> object is not gzipped. * anything else --> do nothing wrt gzip */ @@ -364,11 +364,11 @@ vbf_default_filter_list(void *arg, struct vsb *vsb) else is_gunzip = 1; - /* We won't gunzip unless it is gzip'ed */ + /* We won't gunzip unless it is gzipped */ if (do_gunzip && !is_gzip) do_gunzip = 0; - /* We wont gzip unless if it already is gzip'ed */ + /* We wont gzip unless if it already is gzipped */ if (do_gzip && !is_gunzip) do_gzip = 0; diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index cf9d7b465..9a514e88b 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -603,7 +603,7 @@ h2_rx_settings(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) } /********************************************************************** - * Incoming HEADERS, this is where the parties at... + * Incoming HEADERS, this is where the party's at... */ void v_matchproto_(task_func_t) diff --git a/bin/varnishtest/tests/e00023.vtc b/bin/varnishtest/tests/e00023.vtc index 08517c609..bfc8fd76b 100644 --- a/bin/varnishtest/tests/e00023.vtc +++ b/bin/varnishtest/tests/e00023.vtc @@ -1,4 +1,4 @@ -varnishtest "Include gzip'ed and ungzip'ed (ESI) objects in a gzip'ed ESI object" +varnishtest "Include gzipped and ungzipped (ESI) objects in a gzipped ESI object" server s1 { rxreq diff --git a/bin/varnishtest/tests/e00034.vtc b/bin/varnishtest/tests/e00034.vtc index b0e93ba8e..352c44764 100644 --- a/bin/varnishtest/tests/e00034.vtc +++ b/bin/varnishtest/tests/e00034.vtc @@ -5,7 +5,7 @@ varnishtest "ESI requests, gzip and custom VDPs" # To select the embedding vdps (gzgz / pretendgz), ESI currently only # looks at the cache object's ESI status. So for VDP-processing an -# include of a gzip'ed ESI object, we need to restore the encoding +# include of a gzipped ESI object, we need to restore the encoding # after filtering. Thus, we would currently need a gzip VDP to be able # to use `set resp.filters += " gunzip rot13 gzip"` # diff --git a/bin/varnishtest/tests/r01184.vtc b/bin/varnishtest/tests/r01184.vtc index d5aba3251..0988e65a3 100644 --- a/bin/varnishtest/tests/r01184.vtc +++ b/bin/varnishtest/tests/r01184.vtc @@ -1,4 +1,4 @@ -varnishtest "Corrupt gzip'ed ESI objects" +varnishtest "Corrupt gzipped ESI objects" # First, check that our data is actually good diff --git a/doc/sphinx/phk/gzip.rst b/doc/sphinx/phk/gzip.rst index 4882cd4e7..6a62bf346 100644 --- a/doc/sphinx/phk/gzip.rst +++ b/doc/sphinx/phk/gzip.rst @@ -25,7 +25,7 @@ will not experience any difference, this processing only affects cache hit/miss requests. Unless vcl_recv{} results in "pipe" or "pass", we determine if the -client is capable of receiving gzip'ed content. The test amounts to: +client is capable of receiving gzipped content. The test amounts to: Is there a Accept-Encoding header that mentions gzip, and if is has a q=# number, is it larger than zero. @@ -54,14 +54,14 @@ always set to: Even if this particular client does not support -To always entice the backend into sending us gzip'ed content. +To always entice the backend into sending us gzipped content. Varnish will not gzip any content on its own (but see below), we trust -the backend to know what content can be sensibly gzip'ed (html) and what +the backend to know what content can be sensibly gzipped (html) and what cannot (jpeg) If in vcl_backend_response{} we find out that we are trying to deliver a -gzip'ed object to a client that has not indicated willingness to receive +gzipped object to a client that has not indicated willingness to receive gzip, we will ungzip the object during deliver. Tuning, tweaking and frobbing @@ -84,13 +84,13 @@ gzip-ness of objects during fetch: set beresp.do_gunzip = true; -Will make varnish gunzip an already gzip'ed object from the backend during +Will make varnish gunzip an already gzipped object from the backend during fetch. (I have no idea why/when you would use this...) set beresp.do_gzip = true; Will make varnish gzip the object during fetch from the backend, provided -the backend didn't send us a gzip'ed object. +the backend didn't send us a gzipped object. Remember that a lot of content types cannot sensibly be gzipped, most notably compressed image formats like jpeg, png and similar, so a @@ -115,8 +115,8 @@ In theory, and hopefully in practice, all you read above should apply also when you enable ESI, if not it is a bug you should report. But things are vastly more complicated now. What happens for -instance, when the backend sends a gzip'ed object we ESI process -it and it includes another object which is not gzip'ed, and we want +instance, when the backend sends a gzipped object we ESI process +it and it includes another object which is not gzipped, and we want to send the result gzipped to the client ? Things can get really hairy here, so let me explain it in stages. @@ -139,15 +139,15 @@ the new URL and possibly Host: header, and call vcl_recv{} etc. You can tell that you are in an ESI include by examining the 'req.esi_level' variable in VCL. -The ESI-parsed object is stored gzip'ed under the same conditions as -above: If the backend sends gzip'ed and VCL did not ask for do_gunzip, -or if the backend sends ungzip'ed and VCL asked for do_gzip. +The ESI-parsed object is stored gzipped under the same conditions as +above: If the backend sends gzipped and VCL did not ask for do_gunzip, +or if the backend sends ungzipped and VCL asked for do_gzip. Please note that since we need to insert flush and reset points in the gzip file, it will be slightly larger than a normal gzip file of the same object. -When we encounter gzip'ed include objects which should not be, we +When we encounter gzipped include objects which should not be, we gunzip them, but when we encounter gunzip'ed objects which should be, we gzip them, but only at compression level zero. diff --git a/doc/sphinx/users-guide/compression.rst b/doc/sphinx/users-guide/compression.rst index 3aca6f116..5409b6d76 100644 --- a/doc/sphinx/users-guide/compression.rst +++ b/doc/sphinx/users-guide/compression.rst @@ -34,7 +34,7 @@ Unless the request is a `pass`, Varnish sets `bereq.http.Accept-Encoding` to "gzip" before `vcl_backend_fetch` runs, so the header can be changed in VCL. -If the server responds with gzip'ed content it will be stored in memory +If the server responds with gzipped content it will be stored in memory in its compressed form and `Accept-Encoding` will be added to the `Vary` header. diff --git a/doc/sphinx/users-guide/esi.rst b/doc/sphinx/users-guide/esi.rst index 8920ed7c0..25be4e18a 100644 --- a/doc/sphinx/users-guide/esi.rst +++ b/doc/sphinx/users-guide/esi.rst @@ -209,7 +209,7 @@ Varnish does this compressing all parts of ESI responses separately, and stitching them together on the fly during delivery, which has a negative impact on compression ratio. -When you ``set beresp.do_esi = True;`` on a gzip'ed response, it +When you ``set beresp.do_esi = True;`` on a gzipped response, it will be uncompressed and recompressed part-wise during the fetch. The part-wise compression reduces the opportunities for diff --git a/include/tbl/vsl_tags.h b/include/tbl/vsl_tags.h index 5d50d8afc..9c1aae69d 100644 --- a/include/tbl/vsl_tags.h +++ b/include/tbl/vsl_tags.h @@ -481,7 +481,7 @@ SLTM(VCL_Error, 0, "VCL execution error message", SLTM(Gzip, 0, "G(un)zip performed on object", "A Gzip record is emitted for each instance of gzip or gunzip" " work performed. Worst case, an ESI transaction stored in" - " gzip'ed objects but delivered gunzipped, will run into many of" + " gzipped objects but delivered gunzipped, will run into many of" " these.\n\n" "The format is::\n\n" "\t%c %c %c %d %d %d %d %d\n" diff --git a/lib/libvarnish/vsb.c b/lib/libvarnish/vsb.c index 61f119458..2a405987a 100644 --- a/lib/libvarnish/vsb.c +++ b/lib/libvarnish/vsb.c @@ -87,7 +87,7 @@ _assert_VSB_integrity(const char *fun, const struct vsb *s) KASSERT(s != NULL, ("%s called with a NULL vsb pointer", fun)); KASSERT(s->magic == VSB_MAGIC, - ("%s called with an bogus vsb pointer", fun)); + ("%s called with a bogus vsb pointer", fun)); KASSERT(s->s_buf != NULL, ("%s called with uninitialized or corrupt vsb", fun)); KASSERT(s->s_len < s->s_size, From dridi at varni.sh Wed Sep 11 12:35:51 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 11 Sep 2024 12:35:51 +0000 Subject: [master] 2c3e4a148 spelling: gzipped In-Reply-To: References: <20240819124307.A2C1811DFB6@lists.varnish-cache.org> Message-ID: On Tue, Sep 10, 2024 at 2:15?PM Dridi Boukelmoune wrote: > > On Mon, Aug 19, 2024 at 1:05?PM Poul-Henning Kamp wrote: > > > > > > commit 2c3e4a1485e5bf6a460c3d8ab272ee9888bbfaf1 > > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > > Date: Wed Aug 7 08:30:57 2024 -0400 > > > > spelling: gzipped > > > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > > > diff --git a/include/tbl/obj_attr.h b/include/tbl/obj_attr.h > > index f9441b17a..dbbb2295d 100644 > > --- a/include/tbl/obj_attr.h > > +++ b/include/tbl/obj_attr.h > > @@ -56,7 +56,7 @@ > > > > #ifdef OBJ_FLAG > > /* upper, lower, val */ > > - OBJ_FLAG(GZIPED, gziped, (1<<1)) > > + OBJ_FLAG(GZIPED, gzipped, (1<<1)) > > We don't appear to use the lowercase token anywhere, but is it a good > idea to desync? I didn't touch this one in my post-spelling sweeps, but I'd rather revert it. From dridi at varni.sh Wed Sep 11 18:14:03 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 11 Sep 2024 18:14:03 +0000 Subject: [master] 23abba252 chore(m4): spelling: reentrant In-Reply-To: <20240819124105.853C411CD6F@lists.varnish-cache.org> References: <20240819124105.853C411CD6F@lists.varnish-cache.org> Message-ID: On Mon, Aug 19, 2024 at 12:50?PM Poul-Henning Kamp wrote: > > > commit 23abba252288e7396ab02869ecc2a02115a7d5e1 > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > Date: Wed Aug 7 08:31:00 2024 -0400 > > chore(m4): spelling: reentrant > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > diff --git a/m4/ax_pthread.m4 b/m4/ax_pthread.m4 > index 9f35d1391..39a7a5ed4 100644 > --- a/m4/ax_pthread.m4 > +++ b/m4/ax_pthread.m4 > @@ -244,7 +244,7 @@ AS_IF([test "x$ax_pthread_clang" = "xyes"], > [ax_pthread_flags="-pthread,-lpthread -pthread"]) > > > -# The presence of a feature test macro requesting re-entrant function > +# The presence of a feature test macro requesting reentrant function This one is not technically ours... > # definitions is, on some systems, a strong hint that pthreads support is > # correctly enabled > > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From nils.goroll at uplex.de Thu Sep 12 07:27:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Sep 2024 07:27:06 +0000 (UTC) Subject: [master] e8dac1ab3 Start skeleton release notes for the next version. Message-ID: <20240912072706.D5B1F7206@lists.varnish-cache.org> commit e8dac1ab367f4e5b3e58b923ae96a06cbfcfccc0 Author: Geoff Simmons Date: Tue Sep 25 16:31:17 2018 +0200 Start skeleton release notes for the next version. Restructured so that: * 'Upgrading' is limited to work that has to be done to upgrade from a current deployment to the new version. * 'Changes' is a comprehensive, user-level description of changes and new features. Conflicts: doc/sphinx/whats-new/index.rst diff --git a/doc/sphinx/whats-new/changes-trunk.rst b/doc/sphinx/whats-new/changes-trunk.rst new file mode 100644 index 000000000..2070fadd5 --- /dev/null +++ b/doc/sphinx/whats-new/changes-trunk.rst @@ -0,0 +1,69 @@ +.. _whatsnew_changes_CURRENT: + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +Changes in Varnish **${NEXT_RELEASE}** +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +For information about updating your current Varnish deployment to the +new version, see :ref:`whatsnew_upgrading_CURRENT`. + +A more detailed and technical account of changes in Varnish, with +links to issues that have been fixed and pull requests that have been +merged, may be found in the `change log`_. + +.. _change log: https://github.com/varnishcache/varnish-cache/blob/master/doc/changes.rst + +varnishd +======== + +Parameters +~~~~~~~~~~ + +**XXX changes in -p parameters** + +Other changes in varnishd +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Changes to VCL +============== + +VCL variables +~~~~~~~~~~~~~ + +**XXX new, deprecated or removed variables, or changed semantics** + +Other changes to VCL +~~~~~~~~~~~~~~~~~~~~ + +VMODs +===== + +**XXX changes in the bundled VMODs** + +varnishlog +========== + +**XXX changes concerning varnishlog(1) and/or vsl(7)** + +varnishadm +========== + +**XXX changes concerning varnishadm(1) and/or varnish-cli(7)** + +varnishstat +=========== + +**XXX changes concerning varnishstat(1) and/or varnish-counters(7)** + +varnishtest +=========== + +**XXX changes concerning varnishtest(1) and/or vtc(7)** + +Changes for developers and VMOD authors +======================================= + +**XXX changes concerning VRT, the public APIs, source code organization, +builds etc.** + +*eof* diff --git a/doc/sphinx/whats-new/index.rst b/doc/sphinx/whats-new/index.rst index 51a6b97bd..b275e2bc9 100644 --- a/doc/sphinx/whats-new/index.rst +++ b/doc/sphinx/whats-new/index.rst @@ -13,13 +13,22 @@ This section describes the changes and improvements between different versions of Varnish, and what upgrading between the different versions entail. -Varnish **7.5** +Varnish **$NEXT_RELEASE** ------------------------- **Note: These are working documents for a future release, with running updates for changes in the development branch. For changes in the released versions of Varnish, see the chapters listed below.** +.. toctree:: + :maxdepth: 2 + + changes-trunk + upgrading-trunk + +Varnish **7.5** +--------------- + .. toctree:: :maxdepth: 2 diff --git a/doc/sphinx/whats-new/upgrading-trunk.rst b/doc/sphinx/whats-new/upgrading-trunk.rst new file mode 100644 index 000000000..6143fde99 --- /dev/null +++ b/doc/sphinx/whats-new/upgrading-trunk.rst @@ -0,0 +1,33 @@ +**Note: This is a working document for a future release, with running +updates for changes in the development branch. For changes in the +released versions of Varnish, see:** :ref:`whats-new-index` + +.. _whatsnew_upgrading_CURRENT: + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +Upgrading to Varnish **$NEXT_RELEASE** +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +**XXX: how to upgrade from previous deployments to this +version. Limited to work that has to be done for an upgrade, new +features are listed in "Changes". Explicitly mention what does *not* +have to be changed, especially in VCL. May include, but is not limited +to:** + +* Elements of VCL that have been removed or are deprecated, or whose + semantics have changed. + +* -p parameters that have been removed or are deprecated, or whose + semantics have changed. + +* Changes in the CLI. + +* Changes in the output or interpretation of stats or the log, including + changes affecting varnishncsa/-hist/-top. + +* Changes that may be necessary in VTCs or in the use of varnishtest. + +* Changes in public APIs that may require changes in VMODs or VAPI/VUT + clients. + +*eof* From dridi.boukelmoune at gmail.com Thu Sep 12 08:17:05 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Thu, 12 Sep 2024 08:17:05 +0000 (UTC) Subject: [master] 90a59a742 jail: Code style cleanup in the Linux department Message-ID: <20240912081705.830F162004@lists.varnish-cache.org> commit 90a59a742dc5bd0e42b814896d3e8f0194b31c12 Author: Dridi Boukelmoune Date: Thu Sep 12 10:15:05 2024 +0200 jail: Code style cleanup in the Linux department diff --git a/bin/varnishd/mgt/mgt_jail_linux.c b/bin/varnishd/mgt/mgt_jail_linux.c index 10243049c..93e5c2a6d 100644 --- a/bin/varnishd/mgt/mgt_jail_linux.c +++ b/bin/varnishd/mgt/mgt_jail_linux.c @@ -48,15 +48,24 @@ #include "mgt/mgt.h" #include "common/heritage.h" -static int vjl_init(char **args) { +static int +vjl_init(char **args) +{ + return jail_tech_unix.init(args); } -static void vjl_master(enum jail_master_e jme) { +static void +vjl_master(enum jail_master_e jme) +{ + jail_tech_unix.master(jme); } -static void vjl_subproc(enum jail_subproc_e jse) { +static void +vjl_subproc(enum jail_subproc_e jse) +{ + jail_tech_unix.subproc(jse); /* * On linux mucking about with uid/gid disables core-dumps, @@ -68,15 +77,21 @@ static void vjl_subproc(enum jail_subproc_e jse) { } } -static int vjl_make_subdir(const char *dname, const char *what, struct vsb *vsb) { +static int +vjl_make_subdir(const char *dname, const char *what, struct vsb *vsb) +{ + return jail_tech_unix.make_subdir(dname, what, vsb); } -static int vjl_make_workdir(const char *dname, const char *what, struct vsb *vsb) { +static int +vjl_make_workdir(const char *dname, const char *what, struct vsb *vsb) +{ struct statfs info; if (jail_tech_unix.make_workdir(dname, what, vsb) != 0) return (1); + vjl_master(JAIL_MASTER_FILE); if (statfs(dname, &info) != 0) { if (vsb) @@ -95,7 +110,10 @@ static int vjl_make_workdir(const char *dname, const char *what, struct vsb *vsb return (0); } -static void vjl_fixfd(int fd, enum jail_fixfd_e what) { +static void +vjl_fixfd(int fd, enum jail_fixfd_e what) +{ + jail_tech_unix.fixfd(fd, what); } From nils.goroll at uplex.de Thu Sep 12 08:39:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Sep 2024 08:39:05 +0000 (UTC) Subject: [master] c9f7fa631 changelog polish Message-ID: <20240912083905.576A762EB8@lists.varnish-cache.org> commit c9f7fa63122233cab75240020a4b6c738c5e36da Author: Nils Goroll Date: Thu Sep 12 10:04:37 2024 +0200 changelog polish diff --git a/doc/changes.rst b/doc/changes.rst index 60c4559f1..016688146 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -97,8 +97,8 @@ Varnish Cache NEXT (2024-09-15) added to ``libvarnish``. * The ``Warning: mlock() of VSM failed`` message is now emitted when locking of - shared memory segments (via ``mlock(2)``) fails. As Varnish performance may - severely be impacted if shared memory segments are not resident in RAM, users + shared memory segments (via ``mlock(2)``) fails. As Varnish performance may be + severely impacted if shared memory segments are not resident in RAM, users seeing this message are urged to review the ``RLIMIT_MEMLOCK`` resource control as set via ``ulimit -l`` or ``LimitMEMLOCK`` with ``systemd(1)``. @@ -111,9 +111,9 @@ Varnish Cache NEXT (2024-09-15) * An issue has been addressed in the ``builtin.vcl`` where backend responses would fail if they contained a ``Content-Range`` header when no range was requested. According to `RFC9110`_, this header should just be ignored, yet - some Varnish-Users might prefer stricter checks. Thus, we decided to change + some Varnish users might prefer stricter checks. Thus, we decided to change the ``builtin.vcl`` only and users hitting this issue are advised to call - ``vcl_beresp_range`` in custom VCL. + ``vcl_beresp_range`` from custom VCL. * Additional ``SessError`` VSL events are now generated for various HTTP/2 protocol errors. Some HTTP/2 log events have been changed from ``Debug`` and @@ -142,14 +142,14 @@ Varnish Cache NEXT (2024-09-15) * Backend tasks can now be instructed to queue if the backend has reached its ``max_connections``. This allows tasks to wait for a connection to become - available rather than immediately failing. This feature must be enabled - through new global parameters or individual backend properties: + available rather than immediately fail. This feature must be enabled through + new global parameters or individual backend attributes: * ``backend_wait_timeout`` sets the amount of time a task will wait. * ``backend_wait_limit`` sets the maximum number of tasks that can wait. These parameters can also be set for individual backends using the - ``wait_timeout`` and ``wait_limit`` properties. + ``wait_timeout`` and ``wait_limit`` attributes. Tasks waiting on a backend going sick (either explicitly via the ``backend.set_health`` command or implicitly through the probe) fail @@ -174,12 +174,11 @@ Varnish Cache NEXT (2024-09-15) using ``varnishlog -q``) with numerical values would fail in unexpected ways due to truncation. (`4088`_) -* The ObjWaitExtend() Object API function gained a ``statep`` argument - to optionally return the busy object state consistent with the - current extension. A ``NULL`` value may be passed if the caller does - not require it. +* The ``ObjWaitExtend()`` Object API function gained a ``statep`` argument to + optionally return the busy object state consistent with the current extension. + A ``NULL`` value may be passed if the caller does not require it. -* for backends using the ``.via`` attribute to connect through a +* For backends using the ``.via`` attribute to connect through a *proxy*, the ``connect_timeout``, ``first_byte_timeout`` and ``between_bytes_timeout`` attributes are now inherited from *proxy* unless explicitly given. From nils.goroll at uplex.de Thu Sep 12 08:39:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Sep 2024 08:39:05 +0000 (UTC) Subject: [master] 53155cc7b Add release documentation Message-ID: <20240912083905.6F3C662EBB@lists.varnish-cache.org> commit 53155cc7bf56fe3b3ff8de69328b5bc24f93ea75 Author: Nils Goroll Date: Thu Sep 12 10:37:43 2024 +0200 Add release documentation diff --git a/doc/sphinx/reference/vmod.rst b/doc/sphinx/reference/vmod.rst index 367d7e8ac..e8de061ec 100644 --- a/doc/sphinx/reference/vmod.rst +++ b/doc/sphinx/reference/vmod.rst @@ -859,6 +859,8 @@ You should call ``VSC_*_New()`` when your VMOD is loaded and ``VSC_*.h`` file for the full details about the structure that contains your counters. +.. _ref-vmod-tmpdir: + Temporary Files =============== diff --git a/doc/sphinx/whats-new/changes-trunk.rst b/doc/sphinx/whats-new/changes-trunk.rst index 2070fadd5..45d099559 100644 --- a/doc/sphinx/whats-new/changes-trunk.rst +++ b/doc/sphinx/whats-new/changes-trunk.rst @@ -13,57 +13,152 @@ merged, may be found in the `change log`_. .. _change log: https://github.com/varnishcache/varnish-cache/blob/master/doc/changes.rst +Changes applying to most varnish-cache programs +=============================================== + +The environment variable ``VARNISH_DEFAULT_N`` now provides the default "varnish +name" / "workdir" as otherwise specified by he ``-n`` argument to ``varnishd`` +and ``varnish*`` utilities except ``varnishtest``. + varnishd ======== -Parameters -~~~~~~~~~~ +A new ``linux`` jail has been added (configured via the ``-j`` argument) which is +now the default on Linux. For now, it is almost identical to the ``unix`` jail +with one :ref:`whatsnew_upgrading_CURRENT_linux_jail` added. -**XXX changes in -p parameters** +The port of a *listen_endpoint* given with the ``-a`` argument to ``varnishd`` +can now also be a numerical port range like ``80-89``, besides the existing +options of port number (e.g. ``80``) and service name (e.g. ``http``). With a +port range, Varnish will accept connections on all ports within the range. -Other changes in varnishd -~~~~~~~~~~~~~~~~~~~~~~~~~ +.. _whatsnew_changes_CURRENT_connq: -Changes to VCL -============== +Backend connection queuing +~~~~~~~~~~~~~~~~~~~~~~~~~~ -VCL variables -~~~~~~~~~~~~~ +A feature has been added to instruct backend tasks to queue if the backend has +reached its ``max_connections``. This allows tasks to wait for a connection to +become available rather than immediately fail. This feature must be enabled +through new global parameters or individual backend properties: -**XXX new, deprecated or removed variables, or changed semantics** +* ``backend_wait_timeout`` sets the amount of time a task will wait. +* ``backend_wait_limit`` sets the maximum number of tasks that can wait. -Other changes to VCL -~~~~~~~~~~~~~~~~~~~~ +These parameters can also be set for individual backends using the +``wait_timeout`` and ``wait_limit`` properties. -VMODs -===== +Tasks waiting on a backend going sick (either explicitly via the +``backend.set_health`` command or implicitly through the probe) fail +immediately. -**XXX changes in the bundled VMODs** +Global VSC counters have been added under ``MAIN``: -varnishlog -========== +* ``backend_wait`` counts tasks which waited in queue for a connection. +* ``backend_wait_fail`` counts tasks which waited in queue but failed because + ``wait_timeout`` was reached or the backend went sick. + +Parameters +~~~~~~~~~~ + +The ``backend_wait_timeout`` and ``backend_wait_limit`` parameters have been +added, see :ref:`whatsnew_changes_CURRENT_connq` above for details. -**XXX changes concerning varnishlog(1) and/or vsl(7)** +The size of the buffer to hold panic messages is now tunable through the new +``panic_buffer`` parameter. -varnishadm +Changes to VCL +============== + +The ``wait_timeout`` and ``wait_limit`` backend properties have been added, see +:ref:`whatsnew_changes_CURRENT_connq` above for details. + +For backends using the ``.via`` attribute to connect through a *proxy*, the +``connect_timeout``, ``first_byte_timeout`` and ``between_bytes_timeout`` +attributes are now inherited from *proxy* unless explicitly given. + +varnishlog ========== -**XXX changes concerning varnishadm(1) and/or varnish-cli(7)** +Additional ``SessError`` VSL events are now generated for various HTTP/2 +protocol errors. Some HTTP/2 log events have been changed from ``Debug`` and +``Error`` to ``SessError``. varnishstat =========== -**XXX changes concerning varnishstat(1) and/or varnish-counters(7)** +VSC counters for waiters have been added: + +* ``conns`` to count waits on idle connections +* ``remclose`` to count idle connections closed by the peer +* ``timeout`` to count idle connections which timed out in the waiter +* ``action`` to count idle connections which resulted in a read + +These can be found under ``WAITER..``. + +The ``MAIN.backend_wait`` and ``MAIN.backend_wait_fail`` counters have been +added, see :ref:`whatsnew_changes_CURRENT_connq` above for details. varnishtest =========== -**XXX changes concerning varnishtest(1) and/or vtc(7)** +``varnishtest`` now supports the ``shutdown`` command corresponding to the +``shutdown(2)`` standard C library call. Changes for developers and VMOD authors ======================================= -**XXX changes concerning VRT, the public APIs, source code organization, -builds etc.** +.. _whatsnew_changes_CURRENT_VDP: + +VDP filter API changes +~~~~~~~~~~~~~~~~~~~~~~ + +The Varnish Delivery Processor (VDP) filter API has been generalized to also +accommodate future use for backend request bodies: + +``VDP_Init()`` gained a ``struct busyobj *`` argument for use of VDPs on the +backend side, which is mutually exclusive with the existing ``struct req *`` +argument (one of the two needs to be ``NULL``). ``VDP_Init()`` also gained an +``intmax_t *`` pointer, which needs to point to the known content length of the +body data or ``-1`` for "unknown length". Filters can change this value. + +``struct vdp_ctx`` lost the ``req`` member, but gained ``struct objcore *oc``, +``struct http *hp`` and ``intmax_t *clen`` members. The rationale here is that a +VDP should be concerned mainly with transforming body data (for which ``clen`` +is relevant) and optionally changing (from the ``vdp_init_f``) the headers sent +before the body data, for which ``hp`` is intended. Some VDPs also work directly +on a ``struct objcore *``, so ``oc`` is provided to the first VDP in the chain +only. + +Generic VDPs should specifically not access the request or be concerned with the +object. + +Yet special purpose VDPs still can take from ``VRT_CTX`` whatever references +they need in the ``vdp_init_f`` and store them in their private data. + +Consequent to what as been explained above, ``vdp_init_f`` lost its ``struct +objcore *`` argument. + +VDPs with no ``vdp_bytes_f`` function are now supported if the ``vdp_init_f`` +returns a value greater than zero to signify that the filter is not to be added +to the chain. This is useful to support VDPs which only need to work on headers. + +.. _whatsnew_changes_CURRENT_Obj: + +Object API changes +~~~~~~~~~~~~~~~~~~ + +The ``ObjWaitExtend()`` Object API function gained a ``statep`` argument to +optionally return the busy object state consistent with the current extension. +A ``NULL`` value may be passed if the caller does not require it. + +Other changes relevant for developers +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +``VSS_resolver_range()`` as been added to ``libvarnish`` to implement resolution +of port ranges. + +The implementation of the ``transit_buffer`` has now been made the +responsibility of storage engines. *eof* diff --git a/doc/sphinx/whats-new/upgrading-trunk.rst b/doc/sphinx/whats-new/upgrading-trunk.rst index 6143fde99..d286031ab 100644 --- a/doc/sphinx/whats-new/upgrading-trunk.rst +++ b/doc/sphinx/whats-new/upgrading-trunk.rst @@ -8,26 +8,58 @@ released versions of Varnish, see:** :ref:`whats-new-index` Upgrading to Varnish **$NEXT_RELEASE** %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -**XXX: how to upgrade from previous deployments to this -version. Limited to work that has to be done for an upgrade, new -features are listed in "Changes". Explicitly mention what does *not* -have to be changed, especially in VCL. May include, but is not limited -to:** +In general, upgrading from Varnish 7.5 to **$NEXT_RELEASE** should not require any changes +besides the actual upgrade. -* Elements of VCL that have been removed or are deprecated, or whose - semantics have changed. +The changes mentioned below are considered noteworthy nevertheless: -* -p parameters that have been removed or are deprecated, or whose - semantics have changed. +Noteworthy changes when upgrading varnishd +========================================== -* Changes in the CLI. +Warning about failed memory locking +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -* Changes in the output or interpretation of stats or the log, including - changes affecting varnishncsa/-hist/-top. +The ``Warning: mlock() of VSM failed`` message is now emitted when locking of +shared memory segments (via ``mlock(2)``) fails. As Varnish performance may be +severely impacted if shared memory segments are not resident in RAM, users +seeing this message are urged to review the ``RLIMIT_MEMLOCK`` resource control +as set via ``ulimit -l`` or ``LimitMEMLOCK`` with ``systemd(1)``. This is not +new at all, just now the warning has been added to make administrators more +aware. -* Changes that may be necessary in VTCs or in the use of varnishtest. +.. _whatsnew_upgrading_CURRENT_linux_jail: -* Changes in public APIs that may require changes in VMODs or VAPI/VUT - clients. +Warning if tmpfs is not used +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On Linux (when the new ``linux`` jail is used), the ``Working directory not +mounted on tmpfs partition`` warning is now emitted if the working directory is +found to reside on a file system other than ``tmpfs``. While other file systems +are supported (and might be the right choice where administrators understand how +to avoid blocking disk IO while ``varnishd`` is writing to shared memory), +``tmpfs`` is the failsafe option to avoid performance issues. + +Upgrading VCL +============= + +.. _RFC9110: https://www.rfc-editor.org/rfc/rfc9110.html#section-14.4 + +An issue has been addressed in the ``builtin.vcl`` where backend responses +would fail if they contained a ``Content-Range`` header when no range was +requested. According to `RFC9110`_, this header should just be ignored, yet +some Varnish users might prefer stricter checks. Thus, we decided to change +the ``builtin.vcl`` only and users hitting this issue are advised to call +``vcl_beresp_range`` from custom VCL. + +Changes for developers and VMOD authors +======================================= + +The VDP filter API has changed. See :ref:`whatsnew_changes_CURRENT_VDP` for details. + +The signature of ``ObjWaitExtend()`` has changed. See +:ref:`whatsnew_changes_CURRENT_Obj` for details. + +``varnishd`` now creates a ``worker_tmpdir`` which can be used by VMODs for +temporary files. See :ref:`ref-vmod-event-functions` for details. *eof* From nils.goroll at uplex.de Thu Sep 12 09:03:04 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Sep 2024 09:03:04 +0000 (UTC) Subject: [master] c2bc1c31d Coverity polish: 1610758 Unused value Message-ID: <20240912090304.EA5B76551B@lists.varnish-cache.org> commit c2bc1c31d3ead1099cf24e0cefdc5044215a9ee8 Author: Nils Goroll Date: Thu Sep 12 10:49:34 2024 +0200 Coverity polish: 1610758 Unused value diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index a6a31805e..3b4fa2fa8 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -520,7 +520,6 @@ vbp_scheduler(struct worker *wrk, void *priv) (void)Lck_CondWaitUntil(&vbp_cond, &vbp_mtx, nxt); } else if (vt->due > now) { nxt = vt->due; - vt = NULL; (void)Lck_CondWaitUntil(&vbp_cond, &vbp_mtx, nxt); } else { assert(vt->state == vbp_state_scheduled); From nils.goroll at uplex.de Thu Sep 12 09:29:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Sep 2024 09:29:05 +0000 (UTC) Subject: [master] b36272950 Coverity polish: 1605326 Overflowed integer argument Message-ID: <20240912092905.57924100495@lists.varnish-cache.org> commit b36272950440f0bf98240b7da85a9cec36b81b92 Author: Nils Goroll Date: Thu Sep 12 11:26:47 2024 +0200 Coverity polish: 1605326 Overflowed integer argument Turn l into the return type of WS_Dump() to avoid truncation from 64 to 32 bits. diff --git a/vmod/vmod_vtc.c b/vmod/vmod_vtc.c index 6c38db3ae..2d97b3543 100644 --- a/vmod/vmod_vtc.c +++ b/vmod/vmod_vtc.c @@ -248,7 +248,8 @@ vmod_workspace_dump(VRT_CTX, VCL_ENUM which, VCL_ENUM where, VCL_BYTES off, VCL_BYTES len) { struct ws *ws; - VCL_BYTES l, maxlen = 1024; + unsigned l; + const unsigned maxlen = 1024; unsigned char buf[maxlen]; const char *p, *err; @@ -267,6 +268,7 @@ vmod_workspace_dump(VRT_CTX, VCL_ENUM which, VCL_ENUM where, } l = WS_Dump(ws, *where, off, buf, len); + assert(l <= maxlen); if (l == 0) { switch (errno) { @@ -279,8 +281,7 @@ vmod_workspace_dump(VRT_CTX, VCL_ENUM which, VCL_ENUM where, return (NULL); } - assert(l < maxlen); - p = WS_Copy(ctx->ws, buf, l); + p = WS_Copy(ctx->ws, buf, (int)l); if (p == NULL) { VRT_fail(ctx, "workspace_dump: copy failed"); return (NULL); From nils.goroll at uplex.de Thu Sep 12 09:41:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Sep 2024 09:41:05 +0000 (UTC) Subject: [master] a5810d25d Coverity: Explain vtc_logfail() Message-ID: <20240912094105.33ED0100E98@lists.varnish-cache.org> commit a5810d25d00eaaa97f551a52bb03139c4d610cc8 Author: Nils Goroll Date: Thu Sep 12 11:37:55 2024 +0200 Coverity: Explain vtc_logfail() It seems coverity does not grok vtc_logfail() via vtc_fatal() as panic-ish, because it complains about a possible overflow for l in if (signed <= 0) vtc_fatal(...) l -= signed; Ref CID 1605325 diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c index 5fa44e7be..26bb7302e 100644 --- a/bin/varnishtest/vtc_log.c +++ b/bin/varnishtest/vtc_log.c @@ -109,6 +109,13 @@ vtc_logclose(void *arg) FREE_OBJ(vl); } +#ifdef __COVERITY__ +static void v_noreturn_ +vtc_logfail(void) +{ + __coverity_panic__(); +} +#else static void v_noreturn_ vtc_logfail(void) { @@ -119,6 +126,7 @@ vtc_logfail(void) else exit(fail_out()); } +#endif static const char * const lead[] = { "----", From nils.goroll at uplex.de Thu Sep 12 10:49:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Sep 2024 10:49:05 +0000 (UTC) Subject: [master] cbb8bb437 Coverity polish: 1605323 Overflowed array index write Message-ID: <20240912104905.44F4810378E@lists.varnish-cache.org> commit cbb8bb437b73ab51dedc78c4153a7243800d61d0 Author: Nils Goroll Date: Thu Sep 12 12:48:16 2024 +0200 Coverity polish: 1605323 Overflowed array index write diff --git a/bin/varnishd/mgt/mgt_jail_unix.c b/bin/varnishd/mgt/mgt_jail_unix.c index c6cab790a..8fb6dfc9a 100644 --- a/bin/varnishd/mgt/mgt_jail_unix.c +++ b/bin/varnishd/mgt/mgt_jail_unix.c @@ -215,6 +215,7 @@ vju_subproc(enum jail_subproc_e jse) /* Add the optional extra group for the C-compiler access */ i = getgroups(NGID, gid_list); assert(i >= 0); + assert(i < NGID - 1); gid_list[i++] = vju_cc_gid; AZ(setgroups(i, gid_list)); } From nils.goroll at uplex.de Thu Sep 12 10:55:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Sep 2024 10:55:06 +0000 (UTC) Subject: [master] 0757b3807 Coverity polish: 1605322 Overflowed integer argument Message-ID: <20240912105506.11F90103C5A@lists.varnish-cache.org> commit 0757b380798d4c28b1c70b6a4bb1b12e4abf0530 Author: Nils Goroll Date: Thu Sep 12 12:52:07 2024 +0200 Coverity polish: 1605322 Overflowed integer argument we do want all 32bits of seed to be passed to srand48(), and if the cast results in a a negative number, that's fine. Let's see if Coverity can be appeased this way, otherwise we'll probably need to ignore the report. diff --git a/lib/libvarnish/vrnd.c b/lib/libvarnish/vrnd.c index d72b697ad..87929427b 100644 --- a/lib/libvarnish/vrnd.c +++ b/lib/libvarnish/vrnd.c @@ -196,5 +196,5 @@ VRND_SeedAll(void) AZ(VRND_RandomCrypto(&seed, sizeof seed)); VRND_SeedTestable(seed); AZ(VRND_RandomCrypto(&seed, sizeof seed)); - srand48(seed); + srand48((long)seed); } From nils.goroll at uplex.de Thu Sep 12 10:59:04 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Sep 2024 10:59:04 +0000 (UTC) Subject: [master] c2e533861 Coverity polish: 1605321 Overflowed integer argument Message-ID: <20240912105904.A80B7104003@lists.varnish-cache.org> commit c2e5338618fea1ea359be1acede1d1d96cbfdf4f Author: Nils Goroll Date: Thu Sep 12 12:58:36 2024 +0200 Coverity polish: 1605321 Overflowed integer argument diff --git a/bin/varnishd/http1/cache_http1_pipe.c b/bin/varnishd/http1/cache_http1_pipe.c index dd8b9098b..e361ba020 100644 --- a/bin/varnishd/http1/cache_http1_pipe.c +++ b/bin/varnishd/http1/cache_http1_pipe.c @@ -49,7 +49,7 @@ static struct lock pipestat_mtx; static int rdf(int fd0, int fd1, uint64_t *pcnt) { - int i, j; + ssize_t i, j; char buf[BUFSIZ], *p; i = read(fd0, buf, sizeof buf); From nils.goroll at uplex.de Thu Sep 12 12:05:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Sep 2024 12:05:05 +0000 (UTC) Subject: [master] 8c72c416d Coverity polish: 1605317 Overflowed constant Message-ID: <20240912120505.B127A106293@lists.varnish-cache.org> commit 8c72c416d5c5d5e2791fd39272edf2084b0a903a Author: Nils Goroll Date: Thu Sep 12 14:01:19 2024 +0200 Coverity polish: 1605317 Overflowed constant do not count backend removals for hinting the size of the array diff --git a/vmod/vmod_directors_shard_cfg.c b/vmod/vmod_directors_shard_cfg.c index 1a9e9fcbe..5f0c2f559 100644 --- a/vmod/vmod_directors_shard_cfg.c +++ b/vmod/vmod_directors_shard_cfg.c @@ -553,7 +553,6 @@ shardcfg_apply_change(struct vsl_log *vsl, struct sharddir *shardd, re.hint++; break; case REMOVE_BE: - re.hint--; break; default: INCOMPL(); From nils.goroll at uplex.de Thu Sep 12 12:09:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Sep 2024 12:09:06 +0000 (UTC) Subject: [master] 33d8a53d4 Coverity polish: 1605316 Overflowed constant Message-ID: <20240912120906.603F31066EA@lists.varnish-cache.org> commit 33d8a53d4e4bc43edd8fe5fd8cfd9bc11db078c8 Author: Nils Goroll Date: Thu Sep 12 14:06:48 2024 +0200 Coverity polish: 1605316 Overflowed constant Add assertion that we only process wait events which we have counted, anything else is a bug. diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c index 70b71c07d..fb33c1199 100644 --- a/bin/varnishd/waiter/cache_waiter_epoll.c +++ b/bin/varnishd/waiter/cache_waiter_epoll.c @@ -105,6 +105,7 @@ vwe_thread(void *priv) } CHECK_OBJ_NOTNULL(wp, WAITED_MAGIC); AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_DEL, wp->fd, NULL)); + AN(vwe->nwaited); vwe->nwaited--; AN(Wait_HeapDelete(w, wp)); Lck_Unlock(&vwe->mtx); @@ -138,6 +139,7 @@ vwe_thread(void *priv) continue; } AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_DEL, wp->fd, NULL)); + AN(vwe->nwaited); vwe->nwaited--; if (ep->events & EPOLLIN) { if (ep->events & EPOLLRDHUP && From nils.goroll at uplex.de Thu Sep 12 12:30:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Sep 2024 12:30:06 +0000 (UTC) Subject: [master] 3e8c1812d Coverity polish: 1605314 Overflowed return value Message-ID: <20240912123006.1E2911072CA@lists.varnish-cache.org> commit 3e8c1812deb8a533c57130d39c4aa04e516f60c0 Author: Nils Goroll Date: Thu Sep 12 14:25:09 2024 +0200 Coverity polish: 1605314 Overflowed return value Coverity seems to not understand that read will not return more than len, and thus i + l can never be greated than len. Try to help it... diff --git a/bin/varnishd/http1/cache_http1_vfp.c b/bin/varnishd/http1/cache_http1_vfp.c index d10df7ac5..20f349d1c 100644 --- a/bin/varnishd/http1/cache_http1_vfp.c +++ b/bin/varnishd/http1/cache_http1_vfp.c @@ -81,9 +81,9 @@ v1f_read(const struct vfp_ctx *vc, struct http_conn *htc, void *d, ssize_t len) TOSTRAND(VAS_errtxt(errno))); return (i); } + assert(i <= len); if (i == 0) htc->doclose = SC_RESP_CLOSE; - } return (i + l); } From nils.goroll at uplex.de Thu Sep 12 12:56:04 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Sep 2024 12:56:04 +0000 (UTC) Subject: [master] a2482ced6 Coverity polish: 1605312 Overflowed integer argument Message-ID: <20240912125605.07A441081CE@lists.varnish-cache.org> commit a2482ced6b039d35dc95e3a3e6034f84c3d9e9cc Author: Nils Goroll Date: Thu Sep 12 14:43:37 2024 +0200 Coverity polish: 1605312 Overflowed integer argument another case where apparently it does not understand the count argument of read(2) diff --git a/lib/libvarnish/vlu.c b/lib/libvarnish/vlu.c index 8883a350c..c79c65603 100644 --- a/lib/libvarnish/vlu.c +++ b/lib/libvarnish/vlu.c @@ -123,14 +123,18 @@ LineUpProcess(struct vlu *l) int VLU_Fd(struct vlu *l, int fd) { - int i; + ssize_t i; + size_t sz; CHECK_OBJ_NOTNULL(l, LINEUP_MAGIC); - i = read(fd, l->buf + l->bufp, l->bufl - l->bufp); + assert(l->bufl >= l->bufp); + sz = l->bufl - l->bufp; + i = read(fd, l->buf + l->bufp, sz); if (i == 0) return (-2); if (i < 0) return (-1); + assert(i <= sz); l->bufp += i; return (LineUpProcess(l)); } From nils.goroll at uplex.de Thu Sep 12 13:11:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Sep 2024 13:11:06 +0000 (UTC) Subject: [master] 3862b9d35 Flexelint previous commit Message-ID: <20240912131106.56F96108B48@lists.varnish-cache.org> commit 3862b9d3509de8c197ffae64d4ca5b1a84ee17da Author: Nils Goroll Date: Thu Sep 12 15:10:26 2024 +0200 Flexelint previous commit diff --git a/lib/libvarnish/vlu.c b/lib/libvarnish/vlu.c index c79c65603..c3c15f273 100644 --- a/lib/libvarnish/vlu.c +++ b/lib/libvarnish/vlu.c @@ -134,7 +134,7 @@ VLU_Fd(struct vlu *l, int fd) return (-2); if (i < 0) return (-1); - assert(i <= sz); + assert((size_t)i <= sz); l->bufp += i; return (LineUpProcess(l)); } From nils.goroll at uplex.de Thu Sep 12 13:33:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Sep 2024 13:33:05 +0000 (UTC) Subject: [master] 5353deb8a varnishd.1: Elaborate on the Linux jail Message-ID: <20240912133305.4485410981D@lists.varnish-cache.org> commit 5353deb8a8b432d58081023999c62b350810b574 Author: Thibaut Artis Date: Wed Sep 11 17:22:01 2024 +0200 varnishd.1: Elaborate on the Linux jail diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 1ca94c7a5..13dab58f2 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -456,8 +456,11 @@ specific options. Available jails are: -j - Default on Linux platforms, it overloads the UNIX jail with - Linux-specific mechanisms. + Default on Linux platforms, it extends the UNIX jail with + Linux-specific mechanisms: + + - warn when *workdir* is not in a ``tmpfs`` + - try keeping the process dumpable after dropping privileges -j From nils.goroll at uplex.de Thu Sep 12 13:33:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Sep 2024 13:33:05 +0000 (UTC) Subject: [master] 5a6d61888 Slightly polish previous commit Message-ID: <20240912133305.5A9A6109821@lists.varnish-cache.org> commit 5a6d61888b09653df8717b061991dec5ac367785 Author: Nils Goroll Date: Thu Sep 12 15:31:13 2024 +0200 Slightly polish previous commit diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 13dab58f2..a3c445d14 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -459,8 +459,8 @@ specific options. Available jails are: Default on Linux platforms, it extends the UNIX jail with Linux-specific mechanisms: - - warn when *workdir* is not in a ``tmpfs`` - - try keeping the process dumpable after dropping privileges + - It warns when *workdir* is not on a ``tmpfs``. + - It tries to keep the process dumpable after dropping privileges. -j From nils.goroll at uplex.de Thu Sep 12 14:02:04 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Sep 2024 14:02:04 +0000 (UTC) Subject: [master] 0b938f2a1 Revert "spelling: gzipped" Message-ID: <20240912140204.F2EDF10AB60@lists.varnish-cache.org> commit 0b938f2a1f91947cb0547067ce98488af93c7f21 Author: Nils Goroll Date: Thu Sep 12 15:58:49 2024 +0200 Revert "spelling: gzipped" In accordance with a -commit discussion before we forget: if this was changed, both the upper- and lowercase variants should be changed, but that would break modules/extensions for no good reason. This reverts commit 2c3e4a1485e5bf6a460c3d8ab272ee9888bbfaf1. diff --git a/include/tbl/obj_attr.h b/include/tbl/obj_attr.h index dbbb2295d..f9441b17a 100644 --- a/include/tbl/obj_attr.h +++ b/include/tbl/obj_attr.h @@ -56,7 +56,7 @@ #ifdef OBJ_FLAG /* upper, lower, val */ - OBJ_FLAG(GZIPED, gzipped, (1<<1)) + OBJ_FLAG(GZIPED, gziped, (1<<1)) OBJ_FLAG(CHGCE, chgce, (1<<2)) OBJ_FLAG(IMSCAND, imscand, (1<<3)) OBJ_FLAG(ESIPROC, esiproc, (1<<4)) From dridi at varni.sh Thu Sep 12 14:13:02 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Thu, 12 Sep 2024 14:13:02 +0000 Subject: [master] 0b938f2a1 Revert "spelling: gzipped" In-Reply-To: <20240912140204.F2EDF10AB60@lists.varnish-cache.org> References: <20240912140204.F2EDF10AB60@lists.varnish-cache.org> Message-ID: On Thu, Sep 12, 2024 at 2:10?PM Nils Goroll wrote: > > > commit 0b938f2a1f91947cb0547067ce98488af93c7f21 > Author: Nils Goroll > Date: Thu Sep 12 15:58:49 2024 +0200 > > Revert "spelling: gzipped" > > In accordance with a -commit discussion before we forget: if this was changed, > both the upper- and lowercase variants should be changed, but that would break > modules/extensions for no good reason. Not sure about third parties, but I mentioned in the original commit that we are not (or no longer?) using the lowercase flag names. I think I also mentioned that I was in favor of reverting, so thanks for taking care of it. > This reverts commit 2c3e4a1485e5bf6a460c3d8ab272ee9888bbfaf1. > > diff --git a/include/tbl/obj_attr.h b/include/tbl/obj_attr.h > index dbbb2295d..f9441b17a 100644 > --- a/include/tbl/obj_attr.h > +++ b/include/tbl/obj_attr.h > @@ -56,7 +56,7 @@ > > #ifdef OBJ_FLAG > /* upper, lower, val */ > - OBJ_FLAG(GZIPED, gzipped, (1<<1)) > + OBJ_FLAG(GZIPED, gziped, (1<<1)) > OBJ_FLAG(CHGCE, chgce, (1<<2)) > OBJ_FLAG(IMSCAND, imscand, (1<<3)) > OBJ_FLAG(ESIPROC, esiproc, (1<<4)) > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From nils.goroll at uplex.de Thu Sep 12 14:14:19 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Sep 2024 16:14:19 +0200 Subject: Dridi the spellcheck checker In-Reply-To: References: <20240819124712.05AF3120ADB@lists.varnish-cache.org> Message-ID: <33c5a042-5303-4b94-8650-1e58c50ca705@uplex.de> Thank you, Dridi, for once again being pedantic in a good way :) FTR, when the spell check commits happened I should have looked closer, but yeah, the whole thing somehow went a bit too fast... I will consolidate two replies into one email: >> @@ -25,7 +25,7 @@ keep a place ready for it in the overall architecture. >> >> Now a credible use-case finally appeared, and we have implemented >> "Varnish Extensions" (VTLA: "VEXT"), which can both be used to load >> -ambient VMODs and to implement entirely new functionaly, for instance >> +ambient VMODs and to implement entirely new functionally, for instance > > Isn't the correct word functionality here? Yes. On 10.09.24 16:23, Dridi Boukelmoune wrote: >> For more than one invocation of ``VRT_call()``, VMODs*must* >> - check if ``VRT_handled()`` returns non-zero inbetween calls: >> + check if ``VRT_handled()`` returns non-zero in between calls: > Isn't the correct spelling in-between in this context? I would think so, too. Nils -- Nils Goroll (he/him) ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ From dridi at varni.sh Thu Sep 12 14:18:37 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Thu, 12 Sep 2024 14:18:37 +0000 Subject: Dridi the spellcheck checker In-Reply-To: <33c5a042-5303-4b94-8650-1e58c50ca705@uplex.de> References: <20240819124712.05AF3120ADB@lists.varnish-cache.org> <33c5a042-5303-4b94-8650-1e58c50ca705@uplex.de> Message-ID: On Thu, Sep 12, 2024 at 2:14?PM Nils Goroll wrote: > > Thank you, Dridi, for once again being pedantic in a good way :) > > FTR, when the spell check commits happened I should have looked closer, but > yeah, the whole thing somehow went a bit too fast... Regardless of how a change was committed, having this mailing list is very valuable to get a chance to review after the facts! > I will consolidate two replies into one email: > > >> @@ -25,7 +25,7 @@ keep a place ready for it in the overall architecture. > >> > >> Now a credible use-case finally appeared, and we have implemented > >> "Varnish Extensions" (VTLA: "VEXT"), which can both be used to load > >> -ambient VMODs and to implement entirely new functionaly, for instance > >> +ambient VMODs and to implement entirely new functionally, for instance > > > > Isn't the correct word functionality here? > > Yes. > > On 10.09.24 16:23, Dridi Boukelmoune wrote: > >> For more than one invocation of ``VRT_call()``, VMODs*must* > >> - check if ``VRT_handled()`` returns non-zero inbetween calls: > >> + check if ``VRT_handled()`` returns non-zero in between calls: > > Isn't the correct spelling in-between in this context? > > I would think so, too. I already fixed both and then more ;) Dridi From nils.goroll at uplex.de Thu Sep 12 14:22:31 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Sep 2024 16:22:31 +0200 Subject: Dridi the spellcheck checker In-Reply-To: References: <20240819124712.05AF3120ADB@lists.varnish-cache.org> <33c5a042-5303-4b94-8650-1e58c50ca705@uplex.de> Message-ID: <70d393a4-edfd-416f-875a-00d2acd32376@uplex.de> > Regardless of how a change was committed, having this mailing list is > very valuable to get a chance to review after the facts! Very true. As always, Email is very much underrated. > I already fixed both and then more ;) You did fix functionality "and then more", yes, but not "inbetween". Sorry, I threw both into one basket. Nils -- Nils Goroll (he/him) ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ From dridi at varni.sh Thu Sep 12 14:36:21 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Thu, 12 Sep 2024 14:36:21 +0000 Subject: Dridi the spellcheck checker In-Reply-To: <70d393a4-edfd-416f-875a-00d2acd32376@uplex.de> References: <20240819124712.05AF3120ADB@lists.varnish-cache.org> <33c5a042-5303-4b94-8650-1e58c50ca705@uplex.de> <70d393a4-edfd-416f-875a-00d2acd32376@uplex.de> Message-ID: On Thu, Sep 12, 2024 at 2:22?PM Nils Goroll wrote: > > > Regardless of how a change was committed, having this mailing list is > > very valuable to get a chance to review after the facts! > > Very true. As always, Email is very much underrated. > > > I already fixed both and then more ;) > You did fix functionality "and then more", yes, but not "inbetween". Sorry, I > threw both into one basket. Are you sure? > git show --word-diff 4623d25925fb453462bfc27d1412ca944ce2b422 -- doc/sphinx/[rw]*/ I'm seeing both in my first commit. Dridi From nils.goroll at uplex.de Thu Sep 12 14:39:19 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Sep 2024 16:39:19 +0200 Subject: Dridi the spellcheck checker In-Reply-To: References: <20240819124712.05AF3120ADB@lists.varnish-cache.org> <33c5a042-5303-4b94-8650-1e58c50ca705@uplex.de> <70d393a4-edfd-416f-875a-00d2acd32376@uplex.de> Message-ID: <73dcdd87-e3df-4f6e-be01-5600e9008ea7@uplex.de> On 12.09.24 16:36, Dridi Boukelmoune wrote: >> You did fix functionality "and then more", yes, but not "inbetween". Sorry, I >> threw both into one basket. > Are you sure? Ah, you went for in-between, which is why I did not see it. Thank you! -- Nils Goroll (he/him) ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ From dridi.boukelmoune at gmail.com Fri Sep 13 07:52:09 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 13 Sep 2024 07:52:09 +0000 (UTC) Subject: [master] 5caded903 cifuzz: Update upload action to v4 Message-ID: <20240913075209.4D61810DE21@lists.varnish-cache.org> commit 5caded903ccd8e453e1401de3c1abf8e562ae9ec Author: Thibaut Artis Date: Thu Sep 12 14:10:40 2024 +0200 cifuzz: Update upload action to v4 diff --git a/.github/workflows/cifuzz.yml b/.github/workflows/cifuzz.yml index 411fbe2ef..4895b0ffc 100644 --- a/.github/workflows/cifuzz.yml +++ b/.github/workflows/cifuzz.yml @@ -20,7 +20,7 @@ jobs: dry-run: false language: c - name: Upload Crash - uses: actions/upload-artifact at v1 + uses: actions/upload-artifact at v4 if: failure() && steps.build.outcome == 'success' with: name: artifacts From dridi.boukelmoune at gmail.com Fri Sep 13 07:56:05 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 13 Sep 2024 07:56:05 +0000 (UTC) Subject: [master] 922c8a62e vtc_http: Consistency polish Message-ID: <20240913075605.9D65610E154@lists.varnish-cache.org> commit 922c8a62ec5c46b2562c139fcc0415bde7a78ccf Author: Dridi Boukelmoune Date: Fri Sep 13 09:37:09 2024 +0200 vtc_http: Consistency polish Missing dot added and alphabetic order restored. diff --git a/bin/varnishtest/vtc_http.c b/bin/varnishtest/vtc_http.c index 7f27300e8..352054a59 100644 --- a/bin/varnishtest/vtc_http.c +++ b/bin/varnishtest/vtc_http.c @@ -1592,7 +1592,7 @@ cmd_http_accept(CMD_ARGS) * \-read * Shutdown the read direction. * \-write - * Shutdown the write direction + * Shutdown the write direction. * * The default is both direction. */ @@ -1804,12 +1804,12 @@ const struct cmds http_cmds[] = { /* session */ CMD_HTTP(accept) CMD_HTTP(close) - CMD_HTTP(shutdown) CMD_HTTP(recv) CMD_HTTP(send) CMD_HTTP(send_n) CMD_HTTP(send_urgent) CMD_HTTP(sendhex) + CMD_HTTP(shutdown) CMD_HTTP(timeout) /* spec */ From dridi.boukelmoune at gmail.com Fri Sep 13 14:01:06 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 13 Sep 2024 14:01:06 +0000 (UTC) Subject: [master] 8f11d018f Merge tag 'varnish-7.6.0' Message-ID: <20240913140106.AAA4D11A03C@lists.varnish-cache.org> commit 8f11d018f2d6b9e2b811a83ead706c49dd440b80 Merge: 168ebabab ed1243ca1 Author: Dridi Boukelmoune Date: Fri Sep 13 15:55:59 2024 +0200 Merge tag 'varnish-7.6.0' From simon.stridsberg at varnish-software.com Fri Sep 13 14:53:07 2024 From: simon.stridsberg at varnish-software.com (Simon Stridsberg) Date: Fri, 13 Sep 2024 14:53:07 +0000 (UTC) Subject: [7.6] 986267c47 Build from 7.6 pkg-varnish-cache branch Message-ID: <20240913145307.BB71111C082@lists.varnish-cache.org> commit 986267c4732fc394f7cd7bc878abb7a53f665e15 Author: Simon Stridsberg Date: Fri Sep 13 16:52:15 2024 +0200 Build from 7.6 pkg-varnish-cache branch diff --git a/.circleci/config.yml b/.circleci/config.yml index 185e76fe2..412431a51 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ parameters: default: "HEAD" pkg-commit: type: string - default: "master" + default: "7.6" dist-url: type: string default: "" From dridi at varni.sh Mon Sep 16 12:30:53 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 16 Sep 2024 12:30:53 +0000 Subject: [master] 2c3e4a148 spelling: gzipped In-Reply-To: References: <20240819124307.A2C1811DFB6@lists.varnish-cache.org> Message-ID: On Wed, Sep 11, 2024 at 12:35?PM Dridi Boukelmoune wrote: > > On Tue, Sep 10, 2024 at 2:15?PM Dridi Boukelmoune wrote: > > > > On Mon, Aug 19, 2024 at 1:05?PM Poul-Henning Kamp wrote: > > > > > > > > > commit 2c3e4a1485e5bf6a460c3d8ab272ee9888bbfaf1 > > > Author: Josh Soref <2119212+jsoref at users.noreply.github.com> > > > Date: Wed Aug 7 08:30:57 2024 -0400 > > > > > > spelling: gzipped > > > > > > Signed-off-by: Josh Soref <2119212+jsoref at users.noreply.github.com> > > > > > > diff --git a/include/tbl/obj_attr.h b/include/tbl/obj_attr.h > > > index f9441b17a..dbbb2295d 100644 > > > --- a/include/tbl/obj_attr.h > > > +++ b/include/tbl/obj_attr.h > > > @@ -56,7 +56,7 @@ > > > > > > #ifdef OBJ_FLAG > > > /* upper, lower, val */ > > > - OBJ_FLAG(GZIPED, gziped, (1<<1)) > > > + OBJ_FLAG(GZIPED, gzipped, (1<<1)) > > > > We don't appear to use the lowercase token anywhere, but is it a good > > idea to desync? > > I didn't touch this one in my post-spelling sweeps, but I'd rather revert it. For reference: https://github.com/varnishcache/varnish-cache/commit/0b938f2a1f91947cb0547067ce98488af93c7f21 From nils.goroll at uplex.de Mon Sep 16 13:09:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 16 Sep 2024 13:09:06 +0000 (UTC) Subject: [master] da1d27362 vcc_action: Accept vmod returned strings for return(fail()) Message-ID: <20240916130906.1A6B8117473@lists.varnish-cache.org> commit da1d273629bbfe299c195584f7f8653b167da7e6 Author: Walid Boudebouda Date: Thu Sep 5 18:01:39 2024 +0200 vcc_action: Accept vmod returned strings for return(fail()) Fixes: #4170 diff --git a/bin/varnishtest/tests/v00018.vtc b/bin/varnishtest/tests/v00018.vtc index 7e515e967..3ee4a5221 100644 --- a/bin/varnishtest/tests/v00018.vtc +++ b/bin/varnishtest/tests/v00018.vtc @@ -155,3 +155,13 @@ varnish v1 -errvcl {Expected '.' got ';'} { rr; } } + +varnish v1 -vcl { + backend default none; + + import debug; + + sub vcl_recv { + return (fail(debug.author())); + } +} diff --git a/lib/libvcc/vcc_action.c b/lib/libvcc/vcc_action.c index f7c2ce29f..86141d4a7 100644 --- a/lib/libvcc/vcc_action.c +++ b/lib/libvcc/vcc_action.c @@ -269,7 +269,7 @@ static void vcc_act_return_fail(struct vcc *tl) { SkipToken(tl, '('); - Fb(tl, 1, "VRT_fail(ctx,\n"); + Fb(tl, 1, "VRT_fail(ctx, \"%%s\",\n"); tl->indent += INDENT; vcc_Expr(tl, STRING); tl->indent -= INDENT; From nils.goroll at uplex.de Mon Sep 16 13:19:04 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 16 Sep 2024 13:19:04 +0000 (UTC) Subject: [master] 7b68497c4 obj: List transient flags in a single place Message-ID: <20240916131904.CA6F9117AF4@lists.varnish-cache.org> commit 7b68497c48b1f0381997df5c5f3b52e83e95df9d Author: Dridi Boukelmoune Date: Mon Sep 16 11:16:24 2024 +0200 obj: List transient flags in a single place To avoid repetition and mistakes, and in order to clarify the meaning of certain conditions, a new OC_F_TRANSIENT pseudo-flag is introduced. This was prompted by an incoming change to transit buffer where identifying transient delivery requires again to check the same set of flags. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 10fab7e8d..9c40faffb 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -326,6 +326,8 @@ enum oc_flags { #include "tbl/oc_flags.h" }; +#define OC_F_TRANSIENT (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP) + enum oc_exp_flags { #define OC_EXP_FLAG(U, l, v) OC_EF_##U = v, #include "tbl/oc_exp_flags.h" diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 5321fa06c..f824d2059 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -277,7 +277,7 @@ VDP_DeliverObj(struct vdp_ctx *vdc, struct objcore *oc) AZ(vdc->oc); vdc->hp = NULL; vdc->clen = NULL; - final = oc->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP) ? 1 : 0; + final = oc->flags & OC_F_TRANSIENT ? 1 : 0; r = ObjIterate(vdc->wrk, oc, vdc, vdp_objiterate, final); if (r < 0) return (r); diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 8a1bb400d..6446f3beb 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -831,7 +831,7 @@ HSH_Cancel(struct worker *wrk, struct objcore *oc, struct boc *boc) CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC); - if ((oc->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP)) == 0) + if ((oc->flags & OC_F_TRANSIENT) == 0) return; /* diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index bdf78c162..477b26e2a 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -225,7 +225,7 @@ obj_extend_condwait(const struct objcore *oc) if (oc->boc->transit_buffer == 0) return; - assert(oc->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP)); + assert(oc->flags & OC_F_TRANSIENT); /* NB: strictly signaling progress both ways would be prone to * deadlocks, so instead we wait for signals from the client side * when delivered_so_far so far is updated, but in case the fetch @@ -279,7 +279,7 @@ ObjWaitExtend(const struct worker *wrk, const struct objcore *oc, uint64_t l, rv = oc->boc->fetched_so_far; assert(l <= rv || oc->boc->state == BOS_FAILED); if (oc->boc->transit_buffer > 0) { - assert(oc->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP)); + assert(oc->flags & OC_F_TRANSIENT); /* Signal the new client position */ oc->boc->delivered_so_far = l; PTOK(pthread_cond_signal(&oc->boc->cond)); From nils.goroll at uplex.de Mon Sep 16 13:22:04 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 16 Sep 2024 13:22:04 +0000 (UTC) Subject: [master] 70e87cda7 Minor r'str' fix to vmodtool.py Message-ID: <20240916132204.C611A117DEF@lists.varnish-cache.org> commit 70e87cda78d14f9b03662afe1766d0d9ed77d79b Author: Yuri Astrakhan Date: Fri Sep 13 14:44:05 2024 -0400 Minor r'str' fix to vmodtool.py Python does not like having `\` characters followed by non-special characters, e.g. `\.` - but regex requires that. For this, python has "raw strings", e.g. `r'....'` - this way no `\` have any special meaning to python, only to the regex itself. At least on my 3.12 Python it refused to work without it. diff --git a/lib/libvcc/vmodtool.py b/lib/libvcc/vmodtool.py index 2d3d94640..21d91a7b4 100755 --- a/lib/libvcc/vmodtool.py +++ b/lib/libvcc/vmodtool.py @@ -342,11 +342,11 @@ class ProtoType(): self.retval = CType(['VOID'], st.vcc.enums) self.bname = wl.pop(0) - if not re.match("^[a-zA-Z.][a-zA-Z0-9_]*$", self.bname): + if not re.match(r'^[a-zA-Z.][a-zA-Z0-9_]*$', self.bname): err("%s(): Illegal name\n" % self.bname, warn=False) self.name = prefix + self.bname - if not re.match('^[a-zA-Z_][a-zA-Z0-9_]*$', self.cname()): + if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*$', self.cname()): err("%s(): Illegal C-name\n" % self.cname(), warn=False) if len(wl) == 2 and wl[0] == '(' and wl[1] == ')': @@ -899,10 +899,10 @@ class AliasStanza(Stanza): def parse(self): if len(self.toks) != 3: err("Syntax error, expected: $Alias \n", warn=False) - if not re.match('^\.?[a-zA-Z_][a-zA-Z0-9_]*$', self.toks[1]): + if not re.match(r'^\.?[a-zA-Z_][a-zA-Z0-9_]*$', self.toks[1]): err("%s(): Illegal C-name\n" % self.toks[1], warn=False) if self.toks[1][0] == '.': - if not re.match('^[a-zA-Z_][a-zA-Z0-9_]*\.[a-zA-Z_][a-zA-Z0-9_]*$', + if not re.match(r'^[a-zA-Z_][a-zA-Z0-9_]*\.[a-zA-Z_][a-zA-Z0-9_]*$', self.toks[2]): err("Syntax error, expected: $Alias <.alias> \n", warn=False) From nils.goroll at uplex.de Mon Sep 16 13:43:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 16 Sep 2024 13:43:05 +0000 (UTC) Subject: [master] e41f29a23 Try to make Coverity understand the code Message-ID: <20240916134305.9BF03118A13@lists.varnish-cache.org> commit e41f29a2380080be42e9b18383b5dff877c8d3b4 Author: Nils Goroll Date: Mon Sep 16 15:40:55 2024 +0200 Try to make Coverity understand the code Idea from bugwash discussing #4173 diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c index 1dedb3cc9..1c957b6bd 100644 --- a/lib/libvarnishapi/vsl_cursor.c +++ b/lib/libvarnishapi/vsl_cursor.c @@ -340,6 +340,8 @@ vslc_file_readn(int fd, void *buf, ssize_t n) ssize_t t = 0; ssize_t l; + assert(n > 0); + while (t < n) { l = read(fd, (char *)buf + t, n - t); if (l <= 0) From dridi.boukelmoune at gmail.com Mon Sep 16 15:02:05 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 16 Sep 2024 15:02:05 +0000 (UTC) Subject: [master] 57e660bd4 vbe: Unconditionally clear the backend queue Message-ID: <20240916150205.EA07411B2E2@lists.varnish-cache.org> commit 57e660bd46d141ffcf064f6162daac6405fbb515 Author: Dridi Boukelmoune Date: Mon Sep 16 16:58:00 2024 +0200 vbe: Unconditionally clear the backend queue Refs #4134 diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 4a0e0a424..c7c136bba 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -140,8 +140,9 @@ VBE_Connect_Error(struct VSC_vbe *vsc, int err) /*--------------------------------------------------------------------*/ int -VBE_is_ah_auto (const struct backend *bp) +VBE_is_ah_auto(const struct backend *bp) { + CHECK_OBJ_NOTNULL(bp, BACKEND_MAGIC); return (bp->director->vdir->admin_health == VDI_AH_AUTO); } @@ -150,17 +151,9 @@ void VBE_connwait_signal_all(const struct backend *bp) { struct connwait *cw; - unsigned wait_limit; - vtim_dur wait_tmod; CHECK_OBJ_NOTNULL(bp, BACKEND_MAGIC); - FIND_BE_PARAM(backend_wait_limit, wait_limit, bp); - FIND_BE_TMO(backend_wait_timeout, wait_tmod, bp); - - if (wait_limit == 0 || wait_tmod <= 0) - return; - Lck_Lock(bp->director->mtx); VTAILQ_FOREACH(cw, &bp->cw_head, cw_list) { CHECK_OBJ(cw, CONNWAIT_MAGIC); diff --git a/bin/varnishtest/tests/v00074.vtc b/bin/varnishtest/tests/v00074.vtc index 72104f01d..254718678 100644 --- a/bin/varnishtest/tests/v00074.vtc +++ b/bin/varnishtest/tests/v00074.vtc @@ -190,8 +190,6 @@ varnish v1 -vcl { .host = "${v0_addr}"; .port = "${v0_port}"; .max_connections = 1; - .wait_timeout = 1h; - .wait_limit = 10; } probe default { @@ -205,7 +203,10 @@ varnish v1 -vcl { return(pass); } } -start + varnish v1 -cliok "backend.set_health be healthy" +varnish v1 -cliok "param.set backend_wait_timeout 1h" +varnish v1 -cliok "param.set backend_wait_limit 10" client c9 { txreq @@ -245,8 +246,13 @@ client c12 { # make sure c10-12 are already in the queue barrier b6 sync + +varnish v1 -cliok "param.reset backend_wait_timeout" +varnish v1 -cliok "param.reset backend_wait_limit" + # Going back to probe (auto) that is sick should -# wake up reqs in the wait queue +# wake up reqs in the wait queue, even if queuing +# is currently disabled. varnish v1 -cliok "backend.set_health be auto" client c9 -wait From dridi at varni.sh Mon Sep 16 16:44:11 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Mon, 16 Sep 2024 16:44:11 +0000 Subject: [master] a5810d25d Coverity: Explain vtc_logfail() In-Reply-To: <20240912094105.33ED0100E98@lists.varnish-cache.org> References: <20240912094105.33ED0100E98@lists.varnish-cache.org> Message-ID: On Thu, Sep 12, 2024 at 9:50?AM Nils Goroll wrote: > > > commit a5810d25d00eaaa97f551a52bb03139c4d610cc8 > Author: Nils Goroll > Date: Thu Sep 12 11:37:55 2024 +0200 > > Coverity: Explain vtc_logfail() > > It seems coverity does not grok vtc_logfail() via vtc_fatal() as panic-ish, > because it complains about a possible overflow for l in > > if (signed <= 0) > vtc_fatal(...) > l -= signed; > > Ref CID 1605325 > > diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c > index 5fa44e7be..26bb7302e 100644 > --- a/bin/varnishtest/vtc_log.c > +++ b/bin/varnishtest/vtc_log.c > @@ -109,6 +109,13 @@ vtc_logclose(void *arg) > FREE_OBJ(vl); > } > > +#ifdef __COVERITY__ > +static void v_noreturn_ > +vtc_logfail(void) > +{ > + __coverity_panic__(); > +} > +#else > static void v_noreturn_ > vtc_logfail(void) > { > @@ -119,6 +126,7 @@ vtc_logfail(void) > else > exit(fail_out()); > } > +#endif Out of curiosity, are you against this instead? --->8--- static void v_noreturn_ vtc_logfail(void) { vtc_error = 2; if (!pthread_equal(pthread_self(), vtc_thread)) pthread_exit(NULL); else exit(fail_out()); #ifdef __COVERITY__ __coverity_panic__(); #endif } ---8<--- Alternatively, why not a simple `WRONG("unreachable");` statement instead of a coverity gadget? > static const char * const lead[] = { > "----", > _______________________________________________ > varnish-commit mailing list > varnish-commit at varnish-cache.org > https://www.varnish-cache.org/lists/mailman/listinfo/varnish-commit From dridi.boukelmoune at gmail.com Tue Sep 17 06:37:08 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 17 Sep 2024 06:37:08 +0000 (UTC) Subject: [master] b1cd36eab whats-new: Mention VSM_NOPID Message-ID: <20240917063708.3F9F1114FF0@lists.varnish-cache.org> commit b1cd36eab5b3325998280d0d06e70b7dc1c33ef4 Author: Dridi Boukelmoune Date: Tue Sep 17 08:35:11 2024 +0200 whats-new: Mention VSM_NOPID Spotted by @gquintard who tripped on the PID 1 check that has already proven its worth. diff --git a/doc/sphinx/whats-new/changes-7.6.rst b/doc/sphinx/whats-new/changes-7.6.rst index e3085e8bd..885b911f8 100644 --- a/doc/sphinx/whats-new/changes-7.6.rst +++ b/doc/sphinx/whats-new/changes-7.6.rst @@ -20,6 +20,12 @@ The environment variable ``VARNISH_DEFAULT_N`` now provides the default "varnish name" / "workdir" as otherwise specified by he ``-n`` argument to ``varnishd`` and ``varnish*`` utilities except ``varnishtest``. +Programs attaching to ``varnishd``'s shared memory are now performing more +precise status checks of the ``varnishd`` process. They should in particular +better detect restarts of the process. This comes with signal-based liveness +checks that can be disabled when ``VSM_NOPID`` is exported to the environment +of utilities like ``varnishlog``, ``varnishstat`` or ``varnishncsa``. + varnishd ======== diff --git a/doc/sphinx/whats-new/upgrading-7.6.rst b/doc/sphinx/whats-new/upgrading-7.6.rst index 4f30f4c02..61378e5c1 100644 --- a/doc/sphinx/whats-new/upgrading-7.6.rst +++ b/doc/sphinx/whats-new/upgrading-7.6.rst @@ -9,6 +9,14 @@ besides the actual upgrade. The changes mentioned below are considered noteworthy nevertheless: +Noteworthy changes for container workloads +========================================== + +When ``varnishd`` runs in a different PID namespace on Linux, typically in a +container deployment, ``VSM_NOPID`` must be added to the environment of other +containers attaching themselves to ``varnishd``'s environment variable. This +will otherwise fail liveness checks performed by VSM consumers. + Noteworthy changes when upgrading varnishd ========================================== From nils.goroll at uplex.de Tue Sep 17 07:07:37 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 17 Sep 2024 09:07:37 +0200 Subject: [master] b1cd36eab whats-new: Mention VSM_NOPID In-Reply-To: <20240917063708.3F9F1114FF0@lists.varnish-cache.org> References: <20240917063708.3F9F1114FF0@lists.varnish-cache.org> Message-ID: VSM_NOPID is not reqlly new, is it? -- Nils Goroll (he/him) ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ From dridi.boukelmoune at gmail.com Tue Sep 17 07:29:04 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 17 Sep 2024 07:29:04 +0000 (UTC) Subject: [7.6] cc74ad93a whats-new: Mention VSM_NOPID Message-ID: <20240917072904.DFAF5116E3B@lists.varnish-cache.org> commit cc74ad93a32627e8b24aa2348871fd89630d410d Author: Dridi Boukelmoune Date: Tue Sep 17 08:35:11 2024 +0200 whats-new: Mention VSM_NOPID Spotted by @gquintard who tripped on the PID 1 check that has already proven its worth. diff --git a/doc/sphinx/whats-new/changes-7.6.rst b/doc/sphinx/whats-new/changes-7.6.rst index e3085e8bd..885b911f8 100644 --- a/doc/sphinx/whats-new/changes-7.6.rst +++ b/doc/sphinx/whats-new/changes-7.6.rst @@ -20,6 +20,12 @@ The environment variable ``VARNISH_DEFAULT_N`` now provides the default "varnish name" / "workdir" as otherwise specified by he ``-n`` argument to ``varnishd`` and ``varnish*`` utilities except ``varnishtest``. +Programs attaching to ``varnishd``'s shared memory are now performing more +precise status checks of the ``varnishd`` process. They should in particular +better detect restarts of the process. This comes with signal-based liveness +checks that can be disabled when ``VSM_NOPID`` is exported to the environment +of utilities like ``varnishlog``, ``varnishstat`` or ``varnishncsa``. + varnishd ======== diff --git a/doc/sphinx/whats-new/upgrading-7.6.rst b/doc/sphinx/whats-new/upgrading-7.6.rst index 4f30f4c02..61378e5c1 100644 --- a/doc/sphinx/whats-new/upgrading-7.6.rst +++ b/doc/sphinx/whats-new/upgrading-7.6.rst @@ -9,6 +9,14 @@ besides the actual upgrade. The changes mentioned below are considered noteworthy nevertheless: +Noteworthy changes for container workloads +========================================== + +When ``varnishd`` runs in a different PID namespace on Linux, typically in a +container deployment, ``VSM_NOPID`` must be added to the environment of other +containers attaching themselves to ``varnishd``'s environment variable. This +will otherwise fail liveness checks performed by VSM consumers. + Noteworthy changes when upgrading varnishd ========================================== From dridi.boukelmoune at gmail.com Tue Sep 17 07:29:05 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 17 Sep 2024 07:29:05 +0000 (UTC) Subject: [7.6] 6f1b5befe whats-new: Little mixup Message-ID: <20240917072905.06CD3116E3F@lists.varnish-cache.org> commit 6f1b5befea3c0196e1c6f63037ea5a79b1847e0b Author: Dridi Boukelmoune Date: Tue Sep 17 09:27:06 2024 +0200 whats-new: Little mixup diff --git a/doc/sphinx/whats-new/upgrading-7.6.rst b/doc/sphinx/whats-new/upgrading-7.6.rst index 61378e5c1..bbe2e4abc 100644 --- a/doc/sphinx/whats-new/upgrading-7.6.rst +++ b/doc/sphinx/whats-new/upgrading-7.6.rst @@ -14,8 +14,8 @@ Noteworthy changes for container workloads When ``varnishd`` runs in a different PID namespace on Linux, typically in a container deployment, ``VSM_NOPID`` must be added to the environment of other -containers attaching themselves to ``varnishd``'s environment variable. This -will otherwise fail liveness checks performed by VSM consumers. +containers running programs attaching themselves to ``varnishd``'s shared +memory. This will otherwise fail liveness checks performed by VSM consumers. Noteworthy changes when upgrading varnishd ========================================== From dridi at varni.sh Tue Sep 17 07:56:15 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 17 Sep 2024 07:56:15 +0000 Subject: [master] b1cd36eab whats-new: Mention VSM_NOPID In-Reply-To: References: <20240917063708.3F9F1114FF0@lists.varnish-cache.org> Message-ID: On Tue, Sep 17, 2024 at 7:07?AM Nils Goroll wrote: > > VSM_NOPID is not reqlly new, is it? It is in the sense that it was retired a while ago and reintroduced in this release. See revert commits: https://github.com/varnishcache/varnish-cache/pull/3950/commits Dridi From nils.goroll at uplex.de Tue Sep 17 08:05:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 17 Sep 2024 08:05:06 +0000 (UTC) Subject: [master] 240ef2c66 Start NEXT sections in vrt.h and changes.rst Message-ID: <20240917080506.9F1E1118801@lists.varnish-cache.org> commit 240ef2c66b2f06bd743ef982b50f5ca2419f7624 Author: Nils Goroll Date: Tue Sep 17 10:04:24 2024 +0200 Start NEXT sections in vrt.h and changes.rst to anchor patches diff --git a/doc/changes.rst b/doc/changes.rst index 1fb332bfc..360cb180a 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -34,13 +34,17 @@ http://varnish-cache.org/docs/trunk/whats-new/index.html and via individual releases. These documents are updated as part of the release process. -================================ -Varnish Cache 7.6.0 (2024-09-13) -================================ +=============================== +Varnish Cache NEXT (2025-03-15) +=============================== .. PLEASE keep this roughly in commit order as shown by git-log / tig (new to old) +================================ +Varnish Cache 7.6.0 (2024-09-13) +================================ + * The Varnish Delivery Processor (VDP) filter API has been generalized to also accommodate future use for backend request bodies: diff --git a/include/vrt.h b/include/vrt.h index edd21e3ed..144a7e2ed 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -57,6 +57,7 @@ * Whenever something is deleted or changed in a way which is not * binary/load-time compatible, increment MAJOR version * + * NEXT (2024-03-15) * 20.0 (2024-09-13) * struct vrt_backend.backend_wait_timeout added * struct vrt_backend.backend_wait_limit added From nils.goroll at uplex.de Tue Sep 17 08:41:33 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 17 Sep 2024 10:41:33 +0200 Subject: [master] a5810d25d Coverity: Explain vtc_logfail() In-Reply-To: References: <20240912094105.33ED0100E98@lists.varnish-cache.org> Message-ID: <08ea694e-c616-420f-af9a-4b6cd9e14499@uplex.de> On 16.09.24 18:44, Dridi Boukelmoune wrote: > Out of curiosity, are you against this instead? ... > Alternatively, why not a simple `WRONG("unreachable");` statement > instead of a coverity gadget? I have no idea which of these, if any, work. Neither do I know yet if the __coverity_panic__() I added works. -- Nils Goroll (he/him) ** * * UPLEX - Nils Goroll Systemoptimierung Scheffelstra?e 32 22301 Hamburg tel +49 40 28805731 mob +49 170 2723133 fax +49 40 42949753 xmpp://slink at jabber.int.uplex.de/ http://uplex.de/ From dridi at varni.sh Tue Sep 17 09:06:29 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Tue, 17 Sep 2024 09:06:29 +0000 Subject: [master] a5810d25d Coverity: Explain vtc_logfail() In-Reply-To: <08ea694e-c616-420f-af9a-4b6cd9e14499@uplex.de> References: <20240912094105.33ED0100E98@lists.varnish-cache.org> <08ea694e-c616-420f-af9a-4b6cd9e14499@uplex.de> Message-ID: On Tue, Sep 17, 2024 at 8:41?AM Nils Goroll wrote: > > On 16.09.24 18:44, Dridi Boukelmoune wrote: > > Out of curiosity, are you against this instead? > > ... > > > Alternatively, why not a simple `WRONG("unreachable");` statement > > instead of a coverity gadget? > > I have no idea which of these, if any, work. Neither do I know yet if the > __coverity_panic__() I added works. $ git grep WRONG | grep unreachable flint.lnt:-emacro(527, WRONG) // unreachable code vmod/vmod_debug.c: WRONG("unreachable"); I'll add a WRONG with a comment. From dridi.boukelmoune at gmail.com Wed Sep 18 09:05:07 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 18 Sep 2024 09:05:07 +0000 (UTC) Subject: [master] 0a4d4c479 varnishtest: Unknown escape sequence in huffman_gen.py Message-ID: <20240918090507.CAEE764BFA@lists.varnish-cache.org> commit 0a4d4c4793eb0d5aa42158c54e250c8401fda3b2 Author: Dridi Boukelmoune Date: Wed Sep 18 11:02:52 2024 +0200 varnishtest: Unknown escape sequence in huffman_gen.py diff --git a/bin/varnishtest/huffman_gen.py b/bin/varnishtest/huffman_gen.py index 1320424d2..4b761515b 100755 --- a/bin/varnishtest/huffman_gen.py +++ b/bin/varnishtest/huffman_gen.py @@ -4,7 +4,7 @@ import re import sys #HPH(0x30, 0x00000000, 5) -regex = re.compile("^HPH\((.{4}), (.{10}), +(.{1,3})\)") +regex = re.compile(r"^HPH\((.{4}), (.{10}), +(.{1,3})\)") if len(sys.argv) != 2: print("{} takes one and only one argument".format(sys.argv[0])) From dridi.boukelmoune at gmail.com Wed Sep 18 12:02:04 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Wed, 18 Sep 2024 12:02:04 +0000 (UTC) Subject: [master] 2802dc94d vtc_log: Simplify Coverity helper Message-ID: <20240918120204.BE2CA104971@lists.varnish-cache.org> commit 2802dc94dbfcc74fdb89fd37ebe6efbc7079a3ad Author: Dridi Boukelmoune Date: Wed Sep 18 14:01:08 2024 +0200 vtc_log: Simplify Coverity helper diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c index 26bb7302e..549ffdad8 100644 --- a/bin/varnishtest/vtc_log.c +++ b/bin/varnishtest/vtc_log.c @@ -109,13 +109,6 @@ vtc_logclose(void *arg) FREE_OBJ(vl); } -#ifdef __COVERITY__ -static void v_noreturn_ -vtc_logfail(void) -{ - __coverity_panic__(); -} -#else static void v_noreturn_ vtc_logfail(void) { @@ -125,8 +118,9 @@ vtc_logfail(void) pthread_exit(NULL); else exit(fail_out()); + + WRONG("unreachable"); /* Help Coverity Scan see noreturn. */ } -#endif static const char * const lead[] = { "----", From dridi at varni.sh Wed Sep 18 14:07:03 2024 From: dridi at varni.sh (Dridi Boukelmoune) Date: Wed, 18 Sep 2024 14:07:03 +0000 Subject: [master] a5810d25d Coverity: Explain vtc_logfail() In-Reply-To: References: <20240912094105.33ED0100E98@lists.varnish-cache.org> <08ea694e-c616-420f-af9a-4b6cd9e14499@uplex.de> Message-ID: On Tue, Sep 17, 2024 at 9:06?AM Dridi Boukelmoune wrote: > > On Tue, Sep 17, 2024 at 8:41?AM Nils Goroll wrote: > > > > On 16.09.24 18:44, Dridi Boukelmoune wrote: > > > Out of curiosity, are you against this instead? > > > > ... > > > > > Alternatively, why not a simple `WRONG("unreachable");` statement > > > instead of a coverity gadget? > > > > I have no idea which of these, if any, work. Neither do I know yet if the > > __coverity_panic__() I added works. > > $ git grep WRONG | grep unreachable > flint.lnt:-emacro(527, WRONG) // unreachable code > vmod/vmod_debug.c: WRONG("unreachable"); > > I'll add a WRONG with a comment. For reference: https://github.com/varnishcache/varnish-cache/commit/2802dc94dbfcc74fdb89fd37ebe6efbc7079a3ad From nils.goroll at uplex.de Thu Sep 19 21:40:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 19 Sep 2024 21:40:06 +0000 (UTC) Subject: [master] c41711500 waiter: gc unused typedefs & inject method Message-ID: <20240919214006.AC1499B7E@lists.varnish-cache.org> commit c41711500ce8f93dda925151a58f7e6876e1db93 Author: Nils Goroll Date: Thu Sep 19 23:39:08 2024 +0200 waiter: gc unused typedefs & inject method diff --git a/bin/varnishd/waiter/waiter_priv.h b/bin/varnishd/waiter/waiter_priv.h index ed0d7b0c1..aa85df3a0 100644 --- a/bin/varnishd/waiter/waiter_priv.h +++ b/bin/varnishd/waiter/waiter_priv.h @@ -50,15 +50,12 @@ struct waiter { typedef void waiter_init_f(struct waiter *); typedef void waiter_fini_f(struct waiter *); typedef int waiter_enter_f(void *priv, struct waited *); -typedef void waiter_inject_f(const struct waiter *, struct waited *); -typedef void waiter_evict_f(const struct waiter *, struct waited *); struct waiter_impl { const char *name; waiter_init_f *init; waiter_fini_f *fini; waiter_enter_f *enter; - waiter_inject_f *inject; size_t size; }; From nils.goroll at uplex.de Mon Sep 23 12:23:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 23 Sep 2024 12:23:06 +0000 (UTC) Subject: [master] fb28e92fd cache_obj: gc obsolete comment Message-ID: <20240923122306.6EFD411941B@lists.varnish-cache.org> commit fb28e92fdb34c11aa4a9a890b6130a4366e6af92 Author: Nils Goroll Date: Sun Sep 22 12:46:02 2024 +0200 cache_obj: gc obsolete comment Remove a comment which became obsolete with caf49d8b42b97c3599e01f2e503d24991a1a0bcf diff --git a/bin/varnishd/cache/cache_obj.c b/bin/varnishd/cache/cache_obj.c index 477b26e2a..9ffa373f3 100644 --- a/bin/varnishd/cache/cache_obj.c +++ b/bin/varnishd/cache/cache_obj.c @@ -226,12 +226,6 @@ obj_extend_condwait(const struct objcore *oc) return; assert(oc->flags & OC_F_TRANSIENT); - /* NB: strictly signaling progress both ways would be prone to - * deadlocks, so instead we wait for signals from the client side - * when delivered_so_far so far is updated, but in case the fetch - * thread was not waiting at the time of the signal, we will see - * updates to delivered_so_far after timing out. - */ while (!(oc->flags & OC_F_CANCEL) && oc->boc->fetched_so_far > oc->boc->delivered_so_far + oc->boc->transit_buffer) (void)Lck_CondWait(&oc->boc->cond, &oc->boc->mtx); From phk at FreeBSD.org Mon Sep 23 13:45:05 2024 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 23 Sep 2024 13:45:05 +0000 (UTC) Subject: [master] f666571b4 Cherry-pick (f7d01aae6ec611) from madler/zlib: Message-ID: <20240923134505.F0C7011BEFB@lists.varnish-cache.org> commit f666571b46f8ec66b92a272dd0f9f4f9724f885f Author: Poul-Henning Kamp Date: Mon Sep 23 13:33:05 2024 +0000 Cherry-pick (f7d01aae6ec611) from madler/zlib: Avoid out-of-bounds pointer arithmetic in inflateCopy(). Though it does not matter for code correctness, clang's UBSan injects code that complains about computing a pointer from an array where the result is out-of-bounds for that array, even though the pointer is never dereferenced. Go figure. This commit avoids that possibility when computing distcode in inflateCopy(). diff --git a/lib/libvgz/inflate.c b/lib/libvgz/inflate.c index 5c7494f47..b9545e918 100644 --- a/lib/libvgz/inflate.c +++ b/lib/libvgz/inflate.c @@ -933,7 +933,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { while (state->have < 19) state->lens[order[state->have++]] = 0; state->next = state->codes; - state->lencode = (const code FAR *)(state->next); + state->lencode = state->distcode = (const code FAR *)(state->next); state->lenbits = 7; ret = inflate_table(CODES, state->lens, 19, &(state->next), &(state->lenbits), state->work); From phk at FreeBSD.org Mon Sep 23 13:45:06 2024 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 23 Sep 2024 13:45:06 +0000 (UTC) Subject: [master] 76e5f90e6 Cherry pick 545f194963 from madler/zlib Message-ID: <20240923134506.1763B11BEFE@lists.varnish-cache.org> commit 76e5f90e689fce9237615906aecac51006304e8b Author: Poul-Henning Kamp Date: Mon Sep 23 13:41:41 2024 +0000 Cherry pick 545f194963 from madler/zlib Add old gcc ULONG_LONG_MAX macro to find a 64-bit type in zutil.h. diff --git a/lib/libvgz/zutil.h b/lib/libvgz/zutil.h index a74201f02..c095a04d1 100644 --- a/lib/libvgz/zutil.h +++ b/lib/libvgz/zutil.h @@ -48,6 +48,8 @@ typedef unsigned long ulg; # define Z_U8 unsigned long # elif (ULLONG_MAX == 0xffffffffffffffff) # define Z_U8 unsigned long long +# elif (ULONG_LONG_MAX == 0xffffffffffffffff) +# define Z_U8 unsigned long long # elif (UINT_MAX == 0xffffffffffffffff) # define Z_U8 unsigned # endif From nils.goroll at uplex.de Mon Sep 23 15:31:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 23 Sep 2024 15:31:05 +0000 (UTC) Subject: [master] 44ddf0360 mgt: Make an attempt to raise RLIMIT_MEMLOCK to infinity, set soft limit Message-ID: <20240923153105.0C3DD120162@lists.varnish-cache.org> commit 44ddf036003e30fc4046038eabf95e30b7410fd2 Author: Nils Goroll Date: Mon Sep 23 16:55:20 2024 +0200 mgt: Make an attempt to raise RLIMIT_MEMLOCK to infinity, set soft limit Try to raise the hard limit to infinity, then use as the soft (current) limit whatever the hard (max) limit is. Motivated by #4193 diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 1b8e49f27..4edc1f0a3 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include @@ -636,6 +637,7 @@ main(int argc, char * const *argv) pid_t pid; struct arg_list *alp; int first_arg = 1; + struct rlimit rlim; if (argc == 2 && !strcmp(argv[1], "--optstring")) { printf("%s\n", opt_spec); @@ -873,6 +875,14 @@ main(int argc, char * const *argv) workdir, VAS_errtxt(errno)); VJ_master(JAIL_MASTER_SYSTEM); + /* try to raise to max (ignore error), then set _cur to whatever we got */ + rlim.rlim_cur = 0; + rlim.rlim_max = RLIM_INFINITY; + (void)setrlimit(RLIMIT_MEMLOCK, &rlim); + AZ(getrlimit(RLIMIT_MEMLOCK, &rlim)); + rlim.rlim_cur = rlim.rlim_max; + AZ(setrlimit(RLIMIT_MEMLOCK, &rlim)); + AZ(system("rm -rf vmod_cache vext_cache worker_tmpdir")); VJ_master(JAIL_MASTER_LOW); From nils.goroll at uplex.de Mon Sep 23 15:31:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 23 Sep 2024 15:31:05 +0000 (UTC) Subject: [master] 93c001a25 vsm: Improve information output with the mlock() warning Message-ID: <20240923153105.2BEB6120165@lists.varnish-cache.org> commit 93c001a25437c401bdc45fd5b43a7938c01780ae Author: Nils Goroll Date: Mon Sep 23 17:13:42 2024 +0200 vsm: Improve information output with the mlock() warning Inform about current resource limits to help diagnosis. Example output: Child launched OK Info: Child (792279) said Child starts Info: Child (792279) said Warning: mlock() of VSM failed: Cannot allocate memory (12) Info: Child (792279) said Info: max locked memory (soft): 1048576 bytes Info: Child (792279) said Info: max locked memory (hard): 1048576 bytes Motivated by #4193 diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c index 3a9aeb001..51315f9c3 100644 --- a/bin/varnishd/common/common_vsmw.c +++ b/bin/varnishd/common/common_vsmw.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include "vdef.h" @@ -274,10 +275,22 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg) /*--------------------------------------------------------------------*/ +static void +printlim(const char *name, rlim_t lim) +{ + + fprintf(stderr, "Info: %s: ", name); + if (lim == RLIM_INFINITY) + fprintf(stderr, "unlimited\n"); + else + fprintf(stderr, "%ju bytes\n", (uintmax_t)lim); +} + static struct vsmw_cluster * vsmw_newcluster(struct vsmw *vsmw, size_t len, const char *pfx) { struct vsmw_cluster *vc; + struct rlimit rlim; static int warn = 0; int fd; size_t ps; @@ -311,6 +324,9 @@ vsmw_newcluster(struct vsmw *vsmw, size_t len, const char *pfx) if (mlock(vc->ptr, len) && warn++ == 0) { fprintf(stderr, "Warning: mlock() of VSM failed: %s (%d)\n", VAS_errtxt(errno), errno); + AZ(getrlimit(RLIMIT_MEMLOCK, &rlim)); + printlim("max locked memory (soft)", rlim.rlim_cur); + printlim("max locked memory (hard)", rlim.rlim_max); } return (vc); From nils.goroll at uplex.de Mon Sep 23 16:17:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 23 Sep 2024 16:17:05 +0000 (UTC) Subject: [master] 82ab4ec6b doc: Try to help container / kubernetes users Message-ID: <20240923161705.A252C121F3B@lists.varnish-cache.org> commit 82ab4ec6b08e085f912d238d84555fc2a633611a Author: Nils Goroll Date: Mon Sep 23 18:15:30 2024 +0200 doc: Try to help container / kubernetes users Ref #4193 diff --git a/doc/sphinx/reference/vsm.rst b/doc/sphinx/reference/vsm.rst index 22d4f34ef..c5da4e68e 100644 --- a/doc/sphinx/reference/vsm.rst +++ b/doc/sphinx/reference/vsm.rst @@ -121,3 +121,45 @@ varnishd are no longer relevant across namespaces. To disable liveness checks based on PIDs, the variable ``VSM_NOPID`` needs to be present in the environment of VSM readers. + +Warning: mlock() of VSM failed +------------------------------ + +It is vital for performance of the Varnish Shared Memory model that all VSM be +resident in RAM at all times. At startup, varnish tries to lift the respective +limits and an attempt is made to lock all VSM in memory, but if +``RLIMIT_MEMLOCK`` is configured too low, this fails and a warning similar to +the following is logged to standard error or syslog:: + + Info: Child (814693) said Warning: mlock() of VSM failed: Cannot allocate memory (12) + Info: Child (814693) said Info: max locked memory (soft): 1048576 bytes + Info: Child (814693) said Info: max locked memory (hard): 1048576 bytes + +Where the system configuration ensures that virtual memory is never paged, this +warning can be ignored, but in general it is recommended to set +``RLIMIT_MEMLOCK`` to ``unlimited``. See the ``ulimit`` shell builtin and +``getrlimit(2)``) for details. + +Containers and Memory Locking +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Container runtime environments might require outside configuration to raise +``RLIMIT_MEMLOCK``. + +For _Docker_, a common option is to use the ``--ulimit=memlock=-1`` command line +argument. + +.. _Kubernetes: https://github.com/kubernetes/kubernetes/issues/3595 + +`Kubernetes`_ infamously does not support setting resource controls, so where +the ``mlock()`` warning is seen, one option is to add ``CAP_IPC_LOCK`` to the +container's ``securityContext``:: + + securityContext: + capabilities: + add: + - IPC_LOCK + +Note that this added capability should, with the usual disclaimer that bugs +could exist, not impose any additional risks, in particular not if the system at +hand would not page memory anyway because no swap space is configured. From nils.goroll at uplex.de Mon Sep 23 16:33:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 23 Sep 2024 16:33:05 +0000 (UTC) Subject: [master] e79c02546 Use RLIMIT_MEMLOCK only where available Message-ID: <20240923163305.BF94F122862@lists.varnish-cache.org> commit e79c025469116b0a4b9a4b82dc93f447098677d9 Author: Nils Goroll Date: Mon Sep 23 18:31:56 2024 +0200 Use RLIMIT_MEMLOCK only where available Solaris does not have it and I overlooked the CONFORMING TO section in the Linux manpage. Ref #4193 diff --git a/bin/varnishd/common/common_vsmw.c b/bin/varnishd/common/common_vsmw.c index 51315f9c3..79d1d740a 100644 --- a/bin/varnishd/common/common_vsmw.c +++ b/bin/varnishd/common/common_vsmw.c @@ -275,6 +275,7 @@ vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg) /*--------------------------------------------------------------------*/ +#ifdef RLIMIT_MEMLOCK static void printlim(const char *name, rlim_t lim) { @@ -286,11 +287,22 @@ printlim(const char *name, rlim_t lim) fprintf(stderr, "%ju bytes\n", (uintmax_t)lim); } +static void +printmemlock(void) { + struct rlimit rlim; + + AZ(getrlimit(RLIMIT_MEMLOCK, &rlim)); + printlim("max locked memory (soft)", rlim.rlim_cur); + printlim("max locked memory (hard)", rlim.rlim_max); +} +#else +static void printmemlock(void) {} +#endif + static struct vsmw_cluster * vsmw_newcluster(struct vsmw *vsmw, size_t len, const char *pfx) { struct vsmw_cluster *vc; - struct rlimit rlim; static int warn = 0; int fd; size_t ps; @@ -324,9 +336,7 @@ vsmw_newcluster(struct vsmw *vsmw, size_t len, const char *pfx) if (mlock(vc->ptr, len) && warn++ == 0) { fprintf(stderr, "Warning: mlock() of VSM failed: %s (%d)\n", VAS_errtxt(errno), errno); - AZ(getrlimit(RLIMIT_MEMLOCK, &rlim)); - printlim("max locked memory (soft)", rlim.rlim_cur); - printlim("max locked memory (hard)", rlim.rlim_max); + printmemlock(); } return (vc); diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 4edc1f0a3..d48a3fd06 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -875,6 +875,7 @@ main(int argc, char * const *argv) workdir, VAS_errtxt(errno)); VJ_master(JAIL_MASTER_SYSTEM); +#ifdef RLIMIT_MEMLOCK /* try to raise to max (ignore error), then set _cur to whatever we got */ rlim.rlim_cur = 0; rlim.rlim_max = RLIM_INFINITY; @@ -882,6 +883,7 @@ main(int argc, char * const *argv) AZ(getrlimit(RLIMIT_MEMLOCK, &rlim)); rlim.rlim_cur = rlim.rlim_max; AZ(setrlimit(RLIMIT_MEMLOCK, &rlim)); +#endif AZ(system("rm -rf vmod_cache vext_cache worker_tmpdir")); VJ_master(JAIL_MASTER_LOW); From nils.goroll at uplex.de Mon Sep 23 16:38:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 23 Sep 2024 16:38:05 +0000 (UTC) Subject: [master] f53bb2486 doc polish: fix emphasis Message-ID: <20240923163805.B214C122C2A@lists.varnish-cache.org> commit f53bb248691a17849b46a4dbc897fa5b3861e000 Author: Nils Goroll Date: Mon Sep 23 18:37:28 2024 +0200 doc polish: fix emphasis diff --git a/doc/sphinx/reference/vsm.rst b/doc/sphinx/reference/vsm.rst index c5da4e68e..f2902db89 100644 --- a/doc/sphinx/reference/vsm.rst +++ b/doc/sphinx/reference/vsm.rst @@ -146,7 +146,7 @@ Containers and Memory Locking Container runtime environments might require outside configuration to raise ``RLIMIT_MEMLOCK``. -For _Docker_, a common option is to use the ``--ulimit=memlock=-1`` command line +For `Docker`, a common option is to use the ``--ulimit=memlock=-1`` command line argument. .. _Kubernetes: https://github.com/kubernetes/kubernetes/issues/3595 From nils.goroll at uplex.de Mon Sep 23 16:39:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 23 Sep 2024 16:39:05 +0000 (UTC) Subject: [master] 5f6504609 doc polish: another nit snug in Message-ID: <20240923163905.8E0E9122F3A@lists.varnish-cache.org> commit 5f650460977dd238af8e0d948a95caa85e617f2a Author: Nils Goroll Date: Mon Sep 23 18:38:23 2024 +0200 doc polish: another nit snug in diff --git a/doc/sphinx/reference/vsm.rst b/doc/sphinx/reference/vsm.rst index f2902db89..ad6524519 100644 --- a/doc/sphinx/reference/vsm.rst +++ b/doc/sphinx/reference/vsm.rst @@ -138,7 +138,7 @@ the following is logged to standard error or syslog:: Where the system configuration ensures that virtual memory is never paged, this warning can be ignored, but in general it is recommended to set ``RLIMIT_MEMLOCK`` to ``unlimited``. See the ``ulimit`` shell builtin and -``getrlimit(2)``) for details. +``getrlimit(2)`` for details. Containers and Memory Locking ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From nils.goroll at uplex.de Mon Sep 23 16:41:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 23 Sep 2024 16:41:06 +0000 (UTC) Subject: [master] e87afd691 nit: Use RLIMIT_MEMLOCK only where available Message-ID: <20240923164106.2925C423C@lists.varnish-cache.org> commit e87afd691d820d9fcb28f3621c4a23a3c0bbb780 Author: Nils Goroll Date: Mon Sep 23 18:40:22 2024 +0200 nit: Use RLIMIT_MEMLOCK only where available Not my day for details today, it seems. :( diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index d48a3fd06..4db07013c 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -637,7 +637,6 @@ main(int argc, char * const *argv) pid_t pid; struct arg_list *alp; int first_arg = 1; - struct rlimit rlim; if (argc == 2 && !strcmp(argv[1], "--optstring")) { printf("%s\n", opt_spec); @@ -877,6 +876,7 @@ main(int argc, char * const *argv) VJ_master(JAIL_MASTER_SYSTEM); #ifdef RLIMIT_MEMLOCK /* try to raise to max (ignore error), then set _cur to whatever we got */ + struct rlimit rlim; rlim.rlim_cur = 0; rlim.rlim_max = RLIM_INFINITY; (void)setrlimit(RLIMIT_MEMLOCK, &rlim); From nils.goroll at uplex.de Mon Sep 23 16:45:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 23 Sep 2024 16:45:06 +0000 (UTC) Subject: [master] e134438cd add tests get larger threadpool, necessary for ppc32 Message-ID: <20240923164506.C6A37468D@lists.varnish-cache.org> commit e134438cd8016d82c8f9ad98ab2dadb5ca47a818 Author: Ingvar Hagelund Date: Wed Sep 18 17:14:20 2024 +0200 add tests get larger threadpool, necessary for ppc32 diff --git a/bin/varnishtest/tests/e00016.vtc b/bin/varnishtest/tests/e00016.vtc index 180f1a769..a096c4f12 100644 --- a/bin/varnishtest/tests/e00016.vtc +++ b/bin/varnishtest/tests/e00016.vtc @@ -27,7 +27,7 @@ server s1 { } -start # give enough stack to 32bit systems -varnish v1 -cliok "param.set thread_pool_stack 80k" +varnish v1 -cliok "param.set thread_pool_stack 128k" varnish v1 -cliok "param.set feature +esi_disable_xml_check" varnish v1 -syntax 4.0 -vcl+backend { diff --git a/bin/varnishtest/tests/e00029.vtc b/bin/varnishtest/tests/e00029.vtc index b26b966d7..6f57c5110 100644 --- a/bin/varnishtest/tests/e00029.vtc +++ b/bin/varnishtest/tests/e00029.vtc @@ -16,7 +16,7 @@ server s1 { chunkedlen 0 } -start -varnish v1 -cliok "param.set thread_pool_stack 80k" +varnish v1 -cliok "param.set thread_pool_stack 128k" varnish v1 -cliok "param.set feature +esi_include_onerror" varnish v1 -vcl+backend { sub vcl_backend_response { diff --git a/bin/varnishtest/tests/e00030.vtc b/bin/varnishtest/tests/e00030.vtc index 424207020..c852c0ec0 100644 --- a/bin/varnishtest/tests/e00030.vtc +++ b/bin/varnishtest/tests/e00030.vtc @@ -52,7 +52,7 @@ server s1 { } -start # give enough stack to 32bit systems -varnish v1 -cliok "param.set thread_pool_stack 80k" +varnish v1 -cliok "param.set thread_pool_stack 128k" varnish v1 -cliok "param.set feature +esi_disable_xml_check" varnish v1 -vcl+backend { diff --git a/bin/varnishtest/tests/e00034.vtc b/bin/varnishtest/tests/e00034.vtc index 352c44764..c7c81105b 100644 --- a/bin/varnishtest/tests/e00034.vtc +++ b/bin/varnishtest/tests/e00034.vtc @@ -73,7 +73,7 @@ server s1 { } -start # give enough stack to 32bit systems -varnish v1 -cliok "param.set thread_pool_stack 80k" +varnish v1 -cliok "param.set thread_pool_stack 128k" varnish v1 -syntax 4.1 -vcl+backend { import debug; diff --git a/bin/varnishtest/tests/l00003.vtc b/bin/varnishtest/tests/l00003.vtc index 43e8cc6e7..b851e5fb6 100644 --- a/bin/varnishtest/tests/l00003.vtc +++ b/bin/varnishtest/tests/l00003.vtc @@ -15,7 +15,7 @@ server s1 { } -start # give enough stack to 32bit systems -varnish v1 -cliok "param.set thread_pool_stack 80k" +varnish v1 -cliok "param.set thread_pool_stack 128k" varnish v1 -vcl+backend { sub vcl_backend_response { diff --git a/bin/varnishtest/tests/r01737.vtc b/bin/varnishtest/tests/r01737.vtc index 9b5d23a93..3a8e2cce0 100644 --- a/bin/varnishtest/tests/r01737.vtc +++ b/bin/varnishtest/tests/r01737.vtc @@ -24,7 +24,7 @@ server s1 { } -start # give enough stack to 32bit systems -varnish v1 -cliok "param.set thread_pool_stack 80k" +varnish v1 -cliok "param.set thread_pool_stack 128k" varnish v1 -cliok "param.set feature +esi_disable_xml_check" varnish v1 -vcl+backend { diff --git a/bin/varnishtest/tests/r01781.vtc b/bin/varnishtest/tests/r01781.vtc index 31e21bf9a..543f6b7e5 100644 --- a/bin/varnishtest/tests/r01781.vtc +++ b/bin/varnishtest/tests/r01781.vtc @@ -14,7 +14,7 @@ server s1 { } -start # give enough stack to 32bit systems -varnish v1 -cliok "param.set thread_pool_stack 80k" +varnish v1 -cliok "param.set thread_pool_stack 128k" varnish v1 -cliok "param.set feature +esi_disable_xml_check" varnish v1 -vcl+backend { diff --git a/bin/varnishtest/tests/r01878.vtc b/bin/varnishtest/tests/r01878.vtc index 2d8a0f1fa..05cced617 100644 --- a/bin/varnishtest/tests/r01878.vtc +++ b/bin/varnishtest/tests/r01878.vtc @@ -15,7 +15,7 @@ server s1 { } -start # give enough stack to 32bit systems -varnish v1 -cliok "param.set thread_pool_stack 80k" +varnish v1 -cliok "param.set thread_pool_stack 128k" varnish v1 -vcl+backend { sub vcl_backend_response { diff --git a/bin/varnishtest/tests/r02849.vtc b/bin/varnishtest/tests/r02849.vtc index f7860ab35..0cb0f2f24 100644 --- a/bin/varnishtest/tests/r02849.vtc +++ b/bin/varnishtest/tests/r02849.vtc @@ -23,7 +23,7 @@ server s1 { } -start # give enough stack to 32bit systems -varnish v1 -cliok "param.set thread_pool_stack 80k" +varnish v1 -cliok "param.set thread_pool_stack 128k" varnish v1 -vcl+backend { sub vcl_recv { diff --git a/bin/varnishtest/tests/v00042.vtc b/bin/varnishtest/tests/v00042.vtc index 3d77175ea..3b4639935 100644 --- a/bin/varnishtest/tests/v00042.vtc +++ b/bin/varnishtest/tests/v00042.vtc @@ -34,7 +34,7 @@ server s1 { } -start # give enough stack to 32bit systems -varnish v1 -cliok "param.set thread_pool_stack 80k" +varnish v1 -cliok "param.set thread_pool_stack 128k" varnish v1 -vcl+backend { import debug; diff --git a/bin/varnishtest/tests/v00043.vtc b/bin/varnishtest/tests/v00043.vtc index 0d9e6cb93..a3a64da38 100644 --- a/bin/varnishtest/tests/v00043.vtc +++ b/bin/varnishtest/tests/v00043.vtc @@ -72,7 +72,7 @@ varnish v1 -errvcl "Not available in subroutine 'vcl_init'" { } # give enough stack to 32bit systems -varnish v1 -cliok "param.set thread_pool_stack 80k" +varnish v1 -cliok "param.set thread_pool_stack 128k" varnish v1 -cliok "param.set debug +syncvsl" varnish v1 -vcl+backend { From nils.goroll at uplex.de Mon Sep 23 17:39:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 23 Sep 2024 17:39:06 +0000 (UTC) Subject: [master] 4001ea251 Polish: avoid magic length numbers for fixed strings Message-ID: <20240923173906.240346B20@lists.varnish-cache.org> commit 4001ea251bac6924c2e86bac3d0f2c8d93c96bd8 Author: Nils Goroll Date: Mon Sep 23 19:36:19 2024 +0200 Polish: avoid magic length numbers for fixed strings Motivated by Dridi in https://github.com/varnishcache/varnish-cache/pull/4174#discussion_r1771405374 diff --git a/bin/varnishd/mgt/mgt_jail_unix.c b/bin/varnishd/mgt/mgt_jail_unix.c index 8fb6dfc9a..66e088253 100644 --- a/bin/varnishd/mgt/mgt_jail_unix.c +++ b/bin/varnishd/mgt/mgt_jail_unix.c @@ -132,22 +132,28 @@ vju_init(char **args) ARGV_ERR("Unix Jail: Must be root.\n"); for (;*args != NULL; args++) { - if (!strncmp(*args, "user=", 5)) { - if (vju_getuid((*args) + 5)) + const char * const a_user = "user="; + const size_t l_user = strlen(a_user); + if (!strncmp(*args, a_user, l_user)) { + if (vju_getuid((*args) + l_user)) ARGV_ERR( "Unix jail: %s user not found.\n", (*args) + 5); continue; } - if (!strncmp(*args, "workuser=", 9)) { - if (vju_getwrkuid((*args) + 9)) + const char * const a_workuser = "workuser="; + const size_t l_workuser = strlen(a_workuser); + if (!strncmp(*args, a_workuser, l_workuser)) { + if (vju_getwrkuid((*args) + l_workuser)) ARGV_ERR( "Unix jail: %s user not found.\n", (*args) + 9); continue; } - if (!strncmp(*args, "ccgroup=", 8)) { - if (vju_getccgid((*args) + 8)) + const char * const a_ccgroup = "ccgroup="; + const size_t l_ccgroup = strlen(a_ccgroup); + if (!strncmp(*args, "ccgroup=", l_ccgroup)) { + if (vju_getccgid((*args) + l_ccgroup)) ARGV_ERR( "Unix jail: %s group not found.\n", (*args) + 8); From dridi.boukelmoune at gmail.com Tue Sep 24 07:21:08 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 24 Sep 2024 07:21:08 +0000 (UTC) Subject: [master] 4a5c7fd94 vdef: Add Tforeach() macro for txt Message-ID: <20240924072108.2146E11A4D1@lists.varnish-cache.org> commit 4a5c7fd9472f577505f4b1b255dc2fc8078818bb Author: Dridi Boukelmoune Date: Mon Sep 23 11:02:11 2024 +0200 vdef: Add Tforeach() macro for txt diff --git a/include/vdef.h b/include/vdef.h index b287e595f..95f342cb6 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -265,6 +265,7 @@ typedef struct { #define Tlen(t) (pdiff((t).b, (t).e)) #define Tstr(s) (/*lint -e(446)*/ (txt){(s), (s) + strlen(s)}) #define Tstrcmp(t, s) (strncmp((t).b, (s), Tlen(t))) +#define Tforeach(c, t) for ((c) = (t).b; (c) < (t).e; (c)++) /* #3020 dummy definitions until PR is merged*/ #define LIKELY(x) (x) From dridi.boukelmoune at gmail.com Tue Sep 24 07:21:08 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 24 Sep 2024 07:21:08 +0000 (UTC) Subject: [master] 3f3d70146 http2_hpack: Apply Tforeach() macro Message-ID: <20240924072108.4CC3D11A4D4@lists.varnish-cache.org> commit 3f3d701468564d166529a4b00d09966d688432e0 Author: Dridi Boukelmoune Date: Mon Sep 23 11:03:49 2024 +0200 http2_hpack: Apply Tforeach() macro Courtesy of Coccinelle. diff --git a/bin/varnishd/http2/cache_http2_hpack.c b/bin/varnishd/http2/cache_http2_hpack.c index 0ef67ffe1..694e7fc83 100644 --- a/bin/varnishd/http2/cache_http2_hpack.c +++ b/bin/varnishd/http2/cache_http2_hpack.c @@ -74,7 +74,7 @@ h2h_checkhdr(struct vsl_log *vsl, txt nm, txt val) l = vmin_t(int, Tlen(nm) + 2 + Tlen(val), 20); state = FLD_NAME_FIRST; - for (p = nm.b; p < nm.e; p++) { + Tforeach(p, nm) { switch(state) { case FLD_NAME_FIRST: state = FLD_NAME; @@ -101,7 +101,7 @@ h2h_checkhdr(struct vsl_log *vsl, txt nm, txt val) } state = FLD_VALUE_FIRST; - for (p = val.b; p < val.e; p++) { + Tforeach(p, val) { switch(state) { case FLD_VALUE_FIRST: if (vct_issp(*p)) { @@ -174,7 +174,7 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) disallow_empty = 1; /* Check HTTP token */ - for (p = hdr.b; p < hdr.e; p++) { + Tforeach(p, hdr) { if (!vct_istchar(*p)) return (H2SE_PROTOCOL_ERROR); } @@ -192,7 +192,7 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) } /* Path cannot contain LWS or CTL */ - for (p = hdr.b; p < hdr.e; p++) { + Tforeach(p, hdr) { if (vct_islws(*p) || vct_isctl(*p)) return (H2SE_PROTOCOL_ERROR); } @@ -206,7 +206,7 @@ h2h_addhdr(struct http *hp, struct h2h_decode *d) disallow_empty = 1; /* Check HTTP token */ - for (p = val.b; p < val.e; p++) { + Tforeach(p, val) { if (!vct_istchar(*p)) return (H2SE_PROTOCOL_ERROR); } diff --git a/tools/coccinelle/tforeach.cocci b/tools/coccinelle/tforeach.cocci new file mode 100644 index 000000000..ea6e25301 --- /dev/null +++ b/tools/coccinelle/tforeach.cocci @@ -0,0 +1,14 @@ +/* This patch turns manual loops over a txt into foreach loops */ + +@@ +iterator name Tforeach; +typedef txt; +expression c; +txt t; +@@ + +- for (c = t.b; c < t.e; c++) ++ Tforeach(c, t) +{ + ... +} From dridi.boukelmoune at gmail.com Tue Sep 24 07:26:05 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 24 Sep 2024 07:26:05 +0000 (UTC) Subject: [master] f2c259681 vcc: Internal type for the reserved 'default' symbol Message-ID: <20240924072605.A715311A9ED@lists.varnish-cache.org> commit f2c259681e51e4c0ba0f9905490cb162d78b5bf0 Author: Dridi Boukelmoune Date: Wed Sep 18 18:46:16 2024 +0200 vcc: Internal type for the reserved 'default' symbol A new DEFAULT type identifies symbols called 'default' and types that can have a default symbol carry a pseudo symbol. This approach can reconcile the overloaded default symbols that all share the generic type DEFAULT but carry their respective kinds. This helps remove some of the special casing for default probes and backends whilst offering a sentinel DEFAULT type to safely deal with the special casing where it is actually needed, simplifying the code a wee bit in those areas. Fixes #4177 diff --git a/bin/varnishtest/tests/c00042.vtc b/bin/varnishtest/tests/c00042.vtc index a57ec7797..ed69de824 100644 --- a/bin/varnishtest/tests/c00042.vtc +++ b/bin/varnishtest/tests/c00042.vtc @@ -176,3 +176,9 @@ varnish v1 -errvcl "Cannot stack .via backends" { set bereq.backend = c; } } + +# issue #4177: backend named default with .via property +varnish v1 -vcl { + backend via { .host = "${localhost}"; } + backend default { .via = via; .host = "${localhost}"; } +} diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c index dc7a64b59..3cd4da265 100644 --- a/lib/libvcc/vcc_backend.c +++ b/lib/libvcc/vcc_backend.c @@ -40,19 +40,6 @@ #include "vcc_compile.h" #include "vus.h" -const char * -vcc_default_probe(struct vcc *tl) -{ - - if (tl->default_probe != NULL) - return (tl->default_probe); - VSB_cat(tl->sb, "No default probe defined\n"); - vcc_ErrToken(tl, tl->t); - VSB_cat(tl->sb, " at\n"); - vcc_ErrWhere(tl, tl->t); - return (""); -} - /*-------------------------------------------------------------------- * Struct sockaddr is not really designed to be a compile time * initialized data structure, so we encode it as a byte-string @@ -346,16 +333,16 @@ vcc_ParseProbe(struct vcc *tl) vcc_ExpectVid(tl, "backend probe"); /* ID: name */ ERRCHK(tl); - if (vcc_IdIs(tl->t, "default")) { - vcc_NextToken(tl); - vcc_ParseProbeSpec(tl, NULL, &p); + + sym = VCC_HandleSymbol(tl, PROBE); + ERRCHK(tl); + AN(sym); + vcc_ParseProbeSpec(tl, sym, &p); + + if (sym->type == DEFAULT) tl->default_probe = p; - } else { - sym = VCC_HandleSymbol(tl, PROBE); - ERRCHK(tl); - AN(sym); - vcc_ParseProbeSpec(tl, sym, NULL); - } + else + free(p); } /*-------------------------------------------------------------------- @@ -370,6 +357,7 @@ vcc_ParseHostDef(struct vcc *tl, struct symbol *sym, { const struct token *t_field; const struct token *t_val; + const struct token *t_probe; const struct token *t_host = NULL; const struct token *t_port = NULL; const struct token *t_path = NULL; @@ -526,16 +514,23 @@ vcc_ParseHostDef(struct vcc *tl, struct symbol *sym, free(p); ERRCHK(tl); } else if (vcc_IdIs(t_field, "probe") && tl->t->tok == ID) { - if (vcc_IdIs(tl->t, "default")) { - vcc_NextToken(tl); - (void)vcc_default_probe(tl); - } else { - pb = VCC_SymbolGet(tl, SYM_MAIN, SYM_PROBE, - SYMTAB_EXISTING, XREF_REF); - ERRCHK(tl); - AN(pb); - Fb(tl, 0, "\t.probe = %s,\n", pb->rname); + t_probe = tl->t; + pb = VCC_SymbolGet(tl, SYM_MAIN, SYM_PROBE, + SYMTAB_EXISTING, XREF_REF); + ERRCHK(tl); + AN(pb); + if (pb->type == DEFAULT) { + if (tl->default_probe == NULL) { + VSB_cat(tl->sb, + "No default probe defined\n"); + vcc_ErrToken(tl, t_probe); + VSB_cat(tl->sb, " at\n"); + vcc_ErrWhere(tl, t_probe); + } + pb = PROBE->default_sym; } + ERRCHK(tl); + Fb(tl, 0, "\t.probe = %s,\n", pb->rname); SkipToken(tl, ';'); } else if (vcc_IdIs(t_field, "probe")) { VSB_cat(tl->sb, "Expected '{' or name of probe, got "); @@ -720,7 +715,12 @@ vcc_ParseBackend(struct vcc *tl) ERRCHK(tl); t_be = tl->t; - if (vcc_IdIs(tl->t, "default")) { + + sym = VCC_HandleSymbol(tl, BACKEND); + ERRCHK(tl); + AN(sym); + + if (sym->type == DEFAULT) { if (tl->first_director != NULL) { tl->first_director->noref = 0; tl->first_director = NULL; @@ -732,13 +732,9 @@ vcc_ParseBackend(struct vcc *tl) vcc_ErrWhere(tl, t_first); return; } - vcc_NextToken(tl); dn = "vgc_backend_default"; tl->default_director = dn; } else { - sym = VCC_HandleSymbol(tl, BACKEND); - ERRCHK(tl); - AN(sym); dn = sym->rname; if (tl->default_director == NULL) { tl->first_director = sym; @@ -746,6 +742,7 @@ vcc_ParseBackend(struct vcc *tl) sym->noref = 1; } } + Fh(tl, 0, "\nstatic VCL_BACKEND %s;\n", dn); vcc_ParseHostDef(tl, sym, t_be, dn); if (tl->err) { diff --git a/lib/libvcc/vcc_compile.h b/lib/libvcc/vcc_compile.h index 93fb613c8..132553241 100644 --- a/lib/libvcc/vcc_compile.h +++ b/lib/libvcc/vcc_compile.h @@ -118,6 +118,8 @@ struct type { const char *global_pfx; const char *tostring; vcc_type_t multype; + struct symbol *default_sym; + int stringform; int bodyform; int noindent; @@ -126,6 +128,8 @@ struct type { #define VCC_TYPE(UC, lc) extern const struct type UC[1]; #include "vcc_types.h" +extern const struct type DEFAULT[1]; /* type for pseudo-symbol "default" */ + /*---------------------------------------------------------------------*/ typedef const struct kind *vcc_kind_t; @@ -317,7 +321,6 @@ void vcc_Action_Init(struct vcc *); /* vcc_backend.c */ struct fld_spec; -const char *vcc_default_probe(struct vcc *); void vcc_Backend_Init(struct vcc *tl); void vcc_ParseProbe(struct vcc *tl); void vcc_ParseBackend(struct vcc *tl); diff --git a/lib/libvcc/vcc_expr.c b/lib/libvcc/vcc_expr.c index 7475053aa..420898120 100644 --- a/lib/libvcc/vcc_expr.c +++ b/lib/libvcc/vcc_expr.c @@ -1606,14 +1606,13 @@ vcc_Eval_Default(struct vcc *tl, struct expr **e, struct token *t, (void)sym; (void)t; - if (fmt == PROBE) - *e = vcc_mk_expr(PROBE, "%s", vcc_default_probe(tl)); - else if (fmt == BACKEND) - *e = vcc_mk_expr(BACKEND, "*(VCL_conf.default_director)"); - else { + if (fmt->default_sym == NULL) { VSB_cat(tl->sb, "Symbol 'default' is a reserved word.\n"); vcc_ErrWhere(tl, t); + return; } + + *e = vcc_mk_expr(fmt, "%s", fmt->default_sym->rname); } /*-------------------------------------------------------------------- @@ -1650,6 +1649,6 @@ vcc_Expr_Init(struct vcc *tl) sym = VCC_MkSym(tl, "default", SYM_MAIN, SYM_FUNC, VCL_LOW, VCL_HIGH); AN(sym); - sym->type = BACKEND; // ... can also (sometimes) deliver PROBE + sym->type = DEFAULT; sym->eval = vcc_Eval_Default; } diff --git a/lib/libvcc/vcc_symb.c b/lib/libvcc/vcc_symb.c index fc273f2de..4d4462cd2 100644 --- a/lib/libvcc/vcc_symb.c +++ b/lib/libvcc/vcc_symb.c @@ -363,7 +363,7 @@ VCC_SymbolGet(struct vcc *tl, vcc_ns_t ns, vcc_kind_t kind, vcc_ErrWhere2(tl, t0, tl->t); return (NULL); } - if (kind != SYM_NONE && kind != sym->kind) { + if (kind != SYM_NONE && kind != sym->kind && sym->type != DEFAULT) { VSB_cat(tl->sb, "Symbol '"); vcc_PrintTokens(tl, t0, tl->t); VSB_printf(tl->sb, "' has wrong type (%s), expected %s:", @@ -577,6 +577,11 @@ VCC_HandleSymbol(struct vcc *tl, vcc_type_t fmt) struct token *t; const char *p; + if (vcc_IdIs(tl->t, "default") && fmt->default_sym != NULL) { + vcc_NextToken(tl); + return (fmt->default_sym); + } + kind = VCC_HandleKind(fmt); assert(kind != SYM_NONE); diff --git a/lib/libvcc/vcc_types.c b/lib/libvcc/vcc_types.c index cfa6714b3..ff0b4e4d4 100644 --- a/lib/libvcc/vcc_types.c +++ b/lib/libvcc/vcc_types.c @@ -67,12 +67,23 @@ static const struct vcc_method backend_methods[] = { { VCC_METHOD_MAGIC, NULL }, }; +static struct symbol default_backend[1] = {{ + .magic = SYMBOL_MAGIC, + .name = "default", + .lorev = 0, + .hirev = 99, + .kind = SYM_BACKEND, + .type = DEFAULT, + .rname = "*(VCL_conf.default_director)", +}}; + const struct type BACKEND[1] = {{ .magic = TYPE_MAGIC, .name = "BACKEND", .methods = backend_methods, .global_pfx = "vgc_backend", .tostring = "VRT_BACKEND_string(\v1)", + .default_sym = default_backend, }}; const struct type BLOB[1] = {{ @@ -101,6 +112,11 @@ const struct type BYTES[1] = {{ .multype = REAL, // XXX: wrong }}; +const struct type DEFAULT[1] = {{ + .magic = TYPE_MAGIC, + .name = "DEFAULT", +}}; + const struct type DURATION[1] = {{ .magic = TYPE_MAGIC, .name = "DURATION", @@ -144,10 +160,21 @@ const struct type IP[1] = {{ .tostring = "VRT_IP_string(ctx, \v1)", }}; +static struct symbol default_probe[1] = {{ + .magic = SYMBOL_MAGIC, + .name = "default", + .lorev = 0, + .hirev = 99, + .kind = SYM_PROBE, + .type = DEFAULT, + .rname = "vgc_probe_default", +}}; + const struct type PROBE[1] = {{ .magic = TYPE_MAGIC, .name = "PROBE", .global_pfx = "vgc_probe", + .default_sym = default_probe, }}; const struct type REAL[1] = {{ From dridi.boukelmoune at gmail.com Tue Sep 24 07:58:04 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 24 Sep 2024 07:58:04 +0000 (UTC) Subject: [master] 45faed232 vcc_types: Use VCL_{LOW,HIGH} for default symbols Message-ID: <20240924075804.D968611BE15@lists.varnish-cache.org> commit 45faed23247471b392359dc66f26e131510fc15b Author: Dridi Boukelmoune Date: Tue Sep 24 09:54:40 2024 +0200 vcc_types: Use VCL_{LOW,HIGH} for default symbols I shouldn't have picked the magic values from vmodtool.py in the first place. Refs #4190 diff --git a/lib/libvcc/vcc_types.c b/lib/libvcc/vcc_types.c index ff0b4e4d4..3a3c1dd21 100644 --- a/lib/libvcc/vcc_types.c +++ b/lib/libvcc/vcc_types.c @@ -70,8 +70,8 @@ static const struct vcc_method backend_methods[] = { static struct symbol default_backend[1] = {{ .magic = SYMBOL_MAGIC, .name = "default", - .lorev = 0, - .hirev = 99, + .lorev = VCL_LOW, + .hirev = VCL_HIGH, .kind = SYM_BACKEND, .type = DEFAULT, .rname = "*(VCL_conf.default_director)", @@ -163,8 +163,8 @@ const struct type IP[1] = {{ static struct symbol default_probe[1] = {{ .magic = SYMBOL_MAGIC, .name = "default", - .lorev = 0, - .hirev = 99, + .lorev = VCL_LOW, + .hirev = VCL_HIGH, .kind = SYM_PROBE, .type = DEFAULT, .rname = "vgc_probe_default", From dridi.boukelmoune at gmail.com Fri Sep 27 06:03:11 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 27 Sep 2024 06:03:11 +0000 (UTC) Subject: [master] 351cac103 vtc: Simplify c34 Message-ID: <20240927060311.EAE6C1113C1@lists.varnish-cache.org> commit 351cac103c2d4c09feae870c65f845a8ce9829c3 Author: Dridi Boukelmoune Date: Fri Sep 27 08:00:41 2024 +0200 vtc: Simplify c34 Refs #4137 diff --git a/bin/varnishtest/tests/c00034.vtc b/bin/varnishtest/tests/c00034.vtc index 1331c265d..86152e004 100644 --- a/bin/varnishtest/tests/c00034.vtc +++ b/bin/varnishtest/tests/c00034.vtc @@ -274,16 +274,8 @@ varnish v1 -vcl+backend { } } - # skip built-in vcl_beresp_range - sub vcl_backend_response { - if (bereq.uncacheable) { - return (deliver); - } - call vcl_beresp_stale; - call vcl_beresp_cookie; - call vcl_beresp_control; - call vcl_beresp_vary; - return (deliver); + sub vcl_beresp_range { + return; } } From dridi.boukelmoune at gmail.com Fri Sep 27 06:25:06 2024 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Fri, 27 Sep 2024 06:25:06 +0000 (UTC) Subject: [master] 5dfbc6e2a vsm: Remove PID 1 expectation Message-ID: <20240927062506.38DCA1120BF@lists.varnish-cache.org> commit 5dfbc6e2a0532460522da6868e11762d26a6c3ea Author: Dridi Boukelmoune Date: Fri Sep 27 08:19:31 2024 +0200 vsm: Remove PID 1 expectation It turns out varnishd can be the main process of a Docker container when another process like varnishlog is 'docker exec'uted in the same PID namespace. Spotted by @gquintard. diff --git a/lib/libvarnishapi/vsm.c b/lib/libvarnishapi/vsm.c index 39b601de4..e8a667eb4 100644 --- a/lib/libvarnishapi/vsm.c +++ b/lib/libvarnishapi/vsm.c @@ -494,7 +494,6 @@ vsm_running(struct vsm_set *vs, pid_t pid) if (pid == 0) return (0); - assert(pid > 1); if (kill(pid, 0) == 0) { vs->couldkill = 1; From nils.goroll at uplex.de Sat Sep 28 12:08:08 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 28 Sep 2024 12:08:08 +0000 (UTC) Subject: [master] e89d3e67d Include cleanup (as reported by flexelint locally) Message-ID: <20240928120808.246D068F7@lists.varnish-cache.org> commit e89d3e67d03617309bc541dc473aafaeaf3954e5 Author: Nils Goroll Date: Sat Sep 28 13:52:14 2024 +0200 Include cleanup (as reported by flexelint locally) diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 3b4fa2fa8..502e10310 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -46,7 +46,6 @@ #include "cache_varnishd.h" #include "vbh.h" -#include "vcli_serve.h" #include "vsa.h" #include "vtcp.h" #include "vtim.h" diff --git a/bin/varnishd/cache/cache_conn_pool.c b/bin/varnishd/cache/cache_conn_pool.c index b865c3f94..9e940d42d 100644 --- a/bin/varnishd/cache/cache_conn_pool.c +++ b/bin/varnishd/cache/cache_conn_pool.c @@ -40,7 +40,6 @@ #include "vsa.h" #include "vsha256.h" #include "vtcp.h" -#include "vtree.h" #include "vus.h" #include "vtim.h" #include "waiter/waiter.h" diff --git a/bin/varnishd/mgt/mgt_jail_linux.c b/bin/varnishd/mgt/mgt_jail_linux.c index 93e5c2a6d..d344d7c76 100644 --- a/bin/varnishd/mgt/mgt_jail_linux.c +++ b/bin/varnishd/mgt/mgt_jail_linux.c @@ -42,7 +42,6 @@ #include #include #include -#include #include #include "mgt/mgt.h" From nils.goroll at uplex.de Sat Sep 28 12:10:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 28 Sep 2024 12:10:05 +0000 (UTC) Subject: [master] 9aa6fe768 Flexelinting Message-ID: <20240928121005.858666BE5@lists.varnish-cache.org> commit 9aa6fe76892521e62a98279a51b7a668a0f44567 Author: Nils Goroll Date: Sat Sep 28 14:08:16 2024 +0200 Flexelinting diff --git a/lib/libvcc/vcc_backend.c b/lib/libvcc/vcc_backend.c index 3cd4da265..f0641d25a 100644 --- a/lib/libvcc/vcc_backend.c +++ b/lib/libvcc/vcc_backend.c @@ -704,7 +704,7 @@ void vcc_ParseBackend(struct vcc *tl) { struct token *t_first, *t_be; - struct symbol *sym = NULL; + struct symbol *sym; const char *dn; tl->ndirector++; From nils.goroll at uplex.de Sat Sep 28 12:10:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 28 Sep 2024 12:10:05 +0000 (UTC) Subject: [master] eb8a2e976 Silence Flexelint Message-ID: <20240928121005.9AD706BE7@lists.varnish-cache.org> commit eb8a2e9762b797c7485aacca43034c8f996a5a23 Author: Nils Goroll Date: Sat Sep 28 14:09:27 2024 +0200 Silence Flexelint diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c index 549ffdad8..bb126ec56 100644 --- a/bin/varnishtest/vtc_log.c +++ b/bin/varnishtest/vtc_log.c @@ -119,7 +119,7 @@ vtc_logfail(void) else exit(fail_out()); - WRONG("unreachable"); /* Help Coverity Scan see noreturn. */ + WRONG("unreachable"); /*lint !e527 Help Coverity Scan see noreturn. */ } static const char * const lead[] = { From nils.goroll at uplex.de Sat Sep 28 12:20:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Sat, 28 Sep 2024 12:20:05 +0000 (UTC) Subject: [master] d826fb8e6 typo Message-ID: <20240928122005.898C075D7@lists.varnish-cache.org> commit d826fb8e639664ce9b37f10fa353f96e15d6689e Author: Nils Goroll Date: Sat Sep 28 14:19:23 2024 +0200 typo diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c index bb126ec56..f96dc9690 100644 --- a/bin/varnishtest/vtc_log.c +++ b/bin/varnishtest/vtc_log.c @@ -119,7 +119,7 @@ vtc_logfail(void) else exit(fail_out()); - WRONG("unreachable"); /*lint !e527 Help Coverity Scan see noreturn. */ + WRONG("unreachable"); /*lint !e827 Help Coverity Scan see noreturn. */ } static const char * const lead[] = { From nils.goroll at uplex.de Mon Sep 30 13:38:07 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 13:38:07 +0000 (UTC) Subject: [master] dbd1cdfb2 backend.list -j now outputs IPs/port information Message-ID: <20240930133807.632FD1164D5@lists.varnish-cache.org> commit dbd1cdfb25b6f80f9b4c3e014079e7fbccc006f5 Author: Guillaume Quintard Date: Tue Sep 24 05:48:13 2024 -0400 backend.list -j now outputs IPs/port information diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index fe7e4f4e1..922a4bb13 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -624,12 +624,17 @@ static void v_matchproto_(vdi_list_f) vbe_list(VRT_CTX, const struct director *d, struct vsb *vsb, int pflag, int jflag) { + char buf[VTCP_ADDRBUFSIZE]; struct backend *bp; + struct vrt_endpoint *vep; (void)ctx; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC); + CHECK_OBJ_NOTNULL(bp->endpoint, VRT_ENDPOINT_MAGIC); + + vep = bp->endpoint; if (bp->probe != NULL) VBP_Status(vsb, bp, pflag, jflag); @@ -641,6 +646,17 @@ vbe_list(VRT_CTX, const struct director *d, struct vsb *vsb, int pflag, return; else VSB_cat(vsb, "0/0\thealthy"); + + if (jflag && pflag) { + if (vep->ipv4 != NULL) { + VTCP_name(vep->ipv4, buf, sizeof buf, NULL, 0); + VSB_printf(vsb, "\"ipv4\": \"%s\",\n", buf); + } + if (vep->ipv6 != NULL) { + VTCP_name(vep->ipv6, buf, sizeof buf, NULL, 0); + VSB_printf(vsb, "\"ipv6\": \"%s\",\n", buf); + } + } } /*-------------------------------------------------------------------- diff --git a/bin/varnishtest/tests/b00091.vtc b/bin/varnishtest/tests/b00091.vtc new file mode 100644 index 000000000..9bc1b98ea --- /dev/null +++ b/bin/varnishtest/tests/b00091.vtc @@ -0,0 +1,41 @@ +varnishtest "IPs/port in varnishadm backend.list -j" + +feature cmd "jq -V" + +server s0 { +} -start + + +varnish v1 -vcl+backend { + backend s1 { + .host = "127.0.0.1"; + .port = "2222"; + .probe = { .url = "/"; } + } + backend s2 { + .host = "::1:2:3:4"; + .port = "3333"; + } + + sub vcl_recv { + set req.backend_hint = s0; + set req.backend_hint = s1; + set req.backend_hint = s2; + } +} -start + +shell { + set -e + +varnishadm -n ${v1_name} backend.list -j + varnishadm -n ${v1_name} backend.list -jp > result.json + jq ' + .[3]["vcl1.s0"].ipv4 == "${s0_addr}" or error("wrong s0 ipv4"), + .[3]["vcl1.s1"].ipv4 == "127.0.0.1" or error("wrong s2 ipv4"), + .[3]["vcl1.s2"].ipv4 == null or error("s3 ipv4 exists"), + + .[3]["vcl1.s0"].ipv6 == null or error("s0 ipv6 exists"), + .[3]["vcl1.s1"].ipv6 == null or error("s2 ipv6 exists"), + .[3]["vcl1.s2"].ipv6 == "::1:2:3:4" or error("wrong s3 ipv6") + ' result.json +} From nils.goroll at uplex.de Mon Sep 30 14:06:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 14:06:05 +0000 (UTC) Subject: [master] c6e4b2807 Add an event for directors going sick Message-ID: <20240930140605.91AF5117DD7@lists.varnish-cache.org> commit c6e4b2807afae3ef8d4f62d8ca83c082b61eef4a Author: Nils Goroll Date: Mon Sep 23 19:19:10 2024 +0200 Add an event for directors going sick Pondering solutions for #4183 it became apparent that we need a way to notify directors of changes to the admin_health without crossing the VDI/VBE layering. Besides adding another director callback, one option is to add an event to the existing event facility, which so far was only used for VCL-related events. We now add VDI_EVENT_SICK as a director-specific event, which is cheap and also helps us maintain clean layering. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 922a4bb13..99d10ed70 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -561,6 +561,11 @@ vbe_dir_event(const struct director *d, enum vcl_event_e ev) VRT_VSC_Hide(bp->vsc_seg); } else if (ev == VCL_EVENT_DISCARD) { VRT_DelDirector(&bp->director); + } else if (ev == VDI_EVENT_SICK) { + const struct vdi_ahealth *ah = d->vdir->admin_health; + + if (ah == VDI_AH_SICK || (ah == VDI_AH_AUTO && bp->sick)) + VBE_connwait_signal_all(bp); } } @@ -677,7 +682,6 @@ vbe_healthy(VRT_CTX, VCL_BACKEND d, VCL_TIME *t) return (!bp->sick); } - /*-------------------------------------------------------------------- */ diff --git a/include/vrt.h b/include/vrt.h index 144a7e2ed..13c93be12 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -48,7 +48,7 @@ #define VRT_MAJOR_VERSION 20U -#define VRT_MINOR_VERSION 0U +#define VRT_MINOR_VERSION 1U /*********************************************************************** * Major and minor VRT API versions. @@ -58,6 +58,8 @@ * binary/load-time compatible, increment MAJOR version * * NEXT (2024-03-15) + * 20.1 + * VDI_EVENT_SICK added to enum vcl_event_e * 20.0 (2024-09-13) * struct vrt_backend.backend_wait_timeout added * struct vrt_backend.backend_wait_limit added @@ -494,7 +496,7 @@ struct vmod_data { }; /*********************************************************************** - * VCL events sent to VMODs + * VCL events sent to VMODs and directors */ enum vcl_event_e { @@ -502,6 +504,8 @@ enum vcl_event_e { VCL_EVENT_WARM, VCL_EVENT_COLD, VCL_EVENT_DISCARD, + // Only for directors + VDI_EVENT_SICK, }; typedef int vmod_event_f(VRT_CTX, struct vmod_priv *, enum vcl_event_e); From nils.goroll at uplex.de Mon Sep 30 14:06:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 14:06:05 +0000 (UTC) Subject: [master] d6b6602f7 Use VDI_EVENT_SICK when the admin health changes Message-ID: <20240930140605.B05AF117DD9@lists.varnish-cache.org> commit d6b6602f70c684f023de6d0c01ec9759b0dfd8cd Author: Nils Goroll Date: Mon Sep 23 19:19:16 2024 +0200 Use VDI_EVENT_SICK when the admin health changes Using the new event, we can now selectively notify interested backend types (so far: VBE only) when the admin health changes. This fixes the layer violation introduced with e46f97278f1036f4e648dd64d3f34ada8d29f64a, where a director private pointer was used with a VBE specific function. Fixes #4183 diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 99d10ed70..29084bd2a 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -139,16 +139,8 @@ VBE_Connect_Error(struct VSC_vbe *vsc, int err) /*--------------------------------------------------------------------*/ -int -VBE_is_ah_auto(const struct backend *bp) -{ - - CHECK_OBJ_NOTNULL(bp, BACKEND_MAGIC); - return (bp->director->vdir->admin_health == VDI_AH_AUTO); -} - -void -VBE_connwait_signal_all(const struct backend *bp) +static void +vbe_connwait_signal_all(const struct backend *bp) { struct connwait *cw; @@ -565,7 +557,7 @@ vbe_dir_event(const struct director *d, enum vcl_event_e ev) const struct vdi_ahealth *ah = d->vdir->admin_health; if (ah == VDI_AH_SICK || (ah == VDI_AH_AUTO && bp->sick)) - VBE_connwait_signal_all(bp); + vbe_connwait_signal_all(bp); } } diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 502e10310..501ddb64a 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -190,10 +190,8 @@ VBP_Update_Backend(struct vbp_target *vt) i = (vt->good < vt->threshold); chg = (i != vt->backend->sick); vt->backend->sick = i; - if (i && chg && (vt->backend->director != NULL && - VBE_is_ah_auto(vt->backend))) { - VBE_connwait_signal_all(vt->backend); - } + if (i && chg && vt->backend->director != NULL) + VDI_Event(vt->backend->director, VDI_EVENT_SICK); AN(vt->backend->vcl_name); VSL(SLT_Backend_health, NO_VXID, diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index 042ad9f34..ea1a4efb9 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -477,7 +477,6 @@ static int v_matchproto_(vcl_be_func) do_set_health(struct cli *cli, struct director *d, void *priv) { struct set_health *sh; - struct vrt_ctx *ctx; (void)cli; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); @@ -488,12 +487,7 @@ do_set_health(struct cli *cli, struct director *d, void *priv) if (d->vdir->admin_health != sh->ah) { d->vdir->health_changed = VTIM_real(); d->vdir->admin_health = sh->ah; - ctx = VCL_Get_CliCtx(0); - if (sh->ah == VDI_AH_SICK || (sh->ah == VDI_AH_AUTO && - d->vdir->methods->healthy != NULL && - !d->vdir->methods->healthy(ctx, d, NULL))) { - VBE_connwait_signal_all(d->priv); - } + VDI_Event(d, VDI_EVENT_SICK); } return (0); } diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index e0b88b4aa..e84f90e0f 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -161,11 +161,7 @@ void VCA_Init(void); void VCA_Shutdown(void); /* cache_backend.c */ -struct backend; - void VBE_InitCfg(void); -void VBE_connwait_signal_all(const struct backend *bp); -int VBE_is_ah_auto(const struct backend *bp); /* cache_ban.c */ diff --git a/include/vrt.h b/include/vrt.h index 13c93be12..9d7d90c14 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -496,7 +496,9 @@ struct vmod_data { }; /*********************************************************************** - * VCL events sent to VMODs and directors + * Events sent to VMODs and directors: + * - VCL: VCL temperature events + * - VDI: director events */ enum vcl_event_e { @@ -504,7 +506,7 @@ enum vcl_event_e { VCL_EVENT_WARM, VCL_EVENT_COLD, VCL_EVENT_DISCARD, - // Only for directors + VDI_EVENT_SICK, }; From nils.goroll at uplex.de Mon Sep 30 14:06:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 14:06:05 +0000 (UTC) Subject: [master] e3d1e2914 Function rename suggested by Dridi Message-ID: <20240930140605.CAC8C117DDE@lists.varnish-cache.org> commit e3d1e2914bd6a319b06820347e55ba880447ffce Author: Nils Goroll Date: Mon Sep 23 19:22:50 2024 +0200 Function rename suggested by Dridi diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 29084bd2a..977933772 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -140,7 +140,7 @@ VBE_Connect_Error(struct VSC_vbe *vsc, int err) /*--------------------------------------------------------------------*/ static void -vbe_connwait_signal_all(const struct backend *bp) +vbe_connwait_broadcast(const struct backend *bp) { struct connwait *cw; @@ -557,7 +557,7 @@ vbe_dir_event(const struct director *d, enum vcl_event_e ev) const struct vdi_ahealth *ah = d->vdir->admin_health; if (ah == VDI_AH_SICK || (ah == VDI_AH_AUTO && bp->sick)) - vbe_connwait_signal_all(bp); + vbe_connwait_broadcast(bp); } } From nils.goroll at uplex.de Mon Sep 30 14:19:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 14:19:06 +0000 (UTC) Subject: [master] 4a4a232d7 Declare VDP_ObjIterate extern Message-ID: <20240930141906.F0A25118A1F@lists.varnish-cache.org> commit 4a4a232d737691ec4f38ea7d03cc0bf6296e7325 Author: Nils Goroll Date: Thu Dec 28 10:48:09 2023 +0100 Declare VDP_ObjIterate extern diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index f824d2059..d9c20603a 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -249,8 +249,8 @@ VDP_Close(struct vdp_ctx *vdc, struct objcore *oc, struct boc *boc) /*--------------------------------------------------------------------*/ -static int v_matchproto_(objiterate_f) -vdp_objiterate(void *priv, unsigned flush, const void *ptr, ssize_t len) +int v_matchproto_(objiterate_f) +VDP_ObjIterate(void *priv, unsigned flush, const void *ptr, ssize_t len) { enum vdp_action act; @@ -278,7 +278,7 @@ VDP_DeliverObj(struct vdp_ctx *vdc, struct objcore *oc) vdc->hp = NULL; vdc->clen = NULL; final = oc->flags & OC_F_TRANSIENT ? 1 : 0; - r = ObjIterate(vdc->wrk, oc, vdc, vdp_objiterate, final); + r = ObjIterate(vdc->wrk, oc, vdc, VDP_ObjIterate, final); if (r < 0) return (r); return (0); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index e84f90e0f..7f876290a 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -194,6 +194,7 @@ uint64_t VDP_Close(struct vdp_ctx *, struct objcore *, struct boc *); void VDP_Panic(struct vsb *vsb, const struct vdp_ctx *vdc); int VDP_Push(VRT_CTX, struct vdp_ctx *, struct ws *, const struct vdp *, void *priv); +int VDP_ObjIterate(void *priv, unsigned flush, const void *ptr, ssize_t len); int VDP_DeliverObj(struct vdp_ctx *vdc, struct objcore *oc); extern const struct vdp VDP_gunzip; extern const struct vdp VDP_esi; From nils.goroll at uplex.de Mon Sep 30 14:19:07 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 14:19:07 +0000 (UTC) Subject: [master] c92ac8bb8 Use VDP for bereq.body Message-ID: <20240930141907.1D54C118A22@lists.varnish-cache.org> commit c92ac8bb818691fae0ca9d96fd36b8711dc3b743 Author: Nils Goroll Date: Wed Dec 27 10:36:33 2023 +0100 Use VDP for bereq.body This change introduces additional failure points to V1F_SendReq() when there is insufficient workspace to push the V1L VDP. We adjust r01834.vtc to cover the respective 503 errors and simplify it by example of r02645.vtc. The two test cases now resemble each other a lot, but I would think, at least for now, their purpose is so important that the overhead does not matter. diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c index 99075c9fe..d2859e930 100644 --- a/bin/varnishd/http1/cache_http1_fetch.c +++ b/bin/varnishd/http1/cache_http1_fetch.c @@ -42,25 +42,14 @@ #include "cache_http1.h" -/*-------------------------------------------------------------------- - * Pass the request body to the backend - */ - -static int v_matchproto_(objiterate_f) -vbf_iter_req_body(void *priv, unsigned flush, const void *ptr, ssize_t l) +static int +v1f_stackv1l(struct vdp_ctx *vdc, struct busyobj *bo) { - struct busyobj *bo; - - CAST_OBJ_NOTNULL(bo, priv, BUSYOBJ_MAGIC); + struct vrt_ctx ctx[1]; - if (l > 0) { - if (DO_DEBUG(DBG_SLOW_BEREQ)) - VTIM_sleep(1.0); - (void)V1L_Write(bo->wrk, ptr, l); - if (flush && V1L_Flush(bo->wrk) != SC_NULL) - return (-1); - } - return (0); + INIT_OBJ(ctx, VRT_CTX_MAGIC); + VCL_Bo2Ctx(ctx, bo); + return (VDP_Push(ctx, vdc, ctx->ws, VDP_v1l, NULL)); } /*-------------------------------------------------------------------- @@ -80,7 +69,8 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes, ssize_t i; uint64_t bytes, hdrbytes; struct http_conn *htc; - int do_chunked = 0; + struct vdp_ctx vdc[1]; + intmax_t cl; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); @@ -93,13 +83,35 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes, assert(*htc->rfd > 0); hp = bo->bereq; - if (bo->req != NULL && !bo->req->req_body_status->length_known) { - http_PrintfHeader(hp, "Transfer-Encoding: chunked"); - do_chunked = 1; + if (bo->bereq_body != NULL) + cl = ObjGetLen(wrk, bo->bereq_body); + else if (bo->req != NULL && !bo->req->req_body_status->length_known) + cl = -1; + else if (bo->req != NULL) { + cl = http_GetContentLength(bo->req->http); + if (cl < 0) + cl = 0; } + else + cl = 0; + + VDP_Init(vdc, wrk, bo->vsl, NULL, bo, &cl); + + if (v1f_stackv1l(vdc, bo)) { + VSLb(bo->vsl, SLT_FetchError, "Failure to push V1L"); + VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk)); + (void) VDP_Close(vdc, NULL, NULL); + htc->doclose = SC_OVERLOAD; + return (-1); + } + + assert(cl >= -1); + if (cl < 0) + http_PrintfHeader(hp, "Transfer-Encoding: chunked"); VTCP_blocking(*htc->rfd); /* XXX: we should timeout instead */ /* XXX: need a send_timeout for the backend side */ + // XXX cache_param->http1_iovs ? V1L_Open(wrk, wrk->aws, htc->rfd, bo->vsl, nan(""), 0); hdrbytes = HTTP1_Write(wrk, hp, HTTP1_Req); @@ -108,16 +120,14 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes, if (bo->bereq_body != NULL) { AZ(bo->req); - AZ(do_chunked); + assert(cl >= 0); (void)ObjIterate(bo->wrk, bo->bereq_body, - bo, vbf_iter_req_body, 0); + vdc, VDP_ObjIterate, 0); } else if (bo->req != NULL && bo->req->req_body_status != BS_NONE) { - if (DO_DEBUG(DBG_FLUSH_HEAD)) - (void)V1L_Flush(wrk); - if (do_chunked) + if (cl < 0) V1L_Chunked(wrk); - i = VRB_Iterate(wrk, bo->vsl, bo->req, vbf_iter_req_body, bo); + i = VRB_Iterate(wrk, bo->vsl, bo->req, VDP_ObjIterate, vdc); if (bo->req->req_body_status != BS_CACHED) bo->no_retry = "req.body not cached"; @@ -138,7 +148,7 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes, errno, VAS_errtxt(errno)); bo->req->doclose = SC_RX_BODY; } - if (do_chunked) + if (cl < 0) V1L_EndChunk(wrk); } @@ -146,12 +156,8 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes, CHECK_OBJ_NOTNULL(sc, STREAM_CLOSE_MAGIC); /* Bytes accounting */ - if (bytes < hdrbytes) - *ctr_hdrbytes += bytes; - else { - *ctr_hdrbytes += hdrbytes; - *ctr_bodybytes += bytes - hdrbytes; - } + *ctr_hdrbytes += hdrbytes; + *ctr_bodybytes += VDP_Close(vdc, NULL, NULL); if (sc == SC_NULL && i < 0) sc = SC_TX_ERROR; diff --git a/bin/varnishtest/tests/r01834.vtc b/bin/varnishtest/tests/r01834.vtc index 33437a9b2..b1f689e50 100644 --- a/bin/varnishtest/tests/r01834.vtc +++ b/bin/varnishtest/tests/r01834.vtc @@ -3,9 +3,13 @@ varnishtest "#1834 - Buffer overflow in backend workspace" # This test case checks fetch side for buffer overflow when there is little # workspace left. If failing it would be because we tripped the canary # at the end of the workspace. +# +# status codes are not so important here, we just should not panic +# +# XXX almost the same as r02645.vtc - -server s1 -repeat 64 { +server s1 -repeat 40 { + non_fatal rxreq txresp } -start @@ -19,95 +23,30 @@ varnish v1 -vcl+backend { } sub vcl_backend_fetch { - vtc.workspace_alloc(backend, -1 * std.integer(bereq.http.WS, 256)); + # this relies on deterministic xids. + # first bereq is xid == 1002 + vtc.workspace_alloc(backend, -320 + + 8 * ((bereq.xid - 1002) / 3)); } } -start -client c1 { - # Start with enough workspace to receive a good result - txreq -hdr "WS: 320" +# Start with enough workspace to receive some good results +client c1 -repeat 5 { + txreq -hdr "Connection: close" rxresp expect resp.status == 200 +} -run - # Continue with decreasing workspaces by decrements of 8 (sizeof void*) - txreq -hdr "WS: 312" - rxresp - txreq -hdr "WS: 304" - rxresp - txreq -hdr "WS: 296" - rxresp - txreq -hdr "WS: 288" - rxresp - txreq -hdr "WS: 280" - rxresp - txreq -hdr "WS: 272" - rxresp - txreq -hdr "WS: 264" - rxresp - txreq -hdr "WS: 256" - rxresp - txreq -hdr "WS: 248" - rxresp - txreq -hdr "WS: 240" - rxresp - txreq -hdr "WS: 232" - rxresp - txreq -hdr "WS: 224" - rxresp - txreq -hdr "WS: 216" - rxresp - txreq -hdr "WS: 208" - rxresp - txreq -hdr "WS: 200" - rxresp - txreq -hdr "WS: 192" - rxresp - txreq -hdr "WS: 184" - rxresp - txreq -hdr "WS: 176" - rxresp - txreq -hdr "WS: 168" - rxresp - txreq -hdr "WS: 160" +# tolerance region where we do not care about the return code +# failures should start at around xid 1042 (repeat 10) +client c1 -repeat 15 { + txreq -hdr "Connection: close" rxresp - txreq -hdr "WS: 152" - rxresp - txreq -hdr "WS: 144" - rxresp - txreq -hdr "WS: 136" - rxresp - txreq -hdr "WS: 128" - rxresp - txreq -hdr "WS: 120" - rxresp - txreq -hdr "WS: 112" - rxresp - txreq -hdr "WS: 104" - rxresp - txreq -hdr "WS: 096" - rxresp - txreq -hdr "WS: 088" - rxresp - txreq -hdr "WS: 080" - rxresp - txreq -hdr "WS: 072" - rxresp - txreq -hdr "WS: 064" - rxresp - txreq -hdr "WS: 056" - rxresp - txreq -hdr "WS: 048" - rxresp - txreq -hdr "WS: 040" - rxresp - txreq -hdr "WS: 032" - rxresp - txreq -hdr "WS: 024" - rxresp - txreq -hdr "WS: 016" - rxresp - txreq -hdr "WS: 008" - rxresp - txreq -hdr "WS: 000" +} -run + +# final must all fail. +client c1 -repeat 20 { + txreq -hdr "Connection: close" rxresp + expect resp.status == 503 } -run diff --git a/bin/varnishtest/tests/r02645.vtc b/bin/varnishtest/tests/r02645.vtc index 982c50a96..2f6a86d7d 100644 --- a/bin/varnishtest/tests/r02645.vtc +++ b/bin/varnishtest/tests/r02645.vtc @@ -1,6 +1,9 @@ varnishtest "sweep through tight backend workspace conditions" +# XXX almost the same as r01834.vtc - retire? + server s1 -repeat 100 { + non_fatal rxreq send "HTTP/1.1 200 OK\r\nTransfer-encoding: chunked\r\n\r\n00000004\r\n1234\r\n00000000\r\n\r\n" } -start From nils.goroll at uplex.de Mon Sep 30 14:19:07 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 14:19:07 +0000 (UTC) Subject: [master] 00b190a0a Define (be)?req.filters fields and adjust VRT Message-ID: <20240930141907.4DE69118A27@lists.varnish-cache.org> commit 00b190a0aed482a638fdb9f1207598cca3db6c27 Author: Nils Goroll Date: Mon Dec 11 08:00:35 2023 +0100 Define (be)?req.filters fields and adjust VRT diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 9c40faffb..b52cbab01 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -394,8 +394,11 @@ struct busyobj { struct sess *sp; struct worker *wrk; + /* beresp.body */ struct vfp_ctx *vfc; const char *vfp_filter_list; + /* bereq.body */ + const char *vdp_filter_list; struct ws ws[1]; uintptr_t ws_bo; @@ -537,9 +540,11 @@ struct req { struct objcore *objcore; struct objcore *stale_oc; - /* Deliver pipeline */ + /* resp.body */ struct vdp_ctx *vdc; const char *vdp_filter_list; + /* req.body */ + const char *vfp_filter_list; /* Transaction VSL buffer */ struct vsl_log vsl[1]; diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index c5f4788a3..eef6cb527 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -377,6 +377,10 @@ pan_busyobj(struct vsb *vsb, const struct busyobj *bo) VSB_printf(vsb, "vfp_filter_list = \"%s\",\n", bo->vfp_filter_list); } + if (bo->vdp_filter_list != NULL) { + VSB_printf(vsb, "vdp_filter_list = \"%s\",\n", + bo->vdp_filter_list); + } WS_Panic(vsb, bo->ws); VSB_printf(vsb, "ws_bo = %p,\n", (void *)bo->ws_bo); @@ -464,6 +468,15 @@ pan_req(struct vsb *vsb, const struct req *req) else VSB_printf(vsb, "step = %s\n", req->req_step->name); + if (req->vfp_filter_list != NULL) { + VSB_printf(vsb, "vfp_filter_list = \"%s\",\n", + req->vfp_filter_list); + } + if (req->vdp_filter_list != NULL) { + VSB_printf(vsb, "vdp_filter_list = \"%s\",\n", + req->vdp_filter_list); + } + VSB_printf(vsb, "req_body = %s,\n", req->req_body_status ? req->req_body_status->name : "NULL"); diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index a9c20250d..42e2b93b0 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -252,6 +252,7 @@ Req_Rollback(VRT_CTX) VCL_TaskEnter(req->top->privs); HTTP_Clone(req->http, req->http0); req->vdp_filter_list = NULL; + req->vfp_filter_list = NULL; req->vcf = NULL; if (WS_Overflowed(req->ws)) req->wrk->stats->ws_client_overflow++; diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 20e4eb13b..9dfebfd99 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -909,6 +909,8 @@ cnt_recv_prep(struct req *req, const char *ci) req->is_hitpass = 0; req->err_code = 0; req->err_reason = NULL; + + req->vfp_filter_list = NULL; } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c index 4d777d464..707533eea 100644 --- a/bin/varnishd/cache/cache_vrt_filter.c +++ b/bin/varnishd/cache/cache_vrt_filter.c @@ -400,6 +400,14 @@ VBF_Get_Filter_List(struct busyobj *bo) return (filter_on_ws(bo->ws, vbf_default_filter_list, bo)); } +static const char * +bereq_Empty_Filter(struct busyobj *bo) +{ + + (void)bo; + return (""); +} + /*-------------------------------------------------------------------- */ @@ -433,9 +441,25 @@ resp_Get_Filter_List(struct req *req) return (filter_on_ws(req->ws, resp_default_filter_list, req)); } +static const char * +req_Empty_Filter(struct req *req) +{ + + (void)req; + return (""); +} + +/*--------------------------------------------------------------------*/ +static int +req_filter_can(struct req *req) { + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + + return (req->req_body_status->avail == 1); +} + /*--------------------------------------------------------------------*/ -#define FILTER_VAR(vcl, in, func, fld) \ +#define FILTER_VAR(vcl, in, func, cond, fld) \ VCL_STRING \ VRT_r_##vcl##_filters(VRT_CTX) \ { \ @@ -453,6 +477,10 @@ resp_Get_Filter_List(struct req *req) \ (void)str; \ CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); \ + if (! (cond)) { \ + VRT_fail(ctx, #vcl ".filters not settable"); \ + return; \ + } \ b = VRT_StrandsWS(ctx->in->ws, str, s); \ if (b == NULL) \ WS_MarkOverflow(ctx->in->ws); \ @@ -460,5 +488,7 @@ resp_Get_Filter_List(struct req *req) ctx->in->fld = b; \ } -FILTER_VAR(beresp, bo, VBF_Get_Filter_List, vfp_filter_list) -FILTER_VAR(resp, req, resp_Get_Filter_List, vdp_filter_list) +FILTER_VAR(bereq, bo, bereq_Empty_Filter, 1, vdp_filter_list) +FILTER_VAR(beresp, bo, VBF_Get_Filter_List, 1, vfp_filter_list) +FILTER_VAR(req, req, req_Empty_Filter, req_filter_can(ctx->req), vfp_filter_list) +FILTER_VAR(resp, req, resp_Get_Filter_List, 1, vdp_filter_list) diff --git a/doc/sphinx/reference/vcl_var.rst b/doc/sphinx/reference/vcl_var.rst index e895753b2..86b5e4955 100644 --- a/doc/sphinx/reference/vcl_var.rst +++ b/doc/sphinx/reference/vcl_var.rst @@ -236,6 +236,32 @@ req.esi_level A count of how many levels of ESI requests we're currently at. +.. _req.filters: + +req.filters + + Type: STRING + + Readable from: vcl_recv + + Writable from: vcl_recv + + List of Varnish Fetch Processor (VFP) filters the req.body + will be pulled through. The order left to right signifies + processing from client to cache, iow the leftmost filter is + run first on the body as received from the client after + decoding of any transfer encodings. + + VFP Filters change the body before potentially being cached + (e.g. using ``std.cache_req.body()``) and/or being handled by + the backend side, where it may get processed again by + bereq.filters. + + Trying to set req.filters after processing the request body + (again, for example with ``std.cache_req.body()``) triggers a + VCL error. + + .. _req.grace: req.grace @@ -694,6 +720,20 @@ bereq.connect_timeout established. +.. _bereq.filters: + +bereq.filters + + Type: STRING + + Readable from: vcl_backend_fetch + + Writable from: vcl_backend_fetch + + List of VDP filters the bereq.body will be pushed through when + sending the body to the backend. + + .. _bereq.first_byte_timeout: bereq.first_byte_timeout From nils.goroll at uplex.de Mon Sep 30 14:19:07 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 14:19:07 +0000 (UTC) Subject: [master] b498c01c0 Stack req.filters Message-ID: <20240930141907.681FC118A2C@lists.varnish-cache.org> commit b498c01c0628bcd66bf9f7eace4aad0bdf688b1f Author: Nils Goroll Date: Mon Dec 11 08:42:55 2023 +0100 Stack req.filters diff --git a/bin/varnishd/cache/cache_req_body.c b/bin/varnishd/cache/cache_req_body.c index 0ee49c37a..9a4cd54d7 100644 --- a/bin/varnishd/cache/cache_req_body.c +++ b/bin/varnishd/cache/cache_req_body.c @@ -91,6 +91,16 @@ vrb_pull(struct req *req, ssize_t maxsize, objiterate_f *func, void *priv) INIT_OBJ(ctx, VRT_CTX_MAGIC); VCL_Req2Ctx(ctx, req); + if (req->vfp_filter_list != NULL && + VCL_StackVFP(vfc, req->vcl, req->vfp_filter_list)) { + (void)VFP_Error(vfc, "req.body filters failed"); + req->req_body_status = BS_ERROR; + HSH_DerefBoc(req->wrk, req->body_oc); + AZ(HSH_DerefObjCore(req->wrk, &req->body_oc, 0)); + return (-1); + } + + if (VFP_Open(ctx, vfc) < 0) { req->req_body_status = BS_ERROR; HSH_DerefBoc(req->wrk, req->body_oc); From nils.goroll at uplex.de Mon Sep 30 14:19:07 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 14:19:07 +0000 (UTC) Subject: [master] a54e2f805 Stack bereq.filters Message-ID: <20240930141907.8050C118A3C@lists.varnish-cache.org> commit a54e2f80597bcec727028d85837d38c3d925b0a5 Author: Nils Goroll Date: Thu Dec 28 10:51:10 2023 +0100 Stack bereq.filters diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c index d2859e930..e814a7ab7 100644 --- a/bin/varnishd/http1/cache_http1_fetch.c +++ b/bin/varnishd/http1/cache_http1_fetch.c @@ -96,7 +96,13 @@ V1F_SendReq(struct worker *wrk, struct busyobj *bo, uint64_t *ctr_hdrbytes, cl = 0; VDP_Init(vdc, wrk, bo->vsl, NULL, bo, &cl); - + if (bo->vdp_filter_list != NULL && + VCL_StackVDP(vdc, bo->vcl, bo->vdp_filter_list, NULL, bo)) { + VSLb(bo->vsl, SLT_FetchError, "Failure to push processors"); + VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk)); + htc->doclose = SC_OVERLOAD; + return (-1); + } if (v1f_stackv1l(vdc, bo)) { VSLb(bo->vsl, SLT_FetchError, "Failure to push V1L"); VSLb_ts_busyobj(bo, "Bereq", W_TIM_real(wrk)); From nils.goroll at uplex.de Mon Sep 30 14:19:07 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 14:19:07 +0000 (UTC) Subject: [master] 082f48dff Add testcase for (be)req.filters Message-ID: <20240930141907.9C992118A42@lists.varnish-cache.org> commit 082f48dff261c5e592b5f74dd886c6b34f472d0d Author: Nils Goroll Date: Wed Dec 27 09:52:56 2023 +0100 Add testcase for (be)req.filters diff --git a/bin/varnishtest/tests/m00049.vtc b/bin/varnishtest/tests/m00049.vtc new file mode 100644 index 000000000..db6d607e6 --- /dev/null +++ b/bin/varnishtest/tests/m00049.vtc @@ -0,0 +1,127 @@ +varnishtest "VMOD vfp & vdp - request bodies" + +# this test mirrors m00048.vtc with directions reversed + +server s1 { + rxreq + expect req.body == "Cbagb Snpgb, Pnrfne Genafvg!" + txresp +} -start + +varnish v1 -vcl+backend { + import debug; + + sub vcl_backend_fetch { + set bereq.filters = "rot13 debug.pedantic"; + } +} -start + +client c1 { + txreq -req POST -body "Ponto Facto, Caesar Transit!" + rxresp + expect resp.status == 200 +} -run + +varnish v1 -vsl_catchup + +varnish v1 -vcl+backend { + import debug; + + sub vcl_backend_fetch { + set bereq.filters = "rot13 rot13a"; + } +} + +client c1 { + txreq -req POST -body "Ponto Facto, Caesar Transit!" + rxresp + expect resp.status == 503 +} -run + + +server s1 -wait + +server s1 -repeat 4 { + rxreq + expect req.body == "Ponto Facto, Caesar Transit!" + txresp + + rxreq + expect req.http.rot13 == "please" + expect req.body == "Cbagb Snpgb, Pnrfne Genafvg!" + txresp + + rxreq + expect req.http.rot13 == "please" + expect req.http.back13 == "undoes" + expect req.body == "Ponto Facto, Caesar Transit!" + txresp +} -start + +varnish v1 -vcl+backend { + import debug; + import std; + + sub vcl_recv { + if (req.http.rot13) { + set req.filters = "rot13"; + } + if (req.http.cache) { + std.cache_req_body(1MB); + if (req.http.wrong) { + set req.filters += " rot13"; + } + } + } + + sub vcl_backend_fetch { + if (bereq.http.back13) { + set bereq.filters = "rot13 debug.pedantic"; + } + } +} + +client c1 -repeat 2 { + txreq -req POST -body "Ponto Facto, Caesar Transit!" + rxresp + expect resp.status == 200 + + txreq -req POST -hdr "rot13: please" \ + -body "Ponto Facto, Caesar Transit!" + rxresp + expect resp.status == 200 + + txreq -req POST -hdr "rot13: please" -hdr "back13: undoes" \ + -body "Ponto Facto, Caesar Transit!" + rxresp + expect resp.status == 200 + + txreq -req POST -hdr "Cache: yes" \ + -body "Ponto Facto, Caesar Transit!" + rxresp + expect resp.status == 200 + + txreq -req POST -hdr "Cache: yes" -hdr "rot13: please" \ + -body "Ponto Facto, Caesar Transit!" + rxresp + expect resp.status == 200 + + txreq -req POST -hdr "Cache: yes" \ + -hdr "rot13: please" -hdr "back13: undoes" \ + -body "Ponto Facto, Caesar Transit!" + rxresp + expect resp.status == 200 +} -run + +logexpect l1 -v v1 -g raw { + expect * * VCL_Error "^req\\.filters not settable" +} -start + +client c1 { + txreq -req POST -hdr "Cache: yes" -hdr "Wrong: it.is" \ + -body "No second thoughts after caching" + rxresp + expect resp.status == 503 +} -run + +logexpect l1 -wait From nils.goroll at uplex.de Mon Sep 30 14:19:07 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 14:19:07 +0000 (UTC) Subject: [master] 563c7380f Replace DBG_SLOW_BEREQ Message-ID: <20240930141907.C0417118A4D@lists.varnish-cache.org> commit 563c7380faca2d74544fc75e3c920a5c905131dd Author: Nils Goroll Date: Fri Sep 6 17:42:25 2024 +0200 Replace DBG_SLOW_BEREQ diff --git a/bin/varnishtest/tests/m00048.vtc b/bin/varnishtest/tests/m00048.vtc index 6e4263224..bcd94925a 100644 --- a/bin/varnishtest/tests/m00048.vtc +++ b/bin/varnishtest/tests/m00048.vtc @@ -66,7 +66,7 @@ varnish v1 -vcl+backend { sub vcl_deliver { if (req.http.Rot13) { - set resp.filters = "rot13 debug.pedantic"; + set resp.filters = "debug.slow rot13 debug.pedantic"; } } } @@ -90,7 +90,7 @@ varnish v1 -vcl { sub vcl_synth { set resp.body = "Ponto Facto, Caesar Transit!"; if (req.http.Rot13) { - set resp.filters += "rot13 debug.chunked debug.pedantic"; + set resp.filters += "debug.slow rot13 debug.chunked debug.pedantic"; } return (deliver); } diff --git a/include/tbl/debug_bits.h b/include/tbl/debug_bits.h index cbe11a487..2c6a980c4 100644 --- a/include/tbl/debug_bits.h +++ b/include/tbl/debug_bits.h @@ -52,7 +52,6 @@ DEBUG_BIT(PROCESSORS, processors, "Fetch/Deliver processors") DEBUG_BIT(PROTOCOL, protocol, "Protocol debugging") DEBUG_BIT(VCL_KEEP, vcl_keep, "Keep VCL C and so files") DEBUG_BIT(LCK, lck, "Additional lock statistics") -DEBUG_BIT(SLOW_BEREQ, slow_bereq, "Slow down bereq") // XXX -> filter #undef DEBUG_BIT /*lint -restore */ diff --git a/vmod/vmod_debug.c b/vmod/vmod_debug.c index 3cf1eb7b7..33c1bf45e 100644 --- a/vmod/vmod_debug.c +++ b/vmod/vmod_debug.c @@ -322,6 +322,46 @@ static const struct vdp xyzzy_vdp_pedantic = { .fini = xyzzy_pedantic_fini, }; +/********************************************************************** + * + * this trivial copy/paste/edit filter (of rot13) was specifically made for + * someone who added a DBG_SLOW_BEREQ debug flag. It should actually be turned + * in a proper "bandwidth control" filter, but that exceeds an evening's work, + * so it's kept for later + */ + +static enum vfp_status v_matchproto_(vfp_pull_f) +xyzzy_vfp_slow_pull(struct vfp_ctx *vc, struct vfp_entry *vfe, void *p, + ssize_t *lp) +{ + + (void)vfe; + VTIM_sleep(1.0); + return (VFP_Suck(vc, p, lp)); +} + +static const struct vfp xyzzy_vfp_slow = { + .name = "debug.slow", + .pull = xyzzy_vfp_slow_pull, +}; + +/**********************************************************************/ + +static int v_matchproto_(vdp_bytes_f) +xyzzy_vdp_slow_bytes(struct vdp_ctx *vdc, enum vdp_action act, void **priv, + const void *ptr, ssize_t len) +{ + + (void)priv; + VTIM_sleep(1.0); + return (VDP_bytes(vdc, act, ptr, len)); +} + +static const struct vdp xyzzy_vdp_slow = { + .name = "debug.slow", + .bytes = xyzzy_vdp_slow_bytes +}; + /**********************************************************************/ VCL_STRING v_matchproto_(td_debug_author) @@ -593,6 +633,7 @@ event_load(VRT_CTX, struct vmod_priv *priv) AZ(VRT_AddFilter(ctx, &xyzzy_vfp_rot13, &xyzzy_vdp_rot13)); AZ(VRT_AddFilter(ctx, NULL, &xyzzy_vdp_pedantic)); AZ(VRT_AddFilter(ctx, NULL, &xyzzy_vdp_chunked)); + AZ(VRT_AddFilter(ctx, &xyzzy_vfp_slow, &xyzzy_vdp_slow)); return (0); } @@ -754,6 +795,7 @@ event_discard(VRT_CTX, void *priv) AZ(ctx->msg); + VRT_RemoveFilter(ctx, &xyzzy_vfp_slow, &xyzzy_vdp_slow); VRT_RemoveFilter(ctx, &xyzzy_vfp_rot13, &xyzzy_vdp_rot13); VRT_RemoveFilter(ctx, NULL, &xyzzy_vdp_pedantic); VRT_RemoveFilter(ctx, NULL, &xyzzy_vdp_chunked); From nils.goroll at uplex.de Mon Sep 30 14:28:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 14:28:06 +0000 (UTC) Subject: [master] 75decc80e vev: use sigaction instead of sighandler Message-ID: <20240930142806.2C6B8119A04@lists.varnish-cache.org> commit 75decc80e7b97e394d70a74281c7976e64d678dc Author: Thibaut Artis Date: Fri Apr 12 17:17:40 2024 +0200 vev: use sigaction instead of sighandler diff --git a/lib/libvarnish/vev.c b/lib/libvarnish/vev.c index 8ce5837fb..c9335e6a1 100644 --- a/lib/libvarnish/vev.c +++ b/lib/libvarnish/vev.c @@ -175,10 +175,11 @@ vev_get_sig(int sig) /*--------------------------------------------------------------------*/ static void -vev_sighandler(int sig) +vev_sigaction(int sig, siginfo_t *siginfo, void *ctx) { struct vevsig *es; + (void)ctx; assert(sig < vev_nsig); assert(vev_sigs != NULL); es = &vev_sigs[sig]; @@ -278,8 +279,8 @@ VEV_Start(struct vev_root *evb, struct vev *e) AZ(es->happened); es->vev = e; es->vevb = evb; - es->sigact.sa_flags = e->sig_flags; - es->sigact.sa_handler = vev_sighandler; + es->sigact.sa_flags = e->sig_flags | SA_SIGINFO; + es->sigact.sa_sigaction = vev_sigaction; } else { es = NULL; } From nils.goroll at uplex.de Mon Sep 30 14:28:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 14:28:06 +0000 (UTC) Subject: [master] 24c7d1542 vev: keep a copy of siginfo_t in vevsig Message-ID: <20240930142806.7B23C119A08@lists.varnish-cache.org> commit 24c7d154207dcea0e4ace3e933a8464aa6b634df Author: Thibaut Artis Date: Fri Apr 12 17:37:51 2024 +0200 vev: keep a copy of siginfo_t in vevsig (OBE commit message removed during merge/rebase by @nigoroll) diff --git a/bin/varnishd/mgt/mgt_cli.c b/bin/varnishd/mgt/mgt_cli.c index d0b28bf61..f49bb358d 100644 --- a/bin/varnishd/mgt/mgt_cli.c +++ b/bin/varnishd/mgt/mgt_cli.c @@ -38,6 +38,7 @@ #include #include +#include #include #include #include diff --git a/bin/varnishd/mgt/mgt_vcl.c b/bin/varnishd/mgt/mgt_vcl.c index 9c4e01e9c..cf437cc60 100644 --- a/bin/varnishd/mgt/mgt_vcl.c +++ b/bin/varnishd/mgt/mgt_vcl.c @@ -33,7 +33,10 @@ #include "config.h" +#include +#include #include +#include #include #include #include diff --git a/include/vev.h b/include/vev.h index 197c14882..556abc648 100644 --- a/include/vev.h +++ b/include/vev.h @@ -53,6 +53,7 @@ struct vev { unsigned sig_flags; double timeout; vev_cb_f *callback; + siginfo_t *siginfo; void *priv; /* priv */ diff --git a/lib/libvarnish/vev.c b/lib/libvarnish/vev.c index c9335e6a1..ff097310d 100644 --- a/lib/libvarnish/vev.c +++ b/lib/libvarnish/vev.c @@ -59,6 +59,7 @@ struct vevsig { struct vev *vev; struct sigaction sigact; unsigned char happened; + siginfo_t siginfo[1]; }; static struct vevsig *vev_sigs; @@ -183,8 +184,11 @@ vev_sigaction(int sig, siginfo_t *siginfo, void *ctx) assert(sig < vev_nsig); assert(vev_sigs != NULL); es = &vev_sigs[sig]; - if (!es->happened) + if (!es->happened) { es->vevb->psig++; + memcpy(es->siginfo, siginfo, sizeof *es->siginfo); + es->vev->siginfo = es->siginfo; + } es->happened = 1; } @@ -400,6 +404,10 @@ vev_sched_signal(struct vev_root *evb) e = es->vev; assert(e != NULL); i = e->callback(e, VEV__SIG); + if (e->siginfo != NULL) { + e->siginfo = NULL; + memset(es->siginfo, 0, sizeof *es->siginfo); + } if (i) { VEV_Stop(evb, e); free(e); From nils.goroll at uplex.de Mon Sep 30 14:28:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 14:28:06 +0000 (UTC) Subject: [master] d5ddb5f8c mgt_main: print signal PID of origin in mgt_sigint Message-ID: <20240930142806.9C1B6119A0B@lists.varnish-cache.org> commit d5ddb5f8ce2f43f88b3963a0e97ba6942fedf017 Author: Thibaut Artis Date: Tue May 7 16:26:51 2024 +0200 mgt_main: print signal PID of origin in mgt_sigint diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 4db07013c..d3eb09a90 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -445,9 +445,8 @@ static int v_matchproto_(vev_cb_f) mgt_sigint(const struct vev *e, int what) { - (void)e; (void)what; - MGT_Complain(C_ERR, "Manager got %s", e->name); + MGT_Complain(C_ERR, "Manager got %s from PID %jd", e->name, (intmax_t)e->siginfo->si_pid); (void)fflush(stdout); if (MCH_Running()) MCH_Stop_Child(); From nils.goroll at uplex.de Mon Sep 30 14:28:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 14:28:06 +0000 (UTC) Subject: [master] 0fa953359 vev: Restore a default handler without flags Message-ID: <20240930142806.CCC3E119A10@lists.varnish-cache.org> commit 0fa9533594ec90884da09eb2f1642a70d3814b1b Author: Thibaut Artis Date: Tue Apr 30 11:28:21 2024 +0200 vev: Restore a default handler without flags diff --git a/lib/libvarnish/vev.c b/lib/libvarnish/vev.c index ff097310d..8269e751c 100644 --- a/lib/libvarnish/vev.c +++ b/lib/libvarnish/vev.c @@ -343,7 +343,7 @@ VEV_Stop(struct vev_root *evb, struct vev *e) assert(es->vev == e); es->vev = NULL; es->vevb = NULL; - es->sigact.sa_flags = e->sig_flags; + es->sigact.sa_flags = 0; es->sigact.sa_handler = SIG_DFL; AZ(sigaction(e->sig, &es->sigact, NULL)); es->happened = 0; From nils.goroll at uplex.de Mon Sep 30 14:53:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 14:53:05 +0000 (UTC) Subject: [master] 0c0bfc526 Silence Flexelint Message-ID: <20240930145305.7160311ADD3@lists.varnish-cache.org> commit 0c0bfc5263a32ced1607a36103c9385d017f63d2 Author: Nils Goroll Date: Mon Sep 30 16:51:52 2024 +0200 Silence Flexelint Ref 00b190a0aed482a638fdb9f1207598cca3db6c27 diff --git a/bin/varnishd/cache/cache_vrt_filter.c b/bin/varnishd/cache/cache_vrt_filter.c index 707533eea..6bf1ba06e 100644 --- a/bin/varnishd/cache/cache_vrt_filter.c +++ b/bin/varnishd/cache/cache_vrt_filter.c @@ -459,6 +459,8 @@ req_filter_can(struct req *req) { /*--------------------------------------------------------------------*/ +//lint -emacro(506, FILTER_VAR) constant value boolean +//lint -emacro(774, FILTER_VAR) if always evaluates to false #define FILTER_VAR(vcl, in, func, cond, fld) \ VCL_STRING \ VRT_r_##vcl##_filters(VRT_CTX) \ From nils.goroll at uplex.de Mon Sep 30 15:13:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 15:13:05 +0000 (UTC) Subject: [master] 052a1ef1b Polish cstyle Message-ID: <20240930151305.8D38711B9A2@lists.varnish-cache.org> commit 052a1ef1b69639fddaf4e4201f42d787aaef4f34 Author: Nils Goroll Date: Mon Sep 30 17:11:11 2024 +0200 Polish cstyle diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c index fb33c1199..1a5303221 100644 --- a/bin/varnishd/waiter/cache_waiter_epoll.c +++ b/bin/varnishd/waiter/cache_waiter_epoll.c @@ -133,7 +133,7 @@ vwe_thread(void *priv) Lck_Lock(&vwe->mtx); active = Wait_HeapDelete(w, wp); Lck_Unlock(&vwe->mtx); - if (!active) { + if (active == 0) { VSL(SLT_Debug, NO_VXID, "epoll: spurious event (%d)", wp->fd); continue; From nils.goroll at uplex.de Mon Sep 30 15:13:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 15:13:05 +0000 (UTC) Subject: [master] 48a2a41cb waiter_epoll: Fix locking wrt nwaited counter Message-ID: <20240930151305.B2BE111B9A4@lists.varnish-cache.org> commit 48a2a41cbc97498e044af7b7b3545427e3289872 Author: Nils Goroll Date: Mon Sep 30 17:11:46 2024 +0200 waiter_epoll: Fix locking wrt nwaited counter Fixes #4204 diff --git a/bin/varnishd/waiter/cache_waiter_epoll.c b/bin/varnishd/waiter/cache_waiter_epoll.c index 1a5303221..95bc442f1 100644 --- a/bin/varnishd/waiter/cache_waiter_epoll.c +++ b/bin/varnishd/waiter/cache_waiter_epoll.c @@ -132,6 +132,10 @@ vwe_thread(void *priv) CAST_OBJ_NOTNULL(wp, ep->data.ptr, WAITED_MAGIC); Lck_Lock(&vwe->mtx); active = Wait_HeapDelete(w, wp); + if (active != 0) { + AN(vwe->nwaited); + vwe->nwaited--; + } Lck_Unlock(&vwe->mtx); if (active == 0) { VSL(SLT_Debug, NO_VXID, @@ -139,8 +143,6 @@ vwe_thread(void *priv) continue; } AZ(epoll_ctl(vwe->epfd, EPOLL_CTL_DEL, wp->fd, NULL)); - AN(vwe->nwaited); - vwe->nwaited--; if (ep->events & EPOLLIN) { if (ep->events & EPOLLRDHUP && recv(wp->fd, &c, 1, MSG_PEEK) == 0) From nils.goroll at uplex.de Mon Sep 30 15:53:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 15:53:06 +0000 (UTC) Subject: [master] 965f71621 Move handling of the start cli command to cache_main.c Message-ID: <20240930155306.746E911D0E0@lists.varnish-cache.org> commit 965f7162118fcc1011670146268672960597c305 Author: Martin Blix Grydeland Date: Thu Nov 1 15:24:41 2018 +0100 Move handling of the start cli command to cache_main.c This was privously implemented in cache_acceptor.c. Move it to cache_main to be able to handle other tasks not only acceptor related during start. diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index 452a08c92..78dd7dc53 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -676,15 +676,11 @@ vca_acct(void *arg) /*--------------------------------------------------------------------*/ -static void v_matchproto_(cli_func_t) -ccf_start(struct cli *cli, const char * const *av, void *priv) +void +VCA_Start(struct cli *cli) { struct listen_sock *ls; - (void)cli; - (void)av; - (void)priv; - (void)vca_sock_opt_init(); VTAILQ_FOREACH(ls, &heritage.socks, list) { @@ -752,7 +748,6 @@ ccf_listen_address(struct cli *cli, const char * const *av, void *priv) /*--------------------------------------------------------------------*/ static struct cli_proto vca_cmds[] = { - { CLICMD_SERVER_START, "", ccf_start }, { CLICMD_DEBUG_LISTEN_ADDRESS, "d", ccf_listen_address }, { NULL } }; diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index 303bc5de0..284a58be6 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -366,6 +366,20 @@ cli_quit(int sig) * Run the child process */ +static void v_matchproto_(cli_func_t) +ccf_child_start(struct cli *cli, const char * const *av, void *priv) +{ + (void)av; + (void)priv; + + VCA_Start(cli); +} + +static struct cli_proto child_cmds[] = { + { CLICMD_SERVER_START, "", ccf_child_start }, + { NULL } +}; + void child_main(int sigmagic, size_t altstksz) { @@ -443,6 +457,7 @@ child_main(int sigmagic, size_t altstksz) CLI_AddFuncs(debug_cmds); + CLI_AddFuncs(child_cmds); #if WITH_PERSISTENT_STORAGE /* Wait for persistent storage to load if asked to */ diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 7f876290a..73a10352e 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -158,6 +158,7 @@ struct vcf { /* cache_acceptor.c */ void VCA_Init(void); +void VCA_Start(struct cli *cli); void VCA_Shutdown(void); /* cache_backend.c */ From nils.goroll at uplex.de Mon Sep 30 15:53:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 15:53:06 +0000 (UTC) Subject: [master] d719686dc varnishd: copy acceptor out Message-ID: <20240930155306.8D31E11D0E2@lists.varnish-cache.org> commit d719686dc48fad2b677a1b2eaeca86c78cf1018a Author: Asad Sajjad Ahmed Date: Mon Sep 4 13:07:02 2023 +0200 varnishd: copy acceptor out Copy acceptor code out from the child and the manager to a new directory called 'acceptor'. Updated to varnish-cache as of 48a2a41cbc97498e044af7b7b3545427e3289872 (just after 7.6) Signed-off-by: Asad Sajjad Ahmed Signed-off-by: Nils Goroll diff --git a/bin/varnishd/acceptor/cache_acceptor.c b/bin/varnishd/acceptor/cache_acceptor.c new file mode 100644 index 000000000..8989bc53b --- /dev/null +++ b/bin/varnishd/acceptor/cache_acceptor.c @@ -0,0 +1,834 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2015 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * This source file has the various trickery surrounding the accept/listen + * sockets. + * + */ + +#include "config.h" + +#include +#include +#include + +#include "cache/cache_varnishd.h" +#include "acceptor/cache_acceptor.h" + +#include "cache/cache_transport.h" +#include "cache/cache_pool.h" +#include "common/heritage.h" + +#include "vcli_serve.h" +#include "vsa.h" +#include "vtcp.h" +#include "vtim.h" + +static pthread_t VCA_thread; +static vtim_dur vca_pace = 0.0; +static struct lock pace_mtx; +static unsigned pool_accepting; +static pthread_mutex_t shut_mtx = PTHREAD_MUTEX_INITIALIZER; + +struct wrk_accept { + unsigned magic; +#define WRK_ACCEPT_MAGIC 0x8c4b4d59 + + /* Accept stuff */ + struct sockaddr_storage acceptaddr; + socklen_t acceptaddrlen; + int acceptsock; + struct listen_sock *acceptlsock; +}; + +struct poolsock { + unsigned magic; +#define POOLSOCK_MAGIC 0x1b0a2d38 + VTAILQ_ENTRY(poolsock) list; + struct listen_sock *lsock; + struct pool_task task[1]; + struct pool *pool; +}; + +/*-------------------------------------------------------------------- + * TCP options we want to control + */ + +union sock_arg { + struct linger lg; + struct timeval tv; + int i; +}; + +static struct sock_opt { + int level; + int optname; + const char *strname; + unsigned mod; + socklen_t sz; + union sock_arg arg[1]; +} sock_opts[] = { + /* Note: Setting the mod counter to something not-zero is needed + * to force the setsockopt() calls on startup */ +#define SOCK_OPT(lvl, nam, typ) { lvl, nam, #nam, 1, sizeof(typ) }, + + SOCK_OPT(SOL_SOCKET, SO_LINGER, struct linger) + SOCK_OPT(SOL_SOCKET, SO_KEEPALIVE, int) + SOCK_OPT(SOL_SOCKET, SO_SNDTIMEO, struct timeval) + SOCK_OPT(SOL_SOCKET, SO_RCVTIMEO, struct timeval) + + SOCK_OPT(IPPROTO_TCP, TCP_NODELAY, int) + +#if defined(HAVE_TCP_KEEP) + SOCK_OPT(IPPROTO_TCP, TCP_KEEPIDLE, int) + SOCK_OPT(IPPROTO_TCP, TCP_KEEPCNT, int) + SOCK_OPT(IPPROTO_TCP, TCP_KEEPINTVL, int) +#elif defined(HAVE_TCP_KEEPALIVE) + SOCK_OPT(IPPROTO_TCP, TCP_KEEPALIVE, int) +#endif + +#undef SOCK_OPT +}; + +static const int n_sock_opts = sizeof sock_opts / sizeof sock_opts[0]; + +struct conn_heritage { + unsigned sess_set; + unsigned listen_mod; +}; + +/*-------------------------------------------------------------------- + * We want to get out of any kind of trouble-hit TCP connections as fast + * as absolutely possible, so we set them LINGER disabled, so that even if + * there are outstanding write data on the socket, a close(2) will return + * immediately. + */ +static const struct linger disable_so_linger = { + .l_onoff = 0, +}; + +/* + * We turn on keepalives by default to assist in detecting clients that have + * hung up on connections returning from waitinglists + */ + +static const unsigned enable_so_keepalive = 1; + +/* We disable Nagle's algorithm in favor of low latency setups. + */ + +static const unsigned enable_tcp_nodelay = 1; + +/*-------------------------------------------------------------------- + * lacking a better place, we put some generic periodic updates + * into the vca_acct() loop which we are running anyway + */ +static void +vca_periodic(vtim_real t0) +{ + vtim_real now; + + now = VTIM_real(); + VSC_C_main->uptime = (uint64_t)(now - t0); + + VTIM_postel = FEATURE(FEATURE_HTTP_DATE_POSTEL); +} + +/*-------------------------------------------------------------------- + * Some kernels have bugs/limitations with respect to which options are + * inherited from the accept/listen socket, so we have to keep track of + * which, if any, sockopts we have to set on the accepted socket. + */ + +static int +vca_sock_opt_init(void) +{ + struct sock_opt *so; + union sock_arg tmp; + int n, chg = 0; + size_t sz; + + memset(&tmp, 0, sizeof tmp); + + for (n = 0; n < n_sock_opts; n++) { + so = &sock_opts[n]; + +#define SET_VAL(nm, so, fld, val) \ + do { \ + if (!strcmp(#nm, so->strname)) { \ + assert(so->sz == sizeof so->arg->fld); \ + so->arg->fld = (val); \ + } \ + } while (0) + +#define NEW_VAL(nm, so, fld, val) \ + do { \ + if (!strcmp(#nm, so->strname)) { \ + sz = sizeof tmp.fld; \ + assert(so->sz == sz); \ + tmp.fld = (val); \ + if (memcmp(&so->arg->fld, &(tmp.fld), sz)) { \ + memcpy(&so->arg->fld, &(tmp.fld), sz); \ + so->mod++; \ + chg = 1; \ + } \ + } \ + } while (0) + + SET_VAL(SO_LINGER, so, lg, disable_so_linger); + SET_VAL(SO_KEEPALIVE, so, i, enable_so_keepalive); + NEW_VAL(SO_SNDTIMEO, so, tv, + VTIM_timeval_sock(cache_param->idle_send_timeout)); + NEW_VAL(SO_RCVTIMEO, so, tv, + VTIM_timeval_sock(cache_param->timeout_idle)); + SET_VAL(TCP_NODELAY, so, i, enable_tcp_nodelay); +#if defined(HAVE_TCP_KEEP) + NEW_VAL(TCP_KEEPIDLE, so, i, + (int)cache_param->tcp_keepalive_time); + NEW_VAL(TCP_KEEPCNT, so, i, + (int)cache_param->tcp_keepalive_probes); + NEW_VAL(TCP_KEEPINTVL, so, i, + (int)cache_param->tcp_keepalive_intvl); +#elif defined(HAVE_TCP_KEEPALIVE) + NEW_VAL(TCP_KEEPALIVE, so, i, + (int)cache_param->tcp_keepalive_time); +#endif + } + return (chg); +} + +static void +vca_sock_opt_test(const struct listen_sock *ls, const struct sess *sp) +{ + struct conn_heritage *ch; + struct sock_opt *so; + union sock_arg tmp; + socklen_t l; + int i, n; + + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + + for (n = 0; n < n_sock_opts; n++) { + so = &sock_opts[n]; + ch = &ls->conn_heritage[n]; + if (ch->sess_set) { + VSL(SLT_Debug, sp->vxid, + "sockopt: Not testing nonhereditary %s for %s=%s", + so->strname, ls->name, ls->endpoint); + continue; + } + if (so->level == IPPROTO_TCP && ls->uds) { + VSL(SLT_Debug, sp->vxid, + "sockopt: Not testing incompatible %s for %s=%s", + so->strname, ls->name, ls->endpoint); + continue; + } + memset(&tmp, 0, sizeof tmp); + l = so->sz; + i = getsockopt(sp->fd, so->level, so->optname, &tmp, &l); + if (i == 0 && memcmp(&tmp, so->arg, so->sz)) { + VSL(SLT_Debug, sp->vxid, + "sockopt: Test confirmed %s non heredity for %s=%s", + so->strname, ls->name, ls->endpoint); + ch->sess_set = 1; + } + if (i && errno != ENOPROTOOPT) + VTCP_Assert(i); + } +} + +static void +vca_sock_opt_set(const struct listen_sock *ls, const struct sess *sp) +{ + struct conn_heritage *ch; + struct sock_opt *so; + vxid_t vxid; + int n, sock; + + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + + if (sp != NULL) { + CHECK_OBJ(sp, SESS_MAGIC); + sock = sp->fd; + vxid = sp->vxid; + } else { + sock = ls->sock; + vxid = NO_VXID; + } + + for (n = 0; n < n_sock_opts; n++) { + so = &sock_opts[n]; + ch = &ls->conn_heritage[n]; + if (so->level == IPPROTO_TCP && ls->uds) { + VSL(SLT_Debug, vxid, + "sockopt: Not setting incompatible %s for %s=%s", + so->strname, ls->name, ls->endpoint); + continue; + } + if (sp == NULL && ch->listen_mod == so->mod) { + VSL(SLT_Debug, vxid, + "sockopt: Not setting unmodified %s for %s=%s", + so->strname, ls->name, ls->endpoint); + continue; + } + if (sp != NULL && !ch->sess_set) { + VSL(SLT_Debug, sp->vxid, + "sockopt: %s may be inherited for %s=%s", + so->strname, ls->name, ls->endpoint); + continue; + } + VSL(SLT_Debug, vxid, + "sockopt: Setting %s for %s=%s", + so->strname, ls->name, ls->endpoint); + VTCP_Assert(setsockopt(sock, + so->level, so->optname, so->arg, so->sz)); + if (sp == NULL) + ch->listen_mod = so->mod; + } +} + +/*-------------------------------------------------------------------- + * If accept(2)'ing fails, we pace ourselves to relive any resource + * shortage if possible. + */ + +static void +vca_pace_check(void) +{ + vtim_dur p; + + if (vca_pace == 0.0) + return; + Lck_Lock(&pace_mtx); + p = vca_pace; + Lck_Unlock(&pace_mtx); + if (p > 0.0) + VTIM_sleep(p); +} + +static void +vca_pace_bad(void) +{ + + Lck_Lock(&pace_mtx); + vca_pace += cache_param->acceptor_sleep_incr; + if (vca_pace > cache_param->acceptor_sleep_max) + vca_pace = cache_param->acceptor_sleep_max; + Lck_Unlock(&pace_mtx); +} + +static void +vca_pace_good(void) +{ + + if (vca_pace == 0.0) + return; + Lck_Lock(&pace_mtx); + vca_pace *= cache_param->acceptor_sleep_decay; + if (vca_pace < cache_param->acceptor_sleep_incr) + vca_pace = 0.0; + Lck_Unlock(&pace_mtx); +} + +/*-------------------------------------------------------------------- + * The pool-task for a newly accepted session + * + * Called from assigned worker thread + */ + +static void +vca_mk_tcp(const struct wrk_accept *wa, + struct sess *sp, char *laddr, char *lport, char *raddr, char *rport) +{ + struct suckaddr *sa = NULL; + ssize_t sz; + + AN(SES_Reserve_remote_addr(sp, &sa, &sz)); + AN(sa); + assert(sz == vsa_suckaddr_len); + AN(VSA_Build(sa, &wa->acceptaddr, wa->acceptaddrlen)); + sp->sattr[SA_CLIENT_ADDR] = sp->sattr[SA_REMOTE_ADDR]; + + VTCP_name(sa, raddr, VTCP_ADDRBUFSIZE, rport, VTCP_PORTBUFSIZE); + AN(SES_Set_String_Attr(sp, SA_CLIENT_IP, raddr)); + AN(SES_Set_String_Attr(sp, SA_CLIENT_PORT, rport)); + + + AN(SES_Reserve_local_addr(sp, &sa, &sz)); + AN(VSA_getsockname(sp->fd, sa, sz)); + sp->sattr[SA_SERVER_ADDR] = sp->sattr[SA_LOCAL_ADDR]; + VTCP_name(sa, laddr, VTCP_ADDRBUFSIZE, lport, VTCP_PORTBUFSIZE); +} + +static void +vca_mk_uds(struct wrk_accept *wa, struct sess *sp, char *laddr, char *lport, + char *raddr, char *rport) +{ + struct suckaddr *sa = NULL; + ssize_t sz; + + (void) wa; + AN(SES_Reserve_remote_addr(sp, &sa, &sz)); + AN(sa); + assert(sz == vsa_suckaddr_len); + AZ(SES_Set_remote_addr(sp, bogo_ip)); + sp->sattr[SA_CLIENT_ADDR] = sp->sattr[SA_REMOTE_ADDR]; + sp->sattr[SA_LOCAL_ADDR] = sp->sattr[SA_REMOTE_ADDR]; + sp->sattr[SA_SERVER_ADDR] = sp->sattr[SA_REMOTE_ADDR]; + AN(SES_Set_String_Attr(sp, SA_CLIENT_IP, "0.0.0.0")); + AN(SES_Set_String_Attr(sp, SA_CLIENT_PORT, "0")); + + strcpy(laddr, "0.0.0.0"); + strcpy(raddr, "0.0.0.0"); + strcpy(lport, "0"); + strcpy(rport, "0"); +} + +static void v_matchproto_(task_func_t) +vca_make_session(struct worker *wrk, void *arg) +{ + struct sess *sp; + struct req *req; + struct wrk_accept *wa; + char laddr[VTCP_ADDRBUFSIZE]; + char lport[VTCP_PORTBUFSIZE]; + char raddr[VTCP_ADDRBUFSIZE]; + char rport[VTCP_PORTBUFSIZE]; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CAST_OBJ_NOTNULL(wa, arg, WRK_ACCEPT_MAGIC); + + VTCP_blocking(wa->acceptsock); + + /* Turn accepted socket into a session */ + AN(WS_Reservation(wrk->aws)); + sp = SES_New(wrk->pool); + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + wrk->stats->s_sess++; + + sp->t_open = VTIM_real(); + sp->t_idle = sp->t_open; + sp->vxid = VXID_Get(wrk, VSL_CLIENTMARKER); + + sp->fd = wa->acceptsock; + wa->acceptsock = -1; + sp->listen_sock = wa->acceptlsock; + + assert((size_t)wa->acceptaddrlen <= vsa_suckaddr_len); + + if (wa->acceptlsock->uds) + vca_mk_uds(wa, sp, laddr, lport, raddr, rport); + else + vca_mk_tcp(wa, sp, laddr, lport, raddr, rport); + + AN(wa->acceptlsock->name); + VSL(SLT_Begin, sp->vxid, "sess 0 %s", + wa->acceptlsock->transport->name); + VSL(SLT_SessOpen, sp->vxid, "%s %s %s %s %s %.6f %d", + raddr, rport, wa->acceptlsock->name, laddr, lport, + sp->t_open, sp->fd); + + vca_pace_good(); + wrk->stats->sess_conn++; + + if (wa->acceptlsock->test_heritage) { + vca_sock_opt_test(wa->acceptlsock, sp); + wa->acceptlsock->test_heritage = 0; + } + vca_sock_opt_set(wa->acceptlsock, sp); + + req = Req_New(sp); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + req->htc->rfd = &sp->fd; + + SES_SetTransport(wrk, sp, req, wa->acceptlsock->transport); + WS_Release(wrk->aws, 0); +} + +/*-------------------------------------------------------------------- + * This function accepts on a single socket for a single thread pool. + * + * As long as we can stick the accepted connection to another thread + * we do so, otherwise we put the socket back on the "BACK" pool + * and handle the new connection ourselves. + */ + +static void v_matchproto_(task_func_t) +vca_accept_task(struct worker *wrk, void *arg) +{ + struct wrk_accept wa; + struct poolsock *ps; + struct listen_sock *ls; + int i; + char laddr[VTCP_ADDRBUFSIZE]; + char lport[VTCP_PORTBUFSIZE]; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CAST_OBJ_NOTNULL(ps, arg, POOLSOCK_MAGIC); + ls = ps->lsock; + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + + while (!pool_accepting) + VTIM_sleep(.1); + + /* Dont hold on to (possibly) discarded VCLs */ + if (wrk->wpriv->vcl != NULL) + VCL_Rel(&wrk->wpriv->vcl); + + while (!ps->pool->die) { + INIT_OBJ(&wa, WRK_ACCEPT_MAGIC); + wa.acceptlsock = ls; + + vca_pace_check(); + + wa.acceptaddrlen = sizeof wa.acceptaddr; + do { + i = accept(ls->sock, (void*)&wa.acceptaddr, + &wa.acceptaddrlen); + } while (i < 0 && errno == EAGAIN && !ps->pool->die); + + if (i < 0 && ps->pool->die) + break; + + if (i < 0 && ls->sock == -2) { + /* Shut down in progress */ + sleep(2); + continue; + } + + if (i < 0) { + switch (errno) { + case ECONNABORTED: + wrk->stats->sess_fail_econnaborted++; + break; + case EINTR: + wrk->stats->sess_fail_eintr++; + break; + case EMFILE: + wrk->stats->sess_fail_emfile++; + vca_pace_bad(); + break; + case EBADF: + wrk->stats->sess_fail_ebadf++; + vca_pace_bad(); + break; + case ENOBUFS: + case ENOMEM: + wrk->stats->sess_fail_enomem++; + vca_pace_bad(); + break; + default: + wrk->stats->sess_fail_other++; + vca_pace_bad(); + break; + } + + i = errno; + wrk->stats->sess_fail++; + + if (wa.acceptlsock->uds) { + bstrcpy(laddr, "0.0.0.0"); + bstrcpy(lport, "0"); + } else { + VTCP_myname(ls->sock, laddr, VTCP_ADDRBUFSIZE, + lport, VTCP_PORTBUFSIZE); + } + + VSL(SLT_SessError, NO_VXID, "%s %s %s %d %d \"%s\"", + wa.acceptlsock->name, laddr, lport, + ls->sock, i, VAS_errtxt(i)); + (void)Pool_TrySumstat(wrk); + continue; + } + + wa.acceptsock = i; + + if (!Pool_Task_Arg(wrk, TASK_QUEUE_REQ, + vca_make_session, &wa, sizeof wa)) { + /* + * We couldn't get another thread, so we will handle + * the request in this worker thread, but first we + * must reschedule the listening task so it will be + * taken up by another thread again. + */ + if (!ps->pool->die) { + AZ(Pool_Task(wrk->pool, ps->task, + TASK_QUEUE_VCA)); + return; + } + } + if (!ps->pool->die && DO_DEBUG(DBG_SLOW_ACCEPTOR)) + VTIM_sleep(2.0); + + } + + VSL(SLT_Debug, NO_VXID, "XXX Accept thread dies %p", ps); + FREE_OBJ(ps); +} + +/*-------------------------------------------------------------------- + * Called when a worker and attached thread pool is created, to + * allocate the tasks which will listen to sockets for that pool. + */ + +void +VCA_NewPool(struct pool *pp) +{ + struct listen_sock *ls; + struct poolsock *ps; + + VTAILQ_FOREACH(ls, &heritage.socks, list) { + ALLOC_OBJ(ps, POOLSOCK_MAGIC); + AN(ps); + ps->lsock = ls; + ps->task->func = vca_accept_task; + ps->task->priv = ps; + ps->pool = pp; + VTAILQ_INSERT_TAIL(&pp->poolsocks, ps, list); + AZ(Pool_Task(pp, ps->task, TASK_QUEUE_VCA)); + } +} + +void +VCA_DestroyPool(struct pool *pp) +{ + struct poolsock *ps; + + while (!VTAILQ_EMPTY(&pp->poolsocks)) { + ps = VTAILQ_FIRST(&pp->poolsocks); + VTAILQ_REMOVE(&pp->poolsocks, ps, list); + } +} + +/*--------------------------------------------------------------------*/ + +static void * v_matchproto_() +vca_acct(void *arg) +{ + struct listen_sock *ls; + vtim_real t0; + + // XXX Actually a misnomer now because the accept happens in a pool + // thread. Rename to accept-nanny or so? + THR_SetName("cache-acceptor"); + THR_Init(); + (void)arg; + + t0 = VTIM_real(); + vca_periodic(t0); + + pool_accepting = 1; + + while (1) { + (void)sleep(1); + if (vca_sock_opt_init()) { + PTOK(pthread_mutex_lock(&shut_mtx)); + VTAILQ_FOREACH(ls, &heritage.socks, list) { + if (ls->sock == -2) + continue; // VCA_Shutdown + assert (ls->sock > 0); + vca_sock_opt_set(ls, NULL); + /* If one of the options on a socket has + * changed, also force a retest of whether + * the values are inherited to the + * accepted sockets. This should then + * catch any false positives from previous + * tests that could happen if the set + * value of an option happened to just be + * the OS default for that value, and + * wasn't actually inherited from the + * listening socket. */ + ls->test_heritage = 1; + } + PTOK(pthread_mutex_unlock(&shut_mtx)); + } + vca_periodic(t0); + } + NEEDLESS(return (NULL)); +} + +/*--------------------------------------------------------------------*/ + +void +VCA_Start(struct cli *cli) +{ + struct listen_sock *ls; + + (void)vca_sock_opt_init(); + + VTAILQ_FOREACH(ls, &heritage.socks, list) { + CHECK_OBJ_NOTNULL(ls->transport, TRANSPORT_MAGIC); + assert (ls->sock > 0); // We know where stdin is + if (cache_param->tcp_fastopen && + VTCP_fastopen(ls->sock, cache_param->listen_depth)) + VSL(SLT_Error, NO_VXID, + "Kernel TCP Fast Open: sock=%d, errno=%d %s", + ls->sock, errno, VAS_errtxt(errno)); + if (listen(ls->sock, cache_param->listen_depth)) { + VCLI_SetResult(cli, CLIS_CANT); + VCLI_Out(cli, "Listen failed on socket '%s': %s", + ls->endpoint, VAS_errtxt(errno)); + return; + } + AZ(ls->conn_heritage); + ls->conn_heritage = calloc(n_sock_opts, + sizeof *ls->conn_heritage); + AN(ls->conn_heritage); + ls->test_heritage = 1; + vca_sock_opt_set(ls, NULL); + if (cache_param->accept_filter && VTCP_filter_http(ls->sock)) + VSL(SLT_Error, NO_VXID, + "Kernel filtering: sock=%d, errno=%d %s", + ls->sock, errno, VAS_errtxt(errno)); + } + + PTOK(pthread_create(&VCA_thread, NULL, vca_acct, NULL)); +} + +/*--------------------------------------------------------------------*/ + +static void v_matchproto_(cli_func_t) +ccf_listen_address(struct cli *cli, const char * const *av, void *priv) +{ + struct listen_sock *ls; + char h[VTCP_ADDRBUFSIZE], p[VTCP_PORTBUFSIZE]; + + (void)cli; + (void)av; + (void)priv; + + /* + * This CLI command is primarily used by varnishtest. Don't + * respond until listen(2) has been called, in order to avoid + * a race where varnishtest::client would attempt to connect(2) + * before listen(2) has been called. + */ + while (!pool_accepting) + VTIM_sleep(.1); + + PTOK(pthread_mutex_lock(&shut_mtx)); + VTAILQ_FOREACH(ls, &heritage.socks, list) { + if (!ls->uds) { + VTCP_myname(ls->sock, h, sizeof h, p, sizeof p); + VCLI_Out(cli, "%s %s %s\n", ls->name, h, p); + } + else + VCLI_Out(cli, "%s %s -\n", ls->name, ls->endpoint); + } + PTOK(pthread_mutex_unlock(&shut_mtx)); +} + +/*--------------------------------------------------------------------*/ + +static struct cli_proto vca_cmds[] = { + { CLICMD_DEBUG_LISTEN_ADDRESS, "d", ccf_listen_address }, + { NULL } +}; + +void +VCA_Init(void) +{ + + CLI_AddFuncs(vca_cmds); + Lck_New(&pace_mtx, lck_vcapace); +} + +void +VCA_Shutdown(void) +{ + struct listen_sock *ls; + int i; + + PTOK(pthread_mutex_lock(&shut_mtx)); + VTAILQ_FOREACH(ls, &heritage.socks, list) { + i = ls->sock; + ls->sock = -2; + (void)close(i); + } + PTOK(pthread_mutex_unlock(&shut_mtx)); +} + +/*-------------------------------------------------------------------- + * Transport protocol registration + * + */ + +static VTAILQ_HEAD(,transport) transports = + VTAILQ_HEAD_INITIALIZER(transports); + +static uint16_t next_xport; + +static void +XPORT_Register(struct transport *xp) +{ + + CHECK_OBJ_NOTNULL(xp, TRANSPORT_MAGIC); + AZ(xp->number); + + xp->number = ++next_xport; + VTAILQ_INSERT_TAIL(&transports, xp, list); +} + +void +XPORT_Init(void) +{ + + ASSERT_MGT(); + +#define TRANSPORT_MACRO(name) XPORT_Register(&name##_transport); + TRANSPORTS +#undef TRANSPORT_MACRO +} + +const struct transport * +XPORT_Find(const char *name) +{ + const struct transport *xp; + + ASSERT_MGT(); + + VTAILQ_FOREACH(xp, &transports, list) + if (xp->proto_ident != NULL && + !strcasecmp(xp->proto_ident, name)) + return (xp); + return (NULL); +} + +const struct transport * +XPORT_ByNumber(uint16_t no) +{ + const struct transport *xp; + + VTAILQ_FOREACH(xp, &transports, list) + if (xp->number == no) + return (xp); + return (NULL); +} diff --git a/bin/varnishd/acceptor/cache_acceptor.h b/bin/varnishd/acceptor/cache_acceptor.h new file mode 100644 index 000000000..5e9fbe282 --- /dev/null +++ b/bin/varnishd/acceptor/cache_acceptor.h @@ -0,0 +1,37 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2015 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +/* cache_acceptor.c */ + +void VCA_Init(void); +void VCA_Start(struct cli *cli); +void VCA_Shutdown(void); diff --git a/bin/varnishd/acceptor/mgt_acceptor.c b/bin/varnishd/acceptor/mgt_acceptor.c new file mode 100644 index 000000000..aac7295c2 --- /dev/null +++ b/bin/varnishd/acceptor/mgt_acceptor.c @@ -0,0 +1,383 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2015 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * Acceptor socket management + */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mgt/mgt.h" +#include "acceptor/cache_acceptor.h" // XXX mgt_acceptor should not use +#include "acceptor/mgt_acceptor.h" +#include "common/heritage.h" + +#include "vav.h" +#include "vcli_serve.h" +#include "vsa.h" +#include "vss.h" +#include "vtcp.h" +#include "vus.h" + +struct listen_arg { + unsigned magic; +#define LISTEN_ARG_MAGIC 0xbb2fc333 + VTAILQ_ENTRY(listen_arg) list; + const char *endpoint; + const char *name; + VTAILQ_HEAD(,listen_sock) socks; + const struct transport *transport; + const struct uds_perms *perms; +}; + +struct uds_perms { + unsigned magic; +#define UDS_PERMS_MAGIC 0x84fb5635 + mode_t mode; + uid_t uid; + gid_t gid; +}; + +static VTAILQ_HEAD(,listen_arg) listen_args = + VTAILQ_HEAD_INITIALIZER(listen_args); + +static int +mac_vus_bind(void *priv, const struct sockaddr_un *uds) +{ + return (VUS_bind(uds, priv)); +} + +static int +mac_opensocket(struct listen_sock *ls) +{ + int fail; + const char *err; + + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + if (ls->sock > 0) { + MCH_Fd_Inherit(ls->sock, NULL); + closefd(&ls->sock); + } + if (!ls->uds) + ls->sock = VTCP_bind(ls->addr, NULL); + else + ls->sock = VUS_resolver(ls->endpoint, mac_vus_bind, NULL, &err); + fail = errno; + if (ls->sock < 0) { + AN(fail); + return (fail); + } + if (ls->perms != NULL) { + CHECK_OBJ(ls->perms, UDS_PERMS_MAGIC); + assert(ls->uds); + errno = 0; + if (ls->perms->mode != 0 && + chmod(ls->endpoint, ls->perms->mode) != 0) + return (errno); + if (chown(ls->endpoint, ls->perms->uid, ls->perms->gid) != 0) + return (errno); + } + MCH_Fd_Inherit(ls->sock, "sock"); + return (0); +} + +/*===================================================================== + * Reopen the accept sockets to get rid of listen status. + * returns the highest errno encountered, 0 for success + */ + +int +MAC_reopen_sockets(void) +{ + struct listen_sock *ls; + int err, fail = 0; + + VTAILQ_FOREACH(ls, &heritage.socks, list) { + VJ_master(JAIL_MASTER_PRIVPORT); + err = mac_opensocket(ls); + VJ_master(JAIL_MASTER_LOW); + if (err == 0) + continue; + fail = vmax(fail, err); + MGT_Complain(C_ERR, + "Could not reopen listen socket %s: %s", + ls->endpoint, VAS_errtxt(err)); + } + return (fail); +} + +/*--------------------------------------------------------------------*/ + +static struct listen_sock * +mk_listen_sock(const struct listen_arg *la, const struct suckaddr *sa) +{ + struct listen_sock *ls; + int fail; + + ALLOC_OBJ(ls, LISTEN_SOCK_MAGIC); + AN(ls); + ls->sock = -1; + ls->addr = VSA_Clone(sa); + AN(ls->addr); + REPLACE(ls->endpoint, la->endpoint); + ls->name = la->name; + ls->transport = la->transport; + ls->perms = la->perms; + ls->uds = VUS_is(la->endpoint); + VJ_master(JAIL_MASTER_PRIVPORT); + fail = mac_opensocket(ls); + VJ_master(JAIL_MASTER_LOW); + if (fail) { + VSA_free(&ls->addr); + free(ls->endpoint); + FREE_OBJ(ls); + if (fail != EAFNOSUPPORT) + ARGV_ERR("Could not get socket %s: %s\n", + la->endpoint, VAS_errtxt(fail)); + return (NULL); + } + return (ls); +} + +static int v_matchproto_(vss_resolved_f) +mac_tcp(void *priv, const struct suckaddr *sa) +{ + struct listen_arg *la; + struct listen_sock *ls; + char abuf[VTCP_ADDRBUFSIZE], pbuf[VTCP_PORTBUFSIZE]; + char nbuf[VTCP_ADDRBUFSIZE+VTCP_PORTBUFSIZE+2]; + + CAST_OBJ_NOTNULL(la, priv, LISTEN_ARG_MAGIC); + + VTAILQ_FOREACH(ls, &heritage.socks, list) { + if (!ls->uds && !VSA_Compare(sa, ls->addr)) + ARGV_ERR("-a arguments %s and %s have same address\n", + ls->endpoint, la->endpoint); + } + ls = mk_listen_sock(la, sa); + if (ls == NULL) + return (0); + AZ(ls->uds); + if (VSA_Port(ls->addr) == 0) { + /* + * If the argv port number is zero, we adopt whatever + * port number this VTCP_bind() found us, as if + * it was specified by the argv. + */ + VSA_free(&ls->addr); + ls->addr = VTCP_my_suckaddr(ls->sock); + VTCP_myname(ls->sock, abuf, sizeof abuf, + pbuf, sizeof pbuf); + if (VSA_Get_Proto(sa) == AF_INET6) + bprintf(nbuf, "[%s]:%s", abuf, pbuf); + else + bprintf(nbuf, "%s:%s", abuf, pbuf); + REPLACE(ls->endpoint, nbuf); + } + VTAILQ_INSERT_TAIL(&la->socks, ls, arglist); + VTAILQ_INSERT_TAIL(&heritage.socks, ls, list); + return (0); +} + +static int v_matchproto_(vus_resolved_f) +mac_uds(void *priv, const struct sockaddr_un *uds) +{ + struct listen_arg *la; + struct listen_sock *ls; + + CAST_OBJ_NOTNULL(la, priv, LISTEN_ARG_MAGIC); + (void) uds; + + VTAILQ_FOREACH(ls, &heritage.socks, list) { + if (ls->uds && strcmp(ls->endpoint, la->endpoint) == 0) + ARGV_ERR("-a arguments %s and %s have same address\n", + ls->endpoint, la->endpoint); + } + ls = mk_listen_sock(la, bogo_ip); + if (ls == NULL) + return (0); + AN(ls->uds); + VTAILQ_INSERT_TAIL(&la->socks, ls, arglist); + VTAILQ_INSERT_TAIL(&heritage.socks, ls, list); + return (0); +} + +void +MAC_Arg(const char *spec) +{ + char **av; + struct listen_arg *la; + const char *err; + int error; + const struct transport *xp = NULL; + const char *name; + char name_buf[8]; + static unsigned seq = 0; + struct passwd *pwd = NULL; + struct group *grp = NULL; + mode_t mode = 0; + struct uds_perms *perms; + + av = MGT_NamedArg(spec, &name, "-a"); + AN(av); + + ALLOC_OBJ(la, LISTEN_ARG_MAGIC); + AN(la); + VTAILQ_INIT(&la->socks); + VTAILQ_INSERT_TAIL(&listen_args, la, list); + la->endpoint = av[1]; + + if (name == NULL) { + bprintf(name_buf, "a%u", seq++); + name = strdup(name_buf); + AN(name); + } + la->name = name; + + if (*la->endpoint != '/' && strchr(la->endpoint, '/') != NULL) + ARGV_ERR("Unix domain socket addresses must be" + " absolute paths in -a (%s)\n", la->endpoint); + + if (VUS_is(la->endpoint) && heritage.min_vcl_version < 41) + heritage.min_vcl_version = 41; + + for (int i = 2; av[i] != NULL; i++) { + char *eq, *val; + int len; + + if ((eq = strchr(av[i], '=')) == NULL) { + if (xp != NULL) + ARGV_ERR("Too many protocol sub-args" + " in -a (%s)\n", av[i]); + xp = XPORT_Find(av[i]); + if (xp == NULL) + ARGV_ERR("Unknown protocol '%s'\n", av[i]); + continue; + } + if (la->endpoint[0] != '/') + ARGV_ERR("Invalid sub-arg %s" + " in -a\n", av[i]); + + val = eq + 1; + len = eq - av[i]; + assert(len >= 0); + if (len == 0) + ARGV_ERR("Invalid sub-arg %s in -a\n", av[i]); + + if (strncmp(av[i], "user", len) == 0) { + if (pwd != NULL) + ARGV_ERR("Too many user sub-args in -a (%s)\n", + av[i]); + pwd = getpwnam(val); + if (pwd == NULL) + ARGV_ERR("Unknown user %s in -a\n", val); + continue; + } + + if (strncmp(av[i], "group", len) == 0) { + if (grp != NULL) + ARGV_ERR("Too many group sub-args in -a (%s)\n", + av[i]); + grp = getgrnam(val); + if (grp == NULL) + ARGV_ERR("Unknown group %s in -a\n", val); + continue; + } + + if (strncmp(av[i], "mode", len) == 0) { + long m; + char *p; + + if (mode != 0) + ARGV_ERR("Too many mode sub-args in -a (%s)\n", + av[i]); + if (*val == '\0') + ARGV_ERR("Empty mode sub-arg in -a\n"); + errno = 0; + m = strtol(val, &p, 8); + if (*p != '\0') + ARGV_ERR("Invalid mode sub-arg %s in -a\n", + val); + if (errno) + ARGV_ERR("Cannot parse mode sub-arg %s in -a: " + "%s\n", val, VAS_errtxt(errno)); + if (m <= 0 || m > 0777) + ARGV_ERR("Mode sub-arg %s out of range in -a\n", + val); + mode = (mode_t) m; + continue; + } + + ARGV_ERR("Invalid sub-arg %s in -a\n", av[i]); + } + + if (xp == NULL) + xp = XPORT_Find("http"); + AN(xp); + la->transport = xp; + + if (pwd != NULL || grp != NULL || mode != 0) { + ALLOC_OBJ(perms, UDS_PERMS_MAGIC); + AN(perms); + if (pwd != NULL) + perms->uid = pwd->pw_uid; + else + perms->uid = (uid_t) -1; + if (grp != NULL) + perms->gid = grp->gr_gid; + else + perms->gid = (gid_t) -1; + perms->mode = mode; + la->perms = perms; + } + else + AZ(la->perms); + + if (VUS_is(la->endpoint)) + error = VUS_resolver(av[1], mac_uds, la, &err); + else + error = VSS_resolver_range(av[1], "80", mac_tcp, la, &err); + + if (error) + ARGV_ERR("Got no socket(s) for %s (%s)\n", av[1], err); + else if (VTAILQ_EMPTY(&la->socks)) + ARGV_ERR("Got no socket(s) for %s\n", av[1]); + VAV_Free(av); +} diff --git a/bin/varnishd/acceptor/mgt_acceptor.h b/bin/varnishd/acceptor/mgt_acceptor.h new file mode 100644 index 000000000..61eb43627 --- /dev/null +++ b/bin/varnishd/acceptor/mgt_acceptor.h @@ -0,0 +1,36 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2011 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +/* mgt_acceptor.c */ + +void MAC_Arg(const char *); +int MAC_reopen_sockets(void); From nils.goroll at uplex.de Mon Sep 30 15:53:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 15:53:06 +0000 (UTC) Subject: [master] a3e790a0b acceptor: rename acceptor code with the VCA prefix Message-ID: <20240930155306.A890411D0E5@lists.varnish-cache.org> commit a3e790a0bab92e9d9bf1189020bebf4da3e97de5 Author: Asad Sajjad Ahmed Date: Mon Sep 4 13:14:16 2023 +0200 acceptor: rename acceptor code with the VCA prefix Consistently name acceptor function to VCA/vca prefix, no matter if in the manager or cache worker. Signed-off-by: Asad Sajjad Ahmed Resolved merge conflict due to port range support. Patch edited to reflect name decision from bugswash. Signed-off-by: Nils Goroll diff --git a/bin/varnishd/acceptor/mgt_acceptor.c b/bin/varnishd/acceptor/mgt_acceptor.c index aac7295c2..ce93f7b3e 100644 --- a/bin/varnishd/acceptor/mgt_acceptor.c +++ b/bin/varnishd/acceptor/mgt_acceptor.c @@ -79,13 +79,13 @@ static VTAILQ_HEAD(,listen_arg) listen_args = VTAILQ_HEAD_INITIALIZER(listen_args); static int -mac_vus_bind(void *priv, const struct sockaddr_un *uds) +vca_vus_bind(void *priv, const struct sockaddr_un *uds) { return (VUS_bind(uds, priv)); } static int -mac_opensocket(struct listen_sock *ls) +vca_opensocket(struct listen_sock *ls) { int fail; const char *err; @@ -98,7 +98,7 @@ mac_opensocket(struct listen_sock *ls) if (!ls->uds) ls->sock = VTCP_bind(ls->addr, NULL); else - ls->sock = VUS_resolver(ls->endpoint, mac_vus_bind, NULL, &err); + ls->sock = VUS_resolver(ls->endpoint, vca_vus_bind, NULL, &err); fail = errno; if (ls->sock < 0) { AN(fail); @@ -124,14 +124,14 @@ mac_opensocket(struct listen_sock *ls) */ int -MAC_reopen_sockets(void) +VCA_reopen_sockets(void) { struct listen_sock *ls; int err, fail = 0; VTAILQ_FOREACH(ls, &heritage.socks, list) { VJ_master(JAIL_MASTER_PRIVPORT); - err = mac_opensocket(ls); + err = vca_opensocket(ls); VJ_master(JAIL_MASTER_LOW); if (err == 0) continue; @@ -162,7 +162,7 @@ mk_listen_sock(const struct listen_arg *la, const struct suckaddr *sa) ls->perms = la->perms; ls->uds = VUS_is(la->endpoint); VJ_master(JAIL_MASTER_PRIVPORT); - fail = mac_opensocket(ls); + fail = vca_opensocket(ls); VJ_master(JAIL_MASTER_LOW); if (fail) { VSA_free(&ls->addr); @@ -177,7 +177,7 @@ mk_listen_sock(const struct listen_arg *la, const struct suckaddr *sa) } static int v_matchproto_(vss_resolved_f) -mac_tcp(void *priv, const struct suckaddr *sa) +vca_tcp(void *priv, const struct suckaddr *sa) { struct listen_arg *la; struct listen_sock *ls; @@ -217,7 +217,7 @@ mac_tcp(void *priv, const struct suckaddr *sa) } static int v_matchproto_(vus_resolved_f) -mac_uds(void *priv, const struct sockaddr_un *uds) +vca_uds(void *priv, const struct sockaddr_un *uds) { struct listen_arg *la; struct listen_sock *ls; @@ -240,7 +240,7 @@ mac_uds(void *priv, const struct sockaddr_un *uds) } void -MAC_Arg(const char *spec) +VCA_Arg(const char *spec) { char **av; struct listen_arg *la; @@ -371,9 +371,9 @@ MAC_Arg(const char *spec) AZ(la->perms); if (VUS_is(la->endpoint)) - error = VUS_resolver(av[1], mac_uds, la, &err); + error = VUS_resolver(av[1], vca_uds, la, &err); else - error = VSS_resolver_range(av[1], "80", mac_tcp, la, &err); + error = VSS_resolver_range(av[1], "80", vca_tcp, la, &err); if (error) ARGV_ERR("Got no socket(s) for %s (%s)\n", av[1], err); diff --git a/bin/varnishd/acceptor/mgt_acceptor.h b/bin/varnishd/acceptor/mgt_acceptor.h index 61eb43627..91ef0dcbe 100644 --- a/bin/varnishd/acceptor/mgt_acceptor.h +++ b/bin/varnishd/acceptor/mgt_acceptor.h @@ -32,5 +32,5 @@ /* mgt_acceptor.c */ -void MAC_Arg(const char *); -int MAC_reopen_sockets(void); +void VCA_Arg(const char *); +int VCA_reopen_sockets(void); From nils.goroll at uplex.de Mon Sep 30 15:53:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 15:53:06 +0000 (UTC) Subject: [master] 04354217d acceptor: split mgt_acceptor.c Message-ID: <20240930155306.C630B11D0EA@lists.varnish-cache.org> commit 04354217dc0b94a094e53304037692a6336a58ac Author: Nils Goroll Date: Thu Sep 5 10:40:42 2024 +0200 acceptor: split mgt_acceptor.c Currently, changes between TCP and UDS is inter-mixed when opening listen sockets, so split them up into their own files. This is cleaner and is needed to add QUIC listen endpoint in the future. Signed-off-by: Asad Sajjad Ahmed Conflicts: bin/varnishd/acceptor/mgt_acceptor.c Signed-off-by: Nils Goroll diff --git a/bin/varnishd/acceptor/acceptor_tcp.h b/bin/varnishd/acceptor/acceptor_tcp.h new file mode 100644 index 000000000..350d5f72a --- /dev/null +++ b/bin/varnishd/acceptor/acceptor_tcp.h @@ -0,0 +1,39 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2023 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * Author: Asad Sajjad Ahmed + * + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +struct listen_arg; + +int vca_tcp_config(void); +int vca_tcp_open(char **, struct listen_arg *, const char **); +int vca_tcp_reopen(void); diff --git a/bin/varnishd/acceptor/acceptor_uds.h b/bin/varnishd/acceptor/acceptor_uds.h new file mode 100644 index 000000000..6a04546c3 --- /dev/null +++ b/bin/varnishd/acceptor/acceptor_uds.h @@ -0,0 +1,46 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2023 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * Author: Asad Sajjad Ahmed + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +struct listen_arg; + +struct uds_perms { + unsigned magic; +#define UDS_PERMS_MAGIC 0x84fb5635 + mode_t mode; + uid_t uid; + gid_t gid; +}; + +int vca_uds_config(void); +int vca_uds_open(char **, struct listen_arg *, const char **); +int vca_uds_reopen(void); diff --git a/bin/varnishd/acceptor/mgt_acceptor.c b/bin/varnishd/acceptor/mgt_acceptor.c index ce93f7b3e..1e25a9078 100644 --- a/bin/varnishd/acceptor/mgt_acceptor.c +++ b/bin/varnishd/acceptor/mgt_acceptor.c @@ -41,8 +41,6 @@ #include #include #include -#include -#include #include "mgt/mgt.h" #include "acceptor/cache_acceptor.h" // XXX mgt_acceptor should not use @@ -56,68 +54,9 @@ #include "vtcp.h" #include "vus.h" -struct listen_arg { - unsigned magic; -#define LISTEN_ARG_MAGIC 0xbb2fc333 - VTAILQ_ENTRY(listen_arg) list; - const char *endpoint; - const char *name; - VTAILQ_HEAD(,listen_sock) socks; - const struct transport *transport; - const struct uds_perms *perms; -}; - -struct uds_perms { - unsigned magic; -#define UDS_PERMS_MAGIC 0x84fb5635 - mode_t mode; - uid_t uid; - gid_t gid; -}; - static VTAILQ_HEAD(,listen_arg) listen_args = VTAILQ_HEAD_INITIALIZER(listen_args); -static int -vca_vus_bind(void *priv, const struct sockaddr_un *uds) -{ - return (VUS_bind(uds, priv)); -} - -static int -vca_opensocket(struct listen_sock *ls) -{ - int fail; - const char *err; - - CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); - if (ls->sock > 0) { - MCH_Fd_Inherit(ls->sock, NULL); - closefd(&ls->sock); - } - if (!ls->uds) - ls->sock = VTCP_bind(ls->addr, NULL); - else - ls->sock = VUS_resolver(ls->endpoint, vca_vus_bind, NULL, &err); - fail = errno; - if (ls->sock < 0) { - AN(fail); - return (fail); - } - if (ls->perms != NULL) { - CHECK_OBJ(ls->perms, UDS_PERMS_MAGIC); - assert(ls->uds); - errno = 0; - if (ls->perms->mode != 0 && - chmod(ls->endpoint, ls->perms->mode) != 0) - return (errno); - if (chown(ls->endpoint, ls->perms->uid, ls->perms->gid) != 0) - return (errno); - } - MCH_Fd_Inherit(ls->sock, "sock"); - return (0); -} - /*===================================================================== * Reopen the accept sockets to get rid of listen status. * returns the highest errno encountered, 0 for success @@ -126,118 +65,15 @@ vca_opensocket(struct listen_sock *ls) int VCA_reopen_sockets(void) { - struct listen_sock *ls; - int err, fail = 0; - - VTAILQ_FOREACH(ls, &heritage.socks, list) { - VJ_master(JAIL_MASTER_PRIVPORT); - err = vca_opensocket(ls); - VJ_master(JAIL_MASTER_LOW); - if (err == 0) - continue; - fail = vmax(fail, err); - MGT_Complain(C_ERR, - "Could not reopen listen socket %s: %s", - ls->endpoint, VAS_errtxt(err)); - } - return (fail); -} - -/*--------------------------------------------------------------------*/ - -static struct listen_sock * -mk_listen_sock(const struct listen_arg *la, const struct suckaddr *sa) -{ - struct listen_sock *ls; - int fail; - - ALLOC_OBJ(ls, LISTEN_SOCK_MAGIC); - AN(ls); - ls->sock = -1; - ls->addr = VSA_Clone(sa); - AN(ls->addr); - REPLACE(ls->endpoint, la->endpoint); - ls->name = la->name; - ls->transport = la->transport; - ls->perms = la->perms; - ls->uds = VUS_is(la->endpoint); - VJ_master(JAIL_MASTER_PRIVPORT); - fail = vca_opensocket(ls); - VJ_master(JAIL_MASTER_LOW); - if (fail) { - VSA_free(&ls->addr); - free(ls->endpoint); - FREE_OBJ(ls); - if (fail != EAFNOSUPPORT) - ARGV_ERR("Could not get socket %s: %s\n", - la->endpoint, VAS_errtxt(fail)); - return (NULL); - } - return (ls); -} - -static int v_matchproto_(vss_resolved_f) -vca_tcp(void *priv, const struct suckaddr *sa) -{ - struct listen_arg *la; - struct listen_sock *ls; - char abuf[VTCP_ADDRBUFSIZE], pbuf[VTCP_PORTBUFSIZE]; - char nbuf[VTCP_ADDRBUFSIZE+VTCP_PORTBUFSIZE+2]; + int fail, fail2; - CAST_OBJ_NOTNULL(la, priv, LISTEN_ARG_MAGIC); + fail = vca_tcp_open(); + fail2 = vca_uds_open(); - VTAILQ_FOREACH(ls, &heritage.socks, list) { - if (!ls->uds && !VSA_Compare(sa, ls->addr)) - ARGV_ERR("-a arguments %s and %s have same address\n", - ls->endpoint, la->endpoint); - } - ls = mk_listen_sock(la, sa); - if (ls == NULL) - return (0); - AZ(ls->uds); - if (VSA_Port(ls->addr) == 0) { - /* - * If the argv port number is zero, we adopt whatever - * port number this VTCP_bind() found us, as if - * it was specified by the argv. - */ - VSA_free(&ls->addr); - ls->addr = VTCP_my_suckaddr(ls->sock); - VTCP_myname(ls->sock, abuf, sizeof abuf, - pbuf, sizeof pbuf); - if (VSA_Get_Proto(sa) == AF_INET6) - bprintf(nbuf, "[%s]:%s", abuf, pbuf); - else - bprintf(nbuf, "%s:%s", abuf, pbuf); - REPLACE(ls->endpoint, nbuf); - } - VTAILQ_INSERT_TAIL(&la->socks, ls, arglist); - VTAILQ_INSERT_TAIL(&heritage.socks, ls, list); - return (0); + return (vmax(fail, fail2)); } -static int v_matchproto_(vus_resolved_f) -vca_uds(void *priv, const struct sockaddr_un *uds) -{ - struct listen_arg *la; - struct listen_sock *ls; - - CAST_OBJ_NOTNULL(la, priv, LISTEN_ARG_MAGIC); - (void) uds; - - VTAILQ_FOREACH(ls, &heritage.socks, list) { - if (ls->uds && strcmp(ls->endpoint, la->endpoint) == 0) - ARGV_ERR("-a arguments %s and %s have same address\n", - ls->endpoint, la->endpoint); - } - ls = mk_listen_sock(la, bogo_ip); - if (ls == NULL) - return (0); - AN(ls->uds); - VTAILQ_INSERT_TAIL(&la->socks, ls, arglist); - VTAILQ_INSERT_TAIL(&heritage.socks, ls, list); - return (0); -} +/*--------------------------------------------------------------------*/ void VCA_Arg(const char *spec) @@ -246,14 +82,9 @@ VCA_Arg(const char *spec) struct listen_arg *la; const char *err; int error; - const struct transport *xp = NULL; const char *name; char name_buf[8]; static unsigned seq = 0; - struct passwd *pwd = NULL; - struct group *grp = NULL; - mode_t mode = 0; - struct uds_perms *perms; av = MGT_NamedArg(spec, &name, "-a"); AN(av); @@ -269,111 +100,13 @@ VCA_Arg(const char *spec) name = strdup(name_buf); AN(name); } - la->name = name; - - if (*la->endpoint != '/' && strchr(la->endpoint, '/') != NULL) - ARGV_ERR("Unix domain socket addresses must be" - " absolute paths in -a (%s)\n", la->endpoint); - - if (VUS_is(la->endpoint) && heritage.min_vcl_version < 41) - heritage.min_vcl_version = 41; - - for (int i = 2; av[i] != NULL; i++) { - char *eq, *val; - int len; - - if ((eq = strchr(av[i], '=')) == NULL) { - if (xp != NULL) - ARGV_ERR("Too many protocol sub-args" - " in -a (%s)\n", av[i]); - xp = XPORT_Find(av[i]); - if (xp == NULL) - ARGV_ERR("Unknown protocol '%s'\n", av[i]); - continue; - } - if (la->endpoint[0] != '/') - ARGV_ERR("Invalid sub-arg %s" - " in -a\n", av[i]); - - val = eq + 1; - len = eq - av[i]; - assert(len >= 0); - if (len == 0) - ARGV_ERR("Invalid sub-arg %s in -a\n", av[i]); - - if (strncmp(av[i], "user", len) == 0) { - if (pwd != NULL) - ARGV_ERR("Too many user sub-args in -a (%s)\n", - av[i]); - pwd = getpwnam(val); - if (pwd == NULL) - ARGV_ERR("Unknown user %s in -a\n", val); - continue; - } - - if (strncmp(av[i], "group", len) == 0) { - if (grp != NULL) - ARGV_ERR("Too many group sub-args in -a (%s)\n", - av[i]); - grp = getgrnam(val); - if (grp == NULL) - ARGV_ERR("Unknown group %s in -a\n", val); - continue; - } - - if (strncmp(av[i], "mode", len) == 0) { - long m; - char *p; - - if (mode != 0) - ARGV_ERR("Too many mode sub-args in -a (%s)\n", - av[i]); - if (*val == '\0') - ARGV_ERR("Empty mode sub-arg in -a\n"); - errno = 0; - m = strtol(val, &p, 8); - if (*p != '\0') - ARGV_ERR("Invalid mode sub-arg %s in -a\n", - val); - if (errno) - ARGV_ERR("Cannot parse mode sub-arg %s in -a: " - "%s\n", val, VAS_errtxt(errno)); - if (m <= 0 || m > 0777) - ARGV_ERR("Mode sub-arg %s out of range in -a\n", - val); - mode = (mode_t) m; - continue; - } - - ARGV_ERR("Invalid sub-arg %s in -a\n", av[i]); - } - - if (xp == NULL) - xp = XPORT_Find("http"); - AN(xp); - la->transport = xp; - if (pwd != NULL || grp != NULL || mode != 0) { - ALLOC_OBJ(perms, UDS_PERMS_MAGIC); - AN(perms); - if (pwd != NULL) - perms->uid = pwd->pw_uid; - else - perms->uid = (uid_t) -1; - if (grp != NULL) - perms->gid = grp->gr_gid; - else - perms->gid = (gid_t) -1; - perms->mode = mode; - la->perms = perms; - } - else - AZ(la->perms); + la->name = name; if (VUS_is(la->endpoint)) - error = VUS_resolver(av[1], vca_uds, la, &err); + error = vca_uds_open(av[1], la, &err); else - error = VSS_resolver_range(av[1], "80", vca_tcp, la, &err); + error = vca_tcp_open(av[1], la, &err); if (error) ARGV_ERR("Got no socket(s) for %s (%s)\n", av[1], err); diff --git a/bin/varnishd/acceptor/mgt_acceptor.h b/bin/varnishd/acceptor/mgt_acceptor.h index 91ef0dcbe..5558383c5 100644 --- a/bin/varnishd/acceptor/mgt_acceptor.h +++ b/bin/varnishd/acceptor/mgt_acceptor.h @@ -32,5 +32,16 @@ /* mgt_acceptor.c */ +struct listen_arg { + unsigned magic; +#define LISTEN_ARG_MAGIC 0xbb2fc333 + VTAILQ_ENTRY(listen_arg) list; + const char *endpoint; + const char *name; + VTAILQ_HEAD(,listen_sock) socks; + const struct transport *transport; + const struct uds_perms *perms; +}; + void VCA_Arg(const char *); int VCA_reopen_sockets(void); diff --git a/bin/varnishd/acceptor/mgt_acceptor_tcp.c b/bin/varnishd/acceptor/mgt_acceptor_tcp.c new file mode 100644 index 000000000..279ea7479 --- /dev/null +++ b/bin/varnishd/acceptor/mgt_acceptor_tcp.c @@ -0,0 +1,225 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2023 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * Author: Asad Sajjad Ahmed + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mgt/mgt.h" +#include "common/heritage.h" + +#include "acceptor/cache_acceptor.h" +#include "acceptor/acceptor_tcp.h" +#include "acceptor/mgt_acceptor.h" + +#include "vav.h" +#include "vcli_serve.h" +#include "vsa.h" +#include "vss.h" +#include "vtcp.h" + +int +vca_tcp_config(void) +{ + + return (0); +} + +static int +vca_tcp_opensocket(struct listen_sock *ls) +{ + int fail; + + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + + if (ls->sock > 0) { + MCH_Fd_Inherit(ls->sock, NULL); + closefd(&ls->sock); + } + + ls->sock = VTCP_bind(ls->addr, NULL); + fail = errno; + + if (ls->sock < 0) { + AN(fail); + return (fail); + } + + AZ(ls->perms); + MCH_Fd_Inherit(ls->sock, "sock"); + + return (0); +} + +static int v_matchproto_(vss_resolved_f) +vca_tcp_open_cb(void *priv, const struct suckaddr *sa) +{ + struct listen_arg *la; + struct listen_sock *ls; + char abuf[VTCP_ADDRBUFSIZE], pbuf[VTCP_PORTBUFSIZE]; + char nbuf[VTCP_ADDRBUFSIZE+VTCP_PORTBUFSIZE+2]; + int fail; + + CAST_OBJ_NOTNULL(la, priv, LISTEN_ARG_MAGIC); + + VTAILQ_FOREACH(ls, &heritage.socks, list) { + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + + if (!ls->uds && !VSA_Compare(sa, ls->addr)) + ARGV_ERR("-a arguments %s and %s have same address\n", + ls->endpoint, la->endpoint); + } + + ALLOC_OBJ(ls, LISTEN_SOCK_MAGIC); + AN(ls); + + ls->sock = -1; + + ls->addr = VSA_Clone(sa); + AN(ls->addr); + + REPLACE(ls->endpoint, la->endpoint); + ls->name = la->name; + ls->transport = la->transport; + ls->perms = la->perms; + + VJ_master(JAIL_MASTER_PRIVPORT); + fail = vca_tcp_opensocket(ls); + VJ_master(JAIL_MASTER_LOW); + + if (fail) { + VSA_free(&ls->addr); + free(ls->endpoint); + FREE_OBJ(ls); + if (fail != EAFNOSUPPORT) + ARGV_ERR("Could not get socket %s: %s\n", + la->endpoint, VAS_errtxt(fail)); + return (0); + } + + AZ(ls->uds); + + if (VSA_Port(ls->addr) == 0) { + /* + * If the argv port number is zero, we adopt whatever + * port number this VTCP_bind() found us, as if + * it was specified by the argv. + */ + VSA_free(&ls->addr); + ls->addr = VTCP_my_suckaddr(ls->sock); + VTCP_myname(ls->sock, abuf, sizeof abuf, + pbuf, sizeof pbuf); + if (VSA_Get_Proto(sa) == AF_INET6) + bprintf(nbuf, "[%s]:%s", abuf, pbuf); + else + bprintf(nbuf, "%s:%s", abuf, pbuf); + REPLACE(ls->endpoint, nbuf); + } + + VTAILQ_INSERT_TAIL(&la->socks, ls, arglist); + VTAILQ_INSERT_TAIL(&heritage.socks, ls, list); + + return (0); +} + + +int +vca_tcp_open(char **av, struct listen_arg *la, const char **err) +{ + const struct transport *xp = NULL; + + CHECK_OBJ_NOTNULL(la, LISTEN_ARG_MAGIC); + AN(av); + AN(err); + AZ(la->perms); + + if (strchr(la->endpoint, '/') != NULL) + ARGV_ERR("Unix domain socket addresses must be" + " absolute paths in -a (%s)\n", la->endpoint); + + for (int i = 2; av[i] != NULL; i++) { + if (strchr(av[i], '=') == NULL) { + if (xp != NULL) + ARGV_ERR("Too many protocol sub-args" + " in -a (%s)\n", av[i]); + xp = XPORT_Find(av[i]); + if (xp == NULL) + ARGV_ERR("Unknown protocol '%s'\n", av[i]); + continue; + } + + ARGV_ERR("Invalid sub-arg %s in -a\n", av[i]); + } + + if (xp == NULL) + xp = XPORT_Find("http"); + + AN(xp); + la->transport = xp; + + return (VSS_resolver_range(av[1], "80", vca_tcp_open_cb, la, err)); +} + +int +vca_tcp_reopen(void) +{ + struct listen_sock *ls; + int err, fail = 0; + + VTAILQ_FOREACH(ls, &heritage.socks, list) { + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + + if (ls->uds) + continue; + + VJ_master(JAIL_MASTER_PRIVPORT); + err = vca_tcp_opensocket(ls); + VJ_master(JAIL_MASTER_LOW); + + if (err == 0) + continue; + + fail = vmax(fail, err); + MGT_Complain(C_ERR, "Could not reopen listen socket %s: %s", + ls->endpoint, VAS_errtxt(err)); + } + + return (fail); +} diff --git a/bin/varnishd/acceptor/mgt_acceptor_uds.c b/bin/varnishd/acceptor/mgt_acceptor_uds.c new file mode 100644 index 000000000..93641e39d --- /dev/null +++ b/bin/varnishd/acceptor/mgt_acceptor_uds.c @@ -0,0 +1,301 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2023 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * Author: Asad Sajjad Ahmed + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * Acceptor socket management + */ + +#include "config.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "mgt/mgt.h" +#include "common/heritage.h" + +#include "acceptor/cache_acceptor.h" +#include "acceptor/acceptor_uds.h" +#include "acceptor/mgt_acceptor.h" + +#include "vav.h" +#include "vcli_serve.h" +#include "vsa.h" +#include "vss.h" +#include "vus.h" + +int +vca_uds_config(void) +{ + + return (0); +} + +static int +vca_vus_bind(void *priv, const struct sockaddr_un *uds) +{ + return (VUS_bind(uds, priv)); +} + +static int +vca_uds_opensocket(struct listen_sock *ls) +{ + int fail; + const char *err; + + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + + if (ls->sock > 0) { + MCH_Fd_Inherit(ls->sock, NULL); + closefd(&ls->sock); + } + + ls->sock = VUS_resolver(ls->endpoint, vca_vus_bind, NULL, &err); + fail = errno; + + if (ls->sock < 0) { + AN(fail); + return (fail); + } + + if (ls->perms != NULL) { + CHECK_OBJ(ls->perms, UDS_PERMS_MAGIC); + assert(ls->uds); + errno = 0; + if (ls->perms->mode != 0 && + chmod(ls->endpoint, ls->perms->mode) != 0) + return (errno); + if (chown(ls->endpoint, ls->perms->uid, ls->perms->gid) != 0) + return (errno); + } + + MCH_Fd_Inherit(ls->sock, "sock"); + return (0); +} + +static int v_matchproto_(vus_resolved_f) +vca_uds_open_cb(void *priv, const struct sockaddr_un *uds) +{ + struct listen_arg *la; + struct listen_sock *ls; + int fail; + + CAST_OBJ_NOTNULL(la, priv, LISTEN_ARG_MAGIC); + (void) uds; + + VTAILQ_FOREACH(ls, &heritage.socks, list) { + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + + if (ls->uds && strcmp(ls->endpoint, la->endpoint) == 0) + ARGV_ERR("-a arguments %s and %s have same address\n", + ls->endpoint, la->endpoint); + } + + ALLOC_OBJ(ls, LISTEN_SOCK_MAGIC); + AN(ls); + + ls->sock = -1; + + ls->addr = VSA_Clone(bogo_ip); + AN(ls->addr); + + REPLACE(ls->endpoint, la->endpoint); + ls->name = la->name; + ls->transport = la->transport; + ls->perms = la->perms; + ls->uds = 1; + + VJ_master(JAIL_MASTER_PRIVPORT); + fail = vca_uds_opensocket(ls); + VJ_master(JAIL_MASTER_LOW); + + if (fail) { + VSA_free(&ls->addr); + free(ls->endpoint); + FREE_OBJ(ls); + if (fail != EAFNOSUPPORT) + ARGV_ERR("Could not get socket %s: %s\n", + la->endpoint, VAS_errtxt(fail)); + return (0); + } + + VTAILQ_INSERT_TAIL(&la->socks, ls, arglist); + VTAILQ_INSERT_TAIL(&heritage.socks, ls, list); + + return (0); +} + +int +vca_uds_open(char **av, struct listen_arg *la, const char **err) +{ + + const struct transport *xp = NULL; + struct passwd *pwd = NULL; + struct group *grp = NULL; + struct uds_perms *perms; + mode_t mode = 0; + + CHECK_OBJ_NOTNULL(la, LISTEN_ARG_MAGIC); + AN(av); + AN(err); + + if (*la->endpoint != '/' && strchr(la->endpoint, '/') != NULL) + ARGV_ERR("Unix domain socket addresses must be" + " absolute paths in -a (%s)\n", la->endpoint); + + heritage.min_vcl_version = vmax(heritage.min_vcl_version, 41U); + + for (int i = 2; av[i] != NULL; i++) { + char *eq, *val; + int len; + + if ((eq = strchr(av[i], '=')) == NULL) { + if (xp != NULL) + ARGV_ERR("Too many protocol sub-args" + " in -a (%s)\n", av[i]); + xp = XPORT_Find(av[i]); + if (xp == NULL) + ARGV_ERR("Unknown protocol '%s'\n", av[i]); + continue; + } + if (la->endpoint[0] != '/') + ARGV_ERR("Invalid sub-arg %s" + " in -a\n", av[i]); + + val = eq + 1; + len = eq - av[i]; + assert(len >= 0); + if (len == 0) + ARGV_ERR("Invalid sub-arg %s in -a\n", av[i]); + + if (strncmp(av[i], "user", len) == 0) { + if (pwd != NULL) + ARGV_ERR("Too many user sub-args in -a (%s)\n", + av[i]); + pwd = getpwnam(val); + if (pwd == NULL) + ARGV_ERR("Unknown user %s in -a\n", val); + continue; + } + + if (strncmp(av[i], "group", len) == 0) { + if (grp != NULL) + ARGV_ERR("Too many group sub-args in -a (%s)\n", + av[i]); + grp = getgrnam(val); + if (grp == NULL) + ARGV_ERR("Unknown group %s in -a\n", val); + continue; + } + + if (strncmp(av[i], "mode", len) == 0) { + long m; + char *p; + + if (mode != 0) + ARGV_ERR("Too many mode sub-args in -a (%s)\n", + av[i]); + if (*val == '\0') + ARGV_ERR("Empty mode sub-arg in -a\n"); + errno = 0; + m = strtol(val, &p, 8); + if (*p != '\0') + ARGV_ERR("Invalid mode sub-arg %s in -a\n", + val); + if (errno) + ARGV_ERR("Cannot parse mode sub-arg %s in -a: " + "%s\n", val, VAS_errtxt(errno)); + if (m <= 0 || m > 0777) + ARGV_ERR("Mode sub-arg %s out of range in -a\n", + val); + mode = (mode_t) m; + continue; + } + + ARGV_ERR("Invalid sub-arg %s in -a\n", av[i]); + } + + if (xp == NULL) + xp = XPORT_Find("http"); + + AN(xp); + la->transport = xp; + + if (pwd != NULL || grp != NULL || mode != 0) { + ALLOC_OBJ(perms, UDS_PERMS_MAGIC); + AN(perms); + if (pwd != NULL) + perms->uid = pwd->pw_uid; + else + perms->uid = (uid_t) -1; + if (grp != NULL) + perms->gid = grp->gr_gid; + else + perms->gid = (gid_t) -1; + perms->mode = mode; + la->perms = perms; + } + + return (VUS_resolver(la->endpoint, vca_uds_open_cb, la, err)); +} + +int +vca_uds_reopen(void) +{ + struct listen_sock *ls; + int err, fail = 0; + + VTAILQ_FOREACH(ls, &heritage.socks, list) { + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + + if (!ls->uds) + continue; + + VJ_master(JAIL_MASTER_PRIVPORT); + err = vca_uds_opensocket(ls); + VJ_master(JAIL_MASTER_LOW); + + if (err == 0) + continue; + + fail = vmax(fail, err); + MGT_Complain(C_ERR, "Could not reopen listen socket %s: %s", + ls->endpoint, VAS_errtxt(err)); + } + + return (fail); +} From nils.goroll at uplex.de Mon Sep 30 15:53:06 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 15:53:06 +0000 (UTC) Subject: [master] fc8b7392f acceptor: split acceptor.c Message-ID: <20240930155306.E854511D0ED@lists.varnish-cache.org> commit fc8b7392fbc7ccc75b9c572b57dc7372db1a875e Author: Asad Sajjad Ahmed Date: Tue Sep 5 15:28:34 2023 +0200 acceptor: split acceptor.c Currently, changes between TCP and UDS is inter-mixed, so split them up into their own files. This is cleaner and is needed to support HTTP/3 in the future. Signed-off-by: Asad Sajjad Ahmed Merged b4a094787b25b0d3571dc50829610cafe6b33129 back in Signed-off-by: Nils Goroll diff --git a/bin/varnishd/acceptor/acceptor_priv.h b/bin/varnishd/acceptor/acceptor_priv.h new file mode 100644 index 000000000..5876c00a3 --- /dev/null +++ b/bin/varnishd/acceptor/acceptor_priv.h @@ -0,0 +1,67 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2015 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +struct sockaddr_storage; +struct pool_task; + +enum vca_event { + VCA_EVENT_LADDR, +}; + +struct wrk_accept { + unsigned magic; +#define WRK_ACCEPT_MAGIC 0x8c4b4d59 + + /* Accept stuff */ + struct sockaddr_storage acceptaddr; + socklen_t acceptaddrlen; + int acceptsock; + struct listen_sock *acceptlsock; +}; + +struct poolsock { + unsigned magic; +#define POOLSOCK_MAGIC 0x1b0a2d38 + VTAILQ_ENTRY(poolsock) list; + struct listen_sock *lsock; + struct pool_task task[1]; + struct pool *pool; +}; + +struct conn_heritage { + unsigned sess_set; + unsigned listen_mod; +}; + +void vca_pace_check(void); +void vca_pace_bad(void); +void vca_pace_good(void); diff --git a/bin/varnishd/acceptor/acceptor_tcp.h b/bin/varnishd/acceptor/acceptor_tcp.h index 350d5f72a..402f549bd 100644 --- a/bin/varnishd/acceptor/acceptor_tcp.h +++ b/bin/varnishd/acceptor/acceptor_tcp.h @@ -37,3 +37,10 @@ struct listen_arg; int vca_tcp_config(void); int vca_tcp_open(char **, struct listen_arg *, const char **); int vca_tcp_reopen(void); + +void vca_tcp_init(void); +void vca_tcp_start(struct cli *cli); +void vca_tcp_event(struct cli *cli, enum vca_event event); +void vca_tcp_accept(struct pool *pp); +void vca_tcp_update(pthread_mutex_t *shut_mtx); +void vca_tcp_shutdown(void); diff --git a/bin/varnishd/acceptor/acceptor_uds.h b/bin/varnishd/acceptor/acceptor_uds.h index 6a04546c3..9abcc0f46 100644 --- a/bin/varnishd/acceptor/acceptor_uds.h +++ b/bin/varnishd/acceptor/acceptor_uds.h @@ -44,3 +44,10 @@ struct uds_perms { int vca_uds_config(void); int vca_uds_open(char **, struct listen_arg *, const char **); int vca_uds_reopen(void); + +void vca_uds_init(void); +void vca_uds_start(struct cli *cli); +void vca_uds_event(struct cli *cli, enum vca_event event); +void vca_uds_accept(struct pool *pp); +void vca_uds_update(pthread_mutex_t *shut_mtx); +void vca_uds_shutdown(void); diff --git a/bin/varnishd/acceptor/cache_acceptor.c b/bin/varnishd/acceptor/cache_acceptor.c index 8989bc53b..88c4c22e6 100644 --- a/bin/varnishd/acceptor/cache_acceptor.c +++ b/bin/varnishd/acceptor/cache_acceptor.c @@ -41,6 +41,9 @@ #include "cache/cache_varnishd.h" #include "acceptor/cache_acceptor.h" +#include "acceptor/acceptor_priv.h" +#include "acceptor/acceptor_tcp.h" +#include "acceptor/acceptor_uds.h" #include "cache/cache_transport.h" #include "cache/cache_pool.h" @@ -52,100 +55,11 @@ #include "vtim.h" static pthread_t VCA_thread; -static vtim_dur vca_pace = 0.0; -static struct lock pace_mtx; -static unsigned pool_accepting; +vtim_dur vca_pace = 0.0; +struct lock pace_mtx; +unsigned pool_accepting; static pthread_mutex_t shut_mtx = PTHREAD_MUTEX_INITIALIZER; -struct wrk_accept { - unsigned magic; -#define WRK_ACCEPT_MAGIC 0x8c4b4d59 - - /* Accept stuff */ - struct sockaddr_storage acceptaddr; - socklen_t acceptaddrlen; - int acceptsock; - struct listen_sock *acceptlsock; -}; - -struct poolsock { - unsigned magic; -#define POOLSOCK_MAGIC 0x1b0a2d38 - VTAILQ_ENTRY(poolsock) list; - struct listen_sock *lsock; - struct pool_task task[1]; - struct pool *pool; -}; - -/*-------------------------------------------------------------------- - * TCP options we want to control - */ - -union sock_arg { - struct linger lg; - struct timeval tv; - int i; -}; - -static struct sock_opt { - int level; - int optname; - const char *strname; - unsigned mod; - socklen_t sz; - union sock_arg arg[1]; -} sock_opts[] = { - /* Note: Setting the mod counter to something not-zero is needed - * to force the setsockopt() calls on startup */ -#define SOCK_OPT(lvl, nam, typ) { lvl, nam, #nam, 1, sizeof(typ) }, - - SOCK_OPT(SOL_SOCKET, SO_LINGER, struct linger) - SOCK_OPT(SOL_SOCKET, SO_KEEPALIVE, int) - SOCK_OPT(SOL_SOCKET, SO_SNDTIMEO, struct timeval) - SOCK_OPT(SOL_SOCKET, SO_RCVTIMEO, struct timeval) - - SOCK_OPT(IPPROTO_TCP, TCP_NODELAY, int) - -#if defined(HAVE_TCP_KEEP) - SOCK_OPT(IPPROTO_TCP, TCP_KEEPIDLE, int) - SOCK_OPT(IPPROTO_TCP, TCP_KEEPCNT, int) - SOCK_OPT(IPPROTO_TCP, TCP_KEEPINTVL, int) -#elif defined(HAVE_TCP_KEEPALIVE) - SOCK_OPT(IPPROTO_TCP, TCP_KEEPALIVE, int) -#endif - -#undef SOCK_OPT -}; - -static const int n_sock_opts = sizeof sock_opts / sizeof sock_opts[0]; - -struct conn_heritage { - unsigned sess_set; - unsigned listen_mod; -}; - -/*-------------------------------------------------------------------- - * We want to get out of any kind of trouble-hit TCP connections as fast - * as absolutely possible, so we set them LINGER disabled, so that even if - * there are outstanding write data on the socket, a close(2) will return - * immediately. - */ -static const struct linger disable_so_linger = { - .l_onoff = 0, -}; - -/* - * We turn on keepalives by default to assist in detecting clients that have - * hung up on connections returning from waitinglists - */ - -static const unsigned enable_so_keepalive = 1; - -/* We disable Nagle's algorithm in favor of low latency setups. - */ - -static const unsigned enable_tcp_nodelay = 1; - /*-------------------------------------------------------------------- * lacking a better place, we put some generic periodic updates * into the vca_acct() loop which we are running anyway @@ -161,166 +75,12 @@ vca_periodic(vtim_real t0) VTIM_postel = FEATURE(FEATURE_HTTP_DATE_POSTEL); } -/*-------------------------------------------------------------------- - * Some kernels have bugs/limitations with respect to which options are - * inherited from the accept/listen socket, so we have to keep track of - * which, if any, sockopts we have to set on the accepted socket. - */ - -static int -vca_sock_opt_init(void) -{ - struct sock_opt *so; - union sock_arg tmp; - int n, chg = 0; - size_t sz; - - memset(&tmp, 0, sizeof tmp); - - for (n = 0; n < n_sock_opts; n++) { - so = &sock_opts[n]; - -#define SET_VAL(nm, so, fld, val) \ - do { \ - if (!strcmp(#nm, so->strname)) { \ - assert(so->sz == sizeof so->arg->fld); \ - so->arg->fld = (val); \ - } \ - } while (0) - -#define NEW_VAL(nm, so, fld, val) \ - do { \ - if (!strcmp(#nm, so->strname)) { \ - sz = sizeof tmp.fld; \ - assert(so->sz == sz); \ - tmp.fld = (val); \ - if (memcmp(&so->arg->fld, &(tmp.fld), sz)) { \ - memcpy(&so->arg->fld, &(tmp.fld), sz); \ - so->mod++; \ - chg = 1; \ - } \ - } \ - } while (0) - - SET_VAL(SO_LINGER, so, lg, disable_so_linger); - SET_VAL(SO_KEEPALIVE, so, i, enable_so_keepalive); - NEW_VAL(SO_SNDTIMEO, so, tv, - VTIM_timeval_sock(cache_param->idle_send_timeout)); - NEW_VAL(SO_RCVTIMEO, so, tv, - VTIM_timeval_sock(cache_param->timeout_idle)); - SET_VAL(TCP_NODELAY, so, i, enable_tcp_nodelay); -#if defined(HAVE_TCP_KEEP) - NEW_VAL(TCP_KEEPIDLE, so, i, - (int)cache_param->tcp_keepalive_time); - NEW_VAL(TCP_KEEPCNT, so, i, - (int)cache_param->tcp_keepalive_probes); - NEW_VAL(TCP_KEEPINTVL, so, i, - (int)cache_param->tcp_keepalive_intvl); -#elif defined(HAVE_TCP_KEEPALIVE) - NEW_VAL(TCP_KEEPALIVE, so, i, - (int)cache_param->tcp_keepalive_time); -#endif - } - return (chg); -} - -static void -vca_sock_opt_test(const struct listen_sock *ls, const struct sess *sp) -{ - struct conn_heritage *ch; - struct sock_opt *so; - union sock_arg tmp; - socklen_t l; - int i, n; - - CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - - for (n = 0; n < n_sock_opts; n++) { - so = &sock_opts[n]; - ch = &ls->conn_heritage[n]; - if (ch->sess_set) { - VSL(SLT_Debug, sp->vxid, - "sockopt: Not testing nonhereditary %s for %s=%s", - so->strname, ls->name, ls->endpoint); - continue; - } - if (so->level == IPPROTO_TCP && ls->uds) { - VSL(SLT_Debug, sp->vxid, - "sockopt: Not testing incompatible %s for %s=%s", - so->strname, ls->name, ls->endpoint); - continue; - } - memset(&tmp, 0, sizeof tmp); - l = so->sz; - i = getsockopt(sp->fd, so->level, so->optname, &tmp, &l); - if (i == 0 && memcmp(&tmp, so->arg, so->sz)) { - VSL(SLT_Debug, sp->vxid, - "sockopt: Test confirmed %s non heredity for %s=%s", - so->strname, ls->name, ls->endpoint); - ch->sess_set = 1; - } - if (i && errno != ENOPROTOOPT) - VTCP_Assert(i); - } -} - -static void -vca_sock_opt_set(const struct listen_sock *ls, const struct sess *sp) -{ - struct conn_heritage *ch; - struct sock_opt *so; - vxid_t vxid; - int n, sock; - - CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); - - if (sp != NULL) { - CHECK_OBJ(sp, SESS_MAGIC); - sock = sp->fd; - vxid = sp->vxid; - } else { - sock = ls->sock; - vxid = NO_VXID; - } - - for (n = 0; n < n_sock_opts; n++) { - so = &sock_opts[n]; - ch = &ls->conn_heritage[n]; - if (so->level == IPPROTO_TCP && ls->uds) { - VSL(SLT_Debug, vxid, - "sockopt: Not setting incompatible %s for %s=%s", - so->strname, ls->name, ls->endpoint); - continue; - } - if (sp == NULL && ch->listen_mod == so->mod) { - VSL(SLT_Debug, vxid, - "sockopt: Not setting unmodified %s for %s=%s", - so->strname, ls->name, ls->endpoint); - continue; - } - if (sp != NULL && !ch->sess_set) { - VSL(SLT_Debug, sp->vxid, - "sockopt: %s may be inherited for %s=%s", - so->strname, ls->name, ls->endpoint); - continue; - } - VSL(SLT_Debug, vxid, - "sockopt: Setting %s for %s=%s", - so->strname, ls->name, ls->endpoint); - VTCP_Assert(setsockopt(sock, - so->level, so->optname, so->arg, so->sz)); - if (sp == NULL) - ch->listen_mod = so->mod; - } -} - /*-------------------------------------------------------------------- * If accept(2)'ing fails, we pace ourselves to relive any resource * shortage if possible. */ -static void +void vca_pace_check(void) { vtim_dur p; @@ -334,7 +94,7 @@ vca_pace_check(void) VTIM_sleep(p); } -static void +void vca_pace_bad(void) { @@ -345,7 +105,7 @@ vca_pace_bad(void) Lck_Unlock(&pace_mtx); } -static void +void vca_pace_good(void) { @@ -358,242 +118,6 @@ vca_pace_good(void) Lck_Unlock(&pace_mtx); } -/*-------------------------------------------------------------------- - * The pool-task for a newly accepted session - * - * Called from assigned worker thread - */ - -static void -vca_mk_tcp(const struct wrk_accept *wa, - struct sess *sp, char *laddr, char *lport, char *raddr, char *rport) -{ - struct suckaddr *sa = NULL; - ssize_t sz; - - AN(SES_Reserve_remote_addr(sp, &sa, &sz)); - AN(sa); - assert(sz == vsa_suckaddr_len); - AN(VSA_Build(sa, &wa->acceptaddr, wa->acceptaddrlen)); - sp->sattr[SA_CLIENT_ADDR] = sp->sattr[SA_REMOTE_ADDR]; - - VTCP_name(sa, raddr, VTCP_ADDRBUFSIZE, rport, VTCP_PORTBUFSIZE); - AN(SES_Set_String_Attr(sp, SA_CLIENT_IP, raddr)); - AN(SES_Set_String_Attr(sp, SA_CLIENT_PORT, rport)); - - - AN(SES_Reserve_local_addr(sp, &sa, &sz)); - AN(VSA_getsockname(sp->fd, sa, sz)); - sp->sattr[SA_SERVER_ADDR] = sp->sattr[SA_LOCAL_ADDR]; - VTCP_name(sa, laddr, VTCP_ADDRBUFSIZE, lport, VTCP_PORTBUFSIZE); -} - -static void -vca_mk_uds(struct wrk_accept *wa, struct sess *sp, char *laddr, char *lport, - char *raddr, char *rport) -{ - struct suckaddr *sa = NULL; - ssize_t sz; - - (void) wa; - AN(SES_Reserve_remote_addr(sp, &sa, &sz)); - AN(sa); - assert(sz == vsa_suckaddr_len); - AZ(SES_Set_remote_addr(sp, bogo_ip)); - sp->sattr[SA_CLIENT_ADDR] = sp->sattr[SA_REMOTE_ADDR]; - sp->sattr[SA_LOCAL_ADDR] = sp->sattr[SA_REMOTE_ADDR]; - sp->sattr[SA_SERVER_ADDR] = sp->sattr[SA_REMOTE_ADDR]; - AN(SES_Set_String_Attr(sp, SA_CLIENT_IP, "0.0.0.0")); - AN(SES_Set_String_Attr(sp, SA_CLIENT_PORT, "0")); - - strcpy(laddr, "0.0.0.0"); - strcpy(raddr, "0.0.0.0"); - strcpy(lport, "0"); - strcpy(rport, "0"); -} - -static void v_matchproto_(task_func_t) -vca_make_session(struct worker *wrk, void *arg) -{ - struct sess *sp; - struct req *req; - struct wrk_accept *wa; - char laddr[VTCP_ADDRBUFSIZE]; - char lport[VTCP_PORTBUFSIZE]; - char raddr[VTCP_ADDRBUFSIZE]; - char rport[VTCP_PORTBUFSIZE]; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CAST_OBJ_NOTNULL(wa, arg, WRK_ACCEPT_MAGIC); - - VTCP_blocking(wa->acceptsock); - - /* Turn accepted socket into a session */ - AN(WS_Reservation(wrk->aws)); - sp = SES_New(wrk->pool); - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - wrk->stats->s_sess++; - - sp->t_open = VTIM_real(); - sp->t_idle = sp->t_open; - sp->vxid = VXID_Get(wrk, VSL_CLIENTMARKER); - - sp->fd = wa->acceptsock; - wa->acceptsock = -1; - sp->listen_sock = wa->acceptlsock; - - assert((size_t)wa->acceptaddrlen <= vsa_suckaddr_len); - - if (wa->acceptlsock->uds) - vca_mk_uds(wa, sp, laddr, lport, raddr, rport); - else - vca_mk_tcp(wa, sp, laddr, lport, raddr, rport); - - AN(wa->acceptlsock->name); - VSL(SLT_Begin, sp->vxid, "sess 0 %s", - wa->acceptlsock->transport->name); - VSL(SLT_SessOpen, sp->vxid, "%s %s %s %s %s %.6f %d", - raddr, rport, wa->acceptlsock->name, laddr, lport, - sp->t_open, sp->fd); - - vca_pace_good(); - wrk->stats->sess_conn++; - - if (wa->acceptlsock->test_heritage) { - vca_sock_opt_test(wa->acceptlsock, sp); - wa->acceptlsock->test_heritage = 0; - } - vca_sock_opt_set(wa->acceptlsock, sp); - - req = Req_New(sp); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - req->htc->rfd = &sp->fd; - - SES_SetTransport(wrk, sp, req, wa->acceptlsock->transport); - WS_Release(wrk->aws, 0); -} - -/*-------------------------------------------------------------------- - * This function accepts on a single socket for a single thread pool. - * - * As long as we can stick the accepted connection to another thread - * we do so, otherwise we put the socket back on the "BACK" pool - * and handle the new connection ourselves. - */ - -static void v_matchproto_(task_func_t) -vca_accept_task(struct worker *wrk, void *arg) -{ - struct wrk_accept wa; - struct poolsock *ps; - struct listen_sock *ls; - int i; - char laddr[VTCP_ADDRBUFSIZE]; - char lport[VTCP_PORTBUFSIZE]; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CAST_OBJ_NOTNULL(ps, arg, POOLSOCK_MAGIC); - ls = ps->lsock; - CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); - - while (!pool_accepting) - VTIM_sleep(.1); - - /* Dont hold on to (possibly) discarded VCLs */ - if (wrk->wpriv->vcl != NULL) - VCL_Rel(&wrk->wpriv->vcl); - - while (!ps->pool->die) { - INIT_OBJ(&wa, WRK_ACCEPT_MAGIC); - wa.acceptlsock = ls; - - vca_pace_check(); - - wa.acceptaddrlen = sizeof wa.acceptaddr; - do { - i = accept(ls->sock, (void*)&wa.acceptaddr, - &wa.acceptaddrlen); - } while (i < 0 && errno == EAGAIN && !ps->pool->die); - - if (i < 0 && ps->pool->die) - break; - - if (i < 0 && ls->sock == -2) { - /* Shut down in progress */ - sleep(2); - continue; - } - - if (i < 0) { - switch (errno) { - case ECONNABORTED: - wrk->stats->sess_fail_econnaborted++; - break; - case EINTR: - wrk->stats->sess_fail_eintr++; - break; - case EMFILE: - wrk->stats->sess_fail_emfile++; - vca_pace_bad(); - break; - case EBADF: - wrk->stats->sess_fail_ebadf++; - vca_pace_bad(); - break; - case ENOBUFS: - case ENOMEM: - wrk->stats->sess_fail_enomem++; - vca_pace_bad(); - break; - default: - wrk->stats->sess_fail_other++; - vca_pace_bad(); - break; - } - - i = errno; - wrk->stats->sess_fail++; - - if (wa.acceptlsock->uds) { - bstrcpy(laddr, "0.0.0.0"); - bstrcpy(lport, "0"); - } else { - VTCP_myname(ls->sock, laddr, VTCP_ADDRBUFSIZE, - lport, VTCP_PORTBUFSIZE); - } - - VSL(SLT_SessError, NO_VXID, "%s %s %s %d %d \"%s\"", - wa.acceptlsock->name, laddr, lport, - ls->sock, i, VAS_errtxt(i)); - (void)Pool_TrySumstat(wrk); - continue; - } - - wa.acceptsock = i; - - if (!Pool_Task_Arg(wrk, TASK_QUEUE_REQ, - vca_make_session, &wa, sizeof wa)) { - /* - * We couldn't get another thread, so we will handle - * the request in this worker thread, but first we - * must reschedule the listening task so it will be - * taken up by another thread again. - */ - if (!ps->pool->die) { - AZ(Pool_Task(wrk->pool, ps->task, - TASK_QUEUE_VCA)); - return; - } - } - if (!ps->pool->die && DO_DEBUG(DBG_SLOW_ACCEPTOR)) - VTIM_sleep(2.0); - - } - - VSL(SLT_Debug, NO_VXID, "XXX Accept thread dies %p", ps); - FREE_OBJ(ps); -} - /*-------------------------------------------------------------------- * Called when a worker and attached thread pool is created, to * allocate the tasks which will listen to sockets for that pool. @@ -602,19 +126,9 @@ vca_accept_task(struct worker *wrk, void *arg) void VCA_NewPool(struct pool *pp) { - struct listen_sock *ls; - struct poolsock *ps; - VTAILQ_FOREACH(ls, &heritage.socks, list) { - ALLOC_OBJ(ps, POOLSOCK_MAGIC); - AN(ps); - ps->lsock = ls; - ps->task->func = vca_accept_task; - ps->task->priv = ps; - ps->pool = pp; - VTAILQ_INSERT_TAIL(&pp->poolsocks, ps, list); - AZ(Pool_Task(pp, ps->task, TASK_QUEUE_VCA)); - } + vca_tcp_accept(pp); + vca_uds_accept(pp); } void @@ -633,7 +147,6 @@ VCA_DestroyPool(struct pool *pp) static void * v_matchproto_() vca_acct(void *arg) { - struct listen_sock *ls; vtim_real t0; // XXX Actually a misnomer now because the accept happens in a pool @@ -649,29 +162,11 @@ vca_acct(void *arg) while (1) { (void)sleep(1); - if (vca_sock_opt_init()) { - PTOK(pthread_mutex_lock(&shut_mtx)); - VTAILQ_FOREACH(ls, &heritage.socks, list) { - if (ls->sock == -2) - continue; // VCA_Shutdown - assert (ls->sock > 0); - vca_sock_opt_set(ls, NULL); - /* If one of the options on a socket has - * changed, also force a retest of whether - * the values are inherited to the - * accepted sockets. This should then - * catch any false positives from previous - * tests that could happen if the set - * value of an option happened to just be - * the OS default for that value, and - * wasn't actually inherited from the - * listening socket. */ - ls->test_heritage = 1; - } - PTOK(pthread_mutex_unlock(&shut_mtx)); - } + vca_tcp_update(&shut_mtx); + vca_uds_update(&shut_mtx); vca_periodic(t0); } + NEEDLESS(return (NULL)); } @@ -680,35 +175,11 @@ vca_acct(void *arg) void VCA_Start(struct cli *cli) { - struct listen_sock *ls; - (void)vca_sock_opt_init(); - - VTAILQ_FOREACH(ls, &heritage.socks, list) { - CHECK_OBJ_NOTNULL(ls->transport, TRANSPORT_MAGIC); - assert (ls->sock > 0); // We know where stdin is - if (cache_param->tcp_fastopen && - VTCP_fastopen(ls->sock, cache_param->listen_depth)) - VSL(SLT_Error, NO_VXID, - "Kernel TCP Fast Open: sock=%d, errno=%d %s", - ls->sock, errno, VAS_errtxt(errno)); - if (listen(ls->sock, cache_param->listen_depth)) { - VCLI_SetResult(cli, CLIS_CANT); - VCLI_Out(cli, "Listen failed on socket '%s': %s", - ls->endpoint, VAS_errtxt(errno)); - return; - } - AZ(ls->conn_heritage); - ls->conn_heritage = calloc(n_sock_opts, - sizeof *ls->conn_heritage); - AN(ls->conn_heritage); - ls->test_heritage = 1; - vca_sock_opt_set(ls, NULL); - if (cache_param->accept_filter && VTCP_filter_http(ls->sock)) - VSL(SLT_Error, NO_VXID, - "Kernel filtering: sock=%d, errno=%d %s", - ls->sock, errno, VAS_errtxt(errno)); - } + ASSERT_CLI(); + + vca_tcp_start(cli); + vca_uds_start(cli); PTOK(pthread_create(&VCA_thread, NULL, vca_acct, NULL)); } @@ -721,7 +192,6 @@ ccf_listen_address(struct cli *cli, const char * const *av, void *priv) struct listen_sock *ls; char h[VTCP_ADDRBUFSIZE], p[VTCP_PORTBUFSIZE]; - (void)cli; (void)av; (void)priv; @@ -735,14 +205,12 @@ ccf_listen_address(struct cli *cli, const char * const *av, void *priv) VTIM_sleep(.1); PTOK(pthread_mutex_lock(&shut_mtx)); - VTAILQ_FOREACH(ls, &heritage.socks, list) { if (!ls->uds) { VTCP_myname(ls->sock, h, sizeof h, p, sizeof p); VCLI_Out(cli, "%s %s %s\n", ls->name, h, p); } else VCLI_Out(cli, "%s %s -\n", ls->name, ls->endpoint); - } PTOK(pthread_mutex_unlock(&shut_mtx)); } @@ -759,20 +227,17 @@ VCA_Init(void) CLI_AddFuncs(vca_cmds); Lck_New(&pace_mtx, lck_vcapace); + vca_tcp_init(); + vca_uds_init(); } void VCA_Shutdown(void) { - struct listen_sock *ls; - int i; PTOK(pthread_mutex_lock(&shut_mtx)); - VTAILQ_FOREACH(ls, &heritage.socks, list) { - i = ls->sock; - ls->sock = -2; - (void)close(i); - } + vca_tcp_shutdown(); + vca_uds_shutdown(); PTOK(pthread_mutex_unlock(&shut_mtx)); } diff --git a/bin/varnishd/acceptor/cache_acceptor_tcp.c b/bin/varnishd/acceptor/cache_acceptor_tcp.c new file mode 100644 index 000000000..ad6d91dc4 --- /dev/null +++ b/bin/varnishd/acceptor/cache_acceptor_tcp.c @@ -0,0 +1,629 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2015 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include "config.h" + +#include +#include +#include + +#include "cache/cache_varnishd.h" + +#include "acceptor/cache_acceptor.h" +#include "acceptor/acceptor_priv.h" +#include "acceptor/acceptor_tcp.h" + +#include "cache/cache_transport.h" +#include "cache/cache_pool.h" +#include "common/heritage.h" + +#include "vcli_serve.h" +#include "vsa.h" +#include "vtcp.h" +#include "vtim.h" + +extern vtim_dur vca_pace; +extern struct lock pace_mtx; +extern unsigned pool_accepting; + +/*-------------------------------------------------------------------- + * TCP options we want to control + */ + +union sock_arg { + struct linger lg; + struct timeval tv; + int i; +}; + +static struct sock_opt { + int level; + int optname; + const char *strname; + unsigned mod; + socklen_t sz; + union sock_arg arg[1]; +} sock_opts[] = { + /* Note: Setting the mod counter to something not-zero is needed + * to force the setsockopt() calls on startup */ +#define SOCK_OPT(lvl, nam, typ) { lvl, nam, #nam, 1, sizeof(typ) }, + + SOCK_OPT(SOL_SOCKET, SO_LINGER, struct linger) + SOCK_OPT(SOL_SOCKET, SO_KEEPALIVE, int) + SOCK_OPT(SOL_SOCKET, SO_SNDTIMEO, struct timeval) + SOCK_OPT(SOL_SOCKET, SO_RCVTIMEO, struct timeval) + + SOCK_OPT(IPPROTO_TCP, TCP_NODELAY, int) + +#if defined(HAVE_TCP_KEEP) + SOCK_OPT(IPPROTO_TCP, TCP_KEEPIDLE, int) + SOCK_OPT(IPPROTO_TCP, TCP_KEEPCNT, int) + SOCK_OPT(IPPROTO_TCP, TCP_KEEPINTVL, int) +#elif defined(HAVE_TCP_KEEPALIVE) + SOCK_OPT(IPPROTO_TCP, TCP_KEEPALIVE, int) +#endif + +#undef SOCK_OPT +}; + +static const int n_sock_opts = sizeof sock_opts / sizeof sock_opts[0]; + +/*-------------------------------------------------------------------- + * We want to get out of any kind of trouble-hit TCP connections as fast + * as absolutely possible, so we set them LINGER disabled, so that even if + * there are outstanding write data on the socket, a close(2) will return + * immediately. + */ +static const struct linger disable_so_linger = { + .l_onoff = 0, +}; + +/* + * We turn on keepalives by default to assist in detecting clients that have + * hung up on connections returning from waitinglists + */ +static const unsigned enable_so_keepalive = 1; + +/* We disable Nagle's algorithm in favor of low latency setups. + */ +static const unsigned enable_tcp_nodelay = 1; + +/*-------------------------------------------------------------------- + * Some kernels have bugs/limitations with respect to which options are + * inherited from the accept/listen socket, so we have to keep track of + * which, if any, sockopts we have to set on the accepted socket. + */ + +static int +vca_tcp_sockopt_init(void) +{ + struct sock_opt *so; + union sock_arg tmp; + int n, chg = 0; + size_t sz; + + memset(&tmp, 0, sizeof tmp); + + for (n = 0; n < n_sock_opts; n++) { + so = &sock_opts[n]; + +#define SET_VAL(nm, so, fld, val) \ + do { \ + if (!strcmp(#nm, so->strname)) { \ + assert(so->sz == sizeof so->arg->fld); \ + so->arg->fld = (val); \ + } \ + } while (0) + +#define NEW_VAL(nm, so, fld, val) \ + do { \ + if (!strcmp(#nm, so->strname)) { \ + sz = sizeof tmp.fld; \ + assert(so->sz == sz); \ + tmp.fld = (val); \ + if (memcmp(&so->arg->fld, &(tmp.fld), sz)) { \ + memcpy(&so->arg->fld, &(tmp.fld), sz); \ + so->mod++; \ + chg = 1; \ + } \ + } \ + } while (0) + + SET_VAL(SO_LINGER, so, lg, disable_so_linger); + SET_VAL(SO_KEEPALIVE, so, i, enable_so_keepalive); + NEW_VAL(SO_SNDTIMEO, so, tv, + VTIM_timeval_sock(cache_param->idle_send_timeout)); + NEW_VAL(SO_RCVTIMEO, so, tv, + VTIM_timeval_sock(cache_param->timeout_idle)); + SET_VAL(TCP_NODELAY, so, i, enable_tcp_nodelay); +#if defined(HAVE_TCP_KEEP) + NEW_VAL(TCP_KEEPIDLE, so, i, + (int)cache_param->tcp_keepalive_time); + NEW_VAL(TCP_KEEPCNT, so, i, + (int)cache_param->tcp_keepalive_probes); + NEW_VAL(TCP_KEEPINTVL, so, i, + (int)cache_param->tcp_keepalive_intvl); +#elif defined(HAVE_TCP_KEEPALIVE) + NEW_VAL(TCP_KEEPALIVE, so, i, + (int)cache_param->tcp_keepalive_time); +#endif + } + + return (chg); +} + +static void +vca_tcp_sockopt_test(const struct listen_sock *ls, const struct sess *sp) +{ + struct conn_heritage *ch; + struct sock_opt *so; + union sock_arg tmp; + socklen_t l; + int i, n; + + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + + for (n = 0; n < n_sock_opts; n++) { + so = &sock_opts[n]; + ch = &ls->conn_heritage[n]; + + if (ch->sess_set) { + VSL(SLT_Debug, sp->vxid, + "sockopt: Not testing nonhereditary %s for %s=%s", + so->strname, ls->name, ls->endpoint); + continue; + } + + memset(&tmp, 0, sizeof tmp); + l = so->sz; + i = getsockopt(sp->fd, so->level, so->optname, &tmp, &l); + + if (i == 0 && memcmp(&tmp, so->arg, so->sz)) { + VSL(SLT_Debug, sp->vxid, + "sockopt: Test confirmed %s non heredity for %s=%s", + so->strname, ls->name, ls->endpoint); + ch->sess_set = 1; + } + + if (i && errno != ENOPROTOOPT) + VTCP_Assert(i); + } +} + +static void +vca_tcp_sockopt_set(const struct listen_sock *ls, const struct sess *sp) +{ + struct conn_heritage *ch; + struct sock_opt *so; + vxid_t vxid; + int n, sock; + + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + + if (sp != NULL) { + CHECK_OBJ(sp, SESS_MAGIC); + sock = sp->fd; + vxid = sp->vxid; + } else { + sock = ls->sock; + vxid = NO_VXID; + } + + for (n = 0; n < n_sock_opts; n++) { + so = &sock_opts[n]; + ch = &ls->conn_heritage[n]; + + if (sp == NULL && ch->listen_mod == so->mod) { + VSL(SLT_Debug, vxid, + "sockopt: Not setting unmodified %s for %s=%s", + so->strname, ls->name, ls->endpoint); + continue; + } + + if (sp != NULL && !ch->sess_set) { + VSL(SLT_Debug, sp->vxid, + "sockopt: %s may be inherited for %s=%s", + so->strname, ls->name, ls->endpoint); + continue; + } + + VSL(SLT_Debug, vxid, + "sockopt: Setting %s for %s=%s", + so->strname, ls->name, ls->endpoint); + VTCP_Assert(setsockopt(sock, + so->level, so->optname, so->arg, so->sz)); + + if (sp == NULL) + ch->listen_mod = so->mod; + } +} + +void +vca_tcp_init(void) +{ + +} + +static int +vca_tcp_listen(struct cli *cli, struct listen_sock *ls) +{ + + CHECK_OBJ_NOTNULL(ls->transport, TRANSPORT_MAGIC); + assert (ls->sock > 0); // We know where stdin is + + if (cache_param->tcp_fastopen && + VTCP_fastopen(ls->sock, cache_param->listen_depth)) + VSL(SLT_Error, NO_VXID, + "Kernel TCP Fast Open: sock=%d, errno=%d %s", + ls->sock, errno, VAS_errtxt(errno)); + + if (listen(ls->sock, cache_param->listen_depth)) { + VCLI_SetResult(cli, CLIS_CANT); + VCLI_Out(cli, "Listen failed on socket '%s': %s", + ls->endpoint, VAS_errtxt(errno)); + return (-1); + } + + AZ(ls->conn_heritage); + ls->conn_heritage = calloc(n_sock_opts, + sizeof *ls->conn_heritage); + AN(ls->conn_heritage); + + ls->test_heritage = 1; + vca_tcp_sockopt_set(ls, NULL); + + if (cache_param->accept_filter && VTCP_filter_http(ls->sock)) + VSL(SLT_Error, NO_VXID, + "Kernel filtering: sock=%d, errno=%d %s", + ls->sock, errno, VAS_errtxt(errno)); + + return (0); +} + +void +vca_tcp_start(struct cli *cli) +{ + struct listen_sock *ls; + + ASSERT_CLI(); + + (void)vca_tcp_sockopt_init(); + + VTAILQ_FOREACH(ls, &heritage.socks, list) { + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + + if (ls->uds) + continue; + + if (vca_tcp_listen(cli, ls)) + return; + } +} + +void +vca_tcp_event(struct cli *cli, struct listen_sock *ls, enum vca_event event) +{ + char h[VTCP_ADDRBUFSIZE], p[VTCP_PORTBUFSIZE]; + + if (ls->uds) + return; + + switch (event) { + case VCA_EVENT_LADDR: + VTCP_myname(ls->sock, h, sizeof h, p, sizeof p); + VCLI_Out(cli, "%s %s %s\n", ls->name, h, p); + break; + default: + WRONG("INVALID VCA_EVENT"); + } +} + +/*-------------------------------------------------------------------- + * The pool-task for a newly accepted session + * + * Called from assigned worker thread + */ + +static void +vca_mk_tcp(const struct wrk_accept *wa, + struct sess *sp, char *laddr, char *lport, char *raddr, char *rport) +{ + struct suckaddr *sa = NULL; + ssize_t sz; + + AN(SES_Reserve_remote_addr(sp, &sa, &sz)); + AN(sa); + assert(sz == vsa_suckaddr_len); + AN(VSA_Build(sa, &wa->acceptaddr, wa->acceptaddrlen)); + sp->sattr[SA_CLIENT_ADDR] = sp->sattr[SA_REMOTE_ADDR]; + + VTCP_name(sa, raddr, VTCP_ADDRBUFSIZE, rport, VTCP_PORTBUFSIZE); + AN(SES_Set_String_Attr(sp, SA_CLIENT_IP, raddr)); + AN(SES_Set_String_Attr(sp, SA_CLIENT_PORT, rport)); + + + AN(SES_Reserve_local_addr(sp, &sa, &sz)); + AN(VSA_getsockname(sp->fd, sa, sz)); + sp->sattr[SA_SERVER_ADDR] = sp->sattr[SA_LOCAL_ADDR]; + VTCP_name(sa, laddr, VTCP_ADDRBUFSIZE, lport, VTCP_PORTBUFSIZE); +} + +static void v_matchproto_(task_func_t) +vca_tcp_make_session(struct worker *wrk, void *arg) +{ + char laddr[VTCP_ADDRBUFSIZE]; + char lport[VTCP_PORTBUFSIZE]; + char raddr[VTCP_ADDRBUFSIZE]; + char rport[VTCP_PORTBUFSIZE]; + struct wrk_accept *wa; + struct sess *sp; + struct req *req; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CAST_OBJ_NOTNULL(wa, arg, WRK_ACCEPT_MAGIC); + + VTCP_blocking(wa->acceptsock); + + /* Turn accepted socket into a session */ + AN(WS_Reservation(wrk->aws)); + sp = SES_New(wrk->pool); + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + wrk->stats->s_sess++; + + sp->t_open = VTIM_real(); + sp->t_idle = sp->t_open; + sp->vxid = VXID_Get(wrk, VSL_CLIENTMARKER); + + sp->fd = wa->acceptsock; + wa->acceptsock = -1; + sp->listen_sock = wa->acceptlsock; + + assert((size_t)wa->acceptaddrlen <= vsa_suckaddr_len); + + vca_mk_tcp(wa, sp, laddr, lport, raddr, rport); + + AN(wa->acceptlsock->name); + VSL(SLT_Begin, sp->vxid, "sess 0 %s", + wa->acceptlsock->transport->name); + VSL(SLT_SessOpen, sp->vxid, "%s %s %s %s %s %.6f %d", + raddr, rport, wa->acceptlsock->name, laddr, lport, + sp->t_open, sp->fd); + + vca_pace_good(); + wrk->stats->sess_conn++; + + if (wa->acceptlsock->test_heritage) { + vca_tcp_sockopt_test(wa->acceptlsock, sp); + wa->acceptlsock->test_heritage = 0; + } + + vca_tcp_sockopt_set(wa->acceptlsock, sp); + + req = Req_New(sp); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + req->htc->rfd = &sp->fd; + + SES_SetTransport(wrk, sp, req, wa->acceptlsock->transport); + WS_Release(wrk->aws, 0); +} + +/*-------------------------------------------------------------------- + * This function accepts on a single socket for a single thread pool. + * + * As long as we can stick the accepted connection to another thread + * we do so, otherwise we put the socket back on the "BACK" pool + * and handle the new connection ourselves. + */ + +static void v_matchproto_(task_func_t) +vca_tcp_accept_task(struct worker *wrk, void *arg) +{ + char laddr[VTCP_ADDRBUFSIZE]; + char lport[VTCP_PORTBUFSIZE]; + struct listen_sock *ls; + struct wrk_accept wa; + struct poolsock *ps; + int i; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CAST_OBJ_NOTNULL(ps, arg, POOLSOCK_MAGIC); + ls = ps->lsock; + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + CHECK_OBJ_NOTNULL(ps->pool, POOL_MAGIC); + + while (!pool_accepting) + VTIM_sleep(.1); + + /* Dont hold on to (possibly) discarded VCLs */ + if (wrk->wpriv->vcl != NULL) + VCL_Rel(&wrk->wpriv->vcl); + + while (!ps->pool->die) { + INIT_OBJ(&wa, WRK_ACCEPT_MAGIC); + wa.acceptlsock = ls; + + vca_pace_check(); + + wa.acceptaddrlen = sizeof wa.acceptaddr; + do { + i = accept(ls->sock, (void*)&wa.acceptaddr, + &wa.acceptaddrlen); + } while (i < 0 && errno == EAGAIN && !ps->pool->die); + + if (i < 0 && ps->pool->die) + break; + + if (i < 0 && ls->sock == -2) { + /* Shut down in progress */ + sleep(2); + continue; + } + + if (i < 0) { + switch (errno) { + case ECONNABORTED: + wrk->stats->sess_fail_econnaborted++; + break; + case EINTR: + wrk->stats->sess_fail_eintr++; + break; + case EMFILE: + wrk->stats->sess_fail_emfile++; + vca_pace_bad(); + break; + case EBADF: + wrk->stats->sess_fail_ebadf++; + vca_pace_bad(); + break; + case ENOBUFS: + case ENOMEM: + wrk->stats->sess_fail_enomem++; + vca_pace_bad(); + break; + default: + wrk->stats->sess_fail_other++; + vca_pace_bad(); + break; + } + + i = errno; + wrk->stats->sess_fail++; + + VTCP_myname(ls->sock, laddr, VTCP_ADDRBUFSIZE, + lport, VTCP_PORTBUFSIZE); + + VSL(SLT_SessError, NO_VXID, "%s %s %s %d %d \"%s\"", + wa.acceptlsock->name, laddr, lport, + ls->sock, i, VAS_errtxt(i)); + (void)Pool_TrySumstat(wrk); + continue; + } + + wa.acceptsock = i; + + if (!Pool_Task_Arg(wrk, TASK_QUEUE_REQ, + vca_tcp_make_session, &wa, sizeof wa)) { + /* + * We couldn't get another thread, so we will handle + * the request in this worker thread, but first we + * must reschedule the listening task so it will be + * taken up by another thread again. + */ + if (!ps->pool->die) { + AZ(Pool_Task(wrk->pool, ps->task, + TASK_QUEUE_VCA)); + return; + } + } + if (!ps->pool->die && DO_DEBUG(DBG_SLOW_ACCEPTOR)) + VTIM_sleep(2.0); + + } + + VSL(SLT_Debug, NO_VXID, "XXX Accept thread dies %p", ps); + FREE_OBJ(ps); +} + +void +vca_tcp_accept(struct pool *pp) +{ + struct listen_sock *ls; + struct poolsock *ps; + + VTAILQ_FOREACH(ls, &heritage.socks, list) { + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + + if (ls->uds) + continue; + + ALLOC_OBJ(ps, POOLSOCK_MAGIC); + AN(ps); + ps->lsock = ls; + ps->task->func = vca_tcp_accept_task; + ps->task->priv = ps; + ps->pool = pp; + VTAILQ_INSERT_TAIL(&pp->poolsocks, ps, list); + AZ(Pool_Task(pp, ps->task, TASK_QUEUE_VCA)); + } +} + +void +vca_tcp_update(pthread_mutex_t *shut_mtx) +{ + struct listen_sock *ls; + + if (!vca_tcp_sockopt_init()) + return; + + PTOK(pthread_mutex_lock(shut_mtx)); + + VTAILQ_FOREACH(ls, &heritage.socks, list) { + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + + if (ls->sock == -2 || ls->uds) + continue; // VCA_Shutdown || UDS + assert (ls->sock > 0); + vca_tcp_sockopt_set(ls, NULL); + /* If one of the options on a socket has + * changed, also force a retest of whether + * the values are inherited to the + * accepted sockets. This should then + * catch any false positives from previous + * tests that could happen if the set + * value of an option happened to just be + * the OS default for that value, and + * wasn't actually inherited from the + * listening socket. */ + ls->test_heritage = 1; + } + + PTOK(pthread_mutex_unlock(shut_mtx)); +} + +void +vca_tcp_shutdown(void) +{ + struct listen_sock *ls; + int i; + + VTAILQ_FOREACH(ls, &heritage.socks, list) { + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + + if (ls->uds) + continue; + + i = ls->sock; + ls->sock = -2; + (void)close(i); + } +} diff --git a/bin/varnishd/acceptor/cache_acceptor_uds.c b/bin/varnishd/acceptor/cache_acceptor_uds.c new file mode 100644 index 000000000..bea713abe --- /dev/null +++ b/bin/varnishd/acceptor/cache_acceptor_uds.c @@ -0,0 +1,583 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2015 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * SPDX-License-Identifier: BSD-2-Clause + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include "config.h" + +#include +#include +#include + +#include "cache/cache_varnishd.h" + +#include "acceptor/cache_acceptor.h" +#include "acceptor/acceptor_priv.h" +#include "acceptor/acceptor_uds.h" + +#include "cache/cache_transport.h" +#include "cache/cache_pool.h" +#include "common/heritage.h" + +#include "vcli_serve.h" +#include "vsa.h" +#include "vtcp.h" +#include "vtim.h" + +extern vtim_dur vca_pace; +extern struct lock pace_mtx; +extern unsigned pool_accepting; + +/*-------------------------------------------------------------------- + * We want to get out of any kind of trouble-hit TCP connections as fast + * as absolutely possible, so we set them LINGER disabled, so that even if + * there are outstanding write data on the socket, a close(2) will return + * immediately. + */ +static const struct linger disable_so_linger = { + .l_onoff = 0, +}; + +/* + * We turn on keepalives by default to assist in detecting clients that have + * hung up on connections returning from waitinglists + */ +static const unsigned enable_so_keepalive = 1; + +/*-------------------------------------------------------------------- + * UDS options we want to control + */ + +union sock_arg { + struct linger lg; + struct timeval tv; + int i; +}; + +static struct sock_opt { + int level; + int optname; + const char *strname; + unsigned mod; + socklen_t sz; + union sock_arg arg[1]; +} sock_opts[] = { + /* Note: Setting the mod counter to something not-zero is needed + * to force the setsockopt() calls on startup */ +#define SOCK_OPT(lvl, nam, typ) { lvl, nam, #nam, 1, sizeof(typ) }, + + SOCK_OPT(SOL_SOCKET, SO_LINGER, struct linger) + SOCK_OPT(SOL_SOCKET, SO_KEEPALIVE, int) + SOCK_OPT(SOL_SOCKET, SO_SNDTIMEO, struct timeval) + SOCK_OPT(SOL_SOCKET, SO_RCVTIMEO, struct timeval) + +#undef SOCK_OPT +}; + +static const int n_sock_opts = sizeof sock_opts / sizeof sock_opts[0]; + +/*-------------------------------------------------------------------- + * Some kernels have bugs/limitations with respect to which options are + * inherited from the accept/listen socket, so we have to keep track of + * which, if any, sockopts we have to set on the accepted socket. + */ + +static int +vca_uds_sockopt_init(void) +{ + struct sock_opt *so; + union sock_arg tmp; + int n, chg = 0; + size_t sz; + + memset(&tmp, 0, sizeof tmp); + + for (n = 0; n < n_sock_opts; n++) { + so = &sock_opts[n]; + +#define SET_VAL(nm, so, fld, val) \ + do { \ + if (!strcmp(#nm, so->strname)) { \ + assert(so->sz == sizeof so->arg->fld); \ + so->arg->fld = (val); \ + } \ + } while (0) + +#define NEW_VAL(nm, so, fld, val) \ + do { \ + if (!strcmp(#nm, so->strname)) { \ + sz = sizeof tmp.fld; \ + assert(so->sz == sz); \ + tmp.fld = (val); \ + if (memcmp(&so->arg->fld, &(tmp.fld), sz)) { \ + memcpy(&so->arg->fld, &(tmp.fld), sz); \ + so->mod++; \ + chg = 1; \ + } \ + } \ + } while (0) + + SET_VAL(SO_LINGER, so, lg, disable_so_linger); + SET_VAL(SO_KEEPALIVE, so, i, enable_so_keepalive); + NEW_VAL(SO_SNDTIMEO, so, tv, + VTIM_timeval_sock(cache_param->idle_send_timeout)); + NEW_VAL(SO_RCVTIMEO, so, tv, + VTIM_timeval_sock(cache_param->timeout_idle)); + } + + return (chg); +} + +static void +vca_uds_sockopt_test(const struct listen_sock *ls, const struct sess *sp) +{ + struct conn_heritage *ch; + struct sock_opt *so; + union sock_arg tmp; + socklen_t l; + int i, n; + + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + + for (n = 0; n < n_sock_opts; n++) { + so = &sock_opts[n]; + ch = &ls->conn_heritage[n]; + + if (ch->sess_set) { + VSL(SLT_Debug, sp->vxid, + "sockopt: Not testing nonhereditary %s for %s=%s", + so->strname, ls->name, ls->endpoint); + continue; + } + + memset(&tmp, 0, sizeof tmp); + l = so->sz; + i = getsockopt(sp->fd, so->level, so->optname, &tmp, &l); + + if (i == 0 && memcmp(&tmp, so->arg, so->sz)) { + VSL(SLT_Debug, sp->vxid, + "sockopt: Test confirmed %s non heredity for %s=%s", + so->strname, ls->name, ls->endpoint); + ch->sess_set = 1; + } + + if (i && errno != ENOPROTOOPT) + VTCP_Assert(i); + } +} + +static void +vca_uds_sockopt_set(const struct listen_sock *ls, const struct sess *sp) +{ + struct conn_heritage *ch; + struct sock_opt *so; + vxid_t vxid; + int n, sock; + + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + + if (sp != NULL) { + CHECK_OBJ(sp, SESS_MAGIC); + sock = sp->fd; + vxid = sp->vxid; + } else { + sock = ls->sock; + vxid = NO_VXID; + } + + for (n = 0; n < n_sock_opts; n++) { + so = &sock_opts[n]; + ch = &ls->conn_heritage[n]; + + if (sp == NULL && ch->listen_mod == so->mod) { + VSL(SLT_Debug, vxid, + "sockopt: Not setting unmodified %s for %s=%s", + so->strname, ls->name, ls->endpoint); + continue; + } + + if (sp != NULL && !ch->sess_set) { + VSL(SLT_Debug, sp->vxid, + "sockopt: %s may be inherited for %s=%s", + so->strname, ls->name, ls->endpoint); + continue; + } + + VSL(SLT_Debug, vxid, + "sockopt: Setting %s for %s=%s", + so->strname, ls->name, ls->endpoint); + VTCP_Assert(setsockopt(sock, + so->level, so->optname, so->arg, so->sz)); + + if (sp == NULL) + ch->listen_mod = so->mod; + } +} + +void +vca_uds_init(void) +{ + +} + +static int +vca_uds_listen(struct cli *cli, struct listen_sock *ls) +{ + + CHECK_OBJ_NOTNULL(ls->transport, TRANSPORT_MAGIC); + assert (ls->sock > 0); // We know where stdin is + + if (listen(ls->sock, cache_param->listen_depth)) { + VCLI_SetResult(cli, CLIS_CANT); + VCLI_Out(cli, "Listen failed on socket '%s': %s", + ls->endpoint, VAS_errtxt(errno)); + return (-1); + } + + AZ(ls->conn_heritage); + ls->conn_heritage = calloc(n_sock_opts, + sizeof *ls->conn_heritage); + AN(ls->conn_heritage); + + ls->test_heritage = 1; + vca_uds_sockopt_set(ls, NULL); + + if (cache_param->accept_filter && VTCP_filter_http(ls->sock)) + VSL(SLT_Error, NO_VXID, + "Kernel filtering: sock=%d, errno=%d %s", + ls->sock, errno, VAS_errtxt(errno)); + + return (0); +} + +void +vca_uds_start(struct cli *cli) +{ + struct listen_sock *ls; + + ASSERT_CLI(); + + (void)vca_uds_sockopt_init(); + + VTAILQ_FOREACH(ls, &heritage.socks, list) { + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + + if (!ls->uds) + continue; + + if (vca_uds_listen(cli, ls)) + return; + } +} + +void +vca_uds_event(struct cli *cli, struct listen_sock *ls, enum vca_event event) +{ + + if (!ls->uds) + return; + + switch (event) { + case VCA_EVENT_LADDR: + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + VCLI_Out(cli, "%s %s -\n", ls->name, ls->endpoint); + break; + default: + WRONG("INVALID VCA_EVENT"); + } +} + +static void +vca_mk_uds(struct wrk_accept *wa, struct sess *sp, char *laddr, char *lport, + char *raddr, char *rport) +{ + struct suckaddr *sa = NULL; + ssize_t sz; + + (void) wa; + AN(SES_Reserve_remote_addr(sp, &sa, &sz)); + AN(sa); + assert(sz == vsa_suckaddr_len); + AZ(SES_Set_remote_addr(sp, bogo_ip)); + sp->sattr[SA_CLIENT_ADDR] = sp->sattr[SA_REMOTE_ADDR]; + sp->sattr[SA_LOCAL_ADDR] = sp->sattr[SA_REMOTE_ADDR]; + sp->sattr[SA_SERVER_ADDR] = sp->sattr[SA_REMOTE_ADDR]; + AN(SES_Set_String_Attr(sp, SA_CLIENT_IP, "0.0.0.0")); + AN(SES_Set_String_Attr(sp, SA_CLIENT_PORT, "0")); + + strcpy(laddr, "0.0.0.0"); + strcpy(raddr, "0.0.0.0"); + strcpy(lport, "0"); + strcpy(rport, "0"); +} + +static void v_matchproto_(task_func_t) +vca_make_session(struct worker *wrk, void *arg) +{ + char laddr[VTCP_ADDRBUFSIZE]; + char lport[VTCP_PORTBUFSIZE]; + char raddr[VTCP_ADDRBUFSIZE]; + char rport[VTCP_PORTBUFSIZE]; + struct wrk_accept *wa; + struct sess *sp; + struct req *req; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CAST_OBJ_NOTNULL(wa, arg, WRK_ACCEPT_MAGIC); + + VTCP_blocking(wa->acceptsock); + + /* Turn accepted socket into a session */ + AN(WS_Reservation(wrk->aws)); + sp = SES_New(wrk->pool); + CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); + wrk->stats->s_sess++; + + sp->t_open = VTIM_real(); + sp->t_idle = sp->t_open; + sp->vxid = VXID_Get(wrk, VSL_CLIENTMARKER); + + sp->fd = wa->acceptsock; + wa->acceptsock = -1; + sp->listen_sock = wa->acceptlsock; + + assert((size_t)wa->acceptaddrlen <= vsa_suckaddr_len); + + vca_mk_uds(wa, sp, laddr, lport, raddr, rport); + + AN(wa->acceptlsock->name); + VSL(SLT_Begin, sp->vxid, "sess 0 %s", + wa->acceptlsock->transport->name); + VSL(SLT_SessOpen, sp->vxid, "%s %s %s %s %s %.6f %d", + raddr, rport, wa->acceptlsock->name, laddr, lport, + sp->t_open, sp->fd); + + vca_pace_good(); + wrk->stats->sess_conn++; + + if (wa->acceptlsock->test_heritage) { + vca_uds_sockopt_test(wa->acceptlsock, sp); + wa->acceptlsock->test_heritage = 0; + } + + vca_uds_sockopt_set(wa->acceptlsock, sp); + + req = Req_New(sp); + CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + req->htc->rfd = &sp->fd; + + SES_SetTransport(wrk, sp, req, wa->acceptlsock->transport); + WS_Release(wrk->aws, 0); +} + +/*-------------------------------------------------------------------- + * This function accepts on a single socket for a single thread pool. + * + * As long as we can stick the accepted connection to another thread + * we do so, otherwise we put the socket back on the "BACK" pool + * and handle the new connection ourselves. + */ + +static void v_matchproto_(task_func_t) +vca_uds_accept_task(struct worker *wrk, void *arg) +{ + struct listen_sock *ls; + struct wrk_accept wa; + struct poolsock *ps; + int i; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CAST_OBJ_NOTNULL(ps, arg, POOLSOCK_MAGIC); + ls = ps->lsock; + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + + while (!pool_accepting) + VTIM_sleep(.1); + + /* Dont hold on to (possibly) discarded VCLs */ + if (wrk->wpriv->vcl != NULL) + VCL_Rel(&wrk->wpriv->vcl); + + while (!ps->pool->die) { + INIT_OBJ(&wa, WRK_ACCEPT_MAGIC); + wa.acceptlsock = ls; + + vca_pace_check(); + + wa.acceptaddrlen = sizeof wa.acceptaddr; + do { + i = accept(ls->sock, (void*)&wa.acceptaddr, + &wa.acceptaddrlen); + } while (i < 0 && errno == EAGAIN && !ps->pool->die); + + if (i < 0 && ps->pool->die) + break; + + if (i < 0 && ls->sock == -2) { + /* Shut down in progress */ + sleep(2); + continue; + } + + if (i < 0) { + switch (errno) { + case ECONNABORTED: + wrk->stats->sess_fail_econnaborted++; + break; + case EINTR: + wrk->stats->sess_fail_eintr++; + break; + case EMFILE: + wrk->stats->sess_fail_emfile++; + vca_pace_bad(); + break; + case EBADF: + wrk->stats->sess_fail_ebadf++; + vca_pace_bad(); + break; + case ENOBUFS: + case ENOMEM: + wrk->stats->sess_fail_enomem++; + vca_pace_bad(); + break; + default: + wrk->stats->sess_fail_other++; + vca_pace_bad(); + break; + } + + i = errno; + wrk->stats->sess_fail++; + + VSL(SLT_SessError, NO_VXID, "%s 0.0.0.0 0 %d %d \"%s\"", + wa.acceptlsock->name, ls->sock, i, VAS_errtxt(i)); + (void)Pool_TrySumstat(wrk); + continue; + } + + wa.acceptsock = i; + + if (!Pool_Task_Arg(wrk, TASK_QUEUE_REQ, + vca_make_session, &wa, sizeof wa)) { + /* + * We couldn't get another thread, so we will handle + * the request in this worker thread, but first we + * must reschedule the listening task so it will be + * taken up by another thread again. + */ + if (!ps->pool->die) { + AZ(Pool_Task(wrk->pool, ps->task, + TASK_QUEUE_VCA)); + return; + } + } + if (!ps->pool->die && DO_DEBUG(DBG_SLOW_ACCEPTOR)) + VTIM_sleep(2.0); + + } + + VSL(SLT_Debug, NO_VXID, "XXX Accept thread dies %p", ps); + FREE_OBJ(ps); +} + +void +vca_uds_accept(struct pool *pp) +{ + struct listen_sock *ls; + struct poolsock *ps; + + VTAILQ_FOREACH(ls, &heritage.socks, list) { + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + + if (!ls->uds) + continue; + + ALLOC_OBJ(ps, POOLSOCK_MAGIC); + AN(ps); + ps->lsock = ls; + ps->task->func = vca_uds_accept_task; + ps->task->priv = ps; + ps->pool = pp; + VTAILQ_INSERT_TAIL(&pp->poolsocks, ps, list); + AZ(Pool_Task(pp, ps->task, TASK_QUEUE_VCA)); + } +} + +void +vca_uds_update(pthread_mutex_t *shut_mtx) +{ + struct listen_sock *ls; + + if (!vca_uds_sockopt_init()) + return; + + PTOK(pthread_mutex_lock(shut_mtx)); + + VTAILQ_FOREACH(ls, &heritage.socks, list) { + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + + if (ls->sock == -2 || !ls->uds) + continue; // VCA_Shutdown || TCP + assert (ls->sock > 0); + vca_uds_sockopt_set(ls, NULL); + /* If one of the options on a socket has + * changed, also force a retest of whether + * the values are inherited to the + * accepted sockets. This should then + * catch any false positives from previous + * tests that could happen if the set + * value of an option happened to just be + * the OS default for that value, and + * wasn't actually inherited from the + * listening socket. */ + ls->test_heritage = 1; + } + + PTOK(pthread_mutex_unlock(shut_mtx)); +} + +void +vca_uds_shutdown(void) +{ + struct listen_sock *ls; + int i; + + VTAILQ_FOREACH(ls, &heritage.socks, list) { + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + + if (!ls->uds) + continue; + + i = ls->sock; + ls->sock = -2; + (void)close(i); + } +} From nils.goroll at uplex.de Mon Sep 30 15:53:07 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 15:53:07 +0000 (UTC) Subject: [master] 72aef9995 acceptor: plugable acceptor Message-ID: <20240930155307.17CED11D0F2@lists.varnish-cache.org> commit 72aef9995479ec3a568739c2724f1781664b6f50 Author: Asad Sajjad Ahmed Date: Wed Sep 6 10:22:09 2023 +0200 acceptor: plugable acceptor Make the acceptor code plugable and start off with two acceptor modules TCP and UDS. Adding a new QUIC acceptor module in the future should be easy now. Signed-off-by: Asad Sajjad Ahmed diff --git a/bin/varnishd/acceptor/acceptor_priv.h b/bin/varnishd/acceptor/acceptor_priv.h index 5876c00a3..727e07d1b 100644 --- a/bin/varnishd/acceptor/acceptor_priv.h +++ b/bin/varnishd/acceptor/acceptor_priv.h @@ -33,10 +33,6 @@ struct sockaddr_storage; struct pool_task; -enum vca_event { - VCA_EVENT_LADDR, -}; - struct wrk_accept { unsigned magic; #define WRK_ACCEPT_MAGIC 0x8c4b4d59 diff --git a/bin/varnishd/acceptor/acceptor_tcp.h b/bin/varnishd/acceptor/acceptor_tcp.h index 402f549bd..350d5f72a 100644 --- a/bin/varnishd/acceptor/acceptor_tcp.h +++ b/bin/varnishd/acceptor/acceptor_tcp.h @@ -37,10 +37,3 @@ struct listen_arg; int vca_tcp_config(void); int vca_tcp_open(char **, struct listen_arg *, const char **); int vca_tcp_reopen(void); - -void vca_tcp_init(void); -void vca_tcp_start(struct cli *cli); -void vca_tcp_event(struct cli *cli, enum vca_event event); -void vca_tcp_accept(struct pool *pp); -void vca_tcp_update(pthread_mutex_t *shut_mtx); -void vca_tcp_shutdown(void); diff --git a/bin/varnishd/acceptor/acceptor_uds.h b/bin/varnishd/acceptor/acceptor_uds.h index 9abcc0f46..6a04546c3 100644 --- a/bin/varnishd/acceptor/acceptor_uds.h +++ b/bin/varnishd/acceptor/acceptor_uds.h @@ -44,10 +44,3 @@ struct uds_perms { int vca_uds_config(void); int vca_uds_open(char **, struct listen_arg *, const char **); int vca_uds_reopen(void); - -void vca_uds_init(void); -void vca_uds_start(struct cli *cli); -void vca_uds_event(struct cli *cli, enum vca_event event); -void vca_uds_accept(struct pool *pp); -void vca_uds_update(pthread_mutex_t *shut_mtx); -void vca_uds_shutdown(void); diff --git a/bin/varnishd/acceptor/cache_acceptor.c b/bin/varnishd/acceptor/cache_acceptor.c index 88c4c22e6..375edb074 100644 --- a/bin/varnishd/acceptor/cache_acceptor.c +++ b/bin/varnishd/acceptor/cache_acceptor.c @@ -42,8 +42,6 @@ #include "cache/cache_varnishd.h" #include "acceptor/cache_acceptor.h" #include "acceptor/acceptor_priv.h" -#include "acceptor/acceptor_tcp.h" -#include "acceptor/acceptor_uds.h" #include "cache/cache_transport.h" #include "cache/cache_pool.h" @@ -126,9 +124,12 @@ vca_pace_good(void) void VCA_NewPool(struct pool *pp) { + struct acceptor *vca; - vca_tcp_accept(pp); - vca_uds_accept(pp); + VCA_Foreach(vca) { + CHECK_OBJ_NOTNULL(vca, ACCEPTOR_MAGIC); + vca->accept(pp); + } } void @@ -147,6 +148,7 @@ VCA_DestroyPool(struct pool *pp) static void * v_matchproto_() vca_acct(void *arg) { + struct acceptor *vca; vtim_real t0; // XXX Actually a misnomer now because the accept happens in a pool @@ -162,8 +164,12 @@ vca_acct(void *arg) while (1) { (void)sleep(1); - vca_tcp_update(&shut_mtx); - vca_uds_update(&shut_mtx); + + VCA_Foreach(vca) { + CHECK_OBJ_NOTNULL(vca, ACCEPTOR_MAGIC); + vca->update(&shut_mtx); + } + vca_periodic(t0); } @@ -175,11 +181,14 @@ vca_acct(void *arg) void VCA_Start(struct cli *cli) { + struct acceptor *vca; ASSERT_CLI(); - vca_tcp_start(cli); - vca_uds_start(cli); + VCA_Foreach(vca) { + CHECK_OBJ_NOTNULL(vca, ACCEPTOR_MAGIC); + vca->start(cli); + } PTOK(pthread_create(&VCA_thread, NULL, vca_acct, NULL)); } @@ -190,7 +199,6 @@ static void v_matchproto_(cli_func_t) ccf_listen_address(struct cli *cli, const char * const *av, void *priv) { struct listen_sock *ls; - char h[VTCP_ADDRBUFSIZE], p[VTCP_PORTBUFSIZE]; (void)av; (void)priv; @@ -205,12 +213,17 @@ ccf_listen_address(struct cli *cli, const char * const *av, void *priv) VTIM_sleep(.1); PTOK(pthread_mutex_lock(&shut_mtx)); - if (!ls->uds) { - VTCP_myname(ls->sock, h, sizeof h, p, sizeof p); - VCLI_Out(cli, "%s %s %s\n", ls->name, h, p); - } - else - VCLI_Out(cli, "%s %s -\n", ls->name, ls->endpoint); + + /* + * Varnishtest expects the list of listen sockets to come out in the + * same order as it is specified on the command line. + */ + VTAILQ_FOREACH(ls, &heritage.socks, list) { + CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); + CHECK_OBJ_NOTNULL(ls->vca, ACCEPTOR_MAGIC); + ls->vca->event(cli, ls, VCA_EVENT_LADDR); + } + PTOK(pthread_mutex_unlock(&shut_mtx)); } @@ -224,20 +237,29 @@ static struct cli_proto vca_cmds[] = { void VCA_Init(void) { + struct acceptor *vca; CLI_AddFuncs(vca_cmds); Lck_New(&pace_mtx, lck_vcapace); - vca_tcp_init(); - vca_uds_init(); + + VCA_Foreach(vca) { + CHECK_OBJ_NOTNULL(vca, ACCEPTOR_MAGIC); + vca->init(); + } } void VCA_Shutdown(void) { + struct acceptor *vca; PTOK(pthread_mutex_lock(&shut_mtx)); - vca_tcp_shutdown(); - vca_uds_shutdown(); + + VCA_Foreach(vca) { + CHECK_OBJ_NOTNULL(vca, ACCEPTOR_MAGIC); + vca->shutdown(); + } + PTOK(pthread_mutex_unlock(&shut_mtx)); } diff --git a/bin/varnishd/acceptor/cache_acceptor.h b/bin/varnishd/acceptor/cache_acceptor.h index 5e9fbe282..287c2e8ec 100644 --- a/bin/varnishd/acceptor/cache_acceptor.h +++ b/bin/varnishd/acceptor/cache_acceptor.h @@ -31,7 +31,49 @@ */ /* cache_acceptor.c */ +struct listen_sock; +struct listen_arg; +struct pool; void VCA_Init(void); void VCA_Start(struct cli *cli); void VCA_Shutdown(void); + +enum vca_event { + VCA_EVENT_LADDR, +}; + +typedef int acceptor_config_f(void); +typedef void acceptor_init_f(void); +typedef int acceptor_open_f(char **, struct listen_arg *, + const char **); +typedef int acceptor_reopen_f(void); +typedef void acceptor_start_f(struct cli *); +typedef void acceptor_event_f(struct cli *, struct listen_sock *, + enum vca_event); +typedef void acceptor_accept_f(struct pool *); +typedef void acceptor_update_f(pthread_mutex_t *); +typedef void acceptor_shutdown_f(void); + +struct acceptor { + unsigned magic; +#define ACCEPTOR_MAGIC 0x0611847c + VTAILQ_ENTRY(acceptor) list; + const char *name; + + acceptor_config_f *config; + acceptor_init_f *init; + acceptor_open_f *open; + acceptor_reopen_f *reopen; + acceptor_start_f *start; + acceptor_event_f *event; + acceptor_accept_f *accept; + acceptor_update_f *update; + acceptor_shutdown_f *shutdown; +}; + +#define VCA_Foreach(arg) for (arg = NULL; VCA__iter(&arg);) +int VCA__iter(struct acceptor ** const pp); + +extern struct acceptor TCP_acceptor; +extern struct acceptor UDS_acceptor; diff --git a/bin/varnishd/acceptor/cache_acceptor_tcp.c b/bin/varnishd/acceptor/cache_acceptor_tcp.c index ad6d91dc4..ad4737275 100644 --- a/bin/varnishd/acceptor/cache_acceptor_tcp.c +++ b/bin/varnishd/acceptor/cache_acceptor_tcp.c @@ -268,7 +268,7 @@ vca_tcp_sockopt_set(const struct listen_sock *ls, const struct sess *sp) } } -void +static void vca_tcp_init(void) { @@ -310,7 +310,7 @@ vca_tcp_listen(struct cli *cli, struct listen_sock *ls) return (0); } -void +static void vca_tcp_start(struct cli *cli) { struct listen_sock *ls; @@ -330,14 +330,11 @@ vca_tcp_start(struct cli *cli) } } -void +static void vca_tcp_event(struct cli *cli, struct listen_sock *ls, enum vca_event event) { char h[VTCP_ADDRBUFSIZE], p[VTCP_PORTBUFSIZE]; - if (ls->uds) - return; - switch (event) { case VCA_EVENT_LADDR: VTCP_myname(ls->sock, h, sizeof h, p, sizeof p); @@ -554,7 +551,7 @@ vca_tcp_accept_task(struct worker *wrk, void *arg) FREE_OBJ(ps); } -void +static void vca_tcp_accept(struct pool *pp) { struct listen_sock *ls; @@ -577,7 +574,7 @@ vca_tcp_accept(struct pool *pp) } } -void +static void vca_tcp_update(pthread_mutex_t *shut_mtx) { struct listen_sock *ls; @@ -610,7 +607,7 @@ vca_tcp_update(pthread_mutex_t *shut_mtx) PTOK(pthread_mutex_unlock(shut_mtx)); } -void +static void vca_tcp_shutdown(void) { struct listen_sock *ls; @@ -627,3 +624,17 @@ vca_tcp_shutdown(void) (void)close(i); } } + +struct acceptor TCP_acceptor = { + .magic = ACCEPTOR_MAGIC, + .name = "tcp", + .config = vca_tcp_config, + .init = vca_tcp_init, + .open = vca_tcp_open, + .reopen = vca_tcp_reopen, + .start = vca_tcp_start, + .event = vca_tcp_event, + .accept = vca_tcp_accept, + .update = vca_tcp_update, + .shutdown = vca_tcp_shutdown, +}; diff --git a/bin/varnishd/acceptor/cache_acceptor_uds.c b/bin/varnishd/acceptor/cache_acceptor_uds.c index bea713abe..c18406474 100644 --- a/bin/varnishd/acceptor/cache_acceptor_uds.c +++ b/bin/varnishd/acceptor/cache_acceptor_uds.c @@ -242,7 +242,7 @@ vca_uds_sockopt_set(const struct listen_sock *ls, const struct sess *sp) } } -void +static void vca_uds_init(void) { @@ -278,7 +278,7 @@ vca_uds_listen(struct cli *cli, struct listen_sock *ls) return (0); } -void +static void vca_uds_start(struct cli *cli) { struct listen_sock *ls; @@ -298,13 +298,10 @@ vca_uds_start(struct cli *cli) } } -void +static void vca_uds_event(struct cli *cli, struct listen_sock *ls, enum vca_event event) { - if (!ls->uds) - return; - switch (event) { case VCA_EVENT_LADDR: CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); @@ -508,7 +505,7 @@ vca_uds_accept_task(struct worker *wrk, void *arg) FREE_OBJ(ps); } -void +static void vca_uds_accept(struct pool *pp) { struct listen_sock *ls; @@ -531,7 +528,7 @@ vca_uds_accept(struct pool *pp) } } -void +static void vca_uds_update(pthread_mutex_t *shut_mtx) { struct listen_sock *ls; @@ -564,7 +561,7 @@ vca_uds_update(pthread_mutex_t *shut_mtx) PTOK(pthread_mutex_unlock(shut_mtx)); } -void +static void vca_uds_shutdown(void) { struct listen_sock *ls; @@ -581,3 +578,17 @@ vca_uds_shutdown(void) (void)close(i); } } + +struct acceptor UDS_acceptor = { + .magic = ACCEPTOR_MAGIC, + .name = "uds", + .config = vca_uds_config, + .init = vca_uds_init, + .open = vca_uds_open, + .reopen = vca_uds_reopen, + .start = vca_uds_start, + .event = vca_uds_event, + .accept = vca_uds_accept, + .update = vca_uds_update, + .shutdown = vca_uds_shutdown, +}; diff --git a/bin/varnishd/acceptor/mgt_acceptor.c b/bin/varnishd/acceptor/mgt_acceptor.c index 1e25a9078..d172b4418 100644 --- a/bin/varnishd/acceptor/mgt_acceptor.c +++ b/bin/varnishd/acceptor/mgt_acceptor.c @@ -57,6 +57,36 @@ static VTAILQ_HEAD(,listen_arg) listen_args = VTAILQ_HEAD_INITIALIZER(listen_args); +static VTAILQ_HEAD(,acceptor) acceptors = VTAILQ_HEAD_INITIALIZER(acceptors); + +int +VCA__iter(struct acceptor ** const pvca) +{ + + AN(pvca); + CHECK_OBJ_ORNULL(*pvca, ACCEPTOR_MAGIC); + if (*pvca != NULL) + *pvca = VTAILQ_NEXT(*pvca, list); + else + *pvca = VTAILQ_FIRST(&acceptors); + return (*pvca != NULL); +} + +static struct acceptor * +VCA_Find(const char *name) +{ + struct acceptor *vca; + + VCA_Foreach(vca) { + CHECK_OBJ_NOTNULL(vca, ACCEPTOR_MAGIC); + + if (!strcmp(vca->name, name)) + return (vca); + } + + return (NULL); +} + /*===================================================================== * Reopen the accept sockets to get rid of listen status. * returns the highest errno encountered, 0 for success @@ -65,12 +95,19 @@ static VTAILQ_HEAD(,listen_arg) listen_args = int VCA_reopen_sockets(void) { - int fail, fail2; + struct acceptor *vca; + int fail; + int err; - fail = vca_tcp_open(); - fail2 = vca_uds_open(); + fail = 0; - return (vmax(fail, fail2)); + VCA_Foreach(vca) { + CHECK_OBJ_NOTNULL(vca, ACCEPTOR_MAGIC); + err = vca->reopen(); + fail = vmax(fail, err); + } + + return (fail); } /*--------------------------------------------------------------------*/ @@ -78,6 +115,7 @@ VCA_reopen_sockets(void) void VCA_Arg(const char *spec) { + struct acceptor *vca; char **av; struct listen_arg *la; const char *err; @@ -104,9 +142,12 @@ VCA_Arg(const char *spec) la->name = name; if (VUS_is(la->endpoint)) - error = vca_uds_open(av[1], la, &err); + vca = VCA_Find("uds"); else - error = vca_tcp_open(av[1], la, &err); + vca = VCA_Find("tcp"); + + AN(vca); + error = vca->open(av, la, &err); if (error) ARGV_ERR("Got no socket(s) for %s (%s)\n", av[1], err); @@ -114,3 +155,36 @@ VCA_Arg(const char *spec) ARGV_ERR("Got no socket(s) for %s\n", av[1]); VAV_Free(av); } + +void +VCA_Add(struct acceptor *vca) +{ + + CHECK_OBJ_NOTNULL(vca, ACCEPTOR_MAGIC); + AN(vca->name); + AN(vca->config); + AN(vca->init); + AN(vca->open); + AN(vca->reopen); + AN(vca->start); + AN(vca->event); + AN(vca->accept); + AN(vca->update); + AN(vca->shutdown); + + if (VCA_Find(vca->name) != NULL) + ARGV_ERR("Acceptor '%s' already exist\n", vca->name); + + VTAILQ_INSERT_TAIL(&acceptors, vca, list); + + if (vca->config()) + ARGV_ERR("Acceptor '%s' failed to initialize\n", vca->name); +} + +void +VCA_Config(void) +{ + + VCA_Add(&TCP_acceptor); + VCA_Add(&UDS_acceptor); +} diff --git a/bin/varnishd/acceptor/mgt_acceptor.h b/bin/varnishd/acceptor/mgt_acceptor.h index 5558383c5..ed9d4354d 100644 --- a/bin/varnishd/acceptor/mgt_acceptor.h +++ b/bin/varnishd/acceptor/mgt_acceptor.h @@ -43,5 +43,8 @@ struct listen_arg { const struct uds_perms *perms; }; +void VCA_Add(struct acceptor *); +void VCA_Config(void); + void VCA_Arg(const char *); int VCA_reopen_sockets(void); diff --git a/bin/varnishd/acceptor/mgt_acceptor_tcp.c b/bin/varnishd/acceptor/mgt_acceptor_tcp.c index 279ea7479..43ba9c2f2 100644 --- a/bin/varnishd/acceptor/mgt_acceptor_tcp.c +++ b/bin/varnishd/acceptor/mgt_acceptor_tcp.c @@ -111,6 +111,7 @@ vca_tcp_open_cb(void *priv, const struct suckaddr *sa) AN(ls); ls->sock = -1; + ls->vca = &TCP_acceptor; ls->addr = VSA_Clone(sa); AN(ls->addr); diff --git a/bin/varnishd/acceptor/mgt_acceptor_uds.c b/bin/varnishd/acceptor/mgt_acceptor_uds.c index 93641e39d..e1b0d3242 100644 --- a/bin/varnishd/acceptor/mgt_acceptor_uds.c +++ b/bin/varnishd/acceptor/mgt_acceptor_uds.c @@ -129,6 +129,7 @@ vca_uds_open_cb(void *priv, const struct sockaddr_un *uds) AN(ls); ls->sock = -1; + ls->vca = &UDS_acceptor; ls->addr = VSA_Clone(bogo_ip); AN(ls->addr); diff --git a/bin/varnishd/common/heritage.h b/bin/varnishd/common/heritage.h index 3112c1214..499c91423 100644 --- a/bin/varnishd/common/heritage.h +++ b/bin/varnishd/common/heritage.h @@ -44,6 +44,7 @@ struct listen_sock { #define LISTEN_SOCK_MAGIC 0x999e4b57 VTAILQ_ENTRY(listen_sock) list; VTAILQ_ENTRY(listen_sock) arglist; + VTAILQ_ENTRY(listen_sock) vcalist; int sock; int uds; char *endpoint; @@ -53,6 +54,7 @@ struct listen_sock { const struct uds_perms *perms; unsigned test_heritage; struct conn_heritage *conn_heritage; + struct acceptor *vca; }; VTAILQ_HEAD(listen_sock_head, listen_sock); From nils.goroll at uplex.de Mon Sep 30 15:53:07 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 15:53:07 +0000 (UTC) Subject: [master] eb985308d acceptor: create local acceptor list Message-ID: <20240930155307.4853411D0F7@lists.varnish-cache.org> commit eb985308d850fccc993f3cec3ddc563f8311b489 Author: Asad Sajjad Ahmed Date: Wed Sep 6 10:49:02 2023 +0200 acceptor: create local acceptor list Now that each module is divided, we can limit the traversal of listen sockets to only listen sockets belonging to the respective module. This is done by keeping a local acceptor list of listen sockets in each module. Signed-off-by: Asad Sajjad Ahmed diff --git a/bin/varnishd/acceptor/cache_acceptor.h b/bin/varnishd/acceptor/cache_acceptor.h index 287c2e8ec..6dcc2e2f7 100644 --- a/bin/varnishd/acceptor/cache_acceptor.h +++ b/bin/varnishd/acceptor/cache_acceptor.h @@ -59,6 +59,7 @@ struct acceptor { unsigned magic; #define ACCEPTOR_MAGIC 0x0611847c VTAILQ_ENTRY(acceptor) list; + VTAILQ_HEAD(,listen_sock) socks; const char *name; acceptor_config_f *config; diff --git a/bin/varnishd/acceptor/cache_acceptor_tcp.c b/bin/varnishd/acceptor/cache_acceptor_tcp.c index ad4737275..f33938405 100644 --- a/bin/varnishd/acceptor/cache_acceptor_tcp.c +++ b/bin/varnishd/acceptor/cache_acceptor_tcp.c @@ -319,12 +319,9 @@ vca_tcp_start(struct cli *cli) (void)vca_tcp_sockopt_init(); - VTAILQ_FOREACH(ls, &heritage.socks, list) { + VTAILQ_FOREACH(ls, &TCP_acceptor.socks, vcalist) { CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); - if (ls->uds) - continue; - if (vca_tcp_listen(cli, ls)) return; } @@ -557,12 +554,9 @@ vca_tcp_accept(struct pool *pp) struct listen_sock *ls; struct poolsock *ps; - VTAILQ_FOREACH(ls, &heritage.socks, list) { + VTAILQ_FOREACH(ls, &TCP_acceptor.socks, vcalist) { CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); - if (ls->uds) - continue; - ALLOC_OBJ(ps, POOLSOCK_MAGIC); AN(ps); ps->lsock = ls; @@ -584,11 +578,11 @@ vca_tcp_update(pthread_mutex_t *shut_mtx) PTOK(pthread_mutex_lock(shut_mtx)); - VTAILQ_FOREACH(ls, &heritage.socks, list) { + VTAILQ_FOREACH(ls, &TCP_acceptor.socks, vcalist) { CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); - if (ls->sock == -2 || ls->uds) - continue; // VCA_Shutdown || UDS + if (ls->sock == -2) + continue; // VCA_Shutdown assert (ls->sock > 0); vca_tcp_sockopt_set(ls, NULL); /* If one of the options on a socket has @@ -613,12 +607,9 @@ vca_tcp_shutdown(void) struct listen_sock *ls; int i; - VTAILQ_FOREACH(ls, &heritage.socks, list) { + VTAILQ_FOREACH(ls, &TCP_acceptor.socks, vcalist) { CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); - if (ls->uds) - continue; - i = ls->sock; ls->sock = -2; (void)close(i); diff --git a/bin/varnishd/acceptor/cache_acceptor_uds.c b/bin/varnishd/acceptor/cache_acceptor_uds.c index c18406474..962475945 100644 --- a/bin/varnishd/acceptor/cache_acceptor_uds.c +++ b/bin/varnishd/acceptor/cache_acceptor_uds.c @@ -287,12 +287,9 @@ vca_uds_start(struct cli *cli) (void)vca_uds_sockopt_init(); - VTAILQ_FOREACH(ls, &heritage.socks, list) { + VTAILQ_FOREACH(ls, &UDS_acceptor.socks, vcalist) { CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); - if (!ls->uds) - continue; - if (vca_uds_listen(cli, ls)) return; } @@ -511,12 +508,9 @@ vca_uds_accept(struct pool *pp) struct listen_sock *ls; struct poolsock *ps; - VTAILQ_FOREACH(ls, &heritage.socks, list) { + VTAILQ_FOREACH(ls, &UDS_acceptor.socks, vcalist) { CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); - if (!ls->uds) - continue; - ALLOC_OBJ(ps, POOLSOCK_MAGIC); AN(ps); ps->lsock = ls; @@ -538,11 +532,11 @@ vca_uds_update(pthread_mutex_t *shut_mtx) PTOK(pthread_mutex_lock(shut_mtx)); - VTAILQ_FOREACH(ls, &heritage.socks, list) { + VTAILQ_FOREACH(ls, &UDS_acceptor.socks, vcalist) { CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); - if (ls->sock == -2 || !ls->uds) - continue; // VCA_Shutdown || TCP + if (ls->sock == -2) + continue; // VCA_Shutdown assert (ls->sock > 0); vca_uds_sockopt_set(ls, NULL); /* If one of the options on a socket has @@ -567,12 +561,9 @@ vca_uds_shutdown(void) struct listen_sock *ls; int i; - VTAILQ_FOREACH(ls, &heritage.socks, list) { + VTAILQ_FOREACH(ls, &UDS_acceptor.socks, vcalist) { CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); - if (!ls->uds) - continue; - i = ls->sock; ls->sock = -2; (void)close(i); diff --git a/bin/varnishd/acceptor/mgt_acceptor.c b/bin/varnishd/acceptor/mgt_acceptor.c index d172b4418..8286ae71b 100644 --- a/bin/varnishd/acceptor/mgt_acceptor.c +++ b/bin/varnishd/acceptor/mgt_acceptor.c @@ -175,6 +175,7 @@ VCA_Add(struct acceptor *vca) if (VCA_Find(vca->name) != NULL) ARGV_ERR("Acceptor '%s' already exist\n", vca->name); + VTAILQ_INIT(&vca->socks); VTAILQ_INSERT_TAIL(&acceptors, vca, list); if (vca->config()) diff --git a/bin/varnishd/acceptor/mgt_acceptor_tcp.c b/bin/varnishd/acceptor/mgt_acceptor_tcp.c index 43ba9c2f2..364b56d8f 100644 --- a/bin/varnishd/acceptor/mgt_acceptor_tcp.c +++ b/bin/varnishd/acceptor/mgt_acceptor_tcp.c @@ -99,10 +99,10 @@ vca_tcp_open_cb(void *priv, const struct suckaddr *sa) CAST_OBJ_NOTNULL(la, priv, LISTEN_ARG_MAGIC); - VTAILQ_FOREACH(ls, &heritage.socks, list) { + VTAILQ_FOREACH(ls, &TCP_acceptor.socks, vcalist) { CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); - if (!ls->uds && !VSA_Compare(sa, ls->addr)) + if (!VSA_Compare(sa, ls->addr)) ARGV_ERR("-a arguments %s and %s have same address\n", ls->endpoint, la->endpoint); } @@ -156,6 +156,7 @@ vca_tcp_open_cb(void *priv, const struct suckaddr *sa) VTAILQ_INSERT_TAIL(&la->socks, ls, arglist); VTAILQ_INSERT_TAIL(&heritage.socks, ls, list); + VTAILQ_INSERT_TAIL(&TCP_acceptor.socks, ls, vcalist); return (0); } @@ -204,12 +205,9 @@ vca_tcp_reopen(void) struct listen_sock *ls; int err, fail = 0; - VTAILQ_FOREACH(ls, &heritage.socks, list) { + VTAILQ_FOREACH(ls, &TCP_acceptor.socks, vcalist) { CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); - if (ls->uds) - continue; - VJ_master(JAIL_MASTER_PRIVPORT); err = vca_tcp_opensocket(ls); VJ_master(JAIL_MASTER_LOW); diff --git a/bin/varnishd/acceptor/mgt_acceptor_uds.c b/bin/varnishd/acceptor/mgt_acceptor_uds.c index e1b0d3242..97635b712 100644 --- a/bin/varnishd/acceptor/mgt_acceptor_uds.c +++ b/bin/varnishd/acceptor/mgt_acceptor_uds.c @@ -117,10 +117,10 @@ vca_uds_open_cb(void *priv, const struct sockaddr_un *uds) CAST_OBJ_NOTNULL(la, priv, LISTEN_ARG_MAGIC); (void) uds; - VTAILQ_FOREACH(ls, &heritage.socks, list) { + VTAILQ_FOREACH(ls, &UDS_acceptor.socks, vcalist) { CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); - if (ls->uds && strcmp(ls->endpoint, la->endpoint) == 0) + if (strcmp(ls->endpoint, la->endpoint) == 0) ARGV_ERR("-a arguments %s and %s have same address\n", ls->endpoint, la->endpoint); } @@ -156,6 +156,7 @@ vca_uds_open_cb(void *priv, const struct sockaddr_un *uds) VTAILQ_INSERT_TAIL(&la->socks, ls, arglist); VTAILQ_INSERT_TAIL(&heritage.socks, ls, list); + VTAILQ_INSERT_TAIL(&UDS_acceptor.socks, ls, vcalist); return (0); } @@ -280,12 +281,9 @@ vca_uds_reopen(void) struct listen_sock *ls; int err, fail = 0; - VTAILQ_FOREACH(ls, &heritage.socks, list) { + VTAILQ_FOREACH(ls, &UDS_acceptor.socks, vcalist) { CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); - if (!ls->uds) - continue; - VJ_master(JAIL_MASTER_PRIVPORT); err = vca_uds_opensocket(ls); VJ_master(JAIL_MASTER_LOW); From nils.goroll at uplex.de Mon Sep 30 15:53:07 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 15:53:07 +0000 (UTC) Subject: [master] 05ea6d307 acceptor: simplify UDS address & port Message-ID: <20240930155307.641E911D0FA@lists.varnish-cache.org> commit 05ea6d307fe5670a59b2a1960fb749e0ae28390f Author: Asad Sajjad Ahmed Date: Wed Sep 6 10:54:30 2023 +0200 acceptor: simplify UDS address & port Signed-off-by: Asad Sajjad Ahmed diff --git a/bin/varnishd/acceptor/cache_acceptor_uds.c b/bin/varnishd/acceptor/cache_acceptor_uds.c index 962475945..95515fc43 100644 --- a/bin/varnishd/acceptor/cache_acceptor_uds.c +++ b/bin/varnishd/acceptor/cache_acceptor_uds.c @@ -310,8 +310,7 @@ vca_uds_event(struct cli *cli, struct listen_sock *ls, enum vca_event event) } static void -vca_mk_uds(struct wrk_accept *wa, struct sess *sp, char *laddr, char *lport, - char *raddr, char *rport) +vca_mk_uds(struct wrk_accept *wa, struct sess *sp) { struct suckaddr *sa = NULL; ssize_t sz; @@ -326,20 +325,11 @@ vca_mk_uds(struct wrk_accept *wa, struct sess *sp, char *laddr, char *lport, sp->sattr[SA_SERVER_ADDR] = sp->sattr[SA_REMOTE_ADDR]; AN(SES_Set_String_Attr(sp, SA_CLIENT_IP, "0.0.0.0")); AN(SES_Set_String_Attr(sp, SA_CLIENT_PORT, "0")); - - strcpy(laddr, "0.0.0.0"); - strcpy(raddr, "0.0.0.0"); - strcpy(lport, "0"); - strcpy(rport, "0"); } static void v_matchproto_(task_func_t) vca_make_session(struct worker *wrk, void *arg) { - char laddr[VTCP_ADDRBUFSIZE]; - char lport[VTCP_PORTBUFSIZE]; - char raddr[VTCP_ADDRBUFSIZE]; - char rport[VTCP_PORTBUFSIZE]; struct wrk_accept *wa; struct sess *sp; struct req *req; @@ -365,14 +355,13 @@ vca_make_session(struct worker *wrk, void *arg) assert((size_t)wa->acceptaddrlen <= vsa_suckaddr_len); - vca_mk_uds(wa, sp, laddr, lport, raddr, rport); + vca_mk_uds(wa, sp); AN(wa->acceptlsock->name); VSL(SLT_Begin, sp->vxid, "sess 0 %s", wa->acceptlsock->transport->name); - VSL(SLT_SessOpen, sp->vxid, "%s %s %s %s %s %.6f %d", - raddr, rport, wa->acceptlsock->name, laddr, lport, - sp->t_open, sp->fd); + VSL(SLT_SessOpen, sp->vxid, "0.0.0.0 0 %s 0.0.0.0 0 %.6f %d", + wa->acceptlsock->name, sp->t_open, sp->fd); vca_pace_good(); wrk->stats->sess_conn++; From nils.goroll at uplex.de Mon Sep 30 15:53:07 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 15:53:07 +0000 (UTC) Subject: [master] 86231ff83 varnishd: adopt the new acceptor code Message-ID: <20240930155307.9666C11D105@lists.varnish-cache.org> commit 86231ff8392c62096e8e5adba22137eb8eaa6a8b Author: Asad Sajjad Ahmed Date: Wed Sep 6 13:49:23 2023 +0200 varnishd: adopt the new acceptor code Take in use the new acceptor code, and nuke the old acceptor code to not fail the magic check. Signed-off-by: Asad Sajjad Ahmed Signed-off-by: Nils Goroll - * - * SPDX-License-Identifier: BSD-2-Clause - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * This source file has the various trickery surrounding the accept/listen - * sockets. - * - */ - -#include "config.h" - -#include -#include -#include - -#include "cache_varnishd.h" - -#include "cache_transport.h" -#include "cache_pool.h" -#include "common/heritage.h" - -#include "vcli_serve.h" -#include "vsa.h" -#include "vtcp.h" -#include "vtim.h" - -static pthread_t VCA_thread; -static vtim_dur vca_pace = 0.0; -static struct lock pace_mtx; -static unsigned pool_accepting; -static pthread_mutex_t shut_mtx = PTHREAD_MUTEX_INITIALIZER; - -struct wrk_accept { - unsigned magic; -#define WRK_ACCEPT_MAGIC 0x8c4b4d59 - - /* Accept stuff */ - struct sockaddr_storage acceptaddr; - socklen_t acceptaddrlen; - int acceptsock; - struct listen_sock *acceptlsock; -}; - -struct poolsock { - unsigned magic; -#define POOLSOCK_MAGIC 0x1b0a2d38 - VTAILQ_ENTRY(poolsock) list; - struct listen_sock *lsock; - struct pool_task task[1]; - struct pool *pool; -}; - -/*-------------------------------------------------------------------- - * TCP options we want to control - */ - -union sock_arg { - struct linger lg; - struct timeval tv; - int i; -}; - -static struct sock_opt { - int level; - int optname; - const char *strname; - unsigned mod; - socklen_t sz; - union sock_arg arg[1]; -} sock_opts[] = { - /* Note: Setting the mod counter to something not-zero is needed - * to force the setsockopt() calls on startup */ -#define SOCK_OPT(lvl, nam, typ) { lvl, nam, #nam, 1, sizeof(typ) }, - - SOCK_OPT(SOL_SOCKET, SO_LINGER, struct linger) - SOCK_OPT(SOL_SOCKET, SO_KEEPALIVE, int) - SOCK_OPT(SOL_SOCKET, SO_SNDTIMEO, struct timeval) - SOCK_OPT(SOL_SOCKET, SO_RCVTIMEO, struct timeval) - - SOCK_OPT(IPPROTO_TCP, TCP_NODELAY, int) - -#if defined(HAVE_TCP_KEEP) - SOCK_OPT(IPPROTO_TCP, TCP_KEEPIDLE, int) - SOCK_OPT(IPPROTO_TCP, TCP_KEEPCNT, int) - SOCK_OPT(IPPROTO_TCP, TCP_KEEPINTVL, int) -#elif defined(HAVE_TCP_KEEPALIVE) - SOCK_OPT(IPPROTO_TCP, TCP_KEEPALIVE, int) -#endif - -#undef SOCK_OPT -}; - -static const int n_sock_opts = sizeof sock_opts / sizeof sock_opts[0]; - -struct conn_heritage { - unsigned sess_set; - unsigned listen_mod; -}; - -/*-------------------------------------------------------------------- - * We want to get out of any kind of trouble-hit TCP connections as fast - * as absolutely possible, so we set them LINGER disabled, so that even if - * there are outstanding write data on the socket, a close(2) will return - * immediately. - */ -static const struct linger disable_so_linger = { - .l_onoff = 0, -}; - -/* - * We turn on keepalives by default to assist in detecting clients that have - * hung up on connections returning from waitinglists - */ - -static const unsigned enable_so_keepalive = 1; - -/* We disable Nagle's algorithm in favor of low latency setups. - */ - -static const unsigned enable_tcp_nodelay = 1; - -/*-------------------------------------------------------------------- - * lacking a better place, we put some generic periodic updates - * into the vca_acct() loop which we are running anyway - */ -static void -vca_periodic(vtim_real t0) -{ - vtim_real now; - - now = VTIM_real(); - VSC_C_main->uptime = (uint64_t)(now - t0); - - VTIM_postel = FEATURE(FEATURE_HTTP_DATE_POSTEL); -} - -/*-------------------------------------------------------------------- - * Some kernels have bugs/limitations with respect to which options are - * inherited from the accept/listen socket, so we have to keep track of - * which, if any, sockopts we have to set on the accepted socket. - */ - -static int -vca_sock_opt_init(void) -{ - struct sock_opt *so; - union sock_arg tmp; - int n, chg = 0; - size_t sz; - - memset(&tmp, 0, sizeof tmp); - - for (n = 0; n < n_sock_opts; n++) { - so = &sock_opts[n]; - -#define SET_VAL(nm, so, fld, val) \ - do { \ - if (!strcmp(#nm, so->strname)) { \ - assert(so->sz == sizeof so->arg->fld); \ - so->arg->fld = (val); \ - } \ - } while (0) - -#define NEW_VAL(nm, so, fld, val) \ - do { \ - if (!strcmp(#nm, so->strname)) { \ - sz = sizeof tmp.fld; \ - assert(so->sz == sz); \ - tmp.fld = (val); \ - if (memcmp(&so->arg->fld, &(tmp.fld), sz)) { \ - memcpy(&so->arg->fld, &(tmp.fld), sz); \ - so->mod++; \ - chg = 1; \ - } \ - } \ - } while (0) - - SET_VAL(SO_LINGER, so, lg, disable_so_linger); - SET_VAL(SO_KEEPALIVE, so, i, enable_so_keepalive); - NEW_VAL(SO_SNDTIMEO, so, tv, - VTIM_timeval_sock(cache_param->idle_send_timeout)); - NEW_VAL(SO_RCVTIMEO, so, tv, - VTIM_timeval_sock(cache_param->timeout_idle)); - SET_VAL(TCP_NODELAY, so, i, enable_tcp_nodelay); -#if defined(HAVE_TCP_KEEP) - NEW_VAL(TCP_KEEPIDLE, so, i, - (int)cache_param->tcp_keepalive_time); - NEW_VAL(TCP_KEEPCNT, so, i, - (int)cache_param->tcp_keepalive_probes); - NEW_VAL(TCP_KEEPINTVL, so, i, - (int)cache_param->tcp_keepalive_intvl); -#elif defined(HAVE_TCP_KEEPALIVE) - NEW_VAL(TCP_KEEPALIVE, so, i, - (int)cache_param->tcp_keepalive_time); -#endif - } - return (chg); -} - -static void -vca_sock_opt_test(const struct listen_sock *ls, const struct sess *sp) -{ - struct conn_heritage *ch; - struct sock_opt *so; - union sock_arg tmp; - socklen_t l; - int i, n; - - CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - - for (n = 0; n < n_sock_opts; n++) { - so = &sock_opts[n]; - ch = &ls->conn_heritage[n]; - if (ch->sess_set) { - VSL(SLT_Debug, sp->vxid, - "sockopt: Not testing nonhereditary %s for %s=%s", - so->strname, ls->name, ls->endpoint); - continue; - } - if (so->level == IPPROTO_TCP && ls->uds) { - VSL(SLT_Debug, sp->vxid, - "sockopt: Not testing incompatible %s for %s=%s", - so->strname, ls->name, ls->endpoint); - continue; - } - memset(&tmp, 0, sizeof tmp); - l = so->sz; - i = getsockopt(sp->fd, so->level, so->optname, &tmp, &l); - if (i == 0 && memcmp(&tmp, so->arg, so->sz)) { - VSL(SLT_Debug, sp->vxid, - "sockopt: Test confirmed %s non heredity for %s=%s", - so->strname, ls->name, ls->endpoint); - ch->sess_set = 1; - } - if (i && errno != ENOPROTOOPT) - VTCP_Assert(i); - } -} - -static void -vca_sock_opt_set(const struct listen_sock *ls, const struct sess *sp) -{ - struct conn_heritage *ch; - struct sock_opt *so; - vxid_t vxid; - int n, sock; - - CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); - - if (sp != NULL) { - CHECK_OBJ(sp, SESS_MAGIC); - sock = sp->fd; - vxid = sp->vxid; - } else { - sock = ls->sock; - vxid = NO_VXID; - } - - for (n = 0; n < n_sock_opts; n++) { - so = &sock_opts[n]; - ch = &ls->conn_heritage[n]; - if (so->level == IPPROTO_TCP && ls->uds) { - VSL(SLT_Debug, vxid, - "sockopt: Not setting incompatible %s for %s=%s", - so->strname, ls->name, ls->endpoint); - continue; - } - if (sp == NULL && ch->listen_mod == so->mod) { - VSL(SLT_Debug, vxid, - "sockopt: Not setting unmodified %s for %s=%s", - so->strname, ls->name, ls->endpoint); - continue; - } - if (sp != NULL && !ch->sess_set) { - VSL(SLT_Debug, sp->vxid, - "sockopt: %s may be inherited for %s=%s", - so->strname, ls->name, ls->endpoint); - continue; - } - VSL(SLT_Debug, vxid, - "sockopt: Setting %s for %s=%s", - so->strname, ls->name, ls->endpoint); - VTCP_Assert(setsockopt(sock, - so->level, so->optname, so->arg, so->sz)); - if (sp == NULL) - ch->listen_mod = so->mod; - } -} - -/*-------------------------------------------------------------------- - * If accept(2)'ing fails, we pace ourselves to relive any resource - * shortage if possible. - */ - -static void -vca_pace_check(void) -{ - vtim_dur p; - - if (vca_pace == 0.0) - return; - Lck_Lock(&pace_mtx); - p = vca_pace; - Lck_Unlock(&pace_mtx); - if (p > 0.0) - VTIM_sleep(p); -} - -static void -vca_pace_bad(void) -{ - - Lck_Lock(&pace_mtx); - vca_pace += cache_param->acceptor_sleep_incr; - if (vca_pace > cache_param->acceptor_sleep_max) - vca_pace = cache_param->acceptor_sleep_max; - Lck_Unlock(&pace_mtx); -} - -static void -vca_pace_good(void) -{ - - if (vca_pace == 0.0) - return; - Lck_Lock(&pace_mtx); - vca_pace *= cache_param->acceptor_sleep_decay; - if (vca_pace < cache_param->acceptor_sleep_incr) - vca_pace = 0.0; - Lck_Unlock(&pace_mtx); -} - -/*-------------------------------------------------------------------- - * The pool-task for a newly accepted session - * - * Called from assigned worker thread - */ - -static void -vca_mk_tcp(const struct wrk_accept *wa, - struct sess *sp, char *laddr, char *lport, char *raddr, char *rport) -{ - struct suckaddr *sa = NULL; - ssize_t sz; - - AN(SES_Reserve_remote_addr(sp, &sa, &sz)); - AN(sa); - assert(sz == vsa_suckaddr_len); - AN(VSA_Build(sa, &wa->acceptaddr, wa->acceptaddrlen)); - sp->sattr[SA_CLIENT_ADDR] = sp->sattr[SA_REMOTE_ADDR]; - - VTCP_name(sa, raddr, VTCP_ADDRBUFSIZE, rport, VTCP_PORTBUFSIZE); - AN(SES_Set_String_Attr(sp, SA_CLIENT_IP, raddr)); - AN(SES_Set_String_Attr(sp, SA_CLIENT_PORT, rport)); - - - AN(SES_Reserve_local_addr(sp, &sa, &sz)); - AN(VSA_getsockname(sp->fd, sa, sz)); - sp->sattr[SA_SERVER_ADDR] = sp->sattr[SA_LOCAL_ADDR]; - VTCP_name(sa, laddr, VTCP_ADDRBUFSIZE, lport, VTCP_PORTBUFSIZE); -} - -static void -vca_mk_uds(struct wrk_accept *wa, struct sess *sp, char *laddr, char *lport, - char *raddr, char *rport) -{ - struct suckaddr *sa = NULL; - ssize_t sz; - - (void) wa; - AN(SES_Reserve_remote_addr(sp, &sa, &sz)); - AN(sa); - assert(sz == vsa_suckaddr_len); - AZ(SES_Set_remote_addr(sp, bogo_ip)); - sp->sattr[SA_CLIENT_ADDR] = sp->sattr[SA_REMOTE_ADDR]; - sp->sattr[SA_LOCAL_ADDR] = sp->sattr[SA_REMOTE_ADDR]; - sp->sattr[SA_SERVER_ADDR] = sp->sattr[SA_REMOTE_ADDR]; - AN(SES_Set_String_Attr(sp, SA_CLIENT_IP, "0.0.0.0")); - AN(SES_Set_String_Attr(sp, SA_CLIENT_PORT, "0")); - - strcpy(laddr, "0.0.0.0"); - strcpy(raddr, "0.0.0.0"); - strcpy(lport, "0"); - strcpy(rport, "0"); -} - -static void v_matchproto_(task_func_t) -vca_make_session(struct worker *wrk, void *arg) -{ - struct sess *sp; - struct req *req; - struct wrk_accept *wa; - char laddr[VTCP_ADDRBUFSIZE]; - char lport[VTCP_PORTBUFSIZE]; - char raddr[VTCP_ADDRBUFSIZE]; - char rport[VTCP_PORTBUFSIZE]; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CAST_OBJ_NOTNULL(wa, arg, WRK_ACCEPT_MAGIC); - - VTCP_blocking(wa->acceptsock); - - /* Turn accepted socket into a session */ - AN(WS_Reservation(wrk->aws)); - sp = SES_New(wrk->pool); - CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); - wrk->stats->s_sess++; - - sp->t_open = VTIM_real(); - sp->t_idle = sp->t_open; - sp->vxid = VXID_Get(wrk, VSL_CLIENTMARKER); - - sp->fd = wa->acceptsock; - wa->acceptsock = -1; - sp->listen_sock = wa->acceptlsock; - - assert((size_t)wa->acceptaddrlen <= vsa_suckaddr_len); - - if (wa->acceptlsock->uds) - vca_mk_uds(wa, sp, laddr, lport, raddr, rport); - else - vca_mk_tcp(wa, sp, laddr, lport, raddr, rport); - - AN(wa->acceptlsock->name); - VSL(SLT_Begin, sp->vxid, "sess 0 %s", - wa->acceptlsock->transport->name); - VSL(SLT_SessOpen, sp->vxid, "%s %s %s %s %s %.6f %d", - raddr, rport, wa->acceptlsock->name, laddr, lport, - sp->t_open, sp->fd); - - vca_pace_good(); - wrk->stats->sess_conn++; - - if (wa->acceptlsock->test_heritage) { - vca_sock_opt_test(wa->acceptlsock, sp); - wa->acceptlsock->test_heritage = 0; - } - vca_sock_opt_set(wa->acceptlsock, sp); - - req = Req_New(sp); - CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - req->htc->rfd = &sp->fd; - - SES_SetTransport(wrk, sp, req, wa->acceptlsock->transport); - WS_Release(wrk->aws, 0); -} - -/*-------------------------------------------------------------------- - * This function accepts on a single socket for a single thread pool. - * - * As long as we can stick the accepted connection to another thread - * we do so, otherwise we put the socket back on the "BACK" pool - * and handle the new connection ourselves. - */ - -static void v_matchproto_(task_func_t) -vca_accept_task(struct worker *wrk, void *arg) -{ - struct wrk_accept wa; - struct poolsock *ps; - struct listen_sock *ls; - int i; - char laddr[VTCP_ADDRBUFSIZE]; - char lport[VTCP_PORTBUFSIZE]; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CAST_OBJ_NOTNULL(ps, arg, POOLSOCK_MAGIC); - ls = ps->lsock; - CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); - - while (!pool_accepting) - VTIM_sleep(.1); - - /* Dont hold on to (possibly) discarded VCLs */ - if (wrk->wpriv->vcl != NULL) - VCL_Rel(&wrk->wpriv->vcl); - - while (!ps->pool->die) { - INIT_OBJ(&wa, WRK_ACCEPT_MAGIC); - wa.acceptlsock = ls; - - vca_pace_check(); - - wa.acceptaddrlen = sizeof wa.acceptaddr; - do { - i = accept(ls->sock, (void*)&wa.acceptaddr, - &wa.acceptaddrlen); - } while (i < 0 && errno == EAGAIN && !ps->pool->die); - - if (i < 0 && ps->pool->die) - break; - - if (i < 0 && ls->sock == -2) { - /* Shut down in progress */ - sleep(2); - continue; - } - - if (i < 0) { - switch (errno) { - case ECONNABORTED: - wrk->stats->sess_fail_econnaborted++; - break; - case EINTR: - wrk->stats->sess_fail_eintr++; - break; - case EMFILE: - wrk->stats->sess_fail_emfile++; - vca_pace_bad(); - break; - case EBADF: - wrk->stats->sess_fail_ebadf++; - vca_pace_bad(); - break; - case ENOBUFS: - case ENOMEM: - wrk->stats->sess_fail_enomem++; - vca_pace_bad(); - break; - default: - wrk->stats->sess_fail_other++; - vca_pace_bad(); - break; - } - - i = errno; - wrk->stats->sess_fail++; - - if (wa.acceptlsock->uds) { - bstrcpy(laddr, "0.0.0.0"); - bstrcpy(lport, "0"); - } else { - VTCP_myname(ls->sock, laddr, VTCP_ADDRBUFSIZE, - lport, VTCP_PORTBUFSIZE); - } - - VSL(SLT_SessError, NO_VXID, "%s %s %s %d %d \"%s\"", - wa.acceptlsock->name, laddr, lport, - ls->sock, i, VAS_errtxt(i)); - (void)Pool_TrySumstat(wrk); - continue; - } - - wa.acceptsock = i; - - if (!Pool_Task_Arg(wrk, TASK_QUEUE_REQ, - vca_make_session, &wa, sizeof wa)) { - /* - * We couldn't get another thread, so we will handle - * the request in this worker thread, but first we - * must reschedule the listening task so it will be - * taken up by another thread again. - */ - if (!ps->pool->die) { - AZ(Pool_Task(wrk->pool, ps->task, - TASK_QUEUE_VCA)); - return; - } - } - if (!ps->pool->die && DO_DEBUG(DBG_SLOW_ACCEPTOR)) - VTIM_sleep(2.0); - - } - - VSL(SLT_Debug, NO_VXID, "XXX Accept thread dies %p", ps); - FREE_OBJ(ps); -} - -/*-------------------------------------------------------------------- - * Called when a worker and attached thread pool is created, to - * allocate the tasks which will listen to sockets for that pool. - */ - -void -VCA_NewPool(struct pool *pp) -{ - struct listen_sock *ls; - struct poolsock *ps; - - VTAILQ_FOREACH(ls, &heritage.socks, list) { - ALLOC_OBJ(ps, POOLSOCK_MAGIC); - AN(ps); - ps->lsock = ls; - ps->task->func = vca_accept_task; - ps->task->priv = ps; - ps->pool = pp; - VTAILQ_INSERT_TAIL(&pp->poolsocks, ps, list); - AZ(Pool_Task(pp, ps->task, TASK_QUEUE_VCA)); - } -} - -void -VCA_DestroyPool(struct pool *pp) -{ - struct poolsock *ps; - - while (!VTAILQ_EMPTY(&pp->poolsocks)) { - ps = VTAILQ_FIRST(&pp->poolsocks); - VTAILQ_REMOVE(&pp->poolsocks, ps, list); - } -} - -/*--------------------------------------------------------------------*/ - -static void * v_matchproto_() -vca_acct(void *arg) -{ - struct listen_sock *ls; - vtim_real t0; - - // XXX Actually a misnomer now because the accept happens in a pool - // thread. Rename to accept-nanny or so? - THR_SetName("cache-acceptor"); - THR_Init(); - (void)arg; - - t0 = VTIM_real(); - vca_periodic(t0); - - pool_accepting = 1; - - while (1) { - (void)sleep(1); - if (vca_sock_opt_init()) { - PTOK(pthread_mutex_lock(&shut_mtx)); - VTAILQ_FOREACH(ls, &heritage.socks, list) { - if (ls->sock == -2) - continue; // VCA_Shutdown - assert (ls->sock > 0); - vca_sock_opt_set(ls, NULL); - /* If one of the options on a socket has - * changed, also force a retest of whether - * the values are inherited to the - * accepted sockets. This should then - * catch any false positives from previous - * tests that could happen if the set - * value of an option happened to just be - * the OS default for that value, and - * wasn't actually inherited from the - * listening socket. */ - ls->test_heritage = 1; - } - PTOK(pthread_mutex_unlock(&shut_mtx)); - } - vca_periodic(t0); - } - NEEDLESS(return (NULL)); -} - -/*--------------------------------------------------------------------*/ - -void -VCA_Start(struct cli *cli) -{ - struct listen_sock *ls; - - (void)vca_sock_opt_init(); - - VTAILQ_FOREACH(ls, &heritage.socks, list) { - CHECK_OBJ_NOTNULL(ls->transport, TRANSPORT_MAGIC); - assert (ls->sock > 0); // We know where stdin is - if (cache_param->tcp_fastopen && - VTCP_fastopen(ls->sock, cache_param->listen_depth)) - VSL(SLT_Error, NO_VXID, - "Kernel TCP Fast Open: sock=%d, errno=%d %s", - ls->sock, errno, VAS_errtxt(errno)); - if (listen(ls->sock, cache_param->listen_depth)) { - VCLI_SetResult(cli, CLIS_CANT); - VCLI_Out(cli, "Listen failed on socket '%s': %s", - ls->endpoint, VAS_errtxt(errno)); - return; - } - AZ(ls->conn_heritage); - ls->conn_heritage = calloc(n_sock_opts, - sizeof *ls->conn_heritage); - AN(ls->conn_heritage); - ls->test_heritage = 1; - vca_sock_opt_set(ls, NULL); - if (cache_param->accept_filter && VTCP_filter_http(ls->sock)) - VSL(SLT_Error, NO_VXID, - "Kernel filtering: sock=%d, errno=%d %s", - ls->sock, errno, VAS_errtxt(errno)); - } - - PTOK(pthread_create(&VCA_thread, NULL, vca_acct, NULL)); -} - -/*--------------------------------------------------------------------*/ - -static void v_matchproto_(cli_func_t) -ccf_listen_address(struct cli *cli, const char * const *av, void *priv) -{ - struct listen_sock *ls; - char h[VTCP_ADDRBUFSIZE], p[VTCP_PORTBUFSIZE]; - - (void)cli; - (void)av; - (void)priv; - - /* - * This CLI command is primarily used by varnishtest. Don't - * respond until listen(2) has been called, in order to avoid - * a race where varnishtest::client would attempt to connect(2) - * before listen(2) has been called. - */ - while (!pool_accepting) - VTIM_sleep(.1); - - PTOK(pthread_mutex_lock(&shut_mtx)); - VTAILQ_FOREACH(ls, &heritage.socks, list) { - if (!ls->uds) { - VTCP_myname(ls->sock, h, sizeof h, p, sizeof p); - VCLI_Out(cli, "%s %s %s\n", ls->name, h, p); - } - else - VCLI_Out(cli, "%s %s -\n", ls->name, ls->endpoint); - } - PTOK(pthread_mutex_unlock(&shut_mtx)); -} - -/*--------------------------------------------------------------------*/ - -static struct cli_proto vca_cmds[] = { - { CLICMD_DEBUG_LISTEN_ADDRESS, "d", ccf_listen_address }, - { NULL } -}; - -void -VCA_Init(void) -{ - - CLI_AddFuncs(vca_cmds); - Lck_New(&pace_mtx, lck_vcapace); -} - -void -VCA_Shutdown(void) -{ - struct listen_sock *ls; - int i; - - PTOK(pthread_mutex_lock(&shut_mtx)); - VTAILQ_FOREACH(ls, &heritage.socks, list) { - i = ls->sock; - ls->sock = -2; - (void)close(i); - } - PTOK(pthread_mutex_unlock(&shut_mtx)); -} - -/*-------------------------------------------------------------------- - * Transport protocol registration - * - */ - -static VTAILQ_HEAD(,transport) transports = - VTAILQ_HEAD_INITIALIZER(transports); - -static uint16_t next_xport; - -static void -XPORT_Register(struct transport *xp) -{ - - CHECK_OBJ_NOTNULL(xp, TRANSPORT_MAGIC); - AZ(xp->number); - - xp->number = ++next_xport; - VTAILQ_INSERT_TAIL(&transports, xp, list); -} - -void -XPORT_Init(void) -{ - - ASSERT_MGT(); - -#define TRANSPORT_MACRO(name) XPORT_Register(&name##_transport); - TRANSPORTS -#undef TRANSPORT_MACRO -} - -const struct transport * -XPORT_Find(const char *name) -{ - const struct transport *xp; - - ASSERT_MGT(); - - VTAILQ_FOREACH(xp, &transports, list) - if (xp->proto_ident != NULL && - !strcasecmp(xp->proto_ident, name)) - return (xp); - return (NULL); -} - -const struct transport * -XPORT_ByNumber(uint16_t no) -{ - const struct transport *xp; - - VTAILQ_FOREACH(xp, &transports, list) - if (xp->number == no) - return (xp); - return (NULL); -} diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index 284a58be6..d10a70d31 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -32,6 +32,7 @@ #include "config.h" #include "cache_varnishd.h" +#include "acceptor/cache_acceptor.h" #include #include diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 73a10352e..31fd57eb6 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -156,12 +156,10 @@ struct vcf { /* Prototypes etc ----------------------------------------------------*/ -/* cache_acceptor.c */ -void VCA_Init(void); -void VCA_Start(struct cli *cli); -void VCA_Shutdown(void); - /* cache_backend.c */ +struct backend; + +/* cache_backend_cfg.c */ void VBE_InitCfg(void); /* cache_ban.c */ diff --git a/bin/varnishd/common/heritage.h b/bin/varnishd/common/heritage.h index 499c91423..bf207b17b 100644 --- a/bin/varnishd/common/heritage.h +++ b/bin/varnishd/common/heritage.h @@ -117,7 +117,7 @@ void child_main(int, size_t); /* cache/cache_vcl.c */ int VCL_TestLoad(const char *); -/* cache/cache_acceptor.c */ +/* acceptor/cache_acceptor.c */ struct transport; void XPORT_Init(void); const struct transport *XPORT_Find(const char *name); diff --git a/bin/varnishd/mgt/mgt.h b/bin/varnishd/mgt/mgt.h index 4c40e7ccf..f98862363 100644 --- a/bin/varnishd/mgt/mgt.h +++ b/bin/varnishd/mgt/mgt.h @@ -68,11 +68,6 @@ extern int exit_status; extern const char * const builtin_vcl; -/* mgt_acceptor.c */ - -void MAC_Arg(const char *); -int MAC_reopen_sockets(void); - /* mgt_child.c */ void MCH_Init(void); int MCH_Running(void); diff --git a/bin/varnishd/mgt/mgt_acceptor.c b/bin/varnishd/mgt/mgt_acceptor.c deleted file mode 100644 index b09a9f8f6..000000000 --- a/bin/varnishd/mgt/mgt_acceptor.c +++ /dev/null @@ -1,381 +0,0 @@ -/*- - * Copyright (c) 2006 Verdens Gang AS - * Copyright (c) 2006-2015 Varnish Software AS - * All rights reserved. - * - * Author: Poul-Henning Kamp - * - * SPDX-License-Identifier: BSD-2-Clause - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * Acceptor socket management - */ - -#include "config.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "mgt/mgt.h" -#include "common/heritage.h" - -#include "vav.h" -#include "vcli_serve.h" -#include "vsa.h" -#include "vss.h" -#include "vtcp.h" -#include "vus.h" - -struct listen_arg { - unsigned magic; -#define LISTEN_ARG_MAGIC 0xbb2fc333 - VTAILQ_ENTRY(listen_arg) list; - const char *endpoint; - const char *name; - VTAILQ_HEAD(,listen_sock) socks; - const struct transport *transport; - const struct uds_perms *perms; -}; - -struct uds_perms { - unsigned magic; -#define UDS_PERMS_MAGIC 0x84fb5635 - mode_t mode; - uid_t uid; - gid_t gid; -}; - -static VTAILQ_HEAD(,listen_arg) listen_args = - VTAILQ_HEAD_INITIALIZER(listen_args); - -static int -mac_vus_bind(void *priv, const struct sockaddr_un *uds) -{ - return (VUS_bind(uds, priv)); -} - -static int -mac_opensocket(struct listen_sock *ls) -{ - int fail; - const char *err; - - CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); - if (ls->sock > 0) { - MCH_Fd_Inherit(ls->sock, NULL); - closefd(&ls->sock); - } - if (!ls->uds) - ls->sock = VTCP_bind(ls->addr, NULL); - else - ls->sock = VUS_resolver(ls->endpoint, mac_vus_bind, NULL, &err); - fail = errno; - if (ls->sock < 0) { - AN(fail); - return (fail); - } - if (ls->perms != NULL) { - CHECK_OBJ(ls->perms, UDS_PERMS_MAGIC); - assert(ls->uds); - errno = 0; - if (ls->perms->mode != 0 && - chmod(ls->endpoint, ls->perms->mode) != 0) - return (errno); - if (chown(ls->endpoint, ls->perms->uid, ls->perms->gid) != 0) - return (errno); - } - MCH_Fd_Inherit(ls->sock, "sock"); - return (0); -} - -/*===================================================================== - * Reopen the accept sockets to get rid of listen status. - * returns the highest errno encountered, 0 for success - */ - -int -MAC_reopen_sockets(void) -{ - struct listen_sock *ls; - int err, fail = 0; - - VTAILQ_FOREACH(ls, &heritage.socks, list) { - VJ_master(JAIL_MASTER_PRIVPORT); - err = mac_opensocket(ls); - VJ_master(JAIL_MASTER_LOW); - if (err == 0) - continue; - fail = vmax(fail, err); - MGT_Complain(C_ERR, - "Could not reopen listen socket %s: %s", - ls->endpoint, VAS_errtxt(err)); - } - return (fail); -} - -/*--------------------------------------------------------------------*/ - -static struct listen_sock * -mk_listen_sock(const struct listen_arg *la, const struct suckaddr *sa) -{ - struct listen_sock *ls; - int fail; - - ALLOC_OBJ(ls, LISTEN_SOCK_MAGIC); - AN(ls); - ls->sock = -1; - ls->addr = VSA_Clone(sa); - AN(ls->addr); - REPLACE(ls->endpoint, la->endpoint); - ls->name = la->name; - ls->transport = la->transport; - ls->perms = la->perms; - ls->uds = VUS_is(la->endpoint); - VJ_master(JAIL_MASTER_PRIVPORT); - fail = mac_opensocket(ls); - VJ_master(JAIL_MASTER_LOW); - if (fail) { - VSA_free(&ls->addr); - free(ls->endpoint); - FREE_OBJ(ls); - if (fail != EAFNOSUPPORT) - ARGV_ERR("Could not get socket %s: %s\n", - la->endpoint, VAS_errtxt(fail)); - return (NULL); - } - return (ls); -} - -static int v_matchproto_(vss_resolved_f) -mac_tcp(void *priv, const struct suckaddr *sa) -{ - struct listen_arg *la; - struct listen_sock *ls; - char abuf[VTCP_ADDRBUFSIZE], pbuf[VTCP_PORTBUFSIZE]; - char nbuf[VTCP_ADDRBUFSIZE+VTCP_PORTBUFSIZE+2]; - - CAST_OBJ_NOTNULL(la, priv, LISTEN_ARG_MAGIC); - - VTAILQ_FOREACH(ls, &heritage.socks, list) { - if (!ls->uds && !VSA_Compare(sa, ls->addr)) - ARGV_ERR("-a arguments %s and %s have same address\n", - ls->endpoint, la->endpoint); - } - ls = mk_listen_sock(la, sa); - if (ls == NULL) - return (0); - AZ(ls->uds); - if (VSA_Port(ls->addr) == 0) { - /* - * If the argv port number is zero, we adopt whatever - * port number this VTCP_bind() found us, as if - * it was specified by the argv. - */ - VSA_free(&ls->addr); - ls->addr = VTCP_my_suckaddr(ls->sock); - VTCP_myname(ls->sock, abuf, sizeof abuf, - pbuf, sizeof pbuf); - if (VSA_Get_Proto(sa) == AF_INET6) - bprintf(nbuf, "[%s]:%s", abuf, pbuf); - else - bprintf(nbuf, "%s:%s", abuf, pbuf); - REPLACE(ls->endpoint, nbuf); - } - VTAILQ_INSERT_TAIL(&la->socks, ls, arglist); - VTAILQ_INSERT_TAIL(&heritage.socks, ls, list); - return (0); -} - -static int v_matchproto_(vus_resolved_f) -mac_uds(void *priv, const struct sockaddr_un *uds) -{ - struct listen_arg *la; - struct listen_sock *ls; - - CAST_OBJ_NOTNULL(la, priv, LISTEN_ARG_MAGIC); - (void) uds; - - VTAILQ_FOREACH(ls, &heritage.socks, list) { - if (ls->uds && strcmp(ls->endpoint, la->endpoint) == 0) - ARGV_ERR("-a arguments %s and %s have same address\n", - ls->endpoint, la->endpoint); - } - ls = mk_listen_sock(la, bogo_ip); - if (ls == NULL) - return (0); - AN(ls->uds); - VTAILQ_INSERT_TAIL(&la->socks, ls, arglist); - VTAILQ_INSERT_TAIL(&heritage.socks, ls, list); - return (0); -} - -void -MAC_Arg(const char *spec) -{ - char **av; - struct listen_arg *la; - const char *err; - int error; - const struct transport *xp = NULL; - const char *name; - char name_buf[8]; - static unsigned seq = 0; - struct passwd *pwd = NULL; - struct group *grp = NULL; - mode_t mode = 0; - struct uds_perms *perms; - - av = MGT_NamedArg(spec, &name, "-a"); - AN(av); - - ALLOC_OBJ(la, LISTEN_ARG_MAGIC); - AN(la); - VTAILQ_INIT(&la->socks); - VTAILQ_INSERT_TAIL(&listen_args, la, list); - la->endpoint = av[1]; - - if (name == NULL) { - bprintf(name_buf, "a%u", seq++); - name = strdup(name_buf); - AN(name); - } - la->name = name; - - if (*la->endpoint != '/' && strchr(la->endpoint, '/') != NULL) - ARGV_ERR("Unix domain socket addresses must be" - " absolute paths in -a (%s)\n", la->endpoint); - - if (VUS_is(la->endpoint) && heritage.min_vcl_version < 41) - heritage.min_vcl_version = 41; - - for (int i = 2; av[i] != NULL; i++) { - char *eq, *val; - int len; - - if ((eq = strchr(av[i], '=')) == NULL) { - if (xp != NULL) - ARGV_ERR("Too many protocol sub-args" - " in -a (%s)\n", av[i]); - xp = XPORT_Find(av[i]); - if (xp == NULL) - ARGV_ERR("Unknown protocol '%s'\n", av[i]); - continue; - } - if (la->endpoint[0] != '/') - ARGV_ERR("Invalid sub-arg %s" - " in -a\n", av[i]); - - val = eq + 1; - len = eq - av[i]; - assert(len >= 0); - if (len == 0) - ARGV_ERR("Invalid sub-arg %s in -a\n", av[i]); - - if (strncmp(av[i], "user", len) == 0) { - if (pwd != NULL) - ARGV_ERR("Too many user sub-args in -a (%s)\n", - av[i]); - pwd = getpwnam(val); - if (pwd == NULL) - ARGV_ERR("Unknown user %s in -a\n", val); - continue; - } - - if (strncmp(av[i], "group", len) == 0) { - if (grp != NULL) - ARGV_ERR("Too many group sub-args in -a (%s)\n", - av[i]); - grp = getgrnam(val); - if (grp == NULL) - ARGV_ERR("Unknown group %s in -a\n", val); - continue; - } - - if (strncmp(av[i], "mode", len) == 0) { - long m; - char *p; - - if (mode != 0) - ARGV_ERR("Too many mode sub-args in -a (%s)\n", - av[i]); - if (*val == '\0') - ARGV_ERR("Empty mode sub-arg in -a\n"); - errno = 0; - m = strtol(val, &p, 8); - if (*p != '\0') - ARGV_ERR("Invalid mode sub-arg %s in -a\n", - val); - if (errno) - ARGV_ERR("Cannot parse mode sub-arg %s in -a: " - "%s\n", val, VAS_errtxt(errno)); - if (m <= 0 || m > 0777) - ARGV_ERR("Mode sub-arg %s out of range in -a\n", - val); - mode = (mode_t) m; - continue; - } - - ARGV_ERR("Invalid sub-arg %s in -a\n", av[i]); - } - - if (xp == NULL) - xp = XPORT_Find("http"); - AN(xp); - la->transport = xp; - - if (pwd != NULL || grp != NULL || mode != 0) { - ALLOC_OBJ(perms, UDS_PERMS_MAGIC); - AN(perms); - if (pwd != NULL) - perms->uid = pwd->pw_uid; - else - perms->uid = (uid_t) -1; - if (grp != NULL) - perms->gid = grp->gr_gid; - else - perms->gid = (gid_t) -1; - perms->mode = mode; - la->perms = perms; - } - else - AZ(la->perms); - - if (VUS_is(la->endpoint)) - error = VUS_resolver(av[1], mac_uds, la, &err); - else - error = VSS_resolver_range(av[1], "80", mac_tcp, la, &err); - - if (error) - ARGV_ERR("Got no socket(s) for %s (%s)\n", av[1], err); - else if (VTAILQ_EMPTY(&la->socks)) - ARGV_ERR("Got no socket(s) for %s\n", av[1]); - VAV_Free(av); -} diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 4ab09253c..eadd65f0c 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -47,6 +47,8 @@ #include #include "mgt.h" +#include "acceptor/cache_acceptor.h" +#include "acceptor/mgt_acceptor.h" #include "vapi/vsig.h" @@ -616,7 +618,7 @@ mgt_reap_child(void) /* XXX number of retries? interval? */ for (i = 0; i < 3; i++) { - if (MAC_reopen_sockets() == 0) + if (VCA_reopen_sockets() == 0) break; /* error already logged */ (void)sleep(1); diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index d3eb09a90..8a9143285 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -47,6 +47,8 @@ #include #include "mgt/mgt.h" +#include "acceptor/cache_acceptor.h" +#include "acceptor/mgt_acceptor.h" #include "common/heritage.h" #include "hash/hash_slinger.h" @@ -774,6 +776,7 @@ main(int argc, char * const *argv) /* Various initializations */ VTAILQ_INIT(&heritage.socks); + VCA_Config(); mgt_evb = VEV_New(); AN(mgt_evb); @@ -792,7 +795,7 @@ main(int argc, char * const *argv) VTAILQ_FOREACH(alp, &arglist, list) { switch(alp->arg[0]) { case 'a': - MAC_Arg(alp->val); + VCA_Arg(alp->val); break; case 'f': if (*alp->val != '\0') @@ -945,7 +948,7 @@ main(int argc, char * const *argv) alp->priv = create_pid_file(&pid, "%s/_.pid", workdir); if (VTAILQ_EMPTY(&heritage.socks)) - MAC_Arg(":80\0"); // XXX: extra NUL for FlexeLint + VCA_Arg(":80\0"); // XXX: extra NUL for FlexeLint assert(!VTAILQ_EMPTY(&heritage.socks)); diff --git a/include/tbl/locks.h b/include/tbl/locks.h index 3de8e33e6..b20644ee8 100644 --- a/include/tbl/locks.h +++ b/include/tbl/locks.h @@ -46,7 +46,7 @@ LOCK(probe) LOCK(sess) LOCK(conn_pool) LOCK(vbe) -LOCK(vcapace) +LOCK(accpace) LOCK(vcl) LOCK(vxid) LOCK(waiter) From nils.goroll at uplex.de Mon Sep 30 15:53:07 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 15:53:07 +0000 (UTC) Subject: [master] f52e7c3fd acceptor: Give open functions only the options part of av Message-ID: <20240930155307.B5DA811D114@lists.varnish-cache.org> commit f52e7c3fd2b8b85a8e919ada8c14a521d05d1143 Author: Nils Goroll Date: Thu Sep 5 15:21:43 2024 +0200 acceptor: Give open functions only the options part of av av[0] is already passed as la->name, av[1] as la->endpoint diff --git a/bin/varnishd/acceptor/mgt_acceptor.c b/bin/varnishd/acceptor/mgt_acceptor.c index 8286ae71b..38ba7ac99 100644 --- a/bin/varnishd/acceptor/mgt_acceptor.c +++ b/bin/varnishd/acceptor/mgt_acceptor.c @@ -147,7 +147,7 @@ VCA_Arg(const char *spec) vca = VCA_Find("tcp"); AN(vca); - error = vca->open(av, la, &err); + error = vca->open(av + 2, la, &err); if (error) ARGV_ERR("Got no socket(s) for %s (%s)\n", av[1], err); diff --git a/bin/varnishd/acceptor/mgt_acceptor_tcp.c b/bin/varnishd/acceptor/mgt_acceptor_tcp.c index 364b56d8f..3b41ef140 100644 --- a/bin/varnishd/acceptor/mgt_acceptor_tcp.c +++ b/bin/varnishd/acceptor/mgt_acceptor_tcp.c @@ -176,7 +176,7 @@ vca_tcp_open(char **av, struct listen_arg *la, const char **err) ARGV_ERR("Unix domain socket addresses must be" " absolute paths in -a (%s)\n", la->endpoint); - for (int i = 2; av[i] != NULL; i++) { + for (int i = 0; av[i] != NULL; i++) { if (strchr(av[i], '=') == NULL) { if (xp != NULL) ARGV_ERR("Too many protocol sub-args" @@ -196,7 +196,7 @@ vca_tcp_open(char **av, struct listen_arg *la, const char **err) AN(xp); la->transport = xp; - return (VSS_resolver_range(av[1], "80", vca_tcp_open_cb, la, err)); + return (VSS_resolver_range(la->endpoint, "80", vca_tcp_open_cb, la, err)); } int diff --git a/bin/varnishd/acceptor/mgt_acceptor_uds.c b/bin/varnishd/acceptor/mgt_acceptor_uds.c index 97635b712..325268797 100644 --- a/bin/varnishd/acceptor/mgt_acceptor_uds.c +++ b/bin/varnishd/acceptor/mgt_acceptor_uds.c @@ -181,7 +181,7 @@ vca_uds_open(char **av, struct listen_arg *la, const char **err) heritage.min_vcl_version = vmax(heritage.min_vcl_version, 41U); - for (int i = 2; av[i] != NULL; i++) { + for (int i = 0; av[i] != NULL; i++) { char *eq, *val; int len; From nils.goroll at uplex.de Mon Sep 30 15:53:07 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 15:53:07 +0000 (UTC) Subject: [master] 992614831 acceptor: Improve error reporting Message-ID: <20240930155307.F17BE11D11D@lists.varnish-cache.org> commit 99261483189d462c4609cc9df0a766fb972028b0 Author: Nils Goroll Date: Thu Sep 5 15:32:21 2024 +0200 acceptor: Improve error reporting Include the socket name and make the code more readable by using struct members instead of av[] diff --git a/bin/varnishd/acceptor/mgt_acceptor.c b/bin/varnishd/acceptor/mgt_acceptor.c index 38ba7ac99..11414f1f7 100644 --- a/bin/varnishd/acceptor/mgt_acceptor.c +++ b/bin/varnishd/acceptor/mgt_acceptor.c @@ -149,10 +149,14 @@ VCA_Arg(const char *spec) AN(vca); error = vca->open(av + 2, la, &err); - if (error) - ARGV_ERR("Got no socket(s) for %s (%s)\n", av[1], err); - else if (VTAILQ_EMPTY(&la->socks)) - ARGV_ERR("Got no socket(s) for %s\n", av[1]); + if (error) { + ARGV_ERR("Got no socket(s) for %s=%s (%s)\n", + la->name, la->endpoint, err); + } + else if (VTAILQ_EMPTY(&la->socks)) { + ARGV_ERR("Got no socket(s) for %s=%s\n", + la->name, la->endpoint); + } VAV_Free(av); } diff --git a/bin/varnishtest/tests/c00121.vtc b/bin/varnishtest/tests/c00121.vtc index 8aed31ffa..29812c91f 100644 --- a/bin/varnishtest/tests/c00121.vtc +++ b/bin/varnishtest/tests/c00121.vtc @@ -87,5 +87,5 @@ varnish v1 -errvcl {Backend path: The empty abstract socket name is not supporte } } -shell -err -expect "Error: Got no socket(s) for @ (The empty abstract socket name is not supported)" \ +shell -err -expect "Error: Got no socket(s) for a0=@ (The empty abstract socket name is not supported)" \ "varnishd -n ${tmpdir}/v0 -a @ -b None" From nils.goroll at uplex.de Mon Sep 30 16:02:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 16:02:05 +0000 (UTC) Subject: [master] 8b9b06d7b Overlooked one bit during re-rename Message-ID: <20240930160205.6848111E4BB@lists.varnish-cache.org> commit 8b9b06d7b066a0441398228c0ce79aa1a28c224a Author: Nils Goroll Date: Mon Sep 30 18:00:29 2024 +0200 Overlooked one bit during re-rename diff --git a/include/tbl/locks.h b/include/tbl/locks.h index b20644ee8..3de8e33e6 100644 --- a/include/tbl/locks.h +++ b/include/tbl/locks.h @@ -46,7 +46,7 @@ LOCK(probe) LOCK(sess) LOCK(conn_pool) LOCK(vbe) -LOCK(accpace) +LOCK(vcapace) LOCK(vcl) LOCK(vxid) LOCK(waiter) From nils.goroll at uplex.de Mon Sep 30 16:10:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 16:10:05 +0000 (UTC) Subject: [master] ec46847a6 tell flexelint about the new world order Message-ID: <20240930161005.2799F11EA81@lists.varnish-cache.org> commit ec46847a6fac59306127ab991d1c93d602359362 Author: Nils Goroll Date: Mon Sep 30 18:09:44 2024 +0200 tell flexelint about the new world order diff --git a/bin/varnishd/flint.sh b/bin/varnishd/flint.sh index dde8ac8de..b76ee681b 100644 --- a/bin/varnishd/flint.sh +++ b/bin/varnishd/flint.sh @@ -11,6 +11,7 @@ FLOPS=' -DVARNISH_STATE_DIR="foo" -DVARNISH_VMOD_DIR="foo" -DVARNISH_VCL_DIR="foo" + acceptor/*.c cache/*.c common/*.c hash/*.c From nils.goroll at uplex.de Mon Sep 30 16:15:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 16:15:05 +0000 (UTC) Subject: [master] 563bdd941 sml: Delay freeing of trimmed segments always Message-ID: <20240930161505.C06B511EE5C@lists.varnish-cache.org> commit 563bdd94102421c8c2023638dc0d920603e88c53 Author: Nils Goroll Date: Thu Sep 26 11:53:04 2024 +0200 sml: Delay freeing of trimmed segments always sml_trimstore() already delays freeing of a replaced last, smaller segment for the reallocation case at the bottom of the function: Because a concurrently running iterator might have already taken a reference on a to-be-replaced segment, it can not be freed immediately, but rather is kept around until the busy object is no more. This trivial change applies the same also for a segment which turns out to be unneeded because writing the object ended with a zero length in this segment. Simple, but consequential, see next commit diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c index 9c720aeb4..74657365a 100644 --- a/bin/varnishd/storage/storage_simple.c +++ b/bin/varnishd/storage/storage_simple.c @@ -560,7 +560,8 @@ sml_trimstore(struct worker *wrk, struct objcore *oc) Lck_Lock(&oc->boc->mtx); VTAILQ_REMOVE(&o->list, st, list); Lck_Unlock(&oc->boc->mtx); - sml_stv_free(stv, st); + /* sml_bocdone frees this */ + oc->boc->stevedore_priv = st; return; } From nils.goroll at uplex.de Mon Sep 30 16:15:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 16:15:05 +0000 (UTC) Subject: [master] 5c2a682aa sml: Optimize freebehind (transient) memory usage Message-ID: <20240930161505.DE09E11EE5F@lists.varnish-cache.org> commit 5c2a682aa39af1c61e860f3a62a204d200ac3211 Author: Nils Goroll Date: Thu Sep 26 12:06:03 2024 +0200 sml: Optimize freebehind (transient) memory usage Previously, sml_iterator() could not free the last segment while iterating, because it could have turned out to be over-allocated at any time and be removed. With the change from the previous commit, we can now lift this limitation and take a reference to also the last segment. This is relevant because, due to of the race between the fetch and iterator site of a transient operation, the "is last segment" condition can happen at any point while an object is busy. diff --git a/bin/varnishd/storage/storage_simple.c b/bin/varnishd/storage/storage_simple.c index 74657365a..53eb305de 100644 --- a/bin/varnishd/storage/storage_simple.c +++ b/bin/varnishd/storage/storage_simple.c @@ -395,15 +395,14 @@ sml_iterator(struct worker *wrk, struct objcore *oc, assert(nl > 0); sl += st->len; st = VTAILQ_PREV(st, storagehead, list); - if (VTAILQ_PREV(st, storagehead, list) != NULL) { - if (final && checkpoint != NULL) { - VTAILQ_REMOVE(&obj->list, - checkpoint, list); - sml_stv_free(stv, checkpoint); - } - checkpoint = st; - checkpoint_len = sl; + if (final && checkpoint != NULL) { + VTAILQ_REMOVE(&obj->list, checkpoint, list); + if (checkpoint == boc->stevedore_priv) + boc->stevedore_priv = trim_once; + sml_stv_free(stv, checkpoint); } + checkpoint = st; + checkpoint_len = sl; } CHECK_OBJ_NOTNULL(obj, OBJECT_MAGIC); CHECK_OBJ_NOTNULL(st, STORAGE_MAGIC); From nils.goroll at uplex.de Mon Sep 30 16:54:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 16:54:05 +0000 (UTC) Subject: [master] e142af75d cache_backend: set the backend happy bitfield to all-one if no probe Message-ID: <20240930165405.1842B120949@lists.varnish-cache.org> commit e142af75decb675a554434a48e65e55bd0267f0c Author: Nils Goroll Date: Mon Sep 30 18:46:31 2024 +0200 cache_backend: set the backend happy bitfield to all-one if no probe Fixes #4201 diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 977933772..d3a98ac56 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -893,6 +893,7 @@ VRT_new_backend_clustered(VRT_CTX, struct vsmw_cluster *vc, m = vbe_methods; } else { be->sick = 0; + be->vsc->happy = UINT64_MAX; m = vbe_methods_noprobe; } diff --git a/bin/varnishtest/tests/v00003.vtc b/bin/varnishtest/tests/v00003.vtc index 21fa9607e..b89680d74 100644 --- a/bin/varnishtest/tests/v00003.vtc +++ b/bin/varnishtest/tests/v00003.vtc @@ -17,10 +17,18 @@ varnish v1 -arg "-p vcl_cooldown=1" \ .port = "${s1_port}"; .probe = { .interval = 1s; } } + backend noprobe { + .host = "${s1_addr}"; + .port = "${s1_port}"; + } + sub vcl_backend_fetch { + set bereq.backend = noprobe; + } } -start # We only have one vcl yet varnish v1 -expect VBE.vcl1.default.happy >= 0 +varnish v1 -expect VBE.vcl1.noprobe.happy > 9223372036854775808 varnish v1 -expect !VBE.vcl2.default.happy varnish v1 -cliok "backend.list -p *.*" From nils.goroll at uplex.de Mon Sep 30 20:27:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 20:27:05 +0000 (UTC) Subject: [master] 7dbee0473 Flexelint the new acceptor sources Message-ID: <20240930202705.5DF5F62639@lists.varnish-cache.org> commit 7dbee04736000a085f124626170c776a0e5c48f2 Author: Nils Goroll Date: Mon Sep 30 21:23:29 2024 +0200 Flexelint the new acceptor sources diff --git a/bin/varnishd/acceptor/acceptor_priv.h b/bin/varnishd/acceptor/acceptor_priv.h index 727e07d1b..4f18d23c5 100644 --- a/bin/varnishd/acceptor/acceptor_priv.h +++ b/bin/varnishd/acceptor/acceptor_priv.h @@ -61,3 +61,24 @@ struct conn_heritage { void vca_pace_check(void); void vca_pace_bad(void); void vca_pace_good(void); + + +union sock_arg { + struct linger lg; + struct timeval tv; + int i; +}; + +struct sock_opt { + int level; + int optname; + const char *strname; + unsigned mod; + socklen_t sz; + union sock_arg arg[1]; +}; + +#define SOCK_OPT(lvl, nam, typ) { lvl, nam, #nam, 1, sizeof(typ) }, + +// move to individual pools? +extern unsigned pool_accepting; // cache_acceptor.c diff --git a/bin/varnishd/acceptor/cache_acceptor.c b/bin/varnishd/acceptor/cache_acceptor.c index 375edb074..ca08665c9 100644 --- a/bin/varnishd/acceptor/cache_acceptor.c +++ b/bin/varnishd/acceptor/cache_acceptor.c @@ -48,14 +48,13 @@ #include "common/heritage.h" #include "vcli_serve.h" -#include "vsa.h" -#include "vtcp.h" #include "vtim.h" -static pthread_t VCA_thread; -vtim_dur vca_pace = 0.0; -struct lock pace_mtx; unsigned pool_accepting; + +static pthread_t VCA_thread; +static vtim_dur vca_pace = 0.0; +static struct lock pace_mtx; static pthread_mutex_t shut_mtx = PTHREAD_MUTEX_INITIALIZER; /*-------------------------------------------------------------------- diff --git a/bin/varnishd/acceptor/cache_acceptor_tcp.c b/bin/varnishd/acceptor/cache_acceptor_tcp.c index f33938405..66f15b8e7 100644 --- a/bin/varnishd/acceptor/cache_acceptor_tcp.c +++ b/bin/varnishd/acceptor/cache_acceptor_tcp.c @@ -51,31 +51,13 @@ #include "vtcp.h" #include "vtim.h" -extern vtim_dur vca_pace; -extern struct lock pace_mtx; -extern unsigned pool_accepting; - /*-------------------------------------------------------------------- * TCP options we want to control */ -union sock_arg { - struct linger lg; - struct timeval tv; - int i; -}; - -static struct sock_opt { - int level; - int optname; - const char *strname; - unsigned mod; - socklen_t sz; - union sock_arg arg[1]; -} sock_opts[] = { +static struct sock_opt sock_opts[] = { /* Note: Setting the mod counter to something not-zero is needed * to force the setsockopt() calls on startup */ -#define SOCK_OPT(lvl, nam, typ) { lvl, nam, #nam, 1, sizeof(typ) }, SOCK_OPT(SOL_SOCKET, SO_LINGER, struct linger) SOCK_OPT(SOL_SOCKET, SO_KEEPALIVE, int) @@ -332,6 +314,7 @@ vca_tcp_event(struct cli *cli, struct listen_sock *ls, enum vca_event event) { char h[VTCP_ADDRBUFSIZE], p[VTCP_PORTBUFSIZE]; + (void) ls; // XXX const? switch (event) { case VCA_EVENT_LADDR: VTCP_myname(ls->sock, h, sizeof h, p, sizeof p); diff --git a/bin/varnishd/acceptor/cache_acceptor_uds.c b/bin/varnishd/acceptor/cache_acceptor_uds.c index 95515fc43..513d5161e 100644 --- a/bin/varnishd/acceptor/cache_acceptor_uds.c +++ b/bin/varnishd/acceptor/cache_acceptor_uds.c @@ -51,10 +51,6 @@ #include "vtcp.h" #include "vtim.h" -extern vtim_dur vca_pace; -extern struct lock pace_mtx; -extern unsigned pool_accepting; - /*-------------------------------------------------------------------- * We want to get out of any kind of trouble-hit TCP connections as fast * as absolutely possible, so we set them LINGER disabled, so that even if @@ -75,23 +71,9 @@ static const unsigned enable_so_keepalive = 1; * UDS options we want to control */ -union sock_arg { - struct linger lg; - struct timeval tv; - int i; -}; - -static struct sock_opt { - int level; - int optname; - const char *strname; - unsigned mod; - socklen_t sz; - union sock_arg arg[1]; -} sock_opts[] = { +static struct sock_opt sock_opts[] = { /* Note: Setting the mod counter to something not-zero is needed * to force the setsockopt() calls on startup */ -#define SOCK_OPT(lvl, nam, typ) { lvl, nam, #nam, 1, sizeof(typ) }, SOCK_OPT(SOL_SOCKET, SO_LINGER, struct linger) SOCK_OPT(SOL_SOCKET, SO_KEEPALIVE, int) @@ -299,6 +281,7 @@ static void vca_uds_event(struct cli *cli, struct listen_sock *ls, enum vca_event event) { + (void) ls; // XXX const? switch (event) { case VCA_EVENT_LADDR: CHECK_OBJ_NOTNULL(ls, LISTEN_SOCK_MAGIC); diff --git a/bin/varnishd/acceptor/mgt_acceptor.c b/bin/varnishd/acceptor/mgt_acceptor.c index 11414f1f7..6984dd7d6 100644 --- a/bin/varnishd/acceptor/mgt_acceptor.c +++ b/bin/varnishd/acceptor/mgt_acceptor.c @@ -48,10 +48,6 @@ #include "common/heritage.h" #include "vav.h" -#include "vcli_serve.h" -#include "vsa.h" -#include "vss.h" -#include "vtcp.h" #include "vus.h" static VTAILQ_HEAD(,listen_arg) listen_args = diff --git a/bin/varnishd/acceptor/mgt_acceptor_tcp.c b/bin/varnishd/acceptor/mgt_acceptor_tcp.c index 3b41ef140..e7669b5f6 100644 --- a/bin/varnishd/acceptor/mgt_acceptor_tcp.c +++ b/bin/varnishd/acceptor/mgt_acceptor_tcp.c @@ -49,8 +49,6 @@ #include "acceptor/acceptor_tcp.h" #include "acceptor/mgt_acceptor.h" -#include "vav.h" -#include "vcli_serve.h" #include "vsa.h" #include "vss.h" #include "vtcp.h" diff --git a/bin/varnishd/acceptor/mgt_acceptor_uds.c b/bin/varnishd/acceptor/mgt_acceptor_uds.c index 325268797..c10fa7af6 100644 --- a/bin/varnishd/acceptor/mgt_acceptor_uds.c +++ b/bin/varnishd/acceptor/mgt_acceptor_uds.c @@ -52,10 +52,7 @@ #include "acceptor/acceptor_uds.h" #include "acceptor/mgt_acceptor.h" -#include "vav.h" -#include "vcli_serve.h" #include "vsa.h" -#include "vss.h" #include "vus.h" int From nils.goroll at uplex.de Mon Sep 30 20:36:05 2024 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Sep 2024 20:36:05 +0000 (UTC) Subject: [master] 508306fd0 Last include cleanups for today (Flexelinting) Message-ID: <20240930203605.98E3262D35@lists.varnish-cache.org> commit 508306fd06341c6e960353680245b373a8f406e9 Author: Nils Goroll Date: Mon Sep 30 22:35:46 2024 +0200 Last include cleanups for today (Flexelinting) diff --git a/bin/varnishd/acceptor/mgt_acceptor_tcp.c b/bin/varnishd/acceptor/mgt_acceptor_tcp.c index e7669b5f6..1ebb4f8e5 100644 --- a/bin/varnishd/acceptor/mgt_acceptor_tcp.c +++ b/bin/varnishd/acceptor/mgt_acceptor_tcp.c @@ -35,7 +35,6 @@ #include #include -#include #include #include #include diff --git a/bin/varnishd/storage/storage_persistent_subr.c b/bin/varnishd/storage/storage_persistent_subr.c index 09acea821..9e57f26fa 100644 --- a/bin/varnishd/storage/storage_persistent_subr.c +++ b/bin/varnishd/storage/storage_persistent_subr.c @@ -38,7 +38,6 @@ #include "config.h" #include "cache/cache_varnishd.h" -#include "common/heritage.h" #include