From phk at FreeBSD.org Mon Oct 2 08:49:08 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 2 Oct 2017 08:49:08 +0000 (UTC) Subject: [master] 1291533 Move more of the sharp knives away from $ABI VRT Message-ID: <20171002084908.E134162FB1@lists.varnish-cache.org> commit 1291533566d8def6298e6e8d37d05f5113ad8b3b Author: Poul-Henning Kamp Date: Mon Oct 2 08:36:20 2017 +0000 Move more of the sharp knives away from $ABI VRT diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index a0586c5..e67f89f 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -44,14 +44,6 @@ /*--------------------------------------------------------------------*/ -enum req_fsm_nxt { - REQ_FSM_MORE, - REQ_FSM_DONE, - REQ_FSM_DISEMBARK, -}; - -/*--------------------------------------------------------------------*/ - enum body_status { #define BODYSTATUS(U,l) BS_##U, #include "tbl/body_status.h" @@ -877,26 +869,7 @@ ssize_t VRB_Cache(struct req *, ssize_t maxsize); ssize_t VRB_Iterate(struct req *, objiterate_f *func, void *priv); void VRB_Free(struct req *); -/* cache_req_fsm.c [CNT] */ -enum req_fsm_nxt CNT_Request(struct worker *, struct req *); - /* cache_session.c [SES] */ -void SES_Wait(struct sess *, const struct transport *); -void SES_Ref(struct sess *sp); -void SES_Rel(struct sess *sp); -int SES_Reschedule_Req(struct req *); - -enum htc_status_e { - HTC_S_JUNK = -5, - HTC_S_CLOSE = -4, - HTC_S_TIMEOUT = -3, - HTC_S_OVERFLOW = -2, - HTC_S_EOF = -1, - HTC_S_EMPTY = 0, - HTC_S_MORE = 1, - HTC_S_COMPLETE = 2, - HTC_S_IDLE = 3, -}; void HTC_RxInit(struct http_conn *htc, struct ws *ws); void HTC_RxPipeline(struct http_conn *htc, void *); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index ca00a44..d81edcf 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -156,9 +156,35 @@ int Pool_Task_Any(struct pool_task *task, enum task_prio prio); /* cache_range.c [VRG] */ void VRG_dorange(struct req *req, const char *r); +/* cache_req_fsm.c [CNT] */ + +enum req_fsm_nxt { + REQ_FSM_MORE, + REQ_FSM_DONE, + REQ_FSM_DISEMBARK, +}; + +enum req_fsm_nxt CNT_Request(struct worker *, struct req *); + /* cache_session.c */ void SES_NewPool(struct pool *, unsigned pool_no); void SES_DestroyPool(struct pool *); +void SES_Wait(struct sess *, const struct transport *); +void SES_Ref(struct sess *sp); +void SES_Rel(struct sess *sp); +int SES_Reschedule_Req(struct req *); + +enum htc_status_e { + HTC_S_JUNK = -5, + HTC_S_CLOSE = -4, + HTC_S_TIMEOUT = -3, + HTC_S_OVERFLOW = -2, + HTC_S_EOF = -1, + HTC_S_EMPTY = 0, + HTC_S_MORE = 1, + HTC_S_COMPLETE = 2, + HTC_S_IDLE = 3, +}; /* cache_shmlog.c */ extern struct VSC_main *VSC_C_main; From daghf at varnish-software.com Mon Oct 2 09:10:07 2017 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Mon, 2 Oct 2017 09:10:07 +0000 (UTC) Subject: [master] f0d3be8 Don't mark a stream as scheduled when Pool_Task fails Message-ID: <20171002091007.C9F6162760@lists.varnish-cache.org> commit f0d3be819c5c8e1a2ccf88e8c1df4100a73c70a6 Author: Dag Haavi Finstad Date: Mon Oct 2 11:03:25 2017 +0200 Don't mark a stream as scheduled when Pool_Task fails diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 8828593..3c50d20 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -532,8 +532,10 @@ h2_end_headers(struct worker *wrk, const struct h2_sess *h2, req->task.func = h2_do_req; req->task.priv = req; r2->scheduled = 1; - if (Pool_Task(wrk->pool, &req->task, TASK_QUEUE_STR) != 0) + if (Pool_Task(wrk->pool, &req->task, TASK_QUEUE_STR) != 0) { + r2->scheduled = 0; return (H2SE_REFUSED_STREAM); //rfc7540,l,3326,3329 + } return (0); } diff --git a/bin/varnishtest/tests/t02011.vtc b/bin/varnishtest/tests/t02011.vtc index fd698ce..9c2269e 100644 --- a/bin/varnishtest/tests/t02011.vtc +++ b/bin/varnishtest/tests/t02011.vtc @@ -71,6 +71,9 @@ client c1 { # trigger an update of the stats varnish v1 -cliok "param.set thread_pool_min 3" delay 1 +varnish v1 -vsl_catchup varnish v1 -expect sess_drop == 0 varnish v1 -expect sess_dropped == 0 varnish v1 -expect req_dropped == 1 +varnish v1 -expect MEMPOOL.req0.live == 0 +varnish v1 -expect MEMPOOL.sess0.live == 0 From phk at FreeBSD.org Mon Oct 2 09:23:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 2 Oct 2017 09:23:05 +0000 (UTC) Subject: [master] a51affc Whitespace OCD Message-ID: <20171002092305.7F34962E30@lists.varnish-cache.org> commit a51affc8e2ae3b04376412e3a278b2d1f52365d2 Author: Poul-Henning Kamp Date: Mon Oct 2 08:49:12 2017 +0000 Whitespace OCD diff --git a/bin/varnishd/storage/storage_umem.c b/bin/varnishd/storage/storage_umem.c index bb4ccdb..791a6e5 100644 --- a/bin/varnishd/storage/storage_umem.c +++ b/bin/varnishd/storage/storage_umem.c @@ -268,9 +268,9 @@ smu_open(struct stevedore *st) 0, // align smu_smu_constructor, smu_smu_destructor, - NULL, // reclaim + NULL, // reclaim smu_sc, // callback_data - NULL, // source + NULL, // source 0 // cflags ); AN(smu_sc->smu_cache); From nils.goroll at uplex.de Mon Oct 2 13:41:07 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 2 Oct 2017 13:41:07 +0000 (UTC) Subject: [master] 8633ce1 Fix an off-by-one in the max_restarts check Message-ID: <20171002134107.7CAB76331B@lists.varnish-cache.org> commit 8633ce15f63370e619b89365421ae1c750505e6c Author: Nils Goroll Date: Mon Oct 2 14:47:43 2017 +0200 Fix an off-by-one in the max_restarts check max_restarts is the number of allowed restarts Related to #2405 diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 987f3d0..e36e3aa 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -727,7 +727,7 @@ cnt_restart(struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); req->director_hint = NULL; - if (++req->restarts >= cache_param->max_restarts) { + if (++req->restarts > cache_param->max_restarts) { VSLb(req->vsl, SLT_VCL_Error, "Too many restarts"); req->err_code = 503; req->req_step = R_STP_SYNTH; diff --git a/bin/varnishtest/tests/b00019.vtc b/bin/varnishtest/tests/b00019.vtc index 33d3dbc..5336478 100644 --- a/bin/varnishtest/tests/b00019.vtc +++ b/bin/varnishtest/tests/b00019.vtc @@ -26,15 +26,17 @@ varnish v1 -vcl+backend { } sub vcl_synth { - if (req.restarts == 2) { + # when we end up here, we have _exceeded_ the number + # allowed restarts + if (req.restarts == 3) { set resp.status = 200; - set resp.reason = "restart=2"; - } elsif (req.restarts > 2) { + set resp.reason = "restart=3"; + } elsif (req.restarts > 3) { set resp.status = 501; - set resp.reason = "restart>2"; - } elsif (req.restarts < 2) { + set resp.reason = "restart>3"; + } elsif (req.restarts < 3) { set resp.status = 500; - set resp.reason = "restart<2"; + set resp.reason = "restart<3"; } } } -start diff --git a/doc/changes.rst b/doc/changes.rst index 9b546b1..c1e45b8 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -2,6 +2,10 @@ Varnish Cache Trunk (ongoing) ============================= +* Fixed implementation of the ``max_restarts`` limit: It used to be one + less than the number of allowed restarts, it now is the number of + ``return(restart)``s per request. + ================================ Varnish Cache 5.2.0 (2017-09-15) ================================ From nils.goroll at uplex.de Mon Oct 2 13:41:07 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 2 Oct 2017 13:41:07 +0000 (UTC) Subject: [master] 12e1dcd Turn an impossible condition into an assertion Message-ID: <20171002134107.9A47B6331E@lists.varnish-cache.org> commit 12e1dcd12b8c2da3636ae8a5ceba5faae31c24ef Author: Nils Goroll Date: Mon Oct 2 13:59:50 2017 +0200 Turn an impossible condition into an assertion req->restarts only gets incremented in cnt_restart, so it will never exceed max_restarts here. Related to #2405 diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index e36e3aa..217192b 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -160,9 +160,7 @@ cnt_deliver(struct worker *wrk, struct req *req) VCL_deliver_method(req->vcl, wrk, req, NULL, NULL); VSLb_ts_req(req, "Process", W_TIM_real(wrk)); - /* Stop the insanity before it turns "Hotel California" on us */ - if (req->restarts >= cache_param->max_restarts) - wrk->handling = VCL_RET_DELIVER; + assert(req->restarts <= cache_param->max_restarts); if (wrk->handling != VCL_RET_DELIVER) { (void)HSH_DerefObjCore(wrk, &req->objcore, HSH_RUSH_POLICY); From nils.goroll at uplex.de Mon Oct 2 13:41:07 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 2 Oct 2017 13:41:07 +0000 (UTC) Subject: [master] d2850ab Remove a now obsolete piece of advise Message-ID: <20171002134107.C802E63322@lists.varnish-cache.org> commit d2850abbc87a6a69ebd3201179d02b57ee575995 Author: Nils Goroll Date: Mon Oct 2 14:27:02 2017 +0200 Remove a now obsolete piece of advise Since we decoupled the client and backend side, restarts are completely unrelated to backend requests. Related to #2405 diff --git a/include/tbl/params.h b/include/tbl/params.h index 2a2bbf3..6560666 100644 --- a/include/tbl/params.h +++ b/include/tbl/params.h @@ -786,9 +786,7 @@ PARAM( /* units */ "restarts", /* flags */ 0, /* s-text */ - "Upper limit on how many times a request can restart.\n" - "Be aware that restarts are likely to cause a hit against the " - "backend, so don't increase thoughtlessly.", + "Upper limit on how many times a request can restart.", /* l-text */ "", /* func */ NULL ) From nils.goroll at uplex.de Mon Oct 2 19:16:06 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 2 Oct 2017 19:16:06 +0000 (UTC) Subject: [master] 0b0a8e8 consolidate return keyword documentation and add missing keywords Message-ID: <20171002191606.B663F63818@lists.varnish-cache.org> commit 0b0a8e8bc75f8db85670e3a653a2ef725fbdff1a Author: Nils Goroll Date: Mon Oct 2 19:03:45 2017 +0200 consolidate return keyword documentation and add missing keywords Remove duplication to some extent by moving common return keywords to a separate paragraph. Add fail and vcl(label) keywords Fix some typos diff --git a/doc/sphinx/users-guide/vcl-built-in-subs.rst b/doc/sphinx/users-guide/vcl-built-in-subs.rst index 941f0b8..14a740d 100644 --- a/doc/sphinx/users-guide/vcl-built-in-subs.rst +++ b/doc/sphinx/users-guide/vcl-built-in-subs.rst @@ -7,9 +7,58 @@ Built in subroutines Various built-in subroutines are called during processing of client- and backend requests as well as upon ``vcl.load`` and ``vcl.discard``. -See :ref:`reference-states` for a defailed graphical overview of the +See :ref:`reference-states` for a detailed graphical overview of the states and how they relate to core code functions and VCL subroutines. +Subroutines always terminate with a ``return()`` on a keyword, which +determines how processing continues in the request processing state +machine. + +The behaviour for ``return()`` keywords is identical or at least +similar across subroutines, so differences are only documented where +relevant. + +common return keywords +~~~~~~~~~~~~~~~~~~~~~~ + +.. _fail: + + ``fail`` + Transition to :ref:`vcl_synth` on the client side as for + ``return(synth(503, "VCL Failed"))``, but with any request state + changes undone as if ``std.rollback()`` was called and forcing a + connection close. + + Intended for fatal errors, for which only minimal error handling is + possible. + +.. _synth: + + ``synth(status code, reason)`` + Transition to :ref:`vcl_synth` with ``resp.status`` and + ``resp.reason`` being preset to the arguments of ``synth()``. + +.. _pass: + + ``pass`` + Switch to pass mode. Control will eventually pass to + :ref:`vcl_pass`. + +.. _pipe: + + ``pipe`` + Switch to pipe mode. Control will eventually pass to + :ref:`vcl_pipe`. + +.. _restart: + + ``restart`` + Restart the transaction. Increases the ``req.restarts`` counter. + + If the number of restarts is higher than the *max_restarts* + parameter, control is passed to :ref:`vcl_synth` as for + ``return(synth(503, "Too many restarts"))`` + ----------- client side ----------- @@ -30,15 +79,17 @@ be set as a default for the backend processing side. The `vcl_recv` subroutine may terminate with calling ``return()`` on one of the following keywords: + ``fail`` + see `fail`_ + ``synth(status code, reason)`` - Transition to :ref:`vcl_synth` with ``resp.status`` and - ``resp.reason`` being preset to the arguments of ``synth()``. + see `synth`_ ``pass`` - Switch to pass mode. Control will eventually pass to :ref:`vcl_pass`. + see `pass`_ ``pipe`` - Switch to pipe mode. Control will eventually pass to :ref:`vcl_pipe`. + see `pipe`_ ``hash`` Continue processing the object as a potential candidate for @@ -46,7 +97,13 @@ of the following keywords: ``purge`` Purge the object and it's variants. Control passes through - vcl_hash to vcl_purge. + :ref:`vcl_hash` to :ref:`vcl_purge`. + + ``vcl(label)`` + Switch to vcl labelled *label*. This will continue vcl processing + in this vcl's :ref:`vcl_recv` as if it was the active vcl. + + See the :ref:`varnishadm(1)` ``vcl.label`` command. .. _vcl_pipe: @@ -63,9 +120,11 @@ other VCL subroutine will ever get called after `vcl_pipe`. The `vcl_pipe` subroutine may terminate with calling ``return()`` with one of the following keywords: + ``fail`` + see `fail`_ + ``synth(status code, reason)`` - Transition to :ref:`vcl_synth` with ``resp.status`` and - ``resp.reason`` being preset to the arguments of ``synth()``. + see `synth`_ ``pipe`` Proceed with pipe mode. @@ -83,14 +142,14 @@ submitted over the same client connection are handled normally. The `vcl_pass` subroutine may terminate with calling ``return()`` with one of the following keywords: + ``fail`` + see `fail`_ + ``synth(status code, reason)`` - Transition to :ref:`vcl_synth` with ``resp.status`` and - ``resp.reason`` being preset to the arguments of ``synth()``. + see `synth`_ ``restart`` - Restart the transaction. Increases the restart counter. If the number - of restarts is higher than *max_restarts* Varnish emits a guru meditation - error. + see `restart`_ ``fetch`` Proceed with pass mode - initiate a backend request. @@ -103,14 +162,20 @@ vcl_hash Called after `vcl_recv` to create a hash value for the request. This is used as a key to look up the object in Varnish. -The `vcl_hash` subroutine may only terminate with calling ``return(lookup)``: +The `vcl_hash` subroutine may terminate with calling ``return()`` with one +of the following keywords: + + ``fail`` + see `fail`_ ``lookup`` Look up the object in cache. + Control passes to :ref:`vcl_purge` when coming from a ``purge`` return in `vcl_recv`. - Otherwise control passes to the next subroutine depending on the result of - the cache lookup: + + Otherwise control passes to the next subroutine depending on the + result of the cache lookup: * a hit: pass to :ref:`vcl_hit` @@ -126,19 +191,19 @@ The `vcl_hash` subroutine may only terminate with calling ``return(lookup)``: vcl_purge ~~~~~~~~~ -Called after the purge has been executed and all its variants have been evited. +Called after the purge has been executed and all its variants have been evicted. The `vcl_purge` subroutine may terminate with calling ``return()`` with one of the following keywords: + ``fail`` + see `fail`_ + ``synth(status code, reason)`` - Transition to :ref:`vcl_synth` with ``resp.status`` and - ``resp.reason`` being preset to the arguments of ``synth()``. + see `synth`_ ``restart`` - Restart the transaction. Increases the restart counter. If the number - of restarts is higher than *max_restarts* Varnish emits a guru meditation - error. + see `restart`_ .. _vcl_miss: @@ -155,17 +220,17 @@ the backend processing side. The `vcl_miss` subroutine may terminate with calling ``return()`` with one of the following keywords: + ``fail`` + see `fail`_ + ``synth(status code, reason)`` - Transition to :ref:`vcl_synth` with ``resp.status`` and - ``resp.reason`` being preset to the arguments of ``synth()``. + see `synth`_ ``restart`` - Restart the transaction. Increases the restart counter. If the number - of restarts is higher than *max_restarts* Varnish emits a guru meditation - error. + see `restart`_ ``pass`` - Switch to pass mode. Control will eventually pass to :ref:`vcl_pass`. + see `pass`_ ``fetch`` Retrieve the requested object from the backend. Control will @@ -183,17 +248,17 @@ stale: It can have a zero or negative `ttl` with only `grace` or The `vcl_hit` subroutine may terminate with calling ``return()`` with one of the following keywords: + ``fail`` + see `fail`_ + ``synth(status code, reason)`` - Transition to :ref:`vcl_synth` with ``resp.status`` and - ``resp.reason`` being preset to the arguments of ``synth()``. + see `synth`_ ``restart`` - Restart the transaction. Increases the restart counter. If the number - of restarts is higher than *max_restarts* Varnish emits a guru meditation - error. + see `restart`_ ``pass`` - Switch to pass mode. Control will eventually pass to :ref:`vcl_pass`. + see `pass`_ ``miss`` Synchronously refresh the object from the backend despite the @@ -213,14 +278,14 @@ Called before any object except a `vcl_synth` result is delivered to the client. The `vcl_deliver` subroutine may terminate with calling ``return()`` with one of the following keywords: + ``fail`` + see `fail`_ + ``synth(status code, reason)`` - Transition to :ref:`vcl_synth` with ``resp.status`` and - ``resp.reason`` being preset to the arguments of ``synth()``. + see `synth`_ ``restart`` - Restart the transaction. Increases the restart counter. If the number - of restarts is higher than *max_restarts* Varnish emits a guru meditation - error. + see `restart`_ ``deliver`` Deliver the object to the client. @@ -240,14 +305,15 @@ A `vcl_synth` defined object never enters the cache, contrary to a The subroutine may terminate with calling ``return()`` with one of the following keywords: + ``fail`` + see `fail`_ + ``restart`` - Restart the transaction. Increases the restart counter. If the number - of restarts is higher than *max_restarts* Varnish emits a guru meditation - error. + see `restart`_ ``deliver`` - Directly deliver the object defined by `vcl_synth` to the - client without calling `vcl_deliver`. + Directly deliver the object defined by `vcl_synth` to the client + without calling `vcl_deliver`. ------------ Backend Side @@ -264,6 +330,9 @@ typically alter the request before it gets to the backend. The `vcl_backend_fetch` subroutine may terminate with calling ``return()`` with one of the following keywords: + ``fail`` + see `fail`_ + ``fetch`` Fetch the object from the backend. @@ -323,13 +392,16 @@ For a 304 response, varnish core code amends ``beresp`` before calling `beresp.was_304` marks that this conditional response processing has happened. -Note: Backend conditional requests are independend of client +Note: Backend conditional requests are independent of client conditional requests, so clients may receive 304 responses no matter if a backend request was conditional. The `vcl_backend_response` subroutine may terminate with calling ``return()`` with one of the following keywords: + ``fail`` + see `fail`_ + ``deliver`` For a 304 response, create an updated cache object. Otherwise, fetch the object body from the backend and initiate @@ -365,6 +437,9 @@ using the ``synthetic()`` function. The `vcl_backend_error` subroutine may terminate with calling ``return()`` with one of the following keywords: + ``fail`` + see `fail`_ + ``deliver`` Deliver and possibly cache the object defined in `vcl_backend_error` **as if it was fetched from the backend**, also From nils.goroll at uplex.de Mon Oct 2 19:16:06 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 2 Oct 2017 19:16:06 +0000 (UTC) Subject: [master] 9f193da reorder docs to match generate.py Message-ID: <20171002191606.9D9C063816@lists.varnish-cache.org> commit 9f193da0f37e5d4df1154a30ea4ff0677b3d54c6 Author: Nils Goroll Date: Mon Oct 2 19:01:41 2017 +0200 reorder docs to match generate.py diff --git a/doc/sphinx/users-guide/vcl-built-in-subs.rst b/doc/sphinx/users-guide/vcl-built-in-subs.rst index 38e7c3f..941f0b8 100644 --- a/doc/sphinx/users-guide/vcl-built-in-subs.rst +++ b/doc/sphinx/users-guide/vcl-built-in-subs.rst @@ -30,9 +30,9 @@ be set as a default for the backend processing side. The `vcl_recv` subroutine may terminate with calling ``return()`` on one of the following keywords: - ``hash`` - Continue processing the object as a potential candidate for - caching. Passes the control over to :ref:`vcl_hash`. + ``synth(status code, reason)`` + Transition to :ref:`vcl_synth` with ``resp.status`` and + ``resp.reason`` being preset to the arguments of ``synth()``. ``pass`` Switch to pass mode. Control will eventually pass to :ref:`vcl_pass`. @@ -40,9 +40,9 @@ of the following keywords: ``pipe`` Switch to pipe mode. Control will eventually pass to :ref:`vcl_pipe`. - ``synth(status code, reason)`` - Transition to :ref:`vcl_synth` with ``resp.status`` and - ``resp.reason`` being preset to the arguments of ``synth()``. + ``hash`` + Continue processing the object as a potential candidate for + caching. Passes the control over to :ref:`vcl_hash`. ``purge`` Purge the object and it's variants. Control passes through @@ -63,13 +63,13 @@ other VCL subroutine will ever get called after `vcl_pipe`. The `vcl_pipe` subroutine may terminate with calling ``return()`` with one of the following keywords: - ``pipe`` - Proceed with pipe mode. - ``synth(status code, reason)`` Transition to :ref:`vcl_synth` with ``resp.status`` and ``resp.reason`` being preset to the arguments of ``synth()``. + ``pipe`` + Proceed with pipe mode. + .. _vcl_pass: vcl_pass @@ -83,86 +83,17 @@ submitted over the same client connection are handled normally. The `vcl_pass` subroutine may terminate with calling ``return()`` with one of the following keywords: - ``fetch`` - Proceed with pass mode - initiate a backend request. - - ``restart`` - Restart the transaction. Increases the restart counter. If the number - of restarts is higher than *max_restarts* Varnish emits a guru meditation - error. - ``synth(status code, reason)`` Transition to :ref:`vcl_synth` with ``resp.status`` and ``resp.reason`` being preset to the arguments of ``synth()``. -.. _vcl_hit: - -vcl_hit -~~~~~~~ - -Called when a cache lookup is successful. The object being hit may be -stale: It can have a zero or negative `ttl` with only `grace` or -`keep` time left. - -The `vcl_hit` subroutine may terminate with calling ``return()`` -with one of the following keywords: - - ``deliver`` - Deliver the object. If it is stale, a background fetch to refresh - it is triggered. - - ``miss`` - Synchronously refresh the object from the backend despite the - cache hit. Control will eventually pass to :ref:`vcl_miss`. - - ``pass`` - Switch to pass mode. Control will eventually pass to :ref:`vcl_pass`. - ``restart`` Restart the transaction. Increases the restart counter. If the number of restarts is higher than *max_restarts* Varnish emits a guru meditation error. - ``synth(status code, reason)`` - Transition to :ref:`vcl_synth` with ``resp.status`` and - ``resp.reason`` being preset to the arguments of ``synth()``. - - ``fetch`` (deprecated) - same as ``miss``. Will get removed in a future version, triggers a - VCL_Error log message. - -.. _vcl_miss: - -vcl_miss -~~~~~~~~ - -Called after a cache lookup if the requested document was not found in -the cache or if :ref:`vcl_hit` returned ``fetch``. - -Its purpose is to decide whether or not to attempt to retrieve the -document from the backend. A backend hint may be set as a default for -the backend processing side. - -The `vcl_miss` subroutine may terminate with calling ``return()`` with one -of the following keywords: - ``fetch`` - Retrieve the requested object from the backend. Control will - eventually pass to `vcl_backend_fetch`. - - ``pass`` - Switch to pass mode. Control will eventually pass to :ref:`vcl_pass`. - - ``restart`` - Restart the transaction. Increases the restart counter. If the number - of restarts is higher than *max_restarts* Varnish emits a guru meditation - error. - - ``synth(status code, reason)`` - Transition to :ref:`vcl_synth` with ``resp.status`` and - ``resp.reason`` being preset to the arguments of ``synth()``. - -.. XXX: #1603 hit should not go to miss + Proceed with pass mode - initiate a backend request. .. _vcl_hash: @@ -200,15 +131,78 @@ Called after the purge has been executed and all its variants have been evited. The `vcl_purge` subroutine may terminate with calling ``return()`` with one of the following keywords: + ``synth(status code, reason)`` + Transition to :ref:`vcl_synth` with ``resp.status`` and + ``resp.reason`` being preset to the arguments of ``synth()``. + + ``restart`` + Restart the transaction. Increases the restart counter. If the number + of restarts is higher than *max_restarts* Varnish emits a guru meditation + error. + +.. _vcl_miss: + +vcl_miss +~~~~~~~~ + +Called after a cache lookup if the requested document was not found in +the cache or if :ref:`vcl_hit` returned ``fetch``. + +Its purpose is to decide whether or not to attempt to retrieve the +document from the backend. A backend hint may be set as a default for +the backend processing side. + +The `vcl_miss` subroutine may terminate with calling ``return()`` with one +of the following keywords: + + ``synth(status code, reason)`` + Transition to :ref:`vcl_synth` with ``resp.status`` and + ``resp.reason`` being preset to the arguments of ``synth()``. + ``restart`` Restart the transaction. Increases the restart counter. If the number of restarts is higher than *max_restarts* Varnish emits a guru meditation error. + ``pass`` + Switch to pass mode. Control will eventually pass to :ref:`vcl_pass`. + + ``fetch`` + Retrieve the requested object from the backend. Control will + eventually pass to `vcl_backend_fetch`. + +.. _vcl_hit: + +vcl_hit +~~~~~~~ + +Called when a cache lookup is successful. The object being hit may be +stale: It can have a zero or negative `ttl` with only `grace` or +`keep` time left. + +The `vcl_hit` subroutine may terminate with calling ``return()`` +with one of the following keywords: + ``synth(status code, reason)`` Transition to :ref:`vcl_synth` with ``resp.status`` and ``resp.reason`` being preset to the arguments of ``synth()``. + ``restart`` + Restart the transaction. Increases the restart counter. If the number + of restarts is higher than *max_restarts* Varnish emits a guru meditation + error. + + ``pass`` + Switch to pass mode. Control will eventually pass to :ref:`vcl_pass`. + + ``miss`` + Synchronously refresh the object from the backend despite the + cache hit. Control will eventually pass to :ref:`vcl_miss`. + + ``deliver`` + Deliver the object. If it is stale, a background fetch to refresh + it is triggered. + .. _vcl_deliver: vcl_deliver @@ -219,17 +213,17 @@ Called before any object except a `vcl_synth` result is delivered to the client. The `vcl_deliver` subroutine may terminate with calling ``return()`` with one of the following keywords: - ``deliver`` - Deliver the object to the client. + ``synth(status code, reason)`` + Transition to :ref:`vcl_synth` with ``resp.status`` and + ``resp.reason`` being preset to the arguments of ``synth()``. ``restart`` Restart the transaction. Increases the restart counter. If the number of restarts is higher than *max_restarts* Varnish emits a guru meditation error. - ``synth(status code, reason)`` - Transition to :ref:`vcl_synth` with ``resp.status`` and - ``resp.reason`` being preset to the arguments of ``synth()``. + ``deliver`` + Deliver the object to the client. .. _vcl_synth: @@ -246,15 +240,15 @@ A `vcl_synth` defined object never enters the cache, contrary to a The subroutine may terminate with calling ``return()`` with one of the following keywords: - ``deliver`` - Directly deliver the object defined by `vcl_synth` to the - client without calling `vcl_deliver`. - ``restart`` Restart the transaction. Increases the restart counter. If the number of restarts is higher than *max_restarts* Varnish emits a guru meditation error. + ``deliver`` + Directly deliver the object defined by `vcl_synth` to the + client without calling `vcl_deliver`. + ------------ Backend Side ------------ @@ -342,20 +336,20 @@ The `vcl_backend_response` subroutine may terminate with calling delivery to any waiting client requests, possibly in parallel (streaming). - ``pass(duration)`` - Mark the object as a hit-for-pass for the given duration. Subsequent - lookups hitting this object will be turned into passed transactions, - as if ``vcl_recv`` had returned ``pass``. + ``retry`` + Retry the backend transaction. Increases the `retries` counter. + If the number of retries is higher than *max_retries*, + control will be passed to :ref:`vcl_backend_error`. ``abandon`` Abandon the backend request. Unless the backend request was a background fetch, control is passed to :ref:`vcl_synth` on the client side with ``resp.status`` preset to 503. - ``retry`` - Retry the backend transaction. Increases the `retries` counter. - If the number of retries is higher than *max_retries*, - control will be passed to :ref:`vcl_backend_error`. + ``pass(duration)`` + Mark the object as a hit-for-pass for the given duration. Subsequent + lookups hitting this object will be turned into passed transactions, + as if ``vcl_recv`` had returned ``pass``. .. _vcl_backend_error: From nils.goroll at uplex.de Mon Oct 2 19:16:06 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 2 Oct 2017 19:16:06 +0000 (UTC) Subject: [master] 7491696 add fail to dot graphs, improve vcl(label) a bit Message-ID: <20171002191606.D63AF6381B@lists.varnish-cache.org> commit 7491696a81ab01127cc336ae40b6bd4c7c0f55d8 Author: Nils Goroll Date: Mon Oct 2 21:15:20 2017 +0200 add fail to dot graphs, improve vcl(label) a bit diff --git a/doc/graphviz/cache_fetch.dot b/doc/graphviz/cache_fetch.dot index 5e65495..eb68f72 100644 --- a/doc/graphviz/cache_fetch.dot +++ b/doc/graphviz/cache_fetch.dot @@ -39,7 +39,7 @@ digraph cache_fetch { /* vbf_stp_startfetch() */ v_b_f [ shape=record - label="{vbf_stp_startfetch:|{vcl_backend_fetch\{\}|bereq.*}|{abandon|fetch}}" + label="{vbf_stp_startfetch:|{vcl_backend_fetch\{\}|bereq.*}|{fail|abandon|fetch}}" ] v_b_f:fetch:s -> v_b_hdrs [style=bold] v_b_hdrs [ label="send bereq,\nread beresp (headers)"] @@ -47,7 +47,7 @@ digraph cache_fetch { v_b_hdrs -> v_b_e v_b_r [ shape=record - label="{vbf_stp_startfetch:|{vcl_backend_response\{\}|{bereq.*|beresp.*}}|{{retry|{max?|ok?}}|abandon|{deliver or pass|{304?|other?}}}}" + label="{vbf_stp_startfetch:|{vcl_backend_response\{\}|{bereq.*|beresp.*}}|{fail|{retry|{max?|ok?}}|abandon|{deliver or pass|{304?|other?}}}}" ] v_b_r:retry -> v_b_r_retry [color=purple] v_b_r:max -> v_b_e @@ -96,7 +96,7 @@ digraph cache_fetch { /* vbf_stp_error */ v_b_e [ shape=record - label="{vbf_stp_error:|{vcl_backend_error\{\}|{bereq.*|beresp.*}}|{{retry|{max?|ok?}}|abandon|deliver}}}" + label="{vbf_stp_error:|{vcl_backend_error\{\}|{bereq.*|beresp.*}}|{{retry|{fail|max?|ok?}}|abandon|deliver}}}" ] // v_b_e:deliver aka "backend synth" - goes into cache v_b_e:deliver -> FETCH_DONE [label="\"backend synth\""] diff --git a/doc/graphviz/cache_fetch.svg b/doc/graphviz/cache_fetch.svg index f329e59..aeb38a9 100644 --- a/doc/graphviz/cache_fetch.svg +++ b/doc/graphviz/cache_fetch.svg @@ -4,274 +4,280 @@ - + cache_fetch - + cluster_backend - + RETRY -RETRY +RETRY v_b_f - -vbf_stp_startfetch: - -vcl_backend_fetch{} - -bereq.* - -abandon - -fetch + +vbf_stp_startfetch: + +vcl_backend_fetch{} + +bereq.* + +fail + +abandon + +fetch RETRY->v_b_f - - + + v_b_f_BGFETCH - -BGFETCH + +BGFETCH v_b_f_BGFETCH->v_b_f - - + + v_b_f_FETCH - -FETCH + +FETCH v_b_f_FETCH->v_b_f - - + + v_b_f_FETCH->v_b_f - - + + v_b_hdrs - -send bereq, -read beresp (headers) + +send bereq, +read beresp (headers) v_b_f:fetch:s->v_b_hdrs - - + + v_b_r - -vbf_stp_startfetch: - -vcl_backend_response{} - -bereq.* - -beresp.* - -retry - -max? - -ok? - -abandon - -deliver or pass - -304? - -other? + +vbf_stp_startfetch: + +vcl_backend_response{} + +bereq.* + +beresp.* + +fail + +retry + +max? + +ok? + +abandon + +deliver or pass + +304? + +other? v_b_hdrs->v_b_r - - + + v_b_e - -vbf_stp_error: - -vcl_backend_error{} - -bereq.* - -beresp.* - -retry - -max? - -ok? - -abandon - -deliver + +vbf_stp_error: + +vcl_backend_error{} + +bereq.* + +beresp.* + +retry + +fail + +max? + +ok? + +abandon + +deliver v_b_hdrs->v_b_e - - + + v_b_r:max->v_b_e - - + + v_b_r_retry -RETRY +RETRY v_b_r:retry->v_b_r_retry - - + + vbf_stp_condfetch - -vbf_stp_condfetch: - -copy obj attr - -steal body - -fetch_fail? - -ok? + +vbf_stp_condfetch: + +copy obj attr + +steal body + +fetch_fail? + +ok? v_b_r:fetch_304:s->vbf_stp_condfetch - - + + vbf_stp_fetch - -vbf_stp_fetch: - -setup VFPs - -get object - -error? - -body? + +vbf_stp_fetch: + +setup VFPs + +get object + +error? + +body? v_b_r:non_304:s->vbf_stp_fetch - - + + FETCH_DONE - -FETCH_DONE + +FETCH_DONE v_b_e:deliver->FETCH_DONE - - -"backend synth" + + +"backend synth" FETCH_FAIL - -FETCH_FAIL + +FETCH_FAIL v_b_e:max:s->FETCH_FAIL - - + + v_b_e_retry -RETRY +RETRY v_b_e:retry->v_b_e_retry - - + + vbf_stp_fetchend - -vbf_stp_fetchend: - -finalize object and director - -done + +vbf_stp_fetchend: + +finalize object and director + +done vbf_stp_condfetch:ok:s->vbf_stp_fetchend - - + + vbf_stp_fetchbody - -vbf_stp_fetchbody: - -get storage - -read body, run VFPs - -fetch_fail? - -error? - -ok? + +vbf_stp_fetchbody: + +get storage + +read body, run VFPs + +fetch_fail? + +error? + +ok? vbf_stp_fetch:body:s->vbf_stp_fetchbody - - + + vbf_stp_fetch:body:s->vbf_stp_fetchend - - + + vbf_stp_fetchbody:ok:s->vbf_stp_fetchend - - + + vbf_stp_fetchend:done:s->FETCH_DONE - - + + error -error +error error->FETCH_FAIL - - + + abandon -abandon +abandon abandon->FETCH_FAIL - - + + diff --git a/doc/graphviz/cache_req_fsm.dot b/doc/graphviz/cache_req_fsm.dot index a9bc854..bbec3e5 100644 --- a/doc/graphviz/cache_req_fsm.dot +++ b/doc/graphviz/cache_req_fsm.dot @@ -48,13 +48,13 @@ digraph cache_req_fsm { */ acceptor [shape=hexagon label="Request received"] - label_select [shape=hexagon label="Label selected"] + label_select [shape=hexagon label="LABEL"] ESI_REQ [shape=hexagon label="ESI request"] RESTART [shape=plaintext] ESI_REQ -> recv SYNTH [shape=plaintext] - LABEL [shape=plaintext] + FAIL [shape=plaintext] acceptor -> recv [style=bold] label_select -> recv [style=bold] @@ -62,7 +62,7 @@ digraph cache_req_fsm { /* cnt_deliver() */ deliver [ shape=record - label="{cnt_deliver:|Filter obj.-\>resp.|{vcl_deliver\{\}|{req.*|resp.*}}|{restart|deliver|synth}}" + label="{cnt_deliver:|Filter obj.-\>resp.|{vcl_deliver\{\}|{req.*|resp.*}}|{fail|restart|deliver|synth}}" ] deliver:deliver:s -> V1D_Deliver [style=bold,color=green] @@ -79,9 +79,10 @@ digraph cache_req_fsm { subgraph xcluster_synth { synth [ shape=record - label="{cnt_synth:|{vcl_synth\{\}|{req.*|resp.*}}|{deliver|restart}}" + label="{cnt_synth:|{vcl_synth\{\}|{req.*|resp.*}}|{fail|deliver|restart}}" ] + FAIL -> synth [color=purple] SYNTH -> synth [color=purple] synth:del:s -> V1D_Deliver [color=purple] } @@ -120,7 +121,7 @@ digraph cache_req_fsm { ] lookup2 [ shape=record - label="{cnt_lookup:|{vcl_hit\{\}|{req.*|obj.*}}|{deliver|miss|restart|synth|pass}}" + label="{cnt_lookup:|{vcl_hit\{\}|{req.*|obj.*}}|{fail|deliver|miss|restart|synth|pass}}" ] } lookup:busy:e -> lookup:top:e [label="(waitinglist)", @@ -140,7 +141,7 @@ digraph cache_req_fsm { subgraph xcluster_miss { miss [ shape=record - label="{cnt_miss:|{vcl_miss\{\}|req.*}|{fetch|synth|restart|pass}}" + label="{cnt_miss:|{vcl_miss\{\}|req.*}|{fail|fetch|synth|restart|pass}}" ] } miss:fetch:s -> FETCH [style=bold,color=blue] @@ -150,7 +151,7 @@ digraph cache_req_fsm { subgraph xcluster_pass { pass [ shape=record - label="{cnt_pass:|{vcl_pass\{\}|req.*}|{fetch|synth|restart}}" + label="{cnt_pass:|{vcl_pass\{\}|req.*}|{fail|fetch|synth|restart}}" ] } pass:fetch:s -> FETCH [style=bold, color=red] @@ -159,7 +160,7 @@ digraph cache_req_fsm { subgraph xcluster_pipe { pipe [ shape=record - label="{cnt_pipe:|filter req.*-\>bereq.*|{vcl_pipe\{\}|{req.*|bereq.*}}|{pipe|synth}}" + label="{cnt_pipe:|filter req.*-\>bereq.*|{vcl_pipe\{\}|{req.*|bereq.*}}|{fail|pipe|synth}}" ] pipe_do [ shape=ellipse @@ -175,7 +176,7 @@ digraph cache_req_fsm { shape=record color=grey fontcolor=grey - label="{cnt_restart:|{ok?|max_restarts?}}" + label="{cnt_restart:|{fail|ok?|max_restarts?}}" ] } RESTART -> restart [color=purple] @@ -187,7 +188,7 @@ digraph cache_req_fsm { subgraph xcluster_recv { recv [ shape=record - label="{cnt_recv:|{vcl_recv\{\}|req.*}|{hash|purge|pass|pipe|synth|vcl}}" + label="{cnt_recv:|{vcl_recv\{\}|req.*}|{fail|hash|purge|pass|pipe|synth|vcl}}" ] recv:hash -> hash [style=bold,color=green] hash [ @@ -205,13 +206,13 @@ digraph cache_req_fsm { recv:purge:s -> hash [style=bold,color=purple] recv:vcl:s -> vcl_label - vcl_label [label="LABEL",shape=plaintext] + vcl_label [label="switch to vcl\nLABEL",shape=plaintext] /* cnt_purge */ subgraph xcluster_purge { purge [ shape=record - label="{cnt_purge:|{vcl_purge\{\}|req.*}|{synth|restart}}" + label="{cnt_purge:|{vcl_purge\{\}|req.*}|{fail|synth|restart}}" ] } } diff --git a/doc/graphviz/cache_req_fsm.svg b/doc/graphviz/cache_req_fsm.svg index 26d4564..4afe57d 100644 --- a/doc/graphviz/cache_req_fsm.svg +++ b/doc/graphviz/cache_req_fsm.svg @@ -4,470 +4,494 @@ - + cache_req_fsm - + cluster_backend - + acceptor - -Request received + +Request received recv - -cnt_recv: - -vcl_recv{} - -req.* - -hash - -purge - -pass - -pipe - -synth - -vcl + +cnt_recv: + +vcl_recv{} + +req.* + +fail + +hash + +purge + +pass + +pipe + +synth + +vcl acceptor->recv - - + + label_select - -Label selected + +LABEL label_select->recv - - + + ESI_REQ - -ESI request + +ESI request ESI_REQ->recv - - + + RESTART -RESTART +RESTART restart - -cnt_restart: - -ok? - -max_restarts? + +cnt_restart: + +fail + +ok? + +max_restarts? -RESTART->restart - - +RESTART->restart + + hash - -cnt_recv: - -vcl_hash{} - -req.* - -lookup + +cnt_recv: + +vcl_hash{} + +req.* + +lookup -recv:hash->hash - - +recv:hash->hash + + -recv:pipe->hash - - +recv:pipe->hash + + -recv:pass->hash - - +recv:pass->hash + + -recv:purge:s->hash - - +recv:purge:s->hash + + vcl_label -LABEL +switch to vcl +LABEL -recv:vcl:s->vcl_label - - +recv:vcl:s->vcl_label + + SYNTH -SYNTH +SYNTH synth - -cnt_synth: - -vcl_synth{} - -req.* - -resp.* - -deliver - -restart + +cnt_synth: + +vcl_synth{} + +req.* + +resp.* + +fail + +deliver + +restart -SYNTH->synth - - +SYNTH->synth + + - -LABEL -LABEL + +FAIL +FAIL + + +FAIL->synth + + deliver - -cnt_deliver: - -Filter obj.->resp. - -vcl_deliver{} - -req.* - -resp.* - -restart - -deliver - -synth + +cnt_deliver: + +Filter obj.->resp. + +vcl_deliver{} + +req.* + +resp.* + +fail + +restart + +deliver + +synth V1D_Deliver - -V1D_Deliver + +V1D_Deliver deliver:deliver:s->V1D_Deliver - - + + deliver:deliver:s->V1D_Deliver - - + + deliver:deliver:s->V1D_Deliver - - + + DONE - -DONE + +DONE V1D_Deliver->DONE - - + + stream - -stream? -body + +stream? +body stream->V1D_Deliver - - + + -synth:del:s->V1D_Deliver - - +synth:del:s->V1D_Deliver + + see backend graph -see backend graph +see backend graph BGFETCH - -BGFETCH + +BGFETCH FETCH - -FETCH + +FETCH FETCH_DONE - -FETCH_DONE + +FETCH_DONE -FETCH->FETCH_DONE - - +FETCH->FETCH_DONE + + FETCH_FAIL - -FETCH_FAIL + +FETCH_FAIL -FETCH->FETCH_FAIL - - +FETCH->FETCH_FAIL + + -FETCH_DONE->deliver - - +FETCH_DONE->deliver + + -FETCH_DONE->deliver - - +FETCH_DONE->deliver + + -FETCH_FAIL->synth - - +FETCH_FAIL->synth + + lookup2 - -cnt_lookup: - -vcl_hit{} - -req.* - -obj.* - -deliver - -miss - -restart - -synth - -pass + +cnt_lookup: + +vcl_hit{} + +req.* + +obj.* + +fail + +deliver + +miss + +restart + +synth + +pass -lookup2:deliver:s->deliver:n - - +lookup2:deliver:s->deliver:n + + -lookup2:deliver:s->BGFETCH - - -parallel -if obj expired +lookup2:deliver:s->BGFETCH + + +parallel +if obj expired miss - -cnt_miss: - -vcl_miss{} - -req.* - -fetch - -synth - -restart - -pass + +cnt_miss: + +vcl_miss{} + +req.* + +fail + +fetch + +synth + +restart + +pass -lookup2:miss:s->miss - - +lookup2:miss:s->miss + + pass - -cnt_pass: - -vcl_pass{} - -req.* - -fetch - -synth - -restart + +cnt_pass: + +vcl_pass{} + +req.* + +fail + +fetch + +synth + +restart -lookup2:pass:s->pass - - +lookup2:pass:s->pass + + lookup - -cnt_lookup: - -hash lookup - -hit? - -miss? - -hit-for-pass? - -busy? + +cnt_lookup: + +hash lookup + +hit? + +miss? + +hit-for-pass? + +busy? -lookup:h:s->lookup2 - - +lookup:h:s->lookup2 + + -lookup:busy:e->lookup:top:e - - -(waitinglist) +lookup:busy:e->lookup:top:e + + +(waitinglist) -lookup:miss:s->miss - - +lookup:miss:s->miss + + -lookup:hfp:s->pass - - +lookup:hfp:s->pass + + -miss:fetch:s->FETCH - - +miss:fetch:s->FETCH + + -miss:pass:s->pass - - +miss:pass:s->pass + + -pass:fetch:s->FETCH - - +pass:fetch:s->FETCH + + pipe - -cnt_pipe: - -filter req.*->bereq.* - -vcl_pipe{} - -req.* - -bereq.* - -pipe - -synth + +cnt_pipe: + +filter req.*->bereq.* + +vcl_pipe{} + +req.* + +bereq.* + +fail + +pipe + +synth pipe_do - -send bereq, -copy bytes until close + +send bereq, +copy bytes until close -pipe:pipe->pipe_do - - +pipe:pipe->pipe_do + + -pipe_do->DONE - - +pipe_do->DONE + + -restart:ok:s->recv - - +restart:ok:s->recv + + err_restart -SYNTH +SYNTH -restart:max:s->err_restart - - +restart:max:s->err_restart + + -hash:lookup:w->lookup - - +hash:lookup:w->lookup + + -hash:lookup:s->pass - - +hash:lookup:s->pass + + -hash:lookup:e->pipe - - +hash:lookup:e->pipe + + purge - -cnt_purge: - -vcl_purge{} - -req.* - -synth - -restart + +cnt_purge: + +vcl_purge{} + +req.* + +fail + +synth + +restart -hash:lookup:s->purge:top:n - - +hash:lookup:s->purge:top:n + + From nils.goroll at uplex.de Mon Oct 2 19:31:04 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 2 Oct 2017 19:31:04 +0000 (UTC) Subject: [master] 5b849bf oops I broke distcheck with an rst Message-ID: <20171002193104.9B524630F6@lists.varnish-cache.org> commit 5b849bfce1211279ac5a75397912d149dc8e27f5 Author: Nils Goroll Date: Mon Oct 2 21:29:42 2017 +0200 oops I broke distcheck with an rst diff --git a/doc/changes.rst b/doc/changes.rst index c1e45b8..971d269 100644 --- a/doc/changes.rst +++ b/doc/changes.rst @@ -4,7 +4,7 @@ Varnish Cache Trunk (ongoing) * Fixed implementation of the ``max_restarts`` limit: It used to be one less than the number of allowed restarts, it now is the number of - ``return(restart)``s per request. + ``return(restart)`` calls per request. ================================ Varnish Cache 5.2.0 (2017-09-15) From dridi.boukelmoune at gmail.com Tue Oct 3 12:30:12 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 3 Oct 2017 12:30:12 +0000 (UTC) Subject: [master] 993ca79 Don't reembark h2 streams as http/1 requests Message-ID: <20171003123012.E09E963FAF@lists.varnish-cache.org> commit 993ca79eb8b2890d2e893dbea57364b590d8860e Author: Dridi Boukelmoune Date: Tue Oct 3 14:22:55 2017 +0200 Don't reembark h2 streams as http/1 requests diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index bd0716e..bc3a86d 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -362,7 +362,7 @@ SES_New(struct pool *pp) */ int -SES_Reschedule_Req(struct req *req) +SES_Reschedule_Req(struct req *req, enum task_prio prio) { struct sess *sp; struct pool *pp; @@ -372,10 +372,11 @@ SES_Reschedule_Req(struct req *req) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); pp = sp->pool; CHECK_OBJ_NOTNULL(pp, POOL_MAGIC); + AN(TASK_QUEUE_CLIENT(prio)); AN(req->task.func); - return (Pool_Task(pp, &req->task, TASK_QUEUE_REQ)); + return (Pool_Task(pp, &req->task, prio)); } /*-------------------------------------------------------------------- diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index d81edcf..4e67f7b 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -172,7 +172,7 @@ void SES_DestroyPool(struct pool *); void SES_Wait(struct sess *, const struct transport *); void SES_Ref(struct sess *sp); void SES_Rel(struct sess *sp); -int SES_Reschedule_Req(struct req *); +int SES_Reschedule_Req(struct req *, enum task_prio); enum htc_status_e { HTC_S_JUNK = -5, diff --git a/bin/varnishd/http1/cache_http1_fsm.c b/bin/varnishd/http1/cache_http1_fsm.c index 58748d3..6e8de47 100644 --- a/bin/varnishd/http1/cache_http1_fsm.c +++ b/bin/varnishd/http1/cache_http1_fsm.c @@ -188,7 +188,7 @@ http1_reembark(struct worker *wrk, struct req *req) http1_setstate(sp, H1BUSY); - if (!SES_Reschedule_Req(req)) + if (!SES_Reschedule_Req(req, TASK_QUEUE_REQ)) return; /* Couldn't schedule, ditch */ diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c index f679a3a..567e908 100644 --- a/bin/varnishd/http2/cache_http2_session.c +++ b/bin/varnishd/http2/cache_http2_session.c @@ -357,7 +357,7 @@ h2_reembark(struct worker *wrk, struct req *req) CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); assert(req->transport == &H2_transport); - if (!SES_Reschedule_Req(req)) + if (!SES_Reschedule_Req(req, TASK_QUEUE_STR)) return; /* Couldn't schedule, ditch */ From daghf at varnish-software.com Tue Oct 3 13:43:05 2017 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Tue, 3 Oct 2017 13:43:05 +0000 (UTC) Subject: [master] 053c93c Enforce sequence requirement of header block frames Message-ID: <20171003134305.E0FDB634CE@lists.varnish-cache.org> commit 053c93cd12f79a00d9801a5d0f9e67ea3aa2fcbd Author: Dag Haavi Finstad Date: Fri Sep 29 16:05:31 2017 +0200 Enforce sequence requirement of header block frames This commit enforces that a HEADERS frame without an END_HEADERS flag must be followed by a CONTINUATION frame on the same stream. Sending interleaved/out of sequence header or continuation frames will now be treated as a PROTOCOL_ERROR, as per rfc7540. This also fixes the panics seen in #2387. Fixes: #2387 diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 3c50d20..06353d8 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -508,7 +508,7 @@ h2_do_req(struct worker *wrk, void *priv) } static h2_error -h2_end_headers(struct worker *wrk, const struct h2_sess *h2, +h2_end_headers(struct worker *wrk, struct h2_sess *h2, struct req *req, struct h2_req *r2) { h2_error h2e; @@ -518,6 +518,7 @@ h2_end_headers(struct worker *wrk, const struct h2_sess *h2, h2e = h2h_decode_fini(h2, r2->decode); FREE_OBJ(r2->decode); r2->state = H2_S_CLOS_REM; + h2->new_req = NULL; if (h2e != NULL) { Lck_Lock(&h2->sess->mtx); VSLb(h2->vsl, SLT_Debug, "HPACK/FINI %s", h2e->name); @@ -614,9 +615,7 @@ h2_rx_headers(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) return (0); } -/********************************************************************** - * XXX: Check hard sequence req. for Cont. - */ +/**********************************************************************/ static h2_error __match_proto__(h2_frame_f) h2_rx_continuation(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) @@ -831,6 +830,11 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2, AN(r2); } + if (h2->new_req != NULL && + !(h2->new_req == r2->req && h2f == H2_F_CONTINUATION)) + return (H2CE_PROTOCOL_ERROR); // rfc7540,l,1859,1863 + + h2e = h2f->rxfunc(wrk, h2, r2); if (h2e == 0) return (0); diff --git a/bin/varnishtest/tests/r02387.vtc b/bin/varnishtest/tests/r02387.vtc new file mode 100644 index 0000000..34863a0 --- /dev/null +++ b/bin/varnishtest/tests/r02387.vtc @@ -0,0 +1,36 @@ +varnishtest "2387: Crash on out of order header blocks" + +server s1 { + rxreq + txresp +} -start + +varnish v1 -vcl+backend {} -start + +varnish v1 -cliok "param.set feature +http2" +varnish v1 -cliok "param.set debug +syncvsl" + + +barrier b1 cond 2 +barrier b2 cond 2 + +client c1 { + stream 1 { + txreq -nohdrend + barrier b2 sync + barrier b1 sync + txcont -hdr "bar" "foo" + } -start + stream 3 { + barrier b2 sync + txreq -nohdrend + barrier b1 sync + txcont -hdr "bar" "foo" + + } -run + stream 0 { + rxgoaway + expect goaway.laststream == "3" + expect goaway.err == PROTOCOL_ERROR + } -run +} -run From daghf at varnish-software.com Tue Oct 3 14:10:07 2017 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Tue, 3 Oct 2017 14:10:07 +0000 (UTC) Subject: [master] d5ce30f Drop the extra 100s timeout_idle for h2 frames Message-ID: <20171003141008.0634A63E0E@lists.varnish-cache.org> commit d5ce30f1f7980ec7cad286f64c4620784a8839f2 Author: Dag Haavi Finstad Date: Tue Oct 3 16:08:55 2017 +0200 Drop the extra 100s timeout_idle for h2 frames diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 06353d8..1bc22b7 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -882,7 +882,7 @@ h2_rxframe(struct worker *wrk, struct h2_sess *h2) h2->sess->t_idle = VTIM_real(); hs = HTC_RxStuff(h2->htc, h2_frame_complete, NULL, NULL, NAN, - h2->sess->t_idle + cache_param->timeout_idle + 100, + h2->sess->t_idle + cache_param->timeout_idle, 16384 + 9); // rfc7540,l,4228,4228 if (hs != HTC_S_COMPLETE) { Lck_Lock(&h2->sess->mtx); From daghf at varnish-software.com Tue Oct 3 15:42:06 2017 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Tue, 3 Oct 2017 15:42:06 +0000 (UTC) Subject: [master] 919af66 Fix a tiny mistake introduced in 053c93cd12f Message-ID: <20171003154206.7E06163D0C@lists.varnish-cache.org> commit 919af666ad5f45ff5d2a9451767871c7a6d26b5b Author: Dag Haavi Finstad Date: Tue Oct 3 17:41:12 2017 +0200 Fix a tiny mistake introduced in 053c93cd12f diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 1bc22b7..e3da09f 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -831,7 +831,7 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2, } if (h2->new_req != NULL && - !(h2->new_req == r2->req && h2f == H2_F_CONTINUATION)) + !(r2 && h2->new_req == r2->req && h2f == H2_F_CONTINUATION)) return (H2CE_PROTOCOL_ERROR); // rfc7540,l,1859,1863 From phk at FreeBSD.org Wed Oct 4 09:29:09 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 4 Oct 2017 09:29:09 +0000 (UTC) Subject: [master] 1a0d149 Have I ever mentioned how much I hate the ISO-C people and the ****heads in the gcc&llvm projects who cannot cooperate on usability features ? Message-ID: <20171004092909.997D963822@lists.varnish-cache.org> commit 1a0d149db00a20b2edf19f56793ec99e0e6ccf14 Author: Poul-Henning Kamp Date: Wed Oct 4 09:27:17 2017 +0000 Have I ever mentioned how much I hate the ISO-C people and the ****heads in the gcc&llvm projects who cannot cooperate on usability features ? Fixes #2449 diff --git a/configure.ac b/configure.ac index 5e20c0a..cf8f9b5 100644 --- a/configure.ac +++ b/configure.ac @@ -368,27 +368,6 @@ AC_CHECK_DECL([SO_ACCEPTFILTER], ] ) -# Check for _Static_assert() -# We dont' want to go all C11 on this, and we have vas.h instead -# of assert.h, so check the brutal way - -save_LIBS="${LIBS}" -LIBS="" -AC_CACHE_CHECK([for _Static_assert], - [ac_cv_static_assert], - [AC_RUN_IFELSE( - [AC_LANG_PROGRAM([[ -_Static_assert(1 == sizeof(char), "didn't work"); - ]],[[ - ]])], - [ac_cv_static_assert=yes], - [ac_cv_static_assert=no]) - ]) -if test "$ac_cv_static_assert" = yes; then - AC_DEFINE([STATIC_ASSERT], [1], [Define if _Static_assert is availabel]) -fi -LIBS="${save_LIBS}" - # Older Solaris versions define SO_{RCV,SND}TIMEO, but do not # implement them. # diff --git a/include/vdef.h b/include/vdef.h index 70f8178..12093bf 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -127,8 +127,28 @@ #define NEEDLESS(s) s #endif -#ifndef STATIC_ASSERT -# define _Static_assert(a,b) + +/* + * Most of this nightmare is stolen from FreeBSD's + */ +#if defined(_Static_assert) + /* Nothing, somebody already did this for us */ +#elif defined(__has_extension) && __has_extension(c_static_assert) + /* Nothing, we should be fine */ +#elif (defined(__cplusplus) && __cplusplus >= 201103L) || \ + __has_extension(cxx_static_assert) +# define _Static_assert(x, y) static_assert(x, y) +#elif __GNUC_PREREQ__(4,6) && !defined(__cplusplus) + /* Nothing, gcc 4.6 and higher has _Static_assert built-in */ +#else +# if defined(__COUNTER__) +# define _Static_assert(x, y) __Static_assert(x, __COUNTER__) +# else +# define _Static_assert(x, y) __Static_assert(x, __LINE__) +# endif +# define __Static_assert(x, y) ___Static_assert(x, y) +# define ___Static_assert(x, y) \ + typedef char __assert_ ## y[(x) ? 1 : -1] __unused #endif #endif /* VDEF_H_INCLUDED */ From phk at FreeBSD.org Wed Oct 4 09:35:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 4 Oct 2017 09:35:06 +0000 (UTC) Subject: [master] 5e3387f Try another way Message-ID: <20171004093507.0438963B05@lists.varnish-cache.org> commit 5e3387fa73992cb1179aed69da3507dadbd97cc2 Author: Poul-Henning Kamp Date: Wed Oct 4 09:34:45 2017 +0000 Try another way diff --git a/include/vdef.h b/include/vdef.h index 12093bf..4af293c 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -131,9 +131,12 @@ /* * Most of this nightmare is stolen from FreeBSD's */ +#ifndef __has_extension +# define __has_extension 0 +#endif #if defined(_Static_assert) /* Nothing, somebody already did this for us */ -#elif defined(__has_extension) && __has_extension(c_static_assert) +#elif __has_extension(c_static_assert) /* Nothing, we should be fine */ #elif (defined(__cplusplus) && __cplusplus >= 201103L) || \ __has_extension(cxx_static_assert) From phk at FreeBSD.org Wed Oct 4 09:42:07 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 4 Oct 2017 09:42:07 +0000 (UTC) Subject: [master] 67bb076 Ok, that one was silly Message-ID: <20171004094207.E785863E6B@lists.varnish-cache.org> commit 67bb076e2d6dfb465a8563e3ea4f5b9274d194bf Author: Poul-Henning Kamp Date: Wed Oct 4 09:41:23 2017 +0000 Ok, that one was silly diff --git a/include/vdef.h b/include/vdef.h index 4af293c..7f8af5e 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -132,7 +132,7 @@ * Most of this nightmare is stolen from FreeBSD's */ #ifndef __has_extension -# define __has_extension 0 +# define __has_extension(x) 0 #endif #if defined(_Static_assert) /* Nothing, somebody already did this for us */ From phk at FreeBSD.org Wed Oct 4 09:49:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 4 Oct 2017 09:49:06 +0000 (UTC) Subject: [master] 4ede8c3 Suns C-compiler also lives in its own island. Message-ID: <20171004094906.615E66223A@lists.varnish-cache.org> commit 4ede8c33a97c823158ac0289044a9e156cc060e4 Author: Poul-Henning Kamp Date: Wed Oct 4 09:47:44 2017 +0000 Suns C-compiler also lives in its own island. diff --git a/include/vdef.h b/include/vdef.h index 7f8af5e..a202f0b 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -134,6 +134,9 @@ #ifndef __has_extension # define __has_extension(x) 0 #endif +#ifndef __GNUC_PREREQ__ +# define __GNUC_PREREQ__(x) 0 +#endif #if defined(_Static_assert) /* Nothing, somebody already did this for us */ #elif __has_extension(c_static_assert) From phk at FreeBSD.org Wed Oct 4 09:52:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 4 Oct 2017 09:52:06 +0000 (UTC) Subject: [master] dc34c4f Eventually I'll get this right... Message-ID: <20171004095206.6C89D62496@lists.varnish-cache.org> commit dc34c4f642113c346ce54198d6947028f104624e Author: Poul-Henning Kamp Date: Wed Oct 4 09:51:48 2017 +0000 Eventually I'll get this right... diff --git a/include/vdef.h b/include/vdef.h index a202f0b..2bb0d87 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -135,7 +135,7 @@ # define __has_extension(x) 0 #endif #ifndef __GNUC_PREREQ__ -# define __GNUC_PREREQ__(x) 0 +# define __GNUC_PREREQ__(x,y) 0 #endif #if defined(_Static_assert) /* Nothing, somebody already did this for us */ From phk at FreeBSD.org Wed Oct 4 10:02:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 4 Oct 2017 10:02:05 +0000 (UTC) Subject: [master] e10a004 Sigh... Message-ID: <20171004100205.89342629B6@lists.varnish-cache.org> commit e10a004c2d098042df0568b03f367cbaf313ab3f Author: Poul-Henning Kamp Date: Wed Oct 4 10:01:39 2017 +0000 Sigh... diff --git a/include/vdef.h b/include/vdef.h index 2bb0d87..fd28a58 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -154,7 +154,7 @@ # endif # define __Static_assert(x, y) ___Static_assert(x, y) # define ___Static_assert(x, y) \ - typedef char __assert_ ## y[(x) ? 1 : -1] __unused + typedef char __assert_## y[(x) ? 1 : -1] __unused #endif #endif /* VDEF_H_INCLUDED */ From phk at FreeBSD.org Wed Oct 4 10:08:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 4 Oct 2017 10:08:05 +0000 (UTC) Subject: [master] 50ddc2e I promise this will stop at some point Message-ID: <20171004100805.64A3463C66@lists.varnish-cache.org> commit 50ddc2e9d34c45f82b7fc063f1a0f02db27a73fa Author: Poul-Henning Kamp Date: Wed Oct 4 10:07:12 2017 +0000 I promise this will stop at some point diff --git a/include/vdef.h b/include/vdef.h index fd28a58..f52bdf8 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -122,11 +122,13 @@ #ifdef __SUNPRO_C #define NEEDLESS(s) {} -#define __unused #else #define NEEDLESS(s) s #endif +#ifndef __unused +# define __unused +#endif /* * Most of this nightmare is stolen from FreeBSD's From phk at FreeBSD.org Wed Oct 4 10:12:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 4 Oct 2017 10:12:05 +0000 (UTC) Subject: [master] 687447e As I mentioned... Message-ID: <20171004101205.CABFA63EB2@lists.varnish-cache.org> commit 687447e106e50eb068cc3347a70852905bad010b Author: Poul-Henning Kamp Date: Wed Oct 4 10:11:16 2017 +0000 As I mentioned... diff --git a/include/vdef.h b/include/vdef.h index f52bdf8..af52571 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -127,7 +127,7 @@ #endif #ifndef __unused -# define __unused +# define __unused __attribute__((__unused__)) #endif /* diff --git a/include/vtree.h b/include/vtree.h index 777f234..93323ab 100644 --- a/include/vtree.h +++ b/include/vtree.h @@ -30,10 +30,6 @@ #ifndef _VTREE_H_ #define _VTREE_H_ -#ifndef __unused -#define __unused __attribute__((__unused__)) -#endif - /* * This file defines data structures for different types of trees: * splay trees and red-black trees. From phk at FreeBSD.org Wed Oct 4 10:21:07 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 4 Oct 2017 10:21:07 +0000 (UTC) Subject: [master] 1e533a6 On the other hand... Message-ID: <20171004102107.E60BE63357@lists.varnish-cache.org> commit 1e533a60df47c261c70219c15c5e374656b8ea00 Author: Poul-Henning Kamp Date: Wed Oct 4 10:20:27 2017 +0000 On the other hand... diff --git a/include/vdef.h b/include/vdef.h index af52571..0a23407 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -127,7 +127,11 @@ #endif #ifndef __unused -# define __unused __attribute__((__unused__)) +# if __GNUC_PREREQ__(2, 7) +# define __unused __attribute__((__unused__)) +# else +# define __unused +# endif #endif /* @@ -136,9 +140,7 @@ #ifndef __has_extension # define __has_extension(x) 0 #endif -#ifndef __GNUC_PREREQ__ -# define __GNUC_PREREQ__(x,y) 0 -#endif + #if defined(_Static_assert) /* Nothing, somebody already did this for us */ #elif __has_extension(c_static_assert) From phk at FreeBSD.org Wed Oct 4 10:24:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 4 Oct 2017 10:24:05 +0000 (UTC) Subject: [master] 89acac8 But then again... Message-ID: <20171004102405.6D88A63652@lists.varnish-cache.org> commit 89acac8f1c6f7b28cc5750bee81b506f04cdd74e Author: Poul-Henning Kamp Date: Wed Oct 4 10:23:36 2017 +0000 But then again... diff --git a/include/vdef.h b/include/vdef.h index 0a23407..efb23fb 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -59,18 +59,18 @@ *(fdp) = -1; \ } while (0) -#ifndef __GNUC_PREREQ +#ifndef __GNUC_PREREQ__ # if defined __GNUC__ && defined __GNUC_MINOR__ -# define __GNUC_PREREQ(maj, min) \ +# define __GNUC_PREREQ__(maj, min) \ (__GNUC__ > (maj) || (__GNUC__ == (maj) && __GNUC_MINOR__ >= (min))) # else -# define __GNUC_PREREQ(maj, min) 0 +# define __GNUC_PREREQ__(maj, min) 0 # endif #endif #ifdef __printflike # define __v_printflike(f,a) __printflike(f,a) -#elif __GNUC_PREREQ(2, 95) || defined(__INTEL_COMPILER) +#elif __GNUC_PREREQ__(2, 95) || defined(__INTEL_COMPILER) # define __v_printflike(f,a) __attribute__((format(printf, f, a))) #else # define __v_printflike(f,a) From daghf at varnish-software.com Wed Oct 4 11:37:05 2017 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Wed, 4 Oct 2017 11:37:05 +0000 (UTC) Subject: [master] 58b40c4 Kill h2->[tr]_window Message-ID: <20171004113705.7AFE5633E1@lists.varnish-cache.org> commit 58b40c4f0c9d38951ec37ef713798fee33f427d2 Author: Dag Haavi Finstad Date: Thu Aug 17 15:33:11 2017 +0200 Kill h2->[tr]_window Maintain stream 0 window limits via h2->req0. diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index f04fb0b..3b5ef99 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -144,9 +144,6 @@ struct h2_sess { struct h2_req *mailcall; pthread_cond_t *cond; - int64_t r_window; - int64_t t_window; - struct sess *sess; int refcnt; uint32_t highest_stream; diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index e3da09f..32aaf6f 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -660,7 +660,7 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) Lck_Lock(&h2->sess->mtx); AZ(h2->mailcall); h2->mailcall = r2; - h2->r_window -= h2->rxf_len; + h2->req0->r_window -= h2->rxf_len; r2->r_window -= h2->rxf_len; // req_bodybytes accounted in CNT code. if (r2->cond) @@ -668,8 +668,8 @@ h2_rx_data(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) while (h2->mailcall != NULL && h2->error == 0 && r2->error == 0) AZ(Lck_CondWait(h2->cond, &h2->sess->mtx, 0)); wi = cache_param->h2_rx_window_increment; - if (h2->r_window < cache_param->h2_rx_window_low_water) { - h2->r_window += wi; + if (h2->req0->r_window < cache_param->h2_rx_window_low_water) { + h2->req0->r_window += wi; w1 = 1; } if (r2->r_window < cache_param->h2_rx_window_low_water) { From daghf at varnish-software.com Wed Oct 4 11:37:05 2017 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Wed, 4 Oct 2017 11:37:05 +0000 (UTC) Subject: [master] 3a86dd8 Introduce h2_send_get and h2_send_rel Message-ID: <20171004113705.95AE9633E3@lists.varnish-cache.org> commit 3a86dd8a7b9c0d3fb8b2794d9cc3eec99f24b2df Author: Dag Haavi Finstad Date: Thu Aug 17 15:51:42 2017 +0200 Introduce h2_send_get and h2_send_rel These are versions of H2_Send_Get/H2_Send_Rel that can be called while already holding the lock. diff --git a/bin/varnishd/http2/cache_http2_send.c b/bin/varnishd/http2/cache_http2_send.c index ecbda33..15cbb54 100644 --- a/bin/varnishd/http2/cache_http2_send.c +++ b/bin/varnishd/http2/cache_http2_send.c @@ -36,28 +36,41 @@ #include "vend.h" -void -H2_Send_Get(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) +static void +h2_send_get(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) { - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - Lck_Lock(&h2->sess->mtx); + Lck_AssertHeld(&h2->sess->mtx); r2->wrk = wrk; VTAILQ_INSERT_TAIL(&h2->txqueue, r2, tx_list); while (VTAILQ_FIRST(&h2->txqueue) != r2) AZ(Lck_CondWait(&wrk->cond, &h2->sess->mtx, 0)); r2->wrk = NULL; - Lck_Unlock(&h2->sess->mtx); } void -H2_Send_Rel(struct h2_sess *h2, const struct h2_req *r2) +H2_Send_Get(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) { + CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + Lck_Lock(&h2->sess->mtx); + h2_send_get(wrk, h2, r2); + Lck_Unlock(&h2->sess->mtx); +} + +static void +h2_send_rel(struct h2_sess *h2, const struct h2_req *r2) +{ + CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); + CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); + + Lck_AssertHeld(&h2->sess->mtx); assert(VTAILQ_FIRST(&h2->txqueue) == r2); VTAILQ_REMOVE(&h2->txqueue, r2, tx_list); r2 = VTAILQ_FIRST(&h2->txqueue); @@ -65,6 +78,16 @@ H2_Send_Rel(struct h2_sess *h2, const struct h2_req *r2) CHECK_OBJ_NOTNULL(r2->wrk, WORKER_MAGIC); AZ(pthread_cond_signal(&r2->wrk->cond)); } +} + +void +H2_Send_Rel(struct h2_sess *h2, const struct h2_req *r2) +{ + CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); + CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); + + Lck_Lock(&h2->sess->mtx); + h2_send_rel(h2, r2); Lck_Unlock(&h2->sess->mtx); } From daghf at varnish-software.com Wed Oct 4 11:37:05 2017 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Wed, 4 Oct 2017 11:37:05 +0000 (UTC) Subject: [master] 0d3044d Make senders wait in H2_Send when out of window credits Message-ID: <20171004113705.B8CB8633E9@lists.varnish-cache.org> commit 0d3044d4f3c18935049ad0e954edbacafa055557 Author: Dag Haavi Finstad Date: Tue Oct 3 16:59:03 2017 +0200 Make senders wait in H2_Send when out of window credits diff --git a/bin/varnishd/http2/cache_http2.h b/bin/varnishd/http2/cache_http2.h index 3b5ef99..fd720e6 100644 --- a/bin/varnishd/http2/cache_http2.h +++ b/bin/varnishd/http2/cache_http2.h @@ -217,7 +217,7 @@ h2_error H2_Send_Frame(struct worker *, const struct h2_sess *, h2_frame type, uint8_t flags, uint32_t len, uint32_t stream, const void *); -h2_error H2_Send(struct worker *, const struct h2_req *, +h2_error H2_Send(struct worker *, struct h2_req *, h2_frame type, uint8_t flags, uint32_t len, const void *); /* cache_http2_proto.c */ diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 32aaf6f..705aaa7 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -352,6 +352,10 @@ h2_rx_window_update(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) return (0); Lck_Lock(&h2->sess->mtx); r2->t_window += wu; + if (r2 == h2->req0) + AZ(pthread_cond_broadcast(h2->cond)); + else if (r2->cond != NULL) + AZ(pthread_cond_signal(r2->cond)); Lck_Unlock(&h2->sess->mtx); if (r2->t_window >= (1LLU << 31)) return (H2SE_FLOW_CONTROL_ERROR); diff --git a/bin/varnishd/http2/cache_http2_send.c b/bin/varnishd/http2/cache_http2_send.c index 15cbb54..0c5344d 100644 --- a/bin/varnishd/http2/cache_http2_send.c +++ b/bin/varnishd/http2/cache_http2_send.c @@ -54,7 +54,6 @@ h2_send_get(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) void H2_Send_Get(struct worker *wrk, struct h2_sess *h2, struct h2_req *r2) { - CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -149,14 +148,105 @@ H2_Send_Frame(struct worker *wrk, const struct h2_sess *h2, return (0); } +static int64_t +h2_win_limit(const struct h2_req *r2, const struct h2_sess *h2) +{ + int64_t m; + + CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); + CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); + CHECK_OBJ_NOTNULL(h2->req0, H2_REQ_MAGIC); + + Lck_AssertHeld(&h2->sess->mtx); + m = r2->t_window; + if (m > h2->req0->t_window) + m = h2->req0->t_window; + return (m); +} + +static void +h2_win_charge(struct h2_req *r2, struct h2_sess *h2, + uint32_t w) +{ + CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); + CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); + CHECK_OBJ_NOTNULL(h2->req0, H2_REQ_MAGIC); + + Lck_AssertHeld(&h2->sess->mtx); + r2->t_window -= w; + h2->req0->t_window -= w; +} + +static h2_error +h2_errcheck(const struct h2_req *r2, const struct h2_sess *h2) +{ + CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); + CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); + + if (r2->error) + return (r2->error); + if (h2->error && r2->stream > h2->goaway_last_stream) + return (h2->error); + return (0); +} + +static int64_t +h2_do_window(struct worker *wrk, struct h2_req *r2, + struct h2_sess *h2, int64_t wanted) +{ + int64_t w = 0; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(r2, H2_REQ_MAGIC); + CHECK_OBJ_NOTNULL(h2, H2_SESS_MAGIC); + + if (wanted == 0) + return (0); + + Lck_Lock(&h2->sess->mtx); + if (r2->t_window <= 0 || h2->req0->t_window <= 0) { + h2_send_rel(h2, r2); + while (r2->t_window <= 0 && h2_errcheck(r2, h2) == 0) { + r2->cond = &wrk->cond; + // XXX: timeout handling (subject to send_timeout?) + AZ(Lck_CondWait(r2->cond, &h2->sess->mtx, 0)); + r2->cond = NULL; + } + while (h2->req0->t_window <= 0 && h2_errcheck(r2, h2) == 0) { + // XXX: timeout handling + AZ(Lck_CondWait(h2->cond, &h2->sess->mtx, 0)); + } + + if (h2_errcheck(r2, h2) == 0) { + w = h2_win_limit(r2, h2); + if (w > wanted) + w = wanted; + h2_win_charge(r2, h2, w); + assert (w > 0); + } + h2_send_get(wrk, h2, r2); + } + + if (w == 0 && h2_errcheck(r2, h2) == 0) { + assert(r2->t_window > 0); + assert(h2->req0->t_window > 0); + w = h2_win_limit(r2, h2); + if (w > wanted) + w = wanted; + h2_win_charge(r2, h2, w); + assert (w > 0); + } + Lck_Unlock(&h2->sess->mtx); + return (w); +} + /* * This is the per-stream frame sender. - * XXX: windows * XXX: priority */ h2_error -H2_Send(struct worker *wrk, const struct h2_req *r2, +H2_Send(struct worker *wrk, struct h2_req *r2, h2_frame ftyp, uint8_t flags, uint32_t len, const void *ptr) { h2_error retval; @@ -173,11 +263,9 @@ H2_Send(struct worker *wrk, const struct h2_req *r2, assert(VTAILQ_FIRST(&h2->txqueue) == r2); - if (r2->error) - return (r2->error); - - if (h2->error && r2->stream > h2->goaway_last_stream) - return (h2->error); + retval = h2_errcheck(r2, h2); + if (retval) + return (retval); AN(ftyp); AZ(flags & ~(ftyp->flags)); @@ -189,7 +277,18 @@ H2_Send(struct worker *wrk, const struct h2_req *r2, Lck_Lock(&h2->sess->mtx); mfs = h2->remote_settings.max_frame_size; Lck_Unlock(&h2->sess->mtx); - if (len < mfs) { + + if (ftyp->respect_window) { + tf = h2_do_window(wrk, r2, h2, + (len > mfs) ? mfs : len); + retval = h2_errcheck(r2, h2); + if (retval) + return (retval); + assert(VTAILQ_FIRST(&h2->txqueue) == r2); + } else + tf = mfs; + + if (len <= tf) { retval = H2_Send_Frame(wrk, h2, ftyp, flags, len, r2->stream, ptr); } else { @@ -199,11 +298,22 @@ H2_Send(struct worker *wrk, const struct h2_req *r2, flags &= ~ftyp->final_flags; do { AN(ftyp->continuation); - tf = mfs; + if (!ftyp->respect_window) + tf = mfs; + if (ftyp->respect_window && p != ptr) { + tf = h2_do_window(wrk, r2, h2, + (len > mfs) ? mfs : len); + retval = h2_errcheck(r2, h2); + if (retval) + return (retval); + assert(VTAILQ_FIRST(&h2->txqueue) == r2); + } if (tf < len) { retval = H2_Send_Frame(wrk, h2, ftyp, flags, tf, r2->stream, p); } else { + if (ftyp->respect_window) + assert(tf == len); tf = len; retval = H2_Send_Frame(wrk, h2, ftyp, final_flags, tf, r2->stream, p); diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c index 567e908..63f6f95 100644 --- a/bin/varnishd/http2/cache_http2_session.c +++ b/bin/varnishd/http2/cache_http2_session.c @@ -326,7 +326,7 @@ h2_new_session(struct worker *wrk, void *arg) if (r2->cond != NULL) AZ(pthread_cond_signal(r2->cond)); } - AZ(pthread_cond_signal(h2->cond)); + AZ(pthread_cond_broadcast(h2->cond)); while (1) { again = 0; VTAILQ_FOREACH_SAFE(r2, &h2->streams, list, r22) { From phk at FreeBSD.org Thu Oct 5 08:59:10 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 5 Oct 2017 08:59:10 +0000 (UTC) Subject: [master] d02a5cf Rename the tcp-pool code, it will not be backend specific in the future. Message-ID: <20171005085910.182106331E@lists.varnish-cache.org> commit d02a5cf3990b334cc740c57dc212d26f8f747fdb Author: Poul-Henning Kamp Date: Thu Oct 5 08:14:10 2017 +0000 Rename the tcp-pool code, it will not be backend specific in the future. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index a1dc890..7e5ce45 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -13,7 +13,6 @@ varnishd_SOURCES = \ cache/cache_backend.c \ cache/cache_backend_cfg.c \ cache/cache_backend_probe.c \ - cache/cache_backend_tcp.c \ cache/cache_ban.c \ cache/cache_ban_build.c \ cache/cache_ban_lurker.c \ @@ -43,6 +42,7 @@ varnishd_SOURCES = \ cache/cache_range.c \ cache/cache_session.c \ cache/cache_shmlog.c \ + cache/cache_tcp_pool.c \ cache/cache_vary.c \ cache/cache_vcl.c \ cache/cache_vrt.c \ diff --git a/bin/varnishd/cache/cache_backend_tcp.c b/bin/varnishd/cache/cache_backend_tcp.c deleted file mode 100644 index ce3f052..0000000 --- a/bin/varnishd/cache/cache_backend_tcp.c +++ /dev/null @@ -1,425 +0,0 @@ -/*- - * Copyright (c) 2015 Varnish Software AS - * All rights reserved. - * - * Author: Poul-Henning Kamp - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * TCP connection pools. - * - * These are really a lot more general than just backends, but backends - * are all we use them for, so they live here for now. - * - */ - -#include "config.h" - -#include - -#include "cache_varnishd.h" - -#include "vrt.h" -#include "vsa.h" -#include "vtcp.h" -#include "vtim.h" -#include "waiter/waiter.h" - -#include "cache_director.h" -#include "cache_backend.h" -#include "cache_pool.h" - -struct tcp_pool { - unsigned magic; -#define TCP_POOL_MAGIC 0x28b0e42a - - char *name; - struct suckaddr *ip4; - struct suckaddr *ip6; - - VTAILQ_ENTRY(tcp_pool) list; - int refcnt; - struct lock mtx; - - VTAILQ_HEAD(, vbc) connlist; - int n_conn; - - VTAILQ_HEAD(, vbc) killlist; - int n_kill; - - int n_used; - -}; - -static struct lock pools_mtx; -static VTAILQ_HEAD(, tcp_pool) pools = VTAILQ_HEAD_INITIALIZER(pools); - -/*-------------------------------------------------------------------- - * Waiter-handler - */ - -static void __match_proto__(waiter_handle_f) -tcp_handle(struct waited *w, enum wait_event ev, double now) -{ - struct vbc *vbc; - struct tcp_pool *tp; - - CAST_OBJ_NOTNULL(vbc, w->priv1, VBC_MAGIC); - (void)ev; - (void)now; - CHECK_OBJ_NOTNULL(vbc->tcp_pool, TCP_POOL_MAGIC); - tp = vbc->tcp_pool; - - Lck_Lock(&tp->mtx); - - switch (vbc->state) { - case VBC_STATE_STOLEN: - vbc->state = VBC_STATE_USED; - VTAILQ_REMOVE(&tp->connlist, vbc, list); - AN(vbc->cond); - AZ(pthread_cond_signal(vbc->cond)); - break; - case VBC_STATE_AVAIL: - VTCP_close(&vbc->fd); - VTAILQ_REMOVE(&tp->connlist, vbc, list); - tp->n_conn--; - FREE_OBJ(vbc); - break; - case VBC_STATE_CLEANUP: - VTCP_close(&vbc->fd); - tp->n_kill--; - VTAILQ_REMOVE(&tp->killlist, vbc, list); - memset(vbc, 0x11, sizeof *vbc); - free(vbc); - break; - default: - WRONG("Wrong vbc state"); - } - Lck_Unlock(&tp->mtx); -} - -/*-------------------------------------------------------------------- - * Reference a TCP pool given by {ip4, ip6} pair. Create if it - * doesn't exist already. - */ - -struct tcp_pool * -VBT_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6) -{ - struct tcp_pool *tp; - - Lck_Lock(&pools_mtx); - VTAILQ_FOREACH(tp, &pools, list) { - assert(tp->refcnt > 0); - if (ip4 == NULL) { - if (tp->ip4 != NULL) - continue; - } else { - if (tp->ip4 == NULL) - continue; - if (VSA_Compare(ip4, tp->ip4)) - continue; - } - if (ip6 == NULL) { - if (tp->ip6 != NULL) - continue; - } else { - if (tp->ip6 == NULL) - continue; - if (VSA_Compare(ip6, tp->ip6)) - continue; - } - tp->refcnt++; - Lck_Unlock(&pools_mtx); - return (tp); - } - Lck_Unlock(&pools_mtx); - - ALLOC_OBJ(tp, TCP_POOL_MAGIC); - AN(tp); - if (ip4 != NULL) - tp->ip4 = VSA_Clone(ip4); - if (ip6 != NULL) - tp->ip6 = VSA_Clone(ip6); - tp->refcnt = 1; - Lck_New(&tp->mtx, lck_backend_tcp); - VTAILQ_INIT(&tp->connlist); - VTAILQ_INIT(&tp->killlist); - - Lck_Lock(&pools_mtx); - VTAILQ_INSERT_HEAD(&pools, tp, list); - Lck_Unlock(&pools_mtx); - - return (tp); -} - -/*-------------------------------------------------------------------- - * Release TCP pool, destroy if last reference. - */ - -void -VBT_Rel(struct tcp_pool **tpp) -{ - struct tcp_pool *tp; - struct vbc *vbc, *vbc2; - - TAKE_OBJ_NOTNULL(tp, tpp, TCP_POOL_MAGIC); - - Lck_Lock(&pools_mtx); - assert(tp->refcnt > 0); - if (--tp->refcnt > 0) { - Lck_Unlock(&pools_mtx); - return; - } - AZ(tp->n_used); - VTAILQ_REMOVE(&pools, tp, list); - Lck_Unlock(&pools_mtx); - - free(tp->name); - free(tp->ip4); - free(tp->ip6); - Lck_Lock(&tp->mtx); - VTAILQ_FOREACH_SAFE(vbc, &tp->connlist, list, vbc2) { - VTAILQ_REMOVE(&tp->connlist, vbc, list); - tp->n_conn--; - assert(vbc->state == VBC_STATE_AVAIL); - vbc->state = VBC_STATE_CLEANUP; - (void)shutdown(vbc->fd, SHUT_WR); - VTAILQ_INSERT_TAIL(&tp->killlist, vbc, list); - tp->n_kill++; - } - while (tp->n_kill) { - Lck_Unlock(&tp->mtx); - (void)usleep(20000); - Lck_Lock(&tp->mtx); - } - Lck_Unlock(&tp->mtx); - Lck_Delete(&tp->mtx); - AZ(tp->n_conn); - AZ(tp->n_kill); - - FREE_OBJ(tp); -} - -/*-------------------------------------------------------------------- - * Open a new connection from pool. This is a distinct function since - * probing cannot use a recycled connection. - */ - -int -VBT_Open(const struct tcp_pool *tp, double tmo, const struct suckaddr **sa) -{ - int s; - int msec; - - CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); - - msec = (int)floor(tmo * 1000.0); - if (cache_param->prefer_ipv6) { - *sa = tp->ip6; - s = VTCP_connect(tp->ip6, msec); - if (s >= 0) - return (s); - } - *sa = tp->ip4; - s = VTCP_connect(tp->ip4, msec); - if (s < 0 && !cache_param->prefer_ipv6) { - *sa = tp->ip6; - s = VTCP_connect(tp->ip6, msec); - } - return (s); -} - -/*-------------------------------------------------------------------- - * Recycle a connection. - */ - -void -VBT_Recycle(const struct worker *wrk, struct tcp_pool *tp, struct vbc **vbcp) -{ - struct vbc *vbc; - int i = 0; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); - vbc = *vbcp; - *vbcp = NULL; - CHECK_OBJ_NOTNULL(vbc, VBC_MAGIC); - - assert(vbc->state == VBC_STATE_USED); - assert(vbc->fd > 0); - - Lck_Lock(&tp->mtx); - tp->n_used--; - - vbc->waited->priv1 = vbc; - vbc->waited->fd = vbc->fd; - vbc->waited->idle = VTIM_real(); - vbc->state = VBC_STATE_AVAIL; - vbc->waited->func = tcp_handle; - vbc->waited->tmo = &cache_param->backend_idle_timeout; - if (Wait_Enter(wrk->pool->waiter, vbc->waited)) { - VTCP_close(&vbc->fd); - memset(vbc, 0x33, sizeof *vbc); - free(vbc); - // XXX: stats - vbc = NULL; - } else { - VTAILQ_INSERT_HEAD(&tp->connlist, vbc, list); - i++; - } - - if (vbc != NULL) - tp->n_conn++; - Lck_Unlock(&tp->mtx); - - if (i && DO_DEBUG(DBG_VTC_MODE)) { - /* - * In varnishtest we do not have the luxury of using - * multiple backend connections, so whenever we end up - * in the "pending" case, take a short nap to let the - * waiter catch up and put the vbc back into circulations. - * - * In particular ESI:include related tests suffer random - * failures without this. - * - * In normal operation, the only effect is that we will - * have N+1 backend connections rather than N, which is - * entirely harmless. - */ - (void)usleep(10000); - } -} - -/*-------------------------------------------------------------------- - * Close a connection. - */ - -void -VBT_Close(struct tcp_pool *tp, struct vbc **vbcp) -{ - struct vbc *vbc; - - CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); - vbc = *vbcp; - *vbcp = NULL; - CHECK_OBJ_NOTNULL(vbc, VBC_MAGIC); - - assert(vbc->state == VBC_STATE_USED); - assert(vbc->fd > 0); - - Lck_Lock(&tp->mtx); - tp->n_used--; - if (vbc->state == VBC_STATE_STOLEN) { - (void)shutdown(vbc->fd, SHUT_WR); - vbc->state = VBC_STATE_CLEANUP; - VTAILQ_INSERT_HEAD(&tp->killlist, vbc, list); - tp->n_kill++; - } else { - assert(vbc->state == VBC_STATE_USED); - VTCP_close(&vbc->fd); - memset(vbc, 0x44, sizeof *vbc); - free(vbc); - } - Lck_Unlock(&tp->mtx); -} - -/*-------------------------------------------------------------------- - * Get a connection - */ - -struct vbc * -VBT_Get(struct tcp_pool *tp, double tmo, const struct backend *be, - struct worker *wrk) -{ - struct vbc *vbc; - - CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); - CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - - Lck_Lock(&tp->mtx); - vbc = VTAILQ_FIRST(&tp->connlist); - CHECK_OBJ_ORNULL(vbc, VBC_MAGIC); - if (vbc == NULL || vbc->state == VBC_STATE_STOLEN) - vbc = NULL; - else { - assert(vbc->tcp_pool == tp); - assert(vbc->state == VBC_STATE_AVAIL); - VTAILQ_REMOVE(&tp->connlist, vbc, list); - VTAILQ_INSERT_TAIL(&tp->connlist, vbc, list); - tp->n_conn--; - VSC_C_main->backend_reuse++; - vbc->state = VBC_STATE_STOLEN; - vbc->cond = &wrk->cond; - } - tp->n_used++; // Opening mostly works - Lck_Unlock(&tp->mtx); - - if (vbc != NULL) - return (vbc); - - ALLOC_OBJ(vbc, VBC_MAGIC); - AN(vbc); - INIT_OBJ(vbc->waited, WAITED_MAGIC); - vbc->state = VBC_STATE_USED; - vbc->tcp_pool = tp; - vbc->fd = VBT_Open(tp, tmo, &vbc->addr); - if (vbc->fd < 0) { - FREE_OBJ(vbc); - Lck_Lock(&tp->mtx); - tp->n_used--; // Nope, didn't work after all. - Lck_Unlock(&tp->mtx); - } else - VSC_C_main->backend_conn++; - - return (vbc); -} - -/*-------------------------------------------------------------------- - */ - -void -VBT_Wait(struct worker *wrk, struct vbc *vbc) -{ - struct tcp_pool *tp; - - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(vbc, VBC_MAGIC); - tp = vbc->tcp_pool; - CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); - assert(vbc->cond == &wrk->cond); - Lck_Lock(&tp->mtx); - while (vbc->state == VBC_STATE_STOLEN) - AZ(Lck_CondWait(&wrk->cond, &tp->mtx, 0)); - assert(vbc->state == VBC_STATE_USED); - vbc->cond = NULL; - Lck_Unlock(&tp->mtx); -} - -/*--------------------------------------------------------------------*/ - -void -VBT_Init(void) -{ - Lck_New(&pools_mtx, lck_backend); -} diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c new file mode 100644 index 0000000..ce3f052 --- /dev/null +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -0,0 +1,425 @@ +/*- + * Copyright (c) 2015 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * TCP connection pools. + * + * These are really a lot more general than just backends, but backends + * are all we use them for, so they live here for now. + * + */ + +#include "config.h" + +#include + +#include "cache_varnishd.h" + +#include "vrt.h" +#include "vsa.h" +#include "vtcp.h" +#include "vtim.h" +#include "waiter/waiter.h" + +#include "cache_director.h" +#include "cache_backend.h" +#include "cache_pool.h" + +struct tcp_pool { + unsigned magic; +#define TCP_POOL_MAGIC 0x28b0e42a + + char *name; + struct suckaddr *ip4; + struct suckaddr *ip6; + + VTAILQ_ENTRY(tcp_pool) list; + int refcnt; + struct lock mtx; + + VTAILQ_HEAD(, vbc) connlist; + int n_conn; + + VTAILQ_HEAD(, vbc) killlist; + int n_kill; + + int n_used; + +}; + +static struct lock pools_mtx; +static VTAILQ_HEAD(, tcp_pool) pools = VTAILQ_HEAD_INITIALIZER(pools); + +/*-------------------------------------------------------------------- + * Waiter-handler + */ + +static void __match_proto__(waiter_handle_f) +tcp_handle(struct waited *w, enum wait_event ev, double now) +{ + struct vbc *vbc; + struct tcp_pool *tp; + + CAST_OBJ_NOTNULL(vbc, w->priv1, VBC_MAGIC); + (void)ev; + (void)now; + CHECK_OBJ_NOTNULL(vbc->tcp_pool, TCP_POOL_MAGIC); + tp = vbc->tcp_pool; + + Lck_Lock(&tp->mtx); + + switch (vbc->state) { + case VBC_STATE_STOLEN: + vbc->state = VBC_STATE_USED; + VTAILQ_REMOVE(&tp->connlist, vbc, list); + AN(vbc->cond); + AZ(pthread_cond_signal(vbc->cond)); + break; + case VBC_STATE_AVAIL: + VTCP_close(&vbc->fd); + VTAILQ_REMOVE(&tp->connlist, vbc, list); + tp->n_conn--; + FREE_OBJ(vbc); + break; + case VBC_STATE_CLEANUP: + VTCP_close(&vbc->fd); + tp->n_kill--; + VTAILQ_REMOVE(&tp->killlist, vbc, list); + memset(vbc, 0x11, sizeof *vbc); + free(vbc); + break; + default: + WRONG("Wrong vbc state"); + } + Lck_Unlock(&tp->mtx); +} + +/*-------------------------------------------------------------------- + * Reference a TCP pool given by {ip4, ip6} pair. Create if it + * doesn't exist already. + */ + +struct tcp_pool * +VBT_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6) +{ + struct tcp_pool *tp; + + Lck_Lock(&pools_mtx); + VTAILQ_FOREACH(tp, &pools, list) { + assert(tp->refcnt > 0); + if (ip4 == NULL) { + if (tp->ip4 != NULL) + continue; + } else { + if (tp->ip4 == NULL) + continue; + if (VSA_Compare(ip4, tp->ip4)) + continue; + } + if (ip6 == NULL) { + if (tp->ip6 != NULL) + continue; + } else { + if (tp->ip6 == NULL) + continue; + if (VSA_Compare(ip6, tp->ip6)) + continue; + } + tp->refcnt++; + Lck_Unlock(&pools_mtx); + return (tp); + } + Lck_Unlock(&pools_mtx); + + ALLOC_OBJ(tp, TCP_POOL_MAGIC); + AN(tp); + if (ip4 != NULL) + tp->ip4 = VSA_Clone(ip4); + if (ip6 != NULL) + tp->ip6 = VSA_Clone(ip6); + tp->refcnt = 1; + Lck_New(&tp->mtx, lck_backend_tcp); + VTAILQ_INIT(&tp->connlist); + VTAILQ_INIT(&tp->killlist); + + Lck_Lock(&pools_mtx); + VTAILQ_INSERT_HEAD(&pools, tp, list); + Lck_Unlock(&pools_mtx); + + return (tp); +} + +/*-------------------------------------------------------------------- + * Release TCP pool, destroy if last reference. + */ + +void +VBT_Rel(struct tcp_pool **tpp) +{ + struct tcp_pool *tp; + struct vbc *vbc, *vbc2; + + TAKE_OBJ_NOTNULL(tp, tpp, TCP_POOL_MAGIC); + + Lck_Lock(&pools_mtx); + assert(tp->refcnt > 0); + if (--tp->refcnt > 0) { + Lck_Unlock(&pools_mtx); + return; + } + AZ(tp->n_used); + VTAILQ_REMOVE(&pools, tp, list); + Lck_Unlock(&pools_mtx); + + free(tp->name); + free(tp->ip4); + free(tp->ip6); + Lck_Lock(&tp->mtx); + VTAILQ_FOREACH_SAFE(vbc, &tp->connlist, list, vbc2) { + VTAILQ_REMOVE(&tp->connlist, vbc, list); + tp->n_conn--; + assert(vbc->state == VBC_STATE_AVAIL); + vbc->state = VBC_STATE_CLEANUP; + (void)shutdown(vbc->fd, SHUT_WR); + VTAILQ_INSERT_TAIL(&tp->killlist, vbc, list); + tp->n_kill++; + } + while (tp->n_kill) { + Lck_Unlock(&tp->mtx); + (void)usleep(20000); + Lck_Lock(&tp->mtx); + } + Lck_Unlock(&tp->mtx); + Lck_Delete(&tp->mtx); + AZ(tp->n_conn); + AZ(tp->n_kill); + + FREE_OBJ(tp); +} + +/*-------------------------------------------------------------------- + * Open a new connection from pool. This is a distinct function since + * probing cannot use a recycled connection. + */ + +int +VBT_Open(const struct tcp_pool *tp, double tmo, const struct suckaddr **sa) +{ + int s; + int msec; + + CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); + + msec = (int)floor(tmo * 1000.0); + if (cache_param->prefer_ipv6) { + *sa = tp->ip6; + s = VTCP_connect(tp->ip6, msec); + if (s >= 0) + return (s); + } + *sa = tp->ip4; + s = VTCP_connect(tp->ip4, msec); + if (s < 0 && !cache_param->prefer_ipv6) { + *sa = tp->ip6; + s = VTCP_connect(tp->ip6, msec); + } + return (s); +} + +/*-------------------------------------------------------------------- + * Recycle a connection. + */ + +void +VBT_Recycle(const struct worker *wrk, struct tcp_pool *tp, struct vbc **vbcp) +{ + struct vbc *vbc; + int i = 0; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); + vbc = *vbcp; + *vbcp = NULL; + CHECK_OBJ_NOTNULL(vbc, VBC_MAGIC); + + assert(vbc->state == VBC_STATE_USED); + assert(vbc->fd > 0); + + Lck_Lock(&tp->mtx); + tp->n_used--; + + vbc->waited->priv1 = vbc; + vbc->waited->fd = vbc->fd; + vbc->waited->idle = VTIM_real(); + vbc->state = VBC_STATE_AVAIL; + vbc->waited->func = tcp_handle; + vbc->waited->tmo = &cache_param->backend_idle_timeout; + if (Wait_Enter(wrk->pool->waiter, vbc->waited)) { + VTCP_close(&vbc->fd); + memset(vbc, 0x33, sizeof *vbc); + free(vbc); + // XXX: stats + vbc = NULL; + } else { + VTAILQ_INSERT_HEAD(&tp->connlist, vbc, list); + i++; + } + + if (vbc != NULL) + tp->n_conn++; + Lck_Unlock(&tp->mtx); + + if (i && DO_DEBUG(DBG_VTC_MODE)) { + /* + * In varnishtest we do not have the luxury of using + * multiple backend connections, so whenever we end up + * in the "pending" case, take a short nap to let the + * waiter catch up and put the vbc back into circulations. + * + * In particular ESI:include related tests suffer random + * failures without this. + * + * In normal operation, the only effect is that we will + * have N+1 backend connections rather than N, which is + * entirely harmless. + */ + (void)usleep(10000); + } +} + +/*-------------------------------------------------------------------- + * Close a connection. + */ + +void +VBT_Close(struct tcp_pool *tp, struct vbc **vbcp) +{ + struct vbc *vbc; + + CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); + vbc = *vbcp; + *vbcp = NULL; + CHECK_OBJ_NOTNULL(vbc, VBC_MAGIC); + + assert(vbc->state == VBC_STATE_USED); + assert(vbc->fd > 0); + + Lck_Lock(&tp->mtx); + tp->n_used--; + if (vbc->state == VBC_STATE_STOLEN) { + (void)shutdown(vbc->fd, SHUT_WR); + vbc->state = VBC_STATE_CLEANUP; + VTAILQ_INSERT_HEAD(&tp->killlist, vbc, list); + tp->n_kill++; + } else { + assert(vbc->state == VBC_STATE_USED); + VTCP_close(&vbc->fd); + memset(vbc, 0x44, sizeof *vbc); + free(vbc); + } + Lck_Unlock(&tp->mtx); +} + +/*-------------------------------------------------------------------- + * Get a connection + */ + +struct vbc * +VBT_Get(struct tcp_pool *tp, double tmo, const struct backend *be, + struct worker *wrk) +{ + struct vbc *vbc; + + CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); + CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + + Lck_Lock(&tp->mtx); + vbc = VTAILQ_FIRST(&tp->connlist); + CHECK_OBJ_ORNULL(vbc, VBC_MAGIC); + if (vbc == NULL || vbc->state == VBC_STATE_STOLEN) + vbc = NULL; + else { + assert(vbc->tcp_pool == tp); + assert(vbc->state == VBC_STATE_AVAIL); + VTAILQ_REMOVE(&tp->connlist, vbc, list); + VTAILQ_INSERT_TAIL(&tp->connlist, vbc, list); + tp->n_conn--; + VSC_C_main->backend_reuse++; + vbc->state = VBC_STATE_STOLEN; + vbc->cond = &wrk->cond; + } + tp->n_used++; // Opening mostly works + Lck_Unlock(&tp->mtx); + + if (vbc != NULL) + return (vbc); + + ALLOC_OBJ(vbc, VBC_MAGIC); + AN(vbc); + INIT_OBJ(vbc->waited, WAITED_MAGIC); + vbc->state = VBC_STATE_USED; + vbc->tcp_pool = tp; + vbc->fd = VBT_Open(tp, tmo, &vbc->addr); + if (vbc->fd < 0) { + FREE_OBJ(vbc); + Lck_Lock(&tp->mtx); + tp->n_used--; // Nope, didn't work after all. + Lck_Unlock(&tp->mtx); + } else + VSC_C_main->backend_conn++; + + return (vbc); +} + +/*-------------------------------------------------------------------- + */ + +void +VBT_Wait(struct worker *wrk, struct vbc *vbc) +{ + struct tcp_pool *tp; + + CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); + CHECK_OBJ_NOTNULL(vbc, VBC_MAGIC); + tp = vbc->tcp_pool; + CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); + assert(vbc->cond == &wrk->cond); + Lck_Lock(&tp->mtx); + while (vbc->state == VBC_STATE_STOLEN) + AZ(Lck_CondWait(&wrk->cond, &tp->mtx, 0)); + assert(vbc->state == VBC_STATE_USED); + vbc->cond = NULL; + Lck_Unlock(&tp->mtx); +} + +/*--------------------------------------------------------------------*/ + +void +VBT_Init(void) +{ + Lck_New(&pools_mtx, lck_backend); +} From phk at FreeBSD.org Thu Oct 5 08:59:10 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 5 Oct 2017 08:59:10 +0000 (UTC) Subject: [master] fe362e6 Divorce tcp_pools from backends Message-ID: <20171005085910.319DE63320@lists.varnish-cache.org> commit fe362e60229fd3b6567113e9392627a39e7aeb49 Author: Poul-Henning Kamp Date: Thu Oct 5 08:31:48 2017 +0000 Divorce tcp_pools from backends diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 7e5ce45..895a73c 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -119,6 +119,7 @@ noinst_HEADERS = \ cache/cache_obj.h \ cache/cache_objhead.h \ cache/cache_pool.h \ + cache/cache_tcp_pool.h \ cache/cache_transport.h \ cache/cache_varnishd.h \ cache/cache_vgz.h \ diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 95f2ff2..6473be0 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -43,6 +43,7 @@ #include "cache_director.h" #include "cache_backend.h" +#include "cache_tcp_pool.h" #include "cache_transport.h" #include "http1/cache_http1.h" @@ -95,7 +96,7 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo) bo->htc->doclose = SC_NULL; FIND_TMO(connect_timeout, tmod, bo, bp); - vbc = VBT_Get(bp->tcp_pool, tmod, bp, wrk); + vbc = VBT_Get(bp->tcp_pool, tmod, wrk); if (vbc == NULL) { // XXX: Per backend stats ? VSC_C_main->backend_fail++; diff --git a/bin/varnishd/cache/cache_backend.h b/bin/varnishd/cache/cache_backend.h index 15cd260..58c6d83 100644 --- a/bin/varnishd/cache/cache_backend.h +++ b/bin/varnishd/cache/cache_backend.h @@ -77,27 +77,6 @@ struct backend { }; /*--------------------------------------------------------------------- - * Backend connection -- private to cache_backend*.c - */ - -struct vbc { - unsigned magic; -#define VBC_MAGIC 0x0c5e6592 - int fd; - VTAILQ_ENTRY(vbc) list; - const struct suckaddr *addr; - uint8_t state; -#define VBC_STATE_AVAIL (1<<0) -#define VBC_STATE_USED (1<<1) -#define VBC_STATE_STOLEN (1<<2) -#define VBC_STATE_CLEANUP (1<<3) - struct waited waited[1]; - struct tcp_pool *tcp_pool; - - pthread_cond_t *cond; -}; - -/*--------------------------------------------------------------------- * Prototypes */ @@ -119,17 +98,6 @@ void VBP_Remove(struct backend *b); void VBP_Control(const struct backend *b, int stop); void VBP_Status(struct cli *cli, const struct backend *, int details); -/* cache_backend_tcp.c */ -struct tcp_pool *VBT_Ref(const struct suckaddr *ip4, - const struct suckaddr *ip6); -void VBT_Rel(struct tcp_pool **tpp); -int VBT_Open(const struct tcp_pool *tp, double tmo, const struct suckaddr **sa); -void VBT_Recycle(const struct worker *, struct tcp_pool *, struct vbc **); -void VBT_Close(struct tcp_pool *tp, struct vbc **vbc); -struct vbc *VBT_Get(struct tcp_pool *, double tmo, const struct backend *, - struct worker *); -void VBT_Wait(struct worker *, struct vbc *); - /* cache_vcl.c */ int VCL_AddBackend(struct vcl *, struct backend *); void VCL_DelBackend(struct backend *); diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c index d7f126d..6732aca 100644 --- a/bin/varnishd/cache/cache_backend_cfg.c +++ b/bin/varnishd/cache/cache_backend_cfg.c @@ -46,6 +46,7 @@ #include "cache_director.h" #include "cache_backend.h" +#include "cache_tcp_pool.h" #include "VSC_vbe.h" diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 55f7e27..028fb33 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -53,6 +53,7 @@ #include "cache_director.h" #include "cache_backend.h" +#include "cache_tcp_pool.h" /* Default averaging rate, we want something pretty responsive */ #define AVG_RATE 4 diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index ce3f052..bd8d9ff 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -27,9 +27,6 @@ * * TCP connection pools. * - * These are really a lot more general than just backends, but backends - * are all we use them for, so they live here for now. - * */ #include "config.h" @@ -44,8 +41,7 @@ #include "vtim.h" #include "waiter/waiter.h" -#include "cache_director.h" -#include "cache_backend.h" +#include "cache_tcp_pool.h" #include "cache_pool.h" struct tcp_pool { @@ -348,13 +344,11 @@ VBT_Close(struct tcp_pool *tp, struct vbc **vbcp) */ struct vbc * -VBT_Get(struct tcp_pool *tp, double tmo, const struct backend *be, - struct worker *wrk) +VBT_Get(struct tcp_pool *tp, double tmo, struct worker *wrk) { struct vbc *vbc; CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); - CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); Lck_Lock(&tp->mtx); diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h new file mode 100644 index 0000000..78bfd08 --- /dev/null +++ b/bin/varnishd/cache/cache_tcp_pool.h @@ -0,0 +1,63 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2015 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * TCP connection pools + * + */ + +struct tcp_pool; + +struct vbc { + unsigned magic; +#define VBC_MAGIC 0x0c5e6592 + int fd; + VTAILQ_ENTRY(vbc) list; + const struct suckaddr *addr; + uint8_t state; +#define VBC_STATE_AVAIL (1<<0) +#define VBC_STATE_USED (1<<1) +#define VBC_STATE_STOLEN (1<<2) +#define VBC_STATE_CLEANUP (1<<3) + struct waited waited[1]; + struct tcp_pool *tcp_pool; + + pthread_cond_t *cond; +}; + +/*--------------------------------------------------------------------- + * Prototypes + */ + +/* cache_tcp_pool.c */ +struct tcp_pool *VBT_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6); +void VBT_Rel(struct tcp_pool **tpp); +int VBT_Open(const struct tcp_pool *tp, double tmo, const struct suckaddr **sa); +void VBT_Recycle(const struct worker *, struct tcp_pool *, struct vbc **); +void VBT_Close(struct tcp_pool *tp, struct vbc **vbc); +struct vbc *VBT_Get(struct tcp_pool *, double tmo, struct worker *); +void VBT_Wait(struct worker *, struct vbc *); From phk at FreeBSD.org Thu Oct 5 08:59:10 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 5 Oct 2017 08:59:10 +0000 (UTC) Subject: [master] 56f4586 Rename tcp_pools to VTP Message-ID: <20171005085910.511B963323@lists.varnish-cache.org> commit 56f45866614bc99cd6a287a82e6029a351c9d088 Author: Poul-Henning Kamp Date: Thu Oct 5 08:45:36 2017 +0000 Rename tcp_pools to VTP diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 6473be0..a642304 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -63,10 +63,10 @@ * Get a connection to the backend */ -static struct vbc * +static struct vtp * vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo) { - struct vbc *vbc; + struct vtp *vtp; double tmod; char abuf1[VTCP_ADDRBUFSIZE], abuf2[VTCP_ADDRBUFSIZE]; char pbuf1[VTCP_PORTBUFSIZE], pbuf2[VTCP_PORTBUFSIZE]; @@ -96,16 +96,16 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo) bo->htc->doclose = SC_NULL; FIND_TMO(connect_timeout, tmod, bo, bp); - vbc = VBT_Get(bp->tcp_pool, tmod, wrk); - if (vbc == NULL) { + vtp = VTP_Get(bp->tcp_pool, tmod, wrk); + if (vtp == NULL) { // XXX: Per backend stats ? VSC_C_main->backend_fail++; bo->htc = NULL; return (NULL); } - assert(vbc->fd >= 0); - AN(vbc->addr); + assert(vtp->fd >= 0); + AN(vtp->addr); Lck_Lock(&bp->mtx); bp->n_conn++; @@ -114,21 +114,21 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo) Lck_Unlock(&bp->mtx); if (bp->proxy_header != 0) - VPX_Send_Proxy(vbc->fd, bp->proxy_header, bo->sp); + VPX_Send_Proxy(vtp->fd, bp->proxy_header, bo->sp); - VTCP_myname(vbc->fd, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1); - VTCP_hisname(vbc->fd, abuf2, sizeof abuf2, pbuf2, sizeof pbuf2); + VTCP_myname(vtp->fd, abuf1, sizeof abuf1, pbuf1, sizeof pbuf1); + VTCP_hisname(vtp->fd, abuf2, sizeof abuf2, pbuf2, sizeof pbuf2); VSLb(bo->vsl, SLT_BackendOpen, "%d %s %s %s %s %s", - vbc->fd, bp->display_name, abuf2, pbuf2, abuf1, pbuf1); + vtp->fd, bp->display_name, abuf2, pbuf2, abuf1, pbuf1); INIT_OBJ(bo->htc, HTTP_CONN_MAGIC); - bo->htc->priv = vbc; - bo->htc->rfd = &vbc->fd; + bo->htc->priv = vtp; + bo->htc->rfd = &vtp->fd; FIND_TMO(first_byte_timeout, bo->htc->first_byte_timeout, bo, bp); FIND_TMO(between_bytes_timeout, bo->htc->between_bytes_timeout, bo, bp); - return (vbc); + return (vtp); } static unsigned __match_proto__(vdi_healthy_f) @@ -148,7 +148,7 @@ vbe_dir_finish(const struct director *d, struct worker *wrk, struct busyobj *bo) { struct backend *bp; - struct vbc *vbc; + struct vtp *vtp; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -156,21 +156,21 @@ vbe_dir_finish(const struct director *d, struct worker *wrk, CAST_OBJ_NOTNULL(bp, d->priv, BACKEND_MAGIC); CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC); - CAST_OBJ_NOTNULL(vbc, bo->htc->priv, VBC_MAGIC); + CAST_OBJ_NOTNULL(vtp, bo->htc->priv, VTP_MAGIC); bo->htc->priv = NULL; - if (vbc->state != VBC_STATE_USED) - VBT_Wait(wrk, vbc); + if (vtp->state != VTP_STATE_USED) + VTP_Wait(wrk, vtp); if (bo->htc->doclose != SC_NULL || bp->proxy_header != 0) { - VSLb(bo->vsl, SLT_BackendClose, "%d %s", vbc->fd, + VSLb(bo->vsl, SLT_BackendClose, "%d %s", vtp->fd, bp->display_name); - VBT_Close(bp->tcp_pool, &vbc); + VTP_Close(bp->tcp_pool, &vtp); Lck_Lock(&bp->mtx); } else { - VSLb(bo->vsl, SLT_BackendReuse, "%d %s", vbc->fd, + VSLb(bo->vsl, SLT_BackendReuse, "%d %s", vtp->fd, bp->display_name); Lck_Lock(&bp->mtx); VSC_C_main->backend_recycle++; - VBT_Recycle(wrk, bp->tcp_pool, &vbc); + VTP_Recycle(wrk, bp->tcp_pool, &vtp); } assert(bp->n_conn > 0); bp->n_conn--; @@ -187,7 +187,7 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk, { int i, extrachance = 1; struct backend *bp; - struct vbc *vbc; + struct vtp *vtp; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); @@ -203,21 +203,21 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk, http_PrintfHeader(bo->bereq, "Host: %s", bp->hosthdr); do { - vbc = vbe_dir_getfd(wrk, bp, bo); - if (vbc == NULL) { + vtp = vbe_dir_getfd(wrk, bp, bo); + if (vtp == NULL) { VSLb(bo->vsl, SLT_FetchError, "no backend connection"); return (-1); } AN(bo->htc); - if (vbc->state != VBC_STATE_STOLEN) + if (vtp->state != VTP_STATE_STOLEN) extrachance = 0; i = V1F_SendReq(wrk, bo, &bo->acct.bereq_hdrbytes, 0); - if (vbc->state != VBC_STATE_USED) - VBT_Wait(wrk, vbc); + if (vtp->state != VTP_STATE_USED) + VTP_Wait(wrk, vtp); - assert(vbc->state == VBC_STATE_USED); + assert(vtp->state == VTP_STATE_USED); if (i == 0) i = V1F_FetchRespHdr(bo); @@ -248,15 +248,15 @@ static const struct suckaddr * __match_proto__(vdi_getip_f) vbe_dir_getip(const struct director *d, struct worker *wrk, struct busyobj *bo) { - struct vbc *vbc; + struct vtp *vtp; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(bo->htc, HTTP_CONN_MAGIC); - CAST_OBJ_NOTNULL(vbc, bo->htc->priv, VBC_MAGIC); + CAST_OBJ_NOTNULL(vtp, bo->htc->priv, VTP_MAGIC); - return (vbc->addr); + return (vtp->addr); } /*--------------------------------------------------------------------*/ @@ -268,7 +268,7 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo) enum sess_close retval; struct backend *bp; struct v1p_acct v1a; - struct vbc *vbc; + struct vtp *vtp; CHECK_OBJ_NOTNULL(d, DIRECTOR_MAGIC); CHECK_OBJ_NOTNULL(req, REQ_MAGIC); @@ -283,16 +283,16 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo) req->res_mode = RES_PIPE; - vbc = vbe_dir_getfd(req->wrk, bp, bo); + vtp = vbe_dir_getfd(req->wrk, bp, bo); - if (vbc == NULL) { + if (vtp == NULL) { VSLb(bo->vsl, SLT_FetchError, "no backend connection"); retval = SC_TX_ERROR; } else { i = V1F_SendReq(req->wrk, bo, &v1a.bereq, 1); VSLb_ts_req(req, "Pipe", W_TIM_real(req->wrk)); if (i == 0) - V1P_Process(req, vbc->fd, &v1a); + V1P_Process(req, vtp->fd, &v1a); VSLb_ts_req(req, "PipeSess", W_TIM_real(req->wrk)); bo->htc->doclose = SC_TX_PIPE; vbe_dir_finish(d, req->wrk, bo); diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c index 6732aca..ca828b9 100644 --- a/bin/varnishd/cache/cache_backend_cfg.c +++ b/bin/varnishd/cache/cache_backend_cfg.c @@ -116,9 +116,9 @@ VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt) Lck_Lock(&backends_mtx); VTAILQ_INSERT_TAIL(&backends, b, list); VSC_C_main->n_backend++; - b->tcp_pool = VBT_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr); + b->tcp_pool = VTP_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr); if (vbp != NULL) { - tp = VBT_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr); + tp = VTP_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr); assert(b->tcp_pool == tp); } Lck_Unlock(&backends_mtx); @@ -222,7 +222,7 @@ VBE_Delete(struct backend *be) else VTAILQ_REMOVE(&backends, be, list); VSC_C_main->n_backend--; - VBT_Rel(&be->tcp_pool); + VTP_Rel(&be->tcp_pool); Lck_Unlock(&backends_mtx); #define DA(x) do { if (be->x != NULL) free(be->x); } while (0) diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 028fb33..4bf48ba 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -104,7 +104,7 @@ vbp_delete(struct vbp_target *vt) #define DN(x) /**/ VRT_BACKEND_PROBE_HANDLE(); #undef DN - VBT_Rel(&vt->tcp_pool); + VTP_Rel(&vt->tcp_pool); free(vt->req); FREE_OBJ(vt); } @@ -276,7 +276,7 @@ vbp_poke(struct vbp_target *vt) t_start = t_now = VTIM_real(); t_end = t_start + vt->timeout; - s = VBT_Open(vt->tcp_pool, t_end - t_now, &sa); + s = VTP_Open(vt->tcp_pool, t_end - t_now, &sa); if (s < 0) { /* Got no connection: failed */ return; diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index 778790b..5622bf6 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -259,7 +259,7 @@ child_main(void) HTTP_Init(); VBO_Init(); - VBT_Init(); + VTP_Init(); VBP_Init(); VBE_InitCfg(); Pool_Init(); diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index bd8d9ff..be19ecb 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -56,10 +56,10 @@ struct tcp_pool { int refcnt; struct lock mtx; - VTAILQ_HEAD(, vbc) connlist; + VTAILQ_HEAD(, vtp) connlist; int n_conn; - VTAILQ_HEAD(, vbc) killlist; + VTAILQ_HEAD(, vtp) killlist; int n_kill; int n_used; @@ -76,39 +76,39 @@ static VTAILQ_HEAD(, tcp_pool) pools = VTAILQ_HEAD_INITIALIZER(pools); static void __match_proto__(waiter_handle_f) tcp_handle(struct waited *w, enum wait_event ev, double now) { - struct vbc *vbc; + struct vtp *vtp; struct tcp_pool *tp; - CAST_OBJ_NOTNULL(vbc, w->priv1, VBC_MAGIC); + CAST_OBJ_NOTNULL(vtp, w->priv1, VTP_MAGIC); (void)ev; (void)now; - CHECK_OBJ_NOTNULL(vbc->tcp_pool, TCP_POOL_MAGIC); - tp = vbc->tcp_pool; + CHECK_OBJ_NOTNULL(vtp->tcp_pool, TCP_POOL_MAGIC); + tp = vtp->tcp_pool; Lck_Lock(&tp->mtx); - switch (vbc->state) { - case VBC_STATE_STOLEN: - vbc->state = VBC_STATE_USED; - VTAILQ_REMOVE(&tp->connlist, vbc, list); - AN(vbc->cond); - AZ(pthread_cond_signal(vbc->cond)); + switch (vtp->state) { + case VTP_STATE_STOLEN: + vtp->state = VTP_STATE_USED; + VTAILQ_REMOVE(&tp->connlist, vtp, list); + AN(vtp->cond); + AZ(pthread_cond_signal(vtp->cond)); break; - case VBC_STATE_AVAIL: - VTCP_close(&vbc->fd); - VTAILQ_REMOVE(&tp->connlist, vbc, list); + case VTP_STATE_AVAIL: + VTCP_close(&vtp->fd); + VTAILQ_REMOVE(&tp->connlist, vtp, list); tp->n_conn--; - FREE_OBJ(vbc); + FREE_OBJ(vtp); break; - case VBC_STATE_CLEANUP: - VTCP_close(&vbc->fd); + case VTP_STATE_CLEANUP: + VTCP_close(&vtp->fd); tp->n_kill--; - VTAILQ_REMOVE(&tp->killlist, vbc, list); - memset(vbc, 0x11, sizeof *vbc); - free(vbc); + VTAILQ_REMOVE(&tp->killlist, vtp, list); + memset(vtp, 0x11, sizeof *vtp); + free(vtp); break; default: - WRONG("Wrong vbc state"); + WRONG("Wrong vtp state"); } Lck_Unlock(&tp->mtx); } @@ -119,7 +119,7 @@ tcp_handle(struct waited *w, enum wait_event ev, double now) */ struct tcp_pool * -VBT_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6) +VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6) { struct tcp_pool *tp; @@ -173,10 +173,10 @@ VBT_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6) */ void -VBT_Rel(struct tcp_pool **tpp) +VTP_Rel(struct tcp_pool **tpp) { struct tcp_pool *tp; - struct vbc *vbc, *vbc2; + struct vtp *vtp, *vtp2; TAKE_OBJ_NOTNULL(tp, tpp, TCP_POOL_MAGIC); @@ -194,13 +194,13 @@ VBT_Rel(struct tcp_pool **tpp) free(tp->ip4); free(tp->ip6); Lck_Lock(&tp->mtx); - VTAILQ_FOREACH_SAFE(vbc, &tp->connlist, list, vbc2) { - VTAILQ_REMOVE(&tp->connlist, vbc, list); + VTAILQ_FOREACH_SAFE(vtp, &tp->connlist, list, vtp2) { + VTAILQ_REMOVE(&tp->connlist, vtp, list); tp->n_conn--; - assert(vbc->state == VBC_STATE_AVAIL); - vbc->state = VBC_STATE_CLEANUP; - (void)shutdown(vbc->fd, SHUT_WR); - VTAILQ_INSERT_TAIL(&tp->killlist, vbc, list); + assert(vtp->state == VTP_STATE_AVAIL); + vtp->state = VTP_STATE_CLEANUP; + (void)shutdown(vtp->fd, SHUT_WR); + VTAILQ_INSERT_TAIL(&tp->killlist, vtp, list); tp->n_kill++; } while (tp->n_kill) { @@ -222,7 +222,7 @@ VBT_Rel(struct tcp_pool **tpp) */ int -VBT_Open(const struct tcp_pool *tp, double tmo, const struct suckaddr **sa) +VTP_Open(const struct tcp_pool *tp, double tmo, const struct suckaddr **sa) { int s; int msec; @@ -250,41 +250,41 @@ VBT_Open(const struct tcp_pool *tp, double tmo, const struct suckaddr **sa) */ void -VBT_Recycle(const struct worker *wrk, struct tcp_pool *tp, struct vbc **vbcp) +VTP_Recycle(const struct worker *wrk, struct tcp_pool *tp, struct vtp **vtpp) { - struct vbc *vbc; + struct vtp *vtp; int i = 0; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); - vbc = *vbcp; - *vbcp = NULL; - CHECK_OBJ_NOTNULL(vbc, VBC_MAGIC); + vtp = *vtpp; + *vtpp = NULL; + CHECK_OBJ_NOTNULL(vtp, VTP_MAGIC); - assert(vbc->state == VBC_STATE_USED); - assert(vbc->fd > 0); + assert(vtp->state == VTP_STATE_USED); + assert(vtp->fd > 0); Lck_Lock(&tp->mtx); tp->n_used--; - vbc->waited->priv1 = vbc; - vbc->waited->fd = vbc->fd; - vbc->waited->idle = VTIM_real(); - vbc->state = VBC_STATE_AVAIL; - vbc->waited->func = tcp_handle; - vbc->waited->tmo = &cache_param->backend_idle_timeout; - if (Wait_Enter(wrk->pool->waiter, vbc->waited)) { - VTCP_close(&vbc->fd); - memset(vbc, 0x33, sizeof *vbc); - free(vbc); + vtp->waited->priv1 = vtp; + vtp->waited->fd = vtp->fd; + vtp->waited->idle = VTIM_real(); + vtp->state = VTP_STATE_AVAIL; + vtp->waited->func = tcp_handle; + vtp->waited->tmo = &cache_param->backend_idle_timeout; + if (Wait_Enter(wrk->pool->waiter, vtp->waited)) { + VTCP_close(&vtp->fd); + memset(vtp, 0x33, sizeof *vtp); + free(vtp); // XXX: stats - vbc = NULL; + vtp = NULL; } else { - VTAILQ_INSERT_HEAD(&tp->connlist, vbc, list); + VTAILQ_INSERT_HEAD(&tp->connlist, vtp, list); i++; } - if (vbc != NULL) + if (vtp != NULL) tp->n_conn++; Lck_Unlock(&tp->mtx); @@ -293,7 +293,7 @@ VBT_Recycle(const struct worker *wrk, struct tcp_pool *tp, struct vbc **vbcp) * In varnishtest we do not have the luxury of using * multiple backend connections, so whenever we end up * in the "pending" case, take a short nap to let the - * waiter catch up and put the vbc back into circulations. + * waiter catch up and put the vtp back into circulations. * * In particular ESI:include related tests suffer random * failures without this. @@ -311,30 +311,30 @@ VBT_Recycle(const struct worker *wrk, struct tcp_pool *tp, struct vbc **vbcp) */ void -VBT_Close(struct tcp_pool *tp, struct vbc **vbcp) +VTP_Close(struct tcp_pool *tp, struct vtp **vtpp) { - struct vbc *vbc; + struct vtp *vtp; CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); - vbc = *vbcp; - *vbcp = NULL; - CHECK_OBJ_NOTNULL(vbc, VBC_MAGIC); + vtp = *vtpp; + *vtpp = NULL; + CHECK_OBJ_NOTNULL(vtp, VTP_MAGIC); - assert(vbc->state == VBC_STATE_USED); - assert(vbc->fd > 0); + assert(vtp->state == VTP_STATE_USED); + assert(vtp->fd > 0); Lck_Lock(&tp->mtx); tp->n_used--; - if (vbc->state == VBC_STATE_STOLEN) { - (void)shutdown(vbc->fd, SHUT_WR); - vbc->state = VBC_STATE_CLEANUP; - VTAILQ_INSERT_HEAD(&tp->killlist, vbc, list); + if (vtp->state == VTP_STATE_STOLEN) { + (void)shutdown(vtp->fd, SHUT_WR); + vtp->state = VTP_STATE_CLEANUP; + VTAILQ_INSERT_HEAD(&tp->killlist, vtp, list); tp->n_kill++; } else { - assert(vbc->state == VBC_STATE_USED); - VTCP_close(&vbc->fd); - memset(vbc, 0x44, sizeof *vbc); - free(vbc); + assert(vtp->state == VTP_STATE_USED); + VTCP_close(&vtp->fd); + memset(vtp, 0x44, sizeof *vtp); + free(vtp); } Lck_Unlock(&tp->mtx); } @@ -343,77 +343,77 @@ VBT_Close(struct tcp_pool *tp, struct vbc **vbcp) * Get a connection */ -struct vbc * -VBT_Get(struct tcp_pool *tp, double tmo, struct worker *wrk) +struct vtp * +VTP_Get(struct tcp_pool *tp, double tmo, struct worker *wrk) { - struct vbc *vbc; + struct vtp *vtp; CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); Lck_Lock(&tp->mtx); - vbc = VTAILQ_FIRST(&tp->connlist); - CHECK_OBJ_ORNULL(vbc, VBC_MAGIC); - if (vbc == NULL || vbc->state == VBC_STATE_STOLEN) - vbc = NULL; + vtp = VTAILQ_FIRST(&tp->connlist); + CHECK_OBJ_ORNULL(vtp, VTP_MAGIC); + if (vtp == NULL || vtp->state == VTP_STATE_STOLEN) + vtp = NULL; else { - assert(vbc->tcp_pool == tp); - assert(vbc->state == VBC_STATE_AVAIL); - VTAILQ_REMOVE(&tp->connlist, vbc, list); - VTAILQ_INSERT_TAIL(&tp->connlist, vbc, list); + assert(vtp->tcp_pool == tp); + assert(vtp->state == VTP_STATE_AVAIL); + VTAILQ_REMOVE(&tp->connlist, vtp, list); + VTAILQ_INSERT_TAIL(&tp->connlist, vtp, list); tp->n_conn--; VSC_C_main->backend_reuse++; - vbc->state = VBC_STATE_STOLEN; - vbc->cond = &wrk->cond; + vtp->state = VTP_STATE_STOLEN; + vtp->cond = &wrk->cond; } tp->n_used++; // Opening mostly works Lck_Unlock(&tp->mtx); - if (vbc != NULL) - return (vbc); - - ALLOC_OBJ(vbc, VBC_MAGIC); - AN(vbc); - INIT_OBJ(vbc->waited, WAITED_MAGIC); - vbc->state = VBC_STATE_USED; - vbc->tcp_pool = tp; - vbc->fd = VBT_Open(tp, tmo, &vbc->addr); - if (vbc->fd < 0) { - FREE_OBJ(vbc); + if (vtp != NULL) + return (vtp); + + ALLOC_OBJ(vtp, VTP_MAGIC); + AN(vtp); + INIT_OBJ(vtp->waited, WAITED_MAGIC); + vtp->state = VTP_STATE_USED; + vtp->tcp_pool = tp; + vtp->fd = VTP_Open(tp, tmo, &vtp->addr); + if (vtp->fd < 0) { + FREE_OBJ(vtp); Lck_Lock(&tp->mtx); tp->n_used--; // Nope, didn't work after all. Lck_Unlock(&tp->mtx); } else VSC_C_main->backend_conn++; - return (vbc); + return (vtp); } /*-------------------------------------------------------------------- */ void -VBT_Wait(struct worker *wrk, struct vbc *vbc) +VTP_Wait(struct worker *wrk, struct vtp *vtp) { struct tcp_pool *tp; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(vbc, VBC_MAGIC); - tp = vbc->tcp_pool; + CHECK_OBJ_NOTNULL(vtp, VTP_MAGIC); + tp = vtp->tcp_pool; CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); - assert(vbc->cond == &wrk->cond); + assert(vtp->cond == &wrk->cond); Lck_Lock(&tp->mtx); - while (vbc->state == VBC_STATE_STOLEN) + while (vtp->state == VTP_STATE_STOLEN) AZ(Lck_CondWait(&wrk->cond, &tp->mtx, 0)); - assert(vbc->state == VBC_STATE_USED); - vbc->cond = NULL; + assert(vtp->state == VTP_STATE_USED); + vtp->cond = NULL; Lck_Unlock(&tp->mtx); } /*--------------------------------------------------------------------*/ void -VBT_Init(void) +VTP_Init(void) { Lck_New(&pools_mtx, lck_backend); } diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h index 78bfd08..435b10d 100644 --- a/bin/varnishd/cache/cache_tcp_pool.h +++ b/bin/varnishd/cache/cache_tcp_pool.h @@ -32,17 +32,17 @@ struct tcp_pool; -struct vbc { +struct vtp { unsigned magic; -#define VBC_MAGIC 0x0c5e6592 +#define VTP_MAGIC 0x0c5e6592 int fd; - VTAILQ_ENTRY(vbc) list; + VTAILQ_ENTRY(vtp) list; const struct suckaddr *addr; uint8_t state; -#define VBC_STATE_AVAIL (1<<0) -#define VBC_STATE_USED (1<<1) -#define VBC_STATE_STOLEN (1<<2) -#define VBC_STATE_CLEANUP (1<<3) +#define VTP_STATE_AVAIL (1<<0) +#define VTP_STATE_USED (1<<1) +#define VTP_STATE_STOLEN (1<<2) +#define VTP_STATE_CLEANUP (1<<3) struct waited waited[1]; struct tcp_pool *tcp_pool; @@ -54,10 +54,10 @@ struct vbc { */ /* cache_tcp_pool.c */ -struct tcp_pool *VBT_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6); -void VBT_Rel(struct tcp_pool **tpp); -int VBT_Open(const struct tcp_pool *tp, double tmo, const struct suckaddr **sa); -void VBT_Recycle(const struct worker *, struct tcp_pool *, struct vbc **); -void VBT_Close(struct tcp_pool *tp, struct vbc **vbc); -struct vbc *VBT_Get(struct tcp_pool *, double tmo, struct worker *); -void VBT_Wait(struct worker *, struct vbc *); +struct tcp_pool *VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6); +void VTP_Rel(struct tcp_pool **tpp); +int VTP_Open(const struct tcp_pool *tp, double tmo, const struct suckaddr **sa); +void VTP_Recycle(const struct worker *, struct tcp_pool *, struct vtp **); +void VTP_Close(struct tcp_pool *tp, struct vtp **); +struct vtp *VTP_Get(struct tcp_pool *, double tmo, struct worker *); +void VTP_Wait(struct worker *, struct vtp *); diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 4e67f7b..1dac112 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -58,7 +58,7 @@ void VBE_InitCfg(void); void VBE_Poll(void); /* cache_backend_tcp.c */ -void VBT_Init(void); +void VTP_Init(void); /* cache_backend_poll.c */ void VBP_Init(void); From phk at FreeBSD.org Thu Oct 5 08:59:10 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 5 Oct 2017 08:59:10 +0000 (UTC) Subject: [master] 1afc578 Add a function to cheaply add a reference to an already referenced tcp_pool Message-ID: <20171005085910.6F8D16332B@lists.varnish-cache.org> commit 1afc578c407e74734749d2b2a303521481bcc41d Author: Poul-Henning Kamp Date: Thu Oct 5 08:57:48 2017 +0000 Add a function to cheaply add a reference to an already referenced tcp_pool diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c index ca828b9..15b650b 100644 --- a/bin/varnishd/cache/cache_backend_cfg.c +++ b/bin/varnishd/cache/cache_backend_cfg.c @@ -71,7 +71,6 @@ VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt) struct director *d; struct vsb *vsb; struct vcl *vcl; - struct tcp_pool *tp = NULL; const struct vrt_backend_probe *vbp; int retval; @@ -117,16 +116,14 @@ VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt) VTAILQ_INSERT_TAIL(&backends, b, list); VSC_C_main->n_backend++; b->tcp_pool = VTP_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr); - if (vbp != NULL) { - tp = VTP_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr); - assert(b->tcp_pool == tp); - } Lck_Unlock(&backends_mtx); VBE_fill_director(b); - if (vbp != NULL) - VBP_Insert(b, vbp, tp); + if (vbp != NULL) { + VTP_AddRef(b->tcp_pool); + VBP_Insert(b, vbp, b->tcp_pool); + } retval = VCL_AddBackend(ctx->vcl, b); diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index be19ecb..0fdc57b 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -169,6 +169,21 @@ VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6) } /*-------------------------------------------------------------------- + * Add a reference to a tcp_pool + */ + +void +VTP_AddRef(struct tcp_pool *tp) +{ + CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); + + Lck_Lock(&pools_mtx); + assert(tp->refcnt > 0); + tp->refcnt++; + Lck_Unlock(&pools_mtx); +} + +/*-------------------------------------------------------------------- * Release TCP pool, destroy if last reference. */ diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h index 435b10d..8e83140 100644 --- a/bin/varnishd/cache/cache_tcp_pool.h +++ b/bin/varnishd/cache/cache_tcp_pool.h @@ -55,6 +55,7 @@ struct vtp { /* cache_tcp_pool.c */ struct tcp_pool *VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6); +void VTP_AddRef(struct tcp_pool *); void VTP_Rel(struct tcp_pool **tpp); int VTP_Open(const struct tcp_pool *tp, double tmo, const struct suckaddr **sa); void VTP_Recycle(const struct worker *, struct tcp_pool *, struct vtp **); From nils.goroll at uplex.de Thu Oct 5 11:45:08 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 5 Oct 2017 11:45:08 +0000 (UTC) Subject: [master] 17c92e4 hack up vsctool to work with python 2 and 3 Message-ID: <20171005114508.765E264CF7@lists.varnish-cache.org> commit 17c92e43fda114bf5341e51d752e882238b8fe8c Author: Nils Goroll Date: Thu Oct 5 13:39:23 2017 +0200 hack up vsctool to work with python 2 and 3 StringIO does not exist any more in python3, yet requiring 2.7 would not pave the path forward, so try to be compatible with both. Works for me on Python 2.7.9 and Python 3.4 I would appreciate if someone more fluent in serpentinous programming language reviewed and/or rewrote this. diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py index 854968e..829c6e5 100644 --- a/lib/libvcc/vsctool.py +++ b/lib/libvcc/vsctool.py @@ -37,7 +37,10 @@ import getopt import json import sys import gzip -import StringIO +try: + import StringIO +except ImportError: + import io import collections import struct @@ -54,9 +57,22 @@ PARAMS = { "format": [ "integer", FORMATS], } +# http://python3porting.com/problems.html#bytes-strings-and-unicode +if sys.version_info < (3,): + def b(x): + return x +else: + import codecs + def b(x): + return codecs.latin_1_encode(x)[0] + def gzip_str(s): - out = StringIO.StringIO() - gzip.GzipFile(fileobj=out, mode="w").write(s) + try: + out = StringIO.StringIO() + except NameError: + out = io.BytesIO() + + gzip.GzipFile(fileobj=out, mode="w").write(b(s)) out.seek(4) out.write(struct.pack(" commit 7320bb96adcc32e084020412df32c881c1b7e1a6 Author: Nils Goroll Date: Thu Oct 5 14:25:11 2017 +0200 don't call vsctool in parallel Otherwise we might see garbled output files diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 895a73c..e4e5836 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -218,7 +218,10 @@ BUILT_SOURCES = vhp_hufdec.h DISTCLEANFILES += vhp_hufdec.h ####################################################################### -VSC_main.c VSC_main.h: $(srcdir)/main.vsc $(top_srcdir)/lib/libvcc/vsctool.py + +VSC_main.c: VSC_main.h + +VSC_main.h: $(srcdir)/main.vsc $(top_srcdir)/lib/libvcc/vsctool.py $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/main.vsc $(varnishd_OBJECTS): VSC_main.h @@ -230,7 +233,9 @@ nodist_varnishd_SOURCES += VSC_main.c ####################################################################### -VSC_lck.c VSC_lck.h: $(srcdir)/lck.vsc $(top_srcdir)/lib/libvcc/vsctool.py +VSC_lck.c: VSC_lck.h + +VSC_lck.h: $(srcdir)/lck.vsc $(top_srcdir)/lib/libvcc/vsctool.py $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/lck.vsc $(varnishd_OBJECTS): VSC_lck.h @@ -242,7 +247,9 @@ nodist_varnishd_SOURCES += VSC_lck.c ####################################################################### -VSC_vbe.c VSC_vbe.h: $(srcdir)/vbe.vsc $(top_srcdir)/lib/libvcc/vsctool.py +VSC_vbe.c: VSC_vbe.h + +VSC_vbe.h: $(srcdir)/vbe.vsc $(top_srcdir)/lib/libvcc/vsctool.py $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/vbe.vsc $(varnishd_OBJECTS): VSC_vbe.h @@ -254,7 +261,9 @@ nodist_varnishd_SOURCES += VSC_vbe.c ####################################################################### -VSC_sma.c VSC_sma.h: $(srcdir)/sma.vsc $(top_srcdir)/lib/libvcc/vsctool.py +VSC_sma.c: VSC_sma.h + +VSC_sma.h: $(srcdir)/sma.vsc $(top_srcdir)/lib/libvcc/vsctool.py $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/sma.vsc $(varnishd_OBJECTS): VSC_sma.h @@ -266,7 +275,9 @@ nodist_varnishd_SOURCES += VSC_sma.c ####################################################################### -VSC_smu.c VSC_smu.h: $(srcdir)/smu.vsc $(top_srcdir)/lib/libvcc/vsctool.py +VSC_smu.c: VSC_smu.h + +VSC_smu.h: $(srcdir)/smu.vsc $(top_srcdir)/lib/libvcc/vsctool.py $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/smu.vsc $(varnishd_OBJECTS): VSC_smu.h @@ -278,7 +289,9 @@ nodist_varnishd_SOURCES += VSC_smu.c ####################################################################### -VSC_smf.c VSC_smf.h: $(srcdir)/smf.vsc $(top_srcdir)/lib/libvcc/vsctool.py +VSC_smf.c: VSC_smf.h + +VSC_smf.h: $(srcdir)/smf.vsc $(top_srcdir)/lib/libvcc/vsctool.py $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/smf.vsc $(varnishd_OBJECTS): VSC_smf.h @@ -290,7 +303,9 @@ nodist_varnishd_SOURCES += VSC_smf.c ####################################################################### -VSC_mempool.c VSC_mempool.h: $(srcdir)/mempool.vsc $(top_srcdir)/lib/libvcc/vsctool.py +VSC_mempool.c: VSC_mempool.h + +VSC_mempool.h: $(srcdir)/mempool.vsc $(top_srcdir)/lib/libvcc/vsctool.py $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/mempool.vsc $(varnishd_OBJECTS): VSC_mempool.h @@ -302,7 +317,9 @@ nodist_varnishd_SOURCES += VSC_mempool.c ####################################################################### -VSC_mgt.c VSC_mgt.h: $(srcdir)/mgt.vsc $(top_srcdir)/lib/libvcc/vsctool.py +VSC_mgt.c: VSC_mgt.h + +VSC_mgt.h: $(srcdir)/mgt.vsc $(top_srcdir)/lib/libvcc/vsctool.py $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/mgt.vsc $(varnishd_OBJECTS): VSC_mgt.h From nils.goroll at uplex.de Thu Oct 5 12:42:06 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 5 Oct 2017 12:42:06 +0000 (UTC) Subject: [master] 1a5f3e1 more serialization of generator snakes Message-ID: <20171005124206.0F65D64F0A@lists.varnish-cache.org> commit 1a5f3e1f2d4408cd2337f1516f9d7367d028c33d Author: Nils Goroll Date: Thu Oct 5 14:41:36 2017 +0200 more serialization of generator snakes diff --git a/include/Makefile.am b/include/Makefile.am index dde6a89..9fb1da1 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -99,18 +99,21 @@ nobase_noinst_HEADERS = \ vtcp.h \ vtree.h -GENERATED_H = \ +vmod_abi.h: $(top_srcdir)/lib/libvcc/generate.py $(top_srcdir)/include/vrt.h + mkdir -p tbl + @PYTHON@ $(top_srcdir)/lib/libvcc/generate.py $(top_srcdir) $(top_builddir) + +GEN_H = \ tbl/vrt_stv_var.h \ tbl/vcl_returns.h \ tbl/vcc_types.h \ vrt_obj.h \ vcl.h \ - vcs_version.h \ - vmod_abi.h + vcs_version.h -$(GENERATED_H): $(top_srcdir)/lib/libvcc/generate.py $(top_srcdir)/include/vrt.h - mkdir -p tbl - @PYTHON@ $(top_srcdir)/lib/libvcc/generate.py $(top_srcdir) $(top_builddir) +$(GEN_H): vmod_abi.h + +GENERATED_H = vmod_abi.h $(GEN_H) BUILT_SOURCES = $(GENERATED_H) MAINTAINERCLEANFILES = vcs_version.h diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am index d0e8b64..5ad55d2 100644 --- a/lib/libvarnishapi/Makefile.am +++ b/lib/libvarnishapi/Makefile.am @@ -81,7 +81,9 @@ vsl2rst_SOURCES = \ $(top_srcdir)/include/tbl/vsl_tags.h \ $(top_srcdir)/include/tbl/vsl_tags_http.h -vxp_fixed_token.c vxp_tokens.h: \ +vxp_fixed_token.c: vxp_tokens.h + +vxp_tokens.h: \ $(srcdir)/generate.py @PYTHON@ $(srcdir)/generate.py $(srcdir) $(top_builddir) From nils.goroll at uplex.de Thu Oct 5 13:29:06 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 5 Oct 2017 13:29:06 +0000 (UTC) Subject: [master] 88aab69 Another go at the VSC generated files Message-ID: <20171005132906.7FAD464D26@lists.varnish-cache.org> commit 88aab69974096bf2c024d4ec19c7cb44a869c192 Author: Nils Goroll Date: Thu Oct 5 15:26:37 2017 +0200 Another go at the VSC generated files Should fix distcheck someone(tm) might revisit this again later diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index e4e5836..9873301 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -113,7 +113,6 @@ nodist_varnishd_SOURCES = \ builtin_vcl.c noinst_HEADERS = \ - VSC_main.h \ cache/cache_ban.h \ cache/cache_esi.h \ cache/cache_obj.h \ @@ -224,7 +223,7 @@ VSC_main.c: VSC_main.h VSC_main.h: $(srcdir)/main.vsc $(top_srcdir)/lib/libvcc/vsctool.py $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/main.vsc -$(varnishd_OBJECTS): VSC_main.h +$(varnishd_OBJECTS): VSC_main.c VSC_main.h EXTRA_DIST += main.vsc DISTCLEANFILES += VSC_main.c VSC_main.h @@ -238,7 +237,7 @@ VSC_lck.c: VSC_lck.h VSC_lck.h: $(srcdir)/lck.vsc $(top_srcdir)/lib/libvcc/vsctool.py $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/lck.vsc -$(varnishd_OBJECTS): VSC_lck.h +$(varnishd_OBJECTS): VSC_lck.c VSC_lck.h EXTRA_DIST += lck.vsc DISTCLEANFILES += VSC_lck.c VSC_lck.h @@ -252,7 +251,7 @@ VSC_vbe.c: VSC_vbe.h VSC_vbe.h: $(srcdir)/vbe.vsc $(top_srcdir)/lib/libvcc/vsctool.py $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/vbe.vsc -$(varnishd_OBJECTS): VSC_vbe.h +$(varnishd_OBJECTS): VSC_vbe.c VSC_vbe.h EXTRA_DIST += vbe.vsc DISTCLEANFILES += VSC_vbe.c VSC_vbe.h @@ -266,7 +265,7 @@ VSC_sma.c: VSC_sma.h VSC_sma.h: $(srcdir)/sma.vsc $(top_srcdir)/lib/libvcc/vsctool.py $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/sma.vsc -$(varnishd_OBJECTS): VSC_sma.h +$(varnishd_OBJECTS): VSC_sma.c VSC_sma.h EXTRA_DIST += sma.vsc DISTCLEANFILES += VSC_sma.c VSC_sma.h @@ -280,7 +279,7 @@ VSC_smu.c: VSC_smu.h VSC_smu.h: $(srcdir)/smu.vsc $(top_srcdir)/lib/libvcc/vsctool.py $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/smu.vsc -$(varnishd_OBJECTS): VSC_smu.h +$(varnishd_OBJECTS): VSC_smu.c VSC_smu.h EXTRA_DIST += smu.vsc DISTCLEANFILES += VSC_smu.c VSC_smu.h @@ -294,7 +293,7 @@ VSC_smf.c: VSC_smf.h VSC_smf.h: $(srcdir)/smf.vsc $(top_srcdir)/lib/libvcc/vsctool.py $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/smf.vsc -$(varnishd_OBJECTS): VSC_smf.h +$(varnishd_OBJECTS): VSC_smf.c VSC_smf.h EXTRA_DIST += smf.vsc DISTCLEANFILES += VSC_smf.c VSC_smf.h @@ -308,7 +307,7 @@ VSC_mempool.c: VSC_mempool.h VSC_mempool.h: $(srcdir)/mempool.vsc $(top_srcdir)/lib/libvcc/vsctool.py $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/mempool.vsc -$(varnishd_OBJECTS): VSC_mempool.h +$(varnishd_OBJECTS): VSC_mempool.c VSC_mempool.h EXTRA_DIST += mempool.vsc DISTCLEANFILES += VSC_mempool.c VSC_mempool.h @@ -322,7 +321,7 @@ VSC_mgt.c: VSC_mgt.h VSC_mgt.h: $(srcdir)/mgt.vsc $(top_srcdir)/lib/libvcc/vsctool.py $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/mgt.vsc -$(varnishd_OBJECTS): VSC_mgt.h +$(varnishd_OBJECTS): VSC_mgt.c VSC_mgt.h EXTRA_DIST += mgt.vsc DISTCLEANFILES += VSC_mgt.c VSC_mgt.h From phk at FreeBSD.org Thu Oct 5 14:17:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 5 Oct 2017 14:17:06 +0000 (UTC) Subject: [master] 43a7282 More tcp-pool related renaming Message-ID: <20171005141707.09AED63DE2@lists.varnish-cache.org> commit 43a7282319085e3a7aa9876ff8fa5b61729ccab6 Author: Poul-Henning Kamp Date: Thu Oct 5 13:32:30 2017 +0000 More tcp-pool related renaming diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index 0fdc57b..fa95b18 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -66,8 +66,8 @@ struct tcp_pool { }; -static struct lock pools_mtx; -static VTAILQ_HEAD(, tcp_pool) pools = VTAILQ_HEAD_INITIALIZER(pools); +static struct lock tcp_pools_mtx; +static VTAILQ_HEAD(, tcp_pool) tcp_pools = VTAILQ_HEAD_INITIALIZER(tcp_pools); /*-------------------------------------------------------------------- * Waiter-handler @@ -123,8 +123,9 @@ VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6) { struct tcp_pool *tp; - Lck_Lock(&pools_mtx); - VTAILQ_FOREACH(tp, &pools, list) { + assert(ip4 != NULL || ip6 != NULL); + Lck_Lock(&tcp_pools_mtx); + VTAILQ_FOREACH(tp, &tcp_pools, list) { assert(tp->refcnt > 0); if (ip4 == NULL) { if (tp->ip4 != NULL) @@ -145,10 +146,10 @@ VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6) continue; } tp->refcnt++; - Lck_Unlock(&pools_mtx); + Lck_Unlock(&tcp_pools_mtx); return (tp); } - Lck_Unlock(&pools_mtx); + Lck_Unlock(&tcp_pools_mtx); ALLOC_OBJ(tp, TCP_POOL_MAGIC); AN(tp); @@ -157,13 +158,13 @@ VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6) if (ip6 != NULL) tp->ip6 = VSA_Clone(ip6); tp->refcnt = 1; - Lck_New(&tp->mtx, lck_backend_tcp); + Lck_New(&tp->mtx, lck_tcp_pool); VTAILQ_INIT(&tp->connlist); VTAILQ_INIT(&tp->killlist); - Lck_Lock(&pools_mtx); - VTAILQ_INSERT_HEAD(&pools, tp, list); - Lck_Unlock(&pools_mtx); + Lck_Lock(&tcp_pools_mtx); + VTAILQ_INSERT_HEAD(&tcp_pools, tp, list); + Lck_Unlock(&tcp_pools_mtx); return (tp); } @@ -177,10 +178,10 @@ VTP_AddRef(struct tcp_pool *tp) { CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); - Lck_Lock(&pools_mtx); + Lck_Lock(&tcp_pools_mtx); assert(tp->refcnt > 0); tp->refcnt++; - Lck_Unlock(&pools_mtx); + Lck_Unlock(&tcp_pools_mtx); } /*-------------------------------------------------------------------- @@ -195,15 +196,15 @@ VTP_Rel(struct tcp_pool **tpp) TAKE_OBJ_NOTNULL(tp, tpp, TCP_POOL_MAGIC); - Lck_Lock(&pools_mtx); + Lck_Lock(&tcp_pools_mtx); assert(tp->refcnt > 0); if (--tp->refcnt > 0) { - Lck_Unlock(&pools_mtx); + Lck_Unlock(&tcp_pools_mtx); return; } AZ(tp->n_used); - VTAILQ_REMOVE(&pools, tp, list); - Lck_Unlock(&pools_mtx); + VTAILQ_REMOVE(&tcp_pools, tp, list); + Lck_Unlock(&tcp_pools_mtx); free(tp->name); free(tp->ip4); @@ -430,5 +431,5 @@ VTP_Wait(struct worker *wrk, struct vtp *vtp) void VTP_Init(void) { - Lck_New(&pools_mtx, lck_backend); + Lck_New(&tcp_pools_mtx, lck_tcp_pool); } diff --git a/include/tbl/locks.h b/include/tbl/locks.h index a89b0b4..cf2a6ed 100644 --- a/include/tbl/locks.h +++ b/include/tbl/locks.h @@ -30,7 +30,6 @@ /*lint -save -e525 -e539 */ LOCK(backend) -LOCK(backend_tcp) LOCK(ban) LOCK(busyobj) LOCK(cli) @@ -41,6 +40,7 @@ LOCK(mempool) LOCK(objhdr) LOCK(pipestat) LOCK(sess) +LOCK(tcp_pool) LOCK(vbe) LOCK(vcapace) LOCK(vcl) From phk at FreeBSD.org Thu Oct 5 14:17:07 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 5 Oct 2017 14:17:07 +0000 (UTC) Subject: [master] f6b9c11 Give the tcp-pool API a protocol identifier, so we don't accidentally recycle connections between protocols. Message-ID: <20171005141707.25C9A63DE4@lists.varnish-cache.org> commit f6b9c11a2ed6234a64792e5756215c76df3f6c55 Author: Poul-Henning Kamp Date: Thu Oct 5 14:15:23 2017 +0000 Give the tcp-pool API a protocol identifier, so we don't accidentally recycle connections between protocols. diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index a642304..0956afb 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -163,14 +163,14 @@ vbe_dir_finish(const struct director *d, struct worker *wrk, if (bo->htc->doclose != SC_NULL || bp->proxy_header != 0) { VSLb(bo->vsl, SLT_BackendClose, "%d %s", vtp->fd, bp->display_name); - VTP_Close(bp->tcp_pool, &vtp); + VTP_Close(&vtp); Lck_Lock(&bp->mtx); } else { VSLb(bo->vsl, SLT_BackendReuse, "%d %s", vtp->fd, bp->display_name); Lck_Lock(&bp->mtx); VSC_C_main->backend_recycle++; - VTP_Recycle(wrk, bp->tcp_pool, &vtp); + VTP_Recycle(wrk, &vtp); } assert(bp->n_conn > 0); bp->n_conn--; diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c index 15b650b..efb3dc0 100644 --- a/bin/varnishd/cache/cache_backend_cfg.c +++ b/bin/varnishd/cache/cache_backend_cfg.c @@ -50,6 +50,8 @@ #include "VSC_vbe.h" +const char *vbe_proto_ident = "HTTP Backend"; + static VTAILQ_HEAD(, backend) backends = VTAILQ_HEAD_INITIALIZER(backends); static VTAILQ_HEAD(, backend) cool_backends = VTAILQ_HEAD_INITIALIZER(cool_backends); @@ -115,7 +117,8 @@ VRT_new_backend(VRT_CTX, const struct vrt_backend *vrt) Lck_Lock(&backends_mtx); VTAILQ_INSERT_TAIL(&backends, b, list); VSC_C_main->n_backend++; - b->tcp_pool = VTP_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr); + b->tcp_pool = VTP_Ref(vrt->ipv4_suckaddr, vrt->ipv6_suckaddr, + vbe_proto_ident); Lck_Unlock(&backends_mtx); VBE_fill_director(b); diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index fa95b18..5c351a4 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -48,7 +48,7 @@ struct tcp_pool { unsigned magic; #define TCP_POOL_MAGIC 0x28b0e42a - char *name; + const void *id; struct suckaddr *ip4; struct suckaddr *ip6; @@ -63,7 +63,6 @@ struct tcp_pool { int n_kill; int n_used; - }; static struct lock tcp_pools_mtx; @@ -119,7 +118,7 @@ tcp_handle(struct waited *w, enum wait_event ev, double now) */ struct tcp_pool * -VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6) +VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, const void *id) { struct tcp_pool *tp; @@ -127,6 +126,8 @@ VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6) Lck_Lock(&tcp_pools_mtx); VTAILQ_FOREACH(tp, &tcp_pools, list) { assert(tp->refcnt > 0); + if (tp->id != id) + continue; if (ip4 == NULL) { if (tp->ip4 != NULL) continue; @@ -158,6 +159,7 @@ VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6) if (ip6 != NULL) tp->ip6 = VSA_Clone(ip6); tp->refcnt = 1; + tp->id = id; Lck_New(&tp->mtx, lck_tcp_pool); VTAILQ_INIT(&tp->connlist); VTAILQ_INIT(&tp->killlist); @@ -206,7 +208,6 @@ VTP_Rel(struct tcp_pool **tpp) VTAILQ_REMOVE(&tcp_pools, tp, list); Lck_Unlock(&tcp_pools_mtx); - free(tp->name); free(tp->ip4); free(tp->ip6); Lck_Lock(&tp->mtx); @@ -254,7 +255,9 @@ VTP_Open(const struct tcp_pool *tp, double tmo, const struct suckaddr **sa) } *sa = tp->ip4; s = VTCP_connect(tp->ip4, msec); - if (s < 0 && !cache_param->prefer_ipv6) { + if (s >= 0) + return (s); + if (!cache_param->prefer_ipv6) { *sa = tp->ip6; s = VTCP_connect(tp->ip6, msec); } @@ -266,16 +269,18 @@ VTP_Open(const struct tcp_pool *tp, double tmo, const struct suckaddr **sa) */ void -VTP_Recycle(const struct worker *wrk, struct tcp_pool *tp, struct vtp **vtpp) +VTP_Recycle(const struct worker *wrk, struct vtp **vtpp) { struct vtp *vtp; + struct tcp_pool *tp; int i = 0; CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); - CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); vtp = *vtpp; *vtpp = NULL; CHECK_OBJ_NOTNULL(vtp, VTP_MAGIC); + tp = vtp->tcp_pool; + CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); assert(vtp->state == VTP_STATE_USED); assert(vtp->fd > 0); @@ -327,14 +332,16 @@ VTP_Recycle(const struct worker *wrk, struct tcp_pool *tp, struct vtp **vtpp) */ void -VTP_Close(struct tcp_pool *tp, struct vtp **vtpp) +VTP_Close(struct vtp **vtpp) { struct vtp *vtp; + struct tcp_pool *tp; - CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); vtp = *vtpp; *vtpp = NULL; CHECK_OBJ_NOTNULL(vtp, VTP_MAGIC); + tp = vtp->tcp_pool; + CHECK_OBJ_NOTNULL(tp, TCP_POOL_MAGIC); assert(vtp->state == VTP_STATE_USED); assert(vtp->fd > 0); diff --git a/bin/varnishd/cache/cache_tcp_pool.h b/bin/varnishd/cache/cache_tcp_pool.h index 8e83140..11b0f82 100644 --- a/bin/varnishd/cache/cache_tcp_pool.h +++ b/bin/varnishd/cache/cache_tcp_pool.h @@ -26,7 +26,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * TCP connection pools + * Outgoing TCP connection pools * */ @@ -53,12 +53,45 @@ struct vtp { * Prototypes */ -/* cache_tcp_pool.c */ -struct tcp_pool *VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6); +struct tcp_pool *VTP_Ref(const struct suckaddr *ip4, const struct suckaddr *ip6, const void *id); + /* + * Get a reference to a TCP pool. Either ip4 or ip6 arg must be non-NULL. + * If recycling is to be used, the id pointer distinguishes the pool per protocol. + */ + void VTP_AddRef(struct tcp_pool *); -void VTP_Rel(struct tcp_pool **tpp); -int VTP_Open(const struct tcp_pool *tp, double tmo, const struct suckaddr **sa); -void VTP_Recycle(const struct worker *, struct tcp_pool *, struct vtp **); -void VTP_Close(struct tcp_pool *tp, struct vtp **); + /* + * Get another reference to an already referenced TCP pool. + */ + +void VTP_Rel(struct tcp_pool **); + /* + * Release reference to a TCP pool. When last reference is released + * the pool is destroyed and all cached connections closed. + */ + +int VTP_Open(const struct tcp_pool *, double tmo, const struct suckaddr **); + /* + * Open a new connection and return the adress used. + */ + +void VTP_Close(struct vtp **); + /* + * Close a connection. + */ + +void VTP_Recycle(const struct worker *, struct vtp **); + /* + * Recycle an open connection. + */ + struct vtp *VTP_Get(struct tcp_pool *, double tmo, struct worker *); + /* + * Get a (possibly) recycled connection. + */ + void VTP_Wait(struct worker *, struct vtp *); + /* + * If the connection was recycled (state != VTP_STATE_USED) call this + * function before attempting to receive on the connection. + */ From nils.goroll at uplex.de Thu Oct 5 16:56:06 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 5 Oct 2017 16:56:06 +0000 (UTC) Subject: [master] 042a98b Simplify make rules for vsc files and remove duplication Message-ID: <20171005165606.A953F64D08@lists.varnish-cache.org> commit 042a98bcf1dd9314e70255df923561e4be1a2517 Author: Nils Goroll Date: Thu Oct 5 17:41:31 2017 +0200 Simplify make rules for vsc files and remove duplication We use an implicit rule to tell make how to turn vsc into c and manually define the vsc files in Makefile.am. The implicit rule tells automake to add .vsc to SUFFIXES, which in turn allows us to add the actual vscs to varnishd_SOURCES. To avoid triple configuration of the list of built VSC .c and .h files, we generate these in configure. AC_SUBSTing files into _SOURCES is forbidden by autoconf for a good reason: Adding sources to the build should remain an explicit process to avoid accidentally compiling in code just because someone had files with a magic name lying around. As the implicit .vsc -> .c rule does not allow for additional dependencies, we add in the dependency to vsctool using the auto-generated list of c files. BUILT_SOURCES is used to ensure that, for default make targets, headers generated from vscs are always present before the actual varnishd sources are being build, of which some depend on vsc headers. For subsequent builds, include dependency tracking takes over. The Makefile rules are deliberately simple to hopefully cover all make flavours available on the planet. Notice that, for example, if we limited ourselves to gmake, we could use much simpler substitution rules to generate VSC_GEN_C and _H instead of the de-tour via configure. Portability is also the reason for renaming all vsc files. diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 9873301..87f7f82 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -218,112 +218,25 @@ DISTCLEANFILES += vhp_hufdec.h ####################################################################### -VSC_main.c: VSC_main.h +.vsc.c: + $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $< -VSC_main.h: $(srcdir)/main.vsc $(top_srcdir)/lib/libvcc/vsctool.py - $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/main.vsc +VSC_SRC = \ + VSC_main.vsc \ + VSC_lck.vsc \ + VSC_vbe.vsc \ + VSC_sma.vsc \ + VSC_smu.vsc \ + VSC_smf.vsc \ + VSC_mempool.vsc \ + VSC_mgt.vsc -$(varnishd_OBJECTS): VSC_main.c VSC_main.h +VSC_GEN_C = @VSC_GEN_C@ +VSC_GEN_H = @VSC_GEN_H@ -EXTRA_DIST += main.vsc -DISTCLEANFILES += VSC_main.c VSC_main.h -BUILT_SOURCES += VSC_main.c VSC_main.h -nodist_varnishd_SOURCES += VSC_main.c +$(VSC_GEN_C): $(top_srcdir)/lib/libvcc/vsctool.py -####################################################################### - -VSC_lck.c: VSC_lck.h - -VSC_lck.h: $(srcdir)/lck.vsc $(top_srcdir)/lib/libvcc/vsctool.py - $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/lck.vsc - -$(varnishd_OBJECTS): VSC_lck.c VSC_lck.h - -EXTRA_DIST += lck.vsc -DISTCLEANFILES += VSC_lck.c VSC_lck.h -BUILT_SOURCES += VSC_lck.c VSC_lck.h -nodist_varnishd_SOURCES += VSC_lck.c - -####################################################################### - -VSC_vbe.c: VSC_vbe.h - -VSC_vbe.h: $(srcdir)/vbe.vsc $(top_srcdir)/lib/libvcc/vsctool.py - $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/vbe.vsc - -$(varnishd_OBJECTS): VSC_vbe.c VSC_vbe.h - -EXTRA_DIST += vbe.vsc -DISTCLEANFILES += VSC_vbe.c VSC_vbe.h -BUILT_SOURCES += VSC_vbe.c VSC_vbe.h -nodist_varnishd_SOURCES += VSC_vbe.c - -####################################################################### - -VSC_sma.c: VSC_sma.h - -VSC_sma.h: $(srcdir)/sma.vsc $(top_srcdir)/lib/libvcc/vsctool.py - $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/sma.vsc - -$(varnishd_OBJECTS): VSC_sma.c VSC_sma.h - -EXTRA_DIST += sma.vsc -DISTCLEANFILES += VSC_sma.c VSC_sma.h -BUILT_SOURCES += VSC_sma.c VSC_sma.h -nodist_varnishd_SOURCES += VSC_sma.c - -####################################################################### - -VSC_smu.c: VSC_smu.h - -VSC_smu.h: $(srcdir)/smu.vsc $(top_srcdir)/lib/libvcc/vsctool.py - $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/smu.vsc - -$(varnishd_OBJECTS): VSC_smu.c VSC_smu.h - -EXTRA_DIST += smu.vsc -DISTCLEANFILES += VSC_smu.c VSC_smu.h -BUILT_SOURCES += VSC_smu.c VSC_smu.h -nodist_varnishd_SOURCES += VSC_smu.c - -####################################################################### - -VSC_smf.c: VSC_smf.h - -VSC_smf.h: $(srcdir)/smf.vsc $(top_srcdir)/lib/libvcc/vsctool.py - $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/smf.vsc - -$(varnishd_OBJECTS): VSC_smf.c VSC_smf.h - -EXTRA_DIST += smf.vsc -DISTCLEANFILES += VSC_smf.c VSC_smf.h -BUILT_SOURCES += VSC_smf.c VSC_smf.h -nodist_varnishd_SOURCES += VSC_smf.c - -####################################################################### - -VSC_mempool.c: VSC_mempool.h - -VSC_mempool.h: $(srcdir)/mempool.vsc $(top_srcdir)/lib/libvcc/vsctool.py - $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/mempool.vsc - -$(varnishd_OBJECTS): VSC_mempool.c VSC_mempool.h - -EXTRA_DIST += mempool.vsc -DISTCLEANFILES += VSC_mempool.c VSC_mempool.h -BUILT_SOURCES += VSC_mempool.c VSC_mempool.h -nodist_varnishd_SOURCES += VSC_mempool.c - -####################################################################### - -VSC_mgt.c: VSC_mgt.h - -VSC_mgt.h: $(srcdir)/mgt.vsc $(top_srcdir)/lib/libvcc/vsctool.py - $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $(srcdir)/mgt.vsc - -$(varnishd_OBJECTS): VSC_mgt.c VSC_mgt.h +BUILT_SOURCES += $(VSC_GEN_C) +CLEANFILES = $(VSC_GEN_C) $(VSC_GEN_H) -EXTRA_DIST += mgt.vsc -DISTCLEANFILES += VSC_mgt.c VSC_mgt.h -BUILT_SOURCES += VSC_mgt.c VSC_mgt.h -nodist_varnishd_SOURCES += VSC_mgt.c +varnishd_SOURCES += $(VSC_SRC) diff --git a/bin/varnishd/VSC_lck.vsc b/bin/varnishd/VSC_lck.vsc new file mode 100644 index 0000000..75b69be --- /dev/null +++ b/bin/varnishd/VSC_lck.vsc @@ -0,0 +1,31 @@ +.. + This is *NOT* a RST file but the syntax has been chosen so + that it may become an RST file at some later date. + +.. varnish_vsc_begin:: lck + :oneliner: Lock Counters + :order: 70 + + Counters which track the activity in the different classes + of mutex-locks. + +.. varnish_vsc:: creat + :type: counter + :level: debug + :oneliner: Created locks + + +.. varnish_vsc:: destroy + :type: counter + :level: debug + :oneliner: Destroyed locks + + +.. varnish_vsc:: locks + :type: counter + :level: debug + :oneliner: Lock Operations + + +.. varnish_vsc_end:: lck + diff --git a/bin/varnishd/VSC_main.vsc b/bin/varnishd/VSC_main.vsc new file mode 100644 index 0000000..ca999f7 --- /dev/null +++ b/bin/varnishd/VSC_main.vsc @@ -0,0 +1,731 @@ +.. + This is *NOT* a RST file but the syntax has been chosen so + that it may become an RST file at some later date. + +.. varnish_vsc_begin:: main + :oneliner: Main counters + :order: 10 + :sumfunction: + +.. varnish_vsc:: summs + :level: debug + :oneliner: stat summ operations + +.. varnish_vsc:: uptime + :oneliner: Child process uptime + :format: duration + + How long the child process has been running. + +.. varnish_vsc:: sess_conn + :oneliner: Sessions accepted + + Count of sessions successfully accepted + +.. varnish_vsc:: sess_drop + :oneliner: Sessions dropped + + Count of sessions silently dropped due to lack of worker thread. + +.. varnish_vsc:: sess_fail + :oneliner: Session accept failures + + Count of failures to accept TCP connection. Either the client + changed its mind, or the kernel ran out of some resource like file + descriptors. + +.. varnish_vsc:: client_req_400 + :oneliner: Client requests received, subject to 400 errors + + 400 means we couldn't make sense of the request, it was malformed + in some drastic way. + +.. varnish_vsc:: client_req_417 + :oneliner: Client requests received, subject to 417 errors + + 417 means that something went wrong with an Expect: header. + +.. varnish_vsc:: client_req + :oneliner: Good client requests received + + The count of parseable client requests seen. + +.. varnish_vsc:: cache_hit + :oneliner: Cache hits + + Count of cache hits. A cache hit indicates that an object has been + delivered to a client without fetching it from a backend server. + +.. varnish_vsc:: cache_hitpass + :oneliner: Cache hits for pass. + + Count of hits for pass. A cache hit for pass indicates that Varnish + is going to pass the request to the backend and this decision has + been cached in it self. This counts how many times the cached + decision is being used. + +.. varnish_vsc:: cache_hitmiss + :oneliner: Cache hits for miss. + + Count of hits for miss. A cache hit for miss indicates that Varnish + is going to proceed as for a cache miss without request coalescing, + and this decision has been cached. This counts how many times the + cached decision is being used. + +.. varnish_vsc:: cache_miss + :oneliner: Cache misses + + Count of misses. A cache miss indicates the object was fetched from + the backend before delivering it to the client. + +.. varnish_vsc:: backend_conn + :oneliner: Backend conn. success + + How many backend connections have successfully been established. + +.. varnish_vsc:: backend_unhealthy + :oneliner: Backend conn. not attempted + + +.. varnish_vsc:: backend_busy + :oneliner: Backend conn. too many + + +.. varnish_vsc:: backend_fail + :oneliner: Backend conn. failures + + +.. varnish_vsc:: backend_reuse + :oneliner: Backend conn. reuses + + Count of backend connection reuses. This counter is increased + whenever we reuse a recycled connection. + +.. varnish_vsc:: backend_recycle + :oneliner: Backend conn. recycles + + Count of backend connection recycles. This counter is increased + whenever we have a keep-alive connection that is put back into the + pool of connections. It has not yet been used, but it might be, + unless the backend closes it. + +.. varnish_vsc:: backend_retry + :oneliner: Backend conn. retry + + +.. varnish_vsc:: fetch_head + :oneliner: Fetch no body (HEAD) + + beresp with no body because the request is HEAD. + +.. varnish_vsc:: fetch_length + :oneliner: Fetch with Length + + beresp.body with Content-Length. + +.. varnish_vsc:: fetch_chunked + :oneliner: Fetch chunked + + beresp.body with Chunked. + +.. varnish_vsc:: fetch_eof + :oneliner: Fetch EOF + + beresp.body with EOF. + +.. varnish_vsc:: fetch_bad + :oneliner: Fetch bad T-E + + beresp.body length/fetch could not be determined. + +.. varnish_vsc:: fetch_none + :oneliner: Fetch no body + + beresp.body empty + +.. varnish_vsc:: fetch_1xx + :oneliner: Fetch no body (1xx) + + beresp with no body because of 1XX response. + +.. varnish_vsc:: fetch_204 + :oneliner: Fetch no body (204) + + beresp with no body because of 204 response. + +.. varnish_vsc:: fetch_304 + :oneliner: Fetch no body (304) + + beresp with no body because of 304 response. + +.. varnish_vsc:: fetch_failed + :oneliner: Fetch failed (all causes) + + beresp fetch failed. + +.. varnish_vsc:: fetch_no_thread + :oneliner: Fetch failed (no thread) + + beresp fetch failed, no thread available. + +.. varnish_vsc:: pools + :type: gauge + :oneliner: Number of thread pools + + Number of thread pools. See also parameter thread_pools. NB: Presently + pools cannot be removed once created. + +.. varnish_vsc:: threads + :type: gauge + :oneliner: Total number of threads + + Number of threads in all pools. See also parameters thread_pools, + thread_pool_min and thread_pool_max. + +.. varnish_vsc:: threads_limited + :oneliner: Threads hit max + + Number of times more threads were needed, but limit was reached in + a thread pool. See also parameter thread_pool_max. + +.. varnish_vsc:: threads_created + :oneliner: Threads created + + Total number of threads created in all pools. + +.. varnish_vsc:: threads_destroyed + :oneliner: Threads destroyed + + Total number of threads destroyed in all pools. + +.. varnish_vsc:: threads_failed + :oneliner: Thread creation failed + + Number of times creating a thread failed. See VSL::Debug for + diagnostics. See also parameter thread_fail_delay. + +.. varnish_vsc:: thread_queue_len + :type: gauge + :oneliner: Length of session queue + + Length of session queue waiting for threads. NB: Only updates once + per second. See also parameter thread_queue_limit. + +.. varnish_vsc:: busy_sleep + :oneliner: Number of requests sent to sleep on busy objhdr + + Number of requests sent to sleep without a worker thread because + they found a busy object. + +.. varnish_vsc:: busy_wakeup + :oneliner: Number of requests woken after sleep on busy objhdr + + Number of requests taken off the busy object sleep list and rescheduled. + +.. varnish_vsc:: busy_killed + :oneliner: Number of requests killed after sleep on busy objhdr + + Number of requests killed from the busy object sleep list due to + lack of resources. + +.. varnish_vsc:: sess_queued + :oneliner: Sessions queued for thread + + Number of times session was queued waiting for a thread. See also + parameter thread_queue_limit. + +.. varnish_vsc:: sess_dropped + :oneliner: Sessions dropped for thread + + Number of times an HTTP/1 session was dropped because the queue was + too long already. See also parameter thread_queue_limit. + +.. varnish_vsc:: req_dropped + :oneliner: Requests dropped + + Number of times an HTTP/2 stream was refused because the queue was + too long already. See also parameter thread_queue_limit. + +.. varnish_vsc:: n_object + :type: gauge + :oneliner: object structs made + + Approximate number of HTTP objects (headers + body, if present) in + the cache. + +.. varnish_vsc:: n_vampireobject + :type: gauge + :level: diag + :oneliner: unresurrected objects + + Number of unresurrected objects + +.. varnish_vsc:: n_objectcore + :type: gauge + :oneliner: objectcore structs made + + Approximate number of object metadata elements in the cache. Each + object needs an objectcore, extra objectcores are for hit-for-miss, + hit-for-pass and busy objects. + +.. varnish_vsc:: n_objecthead + :type: gauge + :oneliner: objecthead structs made + + Approximate number of different hash entries in the cache. + +.. varnish_vsc:: n_backend + :type: gauge + :oneliner: Number of backends + + Number of backends known to us. + +.. varnish_vsc:: n_expired + :type: gauge + :oneliner: Number of expired objects + + Number of objects that expired from cache because of old age. + +.. varnish_vsc:: n_lru_nuked + :type: gauge + :oneliner: Number of LRU nuked objects + + How many objects have been forcefully evicted from storage to make + room for a new object. + +.. varnish_vsc:: n_lru_moved + :type: gauge + :level: diag + :oneliner: Number of LRU moved objects + + Number of move operations done on the LRU list. + +.. varnish_vsc:: losthdr + :oneliner: HTTP header overflows + + +.. varnish_vsc:: s_sess + :oneliner: Total sessions seen + + +.. varnish_vsc:: s_pipe + :oneliner: Total pipe sessions seen + + +.. varnish_vsc:: s_pass + :oneliner: Total pass-ed requests seen + + +.. varnish_vsc:: s_fetch + :oneliner: Total backend fetches initiated + + +.. varnish_vsc:: s_synth + :oneliner: Total synthethic responses made + + +.. varnish_vsc:: s_req_hdrbytes + :oneliner: Request header bytes + :format: bytes + + Total request header bytes received + +.. varnish_vsc:: s_req_bodybytes + :oneliner: Request body bytes + :format: bytes + + Total request body bytes received + +.. varnish_vsc:: s_resp_hdrbytes + :oneliner: Response header bytes + :format: bytes + + Total response header bytes transmitted + +.. varnish_vsc:: s_resp_bodybytes + :oneliner: Response body bytes + :format: bytes + + Total response body bytes transmitted + :format: bytes + +.. varnish_vsc:: s_pipe_hdrbytes + :oneliner: Pipe request header bytes + :format: bytes + + Total request bytes received for piped sessions + +.. varnish_vsc:: s_pipe_in + :oneliner: Piped bytes from client + :format: bytes + + Total number of bytes forwarded from clients in pipe sessions + +.. varnish_vsc:: s_pipe_out + :oneliner: Piped bytes to client + :format: bytes + + Total number of bytes forwarded to clients in pipe sessions + +.. varnish_vsc:: sess_closed + :oneliner: Session Closed + + +.. varnish_vsc:: sess_closed_err + :oneliner: Session Closed with error + + Total number of sessions closed with errors. See sc_* diag counters + for detailed breakdown + +.. varnish_vsc:: sess_readahead + :oneliner: Session Read Ahead + + +.. varnish_vsc:: sess_herd + :level: diag + :oneliner: Session herd + + Number of times the timeout_linger triggered + +.. varnish_vsc:: sc_rem_close + :level: diag + :oneliner: Session OK REM_CLOSE + + Number of session closes with REM_CLOSE (Client Closed) + +.. varnish_vsc:: sc_req_close + :level: diag + :oneliner: Session OK REQ_CLOSE + + Number of session closes with REQ_CLOSE (Client requested close) + +.. varnish_vsc:: sc_req_http10 + :level: diag + :oneliner: Session Err REQ_HTTP10 + + Number of session closes with Error REQ_HTTP10 (Proto < HTTP/1.1) + +.. varnish_vsc:: sc_rx_bad + :level: diag + :oneliner: Session Err RX_BAD + + Number of session closes with Error RX_BAD (Received bad req/resp) + +.. varnish_vsc:: sc_rx_body + :level: diag + :oneliner: Session Err RX_BODY + + Number of session closes with Error RX_BODY (Failure receiving req.body) + +.. varnish_vsc:: sc_rx_junk + :level: diag + :oneliner: Session Err RX_JUNK + + Number of session closes with Error RX_JUNK (Received junk data) + +.. varnish_vsc:: sc_rx_overflow + :level: diag + :oneliner: Session Err RX_OVERFLOW + + Number of session closes with Error RX_OVERFLOW (Received buffer overflow) + +.. varnish_vsc:: sc_rx_timeout + :level: diag + :oneliner: Session Err RX_TIMEOUT + + Number of session closes with Error RX_TIMEOUT (Receive timeout) + +.. varnish_vsc:: sc_tx_pipe + :level: diag + :oneliner: Session OK TX_PIPE + + Number of session closes with TX_PIPE (Piped transaction) + +.. varnish_vsc:: sc_tx_error + :level: diag + :oneliner: Session Err TX_ERROR + + Number of session closes with Error TX_ERROR (Error transaction) + +.. varnish_vsc:: sc_tx_eof + :level: diag + :oneliner: Session OK TX_EOF + + Number of session closes with TX_EOF (EOF transmission) + +.. varnish_vsc:: sc_resp_close + :level: diag + :oneliner: Session OK RESP_CLOSE + + Number of session closes with RESP_CLOSE (Backend/VCL requested close) + +.. varnish_vsc:: sc_overload + :level: diag + :oneliner: Session Err OVERLOAD + + Number of session closes with Error OVERLOAD (Out of some resource) + +.. varnish_vsc:: sc_pipe_overflow + :level: diag + :oneliner: Session Err PIPE_OVERFLOW + + Number of session closes with Error PIPE_OVERFLOW (Session pipe overflow) + +.. varnish_vsc:: sc_range_short + :level: diag + :oneliner: Session Err RANGE_SHORT + + Number of session closes with Error RANGE_SHORT (Insufficient data for range) + +.. varnish_vsc:: sc_req_http20 + :level: diag + :oneliner: Session Err REQ_HTTP20 + + Number of session closes with Error REQ_HTTP20 (HTTP2 not accepted) + +.. varnish_vsc:: sc_vcl_failure + :level: diag + :oneliner: Session Err VCL_FAILURE + + Number of session closes with Error VCL_FAILURE (VCL failure) + +.. varnish_vsc:: shm_records + :level: diag + :oneliner: SHM records + + +.. varnish_vsc:: shm_writes + :level: diag + :oneliner: SHM writes + + +.. varnish_vsc:: shm_flushes + :level: diag + :oneliner: SHM flushes due to overflow + + +.. varnish_vsc:: shm_cont + :level: diag + :oneliner: SHM MTX contention + + +.. varnish_vsc:: shm_cycles + :level: diag + :oneliner: SHM cycles through buffer + + +.. varnish_vsc:: backend_req + :oneliner: Backend requests made + + +.. varnish_vsc:: n_vcl + :type: gauge + :oneliner: Number of loaded VCLs in total + + +.. varnish_vsc:: n_vcl_avail + :type: gauge + :level: diag + :oneliner: Number of VCLs available + + +.. varnish_vsc:: n_vcl_discard + :type: gauge + :level: diag + :oneliner: Number of discarded VCLs + + +.. varnish_vsc:: vcl_fail + :oneliner: VCL failures + + Count of failures which prevented VCL from completing. + +.. varnish_vsc:: bans + :type: gauge + :oneliner: Count of bans + + Number of all bans in system, including bans superseded by newer + bans and bans already checked by the ban-lurker. + +.. varnish_vsc:: bans_completed + :type: gauge + :level: diag + :oneliner: Number of bans marked 'completed' + + Number of bans which are no longer active, either because they got + checked by the ban-lurker or superseded by newer identical bans. + +.. varnish_vsc:: bans_obj + :type: gauge + :level: diag + :oneliner: Number of bans using obj.* + + Number of bans which use obj.* variables. These bans can possibly + be washed by the ban-lurker. + +.. varnish_vsc:: bans_req + :type: gauge + :level: diag + :oneliner: Number of bans using req.* + + Number of bans which use req.* variables. These bans can not be + washed by the ban-lurker. + +.. varnish_vsc:: bans_added + :level: diag + :oneliner: Bans added + + Counter of bans added to ban list. + +.. varnish_vsc:: bans_deleted + :level: diag + :oneliner: Bans deleted + + Counter of bans deleted from ban list. + +.. varnish_vsc:: bans_tested + :level: diag + :oneliner: Bans tested against objects (lookup) + + Count of how many bans and objects have been tested against each + other during hash lookup. + +.. varnish_vsc:: bans_obj_killed + :level: diag + :oneliner: Objects killed by bans (lookup) + + Number of objects killed by bans during object lookup. + +.. varnish_vsc:: bans_lurker_tested + :level: diag + :oneliner: Bans tested against objects (lurker) + + Count of how many bans and objects have been tested against each + other by the ban-lurker. + +.. varnish_vsc:: bans_tests_tested + :level: diag + :oneliner: Ban tests tested against objects (lookup) + + Count of how many tests and objects have been tested against each + other during lookup. 'ban req.url == foo && req.http.host == bar' + counts as one in 'bans_tested' and as two in 'bans_tests_tested' + +.. varnish_vsc:: bans_lurker_tests_tested + :level: diag + :oneliner: Ban tests tested against objects (lurker) + + Count of how many tests and objects have been tested against each + other by the ban-lurker. 'ban req.url == foo && req.http.host == + bar' counts as one in 'bans_tested' and as two in 'bans_tests_tested' + +.. varnish_vsc:: bans_lurker_obj_killed + :level: diag + :oneliner: Objects killed by bans (lurker) + + Number of objects killed by the ban-lurker. + +.. varnish_vsc:: bans_lurker_obj_killed_cutoff + :level: diag + :oneliner: Objects killed by bans for cutoff (lurker) + + Number of objects killed by the ban-lurker to keep the number of + bans below ban_cutoff. + +.. varnish_vsc:: bans_dups + :level: diag + :oneliner: Bans superseded by other bans + + Count of bans replaced by later identical bans. + +.. varnish_vsc:: bans_lurker_contention + :level: diag + :oneliner: Lurker gave way for lookup + + Number of times the ban-lurker had to wait for lookups. + +.. varnish_vsc:: bans_persisted_bytes + :type: gauge + :format: bytes + :level: diag + :oneliner: Bytes used by the persisted ban lists + + Number of bytes used by the persisted ban lists. + +.. varnish_vsc:: bans_persisted_fragmentation + :type: gauge + :format: bytes + :level: diag + :oneliner: Extra bytes in persisted ban lists due to fragmentation + + Number of extra bytes accumulated through dropped and completed + bans in the persistent ban lists. + +.. varnish_vsc:: n_purges + :type: gauge + :oneliner: Number of purge operations executed + + +.. varnish_vsc:: n_obj_purged + :type: gauge + :oneliner: Number of purged objects + + +.. varnish_vsc:: exp_mailed + :level: diag + :oneliner: Number of objects mailed to expiry thread + + Number of objects mailed to expiry thread for handling. + +.. varnish_vsc:: exp_received + :level: diag + :oneliner: Number of objects received by expiry thread + + Number of objects received by expiry thread for handling. + +.. varnish_vsc:: hcb_nolock + :level: debug + :oneliner: HCB Lookups without lock + + +.. varnish_vsc:: hcb_lock + :level: debug + :oneliner: HCB Lookups with lock + + +.. varnish_vsc:: hcb_insert + :level: debug + :oneliner: HCB Inserts + + +.. varnish_vsc:: esi_errors + :level: diag + :oneliner: ESI parse errors (unlock) + + +.. varnish_vsc:: esi_warnings + :level: diag + :oneliner: ESI parse warnings (unlock) + + +.. varnish_vsc:: vmods + :type: gauge + :oneliner: Loaded VMODs + + +.. varnish_vsc:: n_gzip + :oneliner: Gzip operations + + +.. varnish_vsc:: n_gunzip + :oneliner: Gunzip operations + + +.. varnish_vsc:: n_test_gunzip + :oneliner: Test gunzip operations + + Those operations occur when Varnish receives a compressed object + from a backend. They are done to verify the gzip stream while it's + inserted in storage. + +.. varnish_vsc_end:: main diff --git a/bin/varnishd/VSC_mempool.vsc b/bin/varnishd/VSC_mempool.vsc new file mode 100644 index 0000000..8c47245 --- /dev/null +++ b/bin/varnishd/VSC_mempool.vsc @@ -0,0 +1,75 @@ +.. + This is *NOT* a RST file but the syntax has been chosen so + that it may become an RST file at some later date. + +.. varnish_vsc_begin:: mempool + :oneliner: Memory Pool Counters + :order: 30 + +.. varnish_vsc:: live + :type: gauge + :level: debug + :oneliner: In use + + +.. varnish_vsc:: pool + :type: gauge + :level: debug + :oneliner: In Pool + + +.. varnish_vsc:: sz_wanted + :type: gauge + :level: debug + :format: bytes + :oneliner: Size requested + + +.. varnish_vsc:: sz_actual + :type: gauge + :level: debug + :format: bytes + :oneliner: Size allocated + + +.. varnish_vsc:: allocs + :type: counter + :level: debug + :oneliner: Allocations + +.. varnish_vsc:: frees + :type: counter + :level: debug + :oneliner: Frees + +.. varnish_vsc:: recycle + :type: counter + :level: debug + :oneliner: Recycled from pool + + +.. varnish_vsc:: timeout + :type: counter + :level: debug + :oneliner: Timed out from pool + + +.. varnish_vsc:: toosmall + :type: counter + :level: debug + :oneliner: Too small to recycle + + +.. varnish_vsc:: surplus + :type: counter + :level: debug + :oneliner: Too many for pool + + +.. varnish_vsc:: randry + :type: counter + :level: debug + :oneliner: Pool ran dry + + +.. varnish_vsc_end:: mempool diff --git a/bin/varnishd/VSC_mgt.vsc b/bin/varnishd/VSC_mgt.vsc new file mode 100644 index 0000000..c3ca867 --- /dev/null +++ b/bin/varnishd/VSC_mgt.vsc @@ -0,0 +1,60 @@ +.. + This is *NOT* a RST file but the syntax has been chosen so + that it may become an RST file at some later date. + +.. varnish_vsc_begin:: mgt + :oneliner: Management Process Counters + :order: 20 + +.. varnish_vsc:: uptime + :type: counter + :format: duration + :level: info + :oneliner: Management process uptime + + Uptime in seconds of the management process + +.. varnish_vsc:: child_start + :type: counter + :level: diag + :oneliner: Child process started + + Number of times the child process has been started + +.. varnish_vsc:: child_exit + :type: counter + :level: diag + :oneliner: Child process normal exit + + Number of times the child process has been cleanly stopped + +.. varnish_vsc:: child_stop + :type: counter + :level: diag + :oneliner: Child process unexpected exit + + Number of times the child process has exited with an + unexpected return code + +.. varnish_vsc:: child_died + :type: counter + :level: diag + :oneliner: Child process died (signal) + + Number of times the child process has died due to signals + +.. varnish_vsc:: child_dump + :type: counter + :level: diag + :oneliner: Child process core dumped + + Number of times the child process has produced core dumps + +.. varnish_vsc:: child_panic + :type: counter + :level: diag + :oneliner: Child process panic + + Number of times the management process has caught a child panic + +.. varnish_vsc_end:: mgt diff --git a/bin/varnishd/VSC_sma.vsc b/bin/varnishd/VSC_sma.vsc new file mode 100644 index 0000000..d2ecc52 --- /dev/null +++ b/bin/varnishd/VSC_sma.vsc @@ -0,0 +1,62 @@ +.. + This is *NOT* a RST file but the syntax has been chosen so + that it may become an RST file at some later date. + +.. varnish_vsc_begin:: sma + :oneliner: Malloc Stevedore Counters + :order: 40 + +.. varnish_vsc:: c_req + :type: counter + :level: info + :oneliner: Allocator requests + + Number of times the storage has been asked to provide a storage segment. + +.. varnish_vsc:: c_fail + :type: counter + :level: info + :oneliner: Allocator failures + + Number of times the storage has failed to provide a storage segment. + +.. varnish_vsc:: c_bytes + :type: counter + :level: info + :format: bytes + :oneliner: Bytes allocated + + Number of total bytes allocated by this storage. + +.. varnish_vsc:: c_freed + :type: counter + :level: info + :format: bytes + :oneliner: Bytes freed + + Number of total bytes returned to this storage. + +.. varnish_vsc:: g_alloc + :type: gauge + :level: info + :oneliner: Allocations outstanding + + Number of storage allocations outstanding. + +.. varnish_vsc:: g_bytes + :type: gauge + :level: info + :format: bytes + :oneliner: Bytes outstanding + + Number of bytes allocated from the storage. + +.. varnish_vsc:: g_space + :type: gauge + :level: info + :format: bytes + :oneliner: Bytes available + + Number of bytes left in the storage. + +.. varnish_vsc_end:: sma diff --git a/bin/varnishd/VSC_smf.vsc b/bin/varnishd/VSC_smf.vsc new file mode 100644 index 0000000..4ec0dcc --- /dev/null +++ b/bin/varnishd/VSC_smf.vsc @@ -0,0 +1,80 @@ +.. + This is *NOT* a RST file but the syntax has been chosen so + that it may become an RST file at some later date. + +.. varnish_vsc_begin:: smf + :oneliner: File Stevedore Counters + :order: 50 + +.. varnish_vsc:: c_req + :type: counter + :level: info + :oneliner: Allocator requests + + Number of times the storage has been asked to provide a storage segment. + +.. varnish_vsc:: c_fail + :type: counter + :level: info + :oneliner: Allocator failures + + Number of times the storage has failed to provide a storage segment. + +.. varnish_vsc:: c_bytes + :type: counter + :level: info + :format: bytes + :oneliner: Bytes allocated + + Number of total bytes allocated by this storage. + +.. varnish_vsc:: c_freed + :type: counter + :level: info + :format: bytes + :oneliner: Bytes freed + + Number of total bytes returned to this storage. + +.. varnish_vsc:: g_alloc + :type: gauge + :level: info + :oneliner: Allocations outstanding + + Number of storage allocations outstanding. + +.. varnish_vsc:: g_bytes + :type: gauge + :level: info + :format: bytes + :oneliner: Bytes outstanding + + Number of bytes allocated from the storage. + +.. varnish_vsc:: g_space + :type: gauge + :level: info + :format: bytes + :oneliner: Bytes available + + Number of bytes left in the storage. + +.. varnish_vsc:: g_smf + :type: gauge + :level: info + :oneliner: N struct smf + + +.. varnish_vsc:: g_smf_frag + :type: gauge + :level: info + :oneliner: N small free smf + + +.. varnish_vsc:: g_smf_large + :type: gauge + :level: info + :oneliner: N large free smf + + +.. varnish_vsc_end:: smf diff --git a/bin/varnishd/VSC_smu.vsc b/bin/varnishd/VSC_smu.vsc new file mode 100644 index 0000000..89af32b --- /dev/null +++ b/bin/varnishd/VSC_smu.vsc @@ -0,0 +1,62 @@ +.. + This is *NOT* a RST file but the syntax has been chosen so + that it may become an RST file at some later date. + +.. varnish_vsc_begin:: smu + :oneliner: Umem Stevedore Counters + :order: 40 + +.. varnish_vsc:: c_req + :type: counter + :level: info + :oneliner: Allocator requests + + Number of times the storage has been asked to provide a storage segment. + +.. varnish_vsc:: c_fail + :type: counter + :level: info + :oneliner: Allocator failures + + Number of times the storage has failed to provide a storage segment. + +.. varnish_vsc:: c_bytes + :type: counter + :level: info + :format: bytes + :oneliner: Bytes allocated + + Number of total bytes allocated by this storage. + +.. varnish_vsc:: c_freed + :type: counter + :level: info + :format: bytes + :oneliner: Bytes freed + + Number of total bytes returned to this storage. + +.. varnish_vsc:: g_alloc + :type: gauge + :level: info + :oneliner: Allocations outstanding + + Number of storage allocations outstanding. + +.. varnish_vsc:: g_bytes + :type: gauge + :level: info + :format: bytes + :oneliner: Bytes outstanding + + Number of bytes allocated from the storage. + +.. varnish_vsc:: g_space + :type: gauge + :level: info + :format: bytes + :oneliner: Bytes available + + Number of bytes left in the storage. + +.. varnish_vsc_end:: smu diff --git a/bin/varnishd/VSC_vbe.vsc b/bin/varnishd/VSC_vbe.vsc new file mode 100644 index 0000000..c5ca37f --- /dev/null +++ b/bin/varnishd/VSC_vbe.vsc @@ -0,0 +1,82 @@ +.. + This is *NOT* a RST file but the syntax has been chosen so + that it may become an RST file at some later date. + +.. varnish_vsc_begin:: vbe + :oneliner: Backend Counters + :order: 60 + +.. varnish_vsc:: happy + :type: bitmap + :format: bitmap + :level: info + :oneliner: Happy health probes + +.. varnish_vsc:: bereq_hdrbytes + :type: counter + :level: info + :format: bytes + :oneliner: Request header bytes + + Total backend request header bytes sent + +.. varnish_vsc:: bereq_bodybytes + :type: counter + :level: info + :format: bytes + :oneliner: Request body bytes + + Total backend request body bytes sent + +.. varnish_vsc:: beresp_hdrbytes + :type: counter + :level: info + :format: bytes + :oneliner: Response header bytes + + Total backend response header bytes received + +.. varnish_vsc:: beresp_bodybytes + :type: counter + :level: info + :format: bytes + :oneliner: Response body bytes + + Total backend response body bytes received + +.. varnish_vsc:: pipe_hdrbytes + :type: counter + :level: info + :format: bytes + :oneliner: Pipe request header bytes + + Total request bytes sent for piped sessions + +.. varnish_vsc:: pipe_out + :type: counter + :level: info + :format: bytes + :oneliner: Piped bytes to backend + + Total number of bytes forwarded to backend in pipe sessions + +.. varnish_vsc:: pipe_in + :type: counter + :level: info + :format: bytes + :oneliner: Piped bytes from backend + + Total number of bytes forwarded from backend in pipe sessions + +.. varnish_vsc:: conn + :type: gauge + :level: info + :oneliner: Concurrent connections to backend + +.. varnish_vsc:: req + :type: counter + :level: info + :oneliner: Backend requests sent + +.. varnish_vsc_end:: vbe + diff --git a/bin/varnishd/lck.vsc b/bin/varnishd/lck.vsc deleted file mode 100644 index 75b69be..0000000 --- a/bin/varnishd/lck.vsc +++ /dev/null @@ -1,31 +0,0 @@ -.. - This is *NOT* a RST file but the syntax has been chosen so - that it may become an RST file at some later date. - -.. varnish_vsc_begin:: lck - :oneliner: Lock Counters - :order: 70 - - Counters which track the activity in the different classes - of mutex-locks. - -.. varnish_vsc:: creat - :type: counter - :level: debug - :oneliner: Created locks - - -.. varnish_vsc:: destroy - :type: counter - :level: debug - :oneliner: Destroyed locks - - -.. varnish_vsc:: locks - :type: counter - :level: debug - :oneliner: Lock Operations - - -.. varnish_vsc_end:: lck - diff --git a/bin/varnishd/main.vsc b/bin/varnishd/main.vsc deleted file mode 100644 index ca999f7..0000000 --- a/bin/varnishd/main.vsc +++ /dev/null @@ -1,731 +0,0 @@ -.. - This is *NOT* a RST file but the syntax has been chosen so - that it may become an RST file at some later date. - -.. varnish_vsc_begin:: main - :oneliner: Main counters - :order: 10 - :sumfunction: - -.. varnish_vsc:: summs - :level: debug - :oneliner: stat summ operations - -.. varnish_vsc:: uptime - :oneliner: Child process uptime - :format: duration - - How long the child process has been running. - -.. varnish_vsc:: sess_conn - :oneliner: Sessions accepted - - Count of sessions successfully accepted - -.. varnish_vsc:: sess_drop - :oneliner: Sessions dropped - - Count of sessions silently dropped due to lack of worker thread. - -.. varnish_vsc:: sess_fail - :oneliner: Session accept failures - - Count of failures to accept TCP connection. Either the client - changed its mind, or the kernel ran out of some resource like file - descriptors. - -.. varnish_vsc:: client_req_400 - :oneliner: Client requests received, subject to 400 errors - - 400 means we couldn't make sense of the request, it was malformed - in some drastic way. - -.. varnish_vsc:: client_req_417 - :oneliner: Client requests received, subject to 417 errors - - 417 means that something went wrong with an Expect: header. - -.. varnish_vsc:: client_req - :oneliner: Good client requests received - - The count of parseable client requests seen. - -.. varnish_vsc:: cache_hit - :oneliner: Cache hits - - Count of cache hits. A cache hit indicates that an object has been - delivered to a client without fetching it from a backend server. - -.. varnish_vsc:: cache_hitpass - :oneliner: Cache hits for pass. - - Count of hits for pass. A cache hit for pass indicates that Varnish - is going to pass the request to the backend and this decision has - been cached in it self. This counts how many times the cached - decision is being used. - -.. varnish_vsc:: cache_hitmiss - :oneliner: Cache hits for miss. - - Count of hits for miss. A cache hit for miss indicates that Varnish - is going to proceed as for a cache miss without request coalescing, - and this decision has been cached. This counts how many times the - cached decision is being used. - -.. varnish_vsc:: cache_miss - :oneliner: Cache misses - - Count of misses. A cache miss indicates the object was fetched from - the backend before delivering it to the client. - -.. varnish_vsc:: backend_conn - :oneliner: Backend conn. success - - How many backend connections have successfully been established. - -.. varnish_vsc:: backend_unhealthy - :oneliner: Backend conn. not attempted - - -.. varnish_vsc:: backend_busy - :oneliner: Backend conn. too many - - -.. varnish_vsc:: backend_fail - :oneliner: Backend conn. failures - - -.. varnish_vsc:: backend_reuse - :oneliner: Backend conn. reuses - - Count of backend connection reuses. This counter is increased - whenever we reuse a recycled connection. - -.. varnish_vsc:: backend_recycle - :oneliner: Backend conn. recycles - - Count of backend connection recycles. This counter is increased - whenever we have a keep-alive connection that is put back into the - pool of connections. It has not yet been used, but it might be, - unless the backend closes it. - -.. varnish_vsc:: backend_retry - :oneliner: Backend conn. retry - - -.. varnish_vsc:: fetch_head - :oneliner: Fetch no body (HEAD) - - beresp with no body because the request is HEAD. - -.. varnish_vsc:: fetch_length - :oneliner: Fetch with Length - - beresp.body with Content-Length. - -.. varnish_vsc:: fetch_chunked - :oneliner: Fetch chunked - - beresp.body with Chunked. - -.. varnish_vsc:: fetch_eof - :oneliner: Fetch EOF - - beresp.body with EOF. - -.. varnish_vsc:: fetch_bad - :oneliner: Fetch bad T-E - - beresp.body length/fetch could not be determined. - -.. varnish_vsc:: fetch_none - :oneliner: Fetch no body - - beresp.body empty - -.. varnish_vsc:: fetch_1xx - :oneliner: Fetch no body (1xx) - - beresp with no body because of 1XX response. - -.. varnish_vsc:: fetch_204 - :oneliner: Fetch no body (204) - - beresp with no body because of 204 response. - -.. varnish_vsc:: fetch_304 - :oneliner: Fetch no body (304) - - beresp with no body because of 304 response. - -.. varnish_vsc:: fetch_failed - :oneliner: Fetch failed (all causes) - - beresp fetch failed. - -.. varnish_vsc:: fetch_no_thread - :oneliner: Fetch failed (no thread) - - beresp fetch failed, no thread available. - -.. varnish_vsc:: pools - :type: gauge - :oneliner: Number of thread pools - - Number of thread pools. See also parameter thread_pools. NB: Presently - pools cannot be removed once created. - -.. varnish_vsc:: threads - :type: gauge - :oneliner: Total number of threads - - Number of threads in all pools. See also parameters thread_pools, - thread_pool_min and thread_pool_max. - -.. varnish_vsc:: threads_limited - :oneliner: Threads hit max - - Number of times more threads were needed, but limit was reached in - a thread pool. See also parameter thread_pool_max. - -.. varnish_vsc:: threads_created - :oneliner: Threads created - - Total number of threads created in all pools. - -.. varnish_vsc:: threads_destroyed - :oneliner: Threads destroyed - - Total number of threads destroyed in all pools. - -.. varnish_vsc:: threads_failed - :oneliner: Thread creation failed - - Number of times creating a thread failed. See VSL::Debug for - diagnostics. See also parameter thread_fail_delay. - -.. varnish_vsc:: thread_queue_len - :type: gauge - :oneliner: Length of session queue - - Length of session queue waiting for threads. NB: Only updates once - per second. See also parameter thread_queue_limit. - -.. varnish_vsc:: busy_sleep - :oneliner: Number of requests sent to sleep on busy objhdr - - Number of requests sent to sleep without a worker thread because - they found a busy object. - -.. varnish_vsc:: busy_wakeup - :oneliner: Number of requests woken after sleep on busy objhdr - - Number of requests taken off the busy object sleep list and rescheduled. - -.. varnish_vsc:: busy_killed - :oneliner: Number of requests killed after sleep on busy objhdr - - Number of requests killed from the busy object sleep list due to - lack of resources. - -.. varnish_vsc:: sess_queued - :oneliner: Sessions queued for thread - - Number of times session was queued waiting for a thread. See also - parameter thread_queue_limit. - -.. varnish_vsc:: sess_dropped - :oneliner: Sessions dropped for thread - - Number of times an HTTP/1 session was dropped because the queue was - too long already. See also parameter thread_queue_limit. - -.. varnish_vsc:: req_dropped - :oneliner: Requests dropped - - Number of times an HTTP/2 stream was refused because the queue was - too long already. See also parameter thread_queue_limit. - -.. varnish_vsc:: n_object - :type: gauge - :oneliner: object structs made - - Approximate number of HTTP objects (headers + body, if present) in - the cache. - -.. varnish_vsc:: n_vampireobject - :type: gauge - :level: diag - :oneliner: unresurrected objects - - Number of unresurrected objects - -.. varnish_vsc:: n_objectcore - :type: gauge - :oneliner: objectcore structs made - - Approximate number of object metadata elements in the cache. Each - object needs an objectcore, extra objectcores are for hit-for-miss, - hit-for-pass and busy objects. - -.. varnish_vsc:: n_objecthead - :type: gauge - :oneliner: objecthead structs made - - Approximate number of different hash entries in the cache. - -.. varnish_vsc:: n_backend - :type: gauge - :oneliner: Number of backends - - Number of backends known to us. - -.. varnish_vsc:: n_expired - :type: gauge - :oneliner: Number of expired objects - - Number of objects that expired from cache because of old age. - -.. varnish_vsc:: n_lru_nuked - :type: gauge - :oneliner: Number of LRU nuked objects - - How many objects have been forcefully evicted from storage to make - room for a new object. - -.. varnish_vsc:: n_lru_moved - :type: gauge - :level: diag - :oneliner: Number of LRU moved objects - - Number of move operations done on the LRU list. - -.. varnish_vsc:: losthdr - :oneliner: HTTP header overflows - - -.. varnish_vsc:: s_sess - :oneliner: Total sessions seen - - -.. varnish_vsc:: s_pipe - :oneliner: Total pipe sessions seen - - -.. varnish_vsc:: s_pass - :oneliner: Total pass-ed requests seen - - -.. varnish_vsc:: s_fetch - :oneliner: Total backend fetches initiated - - -.. varnish_vsc:: s_synth - :oneliner: Total synthethic responses made - - -.. varnish_vsc:: s_req_hdrbytes - :oneliner: Request header bytes - :format: bytes - - Total request header bytes received - -.. varnish_vsc:: s_req_bodybytes - :oneliner: Request body bytes - :format: bytes - - Total request body bytes received - -.. varnish_vsc:: s_resp_hdrbytes - :oneliner: Response header bytes - :format: bytes - - Total response header bytes transmitted - -.. varnish_vsc:: s_resp_bodybytes - :oneliner: Response body bytes - :format: bytes - - Total response body bytes transmitted - :format: bytes - -.. varnish_vsc:: s_pipe_hdrbytes - :oneliner: Pipe request header bytes - :format: bytes - - Total request bytes received for piped sessions - -.. varnish_vsc:: s_pipe_in - :oneliner: Piped bytes from client - :format: bytes - - Total number of bytes forwarded from clients in pipe sessions - -.. varnish_vsc:: s_pipe_out - :oneliner: Piped bytes to client - :format: bytes - - Total number of bytes forwarded to clients in pipe sessions - -.. varnish_vsc:: sess_closed - :oneliner: Session Closed - - -.. varnish_vsc:: sess_closed_err - :oneliner: Session Closed with error - - Total number of sessions closed with errors. See sc_* diag counters - for detailed breakdown - -.. varnish_vsc:: sess_readahead - :oneliner: Session Read Ahead - - -.. varnish_vsc:: sess_herd - :level: diag - :oneliner: Session herd - - Number of times the timeout_linger triggered - -.. varnish_vsc:: sc_rem_close - :level: diag - :oneliner: Session OK REM_CLOSE - - Number of session closes with REM_CLOSE (Client Closed) - -.. varnish_vsc:: sc_req_close - :level: diag - :oneliner: Session OK REQ_CLOSE - - Number of session closes with REQ_CLOSE (Client requested close) - -.. varnish_vsc:: sc_req_http10 - :level: diag - :oneliner: Session Err REQ_HTTP10 - - Number of session closes with Error REQ_HTTP10 (Proto < HTTP/1.1) - -.. varnish_vsc:: sc_rx_bad - :level: diag - :oneliner: Session Err RX_BAD - - Number of session closes with Error RX_BAD (Received bad req/resp) - -.. varnish_vsc:: sc_rx_body - :level: diag - :oneliner: Session Err RX_BODY - - Number of session closes with Error RX_BODY (Failure receiving req.body) - -.. varnish_vsc:: sc_rx_junk - :level: diag - :oneliner: Session Err RX_JUNK - - Number of session closes with Error RX_JUNK (Received junk data) - -.. varnish_vsc:: sc_rx_overflow - :level: diag - :oneliner: Session Err RX_OVERFLOW - - Number of session closes with Error RX_OVERFLOW (Received buffer overflow) - -.. varnish_vsc:: sc_rx_timeout - :level: diag - :oneliner: Session Err RX_TIMEOUT - - Number of session closes with Error RX_TIMEOUT (Receive timeout) - -.. varnish_vsc:: sc_tx_pipe - :level: diag - :oneliner: Session OK TX_PIPE - - Number of session closes with TX_PIPE (Piped transaction) - -.. varnish_vsc:: sc_tx_error - :level: diag - :oneliner: Session Err TX_ERROR - - Number of session closes with Error TX_ERROR (Error transaction) - -.. varnish_vsc:: sc_tx_eof - :level: diag - :oneliner: Session OK TX_EOF - - Number of session closes with TX_EOF (EOF transmission) - -.. varnish_vsc:: sc_resp_close - :level: diag - :oneliner: Session OK RESP_CLOSE - - Number of session closes with RESP_CLOSE (Backend/VCL requested close) - -.. varnish_vsc:: sc_overload - :level: diag - :oneliner: Session Err OVERLOAD - - Number of session closes with Error OVERLOAD (Out of some resource) - -.. varnish_vsc:: sc_pipe_overflow - :level: diag - :oneliner: Session Err PIPE_OVERFLOW - - Number of session closes with Error PIPE_OVERFLOW (Session pipe overflow) - -.. varnish_vsc:: sc_range_short - :level: diag - :oneliner: Session Err RANGE_SHORT - - Number of session closes with Error RANGE_SHORT (Insufficient data for range) - -.. varnish_vsc:: sc_req_http20 - :level: diag - :oneliner: Session Err REQ_HTTP20 - - Number of session closes with Error REQ_HTTP20 (HTTP2 not accepted) - -.. varnish_vsc:: sc_vcl_failure - :level: diag - :oneliner: Session Err VCL_FAILURE - - Number of session closes with Error VCL_FAILURE (VCL failure) - -.. varnish_vsc:: shm_records - :level: diag - :oneliner: SHM records - - -.. varnish_vsc:: shm_writes - :level: diag - :oneliner: SHM writes - - -.. varnish_vsc:: shm_flushes - :level: diag - :oneliner: SHM flushes due to overflow - - -.. varnish_vsc:: shm_cont - :level: diag - :oneliner: SHM MTX contention - - -.. varnish_vsc:: shm_cycles - :level: diag - :oneliner: SHM cycles through buffer - - -.. varnish_vsc:: backend_req - :oneliner: Backend requests made - - -.. varnish_vsc:: n_vcl - :type: gauge - :oneliner: Number of loaded VCLs in total - - -.. varnish_vsc:: n_vcl_avail - :type: gauge - :level: diag - :oneliner: Number of VCLs available - - -.. varnish_vsc:: n_vcl_discard - :type: gauge - :level: diag - :oneliner: Number of discarded VCLs - - -.. varnish_vsc:: vcl_fail - :oneliner: VCL failures - - Count of failures which prevented VCL from completing. - -.. varnish_vsc:: bans - :type: gauge - :oneliner: Count of bans - - Number of all bans in system, including bans superseded by newer - bans and bans already checked by the ban-lurker. - -.. varnish_vsc:: bans_completed - :type: gauge - :level: diag - :oneliner: Number of bans marked 'completed' - - Number of bans which are no longer active, either because they got - checked by the ban-lurker or superseded by newer identical bans. - -.. varnish_vsc:: bans_obj - :type: gauge - :level: diag - :oneliner: Number of bans using obj.* - - Number of bans which use obj.* variables. These bans can possibly - be washed by the ban-lurker. - -.. varnish_vsc:: bans_req - :type: gauge - :level: diag - :oneliner: Number of bans using req.* - - Number of bans which use req.* variables. These bans can not be - washed by the ban-lurker. - -.. varnish_vsc:: bans_added - :level: diag - :oneliner: Bans added - - Counter of bans added to ban list. - -.. varnish_vsc:: bans_deleted - :level: diag - :oneliner: Bans deleted - - Counter of bans deleted from ban list. - -.. varnish_vsc:: bans_tested - :level: diag - :oneliner: Bans tested against objects (lookup) - - Count of how many bans and objects have been tested against each - other during hash lookup. - -.. varnish_vsc:: bans_obj_killed - :level: diag - :oneliner: Objects killed by bans (lookup) - - Number of objects killed by bans during object lookup. - -.. varnish_vsc:: bans_lurker_tested - :level: diag - :oneliner: Bans tested against objects (lurker) - - Count of how many bans and objects have been tested against each - other by the ban-lurker. - -.. varnish_vsc:: bans_tests_tested - :level: diag - :oneliner: Ban tests tested against objects (lookup) - - Count of how many tests and objects have been tested against each - other during lookup. 'ban req.url == foo && req.http.host == bar' - counts as one in 'bans_tested' and as two in 'bans_tests_tested' - -.. varnish_vsc:: bans_lurker_tests_tested - :level: diag - :oneliner: Ban tests tested against objects (lurker) - - Count of how many tests and objects have been tested against each - other by the ban-lurker. 'ban req.url == foo && req.http.host == - bar' counts as one in 'bans_tested' and as two in 'bans_tests_tested' - -.. varnish_vsc:: bans_lurker_obj_killed - :level: diag - :oneliner: Objects killed by bans (lurker) - - Number of objects killed by the ban-lurker. - -.. varnish_vsc:: bans_lurker_obj_killed_cutoff - :level: diag - :oneliner: Objects killed by bans for cutoff (lurker) - - Number of objects killed by the ban-lurker to keep the number of - bans below ban_cutoff. - -.. varnish_vsc:: bans_dups - :level: diag - :oneliner: Bans superseded by other bans - - Count of bans replaced by later identical bans. - -.. varnish_vsc:: bans_lurker_contention - :level: diag - :oneliner: Lurker gave way for lookup - - Number of times the ban-lurker had to wait for lookups. - -.. varnish_vsc:: bans_persisted_bytes - :type: gauge - :format: bytes - :level: diag - :oneliner: Bytes used by the persisted ban lists - - Number of bytes used by the persisted ban lists. - -.. varnish_vsc:: bans_persisted_fragmentation - :type: gauge - :format: bytes - :level: diag - :oneliner: Extra bytes in persisted ban lists due to fragmentation - - Number of extra bytes accumulated through dropped and completed - bans in the persistent ban lists. - -.. varnish_vsc:: n_purges - :type: gauge - :oneliner: Number of purge operations executed - - -.. varnish_vsc:: n_obj_purged - :type: gauge - :oneliner: Number of purged objects - - -.. varnish_vsc:: exp_mailed - :level: diag - :oneliner: Number of objects mailed to expiry thread - - Number of objects mailed to expiry thread for handling. - -.. varnish_vsc:: exp_received - :level: diag - :oneliner: Number of objects received by expiry thread - - Number of objects received by expiry thread for handling. - -.. varnish_vsc:: hcb_nolock - :level: debug - :oneliner: HCB Lookups without lock - - -.. varnish_vsc:: hcb_lock - :level: debug - :oneliner: HCB Lookups with lock - - -.. varnish_vsc:: hcb_insert - :level: debug - :oneliner: HCB Inserts - - -.. varnish_vsc:: esi_errors - :level: diag - :oneliner: ESI parse errors (unlock) - - -.. varnish_vsc:: esi_warnings - :level: diag - :oneliner: ESI parse warnings (unlock) - - -.. varnish_vsc:: vmods - :type: gauge - :oneliner: Loaded VMODs - - -.. varnish_vsc:: n_gzip - :oneliner: Gzip operations - - -.. varnish_vsc:: n_gunzip - :oneliner: Gunzip operations - - -.. varnish_vsc:: n_test_gunzip - :oneliner: Test gunzip operations - - Those operations occur when Varnish receives a compressed object - from a backend. They are done to verify the gzip stream while it's - inserted in storage. - -.. varnish_vsc_end:: main diff --git a/bin/varnishd/mempool.vsc b/bin/varnishd/mempool.vsc deleted file mode 100644 index 8c47245..0000000 --- a/bin/varnishd/mempool.vsc +++ /dev/null @@ -1,75 +0,0 @@ -.. - This is *NOT* a RST file but the syntax has been chosen so - that it may become an RST file at some later date. - -.. varnish_vsc_begin:: mempool - :oneliner: Memory Pool Counters - :order: 30 - -.. varnish_vsc:: live - :type: gauge - :level: debug - :oneliner: In use - - -.. varnish_vsc:: pool - :type: gauge - :level: debug - :oneliner: In Pool - - -.. varnish_vsc:: sz_wanted - :type: gauge - :level: debug - :format: bytes - :oneliner: Size requested - - -.. varnish_vsc:: sz_actual - :type: gauge - :level: debug - :format: bytes - :oneliner: Size allocated - - -.. varnish_vsc:: allocs - :type: counter - :level: debug - :oneliner: Allocations - -.. varnish_vsc:: frees - :type: counter - :level: debug - :oneliner: Frees - -.. varnish_vsc:: recycle - :type: counter - :level: debug - :oneliner: Recycled from pool - - -.. varnish_vsc:: timeout - :type: counter - :level: debug - :oneliner: Timed out from pool - - -.. varnish_vsc:: toosmall - :type: counter - :level: debug - :oneliner: Too small to recycle - - -.. varnish_vsc:: surplus - :type: counter - :level: debug - :oneliner: Too many for pool - - -.. varnish_vsc:: randry - :type: counter - :level: debug - :oneliner: Pool ran dry - - -.. varnish_vsc_end:: mempool diff --git a/bin/varnishd/mgt.vsc b/bin/varnishd/mgt.vsc deleted file mode 100644 index c3ca867..0000000 --- a/bin/varnishd/mgt.vsc +++ /dev/null @@ -1,60 +0,0 @@ -.. - This is *NOT* a RST file but the syntax has been chosen so - that it may become an RST file at some later date. - -.. varnish_vsc_begin:: mgt - :oneliner: Management Process Counters - :order: 20 - -.. varnish_vsc:: uptime - :type: counter - :format: duration - :level: info - :oneliner: Management process uptime - - Uptime in seconds of the management process - -.. varnish_vsc:: child_start - :type: counter - :level: diag - :oneliner: Child process started - - Number of times the child process has been started - -.. varnish_vsc:: child_exit - :type: counter - :level: diag - :oneliner: Child process normal exit - - Number of times the child process has been cleanly stopped - -.. varnish_vsc:: child_stop - :type: counter - :level: diag - :oneliner: Child process unexpected exit - - Number of times the child process has exited with an - unexpected return code - -.. varnish_vsc:: child_died - :type: counter - :level: diag - :oneliner: Child process died (signal) - - Number of times the child process has died due to signals - -.. varnish_vsc:: child_dump - :type: counter - :level: diag - :oneliner: Child process core dumped - - Number of times the child process has produced core dumps - -.. varnish_vsc:: child_panic - :type: counter - :level: diag - :oneliner: Child process panic - - Number of times the management process has caught a child panic - -.. varnish_vsc_end:: mgt diff --git a/bin/varnishd/sma.vsc b/bin/varnishd/sma.vsc deleted file mode 100644 index d2ecc52..0000000 --- a/bin/varnishd/sma.vsc +++ /dev/null @@ -1,62 +0,0 @@ -.. - This is *NOT* a RST file but the syntax has been chosen so - that it may become an RST file at some later date. - -.. varnish_vsc_begin:: sma - :oneliner: Malloc Stevedore Counters - :order: 40 - -.. varnish_vsc:: c_req - :type: counter - :level: info - :oneliner: Allocator requests - - Number of times the storage has been asked to provide a storage segment. - -.. varnish_vsc:: c_fail - :type: counter - :level: info - :oneliner: Allocator failures - - Number of times the storage has failed to provide a storage segment. - -.. varnish_vsc:: c_bytes - :type: counter - :level: info - :format: bytes - :oneliner: Bytes allocated - - Number of total bytes allocated by this storage. - -.. varnish_vsc:: c_freed - :type: counter - :level: info - :format: bytes - :oneliner: Bytes freed - - Number of total bytes returned to this storage. - -.. varnish_vsc:: g_alloc - :type: gauge - :level: info - :oneliner: Allocations outstanding - - Number of storage allocations outstanding. - -.. varnish_vsc:: g_bytes - :type: gauge - :level: info - :format: bytes - :oneliner: Bytes outstanding - - Number of bytes allocated from the storage. - -.. varnish_vsc:: g_space - :type: gauge - :level: info - :format: bytes - :oneliner: Bytes available - - Number of bytes left in the storage. - -.. varnish_vsc_end:: sma diff --git a/bin/varnishd/smf.vsc b/bin/varnishd/smf.vsc deleted file mode 100644 index 4ec0dcc..0000000 --- a/bin/varnishd/smf.vsc +++ /dev/null @@ -1,80 +0,0 @@ -.. - This is *NOT* a RST file but the syntax has been chosen so - that it may become an RST file at some later date. - -.. varnish_vsc_begin:: smf - :oneliner: File Stevedore Counters - :order: 50 - -.. varnish_vsc:: c_req - :type: counter - :level: info - :oneliner: Allocator requests - - Number of times the storage has been asked to provide a storage segment. - -.. varnish_vsc:: c_fail - :type: counter - :level: info - :oneliner: Allocator failures - - Number of times the storage has failed to provide a storage segment. - -.. varnish_vsc:: c_bytes - :type: counter - :level: info - :format: bytes - :oneliner: Bytes allocated - - Number of total bytes allocated by this storage. - -.. varnish_vsc:: c_freed - :type: counter - :level: info - :format: bytes - :oneliner: Bytes freed - - Number of total bytes returned to this storage. - -.. varnish_vsc:: g_alloc - :type: gauge - :level: info - :oneliner: Allocations outstanding - - Number of storage allocations outstanding. - -.. varnish_vsc:: g_bytes - :type: gauge - :level: info - :format: bytes - :oneliner: Bytes outstanding - - Number of bytes allocated from the storage. - -.. varnish_vsc:: g_space - :type: gauge - :level: info - :format: bytes - :oneliner: Bytes available - - Number of bytes left in the storage. - -.. varnish_vsc:: g_smf - :type: gauge - :level: info - :oneliner: N struct smf - - -.. varnish_vsc:: g_smf_frag - :type: gauge - :level: info - :oneliner: N small free smf - - -.. varnish_vsc:: g_smf_large - :type: gauge - :level: info - :oneliner: N large free smf - - -.. varnish_vsc_end:: smf diff --git a/bin/varnishd/smu.vsc b/bin/varnishd/smu.vsc deleted file mode 100644 index 89af32b..0000000 --- a/bin/varnishd/smu.vsc +++ /dev/null @@ -1,62 +0,0 @@ -.. - This is *NOT* a RST file but the syntax has been chosen so - that it may become an RST file at some later date. - -.. varnish_vsc_begin:: smu - :oneliner: Umem Stevedore Counters - :order: 40 - -.. varnish_vsc:: c_req - :type: counter - :level: info - :oneliner: Allocator requests - - Number of times the storage has been asked to provide a storage segment. - -.. varnish_vsc:: c_fail - :type: counter - :level: info - :oneliner: Allocator failures - - Number of times the storage has failed to provide a storage segment. - -.. varnish_vsc:: c_bytes - :type: counter - :level: info - :format: bytes - :oneliner: Bytes allocated - - Number of total bytes allocated by this storage. - -.. varnish_vsc:: c_freed - :type: counter - :level: info - :format: bytes - :oneliner: Bytes freed - - Number of total bytes returned to this storage. - -.. varnish_vsc:: g_alloc - :type: gauge - :level: info - :oneliner: Allocations outstanding - - Number of storage allocations outstanding. - -.. varnish_vsc:: g_bytes - :type: gauge - :level: info - :format: bytes - :oneliner: Bytes outstanding - - Number of bytes allocated from the storage. - -.. varnish_vsc:: g_space - :type: gauge - :level: info - :format: bytes - :oneliner: Bytes available - - Number of bytes left in the storage. - -.. varnish_vsc_end:: smu diff --git a/bin/varnishd/vbe.vsc b/bin/varnishd/vbe.vsc deleted file mode 100644 index c5ca37f..0000000 --- a/bin/varnishd/vbe.vsc +++ /dev/null @@ -1,82 +0,0 @@ -.. - This is *NOT* a RST file but the syntax has been chosen so - that it may become an RST file at some later date. - -.. varnish_vsc_begin:: vbe - :oneliner: Backend Counters - :order: 60 - -.. varnish_vsc:: happy - :type: bitmap - :format: bitmap - :level: info - :oneliner: Happy health probes - -.. varnish_vsc:: bereq_hdrbytes - :type: counter - :level: info - :format: bytes - :oneliner: Request header bytes - - Total backend request header bytes sent - -.. varnish_vsc:: bereq_bodybytes - :type: counter - :level: info - :format: bytes - :oneliner: Request body bytes - - Total backend request body bytes sent - -.. varnish_vsc:: beresp_hdrbytes - :type: counter - :level: info - :format: bytes - :oneliner: Response header bytes - - Total backend response header bytes received - -.. varnish_vsc:: beresp_bodybytes - :type: counter - :level: info - :format: bytes - :oneliner: Response body bytes - - Total backend response body bytes received - -.. varnish_vsc:: pipe_hdrbytes - :type: counter - :level: info - :format: bytes - :oneliner: Pipe request header bytes - - Total request bytes sent for piped sessions - -.. varnish_vsc:: pipe_out - :type: counter - :level: info - :format: bytes - :oneliner: Piped bytes to backend - - Total number of bytes forwarded to backend in pipe sessions - -.. varnish_vsc:: pipe_in - :type: counter - :level: info - :format: bytes - :oneliner: Piped bytes from backend - - Total number of bytes forwarded from backend in pipe sessions - -.. varnish_vsc:: conn - :type: gauge - :level: info - :oneliner: Concurrent connections to backend - -.. varnish_vsc:: req - :type: counter - :level: info - :oneliner: Backend requests sent - -.. varnish_vsc_end:: vbe - diff --git a/configure.ac b/configure.ac index cf8f9b5..a6081a5 100644 --- a/configure.ac +++ b/configure.ac @@ -688,6 +688,14 @@ AC_DEFINE_UNQUOTED([VCC_CC],"$VCC_CC",[C compiler command line for VCL code]) VTC_TESTS="$(cd $srcdir/bin/varnishtest && echo tests/*.vtc)" AC_SUBST(VTC_TESTS) +VSC_SRC="$(cd $srcdir/bin/varnishd && echo *.vsc)" +VSC_GEN_C="$(echo $VSC_SRC | sed 's:\.vsc:.c:g')" +VSC_GEN_H="$(echo $VSC_SRC | sed 's:\.vsc:.h:g')" +# automake does not allow to AC_SUBST _SOURCES files (for a good +# reason), so we automate the cleanup at least +AC_SUBST(VSC_GEN_C) +AC_SUBST(VSC_GEN_H) + # Make sure this include dir exists AC_CONFIG_COMMANDS([mkdir], [$MKDIR_P doc/sphinx/include]) diff --git a/doc/sphinx/Makefile.am b/doc/sphinx/Makefile.am index 3c3142f..490ee50 100644 --- a/doc/sphinx/Makefile.am +++ b/doc/sphinx/Makefile.am @@ -124,14 +124,14 @@ include/params.rst: $(top_builddir)/bin/varnishd/varnishd BUILT_SOURCES += include/params.rst COUNTERS = \ - $(top_srcdir)/bin/varnishd/main.vsc \ - $(top_srcdir)/bin/varnishd/mgt.vsc \ - $(top_srcdir)/bin/varnishd/mempool.vsc \ - $(top_srcdir)/bin/varnishd/sma.vsc \ - $(top_srcdir)/bin/varnishd/smu.vsc \ - $(top_srcdir)/bin/varnishd/smf.vsc \ - $(top_srcdir)/bin/varnishd/vbe.vsc \ - $(top_srcdir)/bin/varnishd/lck.vsc + $(top_srcdir)/bin/varnishd/VSC_main.vsc \ + $(top_srcdir)/bin/varnishd/VSC_mgt.vsc \ + $(top_srcdir)/bin/varnishd/VSC_mempool.vsc \ + $(top_srcdir)/bin/varnishd/VSC_sma.vsc \ + $(top_srcdir)/bin/varnishd/VSC_smu.vsc \ + $(top_srcdir)/bin/varnishd/VSC_smf.vsc \ + $(top_srcdir)/bin/varnishd/VSC_vbe.vsc \ + $(top_srcdir)/bin/varnishd/VSC_lck.vsc include/counters.rst: $(top_srcdir)/lib/libvcc/vsctool.py $(COUNTERS) echo -n '' > $@ From nils.goroll at uplex.de Thu Oct 5 17:36:07 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 5 Oct 2017 17:36:07 +0000 (UTC) Subject: [master] 9f6c0de Try to get the __unused madness out of the way for now Message-ID: <20171005173607.5089664857@lists.varnish-cache.org> commit 9f6c0deac0465475631301c629b8bb46b3d65ab4 Author: Nils Goroll Date: Thu Oct 5 19:35:30 2017 +0200 Try to get the __unused madness out of the way for now diff --git a/bin/varnishd/proxy/cache_proxy_proto.c b/bin/varnishd/proxy/cache_proxy_proto.c index b3aaf1d..b25347f 100644 --- a/bin/varnishd/proxy/cache_proxy_proto.c +++ b/bin/varnishd/proxy/cache_proxy_proto.c @@ -29,12 +29,10 @@ #include "config.h" -#include "cache/cache_varnishd.h" - #include - #include +#include "cache/cache_varnishd.h" #include "cache/cache_transport.h" #include "vend.h" From nils.goroll at uplex.de Fri Oct 6 11:36:05 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 6 Oct 2017 11:36:05 +0000 (UTC) Subject: [master] 75fe1e4 Replace FetchError "no backend connection" with something more useful Message-ID: <20171006113605.F37296434D@lists.varnish-cache.org> commit 75fe1e48ce49cbf66c759d1283cf152d673d5d85 Author: Nils Goroll Date: Fri Oct 6 13:29:46 2017 +0200 Replace FetchError "no backend connection" with something more useful Previously, a FetchError in vbe_dir_getfd() was too general to be useful. Now we log an error with the same identifier as the corresponding statistic: * unhealthy * busy (max_connections reached) * fail (no connection for real) For these, we also log the backend name because we have not yet issued a BackendOpen VSL. A failed WS_Alloc is logged as "out of workspace". diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index 0956afb..b344770 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -77,12 +77,16 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo) AN(bp->vsc); if (!VBE_Healthy(bp, NULL)) { + VSLb(bo->vsl, SLT_FetchError, + "backend %s: unhealthy", bp->display_name); // XXX: per backend stats ? VSC_C_main->backend_unhealthy++; return (NULL); } if (bp->max_connections > 0 && bp->n_conn >= bp->max_connections) { + VSLb(bo->vsl, SLT_FetchError, + "backend %s: busy", bp->display_name); // XXX: per backend stats ? VSC_C_main->backend_busy++; return (NULL); @@ -90,14 +94,18 @@ vbe_dir_getfd(struct worker *wrk, struct backend *bp, struct busyobj *bo) AZ(bo->htc); bo->htc = WS_Alloc(bo->ws, sizeof *bo->htc); - if (bo->htc == NULL) + if (bo->htc == NULL) { + VSLb(bo->vsl, SLT_FetchError, "out of workspace"); /* XXX: counter ? */ return (NULL); + } bo->htc->doclose = SC_NULL; FIND_TMO(connect_timeout, tmod, bo, bp); vtp = VTP_Get(bp->tcp_pool, tmod, wrk); if (vtp == NULL) { + VSLb(bo->vsl, SLT_FetchError, + "backend %s: fail", bp->display_name); // XXX: Per backend stats ? VSC_C_main->backend_fail++; bo->htc = NULL; @@ -204,10 +212,8 @@ vbe_dir_gethdrs(const struct director *d, struct worker *wrk, do { vtp = vbe_dir_getfd(wrk, bp, bo); - if (vtp == NULL) { - VSLb(bo->vsl, SLT_FetchError, "no backend connection"); + if (vtp == NULL) return (-1); - } AN(bo->htc); if (vtp->state != VTP_STATE_STOLEN) extrachance = 0; @@ -286,7 +292,6 @@ vbe_dir_http1pipe(const struct director *d, struct req *req, struct busyobj *bo) vtp = vbe_dir_getfd(req->wrk, bp, bo); if (vtp == NULL) { - VSLb(bo->vsl, SLT_FetchError, "no backend connection"); retval = SC_TX_ERROR; } else { i = V1F_SendReq(req->wrk, bo, &v1a.bereq, 1); diff --git a/bin/varnishtest/tests/r01990.vtc b/bin/varnishtest/tests/r01990.vtc index 2b9dbbd..5ee40bd 100644 --- a/bin/varnishtest/tests/r01990.vtc +++ b/bin/varnishtest/tests/r01990.vtc @@ -23,7 +23,7 @@ varnish v1 -vcl { } -start logexpect l1 -v v1 -g raw { - expect * 1002 FetchError {^no backend connection} + expect * 1002 FetchError {^out of workspace} expect * = BerespStatus {^503} expect * = BerespReason {^Backend fetch failed} expect * = Error {^out of workspace [(]Bo[)]} From nils.goroll at uplex.de Fri Oct 6 13:01:05 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 6 Oct 2017 13:01:05 +0000 (UTC) Subject: [master] c0f7fc3 TASK_PRIV in cli contexts (vcl_init/vcl_fini) needs to be per-vmod Message-ID: <20171006130105.82D8C64DA8@lists.varnish-cache.org> commit c0f7fc3646bd0859c1d15ccc34888e5c095db7d3 Author: Nils Goroll Date: Fri Oct 6 14:55:55 2017 +0200 TASK_PRIV in cli contexts (vcl_init/vcl_fini) needs to be per-vmod Fixes #2451 diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 6ef4221..7792ab1 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -101,6 +101,7 @@ static struct vrt_ctx ctx_cli; static unsigned handling_cli; static struct ws ws_cli; static uintptr_t ws_snapshot_cli; +extern struct vrt_privs cli_task_privs[1]; /*--------------------------------------------------------------------*/ @@ -118,6 +119,7 @@ vcl_get_ctx(unsigned method, int msg) } ctx_cli.ws = &ws_cli; WS_Assert(ctx_cli.ws); + VRTPRIV_init(cli_task_privs); return (&ctx_cli); } @@ -132,7 +134,7 @@ vcl_rel_ctx(struct vrt_ctx **ctx) WS_Reset(&ws_cli, ws_snapshot_cli); INIT_OBJ(*ctx, VRT_CTX_MAGIC); *ctx = NULL; - VRTPRIV_dynamic_kill(NULL, 0); + VRTPRIV_dynamic_kill(cli_task_privs, (uintptr_t)cli_task_privs); } /*--------------------------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index b344756..8e65c8f 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -48,7 +48,7 @@ struct vrt_priv { uintptr_t vmod_id; }; -static struct vmod_priv cli_task_priv; +struct vrt_privs cli_task_privs[1]; /*-------------------------------------------------------------------- */ @@ -105,9 +105,12 @@ vrt_priv_dynamic(VRT_CTX, uintptr_t id, uintptr_t vmod_id) CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); CHECK_OBJ_NOTNULL(ctx->req->sp, SESS_MAGIC); CAST_OBJ_NOTNULL(vps, ctx->req->sp->privs, VRT_PRIVS_MAGIC); - } else { + } else if (ctx->bo) { CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); CAST_OBJ_NOTNULL(vps, ctx->bo->privs, VRT_PRIVS_MAGIC); + } else { + ASSERT_CLI(); + CAST_OBJ_NOTNULL(vps, cli_task_privs, VRT_PRIVS_MAGIC); } VTAILQ_FOREACH(vp, &vps->privs, list) { @@ -130,12 +133,6 @@ VRTPRIV_dynamic_kill(struct vrt_privs *privs, uintptr_t id) { struct vrt_priv *vp, *vp1; - if (privs == NULL && id == 0) { - ASSERT_CLI(); - VRT_priv_fini(&cli_task_priv); - memset(&cli_task_priv, 0, sizeof cli_task_priv); - return; - } CHECK_OBJ_NOTNULL(privs, VRT_PRIVS_MAGIC); AN(id); @@ -163,7 +160,7 @@ VRT_priv_task(VRT_CTX, const void *vmod_id) id = (uintptr_t)ctx->bo; } else { ASSERT_CLI(); - return (&cli_task_priv); + id = (uintptr_t)cli_task_privs; } return (vrt_priv_dynamic(ctx, id, (uintptr_t)vmod_id)); } diff --git a/bin/varnishtest/tests/r02451.vtc b/bin/varnishtest/tests/r02451.vtc new file mode 100644 index 0000000..ef3c45c --- /dev/null +++ b/bin/varnishtest/tests/r02451.vtc @@ -0,0 +1,63 @@ +varnishtest "test PRIV_TASK in vcl_init is per vmod" + +server s1 { + rxreq + txresp -body "ech3Ooj" +} -start + +server s2 { + rxreq + txresp -body "ieQu2qua" +} -start + +server s3 { + rxreq + txresp -body "xiuFi3Pe" +} -start + +varnish v1 -vcl+backend { + import directors; + import debug; + import std; + + sub vcl_init { + new obj = debug.obj(); + new vd = directors.shard(); + debug.test_priv_task("something"); + debug.test_priv_task("to remember"); + + vd.add_backend(s1); + vd.add_backend(s2); + vd.add_backend(s3); + vd.reconfigure(replicas=25); + + std.log("func " + debug.test_priv_task()); + std.log("obj " + obj.test_priv_task()); + } + + sub vcl_recv { + set req.backend_hint = vd.backend(); + return(pass); + } + +} -start + +logexpect l1 -v v1 -g raw -d 1 { + expect 0 0 CLI {^Rd vcl.load} + expect 0 = VCL_Log {^func something to remember} + expect 0 = VCL_Log {^obj something to remember} +} -start + +client c1 { + txreq -url /Boo0aixe + rxresp + expect resp.body == "ech3Ooj" + + txreq -url /eishoSu2 + rxresp + expect resp.body == "ieQu2qua" + + txreq -url /Aunah3uo + rxresp + expect resp.body == "xiuFi3Pe" +} -run From nils.goroll at uplex.de Tue Oct 10 10:59:08 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Tue, 10 Oct 2017 10:59:08 +0000 (UTC) Subject: [master] b60c09b struct vfp_ctx init cleanup Message-ID: <20171010105908.E7296A2837@lists.varnish-cache.org> commit b60c09bfa8cc69e3299cf719ff0f9ffe0b61cc3b Author: Nils Goroll Date: Tue Oct 10 12:49:13 2017 +0200 struct vfp_ctx init cleanup ensure wrk is always set and, for the backend side, move vfp_ctx setup to a single place in vbf_stp_startfetch() Fixes #2453 diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index b7c9947..1bd002e 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -136,8 +136,6 @@ VBO_GetBusyObj(struct worker *wrk, const struct req *req) VRTPRIV_init(bo->privs); - VFP_Setup(bo->vfc); - return (bo); } diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index ca8c89c..c6527e8 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -227,12 +227,8 @@ vbf_stp_mkbereq(struct worker *wrk, struct busyobj *bo) static enum fetch_step vbf_stp_retry(struct worker *wrk, struct busyobj *bo) { - struct vfp_ctx *vfc; - CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC); CHECK_OBJ_NOTNULL(bo, BUSYOBJ_MAGIC); - vfc = bo->vfc; - CHECK_OBJ_NOTNULL(vfc, VFP_CTX_MAGIC); assert(bo->fetch_objcore->boc->state <= BOS_REQ_DONE); @@ -247,9 +243,6 @@ vbf_stp_retry(struct worker *wrk, struct busyobj *bo) bo->do_esi = 0; bo->do_stream = 1; - /* reset fetch processors */ - VFP_Setup(vfc); - // XXX: BereqEnd + BereqAcct ? VSL_ChgId(bo->vsl, "bereq", "retry", VXID_Get(wrk, VSL_BACKENDMARKER)); VSLb_ts_busyobj(bo, "Start", bo->t_prev); @@ -294,6 +287,13 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) assert(bo->fetch_objcore->boc->state <= BOS_REQ_DONE); AZ(bo->htc); + + VFP_Setup(bo->vfc, wrk); + bo->vfc->bo = bo; + bo->vfc->oc = bo->fetch_objcore; + bo->vfc->http = bo->beresp; + bo->vfc->esi_req = bo->bereq; + i = VDI_GetHdr(wrk, bo); now = W_TIM_real(wrk); @@ -384,12 +384,6 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) } } - bo->vfc->bo = bo; - bo->vfc->oc = bo->fetch_objcore; - bo->vfc->wrk = bo->wrk; - bo->vfc->http = bo->beresp; - bo->vfc->esi_req = bo->bereq; - VCL_backend_response_method(bo->vcl, wrk, NULL, bo, NULL); if (wrk->handling == VCL_RET_ABANDON || wrk->handling == VCL_RET_FAIL) { @@ -819,11 +813,11 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) assert(wrk->handling == VCL_RET_DELIVER); - bo->vfc->bo = bo; - bo->vfc->wrk = bo->wrk; - bo->vfc->oc = bo->fetch_objcore; - bo->vfc->http = bo->beresp; - bo->vfc->esi_req = bo->bereq; + assert(bo->vfc->bo == bo); + assert(bo->vfc->wrk == bo->wrk); + assert(bo->vfc->oc == bo->fetch_objcore); + assert(bo->vfc->http == bo->beresp); + assert(bo->vfc->esi_req == bo->bereq); if (vbf_beresp2obj(bo)) { (void)VFP_Error(bo->vfc, "Could not get storage"); diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c index dd36a4a..3426f78 100644 --- a/bin/varnishd/cache/cache_fetch_proc.c +++ b/bin/varnishd/cache/cache_fetch_proc.c @@ -104,11 +104,12 @@ VFP_Extend(const struct vfp_ctx *vc, ssize_t sz) */ void -VFP_Setup(struct vfp_ctx *vc) +VFP_Setup(struct vfp_ctx *vc, struct worker *wrk) { INIT_OBJ(vc, VFP_CTX_MAGIC); VTAILQ_INIT(&vc->vfp); + vc->wrk = wrk; } /********************************************************************** diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 217192b..693ac31 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -92,9 +92,8 @@ cnt_transport(struct worker *wrk, struct req *req) if (req->req_body_status < REQ_BODY_TAKEN) { AN(req->transport->req_body != NULL); - VFP_Setup(req->vfc); + VFP_Setup(req->vfc, wrk); req->vfc->http = req->http; - req->vfc->wrk = wrk; req->transport->req_body(req); } @@ -979,6 +978,7 @@ CNT_Request(struct worker *wrk, struct req *req) AN(req->vsl->wid & VSL_CLIENTMARKER); + /* wrk can have changed for restarts */ req->vfc->wrk = req->wrk = wrk; wrk->vsl = req->vsl; for (nxt = REQ_FSM_MORE; nxt == REQ_FSM_MORE; ) { diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 1dac112..86ea8c1 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -111,7 +111,7 @@ void VFP_Init(void); enum vfp_status VFP_GetStorage(struct vfp_ctx *, ssize_t *sz, uint8_t **ptr); void VFP_Extend(const struct vfp_ctx *, ssize_t sz); struct vfp_entry *VFP_Push(struct vfp_ctx *, const struct vfp *); -void VFP_Setup(struct vfp_ctx *vc); +void VFP_Setup(struct vfp_ctx *vc, struct worker *wrk); int VFP_Open(struct vfp_ctx *bo); void VFP_Close(struct vfp_ctx *bo); diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c index 3304664..6c14193 100644 --- a/bin/varnishd/http1/cache_http1_fetch.c +++ b/bin/varnishd/http1/cache_http1_fetch.c @@ -203,7 +203,7 @@ V1F_FetchRespHdr(struct busyobj *bo) htc->doclose = http_DoConnection(hp); RFC2616_Response_Body(bo->wrk, bo); - bo->vfc->http = bo->beresp; + assert(bo->vfc->http == bo->beresp); if (bo->htc->body_status != BS_NONE && bo->htc->body_status != BS_ERROR) (void)V1F_Setup_Fetch(bo->vfc, bo->htc); From fgsch at lodoss.net Thu Oct 12 06:49:10 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Thu, 12 Oct 2017 06:49:10 +0000 (UTC) Subject: [master] d130e22 Specify that time is in seconds Message-ID: <20171012064910.831E4A3D2C@lists.varnish-cache.org> commit d130e22aa3608318f30c2cf8c9ddf93a4094b5bb Author: Federico G. Schwindt Date: Thu Oct 12 07:46:30 2017 +0100 Specify that time is in seconds Addresses #2456 in a different way. diff --git a/doc/sphinx/reference/varnishncsa.rst b/doc/sphinx/reference/varnishncsa.rst index 07a8b79..9c959d3 100644 --- a/doc/sphinx/reference/varnishncsa.rst +++ b/doc/sphinx/reference/varnishncsa.rst @@ -151,9 +151,10 @@ Supported formatters are: Extended variables. Supported variables are: Varnish:time_firstbyte - Time from when the request processing starts until the first byte - is sent to the client. For backend mode: Time from the request was - sent to the backend to the entire header had been received. + Time from when the request processing starts until the first + byte is sent to the client, in seconds. For backend mode: Time + from the request was sent to the backend to the entire header + had been received. Varnish:hitmiss One of the 'hit' or 'miss' strings, depending on whether the request From nils.goroll at uplex.de Thu Oct 12 20:08:09 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Oct 2017 20:08:09 +0000 (UTC) Subject: [master] 8e9eb0f VFIL_searchpath: Do not call back unless file exists Message-ID: <20171012200809.DD9A1A3EF2@lists.varnish-cache.org> commit 8e9eb0ffe3ddd595fc654d5c03fc6a5e0a0557f9 Author: Nils Goroll Date: Thu Oct 12 21:40:23 2017 +0200 VFIL_searchpath: Do not call back unless file exists diff --git a/lib/libvarnish/vfil.c b/lib/libvarnish/vfil.c index 64f3c56..345096a 100644 --- a/lib/libvarnish/vfil.c +++ b/lib/libvarnish/vfil.c @@ -390,6 +390,8 @@ VFIL_searchpath(const struct vfil_path *vp, vfil_path_func_f *func, void *priv, VSB_clear(vsb); VSB_printf(vsb, "%s/%s", vd->dir, fni); AZ(VSB_finish(vsb)); + if (access(VSB_data(vsb), F_OK)) + continue; i = func(priv, VSB_data(vsb)); if (i <= 0) { e = errno; From nils.goroll at uplex.de Thu Oct 12 20:08:09 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Oct 2017 20:08:09 +0000 (UTC) Subject: [master] f683585 exit VFIL_searchpath upon dlopen error Message-ID: <20171012200809.ECB19A3EF4@lists.varnish-cache.org> commit f683585753ec8ceb2f82eb4fcbf5b51bbbca3b2d Author: Nils Goroll Date: Thu Oct 12 21:58:33 2017 +0200 exit VFIL_searchpath upon dlopen error Together with the previous commit, this fixes potentially misleading error messages. Because we called vcc_path_dlopen on all vmod_path elements irrespective of the kind of dlopen() error, the actual root cause could be hidden. Example: dlerror: .../lib/libvmod_blob/.libs/libvmod_debug.so: cannot open shared object file when the actual error was dlerror: .../lib/libvmod_debug/.libs/libvmod_debug.so: undefined symbol:... diff --git a/lib/libvcc/vcc_vmod.c b/lib/libvcc/vcc_vmod.c index 12d21fd..30a4ae0 100644 --- a/lib/libvcc/vcc_vmod.c +++ b/lib/libvcc/vcc_vmod.c @@ -48,7 +48,7 @@ vcc_path_dlopen(void *priv, const char *fn) hdl = dlopen(fn, RTLD_NOW | RTLD_LOCAL); if (hdl == NULL) - return (1); + return (-1); pp = priv; *pp = hdl; return (0); From nils.goroll at uplex.de Thu Oct 12 22:47:05 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Thu, 12 Oct 2017 22:47:05 +0000 (UTC) Subject: [master] bedaceb Do not possibly underflow rlen Message-ID: <20171012224705.5DFACA2E41@lists.varnish-cache.org> commit bedaceb355a8522f71e3b1b4b963f59e239a9b1f Author: Nils Goroll Date: Fri Oct 13 00:45:24 2017 +0200 Do not possibly underflow rlen for i < 0, rlen could underflow. We are safe because of the check for i < 0 further down, so this change is just a minor cleanup. Fixes #2444 diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index 4bf48ba..e76468d 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -324,7 +324,7 @@ vbp_poke(struct vbp_target *vt) pfd->fd = s; rlen = 0; - do { + while (1) { pfd->events = POLLIN; pfd->revents = 0; tmo = (int)round((t_end - t_now) * 1e3); @@ -341,8 +341,10 @@ vbp_poke(struct vbp_target *vt) sizeof vt->resp_buf - rlen); else i = read(s, buf, sizeof buf); + if (i <= 0) + break; rlen += i; - } while (i > 0); + } VTCP_close(&s); From nils.goroll at uplex.de Fri Oct 13 20:03:07 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Fri, 13 Oct 2017 20:03:07 +0000 (UTC) Subject: [master] 94209b5 move duration conversion to libvarnish for reuse outside vmod_std Message-ID: <20171013200307.CF5F0A329B@lists.varnish-cache.org> commit 94209b5895966a6e2d9f01127f8a455b63d51e70 Author: Nils Goroll Date: Fri Oct 13 14:25:34 2017 +0200 move duration conversion to libvarnish for reuse outside vmod_std diff --git a/include/vnum.h b/include/vnum.h index 8446dae..b245450 100644 --- a/include/vnum.h +++ b/include/vnum.h @@ -31,4 +31,5 @@ /* from libvarnish/vnum.c */ double VNUM(const char *p); double VNUMpfx(const char *p, const char **e); +double VNUM_duration(const char *p); const char *VNUM_2bytes(const char *p, uintmax_t *r, uintmax_t rel); diff --git a/lib/libvarnish/vnum.c b/lib/libvarnish/vnum.c index d7e1343..5e30fc3 100644 --- a/lib/libvarnish/vnum.c +++ b/lib/libvarnish/vnum.c @@ -108,6 +108,61 @@ VNUM(const char *p) /**********************************************************************/ +double +VNUM_duration(const char *p) +{ + const char *t; + double r, sc = 1.0; + + if (p == NULL) + return (nan("")); + + r = VNUMpfx(p, &t); + + if (isnan(r) || t == NULL) + return (nan("")); + + while (isspace(*t)) + t++; + + // keep in sync with vcc_expr.c vcc_TimeUnit() + switch (*t++) { + case 's': + break; + case 'm': + if (*t == 's') { + sc = 1e-3; + t++; + } else + sc = 60.0; + break; + case 'h': + sc = 60.0 * 60.0; + break; + case 'd': + sc = 60.0 * 60.0 * 24.0; + break; + case 'w': + sc = 60.0 * 60.0 * 24.0 * 7.0; + break; + case 'y': + sc = 60.0 * 60.0 * 24.0 * 365.0; + break; + default: + return (nan("")); + } + + while (isspace(*t)) + t++; + + if (*t != '\0') + return (nan("")); + + return (r * sc); +} + +/**********************************************************************/ + const char * VNUM_2bytes(const char *p, uintmax_t *r, uintmax_t rel) { diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index 13b85de..18ac13d 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -48,55 +48,11 @@ VCL_DURATION __match_proto__(td_std_duration) vmod_duration(VRT_CTX, VCL_STRING p, VCL_DURATION d) { - const char *e; - double r; - - CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + double r = VNUM_duration(p); - if (p == NULL) - return (d); + (void) ctx; - while (isspace(*p)) - p++; - - if (*p != '+' && *p != '-' && !isdigit(*p)) - return (d); - - e = NULL; - - r = VNUMpfx(p, &e); - - if (isnan(r) || e == NULL) - return (d); - - while (isspace(*e)) - e++; - - /* NB: Keep this list synchronized with VCC */ - switch (*e++) { - case 's': break; - case 'm': - if (*e == 's') { - r *= 1e-3; - e++; - } else - r *= 60.; - break; - case 'h': r *= 60.*60.; break; - case 'd': r *= 60.*60.*24.; break; - case 'w': r *= 60.*60.*24.*7.; break; - case 'y': r *= 60.*60.*24.*365.; break; - default: - return (d); - } - - while (isspace(*e)) - e++; - - if (*e != '\0') - return (d); - - return (r); + return (isnan(r) ? d : r); } VCL_INT __match_proto__(td_std_integer) From nils.goroll at uplex.de Sun Oct 15 12:32:11 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Sun, 15 Oct 2017 12:32:11 +0000 (UTC) Subject: [master] 17d7c4f When we see a segment overrun, re-check if VSM still valid Message-ID: <20171015123211.36F9DA3FAA@lists.varnish-cache.org> commit 17d7c4f387c5f7dc715d62bd87b770ee6b9319a4 Author: Nils Goroll Date: Sun Oct 15 14:27:18 2017 +0200 When we see a segment overrun, re-check if VSM still valid Trying to tackle #2460 diff --git a/lib/libvarnishapi/vsl_cursor.c b/lib/libvarnishapi/vsl_cursor.c index 0424922..f75e172 100644 --- a/lib/libvarnishapi/vsl_cursor.c +++ b/lib/libvarnishapi/vsl_cursor.c @@ -128,8 +128,12 @@ vslc_vsm_next(const struct VSL_cursor *cursor) while (1) { i = vslc_vsm_check(&c->cursor, &c->next); - if (i <= 0) - return (-3); /* Overrun */ + if (i <= 0) { + if (VSM_StillValid(c->vsm, &c->vf) != VSM_valid) + return (-2); /* VSL abandoned */ + else + return (-3); /* Overrun */ + } t = *(volatile const uint32_t *)c->next.ptr; AN(t); From daghf at varnish-software.com Wed Oct 18 11:08:11 2017 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Wed, 18 Oct 2017 11:08:11 +0000 (UTC) Subject: [master] 9e5dfac whitespace Message-ID: <20171018110811.1E195A31FB@lists.varnish-cache.org> commit 9e5dfacd072302e10ed3b7462226b679074431b0 Author: Dag Haavi Finstad Date: Wed Oct 18 13:07:45 2017 +0200 whitespace diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index 705aaa7..d7d1dce 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -838,7 +838,6 @@ h2_procframe(struct worker *wrk, struct h2_sess *h2, !(r2 && h2->new_req == r2->req && h2f == H2_F_CONTINUATION)) return (H2CE_PROTOCOL_ERROR); // rfc7540,l,1859,1863 - h2e = h2f->rxfunc(wrk, h2, r2); if (h2e == 0) return (0); From phk at FreeBSD.org Sat Oct 21 22:07:11 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 21 Oct 2017 22:07:11 +0000 (UTC) Subject: [master] 82ae65f Move the sigsegv stuff to its own function Message-ID: <20171021220711.80EABA3815@lists.varnish-cache.org> commit 82ae65f8714c52364a630d3f1a909d00aba69fcf Author: Poul-Henning Kamp Date: Sat Oct 21 21:23:25 2017 +0000 Move the sigsegv stuff to its own function diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index 1c1c41b..a8e148d 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -293,6 +293,39 @@ child_signal_handler(int s, siginfo_t *si, void *c) */ static void +mgt_child_sigmagic(void) +{ + struct sigaction sa; + + memset(&sa, 0, sizeof sa); + sa.sa_sigaction = child_signal_handler; + sa.sa_flags = SA_SIGINFO; + (void)sigaction(SIGBUS, &sa, NULL); + (void)sigaction(SIGABRT, &sa, NULL); + +#ifdef HAVE_SIGALTSTACK + size_t sz = SIGSTKSZ + 4096; + if (sz < mgt_param.wthread_stacksize) + sz = mgt_param.wthread_stacksize; + altstack.ss_sp = mmap(NULL, sz, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + AN(altstack.ss_sp != MAP_FAILED); + AN(altstack.ss_sp); + altstack.ss_size = sz; + altstack.ss_flags = 0; + AZ(sigaltstack(&altstack, NULL)); + sa.sa_flags |= SA_ONSTACK; +#endif + (void)sigaction(SIGSEGV, &sa, NULL); +} + + +/*===================================================================== + * Launch the child process + */ + +static void mgt_launch_child(struct cli *cli) { pid_t pid; @@ -300,7 +333,6 @@ mgt_launch_child(struct cli *cli) char *p; struct vev *e; int i, cp[2]; - struct sigaction sa; if (child_state != CH_STOPPED && child_state != CH_DIED) return; @@ -367,30 +399,8 @@ mgt_launch_child(struct cli *cli) heritage.cls = mgt_cls; heritage.ident = VSB_data(vident) + 1; - if (mgt_param.sigsegv_handler) { - memset(&sa, 0, sizeof sa); - sa.sa_sigaction = child_signal_handler; - sa.sa_flags = SA_SIGINFO; - (void)sigaction(SIGBUS, &sa, NULL); - (void)sigaction(SIGABRT, &sa, NULL); - -#ifdef HAVE_SIGALTSTACK - size_t sz = SIGSTKSZ + 4096; - if (sz < mgt_param.wthread_stacksize) - sz = mgt_param.wthread_stacksize; - altstack.ss_sp = mmap(NULL, sz, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, 0); - AN(altstack.ss_sp != MAP_FAILED); - AN(altstack.ss_sp); - altstack.ss_size = sz; - altstack.ss_flags = 0; - AZ(sigaltstack(&altstack, NULL)); - sa.sa_flags |= SA_ONSTACK; -#endif - (void)sigaction(SIGSEGV, &sa, NULL); - - } + if (mgt_param.sigsegv_handler) + mgt_child_sigmagic(); (void)signal(SIGINT, SIG_DFL); (void)signal(SIGTERM, SIG_DFL); From phk at FreeBSD.org Sat Oct 21 22:07:11 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 21 Oct 2017 22:07:11 +0000 (UTC) Subject: [master] 293edb1 Move the worker process stack magic to child_main.c where it belongs. Message-ID: <20171021220711.97333A3817@lists.varnish-cache.org> commit 293edb1e4a25e576f4d0cdac8bebb022a9c90997 Author: Poul-Henning Kamp Date: Sat Oct 21 22:06:12 2017 +0000 Move the worker process stack magic to child_main.c where it belongs. diff --git a/bin/varnishd/cache/cache_main.c b/bin/varnishd/cache/cache_main.c index 5622bf6..cfaec23 100644 --- a/bin/varnishd/cache/cache_main.c +++ b/bin/varnishd/cache/cache_main.c @@ -33,6 +33,9 @@ #include #include +#ifdef HAVE_SIGALTSTACK +# include +#endif #ifdef HAVE_PTHREAD_NP_H # include @@ -121,7 +124,7 @@ THR_GetName(void) */ #ifdef HAVE_SIGALTSTACK #include -extern stack_t altstack; +static stack_t altstack; #endif void @@ -222,10 +225,81 @@ child_malloc_fail(void *p, const char *s) } #endif +/*===================================================================== + * signal handler for child process + */ + +static void __match_proto__() +child_signal_handler(int s, siginfo_t *si, void *c) +{ + char buf[1024]; + struct sigaction sa; + + (void)c; + + /* Don't come back */ + memset(&sa, 0, sizeof sa); + sa.sa_handler = SIG_DFL; + (void)sigaction(SIGSEGV, &sa, NULL); + (void)sigaction(SIGABRT, &sa, NULL); + + bprintf(buf, "Signal %d (%s) received at %p si_code %d", + s, strsignal(s), si->si_addr, si->si_code); + VAS_Fail(__func__, + __FILE__, + __LINE__, + buf, + VAS_WRONG); +} + +/*===================================================================== + * Magic for panicing properly on signals + */ + +static void +child_sigmagic(size_t altstksz) +{ + struct sigaction sa; + + memset(&sa, 0, sizeof sa); + +#ifdef HAVE_SIGALTSTACK + size_t sz = SIGSTKSZ + 4096; + if (sz < altstksz) + sz = altstksz; + altstack.ss_sp = mmap(NULL, sz, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + AN(altstack.ss_sp != MAP_FAILED); + AN(altstack.ss_sp); + altstack.ss_size = sz; + altstack.ss_flags = 0; + sa.sa_flags |= SA_ONSTACK; +#endif + + THR_Init(); + + sa.sa_sigaction = child_signal_handler; + sa.sa_flags |= SA_SIGINFO; + (void)sigaction(SIGBUS, &sa, NULL); + (void)sigaction(SIGABRT, &sa, NULL); + (void)sigaction(SIGSEGV, &sa, NULL); +} + + +/*===================================================================== + * Run the child process + */ + void -child_main(void) +child_main(int sigmagic, size_t altstksz) { + if (sigmagic) + child_sigmagic(altstksz); + (void)signal(SIGINT, SIG_DFL); + (void)signal(SIGTERM, SIG_DFL); + setbuf(stdout, NULL); setbuf(stderr, NULL); printf("Child starts\n"); diff --git a/bin/varnishd/common/heritage.h b/bin/varnishd/common/heritage.h index ebc30fe..9e26079 100644 --- a/bin/varnishd/common/heritage.h +++ b/bin/varnishd/common/heritage.h @@ -109,7 +109,7 @@ void MCH_Fd_Inherit(int fd, const char *what); } while (0) /* cache/cache_main.c */ -void child_main(void); +void child_main(int, size_t); /* cache/cache_vcl.c */ int VCL_TestLoad(const char *); diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index a8e148d..a6f944c 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -85,11 +85,6 @@ static struct vlu *child_std_vlu; static struct vsb *child_panic = NULL; -#ifdef HAVE_SIGALTSTACK -#include -stack_t altstack; -#endif - static void mgt_reap_child(void); /*===================================================================== @@ -262,66 +257,6 @@ child_poker(const struct vev *e, int what) } /*===================================================================== - * signal handler for child process - */ - -static void __match_proto__() -child_signal_handler(int s, siginfo_t *si, void *c) -{ - char buf[1024]; - struct sigaction sa; - - (void)c; - - /* Don't come back */ - memset(&sa, 0, sizeof sa); - sa.sa_handler = SIG_DFL; - (void)sigaction(SIGSEGV, &sa, NULL); - (void)sigaction(SIGABRT, &sa, NULL); - - bprintf(buf, "Signal %d (%s) received at %p si_code %d", - s, strsignal(s), si->si_addr, si->si_code); - VAS_Fail(__func__, - __FILE__, - __LINE__, - buf, - VAS_WRONG); -} - -/*===================================================================== - * Launch the child process - */ - -static void -mgt_child_sigmagic(void) -{ - struct sigaction sa; - - memset(&sa, 0, sizeof sa); - sa.sa_sigaction = child_signal_handler; - sa.sa_flags = SA_SIGINFO; - (void)sigaction(SIGBUS, &sa, NULL); - (void)sigaction(SIGABRT, &sa, NULL); - -#ifdef HAVE_SIGALTSTACK - size_t sz = SIGSTKSZ + 4096; - if (sz < mgt_param.wthread_stacksize) - sz = mgt_param.wthread_stacksize; - altstack.ss_sp = mmap(NULL, sz, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, - -1, 0); - AN(altstack.ss_sp != MAP_FAILED); - AN(altstack.ss_sp); - altstack.ss_size = sz; - altstack.ss_flags = 0; - AZ(sigaltstack(&altstack, NULL)); - sa.sa_flags |= SA_ONSTACK; -#endif - (void)sigaction(SIGSEGV, &sa, NULL); -} - - -/*===================================================================== * Launch the child process */ @@ -399,17 +334,17 @@ mgt_launch_child(struct cli *cli) heritage.cls = mgt_cls; heritage.ident = VSB_data(vident) + 1; - if (mgt_param.sigsegv_handler) - mgt_child_sigmagic(); - (void)signal(SIGINT, SIG_DFL); - (void)signal(SIGTERM, SIG_DFL); - VJ_subproc(JAIL_SUBPROC_WORKER); heritage.proc_vsmw = VSMW_New(heritage.vsm_fd, 0640, "_.index"); AN(heritage.proc_vsmw); - child_main(); + /* + * We pass these two params because child_main needs them + * Well before it has found its own param struct. + */ + child_main(mgt_param.sigsegv_handler, + mgt_param.wthread_stacksize); exit(0); } From phk at FreeBSD.org Sat Oct 21 22:11:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sat, 21 Oct 2017 22:11:05 +0000 (UTC) Subject: [master] d6b8b98 Move extern to common #include Message-ID: <20171021221105.10F2BA3BD6@lists.varnish-cache.org> commit d6b8b982794daa28b04bca5900ba973da7dd2e7e Author: Poul-Henning Kamp Date: Sat Oct 21 22:10:30 2017 +0000 Move extern to common #include diff --git a/bin/varnishd/cache/cache_varnishd.h b/bin/varnishd/cache/cache_varnishd.h index 86ea8c1..8402829 100644 --- a/bin/varnishd/cache/cache_varnishd.h +++ b/bin/varnishd/cache/cache_varnishd.h @@ -219,6 +219,9 @@ void VRTPRIV_init(struct vrt_privs *privs); void VRTPRIV_dynamic_kill(struct vrt_privs *privs, uintptr_t id); void pan_privs(struct vsb *, const struct vrt_privs *); +/* cache_vrt_priv.c */ +extern struct vrt_privs cli_task_privs[1]; + /* cache_vrt_vmod.c */ void VMOD_Init(void); void VMOD_Panic(struct vsb *); diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 7792ab1..547976e 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -101,7 +101,6 @@ static struct vrt_ctx ctx_cli; static unsigned handling_cli; static struct ws ws_cli; static uintptr_t ws_snapshot_cli; -extern struct vrt_privs cli_task_privs[1]; /*--------------------------------------------------------------------*/ From fgsch at lodoss.net Sun Oct 22 07:30:12 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 22 Oct 2017 07:30:12 +0000 (UTC) Subject: [master] f3b7c2a Update after previous commit Message-ID: <20171022073012.BC80CA462E@lists.varnish-cache.org> commit f3b7c2a7bcca329567a0dae8bb282bd89de8bb8b Author: Federico G. Schwindt Date: Sat Oct 21 20:13:30 2017 -0700 Update after previous commit diff --git a/tools/ubsan.suppr b/tools/ubsan.suppr index 5b86d1f..e69de29 100644 --- a/tools/ubsan.suppr +++ b/tools/ubsan.suppr @@ -1,2 +0,0 @@ -# https://github.com/madler/zlib/issues/290 -nonnull-attribute:trees.c From fgsch at lodoss.net Sun Oct 22 07:30:12 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 22 Oct 2017 07:30:12 +0000 (UTC) Subject: [master] 2d6dc0c From upstream: Message-ID: <20171022073012.9A947A462C@lists.varnish-cache.org> commit 2d6dc0c5c67e23af8fe9a30689871dabbb6f57ef Author: Federico G. Schwindt Date: Sat Oct 21 20:12:08 2017 -0700 From upstream: Avoid an undefined behavior of memcpy() in _tr_stored_block(). Allegedly the behavior of memcpy() is undefined if the source pointer is NULL, even if the number of bytes to copy is zero. diff --git a/lib/libvgz/trees.c b/lib/libvgz/trees.c index 10464f6..92a50f2 100644 --- a/lib/libvgz/trees.c +++ b/lib/libvgz/trees.c @@ -874,7 +874,8 @@ void ZLIB_INTERNAL _tr_stored_block(s, buf, stored_len, last) bi_windup(s); /* align on byte boundary */ put_short(s, (ush)stored_len); put_short(s, (ush)~stored_len); - zmemcpy(s->pending_buf + s->pending, (Bytef *)buf, stored_len); + if (stored_len) + zmemcpy(s->pending_buf + s->pending, (Bytef *)buf, stored_len); s->pending += stored_len; #ifdef ZLIB_DEBUG s->compressed_len = (s->compressed_len + 3 + 7) & (ulg)~7L; From fgsch at lodoss.net Sun Oct 22 07:30:12 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Sun, 22 Oct 2017 07:30:12 +0000 (UTC) Subject: [master] 04288d7 Build with clang 5.0 Message-ID: <20171022073012.DB7F4A4631@lists.varnish-cache.org> commit 04288d709e6fd55108286c9dcc77315f788edb01 Author: Federico G. Schwindt Date: Sun Oct 22 00:28:34 2017 -0700 Build with clang 5.0 diff --git a/.travis.yml b/.travis.yml index a3b1cb5..a32b5d5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,13 +13,13 @@ matrix: - os: linux dist: trusty compiler: clang - env: CLANG=4.0 + env: CLANG=5.0 - os: osx osx_image: xcode9 compiler: clang allow_failures: - os: osx - - env: CLANG=4.0 + - env: CLANG=5.0 addons: apt: packages: From phk at FreeBSD.org Sun Oct 22 12:24:09 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 22 Oct 2017 12:24:09 +0000 (UTC) Subject: [master] 806a68b Minor polish for PCLint-Plus Message-ID: <20171022122409.D9B06A447C@lists.varnish-cache.org> commit 806a68b3ff588963b2b97556de556305e0154ad3 Author: Poul-Henning Kamp Date: Sun Oct 22 12:21:24 2017 +0000 Minor polish for PCLint-Plus diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c index efb3dc0..5a30950 100644 --- a/bin/varnishd/cache/cache_backend_cfg.c +++ b/bin/varnishd/cache/cache_backend_cfg.c @@ -50,7 +50,7 @@ #include "VSC_vbe.h" -const char *vbe_proto_ident = "HTTP Backend"; +static const char *vbe_proto_ident = "HTTP Backend"; static VTAILQ_HEAD(, backend) backends = VTAILQ_HEAD_INITIALIZER(backends); static VTAILQ_HEAD(, backend) cool_backends = diff --git a/bin/varnishd/common/common_vsc.c b/bin/varnishd/common/common_vsc.c index 95952ed..1ce53cd 100644 --- a/bin/varnishd/common/common_vsc.c +++ b/bin/varnishd/common/common_vsc.c @@ -118,15 +118,14 @@ VRT_VSC_Destroy(const char *nm, const void *p) vsc_lock(); AN(heritage.proc_vsmw); - VTAILQ_FOREACH(vsg, &vsc_seglist, list) { - if (vsg->ptr != p) - continue; - assert(vsg->nm == nm); - VSMW_Free(heritage.proc_vsmw, &vsg->seg); - VTAILQ_REMOVE(&vsc_seglist, vsg, list); - FREE_OBJ(vsg); - break; - } + VTAILQ_FOREACH(vsg, &vsc_seglist, list) + if (vsg->ptr == p) + break; + AN(vsg); + assert(vsg->nm == nm); + VSMW_Free(heritage.proc_vsmw, &vsg->seg); + VTAILQ_REMOVE(&vsc_seglist, vsg, list); + FREE_OBJ(vsg); if (vsc_unlock != NULL) vsc_unlock(); } diff --git a/bin/varnishd/mgt/mgt_child.c b/bin/varnishd/mgt/mgt_child.c index a6f944c..3174cde 100644 --- a/bin/varnishd/mgt/mgt_child.c +++ b/bin/varnishd/mgt/mgt_child.c @@ -470,7 +470,7 @@ mgt_reap_child(void) r = waitpid(child_pid, &status, 0); } if (r != child_pid) - fprintf(stderr, "WAIT 0x%jx\n", (uintmax_t)r); + fprintf(stderr, "WAIT 0x%jd\n", (intmax_t)r); assert(r == child_pid); MAC_reopen_sockets(); From phk at FreeBSD.org Sun Oct 22 12:24:09 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 22 Oct 2017 12:24:09 +0000 (UTC) Subject: [master] f8ec37d Minor polish for PCLint-Plus Message-ID: <20171022122409.EE409A447E@lists.varnish-cache.org> commit f8ec37d50e98d1faf14982b0a29ccef318f27824 Author: Poul-Henning Kamp Date: Sun Oct 22 12:22:00 2017 +0000 Minor polish for PCLint-Plus diff --git a/lib/libvmod_blob/base64.c b/lib/libvmod_blob/base64.c index a46d88c..f7a8c06 100644 --- a/lib/libvmod_blob/base64.c +++ b/lib/libvmod_blob/base64.c @@ -57,7 +57,7 @@ base64_decode_l(size_t l) } static inline int -decode(char *restrict *restrict dest, char *restrict const buf, +decode(char *restrict *restrict dest, const char *restrict const buf, const size_t buflen, unsigned u, const int n) { char *d; @@ -184,7 +184,7 @@ base64_decode(const enum encoding dec, char *restrict const buf, } if (n) { if (!alpha->padding) - u <<= 6 * (4 - n); + u <<= (6 * (4 - n)); if (decode(&dest, buf, buflen, u, n-term) < 0) return -1; } diff --git a/lib/libvmod_blob/wb.h b/lib/libvmod_blob/wb.h index 7bd1cbf..ce11ba1 100644 --- a/lib/libvmod_blob/wb.h +++ b/lib/libvmod_blob/wb.h @@ -34,26 +34,26 @@ struct wb_s { /* return one byte less for the final zero byte */ static inline const char* -wb_end(struct wb_s *wb) { +wb_end(const struct wb_s *wb) { return wb->ws->r - 1; } /* return the write position */ static inline char* -wb_buf(struct wb_s *wb) { +wb_buf(const struct wb_s *wb) { return wb->w; } /* return one byte less for the final zero byte */ static inline ssize_t -wb_space(struct wb_s *wb) { +wb_space(const struct wb_s *wb) { ssize_t f = wb->ws->r - wb->w; assert(f > 0); return f - 1; } static inline ssize_t -wb_len(struct wb_s *wb) { +wb_len(const struct wb_s *wb) { ssize_t l = wb->w - wb->ws->f; assert(l >= 0); return l; From phk at FreeBSD.org Sun Oct 22 12:24:10 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 22 Oct 2017 12:24:10 +0000 (UTC) Subject: [master] 23ce17a Whitespace OCD Message-ID: <20171022122410.1931BA4481@lists.varnish-cache.org> commit 23ce17a3abe733ef70b4b955e0042220f0fb2b74 Author: Poul-Henning Kamp Date: Sun Oct 22 12:22:54 2017 +0000 Whitespace OCD diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py index 829c6e5..c0507ea 100644 --- a/lib/libvcc/vsctool.py +++ b/lib/libvcc/vsctool.py @@ -60,11 +60,11 @@ PARAMS = { # http://python3porting.com/problems.html#bytes-strings-and-unicode if sys.version_info < (3,): def b(x): - return x + return x else: import codecs def b(x): - return codecs.latin_1_encode(x)[0] + return codecs.latin_1_encode(x)[0] def gzip_str(s): try: From phk at FreeBSD.org Sun Oct 22 22:51:10 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 22 Oct 2017 22:51:10 +0000 (UTC) Subject: [master] 9f26fcd Rewrite and simplify VSL_Glob2Tags(). Message-ID: <20171022225110.B3854A4AEF@lists.varnish-cache.org> commit 9f26fcdd41b2f525d78dd660ff713b5e3655fe0d Author: Poul-Henning Kamp Date: Sun Oct 22 22:46:37 2017 +0000 Rewrite and simplify VSL_Glob2Tags(). Two static checkers and myself had problems understanding the old code. diff --git a/bin/varnishtest/tests/u00006.vtc b/bin/varnishtest/tests/u00006.vtc index 2d16bca..f678bd1 100644 --- a/bin/varnishtest/tests/u00006.vtc +++ b/bin/varnishtest/tests/u00006.vtc @@ -31,10 +31,16 @@ shell -err -expect "-k: Invalid number 'foo'" \ "varnishlog -k foo" shell -err -expect "-L: Range error" \ "varnishlog -L 0" + shell -err -expect {-i: "foo" matches zero tags} \ "varnishlog -i foo" shell -err -expect {-i: "Resp" is ambiguous} \ "varnishlog -i Resp" +shell -err -expect {-i: Syntax error in "Re**sp"} \ + "varnishlog -i Re**sp" +shell -err -expect {-i: "" matches zero tags} \ + "varnishlog -i ''" + shell -err -expect {-I: "foo" matches zero tags} \ "varnishlog -I foo:bar" shell -err -expect {-I: "Resp" is ambiguous} \ diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c index 829dbc3..6642cab 100644 --- a/lib/libvarnishapi/vsl_arg.c +++ b/lib/libvarnishapi/vsl_arg.c @@ -83,40 +83,25 @@ VSL_Name2Tag(const char *name, int l) int VSL_Glob2Tags(const char *glob, int l, VSL_tagfind_f *func, void *priv) { - int i, r, l2; - int pre = 0; - int post = 0; - char buf[64]; + const char *p1 = NULL; + const char *p2 = NULL; + const char *e, *p; + int i, l1 = 0, l2 = 0, r = 0; AN(glob); - if (l < 0) - l = strlen(glob); - if (l == 0 || l > sizeof buf - 1) - return (-1); - if (strchr(glob, '*') != NULL) { - if (glob[0] == '*') { - /* Prefix wildcard */ - pre = 1; - glob++; - l--; - } - if (l > 0 && glob[l - 1] == '*') { - /* Postfix wildcard */ - post = 1; - l--; - } - } - if (pre && post) - /* Support only post or prefix wildcards */ - return (-3); - memcpy(buf, glob, l); - buf[l] = '\0'; - if (strchr(buf, '*') != NULL) - /* No multiple wildcards */ - return (-3); - if (pre == 0 && post == 0) { - /* No wildcards, use VSL_Name2Tag */ - i = VSL_Name2Tag(buf, l); + if (l >= 0) + e = glob + l; + else + e = strchr(glob, '\0'); + if (glob == e) + return (-1); // Empty pattern cannot match + + for(p = glob; p < e; p++) + if (*p == '*') + break; + + if (p == e) { // No wildcard + i = VSL_Name2Tag(glob, l); if (i < 0) return (i); if (func != NULL) @@ -124,22 +109,31 @@ VSL_Glob2Tags(const char *glob, int l, VSL_tagfind_f *func, void *priv) return (1); } - r = 0; + if (p != glob) { // Prefix match + p1 = glob; + l1 = p - p1; + } + + if (p != e - 1) { // Postfix match + p2 = p + 1; + l2 = e - p2; + } + + for(p++; p < e; p++) + if (*p == '*') + return (-3); // More than one wildcard + for (i = 0; i < SLT__MAX; i++) { - if (VSL_tags[i] == NULL) + p = VSL_tags[i]; + if (p == NULL) continue; - l2 = strlen(VSL_tags[i]); - if (l2 < l) + e = strchr(p, '\0'); + if (e - p < l1 + l2) + continue; + if (p1 != NULL && strncasecmp(p, p1, l1)) + continue; + if (p2 != NULL && strncasecmp(e - l2, p2, l2)) continue; - if (pre) { - /* Prefix wildcard match */ - if (strcasecmp(buf, VSL_tags[i] + l2 - l)) - continue; - } else { - /* Postfix wildcard match */ - if (strncasecmp(buf, VSL_tags[i], l)) - continue; - } if (func != NULL) (func)(i, priv); r++; From phk at FreeBSD.org Sun Oct 22 23:09:04 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Sun, 22 Oct 2017 23:09:04 +0000 (UTC) Subject: [master] 9b286d3 Try to cope with SunOS shell issue Message-ID: <20171022230905.064E3A413C@lists.varnish-cache.org> commit 9b286d3468d691f1339f47af8c15af0b3089362d Author: Poul-Henning Kamp Date: Sun Oct 22 23:08:40 2017 +0000 Try to cope with SunOS shell issue diff --git a/bin/varnishtest/tests/u00006.vtc b/bin/varnishtest/tests/u00006.vtc index f678bd1..24f69ca 100644 --- a/bin/varnishtest/tests/u00006.vtc +++ b/bin/varnishtest/tests/u00006.vtc @@ -39,7 +39,7 @@ shell -err -expect {-i: "Resp" is ambiguous} \ shell -err -expect {-i: Syntax error in "Re**sp"} \ "varnishlog -i Re**sp" shell -err -expect {-i: "" matches zero tags} \ - "varnishlog -i ''" + {varnishlog -i ""} shell -err -expect {-I: "foo" matches zero tags} \ "varnishlog -I foo:bar" From fgsch at lodoss.net Mon Oct 23 03:36:06 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 23 Oct 2017 03:36:06 +0000 (UTC) Subject: [master] d5fbaaf Update sanitizer opts to deal with recent changes Message-ID: <20171023033606.2FDB3A4580@lists.varnish-cache.org> commit d5fbaaf4232a15783c9453e41b223326d5fbf2e5 Author: Federico G. Schwindt Date: Sun Oct 22 20:23:03 2017 -0700 Update sanitizer opts to deal with recent changes diff --git a/.travis.yml b/.travis.yml index a32b5d5..5bb8d55 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,9 +41,9 @@ before_install: sudo apt-get install -y clang-$CLANG llvm-$CLANG; export CC=clang-$CLANG; export CONFIGURE_ARGS="--enable-developer-warnings --enable-debugging-symbols --disable-stack-protector --enable-asan --enable-ubsan"; - export ASAN_OPTIONS=abort_on_error=1,detect_odr_violation=1,detect_leaks=1,detect_stack_use_after_return=1,detect_invalid_pointer_pairs=1,handle_segv=0,handle_sigbus=0; - export LSAN_OPTIONS=suppressions=$(pwd)/tools/lsan.suppr; - export UBSAN_OPTIONS=halt_on_error=1,print_stacktrace=1,suppressions=$(pwd)/tools/ubsan.suppr; + export ASAN_OPTIONS=abort_on_error=1,detect_odr_violation=1,detect_leaks=1,detect_stack_use_after_return=1,detect_invalid_pointer_pairs=1,handle_segv=0,handle_sigbus=0,use_sigaltstack=0; + export LSAN_OPTIONS=abort_on_error=1,use_sigaltstack=0,suppressions=$(pwd)/tools/lsan.suppr; + export UBSAN_OPTIONS=halt_on_error=1,print_stacktrace=1,use_sigaltstack=0,suppressions=$(pwd)/tools/ubsan.suppr; fi - ./autogen.sh - ./configure ${CONFIGURE_ARGS} From fgsch at lodoss.net Mon Oct 23 04:20:07 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Mon, 23 Oct 2017 04:20:07 +0000 (UTC) Subject: [master] c6b151d Fix sanitizer builds with gcc Message-ID: <20171023042007.1FB70A44B7@lists.varnish-cache.org> commit c6b151dfa0d86974c1d657817e38922cf26f4c82 Author: Federico G. Schwindt Date: Sun Oct 22 21:05:40 2017 -0700 Fix sanitizer builds with gcc diff --git a/bin/varnishd/hpack/vhp_gen_hufdec.c b/bin/varnishd/hpack/vhp_gen_hufdec.c index 5f86440..bc62567 100644 --- a/bin/varnishd/hpack/vhp_gen_hufdec.c +++ b/bin/varnishd/hpack/vhp_gen_hufdec.c @@ -252,3 +252,11 @@ main(int argc, const char **argv) return (0); } + +#if defined(__SANITIZER) +int __lsan_is_turned_off(void); +int __lsan_is_turned_off() +{ + return (1); +} +#endif diff --git a/bin/varnishd/mgt/mgt_param.c b/bin/varnishd/mgt/mgt_param.c index 25b06df..9cb4900 100644 --- a/bin/varnishd/mgt/mgt_param.c +++ b/bin/varnishd/mgt/mgt_param.c @@ -510,7 +510,7 @@ MCF_InitParams(struct cli *cli) low = sysconf(_SC_THREAD_STACK_MIN); MCF_ParamConf(MCF_MINIMUM, "thread_pool_stack", "%jdb", (intmax_t)low); -#if defined(WITH_SANITIZERS) +#if defined(__SANITIZER) def = 92 * 1024; #else def = 48 * 1024; diff --git a/configure.ac b/configure.ac index a6081a5..9d72283 100644 --- a/configure.ac +++ b/configure.ac @@ -267,8 +267,13 @@ AC_ARG_ENABLE(msan, MSAN_FLAGS="-fsanitize=memory") if test "x$UBSAN_FLAGS$TSAN_FLAGS$ASAN_FLAGS$MSAN_FLAGS" != "x"; then - SAN_CFLAGS="-DWITH_SANITIZERS=1 ${UBSAN_FLAGS} ${TSAN_FLAGS} ${ASAN_FLAGS} ${MSAN_FLAGS} -fPIC -fPIE -fno-omit-frame-pointer" + SAN_CFLAGS="-D__SANITIZER=1 ${UBSAN_FLAGS} ${TSAN_FLAGS} ${ASAN_FLAGS} ${MSAN_FLAGS} -fPIC -fPIE -fno-omit-frame-pointer" SAN_LDFLAGS="${UBSAN_FLAGS} ${TSAN_FLAGS} ${ASAN_FLAGS} ${MSAN_FLAGS} -pie" + case $CC in + gcc*) + SAN_CFLAGS="${SAN_CFLAGS} -fuse-ld=gold" + ;; + esac fi AC_SUBST(SAN_CFLAGS) AC_SUBST(SAN_LDFLAGS) From phk at FreeBSD.org Mon Oct 23 07:05:08 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 23 Oct 2017 07:05:08 +0000 (UTC) Subject: [master] 4135edf Polish the vsl_arg stuff a bit more. Message-ID: <20171023070508.2C4F5A54C4@lists.varnish-cache.org> commit 4135edfe992322419310e9883d335908bd2fd800 Author: Poul-Henning Kamp Date: Mon Oct 23 07:04:25 2017 +0000 Polish the vsl_arg stuff a bit more. diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am index 5ad55d2..41339ed 100644 --- a/lib/libvarnishapi/Makefile.am +++ b/lib/libvarnishapi/Makefile.am @@ -100,17 +100,18 @@ vxp_test_SOURCES = \ $(libvarnishapi_la_SOURCES) \ vxp_test.c -vsl_glob_test_SOURCES = \ - vsl_glob_test.c vsl_glob_test_LDADD = @PCRE_LIBS@ ${RT_LIBS} ${LIBM} libvarnishapi.la vsl_glob_test_CFLAGS = -I$(top_srcdir)/include -TESTS = vjsn_test +TESTS = vjsn_test vsl_glob_test noinst_PROGRAMS += ${TESTS} +vsl_glob_test_SOURCES = vsl_glob_test.c +vsl_glob_test_LDADD = libvarnishapi.la @SAN_LDFLAGS@ + vjsn_test_SOURCES = vjsn.c vjsn_test_CFLAGS = -DVJSN_TEST @SAN_CFLAGS@ vjsn_test_LDADD = libvarnishapi.la @SAN_LDFLAGS@ diff --git a/lib/libvarnishapi/vsl_arg.c b/lib/libvarnishapi/vsl_arg.c index 6642cab..cd7c2b4 100644 --- a/lib/libvarnishapi/vsl_arg.c +++ b/lib/libvarnishapi/vsl_arg.c @@ -146,27 +146,26 @@ VSL_Glob2Tags(const char *glob, int l, VSL_tagfind_f *func, void *priv) int VSL_List2Tags(const char *list, int l, VSL_tagfind_f *func, void *priv) { - const char *p, *q, *e; - int r, t; + const char *p, *b, *e; + int r, t = 0; - if (l < 0) - l = strlen(list); p = list; - e = p + l; - t = 0; + if (l >= 0) + e = p + l; + else + e = strchr(p, '\0'); while (p < e) { while (p < e && *p == ',') p++; if (p == e) break; - q = p; - while (q < e && *q != ',') - q++; - r = VSL_Glob2Tags(p, q - p, func, priv); + b = p; + while (p < e && *p != ',') + p++; + r = VSL_Glob2Tags(b, p - b, func, priv); if (r < 0) return (r); t += r; - p = q; } if (t == 0) return (-1); diff --git a/lib/libvarnishapi/vsl_glob_test.c b/lib/libvarnishapi/vsl_glob_test.c index 3ad2ec8..e4f62a9 100644 --- a/lib/libvarnishapi/vsl_glob_test.c +++ b/lib/libvarnishapi/vsl_glob_test.c @@ -30,26 +30,55 @@ #ifndef __FLEXELINT__ +#include #include #include #include #include #include "vapi/vsl.h" +#include "vdef.h" +#include "vas.h" static void cb(int tag, void *priv) { - (void)priv; - printf("\t%d (%s)\n", tag, VSL_tags[tag]); + if (priv != NULL) + assert(!fnmatch(priv, VSL_tags[tag], FNM_CASEFOLD)); +} + +static int +tst_one(const char *p) +{ + int i; + + printf("Test <%s>\n", p); + i = VSL_Glob2Tags(p, -1, cb, TRUST_ME(p)); + printf(" -> %d\n", i); + return (i); } int main(int argc, char * const *argv) { - int i; + int i, j; + if (argc == 1) { + i = tst_one("Req*"); + assert(i == 10); + j = tst_one("reQ*"); + assert(i == j); + assert(tst_one("*Header") > 0); + assert(tst_one("Req*eader") == 1); + assert(tst_one("*") > 0); + assert(tst_one("a*b*c") == -3); + assert(tst_one("**") == -3); + assert(tst_one("_") == -1); + assert(tst_one("") == -1); + assert(VSL_Glob2Tags("", 0, cb, NULL) == -1); + return (0); + } if (argc != 2) { fprintf(stderr, "vsl_glob_test \n"); exit(1); From phk at FreeBSD.org Mon Oct 23 10:06:08 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 23 Oct 2017 10:06:08 +0000 (UTC) Subject: [master] b692155 Make cache.h include vrt.h and forbid including both. Message-ID: <20171023100608.616C0A5576@lists.varnish-cache.org> commit b6921558b9e5ecb431524f9d5dc9aea5af7bd24b Author: Poul-Henning Kamp Date: Mon Oct 23 10:04:41 2017 +0000 Make cache.h include vrt.h and forbid including both. This is part of the $ABI reorg. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index e67f89f..2f5a379 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -28,6 +28,14 @@ * */ +#ifdef VRT_H_INCLUDED +# error "vrt.h included before cache.h - they are exclusive" +#endif + +#ifdef CACHE_H_INCLUDED +# error "cache.h included multiple times" +#endif + #include #include #include @@ -36,6 +44,10 @@ #include "vdef.h" +#include "vrt.h" + +#define CACHE_H_INCLUDED + #include "miniobj.h" #include "vas.h" #include "vqueue.h" diff --git a/bin/varnishd/cache/cache_backend.c b/bin/varnishd/cache/cache_backend.c index b344770..de03e88 100644 --- a/bin/varnishd/cache/cache_backend.c +++ b/bin/varnishd/cache/cache_backend.c @@ -36,7 +36,6 @@ #include "cache_varnishd.h" -#include "vrt.h" #include "vtcp.h" #include "vtim.h" #include "waiter/waiter.h" diff --git a/bin/varnishd/cache/cache_backend_cfg.c b/bin/varnishd/cache/cache_backend_cfg.c index 5a30950..84a0cb1 100644 --- a/bin/varnishd/cache/cache_backend_cfg.c +++ b/bin/varnishd/cache/cache_backend_cfg.c @@ -39,7 +39,6 @@ #include "cache_varnishd.h" #include "vcli_serve.h" -#include "vrt.h" #include "vcl.h" #include "vtim.h" #include "waiter/waiter.h" diff --git a/bin/varnishd/cache/cache_backend_probe.c b/bin/varnishd/cache/cache_backend_probe.c index e76468d..aa176c0 100644 --- a/bin/varnishd/cache/cache_backend_probe.c +++ b/bin/varnishd/cache/cache_backend_probe.c @@ -45,7 +45,6 @@ #include "binary_heap.h" #include "vcli_serve.h" -#include "vrt.h" #include "vsa.h" #include "vtcp.h" #include "vtim.h" diff --git a/bin/varnishd/cache/cache_director.c b/bin/varnishd/cache/cache_director.c index d9bd52a..3fd9aec 100644 --- a/bin/varnishd/cache/cache_director.c +++ b/bin/varnishd/cache/cache_director.c @@ -39,8 +39,6 @@ #include "cache_director.h" -#include "vrt.h" - /* Resolve director --------------------------------------------------*/ static const struct director * diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index c6527e8..167c8a5 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -35,7 +35,6 @@ #include "cache_objhead.h" #include "hash/hash_slinger.h" #include "storage/storage.h" -#include "vrt.h" #include "vcl.h" #include "vtim.h" diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index c20fa63..869b2ad 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -45,8 +45,6 @@ #include "common/heritage.h" #include "waiter/waiter.h" - -#include "vrt.h" #include "cache_director.h" #include "storage/storage.h" #include "vcli_serve.h" diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 693ac31..8ea3621 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -46,7 +46,6 @@ #include "hash/hash_slinger.h" #include "storage/storage.h" -#include "vrt.h" #include "vcl.h" #include "vsha256.h" #include "vtim.h" diff --git a/bin/varnishd/cache/cache_tcp_pool.c b/bin/varnishd/cache/cache_tcp_pool.c index 5c351a4..04ea7e2 100644 --- a/bin/varnishd/cache/cache_tcp_pool.c +++ b/bin/varnishd/cache/cache_tcp_pool.c @@ -35,7 +35,6 @@ #include "cache_varnishd.h" -#include "vrt.h" #include "vsa.h" #include "vtcp.h" #include "vtim.h" diff --git a/bin/varnishd/cache/cache_vcl.c b/bin/varnishd/cache/cache_vcl.c index 547976e..e61fa76 100644 --- a/bin/varnishd/cache/cache_vcl.c +++ b/bin/varnishd/cache/cache_vcl.c @@ -38,7 +38,6 @@ #include "cache_varnishd.h" #include "common/heritage.h" -#include "vrt.h" #include "vcl.h" #include "waiter/waiter.h" diff --git a/bin/varnishd/cache/cache_vrt.c b/bin/varnishd/cache/cache_vrt.c index e1766c4..7f95ccb 100644 --- a/bin/varnishd/cache/cache_vrt.c +++ b/bin/varnishd/cache/cache_vrt.c @@ -36,7 +36,6 @@ #include "cache_director.h" #include "cache_objhead.h" #include "vav.h" -#include "vrt.h" #include "vcl.h" #include "vrt_obj.h" #include "vsa.h" diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index 8e65c8f..ffab825 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -36,8 +36,6 @@ #include "cache_varnishd.h" -#include "vrt.h" - struct vrt_priv { unsigned magic; #define VRT_PRIV_MAGIC 0x24157a52 diff --git a/bin/varnishd/cache/cache_vrt_re.c b/bin/varnishd/cache/cache_vrt_re.c index ed9d4fa..20a0cf6 100644 --- a/bin/varnishd/cache/cache_vrt_re.c +++ b/bin/varnishd/cache/cache_vrt_re.c @@ -35,8 +35,6 @@ #include "cache_varnishd.h" -#include "vrt.h" - static void Tadd(char **b, char *e, const char *p, int l) { diff --git a/bin/varnishd/cache/cache_vrt_var.c b/bin/varnishd/cache/cache_vrt_var.c index 9d4db8e..0f1d018 100644 --- a/bin/varnishd/cache/cache_vrt_var.c +++ b/bin/varnishd/cache/cache_vrt_var.c @@ -36,7 +36,6 @@ #include "common/heritage.h" #include "cache_director.h" -#include "vrt.h" #include "vrt_obj.h" static char vrt_hostname[255] = ""; diff --git a/bin/varnishd/cache/cache_vrt_vmod.c b/bin/varnishd/cache/cache_vrt_vmod.c index 04502db..f1c7a80 100644 --- a/bin/varnishd/cache/cache_vrt_vmod.c +++ b/bin/varnishd/cache/cache_vrt_vmod.c @@ -39,7 +39,6 @@ #include "vcli_serve.h" #include "vmod_abi.h" -#include "vrt.h" /*-------------------------------------------------------------------- * Modules stuff diff --git a/bin/varnishd/storage/stevedore.c b/bin/varnishd/storage/stevedore.c index cab8ac1..393486d 100644 --- a/bin/varnishd/storage/stevedore.c +++ b/bin/varnishd/storage/stevedore.c @@ -39,7 +39,6 @@ #include #include "storage/storage.h" -#include "vrt.h" #include "vrt_obj.h" diff --git a/bin/varnishd/storage/storage_malloc.c b/bin/varnishd/storage/storage_malloc.c index 0468835..9fe13b5 100644 --- a/bin/varnishd/storage/storage_malloc.c +++ b/bin/varnishd/storage/storage_malloc.c @@ -40,7 +40,6 @@ #include "storage/storage.h" #include "storage/storage_simple.h" -#include "vrt.h" #include "vnum.h" #include "VSC_sma.h" diff --git a/include/vrt.h b/include/vrt.h index bb875e3..6a07e8c 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -31,6 +31,10 @@ * NB: When this file is changed, lib/libvcc/generate.py *MUST* be rerun. */ +#ifdef CACHE_H_INCLUDED +# error "vrt.h included after cache.h - they are inclusive" +#endif + #ifdef VRT_H_INCLUDED # error "vrt.h included multiple times" #endif diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c index 73dd831..030140f 100644 --- a/lib/libvmod_blob/vmod_blob.c +++ b/lib/libvmod_blob/vmod_blob.c @@ -32,7 +32,7 @@ #include #include "vdef.h" -#include "vrt.h" +#include "cache/cache.h" #include "vcc_if.h" #include "vmod_blob.h" diff --git a/lib/libvmod_blob/wb.c b/lib/libvmod_blob/wb.c index 343ca59..e16c421 100644 --- a/lib/libvmod_blob/wb.c +++ b/lib/libvmod_blob/wb.c @@ -28,6 +28,8 @@ #include +#include "vdef.h" +#include "cache/cache.h" #include "wb.h" char * diff --git a/lib/libvmod_blob/wb.h b/lib/libvmod_blob/wb.h index ce11ba1..e2c820d 100644 --- a/lib/libvmod_blob/wb.h +++ b/lib/libvmod_blob/wb.h @@ -25,8 +25,6 @@ * */ -#include "cache/cache.h" - struct wb_s { struct ws *ws; // varnish workspace char *w; // current write position diff --git a/lib/libvmod_debug/vmod_debug.c b/lib/libvmod_debug/vmod_debug.c index e538543..5ed5cd0 100644 --- a/lib/libvmod_debug/vmod_debug.c +++ b/lib/libvmod_debug/vmod_debug.c @@ -37,7 +37,6 @@ #include "cache/cache.h" -#include "vrt.h" #include "vsa.h" #include "vsb.h" #include "vtcp.h" diff --git a/lib/libvmod_debug/vmod_debug_dyn.c b/lib/libvmod_debug/vmod_debug_dyn.c index af61742..8742ff9 100644 --- a/lib/libvmod_debug/vmod_debug_dyn.c +++ b/lib/libvmod_debug/vmod_debug_dyn.c @@ -34,9 +34,6 @@ #include #include "cache/cache.h" - -#include "vrt.h" - #include "cache/cache_director.h" #include "vsa.h" diff --git a/lib/libvmod_debug/vmod_debug_obj.c b/lib/libvmod_debug/vmod_debug_obj.c index 9dfef9d..3364092 100644 --- a/lib/libvmod_debug/vmod_debug_obj.c +++ b/lib/libvmod_debug/vmod_debug_obj.c @@ -33,7 +33,6 @@ #include "cache/cache.h" -#include "vrt.h" #include "vcc_if.h" struct vmod_debug_obj { diff --git a/lib/libvmod_directors/fall_back.c b/lib/libvmod_directors/fall_back.c index 9a4d392..544c168 100644 --- a/lib/libvmod_directors/fall_back.c +++ b/lib/libvmod_directors/fall_back.c @@ -34,7 +34,6 @@ #include "cache/cache.h" #include "cache/cache_director.h" -#include "vrt.h" #include "vcc_if.h" #include "vdir.h" diff --git a/lib/libvmod_directors/hash.c b/lib/libvmod_directors/hash.c index 60fa834..14f36f5 100644 --- a/lib/libvmod_directors/hash.c +++ b/lib/libvmod_directors/hash.c @@ -34,7 +34,6 @@ #include "cache/cache.h" #include "cache/cache_director.h" -#include "vrt.h" #include "vend.h" #include "vsha256.h" diff --git a/lib/libvmod_directors/random.c b/lib/libvmod_directors/random.c index 10e6f53..0608021 100644 --- a/lib/libvmod_directors/random.c +++ b/lib/libvmod_directors/random.c @@ -35,7 +35,6 @@ #include "vbm.h" #include "vrnd.h" -#include "vrt.h" #include "vdir.h" diff --git a/lib/libvmod_directors/round_robin.c b/lib/libvmod_directors/round_robin.c index 8a867cc..64ed115 100644 --- a/lib/libvmod_directors/round_robin.c +++ b/lib/libvmod_directors/round_robin.c @@ -34,7 +34,6 @@ #include "cache/cache.h" #include "cache/cache_director.h" -#include "vrt.h" #include "vcc_if.h" #include "vdir.h" diff --git a/lib/libvmod_directors/shard_cfg.c b/lib/libvmod_directors/shard_cfg.c index 7fc5056..620126f 100644 --- a/lib/libvmod_directors/shard_cfg.c +++ b/lib/libvmod_directors/shard_cfg.c @@ -37,8 +37,6 @@ #include "cache/cache.h" #include "cache/cache_director.h" -#include "vrt.h" - #include "shard_dir.h" #include "shard_cfg.h" #include "shard_hash.h" diff --git a/lib/libvmod_directors/shard_dir.c b/lib/libvmod_directors/shard_dir.c index aa56483..36c901b 100644 --- a/lib/libvmod_directors/shard_dir.c +++ b/lib/libvmod_directors/shard_dir.c @@ -39,7 +39,6 @@ #include "cache/cache.h" #include "cache/cache_director.h" -#include "vrt.h" #include "vbm.h" #include "vrnd.h" diff --git a/lib/libvmod_directors/shard_hash.c b/lib/libvmod_directors/shard_hash.c index 64e7e15..560ceb9 100644 --- a/lib/libvmod_directors/shard_hash.c +++ b/lib/libvmod_directors/shard_hash.c @@ -34,7 +34,6 @@ #include "cache/cache.h" -#include "vrt.h" #include "vsha256.h" #include "vend.h" diff --git a/lib/libvmod_directors/vdir.c b/lib/libvmod_directors/vdir.c index 1581346..c286497 100644 --- a/lib/libvmod_directors/vdir.c +++ b/lib/libvmod_directors/vdir.c @@ -33,7 +33,6 @@ #include "cache/cache.h" #include "cache/cache_director.h" -#include "vrt.h" #include "vbm.h" #include "vdir.h" diff --git a/lib/libvmod_directors/vmod_shard.c b/lib/libvmod_directors/vmod_shard.c index c85c9b4..984c3ed 100644 --- a/lib/libvmod_directors/vmod_shard.c +++ b/lib/libvmod_directors/vmod_shard.c @@ -34,7 +34,6 @@ #include "cache/cache.h" -#include "vrt.h" #include "vend.h" #include "vcc_if.h" diff --git a/lib/libvmod_purge/vmod_purge.c b/lib/libvmod_purge/vmod_purge.c index 4ba039b..0df0c45 100644 --- a/lib/libvmod_purge/vmod_purge.c +++ b/lib/libvmod_purge/vmod_purge.c @@ -28,10 +28,12 @@ #include "config.h" -#include "math.h" - -#include "cache/cache.h" +#include +#include +#include "vdef.h" +#include "vas.h" +#include "miniobj.h" #include "vrt.h" #include "vcc_if.h" diff --git a/lib/libvmod_std/vmod_std.c b/lib/libvmod_std/vmod_std.c index 91f29ca..2887beb 100644 --- a/lib/libvmod_std/vmod_std.c +++ b/lib/libvmod_std/vmod_std.c @@ -41,7 +41,6 @@ #include "cache/cache.h" #include "vrnd.h" -#include "vrt.h" #include "vtcp.h" #include "vsa.h" #include "vtim.h" diff --git a/lib/libvmod_std/vmod_std_conversions.c b/lib/libvmod_std/vmod_std_conversions.c index 18ac13d..c71b771 100644 --- a/lib/libvmod_std/vmod_std_conversions.c +++ b/lib/libvmod_std/vmod_std_conversions.c @@ -40,7 +40,6 @@ #include "cache/cache.h" #include "vnum.h" -#include "vrt.h" #include "vsa.h" #include "vtim.h" #include "vcc_if.h" diff --git a/lib/libvmod_std/vmod_std_fileread.c b/lib/libvmod_std/vmod_std_fileread.c index 8a69921..2be46ff 100644 --- a/lib/libvmod_std/vmod_std_fileread.c +++ b/lib/libvmod_std/vmod_std_fileread.c @@ -44,7 +44,6 @@ #include "cache/cache.h" -#include "vrt.h" #include "vfil.h" #include "vcc_if.h" diff --git a/lib/libvmod_std/vmod_std_querysort.c b/lib/libvmod_std/vmod_std_querysort.c index 3efe155..52a95f9 100644 --- a/lib/libvmod_std/vmod_std_querysort.c +++ b/lib/libvmod_std/vmod_std_querysort.c @@ -33,8 +33,6 @@ #include "cache/cache.h" -#include "vrt.h" - #include "vcc_if.h" static int diff --git a/lib/libvmod_vtc/vmod_vtc.c b/lib/libvmod_vtc/vmod_vtc.c index 840df35..9f8bbf9 100644 --- a/lib/libvmod_vtc/vmod_vtc.c +++ b/lib/libvmod_vtc/vmod_vtc.c @@ -36,7 +36,6 @@ #include "cache/cache.h" -#include "vrt.h" #include "vtcp.h" #include "vtim.h" From daghf at varnish-software.com Mon Oct 23 10:45:07 2017 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Mon, 23 Oct 2017 10:45:07 +0000 (UTC) Subject: [master] 25897d3 h2: Don't panic if we reembark with a request body Message-ID: <20171023104507.8964CA52E3@lists.varnish-cache.org> commit 25897d362b92117888d766ec6f0b8de53e954558 Author: Dag Haavi Finstad Date: Thu Oct 19 13:40:32 2017 +0200 h2: Don't panic if we reembark with a request body This moves the body status decision to h2_end_headers, to make sure that it is only done once per request. We also move the Cookie header concatenation to h2_end_headers, to avoid stepping on any changes that may have taken place in VCL. Fixes: #2305 diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index d7d1dce..f826136 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -480,26 +480,11 @@ h2_do_req(struct worker *wrk, void *priv) { struct req *req; struct h2_req *r2; - const char *b; CAST_OBJ_NOTNULL(req, priv, REQ_MAGIC); CAST_OBJ_NOTNULL(r2, req->transport_priv, H2_REQ_MAGIC); THR_SetRequest(req); - // XXX: Smarter to do this already at HPACK time into tail end of - // XXX: WS, then copy back once all headers received. - // XXX: Have I mentioned H/2 Is hodge-podge ? - http_CollectHdrSep(req->http, H_Cookie, "; "); // rfc7540,l,3114,3120 - - if (req->req_body_status == REQ_BODY_INIT) { - if (!http_GetHdr(req->http, H_Content_Length, &b)) - req->req_body_status = REQ_BODY_WITHOUT_LEN; - else - req->req_body_status = REQ_BODY_WITH_LEN; - } else { - assert (req->req_body_status == REQ_BODY_NONE); - } - req->http->conds = 1; if (CNT_Request(wrk, req) != REQ_FSM_DISEMBARK) { AZ(req->ws->r); @@ -516,6 +501,7 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, struct req *req, struct h2_req *r2) { h2_error h2e; + const char *b; ASSERT_RXTHR(h2); assert(r2->state == H2_S_OPEN); @@ -533,6 +519,20 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, } VSLb_ts_req(req, "Req", req->t_req); + // XXX: Smarter to do this already at HPACK time into tail end of + // XXX: WS, then copy back once all headers received. + // XXX: Have I mentioned H/2 Is hodge-podge ? + http_CollectHdrSep(req->http, H_Cookie, "; "); // rfc7540,l,3114,3120 + + if (req->req_body_status == REQ_BODY_INIT) { + if (!http_GetHdr(req->http, H_Content_Length, &b)) + req->req_body_status = REQ_BODY_WITHOUT_LEN; + else + req->req_body_status = REQ_BODY_WITH_LEN; + } else { + assert (req->req_body_status == REQ_BODY_NONE); + } + req->req_step = R_STP_TRANSPORT; req->task.func = h2_do_req; req->task.priv = req; diff --git a/bin/varnishtest/tests/r02305.vtc b/bin/varnishtest/tests/r02305.vtc new file mode 100644 index 0000000..d0bb78a --- /dev/null +++ b/bin/varnishtest/tests/r02305.vtc @@ -0,0 +1,52 @@ +varnishtest "#2305: h/2 reembark with a request body" + +barrier b1 cond 2 +barrier b2 sock 2 + +server s1 { + rxreq + expect req.url == "/" + barrier b1 sync + delay 2 + txresp +} -start + +varnish v1 -cliok "param.set feature +http2" +varnish v1 -cliok "param.set debug +syncvsl" +varnish v1 -cliok "param.set debug +waitinglist" + +varnish v1 -vcl+backend { + import vtc; + sub vcl_recv { + return (hash); + } + sub vcl_deliver { + if (req.http.sync) { + vtc.barrier_sync("${b2_sock}"); + } + } +} -start + +client c1 { + stream 1 { + txreq + rxresp + expect resp.status == 200 + } -start + stream 3 { + barrier b1 sync + txreq -req POST -hdr sync 1 -body "foo" + rxwinup + # barrier b2 is here to make HEADERS vs WINDOW_UPDATE + # less racy + barrier b2 sync + rxresp + expect resp.status == 200 + } -run +} -run + +varnish v1 -vsl_catchup +varnish v1 -expect MEMPOOL.req0.live == 0 +varnish v1 -expect MEMPOOL.sess0.live == 0 +varnish v1 -expect MEMPOOL.req1.live == 0 +varnish v1 -expect MEMPOOL.sess1.live == 0 From daghf at varnish-software.com Mon Oct 23 10:45:07 2017 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Mon, 23 Oct 2017 10:45:07 +0000 (UTC) Subject: [master] 77a8b1a Return PROTOCOL_ERROR for END_STREAM + content-length Message-ID: <20171023104507.AEF10A52E6@lists.varnish-cache.org> commit 77a8b1aca866ebc1efa1c164c5e961866f2cf191 Author: Dag Haavi Finstad Date: Mon Oct 23 11:37:57 2017 +0200 Return PROTOCOL_ERROR for END_STREAM + content-length diff --git a/bin/varnishd/http2/cache_http2_proto.c b/bin/varnishd/http2/cache_http2_proto.c index f826136..0358acd 100644 --- a/bin/varnishd/http2/cache_http2_proto.c +++ b/bin/varnishd/http2/cache_http2_proto.c @@ -531,6 +531,8 @@ h2_end_headers(struct worker *wrk, struct h2_sess *h2, req->req_body_status = REQ_BODY_WITH_LEN; } else { assert (req->req_body_status == REQ_BODY_NONE); + if (http_GetContentLength(req->http) > 0) + return (H2CE_PROTOCOL_ERROR); //rfc7540,l,1838,1840 } req->req_step = R_STP_TRANSPORT; diff --git a/bin/varnishtest/tests/r02305.vtc b/bin/varnishtest/tests/r02305.vtc index d0bb78a..7da2d48 100644 --- a/bin/varnishtest/tests/r02305.vtc +++ b/bin/varnishtest/tests/r02305.vtc @@ -45,6 +45,27 @@ client c1 { } -run } -run +client c2 { + stream 1 { + txreq -hdr "content-length" "23" + } -run + stream 0 { + rxgoaway + expect goaway.err == PROTOCOL_ERROR + expect goaway.laststream == 1 + + } -run +} -run + +# Allow content-length: 0 +client c2 { + stream 1 { + txreq -hdr "content-length" "0" + rxresp + expect resp.status == 200 + } -run +} -run + varnish v1 -vsl_catchup varnish v1 -expect MEMPOOL.req0.live == 0 varnish v1 -expect MEMPOOL.sess0.live == 0 From daghf at varnish-software.com Mon Oct 23 11:37:06 2017 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Mon, 23 Oct 2017 11:37:06 +0000 (UTC) Subject: [master] 73ad3d9 Move the req-specific PRIV pointers to struct req Message-ID: <20171023113706.47CE0A578D@lists.varnish-cache.org> commit 73ad3d9396d5bedc19103593953d88508f8b9d97 Author: Dag Haavi Finstad Date: Thu Oct 12 15:23:41 2017 +0200 Move the req-specific PRIV pointers to struct req Keeping these in struct sess would necessitate extra locking for h/2 where we have multiple reqs in flight concurrently. Fixes: #2268 diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 2f5a379..95ff511 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -586,6 +586,8 @@ struct req { /* Temporary accounting */ struct acct_req acct; + + struct vrt_privs privs[1]; }; /*-------------------------------------------------------------------- @@ -624,8 +626,6 @@ struct sess { double t_open; /* fd accepted */ double t_idle; /* fd accepted or resp sent */ - struct vrt_privs privs[1]; - }; /* Prototypes etc ----------------------------------------------------*/ diff --git a/bin/varnishd/cache/cache_esi_deliver.c b/bin/varnishd/cache/cache_esi_deliver.c index 4076985..9770315 100644 --- a/bin/varnishd/cache/cache_esi_deliver.c +++ b/bin/varnishd/cache/cache_esi_deliver.c @@ -202,7 +202,7 @@ ved_include(struct req *preq, const char *src, const char *host, AZ(req->wrk); } - VRTPRIV_dynamic_kill(sp->privs, (uintptr_t)req); + VRTPRIV_dynamic_kill(req->privs, (uintptr_t)req); AZ(preq->vcl); preq->vcl = req->vcl; diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 869b2ad..652f48a 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -484,6 +484,8 @@ pan_req(struct vsb *vsb, const struct req *req) VSB_indent(vsb, -2); VSB_printf(vsb, "},\n"); + pan_privs(vsb, req->privs); + VSB_indent(vsb, -2); VSB_printf(vsb, "},\n"); } @@ -521,8 +523,6 @@ pan_sess(struct vsb *vsb, const struct sess *sp) cp = SES_Get_String_Attr(sp, SA_CLIENT_PORT); VSB_printf(vsb, "client = %s %s,\n", ci, cp); - pan_privs(vsb, sp->privs); - VSB_indent(vsb, -2); VSB_printf(vsb, "},\n"); } diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index 66fdbd3..c1e8979 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -134,6 +134,7 @@ Req_New(const struct worker *wrk, struct sess *sp) req->vdpe_nxt = 0; VTAILQ_INIT(&req->vdpe); + VRTPRIV_init(req->privs); return (req); } @@ -190,8 +191,9 @@ Req_Cleanup(struct sess *sp, struct worker *wrk, struct req *req) req->vcl = NULL; } - VRTPRIV_dynamic_kill(sp->privs, (uintptr_t)req); - VRTPRIV_dynamic_kill(sp->privs, (uintptr_t)&req->top); + VRTPRIV_dynamic_kill(req->privs, (uintptr_t)req); + VRTPRIV_dynamic_kill(req->privs, (uintptr_t)&req->top); + assert(VTAILQ_EMPTY(&req->privs->privs)); /* Charge and log byte counters */ if (req->vsl->wid) { diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c index bc3a86d..455788b 100644 --- a/bin/varnishd/cache/cache_session.c +++ b/bin/varnishd/cache/cache_session.c @@ -349,7 +349,6 @@ SES_New(struct pool *pp) sp->t_open = NAN; sp->t_idle = NAN; - VRTPRIV_init(sp->privs); Lck_New(&sp->mtx, lck_sess); CHECK_OBJ_NOTNULL(sp, SESS_MAGIC); return (sp); @@ -545,7 +544,6 @@ SES_Delete(struct sess *sp, enum sess_close reason, double now) if (reason == SC_NULL) reason = (enum sess_close)-sp->fd; - assert(VTAILQ_EMPTY(&sp->privs->privs)); VSL(SLT_SessClose, sp->vxid, "%s %.3f", sess_close_2str(reason, 0), now - sp->t_open); VSL(SLT_End, sp->vxid, "%s", ""); diff --git a/bin/varnishd/cache/cache_vrt_priv.c b/bin/varnishd/cache/cache_vrt_priv.c index ffab825..4e53969 100644 --- a/bin/varnishd/cache/cache_vrt_priv.c +++ b/bin/varnishd/cache/cache_vrt_priv.c @@ -92,24 +92,14 @@ VRTPRIV_init(struct vrt_privs *privs) } static struct vmod_priv * -vrt_priv_dynamic(VRT_CTX, uintptr_t id, uintptr_t vmod_id) +vrt_priv_dynamic(VRT_CTX, struct vrt_privs *vps, uintptr_t id, + uintptr_t vmod_id) { - struct vrt_privs *vps; struct vrt_priv *vp; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); + CHECK_OBJ_NOTNULL(vps, VRT_PRIVS_MAGIC); AN(vmod_id); - if (ctx->req) { - CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); - CHECK_OBJ_NOTNULL(ctx->req->sp, SESS_MAGIC); - CAST_OBJ_NOTNULL(vps, ctx->req->sp->privs, VRT_PRIVS_MAGIC); - } else if (ctx->bo) { - CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); - CAST_OBJ_NOTNULL(vps, ctx->bo->privs, VRT_PRIVS_MAGIC); - } else { - ASSERT_CLI(); - CAST_OBJ_NOTNULL(vps, cli_task_privs, VRT_PRIVS_MAGIC); - } VTAILQ_FOREACH(vp, &vps->privs, list) { CHECK_OBJ_NOTNULL(vp, VRT_PRIV_MAGIC); @@ -148,32 +138,38 @@ struct vmod_priv * VRT_priv_task(VRT_CTX, const void *vmod_id) { uintptr_t id; + struct vrt_privs *vps; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if (ctx->req) { CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); id = (uintptr_t)ctx->req; + CAST_OBJ_NOTNULL(vps, ctx->req->privs, VRT_PRIVS_MAGIC); } else if (ctx->bo) { CHECK_OBJ_NOTNULL(ctx->bo, BUSYOBJ_MAGIC); id = (uintptr_t)ctx->bo; + CAST_OBJ_NOTNULL(vps, ctx->bo->privs, VRT_PRIVS_MAGIC); } else { ASSERT_CLI(); id = (uintptr_t)cli_task_privs; + CAST_OBJ_NOTNULL(vps, cli_task_privs, VRT_PRIVS_MAGIC); } - return (vrt_priv_dynamic(ctx, id, (uintptr_t)vmod_id)); + return (vrt_priv_dynamic(ctx, vps, id, (uintptr_t)vmod_id)); } struct vmod_priv * VRT_priv_top(VRT_CTX, const void *vmod_id) { uintptr_t id; + struct vrt_privs *vps; CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC); if (ctx->req) { CHECK_OBJ_NOTNULL(ctx->req, REQ_MAGIC); CHECK_OBJ_NOTNULL(ctx->req->top, REQ_MAGIC); id = (uintptr_t)&ctx->req->top->top; - return (vrt_priv_dynamic(ctx, id, (uintptr_t)vmod_id)); + CAST_OBJ_NOTNULL(vps, ctx->req->top->privs, VRT_PRIVS_MAGIC); + return (vrt_priv_dynamic(ctx, vps, id, (uintptr_t)vmod_id)); } else WRONG("PRIV_TOP is only accessible in client VCL context"); NEEDLESS(return NULL); From daghf at varnish-software.com Mon Oct 23 11:52:05 2017 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Mon, 23 Oct 2017 11:52:05 +0000 (UTC) Subject: [master] 19080c0 Hold the sess mutex when evaluating r2->cond Message-ID: <20171023115206.07037A5D98@lists.varnish-cache.org> commit 19080c05ebf65a7245f2aded0bbb5e5a95d07201 Author: Dag Haavi Finstad Date: Tue Oct 10 11:46:19 2017 +0200 Hold the sess mutex when evaluating r2->cond The extra locking prevents the race where a request thread is setting r2->cond = NULL and the session thread is testing r2->cond != NULL. Fixes: #2434 diff --git a/bin/varnishd/http2/cache_http2_session.c b/bin/varnishd/http2/cache_http2_session.c index 63f6f95..b7e1783 100644 --- a/bin/varnishd/http2/cache_http2_session.c +++ b/bin/varnishd/http2/cache_http2_session.c @@ -320,6 +320,7 @@ h2_new_session(struct worker *wrk, void *arg) /* Delete all idle streams */ VSLb(h2->vsl, SLT_Debug, "H2 CLEANUP %s", h2->error->name); + Lck_Lock(&h2->sess->mtx); VTAILQ_FOREACH(r2, &h2->streams, list) { if (r2->error == 0) r2->error = h2->error; @@ -327,6 +328,7 @@ h2_new_session(struct worker *wrk, void *arg) AZ(pthread_cond_signal(r2->cond)); } AZ(pthread_cond_broadcast(h2->cond)); + Lck_Unlock(&h2->sess->mtx); while (1) { again = 0; VTAILQ_FOREACH_SAFE(r2, &h2->streams, list, r22) { From geoff at uplex.de Mon Oct 23 16:11:10 2017 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 23 Oct 2017 16:11:10 +0000 (UTC) Subject: [master] 1cd83da Fix #includes in storage_umem.c. Message-ID: <20171023161110.DB49BA5962@lists.varnish-cache.org> commit 1cd83daee3a2d656e8cb84c6fc68577ee7818ea9 Author: Geoff Simmons Date: Mon Oct 23 18:09:39 2017 +0200 Fix #includes in storage_umem.c. Don't include vrt.h after cache_varnishd.h, which includes cache.h. diff --git a/bin/varnishd/storage/storage_umem.c b/bin/varnishd/storage/storage_umem.c index 791a6e5..23244e1 100644 --- a/bin/varnishd/storage/storage_umem.c +++ b/bin/varnishd/storage/storage_umem.c @@ -44,7 +44,6 @@ #include "storage/storage.h" #include "storage/storage_simple.h" -#include "vrt.h" #include "vnum.h" #include "common/heritage.h" From geoff at uplex.de Mon Oct 23 16:17:06 2017 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 23 Oct 2017 16:17:06 +0000 (UTC) Subject: [master] eaa4fdb Get the vsl_glob_test to run on SunOS. Message-ID: <20171023161706.92057A5C59@lists.varnish-cache.org> commit eaa4fdbeabc85abafc1333493971ef55e715b10a Author: Geoff Simmons Date: Mon Oct 23 18:15:06 2017 +0200 Get the vsl_glob_test to run on SunOS. fnmatch(3) appears to know FNM_IGNORECASE but not FNM_CASEFOLD. diff --git a/lib/libvarnishapi/vsl_glob_test.c b/lib/libvarnishapi/vsl_glob_test.c index e4f62a9..ca0b901 100644 --- a/lib/libvarnishapi/vsl_glob_test.c +++ b/lib/libvarnishapi/vsl_glob_test.c @@ -40,6 +40,10 @@ #include "vdef.h" #include "vas.h" +#ifndef FNM_CASEFOLD +#define FNM_CASEFOLD FNM_IGNORECASE +#endif + static void cb(int tag, void *priv) { From geoff at uplex.de Mon Oct 23 18:20:11 2017 From: geoff at uplex.de (Geoff Simmons) Date: Mon, 23 Oct 2017 18:20:11 +0000 (UTC) Subject: [master] 3ad1666 Remove the test of varnishtest -i with an empty argument. Message-ID: <20171023182011.854F5A542A@lists.varnish-cache.org> commit 3ad16662cda0510a88a04fe86f264156583b8f20 Author: Geoff Simmons Date: Mon Oct 23 20:17:36 2017 +0200 Remove the test of varnishtest -i with an empty argument. Getting the quoting right to pass along the empty string in SunOS is not working. The code path is tested in vsl_glob_test. diff --git a/bin/varnishtest/tests/u00006.vtc b/bin/varnishtest/tests/u00006.vtc index 24f69ca..8dcb2f9 100644 --- a/bin/varnishtest/tests/u00006.vtc +++ b/bin/varnishtest/tests/u00006.vtc @@ -38,8 +38,6 @@ shell -err -expect {-i: "Resp" is ambiguous} \ "varnishlog -i Resp" shell -err -expect {-i: Syntax error in "Re**sp"} \ "varnishlog -i Re**sp" -shell -err -expect {-i: "" matches zero tags} \ - {varnishlog -i ""} shell -err -expect {-I: "foo" matches zero tags} \ "varnishlog -I foo:bar" From phk at FreeBSD.org Tue Oct 24 07:14:12 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 24 Oct 2017 07:14:12 +0000 (UTC) Subject: [master] 4c356d6 Make vrt.h include and Message-ID: <20171024071412.853B9A5DA0@lists.varnish-cache.org> commit 4c356d698b7abc59195989bcddcbcb210cc6c096 Author: Poul-Henning Kamp Date: Tue Oct 24 07:12:15 2017 +0000 Make vrt.h include and Originally we decided that the output of VCC should be 100% stand-alone and therefore contain no #includes at all. This was hoped to avoid unspecified trouble with C-compilers at runtime. But C99 is old enough to drink now, so we move forward. The script in tools/include_wash.py checks *.c files and complains about violations of our intended #include orders. diff --git a/tools/include_wash.py b/tools/include_wash.py new file mode 100644 index 0000000..6dd12f6 --- /dev/null +++ b/tools/include_wash.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +from __future__ import print_function + +import os + +def check(fn): + l = [] + for i in open(fn): + i = i.strip() + if len(i) == 0: + continue + if i[0] != "#": + continue + if i.find("include") == -1: + continue + if i.find('"') == -1: + l.append(i.split('<')[1].split('>')[0]) + else: + l.append(i.split('"')[1]) + if "vrt.h" in l: + vrt = l.index("vrt.h") + if not "vdef.h" in l: + print(fn, "vdef.h not included with vrt.h") + vdef = l.index("vdef.h") + if vdef > vrt: + print(fn, "vdef.h included after vrt.h") + for i in ("stddef.h", "stdint.h", "cache/cache.h", "cache.h"): + if i in l: + print(fn, i + " included with vrt.h") + + for i in ("cache/cache.h", "cache.h"): + if i in l: + for i in ( + "stddef.h", "stdint.h", "vrt.h", + "math.h", "pthread.h", "stdarg.h", "sys/types.h", + "vdef.h", "miniobj.h", "vas.h", "vqueue.h", + ): + if i in l: + print(fn, i + " included with cache.h") + +for (dir, dns, fns) in os.walk("."): + for f in fns: + if f[-2:] == ".c": + check(dir + "/" + f) From fgsch at lodoss.net Wed Oct 25 06:56:12 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 25 Oct 2017 06:56:12 +0000 (UTC) Subject: [master] 22a79a5 Actually disable the report with the others Message-ID: <20171025065613.03BB0A528E@lists.varnish-cache.org> commit 22a79a51d5b341ac5999b855606568ca9e9f8222 Author: Federico G. Schwindt Date: Tue Oct 24 23:48:19 2017 -0700 Actually disable the report with the others diff --git a/bin/varnishd/hpack/vhp_gen_hufdec.c b/bin/varnishd/hpack/vhp_gen_hufdec.c index bc62567..5f86440 100644 --- a/bin/varnishd/hpack/vhp_gen_hufdec.c +++ b/bin/varnishd/hpack/vhp_gen_hufdec.c @@ -252,11 +252,3 @@ main(int argc, const char **argv) return (0); } - -#if defined(__SANITIZER) -int __lsan_is_turned_off(void); -int __lsan_is_turned_off() -{ - return (1); -} -#endif diff --git a/tools/lsan.suppr b/tools/lsan.suppr index 041605d..85effbc 100644 --- a/tools/lsan.suppr +++ b/tools/lsan.suppr @@ -11,3 +11,5 @@ leak:HSH_config leak:vcc_ leak:VSL_Setup leak:WRK_BgThread +# +leak:hpack/vhp_gen_hufdec.c From fgsch at lodoss.net Wed Oct 25 06:56:13 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 25 Oct 2017 06:56:13 +0000 (UTC) Subject: [master] 3ec10c8 Ignore vsl_glob_test Message-ID: <20171025065613.1721CA5290@lists.varnish-cache.org> commit 3ec10c8ca8416577c387fae81720b4c069fda412 Author: Federico G. Schwindt Date: Tue Oct 24 23:50:13 2017 -0700 Ignore vsl_glob_test diff --git a/.gitignore b/.gitignore index ba1a4d2..efdd279 100644 --- a/.gitignore +++ b/.gitignore @@ -117,6 +117,7 @@ cscope.*out /lib/libvarnish/vnum_c_test /lib/libvarnishapi/*.log /lib/libvarnishapi/vjsn_test +/lib/libvarnishapi/vsl_glob_test # vtest.sh droppings /tools/tmp/ From fgsch at lodoss.net Wed Oct 25 06:56:13 2017 From: fgsch at lodoss.net (Federico G. Schwindt) Date: Wed, 25 Oct 2017 06:56:13 +0000 (UTC) Subject: [master] 1108bde Fix sanitizer builds and tidy up Message-ID: <20171025065613.33670A5293@lists.varnish-cache.org> commit 1108bdeea0d75bb696856332ef917e0d30a01984 Author: Federico G. Schwindt Date: Tue Oct 24 23:53:25 2017 -0700 Fix sanitizer builds and tidy up diff --git a/lib/libvarnishapi/Makefile.am b/lib/libvarnishapi/Makefile.am index 41339ed..7c88c23 100644 --- a/lib/libvarnishapi/Makefile.am +++ b/lib/libvarnishapi/Makefile.am @@ -87,29 +87,23 @@ vxp_tokens.h: \ $(srcdir)/generate.py @PYTHON@ $(srcdir)/generate.py $(srcdir) $(top_builddir) -EXTRA_PROGRAMS = vxp_test vsl_glob_test - -vxp_test_LDADD = @PCRE_LIBS@ \ - ${RT_LIBS} ${LIBM} ${PTHREAD_LIBS} - -vxp_test_CFLAGS = \ - -DVARNISH_STATE_DIR='"${VARNISH_STATE_DIR}"' \ - -DVXP_DEBUG +EXTRA_PROGRAMS = vxp_test vxp_test_SOURCES = \ $(libvarnishapi_la_SOURCES) \ vxp_test.c - - -vsl_glob_test_LDADD = @PCRE_LIBS@ ${RT_LIBS} ${LIBM} libvarnishapi.la - -vsl_glob_test_CFLAGS = -I$(top_srcdir)/include +vxp_test_CFLAGS = \ + -DVARNISH_STATE_DIR='"${VARNISH_STATE_DIR}"' \ + -DVXP_DEBUG +vxp_test_LDADD = @PCRE_LIBS@ \ + ${RT_LIBS} ${LIBM} ${PTHREAD_LIBS} TESTS = vjsn_test vsl_glob_test noinst_PROGRAMS += ${TESTS} vsl_glob_test_SOURCES = vsl_glob_test.c +vsl_glob_test_CFLAGS = @SAN_CFLAGS@ vsl_glob_test_LDADD = libvarnishapi.la @SAN_LDFLAGS@ vjsn_test_SOURCES = vjsn.c From phk at FreeBSD.org Wed Oct 25 21:28:07 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 25 Oct 2017 21:28:07 +0000 (UTC) Subject: [master] 55e56b3 We no longer need to discard first byte of ident Message-ID: <20171025212807.8C26EA653B@lists.varnish-cache.org> commit 55e56b36a84c22ef4e80f8e4d91cb5978b262afe Author: Poul-Henning Kamp Date: Wed Oct 25 19:55:49 2017 +0000 We no longer need to discard first byte of ident diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index 652f48a..e778d78 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -629,7 +629,7 @@ pan_ic(const char *func, const char *file, int line, const char *cond, VSB_printf(pan_vsb, "version = %s, vrt api = %u.%u\n", VCS_version, VRT_MAJOR_VERSION, VRT_MINOR_VERSION); VSB_printf(pan_vsb, "ident = %s,%s\n", - heritage.ident + 1, Waiter_GetName()); + heritage.ident, Waiter_GetName()); VSB_printf(pan_vsb, "now = %f (mono), %f (real)\n", VTIM_mono(), VTIM_real()); From phk at FreeBSD.org Wed Oct 25 21:28:07 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Wed, 25 Oct 2017 21:28:07 +0000 (UTC) Subject: [master] 7d2c77a Add all the stuff I thought would be in 4c356d698b7abc59195989bcddcbcb210cc6c096 Message-ID: <20171025212807.A15B8A653D@lists.varnish-cache.org> commit 7d2c77ac73f6553ee0820d25ffe7ee750dfd5cdc Author: Poul-Henning Kamp Date: Wed Oct 25 19:56:40 2017 +0000 Add all the stuff I thought would be in 4c356d698b7abc59195989bcddcbcb210cc6c096 diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 95ff511..419ae51 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -39,7 +39,6 @@ #include #include #include -#include #include #include "vdef.h" diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index 1bd002e..b18f867 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -32,7 +32,6 @@ #include "config.h" -#include #include #include "cache_varnishd.h" diff --git a/bin/varnishd/cache/cache_http.c b/bin/varnishd/cache/cache_http.c index 9c4dac8..36bf2b3 100644 --- a/bin/varnishd/cache/cache_http.c +++ b/bin/varnishd/cache/cache_http.c @@ -31,7 +31,6 @@ #include "config.h" -#include #include "cache_varnishd.h" #include diff --git a/bin/varnishd/cache/cache_mempool.c b/bin/varnishd/cache/cache_mempool.c index f2cc1c8..6c150bd 100644 --- a/bin/varnishd/cache/cache_mempool.c +++ b/bin/varnishd/cache/cache_mempool.c @@ -30,7 +30,6 @@ #include "config.h" -#include #include "cache_varnishd.h" #include diff --git a/bin/varnishd/common/common_vsc.c b/bin/varnishd/common/common_vsc.c index 1ce53cd..9379f2d 100644 --- a/bin/varnishd/common/common_vsc.c +++ b/bin/varnishd/common/common_vsc.c @@ -31,18 +31,18 @@ #include #include -#include #include #include #include #include "vdef.h" +#include "vrt.h" + #include "miniobj.h" #include "vas.h" #include "vend.h" #include "vgz.h" #include "vmb.h" -#include "vrt.h" #include "vsmw.h" #include "vqueue.h" #include "vapi/vsc_int.h" diff --git a/bin/varnishd/storage/storage_persistent_subr.c b/bin/varnishd/storage/storage_persistent_subr.c index aedb02b..2e15c3f 100644 --- a/bin/varnishd/storage/storage_persistent_subr.c +++ b/bin/varnishd/storage/storage_persistent_subr.c @@ -35,7 +35,6 @@ #include "config.h" -#include #include "cache/cache_varnishd.h" #include "common/heritage.h" diff --git a/include/vrt.h b/include/vrt.h index 6a07e8c..5b79a80 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -89,6 +89,9 @@ /***********************************************************************/ +#include // NULL, size_t +#include // [u]int%d_t + struct VCL_conf; struct busyobj; struct director; diff --git a/lib/libvarnishapi/vsl_glob_test.c b/lib/libvarnishapi/vsl_glob_test.c index ca0b901..bf09d0b 100644 --- a/lib/libvarnishapi/vsl_glob_test.c +++ b/lib/libvarnishapi/vsl_glob_test.c @@ -41,7 +41,7 @@ #include "vas.h" #ifndef FNM_CASEFOLD -#define FNM_CASEFOLD FNM_IGNORECASE +# define FNM_CASEFOLD FNM_IGNORECASE #endif static void @@ -53,7 +53,7 @@ cb(int tag, void *priv) } static int -tst_one(const char *p) +tst_one_glob(const char *p) { int i; @@ -69,18 +69,23 @@ main(int argc, char * const *argv) int i, j; if (argc == 1) { - i = tst_one("Req*"); + i = tst_one_glob("Req*"); assert(i == 10); - j = tst_one("reQ*"); + j = tst_one_glob("reQ*"); assert(i == j); - assert(tst_one("*Header") > 0); - assert(tst_one("Req*eader") == 1); - assert(tst_one("*") > 0); - assert(tst_one("a*b*c") == -3); - assert(tst_one("**") == -3); - assert(tst_one("_") == -1); - assert(tst_one("") == -1); + assert(tst_one_glob("*Header") > 0); + assert(tst_one_glob("Req*eader") == 1); + assert(tst_one_glob("xyz*y") == -1); + assert(tst_one_glob("*") > 0); + assert(tst_one_glob("a*b*c") == -3); + assert(tst_one_glob("**") == -3); + assert(tst_one_glob("_") == -1); + assert(tst_one_glob("") == -1); assert(VSL_Glob2Tags("", 0, cb, NULL) == -1); + + assert(VSL_List2Tags("Req*,Resp*",-1,cb,NULL) > 0); + assert(VSL_List2Tags(",,,",-1,cb,NULL) == -1); + assert(VSL_List2Tags("xyzzy,,xyzzy",-1,cb,NULL) == -1); return (0); } if (argc != 2) { diff --git a/lib/libvcc/vsctool.py b/lib/libvcc/vsctool.py index c0507ea..7acc7a5 100644 --- a/lib/libvcc/vsctool.py +++ b/lib/libvcc/vsctool.py @@ -172,10 +172,7 @@ class vscset(object): fo = open(fon, "w") genhdr(fo, self.name) fo.write('#include "config.h"\n') - fo.write('#include \n') fo.write('#include \n') - fo.write('#include \n') - fo.write('#include \n') fo.write('#include "vdef.h"\n') fo.write('#include "vas.h"\n') fo.write('#include "vrt.h"\n') diff --git a/lib/libvmod_blob/base64.c b/lib/libvmod_blob/base64.c index f7a8c06..2017029 100644 --- a/lib/libvmod_blob/base64.c +++ b/lib/libvmod_blob/base64.c @@ -28,7 +28,6 @@ #include "config.h" #include -#include #include "base64.h" diff --git a/lib/libvmod_blob/id.c b/lib/libvmod_blob/id.c index 34bb34b..d10e642 100644 --- a/lib/libvmod_blob/id.c +++ b/lib/libvmod_blob/id.c @@ -29,7 +29,6 @@ #include "config.h" #include #include -#include #include "vmod_blob.h" diff --git a/lib/libvmod_blob/vmod_blob.c b/lib/libvmod_blob/vmod_blob.c index 030140f..8beb982 100644 --- a/lib/libvmod_blob/vmod_blob.c +++ b/lib/libvmod_blob/vmod_blob.c @@ -28,10 +28,8 @@ #include "config.h" #include -#include #include -#include "vdef.h" #include "cache/cache.h" #include "vcc_if.h" diff --git a/lib/libvmod_blob/wb.c b/lib/libvmod_blob/wb.c index e16c421..d56b061 100644 --- a/lib/libvmod_blob/wb.c +++ b/lib/libvmod_blob/wb.c @@ -28,7 +28,6 @@ #include -#include "vdef.h" #include "cache/cache.h" #include "wb.h" From phk at FreeBSD.org Thu Oct 26 13:30:16 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 26 Oct 2017 13:30:16 +0000 (UTC) Subject: [master] c2e71d0 Purge forward structs covered by vrt.h (or just lingering) Message-ID: <20171026133016.0C904A60B5@lists.varnish-cache.org> commit c2e71d04b59abbaf37cf036edbaa3b46a9f1e3a8 Author: Poul-Henning Kamp Date: Thu Oct 26 08:55:11 2017 +0000 Purge forward structs covered by vrt.h (or just lingering) diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 419ae51..9ee0917 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -86,29 +86,20 @@ enum { /*--------------------------------------------------------------------*/ struct VSC_lck; +struct VSC_main; +struct backend; struct ban; struct ban_proto; -struct backend; -struct busyobj; struct cli; struct cli_proto; -struct director; -struct iovec; struct mempool; struct objcore; struct objhead; struct pool; struct poolparam; -struct transport; -struct req; struct sess; -struct stevedore; -struct suckaddr; -struct vrt_priv; -struct vsb; +struct transport; struct worker; -struct v1l; -struct VSC_main; #define DIGEST_LEN 32 From phk at FreeBSD.org Thu Oct 26 13:30:16 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 26 Oct 2017 13:30:16 +0000 (UTC) Subject: [master] 62c5b88 Clean up vfp_ctx a bit. Message-ID: <20171026133016.25B31A60B8@lists.varnish-cache.org> commit 62c5b886dbdd2b65ef7629058010526b603efa70 Author: Poul-Henning Kamp Date: Thu Oct 26 09:58:00 2017 +0000 Clean up vfp_ctx a bit. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 9ee0917..dc0a12c 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -175,17 +175,14 @@ VTAILQ_HEAD(vfp_entry_s, vfp_entry); struct vfp_ctx { unsigned magic; #define VFP_CTX_MAGIC 0x61d9d3e5 - struct busyobj *bo; + int failed; + struct http *req; + struct http *resp; struct worker *wrk; struct objcore *oc; - int failed; - struct vfp_entry_s vfp; struct vfp_entry *vfp_nxt; - - struct http *http; - struct http *esi_req; }; /*-------------------------------------------------------------------- diff --git a/bin/varnishd/cache/cache_esi_fetch.c b/bin/varnishd/cache/cache_esi_fetch.c index 071fb79..47ea1c8 100644 --- a/bin/varnishd/cache/cache_esi_fetch.c +++ b/bin/varnishd/cache/cache_esi_fetch.c @@ -84,7 +84,6 @@ vfp_vep_callback(struct vfp_ctx *vc, void *priv, ssize_t l, enum vgz_flag flg) if (l == 0 && flg == VGZ_NORMAL) return (vef->tot); - CHECK_OBJ_NOTNULL(vc->bo, BUSYOBJ_MAGIC); VGZ_Ibuf(vef->vgz, vef->ibuf_o, l); do { dl = 0; @@ -152,13 +151,13 @@ vfp_esi_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe) struct vef_priv *vef; CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); - CHECK_OBJ_NOTNULL(vc->esi_req, HTTP_MAGIC); + CHECK_OBJ_NOTNULL(vc->req, HTTP_MAGIC); CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC); ALLOC_OBJ(vef, VEF_MAGIC); if (vef == NULL) return (VFP_ERROR); vef->vgz = VGZ_NewGzip(vc->wrk->vsl, "G F E"); - vef->vep = VEP_Init(vc, vc->esi_req, vfp_vep_callback, vef); + vef->vep = VEP_Init(vc, vc->req, vfp_vep_callback, vef); vef->ibuf_sz = cache_param->gzip_buffer; vef->ibuf = calloc(1L, vef->ibuf_sz); if (vef->ibuf == NULL) @@ -167,12 +166,12 @@ vfp_esi_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe) vef->ibuf_o = vef->ibuf; vfe->priv1 = vef; - RFC2616_Weaken_Etag(vc->http); - http_Unset(vc->http, H_Content_Length); - http_Unset(vc->http, H_Content_Encoding); - http_SetHeader(vc->http, "Content-Encoding: gzip"); + RFC2616_Weaken_Etag(vc->resp); + http_Unset(vc->resp, H_Content_Length); + http_Unset(vc->resp, H_Content_Encoding); + http_SetHeader(vc->resp, "Content-Encoding: gzip"); - RFC2616_Vary_AE(vc->http); + RFC2616_Vary_AE(vc->resp); return (VFP_OK); } @@ -226,11 +225,11 @@ vfp_esi_init(struct vfp_ctx *vc, struct vfp_entry *vfe) struct vef_priv *vef; CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); - CHECK_OBJ_NOTNULL(vc->esi_req, HTTP_MAGIC); + CHECK_OBJ_NOTNULL(vc->req, HTTP_MAGIC); ALLOC_OBJ(vef, VEF_MAGIC); if (vef == NULL) return (VFP_ERROR); - vef->vep = VEP_Init(vc, vc->esi_req, NULL, NULL); + vef->vep = VEP_Init(vc, vc->req, NULL, NULL); vfe->priv1 = vef; return (VFP_OK); } diff --git a/bin/varnishd/cache/cache_esi_parse.c b/bin/varnishd/cache/cache_esi_parse.c index 5ce78bd..048672b 100644 --- a/bin/varnishd/cache/cache_esi_parse.c +++ b/bin/varnishd/cache/cache_esi_parse.c @@ -1030,7 +1030,7 @@ VEP_Init(struct vfp_ctx *vc, const struct http *req, vep_callback_t *cb, CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); CHECK_OBJ_NOTNULL(req, HTTP_MAGIC); - vep = WS_Alloc(vc->http->ws, sizeof *vep); + vep = WS_Alloc(vc->resp->ws, sizeof *vep); AN(vep); INIT_OBJ(vep, VEP_MAGIC); diff --git a/bin/varnishd/cache/cache_fetch.c b/bin/varnishd/cache/cache_fetch.c index 167c8a5..9c25e70 100644 --- a/bin/varnishd/cache/cache_fetch.c +++ b/bin/varnishd/cache/cache_fetch.c @@ -288,10 +288,9 @@ vbf_stp_startfetch(struct worker *wrk, struct busyobj *bo) AZ(bo->htc); VFP_Setup(bo->vfc, wrk); - bo->vfc->bo = bo; bo->vfc->oc = bo->fetch_objcore; - bo->vfc->http = bo->beresp; - bo->vfc->esi_req = bo->bereq; + bo->vfc->resp = bo->beresp; + bo->vfc->req = bo->bereq; i = VDI_GetHdr(wrk, bo); @@ -812,11 +811,10 @@ vbf_stp_error(struct worker *wrk, struct busyobj *bo) assert(wrk->handling == VCL_RET_DELIVER); - assert(bo->vfc->bo == bo); assert(bo->vfc->wrk == bo->wrk); assert(bo->vfc->oc == bo->fetch_objcore); - assert(bo->vfc->http == bo->beresp); - assert(bo->vfc->esi_req == bo->bereq); + assert(bo->vfc->resp == bo->beresp); + assert(bo->vfc->req == bo->bereq); if (vbf_beresp2obj(bo)) { (void)VFP_Error(bo->vfc, "Could not get storage"); diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c index 3426f78..7d9bf5c 100644 --- a/bin/varnishd/cache/cache_fetch_proc.c +++ b/bin/varnishd/cache/cache_fetch_proc.c @@ -134,7 +134,7 @@ VFP_Open(struct vfp_ctx *vc) struct vfp_entry *vfe; CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); - CHECK_OBJ_NOTNULL(vc->http, HTTP_MAGIC); + CHECK_OBJ_NOTNULL(vc->resp, HTTP_MAGIC); CHECK_OBJ_NOTNULL(vc->wrk, WORKER_MAGIC); AN(vc->wrk->vsl); @@ -201,9 +201,9 @@ VFP_Push(struct vfp_ctx *vc, const struct vfp *vfp) struct vfp_entry *vfe; CHECK_OBJ_NOTNULL(vc, VFP_CTX_MAGIC); - CHECK_OBJ_NOTNULL(vc->http, HTTP_MAGIC); + CHECK_OBJ_NOTNULL(vc->resp, HTTP_MAGIC); - vfe = WS_Alloc(vc->http->ws, sizeof *vfe); + vfe = WS_Alloc(vc->resp->ws, sizeof *vfe); if (vfe == NULL) { (void)VFP_Error(vc, "Workspace overflow"); return (NULL); diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index f2b638a..28e99e3 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -446,11 +446,11 @@ vfp_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe) CHECK_OBJ_NOTNULL(vfe, VFP_ENTRY_MAGIC); if (vfe->vfp == &VFP_gzip) { - if (http_GetHdr(vc->http, H_Content_Encoding, NULL)) + if (http_GetHdr(vc->resp, H_Content_Encoding, NULL)) return (VFP_NULL); vg = VGZ_NewGzip(vc->wrk->vsl, vfe->vfp->priv1); } else { - if (!http_HdrIs(vc->http, H_Content_Encoding, "gzip")) + if (!http_HdrIs(vc->resp, H_Content_Encoding, "gzip")) return (VFP_NULL); if (vfe->vfp == &VFP_gunzip) vg = VGZ_NewGunzip(vc->wrk->vsl, vfe->vfp->priv1); @@ -466,16 +466,16 @@ vfp_gzip_init(struct vfp_ctx *vc, struct vfp_entry *vfe) AZ(vg->m_len); if (vfe->vfp == &VFP_gunzip || vfe->vfp == &VFP_gzip) { - http_Unset(vc->http, H_Content_Encoding); - http_Unset(vc->http, H_Content_Length); - RFC2616_Weaken_Etag(vc->http); + http_Unset(vc->resp, H_Content_Encoding); + http_Unset(vc->resp, H_Content_Length); + RFC2616_Weaken_Etag(vc->resp); } if (vfe->vfp == &VFP_gzip) - http_SetHeader(vc->http, "Content-Encoding: gzip"); + http_SetHeader(vc->resp, "Content-Encoding: gzip"); if (vfe->vfp == &VFP_gzip || vfe->vfp == &VFP_testgunzip) - RFC2616_Vary_AE(vc->http); + RFC2616_Vary_AE(vc->resp); return (VFP_OK); } diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 8ea3621..649b74a 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -92,7 +92,7 @@ cnt_transport(struct worker *wrk, struct req *req) if (req->req_body_status < REQ_BODY_TAKEN) { AN(req->transport->req_body != NULL); VFP_Setup(req->vfc, wrk); - req->vfc->http = req->http; + req->vfc->resp = req->http; // XXX req->transport->req_body(req); } diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c index 6c14193..e823c18 100644 --- a/bin/varnishd/http1/cache_http1_fetch.c +++ b/bin/varnishd/http1/cache_http1_fetch.c @@ -203,7 +203,7 @@ V1F_FetchRespHdr(struct busyobj *bo) htc->doclose = http_DoConnection(hp); RFC2616_Response_Body(bo->wrk, bo); - assert(bo->vfc->http == bo->beresp); + assert(bo->vfc->resp == bo->beresp); if (bo->htc->body_status != BS_NONE && bo->htc->body_status != BS_ERROR) (void)V1F_Setup_Fetch(bo->vfc, bo->htc); From phk at FreeBSD.org Thu Oct 26 13:30:16 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 26 Oct 2017 13:30:16 +0000 (UTC) Subject: [master] 72648a7 Hide struct vfp_ctx properly in cache_filter.h Message-ID: <20171026133016.42C49A60BB@lists.varnish-cache.org> commit 72648a74f8cdc7833582414cfc15ebb2fc0dfe5b Author: Poul-Henning Kamp Date: Thu Oct 26 13:29:10 2017 +0000 Hide struct vfp_ctx properly in cache_filter.h diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index dc0a12c..ff66738 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -167,25 +167,6 @@ struct http { }; /*-------------------------------------------------------------------- - * VFP filter state - */ - -VTAILQ_HEAD(vfp_entry_s, vfp_entry); - -struct vfp_ctx { - unsigned magic; -#define VFP_CTX_MAGIC 0x61d9d3e5 - int failed; - struct http *req; - struct http *resp; - struct worker *wrk; - struct objcore *oc; - - struct vfp_entry_s vfp; - struct vfp_entry *vfp_nxt; -}; - -/*-------------------------------------------------------------------- * HTTP Protocol connection structure * * This is the protocol independent object for a HTTP connection, used @@ -437,7 +418,7 @@ struct busyobj { struct sess *sp; struct worker *wrk; - struct vfp_ctx vfc[1]; + struct vfp_ctx *vfc; struct ws ws[1]; uintptr_t ws_bo; @@ -538,7 +519,7 @@ struct req { double t_req; /* Headers complete */ struct http_conn htc[1]; - struct vfp_ctx vfc[1]; + struct vfp_ctx *vfc; const char *client_identity; /* HTTP request */ diff --git a/bin/varnishd/cache/cache_busyobj.c b/bin/varnishd/cache/cache_busyobj.c index b18f867..3533c66 100644 --- a/bin/varnishd/cache/cache_busyobj.c +++ b/bin/varnishd/cache/cache_busyobj.c @@ -35,7 +35,7 @@ #include #include "cache_varnishd.h" - +#include "cache_filter.h" #include "cache_objhead.h" static struct mempool *vbopool; @@ -121,6 +121,11 @@ VBO_GetBusyObj(struct worker *wrk, const struct req *req) p = (void*)PRNDUP(p); assert(p < bo->end); + bo->vfc = (void*)p; + p += sizeof (*bo->vfc); + p = (void*)PRNDUP(p); + INIT_OBJ(bo->vfc, VFP_CTX_MAGIC); + WS_Init(bo->ws, "bo", p, bo->end - p); bo->do_stream = 1; diff --git a/bin/varnishd/cache/cache_esi_parse.c b/bin/varnishd/cache/cache_esi_parse.c index 048672b..f02a6df 100644 --- a/bin/varnishd/cache/cache_esi_parse.c +++ b/bin/varnishd/cache/cache_esi_parse.c @@ -31,6 +31,7 @@ #include "config.h" #include "cache_varnishd.h" +#include "cache_filter.h" #include "cache_vgz.h" #include "cache_esi.h" diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index 20c770d..84dfc05 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -65,6 +65,25 @@ struct vfp_entry { uint64_t bytes_out; }; +/*-------------------------------------------------------------------- + * VFP filter state + */ + +VTAILQ_HEAD(vfp_entry_s, vfp_entry); + +struct vfp_ctx { + unsigned magic; +#define VFP_CTX_MAGIC 0x61d9d3e5 + int failed; + struct http *req; + struct http *resp; + struct worker *wrk; + struct objcore *oc; + + struct vfp_entry_s vfp; + struct vfp_entry *vfp_nxt; +}; + enum vfp_status VFP_Suck(struct vfp_ctx *, void *p, ssize_t *lp); enum vfp_status VFP_Error(struct vfp_ctx *, const char *fmt, ...) __v_printflike(2, 3); diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index c1e8979..43581e7 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -33,6 +33,7 @@ #include "config.h" #include "cache_varnishd.h" +#include "cache_filter.h" #include #include @@ -122,6 +123,10 @@ Req_New(const struct worker *wrk, struct sess *sp) p += sz; p = (void*)PRNDUP(p); + req->vfc = (void*)p; + p += sizeof (*req->vfc); + INIT_OBJ(req->vfc, VFP_CTX_MAGIC); + assert(p < e); WS_Init(req->ws, "req", p, e - p); @@ -136,6 +141,7 @@ Req_New(const struct worker *wrk, struct sess *sp) VTAILQ_INIT(&req->vdpe); VRTPRIV_init(req->privs); + return (req); } diff --git a/bin/varnishd/http1/cache_http1_fetch.c b/bin/varnishd/http1/cache_http1_fetch.c index e823c18..9efb938 100644 --- a/bin/varnishd/http1/cache_http1_fetch.c +++ b/bin/varnishd/http1/cache_http1_fetch.c @@ -30,6 +30,7 @@ #include "config.h" #include "cache/cache_varnishd.h" +#include "cache/cache_filter.h" #include #include From phk at FreeBSD.org Thu Oct 26 14:02:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Thu, 26 Oct 2017 14:02:05 +0000 (UTC) Subject: [master] 1a78b80 Ditch aborted stack-space monitoring code. Message-ID: <20171026140205.2CFC0A6C36@lists.varnish-cache.org> commit 1a78b8022a9152b36c0994053d743e9bf4d927f9 Author: Poul-Henning Kamp Date: Thu Oct 26 14:00:51 2017 +0000 Ditch aborted stack-space monitoring code. diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index ff66738..a1d6155 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -288,9 +288,6 @@ struct worker { unsigned cur_method; unsigned seen_methods; unsigned handling; - - uintptr_t stack_start; - uintptr_t stack_end; }; /* Stored object ----------------------------------------------------- diff --git a/bin/varnishd/cache/cache_panic.c b/bin/varnishd/cache/cache_panic.c index e778d78..e84c4b7 100644 --- a/bin/varnishd/cache/cache_panic.c +++ b/bin/varnishd/cache/cache_panic.c @@ -318,8 +318,6 @@ pan_wrk(struct vsb *vsb, const struct worker *wrk) return; VSB_indent(vsb, 2); PAN_CheckMagic(vsb, wrk, WORKER_MAGIC); - VSB_printf(vsb, "stack = {0x%jx -> 0x%jx},\n", - (uintmax_t)wrk->stack_start, (uintmax_t)wrk->stack_end); pan_ws(vsb, wrk->aws); m = wrk->cur_method; diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c index 167481f..af65011 100644 --- a/bin/varnishd/cache/cache_wrk.c +++ b/bin/varnishd/cache/cache_wrk.c @@ -99,7 +99,6 @@ WRK_Thread(struct pool *qp, size_t stacksize, unsigned thread_workspace) struct worker *w, ww; struct VSC_main ds; unsigned char ws[thread_workspace]; - uintptr_t u; AN(qp); AN(stacksize); @@ -115,14 +114,6 @@ WRK_Thread(struct pool *qp, size_t stacksize, unsigned thread_workspace) WS_Init(w->aws, "wrk", ws, thread_workspace); - u = getpagesize(); - AN(u); - u -= 1U; - w->stack_start = (((uintptr_t)&qp) + u) & ~u; - - /* XXX: assuming stack grows down. */ - w->stack_end = w->stack_start - stacksize; - VSL(SLT_WorkThread, 0, "%p start", w); Pool_Work_Thread(qp, w); From phk at FreeBSD.org Mon Oct 30 11:08:08 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 30 Oct 2017 11:08:08 +0000 (UTC) Subject: [master] 571a1e3 Wrap VDP delivery context in a struct Message-ID: <20171030110808.280E5A670D@lists.varnish-cache.org> commit 571a1e3375ced741162772813444f2081cbb1325 Author: Poul-Henning Kamp Date: Fri Oct 27 07:32:10 2017 +0000 Wrap VDP delivery context in a struct diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index a1d6155..b64e320 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -532,9 +532,11 @@ struct req { struct objcore *stale_oc; /* Deliver pipeline */ - struct vdp_entry_s vdpe; - struct vdp_entry *vdpe_nxt; - unsigned vdpe_retval; + struct vdp_ctx { + struct vdp_entry_s vdp; + struct vdp_entry *nxt; + unsigned retval; + } vdp[1]; /* Delivery mode */ unsigned res_mode; diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 342696f..2def938 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -39,7 +39,7 @@ * This function picks and calls the next delivery processor from the * list. The return value is the return value of the delivery * processor. Upon seeing a non-zero return value, that lowest value - * observed is latched in req->vdpe_retval and all subsequent calls to + * observed is latched in ->retval and all subsequent calls to * VDP_bytes will return that value directly without calling the next * processor. * @@ -53,30 +53,34 @@ VDP_bytes(struct req *req, enum vdp_action act, const void *ptr, ssize_t len) { int retval; struct vdp_entry *vdpe; + struct vdp_ctx *vdc; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + vdc = req->vdp; assert(act == VDP_NULL || act == VDP_FLUSH); - if (req->vdpe_retval) - return (req->vdpe_retval); - vdpe = req->vdpe_nxt; + if (vdc->retval) + return (vdc->retval); + vdpe = vdc->nxt; CHECK_OBJ_NOTNULL(vdpe, VDP_ENTRY_MAGIC); - req->vdpe_nxt = VTAILQ_NEXT(vdpe, list); + vdc->nxt = VTAILQ_NEXT(vdpe, list); assert(act > VDP_NULL || len > 0); /* Call the present layer, while pointing to the next layer down */ retval = vdpe->vdp->func(req, act, &vdpe->priv, ptr, len); - if (retval && (req->vdpe_retval == 0 || retval < req->vdpe_retval)) - req->vdpe_retval = retval; /* Latch error value */ - req->vdpe_nxt = vdpe; - return (req->vdpe_retval); + if (retval && (vdc->retval == 0 || retval < vdc->retval)) + vdc->retval = retval; /* Latch error value */ + vdc->nxt = vdpe; + return (vdc->retval); } void VDP_push(struct req *req, const struct vdp *vdp, void *priv, int bottom) { struct vdp_entry *vdpe; + struct vdp_ctx *vdc; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); + vdc = req->vdp; AN(vdp); AN(vdp->name); AN(vdp->func); @@ -88,10 +92,10 @@ VDP_push(struct req *req, const struct vdp *vdp, void *priv, int bottom) vdpe->vdp = vdp; vdpe->priv = priv; if (bottom) - VTAILQ_INSERT_TAIL(&req->vdpe, vdpe, list); + VTAILQ_INSERT_TAIL(&vdc->vdp, vdpe, list); else - VTAILQ_INSERT_HEAD(&req->vdpe, vdpe, list); - req->vdpe_nxt = VTAILQ_FIRST(&req->vdpe); + VTAILQ_INSERT_HEAD(&vdc->vdp, vdpe, list); + vdc->nxt = VTAILQ_FIRST(&vdc->vdp); AZ(vdpe->vdp->func(req, VDP_INIT, &vdpe->priv, NULL, 0)); } @@ -100,15 +104,17 @@ void VDP_close(struct req *req) { struct vdp_entry *vdpe; + struct vdp_ctx *vdc; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - while (!VTAILQ_EMPTY(&req->vdpe)) { - vdpe = VTAILQ_FIRST(&req->vdpe); + vdc = req->vdp; + while (!VTAILQ_EMPTY(&vdc->vdp)) { + vdpe = VTAILQ_FIRST(&vdc->vdp); CHECK_OBJ_NOTNULL(vdpe, VDP_ENTRY_MAGIC); - VTAILQ_REMOVE(&req->vdpe, vdpe, list); + VTAILQ_REMOVE(&vdc->vdp, vdpe, list); AZ(vdpe->vdp->func(req, VDP_FINI, &vdpe->priv, NULL, 0)); AZ(vdpe->priv); - req->vdpe_nxt = VTAILQ_FIRST(&req->vdpe); + vdc->nxt = VTAILQ_FIRST(&vdc->vdp); } } diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 28e99e3..2781a87 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -327,7 +327,7 @@ vdp_gunzip(struct req *req, enum vdp_action act, void **priv, * If the size is non-zero AND we are the top * VDP (ie: no ESI), we know what size the output will be. */ - if (u != 0 && VTAILQ_FIRST(&req->vdpe)->vdp == &VDP_gunzip) + if (u != 0 && VTAILQ_FIRST(&req->vdp->vdp)->vdp == &VDP_gunzip) req->resp_len = u; return (0); @@ -355,7 +355,7 @@ vdp_gunzip(struct req *req, enum vdp_action act, void **priv, return (-1); if (vg->m_len == vg->m_sz || vr != VGZ_OK) { if (VDP_bytes(req, VDP_FLUSH, vg->m_buf, vg->m_len)) - return (req->vdpe_retval); + return (req->vdp->retval); vg->m_len = 0; VGZ_Obuf(vg, vg->m_buf, vg->m_sz); } diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index 43581e7..484d6c5 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -137,8 +137,8 @@ Req_New(const struct worker *wrk, struct sess *sp) req->t_prev = NAN; req->t_req = NAN; - req->vdpe_nxt = 0; - VTAILQ_INIT(&req->vdpe); + req->vdp->nxt = 0; + VTAILQ_INIT(&req->vdp->vdp); VRTPRIV_init(req->privs); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index 649b74a..c3b691d 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -790,7 +790,7 @@ cnt_recv(struct worker *wrk, struct req *req) req->director_hint = VCL_DefaultDirector(req->vcl); AN(req->director_hint); - req->vdpe_retval = 0; + req->vdp->retval = 0; req->d_ttl = -1; req->disable_esi = 0; req->hash_always_miss = 0; diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c index 7ed54a6..80e398c 100644 --- a/bin/varnishd/http1/cache_http1_deliver.c +++ b/bin/varnishd/http1/cache_http1_deliver.c @@ -46,7 +46,7 @@ v1d_bytes(struct req *req, enum vdp_action act, void **priv, if (act == VDP_INIT || act == VDP_FINI) return (0); - AZ(req->vdpe_nxt); /* always at the bottom of the pile */ + AZ(req->vdp->nxt); /* always at the bottom of the pile */ if (len > 0) wl = V1L_Write(req->wrk, ptr, len); From phk at FreeBSD.org Mon Oct 30 11:08:08 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 30 Oct 2017 11:08:08 +0000 (UTC) Subject: [master] cd30d11 Move VDP context to cache_filter.h Message-ID: <20171030110808.47DCCA6710@lists.varnish-cache.org> commit cd30d1163b33d907fcab5f8744315d0cbde4ef51 Author: Poul-Henning Kamp Date: Fri Oct 27 07:44:57 2017 +0000 Move VDP context to cache_filter.h diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index b64e320..0e5810b 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -460,8 +460,6 @@ struct busyobj { /*--------------------------------------------------------------------*/ -VTAILQ_HEAD(vdp_entry_s, vdp_entry); - struct req { unsigned magic; #define REQ_MAGIC 0x2751aaa1 @@ -532,11 +530,7 @@ struct req { struct objcore *stale_oc; /* Deliver pipeline */ - struct vdp_ctx { - struct vdp_entry_s vdp; - struct vdp_entry *nxt; - unsigned retval; - } vdp[1]; + struct vdp_ctx *vdc; /* Delivery mode */ unsigned res_mode; diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 2def938..3a5d6ba 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -56,7 +56,7 @@ VDP_bytes(struct req *req, enum vdp_action act, const void *ptr, ssize_t len) struct vdp_ctx *vdc; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - vdc = req->vdp; + vdc = req->vdc; assert(act == VDP_NULL || act == VDP_FLUSH); if (vdc->retval) return (vdc->retval); @@ -80,7 +80,7 @@ VDP_push(struct req *req, const struct vdp *vdp, void *priv, int bottom) struct vdp_ctx *vdc; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - vdc = req->vdp; + vdc = req->vdc; AN(vdp); AN(vdp->name); AN(vdp->func); @@ -107,7 +107,7 @@ VDP_close(struct req *req) struct vdp_ctx *vdc; CHECK_OBJ_NOTNULL(req, REQ_MAGIC); - vdc = req->vdp; + vdc = req->vdc; while (!VTAILQ_EMPTY(&vdc->vdp)) { vdpe = VTAILQ_FIRST(&vdc->vdp); CHECK_OBJ_NOTNULL(vdpe, VDP_ENTRY_MAGIC); diff --git a/bin/varnishd/cache/cache_filter.h b/bin/varnishd/cache/cache_filter.h index 84dfc05..f6a1ffa 100644 --- a/bin/varnishd/cache/cache_filter.h +++ b/bin/varnishd/cache/cache_filter.h @@ -113,5 +113,15 @@ struct vdp_entry { VTAILQ_ENTRY(vdp_entry) list; }; +VTAILQ_HEAD(vdp_entry_s, vdp_entry); + +struct vdp_ctx { + unsigned magic; +#define VDP_CTX_MAGIC 0xee501df7 + struct vdp_entry_s vdp; + struct vdp_entry *nxt; + unsigned retval; +}; + int VDP_bytes(struct req *, enum vdp_action act, const void *ptr, ssize_t len); void VDP_push(struct req *, const struct vdp *, void *priv, int bottom); diff --git a/bin/varnishd/cache/cache_gzip.c b/bin/varnishd/cache/cache_gzip.c index 2781a87..258db64 100644 --- a/bin/varnishd/cache/cache_gzip.c +++ b/bin/varnishd/cache/cache_gzip.c @@ -327,7 +327,7 @@ vdp_gunzip(struct req *req, enum vdp_action act, void **priv, * If the size is non-zero AND we are the top * VDP (ie: no ESI), we know what size the output will be. */ - if (u != 0 && VTAILQ_FIRST(&req->vdp->vdp)->vdp == &VDP_gunzip) + if (u != 0 && VTAILQ_FIRST(&req->vdc->vdp)->vdp == &VDP_gunzip) req->resp_len = u; return (0); @@ -355,7 +355,7 @@ vdp_gunzip(struct req *req, enum vdp_action act, void **priv, return (-1); if (vg->m_len == vg->m_sz || vr != VGZ_OK) { if (VDP_bytes(req, VDP_FLUSH, vg->m_buf, vg->m_len)) - return (req->vdp->retval); + return (req->vdc->retval); vg->m_len = 0; VGZ_Obuf(vg, vg->m_buf, vg->m_sz); } diff --git a/bin/varnishd/cache/cache_req.c b/bin/varnishd/cache/cache_req.c index 484d6c5..2f1014f 100644 --- a/bin/varnishd/cache/cache_req.c +++ b/bin/varnishd/cache/cache_req.c @@ -124,8 +124,13 @@ Req_New(const struct worker *wrk, struct sess *sp) p = (void*)PRNDUP(p); req->vfc = (void*)p; - p += sizeof (*req->vfc); INIT_OBJ(req->vfc, VFP_CTX_MAGIC); + p = (void*)PRNDUP(p + sizeof(*req->vfc)); + + req->vdc = (void*)p; + INIT_OBJ(req->vdc, VDP_CTX_MAGIC); + VTAILQ_INIT(&req->vdc->vdp); + p = (void*)PRNDUP(p + sizeof(*req->vdc)); assert(p < e); @@ -137,11 +142,8 @@ Req_New(const struct worker *wrk, struct sess *sp) req->t_prev = NAN; req->t_req = NAN; - req->vdp->nxt = 0; - VTAILQ_INIT(&req->vdp->vdp); VRTPRIV_init(req->privs); - return (req); } diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index c3b691d..ec0d715 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -790,7 +790,7 @@ cnt_recv(struct worker *wrk, struct req *req) req->director_hint = VCL_DefaultDirector(req->vcl); AN(req->director_hint); - req->vdp->retval = 0; + req->vdc->retval = 0; req->d_ttl = -1; req->disable_esi = 0; req->hash_always_miss = 0; diff --git a/bin/varnishd/http1/cache_http1_deliver.c b/bin/varnishd/http1/cache_http1_deliver.c index 80e398c..9bf5bf8 100644 --- a/bin/varnishd/http1/cache_http1_deliver.c +++ b/bin/varnishd/http1/cache_http1_deliver.c @@ -46,7 +46,7 @@ v1d_bytes(struct req *req, enum vdp_action act, void **priv, if (act == VDP_INIT || act == VDP_FINI) return (0); - AZ(req->vdp->nxt); /* always at the bottom of the pile */ + AZ(req->vdc->nxt); /* always at the bottom of the pile */ if (len > 0) wl = V1L_Write(req->wrk, ptr, len); From phk at FreeBSD.org Mon Oct 30 11:08:08 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Mon, 30 Oct 2017 11:08:08 +0000 (UTC) Subject: [master] eb58d79 More #include polishing Message-ID: <20171030110808.6AC14A6714@lists.varnish-cache.org> commit eb58d799ff68348b9ff35e1313d5fc1703e90697 Author: Poul-Henning Kamp Date: Mon Oct 30 11:07:27 2017 +0000 More #include polishing diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h index 0e5810b..250b0e5 100644 --- a/bin/varnishd/cache/cache.h +++ b/bin/varnishd/cache/cache.h @@ -42,7 +42,6 @@ #include #include "vdef.h" - #include "vrt.h" #define CACHE_H_INCLUDED diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c index 663e7c2..f552b85 100644 --- a/bin/varnishd/cache/cache_acceptor.c +++ b/bin/varnishd/cache/cache_acceptor.c @@ -33,13 +33,13 @@ #include "config.h" -#include "cache_varnishd.h" - #include #include #include #include +#include "cache_varnishd.h" + #include "cache_transport.h" #include "cache_pool.h" #include "common/heritage.h" From nils.goroll at uplex.de Mon Oct 30 12:00:11 2017 From: nils.goroll at uplex.de (Nils Goroll) Date: Mon, 30 Oct 2017 12:00:11 +0000 (UTC) Subject: [master] 26886ef Add some minimal debugging for fetch/deliver processors for a start Message-ID: <20171030120011.65E1CA6AFC@lists.varnish-cache.org> commit 26886ef5c50e6fff0e77cde04ff4671d0d4c54f3 Author: Nils Goroll Date: Mon Oct 30 12:58:26 2017 +0100 Add some minimal debugging for fetch/deliver processors for a start diff --git a/bin/varnishd/cache/cache_deliver_proc.c b/bin/varnishd/cache/cache_deliver_proc.c index 3a5d6ba..2221c89 100644 --- a/bin/varnishd/cache/cache_deliver_proc.c +++ b/bin/varnishd/cache/cache_deliver_proc.c @@ -85,6 +85,9 @@ VDP_push(struct req *req, const struct vdp *vdp, void *priv, int bottom) AN(vdp->name); AN(vdp->func); + if (DO_DEBUG(DBG_PROCESSORS)) + VSLb(req->vsl, SLT_Debug, "VDP_push(%s)", vdp->name); + vdpe = WS_Alloc(req->ws, sizeof *vdpe); if (vdpe == NULL) return; diff --git a/bin/varnishd/cache/cache_fetch_proc.c b/bin/varnishd/cache/cache_fetch_proc.c index 7d9bf5c..84b3abb 100644 --- a/bin/varnishd/cache/cache_fetch_proc.c +++ b/bin/varnishd/cache/cache_fetch_proc.c @@ -141,6 +141,9 @@ VFP_Open(struct vfp_ctx *vc) VTAILQ_FOREACH_REVERSE(vfe, &vc->vfp, vfp_entry_s, list) { if (vfe->vfp->init == NULL) continue; + if (DO_DEBUG(DBG_PROCESSORS)) + VSLb(vc->wrk->vsl, SLT_Debug, "VFP_Open(%s)", + vfe->vfp->name); vfe->closed = vfe->vfp->init(vc, vfe); if (vfe->closed != VFP_OK && vfe->closed != VFP_NULL) { (void)VFP_Error(vc, "Fetch filter %s failed to open", diff --git a/include/tbl/debug_bits.h b/include/tbl/debug_bits.h index 50e4839..3579a5d 100644 --- a/include/tbl/debug_bits.h +++ b/include/tbl/debug_bits.h @@ -48,6 +48,7 @@ DEBUG_BIT(DROP_POOLS, drop_pools, "Drop thread pools (testing)") DEBUG_BIT(SLOW_ACCEPTOR, slow_acceptor, "Slow down Acceptor") DEBUG_BIT(H2_NOCHECK, h2_nocheck, "Disable various H2 checks") DEBUG_BIT(VMOD_SO_KEEP, vmod_so_keep, "Keep copied VMOD libraries") +DEBUG_BIT(PROCESSORS, processors, "Fetch/Deliver processors") #undef DEBUG_BIT /*lint -restore */ From dridi.boukelmoune at gmail.com Mon Oct 30 16:10:09 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 30 Oct 2017 16:10:09 +0000 (UTC) Subject: [master] e097065 Expose the varnishd optstring via -x Message-ID: <20171030161009.0F691A6A75@lists.varnish-cache.org> commit e0970652f9226a5a49887c086812313bd8c72934 Author: Dridi Boukelmoune Date: Mon Sep 25 10:53:08 2017 +0200 Expose the varnishd optstring via -x This helps write scripts like this: optstring=$(varnishd -x optstring) while getopts $optstring opt do case $opt in n) # handle $OPTARG ;; # handle other options *) # ignore unneeded options ;; esac done varnishd "$@" Otherwise if optstring is not kept in sync, getopts will stop processing options if it encounters one that is not specified. diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index aa1bb74..b90e76d 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -74,6 +74,8 @@ static struct vpf_fh *pfh2 = NULL; static struct vfil_path *vcl_path = NULL; static VTAILQ_HEAD(,f_arg) f_args = VTAILQ_HEAD_INITIALIZER(f_args); +static const char opt_spec[] = "a:b:Cdf:Fh:i:I:j:l:M:n:P:p:r:S:s:T:t:VW:x:"; + int optreset; // Some has it, some doesn't. Cheaper than auto* /*--------------------------------------------------------------------*/ @@ -106,6 +108,7 @@ usage(void) printf(FMT, "-x vsl", "VSL record documentation"); printf(FMT, "-x cli", "CLI command documentation"); printf(FMT, "-x builtin", "Builtin VCL program"); + printf(FMT, "-x optstring", "List of getopt options"); printf("\nOperations options:\n"); @@ -297,6 +300,8 @@ mgt_x_arg(const char *x_arg) mgt_DumpRstCli(); else if (!strcmp(x_arg, "builtin")) mgt_DumpBuiltin(); + else if (!strcmp(x_arg, "optstring")) + (void)printf("%s\n", opt_spec); else ARGV_ERR("Invalid -x argument\n"); } @@ -423,8 +428,6 @@ mgt_f_read(const char *fn) VTAILQ_INSERT_TAIL(&f_args, fa, list); } -static const char opt_spec[] = "a:b:Cdf:Fh:i:I:j:l:M:n:P:p:r:S:s:T:t:VW:x:"; - int main(int argc, char * const *argv) { diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 7c31eea..90ab324 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -17,7 +17,7 @@ SYNOPSIS varnishd [-a [name=][address][:port][,PROTO]] [-b host[:port]] [-C] [-d] [-F] [-f config] [-h type[,options]] [-I clifile] [-i identity] [-j jail[,jailoptions]] [-l vsl[,vsm]] [-M address:port] [-n name] [-P file] [-p param=value] [-r param[,param...]] [-S secret-file] [-s [name=]kind[,options]] [-T address[:port]] [-t TTL] [-V] [-W waiter] -varnishd [-x parameter|vsl|cli|builtin] +varnishd [-x parameter|vsl|cli|builtin|optstring] varnishd [-?] @@ -123,6 +123,11 @@ outputs documentation in reStructuredText, aka RST). Print the contents of the default VCL program ``builtin.vcl``. +-x optstring + + Print the optstring parameter to ``getopt(3)`` to help writing + wrapper scripts. + Operations options ------------------ From dridi.boukelmoune at gmail.com Mon Oct 30 16:10:09 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 30 Oct 2017 16:10:09 +0000 (UTC) Subject: [master] 3e10a62 Add a --optstring long options to VUTs Message-ID: <20171030161009.28457A6A77@lists.varnish-cache.org> commit 3e10a62331ff854715f664c18ca1901205df3915 Author: Dridi Boukelmoune Date: Tue Sep 26 14:59:09 2017 +0200 Add a --optstring long options to VUTs This is similar to `varnishd -x optstring`. Refs dc5fe3b diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c index 1438195..61b5a2f 100644 --- a/lib/libvarnishapi/vut.c +++ b/lib/libvarnishapi/vut.c @@ -212,6 +212,10 @@ VUT_Init(const char *progname, int argc, char * const *argv, exit(vut_synopsis(voc)); if (argc == 2 && !strcmp(argv[1], "--options")) exit(vut_options(voc)); + if (argc == 2 && !strcmp(argv[1], "--optstring")) { + (void)printf("%s\n", voc->vopt_optstring); + exit(0); + } vut->progname = progname; vut->g_arg = VSL_g_vxid; From dridi.boukelmoune at gmail.com Mon Oct 30 16:10:09 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 30 Oct 2017 16:10:09 +0000 (UTC) Subject: [master] ecb6e71 Remove stale reference to cli_buffer Message-ID: <20171030161009.4F617A6A7B@lists.varnish-cache.org> commit ecb6e716530e7f0d76fb86d29ceb5f83920be225 Author: Dridi Boukelmoune Date: Mon Oct 9 18:07:46 2017 +0200 Remove stale reference to cli_buffer diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst index 37d4760..5338767 100644 --- a/doc/sphinx/reference/varnish-cli.rst +++ b/doc/sphinx/reference/varnish-cli.rst @@ -90,10 +90,6 @@ Commands may not end with a newline when a shell-style *here document* naturally in the following *here document*. Traditionally EOF or END is used. -When using the here document style of input there are no restrictions -on length. When using newline-terminated commands maximum length is -limited by the varnishd parameter *cli_buffer*. - Quoting pitfalls ---------------- From dridi.boukelmoune at gmail.com Mon Oct 30 16:10:09 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 30 Oct 2017 16:10:09 +0000 (UTC) Subject: [master] 83149f1 Use the correct syntax for RST ``literals`` Message-ID: <20171030161009.73B0EA6A7F@lists.varnish-cache.org> commit 83149f12c9fe05d04fd219824ccd21d5d05e2726 Author: Dridi Boukelmoune Date: Mon Oct 9 18:30:53 2017 +0200 Use the correct syntax for RST ``literals`` diff --git a/doc/sphinx/users-guide/run_cli.rst b/doc/sphinx/users-guide/run_cli.rst index 40c6b4e..a4830b9 100644 --- a/doc/sphinx/users-guide/run_cli.rst +++ b/doc/sphinx/users-guide/run_cli.rst @@ -3,46 +3,46 @@ CLI - bossing Varnish around ============================ -Once `varnishd` is started, you can control it using the command line +Once ``varnishd`` is started, you can control it using the command line interface. -The easiest way to do this, is using `varnishadm` on the -same machine as `varnishd` is running:: +The easiest way to do this, is using ``varnishadm`` on the +same machine as ``varnishd`` is running:: varnishadm help -If you want to run `varnishadm` from a remote system, you can do it +If you want to run ``varnishadm`` from a remote system, you can do it two ways. -You can SSH into the `varnishd` computer and run `varnishadm`:: +You can SSH into the ``varnishd`` computer and run ``varnishadm``:: ssh $http_front_end varnishadm help -But you can also configure `varnishd` to accept remote CLI connections +But you can also configure ``varnishd`` to accept remote CLI connections (using the '-T' and '-S' arguments):: varnishd -T :6082 -S /etc/varnish_secret -And then on the remote system run `varnishadm`:: +And then on the remote system run ``varnishadm``:: varnishadm -T $http_front_end -S /etc/copy_of_varnish_secret help but as you can see, SSH is much more convenient. -If you run `varnishadm` without arguments, it will read CLI commands from -`stdin`, if you give it arguments, it will treat those as the single +If you run ``varnishadm`` without arguments, it will read CLI commands from +``stdin``, if you give it arguments, it will treat those as the single CLI command to execute. The CLI always returns a status code to tell how it went: '200' means OK, anything else means there were some kind of trouble. -`varnishadm` will exit with status 1 and print the status code on +``varnishadm`` will exit with status 1 and print the status code on standard error if it is not 200. What can you do with the CLI ---------------------------- -The CLI gives you almost total control over `varnishd` some of the more important tasks you can perform are: +The CLI gives you almost total control over ``varnishd`` some of the more important tasks you can perform are: * load/use/discard VCL programs * ban (invalidate) cache content @@ -166,7 +166,7 @@ and:: varnish> start -If you start `varnishd` with the '-d' (debugging) argument, you will +If you start ``varnishd`` with the '-d' (debugging) argument, you will always need to start the child process explicitly. Should the child process die, the master process will automatically From dridi.boukelmoune at gmail.com Mon Oct 30 16:10:09 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 30 Oct 2017 16:10:09 +0000 (UTC) Subject: [master] 1908ea2 Don't break paragraph with a comment Message-ID: <20171030161009.99078A6A83@lists.varnish-cache.org> commit 1908ea23499d5048efa87925f83432e6aefe01a1 Author: Dridi Boukelmoune Date: Mon Oct 9 18:33:13 2017 +0200 Don't break paragraph with a comment diff --git a/doc/sphinx/users-guide/run_cli.rst b/doc/sphinx/users-guide/run_cli.rst index a4830b9..d2789db 100644 --- a/doc/sphinx/users-guide/run_cli.rst +++ b/doc/sphinx/users-guide/run_cli.rst @@ -145,12 +145,10 @@ but they can also be examined and changed on the fly from the CLI:: In general it is not a good idea to modify parameters unless you have a good reason, such as performance tuning or security configuration. -Most parameters will take effect instantly, or with a natural delay -of some duration, - .. XXX: Natural delay of some duration sounds vague. benc -but a few of them requires you to restart the +Most parameters will take effect instantly, or with a natural delay +of some duration, but a few of them requires you to restart the child process before they take effect. This is always noted in the description of the parameter. From dridi.boukelmoune at gmail.com Mon Oct 30 16:10:09 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 30 Oct 2017 16:10:09 +0000 (UTC) Subject: [master] 39741c0 Document `-x optstring` and `--optstring` Message-ID: <20171030161009.B7A7FA6A89@lists.varnish-cache.org> commit 39741c03d0d6a1a0778591076994390c74a9fee1 Author: Dridi Boukelmoune Date: Mon Oct 9 19:06:02 2017 +0200 Document `-x optstring` and `--optstring` diff --git a/doc/sphinx/users-guide/run_cli.rst b/doc/sphinx/users-guide/run_cli.rst index d2789db..e0c23e3 100644 --- a/doc/sphinx/users-guide/run_cli.rst +++ b/doc/sphinx/users-guide/run_cli.rst @@ -169,3 +169,47 @@ always need to start the child process explicitly. Should the child process die, the master process will automatically restart it, but you can disable that with the 'auto_restart' parameter. + +The shell, the other CLI +------------------------ + +Besides accessing the CLI via its interface or via ``varnishadm`` there +is the matter of actually running the ``varnishd`` command line, usually +via a shell. See :ref:`run_security` for security concerns around the +``varnishd`` command line. See also :ref:`ref_syntax` about the CLI +syntax and quoting pitfalls when using ``varnishadm``. + +The programs shipped with Varnish can expose their *optstring* in order +to help writing wrapper scripts, in particular to get an opportunity to +hook a task before a program daemonizes. With the exception of +``varnishtest`` and ``varnishadm``, you can write Shell wrappers for +``varnishd`` using the ``-x`` option and other programs using the +``--optstring`` long option. + +This way, when writing a wrapper script you don't need to maintain the +*optstring* in sync when you only need a subset of the options, usually +``-n`` or ``-P``:: + + optstring=$(varnishd -x optstring) + + while getopts "$optstring" opt + do + case $opt in + n) + # handle $OPTARG + ;; + # handle other options + *) + # ignore unneeded options + ;; + esac + done + + varnishd "$@" + + # do something with the options + +You can for example write a wrapper script that blocks until the shared +memory is ready or when the child is started if you need that kind of +synchronization. You can also prevent ``varnishd`` from starting if the +``-S`` option is inadvertently set to not challenge access to the CLI. From dridi.boukelmoune at gmail.com Mon Oct 30 16:10:09 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Mon, 30 Oct 2017 16:10:09 +0000 (UTC) Subject: [master] 1eabc6d Emit --optstring documentation in VUT's --options Message-ID: <20171030161009.DB540A6A99@lists.varnish-cache.org> commit 1eabc6d724ab494276615120817f36432d098341 Author: Dridi Boukelmoune Date: Mon Oct 9 19:06:28 2017 +0200 Emit --optstring documentation in VUT's --options diff --git a/lib/libvarnishapi/vut.c b/lib/libvarnishapi/vut.c index 61b5a2f..46b2aa1 100644 --- a/lib/libvarnishapi/vut.c +++ b/lib/libvarnishapi/vut.c @@ -489,5 +489,8 @@ vut_options(const struct vopt_spec *voc) for (i = 0; i < voc->vopt_list_n; i++) print_opt(&voc->vopt_list[i]); + printf("--optstring\n" + "\tPrint the optstring parameter to ``getopt(3)`` to help" + " writing wrapper scripts.\n\n"); return (0); } From guillaume at varnish-software.com Mon Oct 30 17:38:09 2017 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Mon, 30 Oct 2017 17:38:09 +0000 (UTC) Subject: [master] 83425a0 [doc] default,256m -> malloc,256m Message-ID: <20171030173809.66018A6D3A@lists.varnish-cache.org> commit 83425a059ac9a292b8909a2d3a58aa2c35c5d769 Author: Guillaume Quintard Date: Mon Oct 30 18:12:24 2017 +0100 [doc] default,256m -> malloc,256m diff --git a/doc/sphinx/tutorial/putting_varnish_on_port_80.rst b/doc/sphinx/tutorial/putting_varnish_on_port_80.rst index b1c254b..16d38bc 100644 --- a/doc/sphinx/tutorial/putting_varnish_on_port_80.rst +++ b/doc/sphinx/tutorial/putting_varnish_on_port_80.rst @@ -19,7 +19,7 @@ some text that looks like this:: -T localhost:6082 \ -f /etc/varnish/default.vcl \ -S /etc/varnish/secret \ - -s default,256m" + -s malloc,256m" Change it to:: @@ -27,7 +27,7 @@ Change it to:: -T localhost:6082 \ -f /etc/varnish/default.vcl \ -S /etc/varnish/secret \ - -s default,256m" + -s malloc,256m" Debian (v8+) / Ubuntu (v15.04+) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -40,7 +40,7 @@ Applying changes to the default service is best done by creating a new file [Service] ExecStart= - ExecStart=/usr/sbin/varnishd -a :80 -T localhost:6082 -f /etc/varnish/default.vcl -S /etc/varnish/secret -s default,256m + ExecStart=/usr/sbin/varnishd -a :80 -T localhost:6082 -f /etc/varnish/default.vcl -S /etc/varnish/secret -s malloc,256m This will override the ExecStart part of the default configuration shipped with Varnish Cache. From guillaume at varnish-software.com Mon Oct 30 17:45:06 2017 From: guillaume at varnish-software.com (Guillaume Quintard) Date: Mon, 30 Oct 2017 17:45:06 +0000 (UTC) Subject: [master] 65b82a9 Revert "[doc] default,256m -> malloc,256m" Message-ID: <20171030174507.0B657A6030@lists.varnish-cache.org> commit 65b82a960e0812655e935fcd9360d373b4bbf885 Author: Guillaume Quintard Date: Mon Oct 30 18:44:32 2017 +0100 Revert "[doc] default,256m -> malloc,256m" This reverts commit 83425a059ac9a292b8909a2d3a58aa2c35c5d769. diff --git a/doc/sphinx/tutorial/putting_varnish_on_port_80.rst b/doc/sphinx/tutorial/putting_varnish_on_port_80.rst index 16d38bc..b1c254b 100644 --- a/doc/sphinx/tutorial/putting_varnish_on_port_80.rst +++ b/doc/sphinx/tutorial/putting_varnish_on_port_80.rst @@ -19,7 +19,7 @@ some text that looks like this:: -T localhost:6082 \ -f /etc/varnish/default.vcl \ -S /etc/varnish/secret \ - -s malloc,256m" + -s default,256m" Change it to:: @@ -27,7 +27,7 @@ Change it to:: -T localhost:6082 \ -f /etc/varnish/default.vcl \ -S /etc/varnish/secret \ - -s malloc,256m" + -s default,256m" Debian (v8+) / Ubuntu (v15.04+) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -40,7 +40,7 @@ Applying changes to the default service is best done by creating a new file [Service] ExecStart= - ExecStart=/usr/sbin/varnishd -a :80 -T localhost:6082 -f /etc/varnish/default.vcl -S /etc/varnish/secret -s malloc,256m + ExecStart=/usr/sbin/varnishd -a :80 -T localhost:6082 -f /etc/varnish/default.vcl -S /etc/varnish/secret -s default,256m This will override the ExecStart part of the default configuration shipped with Varnish Cache. From phk at FreeBSD.org Tue Oct 31 09:20:10 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 31 Oct 2017 09:20:10 +0000 (UTC) Subject: [master] f522efc Take common_params out of PACKAGE ABI. Message-ID: <20171031092010.59F60A64E4@lists.varnish-cache.org> commit f522efc3aa7020c63f6e1226cc2f51ea6a53bfe7 Author: Poul-Henning Kamp Date: Tue Oct 31 09:18:20 2017 +0000 Take common_params out of PACKAGE ABI. Sort various lists diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am index 87f7f82..a652cc9 100644 --- a/bin/varnishd/Makefile.am +++ b/bin/varnishd/Makefile.am @@ -35,11 +35,11 @@ varnishd_SOURCES = \ cache/cache_obj.c \ cache/cache_panic.c \ cache/cache_pool.c \ + cache/cache_range.c \ cache/cache_req.c \ cache/cache_req_body.c \ cache/cache_req_fsm.c \ cache/cache_rfc2616.c \ - cache/cache_range.c \ cache/cache_session.c \ cache/cache_shmlog.c \ cache/cache_tcp_pool.c \ @@ -55,10 +55,10 @@ varnishd_SOURCES = \ common/common_vsc.c \ hash/hash_classic.c \ hash/hash_critbit.c \ - hash/mgt_hash.c \ hash/hash_simple_list.c \ - hpack/vhp_table.c \ + hash/mgt_hash.c \ hpack/vhp_decode.c \ + hpack/vhp_table.c \ http1/cache_http1_deliver.c \ http1/cache_http1_fetch.c \ http1/cache_http1_fsm.c \ @@ -70,18 +70,18 @@ varnishd_SOURCES = \ http2/cache_http2_hpack.c \ http2/cache_http2_panic.c \ http2/cache_http2_proto.c \ - http2/cache_http2_session.c \ http2/cache_http2_send.c \ + http2/cache_http2_session.c \ mgt/mgt_acceptor.c \ mgt/mgt_child.c \ mgt/mgt_cli.c \ mgt/mgt_jail.c \ - mgt/mgt_jail_unix.c \ mgt/mgt_jail_solaris.c \ + mgt/mgt_jail_unix.c \ mgt/mgt_main.c \ mgt/mgt_param.c \ - mgt/mgt_param_tbl.c \ mgt/mgt_param_bits.c \ + mgt/mgt_param_tbl.c \ mgt/mgt_param_tcp.c \ mgt/mgt_param_tweak.c \ mgt/mgt_pool.c \ @@ -90,24 +90,24 @@ varnishd_SOURCES = \ mgt/mgt_vcc.c \ mgt/mgt_vcl.c \ proxy/cache_proxy_proto.c \ - storage/stevedore.c \ storage/mgt_stevedore.c \ + storage/mgt_storage_persistent.c \ + storage/stevedore.c \ storage/stevedore_utils.c \ storage/storage_file.c \ storage/storage_lru.c \ storage/storage_malloc.c \ storage/storage_persistent.c \ - storage/mgt_storage_persistent.c \ storage/storage_persistent_silo.c \ storage/storage_persistent_subr.c \ storage/storage_simple.c \ storage/storage_umem.c \ - waiter/mgt_waiter.c \ waiter/cache_waiter.c \ waiter/cache_waiter_epoll.c \ waiter/cache_waiter_kqueue.c \ waiter/cache_waiter_poll.c \ - waiter/cache_waiter_ports.c + waiter/cache_waiter_ports.c \ + waiter/mgt_waiter.c nodist_varnishd_SOURCES = \ builtin_vcl.c @@ -122,6 +122,7 @@ noinst_HEADERS = \ cache/cache_transport.h \ cache/cache_varnishd.h \ cache/cache_vgz.h \ + common/common_param.h \ common/heritage.h \ hash/hash_slinger.h \ hpack/vhp.h \ @@ -130,18 +131,17 @@ noinst_HEADERS = \ mgt/mgt.h \ mgt/mgt_param.h \ storage/storage.h \ - storage/storage_simple.h \ storage/storage_persistent.h \ - waiter/waiter_priv.h \ - waiter/mgt_waiter.h + storage/storage_simple.h \ + waiter/mgt_waiter.h \ + waiter/waiter_priv.h # Headers for use with vmods nobase_pkginclude_HEADERS = \ cache/cache.h \ - cache/cache_filter.h \ cache/cache_backend.h \ cache/cache_director.h \ - common/common_param.h \ + cache/cache_filter.h \ waiter/waiter.h pkgdatadir = ${datarootdir}/${PACKAGE}/vcl @@ -222,14 +222,14 @@ DISTCLEANFILES += vhp_hufdec.h $(PYTHON) $(top_srcdir)/lib/libvcc/vsctool.py -ch $< VSC_SRC = \ - VSC_main.vsc \ VSC_lck.vsc \ - VSC_vbe.vsc \ + VSC_main.vsc \ + VSC_mempool.vsc \ + VSC_mgt.vsc \ VSC_sma.vsc \ - VSC_smu.vsc \ VSC_smf.vsc \ - VSC_mempool.vsc \ - VSC_mgt.vsc + VSC_smu.vsc \ + VSC_vbe.vsc VSC_GEN_C = @VSC_GEN_C@ VSC_GEN_H = @VSC_GEN_H@ From phk at FreeBSD.org Tue Oct 31 09:20:10 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 31 Oct 2017 09:20:10 +0000 (UTC) Subject: [master] bb70950 Add info about namespace/api management Message-ID: <20171031092010.648C8A64E6@lists.varnish-cache.org> commit bb70950cb77b6db4b0c12be0118a06017d2d35d2 Author: Poul-Henning Kamp Date: Tue Oct 31 09:18:42 2017 +0000 Add info about namespace/api management diff --git a/doc/sphinx/phk/apispaces.rst b/doc/sphinx/phk/apispaces.rst new file mode 100644 index 0000000..d561a14 --- /dev/null +++ b/doc/sphinx/phk/apispaces.rst @@ -0,0 +1,161 @@ +.. _phk_api_spaces: + +API spaces +========== + +The reason you cant remember hearing about "API spaces" at university +is that I just made that concept up, as a title for this piece. + +We need a name for the collision where APIs meet namespaces. + +At some point in their career, most C programmers learn that ``j0``, +``j1``, ``y0`` and ``y1`` are names to avoid whereas ``j2`` and +``y2`` up to, but not including ``jn`` and ``yn`` are OK. + +The reason is that somebody back when I was a child thought it would +be really neat if the math library supported Bessel functions, +without thinking about ```` as an API which had to coexist +in the flat namespace of the C language along many other APIs. + +One of the big attractions of Object Oriented programming is that +it solves exactly that problem: +Nobody is confused about ``car->push()`` and ``stack->push()``. + +But Varnish is written in C which has a flat namespace and we must +live with it. + +From the very start, we defined cadastral boundaries in the flat +namespace by assigning VTLA prefixes to various chunks of code. + +``VSB_something`` has to do with the sbufs we adopted from FreeBSD, +``VGC_something`` is Vcc Generated C-source and so on. + +Mostly we have stuck with the 'V' prefix, which for some reason +is almost unused everywhere else, but we also have prominent +exceptions. ``WS_something`` for workspaces for instance. + +As long as all the C-code was in-project, inconsistencies and +the precise location of function prototypes didn't matter much, +it was "just something you had to know". + +Now that we have VMODs, and even more so, now that we want to provide +some semblance of API stability for VMODs, we have a lot of sorting +and some renaming to do, in order to clearly delineate APIs within +our flat namespace and our include files. + +Frederick P. Brooks points out in his classic "The Mythical Man-Month", +that is the difference between a program-product and a programming-product, +and he makes the case that the effort required tripples, going from +the former to the latter. + +Having spent some weeks on what I thought would be a three day task +I suspect that his was an underestimate. + +I will now try to lay out what I think will be our policy on APIs +and name-space sharing going forward, but please understand that +this is mostly just an aspirational goal at this point. + +General namespace rules +----------------------- + +1. Each API or otherwise isolated section of code gets a unique + prefix, ending with an underscore, (``VSB_``, ``V1F_`` etc.) + +2. Public symbols has upper case prefix. + +3. Private symbols use prefix in lower case, both as ``static`` + symbols in source files, and when exposed to other source + files in the same section of code. + +4. Friends-With-Benefit symbols have an additional underscore + after the prefix: ``FOO__bar()`` and are only to be used with + explicit permission, which should be clearly documented in + the relevant #include file. + +VMOD API/ABI levels +------------------- + +Vmods can be written against one of three API/ABI levels, called +respectively ``VRT``, ``PACKAGE`` and ``SOURCE``, defined in +detail below. + +A VMOD which restricts itself to the ``VRT`` API/ABI gets maximum +stability and will, we hope, work without recompilation across +many major and minor releases of Varnish. + +A VMOD which uses the ``PACKAGE`` API, will likely keep working +across minor releases of varnish releases, but will usually +need to be recompiled for new major releases of varnish. + +A VMOD which uses the ``SOURCE`` API is compiled against one +specific version of Varnish, and will not work with another +version until recompiled. + +The VMOD VRT API/ABI +-------------------- + +This API space could also have been called 'inline', because it +is basically what you see in the C-source generated by VCC: + +| Include files allowed: +| +| ``#include "vdef.h"`` +| ``#include "vrt.h"`` +| ``#include "vrt_obj.h"`` +| ``#include "vcl.h"`` + +Any private and Friends-With-Benefits symbols are off-limits +to VMODs, (it is usually stuff VCC needs for the compiled +code, and likely as not, you will crash if you mess with it.) + +The ``"vrt.h"`` contains two #defines which together defines +the level of this API: + +| ``#define VRT_MAJOR_VERSION 6U`` +| ``#define VRT_MINOR_VERSION 2U`` + +A snapshot of these will be automatically compiled into the +VMOD shared library, and they will be checked for compatibility +when the VMOD is imported by the VCL compiler. + +The VMOD PACKAGE API/ABI +------------------------ + +This API space provides access to everything in the ``VRT`` API +space plus the other exposed and supported APIs in varnishd. + +| Include files allowed: +| +| ``#include "cache.h" // NB: includes vdef.h and vrt.h`` +| ``#include "cache_backend.h"`` +| ``#include "cache_director.h"`` +| ``#include "cache_filter.h"`` +| ``#include "waiter/waiter.h"`` + +Any private and Friends-With-Benefits symbols are off-limits +to VMODs. + +In addition to the two-part VRT version, ``"cache.h"`` will +contain two #defines for levels of this API. + +| ``#define PACKAGE_MAJOR_VERSION 1U`` +| ``#define PACKAGE MINOR_VERSION 3U`` + +Compile-time snapshots of these will be checked, along with +their VRT cousins be checked for compatibility on VMOD import. + +The VMOD SOURCE API/ABI +----------------------- + +This API space provides access to private parts of varnishd and its +use is highly discouraged, unless you absolutely have to, + +You can #include any file from the varnish source tree and use +anything you find in them - but don't come crying to us if it +all ends in tears: No refunds at this window. + +A hash value of all the .h files in the source tree will be +compiled into the VMOD and will be checked to match exactly +on VMOD import. + +*phk* diff --git a/doc/sphinx/phk/index.rst b/doc/sphinx/phk/index.rst index 8f21795..7962d9e 100644 --- a/doc/sphinx/phk/index.rst +++ b/doc/sphinx/phk/index.rst @@ -8,6 +8,7 @@ You may or may not want to know what Poul-Henning thinks. .. toctree:: :maxdepth: 1 + apispaces.rst VSV00001.rst somethinghappened.rst trialerror.rst From dridi.boukelmoune at gmail.com Tue Oct 31 09:53:05 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 31 Oct 2017 09:53:05 +0000 (UTC) Subject: [4.1] ff50855 Expose the varnishd optstring via -x Message-ID: <20171031095305.A81F1A61AD@lists.varnish-cache.org> commit ff50855279e06808b09508348332bf18751312d1 Author: Dridi Boukelmoune Date: Mon Sep 25 10:53:08 2017 +0200 Expose the varnishd optstring via -x This helps write scripts like this: optstring=$(varnishd -x optstring) while getopts $optstring opt do case $opt in n) # handle $OPTARG ;; # handle other options *) # ignore unneeded options ;; esac done varnishd "$@" Otherwise if optstring is not kept in sync, getopts will stop processing options if it encounters one that is not specified. diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index 611695f..368c6f4 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -74,6 +74,8 @@ struct VSC_C_mgt *VSC_C_mgt; static struct vpf_fh *pfh = NULL; +static const char opt_spec[] = "a:b:Cdf:Fh:i:j:l:M:n:P:p:r:S:s:T:t:VW:x:"; + /*--------------------------------------------------------------------*/ static void @@ -255,6 +257,7 @@ usage(void) fprintf(stderr, FMT, "", " -W epoll"); #endif fprintf(stderr, FMT, "", " -W poll"); + fprintf(stderr, FMT, "-x optstring", "List of getopt options"); #undef FMT exit(1); @@ -466,7 +469,6 @@ init_params(struct cli *cli) MCF_InitParams(cli); } - /*--------------------------------------------------------------------*/ static void @@ -571,8 +573,7 @@ main(int argc, char * const *argv) init_params(cli); cli_check(cli); - while ((o = getopt(argc, argv, - "a:b:Cdf:Fh:i:j:l:M:n:P:p:r:S:s:T:t:VW:x:")) != -1) { + while ((o = getopt(argc, argv, opt_spec)) != -1) { /* * -j must be the first argument if specified, because * it (may) affect subsequent argument processing. @@ -681,6 +682,10 @@ main(int argc, char * const *argv) mgt_DumpRstVsl(); exit(0); } + if (!strcmp(optarg, "optstring")) { + printf("%s\n", opt_spec); + exit(0); + } usage(); break; default: diff --git a/doc/sphinx/reference/varnishd.rst b/doc/sphinx/reference/varnishd.rst index 4506a63..f1aa09c 100644 --- a/doc/sphinx/reference/varnishd.rst +++ b/doc/sphinx/reference/varnishd.rst @@ -154,6 +154,11 @@ OPTIONS Specifies the waiter type to use. +-x optstring + + Print the optstring parameter to ``getopt(3)`` to help writing + wrapper scripts. + .. _opt_h: Hash Algorithm From dridi.boukelmoune at gmail.com Tue Oct 31 09:53:05 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 31 Oct 2017 09:53:05 +0000 (UTC) Subject: [4.1] 67d56d1 Remove stale reference to cli_buffer Message-ID: <20171031095305.BC18EA61AF@lists.varnish-cache.org> commit 67d56d1729d3fd5a31390bc350207c90855d2b31 Author: Dridi Boukelmoune Date: Mon Oct 9 18:07:46 2017 +0200 Remove stale reference to cli_buffer diff --git a/doc/sphinx/reference/varnish-cli.rst b/doc/sphinx/reference/varnish-cli.rst index 023c7aa..55f1efc 100644 --- a/doc/sphinx/reference/varnish-cli.rst +++ b/doc/sphinx/reference/varnish-cli.rst @@ -89,10 +89,6 @@ Commands may not end with a newline when a shell-style *here document* naturally in the following *here document*. Traditionally EOF or END is used. -When using the here document style of input there are no restrictions -on length. When using newline-terminated commands maximum length is -limited by the varnishd parameter *cli_buffer*. - Quoting pitfalls ---------------- From daghf at varnish-software.com Tue Oct 31 10:34:05 2017 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Tue, 31 Oct 2017 10:34:05 +0000 (UTC) Subject: [master] 6299828 Add a V-prefix to the SHA256 functions Message-ID: <20171031103405.CAF31A601E@lists.varnish-cache.org> commit 6299828f879379d0cf10ab833b24a22f6517f8cd Author: Dag Haavi Finstad Date: Tue Oct 31 11:31:09 2017 +0100 Add a V-prefix to the SHA256 functions diff --git a/bin/varnishd/cache/cache_hash.c b/bin/varnishd/cache/cache_hash.c index 004b86d..2aac129 100644 --- a/bin/varnishd/cache/cache_hash.c +++ b/bin/varnishd/cache/cache_hash.c @@ -177,10 +177,10 @@ HSH_AddString(struct req *req, void *ctx, const char *str) CHECK_OBJ_NOTNULL(req, REQ_MAGIC); AN(ctx); if (str != NULL) { - SHA256_Update(ctx, str, strlen(str)); + VSHA256_Update(ctx, str, strlen(str)); VSLb(req->vsl, SLT_Hash, "%s", str); } else - SHA256_Update(ctx, &str, 1); + VSHA256_Update(ctx, &str, 1); } /*--------------------------------------------------------------------- @@ -191,8 +191,8 @@ HSH_AddString(struct req *req, void *ctx, const char *str) */ static struct hsh_magiclist { - unsigned char was[SHA256_LEN]; - unsigned char now[SHA256_LEN]; + unsigned char was[VSHA256_LEN]; + unsigned char now[VSHA256_LEN]; } hsh_magiclist[] = { { .now = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -241,19 +241,19 @@ hsh_testmagic(void *result) static int nused = 0; for (i = 0; i < nused; i++) - if (!memcmp(hsh_magiclist[i].was, result, SHA256_LEN)) + if (!memcmp(hsh_magiclist[i].was, result, VSHA256_LEN)) break; if (i == nused && i < HSH_NMAGIC) - memcpy(hsh_magiclist[nused++].was, result, SHA256_LEN); + memcpy(hsh_magiclist[nused++].was, result, VSHA256_LEN); if (i == nused) return; assert(i < HSH_NMAGIC); fprintf(stderr, "HASHMAGIC: <"); - for (j = 0; j < SHA256_LEN; j++) + for (j = 0; j < VSHA256_LEN; j++) fprintf(stderr, "%02x", ((unsigned char*)result)[j]); fprintf(stderr, "> -> <"); - memcpy(result, hsh_magiclist[i].now, SHA256_LEN); - for (j = 0; j < SHA256_LEN; j++) + memcpy(result, hsh_magiclist[i].now, VSHA256_LEN); + for (j = 0; j < VSHA256_LEN; j++) fprintf(stderr, "%02x", ((unsigned char*)result)[j]); fprintf(stderr, ">\n"); } @@ -978,7 +978,7 @@ void HSH_Init(const struct hash_slinger *slinger) { - assert(DIGEST_LEN == SHA256_LEN); /* avoid #include pollution */ + assert(DIGEST_LEN == VSHA256_LEN); /* avoid #include pollution */ hash = slinger; if (hash->start != NULL) hash->start(); diff --git a/bin/varnishd/cache/cache_req_fsm.c b/bin/varnishd/cache/cache_req_fsm.c index ec0d715..f0f56cc 100644 --- a/bin/varnishd/cache/cache_req_fsm.c +++ b/bin/varnishd/cache/cache_req_fsm.c @@ -750,7 +750,7 @@ static enum req_fsm_nxt cnt_recv(struct worker *wrk, struct req *req) { unsigned recv_handling; - struct SHA256Context sha256ctx; + struct VSHA256Context sha256ctx; const char *xff; const char *ci, *cp; @@ -841,13 +841,13 @@ cnt_recv(struct worker *wrk, struct req *req) } } - SHA256_Init(&sha256ctx); + VSHA256_Init(&sha256ctx); VCL_hash_method(req->vcl, wrk, req, NULL, &sha256ctx); if (wrk->handling == VCL_RET_FAIL) recv_handling = wrk->handling; else assert(wrk->handling == VCL_RET_LOOKUP); - SHA256_Final(req->digest, &sha256ctx); + VSHA256_Final(req->digest, &sha256ctx); switch (recv_handling) { case VCL_RET_VCL: diff --git a/bin/varnishd/mgt/mgt_main.c b/bin/varnishd/mgt/mgt_main.c index b90e76d..39f1059 100644 --- a/bin/varnishd/mgt/mgt_main.c +++ b/bin/varnishd/mgt/mgt_main.c @@ -264,8 +264,8 @@ mgt_tests(void) assert(VTIM_parse("Sunday, 06-Nov-94 08:49:37 GMT") == 784111777); assert(VTIM_parse("Sun Nov 6 08:49:37 1994") == 784111777); - /* Check that our SHA256 works */ - SHA256_Test(); + /* Check that our VSHA256 works */ + VSHA256_Test(); } static void diff --git a/bin/varnishd/storage/storage_persistent.h b/bin/varnishd/storage/storage_persistent.h index 0383567..7fe612f 100644 --- a/bin/varnishd/storage/storage_persistent.h +++ b/bin/varnishd/storage/storage_persistent.h @@ -122,7 +122,7 @@ struct smp_sign { uint64_t length; /* NB: Must be last */ }; -#define SMP_SIGN_SPACE (sizeof(struct smp_sign) + SHA256_LEN) +#define SMP_SIGN_SPACE (sizeof(struct smp_sign) + VSHA256_LEN) /* * A segment pointer. @@ -168,7 +168,7 @@ struct smp_object { struct smp_signctx { struct smp_sign *ss; - struct SHA256Context ctx; + struct VSHA256Context ctx; uint32_t unique; const char *id; }; diff --git a/bin/varnishd/storage/storage_persistent_subr.c b/bin/varnishd/storage/storage_persistent_subr.c index 2e15c3f..05ce659 100644 --- a/bin/varnishd/storage/storage_persistent_subr.c +++ b/bin/varnishd/storage/storage_persistent_subr.c @@ -84,8 +84,8 @@ smp_def_sign(const struct smp_sc *sc, struct smp_signctx *ctx, int smp_chk_sign(struct smp_signctx *ctx) { - struct SHA256Context cx; - unsigned char sign[SHA256_LEN]; + struct VSHA256Context cx; + unsigned char sign[VSHA256_LEN]; int r = 0; if (strncmp(ctx->id, ctx->ss->ident, sizeof ctx->ss->ident)) @@ -95,13 +95,13 @@ smp_chk_sign(struct smp_signctx *ctx) else if ((uintptr_t)ctx->ss != ctx->ss->mapped) r = 3; else { - SHA256_Init(&ctx->ctx); - SHA256_Update(&ctx->ctx, ctx->ss, + VSHA256_Init(&ctx->ctx); + VSHA256_Update(&ctx->ctx, ctx->ss, offsetof(struct smp_sign, length)); - SHA256_Update(&ctx->ctx, SIGN_DATA(ctx), ctx->ss->length); + VSHA256_Update(&ctx->ctx, SIGN_DATA(ctx), ctx->ss->length); cx = ctx->ctx; - SHA256_Update(&cx, &ctx->ss->length, sizeof(ctx->ss->length)); - SHA256_Final(sign, &cx); + VSHA256_Update(&cx, &ctx->ss->length, sizeof(ctx->ss->length)); + VSHA256_Final(sign, &cx); if (memcmp(sign, SIGN_END(ctx), sizeof sign)) r = 4; } @@ -119,16 +119,16 @@ smp_chk_sign(struct smp_signctx *ctx) static void smp_append_sign(struct smp_signctx *ctx, const void *ptr, uint32_t len) { - struct SHA256Context cx; - unsigned char sign[SHA256_LEN]; + struct VSHA256Context cx; + unsigned char sign[VSHA256_LEN]; if (len != 0) { - SHA256_Update(&ctx->ctx, ptr, len); + VSHA256_Update(&ctx->ctx, ptr, len); ctx->ss->length += len; } cx = ctx->ctx; - SHA256_Update(&cx, &ctx->ss->length, sizeof(ctx->ss->length)); - SHA256_Final(sign, &cx); + VSHA256_Update(&cx, &ctx->ss->length, sizeof(ctx->ss->length)); + VSHA256_Final(sign, &cx); memcpy(SIGN_END(ctx), sign, sizeof sign); } @@ -145,8 +145,8 @@ smp_reset_sign(struct smp_signctx *ctx) strcpy(ctx->ss->ident, ctx->id); ctx->ss->unique = ctx->unique; ctx->ss->mapped = (uintptr_t)ctx->ss; - SHA256_Init(&ctx->ctx); - SHA256_Update(&ctx->ctx, ctx->ss, + VSHA256_Init(&ctx->ctx); + VSHA256_Update(&ctx->ctx, ctx->ss, offsetof(struct smp_sign, length)); smp_append_sign(ctx, NULL, 0); } @@ -358,7 +358,7 @@ smp_valid_silo(struct smp_sc *sc) /* XXX: Sanity check stuff[6] */ - assert(si->stuff[SMP_BAN1_STUFF] > sizeof *si + SHA256_LEN); + assert(si->stuff[SMP_BAN1_STUFF] > sizeof *si + VSHA256_LEN); assert(si->stuff[SMP_BAN2_STUFF] > si->stuff[SMP_BAN1_STUFF]); assert(si->stuff[SMP_SEG1_STUFF] > si->stuff[SMP_BAN2_STUFF]); assert(si->stuff[SMP_SEG2_STUFF] > si->stuff[SMP_SEG1_STUFF]); diff --git a/include/vrt.h b/include/vrt.h index 5b79a80..391a9e3 100644 --- a/include/vrt.h +++ b/include/vrt.h @@ -166,7 +166,7 @@ struct vrt_ctx { /* * method specific argument: - * hash: struct SHA256Context + * hash: struct VSHA256Context * synth+error: struct vsb * */ void *specific; diff --git a/include/vsha256.h b/include/vsha256.h index e3852e9..9e98356 100644 --- a/include/vsha256.h +++ b/include/vsha256.h @@ -26,21 +26,21 @@ * $FreeBSD: head/lib/libmd/sha256.h 154479 2006-01-17 15:35:57Z phk $ */ -#ifndef _SHA256_H_ -#define _SHA256_H_ +#ifndef _VSHA256_H_ +#define _VSHA256_H_ -#define SHA256_LEN 32 -#define SHA256_DIGEST_LENGTH 32 +#define VSHA256_LEN 32 +#define VSHA256_DIGEST_LENGTH 32 -typedef struct SHA256Context { +typedef struct VSHA256Context { uint32_t state[8]; uint64_t count; unsigned char buf[64]; -} SHA256_CTX; +} VSHA256_CTX; -void SHA256_Init(SHA256_CTX *); -void SHA256_Update(SHA256_CTX *, const void *, size_t); -void SHA256_Final(unsigned char [SHA256_LEN], SHA256_CTX *); -void SHA256_Test(void); +void VSHA256_Init(VSHA256_CTX *); +void VSHA256_Update(VSHA256_CTX *, const void *, size_t); +void VSHA256_Final(unsigned char [VSHA256_LEN], VSHA256_CTX *); +void VSHA256_Test(void); -#endif /* !_SHA256_H_ */ +#endif /* !_VSHA256_H_ */ diff --git a/lib/libvarnish/vcli_proto.c b/lib/libvarnish/vcli_proto.c index 8027fa6..3f0b165 100644 --- a/lib/libvarnish/vcli_proto.c +++ b/lib/libvarnish/vcli_proto.c @@ -47,24 +47,24 @@ void VCLI_AuthResponse(int S_fd, const char *challenge, char response[CLI_AUTH_RESPONSE_LEN + 1]) { - SHA256_CTX ctx; - uint8_t buf[SHA256_LEN]; + VSHA256_CTX ctx; + uint8_t buf[VSHA256_LEN]; int i; - assert(CLI_AUTH_RESPONSE_LEN == (SHA256_LEN * 2)); + assert(CLI_AUTH_RESPONSE_LEN == (VSHA256_LEN * 2)); - SHA256_Init(&ctx); - SHA256_Update(&ctx, challenge, 32); - SHA256_Update(&ctx, "\n", 1); + VSHA256_Init(&ctx); + VSHA256_Update(&ctx, challenge, 32); + VSHA256_Update(&ctx, "\n", 1); do { i = read(S_fd, buf, 1); if (i == 1) - SHA256_Update(&ctx, buf, i); + VSHA256_Update(&ctx, buf, i); } while (i > 0); - SHA256_Update(&ctx, challenge, 32); - SHA256_Update(&ctx, "\n", 1); - SHA256_Final(buf, &ctx); - for (i = 0; i < SHA256_LEN; i++) + VSHA256_Update(&ctx, challenge, 32); + VSHA256_Update(&ctx, "\n", 1); + VSHA256_Final(buf, &ctx); + for (i = 0; i < VSHA256_LEN; i++) assert(snprintf(response + 2 * i, 3, "%02x", buf[i]) == 2); } diff --git a/lib/libvarnish/vsha256.c b/lib/libvarnish/vsha256.c index 5b898a9..7836179 100644 --- a/lib/libvarnish/vsha256.c +++ b/lib/libvarnish/vsha256.c @@ -146,7 +146,7 @@ static const uint32_t K[64] = { * the 512-bit input block to produce a new state. */ static void -SHA256_Transform(uint32_t * state, const unsigned char block[64]) +VSHA256_Transform(uint32_t * state, const unsigned char block[64]) { uint32_t W[64]; uint32_t S[8]; @@ -211,7 +211,7 @@ static const unsigned char PAD[64] = { /* Add padding and terminating bit-count. */ static void -SHA256_Pad(SHA256_CTX * ctx) +VSHA256_Pad(VSHA256_CTX * ctx) { size_t r; @@ -225,7 +225,7 @@ SHA256_Pad(SHA256_CTX * ctx) } else { /* Finish the current block and mix. */ memcpy(&ctx->buf[r], PAD, 64 - r); - SHA256_Transform(ctx->state, ctx->buf); + VSHA256_Transform(ctx->state, ctx->buf); /* The start of the final block is all zeroes. */ memset(&ctx->buf[0], 0, 56); @@ -235,12 +235,12 @@ SHA256_Pad(SHA256_CTX * ctx) vbe64enc(&ctx->buf[56], ctx->count); /* Mix in the final block. */ - SHA256_Transform(ctx->state, ctx->buf); + VSHA256_Transform(ctx->state, ctx->buf); } /* SHA-256 initialization. Begins a SHA-256 operation. */ void -SHA256_Init(SHA256_CTX * ctx) +VSHA256_Init(VSHA256_CTX * ctx) { /* Zero bits processed so far */ @@ -259,7 +259,7 @@ SHA256_Init(SHA256_CTX * ctx) /* Add bytes into the hash */ void -SHA256_Update(SHA256_CTX * ctx, const void *in, size_t len) +VSHA256_Update(VSHA256_CTX * ctx, const void *in, size_t len) { uint64_t bitlen; uint32_t r; @@ -282,13 +282,13 @@ SHA256_Update(SHA256_CTX * ctx, const void *in, size_t len) /* Finish the current block */ memcpy(&ctx->buf[r], src, 64 - r); - SHA256_Transform(ctx->state, ctx->buf); + VSHA256_Transform(ctx->state, ctx->buf); src += 64 - r; len -= 64 - r; /* Perform complete blocks */ while (len >= 64) { - SHA256_Transform(ctx->state, src); + VSHA256_Transform(ctx->state, src); src += 64; len -= 64; } @@ -302,14 +302,15 @@ SHA256_Update(SHA256_CTX * ctx, const void *in, size_t len) * and clears the context state. */ void -SHA256_Final(unsigned char digest[static SHA256_DIGEST_LENGTH], SHA256_CTX *ctx) +VSHA256_Final(unsigned char digest[static VSHA256_DIGEST_LENGTH], + VSHA256_CTX *ctx) { /* Add padding */ - SHA256_Pad(ctx); + VSHA256_Pad(ctx); /* Write the hash */ - be32enc_vect(digest, ctx->state, SHA256_DIGEST_LENGTH); + be32enc_vect(digest, ctx->state, VSHA256_DIGEST_LENGTH); /* Clear the context state */ memset((void *)ctx, 0, sizeof(*ctx)); @@ -340,16 +341,16 @@ static const struct sha256test { void -SHA256_Test(void) +VSHA256_Test(void) { - struct SHA256Context c; + struct VSHA256Context c; const struct sha256test *p; unsigned char o[32]; for (p = sha256test; p->input != NULL; p++) { - SHA256_Init(&c); - SHA256_Update(&c, p->input, strlen(p->input)); - SHA256_Final(o, &c); + VSHA256_Init(&c); + VSHA256_Update(&c, p->input, strlen(p->input)); + VSHA256_Final(o, &c); AZ(memcmp(o, p->output, 32)); } } diff --git a/lib/libvmod_directors/hash.c b/lib/libvmod_directors/hash.c index 14f36f5..a8961df 100644 --- a/lib/libvmod_directors/hash.c +++ b/lib/libvmod_directors/hash.c @@ -98,10 +98,10 @@ VCL_BACKEND __match_proto__() vmod_hash_backend(VRT_CTX, struct vmod_directors_hash *rr, const char *arg, ...) { - struct SHA256Context sha_ctx; + struct VSHA256Context sha_ctx; va_list ap; const char *p; - unsigned char sha256[SHA256_LEN]; + unsigned char sha256[VSHA256_LEN]; VCL_BACKEND be; double r; @@ -109,16 +109,16 @@ vmod_hash_backend(VRT_CTX, struct vmod_directors_hash *rr, CHECK_OBJ_ORNULL(ctx->bo, BUSYOBJ_MAGIC); CHECK_OBJ_NOTNULL(rr, VMOD_DIRECTORS_HASH_MAGIC); - SHA256_Init(&sha_ctx); + VSHA256_Init(&sha_ctx); va_start(ap, arg); p = arg; while (p != vrt_magic_string_end) { if (p != NULL && *p != '\0') - SHA256_Update(&sha_ctx, p, strlen(p)); + VSHA256_Update(&sha_ctx, p, strlen(p)); p = va_arg(ap, const char *); } va_end(ap); - SHA256_Final(sha256, &sha_ctx); + VSHA256_Final(sha256, &sha_ctx); r = vbe32dec(sha256); r = scalbn(r, -32); diff --git a/lib/libvmod_directors/shard_hash.c b/lib/libvmod_directors/shard_hash.c index 560ceb9..8ff0eec 100644 --- a/lib/libvmod_directors/shard_hash.c +++ b/lib/libvmod_directors/shard_hash.c @@ -59,16 +59,16 @@ shard_hash_crc32(VCL_STRING s) static uint32_t __match_proto__(hash_func) shard_hash_sha256(VCL_STRING s) { - struct SHA256Context sha256; + struct VSHA256Context sha256; union { unsigned char digest[32]; uint32_t uint32_digest[8]; } sha256_digest; uint32_t r; - SHA256_Init(&sha256); - SHA256_Update(&sha256, s, strlen(s)); - SHA256_Final(sha256_digest.digest, &sha256); + VSHA256_Init(&sha256); + VSHA256_Update(&sha256, s, strlen(s)); + VSHA256_Final(sha256_digest.digest, &sha256); /* * use low 32 bits only From dridi.boukelmoune at gmail.com Tue Oct 31 10:43:05 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 31 Oct 2017 10:43:05 +0000 (UTC) Subject: [master] 8d3c266 Use the complete commit hash in the version string Message-ID: <20171031104305.EB129A647B@lists.varnish-cache.org> commit 8d3c266386c635b5de2943d85b0edc2e8df91095 Author: Dridi Boukelmoune Date: Tue Oct 31 11:40:39 2017 +0100 Use the complete commit hash in the version string Fixes #2468 diff --git a/lib/libvcc/generate.py b/lib/libvcc/generate.py index 2f24b78..fa03724 100755 --- a/lib/libvcc/generate.py +++ b/lib/libvcc/generate.py @@ -1397,7 +1397,7 @@ fp_vclvar.close() if os.path.isdir(os.path.join(srcroot, ".git")): v = subprocess.check_output([ "git --git-dir=" + os.path.join(srcroot, ".git") + - " show -s --pretty=format:%h" + " show -s --pretty=format:%H" ], shell=True, universal_newlines=True) v = v.strip() b = subprocess.check_output([ From daghf at varnish-software.com Tue Oct 31 10:48:05 2017 From: daghf at varnish-software.com (Dag Haavi Finstad) Date: Tue, 31 Oct 2017 10:48:05 +0000 (UTC) Subject: [master] 8844d6b Add compatibility macros for SHA -> VSHA Message-ID: <20171031104805.109FAA6790@lists.varnish-cache.org> commit 8844d6ba3a0ef8e984c6ac70a3acf2cf93f542f2 Author: Dag Haavi Finstad Date: Tue Oct 31 11:46:09 2017 +0100 Add compatibility macros for SHA -> VSHA This should help the transition for VMOD writers that rely on the SHA256_* functions. diff --git a/include/vsha256.h b/include/vsha256.h index 9e98356..8f20505 100644 --- a/include/vsha256.h +++ b/include/vsha256.h @@ -43,4 +43,13 @@ void VSHA256_Update(VSHA256_CTX *, const void *, size_t); void VSHA256_Final(unsigned char [VSHA256_LEN], VSHA256_CTX *); void VSHA256_Test(void); +#define SHA256_LEN VSHA256_LEN +#define SHA256_DIGEST_LENGTH VSHA256_DIGEST_LENGTH +#define SHA256Context VSHA256Context +#define SHA256_CTX VSHA256_CTX +#define SHA256_Init VSHA256_Init +#define SHA256_Update VSHA256_Update +#define SHA256_Final VSHA256_Final +#define SHA256_Test VSHA256_Test + #endif /* !_VSHA256_H_ */ From dridi.boukelmoune at gmail.com Tue Oct 31 11:02:05 2017 From: dridi.boukelmoune at gmail.com (Dridi Boukelmoune) Date: Tue, 31 Oct 2017 11:02:05 +0000 (UTC) Subject: [4.1] 44f5299 Use the complete commit hash in the version string Message-ID: <20171031110205.5E0C3A6CF5@lists.varnish-cache.org> commit 44f5299c374d57f5a8a7d09ca4371a3f868d8822 Author: Dridi Boukelmoune Date: Tue Oct 31 11:40:39 2017 +0100 Use the complete commit hash in the version string Fixes #2468 diff --git a/include/Makefile.am b/include/Makefile.am index ced33c8..2ac2e0f 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -101,7 +101,7 @@ BUILT_SOURCES = vcs_version.h vmod_abi.h MAINTAINERCLEANFILES = vcs_version.h vcs_version.h: FORCE @if [ -d "$(top_srcdir)/.git" ]; then \ - V="$$(git show -s --pretty=format:%h)" \ + V="$$(git show -s --pretty=format:%H)" \ B="$$(git rev-parse --abbrev-ref HEAD)" \ H="$$(head -n 1 vcs_version.h 2>/dev/null || true)"; \ if [ "/* $$V */" != "$$H" ]; then \ From phk at FreeBSD.org Tue Oct 31 12:09:05 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 31 Oct 2017 12:09:05 +0000 (UTC) Subject: [master] 34758f9 Catch up with VSHA256 Message-ID: <20171031120905.B0266A61F2@lists.varnish-cache.org> commit 34758f90e104edd88bfcde79edcab2cacbe144c8 Author: Poul-Henning Kamp Date: Tue Oct 31 12:08:10 2017 +0000 Catch up with VSHA256 diff --git a/lib/libvarnish/flint.lnt b/lib/libvarnish/flint.lnt index 636828a..64d4dd2 100644 --- a/lib/libvarnish/flint.lnt +++ b/lib/libvarnish/flint.lnt @@ -1,10 +1,10 @@ --efunc(835, SHA256_Transform) // A zero has been given as ___ argument to operator '___' --efunc(778, SHA256_Transform) // Constant expression evaluates to 0 in operation '___' --efunc(661, SHA256_Transform) // Possible access of out-of-bounds pointer --efunc(662, SHA256_Transform) // Possible creation of out-of-bounds pointer --efunc(662, SHA256_Update) // Possible creation of out-of-bounds pointer +-efunc(835, VSHA256_Transform) // A zero has been given as ___ argument to operator '___' +-efunc(778, VSHA256_Transform) // Constant expression evaluates to 0 in operation '___' +-efunc(661, VSHA256_Transform) // Possible access of out-of-bounds pointer +-efunc(662, VSHA256_Transform) // Possible creation of out-of-bounds pointer +-efunc(662, VSHA256_Update) // Possible creation of out-of-bounds pointer -efunc(662, be32dec_vect) // Possible creation of out-of-bounds pointer --efunc(670, SHA256_Update) // Possible access beyond array for function '___', +-efunc(670, VSHA256_Update) // Possible access beyond array for function '___', +fan // Allow anon unions -e713 // Loss of precision (assignment) (unsigned long long to long long) From phk at FreeBSD.org Tue Oct 31 12:19:06 2017 From: phk at FreeBSD.org (Poul-Henning Kamp) Date: Tue, 31 Oct 2017 12:19:06 +0000 (UTC) Subject: [master] 2b9a15d Make vdef.h an "include only once" header Message-ID: <20171031121906.1158CA662F@lists.varnish-cache.org> commit 2b9a15d4319b1f471c9a04f8a730b8cb0ca12897 Author: Poul-Henning Kamp Date: Tue Oct 31 12:18:35 2017 +0000 Make vdef.h an "include only once" header diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c index 0da7a4b..0fb3a9b 100644 --- a/bin/varnishtest/vtc_main.c +++ b/bin/varnishtest/vtc_main.c @@ -42,7 +42,6 @@ #include "vtc.h" -#include "vdef.h" #include "vev.h" #include "vfil.h" #include "vnum.h" diff --git a/include/vdef.h b/include/vdef.h index efb23fb..4ab9e73 100644 --- a/include/vdef.h +++ b/include/vdef.h @@ -32,7 +32,9 @@ * */ -#ifndef VDEF_H_INCLUDED +#ifdef VDEF_H_INCLUDED +# error "vdef.h included multiple times" +#endif #define VDEF_H_INCLUDED /* Safe printf into a fixed-size buffer */ @@ -160,5 +162,3 @@ # define ___Static_assert(x, y) \ typedef char __assert_## y[(x) ? 1 : -1] __unused #endif - -#endif /* VDEF_H_INCLUDED */