GCOV - Varnish Cache


/*-
 * Copyright (c) 2006 Verdens Gang AS
 * Copyright (c) 2006-2011 Varnish Software AS
 * All rights reserved.
 *
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
 *
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * Poll backends for collection of health statistics
 *
 * We co-opt threads from the worker pool for probing the backends,
 * but we want to avoid a potentially messy cleanup operation when we
 * retire the backend, so the thread owns the health information, which
 * the backend references, rather than the other way around.
 *
 */

#include "config.h"

#include <poll.h>
#include <stdio.h>
#include <stdlib.h>

#include "cache_varnishd.h"

#include "vbh.h"
#include "vcli_serve.h"
#include "vsa.h"
#include "vtcp.h"
#include "vtim.h"

#include "cache_backend.h"
#include "cache_conn_pool.h"

#include "VSC_vbe.h"

/* Default averaging rate, we want something pretty responsive */
#define AVG_RATE                        4

struct vbp_target {
        unsigned                        magic;
#define VBP_TARGET_MAGIC                0x6b7cb656

        VRT_BACKEND_PROBE_FIELDS()

        struct backend                  *backend;
        struct conn_pool                *conn_pool;

        char                            *req;
        int                             req_len;

        char                            resp_buf[128];
        unsigned                        good;

        /* Collected statistics */
#define BITMAP(n, c, t, b)      uintmax_t       n;
#include "tbl/backend_poll.h"

        vtim_dur                        last;
        vtim_dur                        avg;
        double                          rate;

        vtim_real                       due;
        int                             running;
        int                             heap_idx;
        struct pool_task                task[1];
};

static struct lock                      vbp_mtx;
static pthread_cond_t                   vbp_cond;
static struct vbh                       *vbp_heap;

static const unsigned char vbp_proxy_local[] = {
        0x0d, 0x0a, 0x0d, 0x0a, 0x00, 0x0d, 0x0a, 0x51,
        0x55, 0x49, 0x54, 0x0a, 0x20, 0x00, 0x00, 0x00,
};

/*--------------------------------------------------------------------*/

static void
vbp_delete(struct vbp_target *vt)
{
        CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);

#define DN(x)   /**/
        VRT_BACKEND_PROBE_HANDLE();
#undef DN
        VCP_Rel(&vt->conn_pool);
        free(vt->req);
        FREE_OBJ(vt);
}


/*--------------------------------------------------------------------
 * Record pokings...
 */

static void
vbp_start_poke(struct vbp_target *vt)
{
        CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);

#define BITMAP(n, c, t, b) \
        vt->n <<= 1;
#include "tbl/backend_poll.h"

        vt->last = 0;
        vt->resp_buf[0] = '\0';
}

static void
vbp_has_poked(struct vbp_target *vt)
{
        unsigned i, j;
        uint64_t u;

        CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);

        /* Calculate exponential average */
        if (vt->happy & 1) {
                if (vt->rate < AVG_RATE)
                        vt->rate += 1.0;
                vt->avg += (vt->last - vt->avg) / vt->rate;
        }

        u = vt->happy;
        for (i = j = 0; i < vt->window; i++) {
                if (u & 1)
                        j++;
                u >>= 1;
        }
        vt->good = j;
}

void
VBP_Update_Backend(struct vbp_target *vt)
{
        unsigned i = 0, chg;
        char bits[10];

        CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);

#define BITMAP(n, c, t, b)                      \
        bits[i++] = (vt->n & 1) ? c : '-';
#include "tbl/backend_poll.h"
        bits[i] = '\0';
        assert(i < sizeof bits);

        Lck_Lock(&vbp_mtx);
        if (vt->backend == NULL) {
                Lck_Unlock(&vbp_mtx);
                return;
        }

        i = (vt->good < vt->threshold);
        chg = (i != vt->backend->sick);
        vt->backend->sick = i;

        AN(vt->backend->vcl_name);
        VSL(SLT_Backend_health, NO_VXID,
            "%s %s %s %s %u %u %u %.6f %.6f \"%s\"",
            vt->backend->vcl_name, chg ? "Went" : "Still",
            i ? "sick" : "healthy", bits,
            vt->good, vt->threshold, vt->window,
            vt->last, vt->avg, vt->resp_buf);
        vt->backend->vsc->happy = vt->happy;
        if (chg)
                vt->backend->changed = VTIM_real();
        Lck_Unlock(&vbp_mtx);
}

static void
vbp_reset(struct vbp_target *vt)
{
        unsigned u;

        CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
        vt->avg = 0.0;
        vt->rate = 0.0;
#define BITMAP(n, c, t, b) \
        vt->n = 0;
#include "tbl/backend_poll.h"

        for (u = 0; u < vt->initial; u++) {
                vbp_start_poke(vt);
                vt->happy |= 1;
                vbp_has_poked(vt);
        }
}

/*--------------------------------------------------------------------
 * Poke one backend, once, but possibly at both IPv4 and IPv6 addresses.
 *
 * We do deliberately not use the stuff in cache_backend.c, because we
 * want to measure the backends response without local distractions.
 */

static int
vbp_write(struct vbp_target *vt, int *sock, const void *buf, size_t len)
{
        int i;

        i = write(*sock, buf, len);
        VTCP_Assert(i);
        if (i != len) {
                if (i < 0) {
                        vt->err_xmit |= 1;
                        bprintf(vt->resp_buf, "Write error %d (%s)",
                                errno, VAS_errtxt(errno));
                } else {
                        bprintf(vt->resp_buf,
                                "Short write (%d/%zu) error %d (%s)",
                                i, len, errno, VAS_errtxt(errno));
                }
                VTCP_close(sock);
                return (-1);
        }
        return (0);
}

static int
vbp_write_proxy_v1(struct vbp_target *vt, int *sock)
{
        char buf[105]; /* maximum size for a TCP6 PROXY line with null char */
        char addr[VTCP_ADDRBUFSIZE];
        char port[VTCP_PORTBUFSIZE];
        char vsabuf[vsa_suckaddr_len];
        const struct suckaddr *sua;
        int proto;
        struct vsb vsb;

        sua = VSA_getsockname(*sock, vsabuf, sizeof vsabuf);
        AN(sua);
        AN(VSB_init(&vsb, buf, sizeof buf));

        proto = VSA_Get_Proto(sua);
        if (proto == AF_INET || proto == AF_INET6) {
                VTCP_name(sua, addr, sizeof addr, port, sizeof port);
                VSB_printf(&vsb, "PROXY %s %s %s %s %s\r\n",
                    proto == AF_INET ? "TCP4" : "TCP6",
                    addr, addr, port, port);
        } else {
                VSB_cat(&vsb, "PROXY UNKNOWN\r\n");
        }
        AZ(VSB_finish(&vsb));

        VSB_fini(&vsb);
        return (vbp_write(vt, sock, buf, strlen(buf)));
}

static void
vbp_poke(struct vbp_target *vt)
{
        int s, tmo, i, proxy_header, err;
        vtim_real t_start, t_now, t_end;
        unsigned rlen, resp;
        char buf[8192], *p;
        struct pollfd pfda[1], *pfd = pfda;
        const struct suckaddr *sa;

        t_start = t_now = VTIM_real();
        t_end = t_start + vt->timeout;

        s = VCP_Open(vt->conn_pool, t_end - t_now, &sa, &err);
        if (s < 0) {
                bprintf(vt->resp_buf, "Open error %d (%s)", err, VAS_errtxt(err));
                Lck_Lock(&vbp_mtx);
                if (vt->backend)
                        VBE_Connect_Error(vt->backend->vsc, err);
                Lck_Unlock(&vbp_mtx);
                return;
        }

        i = VSA_Get_Proto(sa);
        if (VSA_Compare(sa, bogo_ip) == 0)
                vt->good_unix |= 1;
        else if (i == AF_INET)
                vt->good_ipv4 |= 1;
        else if (i == AF_INET6)
                vt->good_ipv6 |= 1;
        else
                WRONG("Wrong probe protocol family");

        t_now = VTIM_real();
        tmo = (int)round((t_end - t_now) * 1e3);
        if (tmo <= 0) {
                bprintf(vt->resp_buf,
                        "Open timeout %.3fs exceeded by %.3fs",
                        vt->timeout, t_now - t_end);
                VTCP_close(&s);
                return;
        }

        Lck_Lock(&vbp_mtx);
        if (vt->backend != NULL)
                proxy_header = vt->backend->proxy_header;
        else
                proxy_header = -1;
        Lck_Unlock(&vbp_mtx);

        if (proxy_header < 0) {
                bprintf(vt->resp_buf, "%s", "No backend");
                VTCP_close(&s);
                return;
        }

        /* Send the PROXY header */
        assert(proxy_header <= 2);
        if (proxy_header == 1) {
                if (vbp_write_proxy_v1(vt, &s) != 0)
                        return;
        } else if (proxy_header == 2 &&
            vbp_write(vt, &s, vbp_proxy_local, sizeof vbp_proxy_local) != 0)
                return;

        /* Send the request */
        if (vbp_write(vt, &s, vt->req, vt->req_len) != 0)
                return;

        vt->good_xmit |= 1;

        pfd->fd = s;
        rlen = 0;
        while (1) {
                pfd->events = POLLIN;
                pfd->revents = 0;
                t_now = VTIM_real();
                tmo = (int)round((t_end - t_now) * 1e3);
                if (tmo <= 0) {
                        bprintf(vt->resp_buf,
                            "Poll timeout %.3fs exceeded by %.3fs",
                            vt->timeout, t_now - t_end);
                        i = -1;
                        break;
                }
                i = poll(pfd, 1, tmo);
                if (i <= 0) {
                        if (!i) {
                                if (!vt->exp_close)
                                        break;
                                errno = ETIMEDOUT;
                        }
                        bprintf(vt->resp_buf, "Poll error %d (%s)",
                            errno, VAS_errtxt(errno));
                        i = -1;
                        break;
                }
                if (rlen < sizeof vt->resp_buf)
                        i = read(s, vt->resp_buf + rlen,
                            sizeof vt->resp_buf - rlen);
                else
                        i = read(s, buf, sizeof buf);
                VTCP_Assert(i);
                if (i <= 0) {
                        if (i < 0)
                                bprintf(vt->resp_buf, "Read error %d (%s)",
                                        errno, VAS_errtxt(errno));
                        break;
                }
                rlen += i;
        }

        VTCP_close(&s);

        if (i < 0) {
                /* errno reported above */
                vt->err_recv |= 1;
                return;
        }

        if (rlen == 0) {
                bprintf(vt->resp_buf, "%s", "Empty response");
                return;
        }

        /* So we have a good receive ... */
        t_now = VTIM_real();
        vt->last = t_now - t_start;
        vt->good_recv |= 1;

        /* Now find out if we like the response */
        vt->resp_buf[sizeof vt->resp_buf - 1] = '\0';
        p = strchr(vt->resp_buf, '\r');
        if (p != NULL)
                *p = '\0';
        p = strchr(vt->resp_buf, '\n');
        if (p != NULL)
                *p = '\0';

        i = sscanf(vt->resp_buf, "HTTP/%*f %u ", &resp);

        if (i == 1 && resp == vt->exp_status)
                vt->happy |= 1;
}

/*--------------------------------------------------------------------
 */
static void
vbp_heap_insert(struct vbp_target *vt)
{
        // Lck_AssertHeld(&vbp_mtx);
        VBH_insert(vbp_heap, vt);
        if (VBH_root(vbp_heap) == vt)
                PTOK(pthread_cond_signal(&vbp_cond));
}

/*--------------------------------------------------------------------
 */

static void v_matchproto_(task_func_t)
vbp_task(struct worker *wrk, void *priv)
{
        struct vbp_target *vt;

        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
        CAST_OBJ_NOTNULL(vt, priv, VBP_TARGET_MAGIC);

        AN(vt->running);
        AN(vt->req);
        assert(vt->req_len > 0);

        vbp_start_poke(vt);
        vbp_poke(vt);
        vbp_has_poked(vt);
        VBP_Update_Backend(vt);

        Lck_Lock(&vbp_mtx);
        if (vt->running < 0) {
                assert(vt->heap_idx == VBH_NOIDX);
                vbp_delete(vt);
        } else {
                vt->running = 0;
                if (vt->heap_idx != VBH_NOIDX) {
                        vt->due = VTIM_real() + vt->interval;
                        VBH_delete(vbp_heap, vt->heap_idx);
                        vbp_heap_insert(vt);
                }
        }
        Lck_Unlock(&vbp_mtx);
}

/*--------------------------------------------------------------------
 */

static void * v_matchproto_(bgthread_t)
vbp_thread(struct worker *wrk, void *priv)
{
        vtim_real now, nxt;
        struct vbp_target *vt;
        int r;

        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
        AZ(priv);
        Lck_Lock(&vbp_mtx);
        while (1) {
                now = VTIM_real();
                vt = VBH_root(vbp_heap);
                if (vt == NULL) {
                        nxt = 8.192 + now;
                        (void)Lck_CondWaitUntil(&vbp_cond, &vbp_mtx, nxt);
                } else if (vt->due > now) {
                        nxt = vt->due;
                        vt = NULL;
                        (void)Lck_CondWaitUntil(&vbp_cond, &vbp_mtx, nxt);
                } else {
                        VBH_delete(vbp_heap, vt->heap_idx);
                        vt->due = now + vt->interval;
                        VBH_insert(vbp_heap, vt);
                        if (!vt->running) {
                                vt->running = 1;
                                vt->task->func = vbp_task;
                                vt->task->priv = vt;
                                Lck_Unlock(&vbp_mtx);
                                r = Pool_Task_Any(vt->task, TASK_QUEUE_REQ);
                                Lck_Lock(&vbp_mtx);
                                if (r)
                                        vt->running = 0;
                        }
                }
        }
        NEEDLESS(Lck_Unlock(&vbp_mtx));
        NEEDLESS(return (NULL));
}


/*--------------------------------------------------------------------
 * Cli functions
 */

static void
vbp_bitmap(struct vsb *vsb, char c, uint64_t map, const char *lbl)
{
        int i;
        uint64_t u = (1ULL << 63);

        VSB_cat(vsb, "  ");
        for (i = 0; i < 64; i++) {
                if (map & u)
                        VSB_putc(vsb, c);
                else
                        VSB_putc(vsb, '-');
                map <<= 1;
        }
        VSB_printf(vsb, " %s\n", lbl);
}

/*lint -e{506} constant value boolean */
/*lint -e{774} constant value boolean */
void
VBP_Status(struct vsb *vsb, const struct backend *be, int details, int json)
{
        struct vbp_target *vt;

        CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
        vt = be->probe;
        CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);

        if (!details) {
                if (json)
                        VSB_printf(vsb, "[%u, %u, \"%s\"]",
                            vt->good, vt->window,
                            vt->backend->sick ? "sick" : "healthy");
                else
                        VSB_printf(vsb, "%u/%u\t%s", vt->good, vt->window,
                            vt->backend->sick ? "sick" : "healthy");
                return;
        }

        if (json) {
                VSB_cat(vsb, "{\n");
                VSB_indent(vsb, 2);
#define BITMAP(nn, cc, tt, bb)                                  \
                VSB_printf(vsb, "\"bits_%c\": %ju,\n", cc, vt->nn);
#include "tbl/backend_poll.h"
                VSB_printf(vsb, "\"good\": %u,\n", vt->good);
                VSB_printf(vsb, "\"threshold\": %u,\n", vt->threshold);
                VSB_printf(vsb, "\"window\": %u", vt->window);
                VSB_indent(vsb, -2);
                VSB_cat(vsb, "\n");
                VSB_cat(vsb, "},\n");
                return;
        }

        VSB_printf(vsb,
            "\n Current states  good: %2u threshold: %2u window: %2u\n",
            vt->good, vt->threshold, vt->window);
        VSB_printf(vsb,
            "  Average response time of good probes: %.6f\n", vt->avg);
        VSB_cat(vsb,
            "  Oldest ======================"
            "============================ Newest\n");

#define BITMAP(n, c, t, b)                                      \
                if ((vt->n != 0) || (b))                        \
                        vbp_bitmap(vsb, (c), vt->n, (t));
#include "tbl/backend_poll.h"
}

/*--------------------------------------------------------------------
 * Build request from probe spec
 */

static void
vbp_build_req(struct vbp_target *vt, const struct vrt_backend_probe *vbp,
    const struct backend *be)
{
        struct vsb *vsb;

        vsb = VSB_new_auto();
        AN(vsb);
        VSB_clear(vsb);
        if (vbp->request != NULL) {
                VSB_cat(vsb, vbp->request);
        } else {
                AN(be->hosthdr);
                VSB_printf(vsb,
                    "GET %s HTTP/1.1\r\n"
                    "Host: %s\r\n"
                    "Connection: close\r\n"
                    "\r\n",
                    vbp->url != NULL ?  vbp->url : "/",
                    be->hosthdr);
        }
        AZ(VSB_finish(vsb));
        vt->req = strdup(VSB_data(vsb));
        AN(vt->req);
        vt->req_len = VSB_len(vsb);
        VSB_destroy(&vsb);
}

/*--------------------------------------------------------------------
 * Sanitize and set defaults
 * XXX: we could make these defaults parameters
 */

static void
vbp_set_defaults(struct vbp_target *vt, const struct vrt_backend_probe *vp)
{

#define DN(x)   do { vt->x = vp->x; } while (0)
        VRT_BACKEND_PROBE_HANDLE();
#undef DN

        if (vt->timeout == 0.0)
                vt->timeout = 2.0;
        if (vt->interval == 0.0)
                vt->interval = 5.0;
        if (vt->window == 0)
                vt->window = 8;
        if (vt->threshold == 0)
                vt->threshold = 3;
        if (vt->exp_status == 0)
                vt->exp_status = 200;

        if (vt->initial == ~0U)
                vt->initial = vt->threshold - 1;

        vt->initial = vmin(vt->initial, vt->threshold);
}

/*--------------------------------------------------------------------
 */

void
VBP_Control(const struct backend *be, int enable)
{
        struct vbp_target *vt;

        CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
        vt = be->probe;
        CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);

        vbp_reset(vt);
        VBP_Update_Backend(vt);

        Lck_Lock(&vbp_mtx);
        if (enable) {
                assert(vt->heap_idx == VBH_NOIDX);
                vt->due = VTIM_real();
                vbp_heap_insert(vt);
        } else {
                assert(vt->heap_idx != VBH_NOIDX);
                VBH_delete(vbp_heap, vt->heap_idx);
        }
        Lck_Unlock(&vbp_mtx);
}

/*--------------------------------------------------------------------
 * Insert/Remove/Use called from cache_backend.c
 */

void
VBP_Insert(struct backend *b, const struct vrt_backend_probe *vp,
    struct conn_pool *tp)
{
        struct vbp_target *vt;

        CHECK_OBJ_NOTNULL(b, BACKEND_MAGIC);
        CHECK_OBJ_NOTNULL(vp, VRT_BACKEND_PROBE_MAGIC);

        AZ(b->probe);

        ALLOC_OBJ(vt, VBP_TARGET_MAGIC);
        XXXAN(vt);

        vt->conn_pool = tp;
        VCP_AddRef(vt->conn_pool);
        vt->backend = b;
        b->probe = vt;

        vbp_set_defaults(vt, vp);
        vbp_build_req(vt, vp, b);

        vbp_reset(vt);
}

void
VBP_Remove(struct backend *be)
{
        struct vbp_target *vt;

        CHECK_OBJ_NOTNULL(be, BACKEND_MAGIC);
        vt = be->probe;
        CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);

        Lck_Lock(&vbp_mtx);
        be->sick = 1;
        be->probe = NULL;
        vt->backend = NULL;
        if (vt->running) {
                // task scheduled, it calls vbp_delete()
                vt->running = -1;
                vt = NULL;
        } else if (vt->heap_idx != VBH_NOIDX) {
                // task done, not yet rescheduled
                VBH_delete(vbp_heap, vt->heap_idx);
        }
        Lck_Unlock(&vbp_mtx);
        if (vt != NULL) {
                assert(vt->heap_idx == VBH_NOIDX);
                vbp_delete(vt);
        }
}

/*-------------------------------------------------------------------*/

static int v_matchproto_(vbh_cmp_t)
vbp_cmp(void *priv, const void *a, const void *b)
{
        const struct vbp_target *aa, *bb;
        int ar, br;

        AZ(priv);
        CAST_OBJ_NOTNULL(aa, a, VBP_TARGET_MAGIC);
        CAST_OBJ_NOTNULL(bb, b, VBP_TARGET_MAGIC);

        ar = aa->running == 0;
        br = bb->running == 0;

        if (ar != br)
                return (ar);

        return (aa->due < bb->due);
}

static void v_matchproto_(vbh_update_t)
vbp_update(void *priv, void *p, unsigned u)
{
        struct vbp_target *vt;

        AZ(priv);
        CAST_OBJ_NOTNULL(vt, p, VBP_TARGET_MAGIC);
        vt->heap_idx = u;
}

/*-------------------------------------------------------------------*/

void
VBP_Init(void)
{
        pthread_t thr;

        Lck_New(&vbp_mtx, lck_probe);
        vbp_heap = VBH_new(NULL, vbp_cmp, vbp_update);
        AN(vbp_heap);
        PTOK(pthread_cond_init(&vbp_cond, NULL));
        WRK_BgThread(&thr, "backend-poller", vbp_thread, NULL);
}

		varnish-cache/bin/varnishd/cache/cache_backend_probe.c
0		/*-
1		* Copyright (c) 2006 Verdens Gang AS
2		* Copyright (c) 2006-2011 Varnish Software AS
3		* All rights reserved.
4		*
5		* Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
6		*
7		* SPDX-License-Identifier: BSD-2-Clause
8		*
9		* Redistribution and use in source and binary forms, with or without
10		* modification, are permitted provided that the following conditions
11		* are met:
12		* 1. Redistributions of source code must retain the above copyright
13		* notice, this list of conditions and the following disclaimer.
14		* 2. Redistributions in binary form must reproduce the above copyright
15		* notice, this list of conditions and the following disclaimer in the
16		* documentation and/or other materials provided with the distribution.
17		*
18		* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19		* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20		* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21		* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
22		* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23		* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24		* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25		* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26		* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27		* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28		* SUCH DAMAGE.
29		*
30		* Poll backends for collection of health statistics
31		*
32		* We co-opt threads from the worker pool for probing the backends,
33	18461	* but we want to avoid a potentially messy cleanup operation when we
34	18461	* retire the backend, so the thread owns the health information, which
35	18461	* the backend references, rather than the other way around.
36	18461	*
37	18461	*/
38	18461
39	18461	#include "config.h"
40	18461
41		#include <poll.h>
42		#include <stdio.h>
43		#include <stdlib.h>
44
45		#include "cache_varnishd.h"
46
47		#include "vbh.h"
48		#include "vcli_serve.h"
49		#include "vsa.h"
50		#include "vtcp.h"
51		#include "vtim.h"
52
53		#include "cache_backend.h"
54		#include "cache_conn_pool.h"
55
56		#include "VSC_vbe.h"
57
58		/* Default averaging rate, we want something pretty responsive */
59		#define AVG_RATE 4
60
61		struct vbp_target {
62		unsigned magic;
63		#define VBP_TARGET_MAGIC 0x6b7cb656
64
65		VRT_BACKEND_PROBE_FIELDS()
66
67		struct backend *backend;
68		struct conn_pool *conn_pool;
69
70		char *req;
71		int req_len;
72
73		char resp_buf[128];
74		unsigned good;
75
76		/* Collected statistics */
77		#define BITMAP(n, c, t, b) uintmax_t n;
78		#include "tbl/backend_poll.h"
79
80		vtim_dur last;
81		vtim_dur avg;
82		double rate;
83
84		vtim_real due;
85		int running;
86		int heap_idx;
87		struct pool_task task[1];
88		};
89
90		static struct lock vbp_mtx;
91		static pthread_cond_t vbp_cond;
92		static struct vbh *vbp_heap;
93
94		static const unsigned char vbp_proxy_local[] = {
95		0x0d, 0x0a, 0x0d, 0x0a, 0x00, 0x0d, 0x0a, 0x51,
96		0x55, 0x49, 0x54, 0x0a, 0x20, 0x00, 0x00, 0x00,
97		};
98
99		/--------------------------------------------------------------------/
100
101		static void
102	150	vbp_delete(struct vbp_target *vt)
103		{
104	150	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
105
106		#define DN(x) /**/
107	150	VRT_BACKEND_PROBE_HANDLE();
108		#undef DN
109	150	VCP_Rel(&vt->conn_pool);
110	150	free(vt->req);
111	150	FREE_OBJ(vt);
112	150	}
113
114
115		/*--------------------------------------------------------------------
116		* Record pokings...
117		*/
118
119		static void
120	9671	vbp_start_poke(struct vbp_target *vt)
121		{
122	9671	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
123
124		#define BITMAP(n, c, t, b) \
125		vt->n <<= 1;
126		#include "tbl/backend_poll.h"
127
128	9671	vt->last = 0;
129	9671	vt->resp_buf[0] = '\0';
130	9671	}
131
132		static void
133	9590	vbp_has_poked(struct vbp_target *vt)
134		{
135		unsigned i, j;
136		uint64_t u;
137
138	9590	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
139
140		/* Calculate exponential average */
141	9590	if (vt->happy & 1) {
142	8407	if (vt->rate < AVG_RATE)
143	4408	vt->rate += 1.0;
144	8407	vt->avg += (vt->last - vt->avg) / vt->rate;
145	8407	}
146
147	9590	u = vt->happy;
148	80147	for (i = j = 0; i < vt->window; i++) {
149	70557	if (u & 1)
150	38950	j++;
151	70557	u >>= 1;
152	70557	}
153	9590	vt->good = j;
154	9590	}
155
156		void
157	6590	VBP_Update_Backend(struct vbp_target *vt)
158		{
159	6590	unsigned i = 0, chg;
160		char bits[10];
161
162	6590	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
163
164		#define BITMAP(n, c, t, b) \
165		bits[i++] = (vt->n & 1) ? c : '-';
166		#include "tbl/backend_poll.h"
167	6590	bits[i] = '\0';
168	6590	assert(i < sizeof bits);
169
170	6590	Lck_Lock(&vbp_mtx);
171	6590	if (vt->backend == NULL) {
172	0	Lck_Unlock(&vbp_mtx);
173	0	return;
174		}
175
176	6590	i = (vt->good < vt->threshold);
177	6590	chg = (i != vt->backend->sick);
178	6590	vt->backend->sick = i;
179
180	6590	AN(vt->backend->vcl_name);
181	13180	VSL(SLT_Backend_health, NO_VXID,
182		"%s %s %s %s %u %u %u %.6f %.6f \"%s\"",
183	6590	vt->backend->vcl_name, chg ? "Went" : "Still",
184	6590	i ? "sick" : "healthy", bits,
185	6590	vt->good, vt->threshold, vt->window,
186	6590	vt->last, vt->avg, vt->resp_buf);
187	6590	vt->backend->vsc->happy = vt->happy;
188	6590	if (chg)
189	1685	vt->backend->changed = VTIM_real();
190	6590	Lck_Unlock(&vbp_mtx);
191	6590	}
192
193		static void
194	1875	vbp_reset(struct vbp_target *vt)
195		{
196		unsigned u;
197
198	1875	CHECK_OBJ_NOTNULL(vt, VBP_TARGET_MAGIC);
199	1875	vt->avg = 0.0;
200	1875	vt->rate = 0.0;
201		#define BITMAP(n, c, t, b) \
202		vt->n = 0;
203		#include "tbl/backend_poll.h"
204
205	6750	for (u = 0; u < vt->initial; u++) {
206	4875	vbp_start_poke(vt);
207	4875	vt->happy \|= 1;
208	4875	vbp_has_poked(vt);
209	4875	}
210	1875	}
211
212		/*--------------------------------------------------------------------
213		* Poke one backend, once, but possibly at both IPv4 and IPv6 addresses.
214		*
215		* We do deliberately not use the stuff in cache_backend.c, because we
216		* want to measure the backends response without local distractions.
217		*/
218
219		static int
220	4814	vbp_write(struct vbp_target vt, int sock, const void *buf, size_t len)
221		{
222		int i;
223
224	4814	i = write(*sock, buf, len);
225	4814	VTCP_Assert(i);
226	4814	if (i != len) {
227	0	if (i < 0) {
228	0	vt->err_xmit \|= 1;
229	0	bprintf(vt->resp_buf, "Write error %d (%s)",
230		errno, VAS_errtxt(errno));
231	0	} else {
232	0	bprintf(vt->resp_buf,
233		"Short write (%d/%zu) error %d (%s)",
234		i, len, errno, VAS_errtxt(errno));
235		}
236	0	VTCP_close(sock);
237	0	return (-1);
238		}
239	4814	return (0);
240	4814	}
241
242		static int
243	154	vbp_write_proxy_v1(struct vbp_target vt, int sock)
244		{
245		char buf[105]; /* maximum size for a TCP6 PROXY line with null char */
246		char addr[VTCP_ADDRBUFSIZE];
247		char port[VTCP_PORTBUFSIZE];
248	154	char vsabuf[vsa_suckaddr_len];
249		const struct suckaddr *sua;
250		int proto;
251		struct vsb vsb;
252
253	154	sua = VSA_getsockname(*sock, vsabuf, sizeof vsabuf);
254	154	AN(sua);
255	154	AN(VSB_init(&vsb, buf, sizeof buf));
256
257	154	proto = VSA_Get_Proto(sua);
258	154	if (proto == AF_INET \|\| proto == AF_INET6) {
259	79	VTCP_name(sua, addr, sizeof addr, port, sizeof port);
260	79	VSB_printf(&vsb, "PROXY %s %s %s %s %s\r\n",
261	79	proto == AF_INET ? "TCP4" : "TCP6",
262	79	addr, addr, port, port);
263	79	} else {
264	75	VSB_cat(&vsb, "PROXY UNKNOWN\r\n");
265		}
266	154	AZ(VSB_finish(&vsb));
267
268	154	VSB_fini(&vsb);
269	154	return (vbp_write(vt, sock, buf, strlen(buf)));
270	154	}
271
272		static void
273	4771	vbp_poke(struct vbp_target *vt)
274		{
275		int s, tmo, i, proxy_header, err;
276		vtim_real t_start, t_now, t_end;
277		unsigned rlen, resp;
278		char buf[8192], *p;
279	4771	struct pollfd pfda[1], *pfd = pfda;
280		const struct suckaddr *sa;
281
282	4771	t_start = t_now = VTIM_real();
283	4771	t_end = t_start + vt->timeout;
284
285	4771	s = VCP_Open(vt->conn_pool, t_end - t_now, &sa, &err);
286	4771	if (s < 0) {
287	303	bprintf(vt->resp_buf, "Open error %d (%s)", err, VAS_errtxt(err));
288	303	Lck_Lock(&vbp_mtx);
289	303	if (vt->backend)
290	303	VBE_Connect_Error(vt->backend->vsc, err);
291	303	Lck_Unlock(&vbp_mtx);
292	303	return;
293		}
294
295	4468	i = VSA_Get_Proto(sa);
296	4468	if (VSA_Compare(sa, bogo_ip) == 0)
297	1584	vt->good_unix \|= 1;
298	2884	else if (i == AF_INET)
299	2884	vt->good_ipv4 \|= 1;
300	0	else if (i == AF_INET6)
301	0	vt->good_ipv6 \|= 1;
302		else
303	0	WRONG("Wrong probe protocol family");
304
305	4468	t_now = VTIM_real();
306	4468	tmo = (int)round((t_end - t_now) * 1e3);
307	4468	if (tmo <= 0) {
308	56	bprintf(vt->resp_buf,
309		"Open timeout %.3fs exceeded by %.3fs",
310		vt->timeout, t_now - t_end);
311	56	VTCP_close(&s);
312	56	return;
313		}
314
315	4412	Lck_Lock(&vbp_mtx);
316	4412	if (vt->backend != NULL)
317	4412	proxy_header = vt->backend->proxy_header;
318		else
319	0	proxy_header = -1;
320	4412	Lck_Unlock(&vbp_mtx);
321
322	4412	if (proxy_header < 0) {
323	0	bprintf(vt->resp_buf, "%s", "No backend");
324	0	VTCP_close(&s);
325	0	return;
326		}
327
328		/* Send the PROXY header */
329	4412	assert(proxy_header <= 2);
330	4412	if (proxy_header == 1) {
331	154	if (vbp_write_proxy_v1(vt, &s) != 0)
332	0	return;
333	4412	} else if (proxy_header == 2 &&
334	192	vbp_write(vt, &s, vbp_proxy_local, sizeof vbp_proxy_local) != 0)
335	0	return;
336
337		/* Send the request */
338	4412	if (vbp_write(vt, &s, vt->req, vt->req_len) != 0)
339	0	return;
340
341	4412	vt->good_xmit \|= 1;
342
343	4412	pfd->fd = s;
344	4412	rlen = 0;
345	8807	while (1) {
346	8807	pfd->events = POLLIN;
347	8807	pfd->revents = 0;
348	8807	t_now = VTIM_real();
349	8807	tmo = (int)round((t_end - t_now) * 1e3);
350	8807	if (tmo <= 0) {
351	0	bprintf(vt->resp_buf,
352		"Poll timeout %.3fs exceeded by %.3fs",
353		vt->timeout, t_now - t_end);
354	0	i = -1;
355	0	break;
356		}
357	8807	i = poll(pfd, 1, tmo);
358	8807	if (i <= 0) {
359	505	if (!i) {
360	505	if (!vt->exp_close)
361	107	break;
362	398	errno = ETIMEDOUT;
363	398	}
364	398	bprintf(vt->resp_buf, "Poll error %d (%s)",
365		errno, VAS_errtxt(errno));
366	398	i = -1;
367	398	break;
368		}
369	8302	if (rlen < sizeof vt->resp_buf)
370	15444	i = read(s, vt->resp_buf + rlen,
371	7722	sizeof vt->resp_buf - rlen);
372		else
373	580	i = read(s, buf, sizeof buf);
374	8302	VTCP_Assert(i);
375	8302	if (i <= 0) {
376	3907	if (i < 0)
377	32	bprintf(vt->resp_buf, "Read error %d (%s)",
378		errno, VAS_errtxt(errno));
379	3907	break;
380		}
381	4395	rlen += i;
382		}
383
384	4412	VTCP_close(&s);
385
386	4412	if (i < 0) {
387		/* errno reported above */
388	430	vt->err_recv \|= 1;
389	430	return;
390		}
391
392	3982	if (rlen == 0) {
393	150	bprintf(vt->resp_buf, "%s", "Empty response");
394	150	return;
395		}
396
397		/* So we have a good receive ... */
398	3832	t_now = VTIM_real();
399	3832	vt->last = t_now - t_start;
400	3832	vt->good_recv \|= 1;
401
402		/* Now find out if we like the response */
403	3832	vt->resp_buf[sizeof vt->resp_buf - 1] = '\0';
404	3832	p = strchr(vt->resp_buf, '\r');
405	3832	if (p != NULL)
406	3832	*p = '\0';
407	3832	p = strchr(vt->resp_buf, '\n');
408	3832	if (p != NULL)
409	0	*p = '\0';
410
411	3832	i = sscanf(vt->resp_buf, "HTTP/%*f %u ", &resp);
412
413	3832	if (i == 1 && resp == vt->exp_status)
414	3507	vt->happy \|= 1;
415	4771	}
416
417		/*--------------------------------------------------------------------
418		*/
419		static void
420	5540	vbp_heap_insert(struct vbp_target *vt)
421		{
422		// Lck_AssertHeld(&vbp_mtx);
423	5540	VBH_insert(vbp_heap, vt);
424	5540	if (VBH_root(vbp_heap) == vt)
425	5133	PTOK(pthread_cond_signal(&vbp_cond));
426	5540	}
427
428		/*--------------------------------------------------------------------
429		*/
430
431		static void v_matchproto_(task_func_t)
432	4715	vbp_task(struct worker wrk, void priv)
433		{
434		struct vbp_target *vt;
435
436	4715	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
437	4715	CAST_OBJ_NOTNULL(vt, priv, VBP_TARGET_MAGIC);
438
439	4715	AN(vt->running);
440	4715	AN(vt->req);
441	4715	assert(vt->req_len > 0);
442
443	4715	vbp_start_poke(vt);
444	4715	vbp_poke(vt);
445	4715	vbp_has_poked(vt);
446	4715	VBP_Update_Backend(vt);
447
448	4715	Lck_Lock(&vbp_mtx);
449	4715	if (vt->running < 0) {
450	0	assert(vt->heap_idx == VBH_NOIDX);
451	0	vbp_delete(vt);
452	0	} else {
453	4715	vt->running = 0;
454	4715	if (vt->heap_idx != VBH_NOIDX) {
455	4690	vt->due = VTIM_real() + vt->interval;
456	4690	VBH_delete(vbp_heap, vt->heap_idx);
457	4690	vbp_heap_insert(vt);
458	4690	}
459		}
460	4715	Lck_Unlock(&vbp_mtx);
461	4715	}
462
463		/*--------------------------------------------------------------------
464		*/
465
466		static void * v_matchproto_(bgthread_t)
467	0	vbp_thread(struct worker wrk, void priv)
468		{
469		vtim_real now, nxt;
470		struct vbp_target *vt;
471		int r;
472
473	0	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
474	0	AZ(priv);
475	0	Lck_Lock(&vbp_mtx);
476	42384	while (1) {
477	42384	now = VTIM_real();
478	42384	vt = VBH_root(vbp_heap);
479	42384	if (vt == NULL) {
480	22750	nxt = 8.192 + now;
481	22750	(void)Lck_CondWaitUntil(&vbp_cond, &vbp_mtx, nxt);
482	42384	} else if (vt->due > now) {
483	11956	nxt = vt->due;
484	11956	vt = NULL;
485	11956	(void)Lck_CondWaitUntil(&vbp_cond, &vbp_mtx, nxt);
486	11956	} else {
487	7678	VBH_delete(vbp_heap, vt->heap_idx);
488	7678	vt->due = now + vt->interval;
489	7678	VBH_insert(vbp_heap, vt);
490	7678	if (!vt->running) {
491	4796	vt->running = 1;
492	4796	vt->task->func = vbp_task;
493	4796	vt->task->priv = vt;
494	4796	Lck_Unlock(&vbp_mtx);
495	4796	r = Pool_Task_Any(vt->task, TASK_QUEUE_REQ);
496	4796	Lck_Lock(&vbp_mtx);
497	4796	if (r)
498	0	vt->running = 0;
499	4796	}