GCOV - Varnish Cache


/*-
 * Copyright (c) 2006 Verdens Gang AS
 * Copyright (c) 2006-2015 Varnish Software AS
 * All rights reserved.
 *
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
 *
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include "config.h"

#include "cache_varnishd.h"

#include <stdio.h>
#include <stdlib.h>

#include "vgz.h"
#include "vsl_priv.h"
#include "vmb.h"

#include "common/heritage.h"
#include "common/vsmw.h"

/* ------------------------------------------------------------
 * strands helpers - move elsewhere?
 */

static unsigned
strands_len(const struct strands *s)
{
        unsigned r = 0;
        int i;

        for (i = 0; i < s->n; i++) {
                if (s->p[i] == NULL || *s->p[i] == '\0')
                        continue;
                r += strlen(s->p[i]);
        }

        return (r);
}

/*
 * like VRT_Strands(), but truncating instead of failing for end of buffer
 *
 * returns number of bytes including NUL
 */
static unsigned
strands_cat(char *buf, unsigned bufl, const struct strands *s)
{
        unsigned l = 0, ll;
        int i;

        /* NUL-terminated */
        assert(bufl > 0);
        bufl--;

        for (i = 0; i < s->n && bufl > 0; i++) {
                if (s->p[i] == NULL || *s->p[i] == '\0')
                        continue;
                ll = vmin_t(unsigned, strlen(s->p[i]), bufl);
                memcpy(buf, s->p[i], ll);
                l += ll;
                buf += ll;
                bufl -= ll;
        }
        *buf = '\0';    /* NUL-terminated */
        return (l + 1);
}

/* These cannot be struct lock, which depends on vsm/vsl working */
static pthread_mutex_t vsl_mtx;
static pthread_mutex_t vsc_mtx;
static pthread_mutex_t vsm_mtx;

static struct VSL_head          *vsl_head;
static const uint32_t           *vsl_end;
static uint32_t                 *vsl_ptr;
static unsigned                 vsl_segment_n;
static ssize_t                  vsl_segsize;

struct VSC_main *VSC_C_main;

static void
vsl_sanity(const struct vsl_log *vsl)
{
        AN(vsl);
        AN(vsl->wlp);
        AN(vsl->wlb);
        AN(vsl->wle);
        assert(vsl->wlb <= vsl->wlp);
        assert(vsl->wlp <= vsl->wle);
}

/*--------------------------------------------------------------------
 * Check if the VSL_tag is masked by parameter bitmap
 */

static inline int
vsl_tag_is_masked(enum VSL_tag_e tag)
{
        volatile uint8_t *bm = &cache_param->vsl_mask[0];
        uint8_t b;

        assert(tag > SLT__Bogus);
        assert(tag < SLT__Reserved);
        bm += ((unsigned)tag >> 3);
        b = (0x80 >> ((unsigned)tag & 7));
        return (*bm & b);
}

int
VSL_tag_is_masked(enum VSL_tag_e tag)
{
        return (vsl_tag_is_masked(tag));
}

/*--------------------------------------------------------------------
 * Lay down a header fields, and return pointer to the next record
 */

static inline uint32_t *
vsl_hdr(enum VSL_tag_e tag, uint32_t *p, unsigned len, vxid_t vxid)
{

        AZ((uintptr_t)p & 0x3);
        assert(tag > SLT__Bogus);
        assert(tag < SLT__Reserved);
        AZ(len & ~VSL_LENMASK);

        p[2] = vxid.vxid >> 32;
        p[1] = vxid.vxid;
        p[0] = (((unsigned)tag & VSL_IDMASK) << VSL_IDSHIFT) |
             (VSL_VERSION_3 << VSL_VERSHIFT) |
             len;
        return (VSL_END(p, len));
}

/*--------------------------------------------------------------------
 * Space available in a VSL buffer when accounting for overhead
 */

static unsigned
vsl_space(const struct vsl_log *vsl)
{
        ptrdiff_t mlen;

        mlen = vsl->wle - vsl->wlp;
        assert(mlen >= 0);
        if (mlen < VSL_OVERHEAD + 1)
                return (0);
        mlen -= VSL_OVERHEAD;
        mlen *= sizeof *vsl->wlp;
        if (mlen > cache_param->vsl_reclen)
                mlen = cache_param->vsl_reclen;
        return(mlen);
}

/*--------------------------------------------------------------------
 * Wrap the VSL buffer
 */

static void
vsl_wrap(void)
{

        assert(vsl_ptr >= vsl_head->log);
        assert(vsl_ptr < vsl_end);
        vsl_segment_n += VSL_SEGMENTS - (vsl_segment_n % VSL_SEGMENTS);
        assert(vsl_segment_n % VSL_SEGMENTS == 0);
        vsl_head->offset[0] = 0;
        vsl_head->log[0] = VSL_ENDMARKER;
        VWMB();
        if (vsl_ptr != vsl_head->log) {
                *vsl_ptr = VSL_WRAPMARKER;
                vsl_ptr = vsl_head->log;
        }
        vsl_head->segment_n = vsl_segment_n;
        VSC_C_main->shm_cycles++;
}

/*--------------------------------------------------------------------
 * Reserve bytes for a record, wrap if necessary
 */

static uint32_t *
vsl_get(unsigned len, unsigned records, unsigned flushes)
{
        uint32_t *p;
        int err;

        err = pthread_mutex_trylock(&vsl_mtx);
        if (err == EBUSY) {
                PTOK(pthread_mutex_lock(&vsl_mtx));
                VSC_C_main->shm_cont++;
        } else {
                AZ(err);
        }
        assert(vsl_ptr < vsl_end);
        AZ((uintptr_t)vsl_ptr & 0x3);

        VSC_C_main->shm_writes++;
        VSC_C_main->shm_flushes += flushes;
        VSC_C_main->shm_records += records;
        VSC_C_main->shm_bytes +=
            VSL_BYTES(VSL_OVERHEAD + VSL_WORDS((uint64_t)len));

        /* Wrap if necessary */
        if (VSL_END(vsl_ptr, len) >= vsl_end)
                vsl_wrap();

        p = vsl_ptr;
        vsl_ptr = VSL_END(vsl_ptr, len);
        assert(vsl_ptr < vsl_end);
        AZ((uintptr_t)vsl_ptr & 0x3);

        *vsl_ptr = VSL_ENDMARKER;

        while ((vsl_ptr - vsl_head->log) / vsl_segsize >
            vsl_segment_n % VSL_SEGMENTS) {
                vsl_segment_n++;
                vsl_head->offset[vsl_segment_n % VSL_SEGMENTS] =
                    vsl_ptr - vsl_head->log;
        }

        PTOK(pthread_mutex_unlock(&vsl_mtx));
        /* Implicit VWMB() in mutex op ensures ENDMARKER and new table
           values are seen before new segment number */
        vsl_head->segment_n = vsl_segment_n;

        return (p);
}

/*--------------------------------------------------------------------
 * Stick a finished record into VSL.
 */

static void
vslr(enum VSL_tag_e tag, vxid_t vxid, const char *b, unsigned len)
{
        uint32_t *p;
        unsigned mlen;

        mlen = cache_param->vsl_reclen;

        /* Truncate */
        if (len > mlen)
                len = mlen;

        p = vsl_get(len, 1, 0);

        memcpy(p + VSL_OVERHEAD, b, len);

        /*
         * the vxid needs to be written before the barrier to
         * ensure it is valid when vsl_hdr() marks the record
         * ready by writing p[0]
         */
        p[2] = vxid.vxid >> 32;
        p[1] = vxid.vxid;
        VWMB();
        (void)vsl_hdr(tag, p, len, vxid);
}

/*--------------------------------------------------------------------
 * Add a unbuffered record to VSL
 *
 * NB: This variant should be used sparingly and only for low volume
 * NB: since it significantly adds to the mutex load on the VSL.
 */

void
VSLv(enum VSL_tag_e tag, vxid_t vxid, const char *fmt, va_list ap)
{
        unsigned n, mlen = cache_param->vsl_reclen;
        char buf[mlen];

        AN(fmt);
        if (vsl_tag_is_masked(tag))
                return;

        if (strchr(fmt, '%') == NULL) {
                vslr(tag, vxid, fmt, strlen(fmt) + 1);
        } else {
                n = vsnprintf(buf, mlen, fmt, ap);
                n = vmin(n, mlen - 1);
                buf[n++] = '\0'; /* NUL-terminated */
                vslr(tag, vxid, buf, n);
        }

}

void
VSLs(enum VSL_tag_e tag, vxid_t vxid, const struct strands *s)
{
        unsigned n, mlen = cache_param->vsl_reclen;
        char buf[mlen];

        if (vsl_tag_is_masked(tag))
                return;

        n = strands_cat(buf, mlen, s);

        vslr(tag, vxid, buf, n);
}

void
VSL(enum VSL_tag_e tag, vxid_t vxid, const char *fmt, ...)
{
        va_list ap;

        va_start(ap, fmt);
        VSLv(tag, vxid, fmt, ap);
        va_end(ap);
}

/*--------------------------------------------------------------------*/

void
VSL_Flush(struct vsl_log *vsl, int overflow)
{
        uint32_t *p;
        unsigned l;

        vsl_sanity(vsl);
        l = pdiff(vsl->wlb, vsl->wlp);
        if (l == 0)
                return;

        assert(l >= 8);

        p = vsl_get(l, vsl->wlr, overflow);

        memcpy(p + VSL_OVERHEAD, vsl->wlb, l);
        p[1] = l;
        VWMB();
        p[0] = ((((unsigned)SLT__Batch & 0xff) << VSL_IDSHIFT));
        vsl->wlp = vsl->wlb;
        vsl->wlr = 0;
}

/*--------------------------------------------------------------------
 * Buffered VSLs
 */

static char *
vslb_get(struct vsl_log *vsl, enum VSL_tag_e tag, unsigned *length)
{
        unsigned mlen = cache_param->vsl_reclen;
        char *retval;

        vsl_sanity(vsl);
        if (*length < mlen)
                mlen = *length;

        if (VSL_END(vsl->wlp, mlen) > vsl->wle)
                VSL_Flush(vsl, 1);

        retval = VSL_DATA(vsl->wlp);

        /* If it still doesn't fit, truncate */
        if (VSL_END(vsl->wlp, mlen) > vsl->wle)
                mlen = vsl_space(vsl);

        vsl->wlp = vsl_hdr(tag, vsl->wlp, mlen, vsl->wid);
        vsl->wlr++;
        *length = mlen;
        return (retval);
}

static void
vslb_simple(struct vsl_log *vsl, enum VSL_tag_e tag,
    unsigned length, const char *str)
{
        char *p;

        if (length == 0)
                length = strlen(str);
        length += 1; // NUL
        p = vslb_get(vsl, tag, &length);
        memcpy(p, str, length - 1);
        p[length - 1] = '\0';

        if (DO_DEBUG(DBG_SYNCVSL))
                VSL_Flush(vsl, 0);
}

/*--------------------------------------------------------------------
 * VSL-buffered-txt
 */

void
VSLbt(struct vsl_log *vsl, enum VSL_tag_e tag, txt t)
{

        Tcheck(t);
        if (vsl_tag_is_masked(tag))
                return;

        vslb_simple(vsl, tag, Tlen(t), t.b);
}

/*--------------------------------------------------------------------
 * VSL-buffered-strands
 */
void
VSLbs(struct vsl_log *vsl, enum VSL_tag_e tag, const struct strands *s)
{
        unsigned l;
        char *p;

        if (vsl_tag_is_masked(tag))
                return;

        l = strands_len(s) + 1;
        p = vslb_get(vsl, tag, &l);

        (void)strands_cat(p, l, s);

        if (DO_DEBUG(DBG_SYNCVSL))
                VSL_Flush(vsl, 0);
}

/*--------------------------------------------------------------------
 * VSL-buffered
 */

void
VSLbv(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, va_list ap)
{
        char *p, *p1;
        unsigned n = 0, mlen;
        va_list ap2;

        AN(fmt);
        if (vsl_tag_is_masked(tag))
                return;

        /*
         * If there are no printf-expansions, don't waste time expanding them
         */
        if (strchr(fmt, '%') == NULL) {
                vslb_simple(vsl, tag, 0, fmt);
                return;
        }

        /*
         * If the format is trivial, deal with it directly
         */
        if (!strcmp(fmt, "%s")) {
                p1 = va_arg(ap, char *);
                vslb_simple(vsl, tag, 0, p1);
                return;
        }

        vsl_sanity(vsl);

        mlen = vsl_space(vsl);

        // First attempt, only if any space at all
        if (mlen > 0) {
                p = VSL_DATA(vsl->wlp);
                va_copy(ap2, ap);
                n = vsnprintf(p, mlen, fmt, ap2);
                va_end(ap2);
        }

        // Second attempt, if a flush might help
        if (mlen == 0 || (n + 1 > mlen && n + 1 <= cache_param->vsl_reclen)) {
                VSL_Flush(vsl, 1);
                mlen = vsl_space(vsl);
                p = VSL_DATA(vsl->wlp);
                n = vsnprintf(p, mlen, fmt, ap);
        }
        if (n + 1 < mlen)
                mlen = n + 1;
        (void)vslb_get(vsl, tag, &mlen);

        if (DO_DEBUG(DBG_SYNCVSL))
                VSL_Flush(vsl, 0);
}

void
VSLb(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...)
{
        va_list ap;

        vsl_sanity(vsl);
        va_start(ap, fmt);
        VSLbv(vsl, tag, fmt, ap);
        va_end(ap);
}

#define Tf6 "%ju.%06ju"
#define Ta6(t) (uintmax_t)floor((t)), (uintmax_t)floor((t) * 1e6) % 1000000U

void
VSLb_ts(struct vsl_log *vsl, const char *event, vtim_real first,
    vtim_real *pprev, vtim_real now)
{

        /*
         * XXX: Make an option to turn off some unnecessary timestamp
         * logging. This must be done carefully because some functions
         * (e.g. V1L_Open) takes the last timestamp as its initial
         * value for timeout calculation.
         */
        vsl_sanity(vsl);
        AN(event);
        AN(pprev);
        assert(!isnan(now) && now != 0.);
        VSLb(vsl, SLT_Timestamp, "%s: " Tf6 " " Tf6 " " Tf6,
            event, Ta6(now), Ta6(now - first), Ta6(now - *pprev));
        *pprev = now;
}

void
VSLb_bin(struct vsl_log *vsl, enum VSL_tag_e tag, ssize_t len, const void *ptr)
{
        unsigned mlen;
        char *p;

        vsl_sanity(vsl);
        AN(ptr);
        if (vsl_tag_is_masked(tag))
                return;
        mlen = cache_param->vsl_reclen;

        /* Truncate */
        len = vmin_t(ssize_t, len, mlen);

        assert(vsl->wlp <= vsl->wle);

        /* Flush if necessary */
        if (VSL_END(vsl->wlp, len) > vsl->wle)
                VSL_Flush(vsl, 1);
        assert(VSL_END(vsl->wlp, len) <= vsl->wle);
        p = VSL_DATA(vsl->wlp);
        memcpy(p, ptr, len);
        vsl->wlp = vsl_hdr(tag, vsl->wlp, len, vsl->wid);
        assert(vsl->wlp <= vsl->wle);
        vsl->wlr++;

        if (DO_DEBUG(DBG_SYNCVSL))
                VSL_Flush(vsl, 0);
}

/*--------------------------------------------------------------------
 * Setup a VSL buffer, allocate space if none provided.
 */

void
VSL_Setup(struct vsl_log *vsl, void *ptr, size_t len)
{

        if (ptr == NULL) {
                len = cache_param->vsl_buffer;
                ptr = malloc(len);
                AN(ptr);
        }
        vsl->wlp = ptr;
        vsl->wlb = ptr;
        vsl->wle = ptr;
        vsl->wle += len / sizeof(*vsl->wle);
        vsl->wlr = 0;
        vsl->wid = NO_VXID;
        vsl_sanity(vsl);
}

/*--------------------------------------------------------------------*/

void
VSL_ChgId(struct vsl_log *vsl, const char *typ, const char *why, vxid_t vxid)
{
        vxid_t ovxid;

        vsl_sanity(vsl);
        ovxid = vsl->wid;
        VSLb(vsl, SLT_Link, "%s %ju %s", typ, VXID(vxid), why);
        VSL_End(vsl);
        vsl->wid = vxid;
        VSLb(vsl, SLT_Begin, "%s %ju %s", typ, VXID(ovxid), why);
}

/*--------------------------------------------------------------------*/

void
VSL_End(struct vsl_log *vsl)
{
        txt t;
        char p[] = "";

        vsl_sanity(vsl);
        assert(!IS_NO_VXID(vsl->wid));
        t.b = p;
        t.e = p;
        VSLbt(vsl, SLT_End, t);
        VSL_Flush(vsl, 0);
        vsl->wid = NO_VXID;
}

static void v_matchproto_(vsm_lock_f)
vsm_vsc_lock(void)
{
        PTOK(pthread_mutex_lock(&vsc_mtx));
}

static void v_matchproto_(vsm_lock_f)
vsm_vsc_unlock(void)
{
        PTOK(pthread_mutex_unlock(&vsc_mtx));
}

static void v_matchproto_(vsm_lock_f)
vsm_vsmw_lock(void)
{
        PTOK(pthread_mutex_lock(&vsm_mtx));
}

static void v_matchproto_(vsm_lock_f)
vsm_vsmw_unlock(void)
{
        PTOK(pthread_mutex_unlock(&vsm_mtx));
}

/*--------------------------------------------------------------------*/

void
VSM_Init(void)
{
        unsigned u;

        assert(UINT_MAX % VSL_SEGMENTS == VSL_SEGMENTS - 1);

        PTOK(pthread_mutex_init(&vsl_mtx, &mtxattr_errorcheck));
        PTOK(pthread_mutex_init(&vsc_mtx, &mtxattr_errorcheck));
        PTOK(pthread_mutex_init(&vsm_mtx, &mtxattr_errorcheck));

        vsc_lock = vsm_vsc_lock;
        vsc_unlock = vsm_vsc_unlock;
        vsmw_lock = vsm_vsmw_lock;
        vsmw_unlock = vsm_vsmw_unlock;

        heritage.proc_vsmw = VSMW_New(heritage.vsm_fd, 0640, "_.index");
        AN(heritage.proc_vsmw);

        VSC_C_main = VSC_main_New(NULL, NULL, "");
        AN(VSC_C_main);

        AN(heritage.proc_vsmw);
        vsl_head = VSMW_Allocf(heritage.proc_vsmw, NULL, VSL_CLASS,
            cache_param->vsl_space, VSL_CLASS);
        AN(vsl_head);
        vsl_segsize = ((cache_param->vsl_space - sizeof *vsl_head) /
            sizeof *vsl_end) / VSL_SEGMENTS;
        vsl_end = vsl_head->log + vsl_segsize * VSL_SEGMENTS;
        /* Make segment_n always overflow on first log wrap to make any
           problems with regard to readers on that event visible */
        vsl_segment_n = UINT_MAX - (VSL_SEGMENTS - 1);
        AZ(vsl_segment_n % VSL_SEGMENTS);
        vsl_ptr = vsl_head->log;
        *vsl_ptr = VSL_ENDMARKER;

        memset(vsl_head, 0, sizeof *vsl_head);
        vsl_head->segsize = vsl_segsize;
        vsl_head->offset[0] = 0;
        vsl_head->segment_n = vsl_segment_n;
        for (u = 1; u < VSL_SEGMENTS; u++)
                vsl_head->offset[u] = -1;
        VWMB();
        memcpy(vsl_head->marker, VSL_HEAD_MARKER, sizeof vsl_head->marker);
}

		varnish-cache/bin/varnishd/cache/cache_shmlog.c
0		/*-
1		* Copyright (c) 2006 Verdens Gang AS
2		* Copyright (c) 2006-2015 Varnish Software AS
3		* All rights reserved.
4		*
5		* Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
6		*
7		* SPDX-License-Identifier: BSD-2-Clause
8		*
9		* Redistribution and use in source and binary forms, with or without
10		* modification, are permitted provided that the following conditions
11		* are met:
12		* 1. Redistributions of source code must retain the above copyright
13		* notice, this list of conditions and the following disclaimer.
14		* 2. Redistributions in binary form must reproduce the above copyright
15		* notice, this list of conditions and the following disclaimer in the
16		* documentation and/or other materials provided with the distribution.
17		*
18		* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19		* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20		* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21		* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
22		* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23		* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24		* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25		* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26		* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27		* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28		* SUCH DAMAGE.
29		*/
30
31		#include "config.h"
32
33		#include "cache_varnishd.h"
34
35		#include <stdio.h>
36		#include <stdlib.h>
37
38		#include "vgz.h"
39		#include "vsl_priv.h"
40		#include "vmb.h"
41
42		#include "common/heritage.h"
43		#include "common/vsmw.h"
44
45		/* ------------------------------------------------------------
46		* strands helpers - move elsewhere?
47		*/
48
49		static unsigned
50	1879599	strands_len(const struct strands *s)
51		{
52	1879599	unsigned r = 0;
53		int i;
54
55	3761574	for (i = 0; i < s->n; i++) {
56	1881975	if (s->p[i] == NULL \|\| *s->p[i] == '\0')
57	180196	continue;
58	1701779	r += strlen(s->p[i]);
59	1701779	}
60
61	1879599	return (r);
62		}
63
64		/*
65		* like VRT_Strands(), but truncating instead of failing for end of buffer
66		*
67		* returns number of bytes including NUL
68		*/
69		static unsigned
70	1886942	strands_cat(char buf, unsigned bufl, const struct strands s)
71		{
72	1886942	unsigned l = 0, ll;
73		int i;
74
75		/* NUL-terminated */
76	1886942	assert(bufl > 0);
77	1886942	bufl--;
78
79	3599808	for (i = 0; i < s->n && bufl > 0; i++) {
80	1712876	if (s->p[i] == NULL \|\| *s->p[i] == '\0')
81	104	continue;
82	1712786	ll = vmin_t(unsigned, strlen(s->p[i]), bufl);
83	1712786	memcpy(buf, s->p[i], ll);
84	1712786	l += ll;
85	1712786	buf += ll;
86	1712786	bufl -= ll;
87	1712786	}
88	1886932	buf = '\0'; / NUL-terminated */
89	1886932	return (l + 1);
90		}
91
92		/* These cannot be struct lock, which depends on vsm/vsl working */
93		static pthread_mutex_t vsl_mtx;
94		static pthread_mutex_t vsc_mtx;
95		static pthread_mutex_t vsm_mtx;
96
97		static struct VSL_head *vsl_head;
98		static const uint32_t *vsl_end;
99		static uint32_t *vsl_ptr;
100		static unsigned vsl_segment_n;
101		static ssize_t vsl_segsize;
102
103		struct VSC_main *VSC_C_main;
104
105		static void
106	18327965	vsl_sanity(const struct vsl_log *vsl)
107		{
108	18327965	AN(vsl);
109	18327965	AN(vsl->wlp);
110	18327965	AN(vsl->wlb);
111	18327965	AN(vsl->wle);
112	18327965	assert(vsl->wlb <= vsl->wlp);
113	18327965	assert(vsl->wlp <= vsl->wle);
114	18327965	}
115
116		/*--------------------------------------------------------------------
117		* Check if the VSL_tag is masked by parameter bitmap
118		*/
119
120		static inline int
121	13839255	vsl_tag_is_masked(enum VSL_tag_e tag)
122		{
123	13839255	volatile uint8_t *bm = &cache_param->vsl_mask[0];
124		uint8_t b;
125
126	13839255	assert(tag > SLT__Bogus);
127	13839255	assert(tag < SLT__Reserved);
128	13839255	bm += ((unsigned)tag >> 3);
129	13839255	b = (0x80 >> ((unsigned)tag & 7));
130	13839255	return (*bm & b);
131		}
132
133		int
134	43907	VSL_tag_is_masked(enum VSL_tag_e tag)
135		{
136	43907	return (vsl_tag_is_masked(tag));
137		}
138
139		/*--------------------------------------------------------------------
140		* Lay down a header fields, and return pointer to the next record
141		*/
142
143		static inline uint32_t *
144	11366910	vsl_hdr(enum VSL_tag_e tag, uint32_t *p, unsigned len, vxid_t vxid)
145		{
146
147	11366910	AZ((uintptr_t)p & 0x3);
148	11366910	assert(tag > SLT__Bogus);
149	11366910	assert(tag < SLT__Reserved);
150	11366910	AZ(len & ~VSL_LENMASK);
151
152	11366910	p[2] = vxid.vxid >> 32;
153	11366910	p[1] = vxid.vxid;
154	22733820	p[0] = (((unsigned)tag & VSL_IDMASK) << VSL_IDSHIFT) \|
155	11366910	(VSL_VERSION_3 << VSL_VERSHIFT) \|
156	11366910	len;
157	11366910	return (VSL_END(p, len));
158		}
159
160		/*--------------------------------------------------------------------
161		* Space available in a VSL buffer when accounting for overhead
162		*/
163
164		static unsigned
165	2763310	vsl_space(const struct vsl_log *vsl)
166		{
167		ptrdiff_t mlen;
168
169	2763310	mlen = vsl->wle - vsl->wlp;
170	2763310	assert(mlen >= 0);
171	2763310	if (mlen < VSL_OVERHEAD + 1)
172	120	return (0);
173	2763190	mlen -= VSL_OVERHEAD;
174	2763190	mlen = sizeof vsl->wlp;
175	2763190	if (mlen > cache_param->vsl_reclen)
176	2762378	mlen = cache_param->vsl_reclen;
177	2763190	return(mlen);
178	2763310	}
179
180		/*--------------------------------------------------------------------
181		* Wrap the VSL buffer
182		*/
183
184		static void
185	0	vsl_wrap(void)
186		{
187
188	0	assert(vsl_ptr >= vsl_head->log);
189	0	assert(vsl_ptr < vsl_end);
190	0	vsl_segment_n += VSL_SEGMENTS - (vsl_segment_n % VSL_SEGMENTS);
191	0	assert(vsl_segment_n % VSL_SEGMENTS == 0);
192	0	vsl_head->offset[0] = 0;
193	0	vsl_head->log[0] = VSL_ENDMARKER;
194	0	VWMB();
195	0	if (vsl_ptr != vsl_head->log) {
196	0	*vsl_ptr = VSL_WRAPMARKER;
197	0	vsl_ptr = vsl_head->log;
198	0	}
199	0	vsl_head->segment_n = vsl_segment_n;
200	0	VSC_C_main->shm_cycles++;
201	0	}
202
203		/*--------------------------------------------------------------------
204		* Reserve bytes for a record, wrap if necessary
205		*/
206
207		static uint32_t *
208	3264055	vsl_get(unsigned len, unsigned records, unsigned flushes)
209		{
210		uint32_t *p;
211		int err;
212
213	3264055	err = pthread_mutex_trylock(&vsl_mtx);
214	3264055	if (err == EBUSY) {
215	17152	PTOK(pthread_mutex_lock(&vsl_mtx));
216	17152	VSC_C_main->shm_cont++;
217	17152	} else {
218	3246903	AZ(err);
219		}
220	3264055	assert(vsl_ptr < vsl_end);
221	3264055	AZ((uintptr_t)vsl_ptr & 0x3);
222
223	3264055	VSC_C_main->shm_writes++;
224	3264055	VSC_C_main->shm_flushes += flushes;
225	3264055	VSC_C_main->shm_records += records;
226	3264055	VSC_C_main->shm_bytes +=
227	3264055	VSL_BYTES(VSL_OVERHEAD + VSL_WORDS((uint64_t)len));
228
229		/* Wrap if necessary */
230	3264055	if (VSL_END(vsl_ptr, len) >= vsl_end)
231	0	vsl_wrap();
232
233	3264055	p = vsl_ptr;
234	3264055	vsl_ptr = VSL_END(vsl_ptr, len);
235	3264055	assert(vsl_ptr < vsl_end);
236	3264055	AZ((uintptr_t)vsl_ptr & 0x3);
237
238	3264055	*vsl_ptr = VSL_ENDMARKER;
239
240	3264135	while ((vsl_ptr - vsl_head->log) / vsl_segsize >
241	3264135	vsl_segment_n % VSL_SEGMENTS) {
242	80	vsl_segment_n++;
243	80	vsl_head->offset[vsl_segment_n % VSL_SEGMENTS] =
244	80	vsl_ptr - vsl_head->log;
245		}
246
247	3264055	PTOK(pthread_mutex_unlock(&vsl_mtx));
248		/* Implicit VWMB() in mutex op ensures ENDMARKER and new table
249		values are seen before new segment number */
250	3264055	vsl_head->segment_n = vsl_segment_n;
251
252	3264055	return (p);
253		}
254
255		/*--------------------------------------------------------------------
256		* Stick a finished record into VSL.
257		*/
258
259		static void
260	2222200	vslr(enum VSL_tag_e tag, vxid_t vxid, const char *b, unsigned len)
261		{
262		uint32_t *p;
263		unsigned mlen;
264
265	2222200	mlen = cache_param->vsl_reclen;
266
267		/* Truncate */
268	2222200	if (len > mlen)
269	0	len = mlen;
270
271	2222200	p = vsl_get(len, 1, 0);
272
273	2222200	memcpy(p + VSL_OVERHEAD, b, len);
274
275		/*
276		* the vxid needs to be written before the barrier to
277		* ensure it is valid when vsl_hdr() marks the record
278		* ready by writing p[0]
279		*/
280	2222200	p[2] = vxid.vxid >> 32;
281	2222200	p[1] = vxid.vxid;
282	2222200	VWMB();
283	2222200	(void)vsl_hdr(tag, p, len, vxid);
284	2222200	}
285
286		/*--------------------------------------------------------------------
287		* Add a unbuffered record to VSL
288		*
289		* NB: This variant should be used sparingly and only for low volume
290		* NB: since it significantly adds to the mutex load on the VSL.
291		*/
292
293		void
294	2993853	VSLv(enum VSL_tag_e tag, vxid_t vxid, const char *fmt, va_list ap)
295		{
296	2993853	unsigned n, mlen = cache_param->vsl_reclen;
297	2993853	char buf[mlen];
298
299	2993853	AN(fmt);
300	2993853	if (vsl_tag_is_masked(tag))
301	779342	return;
302
303	2214511	if (strchr(fmt, '%') == NULL) {
304	37896	vslr(tag, vxid, fmt, strlen(fmt) + 1);
305	37896	} else {
306	2176615	n = vsnprintf(buf, mlen, fmt, ap);
307	2176615	n = vmin(n, mlen - 1);
308	2176615	buf[n++] = '\0'; /* NUL-terminated */
309	2176615	vslr(tag, vxid, buf, n);
310		}
311
312	2993853	}
313
314		void
315	7680	VSLs(enum VSL_tag_e tag, vxid_t vxid, const struct strands *s)
316		{
317	7680	unsigned n, mlen = cache_param->vsl_reclen;
318	7680	char buf[mlen];
319
320	7680	if (vsl_tag_is_masked(tag))
321	0	return;
322
323	7680	n = strands_cat(buf, mlen, s);
324
325	7680	vslr(tag, vxid, buf, n);
326	7680	}
327
328		void
329	2928067	VSL(enum VSL_tag_e tag, vxid_t vxid, const char *fmt, ...)
330		{
331		va_list ap;
332
333	2928067	va_start(ap, fmt);
334	2928067	VSLv(tag, vxid, fmt, ap);
335	2928067	va_end(ap);
336	2928067	}
337
338		/--------------------------------------------------------------------/
339
340		void
341	1331697	VSL_Flush(struct vsl_log *vsl, int overflow)
342		{
343		uint32_t *p;
344		unsigned l;
345
346	1331697	vsl_sanity(vsl);
347	1331697	l = pdiff(vsl->wlb, vsl->wlp);
348	1331697	if (l == 0)
349	292312	return;
350
351	1039385	assert(l >= 8);
352
353	1039385	p = vsl_get(l, vsl->wlr, overflow);
354
355	1039385	memcpy(p + VSL_OVERHEAD, vsl->wlb, l);
356	1039385	p[1] = l;
357	1039385	VWMB();
358	1039385	p[0] = ((((unsigned)SLT__Batch & 0xff) << VSL_IDSHIFT));
359	1039385	vsl->wlp = vsl->wlb;
360	1039385	vsl->wlr = 0;
361	1331697	}
362
363		/*--------------------------------------------------------------------
364		* Buffered VSLs
365		*/
366
367		static char *
368	9092947	vslb_get(struct vsl_log vsl, enum VSL_tag_e tag, unsigned length)
369		{
370	9092947	unsigned mlen = cache_param->vsl_reclen;
371		char *retval;
372
373	9092947	vsl_sanity(vsl);
374	9092947	if (*length < mlen)
375	9070567	mlen = *length;
376
377	9092947	if (VSL_END(vsl->wlp, mlen) > vsl->wle)
378	320	VSL_Flush(vsl, 1);
379
380	9092947	retval = VSL_DATA(vsl->wlp);
381
382		/* If it still doesn't fit, truncate */
383	9092947	if (VSL_END(vsl->wlp, mlen) > vsl->wle)
384	80	mlen = vsl_space(vsl);
385
386	9092947	vsl->wlp = vsl_hdr(tag, vsl->wlp, mlen, vsl->wid);
387	9092947	vsl->wlr++;
388	9092947	*length = mlen;
389	9092947	return (retval);
390		}
391
392		static void
393	4454163	vslb_simple(struct vsl_log *vsl, enum VSL_tag_e tag,
394		unsigned length, const char *str)
395		{
396		char *p;
397
398	4454163	if (length == 0)
399	284227	length = strlen(str);
400	4454163	length += 1; // NUL
401	4454163	p = vslb_get(vsl, tag, &length);
402	4454163	memcpy(p, str, length - 1);
403	4454163	p[length - 1] = '\0';
404
405	4454163	if (DO_DEBUG(DBG_SYNCVSL))
406	371374	VSL_Flush(vsl, 0);
407	4454163	}
408
409		/*--------------------------------------------------------------------
410		* VSL-buffered-txt
411		*/
412
413		void
414	5092034	VSLbt(struct vsl_log *vsl, enum VSL_tag_e tag, txt t)
415		{
416
417	5092034	Tcheck(t);
418	5092034	if (vsl_tag_is_masked(tag))
419	683619	return;
420
421	4408415	vslb_simple(vsl, tag, Tlen(t), t.b);
422	5092034	}
423
424		/*--------------------------------------------------------------------
425		* VSL-buffered-strands
426		*/
427		void
428	2229818	VSLbs(struct vsl_log vsl, enum VSL_tag_e tag, const struct strands s)
429		{
430		unsigned l;
431		char *p;
432
433	2229818	if (vsl_tag_is_masked(tag))
434	350150	return;
435
436	1879668	l = strands_len(s) + 1;
437	1879668	p = vslb_get(vsl, tag, &l);
438
439	1879668	(void)strands_cat(p, l, s);
440
441	1879668	if (DO_DEBUG(DBG_SYNCVSL))
442	147250	VSL_Flush(vsl, 0);
443	2229818	}
444
445		/*--------------------------------------------------------------------
446		* VSL-buffered
447		*/
448
449		void
450	3377839	VSLbv(struct vsl_log vsl, enum VSL_tag_e tag, const char fmt, va_list ap)
451		{
452		char p, p1;
453	3377839	unsigned n = 0, mlen;
454		va_list ap2;
455
456	3377839	AN(fmt);
457	3377839	if (vsl_tag_is_masked(tag))
458	568689	return;
459
460		/*
461		* If there are no printf-expansions, don't waste time expanding them
462		*/
463	2809150	if (strchr(fmt, '%') == NULL) {
464	44947	vslb_simple(vsl, tag, 0, fmt);
465	44947	return;
466		}
467
468		/*
469		* If the format is trivial, deal with it directly
470		*/
471	2764203	if (!strcmp(fmt, "%s")) {
472	720	p1 = va_arg(ap, char *);
473	720	vslb_simple(vsl, tag, 0, p1);
474	720	return;
475		}
476
477	2763483	vsl_sanity(vsl);
478
479	2763483	mlen = vsl_space(vsl);
480
481		// First attempt, only if any space at all
482	2763483	if (mlen > 0) {
483	2762761	p = VSL_DATA(vsl->wlp);
484	2762761	va_copy(ap2, ap);
485	2762761	n = vsnprintf(p, mlen, fmt, ap2);
486	2762761	va_end(ap2);
487	2762761	}
488
489		// Second attempt, if a flush might help
490	2763483	if (mlen == 0 \|\| (n + 1 > mlen && n + 1 <= cache_param->vsl_reclen)) {
491	2402	VSL_Flush(vsl, 1);
492	2402	mlen = vsl_space(vsl);
493	2402	p = VSL_DATA(vsl->wlp);
494	2402	n = vsnprintf(p, mlen, fmt, ap);
495	2402	}
496	2763883	if (n + 1 < mlen)
497	2753257	mlen = n + 1;
498	2763883	(void)vslb_get(vsl, tag, &mlen);
499