GCOV - Varnish Cache


/*-
 * Copyright (c) 2006 Verdens Gang AS
 * Copyright (c) 2006-2015 Varnish Software AS
 * All rights reserved.
 *
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
 *
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include "config.h"

#include "cache_varnishd.h"

#include <stdio.h>
#include <stdlib.h>

#include "vgz.h"
#include "vsl_priv.h"
#include "vmb.h"

#include "common/heritage.h"
#include "common/vsmw.h"

/* ------------------------------------------------------------
 * strands helpers - move elsewhere?
 */

static unsigned
strands_len(const struct strands *s)
{
        unsigned r = 0;
        int i;

        for (i = 0; i < s->n; i++) {
                if (s->p[i] == NULL || *s->p[i] == '\0')
                        continue;
                r += strlen(s->p[i]);
        }

        return (r);
}

/*
 * like VRT_Strands(), but truncating instead of failing for end of buffer
 *
 * returns number of bytes including NUL
 */
static unsigned
strands_cat(char *buf, unsigned bufl, const struct strands *s)
{
        unsigned l = 0, ll;
        int i;

        /* NUL-terminated */
        assert(bufl > 0);
        bufl--;

        for (i = 0; i < s->n && bufl > 0; i++) {
                if (s->p[i] == NULL || *s->p[i] == '\0')
                        continue;
                ll = vmin_t(unsigned, strlen(s->p[i]), bufl);
                memcpy(buf, s->p[i], ll);
                l += ll;
                buf += ll;
                bufl -= ll;
        }
        *buf = '\0';    /* NUL-terminated */
        return (l + 1);
}

/* These cannot be struct lock, which depends on vsm/vsl working */
static pthread_mutex_t vsl_mtx;
static pthread_mutex_t vsc_mtx;
static pthread_mutex_t vsm_mtx;

static struct VSL_head          *vsl_head;
static const uint32_t           *vsl_end;
static uint32_t                 *vsl_ptr;
static unsigned                 vsl_segment_n;
static ssize_t                  vsl_segsize;

struct VSC_main *VSC_C_main;

static void
vsl_sanity(const struct vsl_log *vsl)
{
        AN(vsl);
        AN(vsl->wlp);
        AN(vsl->wlb);
        AN(vsl->wle);
        assert(vsl->wlb <= vsl->wlp);
        assert(vsl->wlp <= vsl->wle);
}

/*--------------------------------------------------------------------
 * Check if the VSL_tag is masked by parameter bitmap
 */

static inline int
vsl_tag_is_masked(enum VSL_tag_e tag)
{
        volatile uint8_t *bm = &cache_param->vsl_mask[0];
        uint8_t b;

        assert(tag > SLT__Bogus);
        assert(tag < SLT__Reserved);
        bm += ((unsigned)tag >> 3);
        b = (0x80 >> ((unsigned)tag & 7));
        return (*bm & b);
}

int
VSL_tag_is_masked(enum VSL_tag_e tag)
{
        return (vsl_tag_is_masked(tag));
}

/*--------------------------------------------------------------------
 * Lay down a header fields, and return pointer to the next record
 */

static inline uint32_t *
vsl_hdr(enum VSL_tag_e tag, uint32_t *p, unsigned len, vxid_t vxid)
{

        AZ((uintptr_t)p & 0x3);
        assert(tag > SLT__Bogus);
        assert(tag < SLT__Reserved);
        AZ(len & ~VSL_LENMASK);

        p[2] = vxid.vxid >> 32;
        p[1] = vxid.vxid;
        p[0] = (((unsigned)tag & VSL_IDMASK) << VSL_IDSHIFT) |
             (VSL_VERSION_3 << VSL_VERSHIFT) |
             len;
        return (VSL_END(p, len));
}

/*--------------------------------------------------------------------
 * Space available in a VSL buffer when accounting for overhead
 */

static unsigned
vsl_space(const struct vsl_log *vsl)
{
        ptrdiff_t mlen;

        mlen = vsl->wle - vsl->wlp;
        assert(mlen >= 0);
        if (mlen < VSL_OVERHEAD + 1)
                return (0);
        mlen -= VSL_OVERHEAD;
        mlen *= sizeof *vsl->wlp;
        if (mlen > cache_param->vsl_reclen)
                mlen = cache_param->vsl_reclen;
        return(mlen);
}

/*--------------------------------------------------------------------
 * Wrap the VSL buffer
 */

static void
vsl_wrap(void)
{

        assert(vsl_ptr >= vsl_head->log);
        assert(vsl_ptr < vsl_end);
        vsl_segment_n += VSL_SEGMENTS - (vsl_segment_n % VSL_SEGMENTS);
        assert(vsl_segment_n % VSL_SEGMENTS == 0);
        vsl_head->offset[0] = 0;
        vsl_head->log[0] = VSL_ENDMARKER;
        VWMB();
        if (vsl_ptr != vsl_head->log) {
                *vsl_ptr = VSL_WRAPMARKER;
                vsl_ptr = vsl_head->log;
        }
        vsl_head->segment_n = vsl_segment_n;
        VSC_C_main->shm_cycles++;
}

/*--------------------------------------------------------------------
 * Reserve bytes for a record, wrap if necessary
 */

static uint32_t *
vsl_get(unsigned len, unsigned records, unsigned flushes)
{
        uint32_t *p;
        int err;

        err = pthread_mutex_trylock(&vsl_mtx);
        if (err == EBUSY) {
                PTOK(pthread_mutex_lock(&vsl_mtx));
                VSC_C_main->shm_cont++;
        } else {
                AZ(err);
        }
        assert(vsl_ptr < vsl_end);
        AZ((uintptr_t)vsl_ptr & 0x3);

        VSC_C_main->shm_writes++;
        VSC_C_main->shm_flushes += flushes;
        VSC_C_main->shm_records += records;
        VSC_C_main->shm_bytes +=
            VSL_BYTES(VSL_OVERHEAD + VSL_WORDS((uint64_t)len));

        /* Wrap if necessary */
        if (VSL_END(vsl_ptr, len) >= vsl_end)
                vsl_wrap();

        p = vsl_ptr;
        vsl_ptr = VSL_END(vsl_ptr, len);
        assert(vsl_ptr < vsl_end);
        AZ((uintptr_t)vsl_ptr & 0x3);

        *vsl_ptr = VSL_ENDMARKER;

        while ((vsl_ptr - vsl_head->log) / vsl_segsize >
            vsl_segment_n % VSL_SEGMENTS) {
                vsl_segment_n++;
                vsl_head->offset[vsl_segment_n % VSL_SEGMENTS] =
                    vsl_ptr - vsl_head->log;
        }

        PTOK(pthread_mutex_unlock(&vsl_mtx));
        /* Implicit VWMB() in mutex op ensures ENDMARKER and new table
           values are seen before new segment number */
        vsl_head->segment_n = vsl_segment_n;

        return (p);
}

/*--------------------------------------------------------------------
 * Stick a finished record into VSL.
 */

static void
vslr(enum VSL_tag_e tag, vxid_t vxid, const char *b, unsigned len)
{
        uint32_t *p;
        unsigned mlen;

        mlen = cache_param->vsl_reclen;

        /* Truncate */
        if (len > mlen)
                len = mlen;

        p = vsl_get(len, 1, 0);

        memcpy(p + VSL_OVERHEAD, b, len);

        /*
         * the vxid needs to be written before the barrier to
         * ensure it is valid when vsl_hdr() marks the record
         * ready by writing p[0]
         */
        p[2] = vxid.vxid >> 32;
        p[1] = vxid.vxid;
        VWMB();
        (void)vsl_hdr(tag, p, len, vxid);
}

/*--------------------------------------------------------------------
 * Add a unbuffered record to VSL
 *
 * NB: This variant should be used sparingly and only for low volume
 * NB: since it significantly adds to the mutex load on the VSL.
 */

void
VSLv(enum VSL_tag_e tag, vxid_t vxid, const char *fmt, va_list ap)
{
        unsigned n, mlen = cache_param->vsl_reclen;
        char buf[mlen];

        AN(fmt);
        if (vsl_tag_is_masked(tag))
                return;

        if (strchr(fmt, '%') == NULL) {
                vslr(tag, vxid, fmt, strlen(fmt) + 1);
        } else {
                n = vsnprintf(buf, mlen, fmt, ap);
                n = vmin(n, mlen - 1);
                buf[n++] = '\0'; /* NUL-terminated */
                vslr(tag, vxid, buf, n);
        }

}

void
VSLs(enum VSL_tag_e tag, vxid_t vxid, const struct strands *s)
{
        unsigned n, mlen = cache_param->vsl_reclen;
        char buf[mlen];

        if (vsl_tag_is_masked(tag))
                return;

        n = strands_cat(buf, mlen, s);

        vslr(tag, vxid, buf, n);
}

void
VSL(enum VSL_tag_e tag, vxid_t vxid, const char *fmt, ...)
{
        va_list ap;

        va_start(ap, fmt);
        VSLv(tag, vxid, fmt, ap);
        va_end(ap);
}

/*--------------------------------------------------------------------*/

void
VSL_Flush(struct vsl_log *vsl, int overflow)
{
        uint32_t *p;
        unsigned l;

        vsl_sanity(vsl);
        l = pdiff(vsl->wlb, vsl->wlp);
        if (l == 0)
                return;

        assert(l >= 8);

        p = vsl_get(l, vsl->wlr, overflow);

        memcpy(p + VSL_OVERHEAD, vsl->wlb, l);
        p[1] = l;
        VWMB();
        p[0] = ((((unsigned)SLT__Batch & 0xff) << VSL_IDSHIFT));
        vsl->wlp = vsl->wlb;
        vsl->wlr = 0;
}

/*--------------------------------------------------------------------
 * Buffered VSLs
 */

static char *
vslb_get(struct vsl_log *vsl, enum VSL_tag_e tag, unsigned *length)
{
        unsigned mlen = cache_param->vsl_reclen;
        char *retval;

        vsl_sanity(vsl);
        if (*length < mlen)
                mlen = *length;

        if (VSL_END(vsl->wlp, mlen) > vsl->wle)
                VSL_Flush(vsl, 1);

        retval = VSL_DATA(vsl->wlp);

        /* If it still doesn't fit, truncate */
        if (VSL_END(vsl->wlp, mlen) > vsl->wle)
                mlen = vsl_space(vsl);

        vsl->wlp = vsl_hdr(tag, vsl->wlp, mlen, vsl->wid);
        vsl->wlr++;
        *length = mlen;
        return (retval);
}

static void
vslb_simple(struct vsl_log *vsl, enum VSL_tag_e tag,
    unsigned length, const char *str)
{
        char *p;

        if (length == 0)
                length = strlen(str);
        length += 1; // NUL
        p = vslb_get(vsl, tag, &length);
        memcpy(p, str, length - 1);
        p[length - 1] = '\0';

        if (DO_DEBUG(DBG_SYNCVSL))
                VSL_Flush(vsl, 0);
}

/*--------------------------------------------------------------------
 * VSL-buffered-txt
 */

void
VSLbt(struct vsl_log *vsl, enum VSL_tag_e tag, txt t)
{

        Tcheck(t);
        if (vsl_tag_is_masked(tag))
                return;

        vslb_simple(vsl, tag, Tlen(t), t.b);
}

/*--------------------------------------------------------------------
 * VSL-buffered-strands
 */
void
VSLbs(struct vsl_log *vsl, enum VSL_tag_e tag, const struct strands *s)
{
        unsigned l;
        char *p;

        if (vsl_tag_is_masked(tag))
                return;

        l = strands_len(s) + 1;
        p = vslb_get(vsl, tag, &l);

        (void)strands_cat(p, l, s);

        if (DO_DEBUG(DBG_SYNCVSL))
                VSL_Flush(vsl, 0);
}

/*--------------------------------------------------------------------
 * VSL-buffered
 */

void
VSLbv(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, va_list ap)
{
        char *p, *p1;
        unsigned n = 0, mlen;
        va_list ap2;

        AN(fmt);
        if (vsl_tag_is_masked(tag))
                return;

        /*
         * If there are no printf-expansions, don't waste time expanding them
         */
        if (strchr(fmt, '%') == NULL) {
                vslb_simple(vsl, tag, 0, fmt);
                return;
        }

        /*
         * If the format is trivial, deal with it directly
         */
        if (!strcmp(fmt, "%s")) {
                p1 = va_arg(ap, char *);
                vslb_simple(vsl, tag, 0, p1);
                return;
        }

        vsl_sanity(vsl);

        mlen = vsl_space(vsl);

        // First attempt, only if any space at all
        if (mlen > 0) {
                p = VSL_DATA(vsl->wlp);
                va_copy(ap2, ap);
                n = vsnprintf(p, mlen, fmt, ap2);
                va_end(ap2);
        }

        // Second attempt, if a flush might help
        if (mlen == 0 || (n + 1 > mlen && n + 1 <= cache_param->vsl_reclen)) {
                VSL_Flush(vsl, 1);
                mlen = vsl_space(vsl);
                p = VSL_DATA(vsl->wlp);
                n = vsnprintf(p, mlen, fmt, ap);
        }
        if (n + 1 < mlen)
                mlen = n + 1;
        (void)vslb_get(vsl, tag, &mlen);

        if (DO_DEBUG(DBG_SYNCVSL))
                VSL_Flush(vsl, 0);
}

void
VSLb(struct vsl_log *vsl, enum VSL_tag_e tag, const char *fmt, ...)
{
        va_list ap;

        vsl_sanity(vsl);
        va_start(ap, fmt);
        VSLbv(vsl, tag, fmt, ap);
        va_end(ap);
}

#define Tf6 "%ju.%06ju"
#define Ta6(t) (uintmax_t)floor((t)), (uintmax_t)floor((t) * 1e6) % 1000000U

void
VSLb_ts(struct vsl_log *vsl, const char *event, vtim_real first,
    vtim_real *pprev, vtim_real now)
{

        /*
         * XXX: Make an option to turn off some unnecessary timestamp
         * logging. This must be done carefully because some functions
         * (e.g. V1L_Open) takes the last timestamp as its initial
         * value for timeout calculation.
         */
        vsl_sanity(vsl);
        AN(event);
        AN(pprev);
        assert(!isnan(now) && now != 0.);
        VSLb(vsl, SLT_Timestamp, "%s: " Tf6 " " Tf6 " " Tf6,
            event, Ta6(now), Ta6(now - first), Ta6(now - *pprev));
        *pprev = now;
}

void
VSLb_bin(struct vsl_log *vsl, enum VSL_tag_e tag, ssize_t len, const void *ptr)
{
        unsigned mlen;
        char *p;

        vsl_sanity(vsl);
        AN(ptr);
        if (vsl_tag_is_masked(tag))
                return;
        mlen = cache_param->vsl_reclen;

        /* Truncate */
        len = vmin_t(ssize_t, len, mlen);

        assert(vsl->wlp <= vsl->wle);

        /* Flush if necessary */
        if (VSL_END(vsl->wlp, len) > vsl->wle)
                VSL_Flush(vsl, 1);
        assert(VSL_END(vsl->wlp, len) <= vsl->wle);
        p = VSL_DATA(vsl->wlp);
        memcpy(p, ptr, len);
        vsl->wlp = vsl_hdr(tag, vsl->wlp, len, vsl->wid);
        assert(vsl->wlp <= vsl->wle);
        vsl->wlr++;

        if (DO_DEBUG(DBG_SYNCVSL))
                VSL_Flush(vsl, 0);
}

/*--------------------------------------------------------------------
 * Setup a VSL buffer, allocate space if none provided.
 */

void
VSL_Setup(struct vsl_log *vsl, void *ptr, size_t len)
{

        if (ptr == NULL) {
                len = cache_param->vsl_buffer;
                ptr = malloc(len);
                AN(ptr);
        }
        vsl->wlp = ptr;
        vsl->wlb = ptr;
        vsl->wle = ptr;
        vsl->wle += len / sizeof(*vsl->wle);
        vsl->wlr = 0;
        vsl->wid = NO_VXID;
        vsl_sanity(vsl);
}

/*--------------------------------------------------------------------*/

void
VSL_ChgId(struct vsl_log *vsl, const char *typ, const char *why, vxid_t vxid)
{
        vxid_t ovxid;

        vsl_sanity(vsl);
        ovxid = vsl->wid;
        VSLb(vsl, SLT_Link, "%s %ju %s", typ, VXID(vxid), why);
        VSL_End(vsl);
        vsl->wid = vxid;
        VSLb(vsl, SLT_Begin, "%s %ju %s", typ, VXID(ovxid), why);
}

/*--------------------------------------------------------------------*/

void
VSL_End(struct vsl_log *vsl)
{
        txt t;
        char p[] = "";

        vsl_sanity(vsl);
        assert(!IS_NO_VXID(vsl->wid));
        t.b = p;
        t.e = p;
        VSLbt(vsl, SLT_End, t);
        VSL_Flush(vsl, 0);
        vsl->wid = NO_VXID;
}

static void v_matchproto_(vsm_lock_f)
vsm_vsc_lock(void)
{
        PTOK(pthread_mutex_lock(&vsc_mtx));
}

static void v_matchproto_(vsm_lock_f)
vsm_vsc_unlock(void)
{
        PTOK(pthread_mutex_unlock(&vsc_mtx));
}

static void v_matchproto_(vsm_lock_f)
vsm_vsmw_lock(void)
{
        PTOK(pthread_mutex_lock(&vsm_mtx));
}

static void v_matchproto_(vsm_lock_f)
vsm_vsmw_unlock(void)
{
        PTOK(pthread_mutex_unlock(&vsm_mtx));
}

/*--------------------------------------------------------------------*/

void
VSM_Init(void)
{
        unsigned u;

        assert(UINT_MAX % VSL_SEGMENTS == VSL_SEGMENTS - 1);

        PTOK(pthread_mutex_init(&vsl_mtx, &mtxattr_errorcheck));
        PTOK(pthread_mutex_init(&vsc_mtx, &mtxattr_errorcheck));
        PTOK(pthread_mutex_init(&vsm_mtx, &mtxattr_errorcheck));

        vsc_lock = vsm_vsc_lock;
        vsc_unlock = vsm_vsc_unlock;
        vsmw_lock = vsm_vsmw_lock;
        vsmw_unlock = vsm_vsmw_unlock;

        heritage.proc_vsmw = VSMW_New(heritage.vsm_fd, 0640, "_.index");
        AN(heritage.proc_vsmw);

        VSC_C_main = VSC_main_New(NULL, NULL, "");
        AN(VSC_C_main);

        AN(heritage.proc_vsmw);
        vsl_head = VSMW_Allocf(heritage.proc_vsmw, NULL, VSL_CLASS,
            cache_param->vsl_space, VSL_CLASS);
        AN(vsl_head);
        vsl_segsize = ((cache_param->vsl_space - sizeof *vsl_head) /
            sizeof *vsl_end) / VSL_SEGMENTS;
        vsl_end = vsl_head->log + vsl_segsize * VSL_SEGMENTS;
        /* Make segment_n always overflow on first log wrap to make any
           problems with regard to readers on that event visible */
        vsl_segment_n = UINT_MAX - (VSL_SEGMENTS - 1);
        AZ(vsl_segment_n % VSL_SEGMENTS);
        vsl_ptr = vsl_head->log;
        *vsl_ptr = VSL_ENDMARKER;

        memset(vsl_head, 0, sizeof *vsl_head);
        vsl_head->segsize = vsl_segsize;
        vsl_head->offset[0] = 0;
        vsl_head->segment_n = vsl_segment_n;
        for (u = 1; u < VSL_SEGMENTS; u++)
                vsl_head->offset[u] = -1;
        VWMB();
        memcpy(vsl_head->marker, VSL_HEAD_MARKER, sizeof vsl_head->marker);
}

		varnish-cache/bin/varnishd/cache/cache_shmlog.c
0		/*-
1		* Copyright (c) 2006 Verdens Gang AS
2		* Copyright (c) 2006-2015 Varnish Software AS
3		* All rights reserved.
4		*
5		* Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
6		*
7		* SPDX-License-Identifier: BSD-2-Clause
8		*
9		* Redistribution and use in source and binary forms, with or without
10		* modification, are permitted provided that the following conditions
11		* are met:
12		* 1. Redistributions of source code must retain the above copyright
13		* notice, this list of conditions and the following disclaimer.
14		* 2. Redistributions in binary form must reproduce the above copyright
15		* notice, this list of conditions and the following disclaimer in the
16		* documentation and/or other materials provided with the distribution.
17		*
18		* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19		* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20		* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21		* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
22		* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23		* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24		* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25		* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26		* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27		* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28		* SUCH DAMAGE.
29		*/
30
31		#include "config.h"
32
33		#include "cache_varnishd.h"
34
35		#include <stdio.h>
36		#include <stdlib.h>
37
38		#include "vgz.h"
39		#include "vsl_priv.h"
40		#include "vmb.h"
41
42		#include "common/heritage.h"
43		#include "common/vsmw.h"
44
45		/* ------------------------------------------------------------
46		* strands helpers - move elsewhere?
47		*/
48
49		static unsigned
50	1892652	strands_len(const struct strands *s)
51		{
52	1892652	unsigned r = 0;
53		int i;
54
55	3787676	for (i = 0; i < s->n; i++) {
56	1895024	if (s->p[i] == NULL \|\| *s->p[i] == '\0')
57	180884	continue;
58	1714140	r += strlen(s->p[i]);
59	1714140	}
60
61	1892652	return (r);
62		}
63
64		/*
65		* like VRT_Strands(), but truncating instead of failing for end of buffer
66		*
67		* returns number of bytes including NUL
68		*/
69		static unsigned
70	1900130	strands_cat(char buf, unsigned bufl, const struct strands s)
71		{
72	1900130	unsigned l = 0, ll;
73		int i;
74
75		/* NUL-terminated */
76	1900130	assert(bufl > 0);
77	1900130	bufl--;
78
79	3625471	for (i = 0; i < s->n && bufl > 0; i++) {
80	1725355	if (s->p[i] == NULL \|\| *s->p[i] == '\0')
81	94	continue;
82	1725261	ll = vmin_t(unsigned, strlen(s->p[i]), bufl);
83	1725261	memcpy(buf, s->p[i], ll);
84	1725261	l += ll;
85	1725261	buf += ll;
86	1725261	bufl -= ll;
87	1725261	}
88	1900116	buf = '\0'; / NUL-terminated */
89	1900116	return (l + 1);
90		}
91
92		/* These cannot be struct lock, which depends on vsm/vsl working */
93		static pthread_mutex_t vsl_mtx;
94		static pthread_mutex_t vsc_mtx;
95		static pthread_mutex_t vsm_mtx;
96
97		static struct VSL_head *vsl_head;
98		static const uint32_t *vsl_end;
99		static uint32_t *vsl_ptr;
100		static unsigned vsl_segment_n;
101		static ssize_t vsl_segsize;
102
103		struct VSC_main *VSC_C_main;
104
105		static void
106	18515413	vsl_sanity(const struct vsl_log *vsl)
107		{
108	18515413	AN(vsl);
109	18515413	AN(vsl->wlp);
110	18515413	AN(vsl->wlb);
111	18515413	AN(vsl->wle);
112	18515413	assert(vsl->wlb <= vsl->wlp);
113	18515413	assert(vsl->wlp <= vsl->wle);
114	18515413	}
115
116		/*--------------------------------------------------------------------
117		* Check if the VSL_tag is masked by parameter bitmap
118		*/
119
120		static inline int
121	13909490	vsl_tag_is_masked(enum VSL_tag_e tag)
122		{
123	13909490	volatile uint8_t *bm = &cache_param->vsl_mask[0];
124		uint8_t b;
125
126	13909490	assert(tag > SLT__Bogus);
127	13909490	assert(tag < SLT__Reserved);
128	13909490	bm += ((unsigned)tag >> 3);
129	13909490	b = (0x80 >> ((unsigned)tag & 7));
130	13909490	return (*bm & b);
131		}
132
133		int
134	43893	VSL_tag_is_masked(enum VSL_tag_e tag)
135		{
136	43893	return (vsl_tag_is_masked(tag));
137		}
138
139		/*--------------------------------------------------------------------
140		* Lay down a header fields, and return pointer to the next record
141		*/
142
143		static inline uint32_t *
144	11434112	vsl_hdr(enum VSL_tag_e tag, uint32_t *p, unsigned len, vxid_t vxid)
145		{
146
147	11434112	AZ((uintptr_t)p & 0x3);
148	11434112	assert(tag > SLT__Bogus);
149	11434112	assert(tag < SLT__Reserved);
150	11434112	AZ(len & ~VSL_LENMASK);
151
152	11434112	p[2] = vxid.vxid >> 32;
153	11434112	p[1] = vxid.vxid;
154	22868224	p[0] = (((unsigned)tag & VSL_IDMASK) << VSL_IDSHIFT) \|
155	11434112	(VSL_VERSION_3 << VSL_VERSHIFT) \|
156	11434112	len;
157	11434112	return (VSL_END(p, len));
158		}
159
160		/*--------------------------------------------------------------------
161		* Space available in a VSL buffer when accounting for overhead
162		*/
163
164		static unsigned
165	2786822	vsl_space(const struct vsl_log *vsl)
166		{
167		ptrdiff_t mlen;
168
169	2786822	mlen = vsl->wle - vsl->wlp;
170	2786822	assert(mlen >= 0);
171	2786822	if (mlen < VSL_OVERHEAD + 1)
172	120	return (0);
173	2786702	mlen -= VSL_OVERHEAD;
174	2786702	mlen = sizeof vsl->wlp;
175	2786702	if (mlen > cache_param->vsl_reclen)
176	2785878	mlen = cache_param->vsl_reclen;
177	2786702	return(mlen);
178	2786822	}
179
180		/*--------------------------------------------------------------------
181		* Wrap the VSL buffer
182		*/
183
184		static void
185	0	vsl_wrap(void)
186		{
187
188	0	assert(vsl_ptr >= vsl_head->log);
189	0	assert(vsl_ptr < vsl_end);
190	0	vsl_segment_n += VSL_SEGMENTS - (vsl_segment_n % VSL_SEGMENTS);
191	0	assert(vsl_segment_n % VSL_SEGMENTS == 0);
192	0	vsl_head->offset[0] = 0;
193	0	vsl_head->log[0] = VSL_ENDMARKER;
194	0	VWMB();
195	0	if (vsl_ptr != vsl_head->log) {
196	0	*vsl_ptr = VSL_WRAPMARKER;
197	0	vsl_ptr = vsl_head->log;
198	0	}
199	0	vsl_head->segment_n = vsl_segment_n;
200	0	VSC_C_main->shm_cycles++;
201	0	}
202
203		/*--------------------------------------------------------------------
204		* Reserve bytes for a record, wrap if necessary
205		*/
206
207		static uint32_t *
208	3329503	vsl_get(unsigned len, unsigned records, unsigned flushes)
209		{
210		uint32_t *p;
211		int err;
212
213	3329503	err = pthread_mutex_trylock(&vsl_mtx);
214	3329503	if (err == EBUSY) {
215	18199	PTOK(pthread_mutex_lock(&vsl_mtx));
216	18199	VSC_C_main->shm_cont++;
217	18199	} else {
218	3311304	AZ(err);
219		}
220	3329503	assert(vsl_ptr < vsl_end);
221	3329503	AZ((uintptr_t)vsl_ptr & 0x3);
222
223	3329503	VSC_C_main->shm_writes++;
224	3329503	VSC_C_main->shm_flushes += flushes;
225	3329503	VSC_C_main->shm_records += records;
226	3329503	VSC_C_main->shm_bytes +=
227	3329503	VSL_BYTES(VSL_OVERHEAD + VSL_WORDS((uint64_t)len));
228
229		/* Wrap if necessary */
230	3329503	if (VSL_END(vsl_ptr, len) >= vsl_end)
231	0	vsl_wrap();
232
233	3329503	p = vsl_ptr;
234	3329503	vsl_ptr = VSL_END(vsl_ptr, len);
235	3329503	assert(vsl_ptr < vsl_end);
236	3329503	AZ((uintptr_t)vsl_ptr & 0x3);
237
238	3329503	*vsl_ptr = VSL_ENDMARKER;
239
240	3329583	while ((vsl_ptr - vsl_head->log) / vsl_segsize >
241	3329583	vsl_segment_n % VSL_SEGMENTS) {
242	80	vsl_segment_n++;
243	80	vsl_head->offset[vsl_segment_n % VSL_SEGMENTS] =
244	80	vsl_ptr - vsl_head->log;
245		}
246
247	3329503	PTOK(pthread_mutex_unlock(&vsl_mtx));
248		/* Implicit VWMB() in mutex op ensures ENDMARKER and new table
249		values are seen before new segment number */
250	3329503	vsl_head->segment_n = vsl_segment_n;
251
252	3329503	return (p);
253		}
254
255		/*--------------------------------------------------------------------
256		* Stick a finished record into VSL.
257		*/
258
259		static void
260	2225637	vslr(enum VSL_tag_e tag, vxid_t vxid, const char *b, unsigned len)
261		{
262		uint32_t *p;
263		unsigned mlen;
264
265	2225637	mlen = cache_param->vsl_reclen;
266
267		/* Truncate */
268	2225637	if (len > mlen)
269	0	len = mlen;
270
271	2225637	p = vsl_get(len, 1, 0);
272
273	2225637	memcpy(p + VSL_OVERHEAD, b, len);
274
275		/*
276		* the vxid needs to be written before the barrier to
277		* ensure it is valid when vsl_hdr() marks the record
278		* ready by writing p[0]
279		*/
280	2225637	p[2] = vxid.vxid >> 32;
281	2225637	p[1] = vxid.vxid;
282	2225637	VWMB();
283	2225637	(void)vsl_hdr(tag, p, len, vxid);
284	2225637	}
285
286		/*--------------------------------------------------------------------
287		* Add a unbuffered record to VSL
288		*
289		* NB: This variant should be used sparingly and only for low volume
290		* NB: since it significantly adds to the mutex load on the VSL.
291		*/
292
293		void
294	2999365	VSLv(enum VSL_tag_e tag, vxid_t vxid, const char *fmt, va_list ap)
295		{
296	2999365	unsigned n, mlen = cache_param->vsl_reclen;
297	2999365	char buf[mlen];
298
299	2999365	AN(fmt);
300	2999365	if (vsl_tag_is_masked(tag))
301	781476	return;
302
303	2217889	if (strchr(fmt, '%') == NULL) {
304	37939	vslr(tag, vxid, fmt, strlen(fmt) + 1);
305	37939	} else {
306	2179950	n = vsnprintf(buf, mlen, fmt, ap);
307	2179950	n = vmin(n, mlen - 1);
308	2179950	buf[n++] = '\0'; /* NUL-terminated */
309	2179950	vslr(tag, vxid, buf, n);
310		}
311
312	2999365	}
313
314		void
315	7720	VSLs(enum VSL_tag_e tag, vxid_t vxid, const struct strands *s)
316		{
317	7720	unsigned n, mlen = cache_param->vsl_reclen;
318	7720	char buf[mlen];
319
320	7720	if (vsl_tag_is_masked(tag))
321	0	return;
322
323	7720	n = strands_cat(buf, mlen, s);
324
325	7720	vslr(tag, vxid, buf, n);
326	7720	}
327
328		void
329	2933623	VSL(enum VSL_tag_e tag, vxid_t vxid, const char *fmt, ...)
330		{
331		va_list ap;
332
333	2933623	va_start(ap, fmt);
334	2933623	VSLv(tag, vxid, fmt, ap);
335	2933623	va_end(ap);
336	2933623	}
337
338		/--------------------------------------------------------------------/
339
340		void
341	1402488	VSL_Flush(struct vsl_log *vsl, int overflow)
342		{
343		uint32_t *p;
344		unsigned l;
345
346	1402488	vsl_sanity(vsl);
347	1402488	l = pdiff(vsl->wlb, vsl->wlp);
348	1402488	if (l == 0)
349	301678	return;
350
351	1100810	assert(l >= 8);
352
353	1100810	p = vsl_get(l, vsl->wlr, overflow);
354
355	1100810	memcpy(p + VSL_OVERHEAD, vsl->wlb, l);
356	1100810	p[1] = l;
357	1100810	VWMB();
358	1100810	p[0] = ((((unsigned)SLT__Batch & 0xff) << VSL_IDSHIFT));
359	1100810	vsl->wlp = vsl->wlb;
360	1100810	vsl->wlr = 0;
361	1402488	}
362
363		/*--------------------------------------------------------------------
364		* Buffered VSLs
365		*/
366
367		static char *
368	9156744	vslb_get(struct vsl_log vsl, enum VSL_tag_e tag, unsigned length)
369		{
370	9156744	unsigned mlen = cache_param->vsl_reclen;
371		char *retval;
372
373	9156744	vsl_sanity(vsl);
374	9156744	if (*length < mlen)
375	9134267	mlen = *length;
376
377	9156744	if (VSL_END(vsl->wlp, mlen) > vsl->wle)
378	320	VSL_Flush(vsl, 1);
379
380	9156744	retval = VSL_DATA(vsl->wlp);
381
382		/* If it still doesn't fit, truncate */
383	9156744	if (VSL_END(vsl->wlp, mlen) > vsl->wle)
384	80	mlen = vsl_space(vsl);
385
386	9156744	vsl->wlp = vsl_hdr(tag, vsl->wlp, mlen, vsl->wid);
387	9156744	vsl->wlr++;
388	9156744	*length = mlen;
389	9156744	return (retval);
390		}
391
392		static void
393	4480738	vslb_simple(struct vsl_log *vsl, enum VSL_tag_e tag,
394		unsigned length, const char *str)
395		{
396		char *p;
397
398	4480738	if (length == 0)
399	285547	length = strlen(str);
400	4480738	length += 1; // NUL
401	4480738	p = vslb_get(vsl, tag, &length);
402	4480738	memcpy(p, str, length - 1);
403	4480738	p[length - 1] = '\0';
404
405	4480738	if (DO_DEBUG(DBG_SYNCVSL))
406	397613	VSL_Flush(vsl, 0);
407	4480738	}
408
409		/*--------------------------------------------------------------------
410		* VSL-buffered-txt
411		*/
412
413		void
414	5119017	VSLbt(struct vsl_log *vsl, enum VSL_tag_e tag, txt t)
415		{
416
417	5119017	Tcheck(t);
418	5119017	if (vsl_tag_is_masked(tag))
419	683766	return;
420
421	4435251	vslb_simple(vsl, tag, Tlen(t), t.b);
422	5119017	}
423
424		/*--------------------------------------------------------------------
425		* VSL-buffered-strands
426		*/
427		void
428	2244086	VSLbs(struct vsl_log vsl, enum VSL_tag_e tag, const struct strands s)
429		{
430		unsigned l;
431		char *p;
432
433	2244086	if (vsl_tag_is_masked(tag))
434	351369	return;
435
436	1892717	l = strands_len(s) + 1;
437	1892717	p = vslb_get(vsl, tag, &l);
438
439	1892717	(void)strands_cat(p, l, s);
440
441	1892717	if (DO_DEBUG(DBG_SYNCVSL))
442	160236	VSL_Flush(vsl, 0);
443	2244086	}
444
445		/*--------------------------------------------------------------------
446		* VSL-buffered
447		*/
448
449		void
450	3402998	VSLbv(struct vsl_log vsl, enum VSL_tag_e tag, const char fmt, va_list ap)
451		{
452		char p, p1;
453	3402998	unsigned n = 0, mlen;
454		va_list ap2;
455
456	3402998	AN(fmt);
457	3402998	if (vsl_tag_is_masked(tag))
458	570158	return;
459
460		/*
461		* If there are no printf-expansions, don't waste time expanding them
462		*/
463	2832840	if (strchr(fmt, '%') == NULL) {
464	44945	vslb_simple(vsl, tag, 0, fmt);
465	44945	return;
466		}
467
468		/*
469		* If the format is trivial, deal with it directly
470		*/
471	2787895	if (!strcmp(fmt, "%s")) {
472	720	p1 = va_arg(ap, char *);
473	720	vslb_simple(vsl, tag, 0, p1);
474	720	return;
475		}
476
477	2787175	vsl_sanity(vsl);
478
479	2787175	mlen = vsl_space(vsl);
480
481		// First attempt, only if any space at all
482	2787175	if (mlen > 0) {
483	2786297	p = VSL_DATA(vsl->wlp);
484	2786297	va_copy(ap2, ap);
485	2786297	n = vsnprintf(p, mlen, fmt, ap2);
486	2786297	va_end(ap2);
487	2786297	}
488
489		// Second attempt, if a flush might help
490	2787175	if (mlen == 0 \|\| (n + 1 > mlen && n + 1 <= cache_param->vsl_reclen)) {
491	2804	VSL_Flush(vsl, 1);
492	2804	mlen = vsl_space(vsl);
493	2804	p = VSL_DATA(vsl->wlp);
494	2804	n = vsnprintf(p, mlen, fmt, ap);
495	2804	}
496	2787575	if (n + 1 < mlen)
497	2777143	mlen = n + 1;
498	2787575	(void)vslb_get(vsl, tag, &mlen);
499