GCOV - Varnish Cache


/*-
 * Copyright (c) 2010-2011 Varnish Software AS
 * All rights reserved.
 *
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
 *
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * VSM stuff common to manager and child.
 *
 */

#include "config.h"

#include <fcntl.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/resource.h>
#include <sys/stat.h>

#include "vdef.h"
#include "vas.h"
#include "vsb.h"
#include "miniobj.h"
#include "vqueue.h"

#include "vfil.h"
#include "vrnd.h"

#include "heritage.h"
#include "vsmw.h"

#ifndef MAP_HASSEMAPHORE
#  define MAP_HASSEMAPHORE 0 /* XXX Linux */
#endif

#ifndef MAP_NOSYNC
#  define MAP_NOSYNC 0 /* XXX Linux */
#endif

static void v_matchproto_(vsm_lock_f)
vsmw_dummy_lock(void)
{
}

static int vsmw_haslock;
vsm_lock_f *vsmw_lock = vsmw_dummy_lock;
vsm_lock_f *vsmw_unlock = vsmw_dummy_lock;

#define vsmw_assert_lock()      AN(vsmw_haslock)

#define vsmw_do_lock() vsmw_do_lock_(__func__, __LINE__)

#define vsmw_do_lock_(f, l)                             \
        do {                                    \
                vsmw_lock();                    \
                AZ(vsmw_haslock);               \
                vsmw_haslock = 1;               \
        } while(0)

#define vsmw_do_unlock() vsmw_do_unlock_(__func__, __LINE__)
#define vsmw_do_unlock_(f, l)                           \
        do {                                    \
                AN(vsmw_haslock);               \
                vsmw_haslock = 0;               \
                vsmw_unlock();                  \
        } while(0)

/*--------------------------------------------------------------------*/

struct vsmw_cluster {
        unsigned                        magic;
#define VSMW_CLUSTER_MAGIC              0x28b74c00

        VTAILQ_ENTRY(vsmw_cluster)      list;
        struct vsmwseg                  *cseg;
        char                            *fn;
        size_t                          len;
        void                            *ptr;
        size_t                          next;
        int                             refs;
        int                             named;
};

struct vsmwseg {
        unsigned                        magic;
#define VSMWSEG_MAGIC                   0x7e4ccaea
        VTAILQ_ENTRY(vsmwseg)           list;
        struct vsmw_cluster             *cluster;

        char                            *category;
        size_t                          off;
        size_t                          len;
        char                            *id;
        void                            *ptr;
};

struct vsmw {
        unsigned                        magic;
#define VSMW_MAGIC                      0xc2ca2cd9
        int                             vdirfd;
        int                             mode;
        char                            *idx;
        VTAILQ_HEAD(, vsmw_cluster)     clusters;
        VTAILQ_HEAD(, vsmwseg)          segs;
        struct vsb                      *vsb;
        pid_t                           pid;
        time_t                          birth;
        uint64_t                        nsegs;
        uint64_t                        nsubs;
};

/* Allocations in clusters never start at offset zero */
#define VSM_CLUSTER_OFFSET 16

/*--------------------------------------------------------------------*/

static void
vsmw_idx_head(const struct vsmw *vsmw, int fd)
{
        char buf[64];

        bprintf(buf, "# %jd %jd\n", (intmax_t)vsmw->pid, (intmax_t)vsmw->birth);
        // XXX handle ENOSPC? #2764
        assert(write(fd, buf, strlen(buf)) == strlen(buf));
}

#define ASSERT_SEG_STR(x) do {                  \
                AN(x);                          \
                AZ(strchr(x, '\n'));            \
        } while (0);

static void
vsmw_fmt_index(const struct vsmw *vsmw, const struct vsmwseg *seg, char act)
{

        vsmw_assert_lock();
        CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
        CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
        AN(seg->cluster);
        ASSERT_SEG_STR(seg->category);
        ASSERT_SEG_STR(seg->id);

        VSB_printf(vsmw->vsb, "%c %s %zu %zu %s %s\n",
            act,
            seg->cluster->fn,
            seg->off,
            seg->len,
            seg->category,
            seg->id);
}

/*--------------------------------------------------------------------*/

static void
vsmw_mkent(const struct vsmw *vsmw, const char *pfx)
{
        int fd;
        uint64_t rn;

        AN(pfx);
        vsmw_assert_lock();
        while (1) {
                VSB_clear(vsmw->vsb);
                VSB_printf(vsmw->vsb, "_.%s", pfx);
                AZ(VRND_RandomCrypto(&rn, sizeof rn));
                VSB_printf(vsmw->vsb, ".%016jx", (uintmax_t)rn);
                AZ(VSB_finish(vsmw->vsb));
                fd = openat(vsmw->vdirfd, VSB_data(vsmw->vsb), O_RDONLY);
                if (fd < 0 && errno == ENOENT)
                        return;
                if (fd >= 0)
                        closefd(&fd);
        }
}

/*--------------------------------------------------------------------*/

static void
vsmw_append_record(struct vsmw *vsmw, struct vsmwseg *seg, char act)
{
        int fd;

        vsmw_assert_lock();
        CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
        CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
        fd = openat(vsmw->vdirfd, vsmw->idx, O_APPEND | O_WRONLY);
        assert(fd >= 0);
        VSB_clear(vsmw->vsb);
        vsmw_fmt_index(vsmw, seg, act);
        AZ(VSB_finish(vsmw->vsb));
        XXXAZ(VSB_tofile(vsmw->vsb, fd)); // XXX handle ENOSPC? #2764
        closefd(&fd);
}

/*--------------------------------------------------------------------*/

static void
vsmw_addseg(struct vsmw *vsmw, struct vsmwseg *seg)
{

        vsmw_assert_lock();
        VTAILQ_INSERT_TAIL(&vsmw->segs, seg, list);
        vsmw_append_record(vsmw, seg, '+');
        vsmw->nsegs++;
}

/*--------------------------------------------------------------------*/

static void
vsmw_delseg(struct vsmw *vsmw, struct vsmwseg *seg)
{
        char *t = NULL;
        int fd;
        struct vsmwseg *s2;

        vsmw_assert_lock();
        CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
        CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);

        VTAILQ_REMOVE(&vsmw->segs, seg, list);

        vsmw->nsegs--;
        if (vsmw->nsubs < 10 || vsmw->nsubs * 2 < vsmw->nsegs) {
                vsmw_append_record(vsmw, seg, '-');
                vsmw->nsubs++;
        } else {
                vsmw_mkent(vsmw, vsmw->idx);
                REPLACE(t, VSB_data(vsmw->vsb));
                fd = openat(vsmw->vdirfd,
                    t, O_WRONLY|O_CREAT|O_EXCL, vsmw->mode);
                assert(fd >= 0);
                vsmw_idx_head(vsmw, fd);
                VSB_clear(vsmw->vsb);
                VTAILQ_FOREACH(s2, &vsmw->segs, list)
                        vsmw_fmt_index(vsmw, s2, '+');
                AZ(VSB_finish(vsmw->vsb));
                XXXAZ(VSB_tofile(vsmw->vsb, fd)); // XXX handle ENOSPC? #2764
                closefd(&fd);
                AZ(renameat(vsmw->vdirfd, t, vsmw->vdirfd, vsmw->idx));
                REPLACE(t, NULL);
                vsmw->nsubs = 0;
        }
        REPLACE(seg->category, NULL);
        REPLACE(seg->id, NULL);
        FREE_OBJ(seg);
}

/*--------------------------------------------------------------------*/

#ifdef RLIMIT_MEMLOCK
static void
printlim(const char *name, rlim_t lim)
{

        fprintf(stderr, "Info: %s: ", name);
        if (lim == RLIM_INFINITY)
                fprintf(stderr, "unlimited\n");
        else
                fprintf(stderr, "%ju bytes\n", (uintmax_t)lim);
}

static void
printmemlock(void) {
        struct rlimit rlim;

        AZ(getrlimit(RLIMIT_MEMLOCK, &rlim));
        printlim("max locked memory (soft)", rlim.rlim_cur);
        printlim("max locked memory (hard)", rlim.rlim_max);
}
#else
static void printmemlock(void) {}
#endif

static struct vsmw_cluster *
vsmw_newcluster(struct vsmw *vsmw, size_t len, const char *pfx)
{
        struct vsmw_cluster *vc;
        static int warn = 0;
        int fd;
        size_t ps;

        vsmw_assert_lock();
        ALLOC_OBJ(vc, VSMW_CLUSTER_MAGIC);
        AN(vc);

        vsmw_mkent(vsmw, pfx);
        REPLACE(vc->fn, VSB_data(vsmw->vsb));

        VTAILQ_INSERT_TAIL(&vsmw->clusters, vc, list);

        ps = getpagesize();
        len = RUP2(len, ps);
        vc->len = len;

        fd = openat(vsmw->vdirfd, vc->fn,
            O_RDWR | O_CREAT | O_EXCL, vsmw->mode);
        assert(fd >= 0);

        AZ(VFIL_allocate(fd, (off_t)len, 1));

        vc->ptr = (void *)mmap(NULL, len,
            PROT_READ|PROT_WRITE,
            MAP_HASSEMAPHORE | MAP_NOSYNC | MAP_SHARED,
            fd, 0);

        closefd(&fd);
        assert(vc->ptr != MAP_FAILED);
        if (mlock(vc->ptr, len) && warn++ == 0)  {
                fprintf(stderr, "Warning: mlock() of VSM failed: %s (%d)\n",
                    VAS_errtxt(errno), errno);
                printmemlock();
        }

        return (vc);
}

struct vsmw_cluster *
VSMW_NewCluster(struct vsmw *vsmw, size_t len, const char *pfx)
{
        struct vsmw_cluster *vc;
        struct vsmwseg *seg;

        vsmw_do_lock();
        vc = vsmw_newcluster(vsmw, len + VSM_CLUSTER_OFFSET, pfx);
        AN(vc);
        vc->next += VSM_CLUSTER_OFFSET;

        ALLOC_OBJ(seg, VSMWSEG_MAGIC);
        AN(seg);
        vc->cseg = seg;
        seg->len = vc->len;
        seg->cluster = vc;
        REPLACE(seg->category, "");
        REPLACE(seg->id, "");
        vc->refs++;
        vc->named = 1;
        vsmw_addseg(vsmw, seg);

        vsmw_do_unlock();
        return (vc);
}

static void
vsmw_DestroyCluster_locked(struct vsmw *vsmw, struct vsmw_cluster *vc)
{

        vsmw_assert_lock();
        CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
        CHECK_OBJ_NOTNULL(vc, VSMW_CLUSTER_MAGIC);

        AZ(vc->refs);

        AZ(munmap(vc->ptr, vc->len));
        if (vc->named)
                vsmw_delseg(vsmw, vc->cseg);
        vc->cseg = 0;

        VTAILQ_REMOVE(&vsmw->clusters, vc, list);
        if (unlinkat(vsmw->vdirfd, vc->fn, 0))
                assert (errno == ENOENT);
        REPLACE(vc->fn, NULL);
        FREE_OBJ(vc);
}

void
VSMW_DestroyCluster(struct vsmw *vsmw, struct vsmw_cluster **vsmcp)
{
        struct vsmw_cluster *vc;

        TAKE_OBJ_NOTNULL(vc, vsmcp, VSMW_CLUSTER_MAGIC);

        vsmw_do_lock();
        if (--vc->refs == 0)
                vsmw_DestroyCluster_locked(vsmw, vc);
        vsmw_do_unlock();
}

/*--------------------------------------------------------------------*/

void *
VSMW_Allocv(struct vsmw *vsmw, struct vsmw_cluster *vc,
    const char *category, size_t payload, const char *prefix,
    const char *fmt, va_list va)
{
        struct vsmwseg *seg;
        ssize_t l;

        vsmw_do_lock();
        CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);

        ALLOC_OBJ(seg, VSMWSEG_MAGIC);
        AN(seg);
        REPLACE(seg->category, category);
        seg->len = PRNDUP(payload);

        VSB_clear(vsmw->vsb);
        if (prefix != NULL) {
                assert(prefix[0] != '\0');
                VSB_cat(vsmw->vsb, prefix);
                if (fmt[0] != '\0')
                        VSB_cat(vsmw->vsb, ".");
        }
        l = VSB_len(vsmw->vsb);
        assert(l >= 0);
        VSB_vprintf(vsmw->vsb, fmt, va);
        AZ(VSB_finish(vsmw->vsb));
        assert(fmt[0] == '\0' || l < VSB_len(vsmw->vsb));

        REPLACE(seg->id, VSB_data(vsmw->vsb));

        if (vc == NULL)
                vc = vsmw_newcluster(vsmw, seg->len, category);
        AN(vc);
        vc->refs++;

        seg->cluster = vc;
        seg->off = vc->next;
        vc->next += seg->len;
        assert(vc->next <= vc->len);
        seg->ptr = seg->off + (char*)vc->ptr;

        vsmw_addseg(vsmw, seg);

        vsmw_do_unlock();
        return (seg->ptr);
}

void *
VSMW_Allocf(struct vsmw *vsmw, struct vsmw_cluster *vc,
    const char *category, size_t len, const char *fmt, ...)
{
        va_list ap;
        void *p;

        va_start(ap, fmt);
        p = VSMW_Allocv(vsmw, vc, category, len, NULL, fmt, ap);
        va_end(ap);
        return (p);
}

/*--------------------------------------------------------------------*/

void
VSMW_Free(struct vsmw *vsmw, void **pp)
{
        struct vsmwseg *seg;
        struct vsmw_cluster *cp;

        vsmw_do_lock();
        CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
        AN(pp);
        VTAILQ_FOREACH(seg, &vsmw->segs, list)
                if (seg->ptr == *pp)
                        break;
        AN(seg);
        *pp = NULL;

        cp = seg->cluster;
        CHECK_OBJ_NOTNULL(cp, VSMW_CLUSTER_MAGIC);
        assert(cp->refs > 0);

        vsmw_delseg(vsmw, seg);

        if (!--cp->refs)
                vsmw_DestroyCluster_locked(vsmw, cp);
        vsmw_do_unlock();
}

/*--------------------------------------------------------------------*/

struct vsmw *
VSMW_New(int vdirfd, int mode, const char *idxname)
{
        struct vsmw *vsmw;
        int fd;

        assert(vdirfd > 0);
        assert(mode > 0);
        AN(idxname);

        vsmw_do_lock();
        ALLOC_OBJ(vsmw, VSMW_MAGIC);
        AN(vsmw);

        VTAILQ_INIT(&vsmw->segs);
        VTAILQ_INIT(&vsmw->clusters);
        vsmw->vsb = VSB_new_auto();
        AN(vsmw->vsb);
        REPLACE(vsmw->idx, idxname);
        vsmw->mode = mode;
        vsmw->vdirfd = vdirfd;
        vsmw->pid = getpid();
        vsmw->birth = time(NULL);

        if (unlinkat(vdirfd, vsmw->idx, 0))
                assert (errno == ENOENT);
        fd = openat(vdirfd,
            vsmw->idx, O_APPEND | O_WRONLY | O_CREAT, vsmw->mode);
        assert(fd >= 0);
        vsmw_idx_head(vsmw, fd);
        closefd(&fd);

        vsmw_do_unlock();
        return (vsmw);
}

void
VSMW_Destroy(struct vsmw **pp)
{
        struct vsmw *vsmw;
        struct vsmwseg *seg, *s2;

        vsmw_do_lock();
        TAKE_OBJ_NOTNULL(vsmw, pp, VSMW_MAGIC);
        VTAILQ_FOREACH_SAFE(seg, &vsmw->segs, list, s2)
                vsmw_delseg(vsmw, seg);
        if (unlinkat(vsmw->vdirfd, vsmw->idx, 0))
                assert (errno == ENOENT);
        REPLACE(vsmw->idx, NULL);
        VSB_destroy(&vsmw->vsb);
        closefd(&vsmw->vdirfd);
        FREE_OBJ(vsmw);
        vsmw_do_unlock();
}

		varnish-cache/bin/varnishd/common/common_vsmw.c
0		/*-
1		* Copyright (c) 2010-2011 Varnish Software AS
2		* All rights reserved.
3		*
4		* Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
5		*
6		* SPDX-License-Identifier: BSD-2-Clause
7		*
8		* Redistribution and use in source and binary forms, with or without
9		* modification, are permitted provided that the following conditions
10		* are met:
11		* 1. Redistributions of source code must retain the above copyright
12		* notice, this list of conditions and the following disclaimer.
13		* 2. Redistributions in binary form must reproduce the above copyright
14		* notice, this list of conditions and the following disclaimer in the
15		* documentation and/or other materials provided with the distribution.
16		*
17		* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18		* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19		* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20		* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
21		* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22		* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23		* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24		* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25		* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26		* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27		* SUCH DAMAGE.
28		*
29		* VSM stuff common to manager and child.
30		*
31		*/
32
33		#include "config.h"
34
35		#include <fcntl.h>
36		#include <stdarg.h>
37		#include <stdio.h>
38		#include <stdint.h>
39		#include <stdlib.h>
40		#include <string.h>
41		#include <time.h>
42		#include <unistd.h>
43		#include <sys/mman.h>
44		#include <sys/resource.h>
45		#include <sys/stat.h>
46
47		#include "vdef.h"
48		#include "vas.h"
49		#include "vsb.h"
50		#include "miniobj.h"
51		#include "vqueue.h"
52
53		#include "vfil.h"
54		#include "vrnd.h"
55
56		#include "heritage.h"
57		#include "vsmw.h"
58
59		#ifndef MAP_HASSEMAPHORE
60		# define MAP_HASSEMAPHORE 0 /* XXX Linux */
61		#endif
62
63		#ifndef MAP_NOSYNC
64		# define MAP_NOSYNC 0 /* XXX Linux */
65		#endif
66
67		static void v_matchproto_(vsm_lock_f)
68	845600	vsmw_dummy_lock(void)
69		{
70	845600	}
71
72		static int vsmw_haslock;
73		vsm_lock_f *vsmw_lock = vsmw_dummy_lock;
74		vsm_lock_f *vsmw_unlock = vsmw_dummy_lock;
75
76		#define vsmw_assert_lock() AN(vsmw_haslock)
77
78		#define vsmw_do_lock() vsmw_do_lock_(__func__, __LINE__)
79
80		#define vsmw_do_lock_(f, l) \
81		do { \
82		vsmw_lock(); \
83		AZ(vsmw_haslock); \
84		vsmw_haslock = 1; \
85		} while(0)
86
87		#define vsmw_do_unlock() vsmw_do_unlock_(__func__, __LINE__)
88		#define vsmw_do_unlock_(f, l) \
89		do { \
90		AN(vsmw_haslock); \
91		vsmw_haslock = 0; \
92		vsmw_unlock(); \
93		} while(0)
94
95		/--------------------------------------------------------------------/
96
97		struct vsmw_cluster {
98		unsigned magic;
99		#define VSMW_CLUSTER_MAGIC 0x28b74c00
100
101		VTAILQ_ENTRY(vsmw_cluster) list;
102		struct vsmwseg *cseg;
103		char *fn;
104		size_t len;
105		void *ptr;
106		size_t next;
107		int refs;
108		int named;
109		};
110
111		struct vsmwseg {
112		unsigned magic;
113		#define VSMWSEG_MAGIC 0x7e4ccaea
114		VTAILQ_ENTRY(vsmwseg) list;
115		struct vsmw_cluster *cluster;
116
117		char *category;
118		size_t off;
119		size_t len;
120		char *id;
121		void *ptr;
122		};
123
124		struct vsmw {
125		unsigned magic;
126		#define VSMW_MAGIC 0xc2ca2cd9
127		int vdirfd;
128		int mode;
129		char *idx;
130		VTAILQ_HEAD(, vsmw_cluster) clusters;
131		VTAILQ_HEAD(, vsmwseg) segs;
132		struct vsb *vsb;
133		pid_t pid;
134		time_t birth;
135		uint64_t nsegs;
136		uint64_t nsubs;
137		};
138
139		/* Allocations in clusters never start at offset zero */
140		#define VSM_CLUSTER_OFFSET 16
141
142		/--------------------------------------------------------------------/
143
144		static void
145	76231	vsmw_idx_head(const struct vsmw *vsmw, int fd)
146		{
147		char buf[64];
148
149	76231	bprintf(buf, "# %jd %jd\n", (intmax_t)vsmw->pid, (intmax_t)vsmw->birth);
150		// XXX handle ENOSPC? #2764
151	76231	assert(write(fd, buf, strlen(buf)) == strlen(buf));
152	76231	}
153
154		#define ASSERT_SEG_STR(x) do { \
155		AN(x); \
156		AZ(strchr(x, '\n')); \
157		} while (0);
158
159		static void
160	2213078	vsmw_fmt_index(const struct vsmw vsmw, const struct vsmwseg seg, char act)
161		{
162
163	2213078	vsmw_assert_lock();
164	2213078	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
165	2213078	CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
166	2213078	AN(seg->cluster);
167	2213078	ASSERT_SEG_STR(seg->category);
168	2213078	ASSERT_SEG_STR(seg->id);
169
170	4426156	VSB_printf(vsmw->vsb, "%c %s %zu %zu %s %s\n",
171	2213078	act,
172	2213078	seg->cluster->fn,
173	2213078	seg->off,
174	2213078	seg->len,
175	2213078	seg->category,
176	2213078	seg->id);
177	2213078	}
178
179		/--------------------------------------------------------------------/
180
181		static void
182	1884243	vsmw_mkent(const struct vsmw vsmw, const char pfx)
183		{
184		int fd;
185		uint64_t rn;
186
187	1884243	AN(pfx);
188	1884243	vsmw_assert_lock();
189	1884243	while (1) {
190	1884243	VSB_clear(vsmw->vsb);
191	1884243	VSB_printf(vsmw->vsb, "_.%s", pfx);
192	1884243	AZ(VRND_RandomCrypto(&rn, sizeof rn));
193	1884243	VSB_printf(vsmw->vsb, ".%016jx", (uintmax_t)rn);
194	1884243	AZ(VSB_finish(vsmw->vsb));
195	1884243	fd = openat(vsmw->vdirfd, VSB_data(vsmw->vsb), O_RDONLY);
196	1884243	if (fd < 0 && errno == ENOENT)
197	1884243	return;
198	0	if (fd >= 0)
199	0	closefd(&fd);
200		}
201		}
202
203		/--------------------------------------------------------------------/
204
205		static void
206	2210598	vsmw_append_record(struct vsmw vsmw, struct vsmwseg seg, char act)
207		{
208		int fd;
209
210	2210598	vsmw_assert_lock();
211	2210598	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
212	2210598	CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
213	2210598	fd = openat(vsmw->vdirfd, vsmw->idx, O_APPEND \| O_WRONLY);
214	2210598	assert(fd >= 0);
215	2210598	VSB_clear(vsmw->vsb);
216	2210598	vsmw_fmt_index(vsmw, seg, act);
217	2210598	AZ(VSB_finish(vsmw->vsb));
218	2210598	XXXAZ(VSB_tofile(vsmw->vsb, fd)); // XXX handle ENOSPC? #2764
219	2210598	closefd(&fd);
220	2210598	}
221
222		/--------------------------------------------------------------------/
223
224		static void
225	1935363	vsmw_addseg(struct vsmw vsmw, struct vsmwseg seg)
226		{
227
228	1935363	vsmw_assert_lock();
229	1935363	VTAILQ_INSERT_TAIL(&vsmw->segs, seg, list);
230	1935363	vsmw_append_record(vsmw, seg, '+');
231	1935363	vsmw->nsegs++;
232	1935363	}
233
234		/--------------------------------------------------------------------/
235
236		static void
237	275636	vsmw_delseg(struct vsmw vsmw, struct vsmwseg seg)
238		{
239	275636	char *t = NULL;
240		int fd;
241		struct vsmwseg *s2;
242
243	275636	vsmw_assert_lock();
244	275636	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
245	275636	CHECK_OBJ_NOTNULL(seg, VSMWSEG_MAGIC);
246
247	275636	VTAILQ_REMOVE(&vsmw->segs, seg, list);
248
249	275636	vsmw->nsegs--;
250	275636	if (vsmw->nsubs < 10 \|\| vsmw->nsubs * 2 < vsmw->nsegs) {
251	275236	vsmw_append_record(vsmw, seg, '-');
252	275236	vsmw->nsubs++;
253	275236	} else {
254	400	vsmw_mkent(vsmw, vsmw->idx);
255	400	REPLACE(t, VSB_data(vsmw->vsb));
256	800	fd = openat(vsmw->vdirfd,
257	400	t, O_WRONLY\|O_CREAT\|O_EXCL, vsmw->mode);
258	400	assert(fd >= 0);
259	400	vsmw_idx_head(vsmw, fd);
260	400	VSB_clear(vsmw->vsb);
261	2880	VTAILQ_FOREACH(s2, &vsmw->segs, list)
262	2480	vsmw_fmt_index(vsmw, s2, '+');
263	400	AZ(VSB_finish(vsmw->vsb));
264	400	XXXAZ(VSB_tofile(vsmw->vsb, fd)); // XXX handle ENOSPC? #2764
265	400	closefd(&fd);
266	400	AZ(renameat(vsmw->vdirfd, t, vsmw->vdirfd, vsmw->idx));
267	400	REPLACE(t, NULL);
268	400	vsmw->nsubs = 0;
269		}
270	275636	REPLACE(seg->category, NULL);
271	275636	REPLACE(seg->id, NULL);
272	275636	FREE_OBJ(seg);
273	275636	}
274
275		/--------------------------------------------------------------------/
276
277		#ifdef RLIMIT_MEMLOCK
278		static void
279	76880	printlim(const char *name, rlim_t lim)
280		{
281
282	76880	fprintf(stderr, "Info: %s: ", name);
283	76880	if (lim == RLIM_INFINITY)
284	76880	fprintf(stderr, "unlimited\n");
285		else
286	0	fprintf(stderr, "%ju bytes\n", (uintmax_t)lim);
287	76880	}
288
289		static void
290	38440	printmemlock(void) {
291		struct rlimit rlim;
292
293	38440	AZ(getrlimit(RLIMIT_MEMLOCK, &rlim));
294	38440	printlim("max locked memory (soft)", rlim.rlim_cur);
295	38440	printlim("max locked memory (hard)", rlim.rlim_max);
296	38440	}
297		#else
298		static void printmemlock(void) {}
299		#endif
300
301		static struct vsmw_cluster *
302	1883843	vsmw_newcluster(struct vsmw vsmw, size_t len, const char pfx)
303		{
304		struct vsmw_cluster *vc;
305		static int warn = 0;
306		int fd;
307		size_t ps;
308
309	1883843	vsmw_assert_lock();
310	1883843	ALLOC_OBJ(vc, VSMW_CLUSTER_MAGIC);
311	1883843	AN(vc);
312
313	1883843	vsmw_mkent(vsmw, pfx);
314	1883843	REPLACE(vc->fn, VSB_data(vsmw->vsb));
315
316	1883843	VTAILQ_INSERT_TAIL(&vsmw->clusters, vc, list);
317
318	1883843	ps = getpagesize();
319	1883843	len = RUP2(len, ps);
320	1883843	vc->len = len;
321
322	3767686	fd = openat(vsmw->vdirfd, vc->fn,
323	1883843	O_RDWR \| O_CREAT \| O_EXCL, vsmw->mode);
324	1883843	assert(fd >= 0);
325
326	1883843	AZ(VFIL_allocate(fd, (off_t)len, 1));
327
328	3767686	vc->ptr = (void *)mmap(NULL, len,
329		PROT_READ\|PROT_WRITE,
330		MAP_HASSEMAPHORE \| MAP_NOSYNC \| MAP_SHARED,
331	1883843	fd, 0);
332
333	1883843	closefd(&fd);
334	1883843	assert(vc->ptr != MAP_FAILED);
335	1883843	if (mlock(vc->ptr, len) && warn++ == 0) {
336	76880	fprintf(stderr, "Warning: mlock() of VSM failed: %s (%d)\n",
337	38440	VAS_errtxt(errno), errno);
338	38440	printmemlock();
339	38440	}
340
341	1883843	return (vc);
342		}
343
344		struct vsmw_cluster *
345	48440	VSMW_NewCluster(struct vsmw vsmw, size_t len, const char pfx)
346		{
347		struct vsmw_cluster *vc;
348		struct vsmwseg *seg;
349
350	48440	vsmw_do_lock();
351	48440	vc = vsmw_newcluster(vsmw, len + VSM_CLUSTER_OFFSET, pfx);
352	48440	AN(vc);
353	48440	vc->next += VSM_CLUSTER_OFFSET;
354
355	48440	ALLOC_OBJ(seg, VSMWSEG_MAGIC);
356	48440	AN(seg);
357	48440	vc->cseg = seg;
358	48440	seg->len = vc->len;
359	48440	seg->cluster = vc;
360	48440	REPLACE(seg->category, "");
361	48440	REPLACE(seg->id, "");
362	48440	vc->refs++;
363	48440	vc->named = 1;
364	48440	vsmw_addseg(vsmw, seg);
365
366	48440	vsmw_do_unlock();
367	48440	return (vc);
368		}
369
370		static void
371	80898	vsmw_DestroyCluster_locked(struct vsmw vsmw, struct vsmw_cluster vc)
372		{
373
374	80898	vsmw_assert_lock();
375	80898	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
376	80898	CHECK_OBJ_NOTNULL(vc, VSMW_CLUSTER_MAGIC);
377
378	80898	AZ(vc->refs);
379
380	80898	AZ(munmap(vc->ptr, vc->len));
381	80898	if (vc->named)
382	3098	vsmw_delseg(vsmw, vc->cseg);
383	80898	vc->cseg = 0;
384
385	80898	VTAILQ_REMOVE(&vsmw->clusters, vc, list);
386	80898	if (unlinkat(vsmw->vdirfd, vc->fn, 0))
387	0	assert (errno == ENOENT);
388	80898	REPLACE(vc->fn, NULL);
389	80898	FREE_OBJ(vc);
390	80898	}
391
392		void
393	3098	VSMW_DestroyCluster(struct vsmw vsmw, struct vsmw_cluster *vsmcp)
394		{
395		struct vsmw_cluster *vc;
396
397	3098	TAKE_OBJ_NOTNULL(vc, vsmcp, VSMW_CLUSTER_MAGIC);
398
399	3098	vsmw_do_lock();
400	3098	if (--vc->refs == 0)
401	1720	vsmw_DestroyCluster_locked(vsmw, vc);
402	3098	vsmw_do_unlock();
403	3098	}
404
405		/--------------------------------------------------------------------/
406
407		void *
408	1886923	VSMW_Allocv(struct vsmw vsmw, struct vsmw_cluster vc,
409		const char category, size_t payload, const char prefix,
410		const char *fmt, va_list va)
411		{
412		struct vsmwseg *seg;
413		ssize_t l;
414
415	1886923	vsmw_do_lock();
416	1886923	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
417
418	1886923	ALLOC_OBJ(seg, VSMWSEG_MAGIC);
419	1886923	AN(seg);
420	1886923	REPLACE(seg->category, category);
421	1886923	seg->len = PRNDUP(payload);
422
423	1886923	VSB_clear(vsmw->vsb);
424	1886923	if (prefix != NULL) {
425	1360192	assert(prefix[0] != '\0');
426	1360192	VSB_cat(vsmw->vsb, prefix);
427	1360192	if (fmt[0] != '\0')
428	1246931	VSB_cat(vsmw->vsb, ".");
429	1360192	}
430	1886923	l = VSB_len(vsmw->vsb);
431	1886923	assert(l >= 0);
432	1886923	VSB_vprintf(vsmw->vsb, fmt, va);
433	1886923	AZ(VSB_finish(vsmw->vsb));
434	1886923	assert(fmt[0] == '\0' \|\| l < VSB_len(vsmw->vsb));
435
436	1886923	REPLACE(seg->id, VSB_data(vsmw->vsb));
437
438	1886923	if (vc == NULL)
439	1835403	vc = vsmw_newcluster(vsmw, seg->len, category);
440	1886923	AN(vc);
441	1886923	vc->refs++;
442
443	1886923	seg->cluster = vc;
444	1886923	seg->off = vc->next;
445	1886923	vc->next += seg->len;
446	1886923	assert(vc->next <= vc->len);
447	1886923	seg->ptr = seg->off + (char*)vc->ptr;
448
449	1886923	vsmw_addseg(vsmw, seg);
450
451	1886923	vsmw_do_unlock();
452	1886923	return (seg->ptr);
453		}
454
455		void *
456	229551	VSMW_Allocf(struct vsmw vsmw, struct vsmw_cluster vc,
457		const char category, size_t len, const char fmt, ...)
458		{
459		va_list ap;
460		void *p;
461
462	229551	va_start(ap, fmt);
463	229551	p = VSMW_Allocv(vsmw, vc, category, len, NULL, fmt, ap);
464	229551	va_end(ap);
465	229551	return (p);
466		}
467
468		/--------------------------------------------------------------------/
469
470		void
471	80378	VSMW_Free(struct vsmw vsmw, void *pp)
472		{
473		struct vsmwseg *seg;
474		struct vsmw_cluster *cp;
475
476	80378	vsmw_do_lock();
477	80378	CHECK_OBJ_NOTNULL(vsmw, VSMW_MAGIC);
478	80378	AN(pp);
479	652851	VTAILQ_FOREACH(seg, &vsmw->segs, list)
480	652851	if (seg->ptr == *pp)
481	80378	break;
482	80378	AN(seg);
483	80378	*pp = NULL;
484
485	80378	cp = seg->cluster;
486	80378	CHECK_OBJ_NOTNULL(cp, VSMW_CLUSTER_MAGIC);
487	80378	assert(cp->refs > 0);
488
489	80378	vsmw_delseg(vsmw, seg);
490
491	80378	if (!--cp->refs)
492	79178	vsmw_DestroyCluster_locked(vsmw, cp);
493	80378	vsmw_do_unlock();
494	80378	}
495
496		/--------------------------------------------------------------------/
497
498		struct vsmw *
499	75831	VSMW_New(int vdirfd, int mode, const char *idxname)