GCOV - Varnish Cache


/*-
 * Copyright (c) 2008-2011 Varnish Software AS
 * All rights reserved.
 *
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
 *
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * Persistent storage method
 *
 * XXX: Before we start the client or maybe after it stops, we should give the
 * XXX: stevedores a chance to examine their storage for consistency.
 *
 */

#include "config.h"


#include <stdio.h>
#include <stdlib.h>

#include "cache/cache_varnishd.h"

#include "vsha256.h"
#include "vend.h"
#include "vtim.h"

#include "cache/cache_objhead.h"

#include "storage/storage.h"
#include "storage/storage_simple.h"
#include "storage/storage_persistent.h"

/*
 * We use the top bit to mark objects still needing fixup
 * In theory this may need to be platform dependent
 */

#define NEED_FIXUP      (1U << 31)

/*--------------------------------------------------------------------
 * Write the segmentlist back to the silo.
 *
 * We write the first copy, sync it synchronously, then write the
 * second copy and sync it synchronously.
 *
 * Provided the kernel doesn't lie, that means we will always have
 * at least one valid copy on in the silo.
 */

static void
smp_save_seg(const struct smp_sc *sc, struct smp_signspace *spc)
{
        struct smp_segptr *ss;
        struct smp_seg *sg;
        uint64_t length;

        Lck_AssertHeld(&sc->mtx);
        smp_reset_signspace(spc);
        ss = SIGNSPACE_DATA(spc);
        length = 0;
        VTAILQ_FOREACH(sg, &sc->segments, list) {
                assert(sg->p.offset < sc->mediasize);
                assert(sg->p.offset + sg->p.length <= sc->mediasize);
                *ss = sg->p;
                ss++;
                length += sizeof *ss;
        }
        smp_append_signspace(spc, length);
        smp_sync_sign(&spc->ctx);
}

void
smp_save_segs(struct smp_sc *sc)
{
        struct smp_seg *sg, *sg2;

        CHECK_OBJ_NOTNULL(sc, SMP_SC_MAGIC);
        Lck_AssertHeld(&sc->mtx);

        /*
         * Remove empty segments from the front of the list
         * before we write the segments to disk.
         */
        VTAILQ_FOREACH_SAFE(sg, &sc->segments, list, sg2) {
                CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC);

                if (sg->nobj > 0)
                        break;
                if (sg == sc->cur_seg)
                        continue;
                VTAILQ_REMOVE(&sc->segments, sg, list);
                AN(VTAILQ_EMPTY(&sg->objcores));
                FREE_OBJ(sg);
        }
        smp_save_seg(sc, &sc->seg1);
        smp_save_seg(sc, &sc->seg2);
}

/*--------------------------------------------------------------------
 * Load segments
 *
 * The overall objective is to register the existence of an object, based
 * only on the minimally sized struct smp_object, without causing the
 * main object to be faulted in.
 *
 * XXX: We can test this by mprotecting the main body of the segment
 * XXX: until the first fixup happens, or even just over this loop,
 * XXX: However: the requires that the smp_objects starter further
 * XXX: into the segment than a page so that they do not get hit
 * XXX: by the protection.
 */

void
smp_load_seg(struct worker *wrk, const struct smp_sc *sc,
    struct smp_seg *sg)
{
        struct smp_object *so;
        struct objcore *oc;
        struct ban *ban;
        uint32_t no;
        double t_now = VTIM_real();
        struct smp_signctx ctx[1];

        ASSERT_SILO_THREAD(sc);
        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
        CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC);
        assert(sg->flags & SMP_SEG_MUSTLOAD);
        sg->flags &= ~SMP_SEG_MUSTLOAD;
        AN(sg->p.offset);
        if (sg->p.objlist == 0)
                return;
        smp_def_sign(sc, ctx, sg->p.offset, "SEGHEAD");
        if (smp_chk_sign(ctx))
                return;

        /* test SEGTAIL */
        /* test OBJIDX */
        so = (void*)(sc->base + sg->p.objlist);
        sg->objs = so;
        no = sg->p.lobjlist;
        /* Clear the bogus "hold" count */
        sg->nobj = 0;
        for (;no > 0; so++,no--) {
                if (EXP_WHEN(so) < t_now)
                        continue;
                ban = BAN_FindBan(so->ban);
                AN(ban);
                oc = ObjNew(wrk);
                oc->stobj->stevedore = sc->parent;
                smp_init_oc(oc, sg, no);
                VTAILQ_INSERT_TAIL(&sg->objcores, oc, lru_list);
                oc->stobj->priv2 |= NEED_FIXUP;
                EXP_COPY(oc, so);
                sg->nobj++;
                oc->refcnt++;
                HSH_Insert(wrk, so->hash, oc, ban);
                AN(oc->ban);
                HSH_DerefBoc(wrk, oc);  // XXX Keep it an stream resurrection?
                (void)HSH_DerefObjCore(wrk, &oc, HSH_RUSH_POLICY);
                wrk->stats->n_vampireobject++;
        }
        Pool_Sumstat(wrk);
        sg->flags |= SMP_SEG_LOADED;
}

/*--------------------------------------------------------------------
 * Create a new segment
 */

void
smp_new_seg(struct smp_sc *sc)
{
        struct smp_seg tmpsg;
        struct smp_seg *sg;

        AZ(sc->cur_seg);
        Lck_AssertHeld(&sc->mtx);

        /* XXX: find where it goes in silo */

        INIT_OBJ(&tmpsg, SMP_SEG_MAGIC);
        tmpsg.sc = sc;
        tmpsg.p.offset = sc->free_offset;
        /* XXX: align */
        assert(tmpsg.p.offset >= sc->ident->stuff[SMP_SPC_STUFF]);
        assert(tmpsg.p.offset < sc->mediasize);

        tmpsg.p.length = sc->aim_segl;
        tmpsg.p.length = RDN2(tmpsg.p.length, 8);

        if (smp_segend(&tmpsg) > sc->mediasize)
                /* XXX: Consider truncation in this case */
                tmpsg.p.offset = sc->ident->stuff[SMP_SPC_STUFF];

        assert(smp_segend(&tmpsg) <= sc->mediasize);

        sg = VTAILQ_FIRST(&sc->segments);
        if (sg != NULL && tmpsg.p.offset <= sg->p.offset) {
                if (smp_segend(&tmpsg) > sg->p.offset)
                        /* No more space, return (cur_seg will be NULL) */
                        /* XXX: Consider truncation instead of failing */
                        return;
                assert(smp_segend(&tmpsg) <= sg->p.offset);
        }

        if (tmpsg.p.offset == sc->ident->stuff[SMP_SPC_STUFF])
                printf("Wrapped silo\n");

        ALLOC_OBJ(sg, SMP_SEG_MAGIC);
        if (sg == NULL)
                return;
        *sg = tmpsg;
        VTAILQ_INIT(&sg->objcores);

        sg->p.offset = IRNUP(sc, sg->p.offset);
        sg->p.length -= sg->p.offset - tmpsg.p.offset;
        sg->p.length = IRNDN(sc, sg->p.length);
        assert(sg->p.offset + sg->p.length <= tmpsg.p.offset + tmpsg.p.length);
        sc->free_offset = sg->p.offset + sg->p.length;

        VTAILQ_INSERT_TAIL(&sc->segments, sg, list);

        /* Neuter the new segment in case there is an old one there */
        AN(sg->p.offset);
        smp_def_sign(sc, sg->ctx, sg->p.offset, "SEGHEAD");
        smp_reset_sign(sg->ctx);
        smp_sync_sign(sg->ctx);

        /* Set up our allocation points */
        sc->cur_seg = sg;
        sc->next_bot = sg->p.offset + IRNUP(sc, SMP_SIGN_SPACE);
        sc->next_top = smp_segend(sg);
        sc->next_top -= IRNUP(sc, SMP_SIGN_SPACE);
        IASSERTALIGN(sc, sc->next_bot);
        IASSERTALIGN(sc, sc->next_top);
        sg->objs = (void*)(sc->base + sc->next_top);
}

/*--------------------------------------------------------------------
 * Close a segment
 */

void
smp_close_seg(struct smp_sc *sc, struct smp_seg *sg)
{
        uint64_t left, dst, len;
        void *dp;

        CHECK_OBJ_NOTNULL(sc, SMP_SC_MAGIC);
        Lck_AssertHeld(&sc->mtx);

        CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC);
        assert(sg == sc->cur_seg);
        AN(sg->p.offset);
        sc->cur_seg = NULL;

        if (sg->nalloc == 0) {
                /* If segment is empty, delete instead */
                VTAILQ_REMOVE(&sc->segments, sg, list);
                assert(sg->p.offset >= sc->ident->stuff[SMP_SPC_STUFF]);
                assert(sg->p.offset < sc->mediasize);
                sc->free_offset = sg->p.offset;
                AN(VTAILQ_EMPTY(&sg->objcores));
                FREE_OBJ(sg);
                return;
        }

        /*
         * If there is enough space left, that we can move the smp_objects
         * down without overwriting the present copy, we will do so to
         * compact the segment.
         */
        left = smp_spaceleft(sc, sg);
        len = sizeof(struct smp_object) * sg->p.lobjlist;
        if (len < left) {
                dst = sc->next_bot + IRNUP(sc, SMP_SIGN_SPACE);
                dp = sc->base + dst;
                assert((uintptr_t)dp + len < (uintptr_t)sg->objs);
                memcpy(dp, sg->objs, len);
                sc->next_top = dst;
                sg->objs = dp;
                sg->p.length = (sc->next_top - sg->p.offset)
                     + len + IRNUP(sc, SMP_SIGN_SPACE);
                (void)smp_spaceleft(sc, sg);    /* for the asserts */

        }

        /* Update the segment header */
        sg->p.objlist = sc->next_top;

        /* Write the (empty) OBJIDX signature */
        sc->next_top -= IRNUP(sc, SMP_SIGN_SPACE);
        assert(sc->next_top >= sc->next_bot);
        smp_def_sign(sc, sg->ctx, sc->next_top, "OBJIDX");
        smp_reset_sign(sg->ctx);
        smp_sync_sign(sg->ctx);

        /* Write the (empty) SEGTAIL signature */
        smp_def_sign(sc, sg->ctx,
            sg->p.offset + sg->p.length - IRNUP(sc, SMP_SIGN_SPACE), "SEGTAIL");
        smp_reset_sign(sg->ctx);
        smp_sync_sign(sg->ctx);

        /* Save segment list */
        smp_save_segs(sc);
        sc->free_offset = smp_segend(sg);
}


/*---------------------------------------------------------------------
 */

static struct smp_object *
smp_find_so(const struct smp_seg *sg, unsigned priv2)
{
        struct smp_object *so;

        priv2 &= ~NEED_FIXUP;
        assert(priv2 > 0);
        assert(priv2 <= sg->p.lobjlist);
        so = &sg->objs[sg->p.lobjlist - priv2];
        return (so);
}

/*---------------------------------------------------------------------
 * Check if a given storage structure is valid to use
 */

static int
smp_loaded_st(const struct smp_sc *sc, const struct smp_seg *sg,
    const struct storage *st)
{
        struct smp_seg *sg2;
        const uint8_t *pst;
        uint64_t o;

        (void)sg;               /* XXX: faster: Start search from here */
        pst = (const void *)st;

        if (pst < (sc->base + sc->ident->stuff[SMP_SPC_STUFF]))
                return (0x01);          /* Before silo payload start */
        if (pst > (sc->base + sc->ident->stuff[SMP_END_STUFF]))
                return (0x02);          /* After silo end */

        o = pst - sc->base;

        /* Find which segment contains the storage structure */
        VTAILQ_FOREACH(sg2, &sc->segments, list)
                if (o > sg2->p.offset && (o + sizeof(*st)) < sg2->p.objlist)
                        break;
        if (sg2 == NULL)
                return (0x04);          /* No claiming segment */
        if (!(sg2->flags & SMP_SEG_LOADED))
                return (0x08);          /* Claiming segment not loaded */

        /* It is now safe to access the storage structure */
        if (st->magic != STORAGE_MAGIC)
                return (0x10);          /* Not enough magic */

        if (o + st->space >= sg2->p.objlist)
                return (0x20);          /* Allocation not inside segment */

        if (st->len > st->space)
                return (0x40);          /* Plain bad... */

        /*
         * XXX: We could patch up st->stevedore and st->priv here
         * XXX: but if things go right, we will never need them.
         */
        return (0);
}

/*---------------------------------------------------------------------
 * objcore methods for persistent objects
 */

static void
fix_ptr(const struct smp_seg *sg, const struct storage *st, void **ptr)
{
        // See comment where used below
        uintptr_t u;

        u = (uintptr_t)(*ptr);
        if (u != 0) {
                u -= (uintptr_t)st->priv;
                u += (uintptr_t)sg->sc->base;
        }
        *ptr = (void *)u;
}

struct object * v_matchproto_(sml_getobj_f)
smp_sml_getobj(struct worker *wrk, struct objcore *oc)
{
        struct object *o;
        struct smp_seg *sg;
        struct smp_object *so;
        struct storage *st, *st2;
        uint64_t l;
        int bad;

        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
        AN(oc->stobj->stevedore);

        CAST_OBJ_NOTNULL(sg, oc->stobj->priv, SMP_SEG_MAGIC);
        so = smp_find_so(sg, oc->stobj->priv2);

        /**************************************************************
         * The silo may have been remapped at a different address,
         * because the people who came up with ASLR were unable
         * imagine that there might be beneficial use-cases for
         * always mapping a file at the same specific address.
         *
         * We store the silos base address in struct storage->priv
         * and manually fix all the pointers in struct object and
         * the list of struct storage objects which hold the body.
         * When done, we update the storage->priv, so we can do the
         * same trick next time.
         *
         * This is a prohibitively expensive workaround, but we can
         * live with it, because the role of this stevedore is only
         * to keep the internal stevedore API honest.
         */

        st = (void*)(sg->sc->base + so->ptr);
        fix_ptr(sg, st, (void**)&st->ptr);

        o = (void*)st->ptr;
        fix_ptr(sg, st, (void**)&o->objstore);
        fix_ptr(sg, st, (void**)&o->va_vary);
        fix_ptr(sg, st, (void**)&o->va_headers);
        fix_ptr(sg, st, (void**)&o->list.vtqh_first);
        fix_ptr(sg, st, (void**)&o->list.vtqh_last);
        st->priv = (void*)(sg->sc->base);

        st2 = o->list.vtqh_first;
        while (st2 != NULL) {
                fix_ptr(sg, st2, (void**)&st2->list.vtqe_next);
                fix_ptr(sg, st2, (void**)&st2->list.vtqe_prev);
                fix_ptr(sg, st2, (void**)&st2->ptr);
                st2->priv = (void*)(sg->sc->base);
                st2 = st2->list.vtqe_next;
        }

        /*
         * The object may not be in this segment since we allocate it
         * In a separate operation than the smp_object.  We could check
         * that it is in a later segment, but that would be complicated.
         * XXX: For now, be happy if it is inside the silo
         */
        ASSERT_PTR_IN_SILO(sg->sc, o);
        CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);

        /*
         * If this flag is not set, it will not be, and the lock is not
         * needed to test it.
         */
        if (!(oc->stobj->priv2 & NEED_FIXUP))
                return (o);

        Lck_Lock(&sg->sc->mtx);
        /* Check again, we might have raced. */
        if (oc->stobj->priv2 & NEED_FIXUP) {
                /* We trust caller to have a refcnt for us */

                bad = 0;
                l = 0;
                VTAILQ_FOREACH(st, &o->list, list) {
                        bad |= smp_loaded_st(sg->sc, sg, st);
                        if (bad)
                                break;
                        l += st->len;
                }
                if (l != vbe64dec(o->fa_len))
                        bad |= 0x100;

                if (bad) {
                        EXP_ZERO(oc);
                        EXP_ZERO(so);
                }

                sg->nfixed++;
                wrk->stats->n_object++;
                wrk->stats->n_vampireobject--;
                oc->stobj->priv2 &= ~NEED_FIXUP;
        }
        Lck_Unlock(&sg->sc->mtx);
        return (o);
}

void v_matchproto_(objfree_f)
smp_oc_objfree(struct worker *wrk, struct objcore *oc)
{
        struct smp_seg *sg;
        struct smp_object *so;

        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);

        CAST_OBJ_NOTNULL(sg, oc->stobj->priv, SMP_SEG_MAGIC);
        so = smp_find_so(sg, oc->stobj->priv2);

        Lck_Lock(&sg->sc->mtx);
        EXP_ZERO(so);
        so->ptr = 0;

        assert(sg->nobj > 0);
        sg->nobj--;
        if (oc->stobj->priv2 & NEED_FIXUP) {
                wrk->stats->n_vampireobject--;
        } else {
                assert(sg->nfixed > 0);
                sg->nfixed--;
                wrk->stats->n_object--;
        }
        VTAILQ_REMOVE(&sg->objcores, oc, lru_list);

        Lck_Unlock(&sg->sc->mtx);
        memset(oc->stobj, 0, sizeof oc->stobj);
}

/*--------------------------------------------------------------------*/

void
smp_init_oc(struct objcore *oc, struct smp_seg *sg, unsigned objidx)
{

        AZ(objidx & NEED_FIXUP);
        oc->stobj->priv = sg;
        oc->stobj->priv2 = objidx;
}

/*--------------------------------------------------------------------*/

void v_matchproto_(obj_event_f)
smp_oc_event(struct worker *wrk, void *priv, struct objcore *oc, unsigned ev)
{
        struct stevedore *st;
        struct smp_seg *sg;
        struct smp_object *so;

        CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
        CAST_OBJ_NOTNULL(st, priv, STEVEDORE_MAGIC);
        CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);

        if (oc->stobj->stevedore != st)
                return;

        CAST_OBJ_NOTNULL(sg, oc->stobj->priv, SMP_SEG_MAGIC);
        CHECK_OBJ_NOTNULL(sg->sc, SMP_SC_MAGIC);
        so = smp_find_so(sg, oc->stobj->priv2);

        if (sg == sg->sc->cur_seg) {
                /* Lock necessary, we might race close_seg */
                Lck_Lock(&sg->sc->mtx);
                if (ev & (OEV_BANCHG|OEV_INSERT))
                        so->ban = BAN_Time(oc->ban);
                if (ev & (OEV_TTLCHG|OEV_INSERT))
                        EXP_COPY(so, oc);
                Lck_Unlock(&sg->sc->mtx);
        } else {
                if (ev & (OEV_BANCHG|OEV_INSERT))
                        so->ban = BAN_Time(oc->ban);
                if (ev & (OEV_TTLCHG|OEV_INSERT))
                        EXP_COPY(so, oc);
        }
}


		varnish-cache/bin/varnishd/storage/storage_persistent_silo.c
0		/*-
1		* Copyright (c) 2008-2011 Varnish Software AS
2		* All rights reserved.
3		*
4		* Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
5		*
6		* SPDX-License-Identifier: BSD-2-Clause
7		*
8		* Redistribution and use in source and binary forms, with or without
9		* modification, are permitted provided that the following conditions
10		* are met:
11		* 1. Redistributions of source code must retain the above copyright
12		* notice, this list of conditions and the following disclaimer.
13		* 2. Redistributions in binary form must reproduce the above copyright
14		* notice, this list of conditions and the following disclaimer in the
15		* documentation and/or other materials provided with the distribution.
16		*
17		* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18		* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19		* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20		* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
21		* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22		* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23		* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24		* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25		* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26		* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27		* SUCH DAMAGE.
28		*
29		* Persistent storage method
30		*
31		* XXX: Before we start the client or maybe after it stops, we should give the
32		* XXX: stevedores a chance to examine their storage for consistency.
33		*
34		*/
35
36		#include "config.h"
37
38
39		#include <stdio.h>
40		#include <stdlib.h>
41
42		#include "cache/cache_varnishd.h"
43
44		#include "vsha256.h"
45		#include "vend.h"
46		#include "vtim.h"
47
48		#include "cache/cache_objhead.h"
49
50		#include "storage/storage.h"
51		#include "storage/storage_simple.h"
52		#include "storage/storage_persistent.h"
53
54		/*
55		* We use the top bit to mark objects still needing fixup
56		* In theory this may need to be platform dependent
57		*/
58
59		#define NEED_FIXUP (1U << 31)
60
61		/*--------------------------------------------------------------------
62		* Write the segmentlist back to the silo.
63		*
64		* We write the first copy, sync it synchronously, then write the
65		* second copy and sync it synchronously.
66		*
67		* Provided the kernel doesn't lie, that means we will always have
68		* at least one valid copy on in the silo.
69		*/
70
71		static void
72	4615	smp_save_seg(const struct smp_sc sc, struct smp_signspace spc)
73		{
74		struct smp_segptr *ss;
75		struct smp_seg *sg;
76		uint64_t length;
77
78	4615	Lck_AssertHeld(&sc->mtx);
79	4615	smp_reset_signspace(spc);
80	4615	ss = SIGNSPACE_DATA(spc);
81	4615	length = 0;
82	8671	VTAILQ_FOREACH(sg, &sc->segments, list) {
83	4056	assert(sg->p.offset < sc->mediasize);
84	4056	assert(sg->p.offset + sg->p.length <= sc->mediasize);
85	4056	*ss = sg->p;
86	4056	ss++;
87	4056	length += sizeof *ss;
88	4056	}
89	4615	smp_append_signspace(spc, length);
90	4615	smp_sync_sign(&spc->ctx);
91	4615	}
92
93		void
94	2308	smp_save_segs(struct smp_sc *sc)
95		{
96		struct smp_seg sg, sg2;
97
98	2308	CHECK_OBJ_NOTNULL(sc, SMP_SC_MAGIC);
99	2308	Lck_AssertHeld(&sc->mtx);
100
101		/*
102		* Remove empty segments from the front of the list
103		* before we write the segments to disk.
104		*/
105	2508	VTAILQ_FOREACH_SAFE(sg, &sc->segments, list, sg2) {
106	2108	CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC);
107
108	2108	if (sg->nobj > 0)
109	1908	break;
110	200	if (sg == sc->cur_seg)
111	40	continue;
112	160	VTAILQ_REMOVE(&sc->segments, sg, list);
113	160	AN(VTAILQ_EMPTY(&sg->objcores));
114	160	FREE_OBJ(sg);
115	160	}
116	2308	smp_save_seg(sc, &sc->seg1);
117	2308	smp_save_seg(sc, &sc->seg2);
118	2308	}
119
120		/*--------------------------------------------------------------------
121		* Load segments
122		*
123		* The overall objective is to register the existence of an object, based
124		* only on the minimally sized struct smp_object, without causing the
125		* main object to be faulted in.
126		*
127		* XXX: We can test this by mprotecting the main body of the segment
128		* XXX: until the first fixup happens, or even just over this loop,
129		* XXX: However: the requires that the smp_objects starter further
130		* XXX: into the segment than a page so that they do not get hit
131		* XXX: by the protection.
132		*/
133
134		void
135	600	smp_load_seg(struct worker wrk, const struct smp_sc sc,
136		struct smp_seg *sg)
137		{
138		struct smp_object *so;
139		struct objcore *oc;
140		struct ban *ban;
141		uint32_t no;
142	600	double t_now = VTIM_real();
143		struct smp_signctx ctx[1];
144
145	600	ASSERT_SILO_THREAD(sc);
146	600	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
147	600	CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC);
148	600	assert(sg->flags & SMP_SEG_MUSTLOAD);
149	600	sg->flags &= ~SMP_SEG_MUSTLOAD;
150	600	AN(sg->p.offset);
151	600	if (sg->p.objlist == 0)
152	0	return;
153	600	smp_def_sign(sc, ctx, sg->p.offset, "SEGHEAD");
154	600	if (smp_chk_sign(ctx))
155	0	return;
156
157		/* test SEGTAIL */
158		/* test OBJIDX */
159	600	so = (void*)(sc->base + sg->p.objlist);
160	600	sg->objs = so;
161	600	no = sg->p.lobjlist;
162		/* Clear the bogus "hold" count */
163	600	sg->nobj = 0;
164	1280	for (;no > 0; so++,no--) {
165	680	if (EXP_WHEN(so) < t_now)
166	0	continue;
167	680	ban = BAN_FindBan(so->ban);
168	680	AN(ban);
169	680	oc = ObjNew(wrk);
170	680	oc->stobj->stevedore = sc->parent;
171	680	smp_init_oc(oc, sg, no);
172	680	VTAILQ_INSERT_TAIL(&sg->objcores, oc, lru_list);
173	680	oc->stobj->priv2 \|= NEED_FIXUP;
174	680	EXP_COPY(oc, so);
175	680	sg->nobj++;
176	680	oc->refcnt++;
177	680	HSH_Insert(wrk, so->hash, oc, ban);
178	680	AN(oc->ban);
179	680	HSH_DerefBoc(wrk, oc); // XXX Keep it an stream resurrection?
180	680	(void)HSH_DerefObjCore(wrk, &oc, HSH_RUSH_POLICY);
181	680	wrk->stats->n_vampireobject++;
182	680	}
183	600	Pool_Sumstat(wrk);
184	600	sg->flags \|= SMP_SEG_LOADED;
185	600	}
186
187		/*--------------------------------------------------------------------
188		* Create a new segment
189		*/
190
191		void
192	1640	smp_new_seg(struct smp_sc *sc)
193		{
194		struct smp_seg tmpsg;
195		struct smp_seg *sg;
196
197	1640	AZ(sc->cur_seg);
198	1640	Lck_AssertHeld(&sc->mtx);
199
200		/* XXX: find where it goes in silo */
201
202	1640	INIT_OBJ(&tmpsg, SMP_SEG_MAGIC);
203	1640	tmpsg.sc = sc;
204	1640	tmpsg.p.offset = sc->free_offset;
205		/* XXX: align */
206	1640	assert(tmpsg.p.offset >= sc->ident->stuff[SMP_SPC_STUFF]);
207	1640	assert(tmpsg.p.offset < sc->mediasize);
208
209	1640	tmpsg.p.length = sc->aim_segl;
210	1640	tmpsg.p.length = RDN2(tmpsg.p.length, 8);
211
212	1640	if (smp_segend(&tmpsg) > sc->mediasize)
213		/* XXX: Consider truncation in this case */
214	0	tmpsg.p.offset = sc->ident->stuff[SMP_SPC_STUFF];
215
216	1640	assert(smp_segend(&tmpsg) <= sc->mediasize);
217
218	1640	sg = VTAILQ_FIRST(&sc->segments);
219	1640	if (sg != NULL && tmpsg.p.offset <= sg->p.offset) {
220	0	if (smp_segend(&tmpsg) > sg->p.offset)
221		/* No more space, return (cur_seg will be NULL) */
222		/* XXX: Consider truncation instead of failing */
223	0	return;
224	0	assert(smp_segend(&tmpsg) <= sg->p.offset);
225	0	}
226
227	1640	if (tmpsg.p.offset == sc->ident->stuff[SMP_SPC_STUFF])
228	920	printf("Wrapped silo\n");
229
230	1640	ALLOC_OBJ(sg, SMP_SEG_MAGIC);
231	1640	if (sg == NULL)
232	0	return;
233	1640	*sg = tmpsg;
234	1640	VTAILQ_INIT(&sg->objcores);
235
236	1640	sg->p.offset = IRNUP(sc, sg->p.offset);
237	1640	sg->p.length -= sg->p.offset - tmpsg.p.offset;
238	1640	sg->p.length = IRNDN(sc, sg->p.length);
239	1640	assert(sg->p.offset + sg->p.length <= tmpsg.p.offset + tmpsg.p.length);
240	1640	sc->free_offset = sg->p.offset + sg->p.length;
241
242	1640	VTAILQ_INSERT_TAIL(&sc->segments, sg, list);
243
244		/* Neuter the new segment in case there is an old one there */
245	1640	AN(sg->p.offset);
246	1640	smp_def_sign(sc, sg->ctx, sg->p.offset, "SEGHEAD");
247	1640	smp_reset_sign(sg->ctx);
248	1640	smp_sync_sign(sg->ctx);
249
250		/* Set up our allocation points */
251	1640	sc->cur_seg = sg;
252	1640	sc->next_bot = sg->p.offset + IRNUP(sc, SMP_SIGN_SPACE);
253	1640	sc->next_top = smp_segend(sg);
254	1640	sc->next_top -= IRNUP(sc, SMP_SIGN_SPACE);
255	1640	IASSERTALIGN(sc, sc->next_bot);
256	1640	IASSERTALIGN(sc, sc->next_top);
257	1640	sg->objs = (void*)(sc->base + sc->next_top);
258	1640	}
259
260		/*--------------------------------------------------------------------
261		* Close a segment
262		*/
263
264		void
265	1600	smp_close_seg(struct smp_sc sc, struct smp_seg sg)
266		{
267		uint64_t left, dst, len;
268		void *dp;
269
270	1600	CHECK_OBJ_NOTNULL(sc, SMP_SC_MAGIC);
271	1600	Lck_AssertHeld(&sc->mtx);
272
273	1600	CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC);
274	1600	assert(sg == sc->cur_seg);
275	1600	AN(sg->p.offset);
276	1600	sc->cur_seg = NULL;
277
278	1600	if (sg->nalloc == 0) {
279		/* If segment is empty, delete instead */
280	840	VTAILQ_REMOVE(&sc->segments, sg, list);
281	840	assert(sg->p.offset >= sc->ident->stuff[SMP_SPC_STUFF]);
282	840	assert(sg->p.offset < sc->mediasize);
283	840	sc->free_offset = sg->p.offset;
284	840	AN(VTAILQ_EMPTY(&sg->objcores));
285	840	FREE_OBJ(sg);
286	840	return;
287		}
288
289		/*
290		* If there is enough space left, that we can move the smp_objects
291		* down without overwriting the present copy, we will do so to
292		* compact the segment.
293		*/
294	760	left = smp_spaceleft(sc, sg);
295	760	len = sizeof(struct smp_object) * sg->p.lobjlist;
296	760	if (len < left) {
297	760	dst = sc->next_bot + IRNUP(sc, SMP_SIGN_SPACE);
298	760	dp = sc->base + dst;
299	760	assert((uintptr_t)dp + len < (uintptr_t)sg->objs);
300	760	memcpy(dp, sg->objs, len);
301	760	sc->next_top = dst;
302	760	sg->objs = dp;
303	1520	sg->p.length = (sc->next_top - sg->p.offset)
304	760	+ len + IRNUP(sc, SMP_SIGN_SPACE);
305	760	(void)smp_spaceleft(sc, sg); /* for the asserts */
306
307	760	}
308
309		/* Update the segment header */
310	760	sg->p.objlist = sc->next_top;
311
312		/* Write the (empty) OBJIDX signature */
313	760	sc->next_top -= IRNUP(sc, SMP_SIGN_SPACE);
314	760	assert(sc->next_top >= sc->next_bot);
315	760	smp_def_sign(sc, sg->ctx, sc->next_top, "OBJIDX");
316	760	smp_reset_sign(sg->ctx);
317	760	smp_sync_sign(sg->ctx);
318
319		/* Write the (empty) SEGTAIL signature */
320	1520	smp_def_sign(sc, sg->ctx,
321	760	sg->p.offset + sg->p.length - IRNUP(sc, SMP_SIGN_SPACE), "SEGTAIL");
322	760	smp_reset_sign(sg->ctx);
323	760	smp_sync_sign(sg->ctx);
324
325		/* Save segment list */
326	760	smp_save_segs(sc);
327	760	sc->free_offset = smp_segend(sg);
328	1600	}
329
330
331		/*---------------------------------------------------------------------
332		*/
333
334		static struct smp_object *
335	15240	smp_find_so(const struct smp_seg *sg, unsigned priv2)
336		{
337		struct smp_object *so;
338
339	15240	priv2 &= ~NEED_FIXUP;
340	15240	assert(priv2 > 0);
341	15240	assert(priv2 <= sg->p.lobjlist);
342	15240	so = &sg->objs[sg->p.lobjlist - priv2];
343	15240	return (so);
344		}
345
346		/*---------------------------------------------------------------------
347		* Check if a given storage structure is valid to use
348		*/
349
350		static int
351	80	smp_loaded_st(const struct smp_sc sc, const struct smp_seg sg,
352		const struct storage *st)
353		{
354		struct smp_seg *sg2;
355		const uint8_t *pst;
356		uint64_t o;
357
358	80	(void)sg; /* XXX: faster: Start search from here */
359	80	pst = (const void *)st;
360
361	80	if (pst < (sc->base + sc->ident->stuff[SMP_SPC_STUFF]))
362	0	return (0x01); /* Before silo payload start */
363	80	if (pst > (sc->base + sc->ident->stuff[SMP_END_STUFF]))
364	0	return (0x02); /* After silo end */
365
366	80	o = pst - sc->base;
367
368		/* Find which segment contains the storage structure */
369	160	VTAILQ_FOREACH(sg2, &sc->segments, list)
370	120	if (o > sg2->p.offset && (o + sizeof(*st)) < sg2->p.objlist)
371	40	break;
372	80	if (sg2 == NULL)
373	40	return (0x04); /* No claiming segment */
374	40	if (!(sg2->flags & SMP_SEG_LOADED))
375	0	return (0x08); /* Claiming segment not loaded */
376
377		/* It is now safe to access the storage structure */
378	40	if (st->magic != STORAGE_MAGIC)
379	0	return (0x10); /* Not enough magic */
380
381	40	if (o + st->space >= sg2->p.objlist)
382	0	return (0x20); /* Allocation not inside segment */
383
384	40	if (st->len > st->space)
385	0	return (0x40); /* Plain bad... */
386
387		/*
388		* XXX: We could patch up st->stevedore and st->priv here
389		* XXX: but if things go right, we will never need them.
390		*/
391	40	return (0);
392	80	}
393
394		/*---------------------------------------------------------------------
395		* objcore methods for persistent objects
396		*/
397
398		static void
399	75709	fix_ptr(const struct smp_seg sg, const struct storage st, void **ptr)
400		{
401		// See comment where used below
402		uintptr_t u;
403
404	75709	u = (uintptr_t)(*ptr);
405	75709	if (u != 0) {
406	52189	u -= (uintptr_t)st->priv;
407	52189	u += (uintptr_t)sg->sc->base;
408	52189	}
409	75709	ptr = (void )u;
410	75709	}
411
412		struct object * v_matchproto_(sml_getobj_f)
413	11600	smp_sml_getobj(struct worker wrk, struct objcore oc)
414		{
415		struct object *o;
416		struct smp_seg *sg;
417		struct smp_object *so;
418		struct storage st, st2;
419		uint64_t l;
420		int bad;
421
422	11600	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
423	11600	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
424	11600	AN(oc->stobj->stevedore);
425
426	11600	CAST_OBJ_NOTNULL(sg, oc->stobj->priv, SMP_SEG_MAGIC);
427	11600	so = smp_find_so(sg, oc->stobj->priv2);
428
429		/**************************************************************
430		* The silo may have been remapped at a different address,
431		* because the people who came up with ASLR were unable
432		* imagine that there might be beneficial use-cases for
433		* always mapping a file at the same specific address.
434		*
435		* We store the silos base address in struct storage->priv
436		* and manually fix all the pointers in struct object and
437		* the list of struct storage objects which hold the body.
438		* When done, we update the storage->priv, so we can do the
439		* same trick next time.
440		*
441		* This is a prohibitively expensive workaround, but we can
442		* live with it, because the role of this stevedore is only
443		* to keep the internal stevedore API honest.
444		*/
445
446	11600	st = (void*)(sg->sc->base + so->ptr);
447	11600	fix_ptr(sg, st, (void**)&st->ptr);
448
449	11600	o = (void*)st->ptr;
450	11600	fix_ptr(sg, st, (void**)&o->objstore);
451	11600	fix_ptr(sg, st, (void**)&o->va_vary);
452	11600	fix_ptr(sg, st, (void**)&o->va_headers);
453	11600	fix_ptr(sg, st, (void**)&o->list.vtqh_first);
454	11600	fix_ptr(sg, st, (void**)&o->list.vtqh_last);
455	11600	st->priv = (void*)(sg->sc->base);
456
457	11600	st2 = o->list.vtqh_first;
458	13637	while (st2 != NULL) {
459	2037	fix_ptr(sg, st2, (void**)&st2->list.vtqe_next);
460	2037	fix_ptr(sg, st2, (void**)&st2->list.vtqe_prev);
461	2037	fix_ptr(sg, st2, (void**)&st2->ptr);
462	2037	st2->priv = (void*)(sg->sc->base);
463	2037	st2 = st2->list.vtqe_next;
464		}
465
466		/*
467		* The object may not be in this segment since we allocate it
468		* In a separate operation than the smp_object. We could check
469		* that it is in a later segment, but that would be complicated.
470		* XXX: For now, be happy if it is inside the silo
471		*/
472	11600	ASSERT_PTR_IN_SILO(sg->sc, o);
473	11600	CHECK_OBJ_NOTNULL(o, OBJECT_MAGIC);
474
475		/*
476		* If this flag is not set, it will not be, and the lock is not
477		* needed to test it.
478		*/
479	11600	if (!(oc->stobj->priv2 & NEED_FIXUP))
480	11080	return (o);
481
482	520	Lck_Lock(&sg->sc->mtx);
483		/* Check again, we might have raced. */
484	520	if (oc->stobj->priv2 & NEED_FIXUP) {
485		/* We trust caller to have a refcnt for us */
486
487	520	bad = 0;
488	520	l = 0;
489	560	VTAILQ_FOREACH(st, &o->list, list) {
490	80	bad \|= smp_loaded_st(sg->sc, sg, st);
491	80	if (bad)
492	40	break;
493	40	l += st->len;
494	40	}
495	520	if (l != vbe64dec(o->fa_len))
496	40	bad \|= 0x100;
497
498	520	if (bad) {
499	40	EXP_ZERO(oc);