varnish-cache/bin/varnishd/storage/mgt_storage_persistent.c
0
/*-
1
 * Copyright (c) 2008-2011 Varnish Software AS
2
 * All rights reserved.
3
 *
4
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
5
 *
6
 * SPDX-License-Identifier: BSD-2-Clause
7
 *
8
 * Redistribution and use in source and binary forms, with or without
9
 * modification, are permitted provided that the following conditions
10
 * are met:
11
 * 1. Redistributions of source code must retain the above copyright
12
 *    notice, this list of conditions and the following disclaimer.
13
 * 2. Redistributions in binary form must reproduce the above copyright
14
 *    notice, this list of conditions and the following disclaimer in the
15
 *    documentation and/or other materials provided with the distribution.
16
 *
17
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
21
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27
 * SUCH DAMAGE.
28
 *
29
 * Persistent storage method
30
 *
31
 * XXX: Before we start the client or maybe after it stops, we should give the
32
 * XXX: stevedores a chance to examine their storage for consistency.
33
 *
34
 * XXX: Do we ever free the LRU-lists ?
35
 */
36
37
#include "config.h"
38
39
#include "cache/cache_varnishd.h"
40
#include "common/heritage.h"
41
42
#include <sys/mman.h>
43
44
#include <stdio.h>
45
#include <stdlib.h>
46
47
#include "storage/storage.h"
48
#include "storage/storage_simple.h"
49
50
#include "vsha256.h"
51
52
#include "storage/storage_persistent.h"
53
54
#ifdef HAVE_SYS_PERSONALITY_H
55
#include <sys/personality.h>
56
#endif
57
58
#ifndef MAP_NOCORE
59
#ifdef MAP_CONCEAL
60
#define MAP_NOCORE MAP_CONCEAL /* XXX OpenBSD */
61
#else
62
#define MAP_NOCORE 0 /* XXX Linux */
63
#endif
64
#endif
65
66
#ifndef MAP_NOSYNC
67
#define MAP_NOSYNC 0 /* XXX Linux */
68
#endif
69
70
/*--------------------------------------------------------------------
71
 * Calculate cleaner metrics from silo dimensions
72
 */
73
74
static void
75 950
smp_metrics(struct smp_sc *sc)
76
{
77
78
        /*
79
         * We do not want to loose too big chunks of the silos
80
         * content when we are forced to clean a segment.
81
         *
82
         * For now insist that a segment covers no more than 1% of the silo.
83
         *
84
         * XXX: This should possibly depend on the size of the silo so
85
         * XXX: trivially small silos do not run into trouble along
86
         * XXX: the lines of "one object per segment".
87
         */
88
89 950
        sc->min_nseg = 10;
90 950
        sc->max_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->min_nseg;
91
92 1900
        fprintf(stderr, "min_nseg = %u, max_segl = %ju\n",
93 950
            sc->min_nseg, (uintmax_t)sc->max_segl);
94
95
        /*
96
         * The number of segments are limited by the size of the segment
97
         * table(s) and from that follows the minimum size of a segmement.
98
         */
99
100 950
        sc->max_nseg = smp_stuff_len(sc, SMP_SEG1_STUFF) / sc->min_nseg;
101 950
        sc->min_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->max_nseg;
102
103 4750
        while (sc->min_segl < sizeof(struct object)) {
104 3800
                sc->max_nseg /= 2;
105 3800
                sc->min_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->max_nseg;
106
        }
107
108 1900
        fprintf(stderr, "max_nseg = %u, min_segl = %ju\n",
109 950
            sc->max_nseg, (uintmax_t)sc->min_segl);
110
111
        /*
112
         * Set our initial aim point at the exponential average of the
113
         * two extremes.
114
         *
115
         * XXX: This is a pretty arbitrary choice, but having no idea
116
         * XXX: object count, size distribution or ttl pattern at this
117
         * XXX: point, we have to do something.
118
         */
119
120 950
        sc->aim_nseg =
121 950
           (unsigned) exp((log(sc->min_nseg) + log(sc->max_nseg))*.5);
122 950
        sc->aim_segl = smp_stuff_len(sc, SMP_SPC_STUFF) / sc->aim_nseg;
123
124 1900
        fprintf(stderr, "aim_nseg = %u, aim_segl = %ju\n",
125 950
            sc->aim_nseg, (uintmax_t)sc->aim_segl);
126
127
        /*
128
         * How much space in the free reserve pool ?
129
         */
130 950
        sc->free_reserve = sc->aim_segl * 10;
131
132 950
        fprintf(stderr, "free_reserve = %ju\n", (uintmax_t)sc->free_reserve);
133 950
}
134
135
/*--------------------------------------------------------------------
136
 * Set up persistent storage silo in the master process.
137
 */
138
139
void v_matchproto_(storage_init_f)
140 950
smp_mgt_init(struct stevedore *parent, int ac, char * const *av)
141
{
142
        struct smp_sc           *sc;
143
        void *target;
144
        int i, mmap_flags;
145
146 950
        AZ(av[ac]);
147
148
#ifdef HAVE_SYS_PERSONALITY_H
149
        i = personality(0xffffffff); /* Fetch old personality. */
150
        if (!(i & ADDR_NO_RANDOMIZE)) {
151
                i = personality(i | ADDR_NO_RANDOMIZE);
152
                if (i < 0)
153
                        fprintf(stderr, "WARNING: Could not disable ASLR\n");
154
                else
155
                        fprintf(stderr, "NB: Disabled ASLR for Persistent\n");
156
        }
157
#endif
158
159
        /* Necessary alignment. See also smp_object::__filler__ */
160 950
        assert(sizeof(struct smp_object) % 8 == 0);
161
162
#define SIZOF(foo)       fprintf(stderr, \
163
    "sizeof(%s) = %zu = 0x%zx\n", #foo, sizeof(foo), sizeof(foo));
164 950
        SIZOF(struct smp_ident);
165 950
        SIZOF(struct smp_sign);
166 950
        SIZOF(struct smp_segptr);
167 950
        SIZOF(struct smp_object);
168
#undef SIZOF
169
170
        /* See comments in storage_persistent.h */
171 950
        assert(sizeof(struct smp_ident) == SMP_IDENT_SIZE);
172
173
        /* Allocate softc */
174 950
        ALLOC_OBJ(sc, SMP_SC_MAGIC);
175 950
        XXXAN(sc);
176 950
        sc->parent = parent;
177 950
        sc->fd = -1;
178 950
        VTAILQ_INIT(&sc->segments);
179
180
        /* Argument processing */
181 950
        if (ac != 2)
182 0
                ARGV_ERR("(-spersistent) wrong number of arguments\n");
183
184 950
        i = STV_GetFile(av[0], &sc->fd, &sc->filename, "-spersistent");
185 950
        if (i == 2)
186 0
                ARGV_ERR("(-spersistent) need filename (not directory)\n");
187
188 950
        sc->align = sizeof(void*) * 2;
189 950
        sc->granularity = getpagesize();
190 950
        sc->mediasize = STV_FileSize(sc->fd, av[1], &sc->granularity,
191
            "-spersistent");
192
193 950
        AZ(ftruncate(sc->fd, sc->mediasize));
194
195
        /* Try to determine correct mmap address */
196 950
        target = NULL;
197
198 950
        mmap_flags = MAP_NOCORE | MAP_NOSYNC | MAP_SHARED;
199
200
#ifdef MAP_ALIGNED_SUPER
201 950
        mmap_flags |= MAP_ALIGNED_SUPER;
202
#endif
203
204 1900
        sc->base = (void*)mmap(target, sc->mediasize, PROT_READ|PROT_WRITE,
205 950
            mmap_flags, sc->fd, 0);
206
207 950
        if (sc->base == MAP_FAILED)
208 0
                ARGV_ERR("(-spersistent) failed to mmap (%s) @%p\n",
209
                    VAS_errtxt(errno), target);
210
211 950
        smp_def_sign(sc, &sc->idn, 0, "SILO");
212 950
        sc->ident = SIGN_DATA(&sc->idn);
213
214 950
        i = smp_valid_silo(sc);
215 950
        if (i) {
216 425
                printf("Warning SILO (%s) not reloaded (reason=%d)\n",
217 425
                    sc->filename, i);
218 425
                smp_newsilo(sc);
219 425
        }
220 950
        AZ(smp_valid_silo(sc));
221
222 950
        smp_metrics(sc);
223
224 950
        parent->priv = sc;
225
226
        /* XXX: only for sendfile I guess... */
227 950
        MCH_Fd_Inherit(sc->fd, "storage_persistent");
228 950
}