varnish-cache/lib/libvarnishapi/vsl_cursor.c
1
/*-
2
 * Copyright (c) 2006 Verdens Gang AS
3
 * Copyright (c) 2006-2015 Varnish Software AS
4
 * All rights reserved.
5
 *
6
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
7
 * Author: Martin Blix Grydeland <martin@varnish-software.com>
8
 *
9
 * Redistribution and use in source and binary forms, with or without
10
 * modification, are permitted provided that the following conditions
11
 * are met:
12
 * 1. Redistributions of source code must retain the above copyright
13
 *    notice, this list of conditions and the following disclaimer.
14
 * 2. Redistributions in binary form must reproduce the above copyright
15
 *    notice, this list of conditions and the following disclaimer in the
16
 *    documentation and/or other materials provided with the distribution.
17
 *
18
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
22
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28
 * SUCH DAMAGE.
29
 *
30
 */
31
32
#include "config.h"
33
34
#include <errno.h>
35
#include <fcntl.h>
36
#include <stdint.h>
37
#include <stdio.h>
38
#include <stdlib.h>
39
#include <string.h>
40
#include <sys/stat.h>
41
#include <sys/types.h>
42
#include <unistd.h>
43
44
#include "vdef.h"
45
#include "vas.h"
46
#include "miniobj.h"
47
#include "vmb.h"
48
49
#include "vqueue.h"
50
#include "vre.h"
51
#include "vsl_priv.h"
52
53
#include "vapi/vsl.h"
54
#include "vapi/vsm.h"
55
56
#include "vsl_api.h"
57
58
struct vslc_vsm {
59
        unsigned                        magic;
60
#define VSLC_VSM_MAGIC                  0x4D3903A6
61
62
        struct VSL_cursor               cursor;
63
64
        unsigned                        options;
65
66
        struct vsm                      *vsm;
67
        struct vsm_fantom               vf;
68
69
        const struct VSL_head           *head;
70
        const uint32_t                  *end;
71
        struct VSLC_ptr                 next;
72
};
73
74
static void
75 5264
vslc_vsm_delete(const struct VSL_cursor *cursor)
76
{
77
        struct vslc_vsm *c;
78
79 5264
        CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_VSM_MAGIC);
80 5264
        AZ(VSM_Unmap(c->vsm, &c->vf));
81 5264
        assert(&c->cursor == cursor);
82 5264
        FREE_OBJ(c);
83 5264
}
84
85
/*
86
 * We tolerate the fact that segment_n wraps around eventually: for the default
87
 * vsl_space of 80MB and 8 segments, each segment is 10MB long, so we wrap
88
 * roughly after 40 pebibytes (32bit) or 160 yobibytes (64bit) worth of vsl
89
 * written.
90
 *
91
 * The vsm_check would fail if a vslc paused while this amount of data was
92
 * written
93
 */
94
95
static int
96 1246305
vslc_vsm_check(const struct VSL_cursor *cursor, const struct VSLC_ptr *ptr)
97
{
98
        const struct vslc_vsm *c;
99
        unsigned dist;
100
101 1246305
        CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_VSM_MAGIC);
102 1246305
        assert(&c->cursor == cursor);
103
104 1246305
        if (ptr->ptr == NULL)
105 0
                return (0);
106
107 1246305
        dist = c->head->segment_n - ptr->priv;
108
109 1246305
        if (dist >= VSL_SEGMENTS - 2)
110
                /* Too close to continue */
111 0
                return (0);
112 1246305
        if (dist >= VSL_SEGMENTS - 4)
113
                /* Warning level */
114 0
                return (1);
115
        /* Safe */
116 1246305
        return (2);
117
}
118
119
static int
120 1134516
vslc_vsm_next(const struct VSL_cursor *cursor)
121
{
122
        struct vslc_vsm *c;
123
        int i;
124
        uint32_t t;
125
126 1134516
        CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_VSM_MAGIC);
127 1134516
        assert(&c->cursor == cursor);
128
129
        while (1) {
130 1241137
                i = vslc_vsm_check(&c->cursor, &c->next);
131 1241134
                if (i <= 0) {
132 0
                        if (VSM_StillValid(c->vsm, &c->vf) != VSM_valid)
133 0
                                return (-2); /* VSL abandoned */
134
                        else
135 0
                                return (-3); /* Overrun */
136
                }
137
138 1241134
                t = *(volatile const uint32_t *)c->next.ptr;
139 1241134
                AN(t);
140
141 1241134
                if (t == VSL_WRAPMARKER) {
142
                        /* Wrap around not possible at front */
143 0
                        assert(c->next.ptr != c->head->log);
144 0
                        c->next.ptr = c->head->log;
145 0
                        while (c->next.priv % VSL_SEGMENTS)
146 0
                                c->next.priv++;
147 0
                        continue;
148
                }
149
150 1241134
                if (t == VSL_ENDMARKER) {
151 32787
                        if (VSM_StillValid(c->vsm, &c->vf) != VSM_valid)
152 0
                                return (-2); /* VSL abandoned */
153 32790
                        if (c->options & VSL_COPT_TAILSTOP)
154 11
                                return (-1); /* EOF */
155 32779
                        return (0);     /* No new records available */
156
                }
157
158 1208347
                c->cursor.rec = c->next;
159 1208347
                c->next.ptr = VSL_NEXT(c->next.ptr);
160
161 1208347
                if (VSL_TAG(c->cursor.rec.ptr) == SLT__Batch) {
162 108845
                        if (!(c->options & VSL_COPT_BATCH))
163
                                /* Skip the batch record */
164 106621
                                continue;
165
                        /* Next call will point to the first record past
166
                           the batch */
167 4448
                        c->next.ptr +=
168 2224
                            VSL_WORDS(VSL_BATCHLEN(c->cursor.rec.ptr));
169
                }
170
171 3305178
                while ((c->next.ptr - c->head->log) / c->head->segsize >
172 1101726
                    c->next.priv % VSL_SEGMENTS)
173 0
                        c->next.priv++;
174
175 1101726
                assert(c->next.ptr >= c->head->log);
176 1101726
                assert(c->next.ptr < c->end);
177
178 1101726
                return (1);
179 106621
        }
180
}
181
182
static int
183 5264
vslc_vsm_reset(const struct VSL_cursor *cursor)
184
{
185
        struct vslc_vsm *c;
186
        unsigned u, segment_n;
187
        int i;
188
189 5264
        CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_VSM_MAGIC);
190 5264
        assert(&c->cursor == cursor);
191 5264
        c->cursor.rec.ptr = NULL;
192
193 5264
        segment_n = c->head->segment_n;
194 5264
        VRMB();                 /* Make sure offset table is not stale
195
                                   compared to segment_n */
196
197 5264
        if (c->options & VSL_COPT_TAIL) {
198
                /* Start in the same segment varnishd currently is in and
199
                   run forward until we see the end */
200 4608
                u = c->next.priv = segment_n;
201 4608
                assert(c->head->offset[c->next.priv % VSL_SEGMENTS] >= 0);
202 9216
                c->next.ptr = c->head->log +
203 4608
                    c->head->offset[c->next.priv % VSL_SEGMENTS];
204
                do {
205 959395
                        if (c->head->segment_n - u > 1) {
206
                                /* Give up if varnishd is moving faster
207
                                   than us */
208 0
                                return (-3); /* overrun */
209
                        }
210 959395
                        i = vslc_vsm_next(&c->cursor);
211 959395
                } while (i == 1);
212 4608
                if (i)
213 0
                        return (i);
214
        } else {
215
                /* Starting (VSL_SEGMENTS - 3) behind varnishd. This way
216
                 * even if varnishd advances segment_n immediately, we'll
217
                 * still have a full segment worth of log before the
218
                 * general constraint of at least 2 segments apart will be
219
                 * broken.
220
                 */
221 656
                c->next.priv = segment_n - (VSL_SEGMENTS - 3);
222 4592
                while (c->head->offset[c->next.priv % VSL_SEGMENTS] < 0) {
223
                        /* seg 0 must be initialized */
224 3280
                        assert(c->next.priv % VSL_SEGMENTS != 0);
225 3280
                        c->next.priv++;
226
                }
227 656
                assert(c->head->offset[c->next.priv % VSL_SEGMENTS] >= 0);
228 1312
                c->next.ptr = c->head->log +
229 656
                    c->head->offset[c->next.priv % VSL_SEGMENTS];
230
        }
231 5264
        assert(c->next.ptr >= c->head->log);
232 5264
        assert(c->next.ptr < c->end);
233 5264
        return (0);
234
}
235
236
static const struct vslc_tbl vslc_vsm_tbl = {
237
        .magic          = VSLC_TBL_MAGIC,
238
        .delete         = vslc_vsm_delete,
239
        .next           = vslc_vsm_next,
240
        .reset          = vslc_vsm_reset,
241
        .check          = vslc_vsm_check,
242
};
243
244
struct VSL_cursor *
245 28048
VSL_CursorVSM(struct VSL_data *vsl, struct vsm *vsm, unsigned options)
246
{
247
        struct vslc_vsm *c;
248
        struct vsm_fantom vf;
249
        struct VSL_head *head;
250
        int i;
251
252 28048
        CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
253
254 28048
        if (!VSM_Get(vsm, &vf, VSL_CLASS, NULL)) {
255 22678
                (void)vsl_diag(vsl,
256
                    "No VSL chunk found (child not started ?)");
257 22678
                return (NULL);
258
        }
259 5374
        if (VSM_Map(vsm, &vf)) {
260 110
                (void)vsl_diag(vsl,
261
                    "VSM_Map(): %s", VSM_Error(vsm));
262 110
                return (NULL);
263
        }
264 5264
        AN(vf.b);
265
266 5264
        head = vf.b;
267 5264
        if (memcmp(head->marker, VSL_HEAD_MARKER, sizeof head->marker)) {
268 0
                AZ(VSM_Unmap(vsm, &vf));
269 0
                (void)vsl_diag(vsl, "Not a VSL chunk");
270 0
                return (NULL);
271
        }
272 5264
        ALLOC_OBJ(c, VSLC_VSM_MAGIC);
273 5264
        if (c == NULL) {
274 0
                AZ(VSM_Unmap(vsm, &vf));
275 0
                (void)vsl_diag(vsl, "Out of memory");
276 0
                return (NULL);
277
        }
278 5264
        c->cursor.priv_tbl = &vslc_vsm_tbl;
279 5264
        c->cursor.priv_data = c;
280
281 5264
        c->options = options;
282 5264
        c->vsm = vsm;
283 5264
        c->vf = vf;
284 5264
        c->head = head;
285 5264
        c->end = c->head->log + c->head->segsize * VSL_SEGMENTS;
286 5264
        assert(c->end <= (const uint32_t *)vf.e);
287
288 5264
        i = vslc_vsm_reset(&c->cursor);
289 5264
        if (i) {
290 0
                (void)vsl_diag(vsl, "Cursor initialization failure (%d)", i);
291 0
                FREE_OBJ(c);
292 0
                return (NULL);
293
        }
294
295 5264
        return (&c->cursor);
296
}
297
298
struct vslc_file {
299
        unsigned                        magic;
300
#define VSLC_FILE_MAGIC                 0x1D65FFEF
301
302
        int                             error;
303
        int                             fd;
304
        int                             close_fd;
305
        ssize_t                         buflen;
306
        uint32_t                        *buf;
307
308
        struct VSL_cursor               cursor;
309
310
};
311
312
static void
313 3
vslc_file_delete(const struct VSL_cursor *cursor)
314
{
315
        struct vslc_file *c;
316
317 3
        CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_FILE_MAGIC);
318 3
        assert(&c->cursor == cursor);
319 3
        if (c->close_fd)
320 3
                (void)close(c->fd);
321 3
        if (c->buf != NULL)
322 3
                free(c->buf);
323 3
        FREE_OBJ(c);
324 3
}
325
326
/* Read n bytes from fd into buf */
327
static ssize_t
328 438
vslc_file_readn(int fd, void *buf, size_t n)
329
{
330 438
        ssize_t t = 0;
331
        ssize_t l;
332
333 1311
        while (t < n) {
334 438
                l = read(fd, (char *)buf + t, n - t);
335 438
                if (l <= 0)
336 3
                        return (l);
337 435
                t += l;
338
        }
339 435
        return (t);
340
}
341
342
static int
343 219
vslc_file_next(const struct VSL_cursor *cursor)
344
{
345
        struct vslc_file *c;
346
        ssize_t i;
347
        ssize_t l;
348
349 219
        CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_FILE_MAGIC);
350 219
        assert(&c->cursor == cursor);
351
352 219
        if (c->error)
353 0
                return (c->error);
354
355
        do {
356 219
                c->cursor.rec.ptr = NULL;
357 219
                assert(c->buflen >= 2);
358 219
                i = vslc_file_readn(c->fd, c->buf, VSL_BYTES(2));
359 219
                if (i < 0)
360 0
                        return (-4);    /* I/O error */
361 219
                if (i == 0)
362 3
                        return (-1);    /* EOF */
363 216
                assert(i == VSL_BYTES(2));
364 216
                l = 2 + VSL_WORDS(VSL_LEN(c->buf));
365 216
                if (c->buflen < l) {
366 0
                        while (c->buflen < l)
367 0
                                c->buflen = 2 * l;
368 0
                        c->buf = realloc(c->buf, VSL_BYTES(c->buflen));
369 0
                        AN(c->buf);
370
                }
371 216
                if (l > 2) {
372 216
                        i = vslc_file_readn(c->fd, c->buf + 2,
373 216
                            VSL_BYTES(l - 2));
374 216
                        if (i < 0)
375 0
                                return (-4);    /* I/O error */
376 216
                        if (i == 0)
377 0
                                return (-1);    /* EOF */
378 216
                        assert(i == VSL_BYTES(l - 2));
379
                }
380 216
                c->cursor.rec.ptr = c->buf;
381 216
        } while (VSL_TAG(c->cursor.rec.ptr) == SLT__Batch);
382 216
        return (1);
383
}
384
385
static int
386 0
vslc_file_reset(const struct VSL_cursor *cursor)
387
{
388
        (void)cursor;
389
        /* XXX: Implement me */
390 0
        return (-1);
391
}
392
393
static const struct vslc_tbl vslc_file_tbl = {
394
        .magic          = VSLC_TBL_MAGIC,
395
        .delete         = vslc_file_delete,
396
        .next           = vslc_file_next,
397
        .reset          = vslc_file_reset,
398
        .check          = NULL,
399
};
400
401
struct VSL_cursor *
402 3
VSL_CursorFile(struct VSL_data *vsl, const char *name, unsigned options)
403
{
404
        struct vslc_file *c;
405
        int fd;
406 3
        int close_fd = 0;
407 3
        char buf[] = VSL_FILE_ID;
408
        ssize_t i;
409
410 3
        CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
411 3
        AN(name);
412
        (void)options;
413
414 3
        if (!strcmp(name, "-"))
415 0
                fd = STDIN_FILENO;
416
        else {
417 3
                fd = open(name, O_RDONLY);
418 3
                if (fd < 0) {
419 0
                        vsl_diag(vsl, "Cannot open %s: %s", name,
420 0
                            strerror(errno));
421 0
                        return (NULL);
422
                }
423 3
                close_fd = 1;
424
        }
425
426 3
        i = vslc_file_readn(fd, buf, sizeof buf);
427 3
        if (i <= 0) {
428 0
                if (close_fd)
429 0
                        (void)close(fd);
430 0
                vsl_diag(vsl, "VSL file read error: %s",
431 0
                    i < 0 ? strerror(errno) : "EOF");
432 0
                return (NULL);
433
        }
434 3
        assert(i == sizeof buf);
435 3
        if (memcmp(buf, VSL_FILE_ID, sizeof buf)) {
436 0
                if (close_fd)
437 0
                        (void)close(fd);
438 0
                vsl_diag(vsl, "Not a VSL file: %s", name);
439 0
                return (NULL);
440
        }
441
442 3
        ALLOC_OBJ(c, VSLC_FILE_MAGIC);
443 3
        if (c == NULL) {
444 0
                if (close_fd)
445 0
                        (void)close(fd);
446 0
                vsl_diag(vsl, "Out of memory");
447 0
                return (NULL);
448
        }
449 3
        c->cursor.priv_tbl = &vslc_file_tbl;
450 3
        c->cursor.priv_data = c;
451
452 3
        c->fd = fd;
453 3
        c->close_fd = close_fd;
454 3
        c->buflen = VSL_WORDS(BUFSIZ);
455 3
        c->buf = malloc(VSL_BYTES(c->buflen));
456 3
        AN(c->buf);
457
458 3
        return (&c->cursor);
459
}
460
461
void
462 5267
VSL_DeleteCursor(const struct VSL_cursor *cursor)
463
{
464
        const struct vslc_tbl *tbl;
465
466 5267
        CAST_OBJ_NOTNULL(tbl, cursor->priv_tbl, VSLC_TBL_MAGIC);
467 5267
        if (tbl->delete == NULL)
468 5267
                return;
469 5267
        (tbl->delete)(cursor);
470
}
471
472
int
473 937
VSL_ResetCursor(const struct VSL_cursor *cursor)
474
{
475
        const struct vslc_tbl *tbl;
476
477 937
        CAST_OBJ_NOTNULL(tbl, cursor->priv_tbl, VSLC_TBL_MAGIC);
478 937
        if (tbl->reset == NULL)
479 0
                return (-1);
480 937
        return ((tbl->reset)(cursor));
481
}
482
483
int
484 198502
VSL_Next(const struct VSL_cursor *cursor)
485
{
486
        const struct vslc_tbl *tbl;
487
488 198502
        CAST_OBJ_NOTNULL(tbl, cursor->priv_tbl, VSLC_TBL_MAGIC);
489 198502
        AN(tbl->next);
490 198502
        return ((tbl->next)(cursor));
491
}
492
493
int
494 5383
VSL_Check(const struct VSL_cursor *cursor, const struct VSLC_ptr *ptr)
495
{
496
        const struct vslc_tbl *tbl;
497
498 5383
        CAST_OBJ_NOTNULL(tbl, cursor->priv_tbl, VSLC_TBL_MAGIC);
499 5383
        if (tbl->check == NULL)
500 216
                return (-1);
501 5167
        return ((tbl->check)(cursor, ptr));
502
}