varnish-cache/lib/libvarnishapi/vsl_cursor.c
1
/*-
2
 * Copyright (c) 2006 Verdens Gang AS
3
 * Copyright (c) 2006-2015 Varnish Software AS
4
 * All rights reserved.
5
 *
6
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
7
 * Author: Martin Blix Grydeland <martin@varnish-software.com>
8
 *
9
 * Redistribution and use in source and binary forms, with or without
10
 * modification, are permitted provided that the following conditions
11
 * are met:
12
 * 1. Redistributions of source code must retain the above copyright
13
 *    notice, this list of conditions and the following disclaimer.
14
 * 2. Redistributions in binary form must reproduce the above copyright
15
 *    notice, this list of conditions and the following disclaimer in the
16
 *    documentation and/or other materials provided with the distribution.
17
 *
18
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
22
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28
 * SUCH DAMAGE.
29
 *
30
 */
31
32
#include "config.h"
33
34
#include <errno.h>
35
#include <fcntl.h>
36
#include <stdint.h>
37
#include <stdio.h>
38
#include <stdlib.h>
39
#include <string.h>
40
#include <sys/types.h>
41
#include <unistd.h>
42
43
#include "vdef.h"
44
#include "vas.h"
45
#include "miniobj.h"
46
#include "vmb.h"
47
48
#include "vqueue.h"
49
#include "vre.h"
50
#include "vsl_priv.h"
51
52
#include "vapi/vsl.h"
53
#include "vapi/vsm.h"
54
55
#include "vsl_api.h"
56
57
struct vslc_vsm {
58
        unsigned                        magic;
59
#define VSLC_VSM_MAGIC                  0x4D3903A6
60
61
        struct VSL_cursor               cursor;
62
63
        unsigned                        options;
64
65
        struct vsm                      *vsm;
66
        struct vsm_fantom               vf;
67
68
        const struct VSL_head           *head;
69
        const uint32_t                  *end;
70
        struct VSLC_ptr                 next;
71
};
72
73
static void
74 6388
vslc_vsm_delete(const struct VSL_cursor *cursor)
75
{
76
        struct vslc_vsm *c;
77
78 6388
        CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_VSM_MAGIC);
79 6388
        AZ(VSM_Unmap(c->vsm, &c->vf));
80 6388
        assert(&c->cursor == cursor);
81 6388
        FREE_OBJ(c);
82 6388
}
83
84
/*
85
 * We tolerate the fact that segment_n wraps around eventually: for the default
86
 * vsl_space of 80MB and 8 segments, each segment is 10MB long, so we wrap
87
 * roughly after 40 pebibytes (32bit) or 160 yobibytes (64bit) worth of vsl
88
 * written.
89
 *
90
 * The vsm_check would fail if a vslc paused while this amount of data was
91
 * written
92
 */
93
94
static enum vsl_check v_matchproto_(vslc_check_f)
95 1405968
vslc_vsm_check(const struct VSL_cursor *cursor, const struct VSLC_ptr *ptr)
96
{
97
        const struct vslc_vsm *c;
98
        unsigned dist;
99
100 1405968
        CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_VSM_MAGIC);
101 1405968
        assert(&c->cursor == cursor);
102
103 1405968
        if (ptr->ptr == NULL)
104 0
                return (vsl_check_e_inval);
105
106 1405968
        dist = c->head->segment_n - ptr->priv;
107
108 1405968
        if (dist >= VSL_SEGMENTS - 2)
109
                /* Too close to continue */
110 0
                return (vsl_check_e_inval);
111 1405968
        if (dist >= VSL_SEGMENTS - 4)
112
                /* Warning level */
113 0
                return (vsl_check_warn);
114
        /* Safe */
115 1405968
        return (vsl_check_valid);
116
}
117
118
static enum vsl_status v_matchproto_(vslc_next_f)
119 1276961
vslc_vsm_next(const struct VSL_cursor *cursor)
120
{
121
        struct vslc_vsm *c;
122
        enum vsl_check i;
123
        uint32_t t;
124
125 1276961
        CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_VSM_MAGIC);
126 1276961
        assert(&c->cursor == cursor);
127
128
        while (1) {
129 1513223
                i = vslc_vsm_check(&c->cursor, &c->next);
130 1395097
                if (i < vsl_check_warn) {
131 0
                        if (VSM_StillValid(c->vsm, &c->vf) != VSM_valid)
132 0
                                return (vsl_e_abandon);
133
                        else
134 0
                                return (vsl_e_overrun);
135
                }
136
137 1395097
                t = *(volatile const uint32_t *)c->next.ptr;
138 1395097
                AN(t);
139
140 1395097
                if (t == VSL_WRAPMARKER) {
141
                        /* Wrap around not possible at front */
142 0
                        assert(c->next.ptr != c->head->log);
143 0
                        c->next.ptr = c->head->log;
144 0
                        while (c->next.priv % VSL_SEGMENTS)
145 0
                                c->next.priv++;
146 0
                        continue;
147
                }
148
149 1395097
                if (t == VSL_ENDMARKER) {
150 69148
                        if (VSM_StillValid(c->vsm, &c->vf) != VSM_valid)
151 0
                                return (vsl_e_abandon);
152 69148
                        if (c->options & VSL_COPT_TAILSTOP)
153 40
                                return (vsl_e_eof);
154
                        /* No new records available */
155 69108
                        return (vsl_end);
156
                }
157
158 1325949
                c->cursor.rec = c->next;
159 1325949
                c->next.ptr = VSL_NEXT(c->next.ptr);
160
161 1325949
                if (VSL_TAG(c->cursor.rec.ptr) == SLT__Batch) {
162 122931
                        if (!(c->options & VSL_COPT_BATCH))
163
                                /* Skip the batch record */
164 118131
                                continue;
165
                        /* Next call will point to the first record past
166
                           the batch */
167 9600
                        c->next.ptr +=
168 4800
                            VSL_WORDS(VSL_BATCHLEN(c->cursor.rec.ptr));
169
                }
170
171 3623454
                while ((c->next.ptr - c->head->log) / c->head->segsize >
172 1207818
                    c->next.priv % VSL_SEGMENTS)
173 0
                        c->next.priv++;
174
175 1207818
                assert(c->next.ptr >= c->head->log);
176 1207818
                assert(c->next.ptr < c->end);
177
178 1207818
                return (vsl_more);
179
        }
180
}
181
182
static enum vsl_status v_matchproto_(vslc_reset_f)
183 6388
vslc_vsm_reset(const struct VSL_cursor *cursor)
184
{
185
        struct vslc_vsm *c;
186
        unsigned u, segment_n;
187
        enum vsl_status r;
188
189 6388
        CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_VSM_MAGIC);
190 6388
        assert(&c->cursor == cursor);
191 6388
        c->cursor.rec.ptr = NULL;
192
193 6388
        segment_n = c->head->segment_n;
194
        /* Make sure offset table is not stale compared to segment_n */
195 6388
        VRMB();
196
197 6388
        if (c->options & VSL_COPT_TAIL) {
198
                /* Start in the same segment varnishd currently is in and
199
                   run forward until we see the end */
200 4868
                u = c->next.priv = segment_n;
201 4868
                assert(c->head->offset[c->next.priv % VSL_SEGMENTS] >= 0);
202 9736
                c->next.ptr = c->head->log +
203 4868
                    c->head->offset[c->next.priv % VSL_SEGMENTS];
204
                do {
205 864300
                        if (c->head->segment_n - u > 1) {
206
                                /* Give up if varnishd is moving faster
207
                                   than us */
208 0
                                return (vsl_e_overrun);
209
                        }
210 864300
                        r = vslc_vsm_next(&c->cursor);
211 864300
                } while (r == vsl_more);
212 4868
                if (r != vsl_end)
213 0
                        return (r);
214
        } else {
215
                /* Starting (VSL_SEGMENTS - 3) behind varnishd. This way
216
                 * even if varnishd advances segment_n immediately, we'll
217
                 * still have a full segment worth of log before the
218
                 * general constraint of at least 2 segments apart will be
219
                 * broken.
220
                 */
221 1520
                c->next.priv = segment_n - (VSL_SEGMENTS - 3);
222 10640
                while (c->head->offset[c->next.priv % VSL_SEGMENTS] < 0) {
223
                        /* seg 0 must be initialized */
224 7600
                        assert(c->next.priv % VSL_SEGMENTS != 0);
225 7600
                        c->next.priv++;
226
                }
227 1520
                assert(c->head->offset[c->next.priv % VSL_SEGMENTS] >= 0);
228 3040
                c->next.ptr = c->head->log +
229 1520
                    c->head->offset[c->next.priv % VSL_SEGMENTS];
230
        }
231 6388
        assert(c->next.ptr >= c->head->log);
232 6388
        assert(c->next.ptr < c->end);
233 6388
        return (vsl_end);
234
}
235
236
static const struct vslc_tbl vslc_vsm_tbl = {
237
        .magic          = VSLC_TBL_MAGIC,
238
        .delete         = vslc_vsm_delete,
239
        .next           = vslc_vsm_next,
240
        .reset          = vslc_vsm_reset,
241
        .check          = vslc_vsm_check,
242
};
243
244
struct VSL_cursor *
245 53531
VSL_CursorVSM(struct VSL_data *vsl, struct vsm *vsm, unsigned options)
246
{
247
        struct vslc_vsm *c;
248
        struct vsm_fantom vf;
249
        struct VSL_head *head;
250
        enum vsl_status r;
251
252 53531
        CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
253
254 53531
        if (!VSM_Get(vsm, &vf, VSL_CLASS, NULL)) {
255 46938
                (void)vsl_diag(vsl,
256
                    "No VSL chunk found (child not started ?)");
257 46938
                return (NULL);
258
        }
259 6594
        if (VSM_Map(vsm, &vf)) {
260 205
                (void)vsl_diag(vsl,
261
                    "VSM_Map(): %s", VSM_Error(vsm));
262 205
                return (NULL);
263
        }
264 6389
        AN(vf.b);
265
266 6389
        head = vf.b;
267 6389
        if (memcmp(head->marker, VSL_HEAD_MARKER, sizeof head->marker)) {
268 1
                AZ(VSM_Unmap(vsm, &vf));
269 1
                (void)vsl_diag(vsl, "Not a VSL chunk");
270 1
                return (NULL);
271
        }
272 6388
        ALLOC_OBJ(c, VSLC_VSM_MAGIC);
273 6388
        if (c == NULL) {
274 0
                AZ(VSM_Unmap(vsm, &vf));
275 0
                (void)vsl_diag(vsl, "Out of memory");
276 0
                return (NULL);
277
        }
278 6388
        c->cursor.priv_tbl = &vslc_vsm_tbl;
279 6388
        c->cursor.priv_data = c;
280
281 6388
        c->options = options;
282 6388
        c->vsm = vsm;
283 6388
        c->vf = vf;
284 6388
        c->head = head;
285 6388
        c->end = c->head->log + c->head->segsize * VSL_SEGMENTS;
286 6388
        assert(c->end <= (const uint32_t *)vf.e);
287
288 6388
        r = vslc_vsm_reset(&c->cursor);
289 6388
        if (r != vsl_end) {
290 0
                (void)vsl_diag(vsl, "Cursor initialization failure (%d)", r);
291 0
                FREE_OBJ(c);
292 0
                return (NULL);
293
        }
294
295 6388
        return (&c->cursor);
296
}
297
298
struct vslc_file {
299
        unsigned                        magic;
300
#define VSLC_FILE_MAGIC                 0x1D65FFEF
301
302
        int                             fd;
303
        int                             close_fd;
304
        ssize_t                         buflen;
305
        uint32_t                        *buf;
306
307
        struct VSL_cursor               cursor;
308
309
};
310
311
static void
312 6
vslc_file_delete(const struct VSL_cursor *cursor)
313
{
314
        struct vslc_file *c;
315
316 6
        CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_FILE_MAGIC);
317 6
        assert(&c->cursor == cursor);
318 6
        if (c->close_fd)
319 6
                (void)close(c->fd);
320 6
        if (c->buf != NULL)
321 6
                free(c->buf);
322 6
        FREE_OBJ(c);
323 6
}
324
325
/* Read n bytes from fd into buf */
326
static ssize_t
327 840
vslc_file_readn(int fd, void *buf, size_t n)
328
{
329 840
        ssize_t t = 0;
330
        ssize_t l;
331
332 2514
        while (t < n) {
333 840
                l = read(fd, (char *)buf + t, n - t);
334 840
                if (l <= 0)
335 6
                        return (l);
336 834
                t += l;
337
        }
338 834
        return (t);
339
}
340
341
static enum vsl_status v_matchproto_(vslc_next_f)
342 420
vslc_file_next(const struct VSL_cursor *cursor)
343
{
344
        struct vslc_file *c;
345
        ssize_t i;
346
        ssize_t l;
347
348 420
        CAST_OBJ_NOTNULL(c, cursor->priv_data, VSLC_FILE_MAGIC);
349 420
        assert(&c->cursor == cursor);
350
351
        do {
352 420
                c->cursor.rec.ptr = NULL;
353 420
                assert(c->buflen >= 2);
354 420
                i = vslc_file_readn(c->fd, c->buf, VSL_BYTES(2));
355 420
                if (i < 0)
356 0
                        return (vsl_e_io);
357 420
                if (i == 0)
358 6
                        return (vsl_e_eof);
359 414
                assert(i == VSL_BYTES(2));
360 414
                l = 2 + VSL_WORDS(VSL_LEN(c->buf));
361 414
                if (c->buflen < l) {
362 0
                        while (c->buflen < l)
363 0
                                c->buflen = 2 * l;
364 0
                        c->buf = realloc(c->buf, VSL_BYTES(c->buflen));
365 0
                        AN(c->buf);
366
                }
367 414
                if (l > 2) {
368 414
                        i = vslc_file_readn(c->fd, c->buf + 2,
369 414
                            VSL_BYTES(l - 2));
370 414
                        if (i < 0)
371 0
                                return (vsl_e_io);
372 414
                        if (i == 0)
373 0
                                return (vsl_e_eof);
374 414
                        assert(i == VSL_BYTES(l - 2));
375
                }
376 414
                c->cursor.rec.ptr = c->buf;
377 414
        } while (VSL_TAG(c->cursor.rec.ptr) == SLT__Batch);
378 414
        return (vsl_more);
379
}
380
381
static enum vsl_status v_matchproto_(vslc_reset_f)
382 0
vslc_file_reset(const struct VSL_cursor *cursor)
383
{
384
        (void)cursor;
385
        /* XXX: Implement me */
386 0
        return (vsl_e_eof);
387
}
388
389
static const struct vslc_tbl vslc_file_tbl = {
390
        .magic          = VSLC_TBL_MAGIC,
391
        .delete         = vslc_file_delete,
392
        .next           = vslc_file_next,
393
        .reset          = vslc_file_reset,
394
        .check          = NULL,
395
};
396
397
struct VSL_cursor *
398 6
VSL_CursorFile(struct VSL_data *vsl, const char *name, unsigned options)
399
{
400
        struct vslc_file *c;
401
        int fd;
402 6
        int close_fd = 0;
403 6
        char buf[] = VSL_FILE_ID;
404
        ssize_t i;
405
406 6
        CHECK_OBJ_NOTNULL(vsl, VSL_MAGIC);
407 6
        AN(name);
408
        (void)options;
409
410 6
        if (!strcmp(name, "-"))
411 0
                fd = STDIN_FILENO;
412
        else {
413 6
                fd = open(name, O_RDONLY);
414 6
                if (fd < 0) {
415 0
                        vsl_diag(vsl, "Cannot open %s: %s", name,
416 0
                            strerror(errno));
417 0
                        return (NULL);
418
                }
419 6
                close_fd = 1;
420
        }
421
422 6
        i = vslc_file_readn(fd, buf, sizeof buf);
423 6
        if (i <= 0) {
424 0
                if (close_fd)
425 0
                        (void)close(fd);
426 0
                vsl_diag(vsl, "VSL file read error: %s",
427 0
                    i < 0 ? strerror(errno) : "EOF");
428 0
                return (NULL);
429
        }
430 6
        assert(i == sizeof buf);
431 6
        if (memcmp(buf, VSL_FILE_ID, sizeof buf)) {
432 0
                if (close_fd)
433 0
                        (void)close(fd);
434 0
                vsl_diag(vsl, "Not a VSL file: %s", name);
435 0
                return (NULL);
436
        }
437
438 6
        ALLOC_OBJ(c, VSLC_FILE_MAGIC);
439 6
        if (c == NULL) {
440 0
                if (close_fd)
441 0
                        (void)close(fd);
442 0
                vsl_diag(vsl, "Out of memory");
443 0
                return (NULL);
444
        }
445 6
        c->cursor.priv_tbl = &vslc_file_tbl;
446 6
        c->cursor.priv_data = c;
447
448 6
        c->fd = fd;
449 6
        c->close_fd = close_fd;
450 6
        c->buflen = VSL_WORDS(BUFSIZ);
451 6
        c->buf = malloc(VSL_BYTES(c->buflen));
452 6
        AN(c->buf);
453
454 6
        return (&c->cursor);
455
}
456
457
void
458 6394
VSL_DeleteCursor(const struct VSL_cursor *cursor)
459
{
460
        const struct vslc_tbl *tbl;
461
462 6394
        CAST_OBJ_NOTNULL(tbl, cursor->priv_tbl, VSLC_TBL_MAGIC);
463 6394
        if (tbl->delete == NULL)
464 0
                return;
465 6394
        (tbl->delete)(cursor);
466
}
467
468
enum vsl_status
469 1862
VSL_ResetCursor(const struct VSL_cursor *cursor)
470
{
471
        const struct vslc_tbl *tbl;
472
473 1862
        CAST_OBJ_NOTNULL(tbl, cursor->priv_tbl, VSLC_TBL_MAGIC);
474 1862
        if (tbl->reset == NULL)
475 0
                return (vsl_e_eof);
476 1862
        return ((tbl->reset)(cursor));
477
}
478
479
enum vsl_status
480 464998
VSL_Next(const struct VSL_cursor *cursor)
481
{
482
        const struct vslc_tbl *tbl;
483
484 464998
        CAST_OBJ_NOTNULL(tbl, cursor->priv_tbl, VSLC_TBL_MAGIC);
485 464998
        AN(tbl->next);
486 464998
        return ((tbl->next)(cursor));
487
}
488
489
enum vsl_check
490 11342
VSL_Check(const struct VSL_cursor *cursor, const struct VSLC_ptr *ptr)
491
{
492
        const struct vslc_tbl *tbl;
493
494 11342
        CAST_OBJ_NOTNULL(tbl, cursor->priv_tbl, VSLC_TBL_MAGIC);
495 11342
        if (tbl->check == NULL)
496 414
                return (vsl_check_e_notsupp);
497 10928
        return ((tbl->check)(cursor, ptr));
498
}