varnish-cache/lib/libvcc/vcc_token.c
0
/*-
1
 * Copyright (c) 2006 Verdens Gang AS
2
 * Copyright (c) 2006-2011 Varnish Software AS
3
 * All rights reserved.
4
 *
5
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
6
 *
7
 * SPDX-License-Identifier: BSD-2-Clause
8
 *
9
 * Redistribution and use in source and binary forms, with or without
10
 * modification, are permitted provided that the following conditions
11
 * are met:
12
 * 1. Redistributions of source code must retain the above copyright
13
 *    notice, this list of conditions and the following disclaimer.
14
 * 2. Redistributions in binary form must reproduce the above copyright
15
 *    notice, this list of conditions and the following disclaimer in the
16
 *    documentation and/or other materials provided with the distribution.
17
 *
18
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
22
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28
 * SUCH DAMAGE.
29
 */
30
31
#include "config.h"
32
33
#include <stdlib.h>
34
#include <string.h>
35
36
#include "vcc_compile.h"
37
38
#include "venc.h"
39
#include "vct.h"
40
41
/*--------------------------------------------------------------------*/
42
43
void
44 396
vcc_ErrToken(const struct vcc *tl, const struct token *t)
45
{
46
47 396
        if (t->tok == EOI)
48 0
                VSB_cat(tl->sb, "end of input");
49 396
        else if (t->tok == CSRC)
50 0
                VSB_cat(tl->sb, "C{ ... }C");
51
        else
52 396
                VSB_printf(tl->sb, "'%.*s'", PF(t));
53 396
}
54
55
void
56 0
vcc__ErrInternal(struct vcc *tl, const char *func, unsigned line)
57
{
58
59 0
        VSB_printf(tl->sb, "VCL compiler internal error at %s():%u\n",
60 0
            func, line);
61 0
        tl->err = 1;
62 0
}
63
64
/*--------------------------------------------------------------------
65
 * Find start of source-line of token
66
 */
67
68
static void
69 7776
vcc_iline(const struct token *t, const char **ll, int tail)
70
{
71
        const char *p, *b, *x;
72
73 7776
        b = t->src->b;
74 7776
        if (ll != NULL)
75 7776
                *ll = b;
76 7776
        x = tail ? t->e - 1 : t->b;
77 3531936
        for (p = b; p < x; p++) {
78 3524160
                if (*p == '\n') {
79 97116
                        if (ll != NULL)
80 97116
                                *ll = p + 1;
81 97116
                }
82 3524160
        }
83 7776
}
84
85
/*--------------------------------------------------------------------
86
 * Find and print src+line+pos of this token
87
 */
88
89
static void
90 1154760
vcc_icoord(struct vsb *vsb, const struct token *t, int tail)
91
{
92
        unsigned lin, pos;
93
        const char *p, *b, *x;
94
95 1154760
        lin = 1;
96 1154760
        pos = 0;
97 1154760
        b = t->src->b;
98 1154760
        x = tail ? t->e - 1 : t->b;
99 4373739431
        for (p = b; p < x; p++) {
100 4372584671
                if (*p == '\n') {
101 180767100
                        lin++;
102 180767100
                        pos = 0;
103 4372584671
                } else if (*p == '\t') {
104 87706644
                        pos &= ~7;
105 87706644
                        pos += 8;
106 87706644
                } else
107 4104110927
                        pos++;
108 4372584671
        }
109 1154760
        VSB_cat(vsb, "(");
110 1154760
        if (tail < 2)
111 1153260
                VSB_printf(vsb, "'%s' Line %u ", t->src->name, lin);
112 1154760
        VSB_printf(vsb, "Pos %u)", pos + 1);
113 1154760
}
114
115
/*--------------------------------------------------------------------*/
116
117
void
118 1146984
vcc_Coord(const struct vcc *tl, struct vsb *vsb, const struct token *t)
119
{
120
121 1146984
        if (t == NULL)
122 1146984
                t = tl->t;
123 1146984
        vcc_icoord(vsb, t, 0);
124 1146984
}
125
126
/*--------------------------------------------------------------------
127
 * Output one line of source code, starting at 'l' and ending at the
128
 * first NL or 'le'.
129
 */
130
131
static void
132 6276
vcc_quoteline(const struct vcc *tl, const char *l, const char *le)
133
{
134
        const char *p;
135
        unsigned x, y;
136
137 6276
        x = y = 0;
138 213385
        for (p = l; p < le && *p != '\n'; p++) {
139 207109
                if (*p == '\t') {
140 8292
                        y &= ~7;
141 8292
                        y += 8;
142 74628
                        while (x < y) {
143 66336
                                VSB_putc(tl->sb, ' ');
144 66336
                                x++;
145
                        }
146 8292
                } else {
147 198817
                        x++;
148 198817
                        y++;
149 198817
                        VSB_putc(tl->sb, *p);
150
                }
151 207109
        }
152 6276
        VSB_putc(tl->sb, '\n');
153 6276
}
154
155
/*--------------------------------------------------------------------
156
 * Output a marker line for a sourceline starting at 'l' and ending at
157
 * the first NL or 'le'.  Characters between 'b' and 'e' are marked.
158
 */
159
160
static void
161 6276
vcc_markline(const struct vcc *tl, const char *l, const char *le,
162
    const char *b, const char *e)
163
{
164
        const char *p;
165
        unsigned x, y;
166
        char c;
167
168 6276
        x = y = 0;
169 213385
        for (p = l; p < le && *p != '\n'; p++) {
170 207109
                if (p >= b && p < e)
171 48904
                        c = '#';
172
                else
173 158205
                        c = '-';
174
175 207109
                if (*p == '\t') {
176 8292
                        y &= ~7;
177 8292
                        y += 8;
178 8292
                } else
179 198817
                        y++;
180 472262
                while (x < y) {
181 265153
                        VSB_putc(tl->sb, c);
182 265153
                        x++;
183
                }
184 207109
        }
185 6276
        VSB_putc(tl->sb, '\n');
186 6276
}
187
188
void
189 1668
vcc_Warn(struct vcc *tl)
190
{
191
192 1668
        AN(tl);
193 1668
        AN(tl->err);
194 1668
        VSB_cat(tl->sb, "(That was just a warning)\n");
195 1668
        tl->err = 0;
196 1668
}
197
198
/*--------------------------------------------------------------------*/
199
/* XXX: should take first+last token */
200
201
void
202 1524
vcc_ErrWhere2(struct vcc *tl, const struct token *t, const struct token *t2)
203
{
204
        const char  *l1, *l2, *l3;
205
206 1524
        if (t == NULL) {
207 12
                vcc_ErrWhere(tl, t2);
208 12
                return;
209
        }
210 1512
        vcc_iline(t, &l1, 0);
211 1512
        t2 = VTAILQ_PREV(t2, tokenhead, list);
212 1512
        vcc_iline(t2, &l2, 1);
213
214
215 1512
        if (l1 == l2) {
216 1500
                vcc_icoord(tl->sb, t, 0);
217 1500
                VSB_cat(tl->sb, " -- ");
218 1500
                vcc_icoord(tl->sb, t2, 2);
219 1500
                VSB_putc(tl->sb, '\n');
220
                /* Two tokens on same line */
221 1500
                vcc_quoteline(tl, l1, t->src->e);
222 1500
                vcc_markline(tl, l1, t->src->e, t->b, t2->e);
223 1500
        } else {
224
                /* Two tokens different lines */
225 12
                l3 = strchr(l1, '\n');
226 12
                AN(l3);
227
                /* XXX: t had better be before t2 */
228 12
                vcc_icoord(tl->sb, t, 0);
229 12
                if (l3 + 1 == l2) {
230 0
                        VSB_cat(tl->sb, " -- ");
231 0
                        vcc_icoord(tl->sb, t2, 1);
232 0
                }
233 12
                VSB_putc(tl->sb, '\n');
234 12
                vcc_quoteline(tl, l1, t->src->e);
235 12
                vcc_markline(tl, l1, t->src->e, t->b, t2->e);
236 12
                if (l3 + 1 != l2) {
237 12
                        VSB_cat(tl->sb, "[...]\n");
238 12
                        vcc_icoord(tl->sb, t2, 1);
239 12
                        VSB_putc(tl->sb, '\n');
240 12
                }
241 12
                vcc_quoteline(tl, l2, t->src->e);
242 12
                vcc_markline(tl, l2, t->src->e, t->b, t2->e);
243
        }
244 1512
        VSB_putc(tl->sb, '\n');
245 1512
        tl->err = 1;
246 1524
}
247
248
void
249 4752
vcc_ErrWhere(struct vcc *tl, const struct token *t)
250
{
251
        const char  *l1;
252
253 4752
        vcc_iline(t, &l1, 0);
254 4752
        vcc_icoord(tl->sb, t, 0);
255 4752
        VSB_putc(tl->sb, '\n');
256 4752
        vcc_quoteline(tl, l1, t->src->e);
257 4752
        vcc_markline(tl, l1, t->src->e, t->b, t->e);
258 4752
        VSB_putc(tl->sb, '\n');
259 4752
        tl->err = 1;
260 4752
}
261
262
/*--------------------------------------------------------------------*/
263
264
struct token *
265 37454892
vcc_PeekTokenFrom(const struct vcc *tl, const struct token *t)
266
{
267
        struct token *t2;
268
269 37454892
        CHECK_OBJ_NOTNULL(tl, VCC_MAGIC);
270 37454892
        AN(t);
271 37454892
        assert(t->tok != EOI);
272 37454892
        t2 = VTAILQ_NEXT(t, list);
273 37454892
        AN(t2);
274 37454892
        return (t2);
275
}
276
277
struct token *
278 2712096
vcc_PeekToken(const struct vcc *tl)
279
{
280
281 2712096
        CHECK_OBJ_NOTNULL(tl, VCC_MAGIC);
282 2712096
        return (vcc_PeekTokenFrom(tl, tl->t));
283
}
284
285
void
286 14273364
vcc_NextToken(struct vcc *tl)
287
{
288
289 14273364
        CHECK_OBJ_NOTNULL(tl, VCC_MAGIC);
290 14273364
        tl->t = vcc_PeekTokenFrom(tl, tl->t);
291 14273364
}
292
293
void
294 9989580
vcc__Expect(struct vcc *tl, unsigned tok, unsigned line)
295
{
296 9989580
        if (tl->t->tok == tok)
297 9989412
                return;
298 168
        VSB_printf(tl->sb, "Expected %s got ", vcl_tnames[tok]);
299 168
        vcc_ErrToken(tl, tl->t);
300 168
        VSB_printf(tl->sb, "\n(program line %u), at\n", line);
301 168
        vcc_ErrWhere(tl, tl->t);
302 9989580
}
303
304
/*--------------------------------------------------------------------
305
 * Compare ID token to string, return true of match
306
 */
307
308
int
309 25505532
vcc_IdIs(const struct token *t, const char *p)
310
{
311
        const char *q;
312
313 25505532
        assert(t->tok == ID);
314 34709244
        for (q = t->b; q < t->e && *p != '\0'; p++, q++)
315 32583324
                if (*q != *p)
316 23379612
                        return (0);
317 2125920
        if (q != t->e || *p != '\0')
318 2460
                return (0);
319 2123460
        return (1);
320 25505532
}
321
322
/*--------------------------------------------------------------------
323
 * Check that we have a Varnish identifier
324
 */
325
326
void
327 384
vcc_PrintTokens(const struct vcc *tl,
328
    const struct token *tb, const struct token *te)
329
{
330
331 384
        CHECK_OBJ_NOTNULL(tl, VCC_MAGIC);
332 384
        AN(tb);
333 384
        AN(te);
334 1296
        while (tb != te) {
335 912
                VSB_printf(tl->sb, "%.*s", PF(tb));
336 912
                tb = vcc_PeekTokenFrom(tl, tb);
337 912
                AN(tb);
338
        }
339 384
}
340
341
void
342 1169316
vcc_ExpectVid(struct vcc *tl, const char *what)
343
{
344 1169316
        const char *bad = NULL;
345
        struct token *t2;
346
347 1169316
        ExpectErr(tl, ID);
348 1169304
        ERRCHK(tl);
349
350 1169304
        t2 = vcc_PeekToken(tl);
351 1169304
        AN(t2);
352 1169340
        while (t2->tok == '.') {
353 36
                bad = ".";
354 36
                t2 = vcc_PeekTokenFrom(tl, t2);
355 36
                AN(t2);
356 36
                if (t2->tok != ID)
357 0
                        break;
358 36
                t2 = vcc_PeekTokenFrom(tl, t2);
359 36
                AN(t2);
360
        }
361 1169304
        if (bad == NULL)
362 1169268
                bad = VCT_invalid_name(tl->t->b, tl->t->e);
363 1169304
        if (bad != NULL) {
364 36
                VSB_printf(tl->sb, "Name of %s, '", what);
365 36
                vcc_PrintTokens(tl, tl->t, t2);
366 72
                VSB_printf(tl->sb,
367 36
                    "', contains illegal character '%c'\n", *bad);
368 36
                vcc_ErrWhere2(tl, tl->t, t2);
369 36
                return;
370
        }
371 1169316
}
372
373
/*--------------------------------------------------------------------
374
 * Decode a string
375
 */
376
377
static void
378 1304604
vcc_decstr(struct vcc *tl, unsigned sep)
379
{
380
        char *q;
381
        unsigned int l;
382
383 1304604
        assert(tl->t->tok == CSTR);
384 1304604
        l = pdiff(tl->t->b + sep, tl->t->e - sep);
385 1304604
        tl->t->dec = TlAlloc(tl, l + 1);
386 1304604
        AN(tl->t->dec);
387 1304604
        q = tl->t->dec;
388 1304604
        memcpy(q, tl->t->b + sep, l);
389 1304604
        q[l] = '\0';
390 1304604
}
391
392
/*--------------------------------------------------------------------
393
 * Add a token to the token list.
394
 */
395
396
static void
397 26970396
vcc_addtoken(struct vcc *tl, unsigned tok,
398
    struct source *sp, const char *b, const char *e)
399
{
400
        struct token *t;
401
402 26970396
        t = TlAlloc(tl, sizeof *t);
403 26970396
        assert(t != NULL);
404 26970396
        t->tok = tok;
405 26970396
        t->b = b;
406 26970396
        t->e = e;
407 26970396
        t->src = sp;
408 26970396
        VTAILQ_INSERT_TAIL(&sp->src_tokens, t, src_list);
409 26970396
        tl->t = t;
410 26970396
}
411
412
/*--------------------------------------------------------------------
413
 * Find a delimited token
414
 */
415
416
static const struct delim_def {
417
        const char      *name;
418
        const char      *b;
419
        const char      *e;
420
        unsigned        len;    /* NB: must be the same for both delimiters */
421
        unsigned        crlf;
422
        unsigned        tok;
423
} delim_defs[] = {
424
#define DELIM_DEF(nm, l, r, c, t)               \
425
        { nm, l, r, sizeof (l) - 1, c, t }
426
        DELIM_DEF("long-string", "\"\"\"", "\"\"\"", 1, CSTR),  /* """...""" */
427
        DELIM_DEF("long-string", "{\"", "\"}", 1, CSTR),        /*  {"..."}  */
428
        DELIM_DEF("string", "\"", "\"", 0, CSTR),               /*   "..."   */
429
        DELIM_DEF("inline C source", "C{", "}C", 1, CSRC),      /*  C{...}C  */
430
#undef DELIM_DEF
431
        { NULL }
432
};
433
434
static unsigned
435 26917920
vcc_delim_token(struct vcc *tl, struct source *sp, const char *p,
436
    const char **qp)
437
{
438
        const struct delim_def *dd;
439
        const char *q, *r;
440
441 131633136
        for (dd = delim_defs; dd->name != NULL; dd++)
442 106020156
                if (!strncmp(p, dd->b, dd->len))
443 1304940
                        break;
444
445 26917920
        if (dd->name == NULL)
446 25612980
                return (0);
447
448 1304940
        q = strstr(p + dd->len, dd->e);
449 1304940
        if (q != NULL && !dd->crlf) {
450 957804
                r = strpbrk(p + dd->len, "\r\n");
451 957804
                if (r != NULL && r < q)
452 0
                        q = NULL;
453 957804
        }
454
455 1304940
        if (q == NULL) {
456 48
                vcc_addtoken(tl, EOI, sp, p, p + dd->len);
457 48
                VSB_printf(tl->sb, "Unterminated %s, starting at\n", dd->name);
458 48
                vcc_ErrWhere(tl, tl->t);
459 48
                return (0);
460
        }
461
462 1304892
        assert(q < sp->e);
463 1304892
        vcc_addtoken(tl, dd->tok, sp, p, q + dd->len);
464 1304892
        if (dd->tok == CSTR)
465 1304604
                vcc_decstr(tl, dd->len);
466 1304892
        *qp = q + dd->len;
467 1304892
        return (1);
468 26917920
}
469
470
/*--------------------------------------------------------------------
471
 * Lex a number, either CNUM or FNUM.
472
 * We enforce the RFC8941 restrictions on number of digits here.
473
 */
474
475
static const char *
476 277020
vcc_lex_number(struct vcc *tl, struct source *sp, const char *p)
477
{
478
        const char *q, *r;
479
        char *s;
480
481 848112
        for (q = p; q < sp->e; q++)
482 848112
                if (!vct_isdigit(*q))
483 277020
                        break;
484 277020
        if (*q != '.') {
485 223296
                vcc_addtoken(tl, CNUM, sp, p, q);
486 223296
                if (q - p > 15) {
487 12
                        VSB_cat(tl->sb, "Too many digits for integer.\n");
488 12
                        vcc_ErrWhere(tl, tl->t);
489 12
                        return (NULL);
490
                }
491 223284
                tl->t->num = strtod(p, &s);
492 223284
                assert(s == tl->t->e);
493 223284
                return (q);
494
        }
495 53724
        r = ++q;
496 107772
        for (; r < sp->e; r++)
497 107772
                if (!vct_isdigit(*r))
498 53724
                        break;
499 53724
        vcc_addtoken(tl, FNUM, sp, p, r);
500 53724
        if (q - p > 13 || r - q > 3) {
501 36
                VSB_cat(tl->sb, "Too many digits for real.\n");
502 36
                vcc_ErrWhere(tl, tl->t);
503 36
                return (NULL);
504
        }
505 53688
        if (*tl->t->e == 'e' || *tl->t->e == 'E') {
506 12
                VSB_printf(tl->sb, "Unexpected character '%c'.\n", *tl->t->e);
507 12
                tl->t->e++;
508 12
                vcc_ErrWhere(tl, tl->t);
509 12
                return (NULL);
510
        }
511 53676
        tl->t->num = strtod(p, &s);
512 53676
        assert(s == tl->t->e);
513 53676
        return (r);
514 277020
}
515
516
/*--------------------------------------------------------------------
517
 * Lexical analysis and token generation
518
 */
519
520
void
521 52560
vcc_Lexer(struct vcc *tl, struct source *sp)
522
{
523
        const char *p, *q, *r;
524
        unsigned u;
525
        struct vsb *vsb;
526
        char namebuf[40];
527
528 54361464
        for (p = sp->b; p < sp->e; ) {
529
530
                /* Skip any whitespace */
531 54309180
                if (vct_isspace(*p)) {
532 25622292
                        p++;
533 25622292
                        continue;
534
                }
535
536
                /* Skip '#.*\n' comments */
537 28686888
                if (*p == '#') {
538 80109876
                        while (p < sp->e && *p != '\n')
539 78341280
                                p++;
540 1768596
                        continue;
541
                }
542
543
                /* Skip C-style comments */
544 26918292
                if (*p == '/' && p[1] == '*') {
545 792
                        for (q = p + 2; q < sp->e; q++) {
546 780
                                if (*q == '/' && q[1] == '*') {
547 12
                                        VSB_cat(tl->sb,
548
                                            "/* ... */ comment contains /*\n");
549 12
                                        vcc_addtoken(tl, EOI, sp, p, p + 2);
550 12
                                        vcc_ErrWhere(tl, tl->t);
551 12
                                        vcc_addtoken(tl, EOI, sp, q, q + 2);
552 12
                                        vcc_ErrWhere(tl, tl->t);
553 12
                                        return;
554
                                }
555 768
                                if (*q == '*' && q[1] == '/') {
556 24
                                        p = q + 2;
557 24
                                        break;
558
                                }
559 744
                        }
560 36
                        if (q < sp->e)
561 24
                                continue;
562 12
                        vcc_addtoken(tl, EOI, sp, p, p + 2);
563 12
                        VSB_cat(tl->sb,
564
                            "Unterminated /* ... */ comment, starting at\n");
565 12
                        vcc_ErrWhere(tl, tl->t);
566 12
                        return;
567
                }
568
569
                /* Skip C++-style comments */
570 26918244
                if (*p == '/' && p[1] == '/') {
571 4104
                        while (p < sp->e && *p != '\n')
572 3936
                                p++;
573 168
                        continue;
574
                }
575
576
                /* Recognize BLOB (= SF-binary) */
577 26918076
                if (*p == ':') {
578 156
                        vsb = VSB_new_auto();
579 156
                        AN(vsb);
580 156
                        q = sp->e;
581 156
                        q -= (q - (p + 1)) % 4;
582 156
                        assert(q > p);
583 156
                        r = VENC_Decode_Base64(vsb, p + 1, q);
584 156
                        if (r == NULL) {
585 24
                                vcc_addtoken(tl, CBLOB, sp, p, q + 1);
586 24
                                VSB_cat(tl->sb,
587
                                    "Missing colon at end of BLOB:\n");
588 24
                                vcc_ErrWhere(tl, tl->t);
589 24
                                VSB_destroy(&vsb);
590 24
                                return;
591
                        }
592 132
                        vcc_addtoken(tl, CBLOB, sp, p, r + 1);
593 132
                        if (*r == ':' && ((r - p) % 4) != 1) {
594 36
                                VSB_cat(tl->sb,
595
                                    "BLOB must have n*4 base64 characters\n");
596 36
                                vcc_ErrWhere(tl, tl->t);
597 36
                                VSB_destroy(&vsb);
598 36
                                return;
599
                        }
600 96
                        if (*r == '=') {
601 36
                                VSB_cat(tl->sb,
602
                                    "Wrong padding ('=') in BLOB:\n");
603 36
                                vcc_ErrWhere(tl, tl->t);
604 36
                                VSB_destroy(&vsb);
605 36
                                return;
606
                        }
607 60
                        if (*r != ':') {
608 24
                                VSB_cat(tl->sb, "Illegal BLOB character:\n");
609 24
                                vcc_ErrWhere(tl, tl->t);
610 24
                                VSB_destroy(&vsb);
611 24
                                return;
612
                        }
613 36
                        r++;
614 36
                        AZ(VSB_finish(vsb));
615
616 36
                        bprintf(namebuf, "blob_%u", tl->unique++);
617 72
                        Fh(tl, 0,
618
                            "\nstatic const unsigned char %s_data[%zd] = {\n",
619 36
                            namebuf, VSB_len(vsb));
620 984
                        for (u = 0; u < VSB_len(vsb); u++) {
621 948
                                Fh(tl, 0, "\t0x%02x,", VSB_data(vsb)[u] & 0xff);
622 948
                                if ((u & 7) == 7)
623 108
                                        Fh(tl, 0, "\n");
624 948
                        }
625 36
                        if ((u & 7) != 7)
626 36
                                Fh(tl, 0, "\n");
627 36
                        Fh(tl, 0, "};\n");
628 72
                        Fh(tl, 0,
629
                            "\nstatic const struct vrt_blob %s[1] = {{\n",
630 36
                            namebuf);
631 36
                        Fh(tl, 0, "\t.len =\t%zd,\n", VSB_len(vsb));
632 36
                        Fh(tl, 0, "\t.blob =\t%s_data,\n", namebuf);
633 36
                        Fh(tl, 0, "}};\n");
634 36
                        REPLACE(tl->t->dec, namebuf);
635 36
                        VSB_destroy(&vsb);
636 36
                        p = r;
637 36
                        continue;
638
                }
639
640
                /* Match delimited tokens */
641 26917920
                if (vcc_delim_token(tl, sp, p, &q) != 0) {
642 1304892
                        p = q;
643 1304892
                        continue;
644
                }
645 25613028
                ERRCHK(tl);
646
647
                /* Match for the fixed tokens (see generate.py) */
648 25612980
                u = vcl_fixed_token(p, &q);
649 25612980
                if (u != 0) {
650 13925820
                        vcc_addtoken(tl, u, sp, p, q);
651 13925820
                        p = q;
652 13925820
                        continue;
653
                }
654
655
                /* Match Identifiers */
656 11687160
                if (vct_isident1(*p)) {
657 87482196
                        for (q = p; q < sp->e; q++)
658 87482196
                                if (!vct_isident(*q))
659 11410116
                                        break;
660 11410116
                        vcc_addtoken(tl, ID, sp, p, q);
661 11410116
                        p = q;
662 11410116
                        continue;
663
                }
664
665
                /* Match numbers { [0-9]+ } */
666 277044
                if (vct_isdigit(*p)) {
667 277020
                        p = vcc_lex_number(tl, sp, p);
668 277020
                        if (p == NULL)
669 60
                                return;
670 276960
                        continue;
671
                }
672 24
                vcc_addtoken(tl, EOI, sp, p, p + 1);
673 24
                VSB_cat(tl->sb, "Syntax error at\n");
674 24
                vcc_ErrWhere(tl, tl->t);
675 24
                return;
676
        }
677 52284
        vcc_addtoken(tl, EOI, sp, sp->e, sp->e);
678 52560
}