varnish-cache/lib/libvcc/vcc_token.c
0
/*-
1
 * Copyright (c) 2006 Verdens Gang AS
2
 * Copyright (c) 2006-2011 Varnish Software AS
3
 * All rights reserved.
4
 *
5
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
6
 *
7
 * SPDX-License-Identifier: BSD-2-Clause
8
 *
9
 * Redistribution and use in source and binary forms, with or without
10
 * modification, are permitted provided that the following conditions
11
 * are met:
12
 * 1. Redistributions of source code must retain the above copyright
13
 *    notice, this list of conditions and the following disclaimer.
14
 * 2. Redistributions in binary form must reproduce the above copyright
15
 *    notice, this list of conditions and the following disclaimer in the
16
 *    documentation and/or other materials provided with the distribution.
17
 *
18
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
22
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28
 * SUCH DAMAGE.
29
 */
30
31
#include "config.h"
32
33
#include <stdlib.h>
34
#include <string.h>
35
36
#include "vcc_compile.h"
37
38
#include "venc.h"
39
#include "vct.h"
40
41
/*--------------------------------------------------------------------*/
42
43
void
44 66
vcc_ErrToken(const struct vcc *tl, const struct token *t)
45
{
46
47 66
        if (t->tok == EOI)
48 0
                VSB_cat(tl->sb, "end of input");
49 66
        else if (t->tok == CSRC)
50 0
                VSB_cat(tl->sb, "C{ ... }C");
51
        else
52 66
                VSB_printf(tl->sb, "'%.*s'", PF(t));
53 66
}
54
55
void
56 0
vcc__ErrInternal(struct vcc *tl, const char *func, unsigned line)
57
{
58
59 0
        VSB_printf(tl->sb, "VCL compiler internal error at %s():%u\n",
60 0
            func, line);
61 0
        tl->err = 1;
62 0
}
63
64
/*--------------------------------------------------------------------
65
 * Find start of source-line of token
66
 */
67
68
static void
69 1372
vcc_iline(const struct token *t, const char **ll, int tail)
70
{
71
        const char *p, *b, *x;
72
73 1372
        b = t->src->b;
74 1372
        if (ll != NULL)
75 1372
                *ll = b;
76 1372
        x = tail ? t->e - 1 : t->b;
77 601470
        for (p = b; p < x; p++) {
78 600098
                if (*p == '\n') {
79 16894
                        if (ll != NULL)
80 16894
                                *ll = p + 1;
81 16894
                }
82 600098
        }
83 1372
}
84
85
/*--------------------------------------------------------------------
86
 * Find and print src+line+pos of this token
87
 */
88
89
static void
90 238084
vcc_icoord(struct vsb *vsb, const struct token *t, int tail)
91
{
92
        unsigned lin, pos;
93
        const char *p, *b, *x;
94
95 238084
        lin = 1;
96 238084
        pos = 0;
97 238084
        b = t->src->b;
98 238084
        x = tail ? t->e - 1 : t->b;
99 988002844
        for (p = b; p < x; p++) {
100 987764760
                if (*p == '\n') {
101 41453688
                        lin++;
102 41453688
                        pos = 0;
103 987764760
                } else if (*p == '\t') {
104 19665182
                        pos &= ~7;
105 19665182
                        pos += 8;
106 19665182
                } else
107 926645890
                        pos++;
108 987764760
        }
109 238084
        VSB_cat(vsb, "(");
110 238084
        if (tail < 2)
111 237834
                VSB_printf(vsb, "'%s' Line %u ", t->src->name, lin);
112 238084
        VSB_printf(vsb, "Pos %u)", pos + 1);
113 238084
}
114
115
/*--------------------------------------------------------------------*/
116
117
void
118 236712
vcc_Coord(const struct vcc *tl, struct vsb *vsb, const struct token *t)
119
{
120
121 236712
        if (t == NULL)
122 236712
                t = tl->t;
123 236712
        vcc_icoord(vsb, t, 0);
124 236712
}
125
126
/*--------------------------------------------------------------------
127
 * Output one line of source code, starting at 'l' and ending at the
128
 * first NL or 'le'.
129
 */
130
131
static void
132 1122
vcc_quoteline(const struct vcc *tl, const char *l, const char *le)
133
{
134
        const char *p;
135
        unsigned x, y;
136
137 1122
        x = y = 0;
138 36977
        for (p = l; p < le && *p != '\n'; p++) {
139 35855
                if (*p == '\t') {
140 1524
                        y &= ~7;
141 1524
                        y += 8;
142 13716
                        while (x < y) {
143 12192
                                VSB_putc(tl->sb, ' ');
144 12192
                                x++;
145
                        }
146 1524
                } else {
147 34331
                        x++;
148 34331
                        y++;
149 34331
                        VSB_putc(tl->sb, *p);
150
                }
151 35855
        }
152 1122
        VSB_putc(tl->sb, '\n');
153 1122
}
154
155
/*--------------------------------------------------------------------
156
 * Output a marker line for a sourceline starting at 'l' and ending at
157
 * the first NL or 'le'.  Characters between 'b' and 'e' are marked.
158
 */
159
160
static void
161 1122
vcc_markline(const struct vcc *tl, const char *l, const char *le,
162
    const char *b, const char *e)
163
{
164
        const char *p;
165
        unsigned x, y;
166
        char c;
167
168 1122
        x = y = 0;
169 36977
        for (p = l; p < le && *p != '\n'; p++) {
170 35855
                if (p >= b && p < e)
171 8915
                        c = '#';
172
                else
173 26940
                        c = '-';
174
175 35855
                if (*p == '\t') {
176 1524
                        y &= ~7;
177 1524
                        y += 8;
178 1524
                } else
179 34331
                        y++;
180 82378
                while (x < y) {
181 46523
                        VSB_putc(tl->sb, c);
182 46523
                        x++;
183
                }
184 35855
        }
185 1122
        VSB_putc(tl->sb, '\n');
186 1122
}
187
188
void
189 310
vcc_Warn(struct vcc *tl)
190
{
191
192 310
        AN(tl);
193 310
        AN(tl->err);
194 310
        VSB_cat(tl->sb, "(That was just a warning)\n");
195 310
        tl->err = 0;
196 310
}
197
198
/*--------------------------------------------------------------------*/
199
/* XXX: should take first+last token */
200
201
void
202 254
vcc_ErrWhere2(struct vcc *tl, const struct token *t, const struct token *t2)
203
{
204
        const char  *l1, *l2, *l3;
205
206 254
        if (t == NULL) {
207 2
                vcc_ErrWhere(tl, t2);
208 2
                return;
209
        }
210 252
        vcc_iline(t, &l1, 0);
211 252
        t2 = VTAILQ_PREV(t2, tokenhead, list);
212 252
        vcc_iline(t2, &l2, 1);
213
214
215 252
        if (l1 == l2) {
216 250
                vcc_icoord(tl->sb, t, 0);
217 250
                VSB_cat(tl->sb, " -- ");
218 250
                vcc_icoord(tl->sb, t2, 2);
219 250
                VSB_putc(tl->sb, '\n');
220
                /* Two tokens on same line */
221 250
                vcc_quoteline(tl, l1, t->src->e);
222 250
                vcc_markline(tl, l1, t->src->e, t->b, t2->e);
223 250
        } else {
224
                /* Two tokens different lines */
225 2
                l3 = strchr(l1, '\n');
226 2
                AN(l3);
227
                /* XXX: t had better be before t2 */
228 2
                vcc_icoord(tl->sb, t, 0);
229 2
                if (l3 + 1 == l2) {
230 0
                        VSB_cat(tl->sb, " -- ");
231 0
                        vcc_icoord(tl->sb, t2, 1);
232 0
                }
233 2
                VSB_putc(tl->sb, '\n');
234 2
                vcc_quoteline(tl, l1, t->src->e);
235 2
                vcc_markline(tl, l1, t->src->e, t->b, t2->e);
236 2
                if (l3 + 1 != l2) {
237 2
                        VSB_cat(tl->sb, "[...]\n");
238 2
                        vcc_icoord(tl->sb, t2, 1);
239 2
                        VSB_putc(tl->sb, '\n');
240 2
                }
241 2
                vcc_quoteline(tl, l2, t->src->e);
242 2
                vcc_markline(tl, l2, t->src->e, t->b, t2->e);
243
        }
244 252
        VSB_putc(tl->sb, '\n');
245 252
        tl->err = 1;
246 254
}
247
248
void
249 868
vcc_ErrWhere(struct vcc *tl, const struct token *t)
250
{
251
        const char  *l1;
252
253 868
        vcc_iline(t, &l1, 0);
254 868
        vcc_icoord(tl->sb, t, 0);
255 868
        VSB_putc(tl->sb, '\n');
256 868
        vcc_quoteline(tl, l1, t->src->e);
257 868
        vcc_markline(tl, l1, t->src->e, t->b, t->e);
258 868
        VSB_putc(tl->sb, '\n');
259 868
        tl->err = 1;
260 868
}
261
262
/*--------------------------------------------------------------------*/
263
264
struct token *
265 7690932
vcc_PeekTokenFrom(const struct vcc *tl, const struct token *t)
266
{
267
        struct token *t2;
268
269 7690932
        CHECK_OBJ_NOTNULL(tl, VCC_MAGIC);
270 7690932
        AN(t);
271 7690932
        assert(t->tok != EOI);
272 7690932
        t2 = VTAILQ_NEXT(t, list);
273 7690932
        AN(t2);
274 7690932
        return (t2);
275
}
276
277
struct token *
278 546976
vcc_PeekToken(const struct vcc *tl)
279
{
280
281 546976
        CHECK_OBJ_NOTNULL(tl, VCC_MAGIC);
282 546976
        return (vcc_PeekTokenFrom(tl, tl->t));
283
}
284
285
void
286 2935168
vcc_NextToken(struct vcc *tl)
287
{
288
289 2935168
        CHECK_OBJ_NOTNULL(tl, VCC_MAGIC);
290 2935168
        tl->t = vcc_PeekTokenFrom(tl, tl->t);
291 2935168
}
292
293
void
294 2064704
vcc__Expect(struct vcc *tl, unsigned tok, unsigned line)
295
{
296 2064704
        if (tl->t->tok == tok)
297 2064676
                return;
298 28
        VSB_printf(tl->sb, "Expected %s got ", vcl_tnames[tok]);
299 28
        vcc_ErrToken(tl, tl->t);
300 28
        VSB_printf(tl->sb, "\n(program line %u), at\n", line);
301 28
        vcc_ErrWhere(tl, tl->t);
302 2064704
}
303
304
/*--------------------------------------------------------------------
305
 * Compare ID token to string, return true of match
306
 */
307
308
int
309 5548924
vcc_IdIs(const struct token *t, const char *p)
310
{
311
        const char *q;
312
313 5548924
        assert(t->tok == ID);
314 7403656
        for (q = t->b; q < t->e && *p != '\0'; p++, q++)
315 6977180
                if (*q != *p)
316 5122448
                        return (0);
317 426476
        if (q != t->e || *p != '\0')
318 420
                return (0);
319 426056
        return (1);
320 5548924
}
321
322
/*--------------------------------------------------------------------
323
 * Check that we have a Varnish identifier
324
 */
325
326
void
327 64
vcc_PrintTokens(const struct vcc *tl,
328
    const struct token *tb, const struct token *te)
329
{
330
331 64
        CHECK_OBJ_NOTNULL(tl, VCC_MAGIC);
332 64
        AN(tb);
333 64
        AN(te);
334 216
        while (tb != te) {
335 152
                VSB_printf(tl->sb, "%.*s", PF(tb));
336 152
                tb = vcc_PeekTokenFrom(tl, tb);
337 152
                AN(tb);
338
        }
339 64
}
340
341
void
342 240616
vcc_ExpectVid(struct vcc *tl, const char *what)
343
{
344 240616
        const char *bad = NULL;
345
        struct token *t2;
346
347 240616
        ExpectErr(tl, ID);
348 240614
        ERRCHK(tl);
349
350 240614
        t2 = vcc_PeekToken(tl);
351 240614
        AN(t2);
352 240620
        while (t2->tok == '.') {
353 6
                bad = ".";
354 6
                t2 = vcc_PeekTokenFrom(tl, t2);
355 6
                AN(t2);
356 6
                if (t2->tok != ID)
357 0
                        break;
358 6
                t2 = vcc_PeekTokenFrom(tl, t2);
359 6
                AN(t2);
360
        }
361 240614
        if (bad == NULL)
362 240608
                bad = VCT_invalid_name(tl->t->b, tl->t->e);
363 240614
        if (bad != NULL) {
364 6
                VSB_printf(tl->sb, "Name of %s, '", what);
365 6
                vcc_PrintTokens(tl, tl->t, t2);
366 12
                VSB_printf(tl->sb,
367 6
                    "', contains illegal character '%c'\n", *bad);
368 6
                vcc_ErrWhere2(tl, tl->t, t2);
369 6
                return;
370
        }
371 240616
}
372
373
/*--------------------------------------------------------------------
374
 * Decode a string
375
 */
376
377
static void
378 247930
vcc_decstr(struct vcc *tl, unsigned sep)
379
{
380
        char *q;
381
        unsigned int l;
382
383 247930
        assert(tl->t->tok == CSTR);
384 247930
        l = pdiff(tl->t->b + sep, tl->t->e - sep);
385 247930
        tl->t->dec = TlAlloc(tl, l + 1);
386 247930
        AN(tl->t->dec);
387 247930
        q = tl->t->dec;
388 247930
        memcpy(q, tl->t->b + sep, l);
389 247930
        q[l] = '\0';
390 247930
}
391
392
/*--------------------------------------------------------------------
393
 * Add a token to the token list.
394
 */
395
396
static void
397 5534576
vcc_addtoken(struct vcc *tl, unsigned tok,
398
    struct source *sp, const char *b, const char *e)
399
{
400
        struct token *t;
401
402 5534576
        t = TlAlloc(tl, sizeof *t);
403 5534576
        assert(t != NULL);
404 5534576
        t->tok = tok;
405 5534576
        t->b = b;
406 5534576
        t->e = e;
407 5534576
        t->src = sp;
408 5534576
        VTAILQ_INSERT_TAIL(&sp->src_tokens, t, src_list);
409 5534576
        tl->t = t;
410 5534576
}
411
412
/*--------------------------------------------------------------------
413
 * Find a delimited token
414
 */
415
416
static const struct delim_def {
417
        const char      *name;
418
        const char      *b;
419
        const char      *e;
420
        unsigned        len;    /* NB: must be the same for both delimiters */
421
        unsigned        crlf;
422
        unsigned        tok;
423
} delim_defs[] = {
424
#define DELIM_DEF(nm, l, r, c, t)               \
425
        { nm, l, r, sizeof (l) - 1, c, t }
426
        DELIM_DEF("long-string", "\"\"\"", "\"\"\"", 1, CSTR),  /* """...""" */
427
        DELIM_DEF("long-string", "{\"", "\"}", 1, CSTR),        /*  {"..."}  */
428
        DELIM_DEF("string", "\"", "\"", 0, CSTR),               /*   "..."   */
429
        DELIM_DEF("inline C source", "C{", "}C", 1, CSRC),      /*  C{...}C  */
430
#undef DELIM_DEF
431
        { NULL }
432
};
433
434
static unsigned
435 5525340
vcc_delim_token(struct vcc *tl, struct source *sp, const char *p,
436
    const char **qp)
437
{
438
        const struct delim_def *dd;
439
        const char *q, *r;
440
441 27069704
        for (dd = delim_defs; dd->name != NULL; dd++)
442 21792350
                if (!strncmp(p, dd->b, dd->len))
443 247986
                        break;
444
445 5525340
        if (dd->name == NULL)
446 5277354
                return (0);
447
448 247986
        q = strstr(p + dd->len, dd->e);
449 247986
        if (q != NULL && !dd->crlf) {
450 186870
                r = strpbrk(p + dd->len, "\r\n");
451 186870
                if (r != NULL && r < q)
452 0
                        q = NULL;
453 186870
        }
454
455 247986
        if (q == NULL) {
456 8
                vcc_addtoken(tl, EOI, sp, p, p + dd->len);
457 8
                VSB_printf(tl->sb, "Unterminated %s, starting at\n", dd->name);
458 8
                vcc_ErrWhere(tl, tl->t);
459 8
                return (0);
460
        }
461
462 247978
        assert(q < sp->e);
463 247978
        vcc_addtoken(tl, dd->tok, sp, p, q + dd->len);
464 247978
        if (dd->tok == CSTR)
465 247930
                vcc_decstr(tl, dd->len);
466 247978
        *qp = q + dd->len;
467 247978
        return (1);
468 5525340
}
469
470
/*--------------------------------------------------------------------
471
 * Lex a number, either CNUM or FNUM.
472
 * We enforce the RFC8941 restrictions on number of digits here.
473
 */
474
475
static const char *
476 79322
vcc_lex_number(struct vcc *tl, struct source *sp, const char *p)
477
{
478
        const char *q, *r;
479
        char *s;
480
481 271390
        for (q = p; q < sp->e; q++)
482 271390
                if (!vct_isdigit(*q))
483 79322
                        break;
484 79322
        if (*q != '.') {
485 69864
                vcc_addtoken(tl, CNUM, sp, p, q);
486 69864
                if (q - p > 15) {
487 2
                        VSB_cat(tl->sb, "Too many digits for integer.\n");
488 2
                        vcc_ErrWhere(tl, tl->t);
489 2
                        return (NULL);
490
                }
491 69862
                tl->t->num = strtod(p, &s);
492 69862
                assert(s == tl->t->e);
493 69862
                return (q);
494
        }
495 9458
        r = ++q;
496 18986
        for (; r < sp->e; r++)
497 18986
                if (!vct_isdigit(*r))
498 9458
                        break;
499 9458
        vcc_addtoken(tl, FNUM, sp, p, r);
500 9458
        if (q - p > 13 || r - q > 3) {
501 6
                VSB_cat(tl->sb, "Too many digits for real.\n");
502 6
                vcc_ErrWhere(tl, tl->t);
503 6
                return (NULL);
504
        }
505 9452
        if (*tl->t->e == 'e' || *tl->t->e == 'E') {
506 2
                VSB_printf(tl->sb, "Unexpected character '%c'.\n", *tl->t->e);
507 2
                tl->t->e++;
508 2
                vcc_ErrWhere(tl, tl->t);
509 2
                return (NULL);
510
        }
511 9450
        tl->t->num = strtod(p, &s);
512 9450
        assert(s == tl->t->e);
513 9450
        return (r);
514 79322
}
515
516
/*--------------------------------------------------------------------
517
 * Lexical analysis and token generation
518
 */
519
520
void
521 9250
vcc_Lexer(struct vcc *tl, struct source *sp)
522
{
523
        const char *p, *q, *r;
524
        unsigned u;
525
        struct vsb *vsb;
526
        char namebuf[40];
527
528 11027890
        for (p = sp->b; p < sp->e; ) {
529
530
                /* Skip any whitespace */
531 11018686
                if (vct_isspace(*p)) {
532 5157484
                        p++;
533 5157484
                        continue;
534
                }
535
536
                /* Skip '#.*\n' comments */
537 5861202
                if (*p == '#') {
538 15447056
                        while (p < sp->e && *p != '\n')
539 15111258
                                p++;
540 335798
                        continue;
541
                }
542
543
                /* Skip C-style comments */
544 5525404
                if (*p == '/' && p[1] == '*') {
545 132
                        for (q = p + 2; q < sp->e; q++) {
546 130
                                if (*q == '/' && q[1] == '*') {
547 2
                                        VSB_cat(tl->sb,
548
                                            "/* ... */ comment contains /*\n");
549 2
                                        vcc_addtoken(tl, EOI, sp, p, p + 2);
550 2
                                        vcc_ErrWhere(tl, tl->t);
551 2
                                        vcc_addtoken(tl, EOI, sp, q, q + 2);
552 2
                                        vcc_ErrWhere(tl, tl->t);
553 2
                                        return;
554
                                }
555 128
                                if (*q == '*' && q[1] == '/') {
556 4
                                        p = q + 2;
557 4
                                        break;
558
                                }
559 124
                        }
560 6
                        if (q < sp->e)
561 4
                                continue;
562 2
                        vcc_addtoken(tl, EOI, sp, p, p + 2);
563 2
                        VSB_cat(tl->sb,
564
                            "Unterminated /* ... */ comment, starting at\n");
565 2
                        vcc_ErrWhere(tl, tl->t);
566 2
                        return;
567
                }
568
569
                /* Skip C++-style comments */
570 5525396
                if (*p == '/' && p[1] == '/') {
571 754
                        while (p < sp->e && *p != '\n')
572 724
                                p++;
573 30
                        continue;
574
                }
575
576
                /* Recognize BLOB (= SF-binary) */
577 5525366
                if (*p == ':') {
578 26
                        vsb = VSB_new_auto();
579 26
                        AN(vsb);
580 26
                        q = sp->e;
581 26
                        q -= (q - (p + 1)) % 4;
582 26
                        assert(q > p);
583 26
                        r = VENC_Decode_Base64(vsb, p + 1, q);
584 26
                        if (r == NULL) {
585 4
                                vcc_addtoken(tl, CBLOB, sp, p, q + 1);
586 4
                                VSB_cat(tl->sb,
587
                                    "Missing colon at end of BLOB:\n");
588 4
                                vcc_ErrWhere(tl, tl->t);
589 4
                                VSB_destroy(&vsb);
590 4
                                return;
591
                        }
592 22
                        vcc_addtoken(tl, CBLOB, sp, p, r + 1);
593 22
                        if (*r == ':' && ((r - p) % 4) != 1) {
594 6
                                VSB_cat(tl->sb,
595
                                    "BLOB must have n*4 base64 characters\n");
596 6
                                vcc_ErrWhere(tl, tl->t);
597 6
                                VSB_destroy(&vsb);
598 6
                                return;
599
                        }
600 16
                        if (*r == '=') {
601 6
                                VSB_cat(tl->sb,
602
                                    "Wrong padding ('=') in BLOB:\n");
603 6
                                vcc_ErrWhere(tl, tl->t);
604 6
                                VSB_destroy(&vsb);
605 6
                                return;
606
                        }
607 10
                        if (*r != ':') {
608 4
                                VSB_cat(tl->sb, "Illegal BLOB character:\n");
609 4
                                vcc_ErrWhere(tl, tl->t);
610 4
                                VSB_destroy(&vsb);
611 4
                                return;
612
                        }
613 6
                        r++;
614 6
                        AZ(VSB_finish(vsb));
615
616 6
                        bprintf(namebuf, "blob_%u", tl->unique++);
617 12
                        Fh(tl, 0,
618
                            "\nstatic const unsigned char %s_data[%zd] = {\n",
619 6
                            namebuf, VSB_len(vsb));
620 164
                        for (u = 0; u < VSB_len(vsb); u++) {
621 158
                                Fh(tl, 0, "\t0x%02x,", VSB_data(vsb)[u] & 0xff);
622 158
                                if ((u & 7) == 7)
623 18
                                        Fh(tl, 0, "\n");
624 158
                        }
625 6
                        if ((u & 7) != 7)
626 6
                                Fh(tl, 0, "\n");
627 6
                        Fh(tl, 0, "};\n");
628 12
                        Fh(tl, 0,
629
                            "\nstatic const struct vrt_blob %s[1] = {{\n",
630 6
                            namebuf);
631 6
                        Fh(tl, 0, "\t.magic =\tVRT_BLOB_MAGIC,\n");
632 6
                        Fh(tl, 0, "\t.len =\t%zd,\n", VSB_len(vsb));
633 6
                        Fh(tl, 0, "\t.blob =\t%s_data,\n", namebuf);
634 6
                        Fh(tl, 0, "}};\n");
635 6
                        REPLACE(tl->t->dec, namebuf);
636 6
                        VSB_destroy(&vsb);
637 6
                        p = r;
638 6
                        continue;
639
                }
640
641
                /* Match delimited tokens */
642 5525340
                if (vcc_delim_token(tl, sp, p, &q) != 0) {
643 247978
                        p = q;
644 247978
                        continue;
645
                }
646 5277362
                ERRCHK(tl);
647
648
                /* Match for the fixed tokens (see generate.py) */
649 5277354
                u = vcl_fixed_token(p, &q);
650 5277354
                if (u != 0) {
651 2872162
                        vcc_addtoken(tl, u, sp, p, q);
652 2872162
                        p = q;
653 2872162
                        continue;
654
                }
655
656
                /* Match Identifiers */
657 2405192
                if (vct_isident1(*p)) {
658 18519388
                        for (q = p; q < sp->e; q++)
659 18519388
                                if (!vct_isident(*q))
660 2325866
                                        break;
661 2325866
                        vcc_addtoken(tl, ID, sp, p, q);
662 2325866
                        p = q;
663 2325866
                        continue;
664
                }
665
666
                /* Match numbers { [0-9]+ } */
667 79326
                if (vct_isdigit(*p)) {
668 79322
                        p = vcc_lex_number(tl, sp, p);
669 79322
                        if (p == NULL)
670 10
                                return;
671 79312
                        continue;
672
                }
673 4
                vcc_addtoken(tl, EOI, sp, p, p + 1);
674 4
                VSB_cat(tl->sb, "Syntax error at\n");
675 4
                vcc_ErrWhere(tl, tl->t);
676 4
                return;
677
        }
678 9204
        vcc_addtoken(tl, EOI, sp, sp->e, sp->e);
679 9250
}