varnish-cache/bin/varnishtest/vtc_main.c
1
/*-
2
 * Copyright (c) 2008-2011 Varnish Software AS
3
 * All rights reserved.
4
 *
5
 * Author: Poul-Henning Kamp <phk@phk.freebsd.dk>
6
 *
7
 * Redistribution and use in source and binary forms, with or without
8
 * modification, are permitted provided that the following conditions
9
 * are met:
10
 * 1. Redistributions of source code must retain the above copyright
11
 *    notice, this list of conditions and the following disclaimer.
12
 * 2. Redistributions in binary form must reproduce the above copyright
13
 *    notice, this list of conditions and the following disclaimer in the
14
 *    documentation and/or other materials provided with the distribution.
15
 *
16
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19
 * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
20
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26
 * SUCH DAMAGE.
27
 */
28
29
#include "config.h"
30
31
#include <sys/mman.h>
32
#include <sys/stat.h>
33
#include <sys/wait.h>
34
35
#include <ctype.h>
36
#include <errno.h>
37
#include <poll.h>
38
#include <stdio.h>
39
#include <stdlib.h>
40
#include <string.h>
41
#include <unistd.h>
42
43
#include "vtc.h"
44
45
#include "vev.h"
46
#include "vfil.h"
47
#include "vnum.h"
48
#include "vrnd.h"
49
#include "vss.h"
50
#include "vsub.h"
51
#include "vtcp.h"
52
#include "vtim.h"
53
#include "vct.h"
54
55
static const char *argv0;
56
57
struct vtc_tst {
58
        unsigned                magic;
59
#define TST_MAGIC               0x618d8b88
60
        VTAILQ_ENTRY(vtc_tst)   list;
61
        const char              *filename;
62
        char                    *script;
63
        unsigned                ntodo;
64
};
65
66
struct vtc_job {
67
        unsigned                magic;
68
#define JOB_MAGIC               0x1b5fc419
69
        struct vtc_tst          *tst;
70
        pid_t                   child;
71
        struct vev              *ev;
72
        struct vev              *evt;
73
        char                    *buf;
74
        char                    *tmpdir;
75
        unsigned                bufsiz;
76
        double                  t0;
77
        struct vsb              *diag;
78
        int                     killed;
79
};
80
81
int iflg = 0;
82
unsigned vtc_maxdur = 60;
83
static unsigned vtc_bufsiz = 1024 * 1024;
84
85
static VTAILQ_HEAD(, vtc_tst) tst_head = VTAILQ_HEAD_INITIALIZER(tst_head);
86
static struct vev_root *vb;
87
static int njob = 0;
88
static int npar = 1;                    /* Number of parallel tests */
89
static int vtc_continue;                /* Continue on error */
90
static int vtc_verbosity = 1;           /* Verbosity Level */
91
static int vtc_good;
92
static int vtc_fail;
93
static int vtc_skip;
94
static char *tmppath;
95
static char *cwd = NULL;
96
char *vmod_path = NULL;
97
struct vsb *params_vsb = NULL;
98
int leave_temp;
99
int vtc_witness = 0;
100
101
/**********************************************************************
102
 * Parse a -D option argument into a name/val pair, and insert
103
 * into extmacro list
104
 */
105
106
static int
107 4
parse_D_opt(char *arg)
108
{
109
        char *p, *q;
110
111 4
        p = arg;
112 4
        q = strchr(p, '=');
113 4
        if (!q)
114 0
                return (0);
115 4
        *q++ = '\0';
116 4
        extmacro_def(p, "%s", q);
117
118 4
        return (1);
119
}
120
121
/**********************************************************************
122
 * Print usage
123
 */
124
125
static void
126 8
usage(void)
127
{
128 8
        fprintf(stderr, "usage: %s [options] file ...\n", argv0);
129
#define FMT "    %-28s # %s\n"
130 8
        fprintf(stderr, FMT, "-b size",
131
            "Set internal buffer size (default: 1M)");
132 8
        fprintf(stderr, FMT, "-D name=val", "Define macro");
133 8
        fprintf(stderr, FMT, "-i", "Find varnish binaries in build tree");
134 8
        fprintf(stderr, FMT, "-j jobs", "Run this many tests in parallel");
135 8
        fprintf(stderr, FMT, "-k", "Continue on test failure");
136 8
        fprintf(stderr, FMT, "-L", "Always leave temporary vtc.*");
137 8
        fprintf(stderr, FMT, "-l", "Leave temporary vtc.* if test fails");
138 8
        fprintf(stderr, FMT, "-n iterations", "Run tests this many times");
139 8
        fprintf(stderr, FMT, "-p name=val", "Pass a varnishd parameter");
140 8
        fprintf(stderr, FMT, "-q", "Quiet mode: report only failures");
141 8
        fprintf(stderr, FMT, "-t duration", "Time tests out after this long");
142 8
        fprintf(stderr, FMT, "-v", "Verbose mode: always report test log");
143 8
        fprintf(stderr, FMT, "-W", "Enable the witness facility for locking");
144 8
        exit(1);
145
}
146
147
/**********************************************************************
148
 * CallBack
149
 */
150
151
static int
152 1526
tst_cb(const struct vev *ve, int what)
153
{
154
        struct vtc_job *jp;
155
        char buf[BUFSIZ];
156
        int ecode;
157
        int i, stx;
158
        pid_t px;
159
        double t;
160
        FILE *f;
161
        char *p;
162
        struct vsb *v;
163
164 1526
        CAST_OBJ_NOTNULL(jp, ve->priv, JOB_MAGIC);
165
166
        // printf("CB %p %s %d\n", ve, jp->tst->filename, what);
167 1526
        if (what == 0) {
168 0
                jp->killed = 1;
169 0
                AZ(kill(-jp->child, SIGKILL)); /* XXX: Timeout */
170
        } else {
171 1526
                assert(what & (VEV__RD | VEV__HUP));
172
        }
173
174 1526
        *buf = '\0';
175 1526
        i = read(ve->fd, buf, sizeof buf);
176 1526
        if (i > 0)
177 0
                VSB_bcat(jp->diag, buf, i);
178 1526
        if (i == 0) {
179 1526
                njob--;
180 1526
                px = wait4(jp->child, &stx, 0, NULL);
181 1526
                assert(px == jp->child);
182 1526
                t = VTIM_mono() - jp->t0;
183 1526
                AZ(close(ve->fd));
184
185 1526
                ecode = WTERMSIG(stx);
186 1526
                if (ecode == 0)
187 1526
                        ecode = WEXITSTATUS(stx);
188
189 1526
                AZ(VSB_finish(jp->diag));
190 1526
                v = VSB_new_auto();
191 1526
                AN(v);
192 1526
                VSB_cat(v, jp->buf);
193 1526
                p = strchr(jp->buf, '\0');
194 1526
                if (p > jp->buf && p[-1] != '\n')
195 0
                        VSB_putc(v, '\n');
196 1526
                VSB_quote_pfx(v, "*    diag  0.0 ",
197 1526
                    VSB_data(jp->diag), -1, VSB_QUOTE_NONL);
198 1526
                AZ(VSB_finish(v));
199 1526
                VSB_destroy(&jp->diag);
200 1526
                AZ(munmap(jp->buf, jp->bufsiz));
201
202 1526
                if ((ecode > 1 && vtc_verbosity) || vtc_verbosity > 1)
203 1476
                        printf("%s", VSB_data(v));
204
205 1526
                if (!ecode)
206 1504
                        vtc_good++;
207 22
                else if (ecode == 1)
208 18
                        vtc_skip++;
209
                else
210 4
                        vtc_fail++;
211
212 1526
                if (leave_temp == 0 || (leave_temp == 1 && ecode <= 1)) {
213 1526
                        bprintf(buf, "rm -rf %s", jp->tmpdir);
214 1526
                        AZ(system(buf));
215
                } else {
216 0
                        bprintf(buf, "%s/LOG", jp->tmpdir);
217 0
                        f = fopen(buf, "w");
218 0
                        AN(f);
219 0
                        (void)fprintf(f, "%s\n", VSB_data(v));
220 0
                        AZ(fclose(f));
221
                }
222 1526
                free(jp->tmpdir);
223 1526
                VSB_destroy(&v);
224
225 1526
                if (jp->killed)
226 0
                        printf("#    top  TEST %s TIMED OUT (kill -9)\n",
227 0
                            jp->tst->filename);
228 1526
                if (ecode > 1) {
229 4
                        printf("#    top  TEST %s FAILED (%.3f)",
230 4
                            jp->tst->filename, t);
231 4
                        if (WIFSIGNALED(stx))
232 0
                                printf(" signal=%d\n", WTERMSIG(stx));
233 4
                        else if (WIFEXITED(stx))
234 4
                                printf(" exit=%d\n", WEXITSTATUS(stx));
235 4
                        if (!vtc_continue) {
236
                                /* XXX kill -9 other jobs ? */
237 4
                                exit(2);
238
                        }
239 1522
                } else if (vtc_verbosity) {
240 3036
                        printf("#    top  TEST %s %s (%.3f)\n",
241 1518
                            jp->tst->filename,
242
                            ecode ? "skipped" : "passed", t);
243
                }
244 1522
                if (jp->evt != NULL) {
245 1522
                        VEV_Stop(vb, jp->evt);
246 1522
                        free(jp->evt);
247
                }
248 1522
                FREE_OBJ(jp);
249 1522
                return (1);
250
        }
251 0
        return (0);
252
}
253
254
/**********************************************************************
255
 * Start Test
256
 */
257
258
static void
259 1526
start_test(void)
260
{
261
        struct vtc_tst *tp;
262
        int p[2], retval;
263
        struct vtc_job *jp;
264
        char tmpdir[PATH_MAX];
265
266 1526
        ALLOC_OBJ(jp, JOB_MAGIC);
267 1526
        AN(jp);
268
269 1526
        jp->diag = VSB_new_auto();
270 1526
        AN(jp->diag);
271
272 1526
        jp->bufsiz = vtc_bufsiz;
273
274 1526
        jp->buf = mmap(NULL, jp->bufsiz, PROT_READ|PROT_WRITE,
275
            MAP_ANON | MAP_SHARED, -1, 0);
276 1526
        assert(jp->buf != MAP_FAILED);
277 1526
        memset(jp->buf, 0, jp->bufsiz);
278
279 1526
        VRND_SeedAll();
280 1526
        bprintf(tmpdir, "%s/vtc.%d.%08x", tmppath, (int)getpid(),
281
                (unsigned)random());
282 1526
        AZ(mkdir(tmpdir, 0711));
283
284 1526
        tp = VTAILQ_FIRST(&tst_head);
285 1526
        CHECK_OBJ_NOTNULL(tp, TST_MAGIC);
286 1526
        AN(tp->ntodo);
287 1526
        tp->ntodo--;
288 1526
        VTAILQ_REMOVE(&tst_head, tp, list);
289 1526
        if (tp->ntodo >0)
290 0
                VTAILQ_INSERT_TAIL(&tst_head, tp, list);
291
292 1526
        jp->tst = tp;
293 1526
        jp->tmpdir = strdup(tmpdir);
294 1526
        AN(jp->tmpdir);
295
296 1526
        AZ(pipe(p));
297 1526
        assert(p[0] > STDERR_FILENO);
298 1526
        assert(p[1] > STDERR_FILENO);
299 1526
        jp->t0 = VTIM_mono();
300 1526
        jp->child = fork();
301 3052
        assert(jp->child >= 0);
302 3052
        if (jp->child == 0) {
303 1526
                AZ(setpgid(getpid(), 0));
304 1526
                VFIL_null_fd(STDIN_FILENO);
305 1526
                assert(dup2(p[1], STDOUT_FILENO) == STDOUT_FILENO);
306 1526
                assert(dup2(p[1], STDERR_FILENO) == STDERR_FILENO);
307 1526
                VSUB_closefrom(STDERR_FILENO + 1);
308 3052
                retval = exec_file(jp->tst->filename, jp->tst->script,
309 1526
                    jp->tmpdir, jp->buf, jp->bufsiz);
310 1522
                exit(retval);
311
        }
312 1526
        closefd(&p[1]);
313
314 1526
        jp->ev = VEV_Alloc();
315 1526
        AN(jp->ev);
316 1526
        jp->ev->fd_flags = VEV__RD | VEV__HUP | VEV__ERR;
317 1526
        jp->ev->fd = p[0];
318 1526
        jp->ev->priv = jp;
319 1526
        jp->ev->callback = tst_cb;
320 1526
        AZ(VEV_Start(vb, jp->ev));
321
322 1526
        jp->evt = VEV_Alloc();
323 1526
        AN(jp->evt);
324 1526
        jp->evt->fd = -1;
325 1526
        jp->evt->timeout = vtc_maxdur;
326 1526
        jp->evt->priv = jp;
327 1526
        jp->evt->callback = tst_cb;
328 1526
        AZ(VEV_Start(vb, jp->evt));
329 1526
}
330
331
/**********************************************************************
332
 * i-mode = "we're inside a src-tree"
333
 *
334
 * Find the abs path to top of source dir from Makefile, if that
335
 * fails, fall back on "../../"
336
 *
337
 * Set path to all programs build directories
338
 *
339
 */
340
341
static void
342 1468
i_mode(void)
343
{
344
        const char *sep;
345
        struct vsb *vsb;
346
        char *p, *q;
347
        char *topbuild;
348
349
        /*
350
         * This code has a rather intimate knowledge of auto* generated
351
         * makefiles.
352
         */
353
354 1468
        vsb = VSB_new_auto();
355 1468
        AN(vsb);
356
357 1468
        q = p = VFIL_readfile(NULL, "Makefile", NULL);
358 1468
        if (p == NULL) {
359 0
                fprintf(stderr, "No Makefile to search for -i flag.\n");
360 0
                VSB_printf(vsb, "%s/../..", cwd);
361 0
                AZ(VSB_finish(vsb));
362 0
                topbuild = strdup(VSB_data(vsb));
363 0
                VSB_clear(vsb);
364
        } else {
365 1468
                p = strstr(p, "\nabs_top_builddir");
366 1468
                if (p == NULL) {
367 0
                        fprintf(stderr,
368
                            "could not find 'abs_top_builddir' in Makefile\n");
369 0
                        exit(2);
370
                }
371 1468
                topbuild =  strchr(p + 1, '\n');
372 1468
                if (topbuild == NULL) {
373 0
                        fprintf(stderr,
374
                            "No NL after 'abs_top_builddir' in Makefile\n");
375 0
                        exit(2);
376
                }
377 1468
                *topbuild = '\0';
378 1468
                topbuild = strchr(p, '/');
379 1468
                if (topbuild == NULL) {
380 0
                        fprintf(stderr,
381
                            "No '/' after 'abs_top_builddir' in Makefile\n");
382 0
                        exit(2);
383
                }
384 1468
                topbuild = strdup(topbuild);
385 1468
                free(q);
386
387
        }
388 1468
        AN(topbuild);
389 1468
        extmacro_def("topbuild", "%s", topbuild);
390
        /*
391
         * Build $PATH which can find all programs in the build tree
392
         */
393 1468
        VSB_printf(vsb, "PATH=");
394 1468
        sep = "";
395
#define VTC_PROG(l)                                                     \
396
        do {                                                            \
397
                VSB_printf(vsb, "%s%s/bin/" #l, sep, topbuild);         \
398
                sep = ":";                                              \
399
        } while (0);
400
#include "programs.h"
401
402 1468
        VSB_printf(vsb, ":%s", getenv("PATH"));
403 1468
        AZ(VSB_finish(vsb));
404
405 1468
        AZ(putenv(strdup(VSB_data(vsb))));
406
407
        /*
408
         * Build vmod_path which can find all VMODs in the build tree
409
         */
410 1468
        VSB_clear(vsb);
411 1468
        sep = "";
412
#define VTC_VMOD(l)                                                     \
413
        do {                                                            \
414
                VSB_printf(vsb, "%s%s/lib/libvmod_" #l "/.libs",        \
415
                    sep, topbuild);                                     \
416
                sep = ":";                                              \
417
        } while (0);
418
#include "vmods.h"
419
#undef VTC_VMOD
420 1468
        AZ(VSB_finish(vsb));
421 1468
        vmod_path = strdup(VSB_data(vsb));
422 1468
        AN(vmod_path);
423 1468
        free(topbuild);
424 1468
        VSB_destroy(&vsb);
425
426
        /*
427
         * strict jemalloc checking
428
         */
429 1468
        AZ(putenv(strdup("MALLOC_CONF=abort:true,junk:true")));
430 1468
}
431
432
/**********************************************************************
433
 * Figure out what IP related magic
434
 */
435
436
static int v_matchproto_(vss_resolved_f)
437 1486
bind_cb(void *priv, const struct suckaddr *sa)
438
{
439
        (void)priv;
440 1486
        return (VTCP_bind(sa, NULL));
441
}
442
443
static void
444 1486
ip_magic(void)
445
{
446
        const char *p;
447
        int fd;
448
        char abuf[VTCP_ADDRBUFSIZE];
449
        char pbuf[VTCP_PORTBUFSIZE];
450
451
        /*
452
         * In FreeBSD jails localhost/127.0.0.1 becomes the jails IP#
453
         * XXX: IPv6-only hosts would have similar issue, but it is not
454
         * XXX: obvious how to cope.  Ideally "127.0.0.1" would be
455
         * XXX: "localhost", but that doesn't work out of the box.
456
         * XXX: Things like "prefer_ipv6" parameter complicates things.
457
         */
458 1486
        fd = VSS_resolver("127.0.0.1", NULL, bind_cb, NULL, &p);
459 1486
        assert(fd >= 0);
460 1486
        VTCP_myname(fd, abuf, sizeof abuf, pbuf, sizeof(pbuf));
461 1486
        extmacro_def("localhost", "%s", abuf);
462
463
#if defined (__APPLE__)
464
        /*
465
         * In MacOS a bound socket that is not listening will timeout
466
         * instead of refusing the connection so close it and hope
467
         * for the best.
468
         */
469
        VTCP_close(&fd);
470
#endif
471
472
        /* Expose a backend that is forever down. */
473 1486
        extmacro_def("bad_backend", "%s %s", abuf, pbuf);
474
475
        /*
476
         * We need an IP number which will not repond, ever, and that is a
477
         * lot harder than it sounds.  This IP# is from RFC5737 and a
478
         * C-class broadcast at that.
479
         * If tests involving ${bad_ip} fails and you run linux, you should
480
         * check your /proc/sys/net/ipv4/ip_nonlocal_bind setting.
481
         */
482
483 1486
        extmacro_def("bad_ip", "%s", "192.0.2.255");
484 1486
}
485
486
/**********************************************************************
487
 * Main
488
 */
489
490
static int
491 1534
read_file(const char *fn, int ntest)
492
{
493
        struct vtc_tst *tp;
494
        char *p, *q;
495
496 1534
        p = VFIL_readfile(NULL, fn, NULL);
497 1534
        if (p == NULL) {
498 0
                fprintf(stderr, "Cannot stat file \"%s\": %s\n",
499 0
                    fn, strerror(errno));
500 0
                return (2);
501
        }
502 1534
        for (q = p ;q != NULL && *q != '\0'; q++) {
503 1534
                if (vct_islws(*q))
504 0
                        continue;
505 1534
                if (*q != '#')
506 1534
                        break;
507 0
                q = strchr(q, '\n');
508 0
                if (q == NULL)
509 0
                        break;
510
        }
511
512 1534
        if (q == NULL || *q == '\0') {
513 0
                fprintf(stderr, "File \"%s\" has no content.\n", fn);
514 0
                free(p);
515 0
                return (2);
516
        }
517
518 1546
        if ((strncmp(q, "varnishtest", 11) || !isspace(q[11])) &&
519 16
            (strncmp(q, "vtest", 5) || !isspace(q[5]))) {
520 8
                fprintf(stderr,
521
                    "File \"%s\" doesn't start with"
522
                    " 'vtest' or 'varnishtest'\n", fn);
523 8
                free(p);
524 8
                vtc_skip++;
525 8
                return(2);
526
        }
527 1526
        ALLOC_OBJ(tp, TST_MAGIC);
528 1526
        AN(tp);
529 1526
        tp->filename = fn;
530 1526
        tp->script = p;
531 1526
        tp->ntodo = ntest;
532 1526
        VTAILQ_INSERT_TAIL(&tst_head, tp, list);
533 1526
        return(0);
534
}
535
536
/**********************************************************************
537
 * Main
538
 */
539
540
int
541 1498
main(int argc, char * const *argv)
542
{
543
        int ch, i;
544 1498
        int ntest = 1;                  /* Run tests this many times */
545
        uintmax_t bufsiz;
546
        const char *p;
547
548 1498
        argv0 = strrchr(argv[0], '/');
549 1498
        if (argv0 == NULL)
550 0
                argv0 = argv[0];
551
        else
552 1498
                argv0++;
553
554 1498
        if (getenv("TMPDIR") != NULL)
555 1494
                tmppath = strdup(getenv("TMPDIR"));
556
        else
557 4
                tmppath = strdup("/tmp");
558
559 1498
        cwd = getcwd(NULL, PATH_MAX);
560 1498
        extmacro_def("pwd", "%s", cwd);
561
562 1498
        vmod_path = NULL;
563
564 1498
        params_vsb = VSB_new_auto();
565 1498
        AN(params_vsb);
566 1498
        p = getenv("VTEST_DURATION");
567 1498
        if (p == NULL)
568 1498
                p = getenv("VARNISHTEST_DURATION");
569 1498
        if (p != NULL)
570 0
                vtc_maxdur = atoi(p);
571
572 1498
        setbuf(stdout, NULL);
573 1498
        setbuf(stderr, NULL);
574 5960
        while ((ch = getopt(argc, argv, "b:D:hij:kLln:p:qt:vW")) != -1) {
575 2972
                switch (ch) {
576
                case 'b':
577 0
                        if (VNUM_2bytes(optarg, &bufsiz, 0)) {
578 0
                                fprintf(stderr, "Cannot parse b opt '%s'\n",
579
                                    optarg);
580 0
                                exit(2);
581
                        }
582 0
                        if (bufsiz > UINT_MAX) {
583 0
                                fprintf(stderr, "Invalid b opt '%s'\n",
584
                                    optarg);
585 0
                                exit(2);
586
                        }
587 0
                        vtc_bufsiz = (unsigned)bufsiz;
588 0
                        break;
589
                case 'D':
590 4
                        if (!parse_D_opt(optarg)) {
591 0
                                fprintf(stderr, "Cannot parse D opt '%s'\n",
592
                                    optarg);
593 0
                                exit(2);
594
                        }
595 4
                        break;
596
                case 'i':
597 1468
                        iflg = 1;
598 1468
                        break;
599
                case 'j':
600 0
                        npar = strtoul(optarg, NULL, 0);
601 0
                        break;
602
                case 'L':
603 0
                        leave_temp = 2;
604 0
                        break;
605
                case 'l':
606 0
                        leave_temp = 1;
607 0
                        break;
608
                case 'k':
609 4
                        vtc_continue = !vtc_continue;
610 4
                        break;
611
                case 'n':
612 0
                        ntest = strtoul(optarg, NULL, 0);
613 0
                        break;
614
                case 'p':
615 2
                        VSB_printf(params_vsb, " -p ");
616 2
                        VSB_quote(params_vsb, optarg, -1, 0);
617 2
                        break;
618
                case 'q':
619 4
                        if (vtc_verbosity > 0)
620 4
                                vtc_verbosity--;
621 4
                        break;
622
                case 't':
623 2
                        vtc_maxdur = strtoul(optarg, NULL, 0);
624 2
                        break;
625
                case 'v':
626 1480
                        if (vtc_verbosity < 2)
627 1480
                                vtc_verbosity++;
628 1480
                        break;
629
                case 'W':
630 0
                        vtc_witness++;
631 0
                        break;
632
                default:
633 8
                        usage();
634
                }
635
        }
636 1490
        argc -= optind;
637 1490
        argv += optind;
638
639 1490
        if (argc < 1)
640 0
                usage();
641
642 3020
        for (; argc > 0; argc--, argv++) {
643 1534
                if (!read_file(*argv, ntest))
644 1526
                        continue;
645 8
                if (!vtc_continue)
646 4
                        exit(2);
647
        }
648
649 1486
        AZ(VSB_finish(params_vsb));
650
651 1486
        ip_magic();
652
653 1486
        if (iflg)
654 1468
                i_mode();
655
656 1486
        vb = VEV_New();
657
658 1486
        i = 0;
659 7498
        while (!VTAILQ_EMPTY(&tst_head) || i) {
660 4530
                if (!VTAILQ_EMPTY(&tst_head) && njob < npar) {
661 1526
                        start_test();
662 1526
                        njob++;
663
                        /* Stagger ramp-up */
664 1526
                        if (njob < npar)
665 0
                                (void)usleep(random() % 100000L);
666 1526
                        i = 1;
667 1526
                        continue;
668
                }
669 3004
                i = VEV_Once(vb);
670
        }
671 1482
        if (vtc_continue)
672 4
                fprintf(stderr,
673
                    "%d tests failed, %d tests skipped, %d tests passed\n",
674
                    vtc_fail, vtc_skip, vtc_good);
675 1482
        if (vtc_fail)
676 0
                return (1);
677 1482
        if (vtc_skip && !vtc_good)
678 22
                return (77);
679 1460
        return (0);
680
}