[master] 206ae06 Treat skipped tests as such

[Federico G. Schwindt]

commit 206ae064c0d0abddd711569a44f0a7d976edceb2
Author: Federico G. Schwindt <fgsch at lodoss.net>
Date:   Wed Nov 30 19:27:31 2016 +0000

    Treat skipped tests as such
    
    E.g.
    
    and
    
    0 tests failed, 7 tests skipped, 558 tests passed
    
    This could use some tlc but will do it for now.

diff --git a/bin/varnishtest/vtc.c b/bin/varnishtest/vtc.c
index 7d2219e..3a8522a 100644
--- a/bin/varnishtest/vtc.c
+++ b/bin/varnishtest/vtc.c
@@ -585,7 +585,7 @@ cmd_feature(CMD_ARGS)
 			if (tst) {			\
 				good = 1;		\
 			} else {			\
-				vtc_stop = 1;		\
+				vtc_stop = 2;		\
 			}				\
 		}					\
 	} while (0)
@@ -596,7 +596,7 @@ cmd_feature(CMD_ARGS)
 #ifdef SO_RCVTIMEO_WORKS
 			good = 1;
 #else
-			vtc_stop = 1;
+			vtc_stop = 2;
 #endif
 		}
 
@@ -604,7 +604,7 @@ cmd_feature(CMD_ARGS)
 #if !defined(__APPLE__) || !defined(__MACH__)
 			good = 1;
 #else
-			vtc_stop = 1;
+			vtc_stop = 2;
 #endif
 		}
 		FEATURE("pcre_jit", VRE_has_jit);
@@ -623,7 +623,7 @@ cmd_feature(CMD_ARGS)
 			r = personality(r | ADDR_NO_RANDOMIZE);
 			if (r < 0) {
 				good = 0;
-				vtc_stop = 1;
+				vtc_stop = 2;
 			}
 #endif
 		} else if (!strcmp(*av, "cmd")) {
@@ -636,7 +636,7 @@ cmd_feature(CMD_ARGS)
 			if (WEXITSTATUS(r) == 0)
 				good = 1;
 			else
-				vtc_stop = 1;
+				vtc_stop = 2;
 		}
 		if (good)
 			continue;
@@ -706,7 +706,8 @@ exec_file(const char *fn, const char *script, const char *tmpdir,
 	vtc_thread = pthread_self();
 	parse_string(script, cmds, NULL, vltop);
 	old_err = vtc_error;
-	vtc_stop = 1;
+	if (!vtc_stop)
+		vtc_stop = 1;
 	vtc_log(vltop, 1, "RESETTING after %s", fn);
 	reset_cmds(cmds);
 	vtc_error |= old_err;
@@ -716,5 +717,7 @@ exec_file(const char *fn, const char *script, const char *tmpdir,
 	else
 		vtc_log(vltop, 1, "TEST %s completed", fn);
 
+	if (vtc_stop > 1)
+		return (1);
 	return (vtc_error);
 }
diff --git a/bin/varnishtest/vtc_log.c b/bin/varnishtest/vtc_log.c
index 67918c3..b111d28 100644
--- a/bin/varnishtest/vtc_log.c
+++ b/bin/varnishtest/vtc_log.c
@@ -138,7 +138,7 @@ vtc_log(struct vtclog *vl, int lvl, const char *fmt, ...)
 	if (lvl > 0)
 		return;
 	if (lvl == 0)
-		vtc_error = 1;
+		vtc_error = 2;
 	if (pthread_self() != vtc_thread)
 		pthread_exit(NULL);
 }
@@ -204,7 +204,7 @@ vtc_dump(struct vtclog *vl, int lvl, const char *pfx, const char *str, int len)
 	vl->act = 0;
 	AZ(pthread_mutex_unlock(&vl->mtx));
 	if (lvl == 0) {
-		vtc_error = 1;
+		vtc_error = 2;
 		if (pthread_self() != vtc_thread)
 			pthread_exit(NULL);
 	}
@@ -264,7 +264,7 @@ vtc_hexdump(struct vtclog *vl, int lvl, const char *pfx,
 	vl->act = 0;
 	AZ(pthread_mutex_unlock(&vl->mtx));
 	if (lvl == 0) {
-		vtc_error = 1;
+		vtc_error = 2;
 		if (pthread_self() != vtc_thread)
 			pthread_exit(NULL);
 	}
diff --git a/bin/varnishtest/vtc_main.c b/bin/varnishtest/vtc_main.c
index c6d2a99..3ca934c 100644
--- a/bin/varnishtest/vtc_main.c
+++ b/bin/varnishtest/vtc_main.c
@@ -94,6 +94,7 @@ static int vtc_continue;		/* Continue on error */
 static int vtc_verbosity = 1;		/* Verbosity Level */
 static int vtc_good;
 static int vtc_fail;
+static int vtc_skip;
 static char *tmppath;
 static char *cwd = NULL;
 char *vmod_path = NULL;
@@ -158,6 +159,7 @@ tst_cb(const struct vev *ve, int what)
 {
 	struct vtc_job *jp;
 	char buf[BUFSIZ];
+	int ecode, signo;
 	int i, stx;
 	pid_t px;
 	double t;
@@ -184,17 +186,26 @@ tst_cb(const struct vev *ve, int what)
 		t = VTIM_mono() - jp->t0;
 		AZ(close(ve->fd));
 
-		if (stx && vtc_verbosity)
+		ecode = 2;
+		signo = 0;
+		if (WIFEXITED(stx))
+			ecode = WEXITSTATUS(stx);
+		if (WIFSIGNALED(stx))
+			signo = WTERMSIG(stx);
+
+		if (ecode > 1 && vtc_verbosity)
 			printf("%s\n", jp->buf);
 		else if (vtc_verbosity > 1)
 			printf("%s\n", jp->buf);
 
-		if (stx)
-			vtc_fail++;
-		else
+		if (!ecode)
 			vtc_good++;
+		else if (ecode == 1)
+			vtc_skip++;
+		else
+			vtc_fail++;
 
-		if (leave_temp == 0 || (leave_temp == 1 && !stx)) {
+		if (leave_temp == 0 || (leave_temp == 1 && ecode <= 1)) {
 			bprintf(buf, "rm -rf %s", jp->tmpdir);
 			AZ(system(buf));
 		} else {
@@ -206,19 +217,20 @@ tst_cb(const struct vev *ve, int what)
 		}
 		free(jp->tmpdir);
 
-		if (stx) {
+		if (ecode > 1) {
 			printf("#     top  TEST %s FAILED (%.3f)",
 			    jp->tst->filename, t);
-			if (WIFSIGNALED(stx))
-				printf(" signal=%d", WTERMSIG(stx));
-			printf(" exit=%d\n", WEXITSTATUS(stx));
+			if (signo)
+				printf(" signal=%d", signo);
+			printf(" exit=%d\n", ecode);
 			if (!vtc_continue) {
 				/* XXX kill -9 other jobs ? */
 				exit(2);
 			}
 		} else if (vtc_verbosity) {
-			printf("#     top  TEST %s passed (%.3f)\n",
-			    jp->tst->filename, t);
+			printf("#     top  TEST %s %s (%.3f)\n",
+			    jp->tst->filename,
+			    ecode ? "skipped" : "passed", t);
 		}
 		AZ(munmap(jp->buf, jp->bufsiz));
 		if (jp->evt != NULL)
@@ -640,9 +652,12 @@ main(int argc, char * const *argv)
 		i = vev_schedule_one(vb);
 	}
 	if (vtc_continue)
-		fprintf(stderr, "%d tests failed, %d tests passed\n",
-		    vtc_fail, vtc_good);
+		fprintf(stderr,
+		    "%d tests failed, %d tests skipped, %d tests passed\n",
+		    vtc_fail, vtc_skip, vtc_good);
 	if (vtc_fail)
 		return (1);
+	if (vtc_skip && !vtc_good)
+		return (77);
 	return (0);
 }