[master] 7e3e7f904 Make witness mode a first-class citizen

[Dridi Boukelmoune]

commit 7e3e7f9040ce9527332d928967fc34f8dfcc2843
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Sep 30 08:02:20 2019 +0200

    Make witness mode a first-class citizen
    
    This change introduces a top-level make witness target that builds a dot
    graph and if graphviz is available, an SVG file as well. A shell script
    replaces the previous python script that no longer works. Instead of
    fixing witness.py, which is probably trivial, the shell script does an
    intermediate pass and programmatically looks for cycles using tsort(1).
    
    Checking lock dependencies becomes actionable in a CI context.
    
    The script also takes explicit test directories on purpose, to have the
    ability to aggregate test results from multiple executions. For example
    when the test suite is run on various operating systems or with varying
    privileges to cover feature-conditional tests.

diff --git a/.gitignore b/.gitignore
index 07e77ffa9..f2b991fc5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -144,6 +144,10 @@ cscope.*out
 /cov-int
 /myproject.tgz
 
+# Witness droppings
+witness.dot
+witness.svg
+
 # Flexelint droppings
 _.fl
 _.fl.old
diff --git a/Makefile.am b/Makefile.am
index add21a8ca..c2cbd1c13 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -8,7 +8,13 @@ pkgconfig_DATA = varnishapi.pc
 m4dir = $(datadir)/aclocal
 m4_DATA = varnish.m4 varnish-legacy.m4
 
-CLEANFILES = cscope.in.out cscope.out cscope.po.out
+CLEANFILES = \
+	cscope.in.out \
+	cscope.out \
+	cscope.po.out \
+	witness.dot \
+	witness.svg
+
 EXTRA_DIST = \
 	README.rst \
 	README.Packaging \
@@ -52,4 +58,24 @@ cscope:
 gcov_digest:
 	${PYTHON} tools/gcov_digest.py -o _gcov
 
-.PHONY: cscope
+witness.dot: all
+	$(MAKE) -C bin/varnishtest check AM_VTC_LOG_FLAGS=-pdebug=+witness
+	$(AM_V_GEN) $(srcdir)/tools/witness.sh witness.dot bin/varnishtest
+
+.dot.svg:
+if ! HAVE_DOT
+	@echo ==================================================
+	@echo You need graphviz installed to generate svg output
+	@echo ==================================================
+	@false
+else
+	$(AM_V_GEN) $(DOT) -Tsvg $< >$@
+endif
+
+if HAVE_DOT
+witness: witness.svg
+else
+witness: witness.dot
+endif
+
+.PHONY: cscope witness.dot
diff --git a/bin/varnishtest/witness.py b/bin/varnishtest/witness.py
deleted file mode 100644
index f5b516ee0..000000000
--- a/bin/varnishtest/witness.py
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env python3
-#
-# This script is in the public domain
-#
-# Run instructions:
-#       varnishtest -W -iv -j <pick_a_number> > _.w
-#       python witness.py
-#       dot -Tpng /tmp/_.dot > /tmp/_.png
-
-d = dict()
-a = dict()
-
-fi = open("_.w")
-fo = open("/tmp/_.dot", "w")
-
-fo.write('''digraph {
-    #rotate="90"
-    #page="8.2,11.7"
-    size="8.2,11.7"
-    rankdir="LR"
-    node [fontname="Inconsolata", fontsize="10"]
-    edge [fontname="Inconsolata", fontsize="10"]
-''')
-
-for i in fi:
-    l = "ROOT"
-    j = i.split()
-    if len(j) < 8:
-        continue
-    if j[1][0] != 'v':
-        continue
-    if j[3] != "vsl|":
-        continue
-    if j[5] != "Witness":
-        continue
-    t = j[7:]
-    tt = str(t)
-    if tt in d:
-        continue
-    d[tt] = True
-    for e in t:
-        f = e.split(",")
-        x = '"%s" -> "%s" [label="%s(%s)"]\n' % (l, f[0], f[1], f[2])
-        if not x in a:
-            a[x] = True
-            fo.write(x)
-        l = f[0]
-
-fo.write("}\n")
-
diff --git a/tools/witness.sh b/tools/witness.sh
new file mode 100755
index 000000000..9b8d0bfaa
--- /dev/null
+++ b/tools/witness.sh
@@ -0,0 +1,82 @@
+#!/bin/sh
+
+set -e
+set -u
+
+readonly work_dir=$(mktemp -d)
+trap 'rm -rf "$work_dir"' EXIT
+
+witness_full_paths() {
+	find "$@" -name '*.log' -print0 |
+	xargs -0 awk '$3 == "vsl|" && $5 == "Witness" {
+		printf "%s", "ROOT"
+		for (i = 7; i <= NF; i++) {
+			printf " %s", $i
+		}
+		printf "\n"
+	}' |
+	sort |
+	uniq
+}
+
+witness_edges() {
+	awk '{
+		for (i = 1; i < NF; i++) {
+			printf "%s %s\n", $i, $(i + 1)
+		}
+	}' |
+	sort |
+	uniq
+}
+
+witness_cycles() {
+	! awk -F '[ ,]' '{print $1 " " $(NF - 2)}' |
+	tsort >/dev/null 2>&1
+}
+
+witness_graph() {
+	cat <<-EOF
+	digraph {
+	    size="8.2,11.7"
+	    rankdir="LR"
+	    node [fontname="Inconsolata", fontsize="10"]
+	    edge [fontname="Inconsolata", fontsize="10"]
+	EOF
+
+	awk -F '[ ,]' '{
+		printf "    \"%s\" -> \"%s\" [label=\"%s(%s)\"]\n",
+		    $1, $(NF - 2), $(NF - 1), $NF
+	}' |
+	sort |
+	uniq
+
+	echo '}'
+}
+
+if [ $# -lt 2 ]
+then
+	cat >&2 <<-EOF
+	usage: $0 dot_file test_dirs...
+	EOF
+	exit 1
+fi
+
+dest_file=$1
+shift
+
+witness_full_paths "$@" |
+witness_edges >"$work_dir/witness-edges.txt"
+
+tsort_err=
+
+if witness_cycles <"$work_dir/witness-edges.txt"
+then
+	echo "Error: lock cycle witnessed" >&2
+	tsort_err=1
+fi
+
+witness_graph <"$work_dir/witness-edges.txt" >"$work_dir/witness.dot"
+
+mv "$work_dir/witness.dot" "$dest_file"
+
+exit $tsort_err