[6.0] 2c214ad55 Make witness mode a first-class citizen

[Dridi Boukelmoune]

commit 2c214ad5573726a73a8a6d04970ec1c65759d30f
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon Sep 30 08:02:20 2019 +0200

    Make witness mode a first-class citizen
    
    This change introduces a top-level make witness target that builds a dot
    graph and if graphviz is available, an SVG file as well. A shell script
    replaces the previous python script that no longer works. Instead of
    fixing witness.py, which is probably trivial, the shell script does an
    intermediate pass and programmatically looks for cycles using tsort(1).
    
    Checking lock dependencies becomes actionable in a CI context.
    
    The script also takes explicit test directories on purpose, to have the
    ability to aggregate test results from multiple executions. For example
    when the test suite is run on various operating systems or with varying
    privileges to cover feature-conditional tests.
    
    Conflicts:
            tools/witness.sh
    
    The 6.0 branch does not contain the dT change in the vtc output, so
    columns collected by the shell script were off by one.

diff --git a/.gitignore b/.gitignore
index f6870aa1b..2bac013d4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -139,6 +139,10 @@ cscope.*out
 /cov-int
 /myproject.tgz
 
+# Witness droppings
+witness.dot
+witness.svg
+
 # Flexelint droppings
 _.fl
 _.fl.old
diff --git a/Makefile.am b/Makefile.am
index f019082a2..781d413ce 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -8,7 +8,13 @@ pkgconfig_DATA = varnishapi.pc
 m4dir = $(datadir)/aclocal
 m4_DATA = varnish.m4 varnish-legacy.m4
 
-CLEANFILES = cscope.in.out cscope.out cscope.po.out
+CLEANFILES = \
+	cscope.in.out \
+	cscope.out \
+	cscope.po.out \
+	witness.dot \
+	witness.svg
+
 EXTRA_DIST = \
 	README.rst \
 	README.Packaging \
@@ -48,4 +54,24 @@ cscope:
 gcov_digest:
 	${PYTHON} tools/gcov_digest.py -o _gcov
 
-.PHONY: cscope
+witness.dot: all
+	$(MAKE) -C bin/varnishtest check AM_VTC_LOG_FLAGS=-pdebug=+witness
+	$(AM_V_GEN) $(srcdir)/tools/witness.sh witness.dot bin/varnishtest
+
+.dot.svg:
+if ! HAVE_DOT
+	@echo ==================================================
+	@echo You need graphviz installed to generate svg output
+	@echo ==================================================
+	@false
+else
+	$(AM_V_GEN) $(DOT) -Tsvg $< >$@
+endif
+
+if HAVE_DOT
+witness: witness.svg
+else
+witness: witness.dot
+endif
+
+.PHONY: cscope witness.dot
diff --git a/bin/varnishtest/witness.py b/bin/varnishtest/witness.py
deleted file mode 100644
index f5b516ee0..000000000
--- a/bin/varnishtest/witness.py
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env python3
-#
-# This script is in the public domain
-#
-# Run instructions:
-#       varnishtest -W -iv -j <pick_a_number> > _.w
-#       python witness.py
-#       dot -Tpng /tmp/_.dot > /tmp/_.png
-
-d = dict()
-a = dict()
-
-fi = open("_.w")
-fo = open("/tmp/_.dot", "w")
-
-fo.write('''digraph {
-    #rotate="90"
-    #page="8.2,11.7"
-    size="8.2,11.7"
-    rankdir="LR"
-    node [fontname="Inconsolata", fontsize="10"]
-    edge [fontname="Inconsolata", fontsize="10"]
-''')
-
-for i in fi:
-    l = "ROOT"
-    j = i.split()
-    if len(j) < 8:
-        continue
-    if j[1][0] != 'v':
-        continue
-    if j[3] != "vsl|":
-        continue
-    if j[5] != "Witness":
-        continue
-    t = j[7:]
-    tt = str(t)
-    if tt in d:
-        continue
-    d[tt] = True
-    for e in t:
-        f = e.split(",")
-        x = '"%s" -> "%s" [label="%s(%s)"]\n' % (l, f[0], f[1], f[2])
-        if not x in a:
-            a[x] = True
-            fo.write(x)
-        l = f[0]
-
-fo.write("}\n")
-
diff --git a/tools/witness.sh b/tools/witness.sh
new file mode 100755
index 000000000..814fae4a0
--- /dev/null
+++ b/tools/witness.sh
@@ -0,0 +1,82 @@
+#!/bin/sh
+
+set -e
+set -u
+
+readonly work_dir=$(mktemp -d)
+trap 'rm -rf "$work_dir"' EXIT
+
+witness_full_paths() {
+	find "$@" -name '*.log' -print0 |
+	xargs -0 awk '$4 == "vsl|" && $6 == "Witness" {
+		printf "%s", "ROOT"
+		for (i = 8; i <= NF; i++) {
+			printf " %s", $i
+		}
+		printf "\n"
+	}' |
+	sort |
+	uniq
+}
+
+witness_edges() {
+	awk '{
+		for (i = 1; i < NF; i++) {
+			printf "%s %s\n", $i, $(i + 1)
+		}
+	}' |
+	sort |
+	uniq
+}
+
+witness_cycles() {
+	! awk -F '[ ,]' '{print $1 " " $(NF - 2)}' |
+	tsort >/dev/null 2>&1
+}
+
+witness_graph() {
+	cat <<-EOF
+	digraph {
+	    size="8.2,11.7"
+	    rankdir="LR"
+	    node [fontname="Inconsolata", fontsize="10"]
+	    edge [fontname="Inconsolata", fontsize="10"]
+	EOF
+
+	awk -F '[ ,]' '{
+		printf "    \"%s\" -> \"%s\" [label=\"%s(%s)\"]\n",
+		    $1, $(NF - 2), $(NF - 1), $NF
+	}' |
+	sort |
+	uniq
+
+	echo '}'
+}
+
+if [ $# -lt 2 ]
+then
+	cat >&2 <<-EOF
+	usage: $0 dot_file test_dirs...
+	EOF
+	exit 1
+fi
+
+dest_file=$1
+shift
+
+witness_full_paths "$@" |
+witness_edges >"$work_dir/witness-edges.txt"
+
+tsort_err=
+
+if witness_cycles <"$work_dir/witness-edges.txt"
+then
+	echo "Error: lock cycle witnessed" >&2
+	tsort_err=1
+fi
+
+witness_graph <"$work_dir/witness-edges.txt" >"$work_dir/witness.dot"
+
+mv "$work_dir/witness.dot" "$dest_file"
+
+exit $tsort_err