[master] 657e31f25 Easier regression tracking with vtc-bisect.sh

[Dridi Boukelmoune]

commit 657e31f25a66adc65565478fc95625119da6d867
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Mon May 27 10:34:22 2019 +0200

    Easier regression tracking with vtc-bisect.sh
    
    With this script we can essentially use a VTC to track down a regression
    and see what broke after getting a cup of coffee, lunch, or whatever you
    wish to do while the script is running.
    
    It will search out of the box for a regression between the last tag and
    the current git head, and assumes the last tag passed the test case. It
    is otherwise the maintainer's responsibility to provide the good and/or
    bad commits beforehand.
    
    For more information:
    
        tools/vtc-bisect.sh -h
    
    I had this idea after setting up an automated bisect one time too many
    for #3003 and decided to generalize this in a handy script.
    
    For example:
    
        tools/vtc-bisect.sh -b d6d341609~ bin/varnishtest/tests/r03003.vtc
    
    The script works on a best-effort basis and tries to minimize rebuild
    time, or broken builds. It has minimal error handling and may leave you
    in a dirty git tree stuck in bisect mode as a result.
    
    I also noticed that 6.2.0 was not tagged in the master branch, so the
    example above starts with the varnish-6.1.0 tag. Tagging major releases
    (aka x.y.0 or dot-zero releases) from master is part of the release
    procedure and this facility should give one more reason to enforce the
    rule. Last time it happened before 6.2.0 was 5.2.0 so there's clearly
    something wrong with x.2.0 releases...
    
    This will of course not solve all bisect problems if for example a VTC
    relies on a feature that wasn't available in varnishtest when the
    regression was introduced and we still may need to perform manual git
    bisect operations but hopefully this should save us time from now on.

diff --git a/.gitignore b/.gitignore
index d24c7b41c..07e77ffa9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -127,6 +127,9 @@ cscope.*out
 /lib/libvarnishapi/vsl_glob_test
 /lib/libvarnishapi/vsl_glob_test_coverage
 
+# vtc-bisect.sh default vtc
+/bisect.vtc
+
 # vtest.sh droppings
 /tools/tmp/
 /tools/_vtest_tmp/
diff --git a/tools/vtc-bisect.sh b/tools/vtc-bisect.sh
new file mode 100755
index 000000000..050c3319c
--- /dev/null
+++ b/tools/vtc-bisect.sh
@@ -0,0 +1,130 @@
+#!/bin/sh
+#
+# Copyright (c) 2019 Varnish Software AS
+# All rights reserved.
+#
+# Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+
+set -e
+set -u
+
+usage() {
+	test $# -eq 1 &&
+	printf 'Error: %s.\n\n' "$1"
+
+	cat <<-EOF
+	Usage: $SCRIPT [-b <rev>] [-g <rev>] [-j <jobs>] [<file>]
+	       $SCRIPT [-j <jobs>] -r <file>
+	       $SCRIPT -h
+
+	Automatically look for the change that introduced a regression with
+	the help of a test case.
+
+	Available options:
+	-h         : show this help and exit
+	-b <rev>   : bad revision (defaults to HEAD)
+	-g <rev>   : good revision (defaults to latest tag before bad)
+	-j <jobs>  : number of jobs for make invocations (defaults to 8)
+	-r         : git-bisect(1) run mode
+
+	When <file> is empty or missing, bisect.vtc is expected to be found
+	at the root of the git repository. The current source tree is used
+	and VPATH setups are not supported.
+
+	This script is expected to run from the root of the git repository
+	as well.
+	EOF
+	exit $#
+}
+
+build() {
+	make -s -j"$MAKE_JOBS" all || {
+		./autogen.des
+		make -s -j"$MAKE_JOBS" all
+	}
+}
+
+run() {
+	if build
+	then
+		# ignore unfortunate build-time modifications of srcdir
+		git checkout -- .
+	else
+		git checkout -- .
+		exit 125
+	fi
+
+	bin/varnishtest/varnishtest -i "$VTC_FILE"
+	exit $?
+}
+
+BISECT_GOOD=
+BISECT_BAD=
+MAKE_JOBS=
+RUN_MODE=false
+VTC_FILE=
+
+while getopts b:g:hj:r OPT
+do
+	case $OPT in
+	b) BISECT_BAD=$OPTARG ;;
+	g) BISECT_GOOD=$OPTARG ;;
+	h) usage ;;
+	j) MAKE_JOBS=$OPTARG ;;
+	r) RUN_MODE=true ;;
+	*) usage "wrong usage" >&2 ;;
+	esac
+done
+
+shift $((OPTIND - 1))
+
+test $# -gt 1 && usage "too many arguments" >&2
+test $# -eq 1 && VTC_FILE=$1
+
+BISECT_BAD=${BISECT_BAD:-HEAD}
+MAKE_JOBS=${MAKE_JOBS:-8}
+VTC_FILE=${VTC_FILE:-bisect.vtc}
+
+[ -n "$BISECT_GOOD" ] || BISECT_GOOD=$(git describe --abbrev=0 "$BISECT_BAD")
+
+# run mode short circuit
+"$RUN_MODE" && run
+
+# bisect mode
+ROOT_DIR=$(git rev-parse --show-toplevel)
+
+readonly TMP_DIR=$(mktemp -d)
+
+trap 'rm -rf $TMP_DIR' EXIT
+
+cp "$ROOT_DIR"/tools/vtc-bisect.sh "$TMP_DIR"
+cp "$VTC_FILE" "$TMP_DIR"/bisect.vtc
+
+cd "$ROOT_DIR"
+
+git bisect start
+git bisect good "$BISECT_GOOD"
+git bisect bad "$BISECT_BAD"
+git bisect run "$TMP_DIR"/vtc-bisect.sh -r -j "$MAKE_JOBS" "$TMP_DIR"/bisect.vtc
+git bisect reset