[6.0] 444378371 Limit watchdog to highest priority only

[Reza Naghibi]

commit 444378371c222c26eb98e6fdad3cf0e77faa4cd7
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Feb 25 15:58:27 2021 +0100

    Limit watchdog to highest priority only
    
    The watchdog mechanism currently triggers when any queueing is happening,
    regardless of the priority. Strictly speaking it is only the backend
    fetches that are critical to get executed, and this prevents the thread
    limits to be used as limits on the amount of work the Varnish instance
    should handle.
    
    This can be especially important for instances with H/2 enabled, as these
    connections will be holding threads for extended periods of time, possibly
    triggering the watchdog in benign situations.
    
    This patch limits the watchdog to only trigger for no queue development
    on the highest priority queue.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index ff85761c4..db5467a8d 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -233,6 +233,7 @@ enum task_prio {
 	TASK_QUEUE__END
 };
 
+#define TASK_QUEUE_HIGHEST_PRIORITY TASK_QUEUE_BO
 #define TASK_QUEUE_CLIENT(prio) \
 	(prio == TASK_QUEUE_REQ || prio == TASK_QUEUE_STR)
 
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 2083e4cf7..eacf21c04 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -532,7 +532,9 @@ pool_herder(void *priv)
 		 * Instead we implement a watchdog and kill the worker if
 		 * nothing has been dequeued for that long.
 		 */
-		if (pp->lqueue == 0) {
+		if (VTAILQ_EMPTY(&pp->queues[TASK_QUEUE_HIGHEST_PRIORITY])) {
+			/* Watchdog only applies to no movement on the
+			 * highest priority queue (TASK_QUEUE_BO) */
 			dq = pp->ndequeued + 1;
 		} else if (dq != pp->ndequeued) {
 			dq = pp->ndequeued;
diff --git a/bin/varnishtest/tests/c00104.vtc b/bin/varnishtest/tests/c00104.vtc
new file mode 100644
index 000000000..3af498179
--- /dev/null
+++ b/bin/varnishtest/tests/c00104.vtc
@@ -0,0 +1,32 @@
+varnishtest "Test watchdog only active on queue 0"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -cliok "param.set thread_pools 1"
+varnish v1 -cliok "param.set thread_pool_min 5"
+varnish v1 -cliok "param.set thread_pool_max 5"
+varnish v1 -cliok "param.set thread_pool_watchdog 1"
+varnish v1 -cliok "param.set feature +http2"
+
+varnish v1 -vcl+backend {
+} -start
+
+client c1 {
+	txpri
+	delay 2
+} -start
+
+client c2 {
+	txpri
+	delay 2
+} -start
+
+client c3 {
+	txpri
+	delay 2
+} -start
+
+delay 2