[master] e4ea44564 Limit watchdog to highest priority only

[Martin Blix Grydeland]

commit e4ea44564f512d42f99aca833b4f71dcc8b445c7
Author: Martin Blix Grydeland <martin at varnish-software.com>
Date:   Thu Feb 25 15:58:27 2021 +0100

    Limit watchdog to highest priority only
    
    The watchdog mechanism currently triggers when any queueing is happening,
    regardless of the priority. Strictly speaking it is only the backend
    fetches that are critical to get executed, and this prevents the thread
    limits to be used as limits on the amount of work the Varnish instance
    should handle.
    
    This can be especially important for instances with H/2 enabled, as these
    connections will be holding threads for extended periods of time, possibly
    triggering the watchdog in benign situations.
    
    This patch limits the watchdog to only trigger for no queue development
    on the highest priority queue.

diff --git a/bin/varnishd/cache/cache.h b/bin/varnishd/cache/cache.h
index 7fba1e6f2..d80877e76 100644
--- a/bin/varnishd/cache/cache.h
+++ b/bin/varnishd/cache/cache.h
@@ -220,6 +220,7 @@ enum task_prio {
 	TASK_QUEUE__END
 };
 
+#define TASK_QUEUE_HIGHEST_PRIORITY TASK_QUEUE_BO
 #define TASK_QUEUE_CLIENT(prio) \
 	(prio == TASK_QUEUE_REQ || prio == TASK_QUEUE_STR)
 
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index 3d31072d0..090ada6c3 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -600,7 +600,9 @@ pool_herder(void *priv)
 		 * Instead we implement a watchdog and kill the worker if
 		 * nothing has been dequeued for that long.
 		 */
-		if (pp->lqueue == 0) {
+		if (VTAILQ_EMPTY(&pp->queues[TASK_QUEUE_HIGHEST_PRIORITY])) {
+			/* Watchdog only applies to no movement on the
+			 * highest priority queue (TASK_QUEUE_BO) */
 			dq = pp->ndequeued + 1;
 		} else if (dq != pp->ndequeued) {
 			dq = pp->ndequeued;
diff --git a/bin/varnishtest/tests/c00104.vtc b/bin/varnishtest/tests/c00104.vtc
new file mode 100644
index 000000000..3af498179
--- /dev/null
+++ b/bin/varnishtest/tests/c00104.vtc
@@ -0,0 +1,32 @@
+varnishtest "Test watchdog only active on queue 0"
+
+server s1 {
+	rxreq
+	txresp
+} -start
+
+varnish v1 -cliok "param.set thread_pools 1"
+varnish v1 -cliok "param.set thread_pool_min 5"
+varnish v1 -cliok "param.set thread_pool_max 5"
+varnish v1 -cliok "param.set thread_pool_watchdog 1"
+varnish v1 -cliok "param.set feature +http2"
+
+varnish v1 -vcl+backend {
+} -start
+
+client c1 {
+	txpri
+	delay 2
+} -start
+
+client c2 {
+	txpri
+	delay 2
+} -start
+
+client c3 {
+	txpri
+	delay 2
+} -start
+
+delay 2