[master] 687273347 pool: Wait when we run out of workers

[Dridi Boukelmoune]

commit 6872733475228890799a94e62b689d88e3fa911e
Author: Dridi Boukelmoune <dridi.boukelmoune at gmail.com>
Date:   Thu Feb 11 12:21:33 2021 +0100

    pool: Wait when we run out of workers
    
    Since the removal of dry signals, pools will spin when they run out of
    threads and increment MAIN.threads_limited at a very high rate. That
    spike in CPU consumption will also have detrimental effects on useful
    tasks.
    
    This change introduces a 1s delay when the pool is saturated. This
    allows to periodically exercise the watchdog check. We could wait until
    the watchdog times out but we would also miss potential updates to the
    thread_pool_watchdog parameter.
    
    To avoid spurious increments of MAIN.threads_limited we only take task
    submissions into account and ignore timeouts of the condvar.
    
    Refs #2942
    Refs #3531

diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index dcedcd344..3d31072d0 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -579,6 +579,7 @@ pool_herder(void *priv)
 	unsigned wthread_min;
 	uintmax_t dq = (1ULL << 31);
 	vtim_mono dqt = 0;
+	int r = 0;
 
 	CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
 
@@ -680,11 +681,15 @@ pool_herder(void *priv)
 		if (pp->lqueue == 0) {
 			if (DO_DEBUG(DBG_VTC_MODE))
 				delay = 0.5;
-			(void)Lck_CondWait(&pp->herder_cond, &pp->mtx,
-				VTIM_real() + delay);
-		} else if (pp->nthr >= cache_param->wthread_max)
+			r = Lck_CondWait(&pp->herder_cond, &pp->mtx,
+			    VTIM_real() + delay);
+		} else if (pp->nthr >= cache_param->wthread_max) {
 			/* XXX: unsafe counters */
-			VSC_C_main->threads_limited++;
+			if (r != ETIMEDOUT)
+				VSC_C_main->threads_limited++;
+			r = Lck_CondWait(&pp->herder_cond, &pp->mtx,
+			    VTIM_real() + 1.0);
+		}
 		Lck_Unlock(&pp->mtx);
 	}
 	return (NULL);