[master] 656982a Some experiemental code to deconfigure a thread pool.
Poul-Henning Kamp
phk at FreeBSD.org
Tue Feb 21 00:06:05 CET 2017
commit 656982a5cf7042a8576c2e9d48defcd25d749fbd
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date: Mon Feb 20 23:03:35 2017 +0000
Some experiemental code to deconfigure a thread pool.
The fundamental problem still exists, in that we have no way to
tickle the threads stuck in accept(2), so for now this remains
a debugging feature.
If this (=c00080.vtc) explodes on some platform, start looking
at the Waiter_Destroy() implementation.
diff --git a/bin/varnishd/cache/cache_acceptor.c b/bin/varnishd/cache/cache_acceptor.c
index d764d1b..b570db2 100644
--- a/bin/varnishd/cache/cache_acceptor.c
+++ b/bin/varnishd/cache/cache_acceptor.c
@@ -68,6 +68,7 @@ struct wrk_accept {
struct poolsock {
unsigned magic;
#define POOLSOCK_MAGIC 0x1b0a2d38
+ VTAILQ_ENTRY(poolsock) list;
struct listen_sock *lsock;
struct pool_task task;
struct pool *pool;
@@ -402,7 +403,7 @@ vca_accept_task(struct worker *wrk, void *arg)
while (!pool_accepting)
VTIM_sleep(.1);
- while (1) {
+ while (!ps->pool->die) {
INIT_OBJ(&wa, WRK_ACCEPT_MAGIC);
wa.acceptlsock = ls;
@@ -414,6 +415,12 @@ vca_accept_task(struct worker *wrk, void *arg)
&wa.acceptaddrlen);
} while (i < 0 && errno == EAGAIN);
+ if (i < 0 && ps->pool->die) {
+ VSL(SLT_Debug, 0, "XXX Accept thread dies %p", ps);
+ FREE_OBJ(ps);
+ return;
+ }
+
if (i < 0 && ls->sock == -2) {
/* Shut down in progress */
sleep(2);
@@ -454,7 +461,9 @@ vca_accept_task(struct worker *wrk, void *arg)
* must reschedule the listening task so it will be
* taken up by another thread again.
*/
- AZ(Pool_Task(wrk->pool, &ps->task, TASK_QUEUE_VCA));
+ if (!ps->pool->die)
+ AZ(Pool_Task(wrk->pool, &ps->task,
+ TASK_QUEUE_VCA));
return;
}
@@ -485,10 +494,22 @@ VCA_NewPool(struct pool *pp)
ps->task.func = vca_accept_task;
ps->task.priv = ps;
ps->pool = pp;
+ VTAILQ_INSERT_TAIL(&pp->poolsocks, ps, list);
AZ(Pool_Task(pp, &ps->task, TASK_QUEUE_VCA));
}
}
+void
+VCA_DestroyPool(struct pool *pp)
+{
+ struct poolsock *ps;
+
+ while (!VTAILQ_EMPTY(&pp->poolsocks)) {
+ ps = VTAILQ_FIRST(&pp->poolsocks);
+ VTAILQ_REMOVE(&pp->poolsocks, ps, list);
+ }
+}
+
/*--------------------------------------------------------------------*/
static void *
diff --git a/bin/varnishd/cache/cache_pool.c b/bin/varnishd/cache/cache_pool.c
index be26db4..57b7e8d 100644
--- a/bin/varnishd/cache/cache_pool.c
+++ b/bin/varnishd/cache/cache_pool.c
@@ -161,6 +161,7 @@ pool_mkpool(unsigned pool_no)
Lck_New(&pp->mtx, lck_wq);
VTAILQ_INIT(&pp->idle_queue);
+ VTAILQ_INIT(&pp->poolsocks);
for (i = 0; i < TASK_QUEUE_END; i++)
VTAILQ_INIT(&pp->queues[i]);
AZ(pthread_cond_init(&pp->herder_cond, NULL));
@@ -187,8 +188,9 @@ static void *
pool_poolherder(void *priv)
{
unsigned nwq;
- struct pool *pp;
+ struct pool *pp, *ppx;
uint64_t u;
+ void *rvp;
THR_SetName("pool_poolherder");
(void)priv;
@@ -205,23 +207,42 @@ pool_poolherder(void *priv)
nwq++;
continue;
}
- }
- /* XXX: remove pools */
- if (0) {
+ } else if (nwq > cache_param->wthread_pools &&
+ DO_DEBUG(DBG_DROP_POOLS)) {
Lck_Lock(&pool_mtx);
pp = VTAILQ_FIRST(&pools);
+ AN(pp);
VTAILQ_REMOVE(&pools, pp, list);
+ VTAILQ_INSERT_TAIL(&pools, pp, list);
+ if (!pp->die)
+ nwq--;
Lck_Unlock(&pool_mtx);
- AN(pp);
- MPL_Destroy(&pp->mpl_sess);
- MPL_Destroy(&pp->mpl_req);
- INCOMPL();
+ if (!pp->die) {
+ VSL(SLT_Debug, 0, "XXX Kill Pool %p", pp);
+ pp->die = 1;
+ VCA_DestroyPool(pp);
+ AZ(pthread_cond_signal(&pp->herder_cond));
+ }
}
(void)sleep(1);
u = 0;
+ ppx = NULL;
Lck_Lock(&pool_mtx);
- VTAILQ_FOREACH(pp, &pools, list)
+ VTAILQ_FOREACH(pp, &pools, list) {
+ if (pp->die && pp->nthr == 0)
+ ppx = pp;
u += pp->lqueue;
+ }
+ if (ppx != NULL) {
+ VTAILQ_REMOVE(&pools, ppx, list);
+ AZ(pthread_join(ppx->herder_thr, &rvp));
+ AZ(pthread_cond_destroy(&ppx->herder_cond));
+ free(ppx->a_stat);
+ free(ppx->b_stat);
+ SES_DestroyPool(ppx);
+ FREE_OBJ(ppx);
+ VSC_C_main->pools--;
+ }
Lck_Unlock(&pool_mtx);
VSC_C_main->thread_queue_len = u;
}
diff --git a/bin/varnishd/cache/cache_pool.h b/bin/varnishd/cache/cache_pool.h
index 83359ce..98e43ca 100644
--- a/bin/varnishd/cache/cache_pool.h
+++ b/bin/varnishd/cache/cache_pool.h
@@ -31,11 +31,15 @@
VTAILQ_HEAD(taskhead, pool_task);
+struct poolsock;
+
struct pool {
unsigned magic;
#define POOL_MAGIC 0x606658fa
VTAILQ_ENTRY(pool) list;
+ VTAILQ_HEAD(,poolsock) poolsocks;
+ int die;
pthread_cond_t herder_cond;
pthread_t herder_thr;
@@ -59,4 +63,5 @@ struct pool {
void *pool_herder(void*);
task_func_t pool_stat_summ;
extern struct lock pool_mtx;
-void VCA_NewPool(struct pool *pp);
+void VCA_NewPool(struct pool *);
+void VCA_DestroyPool(struct pool *);
diff --git a/bin/varnishd/cache/cache_priv.h b/bin/varnishd/cache/cache_priv.h
index dd88025..abf5e7f 100644
--- a/bin/varnishd/cache/cache_priv.h
+++ b/bin/varnishd/cache/cache_priv.h
@@ -99,6 +99,7 @@ task_func_t VPX_Proto_Sess;
/* cache_session.c */
void SES_NewPool(struct pool *, unsigned pool_no);
+void SES_DestroyPool(struct pool *);
/* cache_shmlog.c */
void VSM_Init(void);
diff --git a/bin/varnishd/cache/cache_session.c b/bin/varnishd/cache/cache_session.c
index dc671ea..1c0c6d7 100644
--- a/bin/varnishd/cache/cache_session.c
+++ b/bin/varnishd/cache/cache_session.c
@@ -600,3 +600,11 @@ SES_NewPool(struct pool *pp, unsigned pool_no)
pp->waiter = Waiter_New();
}
+
+void
+SES_DestroyPool(struct pool *pp)
+{
+ MPL_Destroy(&pp->mpl_req);
+ MPL_Destroy(&pp->mpl_sess);
+ Waiter_Destroy(&pp->waiter);
+}
diff --git a/bin/varnishd/cache/cache_wrk.c b/bin/varnishd/cache/cache_wrk.c
index e28d993..f3dfc3b 100644
--- a/bin/varnishd/cache/cache_wrk.c
+++ b/bin/varnishd/cache/cache_wrk.c
@@ -465,14 +465,17 @@ pool_herder(void *priv)
struct pool_task *pt;
double t_idle;
struct worker *wrk;
- int delay, wthread_min;
+ double delay;
+ int wthread_min;
CAST_OBJ_NOTNULL(pp, priv, POOL_MAGIC);
THR_SetName("pool_herder");
- while (1) {
+ while (!pp->die || pp->nthr > 0) {
wthread_min = cache_param->wthread_min;
+ if (pp->die)
+ wthread_min = 0;
/* Make more threads if needed and allowed */
if (pp->nthr < wthread_min ||
@@ -501,7 +504,7 @@ pool_herder(void *priv)
AZ(pt->func);
CAST_OBJ_NOTNULL(wrk, pt->priv, WORKER_MAGIC);
- if (wrk->lastused < t_idle ||
+ if (pp->die || wrk->lastused < t_idle ||
pp->nthr > cache_param->wthread_max) {
/* Give it a kiss on the cheek... */
VTAILQ_REMOVE(&pp->idle_queue,
@@ -527,6 +530,14 @@ pool_herder(void *priv)
delay = cache_param->wthread_destroy_delay;
}
+ if (pp->die) {
+ if (delay < 2)
+ delay = 10e-3;
+ else
+ delay = 1;
+ VTIM_sleep(delay);
+ continue;
+ }
Lck_Lock(&pp->mtx);
if (!pp->dry) {
(void)Lck_CondWait(&pp->herder_cond, &pp->mtx,
@@ -538,5 +549,5 @@ pool_herder(void *priv)
}
Lck_Unlock(&pp->mtx);
}
- NEEDLESS(return NULL);
+ return (NULL);
}
diff --git a/bin/varnishtest/tests/c00080.vtc b/bin/varnishtest/tests/c00080.vtc
new file mode 100644
index 0000000..ce96a55
--- /dev/null
+++ b/bin/varnishtest/tests/c00080.vtc
@@ -0,0 +1,36 @@
+varnishtest "Deconfigure thread pool"
+
+server s1 {
+ rxreq
+ txresp
+} -start
+
+varnish v1 -vcl+backend {} -start
+
+varnish v1 -cliok "param.set debug +drop_pools"
+varnish v1 -cliok "param.set thread_pools 1"
+
+delay 2
+
+client c1 {
+ txreq
+ rxresp
+} -run
+
+varnish v1 -vsc *poo*
+
+delay 2
+
+client c1 {
+ txreq
+ rxresp
+} -run
+
+varnish v1 -vsc *poo*
+varnish v1 -expect MAIN.pools == 1
+
+client c1 {
+ txreq
+ rxresp
+} -run
+
diff --git a/bin/varnishtest/vtc_varnish.c b/bin/varnishtest/vtc_varnish.c
index 9616260..e0dce91 100644
--- a/bin/varnishtest/vtc_varnish.c
+++ b/bin/varnishtest/vtc_varnish.c
@@ -823,9 +823,9 @@ do_stat_dump_cb(void *priv, const struct VSC_point * const pt)
strcpy(buf, pt->section->type);
if (pt->section->ident[0] != '\0')
- bprintf(buf, ".%s.%s", pt->section->ident, pt->desc->name);
+ bprintf(buf, "%s.%s", pt->section->ident, pt->desc->name);
else
- bprintf(buf, ".%s", pt->desc->name);
+ bprintf(buf, "MAIN.%s", pt->desc->name);
if (strcmp(dp->arg, "*")) {
if (fnmatch(dp->arg, buf, 0))
More information about the varnish-commit
mailing list