r4216 - in trunk/varnish-cache: bin/varnishd include
phk at projects.linpro.no
phk at projects.linpro.no
Fri Aug 28 17:25:12 CEST 2009
Author: phk
Date: 2009-08-28 17:25:11 +0200 (Fri, 28 Aug 2009)
New Revision: 4216
Modified:
trunk/varnish-cache/bin/varnishd/cache.h
trunk/varnish-cache/bin/varnishd/cache_center.c
trunk/varnish-cache/bin/varnishd/cache_hash.c
trunk/varnish-cache/bin/varnishd/cache_pool.c
trunk/varnish-cache/bin/varnishd/cache_session.c
trunk/varnish-cache/bin/varnishd/hash_critbit.c
trunk/varnish-cache/bin/varnishd/hash_slinger.h
trunk/varnish-cache/bin/varnishd/heritage.h
trunk/varnish-cache/bin/varnishd/mgt_pool.c
trunk/varnish-cache/include/stat_field.h
Log:
More stats work.
Add a parameter, so the worker thread gets forced to dump stats
into the global pool for every N requests it serves.
This should solve the problem where a wget(1) client does not
show up in the stats, until the session ends.
Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h 2009-08-28 14:44:02 UTC (rev 4215)
+++ trunk/varnish-cache/bin/varnishd/cache.h 2009-08-28 15:25:11 UTC (rev 4216)
@@ -199,7 +199,7 @@
#define WORKER_MAGIC 0x6391adcf
struct objhead *nobjhead;
struct objcore *nobjcore;
- struct dstat *stats;
+ struct dstat stats;
double lastused;
@@ -592,7 +592,7 @@
void WRK_Init(void);
int WRK_Queue(struct workreq *wrq);
void WRK_QueueSession(struct sess *sp);
-void WRK_SumStat(const struct worker *w);
+void WRK_SumStat(struct worker *w);
void WRW_Reserve(struct worker *w, int *fd);
unsigned WRW_Flush(struct worker *w);
Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-08-28 14:44:02 UTC (rev 4215)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-08-28 15:25:11 UTC (rev 4216)
@@ -105,7 +105,7 @@
i = poll(pfd, 1, params->session_linger);
if (i == 0) {
WSL(sp->wrk, SLT_Debug, sp->fd, "herding");
- sp->wrk->stats->sess_herd++;
+ sp->wrk->stats.sess_herd++;
SES_Charge(sp);
sp->wrk = NULL;
vca_return_session(sp);
@@ -271,35 +271,38 @@
TCP_linger(sp->fd, 0);
vca_close_session(sp, sp->doclose);
}
+
+ SES_Charge(sp);
+
if (sp->fd < 0) {
- sp->wrk->stats->sess_closed++;
- SES_Charge(sp);
+ sp->wrk->stats.sess_closed++;
sp->wrk = NULL;
SES_Delete(sp);
return (1);
}
+ if (sp->wrk->stats.client_req >= params->wthread_stats_rate)
+ WRK_SumStat(sp->wrk);
/* Reset the workspace to the session-watermark */
WS_Reset(sp->ws, sp->ws_ses);
i = HTC_Reinit(sp->htc);
if (i == 1) {
- sp->wrk->stats->sess_pipeline++;
+ sp->wrk->stats.sess_pipeline++;
sp->step = STP_START;
return (0);
}
if (Tlen(sp->htc->rxbuf)) {
- sp->wrk->stats->sess_readahead++;
+ sp->wrk->stats.sess_readahead++;
sp->step = STP_WAIT;
return (0);
}
if (params->session_linger > 0) {
- sp->wrk->stats->sess_linger++;
+ sp->wrk->stats.sess_linger++;
sp->step = STP_WAIT;
return (0);
}
- sp->wrk->stats->sess_herd++;
- SES_Charge(sp);
+ sp->wrk->stats.sess_herd++;
sp->wrk = NULL;
vca_return_session(sp);
return (1);
@@ -757,7 +760,7 @@
/* If we inserted a new object it's a miss */
if (oc->flags & OC_F_BUSY) {
- sp->wrk->stats->cache_miss++;
+ sp->wrk->stats.cache_miss++;
AZ(oc->obj);
sp->objhead = oh;
@@ -771,7 +774,7 @@
sp->obj = o;
if (oc->flags & OC_F_PASS) {
- sp->wrk->stats->cache_hitpass++;
+ sp->wrk->stats.cache_hitpass++;
WSP(sp, SLT_HitPass, "%u", sp->obj->xid);
HSH_Deref(sp->wrk, &sp->obj);
sp->objcore = NULL;
@@ -780,7 +783,7 @@
return (0);
}
- sp->wrk->stats->cache_hit++;
+ sp->wrk->stats.cache_hit++;
WSP(sp, SLT_Hit, "%u", sp->obj->xid);
sp->step = STP_HIT;
return (0);
@@ -1049,7 +1052,7 @@
AZ(sp->vcl);
/* Update stats of various sorts */
- sp->wrk->stats->client_req++;
+ sp->wrk->stats.client_req++;
sp->t_req = TIM_real();
sp->wrk->lastused = sp->t_req;
sp->acct_req.req++;
Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-28 14:44:02 UTC (rev 4215)
+++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-28 15:25:11 UTC (rev 4216)
@@ -117,7 +117,7 @@
o->entered = NAN;
VTAILQ_INIT(&o->store);
VTAILQ_INIT(&o->esibits);
- sp->wrk->stats->n_object++;
+ sp->wrk->stats.n_object++;
return (o);
}
@@ -164,7 +164,7 @@
VTAILQ_INIT(&oh->waitinglist);
Lck_New(&oh->mtx);
w->nobjhead = oh;
- w->stats->n_objecthead++;
+ w->stats.n_objecthead++;
}
CHECK_OBJ_NOTNULL(w->nobjhead, OBJHEAD_MAGIC);
@@ -182,18 +182,18 @@
Lck_Delete(&w->nobjhead->mtx);
FREE_OBJ(w->nobjhead);
w->nobjhead = NULL;
- w->stats->n_objecthead--;
+ w->stats.n_objecthead--;
}
}
void
-HSH_DeleteObjHead(const struct worker *w, struct objhead *oh)
+HSH_DeleteObjHead(struct worker *w, struct objhead *oh)
{
AZ(oh->refcnt);
assert(VTAILQ_EMPTY(&oh->objcs));
Lck_Delete(&oh->mtx);
- w->stats->n_objecthead--;
+ w->stats.n_objecthead--;
free(oh->hash);
FREE_OBJ(oh);
}
@@ -409,7 +409,7 @@
/* NB: do not deref objhead the new object inherits our reference */
oc->objhead = oh;
Lck_Unlock(&oh->mtx);
- sp->wrk->stats->n_object++;
+ sp->wrk->stats.n_object++;
return (oc);
}
@@ -608,7 +608,7 @@
assert(o->refcnt > 0);
assert(oh->refcnt > 0);
if (o->ws_o->overflow)
- VSL_stats->n_objoverflow++;
+ sp->wrk->stats.n_objoverflow++;
if (params->diag_bitmap & 0x40)
WSP(sp, SLT_Debug,
"Object %u workspace free %u", o->xid, WS_Free(o->ws_o));
@@ -651,7 +651,7 @@
Lck_Lock(&oh->mtx);
VTAILQ_REMOVE(&oh->objcs, oc, list);
- sp->wrk->stats->n_object--;
+ sp->wrk->stats.n_object--;
Lck_Unlock(&oh->mtx);
assert(oh->refcnt > 0);
FREE_OBJ(oc);
@@ -694,7 +694,7 @@
}
void
-HSH_Deref(const struct worker *w, struct object **oo)
+HSH_Deref(struct worker *w, struct object **oo)
{
struct object *o;
struct objhead *oh;
@@ -749,7 +749,7 @@
FREE_OBJ(o);
}
o = NULL;
- w->stats->n_object--;
+ w->stats.n_object--;
if (oc == NULL) {
AZ(oh);
Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_pool.c 2009-08-28 14:44:02 UTC (rev 4215)
+++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2009-08-28 15:25:11 UTC (rev 4216)
@@ -91,23 +91,24 @@
/*--------------------------------------------------------------------*/
static void
-wrk_sumstat(const struct worker *w)
+wrk_sumstat(struct worker *w)
{
Lck_AssertHeld(&wstat_mtx);
#define L0(n)
-#define L1(n) (VSL_stats->n += w->stats->n)
+#define L1(n) (VSL_stats->n += w->stats.n)
#define MAC_STAT(n, t, l, f, d) L##l(n);
#include "stat_field.h"
#undef MAC_STAT
#undef L0
#undef L1
- memset(w->stats, 0, sizeof *w->stats);
+ memset(&w->stats, 0, sizeof w->stats);
}
void
-WRK_SumStat(const struct worker *w)
+WRK_SumStat(struct worker *w)
{
+
Lck_Lock(&wstat_mtx);
wrk_sumstat(w);
Lck_Unlock(&wstat_mtx);
@@ -122,15 +123,12 @@
unsigned char wlog[shm_workspace];
unsigned char ws[sess_workspace];
struct SHA256Context sha256;
- struct dstat stats;
- unsigned stats_clean = 0;
+ int stats_clean;
THR_SetName("cache-worker");
w = &ww;
memset(w, 0, sizeof *w);
- memset(&stats, 0, sizeof stats);
w->magic = WORKER_MAGIC;
- w->stats = &stats;
w->lastused = NAN;
w->wlb = w->wlp = wlog;
w->wle = wlog + sizeof wlog;
@@ -143,6 +141,7 @@
Lck_Lock(&qp->mtx);
qp->nthr++;
+ stats_clean = 1;
while (1) {
CHECK_OBJ_NOTNULL(w, WORKER_MAGIC);
@@ -155,19 +154,17 @@
if (isnan(w->lastused))
w->lastused = TIM_real();
VTAILQ_INSERT_HEAD(&qp->idle, w, list);
- if (!stats_clean) {
+ if (!stats_clean)
WRK_SumStat(w);
- stats_clean = 1;
- }
Lck_CondWait(&w->cond, &qp->mtx);
}
if (w->wrq == NULL)
break;
Lck_Unlock(&qp->mtx);
+ stats_clean = 0;
AN(w->wrq);
AN(w->wrq->func);
w->lastused = NAN;
- stats_clean = 0;
WS_Reset(w->ws, NULL);
w->bereq = NULL;
w->beresp1 = NULL;
@@ -184,14 +181,13 @@
w->wrq = NULL;
if (!Lck_Trylock(&wstat_mtx)) {
wrk_sumstat(w);
- stats_clean = 1;
Lck_Unlock(&wstat_mtx);
+ stats_clean = 1;
}
Lck_Lock(&qp->mtx);
}
qp->nthr--;
Lck_Unlock(&qp->mtx);
- AN(stats_clean);
VSL(SLT_WorkThread, 0, "%p end", w);
if (w->vcl != NULL)
@@ -524,19 +520,16 @@
struct worker ww;
struct sess *sp;
unsigned char logbuf[1024]; /* XXX: size ? */
- struct dstat stats;
CAST_OBJ_NOTNULL(bt, arg, BGTHREAD_MAGIC);
THR_SetName(bt->name);
sp = SES_Alloc(NULL, 0);
XXXAN(sp);
memset(&ww, 0, sizeof ww);
- memset(&stats, 0, sizeof stats);
sp->wrk = &ww;
ww.magic = WORKER_MAGIC;
ww.wlp = ww.wlb = logbuf;
ww.wle = logbuf + sizeof logbuf;
- ww.stats = &stats;
(void)bt->func(sp, bt->priv);
Modified: trunk/varnish-cache/bin/varnishd/cache_session.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_session.c 2009-08-28 14:44:02 UTC (rev 4215)
+++ trunk/varnish-cache/bin/varnishd/cache_session.c 2009-08-28 15:25:11 UTC (rev 4216)
@@ -86,7 +86,7 @@
struct acct *a = &sp->acct_req;
#define ACCT(foo) \
- sp->wrk->stats->s_##foo += a->foo; \
+ sp->wrk->stats.s_##foo += a->foo; \
sp->acct.foo += a->foo; \
a->foo = 0;
#include "acct_fields.h"
Modified: trunk/varnish-cache/bin/varnishd/hash_critbit.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/hash_critbit.c 2009-08-28 14:44:02 UTC (rev 4215)
+++ trunk/varnish-cache/bin/varnishd/hash_critbit.c 2009-08-28 15:25:11 UTC (rev 4216)
@@ -347,12 +347,9 @@
struct objhead *oh, *oh2;
struct hcb_y *y;
struct worker ww;
- struct dstat stats;
memset(&ww, 0, sizeof ww);
- memset(&stats, 0, sizeof stats);
ww.magic = WORKER_MAGIC;
- ww.stats = &stats;
THR_SetName("hcb_cleaner");
(void)priv;
Modified: trunk/varnish-cache/bin/varnishd/hash_slinger.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/hash_slinger.h 2009-08-28 14:44:02 UTC (rev 4215)
+++ trunk/varnish-cache/bin/varnishd/hash_slinger.h 2009-08-28 15:25:11 UTC (rev 4216)
@@ -105,8 +105,8 @@
};
extern unsigned save_hash;
-void HSH_DeleteObjHead(const struct worker *w, struct objhead *oh);
-void HSH_Deref(const struct worker *w, struct object **o);
+void HSH_DeleteObjHead(struct worker *w, struct objhead *oh);
+void HSH_Deref(struct worker *w, struct object **o);
#endif /* VARNISH_CACHE_CHILD */
extern struct hash_slinger hsl_slinger;
Modified: trunk/varnish-cache/bin/varnishd/heritage.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/heritage.h 2009-08-28 14:44:02 UTC (rev 4215)
+++ trunk/varnish-cache/bin/varnishd/heritage.h 2009-08-28 15:25:11 UTC (rev 4216)
@@ -92,6 +92,7 @@
unsigned wthread_add_delay;
unsigned wthread_fail_delay;
unsigned wthread_purge_delay;
+ unsigned wthread_stats_rate;
unsigned overflow_max;
Modified: trunk/varnish-cache/bin/varnishd/mgt_pool.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_pool.c 2009-08-28 14:44:02 UTC (rev 4215)
+++ trunk/varnish-cache/bin/varnishd/mgt_pool.c 2009-08-28 15:25:11 UTC (rev 4216)
@@ -165,6 +165,16 @@
"destroyed and later recreated.\n",
EXPERIMENTAL,
"200", "milliseconds" },
+ { "thread_stats_rate",
+ tweak_uint, &master.wthread_stats_rate, 0, UINT_MAX,
+ "Worker threads accumulate statistics, and dump these into "
+ "the global stats counters if the lock is free when they "
+ "finish a request.\n"
+ "This parameters defines the maximum number of requests "
+ "a worker thread may handle, before it is forced to dump "
+ "its accumulated stats into the global counters.\n",
+ EXPERIMENTAL,
+ "10", "requests" },
{ "overflow_max", tweak_uint, &master.overflow_max, 0, UINT_MAX,
"Percentage permitted overflow queue length.\n"
"\n"
Modified: trunk/varnish-cache/include/stat_field.h
===================================================================
--- trunk/varnish-cache/include/stat_field.h 2009-08-28 14:44:02 UTC (rev 4215)
+++ trunk/varnish-cache/include/stat_field.h 2009-08-28 15:25:11 UTC (rev 4216)
@@ -76,7 +76,7 @@
MAC_STAT(n_objsendfile, uint64_t, 0, 'a', "Objects sent with sendfile")
MAC_STAT(n_objwrite, uint64_t, 0, 'a', "Objects sent with write")
-MAC_STAT(n_objoverflow, uint64_t, 0, 'a', "Objects overflowing workspace")
+MAC_STAT(n_objoverflow, uint64_t, 1, 'a', "Objects overflowing workspace")
MAC_STAT(s_sess, uint64_t, 1, 'a', "Total Sessions")
MAC_STAT(s_req, uint64_t, 1, 'a', "Total Requests")
More information about the varnish-commit
mailing list