r4206 - in trunk/varnish-cache: bin/varnishd bin/varnishtest/tests include lib/libvcl
phk at projects.linpro.no
phk at projects.linpro.no
Wed Aug 19 23:04:03 CEST 2009
Author: phk
Date: 2009-08-19 23:04:03 +0200 (Wed, 19 Aug 2009)
New Revision: 4206
Modified:
trunk/varnish-cache/bin/varnishd/cache_ban.c
trunk/varnish-cache/bin/varnishd/cache_ban.h
trunk/varnish-cache/bin/varnishd/cache_hash.c
trunk/varnish-cache/bin/varnishd/hash_slinger.h
trunk/varnish-cache/bin/varnishd/heritage.h
trunk/varnish-cache/bin/varnishd/mgt_param.c
trunk/varnish-cache/bin/varnishd/storage_persistent.c
trunk/varnish-cache/bin/varnishtest/tests/v00006.vtc
trunk/varnish-cache/include/purge_vars.h
trunk/varnish-cache/lib/libvcl/vcc_action.c
Log:
Add an experimental thread to lurk at the bottom of the ban-list
and try to push objects away from the tail end so that the list
can be shortened.
It won't work if your bans contains any req.* tests, because the
lurker has no request to compare against.
By default the lurker is disabled, you enable it by setting the
'ban_lurker_sleep' parameter to the amount of time the lurker
should sleep between doing things.
If it cannot do anything, it always sleeps for a second.
NB: This feature is on probation, there is currently no guarantee
it will end up in a future varnish release, it depends on the
impact and usability. Feedback much anticipated.
One of the quasi-nasty details of this feature, is the need to work
against the lock-order in cache_hash.c, see details in comments for
HSH_FindBan().
Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-08-19 18:53:46 UTC (rev 4205)
+++ trunk/varnish-cache/bin/varnishd/cache_ban.c 2009-08-19 21:04:03 UTC (rev 4206)
@@ -63,6 +63,7 @@
static VTAILQ_HEAD(banhead,ban) ban_head = VTAILQ_HEAD_INITIALIZER(ban_head);
static struct lock ban_mtx;
static struct ban *ban_magic;
+static pthread_t ban_thread;
/*--------------------------------------------------------------------
* Manipulation of bans
@@ -285,6 +286,8 @@
if (strncmp(a1, pv->name, strlen(pv->name)))
continue;
bt->func = pv->func;
+ if (pv->flag & 2)
+ b->flags |= BAN_F_REQ;
if (pv->flag & 1)
ban_parse_http(bt, a1 + strlen(pv->name));
break;
@@ -419,8 +422,8 @@
}
-int
-BAN_CheckObject(struct object *o, const struct sess *sp)
+static int
+ban_check_object(struct object *o, const struct sess *sp, int has_req)
{
struct ban *b;
struct ban_test *bt;
@@ -445,6 +448,8 @@
for (b = b0; b != o->ban; b = VTAILQ_NEXT(b, list)) {
if (b->flags & BAN_F_GONE)
continue;
+ if (!has_req && (b->flags & BAN_F_REQ))
+ return (0);
VTAILQ_FOREACH(bt, &b->tests, list) {
tests++;
if (bt->func(bt, o, sp))
@@ -484,7 +489,69 @@
}
}
+int
+BAN_CheckObject(struct object *o, const struct sess *sp)
+{
+
+ return ban_check_object(o, sp, 1);
+}
+
/*--------------------------------------------------------------------
+ * Ban tail lurker thread
+ */
+
+static void *
+ban_lurker(struct sess *sp, void *priv)
+{
+ struct ban *b, *bf;
+ struct objcore *oc;
+ struct object *o;
+
+ (void)priv;
+ while (1) {
+ WSL_Flush(sp->wrk, 0);
+ WRK_SumStat(sp->wrk);
+ if (params->ban_lurker_sleep == 0.0) {
+ AZ(sleep(1));
+ continue;
+ }
+ Lck_Lock(&ban_mtx);
+
+ /* First try to route the last ban */
+ bf = BAN_CheckLast();
+ if (bf != NULL) {
+ Lck_Unlock(&ban_mtx);
+ BAN_Free(bf);
+ TIM_sleep(params->ban_lurker_sleep);
+ continue;
+ }
+ /* Then try to poke the first object on the last ban */
+ oc = NULL;
+ while (1) {
+ b = VTAILQ_LAST(&ban_head, banhead);
+ if (b == ban_start)
+ break;
+ oc = VTAILQ_FIRST(&b->objcore);
+ if (oc == NULL)
+ break;
+ HSH_FindBan(sp, &oc);
+ break;
+ }
+ Lck_Unlock(&ban_mtx);
+ if (oc == NULL) {
+ TIM_sleep(1.0);
+ continue;
+ }
+ o = oc->obj;
+ (void)ban_check_object(o, sp, 0);
+ WSP(sp, SLT_Debug, "lurker: %p %g", oc, o->ttl);
+ HSH_Deref(sp->wrk, &o);
+ TIM_sleep(params->ban_lurker_sleep);
+ }
+}
+
+
+/*--------------------------------------------------------------------
* Release a reference
*/
@@ -767,4 +834,5 @@
AN(ban_magic);
ban_magic->flags |= BAN_F_GONE;
BAN_Insert(ban_magic);
+ WRK_BgThread(&ban_thread, "ban-lurker", ban_lurker, NULL);
}
Modified: trunk/varnish-cache/bin/varnishd/cache_ban.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_ban.h 2009-08-19 18:53:46 UTC (rev 4205)
+++ trunk/varnish-cache/bin/varnishd/cache_ban.h 2009-08-19 21:04:03 UTC (rev 4206)
@@ -57,6 +57,7 @@
int flags;
#define BAN_F_GONE (1 << 0)
#define BAN_F_PENDING (1 << 1)
+#define BAN_F_REQ (1 << 2)
VTAILQ_HEAD(,ban_test) tests;
VTAILQ_HEAD(,objcore) objcore;
double t0;
Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-19 18:53:46 UTC (rev 4205)
+++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-08-19 21:04:03 UTC (rev 4206)
@@ -652,7 +652,40 @@
HSH_DeleteObjHead(sp->wrk, oh);
}
+/*******************************************************************
+ * This one is slightly tricky. This is called from the BAN module
+ * to try to wash the object which holds the oldest ban.
+ * We compete against HSH_Deref() which comes in the opposite
+ * locking order, we need to hold the BAN mutex, to stop the
+ * BAN_DestroyObj() call in HSH_Deref(), so that the objhead
+ * will not be removed under us.
+ * NB: Do not call this function any other way or from any other
+ * NB: place in the code. It will not work for you.
+ */
+
void
+HSH_FindBan(struct sess *sp, struct objcore **oc)
+{
+ struct objcore *oc1, *oc2;
+ struct objhead *oh;
+
+ oc1 = *oc;
+ CHECK_OBJ_NOTNULL(oc1, OBJCORE_MAGIC);
+ oh = oc1->objhead;
+ CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+ Lck_Lock(&oh->mtx);
+ VTAILQ_FOREACH(oc2, &oh->objcs, list)
+ if (oc1 == oc2) {
+ oc1->obj->refcnt++;
+ break;
+ }
+ if (oc2 != NULL && oc2->flags & OC_F_PERSISTENT)
+ SMP_Fixup(sp, oh, oc2);
+ Lck_Unlock(&oc1->objhead->mtx);
+ *oc = oc2;
+}
+
+void
HSH_Deref(const struct worker *w, struct object **oo)
{
struct object *o;
Modified: trunk/varnish-cache/bin/varnishd/hash_slinger.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/hash_slinger.h 2009-08-19 18:53:46 UTC (rev 4205)
+++ trunk/varnish-cache/bin/varnishd/hash_slinger.h 2009-08-19 21:04:03 UTC (rev 4206)
@@ -66,6 +66,7 @@
void HSH_BeforeVclHash(struct sess *sp, unsigned hashcount);
void HSH_AfterVclHash(const struct sess *sp);
void HSH_DerefObjCore(struct sess *sp);
+void HSH_FindBan(struct sess *sp, struct objcore **oc);
struct objcore *HSH_Insert(const struct sess *sp);
#ifdef VARNISH_CACHE_CHILD
Modified: trunk/varnish-cache/bin/varnishd/heritage.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/heritage.h 2009-08-19 18:53:46 UTC (rev 4205)
+++ trunk/varnish-cache/bin/varnishd/heritage.h 2009-08-19 21:04:03 UTC (rev 4206)
@@ -195,6 +195,9 @@
/* CLI banner */
unsigned cli_banner;
+
+ /* How long time does the ban lurker sleep */
+ double ban_lurker_sleep;
};
extern volatile struct params *params;
Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_param.c 2009-08-19 18:53:46 UTC (rev 4205)
+++ trunk/varnish-cache/bin/varnishd/mgt_param.c 2009-08-19 21:04:03 UTC (rev 4206)
@@ -758,6 +758,14 @@
"Set to off for compatibility with pre 2.1 versions.\n",
0,
"on", "bool" },
+ { "ban_lurker_sleep", tweak_timeout_double,
+ &master.ban_lurker_sleep, 0.0, UINT_MAX,
+ "How long time does the ban lurker thread sleeps between "
+ "successfull attempts to push the last item up the purge "
+ " list. It always sleeps a second when nothing can be done.\n"
+ "A value of zero disables the ban lurker.",
+ 0,
+ "0.0", "s" },
{ NULL, NULL, NULL }
};
Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-19 18:53:46 UTC (rev 4205)
+++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-08-19 21:04:03 UTC (rev 4206)
@@ -610,6 +610,7 @@
struct smp_seg *sg;
struct smp_object *so;
+ Lck_AssertHeld(&oh->mtx);
(void)sp;
sg = oc->smp_seg;
CHECK_OBJ_NOTNULL(sg, SMP_SEG_MAGIC);
Modified: trunk/varnish-cache/bin/varnishtest/tests/v00006.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/v00006.vtc 2009-08-19 18:53:46 UTC (rev 4205)
+++ trunk/varnish-cache/bin/varnishtest/tests/v00006.vtc 2009-08-19 21:04:03 UTC (rev 4206)
@@ -53,6 +53,9 @@
varnish v1 -cli "vcl.discard vcl1"
+# Give expiry thread a chance to let go
+delay 2
+
# It won't go away as long as the workthread holds a VCL reference
varnish v1 -expect n_backend == 2
varnish v1 -expect n_vcl_avail == 1
Modified: trunk/varnish-cache/include/purge_vars.h
===================================================================
--- trunk/varnish-cache/include/purge_vars.h 2009-08-19 18:53:46 UTC (rev 4205)
+++ trunk/varnish-cache/include/purge_vars.h 2009-08-19 21:04:03 UTC (rev 4206)
@@ -28,9 +28,13 @@
* $Id$
*
* Define which variables we can purge on, and which function does it.
+ *
+ * Middle field is a bitmap:
+ * 1 = HTTP header search
+ * 2 = Needs a client supplied request for test evaluation
*/
-PVAR("req.url", 0, ban_cond_url)
-PVAR("obj.hash", 0, ban_cond_hash)
-PVAR("req.http.", 1, ban_cond_req_http)
-PVAR("obj.http.", 1, ban_cond_obj_http)
+PVAR("req.url", 2|0, ban_cond_url)
+PVAR("obj.hash", 0|0, ban_cond_hash)
+PVAR("req.http.", 2|1, ban_cond_req_http)
+PVAR("obj.http.", 0|1, ban_cond_obj_http)
Modified: trunk/varnish-cache/lib/libvcl/vcc_action.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcc_action.c 2009-08-19 18:53:46 UTC (rev 4205)
+++ trunk/varnish-cache/lib/libvcl/vcc_action.c 2009-08-19 21:04:03 UTC (rev 4206)
@@ -383,7 +383,7 @@
vcc_ErrWhere(tl, tl->t);
return;
}
- if (pv->flag &&
+ if ((pv->flag & 1) &&
tl->t->b + strlen(pv->name) >= tl->t->e) {
vsb_printf(tl->sb, "Missing header name.");
vcc_ErrWhere(tl, tl->t);
More information about the varnish-commit
mailing list