[master] 4073a4b Parameter ban_cutoff to limit the number of active (incomplete) bans
Nils Goroll
nils.goroll at uplex.de
Mon Feb 27 18:11:04 CET 2017
commit 4073a4b65e9757f4218bcf750f0740127e53f283
Author: Nils Goroll <nils.goroll at uplex.de>
Date: Mon Feb 27 17:24:52 2017 +0100
Parameter ban_cutoff to limit the number of active (incomplete) bans
Expurge long tail content from the cache to keep the number of bans
below this value. 0 disables.
This is a safety net to avoid bad response times due to bans being
tested at lookup time. Setting a cutoff trades response time for cache
efficiency. The recommended value is proportional to
rate(bans_lurker_tests_tested) / n_objects while the ban lurker is
working, which is the number of bans the system can sustain. The
additional latency due to request ban testing is in the order of
ban_cutoff / rate(bans_lurker_tests_tested). For example, for
rate(bans_lurker_tests_tested) = 2M/s and a tolerable latency of
100ms, a good value for ban_cutoff may be 200K.
Merges #2131
diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c
index 638298c..9a5b43b 100644
--- a/bin/varnishd/cache/cache_ban_lurker.c
+++ b/bin/varnishd/cache/cache_ban_lurker.c
@@ -196,7 +196,7 @@ ban_lurker_getfirst(struct vsl_log *vsl, struct ban *bt)
static void
ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
- struct banhead_s *obans, struct ban *bd)
+ struct banhead_s *obans, struct ban *bd, int kill)
{
struct ban *bl, *bln;
struct objcore *oc;
@@ -238,16 +238,30 @@ ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
VTAILQ_REMOVE(obans, bl, l_list);
continue;
}
- AZ(bl->flags & BANS_FLAG_REQ);
- tests = 0;
- i = ban_evaluate(wrk, bl->spec, oc, NULL, &tests);
- VSC_C_main->bans_lurker_tested++;
- VSC_C_main->bans_lurker_tests_tested += tests;
+ if (kill == 1)
+ i = 1;
+ else {
+ AZ(bl->flags & BANS_FLAG_REQ);
+ tests = 0;
+ i = ban_evaluate(wrk, bl->spec, oc, NULL,
+ &tests);
+ VSC_C_main->bans_lurker_tested++;
+ VSC_C_main->bans_lurker_tests_tested += tests;
+ }
if (i) {
- VSLb(vsl, SLT_ExpBan, "%u banned by lurker",
- ObjGetXID(wrk, oc));
+ if (kill) {
+ VSLb(vsl, SLT_ExpBan,
+ "%u killed for lurker cutoff",
+ ObjGetXID(wrk, oc));
+ VSC_C_main->
+ bans_lurker_obj_killed_cutoff++;
+ } else {
+ VSLb(vsl, SLT_ExpBan,
+ "%u banned by lurker",
+ ObjGetXID(wrk, oc));
+ VSC_C_main->bans_lurker_obj_killed++;
+ }
HSH_Kill(oc);
- VSC_C_main->bans_lurker_obj_killed++;
break;
}
}
@@ -285,12 +299,15 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
struct ban *b, *bd;
struct banhead_s obans;
double d, dt, n;
+ unsigned count = 0, cutoff = UINT_MAX;
dt = 49.62; // Random, non-magic
if (cache_param->ban_lurker_sleep == 0) {
(void)ban_cleantail(NULL);
return (dt);
}
+ if (cache_param->ban_cutoff > 0)
+ cutoff = cache_param->ban_cutoff;
Lck_Lock(&ban_mtx);
b = ban_start;
@@ -300,14 +317,17 @@ ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
VTAILQ_INIT(&obans);
for (; b != NULL; b = VTAILQ_NEXT(b, list)) {
if (bd != NULL && bd != b)
- ban_lurker_test_ban(wrk, vsl, b, &obans, bd);
+ ban_lurker_test_ban(wrk, vsl, b, &obans, bd,
+ count > cutoff);
if (b->flags & BANS_FLAG_COMPLETED)
continue;
- if (b->flags & BANS_FLAG_REQ) {
+ if (b->flags & BANS_FLAG_REQ &&
+ count <= cutoff) {
if (bd != NULL)
bd = VTAILQ_NEXT(b, list);
continue;
}
+ count++;
n = ban_time(b->spec) - d;
if (n < 0) {
VTAILQ_INSERT_TAIL(&obans, b, l_list);
diff --git a/bin/varnishtest/tests/c00049.vtc b/bin/varnishtest/tests/c00049.vtc
index 43f9e1b..9ff05d7 100644
--- a/bin/varnishtest/tests/c00049.vtc
+++ b/bin/varnishtest/tests/c00049.vtc
@@ -188,3 +188,38 @@ varnish v1 -expect bans_lurker_obj_killed == 4
varnish v1 -expect bans_dups == 0
varnish v1 -expect n_object == 3
+
+# adding more bans than the cutoff purges all untested objects
+# (here: all objects)
+
+varnish v1 -cliok "param.set ban_lurker_age 2"
+varnish v1 -cliok "param.set ban_cutoff 4"
+
+varnish v1 -cliok "ban.list"
+varnish v1 -cliok "ban obj.http.nomatch == 1"
+varnish v1 -cliok "ban obj.http.nomatch == 2"
+varnish v1 -cliok "ban obj.http.nomatch == 3"
+varnish v1 -cliok "ban obj.http.nomatch == 4"
+varnish v1 -cliok "ban obj.http.nomatch == 5"
+varnish v1 -cliok "ban.list"
+varnish v1 -cliok "param.set ban_lurker_age .1"
+delay 3
+
+varnish v1 -cliok "ban.list"
+
+varnish v1 -expect bans == 1
+varnish v1 -expect bans_completed == 1
+varnish v1 -expect bans_req == 0
+varnish v1 -expect bans_obj == 1
+varnish v1 -expect bans_added == 11
+varnish v1 -expect bans_deleted == 10
+varnish v1 -expect bans_tested == 2
+varnish v1 -expect bans_tests_tested == 2
+varnish v1 -expect bans_obj_killed == 1
+varnish v1 -expect bans_lurker_tested == 8
+varnish v1 -expect bans_lurker_tests_tested == 9
+varnish v1 -expect bans_lurker_obj_killed == 4
+varnish v1 -expect bans_lurker_obj_killed_cutoff == 3
+varnish v1 -expect bans_dups == 0
+
+varnish v1 -expect n_object == 0
diff --git a/include/tbl/params.h b/include/tbl/params.h
index 71552ee..49f77ac 100644
--- a/include/tbl/params.h
+++ b/include/tbl/params.h
@@ -137,6 +137,30 @@ PARAM(
)
PARAM(
+ /* name */ ban_cutoff,
+ /* typ */ uint,
+ /* min */ "0",
+ /* max */ NULL,
+ /* default */ "0",
+ /* units */ "bans",
+ /* flags */ EXPERIMENTAL,
+ /* s-text */
+ "Expurge long tail content from the cache to keep the number of bans "
+ "below this value. 0 disables.\n"
+ "This is a safety net to avoid bad response times due to bans being "
+ "tested at lookup time. Setting a cutoff trades response time for "
+ "cache efficiency. The recommended value is proportional to "
+ "rate(bans_lurker_tests_tested) / n_objects while the ban lurker is "
+ "working, which is the number of bans the system can sustain. The "
+ "additional latency due to request ban testing is in the order of "
+ "ban_cutoff / rate(bans_lurker_tests_tested). For example, for "
+ "rate(bans_lurker_tests_tested) = 2M/s and a tolerable latency of "
+ "100ms, a good value for ban_cutoff may be 200K.",
+ /* l-text */ "",
+ /* func */ NULL
+)
+
+PARAM(
/* name */ ban_lurker_age,
/* typ */ timeout,
/* min */ "0",
diff --git a/include/tbl/vsc_f_main.h b/include/tbl/vsc_f_main.h
index 2651435..b1dbb17 100644
--- a/include/tbl/vsc_f_main.h
+++ b/include/tbl/vsc_f_main.h
@@ -571,7 +571,13 @@ VSC_FF(bans_lurker_tests_tested, uint64_t, 0, 'c', 'i', diag,
VSC_FF(bans_lurker_obj_killed, uint64_t, 0, 'c', 'i', diag,
"Objects killed by bans (lurker)",
- "Number of objects killed by ban-lurker."
+ "Number of objects killed by the ban-lurker."
+)
+
+VSC_FF(bans_lurker_obj_killed_cutoff, uint64_t, 0, 'c', 'i', diag,
+ "Objects killed by bans for cutoff (lurker)",
+ "Number of objects killed by the ban-lurker to keep the number of"
+ " bans below ban_cutoff."
)
VSC_FF(bans_dups, uint64_t, 0, 'c', 'i', diag,
More information about the varnish-commit
mailing list