[master] 30b5f02 Split ban_lurker into separate source file.

Poul-Henning Kamp phk at FreeBSD.org
Thu Oct 22 17:40:38 CEST 2015


commit 30b5f02c527cb227737d0a26ed87af8237ba96a9
Author: Poul-Henning Kamp <phk at FreeBSD.org>
Date:   Thu Oct 22 05:44:51 2015 +0000

    Split ban_lurker into separate source file.
    
    No functional changes.

diff --git a/bin/varnishd/Makefile.am b/bin/varnishd/Makefile.am
index e4d2ab5..86906a2 100644
--- a/bin/varnishd/Makefile.am
+++ b/bin/varnishd/Makefile.am
@@ -15,6 +15,7 @@ varnishd_SOURCES = \
 	cache/cache_backend_probe.c \
 	cache/cache_backend_tcp.c \
 	cache/cache_ban.c \
+	cache/cache_ban_lurker.c \
 	cache/cache_busyobj.c \
 	cache/cache_cli.c \
 	cache/cache_deliver_proc.c \
@@ -99,6 +100,7 @@ varnishd_SOURCES = \
 
 noinst_HEADERS = \
 	builtin_vcl.h \
+	cache/cache_ban.h \
 	cache/cache_esi.h \
 	cache/cache_pool.h \
 	cache/cache_priv.h \
diff --git a/bin/varnishd/cache/cache_ban.c b/bin/varnishd/cache/cache_ban.c
index f63c8b4..016c77c 100644
--- a/bin/varnishd/cache/cache_ban.c
+++ b/bin/varnishd/cache/cache_ban.c
@@ -26,38 +26,6 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- * Ban processing
- *
- * A ban consists of a number of conditions (or tests), all of which must be
- * satisfied.  Here are some potential bans we could support:
- *
- *	req.url == "/foo"
- *	req.url ~ ".iso" && obj.size > 10MB
- *	req.http.host ~ "web1.com" && obj.http.set-cookie ~ "USER=29293"
- *
- * We make the "&&" mandatory from the start, leaving the syntax space
- * for latter handling of "||" as well.
- *
- * Bans are compiled into bytestrings as follows:
- *	8 bytes	- double: timestamp		XXX: Byteorder ?
- *	4 bytes - be32: length
- *	1 byte - flags: 0x01: BAN_F_{REQ|OBJ|COMPLETED}
- *	N tests
- * A test have this form:
- *	1 byte - arg (see ban_vars.h col 3 "BANS_ARG_XXX")
- *	(n bytes) - http header name, canonical encoding
- *	lump - comparison arg
- *	1 byte - operation (BANS_OPER_)
- *	(lump) - compiled regexp
- * A lump is:
- *	4 bytes - be32: length
- *	n bytes - content
- *
- * In a perfect world, we should vector through VRE to get to PCRE,
- * but since we rely on PCRE's ability to encode the regexp into a
- * byte string, that would be a little bit artificial, so this is
- * the exception that confirms the rule.
- *
  */
 
 #include "config.h"
@@ -65,6 +33,7 @@
 #include <pcre.h>
 
 #include "cache.h"
+#include "cache_ban.h"
 
 #include "hash/hash_slinger.h"
 #include "vcli.h"
@@ -73,66 +42,13 @@
 #include "vmb.h"
 #include "vtim.h"
 
-/*--------------------------------------------------------------------
- * BAN string defines & magic markers
- */
-
-#define BANS_TIMESTAMP		0
-#define BANS_LENGTH		8
-#define BANS_FLAGS		12
-#define BANS_HEAD_LEN		16
-
-#define BANS_FLAG_REQ		(1<<0)
-#define BANS_FLAG_OBJ		(1<<1)
-#define BANS_FLAG_COMPLETED	(1<<2)
-#define BANS_FLAG_HTTP		(1<<3)
-#define BANS_FLAG_ERROR		(1<<4)
-
-#define BANS_OPER_EQ		0x10
-#define BANS_OPER_NEQ		0x11
-#define BANS_OPER_MATCH		0x12
-#define BANS_OPER_NMATCH	0x13
-
-#define BANS_ARG_URL		0x18
-#define BANS_ARG_REQHTTP	0x19
-#define BANS_ARG_OBJHTTP	0x1a
-#define BANS_ARG_OBJSTATUS	0x1b
-
-/*--------------------------------------------------------------------*/
-
-struct ban {
-	unsigned		magic;
-#define BAN_MAGIC		0x700b08ea
-	VTAILQ_ENTRY(ban)	list;
-	VTAILQ_ENTRY(ban)	l_list;
-	int			refcount;
-	unsigned		flags;		/* BANS_FLAG_* */
-
-	VTAILQ_HEAD(,objcore)	objcore;
-	struct vsb		*vsb;
-	uint8_t			*spec;
-};
-
-VTAILQ_HEAD(banhead_s,ban);
+struct lock ban_mtx;
+int ban_shutdown = 0;
+struct banhead_s ban_head = VTAILQ_HEAD_INITIALIZER(ban_head);
+struct ban * volatile ban_start;
 
-struct ban_test {
-	uint8_t			arg1;
-	const char		*arg1_spec;
-	uint8_t			oper;
-	const char		*arg2;
-	const void		*arg2_spec;
-};
-
-static void ban_info(enum baninfo event, const uint8_t *ban, unsigned len);
-static struct banhead_s ban_head = VTAILQ_HEAD_INITIALIZER(ban_head);
-static struct lock ban_mtx;
 static struct ban *ban_magic;
 static pthread_t ban_thread;
-static struct ban * volatile ban_start;
-static struct objcore oc_marker = { .magic = OBJCORE_MAGIC, };
-static bgthread_t ban_lurker;
-static unsigned ban_batch;
-static int ban_shutdown = 0;
 
 /*--------------------------------------------------------------------
  * Variables we can purge on
@@ -231,7 +147,7 @@ BAN_TailDeref(struct ban **bb)
  * Extract time and length from ban-spec
  */
 
-static double
+double
 ban_time(const uint8_t *banspec)
 {
 	double t;
@@ -241,7 +157,7 @@ ban_time(const uint8_t *banspec)
 	return (t);
 }
 
-static unsigned
+unsigned
 ban_len(const uint8_t *banspec)
 {
 	unsigned u;
@@ -264,7 +180,7 @@ ban_equal(const uint8_t *bs1, const uint8_t *bs2)
 	return (!memcmp(bs1 + BANS_LENGTH, bs2 + BANS_LENGTH, u - BANS_LENGTH));
 }
 
-static void
+void
 ban_mark_completed(struct ban *b)
 {
 	unsigned ln;
@@ -677,7 +593,7 @@ ban_export(void)
 	VSC_C_main->bans_persisted_fragmentation = 0;
 }
 
-static void
+void
 ban_info(enum baninfo event, const uint8_t *ban, unsigned len)
 {
 	if (STV_BanInfo(event, ban, len)) {
@@ -799,36 +715,10 @@ BAN_Time(const struct ban *b)
 }
 
 /*--------------------------------------------------------------------
- * All silos have read their bans, ready for action
- */
-
-void
-BAN_Compile(void)
-{
-
-	ASSERT_CLI();
-	AZ(ban_shutdown);
-
-	Lck_Lock(&ban_mtx);
-
-	/* Do late reporting of ban_magic */
-	AZ(STV_BanInfo(BI_NEW, ban_magic->spec, ban_len(ban_magic->spec)));
-
-	/* All bans have been read from all persistent stevedores. Export
-	   the compiled list */
-	ban_export();
-
-	Lck_Unlock(&ban_mtx);
-
-	ban_start = VTAILQ_FIRST(&ban_head);
-	WRK_BgThread(&ban_thread, "ban-lurker", ban_lurker, NULL);
-}
-
-/*--------------------------------------------------------------------
  * Evaluate ban-spec
  */
 
-static int
+int
 ban_evaluate(struct worker *wrk, const uint8_t *bs, struct objcore *oc,
     const struct http *reqhttp, unsigned *tests)
 {
@@ -971,227 +861,6 @@ BAN_CheckObject(struct worker *wrk, struct objcore *oc, struct req *req)
 	}
 }
 
-static void
-ban_cleantail(void)
-{
-	struct ban *b;
-
-	do {
-		Lck_Lock(&ban_mtx);
-		b = VTAILQ_LAST(&ban_head, banhead_s);
-		if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) {
-			if (b->flags & BANS_FLAG_COMPLETED)
-				VSC_C_main->bans_completed--;
-			if (b->flags & BANS_FLAG_OBJ)
-				VSC_C_main->bans_obj--;
-			if (b->flags & BANS_FLAG_REQ)
-				VSC_C_main->bans_req--;
-			VSC_C_main->bans--;
-			VSC_C_main->bans_deleted++;
-			VTAILQ_REMOVE(&ban_head, b, list);
-			VSC_C_main->bans_persisted_fragmentation +=
-			    ban_len(b->spec);
-			ban_info(BI_DROP, b->spec, ban_len(b->spec));
-		} else {
-			b = NULL;
-		}
-		Lck_Unlock(&ban_mtx);
-		if (b != NULL)
-			BAN_Free(b);
-	} while (b != NULL);
-}
-
-/*--------------------------------------------------------------------
- * Our task here is somewhat tricky:  The canonical locking order is
- * objhead->mtx first, then ban_mtx, because that is the order which
- * makes most sense in HSH_Lookup(), but we come the other way.
- * We optimistically try to get them the other way, and get out of
- * the way if that fails, and retry again later.
- */
-
-static struct objcore *
-ban_lurker_getfirst(struct vsl_log *vsl, struct ban *bt)
-{
-	struct objhead *oh;
-	struct objcore *oc;
-
-	while (1) {
-		Lck_Lock(&ban_mtx);
-		oc = VTAILQ_FIRST(&bt->objcore);
-		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
-		if (oc == &oc_marker) {
-			VTAILQ_REMOVE(&bt->objcore, oc, ban_list);
-			Lck_Unlock(&ban_mtx);
-			return (NULL);
-		}
-		oh = oc->objhead;
-		CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
-		if (!Lck_Trylock(&oh->mtx)) {
-			if (oc->refcnt == 0) {
-				Lck_Unlock(&oh->mtx);
-			} else {
-				/*
-				 * We got the lock, and the oc is not being
-				 * dismantled under our feet, run with it...
-				 */
-				AZ(oc->flags & OC_F_BUSY);
-				oc->refcnt += 1;
-				VTAILQ_REMOVE(&bt->objcore, oc, ban_list);
-				VTAILQ_INSERT_TAIL(&bt->objcore, oc, ban_list);
-				Lck_Unlock(&oh->mtx);
-				Lck_Unlock(&ban_mtx);
-				break;
-			}
-		}
-
-		/* Try again, later */
-		Lck_Unlock(&ban_mtx);
-		VSC_C_main->bans_lurker_contention++;
-		VSL_Flush(vsl, 0);
-		VTIM_sleep(cache_param->ban_lurker_sleep);
-	}
-	return (oc);
-}
-
-static void
-ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
-    struct banhead_s *obans)
-{
-	struct ban *bl, *bln;
-	struct objcore *oc;
-	unsigned tests;
-	int i;
-
-	/*
-	 * First see if there is anything to do, and if so, insert marker
-	 */
-	Lck_Lock(&ban_mtx);
-	oc = VTAILQ_FIRST(&bt->objcore);
-	if (oc != NULL)
-		VTAILQ_INSERT_TAIL(&bt->objcore, &oc_marker, ban_list);
-	Lck_Unlock(&ban_mtx);
-	if (oc == NULL)
-		return;
-
-	while (1) {
-		if (++ban_batch > cache_param->ban_lurker_batch) {
-			VTIM_sleep(cache_param->ban_lurker_sleep);
-			ban_batch = 0;
-		}
-		oc = ban_lurker_getfirst(vsl, bt);
-		if (oc == NULL)
-			return;
-		i = 0;
-		VTAILQ_FOREACH_REVERSE_SAFE(bl, obans, banhead_s, l_list, bln) {
-			if (bl->flags & BANS_FLAG_COMPLETED) {
-				/* Ban was overtaken by new (dup) ban */
-				VTAILQ_REMOVE(obans, bl, l_list);
-				continue;
-			}
-			tests = 0;
-			i = ban_evaluate(wrk, bl->spec, oc, NULL, &tests);
-			VSC_C_main->bans_lurker_tested++;
-			VSC_C_main->bans_lurker_tests_tested += tests;
-			if (i)
-				break;
-		}
-		if (i) {
-			VSLb(vsl, SLT_ExpBan, "%u banned by lurker",
-			    ObjGetXID(wrk, oc));
-
-			EXP_Rearm(oc, oc->exp.t_origin, 0, 0, 0);
-					// XXX ^ fake now
-			VSC_C_main->bans_lurker_obj_killed++;
-		}
-		(void)HSH_DerefObjCore(wrk, &oc);
-	}
-}
-
-/*--------------------------------------------------------------------
- * Ban lurker thread
- */
-
-static int
-ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
-{
-	struct ban *b, *bt;
-	struct banhead_s obans;
-	double d;
-	int i;
-
-	/* Make a list of the bans we can do something about */
-	VTAILQ_INIT(&obans);
-	Lck_Lock(&ban_mtx);
-	b = ban_start;
-	Lck_Unlock(&ban_mtx);
-	i = 0;
-	d = VTIM_real() - cache_param->ban_lurker_age;
-	while (b != NULL) {
-		if (b->flags & BANS_FLAG_COMPLETED) {
-			;
-		} else if (b->flags & BANS_FLAG_REQ) {
-			;
-		} else if (b == VTAILQ_LAST(&ban_head, banhead_s)) {
-			;
-		} else if (ban_time(b->spec) > d) {
-			;
-		} else {
-			VTAILQ_INSERT_TAIL(&obans, b, l_list);
-			i++;
-		}
-		b = VTAILQ_NEXT(b, list);
-	}
-	if (DO_DEBUG(DBG_LURKER))
-		VSLb(vsl, SLT_Debug, "lurker: %d actionable bans", i);
-	if (i == 0)
-		return (0);
-
-	/* Go though all the bans to test the objects */
-	VTAILQ_FOREACH_REVERSE(bt, &ban_head, banhead_s, list) {
-		if (bt == VTAILQ_LAST(&obans, banhead_s)) {
-			if (DO_DEBUG(DBG_LURKER))
-				VSLb(vsl, SLT_Debug,
-				    "Lurk bt completed %p", bt);
-			Lck_Lock(&ban_mtx);
-			/* We can be raced by a new ban */
-			if (!(bt->flags & BANS_FLAG_COMPLETED))
-				ban_mark_completed(bt);
-			Lck_Unlock(&ban_mtx);
-			VTAILQ_REMOVE(&obans, bt, l_list);
-			if (VTAILQ_EMPTY(&obans))
-				break;
-		}
-		if (DO_DEBUG(DBG_LURKER))
-			VSLb(vsl, SLT_Debug, "Lurk bt %p", bt);
-		ban_lurker_test_ban(wrk, vsl, bt, &obans);
-		if (VTAILQ_EMPTY(&obans))
-			break;
-	}
-	return (1);
-}
-
-static void * __match_proto__(bgthread_t)
-ban_lurker(struct worker *wrk, void *priv)
-{
-	struct vsl_log vsl;
-	volatile double d;
-
-	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
-	AZ(priv);
-
-	VSL_Setup(&vsl, NULL, 0);
-
-	while (!ban_shutdown) {
-		d = cache_param->ban_lurker_sleep;
-		if (d <= 0.0 || !ban_lurker_work(wrk, &vsl))
-			d = 0.609;	// Random, non-magic
-		ban_cleantail();
-		VTIM_sleep(d);
-	}
-	pthread_exit(0);
-	NEEDLESS_RETURN(NULL);
-}
-
 /*--------------------------------------------------------------------
  * CLI functions to add bans
  */
@@ -1326,6 +995,34 @@ static struct cli_proto ban_cmds[] = {
 	{ NULL }
 };
 
+/*--------------------------------------------------------------------
+ */
+
+void
+BAN_Compile(void)
+{
+
+	/* All bans have been read from all persistent stevedores. Export
+	 * the compiled list
+	 */
+
+	ASSERT_CLI();
+	AZ(ban_shutdown);
+
+	Lck_Lock(&ban_mtx);
+
+	/* Do late reporting of ban_magic */
+	AZ(STV_BanInfo(BI_NEW, ban_magic->spec, ban_len(ban_magic->spec)));
+
+	ban_export();
+
+	Lck_Unlock(&ban_mtx);
+
+	ban_start = VTAILQ_FIRST(&ban_head);
+	WRK_BgThread(&ban_thread, "ban-lurker", ban_lurker, NULL);
+}
+
+
 void
 BAN_Init(void)
 {
diff --git a/bin/varnishd/cache/cache_ban.h b/bin/varnishd/cache/cache_ban.h
new file mode 100644
index 0000000..fae2b80
--- /dev/null
+++ b/bin/varnishd/cache/cache_ban.h
@@ -0,0 +1,124 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2015 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * Ban processing
+ *
+ * A ban consists of a number of conditions (or tests), all of which must be
+ * satisfied.  Here are some potential bans we could support:
+ *
+ *	req.url == "/foo"
+ *	req.url ~ ".iso" && obj.size > 10MB
+ *	req.http.host ~ "web1.com" && obj.http.set-cookie ~ "USER=29293"
+ *
+ * We make the "&&" mandatory from the start, leaving the syntax space
+ * for latter handling of "||" as well.
+ *
+ * Bans are compiled into bytestrings as follows:
+ *	8 bytes	- double: timestamp		XXX: Byteorder ?
+ *	4 bytes - be32: length
+ *	1 byte - flags: 0x01: BAN_F_{REQ|OBJ|COMPLETED}
+ *	N tests
+ * A test have this form:
+ *	1 byte - arg (see ban_vars.h col 3 "BANS_ARG_XXX")
+ *	(n bytes) - http header name, canonical encoding
+ *	lump - comparison arg
+ *	1 byte - operation (BANS_OPER_)
+ *	(lump) - compiled regexp
+ * A lump is:
+ *	4 bytes - be32: length
+ *	n bytes - content
+ *
+ * In a perfect world, we should vector through VRE to get to PCRE,
+ * but since we rely on PCRE's ability to encode the regexp into a
+ * byte string, that would be a little bit artificial, so this is
+ * the exception that confirms the rule.
+ *
+ */
+
+/*--------------------------------------------------------------------
+ * BAN string defines & magic markers
+ */
+
+#define BANS_TIMESTAMP		0
+#define BANS_LENGTH		8
+#define BANS_FLAGS		12
+#define BANS_HEAD_LEN		16
+
+#define BANS_FLAG_REQ		(1<<0)
+#define BANS_FLAG_OBJ		(1<<1)
+#define BANS_FLAG_COMPLETED	(1<<2)
+#define BANS_FLAG_HTTP		(1<<3)
+#define BANS_FLAG_ERROR		(1<<4)
+
+#define BANS_OPER_EQ		0x10
+#define BANS_OPER_NEQ		0x11
+#define BANS_OPER_MATCH		0x12
+#define BANS_OPER_NMATCH	0x13
+
+#define BANS_ARG_URL		0x18
+#define BANS_ARG_REQHTTP	0x19
+#define BANS_ARG_OBJHTTP	0x1a
+#define BANS_ARG_OBJSTATUS	0x1b
+
+/*--------------------------------------------------------------------*/
+
+struct ban {
+	unsigned		magic;
+#define BAN_MAGIC		0x700b08ea
+	VTAILQ_ENTRY(ban)	list;
+	VTAILQ_ENTRY(ban)	l_list;
+	int			refcount;
+	unsigned		flags;		/* BANS_FLAG_* */
+
+	VTAILQ_HEAD(,objcore)	objcore;
+	struct vsb		*vsb;
+	uint8_t			*spec;
+};
+
+VTAILQ_HEAD(banhead_s,ban);
+
+struct ban_test {
+	uint8_t			arg1;
+	const char		*arg1_spec;
+	uint8_t			oper;
+	const char		*arg2;
+	const void		*arg2_spec;
+};
+
+bgthread_t ban_lurker;
+extern struct lock ban_mtx;
+extern int ban_shutdown;
+extern struct banhead_s ban_head;
+extern struct ban * volatile ban_start;
+
+void ban_mark_completed(struct ban *b);
+unsigned ban_len(const uint8_t *banspec);
+void ban_info(enum baninfo event, const uint8_t *ban, unsigned len);
+int ban_evaluate(struct worker *wrk, const uint8_t *bs, struct objcore *oc,
+    const struct http *reqhttp, unsigned *tests);
+double ban_time(const uint8_t *banspec);
diff --git a/bin/varnishd/cache/cache_ban_lurker.c b/bin/varnishd/cache/cache_ban_lurker.c
new file mode 100644
index 0000000..cfcdd26
--- /dev/null
+++ b/bin/varnishd/cache/cache_ban_lurker.c
@@ -0,0 +1,261 @@
+/*-
+ * Copyright (c) 2006 Verdens Gang AS
+ * Copyright (c) 2006-2015 Varnish Software AS
+ * All rights reserved.
+ *
+ * Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include "config.h"
+
+#include "cache.h"
+#include "cache_ban.h"
+
+#include "hash/hash_slinger.h"
+#include "vtim.h"
+
+static struct objcore oc_marker = { .magic = OBJCORE_MAGIC, };
+static unsigned ban_batch;
+
+static void
+ban_cleantail(void)
+{
+	struct ban *b;
+
+	do {
+		Lck_Lock(&ban_mtx);
+		b = VTAILQ_LAST(&ban_head, banhead_s);
+		if (b != VTAILQ_FIRST(&ban_head) && b->refcount == 0) {
+			if (b->flags & BANS_FLAG_COMPLETED)
+				VSC_C_main->bans_completed--;
+			if (b->flags & BANS_FLAG_OBJ)
+				VSC_C_main->bans_obj--;
+			if (b->flags & BANS_FLAG_REQ)
+				VSC_C_main->bans_req--;
+			VSC_C_main->bans--;
+			VSC_C_main->bans_deleted++;
+			VTAILQ_REMOVE(&ban_head, b, list);
+			VSC_C_main->bans_persisted_fragmentation +=
+			    ban_len(b->spec);
+			ban_info(BI_DROP, b->spec, ban_len(b->spec));
+		} else {
+			b = NULL;
+		}
+		Lck_Unlock(&ban_mtx);
+		if (b != NULL)
+			BAN_Free(b);
+	} while (b != NULL);
+}
+
+/*--------------------------------------------------------------------
+ * Our task here is somewhat tricky:  The canonical locking order is
+ * objhead->mtx first, then ban_mtx, because that is the order which
+ * makes most sense in HSH_Lookup(), but we come the other way.
+ * We optimistically try to get them the other way, and get out of
+ * the way if that fails, and retry again later.
+ */
+
+static struct objcore *
+ban_lurker_getfirst(struct vsl_log *vsl, struct ban *bt)
+{
+	struct objhead *oh;
+	struct objcore *oc;
+
+	while (1) {
+		Lck_Lock(&ban_mtx);
+		oc = VTAILQ_FIRST(&bt->objcore);
+		CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+		if (oc == &oc_marker) {
+			VTAILQ_REMOVE(&bt->objcore, oc, ban_list);
+			Lck_Unlock(&ban_mtx);
+			return (NULL);
+		}
+		oh = oc->objhead;
+		CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+		if (!Lck_Trylock(&oh->mtx)) {
+			if (oc->refcnt == 0) {
+				Lck_Unlock(&oh->mtx);
+			} else {
+				/*
+				 * We got the lock, and the oc is not being
+				 * dismantled under our feet, run with it...
+				 */
+				AZ(oc->flags & OC_F_BUSY);
+				oc->refcnt += 1;
+				VTAILQ_REMOVE(&bt->objcore, oc, ban_list);
+				VTAILQ_INSERT_TAIL(&bt->objcore, oc, ban_list);
+				Lck_Unlock(&oh->mtx);
+				Lck_Unlock(&ban_mtx);
+				break;
+			}
+		}
+
+		/* Try again, later */
+		Lck_Unlock(&ban_mtx);
+		VSC_C_main->bans_lurker_contention++;
+		VSL_Flush(vsl, 0);
+		VTIM_sleep(cache_param->ban_lurker_sleep);
+	}
+	return (oc);
+}
+
+static void
+ban_lurker_test_ban(struct worker *wrk, struct vsl_log *vsl, struct ban *bt,
+    struct banhead_s *obans)
+{
+	struct ban *bl, *bln;
+	struct objcore *oc;
+	unsigned tests;
+	int i;
+
+	/*
+	 * First see if there is anything to do, and if so, insert marker
+	 */
+	Lck_Lock(&ban_mtx);
+	oc = VTAILQ_FIRST(&bt->objcore);
+	if (oc != NULL)
+		VTAILQ_INSERT_TAIL(&bt->objcore, &oc_marker, ban_list);
+	Lck_Unlock(&ban_mtx);
+	if (oc == NULL)
+		return;
+
+	while (1) {
+		if (++ban_batch > cache_param->ban_lurker_batch) {
+			VTIM_sleep(cache_param->ban_lurker_sleep);
+			ban_batch = 0;
+		}
+		oc = ban_lurker_getfirst(vsl, bt);
+		if (oc == NULL)
+			return;
+		i = 0;
+		VTAILQ_FOREACH_REVERSE_SAFE(bl, obans, banhead_s, l_list, bln) {
+			if (bl->flags & BANS_FLAG_COMPLETED) {
+				/* Ban was overtaken by new (dup) ban */
+				VTAILQ_REMOVE(obans, bl, l_list);
+				continue;
+			}
+			tests = 0;
+			i = ban_evaluate(wrk, bl->spec, oc, NULL, &tests);
+			VSC_C_main->bans_lurker_tested++;
+			VSC_C_main->bans_lurker_tests_tested += tests;
+			if (i)
+				break;
+		}
+		if (i) {
+			VSLb(vsl, SLT_ExpBan, "%u banned by lurker",
+			    ObjGetXID(wrk, oc));
+
+			EXP_Rearm(oc, oc->exp.t_origin, 0, 0, 0);
+					// XXX ^ fake now
+			VSC_C_main->bans_lurker_obj_killed++;
+		}
+		(void)HSH_DerefObjCore(wrk, &oc);
+	}
+}
+
+/*--------------------------------------------------------------------
+ * Ban lurker thread
+ */
+
+static int
+ban_lurker_work(struct worker *wrk, struct vsl_log *vsl)
+{
+	struct ban *b, *bt;
+	struct banhead_s obans;
+	double d;
+	int i;
+
+	/* Make a list of the bans we can do something about */
+	VTAILQ_INIT(&obans);
+	Lck_Lock(&ban_mtx);
+	b = ban_start;
+	Lck_Unlock(&ban_mtx);
+	i = 0;
+	d = VTIM_real() - cache_param->ban_lurker_age;
+	while (b != NULL) {
+		if (b->flags & BANS_FLAG_COMPLETED) {
+			;
+		} else if (b->flags & BANS_FLAG_REQ) {
+			;
+		} else if (b == VTAILQ_LAST(&ban_head, banhead_s)) {
+			;
+		} else if (ban_time(b->spec) > d) {
+			;
+		} else {
+			VTAILQ_INSERT_TAIL(&obans, b, l_list);
+			i++;
+		}
+		b = VTAILQ_NEXT(b, list);
+	}
+	if (DO_DEBUG(DBG_LURKER))
+		VSLb(vsl, SLT_Debug, "lurker: %d actionable bans", i);
+	if (i == 0)
+		return (0);
+
+	/* Go though all the bans to test the objects */
+	VTAILQ_FOREACH_REVERSE(bt, &ban_head, banhead_s, list) {
+		if (bt == VTAILQ_LAST(&obans, banhead_s)) {
+			if (DO_DEBUG(DBG_LURKER))
+				VSLb(vsl, SLT_Debug,
+				    "Lurk bt completed %p", bt);
+			Lck_Lock(&ban_mtx);
+			/* We can be raced by a new ban */
+			if (!(bt->flags & BANS_FLAG_COMPLETED))
+				ban_mark_completed(bt);
+			Lck_Unlock(&ban_mtx);
+			VTAILQ_REMOVE(&obans, bt, l_list);
+			if (VTAILQ_EMPTY(&obans))
+				break;
+		}
+		if (DO_DEBUG(DBG_LURKER))
+			VSLb(vsl, SLT_Debug, "Lurk bt %p", bt);
+		ban_lurker_test_ban(wrk, vsl, bt, &obans);
+		if (VTAILQ_EMPTY(&obans))
+			break;
+	}
+	return (1);
+}
+
+void * __match_proto__(bgthread_t)
+ban_lurker(struct worker *wrk, void *priv)
+{
+	struct vsl_log vsl;
+	volatile double d;
+
+	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
+	AZ(priv);
+
+	VSL_Setup(&vsl, NULL, 0);
+
+	while (!ban_shutdown) {
+		d = cache_param->ban_lurker_sleep;
+		if (d <= 0.0 || !ban_lurker_work(wrk, &vsl))
+			d = 0.609;	// Random, non-magic
+		ban_cleantail();
+		VTIM_sleep(d);
+	}
+	pthread_exit(0);
+	NEEDLESS_RETURN(NULL);
+}



More information about the varnish-commit mailing list