r3939 - in trunk/varnish-cache: bin/varnishd bin/varnishtest/tests include

phk at projects.linpro.no phk at projects.linpro.no
Mon Mar 16 14:32:36 CET 2009


Author: phk
Date: 2009-03-16 14:32:36 +0100 (Mon, 16 Mar 2009)
New Revision: 3939

Added:
   trunk/varnish-cache/bin/varnishtest/tests/p00002.vtc
Modified:
   trunk/varnish-cache/bin/varnishd/cache.h
   trunk/varnish-cache/bin/varnishd/cache_ban.c
   trunk/varnish-cache/bin/varnishd/cache_main.c
   trunk/varnish-cache/bin/varnishd/storage_persistent.c
   trunk/varnish-cache/include/persistent.h
Log:
Save and restore bans from persistent storage.

We append bans in all silos all the time.

(Still missing: periodic rewrite of list to get rid of "gone" bans)

When we open the silos, we read in all the bans in the silo before
kicking of the silo thread that reads on the object indicies, this
way we are sure to have the bans referenced from the objects.

Add a testcase to see that duplicate elimination (from multiple silos) work.



Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h	2009-03-16 13:28:32 UTC (rev 3938)
+++ trunk/varnish-cache/bin/varnishd/cache.h	2009-03-16 13:32:36 UTC (rev 3939)
@@ -459,6 +459,9 @@
 void BAN_NewObj(struct object *o);
 void BAN_DestroyObj(struct object *o);
 int BAN_CheckObject(struct object *o, const struct sess *sp);
+void BAN_Reload(double t0, unsigned flags, const char *ban);
+struct ban *BAN_TailRef(void);
+void BAN_Compile(void);
 
 /* cache_center.c [CNT] */
 void CNT_Session(struct sess *sp);

Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_ban.c	2009-03-16 13:28:32 UTC (rev 3938)
+++ trunk/varnish-cache/bin/varnishd/cache_ban.c	2009-03-16 13:32:36 UTC (rev 3939)
@@ -469,6 +469,85 @@
 }
 
 /*--------------------------------------------------------------------
+ * Bans read in from persistent storage on startup
+ */
+
+struct ban *
+BAN_TailRef(void)
+{
+	struct ban *b;
+
+	ASSERT_CLI();
+	b = VTAILQ_LAST(&ban_head, banhead);
+	AN(b);
+	b->refcount++;
+	return (b);
+}
+
+/*--------------------------------------------------------------------
+ * Put a skeleton ban in the list, unless there is an indentical one
+ * already.
+ */
+
+void
+BAN_Reload(double t0, unsigned flags, const char *ban)
+{
+	struct ban *b, *b2;
+
+	ASSERT_CLI();
+
+	(void)flags;		/* for future use */
+	VTAILQ_FOREACH(b, &ban_head, list) {
+		if (b->t0 > t0)
+			continue;
+		if (b->t0 == t0 && !strcmp(b->test, ban))
+			return;
+		if (b->t0 < t0)
+			break;
+	}
+
+	VSL_stats->n_purge++;
+	VSL_stats->n_purge_add++;
+
+	b2 = BAN_New();
+	b2->test = strdup(ban);
+	AN(b2->test);
+	b2->t0 = t0;
+	if (b == NULL)
+		VTAILQ_INSERT_TAIL(&ban_head, b2, list);
+	else
+		VTAILQ_INSERT_BEFORE(b, b2, list);
+}
+
+void
+BAN_Compile(void)
+{
+	struct ban *b;
+	char **av;
+	int i;
+
+	ASSERT_CLI();
+
+	VTAILQ_FOREACH(b, &ban_head, list) {
+		if (!VTAILQ_EMPTY(&b->tests))
+			continue;
+		if (b->test == NULL || *b->test == '\0')
+			continue;
+		av = ParseArgv(b->test, 0);
+		XXXAN(av);
+		XXXAZ(av[0]);
+		for (i = 1; av[i] != NULL; i += 3) {
+			if (i != 1) {
+				AZ(strcmp(av[i], "&&"));
+				i++;
+			}
+			AZ(BAN_AddTest(NULL, b, av[i], av[i + 1], av[i + 2]));
+		}
+	}
+	ban_start = VTAILQ_FIRST(&ban_head);
+}
+
+/*--------------------------------------------------------------------
  * CLI functions to add bans
  */
 
@@ -552,6 +631,7 @@
 ccf_purge_list(struct cli *cli, const char * const *av, void *priv)
 {
 	struct ban *b;
+	char t[64];
 
 	(void)av;
 	(void)priv;
@@ -570,7 +650,8 @@
 	VTAILQ_FOREACH(b, &ban_head, list) {
 		if (b->refcount == 0 && (b->flags & BAN_F_GONE))
 			continue;
-		cli_out(cli, "%5u%s\t%s\n", b->refcount,
+		TIM_format(b->t0, t);
+		cli_out(cli, "%s %10.6f %5u%s\t%s\n", t, b->t0, b->refcount,
 		    b->flags & BAN_F_GONE ? "G" : " ", b->test);
 	}
 
@@ -597,17 +678,12 @@
 void
 BAN_Init(void)
 {
-	const char *aav[6];
+	struct ban *b;
 
 	Lck_New(&ban_mtx);
 	CLI_AddFuncs(PUBLIC_CLI, ban_cmds);
 
-	/* Add an initial ban, since the list can never be empty */
-	aav[0] = NULL;
-	aav[1] = "purge";
-	aav[2] = "req.url";
-	aav[3] = "~";
-	aav[4] = ".";
-	aav[5] = NULL;
-	ccf_purge(NULL, aav, NULL);
+	b = BAN_New();
+	b->flags |= BAN_F_GONE;
+	BAN_Insert(b);
 }

Modified: trunk/varnish-cache/bin/varnishd/cache_main.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_main.c	2009-03-16 13:28:32 UTC (rev 3938)
+++ trunk/varnish-cache/bin/varnishd/cache_main.c	2009-03-16 13:32:36 UTC (rev 3939)
@@ -129,6 +129,8 @@
 	SMS_Init();
 	STV_open();
 
+	BAN_Compile();
+
 	/* Wait for persistent storage to load if asked to */
 	if (params->diag_bitmap & 0x00020000)
 		SMP_Ready();

Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/storage_persistent.c	2009-03-16 13:28:32 UTC (rev 3938)
+++ trunk/varnish-cache/bin/varnishd/storage_persistent.c	2009-03-16 13:32:36 UTC (rev 3939)
@@ -123,6 +123,8 @@
 	struct smp_signctx	ban2;
 	struct smp_signctx	seg1;
 	struct smp_signctx	seg2;
+
+	struct ban		*tailban;
 };
 
 /*
@@ -622,6 +624,7 @@
 		}
 
 fprintf(stderr, "BAN {%g %u %u \"%s\"}\n", t0, flags, length, ptr);
+		BAN_Reload(t0, flags, (const char *)ptr);
 
 		ptr += length;
 	}
@@ -906,6 +909,9 @@
 	if (smp_open_segs(sc, &sc->seg1))
 		AZ(smp_open_segs(sc, &sc->seg2));
 
+	sc->tailban = BAN_TailRef();
+	AN(sc->tailban);
+
 	/* XXX: save segments to ensure consistency between seg1 & seg2 ? */
 
 	/* Open a new segment, so we are ready to write */

Added: trunk/varnish-cache/bin/varnishtest/tests/p00002.vtc
===================================================================
--- trunk/varnish-cache/bin/varnishtest/tests/p00002.vtc	                        (rev 0)
+++ trunk/varnish-cache/bin/varnishtest/tests/p00002.vtc	2009-03-16 13:32:36 UTC (rev 3939)
@@ -0,0 +1,50 @@
+# $Id$
+
+test "Ban a persistent object"
+
+shell "rm -f /tmp/__v1/_.per[12]"
+
+server s1 {
+	rxreq 
+	txresp -hdr "Foo: foo"
+} -start
+
+varnish v1 \
+	-arg "-pdiag_bitmap=0x20000" \
+	-arg "-spersistent,/tmp/__v1/_.per1,10m" \
+	-arg "-spersistent,/tmp/__v1/_.per2,10m" \
+	-vcl+backend { } -start 
+
+client c1 {
+	txreq -url "/"
+	rxresp
+	expect resp.status == 200
+	expect resp.http.X-Varnish == "1001"
+	expect resp.http.foo == "foo"
+} -run
+
+varnish v1 -cliok "purge req.url == / && req.http.jam != session"
+varnish v1 -stop
+server s1 -wait
+
+server s1 {
+	rxreq 
+	txresp -hdr "Foo: bar"
+} -start
+
+varnish v1 -start
+
+varnish v1 -cliok purge.list
+
+varnish v1 -expect n_purge == 1
+
+# client c1 {
+#	txreq -url "/"
+#	rxresp
+#	expect resp.status == 200
+#	expect resp.http.X-Varnish == "1001"
+#	expect resp.http.foo == "bar"
+#} -run
+#
+#varnish v1 -stop
+

Modified: trunk/varnish-cache/include/persistent.h
===================================================================
--- trunk/varnish-cache/include/persistent.h	2009-03-16 13:28:32 UTC (rev 3938)
+++ trunk/varnish-cache/include/persistent.h	2009-03-16 13:32:36 UTC (rev 3939)
@@ -108,7 +108,7 @@
 	char			ident[8];
 	uint32_t		unique;
 	uint64_t		mapped;
-	uint64_t		length;
+	uint64_t		length;		/* NB: Must be last */
 };
 
 #define SMP_SIGN_SIZE		(8 + 4 + 8 + 8)



More information about the varnish-commit mailing list