r3935 - trunk/varnish-cache/bin/varnishd

phk at projects.linpro.no phk at projects.linpro.no
Mon Mar 16 12:55:39 CET 2009


Author: phk
Date: 2009-03-16 12:55:38 +0100 (Mon, 16 Mar 2009)
New Revision: 3935

Modified:
   trunk/varnish-cache/bin/varnishd/cache_ban.c
   trunk/varnish-cache/bin/varnishd/storage_persistent.c
Log:
Implement more convenient primitives for the on-disk signatures.

Wrap the ident struct in a signature as well for consistency.



Modified: trunk/varnish-cache/bin/varnishd/cache_ban.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_ban.c	2009-03-16 09:49:11 UTC (rev 3934)
+++ trunk/varnish-cache/bin/varnishd/cache_ban.c	2009-03-16 11:55:38 UTC (rev 3935)
@@ -332,6 +332,8 @@
 		be->refcount++;
 	} else
 		be = NULL;
+
+	SMP_NewBan(b->t0, b->test);
 	Lck_Unlock(&ban_mtx);
 
 	if (be == NULL)
@@ -349,7 +351,6 @@
 		bi->flags |= BAN_F_GONE;
 		pcount++;
 	}
-	SMP_NewBan(b->t0, b->test);
 	Lck_Lock(&ban_mtx);
 	be->refcount--;
 	VSL_stats->n_purge_dups += pcount;

Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/storage_persistent.c	2009-03-16 09:49:11 UTC (rev 3934)
+++ trunk/varnish-cache/bin/varnishd/storage_persistent.c	2009-03-16 11:55:38 UTC (rev 3935)
@@ -37,6 +37,7 @@
 
 #include <errno.h>
 #include <stdio.h>
+#include <stddef.h>
 #include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
@@ -58,6 +59,13 @@
 #define MAP_NOSYNC 0 /* XXX Linux */
 #endif
 
+struct smp_signctx {
+	struct smp_sign		*ss;
+	struct SHA256Context	ctx;
+	uint32_t		unique;
+	const char		*id;
+};
+
 struct smp_sc;
 
 /* XXX: name confusion with on-media version ? */
@@ -81,6 +89,8 @@
 	uint32_t		maxobj;		/* Max number of objects */
 	struct smp_object	*objs;		/* objdesc copy */
 	uint64_t		next_addr;	/* next write address */
+
+	struct smp_signctx	ctx[1];
 };
 
 VTAILQ_HEAD(smp_seghead, smp_seg);
@@ -107,127 +117,156 @@
 	pthread_t		thread;
 
 	VTAILQ_ENTRY(smp_sc)	list;
+
+	struct smp_signctx	idn;
+	struct smp_signctx	ban1;
+	struct smp_signctx	ban2;
+	struct smp_signctx	seg1;
+	struct smp_signctx	seg2;
 };
 
+/*
+ * silos is unlocked, it only changes during startup when we are 
+ * single-threaded
+ */
 static VTAILQ_HEAD(,smp_sc)	silos = VTAILQ_HEAD_INITIALIZER(silos);
 
+/*********************************************************************
+ * SIGNATURE functions
+ */
+
+#define SIGN_DATA(ctx)	((void *)((ctx)->ss + 1))
+#define SIGN_END(ctx)	((void *)((int8_t *)SIGN_DATA(ctx) + (ctx)->ss->length))
+
 /*--------------------------------------------------------------------
- * Write a sha256hash after a sequence of bytes.
+ * debug
  */
 
 static void
-smp_make_hash(void *ptr, off_t len)
+smp_dump_sign(struct smp_signctx *ctx)
 {
-	struct SHA256Context c;
-	unsigned char *dest;
+	fprintf(stderr, "%p {%s %x %p %ju}\n", ctx, ctx->id, ctx->unique, ctx->ss, ctx->ss->length);
+}
 
-	dest = ptr;
-	dest += len;
-	SHA256_Init(&c);
-	SHA256_Update(&c, ptr, len);
-	SHA256_Final(dest, &c);
+
+/*--------------------------------------------------------------------
+ * Define a signature by location and identifier.
+ */
+
+static void
+smp_def_sign(struct smp_sc *sc, struct smp_signctx *ctx, uint64_t off, const char *id)
+{
+
+	AZ(off & 7);			/* Alignment */
+	assert(strlen(id) < sizeof ctx->ss->ident);
+
+	memset(ctx, 0, sizeof ctx);
+	ctx->ss = (void*)(sc->ptr + off);
+	ctx->unique = sc->unique;
+	ctx->id = id;
+fprintf(stderr, "DEF(%p %p %s)\n", ctx, ctx->ss, id);
 }
 
 /*--------------------------------------------------------------------
- * Check that a sequence of bytes matches the SHA256 stored behind it.
+ * Check that a signature is good, leave state ready for append
  */
-
 static int
-smp_check_hash(void *ptr, off_t len)
+smp_chk_sign(struct smp_signctx *ctx)
 {
-	struct SHA256Context c;
+	struct SHA256Context cx;
 	unsigned char sign[SHA256_LEN];
-	unsigned char *dest;
+	int r = 0;
 
-	dest = ptr;
-	dest += len;
-	SHA256_Init(&c);
-	SHA256_Update(&c, ptr, len);
-	SHA256_Final(sign, &c);
-	return(memcmp(sign, dest, sizeof sign));
+	if (strcmp(ctx->id, ctx->ss->ident))
+		r = 1;
+	else if (ctx->unique != ctx->ss->unique)
+		r = 2;
+	else if ((uintptr_t)ctx->ss != ctx->ss->mapped)
+		r = 3;
+	else {
+		SHA256_Init(&ctx->ctx);
+		SHA256_Update(&ctx->ctx, ctx->ss,
+		    offsetof(struct smp_sign, length));
+		SHA256_Update(&ctx->ctx, SIGN_DATA(ctx), ctx->ss->length);
+		cx = ctx->ctx;
+		SHA256_Update(&cx, &ctx->ss->length, sizeof(ctx->ss->length));
+		SHA256_Final(sign, &cx);
+		if (memcmp(sign, SIGN_END(ctx), sizeof sign))
+			r = 4;
+	} 
+if (r) {
+	fprintf(stderr, "CHK(%p %p %s) = %d\n",
+	    ctx, ctx->ss, ctx->ss->ident, r);
+	smp_dump_sign(ctx);
 }
+	return (r);
+}
 
 /*--------------------------------------------------------------------
- * Create or write a signature block covering a sequence of bytes.
+ * Append data to a signature
  */
-
 static void
-smp_create_sign(const struct smp_sc *sc, uint64_t adr, uint64_t len, const char *id)
+smp_append_sign(struct smp_signctx *ctx, void *ptr, uint32_t len)
 {
-	struct smp_sign *ss;
+	struct SHA256Context cx;
+	unsigned char sign[SHA256_LEN];
 
-	AZ(adr & 0x7);			/* Enforce alignment */
-
-	ss = (void*)(sc->ptr + adr);
-	memset(ss, 0, sizeof *ss);
-	assert(strlen(id) < sizeof ss->ident);
-	strcpy(ss->ident, id);
-	ss->unique = sc->unique;
-	ss->mapped = (uintptr_t)(sc->ptr + adr);
-	ss->length = len;
-	smp_make_hash(ss, sizeof *ss + len);
-fprintf(stderr, "CreateSign(%jx, %jx, %s)\n",
-    adr, len, id);
+	if (len != 0) {
+		SHA256_Update(&ctx->ctx, ptr, len);
+		ctx->ss->length += len;
+	}
+	cx = ctx->ctx;
+	SHA256_Update(&cx, &ctx->ss->length, sizeof(ctx->ss->length));
+	SHA256_Final(sign, &cx);
+	memcpy(SIGN_END(ctx), sign, sizeof sign);
+XXXAZ(smp_chk_sign(ctx));
 }
 
 /*--------------------------------------------------------------------
- * Force a write of a signature block to the backing store.
+ * Reset a signature to empty, prepare for appending.
  */
 
 static void
-smp_sync_sign(const struct smp_sc *sc, uint64_t adr, uint64_t len)
+smp_reset_sign(struct smp_signctx *ctx)
 {
-	int i;
 
-	AZ(adr & 0x7);			/* Enforce alignment */
-
-	i = msync(sc->ptr + adr,
-	    sizeof(struct smp_sign) + len + SHA256_LEN, MS_SYNC);
-fprintf(stderr, "SyncSign(%jx, %jx) = %d %s\n", adr, len, i, strerror(errno));
+	memset(ctx->ss, 0, sizeof *ctx->ss);
+	strcpy(ctx->ss->ident, ctx->id);
+	ctx->ss->unique = ctx->unique;
+	ctx->ss->mapped = (uintptr_t)ctx->ss;
+	SHA256_Init(&ctx->ctx);
+	SHA256_Update(&ctx->ctx, ctx->ss,
+	    offsetof(struct smp_sign, length));
+	smp_append_sign(ctx, NULL, 0);
 }
 
 /*--------------------------------------------------------------------
- * Check a signature block and return zero if OK.
+ * Force a write of a signature block to the backing store.
  */
 
-static int
-smp_check_sign(const struct smp_sc *sc, uint64_t adr, const char *id)
+static void
+smp_sync_sign(const struct smp_signctx *ctx)
 {
-	struct smp_sign *ss;
 
-	AZ(adr & 0x7);			/* Enforce alignment */
+	(void)ctx;
 
-	ss = (void*)(sc->ptr + adr);
-	assert(strlen(id) < sizeof ss->ident);
-	if (strcmp(id, ss->ident))
-		return (1);
-	if (ss->unique != sc->unique)
-		return (2);
-	if (ss->mapped != (uintptr_t)ss)
-		return (3);
-	return (smp_check_hash(ss, sizeof *ss + ss->length));
+#if 0
+	i = msync(sc->ptr + adr,
+	    sizeof(struct smp_sign) + len + SHA256_LEN, MS_SYNC);
+fprintf(stderr, "SyncSign(%jx, %jx) = %d %s\n", adr, len, i, strerror(errno));
+#endif
 }
 
 /*--------------------------------------------------------------------
- * Open a signature block, and return zero if it is valid.
+ * Create and force a new signature to backing store
  */
 
-static int
-smp_open_sign(const struct smp_sc *sc, uint64_t adr, void **ptr, uint64_t *len, const char *id)
+static void
+smp_new_sign(struct smp_sc *sc, struct smp_signctx *ctx, uint64_t off, const char *id)
 {
-	struct smp_sign *ss;
-	int i;
-	
-	AZ(adr & 0x7);			/* Enforce alignment */
-	AN(ptr);
-	AN(len);
-	ss = (void*)(sc->ptr + adr);
-	*ptr = (void*)(sc->ptr + adr + sizeof *ss);
-	*len = ss->length;
-	i = smp_check_sign(sc, adr, id);
-fprintf(stderr, "OpenSign(%jx, %s) -> {%p, %jx, %d}\n",
-    adr, id, *ptr, *len, i);
-	return (i);
+	smp_def_sign(sc, ctx, off, id);
+	smp_reset_sign(ctx);
+	smp_sync_sign(ctx);
 }
 
 /*--------------------------------------------------------------------
@@ -243,9 +282,13 @@
 
 	assert(strlen(SMP_IDENT_STRING) < sizeof si->ident);
 
+	/* Choose a new random number */
 	sc->unique = random();
 
-	si = (void*)sc->ptr;
+	smp_reset_sign(&sc->idn);
+	si = sc->ident;
+printf("NEW: %p\n", si);
+
 	memset(si, 0, sizeof *si);
 	strcpy(si->ident, SMP_IDENT_STRING);
 	si->byte_order = 0x12345678;
@@ -262,12 +305,15 @@
 	si->stuff[SMP_SEG2_STUFF] = si->stuff[SMP_SEG1_STUFF] + 1024*1024;
 	si->stuff[SMP_SPC_STUFF] = si->stuff[SMP_SEG2_STUFF] + 1024*1024;
 	si->stuff[SMP_END_STUFF] = si->mediasize;
-	smp_create_sign(sc, si->stuff[SMP_BAN1_STUFF], 0, "BAN 1");
-	smp_create_sign(sc, si->stuff[SMP_BAN2_STUFF], 0, "BAN 2");
-	smp_create_sign(sc, si->stuff[SMP_SEG1_STUFF], 0, "SEG 1");
-	smp_create_sign(sc, si->stuff[SMP_SEG2_STUFF], 0, "SEG 2");
+	assert(si->stuff[SMP_SPC_STUFF] < si->stuff[SMP_END_STUFF]);
 
-	smp_make_hash(si, sizeof *si);
+	smp_new_sign(sc, &sc->ban1, si->stuff[SMP_BAN1_STUFF], "BAN 1");
+	smp_new_sign(sc, &sc->ban2, si->stuff[SMP_BAN2_STUFF], "BAN 2");
+	smp_new_sign(sc, &sc->seg1, si->stuff[SMP_SEG1_STUFF], "SEG 1");
+	smp_new_sign(sc, &sc->seg2, si->stuff[SMP_SEG2_STUFF], "SEG 2");
+
+	smp_append_sign(&sc->idn, si, sizeof *si);
+	smp_sync_sign(&sc->idn);
 }
 
 /*--------------------------------------------------------------------
@@ -281,14 +327,17 @@
 	int i, j;
 
 	assert(strlen(SMP_IDENT_STRING) < sizeof si->ident);
-	si = (void*)sc->ptr;
+
+	if (smp_chk_sign(&sc->idn))
+		return (1);
+
+	si = sc->ident;
+printf("VALID: %p\n", si);
 	if (strcmp(si->ident, SMP_IDENT_STRING))
-		return (1);
+		return (2);
 	if (si->byte_order != 0x12345678)
-		return (2);
+		return (3);
 	if (si->size != sizeof *si)
-		return (3);
-	if (smp_check_hash(si, sizeof *si))
 		return (4);
 	if (si->major_version != 1)
 		return (5);
@@ -309,15 +358,20 @@
 	assert(si->stuff[SMP_SPC_STUFF] > si->stuff[3]);
 	assert(si->stuff[SMP_END_STUFF] == sc->mediasize);
 
+	smp_def_sign(sc, &sc->ban1, si->stuff[SMP_BAN1_STUFF], "BAN 1");
+	smp_def_sign(sc, &sc->ban2, si->stuff[SMP_BAN2_STUFF], "BAN 2");
+	smp_def_sign(sc, &sc->seg1, si->stuff[SMP_SEG1_STUFF], "SEG 1");
+	smp_def_sign(sc, &sc->seg2, si->stuff[SMP_SEG2_STUFF], "SEG 2");
+
 	/* We must have one valid BAN table */
-	i = smp_check_sign(sc, si->stuff[SMP_BAN1_STUFF], "BAN 1");
-	j = smp_check_sign(sc, si->stuff[SMP_BAN2_STUFF], "BAN 2");
+	i = smp_chk_sign(&sc->ban1);
+	j = smp_chk_sign(&sc->ban2);
 	if (i && j)
 		return (100 + i * 10 + j);
 
 	/* We must have one valid SEG table */
-	i = smp_check_sign(sc, si->stuff[SMP_SEG1_STUFF], "SEG 1");
-	j = smp_check_sign(sc, si->stuff[SMP_SEG2_STUFF], "SEG 2");
+	i = smp_chk_sign(&sc->seg1);
+	j = smp_chk_sign(&sc->seg2);
 	if (i && j)
 		return (200 + i * 10 + j);
 	return (0);
@@ -379,6 +433,9 @@
 	fprintf(stderr, "i = %d ms = %jd g = %u\n",
 	    i, (intmax_t)sc->mediasize, sc->granularity);
 
+	smp_def_sign(sc, &sc->idn, 0, "SILO");
+	sc->ident = SIGN_DATA(&sc->idn);
+
 	i = smp_valid_silo(sc);
 	fprintf(stderr, "Silo: %d\n", i);
 	if (i)
@@ -403,33 +460,33 @@
  */
 
 static void
-smp_save_seg(struct smp_sc *sc, uint64_t adr, const char *id)
+smp_save_seg(struct smp_sc *sc, struct smp_signctx *ctx)
 {
 	struct smp_segptr *ss;
 	struct smp_seg *sg;
-	void *ptr;
 	uint64_t length;
 
-	(void)smp_open_sign(sc, adr, &ptr, &length, id);
-	ss = ptr;
-	length = 0;
+	smp_reset_sign(ctx);
+	ss = SIGN_DATA(ctx);
 	VTAILQ_FOREACH(sg, &sc->segments, list) {
+		assert(sg->offset < sc->mediasize);
+		assert(sg->offset + sg->length <= sc->mediasize);
 		ss->offset = sg->offset;
 		ss->length = sg->length;
 		ss++;
 		length += sizeof *ss;
 fprintf(stderr, "WR SEG %jx %jx\n", sg->offset, sg->length);
 	}
-	smp_create_sign(sc, adr, length, id);
-	smp_sync_sign(sc, adr, length);
+	smp_append_sign(ctx, SIGN_DATA(ctx), length);
+	smp_sync_sign(ctx);
 }
 
 static void
 smp_save_segs(struct smp_sc *sc)
 {
 
-	smp_save_seg(sc, sc->ident->stuff[SMP_SEG1_STUFF], "SEG 1");
-	smp_save_seg(sc, sc->ident->stuff[SMP_SEG2_STUFF], "SEG 2");
+	smp_save_seg(sc, &sc->seg1);
+	smp_save_seg(sc, &sc->seg2);
 }
 
 /*--------------------------------------------------------------------
@@ -473,11 +530,19 @@
  * Add a new ban to all silos
  */
 
+static void
+smp_appendban(struct smp_sc *sc, double t0, const char *ban)
+{
+fprintf(stderr, "silo %p BAN %g %s\n", sc, t0, ban);
+}
+
 void
 SMP_NewBan(double t0, const char *ban)
 {
-	(void)t0;
-	(void)ban;
+	struct smp_sc *sc;
+
+	VTAILQ_FOREACH(sc, &silos, list)
+		smp_appendban(sc, t0, ban);
 }
 
 /*--------------------------------------------------------------------
@@ -554,10 +619,15 @@
 	struct smp_object *so;
 	uint32_t no;
 	double t_now = TIM_real();
+	struct smp_signctx ctx[1];
 
 	(void)sp;
-	if (smp_open_sign(sc, sg->offset, &ptr, &length, "SEGMENT"))
+	AN(sg->offset);
+	smp_def_sign(sc, ctx, sg->offset, "SEGMENT");
+	if (smp_chk_sign(ctx))
 		return;
+	ptr = SIGN_DATA(ctx);
+	length = ctx->ss->length;
 	fprintf(stderr, "Load Seg %p %jx\n", ptr, length);
 	ss = ptr;
 	fprintf(stderr, "Objlist %jx Nalloc %u\n", ss->objlist, ss->nalloc);
@@ -586,22 +656,27 @@
  */
 
 static int
-smp_open_segs(struct smp_sc *sc, int stuff, const char *id)
+smp_open_segs(struct smp_sc *sc, struct smp_signctx *ctx)
 {
-	void *ptr;
 	uint64_t length;
 	struct smp_segptr *ss;
 	struct smp_seg *sg;
+	int i;
 
-	if (smp_open_sign(sc, sc->ident->stuff[stuff], &ptr, &length, id))
-		return (1);
+	i = smp_chk_sign(ctx);	
+	if (i)
+		return (i);
+	ss = SIGN_DATA(ctx);
+	length = ctx->ss->length;
 
-	ss = ptr;
 	for(; length > 0; length -= sizeof *ss, ss ++) {
 		ALLOC_OBJ(sg, SMP_SEG_MAGIC);
 		AN(sg);
 		sg->offset = ss->offset;
 		sg->length = ss->length;
+		/* XXX: check that they are inside silo */
+		/* XXX: check that they don't overlap */
+		/* XXX: check that they are serial */
 		sg->sc = sc;
 		VTAILQ_INSERT_TAIL(&sc->segments, sg, list);
 fprintf(stderr, "RD SEG %jx %jx\n", sg->offset, sg->length);
@@ -617,8 +692,6 @@
 smp_new_seg(struct smp_sc *sc)
 {
 	struct smp_seg *sg, *sg2;
-	void *ptr;
-	uint64_t length;
 
 	ALLOC_OBJ(sg, SMP_SEG_MAGIC);
 	AN(sg);
@@ -630,22 +703,30 @@
 	/* XXX: find where it goes in silo */
 
 	sg2 = VTAILQ_LAST(&sc->segments, smp_seghead);
-	if (sg2 == NULL)
+	if (sg2 == NULL) {
 		sg->offset = sc->ident->stuff[SMP_SPC_STUFF];
-	else
+		assert(sc->ident->stuff[SMP_SPC_STUFF] < sc->mediasize);
+	} else {
 		sg->offset = sg2->offset + sg2->length;
+		assert(sg->offset < sc->mediasize);
+	}
 	sg->length = sc->ident->stuff[SMP_END_STUFF] - sg->offset;
 
+	assert(sg->offset + sg->length <= sc->mediasize);
+
 	VTAILQ_INSERT_TAIL(&sc->segments, sg, list);
 fprintf(stderr, "MK SEG %jx %jx\n", sg->offset, sg->length);
 
 	/* Neuter the new segment in case there is an old one there */
-	(void)smp_open_sign(sc, sg->offset, &ptr, &length, "SEGMENT");
-	memcpy(ptr, &sg->segment, sizeof sg->segment);
-	smp_create_sign(sc, sg->offset, sizeof sg->segment, "SEGMENT");
-	smp_sync_sign(sc, sg->offset, sizeof sg->segment);
+	AN(sg->offset);
+	smp_def_sign(sc, sg->ctx, sg->offset, "SEGMENT");
+	smp_reset_sign(sg->ctx);
+	memcpy(SIGN_DATA(sg->ctx), &sg->segment, sizeof sg->segment);
+	smp_append_sign(sg->ctx, &sg->segment, sizeof sg->segment);
+	smp_sync_sign(sg->ctx);
 
 	/* Then add it to the segment list. */
+	/* XXX: could be done cheaper with append ? */
 	smp_save_segs(sc);
 
 	/* Set up our allocation point */
@@ -664,8 +745,6 @@
 static void
 smp_close_seg(struct smp_sc *sc, struct smp_seg *sg)
 {
-	void *ptr;
-	uint64_t length;
 
 	(void)sc;
 	/* XXX: if segment is empty, delete instead */
@@ -679,9 +758,10 @@
 	sg->segment.nalloc = sg->nalloc;
 
 	/* Write it to silo */
-	(void)smp_open_sign(sc, sg->offset, &ptr, &length, "SEGMENT");
-	memcpy(ptr, &sg->segment, sizeof sg->segment);
-	smp_create_sign(sc, sg->offset, sizeof sg->segment, "SEGMENT");
+	smp_reset_sign(sg->ctx);
+	memcpy(SIGN_DATA(sg->ctx), &sg->segment, sizeof sg->segment);
+	smp_append_sign(sg->ctx, &sg->segment, sizeof sg->segment);
+	smp_sync_sign(sg->ctx);
 
 	sg->next_addr += sizeof *sg->objs * sg->nalloc;
 	sg->length = sg->next_addr - sg->offset;
@@ -730,15 +810,15 @@
 	/* We trust the parent to give us a valid silo, for good measure: */
 	AZ(smp_valid_silo(sc));
 
-	sc->ident = (void*)sc->ptr;
+	sc->ident = SIGN_DATA(&sc->idn);
 
 	/* XXX: read in bans */
 
 	/*
 	 * We attempt seg1 first, and if that fails, try seg2
 	 */
-	if (smp_open_segs(sc, SMP_SEG1_STUFF, "SEG 1"))
-		AZ(smp_open_segs(sc, SMP_SEG2_STUFF, "SEG 2"));
+	if (smp_open_segs(sc, &sc->seg1))
+		AZ(smp_open_segs(sc, &sc->seg2));
 
 	/* Open a new segment, so we are ready to write */
 	smp_new_seg(sc);



More information about the varnish-commit mailing list