r3907 - trunk/varnish-cache/bin/varnishd
phk at projects.linpro.no
phk at projects.linpro.no
Mon Mar 9 15:27:28 CET 2009
Author: phk
Date: 2009-03-09 15:27:28 +0100 (Mon, 09 Mar 2009)
New Revision: 3907
Modified:
trunk/varnish-cache/bin/varnishd/cache.h
trunk/varnish-cache/bin/varnishd/cache_center.c
trunk/varnish-cache/bin/varnishd/cache_hash.c
trunk/varnish-cache/bin/varnishd/hash_slinger.h
trunk/varnish-cache/bin/varnishd/storage_persistent.c
Log:
Add a new HSH_Insert() function which does that.
Clean up the hash generation around lookups a little bit.
Start a background thread on each persistent silo and stuff the
objects we find into the hash.
Mark these with a new objcore flag, and panic if we find this flag
on lookup.
Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h 2009-03-09 13:52:53 UTC (rev 3906)
+++ trunk/varnish-cache/bin/varnishd/cache.h 2009-03-09 14:27:28 UTC (rev 3907)
@@ -282,6 +282,7 @@
#define OC_F_ONLRU (1<<0)
#define OC_F_BUSY (1<<1)
#define OC_F_PASS (1<<2)
+#define OC_F_PERSISTENT (1<<3)
unsigned timer_idx;
VTAILQ_ENTRY(objcore) list;
VTAILQ_ENTRY(objcore) lru_list;
Modified: trunk/varnish-cache/bin/varnishd/cache_center.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_center.c 2009-03-09 13:52:53 UTC (rev 3906)
+++ trunk/varnish-cache/bin/varnishd/cache_center.c 2009-03-09 14:27:28 UTC (rev 3907)
@@ -715,9 +715,10 @@
CHECK_OBJ_NOTNULL(sp->vcl, VCL_CONF_MAGIC);
if (sp->obj == NULL) {
- HSH_Prepare(sp, sp->vcl->nhashcount);
+ HSH_BeforeVclHash(sp, sp->vcl->nhashcount);
VCL_hash_method(sp);
assert(sp->handling == VCL_RET_HASH);
+ HSH_AfterVclHash(sp);
}
oc = HSH_Lookup(sp, &oh);
Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-03-09 13:52:53 UTC (rev 3906)
+++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2009-03-09 14:27:28 UTC (rev 3907)
@@ -234,7 +234,7 @@
}
void
-HSH_Prepare(struct sess *sp, unsigned nhashcount)
+HSH_BeforeVclHash(struct sess *sp, unsigned nhashcount)
{
char *p;
unsigned u;
@@ -258,6 +258,14 @@
}
void
+HSH_AfterVclHash(struct sess *sp)
+{
+
+ HSH_Prealloc(sp);
+ SHA256_Final(sp->wrk->nobjhead->digest, sp->wrk->sha256ctx);
+}
+
+void
HSH_AddString(struct sess *sp, const char *str)
{
int l;
@@ -365,7 +373,43 @@
/**********************************************************************/
+struct objcore *
+HSH_Insert(struct sess *sp)
+{
+ struct worker *w;
+ struct objhead *oh;
+ struct objcore *oc;
+ CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
+ CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
+ AN(hash);
+ w = sp->wrk;
+
+ HSH_Prealloc(sp);
+ if (params->diag_bitmap & 0x80000000)
+ hsh_testmagic(sp->wrk->nobjhead->digest);
+
+ AZ(sp->objhead);
+ AN(w->nobjhead);
+ oh = hash->lookup(sp, w->nobjhead);
+ CHECK_OBJ_NOTNULL(oh, OBJHEAD_MAGIC);
+ if (oh == w->nobjhead)
+ w->nobjhead = NULL;
+ Lck_Lock(&oh->mtx);
+
+ /* Insert (precreated) objcore in objecthead */
+ oc = w->nobjcore;
+ w->nobjcore = NULL;
+ AN(oc->flags & OC_F_BUSY);
+
+ /* XXX: Should this not be ..._HEAD now ? */
+ VTAILQ_INSERT_TAIL(&oh->objcs, oc, list);
+ /* NB: do not deref objhead the new object inherits our reference */
+ Lck_Unlock(&oh->mtx);
+ return (oc);
+}
+
+
struct objcore *
HSH_Lookup(struct sess *sp, struct objhead **poh)
{
@@ -383,7 +427,6 @@
w = sp->wrk;
HSH_Prealloc(sp);
- SHA256_Final(sp->wrk->nobjhead->digest, sp->wrk->sha256ctx);
if (params->diag_bitmap & 0x80000000)
hsh_testmagic(sp->wrk->nobjhead->digest);
@@ -409,6 +452,8 @@
assert(oh->refcnt > 1);
CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
+ XXXAZ(oc->flags & OC_F_PERSISTENT);
+
if (oc->flags & OC_F_BUSY) {
busy_oc = oc;
continue;
Modified: trunk/varnish-cache/bin/varnishd/hash_slinger.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/hash_slinger.h 2009-03-09 13:52:53 UTC (rev 3906)
+++ trunk/varnish-cache/bin/varnishd/hash_slinger.h 2009-03-09 14:27:28 UTC (rev 3907)
@@ -63,10 +63,11 @@
double HSH_Grace(double g);
void HSH_Init(void);
void HSH_AddString(struct sess *sp, const char *str);
-void HSH_Prepare(struct sess *sp, unsigned hashcount);
+void HSH_BeforeVclHash(struct sess *sp, unsigned hashcount);
+void HSH_AfterVclHash(struct sess *sp);
void HSH_DerefObjCore(struct sess *sp);
+struct objcore *HSH_Insert(struct sess *sp);
-
#ifdef VARNISH_CACHE_CHILD
#define DIGEST_LEN 32
Modified: trunk/varnish-cache/bin/varnishd/storage_persistent.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-03-09 13:52:53 UTC (rev 3906)
+++ trunk/varnish-cache/bin/varnishd/storage_persistent.c 2009-03-09 14:27:28 UTC (rev 3907)
@@ -90,6 +90,7 @@
struct smp_seghead segments;
struct smp_seg *cur_seg;
+ pthread_t thread;
};
/*--------------------------------------------------------------------
@@ -327,6 +328,7 @@
assert(sizeof(struct smp_ident) == SMP_IDENT_SIZE);
assert(sizeof(struct smp_sign) == SMP_SIGN_SIZE);
assert(sizeof(struct smp_object) == SMP_OBJECT_SIZE);
+ /* XXX: assert(sizeof smp_object.hash == DIGEST_LEN); */
/* Allocate softc */
ALLOC_OBJ(sc, SMP_SC_MAGIC);
@@ -416,27 +418,32 @@
*/
static void
-smp_load_seg(struct smp_sc *sc, struct smp_seg *sg)
+smp_load_seg(struct sess *sp, struct smp_sc *sc, struct smp_seg *sg)
{
void *ptr;
uint64_t length;
- struct smp_segment *sp;
+ struct smp_segment *ss;
struct smp_object *so;
uint32_t no;
double t_now = TIM_real();
+ (void)sp;
if (smp_open_sign(sc, sg->offset, &ptr, &length, "SEGMENT"))
return;
fprintf(stderr, "Load Seg %p %jx\n", ptr, length);
- sp = ptr;
- fprintf(stderr, "Objlist %jx Nalloc %u\n", sp->objlist, sp->nalloc);
- so = (void*)(sc->ptr + sp->objlist);
- no = sp->nalloc;
+ ss = ptr;
+ fprintf(stderr, "Objlist %jx Nalloc %u\n", ss->objlist, ss->nalloc);
+ so = (void*)(sc->ptr + ss->objlist);
+ no = ss->nalloc;
for (;no > 0; so++,no--) {
if (so->ttl < t_now)
continue;
fprintf(stderr, "OBJ %p dTTL: %g PTR %jx\n",
so, so->ttl - t_now, so->offset);
+ HSH_Prealloc(sp);
+ sp->wrk->nobjcore->flags |= OC_F_PERSISTENT;
+ memcpy(sp->wrk->nobjhead->digest, so->hash, SHA256_LEN);
+ (void)HSH_Insert(sp);
}
}
@@ -552,6 +559,27 @@
}
/*--------------------------------------------------------------------
+ * Silo worker thread
+ */
+
+static void *
+smp_thread(struct sess *sp, void *priv)
+{
+ struct smp_sc *sc;
+ struct smp_seg *sg;
+
+ (void)sp;
+ CAST_OBJ_NOTNULL(sc, priv, SMP_SC_MAGIC);
+
+ VTAILQ_FOREACH(sg, &sc->segments, list)
+ smp_load_seg(sp, sc, sg);
+
+ while (1)
+ sleep (1);
+ return (NULL);
+}
+
+/*--------------------------------------------------------------------
* Open a silo in the worker process
*/
@@ -559,7 +587,6 @@
smp_open(const struct stevedore *st)
{
struct smp_sc *sc;
- struct smp_seg *sg;
CAST_OBJ_NOTNULL(sc, st->priv, SMP_SC_MAGIC);
fprintf(stderr, "Open Silo(%p)\n", st);
@@ -577,10 +604,10 @@
if (smp_open_segs(sc, SMP_SEG1_STUFF, "SEG 1"))
AZ(smp_open_segs(sc, SMP_SEG2_STUFF, "SEG 2"));
- VTAILQ_FOREACH(sg, &sc->segments, list)
- smp_load_seg(sc, sg);
+ /* Open a new segment, so we are ready to write */
+ smp_new_seg(sc);
- smp_new_seg(sc);
+ WRK_BgThread(&sc->thread, "persistence", smp_thread, sc);
}
/*--------------------------------------------------------------------
More information about the varnish-commit
mailing list