r3454 - in trunk/varnish-cache: bin/varnishd include
phk at projects.linpro.no
phk at projects.linpro.no
Wed Dec 3 11:49:34 CET 2008
Author: phk
Date: 2008-12-03 11:49:34 +0100 (Wed, 03 Dec 2008)
New Revision: 3454
Modified:
trunk/varnish-cache/bin/varnishd/Makefile.am
trunk/varnish-cache/bin/varnishd/cache_hash.c
trunk/varnish-cache/bin/varnishd/hash_slinger.h
trunk/varnish-cache/bin/varnishd/mgt_param.c
trunk/varnish-cache/bin/varnishd/varnishd.c
trunk/varnish-cache/include/stat_field.h
Log:
Add preliminary version of lock-less tree based lookup (see below)
Enable SHA256 digests by default, and put it in the objhead. This
increases the size of the objhead by 32 bytes, but may drop
a bit again later, when other now unnecessary fields go away.
Test SHA256 for correct operation on startup.
About the "critbit" lookup:
To enable this, use "-hcritbit" argument.
"Crit Bit" trees, are also known under various other names, the original
version of the idea is probably the PATRICIA tree.
The basic concept is a tree structure which has nodes only where necessary
to tell the indices apart.
Our version of it, has some additional bells and whistles.
First lookups do not require any locks until we reach the objhead
we were looking for, or until we need to insert one which wasn't
there.
Second, the branch nodes are part of the objhead, as all but the
very first will need one, this saves malloc operations big time.
Now the down-sides:
There are still missing bits, amongst these the "cooling off" list,
for objheads that have been dereferenced, but where the branch-node
is not. Currently we just leak that memory.
There is a race relating to node deref and unlocked lookup that is
not closed, weird things may happen until I fix it.
I'd be interested to hear how long it survives before it croaks,
but apart from that, would not advocate that you use it, until
I fix those remaining issues.
Modified: trunk/varnish-cache/bin/varnishd/Makefile.am
===================================================================
--- trunk/varnish-cache/bin/varnishd/Makefile.am 2008-12-03 10:39:15 UTC (rev 3453)
+++ trunk/varnish-cache/bin/varnishd/Makefile.am 2008-12-03 10:49:34 UTC (rev 3454)
@@ -40,6 +40,7 @@
cache_vrt_re.c \
cache_ws.c \
hash_classic.c \
+ hash_critbit.c \
hash_simple_list.c \
instance.c \
mgt_child.c \
Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_hash.c 2008-12-03 10:39:15 UTC (rev 3453)
+++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2008-12-03 10:49:34 UTC (rev 3454)
@@ -251,7 +251,6 @@
struct http *h;
struct objhead *oh;
struct object *o, *busy_o, *grace_o;
- unsigned char sha256[32];
CHECK_OBJ_NOTNULL(sp, SESS_MAGIC);
CHECK_OBJ_NOTNULL(sp->wrk, WORKER_MAGIC);
@@ -259,12 +258,14 @@
AN(hash);
w = sp->wrk;
h = sp->http;
+
+ HSH_Prealloc(sp);
if (params->hash_sha256) {
- SHA256_Final(sha256, sp->wrk->sha256ctx);
+ SHA256_Final(sp->wrk->nobjhead->digest, sp->wrk->sha256ctx);
+ sp->wrk->nobjhead->digest_len = 32;
/* WSP(sp, SLT_Debug, "SHA256: <%.32s>", sha256); */
}
-
- HSH_Prealloc(sp);
+
if (sp->objhead != NULL) {
CHECK_OBJ_NOTNULL(sp->objhead, OBJHEAD_MAGIC);
oh = sp->objhead;
Modified: trunk/varnish-cache/bin/varnishd/hash_slinger.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/hash_slinger.h 2008-12-03 10:39:15 UTC (rev 3453)
+++ trunk/varnish-cache/bin/varnishd/hash_slinger.h 2008-12-03 10:49:34 UTC (rev 3454)
@@ -73,14 +73,24 @@
VTAILQ_HEAD(,object) objects;
char *hash;
unsigned hashlen;
+ unsigned char digest[32];
+ unsigned char digest_len;
VTAILQ_HEAD(, sess) waitinglist;
- /*------------------------------------------------------------
- * The fields below are for the sole private use of the hash
- * implementation.
+ /*----------------------------------------------------
+ * The fields below are for the sole private use of
+ * the hash implementation(s).
*/
- VTAILQ_ENTRY(objhead) hoh_list;
- void *hoh_head;
- unsigned hoh_digest;
+ union {
+ void *filler[3];
+ struct {
+ VTAILQ_ENTRY(objhead) u_n_hoh_list;
+ void *u_n_hoh_head;
+ unsigned u_n_hoh_digest;
+ } n;
+ } u;
+#define hoh_list u.n.u_n_hoh_list
+#define hoh_head u.n.u_n_hoh_head
+#define hoh_digest u.n.u_n_hoh_digest
};
#endif /* VARNISH_CACHE_CHILD */
Modified: trunk/varnish-cache/bin/varnishd/mgt_param.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt_param.c 2008-12-03 10:39:15 UTC (rev 3453)
+++ trunk/varnish-cache/bin/varnishd/mgt_param.c 2008-12-03 10:49:34 UTC (rev 3454)
@@ -713,7 +713,7 @@
{ "hash_sha256", tweak_bool, &master.hash_sha256, 0, 0,
"Use SHA256 compression of hash-strings",
0,
- "off", "bool" },
+ "on", "bool" },
{ "log_hashstring", tweak_bool, &master.log_hash, 0, 0,
"Log the hash string to shared memory log.\n",
0,
Modified: trunk/varnish-cache/bin/varnishd/varnishd.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/varnishd.c 2008-12-03 10:39:15 UTC (rev 3453)
+++ trunk/varnish-cache/bin/varnishd/varnishd.c 2008-12-03 10:49:34 UTC (rev 3454)
@@ -59,6 +59,7 @@
#include "vsb.h"
#include "vpf.h"
+#include "vsha256.h"
#include "cli.h"
#include "cli_priv.h"
@@ -157,11 +158,13 @@
extern struct hash_slinger hsl_slinger;
extern struct hash_slinger hcl_slinger;
+extern struct hash_slinger hcb_slinger;
static const struct choice hsh_choice[] = {
{ "classic", &hcl_slinger },
{ "simple", &hsl_slinger },
{ "simple_list", &hsl_slinger }, /* backwards compat */
+ { "critbit", &hcb_slinger },
{ NULL, NULL }
};
@@ -450,6 +453,11 @@
assert(TIM_parse("Sunday, 06-Nov-94 08:49:37 GMT") == 784111777);
assert(TIM_parse("Sun Nov 6 08:49:37 1994") == 784111777);
+ /*
+ * Check that our SHA256 works
+ */
+ SHA256_Test();
+
memset(cli, 0, sizeof cli);
cli[0].sb = vsb_newauto();
XXXAN(cli[0].sb);
Modified: trunk/varnish-cache/include/stat_field.h
===================================================================
--- trunk/varnish-cache/include/stat_field.h 2008-12-03 10:39:15 UTC (rev 3453)
+++ trunk/varnish-cache/include/stat_field.h 2008-12-03 10:49:34 UTC (rev 3454)
@@ -126,3 +126,7 @@
MAC_STAT(n_purge_obj_test, uint64_t, 'a', "N objects tested")
MAC_STAT(n_purge_re_test, uint64_t, 'a', "N regexps tested against")
MAC_STAT(n_purge_dups, uint64_t, 'a', "N duplicate purges removed")
+
+MAC_STAT(hcb_nolock, uint64_t, 'a', "HCB Lookups without lock")
+MAC_STAT(hcb_lock, uint64_t, 'a', "HCB Lookups with lock")
+MAC_STAT(hcb_insert, uint64_t, 'a', "HCB Inserts")
More information about the varnish-commit
mailing list