r279 - trunk/varnish-cache/bin/varnishd
phk at projects.linpro.no
phk at projects.linpro.no
Mon Jul 3 14:41:14 CEST 2006
Author: phk
Date: 2006-07-03 14:41:14 +0200 (Mon, 03 Jul 2006)
New Revision: 279
Added:
trunk/varnish-cache/bin/varnishd/hash_classic.c
trunk/varnish-cache/bin/varnishd/hash_slinger.h
Modified:
trunk/varnish-cache/bin/varnishd/Makefile.am
trunk/varnish-cache/bin/varnishd/cache.h
trunk/varnish-cache/bin/varnishd/cache_fetch.c
trunk/varnish-cache/bin/varnishd/cache_hash.c
trunk/varnish-cache/bin/varnishd/cache_pool.c
trunk/varnish-cache/bin/varnishd/hash_simple_list.c
trunk/varnish-cache/bin/varnishd/heritage.h
trunk/varnish-cache/bin/varnishd/mgt.h
trunk/varnish-cache/bin/varnishd/rfc2616.c
trunk/varnish-cache/bin/varnishd/varnishd.c
Log:
Add another hash-method with better real-world survival chances: A classic
bucketed hash table of lists. Hash is MD5. Number of buckets and number
of mutexes can be configured at command line.
Modified: trunk/varnish-cache/bin/varnishd/Makefile.am
===================================================================
--- trunk/varnish-cache/bin/varnishd/Makefile.am 2006-07-03 08:30:33 UTC (rev 278)
+++ trunk/varnish-cache/bin/varnishd/Makefile.am 2006-07-03 12:41:14 UTC (rev 279)
@@ -21,6 +21,7 @@
cache_vrt.c \
cli_event.c \
hash_simple_list.c \
+ hash_classic.c \
mgt_child.c \
rfc2616.c \
storage_file.c \
Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h 2006-07-03 08:30:33 UTC (rev 278)
+++ trunk/varnish-cache/bin/varnishd/cache.h 2006-07-03 12:41:14 UTC (rev 279)
@@ -27,21 +27,8 @@
struct worker;
#endif
-/* Hashing -----------------------------------------------------------*/
+#include "hash_slinger.h"
-typedef void hash_init_f(void);
-typedef struct objhead *hash_lookup_f(const char *key, struct objhead *nobj);
-typedef int hash_deref_f(struct objhead *obj);
-
-struct hash_slinger {
- const char *name;
- hash_init_f *init;
- hash_lookup_f *lookup;
- hash_deref_f *deref;
-};
-
-extern struct hash_slinger hsl_slinger;
-
/* Storage -----------------------------------------------------------*/
struct storage {
Modified: trunk/varnish-cache/bin/varnishd/cache_fetch.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-03 08:30:33 UTC (rev 278)
+++ trunk/varnish-cache/bin/varnishd/cache_fetch.c 2006-07-03 12:41:14 UTC (rev 279)
@@ -245,7 +245,7 @@
sp->obj->xid = sp->xid;
fd = VBE_GetFd(sp->backend, &fd_token, sp->xid);
- assert(fd != -1);
+ assert(fd != -1); /* XXX: handle this */
VSL(SLT_Backend, sp->fd, "%d %s", fd, sp->backend->vcl_name);
hp = http_New();
Modified: trunk/varnish-cache/bin/varnishd/cache_hash.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-03 08:30:33 UTC (rev 278)
+++ trunk/varnish-cache/bin/varnishd/cache_hash.c 2006-07-03 12:41:14 UTC (rev 279)
@@ -14,6 +14,7 @@
#include "libvarnish.h"
#include "shmlog.h"
+#include "heritage.h"
#include "cache.h"
static struct hash_slinger *hash;
@@ -129,7 +130,7 @@
HSH_Init(void)
{
- hash = &hsl_slinger;
- if (hash->init != NULL)
- hash->init();
+ hash = heritage.hash;
+ if (hash->start != NULL)
+ hash->start();
}
Modified: trunk/varnish-cache/bin/varnishd/cache_pool.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-03 08:30:33 UTC (rev 278)
+++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-07-03 12:41:14 UTC (rev 279)
@@ -1,5 +1,7 @@
/*
* $Id$
+ *
+ * XXX: automatic thread-pool size adaptation.
*/
#include <stdio.h>
@@ -14,6 +16,7 @@
#include <event.h>
#include "libvarnish.h"
+#include "heritage.h"
#include "shmlog.h"
#include "vcl.h"
#include "cache.h"
@@ -154,9 +157,10 @@
AZ(pthread_cond_init(&shdcnd, NULL));
- for (i = 0; i < 5; i++)
+ for (i = 0; i < heritage.wthread_min; i++) {
AZ(pthread_create(&tp, NULL, CacheWorker, NULL));
- AZ(pthread_detach(tp));
+ AZ(pthread_detach(tp));
+ }
srandomdev();
xids = random();
}
Added: trunk/varnish-cache/bin/varnishd/hash_classic.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/hash_classic.c 2006-07-03 08:30:33 UTC (rev 278)
+++ trunk/varnish-cache/bin/varnishd/hash_classic.c 2006-07-03 12:41:14 UTC (rev 279)
@@ -0,0 +1,188 @@
+/*
+ * $Id$
+ *
+ * A classic bucketed hash
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <pthread.h>
+#include <queue.h>
+#include <sys/types.h>
+#include <md5.h>
+
+#include <libvarnish.h>
+#include <cache.h>
+
+/*--------------------------------------------------------------------*/
+
+struct hcl_entry {
+ TAILQ_ENTRY(hcl_entry) list;
+ char *key;
+ struct objhead *obj;
+ unsigned refcnt;
+ unsigned hash;
+ unsigned mtx;
+};
+
+TAILQ_HEAD(hcl_head, hcl_entry);
+
+static struct hcl_head *hcl_head;
+static unsigned hcl_nhash = 256;
+static unsigned hcl_nmtx = 16;
+static pthread_mutex_t *hcl_mutex;
+
+/*--------------------------------------------------------------------
+ * The ->init method allows the management process to pass arguments
+ */
+
+static int
+hcl_init(const char *p)
+{
+ int i;
+ unsigned u1, u2;
+
+ i = sscanf(p, "%u,%u", &u1, &u2);
+ if (i == 0)
+ return (0);
+ if (u1 == 0 || (i == 2 && (u2 == 0 || u2 > u1))) {
+ fprintf(stderr, "Invallid parameters to hash \"classic\":\n");
+ fprintf(stderr,
+ "\t-h classic,<bucket count>[,<buckets per mutex>]\n");
+ return (1);
+ }
+ hcl_nhash = u1;
+ if (i == 1) {
+ hcl_nmtx = hcl_nhash / 16;
+ if (hcl_nmtx < 1)
+ hcl_nmtx = 1;
+ return(0);
+ } else {
+ hcl_nmtx = hcl_nhash / u2;
+ if (hcl_nmtx < 1)
+ hcl_nmtx = 1;
+ }
+ fprintf(stderr, "Classic hash: %u buckets %u mutexes\n",
+ hcl_nhash, hcl_nmtx);
+ return (0);
+}
+
+/*--------------------------------------------------------------------
+ * The ->start method is called during cache process start and allows
+ * initialization to happen before the first lookup.
+ */
+
+static void
+hcl_start(void)
+{
+ unsigned u;
+
+ hcl_head = calloc(sizeof *hcl_head, hcl_nhash);
+ assert(hcl_head != NULL);
+ hcl_mutex = calloc(sizeof *hcl_mutex, hcl_nmtx);
+ assert(hcl_mutex != NULL);
+
+
+ for (u = 0; u < hcl_nhash; u++)
+ TAILQ_INIT(&hcl_head[u]);
+
+ for (u = 0; u < hcl_nmtx; u++)
+ AZ(pthread_mutex_init(&hcl_mutex[u], NULL));
+}
+
+/*--------------------------------------------------------------------
+ * Lookup and possibly insert element.
+ * If nobj != NULL and the lookup does not find key, nobj is inserted.
+ * If nobj == NULL and the lookup does not find key, NULL is returned.
+ * A reference to the returned object is held.
+ */
+
+static struct objhead *
+hcl_lookup(const char *key, struct objhead *nobj)
+{
+ struct hcl_entry *he, *he2;
+ MD5_CTX c;
+ unsigned char md5[MD5_DIGEST_LENGTH];
+ unsigned u1, u2;
+ int i;
+
+ MD5Init(&c);
+ MD5Update(&c, key, strlen(key));
+ MD5Final(md5, &c);
+ memcpy(&u1, md5, sizeof u1);
+ u1 %= hcl_nhash;
+ memcpy(&u2, md5 + sizeof u1, sizeof u2);
+ u2 %= hcl_nmtx;
+
+ AZ(pthread_mutex_lock(&hcl_mutex[u2]));
+ TAILQ_FOREACH(he, &hcl_head[u1], list) {
+ i = strcmp(key, he->key);
+ if (i < 0)
+ continue;
+ if (i == 0) {
+ he->refcnt++;
+ nobj = he->obj;
+ nobj->hashpriv = he;
+ AZ(pthread_mutex_unlock(&hcl_mutex[u2]));
+ return (nobj);
+ }
+ if (nobj == NULL) {
+ AZ(pthread_mutex_unlock(&hcl_mutex[u2]));
+ return (NULL);
+ }
+ break;
+ }
+ he2 = calloc(sizeof *he2, 1);
+ assert(he2 != NULL);
+ he2->obj = nobj;
+ he2->refcnt = 1;
+ he2->hash = u1;
+ he2->mtx = u2;
+ he2->key = strdup(key);
+ assert(he2->key != NULL);
+ nobj->hashpriv = he2;
+ if (he != NULL)
+ TAILQ_INSERT_BEFORE(he, he2, list);
+ else
+ TAILQ_INSERT_TAIL(&hcl_head[u1], he2, list);
+ AZ(pthread_mutex_unlock(&hcl_mutex[u2]));
+ return (nobj);
+}
+
+/*--------------------------------------------------------------------
+ * Dereference and if no references are left, free.
+ */
+
+static int
+hcl_deref(struct objhead *obj)
+{
+ struct hcl_entry *he;
+ int ret;
+ unsigned mtx;
+
+ assert(obj->hashpriv != NULL);
+ he = obj->hashpriv;
+ mtx = he->mtx;
+ AZ(pthread_mutex_lock(&hcl_mutex[mtx]));
+ if (--he->refcnt == 0) {
+ free(he->key);
+ TAILQ_REMOVE(&hcl_head[he->hash], he, list);
+ free(he);
+ ret = 0;
+ } else
+ ret = 1;
+ AZ(pthread_mutex_unlock(&hcl_mutex[mtx]));
+ return (ret);
+}
+
+/*--------------------------------------------------------------------*/
+
+struct hash_slinger hcl_slinger = {
+ "classic",
+ hcl_init,
+ hcl_start,
+ hcl_lookup,
+ hcl_deref,
+};
Modified: trunk/varnish-cache/bin/varnishd/hash_simple_list.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2006-07-03 08:30:33 UTC (rev 278)
+++ trunk/varnish-cache/bin/varnishd/hash_simple_list.c 2006-07-03 12:41:14 UTC (rev 279)
@@ -112,6 +112,7 @@
struct hash_slinger hsl_slinger = {
"simple_list",
+ NULL,
hsl_init,
hsl_lookup,
hsl_deref,
Added: trunk/varnish-cache/bin/varnishd/hash_slinger.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/hash_slinger.h 2006-07-03 08:30:33 UTC (rev 278)
+++ trunk/varnish-cache/bin/varnishd/hash_slinger.h 2006-07-03 12:41:14 UTC (rev 279)
@@ -0,0 +1,16 @@
+/*
+ * $Id$
+ */
+
+typedef int hash_init_f(const char *);
+typedef void hash_start_f(void);
+typedef struct objhead *hash_lookup_f(const char *key, struct objhead *nobj);
+typedef int hash_deref_f(struct objhead *obj);
+
+struct hash_slinger {
+ const char *name;
+ hash_init_f *init;
+ hash_start_f *start;
+ hash_lookup_f *lookup;
+ hash_deref_f *deref;
+};
Modified: trunk/varnish-cache/bin/varnishd/heritage.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/heritage.h 2006-07-03 08:30:33 UTC (rev 278)
+++ trunk/varnish-cache/bin/varnishd/heritage.h 2006-07-03 12:41:14 UTC (rev 279)
@@ -31,7 +31,13 @@
/* Storage method */
struct stevedore *stevedore;
+ /* Hash method */
+ struct hash_slinger *hash;
+
unsigned default_ttl;
+
+ /* Worker threads */
+ unsigned wthread_min, wthread_max;
};
extern struct heritage heritage;
Modified: trunk/varnish-cache/bin/varnishd/mgt.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/mgt.h 2006-07-03 08:30:33 UTC (rev 278)
+++ trunk/varnish-cache/bin/varnishd/mgt.h 2006-07-03 12:41:14 UTC (rev 279)
@@ -19,5 +19,10 @@
extern struct stevedore sma_stevedore;
extern struct stevedore smf_stevedore;
+#include "hash_slinger.h"
+
+extern struct hash_slinger hsl_slinger;
+extern struct hash_slinger hcl_slinger;
+
void VSL_MgtInit(const char *fn, unsigned size);
extern struct varnish_stats *VSL_stats;
Modified: trunk/varnish-cache/bin/varnishd/rfc2616.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-03 08:30:33 UTC (rev 278)
+++ trunk/varnish-cache/bin/varnishd/rfc2616.c 2006-07-03 12:41:14 UTC (rev 279)
@@ -9,6 +9,7 @@
#include "cache.h"
#include "libvarnish.h"
#include "heritage.h"
+
/*--------------------------------------------------------------------
* From RFC2616, 13.2.3 Age Calculations
*
@@ -106,6 +107,7 @@
switch (http_GetStatus(hp)) {
case 200: /* OK */
sp->obj->valid = 1;
+ /* FALLTHROUGH */
case 203: /* Non-Authoritative Information */
case 300: /* Multiple Choices */
case 301: /* Moved Permanently */
Modified: trunk/varnish-cache/bin/varnishd/varnishd.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-03 08:30:33 UTC (rev 278)
+++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-07-03 12:41:14 UTC (rev 279)
@@ -297,26 +297,47 @@
/*--------------------------------------------------------------------*/
+static int
+cmp_hash(struct hash_slinger *s, const char *p, const char *q)
+{
+ if (strlen(s->name) != q - p)
+ return (1);
+ if (strncmp(s->name, p, q - p))
+ return (1);
+ return (0);
+}
+
static void
-usage(void)
+setup_hash(const char *sflag)
{
- fprintf(stderr, "usage: varnishd [options]\n");
- fprintf(stderr, " %-28s # %s\n", "-b", "backend_IP_number");
- fprintf(stderr, " %-28s # %s\n", "-d", "debug");
- fprintf(stderr, " %-28s # %s\n", "-f", "VCL_file");
- fprintf(stderr, " %-28s # %s\n", "-p number", "TCP listen port");
- fprintf(stderr, " %-28s # %s\n",
- "-s kind[,storageoptions]", "Backend storage specification");
- fprintf(stderr, " %-28s # %s\n", "-t", "Default TTL");
-#if 0
- -c clusterid at cluster_controller
- -m memory_limit
- -s kind[,storage-options]
- -l logfile,logsize
- -u uid
- -a CLI_port
-#endif
- exit(1);
+ const char *p, *q;
+ struct hash_slinger *hp;
+
+ p = strchr(sflag, ',');
+ if (p == NULL)
+ q = p = strchr(sflag, '\0');
+ else
+ q = p + 1;
+ assert(p != NULL);
+ assert(q != NULL);
+ if (!cmp_hash(&hcl_slinger, sflag, p)) {
+ hp = &hcl_slinger;
+ } else if (!cmp_hash(&hsl_slinger, sflag, p)) {
+ hp = &hsl_slinger;
+ } else {
+ fprintf(stderr, "Unknown hash method \"%*.*s\"\n",
+ p - sflag, p - sflag, sflag);
+ exit (2);
+ }
+ heritage.hash = hp;
+ if (hp->init != NULL) {
+ if (hp->init(q))
+ exit (1);
+ } else if (*q) {
+ fprintf(stderr, "Hash method \"%s\" takes no arguments\n",
+ hp->name);
+ exit (1);
+ }
}
/*--------------------------------------------------------------------*/
@@ -361,6 +382,36 @@
/*--------------------------------------------------------------------*/
+static void
+usage(void)
+{
+ fprintf(stderr, "usage: varnishd [options]\n");
+ fprintf(stderr, " %-28s # %s\n", "-b backend",
+ "backend IP or hostname");
+ fprintf(stderr, " %-28s # %s\n", "-d", "debug");
+ fprintf(stderr, " %-28s # %s\n", "-f file", "VCL_file");
+ fprintf(stderr, " %-28s # %s\n",
+ "-h kind[,hashoptions]", "Hash specification");
+ fprintf(stderr, " %-28s # %s\n", "-p number", "TCP listen port");
+ fprintf(stderr, " %-28s # %s\n",
+ "-s kind[,storageoptions]", "Backend storage specification");
+ fprintf(stderr, " %-28s # %s\n", "-t", "Default TTL");
+ fprintf(stderr, " %-28s # %s\n", "-w int[,int]",
+ "Number of worker threads (fixed/{min,max})");
+#if 0
+ -c clusterid at cluster_controller
+ -m memory_limit
+ -s kind[,storage-options]
+ -l logfile,logsize
+ -u uid
+ -a CLI_port
+#endif
+ exit(1);
+}
+
+
+/*--------------------------------------------------------------------*/
+
/* for development purposes */
#include <printf.h>
#include <err.h>
@@ -368,20 +419,24 @@
int
main(int argc, char *argv[])
{
- int o;
+ int o, i;
+ unsigned ua, ub;
const char *portnumber = "8080";
unsigned dflag = 1; /* XXX: debug=on for now */
const char *bflag = NULL;
const char *fflag = NULL;
const char *sflag = "file";
+ const char *hflag = "classic";
register_printf_render_std((const unsigned char *)"HVQ");
VCC_InitCompile();
heritage.default_ttl = 120;
+ heritage.wthread_min = 5;
+ heritage.wthread_max = 5;
- while ((o = getopt(argc, argv, "b:df:p:s:t:")) != -1)
+ while ((o = getopt(argc, argv, "b:df:h:p:s:t:w:")) != -1)
switch (o) {
case 'b':
bflag = optarg;
@@ -392,6 +447,9 @@
case 'f':
fflag = optarg;
break;
+ case 'h':
+ hflag = optarg;
+ break;
case 'p':
portnumber = optarg;
break;
@@ -401,6 +459,15 @@
case 't':
heritage.default_ttl = strtoul(optarg, NULL, 0);
break;
+ case 'w':
+ i = sscanf(optarg, "%u,%u", &ua, &ub);
+ if (i == 0)
+ usage();
+ heritage.wthread_min = ua;
+ heritage.wthread_max = ua;
+ if (i == 2)
+ heritage.wthread_max = ub;
+ break;
default:
usage();
}
@@ -430,6 +497,7 @@
exit (1);
setup_storage(sflag);
+ setup_hash(hflag);
/*
* XXX: Lacking the suspend/resume facility (due to the socket API
More information about the varnish-commit
mailing list