r84 - in trunk/varnish-cache: bin/varnishd include lib/libvcl
phk at projects.linpro.no
phk at projects.linpro.no
Thu Mar 30 09:05:10 CEST 2006
Author: phk
Date: 2006-03-30 09:05:10 +0200 (Thu, 30 Mar 2006)
New Revision: 84
Added:
trunk/varnish-cache/bin/varnishd/cache_pool.c
Modified:
trunk/varnish-cache/bin/varnishd/Makefile.am
trunk/varnish-cache/bin/varnishd/cache.h
trunk/varnish-cache/bin/varnishd/cache_acceptor.c
trunk/varnish-cache/bin/varnishd/cache_httpd.c
trunk/varnish-cache/bin/varnishd/cache_main.c
trunk/varnish-cache/bin/varnishd/cache_vcl.c
trunk/varnish-cache/bin/varnishd/varnishd.c
trunk/varnish-cache/include/vcl_lang.h
trunk/varnish-cache/lib/libvcl/vcl_compile.c
trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c
trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl
trunk/varnish-cache/lib/libvcl/vcl_token_defs.h
Log:
Integrate the VCL code closer in the cache process: The passed
argument will be the session structure.
Add the pool of worker threads (cache_pool). With a unitary nature
of the VCL code, the HTTP parsing can be postponed to the worker thread.
This actually helps us with CPU cache locality as it will reduce the
amount of memory allocated on one CPU but freed on another.
Modified: trunk/varnish-cache/bin/varnishd/Makefile.am
===================================================================
--- trunk/varnish-cache/bin/varnishd/Makefile.am 2006-03-27 14:12:07 UTC (rev 83)
+++ trunk/varnish-cache/bin/varnishd/Makefile.am 2006-03-30 07:05:10 UTC (rev 84)
@@ -8,6 +8,7 @@
cache_acceptor.c \
cache_httpd.c \
cache_main.c \
+ cache_pool.c \
cache_shmlog.c \
cache_vcl.c \
cli_event.c \
Modified: trunk/varnish-cache/bin/varnishd/cache.h
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache.h 2006-03-27 14:12:07 UTC (rev 83)
+++ trunk/varnish-cache/bin/varnishd/cache.h 2006-03-30 07:05:10 UTC (rev 84)
@@ -2,46 +2,16 @@
* $Id$
*/
-#define VCA_RXBUFSIZE 1024
-#define VCA_ADDRBUFSIZE 32
-
-struct sess {
- int fd;
-
- /* formatted ascii client address */
- char addr[VCA_ADDRBUFSIZE];
-
- /* Receive buffer for HTTP header */
- char rcv[VCA_RXBUFSIZE + 1];
- unsigned rcv_len;
-
- /* HTTP request info, points into rcv */
- const char *req_b;
- const char *req_e;
- const char *url_b;
- const char *url_e;
- const char *proto_b;
- const char *proto_e;
- const char *hdr_b;
- const char *hdr_e;
-
- enum {
- HND_Unclass,
- HND_Handle,
- HND_Pass
- } handling;
-
- /* Various internal stuff */
- struct event *rd_e;
- struct sessmem *mem;
-};
-
/* cache_acceptor.c */
void *vca_main(void *arg);
/* cache_httpd.c */
void HttpdAnalyze(struct sess *sp);
+/* cache_pool.c */
+void CacheInitPool(void);
+void DealWithSession(struct sess *sp);
+
/* cache_shmlog.c */
void VSL_Init(void);
#ifdef SHMLOGHEAD_MAGIC
Modified: trunk/varnish-cache/bin/varnishd/cache_acceptor.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-03-27 14:12:07 UTC (rev 83)
+++ trunk/varnish-cache/bin/varnishd/cache_acceptor.c 2006-03-30 07:05:10 UTC (rev 84)
@@ -17,6 +17,7 @@
#include <event.h>
#include "libvarnish.h"
+#include "vcl_lang.h"
#include "heritage.h"
#include "shmlog.h"
#include "cache.h"
@@ -65,10 +66,7 @@
}
sp->hdr_e = p;
event_del(sp->rd_e);
- HttpdAnalyze(sp);
-
- /* XXX: for now, pass everything */
- sp->handling = HND_Pass;
+ DealWithSession(sp);
}
static void
Modified: trunk/varnish-cache/bin/varnishd/cache_httpd.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_httpd.c 2006-03-27 14:12:07 UTC (rev 83)
+++ trunk/varnish-cache/bin/varnishd/cache_httpd.c 2006-03-30 07:05:10 UTC (rev 84)
@@ -9,6 +9,7 @@
#include "libvarnish.h"
#include "shmlog.h"
+#include "vcl_lang.h"
#include "cache.h"
void
@@ -20,23 +21,9 @@
/* First, isolate and possibly identify request type */
p = sp->req_b = sp->rcv;
- if (p[0] == 'G' && p[1] == 'E' && p[2] == 'T' && p[3] == ' ') {
- p = sp->req_e = p + 4;
- sp->handling = HND_Handle;
- } else if (p[0] == 'H' && p[1] == 'E' && p[2] == 'A' && p[3] == 'D'
- && p[4] == ' ') {
- p = sp->req_e = p + 5;
- sp->handling = HND_Handle;
- } else {
- /*
- * We don't bother to identify the rest, we won't handle
- * them in any case
- */
- for (q = p; isalpha(*q); q++)
- ;
- p = sp->req_e = q;
- sp->handling = HND_Pass;
- }
+ for (q = p; isalpha(*q); q++)
+ ;
+ p = sp->req_e = q;
VSLR(SLT_Request, sp->fd, sp->req_b, sp->req_e);
/* Next find the URI */
Modified: trunk/varnish-cache/bin/varnishd/cache_main.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_main.c 2006-03-27 14:12:07 UTC (rev 83)
+++ trunk/varnish-cache/bin/varnishd/cache_main.c 2006-03-30 07:05:10 UTC (rev 84)
@@ -18,6 +18,7 @@
#include "libvarnish.h"
#include "heritage.h"
#include "shmlog.h"
+#include "vcl_lang.h"
#include "cache.h"
#include "cli_event.h"
@@ -100,6 +101,7 @@
printf("Child starts\n");
VSL_Init();
+ CacheInitPool();
AZ(pthread_create(&vca_thread, NULL, vca_main, NULL));
Added: trunk/varnish-cache/bin/varnishd/cache_pool.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-03-27 14:12:07 UTC (rev 83)
+++ trunk/varnish-cache/bin/varnishd/cache_pool.c 2006-03-30 07:05:10 UTC (rev 84)
@@ -0,0 +1,61 @@
+/*
+ * $Id$
+ */
+
+#include <stdio.h>
+#include <assert.h>
+#include <pthread.h>
+
+#include "libvarnish.h"
+#include "vcl_lang.h"
+#include "cache.h"
+
+static TAILQ_HEAD(, sess) shd = TAILQ_HEAD_INITIALIZER(shd);
+
+static pthread_mutex_t shdmtx;
+static pthread_cond_t shdcnd;
+
+static void *
+CacheWorker(void *priv __unused)
+{
+ struct sess *sp;
+
+ while (1) {
+ AZ(pthread_mutex_lock(&shdmtx));
+ while (1) {
+ sp = TAILQ_FIRST(&shd);
+ if (sp != NULL)
+ break;
+ AZ(pthread_cond_wait(&shdcnd, &shdmtx));
+ }
+ TAILQ_REMOVE(&shd, sp, list);
+ AZ(pthread_mutex_unlock(&shdmtx));
+
+ HttpdAnalyze(sp);
+
+ /*
+ * XXX send session to acceptor for reuse/disposal
+ */
+ }
+}
+
+void
+DealWithSession(struct sess *sp)
+{
+ AZ(pthread_mutex_lock(&shdmtx));
+ TAILQ_INSERT_TAIL(&shd, sp, list);
+ AZ(pthread_mutex_unlock(&shdmtx));
+ AZ(pthread_cond_signal(&shdcnd));
+}
+
+void
+CacheInitPool(void)
+{
+ pthread_t tp;
+
+ AZ(pthread_mutex_init(&shdmtx, NULL));
+ AZ(pthread_cond_init(&shdcnd, NULL));
+
+ AZ(pthread_create(&tp, NULL, CacheWorker, NULL));
+ AZ(pthread_detach(tp));
+}
Modified: trunk/varnish-cache/bin/varnishd/cache_vcl.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-03-27 14:12:07 UTC (rev 83)
+++ trunk/varnish-cache/bin/varnishd/cache_vcl.c 2006-03-30 07:05:10 UTC (rev 84)
@@ -142,3 +142,13 @@
}
active_vcl = vcl;
}
+
+/*--------------------------------------------------------------------*/
+
+void
+VCL_pass(VCL_FARGS)
+{
+
+ sess->handling = HND_Pass;
+ sess->done++;
+}
Modified: trunk/varnish-cache/bin/varnishd/varnishd.c
===================================================================
--- trunk/varnish-cache/bin/varnishd/varnishd.c 2006-03-27 14:12:07 UTC (rev 83)
+++ trunk/varnish-cache/bin/varnishd/varnishd.c 2006-03-30 07:05:10 UTC (rev 84)
@@ -68,8 +68,23 @@
buf = NULL;
asprintf(&buf,
- "backend default { set backend.ip = %s; }",
- bflag);
+ "backend default { set backend.ip = %s; }\n"
+ "sub main {\n"
+ " pass;\n"
+#if 0
+ " if (req.request != \"GET\" && req.request != \"HEAD\") {\n"
+ " pass;\n"
+ " }\n"
+ " lookup;\n"
+ " if (!obj.valid) {\n"
+ " fetch;\n"
+ " if (obj.cacheable) {\n"
+ " insert;\n"
+ " }\n"
+ " }\n"
+#endif
+ "}\n"
+ "", bflag);
assert(buf != NULL);
sb = sbuf_new(NULL, NULL, 0, SBUF_AUTOEXTEND);
assert(sb != NULL);
Modified: trunk/varnish-cache/include/vcl_lang.h
===================================================================
--- trunk/varnish-cache/include/vcl_lang.h 2006-03-27 14:12:07 UTC (rev 83)
+++ trunk/varnish-cache/include/vcl_lang.h 2006-03-30 07:05:10 UTC (rev 84)
@@ -5,6 +5,7 @@
* XXX: *MUST* be rerun.
*/
+#include <sys/queue.h>
struct vcl_ref {
unsigned line;
@@ -18,19 +19,42 @@
unsigned mask;
};
-struct client {
- unsigned ip;
-};
+#define VCA_RXBUFSIZE 1024
+#define VCA_ADDRBUFSIZE 32
-struct req {
- char *req;
- char *useragent;
- struct {
- char *path;
- char *host;
- } url;
- double ttlfactor;
- struct backend *backend;
+struct sess {
+ int fd;
+
+ /* formatted ascii client address */
+ char addr[VCA_ADDRBUFSIZE];
+
+ /* Receive buffer for HTTP header */
+ char rcv[VCA_RXBUFSIZE + 1];
+ unsigned rcv_len;
+
+ /* HTTP request info, points into rcv */
+ const char *req_b;
+ const char *req_e;
+ const char *url_b;
+ const char *url_e;
+ const char *proto_b;
+ const char *proto_e;
+ const char *hdr_b;
+ const char *hdr_e;
+
+ enum {
+ HND_Unclass,
+ HND_Handle,
+ HND_Pass
+ } handling;
+
+ char done;
+
+ TAILQ_ENTRY(sess) list;
+
+ /* Various internal stuff */
+ struct event *rd_e;
+ struct sessmem *mem;
};
struct backend {
@@ -41,17 +65,9 @@
int down;
};
-struct obj {
- int exists;
- double ttl;
- unsigned result;
- unsigned size;
- unsigned usage;
-};
+#define VCL_FARGS struct sess *sess
+#define VCL_PASS_ARGS sess
-#define VCL_FARGS struct client *client, struct obj *obj, struct req *req, struct backend *backend
-#define VCL_PASS_ARGS client, obj, req, backend
-
void VCL_count(unsigned);
void VCL_no_cache();
void VCL_no_new_cache();
@@ -59,15 +75,18 @@
int string_match(const char *, const char *);
int VCL_rewrite(const char *, const char *);
int VCL_error(unsigned, const char *);
+void VCL_pass(VCL_FARGS);
int VCL_fetch(void);
int VCL_switch_config(const char *);
typedef void vcl_init_f(void);
+typedef void vcl_func_f(VCL_FARGS);
struct VCL_conf {
unsigned magic;
#define VCL_CONF_MAGIC 0x7406c509 /* from /dev/random */
vcl_init_f *init_func;
+ vcl_func_f *main_func;
struct backend *default_backend;
struct vcl_ref *ref;
unsigned nref;
Modified: trunk/varnish-cache/lib/libvcl/vcl_compile.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-03-27 14:12:07 UTC (rev 83)
+++ trunk/varnish-cache/lib/libvcl/vcl_compile.c 2006-03-30 07:05:10 UTC (rev 84)
@@ -818,6 +818,11 @@
I(tl);
sbuf_printf(tl->fc, "return;\n");
return;
+ case T_PASS:
+ I(tl);
+ sbuf_printf(tl->fc, "VCL_pass(VCL_PASS_ARGS);\n");
+ sbuf_printf(tl->fc, "return;\n");
+ return;
case T_FETCH:
I(tl);
sbuf_printf(tl->fc, "VCL_fetch();\n");
@@ -849,7 +854,8 @@
sbuf_printf(tl->fc, "VCL_function_%*.*s(VCL_PASS_ARGS);\n",
tl->t->e - tl->t->b,
tl->t->e - tl->t->b, tl->t->b);
- /* XXX: check if function finished request */
+ I(tl); sbuf_printf(tl->fc, "if (sess->done)\n");
+ I(tl); sbuf_printf(tl->fc, "\treturn;\n");
NextToken(tl);
return;
case T_REWRITE:
@@ -1364,6 +1370,8 @@
sbuf_printf(tl->fc,
"\t.init_func = VCL_Init,\n");
sbuf_printf(tl->fc,
+ "\t.main_func = VCL_function_main,\n");
+ sbuf_printf(tl->fc,
"\t.default_backend = &VCL_backend_default,\n");
sbuf_printf(tl->fc,
"\t.ref = VCL_ref,\n");
Modified: trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-03-27 14:12:07 UTC (rev 83)
+++ trunk/varnish-cache/lib/libvcl/vcl_fixed_token.c 2006-03-30 07:05:10 UTC (rev 84)
@@ -245,6 +245,11 @@
*q = p + 4;
return (T_PROC);
}
+ if (p[0] == 'p' && p[1] == 'a' && p[2] == 's' &&
+ p[3] == 's' && !isvar(p[4])) {
+ *q = p + 4;
+ return (T_PASS);
+ }
return (0);
case 'r':
if (p[0] == 'r' && p[1] == 'e' && p[2] == 'w' &&
@@ -360,6 +365,7 @@
vcl_tnames[T_NEQ] = "!=";
vcl_tnames[T_NO_CACHE] = "no_cache";
vcl_tnames[T_NO_NEW_CACHE] = "no_new_cache";
+ vcl_tnames[T_PASS] = "pass";
vcl_tnames[T_PROC] = "proc";
vcl_tnames[T_REWRITE] = "rewrite";
vcl_tnames[T_SET] = "set";
@@ -380,6 +386,7 @@
fputs(" * XXX: *MUST* be rerun.\n", f);
fputs(" */\n", f);
fputs("\n", f);
+ fputs("#include <sys/queue.h>\n", f);
fputs("\n", f);
fputs("struct vcl_ref {\n", f);
fputs(" unsigned line;\n", f);
@@ -393,19 +400,40 @@
fputs(" unsigned mask;\n", f);
fputs("};\n", f);
fputs("\n", f);
- fputs("struct client {\n", f);
- fputs(" unsigned ip;\n", f);
- fputs("};\n", f);
+ fputs("#define VCA_RXBUFSIZE 1024\n", f);
+ fputs("#define VCA_ADDRBUFSIZE 32\n", f);
fputs("\n", f);
- fputs("struct req {\n", f);
- fputs(" char *req;\n", f);
- fputs(" char *useragent;\n", f);
- fputs(" struct {\n", f);
- fputs(" char *path;\n", f);
- fputs(" char *host;\n", f);
- fputs(" } url;\n", f);
- fputs(" double ttlfactor;\n", f);
- fputs(" struct backend *backend;\n", f);
+ fputs("struct sess {\n", f);
+ fputs(" int fd;\n", f);
+ fputs("\n", f);
+ fputs(" /* formatted ascii client address */\n", f);
+ fputs(" char addr[VCA_ADDRBUFSIZE];\n", f);
+ fputs("\n", f);
+ fputs(" /* Receive buffer for HTTP header */\n", f);
+ fputs(" char rcv[VCA_RXBUFSIZE + 1];\n", f);
+ fputs(" unsigned rcv_len;\n", f);
+ fputs("\n", f);
+ fputs(" /* HTTP request info, points into rcv */\n", f);
+ fputs(" const char *req_b;\n", f);
+ fputs(" const char *req_e;\n", f);
+ fputs(" const char *url_b;\n", f);
+ fputs(" const char *url_e;\n", f);
+ fputs(" const char *proto_b;\n", f);
+ fputs(" const char *proto_e;\n", f);
+ fputs(" const char *hdr_b;\n", f);
+ fputs(" const char *hdr_e;\n", f);
+ fputs("\n", f);
+ fputs(" enum {\n", f);
+ fputs(" HND_Unclass,\n", f);
+ fputs(" HND_Handle,\n", f);
+ fputs(" HND_Pass\n", f);
+ fputs(" } handling;\n", f);
+ fputs("\n", f);
+ fputs(" char done;\n", f);
+ fputs("\n", f);
+ fputs(" /* Various internal stuff */\n", f);
+ fputs(" struct event *rd_e;\n", f);
+ fputs(" struct sessmem *mem;\n", f);
fputs("};\n", f);
fputs("\n", f);
fputs("struct backend {\n", f);
@@ -416,17 +444,9 @@
fputs(" int down;\n", f);
fputs("};\n", f);
fputs("\n", f);
- fputs("struct obj {\n", f);
- fputs(" int exists;\n", f);
- fputs(" double ttl;\n", f);
- fputs(" unsigned result;\n", f);
- fputs(" unsigned size;\n", f);
- fputs(" unsigned usage;\n", f);
- fputs("};\n", f);
+ fputs("#define VCL_FARGS struct sess *sess\n", f);
+ fputs("#define VCL_PASS_ARGS sess\n", f);
fputs("\n", f);
- fputs("#define VCL_FARGS struct client *client, struct obj *obj, struct req *req, struct backend *backend\n", f);
- fputs("#define VCL_PASS_ARGS client, obj, req, backend\n", f);
- fputs("\n", f);
fputs("void VCL_count(unsigned);\n", f);
fputs("void VCL_no_cache();\n", f);
fputs("void VCL_no_new_cache();\n", f);
@@ -434,15 +454,18 @@
fputs("int string_match(const char *, const char *);\n", f);
fputs("int VCL_rewrite(const char *, const char *);\n", f);
fputs("int VCL_error(unsigned, const char *);\n", f);
+ fputs("void VCL_pass(VCL_FARGS);\n", f);
fputs("int VCL_fetch(void);\n", f);
fputs("int VCL_switch_config(const char *);\n", f);
fputs("\n", f);
fputs("typedef void vcl_init_f(void);\n", f);
+ fputs("typedef void vcl_func_f(VCL_FARGS);\n", f);
fputs("\n", f);
fputs("struct VCL_conf {\n", f);
fputs(" unsigned magic;\n", f);
fputs("#define VCL_CONF_MAGIC 0x7406c509 /* from /dev/random */\n", f);
fputs(" vcl_init_f *init_func;\n", f);
+ fputs(" vcl_func_f *main_func;\n", f);
fputs(" struct backend *default_backend;\n", f);
fputs(" struct vcl_ref *ref;\n", f);
fputs(" unsigned nref;\n", f);
Modified: trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-03-27 14:12:07 UTC (rev 83)
+++ trunk/varnish-cache/lib/libvcl/vcl_gen_fixed_token.tcl 2006-03-30 07:05:10 UTC (rev 84)
@@ -13,6 +13,7 @@
backend
error
+ pass
fetch
call
no_cache
Modified: trunk/varnish-cache/lib/libvcl/vcl_token_defs.h
===================================================================
--- trunk/varnish-cache/lib/libvcl/vcl_token_defs.h 2006-03-27 14:12:07 UTC (rev 83)
+++ trunk/varnish-cache/lib/libvcl/vcl_token_defs.h 2006-03-30 07:05:10 UTC (rev 84)
@@ -14,30 +14,31 @@
#define T_ACL 135
#define T_BACKEND 136
#define T_ERROR 137
-#define T_FETCH 138
-#define T_CALL 139
-#define T_NO_CACHE 140
-#define T_NO_NEW_CACHE 141
-#define T_SET 142
-#define T_REWRITE 143
-#define T_FINISH 144
-#define T_SWITCH_CONFIG 145
-#define T_INC 146
-#define T_DEC 147
-#define T_CAND 148
-#define T_COR 149
-#define T_LEQ 150
-#define T_EQ 151
-#define T_NEQ 152
-#define T_GEQ 153
-#define T_SHR 154
-#define T_SHL 155
-#define T_INCR 156
-#define T_DECR 157
-#define T_MUL 158
-#define T_DIV 159
-#define ID 160
-#define VAR 161
-#define CNUM 162
-#define CSTR 163
-#define EOI 164
+#define T_PASS 138
+#define T_FETCH 139
+#define T_CALL 140
+#define T_NO_CACHE 141
+#define T_NO_NEW_CACHE 142
+#define T_SET 143
+#define T_REWRITE 144
+#define T_FINISH 145
+#define T_SWITCH_CONFIG 146
+#define T_INC 147
+#define T_DEC 148
+#define T_CAND 149
+#define T_COR 150
+#define T_LEQ 151
+#define T_EQ 152
+#define T_NEQ 153
+#define T_GEQ 154
+#define T_SHR 155
+#define T_SHL 156
+#define T_INCR 157
+#define T_DECR 158
+#define T_MUL 159
+#define T_DIV 160
+#define ID 161
+#define VAR 162
+#define CNUM 163
+#define CSTR 164
+#define EOI 165
More information about the varnish-commit
mailing list