r4732 - trunk/varnish-cache/lib/libvcl
phk at varnish-cache.org
phk at varnish-cache.org
Tue Apr 27 16:45:43 CEST 2010
Author: phk
Date: 2010-04-27 16:45:43 +0200 (Tue, 27 Apr 2010)
New Revision: 4732
Added:
trunk/varnish-cache/lib/libvcl/generate.py
Log:
A Python3 version of the vcc_gen_fixed_token.tcl script.
Added: trunk/varnish-cache/lib/libvcl/generate.py
===================================================================
--- trunk/varnish-cache/lib/libvcl/generate.py (rev 0)
+++ trunk/varnish-cache/lib/libvcl/generate.py 2010-04-27 14:45:43 UTC (rev 4732)
@@ -0,0 +1,709 @@
+#!/usr/local/bin/python3.1
+#-
+# Copyright (c) 2006 Verdens Gang AS
+# Copyright (c) 2006-2009 Linpro AS
+# All rights reserved.
+#
+# Author: Poul-Henning Kamp <phk at phk.freebsd.dk>
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# Generate various .c and .h files for the VCL compiler and the interfaces
+# for it.
+#
+# $Id$
+
+#######################################################################
+# These are our tokens
+
+# We could drop all words such as "include", "if" etc, and use the
+# ID type instead, but declaring them tokens makes them reserved words
+# which hopefully makes for better error messages.
+# XXX: does it actually do that ?
+
+tokens = {
+ "T_INC": "++",
+ "T_DEC": "--",
+ "T_CAND": "&&",
+ "T_COR": "||",
+ "T_LEQ": "<=",
+ "T_EQ": "==",
+ "T_NEQ": "!=",
+ "T_GEQ": ">=",
+ "T_SHR": ">>",
+ "T_SHL": "<<",
+ "T_INCR": "+=",
+ "T_DECR": "-=",
+ "T_MUL": "*=",
+ "T_DIV": "/=",
+ "T_NOMATCH": "!~",
+ "T_INCLUDE": "include",
+ "T_IF": "if",
+ "T_ELSEIF": "elseif",
+ "T_ELSIF": "elsif",
+ "T_ELSE": "else",
+
+ # Single char tokens, for convenience on one line
+ None: "{}()*+-/%><=;!&.|~,",
+
+ # These have handwritten recognizers
+ "ID": None,
+ "VAR": None,
+ "CNUM": None,
+ "CSTR": None,
+ "EOI": None,
+ "CSRC": None,
+}
+
+#######################################################################
+# Our methods and actions
+
+returns =(
+ ('recv', ('error', 'pass', 'pipe', 'lookup',)),
+ ('pipe', ('error', 'pipe',)),
+ ('pass', ('error', 'restart', 'pass',)),
+ ('hash', ('hash',)),
+ ('miss', ('error', 'restart', 'pass', 'fetch',)),
+ ('hit', ('error', 'restart', 'pass', 'deliver',)),
+ ('fetch', ('error', 'restart', 'pass', 'deliver',)),
+ ('deliver', ('restart', 'deliver',)),
+ ('error', ('restart', 'deliver',)),
+)
+
+
+#######################################################################
+# Variables available in sessions
+
+sp_variables = (
+ ('client.ip',
+ 'IP', 'RO',
+ ( 'all',),
+ 'const struct sess *'
+ ),
+ ('server.ip',
+ 'IP', 'RO',
+ ( 'all',),
+ 'struct sess *'
+ ),
+ ('server.hostname',
+ 'STRING', 'RO',
+ ( 'all',),
+ 'struct sess *'
+ ),
+ ('server.identity',
+ 'STRING', 'RO',
+ ( 'all',),
+ 'struct sess *'
+ ),
+ ('server.port',
+ 'INT', 'RO',
+ ( 'all',),
+ 'struct sess *'
+ ),
+ ('req.request',
+ 'STRING', 'RW',
+ ( 'all',),
+ 'const struct sess *'
+ ),
+ ('req.url',
+ 'STRING', 'RW',
+ ( 'all',),
+ 'const struct sess *'
+ ),
+ ('req.proto',
+ 'STRING', 'RW',
+ ( 'all',),
+ 'const struct sess *'
+ ),
+ ('req.http.',
+ 'HDR_REQ', 'RW',
+ ( 'all',),
+ 'const struct sess *'
+ ),
+ ('req.hash',
+ 'HASH', 'WO',
+ ( 'hash', 'error',),
+ 'struct sess *'
+ ),
+ ('req.backend',
+ 'BACKEND', 'RW',
+ ( 'all',),
+ 'struct sess *'
+ ),
+ ('req.restarts',
+ 'INT', 'RO',
+ ( 'all',),
+ 'const struct sess *'
+ ),
+ ('req.grace',
+ 'RTIME', 'RW',
+ ( 'all',),
+ 'struct sess *'
+ ),
+ ('req.xid',
+ 'STRING', 'RO',
+ ( 'all',),
+ 'struct sess *'
+ ),
+ ('req.esi',
+ 'BOOL', 'RW',
+ ( 'recv', 'fetch', 'deliver', 'error',),
+ 'struct sess *'
+ ),
+ ('req.backend.healthy',
+ 'BOOL', 'RO',
+ ( 'all',),
+ 'const struct sess *'
+ ),
+ ('bereq.request',
+ 'STRING', 'RW',
+ ( 'pipe', 'pass', 'miss', 'fetch',),
+ 'const struct sess *'
+ ),
+ ('bereq.url',
+ 'STRING', 'RW',
+ ( 'pipe', 'pass', 'miss', 'fetch',),
+ 'const struct sess *'
+ ),
+ ('bereq.proto',
+ 'STRING', 'RW',
+ ( 'pipe', 'pass', 'miss', 'fetch',),
+ 'const struct sess *'
+ ),
+ ('bereq.http.',
+ 'HDR_BEREQ', 'RW',
+ ( 'pipe', 'pass', 'miss', 'fetch',),
+ 'const struct sess *'
+ ),
+ ('bereq.connect_timeout',
+ 'RTIME', 'RW',
+ ( 'pass', 'miss',),
+ 'struct sess *'
+ ),
+ ('bereq.first_byte_timeout',
+ 'RTIME', 'RW',
+ ( 'pass', 'miss',),
+ 'struct sess *'
+ ),
+ ('bereq.between_bytes_timeout',
+ 'RTIME', 'RW',
+ ( 'pass', 'miss',),
+ 'struct sess *'
+ ),
+ ('beresp.proto',
+ 'STRING', 'RW',
+ ( 'fetch',),
+ 'const struct sess *'
+ ),
+ ('beresp.saintmode',
+ 'RTIME', 'WO',
+ ( 'fetch',),
+ 'const struct sess *'
+ ),
+ ('beresp.status',
+ 'INT', 'RW',
+ ( 'fetch',),
+ 'const struct sess *'
+ ),
+ ('beresp.response',
+ 'STRING', 'RW',
+ ( 'fetch',),
+ 'const struct sess *'
+ ),
+ ('beresp.http.',
+ 'HDR_BERESP', 'RW',
+ ( 'fetch',),
+ 'const struct sess *'
+ ),
+ ('beresp.cacheable',
+ 'BOOL', 'RW',
+ ( 'fetch',),
+ 'const struct sess *'
+ ),
+ ('beresp.ttl',
+ 'RTIME', 'RW',
+ ( 'fetch',),
+ 'const struct sess *'
+ ),
+ ('beresp.grace',
+ 'RTIME', 'RW',
+ ( 'fetch',),
+ 'const struct sess *'
+ ),
+ ('obj.proto',
+ 'STRING', 'RW',
+ ( 'hit', 'error',),
+ 'const struct sess *'
+ ),
+ ('obj.status',
+ 'INT', 'RW',
+ ( 'error',),
+ 'const struct sess *'
+ ),
+ ('obj.response',
+ 'STRING', 'RW',
+ ( 'error',),
+ 'const struct sess *'
+ ),
+ ('obj.hits',
+ 'INT', 'RO',
+ ( 'hit', 'deliver',),
+ 'const struct sess *'
+ ),
+ ('obj.http.',
+ 'HDR_OBJ', 'RW',
+ ( 'hit', 'error',),
+ 'const struct sess *'
+ ),
+ ('obj.cacheable',
+ 'BOOL', 'RW',
+ ( 'hit',),
+ 'const struct sess *'
+ ),
+ ('obj.ttl',
+ 'RTIME', 'RW',
+ ( 'hit', 'error',),
+ 'const struct sess *'
+ ),
+ ('obj.grace',
+ 'RTIME', 'RW',
+ ( 'hit', 'error',),
+ 'const struct sess *'
+ ),
+ ('obj.lastuse',
+ 'RTIME', 'RO',
+ ( 'hit', 'deliver', 'error',),
+ 'const struct sess *'
+ ),
+ ('resp.proto',
+ 'STRING', 'RW',
+ ( 'deliver',),
+ 'const struct sess *'
+ ),
+ ('resp.status',
+ 'INT', 'RW',
+ ( 'deliver',),
+ 'const struct sess *'
+ ),
+ ('resp.response',
+ 'STRING', 'RW',
+ ( 'deliver',),
+ 'const struct sess *'
+ ),
+ ('resp.http.',
+ 'HDR_RESP', 'RW',
+ ( 'deliver',),
+ 'const struct sess *'
+ ),
+ ('now',
+ 'TIME', 'RO',
+ ( 'all',),
+ 'const struct sess *'
+ ),
+)
+#######################################################################
+# VCL to C type conversion
+
+vcltypes = {
+ 'IP': "struct sockaddr *",
+ 'STRING': "const char *",
+ 'BOOL': "unsigned",
+ 'BACKEND': "struct director *",
+ 'TIME': "double",
+ 'RTIME': "double",
+ 'INT': "int",
+ 'HDR_RESP': "const char *",
+ 'HDR_OBJ': "const char *",
+ 'HDR_REQ': "const char *",
+ 'HDR_BEREQ': "const char *",
+ 'HOSTNAME': "const char *",
+ 'PORTNAME': "const char *",
+ 'HASH': "const char *",
+ 'SET': "struct vrt_backend_entry *",
+}
+
+#######################################################################
+# Nothing is easily configurable below this line.
+#######################################################################
+
+import sys
+import copy
+
+#######################################################################
+# Emit a function to recognize tokens in a string
+
+def emit_vcl_fixed_token(fo, tokens):
+
+ recog = list()
+ emit = dict()
+ for i in tokens:
+ j = tokens[i]
+ if (j != None):
+ recog.append(j)
+ emit[j] = i
+
+ recog.sort()
+ rrecog = copy.copy(recog)
+ rrecog.sort(key = lambda x: -len(x))
+
+ fo.write("""
+#define M1()\tdo {*q = p + 1; return (p[0]); } while (0)
+#define M2(c,t)\tdo {if (p[1] == (c)) { *q = p + 2; return (t); }} while (0)
+
+unsigned
+vcl_fixed_token(const char *p, const char **q)
+{
+
+\tswitch (p[0]) {
+""")
+ last_initial = None
+ for i in recog:
+ if (i[0] == last_initial):
+ continue
+ last_initial = i[0]
+ fo.write("\tcase '%s':\n" % last_initial)
+ need_ret = True
+ for j in rrecog:
+ if (j[0] != last_initial):
+ continue
+ if len(j) == 2:
+ fo.write("\t\tM2('%s', %s);\n" %
+ (j[1], emit[j]))
+ elif len(j) == 1:
+ fo.write("\t\tM1();\n")
+ need_ret = False
+ else:
+ fo.write("\t\tif (")
+ k = 1
+ l = len(j)
+ while (k < l):
+ fo.write("p[%d] == '%s'" % (k, j[k]))
+ fo.write(" && ")
+ if (k % 3) == 0:
+ fo.write("\n\t\t ")
+ k += 1
+ fo.write("!isvar(p[%d])) {\n" % l)
+ fo.write("\t\t\t*q = p + %d;\n" % l)
+ fo.write("\t\t\treturn (%s);\n" % emit[j])
+ fo.write("\t\t}\n")
+ if need_ret:
+ fo.write("\t\treturn (0);\n")
+ fo.write("\tdefault:\n\t\treturn (0);\n\t}\n}\n")
+
+#######################################################################
+# Emit the vcl_tnames (token->string) conversion array
+
+def emit_vcl_tnames(fo, tokens):
+ fo.write("\nconst char * const vcl_tnames[256] = {\n")
+ l = list(tokens.keys())
+ l.sort()
+ for i in l:
+ j = tokens[i]
+ if j == None:
+ j = i
+ if i[0] == "'":
+ j = i
+ fo.write("\t[%s] = \"%s\",\n" % (i, j))
+ fo.write("};\n")
+
+#######################################################################
+# Read a C-source file and spit out code that outputs it with vsb_cat()
+
+def emit_file(fo, fn):
+ fi = open(fn)
+ fc = fi.read()
+ fi.close()
+
+ w = 66 # Width of lines, after white space prefix
+ maxlen = 10240 # Max length of string literal
+
+ x = 0
+ l = 0
+ fo.write("\n\t/* %s */\n\n" % fn)
+ for c in fc:
+ if l == 0:
+ fo.write("\tvsb_cat(sb, \"")
+ l += 12
+ x += 12
+ if x == 0:
+ fo.write("\t \"")
+ d = c
+ if c == '\n':
+ d = "\\n"
+ elif c == '\t':
+ d = "\\t"
+ elif c == '"':
+ d = "\\\""
+ elif c == '\\':
+ d = "\\\\"
+
+ if c == '\n' and x > w - 20:
+ fo.write(d + "\"\n")
+ x = 0
+ continue
+ if c.isspace() and x > w - 10:
+ fo.write(d + "\"\n")
+ x = 0
+ continue
+
+ fo.write(d)
+ x += len(d)
+ l += len(d)
+ if l > maxlen:
+ fo.write("\");\n")
+ l = 0;
+ x = 0
+ if x > w - 3:
+ fo.write("\"\n")
+ x = 0
+ fo.write("\");\n")
+
+#######################################################################
+
+def polish_tokens(tokens):
+ # Expand single char tokens
+ st = tokens[None]
+ del tokens[None]
+
+ for i in st:
+ tokens["'" + i + "'"] = i
+#######################################################################
+
+def file_header(fo):
+ fo.write("""
+/*
+ * $%s$
+ *
+ * NB: This file is machine generated, DO NOT EDIT!
+ *
+ * Edit and run generate.py instead
+ */
+""" % "Id")
+
+#######################################################################
+
+fo = open("vcc_fixed_token.c", "w")
+
+file_header(fo)
+fo.write("""
+
+#include "config.h"
+#include <stdio.h>
+#include <ctype.h>
+#include "config.h"
+#include "vcc_priv.h"
+#include "vsb.h"
+""")
+
+polish_tokens(tokens)
+emit_vcl_fixed_token(fo, tokens)
+emit_vcl_tnames(fo, tokens)
+
+fo.write("""
+void
+vcl_output_lang_h(struct vsb *sb)
+{
+""")
+
+emit_file(fo, "../../include/vcl.h")
+emit_file(fo, "../../include/vrt.h")
+emit_file(fo, "../../include/vrt_obj.h")
+
+fo.write("""
+}
+""")
+
+fo.close()
+
+#######################################################################
+
+fo = open("vcc_token_defs.h", "w")
+
+file_header(fo)
+
+j = 128
+l = list(tokens.keys())
+l.sort()
+for i in l:
+ if i[0] == "'":
+ continue
+ fo.write("#define\t%s %d\n" % (i, j))
+ j += 1
+ assert j < 256
+
+fo.close()
+
+#######################################################################
+
+rets = dict()
+vcls = list()
+for i in returns:
+ vcls.append(i[0])
+ for j in i[1]:
+ rets[j] = True
+
+
+#######################################################################
+
+fo = open("../../include/vcl_returns.h", "w")
+
+file_header(fo)
+
+fo.write("\n#ifdef VCL_RET_MAC\n")
+l = list(rets.keys())
+l.sort()
+for i in l:
+ fo.write("VCL_RET_MAC(%s, %s)\n" % (i.lower(), i.upper()))
+fo.write("#endif\n")
+fo.write("\n#ifdef VCL_MET_MAC\n")
+for i in returns:
+ fo.write("VCL_MET_MAC(%s,%s,\n" % (i[0].lower(), i[0].upper()))
+ p = " ("
+ for j in i[1]:
+ fo.write(" %s(1U << VCL_RET_%s)\n" % (p, j.upper()))
+ p = "| "
+ fo.write("))\n")
+fo.write("#endif\n")
+fo.close()
+
+#######################################################################
+
+fo = open("../../include/vcl.h", "w")
+
+file_header(fo)
+
+fo.write("""
+struct sess;
+struct cli;
+
+typedef void vcl_init_f(struct cli *);
+typedef void vcl_fini_f(struct cli *);
+typedef int vcl_func_f(struct sess *sp);
+""")
+
+
+fo.write("\n/* VCL Methods */\n")
+n = 0
+for i in returns:
+ fo.write("#define VCL_MET_%s\t\t(1U << %d)\n" % (i[0].upper(), n))
+ n += 1
+
+fo.write("\n#define VCL_MET_MAX\t\t%d\n" % n)
+
+
+fo.write("\n/* VCL Returns */\n")
+n = 0
+l = list(rets.keys())
+l.sort()
+for i in l:
+ fo.write("#define VCL_RET_%s\t\t%d\n" % (i.upper(), n))
+ n += 1
+
+fo.write("\n#define VCL_RET_MAX\t\t%d\n" % n)
+
+
+fo.write("""
+struct VCL_conf {
+ unsigned magic;
+#define VCL_CONF_MAGIC 0x7406c509 /* from /dev/random */
+
+ struct director **director;
+ unsigned ndirector;
+ struct vrt_ref *ref;
+ unsigned nref;
+ unsigned busy;
+ unsigned discard;
+
+ unsigned nsrc;
+ const char **srcname;
+ const char **srcbody;
+
+ vcl_init_f *init_func;
+ vcl_fini_f *fini_func;
+
+ vcl_func_f *recv_func;
+ vcl_func_f *pipe_func;
+ vcl_func_f *pass_func;
+ vcl_func_f *hash_func;
+ vcl_func_f *miss_func;
+ vcl_func_f *hit_func;
+ vcl_func_f *fetch_func;
+ vcl_func_f *deliver_func;
+ vcl_func_f *error_func;
+};
+""")
+
+fo.close()
+
+#######################################################################
+
+fo=open("vcc_obj.c", "w")
+file_header(fo)
+
+fo.write("""
+#include "config.h"
+#include <stdio.h>
+#include "vcc_compile.h"
+
+struct var vcc_vars[] = {
+""")
+
+for i in sp_variables:
+ typ = i[1]
+ if typ[:4] == "HDR_":
+ typ = "HEADER"
+ fo.write("\t{ \"%s\", %s, %d,\n" % (i[0], typ, len(i[0])))
+ if i[2] == "RO" or i[2] == "RW":
+ fo.write('\t "VRT_r_%s(sp)",\n' % i[0].replace(".", "_"))
+ else:
+ fo.write('\t NULL,\n')
+ if i[2] == "WO" or i[2] == "RW":
+ fo.write('\t "VRT_l_%s(sp, ",\n' % i[0].replace(".", "_"))
+ else:
+ fo.write('\t NULL,\n')
+ fo.write('\t V_%s,' % i[2])
+ if typ == "HEADER":
+ fo.write('\t "%s",\n' % i[1])
+ else:
+ fo.write('\t 0,\n') # XXX: shoule be NULL
+ x = i[3]
+ if x[0] == 'all':
+ x = vcls
+ p = ""
+ n = 0
+ for j in x:
+ if n == 0:
+ fo.write("\t ")
+ n += 1
+ fo.write(p + "VCL_MET_" + j.upper())
+ p = " | "
+ if n == 4:
+ fo.write("\n")
+ n = 0
+
+ if n > 0:
+ fo.write("\n")
+ fo.write("\t},\n")
+
+fo.write("\t{ NULL }\n};\n")
+
+fo.close()
Property changes on: trunk/varnish-cache/lib/libvcl/generate.py
___________________________________________________________________
Added: svn:executable
+ *
More information about the varnish-commit
mailing list